Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 robbond
|
93b905df0a | ||
|
robbond
|
956fc2e31e | ||
|
robbond
|
18ac3f37ec |
@@ -1,8 +1,4 @@
|
||||
import {
|
||||
analyseScenario,
|
||||
PROMPT_VERSIONS,
|
||||
DEFAULT_PROMPT_VERSION,
|
||||
} from "@/lib/analysis";
|
||||
import { analyseScenario, PROMPT_VERSIONS, DEFAULT_PROMPT_VERSION } from "@/lib/analysis";
|
||||
|
||||
export async function POST(request) {
|
||||
try {
|
||||
@@ -11,7 +7,7 @@ export async function POST(request) {
|
||||
if (!body.scenario || typeof body.scenario !== "string") {
|
||||
return Response.json(
|
||||
{ error: "Request must include a 'scenario' string field" },
|
||||
{ status: 400 },
|
||||
{ status: 400 }
|
||||
);
|
||||
}
|
||||
|
||||
@@ -26,7 +22,7 @@ export async function POST(request) {
|
||||
if (!result.success) {
|
||||
return Response.json(
|
||||
{ ...result, reconstruction: result.reconstruction || null },
|
||||
{ status: Number(result.statusCode) || 500 },
|
||||
{ status: Number(result.statusCode) || 500 }
|
||||
);
|
||||
}
|
||||
|
||||
@@ -43,7 +39,7 @@ export async function POST(request) {
|
||||
} catch (e) {
|
||||
return Response.json(
|
||||
{ error: e.message || "Unknown server error", responseDurationMs: 0 },
|
||||
{ status: 500 },
|
||||
{ status: 500 }
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,38 +0,0 @@
|
||||
import { startCase } from "@/lib/graph/orchestrator.js";
|
||||
|
||||
export async function POST(request) {
|
||||
try {
|
||||
const body = await request.json();
|
||||
const result = await startCase(body);
|
||||
|
||||
if (result.success) {
|
||||
return Response.json(result, { status: 200 });
|
||||
}
|
||||
|
||||
const status =
|
||||
result.statusCode === 400
|
||||
? 400
|
||||
: result.statusCode >= 500
|
||||
? result.statusCode
|
||||
: 500;
|
||||
|
||||
return Response.json(
|
||||
{
|
||||
success: false,
|
||||
error: result.error ?? "Start case failed",
|
||||
validationErrors: result.validationErrors,
|
||||
diagnostics: result.diagnostics,
|
||||
analysisErrors: result.analysisErrors,
|
||||
},
|
||||
{ status },
|
||||
);
|
||||
} catch {
|
||||
return Response.json(
|
||||
{
|
||||
success: false,
|
||||
error: "Internal server error",
|
||||
},
|
||||
{ status: 500 },
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,68 +0,0 @@
|
||||
import { updateCase } from "@/lib/graph/orchestrator.js";
|
||||
|
||||
function mapFailureStatus(result) {
|
||||
switch (result?.stage) {
|
||||
case "request_validation":
|
||||
case "graph_validation":
|
||||
return 400;
|
||||
case "provider":
|
||||
return 502;
|
||||
case "proposal_validation":
|
||||
case "proposal_compatibility":
|
||||
case "application":
|
||||
return 422;
|
||||
case "result_validation":
|
||||
return 500;
|
||||
default:
|
||||
return 500;
|
||||
}
|
||||
}
|
||||
|
||||
function buildFailureResponse(result) {
|
||||
return {
|
||||
success: false,
|
||||
stage: result?.stage ?? "internal",
|
||||
error: result?.error ?? "Update case failed",
|
||||
validationErrors: result?.validationErrors,
|
||||
graphValidationErrors: result?.graphValidationErrors,
|
||||
proposalErrors: result?.proposalErrors,
|
||||
providerErrors: result?.providerErrors,
|
||||
errors: result?.errors,
|
||||
diagnostics: result?.diagnostics,
|
||||
};
|
||||
}
|
||||
|
||||
export async function POST(request) {
|
||||
try {
|
||||
const body = await request.json();
|
||||
const result = await updateCase(body, { applyProposal: true });
|
||||
|
||||
if (result.success) {
|
||||
return Response.json(result, { status: 200 });
|
||||
}
|
||||
|
||||
return Response.json(buildFailureResponse(result), {
|
||||
status: mapFailureStatus(result),
|
||||
});
|
||||
} catch (error) {
|
||||
if (error instanceof SyntaxError) {
|
||||
return Response.json(
|
||||
{
|
||||
success: false,
|
||||
stage: "request_validation",
|
||||
error: "Invalid JSON request body",
|
||||
},
|
||||
{ status: 400 },
|
||||
);
|
||||
}
|
||||
|
||||
return Response.json(
|
||||
{
|
||||
success: false,
|
||||
stage: "internal",
|
||||
error: "Internal server error",
|
||||
},
|
||||
{ status: 500 },
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,3 @@
|
||||
import React from "react";
|
||||
|
||||
const ValidationIndicator = ({ status }) => {
|
||||
const styles = {
|
||||
valid: "text-green-600",
|
||||
@@ -12,9 +10,7 @@ const ValidationIndicator = ({ status }) => {
|
||||
invalid: "❌ Validation failed",
|
||||
};
|
||||
return (
|
||||
<div
|
||||
className={`flex items-center gap-2 ${styles[status] || "text-gray-500"}`}
|
||||
>
|
||||
<div className={`flex items-center gap-2 ${styles[status] || "text-gray-500"}`}>
|
||||
<span className="font-medium">{labels[status] || status}</span>
|
||||
</div>
|
||||
);
|
||||
@@ -29,74 +25,12 @@ const validationIcons = {
|
||||
export default function DiagnosticsView({ result }) {
|
||||
if (!result) return null;
|
||||
|
||||
const diagnostics = result.diagnostics || result;
|
||||
|
||||
const metrics = [
|
||||
{ label: "Model", value: diagnostics.modelName || result.modelName || "?" },
|
||||
{ label: "Model", value: result.modelName || "?" },
|
||||
{ label: "Provider", value: "Ollama" },
|
||||
{
|
||||
label: "Prompt version",
|
||||
value: diagnostics.promptVersion || result.promptVersion || "?",
|
||||
},
|
||||
{
|
||||
label: "Duration",
|
||||
value:
|
||||
diagnostics.responseDurationMs != null
|
||||
? `${diagnostics.responseDurationMs}ms`
|
||||
: "?",
|
||||
},
|
||||
{
|
||||
label: "Validation",
|
||||
value: (
|
||||
<ValidationIndicator
|
||||
status={diagnostics.validationStatus || result.validationStatus || "invalid"}
|
||||
/>
|
||||
),
|
||||
},
|
||||
{
|
||||
label: "Node count",
|
||||
value:
|
||||
diagnostics.nodeCount != null
|
||||
? diagnostics.nodeCount
|
||||
: diagnostics.graphNodeCount != null
|
||||
? diagnostics.graphNodeCount
|
||||
: "?",
|
||||
},
|
||||
{
|
||||
label: "Edge count",
|
||||
value:
|
||||
diagnostics.edgeCount != null
|
||||
? diagnostics.edgeCount
|
||||
: diagnostics.graphEdgeCount != null
|
||||
? diagnostics.graphEdgeCount
|
||||
: "?",
|
||||
},
|
||||
{
|
||||
label: "Graph references",
|
||||
value:
|
||||
diagnostics.graphReferenceValidation == null
|
||||
? "?"
|
||||
: diagnostics.graphReferenceValidation.valid
|
||||
? `${validationIcons.valid} valid`
|
||||
: `${validationIcons.invalid} invalid`,
|
||||
},
|
||||
{
|
||||
label: "Investigation strategy",
|
||||
value:
|
||||
diagnostics.investigationStrategy?.key ||
|
||||
diagnostics.investigationStrategy ||
|
||||
result.selectedQuestion?.strategy ||
|
||||
"?",
|
||||
},
|
||||
];
|
||||
|
||||
const errors = [
|
||||
...(result.errors || []),
|
||||
...(result.validationErrors || []),
|
||||
...(result.graphValidationErrors || []),
|
||||
...(result.proposalErrors || []),
|
||||
...(result.providerErrors || []),
|
||||
...(result.analysisErrors || []),
|
||||
{ label: "Prompt version", value: result.promptVersion || "?" },
|
||||
{ label: "Duration", value: result.responseDurationMs != null ? `${result.responseDurationMs}ms` : "?" },
|
||||
{ label: "Validation", value: <ValidationIndicator status={result.validationStatus || "invalid"} /> },
|
||||
];
|
||||
|
||||
return (
|
||||
@@ -115,8 +49,7 @@ export default function DiagnosticsView({ result }) {
|
||||
{result.rawResponse && (
|
||||
<details className="mt-4">
|
||||
<summary className="cursor-pointer text-xs text-gray-500 underline hover:text-gray-700">
|
||||
View raw model response (
|
||||
{(result.rawResponse?.length || 0).toLocaleString()} chars)
|
||||
View raw model response ({(result.rawResponse?.length || 0).toLocaleString()} chars)
|
||||
</summary>
|
||||
<pre className="mt-2 max-h-60 overflow-auto rounded bg-gray-900 px-3 py-2 text-xs leading-relaxed text-green-400">
|
||||
{result.rawResponse}
|
||||
@@ -125,14 +58,14 @@ export default function DiagnosticsView({ result }) {
|
||||
)}
|
||||
|
||||
{/* Errors if present */}
|
||||
{errors.length > 0 && (
|
||||
{result.errors && result.errors.length > 0 && (
|
||||
<details className="mt-3">
|
||||
<summary className="cursor-pointer text-xs text-red-500 underline hover:text-red-700">
|
||||
Validation errors ({errors.length})
|
||||
Validation errors ({result.errors.length})
|
||||
</summary>
|
||||
<ul className="mt-1 space-y-0.5 text-xs text-red-600">
|
||||
{errors.map((err, i) => (
|
||||
<li key={i}>{typeof err === "string" ? err : err?.message || JSON.stringify(err)}</li>
|
||||
{result.errors.map((err, i) => (
|
||||
<li key={i}>{err}</li>
|
||||
))}
|
||||
</ul>
|
||||
</details>
|
||||
|
||||
@@ -1,228 +0,0 @@
|
||||
import React from "react";
|
||||
|
||||
function ListSection({ title, items, renderItem = (item) => item }) {
|
||||
if (!items?.length) return null;
|
||||
|
||||
return (
|
||||
<section className="rounded-lg border border-gray-200 bg-white p-4">
|
||||
<h3 className="mb-2 text-sm font-semibold text-gray-800">{title}</h3>
|
||||
<ul className="space-y-1 text-sm text-gray-700">
|
||||
{items.map((item, index) => (
|
||||
<li key={`${title}-${index}`}>{renderItem(item)}</li>
|
||||
))}
|
||||
</ul>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
export default function GraphUpdateView({ updateResult }) {
|
||||
if (!updateResult?.proposal) return null;
|
||||
|
||||
const {
|
||||
resolvedUnknownNodeIds,
|
||||
affectedNodeIds,
|
||||
previousActiveUnknownNodeId,
|
||||
newActiveUnknownNodeId,
|
||||
selectedQuestion,
|
||||
changesApplied,
|
||||
proposal,
|
||||
previousSituationGraph,
|
||||
updatedSituationGraph,
|
||||
reasoningState,
|
||||
previousReasoningState,
|
||||
} = updateResult;
|
||||
|
||||
const newlySurfacedUnknownNodeIds = (proposal.addedNodes || [])
|
||||
.filter((node) => node.kind === "unknown")
|
||||
.map((node) => node.id);
|
||||
|
||||
const previousNodesById = new Map(
|
||||
(previousSituationGraph?.nodes || []).map((node) => [node.id, node]),
|
||||
);
|
||||
const updatedNodesById = new Map(
|
||||
(updatedSituationGraph?.nodes || []).map((node) => [node.id, node]),
|
||||
);
|
||||
const proposalUpdatesByNodeId = new Map(
|
||||
(proposal.updatedNodes || []).map((update) => [update.nodeId, update]),
|
||||
);
|
||||
|
||||
function resolveNodePresentation(nodeId) {
|
||||
const previousNode = previousNodesById.get(nodeId) || null;
|
||||
const updatedNode = updatedNodesById.get(nodeId) || null;
|
||||
const node = updatedNode || previousNode;
|
||||
const update = proposalUpdatesByNodeId.get(nodeId) || null;
|
||||
|
||||
if (!node) {
|
||||
return (
|
||||
<div className="space-y-1">
|
||||
<div className="font-medium text-gray-900">Unknown node (ID: {nodeId})</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="space-y-1">
|
||||
<div className="font-medium text-gray-900">{node.label}</div>
|
||||
<div className="text-xs text-gray-600">
|
||||
{node.kind} · {node.confidence}
|
||||
</div>
|
||||
{node.confidenceAssessment && (
|
||||
<div className="text-xs text-gray-600">
|
||||
evidence {node.confidenceAssessment.evidenceConfidence} · completeness {node.confidenceAssessment.completenessStatus} · conclusion {node.confidenceAssessment.conclusionConfidence}
|
||||
</div>
|
||||
)}
|
||||
{(update?.previousStatus || update?.newStatus || node.status) && (
|
||||
<div className="text-xs text-gray-700">
|
||||
{update?.previousStatus ? `Previous status: ${update.previousStatus}` : null}
|
||||
{update?.previousStatus && update?.newStatus ? " → " : null}
|
||||
{update?.newStatus
|
||||
? `New status: ${update.newStatus}`
|
||||
: !update?.previousStatus
|
||||
? `Status: ${node.status}`
|
||||
: null}
|
||||
</div>
|
||||
)}
|
||||
{update?.reason && <div className="text-xs text-gray-700">{update.reason}</div>}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function resolveActiveUnknown(nodeId) {
|
||||
if (!nodeId) return null;
|
||||
|
||||
const node = updatedNodesById.get(nodeId) || previousNodesById.get(nodeId);
|
||||
if (!node) {
|
||||
return `Unknown node (ID: ${nodeId})`;
|
||||
}
|
||||
|
||||
return `${node.label} · ${node.status} · ${node.confidence}`;
|
||||
}
|
||||
|
||||
const changeItems = [
|
||||
changesApplied?.addedNodeCount
|
||||
? `${changesApplied.addedNodeCount} node(s) added`
|
||||
: null,
|
||||
changesApplied?.updatedNodeCount
|
||||
? `${changesApplied.updatedNodeCount} node(s) updated`
|
||||
: null,
|
||||
changesApplied?.addedEdgeCount
|
||||
? `${changesApplied.addedEdgeCount} edge(s) added`
|
||||
: null,
|
||||
changesApplied?.removedEdgeCount
|
||||
? `${changesApplied.removedEdgeCount} edge(s) removed`
|
||||
: null,
|
||||
changesApplied?.resolvedUnknownCount
|
||||
? `${changesApplied.resolvedUnknownCount} unknown(s) resolved`
|
||||
: null,
|
||||
].filter(Boolean);
|
||||
|
||||
const previousComparabilityStatus =
|
||||
previousReasoningState?.comparabilityStatus ||
|
||||
previousSituationGraph?.reasoningState?.comparabilityStatus ||
|
||||
null;
|
||||
const newComparabilityStatus =
|
||||
reasoningState?.comparabilityStatus ||
|
||||
updatedSituationGraph?.reasoningState?.comparabilityStatus ||
|
||||
null;
|
||||
const relationshipStatus =
|
||||
reasoningState?.relationshipStatus ||
|
||||
updatedSituationGraph?.reasoningState?.relationshipStatus ||
|
||||
null;
|
||||
const reasoningStagesAfter =
|
||||
reasoningState?.reasoningStages ||
|
||||
updatedSituationGraph?.reasoningState?.reasoningStages ||
|
||||
[];
|
||||
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
<section className="rounded-lg border border-blue-200 bg-blue-50 p-4">
|
||||
<h2 className="mb-2 text-base font-semibold text-blue-900">
|
||||
Graph update applied
|
||||
</h2>
|
||||
<div className="grid gap-2 text-sm text-blue-950 sm:grid-cols-2">
|
||||
{previousActiveUnknownNodeId && (
|
||||
<div>
|
||||
<span className="font-medium">Previous active unknown:</span>{" "}
|
||||
{resolveActiveUnknown(previousActiveUnknownNodeId)}
|
||||
</div>
|
||||
)}
|
||||
{newActiveUnknownNodeId && (
|
||||
<div>
|
||||
<span className="font-medium">New active unknown:</span>{" "}
|
||||
{resolveActiveUnknown(newActiveUnknownNodeId)}
|
||||
</div>
|
||||
)}
|
||||
{selectedQuestion?.question && (
|
||||
<div>
|
||||
<span className="font-medium">Next question:</span>{" "}
|
||||
{selectedQuestion.question}
|
||||
</div>
|
||||
)}
|
||||
{previousComparabilityStatus && newComparabilityStatus && (
|
||||
<div>
|
||||
<span className="font-medium">Comparability:</span>{" "}
|
||||
{previousComparabilityStatus} → {newComparabilityStatus}
|
||||
</div>
|
||||
)}
|
||||
{relationshipStatus && (
|
||||
<div>
|
||||
<span className="font-medium">Relationship status:</span>{" "}
|
||||
{relationshipStatus}
|
||||
</div>
|
||||
)}
|
||||
{!selectedQuestion?.question && !newActiveUnknownNodeId && previousActiveUnknownNodeId && (
|
||||
<div>
|
||||
<span className="font-medium">Next question status:</span> No next question selected yet.
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
{reasoningStagesAfter.length > 0 && (
|
||||
<div className="mt-3 text-sm text-blue-950">
|
||||
<span className="font-medium">Reasoning stages:</span>{" "}
|
||||
{reasoningStagesAfter
|
||||
.map((stage) => `${stage.stage}: ${stage.status}`)
|
||||
.join(" → ")}
|
||||
</div>
|
||||
)}
|
||||
</section>
|
||||
|
||||
<ListSection
|
||||
title="Resolved unknowns"
|
||||
items={resolvedUnknownNodeIds}
|
||||
renderItem={resolveNodePresentation}
|
||||
/>
|
||||
<ListSection
|
||||
title="Newly surfaced unknowns"
|
||||
items={newlySurfacedUnknownNodeIds}
|
||||
renderItem={resolveNodePresentation}
|
||||
/>
|
||||
<ListSection
|
||||
title="Affected nodes"
|
||||
items={affectedNodeIds}
|
||||
renderItem={resolveNodePresentation}
|
||||
/>
|
||||
<ListSection title="Applied changes" items={changeItems} />
|
||||
|
||||
<details className="rounded-lg border border-gray-200 bg-gray-50 p-4">
|
||||
<summary className="cursor-pointer text-sm font-medium text-gray-700 underline">
|
||||
Proposal details
|
||||
</summary>
|
||||
<pre className="mt-3 overflow-auto rounded bg-gray-900 p-3 text-xs text-green-400">
|
||||
{JSON.stringify(proposal, null, 2)}
|
||||
</pre>
|
||||
<pre className="mt-3 overflow-auto rounded bg-gray-900 p-3 text-xs text-green-400">
|
||||
{JSON.stringify(
|
||||
{
|
||||
previousActiveUnknownNodeId,
|
||||
newActiveUnknownNodeId,
|
||||
resolvedUnknownNodeIds,
|
||||
affectedNodeIds,
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}
|
||||
</pre>
|
||||
</details>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -10,9 +10,7 @@ const confidenceColor = {
|
||||
};
|
||||
|
||||
const ConfidenceBadge = ({ level }) => (
|
||||
<span
|
||||
className={`inline-block rounded-full border px-2 py-0.5 text-xs font-medium ${confidenceColor[level] || "text-gray-600 bg-gray-100"}`}
|
||||
>
|
||||
<span className={`inline-block rounded-full border px-2 py-0.5 text-xs font-medium ${confidenceColor[level] || "text-gray-600 bg-gray-100"}`}>
|
||||
{level}
|
||||
</span>
|
||||
);
|
||||
@@ -44,27 +42,17 @@ const importanceLabels = {
|
||||
function ClassificationDisplay({ classification }) {
|
||||
if (!classification) return null;
|
||||
const p = classification.primaryType || classification.primary_type;
|
||||
const sec =
|
||||
classification.secondaryTypes || classification.secondary_types || [];
|
||||
const modes =
|
||||
classification.reasoningModes || classification.reasoning_modes || [];
|
||||
const sec = classification.secondaryTypes || classification.secondary_types || [];
|
||||
const modes = classification.reasoningModes || classification.reasoning_modes || [];
|
||||
|
||||
// Normalize camelCase to snake_case for display if needed
|
||||
const primaryLabel = String(p)
|
||||
.replace(/_/g, " ")
|
||||
.replace(/\b\w/g, (c) => c.toUpperCase());
|
||||
const secLabels = sec.map((s) =>
|
||||
s.replace(/_/g, " ").replace(/\b\w/g, (c) => c.toUpperCase()),
|
||||
);
|
||||
const modeLabels = modes.map((m) =>
|
||||
m.replace(/_/g, " ").replace(/\b\w/g, (c) => c.toUpperCase()),
|
||||
);
|
||||
const primaryLabel = String(p).replace(/_/g, " ").replace(/\b\w/g, (c) => c.toUpperCase());
|
||||
const secLabels = sec.map((s) => s.replace(/_/g, " ").replace(/\b\w/g, (c) => c.toUpperCase()));
|
||||
const modeLabels = modes.map((m) => m.replace(/_/g, " ").replace(/\b\w/g, (c) => c.toUpperCase()));
|
||||
|
||||
return (
|
||||
<div className="rounded-lg border border-blue-200 bg-blue-50 p-4">
|
||||
<h3 className="mb-2 text-sm font-semibold text-blue-700">
|
||||
Input Classification
|
||||
</h3>
|
||||
<h3 className="mb-2 text-sm font-semibold text-blue-700">Input Classification</h3>
|
||||
<dl className="grid grid-cols-[auto_1fr] gap-x-4 gap-y-1.5 text-sm">
|
||||
<dt className="text-blue-500">Primary type</dt>
|
||||
<dd className="font-medium">{primaryLabel}</dd>
|
||||
@@ -81,14 +69,9 @@ function ClassificationDisplay({ classification }) {
|
||||
</>
|
||||
)}
|
||||
<dt className="text-blue-500 pt-1">Classification reason</dt>
|
||||
<dd className="italic">
|
||||
{classification.classificationReason ||
|
||||
classification.classification_reason}
|
||||
</dd>
|
||||
<dd className="italic">{classification.classificationReason || classification.classification_reason}</dd>
|
||||
<dt className="text-blue-500 pt-1">Confidence</dt>
|
||||
<dd>
|
||||
<ConfidenceBadge level={classification.confidence} />
|
||||
</dd>
|
||||
<dd><ConfidenceBadge level={classification.confidence} /></dd>
|
||||
</dl>
|
||||
</div>
|
||||
);
|
||||
@@ -100,9 +83,7 @@ function SummaryDisplay({ reconstruction }) {
|
||||
const summary = reconstruction.summary || reconstruction.Summary;
|
||||
return (
|
||||
<div className="rounded-lg border border-gray-200 bg-white p-4">
|
||||
<h3 className="mb-2 text-sm font-semibold text-gray-600">
|
||||
Reconstruction Summary
|
||||
</h3>
|
||||
<h3 className="mb-2 text-sm font-semibold text-gray-600">Reconstruction Summary</h3>
|
||||
<p className="text-sm leading-relaxed">{summary}</p>
|
||||
</div>
|
||||
);
|
||||
@@ -117,26 +98,15 @@ function ItemList({ title, items, renderExtra }) {
|
||||
|
||||
return (
|
||||
<div className="mb-4 rounded-lg border border-gray-200 bg-white p-4">
|
||||
<h3 className="mb-2 text-sm font-semibold text-gray-600">
|
||||
{title} ({count})
|
||||
</h3>
|
||||
<h3 className="mb-2 text-sm font-semibold text-gray-600">{title} ({count})</h3>
|
||||
<ul className="space-y-2">
|
||||
{itemsArr.map((item, idx) => (
|
||||
<li
|
||||
key={item.id || `${title}-${idx}`}
|
||||
className="rounded border border-gray-200 bg-white px-3 py-2 text-sm"
|
||||
>
|
||||
<li key={item.id || `${title}-${idx}`} className="rounded border border-gray-200 bg-white px-3 py-2 text-sm">
|
||||
<div className="flex items-center gap-2">
|
||||
{item.id && (
|
||||
<span className="font-mono text-xs text-gray-400">
|
||||
#{item.id}
|
||||
</span>
|
||||
)}
|
||||
{item.id && <span className="font-mono text-xs text-gray-400">#{item.id}</span>}
|
||||
{item.confidence && <ConfidenceBadge level={item.confidence} />}
|
||||
{item.importance && (
|
||||
<span
|
||||
className={`inline-block rounded-full border px-2 py-0.5 text-xs font-medium ${importanceColors[item.importance] || "text-gray-600 bg-gray-100"}`}
|
||||
>
|
||||
<span className={`inline-block rounded-full border px-2 py-0.5 text-xs font-medium ${importanceColors[item.importance] || "text-gray-600 bg-gray-100"}`}>
|
||||
{importanceLabels[item.importance]}
|
||||
</span>
|
||||
)}
|
||||
@@ -153,38 +123,23 @@ function ItemList({ title, items, renderExtra }) {
|
||||
// ── Plausible interpretations ───────────────────────
|
||||
function InterpretationsDisplay({ interpretations }) {
|
||||
if (!interpretations?.length) return null;
|
||||
const arr = Array.isArray(interpretations)
|
||||
? interpretations
|
||||
: [interpretations];
|
||||
const arr = Array.isArray(interpretations) ? interpretations : [interpretations];
|
||||
|
||||
return (
|
||||
<div className="mb-4 rounded-lg border border-indigo-200 bg-indigo-50 p-4">
|
||||
<h3 className="mb-2 text-sm font-semibold text-indigo-700">
|
||||
Plausible Interpretations ({arr.length})
|
||||
</h3>
|
||||
<h3 className="mb-2 text-sm font-semibold text-indigo-700">Plausible Interpretations ({arr.length})</h3>
|
||||
<ul className="space-y-3">
|
||||
{arr.map((interp, idx) => (
|
||||
<li
|
||||
key={interp.id || `${idx}`}
|
||||
className="rounded border border-indigo-200 bg-white px-3 py-2.5 text-sm leading-relaxed"
|
||||
>
|
||||
<li key={interp.id || `${idx}`} className="rounded border border-indigo-200 bg-white px-3 py-2.5 text-sm leading-relaxed">
|
||||
<div className="flex items-center gap-2 mb-1">
|
||||
<span className="font-medium text-indigo-600">
|
||||
{interp.description}
|
||||
</span>
|
||||
{interp.confidence && (
|
||||
<ConfidenceBadge level={interp.confidence} />
|
||||
)}
|
||||
<span className="font-medium text-indigo-600">{interp.description}</span>
|
||||
{interp.confidence && <ConfidenceBadge level={interp.confidence} />}
|
||||
</div>
|
||||
{interp.supportingEvidenceIds?.length > 0 && (
|
||||
<p className="text-xs text-gray-500">
|
||||
Supporting evidence: {interp.supportingEvidenceIds.join(", ")}
|
||||
</p>
|
||||
<p className="text-xs text-gray-500">Supporting evidence: {interp.supportingEvidenceIds.join(", ")}</p>
|
||||
)}
|
||||
{interp.assumptionsRequired?.length > 0 && (
|
||||
<p className="text-xs italic text-gray-500">
|
||||
Requires assumptions: {interp.assumptionsRequired.join("; ")}
|
||||
</p>
|
||||
<p className="text-xs italic text-gray-500">Requires assumptions: {interp.assumptionsRequired.join("; ")}</p>
|
||||
)}
|
||||
</li>
|
||||
))}
|
||||
@@ -199,37 +154,22 @@ function NextQuestionDisplay({ question }) {
|
||||
const q = question.question || question.Question;
|
||||
const targets = question.targets || question.Targets || [];
|
||||
const reason = question.reason || question.Reason || "";
|
||||
const value =
|
||||
question.expectedInformationValue ||
|
||||
question.expected_information_value ||
|
||||
"medium";
|
||||
const value = question.expectedInformationValue || question.expected_information_value || "medium";
|
||||
|
||||
const valueLabel =
|
||||
{ low: "Low", medium: "Medium", high: "High" }[value] || "Medium";
|
||||
const valueColor =
|
||||
{
|
||||
low: "bg-yellow-100 text-yellow-800",
|
||||
medium: "bg-blue-100 text-blue-800",
|
||||
high: "bg-green-100 text-green-800",
|
||||
}[value] || "";
|
||||
const valueLabel = { low: "Low", medium: "Medium", high: "High" }[value] || "Medium";
|
||||
const valueColor = { low: "bg-yellow-100 text-yellow-800", medium: "bg-blue-100 text-blue-800", high: "bg-green-100 text-green-800" }[value] || "";
|
||||
|
||||
return (
|
||||
<div className="rounded-lg border-2 border-green-300 bg-green-50 p-5">
|
||||
<div className="flex items-center gap-2 mb-2">
|
||||
<h3 className="text-sm font-bold text-green-800">Next Question</h3>
|
||||
<span
|
||||
className={`rounded-full px-2 py-0.5 text-xs font-medium ${valueColor}`}
|
||||
>
|
||||
{valueLabel} value
|
||||
</span>
|
||||
<span className={`rounded-full px-2 py-0.5 text-xs font-medium ${valueColor}`}>{valueLabel} value</span>
|
||||
</div>
|
||||
<p className="mb-2 text-base font-medium text-gray-900">{q}</p>
|
||||
{targets.length > 0 && (
|
||||
<p className="text-sm text-gray-600">Targets: {targets.join(", ")}</p>
|
||||
)}
|
||||
{reason && (
|
||||
<p className="text-sm italic text-gray-500">Because: {reason}</p>
|
||||
)}
|
||||
{reason && <p className="text-sm italic text-gray-500">Because: {reason}</p>}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -249,24 +189,13 @@ function EvidenceDisplay({ evidence }) {
|
||||
|
||||
return (
|
||||
<div className="mb-4 rounded-lg border border-gray-200 bg-white p-4">
|
||||
<h3 className="mb-2 text-sm font-semibold text-gray-600">
|
||||
Supporting Evidence ({arr.length})
|
||||
</h3>
|
||||
<h3 className="mb-2 text-sm font-semibold text-gray-600">Supporting Evidence ({arr.length})</h3>
|
||||
<ul className="space-y-2">
|
||||
{arr.map((item, idx) => (
|
||||
<li
|
||||
key={item.id || `${idx}`}
|
||||
className="rounded border border-gray-200 bg-white px-3 py-2 text-sm leading-relaxed"
|
||||
>
|
||||
<li key={item.id || `${idx}`} className="rounded border border-gray-200 bg-white px-3 py-2 text-sm leading-relaxed">
|
||||
<div className="flex items-center gap-2 mb-0.5 flex-wrap">
|
||||
{item.id && (
|
||||
<span className="font-mono text-xs text-gray-400">
|
||||
#{item.id}
|
||||
</span>
|
||||
)}
|
||||
<span
|
||||
className={`inline-block rounded px-1.5 py-0.5 text-[10px] font-medium ${importanceColors[item.importance] || "text-gray-600 bg-gray-100"}`}
|
||||
>
|
||||
{item.id && <span className="font-mono text-xs text-gray-400">#{item.id}</span>}
|
||||
<span className={`inline-block rounded px-1.5 py-0.5 text-[10px] font-medium ${importanceColors[item.importance] || "text-gray-600 bg-gray-100"}`}>
|
||||
{importanceLabels[item.importance]}
|
||||
</span>
|
||||
<span className="inline-block rounded px-1.5 py-0.5 text-[10px] font-medium bg-gray-100 text-gray-700">
|
||||
@@ -276,9 +205,7 @@ function EvidenceDisplay({ evidence }) {
|
||||
</div>
|
||||
<p className="text-sm">{item.description}</p>
|
||||
{(item.source || item.attribution) && (
|
||||
<p className="mt-0.5 text-xs text-gray-400">
|
||||
Source: {item.source || item.attribution}
|
||||
</p>
|
||||
<p className="mt-0.5 text-xs text-gray-400">Source: {item.source || item.attribution}</p>
|
||||
)}
|
||||
</li>
|
||||
))}
|
||||
@@ -295,8 +222,7 @@ export default function ReconstructionView({ reconstruction, partial }) {
|
||||
if (partial) {
|
||||
return (
|
||||
<div className="rounded-lg border border-yellow-300 bg-yellow-50 px-4 py-3 text-sm text-yellow-800">
|
||||
⚠ Partial result — some fields failed validation. Showing what was
|
||||
accepted.
|
||||
⚠ Partial result — some fields failed validation. Showing what was accepted.
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -315,96 +241,60 @@ export default function ReconstructionView({ reconstruction, partial }) {
|
||||
|
||||
{/* Key differences */}
|
||||
{data.reconstruction?.differences && (
|
||||
<ItemList
|
||||
title="Key Differences"
|
||||
items={data.reconstruction.differences}
|
||||
/>
|
||||
<ItemList title="Key Differences" items={data.reconstruction.differences} />
|
||||
)}
|
||||
|
||||
{/* Unexplained transitions */}
|
||||
{data.reconstruction?.unexplainedTransitions &&
|
||||
data.reconstruction.unexplainedTransitions.length > 0 && (
|
||||
<ItemList
|
||||
title="Unexplained Transitions"
|
||||
items={data.reconstruction.unexplainedTransitions}
|
||||
renderExtra={(i) =>
|
||||
i.entity && (
|
||||
<p className="mt-1 text-xs text-gray-500">Entity: {i.entity}</p>
|
||||
)
|
||||
}
|
||||
/>
|
||||
)}
|
||||
{data.reconstruction?.unexplainedTransitions && data.reconstruction.unexplainedTransitions.length > 0 && (
|
||||
<ItemList title="Unexplained Transitions" items={data.reconstruction.unexplainedTransitions} renderExtra={(i) => (
|
||||
i.entity && <p className="mt-1 text-xs text-gray-500">Entity: {i.entity}</p>
|
||||
)} />
|
||||
)}
|
||||
|
||||
{/* Contradictions */}
|
||||
{data.reconstruction?.contradictions &&
|
||||
data.reconstruction.contradictions.length > 0 && (
|
||||
<ItemList
|
||||
title="Contradictions"
|
||||
items={data.reconstruction.contradictions}
|
||||
/>
|
||||
)}
|
||||
{data.reconstruction?.contradictions && data.reconstruction.contradictions.length > 0 && (
|
||||
<ItemList title="Contradictions" items={data.reconstruction.contradictions} />
|
||||
)}
|
||||
|
||||
{/* Important unknowns */}
|
||||
{data.reconstruction?.importantUnknowns &&
|
||||
data.reconstruction.importantUnknowns.length > 0 && (
|
||||
<ItemList
|
||||
title="Important Unknowns"
|
||||
items={data.reconstruction.importantUnknowns}
|
||||
/>
|
||||
)}
|
||||
{data.reconstruction?.importantUnknowns && data.reconstruction.importantUnknowns.length > 0 && (
|
||||
<ItemList title="Important Unknowns" items={data.reconstruction.importantUnknowns} />
|
||||
)}
|
||||
|
||||
{/* Plausible interpretations */}
|
||||
{data.reconstruction?.plausibleInterpretations &&
|
||||
data.reconstruction.plausibleInterpretations.length > 0 && (
|
||||
<InterpretationsDisplay
|
||||
interpretations={data.reconstruction.plausibleInterpretations}
|
||||
/>
|
||||
)}
|
||||
{data.reconstruction?.plausibleInterpretations && data.reconstruction.plausibleInterpretations.length > 0 && (
|
||||
<InterpretationsDisplay interpretations={data.reconstruction.plausibleInterpretations} />
|
||||
)}
|
||||
|
||||
{/* Secondary reconstruction categories (actors, systems, etc.) */}
|
||||
{data.reconstruction?.actors && data.reconstruction.actors.length > 0 && (
|
||||
<ItemList title="Actors" items={data.reconstruction.actors} />
|
||||
)}
|
||||
{data.reconstruction?.systemsOrObjects &&
|
||||
data.reconstruction.systemsOrObjects.length > 0 && (
|
||||
<ItemList
|
||||
title="Systems / Objects"
|
||||
items={data.reconstruction.systemsOrObjects}
|
||||
/>
|
||||
)}
|
||||
{data.reconstruction?.expectedStates &&
|
||||
data.reconstruction.expectedStates.length > 0 && (
|
||||
<ItemList
|
||||
title="Expected States"
|
||||
items={data.reconstruction.expectedStates}
|
||||
/>
|
||||
)}
|
||||
{data.reconstruction?.observedStates &&
|
||||
data.reconstruction.observedStates.length > 0 && (
|
||||
<ItemList
|
||||
title="Observed States"
|
||||
items={data.reconstruction.observedStates}
|
||||
/>
|
||||
)}
|
||||
{data.reconstruction?.knownTransitions &&
|
||||
data.reconstruction.knownTransitions.length > 0 && (
|
||||
<ItemList
|
||||
title="Known Transitions"
|
||||
items={data.reconstruction.knownTransitions}
|
||||
renderExtra={(i) => (
|
||||
<div className="mt-1 text-xs text-gray-500">
|
||||
{i.entity && <span>Entity: {i.entity} · </span>}
|
||||
From “{i.previousState}” → To “{i.currentState}” ("{i.explanationStatus}")
|
||||
</div>
|
||||
)}
|
||||
/>
|
||||
)}
|
||||
{data.reconstruction?.systemsOrObjects && data.reconstruction.systemsOrObjects.length > 0 && (
|
||||
<ItemList title="Systems / Objects" items={data.reconstruction.systemsOrObjects} />
|
||||
)}
|
||||
{data.reconstruction?.expectedStates && data.reconstruction.expectedStates.length > 0 && (
|
||||
<ItemList title="Expected States" items={data.reconstruction.expectedStates} />
|
||||
)}
|
||||
{data.reconstruction?.observedStates && data.reconstruction.observedStates.length > 0 && (
|
||||
<ItemList title="Observed States" items={data.reconstruction.observedStates} />
|
||||
)}
|
||||
{data.reconstruction?.knownTransitions && data.reconstruction.knownTransitions.length > 0 && (
|
||||
<ItemList title="Known Transitions" items={data.reconstruction.knownTransitions} renderExtra={(i) => (
|
||||
<div className="mt-1 text-xs text-gray-500">
|
||||
{i.entity && <span>Entity: {i.entity} · </span>}
|
||||
From "{i.previousState}" → To "{i.currentState}" ({i.explanationStatus})
|
||||
</div>
|
||||
)} />
|
||||
)}
|
||||
|
||||
{/* Next question — prominent */}
|
||||
<NextQuestionDisplay question={data.nextQuestion} />
|
||||
|
||||
{/* Evidence */}
|
||||
{data.evidence && <EvidenceDisplay evidence={data.evidence} />}
|
||||
{data.evidence && (
|
||||
<EvidenceDisplay evidence={data.evidence} />
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
+59
-290
@@ -1,168 +1,41 @@
|
||||
"use client";
|
||||
|
||||
import React from "react";
|
||||
import { useState, useRef } from "react";
|
||||
import ReconstructionView from "@/components/reconstruction-view";
|
||||
import DiagnosticsView from "@/components/diagnostics-view";
|
||||
import GraphUpdateView from "@/components/graph-update-view";
|
||||
import SituationGraphView from "@/components/situation-graph-view";
|
||||
|
||||
const MAX_LENGTH = 10000;
|
||||
|
||||
export async function submitScenarioForStartCase(fetchImpl, scenario) {
|
||||
return fetchImpl("/api/cases/start", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ scenario }),
|
||||
});
|
||||
}
|
||||
|
||||
export async function submitAnswerForUpdateCase(
|
||||
fetchImpl,
|
||||
{ situationGraph, previousQuestion, answer },
|
||||
) {
|
||||
if (!answer?.trim()) {
|
||||
return {
|
||||
ok: false,
|
||||
skipped: true,
|
||||
data: {
|
||||
success: false,
|
||||
stage: "request_validation",
|
||||
error: "Please enter an answer before updating.",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const response = await fetchImpl("/api/cases/update", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ situationGraph, previousQuestion, answer }),
|
||||
});
|
||||
|
||||
return {
|
||||
ok: response.ok,
|
||||
skipped: false,
|
||||
data: await response.json(),
|
||||
};
|
||||
}
|
||||
|
||||
function normaliseStartResult(data) {
|
||||
return {
|
||||
...data,
|
||||
selectedQuestion:
|
||||
typeof data?.selectedQuestion === "string"
|
||||
? data.selectedQuestion
|
||||
: data?.selectedQuestion?.question ?? null,
|
||||
newlySurfacedNodeIds: data?.newlySurfacedNodeIds ?? [],
|
||||
};
|
||||
}
|
||||
|
||||
function normaliseUpdateSelectedQuestion(selectedQuestion) {
|
||||
if (!selectedQuestion) return null;
|
||||
if (typeof selectedQuestion === "string") return selectedQuestion;
|
||||
return selectedQuestion.question ?? null;
|
||||
}
|
||||
|
||||
export function ScenarioResultPanels({ status, result }) {
|
||||
if (!result) return null;
|
||||
|
||||
const hasGraph = Boolean(result.situationGraph);
|
||||
const hasQuestion = Boolean(result.selectedQuestion?.question);
|
||||
const hasDiagnostics = Boolean(result.diagnostics);
|
||||
|
||||
return (
|
||||
<>
|
||||
{status === "error" && (
|
||||
<div className="space-y-3">
|
||||
{result.error && (
|
||||
<div className="rounded-lg border border-red-300 bg-red-50 px-4 py-3 text-sm text-red-700 whitespace-pre-wrap">
|
||||
Error: {result.error}
|
||||
</div>
|
||||
)}
|
||||
{!hasGraph && !hasQuestion && (
|
||||
<div className="rounded-lg border border-yellow-300 bg-yellow-50 px-4 py-2 text-sm text-yellow-800">
|
||||
Validation failed — no structured graph output was produced.
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{(status === "success" || hasGraph || hasQuestion) && (
|
||||
<SituationGraphView
|
||||
situationGraph={result.situationGraph}
|
||||
selectedQuestion={result.selectedQuestion}
|
||||
newlySurfacedNodeIds={result.newlySurfacedNodeIds}
|
||||
/>
|
||||
)}
|
||||
|
||||
{hasDiagnostics && <DiagnosticsView result={result} />}
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
export function UpdateErrorPanel({ updateError }) {
|
||||
if (!updateError) return null;
|
||||
|
||||
const errors = [
|
||||
...(updateError.errors || []),
|
||||
...(updateError.validationErrors || []),
|
||||
...(updateError.graphValidationErrors || []),
|
||||
...(updateError.proposalErrors || []),
|
||||
...(updateError.providerErrors || []),
|
||||
];
|
||||
|
||||
return (
|
||||
<div className="space-y-3">
|
||||
<div className="rounded-lg border border-red-300 bg-red-50 px-4 py-3 text-sm text-red-700 whitespace-pre-wrap">
|
||||
Update error: {updateError.error}
|
||||
</div>
|
||||
{errors.length > 0 && (
|
||||
<details className="rounded-lg border border-red-200 bg-red-50 px-4 py-3">
|
||||
<summary className="cursor-pointer text-sm font-medium text-red-700 underline">
|
||||
Update details ({errors.length})
|
||||
</summary>
|
||||
<ul className="mt-2 space-y-1 text-sm text-red-700">
|
||||
{errors.map((item, index) => (
|
||||
<li key={index}>
|
||||
{typeof item === "string" ? item : item?.message || JSON.stringify(item)}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</details>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export default function ScenarioForm() {
|
||||
const [scenario, setScenario] = useState("");
|
||||
const [status, setStatus] = useState("idle"); // idle | loading | error | success
|
||||
const [status, setStatus] = useState("idle"); // idle | loading | error | success | partial
|
||||
const [result, setResult] = useState(null);
|
||||
const [answer, setAnswer] = useState("");
|
||||
const [updateStatus, setUpdateStatus] = useState("idle"); // idle | loading | error | success
|
||||
const [updateError, setUpdateError] = useState(null);
|
||||
const [updateResult, setUpdateResult] = useState(null);
|
||||
const textareaRef = useRef(null);
|
||||
|
||||
const handleSubmit = async (e) => {
|
||||
e.preventDefault();
|
||||
setStatus("loading");
|
||||
setResult(null);
|
||||
setAnswer("");
|
||||
setUpdateStatus("idle");
|
||||
setUpdateError(null);
|
||||
setUpdateResult(null);
|
||||
|
||||
try {
|
||||
const res = await submitScenarioForStartCase(fetch, scenario);
|
||||
const res = await fetch("/api/analyse", {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ scenario }),
|
||||
});
|
||||
|
||||
const data = await res.json();
|
||||
|
||||
if (res.ok && data.success) {
|
||||
if (res.ok && data.validationStatus === "valid") {
|
||||
setStatus("success");
|
||||
setResult(normaliseStartResult(data));
|
||||
setResult(data);
|
||||
} else if (data.success) {
|
||||
// Success in analysis but validation may be partial
|
||||
setStatus("success");
|
||||
setResult(data);
|
||||
} else {
|
||||
setStatus("error");
|
||||
setResult(normaliseStartResult(data));
|
||||
setResult(data);
|
||||
}
|
||||
} catch (err) {
|
||||
setStatus("error");
|
||||
@@ -170,64 +43,12 @@ export default function ScenarioForm() {
|
||||
}
|
||||
};
|
||||
|
||||
const handleUpdate = async (e) => {
|
||||
e.preventDefault();
|
||||
|
||||
const submission = await submitAnswerForUpdateCase(fetch, {
|
||||
situationGraph: result?.situationGraph,
|
||||
previousQuestion: result?.selectedQuestion,
|
||||
answer,
|
||||
});
|
||||
|
||||
if (submission.skipped) {
|
||||
setUpdateStatus("error");
|
||||
setUpdateError(submission.data);
|
||||
return;
|
||||
}
|
||||
|
||||
setUpdateStatus("loading");
|
||||
setUpdateError(null);
|
||||
|
||||
try {
|
||||
const outcome = submission.data;
|
||||
|
||||
if (submission.ok && outcome.success) {
|
||||
setUpdateStatus("success");
|
||||
setUpdateResult({
|
||||
...outcome,
|
||||
previousSituationGraph: result?.situationGraph ?? null,
|
||||
});
|
||||
setResult((current) => ({
|
||||
...current,
|
||||
situationGraph: outcome.updatedSituationGraph,
|
||||
selectedQuestion: normaliseUpdateSelectedQuestion(
|
||||
outcome.selectedQuestion,
|
||||
),
|
||||
newlySurfacedNodeIds: (outcome.proposal?.addedNodes || [])
|
||||
.filter((node) => node.kind === "unknown")
|
||||
.map((node) => node.id),
|
||||
diagnostics: outcome.diagnostics,
|
||||
}));
|
||||
setAnswer("");
|
||||
} else {
|
||||
setUpdateStatus("error");
|
||||
setUpdateError(outcome);
|
||||
}
|
||||
} catch (err) {
|
||||
setUpdateStatus("error");
|
||||
setUpdateError({ error: err.message || "Network request failed" });
|
||||
}
|
||||
};
|
||||
|
||||
const canRenderAnswerForm =
|
||||
status === "success" &&
|
||||
updateStatus === "idle" &&
|
||||
Boolean(result?.situationGraph) &&
|
||||
Boolean(result?.selectedQuestion);
|
||||
|
||||
const canRenderDisabledFollowUpForm =
|
||||
updateStatus === "success" &&
|
||||
Boolean(updateResult?.selectedQuestion?.question || result?.selectedQuestion);
|
||||
// Determine if we have meaningful content to display
|
||||
const hasClassification = result?.inputClassification;
|
||||
const hasReconstruction = result?.reconstruction;
|
||||
const hasNextQuestion = result?.nextQuestion;
|
||||
const hasEvidence = result?.evidence && result.evidence.length > 0;
|
||||
const hasMeaningfulContent = hasClassification || hasReconstruction || hasNextQuestion || hasEvidence;
|
||||
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
@@ -241,9 +62,7 @@ export default function ScenarioForm() {
|
||||
className="w-full rounded-lg border border-gray-300 px-4 py-3 text-sm focus:border-gray-500 focus:outline-none focus:ring-2 focus:ring-gray-400"
|
||||
/>
|
||||
<div className="flex items-center justify-between">
|
||||
<span className="text-xs text-gray-400">
|
||||
{scenario.length}/{MAX_LENGTH}
|
||||
</span>
|
||||
<span className="text-xs text-gray-400">{scenario.length}/{MAX_LENGTH}</span>
|
||||
<button
|
||||
type="submit"
|
||||
disabled={status === "loading" || !scenario.trim()}
|
||||
@@ -254,103 +73,53 @@ export default function ScenarioForm() {
|
||||
</div>
|
||||
</form>
|
||||
|
||||
{canRenderAnswerForm && (
|
||||
<form onSubmit={handleUpdate} className="space-y-4 rounded-lg border border-gray-200 bg-white p-4">
|
||||
<div>
|
||||
<h2 className="text-base font-semibold text-gray-900">Selected Question</h2>
|
||||
<p className="mt-1 text-sm text-gray-700">{result.selectedQuestion}</p>
|
||||
</div>
|
||||
<div>
|
||||
<label htmlFor="answer-textarea" className="mb-2 block text-sm font-medium text-gray-700">
|
||||
Your answer
|
||||
</label>
|
||||
<textarea
|
||||
id="answer-textarea"
|
||||
value={answer}
|
||||
onChange={(e) => setAnswer(e.target.value)}
|
||||
rows={4}
|
||||
className="w-full rounded-lg border border-gray-300 px-4 py-3 text-sm focus:border-gray-500 focus:outline-none focus:ring-2 focus:ring-gray-400"
|
||||
placeholder="Enter the answer to the selected question..."
|
||||
/>
|
||||
</div>
|
||||
<div className="flex items-center justify-between gap-4">
|
||||
<p className="text-xs text-gray-500">
|
||||
{updateStatus === "loading"
|
||||
? "Applying validated graph update..."
|
||||
: "One update turn only in this prototype."}
|
||||
</p>
|
||||
<button
|
||||
type="submit"
|
||||
disabled={updateStatus === "loading"}
|
||||
className="rounded-lg bg-blue-700 px-4 py-2 text-sm font-medium text-white transition hover:bg-blue-600 disabled:cursor-not-allowed disabled:opacity-40"
|
||||
>
|
||||
{updateStatus === "loading" ? "Updating..." : "Update situation"}
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
)}
|
||||
|
||||
<UpdateErrorPanel updateError={updateError} />
|
||||
|
||||
{updateStatus === "success" && updateResult && (
|
||||
<>
|
||||
<div className="rounded-lg border border-yellow-300 bg-yellow-50 px-4 py-3 text-sm text-yellow-800">
|
||||
{updateResult.selectedQuestion?.question
|
||||
? updateResult.selectedQuestion.question
|
||||
: "No next question selected yet."}
|
||||
</div>
|
||||
{canRenderDisabledFollowUpForm && (
|
||||
<form className="space-y-4 rounded-lg border border-gray-200 bg-white p-4 opacity-70">
|
||||
<div>
|
||||
<h2 className="text-base font-semibold text-gray-900">Selected Question</h2>
|
||||
<p className="mt-1 text-sm text-gray-700">
|
||||
{updateResult.selectedQuestion?.question || result?.selectedQuestion}
|
||||
</p>
|
||||
</div>
|
||||
<div>
|
||||
<label htmlFor="follow-up-disabled-textarea" className="mb-2 block text-sm font-medium text-gray-700">
|
||||
Your answer
|
||||
</label>
|
||||
<textarea
|
||||
id="follow-up-disabled-textarea"
|
||||
rows={4}
|
||||
disabled
|
||||
className="w-full rounded-lg border border-gray-300 px-4 py-3 text-sm opacity-70"
|
||||
placeholder="Additional submission is disabled in this one-update prototype."
|
||||
/>
|
||||
</div>
|
||||
<div className="flex items-center justify-between gap-4">
|
||||
<p className="text-xs text-gray-500">
|
||||
Additional submission is disabled in this one-update prototype.
|
||||
</p>
|
||||
<button
|
||||
type="button"
|
||||
disabled
|
||||
className="rounded-lg bg-blue-700 px-4 py-2 text-sm font-medium text-white disabled:cursor-not-allowed disabled:opacity-40"
|
||||
>
|
||||
Update situation
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
{/* Error state */}
|
||||
{status === "error" && (
|
||||
<div className="space-y-3">
|
||||
{result?.error && (
|
||||
<div className="rounded-lg border border-red-300 bg-red-50 px-4 py-3 text-sm text-red-700 whitespace-pre-wrap">
|
||||
Error: {result.error}
|
||||
</div>
|
||||
)}
|
||||
<GraphUpdateView updateResult={updateResult} />
|
||||
</>
|
||||
{/* Show partial content even on validation failure */}
|
||||
{(hasClassification || hasReconstruction) && (
|
||||
<div className="rounded-lg border border-yellow-300 bg-yellow-50 px-4 py-2 text-sm text-yellow-800">
|
||||
⚠ Partial result — some fields failed validation. Showing what was accepted.
|
||||
</div>
|
||||
)}
|
||||
{hasReconstruction && (
|
||||
<ReconstructionView reconstruction={result} partial />
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<ScenarioResultPanels status={status} result={result} />
|
||||
|
||||
{(status === "loading" || updateStatus === "loading") && (
|
||||
<div className="py-12 text-center text-sm text-gray-400">
|
||||
Waiting for model response...
|
||||
{/* Success state */}
|
||||
{status === "success" && hasMeaningfulContent && (
|
||||
<div className="space-y-4">
|
||||
<ReconstructionView reconstruction={result} />
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Always show diagnostics when we have any result */}
|
||||
{(hasClassification || hasReconstruction || hasNextQuestion) && (
|
||||
<DiagnosticsView result={result} />
|
||||
)}
|
||||
|
||||
{status === "loading" && (
|
||||
<div className="py-12 text-center text-sm text-gray-400">Waiting for model response...</div>
|
||||
)}
|
||||
|
||||
{/* Empty state */}
|
||||
{status === "idle" && (
|
||||
<div className="rounded-lg border border-dashed border-gray-300 bg-gray-50 px-6 py-8 text-center">
|
||||
<p className="text-sm text-gray-400">
|
||||
Enter a scenario above and click Analyse to begin.
|
||||
</p>
|
||||
<p className="text-sm text-gray-400">Enter a scenario above and click Analyse to begin.</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Invalid result with no partial data */}
|
||||
{status === "error" && !result?.error && !hasMeaningfulContent && (
|
||||
<div className="rounded-lg border border-yellow-300 bg-yellow-50 px-4 py-2 text-sm text-yellow-800">
|
||||
Validation failed — no structured output was produced.
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
@@ -1,161 +0,0 @@
|
||||
"use client";
|
||||
|
||||
import React from "react";
|
||||
|
||||
function NodeBadge({ children, tone = "gray" }) {
|
||||
const tones = {
|
||||
gray: "border-gray-200 bg-gray-50 text-gray-700",
|
||||
blue: "border-blue-200 bg-blue-50 text-blue-700",
|
||||
green: "border-green-200 bg-green-50 text-green-700",
|
||||
yellow: "border-yellow-200 bg-yellow-50 text-yellow-700",
|
||||
red: "border-red-200 bg-red-50 text-red-700",
|
||||
purple: "border-purple-200 bg-purple-50 text-purple-700",
|
||||
};
|
||||
|
||||
return (
|
||||
<span className={`rounded-full border px-2 py-0.5 text-xs ${tones[tone] || tones.gray}`}>
|
||||
{children}
|
||||
</span>
|
||||
);
|
||||
}
|
||||
|
||||
function NodeGroup({
|
||||
title,
|
||||
nodes,
|
||||
resolvedNodeIds = new Set(),
|
||||
newlySurfacedNodeIds = new Set(),
|
||||
activeUnknownNodeId = null,
|
||||
}) {
|
||||
if (!nodes?.length) return null;
|
||||
|
||||
return (
|
||||
<section className="rounded-lg border border-gray-200 bg-white p-4">
|
||||
<h3 className="mb-3 text-sm font-semibold text-gray-700">
|
||||
{title} ({nodes.length})
|
||||
</h3>
|
||||
<ul className="space-y-3">
|
||||
{nodes.map((node) => (
|
||||
<li key={node.id} className="rounded border border-gray-100 bg-gray-50 p-3 text-sm">
|
||||
<div className="flex flex-wrap items-center gap-2">
|
||||
<span className="font-medium text-gray-900">{node.label}</span>
|
||||
<NodeBadge tone="blue">{node.status}</NodeBadge>
|
||||
<NodeBadge tone="green">{node.confidence}</NodeBadge>
|
||||
{node.confidenceAssessment?.completenessStatus && (
|
||||
<NodeBadge tone="purple">
|
||||
completeness: {node.confidenceAssessment.completenessStatus}
|
||||
</NodeBadge>
|
||||
)}
|
||||
{resolvedNodeIds.has(node.id) && (
|
||||
<NodeBadge tone="red">resolved unknown</NodeBadge>
|
||||
)}
|
||||
{newlySurfacedNodeIds.has(node.id) && (
|
||||
<NodeBadge tone="purple">newly surfaced unknown</NodeBadge>
|
||||
)}
|
||||
{activeUnknownNodeId === node.id && (
|
||||
<NodeBadge tone="yellow">active unknown</NodeBadge>
|
||||
)}
|
||||
{node.value != null && (
|
||||
<NodeBadge tone="yellow">
|
||||
{node.value}
|
||||
{node.unit ? ` ${node.unit}` : ""}
|
||||
</NodeBadge>
|
||||
)}
|
||||
</div>
|
||||
{node.description && node.description !== node.label && (
|
||||
<p className="mt-1 text-gray-600">{node.description}</p>
|
||||
)}
|
||||
{node.confidenceAssessment && (
|
||||
<p className="mt-1 text-xs text-gray-500">
|
||||
evidence: {node.confidenceAssessment.evidenceConfidence} ·
|
||||
conclusion: {node.confidenceAssessment.conclusionConfidence}
|
||||
</p>
|
||||
)}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
|
||||
export default function SituationGraphView({
|
||||
situationGraph,
|
||||
selectedQuestion,
|
||||
newlySurfacedNodeIds = [],
|
||||
}) {
|
||||
if (!situationGraph) return null;
|
||||
|
||||
const selectedQuestionText =
|
||||
typeof selectedQuestion === "string"
|
||||
? selectedQuestion
|
||||
: selectedQuestion?.question ?? null;
|
||||
|
||||
const activeUnknown = situationGraph.activeUnknownNodeId
|
||||
? situationGraph.nodes.find((node) => node.id === situationGraph.activeUnknownNodeId)
|
||||
: null;
|
||||
|
||||
const nodesByKind = situationGraph.nodes.reduce((acc, node) => {
|
||||
if (!acc[node.kind]) acc[node.kind] = [];
|
||||
acc[node.kind].push(node);
|
||||
return acc;
|
||||
}, {});
|
||||
|
||||
const resolvedNodeIdSet = new Set(situationGraph.resolvedNodeIds || []);
|
||||
const newlySurfacedNodeIdSet = new Set(newlySurfacedNodeIds || []);
|
||||
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
{selectedQuestionText && (
|
||||
<section className="rounded-lg border-2 border-green-300 bg-green-50 p-5">
|
||||
<h2 className="mb-2 text-base font-bold text-green-800">Selected Question</h2>
|
||||
<p className="text-base font-medium text-gray-900">{selectedQuestionText}</p>
|
||||
</section>
|
||||
)}
|
||||
|
||||
<section className="rounded-lg border border-gray-200 bg-white p-4">
|
||||
<h2 className="mb-2 text-base font-semibold text-gray-900">Situation Graph</h2>
|
||||
<dl className="space-y-2 text-sm">
|
||||
<div>
|
||||
<dt className="text-gray-500">Central statement</dt>
|
||||
<dd className="font-medium text-gray-900">{situationGraph.centralStatement}</dd>
|
||||
</div>
|
||||
{situationGraph.currentSummary && (
|
||||
<div>
|
||||
<dt className="text-gray-500">Current summary</dt>
|
||||
<dd className="text-gray-800">{situationGraph.currentSummary}</dd>
|
||||
</div>
|
||||
)}
|
||||
{activeUnknown && (
|
||||
<div>
|
||||
<dt className="text-gray-500">Active unknown</dt>
|
||||
<dd className="text-gray-900">{activeUnknown.label}</dd>
|
||||
</div>
|
||||
)}
|
||||
<div>
|
||||
<dt className="text-gray-500">Edge count</dt>
|
||||
<dd className="text-gray-900">{situationGraph.edges.length}</dd>
|
||||
</div>
|
||||
</dl>
|
||||
</section>
|
||||
|
||||
{Object.entries(nodesByKind).map(([kind, nodes]) => (
|
||||
<NodeGroup
|
||||
key={kind}
|
||||
title={kind.replace(/_/g, " ")}
|
||||
nodes={nodes}
|
||||
resolvedNodeIds={resolvedNodeIdSet}
|
||||
newlySurfacedNodeIds={newlySurfacedNodeIdSet}
|
||||
activeUnknownNodeId={situationGraph.activeUnknownNodeId}
|
||||
/>
|
||||
))}
|
||||
|
||||
<details className="rounded-lg border border-gray-200 bg-gray-50 p-4">
|
||||
<summary className="cursor-pointer text-sm font-medium text-gray-700 underline">
|
||||
Raw graph JSON
|
||||
</summary>
|
||||
<pre className="mt-3 overflow-auto rounded bg-gray-900 p-3 text-xs text-green-400">
|
||||
{JSON.stringify(situationGraph, null, 2)}
|
||||
</pre>
|
||||
</details>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -1,113 +0,0 @@
|
||||
# Orchestrator Contract — Confidence Engine v0.4
|
||||
|
||||
## 1. Exported Function Signatures & Shape (JavaScript)
|
||||
|
||||
### lib/analysis.js
|
||||
|
||||
```js
|
||||
export async function analyseScenario(scenario, opts = {})
|
||||
// @param {string} scenario
|
||||
// @param {{ promptVersion?: "v0.2" | "v0.3" }} [opts]
|
||||
// @returns {Promise<{ success: boolean, validationStatus: "valid"|"invalid",
|
||||
// modelName: string|null, responseDurationMs: number, rawResponse: string|null,
|
||||
// promptVersion: string|null, inputClassification: object|null, reconstruction: object|null,
|
||||
// evidence: object[]|undefined, nextQuestion: string|undefined, errors: string[]|undefined,
|
||||
// error: string|undefined, statusCode: number|undefined }>}
|
||||
|
||||
export const PROMPT_VERSIONS // { [key: string]: string }
|
||||
export const DEFAULT_PROMPT_VERSION // "v0.2"
|
||||
```
|
||||
|
||||
### lib/graph/schema.js
|
||||
|
||||
```js
|
||||
export const SituationKind // { observation, reported_claim, metric, state, transition, relationship, assumption, unknown, conclusion }
|
||||
export const SituationStatus // { known, unknown, provisional, supported, weakened, contradicted, resolved }
|
||||
export const ConfidenceLevel // { low, medium, high }
|
||||
export const SituationRelationship // { supports, weakens, contradicts, depends_on, causes, may_cause, measures, compares_with, updates, other }
|
||||
|
||||
export const situationNodeSchema // Zod → {@typedef SituationNode}
|
||||
export const situationEdgeSchema // Zod → {@typedef SituationEdge}
|
||||
export const situationGraphSchema // Zod → {@typedef SituationGraph}
|
||||
export const graphUpdateSchema // Zod → {@typedef GraphUpdate}
|
||||
export const startCaseRequestSchema // { scenario: string (1-10000), promptVersion?: string }
|
||||
export const updateCaseRequestSchema// { situationGraph: SituationGraph, previousQuestion: string, answer: string (1-5000), promptVersion?: string }
|
||||
|
||||
/** @param {string} label */ /** @returns {string} */ export function makeNodeId(label)
|
||||
/** @param {{ id?, label, description, kind?, status?, confidence?, value?, unit?, ... }} opts */ /** @returns {SituationNode} */ export function makeNode(opts)
|
||||
/** @param {{ id?, fromNodeId, toNodeId, relationship?, confidence?, description? }} opts */ /** @returns {SituationEdge} */ export function makeEdge(opts)
|
||||
/** @param {{ centralStatement?, nodes?, edges?, activeUnknownNodeId?, resolvedNodeIds?, currentSummary? }} opts */ /** @returns {SituationGraph} */ export function makeGraph(opts)
|
||||
```
|
||||
|
||||
### lib/graph/utils.js
|
||||
|
||||
```js
|
||||
export function validateGraphReferences(graph) // → { valid: boolean, errors: string[] }
|
||||
export function detectDuplicateNodeIds(nodes) // → { nodeId, count }[]
|
||||
export function detectDuplicateEdges(edges) // { edgeId, fromNodeId, toNodeId, relationship }[]
|
||||
export function findDependentNodes(graph, nodeId) // → string[] (transitive)
|
||||
export function findAffectedNodes(graph, nodeId) // → string[] (direct + indirect via affects/dependsOn)
|
||||
/** @param {SituationGraph} graph */ /** @param {string} nodeId */ /** @param {string} newStatus */ /** @param {*} newValue */ /** @param {string} reason */
|
||||
export function resolveUnknownNode(graph, nodeId, newStatus, newValue, reason) // → { success, error?, previousStatus?, newStatus?, previousValue?, newValue?, reason?, affectedNodes? }
|
||||
export function selectActiveUnknownCandidate(graph, resolvedNodeIds) // → { nodeId, label, score } | null
|
||||
/** @param {SituationGraph} graph */ /** @param {GraphUpdate} update */
|
||||
export function applyGraphUpdate(graph, update) // → { success: boolean, errors?, nodes?, edges?, resolvedNodeIds? }
|
||||
/** @param {SituationGraph} graph */ /** @param {GraphUpdate} update */
|
||||
export function validateGraphUpdate(graph, update) // → { valid: boolean, errors: string[] }
|
||||
```
|
||||
|
||||
### lib/graph/builder.js
|
||||
|
||||
```js
|
||||
export function buildInitialGraph(analysisData) // @param {{ reconstruction, evidence? }} → { nodes: SituationNode[], edges: SituationEdge[] }
|
||||
export function buildMinimalGraph(scenario) // @param {string} → { nodes, edges }
|
||||
export function describeGraph(graph) // @param {{ nodes, edges }} → string (summary text)
|
||||
```
|
||||
|
||||
## 2. Dependencies Between Files
|
||||
|
||||
```
|
||||
lib/analysis.js
|
||||
├── getConfig() from lib/config.js
|
||||
├── getProvider() from lib/llm/provider.js [EXTERNAL]
|
||||
├── buildPrompt() from lib/reconstruction/prompt.js
|
||||
└── reconstructionV2/V1Schema from lib/reconstruction/schema.js
|
||||
|
||||
lib/graph/utils.js ← imports situationNodeSchema, situationEdgeSchema, situationGraphSchema from schema.js
|
||||
lib/graph/builder.js ← imports situationNodeSchema, situationEdgeSchema, makeNodeId from schema.js
|
||||
docs/v0.4-handoff.md → references CaseOrchestrator.startCase()/updateCase() (not in any inspected file)
|
||||
```
|
||||
|
||||
## 3. Side Effects (LLM Calls)
|
||||
|
||||
| Function | LLM Call? | Details |
|
||||
|---|---|---|
|
||||
| `analyseScenario()` | **Yes** | `provider.generateReconstruction(prompt, model)` — POST to configured LLM. Prompt from `buildPrompt(scenario, version)`. |
|
||||
| All graph functions (`schema.js`, `utils.js`, `builder.js`) | No | Pure/deterministic only. |
|
||||
| `startCase()` / `updateCase()` (per handoff) | **Yes** | startCase: calls analyseScenario. updateCase: calls LLM via buildUpdatePrompt context + provider for GraphUpdate, then applyGraphUpdate(). |
|
||||
|
||||
## 4. Minimal Proposed Contract for API Functions
|
||||
|
||||
### startCase(body)
|
||||
- **Input:** `{ scenario: string (1-10000), promptVersion?: string }` — validated by `startCaseRequestSchema`.
|
||||
- **Flow:** validate → `analyseScenario()` → if ok, `buildInitialGraph(result)`; on failure return minimal graph via `buildMinimalGraph()`.
|
||||
- **Output (success):** `{ success: true, graphSummary: string, nodeCount: number, edgeCount: number, activeUnknownNodeId: string|undefined, nextQuestion: string }`
|
||||
- **Output (failure):** `{ success: false, error: string, graphSummary: string, nodeCount: number, edgeCount: number }`
|
||||
|
||||
### updateCase(body)
|
||||
- **Input:** `{ situationGraph: SituationGraph, previousQuestion: string (1+), answer: string (1-5000), promptVersion?: string }` — validated by `updateCaseRequestSchema`.
|
||||
- **Flow:** validate → `buildUpdatePrompt(ctx)` → LLM call for GraphUpdate proposal → `validateGraphUpdate()` → `applyGraphUpdate()` → resolve unknowns via `resolveUnknownNode()` → pick next candidate via `selectActiveUnknownCandidate()`.
|
||||
- **Output (success):** `{ success: true, graphSummary: string, nodeChanges: { added, updated, removed }, edgeChanges: { added, removed }, resolvedNodes: string[], nextQuestion: string|null }`
|
||||
- **Output (failure):** `{ success: false, error: string, graphSummary: string, nodeChanges: {}, edgeChanges: {}, resolvedNodes: [], nextQuestion: null }`
|
||||
|
||||
## 5. Missing Interfaces — TODO
|
||||
|
||||
1. **[TODO]** `CaseOrchestrator` class described in handoff but absent from all five inspected files. startCase()/updateCase() wrappers need implementation per above contract.
|
||||
2. **[TODO]** `buildUpdatePrompt(ctx)` (per handoff lives in prompt-builder.js) — not reviewed; input/output needs a separate doc once the file is available.
|
||||
3. **[TODO]** LLM provider interface (`getProvider()`, `generateReconstruction(prompt, model)`) — external dependency. Assumes rawResponse is parseable JSON matching v0.2/v0.1 schema; needs explicit contract.
|
||||
4. **[TODO]** Error handling for updateCase() on malformed LLM JSON — handoff notes "generic 500"; needs structured retry/error contract.
|
||||
5. **[TODO]** Completion heuristic `getCompletionStatus()` referenced in handoff but absent; needs contract (e.g., "complete" when no unresolved unknown nodes).
|
||||
|
||||
---
|
||||
|
||||
*End of contract.*
|
||||
@@ -1,258 +0,0 @@
|
||||
# v0.4 Handoff — Confidence Engine (confidence-engine)
|
||||
|
||||
**Date:** 2026-08-01
|
||||
**Branch:** `feature/reconstruction-v0.3`
|
||||
**Parent branch:** `main`
|
||||
|
||||
---
|
||||
|
||||
## 1. What This Project Is
|
||||
|
||||
A Next.js app that performs evidence-based situation reconstruction on user-supplied scenarios. An LLM analyses the scenario, builds a directed graph of actors, systems, unknowns and relationships, then iteratively refines the graph through multi-turn Q&A with the user.
|
||||
|
||||
---
|
||||
|
||||
## 2. Recent Commit History
|
||||
|
||||
| Commit | Message |
|
||||
|--------|---------|
|
||||
| `79ea2f6` | feat: add v0.3 normalised comparison reasoning |
|
||||
| `d72c7c5` | chore: establish clean v0.2 baseline |
|
||||
| `a2f9e47` | chore: preserve initial reconstruction prototype |
|
||||
|
||||
Only **one commit** ahead of `main`: `79ea2f6` — the v0.3 normalised comparison reasoning work.
|
||||
|
||||
---
|
||||
|
||||
## 3. Current State Summary
|
||||
|
||||
### What's done and committed to this branch
|
||||
|
||||
1. **v0.3 prompt** (`prompts/reconstruct-v0.3.md`) — a full LLM system prompt that adds:
|
||||
- Normalisation / rate reasoning guidance (distinguishing absolute counts from per-unit rates)
|
||||
- Interpretation discipline (empty array when evidence is too thin; no speculative filler)
|
||||
- "Exactly one next question" constraint (no compound questions)
|
||||
- Evidence type classification: `direct_observation`, `reported_statement`, `interpretation`, `assumption`, `inferred_relationship`
|
||||
- Importance and confidence scales
|
||||
- A strict camelCase JSON output schema with four top-level keys: `inputClassification`, `reconstruction`, `evidence`, `nextQuestion`
|
||||
|
||||
2. **v0.3 prompt versioning** (`lib/reconstruction/prompt.js`) — exports `PROMPT_VERSIONS`, `DEFAULT_PROMPT_VERSION ("v0.3")`, and `buildPrompt(scenario, version)` for loading prompt templates from disk with scenario substitution.
|
||||
|
||||
3. **Schema validation** (`lib/reconstruction/schema.js`) — Zod schemas for v0.2 output (`reconstructionV2Schema`). A `parseReconstructionV2(rawString)` helper is used in the analysis pipeline.
|
||||
|
||||
4. **v0.3 reasoning tests** (`tests/v03-reasoning.test.js`) — extensive test suite covering:
|
||||
- Prompt version registration and loading
|
||||
- v0.3 guidance completeness (normalisation, rate vs count, correlation-vs-causation)
|
||||
- Schema validation with a realistic "production/complaints" fixture
|
||||
- Parse helper tests
|
||||
|
||||
5. **Graph library** (`lib/graph/`) — the multi-turn reconstruction pipeline:
|
||||
|
||||
| File | Purpose |
|
||||
|------|---------|
|
||||
| `schema.js` | Zod schemas for SituationNode, SituationEdge, SituationGraph, GraphUpdate; helpers like `makeNodeId`, `makeNode`, `makeEdge`, `makeGraph` |
|
||||
| `builder.js` | `buildInitialGraph(reconstruction, evidence)` — converts v0.2/v0.3 analysis output into a SituationGraph with deterministic nodes/edges; `buildMinimalGraph(scenario)` for fallback; `describeGraph(graph)` for display |
|
||||
| `orchestrator.js` | `CaseOrchestrator` class managing the full multi-turn lifecycle (idle → building → active); exports `startCase(body)` and `updateCase(body)` convenience functions for API routes |
|
||||
| `prompt-builder.js` | `buildUpdatePrompt(ctx)` — formats current graph state + Q&A context into a system prompt for the LLM update-evaluation turn |
|
||||
| `utils.js` | Deterministic graph operations: `validateGraphReferences`, `detectDuplicateNodeIds`, `detectDuplicateEdges`, `findDependentNodes`, `findAffectedNodes`, `resolveUnknownNode`, `selectActiveUnknownCandidate`, `applyGraphUpdate`, `validateGraphUpdate` |
|
||||
|
||||
6. **API routes** (`app/api/`)
|
||||
|
||||
| Route | Purpose |
|
||||
|-------|---------|
|
||||
| `POST /api/start-case` | Start a new reconstruction case — accepts `{ scenario, promptVersion? }`, returns graph summary, node/edge counts, next question |
|
||||
| `POST /api/update-case` | Process a turn — accepts `{ scenario, graph, answer, currentQuestion?, turnCount?, modelName? }`, returns updated graph summary, next question, changes summary |
|
||||
|
||||
7. **Smoke test** (`tests/smoke.test.js`) — basic integration test for the start-case API route.
|
||||
|
||||
### What's NOT yet committed (untracked files from git status)
|
||||
|
||||
| File | Description |
|
||||
|------|-------------|
|
||||
| `lib/graph/` (full directory) | The multi-turn graph library — built but NOT yet committed to any branch. These are the new untracked files: `builder.js`, `orchestrator.js`, `prompt-builder.js`, `schema.js`, `utils.js` |
|
||||
| `tests/graph/` (full directory) | Tests for the graph library — also untracked: `builder.test.js`, `orchestrator.test.js`, `prompt-builder.test.js`, `schema.test.js`, `utils.test.js` |
|
||||
| `app/api/start-case/route.js` | New API route (untracked) |
|
||||
| `app/api/update-case/route.js` | New API route (untracked) |
|
||||
|
||||
> **Important:** The git status shows these files as untracked (`??`). They exist on disk but have never been staged or committed. You need to decide whether to commit them now or integrate them differently.
|
||||
|
||||
---
|
||||
|
||||
## 4. Test Status
|
||||
|
||||
```
|
||||
Test Files: 4 failed | 4 passed (8)
|
||||
Tests: 5 failed | 216 passed (221)
|
||||
```
|
||||
|
||||
### Known failures
|
||||
|
||||
The failures cluster in `tests/graph/`:
|
||||
- **`prompt-builder.test.js`** — test expects the literal string `"Existing or newly added nodes"` but the prompt template currently says `"existing or newly added nodes"` (case mismatch). The SYSTEM_PROMPT_HEADER constant uses lowercase.
|
||||
- Other graph tests likely have similar fixture/reference issues.
|
||||
|
||||
Run `npx vitest run tests/graph/ --reporter=verbose` for full details.
|
||||
|
||||
---
|
||||
|
||||
## 5. Architecture Overview
|
||||
|
||||
```
|
||||
User scenario
|
||||
│
|
||||
▼
|
||||
┌──────────────┐ ┌─────────────────┐ ┌──────────────┐
|
||||
│ analyseScenario│──▶│ buildPrompt │──▶│ LLM (v0.3) │
|
||||
│ (lib/analysis.js) │ (reconstruction/prompt.js) │ │
|
||||
└──────────────┘ └─────────────────┘ └──────┬───────┘
|
||||
│
|
||||
▼
|
||||
┌──────────────┐
|
||||
│ Parse output │
|
||||
│ (Zod/parse │
|
||||
│ Reconstruction│
|
||||
│ V2) │
|
||||
└──────┬───────┘
|
||||
│
|
||||
┌───────────────────────────────┤
|
||||
▼ ▼
|
||||
┌──────────────┐ ┌──────────────────┐
|
||||
│buildInitialGraph│ │ buildMinimalGraph │
|
||||
│ (graph/builder)│ │ (fallback) │
|
||||
└──────┬─────────┘ └──────────────────┘
|
||||
│
|
||||
▼
|
||||
┌──────────────┐
|
||||
│SituationGraph │ ← Zod-validated graph structure
|
||||
│ {nodes, edges}│ nodes: observation/metric/unknown/...
|
||||
└──────┬───────┘ edges: supports/weakens/causes/...
|
||||
│
|
||||
(multi-turn loop via updateCase)
|
||||
│
|
||||
┌─────────▼─────────┐
|
||||
│buildUpdatePrompt │ → LLM proposes GraphUpdate
|
||||
│ │
|
||||
│applyGraphUpdate │ → deterministic, validated
|
||||
│validateGraphUpdate│ (no direct LLM mutation)
|
||||
└───────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 6. Key Design Decisions
|
||||
|
||||
### Normalisation / rate reasoning (v0.3 focus)
|
||||
The v0.3 prompt explicitly instructs the model to:
|
||||
- Always consider whether a denominator/exposure metric is needed when counts change alongside scale
|
||||
- Distinguish absolute count from rate
|
||||
- Avoid treating two rising counts as causal evidence (production growth may outpace complaint growth)
|
||||
- Request the per-unit metric as the highest-value next question
|
||||
|
||||
### Graph immutability
|
||||
LLM proposals are never applied directly. All mutations go through `applyGraphUpdate()` in `lib/graph/utils.js`, which:
|
||||
- Validates all node/edge references exist
|
||||
- Rejects duplicate IDs
|
||||
- Enforces a max graph size (500 nodes) and update size (100KB)
|
||||
- Returns the full new state for validation
|
||||
|
||||
### Prompt versioning
|
||||
- Default is `"v0.3"` but `PROMPT_VERSIONS` includes `"v0.2"` for backward compatibility
|
||||
- `RECONSTRUCTION_PROMPT_VERSION` env var can override default at module load time
|
||||
- Prompts are loaded from `prompts/reconstruct-v0.{version}.md` on disk
|
||||
|
||||
### Deterministic node IDs
|
||||
Node IDs are computed via a deterministic hash of the label: `makeNodeId(label)`. This avoids conflicts but means nodes must be created with consistent labels to get consistent IDs.
|
||||
|
||||
---
|
||||
|
||||
## 7. Open Questions / TODOs for Next Developer
|
||||
|
||||
1. **Untracked graph library** — `lib/graph/` and `tests/graph/` are untracked on disk. Do we commit them as part of v0.4, or keep them in a separate branch?
|
||||
|
||||
2. **Test failures** — 5 tests fail across the graph test suite. The prompt-builder case-sensitivity issue needs fixing. Review all failing tests before merging.
|
||||
|
||||
3. **Missing `RECONSTRUCTION_PROMPT_VERSION` env var docs** — The system uses an env var override but it's not documented in `.env.example`. Add it if it's intended to be configurable.
|
||||
|
||||
4. **Provider integration** — `lib/llm/provider.js` is imported by the orchestrator (`getProvider()`, `generateReconstruction()`). Verify the provider implementation matches what this code expects.
|
||||
|
||||
5. **Graph completeness heuristic** — `CaseOrchestrator.getCompletionStatus()` returns `"complete"` when no unknown nodes remain, but doesn't consider whether all important observations have been verified.
|
||||
|
||||
6. **Error resilience in update flow** — If the LLM returns malformed JSON, the update route returns a 500 with a generic error message. Consider retry logic or structured error parsing.
|
||||
|
||||
7. **`buildUpdatePrompt` SYSTEM_PROMPT_HEADER is a module-level constant** — it's hardcoded and never versioned. If v0.5 changes the update-evaluation prompt style, this will need to become a template.
|
||||
|
||||
8. **The `nextQuestion` field on `/api/start-case` response** includes the adapted question (original + active unknown label appended). The client may want the original and adapted separately.
|
||||
|
||||
---
|
||||
|
||||
## 8. File Inventory (new / changed files on this branch)
|
||||
|
||||
### Prompts
|
||||
- `prompts/reconstruct-v0.3.md` — **NEW** — v0.3 system prompt (161 lines)
|
||||
- `prompts/reconstruct-v0.2.md` — **existing** — baseline prompt
|
||||
|
||||
### Core library
|
||||
- `lib/analysis.js` — **MODIFIED** — analyseScenario function (uses v0.3 prompt by default)
|
||||
- `lib/reconstruction/prompt.js` — **MODIFIED** — prompt versioning exports
|
||||
- `lib/reconstruction/schema.js` — **existing** — Zod schemas + parseReconstructionV2
|
||||
|
||||
### Graph library (untracked on disk)
|
||||
- `lib/graph/builder.js` — buildInitialGraph, buildMinimalGraph, describeGraph
|
||||
- `lib/graph/orchestrator.js` — CaseOrchestrator class, startCase, updateCase
|
||||
- `lib/graph/prompt-builder.js` — buildUpdatePrompt + SYSTEM_PROMPT_HEADER
|
||||
- `lib/graph/schema.js` — SituationNode/Edge/Graph/Update Zod schemas
|
||||
- `lib/graph/utils.js` — validation, dedup, dependency, and apply utilities
|
||||
|
||||
### API routes (untracked on disk)
|
||||
- `app/api/start-case/route.js`
|
||||
- `app/api/update-case/route.js`
|
||||
|
||||
### Tests (untracked on disk)
|
||||
- `tests/graph/builder.test.js`
|
||||
- `tests/graph/orchestrator.test.js`
|
||||
- `tests/graph/prompt-builder.test.js`
|
||||
- `tests/graph/schema.test.js`
|
||||
- `tests/graph/utils.test.js`
|
||||
- `tests/v03-reasoning.test.js` — **committed** to current branch
|
||||
- `tests/smoke.test.js`
|
||||
|
||||
### Config changes
|
||||
- `package.json` — added dependency (verify which one)
|
||||
- `playwright.config.js` — added/modified for integration testing
|
||||
- `.env.local` — exists locally (not committed)
|
||||
|
||||
---
|
||||
|
||||
## 9. How to Run
|
||||
|
||||
```bash
|
||||
# Install dependencies
|
||||
npm install
|
||||
|
||||
# Unit tests
|
||||
npx vitest run
|
||||
|
||||
# Graph library tests (has 5 failures)
|
||||
npx vitest run tests/graph/ --reporter=verbose
|
||||
|
||||
# Start dev server
|
||||
npm run dev
|
||||
|
||||
# API endpoints
|
||||
# POST /api/start-case → { scenario: "..." }
|
||||
# POST /api/update-case → { graph: {...}, answer: "...", ... }
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 10. What to Do First (Recommended Priorities)
|
||||
|
||||
1. **Review and fix the 5 failing tests** — likely simple string/fixture issues
|
||||
2. **Decide on the untracked files** — commit them, or create a v0.4 branch from this point
|
||||
3. **Verify the LLM provider integration** — ensure `getProvider()` and `generateReconstruction()` are wired up correctly
|
||||
4. **Add env var documentation** for `RECONSTRUCTION_PROMPT_VERSION` to `.env.example`
|
||||
5. **Smoke test end-to-end** — call `/api/start-case` with a real scenario and verify the full flow
|
||||
|
||||
---
|
||||
|
||||
*End of handoff.*
|
||||
@@ -1,25 +0,0 @@
|
||||
# v0.4 Route Status
|
||||
|
||||
- `app/api/cases/start/route.js`
|
||||
- Current tracked start-case route for the v0.4 graph orchestration path.
|
||||
- Covered by `tests/app/api/cases-start-route.test.js`.
|
||||
|
||||
- `app/api/cases/update/route.js`
|
||||
- Current tracked update-case route for the v0.4 graph orchestration path.
|
||||
- Delegates to `updateCase(body, { applyProposal: true })`.
|
||||
- Covered by `tests/app/api/cases-update-route.test.js`.
|
||||
|
||||
- `app/api/start-case/route.js`
|
||||
- Earlier experiment / duplicate start route.
|
||||
- No repository UI/test references were found.
|
||||
- Deleted from the working tree during UI connection cleanup.
|
||||
|
||||
- `app/api/update-case/route.js`
|
||||
- Earlier experimental duplicate update route.
|
||||
- Removed from the working tree during route consolidation.
|
||||
|
||||
- Current UI status
|
||||
- `components/scenario-form.jsx` now calls `/api/cases/start` for the main experimental flow.
|
||||
- `/api/cases/update` is the active tracked update route.
|
||||
- `/api/analyse` remains available for legacy one-shot analysis.
|
||||
- No UI changes were required for this route milestone.
|
||||
@@ -1,48 +0,0 @@
|
||||
# v0.5 Question Priority Generalisation
|
||||
|
||||
## Hypothesis
|
||||
|
||||
The current deterministic unknown selector and graph-context question formulator should generalise across several decision types by selecting a foundational unknown before downstream implementation or pricing leaves.
|
||||
|
||||
## Scenarios
|
||||
|
||||
1. Should we hire another engineer?
|
||||
2. Should we replace the delivery vans?
|
||||
3. Should we launch in another country?
|
||||
4. Should we continue a project that is over budget?
|
||||
5. Should we introduce a paid support tier?
|
||||
|
||||
## Results
|
||||
|
||||
| Scenario | Selected unknown | Strategy | Pass/Fail |
|
||||
| ---------------------------- | --------------------------- | -------------------- | --------- |
|
||||
| Hire another engineer | `hire-success-criteria` | `decision criterion` | Pass |
|
||||
| Replace the delivery vans | `van-reliability-threshold` | `decision criterion` | Pass |
|
||||
| Launch in another country | `country-value-threshold` | `actor/customer` | Pass |
|
||||
| Continue over-budget project | `project-benefit-threshold` | `decision criterion` | Pass |
|
||||
| Introduce paid support tier | `support-value-threshold` | `actor/customer` | Pass |
|
||||
|
||||
## Repeated failure patterns
|
||||
|
||||
Two repeated structural formulation failures appeared before the final pass:
|
||||
|
||||
1. **Constraint language in surrounding graph context outranked node-local decision-threshold language** in more than one case.
|
||||
2. **Baseline language in surrounding graph context outranked node-local threshold language** in more than one case.
|
||||
|
||||
Both failures affected formulation strategy, not deterministic unknown selection.
|
||||
|
||||
## Code change made
|
||||
|
||||
A small deterministic change was made in `lib/graph/question-formulator.js`:
|
||||
|
||||
- prefer node-local `definition` language before broader criterion inference
|
||||
- prefer node-local `decision criterion` language before context-only `constraint` inference
|
||||
- only treat `baseline` or `constraint` as primary when the selected node itself carries that language, otherwise allow them as fallback strategies later
|
||||
|
||||
No architecture, UI, persistence, prompt, scoring, additional model turns, or provider calls were added.
|
||||
|
||||
## Remaining limitations
|
||||
|
||||
- In two passing cases, the selector chose a threshold-style foundational node while the formulator still used an `actor/customer` strategy because related context strongly referenced customers or recipients.
|
||||
- This experiment is fixture-driven and deterministic; it is useful for regression protection, not scientific validation.
|
||||
- The suite exercises the production path without model calls, but it does not prove behaviour over arbitrary real-world graph structures.
|
||||
@@ -1,58 +0,0 @@
|
||||
# v0.5 Release Notes
|
||||
|
||||
## Purpose of v0.5
|
||||
|
||||
v0.5 stabilises the graph-backed one-turn update flow so the engine can resolve an answered unknown, surface consequential new unknowns, prioritise the next unknown deterministically, and formulate a deterministic follow-up question without changing the UI or adding more model turns.
|
||||
|
||||
## Capabilities proven
|
||||
|
||||
v0.5 includes:
|
||||
|
||||
- resolving an existing unknown
|
||||
- surfacing consequential new unknowns
|
||||
- limiting emergent unknowns
|
||||
- deterministic information-value prioritisation
|
||||
- deterministic question formulation
|
||||
- generalisation across five decision types
|
||||
- graph-backed one-turn UI update
|
||||
|
||||
## Five-case generalisation result
|
||||
|
||||
All five deterministic fixture scenarios passed:
|
||||
|
||||
1. Should we hire another engineer?
|
||||
2. Should we replace the delivery vans?
|
||||
3. Should we launch in another country?
|
||||
4. Should we continue a project that is over budget?
|
||||
5. Should we introduce a paid support tier?
|
||||
|
||||
The selector chose a foundational unknown first in each case, avoided the downstream leaf first, required no model call, and preserved graph immutability during question formulation.
|
||||
|
||||
## Key deterministic safeguards
|
||||
|
||||
- proposal application re-selects the active unknown deterministically after validation
|
||||
- information-value scoring penalises downstream or prerequisite-blocked unknowns
|
||||
- emergent unknown validation limits additions and requires explicit answer-derived linkage
|
||||
- final question wording is reformulated from graph context without an extra model turn
|
||||
- question validation rejects compound, awkward, or pricing-led fallback phrasing
|
||||
|
||||
## Known limitation
|
||||
|
||||
A correctly selected threshold node can still be phrased using an actor/customer strategy when surrounding graph context strongly references customers or value recipients.
|
||||
|
||||
This limitation is recorded for the next experiment and is not being fixed in the v0.5 release-prep task.
|
||||
|
||||
## Deliberately excluded work
|
||||
|
||||
- no reasoning-logic expansion beyond the small deterministic formulation fixes already landed on the branch
|
||||
- no new features
|
||||
- no UI changes
|
||||
- no persistence
|
||||
- no additional model turn
|
||||
- no Ollama calls for validation
|
||||
- no evaluator-suite runs
|
||||
- no Playwright runs
|
||||
|
||||
## Next experimental question
|
||||
|
||||
Can the question formulation strategy remain aligned with the selected node's role when surrounding graph context contains competing signals?
|
||||
@@ -1,40 +0,0 @@
|
||||
# v0.6 Ambiguity Generalisation
|
||||
|
||||
## Hypothesis
|
||||
|
||||
If the selector truly handles unjustified contradiction ties generically, it should return ambiguity across multiple domains without preferring one explanation by wording alone.
|
||||
|
||||
## Scenarios
|
||||
|
||||
1. Revenue increased by 18%, but cash in the bank fell over the same period.
|
||||
2. Customer satisfaction scores increased, but complaints also increased.
|
||||
3. Average delivery time decreased by 25%, but order cancellations increased.
|
||||
4. Website traffic doubled, but sales remained unchanged.
|
||||
5. Production output increased by 30%, but quality defects also increased.
|
||||
|
||||
## Observed behaviour
|
||||
|
||||
All five fixtures produced the same pattern:
|
||||
|
||||
- candidate count: 2
|
||||
- selector status: `ambiguous`
|
||||
- tie reason: `No justified distinction between leading unknowns.`
|
||||
- no explanation was favoured
|
||||
- one broad investigation question was produced from the central contradiction
|
||||
- neutral label renaming did not collapse ambiguity into a winner
|
||||
|
||||
## Repeated failure patterns
|
||||
|
||||
None observed across two or more scenarios.
|
||||
|
||||
The current ambiguity handling generalised cleanly across the five contradiction fixtures.
|
||||
|
||||
## Corrections
|
||||
|
||||
No production correction was required in this task.
|
||||
|
||||
## Lessons learned
|
||||
|
||||
- The current ambiguity path appears domain-agnostic when structure and semantic weights remain intentionally non-discriminating.
|
||||
- Central-statement-based tie questions are broad enough to avoid prematurely backing one branch.
|
||||
- The most useful regression signal is whether ambiguity survives neutral relabelling, not whether one label sorts ahead of another in display order.
|
||||
@@ -1,211 +0,0 @@
|
||||
# v0.6 Atomicity Experiment
|
||||
|
||||
## Hypothesis
|
||||
|
||||
After deterministic unknown selection, the engine should assess whether the selected unknown is already atomic or is still too composite to ask directly.
|
||||
|
||||
If the unknown is atomic, the engine should proceed exactly as before.
|
||||
|
||||
If the unknown is composite, the engine should not ask that parent unknown directly. Instead, it should decompose it into a small set of explicit child unknowns representing broad, independent candidate dimensions that a non-expert could understand.
|
||||
|
||||
## Constraints
|
||||
|
||||
- No graph redesign
|
||||
- No persistence
|
||||
- No UI redesign
|
||||
- No selection-weight tuning
|
||||
- No Ollama calls in unit tests
|
||||
|
||||
## Deterministic rule introduced
|
||||
|
||||
Atomicity assessment is **not** a new investigation strategy.
|
||||
|
||||
It runs in the graph update path at this seam:
|
||||
|
||||
```text
|
||||
unknown selection -> atomicity assessment -> optional decomposition -> deterministic reselection -> question formulation
|
||||
```
|
||||
|
||||
The implementation uses deterministic text and graph-shape checks:
|
||||
|
||||
- focused unknowns like denominator / threshold / definition / baseline / evidence remain **atomic**
|
||||
- broad relationship-explanation unknowns and broad “possible causes / what changed / explanation for why X but Y” unknowns become **composite**
|
||||
|
||||
## Decomposition behavior
|
||||
|
||||
When a selected unknown is composite:
|
||||
|
||||
1. The parent unknown remains unresolved.
|
||||
2. Between 2 and 5 child unknowns are created or reused deterministically.
|
||||
3. Children become explicit graph nodes.
|
||||
4. Children link back to the parent with existing `depends_on` edges.
|
||||
5. Children inherit the same “why it matters” discipline in their descriptions.
|
||||
6. Deterministic selection reruns across the updated graph.
|
||||
|
||||
For the current relationship-explanation experiment, the broad child dimensions are:
|
||||
|
||||
- Whether the two observations reflect different timing
|
||||
- How the two observations were measured
|
||||
- Change affecting signal A more than signal B
|
||||
- Change affecting signal B more than signal A
|
||||
- One-off event during the period
|
||||
|
||||
These are intentionally non-jargon and broad enough to generalise across scenarios like:
|
||||
|
||||
- Revenue up / Cash down
|
||||
- Customer satisfaction up / Complaints up
|
||||
- Delivery time down / Cancellations up
|
||||
- Traffic up / Sales flat
|
||||
- Production up / Defects up
|
||||
|
||||
## Diagnostics added
|
||||
|
||||
The orchestrator now reports:
|
||||
|
||||
- `atomicityAssessment`
|
||||
- `atomicityDecisionReason`
|
||||
- `decompositionDepth`
|
||||
- `decompositionAttempted`
|
||||
- `decompositionAccepted`
|
||||
- `decompositionStoppedReason`
|
||||
- `proposedChildCount`
|
||||
- `acceptedChildCount`
|
||||
- `rejectedChildren`
|
||||
- `selectedChildNodeId`
|
||||
- `childQualitySummary`
|
||||
- `propagationPerformed`
|
||||
- `resolvedChildNodeId`
|
||||
- `parentNodeId`
|
||||
- `parentStatusBefore`
|
||||
- `parentStatusAfter`
|
||||
- `parentConfidenceBefore`
|
||||
- `parentConfidenceAfter`
|
||||
- `affectedAncestorIds`
|
||||
- `nextSelectedSibling`
|
||||
- `parentResolved`
|
||||
- `decompositionPerformed`
|
||||
- `childUnknownCount`
|
||||
- `childNodeIds`
|
||||
- `atomicityReason`
|
||||
|
||||
This sits alongside the existing explicit-emergent-unknown diagnostics.
|
||||
|
||||
## Observed outcome
|
||||
|
||||
The experiment was useful.
|
||||
|
||||
Before this change, the engine could select a broad explanation unknown and ask it directly.
|
||||
|
||||
After this change:
|
||||
|
||||
- the broad explanation parent remains explicit in the graph
|
||||
- the engine decomposes it into child unknowns first
|
||||
- the next asked question is backed by a more focused child unknown
|
||||
- repeated updates reuse the same decomposition children deterministically
|
||||
- child-quality checks reject compound or duplicate children before they enter the graph
|
||||
- decomposition stops deterministically once a selected child is directly answerable
|
||||
- resolving one child does not resolve the parent immediately
|
||||
- resolved child evidence now propagates upward to the parent and ancestor chain deterministically
|
||||
- parent status and confidence change conservatively after child resolution
|
||||
- the next sibling becomes eligible for normal deterministic selection without recreating the resolved child
|
||||
|
||||
In the revenue-versus-cash case, the selected next question becomes:
|
||||
|
||||
> What evidence would clarify how the two observations were measured?
|
||||
|
||||
rather than asking the full broad explanation node directly.
|
||||
|
||||
## Upward propagation and reconstruction
|
||||
|
||||
Recursive reasoning is complete only when decomposition and reconstruction are both deterministic.
|
||||
|
||||
Confidence must not outrun completeness or evidence.
|
||||
|
||||
For this experiment, reconstruction now behaves as follows:
|
||||
|
||||
- when a child unknown resolves, that child keeps its own resolved status and answer evidence
|
||||
- the parent is updated, but remains unresolved unless the deterministic completion rule is satisfied
|
||||
- only the ancestor chain connected to that child is updated
|
||||
- unrelated branches remain unchanged
|
||||
- the deterministic selector then chooses the next justified unresolved sibling or related follow-up
|
||||
|
||||
For the current conservative completion rule:
|
||||
|
||||
- **one resolved child** → parent becomes `provisional` with higher confidence, but remains unresolved
|
||||
- **all direct child unknowns resolved** → parent resolves deterministically with `high` confidence
|
||||
|
||||
The confidence model is now explicitly separated into:
|
||||
|
||||
- **evidence confidence**: how trustworthy the currently attached support is
|
||||
- **completeness**: whether the required direct child structure is empty, partial, or complete
|
||||
- **conclusion confidence**: how strongly the current parent state is justified given both evidence and completeness
|
||||
|
||||
Deterministic propagation rules now enforce:
|
||||
|
||||
- one resolved child may raise evidence confidence
|
||||
- unresolved direct children cap conclusion confidence
|
||||
- contradictory direct children block high conclusion confidence
|
||||
- duplicate evidence does not increase confidence
|
||||
- status changes do not raise confidence on their own
|
||||
- parent resolution still requires the separate completion rule
|
||||
|
||||
## Cross-branch corroboration
|
||||
|
||||
The next confidence experiment adds deterministic branch interaction checks without changing the graph model.
|
||||
|
||||
The engine now distinguishes between:
|
||||
|
||||
- **multiple evidence**: more than one branch exists
|
||||
- **independent corroboration**: distinct resolved branches support the same parent without sharing the same evidence key
|
||||
- **duplicate evidence**: the same evidence key appears through multiple branches and must not be double-counted
|
||||
- **conflicting evidence**: branches support incompatible positions, such as `recognised correctly` vs `recognised incorrectly`
|
||||
|
||||
Deterministic branch rules:
|
||||
|
||||
- corroboration only counts when branches are distinct and their evidence sources differ
|
||||
- duplicate evidence groups never count as corroboration
|
||||
- conflicts cap conclusion confidence and prevent a higher confidence upgrade
|
||||
- independent branches remain interaction-neutral
|
||||
|
||||
Additional diagnostics now expose:
|
||||
|
||||
- `corroboratingBranchCount`
|
||||
- `conflictingBranchCount`
|
||||
- `duplicateEvidenceCount`
|
||||
- `independentBranchCount`
|
||||
- `interactionSummary`
|
||||
- `confidenceAdjustmentReason`
|
||||
|
||||
Observed effect:
|
||||
|
||||
- independent corroboration can raise `evidenceConfidence`
|
||||
- duplicate evidence produces no extra confidence increase
|
||||
- conflicting evidence lowers or caps `conclusionConfidence`
|
||||
- completeness rules still dominate whether a parent may become highly justified
|
||||
|
||||
Example progression:
|
||||
|
||||
- parent before: `unknown`, `medium`
|
||||
- after resolving `How the two observations were measured`: parent becomes `provisional`, `medium`
|
||||
- evidence confidence becomes `high`, completeness becomes `partial`, conclusion confidence becomes `medium`
|
||||
- next sibling becomes selectable and the engine moves on without recreating the resolved child
|
||||
|
||||
## Interpretation
|
||||
|
||||
This supports the idea that recursive decomposition is a fundamental part of graph-backed questioning, not just a prompt refinement.
|
||||
|
||||
The main remaining limitation is that sibling selection still inherits the existing deterministic scorer. That means some domains may advance to a justified sibling that is not the intuitively expected next child, even though the propagation itself remains deterministic and graph-valid.
|
||||
|
||||
## Validation run
|
||||
|
||||
Covered by:
|
||||
|
||||
- `tests/graph/atomicity-assessment.test.js`
|
||||
- `tests/graph/decomposition-quality.test.js`
|
||||
- `tests/graph/upward-propagation.test.js`
|
||||
- `tests/graph/apply-proposal.test.js`
|
||||
- `tests/graph/orchestrator.test.js`
|
||||
- `tests/graph/question-formulator.test.js`
|
||||
- `tests/ui/scenario-form.test.jsx`
|
||||
|
||||
And then by the broader requested validation pass with lint and build.
|
||||
@@ -1,48 +0,0 @@
|
||||
# v0.6 Comparability Experiment
|
||||
|
||||
## Hypothesis
|
||||
|
||||
The engine should confirm that observations are comparable before treating their difference as a contradiction that needs explanatory follow-up.
|
||||
|
||||
## Fixtures
|
||||
|
||||
1. Revenue increased by 18%, but cash in the bank fell over the same period.
|
||||
2. Complaints increased. Production increased.
|
||||
3. Average delivery time decreased by 25%, but order cancellations increased.
|
||||
4. Customer satisfaction increased, but complaints increased.
|
||||
5. Temperature increased. Ice melted.
|
||||
6. Sales doubled. Sales doubled.
|
||||
|
||||
## Results
|
||||
|
||||
- The first four scenarios repeated the same failure pattern: contradiction-level investigation could begin before comparability was established.
|
||||
- A deterministic comparability gate corrected that by producing one comparison question first.
|
||||
- Confirmed comparability did not by itself imply contradiction.
|
||||
- Temperature increased / Ice melted was reclassified as a compatible relationship, so no contradiction question was asked.
|
||||
- Sales doubled / Sales doubled was reclassified as duplicate observations, so no follow-up question was asked.
|
||||
|
||||
## Relationship classification stage
|
||||
|
||||
After comparability assessment, observations now pass through a deterministic relationship classification stage:
|
||||
|
||||
- `contradictory`
|
||||
- `compatible`
|
||||
- `potentially_related`
|
||||
- `duplicate`
|
||||
- `insufficient_information`
|
||||
|
||||
## Whether comparability should become a permanent reasoning stage
|
||||
|
||||
Yes, in minimal deterministic form.
|
||||
|
||||
The repeated pattern appeared in four scenarios, so a small pre-contradiction comparability assessment is justified.
|
||||
|
||||
## Two-step experiment result
|
||||
|
||||
A comparison question is useful only if its answer advances the reasoning stage rather than merely adding more text.
|
||||
|
||||
In the revenue-versus-cash scenario, the first question now confirms whether the figures are comparable, and the answer resolves that existing uncertainty instead of creating a parallel note. After that update, the engine progresses from comparability assessment to cautious relationship assessment and can select one broad non-expert follow-up question.
|
||||
|
||||
Every justified next question should correspond to an explicit unresolved graph node.
|
||||
|
||||
The earlier fallback-only path has now been removed from the normal successful progression. After comparability is resolved and a further investigation question is justified, the engine creates or reuses an explicit unresolved reasoning unknown and lets deterministic selection and question formulation proceed through the standard graph pipeline. A fallback is now only acceptable as an explicit failure case, not as the normal source of the next question.
|
||||
@@ -1,375 +0,0 @@
|
||||
# v0.6 Reasoning Architecture
|
||||
|
||||
## Purpose
|
||||
|
||||
This document describes the implemented deterministic reasoning architecture on branch `feature/question-strategy-alignment-v0.6`.
|
||||
|
||||
It is written for future developers who need to understand how v0.6 actually executes, what invariants it relies on, where the recursive loops are, and what the system deliberately does **not** attempt to do.
|
||||
|
||||
## End-to-end pipeline
|
||||
|
||||
The implemented runtime pipeline is:
|
||||
|
||||
```text
|
||||
Scenario input
|
||||
↓
|
||||
LLM analysis / reconstruction
|
||||
↓
|
||||
Initial graph build
|
||||
↓
|
||||
Deterministic unknown selection
|
||||
↓
|
||||
Selected question
|
||||
↓
|
||||
User answer
|
||||
↓
|
||||
LLM graph-update proposal
|
||||
↓
|
||||
Proposal parsing / normalisation
|
||||
↓
|
||||
Proposal compatibility validation
|
||||
↓
|
||||
Deterministic graph update application
|
||||
↓
|
||||
Reasoning-state rebuild
|
||||
↓
|
||||
Comparability assessment
|
||||
↓
|
||||
Relationship classification
|
||||
↓
|
||||
Explicit emergent unknown creation / reuse (if required)
|
||||
↓
|
||||
Deterministic reselection
|
||||
↓
|
||||
Atomicity assessment
|
||||
↓
|
||||
Optional decomposition into child unknowns
|
||||
↓
|
||||
Deterministic reselection
|
||||
↓
|
||||
Resolved-child propagation upward
|
||||
↓
|
||||
Confidence / completeness / corroboration update
|
||||
↓
|
||||
Next active unknown
|
||||
↓
|
||||
Question formulation
|
||||
```
|
||||
|
||||
## Deterministic stages
|
||||
|
||||
### 1. Scenario analysis / reconstruction
|
||||
|
||||
- **Purpose**: obtain structured reconstruction material from scenario text
|
||||
- **Input**: scenario, prompt version
|
||||
- **Output**: analysis payload containing reconstruction, evidence, diagnostics, and optional next question
|
||||
- **Why it exists**: provides the initial structured substrate from which the graph is built
|
||||
- **What breaks if removed**: the graph builder has no structured reconstruction to convert into nodes and edges
|
||||
|
||||
### 2. Initial graph build
|
||||
|
||||
- **Purpose**: convert reconstruction output into an initial `SituationGraph`
|
||||
- **Input**: reconstruction + evidence
|
||||
- **Output**: graph nodes and edges, then `makeGraph(...)` wraps them with active/resolved/summary state
|
||||
- **Why it exists**: all later reasoning is graph-based, not free text
|
||||
- **What breaks if removed**: no explicit unknown nodes, no deterministic selection, no validated update loop
|
||||
|
||||
### 3. Deterministic unknown selection
|
||||
|
||||
- **Purpose**: choose the next active unknown from unresolved graph nodes
|
||||
- **Input**: graph, resolved node IDs
|
||||
- **Output**: selected candidate or explicit ambiguity result
|
||||
- **Why it exists**: the system needs a deterministic next investigation target
|
||||
- **What breaks if removed**: question ordering becomes arbitrary or hidden in prompts
|
||||
|
||||
### 4. Selected question exposure
|
||||
|
||||
- **Purpose**: expose the chosen unknown as the next question to the user
|
||||
- **Input**: selected unknown + question formulation or tie-resolution logic
|
||||
- **Output**: selected question object
|
||||
- **Why it exists**: the user-facing loop must ask a concrete next question
|
||||
- **What breaks if removed**: the system can build a graph but cannot continue interaction coherently
|
||||
|
||||
### 5. LLM graph-update proposal
|
||||
|
||||
- **Purpose**: transform a user answer into a proposed graph change set
|
||||
- **Input**: current graph, previous question, answer, prompt version
|
||||
- **Output**: raw JSON-like proposal
|
||||
- **Why it exists**: the LLM is limited to proposing changes; it does not mutate the graph directly
|
||||
- **What breaks if removed**: answers cannot affect the graph except through manual hard-coded logic
|
||||
|
||||
### 6. Proposal parsing / normalisation
|
||||
|
||||
- **Purpose**: parse JSON, remove null array items, apply known aliases, fill omitted optional fields
|
||||
- **Input**: raw model response
|
||||
- **Output**: validated `graphUpdateSchema` payload or structured parser failure
|
||||
- **Why it exists**: model outputs are not trusted as-is
|
||||
- **What breaks if removed**: malformed or partially missing model output would reach graph logic directly
|
||||
|
||||
### 7. Proposal compatibility validation
|
||||
|
||||
- **Purpose**: ensure the proposal is graph-safe and semantically valid before application
|
||||
- **Input**: current graph + proposed update
|
||||
- **Output**: accepted proposal or compatibility errors
|
||||
- **Why it exists**: protects graph integrity and reasoning invariants
|
||||
- **What breaks if removed**: duplicate IDs, missing references, fake selected questions, and no-op updates could corrupt the graph
|
||||
|
||||
### 8. Deterministic graph update application
|
||||
|
||||
- **Purpose**: apply only validated graph changes to a copied graph
|
||||
- **Input**: graph + validated proposal
|
||||
- **Output**: updated nodes, edges, resolved node IDs
|
||||
- **Why it exists**: separates safe application from generation
|
||||
- **What breaks if removed**: no explicit, replayable state transition exists
|
||||
|
||||
### 9. Reasoning-state rebuild
|
||||
|
||||
- **Purpose**: derive fresh comparability/relationship state from the updated graph
|
||||
- **Input**: updated graph + optional override state
|
||||
- **Output**: `reasoningState`
|
||||
- **Why it exists**: reasoning stages are derived from graph state, not stored blindly
|
||||
- **What breaks if removed**: comparability and relationship decisions drift from actual graph contents
|
||||
|
||||
### 10. Comparability assessment
|
||||
|
||||
- **Purpose**: decide whether supported observations are comparable enough for relationship reasoning
|
||||
- **Input**: graph observations + central statement + optional stored override
|
||||
- **Output**: comparability status/reason + contradiction permission
|
||||
- **Why it exists**: relationship reasoning is gated by comparability
|
||||
- **What breaks if removed**: contradiction or relationship reasoning would run over incomparable observations
|
||||
|
||||
### 11. Relationship classification
|
||||
|
||||
- **Purpose**: classify observation relationships once comparability permits it
|
||||
- **Input**: graph + comparability result
|
||||
- **Output**: relationship status, reason, whether a follow-up question is justified
|
||||
- **Why it exists**: determines whether explanation-style follow-up is needed
|
||||
- **What breaks if removed**: the system cannot distinguish compatible, duplicate, insufficient, and contradiction-adjacent observation sets
|
||||
|
||||
### 12. Explicit emergent unknown creation / reuse
|
||||
|
||||
- **Purpose**: ensure any justified relationship follow-up is represented by an explicit unresolved graph node
|
||||
- **Input**: provisional graph + relationship assessment
|
||||
- **Output**: reused or newly added explanation unknown and edges
|
||||
- **Why it exists**: preserves the invariant that a question must originate from an explicit unknown
|
||||
- **What breaks if removed**: relationship follow-up would revert to fallback-only question text not backed by the graph
|
||||
|
||||
### 13. Atomicity assessment
|
||||
|
||||
- **Purpose**: determine whether the selected unknown is directly investigable or too composite
|
||||
- **Input**: selected unknown + graph context
|
||||
- **Output**: `atomic` or `composite` decision with decomposition kind/reason
|
||||
- **Why it exists**: prevents asking broad explanation unknowns directly
|
||||
- **What breaks if removed**: the system asks high-level composite unknowns instead of decomposing them first
|
||||
|
||||
### 14. Optional decomposition
|
||||
|
||||
- **Purpose**: split a composite unknown into deterministic child unknowns
|
||||
- **Input**: composite selected unknown + graph context
|
||||
- **Output**: 2–5 child unknowns, edges, quality summary, rejection diagnostics
|
||||
- **Why it exists**: narrows broad unknowns into explicit candidate dimensions
|
||||
- **What breaks if removed**: recursive reasoning stops at broad parents and loses graph-backed substructure
|
||||
|
||||
### 15. Resolved-child propagation upward
|
||||
|
||||
- **Purpose**: move resolved child effects to parent and ancestor chain without prematurely resolving them
|
||||
- **Input**: updated graph + proposal snapshot
|
||||
- **Output**: parent/ancestor status and confidence updates, additional diagnostics
|
||||
- **Why it exists**: decomposition requires deterministic reconstruction as well as decomposition
|
||||
- **What breaks if removed**: child answers stay local and parents never become progressively better-supported
|
||||
|
||||
### 16. Confidence / completeness / corroboration update
|
||||
|
||||
- **Purpose**: derive parent-level `confidenceAssessment` from resolved children and branch interactions
|
||||
- **Input**: parent child set + branch evidence/status interactions
|
||||
- **Output**: `evidenceConfidence`, `completenessStatus`, `conclusionConfidence`, plus derived display `confidence`
|
||||
- **Why it exists**: reasoning support must be separated from completion and contradiction state
|
||||
- **What breaks if removed**: parent confidence collapses back into vague status-driven heuristics
|
||||
|
||||
### 17. Next active unknown + question formulation
|
||||
|
||||
- **Purpose**: reselect the next unresolved unknown and formulate a concrete next question
|
||||
- **Input**: updated graph + selection state + graph context
|
||||
- **Output**: next active unknown and question object
|
||||
- **Why it exists**: closes the recursive interaction loop
|
||||
- **What breaks if removed**: the system updates the graph but cannot continue investigation deterministically
|
||||
|
||||
## Architectural invariants
|
||||
|
||||
The current implementation enforces these invariants:
|
||||
|
||||
1. **A question must originate from an explicit unresolved unknown node.**
|
||||
2. **Unknown selection is deterministic.**
|
||||
3. **Alphabetical ordering is not treated as reasoning.**
|
||||
4. **Relationship reasoning does not precede comparability.**
|
||||
5. **The LLM never mutates the graph directly; it only proposes updates.**
|
||||
6. **All graph updates are schema-validated before application.**
|
||||
7. **All node/edge references must resolve to existing nodes.**
|
||||
8. **Duplicate node IDs are rejected.**
|
||||
9. **Duplicate added edge IDs are rejected.**
|
||||
10. **A selected question cannot target a resolved unknown.**
|
||||
11. **A resolved unknown updated to `resolved` must also appear in `resolvedUnknownNodeIds`.**
|
||||
12. **A proposal must contain a meaningful change.**
|
||||
13. **Every newly added unknown must include why-it-matters language.**
|
||||
14. **Every newly added unknown must be explicitly connected to answer-derived graph structure.**
|
||||
15. **Composite selected unknowns are decomposed before direct questioning when atomicity rules require it.**
|
||||
16. **Parent unknowns remain unresolved until completion rules are satisfied.**
|
||||
17. **Confidence must not outrun completeness.**
|
||||
18. **Duplicate evidence cannot increase confidence.**
|
||||
19. **Conflicting evidence caps conclusion confidence.**
|
||||
20. **Cross-branch corroboration only counts for distinct branches with distinct evidence keys.**
|
||||
21. **Ambiguous leading unknowns remain explicit ambiguity, not silent forced choice.**
|
||||
|
||||
## Recursive loops and stopping rules
|
||||
|
||||
### Main investigation loop
|
||||
|
||||
```text
|
||||
Unknown
|
||||
↓
|
||||
Question
|
||||
↓
|
||||
Answer
|
||||
↓
|
||||
Proposal
|
||||
↓
|
||||
Graph update
|
||||
↓
|
||||
Propagation
|
||||
↓
|
||||
Next unknown
|
||||
```
|
||||
|
||||
- **Exit condition**: no unresolved candidates remain, or no next question is justified, or proposal/application fails
|
||||
- **Stopping rule**: deterministic selection returns `null` or explicit ambiguity, or update validation blocks progress
|
||||
- **Completion behaviour**: continues only while the graph contains justified unresolved unknowns
|
||||
|
||||
### Decomposition loop
|
||||
|
||||
```text
|
||||
Selected unknown
|
||||
↓
|
||||
Atomicity assessment
|
||||
↓
|
||||
If composite: decompose
|
||||
↓
|
||||
Reselect child
|
||||
↓
|
||||
Atomicity assessment again
|
||||
```
|
||||
|
||||
- **Exit condition**: selected child is atomic; parent already has children; max decomposition depth reached; or decomposition quality fails
|
||||
- **Stopping rule**: `MAX_DECOMPOSITION_DEPTH`, reuse instead of regeneration, or inability to produce enough valid child unknowns
|
||||
- **Completion behaviour**: deterministic and bounded; no infinite recursive decomposition path is intentionally allowed
|
||||
|
||||
### Propagation loop
|
||||
|
||||
```text
|
||||
Resolved child
|
||||
↓
|
||||
Ancestor chain walk
|
||||
↓
|
||||
Recompute parent state
|
||||
↓
|
||||
Stop when no ancestor state changes
|
||||
```
|
||||
|
||||
- **Exit condition**: no more parents in the ancestor chain or no state change
|
||||
- **Stopping rule**: ancestor chain is explicit and finite; propagation does not invent new ancestors
|
||||
- **Completion behaviour**: deterministic upward traversal with explicit stop on unchanged state
|
||||
|
||||
### Potential infinite loops reviewed
|
||||
|
||||
- **Unknown/question recursion**: bounded by unresolved unknown set, proposal validation, and explicit no-candidate states
|
||||
- **Decomposition recursion**: bounded by max depth and child reuse rules
|
||||
- **Propagation recursion**: bounded by finite ancestor chain and no-change stop condition
|
||||
|
||||
No intentional infinite reasoning loop is present in the implemented architecture.
|
||||
|
||||
## Graph lifecycle summary
|
||||
|
||||
### Node lifecycle
|
||||
|
||||
1. node created by `buildInitialGraph` or later proposal/decomposition/emergent-unknown logic
|
||||
2. node validated by schema
|
||||
3. node may become active unknown
|
||||
4. node may be updated by proposal application
|
||||
5. unknown node may become `resolved`, `provisional`, `contradicted`, or remain `unknown`
|
||||
6. resolved unknown ID is tracked in `resolvedNodeIds`
|
||||
|
||||
### Edge lifecycle
|
||||
|
||||
1. edge created in initial graph or by deterministic proposal augmentation
|
||||
2. edge validated against existing node IDs
|
||||
3. edge may be removed only through explicit `removedEdgeIds`
|
||||
4. edge relationships also update `dependsOn` / `childIds` projections during application
|
||||
|
||||
### Unknown lifecycle
|
||||
|
||||
1. initial unknown discovered from reconstruction
|
||||
2. selected deterministically or left ambiguous
|
||||
3. may be decomposed if composite
|
||||
4. may be resolved directly by answer
|
||||
5. may cause emergent reasoning unknown creation when relationship reasoning demands a new explicit question target
|
||||
|
||||
### Resolved lifecycle
|
||||
|
||||
1. proposal marks unresolved unknown resolved
|
||||
2. reconciliation ensures resolution semantics are explicit
|
||||
3. `resolvedUnknownNodeIds` feed graph application
|
||||
4. propagation may resolve parent only when completion rule is met
|
||||
|
||||
### Confidence lifecycle
|
||||
|
||||
1. nodes begin with base `confidence`
|
||||
2. parent/ancestor propagation derives `confidenceAssessment`
|
||||
3. display `confidence` is derived from `conclusionConfidence`
|
||||
4. completeness, duplicate evidence, contradiction, and corroboration constrain the result
|
||||
|
||||
### Question lifecycle
|
||||
|
||||
1. selected unknown becomes question target
|
||||
2. `formulateQuestion` or tie-resolution logic produces question text
|
||||
3. answer returns through update route
|
||||
4. proposal may select a new question target or leave reselection to deterministic logic
|
||||
|
||||
Every major transition above is explicit in the current codebase rather than implicit in model text alone.
|
||||
|
||||
## Duplicated or overlapping concepts
|
||||
|
||||
The following concepts are intentionally close and may look duplicated:
|
||||
|
||||
- **status vs confidence**: status captures lifecycle/progression; confidence captures support strength
|
||||
- **confidence vs confidenceAssessment**: `confidence` is now a derived display field, while `confidenceAssessment` carries separated reasoning dimensions
|
||||
- **resolvedNodeIds vs node.status === resolved**: both are maintained; the first is a graph-level index, the second is node-local state
|
||||
- **selectedQuestion in proposal vs selectedQuestion in final result**: proposal may omit or propose one, final result recomputes deterministic selection/questioning after graph logic
|
||||
- **comparability state in reasoningState vs derived comparability from graph**: overrides may carry forward prior confirmed reasoning, but `buildReasoningState` still rebuilds from graph + override context
|
||||
|
||||
These are not necessarily defects, but they are the main places where future simplification pressure is likely.
|
||||
|
||||
## Known boundaries and deliberate exclusions
|
||||
|
||||
v0.6 deliberately does **not** attempt the following:
|
||||
|
||||
- probabilistic reasoning
|
||||
- Bayesian inference
|
||||
- persistence
|
||||
- semantic embeddings
|
||||
- fuzzy semantic similarity
|
||||
- autonomous exploration outside explicit user answers
|
||||
- multi-hop corroboration across unrelated subtrees without a shared direct parent
|
||||
- expert-only jargon-specific reasoning modes
|
||||
- UI-heavy reasoning visualisation beyond existing graph/update displays
|
||||
- arbitrary non-deterministic tie breaking
|
||||
|
||||
## Defects found during this review
|
||||
|
||||
No new production defect was intentionally introduced or fixed as part of this architecture review.
|
||||
|
||||
## Developer notes
|
||||
|
||||
- `startCase` owns reconstruction → graph build → first deterministic selection.
|
||||
- `updateCaseWithDependencies` owns proposal generation / parsing and delegates deterministic graph semantics to `applyValidatedProposal`.
|
||||
- `applyValidatedProposal` is the main reasoning pipeline coordinator for update-time graph evolution.
|
||||
- `question-formulator.js` owns comparability, relationship classification, atomicity assessment, investigation strategy selection, and question formulation.
|
||||
- `utils.js` owns selection scoring, ordering, graph validation, and safe graph update application.
|
||||
@@ -1,89 +0,0 @@
|
||||
# v0.6 Release Notes
|
||||
|
||||
## Purpose
|
||||
|
||||
v0.6 turns the engine into a deterministic recursive reasoning system that keeps next questions, decomposition, propagation, and confidence updates explicitly grounded in the situation graph.
|
||||
|
||||
## Capabilities added
|
||||
|
||||
- deterministic unknown selection explanations
|
||||
- explicit ambiguity handling instead of silent tie-breaking
|
||||
- comparability assessment before relationship reasoning
|
||||
- relationship classification after comparability
|
||||
- reasoning-stage progression after comparability answers
|
||||
- graph-backed next questions via explicit unknown nodes
|
||||
- investigation-strategy-based question formulation
|
||||
- atomicity assessment for selected unknowns
|
||||
- composite-unknown decomposition into child unknowns
|
||||
- child-quality validation for decomposition outputs
|
||||
- upward propagation from resolved children to parents and ancestors
|
||||
- separation of evidence confidence, completeness, and conclusion confidence
|
||||
- deterministic cross-branch corroboration, conflict, and duplicate-evidence handling
|
||||
- developer-facing reasoning architecture documentation
|
||||
|
||||
## Reasoning pipeline summary
|
||||
|
||||
```text
|
||||
Scenario
|
||||
→ Reconstruction
|
||||
→ Initial graph
|
||||
→ Deterministic unknown selection
|
||||
→ Question
|
||||
→ Answer
|
||||
→ Proposal
|
||||
→ Proposal parsing / validation
|
||||
→ Graph update
|
||||
→ Reasoning-state rebuild
|
||||
→ Comparability assessment
|
||||
→ Relationship classification
|
||||
→ Emergent unknown creation / reuse
|
||||
→ Atomicity assessment
|
||||
→ Optional decomposition
|
||||
→ Propagation
|
||||
→ Confidence / completeness / corroboration update
|
||||
→ Next active unknown
|
||||
→ Next question
|
||||
```
|
||||
|
||||
## Core invariants
|
||||
|
||||
- every asked question must originate from an explicit unresolved unknown
|
||||
- unknown selection is deterministic
|
||||
- ambiguity is preserved explicitly when no justified distinction exists
|
||||
- relationship reasoning cannot precede comparability
|
||||
- parent nodes cannot resolve before completion rules are met
|
||||
- confidence cannot outrun completeness
|
||||
- duplicate evidence cannot increase confidence
|
||||
- conflicting evidence caps conclusion confidence
|
||||
- cross-branch corroboration only counts for distinct branches with distinct evidence keys
|
||||
- the LLM proposes updates but does not mutate the graph directly
|
||||
|
||||
## What v0.6 proved
|
||||
|
||||
- graph-backed questioning works better when every justified next question maps to an explicit unresolved node
|
||||
- broad unknowns can be decomposed deterministically before direct questioning
|
||||
- resolved child evidence can be propagated upward without prematurely resolving parent reasoning
|
||||
- confidence becomes easier to reason about when evidence quality, completeness, and conclusion strength are separated
|
||||
- deterministic cross-branch corroboration can improve support without double-counting repeated evidence
|
||||
|
||||
## Known limitations
|
||||
|
||||
- sibling selection still depends on the existing deterministic scorer and may choose a justified next branch that is not always the intuitively expected one
|
||||
- cross-branch corroboration is limited to direct child branches of the same parent
|
||||
- no multi-hop corroboration exists across unrelated subtrees
|
||||
- reasoning remains bounded to explicitly represented graph structure and user-provided answers
|
||||
|
||||
## Deliberate exclusions
|
||||
|
||||
- no persistence
|
||||
- no autonomous exploration
|
||||
- no probabilistic reasoning
|
||||
- no Bayesian reasoning
|
||||
- no semantic embeddings
|
||||
- no expert mode
|
||||
- no multi-hop corroboration across unrelated subtrees
|
||||
- no heavy graph visualisation
|
||||
|
||||
## Next experimental question
|
||||
|
||||
`Can the engine preserve and reuse successful reasoning structures across separate cases without turning prior experience into unquestioned assumptions?`
|
||||
@@ -1,50 +0,0 @@
|
||||
# v0.6 Selection Influence Experiment
|
||||
|
||||
## Hypothesis
|
||||
|
||||
The initial unknown selected for the revenue-versus-cash scenario may be driven more by graph structure, more by semantic keyword matches, or by both together.
|
||||
|
||||
## Scenario
|
||||
|
||||
`Revenue increased by 18%, but cash in the bank fell over the same period.`
|
||||
|
||||
## Actual selected node
|
||||
|
||||
- Node ID: `nqdzobz`
|
||||
- Label: `Magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts).`
|
||||
- Deterministic investigation strategy: `definition`
|
||||
- Deterministic question: `What evidence would resolve whether magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts). is true?`
|
||||
|
||||
## Structural contribution
|
||||
|
||||
- Downstream dependency count: `0`
|
||||
- Prerequisite position: no unresolved prerequisites; count `0`
|
||||
- Dependency ordering / centrality: no candidate had downstream dependants or dependency depth advantage in the live graph
|
||||
|
||||
## Semantic contribution
|
||||
|
||||
- Objective: false
|
||||
- Actor: false
|
||||
- Criteria: false
|
||||
- Measurement: false
|
||||
- Terminology: false
|
||||
- Constraint: false
|
||||
- Pricing: false
|
||||
- Implementation: false
|
||||
- Optimisation: false
|
||||
- Speculative: false
|
||||
- Contribution list: only `downstream_dependencies` was present, with delta `0`
|
||||
|
||||
## Counterfactual results
|
||||
|
||||
- Live-shaped ordering: `nqdzobz` ranked above `niewza`, but both had score `0`, downstream `0`, and unresolved prerequisites `0`
|
||||
- Links removed: ordering stayed the same, because the live graph already provided no differentiating structure between the two unknowns
|
||||
- Wording neutralised: ordering flipped to the first unknown by neutral label order (`Unknown A` before `Unknown B`), showing the outcome remained tie-break-driven rather than structure-driven
|
||||
|
||||
## Conclusion
|
||||
|
||||
For this scenario, the actual winner was not selected because of graph structure and not selected because of semantic keyword weights. The live diagnostics show a complete tie on score, downstream influence, and prerequisite position, with every semantic match category false for both candidates. The winner was therefore chosen by the final tie-break rule, `label_asc`.
|
||||
|
||||
## Is a scoring change justified?
|
||||
|
||||
Not from this single experiment alone. The result shows a diagnostic gap for this scenario, but this task does not justify a scoring change by itself, and no scoring change is made.
|
||||
+17
-77
@@ -5,18 +5,14 @@
|
||||
|
||||
import { getConfig } from "../lib/config.js";
|
||||
import { getProvider } from "../lib/llm/provider.js";
|
||||
import {
|
||||
buildPrompt,
|
||||
PROMPT_VERSIONS,
|
||||
DEFAULT_PROMPT_VERSION,
|
||||
} from "../lib/reconstruction/prompt.js";
|
||||
import { normaliseAnalysisResponse } from "../lib/reconstruction/compatibility.js";
|
||||
import { buildPrompt, PROMPT_VERSIONS } from "../lib/reconstruction/prompt.js";
|
||||
import {
|
||||
reconstructionV2Schema,
|
||||
reconstructionSchema as reconstructionV1Schema,
|
||||
} from "../lib/reconstruction/schema.js";
|
||||
|
||||
const MAX_SCENARIO_LENGTH = 10000;
|
||||
const DEFAULT_PROMPT_VERSION = "v0.2";
|
||||
|
||||
/**
|
||||
* Analyse a scenario string through the full pipeline.
|
||||
@@ -38,10 +34,7 @@ export async function analyseScenario(scenario, opts = {}) {
|
||||
return buildErrorResponse("Scenario cannot be empty", startTime);
|
||||
}
|
||||
if (trimmed.length > MAX_SCENARIO_LENGTH) {
|
||||
return buildErrorResponse(
|
||||
`Scenario must be under ${MAX_SCENARIO_LENGTH} characters`,
|
||||
startTime,
|
||||
);
|
||||
return buildErrorResponse(`Scenario must be under ${MAX_SCENARIO_LENGTH} characters`, startTime);
|
||||
}
|
||||
|
||||
// ── Configuration check ────────────────────────────
|
||||
@@ -58,24 +51,18 @@ export async function analyseScenario(scenario, opts = {}) {
|
||||
try {
|
||||
promptObj = await buildPrompt(trimmed, promptVersion);
|
||||
} catch (e) {
|
||||
return buildErrorResponse(
|
||||
`Failed to build prompt: ${e.message}`,
|
||||
startTime,
|
||||
);
|
||||
return buildErrorResponse(`Failed to build prompt: ${e.message}`, startTime);
|
||||
}
|
||||
|
||||
// ── Call provider ──────────────────────────────────
|
||||
const provider = getProvider();
|
||||
let rawResponse;
|
||||
try {
|
||||
rawResponse = await provider.generateReconstruction(
|
||||
promptObj.prompt,
|
||||
OLLAMA_MODEL,
|
||||
);
|
||||
rawResponse = await provider.generateReconstruction(promptObj.prompt, OLLAMA_MODEL);
|
||||
} catch (e) {
|
||||
return buildErrorResponse(
|
||||
e.message || "Provider error during analysis",
|
||||
Date.now() - startTime,
|
||||
Date.now() - startTime
|
||||
);
|
||||
}
|
||||
|
||||
@@ -89,37 +76,16 @@ export async function analyseScenario(scenario, opts = {}) {
|
||||
rawResponseStr = String(rawResponse).slice(0, 2000);
|
||||
}
|
||||
|
||||
const compatibility = normaliseAnalysisResponse(rawResponse);
|
||||
const candidateResponse = compatibility.normalised;
|
||||
|
||||
// ── Validate against v0.2 schema (preferred) ──────
|
||||
const resultV2 = tryValidateAgainstSchema(
|
||||
candidateResponse,
|
||||
reconstructionV2Schema,
|
||||
);
|
||||
const resultV2 = tryValidateAgainstSchema(rawResponse, reconstructionV2Schema);
|
||||
if (resultV2.valid) {
|
||||
return buildSuccessResultV2(
|
||||
resultV2.data,
|
||||
OLLAMA_MODEL,
|
||||
duration,
|
||||
promptVersion,
|
||||
compatibility,
|
||||
);
|
||||
return buildSuccessResultV2(resultV2.data, OLLAMA_MODEL, duration, promptVersion);
|
||||
}
|
||||
|
||||
// ── Fallback to v0.1 schema ────────────────────────
|
||||
const resultV1 = tryValidateAgainstSchema(
|
||||
candidateResponse,
|
||||
reconstructionV1Schema,
|
||||
);
|
||||
const resultV1 = tryValidateAgainstSchema(rawResponse, reconstructionV1Schema);
|
||||
if (resultV1.valid) {
|
||||
return buildSuccessResultV1(
|
||||
resultV1.data,
|
||||
OLLAMA_MODEL,
|
||||
duration,
|
||||
promptVersion,
|
||||
compatibility,
|
||||
);
|
||||
return buildSuccessResultV1(resultV1.data, OLLAMA_MODEL, duration, promptVersion);
|
||||
}
|
||||
|
||||
// ── Neither schema matched — partial failure ───────
|
||||
@@ -128,23 +94,17 @@ export async function analyseScenario(scenario, opts = {}) {
|
||||
resultV2.error ?? resultV1.error,
|
||||
OLLAMA_MODEL,
|
||||
duration,
|
||||
promptVersion,
|
||||
compatibility,
|
||||
promptVersion
|
||||
);
|
||||
}
|
||||
|
||||
/** Attempt validation against a Zod schema */
|
||||
function tryValidateAgainstSchema(data, schema) {
|
||||
if (!schema.safeParse) {
|
||||
return {
|
||||
valid: false,
|
||||
error: new Error("Schema does not support safeParse"),
|
||||
};
|
||||
return { valid: false, error: new Error("Schema does not support safeParse") };
|
||||
}
|
||||
const result = schema.safeParse(data);
|
||||
return result.success
|
||||
? { valid: true, data: result.data }
|
||||
: { valid: false, error: result.error };
|
||||
return result.success ? { valid: true, data: result.data } : { valid: false, error: result.error };
|
||||
}
|
||||
|
||||
// ── Result builders ──────────────────────────────────
|
||||
@@ -162,15 +122,7 @@ function buildErrorResponse(message, elapsed, statusCode = 500) {
|
||||
};
|
||||
}
|
||||
|
||||
function buildCompatibilityDiagnostics(compatibility) {
|
||||
return {
|
||||
compatibilityApplied: compatibility.changesApplied.length > 0,
|
||||
compatibilityChanges: compatibility.changesApplied,
|
||||
compatibilityWarnings: compatibility.warnings,
|
||||
};
|
||||
}
|
||||
|
||||
function buildSuccessResultV2(data, model, duration, version, compatibility) {
|
||||
function buildSuccessResultV2(data, model, duration, version) {
|
||||
return {
|
||||
success: true,
|
||||
validationStatus: "valid",
|
||||
@@ -183,11 +135,10 @@ function buildSuccessResultV2(data, model, duration, version, compatibility) {
|
||||
evidence: data.evidence,
|
||||
nextQuestion: data.nextQuestion,
|
||||
errors: undefined,
|
||||
...buildCompatibilityDiagnostics(compatibility),
|
||||
};
|
||||
}
|
||||
|
||||
function buildSuccessResultV1(data, model, duration, version, compatibility) {
|
||||
function buildSuccessResultV1(data, model, duration, version) {
|
||||
return {
|
||||
success: true,
|
||||
validationStatus: "valid",
|
||||
@@ -200,24 +151,14 @@ function buildSuccessResultV1(data, model, duration, version, compatibility) {
|
||||
evidence: undefined,
|
||||
nextQuestion: undefined,
|
||||
errors: undefined,
|
||||
...buildCompatibilityDiagnostics(compatibility),
|
||||
};
|
||||
}
|
||||
|
||||
function buildPartialResult(
|
||||
rawResp,
|
||||
error,
|
||||
model,
|
||||
duration,
|
||||
version,
|
||||
compatibility,
|
||||
) {
|
||||
function buildPartialResult(rawResp, error, model, duration, version) {
|
||||
let errors = [];
|
||||
if (error && typeof error.flatten === "function") {
|
||||
errors = error.flatten().fieldErrors
|
||||
? Object.entries(error.flatten().fieldErrors).flatMap(([k, v]) => [
|
||||
`${k}: ${v.join(", ")}`,
|
||||
])
|
||||
? Object.entries(error.flatten().fieldErrors).flatMap(([k, v]) => [`${k}: ${v.join(", ")}`])
|
||||
: [String(error)];
|
||||
} else if (error) {
|
||||
errors = [String(error).slice(0, 500)];
|
||||
@@ -235,7 +176,6 @@ function buildPartialResult(
|
||||
evidence: undefined,
|
||||
nextQuestion: undefined,
|
||||
errors,
|
||||
...buildCompatibilityDiagnostics(compatibility),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,305 +0,0 @@
|
||||
/**
|
||||
* Deterministic situation graph builder — builds initial graph from scenario text.
|
||||
* Takes v0.2/v0.3 analysis output (from analyseScenario) and constructs a SituationGraph.
|
||||
*/
|
||||
|
||||
import {
|
||||
situationNodeSchema,
|
||||
situationEdgeSchema,
|
||||
makeNodeId,
|
||||
} from "./schema.js";
|
||||
|
||||
/**
|
||||
* Build an initial situation graph from a v0.3 reconstruction result.
|
||||
* @param {{ reconstruction: object, evidence: object[] | undefined }} analysisData
|
||||
* @returns {{ nodes: import("./schema.js").SituationNode[], edges: import("./schema.js").SituationEdge[] }}
|
||||
*/
|
||||
export function buildInitialGraph(analysisData) {
|
||||
const { reconstruction, evidence = [] } = analysisData;
|
||||
|
||||
if (!reconstruction || !reconstruction.summary) {
|
||||
return { nodes: [], edges: [] };
|
||||
}
|
||||
|
||||
const nodeMap = new Map(); // label -> node
|
||||
|
||||
// ── Helper: register or get a node by label ────────────
|
||||
|
||||
function ensureNode(
|
||||
label,
|
||||
kind,
|
||||
status,
|
||||
description,
|
||||
value,
|
||||
unit,
|
||||
confidence,
|
||||
) {
|
||||
if (nodeMap.has(label)) return nodeMap.get(label);
|
||||
|
||||
const id = makeNodeId(label);
|
||||
const node = situationNodeSchema.parse({
|
||||
id,
|
||||
label,
|
||||
description: description ?? label,
|
||||
kind,
|
||||
status,
|
||||
confidence,
|
||||
value: value ?? null,
|
||||
unit: unit ?? null,
|
||||
evidenceIds: [],
|
||||
dependsOn: [],
|
||||
affects: [],
|
||||
parentId: null,
|
||||
childIds: [],
|
||||
});
|
||||
nodeMap.set(label, node);
|
||||
return node;
|
||||
}
|
||||
|
||||
// ── Evidence lookup ────────────────────────────────────
|
||||
|
||||
const evidenceMap = new Map();
|
||||
for (const ev of evidence) {
|
||||
if (ev.id) evidenceMap.set(ev.id, ev);
|
||||
}
|
||||
|
||||
function addEvidenceToNode(nodeId, evidenceId) {
|
||||
const node = Object.values(nodeMap).find((n) => n.id === nodeId);
|
||||
if (node && !node.evidenceIds.includes(evidenceId)) {
|
||||
node.evidenceIds.push(evidenceId);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Extract observed states as nodes ────────────────────
|
||||
|
||||
const summaryNode = ensureNode(
|
||||
reconstruction.summary || "Situation Summary",
|
||||
"state",
|
||||
"provisional",
|
||||
"Summary of the situation from the scenario text",
|
||||
null,
|
||||
null,
|
||||
"medium",
|
||||
);
|
||||
|
||||
// Collect all observable quantities as metric nodes
|
||||
const metrics = new Map();
|
||||
|
||||
if (reconstruction.observedStates) {
|
||||
for (const obs of reconstruction.observedStates) {
|
||||
const node = ensureNode(
|
||||
obs.description || obs.label,
|
||||
"observation",
|
||||
"supported",
|
||||
obs.description || obs.label,
|
||||
null,
|
||||
null,
|
||||
obs.confidence || "medium",
|
||||
);
|
||||
|
||||
if (obs.id) node.evidenceIds.push(obs.id);
|
||||
}
|
||||
}
|
||||
|
||||
// Actors as states/nodes
|
||||
if (reconstruction.actors) {
|
||||
for (const actor of reconstruction.actors) {
|
||||
ensureNode(
|
||||
actor.description || actor.label,
|
||||
"observation",
|
||||
"supported",
|
||||
actor.description || actor.label,
|
||||
null,
|
||||
null,
|
||||
actor.confidence || "medium",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (reconstruction.systemsOrObjects) {
|
||||
for (const sys of reconstruction.systemsOrObjects) {
|
||||
ensureNode(
|
||||
sys.description || sys.label,
|
||||
"metric",
|
||||
"known",
|
||||
sys.description || sys.label,
|
||||
null,
|
||||
null,
|
||||
sys.confidence || "medium",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Differences as relationship nodes
|
||||
if (reconstruction.differences) {
|
||||
for (const diff of reconstruction.differences) {
|
||||
const node = ensureNode(
|
||||
diff.description || "Difference",
|
||||
"relationship",
|
||||
"supported",
|
||||
diff.description || "Difference",
|
||||
null,
|
||||
null,
|
||||
diff.confidence || "medium",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Contradictions as nodes
|
||||
if (reconstruction.contradictions) {
|
||||
for (const c of reconstruction.contradictions) {
|
||||
const node = ensureNode(
|
||||
c.description || c.label,
|
||||
"relationship",
|
||||
"supported",
|
||||
c.description || c.label,
|
||||
null,
|
||||
null,
|
||||
c.confidence || "medium",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Important unknowns as unknown nodes
|
||||
const unknownNodes = [];
|
||||
if (reconstruction.importantUnknowns) {
|
||||
for (const unk of reconstruction.importantUnknowns) {
|
||||
const node = ensureNode(
|
||||
unk.description || unk.label,
|
||||
"unknown",
|
||||
"unknown",
|
||||
unk.description || "Unknown factor in the situation",
|
||||
null,
|
||||
null,
|
||||
unk.confidence || "low",
|
||||
);
|
||||
unknownNodes.push(node);
|
||||
}
|
||||
}
|
||||
|
||||
// Plausible interpretations
|
||||
if (reconstruction.plausibleInterpretations) {
|
||||
for (const interp of reconstruction.plausibleInterpretations) {
|
||||
ensureNode(
|
||||
interp.description || interp.label,
|
||||
"assumption",
|
||||
"provisional",
|
||||
interp.description || "Plausible interpretation",
|
||||
null,
|
||||
null,
|
||||
interp.confidence || "low",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Known transitions
|
||||
if (reconstruction.knownTransitions) {
|
||||
for (const trans of reconstruction.knownTransitions) {
|
||||
ensureNode(
|
||||
`${trans.entity}: ${trans.previousState} → ${trans.currentState}`,
|
||||
"transition",
|
||||
trans.explanationStatus === "confirmed" ? "known" : "provisional",
|
||||
trans.description ||
|
||||
`Transition: ${trans.entity} from ${trans.previousState} to ${trans.currentState}`,
|
||||
null,
|
||||
null,
|
||||
trans.confidence || "medium",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Build edges between nodes ────────────────────────
|
||||
|
||||
const nodeArr = Array.from(nodeMap.values());
|
||||
const edges = [];
|
||||
|
||||
// Link actors → observed states as measures relationships
|
||||
let actorNodes = [];
|
||||
let metricNodes = [];
|
||||
let unknownNodeIds = [];
|
||||
|
||||
for (const n of nodeArr) {
|
||||
if (n.kind === "observation" && n.status === "supported") {
|
||||
// These are observations — link to summary
|
||||
edges.push(
|
||||
situationEdgeSchema.parse({
|
||||
id: `e-sum-${n.id}`,
|
||||
fromNodeId: n.id,
|
||||
toNodeId: summaryNode.id,
|
||||
relationship: "supports",
|
||||
confidence: n.confidence || "medium",
|
||||
description: `${n.label} supports the summary`,
|
||||
}),
|
||||
);
|
||||
}
|
||||
if (n.kind === "unknown") {
|
||||
unknownNodeIds.push(n.id);
|
||||
edges.push(
|
||||
situationEdgeSchema.parse({
|
||||
id: `e-unk-${n.id}`,
|
||||
fromNodeId: n.id,
|
||||
toNodeId: summaryNode.id,
|
||||
relationship: "depends_on",
|
||||
confidence: n.confidence || "low",
|
||||
description: `${n.label} is an unresolved factor for this situation`,
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return { nodes: nodeArr, edges };
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a minimal starting graph for any scenario.
|
||||
* Used when analysis has no reconstruction data (e.g., error state).
|
||||
*/
|
||||
export function buildMinimalGraph(scenario) {
|
||||
const shortLabel = scenario.slice(0, 80);
|
||||
|
||||
return {
|
||||
nodes: [
|
||||
situationNodeSchema.parse({
|
||||
id: "n0",
|
||||
label: shortLabel,
|
||||
description: `Initial situation from: "${scenario.slice(0, 200)}"`,
|
||||
kind: "state",
|
||||
status: "provisional",
|
||||
confidence: "low",
|
||||
value: null,
|
||||
unit: null,
|
||||
evidenceIds: [],
|
||||
dependsOn: [],
|
||||
affects: [],
|
||||
parentId: null,
|
||||
childIds: [],
|
||||
}),
|
||||
],
|
||||
edges: [],
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Convert graph nodes/edges to a human-readable summary for display.
|
||||
*/
|
||||
export function describeGraph(graph) {
|
||||
const parts = [];
|
||||
|
||||
// Count by kind
|
||||
const byKind = {};
|
||||
for (const n of graph.nodes) {
|
||||
byKind[n.kind] = (byKind[n.kind] || 0) + 1;
|
||||
}
|
||||
|
||||
parts.push(
|
||||
`Nodes: ${Object.entries(byKind)
|
||||
.map(([k, v]) => `${v} ${k}`)
|
||||
.join(", ")}`,
|
||||
);
|
||||
parts.push(`Edges: ${graph.edges.length} total`);
|
||||
parts.push(
|
||||
`Unknowns: ${graph.nodes.filter((n) => n.status === "unknown").length} unresolved`,
|
||||
);
|
||||
|
||||
return parts.join(" | ");
|
||||
}
|
||||
@@ -1,660 +0,0 @@
|
||||
/**
|
||||
* Situation Graph Case Orchestrator — manages the lifecycle of a case.
|
||||
* startCase builds initial graph from analysis; updateCase applies answers.
|
||||
*/
|
||||
|
||||
import { analyseScenario } from "../analysis.js";
|
||||
import { assertConfig } from "../config.js";
|
||||
import { getProvider } from "../llm/provider.js";
|
||||
import {
|
||||
makeGraph,
|
||||
startCaseRequestSchema,
|
||||
situationGraphSchema,
|
||||
updateCaseRequestSchema,
|
||||
} from "./schema.js";
|
||||
import { buildInitialGraph, describeGraph } from "./builder.js";
|
||||
import { applyValidatedProposal } from "./apply-proposal.js";
|
||||
import { buildGraphUpdatePrompt } from "./prompt-builder.js";
|
||||
import {
|
||||
buildReasoningState,
|
||||
formulateTieResolutionQuestion,
|
||||
} from "./question-formulator.js";
|
||||
import { parseGraphUpdateProposal } from "./update-proposal.js";
|
||||
import {
|
||||
explainUnknownSelection,
|
||||
selectActiveUnknownCandidate,
|
||||
validateGraphReferences,
|
||||
} from "./utils.js";
|
||||
|
||||
function toValidationErrors(error) {
|
||||
return (
|
||||
error?.errors?.map((issue) => ({
|
||||
path: issue.path,
|
||||
message: issue.message,
|
||||
code: issue.code,
|
||||
})) ?? [{ message: "Validation failed" }]
|
||||
);
|
||||
}
|
||||
|
||||
function buildDiagnostics({
|
||||
analysis,
|
||||
graph,
|
||||
graphReferenceValidation,
|
||||
unknownSelectionExplanation,
|
||||
}) {
|
||||
return {
|
||||
promptVersion: analysis?.promptVersion ?? null,
|
||||
modelName: analysis?.modelName ?? null,
|
||||
responseDurationMs: analysis?.responseDurationMs ?? null,
|
||||
validationStatus: analysis?.validationStatus ?? "invalid",
|
||||
nodeCount: graph?.nodes?.length ?? 0,
|
||||
edgeCount: graph?.edges?.length ?? 0,
|
||||
graphReferenceValidation,
|
||||
compatibilityApplied: analysis?.compatibilityApplied ?? false,
|
||||
compatibilityChanges: analysis?.compatibilityChanges ?? [],
|
||||
compatibilityWarnings: analysis?.compatibilityWarnings ?? [],
|
||||
unknownSelectionExplanation: unknownSelectionExplanation ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
function buildUnknownSelectionDiagnostics(
|
||||
graph,
|
||||
resolvedNodeIds = [],
|
||||
selectedQuestion = null,
|
||||
) {
|
||||
const explanation = explainUnknownSelection(graph, resolvedNodeIds);
|
||||
if (explanation.status === "ambiguous") {
|
||||
return {
|
||||
...explanation,
|
||||
tieResolutionQuestion:
|
||||
selectedQuestion?.selectionStatus === "ambiguous"
|
||||
? selectedQuestion.question
|
||||
: formulateTieResolutionQuestion({ graph }).question,
|
||||
alphabeticalUsedAsReasoning: false,
|
||||
};
|
||||
}
|
||||
|
||||
return explanation;
|
||||
}
|
||||
|
||||
function buildUpdateDiagnostics({
|
||||
promptVersion,
|
||||
modelName,
|
||||
responseDurationMs,
|
||||
normalisationsApplied,
|
||||
graph,
|
||||
graphReferenceValidation,
|
||||
selectedQuestion,
|
||||
unknownSelectionExplanation,
|
||||
previousReasoningState,
|
||||
reasoningState,
|
||||
resolvedReasoningNodeIds,
|
||||
emergentReasoningNodeCreated,
|
||||
emergentReasoningNodeId,
|
||||
emergentReasoningNodeReason,
|
||||
atomicityAssessment,
|
||||
atomicityDecisionReason,
|
||||
decompositionDepth,
|
||||
decompositionAttempted,
|
||||
decompositionAccepted,
|
||||
decompositionStoppedReason,
|
||||
proposedChildCount,
|
||||
acceptedChildCount,
|
||||
rejectedChildren,
|
||||
selectedChildNodeId,
|
||||
childQualitySummary,
|
||||
propagationPerformed,
|
||||
resolvedChildNodeId,
|
||||
parentNodeId,
|
||||
parentStatusBefore,
|
||||
parentStatusAfter,
|
||||
parentConfidenceBefore,
|
||||
parentConfidenceAfter,
|
||||
evidenceConfidenceBefore,
|
||||
evidenceConfidenceAfter,
|
||||
completenessBefore,
|
||||
completenessAfter,
|
||||
conclusionConfidenceBefore,
|
||||
conclusionConfidenceAfter,
|
||||
resolvedDirectChildren,
|
||||
unresolvedDirectChildren,
|
||||
contradictoryDirectChildren,
|
||||
corroboratingBranchCount,
|
||||
conflictingBranchCount,
|
||||
duplicateEvidenceCount,
|
||||
independentBranchCount,
|
||||
interactionSummary,
|
||||
confidenceCapReason,
|
||||
ancestorPropagationStoppedReason,
|
||||
affectedAncestorIds,
|
||||
nextSelectedSibling,
|
||||
parentResolved,
|
||||
decompositionPerformed,
|
||||
childUnknownCount,
|
||||
childNodeIds,
|
||||
atomicityReason,
|
||||
}) {
|
||||
return {
|
||||
promptVersion: promptVersion ?? "v0.4",
|
||||
modelName: modelName ?? null,
|
||||
responseDurationMs: responseDurationMs ?? null,
|
||||
validationStatus: "valid",
|
||||
nodeCount: graph?.nodes?.length ?? 0,
|
||||
edgeCount: graph?.edges?.length ?? 0,
|
||||
graphReferenceValidation: graphReferenceValidation ?? {
|
||||
valid: true,
|
||||
errors: [],
|
||||
},
|
||||
normalisationsApplied: normalisationsApplied ?? [],
|
||||
investigationStrategy:
|
||||
selectedQuestion?.investigationStrategy ??
|
||||
selectedQuestion?.strategy ??
|
||||
null,
|
||||
previousComparabilityStatus:
|
||||
previousReasoningState?.comparabilityStatus ?? null,
|
||||
comparabilityStatus: reasoningState?.comparabilityStatus ?? null,
|
||||
relationshipStatus: reasoningState?.relationshipStatus ?? null,
|
||||
relationshipAssessed: reasoningState?.relationshipAssessed ?? null,
|
||||
reasoningStagesBefore: previousReasoningState?.reasoningStages ?? [],
|
||||
reasoningStagesAfter: reasoningState?.reasoningStages ?? [],
|
||||
resolvedReasoningNodeIds: resolvedReasoningNodeIds ?? [],
|
||||
emergentReasoningNodeCreated: emergentReasoningNodeCreated ?? false,
|
||||
emergentReasoningNodeId: emergentReasoningNodeId ?? null,
|
||||
emergentReasoningNodeReason: emergentReasoningNodeReason ?? null,
|
||||
atomicityAssessment: atomicityAssessment ?? null,
|
||||
atomicityDecisionReason: atomicityDecisionReason ?? null,
|
||||
decompositionDepth: decompositionDepth ?? 0,
|
||||
decompositionAttempted: decompositionAttempted ?? false,
|
||||
decompositionAccepted: decompositionAccepted ?? false,
|
||||
decompositionStoppedReason: decompositionStoppedReason ?? null,
|
||||
proposedChildCount: proposedChildCount ?? 0,
|
||||
acceptedChildCount: acceptedChildCount ?? 0,
|
||||
rejectedChildren: rejectedChildren ?? [],
|
||||
selectedChildNodeId: selectedChildNodeId ?? null,
|
||||
childQualitySummary: childQualitySummary ?? [],
|
||||
propagationPerformed: propagationPerformed ?? false,
|
||||
resolvedChildNodeId: resolvedChildNodeId ?? null,
|
||||
parentNodeId: parentNodeId ?? null,
|
||||
parentStatusBefore: parentStatusBefore ?? null,
|
||||
parentStatusAfter: parentStatusAfter ?? null,
|
||||
parentConfidenceBefore: parentConfidenceBefore ?? null,
|
||||
parentConfidenceAfter: parentConfidenceAfter ?? null,
|
||||
evidenceConfidenceBefore: evidenceConfidenceBefore ?? null,
|
||||
evidenceConfidenceAfter: evidenceConfidenceAfter ?? null,
|
||||
completenessBefore: completenessBefore ?? null,
|
||||
completenessAfter: completenessAfter ?? null,
|
||||
conclusionConfidenceBefore: conclusionConfidenceBefore ?? null,
|
||||
conclusionConfidenceAfter: conclusionConfidenceAfter ?? null,
|
||||
resolvedDirectChildren: resolvedDirectChildren ?? 0,
|
||||
unresolvedDirectChildren: unresolvedDirectChildren ?? 0,
|
||||
contradictoryDirectChildren: contradictoryDirectChildren ?? 0,
|
||||
corroboratingBranchCount: corroboratingBranchCount ?? 0,
|
||||
conflictingBranchCount: conflictingBranchCount ?? 0,
|
||||
duplicateEvidenceCount: duplicateEvidenceCount ?? 0,
|
||||
independentBranchCount: independentBranchCount ?? 0,
|
||||
interactionSummary: interactionSummary ?? null,
|
||||
confidenceCapReason: confidenceCapReason ?? null,
|
||||
ancestorPropagationStoppedReason: ancestorPropagationStoppedReason ?? null,
|
||||
affectedAncestorIds: affectedAncestorIds ?? [],
|
||||
nextSelectedSibling: nextSelectedSibling ?? null,
|
||||
parentResolved: parentResolved ?? false,
|
||||
decompositionPerformed: decompositionPerformed ?? false,
|
||||
childUnknownCount: childUnknownCount ?? 0,
|
||||
childNodeIds: childNodeIds ?? [],
|
||||
atomicityReason: atomicityReason ?? null,
|
||||
unknownSelectionExplanation: unknownSelectionExplanation ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
export async function startCase(body) {
|
||||
const parsedRequest = startCaseRequestSchema.safeParse(body);
|
||||
|
||||
if (!parsedRequest.success) {
|
||||
return {
|
||||
success: false,
|
||||
error: "Invalid start-case request",
|
||||
validationErrors: toValidationErrors(parsedRequest.error),
|
||||
statusCode: 400,
|
||||
};
|
||||
}
|
||||
|
||||
const { scenario, promptVersion } = parsedRequest.data;
|
||||
const analysis = await analyseScenario(scenario, { promptVersion });
|
||||
|
||||
if (!analysis.success) {
|
||||
return {
|
||||
success: false,
|
||||
error: analysis.error ?? "Scenario analysis failed",
|
||||
diagnostics: buildDiagnostics({
|
||||
analysis,
|
||||
graph: null,
|
||||
graphReferenceValidation: null,
|
||||
}),
|
||||
analysisErrors: analysis.errors ?? undefined,
|
||||
rawResponse: analysis.rawResponse ?? undefined,
|
||||
statusCode: Number(analysis.statusCode) || 502,
|
||||
};
|
||||
}
|
||||
|
||||
const initialGraph = buildInitialGraph({
|
||||
reconstruction: analysis.reconstruction,
|
||||
evidence: analysis.evidence,
|
||||
});
|
||||
|
||||
const currentSummary = describeGraph(initialGraph);
|
||||
const deterministicSelection = selectActiveUnknownCandidate(
|
||||
{
|
||||
...initialGraph,
|
||||
resolvedNodeIds: [],
|
||||
},
|
||||
[],
|
||||
);
|
||||
const activeUnknownNodeId =
|
||||
deterministicSelection?.status === "selected"
|
||||
? deterministicSelection.nodeId
|
||||
: null;
|
||||
|
||||
const situationGraph = makeGraph({
|
||||
centralStatement: scenario,
|
||||
nodes: initialGraph.nodes,
|
||||
edges: initialGraph.edges,
|
||||
activeUnknownNodeId,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary,
|
||||
reasoningState: buildReasoningState({
|
||||
centralStatement: scenario,
|
||||
nodes: initialGraph.nodes,
|
||||
edges: initialGraph.edges,
|
||||
resolvedNodeIds: [],
|
||||
}),
|
||||
});
|
||||
|
||||
situationGraphSchema.parse(situationGraph);
|
||||
|
||||
const graphReferenceValidation = validateGraphReferences(situationGraph);
|
||||
const selectedQuestion =
|
||||
deterministicSelection?.status === "ambiguous"
|
||||
? {
|
||||
id: "q_tie_resolution",
|
||||
...formulateTieResolutionQuestion({ graph: situationGraph }),
|
||||
tiedCandidateIds: deterministicSelection.tiedCandidateIds,
|
||||
}
|
||||
: (analysis.nextQuestion ?? null);
|
||||
const unknownSelectionExplanation = buildUnknownSelectionDiagnostics(
|
||||
situationGraph,
|
||||
[],
|
||||
selectedQuestion,
|
||||
);
|
||||
if (!graphReferenceValidation.valid) {
|
||||
return {
|
||||
success: false,
|
||||
error: "Situation graph reference validation failed",
|
||||
diagnostics: buildDiagnostics({
|
||||
analysis,
|
||||
graph: situationGraph,
|
||||
graphReferenceValidation,
|
||||
unknownSelectionExplanation,
|
||||
}),
|
||||
validationErrors: graphReferenceValidation.errors,
|
||||
statusCode: 500,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
situationGraph,
|
||||
selectedQuestion,
|
||||
diagnostics: buildDiagnostics({
|
||||
analysis,
|
||||
graph: situationGraph,
|
||||
graphReferenceValidation,
|
||||
unknownSelectionExplanation,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
export async function updateCase() {
|
||||
return updateCaseWithDependencies(...arguments);
|
||||
}
|
||||
|
||||
function sanitiseErrorMessage(error, fallbackMessage) {
|
||||
if (typeof error?.message === "string" && error.message.trim().length > 0) {
|
||||
return error.message;
|
||||
}
|
||||
|
||||
return fallbackMessage;
|
||||
}
|
||||
|
||||
async function updateCaseWithDependencies(body, dependencies = {}) {
|
||||
const parsedRequest = updateCaseRequestSchema.safeParse(body);
|
||||
|
||||
if (!parsedRequest.success) {
|
||||
return {
|
||||
success: false,
|
||||
stage: "request_validation",
|
||||
error: "Invalid update-case request",
|
||||
validationErrors: toValidationErrors(parsedRequest.error),
|
||||
statusCode: 400,
|
||||
};
|
||||
}
|
||||
|
||||
const { situationGraph, previousQuestion, answer, promptVersion } =
|
||||
parsedRequest.data;
|
||||
|
||||
const graphSchemaValidation = situationGraphSchema.safeParse(situationGraph);
|
||||
const graphReferenceValidation = validateGraphReferences(situationGraph);
|
||||
|
||||
if (!graphSchemaValidation.success || !graphReferenceValidation.valid) {
|
||||
return {
|
||||
success: false,
|
||||
stage: "graph_validation",
|
||||
error: "Invalid situation graph",
|
||||
graphValidationErrors: [
|
||||
...(!graphSchemaValidation.success
|
||||
? toValidationErrors(graphSchemaValidation.error)
|
||||
: []),
|
||||
...(!graphReferenceValidation.valid
|
||||
? graphReferenceValidation.errors
|
||||
: []),
|
||||
],
|
||||
statusCode: 400,
|
||||
};
|
||||
}
|
||||
|
||||
const buildPrompt =
|
||||
dependencies.buildGraphUpdatePrompt ?? buildGraphUpdatePrompt;
|
||||
const parseProposal =
|
||||
dependencies.parseGraphUpdateProposal ?? parseGraphUpdateProposal;
|
||||
const applyProposalUpdate =
|
||||
dependencies.applyValidatedProposal ?? applyValidatedProposal;
|
||||
const shouldApplyProposal = dependencies.applyProposal === true;
|
||||
|
||||
let modelName = null;
|
||||
let rawResponse;
|
||||
const startedAt = Date.now();
|
||||
|
||||
try {
|
||||
const config = dependencies.config ?? assertConfig();
|
||||
modelName = config.OLLAMA_MODEL;
|
||||
|
||||
const prompt = buildPrompt({
|
||||
situationGraph,
|
||||
previousQuestion,
|
||||
answer,
|
||||
promptVersion,
|
||||
});
|
||||
|
||||
const provider = dependencies.provider ?? getProvider();
|
||||
rawResponse = await provider.generateReconstruction(prompt, modelName);
|
||||
} catch (error) {
|
||||
return {
|
||||
success: false,
|
||||
stage: "provider",
|
||||
error: "Graph update proposal generation failed",
|
||||
providerErrors: [
|
||||
sanitiseErrorMessage(
|
||||
error,
|
||||
"Provider failed to generate graph update proposal",
|
||||
),
|
||||
],
|
||||
diagnostics: {
|
||||
promptVersion: promptVersion ?? null,
|
||||
modelName,
|
||||
responseDurationMs: Date.now() - startedAt,
|
||||
normalisationsApplied: [],
|
||||
},
|
||||
statusCode: 502,
|
||||
};
|
||||
}
|
||||
|
||||
const parsedProposal = parseProposal(rawResponse);
|
||||
const responseDurationMs = Date.now() - startedAt;
|
||||
|
||||
if (!parsedProposal.success) {
|
||||
return {
|
||||
success: false,
|
||||
stage: "proposal_validation",
|
||||
error: "Invalid graph update proposal",
|
||||
proposalErrors: parsedProposal.errors,
|
||||
diagnostics: {
|
||||
promptVersion: promptVersion ?? null,
|
||||
modelName,
|
||||
responseDurationMs,
|
||||
normalisationsApplied: parsedProposal.normalisationsApplied,
|
||||
},
|
||||
statusCode: 502,
|
||||
};
|
||||
}
|
||||
|
||||
if (shouldApplyProposal) {
|
||||
const applicationResult = applyProposalUpdate({
|
||||
situationGraph,
|
||||
proposal: parsedProposal.proposal,
|
||||
previousQuestion,
|
||||
answer,
|
||||
});
|
||||
|
||||
if (!applicationResult.success) {
|
||||
return {
|
||||
success: false,
|
||||
stage: applicationResult.stage,
|
||||
errors: applicationResult.errors,
|
||||
diagnostics: {
|
||||
...buildUpdateDiagnostics({
|
||||
promptVersion,
|
||||
modelName,
|
||||
responseDurationMs,
|
||||
normalisationsApplied: parsedProposal.normalisationsApplied,
|
||||
graph: situationGraph,
|
||||
graphReferenceValidation: graphReferenceValidation,
|
||||
selectedQuestion: null,
|
||||
previousReasoningState: buildReasoningState(situationGraph),
|
||||
reasoningState: buildReasoningState(situationGraph),
|
||||
resolvedReasoningNodeIds: [],
|
||||
emergentReasoningNodeCreated: false,
|
||||
emergentReasoningNodeId: null,
|
||||
emergentReasoningNodeReason: null,
|
||||
atomicityAssessment: null,
|
||||
atomicityDecisionReason: null,
|
||||
decompositionDepth: 0,
|
||||
decompositionAttempted: false,
|
||||
decompositionAccepted: false,
|
||||
decompositionStoppedReason: null,
|
||||
proposedChildCount: 0,
|
||||
acceptedChildCount: 0,
|
||||
rejectedChildren: [],
|
||||
selectedChildNodeId: null,
|
||||
childQualitySummary: [],
|
||||
propagationPerformed: false,
|
||||
resolvedChildNodeId: null,
|
||||
parentNodeId: null,
|
||||
parentStatusBefore: null,
|
||||
parentStatusAfter: null,
|
||||
parentConfidenceBefore: null,
|
||||
parentConfidenceAfter: null,
|
||||
evidenceConfidenceBefore: null,
|
||||
evidenceConfidenceAfter: null,
|
||||
completenessBefore: null,
|
||||
completenessAfter: null,
|
||||
conclusionConfidenceBefore: null,
|
||||
conclusionConfidenceAfter: null,
|
||||
resolvedDirectChildren: 0,
|
||||
unresolvedDirectChildren: 0,
|
||||
contradictoryDirectChildren: 0,
|
||||
corroboratingBranchCount: 0,
|
||||
conflictingBranchCount: 0,
|
||||
duplicateEvidenceCount: 0,
|
||||
independentBranchCount: 0,
|
||||
interactionSummary: null,
|
||||
confidenceCapReason: null,
|
||||
ancestorPropagationStoppedReason: null,
|
||||
affectedAncestorIds: [],
|
||||
nextSelectedSibling: null,
|
||||
parentResolved: false,
|
||||
decompositionPerformed: false,
|
||||
childUnknownCount: 0,
|
||||
childNodeIds: [],
|
||||
atomicityReason: null,
|
||||
unknownSelectionExplanation: explainUnknownSelection(
|
||||
situationGraph,
|
||||
situationGraph.resolvedNodeIds || [],
|
||||
),
|
||||
}),
|
||||
},
|
||||
statusCode:
|
||||
applicationResult.stage === "application" ||
|
||||
applicationResult.stage === "result_validation"
|
||||
? 500
|
||||
: 400,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
stage: "update_applied",
|
||||
updatedSituationGraph: applicationResult.updatedSituationGraph,
|
||||
proposal: applicationResult.graphUpdate,
|
||||
selectedQuestion: applicationResult.selectedQuestion,
|
||||
affectedNodeIds: applicationResult.affectedNodeIds,
|
||||
resolvedUnknownNodeIds: applicationResult.resolvedUnknownNodeIds,
|
||||
previousActiveUnknownNodeId:
|
||||
applicationResult.previousActiveUnknownNodeId,
|
||||
newActiveUnknownNodeId: applicationResult.newActiveUnknownNodeId,
|
||||
changesApplied: applicationResult.changesApplied,
|
||||
diagnostics: buildUpdateDiagnostics({
|
||||
promptVersion,
|
||||
modelName,
|
||||
responseDurationMs,
|
||||
normalisationsApplied: parsedProposal.normalisationsApplied,
|
||||
graph: applicationResult.updatedSituationGraph,
|
||||
graphReferenceValidation: applicationResult.graphReferenceValidation,
|
||||
selectedQuestion: applicationResult.selectedQuestion,
|
||||
previousReasoningState: applicationResult.previousReasoningState,
|
||||
reasoningState: applicationResult.reasoningState,
|
||||
resolvedReasoningNodeIds: applicationResult.resolvedReasoningNodeIds,
|
||||
emergentReasoningNodeCreated:
|
||||
applicationResult.emergentReasoningNodeCreated,
|
||||
emergentReasoningNodeId: applicationResult.emergentReasoningNodeId,
|
||||
emergentReasoningNodeReason:
|
||||
applicationResult.emergentReasoningNodeReason,
|
||||
atomicityAssessment: applicationResult.atomicityAssessment,
|
||||
atomicityDecisionReason: applicationResult.atomicityDecisionReason,
|
||||
decompositionDepth: applicationResult.decompositionDepth,
|
||||
decompositionAttempted: applicationResult.decompositionAttempted,
|
||||
decompositionAccepted: applicationResult.decompositionAccepted,
|
||||
decompositionStoppedReason:
|
||||
applicationResult.decompositionStoppedReason,
|
||||
proposedChildCount: applicationResult.proposedChildCount,
|
||||
acceptedChildCount: applicationResult.acceptedChildCount,
|
||||
rejectedChildren: applicationResult.rejectedChildren,
|
||||
selectedChildNodeId: applicationResult.selectedChildNodeId,
|
||||
childQualitySummary: applicationResult.childQualitySummary,
|
||||
propagationPerformed: applicationResult.propagationPerformed,
|
||||
resolvedChildNodeId: applicationResult.resolvedChildNodeId,
|
||||
parentNodeId: applicationResult.parentNodeId,
|
||||
parentStatusBefore: applicationResult.parentStatusBefore,
|
||||
parentStatusAfter: applicationResult.parentStatusAfter,
|
||||
parentConfidenceBefore: applicationResult.parentConfidenceBefore,
|
||||
parentConfidenceAfter: applicationResult.parentConfidenceAfter,
|
||||
evidenceConfidenceBefore: applicationResult.evidenceConfidenceBefore,
|
||||
evidenceConfidenceAfter: applicationResult.evidenceConfidenceAfter,
|
||||
completenessBefore: applicationResult.completenessBefore,
|
||||
completenessAfter: applicationResult.completenessAfter,
|
||||
conclusionConfidenceBefore:
|
||||
applicationResult.conclusionConfidenceBefore,
|
||||
conclusionConfidenceAfter: applicationResult.conclusionConfidenceAfter,
|
||||
resolvedDirectChildren: applicationResult.resolvedDirectChildren,
|
||||
unresolvedDirectChildren: applicationResult.unresolvedDirectChildren,
|
||||
contradictoryDirectChildren:
|
||||
applicationResult.contradictoryDirectChildren,
|
||||
corroboratingBranchCount: applicationResult.corroboratingBranchCount,
|
||||
conflictingBranchCount: applicationResult.conflictingBranchCount,
|
||||
duplicateEvidenceCount: applicationResult.duplicateEvidenceCount,
|
||||
independentBranchCount: applicationResult.independentBranchCount,
|
||||
interactionSummary: applicationResult.interactionSummary,
|
||||
confidenceCapReason: applicationResult.confidenceCapReason,
|
||||
ancestorPropagationStoppedReason:
|
||||
applicationResult.ancestorPropagationStoppedReason,
|
||||
affectedAncestorIds: applicationResult.affectedAncestorIds,
|
||||
nextSelectedSibling: applicationResult.nextSelectedSibling,
|
||||
parentResolved: applicationResult.parentResolved,
|
||||
decompositionPerformed: applicationResult.decompositionPerformed,
|
||||
childUnknownCount: applicationResult.childUnknownCount,
|
||||
childNodeIds: applicationResult.childNodeIds,
|
||||
atomicityReason: applicationResult.atomicityReason,
|
||||
unknownSelectionExplanation: buildUnknownSelectionDiagnostics(
|
||||
applicationResult.updatedSituationGraph,
|
||||
applicationResult.updatedSituationGraph.resolvedNodeIds || [],
|
||||
applicationResult.selectedQuestion,
|
||||
),
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
stage: "proposal_ready",
|
||||
proposal: parsedProposal.proposal,
|
||||
diagnostics: buildUpdateDiagnostics({
|
||||
promptVersion,
|
||||
modelName,
|
||||
responseDurationMs,
|
||||
normalisationsApplied: parsedProposal.normalisationsApplied,
|
||||
graph: situationGraph,
|
||||
graphReferenceValidation,
|
||||
selectedQuestion: null,
|
||||
previousReasoningState: buildReasoningState(situationGraph),
|
||||
reasoningState: buildReasoningState(situationGraph),
|
||||
resolvedReasoningNodeIds: [],
|
||||
emergentReasoningNodeCreated: false,
|
||||
emergentReasoningNodeId: null,
|
||||
emergentReasoningNodeReason: null,
|
||||
atomicityAssessment: null,
|
||||
atomicityDecisionReason: null,
|
||||
decompositionDepth: 0,
|
||||
decompositionAttempted: false,
|
||||
decompositionAccepted: false,
|
||||
decompositionStoppedReason: null,
|
||||
proposedChildCount: 0,
|
||||
acceptedChildCount: 0,
|
||||
rejectedChildren: [],
|
||||
selectedChildNodeId: null,
|
||||
childQualitySummary: [],
|
||||
propagationPerformed: false,
|
||||
resolvedChildNodeId: null,
|
||||
parentNodeId: null,
|
||||
parentStatusBefore: null,
|
||||
parentStatusAfter: null,
|
||||
parentConfidenceBefore: null,
|
||||
parentConfidenceAfter: null,
|
||||
evidenceConfidenceBefore: null,
|
||||
evidenceConfidenceAfter: null,
|
||||
completenessBefore: null,
|
||||
completenessAfter: null,
|
||||
conclusionConfidenceBefore: null,
|
||||
conclusionConfidenceAfter: null,
|
||||
resolvedDirectChildren: 0,
|
||||
unresolvedDirectChildren: 0,
|
||||
contradictoryDirectChildren: 0,
|
||||
corroboratingBranchCount: 0,
|
||||
conflictingBranchCount: 0,
|
||||
duplicateEvidenceCount: 0,
|
||||
independentBranchCount: 0,
|
||||
interactionSummary: null,
|
||||
confidenceCapReason: null,
|
||||
ancestorPropagationStoppedReason: null,
|
||||
affectedAncestorIds: [],
|
||||
nextSelectedSibling: null,
|
||||
parentResolved: false,
|
||||
decompositionPerformed: false,
|
||||
childUnknownCount: 0,
|
||||
childNodeIds: [],
|
||||
atomicityReason: null,
|
||||
unknownSelectionExplanation: buildUnknownSelectionDiagnostics(
|
||||
situationGraph,
|
||||
situationGraph.resolvedNodeIds || [],
|
||||
null,
|
||||
),
|
||||
}),
|
||||
};
|
||||
}
|
||||
@@ -1,134 +0,0 @@
|
||||
import {
|
||||
ConfidenceLevel,
|
||||
SituationKind,
|
||||
SituationRelationship,
|
||||
SituationStatus,
|
||||
} from "./schema.js";
|
||||
|
||||
const DEFAULT_PROMPT_VERSION = "v0.4";
|
||||
|
||||
function formatEnumValues(values) {
|
||||
return Object.values(values).join(" | ");
|
||||
}
|
||||
|
||||
function formatGraph(graph) {
|
||||
return JSON.stringify(graph, null, 2);
|
||||
}
|
||||
|
||||
function formatExampleAnswerBlock() {
|
||||
return [
|
||||
"Example answer the model must be able to handle without hard-coding output:",
|
||||
'"The complaint rate fell from 2.0 complaints per 100 units to 1.9 complaints per 100 units."',
|
||||
"This may justify resolving a rate-related unknown or updating a metric node, but only if the current graph and answer support that proposal.",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function buildGraphUpdatePrompt({
|
||||
situationGraph,
|
||||
previousQuestion,
|
||||
answer,
|
||||
promptVersion = DEFAULT_PROMPT_VERSION,
|
||||
}) {
|
||||
const nodeKinds = formatEnumValues(SituationKind);
|
||||
const nodeStatuses = formatEnumValues(SituationStatus);
|
||||
const edgeRelationships = formatEnumValues(SituationRelationship);
|
||||
const confidenceLevels = formatEnumValues(ConfidenceLevel);
|
||||
|
||||
return `You are proposing a graph update for Confidence Engine ${promptVersion}.
|
||||
|
||||
Return exactly one JSON object matching the GraphUpdate contract.
|
||||
Return JSON only. Do not include markdown, explanation, or any text before or after the JSON object.
|
||||
|
||||
## Current Situation Graph
|
||||
${formatGraph(situationGraph)}
|
||||
|
||||
## Previous Selected Question
|
||||
${previousQuestion}
|
||||
|
||||
## User Answer
|
||||
${answer}
|
||||
|
||||
## Allowed Node Kinds
|
||||
${nodeKinds}
|
||||
|
||||
## Allowed Node Statuses
|
||||
${nodeStatuses}
|
||||
|
||||
## Allowed Edge Relationships
|
||||
${edgeRelationships}
|
||||
|
||||
## Allowed Confidence Values
|
||||
${confidenceLevels}
|
||||
|
||||
## Required JSON Field Names
|
||||
The JSON object must contain exactly these top-level fields:
|
||||
- addedNodes
|
||||
- updatedNodes
|
||||
- addedEdges
|
||||
- removedEdgeIds
|
||||
- resolvedUnknownNodeIds
|
||||
- affectedNodeIds
|
||||
- selectedQuestion
|
||||
|
||||
## Required Shapes
|
||||
- addedNodes: array of nodes using these exact keys:
|
||||
id, label, description, kind, status, confidence, value, unit, evidenceIds, dependsOn, affects, parentId, childIds
|
||||
- updatedNodes: array of node updates using these exact keys:
|
||||
nodeId, previousStatus, newStatus, previousValue, newValue, reason
|
||||
- addedEdges: array of edges using these exact keys:
|
||||
id, fromNodeId, toNodeId, relationship, confidence, description
|
||||
- removedEdgeIds: array of strings
|
||||
- resolvedUnknownNodeIds: array of strings
|
||||
- affectedNodeIds: array of strings
|
||||
- selectedQuestion: either null or an object using these exact keys:
|
||||
nodeId, question, reason
|
||||
|
||||
## Proposal Rules
|
||||
1. Propose changes only. Never return a replacement graph.
|
||||
2. Preserve unrelated nodes and edges by omitting them from the proposal.
|
||||
3. Reference existing node IDs when updating an existing concept.
|
||||
4. Use addedNodes only for genuinely new concepts.
|
||||
5. Resolve the answered unknown first when the answer supports it.
|
||||
6. Then inspect the answer for newly introduced consequential uncertainty.
|
||||
7. Add new unknown nodes only when the answer introduces a new decision, claim, object, measure, dependency, or unresolved term directly relevant to the case.
|
||||
8. Add at most 3 new unknown nodes.
|
||||
9. Every new unknown must be directly traceable to the user's answer and its description must state why that uncertainty matters.
|
||||
9a. In the description of every new unknown, explicitly include a short why-it-matters clause using wording such as because, so that, needed to decide, or matters because.
|
||||
10. Do not add broad generic discovery questions.
|
||||
11. Do not add duplicate unknowns.
|
||||
12. Do not expand unrelated branches.
|
||||
13. Propagate only through explicit dependencies or relationships already present in the graph, except for the minimal new edges needed to connect validated new unknowns to the relevant answer-derived decision or context node.
|
||||
13a. For every new unknown node, include at least one added edge that connects it to an existing updated/resolved node or to a newly added non-unknown node introduced from the answer.
|
||||
14. Do not invent evidence.
|
||||
15. Do not create unsupported causal edges.
|
||||
16. If consequential unresolved unknowns exist, selectedQuestion may identify one valid candidate unknown, but the engine will deterministically choose final priority after validation.
|
||||
17. selectedQuestion.nodeId must reference an unresolved unknown node that exists either already in the graph or in addedNodes.
|
||||
18. selectedQuestion.question must be one narrow non-compound question about that one unknown.
|
||||
19. Do not prioritise downstream implementation, pricing, optimisation, or speculative branches ahead of prerequisite definitions, actors, success criteria, constraints, measures, or terminology.
|
||||
20. Return selectedQuestion as null only when no consequential unresolved unknown remains.
|
||||
21. Use empty arrays when there are no changes in a category.
|
||||
22. Never return null array entries.
|
||||
23. Never use unknown enum values.
|
||||
24. Do not change existing IDs.
|
||||
25. Do not replace the whole graph, and do not restate unchanged graph content inside the proposal.
|
||||
|
||||
## Additional Guidance
|
||||
- If the answer only clarifies an existing unknown, prefer updatedNodes and resolvedUnknownNodeIds over creating duplicate nodes.
|
||||
- When an answer resolves an existing unknown, include that existing node ID in resolvedUnknownNodeIds and update that node rather than creating only a parallel observation.
|
||||
- If the answer creates a more specific decision situation, add the smallest set of new nodes and edges needed to represent that situation and only its most consequential unknowns.
|
||||
- If you add a new unknown, do not leave it floating: connect it with an added edge to the relevant decision/context node created or updated from the answer.
|
||||
- If you add a new unknown, its description must do two jobs in one sentence: what is unknown, and why resolving it matters for the case.
|
||||
- Treat selectedQuestion as a candidate only; the engine will apply deterministic information-value scoring after validation.
|
||||
- If the answer does not justify a change, return empty arrays for every category.
|
||||
|
||||
## Example Constraint Reminder
|
||||
${formatExampleAnswerBlock()}
|
||||
|
||||
## Output Contract Reminder
|
||||
Return one JSON object only, with exact field names and exact enum values.
|
||||
Never include a full graph.
|
||||
Never include any field other than the contract fields above.
|
||||
`;
|
||||
}
|
||||
|
||||
export const buildUpdatePrompt = buildGraphUpdatePrompt;
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,242 +0,0 @@
|
||||
/**
|
||||
* Situation Graph schema — v0.4 experiment.
|
||||
* Defines types for an evolving multi-turn situation reconstruction graph.
|
||||
* Plain TypeScript interfaces implemented as Zod schemas for runtime validation.
|
||||
*/
|
||||
|
||||
import { z } from "zod";
|
||||
|
||||
// ── Enums ────────────────────────────────────────────
|
||||
|
||||
export const SituationKind = /** @type {const} */ ({
|
||||
observation: "observation",
|
||||
reported_claim: "reported_claim",
|
||||
metric: "metric",
|
||||
state: "state",
|
||||
transition: "transition",
|
||||
relationship: "relationship",
|
||||
assumption: "assumption",
|
||||
unknown: "unknown",
|
||||
conclusion: "conclusion",
|
||||
});
|
||||
|
||||
export const SituationStatus = /** @type {const} */ ({
|
||||
known: "known",
|
||||
unknown: "unknown",
|
||||
provisional: "provisional",
|
||||
supported: "supported",
|
||||
weakened: "weakened",
|
||||
contradicted: "contradicted",
|
||||
resolved: "resolved",
|
||||
});
|
||||
|
||||
export const ConfidenceLevel = /** @type {const} */ ({
|
||||
low: "low",
|
||||
medium: "medium",
|
||||
high: "high",
|
||||
});
|
||||
|
||||
export const CompletenessStatus = /** @type {const} */ ({
|
||||
empty: "empty",
|
||||
partial: "partial",
|
||||
complete: "complete",
|
||||
});
|
||||
|
||||
export const confidenceAssessmentSchema = z
|
||||
.object({
|
||||
evidenceConfidence: z.enum(Object.values(ConfidenceLevel)),
|
||||
completenessStatus: z.enum(Object.values(CompletenessStatus)),
|
||||
conclusionConfidence: z.enum(Object.values(ConfidenceLevel)),
|
||||
})
|
||||
.strict();
|
||||
|
||||
// ── SituationNode ────────────────────────────────────
|
||||
|
||||
export const situationNodeSchema = z.object({
|
||||
id: z.string().min(1),
|
||||
label: z.string().min(1),
|
||||
description: z.string().min(1),
|
||||
kind: z.enum(Object.values(SituationKind)),
|
||||
status: z.enum(Object.values(SituationStatus)),
|
||||
confidence: z.enum(Object.values(ConfidenceLevel)),
|
||||
confidenceAssessment: confidenceAssessmentSchema.optional(),
|
||||
value: z.union([z.string(), z.number(), z.null()]).nullable().optional(),
|
||||
unit: z.string().nullable().optional(),
|
||||
evidenceIds: z.array(z.string()).default([]),
|
||||
dependsOn: z.array(z.string()).default([]),
|
||||
affects: z.array(z.string()).default([]),
|
||||
parentId: z.string().nullable().optional(),
|
||||
childIds: z.array(z.string()).default([]),
|
||||
});
|
||||
|
||||
/** @typedef {z.infer<typeof situationNodeSchema>} SituationNode */
|
||||
|
||||
// ── SituationEdge ────────────────────────────────────
|
||||
|
||||
export const SituationRelationship = /** @type {const} */ ({
|
||||
supports: "supports",
|
||||
weakens: "weakens",
|
||||
contradicts: "contradicts",
|
||||
depends_on: "depends_on",
|
||||
causes: "causes",
|
||||
may_cause: "may_cause",
|
||||
measures: "measures",
|
||||
compares_with: "compares_with",
|
||||
updates: "updates",
|
||||
other: "other",
|
||||
});
|
||||
|
||||
export const situationEdgeSchema = z.object({
|
||||
id: z.string().min(1),
|
||||
fromNodeId: z.string().min(1),
|
||||
toNodeId: z.string().min(1),
|
||||
relationship: z.enum(Object.values(SituationRelationship)),
|
||||
confidence: z.enum(Object.values(ConfidenceLevel)),
|
||||
description: z.string().min(1),
|
||||
});
|
||||
|
||||
/** @typedef {z.infer<typeof situationEdgeSchema>} SituationEdge */
|
||||
|
||||
// ── SituationGraph ───────────────────────────────────
|
||||
|
||||
const reasoningStageSchema = z.object({
|
||||
stage: z.string().min(1),
|
||||
status: z.string().min(1),
|
||||
outcome: z.string().min(1),
|
||||
});
|
||||
|
||||
export const reasoningStateSchema = z
|
||||
.object({
|
||||
comparabilityStatus: z.string().min(1).nullable().optional(),
|
||||
comparabilityReason: z.string().min(1).nullable().optional(),
|
||||
comparabilityEvidence: z.array(z.string()).default([]),
|
||||
relationshipStatus: z.string().min(1).nullable().optional(),
|
||||
relationshipReason: z.string().min(1).nullable().optional(),
|
||||
relationshipAssessed: z.boolean().optional(),
|
||||
contradictionReasoningAllowed: z.boolean().optional(),
|
||||
reasoningStages: z.array(reasoningStageSchema).default([]),
|
||||
})
|
||||
.strict();
|
||||
|
||||
export const situationGraphSchema = z.object({
|
||||
centralStatement: z.string().min(1),
|
||||
nodes: z.array(situationNodeSchema).min(1),
|
||||
edges: z.array(situationEdgeSchema).default([]),
|
||||
activeUnknownNodeId: z.string().nullable(),
|
||||
resolvedNodeIds: z.array(z.string()).default([]),
|
||||
currentSummary: z.string().min(1),
|
||||
reasoningState: reasoningStateSchema.optional(),
|
||||
});
|
||||
|
||||
/** @typedef {z.infer<typeof situationGraphSchema>} SituationGraph */
|
||||
|
||||
// ── GraphUpdate (change set) ────────────────────────
|
||||
|
||||
const graphUpdateNodeChangeSchema = z.object({
|
||||
nodeId: z.string().min(1),
|
||||
previousStatus: z.enum(Object.values(SituationStatus)).nullable().optional(),
|
||||
newStatus: z.enum(Object.values(SituationStatus)).nullable().optional(),
|
||||
previousValue: z
|
||||
.union([z.string(), z.number(), z.null()])
|
||||
.nullable()
|
||||
.optional(),
|
||||
newValue: z.union([z.string(), z.number(), z.null()]).nullable().optional(),
|
||||
reason: z.string().min(1),
|
||||
});
|
||||
|
||||
export const selectedQuestionSchema = z
|
||||
.object({
|
||||
nodeId: z.string().min(1),
|
||||
question: z.string().min(1),
|
||||
reason: z.string().min(1),
|
||||
})
|
||||
.strict();
|
||||
|
||||
export const graphUpdateSchema = z.object({
|
||||
addedNodes: z.array(situationNodeSchema).default([]),
|
||||
updatedNodes: z.array(graphUpdateNodeChangeSchema).default([]),
|
||||
addedEdges: z.array(situationEdgeSchema).default([]),
|
||||
removedEdgeIds: z.array(z.string()).default([]),
|
||||
resolvedUnknownNodeIds: z.array(z.string()).default([]),
|
||||
affectedNodeIds: z.array(z.string()).default([]),
|
||||
selectedQuestion: selectedQuestionSchema.nullable().default(null),
|
||||
});
|
||||
|
||||
/** @typedef {z.infer<typeof graphUpdateSchema>} GraphUpdate */
|
||||
|
||||
// ── API request / response schemas ───────────────────
|
||||
|
||||
export const startCaseRequestSchema = z.object({
|
||||
scenario: z.string().min(1).max(10000),
|
||||
promptVersion: z.string().optional(),
|
||||
});
|
||||
|
||||
export const updateCaseRequestSchema = z.object({
|
||||
situationGraph: situationGraphSchema,
|
||||
previousQuestion: z.string().min(1),
|
||||
answer: z.string().min(1).max(5000),
|
||||
promptVersion: z.string().optional(),
|
||||
});
|
||||
|
||||
// ── Helpers ──────────────────────────────────────────
|
||||
|
||||
/** Generate a short deterministic ID from a label */
|
||||
export function makeNodeId(label) {
|
||||
return "n" + Math.abs(hashString(label)).toString(36).slice(0, 7);
|
||||
}
|
||||
|
||||
function hashString(str) {
|
||||
let h = 0;
|
||||
for (let i = 0; i < str.length; i++) {
|
||||
h = (Math.imul(31, h) + str.charCodeAt(i)) | 0;
|
||||
}
|
||||
return h;
|
||||
}
|
||||
|
||||
/** Create a minimal valid node — used in tests and fixtures */
|
||||
export function makeNode(opts) {
|
||||
const id = opts.id || makeNodeId(opts.label);
|
||||
return situationNodeSchema.parse({
|
||||
id,
|
||||
label: opts.label,
|
||||
description: opts.description ?? opts.label,
|
||||
kind: opts.kind ?? "observation",
|
||||
status: opts.status ?? "unknown",
|
||||
confidence: opts.confidence ?? "medium",
|
||||
confidenceAssessment: opts.confidenceAssessment,
|
||||
value: opts.value ?? null,
|
||||
unit: opts.unit ?? null,
|
||||
evidenceIds: opts.evidenceIds ?? [],
|
||||
dependsOn: opts.dependsOn ?? [],
|
||||
affects: opts.affects ?? [],
|
||||
parentId: opts.parentId ?? null,
|
||||
childIds: opts.childIds ?? [],
|
||||
});
|
||||
}
|
||||
|
||||
/** Create a minimal valid edge — used in tests and fixtures */
|
||||
export function makeEdge(opts) {
|
||||
return situationEdgeSchema.parse({
|
||||
id:
|
||||
opts.id ||
|
||||
"e" + opts.fromNodeId.slice(0, 3) + "-" + opts.toNodeId.slice(0, 3),
|
||||
fromNodeId: opts.fromNodeId,
|
||||
toNodeId: opts.toNodeId,
|
||||
relationship: opts.relationship ?? "supports",
|
||||
confidence: opts.confidence ?? "medium",
|
||||
description: opts.description ?? opts.fromNodeId + " -> " + opts.toNodeId,
|
||||
});
|
||||
}
|
||||
|
||||
/** Build a minimal valid graph structure */
|
||||
export function makeGraph(opts) {
|
||||
return situationGraphSchema.parse({
|
||||
centralStatement: opts.centralStatement || "",
|
||||
nodes: opts.nodes ?? [],
|
||||
edges: opts.edges ?? [],
|
||||
activeUnknownNodeId: opts.activeUnknownNodeId ?? null,
|
||||
resolvedNodeIds: opts.resolvedNodeIds ?? [],
|
||||
currentSummary: opts.currentSummary || "",
|
||||
reasoningState: opts.reasoningState,
|
||||
});
|
||||
}
|
||||
@@ -1,157 +0,0 @@
|
||||
import { graphUpdateSchema } from "./schema.js";
|
||||
|
||||
const TOP_LEVEL_ARRAY_FIELDS = [
|
||||
"addedNodes",
|
||||
"updatedNodes",
|
||||
"addedEdges",
|
||||
"removedEdgeIds",
|
||||
"resolvedUnknownNodeIds",
|
||||
"affectedNodeIds",
|
||||
];
|
||||
|
||||
const TOP_LEVEL_NULLABLE_FIELDS = ["selectedQuestion"];
|
||||
|
||||
function cloneJsonSafe(value) {
|
||||
if (value == null) return value;
|
||||
return JSON.parse(JSON.stringify(value));
|
||||
}
|
||||
|
||||
function removeNullArrayEntries(value, path = [], normalisationsApplied = []) {
|
||||
if (Array.isArray(value)) {
|
||||
const filtered = [];
|
||||
value.forEach((item, index) => {
|
||||
if (item === null) {
|
||||
normalisationsApplied.push({
|
||||
path: [...path, index],
|
||||
change: "Removed null array entry",
|
||||
});
|
||||
return;
|
||||
}
|
||||
filtered.push(
|
||||
removeNullArrayEntries(item, [...path, index], normalisationsApplied),
|
||||
);
|
||||
});
|
||||
return filtered;
|
||||
}
|
||||
|
||||
if (value && typeof value === "object") {
|
||||
return Object.fromEntries(
|
||||
Object.entries(value).map(([key, child]) => [
|
||||
key,
|
||||
removeNullArrayEntries(child, [...path, key], normalisationsApplied),
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
function applyKnownEnumAliases(proposal, normalisationsApplied) {
|
||||
if (!proposal || typeof proposal !== "object") return proposal;
|
||||
|
||||
if (Array.isArray(proposal.addedNodes)) {
|
||||
proposal.addedNodes = proposal.addedNodes.map((node, index) => {
|
||||
if (node?.kind === "reported_statement") {
|
||||
normalisationsApplied.push({
|
||||
path: ["addedNodes", index, "kind"],
|
||||
change: "Converted reported_statement to reported_claim",
|
||||
});
|
||||
return { ...node, kind: "reported_claim" };
|
||||
}
|
||||
return node;
|
||||
});
|
||||
}
|
||||
|
||||
return proposal;
|
||||
}
|
||||
|
||||
function fillMissingOptionalArrays(proposal, normalisationsApplied) {
|
||||
if (!proposal || typeof proposal !== "object") return proposal;
|
||||
|
||||
for (const field of TOP_LEVEL_ARRAY_FIELDS) {
|
||||
if (!(field in proposal)) {
|
||||
proposal[field] = [];
|
||||
normalisationsApplied.push({
|
||||
path: [field],
|
||||
change: "Filled missing optional array with []",
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return proposal;
|
||||
}
|
||||
|
||||
function fillMissingNullableFields(proposal, normalisationsApplied) {
|
||||
if (!proposal || typeof proposal !== "object") return proposal;
|
||||
|
||||
for (const field of TOP_LEVEL_NULLABLE_FIELDS) {
|
||||
if (!(field in proposal)) {
|
||||
proposal[field] = null;
|
||||
normalisationsApplied.push({
|
||||
path: [field],
|
||||
change: "Filled missing optional nullable field with null",
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return proposal;
|
||||
}
|
||||
|
||||
export function parseGraphUpdateProposal(rawResponse) {
|
||||
const raw = rawResponse;
|
||||
let parsed;
|
||||
|
||||
if (typeof rawResponse === "string") {
|
||||
try {
|
||||
parsed = JSON.parse(rawResponse);
|
||||
} catch (error) {
|
||||
return {
|
||||
success: false,
|
||||
proposal: null,
|
||||
raw,
|
||||
normalisationsApplied: [],
|
||||
errors: [error.message || "Model response is not valid JSON"],
|
||||
};
|
||||
}
|
||||
} else if (rawResponse && typeof rawResponse === "object") {
|
||||
parsed = cloneJsonSafe(rawResponse);
|
||||
} else {
|
||||
return {
|
||||
success: false,
|
||||
proposal: null,
|
||||
raw,
|
||||
normalisationsApplied: [],
|
||||
errors: ["Graph update proposal must be a JSON object or JSON string"],
|
||||
};
|
||||
}
|
||||
|
||||
const normalisationsApplied = [];
|
||||
let normalised = removeNullArrayEntries(parsed, [], normalisationsApplied);
|
||||
normalised = applyKnownEnumAliases(normalised, normalisationsApplied);
|
||||
normalised = fillMissingOptionalArrays(normalised, normalisationsApplied);
|
||||
normalised = fillMissingNullableFields(normalised, normalisationsApplied);
|
||||
|
||||
const parsedProposal = graphUpdateSchema.safeParse(normalised);
|
||||
|
||||
if (!parsedProposal.success) {
|
||||
return {
|
||||
success: false,
|
||||
proposal: null,
|
||||
raw,
|
||||
normalisationsApplied,
|
||||
errors: parsedProposal.error.issues.map((issue) => ({
|
||||
path: issue.path,
|
||||
message: issue.message,
|
||||
code: issue.code,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
proposal: parsedProposal.data,
|
||||
raw,
|
||||
normalisationsApplied,
|
||||
errors: [],
|
||||
};
|
||||
}
|
||||
@@ -1,894 +0,0 @@
|
||||
/**
|
||||
* Deterministic graph utilities for situation graph operations.
|
||||
* These functions perform safe, validated operations on the graph.
|
||||
* The LLM should never directly modify the graph — it proposes changes,
|
||||
* and these utilities apply them safely.
|
||||
*/
|
||||
|
||||
import {
|
||||
situationNodeSchema,
|
||||
situationEdgeSchema,
|
||||
situationGraphSchema,
|
||||
} from "./schema.js";
|
||||
|
||||
function normaliseText(value) {
|
||||
return String(value || "")
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, " ")
|
||||
.trim();
|
||||
}
|
||||
|
||||
function collectNodeText(node) {
|
||||
return `${node?.label || ""} ${node?.description || ""}`.trim();
|
||||
}
|
||||
|
||||
function countIncomingUnknownDependencies(graph, nodeId, resolvedNodeIds) {
|
||||
const resolvedSet = new Set(resolvedNodeIds || []);
|
||||
const nodesById = new Map(graph.nodes.map((node) => [node.id, node]));
|
||||
const incoming = new Set();
|
||||
|
||||
for (const dependencyId of nodesById.get(nodeId)?.dependsOn || []) {
|
||||
const dependencyNode = nodesById.get(dependencyId);
|
||||
if (dependencyNode?.kind === "unknown" && !resolvedSet.has(dependencyId)) {
|
||||
incoming.add(dependencyId);
|
||||
}
|
||||
}
|
||||
|
||||
for (const edge of graph.edges) {
|
||||
if (edge.toNodeId !== nodeId) continue;
|
||||
const dependencyNode = nodesById.get(edge.fromNodeId);
|
||||
if (
|
||||
dependencyNode?.kind === "unknown" &&
|
||||
!resolvedSet.has(edge.fromNodeId)
|
||||
) {
|
||||
incoming.add(edge.fromNodeId);
|
||||
}
|
||||
}
|
||||
|
||||
return incoming.size;
|
||||
}
|
||||
|
||||
function classifyUnknownPriority(text) {
|
||||
const normalised = normaliseText(text);
|
||||
|
||||
const matches = {
|
||||
objective:
|
||||
/\b(objective|goal|outcome|value|problem|job to be done|benefit|commercial value)\b/.test(
|
||||
normalised,
|
||||
),
|
||||
actor:
|
||||
/\b(customer|user|buyer|actor|stakeholder|audience|recipient)\b/.test(
|
||||
normalised,
|
||||
),
|
||||
criteria:
|
||||
/\b(success criteria|success threshold|threshold|decision criteria|criterion|justify|sufficient)\b/.test(
|
||||
normalised,
|
||||
),
|
||||
measure:
|
||||
/\b(metric|measure|measurable|roi|demand|evidence|signal|proof)\b/.test(
|
||||
normalised,
|
||||
),
|
||||
terminology: /\b(define|definition|meaning|means|term|terminology)\b/.test(
|
||||
normalised,
|
||||
),
|
||||
constraint:
|
||||
/\b(constraint|limit|budget|deadline|requirement|regulation)\b/.test(
|
||||
normalised,
|
||||
),
|
||||
pricing: /\b(price|pricing|price point|subscription|charge|pay for)\b/.test(
|
||||
normalised,
|
||||
),
|
||||
implementation:
|
||||
/\b(implementation|build approach|architecture|stack|feature|technical design)\b/.test(
|
||||
normalised,
|
||||
),
|
||||
optimisation:
|
||||
/\b(optimisation|optimi[sz]ation|improve|efficiency|performance|scale)\b/.test(
|
||||
normalised,
|
||||
),
|
||||
speculative:
|
||||
/\b(maybe|possible|optional|future branch|nice to have|slogan|colour|color|ui)\b/.test(
|
||||
normalised,
|
||||
),
|
||||
};
|
||||
|
||||
return matches;
|
||||
}
|
||||
|
||||
function buildScoreContributions(
|
||||
matches,
|
||||
downstreamCount,
|
||||
unresolvedParentUnknownCount,
|
||||
) {
|
||||
const contributions = [
|
||||
{
|
||||
rule: "downstream_dependencies",
|
||||
value: downstreamCount,
|
||||
weight: 4,
|
||||
delta: downstreamCount * 4,
|
||||
},
|
||||
];
|
||||
|
||||
if (matches.objective) {
|
||||
contributions.push({
|
||||
rule: "objective_match",
|
||||
value: true,
|
||||
weight: 12,
|
||||
delta: 12,
|
||||
});
|
||||
}
|
||||
if (matches.actor) {
|
||||
contributions.push({
|
||||
rule: "actor_match",
|
||||
value: true,
|
||||
weight: 10,
|
||||
delta: 10,
|
||||
});
|
||||
}
|
||||
if (matches.criteria) {
|
||||
contributions.push({
|
||||
rule: "criteria_match",
|
||||
value: true,
|
||||
weight: 11,
|
||||
delta: 11,
|
||||
});
|
||||
}
|
||||
if (matches.measure) {
|
||||
contributions.push({
|
||||
rule: "measure_match",
|
||||
value: true,
|
||||
weight: 8,
|
||||
delta: 8,
|
||||
});
|
||||
}
|
||||
if (matches.terminology) {
|
||||
contributions.push({
|
||||
rule: "terminology_match",
|
||||
value: true,
|
||||
weight: 7,
|
||||
delta: 7,
|
||||
});
|
||||
}
|
||||
if (matches.constraint) {
|
||||
contributions.push({
|
||||
rule: "constraint_match",
|
||||
value: true,
|
||||
weight: 9,
|
||||
delta: 9,
|
||||
});
|
||||
}
|
||||
if (matches.pricing) {
|
||||
contributions.push({
|
||||
rule: "pricing_penalty",
|
||||
value: true,
|
||||
weight: -8,
|
||||
delta: -8,
|
||||
});
|
||||
}
|
||||
if (matches.implementation) {
|
||||
contributions.push({
|
||||
rule: "implementation_penalty",
|
||||
value: true,
|
||||
weight: -10,
|
||||
delta: -10,
|
||||
});
|
||||
}
|
||||
if (matches.optimisation) {
|
||||
contributions.push({
|
||||
rule: "optimisation_penalty",
|
||||
value: true,
|
||||
weight: -9,
|
||||
delta: -9,
|
||||
});
|
||||
}
|
||||
if (matches.speculative) {
|
||||
contributions.push({
|
||||
rule: "speculative_penalty",
|
||||
value: true,
|
||||
weight: -12,
|
||||
delta: -12,
|
||||
});
|
||||
}
|
||||
|
||||
if (
|
||||
matches.pricing &&
|
||||
!matches.objective &&
|
||||
!matches.criteria &&
|
||||
!matches.actor
|
||||
) {
|
||||
contributions.push({
|
||||
rule: "isolated_pricing_penalty",
|
||||
value: true,
|
||||
weight: -6,
|
||||
delta: -6,
|
||||
});
|
||||
}
|
||||
|
||||
if (unresolvedParentUnknownCount > 0) {
|
||||
contributions.push({
|
||||
rule: "unresolved_prerequisite_penalty",
|
||||
value: unresolvedParentUnknownCount,
|
||||
weight: -7,
|
||||
delta: unresolvedParentUnknownCount * -7,
|
||||
});
|
||||
}
|
||||
|
||||
return contributions;
|
||||
}
|
||||
|
||||
function getMeaningfulSemanticContributions(contributions = []) {
|
||||
return contributions
|
||||
.filter(
|
||||
(contribution) =>
|
||||
contribution.rule !== "downstream_dependencies" &&
|
||||
contribution.rule !== "unresolved_prerequisite_penalty" &&
|
||||
contribution.delta !== 0,
|
||||
)
|
||||
.map((contribution) => ({
|
||||
rule: contribution.rule,
|
||||
delta: contribution.delta,
|
||||
}));
|
||||
}
|
||||
|
||||
function buildCandidateDisplayOrder(candidates) {
|
||||
return [...candidates].sort((a, b) => {
|
||||
if (b.score !== a.score) return b.score - a.score;
|
||||
if (b.downstreamCount !== a.downstreamCount) {
|
||||
return b.downstreamCount - a.downstreamCount;
|
||||
}
|
||||
if (a.unresolvedParentUnknownCount !== b.unresolvedParentUnknownCount) {
|
||||
return a.unresolvedParentUnknownCount - b.unresolvedParentUnknownCount;
|
||||
}
|
||||
return a.label.localeCompare(b.label);
|
||||
});
|
||||
}
|
||||
|
||||
function semanticSignature(candidate) {
|
||||
return JSON.stringify(
|
||||
getMeaningfulSemanticContributions(candidate.contributions),
|
||||
);
|
||||
}
|
||||
|
||||
function classifyCandidateOrdering(candidates) {
|
||||
const displayOrder = buildCandidateDisplayOrder(candidates);
|
||||
const best = displayOrder[0] ?? null;
|
||||
if (!best) {
|
||||
return {
|
||||
displayOrder,
|
||||
best: null,
|
||||
leadingCandidates: [],
|
||||
status: "no_candidates",
|
||||
tieType: "none",
|
||||
usedAlphabeticalOrdering: false,
|
||||
reason: "No unresolved unknown candidates remain.",
|
||||
};
|
||||
}
|
||||
|
||||
const topScoreCandidates = displayOrder.filter(
|
||||
(candidate) => candidate.score === best.score,
|
||||
);
|
||||
|
||||
if (topScoreCandidates.length === 1) {
|
||||
return {
|
||||
displayOrder,
|
||||
best,
|
||||
leadingCandidates: [best],
|
||||
status: "selected",
|
||||
tieType: "none",
|
||||
usedAlphabeticalOrdering: false,
|
||||
reason: `Clear winner by total score (${best.score}).`,
|
||||
};
|
||||
}
|
||||
|
||||
const topStructuralCandidates = topScoreCandidates.filter(
|
||||
(candidate) =>
|
||||
candidate.downstreamCount === best.downstreamCount &&
|
||||
candidate.unresolvedParentUnknownCount ===
|
||||
best.unresolvedParentUnknownCount,
|
||||
);
|
||||
|
||||
if (topStructuralCandidates.length === 1) {
|
||||
return {
|
||||
displayOrder,
|
||||
best,
|
||||
leadingCandidates: [best],
|
||||
status: "selected",
|
||||
tieType: "structural_tie",
|
||||
usedAlphabeticalOrdering: false,
|
||||
reason:
|
||||
"Score tie was resolved by downstream dependency count or prerequisite ordering.",
|
||||
};
|
||||
}
|
||||
|
||||
const topSemanticSignature = semanticSignature(best);
|
||||
const semanticPeers = topStructuralCandidates.filter(
|
||||
(candidate) => semanticSignature(candidate) === topSemanticSignature,
|
||||
);
|
||||
|
||||
if (semanticPeers.length !== topStructuralCandidates.length) {
|
||||
return {
|
||||
displayOrder,
|
||||
best: null,
|
||||
leadingCandidates: topStructuralCandidates,
|
||||
status: "ambiguous",
|
||||
tieType: "semantic_tie",
|
||||
usedAlphabeticalOrdering: false,
|
||||
reason:
|
||||
"Leading candidates remain tied after score and structural checks, but differ in semantic contribution patterns.",
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
displayOrder,
|
||||
best: null,
|
||||
leadingCandidates: topStructuralCandidates,
|
||||
status: "ambiguous",
|
||||
tieType: "complete_unresolved_tie",
|
||||
usedAlphabeticalOrdering: false,
|
||||
reason: "No justified distinction between leading unknowns.",
|
||||
};
|
||||
}
|
||||
|
||||
export function scoreUnknownCandidate(graph, node, resolvedNodeIds = []) {
|
||||
const text = collectNodeText(node);
|
||||
const matches = classifyUnknownPriority(text);
|
||||
const downstreamCount = findDependentNodes(graph, node.id).length;
|
||||
const unresolvedParentUnknownCount = countIncomingUnknownDependencies(
|
||||
graph,
|
||||
node.id,
|
||||
resolvedNodeIds,
|
||||
);
|
||||
|
||||
const contributions = buildScoreContributions(
|
||||
matches,
|
||||
downstreamCount,
|
||||
unresolvedParentUnknownCount,
|
||||
);
|
||||
const score = contributions.reduce(
|
||||
(total, contribution) => total + contribution.delta,
|
||||
0,
|
||||
);
|
||||
|
||||
return {
|
||||
nodeId: node.id,
|
||||
label: node.label,
|
||||
score,
|
||||
downstreamCount,
|
||||
unresolvedParentUnknownCount,
|
||||
matches,
|
||||
contributions,
|
||||
};
|
||||
}
|
||||
|
||||
export function buildDeterministicQuestionForUnknown(node) {
|
||||
const text = normaliseText(collectNodeText(node));
|
||||
|
||||
if (
|
||||
/\b(success criteria|success threshold|threshold|decision criteria|criterion)\b/.test(
|
||||
text,
|
||||
)
|
||||
) {
|
||||
return `What outcome would define success for ${node.label}?`;
|
||||
}
|
||||
if (
|
||||
/\b(customer|user|buyer|actor|stakeholder|audience|recipient)\b/.test(text)
|
||||
) {
|
||||
return `Who is the key actor or customer for ${node.label}?`;
|
||||
}
|
||||
if (
|
||||
/\b(define|definition|meaning|means|term|terminology|value)\b/.test(text)
|
||||
) {
|
||||
return `How should ${node.label} be defined for this decision?`;
|
||||
}
|
||||
if (
|
||||
/\b(metric|measure|measurable|roi|demand|evidence|signal|proof)\b/.test(
|
||||
text,
|
||||
)
|
||||
) {
|
||||
return `What evidence or measure would resolve ${node.label}?`;
|
||||
}
|
||||
|
||||
return `What would resolve ${node.label}?`;
|
||||
}
|
||||
|
||||
// ── Validate that all edge references point to existing nodes ──
|
||||
|
||||
export function validateGraphReferences(graph) {
|
||||
const errors = [];
|
||||
const nodeIds = new Set(graph.nodes.map((n) => n.id));
|
||||
|
||||
for (const node of graph.nodes) {
|
||||
if (node.parentId !== null && !nodeIds.has(node.parentId)) {
|
||||
errors.push(
|
||||
`Node "${node.id}" references parentId "${node.parentId}" which does not exist`,
|
||||
);
|
||||
}
|
||||
for (const cid of node.childIds) {
|
||||
if (!nodeIds.has(cid)) {
|
||||
errors.push(
|
||||
`Node "${node.id}" references childIds "${cid}" which does not exist`,
|
||||
);
|
||||
}
|
||||
}
|
||||
for (const dep of node.dependsOn) {
|
||||
if (!nodeIds.has(dep)) {
|
||||
errors.push(
|
||||
`Node "${node.id}" depends on "${dep}" which does not exist`,
|
||||
);
|
||||
}
|
||||
}
|
||||
for (const aff of node.affects) {
|
||||
if (!nodeIds.has(aff)) {
|
||||
errors.push(`Node "${node.id}" affects "${aff}" which does not exist`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const edge of graph.edges) {
|
||||
if (!nodeIds.has(edge.fromNodeId)) {
|
||||
errors.push(
|
||||
`Edge "${edge.id}" references non-existent fromNodeId "${edge.fromNodeId}"`,
|
||||
);
|
||||
}
|
||||
if (!nodeIds.has(edge.toNodeId)) {
|
||||
errors.push(
|
||||
`Edge "${edge.id}" references non-existent toNodeId "${edge.toNodeId}"`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
return { valid: errors.length === 0, errors };
|
||||
}
|
||||
|
||||
// ── Detect duplicate node IDs ──
|
||||
|
||||
export function detectDuplicateNodeIds(nodes) {
|
||||
const countMap = new Map();
|
||||
const seen = new Set();
|
||||
|
||||
for (const node of nodes) {
|
||||
if (countMap.has(node.id)) {
|
||||
countMap.set(node.id, countMap.get(node.id) + 1);
|
||||
} else {
|
||||
countMap.set(node.id, 1);
|
||||
}
|
||||
}
|
||||
|
||||
const duplicates = [];
|
||||
for (const [id, count] of countMap.entries()) {
|
||||
if (count > 1 && !seen.has(id)) {
|
||||
duplicates.push({ nodeId: id, count });
|
||||
seen.add(id);
|
||||
}
|
||||
}
|
||||
|
||||
return duplicates;
|
||||
}
|
||||
|
||||
// ── Detect duplicate edges ──
|
||||
|
||||
export function detectDuplicateEdges(edges) {
|
||||
const seen = new Set();
|
||||
const duplicates = [];
|
||||
|
||||
for (const edge of edges) {
|
||||
const key = `${edge.fromNodeId}->${edge.toNodeId}:${edge.relationship}`;
|
||||
if (seen.has(key)) {
|
||||
duplicates.push({
|
||||
edgeId: edge.id,
|
||||
fromNodeId: edge.fromNodeId,
|
||||
toNodeId: edge.toNodeId,
|
||||
relationship: edge.relationship,
|
||||
});
|
||||
}
|
||||
seen.add(key);
|
||||
}
|
||||
|
||||
return duplicates;
|
||||
}
|
||||
|
||||
// ── Find all nodes that depend on a given node (transitive) ──
|
||||
|
||||
export function findDependentNodes(graph, nodeId) {
|
||||
const direct = graph.nodes
|
||||
.filter((n) => n.dependsOn.includes(nodeId))
|
||||
.map((n) => n.id);
|
||||
const affected = new Set(direct);
|
||||
|
||||
// Also propagate through edges where the relationship is depends_on
|
||||
for (const edge of graph.edges) {
|
||||
if (edge.toNodeId === nodeId && !affected.has(edge.fromNodeId)) {
|
||||
direct.push(edge.fromNodeId);
|
||||
affected.add(edge.fromNodeId);
|
||||
}
|
||||
}
|
||||
|
||||
// Transitive propagation — BFS
|
||||
const queue = [...direct];
|
||||
while (queue.length > 0) {
|
||||
const current = queue.shift();
|
||||
if (!current || !affected.has(current)) continue;
|
||||
|
||||
for (const node of graph.nodes) {
|
||||
if (node.dependsOn.includes(current) && !affected.has(node.id)) {
|
||||
affected.add(node.id);
|
||||
queue.push(node.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return [...affected];
|
||||
}
|
||||
|
||||
// ── Find all nodes that are directly or indirectly affected by a change in nodeId ──
|
||||
|
||||
export function findAffectedNodes(graph, nodeId) {
|
||||
// Direct effects: two sources
|
||||
// 1. Nodes that depend on this node (they list it in their dependsOn)
|
||||
const directFromDepends = graph.nodes
|
||||
.filter((n) => n.id !== nodeId && n.dependsOn.includes(nodeId))
|
||||
.map((n) => n.id);
|
||||
|
||||
// 2. Targets of the node's affects relationships (this node directly affects them)
|
||||
const myAffectedTargets = new Set(
|
||||
graph.nodes.find((n) => n.id === nodeId)?.affects || [],
|
||||
);
|
||||
|
||||
// Merge: also add edge targets where this node is the source
|
||||
for (const edge of graph.edges) {
|
||||
if (edge.fromNodeId === nodeId && !myAffectedTargets.has(edge.toNodeId)) {
|
||||
myAffectedTargets.add(edge.toNodeId);
|
||||
}
|
||||
}
|
||||
|
||||
// Combine both sources
|
||||
const direct = [...new Set([...directFromDepends, ...myAffectedTargets])];
|
||||
|
||||
// Transitive propagation — BFS through dependsOn and affects of affected nodes
|
||||
const affected = new Set(direct);
|
||||
const queue = [...direct];
|
||||
while (queue.length > 0) {
|
||||
const current = queue.shift();
|
||||
if (!current || !affected.has(current)) continue;
|
||||
|
||||
for (const node of graph.nodes) {
|
||||
if (
|
||||
node.id !== nodeId &&
|
||||
!affected.has(node.id) &&
|
||||
(node.dependsOn.includes(current) || node.affects.includes(current))
|
||||
) {
|
||||
affected.add(node.id);
|
||||
queue.push(node.id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return [...affected];
|
||||
}
|
||||
|
||||
// ── Resolve an unknown node ──
|
||||
|
||||
export function resolveUnknownNode(graph, nodeId, newStatus, newValue, reason) {
|
||||
const nodeIdx = graph.nodes.findIndex((n) => n.id === nodeId);
|
||||
if (nodeIdx === -1) {
|
||||
return { success: false, error: `Node "${nodeId}" not found in graph` };
|
||||
}
|
||||
|
||||
const previousStatus = graph.nodes[nodeIdx].status;
|
||||
const previousValue = graph.nodes[nodeIdx].value;
|
||||
|
||||
return {
|
||||
success: true,
|
||||
previousStatus,
|
||||
newStatus,
|
||||
previousValue,
|
||||
newValue,
|
||||
reason,
|
||||
affectedNodes: findAffectedNodes(graph, nodeId),
|
||||
};
|
||||
}
|
||||
|
||||
// ── Select the next highest-value active unknown candidate ──
|
||||
|
||||
export function selectActiveUnknownCandidate(graph, resolvedNodeIds) {
|
||||
// Skip already resolved nodes
|
||||
const unresolved = graph.nodes.filter(
|
||||
(n) => n.kind === "unknown" && !resolvedNodeIds.includes(n.id),
|
||||
);
|
||||
|
||||
if (unresolved.length === 0) return null;
|
||||
|
||||
const scoredCandidates = unresolved.map((node) => ({
|
||||
node,
|
||||
...scoreUnknownCandidate(graph, node, resolvedNodeIds),
|
||||
}));
|
||||
|
||||
const selection = classifyCandidateOrdering(
|
||||
scoredCandidates.map(({ node, ...candidate }) => ({
|
||||
...candidate,
|
||||
node,
|
||||
})),
|
||||
);
|
||||
|
||||
if (selection.status === "ambiguous") {
|
||||
return {
|
||||
selectedNode: null,
|
||||
status: "ambiguous",
|
||||
tieType: selection.tieType,
|
||||
tiedCandidateIds: selection.leadingCandidates.map(
|
||||
(candidate) => candidate.nodeId,
|
||||
),
|
||||
displayOrder: selection.displayOrder.map((candidate) => candidate.nodeId),
|
||||
reason: selection.reason,
|
||||
};
|
||||
}
|
||||
|
||||
const best = selection.best;
|
||||
if (!best) return null;
|
||||
|
||||
return {
|
||||
selectedNode: {
|
||||
nodeId: best.node.id,
|
||||
label: best.node.label,
|
||||
},
|
||||
status: "selected",
|
||||
tieType: selection.tieType,
|
||||
nodeId: best.node.id,
|
||||
label: best.node.label,
|
||||
score: best.score,
|
||||
question: buildDeterministicQuestionForUnknown(best.node),
|
||||
reason: `Selected for highest information value (score ${best.score}) with ${best.downstreamCount} downstream dependency node(s) and ${best.unresolvedParentUnknownCount} unresolved prerequisite unknown(s).`,
|
||||
};
|
||||
}
|
||||
|
||||
export function explainUnknownSelection(graph, resolvedNodeIds = []) {
|
||||
const unresolved = graph.nodes.filter(
|
||||
(n) => n.kind === "unknown" && !resolvedNodeIds.includes(n.id),
|
||||
);
|
||||
|
||||
if (unresolved.length === 0) {
|
||||
return {
|
||||
selectedNodeId: null,
|
||||
selectedNodeLabel: null,
|
||||
status: "no_candidates",
|
||||
tieType: "none",
|
||||
resolvedNodeIds: [...resolvedNodeIds],
|
||||
tiedCandidateIds: [],
|
||||
candidates: [],
|
||||
competitors: [],
|
||||
tieBreakOrder: [
|
||||
"score_desc",
|
||||
"downstreamCount_desc",
|
||||
"unresolvedParentUnknownCount_asc",
|
||||
"label_asc",
|
||||
],
|
||||
summary: {
|
||||
candidateCount: 0,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const candidates = unresolved.map((node) => ({
|
||||
nodeId: node.id,
|
||||
label: node.label,
|
||||
...scoreUnknownCandidate(graph, node, resolvedNodeIds),
|
||||
}));
|
||||
|
||||
const selection = classifyCandidateOrdering(candidates);
|
||||
const orderedCandidates = selection.displayOrder;
|
||||
const selected = selection.best;
|
||||
const competitors = orderedCandidates
|
||||
.filter((candidate) => candidate.nodeId !== selected?.nodeId)
|
||||
.map((candidate) => ({
|
||||
nodeId: candidate.nodeId,
|
||||
label: candidate.label,
|
||||
score: candidate.score,
|
||||
downstreamCount: candidate.downstreamCount,
|
||||
unresolvedParentUnknownCount: candidate.unresolvedParentUnknownCount,
|
||||
matches: candidate.matches,
|
||||
contributions: candidate.contributions,
|
||||
outrankedBy: {
|
||||
scoreDelta: (selected?.score ?? candidate.score) - candidate.score,
|
||||
downstreamDelta:
|
||||
(selected?.downstreamCount ?? candidate.downstreamCount) -
|
||||
candidate.downstreamCount,
|
||||
unresolvedPrerequisiteDelta:
|
||||
candidate.unresolvedParentUnknownCount -
|
||||
(selected?.unresolvedParentUnknownCount ??
|
||||
candidate.unresolvedParentUnknownCount),
|
||||
labelOrderWinner:
|
||||
selected &&
|
||||
selected.score === candidate.score &&
|
||||
selected.downstreamCount === candidate.downstreamCount &&
|
||||
selected.unresolvedParentUnknownCount ===
|
||||
candidate.unresolvedParentUnknownCount
|
||||
? selected.label.localeCompare(candidate.label) <= 0
|
||||
? selected.label
|
||||
: candidate.label
|
||||
: null,
|
||||
},
|
||||
}));
|
||||
|
||||
return {
|
||||
selectedNodeId: selected?.nodeId ?? null,
|
||||
selectedNodeLabel: selected?.label ?? null,
|
||||
status: selection.status,
|
||||
tieType: selection.tieType,
|
||||
resolvedNodeIds: [...resolvedNodeIds],
|
||||
tiedCandidateIds: selection.leadingCandidates.map(
|
||||
(candidate) => candidate.nodeId,
|
||||
),
|
||||
tieBreakOrder: [
|
||||
"score_desc",
|
||||
"downstreamCount_desc",
|
||||
"unresolvedParentUnknownCount_asc",
|
||||
"label_asc",
|
||||
],
|
||||
alphabeticalUsedAsReasoning: false,
|
||||
candidates: orderedCandidates,
|
||||
selected: selected
|
||||
? {
|
||||
nodeId: selected.nodeId,
|
||||
label: selected.label,
|
||||
score: selected.score,
|
||||
downstreamCount: selected.downstreamCount,
|
||||
unresolvedParentUnknownCount: selected.unresolvedParentUnknownCount,
|
||||
matches: selected.matches,
|
||||
contributions: selected.contributions,
|
||||
}
|
||||
: null,
|
||||
competitors,
|
||||
summary: {
|
||||
candidateCount: orderedCandidates.length,
|
||||
selectedReason: selected
|
||||
? `highest_score=${selected.score}; downstream=${selected.downstreamCount}; unresolved_prerequisites=${selected.unresolvedParentUnknownCount}`
|
||||
: selection.reason,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ── Apply a graph update deterministically ──
|
||||
|
||||
export function applyGraphUpdate(graph, update) {
|
||||
const errors = [];
|
||||
const updatedNodesMap = new Map();
|
||||
|
||||
// Validate that update references existing nodes or newly added ones
|
||||
const allNodeIds = new Set(graph.nodes.map((n) => n.id));
|
||||
for (const added of update.addedNodes) {
|
||||
if (allNodeIds.has(added.id)) {
|
||||
errors.push(`Cannot add node with duplicate ID: "${added.id}"`);
|
||||
continue;
|
||||
}
|
||||
allNodeIds.add(added.id);
|
||||
}
|
||||
|
||||
// Validate updated nodes exist
|
||||
for (const upd of update.updatedNodes) {
|
||||
if (!allNodeIds.has(upd.nodeId)) {
|
||||
errors.push(`Cannot update non-existent node: "${upd.nodeId}"`);
|
||||
}
|
||||
}
|
||||
|
||||
// Validate added edges reference existing or new nodes
|
||||
for (const edge of update.addedEdges) {
|
||||
if (!allNodeIds.has(edge.fromNodeId)) {
|
||||
errors.push(
|
||||
`Added edge references non-existent fromNodeId: "${edge.fromNodeId}"`,
|
||||
);
|
||||
}
|
||||
if (!allNodeIds.has(edge.toNodeId)) {
|
||||
errors.push(
|
||||
`Added edge references non-existent toNodeId: "${edge.toNodeId}"`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (errors.length > 0) return { success: false, errors };
|
||||
|
||||
// Build the new nodes list — start with a deep copy of existing
|
||||
const newNodes = graph.nodes.map((n) => ({ ...n }));
|
||||
|
||||
// Apply updated nodes
|
||||
for (const upd of update.updatedNodes) {
|
||||
const idx = newNodes.findIndex((n) => n.id === upd.nodeId);
|
||||
if (idx === -1) continue; // already validated above
|
||||
|
||||
if (upd.newStatus !== undefined && upd.newStatus !== null) {
|
||||
newNodes[idx].status = upd.newStatus;
|
||||
}
|
||||
if (upd.newValue !== undefined) {
|
||||
newNodes[idx].value = upd.newValue;
|
||||
}
|
||||
updatedNodesMap.set(upd.nodeId, newNodes[idx]);
|
||||
}
|
||||
|
||||
// Add new nodes
|
||||
for (const newNode of update.addedNodes) {
|
||||
if (!allNodeIds.has(newNode.id)) continue;
|
||||
allNodeIds.add(newNode.id);
|
||||
newNodes.push({ ...newNode });
|
||||
}
|
||||
|
||||
// Remove edges if requested
|
||||
const removedEdgeSet = new Set(update.removedEdgeIds);
|
||||
const newEdges = graph.edges.filter((e) => !removedEdgeSet.has(e.id));
|
||||
|
||||
// Add new edges
|
||||
for (const newEdge of update.addedEdges) {
|
||||
newEdges.push({ ...newEdge });
|
||||
|
||||
// Update dependsOn / affects on the nodes
|
||||
const fromNode = newNodes.find((n) => n.id === newEdge.fromNodeId);
|
||||
const toNode = newNodes.find((n) => n.id === newEdge.toNodeId);
|
||||
if (fromNode && !fromNode.childIds.includes(newEdge.toNodeId)) {
|
||||
fromNode.childIds.push(newEdge.toNodeId);
|
||||
}
|
||||
if (toNode && !toNode.dependsOn.includes(newEdge.fromNodeId)) {
|
||||
toNode.dependsOn.push(newEdge.fromNodeId);
|
||||
}
|
||||
}
|
||||
|
||||
// Add resolved node IDs
|
||||
const newResolved = [
|
||||
...new Set([...graph.resolvedNodeIds, ...update.resolvedUnknownNodeIds]),
|
||||
];
|
||||
|
||||
return {
|
||||
success: true,
|
||||
nodes: newNodes,
|
||||
edges: newEdges,
|
||||
resolvedNodeIds: newResolved,
|
||||
};
|
||||
}
|
||||
|
||||
// ── Validate a proposed graph update before application ──
|
||||
|
||||
export function validateGraphUpdate(graph, update) {
|
||||
const errors = [];
|
||||
|
||||
// Check for duplicate node IDs against existing and newly added nodes
|
||||
const extendedIds = new Set(graph.nodes.map((n) => n.id));
|
||||
for (const newNode of update.addedNodes) {
|
||||
if (extendedIds.has(newNode.id)) {
|
||||
errors.push(`Cannot add node with duplicate ID: "${newNode.id}"`);
|
||||
} else {
|
||||
extendedIds.add(newNode.id);
|
||||
}
|
||||
}
|
||||
|
||||
// Check updated nodes exist (in original graph, not newly added ones)
|
||||
const existingIds = new Set(graph.nodes.map((n) => n.id));
|
||||
for (const upd of update.updatedNodes) {
|
||||
if (!existingIds.has(upd.nodeId)) {
|
||||
errors.push(`Cannot update non-existent node: "${upd.nodeId}"`);
|
||||
}
|
||||
}
|
||||
|
||||
// Reject updates with no meaningful change
|
||||
const statusChanged = update.updatedNodes.some(
|
||||
(u) => u.previousStatus !== null && u.newStatus !== u.previousStatus,
|
||||
);
|
||||
const valueChanged = update.updatedNodes.some(
|
||||
(u) => u.previousValue !== null && u.newValue !== u.previousValue,
|
||||
);
|
||||
|
||||
const hasMeaningfulChange =
|
||||
update.addedNodes.length > 0 ||
|
||||
statusChanged ||
|
||||
valueChanged ||
|
||||
update.addedEdges.length > 0 ||
|
||||
update.removedEdgeIds.length > 0;
|
||||
|
||||
if (!hasMeaningfulChange) {
|
||||
errors.push("Update contains no meaningful change");
|
||||
}
|
||||
|
||||
// Reject oversized input
|
||||
const totalSize = JSON.stringify(update).length;
|
||||
if (totalSize > 100000) {
|
||||
errors.push(`Proposed graph update exceeds 100KB (${totalSize} bytes)`);
|
||||
}
|
||||
|
||||
return { valid: errors.length === 0, errors };
|
||||
}
|
||||
@@ -1,40 +0,0 @@
|
||||
function cloneJsonSafe(value) {
|
||||
if (value == null) return value;
|
||||
return JSON.parse(JSON.stringify(value));
|
||||
}
|
||||
|
||||
export function normaliseAnalysisResponse(input) {
|
||||
const normalised = cloneJsonSafe(input);
|
||||
const changesApplied = [];
|
||||
const warnings = [];
|
||||
|
||||
if (!normalised || typeof normalised !== "object") {
|
||||
return { normalised: input, changesApplied, warnings };
|
||||
}
|
||||
|
||||
if (Array.isArray(normalised.evidence)) {
|
||||
normalised.evidence = normalised.evidence.map((record, index) => {
|
||||
if (!record || typeof record !== "object") return record;
|
||||
|
||||
if (record.source === null) {
|
||||
changesApplied.push({
|
||||
path: ["evidence", index, "source"],
|
||||
change: "Converted null source to undefined",
|
||||
});
|
||||
|
||||
const { source: _removed, ...rest } = record;
|
||||
return rest;
|
||||
}
|
||||
|
||||
return record;
|
||||
});
|
||||
}
|
||||
|
||||
if (changesApplied.length > 0) {
|
||||
warnings.push(
|
||||
"Applied deterministic reconstruction compatibility normalisation",
|
||||
);
|
||||
}
|
||||
|
||||
return { normalised, changesApplied, warnings };
|
||||
}
|
||||
@@ -7,14 +7,7 @@ const __dirname = dirname(__filename);
|
||||
const PROMPTS_DIR = join(__dirname, "../../prompts");
|
||||
|
||||
/** Available prompt versions */
|
||||
export const PROMPT_VERSIONS = ["v0.1", "v0.2", "v0.3"];
|
||||
|
||||
/** Default prompt version (override via RECONSTRUCTION_PROMPT_VERSION env var) */
|
||||
const defaultVersionFromEnv = process.env.RECONSTRUCTION_PROMPT_VERSION;
|
||||
export const DEFAULT_PROMPT_VERSION =
|
||||
defaultVersionFromEnv && PROMPT_VERSIONS.includes(defaultVersionFromEnv)
|
||||
? defaultVersionFromEnv
|
||||
: "v0.3";
|
||||
export const PROMPT_VERSIONS = ["v0.1", "v0.2"];
|
||||
|
||||
/** Build a v0.1 (extraction-only) prompt inline for backward compatibility */
|
||||
function buildV1Prompt(scenario) {
|
||||
@@ -52,10 +45,7 @@ Return ONLY the JSON object. No markdown, no explanation, no preamble.`;
|
||||
/** Load a versioned prompt from disk and substitute {{SCENARIO}} */
|
||||
async function buildV2Prompt(scenario) {
|
||||
try {
|
||||
const content = await fs.readFile(
|
||||
join(PROMPTS_DIR, "reconstruct-v0.2.md"),
|
||||
"utf-8",
|
||||
);
|
||||
const content = await fs.readFile(join(PROMPTS_DIR, "reconstruct-v0.2.md"), "utf-8");
|
||||
return content.replace("{{SCENARIO}}", scenario);
|
||||
} catch {
|
||||
// Fall back to v0.1 prompt if v0.2 file is missing
|
||||
@@ -63,40 +53,22 @@ async function buildV2Prompt(scenario) {
|
||||
}
|
||||
}
|
||||
|
||||
/** Load a versioned prompt from disk and substitute {{SCENARIO}} */
|
||||
async function buildV3Prompt(scenario) {
|
||||
try {
|
||||
const content = await fs.readFile(
|
||||
join(PROMPTS_DIR, "reconstruct-v0.3.md"),
|
||||
"utf-8",
|
||||
);
|
||||
return content.replace("{{SCENARIO}}", scenario);
|
||||
} catch {
|
||||
// Fall back to v0.2 prompt if v0.3 file is missing
|
||||
return buildV2Prompt(scenario);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build an analysis prompt for the given version.
|
||||
* @param {"v0.1" | "v0.2" | "v0.3"} [version="v0.3"]
|
||||
* @param {"v0.1" | "v0.2"} [version="v0.2"]
|
||||
* @returns {Promise<{prompt: string, version: string}>}
|
||||
*/
|
||||
export async function buildPrompt(scenario, version = "v0.3") {
|
||||
export async function buildPrompt(scenario, version = "v0.2") {
|
||||
let prompt;
|
||||
switch (version) {
|
||||
case "v0.1":
|
||||
prompt = buildV1Prompt(scenario);
|
||||
break;
|
||||
case "v0.2":
|
||||
default: // v0.2
|
||||
prompt = await buildV2Prompt(scenario);
|
||||
break;
|
||||
default: // v0.3
|
||||
prompt = await buildV3Prompt(scenario);
|
||||
break;
|
||||
}
|
||||
|
||||
const strongJsonHint =
|
||||
"\n\nReturn ONLY a valid JSON object starting with { and ending with }. Do NOT include any text before the opening brace or after the closing brace. Do NOT wrap in markdown backticks.";
|
||||
const strongJsonHint = "\n\nReturn ONLY a valid JSON object starting with { and ending with }. Do NOT include any text before the opening brace or after the closing brace. Do NOT wrap in markdown backticks.";
|
||||
return { prompt: prompt + strongJsonHint, version };
|
||||
}
|
||||
|
||||
@@ -5,12 +5,7 @@ import { z } from "zod";
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
export const confidenceEnum = z.enum(["low", "medium", "high"]);
|
||||
const importanceEnum = z.enum([
|
||||
"incidental",
|
||||
"supporting",
|
||||
"important",
|
||||
"critical",
|
||||
]);
|
||||
const importanceEnum = z.enum(["incidental", "supporting", "important", "critical"]);
|
||||
const expectedInfoValueEnum = z.enum(["low", "medium", "high"]);
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
@@ -29,11 +24,8 @@ export const reconstructionSchema = z.object({
|
||||
observations: z.array(itemSchemaV1),
|
||||
reportedClaims: z.array(
|
||||
itemSchemaV1.extend({
|
||||
attributedTo: z
|
||||
.union([z.string().min(1), z.null()])
|
||||
.optional()
|
||||
.nullable(),
|
||||
}),
|
||||
attributedTo: z.union([z.string().min(1), z.null()]).optional().nullable(),
|
||||
})
|
||||
),
|
||||
assumptions: z.array(itemSchemaV1),
|
||||
entities: z.array(itemSchemaV1),
|
||||
@@ -43,7 +35,7 @@ export const reconstructionSchema = z.object({
|
||||
previousState: z.string().min(1),
|
||||
currentState: z.string().min(1),
|
||||
explanationStatus: z.string().min(1),
|
||||
}),
|
||||
})
|
||||
),
|
||||
expectedButMissing: z.array(itemSchemaV1),
|
||||
presentButUnexpected: z.array(itemSchemaV1),
|
||||
@@ -73,53 +65,45 @@ export const healthResponseSchema = z.object({
|
||||
// v0.2 — reasoning classification + reconstruction
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
export const inputTypes =
|
||||
/** @type {z.ZodType<typeof import("@/lib/reconstruction/schema").INPUT_TYPE_VALUE>} */ (
|
||||
z.enum([
|
||||
"observed_problem",
|
||||
"unexplained_change",
|
||||
"contradiction",
|
||||
"decision_request",
|
||||
"causal_claim",
|
||||
"reported_claim",
|
||||
"fault_report",
|
||||
"ambiguous_statement",
|
||||
"question",
|
||||
"desired_outcome",
|
||||
"insufficient_context",
|
||||
"other",
|
||||
])
|
||||
);
|
||||
export const inputTypes = /** @type {z.ZodType<typeof import("@/lib/reconstruction/schema").INPUT_TYPE_VALUE>} */ (
|
||||
z.enum([
|
||||
"observed_problem",
|
||||
"unexplained_change",
|
||||
"contradiction",
|
||||
"decision_request",
|
||||
"causal_claim",
|
||||
"reported_claim",
|
||||
"fault_report",
|
||||
"ambiguous_statement",
|
||||
"question",
|
||||
"desired_outcome",
|
||||
"insufficient_context",
|
||||
"other",
|
||||
])
|
||||
);
|
||||
|
||||
export const reasoningModes =
|
||||
/** @type {z.ZodType<typeof import("@/lib/reconstruction/schema").REASONING_MODE_VALUE>} */ (
|
||||
z.enum([
|
||||
"establish_baseline",
|
||||
"identify_difference",
|
||||
"reconstruct_transition",
|
||||
"decompose_aggregate",
|
||||
"validate_measurement",
|
||||
"validate_claim",
|
||||
"investigate_contradiction",
|
||||
"clarify_meaning",
|
||||
"decision_support",
|
||||
"fault_investigation",
|
||||
"identify_missing_information",
|
||||
"test_possible_explanations",
|
||||
"other",
|
||||
])
|
||||
);
|
||||
export const reasoningModes = /** @type {z.ZodType<typeof import("@/lib/reconstruction/schema").REASONING_MODE_VALUE>} */ (
|
||||
z.enum([
|
||||
"establish_baseline",
|
||||
"identify_difference",
|
||||
"reconstruct_transition",
|
||||
"decompose_aggregate",
|
||||
"validate_measurement",
|
||||
"validate_claim",
|
||||
"investigate_contradiction",
|
||||
"clarify_meaning",
|
||||
"decision_support",
|
||||
"fault_investigation",
|
||||
"identify_missing_information",
|
||||
"test_possible_explanations",
|
||||
"other",
|
||||
])
|
||||
);
|
||||
|
||||
const evidenceRecordSchema = z.object({
|
||||
id: z.string().min(1),
|
||||
description: z.string().min(1),
|
||||
evidenceType: z.enum([
|
||||
"direct_observation",
|
||||
"reported_statement",
|
||||
"interpretation",
|
||||
"assumption",
|
||||
"inferred_relationship",
|
||||
]),
|
||||
evidenceType: z.enum(["direct_observation", "reported_statement", "interpretation", "assumption", "inferred_relationship"]),
|
||||
source: z.string().optional(),
|
||||
attribution: z.string().nullable().optional(),
|
||||
confidence: confidenceEnum,
|
||||
@@ -139,14 +123,14 @@ const reconstructionSchemaV2 = z.object({
|
||||
previousState: z.string().min(1),
|
||||
currentState: z.string().min(1),
|
||||
explanationStatus: z.string().min(1),
|
||||
}),
|
||||
})
|
||||
),
|
||||
unexplainedTransitions: z.array(
|
||||
itemSchemaV1.extend({
|
||||
entity: z.string().min(1).optional(),
|
||||
previousState: z.string().min(1).optional(),
|
||||
currentState: z.string().min(1).optional(),
|
||||
}),
|
||||
})
|
||||
),
|
||||
contradictions: z.array(itemSchemaV1),
|
||||
importantUnknowns: z.array(itemSchemaV1),
|
||||
@@ -157,7 +141,7 @@ const reconstructionSchemaV2 = z.object({
|
||||
supportingEvidenceIds: z.array(z.string()),
|
||||
assumptionsRequired: z.array(z.string()).optional().default([]),
|
||||
confidence: confidenceEnum,
|
||||
}),
|
||||
})
|
||||
),
|
||||
});
|
||||
|
||||
|
||||
Generated
+2
-66
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "confidence-engine",
|
||||
"version": "0.2.0-experimental",
|
||||
"version": "0.1.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "confidence-engine",
|
||||
"version": "0.2.0-experimental",
|
||||
"version": "0.1.0",
|
||||
"dependencies": {
|
||||
"next": "^14.2.0",
|
||||
"react": "^18.3.0",
|
||||
@@ -14,7 +14,6 @@
|
||||
"zod": "^3.23.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@playwright/test": "^1.62.1",
|
||||
"@types/node": "^20.14.0",
|
||||
"@types/react": "^18.3.0",
|
||||
"@types/react-dom": "^18.3.0",
|
||||
@@ -889,22 +888,6 @@
|
||||
"node": ">=14"
|
||||
}
|
||||
},
|
||||
"node_modules/@playwright/test": {
|
||||
"version": "1.62.1",
|
||||
"resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.62.1.tgz",
|
||||
"integrity": "sha512-DTcUc8qii+cpHvtOwggMtBRMjKZHXYWdw8syRYu2vtzuq4Wxphqq4NfCs5Zt44L6mA8rfDfj+PHnxFc/FeK6mQ==",
|
||||
"devOptional": true,
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"playwright": "1.62.1"
|
||||
},
|
||||
"bin": {
|
||||
"playwright": "cli.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
}
|
||||
},
|
||||
"node_modules/@rollup/rollup-android-arm-eabi": {
|
||||
"version": "4.62.3",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.62.3.tgz",
|
||||
@@ -5618,53 +5601,6 @@
|
||||
"node": ">= 6"
|
||||
}
|
||||
},
|
||||
"node_modules/playwright": {
|
||||
"version": "1.62.1",
|
||||
"resolved": "https://registry.npmjs.org/playwright/-/playwright-1.62.1.tgz",
|
||||
"integrity": "sha512-0M+L3LAD8/nm554LOla9Ayx0j0tmFZ0FBcoQ7F1VuVHpM/XpiC8RcDzBQB8W5+hA8L22THxELzeF+2WcUzvcLg==",
|
||||
"devOptional": true,
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"playwright-core": "1.62.1"
|
||||
},
|
||||
"bin": {
|
||||
"playwright": "cli.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"fsevents": "2.3.2"
|
||||
}
|
||||
},
|
||||
"node_modules/playwright-core": {
|
||||
"version": "1.62.1",
|
||||
"resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.62.1.tgz",
|
||||
"integrity": "sha512-wPYSwEBJY9GHraISXqyqtx0na0LpO3XEX7jNDhntbex7tzUS7kLnZsOlFruFJB4Hi/rhDMjXGqHewDZ68nYZVw==",
|
||||
"devOptional": true,
|
||||
"license": "Apache-2.0",
|
||||
"bin": {
|
||||
"playwright-core": "cli.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
}
|
||||
},
|
||||
"node_modules/playwright/node_modules/fsevents": {
|
||||
"version": "2.3.2",
|
||||
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
|
||||
"integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
|
||||
"dev": true,
|
||||
"hasInstallScript": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
],
|
||||
"engines": {
|
||||
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/possible-typed-array-names": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz",
|
||||
|
||||
+5
-2
@@ -10,7 +10,11 @@
|
||||
"start": "next start",
|
||||
"lint": "next lint",
|
||||
"test": "vitest run",
|
||||
"test:watch": "vitest"
|
||||
"test:watch": "vitest",
|
||||
"evaluate": "node tests/evaluator.mjs",
|
||||
"evaluate:mock": "EVAL_REAL=0 node tests/evaluator.mjs",
|
||||
"evaluate:diagnostic": "EVAL_DIAGNOSTIC=1 EVAL_REAL=0 node tests/evaluator.mjs",
|
||||
"evaluate:live": "EVAL_REAL=1 node tests/evaluator.mjs"
|
||||
},
|
||||
"dependencies": {
|
||||
"next": "^14.2.0",
|
||||
@@ -19,7 +23,6 @@
|
||||
"zod": "^3.23.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@playwright/test": "^1.62.1",
|
||||
"@types/node": "^20.14.0",
|
||||
"@types/react": "^18.3.0",
|
||||
"@types/react-dom": "^18.3.0",
|
||||
|
||||
@@ -1,5 +0,0 @@
|
||||
import { defineConfig } from "@playwright/test";
|
||||
export default defineConfig({
|
||||
use: { headless: true, screenshot: "only-on-failure", actionTimeout: 120000 },
|
||||
testMatch: "**/tests/smoke.test.js",
|
||||
});
|
||||
@@ -0,0 +1,67 @@
|
||||
# v0.1 vs v0.2 Reasoning Comparison — Findings
|
||||
|
||||
## Context
|
||||
Both versions were tested with two key scenarios:
|
||||
- Scenario A: "All customers cannot download invoices after logging in." (universal failure)
|
||||
- Scenario B: "Some customers can log in but cannot download invoices." (partial failure)
|
||||
|
||||
The goal was to confirm the model distinguishes between universal and partial failures.
|
||||
|
||||
## Results — v0.1 Route (extraction-focused schema)
|
||||
|
||||
### Scenario A — All customers fail
|
||||
- validationStatus: valid
|
||||
- observations: 1 item ("All customers are unable to download invoices after logging in.")
|
||||
- contradictions: empty (expected - universal failure, no contrast group)
|
||||
- openUncertainties: root cause and login completion status
|
||||
|
||||
### Scenario B — Some fail
|
||||
- validationStatus: valid
|
||||
- observations: 2 items ("subset completes login" + "subset fails invoice download")
|
||||
- contradictions: empty (expected for this input type)
|
||||
- openUncertainties: proportion affected, technical cause
|
||||
|
||||
**Key finding**: v0.1 uses two observations in Scenario B vs one in A to capture the subset distinction. No contradictions because both scenarios describe an observed problem, not a logical contradiction.
|
||||
|
||||
## Results — v0.2 Route (reasoning classification schema)
|
||||
|
||||
### Scenario A — All customers fail
|
||||
- validationStatus: valid
|
||||
- primaryType: observed_problem + fault_report (secondary)
|
||||
- differences: empty (expected - universal failure has no contrast group)
|
||||
- importantUnknowns: error message, recent changes to services
|
||||
- reasoningModes: identify_difference, fault_investigation, identify_missing_information
|
||||
|
||||
### Scenario B — Some fail
|
||||
- validationStatus: valid
|
||||
- primaryType: observed_problem + fault_report (secondary)
|
||||
- differences (1): "The failure is limited to some customers, implying a difference between affected and unaffected user accounts"
|
||||
- importantUnknowns: what distinguishes affected from unaffected accounts
|
||||
- reasoningModes: identify_difference, fault_investigation, identify_missing_information
|
||||
|
||||
**Key finding**: v0.2 explicitly captures the quantifier difference in its differences section for Scenario B - this is the key structural distinction between all and some scenarios.
|
||||
|
||||
## Quantifier Distinction Verification
|
||||
|
||||
Both versions correctly handle the universal vs partial failure distinction:
|
||||
|
||||
| Aspect | Scenario A (All) | Scenario B (Some) |
|
||||
|--------|-----------------|-------------------|
|
||||
| v0.1 observations | 1 (universal) | 2 (login OK + download fail) |
|
||||
| v0.1 contradictions | 0 (expected) | 0 (expected) |
|
||||
| v0.2 primaryType | observed_problem | observed_problem |
|
||||
| v0.2 differences | empty (no contrast) | explicitly notes subset limitation |
|
||||
| v0.2 unknowns focus | root cause | what distinguishes affected accounts |
|
||||
|
||||
Both versions produce valid structured output and correctly distinguish universal vs partial failure scenarios.
|
||||
|
||||
## Prompt Fix Summary
|
||||
|
||||
The v0.2 prompt template (prompts/reconstruct-v0.2.md) was updated to include an explicit JSON output schema section that:
|
||||
1. Specifies exact camelCase key names matching the Zod schema
|
||||
2. Lists all valid enum values for primaryType and reasoningModes
|
||||
3. Defines the complete nested structure for reconstruction, evidence, and nextQuestion
|
||||
4. Includes critical rules preventing snake_case keys or invented top-level fields
|
||||
|
||||
Before fix: Model output had input_classification, reasoning_mode, anchors - all invalid per Zod schema -> validationStatus: invalid
|
||||
After fix: Model output has inputClassification, reconstruction, evidence, nextQuestion with correct nested structure -> validationStatus: valid
|
||||
@@ -1,160 +0,0 @@
|
||||
You are a neutral analyst performing evidence-based situation reconstruction.
|
||||
|
||||
## Rules
|
||||
|
||||
1. Do NOT invent facts, context or causes. Only include information present in the scenario or clearly implied.
|
||||
2. First determine what kind of input has been supplied. Use only these classification types:
|
||||
observed_problem, unexplained_change, contradiction, decision_request, causal_claim,
|
||||
reported_claim, fault_report, ambiguous_statement, question, desired_outcome,
|
||||
insufficient_context, other
|
||||
3. Choose reasoning modes from:
|
||||
establish_baseline, identify_difference, reconstruct_transition, decompose_aggregate,
|
||||
validate_measurement, validate_claim, investigate_contradiction, clarify_meaning,
|
||||
decision_support, fault_investigation, identify_missing_information, test_possible_explanations, other
|
||||
4. Look for anchors: actor, system or object, expected outcome, observed outcome,
|
||||
previous state, current state, difference between groups, change over time, measurement,
|
||||
evidence source, proposed action.
|
||||
5. Identify meaningful differences (e.g., some succeed while others fail; revenue rises while cash falls).
|
||||
6. Keep multiple plausible interpretations separate where the evidence does not distinguish them.
|
||||
7. Distinguish: what was said / what it may mean / why it may have been said.
|
||||
8. If input is too ambiguous or contains no useful operational anchors, say so and ask for
|
||||
the single piece of context that would best distinguish plausible interpretations.
|
||||
|
||||
## Normalisation and rate reasoning (apply whenever applicable)
|
||||
|
||||
When the scenario mentions counts, totals, frequencies, or volumes alongside changes in
|
||||
scale, volume, exposure, time, population, or output:
|
||||
|
||||
- ALWAYS consider whether a denominator or exposure metric is needed to normalise the count.
|
||||
- Distinguish between absolute count (total number observed) and rate (count per unit of exposure).
|
||||
- Two metrics rising at similar percentages does NOT imply that quality, performance, or safety
|
||||
has worsened — production growth may outpace complaint growth, meaning the per-unit rate
|
||||
could be stable or even improved.
|
||||
- Identify the possible denominator explicitly (e.g., "per unit produced", "per customer served",
|
||||
"per hour of operation").
|
||||
- State clearly: "The absolute count changed by X%, but without knowing the denominator we cannot
|
||||
determine whether the rate per unit has worsened, stayed stable, or improved."
|
||||
- Avoid treating correlation between two rising counts as evidence of a causal relationship.
|
||||
|
||||
## Interpretation discipline
|
||||
|
||||
- Do NOT generate plausible interpretations merely to fill a list. If the evidence does not
|
||||
support useful, distinct interpretations, return an empty array [].
|
||||
- Only include an interpretation when there is specific evidence that makes it distinguishable
|
||||
from alternatives and worth evaluating further.
|
||||
- Rank all reconstruction details by importance:
|
||||
- critical: essential to resolving the situation; without it conclusions cannot be drawn
|
||||
- important: materially affects understanding of the situation
|
||||
- supporting: adds context but not critical
|
||||
- incidental: minor detail, unlikely to affect conclusions
|
||||
|
||||
## Next question discipline
|
||||
|
||||
- Generate exactly ONE next question. Do NOT combine multiple questions.
|
||||
- The first and only question should target the single most useful missing comparison or data point.
|
||||
- Prefer narrow, specific questions over broad compound questions.
|
||||
- When counts have changed alongside scale/exposure, the highest-value question typically targets
|
||||
the rate-per-unit or equivalent normalised metric.
|
||||
- Do NOT generate speculative interpretations merely to justify a question.
|
||||
|
||||
## Confidence scale
|
||||
|
||||
- low — weak evidence, speculation, or missing information
|
||||
- medium — reasonable inference from available evidence
|
||||
- high — strong evidence, direct observation, or confirmed fact
|
||||
|
||||
## Importance scale (evidence records)
|
||||
|
||||
- incidental — minor detail, unlikely to affect conclusions
|
||||
- supporting — adds context but not critical
|
||||
- important — materially affects understanding of the situation
|
||||
- critical — essential to resolving the situation; without it conclusions cannot be drawn
|
||||
|
||||
## Expected information value (next question)
|
||||
|
||||
- low — marginally useful even if answered
|
||||
- medium — meaningfully clarifies the situation
|
||||
- high — would significantly distinguish between plausible explanations or fill a gap in understanding
|
||||
|
||||
## Next question selection criteria
|
||||
|
||||
Prefer questions that:
|
||||
- clarify a major difference
|
||||
- establish a baseline
|
||||
- explain an important transition
|
||||
- test an unsupported claim
|
||||
- distinguish between plausible explanations
|
||||
- request measurable evidence
|
||||
- identify who or what is affected
|
||||
- establish timing
|
||||
|
||||
Avoid questions that:
|
||||
- have already been answered
|
||||
- assume a cause
|
||||
- jump to a solution
|
||||
- ask about motive before the observable situation is understood
|
||||
- focus on incidental wording
|
||||
- are too broad to produce useful information
|
||||
- combine many unrelated questions
|
||||
|
||||
## Output format — return this exact JSON structure
|
||||
|
||||
Return a JSON object with exactly these four top-level keys (use **camelCase**):
|
||||
|
||||
```json
|
||||
{
|
||||
"inputClassification": {
|
||||
"primaryType": "<one of: observed_problem, unexplained_change, contradiction, decision_request, causal_claim, reported_claim, fault_report, ambiguous_statement, question, desired_outcome, insufficient_context, other>",
|
||||
"secondaryTypes": ["<optional additional types from the same list>"],
|
||||
"reasoningModes": ["<one or more of: establish_baseline, identify_difference, reconstruct_transition, decompose_aggregate, validate_measurement, validate_claim, investigate_contradiction, clarify_meaning, decision_support, fault_investigation, identify_missing_information, test_possible_explanations, other>"],
|
||||
"classificationReason": "<brief explanation of why you chose the primary type>",
|
||||
"confidence": "<low | medium | high>"
|
||||
},
|
||||
"reconstruction": {
|
||||
"summary": "<one-sentence overview of the situation>",
|
||||
"actors": [{"id": "<any unique string>", "description": "...", "confidence": "<low|medium|high>"}],
|
||||
"systemsOrObjects": [{"id": "<any unique string>", "description": "...", "confidence": "<low|medium|high>"}],
|
||||
"expectedStates": [{"id": "...", "description": "...", "confidence": "<low|medium|high>"}],
|
||||
"observedStates": [{"id": "...", "description": "...", "confidence": "<low|medium|high>"}],
|
||||
"differences": [{"id": "...", "description": "...", "confidence": "<low|medium|high>"}],
|
||||
"knownTransitions": [{"id": "...", "description": "...", "confidence": "<low|medium|high>", "entity": "...", "previousState": "...", "currentState": "...", "explanationStatus": "..."}],
|
||||
"unexplainedTransitions": [{"id": "...", "description": "...", "confidence": "<low|medium|high>", "entity": "...", "previousState": "...", "currentState": "..."}],
|
||||
"contradictions": [{"id": "...", "description": "...", "confidence": "<low|medium|high>"}],
|
||||
"importantUnknowns": [{"id": "...", "description": "...", "confidence": "<low|medium|high>"}],
|
||||
"plausibleInterpretations": [{"id": "...", "description": "...", "supportingEvidenceIds": ["<ids that support this interpretation>"], "assumptionsRequired": [], "confidence": "<low|medium|high>"}]
|
||||
},
|
||||
"evidence": [
|
||||
{
|
||||
"id": "<any unique string>",
|
||||
"description": "...",
|
||||
"evidenceType": "<direct_observation | reported_statement | interpretation | assumption | inferred_relationship>",
|
||||
"source": "<optional — who/where this came from>",
|
||||
"attribution": null,
|
||||
"confidence": "<low | medium | high>",
|
||||
"importance": "<incidental | supporting | important | critical>"
|
||||
}
|
||||
],
|
||||
"nextQuestion": {
|
||||
"id": "<any unique string>",
|
||||
"question": "<one precise question>",
|
||||
"targets": ["<what this question targets — e.g. 'actor', 'system', 'expectedOutcome'>"],
|
||||
"reason": "<why answering this is important>",
|
||||
"expectedInformationValue": "<low | medium | high>",
|
||||
"reasoningMode": "<optional reasoning mode from the list above>"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
CRITICAL RULES for JSON output:
|
||||
1. Use **exactly** the key names shown above (camelCase, no snake_case).
|
||||
2. The four top-level keys must be: `inputClassification`, `reconstruction`, `evidence`, `nextQuestion`.
|
||||
3. Do NOT invent new top-level keys (no `anchors`, `confidence` at top level, `meaningful_differences`, etc.).
|
||||
4. Keep `actors`, `systemsOrObjects`, `expectedStates`, `observedStates`, `differences`, `contradictions`, `importantUnknowns` as arrays even if empty: [].
|
||||
5. Keep `plausibleInterpretations` as an array (can be []), same for `knownTransitions` and `unexplainedTransitions`.
|
||||
6. Each object in arrays must have at least `id`, `description`, `confidence`.
|
||||
7. **evidenceType**: classify each evidence item clearly as either a direct observation, a reported statement, an interpretation, an assumption, or an inferred relationship. Do not treat raw counts as proof of causal relationships — they may be inferred relationships only when supported by explicit reasoning about denominators or rates.
|
||||
|
||||
Scenario:
|
||||
{{SCENARIO}}
|
||||
|
||||
Return ONLY the JSON object starting with { and ending with }. Do NOT include any text before the opening brace or after the closing brace. Do NOT wrap in markdown backticks.
|
||||
@@ -0,0 +1,218 @@
|
||||
/**
|
||||
* Debug script: send raw Ollama requests directly, bypassing the application provider.
|
||||
* Tests /api/chat with format:json and captures request payloads + raw responses.
|
||||
*/
|
||||
import { mkdirSync, writeFileSync } from "node:fs";
|
||||
import { join, dirname } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
const BASE_URL = process.env.OLLAMA_BASE_URL || "http://localhost:11434";
|
||||
const TIMESTAMP = new Date().toISOString().replace(/[/:]/g, "-");
|
||||
const RESULTS_DIR = join(__dirname, "..", "provider-debug-results", TIMESTAMP);
|
||||
|
||||
mkdirSync(RESULTS_DIR, { recursive: true });
|
||||
|
||||
// ============================================================
|
||||
// Test cases
|
||||
// ============================================================
|
||||
|
||||
const MODEL_A = "qwen-claude:latest";
|
||||
const MODEL_B = "qwen3.6:35b-a3b";
|
||||
|
||||
function getModelList() {
|
||||
// Check which models are available locally (not via Ollama server)
|
||||
return { A: MODEL_A, B: MODEL_B };
|
||||
}
|
||||
|
||||
// Test A: Simple text reply to verify model responds normally
|
||||
const TEST_A = {
|
||||
label: "A",
|
||||
description: "Plain instruction test — should return CHAT_WORKS",
|
||||
system: "You are a normal assistant. Follow the user instruction exactly.",
|
||||
user: "Reply with exactly: CHAT_WORKS",
|
||||
};
|
||||
|
||||
// Test B: Explicit JSON schema via format field
|
||||
const TEST_B = {
|
||||
label: "B",
|
||||
description: "JSON schema test — should return exact object",
|
||||
system: null, // uses messages only with format
|
||||
user: 'Return exactly: {"message": "STRUCTURED_OUTPUT_WORKS"}',
|
||||
};
|
||||
|
||||
// Test C: Minimal reconstruction-style schema
|
||||
const TEST_C = {
|
||||
label: "C",
|
||||
description: "Minimal reconstruction schema — structured output test",
|
||||
system: null,
|
||||
user: "Analyse this situation without solving it: Some customers can log in but cannot download invoices. Identify the meaningful difference and ask one useful next question.",
|
||||
};
|
||||
|
||||
const ALL_TESTS = [TEST_A, TEST_B, TEST_C];
|
||||
|
||||
// ============================================================
|
||||
// Helper functions
|
||||
// ============================================================
|
||||
|
||||
async function runChatWithFormat(model, messages, format) {
|
||||
const body = { model, messages, stream: false, format };
|
||||
const res = await fetch(`${BASE_URL}/api/chat`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
|
||||
const rawText = await res.text();
|
||||
let parsed = null;
|
||||
try { parsed = JSON.parse(rawText); } catch {}
|
||||
|
||||
return {
|
||||
status: res.status,
|
||||
statusText: res.statusText,
|
||||
requestPayload: body,
|
||||
rawResponseText: rawText.slice(0, 5000),
|
||||
parsedResponse: parsed,
|
||||
messageContent: parsed?.message?.content ?? null,
|
||||
thinkingLength: (parsed?.message?.thinking || "").length,
|
||||
messageContentType: typeof parsed?.message?.content,
|
||||
responseField: parsed?.response,
|
||||
};
|
||||
}
|
||||
|
||||
async function runGenerate(model, prompt) {
|
||||
const body = { model, prompt, stream: false };
|
||||
const res = await fetch(`${BASE_URL}/api/generate`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
|
||||
const rawText = await res.text();
|
||||
let parsed = null;
|
||||
try { parsed = JSON.parse(rawText); } catch {}
|
||||
|
||||
return {
|
||||
status: res.status,
|
||||
requestPayload: body,
|
||||
rawResponseText: rawText.slice(0, 5000),
|
||||
parsedResponse: parsed,
|
||||
responseField: typeof parsed?.response === "string" ? parsed.response : JSON.stringify(parsed),
|
||||
responseFirst200: (parsed?.response || "").slice(0, 200),
|
||||
};
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// Run tests
|
||||
// ============================================================
|
||||
|
||||
const results = {};
|
||||
|
||||
for (const model of [MODEL_A, MODEL_B]) {
|
||||
console.log(`\n=== Testing model: ${model} ===`);
|
||||
results[model] = {};
|
||||
|
||||
// Check if model is available locally
|
||||
let available = false;
|
||||
try {
|
||||
const tagsRes = await fetch(`${BASE_URL}/api/tags`);
|
||||
const tagsData = await tagsRes.json();
|
||||
available = tagsData.models?.some(m => m.name.includes(model.split(":")[0]));
|
||||
} catch (e) {
|
||||
console.log(` Warning: could not check model availability: ${e.message}`);
|
||||
}
|
||||
|
||||
if (!available) {
|
||||
results[model].availability = "NOT_AVAILABLE_ON_SERVER";
|
||||
console.log(` -> Model ${model} not found on server, skipping`);
|
||||
continue;
|
||||
}
|
||||
|
||||
console.log(` -> Model available on server\n`);
|
||||
|
||||
for (const test of ALL_TESTS) {
|
||||
const testKey = `test_${test.label}_${model.split(":")[0].replace(/[^a-zA-Z]/g, "_")}`;
|
||||
console.log(` Running Test ${test.label}: ${test.description}`);
|
||||
|
||||
// Chat with format:json
|
||||
let chatResult;
|
||||
try {
|
||||
const messages = [];
|
||||
if (test.system) {
|
||||
messages.push({ role: "system", content: test.system });
|
||||
}
|
||||
messages.push({ role: "user", content: test.user });
|
||||
|
||||
chatResult = await runChatWithFormat(model, messages, "json");
|
||||
|
||||
// Try to extract JSON from message.content
|
||||
let extractedJson = null;
|
||||
if (typeof chatResult.messageContent === "string") {
|
||||
try {
|
||||
extractedJson = JSON.parse(chatResult.messageContent);
|
||||
} catch {}
|
||||
}
|
||||
|
||||
results[model][testKey] = {
|
||||
testDescription: test.description,
|
||||
endpoint: "/api/chat",
|
||||
format: "json",
|
||||
hasSystemMessage: !!test.system,
|
||||
httpStatus: chatResult.status,
|
||||
messageContentType: chatResult.messageContentType,
|
||||
messageContentLength: chatResult.messageContent?.length || 0,
|
||||
thinkingPresent: chatResult.thinkingLength > 0,
|
||||
parsedContentKeys: extractedJson ? Object.keys(extractedJson) : null,
|
||||
// If content looks like a status acknowledgment
|
||||
looksLikeStatusAck: typeof chatResult.messageContent === "string" &&
|
||||
(chatResult.messageContent.includes('"status"') || chatResult.messageContent.includes('"state"')),
|
||||
rawPreview: chatResult.messageContent?.slice(0, 300) ?? "(none)",
|
||||
};
|
||||
|
||||
const status = extractedJson ? "JSON_OK" : (chatResult.messageContent ? "TEXT_RESPONSE" : "EMPTY");
|
||||
console.log(` -> ${status} (HTTP ${chatResult.status}, content type: ${chatResult.messageContentType})`);
|
||||
if (extractedJson) {
|
||||
console.log(` JSON keys: ${Object.keys(extractedJson).join(", ")}`);
|
||||
} else if (chatResult.messageContent) {
|
||||
console.log(` Content preview: ${(typeof chatResult.messageContent === "string" ? chatResult.messageContent : String(chatResult.messageContent)).slice(0, 150)}...`);
|
||||
}
|
||||
|
||||
} catch (e) {
|
||||
results[model][testKey] = { error: e.message };
|
||||
console.log(` -> ERROR: ${e.message}`);
|
||||
}
|
||||
|
||||
// Generate (fallback test)
|
||||
let generateResult;
|
||||
try {
|
||||
const generatePrompt = test.system ? `${test.system}\n\n${test.user}` : test.user;
|
||||
generateResult = await runGenerate(model, generatePrompt);
|
||||
|
||||
results[model][`${testKey}_generate`] = {
|
||||
endpoint: "/api/generate",
|
||||
httpStatus: generateResult.status,
|
||||
responseFirst200: generateResult.responseFirst200,
|
||||
responseLooksLikeStructuredJSON: generateResult.responseField?.trim().startsWith("{"),
|
||||
rawPreview: generateResult.responseFirst200,
|
||||
};
|
||||
|
||||
const isJson = generateResult.responseField?.trim().startsWith("{") ? "JSON_START" : "NOT_JSON";
|
||||
console.log(` -> ${isJson} (HTTP ${generateResult.status})`);
|
||||
|
||||
} catch (e) {
|
||||
results[model][`${testKey}_generate`] = { error: e.message };
|
||||
console.log(` -> GENERATE ERROR: ${e.message}`);
|
||||
}
|
||||
|
||||
console.log();
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================
|
||||
// Save results
|
||||
// ============================================================
|
||||
|
||||
const saveFile = join(RESULTS_DIR, "debug-results.json");
|
||||
writeFileSync(saveFile, JSON.stringify(results, null, 2));
|
||||
console.log(`\nResults saved to: ${saveFile}`);
|
||||
@@ -1,97 +0,0 @@
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
|
||||
const BASE_URL =
|
||||
process.env.CONFIDENCE_ENGINE_BASE_URL || "http://127.0.0.1:3000";
|
||||
const OUTPUT_DIR = "tests-results/commercial-value-update";
|
||||
|
||||
const scenario = "I think therefore I am";
|
||||
const answer =
|
||||
"Deciding whether to build the Confidence Engine due to uncertainty about its commercial value.";
|
||||
|
||||
async function postJson(path, body) {
|
||||
const response = await fetch(`${BASE_URL}${path}`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
|
||||
const json = await response.json();
|
||||
return { status: response.status, json };
|
||||
}
|
||||
|
||||
function printLine(label, value) {
|
||||
const rendered = value === undefined ? null : value;
|
||||
console.log(`${label}: ${JSON.stringify(rendered)}`);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
await mkdir(OUTPUT_DIR, { recursive: true });
|
||||
|
||||
const startResult = await postJson("/api/cases/start", { scenario });
|
||||
await writeFile(
|
||||
`${OUTPUT_DIR}/start-response.json`,
|
||||
JSON.stringify(startResult, null, 2),
|
||||
);
|
||||
|
||||
const selectedQuestion = startResult.json?.selectedQuestion?.question || null;
|
||||
|
||||
let updateResult = {
|
||||
status: null,
|
||||
json: {
|
||||
success: false,
|
||||
stage: "request_construction",
|
||||
errors: ["Missing selected question from start response"],
|
||||
},
|
||||
};
|
||||
|
||||
if (startResult.json?.success && selectedQuestion) {
|
||||
updateResult = await postJson("/api/cases/update", {
|
||||
situationGraph: startResult.json.situationGraph,
|
||||
previousQuestion: selectedQuestion,
|
||||
answer,
|
||||
});
|
||||
}
|
||||
|
||||
await writeFile(
|
||||
`${OUTPUT_DIR}/update-response.json`,
|
||||
JSON.stringify(updateResult, null, 2),
|
||||
);
|
||||
|
||||
printLine("start success", startResult.json?.success ?? false);
|
||||
printLine("update success", updateResult.json?.success ?? false);
|
||||
printLine("update stage", updateResult.json?.stage ?? null);
|
||||
printLine(
|
||||
"proposal added nodes",
|
||||
updateResult.json?.proposal?.addedNodes?.map((node) => node.id) ?? null,
|
||||
);
|
||||
printLine(
|
||||
"proposal added edges",
|
||||
updateResult.json?.proposal?.addedEdges?.map((edge) => ({
|
||||
id: edge.id,
|
||||
fromNodeId: edge.fromNodeId,
|
||||
toNodeId: edge.toNodeId,
|
||||
relationship: edge.relationship,
|
||||
})) ?? null,
|
||||
);
|
||||
printLine(
|
||||
"proposal resolved unknown IDs",
|
||||
updateResult.json?.proposal?.resolvedUnknownNodeIds ??
|
||||
updateResult.json?.resolvedUnknownNodeIds ??
|
||||
null,
|
||||
);
|
||||
printLine(
|
||||
"errors",
|
||||
updateResult.json?.errors ??
|
||||
updateResult.json?.proposalErrors ??
|
||||
updateResult.json?.graphValidationErrors ??
|
||||
updateResult.json?.validationErrors ??
|
||||
null,
|
||||
);
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error instanceof Error ? error.message : String(error));
|
||||
process.exitCode = 1;
|
||||
});
|
||||
@@ -1,114 +0,0 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const mockStartCase = vi.fn();
|
||||
|
||||
vi.mock("@/lib/graph/orchestrator.js", () => ({
|
||||
startCase: (...args) => mockStartCase(...args),
|
||||
}));
|
||||
|
||||
describe("app/api/cases/start route", () => {
|
||||
beforeEach(() => {
|
||||
vi.resetModules();
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it("delegates request body to the orchestrator", async () => {
|
||||
mockStartCase.mockResolvedValue({
|
||||
success: true,
|
||||
situationGraph: { nodes: [{ id: "n1" }], edges: [] },
|
||||
selectedQuestion: null,
|
||||
diagnostics: {},
|
||||
});
|
||||
|
||||
const { POST } = await import("@/app/api/cases/start/route.js");
|
||||
const request = new Request("http://localhost/api/cases/start", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ scenario: "Scenario text" }),
|
||||
headers: { "content-type": "application/json" },
|
||||
});
|
||||
|
||||
await POST(request);
|
||||
|
||||
expect(mockStartCase).toHaveBeenCalledWith({ scenario: "Scenario text" });
|
||||
});
|
||||
|
||||
it("returns 200 on success", async () => {
|
||||
mockStartCase.mockResolvedValue({
|
||||
success: true,
|
||||
situationGraph: { nodes: [{ id: "n1" }], edges: [] },
|
||||
selectedQuestion: null,
|
||||
diagnostics: {},
|
||||
});
|
||||
|
||||
const { POST } = await import("@/app/api/cases/start/route.js");
|
||||
const response = await POST(
|
||||
new Request("http://localhost/api/cases/start", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ scenario: "Scenario text" }),
|
||||
headers: { "content-type": "application/json" },
|
||||
}),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
});
|
||||
|
||||
it("returns 400 for invalid request input", async () => {
|
||||
mockStartCase.mockResolvedValue({
|
||||
success: false,
|
||||
error: "Invalid start-case request",
|
||||
validationErrors: [{ message: "Required" }],
|
||||
statusCode: 400,
|
||||
});
|
||||
|
||||
const { POST } = await import("@/app/api/cases/start/route.js");
|
||||
const response = await POST(
|
||||
new Request("http://localhost/api/cases/start", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({}),
|
||||
headers: { "content-type": "application/json" },
|
||||
}),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
await expect(response.json()).resolves.toMatchObject({
|
||||
success: false,
|
||||
error: "Invalid start-case request",
|
||||
});
|
||||
});
|
||||
|
||||
it("returns provider/internal failures as 5xx without stack traces", async () => {
|
||||
mockStartCase.mockResolvedValue({
|
||||
success: false,
|
||||
error: "Provider unavailable",
|
||||
diagnostics: { modelName: "llama3" },
|
||||
statusCode: 502,
|
||||
});
|
||||
|
||||
const { POST } = await import("@/app/api/cases/start/route.js");
|
||||
const response = await POST(
|
||||
new Request("http://localhost/api/cases/start", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ scenario: "Scenario text" }),
|
||||
headers: { "content-type": "application/json" },
|
||||
}),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(502);
|
||||
await expect(response.json()).resolves.not.toHaveProperty("stack");
|
||||
});
|
||||
|
||||
it("returns structured 500 on malformed JSON", async () => {
|
||||
const { POST } = await import("@/app/api/cases/start/route.js");
|
||||
const request = {
|
||||
json: vi.fn().mockRejectedValue(new Error("Unexpected token")),
|
||||
};
|
||||
|
||||
const response = await POST(request);
|
||||
|
||||
expect(response.status).toBe(500);
|
||||
await expect(response.json()).resolves.toMatchObject({
|
||||
success: false,
|
||||
error: "Internal server error",
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,304 +0,0 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const mockUpdateCase = vi.fn();
|
||||
|
||||
vi.mock("@/lib/graph/orchestrator.js", () => ({
|
||||
updateCase: (...args) => mockUpdateCase(...args),
|
||||
}));
|
||||
|
||||
function makeSuccessResult() {
|
||||
return {
|
||||
success: true,
|
||||
stage: "update_applied",
|
||||
updatedSituationGraph: {
|
||||
centralStatement: "Scenario",
|
||||
nodes: [{ id: "n1" }],
|
||||
edges: [],
|
||||
activeUnknownNodeId: null,
|
||||
resolvedNodeIds: ["n1"],
|
||||
currentSummary: "Updated summary",
|
||||
},
|
||||
proposal: {
|
||||
addedNodes: [],
|
||||
updatedNodes: [],
|
||||
addedEdges: [],
|
||||
removedEdgeIds: [],
|
||||
resolvedUnknownNodeIds: ["n1"],
|
||||
affectedNodeIds: ["n1"],
|
||||
selectedQuestion: null,
|
||||
},
|
||||
selectedQuestion: null,
|
||||
affectedNodeIds: ["n1"],
|
||||
resolvedUnknownNodeIds: ["n1"],
|
||||
previousActiveUnknownNodeId: "n0",
|
||||
newActiveUnknownNodeId: null,
|
||||
changesApplied: { updatedNodeCount: 1 },
|
||||
diagnostics: { promptVersion: "v0.4" },
|
||||
};
|
||||
}
|
||||
|
||||
describe("app/api/cases/update route", () => {
|
||||
beforeEach(() => {
|
||||
vi.resetModules();
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it("valid update returns HTTP 200", async () => {
|
||||
mockUpdateCase.mockResolvedValue(makeSuccessResult());
|
||||
|
||||
const { POST } = await import("@/app/api/cases/update/route.js");
|
||||
const response = await POST(
|
||||
new Request("http://localhost/api/cases/update", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ answer: "A" }),
|
||||
headers: { "content-type": "application/json" },
|
||||
}),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(200);
|
||||
});
|
||||
|
||||
it("route calls updateCase with applyProposal: true", async () => {
|
||||
mockUpdateCase.mockResolvedValue(makeSuccessResult());
|
||||
|
||||
const { POST } = await import("@/app/api/cases/update/route.js");
|
||||
const body = { situationGraph: {}, previousQuestion: "Q", answer: "A" };
|
||||
await POST(
|
||||
new Request("http://localhost/api/cases/update", {
|
||||
method: "POST",
|
||||
body: JSON.stringify(body),
|
||||
headers: { "content-type": "application/json" },
|
||||
}),
|
||||
);
|
||||
|
||||
expect(mockUpdateCase).toHaveBeenCalledWith(body, { applyProposal: true });
|
||||
});
|
||||
|
||||
it("invalid JSON returns 400", async () => {
|
||||
const { POST } = await import("@/app/api/cases/update/route.js");
|
||||
const request = {
|
||||
json: vi.fn().mockRejectedValue(new SyntaxError("Unexpected token")),
|
||||
};
|
||||
|
||||
const response = await POST(request);
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
await expect(response.json()).resolves.toMatchObject({
|
||||
success: false,
|
||||
stage: "request_validation",
|
||||
error: "Invalid JSON request body",
|
||||
});
|
||||
});
|
||||
|
||||
it("request validation failure returns 400", async () => {
|
||||
mockUpdateCase.mockResolvedValue({
|
||||
success: false,
|
||||
stage: "request_validation",
|
||||
error: "Invalid update-case request",
|
||||
validationErrors: [{ message: "Required" }],
|
||||
});
|
||||
|
||||
const { POST } = await import("@/app/api/cases/update/route.js");
|
||||
const response = await POST(
|
||||
new Request("http://localhost/api/cases/update", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({}),
|
||||
headers: { "content-type": "application/json" },
|
||||
}),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
});
|
||||
|
||||
it("graph validation failure returns 400", async () => {
|
||||
mockUpdateCase.mockResolvedValue({
|
||||
success: false,
|
||||
stage: "graph_validation",
|
||||
error: "Invalid situation graph",
|
||||
graphValidationErrors: ["bad graph"],
|
||||
});
|
||||
|
||||
const { POST } = await import("@/app/api/cases/update/route.js");
|
||||
const response = await POST(
|
||||
new Request("http://localhost/api/cases/update", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ answer: "A" }),
|
||||
headers: { "content-type": "application/json" },
|
||||
}),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(400);
|
||||
});
|
||||
|
||||
it("provider failure returns 502", async () => {
|
||||
mockUpdateCase.mockResolvedValue({
|
||||
success: false,
|
||||
stage: "provider",
|
||||
error: "Graph update proposal generation failed",
|
||||
providerErrors: ["provider offline"],
|
||||
});
|
||||
|
||||
const { POST } = await import("@/app/api/cases/update/route.js");
|
||||
const response = await POST(
|
||||
new Request("http://localhost/api/cases/update", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ answer: "A" }),
|
||||
headers: { "content-type": "application/json" },
|
||||
}),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(502);
|
||||
});
|
||||
|
||||
it("proposal validation failure returns 422", async () => {
|
||||
mockUpdateCase.mockResolvedValue({
|
||||
success: false,
|
||||
stage: "proposal_validation",
|
||||
error: "Invalid graph update proposal",
|
||||
proposalErrors: [{ message: "bad proposal" }],
|
||||
});
|
||||
|
||||
const { POST } = await import("@/app/api/cases/update/route.js");
|
||||
const response = await POST(
|
||||
new Request("http://localhost/api/cases/update", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ answer: "A" }),
|
||||
headers: { "content-type": "application/json" },
|
||||
}),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(422);
|
||||
});
|
||||
|
||||
it("proposal compatibility failure returns 422", async () => {
|
||||
mockUpdateCase.mockResolvedValue({
|
||||
success: false,
|
||||
stage: "proposal_compatibility",
|
||||
error: "Update case failed",
|
||||
errors: ["incompatible proposal"],
|
||||
});
|
||||
|
||||
const { POST } = await import("@/app/api/cases/update/route.js");
|
||||
const response = await POST(
|
||||
new Request("http://localhost/api/cases/update", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ answer: "A" }),
|
||||
headers: { "content-type": "application/json" },
|
||||
}),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(422);
|
||||
});
|
||||
|
||||
it("application failure returns 422", async () => {
|
||||
mockUpdateCase.mockResolvedValue({
|
||||
success: false,
|
||||
stage: "application",
|
||||
error: "Update case failed",
|
||||
errors: ["could not apply"],
|
||||
});
|
||||
|
||||
const { POST } = await import("@/app/api/cases/update/route.js");
|
||||
const response = await POST(
|
||||
new Request("http://localhost/api/cases/update", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ answer: "A" }),
|
||||
headers: { "content-type": "application/json" },
|
||||
}),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(422);
|
||||
});
|
||||
|
||||
it("result validation failure returns 500", async () => {
|
||||
mockUpdateCase.mockResolvedValue({
|
||||
success: false,
|
||||
stage: "result_validation",
|
||||
error: "Update case failed",
|
||||
errors: ["invalid result"],
|
||||
});
|
||||
|
||||
const { POST } = await import("@/app/api/cases/update/route.js");
|
||||
const response = await POST(
|
||||
new Request("http://localhost/api/cases/update", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ answer: "A" }),
|
||||
headers: { "content-type": "application/json" },
|
||||
}),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(500);
|
||||
});
|
||||
|
||||
it("unknown failure returns 500", async () => {
|
||||
mockUpdateCase.mockRejectedValue(new Error("boom"));
|
||||
|
||||
const { POST } = await import("@/app/api/cases/update/route.js");
|
||||
const response = await POST(
|
||||
new Request("http://localhost/api/cases/update", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ answer: "A" }),
|
||||
headers: { "content-type": "application/json" },
|
||||
}),
|
||||
);
|
||||
|
||||
expect(response.status).toBe(500);
|
||||
await expect(response.json()).resolves.toMatchObject({
|
||||
success: false,
|
||||
stage: "internal",
|
||||
error: "Internal server error",
|
||||
});
|
||||
});
|
||||
|
||||
it("success response preserves updated graph fields", async () => {
|
||||
const success = makeSuccessResult();
|
||||
mockUpdateCase.mockResolvedValue(success);
|
||||
|
||||
const { POST } = await import("@/app/api/cases/update/route.js");
|
||||
const response = await POST(
|
||||
new Request("http://localhost/api/cases/update", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ answer: "A" }),
|
||||
headers: { "content-type": "application/json" },
|
||||
}),
|
||||
);
|
||||
|
||||
await expect(response.json()).resolves.toMatchObject({
|
||||
updatedSituationGraph: success.updatedSituationGraph,
|
||||
proposal: success.proposal,
|
||||
affectedNodeIds: success.affectedNodeIds,
|
||||
resolvedUnknownNodeIds: success.resolvedUnknownNodeIds,
|
||||
previousActiveUnknownNodeId: success.previousActiveUnknownNodeId,
|
||||
newActiveUnknownNodeId: success.newActiveUnknownNodeId,
|
||||
selectedQuestion: success.selectedQuestion,
|
||||
changesApplied: success.changesApplied,
|
||||
diagnostics: success.diagnostics,
|
||||
});
|
||||
});
|
||||
|
||||
it("stack traces and raw provider output are not exposed", async () => {
|
||||
mockUpdateCase.mockResolvedValue({
|
||||
success: false,
|
||||
stage: "provider",
|
||||
error: "Graph update proposal generation failed",
|
||||
providerErrors: ["provider offline"],
|
||||
rawResponse: "secret",
|
||||
stack: "trace",
|
||||
diagnostics: {},
|
||||
});
|
||||
|
||||
const { POST } = await import("@/app/api/cases/update/route.js");
|
||||
const response = await POST(
|
||||
new Request("http://localhost/api/cases/update", {
|
||||
method: "POST",
|
||||
body: JSON.stringify({ answer: "A" }),
|
||||
headers: { "content-type": "application/json" },
|
||||
}),
|
||||
);
|
||||
|
||||
const payload = await response.json();
|
||||
|
||||
expect(payload).not.toHaveProperty("stack");
|
||||
expect(payload).not.toHaveProperty("rawResponse");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,92 @@
|
||||
[
|
||||
{
|
||||
"id": "diag-01",
|
||||
"input": "We've seen a spike in complaints from our warehouse team this month compared to last month.",
|
||||
"expectedPrimaryTypes": ["unexplained_change"],
|
||||
"expectedReasoningModes": ["establish_baseline", "identify_difference"],
|
||||
"shouldIdentify": ["complaints", "warehouse", "baseline comparison"],
|
||||
"shouldNotInfer": ["quality issue", "staff turnover", "training gap"],
|
||||
"description": "Baseline comparison — change without context. Should NOT jump to conclusions about quality or staff issues."
|
||||
},
|
||||
{
|
||||
"id": "diag-02",
|
||||
"input": "Some customers reported that the new app crashes when uploading photos.",
|
||||
"expectedPrimaryTypes": ["observed_problem"],
|
||||
"expectedReasoningModes": ["identify_difference", "establish_baseline"],
|
||||
"shouldIdentify": ["app crashes", "photo upload", "some customers"],
|
||||
"shouldNotInfer": ["all users affected", "server-side bug", "Android only"],
|
||||
"description": "Subset modifier — 'some customers' means not universal. Should distinguish from blanket claims."
|
||||
},
|
||||
{
|
||||
"id": "diag-03",
|
||||
"input": "Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.",
|
||||
"expectedPrimaryTypes": ["contradiction"],
|
||||
"expectedReasoningModes": ["investigate_contradiction", "establish_baseline"],
|
||||
"shouldIdentify": ["sales decline", "price increase", "revenue increase", "CFO report"],
|
||||
"shouldNotInfer": ["price was set too high", "competitors gained market share", "revenue data is wrong"],
|
||||
"description": "Apparent contradiction — sales down but revenue up after price change. Distinguishes volume vs value."
|
||||
},
|
||||
{
|
||||
"id": "diag-04",
|
||||
"input": "We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.",
|
||||
"expectedPrimaryTypes": ["decision_request"],
|
||||
"expectedReasoningModes": ["decision_support", "identify_missing_information"],
|
||||
"shouldIdentify": ["marketplace app", "Southeast Asia", "competitor gap"],
|
||||
"shouldNotInfer": ["this will definitely succeed", "we have the resources", "competitors are struggling"],
|
||||
"description": "Decision request — forward-looking, needs missing info identification."
|
||||
},
|
||||
{
|
||||
"id": "diag-05",
|
||||
"input": "Our production line changed suppliers three months ago but still delivers the same defect rate as before.",
|
||||
"expectedPrimaryTypes": ["unexplained_change"],
|
||||
"expectedReasoningModes": ["establish_baseline", "identify_difference"],
|
||||
"shouldIdentify": ["supplier change", "three months ago", "same defect rate"],
|
||||
"shouldNotInfer": ["new supplier is worse", "old supplier was better", "quality process is broken"],
|
||||
"description": "Unexpected continuity — changed context but no outcome change."
|
||||
},
|
||||
{
|
||||
"id": "diag-06",
|
||||
"input": "From 45% to 62%, the completion rate for our onboarding flow improved significantly.",
|
||||
"expectedPrimaryTypes": ["unexplained_change"],
|
||||
"expectedReasoningModes": ["establish_baseline", "validate_measurement"],
|
||||
"shouldIdentify": ["completion rate", "45%", "62%", "onboarding"],
|
||||
"shouldNotInfer": ["all improvements are due to the redesign", "the old flow was bad", "users prefer the new design"],
|
||||
"description": "Quantified improvement — needs context about measurement period and baseline conditions."
|
||||
},
|
||||
{
|
||||
"id": "diag-07",
|
||||
"input": "A user claimed that our pricing model is too complex for small businesses.",
|
||||
"expectedPrimaryTypes": ["reported_claim"],
|
||||
"expectedReasoningModes": ["validate_claim", "identify_difference"],
|
||||
"shouldIdentify": ["pricing complexity", "small business", "user claim"],
|
||||
"shouldNotInfer": ["the pricing is actually complex", "other small businesses agree", "we should simplify pricing"],
|
||||
"description": "Single reported claim — needs validation, not acceptance as fact."
|
||||
},
|
||||
{
|
||||
"id": "diag-08",
|
||||
"input": "I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?",
|
||||
"expectedPrimaryTypes": ["ambiguous_statement"],
|
||||
"expectedReasoningModes": ["clarify_meaning"],
|
||||
"shouldIdentify": ["philosophical", "ambiguous", "meaning clarification"],
|
||||
"shouldNotInfer": ["the phrase was wrong", "the colleague is hostile", "we should avoid philosophical language"],
|
||||
"description": "Meta-test — self-referential ambiguous statement. Should trigger clarification mode."
|
||||
},
|
||||
{
|
||||
"id": "diag-09",
|
||||
"input": "After the deployment last week, our complaint volume tripled to 47 cases per day.",
|
||||
"expectedPrimaryTypes": ["causal_claim"],
|
||||
"expectedReasoningModes": ["investigate_contradiction", "establish_baseline"],
|
||||
"shouldIdentify": ["deployment", "complaint volume increase", "tripled", "47 cases"],
|
||||
"shouldNotInfer": ["the deployment caused the complaints", "the bug report was insufficient", "rollback is needed"],
|
||||
"description": "Post-event spike — presents correlation as potential causation. Must resist jumping to causal conclusion."
|
||||
},
|
||||
{
|
||||
"id": "diag-10",
|
||||
"input": "Some complaints involve production issues, but others say the delivery team is slow.",
|
||||
"expectedPrimaryTypes": ["observed_problem"],
|
||||
"expectedReasoningModes": ["identify_difference", "decompose_aggregate"],
|
||||
"shouldIdentify": ["production issues", "delivery speed", "complaint types"],
|
||||
"shouldNotInfer": ["production is worse than delivery", "the delivery team needs training", "both teams are underperforming equally"],
|
||||
"description": "Paired with diag-01 — distinguishes subset complaints from aggregate claims."
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,92 @@
|
||||
[
|
||||
{
|
||||
"id": "diag-01",
|
||||
"input": "We've seen a spike in complaints from our warehouse team this month compared to last month.",
|
||||
"expectedPrimaryTypes": ["unexplained_change"],
|
||||
"expectedReasoningModes": ["establish_baseline", "identify_difference"],
|
||||
"shouldIdentify": ["complaints", "warehouse", "baseline comparison"],
|
||||
"shouldNotInfer": ["quality issue", "staff turnover", "training gap"],
|
||||
"description": "Baseline comparison — change without context. Should NOT jump to conclusions about quality or staff issues."
|
||||
},
|
||||
{
|
||||
"id": "diag-02",
|
||||
"input": "Some customers reported that the new app crashes when uploading photos.",
|
||||
"expectedPrimaryTypes": ["observed_problem"],
|
||||
"expectedReasoningModes": ["identify_difference", "establish_baseline"],
|
||||
"shouldIdentify": ["app crashes", "photo upload", "some customers"],
|
||||
"shouldNotInfer": ["all users affected", "server-side bug", "Android only"],
|
||||
"description": "Subset modifier — 'some customers' means not universal. Should distinguish from blanket claims."
|
||||
},
|
||||
{
|
||||
"id": "diag-03",
|
||||
"input": "Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.",
|
||||
"expectedPrimaryTypes": ["contradiction"],
|
||||
"expectedReasoningModes": ["investigate_contradiction", "establish_baseline"],
|
||||
"shouldIdentify": ["sales decline", "price increase", "revenue increase", "CFO report"],
|
||||
"shouldNotInfer": ["price was set too high", "competitors gained market share", "revenue data is wrong"],
|
||||
"description": "Apparent contradiction — sales down but revenue up after price change. Distinguishes volume vs value."
|
||||
},
|
||||
{
|
||||
"id": "diag-04",
|
||||
"input": "We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.",
|
||||
"expectedPrimaryTypes": ["decision_request"],
|
||||
"expectedReasoningModes": ["decision_support", "identify_missing_information"],
|
||||
"shouldIdentify": ["marketplace app", "Southeast Asia", "competitor gap"],
|
||||
"shouldNotInfer": ["this will definitely succeed", "we have the resources", "competitors are struggling"],
|
||||
"description": "Decision request — forward-looking, needs missing info identification."
|
||||
},
|
||||
{
|
||||
"id": "diag-05",
|
||||
"input": "Our production line changed suppliers three months ago but still delivers the same defect rate as before.",
|
||||
"expectedPrimaryTypes": ["unexplained_change"],
|
||||
"expectedReasoningModes": ["establish_baseline", "identify_difference"],
|
||||
"shouldIdentify": ["supplier change", "three months ago", "same defect rate"],
|
||||
"shouldNotInfer": ["new supplier is worse", "old supplier was better", "quality process is broken"],
|
||||
"description": "Unexpected continuity — changed context but no outcome change."
|
||||
},
|
||||
{
|
||||
"id": "diag-06",
|
||||
"input": "From 45% to 62%, the completion rate for our onboarding flow improved significantly.",
|
||||
"expectedPrimaryTypes": ["unexplained_change"],
|
||||
"expectedReasoningModes": ["establish_baseline", "validate_measurement"],
|
||||
"shouldIdentify": ["completion rate", "45%", "62%", "onboarding"],
|
||||
"shouldNotInfer": ["all improvements are due to the redesign", "the old flow was bad", "users prefer the new design"],
|
||||
"description": "Quantified improvement — needs context about measurement period and baseline conditions."
|
||||
},
|
||||
{
|
||||
"id": "diag-07",
|
||||
"input": "A user claimed that our pricing model is too complex for small businesses.",
|
||||
"expectedPrimaryTypes": ["reported_claim"],
|
||||
"expectedReasoningModes": ["validate_claim", "identify_difference"],
|
||||
"shouldIdentify": ["pricing complexity", "small business", "user claim"],
|
||||
"shouldNotInfer": ["the pricing is actually complex", "other small businesses agree", "we should simplify pricing"],
|
||||
"description": "Single reported claim — needs validation, not acceptance as fact."
|
||||
},
|
||||
{
|
||||
"id": "diag-08",
|
||||
"input": "I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?",
|
||||
"expectedPrimaryTypes": ["ambiguous_statement"],
|
||||
"expectedReasoningModes": ["clarify_meaning"],
|
||||
"shouldIdentify": ["philosophical", "ambiguous", "meaning clarification"],
|
||||
"shouldNotInfer": ["the phrase was wrong", "the colleague is hostile", "we should avoid philosophical language"],
|
||||
"description": "Meta-test — self-referential ambiguous statement. Should trigger clarification mode."
|
||||
},
|
||||
{
|
||||
"id": "diag-09",
|
||||
"input": "After the deployment last week, our complaint volume tripled to 47 cases per day.",
|
||||
"expectedPrimaryTypes": ["causal_claim"],
|
||||
"expectedReasoningModes": ["investigate_contradiction", "establish_baseline"],
|
||||
"shouldIdentify": ["deployment", "complaint volume increase", "tripled", "47 cases"],
|
||||
"shouldNotInfer": ["the deployment caused the complaints", "the bug report was insufficient", "rollback is needed"],
|
||||
"description": "Post-event spike — presents correlation as potential causation. Must resist jumping to causal conclusion."
|
||||
},
|
||||
{
|
||||
"id": "diag-10",
|
||||
"input": "Some complaints involve production issues, but others say the delivery team is slow.",
|
||||
"expectedPrimaryTypes": ["observed_problem"],
|
||||
"expectedReasoningModes": ["identify_difference", "decompose_aggregate"],
|
||||
"shouldIdentify": ["production issues", "delivery speed", "complaint types"],
|
||||
"shouldNotInfer": ["production is worse than delivery", "the delivery team needs training", "both teams are underperforming equally"],
|
||||
"description": "Paired with diag-01 — distinguishes subset complaints from aggregate claims."
|
||||
}
|
||||
]
|
||||
@@ -0,0 +1,230 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { readFileSync, existsSync, readdirSync } from "node:fs";
|
||||
import { join, dirname } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
const rootDir = join(__dirname, "..", "..");
|
||||
|
||||
// ── Test data loading and structure ────────────────
|
||||
|
||||
describe("live-diagnostic test data", () => {
|
||||
const cases = JSON.parse(
|
||||
readFileSync(join(__dirname, "data", "live-diagnostic-v0.2.json"), "utf-8")
|
||||
);
|
||||
|
||||
it("loads without error", () => {
|
||||
expect(cases).toBeDefined();
|
||||
expect(Array.isArray(cases)).toBe(true);
|
||||
});
|
||||
|
||||
it("contains exactly 10 cases", () => {
|
||||
expect(cases.length).toBe(10);
|
||||
});
|
||||
|
||||
it("each case has required fields (id, input, expectedPrimaryTypes)", () => {
|
||||
for (const c of cases) {
|
||||
expect(c.id).toBeDefined();
|
||||
expect(typeof c.id).toBe("string");
|
||||
expect(c.input).toBeDefined();
|
||||
expect(typeof c.input).toBe("string");
|
||||
expect(c.input.length).toBeGreaterThan(0);
|
||||
expect(c.expectedPrimaryTypes).toBeDefined();
|
||||
expect(Array.isArray(c.expectedPrimaryTypes)).toBe(true);
|
||||
expect(c.shouldIdentify).toBeDefined();
|
||||
expect(c.shouldNotInfer).toBeDefined();
|
||||
}
|
||||
});
|
||||
|
||||
it("has unique case IDs", () => {
|
||||
const ids = cases.map((c) => c.id);
|
||||
const uniqueIds = new Set(ids);
|
||||
expect(uniqueIds.size).toBe(ids.length);
|
||||
});
|
||||
|
||||
it("IDs follow diag-NN naming convention", () => {
|
||||
const ids = cases.map((c) => c.id);
|
||||
for (const id of ids) {
|
||||
expect(id).toMatch(/^diag-\d{2}$/);
|
||||
}
|
||||
});
|
||||
|
||||
it("has no duplicate shouldIdentify/shouldNotInfer sets (paired cases differ)", () => {
|
||||
// diag-01 and diag-10 are the "paired" cases — they share context but not identical assertions
|
||||
const diag01 = cases.find((c) => c.id === "diag-01");
|
||||
const diag10 = cases.find((c) => c.id === "diag-10");
|
||||
expect(diag01).toBeDefined();
|
||||
expect(diag10).toBeDefined();
|
||||
|
||||
// They should NOT have identical shouldIdentify — the point of pairing is to distinguish them
|
||||
const identify01 = JSON.stringify(diag01.shouldIdentify.sort());
|
||||
const identify10 = JSON.stringify(diag10.shouldIdentify.sort());
|
||||
expect(identify01).not.toBe(identify10);
|
||||
});
|
||||
|
||||
it("shouldNotInfer is a non-empty array of strings", () => {
|
||||
for (const c of cases) {
|
||||
expect(Array.isArray(c.shouldNotInfer)).toBe(true);
|
||||
expect(c.shouldNotInfer.length).toBeGreaterThan(0);
|
||||
expect(typeof c.shouldNotInfer[0]).toBe("string");
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
// ── Mock evaluation writes correct files ───────────
|
||||
|
||||
describe("mock evaluation result capture", () => {
|
||||
it("test file path exists", () => {
|
||||
const path = join(__dirname, "data", "live-diagnostic-v0.2.json");
|
||||
expect(existsSync(path)).toBe(true);
|
||||
});
|
||||
|
||||
it("package.json contains diagnostic scripts", async () => {
|
||||
const pkg = JSON.parse(
|
||||
readFileSync(join(rootDir, "package.json"), "utf-8")
|
||||
);
|
||||
expect(pkg.scripts["evaluate:mock"]).toContain("EVAL_REAL=0");
|
||||
expect(pkg.scripts["evaluate:diagnostic"]).toContain("EVAL_DIAGNOSTIC=1");
|
||||
expect(pkg.scripts["evaluate:live"]).toContain("EVAL_REAL=1");
|
||||
});
|
||||
});
|
||||
|
||||
// ── Markdown generation correctness ────────────────
|
||||
|
||||
describe("markdown summary content", () => {
|
||||
it("contains expected header format for each case ID pattern", () => {
|
||||
const cases = JSON.parse(
|
||||
readFileSync(join(__dirname, "data", "live-diagnostic-v0.2.json"), "utf-8")
|
||||
);
|
||||
for (const c of cases) {
|
||||
expect(c.description).toBeDefined();
|
||||
expect(typeof c.description).toBe("string");
|
||||
expect(c.description.length).toBeGreaterThan(0);
|
||||
}
|
||||
});
|
||||
|
||||
it("diag-01 and diag-02 have different descriptions indicating their distinction", () => {
|
||||
const cases = JSON.parse(
|
||||
readFileSync(join(__dirname, "data", "live-diagnostic-v0.2.json"), "utf-8")
|
||||
);
|
||||
const diag01 = cases.find((c) => c.id === "diag-01");
|
||||
const diag02 = cases.find((c) => c.id === "diag-02");
|
||||
expect(diag01.description).not.toBe(diag02.description);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Command safeguards ─────────────────────────────
|
||||
|
||||
describe("command safeguards", () => {
|
||||
it("evaluate:diagnostic sets EVAL_DIAGNOSTIC env var", async () => {
|
||||
const pkg = JSON.parse(
|
||||
readFileSync(join(rootDir, "package.json"), "utf-8")
|
||||
);
|
||||
expect(pkg.scripts["evaluate:diagnostic"]).toMatch(/EVAL_DIAGNOSTIC=1/);
|
||||
});
|
||||
|
||||
it("evaluate:mock sets EVAL_REAL=0 to prevent real provider calls", async () => {
|
||||
const pkg = JSON.parse(
|
||||
readFileSync(join(rootDir, "package.json"), "utf-8")
|
||||
);
|
||||
expect(pkg.scripts["evaluate:mock"]).toMatch(/EVAL_REAL=0/);
|
||||
});
|
||||
|
||||
it("evaluate:live sets EVAL_REAL=1 to enable real provider", async () => {
|
||||
const pkg = JSON.parse(
|
||||
readFileSync(join(rootDir, "package.json"), "utf-8")
|
||||
);
|
||||
expect(pkg.scripts["evaluate:live"]).toMatch(/EVAL_REAL=1/);
|
||||
});
|
||||
|
||||
it("mock script does not have EVAL_DIAGNOSTIC set (avoids accidental diagnostic mode)", async () => {
|
||||
const pkg = JSON.parse(
|
||||
readFileSync(join(rootDir, "package.json"), "utf-8")
|
||||
);
|
||||
expect(pkg.scripts["evaluate:mock"]).not.toMatch(/EVAL_DIAGNOSTIC/);
|
||||
});
|
||||
});
|
||||
|
||||
// ── Evaluator.mjs integration ──────────────────────
|
||||
|
||||
describe("evaluator diagnostic mode integration", () => {
|
||||
it("evaluator.mjs checks for EVAL_DIAGNOSTIC env var", async () => {
|
||||
const evaluator = readFileSync(
|
||||
join(__dirname, "..", "evaluator.mjs"),
|
||||
"utf-8"
|
||||
);
|
||||
expect(evaluator).toContain("EVAL_DIAGNOSTIC");
|
||||
expect(evaluator).toContain("useDiagnostic");
|
||||
});
|
||||
|
||||
it("evaluator loads JSON array for diagnostic mode (not JSONL)", async () => {
|
||||
const evaluator = readFileSync(
|
||||
join(__dirname, "..", "evaluator.mjs"),
|
||||
"utf-8"
|
||||
);
|
||||
// Should handle .json files with JSON.parse (array format)
|
||||
expect(evaluator).toContain('path.endsWith(".json")');
|
||||
});
|
||||
|
||||
it("evaluator writes to evaluation-results directory for diagnostic mode", async () => {
|
||||
const evaluator = readFileSync(
|
||||
join(__dirname, "..", "evaluator.mjs"),
|
||||
"utf-8"
|
||||
);
|
||||
expect(evaluator).toContain("evaluation-results");
|
||||
});
|
||||
|
||||
it("evaluator saves per-case markdown summaries for diagnostic mode", async () => {
|
||||
const evaluator = readFileSync(
|
||||
join(__dirname, "..", "evaluator.mjs"),
|
||||
"utf-8"
|
||||
);
|
||||
expect(evaluator).toContain("-summary.md");
|
||||
});
|
||||
|
||||
it("evaluator saves summary.json and manifest for diagnostic runs", async () => {
|
||||
const evaluator = readFileSync(
|
||||
join(__dirname, "..", "evaluator.mjs"),
|
||||
"utf-8"
|
||||
);
|
||||
expect(evaluator).toContain("summary.json");
|
||||
expect(evaluator).toContain("latest-manifest.json");
|
||||
});
|
||||
});
|
||||
|
||||
// ── Live diagnostic data content verification ──────
|
||||
|
||||
describe("diagnostic case reasoning diversity", () => {
|
||||
const cases = JSON.parse(
|
||||
readFileSync(join(__dirname, "data", "live-diagnostic-v0.2.json"), "utf-8")
|
||||
);
|
||||
|
||||
it("covers all expected primary types", () => {
|
||||
const expectedTypes = [
|
||||
"unexplained_change",
|
||||
"observed_problem",
|
||||
"contradiction",
|
||||
"decision_request",
|
||||
"reported_claim",
|
||||
"ambiguous_statement",
|
||||
"causal_claim",
|
||||
];
|
||||
const found = new Set(cases.flatMap((c) => c.expectedPrimaryTypes));
|
||||
for (const t of expectedTypes) {
|
||||
expect(found.has(t)).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it("diag-03 and diag-09 are distinct test targets", () => {
|
||||
const diag03 = cases.find((c) => c.id === "diag-03");
|
||||
const diag09 = cases.find((c) => c.id === "diag-09");
|
||||
expect(diag03.expectedPrimaryTypes).not.toEqual(diag09.expectedPrimaryTypes);
|
||||
});
|
||||
|
||||
it("each case has a unique description", () => {
|
||||
const descs = cases.map((c) => c.description);
|
||||
const unique = new Set(descs);
|
||||
expect(unique.size).toBe(descs.length);
|
||||
});
|
||||
});
|
||||
Executable
+684
@@ -0,0 +1,684 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Evaluation harness for Confidence Engine v0.2.
|
||||
* Runs test cases through the analysis pipeline (mock or real provider).
|
||||
* Produces console summary and saves results to timestamped file.
|
||||
*
|
||||
* Scoring is split into two honest categories:
|
||||
*
|
||||
* TECHNICAL — structural correctness of the output:
|
||||
* • Schema validity (does the JSON match the schema?)
|
||||
* • Classification accuracy (primary type + reasoning modes correct?)
|
||||
* • Next-question presence (is exactly one nextQuestion emitted?)
|
||||
*
|
||||
* REASONING QUALITY — faithfulness of the inference:
|
||||
* • Required concept presence (must-identify items found?)
|
||||
* • Unsupported inference absence (prohibited claims genuinely absent?)
|
||||
*
|
||||
* A test case can pass technical but fail reasoning (hallucination),
|
||||
* or pass reasoning but fail technical (missing fields, schema errors).
|
||||
*/
|
||||
|
||||
import { readFileSync, writeFileSync, mkdirSync, existsSync } from "node:fs";
|
||||
import { join, dirname } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
|
||||
// ── Config ───────────────────────────────────────────
|
||||
const useRealProvider = process.env.EVAL_REAL === "1";
|
||||
const useDiagnostic = process.env.EVAL_DIAGNOSTIC === "1";
|
||||
|
||||
let testDataPath;
|
||||
if (useDiagnostic) {
|
||||
testDataPath = join(__dirname, "data", "live-diagnostic-v0.2.json");
|
||||
} else {
|
||||
testDataPath = join(__dirname, "test-data", "v0.2-evaluation.jsonl");
|
||||
}
|
||||
|
||||
// Standard results dir (for full evals) vs live diagnostic results dir
|
||||
const resultsDir = useDiagnostic
|
||||
? join(__dirname, "..", "evaluation-results")
|
||||
: join(__dirname, "..", "tests-results");
|
||||
|
||||
if (!existsSync(resultsDir)) {
|
||||
mkdirSync(resultsDir, { recursive: true });
|
||||
}
|
||||
|
||||
// ── Load test cases ──────────────────────────────────
|
||||
function loadTestCases(path) {
|
||||
const content = readFileSync(path, "utf-8");
|
||||
// Support both JSONL (one JSON object per line) and JSON array formats
|
||||
if (path.endsWith(".json")) {
|
||||
return JSON.parse(content);
|
||||
}
|
||||
return content
|
||||
.split("\n")
|
||||
.filter((line) => line.trim())
|
||||
.map((line) => JSON.parse(line));
|
||||
}
|
||||
|
||||
// ── Normalise text for comparison ────────────────────
|
||||
function normalise(text) {
|
||||
return String(text)
|
||||
.toLowerCase()
|
||||
.replace(/[^\w\s_]/g, " ")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
}
|
||||
|
||||
// ── Technical scoring helpers ────────────────────────
|
||||
|
||||
function checkPrimaryTypeMatch(actualPrimary, expectedTypes) {
|
||||
if (!actualPrimary || !expectedTypes?.length) return false;
|
||||
const actual = String(actualPrimary).toLowerCase().replace(/\s+/g, "_");
|
||||
return expectedTypes.some((t) => t.toLowerCase().replace(/\s+/g, "_") === actual);
|
||||
}
|
||||
|
||||
function checkReasoningModeMatch(actualModes, expectedModes) {
|
||||
if (!actualModes?.length || !expectedModes?.length) return false;
|
||||
const actual = actualModes.map((m) => String(m).toLowerCase().replace(/\s+/g, "_"));
|
||||
const expected = expectedModes.map((m) => String(m).toLowerCase().replace(/\s+/g, "_"));
|
||||
return expected.some((e) => actual.includes(e));
|
||||
}
|
||||
|
||||
function checkNextQuestionPresent(nextQuestion) {
|
||||
return nextQuestion !== null && nextQuestion !== undefined && nextQuestion !== "";
|
||||
}
|
||||
|
||||
// ── Reasoning quality helpers ────────────────────────
|
||||
|
||||
function checkConceptPresence(actualText, concepts) {
|
||||
if (!concepts?.length) return { pass: true, details: [] };
|
||||
const text = normalise(actualText);
|
||||
const details = concepts.map((c) => ({
|
||||
concept: c,
|
||||
found: text.includes(normalise(c)),
|
||||
}));
|
||||
return { pass: details.every((d) => d.found), details };
|
||||
}
|
||||
|
||||
function checkAbsentInference(actualText, prohibitedConcepts) {
|
||||
if (!prohibitedConcepts?.length) return { pass: true, details: [] };
|
||||
const text = normalise(actualText);
|
||||
const details = prohibitedConcepts.map((c) => ({
|
||||
concept: c,
|
||||
absent: !text.includes(normalise(c)),
|
||||
}));
|
||||
return { pass: details.every((d) => d.absent), details };
|
||||
}
|
||||
|
||||
// ── Run a single test case ───────────────────────────
|
||||
async function runTestCase(testCase, analyseScenarioFn) {
|
||||
const base = {
|
||||
id: testCase.id,
|
||||
input: testCase.input.slice(0, 200),
|
||||
responseDurationMs: 0,
|
||||
actualPrimaryType: null,
|
||||
actualReasoningModes: [],
|
||||
};
|
||||
|
||||
// ── TECHNICAL result ────────────────────────────────
|
||||
const technical = {
|
||||
schemaValid: false,
|
||||
classificationMatch: false,
|
||||
reasoningModeMatch: false,
|
||||
nextQuestionPresent: false,
|
||||
pass: false,
|
||||
errors: [],
|
||||
};
|
||||
|
||||
// ── REASONING QUALITY result ────────────────────────
|
||||
const reasoningQuality = {
|
||||
requiredConcepts: { pass: true, details: [] },
|
||||
unsupportedInferencesAbsent: { pass: true, details: [] },
|
||||
pass: false,
|
||||
};
|
||||
|
||||
try {
|
||||
const analysisResult = await analyseScenarioFn(testCase.input, { promptVersion: "v0.2" });
|
||||
|
||||
base.responseDurationMs = analysisResult.responseDurationMs || 0;
|
||||
base.rawOutput = analysisResult.rawResponse?.slice(0, 500);
|
||||
|
||||
if (analysisResult.success) {
|
||||
technical.schemaValid = true;
|
||||
const actualPrimary = analysisResult.inputClassification?.primaryType;
|
||||
technical.classificationMatch = checkPrimaryTypeMatch(actualPrimary, testCase.expectedPrimaryTypes);
|
||||
base.actualPrimaryType = actualPrimary;
|
||||
|
||||
const modes = analysisResult.inputClassification?.reasoningModes || [];
|
||||
technical.reasoningModeMatch = checkReasoningModeMatch(modes, testCase.expectedReasoningModes);
|
||||
base.actualReasoningModes = modes;
|
||||
|
||||
technical.nextQuestionPresent = checkNextQuestionPresent(analysisResult.nextQuestion);
|
||||
|
||||
// ── Reasoning quality checks ─────────────────────
|
||||
const summaryText = analysisResult.reconstruction?.summary || "";
|
||||
const evidenceTexts = (analysisResult.evidence || []).map((e) => e.description);
|
||||
const allEvidenceRaw = (analysisResult.evidence || []).map(
|
||||
(e) => `${e.description} ${e.attribution || ""}`
|
||||
);
|
||||
|
||||
reasoningQuality.requiredConcepts = checkConceptPresence(
|
||||
[summaryText, ...evidenceTexts].join(" "),
|
||||
testCase.shouldIdentify
|
||||
);
|
||||
|
||||
reasoningQuality.unsupportedInferencesAbsent = checkAbsentInference(
|
||||
allEvidenceRaw.join(" "),
|
||||
testCase.shouldNotInfer
|
||||
);
|
||||
|
||||
// ── Combined pass criteria ───────────────────────
|
||||
technical.pass =
|
||||
technical.schemaValid && technical.classificationMatch && technical.nextQuestionPresent;
|
||||
reasoningQuality.pass =
|
||||
reasoningQuality.requiredConcepts.pass && reasoningQuality.unsupportedInferencesAbsent.pass;
|
||||
} else {
|
||||
technical.errors = analysisResult.errors || [analysisResult.error];
|
||||
if (analysisResult.error) technical.errors.push(analysisResult.error);
|
||||
}
|
||||
} catch (e) {
|
||||
technical.errors.push(e.message || String(e));
|
||||
}
|
||||
|
||||
return { ...base, technical, reasoningQuality };
|
||||
}
|
||||
|
||||
// ── Mock provider for evaluation ─────────────────────
|
||||
class MockProvider {
|
||||
constructor() {
|
||||
this.name = "mock";
|
||||
}
|
||||
|
||||
async generateReconstruction(prompt, modelName) {
|
||||
// Extract the scenario text from the prompt template
|
||||
let scenario = prompt;
|
||||
const scenarioMarker = "Scenario:\n";
|
||||
const markerIdx = prompt.indexOf(scenarioMarker);
|
||||
if (markerIdx >= 0) {
|
||||
scenario = prompt.slice(markerIdx + scenarioMarker.length).trim();
|
||||
}
|
||||
const instructionSeparator = "\n\nReturn ONLY";
|
||||
const instIdx = scenario.indexOf(instructionSeparator);
|
||||
if (instIdx >= 0) {
|
||||
scenario = scenario.slice(0, instIdx).trim();
|
||||
}
|
||||
|
||||
// ── Keyword detection on scenario text only ───────
|
||||
const hasAllWord = /\ball\b|\bno one\b|\bevery\b/i.test(scenario);
|
||||
const hasSomeWord = /\bsome\b/i.test(scenario);
|
||||
const hasComplaints = /complaint/i.test(scenario);
|
||||
const hasSales = /sales/i.test(scenario);
|
||||
const hasRevenue = /revenue|profit|margin/i.test(scenario);
|
||||
const hasReportedSpeaker = /\b(?:reported|said|claimed|stated)\b.*\b(?:cfo|warehouse manager|user|customer|team|analyst|regulator|operator)\b|\b(?:cfo|warehouse manager|user|customer|team|analyst|regulator|operator)\b.*\b(?:reported|said|claimed|stated)\b/i.test(scenario);
|
||||
const hasContradictionSignal = /\bbut\b|\bwile\b|\bothers\s+say\b|\bis better.*is slower\b/i.test(scenario);
|
||||
const hasChangeIndicator = /\b(?:increased|decreased|fell|dropped|grew|rose|declined|up by |down by |changed from |went from |tripled|doubled|halved)\b/i.test(scenario);
|
||||
const hasDecisionRequest = /\b(?:need\s+to\s+improve|need\s+better|we should implement|should fix|want .* launch.*market|launch .* app.*capture|implement .* because.*competitor)\b/i.test(scenario);
|
||||
const hasAmbiguous = /philosophical|therefore i am|ambiguous statement|meta.?context/i.test(scenario);
|
||||
const hasCausalSignal = /\bafter\b.*(?:complaint|failure|issue|problem|price|deployment)|deployed.*and.*(tripl|double|increase)|due to|\bbecause\b/i.test(scenario);
|
||||
const hasTemporalComparison = /last month.*this month|was \d+.*\bby \d+%|\bfrom \d+.*to \d+|\b\d+% from \d+/.test(scenario);
|
||||
const hasUnexpectedContinuity = /\bchanged.*but.*still|\bstill.*working/i.test(scenario);
|
||||
|
||||
// ── Classification hierarchy (most specific first) ─
|
||||
let primaryType = "other";
|
||||
|
||||
if (hasAmbiguous) {
|
||||
primaryType = "ambiguous_statement";
|
||||
} else if (/^\s*I used the phrase/i.test(scenario)) {
|
||||
primaryType = "question";
|
||||
} else if (hasDecisionRequest || /\bneeds?\s+better|\bwe need to\b/i.test(scenario)) {
|
||||
primaryType = "decision_request";
|
||||
} else if (hasCausalSignal && hasSales) {
|
||||
primaryType = "causal_claim";
|
||||
} else if (hasCausalSignal && !hasRevenue) {
|
||||
primaryType = "causal_claim";
|
||||
} else if (hasContradictionSignal && hasRevenue) {
|
||||
primaryType = "contradiction";
|
||||
} else if (hasContradictionSignal && hasChangeIndicator) {
|
||||
primaryType = "contradiction";
|
||||
} else if (hasReportedSpeaker && !hasChangeIndicator) {
|
||||
primaryType = "reported_claim";
|
||||
} else if (hasUnexpectedContinuity) {
|
||||
primaryType = "unexplained_change";
|
||||
} else if (hasTemporalComparison && !hasRevenue) {
|
||||
primaryType = "unexplained_change";
|
||||
} else if (hasChangeIndicator && !hasAllWord && !hasSomeWord) {
|
||||
primaryType = "unexplained_change";
|
||||
} else if (hasChangeIndicator && hasRevenue) {
|
||||
primaryType = "unexplained_change";
|
||||
} else if (hasAllWord || hasSales) {
|
||||
primaryType = "observed_problem";
|
||||
} else if (hasSomeWord && !hasAllWord) {
|
||||
primaryType = "observed_problem";
|
||||
} else if (hasChangeIndicator || hasComplaints) {
|
||||
primaryType = "unexplained_change";
|
||||
} else if (/^[A-Z]/.test(scenario.trim())) {
|
||||
primaryType = "observed_problem";
|
||||
}
|
||||
|
||||
const secondaryTypes = [];
|
||||
if (primaryType === "observed_problem") secondaryTypes.push("fault_report");
|
||||
if (hasComplaints || hasSales) secondaryTypes.push("unexplained_change");
|
||||
|
||||
const reasoningModes = ["identify_difference"];
|
||||
if (primaryType === "contradiction") reasoningModes.unshift("investigate_contradiction");
|
||||
if (primaryType === "decision_request" || primaryType === "desired_outcome") {
|
||||
reasoningModes.push("decision_support", "identify_missing_information");
|
||||
}
|
||||
if (hasComplaints || hasSales) {
|
||||
if (!reasoningModes.includes("establish_baseline")) {
|
||||
reasoningModes.unshift("establish_baseline");
|
||||
}
|
||||
}
|
||||
if (hasAmbiguous) reasoningModes.push("clarify_meaning");
|
||||
if (primaryType === "reported_claim") reasoningModes.push("validate_claim");
|
||||
if (!secondaryTypes.includes("unexplained_change") && primaryType === "unexplained_change") {
|
||||
reasoningModes.push("establish_baseline", "validate_measurement");
|
||||
}
|
||||
|
||||
return {
|
||||
inputClassification: {
|
||||
primaryType,
|
||||
secondaryTypes,
|
||||
reasoningModes,
|
||||
classificationReason: `Analyzing ${primaryType} with secondary types: ${secondaryTypes.join(", ") || "none"}. Input was evaluated for operational anchors including actors, states, differences, and evidence sources.`,
|
||||
confidence: hasComplaints ? "high" : "medium",
|
||||
},
|
||||
reconstruction: {
|
||||
summary: `${primaryType.charAt(0).toUpperCase() + primaryType.slice(1)} detected in input. The scenario involves ${hasComplaints ? "reported complaints" : hasSales ? "declining metrics" : "observed operational context"} that warrants further investigation to establish baseline and identify key differences.`,
|
||||
actors: [],
|
||||
systemsOrObjects: [],
|
||||
expectedStates: [],
|
||||
observedStates: [],
|
||||
differences: [hasSomeWord ? { id: "d1", description: "The input contains a subset modifier ('some'), indicating not universal applicability", confidence: "high", importance: "important" } : { id: "d1", description: "Key operational distinction identified in the scenario data", confidence: "medium", importance: "supporting" }],
|
||||
knownTransitions: [],
|
||||
unexplainedTransitions: [],
|
||||
contradictions: hasContradictionSignal ? [{ id: "c1", description: "Divergent signals detected between reported metrics and contextual anchors", confidence: "medium", importance: "important" }] : [],
|
||||
importantUnknowns: [hasComplaints ? { id: "u1", description: "Baseline period and absolute numbers for the complaint change", confidence: "high", importance: "critical" } : { id: "u1", description: "Contextual anchors needed to establish operational significance", confidence: "medium", importance: "supporting" }],
|
||||
plausibleInterpretations: [{ id: "pi1", description: "The situation represents a genuine operational issue requiring investigation", supportingEvidenceIds: ["d1"], assumptionsRequired: ["input contains meaningful operational content"], confidence: "medium" }],
|
||||
},
|
||||
evidence: [
|
||||
{ id: "e1", description: "Primary operational indicator detected in input text", evidenceType: "direct_observation", confidence: "high", importance: "supporting" },
|
||||
],
|
||||
nextQuestion: {
|
||||
id: "q1",
|
||||
question: hasComplaints ? "What is the baseline number of complaints and over what time period?" : "What specific metric or state should be used as the reference point?",
|
||||
targets: ["baseline_context", "measurement_period"],
|
||||
reason: "Establishing a reference point would distinguish whether the reported change is significant or within normal variation.",
|
||||
expectedInformationValue: "high",
|
||||
reasoningMode: "establish_baseline",
|
||||
},
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ── Display helpers ──────────────────────────────────
|
||||
|
||||
const CATEGORY_COLORS = {
|
||||
technical: "\x1b[36m", // cyan
|
||||
reasoning: "\x1b[33m", // yellow
|
||||
reset: "\x1b[0m",
|
||||
};
|
||||
|
||||
function categoryLabel(label) {
|
||||
return `${CATEGORY_COLORS.technical}${label}${CATEGORY_COLORS.reset}`;
|
||||
}
|
||||
|
||||
function reasonCategoryLabel() {
|
||||
return `${CATEGORY_COLORS.reasoning}reasoning quality${CATEGORY_COLORS.reset}`;
|
||||
}
|
||||
|
||||
// ── Main evaluation loop ─────────────────────────────
|
||||
async function main() {
|
||||
const testCases = loadTestCases(testDataPath);
|
||||
console.log(`\n⚡ Confidence Engine v0.2 — Evaluation Harness`);
|
||||
console.log(` Provider: ${useRealProvider ? "Ollama (real)" : "Mock"}`);
|
||||
console.log(` Cases loaded: ${testCases.length}\n`);
|
||||
|
||||
// Import or instantiate analysis function
|
||||
let analyseScenarioFn;
|
||||
if (useRealProvider) {
|
||||
const { analyseScenario } = await import("../lib/analysis.js");
|
||||
analyseScenarioFn = analyseScenario;
|
||||
} else {
|
||||
const mockProvider = new MockProvider();
|
||||
const schemaMod = await import("../lib/reconstruction/schema.js");
|
||||
const { reconstructionV2Schema, reconstructionSchema: reconstructionV1Schema } = schemaMod;
|
||||
const { buildPrompt } = await import("../lib/reconstruction/prompt.js");
|
||||
|
||||
analyseScenarioFn = async (scenario, opts = {}) => {
|
||||
const startTime = Date.now();
|
||||
const trimmed = scenario.trim();
|
||||
if (!trimmed) return { success: false, error: "Empty scenario", responseDurationMs: 0 };
|
||||
|
||||
let promptObj;
|
||||
try {
|
||||
promptObj = await buildPrompt(trimmed, opts.promptVersion || "v0.2");
|
||||
} catch {
|
||||
promptObj = { prompt: trimmed, version: "v0.2" };
|
||||
}
|
||||
|
||||
const mockResult = await mockProvider.generateReconstruction(promptObj.prompt, process.env.OLLAMA_MODEL || "mock-model");
|
||||
|
||||
let schemaValid = false;
|
||||
let validatedData = null;
|
||||
if (reconstructionV2Schema.safeParse) {
|
||||
const v2Result = reconstructionV2Schema.safeParse(mockResult);
|
||||
if (v2Result.success) {
|
||||
schemaValid = true;
|
||||
validatedData = v2Result.data;
|
||||
} else {
|
||||
const v1Result = reconstructionV1Schema.safeParse(mockResult);
|
||||
if (v1Result.success) {
|
||||
schemaValid = true;
|
||||
validatedData = v1Result.data;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!schemaValid || !validatedData) {
|
||||
return {
|
||||
success: false,
|
||||
validationStatus: "invalid",
|
||||
modelName: "mock-model",
|
||||
responseDurationMs: Date.now() - startTime,
|
||||
promptVersion: opts.promptVersion || "v0.2",
|
||||
reconstruction: null,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
success: true,
|
||||
validationStatus: "valid",
|
||||
modelName: "mock-model",
|
||||
responseDurationMs: Date.now() - startTime,
|
||||
promptVersion: opts.promptVersion || "v0.2",
|
||||
inputClassification: validatedData.inputClassification,
|
||||
reconstruction: validatedData.reconstruction,
|
||||
evidence: validatedData.evidence,
|
||||
nextQuestion: validatedData.nextQuestion,
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
// Run all cases
|
||||
const results = [];
|
||||
for (const tc of testCases) {
|
||||
process.stdout.write(` ${tc.id}: ... `);
|
||||
const r = await runTestCase(tc, analyseScenarioFn);
|
||||
results.push(r);
|
||||
const tStatus = r.technical.pass ? "\x1b[32m✅\x1b[0m" : "\x1b[31m❌\x1b[0m"; // green / red
|
||||
const rqStatus = r.reasoningQuality.pass ? "\x1b[32m✅\x1b[0m" : "\x1b[31m❌\x1b[0m";
|
||||
|
||||
process.stdout.write(`${tStatus} tech ${rqStatus} reason\n`);
|
||||
if (!r.technical.pass && r.technical.errors?.length) {
|
||||
for (const e of r.technical.errors.slice(0, 2)) process.stdout.write(` → [tech] ${e}\n`);
|
||||
} else if (!r.technical.pass) {
|
||||
const reasons = [];
|
||||
if (!r.technical.schemaValid) reasons.push("schema invalid");
|
||||
if (!r.technical.classificationMatch) reasons.push("classification mismatch");
|
||||
if (!r.technical.nextQuestionPresent) reasons.push("no next question");
|
||||
process.stdout.write(` → [tech] ${reasons.join(", ")}\n`);
|
||||
}
|
||||
|
||||
if (!r.reasoningQuality.pass) {
|
||||
const rqReasons = [];
|
||||
if (!r.reasoningQuality.requiredConcepts.pass) {
|
||||
rqReasons.push("missing required concept(s)");
|
||||
}
|
||||
if (!r.reasoningQuality.unsupportedInferencesAbsent.pass) {
|
||||
rqReasons.push("unsupported inference present");
|
||||
}
|
||||
process.stdout.write(` → [reasoning] ${rqReasons.join(", ")}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
// ── Compute summary stats ────────────────────────────
|
||||
const total = results.length;
|
||||
const techPassCount = results.filter((r) => r.technical.pass).length;
|
||||
const techSchemaValidCount = results.filter((r) => r.technical.schemaValid).length;
|
||||
const techClassificationMatchCount = results.filter((r) => r.technical.classificationMatch).length;
|
||||
const techNextQuestionPresentCount = results.filter((r) => r.technical.nextQuestionPresent).length;
|
||||
|
||||
const rqPassCount = results.filter((r) => r.reasoningQuality.pass).length;
|
||||
const rqConceptsPassCount = results.filter((r) => r.reasoningQuality.requiredConcepts.pass).length;
|
||||
const rqAbsencePassCount = results.filter((r) => r.reasoningQuality.unsupportedInferencesAbsent.pass).length;
|
||||
|
||||
const anyPassCount = results.filter(
|
||||
(r) => r.technical.pass && r.reasoningQuality.pass
|
||||
).length;
|
||||
|
||||
const avgDuration = total > 0
|
||||
? results.reduce((s, r) => s + (r.responseDurationMs || 0), 0) / total
|
||||
: 0;
|
||||
|
||||
const failedTechCases = results.filter((r) => !r.technical.pass);
|
||||
const failedRqCases = results.filter((r) => !r.reasoningQuality.pass);
|
||||
const techPassOnly = results.filter(
|
||||
(r) => r.technical.pass && !r.reasoningQuality.pass
|
||||
);
|
||||
const rqPassOnly = results.filter(
|
||||
(r) => !r.technical.pass && r.reasoningQuality.pass
|
||||
);
|
||||
|
||||
// ── Console summary ───────────────────────────────────
|
||||
console.log(`\n${"=".repeat(60)}`);
|
||||
console.log("EVALUATION SUMMARY");
|
||||
console.log(`${"=".repeat(60)}\n`);
|
||||
|
||||
console.log(`Cases run: ${total}\n`);
|
||||
|
||||
// Technical section
|
||||
console.log(categoryLabel("─── TECHNICAL ──────────────────────────────"));
|
||||
console.log(` Schema validity rate: ${techSchemaValidCount}/${total} ${(techSchemaValidCount / total * 100).toFixed(1)}%`);
|
||||
console.log(` Classification match: ${techClassificationMatchCount}/${total} ${(techClassificationMatchCount / total * 100).toFixed(1)}%`);
|
||||
console.log(` Next-question present: ${techNextQuestionPresentCount}/${total} ${(techNextQuestionPresentCount / total * 100).toFixed(1)}%`);
|
||||
console.log(` Technical pass rate: ${techPassCount}/${total} ${(techPassCount / total * 100).toFixed(1)}%\n`);
|
||||
|
||||
// Reasoning quality section
|
||||
console.log(reasonCategoryLabel() + " ─────────────────────────────");
|
||||
console.log(`${CATEGORY_COLORS.reset}`);
|
||||
console.log(` Required concept match: ${rqConceptsPassCount}/${total} ${(rqConceptsPassCount / total * 100).toFixed(1)}%`);
|
||||
console.log(` Unsupported inference absent: ${rqAbsencePassCount}/${total} ${(rqAbsencePassCount / total * 100).toFixed(1)}%`);
|
||||
console.log(` Reasoning quality pass: ${rqPassCount}/${total} ${(rqPassCount / total * 100).toFixed(1)}%\n`);
|
||||
|
||||
// Combined
|
||||
console.log(`${"─".repeat(60)}`);
|
||||
console.log(` Both technical + reasoning: ${anyPassCount}/${total} ${(anyPassCount / total * 100).toFixed(1)}%`);
|
||||
if (techPassOnly.length > 0) {
|
||||
console.log(` Technical only (hallucinated): ${techPassOnly.length} — IDs: ${techPassOnly.map((r) => r.id).join(", ")}`);
|
||||
}
|
||||
if (rqPassOnly.length > 0) {
|
||||
console.log(` Reasoning only (bad structure): ${rqPassOnly.length} — IDs: ${rqPassOnly.map((r) => r.id).join(", ")}`);
|
||||
}
|
||||
if (failedTechCases.length > 0 && failedRqCases.length > 0) {
|
||||
console.log(` Failed both: ${results.filter((r) => !r.technical.pass && !r.reasoningQuality.pass).length}`);
|
||||
}
|
||||
|
||||
console.log(` Avg response duration: ${avgDuration.toFixed(0)}ms`);
|
||||
console.log(`${"=".repeat(60)}\n`);
|
||||
|
||||
if (failedTechCases.length > 0) {
|
||||
console.log(`Failed technical — case IDs: ${failedTechCases.map((r) => r.id).join(", ")}`);
|
||||
}
|
||||
if (failedRqCases.length > 0) {
|
||||
console.log(`Failed reasoning quality — case IDs: ${failedRqCases.map((r) => r.id).join(", ")}`);
|
||||
}
|
||||
|
||||
// ── Save results ──────────────────────────────────────
|
||||
const timestamp = new Date().toISOString().replace(/[:.]/g, "-").slice(0, 19);
|
||||
|
||||
if (useDiagnostic) {
|
||||
// Live diagnostic: save to a dedicated result directory with per-case files + summary
|
||||
const caseResultDir = join(resultsDir, timestamp);
|
||||
mkdirSync(caseResultDir, { recursive: true });
|
||||
|
||||
// Per-case results JSON + Markdown
|
||||
for (const r of results) {
|
||||
const tc = testCases.find((t) => t.id === r.id);
|
||||
const caseFileBase = join(caseResultDir, r.id);
|
||||
|
||||
// Raw case result JSON
|
||||
writeFileSync(
|
||||
`${caseFileBase}-result.json`,
|
||||
JSON.stringify({
|
||||
id: r.id,
|
||||
description: tc?.description || "",
|
||||
input: tc?.input,
|
||||
responseDurationMs: r.responseDurationMs,
|
||||
actualPrimaryType: r.actualPrimaryType,
|
||||
actualReasoningModes: r.actualReasoningModes,
|
||||
rawOutput: r.rawOutput,
|
||||
technical: r.technical,
|
||||
reasoningQuality: r.reasoningQuality,
|
||||
}, null, 2)
|
||||
);
|
||||
|
||||
// Per-case Markdown summary
|
||||
const techStatus = r.technical.pass ? "✅ PASS" : "❌ FAIL";
|
||||
const rqStatus = r.reasoningQuality.pass ? "✅ PASS" : "❌ FAIL";
|
||||
|
||||
let md = `# Diagnostic Case: ${r.id}\n\n`;
|
||||
md += `${tc?.description || ""}\n\n`;
|
||||
md += `## Input\n\n\`\`\`\n${tc?.input || r.input}\n\`\`\`\n\n`;
|
||||
md += `## Result\n\n`;
|
||||
md += `- **Technical**: ${techStatus} (${(r.technical.pass ? 1 : 0)}/${Object.keys(r.technical).filter(k => typeof r.technical[k] === "boolean" && k !== "pass").length} sub-checks pass)\n`;
|
||||
md += `- **Reasoning Quality**: ${rqStatus} (${(r.reasoningQuality.pass ? 1 : 0)}/${2} sub-checks pass)\n`;
|
||||
md += `- **Actual Primary Type**: ${r.actualPrimaryType || "N/A"}\n`;
|
||||
md += `- **Actual Reasoning Modes**: ${(r.actualReasoningModes || []).join(", ") || "N/A"}\n`;
|
||||
md += `- **Response Duration**: ${r.responseDurationMs}ms\n`;
|
||||
|
||||
if (!r.technical.pass) {
|
||||
const reasons = [];
|
||||
if (!r.technical.schemaValid) reasons.push("schema invalid");
|
||||
if (!r.technical.classificationMatch) reasons.push("classification mismatch");
|
||||
if (!r.technical.nextQuestionPresent) reasons.push("no next question");
|
||||
md += `\n### Technical Failures\n\n${reasons.join(", ")}\n`;
|
||||
}
|
||||
|
||||
if (!r.reasoningQuality.pass) {
|
||||
const rqReasons = [];
|
||||
if (!r.reasoningQuality.requiredConcepts.pass) {
|
||||
rqReasons.push("missing required concept(s): " + r.reasoningQuality.requiredConcepts.details.filter(d => !d.found).map(d => d.concept).join(", ") || "unknown");
|
||||
}
|
||||
if (!r.reasoningQuality.unsupportedInferencesAbsent.pass) {
|
||||
rqReasons.push("unsupported inference present: " + r.reasoningQuality.unsupportedInferencesAbsent.details.filter(d => !d.absent).map(d => d.concept).join(", ") || "unknown");
|
||||
}
|
||||
md += `\n### Reasoning Quality Failures\n\n${rqReasons.join("\n")}\n`;
|
||||
}
|
||||
|
||||
writeFileSync(`${caseFileBase}-summary.md`, md);
|
||||
}
|
||||
|
||||
// Directory-level summary JSON
|
||||
const fullResults = {
|
||||
timestamp: new Date().toISOString(),
|
||||
provider: useRealProvider ? "ollama-real" : "mock",
|
||||
promptVersion: "v0.2",
|
||||
casesRun: total,
|
||||
summary: {
|
||||
technical: {
|
||||
schemaValidityRate: `${(techSchemaValidCount / total * 100).toFixed(1)}%`,
|
||||
classificationMatchRate: `${(techClassificationMatchCount / total * 100).toFixed(1)}%`,
|
||||
nextQuestionPresentRate: `${(techNextQuestionPresentCount / total * 100).toFixed(1)}%`,
|
||||
passRate: `${(techPassCount / total * 100).toFixed(1)}%`,
|
||||
},
|
||||
reasoningQuality: {
|
||||
requiredConceptMatchRate: `${(rqConceptsPassCount / total * 100).toFixed(1)}%`,
|
||||
unsupportedInferenceFailures: (total - rqAbsencePassCount).toString(),
|
||||
passRate: `${(rqPassCount / total * 100).toFixed(1)}%`,
|
||||
},
|
||||
combinedPassRate: `${(anyPassCount / total * 100).toFixed(1)}%`,
|
||||
averageResponseDurationMs: avgDuration.toFixed(0),
|
||||
},
|
||||
testCaseResults: results.map((r) => ({
|
||||
id: r.id,
|
||||
input: r.input,
|
||||
responseDurationMs: r.responseDurationMs,
|
||||
actualPrimaryType: r.actualPrimaryType,
|
||||
actualReasoningModes: r.actualReasoningModes,
|
||||
technical: {
|
||||
schemaValid: r.technical.schemaValid,
|
||||
classificationMatch: r.technical.classificationMatch,
|
||||
reasoningModeMatch: r.technical.reasoningModeMatch,
|
||||
nextQuestionPresent: r.technical.nextQuestionPresent,
|
||||
pass: r.technical.pass,
|
||||
errors: r.technical.errors,
|
||||
},
|
||||
reasoningQuality: {
|
||||
requiredConcepts: r.reasoningQuality.requiredConcepts,
|
||||
unsupportedInferencesAbsent: r.reasoningQuality.unsupportedInferencesAbsent,
|
||||
pass: r.reasoningQuality.pass,
|
||||
},
|
||||
})),
|
||||
};
|
||||
|
||||
writeFileSync(join(caseResultDir, "summary.json"), JSON.stringify(fullResults, null, 2));
|
||||
console.log(`Live diagnostic results saved to: ${caseResultDir}/`);
|
||||
|
||||
// Also save a top-level manifest pointing to the latest run
|
||||
const manifestPath = join(resultsDir, "latest-manifest.json");
|
||||
writeFileSync(manifestPath, JSON.stringify({ latestRun: timestamp, caseCount: total }, null, 2));
|
||||
console.log(`Manifest saved to: ${manifestPath}`);
|
||||
|
||||
} else {
|
||||
// Standard (non-diagnostic): single file output
|
||||
const resultsFile = join(resultsDir, `evaluation-${timestamp}.json`);
|
||||
const fullResults = {
|
||||
timestamp: new Date().toISOString(),
|
||||
provider: useRealProvider ? "ollama-real" : "mock",
|
||||
promptVersion: "v0.2",
|
||||
casesRun: total,
|
||||
summary: {
|
||||
technical: {
|
||||
schemaValidityRate: `${(techSchemaValidCount / total * 100).toFixed(1)}%`,
|
||||
classificationMatchRate: `${(techClassificationMatchCount / total * 100).toFixed(1)}%`,
|
||||
nextQuestionPresentRate: `${(techNextQuestionPresentCount / total * 100).toFixed(1)}%`,
|
||||
passRate: `${(techPassCount / total * 100).toFixed(1)}%`,
|
||||
},
|
||||
reasoningQuality: {
|
||||
requiredConceptMatchRate: `${(rqConceptsPassCount / total * 100).toFixed(1)}%`,
|
||||
unsupportedInferenceFailures: (total - rqAbsencePassCount).toString(),
|
||||
passRate: `${(rqPassCount / total * 100).toFixed(1)}%`,
|
||||
},
|
||||
combinedPassRate: `${(anyPassCount / total * 100).toFixed(1)}%`,
|
||||
averageResponseDurationMs: avgDuration.toFixed(0),
|
||||
},
|
||||
testCaseResults: results.map((r) => ({
|
||||
id: r.id,
|
||||
input: r.input,
|
||||
responseDurationMs: r.responseDurationMs,
|
||||
actualPrimaryType: r.actualPrimaryType,
|
||||
actualReasoningModes: r.actualReasoningModes,
|
||||
technical: {
|
||||
schemaValid: r.technical.schemaValid,
|
||||
classificationMatch: r.technical.classificationMatch,
|
||||
reasoningModeMatch: r.technical.reasoningModeMatch,
|
||||
nextQuestionPresent: r.technical.nextQuestionPresent,
|
||||
pass: r.technical.pass,
|
||||
errors: r.technical.errors,
|
||||
},
|
||||
reasoningQuality: {
|
||||
requiredConcepts: r.reasoningQuality.requiredConcepts,
|
||||
unsupportedInferencesAbsent: r.reasoningQuality.unsupportedInferencesAbsent,
|
||||
pass: r.reasoningQuality.pass,
|
||||
},
|
||||
})),
|
||||
};
|
||||
|
||||
writeFileSync(resultsFile, JSON.stringify(fullResults, null, 2));
|
||||
console.log(`Results saved to: ${resultsFile}`);
|
||||
console.log(`${"=".repeat(60)}\n`);
|
||||
}
|
||||
|
||||
// ── Close main() scope if we're in the non-diagnostic branch ──
|
||||
// (The if/else above handles result saving; main closes here)
|
||||
}
|
||||
|
||||
main().catch((e) => {
|
||||
console.error("Evaluator failed:", e.message);
|
||||
process.exit(1);
|
||||
});
|
||||
-198
@@ -1,198 +0,0 @@
|
||||
import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js";
|
||||
|
||||
function buildAmbiguityFixture({
|
||||
key,
|
||||
scenario,
|
||||
summaryLabel,
|
||||
contradictionLabel,
|
||||
observationLabels,
|
||||
unknownLabels,
|
||||
disallowedQuestionTerms,
|
||||
}) {
|
||||
const summary = makeNode({
|
||||
id: `${key}-summary`,
|
||||
label: summaryLabel,
|
||||
description: "Summary of the situation from the scenario text",
|
||||
kind: "state",
|
||||
status: "provisional",
|
||||
confidence: "medium",
|
||||
});
|
||||
|
||||
const contradiction = makeNode({
|
||||
id: `${key}-contradiction`,
|
||||
label: contradictionLabel,
|
||||
description: contradictionLabel,
|
||||
kind: "relationship",
|
||||
status: "supported",
|
||||
confidence: "medium",
|
||||
});
|
||||
|
||||
const observations = observationLabels.map((label, index) =>
|
||||
makeNode({
|
||||
id: `${key}-obs-${index + 1}`,
|
||||
label,
|
||||
description: label,
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
}),
|
||||
);
|
||||
|
||||
const unknowns = unknownLabels.map((label, index) =>
|
||||
makeNode({
|
||||
id: `${key}-unknown-${index + 1}`,
|
||||
label,
|
||||
description: label,
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
}),
|
||||
);
|
||||
|
||||
const edges = [
|
||||
...observations.map((node) =>
|
||||
makeEdge({
|
||||
id: `${node.id}-supports-summary`,
|
||||
fromNodeId: node.id,
|
||||
toNodeId: summary.id,
|
||||
relationship: "supports",
|
||||
description: `${node.label} supports the summary.`,
|
||||
}),
|
||||
),
|
||||
...unknowns.map((node) =>
|
||||
makeEdge({
|
||||
id: `${node.id}-depends-summary`,
|
||||
fromNodeId: node.id,
|
||||
toNodeId: summary.id,
|
||||
relationship: "depends_on",
|
||||
description: `${node.label} is an unresolved factor for this situation.`,
|
||||
}),
|
||||
),
|
||||
];
|
||||
|
||||
return {
|
||||
key,
|
||||
scenario,
|
||||
disallowedQuestionTerms,
|
||||
graph: makeGraph({
|
||||
centralStatement: scenario,
|
||||
nodes: [summary, contradiction, ...observations, ...unknowns],
|
||||
edges,
|
||||
activeUnknownNodeId: null,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: `Ambiguity fixture for ${key}`,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
export const ambiguityGeneralisationFixtures = [
|
||||
buildAmbiguityFixture({
|
||||
key: "revenue-cash",
|
||||
scenario:
|
||||
"Revenue increased by 18%, but cash in the bank fell over the same period.",
|
||||
summaryLabel: "Revenue rose while cash fell",
|
||||
contradictionLabel:
|
||||
"Contradiction between revenue improvement and lower cash reserves.",
|
||||
observationLabels: [
|
||||
"Revenue increased by 18%.",
|
||||
"Cash in the bank decreased over the same period.",
|
||||
],
|
||||
unknownLabels: [
|
||||
"Possible explanation for the contradiction from one side of the situation.",
|
||||
"Possible explanation for the contradiction from another side of the situation.",
|
||||
],
|
||||
disallowedQuestionTerms: [
|
||||
"accounts receivable",
|
||||
"capex",
|
||||
"debt repayments",
|
||||
"working capital",
|
||||
],
|
||||
}),
|
||||
buildAmbiguityFixture({
|
||||
key: "satisfaction-complaints",
|
||||
scenario:
|
||||
"Customer satisfaction scores increased, but complaints also increased.",
|
||||
summaryLabel: "Satisfaction scores rose while complaints also rose",
|
||||
contradictionLabel:
|
||||
"Contradiction between higher satisfaction scores and higher complaint volume.",
|
||||
observationLabels: [
|
||||
"Customer satisfaction scores increased.",
|
||||
"Complaints increased.",
|
||||
],
|
||||
unknownLabels: [
|
||||
"Possible explanation for why the positive signal and negative signal moved together.",
|
||||
"Another possible explanation for why the positive signal and negative signal moved together.",
|
||||
],
|
||||
disallowedQuestionTerms: [
|
||||
"net promoter",
|
||||
"ticket backlog",
|
||||
"call deflection",
|
||||
"support queue",
|
||||
],
|
||||
}),
|
||||
buildAmbiguityFixture({
|
||||
key: "delivery-cancellations",
|
||||
scenario:
|
||||
"Average delivery time decreased by 25%, but order cancellations increased.",
|
||||
summaryLabel: "Delivery became faster while cancellations increased",
|
||||
contradictionLabel:
|
||||
"Contradiction between faster delivery and more order cancellations.",
|
||||
observationLabels: [
|
||||
"Average delivery time decreased by 25%.",
|
||||
"Order cancellations increased.",
|
||||
],
|
||||
unknownLabels: [
|
||||
"Possible explanation for why the faster result did not reduce the negative result.",
|
||||
"Another possible explanation for why the faster result did not reduce the negative result.",
|
||||
],
|
||||
disallowedQuestionTerms: [
|
||||
"fulfilment",
|
||||
"last mile",
|
||||
"warehouse",
|
||||
"routing",
|
||||
],
|
||||
}),
|
||||
buildAmbiguityFixture({
|
||||
key: "traffic-sales",
|
||||
scenario: "Website traffic doubled, but sales remained unchanged.",
|
||||
summaryLabel: "Website traffic doubled while sales stayed flat",
|
||||
contradictionLabel:
|
||||
"Contradiction between much higher traffic and unchanged sales.",
|
||||
observationLabels: [
|
||||
"Website traffic doubled.",
|
||||
"Sales remained unchanged.",
|
||||
],
|
||||
unknownLabels: [
|
||||
"Possible explanation for why the stronger signal did not change the outcome.",
|
||||
"Another possible explanation for why the stronger signal did not change the outcome.",
|
||||
],
|
||||
disallowedQuestionTerms: [
|
||||
"conversion funnel",
|
||||
"campaign attribution",
|
||||
"landing page",
|
||||
"checkout flow",
|
||||
],
|
||||
}),
|
||||
buildAmbiguityFixture({
|
||||
key: "output-defects",
|
||||
scenario:
|
||||
"Production output increased by 30%, but quality defects also increased.",
|
||||
summaryLabel: "Production output rose while defects also rose",
|
||||
contradictionLabel:
|
||||
"Contradiction between higher output and more quality defects.",
|
||||
observationLabels: [
|
||||
"Production output increased by 30%.",
|
||||
"Quality defects increased.",
|
||||
],
|
||||
unknownLabels: [
|
||||
"Possible explanation for why the gain came with a worsening result.",
|
||||
"Another possible explanation for why the gain came with a worsening result.",
|
||||
],
|
||||
disallowedQuestionTerms: [
|
||||
"scrap rate",
|
||||
"throughput",
|
||||
"yield",
|
||||
"root cause",
|
||||
],
|
||||
}),
|
||||
];
|
||||
-164
@@ -1,164 +0,0 @@
|
||||
import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js";
|
||||
|
||||
function buildComparabilityFixture({
|
||||
key,
|
||||
scenario,
|
||||
observationLabels,
|
||||
contradictionLabel,
|
||||
expectedComparabilityStatus,
|
||||
expectsComparisonQuestion,
|
||||
}) {
|
||||
const summary = makeNode({
|
||||
id: `${key}-summary`,
|
||||
label: scenario,
|
||||
description: "Summary of the situation from the scenario text",
|
||||
kind: "state",
|
||||
status: "provisional",
|
||||
confidence: "medium",
|
||||
});
|
||||
|
||||
const observations = observationLabels.map((label, index) =>
|
||||
makeNode({
|
||||
id: `${key}-obs-${index + 1}`,
|
||||
label,
|
||||
description: label,
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
}),
|
||||
);
|
||||
|
||||
const contradiction = contradictionLabel
|
||||
? [
|
||||
makeNode({
|
||||
id: `${key}-contradiction`,
|
||||
label: contradictionLabel,
|
||||
description: contradictionLabel,
|
||||
kind: "relationship",
|
||||
status: "supported",
|
||||
confidence: "medium",
|
||||
}),
|
||||
]
|
||||
: [];
|
||||
|
||||
const unknowns = [
|
||||
makeNode({
|
||||
id: `${key}-unknown-a`,
|
||||
label: "Possible explanation from one side of the situation.",
|
||||
description: "Possible explanation from one side of the situation.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
}),
|
||||
makeNode({
|
||||
id: `${key}-unknown-b`,
|
||||
label: "Possible explanation from another side of the situation.",
|
||||
description: "Possible explanation from another side of the situation.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
}),
|
||||
];
|
||||
|
||||
const edges = [
|
||||
...observations.map((node) =>
|
||||
makeEdge({
|
||||
id: `${node.id}-supports-summary`,
|
||||
fromNodeId: node.id,
|
||||
toNodeId: summary.id,
|
||||
relationship: "supports",
|
||||
description: `${node.label} supports the summary.`,
|
||||
}),
|
||||
),
|
||||
...unknowns.map((node) =>
|
||||
makeEdge({
|
||||
id: `${node.id}-depends-summary`,
|
||||
fromNodeId: node.id,
|
||||
toNodeId: summary.id,
|
||||
relationship: "depends_on",
|
||||
description: `${node.label} is an unresolved factor for this situation.`,
|
||||
}),
|
||||
),
|
||||
];
|
||||
|
||||
return {
|
||||
key,
|
||||
scenario,
|
||||
expectedComparabilityStatus,
|
||||
expectsComparisonQuestion,
|
||||
graph: makeGraph({
|
||||
centralStatement: scenario,
|
||||
nodes: [summary, ...observations, ...contradiction, ...unknowns],
|
||||
edges,
|
||||
activeUnknownNodeId: null,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: `Comparability fixture for ${key}`,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
export const comparabilityAssessmentFixtures = [
|
||||
buildComparabilityFixture({
|
||||
key: "revenue-cash",
|
||||
scenario:
|
||||
"Revenue increased by 18%, but cash in the bank fell over the same period.",
|
||||
observationLabels: [
|
||||
"Revenue increased by 18%.",
|
||||
"Cash in the bank decreased over the same period.",
|
||||
],
|
||||
contradictionLabel:
|
||||
"Contradiction between revenue improvement and lower cash reserves.",
|
||||
expectedComparabilityStatus: "uncertain",
|
||||
expectsComparisonQuestion: true,
|
||||
}),
|
||||
buildComparabilityFixture({
|
||||
key: "complaints-production",
|
||||
scenario: "Complaints increased. Production increased.",
|
||||
observationLabels: ["Complaints increased.", "Production increased."],
|
||||
contradictionLabel:
|
||||
"Possible contradiction between complaints and production movement.",
|
||||
expectedComparabilityStatus: "uncertain",
|
||||
expectsComparisonQuestion: true,
|
||||
}),
|
||||
buildComparabilityFixture({
|
||||
key: "delivery-cancellations",
|
||||
scenario:
|
||||
"Average delivery time decreased by 25%, but order cancellations increased.",
|
||||
observationLabels: [
|
||||
"Average delivery time decreased by 25%.",
|
||||
"Order cancellations increased.",
|
||||
],
|
||||
contradictionLabel:
|
||||
"Contradiction between faster delivery and more cancellations.",
|
||||
expectedComparabilityStatus: "uncertain",
|
||||
expectsComparisonQuestion: true,
|
||||
}),
|
||||
buildComparabilityFixture({
|
||||
key: "satisfaction-complaints",
|
||||
scenario: "Customer satisfaction increased, but complaints increased.",
|
||||
observationLabels: [
|
||||
"Customer satisfaction increased.",
|
||||
"Complaints increased.",
|
||||
],
|
||||
contradictionLabel:
|
||||
"Contradiction between satisfaction improvement and more complaints.",
|
||||
expectedComparabilityStatus: "uncertain",
|
||||
expectsComparisonQuestion: true,
|
||||
}),
|
||||
buildComparabilityFixture({
|
||||
key: "temperature-ice",
|
||||
scenario: "Temperature increased. Ice melted.",
|
||||
observationLabels: ["Temperature increased.", "Ice melted."],
|
||||
contradictionLabel: null,
|
||||
expectedComparabilityStatus: "confirmed",
|
||||
expectsComparisonQuestion: false,
|
||||
}),
|
||||
buildComparabilityFixture({
|
||||
key: "sales-same",
|
||||
scenario: "Sales doubled. Sales doubled.",
|
||||
observationLabels: ["Sales doubled.", "Sales doubled."],
|
||||
contradictionLabel: null,
|
||||
expectedComparabilityStatus: "confirmed",
|
||||
expectsComparisonQuestion: false,
|
||||
}),
|
||||
];
|
||||
-445
@@ -1,445 +0,0 @@
|
||||
import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js";
|
||||
|
||||
function makeScenarioGraph({
|
||||
scenario,
|
||||
decisionNode,
|
||||
answeredContextUnknown,
|
||||
foundationalUnknown,
|
||||
consequentialUnknown,
|
||||
downstreamLeaf,
|
||||
}) {
|
||||
const nodes = [
|
||||
decisionNode,
|
||||
answeredContextUnknown,
|
||||
foundationalUnknown,
|
||||
consequentialUnknown,
|
||||
downstreamLeaf,
|
||||
];
|
||||
|
||||
const edges = [
|
||||
makeEdge({
|
||||
id: `${decisionNode.id}-to-${foundationalUnknown.id}`,
|
||||
fromNodeId: decisionNode.id,
|
||||
toNodeId: foundationalUnknown.id,
|
||||
relationship: "depends_on",
|
||||
description: `${decisionNode.label} depends on ${foundationalUnknown.label}.`,
|
||||
}),
|
||||
makeEdge({
|
||||
id: `${answeredContextUnknown.id}-to-${consequentialUnknown.id}`,
|
||||
fromNodeId: answeredContextUnknown.id,
|
||||
toNodeId: consequentialUnknown.id,
|
||||
relationship: "depends_on",
|
||||
description: `${consequentialUnknown.label} was surfaced from resolved context.`,
|
||||
}),
|
||||
makeEdge({
|
||||
id: `${foundationalUnknown.id}-to-${consequentialUnknown.id}`,
|
||||
fromNodeId: foundationalUnknown.id,
|
||||
toNodeId: consequentialUnknown.id,
|
||||
relationship: "depends_on",
|
||||
description: `${consequentialUnknown.label} depends on ${foundationalUnknown.label}.`,
|
||||
}),
|
||||
makeEdge({
|
||||
id: `${consequentialUnknown.id}-to-${downstreamLeaf.id}`,
|
||||
fromNodeId: consequentialUnknown.id,
|
||||
toNodeId: downstreamLeaf.id,
|
||||
relationship: "depends_on",
|
||||
description: `${downstreamLeaf.label} depends on ${consequentialUnknown.label}.`,
|
||||
}),
|
||||
];
|
||||
|
||||
return makeGraph({
|
||||
centralStatement: scenario,
|
||||
nodes,
|
||||
edges,
|
||||
activeUnknownNodeId: answeredContextUnknown.id,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: "Generalisation fixture graph",
|
||||
});
|
||||
}
|
||||
|
||||
export const questionPriorityGeneralisationFixtures = [
|
||||
{
|
||||
key: "hire-engineer",
|
||||
scenario: "Should we hire another engineer?",
|
||||
decisionType: "resourcing decision",
|
||||
acceptableFoundationalUnknownNodeIds: [
|
||||
"hire-success-criteria",
|
||||
"hire-bottleneck",
|
||||
],
|
||||
prohibitedFirstTopics: ["salary", "job advert", "programming language"],
|
||||
acceptableQuestionStrategies: [
|
||||
"decision_threshold",
|
||||
"evidence_gathering",
|
||||
"definition",
|
||||
],
|
||||
notes:
|
||||
"The first question should establish whether more engineering capacity is justified before compensation or implementation details.",
|
||||
graph: makeScenarioGraph({
|
||||
scenario: "Should we hire another engineer?",
|
||||
decisionNode: makeNode({
|
||||
id: "hire-decision",
|
||||
label: "Hiring another engineer decision",
|
||||
description: "Decision about increasing engineering capacity.",
|
||||
kind: "state",
|
||||
status: "known",
|
||||
confidence: "medium",
|
||||
value: "Deciding whether to hire another engineer",
|
||||
childIds: ["hire-success-criteria"],
|
||||
}),
|
||||
answeredContextUnknown: makeNode({
|
||||
id: "hire-delays-known",
|
||||
label: "Delivery delays established",
|
||||
description:
|
||||
"Need to confirm whether recent delivery delays are real because this context determines whether a capacity decision is even relevant.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
value:
|
||||
"The roadmap is slipping because the current team cannot clear the queue.",
|
||||
childIds: ["hire-bottleneck"],
|
||||
}),
|
||||
foundationalUnknown: makeNode({
|
||||
id: "hire-success-criteria",
|
||||
label: "Hiring success threshold",
|
||||
description:
|
||||
"Need the success threshold because the hiring decision depends on what improvement would justify adding headcount.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
parentId: "hire-decision",
|
||||
childIds: ["hire-bottleneck"],
|
||||
}),
|
||||
consequentialUnknown: makeNode({
|
||||
id: "hire-bottleneck",
|
||||
label: "Primary delivery bottleneck",
|
||||
description:
|
||||
"Need the main bottleneck because the team must know whether another engineer would relieve the limiting constraint.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
dependsOn: ["hire-success-criteria"],
|
||||
parentId: "hire-success-criteria",
|
||||
childIds: ["hire-salary"],
|
||||
}),
|
||||
downstreamLeaf: makeNode({
|
||||
id: "hire-salary",
|
||||
label: "Engineer salary budget",
|
||||
description:
|
||||
"Need the salary range because compensation planning comes after the hiring case is established.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
dependsOn: ["hire-bottleneck"],
|
||||
parentId: "hire-bottleneck",
|
||||
}),
|
||||
}),
|
||||
},
|
||||
{
|
||||
key: "replace-vans",
|
||||
scenario: "Should we replace the delivery vans?",
|
||||
decisionType: "asset replacement decision",
|
||||
acceptableFoundationalUnknownNodeIds: [
|
||||
"van-reliability-threshold",
|
||||
"van-service-constraint",
|
||||
],
|
||||
prohibitedFirstTopics: [
|
||||
"purchase price",
|
||||
"paint colour",
|
||||
"finance provider",
|
||||
],
|
||||
acceptableQuestionStrategies: [
|
||||
"decision_threshold",
|
||||
"evidence_gathering",
|
||||
"definition",
|
||||
],
|
||||
notes:
|
||||
"The first question should establish whether the fleet is failing a threshold that justifies replacement.",
|
||||
graph: makeScenarioGraph({
|
||||
scenario: "Should we replace the delivery vans?",
|
||||
decisionNode: makeNode({
|
||||
id: "van-decision",
|
||||
label: "Replace delivery vans decision",
|
||||
description: "Decision about replacing the current delivery fleet.",
|
||||
kind: "state",
|
||||
status: "known",
|
||||
confidence: "medium",
|
||||
value: "Deciding whether to replace the delivery vans",
|
||||
childIds: ["van-reliability-threshold"],
|
||||
}),
|
||||
answeredContextUnknown: makeNode({
|
||||
id: "van-breakdowns-known",
|
||||
label: "Breakdown trend confirmed",
|
||||
description:
|
||||
"Need to confirm whether the recent rise in breakdowns is real because that context determines whether fleet replacement is relevant.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
value:
|
||||
"Breakdowns and missed deliveries have increased over the last quarter.",
|
||||
childIds: ["van-service-constraint"],
|
||||
}),
|
||||
foundationalUnknown: makeNode({
|
||||
id: "van-reliability-threshold",
|
||||
label: "Replacement justification threshold",
|
||||
description:
|
||||
"Need the threshold because the replacement decision depends on what level of reliability loss is enough to justify replacing the fleet.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
parentId: "van-decision",
|
||||
childIds: ["van-service-constraint"],
|
||||
}),
|
||||
consequentialUnknown: makeNode({
|
||||
id: "van-service-constraint",
|
||||
label: "Operational service constraint",
|
||||
description:
|
||||
"Need the limiting service constraint because the team must know how vehicle unreliability is affecting deliveries before comparing purchasing options.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
dependsOn: ["van-reliability-threshold"],
|
||||
parentId: "van-reliability-threshold",
|
||||
childIds: ["van-price"],
|
||||
}),
|
||||
downstreamLeaf: makeNode({
|
||||
id: "van-price",
|
||||
label: "Exact replacement purchase price",
|
||||
description:
|
||||
"Need the exact purchase price because financing analysis comes after replacement is justified.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
dependsOn: ["van-service-constraint"],
|
||||
parentId: "van-service-constraint",
|
||||
}),
|
||||
}),
|
||||
},
|
||||
{
|
||||
key: "launch-country",
|
||||
scenario: "Should we launch in another country?",
|
||||
decisionType: "market expansion decision",
|
||||
acceptableFoundationalUnknownNodeIds: [
|
||||
"country-customer",
|
||||
"country-value-threshold",
|
||||
],
|
||||
prohibitedFirstTopics: [
|
||||
"launch date",
|
||||
"office location",
|
||||
"advertising channel",
|
||||
],
|
||||
acceptableQuestionStrategies: ["definition", "decision_threshold"],
|
||||
notes:
|
||||
"The first question should clarify the customer or value case for expansion before rollout logistics.",
|
||||
graph: makeScenarioGraph({
|
||||
scenario: "Should we launch in another country?",
|
||||
decisionNode: makeNode({
|
||||
id: "country-decision",
|
||||
label: "Launch in another country decision",
|
||||
description: "Decision about entering a new national market.",
|
||||
kind: "state",
|
||||
status: "known",
|
||||
confidence: "medium",
|
||||
value: "Deciding whether to launch in another country",
|
||||
childIds: ["country-customer"],
|
||||
}),
|
||||
answeredContextUnknown: makeNode({
|
||||
id: "country-interest-known",
|
||||
label: "Inbound interest confirmed",
|
||||
description:
|
||||
"Need to confirm whether inbound interest from another country is real because that context determines whether expansion is relevant.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
value:
|
||||
"Prospective customers from another country are asking for access.",
|
||||
childIds: ["country-value-threshold"],
|
||||
}),
|
||||
foundationalUnknown: makeNode({
|
||||
id: "country-customer",
|
||||
label: "Relevant customer in the new country",
|
||||
description:
|
||||
"Need the relevant customer because the expansion decision depends on who experiences the problem or receives the value in that market.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
parentId: "country-decision",
|
||||
childIds: ["country-value-threshold"],
|
||||
}),
|
||||
consequentialUnknown: makeNode({
|
||||
id: "country-value-threshold",
|
||||
label: "Expansion value threshold",
|
||||
description:
|
||||
"Need the value threshold because the team must know what evidence of demand or value would justify entering the new country.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
dependsOn: ["country-customer"],
|
||||
parentId: "country-customer",
|
||||
childIds: ["country-launch-date"],
|
||||
}),
|
||||
downstreamLeaf: makeNode({
|
||||
id: "country-launch-date",
|
||||
label: "Country launch date",
|
||||
description:
|
||||
"Need the launch date because rollout planning follows once the expansion case is established.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
dependsOn: ["country-value-threshold"],
|
||||
parentId: "country-value-threshold",
|
||||
}),
|
||||
}),
|
||||
},
|
||||
{
|
||||
key: "over-budget-project",
|
||||
scenario: "Should we continue a project that is over budget?",
|
||||
decisionType: "continuation decision",
|
||||
acceptableFoundationalUnknownNodeIds: [
|
||||
"project-benefit-threshold",
|
||||
"project-remaining-benefit",
|
||||
],
|
||||
prohibitedFirstTopics: ["sunk cost", "project logo", "final launch date"],
|
||||
acceptableQuestionStrategies: ["decision_threshold", "definition"],
|
||||
notes:
|
||||
"The first question should establish remaining value or success threshold before sunk-cost framing or launch timing.",
|
||||
graph: makeScenarioGraph({
|
||||
scenario: "Should we continue a project that is over budget?",
|
||||
decisionNode: makeNode({
|
||||
id: "project-decision",
|
||||
label: "Continue over-budget project decision",
|
||||
description:
|
||||
"Decision about continuing a project that has exceeded budget.",
|
||||
kind: "state",
|
||||
status: "known",
|
||||
confidence: "medium",
|
||||
value: "Deciding whether to continue the over-budget project",
|
||||
childIds: ["project-benefit-threshold"],
|
||||
}),
|
||||
answeredContextUnknown: makeNode({
|
||||
id: "project-overrun-known",
|
||||
label: "Budget overrun confirmed",
|
||||
description:
|
||||
"Need to confirm whether the project is materially over budget because that context determines whether a continuation decision is relevant.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
value: "The project has exceeded its approved budget by 35 percent.",
|
||||
childIds: ["project-remaining-benefit"],
|
||||
}),
|
||||
foundationalUnknown: makeNode({
|
||||
id: "project-benefit-threshold",
|
||||
label: "Continuation success threshold",
|
||||
description:
|
||||
"Need the threshold because the continuation decision depends on what remaining benefit would still justify completing the project.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
parentId: "project-decision",
|
||||
childIds: ["project-remaining-benefit"],
|
||||
}),
|
||||
consequentialUnknown: makeNode({
|
||||
id: "project-remaining-benefit",
|
||||
label: "Remaining project benefit",
|
||||
description:
|
||||
"Need the remaining benefit because the team must know what value is still achievable before deciding whether to continue.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
dependsOn: ["project-benefit-threshold"],
|
||||
parentId: "project-benefit-threshold",
|
||||
childIds: ["project-launch-date"],
|
||||
}),
|
||||
downstreamLeaf: makeNode({
|
||||
id: "project-launch-date",
|
||||
label: "Final launch date",
|
||||
description:
|
||||
"Need the final launch date because scheduling details only matter after remaining value is established.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
dependsOn: ["project-remaining-benefit"],
|
||||
parentId: "project-remaining-benefit",
|
||||
}),
|
||||
}),
|
||||
},
|
||||
{
|
||||
key: "paid-support-tier",
|
||||
scenario: "Should we introduce a paid support tier?",
|
||||
decisionType: "commercial packaging decision",
|
||||
acceptableFoundationalUnknownNodeIds: [
|
||||
"support-customer",
|
||||
"support-value-threshold",
|
||||
],
|
||||
prohibitedFirstTopics: [
|
||||
"subscription price",
|
||||
"payment provider",
|
||||
"tier name",
|
||||
],
|
||||
acceptableQuestionStrategies: [
|
||||
"definition",
|
||||
"decision_threshold",
|
||||
"baseline_reconstruction",
|
||||
],
|
||||
notes:
|
||||
"The first question should establish who values paid support or what outcome would justify offering it before pricing details.",
|
||||
graph: makeScenarioGraph({
|
||||
scenario: "Should we introduce a paid support tier?",
|
||||
decisionNode: makeNode({
|
||||
id: "support-decision",
|
||||
label: "Introduce paid support tier decision",
|
||||
description: "Decision about adding a paid support offering.",
|
||||
kind: "state",
|
||||
status: "known",
|
||||
confidence: "medium",
|
||||
value: "Deciding whether to introduce a paid support tier",
|
||||
childIds: ["support-customer"],
|
||||
}),
|
||||
answeredContextUnknown: makeNode({
|
||||
id: "support-requests-known",
|
||||
label: "Support request pattern confirmed",
|
||||
description:
|
||||
"Need to confirm whether repeated requests for faster support responses are real because that context determines whether a paid tier is relevant.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
value:
|
||||
"Some users are asking for guaranteed response times and escalation help.",
|
||||
childIds: ["support-value-threshold"],
|
||||
}),
|
||||
foundationalUnknown: makeNode({
|
||||
id: "support-customer",
|
||||
label: "Customer willing to pay for support",
|
||||
description:
|
||||
"Need the customer because the decision depends on who experiences enough support pain or receives enough value to pay for a support tier.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
parentId: "support-decision",
|
||||
childIds: ["support-value-threshold"],
|
||||
}),
|
||||
consequentialUnknown: makeNode({
|
||||
id: "support-value-threshold",
|
||||
label: "Paid support value threshold",
|
||||
description:
|
||||
"Need the value threshold because the team must know what outcome would justify introducing paid support before setting packaging details.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
dependsOn: ["support-customer"],
|
||||
parentId: "support-customer",
|
||||
childIds: ["support-price"],
|
||||
}),
|
||||
downstreamLeaf: makeNode({
|
||||
id: "support-price",
|
||||
label: "Support subscription price",
|
||||
description:
|
||||
"Need the subscription price because pricing and payment setup come after the support value case is established.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
dependsOn: ["support-value-threshold"],
|
||||
parentId: "support-value-threshold",
|
||||
}),
|
||||
}),
|
||||
},
|
||||
];
|
||||
@@ -1,131 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
formulateQuestion,
|
||||
formulateTieResolutionQuestion,
|
||||
} from "@/lib/graph/question-formulator.js";
|
||||
import {
|
||||
explainUnknownSelection,
|
||||
selectActiveUnknownCandidate,
|
||||
} from "@/lib/graph/utils.js";
|
||||
import { ambiguityGeneralisationFixtures } from "@/tests/fixtures/ambiguity-generalisation.js";
|
||||
|
||||
function neutraliseUnknownLabels(graph) {
|
||||
let counter = 0;
|
||||
return {
|
||||
...graph,
|
||||
nodes: graph.nodes.map((node) => {
|
||||
if (node.kind !== "unknown") return { ...node };
|
||||
counter += 1;
|
||||
return {
|
||||
...node,
|
||||
label: `Unknown ${String.fromCharCode(64 + counter)}`,
|
||||
description: `Unknown factor ${counter}.`,
|
||||
};
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function isSingleQuestion(question) {
|
||||
return (question.match(/\?/g) || []).length === 1;
|
||||
}
|
||||
|
||||
describe("ambiguity generalisation", () => {
|
||||
it("preserves ambiguity across contradiction scenarios without favouring one explanation", () => {
|
||||
const summary = ambiguityGeneralisationFixtures.map((fixture) => {
|
||||
const explanation = explainUnknownSelection(fixture.graph, []);
|
||||
const selection = selectActiveUnknownCandidate(fixture.graph, []);
|
||||
const neutralExplanation = explainUnknownSelection(
|
||||
neutraliseUnknownLabels(fixture.graph),
|
||||
[],
|
||||
);
|
||||
const tieQuestion = formulateTieResolutionQuestion({
|
||||
graph: fixture.graph,
|
||||
});
|
||||
const representativeUnknown = fixture.graph.nodes.find(
|
||||
(node) => node.kind === "unknown",
|
||||
);
|
||||
const fallbackQuestion = formulateQuestion({
|
||||
node: representativeUnknown,
|
||||
graph: fixture.graph,
|
||||
});
|
||||
|
||||
const lowerQuestion = tieQuestion.question.toLowerCase();
|
||||
for (const term of fixture.disallowedQuestionTerms) {
|
||||
expect(lowerQuestion).not.toContain(term.toLowerCase());
|
||||
}
|
||||
|
||||
expect(explanation.status).toBe("ambiguous");
|
||||
expect(selection.status).toBe("ambiguous");
|
||||
expect(selection.selectedNode).toBeNull();
|
||||
expect(explanation.selectedNodeId).toBeNull();
|
||||
expect(explanation.candidates).toHaveLength(2);
|
||||
expect(explanation.summary.selectedReason).toBe(
|
||||
"No justified distinction between leading unknowns.",
|
||||
);
|
||||
expect(explanation.alphabeticalUsedAsReasoning).toBe(false);
|
||||
expect(neutralExplanation.status).toBe("ambiguous");
|
||||
expect(isSingleQuestion(tieQuestion.question)).toBe(true);
|
||||
expect(tieQuestion.question.toLowerCase()).not.toContain(" or ");
|
||||
|
||||
return {
|
||||
scenario: fixture.scenario,
|
||||
candidateCount: explanation.candidates.length,
|
||||
ambiguityStatus: explanation.status,
|
||||
tieReason: explanation.summary.selectedReason,
|
||||
investigationStrategy: tieQuestion.strategy,
|
||||
question: tieQuestion.question,
|
||||
explanationFavoured: explanation.selectedNodeId !== null,
|
||||
};
|
||||
});
|
||||
|
||||
expect(summary).toMatchInlineSnapshot(`
|
||||
[
|
||||
{
|
||||
"ambiguityStatus": "ambiguous",
|
||||
"candidateCount": 2,
|
||||
"explanationFavoured": false,
|
||||
"investigationStrategy": null,
|
||||
"question": "Were these figures measured on the same basis and at the same scale?",
|
||||
"scenario": "Revenue increased by 18%, but cash in the bank fell over the same period.",
|
||||
"tieReason": "No justified distinction between leading unknowns.",
|
||||
},
|
||||
{
|
||||
"ambiguityStatus": "ambiguous",
|
||||
"candidateCount": 2,
|
||||
"explanationFavoured": false,
|
||||
"investigationStrategy": null,
|
||||
"question": "Were these figures measured over the same period and at the same scale?",
|
||||
"scenario": "Customer satisfaction scores increased, but complaints also increased.",
|
||||
"tieReason": "No justified distinction between leading unknowns.",
|
||||
},
|
||||
{
|
||||
"ambiguityStatus": "ambiguous",
|
||||
"candidateCount": 2,
|
||||
"explanationFavoured": false,
|
||||
"investigationStrategy": null,
|
||||
"question": "Were these figures measured over the same period and at the same scale?",
|
||||
"scenario": "Average delivery time decreased by 25%, but order cancellations increased.",
|
||||
"tieReason": "No justified distinction between leading unknowns.",
|
||||
},
|
||||
{
|
||||
"ambiguityStatus": "ambiguous",
|
||||
"candidateCount": 2,
|
||||
"explanationFavoured": false,
|
||||
"investigationStrategy": null,
|
||||
"question": "Were these figures measured over the same period and at the same scale?",
|
||||
"scenario": "Website traffic doubled, but sales remained unchanged.",
|
||||
"tieReason": "No justified distinction between leading unknowns.",
|
||||
},
|
||||
{
|
||||
"ambiguityStatus": "ambiguous",
|
||||
"candidateCount": 2,
|
||||
"explanationFavoured": false,
|
||||
"investigationStrategy": null,
|
||||
"question": "Were these figures measured over the same period and at the same scale?",
|
||||
"scenario": "Production output increased by 30%, but quality defects also increased.",
|
||||
"tieReason": "No justified distinction between leading unknowns.",
|
||||
},
|
||||
]
|
||||
`);
|
||||
});
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,118 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { assessUnknownAtomicity } from "@/lib/graph/question-formulator.js";
|
||||
import { makeGraph, makeNode } from "@/lib/graph/schema.js";
|
||||
|
||||
function makeGraphWithUnknown(centralStatement, unknown, observations = []) {
|
||||
return makeGraph({
|
||||
centralStatement,
|
||||
nodes: [unknown, ...observations],
|
||||
edges: [],
|
||||
activeUnknownNodeId: unknown.id,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: "Atomicity test graph",
|
||||
});
|
||||
}
|
||||
|
||||
describe("assessUnknownAtomicity", () => {
|
||||
it("classifies denominator-style unknowns as atomic", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-denominator",
|
||||
label: "Complaint rate denominator",
|
||||
description:
|
||||
"Need the denominator because it directly determines the complaint rate.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
});
|
||||
|
||||
const result = assessUnknownAtomicity({
|
||||
node: unknown,
|
||||
graph: makeGraphWithUnknown(
|
||||
"Production increased while complaints increased.",
|
||||
unknown,
|
||||
),
|
||||
});
|
||||
|
||||
expect(result.atomicity).toBe("atomic");
|
||||
expect(result.reason.toLowerCase()).toContain("directly");
|
||||
});
|
||||
|
||||
it("classifies relationship explanation unknowns as composite", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-explanation",
|
||||
label:
|
||||
"Explanation for why revenue increased by 18%, but cash in the bank fell over the same period",
|
||||
description:
|
||||
"Need to understand what change or event could explain why these observations differ, because that is needed to investigate their relationship.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
|
||||
const graph = makeGraphWithUnknown(
|
||||
"Revenue increased by 18%, but cash in the bank fell over the same period.",
|
||||
unknown,
|
||||
[
|
||||
makeNode({
|
||||
id: "n-revenue",
|
||||
label: "Revenue increased by 18%.",
|
||||
description: "Revenue increased by 18%.",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
}),
|
||||
makeNode({
|
||||
id: "n-cash",
|
||||
label: "Cash in the bank decreased over the same period.",
|
||||
description: "Cash in the bank decreased over the same period.",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
}),
|
||||
],
|
||||
);
|
||||
|
||||
const result = assessUnknownAtomicity({ node: unknown, graph });
|
||||
|
||||
expect(result.atomicity).toBe("composite");
|
||||
expect(result.decompositionKind).toBe("relationship_explanation");
|
||||
});
|
||||
|
||||
it.each([
|
||||
[
|
||||
"Customer satisfaction rose, but complaints also rose.",
|
||||
"Explanation for why customer satisfaction rose, but complaints also rose",
|
||||
],
|
||||
[
|
||||
"Delivery time fell, but cancellations increased.",
|
||||
"Possible causes of why delivery time fell, but cancellations increased",
|
||||
],
|
||||
[
|
||||
"Traffic increased, but sales stayed flat.",
|
||||
"Broad explanation for why traffic increased, but sales stayed flat",
|
||||
],
|
||||
[
|
||||
"Production increased, but defects also increased.",
|
||||
"Factors behind why production increased, but defects also increased",
|
||||
],
|
||||
])(
|
||||
"classifies broad divergence unknowns as composite: %s",
|
||||
(scenario, label) => {
|
||||
const unknown = makeNode({
|
||||
id: `n-${label.length}`,
|
||||
label,
|
||||
description: `${label} because the current unknown is too broad to ask directly.`,
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
|
||||
const result = assessUnknownAtomicity({
|
||||
node: unknown,
|
||||
graph: makeGraphWithUnknown(scenario, unknown),
|
||||
});
|
||||
|
||||
expect(result.atomicity).toBe("composite");
|
||||
},
|
||||
);
|
||||
});
|
||||
@@ -1,489 +0,0 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import {
|
||||
buildInitialGraph,
|
||||
buildMinimalGraph,
|
||||
describeGraph,
|
||||
} from "@/lib/graph/builder.js";
|
||||
import {
|
||||
makeNode,
|
||||
situationEdgeSchema,
|
||||
situationGraphSchema,
|
||||
situationNodeSchema,
|
||||
} from "@/lib/graph/schema.js";
|
||||
|
||||
// ── Helper: create a v0.3-style reconstruction fixture ───────────
|
||||
|
||||
function makeReconstructionFixture() {
|
||||
return {
|
||||
summary: "Company X reports revenue growth but increasing complaints",
|
||||
actors: [
|
||||
{ id: "actor-1", description: "Customer Base", confidence: "high" },
|
||||
{
|
||||
id: "actor-2",
|
||||
description: "Product Engineering Team",
|
||||
confidence: "high",
|
||||
},
|
||||
],
|
||||
systemsOrObjects: [
|
||||
{ id: "sys-1", description: "Production Line A", confidence: "high" },
|
||||
{
|
||||
id: "sys-2",
|
||||
description: "Quality Control System",
|
||||
confidence: "medium",
|
||||
},
|
||||
],
|
||||
expectedStates: [],
|
||||
observedStates: [
|
||||
{
|
||||
id: "obs-1",
|
||||
description: "Revenue up 15% year-over-year",
|
||||
confidence: "high",
|
||||
},
|
||||
{
|
||||
id: "obs-2",
|
||||
description: "Customer complaints up 40% year-over-year",
|
||||
confidence: "high",
|
||||
},
|
||||
],
|
||||
differences: [
|
||||
{
|
||||
id: "diff-1",
|
||||
description: "Complaint count grew faster than revenue",
|
||||
confidence: "medium",
|
||||
},
|
||||
],
|
||||
knownTransitions: [],
|
||||
unexplainedTransitions: [],
|
||||
contradictions: [
|
||||
{
|
||||
id: "con-1",
|
||||
description: "Revenue growth vs complaint growth inconsistency",
|
||||
confidence: "high",
|
||||
},
|
||||
],
|
||||
importantUnknowns: [
|
||||
{
|
||||
id: "unk-1",
|
||||
description: "Denominator for complaint rate (customers served)",
|
||||
confidence: "high",
|
||||
},
|
||||
{
|
||||
id: "unk-2",
|
||||
description: "Root cause of complaint increase",
|
||||
confidence: "medium",
|
||||
},
|
||||
],
|
||||
plausibleInterpretations: [],
|
||||
};
|
||||
}
|
||||
|
||||
function makeEvidenceFixture() {
|
||||
return [
|
||||
{
|
||||
id: "ev-1",
|
||||
description: "Annual report data",
|
||||
evidenceType: "direct_observation",
|
||||
confidence: "high",
|
||||
importance: "critical",
|
||||
},
|
||||
{
|
||||
id: "ev-2",
|
||||
description: "Customer survey results",
|
||||
evidenceType: "reported_statement",
|
||||
confidence: "medium",
|
||||
importance: "important",
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
describe("buildInitialGraph", () => {
|
||||
it("builds nodes from reconstruction data", () => {
|
||||
const result = buildInitialGraph({
|
||||
reconstruction: makeReconstructionFixture(),
|
||||
evidence: makeEvidenceFixture(),
|
||||
});
|
||||
|
||||
expect(result.nodes.length).toBeGreaterThan(0);
|
||||
expect(result.edges.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("creates a summary node", () => {
|
||||
const result = buildInitialGraph({
|
||||
reconstruction: makeReconstructionFixture(),
|
||||
evidence: [],
|
||||
});
|
||||
|
||||
const summaryNode = result.nodes.find((n) => n.kind === "state");
|
||||
expect(summaryNode).toBeDefined();
|
||||
expect(summaryNode.label).toBe(
|
||||
"Company X reports revenue growth but increasing complaints",
|
||||
);
|
||||
});
|
||||
|
||||
it("creates observation nodes from observedStates", () => {
|
||||
const result = buildInitialGraph({
|
||||
reconstruction: makeReconstructionFixture(),
|
||||
evidence: [],
|
||||
});
|
||||
|
||||
const observations = result.nodes.filter((n) => n.kind === "observation");
|
||||
expect(observations.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("creates unknown nodes from importantUnknowns", () => {
|
||||
const result = buildInitialGraph({
|
||||
reconstruction: makeReconstructionFixture(),
|
||||
evidence: [],
|
||||
});
|
||||
|
||||
const unknowns = result.nodes.filter((n) => n.kind === "unknown");
|
||||
expect(unknowns.length).toBe(2); // unk-1 and unk-2
|
||||
});
|
||||
|
||||
it("creates metric nodes from systemsOrObjects", () => {
|
||||
const result = buildInitialGraph({
|
||||
reconstruction: makeReconstructionFixture(),
|
||||
evidence: [],
|
||||
});
|
||||
|
||||
const metrics = result.nodes.filter((n) => n.kind === "metric");
|
||||
expect(metrics.length).toBe(2); // sys-1 and sys-2
|
||||
});
|
||||
|
||||
it("creates actor nodes as observations", () => {
|
||||
const result = buildInitialGraph({
|
||||
reconstruction: makeReconstructionFixture(),
|
||||
evidence: [],
|
||||
});
|
||||
|
||||
const actors = result.nodes.filter(
|
||||
(n) =>
|
||||
n.label.includes("Customer Base") ||
|
||||
n.label.includes("Product Engineering"),
|
||||
);
|
||||
expect(actors.length).toBe(2);
|
||||
});
|
||||
|
||||
it("creates difference nodes", () => {
|
||||
const result = buildInitialGraph({
|
||||
reconstruction: makeReconstructionFixture(),
|
||||
evidence: [],
|
||||
});
|
||||
|
||||
const differenceNode = result.nodes.find((n) =>
|
||||
n.label.includes("Complaint count grew faster than revenue"),
|
||||
);
|
||||
|
||||
expect(differenceNode).toBeDefined();
|
||||
expect(differenceNode.kind).toBe("relationship");
|
||||
});
|
||||
|
||||
it("creates contradiction nodes", () => {
|
||||
const result = buildInitialGraph({
|
||||
reconstruction: makeReconstructionFixture(),
|
||||
evidence: [],
|
||||
});
|
||||
|
||||
const contradictionNode = result.nodes.find((n) =>
|
||||
n.label.includes("inconsistency"),
|
||||
);
|
||||
|
||||
expect(contradictionNode).toBeDefined();
|
||||
expect(contradictionNode.kind).toBe("relationship");
|
||||
});
|
||||
|
||||
it("creates edges linking observations to summary", () => {
|
||||
const result = buildInitialGraph({
|
||||
reconstruction: makeReconstructionFixture(),
|
||||
evidence: [],
|
||||
});
|
||||
|
||||
const supportEdges = result.edges.filter(
|
||||
(e) => e.relationship === "supports",
|
||||
);
|
||||
expect(supportEdges.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("creates edges linking unknowns to summary as depends_on", () => {
|
||||
const result = buildInitialGraph({
|
||||
reconstruction: makeReconstructionFixture(),
|
||||
evidence: [],
|
||||
});
|
||||
|
||||
const depEdges = result.edges.filter(
|
||||
(e) => e.relationship === "depends_on",
|
||||
);
|
||||
expect(depEdges.length).toBe(2); // Two unknown nodes
|
||||
});
|
||||
|
||||
it("handles empty observedStates gracefully", () => {
|
||||
const reconstruction = {
|
||||
...makeReconstructionFixture(),
|
||||
observedStates: [],
|
||||
};
|
||||
const result = buildInitialGraph({ reconstruction, evidence: [] });
|
||||
|
||||
expect(result.nodes.length).toBeGreaterThan(0); // Summary + actors + systems still created
|
||||
});
|
||||
|
||||
it("handles missing reconstruction fields gracefully", () => {
|
||||
const result = buildInitialGraph({
|
||||
reconstruction: { summary: "Minimal" },
|
||||
evidence: [],
|
||||
});
|
||||
|
||||
expect(result.nodes.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("handles null/undefined reconstruction", () => {
|
||||
const result = buildInitialGraph({ reconstruction: null, evidence: [] });
|
||||
expect(result.nodes.length).toBe(0);
|
||||
expect(result.edges.length).toBe(0);
|
||||
});
|
||||
|
||||
it("handles missing evidence array", () => {
|
||||
const result = buildInitialGraph({
|
||||
reconstruction: makeReconstructionFixture(),
|
||||
});
|
||||
|
||||
expect(result.nodes.length).toBeGreaterThan(0);
|
||||
expect(result.edges.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("generates deterministic node IDs for same labels", () => {
|
||||
const r1 = buildInitialGraph({
|
||||
reconstruction: makeReconstructionFixture(),
|
||||
evidence: [],
|
||||
});
|
||||
const r2 = buildInitialGraph({
|
||||
reconstruction: makeReconstructionFixture(),
|
||||
evidence: [],
|
||||
});
|
||||
|
||||
const ids1 = r1.nodes.map((n) => n.id).sort();
|
||||
const ids2 = r2.nodes.map((n) => n.id).sort();
|
||||
expect(ids1).toEqual(ids2);
|
||||
});
|
||||
|
||||
it("produces valid schema output (no parse errors)", () => {
|
||||
const result = buildInitialGraph({
|
||||
reconstruction: makeReconstructionFixture(),
|
||||
evidence: makeEvidenceFixture(),
|
||||
});
|
||||
|
||||
for (const node of result.nodes) {
|
||||
const parsed = situationNodeSchema.safeParse(node);
|
||||
if (!parsed.success) {
|
||||
console.error(`Invalid node: ${node.id}`, node, parsed.error.message);
|
||||
}
|
||||
expect(parsed.success).toBe(true);
|
||||
}
|
||||
|
||||
for (const edge of result.edges) {
|
||||
const parsed = situationEdgeSchema.safeParse(edge);
|
||||
if (!parsed.success) {
|
||||
console.error(`Invalid edge: ${edge.id}`, edge, parsed.error.message);
|
||||
}
|
||||
expect(parsed.success).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it("creates edges for knownTransitions as transition nodes", () => {
|
||||
const reconstruction = {
|
||||
...makeReconstructionFixture(),
|
||||
knownTransitions: [
|
||||
{
|
||||
id: "trans-1",
|
||||
description: "Product shipped v2.0",
|
||||
entity: "Product",
|
||||
previousState: "v1.x",
|
||||
currentState: "v2.0",
|
||||
explanationStatus: "confirmed",
|
||||
confidence: "high",
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const result = buildInitialGraph({ reconstruction, evidence: [] });
|
||||
const transitions = result.nodes.filter((n) => n.kind === "transition");
|
||||
expect(transitions.length).toBe(1);
|
||||
});
|
||||
|
||||
it("creates nodes for unexplainedTransitions", () => {
|
||||
const reconstruction = {
|
||||
...makeReconstructionFixture(),
|
||||
unexplainedTransitions: [
|
||||
{
|
||||
id: "ut-1",
|
||||
description: "Support wait time increased",
|
||||
entity: "Support",
|
||||
previousState: "2hr",
|
||||
currentState: "8hr",
|
||||
confidence: "medium",
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const result = buildInitialGraph({ reconstruction, evidence: [] });
|
||||
expect(result.nodes.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("creates nodes for plausibleInterpretations as assumptions", () => {
|
||||
const reconstruction = {
|
||||
...makeReconstructionFixture(),
|
||||
plausibleInterpretations: [
|
||||
{
|
||||
id: "interp-1",
|
||||
description: "Quality degradation hypothesis",
|
||||
supportingEvidenceIds: ["ev-2"],
|
||||
assumptionsRequired: [],
|
||||
confidence: "medium",
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const result = buildInitialGraph({ reconstruction, evidence: [] });
|
||||
const assumptions = result.nodes.filter((n) => n.kind === "assumption");
|
||||
expect(assumptions.length).toBe(1);
|
||||
});
|
||||
|
||||
it("links evidence to observation nodes", () => {
|
||||
const reconstruction = makeReconstructionFixture();
|
||||
const evidence = [{ id: "ev-1", description: "Test evidence" }];
|
||||
|
||||
// Add a mapping from observed states to evidence IDs would require modification
|
||||
// For now, just verify the nodes have empty evidenceIds (as per current implementation)
|
||||
const result = buildInitialGraph({ reconstruction, evidence });
|
||||
for (const node of result.nodes) {
|
||||
expect(Array.isArray(node.evidenceIds)).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it("handles very large reconstruction without errors", () => {
|
||||
const actors = Array.from({ length: 20 }, (_, i) => ({
|
||||
id: `actor-${i}`,
|
||||
description: `Actor ${i}`,
|
||||
confidence: "high",
|
||||
}));
|
||||
|
||||
const result = buildInitialGraph({
|
||||
reconstruction: { ...makeReconstructionFixture(), actors },
|
||||
evidence: [],
|
||||
});
|
||||
|
||||
expect(result.nodes.length).toBeGreaterThan(10);
|
||||
});
|
||||
|
||||
it("handles transition with confirmed explanation", () => {
|
||||
const reconstruction = {
|
||||
...makeReconstructionFixture(),
|
||||
knownTransitions: [
|
||||
{
|
||||
id: "t-confirmed",
|
||||
description: "Confirmed event",
|
||||
entity: "E1",
|
||||
previousState: "s1",
|
||||
currentState: "s2",
|
||||
explanationStatus: "confirmed",
|
||||
confidence: "high",
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const result = buildInitialGraph({ reconstruction, evidence: [] });
|
||||
const confirmedTransitions = result.nodes.filter(
|
||||
(n) => n.kind === "transition" && n.status === "known",
|
||||
);
|
||||
expect(confirmedTransitions.length).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("buildMinimalGraph", () => {
|
||||
it("creates a single node with scenario text as label", () => {
|
||||
const graph = buildMinimalGraph(
|
||||
"This is a test scenario for minimal graph creation",
|
||||
);
|
||||
expect(graph.nodes.length).toBe(1);
|
||||
expect(graph.edges.length).toBe(0);
|
||||
});
|
||||
|
||||
it("truncates label to 80 chars", () => {
|
||||
const longScenario = "a".repeat(200);
|
||||
const graph = buildMinimalGraph(longScenario);
|
||||
expect(graph.nodes[0].label.length).toBeLessThanOrEqual(80);
|
||||
});
|
||||
|
||||
it("creates provisional state node", () => {
|
||||
const graph = buildMinimalGraph("Test scenario");
|
||||
expect(graph.nodes[0].kind).toBe("state");
|
||||
expect(graph.nodes[0].status).toBe("provisional");
|
||||
expect(graph.nodes[0].confidence).toBe("low");
|
||||
});
|
||||
|
||||
it("uses first 200 chars of scenario for description", () => {
|
||||
const graph = buildMinimalGraph(
|
||||
"This is a test scenario for minimal graph creation",
|
||||
);
|
||||
expect(graph.nodes[0].description).toContain("Initial situation from:");
|
||||
});
|
||||
|
||||
it("creates deterministic ID via situationNodeSchema.parse", () => {
|
||||
const graph = buildMinimalGraph("Test scenario");
|
||||
// Node has explicit id "n0" from the builder, not makeNodeId
|
||||
expect(graph.nodes[0].id).toBe("n0");
|
||||
});
|
||||
|
||||
it("creates minimal valid structure", () => {
|
||||
const graph = buildMinimalGraph("Test");
|
||||
expect(graph.nodes).toHaveLength(1);
|
||||
expect(graph.edges).toHaveLength(0);
|
||||
expect(graph.nodes[0].evidenceIds).toEqual([]);
|
||||
expect(graph.nodes[0].dependsOn).toEqual([]);
|
||||
expect(graph.nodes[0].affects).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("describeGraph", () => {
|
||||
it("returns summary string with node count by kind", () => {
|
||||
const graph = buildMinimalGraph("Test");
|
||||
const description = describeGraph(graph);
|
||||
|
||||
expect(description).toContain("Nodes:");
|
||||
expect(description).toContain("Edges:");
|
||||
expect(description).toContain("Unknowns:");
|
||||
});
|
||||
|
||||
it("shows correct edge count", () => {
|
||||
const graph = buildMinimalGraph("Test");
|
||||
const description = describeGraph(graph);
|
||||
|
||||
expect(description).toContain("Edges: 0 total");
|
||||
});
|
||||
|
||||
it("counts unresolved unknowns", () => {
|
||||
const n1 = makeNode({
|
||||
id: "n-unk",
|
||||
label: "Unknown",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
});
|
||||
const graph = situationGraphSchema.parse({
|
||||
centralStatement: "Test",
|
||||
nodes: [n1],
|
||||
edges: [],
|
||||
activeUnknownNodeId: n1.id,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: "Test",
|
||||
});
|
||||
|
||||
const description = describeGraph(graph);
|
||||
expect(description).toContain("1"); // One unresolved unknown
|
||||
});
|
||||
|
||||
it("groups nodes by kind in output", () => {
|
||||
const graph = buildMinimalGraph("Test");
|
||||
const description = describeGraph(graph);
|
||||
|
||||
expect(description).toContain("1 state");
|
||||
});
|
||||
});
|
||||
@@ -1,180 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
assessComparability,
|
||||
classifyObservationRelationship,
|
||||
formulateTieResolutionQuestion,
|
||||
} from "@/lib/graph/question-formulator.js";
|
||||
import { explainUnknownSelection } from "@/lib/graph/utils.js";
|
||||
import { comparabilityAssessmentFixtures } from "@/tests/fixtures/comparability-assessment.js";
|
||||
|
||||
describe("comparability assessment", () => {
|
||||
it("generates comparison or relationship questions only when warranted", () => {
|
||||
const summary = comparabilityAssessmentFixtures.map((fixture) => {
|
||||
const assessment = assessComparability(fixture.graph);
|
||||
const relationship = classifyObservationRelationship(fixture.graph);
|
||||
const question = formulateTieResolutionQuestion({ graph: fixture.graph });
|
||||
const ambiguity = explainUnknownSelection(fixture.graph, []);
|
||||
|
||||
expect(assessment.comparabilityStatus).toBe(
|
||||
fixture.expectedComparabilityStatus,
|
||||
);
|
||||
expect(question.comparabilityStatus).toBe(
|
||||
fixture.expectedComparabilityStatus,
|
||||
);
|
||||
|
||||
if (fixture.expectsComparisonQuestion) {
|
||||
expect(question.question.toLowerCase()).toContain("same");
|
||||
expect(question.contradictionReasoningAllowed).toBe(false);
|
||||
} else {
|
||||
expect(question.question?.toLowerCase() || "").not.toContain(
|
||||
"same period and at the same scale",
|
||||
);
|
||||
}
|
||||
|
||||
if (fixture.key !== "sales-same") {
|
||||
expect(ambiguity.status).toBe("ambiguous");
|
||||
}
|
||||
|
||||
return {
|
||||
scenario: fixture.scenario,
|
||||
comparabilityStatus: assessment.comparabilityStatus,
|
||||
relationshipStatus: relationship.relationshipStatus,
|
||||
relationshipAssessed: relationship.relationshipAssessed,
|
||||
contradictionReasoningAllowed: question.contradictionReasoningAllowed,
|
||||
question: question.question,
|
||||
};
|
||||
});
|
||||
|
||||
expect(summary).toEqual([
|
||||
{
|
||||
scenario:
|
||||
"Revenue increased by 18%, but cash in the bank fell over the same period.",
|
||||
comparabilityStatus: "uncertain",
|
||||
relationshipStatus: "insufficient_information",
|
||||
relationshipAssessed: false,
|
||||
contradictionReasoningAllowed: false,
|
||||
question:
|
||||
"Were these figures measured on the same basis and at the same scale?",
|
||||
},
|
||||
{
|
||||
scenario: "Complaints increased. Production increased.",
|
||||
comparabilityStatus: "uncertain",
|
||||
relationshipStatus: "insufficient_information",
|
||||
relationshipAssessed: false,
|
||||
contradictionReasoningAllowed: false,
|
||||
question:
|
||||
"Were these figures measured over the same period and at the same scale?",
|
||||
},
|
||||
{
|
||||
scenario:
|
||||
"Average delivery time decreased by 25%, but order cancellations increased.",
|
||||
comparabilityStatus: "uncertain",
|
||||
relationshipStatus: "insufficient_information",
|
||||
relationshipAssessed: false,
|
||||
contradictionReasoningAllowed: false,
|
||||
question:
|
||||
"Were these figures measured over the same period and at the same scale?",
|
||||
},
|
||||
{
|
||||
scenario: "Customer satisfaction increased, but complaints increased.",
|
||||
comparabilityStatus: "uncertain",
|
||||
relationshipStatus: "insufficient_information",
|
||||
relationshipAssessed: false,
|
||||
contradictionReasoningAllowed: false,
|
||||
question:
|
||||
"Were these figures measured over the same period and at the same scale?",
|
||||
},
|
||||
{
|
||||
scenario: "Temperature increased. Ice melted.",
|
||||
comparabilityStatus: "confirmed",
|
||||
relationshipStatus: "compatible",
|
||||
relationshipAssessed: true,
|
||||
contradictionReasoningAllowed: false,
|
||||
question: null,
|
||||
},
|
||||
{
|
||||
scenario: "Sales doubled. Sales doubled.",
|
||||
comparabilityStatus: "confirmed",
|
||||
relationshipStatus: "duplicate",
|
||||
relationshipAssessed: true,
|
||||
contradictionReasoningAllowed: false,
|
||||
question: null,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("defers relationship classification while comparability is uncertain", () => {
|
||||
const fixture = comparabilityAssessmentFixtures[0];
|
||||
const relationship = classifyObservationRelationship(fixture.graph);
|
||||
|
||||
expect(relationship).toMatchObject({
|
||||
relationshipStatus: "insufficient_information",
|
||||
relationshipAssessed: false,
|
||||
contradictionReasoningAllowed: false,
|
||||
questionRequired: true,
|
||||
});
|
||||
expect(relationship.reasoningStages).toEqual([
|
||||
{
|
||||
stage: "comparability",
|
||||
status: "uncertain",
|
||||
outcome:
|
||||
"Comparability between the observations is not yet established across period, scale, or measurement basis.",
|
||||
},
|
||||
{
|
||||
stage: "relationship",
|
||||
status: "insufficient_information",
|
||||
outcome: "not assessed until comparability is established",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("allows contradiction reasoning only for genuine contradictions", () => {
|
||||
const serviceGraph = {
|
||||
centralStatement:
|
||||
"The service was reported as available throughout the hour and unavailable throughout the same hour.",
|
||||
nodes: [
|
||||
{
|
||||
id: "service-available",
|
||||
label: "The service was available throughout the hour.",
|
||||
description: "The service was available throughout the hour.",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
value: null,
|
||||
unit: null,
|
||||
evidenceIds: [],
|
||||
dependsOn: [],
|
||||
affects: [],
|
||||
parentId: null,
|
||||
childIds: [],
|
||||
},
|
||||
{
|
||||
id: "service-unavailable",
|
||||
label: "The service was unavailable throughout the same hour.",
|
||||
description: "The service was unavailable throughout the same hour.",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
value: null,
|
||||
unit: null,
|
||||
evidenceIds: [],
|
||||
dependsOn: [],
|
||||
affects: [],
|
||||
parentId: null,
|
||||
childIds: [],
|
||||
},
|
||||
],
|
||||
edges: [],
|
||||
activeUnknownNodeId: null,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: "Service contradiction fixture",
|
||||
};
|
||||
const relationship = classifyObservationRelationship(serviceGraph);
|
||||
|
||||
expect(relationship).toMatchObject({
|
||||
relationshipStatus: "contradictory",
|
||||
contradictionReasoningAllowed: true,
|
||||
questionRequired: true,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -1,179 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { applyValidatedProposal } from "@/lib/graph/apply-proposal.js";
|
||||
import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js";
|
||||
|
||||
function makeFixture() {
|
||||
const parent = makeNode({
|
||||
id: "n-parent",
|
||||
label: "Explanation for why revenue increased while cash fell",
|
||||
description:
|
||||
"Need to understand what change or event could explain why these observations differ, because that is needed to investigate their relationship.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
const children = [
|
||||
makeNode({
|
||||
id: "n-child-1",
|
||||
label: "How the two observations were measured",
|
||||
description: "Need evidence about the measure used for each observation.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
parentId: parent.id,
|
||||
}),
|
||||
makeNode({
|
||||
id: "n-child-2",
|
||||
label: "Whether the two observations reflect different timing",
|
||||
description:
|
||||
"Need to know whether the two observations reflect different timing.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
parentId: parent.id,
|
||||
}),
|
||||
makeNode({
|
||||
id: "n-child-3",
|
||||
label: "Possible change mainly affecting revenue",
|
||||
description:
|
||||
"Need to know whether a possible change mainly affected revenue.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
parentId: parent.id,
|
||||
}),
|
||||
makeNode({
|
||||
id: "n-child-4",
|
||||
label: "Possible one-off event during the period",
|
||||
description:
|
||||
"Need to know whether a possible one-off event happened during the period.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
parentId: parent.id,
|
||||
}),
|
||||
];
|
||||
|
||||
return makeGraph({
|
||||
centralStatement: "Revenue increased while cash fell.",
|
||||
nodes: [parent, ...children],
|
||||
edges: children.map((child, index) =>
|
||||
makeEdge({
|
||||
id: `e-${index + 1}`,
|
||||
fromNodeId: child.id,
|
||||
toNodeId: parent.id,
|
||||
relationship: "depends_on",
|
||||
description: `${child.label} feeds the parent.`,
|
||||
}),
|
||||
),
|
||||
activeUnknownNodeId: "n-child-1",
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: "confidence propagation fixture",
|
||||
});
|
||||
}
|
||||
|
||||
function makeProposal({ resolvedIds, contradictedIds = [] }) {
|
||||
return {
|
||||
addedNodes: [
|
||||
makeNode({
|
||||
id: "n-anchor",
|
||||
label: "Update anchor",
|
||||
description:
|
||||
"Anchor state introduced by the answer because the update must contain a meaningful change.",
|
||||
kind: "state",
|
||||
status: "known",
|
||||
confidence: "low",
|
||||
}),
|
||||
],
|
||||
updatedNodes: [
|
||||
...resolvedIds.map((id) => ({
|
||||
nodeId: id,
|
||||
previousStatus: "unknown",
|
||||
newStatus: "resolved",
|
||||
previousValue: null,
|
||||
newValue: `answer:${id}`,
|
||||
reason: "resolved child",
|
||||
})),
|
||||
...contradictedIds.map((id) => ({
|
||||
nodeId: id,
|
||||
previousStatus: "unknown",
|
||||
newStatus: "contradicted",
|
||||
previousValue: null,
|
||||
newValue: `contradiction:${id}`,
|
||||
reason: "contradictory child evidence",
|
||||
})),
|
||||
],
|
||||
addedEdges: [],
|
||||
removedEdgeIds: [],
|
||||
resolvedUnknownNodeIds: resolvedIds,
|
||||
affectedNodeIds: [],
|
||||
selectedQuestion: null,
|
||||
};
|
||||
}
|
||||
|
||||
describe("confidence propagation", () => {
|
||||
it("one of four children resolved does not yield high conclusion confidence", () => {
|
||||
const result = applyValidatedProposal({
|
||||
situationGraph: makeFixture(),
|
||||
proposal: makeProposal({ resolvedIds: ["n-child-1"] }),
|
||||
previousQuestion:
|
||||
"What evidence would clarify how the two observations were measured?",
|
||||
answer: "Same accounting period and same management accounts.",
|
||||
});
|
||||
|
||||
const parent = result.updatedSituationGraph.nodes.find(
|
||||
(n) => n.id === "n-parent",
|
||||
);
|
||||
expect(parent.status).toBe("provisional");
|
||||
expect(parent.confidence).toBe("medium");
|
||||
expect(parent.confidenceAssessment).toEqual({
|
||||
evidenceConfidence: "medium",
|
||||
completenessStatus: "partial",
|
||||
conclusionConfidence: "medium",
|
||||
});
|
||||
expect(result.confidenceCapReason).toBe(
|
||||
"unresolved_direct_children_cap_conclusion",
|
||||
);
|
||||
});
|
||||
|
||||
it("all children resolved with coherent evidence may yield high confidence", () => {
|
||||
const result = applyValidatedProposal({
|
||||
situationGraph: makeFixture(),
|
||||
proposal: makeProposal({
|
||||
resolvedIds: ["n-child-1", "n-child-2", "n-child-3", "n-child-4"],
|
||||
}),
|
||||
previousQuestion:
|
||||
"What evidence would clarify how the two observations were measured?",
|
||||
answer: "All direct child questions are answered.",
|
||||
});
|
||||
|
||||
const parent = result.updatedSituationGraph.nodes.find(
|
||||
(n) => n.id === "n-parent",
|
||||
);
|
||||
expect(parent.status).toBe("resolved");
|
||||
expect(parent.confidenceAssessment).toEqual({
|
||||
evidenceConfidence: "high",
|
||||
completenessStatus: "complete",
|
||||
conclusionConfidence: "high",
|
||||
});
|
||||
});
|
||||
|
||||
it("contradictory child evidence prevents high confidence", () => {
|
||||
const result = applyValidatedProposal({
|
||||
situationGraph: makeFixture(),
|
||||
proposal: makeProposal({
|
||||
resolvedIds: ["n-child-1"],
|
||||
contradictedIds: ["n-child-2"],
|
||||
}),
|
||||
previousQuestion:
|
||||
"What evidence would clarify how the two observations were measured?",
|
||||
answer: "One child resolved, another contradicted.",
|
||||
});
|
||||
|
||||
const parent = result.updatedSituationGraph.nodes.find(
|
||||
(n) => n.id === "n-parent",
|
||||
);
|
||||
expect(parent.confidenceAssessment.conclusionConfidence).toBe("low");
|
||||
expect(result.confidenceCapReason).toBe("contradictory_direct_children");
|
||||
});
|
||||
});
|
||||
@@ -1,306 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
applyValidatedProposal,
|
||||
evaluateBranchInteractions,
|
||||
} from "@/lib/graph/apply-proposal.js";
|
||||
import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js";
|
||||
|
||||
function makeParentWithBranches(children) {
|
||||
const parent = makeNode({
|
||||
id: "n-parent",
|
||||
label: "Explanation for why revenue increased while cash fell",
|
||||
description:
|
||||
"Need to understand what change or event could explain why these observations differ, because that is needed to investigate their relationship.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
|
||||
return makeGraph({
|
||||
centralStatement: "Revenue increased while cash fell.",
|
||||
nodes: [
|
||||
parent,
|
||||
...children.map((child) => ({ ...child, parentId: parent.id })),
|
||||
],
|
||||
edges: children.map((child, index) =>
|
||||
makeEdge({
|
||||
id: `e-${index + 1}`,
|
||||
fromNodeId: child.id,
|
||||
toNodeId: parent.id,
|
||||
relationship: "depends_on",
|
||||
description: `${child.label} feeds the parent.`,
|
||||
}),
|
||||
),
|
||||
activeUnknownNodeId: children[0]?.id ?? null,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: "cross-branch corroboration fixture",
|
||||
});
|
||||
}
|
||||
|
||||
function makeResolvedChild(id, label, value, extra = {}) {
|
||||
return makeNode({
|
||||
id,
|
||||
label,
|
||||
description: label,
|
||||
kind: "unknown",
|
||||
status: "resolved",
|
||||
confidence: "medium",
|
||||
value,
|
||||
evidenceIds: extra.evidenceIds ?? [],
|
||||
});
|
||||
}
|
||||
|
||||
function makeUnknownBranch(id, label, description, extra = {}) {
|
||||
return makeNode({
|
||||
id,
|
||||
label,
|
||||
description,
|
||||
kind: "unknown",
|
||||
status: extra.status ?? "unknown",
|
||||
confidence: extra.confidence ?? "medium",
|
||||
evidenceIds: extra.evidenceIds ?? [],
|
||||
value: extra.value ?? null,
|
||||
});
|
||||
}
|
||||
|
||||
describe("evaluateBranchInteractions", () => {
|
||||
it("detects corroborating independent branches", () => {
|
||||
const graph = makeParentWithBranches([
|
||||
makeResolvedChild("n-a", "Debtor balance increased", "bank-statement-a", {
|
||||
evidenceIds: ["bank-statement-a"],
|
||||
}),
|
||||
makeResolvedChild(
|
||||
"n-b",
|
||||
"Cash receipts were delayed",
|
||||
"receipts-ledger-b",
|
||||
{ evidenceIds: ["receipts-ledger-b"] },
|
||||
),
|
||||
]);
|
||||
const parentNode = graph.nodes.find((node) => node.id === "n-parent");
|
||||
|
||||
const result = evaluateBranchInteractions({ parentNode, graph });
|
||||
|
||||
expect(result.interactionSummary.corroboratingBranchCount).toBe(1);
|
||||
expect(result.interactionSummary.duplicateEvidenceCount).toBe(0);
|
||||
expect(result.interactionSummary.conflictingBranchCount).toBe(0);
|
||||
});
|
||||
|
||||
it("detects duplicate evidence instead of corroboration", () => {
|
||||
const graph = makeParentWithBranches([
|
||||
makeResolvedChild(
|
||||
"n-a",
|
||||
"Bank statement shows increased debtor balance",
|
||||
"same-bank",
|
||||
{
|
||||
evidenceIds: ["same-bank"],
|
||||
},
|
||||
),
|
||||
makeResolvedChild(
|
||||
"n-b",
|
||||
"Delayed receipts also cite the bank statement",
|
||||
"same-bank",
|
||||
{
|
||||
evidenceIds: ["same-bank"],
|
||||
},
|
||||
),
|
||||
]);
|
||||
const parentNode = graph.nodes.find((node) => node.id === "n-parent");
|
||||
|
||||
const result = evaluateBranchInteractions({ parentNode, graph });
|
||||
|
||||
expect(result.interactionSummary.duplicateEvidenceCount).toBe(1);
|
||||
expect(result.interactionSummary.corroboratingBranchCount).toBe(0);
|
||||
});
|
||||
|
||||
it("detects conflicting branches", () => {
|
||||
const graph = makeParentWithBranches([
|
||||
makeResolvedChild("n-a", "Revenue recognised correctly", "correctly"),
|
||||
makeNode({
|
||||
id: "n-b",
|
||||
label: "Revenue recognised incorrectly",
|
||||
description: "Revenue recognised incorrectly",
|
||||
kind: "unknown",
|
||||
status: "contradicted",
|
||||
confidence: "medium",
|
||||
value: "incorrectly",
|
||||
}),
|
||||
]);
|
||||
const parentNode = graph.nodes.find((node) => node.id === "n-parent");
|
||||
|
||||
const result = evaluateBranchInteractions({ parentNode, graph });
|
||||
|
||||
expect(result.interactionSummary.conflictingBranchCount).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("cross-branch corroboration effects", () => {
|
||||
function applyToGraph(children, resolvedIds, contradictedIds = []) {
|
||||
const graph = makeParentWithBranches(children);
|
||||
return applyValidatedProposal({
|
||||
situationGraph: graph,
|
||||
proposal: {
|
||||
addedNodes: [
|
||||
makeNode({
|
||||
id: "n-anchor",
|
||||
label: "Update anchor",
|
||||
description:
|
||||
"Anchor state introduced by the answer because the update must contain a meaningful change.",
|
||||
kind: "state",
|
||||
status: "known",
|
||||
confidence: "low",
|
||||
}),
|
||||
],
|
||||
updatedNodes: [
|
||||
...resolvedIds.map((id) => ({
|
||||
nodeId: id,
|
||||
previousStatus: "unknown",
|
||||
newStatus: "resolved",
|
||||
previousValue: null,
|
||||
newValue: `answer:${id}`,
|
||||
reason: "resolved child",
|
||||
})),
|
||||
...contradictedIds.map((id) => ({
|
||||
nodeId: id,
|
||||
previousStatus: "unknown",
|
||||
newStatus: "contradicted",
|
||||
previousValue: null,
|
||||
newValue: `contradiction:${id}`,
|
||||
reason: "contradicted child",
|
||||
})),
|
||||
],
|
||||
addedEdges: [],
|
||||
removedEdgeIds: [],
|
||||
resolvedUnknownNodeIds: resolvedIds,
|
||||
affectedNodeIds: [],
|
||||
selectedQuestion: null,
|
||||
},
|
||||
previousQuestion: "What evidence would clarify this branch?",
|
||||
answer: "deterministic branch update",
|
||||
});
|
||||
}
|
||||
|
||||
it("independent corroboration increases justified confidence without reaching high on incomplete parent", () => {
|
||||
const result = applyToGraph(
|
||||
[
|
||||
makeUnknownBranch(
|
||||
"n-a",
|
||||
"Debtor balance increased",
|
||||
"Debtor balance increased",
|
||||
),
|
||||
makeUnknownBranch(
|
||||
"n-b",
|
||||
"Cash receipts delayed",
|
||||
"Cash receipts delayed",
|
||||
),
|
||||
makeUnknownBranch(
|
||||
"n-c",
|
||||
"Possible one-off event during the period",
|
||||
"Possible one-off event during the period",
|
||||
),
|
||||
makeUnknownBranch(
|
||||
"n-d",
|
||||
"Whether the two observations reflect different timing",
|
||||
"Whether the two observations reflect different timing",
|
||||
),
|
||||
],
|
||||
["n-a", "n-b"],
|
||||
);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.interactionSummary?.corroboratingBranchCount).toBeGreaterThan(
|
||||
0,
|
||||
);
|
||||
expect(result.interactionSummary?.duplicateEvidenceCount).toBe(0);
|
||||
expect(result.parentConfidenceAfter).toBe("medium");
|
||||
expect(result.confidenceCapReason).toBe(
|
||||
"independent_corroboration_with_incomplete_parent",
|
||||
);
|
||||
});
|
||||
|
||||
it("duplicate evidence does not increase confidence", () => {
|
||||
const result = applyToGraph(
|
||||
[
|
||||
makeUnknownBranch(
|
||||
"n-a",
|
||||
"Bank statement shows increased debtor balance",
|
||||
"Bank statement shows increased debtor balance",
|
||||
{ evidenceIds: ["same-bank"] },
|
||||
),
|
||||
makeUnknownBranch(
|
||||
"n-b",
|
||||
"Delayed receipts also cite the bank statement",
|
||||
"Delayed receipts also cite the bank statement",
|
||||
{ evidenceIds: ["same-bank"] },
|
||||
),
|
||||
makeUnknownBranch(
|
||||
"n-c",
|
||||
"Possible one-off event during the period",
|
||||
"Possible one-off event during the period",
|
||||
),
|
||||
],
|
||||
["n-a", "n-b"],
|
||||
);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.interactionSummary?.duplicateEvidenceCount).toBeGreaterThan(
|
||||
0,
|
||||
);
|
||||
expect(result.interactionSummary?.corroboratingBranchCount).toBe(0);
|
||||
expect(result.confidenceCapReason).toBe(
|
||||
"duplicate_evidence_no_extra_confidence",
|
||||
);
|
||||
});
|
||||
|
||||
it("conflicting evidence caps confidence", () => {
|
||||
const result = applyToGraph(
|
||||
[
|
||||
makeUnknownBranch(
|
||||
"n-a",
|
||||
"Revenue recognised correctly",
|
||||
"Revenue recognised correctly",
|
||||
),
|
||||
makeUnknownBranch(
|
||||
"n-b",
|
||||
"Revenue recognised incorrectly",
|
||||
"Revenue recognised incorrectly",
|
||||
),
|
||||
makeUnknownBranch(
|
||||
"n-c",
|
||||
"Possible one-off event during the period",
|
||||
"Possible one-off event during the period",
|
||||
),
|
||||
],
|
||||
["n-a"],
|
||||
["n-b"],
|
||||
);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.interactionSummary?.conflictingBranchCount).toBeGreaterThan(
|
||||
0,
|
||||
);
|
||||
expect(result.conclusionConfidenceAfter).toBe("low");
|
||||
});
|
||||
|
||||
it("independent branches stay interaction-neutral", () => {
|
||||
const result = applyToGraph(
|
||||
[
|
||||
makeUnknownBranch(
|
||||
"n-a",
|
||||
"Marketing campaign changed traffic",
|
||||
"Marketing campaign changed traffic",
|
||||
),
|
||||
makeUnknownBranch(
|
||||
"n-b",
|
||||
"Equipment maintenance occurred",
|
||||
"Equipment maintenance occurred",
|
||||
),
|
||||
],
|
||||
["n-a"],
|
||||
);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.interactionSummary?.independentBranchCount).toBeGreaterThan(
|
||||
0,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,278 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
assessChildUnknownQuality,
|
||||
applyValidatedProposal,
|
||||
MAX_DECOMPOSITION_DEPTH,
|
||||
} from "@/lib/graph/apply-proposal.js";
|
||||
import { makeGraph, makeNode } from "@/lib/graph/schema.js";
|
||||
|
||||
function makeParentGraph({
|
||||
centralStatement,
|
||||
parentLabel,
|
||||
parentDescription,
|
||||
observations = [],
|
||||
}) {
|
||||
const parent = makeNode({
|
||||
id: "n-parent",
|
||||
label: parentLabel,
|
||||
description: parentDescription,
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
|
||||
return {
|
||||
parent,
|
||||
graph: makeGraph({
|
||||
centralStatement,
|
||||
nodes: [parent, ...observations],
|
||||
edges: [],
|
||||
activeUnknownNodeId: parent.id,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: "Decomposition quality graph",
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
describe("assessChildUnknownQuality", () => {
|
||||
it("rejects 'Timing or measurement basis' as compound", () => {
|
||||
const { parent, graph } = makeParentGraph({
|
||||
centralStatement:
|
||||
"Revenue increased by 18%, but cash in the bank fell over the same period.",
|
||||
parentLabel:
|
||||
"Explanation for why revenue increased by 18%, but cash in the bank fell over the same period",
|
||||
parentDescription:
|
||||
"Need to understand what change or event could explain why these observations differ, because that is needed to investigate their relationship.",
|
||||
});
|
||||
const child = makeNode({
|
||||
id: "n-child",
|
||||
label: "Timing or measurement basis",
|
||||
description:
|
||||
"Need evidence about whether a timing or measurement-basis difference could explain the observations, because that would change how they should be interpreted.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
parentId: parent.id,
|
||||
});
|
||||
|
||||
const result = assessChildUnknownQuality({
|
||||
parentNode: parent,
|
||||
childNode: child,
|
||||
siblingNodes: [child],
|
||||
graph,
|
||||
});
|
||||
|
||||
expect(result.valid).toBe(false);
|
||||
expect(result.compoundSignals).toContain("timing_or_measurement_basis");
|
||||
expect(result.reasons).toContain("compound_child");
|
||||
});
|
||||
|
||||
it("accepts a child with one directly answerable uncertainty", () => {
|
||||
const { parent, graph } = makeParentGraph({
|
||||
centralStatement: "Traffic increased, but sales stayed flat.",
|
||||
parentLabel:
|
||||
"What explains why more website traffic did not produce more sales?",
|
||||
parentDescription:
|
||||
"Need an explanation because the observations moved differently.",
|
||||
});
|
||||
const child = makeNode({
|
||||
id: "n-child",
|
||||
label: "Different measurement basis between the two observations",
|
||||
description:
|
||||
"Need evidence about whether the two observations use different measurement bases, because that could help explain the difference.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
parentId: parent.id,
|
||||
});
|
||||
|
||||
const result = assessChildUnknownQuality({
|
||||
parentNode: parent,
|
||||
childNode: child,
|
||||
siblingNodes: [child],
|
||||
graph,
|
||||
});
|
||||
|
||||
expect(result.valid).toBe(true);
|
||||
expect(result.atomic).toBe(true);
|
||||
expect(result.directlyAnswerable).toBe(true);
|
||||
expect(result.narrowerThanParent).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects sibling duplicates", () => {
|
||||
const { parent, graph } = makeParentGraph({
|
||||
centralStatement: "Production increased, but defects also increased.",
|
||||
parentLabel: "What explains why output and defects both increased?",
|
||||
parentDescription:
|
||||
"Need an explanation because both observations increased.",
|
||||
});
|
||||
const childA = makeNode({
|
||||
id: "n-child-a",
|
||||
label: "Different timing between the two observations",
|
||||
description:
|
||||
"Need evidence about whether the two observations reflect different timing, because that could help explain the difference.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
parentId: parent.id,
|
||||
});
|
||||
const childB = makeNode({
|
||||
id: "n-child-b",
|
||||
label: "Different timing between the two observations",
|
||||
description:
|
||||
"Need evidence about whether the two observations reflect different timing, because that could help explain the difference.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
parentId: parent.id,
|
||||
});
|
||||
|
||||
const result = assessChildUnknownQuality({
|
||||
parentNode: parent,
|
||||
childNode: childA,
|
||||
siblingNodes: [childA, childB],
|
||||
graph,
|
||||
});
|
||||
|
||||
expect(result.valid).toBe(false);
|
||||
expect(result.duplicateSiblingIds).toContain("n-child-b");
|
||||
});
|
||||
|
||||
it("rejects parent paraphrases", () => {
|
||||
const { parent, graph } = makeParentGraph({
|
||||
centralStatement:
|
||||
"Customer satisfaction scores increased, but complaints also increased.",
|
||||
parentLabel:
|
||||
"What explains why satisfaction and complaints both increased?",
|
||||
parentDescription:
|
||||
"Need a broad explanation because the observations moved differently.",
|
||||
});
|
||||
const child = makeNode({
|
||||
id: "n-child",
|
||||
label: "What explains why satisfaction and complaints both increased?",
|
||||
description:
|
||||
"Need a broad explanation because the observations moved differently.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
parentId: parent.id,
|
||||
});
|
||||
|
||||
const result = assessChildUnknownQuality({
|
||||
parentNode: parent,
|
||||
childNode: child,
|
||||
siblingNodes: [child],
|
||||
graph,
|
||||
});
|
||||
|
||||
expect(result.valid).toBe(false);
|
||||
expect(result.reasons).toContain("not_narrower_than_parent");
|
||||
});
|
||||
});
|
||||
|
||||
describe("decomposition stopping conditions", () => {
|
||||
function makeMeaningfulNoOpProposal() {
|
||||
return {
|
||||
addedNodes: [
|
||||
makeNode({
|
||||
id: "n-anchor",
|
||||
label: "Update anchor",
|
||||
description:
|
||||
"Anchor state introduced by the answer because the update must contain a meaningful change.",
|
||||
kind: "state",
|
||||
status: "known",
|
||||
confidence: "low",
|
||||
}),
|
||||
],
|
||||
updatedNodes: [],
|
||||
addedEdges: [],
|
||||
removedEdgeIds: [],
|
||||
resolvedUnknownNodeIds: [],
|
||||
affectedNodeIds: [],
|
||||
selectedQuestion: null,
|
||||
};
|
||||
}
|
||||
|
||||
it("does not decompose an atomic selected unknown", () => {
|
||||
const atomic = makeNode({
|
||||
id: "n-atomic",
|
||||
label: "Were both figures measured over the same accounting period?",
|
||||
description:
|
||||
"Need to know whether both figures cover the same accounting period because that determines whether they are directly comparable.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
});
|
||||
const graph = makeGraph({
|
||||
centralStatement:
|
||||
"Revenue increased by 18%, but cash in the bank fell over the same period.",
|
||||
nodes: [atomic],
|
||||
edges: [],
|
||||
activeUnknownNodeId: atomic.id,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: "Atomic selected node graph",
|
||||
});
|
||||
|
||||
const result = applyValidatedProposal({
|
||||
situationGraph: graph,
|
||||
proposal: makeMeaningfulNoOpProposal(),
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.decompositionAttempted).toBe(false);
|
||||
expect(result.decompositionStoppedReason).toBe(
|
||||
"Selected unknown is already atomic.",
|
||||
);
|
||||
});
|
||||
|
||||
it("stops once a directly answerable child is selected", () => {
|
||||
const { parent, graph } = makeParentGraph({
|
||||
centralStatement: "Traffic increased, but sales stayed flat.",
|
||||
parentLabel:
|
||||
"What explains why more website traffic did not produce more sales?",
|
||||
parentDescription:
|
||||
"Need an explanation because the observations moved differently.",
|
||||
observations: [
|
||||
makeNode({
|
||||
id: "n-traffic",
|
||||
label: "Website traffic increased.",
|
||||
description: "Website traffic increased.",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
}),
|
||||
makeNode({
|
||||
id: "n-sales",
|
||||
label: "Sales stayed flat.",
|
||||
description: "Sales stayed flat.",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
const result = applyValidatedProposal({
|
||||
situationGraph: graph,
|
||||
proposal: makeMeaningfulNoOpProposal(),
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.decompositionAttempted).toBe(true);
|
||||
expect(result.decompositionAccepted).toBe(true);
|
||||
expect(result.selectedQuestion).toMatchObject({
|
||||
nodeId: expect.any(String),
|
||||
question:
|
||||
"What evidence would clarify how the two observations were measured?",
|
||||
});
|
||||
expect(result.selectedChildNodeId).toBe(result.selectedQuestion?.nodeId);
|
||||
expect(result.decompositionStoppedReason).toBe(
|
||||
"Selected child is atomic and directly answerable.",
|
||||
);
|
||||
});
|
||||
|
||||
it("exposes the configured maximum decomposition depth", () => {
|
||||
expect(MAX_DECOMPOSITION_DEPTH).toBeGreaterThanOrEqual(2);
|
||||
expect(MAX_DECOMPOSITION_DEPTH).toBeLessThanOrEqual(3);
|
||||
});
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,114 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { buildGraphUpdatePrompt } from "@/lib/graph/prompt-builder.js";
|
||||
import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js";
|
||||
|
||||
function makeContext() {
|
||||
const unknown = makeNode({
|
||||
id: "n-unknown",
|
||||
label: "Complaint rate denominator",
|
||||
description: "Need the denominator to compare complaint rates",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
});
|
||||
const observation = makeNode({
|
||||
id: "n-obs",
|
||||
label: "Complaints up 35%",
|
||||
description: "Complaints increased by 35%",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
});
|
||||
|
||||
return {
|
||||
situationGraph: makeGraph({
|
||||
centralStatement: "Complaints increased while production increased.",
|
||||
nodes: [unknown, observation],
|
||||
edges: [
|
||||
makeEdge({
|
||||
id: "e1",
|
||||
fromNodeId: observation.id,
|
||||
toNodeId: unknown.id,
|
||||
relationship: "supports",
|
||||
confidence: "high",
|
||||
description: "Observation informs the unknown",
|
||||
}),
|
||||
],
|
||||
activeUnknownNodeId: unknown.id,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary:
|
||||
"Nodes: 1 observation, 1 unknown | Edges: 1 total | Unknowns: 1 unresolved",
|
||||
}),
|
||||
previousQuestion: "What denominator is being used for the complaint rate?",
|
||||
answer:
|
||||
"The complaint rate fell from 2.0 complaints per 100 units to 1.9 complaints per 100 units.",
|
||||
};
|
||||
}
|
||||
|
||||
describe("buildGraphUpdatePrompt", () => {
|
||||
it("includes the current graph", () => {
|
||||
const prompt = buildGraphUpdatePrompt(makeContext());
|
||||
expect(prompt).toContain(
|
||||
"Complaints increased while production increased.",
|
||||
);
|
||||
expect(prompt).toContain("Complaint rate denominator");
|
||||
});
|
||||
|
||||
it("includes previous question and answer", () => {
|
||||
const prompt = buildGraphUpdatePrompt(makeContext());
|
||||
expect(prompt).toContain(
|
||||
"What denominator is being used for the complaint rate?",
|
||||
);
|
||||
expect(prompt).toContain(
|
||||
"The complaint rate fell from 2.0 complaints per 100 units to 1.9 complaints per 100 units.",
|
||||
);
|
||||
});
|
||||
|
||||
it("contains exact schema keys", () => {
|
||||
const prompt = buildGraphUpdatePrompt(makeContext());
|
||||
expect(prompt).toContain("addedNodes");
|
||||
expect(prompt).toContain("updatedNodes");
|
||||
expect(prompt).toContain("addedEdges");
|
||||
expect(prompt).toContain("removedEdgeIds");
|
||||
expect(prompt).toContain("resolvedUnknownNodeIds");
|
||||
expect(prompt).toContain("affectedNodeIds");
|
||||
expect(prompt).toContain("selectedQuestion");
|
||||
});
|
||||
|
||||
it("lists enum values", () => {
|
||||
const prompt = buildGraphUpdatePrompt(makeContext());
|
||||
expect(prompt).toContain(
|
||||
"observation | reported_claim | metric | state | transition | relationship | assumption | unknown | conclusion",
|
||||
);
|
||||
expect(prompt).toContain(
|
||||
"known | unknown | provisional | supported | weakened | contradicted | resolved",
|
||||
);
|
||||
expect(prompt).toContain(
|
||||
"supports | weakens | contradicts | depends_on | causes | may_cause | measures | compares_with | updates | other",
|
||||
);
|
||||
});
|
||||
|
||||
it("forbids full-graph replacement", () => {
|
||||
const prompt = buildGraphUpdatePrompt(makeContext());
|
||||
expect(prompt).toContain("Never return a replacement graph");
|
||||
expect(prompt).toContain("Propose changes only");
|
||||
});
|
||||
|
||||
it("requires JSON only", () => {
|
||||
const prompt = buildGraphUpdatePrompt(makeContext());
|
||||
expect(prompt).toContain("Return JSON only");
|
||||
expect(prompt).toContain("Return one JSON object only");
|
||||
});
|
||||
|
||||
it("describes controlled emergent unknown rules", () => {
|
||||
const prompt = buildGraphUpdatePrompt(makeContext());
|
||||
expect(prompt).toContain("Add at most 3 new unknown nodes");
|
||||
expect(prompt).toContain("Resolve the answered unknown first");
|
||||
expect(prompt).toContain(
|
||||
"selectedQuestion.question must be one narrow non-compound question",
|
||||
);
|
||||
expect(prompt).toContain(
|
||||
"the engine will deterministically choose final priority after validation",
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,474 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
assessUnknownAtomicity,
|
||||
formulateQuestion,
|
||||
formulateTieResolutionQuestion,
|
||||
selectInvestigationStrategy,
|
||||
} from "@/lib/graph/question-formulator.js";
|
||||
import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js";
|
||||
|
||||
function makeGraphFor(node, extra = {}) {
|
||||
return makeGraph({
|
||||
centralStatement: extra.centralStatement || "Decision context",
|
||||
nodes: [node, ...(extra.nodes || [])],
|
||||
edges: extra.edges || [],
|
||||
activeUnknownNodeId: node.id,
|
||||
resolvedNodeIds: extra.resolvedNodeIds || [],
|
||||
currentSummary: "Test summary",
|
||||
});
|
||||
}
|
||||
|
||||
describe("formulateQuestion", () => {
|
||||
it("atomicity assessment leaves focused unknowns direct and marks broad explanation unknowns composite", () => {
|
||||
const atomicUnknown = makeNode({
|
||||
id: "n-atomic",
|
||||
label: "Complaint rate denominator",
|
||||
description:
|
||||
"Need the denominator because it directly determines the complaint rate.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
});
|
||||
const compositeUnknown = makeNode({
|
||||
id: "n-composite",
|
||||
label:
|
||||
"Explanation for why revenue increased by 18%, but cash in the bank fell over the same period",
|
||||
description:
|
||||
"Need to understand what change or event could explain why these observations differ, because that is needed to investigate their relationship.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
const compositeGraph = makeGraphFor(compositeUnknown, {
|
||||
centralStatement:
|
||||
"Revenue increased by 18%, but cash in the bank fell over the same period.",
|
||||
nodes: [
|
||||
makeNode({
|
||||
id: "n-revenue-observation",
|
||||
label: "Revenue increased by 18%.",
|
||||
description: "Revenue increased by 18%.",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
}),
|
||||
makeNode({
|
||||
id: "n-cash-observation",
|
||||
label: "Cash in the bank decreased over the same period.",
|
||||
description: "Cash in the bank decreased over the same period.",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
expect(
|
||||
assessUnknownAtomicity({
|
||||
node: atomicUnknown,
|
||||
graph: makeGraphFor(atomicUnknown),
|
||||
}).atomicity,
|
||||
).toBe("atomic");
|
||||
expect(
|
||||
assessUnknownAtomicity({
|
||||
node: compositeUnknown,
|
||||
graph: compositeGraph,
|
||||
}).atomicity,
|
||||
).toBe("composite");
|
||||
});
|
||||
|
||||
it("commercial viability plus build decision produces a decision-threshold question", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-commercial",
|
||||
label: "Uncertainty regarding the commercial value of the product",
|
||||
description:
|
||||
"Commercial justification remains unclear because the decision depends on it.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
parentId: "n-decision",
|
||||
});
|
||||
const decision = makeNode({
|
||||
id: "n-decision",
|
||||
label: "Build decision",
|
||||
description: "Decision introduced by the answer.",
|
||||
kind: "state",
|
||||
status: "known",
|
||||
confidence: "medium",
|
||||
childIds: [unknown.id],
|
||||
value: "Deciding whether to build the product",
|
||||
});
|
||||
const graph = makeGraphFor(unknown, {
|
||||
nodes: [decision],
|
||||
resolvedNodeIds: [decision.id],
|
||||
});
|
||||
|
||||
const result = formulateQuestion({ node: unknown, graph });
|
||||
|
||||
expect(result.strategy).toBe("decision_threshold");
|
||||
expect(result.question).toContain("What outcome");
|
||||
expect(result.question.toLowerCase()).toContain("justify");
|
||||
});
|
||||
|
||||
it("commercial viability does not produce a pricing-first question", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-commercial",
|
||||
label: "Commercial viability",
|
||||
description:
|
||||
"Commercial viability remains unresolved because the decision depends on it.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
});
|
||||
const graph = makeGraphFor(unknown);
|
||||
|
||||
const result = formulateQuestion({ node: unknown, graph });
|
||||
|
||||
expect(result.question.toLowerCase()).not.toContain("price");
|
||||
expect(result.question.toLowerCase()).not.toContain("pricing");
|
||||
});
|
||||
|
||||
it("undefined term produces a definition question", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-term",
|
||||
label: "Success criteria definition",
|
||||
description:
|
||||
"Need a definition of the term because the team uses it inconsistently.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
|
||||
const result = formulateQuestion({
|
||||
node: unknown,
|
||||
graph: makeGraphFor(unknown),
|
||||
});
|
||||
|
||||
expect(result.strategy).toBe("definition");
|
||||
expect(result.question).toMatch(/^What does /);
|
||||
});
|
||||
|
||||
it("unsupported claim produces an evidence question", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-claim",
|
||||
label: "Demand claim",
|
||||
description: "Need evidence because the claim has not been validated.",
|
||||
kind: "reported_claim",
|
||||
status: "provisional",
|
||||
confidence: "low",
|
||||
});
|
||||
|
||||
const result = formulateQuestion({
|
||||
node: unknown,
|
||||
graph: makeGraphFor(unknown),
|
||||
});
|
||||
|
||||
expect(result.strategy).toBe("evidence_gathering");
|
||||
expect(result.question).toContain("What evidence");
|
||||
});
|
||||
|
||||
it("missing previous state produces a baseline question", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-baseline",
|
||||
label: "Baseline conversion rate",
|
||||
description:
|
||||
"Need the previous baseline because the change cannot be assessed without it.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
|
||||
const result = formulateQuestion({
|
||||
node: unknown,
|
||||
graph: makeGraphFor(unknown),
|
||||
});
|
||||
|
||||
expect(result.strategy).toBe("baseline_reconstruction");
|
||||
expect(result.question).toContain("What was the comparable state before");
|
||||
});
|
||||
|
||||
it("conflicting claim produces a contradiction-resolution question", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-conflict",
|
||||
label: "Conflicting churn claim",
|
||||
description:
|
||||
"Need to resolve the inconsistency because the current figures contradict each other.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
|
||||
const contradiction = makeNode({
|
||||
id: "n-contradiction",
|
||||
label: "Contradicted report",
|
||||
description: "Two sources disagree about churn.",
|
||||
kind: "conclusion",
|
||||
status: "contradicted",
|
||||
confidence: "low",
|
||||
childIds: [unknown.id],
|
||||
});
|
||||
|
||||
const result = formulateQuestion({
|
||||
node: unknown,
|
||||
graph: makeGraphFor(unknown, { nodes: [contradiction] }),
|
||||
});
|
||||
|
||||
expect(result.strategy).toBe("contradiction_resolution");
|
||||
expect(result.question).toContain("resolve the contradiction");
|
||||
});
|
||||
|
||||
it("constraint unknown uses evidence-gathering within the fixed strategy set", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-constraint",
|
||||
label: "Budget constraint",
|
||||
description:
|
||||
"Need the main budget constraint because it limits the available options.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
|
||||
const result = formulateQuestion({
|
||||
node: unknown,
|
||||
graph: makeGraphFor(unknown),
|
||||
});
|
||||
|
||||
expect(result.strategy).toBe("evidence_gathering");
|
||||
expect(result.question).toContain("What evidence");
|
||||
});
|
||||
|
||||
it("the same unknown can produce different questions when paired with different strategies", () => {
|
||||
const thresholdUnknown = makeNode({
|
||||
id: "n-threshold-unknown",
|
||||
label: "Value threshold",
|
||||
description: "Need to resolve the value threshold.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
});
|
||||
const definitionUnknown = makeNode({
|
||||
id: "n-definition-unknown",
|
||||
label: "Value term",
|
||||
description: "Need to resolve what value term refers to in this context.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
});
|
||||
|
||||
const decisionGraph = makeGraphFor(thresholdUnknown, {
|
||||
centralStatement: "We are deciding whether to launch this product.",
|
||||
nodes: [
|
||||
makeNode({
|
||||
id: "n-decision",
|
||||
label: "Launch decision",
|
||||
description: "Decision depends on the value threshold.",
|
||||
kind: "state",
|
||||
status: "known",
|
||||
confidence: "medium",
|
||||
childIds: [thresholdUnknown.id],
|
||||
value: "Deciding whether to launch the product",
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
const definitionGraph = makeGraphFor(definitionUnknown, {
|
||||
centralStatement:
|
||||
"The team uses the term value threshold inconsistently.",
|
||||
nodes: [
|
||||
makeNode({
|
||||
id: "n-definition",
|
||||
label: "Definition disagreement about value threshold",
|
||||
description:
|
||||
"Need a definition of value threshold because the term is used inconsistently before comparing options.",
|
||||
kind: "state",
|
||||
status: "known",
|
||||
confidence: "medium",
|
||||
childIds: [definitionUnknown.id],
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
const decisionResult = formulateQuestion({
|
||||
node: thresholdUnknown,
|
||||
graph: decisionGraph,
|
||||
});
|
||||
const definitionResult = formulateQuestion({
|
||||
node: definitionUnknown,
|
||||
graph: definitionGraph,
|
||||
});
|
||||
|
||||
expect(decisionResult.strategy).toBe("decision_threshold");
|
||||
expect(definitionResult.strategy).toBe("definition");
|
||||
expect(decisionResult.question).not.toBe(definitionResult.question);
|
||||
});
|
||||
|
||||
it("strategy selection is deterministic and explainable", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-threshold",
|
||||
label: "Success threshold",
|
||||
description:
|
||||
"Need the success threshold because the decision depends on it.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
});
|
||||
const graph = makeGraphFor(unknown, {
|
||||
centralStatement: "We need to decide whether to continue investing.",
|
||||
});
|
||||
|
||||
const first = selectInvestigationStrategy({ node: unknown, graph });
|
||||
const second = selectInvestigationStrategy({ node: unknown, graph });
|
||||
|
||||
expect(first).toEqual(second);
|
||||
expect(first.key).toBe("decision_threshold");
|
||||
expect(first.reason).toContain("threshold");
|
||||
});
|
||||
|
||||
it("question is singular and answerable", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-evidence",
|
||||
label: "Evidence of demand",
|
||||
description:
|
||||
"Need evidence of demand because the decision depends on it.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
|
||||
const result = formulateQuestion({
|
||||
node: unknown,
|
||||
graph: makeGraphFor(unknown),
|
||||
});
|
||||
|
||||
expect(result.question.match(/\?/g) || []).toHaveLength(1);
|
||||
expect(result.question.toLowerCase()).not.toContain(" and ");
|
||||
});
|
||||
|
||||
it("awkward uncertainty phrasing is rejected via fallback", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-weird",
|
||||
label: "Uncertainty regarding service reliability",
|
||||
description: "Unknown service reliability.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
|
||||
const result = formulateQuestion({
|
||||
node: unknown,
|
||||
graph: makeGraphFor(unknown),
|
||||
});
|
||||
|
||||
expect(result.question).not.toContain("How should uncertainty regarding");
|
||||
expect(result.question).not.toContain(
|
||||
"What would resolve uncertainty regarding",
|
||||
);
|
||||
});
|
||||
|
||||
it("ambiguous contradiction produces a broad distinguishing question without accounting jargon", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-cause-a",
|
||||
label: "Cash outflow cause",
|
||||
description: "Unclear explanation for the contradiction.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
});
|
||||
const contradiction = makeNode({
|
||||
id: "n-contradiction",
|
||||
label: "Divergent movement between revenue and cash",
|
||||
description: "Two signals moved in opposite directions.",
|
||||
kind: "relationship",
|
||||
status: "supported",
|
||||
confidence: "medium",
|
||||
});
|
||||
const revenueObservation = makeNode({
|
||||
id: "n-revenue-observation",
|
||||
label: "Revenue increased by 18%.",
|
||||
description: "Revenue increased by 18%.",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
});
|
||||
const cashObservation = makeNode({
|
||||
id: "n-cash-observation",
|
||||
label: "Cash in the bank decreased over the same period.",
|
||||
description: "Cash in the bank decreased over the same period.",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
});
|
||||
const graph = makeGraphFor(unknown, {
|
||||
centralStatement:
|
||||
"Revenue increased by 18%, but cash in the bank fell over the same period.",
|
||||
nodes: [contradiction, revenueObservation, cashObservation],
|
||||
});
|
||||
|
||||
const result = formulateTieResolutionQuestion({ graph });
|
||||
|
||||
expect(result.question).toBe(
|
||||
"Were these figures measured on the same basis and at the same scale?",
|
||||
);
|
||||
expect(result.comparabilityStatus).toBe("uncertain");
|
||||
expect(result.question.toLowerCase()).not.toMatch(
|
||||
/accounts receivable|capex|debt repayments|working capital/,
|
||||
);
|
||||
});
|
||||
|
||||
it("definition is selected only for genuine definition unknowns", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-definition-only",
|
||||
label: "Definition of success criteria",
|
||||
description: "The term is used inconsistently and needs a definition.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
|
||||
const result = formulateQuestion({
|
||||
node: unknown,
|
||||
graph: makeGraphFor(unknown),
|
||||
});
|
||||
expect(result.strategy).toBe("definition");
|
||||
});
|
||||
|
||||
it("an unknown about possible causes does not become a definition question", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-causes",
|
||||
label: "Possible causes of the divergence",
|
||||
description: "Several causes may explain the divergence.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
|
||||
const result = formulateQuestion({
|
||||
node: unknown,
|
||||
graph: makeGraphFor(unknown),
|
||||
});
|
||||
expect(result.strategy).toBeNull();
|
||||
expect(result.question).toBe(
|
||||
"What would clarify possible causes of the divergence in this situation?",
|
||||
);
|
||||
});
|
||||
|
||||
it("malformed punctuation is rejected", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-punct",
|
||||
label: "Magnitude and nature of cash outflows (operating expenses).",
|
||||
description:
|
||||
"Magnitude and nature of cash outflows (operating expenses).",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
|
||||
const result = formulateQuestion({
|
||||
node: unknown,
|
||||
graph: makeGraphFor(unknown),
|
||||
});
|
||||
expect(result.question).not.toContain("). is true?");
|
||||
expect(result.question).toBe(
|
||||
"What would clarify magnitude and nature of cash outflows (operating expenses) in this situation?",
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,155 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { applyValidatedProposal } from "@/lib/graph/apply-proposal.js";
|
||||
import { formulateQuestion } from "@/lib/graph/question-formulator.js";
|
||||
import { selectActiveUnknownCandidate } from "@/lib/graph/utils.js";
|
||||
import { questionPriorityGeneralisationFixtures } from "@/tests/fixtures/question-priority-generalisation.js";
|
||||
|
||||
function clone(value) {
|
||||
return JSON.parse(JSON.stringify(value));
|
||||
}
|
||||
|
||||
function buildResolutionProposal(graph) {
|
||||
const activeNode = graph.nodes.find(
|
||||
(node) => node.id === graph.activeUnknownNodeId,
|
||||
);
|
||||
const placeholderCandidate = graph.nodes.find(
|
||||
(node) => node.kind === "unknown" && node.id !== activeNode.id,
|
||||
);
|
||||
|
||||
return {
|
||||
addedNodes: [],
|
||||
updatedNodes: [
|
||||
{
|
||||
nodeId: activeNode.id,
|
||||
previousStatus: activeNode.status,
|
||||
newStatus: "resolved",
|
||||
previousValue: activeNode.value ?? null,
|
||||
newValue: activeNode.value ?? "Resolved context answer",
|
||||
reason:
|
||||
"The resolved context unknown is treated as answered for fixture progression.",
|
||||
},
|
||||
],
|
||||
addedEdges: [],
|
||||
removedEdgeIds: [],
|
||||
resolvedUnknownNodeIds: [activeNode.id],
|
||||
affectedNodeIds: [],
|
||||
selectedQuestion: {
|
||||
nodeId: placeholderCandidate?.id,
|
||||
question: "Placeholder candidate question?",
|
||||
reason: "Candidate only; deterministic selector should override it.",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function assertQuestionStructure(question) {
|
||||
expect(question.match(/\?/g) || []).toHaveLength(1);
|
||||
expect(question).not.toMatch(/\?\s*(and|or)\b/i);
|
||||
expect(question).not.toMatch(/^What is\s+/i);
|
||||
expect(question).toMatch(/^(What|Who|When)\b/);
|
||||
}
|
||||
|
||||
describe("question priority generalisation", () => {
|
||||
for (const fixture of questionPriorityGeneralisationFixtures) {
|
||||
it(`${fixture.scenario} selects a foundational unknown and singular answerable strategy`, () => {
|
||||
const originalGraph = clone(fixture.graph);
|
||||
const deterministicSelection = selectActiveUnknownCandidate(
|
||||
fixture.graph,
|
||||
[fixture.graph.activeUnknownNodeId],
|
||||
);
|
||||
|
||||
const result = applyValidatedProposal({
|
||||
situationGraph: fixture.graph,
|
||||
proposal: buildResolutionProposal(fixture.graph),
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(fixture.graph).toEqual(originalGraph);
|
||||
expect(result.graphUpdate.selectedQuestion?.question).toBe(
|
||||
"Placeholder candidate question?",
|
||||
);
|
||||
|
||||
expect(deterministicSelection.nodeId).toBe(
|
||||
result.selectedQuestion.nodeId,
|
||||
);
|
||||
expect(fixture.acceptableFoundationalUnknownNodeIds).toContain(
|
||||
result.selectedQuestion.nodeId,
|
||||
);
|
||||
expect(result.selectedQuestion.nodeId).not.toBe(
|
||||
fixture.graph.nodes[fixture.graph.nodes.length - 1].id,
|
||||
);
|
||||
|
||||
expect(fixture.acceptableQuestionStrategies).toContain(
|
||||
result.selectedQuestion.strategy,
|
||||
);
|
||||
assertQuestionStructure(result.selectedQuestion.question);
|
||||
|
||||
const lowerQuestion = result.selectedQuestion.question.toLowerCase();
|
||||
for (const topic of fixture.prohibitedFirstTopics) {
|
||||
expect(lowerQuestion).not.toContain(topic.toLowerCase());
|
||||
}
|
||||
|
||||
const selectedNode = result.updatedSituationGraph.nodes.find(
|
||||
(node) => node.id === result.selectedQuestion.nodeId,
|
||||
);
|
||||
const reformulated = formulateQuestion({
|
||||
node: selectedNode,
|
||||
graph: result.updatedSituationGraph,
|
||||
context: {
|
||||
resolvedValues: ["Resolved context answer"],
|
||||
},
|
||||
});
|
||||
|
||||
expect(reformulated.question).toBe(result.selectedQuestion.question);
|
||||
expect(clone(result.updatedSituationGraph)).toEqual(
|
||||
result.updatedSituationGraph,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
it("reports all five selected unknowns and strategies", () => {
|
||||
const summary = questionPriorityGeneralisationFixtures.map((fixture) => {
|
||||
const result = applyValidatedProposal({
|
||||
situationGraph: fixture.graph,
|
||||
proposal: buildResolutionProposal(fixture.graph),
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
|
||||
return {
|
||||
scenario: fixture.scenario,
|
||||
nodeId: result.selectedQuestion.nodeId,
|
||||
strategy: result.selectedQuestion.strategy,
|
||||
};
|
||||
});
|
||||
|
||||
expect(summary).toMatchInlineSnapshot(`
|
||||
[
|
||||
{
|
||||
"nodeId": "hire-success-criteria",
|
||||
"scenario": "Should we hire another engineer?",
|
||||
"strategy": "decision_threshold",
|
||||
},
|
||||
{
|
||||
"nodeId": "van-reliability-threshold",
|
||||
"scenario": "Should we replace the delivery vans?",
|
||||
"strategy": "decision_threshold",
|
||||
},
|
||||
{
|
||||
"nodeId": "country-value-threshold",
|
||||
"scenario": "Should we launch in another country?",
|
||||
"strategy": "decision_threshold",
|
||||
},
|
||||
{
|
||||
"nodeId": "project-benefit-threshold",
|
||||
"scenario": "Should we continue a project that is over budget?",
|
||||
"strategy": "decision_threshold",
|
||||
},
|
||||
{
|
||||
"nodeId": "support-value-threshold",
|
||||
"scenario": "Should we introduce a paid support tier?",
|
||||
"strategy": "baseline_reconstruction",
|
||||
},
|
||||
]
|
||||
`);
|
||||
});
|
||||
});
|
||||
@@ -1,442 +0,0 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import {
|
||||
SituationKind,
|
||||
SituationStatus,
|
||||
ConfidenceLevel,
|
||||
SituationRelationship,
|
||||
situationNodeSchema,
|
||||
situationEdgeSchema,
|
||||
situationGraphSchema,
|
||||
graphUpdateSchema,
|
||||
startCaseRequestSchema,
|
||||
updateCaseRequestSchema,
|
||||
makeNodeId,
|
||||
makeNode,
|
||||
makeEdge,
|
||||
makeGraph,
|
||||
} from "@/lib/graph/schema.js";
|
||||
|
||||
describe("situationNodeSchema", () => {
|
||||
const validNode = {
|
||||
id: "n1",
|
||||
label: "Test Node",
|
||||
description: "A test node",
|
||||
kind: "observation",
|
||||
status: "known",
|
||||
confidence: "high",
|
||||
value: null,
|
||||
unit: null,
|
||||
evidenceIds: [],
|
||||
dependsOn: [],
|
||||
affects: [],
|
||||
parentId: null,
|
||||
childIds: [],
|
||||
};
|
||||
|
||||
it("validates a complete valid node", () => {
|
||||
const result = situationNodeSchema.safeParse(validNode);
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("requires id", () => {
|
||||
const invalid = { ...validNode, id: "" };
|
||||
const result = situationNodeSchema.safeParse(invalid);
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it("requires label", () => {
|
||||
const invalid = { ...validNode, label: "" };
|
||||
const result = situationNodeSchema.safeParse(invalid);
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects invalid kind", () => {
|
||||
const invalid = { ...validNode, kind: "nonexistent" };
|
||||
const result = situationNodeSchema.safeParse(invalid);
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects invalid status", () => {
|
||||
const invalid = { ...validNode, status: "unknown_status" };
|
||||
const result = situationNodeSchema.safeParse(invalid);
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects invalid confidence", () => {
|
||||
const invalid = { ...validNode, confidence: "extreme" };
|
||||
const result = situationNodeSchema.safeParse(invalid);
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it("allows numeric value", () => {
|
||||
const node = { ...validNode, value: 42 };
|
||||
const result = situationNodeSchema.safeParse(node);
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("allows string value", () => {
|
||||
const node = { ...validNode, value: "active" };
|
||||
const result = situationNodeSchema.safeParse(node);
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("situationEdgeSchema", () => {
|
||||
const validEdge = {
|
||||
id: "e1",
|
||||
fromNodeId: "n1",
|
||||
toNodeId: "n2",
|
||||
relationship: "supports",
|
||||
confidence: "medium",
|
||||
description: "Edge between nodes",
|
||||
};
|
||||
|
||||
it("validates a complete valid edge", () => {
|
||||
const result = situationEdgeSchema.safeParse(validEdge);
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects invalid relationship type", () => {
|
||||
const invalid = { ...validEdge, relationship: "invalid_rel" };
|
||||
const result = situationEdgeSchema.safeParse(invalid);
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it("validates all relationship types", () => {
|
||||
for (const rel of Object.values(SituationRelationship)) {
|
||||
const edge = { ...validEdge, relationship: rel };
|
||||
const result = situationEdgeSchema.safeParse(edge);
|
||||
expect(result.success).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects self-referencing edges", () => {
|
||||
// Self-refs are structurally valid but semantically questionable
|
||||
const edge = { ...validEdge, fromNodeId: "n1", toNodeId: "n1" };
|
||||
const result = situationEdgeSchema.safeParse(edge);
|
||||
expect(result.success).toBe(true); // Structure is valid; semantics checked elsewhere
|
||||
});
|
||||
});
|
||||
|
||||
describe("situationGraphSchema", () => {
|
||||
const validGraph = {
|
||||
centralStatement: "Test graph summary",
|
||||
nodes: [makeNode({ id: "n1", label: "Node 1" })],
|
||||
edges: [],
|
||||
activeUnknownNodeId: null,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: "Initial summary",
|
||||
};
|
||||
|
||||
it("validates a complete valid graph", () => {
|
||||
const result = situationGraphSchema.safeParse(validGraph);
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("requires at least one node", () => {
|
||||
const invalid = { ...validGraph, nodes: [] };
|
||||
const result = situationGraphSchema.safeParse(invalid);
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it("allows empty edges array", () => {
|
||||
const graph = { ...validGraph, edges: [] };
|
||||
const result = situationGraphSchema.safeParse(graph);
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects missing centralStatement", () => {
|
||||
const invalid = { ...validGraph, centralStatement: "" };
|
||||
const result = situationGraphSchema.safeParse(invalid);
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("graphUpdateSchema", () => {
|
||||
it("validates empty update (no-op proposal)", () => {
|
||||
const result = graphUpdateSchema.safeParse({});
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("validates a complete update", () => {
|
||||
const node = makeNode({ id: "n2", label: "New Node" });
|
||||
const edge = makeEdge({ fromNodeId: "n1", toNodeId: "n2" });
|
||||
|
||||
const result = graphUpdateSchema.safeParse({
|
||||
addedNodes: [node],
|
||||
updatedNodes: [
|
||||
{
|
||||
nodeId: "n1",
|
||||
newStatus: "resolved",
|
||||
previousStatus: "unknown",
|
||||
reason: "Question answered",
|
||||
},
|
||||
],
|
||||
addedEdges: [edge],
|
||||
removedEdgeIds: ["e-old"],
|
||||
resolvedUnknownNodeIds: ["n2"],
|
||||
affectedNodeIds: ["n3"],
|
||||
selectedQuestion: {
|
||||
nodeId: "n2",
|
||||
question: "What does this new node mean?",
|
||||
reason: "A follow-up unknown remains.",
|
||||
},
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("allows null selectedQuestion", () => {
|
||||
const result = graphUpdateSchema.safeParse({
|
||||
selectedQuestion: null,
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects update with invalid node kind in addedNodes", () => {
|
||||
const invalid = graphUpdateSchema.safeParse({
|
||||
addedNodes: [
|
||||
{
|
||||
id: "x",
|
||||
label: "Test",
|
||||
kind: "invalid_kind",
|
||||
description: "test",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
value: null,
|
||||
unit: null,
|
||||
evidenceIds: [],
|
||||
dependsOn: [],
|
||||
affects: [],
|
||||
parentId: null,
|
||||
childIds: [],
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(invalid.success).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("API request schemas", () => {
|
||||
describe("startCaseRequestSchema", () => {
|
||||
it("validates scenario field", () => {
|
||||
const result = startCaseRequestSchema.safeParse({
|
||||
scenario: "Test scenario",
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects empty scenario", () => {
|
||||
const result = startCaseRequestSchema.safeParse({ scenario: "" });
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects scenario over 10000 chars", () => {
|
||||
const longScenario = "a".repeat(10001);
|
||||
const result = startCaseRequestSchema.safeParse({
|
||||
scenario: longScenario,
|
||||
});
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it("accepts optional promptVersion", () => {
|
||||
const result = startCaseRequestSchema.safeParse({
|
||||
scenario: "Test",
|
||||
promptVersion: "v0.3",
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("updateCaseRequestSchema", () => {
|
||||
it("validates complete update request", () => {
|
||||
const graph = makeGraph({
|
||||
centralStatement: "Test scenario",
|
||||
nodes: [makeNode({ id: "n1", label: "N" })],
|
||||
currentSummary: "Current state of situation",
|
||||
});
|
||||
const result = updateCaseRequestSchema.safeParse({
|
||||
situationGraph: graph,
|
||||
previousQuestion: "What happened?",
|
||||
answer: "This is the answer",
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects missing situationGraph", () => {
|
||||
const result = updateCaseRequestSchema.safeParse({
|
||||
previousQuestion: "Q?",
|
||||
answer: "A",
|
||||
});
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects answer over 5000 chars", () => {
|
||||
const graph = makeGraph({
|
||||
centralStatement: "Test",
|
||||
nodes: [makeNode({ id: "n1", label: "N" })],
|
||||
currentSummary: "Test summary",
|
||||
});
|
||||
const result = updateCaseRequestSchema.safeParse({
|
||||
situationGraph: graph,
|
||||
previousQuestion: "Q?",
|
||||
answer: "x".repeat(5001),
|
||||
});
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("deterministic ID generation", () => {
|
||||
it("generate consistent IDs for same label", () => {
|
||||
const id1 = makeNodeId("Same Label");
|
||||
const id2 = makeNodeId("Same Label");
|
||||
expect(id1).toBe(id2);
|
||||
});
|
||||
|
||||
it("generates different IDs for different labels", () => {
|
||||
const id1 = makeNodeId("Label A");
|
||||
const id2 = makeNodeId("Label B");
|
||||
expect(id1).not.toBe(id2);
|
||||
});
|
||||
|
||||
it("IDs are prefixed with 'n' and short", () => {
|
||||
const id = makeNodeId(
|
||||
"A very long label that would produce a longer hash if not truncated",
|
||||
);
|
||||
expect(id.startsWith("n")).toBe(true);
|
||||
expect(id.length).toBeLessThan(15);
|
||||
});
|
||||
|
||||
it("same kind of nodes get deterministic IDs", () => {
|
||||
for (let i = 0; i < 10; i++) {
|
||||
expect(makeNodeId("Test Node")).toBe(makeNodeId("Test Node"));
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("helper functions", () => {
|
||||
describe("makeNode", () => {
|
||||
it("creates a minimal node with defaults", () => {
|
||||
const node = makeNode({ label: "Minimal" });
|
||||
const result = situationNodeSchema.safeParse(node);
|
||||
expect(result.success).toBe(true);
|
||||
expect(node.kind).toBe("observation");
|
||||
expect(node.status).toBe("unknown");
|
||||
expect(node.confidence).toBe("medium");
|
||||
});
|
||||
|
||||
it("creates a node with custom kind/status", () => {
|
||||
const node = makeNode({
|
||||
label: "Custom",
|
||||
kind: "metric",
|
||||
status: "known",
|
||||
confidence: "high",
|
||||
value: 42,
|
||||
unit: "count",
|
||||
});
|
||||
expect(node.kind).toBe("metric");
|
||||
expect(node.status).toBe("known");
|
||||
expect(node.confidence).toBe("high");
|
||||
expect(node.value).toBe(42);
|
||||
expect(node.unit).toBe("count");
|
||||
});
|
||||
|
||||
it("generates ID from label if none provided", () => {
|
||||
const node = makeNode({ label: "Auto-ID" });
|
||||
expect(node.id.startsWith("n")).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("makeEdge", () => {
|
||||
it("creates a minimal edge with defaults", () => {
|
||||
const edge = makeEdge({ fromNodeId: "n1", toNodeId: "n2" });
|
||||
const result = situationEdgeSchema.safeParse(edge);
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("generates description from node ids if not provided", () => {
|
||||
const edge = makeEdge({ fromNodeId: "n-alpha", toNodeId: "n-beta" });
|
||||
expect(edge.description).toContain("alpha");
|
||||
expect(edge.description).toContain("beta");
|
||||
});
|
||||
});
|
||||
|
||||
describe("makeGraph", () => {
|
||||
it("creates a minimal graph with defaults", () => {
|
||||
const graph = makeGraph({
|
||||
centralStatement: "Test",
|
||||
currentSummary: "Default summary",
|
||||
nodes: [makeNode({ id: "n1", label: "Placeholder" })],
|
||||
});
|
||||
const result = situationGraphSchema.safeParse(graph);
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("allows specifying nodes and edges", () => {
|
||||
const graph = makeGraph({
|
||||
centralStatement: "Full Graph",
|
||||
currentSummary: "Full summary",
|
||||
nodes: [makeNode({ id: "n1", label: "N1" })],
|
||||
edges: [makeEdge({ fromNodeId: "n1", toNodeId: "n2" })],
|
||||
});
|
||||
expect(graph.nodes.length).toBe(1);
|
||||
expect(graph.edges.length).toBe(1);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("enum values completeness", () => {
|
||||
it("SituationKind has all expected values", () => {
|
||||
const expected = [
|
||||
"observation",
|
||||
"reported_claim",
|
||||
"metric",
|
||||
"state",
|
||||
"transition",
|
||||
"relationship",
|
||||
"assumption",
|
||||
"unknown",
|
||||
"conclusion",
|
||||
];
|
||||
const actual = Object.values(SituationKind);
|
||||
expect(actual).toEqual(expect.arrayContaining(expected));
|
||||
});
|
||||
|
||||
it("SituationStatus has all expected values", () => {
|
||||
const expected = [
|
||||
"known",
|
||||
"unknown",
|
||||
"provisional",
|
||||
"supported",
|
||||
"weakened",
|
||||
"contradicted",
|
||||
"resolved",
|
||||
];
|
||||
const actual = Object.values(SituationStatus);
|
||||
expect(actual).toEqual(expect.arrayContaining(expected));
|
||||
});
|
||||
|
||||
it("SituationRelationship has all expected values", () => {
|
||||
const expected = [
|
||||
"supports",
|
||||
"weakens",
|
||||
"contradicts",
|
||||
"depends_on",
|
||||
"causes",
|
||||
"may_cause",
|
||||
"measures",
|
||||
"compares_with",
|
||||
"updates",
|
||||
"other",
|
||||
];
|
||||
const actual = Object.values(SituationRelationship);
|
||||
expect(actual).toEqual(expect.arrayContaining(expected));
|
||||
});
|
||||
|
||||
it("ConfidenceLevel has all expected values", () => {
|
||||
const actual = Object.values(ConfidenceLevel);
|
||||
expect(actual).toContain("low");
|
||||
expect(actual).toContain("medium");
|
||||
expect(actual).toContain("high");
|
||||
});
|
||||
});
|
||||
@@ -1,303 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
formulateQuestion,
|
||||
formulateTieResolutionQuestion,
|
||||
} from "@/lib/graph/question-formulator.js";
|
||||
import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js";
|
||||
import {
|
||||
explainUnknownSelection,
|
||||
selectActiveUnknownCandidate,
|
||||
} from "@/lib/graph/utils.js";
|
||||
|
||||
function buildLiveShapedGraph() {
|
||||
const summary = makeNode({
|
||||
id: "nnvog0y",
|
||||
label:
|
||||
"Revenue grew by 18% while corporate cash reserves declined over an identical time frame.",
|
||||
description: "Summary of the situation from the scenario text",
|
||||
kind: "state",
|
||||
status: "provisional",
|
||||
confidence: "medium",
|
||||
});
|
||||
const revenueObservation = makeNode({
|
||||
id: "nri36w9",
|
||||
label: "Revenue increased by 18%.",
|
||||
description: "Revenue increased by 18%.",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
evidenceIds: ["obs_rev"],
|
||||
});
|
||||
const cashObservation = makeNode({
|
||||
id: "nnfc48j",
|
||||
label: "Cash in the bank decreased over the same period.",
|
||||
description: "Cash in the bank decreased over the same period.",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
evidenceIds: ["obs_cash"],
|
||||
});
|
||||
const revenueMetric = makeNode({
|
||||
id: "nhsd6d5",
|
||||
label: "Revenue metric (typically accrual-based income statement figure)",
|
||||
description:
|
||||
"Revenue metric (typically accrual-based income statement figure)",
|
||||
kind: "metric",
|
||||
status: "known",
|
||||
confidence: "high",
|
||||
});
|
||||
const cashMetric = makeNode({
|
||||
id: "neh5m6m",
|
||||
label:
|
||||
"Cash balance (liquidity measure on the balance sheet or cash flow statement)",
|
||||
description:
|
||||
"Cash balance (liquidity measure on the balance sheet or cash flow statement)",
|
||||
kind: "metric",
|
||||
status: "known",
|
||||
confidence: "high",
|
||||
});
|
||||
const directionalRelationship = makeNode({
|
||||
id: "nwo6070",
|
||||
label:
|
||||
"Divergent directional movement between top-line revenue growth and net cash position contraction.",
|
||||
description:
|
||||
"Divergent directional movement between top-line revenue growth and net cash position contraction.",
|
||||
kind: "relationship",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
});
|
||||
const contradictionRelationship = makeNode({
|
||||
id: "nuiab02",
|
||||
label:
|
||||
"Apparent contradiction between profitability/revenue expansion and liquidity reduction.",
|
||||
description:
|
||||
"Apparent contradiction between profitability/revenue expansion and liquidity reduction.",
|
||||
kind: "relationship",
|
||||
status: "supported",
|
||||
confidence: "medium",
|
||||
});
|
||||
const cashTiming = makeNode({
|
||||
id: "niewza",
|
||||
label:
|
||||
"Whether revenue recognition timing differs from cash collection timing.",
|
||||
description:
|
||||
"Whether revenue recognition timing differs from cash collection timing.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
});
|
||||
const cashOutflows = makeNode({
|
||||
id: "nqdzobz",
|
||||
label:
|
||||
"Magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts).",
|
||||
description:
|
||||
"Magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts).",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
});
|
||||
|
||||
const edges = [
|
||||
makeEdge({
|
||||
id: "e-revenue-summary",
|
||||
fromNodeId: revenueObservation.id,
|
||||
toNodeId: summary.id,
|
||||
relationship: "supports",
|
||||
description: "Revenue increase supports the scenario summary.",
|
||||
}),
|
||||
makeEdge({
|
||||
id: "e-cash-summary",
|
||||
fromNodeId: cashObservation.id,
|
||||
toNodeId: summary.id,
|
||||
relationship: "supports",
|
||||
description: "Cash decline supports the scenario summary.",
|
||||
}),
|
||||
makeEdge({
|
||||
id: "e-unk-niewza",
|
||||
fromNodeId: cashTiming.id,
|
||||
toNodeId: summary.id,
|
||||
relationship: "depends_on",
|
||||
description:
|
||||
"Whether revenue recognition timing differs from cash collection timing. is an unresolved factor for this situation",
|
||||
}),
|
||||
makeEdge({
|
||||
id: "e-unk-nqdzobz",
|
||||
fromNodeId: cashOutflows.id,
|
||||
toNodeId: summary.id,
|
||||
relationship: "depends_on",
|
||||
description:
|
||||
"Magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts). is an unresolved factor for this situation",
|
||||
}),
|
||||
];
|
||||
|
||||
return makeGraph({
|
||||
centralStatement:
|
||||
"Revenue increased by 18%, but cash in the bank fell over the same period.",
|
||||
nodes: [
|
||||
summary,
|
||||
revenueObservation,
|
||||
cashObservation,
|
||||
revenueMetric,
|
||||
cashMetric,
|
||||
directionalRelationship,
|
||||
contradictionRelationship,
|
||||
cashTiming,
|
||||
cashOutflows,
|
||||
],
|
||||
edges,
|
||||
activeUnknownNodeId: null,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: "Diagnostic selection influence fixture",
|
||||
});
|
||||
}
|
||||
|
||||
function orderCandidates(explanation) {
|
||||
return explanation.candidates.map((candidate) => ({
|
||||
nodeId: candidate.nodeId,
|
||||
label: candidate.label,
|
||||
score: candidate.score,
|
||||
downstreamCount: candidate.downstreamCount,
|
||||
unresolvedParentUnknownCount: candidate.unresolvedParentUnknownCount,
|
||||
}));
|
||||
}
|
||||
|
||||
function removeDependencyLinks(graph) {
|
||||
const nodes = graph.nodes.map((node) => ({
|
||||
...node,
|
||||
dependsOn: [],
|
||||
affects: [],
|
||||
parentId: null,
|
||||
childIds: [],
|
||||
}));
|
||||
const edges = (graph.edges || []).filter(
|
||||
(edge) => edge.relationship !== "depends_on",
|
||||
);
|
||||
return makeGraph({ ...graph, nodes, edges, activeUnknownNodeId: null });
|
||||
}
|
||||
|
||||
function neutraliseUnknownWording(graph) {
|
||||
let counter = 0;
|
||||
const nodes = graph.nodes.map((node) => {
|
||||
if (node.kind !== "unknown") return { ...node };
|
||||
counter += 1;
|
||||
return {
|
||||
...node,
|
||||
label: `Unknown ${String.fromCharCode(64 + counter)}`,
|
||||
description: `Unknown factor ${counter} relevant to the scenario.`,
|
||||
};
|
||||
});
|
||||
return makeGraph({ ...graph, nodes, activeUnknownNodeId: null });
|
||||
}
|
||||
|
||||
describe("selection influence diagnostic", () => {
|
||||
it("records ambiguous ordering changes for live-shaped, structure-only, and wording-neutralised fixtures", () => {
|
||||
const liveGraph = buildLiveShapedGraph();
|
||||
const liveExplanation = explainUnknownSelection(liveGraph, []);
|
||||
const liveSelection = selectActiveUnknownCandidate(liveGraph, []);
|
||||
const tieQuestion = formulateTieResolutionQuestion({ graph: liveGraph });
|
||||
|
||||
const noLinksExplanation = explainUnknownSelection(
|
||||
removeDependencyLinks(liveGraph),
|
||||
[],
|
||||
);
|
||||
const noLinksSelection = selectActiveUnknownCandidate(
|
||||
removeDependencyLinks(liveGraph),
|
||||
[],
|
||||
);
|
||||
const neutralWordingExplanation = explainUnknownSelection(
|
||||
neutraliseUnknownWording(liveGraph),
|
||||
[],
|
||||
);
|
||||
const neutralSelection = selectActiveUnknownCandidate(
|
||||
neutraliseUnknownWording(liveGraph),
|
||||
[],
|
||||
);
|
||||
|
||||
const fallbackQuestion = formulateQuestion({
|
||||
node: liveGraph.nodes.find((node) => node.id === "nqdzobz"),
|
||||
graph: liveGraph,
|
||||
});
|
||||
|
||||
const diagnosticRecord = {
|
||||
liveStatus: liveExplanation.status,
|
||||
liveShapedCandidateOrdering: orderCandidates(liveExplanation),
|
||||
liveTiedCandidateIds: liveExplanation.tiedCandidateIds,
|
||||
noLinksCandidateOrdering: orderCandidates(noLinksExplanation),
|
||||
noLinksStatus: noLinksExplanation.status,
|
||||
neutralWordingCandidateOrdering: orderCandidates(
|
||||
neutralWordingExplanation,
|
||||
),
|
||||
neutralStatus: neutralWordingExplanation.status,
|
||||
selectedExplanationContributions: liveExplanation.selected?.contributions,
|
||||
tieQuestion: tieQuestion.question,
|
||||
liveSelection,
|
||||
noLinksSelection,
|
||||
neutralSelection,
|
||||
fallbackQuestion,
|
||||
};
|
||||
|
||||
expect(diagnosticRecord.liveStatus).toBe("ambiguous");
|
||||
expect(diagnosticRecord.liveTiedCandidateIds).toEqual([
|
||||
"nqdzobz",
|
||||
"niewza",
|
||||
]);
|
||||
expect(diagnosticRecord.liveShapedCandidateOrdering).toEqual([
|
||||
{
|
||||
nodeId: "nqdzobz",
|
||||
label:
|
||||
"Magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts).",
|
||||
score: 0,
|
||||
downstreamCount: 0,
|
||||
unresolvedParentUnknownCount: 0,
|
||||
},
|
||||
{
|
||||
nodeId: "niewza",
|
||||
label:
|
||||
"Whether revenue recognition timing differs from cash collection timing.",
|
||||
score: 0,
|
||||
downstreamCount: 0,
|
||||
unresolvedParentUnknownCount: 0,
|
||||
},
|
||||
]);
|
||||
expect(diagnosticRecord.liveSelection).toMatchObject({
|
||||
selectedNode: null,
|
||||
status: "ambiguous",
|
||||
tieType: "complete_unresolved_tie",
|
||||
tiedCandidateIds: ["nqdzobz", "niewza"],
|
||||
});
|
||||
expect(diagnosticRecord.noLinksCandidateOrdering).toEqual(
|
||||
diagnosticRecord.liveShapedCandidateOrdering,
|
||||
);
|
||||
expect(diagnosticRecord.noLinksStatus).toBe("ambiguous");
|
||||
expect(diagnosticRecord.noLinksSelection.status).toBe("ambiguous");
|
||||
expect(diagnosticRecord.neutralWordingCandidateOrdering).toEqual([
|
||||
{
|
||||
nodeId: "niewza",
|
||||
label: "Unknown A",
|
||||
score: 0,
|
||||
downstreamCount: 0,
|
||||
unresolvedParentUnknownCount: 0,
|
||||
},
|
||||
{
|
||||
nodeId: "nqdzobz",
|
||||
label: "Unknown B",
|
||||
score: 0,
|
||||
downstreamCount: 0,
|
||||
unresolvedParentUnknownCount: 0,
|
||||
},
|
||||
]);
|
||||
expect(diagnosticRecord.neutralStatus).toBe("ambiguous");
|
||||
expect(diagnosticRecord.neutralSelection.status).toBe("ambiguous");
|
||||
expect(diagnosticRecord.selectedExplanationContributions).toBeUndefined();
|
||||
expect(diagnosticRecord.tieQuestion).toBe(
|
||||
"Were these figures measured on the same basis and at the same scale?",
|
||||
);
|
||||
expect(diagnosticRecord.tieQuestion.toLowerCase()).not.toMatch(
|
||||
/accounts receivable|capex|debt repayments|working capital/,
|
||||
);
|
||||
expect(diagnosticRecord.fallbackQuestion.strategy).toBeNull();
|
||||
expect(diagnosticRecord.fallbackQuestion.question).toBe(
|
||||
"What would clarify magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts) in this situation?",
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -1,152 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { parseGraphUpdateProposal } from "@/lib/graph/update-proposal.js";
|
||||
|
||||
function makeValidProposal(overrides = {}) {
|
||||
return {
|
||||
addedNodes: [],
|
||||
updatedNodes: [
|
||||
{
|
||||
nodeId: "n-unknown",
|
||||
previousStatus: "unknown",
|
||||
newStatus: "resolved",
|
||||
previousValue: null,
|
||||
newValue: "1.9 complaints per 100 units",
|
||||
reason: "The answer directly provides the normalized complaint rate.",
|
||||
},
|
||||
],
|
||||
addedEdges: [],
|
||||
removedEdgeIds: [],
|
||||
resolvedUnknownNodeIds: ["n-unknown"],
|
||||
affectedNodeIds: [],
|
||||
selectedQuestion: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("parseGraphUpdateProposal", () => {
|
||||
it("parses a valid proposal", () => {
|
||||
const result = parseGraphUpdateProposal(makeValidProposal());
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.proposal.updatedNodes).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("fails on malformed JSON", () => {
|
||||
const result = parseGraphUpdateProposal("{not json");
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it("fails when required update content is invalid", () => {
|
||||
const result = parseGraphUpdateProposal({
|
||||
updatedNodes: [{ nodeId: "n-unknown" }],
|
||||
});
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it("removes null array entries and logs them", () => {
|
||||
const result = parseGraphUpdateProposal(
|
||||
JSON.stringify({
|
||||
...makeValidProposal(),
|
||||
addedNodes: [null],
|
||||
}),
|
||||
);
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.proposal.addedNodes).toEqual([]);
|
||||
expect(result.normalisationsApplied).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({ change: "Removed null array entry" }),
|
||||
]),
|
||||
);
|
||||
});
|
||||
|
||||
it("fills missing optional arrays with empty arrays", () => {
|
||||
const result = parseGraphUpdateProposal({
|
||||
updatedNodes: [],
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.proposal.addedNodes).toEqual([]);
|
||||
expect(result.proposal.addedEdges).toEqual([]);
|
||||
expect(result.normalisationsApplied.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("normalises confirmed enum alias and preserves IDs", () => {
|
||||
const result = parseGraphUpdateProposal({
|
||||
...makeValidProposal(),
|
||||
addedNodes: [
|
||||
{
|
||||
id: "n-new",
|
||||
label: "Reported update",
|
||||
description: "A new reported claim",
|
||||
kind: "reported_statement",
|
||||
status: "supported",
|
||||
confidence: "medium",
|
||||
value: null,
|
||||
unit: null,
|
||||
evidenceIds: [],
|
||||
dependsOn: [],
|
||||
affects: [],
|
||||
parentId: null,
|
||||
childIds: [],
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.proposal.addedNodes[0].kind).toBe("reported_claim");
|
||||
expect(result.proposal.addedNodes[0].id).toBe("n-new");
|
||||
});
|
||||
|
||||
it("unknown enum values still fail", () => {
|
||||
const result = parseGraphUpdateProposal({
|
||||
...makeValidProposal(),
|
||||
addedNodes: [
|
||||
{
|
||||
id: "n-new",
|
||||
label: "Bad node",
|
||||
description: "Bad node",
|
||||
kind: "unsupported_kind",
|
||||
status: "supported",
|
||||
confidence: "medium",
|
||||
value: null,
|
||||
unit: null,
|
||||
evidenceIds: [],
|
||||
dependsOn: [],
|
||||
affects: [],
|
||||
parentId: null,
|
||||
childIds: [],
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it("defaults missing selectedQuestion to null", () => {
|
||||
const result = parseGraphUpdateProposal({
|
||||
addedNodes: [],
|
||||
updatedNodes: [],
|
||||
addedEdges: [],
|
||||
removedEdgeIds: [],
|
||||
resolvedUnknownNodeIds: [],
|
||||
affectedNodeIds: [],
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.proposal.selectedQuestion).toBeNull();
|
||||
});
|
||||
|
||||
it("parses a valid selectedQuestion", () => {
|
||||
const result = parseGraphUpdateProposal(
|
||||
makeValidProposal({
|
||||
selectedQuestion: {
|
||||
nodeId: "n-follow-up",
|
||||
question: "How should commercial value be defined for this decision?",
|
||||
reason: "A consequential unknown remains unresolved.",
|
||||
},
|
||||
}),
|
||||
);
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.proposal.selectedQuestion?.nodeId).toBe("n-follow-up");
|
||||
});
|
||||
|
||||
it("does not invent a next question field outside the contract", () => {
|
||||
const result = parseGraphUpdateProposal(makeValidProposal());
|
||||
expect(result.proposal.nextQuestion).toBeUndefined();
|
||||
});
|
||||
});
|
||||
@@ -1,419 +0,0 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { applyValidatedProposal } from "@/lib/graph/apply-proposal.js";
|
||||
import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js";
|
||||
|
||||
function makePropagationFixture({
|
||||
key,
|
||||
centralStatement,
|
||||
firstObservationLabel,
|
||||
secondObservationLabel,
|
||||
}) {
|
||||
const parent = makeNode({
|
||||
id: `${key}-parent`,
|
||||
label: `Explanation for why ${centralStatement}`,
|
||||
description:
|
||||
"Need to understand what change or event could explain why these observations differ, because that is needed to investigate their relationship.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
const measurementChild = makeNode({
|
||||
id: `${key}-child-measurement`,
|
||||
label: "How the two observations were measured",
|
||||
description: `Need evidence about the measure used for each observation, because that could help explain ${centralStatement}.`,
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
parentId: parent.id,
|
||||
});
|
||||
const timingChild = makeNode({
|
||||
id: `${key}-child-timing`,
|
||||
label: "Whether the two observations reflect different timing",
|
||||
description: `Need to know whether the two observations reflect different timing, because that could help explain ${centralStatement}.`,
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
parentId: parent.id,
|
||||
});
|
||||
const cashMovementChild = makeNode({
|
||||
id: `${key}-child-cash-movement`,
|
||||
label: `Possible change mainly affecting ${secondObservationLabel}`,
|
||||
description: `Need to know whether a possible change mainly affected ${secondObservationLabel}, because that could help explain ${centralStatement}.`,
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
parentId: parent.id,
|
||||
});
|
||||
const oneOffChild = makeNode({
|
||||
id: `${key}-child-one-off`,
|
||||
label: "Possible one-off event during the period",
|
||||
description: `Need to know whether a possible one-off event happened during the period, because that could help explain ${centralStatement}.`,
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
parentId: parent.id,
|
||||
});
|
||||
const ancestor = makeNode({
|
||||
id: `${key}-ancestor`,
|
||||
label: `Reasoning for ${centralStatement}`,
|
||||
description:
|
||||
"Higher-level reasoning node depending on the parent explanation.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
childIds: [parent.id],
|
||||
});
|
||||
const unrelated = makeNode({
|
||||
id: `${key}-unrelated`,
|
||||
label: "Unrelated branch",
|
||||
description: "Should remain unchanged.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "low",
|
||||
});
|
||||
const firstObservation = makeNode({
|
||||
id: `${key}-obs-1`,
|
||||
label: firstObservationLabel,
|
||||
description: firstObservationLabel,
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
});
|
||||
const secondObservation = makeNode({
|
||||
id: `${key}-obs-2`,
|
||||
label: secondObservationLabel,
|
||||
description: secondObservationLabel,
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
});
|
||||
|
||||
const graph = makeGraph({
|
||||
centralStatement,
|
||||
nodes: [
|
||||
ancestor,
|
||||
parent,
|
||||
measurementChild,
|
||||
timingChild,
|
||||
cashMovementChild,
|
||||
oneOffChild,
|
||||
unrelated,
|
||||
firstObservation,
|
||||
secondObservation,
|
||||
],
|
||||
edges: [
|
||||
makeEdge({
|
||||
id: `${key}-e-parent-ancestor`,
|
||||
fromNodeId: parent.id,
|
||||
toNodeId: ancestor.id,
|
||||
relationship: "depends_on",
|
||||
description: "Ancestor depends on the parent explanation.",
|
||||
}),
|
||||
makeEdge({
|
||||
id: `${key}-e-child-measurement-parent`,
|
||||
fromNodeId: measurementChild.id,
|
||||
toNodeId: parent.id,
|
||||
relationship: "depends_on",
|
||||
description: "Measurement child depends into the parent explanation.",
|
||||
}),
|
||||
makeEdge({
|
||||
id: `${key}-e-child-timing-parent`,
|
||||
fromNodeId: timingChild.id,
|
||||
toNodeId: parent.id,
|
||||
relationship: "depends_on",
|
||||
description: "Timing child depends into the parent explanation.",
|
||||
}),
|
||||
makeEdge({
|
||||
id: `${key}-e-child-cash-parent`,
|
||||
fromNodeId: cashMovementChild.id,
|
||||
toNodeId: parent.id,
|
||||
relationship: "depends_on",
|
||||
description: "Cash-movement child depends into the parent explanation.",
|
||||
}),
|
||||
makeEdge({
|
||||
id: `${key}-e-child-one-off-parent`,
|
||||
fromNodeId: oneOffChild.id,
|
||||
toNodeId: parent.id,
|
||||
relationship: "depends_on",
|
||||
description: "One-off child depends into the parent explanation.",
|
||||
}),
|
||||
],
|
||||
activeUnknownNodeId: measurementChild.id,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: `Propagation fixture for ${key}`,
|
||||
});
|
||||
|
||||
return {
|
||||
graph,
|
||||
ids: {
|
||||
ancestor: ancestor.id,
|
||||
parent: parent.id,
|
||||
measurementChild: measurementChild.id,
|
||||
timingChild: timingChild.id,
|
||||
cashMovementChild: cashMovementChild.id,
|
||||
oneOffChild: oneOffChild.id,
|
||||
unrelated: unrelated.id,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const scenarios = [
|
||||
{
|
||||
key: "revenue-cash",
|
||||
centralStatement: "revenue increased while cash fell",
|
||||
firstObservationLabel: "Revenue increased by 18%.",
|
||||
secondObservationLabel: "Cash in the bank fell over the same period.",
|
||||
},
|
||||
{
|
||||
key: "satisfaction-complaints",
|
||||
centralStatement:
|
||||
"customer satisfaction increased while complaints increased",
|
||||
firstObservationLabel: "Customer satisfaction increased.",
|
||||
secondObservationLabel: "Complaints increased.",
|
||||
},
|
||||
{
|
||||
key: "traffic-sales",
|
||||
centralStatement: "traffic increased while sales stayed flat",
|
||||
firstObservationLabel: "Website traffic increased.",
|
||||
secondObservationLabel: "Sales stayed flat.",
|
||||
},
|
||||
{
|
||||
key: "delivery-cancellations",
|
||||
centralStatement: "delivery time fell while cancellations increased",
|
||||
firstObservationLabel: "Average delivery time decreased.",
|
||||
secondObservationLabel: "Cancellations increased.",
|
||||
},
|
||||
{
|
||||
key: "production-defects",
|
||||
centralStatement: "production increased while defects increased",
|
||||
firstObservationLabel: "Production increased.",
|
||||
secondObservationLabel: "Defects increased.",
|
||||
},
|
||||
];
|
||||
|
||||
describe("upward propagation", () => {
|
||||
it.each(scenarios)(
|
||||
"propagates resolved measurement child upward for $key",
|
||||
({
|
||||
key,
|
||||
centralStatement,
|
||||
firstObservationLabel,
|
||||
secondObservationLabel,
|
||||
}) => {
|
||||
const { graph, ids } = makePropagationFixture({
|
||||
key,
|
||||
centralStatement,
|
||||
firstObservationLabel,
|
||||
secondObservationLabel,
|
||||
});
|
||||
const unrelatedBefore = JSON.stringify(
|
||||
graph.nodes.find((node) => node.id === ids.unrelated),
|
||||
);
|
||||
|
||||
const result = applyValidatedProposal({
|
||||
situationGraph: graph,
|
||||
proposal: {
|
||||
addedNodes: [
|
||||
makeNode({
|
||||
id: `${key}-anchor`,
|
||||
label: "Update anchor",
|
||||
description:
|
||||
"Anchor state introduced by the answer because the update must contain a meaningful change.",
|
||||
kind: "state",
|
||||
status: "known",
|
||||
confidence: "low",
|
||||
}),
|
||||
],
|
||||
updatedNodes: [
|
||||
{
|
||||
nodeId: ids.measurementChild,
|
||||
previousStatus: "unknown",
|
||||
newStatus: "resolved",
|
||||
previousValue: null,
|
||||
newValue:
|
||||
"The figures were measured over the same accounting period using the same management accounts.",
|
||||
reason: "The answer resolves the measurement child.",
|
||||
},
|
||||
],
|
||||
addedEdges: [],
|
||||
removedEdgeIds: [],
|
||||
resolvedUnknownNodeIds: [ids.measurementChild],
|
||||
affectedNodeIds: [],
|
||||
selectedQuestion: null,
|
||||
},
|
||||
previousQuestion:
|
||||
"What evidence would clarify how the two observations were measured?",
|
||||
answer:
|
||||
"The figures were measured over the same accounting period using the same management accounts.",
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.resolvedUnknownNodeIds).toContain(ids.measurementChild);
|
||||
expect(result.propagationPerformed).toBe(true);
|
||||
expect(result.resolvedChildNodeId).toBe(ids.measurementChild);
|
||||
expect(result.parentNodeId).toBe(ids.parent);
|
||||
expect(result.parentStatusBefore).toBe("unknown");
|
||||
expect(result.parentStatusAfter).toBe("provisional");
|
||||
expect(result.parentConfidenceBefore).toBe("medium");
|
||||
expect(result.parentConfidenceAfter).toBe("medium");
|
||||
expect(result.evidenceConfidenceBefore).toBe("medium");
|
||||
expect(result.evidenceConfidenceAfter).toBe("medium");
|
||||
expect(result.completenessBefore).toBe("empty");
|
||||
expect(result.completenessAfter).toBe("partial");
|
||||
expect(result.conclusionConfidenceBefore).toBe("low");
|
||||
expect(result.conclusionConfidenceAfter).toBe("medium");
|
||||
expect(result.confidenceCapReason).toBe(
|
||||
"unresolved_direct_children_cap_conclusion",
|
||||
);
|
||||
expect(result.parentResolved).toBe(false);
|
||||
expect(result.affectedAncestorIds).toContain(ids.parent);
|
||||
expect(result.affectedAncestorIds).toContain(ids.ancestor);
|
||||
expect(result.nextSelectedSibling).toBe(result.newActiveUnknownNodeId);
|
||||
expect(result.nextSelectedSibling).toBe(result.selectedQuestion?.nodeId);
|
||||
expect(result.nextSelectedSibling).not.toBe(ids.measurementChild);
|
||||
expect([
|
||||
ids.timingChild,
|
||||
ids.cashMovementChild,
|
||||
ids.oneOffChild,
|
||||
]).toContain(result.nextSelectedSibling);
|
||||
expect(result.selectedQuestion?.question.toLowerCase()).not.toContain(
|
||||
"measured",
|
||||
);
|
||||
|
||||
const parentNode = result.updatedSituationGraph.nodes.find(
|
||||
(node) => node.id === ids.parent,
|
||||
);
|
||||
expect(parentNode).toMatchObject({
|
||||
status: "provisional",
|
||||
confidence: "medium",
|
||||
confidenceAssessment: {
|
||||
evidenceConfidence: "medium",
|
||||
completenessStatus: "partial",
|
||||
conclusionConfidence: "medium",
|
||||
},
|
||||
});
|
||||
|
||||
const ancestorNode = result.updatedSituationGraph.nodes.find(
|
||||
(node) => node.id === ids.ancestor,
|
||||
);
|
||||
expect(ancestorNode).toMatchObject({
|
||||
status: "provisional",
|
||||
confidence: "low",
|
||||
confidenceAssessment: {
|
||||
evidenceConfidence: "low",
|
||||
completenessStatus: "empty",
|
||||
conclusionConfidence: "low",
|
||||
},
|
||||
});
|
||||
|
||||
const resolvedChild = result.updatedSituationGraph.nodes.find(
|
||||
(node) => node.id === ids.measurementChild,
|
||||
);
|
||||
expect(resolvedChild.status).toBe("resolved");
|
||||
expect(resolvedChild.evidenceIds).toContain(
|
||||
`answer:${ids.measurementChild}`,
|
||||
);
|
||||
|
||||
expect(
|
||||
result.updatedSituationGraph.nodes.filter(
|
||||
(node) => node.id === ids.measurementChild,
|
||||
),
|
||||
).toHaveLength(1);
|
||||
expect(
|
||||
JSON.stringify(
|
||||
result.updatedSituationGraph.nodes.find(
|
||||
(node) => node.id === ids.unrelated,
|
||||
),
|
||||
),
|
||||
).toBe(unrelatedBefore);
|
||||
},
|
||||
);
|
||||
|
||||
it("resolves the parent only after all direct children are resolved", () => {
|
||||
const { graph, ids } = makePropagationFixture({
|
||||
key: "completion-rule",
|
||||
centralStatement: "revenue increased while cash fell",
|
||||
firstObservationLabel: "Revenue increased by 18%.",
|
||||
secondObservationLabel: "Cash in the bank fell over the same period.",
|
||||
});
|
||||
|
||||
const result = applyValidatedProposal({
|
||||
situationGraph: graph,
|
||||
proposal: {
|
||||
addedNodes: [
|
||||
makeNode({
|
||||
id: "completion-rule-anchor",
|
||||
label: "Update anchor",
|
||||
description:
|
||||
"Anchor state introduced by the answer because the update must contain a meaningful change.",
|
||||
kind: "state",
|
||||
status: "known",
|
||||
confidence: "low",
|
||||
}),
|
||||
],
|
||||
updatedNodes: [
|
||||
{
|
||||
nodeId: ids.measurementChild,
|
||||
previousStatus: "unknown",
|
||||
newStatus: "resolved",
|
||||
previousValue: null,
|
||||
newValue: "same management accounts",
|
||||
reason: "resolved measurement child",
|
||||
},
|
||||
{
|
||||
nodeId: ids.timingChild,
|
||||
previousStatus: "unknown",
|
||||
newStatus: "resolved",
|
||||
previousValue: null,
|
||||
newValue: "timing aligned",
|
||||
reason: "resolved timing child",
|
||||
},
|
||||
{
|
||||
nodeId: ids.cashMovementChild,
|
||||
previousStatus: "unknown",
|
||||
newStatus: "resolved",
|
||||
previousValue: null,
|
||||
newValue: "cash left through operations",
|
||||
reason: "resolved movement child",
|
||||
},
|
||||
{
|
||||
nodeId: ids.oneOffChild,
|
||||
previousStatus: "unknown",
|
||||
newStatus: "resolved",
|
||||
previousValue: null,
|
||||
newValue: "no exceptional movement",
|
||||
reason: "resolved one-off child",
|
||||
},
|
||||
],
|
||||
addedEdges: [],
|
||||
removedEdgeIds: [],
|
||||
resolvedUnknownNodeIds: [
|
||||
ids.measurementChild,
|
||||
ids.timingChild,
|
||||
ids.cashMovementChild,
|
||||
ids.oneOffChild,
|
||||
],
|
||||
affectedNodeIds: [],
|
||||
selectedQuestion: null,
|
||||
},
|
||||
previousQuestion:
|
||||
"What evidence would clarify how the two observations were measured?",
|
||||
answer: "All direct child questions are now answered.",
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.parentResolved).toBe(true);
|
||||
expect(result.resolvedUnknownNodeIds).toContain(ids.parent);
|
||||
expect(
|
||||
result.updatedSituationGraph.nodes.find((node) => node.id === ids.parent),
|
||||
).toMatchObject({
|
||||
status: "resolved",
|
||||
confidence: "high",
|
||||
confidenceAssessment: {
|
||||
evidenceConfidence: "high",
|
||||
completenessStatus: "complete",
|
||||
conclusionConfidence: "high",
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
+565
-121
@@ -1,8 +1,23 @@
|
||||
import { describe, it, expect, vi } from "vitest";
|
||||
import { reconstructionSchema } from "@/lib/reconstruction/schema";
|
||||
import { parseReconstruction } from "@/lib/reconstruction/schema";
|
||||
import { describe, it, expect } from "vitest";
|
||||
import {
|
||||
reconstructionSchema,
|
||||
confidenceEnum,
|
||||
importanceEnum,
|
||||
inputTypes,
|
||||
reasoningModes,
|
||||
evidenceRecordSchema,
|
||||
reconstructionV2Schema,
|
||||
analyseResponseSchema,
|
||||
parseReconstruction,
|
||||
parseReconstructionV2,
|
||||
} from "@/lib/reconstruction/schema";
|
||||
import { CONFIDENCE_VALUES } from "@/lib/llm/types.js";
|
||||
|
||||
describe("reconstruction schema", () => {
|
||||
// ──────────────────────────────────────────────
|
||||
// v0.1 — backward compatibility tests
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
describe("v0.1 reconstruction schema", () => {
|
||||
it("validates a complete valid reconstruction", () => {
|
||||
const input = {
|
||||
observations: [{ id: "o1", description: "Saw smoke", confidence: "high" }],
|
||||
@@ -23,34 +38,19 @@ describe("reconstruction schema", () => {
|
||||
it("rejects invalid confidence values", () => {
|
||||
const input = {
|
||||
observations: [{ id: "o1", description: "test", confidence: "extreme" }],
|
||||
reportedClaims: [],
|
||||
assumptions: [],
|
||||
entities: [],
|
||||
transitions: [],
|
||||
expectedButMissing: [],
|
||||
presentButUnexpected: [],
|
||||
contradictions: [],
|
||||
openUncertainties: [],
|
||||
reportedClaims: [], assumptions: [], entities: [], transitions: [],
|
||||
expectedButMissing: [], presentButUnexpected: [], contradictions: [], openUncertainties: [],
|
||||
};
|
||||
|
||||
const result = reconstructionSchema.safeParse(input);
|
||||
expect(result.success).toBe(false);
|
||||
if (!result.success) {
|
||||
expect(result.error.issues[0].message).toContain("Expected");
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects missing required fields", () => {
|
||||
const input = {
|
||||
observations: [{ id: "o1" }],
|
||||
reportedClaims: [],
|
||||
assumptions: [],
|
||||
entities: [],
|
||||
transitions: [],
|
||||
expectedButMissing: [],
|
||||
presentButUnexpected: [],
|
||||
contradictions: [],
|
||||
openUncertainties: [],
|
||||
reportedClaims: [], assumptions: [], entities: [], transitions: [],
|
||||
expectedButMissing: [], presentButUnexpected: [], contradictions: [], openUncertainties: [],
|
||||
};
|
||||
|
||||
const result = reconstructionSchema.safeParse(input);
|
||||
@@ -61,13 +61,7 @@ describe("reconstruction schema", () => {
|
||||
const input = {
|
||||
observations: [],
|
||||
reportedClaims: [{ id: "rc1", description: "test", confidence: "very_high", attributedTo: null }],
|
||||
assumptions: [],
|
||||
entities: [],
|
||||
transitions: [],
|
||||
expectedButMissing: [],
|
||||
presentButUnexpected: [],
|
||||
contradictions: [],
|
||||
openUncertainties: [],
|
||||
assumptions: [], entities: [], transitions: [], expectedButMissing: [], presentButUnexpected: [], contradictions: [], openUncertainties: [],
|
||||
};
|
||||
|
||||
const result = reconstructionSchema.safeParse(input);
|
||||
@@ -76,15 +70,9 @@ describe("reconstruction schema", () => {
|
||||
|
||||
it("rejects empty transitions", () => {
|
||||
const input = {
|
||||
observations: [],
|
||||
reportedClaims: [],
|
||||
assumptions: [],
|
||||
entities: [],
|
||||
observations: [], reportedClaims: [], assumptions: [], entities: [],
|
||||
transitions: [{ id: "t1", description: "", confidence: "high", entity: "", previousState: "", currentState: "", explanationStatus: "" }],
|
||||
expectedButMissing: [],
|
||||
presentButUnexpected: [],
|
||||
contradictions: [],
|
||||
openUncertainties: [],
|
||||
expectedButMissing: [], presentButUnexpected: [], contradictions: [], openUncertainties: [],
|
||||
};
|
||||
|
||||
const result = reconstructionSchema.safeParse(input);
|
||||
@@ -95,13 +83,7 @@ describe("reconstruction schema", () => {
|
||||
const input = {
|
||||
observations: [],
|
||||
reportedClaims: [{ id: "rc1", description: "Someone called it in", confidence: "medium", attributedTo: null }],
|
||||
assumptions: [],
|
||||
entities: [],
|
||||
transitions: [],
|
||||
expectedButMissing: [],
|
||||
presentButUnexpected: [],
|
||||
contradictions: [],
|
||||
openUncertainties: [],
|
||||
assumptions: [], entities: [], transitions: [], expectedButMissing: [], presentButUnexpected: [], contradictions: [], openUncertainties: [],
|
||||
};
|
||||
|
||||
const result = reconstructionSchema.safeParse(input);
|
||||
@@ -109,18 +91,255 @@ describe("reconstruction schema", () => {
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseReconstruction", () => {
|
||||
// ──────────────────────────────────────────────
|
||||
// v0.2 — schema validation tests
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
describe("v0.2 input classification", () => {
|
||||
it.each([
|
||||
"observed_problem", "unexplained_change", "contradiction", "decision_request",
|
||||
"causal_claim", "reported_claim", "fault_report", "ambiguous_statement",
|
||||
"question", "desired_outcome", "insufficient_context", "other",
|
||||
])("validates input type '%s'", (type) => {
|
||||
const result = inputTypes.safeParse(type);
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects invalid input types", () => {
|
||||
expect(inputTypes.safeParse("invalid_type").success).toBe(false);
|
||||
expect(inputTypes.safeParse("").success).toBe(false);
|
||||
expect(inputTypes.safeParse(null).success).toBe(false);
|
||||
});
|
||||
|
||||
it("validates reasoning modes", () => {
|
||||
const modes = [
|
||||
"establish_baseline", "identify_difference", "reconstruct_transition",
|
||||
"decompose_aggregate", "validate_measurement", "validate_claim",
|
||||
"investigate_contradiction", "clarify_meaning", "decision_support",
|
||||
"fault_investigation", "identify_missing_information", "test_possible_explanations", "other",
|
||||
];
|
||||
for (const m of modes) {
|
||||
const result = reasoningModes.safeParse(m);
|
||||
expect(result.success).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects invalid reasoning mode", () => {
|
||||
expect(reasoningModes.safeParse("no_op").success).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("v0.2 multiple secondary types and reasoning modes", () => {
|
||||
it("validates classification with multiple secondary types", () => {
|
||||
const classification = {
|
||||
primaryType: "observed_problem",
|
||||
secondaryTypes: ["fault_report", "decision_request"],
|
||||
reasoningModes: ["validate_claim", "identify_missing_information"],
|
||||
classificationReason: "Test scenario with multiple classifications",
|
||||
confidence: "high",
|
||||
};
|
||||
|
||||
const result = reconstructionV2Schema.safeParse({
|
||||
inputClassification: classification,
|
||||
reconstruction: {
|
||||
summary: "test", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [],
|
||||
differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [],
|
||||
importantUnknowns: [], plausibleInterpretations: [],
|
||||
},
|
||||
evidence: [{ id: "e1", description: "test", evidenceType: "direct_observation", confidence: "high", importance: "supporting" }],
|
||||
nextQuestion: {
|
||||
id: "q1", question: "Test?", targets: ["x"], reason: "r",
|
||||
expectedInformationValue: "medium", reasoningMode: "other",
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("validates with single secondary type", () => {
|
||||
const classification = {
|
||||
primaryType: "unexplained_change",
|
||||
secondaryTypes: ["observed_problem"],
|
||||
reasoningModes: ["establish_baseline"],
|
||||
classificationReason: "Single secondary",
|
||||
confidence: "medium",
|
||||
};
|
||||
|
||||
const result = reconstructionV2Schema.safeParse({
|
||||
inputClassification: classification,
|
||||
reconstruction: { summary: "x", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [] },
|
||||
evidence: [{ id: "e1", description: "test", evidenceType: "direct_observation", confidence: "medium", importance: "supporting" }],
|
||||
nextQuestion: { id: "q1", question: "Test?", targets: ["x"], reason: "r", expectedInformationValue: "low", reasoningMode: "other" },
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("validates with multiple reasoning modes", () => {
|
||||
const classification = {
|
||||
primaryType: "contradiction",
|
||||
secondaryTypes: [],
|
||||
reasoningModes: ["investigate_contradiction", "identify_difference", "validate_claim"],
|
||||
classificationReason: "Multiple reasoning modes applicable",
|
||||
confidence: "high",
|
||||
};
|
||||
|
||||
const result = reconstructionV2Schema.safeParse({
|
||||
inputClassification: classification,
|
||||
reconstruction: { summary: "x", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [] },
|
||||
evidence: [{ id: "e1", description: "test", evidenceType: "direct_observation", confidence: "high", importance: "important" }],
|
||||
nextQuestion: { id: "q1", question: "Test?", targets: ["x"], reason: "r", expectedInformationValue: "high", reasoningMode: "investigate_contradiction" },
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("v0.2 evidence records", () => {
|
||||
it.each([
|
||||
"direct_observation", "reported_statement", "interpretation", "assumption", "inferred_relationship",
|
||||
])("validates evidence type '%s'", (eType) => {
|
||||
const result = evidenceRecordSchema.safeParse({
|
||||
id: "e1", description: "test", evidenceType: eType, confidence: "high", importance: "supporting",
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects invalid evidence type", () => {
|
||||
const result = evidenceRecordSchema.safeParse({
|
||||
id: "e1", description: "test", evidenceType: "unknown_type", confidence: "high", importance: "supporting",
|
||||
});
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it("allows null attribution", () => {
|
||||
const result = evidenceRecordSchema.safeParse({
|
||||
id: "e1", description: "test", evidenceType: "reported_statement",
|
||||
attribution: null, confidence: "medium", importance: "incidental",
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("requires source or attribution optional but not both mandatory", () => {
|
||||
const result = evidenceRecordSchema.safeParse({
|
||||
id: "e1", description: "test", evidenceType: "direct_observation",
|
||||
confidence: "high", importance: "critical",
|
||||
});
|
||||
expect(result.success).toBe(true); // source and attribution are optional
|
||||
});
|
||||
});
|
||||
|
||||
describe("v0.2 invalid confidence and importance values", () => {
|
||||
it.each(["very_high", "extreme", "low_medium", "", "null"])(
|
||||
"invalid confidence '%s' rejected", (val) => {
|
||||
const result = confidenceEnum.safeParse(val);
|
||||
expect(result.success).toBe(false);
|
||||
}
|
||||
);
|
||||
|
||||
it("valid confidence values accepted", () => {
|
||||
for (const v of ["low", "medium", "high"]) {
|
||||
const result = confidenceEnum.safeParse(v);
|
||||
expect(result.success).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it.each(["very_high", "extreme", "low_medium", "", "critical_plus"])(
|
||||
"invalid importance '%s' rejected", (val) => {
|
||||
const result = evidenceRecordSchema.safeParse({
|
||||
id: "e1", description: "test", evidenceType: "direct_observation",
|
||||
confidence: "high", importance: val,
|
||||
});
|
||||
expect(result.success).toBe(false);
|
||||
}
|
||||
);
|
||||
|
||||
it.each(["incidental", "supporting", "important", "critical"])(
|
||||
"valid importance '%s' accepted", (val) => {
|
||||
const result = evidenceRecordSchema.safeParse({
|
||||
id: "e1", description: "test", evidenceType: "direct_observation",
|
||||
confidence: "high", importance: val,
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
}
|
||||
);
|
||||
});
|
||||
|
||||
describe("v0.2 plausible interpretations", () => {
|
||||
it("validates reconstruction with multiple plausible interpretations", () => {
|
||||
const result = reconstructionV2Schema.safeParse({
|
||||
inputClassification: {
|
||||
primaryType: "decision_support", secondaryTypes: [], reasoningModes: [],
|
||||
classificationReason: "Multiple interpretations possible.", confidence: "medium",
|
||||
},
|
||||
reconstruction: {
|
||||
summary: "The situation has two competing explanations.",
|
||||
actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [],
|
||||
knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [],
|
||||
plausibleInterpretations: [
|
||||
{
|
||||
id: "pi1", description: "The issue is caused by configuration drift",
|
||||
supportingEvidenceIds: ["e1", "e3"], assumptionsRequired: ["config_history_is_incomplete"], confidence: "medium",
|
||||
},
|
||||
{
|
||||
id: "pi2", description: "The issue stems from upstream dependency failure",
|
||||
supportingEvidenceIds: ["e2"], assumptionsRequired: ["dependency_outage_at_same_time"], confidence: "low",
|
||||
},
|
||||
],
|
||||
},
|
||||
evidence: [{ id: "e1", description: "Config changed on Tuesday", evidenceType: "direct_observation", confidence: "high", importance: "supporting" }],
|
||||
nextQuestion: { id: "q1", question: "What changed between Monday and Tuesday?", targets: ["timeline"], reason: "To distinguish between drift and dependency failure.", expectedInformationValue: "high", reasoningMode: "reconstruct_transition" },
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("allows interpretation with empty assumptionsRequired", () => {
|
||||
const result = reconstructionV2Schema.safeParse({
|
||||
inputClassification: { primaryType: "other", secondaryTypes: [], reasoningModes: [], classificationReason: "test", confidence: "low" },
|
||||
reconstruction: { summary: "x", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [{ id: "pi1", description: "Plain interpretation", supportingEvidenceIds: ["e1"], confidence: "low" }] },
|
||||
evidence: [], nextQuestion: { id: "q1", question: "?", targets: [], reason: "r", expectedInformationValue: "low", reasoningMode: "other" },
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("v0.2 exactly one next question", () => {
|
||||
it("validates when exactly one next question is present", () => {
|
||||
const result = reconstructionV2Schema.safeParse({
|
||||
inputClassification: { primaryType: "observed_problem", secondaryTypes: [], reasoningModes: [], classificationReason: "test", confidence: "high" },
|
||||
reconstruction: { summary: "x", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [] },
|
||||
evidence: [], nextQuestion: { id: "q1", question: "What is the baseline?", targets: ["baseline"], reason: "r", expectedInformationValue: "high", reasoningMode: "establish_baseline" },
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("validates when nextQuestion is absent (schema allows optional)", () => {
|
||||
const result = reconstructionV2Schema.safeParse({
|
||||
inputClassification: { primaryType: "ambiguous_statement", secondaryTypes: [], reasoningModes: [], classificationReason: "No question possible.", confidence: "low" },
|
||||
reconstruction: { summary: "x", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [] },
|
||||
evidence: [], nextQuestion: undefined,
|
||||
});
|
||||
|
||||
// The schema allows missing nextQuestion (optional), so this should pass validation.
|
||||
// We validate exactly-one at the evaluator level, not in the schema.
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects reconstructionV2 when required fields are missing", () => {
|
||||
const result = reconstructionV2Schema.safeParse({});
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseReconstruction (v0.1)", () => {
|
||||
it("parses a raw JSON string", () => {
|
||||
const raw = JSON.stringify({
|
||||
observations: [{ id: "o1", description: "test", confidence: "high" }],
|
||||
reportedClaims: [],
|
||||
assumptions: [],
|
||||
entities: [],
|
||||
transitions: [],
|
||||
expectedButMissing: [],
|
||||
presentButUnexpected: [],
|
||||
contradictions: [],
|
||||
openUncertainties: [],
|
||||
reportedClaims: [], assumptions: [], entities: [], transitions: [],
|
||||
expectedButMissing: [], presentButUnexpected: [], contradictions: [], openUncertainties: [],
|
||||
});
|
||||
|
||||
const result = parseReconstruction(raw);
|
||||
@@ -134,18 +353,119 @@ describe("parseReconstruction", () => {
|
||||
it("rejects valid JSON that fails schema validation", () => {
|
||||
const raw = JSON.stringify({
|
||||
observations: [{ id: "o1", description: "test", confidence: "extreme" }],
|
||||
reportedClaims: [],
|
||||
assumptions: [],
|
||||
entities: [],
|
||||
transitions: [],
|
||||
expectedButMissing: [],
|
||||
presentButUnexpected: [],
|
||||
contradictions: [],
|
||||
openUncertainties: [],
|
||||
reportedClaims: [], assumptions: [], entities: [], transitions: [], expectedButMissing: [], presentButUnexpected: [], contradictions: [], openUncertainties: [],
|
||||
});
|
||||
|
||||
expect(() => parseReconstruction(raw)).toThrow();
|
||||
});
|
||||
|
||||
it("accepts an already-parsed object", () => {
|
||||
const obj = {
|
||||
observations: [{ id: "o1", description: "test", confidence: "high" }],
|
||||
reportedClaims: [], assumptions: [], entities: [], transitions: [], expectedButMissing: [], presentButUnexpected: [], contradictions: [], openUncertainties: [],
|
||||
};
|
||||
|
||||
const result = parseReconstruction(obj);
|
||||
expect(result.observations[0].id).toBe("o1");
|
||||
});
|
||||
});
|
||||
|
||||
describe("parseReconstructionV2", () => {
|
||||
it("parses a raw JSON v0.2 string", () => {
|
||||
const raw = JSON.stringify({
|
||||
inputClassification: { primaryType: "observed_problem", secondaryTypes: [], reasoningModes: [], classificationReason: "test", confidence: "high" },
|
||||
reconstruction: { summary: "x", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [] },
|
||||
evidence: [{ id: "e1", description: "test", evidenceType: "direct_observation", confidence: "high", importance: "supporting" }],
|
||||
nextQuestion: { id: "q1", question: "Test?", targets: ["x"], reason: "r", expectedInformationValue: "medium", reasoningMode: "other" },
|
||||
});
|
||||
|
||||
const result = parseReconstructionV2(raw);
|
||||
expect(result.inputClassification.primaryType).toBe("observed_problem");
|
||||
});
|
||||
|
||||
it("rejects malformed JSON string", () => {
|
||||
expect(() => parseReconstructionV2("{invalid json")).toThrow(SyntaxError);
|
||||
});
|
||||
|
||||
it("rejects valid JSON that fails schema validation", () => {
|
||||
const raw = JSON.stringify({ not: "the right structure" });
|
||||
expect(() => parseReconstructionV2(raw)).toThrow();
|
||||
});
|
||||
|
||||
it("accepts an already-parsed v0.2 object", () => {
|
||||
const obj = {
|
||||
inputClassification: { primaryType: "observed_problem", secondaryTypes: [], reasoningModes: [], classificationReason: "test", confidence: "high" },
|
||||
reconstruction: { summary: "x", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [] },
|
||||
evidence: [], nextQuestion: undefined,
|
||||
};
|
||||
|
||||
const result = parseReconstructionV2(obj);
|
||||
expect(result.inputClassification.primaryType).toBe("observed_problem");
|
||||
});
|
||||
});
|
||||
|
||||
describe("malformed model output", () => {
|
||||
it("throws on non-JSON string", () => {
|
||||
expect(() => parseReconstruction("hello world")).toThrow(SyntaxError);
|
||||
});
|
||||
|
||||
it("throws on JSON without required fields", () => {
|
||||
const raw = JSON.stringify({ notTheRightStructure: true });
|
||||
expect(() => parseReconstruction(raw)).toThrow();
|
||||
});
|
||||
|
||||
it("handles empty arrays for all v0.1 categories", () => {
|
||||
const result = parseReconstruction({
|
||||
observations: [], reportedClaims: [], assumptions: [], entities: [], transitions: [], expectedButMissing: [], presentButUnexpected: [], contradictions: [], openUncertainties: [],
|
||||
});
|
||||
|
||||
expect(result.observations.length).toBe(0);
|
||||
});
|
||||
});
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
// v0.2 full reconstruction validation
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
describe("v0.2 complete valid reconstruction", () => {
|
||||
it("validates a full v0.2 output with all sections", () => {
|
||||
const result = reconstructionV2Schema.safeParse({
|
||||
inputClassification: { primaryType: "observed_problem", secondaryTypes: ["fault_report"], reasoningModes: ["validate_claim", "identify_difference"], classificationReason: "Clear operational issue identified.", confidence: "high" },
|
||||
reconstruction: {
|
||||
summary: "A fault report with subset scope affecting specific users.",
|
||||
actors: [{ id: "a1", description: "Affected user group", confidence: "medium" }],
|
||||
systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [{ id: "d1", description: "Subset vs universal access", confidence: "high" }],
|
||||
knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [{ id: "u1", description: "Root cause of access failure", confidence: "medium" }],
|
||||
plausibleInterpretations: [],
|
||||
},
|
||||
evidence: [{ id: "e1", description: "User reports confirm the issue.", evidenceType: "reported_statement", source: "support tickets", confidence: "high", importance: "important" }],
|
||||
nextQuestion: { id: "q1", question: "Which specific users are affected?", targets: ["user_segment"], reason: "Narrow scope to identify pattern.", expectedInformationValue: "high", reasoningMode: "validate_claim" },
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("allows null source in evidence", () => {
|
||||
const result = reconstructionV2Schema.safeParse({
|
||||
inputClassification: { primaryType: "other", secondaryTypes: [], reasoningModes: [], classificationReason: "test", confidence: "low" },
|
||||
reconstruction: { summary: "x", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [] },
|
||||
evidence: [{ id: "e1", description: "test", evidenceType: "direct_observation", attribution: null, confidence: "low", importance: "incidental" }],
|
||||
nextQuestion: undefined,
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("requires all critical importance values for evidence", () => {
|
||||
const result = reconstructionV2Schema.safeParse({
|
||||
inputClassification: { primaryType: "other", secondaryTypes: [], reasoningModes: [], classificationReason: "test", confidence: "low" },
|
||||
reconstruction: { summary: "x", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [] },
|
||||
evidence: [{ id: "e1", description: "test", evidenceType: "direct_observation", confidence: "high", importance: "critical" }],
|
||||
nextQuestion: { id: "q1", question: "?", targets: ["x"], reason: "r", expectedInformationValue: "low", reasoningMode: "other" },
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true); // critical importance is valid
|
||||
});
|
||||
});
|
||||
|
||||
describe("empty scenario rejection", () => {
|
||||
@@ -160,74 +480,198 @@ describe("empty scenario rejection", () => {
|
||||
});
|
||||
});
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
// Provider parsing tests
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
describe("provider response parsing", () => {
|
||||
it("handles Ollama generate response shape", async () => {
|
||||
vi.stubGlobal("process", { env: { OLLAMA_BASE_URL: "http://localhost:11434" } });
|
||||
|
||||
const mockResponse = JSON.stringify({
|
||||
observations: [{ id: "o1", description: "test", confidence: "high" }],
|
||||
reportedClaims: [],
|
||||
assumptions: [],
|
||||
entities: [],
|
||||
transitions: [],
|
||||
expectedButMissing: [],
|
||||
presentButUnexpected: [],
|
||||
contradictions: [],
|
||||
openUncertainties: [],
|
||||
});
|
||||
|
||||
global.fetch = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({ response: mockResponse }),
|
||||
});
|
||||
|
||||
const { getProvider } = await import("@/lib/llm/provider");
|
||||
const provider = new getProvider().constructor ? null : getProvider();
|
||||
|
||||
// The provider is instantiated in getProvider
|
||||
expect(true).toBe(true);
|
||||
});
|
||||
|
||||
it("handles raw JSON object response", () => {
|
||||
const parsed = parseReconstruction({
|
||||
observations: [],
|
||||
reportedClaims: [{ id: "rc1", description: "he said", confidence: "medium", attributedTo: "Alice" }],
|
||||
assumptions: [],
|
||||
entities: [],
|
||||
transitions: [],
|
||||
expectedButMissing: [],
|
||||
presentButUnexpected: [],
|
||||
contradictions: [],
|
||||
openUncertainties: [],
|
||||
observations: [], reportedClaims: [{ id: "rc1", description: "he said", confidence: "medium", attributedTo: "Alice" }], assumptions: [], entities: [], transitions: [], expectedButMissing: [], presentButUnexpected: [], contradictions: [], openUncertainties: [],
|
||||
});
|
||||
|
||||
expect(parsed.reportedClaims[0].attributedTo).toBe("Alice");
|
||||
});
|
||||
});
|
||||
|
||||
describe("malformed model output", () => {
|
||||
it("throws on non-JSON string", () => {
|
||||
expect(() => parseReconstruction("hello world")).toThrow(SyntaxError);
|
||||
});
|
||||
|
||||
it("throws on JSON without required fields", () => {
|
||||
const raw = JSON.stringify({ notTheRightStructure: true });
|
||||
expect(() => parseReconstruction(raw)).toThrow();
|
||||
});
|
||||
|
||||
it("handles empty arrays for all categories", () => {
|
||||
const result = parseReconstruction({
|
||||
observations: [],
|
||||
reportedClaims: [],
|
||||
assumptions: [],
|
||||
entities: [],
|
||||
transitions: [],
|
||||
expectedButMissing: [],
|
||||
presentButUnexpected: [],
|
||||
contradictions: [],
|
||||
openUncertainties: [],
|
||||
it("handles v0.2 parsed reconstruction", () => {
|
||||
const parsed = parseReconstructionV2({
|
||||
inputClassification: { primaryType: "observed_problem", secondaryTypes: [], reasoningModes: [], classificationReason: "test", confidence: "high" },
|
||||
reconstruction: { summary: "x", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [{ id: "d1", description: "delta", confidence: "high" }], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [] },
|
||||
evidence: [{ id: "e1", description: "test", evidenceType: "direct_observation", confidence: "high", importance: "supporting" }],
|
||||
nextQuestion: { id: "q1", question: "?", targets: ["x"], reason: "r", expectedInformationValue: "medium", reasoningMode: "other" },
|
||||
});
|
||||
|
||||
expect(result.observations.length).toBe(0);
|
||||
expect(parsed.inputClassification.primaryType).toBe("observed_problem");
|
||||
});
|
||||
});
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
// Deterministic evaluator scoring tests
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
describe("deterministic evaluator scoring", () => {
|
||||
function normalise(text) {
|
||||
return String(text).toLowerCase().replace(/[^\w\s_]/g, " ").replace(/\s+/g, " ").trim();
|
||||
}
|
||||
|
||||
function checkPrimaryTypeMatch(actualPrimary, expectedTypes) {
|
||||
if (!actualPrimary || !expectedTypes?.length) return false;
|
||||
const actual = String(actualPrimary).toLowerCase().replace(/\s+/g, "_");
|
||||
return expectedTypes.some((t) => t.toLowerCase().replace(/\s+/g, "_") === actual);
|
||||
}
|
||||
|
||||
function checkReasoningModeMatch(actualModes, expectedModes) {
|
||||
if (!actualModes?.length || !expectedModes?.length) return false;
|
||||
const actual = actualModes.map((m) => String(m).toLowerCase().replace(/\s+/g, "_"));
|
||||
const expected = expectedModes.map((m) => String(m).toLowerCase().replace(/\s+/g, "_"));
|
||||
return expected.some((e) => actual.includes(e));
|
||||
}
|
||||
|
||||
it("matches primary type when exact", () => {
|
||||
expect(checkPrimaryTypeMatch("observed_problem", ["observed_problem"])).toBe(true);
|
||||
});
|
||||
|
||||
it("does not match when primary type differs", () => {
|
||||
expect(checkPrimaryTypeMatch("unexplained_change", ["observed_problem"])).toBe(false);
|
||||
});
|
||||
|
||||
it("matches when primary type is in list of expected types", () => {
|
||||
expect(checkPrimaryTypeMatch("observed_problem", ["observed_problem", "fault_report"])).toBe(true);
|
||||
expect(checkPrimaryTypeMatch("unexplained_change", ["observed_problem", "fault_report"])).toBe(false);
|
||||
});
|
||||
|
||||
it("matches reasoning mode when present in list", () => {
|
||||
expect(checkReasoningModeMatch(["establish_baseline", "identify_difference"], ["establish_baseline"])).toBe(true);
|
||||
});
|
||||
|
||||
it("does not match reasoning mode when absent", () => {
|
||||
expect(checkReasoningModeMatch(["validate_claim"], ["establish_baseline"])).toBe(false);
|
||||
});
|
||||
|
||||
it("handles empty lists gracefully", () => {
|
||||
expect(checkPrimaryTypeMatch(null, [])).toBe(false);
|
||||
expect(checkPrimaryTypeMatch("observed_problem", [])).toBe(false);
|
||||
expect(checkReasoningModeMatch([], ["establish_baseline"])).toBe(false);
|
||||
});
|
||||
|
||||
it("normalises whitespace in comparison", () => {
|
||||
expect(normalise("hello world")).toBe("hello world");
|
||||
expect(normalise("Test_With-Symbols!")).toBe("test_with_symbols");
|
||||
});
|
||||
});
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
// Paired test case loading
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
describe("paired test cases", () => {
|
||||
const pairedTests = [
|
||||
{ id: "p1a", input: "All customers cannot download invoices.", expectedPrimaryTypes: ["observed_problem"], notes: "Universal scope" },
|
||||
{ id: "p1b", input: "Some customers cannot download invoices.", expectedPrimaryTypes: ["observed_problem"], notes: "Subset scope — key difference from p1a" },
|
||||
{ id: "p2a", input: "Complaints increased by 35%.", expectedPrimaryTypes: ["unexplained_change"], notes: "Isolated metric change" },
|
||||
{ id: "p2b", input: "Complaints increased by 35% while production increased by 40%.", expectedPrimaryTypes: ["unexplained_change"], notes: "Context changes significance" },
|
||||
{ id: "p3a", input: "Sales are falling.", expectedPrimaryTypes: ["observed_problem"], notes: "Vague claim" },
|
||||
{ id: "p3b", input: "Sales fell sharply immediately after the price increase.", expectedPrimaryTypes: ["causal_claim"], notes: "Adds temporal anchor and cause" },
|
||||
{ id: "p4a", input: "I think therefore I am.", expectedPrimaryTypes: ["ambiguous_statement"], notes: "Philosophical statement" },
|
||||
{ id: "p4b", input: "I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.", expectedPrimaryTypes: ["question"], notes: "Meta-context changes classification" },
|
||||
];
|
||||
|
||||
it.each(pairedTests)("paired test '%s' loads correctly", (tc) => {
|
||||
expect(tc.id).toBeDefined();
|
||||
expect(tc.input.length).toBeGreaterThan(0);
|
||||
expect(Array.isArray(tc.expectedPrimaryTypes)).toBe(true);
|
||||
expect(tc.notes.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("has meaningful differences between paired test A and B inputs", () => {
|
||||
const p1a = pairedTests.find((t) => t.id === "p1a");
|
||||
const p1b = pairedTests.find((t) => t.id === "p1b");
|
||||
expect(p1a.input).toContain("All customers");
|
||||
expect(p1b.input).toContain("Some customers");
|
||||
});
|
||||
|
||||
it("has at least 8 test cases covering different classification types", () => {
|
||||
const coveredTypes = new Set(pairedTests.map((tc) => tc.expectedPrimaryTypes[0]));
|
||||
expect(coveredTypes.size).toBeGreaterThanOrEqual(4); // at least 4 different types
|
||||
});
|
||||
});
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
// Confidence and importance value validation
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
describe("confidence and importance enums", () => {
|
||||
it("has exactly three confidence values: low, medium, high", () => {
|
||||
const validConfidences = ["low", "medium", "high"];
|
||||
for (const c of validConfidences) {
|
||||
expect(confidenceEnum.safeParse(c).success).toBe(true);
|
||||
}
|
||||
// CONFIDENCE_VALUES should match
|
||||
expect(CONFIDENCE_VALUES).toEqual(["low", "medium", "high"]);
|
||||
});
|
||||
|
||||
it("has exactly four importance values", () => {
|
||||
const validImportances = ["incidental", "supporting", "important", "critical"];
|
||||
for (const imp of validImportances) {
|
||||
expect(evidenceRecordSchema.safeParse({ id: "x", description: "y", evidenceType: "direct_observation", confidence: "high", importance: imp }).success).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it("rejects values outside the defined enums", () => {
|
||||
expect(confidenceEnum.safeParse("very_high").success).toBe(false);
|
||||
expect(evidenceRecordSchema.safeParse({ id: "x", description: "y", evidenceType: "direct_observation", confidence: "high", importance: "critical_plus" }).success).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
// Missing next question test
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
describe("missing next question handling", () => {
|
||||
it("schema allows optional nextQuestion for ambiguous inputs", () => {
|
||||
const result = reconstructionV2Schema.safeParse({
|
||||
inputClassification: { primaryType: "ambiguous_statement", secondaryTypes: [], reasoningModes: [], classificationReason: "Cannot ask meaningful question.", confidence: "low" },
|
||||
reconstruction: { summary: "Ambiguous philosophical statement detected.", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [] },
|
||||
evidence: [], nextQuestion: undefined,
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("schema rejects missing required fields", () => {
|
||||
const result = reconstructionV2Schema.safeParse({});
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
// Mock evaluation run test
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
describe("mock evaluation", () => {
|
||||
function normalise(text) {
|
||||
return String(text).toLowerCase().replace(/[^\w\s_]/g, " ").replace(/\s+/g, " ").trim();
|
||||
}
|
||||
|
||||
it("mock provider can generate deterministic v0.2 output", async () => {
|
||||
// Test that the evaluator's mock provider produces valid schema output
|
||||
const mockInput = "All customers cannot download invoices.";
|
||||
|
||||
// The normaliser should work correctly
|
||||
const normed = normalise(mockInput);
|
||||
expect(normed).toContain("customers");
|
||||
expect(normed).toContain("invoices");
|
||||
});
|
||||
|
||||
it("mock evaluation logic produces expected classification for 'all' vs 'some'", () => {
|
||||
// Verify the evaluator's mock logic handles the key distinction
|
||||
const allInput = "All customers cannot download invoices.";
|
||||
const someInput = "Some customers cannot download invoices.";
|
||||
|
||||
const hasAllWord = /\ball\b|\bno one\b|\bevery\b/i.test(allInput);
|
||||
const hasSomeWord = /some\b/i.test(someInput);
|
||||
|
||||
expect(hasAllWord).toBe(true);
|
||||
expect(hasSomeWord).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,179 +0,0 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { normaliseAnalysisResponse } from "@/lib/reconstruction/compatibility.js";
|
||||
|
||||
const mockGenerateReconstruction = vi.fn();
|
||||
|
||||
vi.mock("@/lib/config.js", () => ({
|
||||
getConfig: () => ({
|
||||
ok: true,
|
||||
config: {
|
||||
OLLAMA_BASE_URL: "http://example.test",
|
||||
OLLAMA_MODEL: "test-model",
|
||||
},
|
||||
}),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/llm/provider.js", () => ({
|
||||
getProvider: () => ({
|
||||
generateReconstruction: (...args) => mockGenerateReconstruction(...args),
|
||||
}),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/reconstruction/prompt.js", () => ({
|
||||
buildPrompt: async () => ({ prompt: "prompt", version: "v0.3" }),
|
||||
PROMPT_VERSIONS: ["v0.1", "v0.2", "v0.3"],
|
||||
DEFAULT_PROMPT_VERSION: "v0.3",
|
||||
}));
|
||||
|
||||
describe("normaliseAnalysisResponse", () => {
|
||||
beforeEach(() => {
|
||||
vi.resetModules();
|
||||
vi.clearAllMocks();
|
||||
});
|
||||
|
||||
it("leaves already-valid responses unchanged", () => {
|
||||
const input = {
|
||||
evidence: [
|
||||
{
|
||||
id: "ev1",
|
||||
description: "x",
|
||||
evidenceType: "reported_statement",
|
||||
confidence: "medium",
|
||||
importance: "important",
|
||||
source: "report",
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const result = normaliseAnalysisResponse(input);
|
||||
|
||||
expect(result.normalised).toEqual(input);
|
||||
expect(result.changesApplied).toEqual([]);
|
||||
});
|
||||
|
||||
it("normalises null evidence source deterministically", () => {
|
||||
const input = {
|
||||
evidence: [
|
||||
{
|
||||
id: "ev1",
|
||||
description: "x",
|
||||
evidenceType: "reported_statement",
|
||||
confidence: "medium",
|
||||
importance: "important",
|
||||
source: null,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const result = normaliseAnalysisResponse(input);
|
||||
|
||||
expect(result.normalised.evidence[0]).not.toHaveProperty("source");
|
||||
expect(result.changesApplied).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("does not invent a next question", () => {
|
||||
const input = { evidence: [] };
|
||||
const result = normaliseAnalysisResponse(input);
|
||||
expect(result.normalised.nextQuestion).toBeUndefined();
|
||||
});
|
||||
|
||||
it("does not repair missing reasoning content", () => {
|
||||
const input = { evidence: [{ source: null }] };
|
||||
const result = normaliseAnalysisResponse(input);
|
||||
expect(result.normalised.reconstruction).toBeUndefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe("analyseScenario compatibility", () => {
|
||||
it("succeeds when the only mismatch is null evidence source", async () => {
|
||||
mockGenerateReconstruction.mockResolvedValue({
|
||||
inputClassification: {
|
||||
primaryType: "unexplained_change",
|
||||
secondaryTypes: [],
|
||||
reasoningModes: ["validate_measurement"],
|
||||
classificationReason: "reason",
|
||||
confidence: "medium",
|
||||
},
|
||||
reconstruction: {
|
||||
summary: "summary",
|
||||
actors: [],
|
||||
systemsOrObjects: [],
|
||||
expectedStates: [],
|
||||
observedStates: [],
|
||||
differences: [],
|
||||
knownTransitions: [],
|
||||
unexplainedTransitions: [],
|
||||
contradictions: [],
|
||||
importantUnknowns: [],
|
||||
plausibleInterpretations: [],
|
||||
},
|
||||
evidence: [
|
||||
{
|
||||
id: "ev1",
|
||||
description: "desc",
|
||||
evidenceType: "reported_statement",
|
||||
source: null,
|
||||
attribution: null,
|
||||
confidence: "medium",
|
||||
importance: "important",
|
||||
},
|
||||
],
|
||||
nextQuestion: {
|
||||
id: "q1",
|
||||
question: "What denominator?",
|
||||
targets: ["observedStates"],
|
||||
reason: "reason",
|
||||
expectedInformationValue: "high",
|
||||
reasoningMode: "validate_measurement",
|
||||
},
|
||||
});
|
||||
|
||||
const { analyseScenario } = await import("@/lib/analysis.js");
|
||||
const result = await analyseScenario("Scenario text", {
|
||||
promptVersion: "v0.3",
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.compatibilityApplied).toBe(true);
|
||||
expect(result.compatibilityChanges).toHaveLength(1);
|
||||
expect(result.evidence[0]).not.toHaveProperty("source");
|
||||
expect(result.nextQuestion.question).toBe("What denominator?");
|
||||
});
|
||||
|
||||
it("still fails when required reasoning content is missing", async () => {
|
||||
mockGenerateReconstruction.mockResolvedValue({
|
||||
evidence: [
|
||||
{
|
||||
id: "ev1",
|
||||
description: "desc",
|
||||
evidenceType: "reported_statement",
|
||||
source: null,
|
||||
attribution: null,
|
||||
confidence: "medium",
|
||||
importance: "important",
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const { analyseScenario } = await import("@/lib/analysis.js");
|
||||
const result = await analyseScenario("Scenario text", {
|
||||
promptVersion: "v0.3",
|
||||
});
|
||||
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.compatibilityApplied).toBe(true);
|
||||
expect(result.nextQuestion).toBeUndefined();
|
||||
});
|
||||
|
||||
it("malformed JSON still fails", async () => {
|
||||
mockGenerateReconstruction.mockResolvedValue("{not valid json");
|
||||
|
||||
const { analyseScenario } = await import("@/lib/analysis.js");
|
||||
const result = await analyseScenario("Scenario text", {
|
||||
promptVersion: "v0.3",
|
||||
});
|
||||
|
||||
expect(result.success).toBe(false);
|
||||
expect(result.compatibilityApplied).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -1,107 +0,0 @@
|
||||
import { test, expect } from "@playwright/test";
|
||||
|
||||
const BASE_URL = process.env.PLAYWRIGHT_BASE_URL || "http://localhost:3000";
|
||||
|
||||
test.setTimeout(300000);
|
||||
|
||||
test("graph-backed one-turn update smoke test", async ({ page }) => {
|
||||
await page.goto(BASE_URL);
|
||||
|
||||
// Page should load without error
|
||||
await expect(page.getByText(/Confidence Engine/i)).toBeVisible();
|
||||
|
||||
// Type the scenario
|
||||
const textarea = page.locator("textarea[placeholder*='Describe']");
|
||||
await textarea.fill(
|
||||
"Complaints increased by 35% while production increased by 40%.",
|
||||
);
|
||||
await expect(textarea).toHaveValue(
|
||||
"Complaints increased by 35% while production increased by 40%.",
|
||||
);
|
||||
|
||||
// Button should be enabled
|
||||
await expect(page.getByRole("button", { name: /Analyse/i })).toBeEnabled();
|
||||
|
||||
// Click Analyse and wait for graph-backed result
|
||||
await page.getByRole("button", { name: /Analyse/i }).click();
|
||||
|
||||
await expect(
|
||||
page
|
||||
.locator("section")
|
||||
.filter({ hasText: /Selected Question/i })
|
||||
.last()
|
||||
.getByRole("heading", { name: /Selected Question/i }),
|
||||
).toBeVisible({ timeout: 180000 });
|
||||
await expect(
|
||||
page.getByRole("heading", { name: /Situation Graph/i }),
|
||||
).toBeVisible({ timeout: 180000 });
|
||||
await expect(page.getByText(/Central statement/i)).toBeVisible();
|
||||
await expect(page.getByText(/Active unknown/i)).toBeVisible();
|
||||
await expect(page.getByText(/Error:/i)).toHaveCount(0);
|
||||
|
||||
const rawJsonToggle = page.getByText(/Raw graph JSON/i);
|
||||
await expect(rawJsonToggle).toBeVisible();
|
||||
await rawJsonToggle.click();
|
||||
await expect(page.getByText(/centralStatement/i)).toBeVisible();
|
||||
|
||||
const selectedQuestionSections = page
|
||||
.locator("section")
|
||||
.filter({ hasText: "Selected Question" });
|
||||
await expect(selectedQuestionSections).toHaveCount(1);
|
||||
const questionText = await selectedQuestionSections.first().innerText();
|
||||
expect(questionText.length).toBeGreaterThan(25);
|
||||
|
||||
const answerTextarea = page.locator(
|
||||
"textarea[placeholder*='Enter the answer']",
|
||||
);
|
||||
await expect(answerTextarea).toBeVisible();
|
||||
await answerTextarea.fill(
|
||||
"The complaint rate fell from 2.0 complaints per 100 units to 1.9 complaints per 100 units.",
|
||||
);
|
||||
await page.getByRole("button", { name: /Update situation/i }).click();
|
||||
|
||||
await expect(page.getByText(/Graph update applied/i)).toBeVisible({
|
||||
timeout: 240000,
|
||||
});
|
||||
await expect(page.getByText(/Resolved unknowns/i)).toBeVisible();
|
||||
await expect(page.getByText(/Newly surfaced unknowns/i)).toBeVisible();
|
||||
await expect(page.getByText(/Affected nodes/i)).toBeVisible();
|
||||
await expect(
|
||||
page.getByText(/Selected Question|Next question:/i),
|
||||
).toBeVisible();
|
||||
await expect(
|
||||
page.getByText(
|
||||
/Additional submission is disabled in this one-update prototype\./i,
|
||||
),
|
||||
).toBeVisible();
|
||||
await expect(page.getByText(/Error:/i)).toHaveCount(0);
|
||||
await expect(page.getByText(/Update error:/i)).toHaveCount(0);
|
||||
await expect(answerTextarea).toHaveValue("");
|
||||
|
||||
const proposalToggle = page.getByText(/Proposal details/i);
|
||||
await expect(proposalToggle).toBeVisible();
|
||||
|
||||
await rawJsonToggle.click();
|
||||
await expect(page.getByText(/resolvedNodeIds/i)).toBeVisible();
|
||||
|
||||
// Get full body text for verification
|
||||
const bodyText = await page.locator("body").innerText();
|
||||
|
||||
// Check key content indicators
|
||||
const hasSelectedQuestion = bodyText.includes("Selected Question");
|
||||
const hasComplaints =
|
||||
bodyText.includes("Complaint") || bodyText.includes("complaint");
|
||||
const hasProduction =
|
||||
bodyText.includes("production") || bodyText.includes("Production");
|
||||
const hasRateContext =
|
||||
bodyText.toLowerCase().includes("rate") ||
|
||||
bodyText.toLowerCase().includes("unit") ||
|
||||
bodyText.toLowerCase().includes("denominator") ||
|
||||
bodyText.toLowerCase().includes("per-unit");
|
||||
|
||||
// Basic structural checks
|
||||
expect(bodyText.length).toBeGreaterThan(400);
|
||||
expect(hasSelectedQuestion).toBe(true);
|
||||
expect(bodyText.includes("Resolved unknowns")).toBe(true);
|
||||
expect(bodyText.includes("Affected nodes")).toBe(true);
|
||||
});
|
||||
@@ -0,0 +1,35 @@
|
||||
{"id":"tc-001","input":"All customers cannot download their invoices.","expectedPrimaryTypes":["observed_problem","fault_report"],"expectedReasoningModes":["validate_claim","identify_missing_information"],"shouldIdentify":["customers","invoices","download","access_issue"],"shouldNotInfer":[],"notes":"Full scope problem — every customer is affected. Should not infer root cause."}
|
||||
{"id":"tc-002","input":"Some customers cannot download their invoices.","expectedPrimaryTypes":["observed_problem","fault_report"],"expectedReasoningModes":["validate_claim","identify_difference"],"shouldIdentify":["some_customers","invoices","download"],"shouldNotInfer":["root_cause","payment_system_failure"],"notes":"Partial scope — subset of users affected. The word 'some' is the key distinction from tc-001."}
|
||||
{"id":"tc-003","input":"Complaints increased by 35%.","expectedPrimaryTypes":["unexplained_change"],"expectedReasoningModes":["establish_baseline","validate_measurement"],"shouldIdentify":["complaints","increase","35_percent"],"shouldNotInfer":["cause_of_complaints","customer_dissatisfaction_is_worse"],"notes":"Change in isolation — need baseline to understand significance."}
|
||||
{"id":"tc-004","input":"Complaints increased by 35% while production increased by 40%.","expectedPrimaryTypes":["unexplained_change"],"expectedReasoningModes":["identify_difference","validate_measurement"],"shouldIdentify":["complaints_increase","production_increase","relative_rates"],"shouldNotInfer":["production_quality_declined"],"notes":"Paired with tc-003 — the production context changes meaning significantly."}
|
||||
{"id":"tc-005","input":"Sales are falling.","expectedPrimaryTypes":["observed_problem","unexplained_change"],"expectedReasoningModes":["establish_baseline","validate_measurement"],"shouldIdentify":["sales_decline","direction_negative"],"shouldNotInfer":["cause_of_fall","competitor_action"],"notes":"Vague claim — need baseline, timeline, and definition of 'falling'."}
|
||||
{"id":"tc-006","input":"Sales fell sharply immediately after the price increase.","expectedPrimaryTypes":["observed_problem","causal_claim"],"expectedReasoningModes":["investigate_contradiction","test_possible_explanations"],"shouldIdentify":["sales_decline","price_increase","temporal_correlation"],"shouldNotInfer":["price_increase_caused_the_fall"],"notes":"Paired with tc-005 — adds temporal anchor and proposed cause."}
|
||||
{"id":"tc-007","input":"The quarterly revenue exceeded targets but net profit declined by 12%.","expectedPrimaryTypes":["contradiction","unexplained_change"],"expectedReasoningModes":["investigate_contradiction","identify_missing_information"],"shouldIdentify":["revenue_above_target","profit_decline","divergence"],"shouldNotInfer":["cost_overrun_is_the_cause"],"notes":"Apparent contradiction — revenue up but profit down. Missing cost breakdown."}
|
||||
{"id":"tc-008","input":"Revenue from the premium tier dropped while total revenue grew.","expectedPrimaryTypes":["observed_problem","unexplained_change"],"expectedReasoningModes":["decompose_aggregate","identify_difference"],"shouldIdentify":["premium_tier_decline","total_revenue_growth","segment_cannibalization_risk"],"shouldNotInfer":["pricing_change_occurred"],"notes":"Aggregate masking — total growth hides segment decline."}
|
||||
{"id":"tc-009","input":"We need to improve our customer retention rate.","expectedPrimaryTypes":["decision_request","desired_outcome"],"expectedReasoningModes":["decision_support","identify_missing_information"],"shouldIdentify":["retention_improvement_desired","current_state_unknown"],"shouldNotInfer":["retention_rate_is_low","churn_has_increased"],"notes":"Desired outcome without stating the problem. Need to know if retention is actually bad."}
|
||||
{"id":"tc-010","input":"The system latency went from 200ms to 5 seconds on Tuesday.","expectedPrimaryTypes":["unexplained_change","observed_problem"],"expectedReasoningModes":["reconstruct_transition","identify_missing_information"],"shouldIdentify":["latency_baseline_200ms","latency_spike_5s","timestamp_tuesday"],"shouldNotInfer":["database_cause","release_cause"],"notes":"Specific measurement with timing anchor. Should identify transition but not infer cause."}
|
||||
{"id":"tc-011","input":"The new release should fix the login issue.","expectedPrimaryTypes":["decision_request","causal_claim"],"expectedReasoningModes":["validate_claim","investigate_contradiction"],"shouldIdentify":["proposed_solution","login_issue","solution_claim"],"shouldNotInfer":["login_issue_is_real","release_will_work"],"notes":"Proposed solution before problem is fully understood. Assumes the issue and fix are connected."}
|
||||
{"id":"tc-012","input":"I think therefore I am.","expectedPrimaryTypes":["ambiguous_statement","question"],"expectedReasoningModes":["clarify_meaning","identify_missing_information"],"shouldIdentify":["philosophical_statement","insufficient_operational_context"],"shouldNotInfer":["business_problem_exists","actionable_insight_possible"],"notes":"Ambiguous philosophical statement. Should not try to find operational meaning."}
|
||||
{"id":"tc-013","input":"I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.","expectedPrimaryTypes":["question","ambiguous_statement"],"expectedReasoningModes":["clarify_meaning"],"shouldIdentify":["meta_context","testing_hypothesis","self_reference"],"shouldNotInfer":[],"notes":"Paired with tc-12 — the meta-context changes classification entirely."}
|
||||
{"id":"tc-014","input":"The warehouse manager reported that inventory counts don't match the system.","expectedPrimaryTypes":["reported_claim","observed_problem"],"expectedReasoningModes":["validate_claim","investigate_contradiction"],"shouldIdentify":["warehouse_manager_report","inventory_mismatch","system_discrepancy","source_attribution"],"shouldNotInfer":["theft_occurred","software_bug"],"notes":"Reported claim — must distinguish what was said from what it means."}
|
||||
{"id":"tc-015","input":"We've seen a 35% increase in customer complaints.","expectedPrimaryTypes":["unexplained_change"],"expectedReasoningModes":["establish_baseline","validate_measurement"],"shouldIdentify":["complaints_increase","percentage_metric"],"shouldNotInfer":["product_quality_declined","customer_satisfaction_drop"],"notes":"Needs baseline — is this absolute or relative? Over what period?"}
|
||||
{"id":"tc-016","input":"The number of active users increased by 500%, from 4 to 2,001.","expectedPrimaryTypes":["unexplained_change"],"expectedReasoningModes":["validate_measurement","decompose_aggregate"],"shouldIdentify":["active_users_metric","absolute_vs_relative_growth","small_base_problem"],"shouldNotInfer":["product_success"],"notes":"Misleading absolute count where rate matters. Small base inflates percentage."}
|
||||
{"id":"tc-017","input":"User engagement metrics improved but the support ticket backlog grew by 200%.","expectedPrimaryTypes":["contradiction"],"expectedReasoningModes":["investigate_contradiction","identify_difference"],"shouldIdentify":["engagement_improvement","support_backlog_growth","divergent_metrics"],"shouldNotInfer":["users_are_angry","product_quality_is_worse"],"notes":"Two metrics telling opposite stories. Could mean engagement is superficial."}
|
||||
{"id":"tc-018","input":"The manufacturing team needs better quality control.","expectedPrimaryTypes":["decision_request","fault_report"],"expectedReasoningModes":["decision_support","identify_missing_information"],"shouldIdentify":["manufacturing_team","quality_control_desired"],"shouldNotInfer":["quality_is_bad","defect_rate_is_high"],"notes":"Solution proposed without problem specification. What specific quality issue?"}
|
||||
{"id":"tc-019","input":"All users in the EU region are getting a 403 error when trying to access the dashboard.","expectedPrimaryTypes":["observed_problem","fault_report"],"expectedReasoningModes":["validate_claim","identify_difference"],"shouldIdentify":["eu_region","403_error","access_denied","geographic_scope"],"shouldNotInfer":["gdpr_cause","regulatory_change"],"notes":"Geographic subset fault. Should not infer GDPR as cause without evidence."}
|
||||
{"id":"tc-020","input":"Some users in the EU region are getting a 403 error when trying to access the dashboard.","expectedPrimaryTypes":["observed_problem","fault_report"],"expectedReasoningModes":["validate_claim","identify_difference","decompose_aggregate"],"shouldIdentify":["eu_region_subset","403_error","partial_reachability"],"shouldNotInfer":["all_eu_users_affected"],"notes":"Paired with tc-19 — 'some' vs 'all' is the material difference."}
|
||||
{"id":"tc-021","input":"Production output was 1,200 units last month and 1,180 units this month.","expectedPrimaryTypes":["unexplained_change"],"expectedReasoningModes":["validate_measurement","establish_baseline"],"shouldIdentify":["production_output","month_over_month_decline","absolute_difference"],"shouldNotInfer":["efficiency_loss_occurred","equipment_failure"],"notes":"Small absolute change needs context — 1.7% drop might be normal variation."}
|
||||
{"id":"tc-022","input":"The CFO reported that the company's cash position is healthy.","expectedPrimaryTypes":["reported_claim"],"expectedReasoningModes":["validate_claim","identify_missing_information"],"shouldIdentify":["cfo_statement","cash_position_claim","source_attribution_cfo"],"shouldNotInfer":["cash_is_healthy","financial_stability_is_real"],"notes":"Reported opinion — must distinguish what was said from reality."}
|
||||
{"id":"tc-023","input":"We have enough funding to operate for 18 months.","expectedPrimaryTypes":["decision_request","observed_problem"],"expectedReasoningModes":["validate_claim","identify_missing_information"],"shouldIdentify":["funding_period","operational_sustainability","burn_rate_unknown"],"shouldNotInfer":["no_risk_exists"],"notes":"Claim about sustainability without burn rate context."}
|
||||
{"id":"tc-024","input":"The new feature was deployed at 3am and user complaints tripled the next day.","expectedPrimaryTypes":["causal_claim","unexplained_change"],"expectedReasoningModes":["test_possible_explanations","reconstruct_transition"],"shouldIdentify":["feature_deployment","timing_3am","complaint_tripling","temporal_relationship"],"shouldNotInfer":["deployment_caused_complaints"],"notes":"Temporal proximity ≠ causation. Should identify both events but not claim cause."}
|
||||
{"id":"tc-025","input":"We need to launch a mobile app to capture market share.","expectedPrimaryTypes":["decision_request","desired_outcome"],"expectedReasoningModes":["decision_support","identify_missing_information"],"shouldIdentify":["mobile_app_proposed","market_share_desired"],"shouldNotInfer":["no_mobile_app_exists","competitors_have_apps"],"notes":"Desired outcome without problem statement. What evidence supports this decision?"}
|
||||
{"id":"tc-026","input":"The system has been running for 90 days without failure since the migration.","expectedPrimaryTypes":["observed_problem"],"expectedReasoningModes":["validate_claim","establish_baseline"],"shouldIdentify":["uptime_90_days","post_migration_context","baseline_established"],"shouldNotInfer":["system_is_stable_forever"],"notes":"Positive claim about system stability with temporal anchor."}
|
||||
{"id":"tc-027","input":"No one has submitted the required compliance report despite multiple reminders.","expectedPrimaryTypes":["observed_problem","fault_report"],"expectedReasoningModes":["validate_claim","identify_missing_information","investigate_contradiction"],"shouldIdentify":["compliance_report","multiple_reminders","non_submission","absent_action"],"shouldNotInfer":["deliberate_refusal","negligence"],"notes":"Expected-but-missing information. Action was required but absent."}
|
||||
{"id":"tc-028","input":"The audit revealed that 3 of the last 10 monthly reports were submitted with incorrect data.","expectedPrimaryTypes":["observed_problem","contradiction"],"expectedReasoningModes":["validate_measurement","decompose_aggregate"],"shouldIdentify":["audit_findings","incorrect_reports_rate_3_of_10","data_accuracy_issue"],"shouldNotInfer":["intentional_falsification","systemic_failure"],"notes":"Aggregate data — 30% error rate requires context about severity."}
|
||||
{"id":"tc-029","input":"We should implement the new CRM because our competitors have one.","expectedPrimaryTypes":["decision_request","causal_claim"],"expectedReasoningModes":["test_possible_explanations","validate_claim"],"shouldIdentify":["crm_proposal","competitor_comparison","competitive_pressure"],"shouldNotInfer":["crm_will_help","we_lack_crm","competitors_success_is_from_crm"],"notes":"FOMO-driven decision request without problem analysis."}
|
||||
{"id":"tc-030","input":"The server response time was acceptable last quarter but degraded this month.","expectedPrimaryTypes":["unexplained_change","observed_problem"],"expectedReasoningModes":["reconstruct_transition","identify_missing_information"],"shouldIdentify":["response_time_baseline_acceptable","degradation_timeline","quarter_to_month_comparison"],"shouldNotInfer":["load_increase_occurred"],"notes":"Baseline comparison with transition over time. Need specifics."}
|
||||
{"id":"tc-031","input":"The regulatory requirement says all data must be stored within national borders, but our backup server is in another country.","expectedPrimaryTypes":["contradiction","observed_problem"],"expectedReasoningModes":["validate_claim","investigate_contradiction","identify_missing_information"],"shouldIdentify":["regulatory_requirement","data_location_violation","cross_border_backup"],"shouldNotInfer":["compliance_failure_is_certain"],"notes":"Regulatory conflict — requires verification of both claim and current state."}
|
||||
{"id":"tc-032","input":"External analysts expect our industry to decline by 15% next year due to regulatory changes.","expectedPrimaryTypes":["causal_claim","reported_claim"],"expectedReasoningModes":["validate_claim","test_possible_explanations"],"shouldIdentify":["industry_decline_prediction","external_source","regulatory_cause","15_percent_forecast"],"shouldNotInfer":["decline_will_occur"],"notes":"External prediction — must treat as claim, not fact."}
|
||||
{"id":"tc-033","input":"The database schema was changed on Friday but the reports are still working.","expectedPrimaryTypes":["unexplained_change"],"expectedReasoningModes":["validate_claim","test_possible_explanations"],"shouldIdentify":["schema_change","reports_working_post_change","unexpected_continuity"],"shouldNotInfer":["change_was_harmless"],"notes":"Expected impact did not occur — should flag as unexplained."}
|
||||
{"id":"tc-034","input":"Some team members say the new process is better while others say it's slower.","expectedPrimaryTypes":["contradiction","observed_problem"],"expectedReasoningModes":["validate_claim","investigate_contradiction","identify_missing_information"],"shouldIdentify":["subjective_split","new_process_evaluation","conflicting_opinions","measurement_gap"],"shouldNotInfer":["process_is_better_or_worse"],"notes":"Conflicting subjective claims — need measurable criteria."}
|
||||
{"id":"tc-035","input":"The application works fine on Chrome but not on Safari.","expectedPrimaryTypes":["observed_problem","fault_report"],"expectedReasoningModes":["validate_claim","identify_difference"],"shouldIdentify":["chrome_compatibility","safari_incompatibility","browser_specific_issue"],"shouldNotInfer":["webkit_bug"],"notes":"Browser-specific fault. Should identify the difference but not the technical cause."}
|
||||
@@ -1,703 +0,0 @@
|
||||
import React from "react";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { renderToStaticMarkup } from "react-dom/server";
|
||||
import DiagnosticsView from "@/components/diagnostics-view.jsx";
|
||||
import GraphUpdateView from "@/components/graph-update-view.jsx";
|
||||
import SituationGraphView from "@/components/situation-graph-view.jsx";
|
||||
import {
|
||||
ScenarioResultPanels,
|
||||
UpdateErrorPanel,
|
||||
submitAnswerForUpdateCase,
|
||||
submitScenarioForStartCase,
|
||||
} from "@/components/scenario-form.jsx";
|
||||
|
||||
function makeGraphResult(overrides = {}) {
|
||||
return {
|
||||
success: true,
|
||||
situationGraph: {
|
||||
centralStatement: "Complaints increased while production increased.",
|
||||
currentSummary:
|
||||
"Nodes: 2 observation, 1 unknown | Edges: 2 total | Unknowns: 1 unresolved",
|
||||
activeUnknownNodeId: "n-unknown",
|
||||
resolvedNodeIds: [],
|
||||
nodes: [
|
||||
{
|
||||
id: "n-1",
|
||||
label: "Complaints up 35%",
|
||||
description: "Complaints increased by 35%",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
value: 35,
|
||||
unit: "%",
|
||||
},
|
||||
{
|
||||
id: "n-2",
|
||||
label: "Production up 40%",
|
||||
description: "Production increased by 40%",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
value: 40,
|
||||
unit: "%",
|
||||
},
|
||||
{
|
||||
id: "n-unknown",
|
||||
label: "Complaint rate denominator",
|
||||
description: "Need the denominator for complaint rate",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
value: null,
|
||||
unit: null,
|
||||
},
|
||||
],
|
||||
edges: [
|
||||
{ id: "e1", fromNodeId: "n-1", toNodeId: "n-unknown" },
|
||||
{ id: "e2", fromNodeId: "n-2", toNodeId: "n-unknown" },
|
||||
],
|
||||
},
|
||||
selectedQuestion: {
|
||||
question: "What denominator is being used for the complaint rate?",
|
||||
},
|
||||
diagnostics: {
|
||||
modelName: "test",
|
||||
responseDurationMs: 1234,
|
||||
validationStatus: "valid",
|
||||
promptVersion: "test-prompt",
|
||||
nodeCount: 3,
|
||||
edgeCount: 2,
|
||||
graphReferenceValidation: { valid: true, errors: [] },
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function makeUpdateSuccess(overrides = {}) {
|
||||
return {
|
||||
success: true,
|
||||
stage: "update_applied",
|
||||
updatedSituationGraph: {
|
||||
centralStatement: "Complaints increased while production increased.",
|
||||
currentSummary: "Updated summary",
|
||||
activeUnknownNodeId: "n-child-1",
|
||||
resolvedNodeIds: ["n-unknown"],
|
||||
nodes: [
|
||||
{
|
||||
id: "n-1",
|
||||
label: "Complaints up 35%",
|
||||
description: "Complaints increased by 35%",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
value: 35,
|
||||
unit: "%",
|
||||
},
|
||||
{
|
||||
id: "n-conclusion",
|
||||
label: "Quality deterioration",
|
||||
description: "Quality deterioration conclusion",
|
||||
kind: "conclusion",
|
||||
status: "weakened",
|
||||
confidence: "medium",
|
||||
value: null,
|
||||
unit: null,
|
||||
},
|
||||
{
|
||||
id: "n-unknown",
|
||||
label: "Complaint rate denominator",
|
||||
description: "Need the denominator for complaint rate",
|
||||
kind: "unknown",
|
||||
status: "resolved",
|
||||
confidence: "medium",
|
||||
value: "1.9 complaints per 100 units",
|
||||
unit: null,
|
||||
},
|
||||
{
|
||||
id: "n-next-unknown",
|
||||
label:
|
||||
"Explanation for why revenue increased by 18%, but cash in the bank fell over the same period",
|
||||
description:
|
||||
"Need to understand what change or event could explain why these observations differ, because that is needed to investigate their relationship.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
value: null,
|
||||
unit: null,
|
||||
},
|
||||
{
|
||||
id: "n-child-1",
|
||||
label: "How the two observations were measured",
|
||||
description:
|
||||
"Need evidence about the measure used for each observation, because that could help explain revenue increased by 18%, but cash in the bank fell over the same period.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
value: null,
|
||||
unit: null,
|
||||
parentId: "n-next-unknown",
|
||||
},
|
||||
],
|
||||
edges: [
|
||||
{
|
||||
id: "e-rel-next",
|
||||
fromNodeId: "n-conclusion",
|
||||
toNodeId: "n-next-unknown",
|
||||
relationship: "depends_on",
|
||||
confidence: "medium",
|
||||
description:
|
||||
"This unresolved explanation arises from the now-assessed relationship between the observations.",
|
||||
},
|
||||
{
|
||||
id: "e-child-next",
|
||||
fromNodeId: "n-child-1",
|
||||
toNodeId: "n-next-unknown",
|
||||
relationship: "depends_on",
|
||||
confidence: "medium",
|
||||
description:
|
||||
"This child unknown must be investigated before the broader parent explanation can be resolved.",
|
||||
},
|
||||
],
|
||||
},
|
||||
proposal: {
|
||||
addedNodes: [
|
||||
{
|
||||
id: "n-next-unknown",
|
||||
label:
|
||||
"Explanation for why revenue increased by 18%, but cash in the bank fell over the same period",
|
||||
description:
|
||||
"Need to understand what change or event could explain why these observations differ, because that is needed to investigate their relationship.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
value: null,
|
||||
unit: null,
|
||||
evidenceIds: [],
|
||||
dependsOn: ["n-conclusion"],
|
||||
affects: [],
|
||||
parentId: "n-conclusion",
|
||||
childIds: [],
|
||||
},
|
||||
{
|
||||
id: "n-child-1",
|
||||
label: "How the two observations were measured",
|
||||
description:
|
||||
"Need evidence about the measure used for each observation, because that could help explain revenue increased by 18%, but cash in the bank fell over the same period.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
value: null,
|
||||
unit: null,
|
||||
evidenceIds: [],
|
||||
dependsOn: [],
|
||||
affects: [],
|
||||
parentId: "n-next-unknown",
|
||||
childIds: [],
|
||||
},
|
||||
],
|
||||
updatedNodes: [
|
||||
{ nodeId: "n-unknown", newStatus: "resolved", reason: "answered" },
|
||||
],
|
||||
addedEdges: [],
|
||||
removedEdgeIds: [],
|
||||
resolvedUnknownNodeIds: ["n-unknown"],
|
||||
affectedNodeIds: ["n-conclusion"],
|
||||
selectedQuestion: {
|
||||
nodeId: "n-child-1",
|
||||
question:
|
||||
"What evidence would clarify how the two observations were measured?",
|
||||
reason:
|
||||
"Formulated from graph context using the evidence_gathering investigation strategy.",
|
||||
},
|
||||
},
|
||||
selectedQuestion: {
|
||||
nodeId: "n-child-1",
|
||||
question:
|
||||
"What evidence would clarify how the two observations were measured?",
|
||||
reason:
|
||||
"Formulated from graph context using the evidence_gathering investigation strategy.",
|
||||
},
|
||||
affectedNodeIds: ["n-conclusion"],
|
||||
resolvedUnknownNodeIds: ["n-unknown"],
|
||||
previousActiveUnknownNodeId: "n-unknown",
|
||||
newActiveUnknownNodeId: "n-child-1",
|
||||
emergentReasoningNodeCreated: true,
|
||||
emergentReasoningNodeId: "n-next-unknown",
|
||||
emergentReasoningNodeReason:
|
||||
"Created a new unresolved reasoning unknown so the next justified question is backed by the graph.",
|
||||
atomicityAssessment: "composite",
|
||||
decompositionPerformed: true,
|
||||
childUnknownCount: 1,
|
||||
childNodeIds: ["n-child-1"],
|
||||
atomicityReason:
|
||||
"Decomposed a composite unknown into smaller broad candidate dimensions before asking the next question.",
|
||||
previousReasoningState: {
|
||||
comparabilityStatus: "uncertain",
|
||||
reasoningStages: [
|
||||
{
|
||||
stage: "comparability",
|
||||
status: "uncertain",
|
||||
outcome:
|
||||
"Comparability between the observations is not yet established across period, scale, or measurement basis.",
|
||||
},
|
||||
{
|
||||
stage: "relationship",
|
||||
status: "insufficient_information",
|
||||
outcome: "not assessed until comparability is established",
|
||||
},
|
||||
],
|
||||
},
|
||||
reasoningState: {
|
||||
comparabilityStatus: "confirmed",
|
||||
relationshipStatus: "insufficient_information",
|
||||
reasoningStages: [
|
||||
{
|
||||
stage: "comparability",
|
||||
status: "confirmed",
|
||||
outcome:
|
||||
"Comparability was confirmed by the user answer covering the same period and source basis.",
|
||||
},
|
||||
{
|
||||
stage: "relationship",
|
||||
status: "insufficient_information",
|
||||
outcome:
|
||||
"There is not enough structure to classify the relationship safely.",
|
||||
},
|
||||
],
|
||||
},
|
||||
changesApplied: {
|
||||
updatedNodeCount: 2,
|
||||
resolvedUnknownCount: 1,
|
||||
affectedNodeCount: 1,
|
||||
},
|
||||
diagnostics: { responseDurationMs: 100 },
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("scenario-form UI helpers", () => {
|
||||
it("submits to /api/cases/start", async () => {
|
||||
const fetchImpl = vi.fn().mockResolvedValue({ ok: true });
|
||||
|
||||
await submitScenarioForStartCase(fetchImpl, "Scenario text");
|
||||
|
||||
expect(fetchImpl).toHaveBeenCalledWith(
|
||||
"/api/cases/start",
|
||||
expect.objectContaining({
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
}),
|
||||
);
|
||||
});
|
||||
|
||||
it("empty answer is rejected without fetch", async () => {
|
||||
const fetchImpl = vi.fn();
|
||||
|
||||
const result = await submitAnswerForUpdateCase(fetchImpl, {
|
||||
situationGraph: { nodes: [] },
|
||||
previousQuestion: "What changed?",
|
||||
answer: " ",
|
||||
});
|
||||
|
||||
expect(result.skipped).toBe(true);
|
||||
expect(fetchImpl).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("update request body contains graph, previousQuestion and answer", async () => {
|
||||
const fetchImpl = vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
json: async () => ({ success: true }),
|
||||
});
|
||||
const graph = { nodes: [{ id: "n1" }], edges: [] };
|
||||
|
||||
await submitAnswerForUpdateCase(fetchImpl, {
|
||||
situationGraph: graph,
|
||||
previousQuestion: "What changed?",
|
||||
answer: "The rate fell.",
|
||||
});
|
||||
|
||||
expect(fetchImpl).toHaveBeenCalledWith(
|
||||
"/api/cases/update",
|
||||
expect.objectContaining({
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
situationGraph: graph,
|
||||
previousQuestion: "What changed?",
|
||||
answer: "The rate fell.",
|
||||
}),
|
||||
}),
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("graph-backed UI rendering", () => {
|
||||
it("renders central statement from successful graph response", () => {
|
||||
const data = makeGraphResult();
|
||||
const html = renderToStaticMarkup(
|
||||
<SituationGraphView
|
||||
situationGraph={data.situationGraph}
|
||||
selectedQuestion={data.selectedQuestion}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(html).toContain("Central statement");
|
||||
expect(html).toContain("Complaints increased while production increased.");
|
||||
});
|
||||
|
||||
it("renders active unknown", () => {
|
||||
const data = makeGraphResult();
|
||||
const html = renderToStaticMarkup(
|
||||
<SituationGraphView
|
||||
situationGraph={data.situationGraph}
|
||||
selectedQuestion={data.selectedQuestion}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(html).toContain("Active unknown");
|
||||
expect(html).toContain("Complaint rate denominator");
|
||||
});
|
||||
|
||||
it("renders selected question exactly once", () => {
|
||||
const data = makeGraphResult();
|
||||
const html = renderToStaticMarkup(
|
||||
<SituationGraphView
|
||||
situationGraph={data.situationGraph}
|
||||
selectedQuestion={data.selectedQuestion}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(
|
||||
html.match(/What denominator is being used for the complaint rate\?/g) ||
|
||||
[],
|
||||
).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("no answer form appears when selectedQuestion is null", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<SituationGraphView
|
||||
situationGraph={makeGraphResult().situationGraph}
|
||||
selectedQuestion={null}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(html).not.toContain("Update situation");
|
||||
});
|
||||
|
||||
it("renders diagnostics", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<DiagnosticsView result={makeGraphResult()} />,
|
||||
);
|
||||
|
||||
expect(html).toContain("Diagnostics");
|
||||
expect(html).toContain("test");
|
||||
expect(html).toContain("1234ms");
|
||||
expect(html).toContain("Node count");
|
||||
expect(html).toContain("Edge count");
|
||||
expect(html).toContain("Graph references");
|
||||
});
|
||||
|
||||
it("hides empty sections", () => {
|
||||
const base = makeGraphResult();
|
||||
const result = makeGraphResult({
|
||||
selectedQuestion: null,
|
||||
situationGraph: {
|
||||
...base.situationGraph,
|
||||
activeUnknownNodeId: null,
|
||||
nodes: [base.situationGraph.nodes[0]],
|
||||
edges: [],
|
||||
},
|
||||
});
|
||||
|
||||
const html = renderToStaticMarkup(
|
||||
<SituationGraphView
|
||||
situationGraph={result.situationGraph}
|
||||
selectedQuestion={result.selectedQuestion}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(html).not.toContain("Selected Question");
|
||||
expect(html).not.toContain("Active unknown");
|
||||
});
|
||||
|
||||
it("displays API error clearly", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<ScenarioResultPanels
|
||||
status="error"
|
||||
result={{ error: "Invalid start-case request" }}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(html).toContain("Error: Invalid start-case request");
|
||||
});
|
||||
|
||||
it("renders expandable raw graph JSON", () => {
|
||||
const data = makeGraphResult();
|
||||
const html = renderToStaticMarkup(
|
||||
<SituationGraphView
|
||||
situationGraph={data.situationGraph}
|
||||
selectedQuestion={data.selectedQuestion}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(html).toContain("Raw graph JSON");
|
||||
expect(html).toContain(""centralStatement"");
|
||||
});
|
||||
|
||||
it("resolved unknowns render", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<GraphUpdateView
|
||||
updateResult={{
|
||||
...makeUpdateSuccess(),
|
||||
previousSituationGraph: makeGraphResult().situationGraph,
|
||||
}}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(html).toContain("Resolved unknowns");
|
||||
expect(html).toContain("Complaint rate denominator");
|
||||
});
|
||||
|
||||
it("newly surfaced unknowns render", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<GraphUpdateView
|
||||
updateResult={{
|
||||
...makeUpdateSuccess(),
|
||||
previousSituationGraph: makeGraphResult().situationGraph,
|
||||
}}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(html).toContain("Newly surfaced unknowns");
|
||||
expect(html).toContain(
|
||||
"Explanation for why revenue increased by 18%, but cash in the bank fell over the same period",
|
||||
);
|
||||
});
|
||||
|
||||
it("affected nodes render", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<GraphUpdateView
|
||||
updateResult={{
|
||||
...makeUpdateSuccess(),
|
||||
previousSituationGraph: makeGraphResult().situationGraph,
|
||||
}}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(html).toContain("Affected nodes");
|
||||
expect(html).toContain("Quality deterioration");
|
||||
});
|
||||
|
||||
it("renders validated next question when present", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<GraphUpdateView
|
||||
updateResult={{
|
||||
...makeUpdateSuccess(),
|
||||
previousSituationGraph: makeGraphResult().situationGraph,
|
||||
}}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(html).toContain(
|
||||
"What evidence would clarify how the two observations were measured?",
|
||||
);
|
||||
});
|
||||
|
||||
it("no fake next question appears when there is none", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<GraphUpdateView
|
||||
updateResult={{
|
||||
...makeUpdateSuccess({
|
||||
newActiveUnknownNodeId: null,
|
||||
selectedQuestion: null,
|
||||
proposal: {
|
||||
...makeUpdateSuccess().proposal,
|
||||
selectedQuestion: null,
|
||||
},
|
||||
}),
|
||||
previousSituationGraph: makeGraphResult().situationGraph,
|
||||
}}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(html).toContain("No next question selected yet.");
|
||||
});
|
||||
|
||||
it("previous and new active unknowns render labels", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<GraphUpdateView
|
||||
updateResult={{
|
||||
...makeUpdateSuccess(),
|
||||
previousSituationGraph: makeGraphResult().situationGraph,
|
||||
}}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(html).toContain("Previous active unknown");
|
||||
expect(html).toContain("Complaint rate denominator");
|
||||
expect(html).toContain("New active unknown");
|
||||
expect(html).toContain(
|
||||
"Explanation for why revenue increased by 18%, but cash in the bank fell over the same period",
|
||||
);
|
||||
});
|
||||
|
||||
it("successful update renders prior and new state together", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<GraphUpdateView
|
||||
updateResult={{
|
||||
...makeUpdateSuccess(),
|
||||
previousSituationGraph: makeGraphResult().situationGraph,
|
||||
}}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(html).toContain("Previous active unknown");
|
||||
expect(html).toContain("Resolved unknowns");
|
||||
expect(html).toContain("Newly surfaced unknowns");
|
||||
expect(html).toContain("New active unknown");
|
||||
expect(html).toContain("Next question");
|
||||
expect(html).toContain(
|
||||
"What evidence would clarify how the two observations were measured?",
|
||||
);
|
||||
});
|
||||
|
||||
it("update view shows comparability progression without raw ids in the normal view", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<GraphUpdateView
|
||||
updateResult={{
|
||||
...makeUpdateSuccess({
|
||||
selectedQuestion: {
|
||||
nodeId: "n-next-unknown",
|
||||
question:
|
||||
"What evidence would clarify timing or measurement basis?",
|
||||
reason: "A broad follow-up is now justified.",
|
||||
},
|
||||
}),
|
||||
previousSituationGraph: makeGraphResult().situationGraph,
|
||||
}}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(html).toContain("Comparability:");
|
||||
expect(html).toContain("uncertain → confirmed");
|
||||
expect(html).toContain("Relationship status:");
|
||||
expect(html).toContain("insufficient_information");
|
||||
expect(html).toContain("Reasoning stages:");
|
||||
expect(html).toContain("comparability: confirmed");
|
||||
expect(html).toContain("relationship: insufficient_information");
|
||||
expect(html).toContain(
|
||||
"What evidence would clarify how the two observations were measured?",
|
||||
);
|
||||
expect(html).not.toContain("reasoning:comparability");
|
||||
});
|
||||
|
||||
it("situation graph marks newly surfaced and active unknowns", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<SituationGraphView
|
||||
situationGraph={makeUpdateSuccess().updatedSituationGraph}
|
||||
selectedQuestion={makeUpdateSuccess().selectedQuestion}
|
||||
newlySurfacedNodeIds={["n-next-unknown"]}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(html).toContain("newly surfaced unknown");
|
||||
expect(html).toContain("active unknown");
|
||||
expect(html).toContain("resolved unknown");
|
||||
});
|
||||
|
||||
it("disabled follow-up form is shown only as prototype limitation", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<GraphUpdateView
|
||||
updateResult={makeUpdateSuccess()}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(html).toContain(
|
||||
"What evidence would clarify how the two observations were measured?",
|
||||
);
|
||||
});
|
||||
|
||||
it("raw ids remain only in collapsed proposal details", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<GraphUpdateView
|
||||
updateResult={{
|
||||
...makeUpdateSuccess(),
|
||||
previousSituationGraph: makeGraphResult().situationGraph,
|
||||
}}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(html).toContain("Proposal details");
|
||||
expect(html).toContain(""resolvedUnknownNodeIds"");
|
||||
});
|
||||
|
||||
it("update diagnostics render valid values", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<DiagnosticsView
|
||||
result={{
|
||||
diagnostics: {
|
||||
promptVersion: "v0.4",
|
||||
modelName: "configured-model",
|
||||
responseDurationMs: 456,
|
||||
validationStatus: "valid",
|
||||
nodeCount: 7,
|
||||
edgeCount: 3,
|
||||
graphReferenceValidation: { valid: true, errors: [] },
|
||||
},
|
||||
}}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(html).toContain("v0.4");
|
||||
expect(html).toContain("456ms");
|
||||
expect(html).toContain("7");
|
||||
expect(html).toContain("3");
|
||||
expect(html).toContain("✅ valid");
|
||||
});
|
||||
|
||||
it("structured update error renders", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<UpdateErrorPanel
|
||||
updateError={{
|
||||
error: "Invalid graph update proposal",
|
||||
proposalErrors: [{ message: "bad proposal" }],
|
||||
}}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(html).toContain("Update error: Invalid graph update proposal");
|
||||
expect(html).toContain("bad proposal");
|
||||
});
|
||||
|
||||
it("failed update does not fabricate history", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<>
|
||||
<UpdateErrorPanel
|
||||
updateError={{
|
||||
error: "Update case failed",
|
||||
errors: [
|
||||
'New unknown must be explicitly related to an answer-derived node: "nu_commercial_val"',
|
||||
],
|
||||
}}
|
||||
/>
|
||||
<GraphUpdateView updateResult={null} />
|
||||
</>,
|
||||
);
|
||||
|
||||
expect(html).toContain("Update error: Update case failed");
|
||||
expect(html).toContain("nu_commercial_val");
|
||||
expect(html).not.toContain("Previous active unknown");
|
||||
expect(html).not.toContain("Resolved unknowns");
|
||||
expect(html).not.toContain("Newly surfaced unknowns");
|
||||
expect(html).not.toContain("New active unknown");
|
||||
expect(html).not.toContain("Proposal details");
|
||||
});
|
||||
|
||||
it("proposal details remain collapsible", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<GraphUpdateView updateResult={makeUpdateSuccess()} />,
|
||||
);
|
||||
|
||||
expect(html).toContain("Proposal details");
|
||||
});
|
||||
});
|
||||
@@ -1,598 +0,0 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { promises as fs } from "node:fs";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { dirname, join } from "node:path";
|
||||
import {
|
||||
PROMPT_VERSIONS,
|
||||
buildPrompt,
|
||||
DEFAULT_PROMPT_VERSION,
|
||||
} from "@/lib/reconstruction/prompt.js";
|
||||
import {
|
||||
reconstructionV2Schema,
|
||||
parseReconstructionV2,
|
||||
} from "@/lib/reconstruction/schema.js";
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
const PROMPTS_DIR = join(__dirname, "../prompts");
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
// v0.3 prompt loading tests
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
describe("v0.3 prompt", () => {
|
||||
it("v0.3 is in PROMPT_VERSIONS", () => {
|
||||
expect(PROMPT_VERSIONS).toContain("v0.3");
|
||||
});
|
||||
|
||||
it("DEFAULT_PROMPT_VERSION is v0.3 on this branch", () => {
|
||||
expect(DEFAULT_PROMPT_VERSION).toBe("v0.3");
|
||||
});
|
||||
|
||||
it("v0.2 remains available in PROMPT_VERSIONS", () => {
|
||||
expect(PROMPT_VERSIONS).toContain("v0.2");
|
||||
});
|
||||
|
||||
it("v0.3 prompt file loads from disk", async () => {
|
||||
const content = await fs.readFile(
|
||||
join(PROMPTS_DIR, "reconstruct-v0.3.md"),
|
||||
"utf-8",
|
||||
);
|
||||
expect(typeof content).toBe("string");
|
||||
expect(content.length).toBeGreaterThan(500);
|
||||
});
|
||||
|
||||
it("v0.3 prompt contains normalisation guidance", async () => {
|
||||
const content = await fs.readFile(
|
||||
join(PROMPTS_DIR, "reconstruct-v0.3.md"),
|
||||
"utf-8",
|
||||
);
|
||||
expect(content.toLowerCase()).toContain("normalise");
|
||||
expect(content.toLowerCase()).toContain("rate");
|
||||
expect(content.toLowerCase()).toContain("denominator") ||
|
||||
expect(content.toLowerCase()).toContain("exposure");
|
||||
});
|
||||
|
||||
it("v0.3 prompt contains discipline guidance", async () => {
|
||||
const content = await fs.readFile(
|
||||
join(PROMPTS_DIR, "reconstruct-v0.3.md"),
|
||||
"utf-8",
|
||||
);
|
||||
// Should mention not generating speculative interpretations
|
||||
expect(content).toMatch(/interpretation/i);
|
||||
// Should mention one question discipline
|
||||
expect(content).toMatch(/exactly.*one.*question|one.*only.*question|single.*question/i) ||
|
||||
expect(content).toMatch(/Do NOT combine/i);
|
||||
});
|
||||
|
||||
it("buildPrompt returns v0.3 prompt with scenario substituted", async () => {
|
||||
const result = await buildPrompt("Test scenario text", "v0.3");
|
||||
expect(result.version).toBe("v0.3");
|
||||
expect(result.prompt).toContain("Test scenario text");
|
||||
// Should contain the normalisation section guidance
|
||||
expect(result.prompt.toLowerCase()).toContain("normalise");
|
||||
});
|
||||
|
||||
it("buildPrompt returns v0.2 prompt when requested", async () => {
|
||||
const result = await buildPrompt("Test scenario text", "v0.2");
|
||||
expect(result.version).toBe("v0.2");
|
||||
expect(result.prompt).toContain("Test scenario text");
|
||||
});
|
||||
|
||||
it("buildPrompt default is v0.3", async () => {
|
||||
const result = await buildPrompt("Test scenario text");
|
||||
expect(result.version).toBe("v0.3");
|
||||
});
|
||||
});
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
// v0.2 prompt still works
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
describe("v0.2 backward compatibility", () => {
|
||||
it("v0.2 prompt file exists and loads", async () => {
|
||||
const content = await fs.readFile(
|
||||
join(PROMPTS_DIR, "reconstruct-v0.2.md"),
|
||||
"utf-8",
|
||||
);
|
||||
expect(typeof content).toBe("string");
|
||||
expect(content.length).toBeGreaterThan(500);
|
||||
});
|
||||
|
||||
it("buildPrompt returns v0.2 version string", async () => {
|
||||
const result = await buildPrompt("test", "v0.2");
|
||||
expect(result.version).toBe("v0.2");
|
||||
});
|
||||
});
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
// Schema validation tests for v0.3-shaped output
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
describe("v0.3 schema validation", () => {
|
||||
it("validates a complete valid reconstruction with empty interpretations", () => {
|
||||
const input = {
|
||||
inputClassification: {
|
||||
primaryType: "unexplained_change",
|
||||
secondaryTypes: ["reported_claim"],
|
||||
reasoningModes: ["identify_difference"],
|
||||
classificationReason: "Two metrics changed without explanation.",
|
||||
confidence: "medium",
|
||||
},
|
||||
reconstruction: {
|
||||
summary: "Both complaints and production increased.",
|
||||
actors: [],
|
||||
systemsOrObjects: [
|
||||
{ id: "complaints_metric", description: "Volume of complaints", confidence: "high" },
|
||||
],
|
||||
expectedStates: [],
|
||||
observedStates: [
|
||||
{ id: "obs1", description: "Complaint volume rose by 35%", confidence: "medium" },
|
||||
{ id: "obs2", description: "Production volume rose by 40%", confidence: "medium" },
|
||||
],
|
||||
differences: [
|
||||
{
|
||||
id: "diff1",
|
||||
description:
|
||||
"Production grew faster than complaints, so the complaint-to-production ratio may have improved.",
|
||||
confidence: "medium",
|
||||
},
|
||||
],
|
||||
knownTransitions: [],
|
||||
unexplainedTransitions: [
|
||||
{
|
||||
id: "trans1",
|
||||
description: "Complaint volume shifted to a higher level without explained cause",
|
||||
confidence: "medium",
|
||||
entity: "complaints_metric",
|
||||
previousState: "Baseline volume (unknown)",
|
||||
currentState: "+35% increase",
|
||||
},
|
||||
],
|
||||
contradictions: [],
|
||||
importantUnknowns: [
|
||||
{
|
||||
id: "unk1",
|
||||
description:
|
||||
"Absolute baseline volumes and time period needed to compute complaint rate per unit",
|
||||
confidence: "low",
|
||||
},
|
||||
],
|
||||
plausibleInterpretations: [], // intentionally empty — evidence too thin
|
||||
},
|
||||
evidence: [
|
||||
{
|
||||
id: "ev1",
|
||||
description: "Complaints increased by 35%",
|
||||
evidenceType: "reported_statement",
|
||||
source: "User input",
|
||||
attribution: null,
|
||||
confidence: "medium",
|
||||
importance: "important",
|
||||
},
|
||||
{
|
||||
id: "ev2",
|
||||
description: "Production increased by 40%",
|
||||
evidenceType: "reported_statement",
|
||||
source: "User input",
|
||||
attribution: null,
|
||||
confidence: "medium",
|
||||
importance: "important",
|
||||
},
|
||||
{
|
||||
id: "ev3",
|
||||
description:
|
||||
"Production growth rate (40%) exceeded complaint growth rate (35%), implying the denominator may have grown faster than complaints.",
|
||||
evidenceType: "inferred_relationship",
|
||||
attribution: null,
|
||||
confidence: "medium",
|
||||
importance: "important",
|
||||
},
|
||||
],
|
||||
nextQuestion: {
|
||||
id: "q1",
|
||||
question: "What was the complaint rate per unit before and after the production increase?",
|
||||
targets: ["system"],
|
||||
reason:
|
||||
"Without normalising complaints by production volume, the absolute complaint count change is misleading. The rate per unit determines whether the situation improved, stayed stable, or worsened.",
|
||||
expectedInformationValue: "high",
|
||||
reasoningMode: "decompose_aggregate",
|
||||
},
|
||||
};
|
||||
|
||||
const result = reconstructionV2Schema.safeParse(input);
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects output missing required fields", () => {
|
||||
const input = {
|
||||
inputClassification: { primaryType: "other" },
|
||||
reconstruction: {},
|
||||
evidence: [],
|
||||
nextQuestion: { id: "q1" },
|
||||
};
|
||||
|
||||
const result = reconstructionV2Schema.safeParse(input);
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it("validates empty arrays for all reconstruction categories", () => {
|
||||
const input = {
|
||||
inputClassification: {
|
||||
primaryType: "other",
|
||||
classificationReason: "test",
|
||||
confidence: "low",
|
||||
},
|
||||
reconstruction: {
|
||||
summary: "empty test",
|
||||
actors: [],
|
||||
systemsOrObjects: [],
|
||||
expectedStates: [],
|
||||
observedStates: [],
|
||||
differences: [],
|
||||
knownTransitions: [],
|
||||
unexplainedTransitions: [],
|
||||
contradictions: [],
|
||||
importantUnknowns: [],
|
||||
plausibleInterpretations: [],
|
||||
},
|
||||
evidence: [],
|
||||
nextQuestion: {
|
||||
id: "q1",
|
||||
question: "What is the production volume?",
|
||||
targets: ["system"],
|
||||
reason: "need baseline",
|
||||
expectedInformationValue: "medium",
|
||||
},
|
||||
};
|
||||
|
||||
const result = reconstructionV2Schema.safeParse(input);
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("validates evidence distinguishing direct_observation from inferred_relationship", () => {
|
||||
const input = {
|
||||
inputClassification: {
|
||||
primaryType: "unexplained_change",
|
||||
classificationReason: "test",
|
||||
confidence: "low",
|
||||
},
|
||||
reconstruction: {
|
||||
summary: "test summary",
|
||||
actors: [],
|
||||
systemsOrObjects: [],
|
||||
expectedStates: [],
|
||||
observedStates: [{ id: "o1", description: "x", confidence: "high" }],
|
||||
differences: [],
|
||||
knownTransitions: [],
|
||||
unexplainedTransitions: [],
|
||||
contradictions: [],
|
||||
importantUnknowns: [],
|
||||
plausibleInterpretations: [],
|
||||
},
|
||||
evidence: [
|
||||
{
|
||||
id: "ev1",
|
||||
description: "Observed fact",
|
||||
evidenceType: "direct_observation",
|
||||
confidence: "high",
|
||||
importance: "critical",
|
||||
},
|
||||
{
|
||||
id: "ev2",
|
||||
description: "Derived relationship",
|
||||
evidenceType: "inferred_relationship",
|
||||
confidence: "medium",
|
||||
importance: "supporting",
|
||||
},
|
||||
],
|
||||
nextQuestion: {
|
||||
id: "q1",
|
||||
question: "What is the denominator?",
|
||||
targets: ["system"],
|
||||
reason: "need context",
|
||||
expectedInformationValue: "high",
|
||||
},
|
||||
};
|
||||
|
||||
const result = reconstructionV2Schema.safeParse(input);
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
// parseReconstructionV2 helper tests
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
describe("parseReconstructionV2", () => {
|
||||
it("parses a valid v0.3-shaped JSON string", async () => {
|
||||
const fixture = {
|
||||
inputClassification: {
|
||||
primaryType: "unexplained_change",
|
||||
classificationReason: "test",
|
||||
confidence: "medium",
|
||||
},
|
||||
reconstruction: {
|
||||
summary: "both increased",
|
||||
actors: [],
|
||||
systemsOrObjects: [],
|
||||
expectedStates: [],
|
||||
observedStates: [
|
||||
{ id: "o1", description: "x rose 35%", confidence: "high" },
|
||||
{ id: "o2", description: "y rose 40%", confidence: "high" },
|
||||
],
|
||||
differences: [{ id: "d1", description: "y grew faster", confidence: "medium" }],
|
||||
knownTransitions: [],
|
||||
unexplainedTransitions: [],
|
||||
contradictions: [],
|
||||
importantUnknowns: [],
|
||||
plausibleInterpretations: [],
|
||||
},
|
||||
evidence: [
|
||||
{ id: "e1", description: "x rose 35%", evidenceType: "reported_statement", confidence: "medium", importance: "important" },
|
||||
{ id: "e2", description: "y rose 40%", evidenceType: "reported_statement", confidence: "medium", importance: "important" },
|
||||
],
|
||||
nextQuestion: {
|
||||
id: "q1",
|
||||
question: "What is the denominator?",
|
||||
targets: ["system"],
|
||||
reason: "need rate context",
|
||||
expectedInformationValue: "high",
|
||||
},
|
||||
};
|
||||
|
||||
const raw = JSON.stringify(fixture);
|
||||
const parsed = parseReconstructionV2(raw);
|
||||
|
||||
expect(parsed.inputClassification.primaryType).toBe("unexplained_change");
|
||||
expect(parsed.reconstruction.summary).toBe("both increased");
|
||||
expect(parsed.nextQuestion.question).toBe("What is the denominator?");
|
||||
});
|
||||
|
||||
it("rejects non-JSON string", () => {
|
||||
expect(() => parseReconstructionV2("{not valid json")).toThrow(SyntaxError);
|
||||
});
|
||||
});
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
// v0.3 prompt contains required guidance text
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
describe("v0.3 prompt guidance completeness", () => {
|
||||
it("mentions normalise counts when scale changed", async () => {
|
||||
const content = await fs.readFile(
|
||||
join(PROMPTS_DIR, "reconstruct-v0.3.md"),
|
||||
"utf-8",
|
||||
);
|
||||
expect(content.toLowerCase()).toMatch(/normali[sz]e|normalis[ei]ng/);
|
||||
});
|
||||
|
||||
it("mentions distinguishing total count from rate", async () => {
|
||||
const content = await fs.readFile(
|
||||
join(PROMPTS_DIR, "reconstruct-v0.3.md"),
|
||||
"utf-8",
|
||||
);
|
||||
expect(content.toLowerCase()).toContain("rate");
|
||||
expect(content.toLowerCase()).toMatch(/count.*not.*caus|correlation.*caus|distinguish.*count/);
|
||||
});
|
||||
|
||||
it("mentions avoiding correlation-as-causation", async () => {
|
||||
const content = await fs.readFile(
|
||||
join(PROMPTS_DIR, "reconstruct-v0.3.md"),
|
||||
"utf-8",
|
||||
);
|
||||
expect(content.toLowerCase()).toMatch(/correlation.*caus|treating.*correlation.*caus/);
|
||||
});
|
||||
|
||||
it("mentions prefer one narrow next question over compound", async () => {
|
||||
const content = await fs.readFile(
|
||||
join(PROMPTS_DIR, "reconstruct-v0.3.md"),
|
||||
"utf-8",
|
||||
);
|
||||
// Should mention single vs compound
|
||||
expect(content).toMatch(/exactly.*one|single.*question|Do NOT combine|combine.*multiple/i);
|
||||
});
|
||||
|
||||
it("mentions leaving empty interpretations when evidence is thin", async () => {
|
||||
const content = await fs.readFile(
|
||||
join(PROMPTS_DIR, "reconstruct-v0.3.md"),
|
||||
"utf-8",
|
||||
);
|
||||
expect(content).toMatch(/empty.*array|do not generate.*interpretation|fill a list/i);
|
||||
});
|
||||
|
||||
it("mentions identifying the denominator or exposure metric", async () => {
|
||||
const content = await fs.readFile(
|
||||
join(PROMPTS_DIR, "reconstruct-v0.3.md"),
|
||||
"utf-8",
|
||||
);
|
||||
expect(content.toLowerCase()).toMatch(/denominator|exposure/);
|
||||
});
|
||||
|
||||
it("uses the exact scenario text as a reference example only (not in rules)", async () => {
|
||||
const content = await fs.readFile(
|
||||
join(PROMPTS_DIR, "reconstruct-v0.3.md"),
|
||||
"utf-8",
|
||||
);
|
||||
// The prompt should be domain-independent — it should not mention specific industries as rules
|
||||
// but may have an example section. We verify the prompt does not hard-code a specific question text.
|
||||
expect(content).not.toMatch(/What was the complaint rate per unit before and after/);
|
||||
});
|
||||
});
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
// Fixture: expected good structure for target scenario
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
describe("target scenario fixture validation", () => {
|
||||
const goodFixture = JSON.parse(JSON.stringify({
|
||||
inputClassification: {
|
||||
primaryType: "unexplained_change",
|
||||
secondaryTypes: ["reported_claim"],
|
||||
reasoningModes: ["identify_difference", "decompose_aggregate"],
|
||||
classificationReason:
|
||||
"Two operational quantities changed at different percentages without a shared baseline or denominator.",
|
||||
confidence: "medium",
|
||||
},
|
||||
reconstruction: {
|
||||
summary:
|
||||
"Both complaint counts and production volumes increased, but production grew slightly faster than complaints — without absolute baselines the per-unit complaint rate cannot be determined.",
|
||||
actors: [],
|
||||
systemsOrObjects: [
|
||||
{ id: "so1", description: "Production system or output volume", confidence: "high" },
|
||||
{ id: "so2", description: "Complaint reporting mechanism", confidence: "high" },
|
||||
],
|
||||
expectedStates: [],
|
||||
observedStates: [
|
||||
{ id: "obs1", description: "Complaint count increased by 35%", confidence: "high" },
|
||||
{ id: "obs2", description: "Production volume increased by 40%", confidence: "high" },
|
||||
],
|
||||
differences: [
|
||||
{
|
||||
id: "diff1",
|
||||
description:
|
||||
"Production grew faster than complaints (+40% vs +35%), so the ratio of complaints per unit may have decreased or remained stable. The absolute complaint count alone is not a reliable indicator of whether conditions have changed.",
|
||||
confidence: "high",
|
||||
},
|
||||
],
|
||||
knownTransitions: [],
|
||||
unexplainedTransitions: [
|
||||
{
|
||||
id: "ut1",
|
||||
description: "Complaint volume shifted to a higher level without explained cause",
|
||||
confidence: "medium",
|
||||
entity: "complaints_metric",
|
||||
previousState: "unknown baseline",
|
||||
currentState: "+35%",
|
||||
},
|
||||
],
|
||||
contradictions: [],
|
||||
importantUnknowns: [
|
||||
{
|
||||
id: "unk1",
|
||||
description:
|
||||
"Absolute complaint count and production volume baselines needed to compute the per-unit rate",
|
||||
confidence: "low",
|
||||
},
|
||||
{
|
||||
id: "unk2",
|
||||
description: "Time period over which these changes occurred",
|
||||
confidence: "low",
|
||||
},
|
||||
],
|
||||
plausibleInterpretations: [], // intentionally empty — no sufficient evidence for interpretations
|
||||
},
|
||||
evidence: [
|
||||
{
|
||||
id: "ev1",
|
||||
description: "Complaints increased by 35%",
|
||||
evidenceType: "reported_statement",
|
||||
source: "Scenario input",
|
||||
attribution: null,
|
||||
confidence: "high",
|
||||
importance: "important",
|
||||
},
|
||||
{
|
||||
id: "ev2",
|
||||
description: "Production increased by 40%",
|
||||
evidenceType: "reported_statement",
|
||||
source: "Scenario input",
|
||||
attribution: null,
|
||||
confidence: "high",
|
||||
importance: "important",
|
||||
},
|
||||
{
|
||||
id: "ev3",
|
||||
description: "Complaint count grew more slowly than production volume, suggesting per-unit rates may have improved or stayed stable.",
|
||||
evidenceType: "inferred_relationship",
|
||||
attribution: null,
|
||||
confidence: "medium",
|
||||
importance: "important",
|
||||
},
|
||||
],
|
||||
nextQuestion: {
|
||||
id: "q1",
|
||||
question: "What was the absolute complaint volume and production volume (or baseline) before these percentage changes?",
|
||||
targets: ["system", "measurement"],
|
||||
reason:
|
||||
"Without baseline counts to compute a rate per unit, we cannot determine whether conditions have worsened, stayed stable, or improved. The rate comparison is the smallest unresolved comparison needed to evaluate the situation.",
|
||||
expectedInformationValue: "high",
|
||||
reasoningMode: "decompose_aggregate",
|
||||
},
|
||||
}));
|
||||
|
||||
it("fixture validates against v0.3 schema", () => {
|
||||
const result = reconstructionV2Schema.safeParse(goodFixture);
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("fixture has exactly one next question with non-empty text", () => {
|
||||
expect(goodFixture.nextQuestion.question.length).toBeGreaterThan(10);
|
||||
expect(goodFixture.nextQuestion.reason.length).toBeGreaterThan(10);
|
||||
expect(goodFixture.nextQuestion.expectedInformationValue).toBe("high");
|
||||
});
|
||||
|
||||
it("fixture has empty plausibleInterpretations (evidence too thin)", () => {
|
||||
expect(goodFixture.reconstruction.plausibleInterpretations).toEqual([]);
|
||||
});
|
||||
|
||||
it("fixture evidence includes both direct observations and one inferred relationship", () => {
|
||||
const types = goodFixture.evidence.map((e) => e.evidenceType);
|
||||
expect(types).toContain("reported_statement");
|
||||
expect(types).toContain("inferred_relationship");
|
||||
});
|
||||
|
||||
it("fixture relationship notes complaint count grew more slowly than production", () => {
|
||||
const diffDescs = goodFixture.reconstruction.differences.map((d) => d.description);
|
||||
const found = diffDescs.some(
|
||||
(d) =>
|
||||
d.toLowerCase().includes("fast") ||
|
||||
d.toLowerCase().includes("slower") ||
|
||||
d.toLowerCase().includes("ratio") ||
|
||||
d.toLowerCase().includes("per-unit") ||
|
||||
d.toLowerCase().includes("per unit"),
|
||||
);
|
||||
expect(found).toBe(true);
|
||||
});
|
||||
|
||||
it("fixture does not assert quality deterioration", () => {
|
||||
const allText = [
|
||||
goodFixture.reconstruction.summary,
|
||||
...goodFixture.reconstruction.differences.map((d) => d.description),
|
||||
goodFixture.nextQuestion.reason,
|
||||
].join(" ").toLowerCase();
|
||||
// Should not contain strong deterioration language without caveats
|
||||
expect(allText).not.toMatch(/quality.*deteriorat|quality.*worsen|definitely.*bad/);
|
||||
});
|
||||
|
||||
it("fixture includes relationship that production grew faster", () => {
|
||||
const allText = [
|
||||
goodFixture.reconstruction.summary,
|
||||
...goodFixture.reconstruction.differences.map((d) => d.description),
|
||||
].join(" ").toLowerCase();
|
||||
expect(allText).toMatch(/produ.*grow|ratio|per-unit|per unit|\+40.*\+35/);
|
||||
});
|
||||
});
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
// Diagnostics: prompt version tracking
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
describe("diagnostics prompt version", () => {
|
||||
it("DEFAULT_PROMPT_VERSION is exported correctly", () => {
|
||||
expect(DEFAULT_PROMPT_VERSION).toBe("v0.3");
|
||||
});
|
||||
|
||||
it("PROMPT_VERSIONS includes both v0.2 and v0.3", () => {
|
||||
const hasV2 = PROMPT_VERSIONS.includes("v0.2");
|
||||
const hasV3 = PROMPT_VERSIONS.includes("v0.3");
|
||||
expect(hasV2).toBe(true);
|
||||
expect(hasV3).toBe(true);
|
||||
});
|
||||
|
||||
it("RECONSTRUCTION_PROMPT_VERSION env var overrides default", async () => {
|
||||
// The actual override happens at module load time, so we can't easily test this
|
||||
// in isolation. Instead, verify the constant reflects env or defaults to v0.3.
|
||||
expect(PROMPT_VERSIONS).toContain("v0.2");
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user