Compare commits
37
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
59631f2e72 | ||
|
|
b73760d5a7 | ||
|
|
fe6a9925cb | ||
|
|
34c25fcb43 | ||
|
|
c27320984c | ||
|
|
3e2edd2edc | ||
|
|
b00928d6fb | ||
|
|
42d4da3496 | ||
|
|
db994d7764 | ||
|
|
ef04b9e494 | ||
|
|
3c0f7f5a45 | ||
|
|
449cf996dc | ||
|
|
5049435005 | ||
|
|
e0d9019c2a | ||
|
|
b2ffc54964 | ||
|
|
1d64144e01 | ||
|
|
49765e95a0 | ||
|
|
d52690cf2b | ||
|
|
0723c2f49a | ||
|
|
b1c633ba5c | ||
|
|
25a989450c | ||
|
|
7d408701b5 | ||
|
|
c97f5f7303 | ||
|
|
0c7558d31f | ||
|
|
b84989b96a | ||
|
|
51ce356218 | ||
|
|
a1f6d0c2b9 | ||
|
|
586802950d | ||
|
|
5ef9710293 | ||
|
|
a79a7bd524 | ||
|
|
2e4c624a8a | ||
|
|
781d6a462f | ||
|
|
48ce66dddb | ||
|
|
4affadab4b | ||
|
|
392564ed61 | ||
|
|
72ef175971 | ||
|
|
904aec7616 |
@@ -80,6 +80,14 @@ export default function DiagnosticsView({ result }) {
|
||||
? `${validationIcons.valid} valid`
|
||||
: `${validationIcons.invalid} invalid`,
|
||||
},
|
||||
{
|
||||
label: "Investigation strategy",
|
||||
value:
|
||||
diagnostics.investigationStrategy?.key ||
|
||||
diagnostics.investigationStrategy ||
|
||||
result.selectedQuestion?.strategy ||
|
||||
"?",
|
||||
},
|
||||
];
|
||||
|
||||
const errors = [
|
||||
|
||||
@@ -23,12 +23,19 @@ export default function GraphUpdateView({ updateResult }) {
|
||||
affectedNodeIds,
|
||||
previousActiveUnknownNodeId,
|
||||
newActiveUnknownNodeId,
|
||||
selectedQuestion,
|
||||
changesApplied,
|
||||
proposal,
|
||||
previousSituationGraph,
|
||||
updatedSituationGraph,
|
||||
reasoningState,
|
||||
previousReasoningState,
|
||||
} = updateResult;
|
||||
|
||||
const newlySurfacedUnknownNodeIds = (proposal.addedNodes || [])
|
||||
.filter((node) => node.kind === "unknown")
|
||||
.map((node) => node.id);
|
||||
|
||||
const previousNodesById = new Map(
|
||||
(previousSituationGraph?.nodes || []).map((node) => [node.id, node]),
|
||||
);
|
||||
@@ -59,6 +66,11 @@ export default function GraphUpdateView({ updateResult }) {
|
||||
<div className="text-xs text-gray-600">
|
||||
{node.kind} · {node.confidence}
|
||||
</div>
|
||||
{node.confidenceAssessment && (
|
||||
<div className="text-xs text-gray-600">
|
||||
evidence {node.confidenceAssessment.evidenceConfidence} · completeness {node.confidenceAssessment.completenessStatus} · conclusion {node.confidenceAssessment.conclusionConfidence}
|
||||
</div>
|
||||
)}
|
||||
{(update?.previousStatus || update?.newStatus || node.status) && (
|
||||
<div className="text-xs text-gray-700">
|
||||
{update?.previousStatus ? `Previous status: ${update.previousStatus}` : null}
|
||||
@@ -104,6 +116,23 @@ export default function GraphUpdateView({ updateResult }) {
|
||||
: null,
|
||||
].filter(Boolean);
|
||||
|
||||
const previousComparabilityStatus =
|
||||
previousReasoningState?.comparabilityStatus ||
|
||||
previousSituationGraph?.reasoningState?.comparabilityStatus ||
|
||||
null;
|
||||
const newComparabilityStatus =
|
||||
reasoningState?.comparabilityStatus ||
|
||||
updatedSituationGraph?.reasoningState?.comparabilityStatus ||
|
||||
null;
|
||||
const relationshipStatus =
|
||||
reasoningState?.relationshipStatus ||
|
||||
updatedSituationGraph?.reasoningState?.relationshipStatus ||
|
||||
null;
|
||||
const reasoningStagesAfter =
|
||||
reasoningState?.reasoningStages ||
|
||||
updatedSituationGraph?.reasoningState?.reasoningStages ||
|
||||
[];
|
||||
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
<section className="rounded-lg border border-blue-200 bg-blue-50 p-4">
|
||||
@@ -123,13 +152,38 @@ export default function GraphUpdateView({ updateResult }) {
|
||||
{resolveActiveUnknown(newActiveUnknownNodeId)}
|
||||
</div>
|
||||
)}
|
||||
{!newActiveUnknownNodeId && previousActiveUnknownNodeId && (
|
||||
{selectedQuestion?.question && (
|
||||
<div>
|
||||
<span className="font-medium">Next question status:</span> No next
|
||||
question selected yet.
|
||||
<span className="font-medium">Next question:</span>{" "}
|
||||
{selectedQuestion.question}
|
||||
</div>
|
||||
)}
|
||||
{previousComparabilityStatus && newComparabilityStatus && (
|
||||
<div>
|
||||
<span className="font-medium">Comparability:</span>{" "}
|
||||
{previousComparabilityStatus} → {newComparabilityStatus}
|
||||
</div>
|
||||
)}
|
||||
{relationshipStatus && (
|
||||
<div>
|
||||
<span className="font-medium">Relationship status:</span>{" "}
|
||||
{relationshipStatus}
|
||||
</div>
|
||||
)}
|
||||
{!selectedQuestion?.question && !newActiveUnknownNodeId && previousActiveUnknownNodeId && (
|
||||
<div>
|
||||
<span className="font-medium">Next question status:</span> No next question selected yet.
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
{reasoningStagesAfter.length > 0 && (
|
||||
<div className="mt-3 text-sm text-blue-950">
|
||||
<span className="font-medium">Reasoning stages:</span>{" "}
|
||||
{reasoningStagesAfter
|
||||
.map((stage) => `${stage.stage}: ${stage.status}`)
|
||||
.join(" → ")}
|
||||
</div>
|
||||
)}
|
||||
</section>
|
||||
|
||||
<ListSection
|
||||
@@ -137,6 +191,11 @@ export default function GraphUpdateView({ updateResult }) {
|
||||
items={resolvedUnknownNodeIds}
|
||||
renderItem={resolveNodePresentation}
|
||||
/>
|
||||
<ListSection
|
||||
title="Newly surfaced unknowns"
|
||||
items={newlySurfacedUnknownNodeIds}
|
||||
renderItem={resolveNodePresentation}
|
||||
/>
|
||||
<ListSection
|
||||
title="Affected nodes"
|
||||
items={affectedNodeIds}
|
||||
|
||||
@@ -52,9 +52,16 @@ function normaliseStartResult(data) {
|
||||
typeof data?.selectedQuestion === "string"
|
||||
? data.selectedQuestion
|
||||
: data?.selectedQuestion?.question ?? null,
|
||||
newlySurfacedNodeIds: data?.newlySurfacedNodeIds ?? [],
|
||||
};
|
||||
}
|
||||
|
||||
function normaliseUpdateSelectedQuestion(selectedQuestion) {
|
||||
if (!selectedQuestion) return null;
|
||||
if (typeof selectedQuestion === "string") return selectedQuestion;
|
||||
return selectedQuestion.question ?? null;
|
||||
}
|
||||
|
||||
export function ScenarioResultPanels({ status, result }) {
|
||||
if (!result) return null;
|
||||
|
||||
@@ -83,6 +90,7 @@ export function ScenarioResultPanels({ status, result }) {
|
||||
<SituationGraphView
|
||||
situationGraph={result.situationGraph}
|
||||
selectedQuestion={result.selectedQuestion}
|
||||
newlySurfacedNodeIds={result.newlySurfacedNodeIds}
|
||||
/>
|
||||
)}
|
||||
|
||||
@@ -192,7 +200,12 @@ export default function ScenarioForm() {
|
||||
setResult((current) => ({
|
||||
...current,
|
||||
situationGraph: outcome.updatedSituationGraph,
|
||||
selectedQuestion: null,
|
||||
selectedQuestion: normaliseUpdateSelectedQuestion(
|
||||
outcome.selectedQuestion,
|
||||
),
|
||||
newlySurfacedNodeIds: (outcome.proposal?.addedNodes || [])
|
||||
.filter((node) => node.kind === "unknown")
|
||||
.map((node) => node.id),
|
||||
diagnostics: outcome.diagnostics,
|
||||
}));
|
||||
setAnswer("");
|
||||
@@ -208,9 +221,14 @@ export default function ScenarioForm() {
|
||||
|
||||
const canRenderAnswerForm =
|
||||
status === "success" &&
|
||||
updateStatus === "idle" &&
|
||||
Boolean(result?.situationGraph) &&
|
||||
Boolean(result?.selectedQuestion);
|
||||
|
||||
const canRenderDisabledFollowUpForm =
|
||||
updateStatus === "success" &&
|
||||
Boolean(updateResult?.selectedQuestion?.question || result?.selectedQuestion);
|
||||
|
||||
return (
|
||||
<div className="space-y-6">
|
||||
<form onSubmit={handleSubmit} className="space-y-4">
|
||||
@@ -277,8 +295,44 @@ export default function ScenarioForm() {
|
||||
{updateStatus === "success" && updateResult && (
|
||||
<>
|
||||
<div className="rounded-lg border border-yellow-300 bg-yellow-50 px-4 py-3 text-sm text-yellow-800">
|
||||
No next question selected yet.
|
||||
{updateResult.selectedQuestion?.question
|
||||
? updateResult.selectedQuestion.question
|
||||
: "No next question selected yet."}
|
||||
</div>
|
||||
{canRenderDisabledFollowUpForm && (
|
||||
<form className="space-y-4 rounded-lg border border-gray-200 bg-white p-4 opacity-70">
|
||||
<div>
|
||||
<h2 className="text-base font-semibold text-gray-900">Selected Question</h2>
|
||||
<p className="mt-1 text-sm text-gray-700">
|
||||
{updateResult.selectedQuestion?.question || result?.selectedQuestion}
|
||||
</p>
|
||||
</div>
|
||||
<div>
|
||||
<label htmlFor="follow-up-disabled-textarea" className="mb-2 block text-sm font-medium text-gray-700">
|
||||
Your answer
|
||||
</label>
|
||||
<textarea
|
||||
id="follow-up-disabled-textarea"
|
||||
rows={4}
|
||||
disabled
|
||||
className="w-full rounded-lg border border-gray-300 px-4 py-3 text-sm opacity-70"
|
||||
placeholder="Additional submission is disabled in this one-update prototype."
|
||||
/>
|
||||
</div>
|
||||
<div className="flex items-center justify-between gap-4">
|
||||
<p className="text-xs text-gray-500">
|
||||
Additional submission is disabled in this one-update prototype.
|
||||
</p>
|
||||
<button
|
||||
type="button"
|
||||
disabled
|
||||
className="rounded-lg bg-blue-700 px-4 py-2 text-sm font-medium text-white disabled:cursor-not-allowed disabled:opacity-40"
|
||||
>
|
||||
Update situation
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
)}
|
||||
<GraphUpdateView updateResult={updateResult} />
|
||||
</>
|
||||
)}
|
||||
|
||||
@@ -8,6 +8,8 @@ function NodeBadge({ children, tone = "gray" }) {
|
||||
blue: "border-blue-200 bg-blue-50 text-blue-700",
|
||||
green: "border-green-200 bg-green-50 text-green-700",
|
||||
yellow: "border-yellow-200 bg-yellow-50 text-yellow-700",
|
||||
red: "border-red-200 bg-red-50 text-red-700",
|
||||
purple: "border-purple-200 bg-purple-50 text-purple-700",
|
||||
};
|
||||
|
||||
return (
|
||||
@@ -17,7 +19,13 @@ function NodeBadge({ children, tone = "gray" }) {
|
||||
);
|
||||
}
|
||||
|
||||
function NodeGroup({ title, nodes }) {
|
||||
function NodeGroup({
|
||||
title,
|
||||
nodes,
|
||||
resolvedNodeIds = new Set(),
|
||||
newlySurfacedNodeIds = new Set(),
|
||||
activeUnknownNodeId = null,
|
||||
}) {
|
||||
if (!nodes?.length) return null;
|
||||
|
||||
return (
|
||||
@@ -32,6 +40,20 @@ function NodeGroup({ title, nodes }) {
|
||||
<span className="font-medium text-gray-900">{node.label}</span>
|
||||
<NodeBadge tone="blue">{node.status}</NodeBadge>
|
||||
<NodeBadge tone="green">{node.confidence}</NodeBadge>
|
||||
{node.confidenceAssessment?.completenessStatus && (
|
||||
<NodeBadge tone="purple">
|
||||
completeness: {node.confidenceAssessment.completenessStatus}
|
||||
</NodeBadge>
|
||||
)}
|
||||
{resolvedNodeIds.has(node.id) && (
|
||||
<NodeBadge tone="red">resolved unknown</NodeBadge>
|
||||
)}
|
||||
{newlySurfacedNodeIds.has(node.id) && (
|
||||
<NodeBadge tone="purple">newly surfaced unknown</NodeBadge>
|
||||
)}
|
||||
{activeUnknownNodeId === node.id && (
|
||||
<NodeBadge tone="yellow">active unknown</NodeBadge>
|
||||
)}
|
||||
{node.value != null && (
|
||||
<NodeBadge tone="yellow">
|
||||
{node.value}
|
||||
@@ -42,6 +64,12 @@ function NodeGroup({ title, nodes }) {
|
||||
{node.description && node.description !== node.label && (
|
||||
<p className="mt-1 text-gray-600">{node.description}</p>
|
||||
)}
|
||||
{node.confidenceAssessment && (
|
||||
<p className="mt-1 text-xs text-gray-500">
|
||||
evidence: {node.confidenceAssessment.evidenceConfidence} ·
|
||||
conclusion: {node.confidenceAssessment.conclusionConfidence}
|
||||
</p>
|
||||
)}
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
@@ -52,6 +80,7 @@ function NodeGroup({ title, nodes }) {
|
||||
export default function SituationGraphView({
|
||||
situationGraph,
|
||||
selectedQuestion,
|
||||
newlySurfacedNodeIds = [],
|
||||
}) {
|
||||
if (!situationGraph) return null;
|
||||
|
||||
@@ -70,6 +99,9 @@ export default function SituationGraphView({
|
||||
return acc;
|
||||
}, {});
|
||||
|
||||
const resolvedNodeIdSet = new Set(situationGraph.resolvedNodeIds || []);
|
||||
const newlySurfacedNodeIdSet = new Set(newlySurfacedNodeIds || []);
|
||||
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
{selectedQuestionText && (
|
||||
@@ -110,6 +142,9 @@ export default function SituationGraphView({
|
||||
key={kind}
|
||||
title={kind.replace(/_/g, " ")}
|
||||
nodes={nodes}
|
||||
resolvedNodeIds={resolvedNodeIdSet}
|
||||
newlySurfacedNodeIds={newlySurfacedNodeIdSet}
|
||||
activeUnknownNodeId={situationGraph.activeUnknownNodeId}
|
||||
/>
|
||||
))}
|
||||
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
# v0.5 Question Priority Generalisation
|
||||
|
||||
## Hypothesis
|
||||
|
||||
The current deterministic unknown selector and graph-context question formulator should generalise across several decision types by selecting a foundational unknown before downstream implementation or pricing leaves.
|
||||
|
||||
## Scenarios
|
||||
|
||||
1. Should we hire another engineer?
|
||||
2. Should we replace the delivery vans?
|
||||
3. Should we launch in another country?
|
||||
4. Should we continue a project that is over budget?
|
||||
5. Should we introduce a paid support tier?
|
||||
|
||||
## Results
|
||||
|
||||
| Scenario | Selected unknown | Strategy | Pass/Fail |
|
||||
| ---------------------------- | --------------------------- | -------------------- | --------- |
|
||||
| Hire another engineer | `hire-success-criteria` | `decision criterion` | Pass |
|
||||
| Replace the delivery vans | `van-reliability-threshold` | `decision criterion` | Pass |
|
||||
| Launch in another country | `country-value-threshold` | `actor/customer` | Pass |
|
||||
| Continue over-budget project | `project-benefit-threshold` | `decision criterion` | Pass |
|
||||
| Introduce paid support tier | `support-value-threshold` | `actor/customer` | Pass |
|
||||
|
||||
## Repeated failure patterns
|
||||
|
||||
Two repeated structural formulation failures appeared before the final pass:
|
||||
|
||||
1. **Constraint language in surrounding graph context outranked node-local decision-threshold language** in more than one case.
|
||||
2. **Baseline language in surrounding graph context outranked node-local threshold language** in more than one case.
|
||||
|
||||
Both failures affected formulation strategy, not deterministic unknown selection.
|
||||
|
||||
## Code change made
|
||||
|
||||
A small deterministic change was made in `lib/graph/question-formulator.js`:
|
||||
|
||||
- prefer node-local `definition` language before broader criterion inference
|
||||
- prefer node-local `decision criterion` language before context-only `constraint` inference
|
||||
- only treat `baseline` or `constraint` as primary when the selected node itself carries that language, otherwise allow them as fallback strategies later
|
||||
|
||||
No architecture, UI, persistence, prompt, scoring, additional model turns, or provider calls were added.
|
||||
|
||||
## Remaining limitations
|
||||
|
||||
- In two passing cases, the selector chose a threshold-style foundational node while the formulator still used an `actor/customer` strategy because related context strongly referenced customers or recipients.
|
||||
- This experiment is fixture-driven and deterministic; it is useful for regression protection, not scientific validation.
|
||||
- The suite exercises the production path without model calls, but it does not prove behaviour over arbitrary real-world graph structures.
|
||||
@@ -0,0 +1,58 @@
|
||||
# v0.5 Release Notes
|
||||
|
||||
## Purpose of v0.5
|
||||
|
||||
v0.5 stabilises the graph-backed one-turn update flow so the engine can resolve an answered unknown, surface consequential new unknowns, prioritise the next unknown deterministically, and formulate a deterministic follow-up question without changing the UI or adding more model turns.
|
||||
|
||||
## Capabilities proven
|
||||
|
||||
v0.5 includes:
|
||||
|
||||
- resolving an existing unknown
|
||||
- surfacing consequential new unknowns
|
||||
- limiting emergent unknowns
|
||||
- deterministic information-value prioritisation
|
||||
- deterministic question formulation
|
||||
- generalisation across five decision types
|
||||
- graph-backed one-turn UI update
|
||||
|
||||
## Five-case generalisation result
|
||||
|
||||
All five deterministic fixture scenarios passed:
|
||||
|
||||
1. Should we hire another engineer?
|
||||
2. Should we replace the delivery vans?
|
||||
3. Should we launch in another country?
|
||||
4. Should we continue a project that is over budget?
|
||||
5. Should we introduce a paid support tier?
|
||||
|
||||
The selector chose a foundational unknown first in each case, avoided the downstream leaf first, required no model call, and preserved graph immutability during question formulation.
|
||||
|
||||
## Key deterministic safeguards
|
||||
|
||||
- proposal application re-selects the active unknown deterministically after validation
|
||||
- information-value scoring penalises downstream or prerequisite-blocked unknowns
|
||||
- emergent unknown validation limits additions and requires explicit answer-derived linkage
|
||||
- final question wording is reformulated from graph context without an extra model turn
|
||||
- question validation rejects compound, awkward, or pricing-led fallback phrasing
|
||||
|
||||
## Known limitation
|
||||
|
||||
A correctly selected threshold node can still be phrased using an actor/customer strategy when surrounding graph context strongly references customers or value recipients.
|
||||
|
||||
This limitation is recorded for the next experiment and is not being fixed in the v0.5 release-prep task.
|
||||
|
||||
## Deliberately excluded work
|
||||
|
||||
- no reasoning-logic expansion beyond the small deterministic formulation fixes already landed on the branch
|
||||
- no new features
|
||||
- no UI changes
|
||||
- no persistence
|
||||
- no additional model turn
|
||||
- no Ollama calls for validation
|
||||
- no evaluator-suite runs
|
||||
- no Playwright runs
|
||||
|
||||
## Next experimental question
|
||||
|
||||
Can the question formulation strategy remain aligned with the selected node's role when surrounding graph context contains competing signals?
|
||||
@@ -0,0 +1,40 @@
|
||||
# v0.6 Ambiguity Generalisation
|
||||
|
||||
## Hypothesis
|
||||
|
||||
If the selector truly handles unjustified contradiction ties generically, it should return ambiguity across multiple domains without preferring one explanation by wording alone.
|
||||
|
||||
## Scenarios
|
||||
|
||||
1. Revenue increased by 18%, but cash in the bank fell over the same period.
|
||||
2. Customer satisfaction scores increased, but complaints also increased.
|
||||
3. Average delivery time decreased by 25%, but order cancellations increased.
|
||||
4. Website traffic doubled, but sales remained unchanged.
|
||||
5. Production output increased by 30%, but quality defects also increased.
|
||||
|
||||
## Observed behaviour
|
||||
|
||||
All five fixtures produced the same pattern:
|
||||
|
||||
- candidate count: 2
|
||||
- selector status: `ambiguous`
|
||||
- tie reason: `No justified distinction between leading unknowns.`
|
||||
- no explanation was favoured
|
||||
- one broad investigation question was produced from the central contradiction
|
||||
- neutral label renaming did not collapse ambiguity into a winner
|
||||
|
||||
## Repeated failure patterns
|
||||
|
||||
None observed across two or more scenarios.
|
||||
|
||||
The current ambiguity handling generalised cleanly across the five contradiction fixtures.
|
||||
|
||||
## Corrections
|
||||
|
||||
No production correction was required in this task.
|
||||
|
||||
## Lessons learned
|
||||
|
||||
- The current ambiguity path appears domain-agnostic when structure and semantic weights remain intentionally non-discriminating.
|
||||
- Central-statement-based tie questions are broad enough to avoid prematurely backing one branch.
|
||||
- The most useful regression signal is whether ambiguity survives neutral relabelling, not whether one label sorts ahead of another in display order.
|
||||
@@ -0,0 +1,211 @@
|
||||
# v0.6 Atomicity Experiment
|
||||
|
||||
## Hypothesis
|
||||
|
||||
After deterministic unknown selection, the engine should assess whether the selected unknown is already atomic or is still too composite to ask directly.
|
||||
|
||||
If the unknown is atomic, the engine should proceed exactly as before.
|
||||
|
||||
If the unknown is composite, the engine should not ask that parent unknown directly. Instead, it should decompose it into a small set of explicit child unknowns representing broad, independent candidate dimensions that a non-expert could understand.
|
||||
|
||||
## Constraints
|
||||
|
||||
- No graph redesign
|
||||
- No persistence
|
||||
- No UI redesign
|
||||
- No selection-weight tuning
|
||||
- No Ollama calls in unit tests
|
||||
|
||||
## Deterministic rule introduced
|
||||
|
||||
Atomicity assessment is **not** a new investigation strategy.
|
||||
|
||||
It runs in the graph update path at this seam:
|
||||
|
||||
```text
|
||||
unknown selection -> atomicity assessment -> optional decomposition -> deterministic reselection -> question formulation
|
||||
```
|
||||
|
||||
The implementation uses deterministic text and graph-shape checks:
|
||||
|
||||
- focused unknowns like denominator / threshold / definition / baseline / evidence remain **atomic**
|
||||
- broad relationship-explanation unknowns and broad “possible causes / what changed / explanation for why X but Y” unknowns become **composite**
|
||||
|
||||
## Decomposition behavior
|
||||
|
||||
When a selected unknown is composite:
|
||||
|
||||
1. The parent unknown remains unresolved.
|
||||
2. Between 2 and 5 child unknowns are created or reused deterministically.
|
||||
3. Children become explicit graph nodes.
|
||||
4. Children link back to the parent with existing `depends_on` edges.
|
||||
5. Children inherit the same “why it matters” discipline in their descriptions.
|
||||
6. Deterministic selection reruns across the updated graph.
|
||||
|
||||
For the current relationship-explanation experiment, the broad child dimensions are:
|
||||
|
||||
- Whether the two observations reflect different timing
|
||||
- How the two observations were measured
|
||||
- Change affecting signal A more than signal B
|
||||
- Change affecting signal B more than signal A
|
||||
- One-off event during the period
|
||||
|
||||
These are intentionally non-jargon and broad enough to generalise across scenarios like:
|
||||
|
||||
- Revenue up / Cash down
|
||||
- Customer satisfaction up / Complaints up
|
||||
- Delivery time down / Cancellations up
|
||||
- Traffic up / Sales flat
|
||||
- Production up / Defects up
|
||||
|
||||
## Diagnostics added
|
||||
|
||||
The orchestrator now reports:
|
||||
|
||||
- `atomicityAssessment`
|
||||
- `atomicityDecisionReason`
|
||||
- `decompositionDepth`
|
||||
- `decompositionAttempted`
|
||||
- `decompositionAccepted`
|
||||
- `decompositionStoppedReason`
|
||||
- `proposedChildCount`
|
||||
- `acceptedChildCount`
|
||||
- `rejectedChildren`
|
||||
- `selectedChildNodeId`
|
||||
- `childQualitySummary`
|
||||
- `propagationPerformed`
|
||||
- `resolvedChildNodeId`
|
||||
- `parentNodeId`
|
||||
- `parentStatusBefore`
|
||||
- `parentStatusAfter`
|
||||
- `parentConfidenceBefore`
|
||||
- `parentConfidenceAfter`
|
||||
- `affectedAncestorIds`
|
||||
- `nextSelectedSibling`
|
||||
- `parentResolved`
|
||||
- `decompositionPerformed`
|
||||
- `childUnknownCount`
|
||||
- `childNodeIds`
|
||||
- `atomicityReason`
|
||||
|
||||
This sits alongside the existing explicit-emergent-unknown diagnostics.
|
||||
|
||||
## Observed outcome
|
||||
|
||||
The experiment was useful.
|
||||
|
||||
Before this change, the engine could select a broad explanation unknown and ask it directly.
|
||||
|
||||
After this change:
|
||||
|
||||
- the broad explanation parent remains explicit in the graph
|
||||
- the engine decomposes it into child unknowns first
|
||||
- the next asked question is backed by a more focused child unknown
|
||||
- repeated updates reuse the same decomposition children deterministically
|
||||
- child-quality checks reject compound or duplicate children before they enter the graph
|
||||
- decomposition stops deterministically once a selected child is directly answerable
|
||||
- resolving one child does not resolve the parent immediately
|
||||
- resolved child evidence now propagates upward to the parent and ancestor chain deterministically
|
||||
- parent status and confidence change conservatively after child resolution
|
||||
- the next sibling becomes eligible for normal deterministic selection without recreating the resolved child
|
||||
|
||||
In the revenue-versus-cash case, the selected next question becomes:
|
||||
|
||||
> What evidence would clarify how the two observations were measured?
|
||||
|
||||
rather than asking the full broad explanation node directly.
|
||||
|
||||
## Upward propagation and reconstruction
|
||||
|
||||
Recursive reasoning is complete only when decomposition and reconstruction are both deterministic.
|
||||
|
||||
Confidence must not outrun completeness or evidence.
|
||||
|
||||
For this experiment, reconstruction now behaves as follows:
|
||||
|
||||
- when a child unknown resolves, that child keeps its own resolved status and answer evidence
|
||||
- the parent is updated, but remains unresolved unless the deterministic completion rule is satisfied
|
||||
- only the ancestor chain connected to that child is updated
|
||||
- unrelated branches remain unchanged
|
||||
- the deterministic selector then chooses the next justified unresolved sibling or related follow-up
|
||||
|
||||
For the current conservative completion rule:
|
||||
|
||||
- **one resolved child** → parent becomes `provisional` with higher confidence, but remains unresolved
|
||||
- **all direct child unknowns resolved** → parent resolves deterministically with `high` confidence
|
||||
|
||||
The confidence model is now explicitly separated into:
|
||||
|
||||
- **evidence confidence**: how trustworthy the currently attached support is
|
||||
- **completeness**: whether the required direct child structure is empty, partial, or complete
|
||||
- **conclusion confidence**: how strongly the current parent state is justified given both evidence and completeness
|
||||
|
||||
Deterministic propagation rules now enforce:
|
||||
|
||||
- one resolved child may raise evidence confidence
|
||||
- unresolved direct children cap conclusion confidence
|
||||
- contradictory direct children block high conclusion confidence
|
||||
- duplicate evidence does not increase confidence
|
||||
- status changes do not raise confidence on their own
|
||||
- parent resolution still requires the separate completion rule
|
||||
|
||||
## Cross-branch corroboration
|
||||
|
||||
The next confidence experiment adds deterministic branch interaction checks without changing the graph model.
|
||||
|
||||
The engine now distinguishes between:
|
||||
|
||||
- **multiple evidence**: more than one branch exists
|
||||
- **independent corroboration**: distinct resolved branches support the same parent without sharing the same evidence key
|
||||
- **duplicate evidence**: the same evidence key appears through multiple branches and must not be double-counted
|
||||
- **conflicting evidence**: branches support incompatible positions, such as `recognised correctly` vs `recognised incorrectly`
|
||||
|
||||
Deterministic branch rules:
|
||||
|
||||
- corroboration only counts when branches are distinct and their evidence sources differ
|
||||
- duplicate evidence groups never count as corroboration
|
||||
- conflicts cap conclusion confidence and prevent a higher confidence upgrade
|
||||
- independent branches remain interaction-neutral
|
||||
|
||||
Additional diagnostics now expose:
|
||||
|
||||
- `corroboratingBranchCount`
|
||||
- `conflictingBranchCount`
|
||||
- `duplicateEvidenceCount`
|
||||
- `independentBranchCount`
|
||||
- `interactionSummary`
|
||||
- `confidenceAdjustmentReason`
|
||||
|
||||
Observed effect:
|
||||
|
||||
- independent corroboration can raise `evidenceConfidence`
|
||||
- duplicate evidence produces no extra confidence increase
|
||||
- conflicting evidence lowers or caps `conclusionConfidence`
|
||||
- completeness rules still dominate whether a parent may become highly justified
|
||||
|
||||
Example progression:
|
||||
|
||||
- parent before: `unknown`, `medium`
|
||||
- after resolving `How the two observations were measured`: parent becomes `provisional`, `medium`
|
||||
- evidence confidence becomes `high`, completeness becomes `partial`, conclusion confidence becomes `medium`
|
||||
- next sibling becomes selectable and the engine moves on without recreating the resolved child
|
||||
|
||||
## Interpretation
|
||||
|
||||
This supports the idea that recursive decomposition is a fundamental part of graph-backed questioning, not just a prompt refinement.
|
||||
|
||||
The main remaining limitation is that sibling selection still inherits the existing deterministic scorer. That means some domains may advance to a justified sibling that is not the intuitively expected next child, even though the propagation itself remains deterministic and graph-valid.
|
||||
|
||||
## Validation run
|
||||
|
||||
Covered by:
|
||||
|
||||
- `tests/graph/atomicity-assessment.test.js`
|
||||
- `tests/graph/decomposition-quality.test.js`
|
||||
- `tests/graph/upward-propagation.test.js`
|
||||
- `tests/graph/apply-proposal.test.js`
|
||||
- `tests/graph/orchestrator.test.js`
|
||||
- `tests/graph/question-formulator.test.js`
|
||||
- `tests/ui/scenario-form.test.jsx`
|
||||
|
||||
And then by the broader requested validation pass with lint and build.
|
||||
@@ -0,0 +1,48 @@
|
||||
# v0.6 Comparability Experiment
|
||||
|
||||
## Hypothesis
|
||||
|
||||
The engine should confirm that observations are comparable before treating their difference as a contradiction that needs explanatory follow-up.
|
||||
|
||||
## Fixtures
|
||||
|
||||
1. Revenue increased by 18%, but cash in the bank fell over the same period.
|
||||
2. Complaints increased. Production increased.
|
||||
3. Average delivery time decreased by 25%, but order cancellations increased.
|
||||
4. Customer satisfaction increased, but complaints increased.
|
||||
5. Temperature increased. Ice melted.
|
||||
6. Sales doubled. Sales doubled.
|
||||
|
||||
## Results
|
||||
|
||||
- The first four scenarios repeated the same failure pattern: contradiction-level investigation could begin before comparability was established.
|
||||
- A deterministic comparability gate corrected that by producing one comparison question first.
|
||||
- Confirmed comparability did not by itself imply contradiction.
|
||||
- Temperature increased / Ice melted was reclassified as a compatible relationship, so no contradiction question was asked.
|
||||
- Sales doubled / Sales doubled was reclassified as duplicate observations, so no follow-up question was asked.
|
||||
|
||||
## Relationship classification stage
|
||||
|
||||
After comparability assessment, observations now pass through a deterministic relationship classification stage:
|
||||
|
||||
- `contradictory`
|
||||
- `compatible`
|
||||
- `potentially_related`
|
||||
- `duplicate`
|
||||
- `insufficient_information`
|
||||
|
||||
## Whether comparability should become a permanent reasoning stage
|
||||
|
||||
Yes, in minimal deterministic form.
|
||||
|
||||
The repeated pattern appeared in four scenarios, so a small pre-contradiction comparability assessment is justified.
|
||||
|
||||
## Two-step experiment result
|
||||
|
||||
A comparison question is useful only if its answer advances the reasoning stage rather than merely adding more text.
|
||||
|
||||
In the revenue-versus-cash scenario, the first question now confirms whether the figures are comparable, and the answer resolves that existing uncertainty instead of creating a parallel note. After that update, the engine progresses from comparability assessment to cautious relationship assessment and can select one broad non-expert follow-up question.
|
||||
|
||||
Every justified next question should correspond to an explicit unresolved graph node.
|
||||
|
||||
The earlier fallback-only path has now been removed from the normal successful progression. After comparability is resolved and a further investigation question is justified, the engine creates or reuses an explicit unresolved reasoning unknown and lets deterministic selection and question formulation proceed through the standard graph pipeline. A fallback is now only acceptable as an explicit failure case, not as the normal source of the next question.
|
||||
@@ -0,0 +1,375 @@
|
||||
# v0.6 Reasoning Architecture
|
||||
|
||||
## Purpose
|
||||
|
||||
This document describes the implemented deterministic reasoning architecture on branch `feature/question-strategy-alignment-v0.6`.
|
||||
|
||||
It is written for future developers who need to understand how v0.6 actually executes, what invariants it relies on, where the recursive loops are, and what the system deliberately does **not** attempt to do.
|
||||
|
||||
## End-to-end pipeline
|
||||
|
||||
The implemented runtime pipeline is:
|
||||
|
||||
```text
|
||||
Scenario input
|
||||
↓
|
||||
LLM analysis / reconstruction
|
||||
↓
|
||||
Initial graph build
|
||||
↓
|
||||
Deterministic unknown selection
|
||||
↓
|
||||
Selected question
|
||||
↓
|
||||
User answer
|
||||
↓
|
||||
LLM graph-update proposal
|
||||
↓
|
||||
Proposal parsing / normalisation
|
||||
↓
|
||||
Proposal compatibility validation
|
||||
↓
|
||||
Deterministic graph update application
|
||||
↓
|
||||
Reasoning-state rebuild
|
||||
↓
|
||||
Comparability assessment
|
||||
↓
|
||||
Relationship classification
|
||||
↓
|
||||
Explicit emergent unknown creation / reuse (if required)
|
||||
↓
|
||||
Deterministic reselection
|
||||
↓
|
||||
Atomicity assessment
|
||||
↓
|
||||
Optional decomposition into child unknowns
|
||||
↓
|
||||
Deterministic reselection
|
||||
↓
|
||||
Resolved-child propagation upward
|
||||
↓
|
||||
Confidence / completeness / corroboration update
|
||||
↓
|
||||
Next active unknown
|
||||
↓
|
||||
Question formulation
|
||||
```
|
||||
|
||||
## Deterministic stages
|
||||
|
||||
### 1. Scenario analysis / reconstruction
|
||||
|
||||
- **Purpose**: obtain structured reconstruction material from scenario text
|
||||
- **Input**: scenario, prompt version
|
||||
- **Output**: analysis payload containing reconstruction, evidence, diagnostics, and optional next question
|
||||
- **Why it exists**: provides the initial structured substrate from which the graph is built
|
||||
- **What breaks if removed**: the graph builder has no structured reconstruction to convert into nodes and edges
|
||||
|
||||
### 2. Initial graph build
|
||||
|
||||
- **Purpose**: convert reconstruction output into an initial `SituationGraph`
|
||||
- **Input**: reconstruction + evidence
|
||||
- **Output**: graph nodes and edges, then `makeGraph(...)` wraps them with active/resolved/summary state
|
||||
- **Why it exists**: all later reasoning is graph-based, not free text
|
||||
- **What breaks if removed**: no explicit unknown nodes, no deterministic selection, no validated update loop
|
||||
|
||||
### 3. Deterministic unknown selection
|
||||
|
||||
- **Purpose**: choose the next active unknown from unresolved graph nodes
|
||||
- **Input**: graph, resolved node IDs
|
||||
- **Output**: selected candidate or explicit ambiguity result
|
||||
- **Why it exists**: the system needs a deterministic next investigation target
|
||||
- **What breaks if removed**: question ordering becomes arbitrary or hidden in prompts
|
||||
|
||||
### 4. Selected question exposure
|
||||
|
||||
- **Purpose**: expose the chosen unknown as the next question to the user
|
||||
- **Input**: selected unknown + question formulation or tie-resolution logic
|
||||
- **Output**: selected question object
|
||||
- **Why it exists**: the user-facing loop must ask a concrete next question
|
||||
- **What breaks if removed**: the system can build a graph but cannot continue interaction coherently
|
||||
|
||||
### 5. LLM graph-update proposal
|
||||
|
||||
- **Purpose**: transform a user answer into a proposed graph change set
|
||||
- **Input**: current graph, previous question, answer, prompt version
|
||||
- **Output**: raw JSON-like proposal
|
||||
- **Why it exists**: the LLM is limited to proposing changes; it does not mutate the graph directly
|
||||
- **What breaks if removed**: answers cannot affect the graph except through manual hard-coded logic
|
||||
|
||||
### 6. Proposal parsing / normalisation
|
||||
|
||||
- **Purpose**: parse JSON, remove null array items, apply known aliases, fill omitted optional fields
|
||||
- **Input**: raw model response
|
||||
- **Output**: validated `graphUpdateSchema` payload or structured parser failure
|
||||
- **Why it exists**: model outputs are not trusted as-is
|
||||
- **What breaks if removed**: malformed or partially missing model output would reach graph logic directly
|
||||
|
||||
### 7. Proposal compatibility validation
|
||||
|
||||
- **Purpose**: ensure the proposal is graph-safe and semantically valid before application
|
||||
- **Input**: current graph + proposed update
|
||||
- **Output**: accepted proposal or compatibility errors
|
||||
- **Why it exists**: protects graph integrity and reasoning invariants
|
||||
- **What breaks if removed**: duplicate IDs, missing references, fake selected questions, and no-op updates could corrupt the graph
|
||||
|
||||
### 8. Deterministic graph update application
|
||||
|
||||
- **Purpose**: apply only validated graph changes to a copied graph
|
||||
- **Input**: graph + validated proposal
|
||||
- **Output**: updated nodes, edges, resolved node IDs
|
||||
- **Why it exists**: separates safe application from generation
|
||||
- **What breaks if removed**: no explicit, replayable state transition exists
|
||||
|
||||
### 9. Reasoning-state rebuild
|
||||
|
||||
- **Purpose**: derive fresh comparability/relationship state from the updated graph
|
||||
- **Input**: updated graph + optional override state
|
||||
- **Output**: `reasoningState`
|
||||
- **Why it exists**: reasoning stages are derived from graph state, not stored blindly
|
||||
- **What breaks if removed**: comparability and relationship decisions drift from actual graph contents
|
||||
|
||||
### 10. Comparability assessment
|
||||
|
||||
- **Purpose**: decide whether supported observations are comparable enough for relationship reasoning
|
||||
- **Input**: graph observations + central statement + optional stored override
|
||||
- **Output**: comparability status/reason + contradiction permission
|
||||
- **Why it exists**: relationship reasoning is gated by comparability
|
||||
- **What breaks if removed**: contradiction or relationship reasoning would run over incomparable observations
|
||||
|
||||
### 11. Relationship classification
|
||||
|
||||
- **Purpose**: classify observation relationships once comparability permits it
|
||||
- **Input**: graph + comparability result
|
||||
- **Output**: relationship status, reason, whether a follow-up question is justified
|
||||
- **Why it exists**: determines whether explanation-style follow-up is needed
|
||||
- **What breaks if removed**: the system cannot distinguish compatible, duplicate, insufficient, and contradiction-adjacent observation sets
|
||||
|
||||
### 12. Explicit emergent unknown creation / reuse
|
||||
|
||||
- **Purpose**: ensure any justified relationship follow-up is represented by an explicit unresolved graph node
|
||||
- **Input**: provisional graph + relationship assessment
|
||||
- **Output**: reused or newly added explanation unknown and edges
|
||||
- **Why it exists**: preserves the invariant that a question must originate from an explicit unknown
|
||||
- **What breaks if removed**: relationship follow-up would revert to fallback-only question text not backed by the graph
|
||||
|
||||
### 13. Atomicity assessment
|
||||
|
||||
- **Purpose**: determine whether the selected unknown is directly investigable or too composite
|
||||
- **Input**: selected unknown + graph context
|
||||
- **Output**: `atomic` or `composite` decision with decomposition kind/reason
|
||||
- **Why it exists**: prevents asking broad explanation unknowns directly
|
||||
- **What breaks if removed**: the system asks high-level composite unknowns instead of decomposing them first
|
||||
|
||||
### 14. Optional decomposition
|
||||
|
||||
- **Purpose**: split a composite unknown into deterministic child unknowns
|
||||
- **Input**: composite selected unknown + graph context
|
||||
- **Output**: 2–5 child unknowns, edges, quality summary, rejection diagnostics
|
||||
- **Why it exists**: narrows broad unknowns into explicit candidate dimensions
|
||||
- **What breaks if removed**: recursive reasoning stops at broad parents and loses graph-backed substructure
|
||||
|
||||
### 15. Resolved-child propagation upward
|
||||
|
||||
- **Purpose**: move resolved child effects to parent and ancestor chain without prematurely resolving them
|
||||
- **Input**: updated graph + proposal snapshot
|
||||
- **Output**: parent/ancestor status and confidence updates, additional diagnostics
|
||||
- **Why it exists**: decomposition requires deterministic reconstruction as well as decomposition
|
||||
- **What breaks if removed**: child answers stay local and parents never become progressively better-supported
|
||||
|
||||
### 16. Confidence / completeness / corroboration update
|
||||
|
||||
- **Purpose**: derive parent-level `confidenceAssessment` from resolved children and branch interactions
|
||||
- **Input**: parent child set + branch evidence/status interactions
|
||||
- **Output**: `evidenceConfidence`, `completenessStatus`, `conclusionConfidence`, plus derived display `confidence`
|
||||
- **Why it exists**: reasoning support must be separated from completion and contradiction state
|
||||
- **What breaks if removed**: parent confidence collapses back into vague status-driven heuristics
|
||||
|
||||
### 17. Next active unknown + question formulation
|
||||
|
||||
- **Purpose**: reselect the next unresolved unknown and formulate a concrete next question
|
||||
- **Input**: updated graph + selection state + graph context
|
||||
- **Output**: next active unknown and question object
|
||||
- **Why it exists**: closes the recursive interaction loop
|
||||
- **What breaks if removed**: the system updates the graph but cannot continue investigation deterministically
|
||||
|
||||
## Architectural invariants
|
||||
|
||||
The current implementation enforces these invariants:
|
||||
|
||||
1. **A question must originate from an explicit unresolved unknown node.**
|
||||
2. **Unknown selection is deterministic.**
|
||||
3. **Alphabetical ordering is not treated as reasoning.**
|
||||
4. **Relationship reasoning does not precede comparability.**
|
||||
5. **The LLM never mutates the graph directly; it only proposes updates.**
|
||||
6. **All graph updates are schema-validated before application.**
|
||||
7. **All node/edge references must resolve to existing nodes.**
|
||||
8. **Duplicate node IDs are rejected.**
|
||||
9. **Duplicate added edge IDs are rejected.**
|
||||
10. **A selected question cannot target a resolved unknown.**
|
||||
11. **A resolved unknown updated to `resolved` must also appear in `resolvedUnknownNodeIds`.**
|
||||
12. **A proposal must contain a meaningful change.**
|
||||
13. **Every newly added unknown must include why-it-matters language.**
|
||||
14. **Every newly added unknown must be explicitly connected to answer-derived graph structure.**
|
||||
15. **Composite selected unknowns are decomposed before direct questioning when atomicity rules require it.**
|
||||
16. **Parent unknowns remain unresolved until completion rules are satisfied.**
|
||||
17. **Confidence must not outrun completeness.**
|
||||
18. **Duplicate evidence cannot increase confidence.**
|
||||
19. **Conflicting evidence caps conclusion confidence.**
|
||||
20. **Cross-branch corroboration only counts for distinct branches with distinct evidence keys.**
|
||||
21. **Ambiguous leading unknowns remain explicit ambiguity, not silent forced choice.**
|
||||
|
||||
## Recursive loops and stopping rules
|
||||
|
||||
### Main investigation loop
|
||||
|
||||
```text
|
||||
Unknown
|
||||
↓
|
||||
Question
|
||||
↓
|
||||
Answer
|
||||
↓
|
||||
Proposal
|
||||
↓
|
||||
Graph update
|
||||
↓
|
||||
Propagation
|
||||
↓
|
||||
Next unknown
|
||||
```
|
||||
|
||||
- **Exit condition**: no unresolved candidates remain, or no next question is justified, or proposal/application fails
|
||||
- **Stopping rule**: deterministic selection returns `null` or explicit ambiguity, or update validation blocks progress
|
||||
- **Completion behaviour**: continues only while the graph contains justified unresolved unknowns
|
||||
|
||||
### Decomposition loop
|
||||
|
||||
```text
|
||||
Selected unknown
|
||||
↓
|
||||
Atomicity assessment
|
||||
↓
|
||||
If composite: decompose
|
||||
↓
|
||||
Reselect child
|
||||
↓
|
||||
Atomicity assessment again
|
||||
```
|
||||
|
||||
- **Exit condition**: selected child is atomic; parent already has children; max decomposition depth reached; or decomposition quality fails
|
||||
- **Stopping rule**: `MAX_DECOMPOSITION_DEPTH`, reuse instead of regeneration, or inability to produce enough valid child unknowns
|
||||
- **Completion behaviour**: deterministic and bounded; no infinite recursive decomposition path is intentionally allowed
|
||||
|
||||
### Propagation loop
|
||||
|
||||
```text
|
||||
Resolved child
|
||||
↓
|
||||
Ancestor chain walk
|
||||
↓
|
||||
Recompute parent state
|
||||
↓
|
||||
Stop when no ancestor state changes
|
||||
```
|
||||
|
||||
- **Exit condition**: no more parents in the ancestor chain or no state change
|
||||
- **Stopping rule**: ancestor chain is explicit and finite; propagation does not invent new ancestors
|
||||
- **Completion behaviour**: deterministic upward traversal with explicit stop on unchanged state
|
||||
|
||||
### Potential infinite loops reviewed
|
||||
|
||||
- **Unknown/question recursion**: bounded by unresolved unknown set, proposal validation, and explicit no-candidate states
|
||||
- **Decomposition recursion**: bounded by max depth and child reuse rules
|
||||
- **Propagation recursion**: bounded by finite ancestor chain and no-change stop condition
|
||||
|
||||
No intentional infinite reasoning loop is present in the implemented architecture.
|
||||
|
||||
## Graph lifecycle summary
|
||||
|
||||
### Node lifecycle
|
||||
|
||||
1. node created by `buildInitialGraph` or later proposal/decomposition/emergent-unknown logic
|
||||
2. node validated by schema
|
||||
3. node may become active unknown
|
||||
4. node may be updated by proposal application
|
||||
5. unknown node may become `resolved`, `provisional`, `contradicted`, or remain `unknown`
|
||||
6. resolved unknown ID is tracked in `resolvedNodeIds`
|
||||
|
||||
### Edge lifecycle
|
||||
|
||||
1. edge created in initial graph or by deterministic proposal augmentation
|
||||
2. edge validated against existing node IDs
|
||||
3. edge may be removed only through explicit `removedEdgeIds`
|
||||
4. edge relationships also update `dependsOn` / `childIds` projections during application
|
||||
|
||||
### Unknown lifecycle
|
||||
|
||||
1. initial unknown discovered from reconstruction
|
||||
2. selected deterministically or left ambiguous
|
||||
3. may be decomposed if composite
|
||||
4. may be resolved directly by answer
|
||||
5. may cause emergent reasoning unknown creation when relationship reasoning demands a new explicit question target
|
||||
|
||||
### Resolved lifecycle
|
||||
|
||||
1. proposal marks unresolved unknown resolved
|
||||
2. reconciliation ensures resolution semantics are explicit
|
||||
3. `resolvedUnknownNodeIds` feed graph application
|
||||
4. propagation may resolve parent only when completion rule is met
|
||||
|
||||
### Confidence lifecycle
|
||||
|
||||
1. nodes begin with base `confidence`
|
||||
2. parent/ancestor propagation derives `confidenceAssessment`
|
||||
3. display `confidence` is derived from `conclusionConfidence`
|
||||
4. completeness, duplicate evidence, contradiction, and corroboration constrain the result
|
||||
|
||||
### Question lifecycle
|
||||
|
||||
1. selected unknown becomes question target
|
||||
2. `formulateQuestion` or tie-resolution logic produces question text
|
||||
3. answer returns through update route
|
||||
4. proposal may select a new question target or leave reselection to deterministic logic
|
||||
|
||||
Every major transition above is explicit in the current codebase rather than implicit in model text alone.
|
||||
|
||||
## Duplicated or overlapping concepts
|
||||
|
||||
The following concepts are intentionally close and may look duplicated:
|
||||
|
||||
- **status vs confidence**: status captures lifecycle/progression; confidence captures support strength
|
||||
- **confidence vs confidenceAssessment**: `confidence` is now a derived display field, while `confidenceAssessment` carries separated reasoning dimensions
|
||||
- **resolvedNodeIds vs node.status === resolved**: both are maintained; the first is a graph-level index, the second is node-local state
|
||||
- **selectedQuestion in proposal vs selectedQuestion in final result**: proposal may omit or propose one, final result recomputes deterministic selection/questioning after graph logic
|
||||
- **comparability state in reasoningState vs derived comparability from graph**: overrides may carry forward prior confirmed reasoning, but `buildReasoningState` still rebuilds from graph + override context
|
||||
|
||||
These are not necessarily defects, but they are the main places where future simplification pressure is likely.
|
||||
|
||||
## Known boundaries and deliberate exclusions
|
||||
|
||||
v0.6 deliberately does **not** attempt the following:
|
||||
|
||||
- probabilistic reasoning
|
||||
- Bayesian inference
|
||||
- persistence
|
||||
- semantic embeddings
|
||||
- fuzzy semantic similarity
|
||||
- autonomous exploration outside explicit user answers
|
||||
- multi-hop corroboration across unrelated subtrees without a shared direct parent
|
||||
- expert-only jargon-specific reasoning modes
|
||||
- UI-heavy reasoning visualisation beyond existing graph/update displays
|
||||
- arbitrary non-deterministic tie breaking
|
||||
|
||||
## Defects found during this review
|
||||
|
||||
No new production defect was intentionally introduced or fixed as part of this architecture review.
|
||||
|
||||
## Developer notes
|
||||
|
||||
- `startCase` owns reconstruction → graph build → first deterministic selection.
|
||||
- `updateCaseWithDependencies` owns proposal generation / parsing and delegates deterministic graph semantics to `applyValidatedProposal`.
|
||||
- `applyValidatedProposal` is the main reasoning pipeline coordinator for update-time graph evolution.
|
||||
- `question-formulator.js` owns comparability, relationship classification, atomicity assessment, investigation strategy selection, and question formulation.
|
||||
- `utils.js` owns selection scoring, ordering, graph validation, and safe graph update application.
|
||||
@@ -0,0 +1,89 @@
|
||||
# v0.6 Release Notes
|
||||
|
||||
## Purpose
|
||||
|
||||
v0.6 turns the engine into a deterministic recursive reasoning system that keeps next questions, decomposition, propagation, and confidence updates explicitly grounded in the situation graph.
|
||||
|
||||
## Capabilities added
|
||||
|
||||
- deterministic unknown selection explanations
|
||||
- explicit ambiguity handling instead of silent tie-breaking
|
||||
- comparability assessment before relationship reasoning
|
||||
- relationship classification after comparability
|
||||
- reasoning-stage progression after comparability answers
|
||||
- graph-backed next questions via explicit unknown nodes
|
||||
- investigation-strategy-based question formulation
|
||||
- atomicity assessment for selected unknowns
|
||||
- composite-unknown decomposition into child unknowns
|
||||
- child-quality validation for decomposition outputs
|
||||
- upward propagation from resolved children to parents and ancestors
|
||||
- separation of evidence confidence, completeness, and conclusion confidence
|
||||
- deterministic cross-branch corroboration, conflict, and duplicate-evidence handling
|
||||
- developer-facing reasoning architecture documentation
|
||||
|
||||
## Reasoning pipeline summary
|
||||
|
||||
```text
|
||||
Scenario
|
||||
→ Reconstruction
|
||||
→ Initial graph
|
||||
→ Deterministic unknown selection
|
||||
→ Question
|
||||
→ Answer
|
||||
→ Proposal
|
||||
→ Proposal parsing / validation
|
||||
→ Graph update
|
||||
→ Reasoning-state rebuild
|
||||
→ Comparability assessment
|
||||
→ Relationship classification
|
||||
→ Emergent unknown creation / reuse
|
||||
→ Atomicity assessment
|
||||
→ Optional decomposition
|
||||
→ Propagation
|
||||
→ Confidence / completeness / corroboration update
|
||||
→ Next active unknown
|
||||
→ Next question
|
||||
```
|
||||
|
||||
## Core invariants
|
||||
|
||||
- every asked question must originate from an explicit unresolved unknown
|
||||
- unknown selection is deterministic
|
||||
- ambiguity is preserved explicitly when no justified distinction exists
|
||||
- relationship reasoning cannot precede comparability
|
||||
- parent nodes cannot resolve before completion rules are met
|
||||
- confidence cannot outrun completeness
|
||||
- duplicate evidence cannot increase confidence
|
||||
- conflicting evidence caps conclusion confidence
|
||||
- cross-branch corroboration only counts for distinct branches with distinct evidence keys
|
||||
- the LLM proposes updates but does not mutate the graph directly
|
||||
|
||||
## What v0.6 proved
|
||||
|
||||
- graph-backed questioning works better when every justified next question maps to an explicit unresolved node
|
||||
- broad unknowns can be decomposed deterministically before direct questioning
|
||||
- resolved child evidence can be propagated upward without prematurely resolving parent reasoning
|
||||
- confidence becomes easier to reason about when evidence quality, completeness, and conclusion strength are separated
|
||||
- deterministic cross-branch corroboration can improve support without double-counting repeated evidence
|
||||
|
||||
## Known limitations
|
||||
|
||||
- sibling selection still depends on the existing deterministic scorer and may choose a justified next branch that is not always the intuitively expected one
|
||||
- cross-branch corroboration is limited to direct child branches of the same parent
|
||||
- no multi-hop corroboration exists across unrelated subtrees
|
||||
- reasoning remains bounded to explicitly represented graph structure and user-provided answers
|
||||
|
||||
## Deliberate exclusions
|
||||
|
||||
- no persistence
|
||||
- no autonomous exploration
|
||||
- no probabilistic reasoning
|
||||
- no Bayesian reasoning
|
||||
- no semantic embeddings
|
||||
- no expert mode
|
||||
- no multi-hop corroboration across unrelated subtrees
|
||||
- no heavy graph visualisation
|
||||
|
||||
## Next experimental question
|
||||
|
||||
`Can the engine preserve and reuse successful reasoning structures across separate cases without turning prior experience into unquestioned assumptions?`
|
||||
@@ -0,0 +1,50 @@
|
||||
# v0.6 Selection Influence Experiment
|
||||
|
||||
## Hypothesis
|
||||
|
||||
The initial unknown selected for the revenue-versus-cash scenario may be driven more by graph structure, more by semantic keyword matches, or by both together.
|
||||
|
||||
## Scenario
|
||||
|
||||
`Revenue increased by 18%, but cash in the bank fell over the same period.`
|
||||
|
||||
## Actual selected node
|
||||
|
||||
- Node ID: `nqdzobz`
|
||||
- Label: `Magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts).`
|
||||
- Deterministic investigation strategy: `definition`
|
||||
- Deterministic question: `What evidence would resolve whether magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts). is true?`
|
||||
|
||||
## Structural contribution
|
||||
|
||||
- Downstream dependency count: `0`
|
||||
- Prerequisite position: no unresolved prerequisites; count `0`
|
||||
- Dependency ordering / centrality: no candidate had downstream dependants or dependency depth advantage in the live graph
|
||||
|
||||
## Semantic contribution
|
||||
|
||||
- Objective: false
|
||||
- Actor: false
|
||||
- Criteria: false
|
||||
- Measurement: false
|
||||
- Terminology: false
|
||||
- Constraint: false
|
||||
- Pricing: false
|
||||
- Implementation: false
|
||||
- Optimisation: false
|
||||
- Speculative: false
|
||||
- Contribution list: only `downstream_dependencies` was present, with delta `0`
|
||||
|
||||
## Counterfactual results
|
||||
|
||||
- Live-shaped ordering: `nqdzobz` ranked above `niewza`, but both had score `0`, downstream `0`, and unresolved prerequisites `0`
|
||||
- Links removed: ordering stayed the same, because the live graph already provided no differentiating structure between the two unknowns
|
||||
- Wording neutralised: ordering flipped to the first unknown by neutral label order (`Unknown A` before `Unknown B`), showing the outcome remained tie-break-driven rather than structure-driven
|
||||
|
||||
## Conclusion
|
||||
|
||||
For this scenario, the actual winner was not selected because of graph structure and not selected because of semantic keyword weights. The live diagnostics show a complete tie on score, downstream influence, and prerequisite position, with every semantic match category false for both candidates. The winner was therefore chosen by the final tie-break rule, `label_asc`.
|
||||
|
||||
## Is a scoring change justified?
|
||||
|
||||
Not from this single experiment alone. The result shows a diagnostic gap for this scenario, but this task does not justify a scoring change by itself, and no scoring change is made.
|
||||
@@ -0,0 +1,136 @@
|
||||
# v0.7 Observation Report
|
||||
|
||||
**Date**: 2026-08-03 | **Commit**: c273209 | **Branch**: feature/reasoning-pattern-memory-v0.7
|
||||
|
||||
## Summary Table
|
||||
|
||||
| Scenario | Name | Start | Update | Nodes | Unknowns | Rating |
|
||||
|----------|------|-------|--------|-------|----------|--------|
|
||||
| scenario-1 | Confidence Engine commercial validation | pass | fail(400) | 9 | 3 | flow failure |
|
||||
| scenario-2 | Hiring | pass | fail(400) | 18 | 8 | flow failure |
|
||||
| scenario-3 | Vehicle replacement | pass | fail(400) | 15 | 8 | flow failure |
|
||||
| scenario-4 | Welsh Government-style programme decision | pass | fail(400) | 10 | 3 | flow failure |
|
||||
| scenario-5 | Operational contradiction | pass | fail(400) | 7 | 2 | flow failure |
|
||||
| scenario-6 | Personal decision | fail | skipped | 0 | 0 | flow failure |
|
||||
|
||||
## Per-Scenario Findings
|
||||
|
||||
### scenario-1: Confidence Engine commercial validation
|
||||
|
||||
- **Overall**: Start=pass, Update=fail(400), Rating=flow failure
|
||||
- Pattern: N/A | Nodes: 9 | Edges: 0
|
||||
- Validation: valid | Duration: 63386ms
|
||||
- Unknown IDs: nirkgb4, n36c0cc, nzeyzkz
|
||||
- Error: [N/A] Invalid update-case request
|
||||
|
||||
- **Assessment**:
|
||||
- reasoning-pattern fit: fail
|
||||
- one-concept simplicity: fail
|
||||
- plain-language clarity: fail
|
||||
- logical progression: fail (No question generated)
|
||||
- graph-backed: fail
|
||||
- premature-specialism avoided: fail
|
||||
|
||||
### scenario-2: Hiring
|
||||
|
||||
- **Overall**: Start=pass, Update=fail(400), Rating=flow failure
|
||||
- Pattern: N/A | Nodes: 18 | Edges: 5
|
||||
- Validation: valid | Duration: 146476ms
|
||||
- Unknown IDs: n7yonyv, npci7a7, nug9wj2, nz0vpey, nz8pwyc, newxmzu, nw14mjj, n25mnp3
|
||||
- Error: [N/A] Invalid update-case request
|
||||
|
||||
- **Assessment**:
|
||||
- reasoning-pattern fit: fail
|
||||
- one-concept simplicity: fail
|
||||
- plain-language clarity: fail
|
||||
- logical progression: fail (No question generated)
|
||||
- graph-backed: fail
|
||||
- premature-specialism avoided: fail
|
||||
|
||||
### scenario-3: Vehicle replacement
|
||||
|
||||
- **Overall**: Start=pass, Update=fail(400), Rating=flow failure
|
||||
- Pattern: N/A | Nodes: 15 | Edges: 5
|
||||
- Validation: valid | Duration: 81460ms
|
||||
- Unknown IDs: ng5yr11, nogqips, n499gin, n8fbv3p, nf2f6zx, n4feiap, nvwthlt, nqrxjli
|
||||
- Error: [N/A] Invalid update-case request
|
||||
|
||||
- **Assessment**:
|
||||
- reasoning-pattern fit: fail
|
||||
- one-concept simplicity: fail
|
||||
- plain-language clarity: fail
|
||||
- logical progression: fail (No question generated)
|
||||
- graph-backed: fail
|
||||
- premature-specialism avoided: fail
|
||||
|
||||
### scenario-4: Welsh Government-style programme decision
|
||||
|
||||
- **Overall**: Start=pass, Update=fail(400), Rating=flow failure
|
||||
- Pattern: N/A | Nodes: 10 | Edges: 0
|
||||
- Validation: valid | Duration: 129682ms
|
||||
- Unknown IDs: nrrm3qn, nefmpat, n6rtwg1
|
||||
- Error: [N/A] Invalid update-case request
|
||||
|
||||
- **Assessment**:
|
||||
- reasoning-pattern fit: fail
|
||||
- one-concept simplicity: fail
|
||||
- plain-language clarity: fail
|
||||
- logical progression: fail (No question generated)
|
||||
- graph-backed: fail
|
||||
- premature-specialism avoided: fail
|
||||
|
||||
### scenario-5: Operational contradiction
|
||||
|
||||
- **Overall**: Start=pass, Update=fail(400), Rating=flow failure
|
||||
- Pattern: N/A | Nodes: 7 | Edges: 0
|
||||
- Validation: valid | Duration: 70579ms
|
||||
- Unknown IDs: n6gm2cv, nylhu9g
|
||||
- Error: [N/A] Invalid update-case request
|
||||
|
||||
- **Assessment**:
|
||||
- reasoning-pattern fit: fail
|
||||
- one-concept simplicity: fail
|
||||
- plain-language clarity: fail
|
||||
- logical progression: fail (No question generated)
|
||||
- graph-backed: fail
|
||||
- premature-specialism avoided: fail
|
||||
|
||||
### scenario-6: Personal decision
|
||||
|
||||
- **Overall**: Start=fail, Update=skipped, Rating=flow failure
|
||||
- Pattern: N/A | Nodes: 0 | Edges: 0
|
||||
- Validation: invalid | Duration: 72547ms
|
||||
|
||||
- **Assessment**:
|
||||
- reasoning-pattern fit: fail
|
||||
- one-concept simplicity: fail
|
||||
- plain-language clarity: fail
|
||||
- logical progression: fail (No question generated)
|
||||
- graph-backed: fail
|
||||
- premature-specialism avoided: fail
|
||||
|
||||
## Failure Pattern Analysis
|
||||
|
||||
### Start Phase
|
||||
- **5/6 succeeded**, 1/6 failed
|
||||
- scenario-6: Scenario analysis failed
|
||||
|
||||
### Update Phase
|
||||
- **0/6 succeeded**, 5/6 failed, 1/6 skipped
|
||||
|
||||
- **N/A** (5 failures):
|
||||
- scenario-1: Invalid update-case request
|
||||
- scenario-2: Invalid update-case request
|
||||
- scenario-3: Invalid update-case request
|
||||
- scenario-4: Invalid update-case request
|
||||
- scenario-5: Invalid update-case request
|
||||
|
||||
## What's Stable
|
||||
|
||||
- ✅ **Graph construction**: 5/6 start success across all scenario types (commercial, operational, personal, policy)
|
||||
|
||||
## Recommendations
|
||||
|
||||
1. **Fix update failures** (5/6): Primary focus area. Most failures in proposal_compatibility and delta detection.
|
||||
- Monitor reasoning pattern inference reliability across different scenario domains.
|
||||
- Consider adding timeout guards for long-running LLM calls (some exceeded 60s).
|
||||
@@ -0,0 +1,288 @@
|
||||
## Post-update selection invariant
|
||||
|
||||
After every successful graph update, the full deterministic question-selection pipeline must run again whenever eligible unresolved unknowns remain.
|
||||
|
||||
The active reasoning pattern constrains which graph nodes may participate in reasoning.
|
||||
|
||||
That means the update path must not stop at graph mutation, child resolution, emergent unknown creation, decomposition, or upward propagation. It must continue through:
|
||||
|
||||
```text
|
||||
updated graph
|
||||
→ rebuild reasoning state
|
||||
→ identify unresolved candidates
|
||||
→ select active unknown
|
||||
→ atomicity assessment
|
||||
→ answerability assessment
|
||||
→ decompose if required
|
||||
→ reselect
|
||||
→ reasoning-pattern selection
|
||||
→ investigation-strategy selection
|
||||
→ question-family selection
|
||||
→ question formulation
|
||||
→ complexity validation
|
||||
→ selectedQuestion
|
||||
```
|
||||
|
||||
Returning no question is only valid when no eligible unresolved candidate remains, the case is complete, ambiguity cannot be safely resolved, or question formulation fails validation with an explicit deterministic reason.
|
||||
|
||||
## Graph validity vs reasoning-pattern validity
|
||||
|
||||
These are separate requirements.
|
||||
|
||||
- **Graph validity** means references, IDs, node shapes, and update semantics are structurally correct.
|
||||
- **Reasoning-pattern validity** means selectable investigation nodes are compatible with the current reasoning mode.
|
||||
|
||||
A graph can be structurally valid while still being reasoning-invalid.
|
||||
|
||||
Example: a decision investigation may still contain an unresolved comparison-style node such as `How the two observations were measured`. That node is structurally well-formed, but it is not allowed to participate as an active investigation target unless the reasoning pattern has actually shifted into comparison, contradiction, or explanation work.
|
||||
|
||||
The engine therefore needs both invariants:
|
||||
|
||||
1. the graph must be structurally valid
|
||||
2. every selectable unknown must be compatible with the active reasoning pattern
|
||||
|
||||
# v0.7 Question Simplicity Experiment
|
||||
|
||||
## Observed failure
|
||||
|
||||
The first v0.7 UI scenario exposed a reasoning failure where the selected unknown could still be directionally correct while the resulting question was too large to answer in one coherent response.
|
||||
|
||||
Example failure:
|
||||
|
||||
> Have you measured the current financial or operational cost to users who lack justified confidence, and what baseline budget do they currently allocate for comparable decision-support methods?
|
||||
|
||||
This question bundled multiple investigations:
|
||||
|
||||
- cost
|
||||
- user impact
|
||||
- existing alternatives
|
||||
- current budget
|
||||
|
||||
That violated the intended one-step reasoning discipline.
|
||||
|
||||
## Principle
|
||||
|
||||
**A correct unknown paired with an unanswerably broad question is still a reasoning failure.**
|
||||
|
||||
The engine should ask one question about one primary concept at a time.
|
||||
|
||||
**The reconstruction model may suggest a question, but only the graph-backed deterministic pipeline may select the user-facing question.**
|
||||
|
||||
**A node is only questionable if it is independently answerable.**
|
||||
|
||||
## One-question / one-concept rule
|
||||
|
||||
Every user-facing question should:
|
||||
|
||||
- contain one question mark
|
||||
- target one unresolved graph node
|
||||
- ask for one primary concept
|
||||
- request one coherent answer
|
||||
- avoid joined investigations
|
||||
- minimise cognitive effort while still reducing meaningful uncertainty
|
||||
|
||||
## Deterministic cognitive-load rules
|
||||
|
||||
The new deterministic question-complexity assessment marks a question as too broad when it shows signals such as:
|
||||
|
||||
- multiple requested answers joined by `and`
|
||||
- distinct measures combined in one prompt, such as cost plus budget
|
||||
- comma-list phrasing that expands the request into several sub-questions
|
||||
- more than one primary concept
|
||||
- abstract noun chains that make the question hard to parse on first reading
|
||||
- very long question length
|
||||
|
||||
The assessment returns:
|
||||
|
||||
- `acceptable`
|
||||
- `primaryConceptCount`
|
||||
- `compoundQuestionSignals`
|
||||
- `abstractTermCount`
|
||||
- `cognitiveLoad`
|
||||
- `reasons`
|
||||
|
||||
## Decomposition-before-rewording rule
|
||||
|
||||
The engine now treats broad commercial-validation unknowns as composite.
|
||||
|
||||
If the selected unknown still spans multiple validation dimensions, the system should not simply shorten the sentence. It should first decompose the unknown into smaller child unknowns and then select one foundational child.
|
||||
|
||||
For the current scenario, this meant creating child unknowns such as:
|
||||
|
||||
- who experiences the problem
|
||||
- what happens when it is not resolved
|
||||
- how often it happens
|
||||
- how people deal with it today
|
||||
- whether people actively look for help
|
||||
|
||||
The selector then reaches the first foundational child through prerequisite ordering encoded in the decomposition graph rather than through global scoring changes.
|
||||
|
||||
## Atomicity vs answerability
|
||||
|
||||
These are different reasoning properties.
|
||||
|
||||
- **Atomicity** asks: does this node describe one investigation or several bundled investigations?
|
||||
- **Answerability** asks: even if the wording looks singular, can this node be answered directly without first resolving multiple prerequisite dimensions?
|
||||
|
||||
A node can appear atomic in wording but still fail answerability.
|
||||
|
||||
Examples include broad evaluation containers such as product validation, customer value, business case, technical feasibility, or commercial justification. These often compress several prerequisite investigations into one conclusion-shaped unknown.
|
||||
|
||||
That means atomicity alone is not enough.
|
||||
|
||||
The engine now decomposes whenever either of these is true:
|
||||
|
||||
- the unknown is not atomic
|
||||
- the unknown is not independently answerable
|
||||
|
||||
This prevents a broad container node from becoming the selected question target even when its wording looks grammatically singular.
|
||||
|
||||
## Reasoning Pattern
|
||||
|
||||
The next failure exposed a deeper issue: even after atomicity and answerability were added, the engine could still choose a question template from the wrong reasoning family.
|
||||
|
||||
The live failure was an explanation-style prompt appearing in a commercial validation scenario:
|
||||
|
||||
> What changed during the period that could help explain why ...
|
||||
|
||||
That was wrong not because of wording, but because the engine had selected an **explanation family** when the actual task was a **decision investigation**.
|
||||
|
||||
To correct that, the deterministic pipeline now explicitly inserts a reasoning-pattern stage:
|
||||
|
||||
```text
|
||||
selected unknown
|
||||
→ atomicity
|
||||
→ answerability
|
||||
→ reasoning pattern
|
||||
→ investigation strategy
|
||||
→ question family
|
||||
→ question
|
||||
```
|
||||
|
||||
This matters because each stage must constrain the next.
|
||||
|
||||
- **Reasoning Pattern** decides what kind of reasoning is happening
|
||||
- **Investigation Strategy** decides how to reduce uncertainty within that pattern
|
||||
- **Question Family** decides what template space is allowed
|
||||
- **Question** is the final concrete wording
|
||||
|
||||
Without this stage separation, strategy and template selection can leak across domains and reuse relationship/explanation prompts too broadly.
|
||||
|
||||
## Deterministic reasoning-pattern vocabulary
|
||||
|
||||
The current deterministic pattern vocabulary is intentionally small:
|
||||
|
||||
- decision
|
||||
- explanation
|
||||
- contradiction
|
||||
- definition
|
||||
- diagnosis
|
||||
- comparison
|
||||
- prioritisation
|
||||
|
||||
Pattern selection uses graph structure rather than wording alone, including:
|
||||
|
||||
- node kind
|
||||
- relationship / observation topology
|
||||
- parent context
|
||||
- reasoning state
|
||||
- selected unknown role in the graph
|
||||
|
||||
## Question-family mapping
|
||||
|
||||
Patterns now constrain which question families are allowed.
|
||||
|
||||
- **decision**
|
||||
- decision_foundation
|
||||
- decision_evidence
|
||||
- decision_threshold
|
||||
- definition
|
||||
- **explanation**
|
||||
- explanation
|
||||
- comparison
|
||||
- **contradiction**
|
||||
- contradiction
|
||||
- comparison
|
||||
- explanation
|
||||
- **definition**
|
||||
- definition
|
||||
- **diagnosis**
|
||||
- diagnosis
|
||||
- comparison
|
||||
- **comparison**
|
||||
- comparison
|
||||
- **prioritisation**
|
||||
- prioritisation
|
||||
- decision_threshold
|
||||
|
||||
Most importantly:
|
||||
|
||||
- explanation templates are only allowed for `explanation` or `contradiction`
|
||||
- decision investigations cannot emit explanation-family questions
|
||||
|
||||
## Live correction
|
||||
|
||||
For the commercial-method scenario, the engine now classifies the reasoning as a **decision** pattern rather than an explanation pattern.
|
||||
|
||||
That means explanation-family templates are explicitly rejected, and the selected child unknown must be questioned using a decision-compatible family instead.
|
||||
|
||||
## UI result
|
||||
|
||||
The long compound question no longer survives as the first follow-up in the tested path.
|
||||
|
||||
The new first-step question is:
|
||||
|
||||
> Who experiences this problem?
|
||||
|
||||
This question:
|
||||
|
||||
- asks one thing
|
||||
- is understandable immediately
|
||||
- stays graph-backed
|
||||
- avoids pricing or budget before problem existence is established
|
||||
|
||||
## Start-case authority rule
|
||||
|
||||
There were previously two question paths during initial analysis:
|
||||
|
||||
- reconstruction model `nextQuestion`
|
||||
- graph-backed unknown selection and question formulation
|
||||
|
||||
The defect was that `startCase` copied the reconstruction `nextQuestion` directly into the normal UI.
|
||||
|
||||
That path is now closed.
|
||||
|
||||
Initial user-facing questioning now follows this pipeline:
|
||||
|
||||
```text
|
||||
reconstruction
|
||||
→ graph build
|
||||
→ unresolved unknown selection
|
||||
→ atomicity assessment
|
||||
→ decomposition if needed
|
||||
→ investigation strategy
|
||||
→ question formulation
|
||||
→ complexity validation
|
||||
→ selectedQuestion
|
||||
```
|
||||
|
||||
The reconstruction question is still retained in diagnostics as provenance, but it is not authoritative.
|
||||
|
||||
## Live result
|
||||
|
||||
Running the commercial-method scenario through the real environment now:
|
||||
|
||||
- succeeds without the enum compatibility failure
|
||||
- does not show the broad reconstruction question in the UI path
|
||||
- surfaces a graph-backed first question instead
|
||||
- keeps the reconstruction question only in diagnostics
|
||||
|
||||
For the tested scenario, the user-facing first question remained:
|
||||
|
||||
> Who experiences this problem?
|
||||
|
||||
## Remaining limitations
|
||||
|
||||
- question-complexity assessment is still conservative and pattern-based rather than semantic in a richer linguistic sense
|
||||
- plain-language simplification currently uses a small deterministic replacement set
|
||||
- broader prerequisite ordering is strongest for decomposition structures that explicitly encode those dependencies
|
||||
+3155
-8
File diff suppressed because it is too large
Load Diff
+669
-16
@@ -13,10 +13,19 @@ import {
|
||||
updateCaseRequestSchema,
|
||||
} from "./schema.js";
|
||||
import { buildInitialGraph, describeGraph } from "./builder.js";
|
||||
import { applyValidatedProposal } from "./apply-proposal.js";
|
||||
import {
|
||||
applyValidatedProposal,
|
||||
determineGraphBackedQuestion,
|
||||
} from "./apply-proposal.js";
|
||||
import { buildGraphUpdatePrompt } from "./prompt-builder.js";
|
||||
import {
|
||||
buildReasoningState,
|
||||
formulateQuestion,
|
||||
formulateTieResolutionQuestion,
|
||||
} from "./question-formulator.js";
|
||||
import { parseGraphUpdateProposal } from "./update-proposal.js";
|
||||
import {
|
||||
explainUnknownSelection,
|
||||
selectActiveUnknownCandidate,
|
||||
validateGraphReferences,
|
||||
} from "./utils.js";
|
||||
@@ -31,7 +40,39 @@ function toValidationErrors(error) {
|
||||
);
|
||||
}
|
||||
|
||||
function buildDiagnostics({ analysis, graph, graphReferenceValidation }) {
|
||||
function buildDiagnostics({
|
||||
analysis,
|
||||
graph,
|
||||
graphReferenceValidation,
|
||||
unknownSelectionExplanation,
|
||||
reconstructionQuestion,
|
||||
reconstructionQuestionAccepted,
|
||||
reconstructionQuestionRejectionReasons,
|
||||
finalGraphBackedQuestion,
|
||||
selectedUnknownNodeId,
|
||||
decompositionApplied,
|
||||
questionComplexityAssessment,
|
||||
answerabilityAssessment,
|
||||
independentlyAnswerable,
|
||||
prerequisiteConceptCount,
|
||||
decompositionTriggeredByAnswerability,
|
||||
decompositionReason,
|
||||
selectedContainerUnknown,
|
||||
selectedChildUnknown,
|
||||
reasoningPattern,
|
||||
questionFamily,
|
||||
allowedQuestionFamilies,
|
||||
rejectedQuestionFamilies,
|
||||
selectedQuestionTemplate,
|
||||
reasoningPatternReason,
|
||||
reasoningPatternValidation,
|
||||
patternCompatibleNodeCount,
|
||||
incompatibleNodeIds,
|
||||
compatibilityFailures,
|
||||
replacementActions,
|
||||
graphReasoningIntegrity,
|
||||
noQuestionReason,
|
||||
}) {
|
||||
return {
|
||||
promptVersion: analysis?.promptVersion ?? null,
|
||||
modelName: analysis?.modelName ?? null,
|
||||
@@ -43,9 +84,80 @@ function buildDiagnostics({ analysis, graph, graphReferenceValidation }) {
|
||||
compatibilityApplied: analysis?.compatibilityApplied ?? false,
|
||||
compatibilityChanges: analysis?.compatibilityChanges ?? [],
|
||||
compatibilityWarnings: analysis?.compatibilityWarnings ?? [],
|
||||
unknownSelectionExplanation: unknownSelectionExplanation ?? null,
|
||||
reconstructionQuestion: reconstructionQuestion ?? null,
|
||||
reconstructionQuestionAccepted: reconstructionQuestionAccepted ?? null,
|
||||
reconstructionQuestionRejectionReasons:
|
||||
reconstructionQuestionRejectionReasons ?? [],
|
||||
finalGraphBackedQuestion: finalGraphBackedQuestion ?? null,
|
||||
selectedUnknownNodeId: selectedUnknownNodeId ?? null,
|
||||
decompositionApplied: decompositionApplied ?? false,
|
||||
questionComplexityAssessment: questionComplexityAssessment ?? null,
|
||||
answerabilityAssessment: answerabilityAssessment ?? null,
|
||||
independentlyAnswerable: independentlyAnswerable ?? null,
|
||||
prerequisiteConceptCount: prerequisiteConceptCount ?? null,
|
||||
decompositionTriggeredByAnswerability:
|
||||
decompositionTriggeredByAnswerability ?? false,
|
||||
decompositionReason: decompositionReason ?? null,
|
||||
selectedContainerUnknown: selectedContainerUnknown ?? null,
|
||||
selectedChildUnknown: selectedChildUnknown ?? null,
|
||||
reasoningPattern: reasoningPattern ?? null,
|
||||
questionFamily: questionFamily ?? null,
|
||||
allowedQuestionFamilies: allowedQuestionFamilies ?? [],
|
||||
rejectedQuestionFamilies: rejectedQuestionFamilies ?? [],
|
||||
selectedQuestionTemplate: selectedQuestionTemplate ?? null,
|
||||
reasoningPatternReason: reasoningPatternReason ?? null,
|
||||
reasoningPatternValidation: reasoningPatternValidation ?? null,
|
||||
patternCompatibleNodeCount: patternCompatibleNodeCount ?? 0,
|
||||
incompatibleNodeIds: incompatibleNodeIds ?? [],
|
||||
compatibilityFailures: compatibilityFailures ?? [],
|
||||
replacementActions: replacementActions ?? [],
|
||||
graphReasoningIntegrity: graphReasoningIntegrity ?? null,
|
||||
noQuestionReason: noQuestionReason ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
function fallbackStartCaseReasoningPatternValidation(
|
||||
selectedQuestion,
|
||||
existingValidation,
|
||||
) {
|
||||
if (existingValidation) {
|
||||
return existingValidation;
|
||||
}
|
||||
|
||||
if (!selectedQuestion?.reasoningPattern) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
activePattern: selectedQuestion.reasoningPattern,
|
||||
valid: Boolean(selectedQuestion.question),
|
||||
reason: selectedQuestion.question
|
||||
? "Initial graph-backed selection produced a reasoning-pattern-compatible question."
|
||||
: "Initial graph-backed selection did not produce a valid question for the inferred reasoning pattern.",
|
||||
};
|
||||
}
|
||||
|
||||
function buildUnknownSelectionDiagnostics(
|
||||
graph,
|
||||
resolvedNodeIds = [],
|
||||
selectedQuestion = null,
|
||||
) {
|
||||
const explanation = explainUnknownSelection(graph, resolvedNodeIds);
|
||||
if (explanation.status === "ambiguous") {
|
||||
return {
|
||||
...explanation,
|
||||
tieResolutionQuestion:
|
||||
selectedQuestion?.selectionStatus === "ambiguous"
|
||||
? selectedQuestion.question
|
||||
: formulateTieResolutionQuestion({ graph }).question,
|
||||
alphabeticalUsedAsReasoning: false,
|
||||
};
|
||||
}
|
||||
|
||||
return explanation;
|
||||
}
|
||||
|
||||
function buildUpdateDiagnostics({
|
||||
promptVersion,
|
||||
modelName,
|
||||
@@ -53,6 +165,82 @@ function buildUpdateDiagnostics({
|
||||
normalisationsApplied,
|
||||
graph,
|
||||
graphReferenceValidation,
|
||||
selectedQuestion,
|
||||
unknownSelectionExplanation,
|
||||
previousReasoningState,
|
||||
reasoningState,
|
||||
resolvedReasoningNodeIds,
|
||||
emergentReasoningNodeCreated,
|
||||
emergentReasoningNodeId,
|
||||
emergentReasoningNodeReason,
|
||||
atomicityAssessment,
|
||||
atomicityDecisionReason,
|
||||
decompositionDepth,
|
||||
decompositionAttempted,
|
||||
decompositionAccepted,
|
||||
decompositionStoppedReason,
|
||||
proposedChildCount,
|
||||
acceptedChildCount,
|
||||
rejectedChildren,
|
||||
selectedChildNodeId,
|
||||
childQualitySummary,
|
||||
propagationPerformed,
|
||||
resolvedChildNodeId,
|
||||
parentNodeId,
|
||||
parentStatusBefore,
|
||||
parentStatusAfter,
|
||||
parentConfidenceBefore,
|
||||
parentConfidenceAfter,
|
||||
evidenceConfidenceBefore,
|
||||
evidenceConfidenceAfter,
|
||||
completenessBefore,
|
||||
completenessAfter,
|
||||
conclusionConfidenceBefore,
|
||||
conclusionConfidenceAfter,
|
||||
resolvedDirectChildren,
|
||||
unresolvedDirectChildren,
|
||||
contradictoryDirectChildren,
|
||||
corroboratingBranchCount,
|
||||
conflictingBranchCount,
|
||||
duplicateEvidenceCount,
|
||||
independentBranchCount,
|
||||
interactionSummary,
|
||||
confidenceCapReason,
|
||||
ancestorPropagationStoppedReason,
|
||||
affectedAncestorIds,
|
||||
nextSelectedSibling,
|
||||
parentResolved,
|
||||
decompositionPerformed,
|
||||
childUnknownCount,
|
||||
childNodeIds,
|
||||
atomicityReason,
|
||||
questionComplexityAccepted,
|
||||
primaryConceptCount,
|
||||
cognitiveLoad,
|
||||
complexityReasons,
|
||||
decompositionTriggeredByQuestionComplexity,
|
||||
previousQuestion,
|
||||
finalQuestion,
|
||||
selectedUnknownBefore,
|
||||
selectedUnknownAfter,
|
||||
plainLanguageNormalisations,
|
||||
reasoningPattern,
|
||||
questionFamily,
|
||||
allowedQuestionFamilies,
|
||||
rejectedQuestionFamilies,
|
||||
selectedQuestionTemplate,
|
||||
reasoningPatternReason,
|
||||
unresolvedCandidateCount,
|
||||
eligibleCandidateCount,
|
||||
candidateNodeIds,
|
||||
resolvedCurrentTurnNodeIds,
|
||||
noQuestionReason,
|
||||
reasoningPatternValidation,
|
||||
patternCompatibleNodeCount,
|
||||
incompatibleNodeIds,
|
||||
compatibilityFailures,
|
||||
replacementActions,
|
||||
graphReasoningIntegrity,
|
||||
}) {
|
||||
return {
|
||||
promptVersion: promptVersion ?? "v0.4",
|
||||
@@ -66,6 +254,91 @@ function buildUpdateDiagnostics({
|
||||
errors: [],
|
||||
},
|
||||
normalisationsApplied: normalisationsApplied ?? [],
|
||||
investigationStrategy:
|
||||
selectedQuestion?.investigationStrategy ??
|
||||
selectedQuestion?.strategy ??
|
||||
null,
|
||||
previousComparabilityStatus:
|
||||
previousReasoningState?.comparabilityStatus ?? null,
|
||||
comparabilityStatus: reasoningState?.comparabilityStatus ?? null,
|
||||
relationshipStatus: reasoningState?.relationshipStatus ?? null,
|
||||
relationshipAssessed: reasoningState?.relationshipAssessed ?? null,
|
||||
reasoningStagesBefore: previousReasoningState?.reasoningStages ?? [],
|
||||
reasoningStagesAfter: reasoningState?.reasoningStages ?? [],
|
||||
resolvedReasoningNodeIds: resolvedReasoningNodeIds ?? [],
|
||||
emergentReasoningNodeCreated: emergentReasoningNodeCreated ?? false,
|
||||
emergentReasoningNodeId: emergentReasoningNodeId ?? null,
|
||||
emergentReasoningNodeReason: emergentReasoningNodeReason ?? null,
|
||||
atomicityAssessment: atomicityAssessment ?? null,
|
||||
atomicityDecisionReason: atomicityDecisionReason ?? null,
|
||||
decompositionDepth: decompositionDepth ?? 0,
|
||||
decompositionAttempted: decompositionAttempted ?? false,
|
||||
decompositionAccepted: decompositionAccepted ?? false,
|
||||
decompositionStoppedReason: decompositionStoppedReason ?? null,
|
||||
proposedChildCount: proposedChildCount ?? 0,
|
||||
acceptedChildCount: acceptedChildCount ?? 0,
|
||||
rejectedChildren: rejectedChildren ?? [],
|
||||
selectedChildNodeId: selectedChildNodeId ?? null,
|
||||
childQualitySummary: childQualitySummary ?? [],
|
||||
propagationPerformed: propagationPerformed ?? false,
|
||||
resolvedChildNodeId: resolvedChildNodeId ?? null,
|
||||
parentNodeId: parentNodeId ?? null,
|
||||
parentStatusBefore: parentStatusBefore ?? null,
|
||||
parentStatusAfter: parentStatusAfter ?? null,
|
||||
parentConfidenceBefore: parentConfidenceBefore ?? null,
|
||||
parentConfidenceAfter: parentConfidenceAfter ?? null,
|
||||
evidenceConfidenceBefore: evidenceConfidenceBefore ?? null,
|
||||
evidenceConfidenceAfter: evidenceConfidenceAfter ?? null,
|
||||
completenessBefore: completenessBefore ?? null,
|
||||
completenessAfter: completenessAfter ?? null,
|
||||
conclusionConfidenceBefore: conclusionConfidenceBefore ?? null,
|
||||
conclusionConfidenceAfter: conclusionConfidenceAfter ?? null,
|
||||
resolvedDirectChildren: resolvedDirectChildren ?? 0,
|
||||
unresolvedDirectChildren: unresolvedDirectChildren ?? 0,
|
||||
contradictoryDirectChildren: contradictoryDirectChildren ?? 0,
|
||||
corroboratingBranchCount: corroboratingBranchCount ?? 0,
|
||||
conflictingBranchCount: conflictingBranchCount ?? 0,
|
||||
duplicateEvidenceCount: duplicateEvidenceCount ?? 0,
|
||||
independentBranchCount: independentBranchCount ?? 0,
|
||||
interactionSummary: interactionSummary ?? null,
|
||||
confidenceCapReason: confidenceCapReason ?? null,
|
||||
ancestorPropagationStoppedReason: ancestorPropagationStoppedReason ?? null,
|
||||
affectedAncestorIds: affectedAncestorIds ?? [],
|
||||
nextSelectedSibling: nextSelectedSibling ?? null,
|
||||
parentResolved: parentResolved ?? false,
|
||||
decompositionPerformed: decompositionPerformed ?? false,
|
||||
childUnknownCount: childUnknownCount ?? 0,
|
||||
childNodeIds: childNodeIds ?? [],
|
||||
atomicityReason: atomicityReason ?? null,
|
||||
questionComplexityAccepted: questionComplexityAccepted ?? null,
|
||||
primaryConceptCount: primaryConceptCount ?? null,
|
||||
cognitiveLoad: cognitiveLoad ?? null,
|
||||
complexityReasons: complexityReasons ?? [],
|
||||
decompositionTriggeredByQuestionComplexity:
|
||||
decompositionTriggeredByQuestionComplexity ?? false,
|
||||
previousQuestion: previousQuestion ?? null,
|
||||
finalQuestion: finalQuestion ?? null,
|
||||
selectedUnknownBefore: selectedUnknownBefore ?? null,
|
||||
selectedUnknownAfter: selectedUnknownAfter ?? null,
|
||||
plainLanguageNormalisations: plainLanguageNormalisations ?? [],
|
||||
reasoningPattern: reasoningPattern ?? null,
|
||||
questionFamily: questionFamily ?? null,
|
||||
allowedQuestionFamilies: allowedQuestionFamilies ?? [],
|
||||
rejectedQuestionFamilies: rejectedQuestionFamilies ?? [],
|
||||
selectedQuestionTemplate: selectedQuestionTemplate ?? null,
|
||||
reasoningPatternReason: reasoningPatternReason ?? null,
|
||||
unresolvedCandidateCount: unresolvedCandidateCount ?? 0,
|
||||
eligibleCandidateCount: eligibleCandidateCount ?? 0,
|
||||
candidateNodeIds: candidateNodeIds ?? [],
|
||||
resolvedCurrentTurnNodeIds: resolvedCurrentTurnNodeIds ?? [],
|
||||
noQuestionReason: noQuestionReason ?? null,
|
||||
reasoningPatternValidation: reasoningPatternValidation ?? null,
|
||||
patternCompatibleNodeCount: patternCompatibleNodeCount ?? 0,
|
||||
incompatibleNodeIds: incompatibleNodeIds ?? [],
|
||||
compatibilityFailures: compatibilityFailures ?? [],
|
||||
replacementActions: replacementActions ?? [],
|
||||
graphReasoningIntegrity: graphReasoningIntegrity ?? null,
|
||||
unknownSelectionExplanation: unknownSelectionExplanation ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -105,27 +378,40 @@ export async function startCase(body) {
|
||||
});
|
||||
|
||||
const currentSummary = describeGraph(initialGraph);
|
||||
const activeUnknownNodeId =
|
||||
selectActiveUnknownCandidate(
|
||||
{
|
||||
...initialGraph,
|
||||
resolvedNodeIds: [],
|
||||
},
|
||||
[],
|
||||
)?.nodeId ?? null;
|
||||
|
||||
const situationGraph = makeGraph({
|
||||
const initialSituationGraph = makeGraph({
|
||||
centralStatement: scenario,
|
||||
nodes: initialGraph.nodes,
|
||||
edges: initialGraph.edges,
|
||||
activeUnknownNodeId,
|
||||
activeUnknownNodeId: null,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary,
|
||||
reasoningState: buildReasoningState({
|
||||
centralStatement: scenario,
|
||||
nodes: initialGraph.nodes,
|
||||
edges: initialGraph.edges,
|
||||
resolvedNodeIds: [],
|
||||
}),
|
||||
});
|
||||
|
||||
situationGraphSchema.parse(situationGraph);
|
||||
situationGraphSchema.parse(initialSituationGraph);
|
||||
|
||||
const graphReferenceValidation = validateGraphReferences(situationGraph);
|
||||
const graphReferenceValidation = validateGraphReferences(
|
||||
initialSituationGraph,
|
||||
);
|
||||
const initialQuestionResult = determineGraphBackedQuestion({
|
||||
situationGraph: initialSituationGraph,
|
||||
});
|
||||
const situationGraph = initialQuestionResult.success
|
||||
? initialQuestionResult.updatedSituationGraph
|
||||
: initialSituationGraph;
|
||||
const selectedQuestion = initialQuestionResult.success
|
||||
? initialQuestionResult.selectedQuestion
|
||||
: null;
|
||||
const unknownSelectionExplanation = buildUnknownSelectionDiagnostics(
|
||||
situationGraph,
|
||||
[],
|
||||
selectedQuestion,
|
||||
);
|
||||
if (!graphReferenceValidation.valid) {
|
||||
return {
|
||||
success: false,
|
||||
@@ -134,6 +420,65 @@ export async function startCase(body) {
|
||||
analysis,
|
||||
graph: situationGraph,
|
||||
graphReferenceValidation,
|
||||
unknownSelectionExplanation,
|
||||
reconstructionQuestion: analysis.nextQuestion?.question ?? null,
|
||||
reconstructionQuestionAccepted: false,
|
||||
reconstructionQuestionRejectionReasons:
|
||||
analysis.nextQuestion?.question != null
|
||||
? [
|
||||
"reconstruction_question_not_authoritative",
|
||||
"graph_backed_pipeline_required",
|
||||
]
|
||||
: [],
|
||||
finalGraphBackedQuestion: selectedQuestion?.question ?? null,
|
||||
selectedUnknownNodeId:
|
||||
initialQuestionResult.selectedUnknownAfter ?? null,
|
||||
decompositionApplied:
|
||||
initialQuestionResult.decompositionPerformed ?? false,
|
||||
questionComplexityAssessment:
|
||||
initialQuestionResult.questionComplexityAssessment ?? null,
|
||||
answerabilityAssessment:
|
||||
initialQuestionResult.answerabilityAssessment ?? null,
|
||||
independentlyAnswerable:
|
||||
initialQuestionResult.independentlyAnswerable ?? null,
|
||||
prerequisiteConceptCount:
|
||||
initialQuestionResult.prerequisiteConceptCount ?? null,
|
||||
decompositionTriggeredByAnswerability:
|
||||
initialQuestionResult.decompositionTriggeredByAnswerability ?? false,
|
||||
decompositionReason:
|
||||
initialQuestionResult.selectedQuestion?.reason ?? null,
|
||||
selectedContainerUnknown:
|
||||
initialQuestionResult.selectedContainerUnknown ?? null,
|
||||
selectedChildUnknown:
|
||||
initialQuestionResult.selectedChildUnknown ?? null,
|
||||
reasoningPattern:
|
||||
initialQuestionResult.selectedQuestion?.reasoningPattern ?? null,
|
||||
questionFamily:
|
||||
initialQuestionResult.selectedQuestion?.questionFamily ?? null,
|
||||
allowedQuestionFamilies:
|
||||
initialQuestionResult.selectedQuestion?.allowedQuestionFamilies ?? [],
|
||||
rejectedQuestionFamilies:
|
||||
initialQuestionResult.selectedQuestion?.rejectedQuestionFamilies ??
|
||||
[],
|
||||
selectedQuestionTemplate:
|
||||
initialQuestionResult.selectedQuestion?.selectedQuestionTemplate ??
|
||||
null,
|
||||
reasoningPatternReason:
|
||||
initialQuestionResult.selectedQuestion?.reasoningPatternReason ??
|
||||
null,
|
||||
reasoningPatternValidation: fallbackStartCaseReasoningPatternValidation(
|
||||
initialQuestionResult.selectedQuestion,
|
||||
initialQuestionResult.reasoningPatternValidation,
|
||||
),
|
||||
patternCompatibleNodeCount:
|
||||
initialQuestionResult.patternCompatibleNodeCount ?? 0,
|
||||
incompatibleNodeIds: initialQuestionResult.incompatibleNodeIds ?? [],
|
||||
compatibilityFailures:
|
||||
initialQuestionResult.compatibilityFailures ?? [],
|
||||
replacementActions: initialQuestionResult.replacementActions ?? [],
|
||||
graphReasoningIntegrity:
|
||||
initialQuestionResult.graphReasoningIntegrity ?? null,
|
||||
noQuestionReason: initialQuestionResult.noQuestionReason ?? null,
|
||||
}),
|
||||
validationErrors: graphReferenceValidation.errors,
|
||||
statusCode: 500,
|
||||
@@ -143,11 +488,65 @@ export async function startCase(body) {
|
||||
return {
|
||||
success: true,
|
||||
situationGraph,
|
||||
selectedQuestion: analysis.nextQuestion ?? null,
|
||||
selectedQuestion,
|
||||
diagnostics: buildDiagnostics({
|
||||
analysis,
|
||||
graph: situationGraph,
|
||||
graphReferenceValidation,
|
||||
unknownSelectionExplanation,
|
||||
reconstructionQuestion: analysis.nextQuestion?.question ?? null,
|
||||
reconstructionQuestionAccepted: false,
|
||||
reconstructionQuestionRejectionReasons:
|
||||
analysis.nextQuestion?.question != null
|
||||
? [
|
||||
"reconstruction_question_not_authoritative",
|
||||
"graph_backed_pipeline_required",
|
||||
]
|
||||
: [],
|
||||
finalGraphBackedQuestion: selectedQuestion?.question ?? null,
|
||||
selectedUnknownNodeId: initialQuestionResult.selectedUnknownAfter ?? null,
|
||||
decompositionApplied:
|
||||
initialQuestionResult.decompositionPerformed ?? false,
|
||||
questionComplexityAssessment:
|
||||
initialQuestionResult.questionComplexityAssessment ?? null,
|
||||
answerabilityAssessment:
|
||||
initialQuestionResult.answerabilityAssessment ?? null,
|
||||
independentlyAnswerable:
|
||||
initialQuestionResult.independentlyAnswerable ?? null,
|
||||
prerequisiteConceptCount:
|
||||
initialQuestionResult.prerequisiteConceptCount ?? null,
|
||||
decompositionTriggeredByAnswerability:
|
||||
initialQuestionResult.decompositionTriggeredByAnswerability ?? false,
|
||||
decompositionReason:
|
||||
initialQuestionResult.selectedQuestion?.reason ?? null,
|
||||
selectedContainerUnknown:
|
||||
initialQuestionResult.selectedContainerUnknown ?? null,
|
||||
selectedChildUnknown: initialQuestionResult.selectedChildUnknown ?? null,
|
||||
reasoningPattern:
|
||||
initialQuestionResult.selectedQuestion?.reasoningPattern ?? null,
|
||||
questionFamily:
|
||||
initialQuestionResult.selectedQuestion?.questionFamily ?? null,
|
||||
allowedQuestionFamilies:
|
||||
initialQuestionResult.selectedQuestion?.allowedQuestionFamilies ?? [],
|
||||
rejectedQuestionFamilies:
|
||||
initialQuestionResult.selectedQuestion?.rejectedQuestionFamilies ?? [],
|
||||
selectedQuestionTemplate:
|
||||
initialQuestionResult.selectedQuestion?.selectedQuestionTemplate ??
|
||||
null,
|
||||
reasoningPatternReason:
|
||||
initialQuestionResult.selectedQuestion?.reasoningPatternReason ?? null,
|
||||
reasoningPatternValidation: fallbackStartCaseReasoningPatternValidation(
|
||||
initialQuestionResult.selectedQuestion,
|
||||
initialQuestionResult.reasoningPatternValidation,
|
||||
),
|
||||
patternCompatibleNodeCount:
|
||||
initialQuestionResult.patternCompatibleNodeCount ?? 0,
|
||||
incompatibleNodeIds: initialQuestionResult.incompatibleNodeIds ?? [],
|
||||
compatibilityFailures: initialQuestionResult.compatibilityFailures ?? [],
|
||||
replacementActions: initialQuestionResult.replacementActions ?? [],
|
||||
graphReasoningIntegrity:
|
||||
initialQuestionResult.graphReasoningIntegrity ?? null,
|
||||
noQuestionReason: initialQuestionResult.noQuestionReason ?? null,
|
||||
}),
|
||||
};
|
||||
}
|
||||
@@ -269,6 +668,8 @@ async function updateCaseWithDependencies(body, dependencies = {}) {
|
||||
const applicationResult = applyProposalUpdate({
|
||||
situationGraph,
|
||||
proposal: parsedProposal.proposal,
|
||||
previousQuestion,
|
||||
answer,
|
||||
});
|
||||
|
||||
if (!applicationResult.success) {
|
||||
@@ -284,6 +685,79 @@ async function updateCaseWithDependencies(body, dependencies = {}) {
|
||||
normalisationsApplied: parsedProposal.normalisationsApplied,
|
||||
graph: situationGraph,
|
||||
graphReferenceValidation: graphReferenceValidation,
|
||||
selectedQuestion: null,
|
||||
previousReasoningState: buildReasoningState(situationGraph),
|
||||
reasoningState: buildReasoningState(situationGraph),
|
||||
resolvedReasoningNodeIds: [],
|
||||
emergentReasoningNodeCreated: false,
|
||||
emergentReasoningNodeId: null,
|
||||
emergentReasoningNodeReason: null,
|
||||
atomicityAssessment: null,
|
||||
atomicityDecisionReason: null,
|
||||
decompositionDepth: 0,
|
||||
decompositionAttempted: false,
|
||||
decompositionAccepted: false,
|
||||
decompositionStoppedReason: null,
|
||||
proposedChildCount: 0,
|
||||
acceptedChildCount: 0,
|
||||
rejectedChildren: [],
|
||||
selectedChildNodeId: null,
|
||||
childQualitySummary: [],
|
||||
propagationPerformed: false,
|
||||
resolvedChildNodeId: null,
|
||||
parentNodeId: null,
|
||||
parentStatusBefore: null,
|
||||
parentStatusAfter: null,
|
||||
parentConfidenceBefore: null,
|
||||
parentConfidenceAfter: null,
|
||||
evidenceConfidenceBefore: null,
|
||||
evidenceConfidenceAfter: null,
|
||||
completenessBefore: null,
|
||||
completenessAfter: null,
|
||||
conclusionConfidenceBefore: null,
|
||||
conclusionConfidenceAfter: null,
|
||||
resolvedDirectChildren: 0,
|
||||
unresolvedDirectChildren: 0,
|
||||
contradictoryDirectChildren: 0,
|
||||
corroboratingBranchCount: 0,
|
||||
conflictingBranchCount: 0,
|
||||
duplicateEvidenceCount: 0,
|
||||
independentBranchCount: 0,
|
||||
interactionSummary: null,
|
||||
confidenceCapReason: null,
|
||||
ancestorPropagationStoppedReason: null,
|
||||
affectedAncestorIds: [],
|
||||
nextSelectedSibling: null,
|
||||
parentResolved: false,
|
||||
decompositionPerformed: false,
|
||||
childUnknownCount: 0,
|
||||
childNodeIds: [],
|
||||
atomicityReason: null,
|
||||
questionComplexityAccepted: null,
|
||||
primaryConceptCount: null,
|
||||
cognitiveLoad: null,
|
||||
complexityReasons: [],
|
||||
decompositionTriggeredByQuestionComplexity: false,
|
||||
previousQuestion,
|
||||
finalQuestion: null,
|
||||
selectedUnknownBefore: null,
|
||||
selectedUnknownAfter: null,
|
||||
unresolvedCandidateCount: 0,
|
||||
eligibleCandidateCount: 0,
|
||||
candidateNodeIds: [],
|
||||
resolvedCurrentTurnNodeIds: [],
|
||||
noQuestionReason: null,
|
||||
reasoningPatternValidation: null,
|
||||
patternCompatibleNodeCount: 0,
|
||||
incompatibleNodeIds: [],
|
||||
compatibilityFailures: [],
|
||||
replacementActions: [],
|
||||
graphReasoningIntegrity: null,
|
||||
plainLanguageNormalisations: [],
|
||||
unknownSelectionExplanation: explainUnknownSelection(
|
||||
situationGraph,
|
||||
situationGraph.resolvedNodeIds || [],
|
||||
),
|
||||
}),
|
||||
},
|
||||
statusCode:
|
||||
@@ -299,6 +773,7 @@ async function updateCaseWithDependencies(body, dependencies = {}) {
|
||||
stage: "update_applied",
|
||||
updatedSituationGraph: applicationResult.updatedSituationGraph,
|
||||
proposal: applicationResult.graphUpdate,
|
||||
selectedQuestion: applicationResult.selectedQuestion,
|
||||
affectedNodeIds: applicationResult.affectedNodeIds,
|
||||
resolvedUnknownNodeIds: applicationResult.resolvedUnknownNodeIds,
|
||||
previousActiveUnknownNodeId:
|
||||
@@ -312,6 +787,104 @@ async function updateCaseWithDependencies(body, dependencies = {}) {
|
||||
normalisationsApplied: parsedProposal.normalisationsApplied,
|
||||
graph: applicationResult.updatedSituationGraph,
|
||||
graphReferenceValidation: applicationResult.graphReferenceValidation,
|
||||
selectedQuestion: applicationResult.selectedQuestion,
|
||||
previousReasoningState: applicationResult.previousReasoningState,
|
||||
reasoningState: applicationResult.reasoningState,
|
||||
resolvedReasoningNodeIds: applicationResult.resolvedReasoningNodeIds,
|
||||
emergentReasoningNodeCreated:
|
||||
applicationResult.emergentReasoningNodeCreated,
|
||||
emergentReasoningNodeId: applicationResult.emergentReasoningNodeId,
|
||||
emergentReasoningNodeReason:
|
||||
applicationResult.emergentReasoningNodeReason,
|
||||
atomicityAssessment: applicationResult.atomicityAssessment,
|
||||
atomicityDecisionReason: applicationResult.atomicityDecisionReason,
|
||||
decompositionDepth: applicationResult.decompositionDepth,
|
||||
decompositionAttempted: applicationResult.decompositionAttempted,
|
||||
decompositionAccepted: applicationResult.decompositionAccepted,
|
||||
decompositionStoppedReason:
|
||||
applicationResult.decompositionStoppedReason,
|
||||
proposedChildCount: applicationResult.proposedChildCount,
|
||||
acceptedChildCount: applicationResult.acceptedChildCount,
|
||||
rejectedChildren: applicationResult.rejectedChildren,
|
||||
selectedChildNodeId: applicationResult.selectedChildNodeId,
|
||||
childQualitySummary: applicationResult.childQualitySummary,
|
||||
propagationPerformed: applicationResult.propagationPerformed,
|
||||
resolvedChildNodeId: applicationResult.resolvedChildNodeId,
|
||||
parentNodeId: applicationResult.parentNodeId,
|
||||
parentStatusBefore: applicationResult.parentStatusBefore,
|
||||
parentStatusAfter: applicationResult.parentStatusAfter,
|
||||
parentConfidenceBefore: applicationResult.parentConfidenceBefore,
|
||||
parentConfidenceAfter: applicationResult.parentConfidenceAfter,
|
||||
evidenceConfidenceBefore: applicationResult.evidenceConfidenceBefore,
|
||||
evidenceConfidenceAfter: applicationResult.evidenceConfidenceAfter,
|
||||
completenessBefore: applicationResult.completenessBefore,
|
||||
completenessAfter: applicationResult.completenessAfter,
|
||||
conclusionConfidenceBefore:
|
||||
applicationResult.conclusionConfidenceBefore,
|
||||
conclusionConfidenceAfter: applicationResult.conclusionConfidenceAfter,
|
||||
resolvedDirectChildren: applicationResult.resolvedDirectChildren,
|
||||
unresolvedDirectChildren: applicationResult.unresolvedDirectChildren,
|
||||
contradictoryDirectChildren:
|
||||
applicationResult.contradictoryDirectChildren,
|
||||
corroboratingBranchCount: applicationResult.corroboratingBranchCount,
|
||||
conflictingBranchCount: applicationResult.conflictingBranchCount,
|
||||
duplicateEvidenceCount: applicationResult.duplicateEvidenceCount,
|
||||
independentBranchCount: applicationResult.independentBranchCount,
|
||||
interactionSummary: applicationResult.interactionSummary,
|
||||
confidenceCapReason: applicationResult.confidenceCapReason,
|
||||
ancestorPropagationStoppedReason:
|
||||
applicationResult.ancestorPropagationStoppedReason,
|
||||
affectedAncestorIds: applicationResult.affectedAncestorIds,
|
||||
nextSelectedSibling: applicationResult.nextSelectedSibling,
|
||||
parentResolved: applicationResult.parentResolved,
|
||||
decompositionPerformed: applicationResult.decompositionPerformed,
|
||||
childUnknownCount: applicationResult.childUnknownCount,
|
||||
childNodeIds: applicationResult.childNodeIds,
|
||||
atomicityReason: applicationResult.atomicityReason,
|
||||
questionComplexityAccepted:
|
||||
applicationResult.questionComplexityAccepted,
|
||||
primaryConceptCount: applicationResult.primaryConceptCount,
|
||||
cognitiveLoad: applicationResult.cognitiveLoad,
|
||||
complexityReasons: applicationResult.complexityReasons,
|
||||
decompositionTriggeredByQuestionComplexity:
|
||||
applicationResult.decompositionTriggeredByQuestionComplexity,
|
||||
previousQuestion: applicationResult.previousQuestion,
|
||||
finalQuestion: applicationResult.finalQuestion,
|
||||
selectedUnknownBefore: applicationResult.selectedUnknownBefore,
|
||||
selectedUnknownAfter: applicationResult.selectedUnknownAfter,
|
||||
unresolvedCandidateCount: applicationResult.unresolvedCandidateCount,
|
||||
eligibleCandidateCount: applicationResult.eligibleCandidateCount,
|
||||
candidateNodeIds: applicationResult.candidateNodeIds,
|
||||
resolvedCurrentTurnNodeIds:
|
||||
applicationResult.resolvedCurrentTurnNodeIds,
|
||||
noQuestionReason: applicationResult.noQuestionReason,
|
||||
reasoningPatternValidation:
|
||||
applicationResult.reasoningPatternValidation,
|
||||
patternCompatibleNodeCount:
|
||||
applicationResult.patternCompatibleNodeCount,
|
||||
incompatibleNodeIds: applicationResult.incompatibleNodeIds,
|
||||
compatibilityFailures: applicationResult.compatibilityFailures,
|
||||
replacementActions: applicationResult.replacementActions,
|
||||
graphReasoningIntegrity: applicationResult.graphReasoningIntegrity,
|
||||
plainLanguageNormalisations:
|
||||
applicationResult.plainLanguageNormalisations,
|
||||
reasoningPattern:
|
||||
applicationResult.selectedQuestion?.reasoningPattern ?? null,
|
||||
questionFamily:
|
||||
applicationResult.selectedQuestion?.questionFamily ?? null,
|
||||
allowedQuestionFamilies:
|
||||
applicationResult.selectedQuestion?.allowedQuestionFamilies ?? [],
|
||||
rejectedQuestionFamilies:
|
||||
applicationResult.selectedQuestion?.rejectedQuestionFamilies ?? [],
|
||||
selectedQuestionTemplate:
|
||||
applicationResult.selectedQuestion?.selectedQuestionTemplate ?? null,
|
||||
reasoningPatternReason:
|
||||
applicationResult.selectedQuestion?.reasoningPatternReason ?? null,
|
||||
unknownSelectionExplanation: buildUnknownSelectionDiagnostics(
|
||||
applicationResult.updatedSituationGraph,
|
||||
applicationResult.updatedSituationGraph.resolvedNodeIds || [],
|
||||
applicationResult.selectedQuestion,
|
||||
),
|
||||
}),
|
||||
};
|
||||
}
|
||||
@@ -327,6 +900,86 @@ async function updateCaseWithDependencies(body, dependencies = {}) {
|
||||
normalisationsApplied: parsedProposal.normalisationsApplied,
|
||||
graph: situationGraph,
|
||||
graphReferenceValidation,
|
||||
selectedQuestion: null,
|
||||
previousReasoningState: buildReasoningState(situationGraph),
|
||||
reasoningState: buildReasoningState(situationGraph),
|
||||
resolvedReasoningNodeIds: [],
|
||||
emergentReasoningNodeCreated: false,
|
||||
emergentReasoningNodeId: null,
|
||||
emergentReasoningNodeReason: null,
|
||||
atomicityAssessment: null,
|
||||
atomicityDecisionReason: null,
|
||||
decompositionDepth: 0,
|
||||
decompositionAttempted: false,
|
||||
decompositionAccepted: false,
|
||||
decompositionStoppedReason: null,
|
||||
proposedChildCount: 0,
|
||||
acceptedChildCount: 0,
|
||||
rejectedChildren: [],
|
||||
selectedChildNodeId: null,
|
||||
childQualitySummary: [],
|
||||
propagationPerformed: false,
|
||||
resolvedChildNodeId: null,
|
||||
parentNodeId: null,
|
||||
parentStatusBefore: null,
|
||||
parentStatusAfter: null,
|
||||
parentConfidenceBefore: null,
|
||||
parentConfidenceAfter: null,
|
||||
evidenceConfidenceBefore: null,
|
||||
evidenceConfidenceAfter: null,
|
||||
completenessBefore: null,
|
||||
completenessAfter: null,
|
||||
conclusionConfidenceBefore: null,
|
||||
conclusionConfidenceAfter: null,
|
||||
resolvedDirectChildren: 0,
|
||||
unresolvedDirectChildren: 0,
|
||||
contradictoryDirectChildren: 0,
|
||||
corroboratingBranchCount: 0,
|
||||
conflictingBranchCount: 0,
|
||||
duplicateEvidenceCount: 0,
|
||||
independentBranchCount: 0,
|
||||
interactionSummary: null,
|
||||
confidenceCapReason: null,
|
||||
ancestorPropagationStoppedReason: null,
|
||||
affectedAncestorIds: [],
|
||||
nextSelectedSibling: null,
|
||||
parentResolved: false,
|
||||
decompositionPerformed: false,
|
||||
childUnknownCount: 0,
|
||||
childNodeIds: [],
|
||||
atomicityReason: null,
|
||||
questionComplexityAccepted: null,
|
||||
primaryConceptCount: null,
|
||||
cognitiveLoad: null,
|
||||
complexityReasons: [],
|
||||
decompositionTriggeredByQuestionComplexity: false,
|
||||
previousQuestion,
|
||||
finalQuestion: null,
|
||||
selectedUnknownBefore: null,
|
||||
selectedUnknownAfter: null,
|
||||
unresolvedCandidateCount: 0,
|
||||
eligibleCandidateCount: 0,
|
||||
candidateNodeIds: [],
|
||||
resolvedCurrentTurnNodeIds: [],
|
||||
noQuestionReason: null,
|
||||
reasoningPatternValidation: null,
|
||||
patternCompatibleNodeCount: 0,
|
||||
incompatibleNodeIds: [],
|
||||
compatibilityFailures: [],
|
||||
replacementActions: [],
|
||||
graphReasoningIntegrity: null,
|
||||
plainLanguageNormalisations: [],
|
||||
reasoningPattern: null,
|
||||
questionFamily: null,
|
||||
allowedQuestionFamilies: [],
|
||||
rejectedQuestionFamilies: [],
|
||||
selectedQuestionTemplate: null,
|
||||
reasoningPatternReason: null,
|
||||
unknownSelectionExplanation: buildUnknownSelectionDiagnostics(
|
||||
situationGraph,
|
||||
situationGraph.resolvedNodeIds || [],
|
||||
null,
|
||||
),
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
+31
-12
@@ -68,6 +68,7 @@ The JSON object must contain exactly these top-level fields:
|
||||
- removedEdgeIds
|
||||
- resolvedUnknownNodeIds
|
||||
- affectedNodeIds
|
||||
- selectedQuestion
|
||||
|
||||
## Required Shapes
|
||||
- addedNodes: array of nodes using these exact keys:
|
||||
@@ -79,27 +80,45 @@ The JSON object must contain exactly these top-level fields:
|
||||
- removedEdgeIds: array of strings
|
||||
- resolvedUnknownNodeIds: array of strings
|
||||
- affectedNodeIds: array of strings
|
||||
- selectedQuestion: either null or an object using these exact keys:
|
||||
nodeId, question, reason
|
||||
|
||||
## Proposal Rules
|
||||
1. Propose changes only. Never return a replacement graph.
|
||||
2. Preserve unrelated nodes and edges by omitting them from the proposal.
|
||||
3. Reference existing node IDs when updating an existing concept.
|
||||
4. Use addedNodes only for genuinely new concepts.
|
||||
5. Resolve the active unknown when the answer supports it.
|
||||
6. Propagate only through explicit dependencies or relationships already present in the graph.
|
||||
7. Do not invent evidence.
|
||||
8. Do not create unsupported causal edges.
|
||||
9. Do not ask more than one next question. In this contract you are not returning any next-question field at all.
|
||||
10. Use empty arrays when there are no changes in a category.
|
||||
11. Never return null array entries.
|
||||
12. Never use unknown enum values.
|
||||
13. Do not change existing IDs.
|
||||
14. Do not replace the whole graph, and do not restate unchanged graph content inside the proposal.
|
||||
5. Resolve the answered unknown first when the answer supports it.
|
||||
6. Then inspect the answer for newly introduced consequential uncertainty.
|
||||
7. Add new unknown nodes only when the answer introduces a new decision, claim, object, measure, dependency, or unresolved term directly relevant to the case.
|
||||
8. Add at most 3 new unknown nodes.
|
||||
9. Every new unknown must be directly traceable to the user's answer and its description must state why that uncertainty matters.
|
||||
9a. In the description of every new unknown, explicitly include a short why-it-matters clause using wording such as because, so that, needed to decide, or matters because.
|
||||
10. Do not add broad generic discovery questions.
|
||||
11. Do not add duplicate unknowns.
|
||||
12. Do not expand unrelated branches.
|
||||
13. Propagate only through explicit dependencies or relationships already present in the graph, except for the minimal new edges needed to connect validated new unknowns to the relevant answer-derived decision or context node.
|
||||
13a. For every new unknown node, include at least one added edge that connects it to an existing updated/resolved node or to a newly added non-unknown node introduced from the answer.
|
||||
14. Do not invent evidence.
|
||||
15. Do not create unsupported causal edges.
|
||||
16. If consequential unresolved unknowns exist, selectedQuestion may identify one valid candidate unknown, but the engine will deterministically choose final priority after validation.
|
||||
17. selectedQuestion.nodeId must reference an unresolved unknown node that exists either already in the graph or in addedNodes.
|
||||
18. selectedQuestion.question must be one narrow non-compound question about that one unknown.
|
||||
19. Do not prioritise downstream implementation, pricing, optimisation, or speculative branches ahead of prerequisite definitions, actors, success criteria, constraints, measures, or terminology.
|
||||
20. Return selectedQuestion as null only when no consequential unresolved unknown remains.
|
||||
21. Use empty arrays when there are no changes in a category.
|
||||
22. Never return null array entries.
|
||||
23. Never use unknown enum values.
|
||||
24. Do not change existing IDs.
|
||||
25. Do not replace the whole graph, and do not restate unchanged graph content inside the proposal.
|
||||
|
||||
## Additional Guidance
|
||||
- If the answer only clarifies an existing unknown, prefer updatedNodes and resolvedUnknownNodeIds over creating duplicate nodes.
|
||||
- When an answer resolves an existing unknown, include that existing node ID in resolvedUnknownNodeIds and update that node rather than creating only a parallel observation.
|
||||
- If a new metric or observation is necessary, add the smallest set of nodes and edges needed.
|
||||
- If the answer creates a more specific decision situation, add the smallest set of new nodes and edges needed to represent that situation and only its most consequential unknowns.
|
||||
- If you add a new unknown, do not leave it floating: connect it with an added edge to the relevant decision/context node created or updated from the answer.
|
||||
- If you add a new unknown, its description must do two jobs in one sentence: what is unknown, and why resolving it matters for the case.
|
||||
- Treat selectedQuestion as a candidate only; the engine will apply deterministic information-value scoring after validation.
|
||||
- If the answer does not justify a change, return empty arrays for every category.
|
||||
|
||||
## Example Constraint Reminder
|
||||
@@ -108,7 +127,7 @@ ${formatExampleAnswerBlock()}
|
||||
## Output Contract Reminder
|
||||
Return one JSON object only, with exact field names and exact enum values.
|
||||
Never include a full graph.
|
||||
Never include a nextQuestion field.
|
||||
Never include any field other than the contract fields above.
|
||||
`;
|
||||
}
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+53
-2
@@ -36,6 +36,20 @@ export const ConfidenceLevel = /** @type {const} */ ({
|
||||
high: "high",
|
||||
});
|
||||
|
||||
export const CompletenessStatus = /** @type {const} */ ({
|
||||
empty: "empty",
|
||||
partial: "partial",
|
||||
complete: "complete",
|
||||
});
|
||||
|
||||
export const confidenceAssessmentSchema = z
|
||||
.object({
|
||||
evidenceConfidence: z.enum(Object.values(ConfidenceLevel)),
|
||||
completenessStatus: z.enum(Object.values(CompletenessStatus)),
|
||||
conclusionConfidence: z.enum(Object.values(ConfidenceLevel)),
|
||||
})
|
||||
.strict();
|
||||
|
||||
// ── SituationNode ────────────────────────────────────
|
||||
|
||||
export const situationNodeSchema = z.object({
|
||||
@@ -45,6 +59,7 @@ export const situationNodeSchema = z.object({
|
||||
kind: z.enum(Object.values(SituationKind)),
|
||||
status: z.enum(Object.values(SituationStatus)),
|
||||
confidence: z.enum(Object.values(ConfidenceLevel)),
|
||||
confidenceAssessment: confidenceAssessmentSchema.optional(),
|
||||
value: z.union([z.string(), z.number(), z.null()]).nullable().optional(),
|
||||
unit: z.string().nullable().optional(),
|
||||
evidenceIds: z.array(z.string()).default([]),
|
||||
@@ -84,6 +99,25 @@ export const situationEdgeSchema = z.object({
|
||||
|
||||
// ── SituationGraph ───────────────────────────────────
|
||||
|
||||
const reasoningStageSchema = z.object({
|
||||
stage: z.string().min(1),
|
||||
status: z.string().min(1),
|
||||
outcome: z.string().min(1),
|
||||
});
|
||||
|
||||
export const reasoningStateSchema = z
|
||||
.object({
|
||||
comparabilityStatus: z.string().min(1).nullable().optional(),
|
||||
comparabilityReason: z.string().min(1).nullable().optional(),
|
||||
comparabilityEvidence: z.array(z.string()).default([]),
|
||||
relationshipStatus: z.string().min(1).nullable().optional(),
|
||||
relationshipReason: z.string().min(1).nullable().optional(),
|
||||
relationshipAssessed: z.boolean().optional(),
|
||||
contradictionReasoningAllowed: z.boolean().optional(),
|
||||
reasoningStages: z.array(reasoningStageSchema).default([]),
|
||||
})
|
||||
.strict();
|
||||
|
||||
export const situationGraphSchema = z.object({
|
||||
centralStatement: z.string().min(1),
|
||||
nodes: z.array(situationNodeSchema).min(1),
|
||||
@@ -91,6 +125,7 @@ export const situationGraphSchema = z.object({
|
||||
activeUnknownNodeId: z.string().nullable(),
|
||||
resolvedNodeIds: z.array(z.string()).default([]),
|
||||
currentSummary: z.string().min(1),
|
||||
reasoningState: reasoningStateSchema.optional(),
|
||||
});
|
||||
|
||||
/** @typedef {z.infer<typeof situationGraphSchema>} SituationGraph */
|
||||
@@ -101,11 +136,22 @@ const graphUpdateNodeChangeSchema = z.object({
|
||||
nodeId: z.string().min(1),
|
||||
previousStatus: z.enum(Object.values(SituationStatus)).nullable().optional(),
|
||||
newStatus: z.enum(Object.values(SituationStatus)).nullable().optional(),
|
||||
previousValue: z.union([z.string(), z.number(), z.null()]).nullable().optional(),
|
||||
previousValue: z
|
||||
.union([z.string(), z.number(), z.null()])
|
||||
.nullable()
|
||||
.optional(),
|
||||
newValue: z.union([z.string(), z.number(), z.null()]).nullable().optional(),
|
||||
reason: z.string().min(1),
|
||||
});
|
||||
|
||||
export const selectedQuestionSchema = z
|
||||
.object({
|
||||
nodeId: z.string().min(1),
|
||||
question: z.string().min(1),
|
||||
reason: z.string().min(1),
|
||||
})
|
||||
.strict();
|
||||
|
||||
export const graphUpdateSchema = z.object({
|
||||
addedNodes: z.array(situationNodeSchema).default([]),
|
||||
updatedNodes: z.array(graphUpdateNodeChangeSchema).default([]),
|
||||
@@ -113,6 +159,7 @@ export const graphUpdateSchema = z.object({
|
||||
removedEdgeIds: z.array(z.string()).default([]),
|
||||
resolvedUnknownNodeIds: z.array(z.string()).default([]),
|
||||
affectedNodeIds: z.array(z.string()).default([]),
|
||||
selectedQuestion: selectedQuestionSchema.nullable().default(null),
|
||||
});
|
||||
|
||||
/** @typedef {z.infer<typeof graphUpdateSchema>} GraphUpdate */
|
||||
@@ -156,6 +203,7 @@ export function makeNode(opts) {
|
||||
kind: opts.kind ?? "observation",
|
||||
status: opts.status ?? "unknown",
|
||||
confidence: opts.confidence ?? "medium",
|
||||
confidenceAssessment: opts.confidenceAssessment,
|
||||
value: opts.value ?? null,
|
||||
unit: opts.unit ?? null,
|
||||
evidenceIds: opts.evidenceIds ?? [],
|
||||
@@ -169,7 +217,9 @@ export function makeNode(opts) {
|
||||
/** Create a minimal valid edge — used in tests and fixtures */
|
||||
export function makeEdge(opts) {
|
||||
return situationEdgeSchema.parse({
|
||||
id: opts.id || "e" + opts.fromNodeId.slice(0,3) + "-" + opts.toNodeId.slice(0,3),
|
||||
id:
|
||||
opts.id ||
|
||||
"e" + opts.fromNodeId.slice(0, 3) + "-" + opts.toNodeId.slice(0, 3),
|
||||
fromNodeId: opts.fromNodeId,
|
||||
toNodeId: opts.toNodeId,
|
||||
relationship: opts.relationship ?? "supports",
|
||||
@@ -187,5 +237,6 @@ export function makeGraph(opts) {
|
||||
activeUnknownNodeId: opts.activeUnknownNodeId ?? null,
|
||||
resolvedNodeIds: opts.resolvedNodeIds ?? [],
|
||||
currentSummary: opts.currentSummary || "",
|
||||
reasoningState: opts.reasoningState,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -9,6 +9,8 @@ const TOP_LEVEL_ARRAY_FIELDS = [
|
||||
"affectedNodeIds",
|
||||
];
|
||||
|
||||
const TOP_LEVEL_NULLABLE_FIELDS = ["selectedQuestion"];
|
||||
|
||||
function cloneJsonSafe(value) {
|
||||
if (value == null) return value;
|
||||
return JSON.parse(JSON.stringify(value));
|
||||
@@ -79,6 +81,22 @@ function fillMissingOptionalArrays(proposal, normalisationsApplied) {
|
||||
return proposal;
|
||||
}
|
||||
|
||||
function fillMissingNullableFields(proposal, normalisationsApplied) {
|
||||
if (!proposal || typeof proposal !== "object") return proposal;
|
||||
|
||||
for (const field of TOP_LEVEL_NULLABLE_FIELDS) {
|
||||
if (!(field in proposal)) {
|
||||
proposal[field] = null;
|
||||
normalisationsApplied.push({
|
||||
path: [field],
|
||||
change: "Filled missing optional nullable field with null",
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return proposal;
|
||||
}
|
||||
|
||||
export function parseGraphUpdateProposal(rawResponse) {
|
||||
const raw = rawResponse;
|
||||
let parsed;
|
||||
@@ -111,6 +129,7 @@ export function parseGraphUpdateProposal(rawResponse) {
|
||||
let normalised = removeNullArrayEntries(parsed, [], normalisationsApplied);
|
||||
normalised = applyKnownEnumAliases(normalised, normalisationsApplied);
|
||||
normalised = fillMissingOptionalArrays(normalised, normalisationsApplied);
|
||||
normalised = fillMissingNullableFields(normalised, normalisationsApplied);
|
||||
|
||||
const parsedProposal = graphUpdateSchema.safeParse(normalised);
|
||||
|
||||
|
||||
+609
-63
@@ -5,26 +5,416 @@
|
||||
* and these utilities apply them safely.
|
||||
*/
|
||||
|
||||
import { situationNodeSchema, situationEdgeSchema, situationGraphSchema } from "./schema.js";
|
||||
import {
|
||||
situationNodeSchema,
|
||||
situationEdgeSchema,
|
||||
situationGraphSchema,
|
||||
} from "./schema.js";
|
||||
|
||||
function normaliseText(value) {
|
||||
return String(value || "")
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, " ")
|
||||
.trim();
|
||||
}
|
||||
|
||||
function collectNodeText(node) {
|
||||
return `${node?.label || ""} ${node?.description || ""}`.trim();
|
||||
}
|
||||
|
||||
function countIncomingUnknownDependencies(graph, nodeId, resolvedNodeIds) {
|
||||
const resolvedSet = new Set(resolvedNodeIds || []);
|
||||
const nodesById = new Map(graph.nodes.map((node) => [node.id, node]));
|
||||
const incoming = new Set();
|
||||
|
||||
for (const dependencyId of nodesById.get(nodeId)?.dependsOn || []) {
|
||||
const dependencyNode = nodesById.get(dependencyId);
|
||||
if (dependencyNode?.kind === "unknown" && !resolvedSet.has(dependencyId)) {
|
||||
incoming.add(dependencyId);
|
||||
}
|
||||
}
|
||||
|
||||
for (const edge of graph.edges) {
|
||||
if (edge.toNodeId !== nodeId) continue;
|
||||
const dependencyNode = nodesById.get(edge.fromNodeId);
|
||||
if (
|
||||
dependencyNode?.kind === "unknown" &&
|
||||
!resolvedSet.has(edge.fromNodeId)
|
||||
) {
|
||||
incoming.add(edge.fromNodeId);
|
||||
}
|
||||
}
|
||||
|
||||
return incoming.size;
|
||||
}
|
||||
|
||||
function classifyUnknownPriority(text) {
|
||||
const normalised = normaliseText(text);
|
||||
|
||||
const matches = {
|
||||
objective:
|
||||
/\b(objective|goal|outcome|value|problem|job to be done|benefit|commercial value)\b/.test(
|
||||
normalised,
|
||||
),
|
||||
actor:
|
||||
/\b(customer|user|buyer|actor|stakeholder|audience|recipient)\b/.test(
|
||||
normalised,
|
||||
),
|
||||
criteria:
|
||||
/\b(success criteria|success threshold|threshold|decision criteria|criterion|justify|sufficient)\b/.test(
|
||||
normalised,
|
||||
),
|
||||
measure:
|
||||
/\b(metric|measure|measurable|roi|demand|evidence|signal|proof)\b/.test(
|
||||
normalised,
|
||||
),
|
||||
terminology: /\b(define|definition|meaning|means|term|terminology)\b/.test(
|
||||
normalised,
|
||||
),
|
||||
constraint:
|
||||
/\b(constraint|limit|budget|deadline|requirement|regulation)\b/.test(
|
||||
normalised,
|
||||
),
|
||||
pricing: /\b(price|pricing|price point|subscription|charge|pay for)\b/.test(
|
||||
normalised,
|
||||
),
|
||||
implementation:
|
||||
/\b(implementation|build approach|architecture|stack|feature|technical design)\b/.test(
|
||||
normalised,
|
||||
),
|
||||
optimisation:
|
||||
/\b(optimisation|optimi[sz]ation|improve|efficiency|performance|scale)\b/.test(
|
||||
normalised,
|
||||
),
|
||||
speculative:
|
||||
/\b(maybe|possible|optional|future branch|nice to have|slogan|colour|color|ui)\b/.test(
|
||||
normalised,
|
||||
),
|
||||
};
|
||||
|
||||
return matches;
|
||||
}
|
||||
|
||||
function buildScoreContributions(
|
||||
matches,
|
||||
downstreamCount,
|
||||
unresolvedParentUnknownCount,
|
||||
) {
|
||||
const contributions = [
|
||||
{
|
||||
rule: "downstream_dependencies",
|
||||
value: downstreamCount,
|
||||
weight: 4,
|
||||
delta: downstreamCount * 4,
|
||||
},
|
||||
];
|
||||
|
||||
if (matches.objective) {
|
||||
contributions.push({
|
||||
rule: "objective_match",
|
||||
value: true,
|
||||
weight: 12,
|
||||
delta: 12,
|
||||
});
|
||||
}
|
||||
if (matches.actor) {
|
||||
contributions.push({
|
||||
rule: "actor_match",
|
||||
value: true,
|
||||
weight: 10,
|
||||
delta: 10,
|
||||
});
|
||||
}
|
||||
if (matches.criteria) {
|
||||
contributions.push({
|
||||
rule: "criteria_match",
|
||||
value: true,
|
||||
weight: 11,
|
||||
delta: 11,
|
||||
});
|
||||
}
|
||||
if (matches.measure) {
|
||||
contributions.push({
|
||||
rule: "measure_match",
|
||||
value: true,
|
||||
weight: 8,
|
||||
delta: 8,
|
||||
});
|
||||
}
|
||||
if (matches.terminology) {
|
||||
contributions.push({
|
||||
rule: "terminology_match",
|
||||
value: true,
|
||||
weight: 7,
|
||||
delta: 7,
|
||||
});
|
||||
}
|
||||
if (matches.constraint) {
|
||||
contributions.push({
|
||||
rule: "constraint_match",
|
||||
value: true,
|
||||
weight: 9,
|
||||
delta: 9,
|
||||
});
|
||||
}
|
||||
if (matches.pricing) {
|
||||
contributions.push({
|
||||
rule: "pricing_penalty",
|
||||
value: true,
|
||||
weight: -8,
|
||||
delta: -8,
|
||||
});
|
||||
}
|
||||
if (matches.implementation) {
|
||||
contributions.push({
|
||||
rule: "implementation_penalty",
|
||||
value: true,
|
||||
weight: -10,
|
||||
delta: -10,
|
||||
});
|
||||
}
|
||||
if (matches.optimisation) {
|
||||
contributions.push({
|
||||
rule: "optimisation_penalty",
|
||||
value: true,
|
||||
weight: -9,
|
||||
delta: -9,
|
||||
});
|
||||
}
|
||||
if (matches.speculative) {
|
||||
contributions.push({
|
||||
rule: "speculative_penalty",
|
||||
value: true,
|
||||
weight: -12,
|
||||
delta: -12,
|
||||
});
|
||||
}
|
||||
|
||||
if (
|
||||
matches.pricing &&
|
||||
!matches.objective &&
|
||||
!matches.criteria &&
|
||||
!matches.actor
|
||||
) {
|
||||
contributions.push({
|
||||
rule: "isolated_pricing_penalty",
|
||||
value: true,
|
||||
weight: -6,
|
||||
delta: -6,
|
||||
});
|
||||
}
|
||||
|
||||
if (unresolvedParentUnknownCount > 0) {
|
||||
contributions.push({
|
||||
rule: "unresolved_prerequisite_penalty",
|
||||
value: unresolvedParentUnknownCount,
|
||||
weight: -7,
|
||||
delta: unresolvedParentUnknownCount * -7,
|
||||
});
|
||||
}
|
||||
|
||||
return contributions;
|
||||
}
|
||||
|
||||
function getMeaningfulSemanticContributions(contributions = []) {
|
||||
return contributions
|
||||
.filter(
|
||||
(contribution) =>
|
||||
contribution.rule !== "downstream_dependencies" &&
|
||||
contribution.rule !== "unresolved_prerequisite_penalty" &&
|
||||
contribution.delta !== 0,
|
||||
)
|
||||
.map((contribution) => ({
|
||||
rule: contribution.rule,
|
||||
delta: contribution.delta,
|
||||
}));
|
||||
}
|
||||
|
||||
function buildCandidateDisplayOrder(candidates) {
|
||||
return [...candidates].sort((a, b) => {
|
||||
if (b.score !== a.score) return b.score - a.score;
|
||||
if (b.downstreamCount !== a.downstreamCount) {
|
||||
return b.downstreamCount - a.downstreamCount;
|
||||
}
|
||||
if (a.unresolvedParentUnknownCount !== b.unresolvedParentUnknownCount) {
|
||||
return a.unresolvedParentUnknownCount - b.unresolvedParentUnknownCount;
|
||||
}
|
||||
return a.label.localeCompare(b.label);
|
||||
});
|
||||
}
|
||||
|
||||
function semanticSignature(candidate) {
|
||||
return JSON.stringify(
|
||||
getMeaningfulSemanticContributions(candidate.contributions),
|
||||
);
|
||||
}
|
||||
|
||||
function classifyCandidateOrdering(candidates) {
|
||||
const displayOrder = buildCandidateDisplayOrder(candidates);
|
||||
const best = displayOrder[0] ?? null;
|
||||
if (!best) {
|
||||
return {
|
||||
displayOrder,
|
||||
best: null,
|
||||
leadingCandidates: [],
|
||||
status: "no_candidates",
|
||||
tieType: "none",
|
||||
usedAlphabeticalOrdering: false,
|
||||
reason: "No unresolved unknown candidates remain.",
|
||||
};
|
||||
}
|
||||
|
||||
const topScoreCandidates = displayOrder.filter(
|
||||
(candidate) => candidate.score === best.score,
|
||||
);
|
||||
|
||||
if (topScoreCandidates.length === 1) {
|
||||
return {
|
||||
displayOrder,
|
||||
best,
|
||||
leadingCandidates: [best],
|
||||
status: "selected",
|
||||
tieType: "none",
|
||||
usedAlphabeticalOrdering: false,
|
||||
reason: `Clear winner by total score (${best.score}).`,
|
||||
};
|
||||
}
|
||||
|
||||
const topStructuralCandidates = topScoreCandidates.filter(
|
||||
(candidate) =>
|
||||
candidate.downstreamCount === best.downstreamCount &&
|
||||
candidate.unresolvedParentUnknownCount ===
|
||||
best.unresolvedParentUnknownCount,
|
||||
);
|
||||
|
||||
if (topStructuralCandidates.length === 1) {
|
||||
return {
|
||||
displayOrder,
|
||||
best,
|
||||
leadingCandidates: [best],
|
||||
status: "selected",
|
||||
tieType: "structural_tie",
|
||||
usedAlphabeticalOrdering: false,
|
||||
reason:
|
||||
"Score tie was resolved by downstream dependency count or prerequisite ordering.",
|
||||
};
|
||||
}
|
||||
|
||||
const topSemanticSignature = semanticSignature(best);
|
||||
const semanticPeers = topStructuralCandidates.filter(
|
||||
(candidate) => semanticSignature(candidate) === topSemanticSignature,
|
||||
);
|
||||
|
||||
if (semanticPeers.length !== topStructuralCandidates.length) {
|
||||
return {
|
||||
displayOrder,
|
||||
best: null,
|
||||
leadingCandidates: topStructuralCandidates,
|
||||
status: "ambiguous",
|
||||
tieType: "semantic_tie",
|
||||
usedAlphabeticalOrdering: false,
|
||||
reason:
|
||||
"Leading candidates remain tied after score and structural checks, but differ in semantic contribution patterns.",
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
displayOrder,
|
||||
best: null,
|
||||
leadingCandidates: topStructuralCandidates,
|
||||
status: "ambiguous",
|
||||
tieType: "complete_unresolved_tie",
|
||||
usedAlphabeticalOrdering: false,
|
||||
reason: "No justified distinction between leading unknowns.",
|
||||
};
|
||||
}
|
||||
|
||||
export function scoreUnknownCandidate(graph, node, resolvedNodeIds = []) {
|
||||
const text = collectNodeText(node);
|
||||
const matches = classifyUnknownPriority(text);
|
||||
const downstreamCount = findDependentNodes(graph, node.id).length;
|
||||
const unresolvedParentUnknownCount = countIncomingUnknownDependencies(
|
||||
graph,
|
||||
node.id,
|
||||
resolvedNodeIds,
|
||||
);
|
||||
|
||||
const contributions = buildScoreContributions(
|
||||
matches,
|
||||
downstreamCount,
|
||||
unresolvedParentUnknownCount,
|
||||
);
|
||||
const score = contributions.reduce(
|
||||
(total, contribution) => total + contribution.delta,
|
||||
0,
|
||||
);
|
||||
|
||||
return {
|
||||
nodeId: node.id,
|
||||
label: node.label,
|
||||
score,
|
||||
downstreamCount,
|
||||
unresolvedParentUnknownCount,
|
||||
matches,
|
||||
contributions,
|
||||
};
|
||||
}
|
||||
|
||||
export function buildDeterministicQuestionForUnknown(node) {
|
||||
const text = normaliseText(collectNodeText(node));
|
||||
|
||||
if (
|
||||
/\b(success criteria|success threshold|threshold|decision criteria|criterion)\b/.test(
|
||||
text,
|
||||
)
|
||||
) {
|
||||
return `What outcome would define success for ${node.label}?`;
|
||||
}
|
||||
if (
|
||||
/\b(customer|user|buyer|actor|stakeholder|audience|recipient)\b/.test(text)
|
||||
) {
|
||||
return `Who is the key actor or customer for ${node.label}?`;
|
||||
}
|
||||
if (
|
||||
/\b(define|definition|meaning|means|term|terminology|value)\b/.test(text)
|
||||
) {
|
||||
return `How should ${node.label} be defined for this decision?`;
|
||||
}
|
||||
if (
|
||||
/\b(metric|measure|measurable|roi|demand|evidence|signal|proof)\b/.test(
|
||||
text,
|
||||
)
|
||||
) {
|
||||
return `What evidence or measure would resolve ${node.label}?`;
|
||||
}
|
||||
|
||||
return `What would resolve ${node.label}?`;
|
||||
}
|
||||
|
||||
// ── Validate that all edge references point to existing nodes ──
|
||||
|
||||
export function validateGraphReferences(graph) {
|
||||
const errors = [];
|
||||
const nodeIds = new Set(graph.nodes.map((n) => n.id));
|
||||
|
||||
|
||||
for (const node of graph.nodes) {
|
||||
if (node.parentId !== null && !nodeIds.has(node.parentId)) {
|
||||
errors.push(`Node "${node.id}" references parentId "${node.parentId}" which does not exist`);
|
||||
errors.push(
|
||||
`Node "${node.id}" references parentId "${node.parentId}" which does not exist`,
|
||||
);
|
||||
}
|
||||
for (const cid of node.childIds) {
|
||||
if (!nodeIds.has(cid)) {
|
||||
errors.push(`Node "${node.id}" references childIds "${cid}" which does not exist`);
|
||||
errors.push(
|
||||
`Node "${node.id}" references childIds "${cid}" which does not exist`,
|
||||
);
|
||||
}
|
||||
}
|
||||
for (const dep of node.dependsOn) {
|
||||
if (!nodeIds.has(dep)) {
|
||||
errors.push(`Node "${node.id}" depends on "${dep}" which does not exist`);
|
||||
errors.push(
|
||||
`Node "${node.id}" depends on "${dep}" which does not exist`,
|
||||
);
|
||||
}
|
||||
}
|
||||
for (const aff of node.affects) {
|
||||
@@ -33,16 +423,20 @@ export function validateGraphReferences(graph) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
for (const edge of graph.edges) {
|
||||
if (!nodeIds.has(edge.fromNodeId)) {
|
||||
errors.push(`Edge "${edge.id}" references non-existent fromNodeId "${edge.fromNodeId}"`);
|
||||
errors.push(
|
||||
`Edge "${edge.id}" references non-existent fromNodeId "${edge.fromNodeId}"`,
|
||||
);
|
||||
}
|
||||
if (!nodeIds.has(edge.toNodeId)) {
|
||||
errors.push(`Edge "${edge.id}" references non-existent toNodeId "${edge.toNodeId}"`);
|
||||
errors.push(
|
||||
`Edge "${edge.id}" references non-existent toNodeId "${edge.toNodeId}"`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
return { valid: errors.length === 0, errors };
|
||||
}
|
||||
|
||||
@@ -76,24 +470,31 @@ export function detectDuplicateNodeIds(nodes) {
|
||||
export function detectDuplicateEdges(edges) {
|
||||
const seen = new Set();
|
||||
const duplicates = [];
|
||||
|
||||
|
||||
for (const edge of edges) {
|
||||
const key = `${edge.fromNodeId}->${edge.toNodeId}:${edge.relationship}`;
|
||||
if (seen.has(key)) {
|
||||
duplicates.push({ edgeId: edge.id, fromNodeId: edge.fromNodeId, toNodeId: edge.toNodeId, relationship: edge.relationship });
|
||||
duplicates.push({
|
||||
edgeId: edge.id,
|
||||
fromNodeId: edge.fromNodeId,
|
||||
toNodeId: edge.toNodeId,
|
||||
relationship: edge.relationship,
|
||||
});
|
||||
}
|
||||
seen.add(key);
|
||||
}
|
||||
|
||||
|
||||
return duplicates;
|
||||
}
|
||||
|
||||
// ── Find all nodes that depend on a given node (transitive) ──
|
||||
|
||||
export function findDependentNodes(graph, nodeId) {
|
||||
const direct = graph.nodes.filter((n) => n.dependsOn.includes(nodeId)).map((n) => n.id);
|
||||
const direct = graph.nodes
|
||||
.filter((n) => n.dependsOn.includes(nodeId))
|
||||
.map((n) => n.id);
|
||||
const affected = new Set(direct);
|
||||
|
||||
|
||||
// Also propagate through edges where the relationship is depends_on
|
||||
for (const edge of graph.edges) {
|
||||
if (edge.toNodeId === nodeId && !affected.has(edge.fromNodeId)) {
|
||||
@@ -101,13 +502,13 @@ export function findDependentNodes(graph, nodeId) {
|
||||
affected.add(edge.fromNodeId);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Transitive propagation — BFS
|
||||
const queue = [...direct];
|
||||
while (queue.length > 0) {
|
||||
const current = queue.shift();
|
||||
if (!current || !affected.has(current)) continue;
|
||||
|
||||
|
||||
for (const node of graph.nodes) {
|
||||
if (node.dependsOn.includes(current) && !affected.has(node.id)) {
|
||||
affected.add(node.id);
|
||||
@@ -115,7 +516,7 @@ export function findDependentNodes(graph, nodeId) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
return [...affected];
|
||||
}
|
||||
|
||||
@@ -124,10 +525,14 @@ export function findDependentNodes(graph, nodeId) {
|
||||
export function findAffectedNodes(graph, nodeId) {
|
||||
// Direct effects: two sources
|
||||
// 1. Nodes that depend on this node (they list it in their dependsOn)
|
||||
const directFromDepends = graph.nodes.filter((n) => n.id !== nodeId && n.dependsOn.includes(nodeId)).map((n) => n.id);
|
||||
const directFromDepends = graph.nodes
|
||||
.filter((n) => n.id !== nodeId && n.dependsOn.includes(nodeId))
|
||||
.map((n) => n.id);
|
||||
|
||||
// 2. Targets of the node's affects relationships (this node directly affects them)
|
||||
const myAffectedTargets = new Set(graph.nodes.find((n) => n.id === nodeId)?.affects || []);
|
||||
const myAffectedTargets = new Set(
|
||||
graph.nodes.find((n) => n.id === nodeId)?.affects || [],
|
||||
);
|
||||
|
||||
// Merge: also add edge targets where this node is the source
|
||||
for (const edge of graph.edges) {
|
||||
@@ -147,7 +552,11 @@ export function findAffectedNodes(graph, nodeId) {
|
||||
if (!current || !affected.has(current)) continue;
|
||||
|
||||
for (const node of graph.nodes) {
|
||||
if (node.id !== nodeId && !affected.has(node.id) && (node.dependsOn.includes(current) || node.affects.includes(current))) {
|
||||
if (
|
||||
node.id !== nodeId &&
|
||||
!affected.has(node.id) &&
|
||||
(node.dependsOn.includes(current) || node.affects.includes(current))
|
||||
) {
|
||||
affected.add(node.id);
|
||||
queue.push(node.id);
|
||||
}
|
||||
@@ -164,10 +573,10 @@ export function resolveUnknownNode(graph, nodeId, newStatus, newValue, reason) {
|
||||
if (nodeIdx === -1) {
|
||||
return { success: false, error: `Node "${nodeId}" not found in graph` };
|
||||
}
|
||||
|
||||
|
||||
const previousStatus = graph.nodes[nodeIdx].status;
|
||||
const previousValue = graph.nodes[nodeIdx].value;
|
||||
|
||||
|
||||
return {
|
||||
success: true,
|
||||
previousStatus,
|
||||
@@ -184,26 +593,158 @@ export function resolveUnknownNode(graph, nodeId, newStatus, newValue, reason) {
|
||||
export function selectActiveUnknownCandidate(graph, resolvedNodeIds) {
|
||||
// Skip already resolved nodes
|
||||
const unresolved = graph.nodes.filter(
|
||||
(n) => n.kind === "unknown" && !resolvedNodeIds.includes(n.id)
|
||||
(n) => n.kind === "unknown" && !resolvedNodeIds.includes(n.id),
|
||||
);
|
||||
|
||||
|
||||
if (unresolved.length === 0) return null;
|
||||
|
||||
// Prioritise: critical unknowns first, then those that are depended upon most
|
||||
const dependencyCount = unresolved.map((n) => {
|
||||
const deps = findDependentNodes(graph, n.id).length;
|
||||
const importanceOrder = { critical: 3, important: 2, supporting: 1, incidental: 0 };
|
||||
const impScore = importanceOrder[n.confidence] || 0;
|
||||
return { node: n, score: deps * 2 + impScore };
|
||||
});
|
||||
|
||||
dependencyCount.sort((a, b) => b.score - a.score);
|
||||
|
||||
// Return the highest-scoring unresolved unknown
|
||||
const best = dependencyCount[0];
|
||||
|
||||
const scoredCandidates = unresolved.map((node) => ({
|
||||
node,
|
||||
...scoreUnknownCandidate(graph, node, resolvedNodeIds),
|
||||
}));
|
||||
|
||||
const selection = classifyCandidateOrdering(
|
||||
scoredCandidates.map(({ node, ...candidate }) => ({
|
||||
...candidate,
|
||||
node,
|
||||
})),
|
||||
);
|
||||
|
||||
if (selection.status === "ambiguous") {
|
||||
return {
|
||||
selectedNode: null,
|
||||
status: "ambiguous",
|
||||
tieType: selection.tieType,
|
||||
tiedCandidateIds: selection.leadingCandidates.map(
|
||||
(candidate) => candidate.nodeId,
|
||||
),
|
||||
displayOrder: selection.displayOrder.map((candidate) => candidate.nodeId),
|
||||
reason: selection.reason,
|
||||
};
|
||||
}
|
||||
|
||||
const best = selection.best;
|
||||
if (!best) return null;
|
||||
|
||||
return { nodeId: best.node.id, label: best.node.label, score: best.score };
|
||||
|
||||
return {
|
||||
selectedNode: {
|
||||
nodeId: best.node.id,
|
||||
label: best.node.label,
|
||||
},
|
||||
status: "selected",
|
||||
tieType: selection.tieType,
|
||||
nodeId: best.node.id,
|
||||
label: best.node.label,
|
||||
score: best.score,
|
||||
question: buildDeterministicQuestionForUnknown(best.node),
|
||||
reason: `Selected for highest information value (score ${best.score}) with ${best.downstreamCount} downstream dependency node(s) and ${best.unresolvedParentUnknownCount} unresolved prerequisite unknown(s).`,
|
||||
};
|
||||
}
|
||||
|
||||
export function explainUnknownSelection(graph, resolvedNodeIds = []) {
|
||||
const unresolved = graph.nodes.filter(
|
||||
(n) => n.kind === "unknown" && !resolvedNodeIds.includes(n.id),
|
||||
);
|
||||
|
||||
if (unresolved.length === 0) {
|
||||
return {
|
||||
selectedNodeId: null,
|
||||
selectedNodeLabel: null,
|
||||
status: "no_candidates",
|
||||
tieType: "none",
|
||||
resolvedNodeIds: [...resolvedNodeIds],
|
||||
tiedCandidateIds: [],
|
||||
candidates: [],
|
||||
competitors: [],
|
||||
tieBreakOrder: [
|
||||
"score_desc",
|
||||
"downstreamCount_desc",
|
||||
"unresolvedParentUnknownCount_asc",
|
||||
"label_asc",
|
||||
],
|
||||
summary: {
|
||||
candidateCount: 0,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const candidates = unresolved.map((node) => ({
|
||||
nodeId: node.id,
|
||||
label: node.label,
|
||||
...scoreUnknownCandidate(graph, node, resolvedNodeIds),
|
||||
}));
|
||||
|
||||
const selection = classifyCandidateOrdering(candidates);
|
||||
const orderedCandidates = selection.displayOrder;
|
||||
const selected = selection.best;
|
||||
const competitors = orderedCandidates
|
||||
.filter((candidate) => candidate.nodeId !== selected?.nodeId)
|
||||
.map((candidate) => ({
|
||||
nodeId: candidate.nodeId,
|
||||
label: candidate.label,
|
||||
score: candidate.score,
|
||||
downstreamCount: candidate.downstreamCount,
|
||||
unresolvedParentUnknownCount: candidate.unresolvedParentUnknownCount,
|
||||
matches: candidate.matches,
|
||||
contributions: candidate.contributions,
|
||||
outrankedBy: {
|
||||
scoreDelta: (selected?.score ?? candidate.score) - candidate.score,
|
||||
downstreamDelta:
|
||||
(selected?.downstreamCount ?? candidate.downstreamCount) -
|
||||
candidate.downstreamCount,
|
||||
unresolvedPrerequisiteDelta:
|
||||
candidate.unresolvedParentUnknownCount -
|
||||
(selected?.unresolvedParentUnknownCount ??
|
||||
candidate.unresolvedParentUnknownCount),
|
||||
labelOrderWinner:
|
||||
selected &&
|
||||
selected.score === candidate.score &&
|
||||
selected.downstreamCount === candidate.downstreamCount &&
|
||||
selected.unresolvedParentUnknownCount ===
|
||||
candidate.unresolvedParentUnknownCount
|
||||
? selected.label.localeCompare(candidate.label) <= 0
|
||||
? selected.label
|
||||
: candidate.label
|
||||
: null,
|
||||
},
|
||||
}));
|
||||
|
||||
return {
|
||||
selectedNodeId: selected?.nodeId ?? null,
|
||||
selectedNodeLabel: selected?.label ?? null,
|
||||
status: selection.status,
|
||||
tieType: selection.tieType,
|
||||
resolvedNodeIds: [...resolvedNodeIds],
|
||||
tiedCandidateIds: selection.leadingCandidates.map(
|
||||
(candidate) => candidate.nodeId,
|
||||
),
|
||||
tieBreakOrder: [
|
||||
"score_desc",
|
||||
"downstreamCount_desc",
|
||||
"unresolvedParentUnknownCount_asc",
|
||||
"label_asc",
|
||||
],
|
||||
alphabeticalUsedAsReasoning: false,
|
||||
candidates: orderedCandidates,
|
||||
selected: selected
|
||||
? {
|
||||
nodeId: selected.nodeId,
|
||||
label: selected.label,
|
||||
score: selected.score,
|
||||
downstreamCount: selected.downstreamCount,
|
||||
unresolvedParentUnknownCount: selected.unresolvedParentUnknownCount,
|
||||
matches: selected.matches,
|
||||
contributions: selected.contributions,
|
||||
}
|
||||
: null,
|
||||
competitors,
|
||||
summary: {
|
||||
candidateCount: orderedCandidates.length,
|
||||
selectedReason: selected
|
||||
? `highest_score=${selected.score}; downstream=${selected.downstreamCount}; unresolved_prerequisites=${selected.unresolvedParentUnknownCount}`
|
||||
: selection.reason,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ── Apply a graph update deterministically ──
|
||||
@@ -211,7 +752,7 @@ export function selectActiveUnknownCandidate(graph, resolvedNodeIds) {
|
||||
export function applyGraphUpdate(graph, update) {
|
||||
const errors = [];
|
||||
const updatedNodesMap = new Map();
|
||||
|
||||
|
||||
// Validate that update references existing nodes or newly added ones
|
||||
const allNodeIds = new Set(graph.nodes.map((n) => n.id));
|
||||
for (const added of update.addedNodes) {
|
||||
@@ -221,34 +762,38 @@ export function applyGraphUpdate(graph, update) {
|
||||
}
|
||||
allNodeIds.add(added.id);
|
||||
}
|
||||
|
||||
|
||||
// Validate updated nodes exist
|
||||
for (const upd of update.updatedNodes) {
|
||||
if (!allNodeIds.has(upd.nodeId)) {
|
||||
errors.push(`Cannot update non-existent node: "${upd.nodeId}"`);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Validate added edges reference existing or new nodes
|
||||
for (const edge of update.addedEdges) {
|
||||
if (!allNodeIds.has(edge.fromNodeId)) {
|
||||
errors.push(`Added edge references non-existent fromNodeId: "${edge.fromNodeId}"`);
|
||||
errors.push(
|
||||
`Added edge references non-existent fromNodeId: "${edge.fromNodeId}"`,
|
||||
);
|
||||
}
|
||||
if (!allNodeIds.has(edge.toNodeId)) {
|
||||
errors.push(`Added edge references non-existent toNodeId: "${edge.toNodeId}"`);
|
||||
errors.push(
|
||||
`Added edge references non-existent toNodeId: "${edge.toNodeId}"`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
if (errors.length > 0) return { success: false, errors };
|
||||
|
||||
|
||||
// Build the new nodes list — start with a deep copy of existing
|
||||
const newNodes = graph.nodes.map((n) => ({ ...n }));
|
||||
|
||||
|
||||
// Apply updated nodes
|
||||
for (const upd of update.updatedNodes) {
|
||||
const idx = newNodes.findIndex((n) => n.id === upd.nodeId);
|
||||
if (idx === -1) continue; // already validated above
|
||||
|
||||
|
||||
if (upd.newStatus !== undefined && upd.newStatus !== null) {
|
||||
newNodes[idx].status = upd.newStatus;
|
||||
}
|
||||
@@ -257,22 +802,22 @@ export function applyGraphUpdate(graph, update) {
|
||||
}
|
||||
updatedNodesMap.set(upd.nodeId, newNodes[idx]);
|
||||
}
|
||||
|
||||
|
||||
// Add new nodes
|
||||
for (const newNode of update.addedNodes) {
|
||||
if (!allNodeIds.has(newNode.id)) continue;
|
||||
allNodeIds.add(newNode.id);
|
||||
newNodes.push({ ...newNode });
|
||||
}
|
||||
|
||||
|
||||
// Remove edges if requested
|
||||
const removedEdgeSet = new Set(update.removedEdgeIds);
|
||||
const newEdges = graph.edges.filter((e) => !removedEdgeSet.has(e.id));
|
||||
|
||||
|
||||
// Add new edges
|
||||
for (const newEdge of update.addedEdges) {
|
||||
newEdges.push({ ...newEdge });
|
||||
|
||||
|
||||
// Update dependsOn / affects on the nodes
|
||||
const fromNode = newNodes.find((n) => n.id === newEdge.fromNodeId);
|
||||
const toNode = newNodes.find((n) => n.id === newEdge.toNodeId);
|
||||
@@ -283,10 +828,12 @@ export function applyGraphUpdate(graph, update) {
|
||||
toNode.dependsOn.push(newEdge.fromNodeId);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Add resolved node IDs
|
||||
const newResolved = [...new Set([...graph.resolvedNodeIds, ...update.resolvedUnknownNodeIds])];
|
||||
|
||||
const newResolved = [
|
||||
...new Set([...graph.resolvedNodeIds, ...update.resolvedUnknownNodeIds]),
|
||||
];
|
||||
|
||||
return {
|
||||
success: true,
|
||||
nodes: newNodes,
|
||||
@@ -299,7 +846,7 @@ export function applyGraphUpdate(graph, update) {
|
||||
|
||||
export function validateGraphUpdate(graph, update) {
|
||||
const errors = [];
|
||||
|
||||
|
||||
// Check for duplicate node IDs against existing and newly added nodes
|
||||
const extendedIds = new Set(graph.nodes.map((n) => n.id));
|
||||
for (const newNode of update.addedNodes) {
|
||||
@@ -309,7 +856,7 @@ export function validateGraphUpdate(graph, update) {
|
||||
extendedIds.add(newNode.id);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Check updated nodes exist (in original graph, not newly added ones)
|
||||
const existingIds = new Set(graph.nodes.map((n) => n.id));
|
||||
for (const upd of update.updatedNodes) {
|
||||
@@ -317,13 +864,13 @@ export function validateGraphUpdate(graph, update) {
|
||||
errors.push(`Cannot update non-existent node: "${upd.nodeId}"`);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// Reject updates with no meaningful change
|
||||
const statusChanged = update.updatedNodes.some(
|
||||
(u) => u.previousStatus !== null && u.newStatus !== u.previousStatus
|
||||
(u) => u.previousStatus !== null && u.newStatus !== u.previousStatus,
|
||||
);
|
||||
const valueChanged = update.updatedNodes.some(
|
||||
(u) => u.previousValue !== null && u.newValue !== u.previousValue
|
||||
(u) => (u.previousValue ?? null) !== (u.newValue ?? null),
|
||||
);
|
||||
|
||||
const hasMeaningfulChange =
|
||||
@@ -336,13 +883,12 @@ export function validateGraphUpdate(graph, update) {
|
||||
if (!hasMeaningfulChange) {
|
||||
errors.push("Update contains no meaningful change");
|
||||
}
|
||||
|
||||
|
||||
// Reject oversized input
|
||||
const totalSize = JSON.stringify(update).length;
|
||||
if (totalSize > 100000) {
|
||||
errors.push(`Proposed graph update exceeds 100KB (${totalSize} bytes)`);
|
||||
}
|
||||
|
||||
|
||||
return { valid: errors.length === 0, errors };
|
||||
}
|
||||
|
||||
|
||||
@@ -16,6 +16,18 @@ export function normaliseAnalysisResponse(input) {
|
||||
normalised.evidence = normalised.evidence.map((record, index) => {
|
||||
if (!record || typeof record !== "object") return record;
|
||||
|
||||
if (record.evidenceType === "reported_claim") {
|
||||
changesApplied.push({
|
||||
path: ["evidence", index, "evidenceType"],
|
||||
change: "Converted reported_claim to reported_statement",
|
||||
});
|
||||
|
||||
record = {
|
||||
...record,
|
||||
evidenceType: "reported_statement",
|
||||
};
|
||||
}
|
||||
|
||||
if (record.source === null) {
|
||||
changesApplied.push({
|
||||
path: ["evidence", index, "source"],
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
|
||||
const BASE_URL =
|
||||
process.env.CONFIDENCE_ENGINE_BASE_URL || "http://127.0.0.1:3000";
|
||||
const OUTPUT_DIR = "tests-results/commercial-value-update";
|
||||
|
||||
const scenario = "I think therefore I am";
|
||||
const answer =
|
||||
"Deciding whether to build the Confidence Engine due to uncertainty about its commercial value.";
|
||||
|
||||
async function postJson(path, body) {
|
||||
const response = await fetch(`${BASE_URL}${path}`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
},
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
|
||||
const json = await response.json();
|
||||
return { status: response.status, json };
|
||||
}
|
||||
|
||||
function printLine(label, value) {
|
||||
const rendered = value === undefined ? null : value;
|
||||
console.log(`${label}: ${JSON.stringify(rendered)}`);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
await mkdir(OUTPUT_DIR, { recursive: true });
|
||||
|
||||
const startResult = await postJson("/api/cases/start", { scenario });
|
||||
await writeFile(
|
||||
`${OUTPUT_DIR}/start-response.json`,
|
||||
JSON.stringify(startResult, null, 2),
|
||||
);
|
||||
|
||||
const selectedQuestion = startResult.json?.selectedQuestion?.question || null;
|
||||
|
||||
let updateResult = {
|
||||
status: null,
|
||||
json: {
|
||||
success: false,
|
||||
stage: "request_construction",
|
||||
errors: ["Missing selected question from start response"],
|
||||
},
|
||||
};
|
||||
|
||||
if (startResult.json?.success && selectedQuestion) {
|
||||
updateResult = await postJson("/api/cases/update", {
|
||||
situationGraph: startResult.json.situationGraph,
|
||||
previousQuestion: selectedQuestion,
|
||||
answer,
|
||||
});
|
||||
}
|
||||
|
||||
await writeFile(
|
||||
`${OUTPUT_DIR}/update-response.json`,
|
||||
JSON.stringify(updateResult, null, 2),
|
||||
);
|
||||
|
||||
printLine("start success", startResult.json?.success ?? false);
|
||||
printLine("update success", updateResult.json?.success ?? false);
|
||||
printLine("update stage", updateResult.json?.stage ?? null);
|
||||
printLine(
|
||||
"proposal added nodes",
|
||||
updateResult.json?.proposal?.addedNodes?.map((node) => node.id) ?? null,
|
||||
);
|
||||
printLine(
|
||||
"proposal added edges",
|
||||
updateResult.json?.proposal?.addedEdges?.map((edge) => ({
|
||||
id: edge.id,
|
||||
fromNodeId: edge.fromNodeId,
|
||||
toNodeId: edge.toNodeId,
|
||||
relationship: edge.relationship,
|
||||
})) ?? null,
|
||||
);
|
||||
printLine(
|
||||
"proposal resolved unknown IDs",
|
||||
updateResult.json?.proposal?.resolvedUnknownNodeIds ??
|
||||
updateResult.json?.resolvedUnknownNodeIds ??
|
||||
null,
|
||||
);
|
||||
printLine(
|
||||
"errors",
|
||||
updateResult.json?.errors ??
|
||||
updateResult.json?.proposalErrors ??
|
||||
updateResult.json?.graphValidationErrors ??
|
||||
updateResult.json?.validationErrors ??
|
||||
null,
|
||||
);
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error instanceof Error ? error.message : String(error));
|
||||
process.exitCode = 1;
|
||||
});
|
||||
@@ -25,7 +25,9 @@ function makeSuccessResult() {
|
||||
removedEdgeIds: [],
|
||||
resolvedUnknownNodeIds: ["n1"],
|
||||
affectedNodeIds: ["n1"],
|
||||
selectedQuestion: null,
|
||||
},
|
||||
selectedQuestion: null,
|
||||
affectedNodeIds: ["n1"],
|
||||
resolvedUnknownNodeIds: ["n1"],
|
||||
previousActiveUnknownNodeId: "n0",
|
||||
@@ -268,6 +270,7 @@ describe("app/api/cases/update route", () => {
|
||||
resolvedUnknownNodeIds: success.resolvedUnknownNodeIds,
|
||||
previousActiveUnknownNodeId: success.previousActiveUnknownNodeId,
|
||||
newActiveUnknownNodeId: success.newActiveUnknownNodeId,
|
||||
selectedQuestion: success.selectedQuestion,
|
||||
changesApplied: success.changesApplied,
|
||||
diagnostics: success.diagnostics,
|
||||
});
|
||||
|
||||
+198
@@ -0,0 +1,198 @@
|
||||
import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js";
|
||||
|
||||
function buildAmbiguityFixture({
|
||||
key,
|
||||
scenario,
|
||||
summaryLabel,
|
||||
contradictionLabel,
|
||||
observationLabels,
|
||||
unknownLabels,
|
||||
disallowedQuestionTerms,
|
||||
}) {
|
||||
const summary = makeNode({
|
||||
id: `${key}-summary`,
|
||||
label: summaryLabel,
|
||||
description: "Summary of the situation from the scenario text",
|
||||
kind: "state",
|
||||
status: "provisional",
|
||||
confidence: "medium",
|
||||
});
|
||||
|
||||
const contradiction = makeNode({
|
||||
id: `${key}-contradiction`,
|
||||
label: contradictionLabel,
|
||||
description: contradictionLabel,
|
||||
kind: "relationship",
|
||||
status: "supported",
|
||||
confidence: "medium",
|
||||
});
|
||||
|
||||
const observations = observationLabels.map((label, index) =>
|
||||
makeNode({
|
||||
id: `${key}-obs-${index + 1}`,
|
||||
label,
|
||||
description: label,
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
}),
|
||||
);
|
||||
|
||||
const unknowns = unknownLabels.map((label, index) =>
|
||||
makeNode({
|
||||
id: `${key}-unknown-${index + 1}`,
|
||||
label,
|
||||
description: label,
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
}),
|
||||
);
|
||||
|
||||
const edges = [
|
||||
...observations.map((node) =>
|
||||
makeEdge({
|
||||
id: `${node.id}-supports-summary`,
|
||||
fromNodeId: node.id,
|
||||
toNodeId: summary.id,
|
||||
relationship: "supports",
|
||||
description: `${node.label} supports the summary.`,
|
||||
}),
|
||||
),
|
||||
...unknowns.map((node) =>
|
||||
makeEdge({
|
||||
id: `${node.id}-depends-summary`,
|
||||
fromNodeId: node.id,
|
||||
toNodeId: summary.id,
|
||||
relationship: "depends_on",
|
||||
description: `${node.label} is an unresolved factor for this situation.`,
|
||||
}),
|
||||
),
|
||||
];
|
||||
|
||||
return {
|
||||
key,
|
||||
scenario,
|
||||
disallowedQuestionTerms,
|
||||
graph: makeGraph({
|
||||
centralStatement: scenario,
|
||||
nodes: [summary, contradiction, ...observations, ...unknowns],
|
||||
edges,
|
||||
activeUnknownNodeId: null,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: `Ambiguity fixture for ${key}`,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
export const ambiguityGeneralisationFixtures = [
|
||||
buildAmbiguityFixture({
|
||||
key: "revenue-cash",
|
||||
scenario:
|
||||
"Revenue increased by 18%, but cash in the bank fell over the same period.",
|
||||
summaryLabel: "Revenue rose while cash fell",
|
||||
contradictionLabel:
|
||||
"Contradiction between revenue improvement and lower cash reserves.",
|
||||
observationLabels: [
|
||||
"Revenue increased by 18%.",
|
||||
"Cash in the bank decreased over the same period.",
|
||||
],
|
||||
unknownLabels: [
|
||||
"Possible explanation for the contradiction from one side of the situation.",
|
||||
"Possible explanation for the contradiction from another side of the situation.",
|
||||
],
|
||||
disallowedQuestionTerms: [
|
||||
"accounts receivable",
|
||||
"capex",
|
||||
"debt repayments",
|
||||
"working capital",
|
||||
],
|
||||
}),
|
||||
buildAmbiguityFixture({
|
||||
key: "satisfaction-complaints",
|
||||
scenario:
|
||||
"Customer satisfaction scores increased, but complaints also increased.",
|
||||
summaryLabel: "Satisfaction scores rose while complaints also rose",
|
||||
contradictionLabel:
|
||||
"Contradiction between higher satisfaction scores and higher complaint volume.",
|
||||
observationLabels: [
|
||||
"Customer satisfaction scores increased.",
|
||||
"Complaints increased.",
|
||||
],
|
||||
unknownLabels: [
|
||||
"Possible explanation for why the positive signal and negative signal moved together.",
|
||||
"Another possible explanation for why the positive signal and negative signal moved together.",
|
||||
],
|
||||
disallowedQuestionTerms: [
|
||||
"net promoter",
|
||||
"ticket backlog",
|
||||
"call deflection",
|
||||
"support queue",
|
||||
],
|
||||
}),
|
||||
buildAmbiguityFixture({
|
||||
key: "delivery-cancellations",
|
||||
scenario:
|
||||
"Average delivery time decreased by 25%, but order cancellations increased.",
|
||||
summaryLabel: "Delivery became faster while cancellations increased",
|
||||
contradictionLabel:
|
||||
"Contradiction between faster delivery and more order cancellations.",
|
||||
observationLabels: [
|
||||
"Average delivery time decreased by 25%.",
|
||||
"Order cancellations increased.",
|
||||
],
|
||||
unknownLabels: [
|
||||
"Possible explanation for why the faster result did not reduce the negative result.",
|
||||
"Another possible explanation for why the faster result did not reduce the negative result.",
|
||||
],
|
||||
disallowedQuestionTerms: [
|
||||
"fulfilment",
|
||||
"last mile",
|
||||
"warehouse",
|
||||
"routing",
|
||||
],
|
||||
}),
|
||||
buildAmbiguityFixture({
|
||||
key: "traffic-sales",
|
||||
scenario: "Website traffic doubled, but sales remained unchanged.",
|
||||
summaryLabel: "Website traffic doubled while sales stayed flat",
|
||||
contradictionLabel:
|
||||
"Contradiction between much higher traffic and unchanged sales.",
|
||||
observationLabels: [
|
||||
"Website traffic doubled.",
|
||||
"Sales remained unchanged.",
|
||||
],
|
||||
unknownLabels: [
|
||||
"Possible explanation for why the stronger signal did not change the outcome.",
|
||||
"Another possible explanation for why the stronger signal did not change the outcome.",
|
||||
],
|
||||
disallowedQuestionTerms: [
|
||||
"conversion funnel",
|
||||
"campaign attribution",
|
||||
"landing page",
|
||||
"checkout flow",
|
||||
],
|
||||
}),
|
||||
buildAmbiguityFixture({
|
||||
key: "output-defects",
|
||||
scenario:
|
||||
"Production output increased by 30%, but quality defects also increased.",
|
||||
summaryLabel: "Production output rose while defects also rose",
|
||||
contradictionLabel:
|
||||
"Contradiction between higher output and more quality defects.",
|
||||
observationLabels: [
|
||||
"Production output increased by 30%.",
|
||||
"Quality defects increased.",
|
||||
],
|
||||
unknownLabels: [
|
||||
"Possible explanation for why the gain came with a worsening result.",
|
||||
"Another possible explanation for why the gain came with a worsening result.",
|
||||
],
|
||||
disallowedQuestionTerms: [
|
||||
"scrap rate",
|
||||
"throughput",
|
||||
"yield",
|
||||
"root cause",
|
||||
],
|
||||
}),
|
||||
];
|
||||
+164
@@ -0,0 +1,164 @@
|
||||
import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js";
|
||||
|
||||
function buildComparabilityFixture({
|
||||
key,
|
||||
scenario,
|
||||
observationLabels,
|
||||
contradictionLabel,
|
||||
expectedComparabilityStatus,
|
||||
expectsComparisonQuestion,
|
||||
}) {
|
||||
const summary = makeNode({
|
||||
id: `${key}-summary`,
|
||||
label: scenario,
|
||||
description: "Summary of the situation from the scenario text",
|
||||
kind: "state",
|
||||
status: "provisional",
|
||||
confidence: "medium",
|
||||
});
|
||||
|
||||
const observations = observationLabels.map((label, index) =>
|
||||
makeNode({
|
||||
id: `${key}-obs-${index + 1}`,
|
||||
label,
|
||||
description: label,
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
}),
|
||||
);
|
||||
|
||||
const contradiction = contradictionLabel
|
||||
? [
|
||||
makeNode({
|
||||
id: `${key}-contradiction`,
|
||||
label: contradictionLabel,
|
||||
description: contradictionLabel,
|
||||
kind: "relationship",
|
||||
status: "supported",
|
||||
confidence: "medium",
|
||||
}),
|
||||
]
|
||||
: [];
|
||||
|
||||
const unknowns = [
|
||||
makeNode({
|
||||
id: `${key}-unknown-a`,
|
||||
label: "Possible explanation from one side of the situation.",
|
||||
description: "Possible explanation from one side of the situation.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
}),
|
||||
makeNode({
|
||||
id: `${key}-unknown-b`,
|
||||
label: "Possible explanation from another side of the situation.",
|
||||
description: "Possible explanation from another side of the situation.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
}),
|
||||
];
|
||||
|
||||
const edges = [
|
||||
...observations.map((node) =>
|
||||
makeEdge({
|
||||
id: `${node.id}-supports-summary`,
|
||||
fromNodeId: node.id,
|
||||
toNodeId: summary.id,
|
||||
relationship: "supports",
|
||||
description: `${node.label} supports the summary.`,
|
||||
}),
|
||||
),
|
||||
...unknowns.map((node) =>
|
||||
makeEdge({
|
||||
id: `${node.id}-depends-summary`,
|
||||
fromNodeId: node.id,
|
||||
toNodeId: summary.id,
|
||||
relationship: "depends_on",
|
||||
description: `${node.label} is an unresolved factor for this situation.`,
|
||||
}),
|
||||
),
|
||||
];
|
||||
|
||||
return {
|
||||
key,
|
||||
scenario,
|
||||
expectedComparabilityStatus,
|
||||
expectsComparisonQuestion,
|
||||
graph: makeGraph({
|
||||
centralStatement: scenario,
|
||||
nodes: [summary, ...observations, ...contradiction, ...unknowns],
|
||||
edges,
|
||||
activeUnknownNodeId: null,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: `Comparability fixture for ${key}`,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
export const comparabilityAssessmentFixtures = [
|
||||
buildComparabilityFixture({
|
||||
key: "revenue-cash",
|
||||
scenario:
|
||||
"Revenue increased by 18%, but cash in the bank fell over the same period.",
|
||||
observationLabels: [
|
||||
"Revenue increased by 18%.",
|
||||
"Cash in the bank decreased over the same period.",
|
||||
],
|
||||
contradictionLabel:
|
||||
"Contradiction between revenue improvement and lower cash reserves.",
|
||||
expectedComparabilityStatus: "uncertain",
|
||||
expectsComparisonQuestion: true,
|
||||
}),
|
||||
buildComparabilityFixture({
|
||||
key: "complaints-production",
|
||||
scenario: "Complaints increased. Production increased.",
|
||||
observationLabels: ["Complaints increased.", "Production increased."],
|
||||
contradictionLabel:
|
||||
"Possible contradiction between complaints and production movement.",
|
||||
expectedComparabilityStatus: "uncertain",
|
||||
expectsComparisonQuestion: true,
|
||||
}),
|
||||
buildComparabilityFixture({
|
||||
key: "delivery-cancellations",
|
||||
scenario:
|
||||
"Average delivery time decreased by 25%, but order cancellations increased.",
|
||||
observationLabels: [
|
||||
"Average delivery time decreased by 25%.",
|
||||
"Order cancellations increased.",
|
||||
],
|
||||
contradictionLabel:
|
||||
"Contradiction between faster delivery and more cancellations.",
|
||||
expectedComparabilityStatus: "uncertain",
|
||||
expectsComparisonQuestion: true,
|
||||
}),
|
||||
buildComparabilityFixture({
|
||||
key: "satisfaction-complaints",
|
||||
scenario: "Customer satisfaction increased, but complaints increased.",
|
||||
observationLabels: [
|
||||
"Customer satisfaction increased.",
|
||||
"Complaints increased.",
|
||||
],
|
||||
contradictionLabel:
|
||||
"Contradiction between satisfaction improvement and more complaints.",
|
||||
expectedComparabilityStatus: "uncertain",
|
||||
expectsComparisonQuestion: true,
|
||||
}),
|
||||
buildComparabilityFixture({
|
||||
key: "temperature-ice",
|
||||
scenario: "Temperature increased. Ice melted.",
|
||||
observationLabels: ["Temperature increased.", "Ice melted."],
|
||||
contradictionLabel: null,
|
||||
expectedComparabilityStatus: "confirmed",
|
||||
expectsComparisonQuestion: false,
|
||||
}),
|
||||
buildComparabilityFixture({
|
||||
key: "sales-same",
|
||||
scenario: "Sales doubled. Sales doubled.",
|
||||
observationLabels: ["Sales doubled.", "Sales doubled."],
|
||||
contradictionLabel: null,
|
||||
expectedComparabilityStatus: "confirmed",
|
||||
expectsComparisonQuestion: false,
|
||||
}),
|
||||
];
|
||||
+445
@@ -0,0 +1,445 @@
|
||||
import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js";
|
||||
|
||||
function makeScenarioGraph({
|
||||
scenario,
|
||||
decisionNode,
|
||||
answeredContextUnknown,
|
||||
foundationalUnknown,
|
||||
consequentialUnknown,
|
||||
downstreamLeaf,
|
||||
}) {
|
||||
const nodes = [
|
||||
decisionNode,
|
||||
answeredContextUnknown,
|
||||
foundationalUnknown,
|
||||
consequentialUnknown,
|
||||
downstreamLeaf,
|
||||
];
|
||||
|
||||
const edges = [
|
||||
makeEdge({
|
||||
id: `${decisionNode.id}-to-${foundationalUnknown.id}`,
|
||||
fromNodeId: decisionNode.id,
|
||||
toNodeId: foundationalUnknown.id,
|
||||
relationship: "depends_on",
|
||||
description: `${decisionNode.label} depends on ${foundationalUnknown.label}.`,
|
||||
}),
|
||||
makeEdge({
|
||||
id: `${answeredContextUnknown.id}-to-${consequentialUnknown.id}`,
|
||||
fromNodeId: answeredContextUnknown.id,
|
||||
toNodeId: consequentialUnknown.id,
|
||||
relationship: "depends_on",
|
||||
description: `${consequentialUnknown.label} was surfaced from resolved context.`,
|
||||
}),
|
||||
makeEdge({
|
||||
id: `${foundationalUnknown.id}-to-${consequentialUnknown.id}`,
|
||||
fromNodeId: foundationalUnknown.id,
|
||||
toNodeId: consequentialUnknown.id,
|
||||
relationship: "depends_on",
|
||||
description: `${consequentialUnknown.label} depends on ${foundationalUnknown.label}.`,
|
||||
}),
|
||||
makeEdge({
|
||||
id: `${consequentialUnknown.id}-to-${downstreamLeaf.id}`,
|
||||
fromNodeId: consequentialUnknown.id,
|
||||
toNodeId: downstreamLeaf.id,
|
||||
relationship: "depends_on",
|
||||
description: `${downstreamLeaf.label} depends on ${consequentialUnknown.label}.`,
|
||||
}),
|
||||
];
|
||||
|
||||
return makeGraph({
|
||||
centralStatement: scenario,
|
||||
nodes,
|
||||
edges,
|
||||
activeUnknownNodeId: answeredContextUnknown.id,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: "Generalisation fixture graph",
|
||||
});
|
||||
}
|
||||
|
||||
export const questionPriorityGeneralisationFixtures = [
|
||||
{
|
||||
key: "hire-engineer",
|
||||
scenario: "Should we hire another engineer?",
|
||||
decisionType: "resourcing decision",
|
||||
acceptableFoundationalUnknownNodeIds: [
|
||||
"hire-success-criteria",
|
||||
"hire-bottleneck",
|
||||
],
|
||||
prohibitedFirstTopics: ["salary", "job advert", "programming language"],
|
||||
acceptableQuestionStrategies: [
|
||||
"decision_threshold",
|
||||
"evidence_gathering",
|
||||
"definition",
|
||||
],
|
||||
notes:
|
||||
"The first question should establish whether more engineering capacity is justified before compensation or implementation details.",
|
||||
graph: makeScenarioGraph({
|
||||
scenario: "Should we hire another engineer?",
|
||||
decisionNode: makeNode({
|
||||
id: "hire-decision",
|
||||
label: "Hiring another engineer decision",
|
||||
description: "Decision about increasing engineering capacity.",
|
||||
kind: "state",
|
||||
status: "known",
|
||||
confidence: "medium",
|
||||
value: "Deciding whether to hire another engineer",
|
||||
childIds: ["hire-success-criteria"],
|
||||
}),
|
||||
answeredContextUnknown: makeNode({
|
||||
id: "hire-delays-known",
|
||||
label: "Delivery delays established",
|
||||
description:
|
||||
"Need to confirm whether recent delivery delays are real because this context determines whether a capacity decision is even relevant.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
value:
|
||||
"The roadmap is slipping because the current team cannot clear the queue.",
|
||||
childIds: ["hire-bottleneck"],
|
||||
}),
|
||||
foundationalUnknown: makeNode({
|
||||
id: "hire-success-criteria",
|
||||
label: "Hiring success threshold",
|
||||
description:
|
||||
"Need the success threshold because the hiring decision depends on what improvement would justify adding headcount.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
parentId: "hire-decision",
|
||||
childIds: ["hire-bottleneck"],
|
||||
}),
|
||||
consequentialUnknown: makeNode({
|
||||
id: "hire-bottleneck",
|
||||
label: "Primary delivery bottleneck",
|
||||
description:
|
||||
"Need the main bottleneck because the team must know whether another engineer would relieve the limiting constraint.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
dependsOn: ["hire-success-criteria"],
|
||||
parentId: "hire-success-criteria",
|
||||
childIds: ["hire-salary"],
|
||||
}),
|
||||
downstreamLeaf: makeNode({
|
||||
id: "hire-salary",
|
||||
label: "Engineer salary budget",
|
||||
description:
|
||||
"Need the salary range because compensation planning comes after the hiring case is established.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
dependsOn: ["hire-bottleneck"],
|
||||
parentId: "hire-bottleneck",
|
||||
}),
|
||||
}),
|
||||
},
|
||||
{
|
||||
key: "replace-vans",
|
||||
scenario: "Should we replace the delivery vans?",
|
||||
decisionType: "asset replacement decision",
|
||||
acceptableFoundationalUnknownNodeIds: [
|
||||
"van-reliability-threshold",
|
||||
"van-service-constraint",
|
||||
],
|
||||
prohibitedFirstTopics: [
|
||||
"purchase price",
|
||||
"paint colour",
|
||||
"finance provider",
|
||||
],
|
||||
acceptableQuestionStrategies: [
|
||||
"decision_threshold",
|
||||
"evidence_gathering",
|
||||
"definition",
|
||||
],
|
||||
notes:
|
||||
"The first question should establish whether the fleet is failing a threshold that justifies replacement.",
|
||||
graph: makeScenarioGraph({
|
||||
scenario: "Should we replace the delivery vans?",
|
||||
decisionNode: makeNode({
|
||||
id: "van-decision",
|
||||
label: "Replace delivery vans decision",
|
||||
description: "Decision about replacing the current delivery fleet.",
|
||||
kind: "state",
|
||||
status: "known",
|
||||
confidence: "medium",
|
||||
value: "Deciding whether to replace the delivery vans",
|
||||
childIds: ["van-reliability-threshold"],
|
||||
}),
|
||||
answeredContextUnknown: makeNode({
|
||||
id: "van-breakdowns-known",
|
||||
label: "Breakdown trend confirmed",
|
||||
description:
|
||||
"Need to confirm whether the recent rise in breakdowns is real because that context determines whether fleet replacement is relevant.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
value:
|
||||
"Breakdowns and missed deliveries have increased over the last quarter.",
|
||||
childIds: ["van-service-constraint"],
|
||||
}),
|
||||
foundationalUnknown: makeNode({
|
||||
id: "van-reliability-threshold",
|
||||
label: "Replacement justification threshold",
|
||||
description:
|
||||
"Need the threshold because the replacement decision depends on what level of reliability loss is enough to justify replacing the fleet.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
parentId: "van-decision",
|
||||
childIds: ["van-service-constraint"],
|
||||
}),
|
||||
consequentialUnknown: makeNode({
|
||||
id: "van-service-constraint",
|
||||
label: "Operational service constraint",
|
||||
description:
|
||||
"Need the limiting service constraint because the team must know how vehicle unreliability is affecting deliveries before comparing purchasing options.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
dependsOn: ["van-reliability-threshold"],
|
||||
parentId: "van-reliability-threshold",
|
||||
childIds: ["van-price"],
|
||||
}),
|
||||
downstreamLeaf: makeNode({
|
||||
id: "van-price",
|
||||
label: "Exact replacement purchase price",
|
||||
description:
|
||||
"Need the exact purchase price because financing analysis comes after replacement is justified.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
dependsOn: ["van-service-constraint"],
|
||||
parentId: "van-service-constraint",
|
||||
}),
|
||||
}),
|
||||
},
|
||||
{
|
||||
key: "launch-country",
|
||||
scenario: "Should we launch in another country?",
|
||||
decisionType: "market expansion decision",
|
||||
acceptableFoundationalUnknownNodeIds: [
|
||||
"country-customer",
|
||||
"country-value-threshold",
|
||||
],
|
||||
prohibitedFirstTopics: [
|
||||
"launch date",
|
||||
"office location",
|
||||
"advertising channel",
|
||||
],
|
||||
acceptableQuestionStrategies: ["definition", "decision_threshold"],
|
||||
notes:
|
||||
"The first question should clarify the customer or value case for expansion before rollout logistics.",
|
||||
graph: makeScenarioGraph({
|
||||
scenario: "Should we launch in another country?",
|
||||
decisionNode: makeNode({
|
||||
id: "country-decision",
|
||||
label: "Launch in another country decision",
|
||||
description: "Decision about entering a new national market.",
|
||||
kind: "state",
|
||||
status: "known",
|
||||
confidence: "medium",
|
||||
value: "Deciding whether to launch in another country",
|
||||
childIds: ["country-customer"],
|
||||
}),
|
||||
answeredContextUnknown: makeNode({
|
||||
id: "country-interest-known",
|
||||
label: "Inbound interest confirmed",
|
||||
description:
|
||||
"Need to confirm whether inbound interest from another country is real because that context determines whether expansion is relevant.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
value:
|
||||
"Prospective customers from another country are asking for access.",
|
||||
childIds: ["country-value-threshold"],
|
||||
}),
|
||||
foundationalUnknown: makeNode({
|
||||
id: "country-customer",
|
||||
label: "Relevant customer in the new country",
|
||||
description:
|
||||
"Need the relevant customer because the expansion decision depends on who experiences the problem or receives the value in that market.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
parentId: "country-decision",
|
||||
childIds: ["country-value-threshold"],
|
||||
}),
|
||||
consequentialUnknown: makeNode({
|
||||
id: "country-value-threshold",
|
||||
label: "Expansion value threshold",
|
||||
description:
|
||||
"Need the value threshold because the team must know what evidence of demand or value would justify entering the new country.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
dependsOn: ["country-customer"],
|
||||
parentId: "country-customer",
|
||||
childIds: ["country-launch-date"],
|
||||
}),
|
||||
downstreamLeaf: makeNode({
|
||||
id: "country-launch-date",
|
||||
label: "Country launch date",
|
||||
description:
|
||||
"Need the launch date because rollout planning follows once the expansion case is established.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
dependsOn: ["country-value-threshold"],
|
||||
parentId: "country-value-threshold",
|
||||
}),
|
||||
}),
|
||||
},
|
||||
{
|
||||
key: "over-budget-project",
|
||||
scenario: "Should we continue a project that is over budget?",
|
||||
decisionType: "continuation decision",
|
||||
acceptableFoundationalUnknownNodeIds: [
|
||||
"project-benefit-threshold",
|
||||
"project-remaining-benefit",
|
||||
],
|
||||
prohibitedFirstTopics: ["sunk cost", "project logo", "final launch date"],
|
||||
acceptableQuestionStrategies: ["decision_threshold", "definition"],
|
||||
notes:
|
||||
"The first question should establish remaining value or success threshold before sunk-cost framing or launch timing.",
|
||||
graph: makeScenarioGraph({
|
||||
scenario: "Should we continue a project that is over budget?",
|
||||
decisionNode: makeNode({
|
||||
id: "project-decision",
|
||||
label: "Continue over-budget project decision",
|
||||
description:
|
||||
"Decision about continuing a project that has exceeded budget.",
|
||||
kind: "state",
|
||||
status: "known",
|
||||
confidence: "medium",
|
||||
value: "Deciding whether to continue the over-budget project",
|
||||
childIds: ["project-benefit-threshold"],
|
||||
}),
|
||||
answeredContextUnknown: makeNode({
|
||||
id: "project-overrun-known",
|
||||
label: "Budget overrun confirmed",
|
||||
description:
|
||||
"Need to confirm whether the project is materially over budget because that context determines whether a continuation decision is relevant.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
value: "The project has exceeded its approved budget by 35 percent.",
|
||||
childIds: ["project-remaining-benefit"],
|
||||
}),
|
||||
foundationalUnknown: makeNode({
|
||||
id: "project-benefit-threshold",
|
||||
label: "Continuation success threshold",
|
||||
description:
|
||||
"Need the threshold because the continuation decision depends on what remaining benefit would still justify completing the project.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
parentId: "project-decision",
|
||||
childIds: ["project-remaining-benefit"],
|
||||
}),
|
||||
consequentialUnknown: makeNode({
|
||||
id: "project-remaining-benefit",
|
||||
label: "Remaining project benefit",
|
||||
description:
|
||||
"Need the remaining benefit because the team must know what value is still achievable before deciding whether to continue.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
dependsOn: ["project-benefit-threshold"],
|
||||
parentId: "project-benefit-threshold",
|
||||
childIds: ["project-launch-date"],
|
||||
}),
|
||||
downstreamLeaf: makeNode({
|
||||
id: "project-launch-date",
|
||||
label: "Final launch date",
|
||||
description:
|
||||
"Need the final launch date because scheduling details only matter after remaining value is established.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
dependsOn: ["project-remaining-benefit"],
|
||||
parentId: "project-remaining-benefit",
|
||||
}),
|
||||
}),
|
||||
},
|
||||
{
|
||||
key: "paid-support-tier",
|
||||
scenario: "Should we introduce a paid support tier?",
|
||||
decisionType: "commercial packaging decision",
|
||||
acceptableFoundationalUnknownNodeIds: [
|
||||
"support-customer",
|
||||
"support-value-threshold",
|
||||
],
|
||||
prohibitedFirstTopics: [
|
||||
"subscription price",
|
||||
"payment provider",
|
||||
"tier name",
|
||||
],
|
||||
acceptableQuestionStrategies: [
|
||||
"definition",
|
||||
"decision_threshold",
|
||||
"baseline_reconstruction",
|
||||
],
|
||||
notes:
|
||||
"The first question should establish who values paid support or what outcome would justify offering it before pricing details.",
|
||||
graph: makeScenarioGraph({
|
||||
scenario: "Should we introduce a paid support tier?",
|
||||
decisionNode: makeNode({
|
||||
id: "support-decision",
|
||||
label: "Introduce paid support tier decision",
|
||||
description: "Decision about adding a paid support offering.",
|
||||
kind: "state",
|
||||
status: "known",
|
||||
confidence: "medium",
|
||||
value: "Deciding whether to introduce a paid support tier",
|
||||
childIds: ["support-customer"],
|
||||
}),
|
||||
answeredContextUnknown: makeNode({
|
||||
id: "support-requests-known",
|
||||
label: "Support request pattern confirmed",
|
||||
description:
|
||||
"Need to confirm whether repeated requests for faster support responses are real because that context determines whether a paid tier is relevant.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
value:
|
||||
"Some users are asking for guaranteed response times and escalation help.",
|
||||
childIds: ["support-value-threshold"],
|
||||
}),
|
||||
foundationalUnknown: makeNode({
|
||||
id: "support-customer",
|
||||
label: "Customer willing to pay for support",
|
||||
description:
|
||||
"Need the customer because the decision depends on who experiences enough support pain or receives enough value to pay for a support tier.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
parentId: "support-decision",
|
||||
childIds: ["support-value-threshold"],
|
||||
}),
|
||||
consequentialUnknown: makeNode({
|
||||
id: "support-value-threshold",
|
||||
label: "Paid support value threshold",
|
||||
description:
|
||||
"Need the value threshold because the team must know what outcome would justify introducing paid support before setting packaging details.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
dependsOn: ["support-customer"],
|
||||
parentId: "support-customer",
|
||||
childIds: ["support-price"],
|
||||
}),
|
||||
downstreamLeaf: makeNode({
|
||||
id: "support-price",
|
||||
label: "Support subscription price",
|
||||
description:
|
||||
"Need the subscription price because pricing and payment setup come after the support value case is established.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
dependsOn: ["support-value-threshold"],
|
||||
parentId: "support-value-threshold",
|
||||
}),
|
||||
}),
|
||||
},
|
||||
];
|
||||
@@ -0,0 +1,131 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
formulateQuestion,
|
||||
formulateTieResolutionQuestion,
|
||||
} from "@/lib/graph/question-formulator.js";
|
||||
import {
|
||||
explainUnknownSelection,
|
||||
selectActiveUnknownCandidate,
|
||||
} from "@/lib/graph/utils.js";
|
||||
import { ambiguityGeneralisationFixtures } from "@/tests/fixtures/ambiguity-generalisation.js";
|
||||
|
||||
function neutraliseUnknownLabels(graph) {
|
||||
let counter = 0;
|
||||
return {
|
||||
...graph,
|
||||
nodes: graph.nodes.map((node) => {
|
||||
if (node.kind !== "unknown") return { ...node };
|
||||
counter += 1;
|
||||
return {
|
||||
...node,
|
||||
label: `Unknown ${String.fromCharCode(64 + counter)}`,
|
||||
description: `Unknown factor ${counter}.`,
|
||||
};
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function isSingleQuestion(question) {
|
||||
return (question.match(/\?/g) || []).length === 1;
|
||||
}
|
||||
|
||||
describe("ambiguity generalisation", () => {
|
||||
it("preserves ambiguity across contradiction scenarios without favouring one explanation", () => {
|
||||
const summary = ambiguityGeneralisationFixtures.map((fixture) => {
|
||||
const explanation = explainUnknownSelection(fixture.graph, []);
|
||||
const selection = selectActiveUnknownCandidate(fixture.graph, []);
|
||||
const neutralExplanation = explainUnknownSelection(
|
||||
neutraliseUnknownLabels(fixture.graph),
|
||||
[],
|
||||
);
|
||||
const tieQuestion = formulateTieResolutionQuestion({
|
||||
graph: fixture.graph,
|
||||
});
|
||||
const representativeUnknown = fixture.graph.nodes.find(
|
||||
(node) => node.kind === "unknown",
|
||||
);
|
||||
const fallbackQuestion = formulateQuestion({
|
||||
node: representativeUnknown,
|
||||
graph: fixture.graph,
|
||||
});
|
||||
|
||||
const lowerQuestion = tieQuestion.question.toLowerCase();
|
||||
for (const term of fixture.disallowedQuestionTerms) {
|
||||
expect(lowerQuestion).not.toContain(term.toLowerCase());
|
||||
}
|
||||
|
||||
expect(explanation.status).toBe("ambiguous");
|
||||
expect(selection.status).toBe("ambiguous");
|
||||
expect(selection.selectedNode).toBeNull();
|
||||
expect(explanation.selectedNodeId).toBeNull();
|
||||
expect(explanation.candidates).toHaveLength(2);
|
||||
expect(explanation.summary.selectedReason).toBe(
|
||||
"No justified distinction between leading unknowns.",
|
||||
);
|
||||
expect(explanation.alphabeticalUsedAsReasoning).toBe(false);
|
||||
expect(neutralExplanation.status).toBe("ambiguous");
|
||||
expect(isSingleQuestion(tieQuestion.question)).toBe(true);
|
||||
expect(tieQuestion.question.toLowerCase()).not.toContain(" or ");
|
||||
|
||||
return {
|
||||
scenario: fixture.scenario,
|
||||
candidateCount: explanation.candidates.length,
|
||||
ambiguityStatus: explanation.status,
|
||||
tieReason: explanation.summary.selectedReason,
|
||||
investigationStrategy: tieQuestion.strategy,
|
||||
question: tieQuestion.question,
|
||||
explanationFavoured: explanation.selectedNodeId !== null,
|
||||
};
|
||||
});
|
||||
|
||||
expect(summary).toMatchInlineSnapshot(`
|
||||
[
|
||||
{
|
||||
"ambiguityStatus": "ambiguous",
|
||||
"candidateCount": 2,
|
||||
"explanationFavoured": false,
|
||||
"investigationStrategy": null,
|
||||
"question": "Were these figures measured on the same basis and at the same scale?",
|
||||
"scenario": "Revenue increased by 18%, but cash in the bank fell over the same period.",
|
||||
"tieReason": "No justified distinction between leading unknowns.",
|
||||
},
|
||||
{
|
||||
"ambiguityStatus": "ambiguous",
|
||||
"candidateCount": 2,
|
||||
"explanationFavoured": false,
|
||||
"investigationStrategy": null,
|
||||
"question": "Were these figures measured over the same period and at the same scale?",
|
||||
"scenario": "Customer satisfaction scores increased, but complaints also increased.",
|
||||
"tieReason": "No justified distinction between leading unknowns.",
|
||||
},
|
||||
{
|
||||
"ambiguityStatus": "ambiguous",
|
||||
"candidateCount": 2,
|
||||
"explanationFavoured": false,
|
||||
"investigationStrategy": null,
|
||||
"question": "Were these figures measured over the same period and at the same scale?",
|
||||
"scenario": "Average delivery time decreased by 25%, but order cancellations increased.",
|
||||
"tieReason": "No justified distinction between leading unknowns.",
|
||||
},
|
||||
{
|
||||
"ambiguityStatus": "ambiguous",
|
||||
"candidateCount": 2,
|
||||
"explanationFavoured": false,
|
||||
"investigationStrategy": null,
|
||||
"question": "Were these figures measured over the same period and at the same scale?",
|
||||
"scenario": "Website traffic doubled, but sales remained unchanged.",
|
||||
"tieReason": "No justified distinction between leading unknowns.",
|
||||
},
|
||||
{
|
||||
"ambiguityStatus": "ambiguous",
|
||||
"candidateCount": 2,
|
||||
"explanationFavoured": false,
|
||||
"investigationStrategy": null,
|
||||
"question": "Were these figures measured over the same period and at the same scale?",
|
||||
"scenario": "Production output increased by 30%, but quality defects also increased.",
|
||||
"tieReason": "No justified distinction between leading unknowns.",
|
||||
},
|
||||
]
|
||||
`);
|
||||
});
|
||||
});
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,118 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { assessUnknownAtomicity } from "@/lib/graph/question-formulator.js";
|
||||
import { makeGraph, makeNode } from "@/lib/graph/schema.js";
|
||||
|
||||
function makeGraphWithUnknown(centralStatement, unknown, observations = []) {
|
||||
return makeGraph({
|
||||
centralStatement,
|
||||
nodes: [unknown, ...observations],
|
||||
edges: [],
|
||||
activeUnknownNodeId: unknown.id,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: "Atomicity test graph",
|
||||
});
|
||||
}
|
||||
|
||||
describe("assessUnknownAtomicity", () => {
|
||||
it("classifies denominator-style unknowns as atomic", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-denominator",
|
||||
label: "Complaint rate denominator",
|
||||
description:
|
||||
"Need the denominator because it directly determines the complaint rate.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
});
|
||||
|
||||
const result = assessUnknownAtomicity({
|
||||
node: unknown,
|
||||
graph: makeGraphWithUnknown(
|
||||
"Production increased while complaints increased.",
|
||||
unknown,
|
||||
),
|
||||
});
|
||||
|
||||
expect(result.atomicity).toBe("atomic");
|
||||
expect(result.reason.toLowerCase()).toContain("directly");
|
||||
});
|
||||
|
||||
it("classifies relationship explanation unknowns as composite", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-explanation",
|
||||
label:
|
||||
"Explanation for why revenue increased by 18%, but cash in the bank fell over the same period",
|
||||
description:
|
||||
"Need to understand what change or event could explain why these observations differ, because that is needed to investigate their relationship.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
|
||||
const graph = makeGraphWithUnknown(
|
||||
"Revenue increased by 18%, but cash in the bank fell over the same period.",
|
||||
unknown,
|
||||
[
|
||||
makeNode({
|
||||
id: "n-revenue",
|
||||
label: "Revenue increased by 18%.",
|
||||
description: "Revenue increased by 18%.",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
}),
|
||||
makeNode({
|
||||
id: "n-cash",
|
||||
label: "Cash in the bank decreased over the same period.",
|
||||
description: "Cash in the bank decreased over the same period.",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
}),
|
||||
],
|
||||
);
|
||||
|
||||
const result = assessUnknownAtomicity({ node: unknown, graph });
|
||||
|
||||
expect(result.atomicity).toBe("composite");
|
||||
expect(result.decompositionKind).toBe("relationship_explanation");
|
||||
});
|
||||
|
||||
it.each([
|
||||
[
|
||||
"Customer satisfaction rose, but complaints also rose.",
|
||||
"Explanation for why customer satisfaction rose, but complaints also rose",
|
||||
],
|
||||
[
|
||||
"Delivery time fell, but cancellations increased.",
|
||||
"Possible causes of why delivery time fell, but cancellations increased",
|
||||
],
|
||||
[
|
||||
"Traffic increased, but sales stayed flat.",
|
||||
"Broad explanation for why traffic increased, but sales stayed flat",
|
||||
],
|
||||
[
|
||||
"Production increased, but defects also increased.",
|
||||
"Factors behind why production increased, but defects also increased",
|
||||
],
|
||||
])(
|
||||
"classifies broad divergence unknowns as composite: %s",
|
||||
(scenario, label) => {
|
||||
const unknown = makeNode({
|
||||
id: `n-${label.length}`,
|
||||
label,
|
||||
description: `${label} because the current unknown is too broad to ask directly.`,
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
|
||||
const result = assessUnknownAtomicity({
|
||||
node: unknown,
|
||||
graph: makeGraphWithUnknown(scenario, unknown),
|
||||
});
|
||||
|
||||
expect(result.atomicity).toBe("composite");
|
||||
},
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,180 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
assessComparability,
|
||||
classifyObservationRelationship,
|
||||
formulateTieResolutionQuestion,
|
||||
} from "@/lib/graph/question-formulator.js";
|
||||
import { explainUnknownSelection } from "@/lib/graph/utils.js";
|
||||
import { comparabilityAssessmentFixtures } from "@/tests/fixtures/comparability-assessment.js";
|
||||
|
||||
describe("comparability assessment", () => {
|
||||
it("generates comparison or relationship questions only when warranted", () => {
|
||||
const summary = comparabilityAssessmentFixtures.map((fixture) => {
|
||||
const assessment = assessComparability(fixture.graph);
|
||||
const relationship = classifyObservationRelationship(fixture.graph);
|
||||
const question = formulateTieResolutionQuestion({ graph: fixture.graph });
|
||||
const ambiguity = explainUnknownSelection(fixture.graph, []);
|
||||
|
||||
expect(assessment.comparabilityStatus).toBe(
|
||||
fixture.expectedComparabilityStatus,
|
||||
);
|
||||
expect(question.comparabilityStatus).toBe(
|
||||
fixture.expectedComparabilityStatus,
|
||||
);
|
||||
|
||||
if (fixture.expectsComparisonQuestion) {
|
||||
expect(question.question.toLowerCase()).toContain("same");
|
||||
expect(question.contradictionReasoningAllowed).toBe(false);
|
||||
} else {
|
||||
expect(question.question?.toLowerCase() || "").not.toContain(
|
||||
"same period and at the same scale",
|
||||
);
|
||||
}
|
||||
|
||||
if (fixture.key !== "sales-same") {
|
||||
expect(ambiguity.status).toBe("ambiguous");
|
||||
}
|
||||
|
||||
return {
|
||||
scenario: fixture.scenario,
|
||||
comparabilityStatus: assessment.comparabilityStatus,
|
||||
relationshipStatus: relationship.relationshipStatus,
|
||||
relationshipAssessed: relationship.relationshipAssessed,
|
||||
contradictionReasoningAllowed: question.contradictionReasoningAllowed,
|
||||
question: question.question,
|
||||
};
|
||||
});
|
||||
|
||||
expect(summary).toEqual([
|
||||
{
|
||||
scenario:
|
||||
"Revenue increased by 18%, but cash in the bank fell over the same period.",
|
||||
comparabilityStatus: "uncertain",
|
||||
relationshipStatus: "insufficient_information",
|
||||
relationshipAssessed: false,
|
||||
contradictionReasoningAllowed: false,
|
||||
question:
|
||||
"Were these figures measured on the same basis and at the same scale?",
|
||||
},
|
||||
{
|
||||
scenario: "Complaints increased. Production increased.",
|
||||
comparabilityStatus: "uncertain",
|
||||
relationshipStatus: "insufficient_information",
|
||||
relationshipAssessed: false,
|
||||
contradictionReasoningAllowed: false,
|
||||
question:
|
||||
"Were these figures measured over the same period and at the same scale?",
|
||||
},
|
||||
{
|
||||
scenario:
|
||||
"Average delivery time decreased by 25%, but order cancellations increased.",
|
||||
comparabilityStatus: "uncertain",
|
||||
relationshipStatus: "insufficient_information",
|
||||
relationshipAssessed: false,
|
||||
contradictionReasoningAllowed: false,
|
||||
question:
|
||||
"Were these figures measured over the same period and at the same scale?",
|
||||
},
|
||||
{
|
||||
scenario: "Customer satisfaction increased, but complaints increased.",
|
||||
comparabilityStatus: "uncertain",
|
||||
relationshipStatus: "insufficient_information",
|
||||
relationshipAssessed: false,
|
||||
contradictionReasoningAllowed: false,
|
||||
question:
|
||||
"Were these figures measured over the same period and at the same scale?",
|
||||
},
|
||||
{
|
||||
scenario: "Temperature increased. Ice melted.",
|
||||
comparabilityStatus: "confirmed",
|
||||
relationshipStatus: "compatible",
|
||||
relationshipAssessed: true,
|
||||
contradictionReasoningAllowed: false,
|
||||
question: null,
|
||||
},
|
||||
{
|
||||
scenario: "Sales doubled. Sales doubled.",
|
||||
comparabilityStatus: "confirmed",
|
||||
relationshipStatus: "duplicate",
|
||||
relationshipAssessed: true,
|
||||
contradictionReasoningAllowed: false,
|
||||
question: null,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("defers relationship classification while comparability is uncertain", () => {
|
||||
const fixture = comparabilityAssessmentFixtures[0];
|
||||
const relationship = classifyObservationRelationship(fixture.graph);
|
||||
|
||||
expect(relationship).toMatchObject({
|
||||
relationshipStatus: "insufficient_information",
|
||||
relationshipAssessed: false,
|
||||
contradictionReasoningAllowed: false,
|
||||
questionRequired: true,
|
||||
});
|
||||
expect(relationship.reasoningStages).toEqual([
|
||||
{
|
||||
stage: "comparability",
|
||||
status: "uncertain",
|
||||
outcome:
|
||||
"Comparability between the observations is not yet established across period, scale, or measurement basis.",
|
||||
},
|
||||
{
|
||||
stage: "relationship",
|
||||
status: "insufficient_information",
|
||||
outcome: "not assessed until comparability is established",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("allows contradiction reasoning only for genuine contradictions", () => {
|
||||
const serviceGraph = {
|
||||
centralStatement:
|
||||
"The service was reported as available throughout the hour and unavailable throughout the same hour.",
|
||||
nodes: [
|
||||
{
|
||||
id: "service-available",
|
||||
label: "The service was available throughout the hour.",
|
||||
description: "The service was available throughout the hour.",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
value: null,
|
||||
unit: null,
|
||||
evidenceIds: [],
|
||||
dependsOn: [],
|
||||
affects: [],
|
||||
parentId: null,
|
||||
childIds: [],
|
||||
},
|
||||
{
|
||||
id: "service-unavailable",
|
||||
label: "The service was unavailable throughout the same hour.",
|
||||
description: "The service was unavailable throughout the same hour.",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
value: null,
|
||||
unit: null,
|
||||
evidenceIds: [],
|
||||
dependsOn: [],
|
||||
affects: [],
|
||||
parentId: null,
|
||||
childIds: [],
|
||||
},
|
||||
],
|
||||
edges: [],
|
||||
activeUnknownNodeId: null,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: "Service contradiction fixture",
|
||||
};
|
||||
const relationship = classifyObservationRelationship(serviceGraph);
|
||||
|
||||
expect(relationship).toMatchObject({
|
||||
relationshipStatus: "contradictory",
|
||||
contradictionReasoningAllowed: true,
|
||||
questionRequired: true,
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,179 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { applyValidatedProposal } from "@/lib/graph/apply-proposal.js";
|
||||
import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js";
|
||||
|
||||
function makeFixture() {
|
||||
const parent = makeNode({
|
||||
id: "n-parent",
|
||||
label: "Explanation for why revenue increased while cash fell",
|
||||
description:
|
||||
"Need to understand what change or event could explain why these observations differ, because that is needed to investigate their relationship.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
const children = [
|
||||
makeNode({
|
||||
id: "n-child-1",
|
||||
label: "How the two observations were measured",
|
||||
description: "Need evidence about the measure used for each observation.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
parentId: parent.id,
|
||||
}),
|
||||
makeNode({
|
||||
id: "n-child-2",
|
||||
label: "Whether the two observations reflect different timing",
|
||||
description:
|
||||
"Need to know whether the two observations reflect different timing.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
parentId: parent.id,
|
||||
}),
|
||||
makeNode({
|
||||
id: "n-child-3",
|
||||
label: "Possible change mainly affecting revenue",
|
||||
description:
|
||||
"Need to know whether a possible change mainly affected revenue.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
parentId: parent.id,
|
||||
}),
|
||||
makeNode({
|
||||
id: "n-child-4",
|
||||
label: "Possible one-off event during the period",
|
||||
description:
|
||||
"Need to know whether a possible one-off event happened during the period.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
parentId: parent.id,
|
||||
}),
|
||||
];
|
||||
|
||||
return makeGraph({
|
||||
centralStatement: "Revenue increased while cash fell.",
|
||||
nodes: [parent, ...children],
|
||||
edges: children.map((child, index) =>
|
||||
makeEdge({
|
||||
id: `e-${index + 1}`,
|
||||
fromNodeId: child.id,
|
||||
toNodeId: parent.id,
|
||||
relationship: "depends_on",
|
||||
description: `${child.label} feeds the parent.`,
|
||||
}),
|
||||
),
|
||||
activeUnknownNodeId: "n-child-1",
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: "confidence propagation fixture",
|
||||
});
|
||||
}
|
||||
|
||||
function makeProposal({ resolvedIds, contradictedIds = [] }) {
|
||||
return {
|
||||
addedNodes: [
|
||||
makeNode({
|
||||
id: "n-anchor",
|
||||
label: "Update anchor",
|
||||
description:
|
||||
"Anchor state introduced by the answer because the update must contain a meaningful change.",
|
||||
kind: "state",
|
||||
status: "known",
|
||||
confidence: "low",
|
||||
}),
|
||||
],
|
||||
updatedNodes: [
|
||||
...resolvedIds.map((id) => ({
|
||||
nodeId: id,
|
||||
previousStatus: "unknown",
|
||||
newStatus: "resolved",
|
||||
previousValue: null,
|
||||
newValue: `answer:${id}`,
|
||||
reason: "resolved child",
|
||||
})),
|
||||
...contradictedIds.map((id) => ({
|
||||
nodeId: id,
|
||||
previousStatus: "unknown",
|
||||
newStatus: "contradicted",
|
||||
previousValue: null,
|
||||
newValue: `contradiction:${id}`,
|
||||
reason: "contradictory child evidence",
|
||||
})),
|
||||
],
|
||||
addedEdges: [],
|
||||
removedEdgeIds: [],
|
||||
resolvedUnknownNodeIds: resolvedIds,
|
||||
affectedNodeIds: [],
|
||||
selectedQuestion: null,
|
||||
};
|
||||
}
|
||||
|
||||
describe("confidence propagation", () => {
|
||||
it("one of four children resolved does not yield high conclusion confidence", () => {
|
||||
const result = applyValidatedProposal({
|
||||
situationGraph: makeFixture(),
|
||||
proposal: makeProposal({ resolvedIds: ["n-child-1"] }),
|
||||
previousQuestion:
|
||||
"What evidence would clarify how the two observations were measured?",
|
||||
answer: "Same accounting period and same management accounts.",
|
||||
});
|
||||
|
||||
const parent = result.updatedSituationGraph.nodes.find(
|
||||
(n) => n.id === "n-parent",
|
||||
);
|
||||
expect(parent.status).toBe("provisional");
|
||||
expect(parent.confidence).toBe("medium");
|
||||
expect(parent.confidenceAssessment).toEqual({
|
||||
evidenceConfidence: "medium",
|
||||
completenessStatus: "partial",
|
||||
conclusionConfidence: "medium",
|
||||
});
|
||||
expect(result.confidenceCapReason).toBe(
|
||||
"unresolved_direct_children_cap_conclusion",
|
||||
);
|
||||
});
|
||||
|
||||
it("all children resolved with coherent evidence may yield high confidence", () => {
|
||||
const result = applyValidatedProposal({
|
||||
situationGraph: makeFixture(),
|
||||
proposal: makeProposal({
|
||||
resolvedIds: ["n-child-1", "n-child-2", "n-child-3", "n-child-4"],
|
||||
}),
|
||||
previousQuestion:
|
||||
"What evidence would clarify how the two observations were measured?",
|
||||
answer: "All direct child questions are answered.",
|
||||
});
|
||||
|
||||
const parent = result.updatedSituationGraph.nodes.find(
|
||||
(n) => n.id === "n-parent",
|
||||
);
|
||||
expect(parent.status).toBe("resolved");
|
||||
expect(parent.confidenceAssessment).toEqual({
|
||||
evidenceConfidence: "high",
|
||||
completenessStatus: "complete",
|
||||
conclusionConfidence: "high",
|
||||
});
|
||||
});
|
||||
|
||||
it("contradictory child evidence prevents high confidence", () => {
|
||||
const result = applyValidatedProposal({
|
||||
situationGraph: makeFixture(),
|
||||
proposal: makeProposal({
|
||||
resolvedIds: ["n-child-1"],
|
||||
contradictedIds: ["n-child-2"],
|
||||
}),
|
||||
previousQuestion:
|
||||
"What evidence would clarify how the two observations were measured?",
|
||||
answer: "One child resolved, another contradicted.",
|
||||
});
|
||||
|
||||
const parent = result.updatedSituationGraph.nodes.find(
|
||||
(n) => n.id === "n-parent",
|
||||
);
|
||||
expect(parent.confidenceAssessment.conclusionConfidence).toBe("low");
|
||||
expect(result.confidenceCapReason).toBe("contradictory_direct_children");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,306 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
applyValidatedProposal,
|
||||
evaluateBranchInteractions,
|
||||
} from "@/lib/graph/apply-proposal.js";
|
||||
import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js";
|
||||
|
||||
function makeParentWithBranches(children) {
|
||||
const parent = makeNode({
|
||||
id: "n-parent",
|
||||
label: "Explanation for why revenue increased while cash fell",
|
||||
description:
|
||||
"Need to understand what change or event could explain why these observations differ, because that is needed to investigate their relationship.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
|
||||
return makeGraph({
|
||||
centralStatement: "Revenue increased while cash fell.",
|
||||
nodes: [
|
||||
parent,
|
||||
...children.map((child) => ({ ...child, parentId: parent.id })),
|
||||
],
|
||||
edges: children.map((child, index) =>
|
||||
makeEdge({
|
||||
id: `e-${index + 1}`,
|
||||
fromNodeId: child.id,
|
||||
toNodeId: parent.id,
|
||||
relationship: "depends_on",
|
||||
description: `${child.label} feeds the parent.`,
|
||||
}),
|
||||
),
|
||||
activeUnknownNodeId: children[0]?.id ?? null,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: "cross-branch corroboration fixture",
|
||||
});
|
||||
}
|
||||
|
||||
function makeResolvedChild(id, label, value, extra = {}) {
|
||||
return makeNode({
|
||||
id,
|
||||
label,
|
||||
description: label,
|
||||
kind: "unknown",
|
||||
status: "resolved",
|
||||
confidence: "medium",
|
||||
value,
|
||||
evidenceIds: extra.evidenceIds ?? [],
|
||||
});
|
||||
}
|
||||
|
||||
function makeUnknownBranch(id, label, description, extra = {}) {
|
||||
return makeNode({
|
||||
id,
|
||||
label,
|
||||
description,
|
||||
kind: "unknown",
|
||||
status: extra.status ?? "unknown",
|
||||
confidence: extra.confidence ?? "medium",
|
||||
evidenceIds: extra.evidenceIds ?? [],
|
||||
value: extra.value ?? null,
|
||||
});
|
||||
}
|
||||
|
||||
describe("evaluateBranchInteractions", () => {
|
||||
it("detects corroborating independent branches", () => {
|
||||
const graph = makeParentWithBranches([
|
||||
makeResolvedChild("n-a", "Debtor balance increased", "bank-statement-a", {
|
||||
evidenceIds: ["bank-statement-a"],
|
||||
}),
|
||||
makeResolvedChild(
|
||||
"n-b",
|
||||
"Cash receipts were delayed",
|
||||
"receipts-ledger-b",
|
||||
{ evidenceIds: ["receipts-ledger-b"] },
|
||||
),
|
||||
]);
|
||||
const parentNode = graph.nodes.find((node) => node.id === "n-parent");
|
||||
|
||||
const result = evaluateBranchInteractions({ parentNode, graph });
|
||||
|
||||
expect(result.interactionSummary.corroboratingBranchCount).toBe(1);
|
||||
expect(result.interactionSummary.duplicateEvidenceCount).toBe(0);
|
||||
expect(result.interactionSummary.conflictingBranchCount).toBe(0);
|
||||
});
|
||||
|
||||
it("detects duplicate evidence instead of corroboration", () => {
|
||||
const graph = makeParentWithBranches([
|
||||
makeResolvedChild(
|
||||
"n-a",
|
||||
"Bank statement shows increased debtor balance",
|
||||
"same-bank",
|
||||
{
|
||||
evidenceIds: ["same-bank"],
|
||||
},
|
||||
),
|
||||
makeResolvedChild(
|
||||
"n-b",
|
||||
"Delayed receipts also cite the bank statement",
|
||||
"same-bank",
|
||||
{
|
||||
evidenceIds: ["same-bank"],
|
||||
},
|
||||
),
|
||||
]);
|
||||
const parentNode = graph.nodes.find((node) => node.id === "n-parent");
|
||||
|
||||
const result = evaluateBranchInteractions({ parentNode, graph });
|
||||
|
||||
expect(result.interactionSummary.duplicateEvidenceCount).toBe(1);
|
||||
expect(result.interactionSummary.corroboratingBranchCount).toBe(0);
|
||||
});
|
||||
|
||||
it("detects conflicting branches", () => {
|
||||
const graph = makeParentWithBranches([
|
||||
makeResolvedChild("n-a", "Revenue recognised correctly", "correctly"),
|
||||
makeNode({
|
||||
id: "n-b",
|
||||
label: "Revenue recognised incorrectly",
|
||||
description: "Revenue recognised incorrectly",
|
||||
kind: "unknown",
|
||||
status: "contradicted",
|
||||
confidence: "medium",
|
||||
value: "incorrectly",
|
||||
}),
|
||||
]);
|
||||
const parentNode = graph.nodes.find((node) => node.id === "n-parent");
|
||||
|
||||
const result = evaluateBranchInteractions({ parentNode, graph });
|
||||
|
||||
expect(result.interactionSummary.conflictingBranchCount).toBe(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("cross-branch corroboration effects", () => {
|
||||
function applyToGraph(children, resolvedIds, contradictedIds = []) {
|
||||
const graph = makeParentWithBranches(children);
|
||||
return applyValidatedProposal({
|
||||
situationGraph: graph,
|
||||
proposal: {
|
||||
addedNodes: [
|
||||
makeNode({
|
||||
id: "n-anchor",
|
||||
label: "Update anchor",
|
||||
description:
|
||||
"Anchor state introduced by the answer because the update must contain a meaningful change.",
|
||||
kind: "state",
|
||||
status: "known",
|
||||
confidence: "low",
|
||||
}),
|
||||
],
|
||||
updatedNodes: [
|
||||
...resolvedIds.map((id) => ({
|
||||
nodeId: id,
|
||||
previousStatus: "unknown",
|
||||
newStatus: "resolved",
|
||||
previousValue: null,
|
||||
newValue: `answer:${id}`,
|
||||
reason: "resolved child",
|
||||
})),
|
||||
...contradictedIds.map((id) => ({
|
||||
nodeId: id,
|
||||
previousStatus: "unknown",
|
||||
newStatus: "contradicted",
|
||||
previousValue: null,
|
||||
newValue: `contradiction:${id}`,
|
||||
reason: "contradicted child",
|
||||
})),
|
||||
],
|
||||
addedEdges: [],
|
||||
removedEdgeIds: [],
|
||||
resolvedUnknownNodeIds: resolvedIds,
|
||||
affectedNodeIds: [],
|
||||
selectedQuestion: null,
|
||||
},
|
||||
previousQuestion: "What evidence would clarify this branch?",
|
||||
answer: "deterministic branch update",
|
||||
});
|
||||
}
|
||||
|
||||
it("independent corroboration increases justified confidence without reaching high on incomplete parent", () => {
|
||||
const result = applyToGraph(
|
||||
[
|
||||
makeUnknownBranch(
|
||||
"n-a",
|
||||
"Debtor balance increased",
|
||||
"Debtor balance increased",
|
||||
),
|
||||
makeUnknownBranch(
|
||||
"n-b",
|
||||
"Cash receipts delayed",
|
||||
"Cash receipts delayed",
|
||||
),
|
||||
makeUnknownBranch(
|
||||
"n-c",
|
||||
"Possible one-off event during the period",
|
||||
"Possible one-off event during the period",
|
||||
),
|
||||
makeUnknownBranch(
|
||||
"n-d",
|
||||
"Whether the two observations reflect different timing",
|
||||
"Whether the two observations reflect different timing",
|
||||
),
|
||||
],
|
||||
["n-a", "n-b"],
|
||||
);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.interactionSummary?.corroboratingBranchCount).toBeGreaterThan(
|
||||
0,
|
||||
);
|
||||
expect(result.interactionSummary?.duplicateEvidenceCount).toBe(0);
|
||||
expect(result.parentConfidenceAfter).toBe("medium");
|
||||
expect(result.confidenceCapReason).toBe(
|
||||
"independent_corroboration_with_incomplete_parent",
|
||||
);
|
||||
});
|
||||
|
||||
it("duplicate evidence does not increase confidence", () => {
|
||||
const result = applyToGraph(
|
||||
[
|
||||
makeUnknownBranch(
|
||||
"n-a",
|
||||
"Bank statement shows increased debtor balance",
|
||||
"Bank statement shows increased debtor balance",
|
||||
{ evidenceIds: ["same-bank"] },
|
||||
),
|
||||
makeUnknownBranch(
|
||||
"n-b",
|
||||
"Delayed receipts also cite the bank statement",
|
||||
"Delayed receipts also cite the bank statement",
|
||||
{ evidenceIds: ["same-bank"] },
|
||||
),
|
||||
makeUnknownBranch(
|
||||
"n-c",
|
||||
"Possible one-off event during the period",
|
||||
"Possible one-off event during the period",
|
||||
),
|
||||
],
|
||||
["n-a", "n-b"],
|
||||
);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.interactionSummary?.duplicateEvidenceCount).toBeGreaterThan(
|
||||
0,
|
||||
);
|
||||
expect(result.interactionSummary?.corroboratingBranchCount).toBe(0);
|
||||
expect(result.confidenceCapReason).toBe(
|
||||
"duplicate_evidence_no_extra_confidence",
|
||||
);
|
||||
});
|
||||
|
||||
it("conflicting evidence caps confidence", () => {
|
||||
const result = applyToGraph(
|
||||
[
|
||||
makeUnknownBranch(
|
||||
"n-a",
|
||||
"Revenue recognised correctly",
|
||||
"Revenue recognised correctly",
|
||||
),
|
||||
makeUnknownBranch(
|
||||
"n-b",
|
||||
"Revenue recognised incorrectly",
|
||||
"Revenue recognised incorrectly",
|
||||
),
|
||||
makeUnknownBranch(
|
||||
"n-c",
|
||||
"Possible one-off event during the period",
|
||||
"Possible one-off event during the period",
|
||||
),
|
||||
],
|
||||
["n-a"],
|
||||
["n-b"],
|
||||
);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.interactionSummary?.conflictingBranchCount).toBeGreaterThan(
|
||||
0,
|
||||
);
|
||||
expect(result.conclusionConfidenceAfter).toBe("low");
|
||||
});
|
||||
|
||||
it("independent branches stay interaction-neutral", () => {
|
||||
const result = applyToGraph(
|
||||
[
|
||||
makeUnknownBranch(
|
||||
"n-a",
|
||||
"Marketing campaign changed traffic",
|
||||
"Marketing campaign changed traffic",
|
||||
),
|
||||
makeUnknownBranch(
|
||||
"n-b",
|
||||
"Equipment maintenance occurred",
|
||||
"Equipment maintenance occurred",
|
||||
),
|
||||
],
|
||||
["n-a"],
|
||||
);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.interactionSummary?.independentBranchCount).toBeGreaterThan(
|
||||
0,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,278 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
assessChildUnknownQuality,
|
||||
applyValidatedProposal,
|
||||
MAX_DECOMPOSITION_DEPTH,
|
||||
} from "@/lib/graph/apply-proposal.js";
|
||||
import { makeGraph, makeNode } from "@/lib/graph/schema.js";
|
||||
|
||||
function makeParentGraph({
|
||||
centralStatement,
|
||||
parentLabel,
|
||||
parentDescription,
|
||||
observations = [],
|
||||
}) {
|
||||
const parent = makeNode({
|
||||
id: "n-parent",
|
||||
label: parentLabel,
|
||||
description: parentDescription,
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
|
||||
return {
|
||||
parent,
|
||||
graph: makeGraph({
|
||||
centralStatement,
|
||||
nodes: [parent, ...observations],
|
||||
edges: [],
|
||||
activeUnknownNodeId: parent.id,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: "Decomposition quality graph",
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
describe("assessChildUnknownQuality", () => {
|
||||
it("rejects 'Timing or measurement basis' as compound", () => {
|
||||
const { parent, graph } = makeParentGraph({
|
||||
centralStatement:
|
||||
"Revenue increased by 18%, but cash in the bank fell over the same period.",
|
||||
parentLabel:
|
||||
"Explanation for why revenue increased by 18%, but cash in the bank fell over the same period",
|
||||
parentDescription:
|
||||
"Need to understand what change or event could explain why these observations differ, because that is needed to investigate their relationship.",
|
||||
});
|
||||
const child = makeNode({
|
||||
id: "n-child",
|
||||
label: "Timing or measurement basis",
|
||||
description:
|
||||
"Need evidence about whether a timing or measurement-basis difference could explain the observations, because that would change how they should be interpreted.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
parentId: parent.id,
|
||||
});
|
||||
|
||||
const result = assessChildUnknownQuality({
|
||||
parentNode: parent,
|
||||
childNode: child,
|
||||
siblingNodes: [child],
|
||||
graph,
|
||||
});
|
||||
|
||||
expect(result.valid).toBe(false);
|
||||
expect(result.compoundSignals).toContain("timing_or_measurement_basis");
|
||||
expect(result.reasons).toContain("compound_child");
|
||||
});
|
||||
|
||||
it("accepts a child with one directly answerable uncertainty", () => {
|
||||
const { parent, graph } = makeParentGraph({
|
||||
centralStatement: "Traffic increased, but sales stayed flat.",
|
||||
parentLabel:
|
||||
"What explains why more website traffic did not produce more sales?",
|
||||
parentDescription:
|
||||
"Need an explanation because the observations moved differently.",
|
||||
});
|
||||
const child = makeNode({
|
||||
id: "n-child",
|
||||
label: "Different measurement basis between the two observations",
|
||||
description:
|
||||
"Need evidence about whether the two observations use different measurement bases, because that could help explain the difference.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
parentId: parent.id,
|
||||
});
|
||||
|
||||
const result = assessChildUnknownQuality({
|
||||
parentNode: parent,
|
||||
childNode: child,
|
||||
siblingNodes: [child],
|
||||
graph,
|
||||
});
|
||||
|
||||
expect(result.valid).toBe(true);
|
||||
expect(result.atomic).toBe(true);
|
||||
expect(result.directlyAnswerable).toBe(true);
|
||||
expect(result.narrowerThanParent).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects sibling duplicates", () => {
|
||||
const { parent, graph } = makeParentGraph({
|
||||
centralStatement: "Production increased, but defects also increased.",
|
||||
parentLabel: "What explains why output and defects both increased?",
|
||||
parentDescription:
|
||||
"Need an explanation because both observations increased.",
|
||||
});
|
||||
const childA = makeNode({
|
||||
id: "n-child-a",
|
||||
label: "Different timing between the two observations",
|
||||
description:
|
||||
"Need evidence about whether the two observations reflect different timing, because that could help explain the difference.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
parentId: parent.id,
|
||||
});
|
||||
const childB = makeNode({
|
||||
id: "n-child-b",
|
||||
label: "Different timing between the two observations",
|
||||
description:
|
||||
"Need evidence about whether the two observations reflect different timing, because that could help explain the difference.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
parentId: parent.id,
|
||||
});
|
||||
|
||||
const result = assessChildUnknownQuality({
|
||||
parentNode: parent,
|
||||
childNode: childA,
|
||||
siblingNodes: [childA, childB],
|
||||
graph,
|
||||
});
|
||||
|
||||
expect(result.valid).toBe(false);
|
||||
expect(result.duplicateSiblingIds).toContain("n-child-b");
|
||||
});
|
||||
|
||||
it("rejects parent paraphrases", () => {
|
||||
const { parent, graph } = makeParentGraph({
|
||||
centralStatement:
|
||||
"Customer satisfaction scores increased, but complaints also increased.",
|
||||
parentLabel:
|
||||
"What explains why satisfaction and complaints both increased?",
|
||||
parentDescription:
|
||||
"Need a broad explanation because the observations moved differently.",
|
||||
});
|
||||
const child = makeNode({
|
||||
id: "n-child",
|
||||
label: "What explains why satisfaction and complaints both increased?",
|
||||
description:
|
||||
"Need a broad explanation because the observations moved differently.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
parentId: parent.id,
|
||||
});
|
||||
|
||||
const result = assessChildUnknownQuality({
|
||||
parentNode: parent,
|
||||
childNode: child,
|
||||
siblingNodes: [child],
|
||||
graph,
|
||||
});
|
||||
|
||||
expect(result.valid).toBe(false);
|
||||
expect(result.reasons).toContain("not_narrower_than_parent");
|
||||
});
|
||||
});
|
||||
|
||||
describe("decomposition stopping conditions", () => {
|
||||
function makeMeaningfulNoOpProposal() {
|
||||
return {
|
||||
addedNodes: [
|
||||
makeNode({
|
||||
id: "n-anchor",
|
||||
label: "Update anchor",
|
||||
description:
|
||||
"Anchor state introduced by the answer because the update must contain a meaningful change.",
|
||||
kind: "state",
|
||||
status: "known",
|
||||
confidence: "low",
|
||||
}),
|
||||
],
|
||||
updatedNodes: [],
|
||||
addedEdges: [],
|
||||
removedEdgeIds: [],
|
||||
resolvedUnknownNodeIds: [],
|
||||
affectedNodeIds: [],
|
||||
selectedQuestion: null,
|
||||
};
|
||||
}
|
||||
|
||||
it("does not decompose an atomic selected unknown", () => {
|
||||
const atomic = makeNode({
|
||||
id: "n-atomic",
|
||||
label: "Were both figures measured over the same accounting period?",
|
||||
description:
|
||||
"Need to know whether both figures cover the same accounting period because that determines whether they are directly comparable.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
});
|
||||
const graph = makeGraph({
|
||||
centralStatement:
|
||||
"Revenue increased by 18%, but cash in the bank fell over the same period.",
|
||||
nodes: [atomic],
|
||||
edges: [],
|
||||
activeUnknownNodeId: atomic.id,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: "Atomic selected node graph",
|
||||
});
|
||||
|
||||
const result = applyValidatedProposal({
|
||||
situationGraph: graph,
|
||||
proposal: makeMeaningfulNoOpProposal(),
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.decompositionAttempted).toBe(false);
|
||||
expect(result.decompositionStoppedReason).toBe(
|
||||
"Selected unknown is already atomic.",
|
||||
);
|
||||
});
|
||||
|
||||
it("stops once a directly answerable child is selected", () => {
|
||||
const { parent, graph } = makeParentGraph({
|
||||
centralStatement: "Traffic increased, but sales stayed flat.",
|
||||
parentLabel:
|
||||
"What explains why more website traffic did not produce more sales?",
|
||||
parentDescription:
|
||||
"Need an explanation because the observations moved differently.",
|
||||
observations: [
|
||||
makeNode({
|
||||
id: "n-traffic",
|
||||
label: "Website traffic increased.",
|
||||
description: "Website traffic increased.",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
}),
|
||||
makeNode({
|
||||
id: "n-sales",
|
||||
label: "Sales stayed flat.",
|
||||
description: "Sales stayed flat.",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
const result = applyValidatedProposal({
|
||||
situationGraph: graph,
|
||||
proposal: makeMeaningfulNoOpProposal(),
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.decompositionAttempted).toBe(true);
|
||||
expect(result.decompositionAccepted).toBe(true);
|
||||
expect(result.selectedQuestion).toMatchObject({
|
||||
nodeId: expect.any(String),
|
||||
question:
|
||||
"What evidence would clarify how the two observations were measured?",
|
||||
});
|
||||
expect(result.selectedChildNodeId).toBe(result.selectedQuestion?.nodeId);
|
||||
expect(result.decompositionStoppedReason).toBe(
|
||||
"Selected child is atomic and directly answerable.",
|
||||
);
|
||||
});
|
||||
|
||||
it("exposes the configured maximum decomposition depth", () => {
|
||||
expect(MAX_DECOMPOSITION_DEPTH).toBeGreaterThanOrEqual(2);
|
||||
expect(MAX_DECOMPOSITION_DEPTH).toBeLessThanOrEqual(3);
|
||||
});
|
||||
});
|
||||
@@ -1,4 +1,5 @@
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
import { applyValidatedProposal } from "@/lib/graph/apply-proposal.js";
|
||||
import { validateGraphReferences } from "@/lib/graph/utils.js";
|
||||
import { makeGraph, makeNode } from "@/lib/graph/schema.js";
|
||||
|
||||
@@ -56,6 +57,107 @@ function makeAnalysisResult(overrides = {}) {
|
||||
};
|
||||
}
|
||||
|
||||
function makeCommercialAnalysisResult(overrides = {}) {
|
||||
return makeAnalysisResult({
|
||||
reconstruction: {
|
||||
summary:
|
||||
"A new reasoning method may become a commercial product, but problem existence and value remain unresolved.",
|
||||
actors: [],
|
||||
systemsOrObjects: [],
|
||||
expectedStates: [],
|
||||
observedStates: [],
|
||||
differences: [],
|
||||
knownTransitions: [],
|
||||
unexplainedTransitions: [],
|
||||
contradictions: [],
|
||||
importantUnknowns: [
|
||||
{
|
||||
id: "unk-commercial",
|
||||
label:
|
||||
"Commercial justification for whether continuing development is commercially justified",
|
||||
description:
|
||||
"Need to know whether this solves a genuine problem, whether people would value it enough to pay for it, and whether it is commercially justified before continuing development.",
|
||||
confidence: "high",
|
||||
},
|
||||
],
|
||||
plausibleInterpretations: [],
|
||||
},
|
||||
nextQuestion: {
|
||||
id: "q-commercial",
|
||||
question:
|
||||
"What specific validation metrics, pilot feedback, or competitive benchmarking results have you collected to measure whether the method solves a recognized problem and how target users evaluate its practical utility compared to existing tools?",
|
||||
reason: "Model-proposed broad validation question",
|
||||
},
|
||||
...overrides,
|
||||
});
|
||||
}
|
||||
|
||||
function makeCommercialTieAnalysisResult(overrides = {}) {
|
||||
return makeAnalysisResult({
|
||||
reconstruction: {
|
||||
summary:
|
||||
"A new reasoning method may become a commercial product, but multiple broad decision unknowns remain unresolved.",
|
||||
actors: [],
|
||||
systemsOrObjects: [],
|
||||
expectedStates: [],
|
||||
observedStates: [],
|
||||
differences: [],
|
||||
knownTransitions: [],
|
||||
unexplainedTransitions: [],
|
||||
contradictions: [],
|
||||
importantUnknowns: [
|
||||
{
|
||||
id: "unk-fit-pay",
|
||||
label:
|
||||
"Evidence of genuine problem-solution fit and actual willingness to pay among target users",
|
||||
description:
|
||||
"Need to know whether there is real problem-solution fit and willingness to pay among target users before continuing development.",
|
||||
confidence: "high",
|
||||
},
|
||||
{
|
||||
id: "unk-distinction",
|
||||
label:
|
||||
"Clear, measurable distinction between the method and existing AI tools that justifies separate commercial value",
|
||||
description:
|
||||
"Need to know whether there is a clear measurable distinction from existing AI tools before continuing development.",
|
||||
confidence: "high",
|
||||
},
|
||||
],
|
||||
plausibleInterpretations: [],
|
||||
},
|
||||
nextQuestion: {
|
||||
id: "q-commercial-tie",
|
||||
question:
|
||||
"What specific validation metrics, pilot feedback, or competitive benchmarking results have you collected?",
|
||||
reason: "Model-proposed broad validation question",
|
||||
},
|
||||
...overrides,
|
||||
});
|
||||
}
|
||||
|
||||
function makeCommercialUpdateGraph() {
|
||||
const parent = makeNode({
|
||||
id: "n-commercial-parent",
|
||||
label:
|
||||
"Commercial justification for whether continuing development is commercially justified",
|
||||
description:
|
||||
"Need to know whether this solves a genuine problem, whether people would value it enough to pay for it, and whether it is commercially justified before continuing development.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
|
||||
return makeGraph({
|
||||
centralStatement:
|
||||
"I have developed a new reasoning method that aims to help people determine whether they have enough justified confidence to make a decision. I believe it could become a commercial product, but I do not yet know whether it solves a genuine problem, whether people would value it enough to pay for it, or whether it is fundamentally different from existing AI tools. Before investing significant time and money into building it further, I want to determine whether continuing development is commercially justified.",
|
||||
nodes: [parent],
|
||||
edges: [],
|
||||
activeUnknownNodeId: parent.id,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: "Commercial update scenario",
|
||||
});
|
||||
}
|
||||
|
||||
function makeUpdateGraph() {
|
||||
const unknown = makeNode({
|
||||
id: "n-unknown",
|
||||
@@ -113,10 +215,95 @@ function makeProposal(overrides = {}) {
|
||||
removedEdgeIds: [],
|
||||
resolvedUnknownNodeIds: ["n-unknown"],
|
||||
affectedNodeIds: [],
|
||||
selectedQuestion: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function makeComparabilityScenarioGraph() {
|
||||
return makeGraph({
|
||||
centralStatement:
|
||||
"Revenue increased by 18%, but cash in the bank fell over the same period.",
|
||||
nodes: [
|
||||
makeNode({
|
||||
id: "n-comparability-unknown",
|
||||
label: "Whether the figures are comparable",
|
||||
description:
|
||||
"Need to know whether the figures use the same period, basis, and scale before comparing them.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
}),
|
||||
makeNode({
|
||||
id: "n-revenue-observation",
|
||||
label: "Revenue increased by 18%.",
|
||||
description: "Revenue increased by 18%.",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
}),
|
||||
makeNode({
|
||||
id: "n-cash-observation",
|
||||
label: "Cash in the bank decreased over the same period.",
|
||||
description: "Cash in the bank decreased over the same period.",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
}),
|
||||
],
|
||||
edges: [],
|
||||
activeUnknownNodeId: "n-comparability-unknown",
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: "Comparability scenario",
|
||||
reasoningState: {
|
||||
comparabilityStatus: "uncertain",
|
||||
comparabilityReason:
|
||||
"Comparability between the observations is not yet established across period, scale, or measurement basis.",
|
||||
comparabilityEvidence: [],
|
||||
relationshipStatus: "insufficient_information",
|
||||
relationshipReason:
|
||||
"Relationship classification is deferred until comparability is established.",
|
||||
relationshipAssessed: false,
|
||||
contradictionReasoningAllowed: false,
|
||||
reasoningStages: [
|
||||
{
|
||||
stage: "comparability",
|
||||
status: "uncertain",
|
||||
outcome:
|
||||
"Comparability between the observations is not yet established across period, scale, or measurement basis.",
|
||||
},
|
||||
{
|
||||
stage: "relationship",
|
||||
status: "insufficient_information",
|
||||
outcome: "not assessed until comparability is established",
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
function makeComparabilityProposal() {
|
||||
return {
|
||||
addedNodes: [],
|
||||
updatedNodes: [
|
||||
{
|
||||
nodeId: "n-comparability-unknown",
|
||||
previousStatus: "unknown",
|
||||
newStatus: "resolved",
|
||||
previousValue: null,
|
||||
newValue:
|
||||
"Both figures cover the same accounting period and are taken from the same management accounts.",
|
||||
reason: "The answer confirms comparability.",
|
||||
},
|
||||
],
|
||||
addedEdges: [],
|
||||
removedEdgeIds: [],
|
||||
resolvedUnknownNodeIds: ["n-comparability-unknown"],
|
||||
affectedNodeIds: [],
|
||||
selectedQuestion: null,
|
||||
};
|
||||
}
|
||||
|
||||
describe("lib/graph/orchestrator startCase", () => {
|
||||
beforeEach(() => {
|
||||
vi.resetModules();
|
||||
@@ -177,6 +364,96 @@ describe("lib/graph/orchestrator startCase", () => {
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.situationGraph.activeUnknownNodeId).toBeTruthy();
|
||||
expect(result.diagnostics.unknownSelectionExplanation?.selectedNodeId).toBe(
|
||||
result.situationGraph.activeUnknownNodeId,
|
||||
);
|
||||
});
|
||||
|
||||
it("returns an ambiguous tie result instead of choosing by label order", async () => {
|
||||
mockAnalyseScenario.mockResolvedValue(
|
||||
makeAnalysisResult({
|
||||
reconstruction: {
|
||||
summary: "Revenue up while cash falls",
|
||||
actors: [],
|
||||
systemsOrObjects: [],
|
||||
expectedStates: [],
|
||||
observedStates: [
|
||||
{
|
||||
id: "obs-1",
|
||||
label: "Revenue increased by 18%.",
|
||||
description: "Revenue increased by 18%.",
|
||||
confidence: "high",
|
||||
},
|
||||
{
|
||||
id: "obs-2",
|
||||
label: "Cash in the bank decreased over the same period.",
|
||||
description: "Cash in the bank decreased over the same period.",
|
||||
confidence: "high",
|
||||
},
|
||||
],
|
||||
differences: [],
|
||||
knownTransitions: [],
|
||||
unexplainedTransitions: [],
|
||||
contradictions: [
|
||||
{
|
||||
id: "c-1",
|
||||
label:
|
||||
"Apparent contradiction between profitability/revenue expansion and liquidity reduction.",
|
||||
description:
|
||||
"Apparent contradiction between profitability/revenue expansion and liquidity reduction.",
|
||||
confidence: "medium",
|
||||
},
|
||||
],
|
||||
importantUnknowns: [
|
||||
{
|
||||
id: "unk-1",
|
||||
label:
|
||||
"Whether revenue recognition timing differs from cash collection timing.",
|
||||
description:
|
||||
"Whether revenue recognition timing differs from cash collection timing.",
|
||||
confidence: "high",
|
||||
},
|
||||
{
|
||||
id: "unk-2",
|
||||
label:
|
||||
"Magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts).",
|
||||
description:
|
||||
"Magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts).",
|
||||
confidence: "high",
|
||||
},
|
||||
],
|
||||
plausibleInterpretations: [],
|
||||
},
|
||||
}),
|
||||
);
|
||||
const { startCase } = await import("@/lib/graph/orchestrator.js");
|
||||
|
||||
const result = await startCase({
|
||||
scenario:
|
||||
"Revenue increased by 18%, but cash in the bank fell over the same period.",
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.situationGraph.activeUnknownNodeId).toBeNull();
|
||||
expect(result.selectedQuestion).toMatchObject({
|
||||
id: "q_tie_resolution",
|
||||
selectionStatus: "ambiguous",
|
||||
question:
|
||||
"Were these figures measured on the same basis and at the same scale?",
|
||||
tiedCandidateIds: expect.arrayContaining([expect.any(String)]),
|
||||
comparabilityStatus: "uncertain",
|
||||
relationshipStatus: "insufficient_information",
|
||||
relationshipAssessed: false,
|
||||
contradictionReasoningAllowed: false,
|
||||
});
|
||||
expect(result.diagnostics.unknownSelectionExplanation).toMatchObject({
|
||||
status: "ambiguous",
|
||||
tieType: "complete_unresolved_tie",
|
||||
selectedNodeId: null,
|
||||
alphabeticalUsedAsReasoning: false,
|
||||
tieResolutionQuestion:
|
||||
"Were these figures measured on the same basis and at the same scale?",
|
||||
});
|
||||
});
|
||||
|
||||
it("returns structured failure when graph reference validation fails", async () => {
|
||||
@@ -226,9 +503,15 @@ describe("lib/graph/orchestrator startCase", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("returns null selectedQuestion when analysis has no nextQuestion", async () => {
|
||||
it("returns null selectedQuestion when neither analysis nor graph path yields a question", async () => {
|
||||
mockAnalyseScenario.mockResolvedValue(
|
||||
makeAnalysisResult({ nextQuestion: undefined }),
|
||||
makeAnalysisResult({
|
||||
nextQuestion: undefined,
|
||||
reconstruction: {
|
||||
...makeAnalysisResult().reconstruction,
|
||||
importantUnknowns: [],
|
||||
},
|
||||
}),
|
||||
);
|
||||
const { startCase } = await import("@/lib/graph/orchestrator.js");
|
||||
|
||||
@@ -238,6 +521,73 @@ describe("lib/graph/orchestrator startCase", () => {
|
||||
expect(result.selectedQuestion).toBeNull();
|
||||
});
|
||||
|
||||
it("uses the graph-backed question path instead of the reconstruction nextQuestion on startCase", async () => {
|
||||
mockAnalyseScenario.mockResolvedValue(makeCommercialAnalysisResult());
|
||||
const { startCase } = await import("@/lib/graph/orchestrator.js");
|
||||
|
||||
const result = await startCase({
|
||||
scenario:
|
||||
"I have developed a new reasoning method that aims to help people determine whether they have enough justified confidence to make a decision.",
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.selectedQuestion?.question).toBe(
|
||||
"Who experiences this problem?",
|
||||
);
|
||||
expect(result.selectedQuestion?.nodeId).toBe(
|
||||
result.situationGraph.activeUnknownNodeId,
|
||||
);
|
||||
expect(result.selectedQuestion?.question).not.toContain(
|
||||
"validation metrics, pilot feedback, or competitive benchmarking",
|
||||
);
|
||||
expect(result.diagnostics.reconstructionQuestion).toContain(
|
||||
"validation metrics, pilot feedback, or competitive benchmarking",
|
||||
);
|
||||
expect(result.diagnostics.reconstructionQuestionAccepted).toBe(false);
|
||||
expect(result.diagnostics.reconstructionQuestionRejectionReasons).toContain(
|
||||
"graph_backed_pipeline_required",
|
||||
);
|
||||
expect(result.diagnostics.finalGraphBackedQuestion).toBe(
|
||||
"Who experiences this problem?",
|
||||
);
|
||||
expect(result.diagnostics.selectedUnknownNodeId).toBe(
|
||||
result.situationGraph.activeUnknownNodeId,
|
||||
);
|
||||
});
|
||||
|
||||
it("reselects and decomposes a tied commercial start-case candidate instead of returning a silent null question", async () => {
|
||||
mockAnalyseScenario.mockResolvedValue(makeCommercialTieAnalysisResult());
|
||||
const { startCase } = await import("@/lib/graph/orchestrator.js");
|
||||
|
||||
const result = await startCase({
|
||||
scenario:
|
||||
"I have developed a new reasoning method that aims to help people determine whether they have enough justified confidence to make a decision. I believe it could become a commercial product, but I do not yet know whether it solves a genuine problem, whether people would value it enough to pay for it, or whether it is fundamentally different from existing AI tools. Before investing significant time and money into building it further, I want to determine whether continuing development is commercially justified.",
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(
|
||||
result.situationGraph.nodes.some((node) => node.kind === "unknown"),
|
||||
).toBe(true);
|
||||
expect(result.selectedQuestion).not.toBeNull();
|
||||
expect(result.selectedQuestion?.question).toBe(
|
||||
"Who experiences this problem?",
|
||||
);
|
||||
expect(result.selectedQuestion?.nodeId).toBe(
|
||||
result.situationGraph.activeUnknownNodeId,
|
||||
);
|
||||
expect(result.diagnostics.noQuestionReason).toBeNull();
|
||||
expect(result.diagnostics.selectedContainerUnknown).toBeTruthy();
|
||||
expect(result.diagnostics.selectedChildUnknown).toBeTruthy();
|
||||
expect(result.diagnostics.decompositionApplied).toBe(true);
|
||||
expect(result.diagnostics.reasoningPatternValidation).toMatchObject({
|
||||
activePattern: "decision",
|
||||
valid: true,
|
||||
});
|
||||
expect(result.selectedQuestion?.question.toLowerCase()).not.toMatch(
|
||||
/pay|price|pricing|budget|benchmark/,
|
||||
);
|
||||
});
|
||||
|
||||
it("includes compatibility diagnostics when provided by analysis", async () => {
|
||||
mockAnalyseScenario.mockResolvedValue(
|
||||
makeAnalysisResult({
|
||||
@@ -261,6 +611,69 @@ describe("lib/graph/orchestrator startCase", () => {
|
||||
expect(result.diagnostics.compatibilityChanges).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("preserves selected question bytes while adding selection explanation diagnostics", async () => {
|
||||
const { updateCase } = await import("@/lib/graph/orchestrator.js");
|
||||
const provider = {
|
||||
generateReconstruction: vi.fn().mockResolvedValue(
|
||||
makeProposal({
|
||||
addedNodes: [
|
||||
makeNode({
|
||||
id: "n-build-decision",
|
||||
label: "Build Confidence Engine decision",
|
||||
description: "Decision introduced by the answer.",
|
||||
kind: "state",
|
||||
status: "supported",
|
||||
confidence: "medium",
|
||||
}),
|
||||
makeNode({
|
||||
id: "n-commercial-value",
|
||||
label: "Commercial value definition",
|
||||
description:
|
||||
"Need a concrete definition because the decision depends on it.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
}),
|
||||
],
|
||||
addedEdges: [
|
||||
{
|
||||
id: "e-build-commercial-value",
|
||||
fromNodeId: "n-build-decision",
|
||||
toNodeId: "n-commercial-value",
|
||||
relationship: "depends_on",
|
||||
confidence: "medium",
|
||||
description:
|
||||
"The decision depends on commercial value definition.",
|
||||
},
|
||||
],
|
||||
selectedQuestion: {
|
||||
nodeId: "n-commercial-value",
|
||||
question:
|
||||
"How should commercial value be defined for this decision?",
|
||||
reason: "Consequential unresolved uncertainty remains.",
|
||||
},
|
||||
}),
|
||||
),
|
||||
};
|
||||
|
||||
const first = await updateCase(makeUpdateRequest(), {
|
||||
provider,
|
||||
config: MOCK_CONFIG,
|
||||
applyProposal: true,
|
||||
});
|
||||
const second = await updateCase(makeUpdateRequest(), {
|
||||
provider,
|
||||
config: MOCK_CONFIG,
|
||||
applyProposal: true,
|
||||
});
|
||||
|
||||
expect(first.selectedQuestion.question).toBe(
|
||||
second.selectedQuestion.question,
|
||||
);
|
||||
expect(first.selectedQuestion.reason).toBe(second.selectedQuestion.reason);
|
||||
expect(first.diagnostics.unknownSelectionExplanation).toBeTruthy();
|
||||
});
|
||||
|
||||
it("produces a validated update proposal for a valid request", async () => {
|
||||
const { updateCase } = await import("@/lib/graph/orchestrator.js");
|
||||
const provider = {
|
||||
@@ -529,6 +942,218 @@ describe("lib/graph/orchestrator startCase", () => {
|
||||
expect(result.proposal.nextQuestion).toBeUndefined();
|
||||
});
|
||||
|
||||
it("returns selectedQuestion from applied update proposal", async () => {
|
||||
const { updateCase } = await import("@/lib/graph/orchestrator.js");
|
||||
const provider = {
|
||||
generateReconstruction: vi.fn().mockResolvedValue(
|
||||
makeProposal({
|
||||
addedNodes: [
|
||||
makeNode({
|
||||
id: "n-build-decision",
|
||||
label: "Build Confidence Engine decision",
|
||||
description: "Decision introduced by the answer.",
|
||||
kind: "state",
|
||||
status: "supported",
|
||||
confidence: "medium",
|
||||
}),
|
||||
makeNode({
|
||||
id: "n-commercial-value",
|
||||
label: "Commercial value definition",
|
||||
description:
|
||||
"Need a concrete definition because the decision depends on it.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
}),
|
||||
],
|
||||
addedEdges: [
|
||||
{
|
||||
id: "e-build-commercial-value",
|
||||
fromNodeId: "n-build-decision",
|
||||
toNodeId: "n-commercial-value",
|
||||
relationship: "depends_on",
|
||||
confidence: "medium",
|
||||
description:
|
||||
"The decision depends on commercial value definition.",
|
||||
},
|
||||
],
|
||||
selectedQuestion: {
|
||||
nodeId: "n-commercial-value",
|
||||
question:
|
||||
"How should commercial value be defined for this decision?",
|
||||
reason: "Consequential unresolved uncertainty remains.",
|
||||
},
|
||||
}),
|
||||
),
|
||||
};
|
||||
|
||||
const result = await updateCase(makeUpdateRequest(), {
|
||||
provider,
|
||||
config: MOCK_CONFIG,
|
||||
applyProposal: true,
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.selectedQuestion?.nodeId).toBe("n-commercial-value");
|
||||
expect(result.newActiveUnknownNodeId).toBe("n-commercial-value");
|
||||
expect(result.selectedQuestion?.question).not.toBe(
|
||||
"How should commercial value be defined for this decision?",
|
||||
);
|
||||
expect(result.selectedQuestion?.question.toLowerCase()).not.toContain(
|
||||
"how should uncertainty regarding",
|
||||
);
|
||||
});
|
||||
|
||||
it("deterministically prioritises customer value over pricing follow-up", async () => {
|
||||
const { updateCase } = await import("@/lib/graph/orchestrator.js");
|
||||
const provider = {
|
||||
generateReconstruction: vi.fn().mockResolvedValue(
|
||||
makeProposal({
|
||||
addedNodes: [
|
||||
makeNode({
|
||||
id: "n-value",
|
||||
label: "Customer value",
|
||||
description:
|
||||
"Need customer value because purchase decisions depend on it.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
}),
|
||||
makeNode({
|
||||
id: "n-price",
|
||||
label: "Target price point",
|
||||
description:
|
||||
"Need a target price point because revenue assumptions depend on it.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
dependsOn: ["n-value"],
|
||||
}),
|
||||
makeNode({
|
||||
id: "n-decision",
|
||||
label: "Build Confidence Engine decision",
|
||||
description: "Decision introduced by the answer.",
|
||||
kind: "state",
|
||||
status: "supported",
|
||||
confidence: "medium",
|
||||
}),
|
||||
],
|
||||
addedEdges: [
|
||||
{
|
||||
id: "e-decision-value",
|
||||
fromNodeId: "n-decision",
|
||||
toNodeId: "n-value",
|
||||
relationship: "depends_on",
|
||||
confidence: "medium",
|
||||
description: "The decision depends on customer value.",
|
||||
},
|
||||
{
|
||||
id: "e-value-price",
|
||||
fromNodeId: "n-value",
|
||||
toNodeId: "n-price",
|
||||
relationship: "depends_on",
|
||||
confidence: "medium",
|
||||
description: "Pricing depends on customer value.",
|
||||
},
|
||||
{
|
||||
id: "e-decision-price",
|
||||
fromNodeId: "n-decision",
|
||||
toNodeId: "n-price",
|
||||
relationship: "depends_on",
|
||||
confidence: "low",
|
||||
description: "The decision references pricing assumptions.",
|
||||
},
|
||||
],
|
||||
selectedQuestion: {
|
||||
nodeId: "n-price",
|
||||
question: "What is the price point?",
|
||||
reason: "Model chose pricing.",
|
||||
},
|
||||
}),
|
||||
),
|
||||
};
|
||||
|
||||
const result = await updateCase(makeUpdateRequest(), {
|
||||
provider,
|
||||
config: MOCK_CONFIG,
|
||||
applyProposal: true,
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.selectedQuestion?.nodeId).toBe("n-value");
|
||||
});
|
||||
|
||||
it("keeps a follow-up question when a resolved child still has an eligible sibling", async () => {
|
||||
const { updateCase } = await import("@/lib/graph/orchestrator.js");
|
||||
const initialGraph = makeCommercialUpdateGraph();
|
||||
const firstPass = applyValidatedProposal({
|
||||
situationGraph: initialGraph,
|
||||
proposal: {
|
||||
addedNodes: [
|
||||
makeNode({
|
||||
id: "n-anchor",
|
||||
label: "Update anchor",
|
||||
description:
|
||||
"Anchor state introduced by the answer because the update must contain a meaningful change.",
|
||||
kind: "state",
|
||||
status: "known",
|
||||
confidence: "low",
|
||||
}),
|
||||
],
|
||||
updatedNodes: [],
|
||||
addedEdges: [],
|
||||
removedEdgeIds: [],
|
||||
resolvedUnknownNodeIds: [],
|
||||
affectedNodeIds: [],
|
||||
selectedQuestion: null,
|
||||
},
|
||||
});
|
||||
|
||||
expect(firstPass.success).toBe(true);
|
||||
|
||||
const provider = {
|
||||
generateReconstruction: vi.fn().mockResolvedValue({
|
||||
addedNodes: [],
|
||||
updatedNodes: [
|
||||
{
|
||||
nodeId: firstPass.selectedQuestion.nodeId,
|
||||
previousStatus: "unknown",
|
||||
newStatus: "resolved",
|
||||
previousValue: null,
|
||||
newValue:
|
||||
"I experience it myself when deciding whether a project or investment is justified.",
|
||||
reason: "The answer resolves the first child.",
|
||||
},
|
||||
],
|
||||
addedEdges: [],
|
||||
removedEdgeIds: [],
|
||||
resolvedUnknownNodeIds: [firstPass.selectedQuestion.nodeId],
|
||||
affectedNodeIds: [],
|
||||
selectedQuestion: null,
|
||||
}),
|
||||
};
|
||||
|
||||
const result = await updateCase(
|
||||
{
|
||||
situationGraph: firstPass.updatedSituationGraph,
|
||||
previousQuestion: firstPass.selectedQuestion.question,
|
||||
answer:
|
||||
"I experience it myself when deciding whether a project or investment is justified.",
|
||||
promptVersion: "v0.4",
|
||||
},
|
||||
{
|
||||
provider,
|
||||
config: MOCK_CONFIG,
|
||||
applyProposal: true,
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.selectedQuestion).toBeTruthy();
|
||||
expect(result.newActiveUnknownNodeId).toBe(result.selectedQuestion?.nodeId);
|
||||
expect(result.diagnostics.noQuestionReason).toBeNull();
|
||||
});
|
||||
|
||||
it("defaults to proposal-only mode", async () => {
|
||||
const { updateCase } = await import("@/lib/graph/orchestrator.js");
|
||||
const applyValidatedProposal = vi.fn();
|
||||
@@ -635,16 +1260,192 @@ describe("lib/graph/orchestrator startCase", () => {
|
||||
});
|
||||
});
|
||||
|
||||
it("startCase behaviour remains unchanged", async () => {
|
||||
it("advances reasoning after comparability is resolved by the update answer", async () => {
|
||||
const { updateCase } = await import("@/lib/graph/orchestrator.js");
|
||||
const provider = {
|
||||
generateReconstruction: vi
|
||||
.fn()
|
||||
.mockResolvedValue(makeComparabilityProposal()),
|
||||
};
|
||||
|
||||
const result = await updateCase(
|
||||
{
|
||||
situationGraph: makeComparabilityScenarioGraph(),
|
||||
previousQuestion:
|
||||
"Were these figures measured on the same basis and at the same scale?",
|
||||
answer:
|
||||
"Yes. Both figures cover the same accounting period and are taken from the same management accounts.",
|
||||
promptVersion: "v0.4",
|
||||
},
|
||||
{
|
||||
provider,
|
||||
config: MOCK_CONFIG,
|
||||
applyProposal: true,
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.resolvedUnknownNodeIds).toEqual(["n-comparability-unknown"]);
|
||||
expect(result.diagnostics).toMatchObject({
|
||||
previousComparabilityStatus: "uncertain",
|
||||
comparabilityStatus: "confirmed",
|
||||
relationshipStatus: "potentially_related",
|
||||
relationshipAssessed: true,
|
||||
resolvedReasoningNodeIds: ["reasoning:comparability"],
|
||||
emergentReasoningNodeCreated: true,
|
||||
atomicityAssessment: "composite",
|
||||
decompositionPerformed: true,
|
||||
childUnknownCount: 5,
|
||||
});
|
||||
expect(result.diagnostics.emergentReasoningNodeId).toBeTruthy();
|
||||
expect(result.diagnostics.childNodeIds).toHaveLength(5);
|
||||
expect(result.diagnostics.atomicityReason).toBeTruthy();
|
||||
expect(result.diagnostics.emergentReasoningNodeReason).toContain(
|
||||
"backed by the graph",
|
||||
);
|
||||
expect(result.diagnostics.reasoningStagesBefore).toEqual([
|
||||
{
|
||||
stage: "comparability",
|
||||
status: "uncertain",
|
||||
outcome:
|
||||
"Comparability between the observations is not yet established across period, scale, or measurement basis.",
|
||||
},
|
||||
{
|
||||
stage: "relationship",
|
||||
status: "insufficient_information",
|
||||
outcome: "not assessed until comparability is established",
|
||||
},
|
||||
]);
|
||||
expect(result.diagnostics.reasoningStagesAfter).toEqual([
|
||||
{
|
||||
stage: "comparability",
|
||||
status: "confirmed",
|
||||
outcome:
|
||||
"Comparability was confirmed by the user answer covering the same period and source basis.",
|
||||
},
|
||||
{
|
||||
stage: "relationship",
|
||||
status: "potentially_related",
|
||||
outcome:
|
||||
"The observations concern connected business signals but do not establish a direct contradiction or cause.",
|
||||
},
|
||||
]);
|
||||
expect(result.selectedQuestion?.nodeId).toBe(result.newActiveUnknownNodeId);
|
||||
expect(result.selectedQuestion?.question).toBe(
|
||||
"What evidence would clarify how the two observations were measured?",
|
||||
);
|
||||
expect(result.selectedQuestion?.question.toLowerCase()).not.toMatch(
|
||||
/same basis|dso|receivables|debtor days|working capital/,
|
||||
);
|
||||
});
|
||||
|
||||
it("reselects the other-people sibling after the first commercial child is answered", async () => {
|
||||
const { updateCase } = await import("@/lib/graph/orchestrator.js");
|
||||
|
||||
const seededGraph = applyValidatedProposal({
|
||||
situationGraph: makeCommercialUpdateGraph(),
|
||||
proposal: {
|
||||
addedNodes: [
|
||||
makeNode({
|
||||
id: "n-anchor",
|
||||
label: "Update anchor",
|
||||
description:
|
||||
"Anchor state introduced by the answer because the update must contain a meaningful change.",
|
||||
kind: "state",
|
||||
status: "known",
|
||||
confidence: "low",
|
||||
}),
|
||||
],
|
||||
updatedNodes: [],
|
||||
addedEdges: [],
|
||||
removedEdgeIds: [],
|
||||
resolvedUnknownNodeIds: [],
|
||||
affectedNodeIds: [],
|
||||
selectedQuestion: null,
|
||||
},
|
||||
});
|
||||
|
||||
expect(seededGraph.success).toBe(true);
|
||||
const resolvedFirstChildNodeId = seededGraph.selectedQuestion?.nodeId;
|
||||
const resolvedFirstQuestion = seededGraph.selectedQuestion?.question;
|
||||
|
||||
const initial = await updateCase(
|
||||
{
|
||||
situationGraph: seededGraph.updatedSituationGraph,
|
||||
previousQuestion: resolvedFirstQuestion,
|
||||
answer:
|
||||
"I experience it myself when I am trying to decide whether a project, idea or investment is justified, but I do not yet know how common that problem is for other people.",
|
||||
promptVersion: "v0.4",
|
||||
},
|
||||
{
|
||||
applyProposal: true,
|
||||
config: MOCK_CONFIG,
|
||||
provider: {
|
||||
generateReconstruction: vi.fn().mockResolvedValue({
|
||||
addedNodes: [],
|
||||
updatedNodes: [
|
||||
{
|
||||
nodeId: resolvedFirstChildNodeId,
|
||||
previousStatus: "unknown",
|
||||
newStatus: "resolved",
|
||||
previousValue: null,
|
||||
newValue:
|
||||
"I experience it myself when I am trying to decide whether a project, idea or investment is justified, but I do not yet know how common that problem is for other people.",
|
||||
reason: "The answer confirms a self-observed instance.",
|
||||
},
|
||||
],
|
||||
addedEdges: [],
|
||||
removedEdgeIds: [],
|
||||
resolvedUnknownNodeIds: [resolvedFirstChildNodeId],
|
||||
affectedNodeIds: [],
|
||||
selectedQuestion: null,
|
||||
}),
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
expect(initial.success).toBe(true);
|
||||
expect(initial.selectedQuestion?.nodeId).toBe(
|
||||
initial.newActiveUnknownNodeId,
|
||||
);
|
||||
expect(initial.selectedQuestion?.question).toBe(
|
||||
"What makes you think other people experience this problem too?",
|
||||
);
|
||||
expect(initial.selectedQuestion?.reasoningPattern).toBe("decision");
|
||||
expect(initial.diagnostics.unresolvedCandidateCount).toBeGreaterThan(0);
|
||||
expect(initial.diagnostics.eligibleCandidateCount).toBeGreaterThan(0);
|
||||
expect(initial.diagnostics.candidateNodeIds).toContain(
|
||||
initial.selectedQuestion?.nodeId,
|
||||
);
|
||||
expect(initial.diagnostics.resolvedCurrentTurnNodeIds).toContain(
|
||||
resolvedFirstChildNodeId,
|
||||
);
|
||||
expect(initial.diagnostics.noQuestionReason).toBeNull();
|
||||
expect(initial.diagnostics.reasoningPatternValidation).toMatchObject({
|
||||
activePattern: "decision",
|
||||
valid: true,
|
||||
});
|
||||
expect(initial.diagnostics.graphReasoningIntegrity).toBe("valid");
|
||||
expect(initial.diagnostics.incompatibleNodeIds).toEqual([]);
|
||||
expect(initial.selectedQuestion?.question.toLowerCase()).not.toMatch(
|
||||
/price|budget|market size|pilot metrics|benchmark|technical differentiation/,
|
||||
);
|
||||
});
|
||||
|
||||
it("startCase no longer copies analysis nextQuestion directly when a graph-backed question exists", async () => {
|
||||
mockAnalyseScenario.mockResolvedValue(makeAnalysisResult());
|
||||
const { startCase } = await import("@/lib/graph/orchestrator.js");
|
||||
|
||||
const result = await startCase({ scenario: "Scenario text" });
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.selectedQuestion).toEqual({
|
||||
id: "q-1",
|
||||
question: "What denominator is being used for the complaint rate?",
|
||||
expect(result.selectedQuestion).toMatchObject({
|
||||
nodeId: result.situationGraph.activeUnknownNodeId,
|
||||
question:
|
||||
"What would clarify need the denominator for complaint rate in this situation?",
|
||||
});
|
||||
expect(result.diagnostics.reconstructionQuestion).toBe(
|
||||
"What denominator is being used for the complaint rate?",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -72,6 +72,7 @@ describe("buildGraphUpdatePrompt", () => {
|
||||
expect(prompt).toContain("removedEdgeIds");
|
||||
expect(prompt).toContain("resolvedUnknownNodeIds");
|
||||
expect(prompt).toContain("affectedNodeIds");
|
||||
expect(prompt).toContain("selectedQuestion");
|
||||
});
|
||||
|
||||
it("lists enum values", () => {
|
||||
@@ -98,4 +99,16 @@ describe("buildGraphUpdatePrompt", () => {
|
||||
expect(prompt).toContain("Return JSON only");
|
||||
expect(prompt).toContain("Return one JSON object only");
|
||||
});
|
||||
|
||||
it("describes controlled emergent unknown rules", () => {
|
||||
const prompt = buildGraphUpdatePrompt(makeContext());
|
||||
expect(prompt).toContain("Add at most 3 new unknown nodes");
|
||||
expect(prompt).toContain("Resolve the answered unknown first");
|
||||
expect(prompt).toContain(
|
||||
"selectedQuestion.question must be one narrow non-compound question",
|
||||
);
|
||||
expect(prompt).toContain(
|
||||
"the engine will deterministically choose final priority after validation",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,539 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
assessUnknownAtomicity,
|
||||
formulateQuestion,
|
||||
formulateTieResolutionQuestion,
|
||||
selectReasoningPattern,
|
||||
selectInvestigationStrategy,
|
||||
} from "@/lib/graph/question-formulator.js";
|
||||
import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js";
|
||||
|
||||
function makeGraphFor(node, extra = {}) {
|
||||
return makeGraph({
|
||||
centralStatement: extra.centralStatement || "Decision context",
|
||||
nodes: [node, ...(extra.nodes || [])],
|
||||
edges: extra.edges || [],
|
||||
activeUnknownNodeId: node.id,
|
||||
resolvedNodeIds: extra.resolvedNodeIds || [],
|
||||
currentSummary: "Test summary",
|
||||
});
|
||||
}
|
||||
|
||||
describe("formulateQuestion", () => {
|
||||
it("atomicity assessment leaves focused unknowns direct and marks broad explanation unknowns composite", () => {
|
||||
const atomicUnknown = makeNode({
|
||||
id: "n-atomic",
|
||||
label: "Complaint rate denominator",
|
||||
description:
|
||||
"Need the denominator because it directly determines the complaint rate.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
});
|
||||
const compositeUnknown = makeNode({
|
||||
id: "n-composite",
|
||||
label:
|
||||
"Explanation for why revenue increased by 18%, but cash in the bank fell over the same period",
|
||||
description:
|
||||
"Need to understand what change or event could explain why these observations differ, because that is needed to investigate their relationship.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
const compositeGraph = makeGraphFor(compositeUnknown, {
|
||||
centralStatement:
|
||||
"Revenue increased by 18%, but cash in the bank fell over the same period.",
|
||||
nodes: [
|
||||
makeNode({
|
||||
id: "n-revenue-observation",
|
||||
label: "Revenue increased by 18%.",
|
||||
description: "Revenue increased by 18%.",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
}),
|
||||
makeNode({
|
||||
id: "n-cash-observation",
|
||||
label: "Cash in the bank decreased over the same period.",
|
||||
description: "Cash in the bank decreased over the same period.",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
expect(
|
||||
assessUnknownAtomicity({
|
||||
node: atomicUnknown,
|
||||
graph: makeGraphFor(atomicUnknown),
|
||||
}).atomicity,
|
||||
).toBe("atomic");
|
||||
expect(
|
||||
assessUnknownAtomicity({
|
||||
node: compositeUnknown,
|
||||
graph: compositeGraph,
|
||||
}).atomicity,
|
||||
).toBe("composite");
|
||||
});
|
||||
|
||||
it("commercial viability plus build decision produces a decision-threshold question", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-commercial",
|
||||
label: "Uncertainty regarding the commercial value of the product",
|
||||
description:
|
||||
"Commercial justification remains unclear because the decision depends on it.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
parentId: "n-decision",
|
||||
});
|
||||
const decision = makeNode({
|
||||
id: "n-decision",
|
||||
label: "Build decision",
|
||||
description: "Decision introduced by the answer.",
|
||||
kind: "state",
|
||||
status: "known",
|
||||
confidence: "medium",
|
||||
childIds: [unknown.id],
|
||||
value: "Deciding whether to build the product",
|
||||
});
|
||||
const graph = makeGraphFor(unknown, {
|
||||
nodes: [decision],
|
||||
resolvedNodeIds: [decision.id],
|
||||
});
|
||||
|
||||
const result = formulateQuestion({ node: unknown, graph });
|
||||
|
||||
expect(result.strategy).toBe("decision_threshold");
|
||||
expect(result.reasoningPattern).toBe("decision");
|
||||
expect(result.questionFamily).toBe("decision_threshold");
|
||||
expect(result.question).toContain("What outcome");
|
||||
expect(result.question.toLowerCase()).toContain("justify");
|
||||
});
|
||||
|
||||
it("commercial viability does not produce a pricing-first question", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-commercial",
|
||||
label: "Commercial viability",
|
||||
description:
|
||||
"Commercial viability remains unresolved because the decision depends on it.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
});
|
||||
const graph = makeGraphFor(unknown);
|
||||
|
||||
const result = formulateQuestion({ node: unknown, graph });
|
||||
|
||||
expect(result.question.toLowerCase()).not.toContain("price");
|
||||
expect(result.question.toLowerCase()).not.toContain("pricing");
|
||||
});
|
||||
|
||||
it("undefined term produces a definition question", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-term",
|
||||
label: "Success criteria definition",
|
||||
description:
|
||||
"Need a definition of the term because the team uses it inconsistently.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
|
||||
const result = formulateQuestion({
|
||||
node: unknown,
|
||||
graph: makeGraphFor(unknown),
|
||||
});
|
||||
|
||||
expect(result.strategy).toBe("definition");
|
||||
expect(result.reasoningPattern).toBe("definition");
|
||||
expect(result.questionFamily).toBe("definition");
|
||||
expect(result.question).toMatch(/^What does /);
|
||||
});
|
||||
|
||||
it("unsupported claim produces an evidence question", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-claim",
|
||||
label: "Demand claim",
|
||||
description: "Need evidence because the claim has not been validated.",
|
||||
kind: "reported_claim",
|
||||
status: "provisional",
|
||||
confidence: "low",
|
||||
});
|
||||
|
||||
const result = formulateQuestion({
|
||||
node: unknown,
|
||||
graph: makeGraphFor(unknown),
|
||||
});
|
||||
|
||||
expect(result.strategy).toBe("evidence_gathering");
|
||||
expect(result.question).toContain("What evidence");
|
||||
});
|
||||
|
||||
it("missing previous state produces a baseline question", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-baseline",
|
||||
label: "Baseline conversion rate",
|
||||
description:
|
||||
"Need the previous baseline because the change cannot be assessed without it.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
|
||||
const result = formulateQuestion({
|
||||
node: unknown,
|
||||
graph: makeGraphFor(unknown),
|
||||
});
|
||||
|
||||
expect(result.strategy).toBe("baseline_reconstruction");
|
||||
expect(result.question).toContain("What was the comparable state before");
|
||||
});
|
||||
|
||||
it("conflicting claim produces a contradiction-resolution question", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-conflict",
|
||||
label: "Conflicting churn claim",
|
||||
description:
|
||||
"Need to resolve the inconsistency because the current figures contradict each other.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
|
||||
const contradiction = makeNode({
|
||||
id: "n-contradiction",
|
||||
label: "Contradicted report",
|
||||
description: "Two sources disagree about churn.",
|
||||
kind: "conclusion",
|
||||
status: "contradicted",
|
||||
confidence: "low",
|
||||
childIds: [unknown.id],
|
||||
});
|
||||
|
||||
const result = formulateQuestion({
|
||||
node: unknown,
|
||||
graph: makeGraphFor(unknown, { nodes: [contradiction] }),
|
||||
});
|
||||
|
||||
expect(result.strategy).toBe("contradiction_resolution");
|
||||
expect(result.reasoningPattern).toBe("contradiction");
|
||||
expect(result.question).toContain("resolve the contradiction");
|
||||
});
|
||||
|
||||
it("reasoning pattern selection marks commercial validation as decision rather than explanation", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-commercial-pattern",
|
||||
label:
|
||||
"Whether the method addresses a genuine, high-priority problem for a specific audience",
|
||||
description:
|
||||
"Need to know whether this solves a real problem for a clear audience before continuing development.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
});
|
||||
const graph = makeGraphFor(unknown, {
|
||||
centralStatement:
|
||||
"Before investing more, we need to know whether continuing development is commercially justified.",
|
||||
});
|
||||
|
||||
const result = selectReasoningPattern({ node: unknown, graph });
|
||||
|
||||
expect(result.pattern).toBe("decision");
|
||||
});
|
||||
|
||||
it("comparison scenario selects the comparison pattern", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-comparison-pattern",
|
||||
label: "How the two observations were measured",
|
||||
description:
|
||||
"Need evidence about the measure used for each observation, because that could help explain the difference.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
const graph = makeGraphFor(unknown, {
|
||||
centralStatement: "Traffic increased, but sales stayed flat.",
|
||||
nodes: [
|
||||
makeNode({
|
||||
id: "n-traffic-observation",
|
||||
label: "Traffic increased.",
|
||||
description: "Traffic increased.",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
}),
|
||||
makeNode({
|
||||
id: "n-sales-observation",
|
||||
label: "Sales stayed flat.",
|
||||
description: "Sales stayed flat.",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
const result = selectReasoningPattern({ node: unknown, graph });
|
||||
|
||||
expect(result.pattern).toBe("comparison");
|
||||
});
|
||||
|
||||
it("constraint unknown uses evidence-gathering within the fixed strategy set", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-constraint",
|
||||
label: "Budget constraint",
|
||||
description:
|
||||
"Need the main budget constraint because it limits the available options.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
|
||||
const result = formulateQuestion({
|
||||
node: unknown,
|
||||
graph: makeGraphFor(unknown),
|
||||
});
|
||||
|
||||
expect(result.strategy).toBe("evidence_gathering");
|
||||
expect(result.question).toContain("What evidence");
|
||||
});
|
||||
|
||||
it("the same unknown can produce different questions when paired with different strategies", () => {
|
||||
const thresholdUnknown = makeNode({
|
||||
id: "n-threshold-unknown",
|
||||
label: "Value threshold",
|
||||
description: "Need to resolve the value threshold.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
});
|
||||
const definitionUnknown = makeNode({
|
||||
id: "n-definition-unknown",
|
||||
label: "Value term",
|
||||
description: "Need to resolve what value term refers to in this context.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
});
|
||||
|
||||
const decisionGraph = makeGraphFor(thresholdUnknown, {
|
||||
centralStatement: "We are deciding whether to launch this product.",
|
||||
nodes: [
|
||||
makeNode({
|
||||
id: "n-decision",
|
||||
label: "Launch decision",
|
||||
description: "Decision depends on the value threshold.",
|
||||
kind: "state",
|
||||
status: "known",
|
||||
confidence: "medium",
|
||||
childIds: [thresholdUnknown.id],
|
||||
value: "Deciding whether to launch the product",
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
const definitionGraph = makeGraphFor(definitionUnknown, {
|
||||
centralStatement:
|
||||
"The team uses the term value threshold inconsistently.",
|
||||
nodes: [
|
||||
makeNode({
|
||||
id: "n-definition",
|
||||
label: "Definition disagreement about value threshold",
|
||||
description:
|
||||
"Need a definition of value threshold because the term is used inconsistently before comparing options.",
|
||||
kind: "state",
|
||||
status: "known",
|
||||
confidence: "medium",
|
||||
childIds: [definitionUnknown.id],
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
const decisionResult = formulateQuestion({
|
||||
node: thresholdUnknown,
|
||||
graph: decisionGraph,
|
||||
});
|
||||
const definitionResult = formulateQuestion({
|
||||
node: definitionUnknown,
|
||||
graph: definitionGraph,
|
||||
});
|
||||
|
||||
expect(decisionResult.strategy).toBe("decision_threshold");
|
||||
expect(definitionResult.strategy).toBe("definition");
|
||||
expect(decisionResult.question).not.toBe(definitionResult.question);
|
||||
});
|
||||
|
||||
it("strategy selection is deterministic and explainable", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-threshold",
|
||||
label: "Success threshold",
|
||||
description:
|
||||
"Need the success threshold because the decision depends on it.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
});
|
||||
const graph = makeGraphFor(unknown, {
|
||||
centralStatement: "We need to decide whether to continue investing.",
|
||||
});
|
||||
|
||||
const first = selectInvestigationStrategy({ node: unknown, graph });
|
||||
const second = selectInvestigationStrategy({ node: unknown, graph });
|
||||
|
||||
expect(first).toEqual(second);
|
||||
expect(first.key).toBe("decision_threshold");
|
||||
expect(first.reason).toContain("threshold");
|
||||
});
|
||||
|
||||
it("question is singular and answerable", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-evidence",
|
||||
label: "Evidence of demand",
|
||||
description:
|
||||
"Need evidence of demand because the decision depends on it.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
|
||||
const result = formulateQuestion({
|
||||
node: unknown,
|
||||
graph: makeGraphFor(unknown),
|
||||
});
|
||||
|
||||
expect(result.question.match(/\?/g) || []).toHaveLength(1);
|
||||
expect(result.question.toLowerCase()).not.toContain(" and ");
|
||||
});
|
||||
|
||||
it("awkward uncertainty phrasing is rejected via fallback", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-weird",
|
||||
label: "Uncertainty regarding service reliability",
|
||||
description: "Unknown service reliability.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
|
||||
const result = formulateQuestion({
|
||||
node: unknown,
|
||||
graph: makeGraphFor(unknown),
|
||||
});
|
||||
|
||||
expect(result.question).not.toContain("How should uncertainty regarding");
|
||||
expect(result.question).not.toContain(
|
||||
"What would resolve uncertainty regarding",
|
||||
);
|
||||
});
|
||||
|
||||
it("ambiguous contradiction produces a broad distinguishing question without accounting jargon", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-cause-a",
|
||||
label: "Cash outflow cause",
|
||||
description: "Unclear explanation for the contradiction.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
});
|
||||
const contradiction = makeNode({
|
||||
id: "n-contradiction",
|
||||
label: "Divergent movement between revenue and cash",
|
||||
description: "Two signals moved in opposite directions.",
|
||||
kind: "relationship",
|
||||
status: "supported",
|
||||
confidence: "medium",
|
||||
});
|
||||
const revenueObservation = makeNode({
|
||||
id: "n-revenue-observation",
|
||||
label: "Revenue increased by 18%.",
|
||||
description: "Revenue increased by 18%.",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
});
|
||||
const cashObservation = makeNode({
|
||||
id: "n-cash-observation",
|
||||
label: "Cash in the bank decreased over the same period.",
|
||||
description: "Cash in the bank decreased over the same period.",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
});
|
||||
const graph = makeGraphFor(unknown, {
|
||||
centralStatement:
|
||||
"Revenue increased by 18%, but cash in the bank fell over the same period.",
|
||||
nodes: [contradiction, revenueObservation, cashObservation],
|
||||
});
|
||||
|
||||
const result = formulateTieResolutionQuestion({ graph });
|
||||
|
||||
expect(result.question).toBe(
|
||||
"Were these figures measured on the same basis and at the same scale?",
|
||||
);
|
||||
expect(result.comparabilityStatus).toBe("uncertain");
|
||||
expect(result.question.toLowerCase()).not.toMatch(
|
||||
/accounts receivable|capex|debt repayments|working capital/,
|
||||
);
|
||||
});
|
||||
|
||||
it("definition is selected only for genuine definition unknowns", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-definition-only",
|
||||
label: "Definition of success criteria",
|
||||
description: "The term is used inconsistently and needs a definition.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
|
||||
const result = formulateQuestion({
|
||||
node: unknown,
|
||||
graph: makeGraphFor(unknown),
|
||||
});
|
||||
expect(result.strategy).toBe("definition");
|
||||
});
|
||||
|
||||
it("an unknown about possible causes does not become a definition question", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-causes",
|
||||
label: "Possible causes of the divergence",
|
||||
description: "Several causes may explain the divergence.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
|
||||
const result = formulateQuestion({
|
||||
node: unknown,
|
||||
graph: makeGraphFor(unknown),
|
||||
});
|
||||
expect(result.reasoningPattern).toBe("diagnosis");
|
||||
expect(result.strategy).toBeNull();
|
||||
expect(result.question).toBe(
|
||||
"What would clarify possible causes of the divergence in this situation?",
|
||||
);
|
||||
});
|
||||
|
||||
it("malformed punctuation is rejected", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-punct",
|
||||
label: "Magnitude and nature of cash outflows (operating expenses).",
|
||||
description:
|
||||
"Magnitude and nature of cash outflows (operating expenses).",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
|
||||
const result = formulateQuestion({
|
||||
node: unknown,
|
||||
graph: makeGraphFor(unknown),
|
||||
});
|
||||
expect(result.question).not.toContain("). is true?");
|
||||
expect(result.question).toBe(
|
||||
"What would clarify magnitude and nature of cash outflows (operating expenses) in this situation?",
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,155 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { applyValidatedProposal } from "@/lib/graph/apply-proposal.js";
|
||||
import { formulateQuestion } from "@/lib/graph/question-formulator.js";
|
||||
import { selectActiveUnknownCandidate } from "@/lib/graph/utils.js";
|
||||
import { questionPriorityGeneralisationFixtures } from "@/tests/fixtures/question-priority-generalisation.js";
|
||||
|
||||
function clone(value) {
|
||||
return JSON.parse(JSON.stringify(value));
|
||||
}
|
||||
|
||||
function buildResolutionProposal(graph) {
|
||||
const activeNode = graph.nodes.find(
|
||||
(node) => node.id === graph.activeUnknownNodeId,
|
||||
);
|
||||
const placeholderCandidate = graph.nodes.find(
|
||||
(node) => node.kind === "unknown" && node.id !== activeNode.id,
|
||||
);
|
||||
|
||||
return {
|
||||
addedNodes: [],
|
||||
updatedNodes: [
|
||||
{
|
||||
nodeId: activeNode.id,
|
||||
previousStatus: activeNode.status,
|
||||
newStatus: "resolved",
|
||||
previousValue: activeNode.value ?? null,
|
||||
newValue: activeNode.value ?? "Resolved context answer",
|
||||
reason:
|
||||
"The resolved context unknown is treated as answered for fixture progression.",
|
||||
},
|
||||
],
|
||||
addedEdges: [],
|
||||
removedEdgeIds: [],
|
||||
resolvedUnknownNodeIds: [activeNode.id],
|
||||
affectedNodeIds: [],
|
||||
selectedQuestion: {
|
||||
nodeId: placeholderCandidate?.id,
|
||||
question: "Placeholder candidate question?",
|
||||
reason: "Candidate only; deterministic selector should override it.",
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function assertQuestionStructure(question) {
|
||||
expect(question.match(/\?/g) || []).toHaveLength(1);
|
||||
expect(question).not.toMatch(/\?\s*(and|or)\b/i);
|
||||
expect(question).not.toMatch(/^What is\s+/i);
|
||||
expect(question).toMatch(/^(What|Who|When)\b/);
|
||||
}
|
||||
|
||||
describe("question priority generalisation", () => {
|
||||
for (const fixture of questionPriorityGeneralisationFixtures) {
|
||||
it(`${fixture.scenario} selects a foundational unknown and singular answerable strategy`, () => {
|
||||
const originalGraph = clone(fixture.graph);
|
||||
const deterministicSelection = selectActiveUnknownCandidate(
|
||||
fixture.graph,
|
||||
[fixture.graph.activeUnknownNodeId],
|
||||
);
|
||||
|
||||
const result = applyValidatedProposal({
|
||||
situationGraph: fixture.graph,
|
||||
proposal: buildResolutionProposal(fixture.graph),
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(fixture.graph).toEqual(originalGraph);
|
||||
expect(result.graphUpdate.selectedQuestion?.question).toBe(
|
||||
"Placeholder candidate question?",
|
||||
);
|
||||
|
||||
expect(deterministicSelection.nodeId).toBe(
|
||||
result.selectedQuestion.nodeId,
|
||||
);
|
||||
expect(fixture.acceptableFoundationalUnknownNodeIds).toContain(
|
||||
result.selectedQuestion.nodeId,
|
||||
);
|
||||
expect(result.selectedQuestion.nodeId).not.toBe(
|
||||
fixture.graph.nodes[fixture.graph.nodes.length - 1].id,
|
||||
);
|
||||
|
||||
expect(fixture.acceptableQuestionStrategies).toContain(
|
||||
result.selectedQuestion.strategy,
|
||||
);
|
||||
assertQuestionStructure(result.selectedQuestion.question);
|
||||
|
||||
const lowerQuestion = result.selectedQuestion.question.toLowerCase();
|
||||
for (const topic of fixture.prohibitedFirstTopics) {
|
||||
expect(lowerQuestion).not.toContain(topic.toLowerCase());
|
||||
}
|
||||
|
||||
const selectedNode = result.updatedSituationGraph.nodes.find(
|
||||
(node) => node.id === result.selectedQuestion.nodeId,
|
||||
);
|
||||
const reformulated = formulateQuestion({
|
||||
node: selectedNode,
|
||||
graph: result.updatedSituationGraph,
|
||||
context: {
|
||||
resolvedValues: ["Resolved context answer"],
|
||||
},
|
||||
});
|
||||
|
||||
expect(reformulated.question).toBe(result.selectedQuestion.question);
|
||||
expect(clone(result.updatedSituationGraph)).toEqual(
|
||||
result.updatedSituationGraph,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
it("reports all five selected unknowns and strategies", () => {
|
||||
const summary = questionPriorityGeneralisationFixtures.map((fixture) => {
|
||||
const result = applyValidatedProposal({
|
||||
situationGraph: fixture.graph,
|
||||
proposal: buildResolutionProposal(fixture.graph),
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
|
||||
return {
|
||||
scenario: fixture.scenario,
|
||||
nodeId: result.selectedQuestion.nodeId,
|
||||
strategy: result.selectedQuestion.strategy,
|
||||
};
|
||||
});
|
||||
|
||||
expect(summary).toMatchInlineSnapshot(`
|
||||
[
|
||||
{
|
||||
"nodeId": "hire-success-criteria",
|
||||
"scenario": "Should we hire another engineer?",
|
||||
"strategy": "decision_threshold",
|
||||
},
|
||||
{
|
||||
"nodeId": "van-reliability-threshold",
|
||||
"scenario": "Should we replace the delivery vans?",
|
||||
"strategy": "decision_threshold",
|
||||
},
|
||||
{
|
||||
"nodeId": "country-value-threshold",
|
||||
"scenario": "Should we launch in another country?",
|
||||
"strategy": "decision_threshold",
|
||||
},
|
||||
{
|
||||
"nodeId": "project-benefit-threshold",
|
||||
"scenario": "Should we continue a project that is over budget?",
|
||||
"strategy": "decision_threshold",
|
||||
},
|
||||
{
|
||||
"nodeId": "support-value-threshold",
|
||||
"scenario": "Should we introduce a paid support tier?",
|
||||
"strategy": "baseline_reconstruction",
|
||||
},
|
||||
]
|
||||
`);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,208 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
assessQuestionComplexity,
|
||||
assessUnknownAtomicity,
|
||||
formulateQuestion,
|
||||
} from "@/lib/graph/question-formulator.js";
|
||||
import { applyValidatedProposal } from "@/lib/graph/apply-proposal.js";
|
||||
import { makeGraph, makeNode } from "@/lib/graph/schema.js";
|
||||
|
||||
const SCENARIO =
|
||||
"I have developed a new reasoning method that aims to help people determine whether they have enough justified confidence to make a decision. I believe it could become a commercial product, but I do not yet know whether it solves a genuine problem, whether people would value it enough to pay for it, or whether it is fundamentally different from existing AI tools. Before investing significant time and money into building it further, I want to determine whether continuing development is commercially justified.";
|
||||
|
||||
function makeCommercialValidationGraph() {
|
||||
const parent = makeNode({
|
||||
id: "n-commercial-parent",
|
||||
label:
|
||||
"Commercial justification for whether continuing development is commercially justified",
|
||||
description:
|
||||
"Need to know whether this solves a genuine problem, whether people would value it enough to pay for it, and whether it is commercially justified before continuing development.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
|
||||
return makeGraph({
|
||||
centralStatement: SCENARIO,
|
||||
nodes: [parent],
|
||||
edges: [],
|
||||
activeUnknownNodeId: parent.id,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: "Commercial validation question simplicity fixture",
|
||||
});
|
||||
}
|
||||
|
||||
function makeMeaningfulNoOpProposal() {
|
||||
return {
|
||||
addedNodes: [
|
||||
makeNode({
|
||||
id: "n-anchor",
|
||||
label: "Update anchor",
|
||||
description:
|
||||
"Anchor state introduced by the answer because the update must contain a meaningful change.",
|
||||
kind: "state",
|
||||
status: "known",
|
||||
confidence: "low",
|
||||
}),
|
||||
],
|
||||
updatedNodes: [],
|
||||
addedEdges: [],
|
||||
removedEdgeIds: [],
|
||||
resolvedUnknownNodeIds: [],
|
||||
affectedNodeIds: [],
|
||||
selectedQuestion: null,
|
||||
};
|
||||
}
|
||||
|
||||
describe("question simplicity", () => {
|
||||
it("rejects the original long compound financial-cost plus budget question", () => {
|
||||
const graph = makeCommercialValidationGraph();
|
||||
const unknown = graph.nodes.find(
|
||||
(node) => node.id === "n-commercial-parent",
|
||||
);
|
||||
const question =
|
||||
"Have you measured the current financial or operational cost to users who lack justified confidence, and what baseline budget do they currently allocate for comparable decision-support methods?";
|
||||
|
||||
const result = assessQuestionComplexity({
|
||||
question,
|
||||
selectedUnknown: unknown,
|
||||
graph,
|
||||
});
|
||||
|
||||
expect(result.acceptable).toBe(false);
|
||||
expect(result.primaryConceptCount).toBeGreaterThan(1);
|
||||
expect(result.cognitiveLoad).toBe("high");
|
||||
expect(result.reasons).toContain("multiple_requested_answers");
|
||||
expect(result.reasons).toContain("cost_and_budget_combined");
|
||||
});
|
||||
|
||||
it("accepts one simple concept question", () => {
|
||||
const graph = makeCommercialValidationGraph();
|
||||
const unknown = makeNode({
|
||||
id: "n-who",
|
||||
label: "Who experiences this problem",
|
||||
description:
|
||||
"Need to know who experiences this problem, because that must be clear before deciding whether it is commercially justified.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
parentId: "n-commercial-parent",
|
||||
});
|
||||
|
||||
const result = formulateQuestion({ node: unknown, graph });
|
||||
|
||||
expect(result.question).toBe("Who experiences this problem?");
|
||||
expect(result.questionComplexity.acceptable).toBe(true);
|
||||
expect(result.questionComplexity.primaryConceptCount).toBe(1);
|
||||
expect(result.question.match(/\?/g) || []).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("classifies broad commercial-validation unknowns as composite", () => {
|
||||
const graph = makeCommercialValidationGraph();
|
||||
const unknown = graph.nodes.find(
|
||||
(node) => node.id === "n-commercial-parent",
|
||||
);
|
||||
|
||||
const result = assessUnknownAtomicity({ node: unknown, graph });
|
||||
|
||||
expect(result.atomicity).toBe("composite");
|
||||
expect(result.decompositionKind).toBe("commercial_validation");
|
||||
});
|
||||
|
||||
it("decomposes a broad commercial-validation unknown instead of merely rewording it", () => {
|
||||
const graph = makeCommercialValidationGraph();
|
||||
|
||||
const result = applyValidatedProposal({
|
||||
situationGraph: graph,
|
||||
proposal: makeMeaningfulNoOpProposal(),
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.decompositionPerformed).toBe(true);
|
||||
expect(result.selectedUnknownBefore).toBe("n-commercial-parent");
|
||||
expect(result.selectedUnknownAfter).not.toBe("n-commercial-parent");
|
||||
expect(result.childNodeIds.length).toBeGreaterThanOrEqual(2);
|
||||
expect(result.selectedQuestion.nodeId).toBe(result.selectedUnknownAfter);
|
||||
expect(result.selectedQuestion.question).toBe(
|
||||
"Who experiences this problem?",
|
||||
);
|
||||
expect(result.selectedQuestion.reasoningPattern).toBe("decision");
|
||||
expect(result.selectedQuestion.questionFamily).toBe("decision_foundation");
|
||||
expect(result.selectedQuestion.selectedQuestionTemplate).toBe(
|
||||
"decision_foundation_direct_child",
|
||||
);
|
||||
expect(result.selectedQuestion.question.match(/\?/g) || []).toHaveLength(1);
|
||||
expect(result.questionComplexityAccepted).toBe(true);
|
||||
expect(result.primaryConceptCount).toBe(1);
|
||||
});
|
||||
|
||||
it("selects a foundational child rather than price or budget", () => {
|
||||
const graph = makeCommercialValidationGraph();
|
||||
|
||||
const result = applyValidatedProposal({
|
||||
situationGraph: graph,
|
||||
proposal: makeMeaningfulNoOpProposal(),
|
||||
});
|
||||
|
||||
const selectedNode = result.updatedSituationGraph.nodes.find(
|
||||
(node) => node.id === result.selectedUnknownAfter,
|
||||
);
|
||||
|
||||
expect(selectedNode.label).toBe("Who experiences this problem");
|
||||
expect(selectedNode.label.toLowerCase()).not.toMatch(/pay|price|budget/);
|
||||
expect(result.selectedQuestion.question.toLowerCase()).not.toMatch(
|
||||
/pay|price|budget/,
|
||||
);
|
||||
});
|
||||
|
||||
it("plain-language replacements simplify formal phrasing when safe", () => {
|
||||
const graph = makeCommercialValidationGraph();
|
||||
const unknown = makeNode({
|
||||
id: "n-actor-formal",
|
||||
label: "Relevant customer or user",
|
||||
description:
|
||||
"Need to identify the relevant customer, user, or value recipient because the decision depends on it.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
|
||||
const result = formulateQuestion({ node: unknown, graph });
|
||||
|
||||
const complexity = assessQuestionComplexity({
|
||||
question:
|
||||
"What evidence would clarify the relevant customer, user, or value recipient?",
|
||||
selectedUnknown: unknown,
|
||||
graph,
|
||||
});
|
||||
|
||||
expect(complexity.acceptable).toBe(false);
|
||||
expect(complexity.reasons).toContain("list_like_question");
|
||||
});
|
||||
|
||||
it("does not remove scenario-relevant jargon blindly", () => {
|
||||
const graph = makeGraph({
|
||||
centralStatement:
|
||||
"The team is deciding whether to continue a decision-support product.",
|
||||
nodes: [
|
||||
makeNode({
|
||||
id: "n-jargon",
|
||||
label: "Decision support methods",
|
||||
description:
|
||||
"Need evidence about decision support methods because the comparison depends on it.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
}),
|
||||
],
|
||||
edges: [],
|
||||
activeUnknownNodeId: "n-jargon",
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: "Jargon retention fixture",
|
||||
});
|
||||
|
||||
const result = formulateQuestion({ node: graph.nodes[0], graph });
|
||||
|
||||
expect(result.question.toLowerCase()).toContain("decision support methods");
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,201 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
formulateQuestion,
|
||||
formulateTieResolutionQuestion,
|
||||
selectReasoningPattern,
|
||||
} from "@/lib/graph/question-formulator.js";
|
||||
import { makeGraph, makeNode } from "@/lib/graph/schema.js";
|
||||
|
||||
function makeGraphFor(node, extra = {}) {
|
||||
return makeGraph({
|
||||
centralStatement: extra.centralStatement || "Decision context",
|
||||
nodes: [node, ...(extra.nodes || [])],
|
||||
edges: extra.edges || [],
|
||||
activeUnknownNodeId: node.id,
|
||||
resolvedNodeIds: extra.resolvedNodeIds || [],
|
||||
currentSummary: "Test summary",
|
||||
reasoningState: extra.reasoningState,
|
||||
});
|
||||
}
|
||||
|
||||
describe("reasoning pattern selection", () => {
|
||||
it("commercial-method scenario selects the decision pattern and rejects explanation family", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-commercial-method",
|
||||
label:
|
||||
"Whether the method addresses a genuine, high-priority problem for a specific audience",
|
||||
description:
|
||||
"Need to know whether this solves a real problem for a clear audience before continuing development.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
});
|
||||
const graph = makeGraphFor(unknown, {
|
||||
centralStatement:
|
||||
"Before investing significant time and money, we need to know whether continuing development is commercially justified.",
|
||||
});
|
||||
|
||||
const result = formulateQuestion({ node: unknown, graph });
|
||||
|
||||
expect(result.reasoningPattern).toBe("decision");
|
||||
expect(result.questionFamily).not.toBe("explanation");
|
||||
expect(result.allowedQuestionFamilies).toContain("decision_foundation");
|
||||
expect(result.rejectedQuestionFamilies).toContain("explanation");
|
||||
});
|
||||
|
||||
it("revenue and cash relationship scenario allows the explanation family", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-revenue-cash-explanation",
|
||||
label:
|
||||
"Explanation for why revenue increased by 18%, but cash in the bank fell over the same period",
|
||||
description:
|
||||
"Need to understand what change or event could explain why these observations differ.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
const graph = makeGraphFor(unknown, {
|
||||
centralStatement:
|
||||
"Revenue increased by 18%, but cash in the bank fell over the same period.",
|
||||
nodes: [
|
||||
makeNode({
|
||||
id: "n-revenue-observation",
|
||||
label: "Revenue increased by 18%.",
|
||||
description: "Revenue increased by 18%.",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
}),
|
||||
makeNode({
|
||||
id: "n-cash-observation",
|
||||
label: "Cash in the bank fell over the same period.",
|
||||
description: "Cash in the bank fell over the same period.",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
const result = formulateQuestion({ node: unknown, graph });
|
||||
|
||||
expect(result.reasoningPattern).toBe("explanation");
|
||||
expect(result.allowedQuestionFamilies).toContain("explanation");
|
||||
});
|
||||
|
||||
it("duplicate observations reject explanation family during tie resolution", () => {
|
||||
const graph = makeGraph({
|
||||
centralStatement: "The same figure was repeated twice.",
|
||||
nodes: [
|
||||
makeNode({
|
||||
id: "n-obs-1",
|
||||
label: "Revenue increased by 10%.",
|
||||
description: "Revenue increased by 10%.",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
}),
|
||||
makeNode({
|
||||
id: "n-obs-2",
|
||||
label: "Revenue increased by 10%.",
|
||||
description: "Revenue increased by 10%.",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
}),
|
||||
],
|
||||
edges: [],
|
||||
activeUnknownNodeId: null,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: "Duplicate observation fixture",
|
||||
});
|
||||
|
||||
const result = formulateTieResolutionQuestion({ graph });
|
||||
|
||||
expect(result.questionFamily).not.toBe("explanation");
|
||||
expect(result.rejectedQuestionFamilies).toContain("explanation");
|
||||
});
|
||||
|
||||
it("definition scenario selects the definition pattern", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-definition-pattern",
|
||||
label: "Definition of justified confidence",
|
||||
description: "The term needs clearer boundaries.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
|
||||
const result = selectReasoningPattern({
|
||||
node: unknown,
|
||||
graph: makeGraphFor(unknown),
|
||||
});
|
||||
|
||||
expect(result.pattern).toBe("definition");
|
||||
});
|
||||
|
||||
it("comparison scenario selects the comparison pattern", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-comparison-pattern-2",
|
||||
label: "How the two observations were measured",
|
||||
description:
|
||||
"Need evidence about the measure used for each observation before comparing them.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
const graph = makeGraphFor(unknown, {
|
||||
centralStatement: "Traffic increased, but sales stayed flat.",
|
||||
nodes: [
|
||||
makeNode({
|
||||
id: "n-traffic-2",
|
||||
label: "Traffic increased.",
|
||||
description: "Traffic increased.",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
}),
|
||||
makeNode({
|
||||
id: "n-sales-2",
|
||||
label: "Sales stayed flat.",
|
||||
description: "Sales stayed flat.",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
}),
|
||||
],
|
||||
reasoningState: {
|
||||
comparabilityStatus: "uncertain",
|
||||
relationshipStatus: "insufficient_information",
|
||||
},
|
||||
});
|
||||
|
||||
const result = selectReasoningPattern({ node: unknown, graph });
|
||||
|
||||
expect(result.pattern).toBe("comparison");
|
||||
});
|
||||
|
||||
it("question family stays compatible with the reasoning pattern", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-decision-family-compatibility",
|
||||
label: "Who experiences this problem",
|
||||
description:
|
||||
"Need to know who experiences this problem before continuing development.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
const graph = makeGraphFor(unknown, {
|
||||
centralStatement:
|
||||
"We need to know whether continuing development is commercially justified.",
|
||||
});
|
||||
|
||||
const result = formulateQuestion({ node: unknown, graph });
|
||||
|
||||
expect(result.reasoningPattern).toBe("decision");
|
||||
expect(result.allowedQuestionFamilies).toContain(result.questionFamily);
|
||||
expect(result.rejectedQuestionFamilies).not.toContain(
|
||||
result.questionFamily,
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,282 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { applyValidatedProposal } from "@/lib/graph/apply-proposal.js";
|
||||
import {
|
||||
classifyObservationRelationship,
|
||||
formulateQuestion,
|
||||
selectReasoningPattern,
|
||||
} from "@/lib/graph/question-formulator.js";
|
||||
import { makeGraph, makeNode } from "@/lib/graph/schema.js";
|
||||
|
||||
const COMMERCIAL_SCENARIO =
|
||||
"I have developed a new reasoning method that aims to help people determine whether they have enough justified confidence to make a decision. I believe it could become a commercial product, but I do not yet know whether it solves a genuine problem, whether people would value it enough to pay for it, or whether it is fundamentally different from existing AI tools. Before investing significant time and money into building it further, I want to determine whether continuing development is commercially justified.";
|
||||
|
||||
function makeCommercialGraph() {
|
||||
const parent = makeNode({
|
||||
id: "n-commercial-parent",
|
||||
label:
|
||||
"Commercial justification for whether continuing development is commercially justified",
|
||||
description:
|
||||
"Need to know whether this solves a genuine problem, whether people would value it enough to pay for it, and whether it is commercially justified before continuing development.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
|
||||
return makeGraph({
|
||||
centralStatement: COMMERCIAL_SCENARIO,
|
||||
nodes: [parent],
|
||||
edges: [],
|
||||
activeUnknownNodeId: parent.id,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: "Commercial pattern fixture",
|
||||
});
|
||||
}
|
||||
|
||||
function makeSeedProposal() {
|
||||
return {
|
||||
addedNodes: [
|
||||
makeNode({
|
||||
id: "n-anchor",
|
||||
label: "Update anchor",
|
||||
description:
|
||||
"Anchor state introduced by the answer because the update must contain a meaningful change.",
|
||||
kind: "state",
|
||||
status: "known",
|
||||
confidence: "low",
|
||||
}),
|
||||
],
|
||||
updatedNodes: [],
|
||||
addedEdges: [],
|
||||
removedEdgeIds: [],
|
||||
resolvedUnknownNodeIds: [],
|
||||
affectedNodeIds: [],
|
||||
selectedQuestion: null,
|
||||
};
|
||||
}
|
||||
|
||||
describe("reasoning-pattern validation", () => {
|
||||
it("keeps the commercial-method scenario in decision mode without comparability-style unknowns", () => {
|
||||
const seeded = applyValidatedProposal({
|
||||
situationGraph: makeCommercialGraph(),
|
||||
proposal: makeSeedProposal(),
|
||||
});
|
||||
|
||||
expect(seeded.success).toBe(true);
|
||||
expect(seeded.selectedQuestion?.reasoningPattern).toBe("decision");
|
||||
expect(
|
||||
seeded.updatedSituationGraph.nodes.some((node) =>
|
||||
/two observations|measured|different timing/i.test(node.label),
|
||||
),
|
||||
).toBe(false);
|
||||
|
||||
const followUp = applyValidatedProposal({
|
||||
situationGraph: seeded.updatedSituationGraph,
|
||||
proposal: {
|
||||
addedNodes: [],
|
||||
updatedNodes: [
|
||||
{
|
||||
nodeId: seeded.selectedQuestion.nodeId,
|
||||
previousStatus: "unknown",
|
||||
newStatus: "resolved",
|
||||
previousValue: null,
|
||||
newValue:
|
||||
"I experience it myself when I am trying to decide whether a project, idea or investment is justified, but I do not yet know how common that problem is for other people.",
|
||||
reason: "Answered by the user.",
|
||||
},
|
||||
],
|
||||
addedEdges: [],
|
||||
removedEdgeIds: [],
|
||||
resolvedUnknownNodeIds: [seeded.selectedQuestion.nodeId],
|
||||
affectedNodeIds: [],
|
||||
selectedQuestion: null,
|
||||
},
|
||||
previousQuestion: seeded.selectedQuestion.question,
|
||||
answer:
|
||||
"I experience it myself when I am trying to decide whether a project, idea or investment is justified, but I do not yet know how common that problem is for other people.",
|
||||
});
|
||||
|
||||
expect(followUp.success).toBe(true);
|
||||
expect(followUp.selectedQuestion?.reasoningPattern).toBe("decision");
|
||||
expect(followUp.reasoningPatternValidation).toMatchObject({
|
||||
activePattern: "decision",
|
||||
valid: true,
|
||||
});
|
||||
expect(followUp.graphReasoningIntegrity).toBe("valid");
|
||||
expect(followUp.incompatibleNodeIds).toEqual([]);
|
||||
expect(followUp.selectedQuestion?.question.toLowerCase()).not.toMatch(
|
||||
/two observations|measured|same basis|same scale|different timing/,
|
||||
);
|
||||
});
|
||||
|
||||
it("allows comparability-style reasoning in an explanation scenario", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-explanation",
|
||||
label:
|
||||
"Explanation for why revenue increased by 18%, but cash in the bank fell over the same period",
|
||||
description:
|
||||
"Need to understand what change or event could explain why these observations differ.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
const graph = makeGraph({
|
||||
centralStatement:
|
||||
"Revenue increased by 18%, but cash in the bank fell over the same period.",
|
||||
nodes: [
|
||||
unknown,
|
||||
makeNode({
|
||||
id: "n-revenue",
|
||||
label: "Revenue increased by 18%.",
|
||||
description: "Revenue increased by 18%.",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
}),
|
||||
makeNode({
|
||||
id: "n-cash",
|
||||
label: "Cash in the bank fell over the same period.",
|
||||
description: "Cash in the bank fell over the same period.",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
}),
|
||||
],
|
||||
edges: [],
|
||||
activeUnknownNodeId: unknown.id,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: "Explanation fixture",
|
||||
});
|
||||
|
||||
const pattern = selectReasoningPattern({ node: unknown, graph });
|
||||
const question = formulateQuestion({ node: unknown, graph });
|
||||
const relationship = classifyObservationRelationship(graph);
|
||||
|
||||
expect(pattern.pattern).toBe("explanation");
|
||||
expect(question.reasoningPattern).toBe("explanation");
|
||||
expect(relationship.questionRequired).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects explanation-family tie resolution for duplicate observations", () => {
|
||||
const graph = makeGraph({
|
||||
centralStatement: "Sales doubled. Sales doubled.",
|
||||
nodes: [
|
||||
makeNode({
|
||||
id: "n-sales-1",
|
||||
label: "Sales doubled.",
|
||||
description: "Sales doubled.",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
}),
|
||||
makeNode({
|
||||
id: "n-sales-2",
|
||||
label: "Sales doubled.",
|
||||
description: "Sales doubled.",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
}),
|
||||
],
|
||||
edges: [],
|
||||
activeUnknownNodeId: null,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: "Duplicate observation fixture",
|
||||
});
|
||||
|
||||
const relationship = classifyObservationRelationship(graph);
|
||||
|
||||
expect(relationship.relationshipStatus).toBe("duplicate");
|
||||
expect(relationship.questionRequired).toBe(false);
|
||||
});
|
||||
|
||||
it("keeps definition scenarios inside compatible node families", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-definition",
|
||||
label: "Definition of justified confidence",
|
||||
description: "The term needs clearer boundaries.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
const graph = makeGraph({
|
||||
centralStatement: "The team uses justified confidence inconsistently.",
|
||||
nodes: [unknown],
|
||||
edges: [],
|
||||
activeUnknownNodeId: unknown.id,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: "Definition fixture",
|
||||
});
|
||||
|
||||
const pattern = selectReasoningPattern({ node: unknown, graph });
|
||||
const result = formulateQuestion({ node: unknown, graph });
|
||||
|
||||
expect(pattern.pattern).toBe("definition");
|
||||
expect(result.reasoningPattern).toBe("definition");
|
||||
expect(result.allowedQuestionFamilies).toEqual(["definition"]);
|
||||
});
|
||||
|
||||
it("replaces an incompatible decision-mode active unknown with a pattern-compatible candidate", () => {
|
||||
const graph = makeGraph({
|
||||
centralStatement:
|
||||
"Before investing more, we need to know whether continuing development is commercially justified.",
|
||||
nodes: [
|
||||
makeNode({
|
||||
id: "n-commercial-parent",
|
||||
label:
|
||||
"Commercial justification for whether continuing development is commercially justified",
|
||||
description:
|
||||
"Need to know whether this solves a genuine problem before continuing development.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
}),
|
||||
makeNode({
|
||||
id: "n-incompatible-child",
|
||||
label: "How the two observations were measured",
|
||||
description:
|
||||
"Need evidence about the measure used for each observation before comparing them.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
parentId: "n-commercial-parent",
|
||||
}),
|
||||
],
|
||||
edges: [],
|
||||
activeUnknownNodeId: "n-incompatible-child",
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: "Incompatible active unknown fixture",
|
||||
});
|
||||
|
||||
const result = applyValidatedProposal({
|
||||
situationGraph: graph,
|
||||
proposal: makeSeedProposal(),
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.reasoningPatternValidation).toMatchObject({
|
||||
activePattern: "decision",
|
||||
valid: true,
|
||||
});
|
||||
expect(result.graphReasoningIntegrity).toBe("valid");
|
||||
expect(result.incompatibleNodeIds).toContain("n-incompatible-child");
|
||||
expect(result.compatibilityFailures).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
nodeId: "n-incompatible-child",
|
||||
activePattern: "decision",
|
||||
nodePattern: "comparison",
|
||||
}),
|
||||
]),
|
||||
);
|
||||
expect(result.replacementActions).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
rejectedNodeId: "n-incompatible-child",
|
||||
replacementNodeId: result.selectedQuestion?.nodeId,
|
||||
}),
|
||||
]),
|
||||
);
|
||||
expect(result.selectedQuestion?.nodeId).not.toBe("n-incompatible-child");
|
||||
expect(result.selectedQuestion?.reasoningPattern).toBe("decision");
|
||||
});
|
||||
});
|
||||
+85
-15
@@ -161,21 +161,56 @@ describe("graphUpdateSchema", () => {
|
||||
it("validates a complete update", () => {
|
||||
const node = makeNode({ id: "n2", label: "New Node" });
|
||||
const edge = makeEdge({ fromNodeId: "n1", toNodeId: "n2" });
|
||||
|
||||
|
||||
const result = graphUpdateSchema.safeParse({
|
||||
addedNodes: [node],
|
||||
updatedNodes: [{ nodeId: "n1", newStatus: "resolved", previousStatus: "unknown", reason: "Question answered" }],
|
||||
updatedNodes: [
|
||||
{
|
||||
nodeId: "n1",
|
||||
newStatus: "resolved",
|
||||
previousStatus: "unknown",
|
||||
reason: "Question answered",
|
||||
},
|
||||
],
|
||||
addedEdges: [edge],
|
||||
removedEdgeIds: ["e-old"],
|
||||
resolvedUnknownNodeIds: ["n2"],
|
||||
affectedNodeIds: ["n3"],
|
||||
selectedQuestion: {
|
||||
nodeId: "n2",
|
||||
question: "What does this new node mean?",
|
||||
reason: "A follow-up unknown remains.",
|
||||
},
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("allows null selectedQuestion", () => {
|
||||
const result = graphUpdateSchema.safeParse({
|
||||
selectedQuestion: null,
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects update with invalid node kind in addedNodes", () => {
|
||||
const invalid = graphUpdateSchema.safeParse({
|
||||
addedNodes: [{ id: "x", label: "Test", kind: "invalid_kind", description: "test", status: "unknown", confidence: "medium", value: null, unit: null, evidenceIds: [], dependsOn: [], affects: [], parentId: null, childIds: [] }],
|
||||
addedNodes: [
|
||||
{
|
||||
id: "x",
|
||||
label: "Test",
|
||||
kind: "invalid_kind",
|
||||
description: "test",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
value: null,
|
||||
unit: null,
|
||||
evidenceIds: [],
|
||||
dependsOn: [],
|
||||
affects: [],
|
||||
parentId: null,
|
||||
childIds: [],
|
||||
},
|
||||
],
|
||||
});
|
||||
expect(invalid.success).toBe(false);
|
||||
});
|
||||
@@ -184,7 +219,9 @@ describe("graphUpdateSchema", () => {
|
||||
describe("API request schemas", () => {
|
||||
describe("startCaseRequestSchema", () => {
|
||||
it("validates scenario field", () => {
|
||||
const result = startCaseRequestSchema.safeParse({ scenario: "Test scenario" });
|
||||
const result = startCaseRequestSchema.safeParse({
|
||||
scenario: "Test scenario",
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
@@ -195,14 +232,16 @@ describe("API request schemas", () => {
|
||||
|
||||
it("rejects scenario over 10000 chars", () => {
|
||||
const longScenario = "a".repeat(10001);
|
||||
const result = startCaseRequestSchema.safeParse({ scenario: longScenario });
|
||||
const result = startCaseRequestSchema.safeParse({
|
||||
scenario: longScenario,
|
||||
});
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it("accepts optional promptVersion", () => {
|
||||
const result = startCaseRequestSchema.safeParse({
|
||||
scenario: "Test",
|
||||
promptVersion: "v0.3"
|
||||
const result = startCaseRequestSchema.safeParse({
|
||||
scenario: "Test",
|
||||
promptVersion: "v0.3",
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
@@ -213,7 +252,7 @@ describe("API request schemas", () => {
|
||||
const graph = makeGraph({
|
||||
centralStatement: "Test scenario",
|
||||
nodes: [makeNode({ id: "n1", label: "N" })],
|
||||
currentSummary: "Current state of situation"
|
||||
currentSummary: "Current state of situation",
|
||||
});
|
||||
const result = updateCaseRequestSchema.safeParse({
|
||||
situationGraph: graph,
|
||||
@@ -235,7 +274,7 @@ describe("API request schemas", () => {
|
||||
const graph = makeGraph({
|
||||
centralStatement: "Test",
|
||||
nodes: [makeNode({ id: "n1", label: "N" })],
|
||||
currentSummary: "Test summary"
|
||||
currentSummary: "Test summary",
|
||||
});
|
||||
const result = updateCaseRequestSchema.safeParse({
|
||||
situationGraph: graph,
|
||||
@@ -261,7 +300,9 @@ describe("deterministic ID generation", () => {
|
||||
});
|
||||
|
||||
it("IDs are prefixed with 'n' and short", () => {
|
||||
const id = makeNodeId("A very long label that would produce a longer hash if not truncated");
|
||||
const id = makeNodeId(
|
||||
"A very long label that would produce a longer hash if not truncated",
|
||||
);
|
||||
expect(id.startsWith("n")).toBe(true);
|
||||
expect(id.length).toBeLessThan(15);
|
||||
});
|
||||
@@ -325,7 +366,7 @@ describe("helper functions", () => {
|
||||
const graph = makeGraph({
|
||||
centralStatement: "Test",
|
||||
currentSummary: "Default summary",
|
||||
nodes: [makeNode({ id: "n1", label: "Placeholder" })]
|
||||
nodes: [makeNode({ id: "n1", label: "Placeholder" })],
|
||||
});
|
||||
const result = situationGraphSchema.safeParse(graph);
|
||||
expect(result.success).toBe(true);
|
||||
@@ -346,19 +387,48 @@ describe("helper functions", () => {
|
||||
|
||||
describe("enum values completeness", () => {
|
||||
it("SituationKind has all expected values", () => {
|
||||
const expected = ["observation", "reported_claim", "metric", "state", "transition", "relationship", "assumption", "unknown", "conclusion"];
|
||||
const expected = [
|
||||
"observation",
|
||||
"reported_claim",
|
||||
"metric",
|
||||
"state",
|
||||
"transition",
|
||||
"relationship",
|
||||
"assumption",
|
||||
"unknown",
|
||||
"conclusion",
|
||||
];
|
||||
const actual = Object.values(SituationKind);
|
||||
expect(actual).toEqual(expect.arrayContaining(expected));
|
||||
});
|
||||
|
||||
it("SituationStatus has all expected values", () => {
|
||||
const expected = ["known", "unknown", "provisional", "supported", "weakened", "contradicted", "resolved"];
|
||||
const expected = [
|
||||
"known",
|
||||
"unknown",
|
||||
"provisional",
|
||||
"supported",
|
||||
"weakened",
|
||||
"contradicted",
|
||||
"resolved",
|
||||
];
|
||||
const actual = Object.values(SituationStatus);
|
||||
expect(actual).toEqual(expect.arrayContaining(expected));
|
||||
});
|
||||
|
||||
it("SituationRelationship has all expected values", () => {
|
||||
const expected = ["supports", "weakens", "contradicts", "depends_on", "causes", "may_cause", "measures", "compares_with", "updates", "other"];
|
||||
const expected = [
|
||||
"supports",
|
||||
"weakens",
|
||||
"contradicts",
|
||||
"depends_on",
|
||||
"causes",
|
||||
"may_cause",
|
||||
"measures",
|
||||
"compares_with",
|
||||
"updates",
|
||||
"other",
|
||||
];
|
||||
const actual = Object.values(SituationRelationship);
|
||||
expect(actual).toEqual(expect.arrayContaining(expected));
|
||||
});
|
||||
|
||||
@@ -0,0 +1,303 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
formulateQuestion,
|
||||
formulateTieResolutionQuestion,
|
||||
} from "@/lib/graph/question-formulator.js";
|
||||
import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js";
|
||||
import {
|
||||
explainUnknownSelection,
|
||||
selectActiveUnknownCandidate,
|
||||
} from "@/lib/graph/utils.js";
|
||||
|
||||
function buildLiveShapedGraph() {
|
||||
const summary = makeNode({
|
||||
id: "nnvog0y",
|
||||
label:
|
||||
"Revenue grew by 18% while corporate cash reserves declined over an identical time frame.",
|
||||
description: "Summary of the situation from the scenario text",
|
||||
kind: "state",
|
||||
status: "provisional",
|
||||
confidence: "medium",
|
||||
});
|
||||
const revenueObservation = makeNode({
|
||||
id: "nri36w9",
|
||||
label: "Revenue increased by 18%.",
|
||||
description: "Revenue increased by 18%.",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
evidenceIds: ["obs_rev"],
|
||||
});
|
||||
const cashObservation = makeNode({
|
||||
id: "nnfc48j",
|
||||
label: "Cash in the bank decreased over the same period.",
|
||||
description: "Cash in the bank decreased over the same period.",
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
evidenceIds: ["obs_cash"],
|
||||
});
|
||||
const revenueMetric = makeNode({
|
||||
id: "nhsd6d5",
|
||||
label: "Revenue metric (typically accrual-based income statement figure)",
|
||||
description:
|
||||
"Revenue metric (typically accrual-based income statement figure)",
|
||||
kind: "metric",
|
||||
status: "known",
|
||||
confidence: "high",
|
||||
});
|
||||
const cashMetric = makeNode({
|
||||
id: "neh5m6m",
|
||||
label:
|
||||
"Cash balance (liquidity measure on the balance sheet or cash flow statement)",
|
||||
description:
|
||||
"Cash balance (liquidity measure on the balance sheet or cash flow statement)",
|
||||
kind: "metric",
|
||||
status: "known",
|
||||
confidence: "high",
|
||||
});
|
||||
const directionalRelationship = makeNode({
|
||||
id: "nwo6070",
|
||||
label:
|
||||
"Divergent directional movement between top-line revenue growth and net cash position contraction.",
|
||||
description:
|
||||
"Divergent directional movement between top-line revenue growth and net cash position contraction.",
|
||||
kind: "relationship",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
});
|
||||
const contradictionRelationship = makeNode({
|
||||
id: "nuiab02",
|
||||
label:
|
||||
"Apparent contradiction between profitability/revenue expansion and liquidity reduction.",
|
||||
description:
|
||||
"Apparent contradiction between profitability/revenue expansion and liquidity reduction.",
|
||||
kind: "relationship",
|
||||
status: "supported",
|
||||
confidence: "medium",
|
||||
});
|
||||
const cashTiming = makeNode({
|
||||
id: "niewza",
|
||||
label:
|
||||
"Whether revenue recognition timing differs from cash collection timing.",
|
||||
description:
|
||||
"Whether revenue recognition timing differs from cash collection timing.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
});
|
||||
const cashOutflows = makeNode({
|
||||
id: "nqdzobz",
|
||||
label:
|
||||
"Magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts).",
|
||||
description:
|
||||
"Magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts).",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
});
|
||||
|
||||
const edges = [
|
||||
makeEdge({
|
||||
id: "e-revenue-summary",
|
||||
fromNodeId: revenueObservation.id,
|
||||
toNodeId: summary.id,
|
||||
relationship: "supports",
|
||||
description: "Revenue increase supports the scenario summary.",
|
||||
}),
|
||||
makeEdge({
|
||||
id: "e-cash-summary",
|
||||
fromNodeId: cashObservation.id,
|
||||
toNodeId: summary.id,
|
||||
relationship: "supports",
|
||||
description: "Cash decline supports the scenario summary.",
|
||||
}),
|
||||
makeEdge({
|
||||
id: "e-unk-niewza",
|
||||
fromNodeId: cashTiming.id,
|
||||
toNodeId: summary.id,
|
||||
relationship: "depends_on",
|
||||
description:
|
||||
"Whether revenue recognition timing differs from cash collection timing. is an unresolved factor for this situation",
|
||||
}),
|
||||
makeEdge({
|
||||
id: "e-unk-nqdzobz",
|
||||
fromNodeId: cashOutflows.id,
|
||||
toNodeId: summary.id,
|
||||
relationship: "depends_on",
|
||||
description:
|
||||
"Magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts). is an unresolved factor for this situation",
|
||||
}),
|
||||
];
|
||||
|
||||
return makeGraph({
|
||||
centralStatement:
|
||||
"Revenue increased by 18%, but cash in the bank fell over the same period.",
|
||||
nodes: [
|
||||
summary,
|
||||
revenueObservation,
|
||||
cashObservation,
|
||||
revenueMetric,
|
||||
cashMetric,
|
||||
directionalRelationship,
|
||||
contradictionRelationship,
|
||||
cashTiming,
|
||||
cashOutflows,
|
||||
],
|
||||
edges,
|
||||
activeUnknownNodeId: null,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: "Diagnostic selection influence fixture",
|
||||
});
|
||||
}
|
||||
|
||||
function orderCandidates(explanation) {
|
||||
return explanation.candidates.map((candidate) => ({
|
||||
nodeId: candidate.nodeId,
|
||||
label: candidate.label,
|
||||
score: candidate.score,
|
||||
downstreamCount: candidate.downstreamCount,
|
||||
unresolvedParentUnknownCount: candidate.unresolvedParentUnknownCount,
|
||||
}));
|
||||
}
|
||||
|
||||
function removeDependencyLinks(graph) {
|
||||
const nodes = graph.nodes.map((node) => ({
|
||||
...node,
|
||||
dependsOn: [],
|
||||
affects: [],
|
||||
parentId: null,
|
||||
childIds: [],
|
||||
}));
|
||||
const edges = (graph.edges || []).filter(
|
||||
(edge) => edge.relationship !== "depends_on",
|
||||
);
|
||||
return makeGraph({ ...graph, nodes, edges, activeUnknownNodeId: null });
|
||||
}
|
||||
|
||||
function neutraliseUnknownWording(graph) {
|
||||
let counter = 0;
|
||||
const nodes = graph.nodes.map((node) => {
|
||||
if (node.kind !== "unknown") return { ...node };
|
||||
counter += 1;
|
||||
return {
|
||||
...node,
|
||||
label: `Unknown ${String.fromCharCode(64 + counter)}`,
|
||||
description: `Unknown factor ${counter} relevant to the scenario.`,
|
||||
};
|
||||
});
|
||||
return makeGraph({ ...graph, nodes, activeUnknownNodeId: null });
|
||||
}
|
||||
|
||||
describe("selection influence diagnostic", () => {
|
||||
it("records ambiguous ordering changes for live-shaped, structure-only, and wording-neutralised fixtures", () => {
|
||||
const liveGraph = buildLiveShapedGraph();
|
||||
const liveExplanation = explainUnknownSelection(liveGraph, []);
|
||||
const liveSelection = selectActiveUnknownCandidate(liveGraph, []);
|
||||
const tieQuestion = formulateTieResolutionQuestion({ graph: liveGraph });
|
||||
|
||||
const noLinksExplanation = explainUnknownSelection(
|
||||
removeDependencyLinks(liveGraph),
|
||||
[],
|
||||
);
|
||||
const noLinksSelection = selectActiveUnknownCandidate(
|
||||
removeDependencyLinks(liveGraph),
|
||||
[],
|
||||
);
|
||||
const neutralWordingExplanation = explainUnknownSelection(
|
||||
neutraliseUnknownWording(liveGraph),
|
||||
[],
|
||||
);
|
||||
const neutralSelection = selectActiveUnknownCandidate(
|
||||
neutraliseUnknownWording(liveGraph),
|
||||
[],
|
||||
);
|
||||
|
||||
const fallbackQuestion = formulateQuestion({
|
||||
node: liveGraph.nodes.find((node) => node.id === "nqdzobz"),
|
||||
graph: liveGraph,
|
||||
});
|
||||
|
||||
const diagnosticRecord = {
|
||||
liveStatus: liveExplanation.status,
|
||||
liveShapedCandidateOrdering: orderCandidates(liveExplanation),
|
||||
liveTiedCandidateIds: liveExplanation.tiedCandidateIds,
|
||||
noLinksCandidateOrdering: orderCandidates(noLinksExplanation),
|
||||
noLinksStatus: noLinksExplanation.status,
|
||||
neutralWordingCandidateOrdering: orderCandidates(
|
||||
neutralWordingExplanation,
|
||||
),
|
||||
neutralStatus: neutralWordingExplanation.status,
|
||||
selectedExplanationContributions: liveExplanation.selected?.contributions,
|
||||
tieQuestion: tieQuestion.question,
|
||||
liveSelection,
|
||||
noLinksSelection,
|
||||
neutralSelection,
|
||||
fallbackQuestion,
|
||||
};
|
||||
|
||||
expect(diagnosticRecord.liveStatus).toBe("ambiguous");
|
||||
expect(diagnosticRecord.liveTiedCandidateIds).toEqual([
|
||||
"nqdzobz",
|
||||
"niewza",
|
||||
]);
|
||||
expect(diagnosticRecord.liveShapedCandidateOrdering).toEqual([
|
||||
{
|
||||
nodeId: "nqdzobz",
|
||||
label:
|
||||
"Magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts).",
|
||||
score: 0,
|
||||
downstreamCount: 0,
|
||||
unresolvedParentUnknownCount: 0,
|
||||
},
|
||||
{
|
||||
nodeId: "niewza",
|
||||
label:
|
||||
"Whether revenue recognition timing differs from cash collection timing.",
|
||||
score: 0,
|
||||
downstreamCount: 0,
|
||||
unresolvedParentUnknownCount: 0,
|
||||
},
|
||||
]);
|
||||
expect(diagnosticRecord.liveSelection).toMatchObject({
|
||||
selectedNode: null,
|
||||
status: "ambiguous",
|
||||
tieType: "complete_unresolved_tie",
|
||||
tiedCandidateIds: ["nqdzobz", "niewza"],
|
||||
});
|
||||
expect(diagnosticRecord.noLinksCandidateOrdering).toEqual(
|
||||
diagnosticRecord.liveShapedCandidateOrdering,
|
||||
);
|
||||
expect(diagnosticRecord.noLinksStatus).toBe("ambiguous");
|
||||
expect(diagnosticRecord.noLinksSelection.status).toBe("ambiguous");
|
||||
expect(diagnosticRecord.neutralWordingCandidateOrdering).toEqual([
|
||||
{
|
||||
nodeId: "niewza",
|
||||
label: "Unknown A",
|
||||
score: 0,
|
||||
downstreamCount: 0,
|
||||
unresolvedParentUnknownCount: 0,
|
||||
},
|
||||
{
|
||||
nodeId: "nqdzobz",
|
||||
label: "Unknown B",
|
||||
score: 0,
|
||||
downstreamCount: 0,
|
||||
unresolvedParentUnknownCount: 0,
|
||||
},
|
||||
]);
|
||||
expect(diagnosticRecord.neutralStatus).toBe("ambiguous");
|
||||
expect(diagnosticRecord.neutralSelection.status).toBe("ambiguous");
|
||||
expect(diagnosticRecord.selectedExplanationContributions).toBeUndefined();
|
||||
expect(diagnosticRecord.tieQuestion).toBe(
|
||||
"Were these figures measured on the same basis and at the same scale?",
|
||||
);
|
||||
expect(diagnosticRecord.tieQuestion.toLowerCase()).not.toMatch(
|
||||
/accounts receivable|capex|debt repayments|working capital/,
|
||||
);
|
||||
expect(diagnosticRecord.fallbackQuestion.strategy).toBeNull();
|
||||
expect(diagnosticRecord.fallbackQuestion.question).toBe(
|
||||
"What would clarify magnitude and nature of cash outflows (operating expenses, debt repayments, capex, or working capital shifts) in this situation?",
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,138 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
assessUnknownAnswerability,
|
||||
assessUnknownAtomicity,
|
||||
} from "@/lib/graph/question-formulator.js";
|
||||
import { applyValidatedProposal } from "@/lib/graph/apply-proposal.js";
|
||||
import { makeGraph, makeNode } from "@/lib/graph/schema.js";
|
||||
|
||||
const COMMERCIAL_SCENARIO =
|
||||
"I have developed a new reasoning method that aims to help people determine whether they have enough justified confidence to make a decision. I believe it could become a commercial product, but I do not yet know whether it solves a genuine problem, whether people would value it enough to pay for it, or whether it is fundamentally different from existing AI tools. Before investing significant time and money into building it further, I want to determine whether continuing development is commercially justified.";
|
||||
|
||||
function makeMeaningfulNoOpProposal() {
|
||||
return {
|
||||
addedNodes: [
|
||||
makeNode({
|
||||
id: "n-anchor",
|
||||
label: "Update anchor",
|
||||
description:
|
||||
"Anchor state introduced by the answer because the update must contain a meaningful change.",
|
||||
kind: "state",
|
||||
status: "known",
|
||||
confidence: "low",
|
||||
}),
|
||||
],
|
||||
updatedNodes: [],
|
||||
addedEdges: [],
|
||||
removedEdgeIds: [],
|
||||
resolvedUnknownNodeIds: [],
|
||||
affectedNodeIds: [],
|
||||
selectedQuestion: null,
|
||||
};
|
||||
}
|
||||
|
||||
function makeCommercialContainerGraph() {
|
||||
const parent = makeNode({
|
||||
id: "n-commercial-parent",
|
||||
label:
|
||||
"Commercial justification for whether continuing development is commercially justified",
|
||||
description:
|
||||
"Need to know whether this solves a genuine problem, whether people would value it enough to pay for it, and whether it is commercially justified before continuing development.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
|
||||
return makeGraph({
|
||||
centralStatement: COMMERCIAL_SCENARIO,
|
||||
nodes: [parent],
|
||||
edges: [],
|
||||
activeUnknownNodeId: parent.id,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: "Commercial answerability fixture",
|
||||
});
|
||||
}
|
||||
|
||||
describe("assessUnknownAnswerability", () => {
|
||||
it("flags commercial-validation container unknowns as non-answerable", () => {
|
||||
const graph = makeCommercialContainerGraph();
|
||||
const unknown = graph.nodes[0];
|
||||
|
||||
const atomicity = assessUnknownAtomicity({ node: unknown, graph });
|
||||
const answerability = assessUnknownAnswerability({ node: unknown, graph });
|
||||
|
||||
expect(atomicity.atomicity).toBe("composite");
|
||||
expect(answerability.independentlyAnswerable).toBe(false);
|
||||
expect(answerability.decompositionRequired).toBe(true);
|
||||
expect(answerability.prerequisiteConceptCount).toBeGreaterThan(1);
|
||||
});
|
||||
|
||||
it("keeps one-concept denominator unknowns independently answerable", () => {
|
||||
const unknown = makeNode({
|
||||
id: "n-denominator",
|
||||
label: "Complaint rate denominator",
|
||||
description:
|
||||
"Need the denominator because it directly determines the complaint rate.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
});
|
||||
const graph = makeGraph({
|
||||
centralStatement: "Production increased while complaints increased.",
|
||||
nodes: [unknown],
|
||||
edges: [],
|
||||
activeUnknownNodeId: unknown.id,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: "Denominator answerability fixture",
|
||||
});
|
||||
|
||||
const result = assessUnknownAnswerability({ node: unknown, graph });
|
||||
|
||||
expect(result.independentlyAnswerable).toBe(true);
|
||||
expect(result.decompositionRequired).toBe(false);
|
||||
expect(result.prerequisiteConceptCount).toBeLessThanOrEqual(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("answerability-triggered decomposition", () => {
|
||||
it("decomposes a non-answerable parent into independently answerable child investigations", () => {
|
||||
const graph = makeCommercialContainerGraph();
|
||||
|
||||
const result = applyValidatedProposal({
|
||||
situationGraph: graph,
|
||||
proposal: makeMeaningfulNoOpProposal(),
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.decompositionPerformed).toBe(true);
|
||||
expect(result.decompositionTriggeredByAnswerability).toBe(true);
|
||||
expect(result.selectedContainerUnknown).toBe("n-commercial-parent");
|
||||
expect(result.selectedChildUnknown).toBe(result.selectedUnknownAfter);
|
||||
expect(result.independentlyAnswerable).toBe(false);
|
||||
expect(result.prerequisiteConceptCount).toBeGreaterThan(1);
|
||||
expect(result.selectedUnknownAfter).not.toBe("n-commercial-parent");
|
||||
expect(result.selectedQuestion.question).toBe(
|
||||
"Who experiences this problem?",
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps the parent unresolved while selecting a child unknown", () => {
|
||||
const graph = makeCommercialContainerGraph();
|
||||
|
||||
const result = applyValidatedProposal({
|
||||
situationGraph: graph,
|
||||
proposal: makeMeaningfulNoOpProposal(),
|
||||
});
|
||||
|
||||
const parentNode = result.updatedSituationGraph.nodes.find(
|
||||
(node) => node.id === "n-commercial-parent",
|
||||
);
|
||||
const selectedChild = result.updatedSituationGraph.nodes.find(
|
||||
(node) => node.id === result.selectedUnknownAfter,
|
||||
);
|
||||
|
||||
expect(parentNode.status).toBe("unknown");
|
||||
expect(selectedChild.parentId).toBe(parentNode.id);
|
||||
expect(selectedChild.status).toBe("unknown");
|
||||
});
|
||||
});
|
||||
@@ -18,6 +18,7 @@ function makeValidProposal(overrides = {}) {
|
||||
removedEdgeIds: [],
|
||||
resolvedUnknownNodeIds: ["n-unknown"],
|
||||
affectedNodeIds: [],
|
||||
selectedQuestion: null,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
@@ -117,7 +118,34 @@ describe("parseGraphUpdateProposal", () => {
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it("does not invent a next question", () => {
|
||||
it("defaults missing selectedQuestion to null", () => {
|
||||
const result = parseGraphUpdateProposal({
|
||||
addedNodes: [],
|
||||
updatedNodes: [],
|
||||
addedEdges: [],
|
||||
removedEdgeIds: [],
|
||||
resolvedUnknownNodeIds: [],
|
||||
affectedNodeIds: [],
|
||||
});
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.proposal.selectedQuestion).toBeNull();
|
||||
});
|
||||
|
||||
it("parses a valid selectedQuestion", () => {
|
||||
const result = parseGraphUpdateProposal(
|
||||
makeValidProposal({
|
||||
selectedQuestion: {
|
||||
nodeId: "n-follow-up",
|
||||
question: "How should commercial value be defined for this decision?",
|
||||
reason: "A consequential unknown remains unresolved.",
|
||||
},
|
||||
}),
|
||||
);
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.proposal.selectedQuestion?.nodeId).toBe("n-follow-up");
|
||||
});
|
||||
|
||||
it("does not invent a next question field outside the contract", () => {
|
||||
const result = parseGraphUpdateProposal(makeValidProposal());
|
||||
expect(result.proposal.nextQuestion).toBeUndefined();
|
||||
});
|
||||
|
||||
@@ -0,0 +1,419 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { applyValidatedProposal } from "@/lib/graph/apply-proposal.js";
|
||||
import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js";
|
||||
|
||||
function makePropagationFixture({
|
||||
key,
|
||||
centralStatement,
|
||||
firstObservationLabel,
|
||||
secondObservationLabel,
|
||||
}) {
|
||||
const parent = makeNode({
|
||||
id: `${key}-parent`,
|
||||
label: `Explanation for why ${centralStatement}`,
|
||||
description:
|
||||
"Need to understand what change or event could explain why these observations differ, because that is needed to investigate their relationship.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
});
|
||||
const measurementChild = makeNode({
|
||||
id: `${key}-child-measurement`,
|
||||
label: "How the two observations were measured",
|
||||
description: `Need evidence about the measure used for each observation, because that could help explain ${centralStatement}.`,
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
parentId: parent.id,
|
||||
});
|
||||
const timingChild = makeNode({
|
||||
id: `${key}-child-timing`,
|
||||
label: "Whether the two observations reflect different timing",
|
||||
description: `Need to know whether the two observations reflect different timing, because that could help explain ${centralStatement}.`,
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
parentId: parent.id,
|
||||
});
|
||||
const cashMovementChild = makeNode({
|
||||
id: `${key}-child-cash-movement`,
|
||||
label: `Possible change mainly affecting ${secondObservationLabel}`,
|
||||
description: `Need to know whether a possible change mainly affected ${secondObservationLabel}, because that could help explain ${centralStatement}.`,
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
parentId: parent.id,
|
||||
});
|
||||
const oneOffChild = makeNode({
|
||||
id: `${key}-child-one-off`,
|
||||
label: "Possible one-off event during the period",
|
||||
description: `Need to know whether a possible one-off event happened during the period, because that could help explain ${centralStatement}.`,
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
parentId: parent.id,
|
||||
});
|
||||
const ancestor = makeNode({
|
||||
id: `${key}-ancestor`,
|
||||
label: `Reasoning for ${centralStatement}`,
|
||||
description:
|
||||
"Higher-level reasoning node depending on the parent explanation.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
childIds: [parent.id],
|
||||
});
|
||||
const unrelated = makeNode({
|
||||
id: `${key}-unrelated`,
|
||||
label: "Unrelated branch",
|
||||
description: "Should remain unchanged.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "low",
|
||||
});
|
||||
const firstObservation = makeNode({
|
||||
id: `${key}-obs-1`,
|
||||
label: firstObservationLabel,
|
||||
description: firstObservationLabel,
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
});
|
||||
const secondObservation = makeNode({
|
||||
id: `${key}-obs-2`,
|
||||
label: secondObservationLabel,
|
||||
description: secondObservationLabel,
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
});
|
||||
|
||||
const graph = makeGraph({
|
||||
centralStatement,
|
||||
nodes: [
|
||||
ancestor,
|
||||
parent,
|
||||
measurementChild,
|
||||
timingChild,
|
||||
cashMovementChild,
|
||||
oneOffChild,
|
||||
unrelated,
|
||||
firstObservation,
|
||||
secondObservation,
|
||||
],
|
||||
edges: [
|
||||
makeEdge({
|
||||
id: `${key}-e-parent-ancestor`,
|
||||
fromNodeId: parent.id,
|
||||
toNodeId: ancestor.id,
|
||||
relationship: "depends_on",
|
||||
description: "Ancestor depends on the parent explanation.",
|
||||
}),
|
||||
makeEdge({
|
||||
id: `${key}-e-child-measurement-parent`,
|
||||
fromNodeId: measurementChild.id,
|
||||
toNodeId: parent.id,
|
||||
relationship: "depends_on",
|
||||
description: "Measurement child depends into the parent explanation.",
|
||||
}),
|
||||
makeEdge({
|
||||
id: `${key}-e-child-timing-parent`,
|
||||
fromNodeId: timingChild.id,
|
||||
toNodeId: parent.id,
|
||||
relationship: "depends_on",
|
||||
description: "Timing child depends into the parent explanation.",
|
||||
}),
|
||||
makeEdge({
|
||||
id: `${key}-e-child-cash-parent`,
|
||||
fromNodeId: cashMovementChild.id,
|
||||
toNodeId: parent.id,
|
||||
relationship: "depends_on",
|
||||
description: "Cash-movement child depends into the parent explanation.",
|
||||
}),
|
||||
makeEdge({
|
||||
id: `${key}-e-child-one-off-parent`,
|
||||
fromNodeId: oneOffChild.id,
|
||||
toNodeId: parent.id,
|
||||
relationship: "depends_on",
|
||||
description: "One-off child depends into the parent explanation.",
|
||||
}),
|
||||
],
|
||||
activeUnknownNodeId: measurementChild.id,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: `Propagation fixture for ${key}`,
|
||||
});
|
||||
|
||||
return {
|
||||
graph,
|
||||
ids: {
|
||||
ancestor: ancestor.id,
|
||||
parent: parent.id,
|
||||
measurementChild: measurementChild.id,
|
||||
timingChild: timingChild.id,
|
||||
cashMovementChild: cashMovementChild.id,
|
||||
oneOffChild: oneOffChild.id,
|
||||
unrelated: unrelated.id,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const scenarios = [
|
||||
{
|
||||
key: "revenue-cash",
|
||||
centralStatement: "revenue increased while cash fell",
|
||||
firstObservationLabel: "Revenue increased by 18%.",
|
||||
secondObservationLabel: "Cash in the bank fell over the same period.",
|
||||
},
|
||||
{
|
||||
key: "satisfaction-complaints",
|
||||
centralStatement:
|
||||
"customer satisfaction increased while complaints increased",
|
||||
firstObservationLabel: "Customer satisfaction increased.",
|
||||
secondObservationLabel: "Complaints increased.",
|
||||
},
|
||||
{
|
||||
key: "traffic-sales",
|
||||
centralStatement: "traffic increased while sales stayed flat",
|
||||
firstObservationLabel: "Website traffic increased.",
|
||||
secondObservationLabel: "Sales stayed flat.",
|
||||
},
|
||||
{
|
||||
key: "delivery-cancellations",
|
||||
centralStatement: "delivery time fell while cancellations increased",
|
||||
firstObservationLabel: "Average delivery time decreased.",
|
||||
secondObservationLabel: "Cancellations increased.",
|
||||
},
|
||||
{
|
||||
key: "production-defects",
|
||||
centralStatement: "production increased while defects increased",
|
||||
firstObservationLabel: "Production increased.",
|
||||
secondObservationLabel: "Defects increased.",
|
||||
},
|
||||
];
|
||||
|
||||
describe("upward propagation", () => {
|
||||
it.each(scenarios)(
|
||||
"propagates resolved measurement child upward for $key",
|
||||
({
|
||||
key,
|
||||
centralStatement,
|
||||
firstObservationLabel,
|
||||
secondObservationLabel,
|
||||
}) => {
|
||||
const { graph, ids } = makePropagationFixture({
|
||||
key,
|
||||
centralStatement,
|
||||
firstObservationLabel,
|
||||
secondObservationLabel,
|
||||
});
|
||||
const unrelatedBefore = JSON.stringify(
|
||||
graph.nodes.find((node) => node.id === ids.unrelated),
|
||||
);
|
||||
|
||||
const result = applyValidatedProposal({
|
||||
situationGraph: graph,
|
||||
proposal: {
|
||||
addedNodes: [
|
||||
makeNode({
|
||||
id: `${key}-anchor`,
|
||||
label: "Update anchor",
|
||||
description:
|
||||
"Anchor state introduced by the answer because the update must contain a meaningful change.",
|
||||
kind: "state",
|
||||
status: "known",
|
||||
confidence: "low",
|
||||
}),
|
||||
],
|
||||
updatedNodes: [
|
||||
{
|
||||
nodeId: ids.measurementChild,
|
||||
previousStatus: "unknown",
|
||||
newStatus: "resolved",
|
||||
previousValue: null,
|
||||
newValue:
|
||||
"The figures were measured over the same accounting period using the same management accounts.",
|
||||
reason: "The answer resolves the measurement child.",
|
||||
},
|
||||
],
|
||||
addedEdges: [],
|
||||
removedEdgeIds: [],
|
||||
resolvedUnknownNodeIds: [ids.measurementChild],
|
||||
affectedNodeIds: [],
|
||||
selectedQuestion: null,
|
||||
},
|
||||
previousQuestion:
|
||||
"What evidence would clarify how the two observations were measured?",
|
||||
answer:
|
||||
"The figures were measured over the same accounting period using the same management accounts.",
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.resolvedUnknownNodeIds).toContain(ids.measurementChild);
|
||||
expect(result.propagationPerformed).toBe(true);
|
||||
expect(result.resolvedChildNodeId).toBe(ids.measurementChild);
|
||||
expect(result.parentNodeId).toBe(ids.parent);
|
||||
expect(result.parentStatusBefore).toBe("unknown");
|
||||
expect(result.parentStatusAfter).toBe("provisional");
|
||||
expect(result.parentConfidenceBefore).toBe("medium");
|
||||
expect(result.parentConfidenceAfter).toBe("medium");
|
||||
expect(result.evidenceConfidenceBefore).toBe("medium");
|
||||
expect(result.evidenceConfidenceAfter).toBe("medium");
|
||||
expect(result.completenessBefore).toBe("empty");
|
||||
expect(result.completenessAfter).toBe("partial");
|
||||
expect(result.conclusionConfidenceBefore).toBe("low");
|
||||
expect(result.conclusionConfidenceAfter).toBe("medium");
|
||||
expect(result.confidenceCapReason).toBe(
|
||||
"unresolved_direct_children_cap_conclusion",
|
||||
);
|
||||
expect(result.parentResolved).toBe(false);
|
||||
expect(result.affectedAncestorIds).toContain(ids.parent);
|
||||
expect(result.affectedAncestorIds).toContain(ids.ancestor);
|
||||
expect(result.nextSelectedSibling).toBe(result.newActiveUnknownNodeId);
|
||||
expect(result.nextSelectedSibling).toBe(result.selectedQuestion?.nodeId);
|
||||
expect(result.nextSelectedSibling).not.toBe(ids.measurementChild);
|
||||
expect([
|
||||
ids.timingChild,
|
||||
ids.cashMovementChild,
|
||||
ids.oneOffChild,
|
||||
]).toContain(result.nextSelectedSibling);
|
||||
expect(result.selectedQuestion?.question.toLowerCase()).not.toContain(
|
||||
"measured",
|
||||
);
|
||||
|
||||
const parentNode = result.updatedSituationGraph.nodes.find(
|
||||
(node) => node.id === ids.parent,
|
||||
);
|
||||
expect(parentNode).toMatchObject({
|
||||
status: "provisional",
|
||||
confidence: "medium",
|
||||
confidenceAssessment: {
|
||||
evidenceConfidence: "medium",
|
||||
completenessStatus: "partial",
|
||||
conclusionConfidence: "medium",
|
||||
},
|
||||
});
|
||||
|
||||
const ancestorNode = result.updatedSituationGraph.nodes.find(
|
||||
(node) => node.id === ids.ancestor,
|
||||
);
|
||||
expect(ancestorNode).toMatchObject({
|
||||
status: "provisional",
|
||||
confidence: "low",
|
||||
confidenceAssessment: {
|
||||
evidenceConfidence: "low",
|
||||
completenessStatus: "empty",
|
||||
conclusionConfidence: "low",
|
||||
},
|
||||
});
|
||||
|
||||
const resolvedChild = result.updatedSituationGraph.nodes.find(
|
||||
(node) => node.id === ids.measurementChild,
|
||||
);
|
||||
expect(resolvedChild.status).toBe("resolved");
|
||||
expect(resolvedChild.evidenceIds).toContain(
|
||||
`answer:${ids.measurementChild}`,
|
||||
);
|
||||
|
||||
expect(
|
||||
result.updatedSituationGraph.nodes.filter(
|
||||
(node) => node.id === ids.measurementChild,
|
||||
),
|
||||
).toHaveLength(1);
|
||||
expect(
|
||||
JSON.stringify(
|
||||
result.updatedSituationGraph.nodes.find(
|
||||
(node) => node.id === ids.unrelated,
|
||||
),
|
||||
),
|
||||
).toBe(unrelatedBefore);
|
||||
},
|
||||
);
|
||||
|
||||
it("resolves the parent only after all direct children are resolved", () => {
|
||||
const { graph, ids } = makePropagationFixture({
|
||||
key: "completion-rule",
|
||||
centralStatement: "revenue increased while cash fell",
|
||||
firstObservationLabel: "Revenue increased by 18%.",
|
||||
secondObservationLabel: "Cash in the bank fell over the same period.",
|
||||
});
|
||||
|
||||
const result = applyValidatedProposal({
|
||||
situationGraph: graph,
|
||||
proposal: {
|
||||
addedNodes: [
|
||||
makeNode({
|
||||
id: "completion-rule-anchor",
|
||||
label: "Update anchor",
|
||||
description:
|
||||
"Anchor state introduced by the answer because the update must contain a meaningful change.",
|
||||
kind: "state",
|
||||
status: "known",
|
||||
confidence: "low",
|
||||
}),
|
||||
],
|
||||
updatedNodes: [
|
||||
{
|
||||
nodeId: ids.measurementChild,
|
||||
previousStatus: "unknown",
|
||||
newStatus: "resolved",
|
||||
previousValue: null,
|
||||
newValue: "same management accounts",
|
||||
reason: "resolved measurement child",
|
||||
},
|
||||
{
|
||||
nodeId: ids.timingChild,
|
||||
previousStatus: "unknown",
|
||||
newStatus: "resolved",
|
||||
previousValue: null,
|
||||
newValue: "timing aligned",
|
||||
reason: "resolved timing child",
|
||||
},
|
||||
{
|
||||
nodeId: ids.cashMovementChild,
|
||||
previousStatus: "unknown",
|
||||
newStatus: "resolved",
|
||||
previousValue: null,
|
||||
newValue: "cash left through operations",
|
||||
reason: "resolved movement child",
|
||||
},
|
||||
{
|
||||
nodeId: ids.oneOffChild,
|
||||
previousStatus: "unknown",
|
||||
newStatus: "resolved",
|
||||
previousValue: null,
|
||||
newValue: "no exceptional movement",
|
||||
reason: "resolved one-off child",
|
||||
},
|
||||
],
|
||||
addedEdges: [],
|
||||
removedEdgeIds: [],
|
||||
resolvedUnknownNodeIds: [
|
||||
ids.measurementChild,
|
||||
ids.timingChild,
|
||||
ids.cashMovementChild,
|
||||
ids.oneOffChild,
|
||||
],
|
||||
affectedNodeIds: [],
|
||||
selectedQuestion: null,
|
||||
},
|
||||
previousQuestion:
|
||||
"What evidence would clarify how the two observations were measured?",
|
||||
answer: "All direct child questions are now answered.",
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.parentResolved).toBe(true);
|
||||
expect(result.resolvedUnknownNodeIds).toContain(ids.parent);
|
||||
expect(
|
||||
result.updatedSituationGraph.nodes.find((node) => node.id === ids.parent),
|
||||
).toMatchObject({
|
||||
status: "resolved",
|
||||
confidence: "high",
|
||||
confidenceAssessment: {
|
||||
evidenceConfidence: "high",
|
||||
completenessStatus: "complete",
|
||||
conclusionConfidence: "high",
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
+480
-148
File diff suppressed because it is too large
Load Diff
@@ -71,6 +71,33 @@ describe("normaliseAnalysisResponse", () => {
|
||||
expect(result.changesApplied).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("normalises reported_claim evidenceType to reported_statement", () => {
|
||||
const input = {
|
||||
evidence: [
|
||||
{
|
||||
id: "ev1",
|
||||
description: "x",
|
||||
evidenceType: "reported_claim",
|
||||
confidence: "medium",
|
||||
importance: "important",
|
||||
source: "report",
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
const result = normaliseAnalysisResponse(input);
|
||||
|
||||
expect(result.normalised.evidence[0].evidenceType).toBe(
|
||||
"reported_statement",
|
||||
);
|
||||
expect(result.changesApplied).toEqual([
|
||||
{
|
||||
path: ["evidence", 0, "evidenceType"],
|
||||
change: "Converted reported_claim to reported_statement",
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("does not invent a next question", () => {
|
||||
const input = { evidence: [] };
|
||||
const result = normaliseAnalysisResponse(input);
|
||||
@@ -165,6 +192,62 @@ describe("analyseScenario compatibility", () => {
|
||||
expect(result.nextQuestion).toBeUndefined();
|
||||
});
|
||||
|
||||
it("succeeds when reported_claim is the only evidenceType mismatch", async () => {
|
||||
mockGenerateReconstruction.mockResolvedValue({
|
||||
inputClassification: {
|
||||
primaryType: "unexplained_change",
|
||||
secondaryTypes: [],
|
||||
reasoningModes: ["validate_measurement"],
|
||||
classificationReason: "reason",
|
||||
confidence: "medium",
|
||||
},
|
||||
reconstruction: {
|
||||
summary: "summary",
|
||||
actors: [],
|
||||
systemsOrObjects: [],
|
||||
expectedStates: [],
|
||||
observedStates: [],
|
||||
differences: [],
|
||||
knownTransitions: [],
|
||||
unexplainedTransitions: [],
|
||||
contradictions: [],
|
||||
importantUnknowns: [],
|
||||
plausibleInterpretations: [],
|
||||
},
|
||||
evidence: [
|
||||
{
|
||||
id: "ev1",
|
||||
description: "desc",
|
||||
evidenceType: "reported_claim",
|
||||
attribution: null,
|
||||
confidence: "medium",
|
||||
importance: "important",
|
||||
},
|
||||
],
|
||||
nextQuestion: {
|
||||
id: "q1",
|
||||
question: "What denominator?",
|
||||
targets: ["observedStates"],
|
||||
reason: "reason",
|
||||
expectedInformationValue: "high",
|
||||
reasoningMode: "validate_measurement",
|
||||
},
|
||||
});
|
||||
|
||||
const { analyseScenario } = await import("@/lib/analysis.js");
|
||||
const result = await analyseScenario("Scenario text", {
|
||||
promptVersion: "v0.3",
|
||||
});
|
||||
|
||||
expect(result.success).toBe(true);
|
||||
expect(result.compatibilityApplied).toBe(true);
|
||||
expect(result.compatibilityChanges).toContainEqual({
|
||||
path: ["evidence", 0, "evidenceType"],
|
||||
change: "Converted reported_claim to reported_statement",
|
||||
});
|
||||
expect(result.evidence[0].evidenceType).toBe("reported_statement");
|
||||
});
|
||||
|
||||
it("malformed JSON still fails", async () => {
|
||||
mockGenerateReconstruction.mockResolvedValue("{not valid json");
|
||||
|
||||
|
||||
+7
-1
@@ -64,9 +64,15 @@ test("graph-backed one-turn update smoke test", async ({ page }) => {
|
||||
timeout: 240000,
|
||||
});
|
||||
await expect(page.getByText(/Resolved unknowns/i)).toBeVisible();
|
||||
await expect(page.getByText(/Newly surfaced unknowns/i)).toBeVisible();
|
||||
await expect(page.getByText(/Affected nodes/i)).toBeVisible();
|
||||
await expect(
|
||||
page.getByText(/No next question selected yet\./i),
|
||||
page.getByText(/Selected Question|Next question:/i),
|
||||
).toBeVisible();
|
||||
await expect(
|
||||
page.getByText(
|
||||
/Additional submission is disabled in this one-update prototype\./i,
|
||||
),
|
||||
).toBeVisible();
|
||||
await expect(page.getByText(/Error:/i)).toHaveCount(0);
|
||||
await expect(page.getByText(/Update error:/i)).toHaveCount(0);
|
||||
|
||||
@@ -80,7 +80,7 @@ function makeUpdateSuccess(overrides = {}) {
|
||||
updatedSituationGraph: {
|
||||
centralStatement: "Complaints increased while production increased.",
|
||||
currentSummary: "Updated summary",
|
||||
activeUnknownNodeId: "n-next-unknown",
|
||||
activeUnknownNodeId: "n-child-1",
|
||||
resolvedNodeIds: ["n-unknown"],
|
||||
nodes: [
|
||||
{
|
||||
@@ -113,11 +113,88 @@ function makeUpdateSuccess(overrides = {}) {
|
||||
value: "1.9 complaints per 100 units",
|
||||
unit: null,
|
||||
},
|
||||
{
|
||||
id: "n-next-unknown",
|
||||
label:
|
||||
"Explanation for why revenue increased by 18%, but cash in the bank fell over the same period",
|
||||
description:
|
||||
"Need to understand what change or event could explain why these observations differ, because that is needed to investigate their relationship.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
value: null,
|
||||
unit: null,
|
||||
},
|
||||
{
|
||||
id: "n-child-1",
|
||||
label: "How the two observations were measured",
|
||||
description:
|
||||
"Need evidence about the measure used for each observation, because that could help explain revenue increased by 18%, but cash in the bank fell over the same period.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
value: null,
|
||||
unit: null,
|
||||
parentId: "n-next-unknown",
|
||||
},
|
||||
],
|
||||
edges: [
|
||||
{
|
||||
id: "e-rel-next",
|
||||
fromNodeId: "n-conclusion",
|
||||
toNodeId: "n-next-unknown",
|
||||
relationship: "depends_on",
|
||||
confidence: "medium",
|
||||
description:
|
||||
"This unresolved explanation arises from the now-assessed relationship between the observations.",
|
||||
},
|
||||
{
|
||||
id: "e-child-next",
|
||||
fromNodeId: "n-child-1",
|
||||
toNodeId: "n-next-unknown",
|
||||
relationship: "depends_on",
|
||||
confidence: "medium",
|
||||
description:
|
||||
"This child unknown must be investigated before the broader parent explanation can be resolved.",
|
||||
},
|
||||
],
|
||||
edges: [],
|
||||
},
|
||||
proposal: {
|
||||
addedNodes: [],
|
||||
addedNodes: [
|
||||
{
|
||||
id: "n-next-unknown",
|
||||
label:
|
||||
"Explanation for why revenue increased by 18%, but cash in the bank fell over the same period",
|
||||
description:
|
||||
"Need to understand what change or event could explain why these observations differ, because that is needed to investigate their relationship.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
value: null,
|
||||
unit: null,
|
||||
evidenceIds: [],
|
||||
dependsOn: ["n-conclusion"],
|
||||
affects: [],
|
||||
parentId: "n-conclusion",
|
||||
childIds: [],
|
||||
},
|
||||
{
|
||||
id: "n-child-1",
|
||||
label: "How the two observations were measured",
|
||||
description:
|
||||
"Need evidence about the measure used for each observation, because that could help explain revenue increased by 18%, but cash in the bank fell over the same period.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
value: null,
|
||||
unit: null,
|
||||
evidenceIds: [],
|
||||
dependsOn: [],
|
||||
affects: [],
|
||||
parentId: "n-next-unknown",
|
||||
childIds: [],
|
||||
},
|
||||
],
|
||||
updatedNodes: [
|
||||
{ nodeId: "n-unknown", newStatus: "resolved", reason: "answered" },
|
||||
],
|
||||
@@ -125,11 +202,69 @@ function makeUpdateSuccess(overrides = {}) {
|
||||
removedEdgeIds: [],
|
||||
resolvedUnknownNodeIds: ["n-unknown"],
|
||||
affectedNodeIds: ["n-conclusion"],
|
||||
selectedQuestion: {
|
||||
nodeId: "n-child-1",
|
||||
question:
|
||||
"What evidence would clarify how the two observations were measured?",
|
||||
reason:
|
||||
"Formulated from graph context using the evidence_gathering investigation strategy.",
|
||||
},
|
||||
},
|
||||
selectedQuestion: {
|
||||
nodeId: "n-child-1",
|
||||
question:
|
||||
"What evidence would clarify how the two observations were measured?",
|
||||
reason:
|
||||
"Formulated from graph context using the evidence_gathering investigation strategy.",
|
||||
},
|
||||
affectedNodeIds: ["n-conclusion"],
|
||||
resolvedUnknownNodeIds: ["n-unknown"],
|
||||
previousActiveUnknownNodeId: "n-unknown",
|
||||
newActiveUnknownNodeId: "n-next-unknown",
|
||||
newActiveUnknownNodeId: "n-child-1",
|
||||
emergentReasoningNodeCreated: true,
|
||||
emergentReasoningNodeId: "n-next-unknown",
|
||||
emergentReasoningNodeReason:
|
||||
"Created a new unresolved reasoning unknown so the next justified question is backed by the graph.",
|
||||
atomicityAssessment: "composite",
|
||||
decompositionPerformed: true,
|
||||
childUnknownCount: 1,
|
||||
childNodeIds: ["n-child-1"],
|
||||
atomicityReason:
|
||||
"Decomposed a composite unknown into smaller broad candidate dimensions before asking the next question.",
|
||||
previousReasoningState: {
|
||||
comparabilityStatus: "uncertain",
|
||||
reasoningStages: [
|
||||
{
|
||||
stage: "comparability",
|
||||
status: "uncertain",
|
||||
outcome:
|
||||
"Comparability between the observations is not yet established across period, scale, or measurement basis.",
|
||||
},
|
||||
{
|
||||
stage: "relationship",
|
||||
status: "insufficient_information",
|
||||
outcome: "not assessed until comparability is established",
|
||||
},
|
||||
],
|
||||
},
|
||||
reasoningState: {
|
||||
comparabilityStatus: "confirmed",
|
||||
relationshipStatus: "insufficient_information",
|
||||
reasoningStages: [
|
||||
{
|
||||
stage: "comparability",
|
||||
status: "confirmed",
|
||||
outcome:
|
||||
"Comparability was confirmed by the user answer covering the same period and source basis.",
|
||||
},
|
||||
{
|
||||
stage: "relationship",
|
||||
status: "insufficient_information",
|
||||
outcome:
|
||||
"There is not enough structure to classify the relationship safely.",
|
||||
},
|
||||
],
|
||||
},
|
||||
changesApplied: {
|
||||
updatedNodeCount: 2,
|
||||
resolvedUnknownCount: 1,
|
||||
@@ -140,6 +275,94 @@ function makeUpdateSuccess(overrides = {}) {
|
||||
};
|
||||
}
|
||||
|
||||
function makeCommercialUpdateSuccess(overrides = {}) {
|
||||
return {
|
||||
success: true,
|
||||
stage: "update_applied",
|
||||
updatedSituationGraph: {
|
||||
centralStatement:
|
||||
"I have developed a new reasoning method that aims to help people determine whether they have enough justified confidence to make a decision. I believe it could become a commercial product, but I do not yet know whether it solves a genuine problem, whether people would value it enough to pay for it, or whether it is fundamentally different from existing AI tools. Before investing significant time and money into building it further, I want to determine whether continuing development is commercially justified.",
|
||||
currentSummary: "Commercial update summary",
|
||||
activeUnknownNodeId: "n-other-people",
|
||||
resolvedNodeIds: ["n-who"],
|
||||
nodes: [
|
||||
{
|
||||
id: "n-commercial-parent",
|
||||
label:
|
||||
"Commercial justification for whether continuing development is commercially justified",
|
||||
description:
|
||||
"Need to know whether this solves a genuine problem, whether people would value it enough to pay for it, and whether it is commercially justified before continuing development.",
|
||||
kind: "unknown",
|
||||
status: "provisional",
|
||||
confidence: "medium",
|
||||
},
|
||||
{
|
||||
id: "n-who",
|
||||
label: "Who experiences this problem",
|
||||
description:
|
||||
"Need to know who experiences this problem, because that must be clear before deciding whether it is commercially justified.",
|
||||
kind: "unknown",
|
||||
status: "resolved",
|
||||
confidence: "medium",
|
||||
value:
|
||||
"I experience it myself when I am trying to decide whether a project, idea or investment is justified, but I do not yet know how common that problem is for other people.",
|
||||
parentId: "n-commercial-parent",
|
||||
},
|
||||
{
|
||||
id: "n-other-people",
|
||||
label: "Whether other people experience this problem",
|
||||
description:
|
||||
"Need to know whether other people experience this problem, because that must be established before deciding whether the problem is broadly important.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
parentId: "n-commercial-parent",
|
||||
},
|
||||
],
|
||||
edges: [
|
||||
{
|
||||
id: "e-other-parent",
|
||||
fromNodeId: "n-other-people",
|
||||
toNodeId: "n-commercial-parent",
|
||||
relationship: "depends_on",
|
||||
confidence: "medium",
|
||||
description:
|
||||
"This child unknown must be investigated before the broader parent explanation can be resolved.",
|
||||
},
|
||||
],
|
||||
},
|
||||
proposal: {
|
||||
addedNodes: [],
|
||||
updatedNodes: [{ nodeId: "n-who", newStatus: "resolved", reason: "answered" }],
|
||||
addedEdges: [],
|
||||
removedEdgeIds: [],
|
||||
resolvedUnknownNodeIds: ["n-who"],
|
||||
affectedNodeIds: ["n-commercial-parent"],
|
||||
selectedQuestion: null,
|
||||
},
|
||||
selectedQuestion: {
|
||||
nodeId: "n-other-people",
|
||||
question: "What makes you think other people experience this problem too?",
|
||||
reason:
|
||||
"Formulated as a direct foundational question because this child unknown should be answered one step at a time.",
|
||||
reasoningPattern: "decision",
|
||||
questionFamily: "decision_foundation",
|
||||
},
|
||||
previousActiveUnknownNodeId: "n-who",
|
||||
newActiveUnknownNodeId: "n-other-people",
|
||||
affectedNodeIds: ["n-commercial-parent"],
|
||||
resolvedUnknownNodeIds: ["n-who"],
|
||||
diagnostics: {
|
||||
unresolvedCandidateCount: 2,
|
||||
eligibleCandidateCount: 1,
|
||||
candidateNodeIds: ["n-other-people"],
|
||||
resolvedCurrentTurnNodeIds: ["n-who"],
|
||||
noQuestionReason: null,
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
describe("scenario-form UI helpers", () => {
|
||||
it("submits to /api/cases/start", async () => {
|
||||
const fetchImpl = vi.fn().mockResolvedValue({ ok: true });
|
||||
@@ -249,6 +472,44 @@ describe("graph-backed UI rendering", () => {
|
||||
expect(html).not.toContain("Update situation");
|
||||
});
|
||||
|
||||
it("renders the initial graph-backed commercial question when start-case reselection succeeds", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<ScenarioResultPanels
|
||||
result={makeGraphResult({
|
||||
situationGraph: {
|
||||
...makeGraphResult().situationGraph,
|
||||
activeUnknownNodeId: "n-who",
|
||||
nodes: [
|
||||
...makeGraphResult().situationGraph.nodes,
|
||||
{
|
||||
id: "n-who",
|
||||
label: "Who experiences this problem",
|
||||
description:
|
||||
"Need to know who experiences this problem, because that must be clear before deciding whether it is commercially justified.",
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "medium",
|
||||
parentId: "n-parent",
|
||||
},
|
||||
],
|
||||
},
|
||||
selectedQuestion: {
|
||||
nodeId: "n-who",
|
||||
question: "Who experiences this problem?",
|
||||
reason: "Graph-backed commercial child selection.",
|
||||
},
|
||||
diagnostics: {
|
||||
...makeGraphResult().diagnostics,
|
||||
noQuestionReason: null,
|
||||
},
|
||||
})}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(html).toContain("Who experiences this problem?");
|
||||
expect(html).toContain("Selected Question");
|
||||
});
|
||||
|
||||
it("renders diagnostics", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<DiagnosticsView result={makeGraphResult()} />,
|
||||
@@ -323,6 +584,22 @@ describe("graph-backed UI rendering", () => {
|
||||
expect(html).toContain("Complaint rate denominator");
|
||||
});
|
||||
|
||||
it("newly surfaced unknowns render", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<GraphUpdateView
|
||||
updateResult={{
|
||||
...makeUpdateSuccess(),
|
||||
previousSituationGraph: makeGraphResult().situationGraph,
|
||||
}}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(html).toContain("Newly surfaced unknowns");
|
||||
expect(html).toContain(
|
||||
"Explanation for why revenue increased by 18%, but cash in the bank fell over the same period",
|
||||
);
|
||||
});
|
||||
|
||||
it("affected nodes render", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<GraphUpdateView
|
||||
@@ -337,11 +614,33 @@ describe("graph-backed UI rendering", () => {
|
||||
expect(html).toContain("Quality deterioration");
|
||||
});
|
||||
|
||||
it("no fake next question appears", () => {
|
||||
it("renders validated next question when present", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<GraphUpdateView
|
||||
updateResult={{
|
||||
...makeUpdateSuccess({ newActiveUnknownNodeId: null }),
|
||||
...makeUpdateSuccess(),
|
||||
previousSituationGraph: makeGraphResult().situationGraph,
|
||||
}}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(html).toContain(
|
||||
"What evidence would clarify how the two observations were measured?",
|
||||
);
|
||||
});
|
||||
|
||||
it("no fake next question appears when there is none", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<GraphUpdateView
|
||||
updateResult={{
|
||||
...makeUpdateSuccess({
|
||||
newActiveUnknownNodeId: null,
|
||||
selectedQuestion: null,
|
||||
proposal: {
|
||||
...makeUpdateSuccess().proposal,
|
||||
selectedQuestion: null,
|
||||
},
|
||||
}),
|
||||
previousSituationGraph: makeGraphResult().situationGraph,
|
||||
}}
|
||||
/>,
|
||||
@@ -363,7 +662,85 @@ describe("graph-backed UI rendering", () => {
|
||||
expect(html).toContain("Previous active unknown");
|
||||
expect(html).toContain("Complaint rate denominator");
|
||||
expect(html).toContain("New active unknown");
|
||||
expect(html).toContain("Unknown node (ID: n-next-unknown)");
|
||||
expect(html).toContain(
|
||||
"Explanation for why revenue increased by 18%, but cash in the bank fell over the same period",
|
||||
);
|
||||
});
|
||||
|
||||
it("successful update renders prior and new state together", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<GraphUpdateView
|
||||
updateResult={{
|
||||
...makeUpdateSuccess(),
|
||||
previousSituationGraph: makeGraphResult().situationGraph,
|
||||
}}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(html).toContain("Previous active unknown");
|
||||
expect(html).toContain("Resolved unknowns");
|
||||
expect(html).toContain("Newly surfaced unknowns");
|
||||
expect(html).toContain("New active unknown");
|
||||
expect(html).toContain("Next question");
|
||||
expect(html).toContain(
|
||||
"What evidence would clarify how the two observations were measured?",
|
||||
);
|
||||
});
|
||||
|
||||
it("update view shows comparability progression without raw ids in the normal view", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<GraphUpdateView
|
||||
updateResult={{
|
||||
...makeUpdateSuccess({
|
||||
selectedQuestion: {
|
||||
nodeId: "n-next-unknown",
|
||||
question:
|
||||
"What evidence would clarify timing or measurement basis?",
|
||||
reason: "A broad follow-up is now justified.",
|
||||
},
|
||||
}),
|
||||
previousSituationGraph: makeGraphResult().situationGraph,
|
||||
}}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(html).toContain("Comparability:");
|
||||
expect(html).toContain("uncertain → confirmed");
|
||||
expect(html).toContain("Relationship status:");
|
||||
expect(html).toContain("insufficient_information");
|
||||
expect(html).toContain("Reasoning stages:");
|
||||
expect(html).toContain("comparability: confirmed");
|
||||
expect(html).toContain("relationship: insufficient_information");
|
||||
expect(html).toContain(
|
||||
"What evidence would clarify how the two observations were measured?",
|
||||
);
|
||||
expect(html).not.toContain("reasoning:comparability");
|
||||
});
|
||||
|
||||
it("situation graph marks newly surfaced and active unknowns", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<SituationGraphView
|
||||
situationGraph={makeUpdateSuccess().updatedSituationGraph}
|
||||
selectedQuestion={makeUpdateSuccess().selectedQuestion}
|
||||
newlySurfacedNodeIds={["n-next-unknown"]}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(html).toContain("newly surfaced unknown");
|
||||
expect(html).toContain("active unknown");
|
||||
expect(html).toContain("resolved unknown");
|
||||
});
|
||||
|
||||
it("disabled follow-up form is shown only as prototype limitation", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<GraphUpdateView
|
||||
updateResult={makeUpdateSuccess()}
|
||||
/>,
|
||||
);
|
||||
|
||||
expect(html).toContain(
|
||||
"What evidence would clarify how the two observations were measured?",
|
||||
);
|
||||
});
|
||||
|
||||
it("raw ids remain only in collapsed proposal details", () => {
|
||||
@@ -418,6 +795,30 @@ describe("graph-backed UI rendering", () => {
|
||||
expect(html).toContain("bad proposal");
|
||||
});
|
||||
|
||||
it("failed update does not fabricate history", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<>
|
||||
<UpdateErrorPanel
|
||||
updateError={{
|
||||
error: "Update case failed",
|
||||
errors: [
|
||||
'New unknown must be explicitly related to an answer-derived node: "nu_commercial_val"',
|
||||
],
|
||||
}}
|
||||
/>
|
||||
<GraphUpdateView updateResult={null} />
|
||||
</>,
|
||||
);
|
||||
|
||||
expect(html).toContain("Update error: Update case failed");
|
||||
expect(html).toContain("nu_commercial_val");
|
||||
expect(html).not.toContain("Previous active unknown");
|
||||
expect(html).not.toContain("Resolved unknowns");
|
||||
expect(html).not.toContain("Newly surfaced unknowns");
|
||||
expect(html).not.toContain("New active unknown");
|
||||
expect(html).not.toContain("Proposal details");
|
||||
});
|
||||
|
||||
it("proposal details remain collapsible", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<GraphUpdateView updateResult={makeUpdateSuccess()} />,
|
||||
@@ -425,4 +826,16 @@ describe("graph-backed UI rendering", () => {
|
||||
|
||||
expect(html).toContain("Proposal details");
|
||||
});
|
||||
|
||||
it("does not show the no-question fallback when a commercial follow-up sibling exists", () => {
|
||||
const html = renderToStaticMarkup(
|
||||
<GraphUpdateView updateResult={makeCommercialUpdateSuccess()} />,
|
||||
);
|
||||
|
||||
expect(html).toContain(
|
||||
"What makes you think other people experience this problem too?",
|
||||
);
|
||||
expect(html).toContain("New active unknown");
|
||||
expect(html).not.toContain("No next question selected yet.");
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user