test(confidence-engine): add progressed investigation experiment fixture
Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
Claude Code
@@ -0,0 +1,189 @@
|
||||
/**
|
||||
* Progressed investigation fixture apparatus verification.
|
||||
*
|
||||
* Proves that the canonical progressed-investigation fixture:
|
||||
* 1. loads successfully as valid JSON
|
||||
* 2. contains >= 1 admitted known/supported evidence item
|
||||
* 3. contains >= 1 unresolved Open Question (kind=unknown, status=unknown)
|
||||
* 4. the unresolved Open Question is not admitted evidence
|
||||
* 5. at least one eligible Finding exists (null disposition)
|
||||
* 6. unresolved Open Question is not represented as an eligible Finding
|
||||
* 7. resolvedNodeIds does NOT include any unknown node id
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from "vitest";
|
||||
import fixture from "../fixtures/progressed-investigation-manufacturing.json";
|
||||
|
||||
describe("Progressed investigation fixture apparatus", () => {
|
||||
/* ── 1. Fixture loads successfully ─────────────────── */
|
||||
|
||||
it("loads as valid JSON with required top-level keys", () => {
|
||||
expect(fixture).toHaveProperty("success", true);
|
||||
expect(fixture).toHaveProperty("stage", "update_applied");
|
||||
expect(fixture).toHaveProperty("situationGraph");
|
||||
expect(fixture.situationGraph).toHaveProperty("nodes");
|
||||
expect(fixture.situationGraph).toHaveProperty("edges");
|
||||
expect(fixture.situationGraph).toHaveProperty("activeUnknownNodeId");
|
||||
expect(fixture.situationGraph).toHaveProperty("resolvedNodeIds");
|
||||
expect(fixture.situationGraph).toHaveProperty("centralStatement");
|
||||
});
|
||||
|
||||
/* ── 2. Contains admitted evidence (>= 1 supported/known node) ─── */
|
||||
|
||||
it("contains at least one admitted known or supported evidence node", () => {
|
||||
const admitted = fixture.situationGraph.nodes.filter(
|
||||
(n) => n.status === "supported" || n.status === "known"
|
||||
);
|
||||
expect(admitted.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("admitted nodes are not unknown kind", () => {
|
||||
const admitted = fixture.situationGraph.nodes.filter(
|
||||
(n) => ["supported", "known"].includes(n.status)
|
||||
);
|
||||
for (const node of admitted) {
|
||||
expect(node.kind).not.toBe("unknown");
|
||||
}
|
||||
});
|
||||
|
||||
/* ── 3. Contains unresolved Open Question (>= 1) ─────── */
|
||||
|
||||
it("contains at least one unresolved Open Question node", () => {
|
||||
const unresolved = fixture.situationGraph.nodes.filter(
|
||||
(n) => n.kind === "unknown" && n.status === "unknown"
|
||||
);
|
||||
expect(unresolved.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
/* ── 4. Unresolved Open Question is not admitted evidence ─── */
|
||||
|
||||
it("no node is simultaneously unresolved unknown AND admitted evidence", () => {
|
||||
const allNodes = fixture.situationGraph.nodes;
|
||||
for (const node of allNodes) {
|
||||
if (node.kind === "unknown" && node.status === "unknown") {
|
||||
// An unresolved unknown must not have admitted status
|
||||
expect(["supported", "known"]).not.toContain(node.status);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("unresolved and admitted node ID sets are disjoint", () => {
|
||||
const unresolvedIds = new Set(
|
||||
fixture.situationGraph.nodes
|
||||
.filter((n) => n.kind === "unknown" && n.status === "unknown")
|
||||
.map((n) => n.id)
|
||||
);
|
||||
const admittedIds = new Set(
|
||||
fixture.situationGraph.nodes
|
||||
.filter((n) => ["supported", "known"].includes(n.status))
|
||||
.map((n) => n.id)
|
||||
);
|
||||
for (const id of unresolvedIds) {
|
||||
expect(admittedIds.has(id)).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
it("unresolved unknown nodes are distinct from admitted evidence", () => {
|
||||
const unresolvedIds = new Set(
|
||||
fixture.situationGraph.nodes
|
||||
.filter((n) => n.kind === "unknown" && n.status === "unknown")
|
||||
.map((n) => n.id)
|
||||
);
|
||||
const admittedIds = new Set(
|
||||
fixture.situationGraph.nodes
|
||||
.filter((n) => ["supported", "known"].includes(n.status))
|
||||
.map((n) => n.id)
|
||||
);
|
||||
for (const id of unresolvedIds) {
|
||||
expect(admittedIds.has(id)).toBe(false);
|
||||
}
|
||||
});
|
||||
|
||||
/* ── 5. Unresolved question excluded from resolvedNodeIds ─── */
|
||||
|
||||
it("resolvedNodeIds does not contain any unresolved unknown node id", () => {
|
||||
const unresolvedIds = fixture.situationGraph.nodes
|
||||
.filter((n) => n.kind === "unknown" && n.status === "unknown")
|
||||
.map((n) => n.id);
|
||||
for (const nodeId of unresolvedIds) {
|
||||
expect(fixture.situationGraph.resolvedNodeIds).not.toContain(nodeId);
|
||||
}
|
||||
});
|
||||
|
||||
/* ── 6. Eligible Finding exists ─────────────────────── */
|
||||
|
||||
it("findings array exists and has eligible items", () => {
|
||||
expect(fixture.findings).toBeDefined();
|
||||
expect(Array.isArray(fixture.findings)).toBe(true);
|
||||
const eligible = fixture.findings.filter(
|
||||
(f) => f.status === "provisional" && f.userDisposition === null
|
||||
);
|
||||
expect(eligible.length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
/* ── 7. Unresolved Open Question is not represented as a Finding ─── */
|
||||
|
||||
it("unresolved unknown nodes are not mapped to findings propositions", () => {
|
||||
const unresolvedNodeIds = new Set(
|
||||
fixture.situationGraph.nodes
|
||||
.filter((n) => n.kind === "unknown" && n.status === "unknown")
|
||||
.map((n) => n.id)
|
||||
);
|
||||
// originTargetNodeId links findings to unknown nodes
|
||||
// A finding whose originatingTargetNodeId is an unresolved unknown is fine
|
||||
// but the proposition should not BE the same as the node label
|
||||
const eligibleFindings = fixture.findings.filter(
|
||||
(f) => f.status === "provisional" && f.userDisposition === null
|
||||
);
|
||||
const unresolvedNodes = fixture.situationGraph.nodes.filter(
|
||||
(n) => n.kind === "unknown" && n.status === "unknown"
|
||||
);
|
||||
|
||||
for (const finding of eligibleFindings) {
|
||||
// The proposition is an admitted evidence item, not the unknown itself
|
||||
expect(finding.proposition).not.toContain("Whether");
|
||||
// Verify the originating node is NOT resolved — it remains open
|
||||
const targetNode = unresolvedNodes.find(
|
||||
(n) => n.id === finding.originatingTargetNodeId
|
||||
);
|
||||
if (targetNode) {
|
||||
expect(targetNode.status).toBe("unknown");
|
||||
expect(targetNode.kind).toBe("unknown");
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
/* ── 8. Canonical shape integrity ──────────────────── */
|
||||
|
||||
it("activeUnknownNodeId references a valid unresolved unknown node", () => {
|
||||
const unresolvedIds = fixture.situationGraph.nodes
|
||||
.filter((n) => n.kind === "unknown" && n.status === "unknown")
|
||||
.map((n) => n.id);
|
||||
expect(unresolvedIds).toContain(fixture.situationGraph.activeUnknownNodeId);
|
||||
});
|
||||
|
||||
it("all nodes pass the canonical situationNode schema invariants", () => {
|
||||
for (const node of fixture.situationGraph.nodes) {
|
||||
expect(node).toHaveProperty("id");
|
||||
expect(node).toHaveProperty("label");
|
||||
expect(node).toHaveProperty("description");
|
||||
expect(["observation","reported_claim","metric","state","transition","relationship","assumption","unknown","conclusion","option"]).toContain(node.kind);
|
||||
expect(["known","unknown","provisional","supported","weakened","contradicted","resolved"]).toContain(node.status);
|
||||
expect(["low","medium","high"]).toContain(node.confidence);
|
||||
expect(Array.isArray(node.evidenceIds)).toBe(true);
|
||||
expect(Array.isArray(node.dependsOn)).toBe(true);
|
||||
expect(Array.isArray(node.affects)).toBe(true);
|
||||
}
|
||||
});
|
||||
|
||||
it("all edges reference valid nodes", () => {
|
||||
const nodeIds = new Set(fixture.situationGraph.nodes.map((n) => n.id));
|
||||
for (const edge of fixture.situationGraph.edges) {
|
||||
expect(edge).toHaveProperty("id");
|
||||
expect(nodeIds.has(edge.fromNodeId)).toBe(true);
|
||||
expect(nodeIds.has(edge.toNodeId)).toBe(true);
|
||||
expect(edge).toHaveProperty("relationship");
|
||||
expect(edge).toHaveProperty("confidence");
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,241 @@
|
||||
{
|
||||
"success": true,
|
||||
"stage": "update_applied",
|
||||
"situationGraph": {
|
||||
"centralStatement": "Complaint counts increased while production also increased.",
|
||||
"nodes": [
|
||||
{
|
||||
"id": "n-obs-1",
|
||||
"label": "Customer complaints rose by 35% over the last six months",
|
||||
"description": "Customer complaints rose by 35% over the last six months",
|
||||
"kind": "observation",
|
||||
"status": "supported",
|
||||
"confidence": "high",
|
||||
"value": null,
|
||||
"unit": null,
|
||||
"evidenceIds": [
|
||||
"complaint-data"
|
||||
],
|
||||
"dependsOn": [],
|
||||
"affects": [],
|
||||
"parentId": null,
|
||||
"childIds": []
|
||||
},
|
||||
{
|
||||
"id": "n-obs-2",
|
||||
"label": "Production volume increased by 40% in the same period",
|
||||
"description": "Production volume increased by 40% in the same period",
|
||||
"kind": "observation",
|
||||
"status": "supported",
|
||||
"confidence": "high",
|
||||
"value": null,
|
||||
"unit": null,
|
||||
"evidenceIds": [
|
||||
"production-data"
|
||||
],
|
||||
"dependsOn": [],
|
||||
"affects": [],
|
||||
"parentId": null,
|
||||
"childIds": []
|
||||
},
|
||||
{
|
||||
"id": "n-obs-3",
|
||||
"label": "A new CRM tagging system was introduced three months ago",
|
||||
"description": "A new CRM tagging system was introduced three months ago, coinciding with the complaint category changes",
|
||||
"kind": "observation",
|
||||
"status": "known",
|
||||
"confidence": "high",
|
||||
"value": null,
|
||||
"unit": null,
|
||||
"evidenceIds": [
|
||||
"crm-audit"
|
||||
],
|
||||
"dependsOn": [],
|
||||
"affects": [],
|
||||
"parentId": null,
|
||||
"childIds": []
|
||||
},
|
||||
{
|
||||
"id": "n-unknown-1",
|
||||
"label": "Whether complaints increased per-unit or proportionally with production volume",
|
||||
"description": "Need to know the complaint rate normalized by production volume to determine if quality deteriorated or if growth explains the rise",
|
||||
"kind": "unknown",
|
||||
"status": "unknown",
|
||||
"confidence": "high",
|
||||
"value": null,
|
||||
"unit": null,
|
||||
"evidenceIds": [],
|
||||
"dependsOn": [],
|
||||
"affects": [
|
||||
"n-conclusion"
|
||||
],
|
||||
"parentId": null,
|
||||
"childIds": []
|
||||
},
|
||||
{
|
||||
"id": "n-unknown-2",
|
||||
"label": "Whether the new CRM tagging system changed what gets counted as a complaint",
|
||||
"description": "The complaint category changes may be an artifact of the new CRM system rather than genuine quality change",
|
||||
"kind": "unknown",
|
||||
"status": "unknown",
|
||||
"confidence": "medium",
|
||||
"value": null,
|
||||
"unit": null,
|
||||
"evidenceIds": [],
|
||||
"dependsOn": [],
|
||||
"affects": [
|
||||
"n-unknown-1"
|
||||
],
|
||||
"parentId": null,
|
||||
"childIds": []
|
||||
},
|
||||
{
|
||||
"id": "n-conclusion",
|
||||
"label": "Whether automated quality inspection investment is justified",
|
||||
"description": "Decision node for whether to spend about £120,000 on automated quality inspection",
|
||||
"kind": "state",
|
||||
"status": "provisional",
|
||||
"confidence": "medium",
|
||||
"value": null,
|
||||
"unit": null,
|
||||
"evidenceIds": [],
|
||||
"dependsOn": [
|
||||
"n-unknown-1"
|
||||
],
|
||||
"affects": [],
|
||||
"parentId": null,
|
||||
"childIds": []
|
||||
},
|
||||
{
|
||||
"id": "nz92pkx-summary",
|
||||
"label": "Complaint counts increased while production also increased.",
|
||||
"description": "Summary of the situation from the scenario text",
|
||||
"kind": "state",
|
||||
"status": "provisional",
|
||||
"confidence": "medium",
|
||||
"value": null,
|
||||
"unit": null,
|
||||
"evidenceIds": [],
|
||||
"dependsOn": [
|
||||
"n-obs-1",
|
||||
"n-obs-2",
|
||||
"n-obs-3"
|
||||
],
|
||||
"affects": [
|
||||
"n-conclusion"
|
||||
],
|
||||
"parentId": null,
|
||||
"childIds": []
|
||||
}
|
||||
],
|
||||
"edges": [
|
||||
{
|
||||
"id": "e-evidence-n-obs-1",
|
||||
"fromNodeId": "n-obs-1",
|
||||
"toNodeId": "nz92pkx-summary",
|
||||
"relationship": "supports",
|
||||
"confidence": "high",
|
||||
"description": "Customer complaints rose by 35% over the last six months supports the summary"
|
||||
},
|
||||
{
|
||||
"id": "e-evidence-n-obs-2",
|
||||
"fromNodeId": "n-obs-2",
|
||||
"toNodeId": "nz92pkx-summary",
|
||||
"relationship": "supports",
|
||||
"confidence": "high",
|
||||
"description": "Production volume increased by 40% in the same period supports the summary"
|
||||
},
|
||||
{
|
||||
"id": "e-evidence-n-obs-3",
|
||||
"fromNodeId": "n-obs-3",
|
||||
"toNodeId": "nz92pkx-summary",
|
||||
"relationship": "supports",
|
||||
"confidence": "high",
|
||||
"description": "A new CRM tagging system was introduced three months ago supports the summary"
|
||||
},
|
||||
{
|
||||
"id": "e-unk-n-unknown-1",
|
||||
"fromNodeId": "n-unknown-1",
|
||||
"toNodeId": "nz92pkx-summary",
|
||||
"relationship": "depends_on",
|
||||
"confidence": "high",
|
||||
"description": "Whether complaints increased per-unit or proportionally with production volume is an unresolved factor for this situation"
|
||||
},
|
||||
{
|
||||
"id": "e-unk-n-unknown-2",
|
||||
"fromNodeId": "n-unknown-2",
|
||||
"toNodeId": "nz92pkx-summary",
|
||||
"relationship": "depends_on",
|
||||
"confidence": "medium",
|
||||
"description": "Whether the new CRM tagging system changed what gets counted as a complaint is an unresolved factor for this situation"
|
||||
},
|
||||
{
|
||||
"id": "e-unk-conclusion",
|
||||
"fromNodeId": "n-unknown-1",
|
||||
"toNodeId": "n-conclusion",
|
||||
"relationship": "depends_on",
|
||||
"confidence": "medium",
|
||||
"description": "Whether complaints increased per-unit or proportionally with production volume affects the investment decision"
|
||||
}
|
||||
],
|
||||
"activeUnknownNodeId": "n-unknown-1",
|
||||
"resolvedNodeIds": [],
|
||||
"currentSummary": "Nodes: 4 observation, 2 unknown, 2 state | Edges: 6 total | Unknowns: 2 unresolved"
|
||||
},
|
||||
"findings": [
|
||||
{
|
||||
"id": "finding-complaint-rise",
|
||||
"proposition": "Customer complaints have increased by 35% over the last six months while production volume grew by 40%.",
|
||||
"status": "provisional",
|
||||
"userDisposition": null,
|
||||
"originatingTargetNodeId": "n-unknown-1",
|
||||
"contributionId": "contrib-admission-1",
|
||||
"sourceObservation": "user_input"
|
||||
},
|
||||
{
|
||||
"id": "finding-crm-change",
|
||||
"proposition": "A new CRM tagging system was introduced three months ago, coinciding with complaint category changes.",
|
||||
"status": "provisional",
|
||||
"userDisposition": null,
|
||||
"originatingTargetNodeId": "n-unknown-2",
|
||||
"contributionId": "contrib-admission-2",
|
||||
"sourceObservation": "user_input"
|
||||
}
|
||||
],
|
||||
"selectedQuestion": {
|
||||
"nodeId": "n-unknown-1",
|
||||
"question": "What is the complaint rate normalized by production volume, and has it increased per unit?",
|
||||
"reason": "The admitted evidence shows absolute increases but does not establish whether quality deteriorated or growth explains the rise.",
|
||||
"strategy": "evidence_gathering",
|
||||
"investigationStrategy": {
|
||||
"key": "evidence_gathering",
|
||||
"reason": "Resolving the unknown requires normalized rate data.",
|
||||
"nodeId": "n-unknown-1",
|
||||
"nodeLabel": "Whether complaints increased per-unit or proportionally with production volume",
|
||||
"meaning": "whether complaints increased per-unit or proportionally with production volume",
|
||||
"actionPhrase": "deciding whether to spend about £120,000 on automated quality inspection now",
|
||||
"relatedNodeIds": [
|
||||
"n-conclusion"
|
||||
],
|
||||
"centralStatement": "Complaint counts increased while production also increased."
|
||||
},
|
||||
"reasoningPattern": "decision",
|
||||
"reasoningPatternReason": "Selected decision because the active unknown sits inside an investment justification context."
|
||||
},
|
||||
"diagnostics": {
|
||||
"promptVersion": "v0.4",
|
||||
"modelName": "configured-model",
|
||||
"validationStatus": "valid",
|
||||
"nodeCount": 6,
|
||||
"edgeCount": 6,
|
||||
"graphReferenceValidation": {
|
||||
"valid": true,
|
||||
"errors": []
|
||||
},
|
||||
"unknownSelectionExplanation": {
|
||||
"selectedNodeId": "n-unknown-1",
|
||||
"status": "selected",
|
||||
"tieType": "none"
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user