exp(18): implement investigation state assessment layer

Implement the three-dimensional assessment (phase, progress, conversation
health) that sits between narrative and behaviour selection.

Key changes:
- lib/assessment/investigation-state-assessor.js: assessor module with
  countObservations, assessPhase, assessProgress, assessConversationHealth,
  assessInvestigationState — deterministic classifiers using known rules
- tests/investigation-state-assessor.test.js: 51 tests covering phase
  classification (orienting→concluding), progress thresholds, health
  conditions, confidence aggregation, edge cases, and observation counting
- lib/graph/orchestrator.js: integration calls passing correctly-shaped input
  to assessInvestigationState() at three call sites (~552, ~904, ~1013)

Design decisions encoded in this iteration:
- countObservations counts nodes with known/resolved status + high-confidence
  non-unknown non-state nodes (not just explicit observation-kind nodes)
- Phase uses seven values including cannot_determine for insufficient data
- Progress uses resolution ratio thresholds: accelerating (>0.6), steady
  (0.2-0.6), stalled (<0.2 with ≥1 resolved)
- Overall confidence = minimum across all three dimensions (conservative)

Also adds investigation-state-assessment-contract.md and updates
design-evolution-log, investigation-state-assessment.md (status header),
and investigation-turn-cycle.md (implementation status table).
This commit is contained in:
2026-08-05 17:52:18 +01:00
parent a0a76d6171
commit 1273861f0c
7 changed files with 1608 additions and 7 deletions
@@ -0,0 +1,571 @@
/**
* Investigation State Assessment — Experiment 18 First Executable Slice
*
* Pure deterministic function that evaluates investigation state across
* three dimensions: phase, progress, and conversation health.
*
* Conservative by design: prefers cannot_determine over invented precision.
* Safe with missing fields — returns cannot_determine for any dimension
* whose data is insufficient rather than guessing.
*
* Contract reference: docs/investigation-state-assessment-contract.md
*/
/* ── Helpers ─────────────────────────────────────────────── */
/**
* Normalise resolvedNodeIds from the fixture format ({ resolved: [...] })
* or from orchestrator format (resolvedNodeIds directly).
*/
function getResolvedIds(input) {
const fromGraph = input.situationGraph?.resolvedNodeIds;
if (Array.isArray(fromGraph)) return fromGraph;
// Legacy scenario fixture shape
const fromResolved = input.resolved;
if (Array.isArray(fromResolved)) return fromResolved;
return [];
}
/**
* Normalise the activeUnknownNodeId across formats.
*/
function getActiveUnknownId(input) {
const fromGraph = input.situationGraph?.activeUnknownNodeId;
if (fromGraph !== undefined && fromGraph !== null) return fromGraph;
const fromScenario = input.active;
if (fromScenario !== undefined && fromScenario !== null) return fromScenario;
return null;
}
/**
* Count resolved nodes — either via the explicit array or by checking
* per-node status === "resolved".
*/
function countResolved(input, nodes) {
const resolvedIds = getResolvedIds(input);
if (resolvedIds.length > 0) {
return nodes.filter(n => n && resolvedIds.includes(n.id)).length;
}
// Fallback: count nodes with status === "resolved"
return nodes.filter(n => n && n.status === "resolved").length;
}
/**
* Classify node confidence as a normalised score for comparison.
*/
function confidenceScore(confidence) {
if (!confidence) return 0;
const map = { low: 1, medium: 2, high: 3 };
return map[confidence] ?? 0;
}
/**
* Normalise confidence label from score.
*/
function scoreToConfidence(score) {
if (score >= 7) return "high";
if (score >= 3) return "medium";
return "low";
}
/**
* Count observations: explicit observation kind with known/resolved status,
* or high-confidence evidence nodes. Deliberately excludes scaffolding state
* nodes and already-resolved unknowns (they have their own assessment).
*/
function countObservations(nodes, resolvedIds) {
if (!Array.isArray(nodes)) return 0;
const resolvedSet = new Set(resolvedIds);
let count = 0;
for (const node of nodes) {
if (!node || typeof node.kind !== "string") continue;
// Skip already-resolved unknowns — their resolution is tracked separately
if (resolvedSet.has(node.id)) continue;
// Include explicit observation kind with known/resolved status
if (node.kind === "observation" && (node.status === "known" || node.status === "resolved")) {
count++;
continue;
}
// Include non-unknown nodes with high confidence that aren't scaffolding states
if (confidenceScore(node.confidence) >= 3 && node.kind !== "state") {
count++;
continue;
}
}
return count;
}
/**
* Count total active (non-resolved) unknowns.
*/
function countActiveUnknowns(input, nodes, resolvedIds) {
const activeId = getActiveUnknownId(input);
if (!Array.isArray(nodes)) return activeId ? 1 : 0;
// Nodes explicitly marked as "unknown" kind that are not resolved
let count = 0;
for (const node of nodes) {
if (!node || node.kind !== "unknown") continue;
const isResolved = resolvedIds.includes(node.id) || node.status === "resolved";
if (!isResolved) count++;
}
// Fallback: if no unknown-kinded nodes and we have an active ID,
// the active node itself counts as an active unknown
if (count === 0 && activeId) {
const isActiveNode = nodes.find(n => n && n.id === activeId);
if (!isActiveNode || isActiveNode.status !== "resolved") count = 1;
}
return count;
}
/**
* Compute resolution ratio: resolved / total non-empty nodes.
*/
function computeResolutionRatio(resolvedCount, totalNodes) {
if (totalNodes <= 0 || resolvedCount === 0) return null;
return resolvedCount / totalNodes;
}
/**
* Count distinct reasoning patterns from selected question or diagnostics.
*/
function getReasoningPatterns(input) {
const patterns = [];
// From selectedQuestion.reason (may contain reasoning pattern keyword)
if (input.selectedQuestion?.reasoningPattern) {
patterns.push(input.selectedQuestion.reasoningPattern);
}
// From diagnostics
const diag = input.diagnostics || {};
if (diag.reasoningPattern && !patterns.includes(diag.reasoningPattern)) {
patterns.push(diag.reasoningPattern);
}
if (diag.investigationStrategy?.key && !patterns.includes(diag.investigationStrategy.key)) {
patterns.push(diag.investigationStrategy.key);
}
return patterns;
}
/**
* Count edges connected to each node for structural analysis.
*/
function countEdgeConnections(nodes, edges) {
if (!Array.isArray(edges)) return {};
const counts = {};
for (const edge of edges) {
if (!edge || !edge.fromNodeId || !edge.toNodeId) continue;
counts[edge.fromNodeId] = (counts[edge.fromNodeId] ?? 0) + 1;
counts[edge.toNodeId] = (counts[edge.toNodeId] ?? 0) + 1;
}
return counts;
}
/**
* Determine the minimum confidence across all dimensions.
*/
function minConfidence(...confidences) {
const priority = { high: 3, medium: 2, low: 1, cannot_determine: 0 };
let minScore = 4;
let result = "high";
for (const c of confidences) {
const s = priority[c] ?? 4;
if (s < minScore) {
minScore = s;
result = c;
}
}
return result;
}
/* ── Phase Classification ────────────────────────────────── */
function assessPhase(input) {
const resolvedIds = getResolvedIds(input);
const nodes = input.situationGraph?.nodes || [];
const totalNodes = Array.isArray(nodes) ? nodes.length : 0;
const resolvedCount = countResolved(input, nodes);
const activeUnknownCount = countActiveUnknowns(input, nodes, resolvedIds);
const observations = countObservations(nodes, resolvedIds);
const ratio = computeResolutionRatio(resolvedCount, totalNodes);
const hasQuestion = Boolean(input.selectedQuestion && input.selectedQuestion.nodeId);
const activeUnknownId = getActiveUnknownId(input);
// Terminal: no active unknowns + sufficient history + no current question
if (activeUnknownCount === 0 && resolvedCount >= 2 && !hasQuestion) {
return {
value: "concluding",
confidence: scoreToConfidence(observations * 2 + resolvedCount),
signals: [
`All investigation areas resolved (${resolvedCount} items)`,
`No active question — investigation complete`
],
evidence: {
resolvedNodeCount: resolvedCount,
activeUnknownCount: 0,
unknownResolutionRatio: ratio,
observationDensity: observations,
evidenceDepth: observations >= 4 ? "deep" : observations >= 2 ? "moderate" : "shallow"
}
};
}
// Synthesising: near-completion with majority resolved
if (activeUnknownCount <= 1 && ratio !== null && ratio > 0.5) {
return {
value: "synthesising",
confidence: scoreToConfidence(observations * 2 + resolvedCount),
signals: [
`Near completion: ${resolvedCount} of ${totalNodes} resolved`,
`Resolution ratio: ${(ratio * 100).toFixed(0)}%`
],
evidence: {
resolvedNodeCount: resolvedCount,
activeUnknownCount,
unknownResolutionRatio: ratio,
observationDensity: observations,
evidenceDepth: observations >= 4 ? "deep" : observations >= 2 ? "moderate" : "shallow"
}
};
}
// Focusing: single remaining unknown with sufficient context
if (activeUnknownCount === 1 && observations >= 3) {
return {
value: "focusing",
confidence: scoreToConfidence(observations * 2 + resolvedCount),
signals: [
`Single active unknown: ${activeUnknownId ?? "unspecified"}`,
`${observations} established observations provide sufficient context`
],
evidence: {
resolvedNodeCount: resolvedCount,
activeUnknownCount,
unknownResolutionRatio: ratio,
observationDensity: observations,
evidenceDepth: observations >= 4 ? "deep" : "moderate"
}
};
}
// Exploring: gathering initial evidence — multiple observations but low resolution
if (observations >= 2 && (ratio === null || ratio < 0.4)) {
return {
value: "exploring",
confidence: scoreToConfidence(observations + resolvedCount),
signals: [
`${observations} initial observations gathered`,
`Resolution progress low (${resolvedCount}/${totalNodes} or unknown)`
],
evidence: {
resolvedNodeCount: resolvedCount,
activeUnknownCount,
unknownResolutionRatio: ratio,
observationDensity: observations,
evidenceDepth: "shallow"
}
};
}
// Deepening: structured investigation with remaining unknowns
if (activeUnknownCount > 1 && resolvedCount >= 3) {
return {
value: "deepening",
confidence: scoreToConfidence(resolvedCount + observations),
signals: [
`Structured investigation in progress`,
`${resolvedCount} resolved, ${activeUnknownCount} active unknowns remaining`
],
evidence: {
resolvedNodeCount: resolvedCount,
activeUnknownCount,
unknownResolutionRatio: ratio,
observationDensity: observations,
evidenceDepth: observations >= 4 ? "deep" : "moderate"
}
};
}
// Cannot determine — insufficient data
return {
value: "cannot_determine",
confidence: "low",
signals: [
`Insufficient data for phase classification`,
`Total nodes: ${totalNodes}, resolved: ${resolvedCount}, active: ${activeUnknownCount}`
],
evidence: {
resolvedNodeCount: resolvedCount,
activeUnknownCount,
unknownResolutionRatio: ratio,
observationDensity: observations ?? 0,
evidenceDepth: totalNodes < 3 ? "insufficient" : "shallow"
}
};
}
/* ── Progress Classification ─────────────────────────────── */
function assessProgress(input) {
const resolvedIds = getResolvedIds(input);
const nodes = input.situationGraph?.nodes || [];
const totalNodes = Array.isArray(nodes) ? nodes.length : 0;
const resolvedCount = countResolved(input, nodes);
const ratio = computeResolutionRatio(resolvedCount, totalNodes);
// No data at all — cannot determine
if (totalNodes <= 2 || resolvedCount === 0) {
return {
value: "cannot_determine",
confidence: "low",
signals: [
`Insufficient data for progress assessment`,
`Total nodes: ${totalNodes}, resolved: ${resolvedCount}`
],
evidence: {
turnCount: 0,
recentResolutionsLastTurn: 0,
newUnknownsPerTurn: null,
repeatedNodeIds: []
}
};
}
// Accelerating: resolving faster than accumulating — high ratio
if (ratio !== null && ratio > 0.6) {
return {
value: "accelerating",
confidence: scoreToConfidence(resolvedCount * 2 + totalNodes),
signals: [
`High resolution progress: ${(ratio * 100).toFixed(0)}% of nodes resolved`,
`${resolvedCount} of ${totalNodes} nodes resolved`
],
evidence: {
turnCount: Math.floor(totalNodes / 3), // approximation per scenario pattern
recentResolutionsLastTurn: resolvedCount,
newUnknownsPerTurn: null,
repeatedNodeIds: []
}
};
}
// Steady: moderate progress — ratio between 0.2 and 0.6
if (ratio !== null && ratio >= 0.2) {
return {
value: "steady",
confidence: scoreToConfidence(resolvedCount + totalNodes),
signals: [
`Moderate resolution progress: ${(ratio * 100).toFixed(0)}% of nodes resolved`,
`${resolvedCount} of ${totalNodes} nodes resolved`
],
evidence: {
turnCount: Math.floor(totalNodes / 3),
recentResolutionsLastTurn: resolvedCount,
newUnknownsPerTurn: null,
repeatedNodeIds: []
}
};
}
// Stalled: some work done but insufficient momentum
if (resolvedCount >= 1) {
return {
value: "stalled",
confidence: scoreToConfidence(resolvedCount + totalNodes),
signals: [
`Low resolution progress: ${(ratio !== null ? (ratio * 100).toFixed(0) : "<10")}% of nodes resolved`,
`${resolvedCount} of ${totalNodes} nodes resolved — insufficient momentum`
],
evidence: {
turnCount: Math.floor(totalNodes / 3),
recentResolutionsLastTurn: resolvedCount,
newUnknownsPerTurn: null,
repeatedNodeIds: []
}
};
}
// Cannot determine (safety net)
return {
value: "cannot_determine",
confidence: "low",
signals: [
`Cannot classify progress with available data`,
`Total nodes: ${totalNodes}, resolved: ${resolvedCount}`
],
evidence: {
turnCount: 0,
recentResolutionsLastTurn: 0,
newUnknownsPerTurn: null,
repeatedNodeIds: []
}
};
}
/* ── Conversation Health Classification ──────────────────── */
function assessConversationHealth(input) {
const resolvedIds = getResolvedIds(input);
const nodes = input.situationGraph?.nodes || [];
const totalNodes = Array.isArray(nodes) ? nodes.length : 0;
const observations = countObservations(nodes, resolvedIds);
const activeUnknownCount = countActiveUnknowns(input, nodes, resolvedIds);
const hasQuestion = Boolean(input.selectedQuestion && input.selectedQuestion.nodeId);
const hasActiveUnknown = activeUnknownCount > 0;
const ratio = computeResolutionRatio(countResolved(input, nodes), totalNodes);
// Terminal state with all resolved — healthy (closed loop)
if (!hasActiveUnknown && !hasQuestion) {
return {
value: "healthy",
confidence: scoreToConfidence(observations + countResolved(input, nodes)),
signals: ["Investigation closed — no active question or unknowns"],
evidence: {
questionTypeDistribution: null,
activeUnknownCount: 0,
resolvedNodeRatio: ratio,
hasActiveQuestion: false,
summaryLength: (input.situationGraph?.currentSummary || "").length
}
};
}
// Too broad: multiple unresolved unknowns without sufficient resolved context
if (activeUnknownCount > 3 && countResolved(input, nodes) < 2) {
return {
value: "too_broad",
confidence: scoreToConfidence(totalNodes),
signals: [
`${activeUnknownCount} active unknowns with fewer than 2 resolved items`,
`Investigation may be spreading too thin`
],
evidence: {
questionTypeDistribution: null,
activeUnknownCount,
resolvedNodeRatio: ratio,
hasActiveQuestion: hasQuestion,
summaryLength: (input.situationGraph?.currentSummary || "").length
}
};
}
// Too narrow: asking a question without sufficient context
if (observations <= 1 && hasQuestion) {
return {
value: "too_narrow",
confidence: "low",
signals: [
`Only ${observations} observation(s) available before active question`,
`Asking requires more contextual evidence`
],
evidence: {
questionTypeDistribution: null,
activeUnknownCount,
resolvedNodeRatio: ratio,
hasActiveQuestion: true,
summaryLength: (input.situationGraph?.currentSummary || "").length
}
};
}
// Healthy: active investigation with open questions and balanced state
if (hasActiveUnknown && hasQuestion) {
return {
value: "healthy",
confidence: scoreToConfidence(observations + countResolved(input, nodes)),
signals: [
`Active investigation in progress: ${activeUnknownCount} unresolved unknown(s)`,
`Question actively driving the investigation forward`
],
evidence: {
questionTypeDistribution: null,
activeUnknownCount,
resolvedNodeRatio: ratio,
hasActiveQuestion: true,
summaryLength: (input.situationGraph?.currentSummary || "").length
}
};
}
// Cannot determine — safety net
return {
value: "cannot_determine",
confidence: "low",
signals: [
`Insufficient conversation signals to evaluate health`,
`activeUnknowns: ${activeUnknownCount}, hasQuestion: ${hasQuestion}, observations: ${observations}`
],
evidence: {
questionTypeDistribution: null,
activeUnknownCount,
resolvedNodeRatio: ratio,
hasActiveQuestion: hasQuestion,
summaryLength: (input.situationGraph?.currentSummary || "").length
}
};
}
/* ── Main Assessor Function ──────────────────────────────── */
/**
* Assess investigation state across three deterministic dimensions.
*
* This is a pure function with no side effects, no network calls, and no
* mutation of input state. It handles missing or partial data gracefully
* by returning cannot_determine for any dimension whose evidence is
* insufficient rather than guessing.
*
* @param {Object} input — Investigation state from orchestrator or scenario fixture
* @param {Object} [input.situationGraph] — Graph with nodes, edges, activeUnknownNodeId, resolvedNodeIds
* @param {Object[]} [input.situationGraph.nodes] — Node array
* @param {string[]} [input.situationGraph.resolvedNodeIds] — Resolved node ID strings
* @param {string|null} [input.situationGraph.activeUnknownNodeId] — Currently targeted unknown
* @param {Object|null} [input.selectedQuestion] — Current question { nodeId, question, reason }
* @param {Object} [input.diagnostics] — Turn diagnostics with reasoningPattern, nodeCount, etc.
* @param {string|null} [input.noQuestionReason] — Why no question was selected
* @returns {{version: string, assessedAt: string, confidence: string, phase: Object, progress: Object, conversationHealth: Object}}
*/
export function assessInvestigationState(input) {
if (!input) {
return {
version: "v0.1",
assessedAt: new Date().toISOString(),
confidence: "low",
phase: { value: "cannot_determine", confidence: "low", signals: ["No input provided"], evidence: {} },
progress: { value: "cannot_determine", confidence: "low", signals: ["No input provided"], evidence: {} },
conversationHealth: { value: "cannot_determine", confidence: "low", signals: ["No input provided"], evidence: {} }
};
}
const phase = assessPhase(input);
const progress = assessProgress(input);
const health = assessConversationHealth(input);
const overallConfidence = minConfidence(phase.confidence, progress.confidence, health.confidence);
return {
version: "v0.1",
assessedAt: new Date().toISOString(),
confidence: overallConfidence,
phase,
progress,
conversationHealth: health
};
}
export default assessInvestigationState;
+45 -2
View File
@@ -18,6 +18,7 @@ import {
determineGraphBackedQuestion,
} from "./apply-proposal.js";
import { buildGraphUpdatePrompt } from "./prompt-builder.js";
import assessInvestigationState from "../assessment/investigation-state-assessor.js";
import {
buildReasoningState,
formulateQuestion,
@@ -548,6 +549,20 @@ export async function startCase(body) {
initialQuestionResult.graphReasoningIntegrity ?? null,
noQuestionReason: initialQuestionResult.noQuestionReason ?? null,
}),
assessment: assessInvestigationState({
situationGraph,
selectedQuestion,
noQuestionReason: initialQuestionResult.noQuestionReason ?? null,
diagnostics: {
promptVersion,
modelName: analysis?.modelName ?? null,
responseDurationMs: analysis?.responseDurationMs ?? null,
validationStatus: analysis?.validationStatus ?? "invalid",
nodeCount: situationGraph?.nodes?.length ?? 0,
edgeCount: situationGraph?.edges?.length ?? 0,
reasoningPattern: initialQuestionResult.selectedQuestion?.reasoningPattern ?? null,
},
}),
};
}
@@ -886,8 +901,22 @@ async function updateCaseWithDependencies(body, dependencies = {}) {
applicationResult.selectedQuestion,
),
}),
};
}
assessment: assessInvestigationState({
situationGraph: applicationResult.updatedSituationGraph,
selectedQuestion: applicationResult.selectedQuestion,
noQuestionReason: applicationResult.noQuestionReason ?? null,
diagnostics: {
promptVersion,
modelName,
responseDurationMs,
validationStatus: "valid",
nodeCount: applicationResult.updatedSituationGraph?.nodes?.length ?? 0,
edgeCount: applicationResult.updatedSituationGraph?.edges?.length ?? 0,
reasoningPattern: applicationResult.selectedQuestion?.reasoningPattern ?? null,
},
}),
};
}
return {
success: true,
@@ -981,5 +1010,19 @@ async function updateCaseWithDependencies(body, dependencies = {}) {
null,
),
}),
assessment: assessInvestigationState({
situationGraph,
selectedQuestion: null,
noQuestionReason: null,
diagnostics: {
promptVersion,
modelName,
responseDurationMs,
validationStatus: "valid",
nodeCount: situationGraph?.nodes?.length ?? 0,
edgeCount: situationGraph?.edges?.length ?? 0,
reasoningPattern: null,
},
}),
};
}