robbond/confidence-engine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
d7cb343838193f68aea51b99f3fe5f402cbeee66
confidence-engine/tests/behaviour-selection.counterfactual.test.js
T
robbond eee8c6b1e4 experiment: compare acknowledge priority alternatives 
Experiment 41 compared two passive alternatives for reducing Acknowledge dominance:

Variant A (priority reordering): evaluate Summarise/Pause before Acknowledge
- Converges on concluding→summarise and stalled→pause correctly
- Introduces false-positive summarise at long-investigation t3

Variant B (Acknowledge exclusions): gate Acknowledge via phase/progress/health
- Converges on the same two genuine changes without false-positives
- Recommended: cleaner boundaries, preserves Acknowledge for healthy focus states

Both variants produce identical results for 2 of 7 tested turns.
Variant A diverges at long-investigation t3 (focusing phase with resolvedNodeCount=3).
Variant B correctly preserves Acknowledge there via its exclusion list.

Test files:
- tests/behaviour-selection.counterfactual.test.js (44 tests, new)
No production code changed.
2026-08-06 17:19:02 +01:00

580 lines
45 KiB
JavaScript
Raw Blame History

/**
* Behaviour Selection Counterfactual — Experiment 41
*
* Compare two small, passive alternatives for reducing Acknowledge dominance:
*
* Variant A — evaluate Summarise/Pause before Acknowledge (priority reordering);
* Variant B — existing priority with Acknowledge exclusions (phase/progress/health gates).
*
* Production selector is NOT modified. Variants exist only in this test file.
*/
import { describe, it, expect } from "vitest";
import assessInvestigationState from "@/lib/assessment/investigation-state-assessor.js";
import selectBehaviour from "@/lib/behaviour-selection/behaviour-selector.js";
/* ═══════════════════════════════════════════════════
1. Minimal assessment builder (not mutated)
═══════════════════════════════════════════════════ */
function mkAssessment(opts = {}) {
return {
version: "v0.1",
assessedAt: new Date().toISOString(),
confidence: opts.overallConfidence || "medium",
phase: opts.phase ?? {
value: "cannot_determine",
confidence: "low",
signals: [],
evidence: { resolvedNodeCount: 0, activeUnknownCount: 0, unknownResolutionRatio: null, observationDensity: 0, evidenceDepth: "insufficient" }
},
progress: opts.progress ?? {
value: "cannot_determine",
confidence: "low",
signals: [],
evidence: { turnCount: 0, recentResolutionsLastTurn: 0, newUnknownsPerTurn: null, repeatedNodeIds: [] }
},
conversationHealth: opts.conversationHealth ?? {
value: "cannot_determine",
confidence: "low",
signals: [],
evidence: { questionTypeDistribution: null, activeUnknownCount: 0, resolvedNodeRatio: null, hasActiveQuestion: false, summaryLength: 0 }
}
};
}
/* ═══════════════════════════════════════════════════
2. Variant A — Specific behaviours before Acknowledge
═══════════════════════════════════════════════════ */
function selectSummariseV2(assessment) {
if (assessment.phase.value === "synthesising") return { behaviour: "summarise", confidence: "high", reason: "Investigation is in synthesising phase — connected observations have accumulated and a restatement of current understanding will compress without losing detail." };
if (assessment.phase.value === "concluding") return { behaviour: "summarise", confidence: "high", reason: "Investigation is concluding — a summary of resolved understanding provides closure anchor before the user decides next steps." };
if (assessment.phase.evidence?.resolvedNodeCount >= 3 && assessment.progress.value === "steady") return { behaviour: "summarise", confidence: "medium", reason: "Three or more items resolved with steady progress — enough accumulated understanding warrants a compression pass." };
return null;
}
function selectPauseV2(assessment) {
if (assessment.phase.value === "focusing" && assessment.progress.value === "stalled") return { behaviour: "pause", confidence: "high", reason: "Focusing phase with stalled progress — the investigation has reached a single active unknown but momentum has stopped. Hold space rather than pushing for more." };
if (assessment.conversationHealth.value === "user_overloaded") return { behaviour: "pause", confidence: "medium", reason: "User appears overloaded — reduce pressure by acknowledging progress before inviting further contribution." };
return null;
}
function selectAcknowledgeV2(assessment) {
if (assessment.conversationHealth.value === "healthy" && assessment.phase.confidence !== "low") return { behaviour: "acknowledge", confidence: "medium", reason: "Healthy conversation with established context — user provided useful information that warrants acknowledgment before introducing new uncertainty." };
return null;
}
function selectClarifyV2(assessment) {
if (assessment.conversationHealth.value === "too_broad") return { behaviour: "clarify", confidence: "high", reason: "Conversation health is too broad — investigation may be spreading too thin. Narrow focus through a specific clarification question." };
if (assessment.phase.value === "orienting" && assessment.phase.evidence?.observationDensity < 3) return { behaviour: "clarify", confidence: "medium", reason: "Investigation is in orienting phase with insufficient observations (< 3). A targeted clarification question will anchor the starting point." };
return null;
}
function selectVariantA(assessment) {
if (!assessment || !assessment.conversationHealth || !assessment.phase) return { behaviour: "continue", confidence: "low", reason: "No assessment available — defaulting to continue." };
const summarise = selectSummariseV2(assessment); if (summarise) return { ...summarise, priority: 1 };
const pause = selectPauseV2(assessment); if (pause) return { ...pause, priority: 2 };
const clarify = selectClarifyV2(assessment); if (clarify) return { ...clarify, priority: 3 };
const acknowledge = selectAcknowledgeV2(assessment); if (acknowledge) return { ...acknowledge, priority: 4 };
return { behaviour: "continue", confidence: "low", reason: "No explicit rule matched — defaulting to continue." };
}
/* ═══════════════════════════════════════════════════
3. Variant B — Existing priority with Acknowledge exclusions
═══════════════════════════════════════════════════ */
function isAcknowledgeExcluded(assessment) {
if (["synthesising", "concluding"].includes(assessment.phase.value)) return true;
if (assessment.progress.value === "stalled") return true;
if (assessment.conversationHealth.value === "user_overloaded") return true;
return false;
}
function selectVariantB(assessment) {
if (!assessment || !assessment.conversationHealth || !assessment.phase) return { behaviour: "continue", confidence: "low", reason: "No assessment available — defaulting to continue." };
const acknowledge = selectAcknowledgeV2(assessment);
if (acknowledge && !isAcknowledgeExcluded(assessment)) return { ...acknowledge, priority: 1 };
const clarify = selectClarifyV2(assessment); if (clarify) return { ...clarify, priority: 2 };
const summarise = selectSummariseV2(assessment); if (summarise) return { ...summarise, priority: 3 };
const pause = selectPauseV2(assessment); if (pause) return { ...pause, priority: 4 };
return { behaviour: "continue", confidence: "low", reason: "No explicit rule matched — defaulting to continue." };
}
/* ═══════════════════════════════════════════════════
4. Real-scenario fixtures (imported from Exp 39/40)
═══════════════════════════════════════════════════ */
function mkN(id, label, opts = {}) {
const kind = opts.kind || "unknown";
const status = opts.status || (kind === "unknown" ? "unknown" : "known");
const confidence = opts.confidence || (kind === "unknown" ? "low" : "high");
return { id, label, description: label, kind, status, confidence, evidenceIds: [], dependsOn: [], affects: [], childIds: [] };
}
function buildInput(turn) {
return {
situationGraph: { centralStatement: turn.centralStatement, currentSummary: turn.currentSummary, nodes: turn.nodes, edges: [], activeUnknownNodeId: turn.activeUnknownNodeId, resolvedNodeIds: turn.resolvedNodeIds },
selectedQuestion: turn.selectedQuestion, noQuestionReason: turn.noQuestionReason,
diagnostics: { promptVersion: "v0.4", modelName: "mock-ollama", responseDurationMs: 0, validationStatus: "valid", nodeCount: turn.nodes.length, edgeCount: 0, reasoningPattern: turn.diagnosticReasoningPattern || null }
};
}
function getRealTurns() {
return [
{ scenario: "long-investigation", turnNumber: 0, centralStatement: "Should we enter the European market with our SaaS analytics platform?", nodes: [mkN("obs-1", "Current revenue is $2M ARR in the US market only", { kind: "observation", status: "known", confidence: "high" }), mkN("state-1", "Evaluating European market entry", { kind: "state", status: "provisional", confidence: "medium" }), mkN("u-1", "Whether there is genuine demand for our category in Europe")], resolvedNodeIds: [], activeUnknownNodeId: "u-1", selectedQuestion: { nodeId: "u-1", question: "How large and mature is the analytics SaaS market in Europe?", reason: "market_validity" }, currentSummary: "We are US-based. The first question before any expansion is whether demand exists.", diagnosticReasoningPattern: "market_validity" },
{ scenario: "long-investigation", turnNumber: 3, centralStatement: "Should we enter the European market with our SaaS analytics platform?", nodes: [mkN("obs-1", "Current revenue is $2M ARR in the US market only", { kind: "observation", status: "known", confidence: "high" }), mkN("obs-2", "European analytics SaaS market valued at approximately €8B and growing 15% annually", { kind: "observation", status: "known", confidence: "medium" }), mkN("obs-3", "Our platform does not currently support EU data residency requirements", { kind: "observation", status: "known", confidence: "high" }), mkN("obs-4", "Achieving compliance would require approximately 6 months and $500K engineering investment", { kind: "observation", status: "known", confidence: "medium" }), mkN("state-1", "Evaluating European market entry", { kind: "state", status: "provisional", confidence: "medium" }), mkN("u-1", "Whether there is genuine demand for our category in Europe", { status: "resolved", confidence: "medium" }), mkN("u-2", "Whether our product is suitable for European compliance requirements", { status: "resolved", confidence: "high" }), mkN("u-3", "Whether the cost of achieving compliance is justified by the market size", { status: "resolved", confidence: "medium" }), mkN("u-4", "Whether we have competitive differentiation against existing European players")], resolvedNodeIds: ["u-1", "u-2", "u-3"], activeUnknownNodeId: "u-4", selectedQuestion: { nodeId: "u-4", question: "What differentiates our platform against established European competitors?", reason: "competitive_analysis" }, currentSummary: "Compliance is feasible. The remaining question is competitive edge.", diagnosticReasoningPattern: "competitive_analysis" },
{ scenario: "long-investigation", turnNumber: 4, centralStatement: "Should we enter the European market with our SaaS analytics platform?", nodes: [mkN("obs-1", "Current revenue is $2M ARR in the US market only", { kind: "observation", status: "known", confidence: "high" }), mkN("obs-2", "European analytics SaaS market valued at approximately €8B and growing 15% annually", { kind: "observation", status: "known", confidence: "medium" }), mkN("obs-3", "Our platform does not currently support EU data residency requirements", { kind: "observation", status: "known", confidence: "high" }), mkN("obs-4", "Achieving compliance would require approximately 6 months and $500K engineering investment", { kind: "observation", status: "known", confidence: "medium" }), mkN("obs-5", "Our real-time collaboration feature has no direct European equivalent", { kind: "observation", status: "provisional", confidence: "medium" }), mkN("state-1", "Evaluating European market entry", { kind: "state", status: "provisional", confidence: "medium" }), mkN("u-1", "Whether there is genuine demand for our category in Europe", { status: "resolved", confidence: "medium" }), mkN("u-2", "Whether our product is suitable for European compliance requirements", { status: "resolved", confidence: "high" }), mkN("u-3", "Whether the cost of achieving compliance is justified by the market size", { status: "resolved", confidence: "medium" }), mkN("u-4", "Whether we have competitive differentiation against existing European players", { status: "resolved", confidence: "medium" })], resolvedNodeIds: ["u-1", "u-2", "u-3", "u-4"], activeUnknownNodeId: null, selectedQuestion: null, noQuestionReason: "All investigation areas resolved.", currentSummary: "European market entry is justified if compliance is achieved and the real-time collaboration feature is positioned as differentiator.", diagnosticReasoningPattern: null },
{ scenario: "contradictory-evidence", turnNumber: 0, centralStatement: "Two consultants give opposite recommendations on which supplier to choose for a $2M procurement.", nodes: [mkN("obs-1", "Consultant A recommends Supplier X: lower cost, proven track record", { kind: "observation", status: "known", confidence: "high" }), mkN("obs-2", "Consultant B recommends Supplier Y: better integration capability, higher risk but long-term upside", { kind: "observation", status: "known", confidence: "high" }), mkN("obs-3", "Supplier X has 15+ years in the sector; Supplier Y has 2 years and mixed client reviews", { kind: "observation", status: "known", confidence: "medium" }), mkN("obs-4", "Our current infrastructure is compatible with neither supplier out of the box", { kind: "observation", status: "known", confidence: "high" }), mkN("state-1", "Evaluating $2M procurement against conflicting expert advice", { kind: "state", status: "provisional", confidence: "medium" }), mkN("u-1", "Whether the conflict is genuine or reflects different evaluation criteria")], resolvedNodeIds: [], activeUnknownNodeId: "u-1", selectedQuestion: { nodeId: "u-1", question: "Are the consultants evaluating the same criteria, or are they measuring different things?", reason: "comparability_check" }, currentSummary: "Conflicting recommendations from two experts. The first uncertainty is whether we are comparing the same dimensions.", diagnosticReasoningPattern: "comparability_check" },
{ scenario: "contradictory-evidence", turnNumber: 1, centralStatement: "Two consultants give opposite recommendations on which supplier to choose for a $2M procurement.", nodes: [mkN("obs-1", "Consultant A recommends Supplier X: lower cost, proven track record", { kind: "observation", status: "known", confidence: "high" }), mkN("obs-2", "Consultant B recommends Supplier Y: better integration capability, higher risk but long-term upside", { kind: "observation", status: "known", confidence: "high" }), mkN("obs-3", "Supplier X has 15+ years in the sector; Supplier Y has 2 years and mixed client reviews", { kind: "observation", status: "known", confidence: "medium" }), mkN("obs-4", "Our current infrastructure is compatible with neither supplier out of the box", { kind: "observation", status: "known", confidence: "high" }), mkN("obs-5", "The consultants used different evaluation weights: cost 60% vs integration 60%", { kind: "observation", status: "known", confidence: "medium" }), mkN("state-1", "Evaluating $2M procurement against conflicting expert advice", { kind: "state", status: "provisional", confidence: "medium" }), mkN("u-1", "Whether the conflict is genuine or reflects different evaluation criteria", { status: "resolved", confidence: "high" }), mkN("u-2", "Which supplier's strengths align with our strategic priorities")], resolvedNodeIds: ["u-1"], activeUnknownNodeId: "u-2", selectedQuestion: { nodeId: "u-2", question: "Does cost or integration capability matter more to the organisation over a 3-year horizon?", reason: "evidence_quality" }, currentSummary: "The conflict reflects different evaluation weights. The next uncertainty is strategic alignment.", diagnosticReasoningPattern: "evidence_quality" },
{ scenario: "contradictory-evidence", turnNumber: 2, centralStatement: "Two consultants give opposite recommendations on which supplier to choose for a $2M procurement.", nodes: [mkN("obs-1", "Consultant A recommends Supplier X: lower cost, proven track record", { kind: "observation", status: "known", confidence: "high" }), mkN("obs-2", "Consultant B recommends Supplier Y: better integration capability, higher risk but long-term upside", { kind: "observation", status: "known", confidence: "high" }), mkN("obs-3", "Supplier X has 15+ years in the sector; Supplier Y has 2 years and mixed client reviews", { kind: "observation", status: "known", confidence: "medium" }), mkN("obs-4", "Our current infrastructure is compatible with neither supplier out of the box", { kind: "observation", status: "known", confidence: "high" }), mkN("obs-5", "The consultants used different evaluation weights: cost 60% vs integration 60%", { kind: "observation", status: "known", confidence: "medium" }), mkN("obs-6", "Our strategic plan prioritises long-term capability over short-term cost savings", { kind: "observation", status: "known", confidence: "high" }), mkN("state-1", "Evaluating $2M procurement against conflicting expert advice", { kind: "state", status: "provisional", confidence: "medium" }), mkN("u-1", "Whether the conflict is genuine or reflects different evaluation criteria", { status: "resolved", confidence: "high" }), mkN("u-2", "Which supplier's strengths align with our strategic priorities", { status: "resolved", confidence: "medium" }), mkN("u-3", "Whether the integration risk of Supplier Y is manageable with internal resources")], resolvedNodeIds: ["u-1", "u-2"], activeUnknownNodeId: "u-3", selectedQuestion: { nodeId: "u-3", question: "Do we have the internal capacity to manage Supplier Y's integration risk?", reason: "alternative_explanation" }, currentSummary: "Strategic priorities favour integration capability. The remaining uncertainty is operational feasibility.", diagnosticReasoningPattern: "alternative_explanation" },
{ scenario: "short-early", turnNumber: 0, centralStatement: "A manufacturing company reports complaints increased by 35% while production increased by 40%.", nodes: [mkN("obs-1", "Complaints increased by 35%", { kind: "observation", status: "known", confidence: "high" }), mkN("obs-2", "Production increased by 40%", { kind: "observation", status: "known", confidence: "high" }), mkN("state-1", "Current situation", { kind: "state", status: "provisional", confidence: "medium" }), mkN("u-1", "Whether the two figures cover the same period")], resolvedNodeIds: [], activeUnknownNodeId: "u-1", selectedQuestion: { nodeId: "u-1", question: "Were the complaint and production figures measured over the same period?", reason: "comparability_check" }, currentSummary: "Two changes have been reported, but we do not yet know whether the figures are directly comparable.", diagnosticReasoningPattern: "comparability_check" }
];
}
/* ═══════════════════════════════════════════════════
5. Evaluation helper
═══════════════════════════════════════════════════ */
function evaluateTurn(turn) {
const input = buildInput(turn);
const assessment = assessInvestigationState(input);
// Deep-clone to verify immutability later
const originalHealthValue = assessment.conversationHealth.value;
const originalPhaseConfidence = assessment.phase.confidence;
const originalProgressValue = assessment.progress.value;
const existingSel = selectBehaviour(assessment);
const varASel = selectVariantA(assessment);
const varBSel = selectVariantB(assessment);
// Verify immutability
expect(assessment.conversationHealth.value).toBe(originalHealthValue);
expect(assessment.phase.confidence).toBe(originalPhaseConfidence);
expect(assessment.progress.value).toBe(originalProgressValue);
return {
scenario: turn.scenario,
turnNumber: turn.turnNumber,
centralStatementShort: turn.centralStatement.substring(0, 55) + (turn.centralStatement.length > 55 ? "…" : ""),
phase: assessment.phase.value,
phaseConfidence: assessment.phase.confidence,
progress: assessment.progress.value,
health: assessment.conversationHealth.value,
existingSelection: existingSel.behaviour,
variantASelection: varASel.behaviour,
variantBSelection: varBSel.behaviour,
_assessment: assessment
};
}
/* ═══════════════════════════════════════════════════
6. Classification helper
═══════════════════════════════════════════════════ */
function classifyChange(turnIdx, originalSel, newSel) {
if (originalSel === newSel) return "unchanged";
const turn = getRealTurns()[turnIdx];
const input = buildInput(turn);
const assessment = assessInvestigationState(input);
const { phase, progress, health } = { phase: assessment.phase.value, progress: assessment.progress.value, health: assessment.conversationHealth.value };
if (newSel === "summarise" && ["synthesising", "concluding"].includes(phase)) return "sensible";
if (newSel === "summarise" && phase.evidence?.resolvedNodeCount >= 3 && progress === "steady") return "sensible";
if (newSel === "pause" && ((phase === "focusing" && progress === "stalled") || health === "user_overloaded")) return "sensible";
if (newSel === "acknowledge" && health === "healthy" && assessment.phase.confidence !== "low") return "sensible";
// Check whether original was sensible and new one is questionable
const origWasSensible = (originalSel === "acknowledge" && health === "healthy" && assessment.phase.confidence !== "low");
if (origWasSensible && newSel !== "acknowledge") {
return "questionable"; // Original was sensible, the variant changed it to something less fitting
}
if (newSel === "continue") return "sensible";
if (newSel === "summarise" || newSel === "pause") return "questionable"; // no clear rule match found
return "cannot determine";
}
/* ═══════════════════════════════════════════════════
Test suite — Experiment 41
═══════════════════════════════════════════════════ */
describe("Experiment 41 — Counterfactual: Acknowledge Priority Alternatives", () => {
/* ── Variant A contract tests ──────────────── */
describe("Variant A implements only priority reordering", () => {
it("prioritises Summarise before Acknowledge", () => {
const result = selectVariantA(mkAssessment({ phase: { value: "concluding", confidence: "high" }, progress: { value: "steady", confidence: "medium" }, conversationHealth: { value: "healthy", confidence: "high" } }));
expect(result.behaviour).toBe("summarise");
});
it("prioritises Pause before Acknowledge when conditions match", () => {
const result = selectVariantA(mkAssessment({ phase: { value: "focusing", confidence: "high" }, progress: { value: "stalled",confidence: "high" }, conversationHealth: { value: "healthy", confidence: "high" } }));
expect(result.behaviour).toBe("pause");
});
it("does not change non-conflicting selections", () => {
const result = selectVariantA(mkAssessment({ phase: { value: "exploring", confidence: "low" }, progress: { value: "cannot_determine", confidence: "low" }, conversationHealth: { value: "too_broad", confidence: "high" } }));
expect(result.behaviour).toBe("clarify"); // no ack conditions met, clarify is only match
});
it("preserves Continue fallback", () => {
const result = selectVariantA(mkAssessment({ phase: { value: "exploring", confidence: "low" }, progress: { value: "cannot_determine", confidence: "low" }, conversationHealth: { value: "too_narrow", confidence: "low" } }));
expect(result.behaviour).toBe("continue");
});
it("preserves Clarify rule when health=too_broad", () => {
const result = selectVariantA(mkAssessment({ phase: { value: "orienting", confidence: "low" }, progress: { value: "cannot_determine", confidence: "low" }, conversationHealth: { value: "too_broad", confidence: "high" } }));
expect(result.behaviour).toBe("clarify");
});
});
/* ── Variant B contract tests ──────────────── */
describe("Variant B implements only Acknowledge exclusions", () => {
it("blocks Acknowledge when phase=synthesising", () => {
const result = selectVariantB(mkAssessment({ phase: { value: "synthesising", confidence: "high" }, progress: { value: "steady", confidence: "medium" }, conversationHealth: { value: "healthy", confidence: "high" } }));
expect(result.behaviour).toBe("summarise"); // Acknowledge excluded, summarise fires next
});
it("blocks Acknowledge when phase=concluding", () => {
const result = selectVariantB(mkAssessment({ phase: { value: "concluding", confidence: "high" }, progress: { value: "steady", confidence: "medium" }, conversationHealth: { value: "healthy", confidence: "high" } }));
expect(result.behaviour).toBe("summarise"); // Acknowledge excluded, summarise fires next
});
it("blocks Acknowledge when progress=stalled", () => {
const result = selectVariantB(mkAssessment({ phase: { value: "focusing", confidence: "high" }, progress: { value: "stalled", confidence: "high" }, conversationHealth: { value: "healthy", confidence: "medium" } }));
expect(result.behaviour).toBe("pause"); // Acknowledge excluded, pause fires next
});
it("blocks Acknowledge when health=user_overloaded", () => {
const result = selectVariantB(mkAssessment({ phase: { value: "exploring", confidence: "medium" }, progress: { value: "steady", confidence: "medium" }, conversationHealth: { value: "user_overloaded", confidence: "medium" } }));
expect(result.behaviour).toBe("pause"); // Acknowledge excluded, pause fires next
});
it("preserves Acknowledge when no exclusion conditions match", () => {
const result = selectVariantB(mkAssessment({ phase: { value: "focusing", confidence: "high" }, progress: { value: "steady", confidence: "medium" }, conversationHealth: { value: "healthy", confidence: "high" } }));
expect(result.behaviour).toBe("acknowledge"); // focusing≠synthesising/concluding, progress=steady≠stalled, health=healthy≠user_overloaded
});
it("preserves Continue fallback", () => {
const result = selectVariantB(mkAssessment({ phase: { value: "exploring", confidence: "low" }, progress: { value: "cannot_determine", confidence: "low" }, conversationHealth: { value: "too_narrow", confidence: "low" } }));
expect(result.behaviour).toBe("continue");
});
it("preserves Clarify rule when health=too_broad and phase low confidence", () => {
const result = selectVariantB(mkAssessment({ phase: { value: "orienting", confidence: "low" }, progress: { value: "cannot_determine", confidence: "low" }, conversationHealth: { value: "too_broad", confidence: "high" } }));
expect(result.behaviour).toBe("clarify");
});
it("preserves Summarise rule when phase=synthesising and Acknowledge excluded", () => {
const result = selectVariantB(mkAssessment({ phase: { value: "synthesising", confidence: "high" }, progress: { value: "steady", confidence: "medium" }, conversationHealth: { value: "healthy", confidence: "high" } }));
expect(result.behaviour).toBe("summarise"); // Acknowledge excluded, summarise fires at priority 3
});
});
/* ── Key scenario reachability tests ───────── */
describe("Key scenario reachability", () => {
it("concluding state can reach Summarise under Variant A", () => {
const result = selectVariantA(mkAssessment({ phase: { value: "concluding", confidence: "high" }, progress: { value: "steady", confidence: "medium" }, conversationHealth: { value: "healthy", confidence: "high" } }));
expect(result.behaviour).toBe("summarise");
});
it("stalled focusing state can reach Pause under Variant A", () => {
const result = selectVariantA(mkAssessment({ phase: { value: "focusing", confidence: "high" }, progress: { value: "stalled", confidence: "high" }, conversationHealth: { value: "healthy", confidence: "high" } }));
expect(result.behaviour).toBe("pause");
});
it("healthy ordinary progress can still reach Acknowledge under Variant A", () => {
const result = selectVariantA(mkAssessment({ phase: { value: "focusing", confidence: "high" }, progress: { value: "steady", confidence: "medium" }, conversationHealth: { value: "healthy", confidence: "high" } }));
expect(result.behaviour).toBe("acknowledge"); // summarise/pause not eligible, ack is 4th but wins
});
it("concluding state can reach Summarise under Variant B", () => {
const result = selectVariantB(mkAssessment({ phase: { value: "concluding", confidence: "high" }, progress: { value: "steady", confidence: "medium" }, conversationHealth: { value: "healthy", confidence: "high" } }));
expect(result.behaviour).toBe("summarise");
});
it("stalled focusing state can reach Pause under Variant B", () => {
const result = selectVariantB(mkAssessment({ phase: { value: "focusing", confidence: "high" }, progress: { value: "stalled", confidence: "high" }, conversationHealth: { value: "healthy", confidence: "medium" } }));
expect(result.behaviour).toBe("pause");
});
it("healthy ordinary progress can still reach Acknowledge under Variant B", () => {
const result = selectVariantB(mkAssessment({ phase: { value: "focusing", confidence: "high" }, progress: { value: "steady", confidence: "medium" }, conversationHealth: { value: "healthy", confidence: "high" } }));
expect(result.behaviour).toBe("acknowledge"); // no exclusion conditions match (focusing≠synthesising/concluding, steady≠stalled)
});
it("fallback remains Continue when no rule matches", () => {
const resultA = selectVariantA(mkAssessment({ phase: { value: "exploring", confidence: "low" }, progress: { value: "cannot_determine", confidence: "low" }, conversationHealth: { value: "too_narrow", confidence: "low" } }));
const resultB = selectVariantB(mkAssessment({ phase: { value: "exploring", confidence: "low" }, progress: { value: "cannot_determine", confidence: "low" }, conversationHealth: { value: "too_narrow", confidence: "low" } }));
expect(resultA.behaviour).toBe("continue");
expect(resultB.behaviour).toBe("continue");
});
it("early low-confidence state falls back safely under both variants", () => {
const result = selectVariantA(mkAssessment({ phase: { value: "cannot_determine", confidence: "low" }, progress: { value: "cannot_determine", confidence: "low" }, conversationHealth: { value: "too_narrow", confidence: "low" } }));
expect(result.behaviour).toBe("continue");
const resultB = selectVariantB(mkAssessment({ phase: { value: "cannot_determine", confidence: "low" }, progress: { value: "cannot_determine", confidence: "low" }, conversationHealth: { value: "too_narrow", confidence: "low" } }));
expect(resultB.behaviour).toBe("continue");
});
});
/* ── Existing selector unchanged ───────────── */
describe("Existing selector results remain unchanged", () => {
it("returns acknowledge for healthy phase with confidence (base behaviour)", () => {
const result = selectBehaviour(mkAssessment({ phase: { value: "focusing", confidence: "high" }, progress: { value: "steady", confidence: "medium" }, conversationHealth: { value: "healthy", confidence: "high" } }));
expect(result.behaviour).toBe("acknowledge");
});
it("returns continue for null input (base behaviour)", () => {
const result = selectBehaviour(null);
expect(result.behaviour).toBe("continue");
});
it("only outputs valid behaviours (base contract)", () => {
const behaviours = [];
for (const phase of ["cannot_determine", "exploring", "orienting", "deepening", "focusing", "synthesising", "concluding"]) {
for (const conf of ["high", "medium", "low"]) {
const result = selectBehaviour(mkAssessment({ phase: { value: phase, confidence: conf }, progress: { value: "cannot_determine", confidence: "low" }, conversationHealth: { value: "cannot_determine", confidence: "low" } }));
behaviours.push(result.behaviour);
}
}
for (const b of behaviours) expect(["acknowledge", "clarify", "summarise", "pause", "continue"]).toContain(b);
});
});
/* ── Seven real assessment turns comparison ─ */
describe("Seven real assessment turns compared across all selectors", () => {
let results;
beforeEach(() => {
const turns = getRealTurns();
results = turns.map(t => evaluateTurn(t));
});
it("evaluates all seven turns", () => expect(results).toHaveLength(7));
it("outputs are deterministic across calls", () => {
for (let i = 0; i < results.length; i++) {
const turn2 = getRealTurns()[i];
const result2 = evaluateTurn(turn2);
expect(result2.existingSelection).toBe(results[i].existingSelection);
expect(result2.variantASelection).toBe(results[i].variantASelection);
expect(result2.variantBSelection).toBe(results[i].variantBSelection);
}
});
it("inputs are not mutated", () => {
const turns = getRealTurns();
for (const turn of turns) {
const input = buildInput(turn);
const snapshot = JSON.stringify(input);
assessInvestigationState(input);
selectVariantA(assessInvestigationState(JSON.parse(snapshot)));
selectVariantB(assessInvestigationState(JSON.parse(snapshot)));
expect(JSON.stringify(input)).toBe(snapshot);
}
});
it("reports assessor output for each turn", () => {
for (const r of results) {
console.log(`\n=== ${r.scenario} turn ${r.turnNumber} ===`);
console.log(` phase=${r.phase}(conf:${r.phaseConfidence}), progress=${r.progress}, health=${r.health}`);
console.log(` existing: ${r.existingSelection} | variant A: ${r.variantASelection} | variant B: ${r.variantBSelection}`);
}
});
it("records whether each change appears sensible/questionable", () => {
const classifications = {};
for (let i = 0; i < results.length; i++) {
const r = results[i];
classifications[`existing-${r.scenario}-t${r.turnNumber}`] = "unchanged"; // baseline
if (r.existingSelection !== r.variantASelection) {
classifications[`variantA-${r.scenario}-t${r.turnNumber}`] = classifyChange(i, r.existingSelection, r.variantASelection);
} else {
classifications[`variantA-${r.scenario}-t${r.turnNumber}`] = "unchanged";
}
if (r.existingSelection !== r.variantBSelection) {
classifications[`variantB-${r.scenario}-t${r.turnNumber}`] = classifyChange(i, r.existingSelection, r.variantBSelection);
} else {
classifications[`variantB-${r.scenario}-t${r.turnNumber}`] = "unchanged";
}
}
console.log("\n=== Selection Classifications ===");
for (const [key, val] of Object.entries(classifications)) {
console.log(` ${key}: ${val}`);
}
global._exp41_classifications = classifications;
});
it("turn-level summary", () => {
const distA = { acknowledge: 0, clarify: 0, summarise: 0, pause: 0, continue: 0 };
const distB = { acknowledge: 0, clarify: 0, summarise: 0, pause: 0, continue: 0 };
for (const r of results) {
distA[r.variantASelection]++;
distB[r.variantBSelection]++;
}
console.log("\n=== Existing Distribution ===");
const existingDist = { acknowledge: 0, clarify: 0, summarise: 0, pause: 0, continue: 0 };
for (const r of results) existingDist[r.existingSelection]++;
for (const [beh, count] of Object.entries(existingDist)) console.log(` ${beh}: ${count} (${((count/7)*100).toFixed(0)}%)`);
console.log("\n=== Variant A Distribution ===");
for (const [beh, count] of Object.entries(distA)) console.log(` ${beh}: ${count} (${((count/7)*100).toFixed(0)}%)`);
console.log("\n=== Variant B Distribution ===");
for (const [beh, count] of Object.entries(distB)) console.log(` ${beh}: ${count} (${((count/7)*100).toFixed(0)}%)`);
global._exp41_distA = distA;
global._exp41_distB = distB;
});
});
/* ── Synthetic safety checks ───────────────── */
describe("Synthetic safety checks", () => {
it("healthy mid-investigation progress can reach Acknowledge under both variants", () => {
const a = selectVariantA(mkAssessment({ phase: { value: "focusing", confidence: "high" }, progress: { value: "steady", confidence: "medium" }, conversationHealth: { value: "healthy", confidence: "high" } }));
const b = selectVariantB(mkAssessment({ phase: { value: "focusing", confidence: "high" }, progress: { value: "steady", confidence: "medium" }, conversationHealth: { value: "healthy", confidence: "high" } }));
expect(a.behaviour).toBe("acknowledge");
expect(b.behaviour).toBe("acknowledge");
});
it("concluding state reaches Summarise under both variants", () => {
const a = selectVariantA(mkAssessment({ phase: { value: "concluding", confidence: "high" }, progress: { value: "steady", confidence: "medium" }, conversationHealth: { value: "healthy", confidence: "high" } }));
const b = selectVariantB(mkAssessment({ phase: { value: "concluding", confidence: "high" }, progress: { value: "steady", confidence: "medium" }, conversationHealth: { value: "healthy", confidence: "high" } }));
expect(a.behaviour).toBe("summarise");
expect(b.behaviour).toBe("summarise");
});
it("focusing + stalled reaches Pause under both variants", () => {
const a = selectVariantA(mkAssessment({ phase: { value: "focusing", confidence: "high" }, progress: { value: "stalled", confidence: "high" }, conversationHealth: { value: "healthy", confidence: "medium" } }));
const b = selectVariantB(mkAssessment({ phase: { value: "focusing", confidence: "high" }, progress: { value: "stalled", confidence: "high" }, conversationHealth: { value: "healthy", confidence: "medium" } }));
expect(a.behaviour).toBe("pause");
expect(b.behaviour).toBe("pause");
});
it("user_overloaded reaches Pause under both variants", () => {
const a = selectVariantA(mkAssessment({ phase: { value: "exploring", confidence: "medium" }, progress: { value: "steady", confidence: "medium" }, conversationHealth: { value: "user_overloaded", confidence: "medium" } }));
const b = selectVariantB(mkAssessment({ phase: { value: "exploring", confidence: "medium" }, progress: { value: "steady", confidence: "medium" }, conversationHealth: { value: "user_overloaded", confidence: "medium" } }));
expect(a.behaviour).toBe("pause");
expect(b.behaviour).toBe("pause");
});
it("early low-confidence state falls back to Continue under both variants", () => {
const a = selectVariantA(mkAssessment({ phase: { value: "exploring", confidence: "low" }, progress: { value: "cannot_determine", confidence: "low" }, conversationHealth: { value: "too_narrow", confidence: "low" } }));
const b = selectVariantB(mkAssessment({ phase: { value: "exploring", confidence: "low" }, progress: { value: "cannot_determine", confidence: "low" }, conversationHealth: { value: "too_narrow", confidence: "low" } }));
expect(a.behaviour).toBe("continue");
expect(b.behaviour).toBe("continue");
});
it("state where no explicit rule matches returns Continue under both variants", () => {
const a = selectVariantA(mkAssessment({ phase: { value: "deepening", confidence: "medium" }, progress: { value: "steady", confidence: "medium" } }));
const b = selectVariantB(mkAssessment({ phase: { value: "deepening", confidence: "medium" }, progress: { value: "steady", confidence: "medium" } }));
expect(a.behaviour).toBe("continue");
expect(b.behaviour).toBe("continue");
});
it("neither variant causes Pause too early in normal progress", () => {
const a = selectVariantA(mkAssessment({ phase: { value: "focusing", confidence: "high" }, progress: { value: "steady", confidence: "medium" }, conversationHealth: { value: "healthy", confidence: "high" } }));
const b = selectVariantB(mkAssessment({ phase: { value: "focusing", confidence: "high" }, progress: { value: "steady", confidence: "medium" }, conversationHealth: { value: "healthy", confidence: "high" } }));
expect(a.behaviour).not.toBe("pause");
expect(b.behaviour).not.toBe("pause");
});
it("neither variant causes Summarise too early in normal progress", () => {
const a = selectVariantA(mkAssessment({ phase: { value: "focusing", confidence: "high" }, progress: { value: "steady", confidence: "medium" }, conversationHealth: { value: "healthy", confidence: "high" } }));
expect(a.behaviour).not.toBe("summarise");
});
it("early incomplete states fall back safely under both variants", () => {
const a = selectVariantA(mkAssessment({ phase: { value: "cannot_determine", confidence: "low" }, progress: { value: "cannot_determine", confidence: "low" }, conversationHealth: { value: "too_narrow", confidence: "low" } }));
const b = selectVariantB(mkAssessment({ phase: { value: "cannot_determine", confidence: "low" }, progress: { value: "cannot_determine", confidence: "low" }, conversationHealth: { value: "too_narrow", confidence: "low" } }));
expect(a.behaviour).toBe("continue");
expect(b.behaviour).toBe("continue");
});
});
/* ── Both variants produce identical results across tested turns ─ */
describe("Variant A vs Variant B — divergence analysis", () => {
it("both variants converge on the same two genuine changes (concluding→summarise, stalled→pause) despite different mechanisms", () => {
const turns = getRealTurns();
// Variant A diverges at long-investigation t3 (focusing with 3 resolved items)
// because its resolvedNodeCount >= 3 rule fires without phase context — a side-effect.
// Variant B correctly preserves Acknowledge there via explicit exclusion list.
// Both converge on the two genuine changes: concluding→summarise and stalled→pause.
const divergeIdx = 1; // long-investigation t3 (Variant A over-summarises here)
expect(selectVariantA(assessInvestigationState(buildInput(turns[divergeIdx]))).behaviour).toBe("summarise");
expect(selectVariantB(assessInvestigationState(buildInput(turns[divergeIdx]))).behaviour).toBe("acknowledge");
// Both converge: concluding turn
const a4 = selectVariantA(assessInvestigationState(buildInput(turns[2])));
const b4 = selectVariantB(assessInvestigationState(buildInput(turns[2])));
expect(a4.behaviour).toBe("summarise");
expect(b4.behaviour).toBe("summarise");
// Both converge: stalled focusing turn
const c1 = selectVariantA(assessInvestigationState(buildInput(turns[4])));
const d1 = selectVariantB(assessInvestigationState(buildInput(turns[4])));
expect(c1.behaviour).toBe("pause");
expect(d1.behaviour).toBe("pause");
});
});
/* ── Specific expected outcomes ────────────── */
describe("Specific expected outcomes", () => {
it("concluding turn (long-investigation t4) selects Summarise under both variants", () => {
const turns = getRealTurns();
const r = evaluateTurn(turns[2]); // long-investigation t4
expect(r.variantASelection).toBe("summarise");
expect(r.variantBSelection).toBe("summarise");
});
it("stalled focusing turn (contradictory-evidence t1) selects Pause under both variants", () => {
const turns = getRealTurns();
const r = evaluateTurn(turns[4]); // contradictory-evidence t1
expect(r.variantASelection).toBe("pause");
expect(r.variantBSelection).toBe("pause");
});
it("ordinary healthy progress still allows Acknowledge under Variant B (Variant A over-summarises mid-focus)", () => {
const turns = getRealTurns();
// long-investigation t3: Variant A incorrectly produces summarise because resolvedNodeCount>=3 fires before acknowledging (side-effect)
expect(evaluateTurn(turns[1]).variantASelection).toBe("summarise");
expect(evaluateTurn(turns[1]).variantBSelection).toBe("acknowledge");
// contradictory-t0 and t2: both preserve acknowledge
for (const idx of [3, 5]) {
const r = evaluateTurn(turns[idx]);
expect(r.variantASelection).toBe("acknowledge");
expect(r.variantBSelection).toBe("acknowledge");
}
});
it("early incomplete states fall back to Continue under both variants", () => {
const turns = getRealTurns();
for (const idx of [0, 6]) { // long-investigation t0, short-early-t0
const r = evaluateTurn(turns[idx]);
expect(r.variantASelection).toBe("continue");
expect(r.variantBSelection).toBe("continue");
}
});
});
});
Reference in New Issue View Git Blame Copy Permalink