730 lines
38 KiB
JavaScript
730 lines
38 KiB
JavaScript
import { describe, it, expect } from "vitest";
|
|
import assessInvestigationState from "@/lib/assessment/investigation-state-assessor.js";
|
|
import selectBehaviour, { BEHAVIOUR_OPTIONS } from "@/lib/behaviour-selection/behaviour-selector.js";
|
|
|
|
/* ═══════════════════════════════════════════════════
|
|
Experiment 40 — Behaviour Reachability Diagnostic
|
|
═══════════════════════════════════════════════════ */
|
|
|
|
/* ── Helpers from the real-assessment test (shared fixtures) ─ */
|
|
|
|
function mkN(id, label, opts = {}) {
|
|
const kind = opts.kind || "unknown";
|
|
const status = opts.status || (kind === "unknown" ? "unknown" : "known");
|
|
const confidence = opts.confidence || (kind === "unknown" ? "low" : "high");
|
|
return {
|
|
id, label, description: label, kind, status, confidence,
|
|
evidenceIds: [], dependsOn: [], affects: [], childIds: []
|
|
};
|
|
}
|
|
|
|
function getEvaluationScenarios() {
|
|
return {
|
|
"long-investigation": [
|
|
{
|
|
centralStatement: "Should we enter the European market with our SaaS analytics platform?",
|
|
nodes: [
|
|
mkN("obs-1", "Current revenue is $2M ARR in the US market only", { kind: "observation", status: "known", confidence: "high" }),
|
|
mkN("state-1", "Evaluating European market entry", { kind: "state", status: "provisional", confidence: "medium" }),
|
|
mkN("u-1", "Whether there is genuine demand for our category in Europe")
|
|
],
|
|
resolvedNodeIds: [], activeUnknownNodeId: "u-1",
|
|
selectedQuestion: { nodeId: "u-1", question: "How large and mature is the analytics SaaS market in Europe?", reason: "market_validity" },
|
|
currentSummary: "We are US-based. The first question before any expansion is whether demand exists.",
|
|
diagnosticReasoningPattern: "market_validity"
|
|
},
|
|
{
|
|
centralStatement: "Should we enter the European market with our SaaS analytics platform?",
|
|
nodes: [
|
|
mkN("obs-1", "Current revenue is $2M ARR in the US market only", { kind: "observation", status: "known", confidence: "high" }),
|
|
mkN("obs-2", "European analytics SaaS market valued at approximately €8B and growing 15% annually", { kind: "observation", status: "known", confidence: "medium" }),
|
|
mkN("obs-3", "Our platform does not currently support EU data residency requirements", { kind: "observation", status: "known", confidence: "high" }),
|
|
mkN("obs-4", "Achieving compliance would require approximately 6 months and $500K engineering investment", { kind: "observation", status: "known", confidence: "medium" }),
|
|
mkN("state-1", "Evaluating European market entry", { kind: "state", status: "provisional", confidence: "medium" }),
|
|
mkN("u-1", "Whether there is genuine demand for our category in Europe", { status: "resolved", confidence: "medium" }),
|
|
mkN("u-2", "Whether our product is suitable for European compliance requirements", { status: "resolved", confidence: "high" }),
|
|
mkN("u-3", "Whether the cost of achieving compliance is justified by the market size", { status: "resolved", confidence: "medium" }),
|
|
mkN("u-4", "Whether we have competitive differentiation against existing European players")
|
|
],
|
|
resolvedNodeIds: ["u-1", "u-2", "u-3"], activeUnknownNodeId: "u-4",
|
|
selectedQuestion: { nodeId: "u-4", question: "What differentiates our platform against established European competitors?", reason: "competitive_analysis" },
|
|
currentSummary: "Compliance is feasible. The remaining question is competitive edge.",
|
|
diagnosticReasoningPattern: "competitive_analysis"
|
|
},
|
|
{
|
|
centralStatement: "Should we enter the European market with our SaaS analytics platform?",
|
|
nodes: [
|
|
mkN("obs-1", "Current revenue is $2M ARR in the US market only", { kind: "observation", status: "known", confidence: "high" }),
|
|
mkN("obs-2", "European analytics SaaS market valued at approximately €8B and growing 15% annually", { kind: "observation", status: "known", confidence: "medium" }),
|
|
mkN("obs-3", "Our platform does not currently support EU data residency requirements", { kind: "observation", status: "known", confidence: "high" }),
|
|
mkN("obs-4", "Achieving compliance would require approximately 6 months and $500K engineering investment", { kind: "observation", status: "known", confidence: "medium" }),
|
|
mkN("obs-5", "Our real-time collaboration feature has no direct European equivalent", { kind: "observation", status: "provisional", confidence: "medium" }),
|
|
mkN("state-1", "Evaluating European market entry", { kind: "state", status: "provisional", confidence: "medium" }),
|
|
mkN("u-1", "Whether there is genuine demand for our category in Europe", { status: "resolved", confidence: "medium" }),
|
|
mkN("u-2", "Whether our product is suitable for European compliance requirements", { status: "resolved", confidence: "high" }),
|
|
mkN("u-3", "Whether the cost of achieving compliance is justified by the market size", { status: "resolved", confidence: "medium" }),
|
|
mkN("u-4", "Whether we have competitive differentiation against existing European players", { status: "resolved", confidence: "medium" })
|
|
],
|
|
resolvedNodeIds: ["u-1", "u-2", "u-3", "u-4"], activeUnknownNodeId: null,
|
|
selectedQuestion: null, noQuestionReason: "All investigation areas resolved.",
|
|
currentSummary: "European market entry is justified if compliance is achieved and the real-time collaboration feature is positioned as differentiator.",
|
|
diagnosticReasoningPattern: null
|
|
}
|
|
],
|
|
|
|
"contradictory-evidence": [
|
|
{
|
|
centralStatement: "Two consultants give opposite recommendations on which supplier to choose for a $2M procurement.",
|
|
nodes: [
|
|
mkN("obs-1", "Consultant A recommends Supplier X: lower cost, proven track record", { kind: "observation", status: "known", confidence: "high" }),
|
|
mkN("obs-2", "Consultant B recommends Supplier Y: better integration capability, higher risk but long-term upside", { kind: "observation", status: "known", confidence: "high" }),
|
|
mkN("obs-3", "Supplier X has 15+ years in the sector; Supplier Y has 2 years and mixed client reviews", { kind: "observation", status: "known", confidence: "medium" }),
|
|
mkN("obs-4", "Our current infrastructure is compatible with neither supplier out of the box", { kind: "observation", status: "known", confidence: "high" }),
|
|
mkN("state-1", "Evaluating $2M procurement against conflicting expert advice", { kind: "state", status: "provisional", confidence: "medium" }),
|
|
mkN("u-1", "Whether the conflict is genuine or reflects different evaluation criteria")
|
|
],
|
|
resolvedNodeIds: [], activeUnknownNodeId: "u-1",
|
|
selectedQuestion: { nodeId: "u-1", question: "Are the consultants evaluating the same criteria, or are they measuring different things?", reason: "comparability_check" },
|
|
currentSummary: "Conflicting recommendations from two experts. The first uncertainty is whether we are comparing the same dimensions.",
|
|
diagnosticReasoningPattern: "comparability_check"
|
|
},
|
|
{
|
|
centralStatement: "Two consultants give opposite recommendations on which supplier to choose for a $2M procurement.",
|
|
nodes: [
|
|
mkN("obs-1", "Consultant A recommends Supplier X: lower cost, proven track record", { kind: "observation", status: "known", confidence: "high" }),
|
|
mkN("obs-2", "Consultant B recommends Supplier Y: better integration capability, higher risk but long-term upside", { kind: "observation", status: "known", confidence: "high" }),
|
|
mkN("obs-3", "Supplier X has 15+ years in the sector; Supplier Y has 2 years and mixed client reviews", { kind: "observation", status: "known", confidence: "medium" }),
|
|
mkN("obs-4", "Our current infrastructure is compatible with neither supplier out of the box", { kind: "observation", status: "known", confidence: "high" }),
|
|
mkN("obs-5", "The consultants used different evaluation weights: cost 60% vs integration 60%", { kind: "observation", status: "known", confidence: "medium" }),
|
|
mkN("state-1", "Evaluating $2M procurement against conflicting expert advice", { kind: "state", status: "provisional", confidence: "medium" }),
|
|
mkN("u-1", "Whether the conflict is genuine or reflects different evaluation criteria", { status: "resolved", confidence: "high" }),
|
|
mkN("u-2", "Which supplier's strengths align with our strategic priorities")
|
|
],
|
|
resolvedNodeIds: ["u-1"], activeUnknownNodeId: "u-2",
|
|
selectedQuestion: { nodeId: "u-2", question: "Does cost or integration capability matter more to the organisation over a 3-year horizon?", reason: "evidence_quality" },
|
|
currentSummary: "The conflict reflects different evaluation weights. The next uncertainty is strategic alignment.",
|
|
diagnosticReasoningPattern: "evidence_quality"
|
|
},
|
|
{
|
|
centralStatement: "Two consultants give opposite recommendations on which supplier to choose for a $2M procurement.",
|
|
nodes: [
|
|
mkN("obs-1", "Consultant A recommends Supplier X: lower cost, proven track record", { kind: "observation", status: "known", confidence: "high" }),
|
|
mkN("obs-2", "Consultant B recommends Supplier Y: better integration capability, higher risk but long-term upside", { kind: "observation", status: "known", confidence: "high" }),
|
|
mkN("obs-3", "Supplier X has 15+ years in the sector; Supplier Y has 2 years and mixed client reviews", { kind: "observation", status: "known", confidence: "medium" }),
|
|
mkN("obs-4", "Our current infrastructure is compatible with neither supplier out of the box", { kind: "observation", status: "known", confidence: "high" }),
|
|
mkN("obs-5", "The consultants used different evaluation weights: cost 60% vs integration 60%", { kind: "observation", status: "known", confidence: "medium" }),
|
|
mkN("obs-6", "Our strategic plan prioritises long-term capability over short-term cost savings", { kind: "observation", status: "known", confidence: "high" }),
|
|
mkN("state-1", "Evaluating $2M procurement against conflicting expert advice", { kind: "state", status: "provisional", confidence: "medium" }),
|
|
mkN("u-1", "Whether the conflict is genuine or reflects different evaluation criteria", { status: "resolved", confidence: "high" }),
|
|
mkN("u-2", "Which supplier's strengths align with our strategic priorities", { status: "resolved", confidence: "medium" }),
|
|
mkN("u-3", "Whether the integration risk of Supplier Y is manageable with internal resources")
|
|
],
|
|
resolvedNodeIds: ["u-1", "u-2"], activeUnknownNodeId: "u-3",
|
|
selectedQuestion: { nodeId: "u-3", question: "Do we have the internal capacity to manage Supplier Y's integration risk?", reason: "alternative_explanation" },
|
|
currentSummary: "Strategic priorities favour integration capability. The remaining uncertainty is operational feasibility.",
|
|
diagnosticReasoningPattern: "alternative_explanation"
|
|
}
|
|
],
|
|
|
|
"short-early": [
|
|
{
|
|
centralStatement: "A manufacturing company reports complaints increased by 35% while production increased by 40%.",
|
|
nodes: [
|
|
mkN("obs-1", "Complaints increased by 35%", { kind: "observation", status: "known", confidence: "high" }),
|
|
mkN("obs-2", "Production increased by 40%", { kind: "observation", status: "known", confidence: "high" }),
|
|
mkN("state-1", "Current situation", { kind: "state", status: "provisional", confidence: "medium" }),
|
|
mkN("u-1", "Whether the two figures cover the same period")
|
|
],
|
|
resolvedNodeIds: [], activeUnknownNodeId: "u-1",
|
|
selectedQuestion: { nodeId: "u-1", question: "Were the complaint and production figures measured over the same period?", reason: "comparability_check" },
|
|
currentSummary: "Two changes have been reported, but we do not yet know whether the figures are directly comparable.",
|
|
diagnosticReasoningPattern: "comparability_check"
|
|
}
|
|
]
|
|
};
|
|
}
|
|
|
|
function buildAssessmentInput(turn) {
|
|
return {
|
|
situationGraph: {
|
|
centralStatement: turn.centralStatement,
|
|
currentSummary: turn.currentSummary,
|
|
nodes: turn.nodes,
|
|
edges: turn.edges || [],
|
|
activeUnknownNodeId: turn.activeUnknownNodeId,
|
|
resolvedNodeIds: turn.resolvedNodeIds
|
|
},
|
|
selectedQuestion: turn.selectedQuestion,
|
|
noQuestionReason: turn.noQuestionReason,
|
|
diagnostics: {
|
|
promptVersion: "v0.4",
|
|
modelName: "mock-ollama",
|
|
responseDurationMs: 0,
|
|
validationStatus: "valid",
|
|
nodeCount: turn.nodes.length,
|
|
edgeCount: (turn.edges || []).length,
|
|
reasoningPattern: turn.diagnosticReasoningPattern || null
|
|
}
|
|
};
|
|
}
|
|
|
|
/* ═══════════════════════════════════════════════════
|
|
Diagnostic audit helper — checks every rule per turn
|
|
═══════════════════════════════════════════════════ */
|
|
|
|
const BEHAVIOUR_CHECKERS = [
|
|
{ key: "acknowledge", fn: (a) => a.conversationHealth.value === "healthy" && a.phase.confidence !== "low" },
|
|
{ key: "clarify", fn: (a) => a.conversationHealth.value === "too_broad" || (a.phase.value === "orienting" && (a.phase.evidence?.observationDensity ?? Infinity) < 3) },
|
|
{ key: "summarise", fn: (a) => a.phase.value === "synthesising" || a.phase.value === "concluding" || (a.phase.evidence?.resolvedNodeCount >= 3 && a.progress.value === "steady") },
|
|
{ key: "pause", fn: (a) => (a.phase.value === "focusing" && a.progress.value === "stalled") || a.conversationHealth.value === "user_overloaded" },
|
|
{ key: "continue", fn: () => true }, // default — always eligible as fallback
|
|
];
|
|
|
|
/**
|
|
* Audit one assessment against every behaviour rule.
|
|
* Returns { behaviour, eligible, selected, blockedBy, reason } for each behaviour.
|
|
*/
|
|
function auditBehaviour(assessment) {
|
|
const results = [];
|
|
const actualResult = selectBehaviour(assessment);
|
|
|
|
for (const { key, fn } of BEHAVIOUR_CHECKERS) {
|
|
if (key === "continue") continue; // handled separately below
|
|
const eligible = fn(assessment);
|
|
let blockedBy = null;
|
|
let selected = false;
|
|
let reason = "";
|
|
|
|
if (eligible) {
|
|
// Check whether any earlier-priority behaviour fires first
|
|
const priorityOrder = ["acknowledge", "clarify", "summarise", "pause"];
|
|
const myIndex = priorityOrder.indexOf(key);
|
|
for (let i = 0; i < myIndex; i++) {
|
|
if (BEHAVIOUR_CHECKERS[i].fn(assessment)) {
|
|
blockedBy = BEHAVIOUR_CHECKERS[i].key;
|
|
break;
|
|
}
|
|
}
|
|
selected = (blockedBy === null);
|
|
reason = eligible ? (blockedBy ? `eligible but blocked by ${blockedBy}` : "eligible and selected") : "";
|
|
} else {
|
|
reason = `${key} rule conditions not met`;
|
|
}
|
|
|
|
results.push({ behaviour: key, eligible, selected, blockedBy, reason });
|
|
}
|
|
|
|
// Continue is always the fallback when nothing else fires
|
|
const continueEligible = !results.some(r => r.eligible && r.selected);
|
|
if (continueEligible) {
|
|
results.push({ behaviour: "continue", eligible: true, selected: true, blockedBy: null, reason: `No specific rule matched — defaulting to continue` });
|
|
} else {
|
|
const blockingBehaviour = results.find(r => r.eligible && r.selected);
|
|
results.push({
|
|
behaviour: "continue",
|
|
eligible: false,
|
|
selected: false,
|
|
blockedBy: blockingBehaviour ? blockingBehaviour.behaviour : null,
|
|
reason: blockingBehaviour ? `blocked by ${blockingBehaviour.behaviour}` : "specific rule matched"
|
|
});
|
|
}
|
|
|
|
return results;
|
|
}
|
|
|
|
/* ═══════════════════════════════════════════════════
|
|
Synthetic minimal assessment builders
|
|
═══════════════════════════════════════════════════ */
|
|
|
|
function mkMinimal(opts = {}) {
|
|
return {
|
|
version: "v0.1",
|
|
assessedAt: new Date().toISOString(),
|
|
confidence: opts.overallConfidence || "medium",
|
|
phase: opts.phase ?? { value: "cannot_determine", confidence: "low", signals: [], evidence: {} },
|
|
progress: opts.progress ?? { value: "cannot_determine", confidence: "low", signals: [], evidence: {} },
|
|
conversationHealth: opts.conversationHealth ?? { value: "cannot_determine", confidence: "low", signals: [], evidence: {} }
|
|
};
|
|
}
|
|
|
|
/* ═══════════════════════════════════════════════════
|
|
Test suite — Experiment 40
|
|
═══════════════════════════════════════════════════ */
|
|
|
|
describe("Experiment 40 — Behaviour Reachability Diagnostic", () => {
|
|
|
|
/* ── Scenario A: Real-scenario reachability audit ─────── */
|
|
|
|
describe("Real-scenario reachability (all Experiment 39 turns)", () => {
|
|
let allAudits = {};
|
|
|
|
function runAllAudits() {
|
|
if (Object.keys(allAudits).length > 0) return allAudits;
|
|
const scenarios = getEvaluationScenarios();
|
|
for (const [scenarioName, turns] of Object.entries(scenarios)) {
|
|
allAudits[scenarioName] = [];
|
|
for (let i = 0; i < turns.length; i++) {
|
|
const input = buildAssessmentInput(turns[i]);
|
|
const assessment = assessInvestigationState(input);
|
|
const behaviourResult = selectBehaviour(assessment);
|
|
allAudits[scenarioName].push({
|
|
turn: i,
|
|
centralStatement: (turns[i].centralStatement || "").substring(0, 50),
|
|
phase: assessment.phase.value,
|
|
progress: assessment.progress.value,
|
|
health: assessment.conversationHealth.value,
|
|
phaseConfidence: assessment.phase.confidence,
|
|
selectedBehaviour: behaviourResult.behaviour,
|
|
behaviours: auditBehaviour(assessment)
|
|
});
|
|
}
|
|
}
|
|
return allAudits;
|
|
}
|
|
|
|
it("audits every turn across all scenarios", () => {
|
|
const audits = runAllAudits();
|
|
expect(audits["long-investigation"]).toHaveLength(3);
|
|
expect(audits["contradictory-evidence"]).toHaveLength(3);
|
|
expect(audits["short-early"]).toHaveLength(1);
|
|
});
|
|
|
|
it("summarises reachability for each behaviour across real turns", () => {
|
|
const audits = runAllAudits();
|
|
const reachability = {};
|
|
|
|
for (const beh of ["acknowledge", "clarify", "summarise", "pause", "continue"]) {
|
|
let eligibleCount = 0;
|
|
let selectedCount = 0;
|
|
let blockedCount = 0;
|
|
const blockingSources = new Set();
|
|
const neverEligibleScenarios = new Set();
|
|
|
|
for (const [scenarioName, turns] of Object.entries(audits)) {
|
|
let scenarioEverEligible = false;
|
|
for (const turnAudit of turns) {
|
|
const behAudit = turnAudit.behaviours.find(b => b.behaviour === beh);
|
|
if (behAudit.eligible) {
|
|
eligibleCount++;
|
|
scenarioEverEligible = true;
|
|
if (behAudit.selected) selectedCount++;
|
|
else { blockedCount++; blockingSources.add(behAudit.blockedBy); }
|
|
}
|
|
}
|
|
if (!scenarioEverEligible) neverEligibleScenarios.add(scenarioName);
|
|
}
|
|
|
|
reachability[beh] = {
|
|
eligible: eligibleCount,
|
|
selected: selectedCount,
|
|
blocked: blockedCount,
|
|
blockingBy: Array.from(blockingSources),
|
|
neverEligibleInScenarios: Array.from(neverEligibleScenarios)
|
|
};
|
|
}
|
|
|
|
console.log("\n=== Experiment 40 — Reachability Summary ===");
|
|
for (const [beh, data] of Object.entries(reachability)) {
|
|
console.log(`\n${beh.toUpperCase()}:`);
|
|
console.log(` eligible: ${data.eligible}`);
|
|
console.log(` selected: ${data.selected}`);
|
|
console.log(` blocked: ${data.blocked} (by: ${data.blockingBy.join(", ") || "none"})`);
|
|
if (data.neverEligibleInScenarios.length) {
|
|
console.log(` never eligible in: ${data.neverEligibleInScenarios.join(", ")}`);
|
|
}
|
|
}
|
|
|
|
// Store for downstream assertions
|
|
global._exp40_reachability = reachability;
|
|
global._exp40_audits = audits;
|
|
|
|
return reachability;
|
|
});
|
|
|
|
it("Acknowledge is eligible in 5 of 7 turns and selected in all 5", () => {
|
|
const reachability = global._exp40_reachability || {};
|
|
expect(reachability.acknowledge?.eligible).toBe(5);
|
|
expect(reachability.acknowledge?.selected).toBe(5);
|
|
});
|
|
|
|
it("Summarise is eligible in 2 of 7 turns and always blocked", () => {
|
|
const reachability = global._exp40_reachability || {};
|
|
// Terminal state has phase=concluding, so summarise IS eligible
|
|
expect(reachability.summarise?.eligible).toBe(2);
|
|
// But acknowledge (priority 1) fires first because health=healthy
|
|
expect(reachability.summarise?.blocked).toBe(2);
|
|
expect(reachability.summarise?.blockingBy).toContain("acknowledge");
|
|
});
|
|
|
|
it("Clarify is eligible in 0 of 7 turns — never triggered", () => {
|
|
const reachability = global._exp40_reachability || {};
|
|
expect(reachability.clarify?.eligible).toBe(0);
|
|
});
|
|
|
|
it("Pause is eligible in 1 of 7 turns but always blocked", () => {
|
|
const reachability = global._exp40_reachability || {};
|
|
// Contradictory turn 1: phase=focusing, progress=stalled → pause rule fires
|
|
expect(reachability.pause?.eligible).toBe(1);
|
|
// But acknowledge (priority 1) also fires (health=healthy), blocking pause
|
|
expect(reachability.pause?.blocked).toBe(1);
|
|
expect(reachability.pause?.blockingBy).toContain("acknowledge");
|
|
});
|
|
|
|
it("Continue is eligible in 2 of 7 turns and selected in all 2", () => {
|
|
const reachability = global._exp40_reachability || {};
|
|
expect(reachability.continue?.eligible).toBe(2);
|
|
expect(reachability.continue?.selected).toBe(2);
|
|
});
|
|
|
|
it("all scenarios and turns are audited with full detail", () => {
|
|
const audits = global._exp40_audits || {};
|
|
for (const [scenarioName, turns] of Object.entries(audits)) {
|
|
for (const t of turns) {
|
|
expect(t.phase).toBeDefined();
|
|
expect(t.progress).toBeDefined();
|
|
expect(t.health).toBeDefined();
|
|
expect(["acknowledge", "clarify", "summarise", "pause", "continue"].includes(t.selectedBehaviour)).toBe(true);
|
|
expect(t.behaviours).toHaveLength(5); // all five behaviours checked
|
|
}
|
|
}
|
|
});
|
|
|
|
it("prints turn-level detail for the terminal long-investigation state", () => {
|
|
const audits = global._exp40_audits || {};
|
|
const terminalTurn = audits["long-investigation"][2];
|
|
expect(terminalTurn.phase).toBe("concluding");
|
|
expect(terminalTurn.health).toBe("healthy");
|
|
|
|
console.log("\n=== Terminal Turn Detail ===");
|
|
console.log(`Scenario: long-investigation, turn ${terminalTurn.turn}`);
|
|
console.log(` phase=${terminalTurn.phase}, progress=${terminalTurn.progress}, health=${terminalTurn.health}`);
|
|
for (const b of terminalTurn.behaviours) {
|
|
const marker = b.selected ? "▶" : b.eligible ? "⚠ blocked" : "✗";
|
|
console.log(` ${marker} ${b.behaviour}: eligible=${b.eligible}${b.blockedBy ? ` blocked by ${b.blockedBy}` : ""}`);
|
|
}
|
|
|
|
// Summarise should be eligible (concluding phase) but blocked
|
|
const summariseAudit = terminalTurn.behaviours.find(b => b.behaviour === "summarise");
|
|
expect(summariseAudit.eligible).toBe(true);
|
|
expect(summariseAudit.blockedBy).toBe("acknowledge");
|
|
|
|
// The selected behaviour is Acknowledge
|
|
expect(terminalTurn.selectedBehaviour).toBe("acknowledge");
|
|
});
|
|
|
|
it("prints turn-level detail for pause-eligible turn (contradictory turn 1)", () => {
|
|
const audits = global._exp40_audits || {};
|
|
const pauseEligibleTurn = audits["contradictory-evidence"][1];
|
|
expect(pauseEligibleTurn.phase).toBe("focusing");
|
|
expect(pauseEligibleTurn.progress).toBe("stalled");
|
|
|
|
console.log("\n=== Pause-Eligible Turn Detail ===");
|
|
console.log(`Scenario: contradictory-evidence, turn ${pauseEligibleTurn.turn}`);
|
|
console.log(` phase=${pauseEligibleTurn.phase}, progress=${pauseEligibleTurn.progress}, health=${pauseEligibleTurn.health}`);
|
|
for (const b of pauseEligibleTurn.behaviours) {
|
|
const marker = b.selected ? "▶" : b.eligible ? "⚠ blocked" : "✗";
|
|
console.log(` ${marker} ${b.behaviour}: eligible=${b.eligible}${b.blockedBy ? ` blocked by ${b.blockedBy}` : ""}`);
|
|
}
|
|
|
|
const pauseAudit = pauseEligibleTurn.behaviours.find(b => b.behaviour === "pause");
|
|
expect(pauseAudit.eligible).toBe(true);
|
|
expect(pauseAudit.blockedBy).toBe("acknowledge");
|
|
});
|
|
|
|
it("Clarify never eligible — no scenario produces too_broad health", () => {
|
|
const audits = global._exp40_audits || {};
|
|
for (const [scenarioName, turns] of Object.entries(audits)) {
|
|
for (const t of turns) {
|
|
expect(t.health).not.toBe("too_broad");
|
|
}
|
|
}
|
|
});
|
|
|
|
it("Clarify never eligible — no scenario produces orienting phase with < 3 obs", () => {
|
|
const audits = global._exp40_audits || {};
|
|
for (const [scenarioName, turns] of Object.entries(audits)) {
|
|
for (const t of turns) {
|
|
expect(t.phase).not.toBe("orienting");
|
|
}
|
|
}
|
|
});
|
|
|
|
it("Summarise eligible ONLY in the terminal long-investigation turn", () => {
|
|
const audits = global._exp40_audits || {};
|
|
let summariseTurns = [];
|
|
for (const [scenarioName, turns] of Object.entries(audits)) {
|
|
for (let i = 0; i < turns.length; i++) {
|
|
const sBeh = turns[i].behaviours.find(b => b.behaviour === "summarise");
|
|
if (sBeh.eligible) summariseTurns.push(`${scenarioName}-t${i}`);
|
|
}
|
|
}
|
|
expect(summariseTurns).toEqual(["long-investigation-t1", "long-investigation-t2"]);
|
|
});
|
|
|
|
it("Pause eligible ONLY in contradictory-evidence turn 1", () => {
|
|
const audits = global._exp40_audits || {};
|
|
let pauseTurns = [];
|
|
for (const [scenarioName, turns] of Object.entries(audits)) {
|
|
for (let i = 0; i < turns.length; i++) {
|
|
const pBeh = turns[i].behaviours.find(b => b.behaviour === "pause");
|
|
if (pBeh.eligible) pauseTurns.push(`${scenarioName}-t${i}`);
|
|
}
|
|
}
|
|
expect(pauseTurns).toEqual(["contradictory-evidence-t1"]);
|
|
});
|
|
|
|
it("terminal long-investigation turn IS included in the audit (confirms Experiment 39 coverage)", () => {
|
|
const audits = global._exp40_audits || {};
|
|
// Exp 39 tested turns: 0, 3, and 4 (complete). Turn 4 IS the terminal.
|
|
// Our getEvaluationScenarios maps to indices 0, 1, 2 — index 2 is the terminal state.
|
|
const terminalTurn = audits["long-investigation"][2];
|
|
expect(terminalTurn.selectedQuestion).toBeFalsy();
|
|
expect(terminalTurn.activeUnknownNodeId).toBeFalsy();
|
|
// Phase must be concluding for a terminal assessment
|
|
expect(terminalTurn.phase).toBe("concluding");
|
|
expect(terminalTurn.centralStatement).toContain("European market");
|
|
});
|
|
|
|
it("Acknowledge dominance caused by combination of broad eligibility, priority order, and scenario distribution", () => {
|
|
const reachability = global._exp40_reachability || {};
|
|
// Acknowledge eligible 5/7 — that's the broad eligibility factor
|
|
expect(reachability.acknowledge.eligible).toBe(5);
|
|
// It is selected every time it is eligible — that's the priority order factor
|
|
expect(reachability.acknowledge.blocked).toBe(0);
|
|
// 5/7 = 71% — scenario distribution (3 scenarios, most turns have healthy health)
|
|
const totalTurns = global._exp40_audits
|
|
? Object.values(global._exp40_audits).reduce((sum, arr) => sum + arr.length, 0)
|
|
: 7;
|
|
expect(reachability.acknowledge.eligible / totalTurns).toBeGreaterThan(0.5);
|
|
});
|
|
});
|
|
|
|
/* ── Scenario B: Synthetic isolated reachability ─────── */
|
|
|
|
describe("Synthetic reachability — each behaviour in isolation", () => {
|
|
it("Acknowledge is reachable when health=healthy and phase confidence ≠ low", () => {
|
|
const result = selectBehaviour(mkMinimal({
|
|
phase: { value: "exploring", confidence: "medium" },
|
|
conversationHealth: { value: "healthy", confidence: "high" }
|
|
}));
|
|
expect(result.behaviour).toBe("acknowledge");
|
|
});
|
|
|
|
it("Clarify is reachable when health=too_broad (no other rule fires first)", () => {
|
|
const result = selectBehaviour(mkMinimal({
|
|
phase: { value: "exploring", confidence: "low" }, // low → no acknowledge
|
|
conversationHealth: { value: "too_broad", confidence: "high" }
|
|
}));
|
|
expect(result.behaviour).toBe("clarify");
|
|
});
|
|
|
|
it("Clarify is reachable when phase=orienting and observationDensity < 3", () => {
|
|
const result = selectBehaviour(mkMinimal({
|
|
phase: { value: "orienting", confidence: "low", evidence: { observationDensity: 1 } },
|
|
conversationHealth: { value: "cannot_determine", confidence: "low" }
|
|
}));
|
|
expect(result.behaviour).toBe("clarify");
|
|
});
|
|
|
|
it("Summarise is reachable when phase=synthesising (no health trigger to block)", () => {
|
|
const result = selectBehaviour(mkMinimal({
|
|
phase: { value: "synthesising", confidence: "high" },
|
|
conversationHealth: { value: "cannot_determine", confidence: "low" } // no healthy → no acknowledge
|
|
}));
|
|
expect(result.behaviour).toBe("summarise");
|
|
});
|
|
|
|
it("Summarise is reachable when phase=concluding (no health trigger to block)", () => {
|
|
const result = selectBehaviour(mkMinimal({
|
|
phase: { value: "concluding", confidence: "high" },
|
|
conversationHealth: { value: "cannot_determine", confidence: "low" } // no healthy → no acknowledge
|
|
}));
|
|
expect(result.behaviour).toBe("summarise");
|
|
});
|
|
|
|
it("Summarise is reachable when resolvedNodeCount >= 3 and progress=steady (no health trigger)", () => {
|
|
const result = selectBehaviour(mkMinimal({
|
|
phase: { value: "focusing", confidence: "high", evidence: { resolvedNodeCount: 4 } },
|
|
progress: { value: "steady", confidence: "medium" },
|
|
conversationHealth: { value: "too_narrow", confidence: "low" } // no healthy → no acknowledge
|
|
}));
|
|
expect(result.behaviour).toBe("summarise");
|
|
});
|
|
|
|
it("Pause is reachable when phase=focusing + progress=stalled (no health trigger)", () => {
|
|
const result = selectBehaviour(mkMinimal({
|
|
phase: { value: "focusing", confidence: "high" },
|
|
progress: { value: "stalled", confidence: "high" },
|
|
conversationHealth: { value: "cannot_determine", confidence: "low" } // no healthy → no acknowledge
|
|
}));
|
|
expect(result.behaviour).toBe("pause");
|
|
});
|
|
|
|
it("Pause is reachable when health=user_overloaded", () => {
|
|
const result = selectBehaviour(mkMinimal({
|
|
phase: { value: "exploring", confidence: "medium" },
|
|
conversationHealth: { value: "user_overloaded", confidence: "medium" }
|
|
}));
|
|
expect(result.behaviour).toBe("pause");
|
|
});
|
|
|
|
it("Continue is reachable when no rule conditions match", () => {
|
|
const result = selectBehaviour(mkMinimal({
|
|
phase: { value: "exploring", confidence: "low" },
|
|
progress: { value: "cannot_determine", confidence: "low" },
|
|
conversationHealth: { value: "cannot_determine", confidence: "low" }
|
|
}));
|
|
expect(result.behaviour).toBe("continue");
|
|
});
|
|
|
|
it("each synthetic case confirms the rule is independently reachable", () => {
|
|
const tests = [
|
|
{ beh: "acknowledge", a: mkMinimal({ phase: { value: "focusing", confidence: "high" }, conversationHealth: { value: "healthy", confidence: "high" } }) },
|
|
{ beh: "clarify", a: mkMinimal({ phase: { value: "exploring", confidence: "low" }, conversationHealth: { value: "too_broad", confidence: "high" } }) },
|
|
{ beh: "summarise", a: mkMinimal({ phase: { value: "concluding", confidence: "high" }, conversationHealth: { value: "cannot_determine", confidence: "low" } }) },
|
|
{ beh: "pause", a: mkMinimal({ phase: { value: "focusing", confidence: "high" }, progress: { value: "stalled", confidence: "high" }, conversationHealth: { value: "cannot_determine", confidence: "low" } }) },
|
|
{ beh: "continue", a: mkMinimal({ phase: { value: "deepening", confidence: "medium" }, progress: { value: "steady", confidence: "medium" }, conversationHealth: { value: "cannot_determine", confidence: "low" } }) },
|
|
];
|
|
|
|
const results = {};
|
|
for (const t of tests) {
|
|
results[t.beh] = selectBehaviour(t.a).behaviour;
|
|
}
|
|
|
|
expect(results.acknowledge).toBe("acknowledge");
|
|
expect(results.clarify).toBe("clarify");
|
|
expect(results.summarise).toBe("summarise");
|
|
expect(results.pause).toBe("pause");
|
|
expect(results.continue).toBe("continue");
|
|
|
|
console.log("\n=== Synthetic Reachability ===");
|
|
for (const [beh, result] of Object.entries(results)) {
|
|
console.log(` ${beh}: ${result === beh ? "✓ reachable" : "✗ NOT reachable"}`);
|
|
}
|
|
});
|
|
});
|
|
|
|
/* ── Diagnostic correctness tests ────────────────────── */
|
|
|
|
describe("Diagnostic accuracy", () => {
|
|
it("selected behaviour matches the real selector output", () => {
|
|
const audits = global._exp40_audits || {};
|
|
let allMatch = true;
|
|
for (const [scenarioName, turns] of Object.entries(audits)) {
|
|
for (const t of turns) {
|
|
const input = buildAssessmentInput(getEvaluationScenarios()[scenarioName][t.turn]);
|
|
const assessment = assessInvestigationState(input);
|
|
const selectorResult = selectBehaviour(assessment);
|
|
if (selectorResult.behaviour !== t.selectedBehaviour) allMatch = false;
|
|
}
|
|
}
|
|
expect(allMatch).toBe(true);
|
|
});
|
|
|
|
it("identical inputs produce identical diagnostics every time", () => {
|
|
const scenarios = getEvaluationScenarios();
|
|
for (const [scenarioName, turns] of Object.entries(scenarios)) {
|
|
for (let i = 0; i < turns.length; i++) {
|
|
const input = buildAssessmentInput(turns[i]);
|
|
const assessment = assessInvestigationState(input);
|
|
const diagnostic1 = auditBehaviour(assessment);
|
|
const diagnostic2 = auditBehaviour(assessment);
|
|
expect(JSON.stringify(diagnostic1)).toBe(JSON.stringify(diagnostic2));
|
|
}
|
|
}
|
|
});
|
|
|
|
it("inputs are not mutated by audit function", () => {
|
|
const scenarios = getEvaluationScenarios();
|
|
for (const [scenarioName, turns] of Object.entries(scenarios)) {
|
|
for (let i = 0; i < turns.length; i++) {
|
|
const input = buildAssessmentInput(turns[i]);
|
|
const snapshot = JSON.stringify(input);
|
|
const assessment = assessInvestigationState(JSON.parse(snapshot));
|
|
auditBehaviour(assessment);
|
|
expect(JSON.stringify(input)).toBe(snapshot);
|
|
}
|
|
}
|
|
});
|
|
|
|
it("eligible behaviours are identified correctly for each turn", () => {
|
|
const audits = global._exp40_audits || {};
|
|
let correctCount = 0;
|
|
let totalCount = 0;
|
|
|
|
for (const [scenarioName, turns] of Object.entries(audits)) {
|
|
for (const t of turns) {
|
|
const input = buildAssessmentInput(getEvaluationScenarios()[scenarioName][t.turn]);
|
|
const assessment = assessInvestigationState(input);
|
|
|
|
for (const behaviour of ["acknowledge", "clarify", "summarise", "pause"]) {
|
|
totalCount++;
|
|
const expectedEligible = auditBehaviour(assessment).find(b => b.behaviour === behaviour).eligible;
|
|
if (expectedEligible === t.behaviours.find(b => b.behaviour === behaviour).eligible) {
|
|
correctCount++;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
expect(correctCount).toBe(totalCount);
|
|
});
|
|
|
|
it("blocked behaviours name the correct earlier winning rule", () => {
|
|
const audits = global._exp40_audits || {};
|
|
for (const [scenarioName, turns] of Object.entries(audits)) {
|
|
for (const t of turns) {
|
|
const blockedBehaviours = t.behaviours.filter(b => b.eligible && !b.selected);
|
|
for (const b of blockedBehaviours) {
|
|
// The blocking behaviour must have fired BEFORE this one in priority order
|
|
const priorityOrder = ["acknowledge", "clarify", "summarise", "pause"];
|
|
const blockerIdx = priorityOrder.indexOf(b.blockedBy);
|
|
const behIdx = priorityOrder.indexOf(b.behaviour);
|
|
expect(blockerIdx).toBeLessThan(behIdx);
|
|
}
|
|
}
|
|
}
|
|
});
|
|
|
|
it("all Experiment 39 turns are audited (7 total)", () => {
|
|
const audits = global._exp40_audits || {};
|
|
let totalTurns = 0;
|
|
for (const turns of Object.values(audits)) {
|
|
totalTurns += turns.length;
|
|
}
|
|
expect(totalTurns).toBe(7);
|
|
});
|
|
});
|
|
|
|
/* ── Classification summary ──────────────────────────── */
|
|
|
|
describe("Reachability classification", () => {
|
|
it("classifies each missing behaviour for the final report", () => {
|
|
const reachability = global._exp40_reachability || {};
|
|
|
|
// Summarise: eligible in real turn but blocked → eligible_but_blocked
|
|
expect(reachability.summarise.eligible).toBeGreaterThan(0);
|
|
expect(reachability.summarise.blocked).toBeGreaterThan(0);
|
|
|
|
// Clarify: never eligible in tested scenarios, reachable only synthetically
|
|
expect(reachability.clarify.eligible).toBe(0);
|
|
|
|
// Pause: eligible in real turn but blocked
|
|
expect(reachability.pause.eligible).toBeGreaterThan(0);
|
|
expect(reachability.pause.blocked).toBeGreaterThan(0);
|
|
|
|
console.log("\n=== Experiment 40 — Final Classifications ===");
|
|
console.log(`Summarise: eligible_but_blocked (blocked by acknowledge)`);
|
|
console.log(`Clarify: never_eligible_in_tested_scenarios (reachable only in synthetic case)`);
|
|
console.log(`Pause: eligible_but_blocked (blocked by acknowledge)`);
|
|
console.log(`Acknowledge: dominant because health=healthy is the most common state`);
|
|
|
|
global._exp40_classifications = {
|
|
summarise: "eligible_but_blocked",
|
|
clarify: "never_eligible_in_tested_scenarios",
|
|
pause: "eligible_but_blocked"
|
|
};
|
|
});
|
|
});
|
|
});
|