test(confidence-engine): capture deep-investigation checkpoints
This commit is contained in:
@@ -0,0 +1,228 @@
|
||||
/**
|
||||
* Deterministic checkpoint extraction tests for live-update-experiment-helper.cjs.
|
||||
*
|
||||
* Exercises the actual exported extractLiveExperimentCheckpoint seam — not a
|
||||
* duplicated simulation. All assertions verify observable behaviour of the
|
||||
* canonical extraction path.
|
||||
*
|
||||
* Tests:
|
||||
* 1. CU narrative field is "UNAVAILABLE" (NOT captured - synthesis is a separate seam)
|
||||
* 2. Structural counts are captured deterministically from graph state
|
||||
* 3. Timing fields pass through unchanged
|
||||
* 4. Missing optional evidence fields are "UNAVAILABLE" (not fabricated)
|
||||
* 5. Existing answer/category/validation checkpoint behaviour preserved
|
||||
* 6. Absent updated graph produces null structural checkpoint
|
||||
*/
|
||||
|
||||
const { extractLiveExperimentCheckpoint, _extractReasoningState } = require("./live-update-experiment-helper.cjs");
|
||||
|
||||
// ---- deriveCategory kept in test because it is NOT exported from the helper ----
|
||||
// (It is part of the answer-meaning extraction infrastructure, not the checkpoint seam)
|
||||
function deriveCategory(text) {
|
||||
if (!text || !text.trim()) return "none";
|
||||
const lower = text.toLowerCase();
|
||||
if (/normally\s+(?:avoid|skip|not\s+take|would\s+n't|can\'t)/i.test(lower) &&
|
||||
/(might|could|would\s+(?:accept|allow|take|do))/i.test(lower)) return "conditional_tradeoff";
|
||||
if (/but\s+i?\s*(don\'t|cannot|can\'t|won\'t|will\s+not)/i.test(lower)) return "qualified_support";
|
||||
if (/(hard\s+constraint|must\s+(?:not|never|always)|can\'?\s*t(?:o)\s*(?:not|be\s+able\s+to)|absolutely\s+cannot)/i.test(lower)) return "hard_constraint";
|
||||
if (/(must\s+(?:have|do|get)|absolutely\s+(?:need|require)|cannot\s+proceed\s+without)/i.test(lower)) return "strong_preference";
|
||||
if (/more\s+to\s+me|matters\s+more|higher\s+priority|top\s*priority/i.test(lower)) return "relative_importance";
|
||||
if (/support(s)?\b|confirm(s)?\b|validates?\b/i.test(lower)) return "supports_decision";
|
||||
if (/contradicts?\b|against\s+it\b|i\s*don\'?\s*t\s*(?:think\s+so|agree)\b/i.test(lower)) return "contradicts_decision";
|
||||
return "cannot_determine";
|
||||
}
|
||||
|
||||
function makeGraphFixture(nodes, edges) {
|
||||
return { nodes: nodes || [], edges: edges || [], resolvedNodeIds: [] };
|
||||
}
|
||||
|
||||
function successResult(mElapsed, e2e) {
|
||||
var nodes = [
|
||||
{ id: "a", kind: "unknown", label: "A", description: "Node A", status: "known" },
|
||||
{ id: "b", kind: "unknown", label: "B", description: "Node B", status: "supported" },
|
||||
{ id: "c", kind: "unknown", label: "C", description: "Node C", status: "unknown" },
|
||||
];
|
||||
var edges = [
|
||||
{ fromNodeId: "a", toNodeId: "b", relationship: "supports" },
|
||||
{ fromNodeId: "b", toNodeId: "c", relationship: "depends_on" },
|
||||
];
|
||||
return {
|
||||
success: true,
|
||||
stage: "update_applied",
|
||||
updatedSituationGraph: makeGraphFixture(nodes, edges),
|
||||
appliedGraph: null,
|
||||
proposal: {
|
||||
answerMeaning: {
|
||||
userSupportedMeaning: "User confirms supporting evidence matters",
|
||||
possibleInference: null,
|
||||
supportCategory: "supports_decision",
|
||||
resolutionGuidance: null,
|
||||
},
|
||||
selectedQuestion: { id: "q1", question: "Next question?" },
|
||||
},
|
||||
proposalValidation: { success: true, errors: [] },
|
||||
compatibilityApplied: true,
|
||||
behaviourSelection: "deterministic-choice",
|
||||
reasoningState: {
|
||||
turnCount: 2,
|
||||
health: "healthy",
|
||||
phase: "deep-investigation",
|
||||
progress: "resolving unknowns",
|
||||
unknownStatuses: [{ id: "c", status: "unknown" }],
|
||||
},
|
||||
modelElapsedMs: mElapsed,
|
||||
endToEndElapsedMs: e2e,
|
||||
providerGenerateCalls: 1,
|
||||
};
|
||||
}
|
||||
|
||||
function failureResult() {
|
||||
return {
|
||||
success: false,
|
||||
stage: "proposal_validation",
|
||||
proposal: null,
|
||||
proposalValidation: { success: false, errors: ["invalid proposal"] },
|
||||
compatibilityApplied: undefined,
|
||||
behaviourSelection: null,
|
||||
reasoningState: null,
|
||||
modelElapsedMs: null,
|
||||
endToEndElapsedMs: null,
|
||||
providerGenerateCalls: 0,
|
||||
updatedSituationGraph: null,
|
||||
appliedGraph: null,
|
||||
};
|
||||
}
|
||||
|
||||
// ---- assertion helpers ----
|
||||
var failures = 0;
|
||||
function assert(cond, msg) {
|
||||
if (!cond) { console.error("FAIL:", msg); failures++; }
|
||||
else { console.log("PASS:", msg); }
|
||||
}
|
||||
function eq(a, e, msg) {
|
||||
var aS = JSON.stringify(a);
|
||||
var eS = JSON.stringify(e);
|
||||
if (aS !== eS) { console.error("FAIL [" + msg + "]: expected " + eS + ", got " + aS); failures++; }
|
||||
else { console.log("PASS [" + msg + "]: values match"); }
|
||||
}
|
||||
|
||||
// ---- SUCCESS PATH TESTS ----
|
||||
console.log("\n=== CHECKPOINT EXTRACTION - SUCCESS PATH ===\n");
|
||||
|
||||
var successParams = {
|
||||
graph: makeGraphFixture([
|
||||
{ id: "a", kind: "unknown", label: "A", description: "Node A", status: "known" },
|
||||
{ id: "b", kind: "unknown", label: "B", description: "Node B", status: "supported" },
|
||||
], [
|
||||
{ fromNodeId: "a", toNodeId: "b", relationship: "supports" },
|
||||
]),
|
||||
previousQuestion: "Is risk a hard constraint?",
|
||||
answer: "Risk matters more to me.",
|
||||
};
|
||||
|
||||
var checkpoint = extractLiveExperimentCheckpoint(successResult(123, 456), successParams);
|
||||
|
||||
// 1. CU text is "UNAVAILABLE" (not captured - separate seam)
|
||||
assert(checkpoint.currentUnderstanding === "UNAVAILABLE", "CU narrative is UNAVAILABLE (not captured)");
|
||||
|
||||
// 2. Structural counts match updated graph fixture
|
||||
eq(checkpoint.graphCheckpoint.nodeCount, 3, "nodeCount matches updated graph nodes length");
|
||||
eq(checkpoint.graphCheckpoint.edgeCount, 2, "edgeCount matches updated graph edges length");
|
||||
assert(Array.isArray(checkpoint.graphCheckpoint.resolvedNodeIds), "resolvedNodeIds is array");
|
||||
|
||||
// 3. Timing fields present and correct type
|
||||
assert(typeof checkpoint.modelElapsedMs === "number", "modelElapsedMs is number");
|
||||
eq(checkpoint.modelElapsedMs, 123, "modelElapsedMs value matches provider timing");
|
||||
assert(typeof checkpoint.endToEndElapsedMs === "number", "endToEndElapsedMs is number");
|
||||
eq(checkpoint.endToEndElapsedMs, 456, "endToEndElapsedMs value matches wrapper timing");
|
||||
|
||||
// 4. Findings count from input graph (not production path)
|
||||
eq(checkpoint.findingsCount.total, 2, "findings total matches input graph length");
|
||||
assert(checkpoint.findingsCount.eligible === "UNAVAILABLE", "eligible finding count is UNAVAILABLE");
|
||||
|
||||
// 5. CU evidence propositions UNAVAILABLE
|
||||
assert(checkpoint.eligibleFindingPropositions === "UNAVAILABLE", "eligibleFindingPropositions is UNAVAILABLE");
|
||||
|
||||
// 6. Answer meaning extraction preserved
|
||||
eq(checkpoint.userSupportedMeaning, "User confirms supporting evidence matters", "userSupportedMeaning extracted correctly");
|
||||
eq(checkpoint.possibleInference, null, "possibleInference is null when absent");
|
||||
eq(checkpoint.rawAnswerCategory, "relative_importance", "raw answer category derived from 'matters more'");
|
||||
|
||||
// 7. Proposal validation preserved
|
||||
assert(checkpoint.proposalValidation.success === true, "proposalValidation.success is true on success");
|
||||
eq(Object.keys(checkpoint.proposalValidation).length, 2, "proposalValidation has {success, errors} keys");
|
||||
|
||||
// 8. Compatibility guard captured correctly
|
||||
eq(checkpoint.compatibilityGuard.passed, true, "compatibilityGuard.passed reflects production result");
|
||||
eq(checkpoint.compatibilityGuard.warnings.length, 0, "compatibilityGuard warnings empty");
|
||||
eq(checkpoint.compatibilityGuard.violations.length, 0, "compatibilityGuard violations empty");
|
||||
|
||||
// 9. Graph mutation null when appliedGraph absent (existing behaviour)
|
||||
assert(checkpoint.graphMutation === null, "graphMutation is null when appliedGraph absent");
|
||||
|
||||
// 10. Reasoning state preserved
|
||||
assert(checkpoint.reasoningState !== null, "reasoningState present on success");
|
||||
eq(checkpoint.reasoningState.turnCount, 2, "reasoningState turnCount preserved");
|
||||
eq(checkpoint.reasoningState.health, "healthy", "reasoningState health preserved");
|
||||
eq(checkpoint.reasoningState.phase, "deep-investigation", "reasoningState phase preserved");
|
||||
assert(Array.isArray(checkpoint.reasoningState.unknownStatuses), "unknownStatuses is array");
|
||||
eq(checkpoint.reasoningState.unknownStatuses.length, 1, "one unknown status tracked");
|
||||
|
||||
// 11. selectedQuestion and behaviourSelection captured
|
||||
assert(checkpoint.selectedQuestion !== null, "selectedQuestion captured");
|
||||
eq(checkpoint.selectedQuestion.question, "Next question?", "selectedQuestion text correct");
|
||||
eq(checkpoint.behaviourSelection, "deterministic-choice", "behaviourSelection captured correctly");
|
||||
|
||||
// 12. Provider call tracking
|
||||
eq(checkpoint.providerGenerateCalls, 1, "providerGenerateCalls incremented once");
|
||||
|
||||
// ---- ABSENT GRAPH PATH TESTS (spec item #6) ----
|
||||
console.log("\n=== CHECKPOINT EXTRACTION - ABSENT UPDATED GRAPH ===\n");
|
||||
|
||||
// Use a result that has no updatedSituationGraph to test null structural checkpoint
|
||||
var absentResult = {
|
||||
success: true,
|
||||
stage: "proposal_validated",
|
||||
updatedSituationGraph: null,
|
||||
appliedGraph: null,
|
||||
proposal: { answerMeaning: { userSupportedMeaning: "", possibleInference: null }, selectedQuestion: null },
|
||||
proposalValidation: { success: true, errors: [] },
|
||||
compatibilityApplied: false,
|
||||
behaviourSelection: null,
|
||||
reasoningState: null,
|
||||
modelElapsedMs: 200,
|
||||
endToEndElapsedMs: 800,
|
||||
providerGenerateCalls: 1,
|
||||
};
|
||||
|
||||
var absentGraphParams = {
|
||||
graph: null,
|
||||
previousQuestion: "Is risk a hard constraint?",
|
||||
answer: "Risk matters more to me.",
|
||||
};
|
||||
|
||||
var absentCheckpoint = extractLiveExperimentCheckpoint(absentResult, absentGraphParams);
|
||||
|
||||
// Structural checkpoint is null when updatedSituationGraph absent
|
||||
assert(absentCheckpoint.graphCheckpoint === null, "graphCheckpoint is null when updated graph absent");
|
||||
|
||||
// CU still unavailable without production contract change
|
||||
assert(absentCheckpoint.currentUnderstanding === "UNAVAILABLE", "CU remains UNAVAILABLE with absent graph");
|
||||
|
||||
// Evidence propositions still unavailable
|
||||
assert(absentCheckpoint.eligibleFindingPropositions === "UNAVAILABLE", "eligble props still UNAVAILABLE");
|
||||
|
||||
// Timing fields still pass through
|
||||
eq(absentCheckpoint.modelElapsedMs, 200, "modelElapsedMs passes through in absent-graph path");
|
||||
eq(absentCheckpoint.endToEndElapsedMs, 800, "endToEndElapsedMs passes through in absent-graph path");
|
||||
|
||||
// Reasoning state is null when reasoningState absent in result
|
||||
assert(absentCheckpoint.reasoningState === null, "reasoningState is null when absent in result");
|
||||
|
||||
// Test _extractReasoningState directly with null input
|
||||
var nullRs = _extractReasoningState(null);
|
||||
assert(nullRs === null, "_extractReasoningState returns null for null input");
|
||||
|
||||
// ---- exit with failure count ----
|
||||
console.log("\n--- Total failures: " + failures + " ---\n");
|
||||
process.exit(failures > 0 ? 1 : 0);
|
||||
@@ -31,85 +31,78 @@ function assertRequiredEnv(name) {
|
||||
}
|
||||
|
||||
/**
|
||||
* Run one live experiment through the real updateCase() production path.
|
||||
* Extract checkpoint data from a raw updateCase() result object.
|
||||
*
|
||||
* @param {object} params
|
||||
* @param {object} params.graph — SituationGraph fixture (must match situationGraphSchema)
|
||||
* @param {string} params.previousQuestion
|
||||
* @param {string} params.answer
|
||||
* @returns {Promise<object>} Checkpoints:
|
||||
* - userSupportedMeaning — extracted from answer meaning
|
||||
* - possibleInference — extracted from answer meaning
|
||||
* - rawAnswerCategory — deterministic category derived from raw answer text
|
||||
* - proposedMeaningCategory — deterministic category derived from userSupportedMeaning
|
||||
* - proposalValidation — { success, errors }
|
||||
* - compatibilityGuard — { passed, warnings, violations }
|
||||
* - graphMutation — { nodes: [...], edges: [...] } | null
|
||||
* - selectedQuestion — { id?, question? } | null
|
||||
* - behaviourSelection — string | null
|
||||
* - reasoningState — { turnCount, health, phase, progress, unknownStatuses }
|
||||
* Pure function — no side effects, no I/O. Intended for use both by
|
||||
* `runLiveExperiment()` (apparatus wrapper) and by deterministic tests
|
||||
* that pass mock results directly.
|
||||
*
|
||||
* @param {object} result — raw result from updateCase()
|
||||
* @param {object} params — { graph?, answer? }
|
||||
* - graph: SituationGraph input fixture (for findingsCount total derived from input)
|
||||
* - answer: previous question text for deriveCategory fallback
|
||||
* @returns {object} checkpoint object per spec contract
|
||||
*/
|
||||
async function runLiveExperiment({ graph, previousQuestion, answer }) {
|
||||
const baseUrl = assertRequiredEnv("OLLAMA_BASE_URL");
|
||||
const model = assertRequiredEnv("OLLAMA_MODEL");
|
||||
function extractLiveExperimentCheckpoint(result, params) {
|
||||
const answerMeaning = result?.proposal?.answerMeaning || {};
|
||||
|
||||
if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") {
|
||||
throw new Error(
|
||||
`Live experiment harness refuses to use localhost fallback. ` +
|
||||
`OLLAMA_BASE_URL=${baseUrl}. Configure a real host in .env.local.`
|
||||
);
|
||||
}
|
||||
|
||||
const { updateCase } = await import(
|
||||
"../../lib/graph/orchestrator.js"
|
||||
);
|
||||
|
||||
const { getProvider } = await import("../../lib/llm/provider.js");
|
||||
const provider = getProvider();
|
||||
let modelElapsedMs = null;
|
||||
let providerGenerateCalls = 0;
|
||||
|
||||
const timedProvider = {
|
||||
async generateReconstruction(prompt, modelName) {
|
||||
providerGenerateCalls++;
|
||||
const startedAt = Date.now();
|
||||
const response = await provider.generateReconstruction(prompt, modelName);
|
||||
modelElapsedMs = Date.now() - startedAt;
|
||||
return response;
|
||||
},
|
||||
};
|
||||
|
||||
const endToEndStartedAt = Date.now();
|
||||
const result = await updateCase(
|
||||
{ situationGraph: graph, previousQuestion, answer },
|
||||
{ provider: timedProvider },
|
||||
);
|
||||
const endToEndElapsedMs = Date.now() - endToEndStartedAt;
|
||||
|
||||
// Extract the standard checkpoints
|
||||
const answerMeaning = result?.proposal?.answerMeaning || {};
|
||||
const rawAnswerCat = _deriveCategory(answer);
|
||||
const supportedCat = _deriveCategory(answerMeaning.userSupportedMeaning || "");
|
||||
// Graph structural checkpoint (available from updatedSituationGraph only when present)
|
||||
const updatedGraph = result?.updatedSituationGraph;
|
||||
const graphCheckpoint = updatedGraph
|
||||
? {
|
||||
nodeCount: updatedGraph.nodes?.length ?? 0,
|
||||
edgeCount: updatedGraph.edges?.length ?? 0,
|
||||
resolvedNodeIds: updatedGraph.resolvedNodeIds ?? [],
|
||||
}
|
||||
: null;
|
||||
|
||||
return {
|
||||
modelElapsedMs,
|
||||
endToEndElapsedMs,
|
||||
providerGenerateCalls,
|
||||
modelName: model,
|
||||
// ── timing (available via wrapper) ────────────────────
|
||||
modelElapsedMs: result?.modelElapsedMs !== undefined ? result.modelElapsedMs : null,
|
||||
endToEndElapsedMs: result?.endToEndElapsedMs !== undefined ? result.endToEndElapsedMs : null,
|
||||
providerGenerateCalls: result != null && result.providerGenerateCalls != null ? result.providerGenerateCalls : 0,
|
||||
modelName: result?.modelName || null,
|
||||
|
||||
// ── answer meaning extraction ─────────────────────────
|
||||
userSupportedMeaning: answerMeaning.userSupportedMeaning ?? null,
|
||||
possibleInference: answerMeaning.possibleInference ?? null,
|
||||
rawAnswerCategory: rawAnswerCat,
|
||||
proposedMeaningCategory: supportedCat,
|
||||
proposalValidation: result?.proposalValidation ?? { success: false, errors: [] },
|
||||
compatibilityGuard: result?.compatibilityApplied !== undefined
|
||||
rawAnswerCategory: _deriveCategory(params?.answer),
|
||||
proposedMeaningCategory: _deriveCategory(answerMeaning.userSupportedMeaning || ""),
|
||||
|
||||
// ── validation / guard checkpoints ───────────────────
|
||||
proposalValidation: result?.proposalValidation ?? { success: false, errors: [] },
|
||||
compatibilityGuard: result?.compatibilityApplied !== undefined
|
||||
? { passed: result.compatibilityApplied, warnings: [], violations: [] }
|
||||
: { passed: false, warnings: [], violations: [] },
|
||||
graphMutation: result?.appliedGraph
|
||||
|
||||
// ── graph mutation checkpoint (from appliedGraph) ────
|
||||
graphMutation: result?.appliedGraph
|
||||
? { nodes: result.appliedGraph.nodes ?? null, edges: result.appliedGraph.edges ?? null }
|
||||
: null,
|
||||
selectedQuestion: result?.proposal?.selectedQuestion ?? null,
|
||||
behaviourSelection: result?.behaviourSelection ?? null,
|
||||
reasoningState: _extractReasoningState(result),
|
||||
|
||||
// ── graph structural checkpoint (available from updatedSituationGraph) ─
|
||||
graphCheckpoint,
|
||||
|
||||
// ── findings checkpoint (from input graph fixture, not production path) ─
|
||||
findingsCount: {
|
||||
total: (params?.graph && Array.isArray(params.graph))
|
||||
? params.graph.length
|
||||
: (params?.graph && params.graph.nodes != null)
|
||||
? params.graph.nodes.length
|
||||
: 0,
|
||||
eligible: "UNAVAILABLE", // not canonically observable without production contract change
|
||||
},
|
||||
|
||||
// ── CU narrative (NOT captured — synthesis is a separate seam from updateCase()) ─
|
||||
currentUnderstanding: "UNAVAILABLE",
|
||||
|
||||
// ── candidate evidence (UNAVAILABLE without changing production code) ────────
|
||||
eligibleFindingPropositions: "UNAVAILABLE",
|
||||
|
||||
// ── helper internals (for diagnostic / test verification) ────────────────────
|
||||
selectedQuestion: result?.proposal?.selectedQuestion ?? null,
|
||||
behaviourSelection: result?.behaviourSelection ?? null,
|
||||
reasoningState: _extractReasoningState(result),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -158,6 +151,10 @@ function _deriveCategory(text) {
|
||||
return "cannot_determine";
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract reasoning state from a raw updateCase() result object.
|
||||
* Exported for deterministic test verification of preserved behaviour.
|
||||
*/
|
||||
function _extractReasoningState(result) {
|
||||
const rs = result?.reasoningState;
|
||||
if (!rs) return null;
|
||||
@@ -170,4 +167,47 @@ function _extractReasoningState(result) {
|
||||
};
|
||||
}
|
||||
|
||||
module.exports = { runLiveExperiment };
|
||||
/**
|
||||
* Run one live experiment through the real updateCase() production path.
|
||||
*/
|
||||
async function runLiveExperiment({ graph, previousQuestion, answer }) {
|
||||
const baseUrl = assertRequiredEnv("OLLAMA_BASE_URL");
|
||||
const model = assertRequiredEnv("OLLAMA_MODEL");
|
||||
|
||||
if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") {
|
||||
throw new Error(
|
||||
`Live experiment harness refuses to use localhost fallback. ` +
|
||||
`OLLAMA_BASE_URL=${baseUrl}. Configure a real host in .env.local.`
|
||||
);
|
||||
}
|
||||
|
||||
const { updateCase } = await import(
|
||||
"../../lib/graph/orchestrator.js"
|
||||
);
|
||||
|
||||
const { getProvider } = await import("../../lib/llm/provider.js");
|
||||
const provider = getProvider();
|
||||
let modelElapsedMs = null;
|
||||
let providerGenerateCalls = 0;
|
||||
|
||||
const timedProvider = {
|
||||
async generateReconstruction(prompt, modelName) {
|
||||
providerGenerateCalls++;
|
||||
const startedAt = Date.now();
|
||||
const response = await provider.generateReconstruction(prompt, modelName);
|
||||
modelElapsedMs = Date.now() - startedAt;
|
||||
return response;
|
||||
},
|
||||
};
|
||||
|
||||
const endToEndStartedAt = Date.now();
|
||||
const result = await updateCase(
|
||||
{ situationGraph: graph, previousQuestion, answer },
|
||||
{ provider: timedProvider },
|
||||
);
|
||||
const endToEndElapsedMs = Date.now() - endToEndStartedAt;
|
||||
|
||||
return extractLiveExperimentCheckpoint(result, { graph, answer });
|
||||
}
|
||||
|
||||
module.exports = { runLiveExperiment, extractLiveExperimentCheckpoint, _extractReasoningState };
|
||||
|
||||
Reference in New Issue
Block a user