robbond
99f200303a
Historical contributions for a different Open Question node no longer suppress fresh focused deconstruction of the current target. Only contributions whose targetNodeId matches the current target are counted towards suppressing redundant deconstruction.
517 lines
20 KiB
JavaScript
517 lines
20 KiB
JavaScript
/**
|
|
* Canonical live-update experiment harness.
|
|
*
|
|
* Loads .env.local, validates Ollama configuration, invokes updateCase()
|
|
* through the real production path, and captures the standard reasoning
|
|
* checkpoints needed by reasoning experiments.
|
|
*
|
|
* Usage (from a test file):
|
|
* const { runLiveExperiment } = require("./tests/graph/live-update-experiment-helper.cjs");
|
|
*
|
|
* const result = await runLiveExperiment({
|
|
* graph: /* SituationGraph fixture *\/,
|
|
* previousQuestion: "Is risk a hard constraint?",
|
|
* answer: "Risk matters more to me.",
|
|
* });
|
|
*/
|
|
|
|
const dotenv = require("dotenv");
|
|
dotenv.config({ path: ".env.local" });
|
|
|
|
// Re-export production helper for apparatus use — avoids fabricating Finding propositions.
|
|
const { deriveFindingsFromContributions } = require("../../lib/graph/finding-helpers.js");
|
|
|
|
function assertRequiredEnv(name) {
|
|
const value = process.env[name];
|
|
if (!value) {
|
|
throw new Error(
|
|
`Live experiment requires ${name}. Set it in .env.local.\n` +
|
|
`Found: OLLAMA_BASE_URL=${process.env.OLLAMA_BASE_URL ?? "(missing)"}, ` +
|
|
`OLLAMA_MODEL=${process.env.OLLAMA_MODEL ?? "(missing)"}`
|
|
);
|
|
}
|
|
return value;
|
|
}
|
|
|
|
/**
|
|
* Extract checkpoint data from a raw updateCase() result object.
|
|
*
|
|
* Pure function — no side effects, no I/O. Intended for use both by
|
|
* `runLiveExperiment()` (apparatus wrapper) and by deterministic tests
|
|
* that pass mock results directly.
|
|
*
|
|
* @param {object} result — raw result from updateCase()
|
|
* @param {object} params — { graph?, answer? }
|
|
* - graph: SituationGraph input fixture (for findingsCount total derived from input)
|
|
* - answer: previous question text for deriveCategory fallback
|
|
* @returns {object} checkpoint object per spec contract
|
|
*/
|
|
function extractLiveExperimentCheckpoint(result, params) {
|
|
const answerMeaning = result?.proposal?.answerMeaning || {};
|
|
|
|
// Graph structural checkpoint (available from updatedSituationGraph only when present)
|
|
const updatedGraph = result?.updatedSituationGraph;
|
|
const graphCheckpoint = updatedGraph
|
|
? {
|
|
nodeCount: updatedGraph.nodes?.length ?? 0,
|
|
edgeCount: updatedGraph.edges?.length ?? 0,
|
|
resolvedNodeIds: updatedGraph.resolvedNodeIds ?? [],
|
|
}
|
|
: null;
|
|
|
|
return {
|
|
// ── timing (available via wrapper) ────────────────────
|
|
modelElapsedMs: result?.modelElapsedMs !== undefined ? result.modelElapsedMs : null,
|
|
endToEndElapsedMs: result?.endToEndElapsedMs !== undefined ? result.endToEndElapsedMs : null,
|
|
providerGenerateCalls: result != null && result.providerGenerateCalls != null ? result.providerGenerateCalls : 0,
|
|
modelName: result?.modelName || null,
|
|
|
|
// ── answer meaning extraction ─────────────────────────
|
|
userSupportedMeaning: answerMeaning.userSupportedMeaning ?? null,
|
|
possibleInference: answerMeaning.possibleInference ?? null,
|
|
rawAnswerCategory: _deriveCategory(params?.answer),
|
|
proposedMeaningCategory: _deriveCategory(answerMeaning.userSupportedMeaning || ""),
|
|
|
|
// ── validation / guard checkpoints ───────────────────
|
|
proposalValidation: result?.proposalValidation ?? { success: false, errors: [] },
|
|
compatibilityGuard: result?.compatibilityApplied !== undefined
|
|
? { passed: result.compatibilityApplied, warnings: [], violations: [] }
|
|
: { passed: false, warnings: [], violations: [] },
|
|
|
|
// ── graph mutation checkpoint (from appliedGraph) ────
|
|
graphMutation: result?.appliedGraph
|
|
? { nodes: result.appliedGraph.nodes ?? null, edges: result.appliedGraph.edges ?? null }
|
|
: null,
|
|
|
|
// ── graph structural checkpoint (available from updatedSituationGraph) ─
|
|
graphCheckpoint,
|
|
|
|
// ── findings checkpoint (from input graph fixture, not production path) ─
|
|
findingsCount: {
|
|
total: (params?.graph && Array.isArray(params.graph))
|
|
? params.graph.length
|
|
: (params?.graph && params.graph.nodes != null)
|
|
? params.graph.nodes.length
|
|
: 0,
|
|
eligible: "UNAVAILABLE", // not canonically observable without production contract change
|
|
},
|
|
|
|
// ── CU narrative (NOT captured — synthesis is a separate seam from updateCase()) ─
|
|
currentUnderstanding: "UNAVAILABLE",
|
|
|
|
// ── candidate evidence (UNAVAILABLE without changing production code) ────────
|
|
eligibleFindingPropositions: "UNAVAILABLE",
|
|
|
|
// ── helper internals (for diagnostic / test verification) ────────────────────
|
|
selectedQuestion: result?.proposal?.selectedQuestion ?? null,
|
|
behaviourSelection: result?.behaviourSelection ?? null,
|
|
reasoningState: _extractReasoningState(result),
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Minimal deterministic category derivation from free-text (mirrors the production pipeline).
|
|
*/
|
|
function _deriveCategory(text) {
|
|
if (!text || !text.trim()) return "none";
|
|
|
|
const lower = text.toLowerCase();
|
|
|
|
// Conditional patterns
|
|
if (/normally\s+(?:avoid|skip|not\s+take|would\s+n't|can\'t)/i.test(lower) &&
|
|
/(might|could|would\s+(?:accept|allow|take|do))/i.test(lower)) {
|
|
return "conditional_tradeoff";
|
|
}
|
|
|
|
// Contrast / but patterns
|
|
if (/but\s+i?\s*(don\'t|cannot|can\'t|won\'t|will\s+not)/i.test(lower)) {
|
|
return "qualified_support";
|
|
}
|
|
|
|
// Explicit hard constraint
|
|
if (/(hard\s+constraint|must\s+(?:not|never|always)|can\'?\s*t(?:o)\s*(?:not|be\s+able\s+to)|absolutely\s+cannot)/i.test(lower)) {
|
|
return "hard_constraint";
|
|
}
|
|
|
|
// Strong preference / must positive
|
|
if (/(must\s+(?:have|do|get)|absolutely\s+(?:need|require)|cannot\s+proceed\s+without)/i.test(lower)) {
|
|
return "strong_preference";
|
|
}
|
|
|
|
// Relative importance
|
|
if (/more\s+to\s+me|matters\s+more|higher\s+priority|top\s*priority/i.test(lower)) {
|
|
return "relative_importance";
|
|
}
|
|
|
|
// Support / contraindicate
|
|
if (/support(s)?\b|confirm(s)?\b|validates?\b/i.test(lower)) {
|
|
return "supports_decision";
|
|
}
|
|
if (/contradicts?\b|against\s+it\b|i\s*don\'?\s*t\s*(?:think\s+so|agree)\b/i.test(lower)) {
|
|
return "contradicts_decision";
|
|
}
|
|
|
|
return "cannot_determine";
|
|
}
|
|
|
|
/**
|
|
* Extract reasoning state from a raw updateCase() result object.
|
|
* Exported for deterministic test verification of preserved behaviour.
|
|
*/
|
|
function _extractReasoningState(result) {
|
|
const rs = result?.reasoningState;
|
|
if (!rs) return null;
|
|
return {
|
|
turnCount: rs.turnCount ?? null,
|
|
health: rs.health ?? null,
|
|
phase: rs.phase ?? null,
|
|
progress: rs.progress ?? null,
|
|
unknownStatuses: (rs.unknownStatuses || []).map(s => s.id + ":" + s.status),
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Run one live experiment through the real updateCase() production path.
|
|
*/
|
|
async function runLiveExperiment({ graph, previousQuestion, answer, findings = [] }) {
|
|
const baseUrl = assertRequiredEnv("OLLAMA_BASE_URL");
|
|
const model = assertRequiredEnv("OLLAMA_MODEL");
|
|
|
|
if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") {
|
|
throw new Error(
|
|
`Live experiment harness refuses to use localhost fallback. ` +
|
|
`OLLAMA_BASE_URL=${baseUrl}. Configure a real host in .env.local.`
|
|
);
|
|
}
|
|
|
|
const { updateCase } = await import(
|
|
"../../lib/graph/orchestrator.js"
|
|
);
|
|
|
|
const { getProvider } = await import("../../lib/llm/provider.js");
|
|
const provider = getProvider();
|
|
let modelElapsedMs = null;
|
|
let providerGenerateCalls = 0;
|
|
|
|
const timedProvider = {
|
|
async generateReconstruction(prompt, modelName, outputSchema) {
|
|
providerGenerateCalls++;
|
|
const startedAt = Date.now();
|
|
const response = await provider.generateReconstruction(prompt, modelName, outputSchema);
|
|
modelElapsedMs = Date.now() - startedAt;
|
|
return response;
|
|
},
|
|
};
|
|
|
|
const endToEndStartedAt = Date.now();
|
|
const result = await updateCase(
|
|
{ situationGraph: graph, previousQuestion, answer },
|
|
{ provider: timedProvider },
|
|
);
|
|
const endToEndElapsedMs = Date.now() - endToEndStartedAt;
|
|
|
|
// ── accumulated state for longitudinal multi-episode use ───
|
|
const updatedGraph = result?.updatedSituationGraph ?? null;
|
|
const postEpisodeFs = Array.isArray(result?.findings) ? [...result.findings] : [];
|
|
|
|
return {
|
|
/* ── CU narrative (UNAVAILABLE — synthesis is a separate seam from updateCase()) ─── */
|
|
currentUnderstanding: "UNAVAILABLE",
|
|
|
|
/* ── updatedSituationGraph (NEW for longitudinal state feeding) ─── */
|
|
updatedSituationGraph: updatedGraph,
|
|
|
|
/* ── postEpisodeFindings (NEW — accumulated Findings exposed for Episode N+1) ─── */
|
|
postEpisodeFindings: postEpisodeFs,
|
|
|
|
/* ── graph checkpoint (from production result) ─── */
|
|
graphCheckpoint: updatedGraph
|
|
? {
|
|
nodeCount: updatedGraph.nodes?.length ?? 0,
|
|
edgeCount: updatedGraph.edges?.length ?? 0,
|
|
resolvedNodeIds: updatedGraph.resolvedNodeIds ?? [],
|
|
}
|
|
: { nodeCount: 0, edgeCount: 0, resolvedNodeIds: [] },
|
|
|
|
/* ── findings checkpoint (count from accumulated Findings) ─── */
|
|
findingsCheckpoint: {
|
|
total: postEpisodeFs.length,
|
|
eligible: "UNAVAILABLE",
|
|
},
|
|
|
|
/* ── Timing ─── */
|
|
modelElapsedMs: result?.modelElapsedMs !== undefined ? result.modelElapsedMs : null,
|
|
endToEndElapsedMs: result?.endToEndElapsedMs !== undefined ? result.endToEndElapsedMs : null,
|
|
providerGenerateCalls: result != null && result.providerGenerateCalls != null ? result.providerGenerateCalls : 0,
|
|
modelName: result?.modelName || null,
|
|
|
|
/* ── answer meaning (from production result) ─── */
|
|
userSupportedMeaning: result?.proposal?.answerMeaning?.userSupportedMeaning ?? null,
|
|
possibleInference: result?.proposal?.answerMeaning?.possibleInference ?? null,
|
|
|
|
/* ── helper internals (for diagnostic / test verification) ─── */
|
|
selectedQuestion: result?.proposal?.selectedQuestion ?? null,
|
|
behaviourSelection: result?.behaviourSelection ?? null,
|
|
reasoningState: _extractReasoningState(result),
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Compose the canonical post-episode deep-investigation checkpoint sequence:
|
|
* updateCase → synthesizeCurrentUnderstanding
|
|
*
|
|
* Mirrors the production flow in ScenarioForm.handleDoneForNowPromotion / executeEpisodeDone:
|
|
* 1. Call the episode done server seam (POST /api/cases/update, episodeMode=true)
|
|
* which updates the SituationGraph and accumulates Findings
|
|
* 2. Use the updated graph + accumulated findings to synthesize Current Understanding
|
|
* via the same seam as production (POST /api/cases/synthesis → synthesizeCurrentUnderstanding)
|
|
*
|
|
* Returns a combined checkpoint containing:
|
|
* - extractLiveExperimentCheckpoint(...) fields from the episode done result
|
|
* - currentUnderstanding from the synthesis result
|
|
* - findingsCheckpoint with accurate post-episode accumulated findings count
|
|
*/
|
|
/**
|
|
* Compose the canonical post-episode deep-investigation checkpoint sequence.
|
|
* Uses production-focused deconstruct for Contribution derivation (not synthetic raw-answer path).
|
|
*/
|
|
async function composeDeepInvestigationCheckpoint({ situationGraph, previousQuestion, answer, findings = [], contributions = [], targetNodeId, episodeMode }) {
|
|
dotenv.config({ path: ".env.local" });
|
|
|
|
const baseUrl = assertRequiredEnv("OLLAMA_BASE_URL");
|
|
const model = assertRequiredEnv("OLLAMA_MODEL");
|
|
|
|
if (baseUrl === "http://localhost:11434" || baseUrl === "http://127.0.0.1:11434") {
|
|
throw new Error(
|
|
`Live experiment harness refuses to use localhost fallback. ` +
|
|
`OLLAMA_BASE_URL=${baseUrl}. Configure a real host in .env.local.`
|
|
);
|
|
}
|
|
|
|
/* ── Step 1: episode done (mirrors executeEpisodeDone server seam) ─── */
|
|
const { updateCase, reconsiderCompletedEpisode } = await import(
|
|
"../../lib/graph/orchestrator.js"
|
|
);
|
|
const { getProvider } = await import("../../lib/llm/provider.js");
|
|
|
|
const provider = getProvider();
|
|
let modelElapsedMs = null;
|
|
let providerGenerateCalls = 0;
|
|
|
|
const timedProvider = {
|
|
async generateReconstruction(prompt, modelName, outputSchema) {
|
|
providerGenerateCalls++;
|
|
const startedAt = Date.now();
|
|
const response = await provider.generateReconstruction(prompt, modelName, outputSchema);
|
|
modelElapsedMs = Date.now() - startedAt;
|
|
return response;
|
|
},
|
|
};
|
|
|
|
const endToEndStartedAt = Date.now();
|
|
|
|
let episodeResult;
|
|
|
|
/* ── Production-equivalent focused-episode seam: targetNodeId triggers the same path as executeEpisodeDone → handleEpisodeMode ─── */
|
|
|
|
/* ── Canonical deconstruction: use focusedDeconstruct for Contribution derivation ─── */
|
|
const effectiveContributions = contributions.length > 0 ? [...contributions] : [];
|
|
|
|
/* ── Target-scoped filtering: only contributions belonging to the current target suppress fresh deconstruction ─── */
|
|
let effectiveContributionsForTarget = effectiveContributions;
|
|
if (targetNodeId && effectiveContributions.length > 0) {
|
|
effectiveContributionsForTarget = effectiveContributions.filter(
|
|
(c) => c?.targetNodeId === targetNodeId,
|
|
);
|
|
}
|
|
|
|
let newFindingsFromContributions = [];
|
|
|
|
if (effectiveContributionsForTarget.length === 0 && answer && targetNodeId) {
|
|
const contribSeq = effectiveContributions.length + 1;
|
|
|
|
/* ── Load production deconstruction seam (dynamic import to resolve ESM path aliases) ─── */
|
|
const { focusedDeconstruct } = await import("../../lib/graph/focused-investigation.js");
|
|
|
|
// Extract node context for deconstruction prompt
|
|
const targetNode = situationGraph.nodes.find(n => n.id === targetNodeId);
|
|
const targetLabel = targetNode?.label ?? "";
|
|
const targetDescription = targetNode?.description ?? "";
|
|
const centralStatement = situationGraph.centralStatement ?? "";
|
|
|
|
// Call production-focused deconstruct — exactly one provider generation
|
|
const focused = await focusedDeconstruct({
|
|
provider: timedProvider,
|
|
modelName: model,
|
|
targetNodeId,
|
|
targetLabel,
|
|
targetDescription,
|
|
centralStatement,
|
|
question: previousQuestion ?? "",
|
|
answer,
|
|
});
|
|
|
|
if (!focused.success) {
|
|
throw new Error(`focusedDeconstruct failed for episode ${contribSeq}: ${JSON.stringify(focused.errors)}`);
|
|
}
|
|
|
|
// Map deconstruction result → canonical Contribution shape
|
|
const derivedContrib = {
|
|
id: `contrib-ep-${String(contribSeq).padStart(4, "0")}`,
|
|
targetNodeId: focused.targetNodeId ?? targetNodeId,
|
|
question: previousQuestion ?? "",
|
|
answer: answer,
|
|
sequence: contribSeq,
|
|
observations: focused.observations,
|
|
uncertainties: focused.uncertainties,
|
|
assumptions: focused.assumptions,
|
|
relationships: focused.relationships,
|
|
possibleFollowUpQuestions: focused.possibleFollowUpQuestions,
|
|
};
|
|
effectiveContributions.push(derivedContrib);
|
|
|
|
// Derive canonical Findings from the deconstruction-derived Contribution (production seam)
|
|
newFindingsFromContributions = deriveFindingsFromContributions([derivedContrib]).findings;
|
|
}
|
|
|
|
if (episodeMode === true && targetNodeId) {
|
|
// Mirrors produceEpisodeAwareGraph: prepareCompletedEpisode → reconsiderCompletedEpisode → applyValidatedProposal
|
|
const { prepareCompletedEpisode } = await import("../../lib/graph/episode-preparation.js");
|
|
const { applyValidatedProposal } = await import("../../lib/graph/apply-proposal.js");
|
|
|
|
const prepared = prepareCompletedEpisode({
|
|
situationGraph,
|
|
targetNodeId,
|
|
contributions: effectiveContributions,
|
|
findings,
|
|
});
|
|
|
|
if (!prepared?.turns?.length && !prepared?.eligibleCanonicalFindings?.length) {
|
|
throw new Error("no_episodic_content: prepared episode has no turns or eligible findings");
|
|
}
|
|
|
|
const reasoning = await reconsiderCompletedEpisode(prepared, { provider: timedProvider });
|
|
if (!reasoning.success) {
|
|
throw new Error(`reconsiderCompletedEpisode failed: ${reasoning.error}`);
|
|
}
|
|
|
|
const application = await applyValidatedProposal({
|
|
situationGraph,
|
|
proposal: reasoning.proposal,
|
|
evidenceContext: {
|
|
isCompletedEpisode: true,
|
|
episodeEvidence: prepared,
|
|
},
|
|
});
|
|
|
|
if (!application.success) {
|
|
throw new Error(`applyValidatedProposal failed: ${application.error}`);
|
|
}
|
|
|
|
const resolvedNodeIds = new Set(application.updatedSituationGraph.resolvedNodeIds ?? []);
|
|
resolvedNodeIds.add(targetNodeId);
|
|
|
|
episodeResult = {
|
|
success: true,
|
|
updatedSituationGraph: {
|
|
...application.updatedSituationGraph,
|
|
resolvedNodeIds: [...resolvedNodeIds],
|
|
},
|
|
appendedFindings: [], // no regular update findings in episode path
|
|
proposal: reasoning.proposal,
|
|
};
|
|
} else {
|
|
const result = await updateCase(
|
|
{ situationGraph, previousQuestion, answer },
|
|
{ provider: timedProvider },
|
|
);
|
|
episodeResult = result;
|
|
}
|
|
|
|
const endToEndElapsedMs = Date.now() - endToEndStartedAt;
|
|
|
|
/* ── Step 2: Current Understanding synthesis (mirrors production synthesizeFn) ─── */
|
|
const updatedGraph = episodeResult.updatedSituationGraph;
|
|
|
|
// Accumulate prior findings + newly derived Findings (from new Contribution) for CU synthesis
|
|
const priorFindings = Array.isArray(findings) ? [...findings] : [];
|
|
const accumulatedEpisodes = episodeResult.appendedFindings ?? [];
|
|
const postEpisodeFindings = [...priorFindings, ...newFindingsFromContributions, ...accumulatedEpisodes];
|
|
|
|
let currentUnderstanding = null;
|
|
let synthesisErrorDiagnostic = null;
|
|
|
|
if (episodeResult.success && updatedGraph) {
|
|
// Reuse the same provider/model plumbing — no re-creation, just reuse
|
|
const { synthesizeCurrentUnderstanding } = await import("../../lib/graph/current-understanding-synthesis.js");
|
|
|
|
try {
|
|
const synthesized = await synthesizeCurrentUnderstanding(
|
|
{ situationGraph: updatedGraph, findings: postEpisodeFindings },
|
|
{ provider: timedProvider, modelName: model },
|
|
);
|
|
|
|
currentUnderstanding = synthesized?.currentUnderstanding ?? null;
|
|
} catch (synthErr) {
|
|
// Expose the safe structural diagnostic from synthesis validation failure
|
|
if (synthErr?.sanitizedDiagnostic) {
|
|
synthesisErrorDiagnostic = synthErr.sanitizedDiagnostic;
|
|
}
|
|
currentUnderstanding = null;
|
|
}
|
|
}
|
|
|
|
// Consistent graphCheckpoint for both success and failure paths
|
|
const graphCheckpoint = updatedGraph
|
|
? {
|
|
nodeCount: updatedGraph.nodes?.length ?? 0,
|
|
edgeCount: updatedGraph.edges?.length ?? 0,
|
|
resolvedNodeIds: updatedGraph.resolvedNodeIds ?? [],
|
|
}
|
|
: { nodeCount: 0, edgeCount: 0, resolvedNodeIds: [] };
|
|
|
|
// Accurate post-episode accumulated findings count
|
|
const { filterEligibleFindings } = await import("../../lib/graph/current-understanding-synthesis.js");
|
|
const eligibleCount = postEpisodeFindings.length > 0
|
|
? filterEligibleFindings(postEpisodeFindings).length
|
|
: 0;
|
|
|
|
return {
|
|
/* ── CU synthesis result (mirrors production synthesizeFn) ─── */
|
|
currentUnderstanding,
|
|
|
|
/* ── updatedSituationGraph (NEW — for longitudinal Episode N+1 state feeding) ─── */
|
|
updatedSituationGraph: updatedGraph,
|
|
|
|
/* ── postEpisodeContributions (NEW — accumulated Contributions for longitudinal tracking) ─── */
|
|
postEpisodeContributions: effectiveContributions,
|
|
|
|
/* ── postEpisodeFindings (NEW — accumulated Findings for Episode N+1) ─── */
|
|
postEpisodeFindings: postEpisodeFindings,
|
|
|
|
/* ── Accurate post-episode accumulated findings count ─── */
|
|
findingsCheckpoint: {
|
|
total: postEpisodeFindings.length,
|
|
eligible: eligibleCount,
|
|
},
|
|
|
|
/* ── graph checkpoint (from production result) ─── */
|
|
graphCheckpoint,
|
|
|
|
/* ── Timing ─── */
|
|
modelElapsedMs,
|
|
endToEndElapsedMs,
|
|
providerGenerateCalls,
|
|
modelName: episodeResult.modelName ?? null,
|
|
|
|
/* ── CU synthesis error diagnostic (NEW — safe structural shape on validation failure) ─── */
|
|
synthesisErrorDiagnostic,
|
|
|
|
/* ── Answer meaning (from production result) ─── */
|
|
userSupportedMeaning: episodeResult?.proposal?.answerMeaning?.userSupportedMeaning ?? null,
|
|
possibleInference: episodeResult?.proposal?.answerMeaning?.possibleInference ?? null,
|
|
};
|
|
}
|
|
|
|
module.exports = { runLiveExperiment, extractLiveExperimentCheckpoint, _extractReasoningState, composeDeepInvestigationCheckpoint };
|