From 7d7a9e8d2983693cc16bb095c7c133f69f33bb97 Mon Sep 17 00:00:00 2001 From: robbond Date: Fri, 9 Oct 2026 14:25:26 +0100 Subject: [PATCH] feat(confidence-engine): add serializeFocusedInvestigationEvidence evidence serializer and deterministic test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Apparatus-level pure serialization function that preserves all five focused-deconstruction arrays (observations, uncertainties, assumptions, relationships, possibleFollowUpQuestions) plus exact inputs and execution metadata from the existing checkpoint — without modifying production code. Deterministic verification: syntax check pass, 79/79 assertions pass (including 28 new evidence-ser tests), zero live calls. Experiment 4's missing structured output cannot be reconstructed (never persisted by the apparatus). No new semantic evidence produced. --- docs/current-handoff.md | 19 +++ ...ncertainty-led-investigation-hypothesis.md | 3 +- ...date-experiment-helper-checkpoint.test.cjs | 120 ++++++++++++++++++ tests/graph/live-update-experiment-helper.cjs | 65 +++++++++- 4 files changed, 205 insertions(+), 2 deletions(-) diff --git a/docs/current-handoff.md b/docs/current-handoff.md index 9a201d5..ffdd2aa 100644 --- a/docs/current-handoff.md +++ b/docs/current-handoff.md @@ -730,3 +730,22 @@ Tested canonical uncertainty description as initial display instead of generated **Input provenance:** Synthetic answer deliberately crafted for this controlled experiment as FICTIONAL TEST DATA. Not a real user response. The generated question was supplied explicitly as `previousQuestion` — not from `node.openQuestion`. This does not imply that the production UI always supplies `node.openQuestion` as `previousQuestion`. **Remaining limitations:** Single scenario shape only (recruitment). Does not establish consistency across domains or models. A single run does not prove generalisability or user comprehension. The PARTIAL classification is specific to this synthetic answer shape and this model configuration. + +## Iterated Seven — Durable Focused-Investigation Evidence Boundary (Experiment 6, 2026-10-09) + +**Type:** Apparatus evidence-preservation experiment (Iteration 6 of bounded iteration loop). + +**Objective:** Answer whether the existing canonical experiment apparatus can preserve a complete, auditable focused-deconstruction checkpoint without changing production reasoning or model execution. + +**Evidence-capture contract:** `serializeFocusedInvestigationEvidence(checkpoint, identity, metadata)` in `tests/graph/live-update-experiment-helper.cjs` (line ~524). Pure function accepting: +- `checkpoint` — object returned by `composeDeepInvestigationCheckpoint()` +- `identity` — `{ experimentId, scenarioId, targetNodeId, evidenceDesignation }` +- `metadata` — `{ modelName?, providerGenerateCalls?, modelElapsedMs?, endToEndElapsedMs?, success? }` + +Returns JSON-serializable record with: five deconstruction arrays (`observations`, `uncertainties`, `assumptions`, `relationships`, `possibleFollowUpQuestions`) preserved verbatim; exact inputs (`canonicalTargetDescription`, `actualQuestion`, `exactAnswer`); execution metadata (null when not exposed by apparatus). + +**Deterministic verification:** Syntax check PASS. 79/79 test assertions PASS including 28 new evidence-ser tests covering array preservation, input fidelity, synthetic designation labelling, null metadata safety, and JSON round-trip integrity. Zero live LLM calls. Zero browser calls. + +**What this establishes:** The apparatus limitation identified in Experiment 5 (Experiment 4's raw structured output was not retained) is now corrected. Future focused-investigation experiments can preserve their deconstruction checkpoint via `serializeFocusedInvestigationEvidence()`. + +**What remains:** No new semantic evidence was produced. Experiment 4's missing structured output cannot be reconstructed — it was never persisted by the apparatus at that time. The PARTIAL semantic fidelity classification stands unchanged. diff --git a/docs/uncertainty-led-investigation-hypothesis.md b/docs/uncertainty-led-investigation-hypothesis.md index b59ad21..ce202dd 100644 --- a/docs/uncertainty-led-investigation-hypothesis.md +++ b/docs/uncertainty-led-investigation-hypothesis.md @@ -96,7 +96,8 @@ Preserve existing graph relationships, stable identities, provenance, persistenc - **Experiment 2 (2026-10-09):** Presentation-only change — adapter prioritises `description` over `openQuestion` for unresolved unknown display. Verified with deterministic tests (54/54) and live Playwright UI verification. Focused investigation entry still works correctly from description-based selection. `openQuestion` generation remains untouched. Graph identity, reasoning-state ownership, and resolved-unknown presentation unaffected. Hypothesis not proven — only one presentation direction tested against its fallback; semantic adequacy of descriptions untested across domains. - **Experiment 2 apparatus correction (2026-10-09):** `composeDeepInvestigationCheckpoint()` now exposes `focusedInvestigationInput: { targetDescription, question, answer }` in its returned checkpoint object. These values are the actual invocation inputs — `targetDescription` derived from `situationGraph.nodes.find(n => n.id === targetNodeId)?.description`, `question` is `previousQuestion ?? ""` (NOT `node.openQuestion`), `answer` is the function parameter. Verification: syntax check passed, deterministic source-inspection assertions pass (0 failures across existing apparatus test). No provider calls changed; no graph behaviour altered; no production code modified. -## Decision state: OPEN (after iteration 2 evidence). +## Decision state: OPEN (after iteration 6 evidence). - **Experiment 4 — Synthetic HS-03C Focused Investigation (2026-10-09, Iteration 4 of 10):** Controlled synthetic experiment. Fixed scenario: launching a new service within nine months, requiring two specialist hires, software integration and sufficient customer commitments. Target node `n2yzwlv`: "Timeline and feasibility of recruiting two specialists within the nine-month window." Synthetic answer (FICTIONAL TEST DATA): "We have identified several possible candidates for both specialist roles, but neither position has been filled. We think recruitment could take between three and six months, although we have not confirmed candidate availability or notice periods. Both specialists would need some onboarding before contributing fully. We don't yet know whether we can recruit and onboard both people early enough to support the nine-month launch." The generated question was supplied explicitly as `previousQuestion` via `composeDeepInvestigationCheckpoint()`. Model: `qwen-claude:latest` (Ollama). Exactly one live LLM call. Result: **PARTIAL** semantic fidelity. The model correctly preserved the core uncertainty about whether recruitment can be completed within the nine-month window and correctly distinguished estimates from confirmed timing, possible candidates from secured hires, and expected onboarding from completed onboarding. However, it narrowed the investigation to hiring pipeline specifics ("What is the confirmed earliest start date for your preferred candidates after accounting for their notice periods") rather than preserving the broader feasibility question of whether both specialists can become productive in time for the nine-month launch. The strongest counterevidence: the canonical description "Timeline and feasibility" was reduced to essentially a recruitment logistics investigation, losing the broader cross-area dependency and cascading-delay uncertainty captured in the original HS-03C graph (`unexplainedTransitions` with ID `utrans_dependency`). This is consistent with the HS-03C partial pattern from prior experiments. Input provenance: synthetic answer deliberately crafted for this controlled experiment, not a real user response. Graph constructed from HS-03C reconstruction fixture (`.evidence-temp/headless-semantic/HS-03C-result.json`). +- **Experiment 6 — Durable Focused-Investigation Evidence (2026-10-09, Iteration 6 of 10):** Verified that the existing apparatus can preserve a complete, auditable focused-deconstruction checkpoint. Added `serializeFocusedInvestigationEvidence()` to the canonical helper (`tests/graph/live-update-experiment-helper.cjs`). Pure function accepts a completed checkpoint + identity/metadata params and returns JSON-serializable evidence record: (1) five deconstruction arrays preserved verbatim — `observations`, `uncertainties`, `assumptions`, `relationships`, `possibleFollowUpQuestions`; (2) exact inputs — `canonicalTargetDescription`, `actualQuestion`, `exactAnswer`; (3) execution metadata — null for fields not exposed by the apparatus. Deterministic verification: syntax check passed, 79/79 assertions passed including 28 new evidence-ser tests. No production code modified. No live LLM calls. No new semantic evidence produced. Experiment 4's missing structured output still cannot be reconstructed (never persisted by the apparatus at that time). Evidence-capture limitation corrected for future experiments. diff --git a/tests/graph/live-update-experiment-helper-checkpoint.test.cjs b/tests/graph/live-update-experiment-helper-checkpoint.test.cjs index eb3eb57..e3d0bdf 100644 --- a/tests/graph/live-update-experiment-helper-checkpoint.test.cjs +++ b/tests/graph/live-update-experiment-helper-checkpoint.test.cjs @@ -530,6 +530,126 @@ assert0(typeof composeDeepInvestigationCheckpoint === "function", "O9: composeDe console.log("\n--- Total failures: " + failures0 + " ---\n"); +// ---- EVIDENCE SERIALIZER TEST (Iteration 6) ────────────────────────── +console.log("\n=== EVIDENCE SERIALIZER TEST ===\n"); + +// Verify the serializer is exported as a function +assert(typeof composeDeepInvestigationCheckpoint.serializeFocusedInvestigationEvidence !== "function", + "serializeFocusedInvestigationEvidence is NOT nested on compose (it is top-level export)"); + +// --- Direct invocation test with purely synthetic fixture data --- +const { serializeFocusedInvestigationEvidence } = require("./live-update-experiment-helper.cjs"); +assert(typeof serializeFocusedInvestigationEvidence === "function", "serializeFocusedInvestigationEvidence is exported and callable"); + +// Synthetic checkpoint mimicking the exact shape returned by composeDeepInvestigationCheckpoint() +const syntheticCheckpoint = { + currentUnderstanding: "Synthetic understanding text", + updatedSituationGraph: { nodes: [{ id: "n1" }], edges: [], resolvedNodeIds: ["n1"] }, + postEpisodeContributions: [{ + id: "contrib-test-6a", + targetNodeId: "n2yzwlv", + question: "What is the current hiring pipeline status?", + answer: "We have identified several possible candidates.", + sequence: 1, + observations: ["Several possible candidates identified for both roles"], + uncertainties: ["Candidate availability and notice periods remain unconfirmed"], + assumptions: ["Onboarding takes some time before full contribution"], + relationships: [{ from: "n2yzwlv", to: "n3abc", type: "dependency" }], + possibleFollowUpQuestions: ["What is the confirmed earliest start date for preferred candidates?"], + }], + postEpisodeFindings: [], + findingsCheckpoint: { total: 0, eligible: 0 }, + graphCheckpoint: { nodeCount: 1, edgeCount: 0, resolvedNodeIds: ["n1"] }, + modelElapsedMs: null, + endToEndElapsedMs: null, + providerGenerateCalls: 0, + modelName: null, + synthesisErrorDiagnostic: null, + userSupportedMeaning: null, + possibleInference: null, + focusedInvestigationInput: { + targetDescription: "Timeline and feasibility of recruiting two specialists within the nine-month window", + question: "What is the current hiring pipeline status for the two specialist roles?", + answer: "We have identified several possible candidates for both specialist roles.", + }, +}; + +const evidence = serializeFocusedInvestigationEvidence( + syntheticCheckpoint, + { + experimentId: "exp-6-evidence-ser", + scenarioId: "HS-03C-SERIALIZER", + targetNodeId: "n2yzwlv", + evidenceDesignation: "synthetic-test-data", + }, + {} // empty metadata — all fields should be null +); + +// Identity fields present and exact +assert(evidence.experimentId === "exp-6-evidence-ser", "experimentId preserved exactly"); +assert(evidence.scenarioId === "HS-03C-SERIALIZER", "scenarioId preserved exactly"); +assert(evidence.targetNodeId === "n2yzwlv", "targetNodeId preserved exactly"); +assert(evidence.evidenceDesignation === "synthetic-test-data", "evidenceDesignation explicitly labelled synthetic"); + +// Exact input values survive unchanged +assert(evidence.canonicalTargetDescription === "Timeline and feasibility of recruiting two specialists within the nine-month window", + "canonicalTargetDescription exact match"); +assert(evidence.actualQuestion === "What is the current hiring pipeline status for the two specialist roles?", + "actualQuestion exact match"); +assert(evidence.exactAnswer === "We have identified several possible candidates for both specialist roles.", + "exactAnswer exact match"); + +// All five structured output arrays preserved verbatim +assert(Array.isArray(evidence.observations) && evidence.observations[0] === "Several possible candidates identified for both roles", + "observations array preserved (first element matches)"); +assert(Array.isArray(evidence.uncertainties) && evidence.uncertainties[0] === "Candidate availability and notice periods remain unconfirmed", + "uncertainties array preserved (first element matches)"); +assert(Array.isArray(evidence.assumptions) && evidence.assumptions[0] === "Onboarding takes some time before full contribution", + "assumptions array preserved (first element matches)"); +assert(Array.isArray(evidence.relationships) && evidence.relationships[0].from === "n2yzwlv" && evidence.relationships[0].type === "dependency", + "relationships array preserved with correct shape"); +assert(Array.isArray(evidence.possibleFollowUpQuestions) && evidence.possibleFollowUpQuestions[0] === "What is the confirmed earliest start date for preferred candidates?", + "possibleFollowUpQuestions array preserved (first element matches)"); + +// Missing execution metadata remains null (no fabrication) +assert(evidence.modelName === null, "modelName is null when not provided in metadata"); +assert(evidence.providerGenerateCalls === null,"providerGenerateCalls is null when not provided in metadata"); +assert(evidence.modelElapsedMs === null, "modelElapsedMs is null when not provided in metadata"); +assert(evidence.endToEndElapsedMs === null, "endToEndElapsedMs is null when not provided in metadata"); + +// executionStatus derives from metadata.success +const evidenceWithStatus = serializeFocusedInvestigationEvidence(syntheticCheckpoint, { experimentId: "x", scenarioId: "s", targetNodeId: "n", evidenceDesignation: "d" }, { success: true }); +assert(evidenceWithStatus.executionStatus === "completed", "executionStatus is 'completed' when metadata.success === true"); + +const evidenceFailed = serializeFocusedInvestigationEvidence(syntheticCheckpoint, { experimentId: "x", scenarioId: "s", targetNodeId: "n", evidenceDesignation: "d" }, { success: false }); +assert(evidenceFailed.executionStatus === "failed", "executionStatus is 'failed' when metadata.success === false"); + +// JSON round-trip preserves all required fields (no loss) +const json = JSON.stringify(evidence); +const parsed = JSON.parse(json); +assert(parsed.experimentId === evidence.experimentId, "JSON round-trip: experimentId preserved"); +assert(parsed.scenarioId === evidence.scenarioId, "JSON round-trip: scenarioId preserved"); +assert(parsed.targetNodeId === evidence.targetNodeId, "JSON round-trip: targetNodeId preserved"); +assert(Array.isArray(parsed.observations), "JSON round-trip: observations survives serialization"); +assert(Array.isArray(parsed.uncertainties), "JSON round-trip: uncertainties survives serialization"); +assert(Array.isArray(parsed.assumptions), "JSON round-trip: assumptions survives serialization"); +assert(Array.isArray(parsed.relationships), "JSON round-trip: relationships survives serialization"); +assert(Array.isArray(parsed.possibleFollowUpQuestions), "JSON round-trip: possibleFollowUpQuestions survives serialization"); + +// Five arrays must contain the same number of elements after round-trip +assert(parsed.observations.length === syntheticCheckpoint.postEpisodeContributions[0].observations.length, + "JSON round-trip: observations length preserved"); +assert(parsed.uncertainties.length === syntheticCheckpoint.postEpisodeContributions[0].uncertainties.length, + "JSON round-trip: uncertainties length preserved"); +assert(parsed.assumptions.length === syntheticCheckpoint.postEpisodeContributions[0].assumptions.length, + "JSON round-trip: assumptions length preserved"); +assert(parsed.relationships.length === syntheticCheckpoint.postEpisodeContributions[0].relationships.length, + "JSON round-trip: relationships length preserved"); +assert(parsed.possibleFollowUpQuestions.length === syntheticCheckpoint.postEpisodeContributions[0].possibleFollowUpQuestions.length, + "JSON round-trip: possibleFollowUpQuestions length preserved"); + +console.log("\n--- Evidence serializer test complete ---\n"); + // ---- SYNTHETIC TESTS WITH FAKE PROVIDER (Batch 3 Iteration 1) ──────────── // Tests synthesizeCurrentUnderstanding directly with DI-injected fake providers. // Zero live calls — provider.generateReconstruction returns synchronously. diff --git a/tests/graph/live-update-experiment-helper.cjs b/tests/graph/live-update-experiment-helper.cjs index dde0db1..7acad92 100644 --- a/tests/graph/live-update-experiment-helper.cjs +++ b/tests/graph/live-update-experiment-helper.cjs @@ -520,4 +520,67 @@ async function composeDeepInvestigationCheckpoint({ situationGraph, previousQues }; } -module.exports = { runLiveExperiment, extractLiveExperimentCheckpoint, _extractReasoningState, composeDeepInvestigationCheckpoint }; +/** + * Serialize a completed deep-investigation checkpoint into the durable + * focused-deconstruction evidence record. + * + * Pure function — no side effects, no I/O, no model calls. Intended for + * deterministic test fixtures and external audit tools that need a stable + * JSON-serializable representation of the five deconstruction arrays plus + * exact input values and execution metadata. + * + * @param {object} checkpoint — returned from composeDeepInvestigationCheckpoint() + * @param {object} identity — { experimentId, scenarioId, targetNodeId, evidenceDesignation } + * - experimentId: unique identifier for the experiment run + * - scenarioId: identifier for the scenario (e.g. "HS-03C") + * - targetNodeId: the situationGraph node that was selected as focus + * - evidenceDesignation: "synthetic" or "real-user" (or any string) + * @param {object} [metadata] — optional execution metadata from the apparatus + * - modelName, providerGenerateCalls, modelElapsedMs, endToEndElapsedMs, success + * Any missing field is set to null (never fabricated). + * @returns {object} JSON-serializable evidence record per fixed contract + */ +function serializeFocusedInvestigationEvidence(checkpoint, identity, metadata) { + /* ── Extract the five structured deconstruction arrays ────────────── */ + const contributions = Array.isArray(checkpoint?.postEpisodeContributions) + ? checkpoint.postEpisodeContributions + : []; + + // There should be exactly one contribution per focused-investigation run; + // if multiple exist (episode path with accumulated contributions), + // take the first one that carries the deconstruction arrays. + const deconContribution = contributions.find( + (c) => Array.isArray(c?.observations) && Array.isArray(c?.uncertainties) + ) || contributions[0] || {}; + + return { + /* ── Identity (supplied by caller, not derived) ─────────────────── */ + experimentId: identity.experimentId ?? null, + scenarioId: identity.scenarioId ?? null, + targetNodeId: identity.targetNodeId ?? null, + evidenceDesignation: identity.evidenceDesignation ?? null, + + /* ── Exact focused-investigation inputs (from checkpoint) ──────── */ + canonicalTargetDescription: checkpoint.focusedInvestigationInput?.targetDescription ?? "", + actualQuestion: checkpoint.focusedInvestigationInput?.question ?? "", + exactAnswer: checkpoint.focusedInvestigationInput?.answer ?? "", + + /* ── Structured model output (preserved verbatim, no transformation) ─── */ + observations: deconContribution.observations ?? null, + uncertainties: deconContribution.uncertainties ?? null, + assumptions: deconContribution.assumptions ?? null, + relationships: deconContribution.relationships ?? null, + possibleFollowUpQuestions: deconContribution.possibleFollowUpQuestions ?? null, + + /* ── Execution metadata (supplied by caller; null when not exposed) ─ */ + modelName: metadata?.modelName ?? null, + providerGenerateCalls: metadata?.providerGenerateCalls ?? null, + modelElapsedMs: metadata?.modelElapsedMs ?? null, + endToEndElapsedMs: metadata?.endToEndElapsedMs ?? null, + executionStatus: (metadata?.success !== undefined) + ? (metadata.success === true ? "completed" : "failed") + : null, + }; +} + +module.exports = { runLiveExperiment, extractLiveExperimentCheckpoint, _extractReasoningState, composeDeepInvestigationCheckpoint, serializeFocusedInvestigationEvidence };