From fe0f337a998bbe855839586b3adfdc3937fada0c Mon Sep 17 00:00:00 2001 From: robbond Date: Wed, 14 Oct 2026 09:39:11 +0100 Subject: [PATCH] docs(experiments): checkpoint uncertainty recognition investigation --- docs/uncertainty-information-flow-handoff.md | 66 ++++++++ ...tainty-recognition-investigation-report.md | 147 ++++++++++++++++++ .../reconstruct-v0.5-d-deconstruct-no23.md | 35 +++++ scripts/start-case-experiment-helper.cjs | 12 +- 4 files changed, 255 insertions(+), 5 deletions(-) create mode 100644 docs/uncertainty-information-flow-handoff.md create mode 100644 docs/uncertainty-recognition-investigation-report.md create mode 100644 prompts/reconstruct-v0.5-d-deconstruct-no23.md diff --git a/docs/uncertainty-information-flow-handoff.md b/docs/uncertainty-information-flow-handoff.md new file mode 100644 index 0000000..f91e858 --- /dev/null +++ b/docs/uncertainty-information-flow-handoff.md @@ -0,0 +1,66 @@ +# Uncertainty Information Flow Audit — Programme Handoff + +**Working title:** Confidence Engine — Uncertainty Information Flow Audit +**Date prepared:** 2026-10-14 +**Prepared for:** Next independent investigation (following the completed 10-iteration uncertainty recognition quality investigation) + +--- + +## Primary Question + +"At which point, if any, does Confidence Engine constrain, transform, discard or fail to preserve meaningful uncertainty recognition?" + +## Context + +The previous 10-iteration programme demonstrated a quality gap between minimal direct-model uncertainty recognition and Confidence Engine-mediated uncertainty recognition. The precise limiting mechanism remains unresolved (see `uncertainty-recognition-investigation-report.md` sections D–G). This new programme is designed to map the actual production information flow and identify specific intervention points without presuming failure locations. + +## Investigation Boundaries (Not Presumed Failure Points) + +1. User input and scenario representation +2. Prompt construction +3. Raw LLM request and response +4. Parsing and validation +5. Internal graph representation +6. User-facing presentation + +These boundaries define investigation scope, not expectations of where problems will be found. + +## Recommended Approach + +- Up to 10 iterations +- One bounded investigative question per iteration +- ChatGPT controls programme progression between iterations +- Claude Code executes one task per invocation and reports evidence +- Each `/rdbHandoff` invocation is a fresh session — work from repository evidence and this handoff +- Read-only investigation before any interventions +- Controlled experiments only when justified by evidence from prior iterations +- Preserve raw evidence and request provenance for every iteration +- No speculative product changes + +## First Iteration Guidance + +The first iteration should establish the actual production information flow and available capture boundaries: + +1. Map the complete code path from scenario file input to model request construction +2. Document what data is accessible at each stage of the pipeline (raw scenario, post-parsing, pre-template, post-template, pre-request, post-response, post-parsing) +3. Identify any existing hooks, logs, or capture mechanisms for raw requests/responses +4. Determine which stages can produce evidence without source modifications + +The output should be a factual map of available capture points and data accessibility, not an analysis of what is "wrong." + +## Existing Evidence to Consider + +- `docs/uncertainty-recognition-investigation-report.md` — Full report from the previous programme +- `.evidence-temp/iteration10/architecture-finding.txt` — Definitive CE prompt-path analysis showing template applicability at every invocation point +- `.evidence-temp/headless-semantic/` — Extensive supporting evidence from HS series experiments + +## What This Programme Is Not + +- A redesign of Confidence Engine +- A directive to change the model or provider configuration +- An assumption that any specific boundary is broken or deficient +- A continuation of the previous programme's quality-gap investigation (that closed at Iteration 10) + +## Start Signal + +Do not begin this programme from this document alone. Wait for an explicit `/rdbHandoff` invocation containing the start signal and first iteration directive. diff --git a/docs/uncertainty-recognition-investigation-report.md b/docs/uncertainty-recognition-investigation-report.md new file mode 100644 index 0000000..ad0f84e --- /dev/null +++ b/docs/uncertainty-recognition-investigation-report.md @@ -0,0 +1,147 @@ +# Uncertainty Recognition Quality Investigation — Report + +**Date:** 2026-10-14 +**Branch:** `experiment/uncertainty-led-investigation` +**Starting HEAD:** `e595ee6` feat(experiments): integrate Condition D/E apparatus +**Status:** CLOSED — 10-iteration programme completed and checkpointed + +--- + +## A. Investigation Objective + +This investigation examined why the configured local LLM (`qwen-claude:latest` via Ollama) produced less useful uncertainty recognition through Confidence Engine's reconstruction pipeline than through a minimal direct prompt. + +The canonical scenario used was `RFG-03-independent-decisions-input.json`, concerning three independent decisions (replacing internal reporting software, recruiting an additional customer-support employee, refurbishing a meeting room). The user's goal is to understand what remains uncertain before deciding what to do. + +**Observed phenomenon:** A minimal direct-model prompt produced two outputs containing approximately 21 and 15 uncertainties with meaningful domain-specific unknowns. Confidence Engine reconstruction under Conditions D/E generally produced three generic uncertainties that largely restated the user's stated decisions without adding new insights. + +--- + +## B. Experimental Programme (Iterations 1–10) + +### Iteration 3 — Baseline Direct-Model Prompt +**Status:** Complete and reproducible +**Evidence:** `.evidence-temp/experiment3/` +**Prompt:** Minimal raw prompt with no rule-set framing or forced JSON output. +**Result:** Two model outputs: raw-run-1 (29 lines, ~15+ domain-specific uncertainties), raw-run-2 (21 lines, similar richness). Scenario SHA-256: `dc9a600f9fb1e7189dd56086a07f357893396a5b0782f4781c95d4aa29fdcf06`. + +### Iteration 5 — Condition D Rule-21 Removal +**Status:** Complete +**Evidence:** `.evidence-temp/iteration5/` +**Treatment:** Removed rule 21 from the deconstruction prompt. +**Result:** Four CE uncertainties including a cross-domain "worthwhile" framework question. No material improvement over control in domain-specific discovery. Evidence: `D-no21-freeform.txt` (103 lines), `D-no21.json`, `D-original-freeform.txt`, `D-original.json`. + +### Iteration 6 — Condition D Rule-23 Removal + Prompt Variant +**Status:** Complete +**Evidence:** `.evidence-temp/iteration6/` +**Treatment:** Removed rule 23; tested with new prompt variant (`reconstruct-v0.5-d-deconstruct-no23.md`). +**Result:** No material improvement. Both `run-A-original-D.json` and `run-B-no-rule-23.json` validated as freeform with zero graph nodes. + +### Iteration 8 — Invalid Run + Recovery +**Status:** Validated invalidity; recovery run complete +**Evidence:** `.evidence-temp/iteration8/` (invalid), `.evidence-temp/iteration8-recovery/` (valid) +**Invalidity cause:** Scenario filename was treated as scenario text instead of JSON content. +**Recovery result:** 4 uncertainty items with modest structural enrichment (expectedStates, differences, tighter relationships) but no new domain-specific unknowns. + +### Iterations 9–10 — Follow-up and Block +**Iteration 9:** Partial evidence; focused-investigation dependency analysis. `.evidence-temp/iteration9/` contains a `b-variant/` subdirectory (incomplete). +**Iteration 10:** BLOCKED before model execution. The existing experimental helper (`scripts/start-case-experiment-helper.cjs`) has no mechanism to pass the exact minimal raw prompt through the CE execution path without source modifications. Every invocation applies a structured prompt template. See `.evidence-temp/iteration10/final-report.txt` and `architecture-finding.txt` for the definitive architecture analysis. + +### Headless-Semantic Evidence (HS-01 through HS-03C) +**Status:** Extensive supporting evidence preserved +**Evidence:** `.evidence-temp/headless-semantic/` (28 files, 2788 lines total) +Covers: Question-free reconstruction comparison (CE-ULI-09), domain-preservation instruction tests (CE-ULI-10, CE-ULI-11), and focused-investigation evidence persistence testing. + +--- + +## C. Established Findings + +These findings are directly supported by the evidence: + +1. **Direct-model prompt quality gap:** Minimal direct prompts produced outputs with approximately 21 and 15 domain-specific uncertainties across three decisions, while CE reconstruction under Conditions D/E generally produced three generic restatements of the user's own decisions. + +2. **Rule removal had no material effect:** Removing Rule 21 (Iteration 5) and Rule 23 (Iteration 6) did not materially improve uncertainty recognition quality. + +3. **Discovery permission instruction produced modest structural enrichment:** The Iteration 8 recovery (adding explicit permission to recognise plausible unstated uncertainties) added four items including a cross-domain evaluation framework question, but no new domain-specific unknowns were discovered beyond what the user stated. + +4. **CE execution path is structurally incapable of raw-prompt pass-through:** Every invocation applies a template-based prompt construction that adds rule sets and forced-JSON constraints absent from the direct-model baseline. This was definitively established in Iteration 10's architecture-finding evidence. + +5. **Question-free reconstruction does not prevent structural compression:** CE-ULI-10 demonstrated that removing the question-generation requirement still produced structural compression of uncertainties into single compound descriptions for the tested scenario shape. + +6. **Explicit domain-separation instructions are ignored:** CE-ULI-11 showed identical compression under both condition B and C despite explicit instruction not to combine domains. + +--- + +## D. Unresolved Hypotheses + +**H1 — Model capability:** The model may be capable of richer uncertainty recognition but its output quality is constrained by the surrounding apparatus (prompt structure, output format constraints). *Untested.* Not equivalent to H2 or H3. + +**H2 — Prompt effectiveness:** The template-based prompt construction (rule sets, forced JSON, deconstruction framing) suppresses the model's ability to produce rich uncertainty recognition relative to a minimal direct prompt. *Supported by evidence of quality gap but not proven as causal.* No controlled experiment eliminated the structural difference while preserving all other conditions. + +**H3 — Confidence Engine orchestration:** Something in the CE pipeline (prompt construction, parsing, validation, graph representation, or presentation) constrains, transforms, or discards meaningful uncertainty recognition between model output and user-facing display. *Supported by evidence but not isolated.* Iteration 10 established that the CE execution path always applies templates, making controlled elimination of specific pipeline stages infeasible without source modifications. + +**Important:** The precise cause of the quality difference remains unresolved. The evidence demonstrates a quality gap and several plausible explanations, not definitive causation. It is not established that prompt construction, JSON formatting, or orchestration individually or collectively suppress uncertainty recognition — only that these are all uncontrolled confounders in the comparison between CE-executed requests and direct model calls. + +--- + +## E. Experimental Apparatus Limitations + +1. **Template-based prompt construction:** All CE invocation paths apply a template (v0.5, b/c/d/e variants). No raw-prompt pass-through mode exists in `lib/analysis.js` or `start-case-experiment-helper.cjs`. + +2. **No raw-prompt pass-through in the helper:** Condition selector loads hardcoded template filenames. The `--experiment-instruction` option only appends — it cannot replace or suppress base structure. + +3. **`--file` input-loading requirement:** Scenarios must be loaded from JSON files. Line-input and other loading modes are not supported by the helper. + +4. **`--reconstruction-only` requirement:** To bypass the graph pipeline, `--reconstruction-only` is required. Its absence routes through the full production path. + +5. **Generation-setting comparability limitations:** Direct-model baseline runs (experiment3) used different generation settings (no forced schema) than CE execution (forced JSON schemas). Temperature, top_p, and other parameters are not controlled or logged for direct runs. + +6. **Missing evidence:** Iteration 9's b-variant evidence is incomplete (empty subdirectory). Evidence for some earlier iterations beyond condition outputs alone is unavailable. + +7. **Single-model limitation:** All experiments used `qwen-claude:latest` only. Results may not generalise to other models. + +--- + +## F. Product Principles + +The following principles must govern any future work on uncertainty recognition: + +- The user owns investigation direction. +- The engine facilitates rather than steers. +- Recognising an uncertainty is not inventing a fact. +- Recognising an uncertainty is not selecting it. +- The engine must not automatically decide what the user should investigate next. +- Fidelity to the user's situation and recognition of plausible unstated unknowns can coexist. +- Do not optimise for the number of uncertainty items. Optimise for meaningful, distinct, well-grounded uncertainty recognition. + +Richer uncertainty recognition must not be confused with engine-owned prioritisation or investigation steering. The value of additional uncertainties is in user understanding, not in expanding the investigation scope autonomously. + +--- + +## G. Programme Conclusion + +The local model (`qwen-claude:latest`) demonstrated richer uncertainty recognition when invoked via a minimal direct prompt independently than when Confidence Engine mediated the invocation through its reconstruction pipeline (Conditions D/E). Confidence Engine did not reliably reproduce that behaviour across any tested condition or prompt variant. + +The precise limiting mechanism remains unresolved — whether it is model capability, prompt effectiveness, orchestration constraints, or their interaction. The original 10-iteration investigation programme is closed. + +--- + +## Referenced Evidence Files + +| Artefact | Path | Status | +|----------|------|--------| +| Raw minimal prompt (experiment3) | `.evidence-temp/experiment3/raw-prompt.txt` (858B, 13 lines) | Complete | +| Direct-model run 1 | `.evidence-temp/experiment3/raw-run-1-output.txt` (3520B, 29 lines) | Complete | +| Direct-model run 2 | `.evidence-temp/experiment3/raw-run-2-output.txt` (1855B, 21 lines) | Complete | +| Scenario SHA-256 | `.evidence-temp/iteration10/scenario-verification.txt` | `dc9a600f...fdcf06` | +| Condition D / no21 evidence | `.evidence-temp/iteration5/D-no21-freeform.txt`, `D-no21.json` | Complete | +| Condition D original (iter 5) | `.evidence-temp/iteration5/D-original-freeform.txt`, `D-original.json` | Complete | +| Rule-23 removal prompt | `.evidence-temp/iteration6/prompt-variants/reconstruct-v0.5-d-deconstruct-no23.md` | Complete | +| Run A (original D, iter 6) | `.evidence-temp/iteration6/run-A-original-D.json` | Complete | +| Run B (no rule 23, iter 6) | `.evidence-temp/iteration6/run-B-no-rule-23.json` | Complete | +| Iteration 8 invalid run | `.evidence-temp/iteration8/run-A-original-d.json`, `run-B-discovery-permission.json` | Invalid | +| Discovery permission prompt | `.evidence-temp/iteration8/reconstruct-v0.5-d-discovery-permission.md` | Complete | +| Iteration 8 recovery log | `.evidence-temp/iteration8-recovery/experiment_log.txt`, `b-comparison.txt` | Complete | +| Iteration 10 final report | `.evidence-temp/iteration10/final-report.txt` | Complete (BLOCKED) | +| Iteration 10 architecture finding | `.evidence-temp/iteration10/architecture-finding.txt` | Complete | +| Headless-semantic series | `.evidence-temp/headless-semantic/` (28 files) | Extensive, preserved | diff --git a/prompts/reconstruct-v0.5-d-deconstruct-no23.md b/prompts/reconstruct-v0.5-d-deconstruct-no23.md new file mode 100644 index 0000000..372f84b --- /dev/null +++ b/prompts/reconstruct-v0.5-d-deconstruct-no23.md @@ -0,0 +1,35 @@ +You are a neutral analyst performing deconstruction of the user's situation. + +Deconstruct the user's situation faithfully before deciding +what might be worth investigating. + +Identify the central concern and intended outcome. + +Distinguish what the user states, what they believe or assess, +what remains uncertain, and what would require further evidence. + +Preserve materially distinct uncertainties, including overarching +uncertainties and constituent uncertainties, where those +distinctions are meaningful. + +Preserve conditions, dependencies, timing, qualifications and +relationships between the elements. + +Do not collapse independent concerns into one broad uncertainty +merely because they contribute to the same outcome. + +Do not create artificial uncertainties to increase their number. + + +Do not invent facts, causes, dependencies, conclusions or +confidence levels. + +Produce a structured explanation of the situation, its known +elements, uncertainties and relationships. + +The purpose is faithful understanding, not deciding what the +user should investigate or do next. + +Scenario: + +{{SCENARIO}} diff --git a/scripts/start-case-experiment-helper.cjs b/scripts/start-case-experiment-helper.cjs index 42d281b..1fde765 100755 --- a/scripts/start-case-experiment-helper.cjs +++ b/scripts/start-case-experiment-helper.cjs @@ -71,7 +71,9 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti } else if (condition === "D") { altPromptFile = treatment === "no21" ? "reconstruct-v0.5-d-deconstruct-no21.md" - : "reconstruct-v0.5-d-deconstruct.md"; + : treatment === "no23" + ? "reconstruct-v0.5-d-deconstruct-no23.md" + : "reconstruct-v0.5-d-deconstruct.md"; } else { // condition === "E" altPromptFile = "reconstruct-v0.5-e-investigability-first.md"; @@ -83,9 +85,9 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti // ── Treatment validation (programmatic API) ──────────────── if (condition === "D") { - const validTreatments = ["no21"]; + const validTreatments = ["no21", "no23"]; if (treatment != null && !validTreatments.includes(treatment)) { - throw new Error(`Invalid treatment: ${treatment}. Must be one of: no21`); + throw new Error(`Invalid treatment: ${treatment}. Must be one of: no21, no23`); } } else if (condition === "E") { const validTreatments = ["e-investigability-first"]; @@ -394,10 +396,10 @@ if (require.main === module) { } // ── Validate treatment value ── - const validDTreatments = ["no21"]; + const validDTreatments = ["no21", "no23"]; const validETreatments = ["e-investigability-first"]; if (treatment !== null && condition === "D" && !validDTreatments.includes(treatment)) { - throw new Error(`Invalid treatment: ${treatment}. Must be one of: no21`); + throw new Error(`Invalid treatment: ${treatment}. Must be one of: no21, no23`); } if (treatment !== null && condition === "E" && !validETreatments.includes(treatment)) { throw new Error(`Invalid treatment: ${treatment}. Must be one of: e-investigability-first`);