docs: orchestrator and handoff

2026-08-02 10:06:48 +01:00
parent a9bce79658
commit c3de80f203
2 changed files with 371 additions and 0 deletions
@@ -0,0 +1,113 @@
 # Orchestrator Contract — Confidence Engine v0.4
 ## 1. Exported Function Signatures & Shape (JavaScript)
 ### lib/analysis.js
 ```js
 export async function analyseScenario(scenario, opts = {})
 // @param {string} scenario
 // @param {{ promptVersion?: "v0.2" | "v0.3" }} [opts]
 // @returns {Promise<{ success: boolean, validationStatus: "valid"|"invalid",
 //     modelName: string|null, responseDurationMs: number, rawResponse: string|null,
 //     promptVersion: string|null, inputClassification: object|null, reconstruction: object|null,
 //     evidence: object[]|undefined, nextQuestion: string|undefined, errors: string[]|undefined,
 //     error: string|undefined, statusCode: number|undefined }>}
 export const PROMPT_VERSIONS    // { [key: string]: string }
 export const DEFAULT_PROMPT_VERSION  // "v0.2"
 ```
 ### lib/graph/schema.js
 ```js
 export const SituationKind          // { observation, reported_claim, metric, state, transition, relationship, assumption, unknown, conclusion }
 export const SituationStatus        // { known, unknown, provisional, supported, weakened, contradicted, resolved }
 export const ConfidenceLevel        // { low, medium, high }
 export const SituationRelationship  // { supports, weakens, contradicts, depends_on, causes, may_cause, measures, compares_with, updates, other }
 export const situationNodeSchema    // Zod → {@typedef SituationNode}
 export const situationEdgeSchema    // Zod → {@typedef SituationEdge}
 export const situationGraphSchema   // Zod → {@typedef SituationGraph}
 export const graphUpdateSchema      // Zod → {@typedef GraphUpdate}
 export const startCaseRequestSchema // { scenario: string (1-10000), promptVersion?: string }
 export const updateCaseRequestSchema// { situationGraph: SituationGraph, previousQuestion: string, answer: string (1-5000), promptVersion?: string }
 /** @param {string} label */ /** @returns {string} */ export function makeNodeId(label)
 /** @param {{ id?, label, description, kind?, status?, confidence?, value?, unit?, ... }} opts */ /** @returns {SituationNode} */ export function makeNode(opts)
 /** @param {{ id?, fromNodeId, toNodeId, relationship?, confidence?, description? }} opts */ /** @returns {SituationEdge} */ export function makeEdge(opts)
 /** @param {{ centralStatement?, nodes?, edges?, activeUnknownNodeId?, resolvedNodeIds?, currentSummary? }} opts */ /** @returns {SituationGraph} */ export function makeGraph(opts)
 ```
 ### lib/graph/utils.js
 ```js
 export function validateGraphReferences(graph)     // → { valid: boolean, errors: string[] }
 export function detectDuplicateNodeIds(nodes)      // → { nodeId, count }[]
 export function detectDuplicateEdges(edges)        // { edgeId, fromNodeId, toNodeId, relationship }[]
 export function findDependentNodes(graph, nodeId)  // → string[] (transitive)
 export function findAffectedNodes(graph, nodeId)   // → string[] (direct + indirect via affects/dependsOn)
 /** @param {SituationGraph} graph */ /** @param {string} nodeId */ /** @param {string} newStatus */ /** @param {*} newValue */ /** @param {string} reason */
 export function resolveUnknownNode(graph, nodeId, newStatus, newValue, reason)  // → { success, error?, previousStatus?, newStatus?, previousValue?, newValue?, reason?, affectedNodes? }
 export function selectActiveUnknownCandidate(graph, resolvedNodeIds)  // → { nodeId, label, score } | null
 /** @param {SituationGraph} graph */ /** @param {GraphUpdate} update */
 export function applyGraphUpdate(graph, update)  // → { success: boolean, errors?, nodes?, edges?, resolvedNodeIds? }
 /** @param {SituationGraph} graph */ /** @param {GraphUpdate} update */
 export function validateGraphUpdate(graph, update)  // → { valid: boolean, errors: string[] }
 ```
 ### lib/graph/builder.js
 ```js
 export function buildInitialGraph(analysisData)  // @param {{ reconstruction, evidence? }} → { nodes: SituationNode[], edges: SituationEdge[] }
 export function buildMinimalGraph(scenario)       // @param {string} → { nodes, edges }
 export function describeGraph(graph)              // @param {{ nodes, edges }} → string (summary text)
 ```
 ## 2. Dependencies Between Files
 ```
 lib/analysis.js
  ├── getConfig()         from lib/config.js
  ├── getProvider()       from lib/llm/provider.js   [EXTERNAL]
  ├── buildPrompt()       from lib/reconstruction/prompt.js
  └── reconstructionV2/V1Schema  from lib/reconstruction/schema.js
 lib/graph/utils.js      ← imports situationNodeSchema, situationEdgeSchema, situationGraphSchema from schema.js
 lib/graph/builder.js    ← imports situationNodeSchema, situationEdgeSchema, makeNodeId from schema.js
 docs/v0.4-handoff.md   → references CaseOrchestrator.startCase()/updateCase() (not in any inspected file)
 ```
 ## 3. Side Effects (LLM Calls)
 | Function | LLM Call? | Details |
 |---|---|---|
 | `analyseScenario()` | **Yes** | `provider.generateReconstruction(prompt, model)` — POST to configured LLM. Prompt from `buildPrompt(scenario, version)`. |
 | All graph functions (`schema.js`, `utils.js`, `builder.js`) | No | Pure/deterministic only. |
 | `startCase()` / `updateCase()` (per handoff) | **Yes** | startCase: calls analyseScenario. updateCase: calls LLM via buildUpdatePrompt context + provider for GraphUpdate, then applyGraphUpdate(). |
 ## 4. Minimal Proposed Contract for API Functions
 ### startCase(body)
 - **Input:** `{ scenario: string (1-10000), promptVersion?: string }` — validated by `startCaseRequestSchema`.
 - **Flow:** validate → `analyseScenario()` → if ok, `buildInitialGraph(result)`; on failure return minimal graph via `buildMinimalGraph()`.
 - **Output (success):** `{ success: true, graphSummary: string, nodeCount: number, edgeCount: number, activeUnknownNodeId: string|undefined, nextQuestion: string }`
 - **Output (failure):** `{ success: false, error: string, graphSummary: string, nodeCount: number, edgeCount: number }`
 ### updateCase(body)
 - **Input:** `{ situationGraph: SituationGraph, previousQuestion: string (1+), answer: string (1-5000), promptVersion?: string }` — validated by `updateCaseRequestSchema`.
 - **Flow:** validate → `buildUpdatePrompt(ctx)` → LLM call for GraphUpdate proposal → `validateGraphUpdate()` → `applyGraphUpdate()` → resolve unknowns via `resolveUnknownNode()` → pick next candidate via `selectActiveUnknownCandidate()`.
 - **Output (success):** `{ success: true, graphSummary: string, nodeChanges: { added, updated, removed }, edgeChanges: { added, removed }, resolvedNodes: string[], nextQuestion: string|null }`
 - **Output (failure):** `{ success: false, error: string, graphSummary: string, nodeChanges: {}, edgeChanges: {}, resolvedNodes: [], nextQuestion: null }`
 ## 5. Missing Interfaces — TODO
 1. **[TODO]** `CaseOrchestrator` class described in handoff but absent from all five inspected files. startCase()/updateCase() wrappers need implementation per above contract.
 2. **[TODO]** `buildUpdatePrompt(ctx)` (per handoff lives in prompt-builder.js) — not reviewed; input/output needs a separate doc once the file is available.
 3. **[TODO]** LLM provider interface (`getProvider()`, `generateReconstruction(prompt, model)`) — external dependency. Assumes rawResponse is parseable JSON matching v0.2/v0.1 schema; needs explicit contract.
 4. **[TODO]** Error handling for updateCase() on malformed LLM JSON — handoff notes "generic 500"; needs structured retry/error contract.
 5. **[TODO]** Completion heuristic `getCompletionStatus()` referenced in handoff but absent; needs contract (e.g., "complete" when no unresolved unknown nodes).
 ---
 *End of contract.*
@@ -0,0 +1,258 @@
 # v0.4 Handoff — Confidence Engine (confidence-engine)
 **Date:** 2026-08-01
 **Branch:** `feature/reconstruction-v0.3`
 **Parent branch:** `main`
 ---
 ## 1. What This Project Is
 A Next.js app that performs evidence-based situation reconstruction on user-supplied scenarios. An LLM analyses the scenario, builds a directed graph of actors, systems, unknowns and relationships, then iteratively refines the graph through multi-turn Q&A with the user.
 ---
 ## 2. Recent Commit History
 | Commit | Message |
 |--------|---------|
 | `79ea2f6` | feat: add v0.3 normalised comparison reasoning |
 | `d72c7c5` | chore: establish clean v0.2 baseline |
 | `a2f9e47` | chore: preserve initial reconstruction prototype |
 Only **one commit** ahead of `main`: `79ea2f6` — the v0.3 normalised comparison reasoning work.
 ---
 ## 3. Current State Summary
 ### What's done and committed to this branch
 1. **v0.3 prompt** (`prompts/reconstruct-v0.3.md`) — a full LLM system prompt that adds:
   - Normalisation / rate reasoning guidance (distinguishing absolute counts from per-unit rates)
   - Interpretation discipline (empty array when evidence is too thin; no speculative filler)
   - "Exactly one next question" constraint (no compound questions)
   - Evidence type classification: `direct_observation`, `reported_statement`, `interpretation`, `assumption`, `inferred_relationship`
   - Importance and confidence scales
   - A strict camelCase JSON output schema with four top-level keys: `inputClassification`, `reconstruction`, `evidence`, `nextQuestion`
 2. **v0.3 prompt versioning** (`lib/reconstruction/prompt.js`) — exports `PROMPT_VERSIONS`, `DEFAULT_PROMPT_VERSION ("v0.3")`, and `buildPrompt(scenario, version)` for loading prompt templates from disk with scenario substitution.
 3. **Schema validation** (`lib/reconstruction/schema.js`) — Zod schemas for v0.2 output (`reconstructionV2Schema`). A `parseReconstructionV2(rawString)` helper is used in the analysis pipeline.
 4. **v0.3 reasoning tests** (`tests/v03-reasoning.test.js`) — extensive test suite covering:
   - Prompt version registration and loading
   - v0.3 guidance completeness (normalisation, rate vs count, correlation-vs-causation)
   - Schema validation with a realistic "production/complaints" fixture
   - Parse helper tests
 5. **Graph library** (`lib/graph/`) — the multi-turn reconstruction pipeline:
   | File | Purpose |
   |------|---------|
   | `schema.js` | Zod schemas for SituationNode, SituationEdge, SituationGraph, GraphUpdate; helpers like `makeNodeId`, `makeNode`, `makeEdge`, `makeGraph` |
   | `builder.js` | `buildInitialGraph(reconstruction, evidence)` — converts v0.2/v0.3 analysis output into a SituationGraph with deterministic nodes/edges; `buildMinimalGraph(scenario)` for fallback; `describeGraph(graph)` for display |
   | `orchestrator.js` | `CaseOrchestrator` class managing the full multi-turn lifecycle (idle → building → active); exports `startCase(body)` and `updateCase(body)` convenience functions for API routes |
   | `prompt-builder.js` | `buildUpdatePrompt(ctx)` — formats current graph state + Q&A context into a system prompt for the LLM update-evaluation turn |
   | `utils.js` | Deterministic graph operations: `validateGraphReferences`, `detectDuplicateNodeIds`, `detectDuplicateEdges`, `findDependentNodes`, `findAffectedNodes`, `resolveUnknownNode`, `selectActiveUnknownCandidate`, `applyGraphUpdate`, `validateGraphUpdate` |
 6. **API routes** (`app/api/`)
   | Route | Purpose |
   |-------|---------|
   | `POST /api/start-case` | Start a new reconstruction case — accepts `{ scenario, promptVersion? }`, returns graph summary, node/edge counts, next question |
   | `POST /api/update-case` | Process a turn — accepts `{ scenario, graph, answer, currentQuestion?, turnCount?, modelName? }`, returns updated graph summary, next question, changes summary |
 7. **Smoke test** (`tests/smoke.test.js`) — basic integration test for the start-case API route.
 ### What's NOT yet committed (untracked files from git status)
 | File | Description |
 |------|-------------|
 | `lib/graph/` (full directory) | The multi-turn graph library — built but NOT yet committed to any branch. These are the new untracked files: `builder.js`, `orchestrator.js`, `prompt-builder.js`, `schema.js`, `utils.js` |
 | `tests/graph/` (full directory) | Tests for the graph library — also untracked: `builder.test.js`, `orchestrator.test.js`, `prompt-builder.test.js`, `schema.test.js`, `utils.test.js` |
 | `app/api/start-case/route.js` | New API route (untracked) |
 | `app/api/update-case/route.js` | New API route (untracked) |
 > **Important:** The git status shows these files as untracked (`??`). They exist on disk but have never been staged or committed. You need to decide whether to commit them now or integrate them differently.
 ---
 ## 4. Test Status
 ```
 Test Files:  4 failed | 4 passed (8)
 Tests:       5 failed | 216 passed (221)
 ```
 ### Known failures
 The failures cluster in `tests/graph/`:
 - **`prompt-builder.test.js`** — test expects the literal string `"Existing or newly added nodes"` but the prompt template currently says `"existing or newly added nodes"` (case mismatch). The SYSTEM_PROMPT_HEADER constant uses lowercase.
 - Other graph tests likely have similar fixture/reference issues.
 Run `npx vitest run tests/graph/ --reporter=verbose` for full details.
 ---
 ## 5. Architecture Overview
 ```
 User scenario
      │
      ▼
 ┌──────────────┐     ┌─────────────────┐     ┌──────────────┐
 │ analyseScenario│──▶│ buildPrompt     │──▶│ LLM (v0.3)   │
 │  (lib/analysis.js) │  (reconstruction/prompt.js) │              │
 └──────────────┘     └─────────────────┘     └──────┬───────┘
                                                      │
                                                      ▼
                                              ┌──────────────┐
                                              │ Parse output │
                                              │ (Zod/parse   │
                                              │  Reconstruction│
                                              │  V2)          │
                                              └──────┬───────┘
                                                     │
                     ┌───────────────────────────────┤
                     ▼                               ▼
              ┌──────────────┐           ┌──────────────────┐
              │buildInitialGraph│         │ buildMinimalGraph │
              │  (graph/builder)│         │  (fallback)       │
              └──────┬─────────┘           └──────────────────┘
                     │
                     ▼
              ┌──────────────┐
              │SituationGraph │  ← Zod-validated graph structure
              │  {nodes, edges}│     nodes: observation/metric/unknown/...
              └──────┬───────┘     edges: supports/weakens/causes/...
                     │
          (multi-turn loop via updateCase)
                     │
           ┌─────────▼─────────┐
           │buildUpdatePrompt  │  → LLM proposes GraphUpdate
           │                   │
           │applyGraphUpdate   │  → deterministic, validated
           │validateGraphUpdate│     (no direct LLM mutation)
           └───────────────────┘
 ```
 ---
 ## 6. Key Design Decisions
 ### Normalisation / rate reasoning (v0.3 focus)
 The v0.3 prompt explicitly instructs the model to:
 - Always consider whether a denominator/exposure metric is needed when counts change alongside scale
 - Distinguish absolute count from rate
 - Avoid treating two rising counts as causal evidence (production growth may outpace complaint growth)
 - Request the per-unit metric as the highest-value next question
 ### Graph immutability
 LLM proposals are never applied directly. All mutations go through `applyGraphUpdate()` in `lib/graph/utils.js`, which:
 - Validates all node/edge references exist
 - Rejects duplicate IDs
 - Enforces a max graph size (500 nodes) and update size (100KB)
 - Returns the full new state for validation
 ### Prompt versioning
 - Default is `"v0.3"` but `PROMPT_VERSIONS` includes `"v0.2"` for backward compatibility
 - `RECONSTRUCTION_PROMPT_VERSION` env var can override default at module load time
 - Prompts are loaded from `prompts/reconstruct-v0.{version}.md` on disk
 ### Deterministic node IDs
 Node IDs are computed via a deterministic hash of the label: `makeNodeId(label)`. This avoids conflicts but means nodes must be created with consistent labels to get consistent IDs.
 ---
 ## 7. Open Questions / TODOs for Next Developer
 1. **Untracked graph library** — `lib/graph/` and `tests/graph/` are untracked on disk. Do we commit them as part of v0.4, or keep them in a separate branch?
 2. **Test failures** — 5 tests fail across the graph test suite. The prompt-builder case-sensitivity issue needs fixing. Review all failing tests before merging.
 3. **Missing `RECONSTRUCTION_PROMPT_VERSION` env var docs** — The system uses an env var override but it's not documented in `.env.example`. Add it if it's intended to be configurable.
 4. **Provider integration** — `lib/llm/provider.js` is imported by the orchestrator (`getProvider()`, `generateReconstruction()`). Verify the provider implementation matches what this code expects.
 5. **Graph completeness heuristic** — `CaseOrchestrator.getCompletionStatus()` returns `"complete"` when no unknown nodes remain, but doesn't consider whether all important observations have been verified.
 6. **Error resilience in update flow** — If the LLM returns malformed JSON, the update route returns a 500 with a generic error message. Consider retry logic or structured error parsing.
 7. **`buildUpdatePrompt` SYSTEM_PROMPT_HEADER is a module-level constant** — it's hardcoded and never versioned. If v0.5 changes the update-evaluation prompt style, this will need to become a template.
 8. **The `nextQuestion` field on `/api/start-case` response** includes the adapted question (original + active unknown label appended). The client may want the original and adapted separately.
 ---
 ## 8. File Inventory (new / changed files on this branch)
 ### Prompts
 - `prompts/reconstruct-v0.3.md` — **NEW** — v0.3 system prompt (161 lines)
 - `prompts/reconstruct-v0.2.md` — **existing** — baseline prompt
 ### Core library
 - `lib/analysis.js` — **MODIFIED** — analyseScenario function (uses v0.3 prompt by default)
 - `lib/reconstruction/prompt.js` — **MODIFIED** — prompt versioning exports
 - `lib/reconstruction/schema.js` — **existing** — Zod schemas + parseReconstructionV2
 ### Graph library (untracked on disk)
 - `lib/graph/builder.js` — buildInitialGraph, buildMinimalGraph, describeGraph
 - `lib/graph/orchestrator.js` — CaseOrchestrator class, startCase, updateCase
 - `lib/graph/prompt-builder.js` — buildUpdatePrompt + SYSTEM_PROMPT_HEADER
 - `lib/graph/schema.js` — SituationNode/Edge/Graph/Update Zod schemas
 - `lib/graph/utils.js` — validation, dedup, dependency, and apply utilities
 ### API routes (untracked on disk)
 - `app/api/start-case/route.js`
 - `app/api/update-case/route.js`
 ### Tests (untracked on disk)
 - `tests/graph/builder.test.js`
 - `tests/graph/orchestrator.test.js`
 - `tests/graph/prompt-builder.test.js`
 - `tests/graph/schema.test.js`
 - `tests/graph/utils.test.js`
 - `tests/v03-reasoning.test.js` — **committed** to current branch
 - `tests/smoke.test.js`
 ### Config changes
 - `package.json` — added dependency (verify which one)
 - `playwright.config.js` — added/modified for integration testing
 - `.env.local` — exists locally (not committed)
 ---
 ## 9. How to Run
 ```bash
 # Install dependencies
 npm install
 # Unit tests
 npx vitest run
 # Graph library tests (has 5 failures)
 npx vitest run tests/graph/ --reporter=verbose
 # Start dev server
 npm run dev
 # API endpoints
 # POST /api/start-case    → { scenario: "..." }
 # POST /api/update-case   → { graph: {...}, answer: "...", ... }
 ```
 ---
 ## 10. What to Do First (Recommended Priorities)
 1. **Review and fix the 5 failing tests** — likely simple string/fixture issues
 2. **Decide on the untracked files** — commit them, or create a v0.4 branch from this point
 3. **Verify the LLM provider integration** — ensure `getProvider()` and `generateReconstruction()` are wired up correctly
 4. **Add env var documentation** for `RECONSTRUCTION_PROMPT_VERSION` to `.env.example`
 5. **Smoke test end-to-end** — call `/api/start-case` with a real scenario and verify the full flow
 ---
 *End of handoff.*