diff --git a/docs/orchestrator-contract.md b/docs/orchestrator-contract.md new file mode 100644 index 0000000..a03506b --- /dev/null +++ b/docs/orchestrator-contract.md @@ -0,0 +1,113 @@ +# Orchestrator Contract — Confidence Engine v0.4 + +## 1. Exported Function Signatures & Shape (JavaScript) + +### lib/analysis.js + +```js +export async function analyseScenario(scenario, opts = {}) +// @param {string} scenario +// @param {{ promptVersion?: "v0.2" | "v0.3" }} [opts] +// @returns {Promise<{ success: boolean, validationStatus: "valid"|"invalid", +// modelName: string|null, responseDurationMs: number, rawResponse: string|null, +// promptVersion: string|null, inputClassification: object|null, reconstruction: object|null, +// evidence: object[]|undefined, nextQuestion: string|undefined, errors: string[]|undefined, +// error: string|undefined, statusCode: number|undefined }>} + +export const PROMPT_VERSIONS // { [key: string]: string } +export const DEFAULT_PROMPT_VERSION // "v0.2" +``` + +### lib/graph/schema.js + +```js +export const SituationKind // { observation, reported_claim, metric, state, transition, relationship, assumption, unknown, conclusion } +export const SituationStatus // { known, unknown, provisional, supported, weakened, contradicted, resolved } +export const ConfidenceLevel // { low, medium, high } +export const SituationRelationship // { supports, weakens, contradicts, depends_on, causes, may_cause, measures, compares_with, updates, other } + +export const situationNodeSchema // Zod → {@typedef SituationNode} +export const situationEdgeSchema // Zod → {@typedef SituationEdge} +export const situationGraphSchema // Zod → {@typedef SituationGraph} +export const graphUpdateSchema // Zod → {@typedef GraphUpdate} +export const startCaseRequestSchema // { scenario: string (1-10000), promptVersion?: string } +export const updateCaseRequestSchema// { situationGraph: SituationGraph, previousQuestion: string, answer: string (1-5000), promptVersion?: string } + +/** @param {string} label */ /** @returns {string} */ export function makeNodeId(label) +/** @param {{ id?, label, description, kind?, status?, confidence?, value?, unit?, ... }} opts */ /** @returns {SituationNode} */ export function makeNode(opts) +/** @param {{ id?, fromNodeId, toNodeId, relationship?, confidence?, description? }} opts */ /** @returns {SituationEdge} */ export function makeEdge(opts) +/** @param {{ centralStatement?, nodes?, edges?, activeUnknownNodeId?, resolvedNodeIds?, currentSummary? }} opts */ /** @returns {SituationGraph} */ export function makeGraph(opts) +``` + +### lib/graph/utils.js + +```js +export function validateGraphReferences(graph) // → { valid: boolean, errors: string[] } +export function detectDuplicateNodeIds(nodes) // → { nodeId, count }[] +export function detectDuplicateEdges(edges) // { edgeId, fromNodeId, toNodeId, relationship }[] +export function findDependentNodes(graph, nodeId) // → string[] (transitive) +export function findAffectedNodes(graph, nodeId) // → string[] (direct + indirect via affects/dependsOn) +/** @param {SituationGraph} graph */ /** @param {string} nodeId */ /** @param {string} newStatus */ /** @param {*} newValue */ /** @param {string} reason */ +export function resolveUnknownNode(graph, nodeId, newStatus, newValue, reason) // → { success, error?, previousStatus?, newStatus?, previousValue?, newValue?, reason?, affectedNodes? } +export function selectActiveUnknownCandidate(graph, resolvedNodeIds) // → { nodeId, label, score } | null +/** @param {SituationGraph} graph */ /** @param {GraphUpdate} update */ +export function applyGraphUpdate(graph, update) // → { success: boolean, errors?, nodes?, edges?, resolvedNodeIds? } +/** @param {SituationGraph} graph */ /** @param {GraphUpdate} update */ +export function validateGraphUpdate(graph, update) // → { valid: boolean, errors: string[] } +``` + +### lib/graph/builder.js + +```js +export function buildInitialGraph(analysisData) // @param {{ reconstruction, evidence? }} → { nodes: SituationNode[], edges: SituationEdge[] } +export function buildMinimalGraph(scenario) // @param {string} → { nodes, edges } +export function describeGraph(graph) // @param {{ nodes, edges }} → string (summary text) +``` + +## 2. Dependencies Between Files + +``` +lib/analysis.js + ├── getConfig() from lib/config.js + ├── getProvider() from lib/llm/provider.js [EXTERNAL] + ├── buildPrompt() from lib/reconstruction/prompt.js + └── reconstructionV2/V1Schema from lib/reconstruction/schema.js + +lib/graph/utils.js ← imports situationNodeSchema, situationEdgeSchema, situationGraphSchema from schema.js +lib/graph/builder.js ← imports situationNodeSchema, situationEdgeSchema, makeNodeId from schema.js +docs/v0.4-handoff.md → references CaseOrchestrator.startCase()/updateCase() (not in any inspected file) +``` + +## 3. Side Effects (LLM Calls) + +| Function | LLM Call? | Details | +|---|---|---| +| `analyseScenario()` | **Yes** | `provider.generateReconstruction(prompt, model)` — POST to configured LLM. Prompt from `buildPrompt(scenario, version)`. | +| All graph functions (`schema.js`, `utils.js`, `builder.js`) | No | Pure/deterministic only. | +| `startCase()` / `updateCase()` (per handoff) | **Yes** | startCase: calls analyseScenario. updateCase: calls LLM via buildUpdatePrompt context + provider for GraphUpdate, then applyGraphUpdate(). | + +## 4. Minimal Proposed Contract for API Functions + +### startCase(body) +- **Input:** `{ scenario: string (1-10000), promptVersion?: string }` — validated by `startCaseRequestSchema`. +- **Flow:** validate → `analyseScenario()` → if ok, `buildInitialGraph(result)`; on failure return minimal graph via `buildMinimalGraph()`. +- **Output (success):** `{ success: true, graphSummary: string, nodeCount: number, edgeCount: number, activeUnknownNodeId: string|undefined, nextQuestion: string }` +- **Output (failure):** `{ success: false, error: string, graphSummary: string, nodeCount: number, edgeCount: number }` + +### updateCase(body) +- **Input:** `{ situationGraph: SituationGraph, previousQuestion: string (1+), answer: string (1-5000), promptVersion?: string }` — validated by `updateCaseRequestSchema`. +- **Flow:** validate → `buildUpdatePrompt(ctx)` → LLM call for GraphUpdate proposal → `validateGraphUpdate()` → `applyGraphUpdate()` → resolve unknowns via `resolveUnknownNode()` → pick next candidate via `selectActiveUnknownCandidate()`. +- **Output (success):** `{ success: true, graphSummary: string, nodeChanges: { added, updated, removed }, edgeChanges: { added, removed }, resolvedNodes: string[], nextQuestion: string|null }` +- **Output (failure):** `{ success: false, error: string, graphSummary: string, nodeChanges: {}, edgeChanges: {}, resolvedNodes: [], nextQuestion: null }` + +## 5. Missing Interfaces — TODO + +1. **[TODO]** `CaseOrchestrator` class described in handoff but absent from all five inspected files. startCase()/updateCase() wrappers need implementation per above contract. +2. **[TODO]** `buildUpdatePrompt(ctx)` (per handoff lives in prompt-builder.js) — not reviewed; input/output needs a separate doc once the file is available. +3. **[TODO]** LLM provider interface (`getProvider()`, `generateReconstruction(prompt, model)`) — external dependency. Assumes rawResponse is parseable JSON matching v0.2/v0.1 schema; needs explicit contract. +4. **[TODO]** Error handling for updateCase() on malformed LLM JSON — handoff notes "generic 500"; needs structured retry/error contract. +5. **[TODO]** Completion heuristic `getCompletionStatus()` referenced in handoff but absent; needs contract (e.g., "complete" when no unresolved unknown nodes). + +--- + +*End of contract.* diff --git a/docs/v0.4-handoff.md b/docs/v0.4-handoff.md new file mode 100644 index 0000000..6f3b290 --- /dev/null +++ b/docs/v0.4-handoff.md @@ -0,0 +1,258 @@ +# v0.4 Handoff — Confidence Engine (confidence-engine) + +**Date:** 2026-08-01 +**Branch:** `feature/reconstruction-v0.3` +**Parent branch:** `main` + +--- + +## 1. What This Project Is + +A Next.js app that performs evidence-based situation reconstruction on user-supplied scenarios. An LLM analyses the scenario, builds a directed graph of actors, systems, unknowns and relationships, then iteratively refines the graph through multi-turn Q&A with the user. + +--- + +## 2. Recent Commit History + +| Commit | Message | +|--------|---------| +| `79ea2f6` | feat: add v0.3 normalised comparison reasoning | +| `d72c7c5` | chore: establish clean v0.2 baseline | +| `a2f9e47` | chore: preserve initial reconstruction prototype | + +Only **one commit** ahead of `main`: `79ea2f6` — the v0.3 normalised comparison reasoning work. + +--- + +## 3. Current State Summary + +### What's done and committed to this branch + +1. **v0.3 prompt** (`prompts/reconstruct-v0.3.md`) — a full LLM system prompt that adds: + - Normalisation / rate reasoning guidance (distinguishing absolute counts from per-unit rates) + - Interpretation discipline (empty array when evidence is too thin; no speculative filler) + - "Exactly one next question" constraint (no compound questions) + - Evidence type classification: `direct_observation`, `reported_statement`, `interpretation`, `assumption`, `inferred_relationship` + - Importance and confidence scales + - A strict camelCase JSON output schema with four top-level keys: `inputClassification`, `reconstruction`, `evidence`, `nextQuestion` + +2. **v0.3 prompt versioning** (`lib/reconstruction/prompt.js`) — exports `PROMPT_VERSIONS`, `DEFAULT_PROMPT_VERSION ("v0.3")`, and `buildPrompt(scenario, version)` for loading prompt templates from disk with scenario substitution. + +3. **Schema validation** (`lib/reconstruction/schema.js`) — Zod schemas for v0.2 output (`reconstructionV2Schema`). A `parseReconstructionV2(rawString)` helper is used in the analysis pipeline. + +4. **v0.3 reasoning tests** (`tests/v03-reasoning.test.js`) — extensive test suite covering: + - Prompt version registration and loading + - v0.3 guidance completeness (normalisation, rate vs count, correlation-vs-causation) + - Schema validation with a realistic "production/complaints" fixture + - Parse helper tests + +5. **Graph library** (`lib/graph/`) — the multi-turn reconstruction pipeline: + + | File | Purpose | + |------|---------| + | `schema.js` | Zod schemas for SituationNode, SituationEdge, SituationGraph, GraphUpdate; helpers like `makeNodeId`, `makeNode`, `makeEdge`, `makeGraph` | + | `builder.js` | `buildInitialGraph(reconstruction, evidence)` — converts v0.2/v0.3 analysis output into a SituationGraph with deterministic nodes/edges; `buildMinimalGraph(scenario)` for fallback; `describeGraph(graph)` for display | + | `orchestrator.js` | `CaseOrchestrator` class managing the full multi-turn lifecycle (idle → building → active); exports `startCase(body)` and `updateCase(body)` convenience functions for API routes | + | `prompt-builder.js` | `buildUpdatePrompt(ctx)` — formats current graph state + Q&A context into a system prompt for the LLM update-evaluation turn | + | `utils.js` | Deterministic graph operations: `validateGraphReferences`, `detectDuplicateNodeIds`, `detectDuplicateEdges`, `findDependentNodes`, `findAffectedNodes`, `resolveUnknownNode`, `selectActiveUnknownCandidate`, `applyGraphUpdate`, `validateGraphUpdate` | + +6. **API routes** (`app/api/`) + + | Route | Purpose | + |-------|---------| + | `POST /api/start-case` | Start a new reconstruction case — accepts `{ scenario, promptVersion? }`, returns graph summary, node/edge counts, next question | + | `POST /api/update-case` | Process a turn — accepts `{ scenario, graph, answer, currentQuestion?, turnCount?, modelName? }`, returns updated graph summary, next question, changes summary | + +7. **Smoke test** (`tests/smoke.test.js`) — basic integration test for the start-case API route. + +### What's NOT yet committed (untracked files from git status) + +| File | Description | +|------|-------------| +| `lib/graph/` (full directory) | The multi-turn graph library — built but NOT yet committed to any branch. These are the new untracked files: `builder.js`, `orchestrator.js`, `prompt-builder.js`, `schema.js`, `utils.js` | +| `tests/graph/` (full directory) | Tests for the graph library — also untracked: `builder.test.js`, `orchestrator.test.js`, `prompt-builder.test.js`, `schema.test.js`, `utils.test.js` | +| `app/api/start-case/route.js` | New API route (untracked) | +| `app/api/update-case/route.js` | New API route (untracked) | + +> **Important:** The git status shows these files as untracked (`??`). They exist on disk but have never been staged or committed. You need to decide whether to commit them now or integrate them differently. + +--- + +## 4. Test Status + +``` +Test Files: 4 failed | 4 passed (8) +Tests: 5 failed | 216 passed (221) +``` + +### Known failures + +The failures cluster in `tests/graph/`: +- **`prompt-builder.test.js`** — test expects the literal string `"Existing or newly added nodes"` but the prompt template currently says `"existing or newly added nodes"` (case mismatch). The SYSTEM_PROMPT_HEADER constant uses lowercase. +- Other graph tests likely have similar fixture/reference issues. + +Run `npx vitest run tests/graph/ --reporter=verbose` for full details. + +--- + +## 5. Architecture Overview + +``` +User scenario + │ + ▼ +┌──────────────┐ ┌─────────────────┐ ┌──────────────┐ +│ analyseScenario│──▶│ buildPrompt │──▶│ LLM (v0.3) │ +│ (lib/analysis.js) │ (reconstruction/prompt.js) │ │ +└──────────────┘ └─────────────────┘ └──────┬───────┘ + │ + ▼ + ┌──────────────┐ + │ Parse output │ + │ (Zod/parse │ + │ Reconstruction│ + │ V2) │ + └──────┬───────┘ + │ + ┌───────────────────────────────┤ + ▼ ▼ + ┌──────────────┐ ┌──────────────────┐ + │buildInitialGraph│ │ buildMinimalGraph │ + │ (graph/builder)│ │ (fallback) │ + └──────┬─────────┘ └──────────────────┘ + │ + ▼ + ┌──────────────┐ + │SituationGraph │ ← Zod-validated graph structure + │ {nodes, edges}│ nodes: observation/metric/unknown/... + └──────┬───────┘ edges: supports/weakens/causes/... + │ + (multi-turn loop via updateCase) + │ + ┌─────────▼─────────┐ + │buildUpdatePrompt │ → LLM proposes GraphUpdate + │ │ + │applyGraphUpdate │ → deterministic, validated + │validateGraphUpdate│ (no direct LLM mutation) + └───────────────────┘ +``` + +--- + +## 6. Key Design Decisions + +### Normalisation / rate reasoning (v0.3 focus) +The v0.3 prompt explicitly instructs the model to: +- Always consider whether a denominator/exposure metric is needed when counts change alongside scale +- Distinguish absolute count from rate +- Avoid treating two rising counts as causal evidence (production growth may outpace complaint growth) +- Request the per-unit metric as the highest-value next question + +### Graph immutability +LLM proposals are never applied directly. All mutations go through `applyGraphUpdate()` in `lib/graph/utils.js`, which: +- Validates all node/edge references exist +- Rejects duplicate IDs +- Enforces a max graph size (500 nodes) and update size (100KB) +- Returns the full new state for validation + +### Prompt versioning +- Default is `"v0.3"` but `PROMPT_VERSIONS` includes `"v0.2"` for backward compatibility +- `RECONSTRUCTION_PROMPT_VERSION` env var can override default at module load time +- Prompts are loaded from `prompts/reconstruct-v0.{version}.md` on disk + +### Deterministic node IDs +Node IDs are computed via a deterministic hash of the label: `makeNodeId(label)`. This avoids conflicts but means nodes must be created with consistent labels to get consistent IDs. + +--- + +## 7. Open Questions / TODOs for Next Developer + +1. **Untracked graph library** — `lib/graph/` and `tests/graph/` are untracked on disk. Do we commit them as part of v0.4, or keep them in a separate branch? + +2. **Test failures** — 5 tests fail across the graph test suite. The prompt-builder case-sensitivity issue needs fixing. Review all failing tests before merging. + +3. **Missing `RECONSTRUCTION_PROMPT_VERSION` env var docs** — The system uses an env var override but it's not documented in `.env.example`. Add it if it's intended to be configurable. + +4. **Provider integration** — `lib/llm/provider.js` is imported by the orchestrator (`getProvider()`, `generateReconstruction()`). Verify the provider implementation matches what this code expects. + +5. **Graph completeness heuristic** — `CaseOrchestrator.getCompletionStatus()` returns `"complete"` when no unknown nodes remain, but doesn't consider whether all important observations have been verified. + +6. **Error resilience in update flow** — If the LLM returns malformed JSON, the update route returns a 500 with a generic error message. Consider retry logic or structured error parsing. + +7. **`buildUpdatePrompt` SYSTEM_PROMPT_HEADER is a module-level constant** — it's hardcoded and never versioned. If v0.5 changes the update-evaluation prompt style, this will need to become a template. + +8. **The `nextQuestion` field on `/api/start-case` response** includes the adapted question (original + active unknown label appended). The client may want the original and adapted separately. + +--- + +## 8. File Inventory (new / changed files on this branch) + +### Prompts +- `prompts/reconstruct-v0.3.md` — **NEW** — v0.3 system prompt (161 lines) +- `prompts/reconstruct-v0.2.md` — **existing** — baseline prompt + +### Core library +- `lib/analysis.js` — **MODIFIED** — analyseScenario function (uses v0.3 prompt by default) +- `lib/reconstruction/prompt.js` — **MODIFIED** — prompt versioning exports +- `lib/reconstruction/schema.js` — **existing** — Zod schemas + parseReconstructionV2 + +### Graph library (untracked on disk) +- `lib/graph/builder.js` — buildInitialGraph, buildMinimalGraph, describeGraph +- `lib/graph/orchestrator.js` — CaseOrchestrator class, startCase, updateCase +- `lib/graph/prompt-builder.js` — buildUpdatePrompt + SYSTEM_PROMPT_HEADER +- `lib/graph/schema.js` — SituationNode/Edge/Graph/Update Zod schemas +- `lib/graph/utils.js` — validation, dedup, dependency, and apply utilities + +### API routes (untracked on disk) +- `app/api/start-case/route.js` +- `app/api/update-case/route.js` + +### Tests (untracked on disk) +- `tests/graph/builder.test.js` +- `tests/graph/orchestrator.test.js` +- `tests/graph/prompt-builder.test.js` +- `tests/graph/schema.test.js` +- `tests/graph/utils.test.js` +- `tests/v03-reasoning.test.js` — **committed** to current branch +- `tests/smoke.test.js` + +### Config changes +- `package.json` — added dependency (verify which one) +- `playwright.config.js` — added/modified for integration testing +- `.env.local` — exists locally (not committed) + +--- + +## 9. How to Run + +```bash +# Install dependencies +npm install + +# Unit tests +npx vitest run + +# Graph library tests (has 5 failures) +npx vitest run tests/graph/ --reporter=verbose + +# Start dev server +npm run dev + +# API endpoints +# POST /api/start-case → { scenario: "..." } +# POST /api/update-case → { graph: {...}, answer: "...", ... } +``` + +--- + +## 10. What to Do First (Recommended Priorities) + +1. **Review and fix the 5 failing tests** — likely simple string/fixture issues +2. **Decide on the untracked files** — commit them, or create a v0.4 branch from this point +3. **Verify the LLM provider integration** — ensure `getProvider()` and `generateReconstruction()` are wired up correctly +4. **Add env var documentation** for `RECONSTRUCTION_PROMPT_VERSION` to `.env.example` +5. **Smoke test end-to-end** — call `/api/start-case` with a real scenario and verify the full flow + +--- + +*End of handoff.*