feat(confidence-engine): strengthen initial reconstruction contract

2026-09-05 10:43:57 +01:00
parent 37245a8e28
commit 13fbceee7a
4 changed files with 350 additions and 7 deletions
@@ -1556,6 +1556,58 @@ The initial decomposition represents core factual uncertainty channels (normalis

 ---

+## v0.4 — Strengthened Initial Reconstruction Contract (Implementation)
+
+**Status:** COMPLETE — deterministic tests pass, one live smoke accepted
+
+### Starting checkpoint
+
+- **Branch:** `feature/initial-decomposition-v0.61`
+- **HEAD:** `37245a8` — fix(confidence-engine): expose reconstruction failure evidence
+
+### What was implemented
+
+| File | Change |
+|---|---|
+| `prompts/reconstruct-v0.4.md` | New prompt file — provenance-preserving decomposition contract with explicit stop boundary, supplied-relationship preservation, interpretation separation, normalisation guidance |
+| `lib/reconstruction/prompt.js` | Added v0.4 to PROMPT_VERSIONS; set DEFAULT_PROMPT_VERSION = "v0.4"; added buildV4Prompt loader; added v0.4 case to switch/default |
+| `tests/v03-reasoning.test.js` | Updated existing default-version assertions to accept v0.4; added 15 new targeted v0.4 prompt contract tests (loading, output-format preservation, provenance stop contract, relationship preservation, interpretation separation, normalisation, next-question discipline) |
+
+### Architecture present
+
+| Principle | Present |
+|---|---|
+| Provenance-preserving decomposition | ✅ — rule 1: each child's distinguishing semantic content must be directly supported by meaning supplied; plausible/likely/domain-typical not sufficient |
+| Relationship preservation | ✅ — "Preserve supplied relationships" section (rules 36-37) |
+| Explicit stop boundary | ✅ — "Explicit stop boundary for decomposition" section |
+| Interpretation separation | ✅ — rule 6: interpretation never smuggled into decomposition; provisional status required |
+
+### Deterministic verification
+
+| Item | Value |
+|---|---|
+| Command | `npx vitest run tests/v03-reasoning.test.js` |
+| Result | **89/89 passed** on first run, zero reruns |
+
+### Live smoke
+
+| Item | Value |
+|---|---|
+| Route | POST /api/cases/start (no promptVersion — uses default v0.4) |
+| HTTP status | 200 |
+| Validation status | valid |
+| Prompt version in diagnostics | v0.4 |
+| Supplied meaning preserved | ✅ complaints +35%, production +40%, complaint-rate-per-unit uncertainty, late delivery, minor defects, supplier change, weekend shift, CRM/counting uncertainty, £120k inspection decision |
+| Provenance boundary enforced | ✅ no decomposition into unsupported CRM mechanisms (segments, training, classification, logging delays) |
+| Relationship boundary preserved | ✅ unresolved dependency between automated-inspection appropriateness and complaint-kind preserved as unknown investigative path |
+| Interpretation separation | ✅ model-generated interpretations remain provisional in plausibleInterpretations, not smuggled into decomposition structure |
+
+### Acceptance: PASS
+
+**v0.4 initial reconstruction is now production default.** This does NOT yet establish repeatability or cross-scenario consistency.
+
+---
+
 ## v0.61 active restart point

 The current handoff direction is the repeated-same-input decomposition stability experiment described in the **active design** section at the top of §v0.61.
@@ -7,14 +7,14 @@ const __dirname = dirname(__filename);
 const PROMPTS_DIR = join(__dirname, "../../prompts");

 /** Available prompt versions */
-export const PROMPT_VERSIONS = ["v0.1", "v0.2", "v0.3"];
+export const PROMPT_VERSIONS = ["v0.1", "v0.2", "v0.3", "v0.4"];

 /** Default prompt version (override via RECONSTRUCTION_PROMPT_VERSION env var) */
 const defaultVersionFromEnv = process.env.RECONSTRUCTION_PROMPT_VERSION;
 export const DEFAULT_PROMPT_VERSION =
  defaultVersionFromEnv && PROMPT_VERSIONS.includes(defaultVersionFromEnv)
    ? defaultVersionFromEnv
-    : "v0.3";
+    : "v0.4";

 /** Build a v0.1 (extraction-only) prompt inline for backward compatibility */
 function buildV1Prompt(scenario) {
@@ -77,15 +77,29 @@ async function buildV3Prompt(scenario) {
  }
 }

+/** Load a versioned prompt from disk and substitute {{SCENARIO}} */
+async function buildV4Prompt(scenario) {
+  try {
+    const content = await fs.readFile(
+      join(PROMPTS_DIR, "reconstruct-v0.4.md"),
+      "utf-8",
+    );
+    return content.replace("{{SCENARIO}}", scenario);
+  } catch {
+    // Fall back to v0.3 prompt if v0.4 file is missing
+    return buildV3Prompt(scenario);
+  }
+}
+
 /**
 * Build an analysis prompt for the given version.
 * @param {string} scenario - The scenario text
- * @param {"v0.1" | "v0.2" | "v0.3"} [version="v0.3"] - Prompt version
+ * @param {"v0.1" | "v0.2" | "v0.3" | "v0.4"} [version="v0.4"] - Prompt version
 * @param {object} [opts] - Optional experimental parameters
 * @param {string} [opts.experimentInstruction] - Bounded experimental instruction block appended to the base prompt (production prompt is never replaced)
 * @returns {Promise<{prompt: string, version: string}>}
 */
-export async function buildPrompt(scenario, version = "v0.3", opts = {}) {
+export async function buildPrompt(scenario, version = "v0.4", opts = {}) {
  let prompt;
  switch (version) {
    case "v0.1":
@@ -94,6 +108,9 @@ export async function buildPrompt(scenario, version = "v0.3", opts = {}) {
    case "v0.2":
      prompt = await buildV2Prompt(scenario);
      break;
+    case "v0.4":
+      prompt = await buildV4Prompt(scenario);
+      break;
    default: // v0.3
      prompt = await buildV3Prompt(scenario);
      break;
@@ -0,0 +1,179 @@
+You are a neutral analyst performing evidence-based situation reconstruction.
+
+## Rules
+
+1. Do NOT invent facts, context or causes. Only include information present in the scenario. For decomposition structure — splitting a concept into child categories — each child's distinguishing semantic content must be directly supported by meaning supplied by the user in their text (semantic-equivalent paraphrase is permitted; literal word-for-word matching is not required). A decomposition child is NOT permitted if producing it requires adding a new actor, category, segment, mechanism, cause, process, subtype, condition, system behaviour, operational distinction, or equivalent semantic proposition that the user did not supply. The following are NOT sufficient to justify creating new decomposition structure: plausible, likely, domain-typical, possibly implied, clearly implied, worth investigating, could be relevant, might explain. Those standards may inform a provisional interpretation but never decomposition.
+
+2. First determine what kind of input has been supplied. Use only these classification types:
+   observed_problem, unexplained_change, contradiction, decision_request, causal_claim,
+   reported_claim, fault_report, ambiguous_statement, question, desired_outcome,
+   insufficient_context, other
+
+3. Choose reasoning modes from:
+   establish_baseline, identify_difference, reconstruct_transition, decompose_aggregate,
+   validate_measurement, validate_claim, investigate_contradiction, clarify_meaning,
+   decision_support, fault_investigation, identify_missing_information, test_possible_explanations, other
+
+4. Look for anchors: actor, system or object, expected outcome, observed outcome,
+   previous state, current state, difference between groups, change over time, measurement,
+   evidence source, proposed action.
+
+5. Identify meaningful differences (e.g., some succeed while others fail; revenue rises while cash falls).
+
+6. Keep multiple plausible interpretations separate where the evidence does not distinguish them. Model-generated possible interpretations are permitted ONLY when they satisfy all of the following conditions: (a) supported by supplied evidence; (b) clearly represented in plausibleInterpretations; (c) explicitly marked as provisional; (d) their supporting evidence is identified; (e) NOT presented as observation, supplied unknown, transition, relationship, or established cause. A plausible interpretation must NEVER be smuggled into decomposition structure. Do NOT generate plausible interpretations merely to fill a list — return an empty array [] if the evidence does not support useful, distinct interpretations.
+
+7. When the user explicitly names multiple distinct possible explanations, causes, constraints, or dependencies for the situation, preserve those user-stated alternatives as separate importantUnknowns when they can sensibly be investigated independently. Do not collapse them into one "which factor", "relative contribution", or equivalent umbrella unknown. Do not turn a user-stated possibility into an asserted plausible interpretation — preserve its uncertain status. Only split concepts when the user has presented materially distinct dimensions that each warrant independent investigation.
+
+8. Distinguish: what was said / what it may mean / why it may have been said.
+
+9. If input is too ambiguous or contains no useful operational anchors, say so and ask for
+   the single piece of context that would best distinguish plausible interpretations.
+
+10. Do not split concepts merely to increase the number of unknowns — only separate when the user has presented materially distinct dimensions worth independent investigation.
+
+## Preserve supplied relationships
+
+Reconstruction must preserve not only supplied semantic units but also relationships the user supplies between them. Preserved relationship types include: dependency, condition, comparison, alternative, constraint, sequence, reported causal claim, decision contingency. Splitting two supplied concepts must NOT erase the relationship between them. A supplied unresolved decision dependency with no dedicated schema field MUST be preserved explicitly in summary and as an importantUnknown whose wording states the dependency, keeping its uncertainty intact. Do not invent new schema fields for this purpose.
+
+## Explicit stop boundary for decomposition
+
+Once the supplied meaning of an observation, uncertainty, relationship or transition has been faithfully represented, STOP. Do not recursively decompose unless the supplied user material itself contains further distinct semantic structure. For each candidate decomposition child: identify the supplied meaning that supports its distinguishing content — if that content adds no new semantic proposition beyond what the user supplied, allow it; if it requires adding new semantic content, stop and do not create the child.
+
+## Normalisation and rate reasoning (apply whenever applicable)
+
+When the scenario mentions counts, totals, frequencies, or volumes alongside changes in
+scale, volume, exposure, time, population, or output:
+
+- ALWAYS consider whether a denominator or exposure metric is needed to normalise the count.
+- Distinguish between absolute count (total number observed) and rate (count per unit of exposure).
+- Two metrics rising at similar percentages does NOT imply that quality, performance, or safety
+  has worsened — production growth may outpace complaint growth, meaning the per-unit rate
+  could be stable or even improved.
+- Identify the possible denominator explicitly (e.g., "per unit produced", "per customer served",
+  "per hour of operation").
+- State clearly: "The absolute count changed by X%, but without knowing the denominator we cannot
+  determine whether the rate per unit has worsened, stayed stable, or improved."
+- Avoid treating correlation between two rising counts as evidence of a causal relationship.
+
+## Interpretation discipline
+
+- Do NOT generate plausible interpretations merely to fill a list. If the evidence does not
+  support useful, distinct interpretations, return an empty array [].
+- Only include an interpretation when there is specific evidence that makes it distinguishable
+  from alternatives and worth evaluating further.
+- Rank all reconstruction details by importance:
+  - critical: essential to resolving the situation; without it conclusions cannot be drawn
+  - important: materially affects understanding of the situation
+  - supporting: adds context but not critical
+  - incidental: minor detail, unlikely to affect conclusions
+
+## Next question discipline
+
+- Generate exactly ONE next question. Do NOT combine multiple questions.
+- The first and only question should target the single most useful missing comparison or data point.
+- Prefer narrow, specific questions over broad compound questions.
+- When counts have changed alongside scale/exposure, the highest-value question typically targets
+  the rate-per-unit or equivalent normalised metric.
+- Do NOT generate speculative interpretations merely to justify a question.
+
+## Confidence scale
+
+- low — weak evidence, speculation, or missing information
+- medium — reasonable inference from available evidence
+- high — strong evidence, direct observation, or confirmed fact
+
+## Importance scale (evidence records)
+
+- incidental — minor detail, unlikely to affect conclusions
+- supporting — adds context but not critical
+- important — materially affects understanding of the situation
+- critical — essential to resolving the situation; without it conclusions cannot be drawn
+
+## Expected information value (next question)
+
+- low — marginally useful even if answered
+- medium — meaningfully clarifies the situation
+- high — would significantly distinguish between plausible explanations or fill a gap in understanding
+
+## Next question selection criteria
+
+Prefer questions that:
+- clarify a major difference
+- establish a baseline
+- explain an important transition
+- test an unsupported claim
+- distinguish between plausible explanations
+- request measurable evidence
+- identify who or what is affected
+- establish timing
+
+Avoid questions that:
+- have already been answered
+- assume a cause
+- jump to a solution
+- ask about motive before the observable situation is understood
+- focus on incidental wording
+- are too broad to produce useful information
+- combine many unrelated questions
+
+## Output format — return this exact JSON structure
+
+Return a JSON object with exactly these four top-level keys (use **camelCase**):
+
+```json
+{
+  "inputClassification": {
+    "primaryType": "<one of: observed_problem, unexplained_change, contradiction, decision_request, causal_claim, reported_claim, fault_report, ambiguous_statement, question, desired_outcome, insufficient_context, other>",
+    "secondaryTypes": ["<optional additional types from the same list>"],
+    "reasoningModes": ["<one or more of: establish_baseline, identify_difference, reconstruct_transition, decompose_aggregate, validate_measurement, validate_claim, investigate_contradiction, clarify_meaning, decision_support, fault_investigation, identify_missing_information, test_possible_explanations, other>"],
+    "classificationReason": "<brief explanation of why you chose the primary type>",
+    "confidence": "<low | medium | high>"
+  },
+  "reconstruction": {
+    "summary": "<one-sentence overview of the situation>",
+    "actors": [{"id": "<any unique string>", "description": "...", "confidence": "<low|medium|high>"}],
+    "systemsOrObjects": [{"id": "<any unique string>", "description": "...", "confidence": "<low|medium|high>"}],
+    "expectedStates": [{"id": "...", "description": "...", "confidence": "<low|medium|high>"}],
+    "observedStates": [{"id": "...", "description": "...", "confidence": "<low|medium|high>"}],
+    "differences": [{"id": "...", "description": "...", "confidence": "<low|medium|high>"}],
+    "knownTransitions": [{"id": "...", "description": "...", "confidence": "<low|medium|high>", "entity": "...", "previousState": "...", "currentState": "...", "explanationStatus": "..."}],
+    "unexplainedTransitions": [{"id": "...", "description": "...", "confidence": "<low|medium|high>", "entity": "...", "previousState": "...", "currentState": "..."}],
+    "contradictions": [{"id": "...", "description": "...", "confidence": "<low|medium|high>"}],
+    "importantUnknowns": [{"id": "...", "description": "...", "confidence": "<low|medium|high>"}],
+    "plausibleInterpretations": [{"id": "...", "description": "...", "supportingEvidenceIds": ["<ids that support this interpretation>"], "assumptionsRequired": [], "confidence": "<low|medium|high>"}]
+  },
+  "evidence": [
+    {
+      "id": "<any unique string>",
+      "description": "...",
+      "evidenceType": "<direct_observation | reported_statement | interpretation | assumption | inferred_relationship>",
+      "source": "<optional — who/where this came from>",
+      "attribution": null,
+      "confidence": "<low | medium | high>",
+      "importance": "<incidental | supporting | important | critical>"
+    }
+  ],
+  "nextQuestion": {
+    "id": "<any unique string>",
+    "question": "<one precise question>",
+    "targets": ["<what this question targets — e.g. 'actor', 'system', 'expectedOutcome'>"],
+    "reason": "<why answering this is important>",
+    "expectedInformationValue": "<low | medium | high>",
+    "reasoningMode": "<optional reasoning mode from the list above>"
+  }
+}
+```
+
+CRITICAL RULES for JSON output:
+1. Use **exactly** the key names shown above (camelCase, no snake_case).
+2. The four top-level keys must be: `inputClassification`, `reconstruction`, `evidence`, `nextQuestion`.
+3. Do NOT invent new top-level keys (no `anchors`, `confidence` at top level, `meaningful_differences`, etc.).
+4. Keep `actors`, `systemsOrObjects`, `expectedStates`, `observedStates`, `differences`, `contradictions`, `importantUnknowns` as arrays even if empty: [].
+5. Keep `plausibleInterpretations` as an array (can be []), same for `knownTransitions` and `unexplainedTransitions`.
+6. Each object in arrays must have at least `id`, `description`, `confidence`.
+7. **evidenceType**: classify each evidence item clearly as either a direct observation, a reported statement, an interpretation, an assumption, or an inferred relationship. Do not treat raw counts as proof of causal relationships — they may be inferred relationships only when supported by explicit reasoning about denominators or rates.
+
+Scenario:
+{{SCENARIO}}
+
+Return ONLY the JSON object starting with { and ending with }. Do NOT include any text before the opening brace or after the closing brace. Do NOT wrap in markdown backticks.
@@ -25,8 +25,9 @@ describe("v0.3 prompt", () => {
    expect(PROMPT_VERSIONS).toContain("v0.3");
  });

-  it("DEFAULT_PROMPT_VERSION is v0.3 on this branch", () => {
-    expect(DEFAULT_PROMPT_VERSION).toBe("v0.3");
+  it("DEFAULT_PROMPT_VERSION was v0.3 on earlier branches (now v0.4)", () => {
+    // This test documents that the old default was v0.3; the new default is v0.4
+    expect(["v0.3", "v0.4"]).toContain(DEFAULT_PROMPT_VERSION);
  });

  it("v0.2 remains available in PROMPT_VERSIONS", () => {
@@ -675,7 +676,8 @@ describe("target scenario fixture validation", () => {

 describe("diagnostics prompt version", () => {
  it("DEFAULT_PROMPT_VERSION is exported correctly", () => {
-    expect(DEFAULT_PROMPT_VERSION).toBe("v0.3");
+    // The current default is v0.4 (was v0.3)
+    expect(["v0.3", "v0.4"]).toContain(DEFAULT_PROMPT_VERSION);
  });

  it("PROMPT_VERSIONS includes both v0.2 and v0.3", () => {
@@ -794,6 +796,99 @@ describe("Finding disposition toggle — state machine", () => {
  });
 });

+// ──────────────────────────────────────────────
+// v0.4 prompt loading and contract tests
+// ──────────────────────────────────────────────
+
+describe("v0.4 prompt", () => {
+  it("DEFAULT_PROMPT_VERSION is v0.4 on this branch", () => {
+    expect(DEFAULT_PROMPT_VERSION).toBe("v0.4");
+  });
+
+  it("v0.4 is in PROMPT_VERSIONS", () => {
+    expect(PROMPT_VERSIONS).toContain("v0.4");
+  });
+
+  it("v0.4 prompt file loads from disk", async () => {
+    const content = await fs.readFile(
+      join(PROMPTS_DIR, "reconstruct-v0.4.md"),
+      "utf-8",
+    );
+    expect(typeof content).toBe("string");
+    expect(content.length).toBeGreaterThan(500);
+  });
+
+  it("buildPrompt returns v0.4 prompt with scenario substituted", async () => {
+    const result = await buildPrompt("Test v0.4 scenario", "v0.4");
+    expect(result.version).toBe("v0.4");
+    expect(result.prompt).toContain("Test v0.4 scenario");
+  });
+
+  it("explicit v0.3 still resolves to unchanged v0.3 prompt", async () => {
+    const result = await buildPrompt("V0.3 test text", "v0.3");
+    expect(result.version).toBe("v0.3");
+    expect(result.prompt).toContain("V0.3 test text");
+  });
+
+  it("v0.4 contains the existing required JSON/output contract", async () => {
+    const result = await buildPrompt("test", "v0.4");
+    // All four top-level keys must be referenced in output format section
+    expect(result.prompt).toContain("inputClassification");
+    expect(result.prompt).toContain("reconstruction");
+    expect(result.prompt).toContain("evidence");
+    expect(result.prompt).toContain("nextQuestion");
+    expect(result.prompt).toContain("plausibleInterpretations");
+    expect(result.prompt).toContain("importantUnknowns");
+    // Must reference evidenceType values
+    expect(result.prompt).toContain("direct_observation");
+    expect(result.prompt).toContain("reported_statement");
+  });
+
+  it("v0.4 contains the provenance stop contract", async () => {
+    const result = await buildPrompt("test", "v0.4");
+    // Must contain explicit stop boundary language
+    expect(result.prompt.toLowerCase()).toMatch(/stop/i);
+    expect(result.prompt).toMatch(/explicit.*stop|do not recursively/i);
+    // Must restrict decomposition to supplied meaning only
+    expect(result.prompt).toMatch(/directly supported by meaning supplied/);
+  });
+
+  it("v0.4 contains the supplied-relationship preservation contract", async () => {
+    const result = await buildPrompt("test", "v0.4");
+    // Must contain relationship preservation section
+    expect(result.prompt.toLowerCase()).toMatch(/preserve.*supplied.*relationship|preserved.*relationship/);
+    // Must mention dependency preservation specifically
+    expect(result.prompt).toMatch(/dependenc/i);
+  });
+
+  it("v0.4 explicitly separates decomposition provenance from plausible interpretation", async () => {
+    const result = await buildPrompt("test", "v0.4");
+    // Must reference both concepts distinctly
+    expect(result.prompt).toMatch(/interpretation.*never.*smuggled|decomposition.*interpretation|plausible.*interpretations.*separate/i);
+  });
+
+  it("v0.4 still contains normalisation guidance", async () => {
+    const result = await buildPrompt("test", "v0.4");
+    expect(result.prompt).toMatch(/normali[sz]e/i);
+    expect(result.prompt.toLowerCase()).toContain("rate");
+    expect(result.prompt.toLowerCase()).toMatch(/denominator|exposure/);
+  });
+
+  it("v0.4 still contains exactly-one-next-question discipline", async () => {
+    const result = await buildPrompt("test", "v0.4");
+    expect(result.prompt).toMatch(/exactly.*one.*question|Do NOT combine/i);
+  });
+
+  it("default buildPrompt (no version arg) resolves to v0.4", async () => {
+    const result = await buildPrompt("Default version test");
+    expect(result.version).toBe(DEFAULT_PROMPT_VERSION);
+    expect(DEFAULT_PROMPT_VERSION).toBe("v0.4");
+    // Should not contain v0.3 schema-specific differences array (it does have it)
+    // but the prompt should be from the v0.4 file which has the stop boundary text
+    expect(result.prompt.toLowerCase()).toMatch(/stop|explicit.*stop/);
+  });
+});
+
 // ── Disposition prop chain verification ────────────────────────

 describe("Disposition prop chain — ScenarioForm → ReasoningWorkspace → FocusedQuestionBody", () => {