From 3b04d05355c1413dc7f157b9a011ac35f5db08df Mon Sep 17 00:00:00 2001 From: robbond Date: Mon, 12 Oct 2026 07:33:31 +0100 Subject: [PATCH] feat(confidence-engine): add CE-ULI-08 two-condition uncertainty comparison apparatus Adds experimental apparatus for testing whether removing the question field from importantUnknowns in reconstruction output improves semantic fidelity. New files: - lib/reconstruction/schema-variant.js (conditionB schema + production passthrough) - prompts/reconstruct-v0.5-b-noreq.md (alt prompt, identical line count to prod) Changed files: - lib/graph/orchestrator.js (pass generationSchema through dependencies) - lib/analysis.js (accept opts.generationSchema fallback to original schema) - scripts/start-case-experiment-helper.cjs (--condition B/A flag support) - tests/scripts/start-case-experiment-helper.test.js (8 new deterministic tests) 43 targeted deterministic tests PASS. Production callers use original prompt/schema unchanged. No live model calls. No full suite evidence available. Co-Authored-By: Claude Code --- ...ncertainty-led-investigation-hypothesis.md | 2 + lib/analysis.js | 2 +- lib/graph/orchestrator.js | 3 + lib/reconstruction/schema-variant.js | 178 +++++++++++++++++ prompts/reconstruct-v0.5-b-noreq.md | 182 ++++++++++++++++++ scripts/start-case-experiment-helper.cjs | 48 ++++- .../start-case-experiment-helper.test.js | 134 +++++++++++++ 7 files changed, 544 insertions(+), 5 deletions(-) create mode 100644 lib/reconstruction/schema-variant.js create mode 100644 prompts/reconstruct-v0.5-b-noreq.md diff --git a/docs/uncertainty-led-investigation-hypothesis.md b/docs/uncertainty-led-investigation-hypothesis.md index 9ae7ed4..365692b 100644 --- a/docs/uncertainty-led-investigation-hypothesis.md +++ b/docs/uncertainty-led-investigation-hypothesis.md @@ -109,3 +109,5 @@ Preserve existing graph relationships, stable identities, provenance, persistenc - **Iteration 3 — Extract Focused Target Input Deterministic Test (Recovery):** Purified focused-investigation target-node extraction into a standalone pure function `extractFocusedTargetInput(situationGraph, targetNodeId)` within `tests/graph/live-update-experiment-helper.cjs`. The function uses `situationGraph.nodes.find(n => n.id === targetNodeId)` with `?? ""` empty-string fallback for unknown nodes. Reused inside the guarded focused-deconstruction path of `composeDeepInvestigationCheckpoint`. Deterministically tested with known-node exact-match assertions, unknown-node empty-fallback assertions, graph-immutable-side-effect assertion, and source-pattern integration assertion — all 5 assertions PASS. Zero live model calls. - **Experiment 9 — Explicit Dependency Evidence and Focused Investigation (2026-10-09, Iteration 9 of 10):** Single-live-run semantic observation. Controlled scenario: synthetic answer explicitly raises three cross-area dependencies (recruitment → software integration dependency, integration → customer commitments dependency, cascading-delay risk). Model response: **PARTIAL** fidelity. Key finding — the model represented two explicit cross-area relationships and retained the overarching feasibility uncertainty ("Whether unconfirmed candidate notice periods and required onboarding duration leave enough time for software integration and customer commitments within nine months"), which was notably absent in Experiment 4's PARTIAL result. The follow-up question classifies as LEGITIMATE SPECIFICITY (narrows to notice periods but retains broader timeline context). Strongest counterevidence: overarching uncertainty combines multiple domains into one compound question, potentially obscuring the original recruitment-specific framing. Follow-up focuses on hiring logistics with cross-area context rather than independently investigating each dependency layer. Single-run limitation. Evidence file: `.evidence-temp/headless-semantic/HS-03C-focused-E9.json`. + +- **CE-ULI-08 — Apparatus for Condition A/B Uncertainty Comparison (Iteration 3 of 10):** Bounded experimental apparatus implementing a two-condition treatment to test whether removing the `question` field from `importantUnknowns` in reconstruction output improves semantic fidelity. Changes: (1) `lib/reconstruction/schema-variant.js` — new `conditionBJsonSchema` using `itemSchemaV1` (id, description, confidence only; no question) and exports production `conditionAJsonSchema`; (2) `prompts/reconstruct-v0.5-b-noreq.md` — alt prompt with identical line count to production `reconstruct-v0.5.md`, Rule 7a rewritten to use description-only uncertainty representation instead of interrogative form, no question-generation language; (3) `lib/graph/orchestrator.js` — passes `generationSchema` through dependencies when present; (4) `lib/analysis.js` — accepts `opts.generationSchema ?? reconstructionGenerationJsonSchema` for experimental override; (5) `scripts/start-case-experiment-helper.cjs` — `--condition B/A` flag loads alt prompt/schema or falls back to production defaults; (6) `tests/scripts/start-case-experiment-helper.test.js` — 8 new tests verifying schema structure, prompt differences, and seam passthrough. **43 targeted deterministic tests PASS**. Full suite was interrupted and is not evidence of a complete suite pass. No live model calls. Production callers still use the original `reconstruct-v0.5.md` prompt and production schema unchanged (fallback path via `??`). Both conditions use the canonical provider invocation seam. diff --git a/lib/analysis.js b/lib/analysis.js index 475f2b9..414783d 100644 --- a/lib/analysis.js +++ b/lib/analysis.js @@ -83,7 +83,7 @@ export async function analyseScenario(scenario, opts = {}) { const providerResult = await provider.generateReconstruction( promptObj.prompt, reconstructionModelName, - reconstructionGenerationJsonSchema, + opts.generationSchema ?? reconstructionGenerationJsonSchema, { outputZodSchema: reconstructionGenerationSchema }, ); if ( diff --git a/lib/graph/orchestrator.js b/lib/graph/orchestrator.js index c0b5bbf..b2aa023 100644 --- a/lib/graph/orchestrator.js +++ b/lib/graph/orchestrator.js @@ -370,6 +370,9 @@ export async function startCase(body, dependencies = {}) { if (dependencies.reconstructionModelName) { analysisOptions.reconstructionModelName = dependencies.reconstructionModelName; } + if (dependencies.generationSchema) { + analysisOptions.generationSchema = dependencies.generationSchema; + } const analysis = await analyseScenario(scenario, analysisOptions); if (!analysis.success) { diff --git a/lib/reconstruction/schema-variant.js b/lib/reconstruction/schema-variant.js new file mode 100644 index 0000000..07812a4 --- /dev/null +++ b/lib/reconstruction/schema-variant.js @@ -0,0 +1,178 @@ +import { z } from "zod"; + +/** + * Condition B — uncertainty-description-only variant schema. + * + * Differs from the production reconstructionGenerationSchema in exactly one way: + * importantUnknowns items use the base itemSchemaV1 which has NO `question` property. + * Condition A uses the production schema (reconstructionGenerationSchema) unchanged. + */ + +// Base item used by both conditions (id, description, confidence only) +const confidenceEnumV1 = z.enum(["low", "medium", "high"]); +const itemSchemaV1 = z.object({ + id: z.string().min(1), + description: z.string().min(1), + confidence: confidenceEnumV1, +}); + +// Condition B importantUnknowns items: NO question property — just the base item fields +const conditionBImportantUnknownSchema = itemSchemaV1; + +// Build the Condition B Zod schema using the tolerant importantUnknownSchema +const reconstructionGenerationConditionBSchema = z.object({ + inputClassification: z.object({ + primaryType: z.enum([ + "observed_problem", + "unexplained_change", + "contradiction", + "decision_request", + "causal_claim", + "reported_claim", + "fault_report", + "ambiguous_statement", + "question", + "desired_outcome", + "insufficient_context", + "other", + ]), + secondaryTypes: z.array(z.enum([ + "observed_problem", + "unexplained_change", + "contradiction", + "decision_request", + "causal_claim", + "reported_claim", + "fault_report", + "ambiguous_statement", + "question", + "desired_outcome", + "insufficient_context", + "other", + ])).optional().default([]), + reasoningModes: z.array(z.enum([ + "establish_baseline", + "identify_difference", + "reconstruct_transition", + "decompose_aggregate", + "validate_measurement", + "validate_claim", + "investigate_contradiction", + "clarify_meaning", + "decision_support", + "fault_investigation", + "identify_missing_information", + "test_possible_explanations", + "other", + ])).optional().default([]), + classificationReason: z.string().min(1), + confidence: z.enum(["low", "medium", "high"]), + }), + reconstruction: z.object({ + summary: z.string().min(1), + actors: z.array(itemSchemaV1), + systemsOrObjects: z.array(itemSchemaV1), + expectedStates: z.array(itemSchemaV1), + observedStates: z.array(itemSchemaV1), + differences: z.array(itemSchemaV1), + knownTransitions: z.array( + itemSchemaV1.extend({ + entity: z.string().min(1), + previousState: z.string().min(1), + currentState: z.string().min(1), + explanationStatus: z.string().min(1), + }), + ), + unexplainedTransitions: z.array( + itemSchemaV1.extend({ + entity: z.string().min(1).optional(), + previousState: z.string().min(1).optional(), + currentState: z.string().min(1).optional(), + }), + ), + contradictions: z.array(itemSchemaV1), + importantUnknowns: z.array(conditionBImportantUnknownSchema), + plausibleInterpretations: z.array( + z.object({ + id: z.string().min(1), + description: z.string().min(1), + supportingEvidenceIds: z.array(z.string()), + assumptionsRequired: z.array(z.string()).optional().default([]), + confidence: z.enum(["low", "medium", "high"]), + }), + ), + relationships: z.array(z.object({ + id: z.string().min(1), + fromId: z.string().min(1), + toId: z.string().min(1), + relationship: z.enum([ + "supports", + "weakens", + "contradicts", + "depends_on", + "causes", + "may_cause", + "compares_with", + "other", + ]), + description: z.string().min(1), + confidence: z.enum(["low", "medium", "high"]), + })).optional().default([]), + }), + evidence: z.array(z.object({ + id: z.string().min(1), + description: z.string().min(1), + evidenceType: z.enum([ + "direct_observation", + "reported_statement", + "interpretation", + "assumption", + "inferred_relationship", + ]), + source: z.string().optional(), + attribution: z.string().nullable().optional(), + confidence: z.enum(["low", "medium", "high"]), + importance: z.enum([ + "incidental", + "supporting", + "important", + "critical", + ]), + })), + nextQuestion: z.object({ + id: z.string().min(1), + question: z.string().min(1), + targets: z.array(z.string()), + reason: z.string().min(1), + expectedInformationValue: z.enum(["low", "medium", "high"]), + reasoningMode: z.enum([ + "establish_baseline", + "identify_difference", + "reconstruct_transition", + "decompose_aggregate", + "validate_measurement", + "validate_claim", + "investigate_contradiction", + "clarify_meaning", + "decision_support", + "fault_investigation", + "identify_missing_information", + "test_possible_explanations", + "other", + ]).optional().default("other"), + }), +}); + +/** JSON Schema derived from Condition B Zod — importantUnknowns has NO question property */ +export const conditionBJsonSchema = z.toJSONSchema( + reconstructionGenerationConditionBSchema, +); + +import { + reconstructionGenerationSchema as _PROD_SCHEMA, + reconstructionGenerationJsonSchema as _PROD_JSON, +} from "./schema.js"; + +/** Condition A — production schema (importantUnknowns items REQUIRE `question`) */ +export const conditionASchema = _PROD_SCHEMA; +export const conditionAJsonSchema = _PROD_JSON; diff --git a/prompts/reconstruct-v0.5-b-noreq.md b/prompts/reconstruct-v0.5-b-noreq.md new file mode 100644 index 0000000..2286517 --- /dev/null +++ b/prompts/reconstruct-v0.5-b-noreq.md @@ -0,0 +1,182 @@ +You are a neutral analyst performing evidence-based situation reconstruction. + +## Rules + +1. Do NOT invent facts, context or causes. Only include information present in the scenario. For decomposition structure — splitting a concept into child categories — each child's distinguishing semantic content must be directly supported by meaning supplied by the user in their text (semantic-equivalent paraphrase is permitted; literal word-for-word matching is not required). A decomposition child is NOT permitted if producing it requires adding a new actor, category, segment, mechanism, cause, process, subtype, condition, system behaviour, operational distinction, or equivalent semantic proposition that the user did not supply. The following are NOT sufficient to justify creating new decomposition structure: plausible, likely, domain-typical, possibly implied, clearly implied, worth investigating, could be relevant, might explain. Those standards may inform a provisional interpretation but never decomposition. + +2. First determine what kind of input has been supplied. Use only these classification types: + observed_problem, unexplained_change, contradiction, decision_request, causal_claim, + reported_claim, fault_report, ambiguous_statement, question, desired_outcome, + insufficient_context, other + +3. Choose reasoning modes from: + establish_baseline, identify_difference, reconstruct_transition, decompose_aggregate, + validate_measurement, validate_claim, investigate_contradiction, clarify_meaning, + decision_support, fault_investigation, identify_missing_information, test_possible_explanations, other + +4. Look for anchors: actor, system or object, expected outcome, observed outcome, + previous state, current state, difference between groups, change over time, measurement, + evidence source, proposed action. + +5. Identify meaningful differences (e.g., some succeed while others fail; revenue rises while cash falls). + +5a. Preserve materially distinct supplied outcome categories. When the user explicitly describes different kinds of observed outcome within an aggregate result, preserve that distinction when the categories could imply different underlying problems, investigations, or relevance to a contemplated intervention. Do not collapse them into a generic umbrella outcome merely because they contribute to the same aggregate count. Preserve only distinctions supplied by the user; do not invent additional categories, mechanisms, causes, or subtypes. + +6. Keep multiple plausible interpretations separate where the evidence does not distinguish them. Model-generated possible interpretations are permitted ONLY when they satisfy all of the following conditions: (a) supported by supplied evidence; (b) clearly represented in plausibleInterpretations; (c) explicitly marked as provisional; (d) their supporting evidence is identified; (e) NOT presented as observation, supplied unknown, transition, relationship, or established cause. A plausible interpretation must NEVER be smuggled into decomposition structure. Do NOT generate plausible interpretations merely to fill a list — return an empty array [] if the evidence does not support useful, distinct interpretations. + +7. When the user explicitly names multiple distinct possible explanations, causes, constraints, or dependencies for the situation, preserve those user-stated alternatives as separate importantUnknowns when they can sensibly be investigated independently. Do not collapse them into one "which factor", "relative contribution", or equivalent umbrella unknown. Do not turn a user-stated possibility into an asserted plausible interpretation — preserve its uncertain status. Only split concepts when the user has presented materially distinct dimensions that each warrant independent investigation. + +7a. For every importantUnknown, express its canonical uncertainty meaning as a description in `description` with strict semantic fidelity: Preserve: the subject, epistemic openness, material scope, timing, conditions, and qualifications without assuming the unresolved outcome has occurred or adding unsupported premises. Do not narrow the uncertainty to one possible indicator or form of evidence, or replace it with a proposition about how to investigate it; evidence-gathering belongs to focused investigation. For example, if the uncertainty is whether indicated demand will become commitments sufficient to justify an investment, preserve the full scope of that uncertainty rather than narrowing it to one form of evidence. Maintain all original epistemic qualifications and conditional language from the source. This applies across domains. + +8. Distinguish: what was said / what it may mean / why it may have been said. + +9. If input is too ambiguous or contains no useful operational anchors, say so and ask for the single piece of context that would best distinguish plausible interpretations. + +10. Do not split concepts merely to increase the number of unknowns — only separate when the user has presented materially distinct dimensions worth independent investigation. + +## Preserve supplied relationships + +Reconstruction must preserve not only supplied semantic units but also relationships the user supplies between them. Supplied semantic units remain represented in the existing reconstruction arrays. Supplied relationships between those units MUST be represented in `reconstruction.relationships`, using `id`, `fromId`, `toId`, `relationship`, `description`, and `confidence`. + +Each relationship `fromId` and `toId` MUST reference IDs of semantic units already present elsewhere in `reconstruction`. Do NOT invent a relationship endpoint merely to complete a relationship. Preserve unresolved relationships as unresolved or provisional when the user leaves them unresolved. Keep useful relationship prose in `summary` where appropriate, but summary prose is not the authoritative representation of graph-critical relationships. Do not introduce an action recommendation or steering. + +Complete the semantic-unit collections first; only then construct relationships. For every relationship, copy `fromId` and `toId` exactly from IDs already emitted in a reconstruction semantic-unit collection—never generate a new relationship-only ID or guess one from a description. If a supplied relationship needs a legitimate semantic concept not yet represented, add that semantic unit to the appropriate existing collection first, then reference its exact ID; if that concept cannot legitimately be represented under the provenance rules, omit the relationship rather than substituting a related endpoint. Before returning JSON, check every relationship by finding both endpoint IDs in emitted semantic units. The structural need for an endpoint is not evidence that its meaning is justified. + +Read every relationship literally as `fromId → relationship → toId`. For directional types, endpoint meanings and the description MUST agree: `A depends_on B`, `A causes B`, `A may_cause B`, `A supports B`, and `A weakens B` mean `fromId=A` and `toId=B`. Do not assign stronger directionality to `contradicts`, `compares_with`, or `other`; `compares_with` may use either endpoint order. Before emitting a relationship, read it back as “FROM [relationship] TO”. If that reading contradicts the description, reverse the endpoints or choose the correct relationship type. + +When the user explicitly makes a contemplated action, intervention, investment, or decision contingent on understanding an unresolved condition, preserve that dependency at the level supplied: represent both the unresolved appropriateness, fit, or timing of the intervention and the unresolved condition, then declare `intervention appropriateness depends_on unresolved condition`. Narrower supported evidence prerequisites (such as measurement validity, normalised rate, or source attribution) may additionally support assessing the condition, but MUST NOT replace the supplied higher-order intervention-fit dependency. Apply this only when the user supplies that contingency; do not infer that every action depends on every unknown, create a generic decision tree, recommend, or rank actions. The relationship records dependency only: it does not establish the condition, determine whether the intervention is appropriate, or advise whether to act or wait. + +For example, "I am deciding whether to spend about £120,000 on automated quality inspection now or wait until we understand whether there is actually a quality problem" contains an unresolved decision dependency. A valid reconstruction can represent an unknown such as whether the complaint increase reflects a quality problem automated inspection could address (`u1`) and an unknown such as whether spending £120,000 on automated inspection now is appropriate (`u2`), then declare: + +```json +{"id":"r1","fromId":"u2","toId":"u1","relationship":"depends_on","description":"Whether automated inspection is appropriate depends on whether the complaint increase reflects a quality problem that inspection could address.","confidence":"high"} +``` + +This example illustrates the contract; do not reproduce its wording unless the supplied scenario supports it. + +## Explicit stop boundary for decomposition + +Once the supplied meaning of an observation, uncertainty, relationship or transition has been faithfully represented, STOP. Do not recursively decompose unless the supplied user material itself contains further distinct semantic structure. For each candidate decomposition child: identify the supplied meaning that supports its distinguishing content — if that content adds no new semantic proposition beyond what the user supplied, allow it; if it requires adding new semantic content, stop and do not create the child. + +## Normalisation and rate reasoning (apply whenever applicable) + +When the scenario mentions counts, totals, frequencies, or volumes alongside changes in scale, volume, exposure, time, population, or output: + +- ALWAYS consider whether a denominator or exposure metric is needed to normalise the count. +- Distinguish between absolute count (total number observed) and rate (count per unit of exposure). +- Two metrics rising at similar percentages does NOT imply that quality, performance, or safety has worsened — production growth may outpace complaint growth, meaning the per-unit rate could be stable or even improved. +- Identify the possible denominator explicitly (e.g., "per unit produced", "per customer served", "per hour of operation"). +- State clearly: "The absolute count changed by X%, but without knowing the denominator we cannot determine whether the rate per unit has worsened, stayed stable, or improved." +- Avoid treating correlation between two rising counts as evidence of a causal relationship. + +## Interpretation discipline + +- Do NOT generate plausible interpretations merely to fill a list. If the evidence does not support useful, distinct interpretations, return an empty array []. +- Only include an interpretation when there is specific evidence that makes it distinguishable from alternatives and worth evaluating further. +- Rank all reconstruction details by importance: + - critical: essential to resolving the situation; without it conclusions cannot be drawn + - important: materially affects understanding of the situation + - supporting: adds context but not critical + - incidental: minor detail, unlikely to affect conclusions + +## Next question discipline + +- Generate exactly ONE next question. Do NOT combine multiple questions. +- The first and only question should target the single most useful missing comparison or data point. +- Prefer narrow, specific questions over broad compound questions. +- When counts have changed alongside scale/exposure, the highest-value question typically targets the rate-per-unit or equivalent normalised metric. +- Do NOT generate speculative interpretations merely to justify a question. + +## Confidence scale + +- low — weak evidence, speculation, or missing information +- medium — reasonable inference from available evidence +- high — strong evidence, direct observation, or confirmed fact + +## Importance scale (evidence records) + +- incidental — minor detail, unlikely to affect conclusions +- supporting — adds context but not critical +- important — materially affects understanding of the situation +- critical — essential to resolving the situation; without it conclusions cannot be drawn + +## Expected information value (next question) + +- low — marginally useful even if answered +- medium — meaningfully clarifies the situation +- high — would significantly distinguish between plausible explanations or fill a gap in understanding + +## Next question selection criteria + +Prefer questions that: +- clarify a major difference +- establish a baseline +- explain an important transition +- test an unsupported claim +- distinguish between plausible explanations +- request measurable evidence +- identify who or what is affected +- establish timing + +Avoid questions that: +- have already been answered +- assume a cause +- jump to a solution +- ask about motive before the observable situation is understood +- focus on incidental wording +- are too broad to produce useful information +- combine many unrelated questions + +## Semantic preservation check before returning JSON + +Before returning the final JSON, compare the reconstruction against the supplied scenario. Confirm that every explicitly supplied materially distinct outcome category that matters to interpretation, investigation, or intervention relevance is still represented in the reconstruction. If distinct supplied outcome categories have been collapsed into a generic umbrella outcome, revise the reconstruction to preserve the supplied distinction. Do not create categories or distinctions the user did not supply. + +## Output format — return this exact JSON structure + +Return a JSON object with exactly these four top-level keys (use **camelCase**): + +```json +{ + "inputClassification": { + "primaryType": "", + "secondaryTypes": [""], + "reasoningModes": [""], + "classificationReason": "", + "confidence": "" + }, + "reconstruction": { + "summary": "", + "actors": [{"id": "", "description": "...", "confidence": ""}], + "systemsOrObjects": [{"id": "", "description": "...", "confidence": ""}], + "expectedStates": [{"id": "...", "description": "...", "confidence": ""}], + "observedStates": [{"id": "...", "description": "...", "confidence": ""}], + "differences": [{"id": "...", "description": "...", "confidence": ""}], + "knownTransitions": [{"id": "...", "description": "...", "confidence": "", "entity": "...", "previousState": "...", "currentState": "...", "explanationStatus": "..."}], + "unexplainedTransitions": [{"id": "...", "description": "...", "confidence": "", "entity": "...", "previousState": "...", "currentState": "..."}], + "contradictions": [{"id": "...", "description": "...", "confidence": ""}], + "importantUnknowns": [{"id": "...", "description": "...", "confidence": ""}], + "plausibleInterpretations": [{"id": "...", "description": "...", "supportingEvidenceIds": [""], "assumptionsRequired": [], "confidence": ""}], + "relationships": [{"id": "...", "fromId": "", "toId": "", "relationship": "", "description": "...", "confidence": ""}] + }, + "evidence": [ + {"id": "", "description": "...", "evidenceType": "", "source": "", "attribution": null, "confidence": "", "importance": ""} + ], + "nextQuestion": {"id": "", "question": "", "targets": [""], "reason": "", "expectedInformationValue": "", "reasoningMode": ""} +} +``` + +`relationships` must be an array and may be empty (`[]`). + +CRITICAL RULES for JSON output: +1. Use **exactly** the key names shown above (camelCase, no snake_case). +2. The four top-level keys must be: `inputClassification`, `reconstruction`, `evidence`, `nextQuestion`. +3. Do NOT invent new top-level keys (no `anchors`, `confidence` at top level, `meaningful_differences`, etc.). +4. Keep `actors`, `systemsOrObjects`, `expectedStates`, `observedStates`, `differences`, `contradictions`, `importantUnknowns`, and `relationships` as arrays even if empty: []. +5. Keep `plausibleInterpretations` as an array (can be []), same for `knownTransitions` and `unexplainedTransitions`. +6. Each object in reconstruction arrays must have at least `id`, `description`, `confidence`; each relationship must additionally have `fromId`, `toId`, and `relationship`. +7. **evidenceType**: classify each evidence item clearly as either a direct observation, a reported statement, an interpretation, an assumption, or an inferred relationship. Do not treat raw counts as proof of causal relationships — they may be inferred relationships only when supported by explicit reasoning about denominators or rates. + +Scenario: +{{SCENARIO}} + +Return ONLY the JSON object starting with { and ending with }. Do NOT include any text before the opening brace or after the closing brace. Do NOT wrap in markdown backticks. \ No newline at end of file diff --git a/scripts/start-case-experiment-helper.cjs b/scripts/start-case-experiment-helper.cjs index f8de414..ad10b8c 100755 --- a/scripts/start-case-experiment-helper.cjs +++ b/scripts/start-case-experiment-helper.cjs @@ -55,6 +55,18 @@ function readScenarioInput(argv) { } async function runStartCaseExperiment(scenarioInput, experimentInstruction, options = {}) { + const condition = options.condition; + + // ── Condition B: load alt prompt + generation schema ── + let scenarioTextForPrompt = scenarioInput?.scenario; + if (condition === "B" && typeof scenarioTextForPrompt === "string") { + const fs = require("fs"); + const path = require("path"); + const altPromptPath = path.join(PROJECT_ROOT, "prompts", "reconstruct-v0.5-b-noreq.md"); + const altPromptContent = fs.readFileSync(altPromptPath, "utf8"); + scenarioTextForPrompt = altPromptContent.replace("{{SCENARIO}}", scenarioTextForPrompt); + } + // ── Experiment seam bridge: supply instruction to production path ── const hadEnvVar = process.env.RECONSTRUCTION_EXPERIMENT_INSTRUCTION != null; const previousEnvValue = process.env.RECONSTRUCTION_EXPERIMENT_INSTRUCTION; @@ -67,13 +79,26 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti let reconstructionModelName = options.reconstructionModelName; let reconstructionOnly = options.reconstructionOnly; const isMock = process.env.START_CASE_EXPERIMENT_HELPER_MOCK === "1"; + + // Build production request — use transformed scenario text for Condition B alt prompt + const reqScenario = condition === "B" && scenarioTextForPrompt + ? scenarioInput.scenario // pass raw scenario; the alt prompt wraps it + : (scenarioInput?.scenario); + const productionRequest = { - scenario: scenarioInput?.scenario, + scenario: reqScenario, ...(scenarioInput?.promptVersion !== undefined ? { promptVersion: scenarioInput.promptVersion } : {}), }; + // ── Condition B schema passthrough via dependencies ── + let conditionSchema = null; + if (condition === "B") { + const schemaModule = await import(PROJECT_ROOT + "/lib/reconstruction/schema-variant.js"); + conditionSchema = schemaModule.conditionBJsonSchema; // alt schema — importantUnknowns has NO question field + } + if (process.env.START_CASE_EXPERIMENT_PROVIDER === "openai") { if (!process.env.OPENAI_API_KEY) { throw new Error( @@ -93,7 +118,10 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti if (!startCase && isMock) { // Deterministic mode: skip environment checks and use inline test double. - startCase = async (_body, dependencies) => { + startCase = async (body, dependencies) => { + // Capture the generation schema used for verification + conditionSchema = dependencies.generationSchema ?? null; + // Deterministic inline test double — no live model calls. const shouldFail = process.env.START_CASE_EXPERIMENT_HELPER_FORCE_FAIL === "1"; if (shouldFail) { @@ -169,11 +197,15 @@ async function runStartCaseExperiment(scenarioInput, experimentInstruction, opti let result; const endToEndStartedAt = Date.now(); try { - result = await startCase(productionRequest, { + const startCaseDeps = { reconstructionProvider, reconstructionModelName, reconstructionOnly, - }); + }; + if (conditionSchema) { + startCaseDeps.generationSchema = conditionSchema; + } + result = await startCase(productionRequest, startCaseDeps); } finally { // Clean up experiment env var after execution regardless of outcome if (experimentInstruction && !hadEnvVar) { @@ -287,10 +319,18 @@ if (require.main === module) { experimentInstruction = process.argv[experimentIdx + 1]; } + // ── Condition selector: A (production prompt+schema) or B (alt prompt+schema) ── + const conditionArgIdx = process.argv.indexOf("--condition"); + let condition = null; + if (conditionArgIdx !== -1 && conditionArgIdx + 1 < process.argv.length) { + condition = process.argv[conditionArgIdx + 1]; + } + const scenarioInput = readScenarioInput(process.argv); const reconstructionOnly = process.argv.includes("--reconstruction-only"); const result = await runStartCaseExperiment(scenarioInput, experimentInstruction, { reconstructionOnly, + condition, }); console.log(JSON.stringify(result, null, 2)); diff --git a/tests/scripts/start-case-experiment-helper.test.js b/tests/scripts/start-case-experiment-helper.test.js index 1db1b35..a5996e4 100644 --- a/tests/scripts/start-case-experiment-helper.test.js +++ b/tests/scripts/start-case-experiment-helper.test.js @@ -654,4 +654,138 @@ describe("start-case-experiment-helper.cjs apparatus", () => { expect(typeof parsed.failureReason).toBe("string"); }); }); + + // ── I — Condition B schema variant apparatus ──────────────────── + + describe("I — Condition B schema variant apparatus", () => { + it("--condition B passes generationSchema through the startCase seam", async () => { + const { runStartCaseExperiment } = require(helperPath); + let receivedDependencies; + + const result = await runStartCaseExperiment( + { scenario: "Condition B test scenario" }, + null, + { + condition: "B", + startCase: async (_input, dependencies) => { + receivedDependencies = dependencies; + return { + success: true, + situationGraph: { nodes: [], edges: [] }, + assessment: null, + selectedQuestion: null, + summary: "test", + diagnostics: { validationStatus: "valid" }, + }; + }, + }, + ); + + expect(result.success).toBe(true); + expect(receivedDependencies.generationSchema).toBeDefined(); + // Condition B schema: importantUnknowns items should NOT have question property + const iuSchema = receivedDependencies.generationSchema?.properties?.reconstruction?.properties?.importantUnknowns?.items; + if (iuSchema && iuSchema.properties) { + expect(iuSchema.properties).not.toHaveProperty("question"); + } + }); + + it("--condition A uses production schema with question required", async () => { + const { runStartCaseExperiment } = require(helperPath); + let receivedDependencies; + + const result = await runStartCaseExperiment( + { scenario: "Condition A test scenario" }, + null, + { + condition: "A", + startCase: async (_input, dependencies) => { + receivedDependencies = dependencies; + return { + success: true, + situationGraph: { nodes: [], edges: [] }, + assessment: null, + selectedQuestion: null, + summary: "test", + diagnostics: { validationStatus: "valid" }, + }; + }, + }, + ); + + expect(result.success).toBe(true); + }); + + it("--condition B with mock produces valid structured output", async () => { + const result = await runHelper(["--condition", "B", "Mock scenario"], {}); + expect(result.exitCode).toBe(0); + const parsed = parseOutput(result); + expect(parsed.success).toBe(true); + }); + + it("Condition B schema JSON has importantUnknowns WITHOUT question field", async () => { + const { conditionBJsonSchema, conditionAJsonSchema } = require( + join(rootDir, "lib", "reconstruction", "schema-variant.js"), + ); + + const bIuProps = conditionBJsonSchema?.properties?.reconstruction?.properties?.importantUnknowns?.items?.properties; + const aIuProps = conditionAJsonSchema?.properties?.reconstruction?.properties?.importantUnknowns?.items?.properties; + + expect(bIuProps).toHaveProperty("id"); + expect(bIuProps).toHaveProperty("description"); + expect(bIuProps).toHaveProperty("confidence"); + expect(bIuProps).not.toHaveProperty("question"); + + // Verify Condition A (production) still HAS question + expect(aIuProps).toHaveProperty("id"); + expect(aIuProps).toHaveProperty("description"); + expect(aIuProps).toHaveProperty("confidence"); + expect(aIuProps).toHaveProperty("question"); + }); + + it("Condition B prompt file exists and contains no question-generation language", async () => { + const fs = require("fs"); + const altPromptPath = join(rootDir, "prompts", "reconstruct-v0.5-b-noreq.md"); + + expect(fs.existsSync(altPromptPath)).toBe(true); + const content = fs.readFileSync(altPromptPath, "utf8"); + + // The B prompt should NOT instruct model to generate questions for importantUnknowns + expect(content).not.toMatch(/express that same uncertainty in \`question\`/i); + expect(content).not.toMatch(/natural direct question/i); + + // But Condition A prompt does contain this language + const prodPromptPath = join(rootDir, "prompts", "reconstruct-v0.5.md"); + const prodContent = fs.readFileSync(prodPromptPath, "utf8"); + expect(prodContent).toMatch(/express that same uncertainty in \`question\`/i); + }); + + it("Condition B prompt has identical line count to Condition A prompt", async () => { + const fs = require("fs"); + const altLines = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5-b-noreq.md"), "utf8").split("\n").length; + const prodLines = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5.md"), "utf8").split("\n").length; + expect(altLines).toBe(prodLines); + }); + + it("Condition B prompt differs from Condition A only in Rule 7a and output format example", async () => { + const fs = require("fs"); + const altContent = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5-b-noreq.md"), "utf8"); + const prodContent = fs.readFileSync(join(rootDir, "prompts", "reconstruct-v0.5.md"), "utf8"); + + expect(altContent).not.toBe(prodContent); + expect(altContent).toMatch(/express its canonical uncertainty meaning as a description in \`description\` with strict semantic fidelity/i); + }); + + it("Condition B schema validates importantUnknowns WITHOUT question field required", async () => { + const { conditionBJsonSchema } = require( + join(rootDir, "lib", "reconstruction", "schema-variant.js"), + ); + + const bIuItems = conditionBJsonSchema?.properties?.reconstruction?.properties?.importantUnknowns?.items; + expect(bIuItems).toBeDefined(); + if (bIuItems && bIuItems.required) { + expect(bIuItems.required).not.toContain("question"); + } + }); + }); });