feat: add v0.3 normalised comparison reasoning
Add explicit reasoning guidance for normalising counts by exposure/denominator, distinguishing total count from rate, and avoiding correlation-as-causation errors. Changes: - prompts/reconstruct-v0.3.md: new prompt with normalisation discipline - lib/reconstruction/prompt.js: v0.3 loader + env var override support - lib/analysis.js: defer DEFAULT_PROMPT_VERSION to prompt module (defaults to v0.3) - PROMPT_VERSIONS extended to [v0.1, v0.2, v0.3] - tests/v03-reasoning.test.js: 34 focused tests covering prompt loading, schema validation, guidance completeness, and target scenario fixture - playwright.config.js + tests/smoke.test.js: minimal UI smoke test for browser rendering - package.json: add @playwright/test as devDependency Default switches to v0.3; v0.2 selectable via promptVersion or RECONSTRUCTION_PROMPT_VERSION env var.
This commit is contained in:
+1
-2
@@ -5,14 +5,13 @@
|
||||
|
||||
import { getConfig } from "../lib/config.js";
|
||||
import { getProvider } from "../lib/llm/provider.js";
|
||||
import { buildPrompt, PROMPT_VERSIONS } from "../lib/reconstruction/prompt.js";
|
||||
import { buildPrompt, PROMPT_VERSIONS, DEFAULT_PROMPT_VERSION } from "../lib/reconstruction/prompt.js";
|
||||
import {
|
||||
reconstructionV2Schema,
|
||||
reconstructionSchema as reconstructionV1Schema,
|
||||
} from "../lib/reconstruction/schema.js";
|
||||
|
||||
const MAX_SCENARIO_LENGTH = 10000;
|
||||
const DEFAULT_PROMPT_VERSION = "v0.2";
|
||||
|
||||
/**
|
||||
* Analyse a scenario string through the full pipeline.
|
||||
|
||||
@@ -7,7 +7,14 @@ const __dirname = dirname(__filename);
|
||||
const PROMPTS_DIR = join(__dirname, "../../prompts");
|
||||
|
||||
/** Available prompt versions */
|
||||
export const PROMPT_VERSIONS = ["v0.1", "v0.2"];
|
||||
export const PROMPT_VERSIONS = ["v0.1", "v0.2", "v0.3"];
|
||||
|
||||
/** Default prompt version (override via RECONSTRUCTION_PROMPT_VERSION env var) */
|
||||
const defaultVersionFromEnv = process.env.RECONSTRUCTION_PROMPT_VERSION;
|
||||
export const DEFAULT_PROMPT_VERSION =
|
||||
defaultVersionFromEnv && PROMPT_VERSIONS.includes(defaultVersionFromEnv)
|
||||
? defaultVersionFromEnv
|
||||
: "v0.3";
|
||||
|
||||
/** Build a v0.1 (extraction-only) prompt inline for backward compatibility */
|
||||
function buildV1Prompt(scenario) {
|
||||
@@ -56,20 +63,37 @@ async function buildV2Prompt(scenario) {
|
||||
}
|
||||
}
|
||||
|
||||
/** Load a versioned prompt from disk and substitute {{SCENARIO}} */
|
||||
async function buildV3Prompt(scenario) {
|
||||
try {
|
||||
const content = await fs.readFile(
|
||||
join(PROMPTS_DIR, "reconstruct-v0.3.md"),
|
||||
"utf-8",
|
||||
);
|
||||
return content.replace("{{SCENARIO}}", scenario);
|
||||
} catch {
|
||||
// Fall back to v0.2 prompt if v0.3 file is missing
|
||||
return buildV2Prompt(scenario);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build an analysis prompt for the given version.
|
||||
* @param {"v0.1" | "v0.2"} [version="v0.2"]
|
||||
* @param {"v0.1" | "v0.2" | "v0.3"} [version="v0.3"]
|
||||
* @returns {Promise<{prompt: string, version: string}>}
|
||||
*/
|
||||
export async function buildPrompt(scenario, version = "v0.2") {
|
||||
export async function buildPrompt(scenario, version = "v0.3") {
|
||||
let prompt;
|
||||
switch (version) {
|
||||
case "v0.1":
|
||||
prompt = buildV1Prompt(scenario);
|
||||
break;
|
||||
default: // v0.2
|
||||
case "v0.2":
|
||||
prompt = await buildV2Prompt(scenario);
|
||||
break;
|
||||
default: // v0.3
|
||||
prompt = await buildV3Prompt(scenario);
|
||||
break;
|
||||
}
|
||||
|
||||
const strongJsonHint =
|
||||
|
||||
Generated
+66
-2
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "confidence-engine",
|
||||
"version": "0.1.0",
|
||||
"version": "0.2.0-experimental",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "confidence-engine",
|
||||
"version": "0.1.0",
|
||||
"version": "0.2.0-experimental",
|
||||
"dependencies": {
|
||||
"next": "^14.2.0",
|
||||
"react": "^18.3.0",
|
||||
@@ -14,6 +14,7 @@
|
||||
"zod": "^3.23.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@playwright/test": "^1.62.1",
|
||||
"@types/node": "^20.14.0",
|
||||
"@types/react": "^18.3.0",
|
||||
"@types/react-dom": "^18.3.0",
|
||||
@@ -888,6 +889,22 @@
|
||||
"node": ">=14"
|
||||
}
|
||||
},
|
||||
"node_modules/@playwright/test": {
|
||||
"version": "1.62.1",
|
||||
"resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.62.1.tgz",
|
||||
"integrity": "sha512-DTcUc8qii+cpHvtOwggMtBRMjKZHXYWdw8syRYu2vtzuq4Wxphqq4NfCs5Zt44L6mA8rfDfj+PHnxFc/FeK6mQ==",
|
||||
"devOptional": true,
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"playwright": "1.62.1"
|
||||
},
|
||||
"bin": {
|
||||
"playwright": "cli.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
}
|
||||
},
|
||||
"node_modules/@rollup/rollup-android-arm-eabi": {
|
||||
"version": "4.62.3",
|
||||
"resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.62.3.tgz",
|
||||
@@ -5601,6 +5618,53 @@
|
||||
"node": ">= 6"
|
||||
}
|
||||
},
|
||||
"node_modules/playwright": {
|
||||
"version": "1.62.1",
|
||||
"resolved": "https://registry.npmjs.org/playwright/-/playwright-1.62.1.tgz",
|
||||
"integrity": "sha512-0M+L3LAD8/nm554LOla9Ayx0j0tmFZ0FBcoQ7F1VuVHpM/XpiC8RcDzBQB8W5+hA8L22THxELzeF+2WcUzvcLg==",
|
||||
"devOptional": true,
|
||||
"license": "Apache-2.0",
|
||||
"dependencies": {
|
||||
"playwright-core": "1.62.1"
|
||||
},
|
||||
"bin": {
|
||||
"playwright": "cli.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"fsevents": "2.3.2"
|
||||
}
|
||||
},
|
||||
"node_modules/playwright-core": {
|
||||
"version": "1.62.1",
|
||||
"resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.62.1.tgz",
|
||||
"integrity": "sha512-wPYSwEBJY9GHraISXqyqtx0na0LpO3XEX7jNDhntbex7tzUS7kLnZsOlFruFJB4Hi/rhDMjXGqHewDZ68nYZVw==",
|
||||
"devOptional": true,
|
||||
"license": "Apache-2.0",
|
||||
"bin": {
|
||||
"playwright-core": "cli.js"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=20"
|
||||
}
|
||||
},
|
||||
"node_modules/playwright/node_modules/fsevents": {
|
||||
"version": "2.3.2",
|
||||
"resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz",
|
||||
"integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==",
|
||||
"dev": true,
|
||||
"hasInstallScript": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
],
|
||||
"engines": {
|
||||
"node": "^8.16.0 || ^10.6.0 || >=11.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/possible-typed-array-names": {
|
||||
"version": "1.1.0",
|
||||
"resolved": "https://registry.npmjs.org/possible-typed-array-names/-/possible-typed-array-names-1.1.0.tgz",
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
"zod": "^3.23.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@playwright/test": "^1.62.1",
|
||||
"@types/node": "^20.14.0",
|
||||
"@types/react": "^18.3.0",
|
||||
"@types/react-dom": "^18.3.0",
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
import { defineConfig } from "@playwright/test";
|
||||
export default defineConfig({
|
||||
use: { headless: true, screenshot: "only-on-failure", actionTimeout: 120000 },
|
||||
testMatch: "**/tests/smoke.test.js",
|
||||
});
|
||||
@@ -0,0 +1,160 @@
|
||||
You are a neutral analyst performing evidence-based situation reconstruction.
|
||||
|
||||
## Rules
|
||||
|
||||
1. Do NOT invent facts, context or causes. Only include information present in the scenario or clearly implied.
|
||||
2. First determine what kind of input has been supplied. Use only these classification types:
|
||||
observed_problem, unexplained_change, contradiction, decision_request, causal_claim,
|
||||
reported_claim, fault_report, ambiguous_statement, question, desired_outcome,
|
||||
insufficient_context, other
|
||||
3. Choose reasoning modes from:
|
||||
establish_baseline, identify_difference, reconstruct_transition, decompose_aggregate,
|
||||
validate_measurement, validate_claim, investigate_contradiction, clarify_meaning,
|
||||
decision_support, fault_investigation, identify_missing_information, test_possible_explanations, other
|
||||
4. Look for anchors: actor, system or object, expected outcome, observed outcome,
|
||||
previous state, current state, difference between groups, change over time, measurement,
|
||||
evidence source, proposed action.
|
||||
5. Identify meaningful differences (e.g., some succeed while others fail; revenue rises while cash falls).
|
||||
6. Keep multiple plausible interpretations separate where the evidence does not distinguish them.
|
||||
7. Distinguish: what was said / what it may mean / why it may have been said.
|
||||
8. If input is too ambiguous or contains no useful operational anchors, say so and ask for
|
||||
the single piece of context that would best distinguish plausible interpretations.
|
||||
|
||||
## Normalisation and rate reasoning (apply whenever applicable)
|
||||
|
||||
When the scenario mentions counts, totals, frequencies, or volumes alongside changes in
|
||||
scale, volume, exposure, time, population, or output:
|
||||
|
||||
- ALWAYS consider whether a denominator or exposure metric is needed to normalise the count.
|
||||
- Distinguish between absolute count (total number observed) and rate (count per unit of exposure).
|
||||
- Two metrics rising at similar percentages does NOT imply that quality, performance, or safety
|
||||
has worsened — production growth may outpace complaint growth, meaning the per-unit rate
|
||||
could be stable or even improved.
|
||||
- Identify the possible denominator explicitly (e.g., "per unit produced", "per customer served",
|
||||
"per hour of operation").
|
||||
- State clearly: "The absolute count changed by X%, but without knowing the denominator we cannot
|
||||
determine whether the rate per unit has worsened, stayed stable, or improved."
|
||||
- Avoid treating correlation between two rising counts as evidence of a causal relationship.
|
||||
|
||||
## Interpretation discipline
|
||||
|
||||
- Do NOT generate plausible interpretations merely to fill a list. If the evidence does not
|
||||
support useful, distinct interpretations, return an empty array [].
|
||||
- Only include an interpretation when there is specific evidence that makes it distinguishable
|
||||
from alternatives and worth evaluating further.
|
||||
- Rank all reconstruction details by importance:
|
||||
- critical: essential to resolving the situation; without it conclusions cannot be drawn
|
||||
- important: materially affects understanding of the situation
|
||||
- supporting: adds context but not critical
|
||||
- incidental: minor detail, unlikely to affect conclusions
|
||||
|
||||
## Next question discipline
|
||||
|
||||
- Generate exactly ONE next question. Do NOT combine multiple questions.
|
||||
- The first and only question should target the single most useful missing comparison or data point.
|
||||
- Prefer narrow, specific questions over broad compound questions.
|
||||
- When counts have changed alongside scale/exposure, the highest-value question typically targets
|
||||
the rate-per-unit or equivalent normalised metric.
|
||||
- Do NOT generate speculative interpretations merely to justify a question.
|
||||
|
||||
## Confidence scale
|
||||
|
||||
- low — weak evidence, speculation, or missing information
|
||||
- medium — reasonable inference from available evidence
|
||||
- high — strong evidence, direct observation, or confirmed fact
|
||||
|
||||
## Importance scale (evidence records)
|
||||
|
||||
- incidental — minor detail, unlikely to affect conclusions
|
||||
- supporting — adds context but not critical
|
||||
- important — materially affects understanding of the situation
|
||||
- critical — essential to resolving the situation; without it conclusions cannot be drawn
|
||||
|
||||
## Expected information value (next question)
|
||||
|
||||
- low — marginally useful even if answered
|
||||
- medium — meaningfully clarifies the situation
|
||||
- high — would significantly distinguish between plausible explanations or fill a gap in understanding
|
||||
|
||||
## Next question selection criteria
|
||||
|
||||
Prefer questions that:
|
||||
- clarify a major difference
|
||||
- establish a baseline
|
||||
- explain an important transition
|
||||
- test an unsupported claim
|
||||
- distinguish between plausible explanations
|
||||
- request measurable evidence
|
||||
- identify who or what is affected
|
||||
- establish timing
|
||||
|
||||
Avoid questions that:
|
||||
- have already been answered
|
||||
- assume a cause
|
||||
- jump to a solution
|
||||
- ask about motive before the observable situation is understood
|
||||
- focus on incidental wording
|
||||
- are too broad to produce useful information
|
||||
- combine many unrelated questions
|
||||
|
||||
## Output format — return this exact JSON structure
|
||||
|
||||
Return a JSON object with exactly these four top-level keys (use **camelCase**):
|
||||
|
||||
```json
|
||||
{
|
||||
"inputClassification": {
|
||||
"primaryType": "<one of: observed_problem, unexplained_change, contradiction, decision_request, causal_claim, reported_claim, fault_report, ambiguous_statement, question, desired_outcome, insufficient_context, other>",
|
||||
"secondaryTypes": ["<optional additional types from the same list>"],
|
||||
"reasoningModes": ["<one or more of: establish_baseline, identify_difference, reconstruct_transition, decompose_aggregate, validate_measurement, validate_claim, investigate_contradiction, clarify_meaning, decision_support, fault_investigation, identify_missing_information, test_possible_explanations, other>"],
|
||||
"classificationReason": "<brief explanation of why you chose the primary type>",
|
||||
"confidence": "<low | medium | high>"
|
||||
},
|
||||
"reconstruction": {
|
||||
"summary": "<one-sentence overview of the situation>",
|
||||
"actors": [{"id": "<any unique string>", "description": "...", "confidence": "<low|medium|high>"}],
|
||||
"systemsOrObjects": [{"id": "<any unique string>", "description": "...", "confidence": "<low|medium|high>"}],
|
||||
"expectedStates": [{"id": "...", "description": "...", "confidence": "<low|medium|high>"}],
|
||||
"observedStates": [{"id": "...", "description": "...", "confidence": "<low|medium|high>"}],
|
||||
"differences": [{"id": "...", "description": "...", "confidence": "<low|medium|high>"}],
|
||||
"knownTransitions": [{"id": "...", "description": "...", "confidence": "<low|medium|high>", "entity": "...", "previousState": "...", "currentState": "...", "explanationStatus": "..."}],
|
||||
"unexplainedTransitions": [{"id": "...", "description": "...", "confidence": "<low|medium|high>", "entity": "...", "previousState": "...", "currentState": "..."}],
|
||||
"contradictions": [{"id": "...", "description": "...", "confidence": "<low|medium|high>"}],
|
||||
"importantUnknowns": [{"id": "...", "description": "...", "confidence": "<low|medium|high>"}],
|
||||
"plausibleInterpretations": [{"id": "...", "description": "...", "supportingEvidenceIds": ["<ids that support this interpretation>"], "assumptionsRequired": [], "confidence": "<low|medium|high>"}]
|
||||
},
|
||||
"evidence": [
|
||||
{
|
||||
"id": "<any unique string>",
|
||||
"description": "...",
|
||||
"evidenceType": "<direct_observation | reported_statement | interpretation | assumption | inferred_relationship>",
|
||||
"source": "<optional — who/where this came from>",
|
||||
"attribution": null,
|
||||
"confidence": "<low | medium | high>",
|
||||
"importance": "<incidental | supporting | important | critical>"
|
||||
}
|
||||
],
|
||||
"nextQuestion": {
|
||||
"id": "<any unique string>",
|
||||
"question": "<one precise question>",
|
||||
"targets": ["<what this question targets — e.g. 'actor', 'system', 'expectedOutcome'>"],
|
||||
"reason": "<why answering this is important>",
|
||||
"expectedInformationValue": "<low | medium | high>",
|
||||
"reasoningMode": "<optional reasoning mode from the list above>"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
CRITICAL RULES for JSON output:
|
||||
1. Use **exactly** the key names shown above (camelCase, no snake_case).
|
||||
2. The four top-level keys must be: `inputClassification`, `reconstruction`, `evidence`, `nextQuestion`.
|
||||
3. Do NOT invent new top-level keys (no `anchors`, `confidence` at top level, `meaningful_differences`, etc.).
|
||||
4. Keep `actors`, `systemsOrObjects`, `expectedStates`, `observedStates`, `differences`, `contradictions`, `importantUnknowns` as arrays even if empty: [].
|
||||
5. Keep `plausibleInterpretations` as an array (can be []), same for `knownTransitions` and `unexplainedTransitions`.
|
||||
6. Each object in arrays must have at least `id`, `description`, `confidence`.
|
||||
7. **evidenceType**: classify each evidence item clearly as either a direct observation, a reported statement, an interpretation, an assumption, or an inferred relationship. Do not treat raw counts as proof of causal relationships — they may be inferred relationships only when supported by explicit reasoning about denominators or rates.
|
||||
|
||||
Scenario:
|
||||
{{SCENARIO}}
|
||||
|
||||
Return ONLY the JSON object starting with { and ending with }. Do NOT include any text before the opening brace or after the closing brace. Do NOT wrap in markdown backticks.
|
||||
@@ -0,0 +1,55 @@
|
||||
import { test, expect } from "@playwright/test";
|
||||
|
||||
test("v0.3 UI smoke test with live model response", async ({ page }) => {
|
||||
await page.goto("http://localhost:3000");
|
||||
|
||||
// Page should load without error
|
||||
await expect(page.getByText(/Confidence Engine/i)).toBeVisible();
|
||||
|
||||
// Type the scenario
|
||||
const textarea = page.locator("textarea[placeholder*='Describe']");
|
||||
await textarea.fill("Complaints increased by 35% while production increased by 40%.");
|
||||
|
||||
// Button should be enabled
|
||||
await expect(page.getByRole("button", { name: /Analyse/i })).toBeEnabled();
|
||||
|
||||
// Click Analyse and wait for diagnostics panel
|
||||
await page.getByRole("button", { name: /Analyse/i }).click();
|
||||
|
||||
// Wait for result section (ReconstructionView rendered)
|
||||
await expect(page.getByRole("heading", { name: /Next Question/i })).toBeVisible({ timeout: 180000 });
|
||||
|
||||
// Take screenshot of result page
|
||||
await page.screenshot({ path: "tests-results/smoke-v0.3.png", fullPage: true });
|
||||
|
||||
// Verify diagnostics panel exists and contains relevant info
|
||||
const diagPanel = page.locator('details summary').first();
|
||||
if (await diagPanel.isVisible()) {
|
||||
console.log("Raw response viewer:", await diagPanel.innerText().catch(() => "not visible"));
|
||||
}
|
||||
|
||||
// Get full body text for verification
|
||||
const bodyText = await page.locator("body").innerText();
|
||||
|
||||
console.log("\n=== UI Smoke Test Results ===");
|
||||
console.log("Page title:", await page.title());
|
||||
console.log("Body content length:", bodyText.length);
|
||||
|
||||
// Check key content indicators
|
||||
const hasNextQ = bodyText.includes("Next Question");
|
||||
const hasComplaints = bodyText.includes("Complaint") || bodyText.includes("complaint");
|
||||
const hasProduction = bodyText.includes("production") || bodyText.includes("Production");
|
||||
const hasRateContext = bodyText.toLowerCase().includes("rate") ||
|
||||
bodyText.toLowerCase().includes("unit") ||
|
||||
bodyText.toLowerCase().includes("denominator") ||
|
||||
bodyText.toLowerCase().includes("per-unit");
|
||||
|
||||
console.log("Has Next Question heading:", hasNextQ);
|
||||
console.log("Has complaints reference:", hasComplaints);
|
||||
console.log("Has production reference:", hasProduction);
|
||||
console.log("Has rate context (rate/unit/denominator):", hasRateContext);
|
||||
|
||||
// Basic structural checks
|
||||
expect(bodyText.length).toBeGreaterThan(200);
|
||||
expect(hasNextQ).toBe(true);
|
||||
}, { timeout: 300000 });
|
||||
@@ -0,0 +1,598 @@
|
||||
import { describe, it, expect } from "vitest";
|
||||
import { promises as fs } from "node:fs";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { dirname, join } from "node:path";
|
||||
import {
|
||||
PROMPT_VERSIONS,
|
||||
buildPrompt,
|
||||
DEFAULT_PROMPT_VERSION,
|
||||
} from "@/lib/reconstruction/prompt.js";
|
||||
import {
|
||||
reconstructionV2Schema,
|
||||
parseReconstructionV2,
|
||||
} from "@/lib/reconstruction/schema.js";
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = dirname(__filename);
|
||||
const PROMPTS_DIR = join(__dirname, "../prompts");
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
// v0.3 prompt loading tests
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
describe("v0.3 prompt", () => {
|
||||
it("v0.3 is in PROMPT_VERSIONS", () => {
|
||||
expect(PROMPT_VERSIONS).toContain("v0.3");
|
||||
});
|
||||
|
||||
it("DEFAULT_PROMPT_VERSION is v0.3 on this branch", () => {
|
||||
expect(DEFAULT_PROMPT_VERSION).toBe("v0.3");
|
||||
});
|
||||
|
||||
it("v0.2 remains available in PROMPT_VERSIONS", () => {
|
||||
expect(PROMPT_VERSIONS).toContain("v0.2");
|
||||
});
|
||||
|
||||
it("v0.3 prompt file loads from disk", async () => {
|
||||
const content = await fs.readFile(
|
||||
join(PROMPTS_DIR, "reconstruct-v0.3.md"),
|
||||
"utf-8",
|
||||
);
|
||||
expect(typeof content).toBe("string");
|
||||
expect(content.length).toBeGreaterThan(500);
|
||||
});
|
||||
|
||||
it("v0.3 prompt contains normalisation guidance", async () => {
|
||||
const content = await fs.readFile(
|
||||
join(PROMPTS_DIR, "reconstruct-v0.3.md"),
|
||||
"utf-8",
|
||||
);
|
||||
expect(content.toLowerCase()).toContain("normalise");
|
||||
expect(content.toLowerCase()).toContain("rate");
|
||||
expect(content.toLowerCase()).toContain("denominator") ||
|
||||
expect(content.toLowerCase()).toContain("exposure");
|
||||
});
|
||||
|
||||
it("v0.3 prompt contains discipline guidance", async () => {
|
||||
const content = await fs.readFile(
|
||||
join(PROMPTS_DIR, "reconstruct-v0.3.md"),
|
||||
"utf-8",
|
||||
);
|
||||
// Should mention not generating speculative interpretations
|
||||
expect(content).toMatch(/interpretation/i);
|
||||
// Should mention one question discipline
|
||||
expect(content).toMatch(/exactly.*one.*question|one.*only.*question|single.*question/i) ||
|
||||
expect(content).toMatch(/Do NOT combine/i);
|
||||
});
|
||||
|
||||
it("buildPrompt returns v0.3 prompt with scenario substituted", async () => {
|
||||
const result = await buildPrompt("Test scenario text", "v0.3");
|
||||
expect(result.version).toBe("v0.3");
|
||||
expect(result.prompt).toContain("Test scenario text");
|
||||
// Should contain the normalisation section guidance
|
||||
expect(result.prompt.toLowerCase()).toContain("normalise");
|
||||
});
|
||||
|
||||
it("buildPrompt returns v0.2 prompt when requested", async () => {
|
||||
const result = await buildPrompt("Test scenario text", "v0.2");
|
||||
expect(result.version).toBe("v0.2");
|
||||
expect(result.prompt).toContain("Test scenario text");
|
||||
});
|
||||
|
||||
it("buildPrompt default is v0.3", async () => {
|
||||
const result = await buildPrompt("Test scenario text");
|
||||
expect(result.version).toBe("v0.3");
|
||||
});
|
||||
});
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
// v0.2 prompt still works
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
describe("v0.2 backward compatibility", () => {
|
||||
it("v0.2 prompt file exists and loads", async () => {
|
||||
const content = await fs.readFile(
|
||||
join(PROMPTS_DIR, "reconstruct-v0.2.md"),
|
||||
"utf-8",
|
||||
);
|
||||
expect(typeof content).toBe("string");
|
||||
expect(content.length).toBeGreaterThan(500);
|
||||
});
|
||||
|
||||
it("buildPrompt returns v0.2 version string", async () => {
|
||||
const result = await buildPrompt("test", "v0.2");
|
||||
expect(result.version).toBe("v0.2");
|
||||
});
|
||||
});
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
// Schema validation tests for v0.3-shaped output
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
describe("v0.3 schema validation", () => {
|
||||
it("validates a complete valid reconstruction with empty interpretations", () => {
|
||||
const input = {
|
||||
inputClassification: {
|
||||
primaryType: "unexplained_change",
|
||||
secondaryTypes: ["reported_claim"],
|
||||
reasoningModes: ["identify_difference"],
|
||||
classificationReason: "Two metrics changed without explanation.",
|
||||
confidence: "medium",
|
||||
},
|
||||
reconstruction: {
|
||||
summary: "Both complaints and production increased.",
|
||||
actors: [],
|
||||
systemsOrObjects: [
|
||||
{ id: "complaints_metric", description: "Volume of complaints", confidence: "high" },
|
||||
],
|
||||
expectedStates: [],
|
||||
observedStates: [
|
||||
{ id: "obs1", description: "Complaint volume rose by 35%", confidence: "medium" },
|
||||
{ id: "obs2", description: "Production volume rose by 40%", confidence: "medium" },
|
||||
],
|
||||
differences: [
|
||||
{
|
||||
id: "diff1",
|
||||
description:
|
||||
"Production grew faster than complaints, so the complaint-to-production ratio may have improved.",
|
||||
confidence: "medium",
|
||||
},
|
||||
],
|
||||
knownTransitions: [],
|
||||
unexplainedTransitions: [
|
||||
{
|
||||
id: "trans1",
|
||||
description: "Complaint volume shifted to a higher level without explained cause",
|
||||
confidence: "medium",
|
||||
entity: "complaints_metric",
|
||||
previousState: "Baseline volume (unknown)",
|
||||
currentState: "+35% increase",
|
||||
},
|
||||
],
|
||||
contradictions: [],
|
||||
importantUnknowns: [
|
||||
{
|
||||
id: "unk1",
|
||||
description:
|
||||
"Absolute baseline volumes and time period needed to compute complaint rate per unit",
|
||||
confidence: "low",
|
||||
},
|
||||
],
|
||||
plausibleInterpretations: [], // intentionally empty — evidence too thin
|
||||
},
|
||||
evidence: [
|
||||
{
|
||||
id: "ev1",
|
||||
description: "Complaints increased by 35%",
|
||||
evidenceType: "reported_statement",
|
||||
source: "User input",
|
||||
attribution: null,
|
||||
confidence: "medium",
|
||||
importance: "important",
|
||||
},
|
||||
{
|
||||
id: "ev2",
|
||||
description: "Production increased by 40%",
|
||||
evidenceType: "reported_statement",
|
||||
source: "User input",
|
||||
attribution: null,
|
||||
confidence: "medium",
|
||||
importance: "important",
|
||||
},
|
||||
{
|
||||
id: "ev3",
|
||||
description:
|
||||
"Production growth rate (40%) exceeded complaint growth rate (35%), implying the denominator may have grown faster than complaints.",
|
||||
evidenceType: "inferred_relationship",
|
||||
attribution: null,
|
||||
confidence: "medium",
|
||||
importance: "important",
|
||||
},
|
||||
],
|
||||
nextQuestion: {
|
||||
id: "q1",
|
||||
question: "What was the complaint rate per unit before and after the production increase?",
|
||||
targets: ["system"],
|
||||
reason:
|
||||
"Without normalising complaints by production volume, the absolute complaint count change is misleading. The rate per unit determines whether the situation improved, stayed stable, or worsened.",
|
||||
expectedInformationValue: "high",
|
||||
reasoningMode: "decompose_aggregate",
|
||||
},
|
||||
};
|
||||
|
||||
const result = reconstructionV2Schema.safeParse(input);
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("rejects output missing required fields", () => {
|
||||
const input = {
|
||||
inputClassification: { primaryType: "other" },
|
||||
reconstruction: {},
|
||||
evidence: [],
|
||||
nextQuestion: { id: "q1" },
|
||||
};
|
||||
|
||||
const result = reconstructionV2Schema.safeParse(input);
|
||||
expect(result.success).toBe(false);
|
||||
});
|
||||
|
||||
it("validates empty arrays for all reconstruction categories", () => {
|
||||
const input = {
|
||||
inputClassification: {
|
||||
primaryType: "other",
|
||||
classificationReason: "test",
|
||||
confidence: "low",
|
||||
},
|
||||
reconstruction: {
|
||||
summary: "empty test",
|
||||
actors: [],
|
||||
systemsOrObjects: [],
|
||||
expectedStates: [],
|
||||
observedStates: [],
|
||||
differences: [],
|
||||
knownTransitions: [],
|
||||
unexplainedTransitions: [],
|
||||
contradictions: [],
|
||||
importantUnknowns: [],
|
||||
plausibleInterpretations: [],
|
||||
},
|
||||
evidence: [],
|
||||
nextQuestion: {
|
||||
id: "q1",
|
||||
question: "What is the production volume?",
|
||||
targets: ["system"],
|
||||
reason: "need baseline",
|
||||
expectedInformationValue: "medium",
|
||||
},
|
||||
};
|
||||
|
||||
const result = reconstructionV2Schema.safeParse(input);
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("validates evidence distinguishing direct_observation from inferred_relationship", () => {
|
||||
const input = {
|
||||
inputClassification: {
|
||||
primaryType: "unexplained_change",
|
||||
classificationReason: "test",
|
||||
confidence: "low",
|
||||
},
|
||||
reconstruction: {
|
||||
summary: "test summary",
|
||||
actors: [],
|
||||
systemsOrObjects: [],
|
||||
expectedStates: [],
|
||||
observedStates: [{ id: "o1", description: "x", confidence: "high" }],
|
||||
differences: [],
|
||||
knownTransitions: [],
|
||||
unexplainedTransitions: [],
|
||||
contradictions: [],
|
||||
importantUnknowns: [],
|
||||
plausibleInterpretations: [],
|
||||
},
|
||||
evidence: [
|
||||
{
|
||||
id: "ev1",
|
||||
description: "Observed fact",
|
||||
evidenceType: "direct_observation",
|
||||
confidence: "high",
|
||||
importance: "critical",
|
||||
},
|
||||
{
|
||||
id: "ev2",
|
||||
description: "Derived relationship",
|
||||
evidenceType: "inferred_relationship",
|
||||
confidence: "medium",
|
||||
importance: "supporting",
|
||||
},
|
||||
],
|
||||
nextQuestion: {
|
||||
id: "q1",
|
||||
question: "What is the denominator?",
|
||||
targets: ["system"],
|
||||
reason: "need context",
|
||||
expectedInformationValue: "high",
|
||||
},
|
||||
};
|
||||
|
||||
const result = reconstructionV2Schema.safeParse(input);
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
// parseReconstructionV2 helper tests
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
describe("parseReconstructionV2", () => {
|
||||
it("parses a valid v0.3-shaped JSON string", async () => {
|
||||
const fixture = {
|
||||
inputClassification: {
|
||||
primaryType: "unexplained_change",
|
||||
classificationReason: "test",
|
||||
confidence: "medium",
|
||||
},
|
||||
reconstruction: {
|
||||
summary: "both increased",
|
||||
actors: [],
|
||||
systemsOrObjects: [],
|
||||
expectedStates: [],
|
||||
observedStates: [
|
||||
{ id: "o1", description: "x rose 35%", confidence: "high" },
|
||||
{ id: "o2", description: "y rose 40%", confidence: "high" },
|
||||
],
|
||||
differences: [{ id: "d1", description: "y grew faster", confidence: "medium" }],
|
||||
knownTransitions: [],
|
||||
unexplainedTransitions: [],
|
||||
contradictions: [],
|
||||
importantUnknowns: [],
|
||||
plausibleInterpretations: [],
|
||||
},
|
||||
evidence: [
|
||||
{ id: "e1", description: "x rose 35%", evidenceType: "reported_statement", confidence: "medium", importance: "important" },
|
||||
{ id: "e2", description: "y rose 40%", evidenceType: "reported_statement", confidence: "medium", importance: "important" },
|
||||
],
|
||||
nextQuestion: {
|
||||
id: "q1",
|
||||
question: "What is the denominator?",
|
||||
targets: ["system"],
|
||||
reason: "need rate context",
|
||||
expectedInformationValue: "high",
|
||||
},
|
||||
};
|
||||
|
||||
const raw = JSON.stringify(fixture);
|
||||
const parsed = parseReconstructionV2(raw);
|
||||
|
||||
expect(parsed.inputClassification.primaryType).toBe("unexplained_change");
|
||||
expect(parsed.reconstruction.summary).toBe("both increased");
|
||||
expect(parsed.nextQuestion.question).toBe("What is the denominator?");
|
||||
});
|
||||
|
||||
it("rejects non-JSON string", () => {
|
||||
expect(() => parseReconstructionV2("{not valid json")).toThrow(SyntaxError);
|
||||
});
|
||||
});
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
// v0.3 prompt contains required guidance text
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
describe("v0.3 prompt guidance completeness", () => {
|
||||
it("mentions normalise counts when scale changed", async () => {
|
||||
const content = await fs.readFile(
|
||||
join(PROMPTS_DIR, "reconstruct-v0.3.md"),
|
||||
"utf-8",
|
||||
);
|
||||
expect(content.toLowerCase()).toMatch(/normali[sz]e|normalis[ei]ng/);
|
||||
});
|
||||
|
||||
it("mentions distinguishing total count from rate", async () => {
|
||||
const content = await fs.readFile(
|
||||
join(PROMPTS_DIR, "reconstruct-v0.3.md"),
|
||||
"utf-8",
|
||||
);
|
||||
expect(content.toLowerCase()).toContain("rate");
|
||||
expect(content.toLowerCase()).toMatch(/count.*not.*caus|correlation.*caus|distinguish.*count/);
|
||||
});
|
||||
|
||||
it("mentions avoiding correlation-as-causation", async () => {
|
||||
const content = await fs.readFile(
|
||||
join(PROMPTS_DIR, "reconstruct-v0.3.md"),
|
||||
"utf-8",
|
||||
);
|
||||
expect(content.toLowerCase()).toMatch(/correlation.*caus|treating.*correlation.*caus/);
|
||||
});
|
||||
|
||||
it("mentions prefer one narrow next question over compound", async () => {
|
||||
const content = await fs.readFile(
|
||||
join(PROMPTS_DIR, "reconstruct-v0.3.md"),
|
||||
"utf-8",
|
||||
);
|
||||
// Should mention single vs compound
|
||||
expect(content).toMatch(/exactly.*one|single.*question|Do NOT combine|combine.*multiple/i);
|
||||
});
|
||||
|
||||
it("mentions leaving empty interpretations when evidence is thin", async () => {
|
||||
const content = await fs.readFile(
|
||||
join(PROMPTS_DIR, "reconstruct-v0.3.md"),
|
||||
"utf-8",
|
||||
);
|
||||
expect(content).toMatch(/empty.*array|do not generate.*interpretation|fill a list/i);
|
||||
});
|
||||
|
||||
it("mentions identifying the denominator or exposure metric", async () => {
|
||||
const content = await fs.readFile(
|
||||
join(PROMPTS_DIR, "reconstruct-v0.3.md"),
|
||||
"utf-8",
|
||||
);
|
||||
expect(content.toLowerCase()).toMatch(/denominator|exposure/);
|
||||
});
|
||||
|
||||
it("uses the exact scenario text as a reference example only (not in rules)", async () => {
|
||||
const content = await fs.readFile(
|
||||
join(PROMPTS_DIR, "reconstruct-v0.3.md"),
|
||||
"utf-8",
|
||||
);
|
||||
// The prompt should be domain-independent — it should not mention specific industries as rules
|
||||
// but may have an example section. We verify the prompt does not hard-code a specific question text.
|
||||
expect(content).not.toMatch(/What was the complaint rate per unit before and after/);
|
||||
});
|
||||
});
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
// Fixture: expected good structure for target scenario
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
describe("target scenario fixture validation", () => {
|
||||
const goodFixture = JSON.parse(JSON.stringify({
|
||||
inputClassification: {
|
||||
primaryType: "unexplained_change",
|
||||
secondaryTypes: ["reported_claim"],
|
||||
reasoningModes: ["identify_difference", "decompose_aggregate"],
|
||||
classificationReason:
|
||||
"Two operational quantities changed at different percentages without a shared baseline or denominator.",
|
||||
confidence: "medium",
|
||||
},
|
||||
reconstruction: {
|
||||
summary:
|
||||
"Both complaint counts and production volumes increased, but production grew slightly faster than complaints — without absolute baselines the per-unit complaint rate cannot be determined.",
|
||||
actors: [],
|
||||
systemsOrObjects: [
|
||||
{ id: "so1", description: "Production system or output volume", confidence: "high" },
|
||||
{ id: "so2", description: "Complaint reporting mechanism", confidence: "high" },
|
||||
],
|
||||
expectedStates: [],
|
||||
observedStates: [
|
||||
{ id: "obs1", description: "Complaint count increased by 35%", confidence: "high" },
|
||||
{ id: "obs2", description: "Production volume increased by 40%", confidence: "high" },
|
||||
],
|
||||
differences: [
|
||||
{
|
||||
id: "diff1",
|
||||
description:
|
||||
"Production grew faster than complaints (+40% vs +35%), so the ratio of complaints per unit may have decreased or remained stable. The absolute complaint count alone is not a reliable indicator of whether conditions have changed.",
|
||||
confidence: "high",
|
||||
},
|
||||
],
|
||||
knownTransitions: [],
|
||||
unexplainedTransitions: [
|
||||
{
|
||||
id: "ut1",
|
||||
description: "Complaint volume shifted to a higher level without explained cause",
|
||||
confidence: "medium",
|
||||
entity: "complaints_metric",
|
||||
previousState: "unknown baseline",
|
||||
currentState: "+35%",
|
||||
},
|
||||
],
|
||||
contradictions: [],
|
||||
importantUnknowns: [
|
||||
{
|
||||
id: "unk1",
|
||||
description:
|
||||
"Absolute complaint count and production volume baselines needed to compute the per-unit rate",
|
||||
confidence: "low",
|
||||
},
|
||||
{
|
||||
id: "unk2",
|
||||
description: "Time period over which these changes occurred",
|
||||
confidence: "low",
|
||||
},
|
||||
],
|
||||
plausibleInterpretations: [], // intentionally empty — no sufficient evidence for interpretations
|
||||
},
|
||||
evidence: [
|
||||
{
|
||||
id: "ev1",
|
||||
description: "Complaints increased by 35%",
|
||||
evidenceType: "reported_statement",
|
||||
source: "Scenario input",
|
||||
attribution: null,
|
||||
confidence: "high",
|
||||
importance: "important",
|
||||
},
|
||||
{
|
||||
id: "ev2",
|
||||
description: "Production increased by 40%",
|
||||
evidenceType: "reported_statement",
|
||||
source: "Scenario input",
|
||||
attribution: null,
|
||||
confidence: "high",
|
||||
importance: "important",
|
||||
},
|
||||
{
|
||||
id: "ev3",
|
||||
description: "Complaint count grew more slowly than production volume, suggesting per-unit rates may have improved or stayed stable.",
|
||||
evidenceType: "inferred_relationship",
|
||||
attribution: null,
|
||||
confidence: "medium",
|
||||
importance: "important",
|
||||
},
|
||||
],
|
||||
nextQuestion: {
|
||||
id: "q1",
|
||||
question: "What was the absolute complaint volume and production volume (or baseline) before these percentage changes?",
|
||||
targets: ["system", "measurement"],
|
||||
reason:
|
||||
"Without baseline counts to compute a rate per unit, we cannot determine whether conditions have worsened, stayed stable, or improved. The rate comparison is the smallest unresolved comparison needed to evaluate the situation.",
|
||||
expectedInformationValue: "high",
|
||||
reasoningMode: "decompose_aggregate",
|
||||
},
|
||||
}));
|
||||
|
||||
it("fixture validates against v0.3 schema", () => {
|
||||
const result = reconstructionV2Schema.safeParse(goodFixture);
|
||||
expect(result.success).toBe(true);
|
||||
});
|
||||
|
||||
it("fixture has exactly one next question with non-empty text", () => {
|
||||
expect(goodFixture.nextQuestion.question.length).toBeGreaterThan(10);
|
||||
expect(goodFixture.nextQuestion.reason.length).toBeGreaterThan(10);
|
||||
expect(goodFixture.nextQuestion.expectedInformationValue).toBe("high");
|
||||
});
|
||||
|
||||
it("fixture has empty plausibleInterpretations (evidence too thin)", () => {
|
||||
expect(goodFixture.reconstruction.plausibleInterpretations).toEqual([]);
|
||||
});
|
||||
|
||||
it("fixture evidence includes both direct observations and one inferred relationship", () => {
|
||||
const types = goodFixture.evidence.map((e) => e.evidenceType);
|
||||
expect(types).toContain("reported_statement");
|
||||
expect(types).toContain("inferred_relationship");
|
||||
});
|
||||
|
||||
it("fixture relationship notes complaint count grew more slowly than production", () => {
|
||||
const diffDescs = goodFixture.reconstruction.differences.map((d) => d.description);
|
||||
const found = diffDescs.some(
|
||||
(d) =>
|
||||
d.toLowerCase().includes("fast") ||
|
||||
d.toLowerCase().includes("slower") ||
|
||||
d.toLowerCase().includes("ratio") ||
|
||||
d.toLowerCase().includes("per-unit") ||
|
||||
d.toLowerCase().includes("per unit"),
|
||||
);
|
||||
expect(found).toBe(true);
|
||||
});
|
||||
|
||||
it("fixture does not assert quality deterioration", () => {
|
||||
const allText = [
|
||||
goodFixture.reconstruction.summary,
|
||||
...goodFixture.reconstruction.differences.map((d) => d.description),
|
||||
goodFixture.nextQuestion.reason,
|
||||
].join(" ").toLowerCase();
|
||||
// Should not contain strong deterioration language without caveats
|
||||
expect(allText).not.toMatch(/quality.*deteriorat|quality.*worsen|definitely.*bad/);
|
||||
});
|
||||
|
||||
it("fixture includes relationship that production grew faster", () => {
|
||||
const allText = [
|
||||
goodFixture.reconstruction.summary,
|
||||
...goodFixture.reconstruction.differences.map((d) => d.description),
|
||||
].join(" ").toLowerCase();
|
||||
expect(allText).toMatch(/produ.*grow|ratio|per-unit|per unit|\+40.*\+35/);
|
||||
});
|
||||
});
|
||||
|
||||
// ──────────────────────────────────────────────
|
||||
// Diagnostics: prompt version tracking
|
||||
// ──────────────────────────────────────────────
|
||||
|
||||
describe("diagnostics prompt version", () => {
|
||||
it("DEFAULT_PROMPT_VERSION is exported correctly", () => {
|
||||
expect(DEFAULT_PROMPT_VERSION).toBe("v0.3");
|
||||
});
|
||||
|
||||
it("PROMPT_VERSIONS includes both v0.2 and v0.3", () => {
|
||||
const hasV2 = PROMPT_VERSIONS.includes("v0.2");
|
||||
const hasV3 = PROMPT_VERSIONS.includes("v0.3");
|
||||
expect(hasV2).toBe(true);
|
||||
expect(hasV3).toBe(true);
|
||||
});
|
||||
|
||||
it("RECONSTRUCTION_PROMPT_VERSION env var overrides default", async () => {
|
||||
// The actual override happens at module load time, so we can't easily test this
|
||||
// in isolation. Instead, verify the constant reflects env or defaults to v0.3.
|
||||
expect(PROMPT_VERSIONS).toContain("v0.2");
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user