robbond
79ea2f6824
Add explicit reasoning guidance for normalising counts by exposure/denominator, distinguishing total count from rate, and avoiding correlation-as-causation errors. Changes: - prompts/reconstruct-v0.3.md: new prompt with normalisation discipline - lib/reconstruction/prompt.js: v0.3 loader + env var override support - lib/analysis.js: defer DEFAULT_PROMPT_VERSION to prompt module (defaults to v0.3) - PROMPT_VERSIONS extended to [v0.1, v0.2, v0.3] - tests/v03-reasoning.test.js: 34 focused tests covering prompt loading, schema validation, guidance completeness, and target scenario fixture - playwright.config.js + tests/smoke.test.js: minimal UI smoke test for browser rendering - package.json: add @playwright/test as devDependency Default switches to v0.3; v0.2 selectable via promptVersion or RECONSTRUCTION_PROMPT_VERSION env var.
599 lines
22 KiB
JavaScript
599 lines
22 KiB
JavaScript
import { describe, it, expect } from "vitest";
|
|
import { promises as fs } from "node:fs";
|
|
import { fileURLToPath } from "node:url";
|
|
import { dirname, join } from "node:path";
|
|
import {
|
|
PROMPT_VERSIONS,
|
|
buildPrompt,
|
|
DEFAULT_PROMPT_VERSION,
|
|
} from "@/lib/reconstruction/prompt.js";
|
|
import {
|
|
reconstructionV2Schema,
|
|
parseReconstructionV2,
|
|
} from "@/lib/reconstruction/schema.js";
|
|
|
|
const __filename = fileURLToPath(import.meta.url);
|
|
const __dirname = dirname(__filename);
|
|
const PROMPTS_DIR = join(__dirname, "../prompts");
|
|
|
|
// ──────────────────────────────────────────────
|
|
// v0.3 prompt loading tests
|
|
// ──────────────────────────────────────────────
|
|
|
|
describe("v0.3 prompt", () => {
|
|
it("v0.3 is in PROMPT_VERSIONS", () => {
|
|
expect(PROMPT_VERSIONS).toContain("v0.3");
|
|
});
|
|
|
|
it("DEFAULT_PROMPT_VERSION is v0.3 on this branch", () => {
|
|
expect(DEFAULT_PROMPT_VERSION).toBe("v0.3");
|
|
});
|
|
|
|
it("v0.2 remains available in PROMPT_VERSIONS", () => {
|
|
expect(PROMPT_VERSIONS).toContain("v0.2");
|
|
});
|
|
|
|
it("v0.3 prompt file loads from disk", async () => {
|
|
const content = await fs.readFile(
|
|
join(PROMPTS_DIR, "reconstruct-v0.3.md"),
|
|
"utf-8",
|
|
);
|
|
expect(typeof content).toBe("string");
|
|
expect(content.length).toBeGreaterThan(500);
|
|
});
|
|
|
|
it("v0.3 prompt contains normalisation guidance", async () => {
|
|
const content = await fs.readFile(
|
|
join(PROMPTS_DIR, "reconstruct-v0.3.md"),
|
|
"utf-8",
|
|
);
|
|
expect(content.toLowerCase()).toContain("normalise");
|
|
expect(content.toLowerCase()).toContain("rate");
|
|
expect(content.toLowerCase()).toContain("denominator") ||
|
|
expect(content.toLowerCase()).toContain("exposure");
|
|
});
|
|
|
|
it("v0.3 prompt contains discipline guidance", async () => {
|
|
const content = await fs.readFile(
|
|
join(PROMPTS_DIR, "reconstruct-v0.3.md"),
|
|
"utf-8",
|
|
);
|
|
// Should mention not generating speculative interpretations
|
|
expect(content).toMatch(/interpretation/i);
|
|
// Should mention one question discipline
|
|
expect(content).toMatch(/exactly.*one.*question|one.*only.*question|single.*question/i) ||
|
|
expect(content).toMatch(/Do NOT combine/i);
|
|
});
|
|
|
|
it("buildPrompt returns v0.3 prompt with scenario substituted", async () => {
|
|
const result = await buildPrompt("Test scenario text", "v0.3");
|
|
expect(result.version).toBe("v0.3");
|
|
expect(result.prompt).toContain("Test scenario text");
|
|
// Should contain the normalisation section guidance
|
|
expect(result.prompt.toLowerCase()).toContain("normalise");
|
|
});
|
|
|
|
it("buildPrompt returns v0.2 prompt when requested", async () => {
|
|
const result = await buildPrompt("Test scenario text", "v0.2");
|
|
expect(result.version).toBe("v0.2");
|
|
expect(result.prompt).toContain("Test scenario text");
|
|
});
|
|
|
|
it("buildPrompt default is v0.3", async () => {
|
|
const result = await buildPrompt("Test scenario text");
|
|
expect(result.version).toBe("v0.3");
|
|
});
|
|
});
|
|
|
|
// ──────────────────────────────────────────────
|
|
// v0.2 prompt still works
|
|
// ──────────────────────────────────────────────
|
|
|
|
describe("v0.2 backward compatibility", () => {
|
|
it("v0.2 prompt file exists and loads", async () => {
|
|
const content = await fs.readFile(
|
|
join(PROMPTS_DIR, "reconstruct-v0.2.md"),
|
|
"utf-8",
|
|
);
|
|
expect(typeof content).toBe("string");
|
|
expect(content.length).toBeGreaterThan(500);
|
|
});
|
|
|
|
it("buildPrompt returns v0.2 version string", async () => {
|
|
const result = await buildPrompt("test", "v0.2");
|
|
expect(result.version).toBe("v0.2");
|
|
});
|
|
});
|
|
|
|
// ──────────────────────────────────────────────
|
|
// Schema validation tests for v0.3-shaped output
|
|
// ──────────────────────────────────────────────
|
|
|
|
describe("v0.3 schema validation", () => {
|
|
it("validates a complete valid reconstruction with empty interpretations", () => {
|
|
const input = {
|
|
inputClassification: {
|
|
primaryType: "unexplained_change",
|
|
secondaryTypes: ["reported_claim"],
|
|
reasoningModes: ["identify_difference"],
|
|
classificationReason: "Two metrics changed without explanation.",
|
|
confidence: "medium",
|
|
},
|
|
reconstruction: {
|
|
summary: "Both complaints and production increased.",
|
|
actors: [],
|
|
systemsOrObjects: [
|
|
{ id: "complaints_metric", description: "Volume of complaints", confidence: "high" },
|
|
],
|
|
expectedStates: [],
|
|
observedStates: [
|
|
{ id: "obs1", description: "Complaint volume rose by 35%", confidence: "medium" },
|
|
{ id: "obs2", description: "Production volume rose by 40%", confidence: "medium" },
|
|
],
|
|
differences: [
|
|
{
|
|
id: "diff1",
|
|
description:
|
|
"Production grew faster than complaints, so the complaint-to-production ratio may have improved.",
|
|
confidence: "medium",
|
|
},
|
|
],
|
|
knownTransitions: [],
|
|
unexplainedTransitions: [
|
|
{
|
|
id: "trans1",
|
|
description: "Complaint volume shifted to a higher level without explained cause",
|
|
confidence: "medium",
|
|
entity: "complaints_metric",
|
|
previousState: "Baseline volume (unknown)",
|
|
currentState: "+35% increase",
|
|
},
|
|
],
|
|
contradictions: [],
|
|
importantUnknowns: [
|
|
{
|
|
id: "unk1",
|
|
description:
|
|
"Absolute baseline volumes and time period needed to compute complaint rate per unit",
|
|
confidence: "low",
|
|
},
|
|
],
|
|
plausibleInterpretations: [], // intentionally empty — evidence too thin
|
|
},
|
|
evidence: [
|
|
{
|
|
id: "ev1",
|
|
description: "Complaints increased by 35%",
|
|
evidenceType: "reported_statement",
|
|
source: "User input",
|
|
attribution: null,
|
|
confidence: "medium",
|
|
importance: "important",
|
|
},
|
|
{
|
|
id: "ev2",
|
|
description: "Production increased by 40%",
|
|
evidenceType: "reported_statement",
|
|
source: "User input",
|
|
attribution: null,
|
|
confidence: "medium",
|
|
importance: "important",
|
|
},
|
|
{
|
|
id: "ev3",
|
|
description:
|
|
"Production growth rate (40%) exceeded complaint growth rate (35%), implying the denominator may have grown faster than complaints.",
|
|
evidenceType: "inferred_relationship",
|
|
attribution: null,
|
|
confidence: "medium",
|
|
importance: "important",
|
|
},
|
|
],
|
|
nextQuestion: {
|
|
id: "q1",
|
|
question: "What was the complaint rate per unit before and after the production increase?",
|
|
targets: ["system"],
|
|
reason:
|
|
"Without normalising complaints by production volume, the absolute complaint count change is misleading. The rate per unit determines whether the situation improved, stayed stable, or worsened.",
|
|
expectedInformationValue: "high",
|
|
reasoningMode: "decompose_aggregate",
|
|
},
|
|
};
|
|
|
|
const result = reconstructionV2Schema.safeParse(input);
|
|
expect(result.success).toBe(true);
|
|
});
|
|
|
|
it("rejects output missing required fields", () => {
|
|
const input = {
|
|
inputClassification: { primaryType: "other" },
|
|
reconstruction: {},
|
|
evidence: [],
|
|
nextQuestion: { id: "q1" },
|
|
};
|
|
|
|
const result = reconstructionV2Schema.safeParse(input);
|
|
expect(result.success).toBe(false);
|
|
});
|
|
|
|
it("validates empty arrays for all reconstruction categories", () => {
|
|
const input = {
|
|
inputClassification: {
|
|
primaryType: "other",
|
|
classificationReason: "test",
|
|
confidence: "low",
|
|
},
|
|
reconstruction: {
|
|
summary: "empty test",
|
|
actors: [],
|
|
systemsOrObjects: [],
|
|
expectedStates: [],
|
|
observedStates: [],
|
|
differences: [],
|
|
knownTransitions: [],
|
|
unexplainedTransitions: [],
|
|
contradictions: [],
|
|
importantUnknowns: [],
|
|
plausibleInterpretations: [],
|
|
},
|
|
evidence: [],
|
|
nextQuestion: {
|
|
id: "q1",
|
|
question: "What is the production volume?",
|
|
targets: ["system"],
|
|
reason: "need baseline",
|
|
expectedInformationValue: "medium",
|
|
},
|
|
};
|
|
|
|
const result = reconstructionV2Schema.safeParse(input);
|
|
expect(result.success).toBe(true);
|
|
});
|
|
|
|
it("validates evidence distinguishing direct_observation from inferred_relationship", () => {
|
|
const input = {
|
|
inputClassification: {
|
|
primaryType: "unexplained_change",
|
|
classificationReason: "test",
|
|
confidence: "low",
|
|
},
|
|
reconstruction: {
|
|
summary: "test summary",
|
|
actors: [],
|
|
systemsOrObjects: [],
|
|
expectedStates: [],
|
|
observedStates: [{ id: "o1", description: "x", confidence: "high" }],
|
|
differences: [],
|
|
knownTransitions: [],
|
|
unexplainedTransitions: [],
|
|
contradictions: [],
|
|
importantUnknowns: [],
|
|
plausibleInterpretations: [],
|
|
},
|
|
evidence: [
|
|
{
|
|
id: "ev1",
|
|
description: "Observed fact",
|
|
evidenceType: "direct_observation",
|
|
confidence: "high",
|
|
importance: "critical",
|
|
},
|
|
{
|
|
id: "ev2",
|
|
description: "Derived relationship",
|
|
evidenceType: "inferred_relationship",
|
|
confidence: "medium",
|
|
importance: "supporting",
|
|
},
|
|
],
|
|
nextQuestion: {
|
|
id: "q1",
|
|
question: "What is the denominator?",
|
|
targets: ["system"],
|
|
reason: "need context",
|
|
expectedInformationValue: "high",
|
|
},
|
|
};
|
|
|
|
const result = reconstructionV2Schema.safeParse(input);
|
|
expect(result.success).toBe(true);
|
|
});
|
|
});
|
|
|
|
// ──────────────────────────────────────────────
|
|
// parseReconstructionV2 helper tests
|
|
// ──────────────────────────────────────────────
|
|
|
|
describe("parseReconstructionV2", () => {
|
|
it("parses a valid v0.3-shaped JSON string", async () => {
|
|
const fixture = {
|
|
inputClassification: {
|
|
primaryType: "unexplained_change",
|
|
classificationReason: "test",
|
|
confidence: "medium",
|
|
},
|
|
reconstruction: {
|
|
summary: "both increased",
|
|
actors: [],
|
|
systemsOrObjects: [],
|
|
expectedStates: [],
|
|
observedStates: [
|
|
{ id: "o1", description: "x rose 35%", confidence: "high" },
|
|
{ id: "o2", description: "y rose 40%", confidence: "high" },
|
|
],
|
|
differences: [{ id: "d1", description: "y grew faster", confidence: "medium" }],
|
|
knownTransitions: [],
|
|
unexplainedTransitions: [],
|
|
contradictions: [],
|
|
importantUnknowns: [],
|
|
plausibleInterpretations: [],
|
|
},
|
|
evidence: [
|
|
{ id: "e1", description: "x rose 35%", evidenceType: "reported_statement", confidence: "medium", importance: "important" },
|
|
{ id: "e2", description: "y rose 40%", evidenceType: "reported_statement", confidence: "medium", importance: "important" },
|
|
],
|
|
nextQuestion: {
|
|
id: "q1",
|
|
question: "What is the denominator?",
|
|
targets: ["system"],
|
|
reason: "need rate context",
|
|
expectedInformationValue: "high",
|
|
},
|
|
};
|
|
|
|
const raw = JSON.stringify(fixture);
|
|
const parsed = parseReconstructionV2(raw);
|
|
|
|
expect(parsed.inputClassification.primaryType).toBe("unexplained_change");
|
|
expect(parsed.reconstruction.summary).toBe("both increased");
|
|
expect(parsed.nextQuestion.question).toBe("What is the denominator?");
|
|
});
|
|
|
|
it("rejects non-JSON string", () => {
|
|
expect(() => parseReconstructionV2("{not valid json")).toThrow(SyntaxError);
|
|
});
|
|
});
|
|
|
|
// ──────────────────────────────────────────────
|
|
// v0.3 prompt contains required guidance text
|
|
// ──────────────────────────────────────────────
|
|
|
|
describe("v0.3 prompt guidance completeness", () => {
|
|
it("mentions normalise counts when scale changed", async () => {
|
|
const content = await fs.readFile(
|
|
join(PROMPTS_DIR, "reconstruct-v0.3.md"),
|
|
"utf-8",
|
|
);
|
|
expect(content.toLowerCase()).toMatch(/normali[sz]e|normalis[ei]ng/);
|
|
});
|
|
|
|
it("mentions distinguishing total count from rate", async () => {
|
|
const content = await fs.readFile(
|
|
join(PROMPTS_DIR, "reconstruct-v0.3.md"),
|
|
"utf-8",
|
|
);
|
|
expect(content.toLowerCase()).toContain("rate");
|
|
expect(content.toLowerCase()).toMatch(/count.*not.*caus|correlation.*caus|distinguish.*count/);
|
|
});
|
|
|
|
it("mentions avoiding correlation-as-causation", async () => {
|
|
const content = await fs.readFile(
|
|
join(PROMPTS_DIR, "reconstruct-v0.3.md"),
|
|
"utf-8",
|
|
);
|
|
expect(content.toLowerCase()).toMatch(/correlation.*caus|treating.*correlation.*caus/);
|
|
});
|
|
|
|
it("mentions prefer one narrow next question over compound", async () => {
|
|
const content = await fs.readFile(
|
|
join(PROMPTS_DIR, "reconstruct-v0.3.md"),
|
|
"utf-8",
|
|
);
|
|
// Should mention single vs compound
|
|
expect(content).toMatch(/exactly.*one|single.*question|Do NOT combine|combine.*multiple/i);
|
|
});
|
|
|
|
it("mentions leaving empty interpretations when evidence is thin", async () => {
|
|
const content = await fs.readFile(
|
|
join(PROMPTS_DIR, "reconstruct-v0.3.md"),
|
|
"utf-8",
|
|
);
|
|
expect(content).toMatch(/empty.*array|do not generate.*interpretation|fill a list/i);
|
|
});
|
|
|
|
it("mentions identifying the denominator or exposure metric", async () => {
|
|
const content = await fs.readFile(
|
|
join(PROMPTS_DIR, "reconstruct-v0.3.md"),
|
|
"utf-8",
|
|
);
|
|
expect(content.toLowerCase()).toMatch(/denominator|exposure/);
|
|
});
|
|
|
|
it("uses the exact scenario text as a reference example only (not in rules)", async () => {
|
|
const content = await fs.readFile(
|
|
join(PROMPTS_DIR, "reconstruct-v0.3.md"),
|
|
"utf-8",
|
|
);
|
|
// The prompt should be domain-independent — it should not mention specific industries as rules
|
|
// but may have an example section. We verify the prompt does not hard-code a specific question text.
|
|
expect(content).not.toMatch(/What was the complaint rate per unit before and after/);
|
|
});
|
|
});
|
|
|
|
// ──────────────────────────────────────────────
|
|
// Fixture: expected good structure for target scenario
|
|
// ──────────────────────────────────────────────
|
|
|
|
describe("target scenario fixture validation", () => {
|
|
const goodFixture = JSON.parse(JSON.stringify({
|
|
inputClassification: {
|
|
primaryType: "unexplained_change",
|
|
secondaryTypes: ["reported_claim"],
|
|
reasoningModes: ["identify_difference", "decompose_aggregate"],
|
|
classificationReason:
|
|
"Two operational quantities changed at different percentages without a shared baseline or denominator.",
|
|
confidence: "medium",
|
|
},
|
|
reconstruction: {
|
|
summary:
|
|
"Both complaint counts and production volumes increased, but production grew slightly faster than complaints — without absolute baselines the per-unit complaint rate cannot be determined.",
|
|
actors: [],
|
|
systemsOrObjects: [
|
|
{ id: "so1", description: "Production system or output volume", confidence: "high" },
|
|
{ id: "so2", description: "Complaint reporting mechanism", confidence: "high" },
|
|
],
|
|
expectedStates: [],
|
|
observedStates: [
|
|
{ id: "obs1", description: "Complaint count increased by 35%", confidence: "high" },
|
|
{ id: "obs2", description: "Production volume increased by 40%", confidence: "high" },
|
|
],
|
|
differences: [
|
|
{
|
|
id: "diff1",
|
|
description:
|
|
"Production grew faster than complaints (+40% vs +35%), so the ratio of complaints per unit may have decreased or remained stable. The absolute complaint count alone is not a reliable indicator of whether conditions have changed.",
|
|
confidence: "high",
|
|
},
|
|
],
|
|
knownTransitions: [],
|
|
unexplainedTransitions: [
|
|
{
|
|
id: "ut1",
|
|
description: "Complaint volume shifted to a higher level without explained cause",
|
|
confidence: "medium",
|
|
entity: "complaints_metric",
|
|
previousState: "unknown baseline",
|
|
currentState: "+35%",
|
|
},
|
|
],
|
|
contradictions: [],
|
|
importantUnknowns: [
|
|
{
|
|
id: "unk1",
|
|
description:
|
|
"Absolute complaint count and production volume baselines needed to compute the per-unit rate",
|
|
confidence: "low",
|
|
},
|
|
{
|
|
id: "unk2",
|
|
description: "Time period over which these changes occurred",
|
|
confidence: "low",
|
|
},
|
|
],
|
|
plausibleInterpretations: [], // intentionally empty — no sufficient evidence for interpretations
|
|
},
|
|
evidence: [
|
|
{
|
|
id: "ev1",
|
|
description: "Complaints increased by 35%",
|
|
evidenceType: "reported_statement",
|
|
source: "Scenario input",
|
|
attribution: null,
|
|
confidence: "high",
|
|
importance: "important",
|
|
},
|
|
{
|
|
id: "ev2",
|
|
description: "Production increased by 40%",
|
|
evidenceType: "reported_statement",
|
|
source: "Scenario input",
|
|
attribution: null,
|
|
confidence: "high",
|
|
importance: "important",
|
|
},
|
|
{
|
|
id: "ev3",
|
|
description: "Complaint count grew more slowly than production volume, suggesting per-unit rates may have improved or stayed stable.",
|
|
evidenceType: "inferred_relationship",
|
|
attribution: null,
|
|
confidence: "medium",
|
|
importance: "important",
|
|
},
|
|
],
|
|
nextQuestion: {
|
|
id: "q1",
|
|
question: "What was the absolute complaint volume and production volume (or baseline) before these percentage changes?",
|
|
targets: ["system", "measurement"],
|
|
reason:
|
|
"Without baseline counts to compute a rate per unit, we cannot determine whether conditions have worsened, stayed stable, or improved. The rate comparison is the smallest unresolved comparison needed to evaluate the situation.",
|
|
expectedInformationValue: "high",
|
|
reasoningMode: "decompose_aggregate",
|
|
},
|
|
}));
|
|
|
|
it("fixture validates against v0.3 schema", () => {
|
|
const result = reconstructionV2Schema.safeParse(goodFixture);
|
|
expect(result.success).toBe(true);
|
|
});
|
|
|
|
it("fixture has exactly one next question with non-empty text", () => {
|
|
expect(goodFixture.nextQuestion.question.length).toBeGreaterThan(10);
|
|
expect(goodFixture.nextQuestion.reason.length).toBeGreaterThan(10);
|
|
expect(goodFixture.nextQuestion.expectedInformationValue).toBe("high");
|
|
});
|
|
|
|
it("fixture has empty plausibleInterpretations (evidence too thin)", () => {
|
|
expect(goodFixture.reconstruction.plausibleInterpretations).toEqual([]);
|
|
});
|
|
|
|
it("fixture evidence includes both direct observations and one inferred relationship", () => {
|
|
const types = goodFixture.evidence.map((e) => e.evidenceType);
|
|
expect(types).toContain("reported_statement");
|
|
expect(types).toContain("inferred_relationship");
|
|
});
|
|
|
|
it("fixture relationship notes complaint count grew more slowly than production", () => {
|
|
const diffDescs = goodFixture.reconstruction.differences.map((d) => d.description);
|
|
const found = diffDescs.some(
|
|
(d) =>
|
|
d.toLowerCase().includes("fast") ||
|
|
d.toLowerCase().includes("slower") ||
|
|
d.toLowerCase().includes("ratio") ||
|
|
d.toLowerCase().includes("per-unit") ||
|
|
d.toLowerCase().includes("per unit"),
|
|
);
|
|
expect(found).toBe(true);
|
|
});
|
|
|
|
it("fixture does not assert quality deterioration", () => {
|
|
const allText = [
|
|
goodFixture.reconstruction.summary,
|
|
...goodFixture.reconstruction.differences.map((d) => d.description),
|
|
goodFixture.nextQuestion.reason,
|
|
].join(" ").toLowerCase();
|
|
// Should not contain strong deterioration language without caveats
|
|
expect(allText).not.toMatch(/quality.*deteriorat|quality.*worsen|definitely.*bad/);
|
|
});
|
|
|
|
it("fixture includes relationship that production grew faster", () => {
|
|
const allText = [
|
|
goodFixture.reconstruction.summary,
|
|
...goodFixture.reconstruction.differences.map((d) => d.description),
|
|
].join(" ").toLowerCase();
|
|
expect(allText).toMatch(/produ.*grow|ratio|per-unit|per unit|\+40.*\+35/);
|
|
});
|
|
});
|
|
|
|
// ──────────────────────────────────────────────
|
|
// Diagnostics: prompt version tracking
|
|
// ──────────────────────────────────────────────
|
|
|
|
describe("diagnostics prompt version", () => {
|
|
it("DEFAULT_PROMPT_VERSION is exported correctly", () => {
|
|
expect(DEFAULT_PROMPT_VERSION).toBe("v0.3");
|
|
});
|
|
|
|
it("PROMPT_VERSIONS includes both v0.2 and v0.3", () => {
|
|
const hasV2 = PROMPT_VERSIONS.includes("v0.2");
|
|
const hasV3 = PROMPT_VERSIONS.includes("v0.3");
|
|
expect(hasV2).toBe(true);
|
|
expect(hasV3).toBe(true);
|
|
});
|
|
|
|
it("RECONSTRUCTION_PROMPT_VERSION env var overrides default", async () => {
|
|
// The actual override happens at module load time, so we can't easily test this
|
|
// in isolation. Instead, verify the constant reflects env or defaults to v0.3.
|
|
expect(PROMPT_VERSIONS).toContain("v0.2");
|
|
});
|
|
});
|