import { describe, it, expect } from "vitest"; import { promises as fs } from "node:fs"; import { fileURLToPath } from "node:url"; import { dirname, join } from "node:path"; import { PROMPT_VERSIONS, buildPrompt, DEFAULT_PROMPT_VERSION, } from "@/lib/reconstruction/prompt.js"; import { reconstructionV2Schema, parseReconstructionV2, } from "@/lib/reconstruction/schema.js"; const __filename = fileURLToPath(import.meta.url); const __dirname = dirname(__filename); const PROMPTS_DIR = join(__dirname, "../prompts"); // ────────────────────────────────────────────── // v0.3 prompt loading tests // ────────────────────────────────────────────── describe("v0.3 prompt", () => { it("v0.3 is in PROMPT_VERSIONS", () => { expect(PROMPT_VERSIONS).toContain("v0.3"); }); it("DEFAULT_PROMPT_VERSION is v0.3 on this branch", () => { expect(DEFAULT_PROMPT_VERSION).toBe("v0.3"); }); it("v0.2 remains available in PROMPT_VERSIONS", () => { expect(PROMPT_VERSIONS).toContain("v0.2"); }); it("v0.3 prompt file loads from disk", async () => { const content = await fs.readFile( join(PROMPTS_DIR, "reconstruct-v0.3.md"), "utf-8", ); expect(typeof content).toBe("string"); expect(content.length).toBeGreaterThan(500); }); it("v0.3 prompt contains normalisation guidance", async () => { const content = await fs.readFile( join(PROMPTS_DIR, "reconstruct-v0.3.md"), "utf-8", ); expect(content.toLowerCase()).toContain("normalise"); expect(content.toLowerCase()).toContain("rate"); expect(content.toLowerCase()).toContain("denominator") || expect(content.toLowerCase()).toContain("exposure"); }); it("v0.3 prompt contains discipline guidance", async () => { const content = await fs.readFile( join(PROMPTS_DIR, "reconstruct-v0.3.md"), "utf-8", ); // Should mention not generating speculative interpretations expect(content).toMatch(/interpretation/i); // Should mention one question discipline expect(content).toMatch(/exactly.*one.*question|one.*only.*question|single.*question/i) || expect(content).toMatch(/Do NOT combine/i); }); it("buildPrompt returns v0.3 prompt with scenario substituted", async () => { const result = await buildPrompt("Test scenario text", "v0.3"); expect(result.version).toBe("v0.3"); expect(result.prompt).toContain("Test scenario text"); // Should contain the normalisation section guidance expect(result.prompt.toLowerCase()).toContain("normalise"); }); it("buildPrompt returns v0.2 prompt when requested", async () => { const result = await buildPrompt("Test scenario text", "v0.2"); expect(result.version).toBe("v0.2"); expect(result.prompt).toContain("Test scenario text"); }); it("buildPrompt default is v0.3", async () => { const result = await buildPrompt("Test scenario text"); expect(result.version).toBe("v0.3"); }); }); // ────────────────────────────────────────────── // v0.2 prompt still works // ────────────────────────────────────────────── describe("v0.2 backward compatibility", () => { it("v0.2 prompt file exists and loads", async () => { const content = await fs.readFile( join(PROMPTS_DIR, "reconstruct-v0.2.md"), "utf-8", ); expect(typeof content).toBe("string"); expect(content.length).toBeGreaterThan(500); }); it("buildPrompt returns v0.2 version string", async () => { const result = await buildPrompt("test", "v0.2"); expect(result.version).toBe("v0.2"); }); }); // ────────────────────────────────────────────── // Schema validation tests for v0.3-shaped output // ────────────────────────────────────────────── describe("v0.3 schema validation", () => { it("validates a complete valid reconstruction with empty interpretations", () => { const input = { inputClassification: { primaryType: "unexplained_change", secondaryTypes: ["reported_claim"], reasoningModes: ["identify_difference"], classificationReason: "Two metrics changed without explanation.", confidence: "medium", }, reconstruction: { summary: "Both complaints and production increased.", actors: [], systemsOrObjects: [ { id: "complaints_metric", description: "Volume of complaints", confidence: "high" }, ], expectedStates: [], observedStates: [ { id: "obs1", description: "Complaint volume rose by 35%", confidence: "medium" }, { id: "obs2", description: "Production volume rose by 40%", confidence: "medium" }, ], differences: [ { id: "diff1", description: "Production grew faster than complaints, so the complaint-to-production ratio may have improved.", confidence: "medium", }, ], knownTransitions: [], unexplainedTransitions: [ { id: "trans1", description: "Complaint volume shifted to a higher level without explained cause", confidence: "medium", entity: "complaints_metric", previousState: "Baseline volume (unknown)", currentState: "+35% increase", }, ], contradictions: [], importantUnknowns: [ { id: "unk1", description: "Absolute baseline volumes and time period needed to compute complaint rate per unit", confidence: "low", }, ], plausibleInterpretations: [], // intentionally empty — evidence too thin }, evidence: [ { id: "ev1", description: "Complaints increased by 35%", evidenceType: "reported_statement", source: "User input", attribution: null, confidence: "medium", importance: "important", }, { id: "ev2", description: "Production increased by 40%", evidenceType: "reported_statement", source: "User input", attribution: null, confidence: "medium", importance: "important", }, { id: "ev3", description: "Production growth rate (40%) exceeded complaint growth rate (35%), implying the denominator may have grown faster than complaints.", evidenceType: "inferred_relationship", attribution: null, confidence: "medium", importance: "important", }, ], nextQuestion: { id: "q1", question: "What was the complaint rate per unit before and after the production increase?", targets: ["system"], reason: "Without normalising complaints by production volume, the absolute complaint count change is misleading. The rate per unit determines whether the situation improved, stayed stable, or worsened.", expectedInformationValue: "high", reasoningMode: "decompose_aggregate", }, }; const result = reconstructionV2Schema.safeParse(input); expect(result.success).toBe(true); }); it("rejects output missing required fields", () => { const input = { inputClassification: { primaryType: "other" }, reconstruction: {}, evidence: [], nextQuestion: { id: "q1" }, }; const result = reconstructionV2Schema.safeParse(input); expect(result.success).toBe(false); }); it("validates empty arrays for all reconstruction categories", () => { const input = { inputClassification: { primaryType: "other", classificationReason: "test", confidence: "low", }, reconstruction: { summary: "empty test", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [], }, evidence: [], nextQuestion: { id: "q1", question: "What is the production volume?", targets: ["system"], reason: "need baseline", expectedInformationValue: "medium", }, }; const result = reconstructionV2Schema.safeParse(input); expect(result.success).toBe(true); }); it("validates evidence distinguishing direct_observation from inferred_relationship", () => { const input = { inputClassification: { primaryType: "unexplained_change", classificationReason: "test", confidence: "low", }, reconstruction: { summary: "test summary", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [{ id: "o1", description: "x", confidence: "high" }], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [], }, evidence: [ { id: "ev1", description: "Observed fact", evidenceType: "direct_observation", confidence: "high", importance: "critical", }, { id: "ev2", description: "Derived relationship", evidenceType: "inferred_relationship", confidence: "medium", importance: "supporting", }, ], nextQuestion: { id: "q1", question: "What is the denominator?", targets: ["system"], reason: "need context", expectedInformationValue: "high", }, }; const result = reconstructionV2Schema.safeParse(input); expect(result.success).toBe(true); }); }); // ────────────────────────────────────────────── // parseReconstructionV2 helper tests // ────────────────────────────────────────────── describe("parseReconstructionV2", () => { it("parses a valid v0.3-shaped JSON string", async () => { const fixture = { inputClassification: { primaryType: "unexplained_change", classificationReason: "test", confidence: "medium", }, reconstruction: { summary: "both increased", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [ { id: "o1", description: "x rose 35%", confidence: "high" }, { id: "o2", description: "y rose 40%", confidence: "high" }, ], differences: [{ id: "d1", description: "y grew faster", confidence: "medium" }], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [], }, evidence: [ { id: "e1", description: "x rose 35%", evidenceType: "reported_statement", confidence: "medium", importance: "important" }, { id: "e2", description: "y rose 40%", evidenceType: "reported_statement", confidence: "medium", importance: "important" }, ], nextQuestion: { id: "q1", question: "What is the denominator?", targets: ["system"], reason: "need rate context", expectedInformationValue: "high", }, }; const raw = JSON.stringify(fixture); const parsed = parseReconstructionV2(raw); expect(parsed.inputClassification.primaryType).toBe("unexplained_change"); expect(parsed.reconstruction.summary).toBe("both increased"); expect(parsed.nextQuestion.question).toBe("What is the denominator?"); }); it("rejects non-JSON string", () => { expect(() => parseReconstructionV2("{not valid json")).toThrow(SyntaxError); }); }); // ────────────────────────────────────────────── // v0.3 prompt contains required guidance text // ────────────────────────────────────────────── describe("v0.3 prompt guidance completeness", () => { it("mentions normalise counts when scale changed", async () => { const content = await fs.readFile( join(PROMPTS_DIR, "reconstruct-v0.3.md"), "utf-8", ); expect(content.toLowerCase()).toMatch(/normali[sz]e|normalis[ei]ng/); }); it("mentions distinguishing total count from rate", async () => { const content = await fs.readFile( join(PROMPTS_DIR, "reconstruct-v0.3.md"), "utf-8", ); expect(content.toLowerCase()).toContain("rate"); expect(content.toLowerCase()).toMatch(/count.*not.*caus|correlation.*caus|distinguish.*count/); }); it("mentions avoiding correlation-as-causation", async () => { const content = await fs.readFile( join(PROMPTS_DIR, "reconstruct-v0.3.md"), "utf-8", ); expect(content.toLowerCase()).toMatch(/correlation.*caus|treating.*correlation.*caus/); }); it("mentions prefer one narrow next question over compound", async () => { const content = await fs.readFile( join(PROMPTS_DIR, "reconstruct-v0.3.md"), "utf-8", ); // Should mention single vs compound expect(content).toMatch(/exactly.*one|single.*question|Do NOT combine|combine.*multiple/i); }); it("mentions leaving empty interpretations when evidence is thin", async () => { const content = await fs.readFile( join(PROMPTS_DIR, "reconstruct-v0.3.md"), "utf-8", ); expect(content).toMatch(/empty.*array|do not generate.*interpretation|fill a list/i); }); it("mentions identifying the denominator or exposure metric", async () => { const content = await fs.readFile( join(PROMPTS_DIR, "reconstruct-v0.3.md"), "utf-8", ); expect(content.toLowerCase()).toMatch(/denominator|exposure/); }); it("uses the exact scenario text as a reference example only (not in rules)", async () => { const content = await fs.readFile( join(PROMPTS_DIR, "reconstruct-v0.3.md"), "utf-8", ); // The prompt should be domain-independent — it should not mention specific industries as rules // but may have an example section. We verify the prompt does not hard-code a specific question text. expect(content).not.toMatch(/What was the complaint rate per unit before and after/); }); }); // ────────────────────────────────────────────── // Fixture: expected good structure for target scenario // ────────────────────────────────────────────── describe("target scenario fixture validation", () => { const goodFixture = JSON.parse(JSON.stringify({ inputClassification: { primaryType: "unexplained_change", secondaryTypes: ["reported_claim"], reasoningModes: ["identify_difference", "decompose_aggregate"], classificationReason: "Two operational quantities changed at different percentages without a shared baseline or denominator.", confidence: "medium", }, reconstruction: { summary: "Both complaint counts and production volumes increased, but production grew slightly faster than complaints — without absolute baselines the per-unit complaint rate cannot be determined.", actors: [], systemsOrObjects: [ { id: "so1", description: "Production system or output volume", confidence: "high" }, { id: "so2", description: "Complaint reporting mechanism", confidence: "high" }, ], expectedStates: [], observedStates: [ { id: "obs1", description: "Complaint count increased by 35%", confidence: "high" }, { id: "obs2", description: "Production volume increased by 40%", confidence: "high" }, ], differences: [ { id: "diff1", description: "Production grew faster than complaints (+40% vs +35%), so the ratio of complaints per unit may have decreased or remained stable. The absolute complaint count alone is not a reliable indicator of whether conditions have changed.", confidence: "high", }, ], knownTransitions: [], unexplainedTransitions: [ { id: "ut1", description: "Complaint volume shifted to a higher level without explained cause", confidence: "medium", entity: "complaints_metric", previousState: "unknown baseline", currentState: "+35%", }, ], contradictions: [], importantUnknowns: [ { id: "unk1", description: "Absolute complaint count and production volume baselines needed to compute the per-unit rate", confidence: "low", }, { id: "unk2", description: "Time period over which these changes occurred", confidence: "low", }, ], plausibleInterpretations: [], // intentionally empty — no sufficient evidence for interpretations }, evidence: [ { id: "ev1", description: "Complaints increased by 35%", evidenceType: "reported_statement", source: "Scenario input", attribution: null, confidence: "high", importance: "important", }, { id: "ev2", description: "Production increased by 40%", evidenceType: "reported_statement", source: "Scenario input", attribution: null, confidence: "high", importance: "important", }, { id: "ev3", description: "Complaint count grew more slowly than production volume, suggesting per-unit rates may have improved or stayed stable.", evidenceType: "inferred_relationship", attribution: null, confidence: "medium", importance: "important", }, ], nextQuestion: { id: "q1", question: "What was the absolute complaint volume and production volume (or baseline) before these percentage changes?", targets: ["system", "measurement"], reason: "Without baseline counts to compute a rate per unit, we cannot determine whether conditions have worsened, stayed stable, or improved. The rate comparison is the smallest unresolved comparison needed to evaluate the situation.", expectedInformationValue: "high", reasoningMode: "decompose_aggregate", }, })); it("fixture validates against v0.3 schema", () => { const result = reconstructionV2Schema.safeParse(goodFixture); expect(result.success).toBe(true); }); it("fixture has exactly one next question with non-empty text", () => { expect(goodFixture.nextQuestion.question.length).toBeGreaterThan(10); expect(goodFixture.nextQuestion.reason.length).toBeGreaterThan(10); expect(goodFixture.nextQuestion.expectedInformationValue).toBe("high"); }); it("fixture has empty plausibleInterpretations (evidence too thin)", () => { expect(goodFixture.reconstruction.plausibleInterpretations).toEqual([]); }); it("fixture evidence includes both direct observations and one inferred relationship", () => { const types = goodFixture.evidence.map((e) => e.evidenceType); expect(types).toContain("reported_statement"); expect(types).toContain("inferred_relationship"); }); it("fixture relationship notes complaint count grew more slowly than production", () => { const diffDescs = goodFixture.reconstruction.differences.map((d) => d.description); const found = diffDescs.some( (d) => d.toLowerCase().includes("fast") || d.toLowerCase().includes("slower") || d.toLowerCase().includes("ratio") || d.toLowerCase().includes("per-unit") || d.toLowerCase().includes("per unit"), ); expect(found).toBe(true); }); it("fixture does not assert quality deterioration", () => { const allText = [ goodFixture.reconstruction.summary, ...goodFixture.reconstruction.differences.map((d) => d.description), goodFixture.nextQuestion.reason, ].join(" ").toLowerCase(); // Should not contain strong deterioration language without caveats expect(allText).not.toMatch(/quality.*deteriorat|quality.*worsen|definitely.*bad/); }); it("fixture includes relationship that production grew faster", () => { const allText = [ goodFixture.reconstruction.summary, ...goodFixture.reconstruction.differences.map((d) => d.description), ].join(" ").toLowerCase(); expect(allText).toMatch(/produ.*grow|ratio|per-unit|per unit|\+40.*\+35/); }); }); // ────────────────────────────────────────────── // Diagnostics: prompt version tracking // ────────────────────────────────────────────── describe("diagnostics prompt version", () => { it("DEFAULT_PROMPT_VERSION is exported correctly", () => { expect(DEFAULT_PROMPT_VERSION).toBe("v0.3"); }); it("PROMPT_VERSIONS includes both v0.2 and v0.3", () => { const hasV2 = PROMPT_VERSIONS.includes("v0.2"); const hasV3 = PROMPT_VERSIONS.includes("v0.3"); expect(hasV2).toBe(true); expect(hasV3).toBe(true); }); it("RECONSTRUCTION_PROMPT_VERSION env var overrides default", async () => { // The actual override happens at module load time, so we can't easily test this // in isolation. Instead, verify the constant reflects env or defaults to v0.3. expect(PROMPT_VERSIONS).toContain("v0.2"); }); });