import { describe, it, expect } from "vitest"; import { reconstructionSchema, confidenceEnum, importanceEnum, inputTypes, reasoningModes, evidenceRecordSchema, reconstructionV2Schema, analyseResponseSchema, parseReconstruction, parseReconstructionV2, } from "@/lib/reconstruction/schema"; import { CONFIDENCE_VALUES } from "@/lib/llm/types.js"; // ────────────────────────────────────────────── // v0.1 — backward compatibility tests // ────────────────────────────────────────────── describe("v0.1 reconstruction schema", () => { it("validates a complete valid reconstruction", () => { const input = { observations: [{ id: "o1", description: "Saw smoke", confidence: "high" }], reportedClaims: [{ id: "rc1", description: "He said the alarm went off", confidence: "medium", attributedTo: "John" }], assumptions: [{ id: "a1", description: "It was a fire", confidence: "low" }], entities: [{ id: "e1", description: "John", confidence: "high" }], transitions: [{ id: "t1", description: "John left the room", confidence: "medium", entity: "John", previousState: "present", currentState: "gone", explanationStatus: "confirmed" }], expectedButMissing: [{ id: "eb1", description: "No one called 911", confidence: "high" }], presentButUnexpected: [{ id: "pb1", description: "The lights were on", confidence: "low" }], contradictions: [{ id: "c1", description: "Said he was home but car is gone", confidence: "medium" }], openUncertainties: [{ id: "ou1", description: "Who was in the room?", confidence: "high" }], }; const result = reconstructionSchema.safeParse(input); expect(result.success).toBe(true); }); it("rejects invalid confidence values", () => { const input = { observations: [{ id: "o1", description: "test", confidence: "extreme" }], reportedClaims: [], assumptions: [], entities: [], transitions: [], expectedButMissing: [], presentButUnexpected: [], contradictions: [], openUncertainties: [], }; const result = reconstructionSchema.safeParse(input); expect(result.success).toBe(false); }); it("rejects missing required fields", () => { const input = { observations: [{ id: "o1" }], reportedClaims: [], assumptions: [], entities: [], transitions: [], expectedButMissing: [], presentButUnexpected: [], contradictions: [], openUncertainties: [], }; const result = reconstructionSchema.safeParse(input); expect(result.success).toBe(false); }); it("rejects invalid confidence values in reportedClaims", () => { const input = { observations: [], reportedClaims: [{ id: "rc1", description: "test", confidence: "very_high", attributedTo: null }], assumptions: [], entities: [], transitions: [], expectedButMissing: [], presentButUnexpected: [], contradictions: [], openUncertainties: [], }; const result = reconstructionSchema.safeParse(input); expect(result.success).toBe(false); }); it("rejects empty transitions", () => { const input = { observations: [], reportedClaims: [], assumptions: [], entities: [], transitions: [{ id: "t1", description: "", confidence: "high", entity: "", previousState: "", currentState: "", explanationStatus: "" }], expectedButMissing: [], presentButUnexpected: [], contradictions: [], openUncertainties: [], }; const result = reconstructionSchema.safeParse(input); expect(result.success).toBe(false); }); it("allows null attributedTo on reported claims", () => { const input = { observations: [], reportedClaims: [{ id: "rc1", description: "Someone called it in", confidence: "medium", attributedTo: null }], assumptions: [], entities: [], transitions: [], expectedButMissing: [], presentButUnexpected: [], contradictions: [], openUncertainties: [], }; const result = reconstructionSchema.safeParse(input); expect(result.success).toBe(true); }); }); // ────────────────────────────────────────────── // v0.2 — schema validation tests // ────────────────────────────────────────────── describe("v0.2 input classification", () => { it.each([ "observed_problem", "unexplained_change", "contradiction", "decision_request", "causal_claim", "reported_claim", "fault_report", "ambiguous_statement", "question", "desired_outcome", "insufficient_context", "other", ])("validates input type '%s'", (type) => { const result = inputTypes.safeParse(type); expect(result.success).toBe(true); }); it("rejects invalid input types", () => { expect(inputTypes.safeParse("invalid_type").success).toBe(false); expect(inputTypes.safeParse("").success).toBe(false); expect(inputTypes.safeParse(null).success).toBe(false); }); it("validates reasoning modes", () => { const modes = [ "establish_baseline", "identify_difference", "reconstruct_transition", "decompose_aggregate", "validate_measurement", "validate_claim", "investigate_contradiction", "clarify_meaning", "decision_support", "fault_investigation", "identify_missing_information", "test_possible_explanations", "other", ]; for (const m of modes) { const result = reasoningModes.safeParse(m); expect(result.success).toBe(true); } }); it("rejects invalid reasoning mode", () => { expect(reasoningModes.safeParse("no_op").success).toBe(false); }); }); describe("v0.2 multiple secondary types and reasoning modes", () => { it("validates classification with multiple secondary types", () => { const classification = { primaryType: "observed_problem", secondaryTypes: ["fault_report", "decision_request"], reasoningModes: ["validate_claim", "identify_missing_information"], classificationReason: "Test scenario with multiple classifications", confidence: "high", }; const result = reconstructionV2Schema.safeParse({ inputClassification: classification, reconstruction: { summary: "test", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [], }, evidence: [{ id: "e1", description: "test", evidenceType: "direct_observation", confidence: "high", importance: "supporting" }], nextQuestion: { id: "q1", question: "Test?", targets: ["x"], reason: "r", expectedInformationValue: "medium", reasoningMode: "other", }, }); expect(result.success).toBe(true); }); it("validates with single secondary type", () => { const classification = { primaryType: "unexplained_change", secondaryTypes: ["observed_problem"], reasoningModes: ["establish_baseline"], classificationReason: "Single secondary", confidence: "medium", }; const result = reconstructionV2Schema.safeParse({ inputClassification: classification, reconstruction: { summary: "x", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [] }, evidence: [{ id: "e1", description: "test", evidenceType: "direct_observation", confidence: "medium", importance: "supporting" }], nextQuestion: { id: "q1", question: "Test?", targets: ["x"], reason: "r", expectedInformationValue: "low", reasoningMode: "other" }, }); expect(result.success).toBe(true); }); it("validates with multiple reasoning modes", () => { const classification = { primaryType: "contradiction", secondaryTypes: [], reasoningModes: ["investigate_contradiction", "identify_difference", "validate_claim"], classificationReason: "Multiple reasoning modes applicable", confidence: "high", }; const result = reconstructionV2Schema.safeParse({ inputClassification: classification, reconstruction: { summary: "x", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [] }, evidence: [{ id: "e1", description: "test", evidenceType: "direct_observation", confidence: "high", importance: "important" }], nextQuestion: { id: "q1", question: "Test?", targets: ["x"], reason: "r", expectedInformationValue: "high", reasoningMode: "investigate_contradiction" }, }); expect(result.success).toBe(true); }); }); describe("v0.2 evidence records", () => { it.each([ "direct_observation", "reported_statement", "interpretation", "assumption", "inferred_relationship", ])("validates evidence type '%s'", (eType) => { const result = evidenceRecordSchema.safeParse({ id: "e1", description: "test", evidenceType: eType, confidence: "high", importance: "supporting", }); expect(result.success).toBe(true); }); it("rejects invalid evidence type", () => { const result = evidenceRecordSchema.safeParse({ id: "e1", description: "test", evidenceType: "unknown_type", confidence: "high", importance: "supporting", }); expect(result.success).toBe(false); }); it("allows null attribution", () => { const result = evidenceRecordSchema.safeParse({ id: "e1", description: "test", evidenceType: "reported_statement", attribution: null, confidence: "medium", importance: "incidental", }); expect(result.success).toBe(true); }); it("requires source or attribution optional but not both mandatory", () => { const result = evidenceRecordSchema.safeParse({ id: "e1", description: "test", evidenceType: "direct_observation", confidence: "high", importance: "critical", }); expect(result.success).toBe(true); // source and attribution are optional }); }); describe("v0.2 invalid confidence and importance values", () => { it.each(["very_high", "extreme", "low_medium", "", "null"])( "invalid confidence '%s' rejected", (val) => { const result = confidenceEnum.safeParse(val); expect(result.success).toBe(false); } ); it("valid confidence values accepted", () => { for (const v of ["low", "medium", "high"]) { const result = confidenceEnum.safeParse(v); expect(result.success).toBe(true); } }); it.each(["very_high", "extreme", "low_medium", "", "critical_plus"])( "invalid importance '%s' rejected", (val) => { const result = evidenceRecordSchema.safeParse({ id: "e1", description: "test", evidenceType: "direct_observation", confidence: "high", importance: val, }); expect(result.success).toBe(false); } ); it.each(["incidental", "supporting", "important", "critical"])( "valid importance '%s' accepted", (val) => { const result = evidenceRecordSchema.safeParse({ id: "e1", description: "test", evidenceType: "direct_observation", confidence: "high", importance: val, }); expect(result.success).toBe(true); } ); }); describe("v0.2 plausible interpretations", () => { it("validates reconstruction with multiple plausible interpretations", () => { const result = reconstructionV2Schema.safeParse({ inputClassification: { primaryType: "decision_support", secondaryTypes: [], reasoningModes: [], classificationReason: "Multiple interpretations possible.", confidence: "medium", }, reconstruction: { summary: "The situation has two competing explanations.", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [ { id: "pi1", description: "The issue is caused by configuration drift", supportingEvidenceIds: ["e1", "e3"], assumptionsRequired: ["config_history_is_incomplete"], confidence: "medium", }, { id: "pi2", description: "The issue stems from upstream dependency failure", supportingEvidenceIds: ["e2"], assumptionsRequired: ["dependency_outage_at_same_time"], confidence: "low", }, ], }, evidence: [{ id: "e1", description: "Config changed on Tuesday", evidenceType: "direct_observation", confidence: "high", importance: "supporting" }], nextQuestion: { id: "q1", question: "What changed between Monday and Tuesday?", targets: ["timeline"], reason: "To distinguish between drift and dependency failure.", expectedInformationValue: "high", reasoningMode: "reconstruct_transition" }, }); expect(result.success).toBe(true); }); it("allows interpretation with empty assumptionsRequired", () => { const result = reconstructionV2Schema.safeParse({ inputClassification: { primaryType: "other", secondaryTypes: [], reasoningModes: [], classificationReason: "test", confidence: "low" }, reconstruction: { summary: "x", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [{ id: "pi1", description: "Plain interpretation", supportingEvidenceIds: ["e1"], confidence: "low" }] }, evidence: [], nextQuestion: { id: "q1", question: "?", targets: [], reason: "r", expectedInformationValue: "low", reasoningMode: "other" }, }); expect(result.success).toBe(true); }); }); describe("v0.2 exactly one next question", () => { it("validates when exactly one next question is present", () => { const result = reconstructionV2Schema.safeParse({ inputClassification: { primaryType: "observed_problem", secondaryTypes: [], reasoningModes: [], classificationReason: "test", confidence: "high" }, reconstruction: { summary: "x", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [] }, evidence: [], nextQuestion: { id: "q1", question: "What is the baseline?", targets: ["baseline"], reason: "r", expectedInformationValue: "high", reasoningMode: "establish_baseline" }, }); expect(result.success).toBe(true); }); it("validates when nextQuestion is absent (schema allows optional)", () => { const result = reconstructionV2Schema.safeParse({ inputClassification: { primaryType: "ambiguous_statement", secondaryTypes: [], reasoningModes: [], classificationReason: "No question possible.", confidence: "low" }, reconstruction: { summary: "x", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [] }, evidence: [], nextQuestion: undefined, }); // The schema allows missing nextQuestion (optional), so this should pass validation. // We validate exactly-one at the evaluator level, not in the schema. expect(result.success).toBe(true); }); it("rejects reconstructionV2 when required fields are missing", () => { const result = reconstructionV2Schema.safeParse({}); expect(result.success).toBe(false); }); }); describe("parseReconstruction (v0.1)", () => { it("parses a raw JSON string", () => { const raw = JSON.stringify({ observations: [{ id: "o1", description: "test", confidence: "high" }], reportedClaims: [], assumptions: [], entities: [], transitions: [], expectedButMissing: [], presentButUnexpected: [], contradictions: [], openUncertainties: [], }); const result = parseReconstruction(raw); expect(result.observations[0].id).toBe("o1"); }); it("rejects malformed JSON string", () => { expect(() => parseReconstruction("{invalid json")).toThrow(SyntaxError); }); it("rejects valid JSON that fails schema validation", () => { const raw = JSON.stringify({ observations: [{ id: "o1", description: "test", confidence: "extreme" }], reportedClaims: [], assumptions: [], entities: [], transitions: [], expectedButMissing: [], presentButUnexpected: [], contradictions: [], openUncertainties: [], }); expect(() => parseReconstruction(raw)).toThrow(); }); it("accepts an already-parsed object", () => { const obj = { observations: [{ id: "o1", description: "test", confidence: "high" }], reportedClaims: [], assumptions: [], entities: [], transitions: [], expectedButMissing: [], presentButUnexpected: [], contradictions: [], openUncertainties: [], }; const result = parseReconstruction(obj); expect(result.observations[0].id).toBe("o1"); }); }); describe("parseReconstructionV2", () => { it("parses a raw JSON v0.2 string", () => { const raw = JSON.stringify({ inputClassification: { primaryType: "observed_problem", secondaryTypes: [], reasoningModes: [], classificationReason: "test", confidence: "high" }, reconstruction: { summary: "x", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [] }, evidence: [{ id: "e1", description: "test", evidenceType: "direct_observation", confidence: "high", importance: "supporting" }], nextQuestion: { id: "q1", question: "Test?", targets: ["x"], reason: "r", expectedInformationValue: "medium", reasoningMode: "other" }, }); const result = parseReconstructionV2(raw); expect(result.inputClassification.primaryType).toBe("observed_problem"); }); it("rejects malformed JSON string", () => { expect(() => parseReconstructionV2("{invalid json")).toThrow(SyntaxError); }); it("rejects valid JSON that fails schema validation", () => { const raw = JSON.stringify({ not: "the right structure" }); expect(() => parseReconstructionV2(raw)).toThrow(); }); it("accepts an already-parsed v0.2 object", () => { const obj = { inputClassification: { primaryType: "observed_problem", secondaryTypes: [], reasoningModes: [], classificationReason: "test", confidence: "high" }, reconstruction: { summary: "x", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [] }, evidence: [], nextQuestion: undefined, }; const result = parseReconstructionV2(obj); expect(result.inputClassification.primaryType).toBe("observed_problem"); }); }); describe("malformed model output", () => { it("throws on non-JSON string", () => { expect(() => parseReconstruction("hello world")).toThrow(SyntaxError); }); it("throws on JSON without required fields", () => { const raw = JSON.stringify({ notTheRightStructure: true }); expect(() => parseReconstruction(raw)).toThrow(); }); it("handles empty arrays for all v0.1 categories", () => { const result = parseReconstruction({ observations: [], reportedClaims: [], assumptions: [], entities: [], transitions: [], expectedButMissing: [], presentButUnexpected: [], contradictions: [], openUncertainties: [], }); expect(result.observations.length).toBe(0); }); }); // ────────────────────────────────────────────── // v0.2 full reconstruction validation // ────────────────────────────────────────────── describe("v0.2 complete valid reconstruction", () => { it("validates a full v0.2 output with all sections", () => { const result = reconstructionV2Schema.safeParse({ inputClassification: { primaryType: "observed_problem", secondaryTypes: ["fault_report"], reasoningModes: ["validate_claim", "identify_difference"], classificationReason: "Clear operational issue identified.", confidence: "high" }, reconstruction: { summary: "A fault report with subset scope affecting specific users.", actors: [{ id: "a1", description: "Affected user group", confidence: "medium" }], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [{ id: "d1", description: "Subset vs universal access", confidence: "high" }], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [{ id: "u1", description: "Root cause of access failure", confidence: "medium" }], plausibleInterpretations: [], }, evidence: [{ id: "e1", description: "User reports confirm the issue.", evidenceType: "reported_statement", source: "support tickets", confidence: "high", importance: "important" }], nextQuestion: { id: "q1", question: "Which specific users are affected?", targets: ["user_segment"], reason: "Narrow scope to identify pattern.", expectedInformationValue: "high", reasoningMode: "validate_claim" }, }); expect(result.success).toBe(true); }); it("allows null source in evidence", () => { const result = reconstructionV2Schema.safeParse({ inputClassification: { primaryType: "other", secondaryTypes: [], reasoningModes: [], classificationReason: "test", confidence: "low" }, reconstruction: { summary: "x", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [] }, evidence: [{ id: "e1", description: "test", evidenceType: "direct_observation", attribution: null, confidence: "low", importance: "incidental" }], nextQuestion: undefined, }); expect(result.success).toBe(true); }); it("requires all critical importance values for evidence", () => { const result = reconstructionV2Schema.safeParse({ inputClassification: { primaryType: "other", secondaryTypes: [], reasoningModes: [], classificationReason: "test", confidence: "low" }, reconstruction: { summary: "x", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [] }, evidence: [{ id: "e1", description: "test", evidenceType: "direct_observation", confidence: "high", importance: "critical" }], nextQuestion: { id: "q1", question: "?", targets: ["x"], reason: "r", expectedInformationValue: "low", reasoningMode: "other" }, }); expect(result.success).toBe(true); // critical importance is valid }); }); describe("empty scenario rejection", () => { it("rejects empty string", () => { const trimmed = "".trim(); expect(trimmed.length).toBe(0); }); it("rejects whitespace-only string", () => { const trimmed = " \n\t ".trim(); expect(trimmed.length).toBe(0); }); }); // ────────────────────────────────────────────── // Provider parsing tests // ────────────────────────────────────────────── describe("provider response parsing", () => { it("handles raw JSON object response", () => { const parsed = parseReconstruction({ observations: [], reportedClaims: [{ id: "rc1", description: "he said", confidence: "medium", attributedTo: "Alice" }], assumptions: [], entities: [], transitions: [], expectedButMissing: [], presentButUnexpected: [], contradictions: [], openUncertainties: [], }); expect(parsed.reportedClaims[0].attributedTo).toBe("Alice"); }); it("handles v0.2 parsed reconstruction", () => { const parsed = parseReconstructionV2({ inputClassification: { primaryType: "observed_problem", secondaryTypes: [], reasoningModes: [], classificationReason: "test", confidence: "high" }, reconstruction: { summary: "x", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [{ id: "d1", description: "delta", confidence: "high" }], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [] }, evidence: [{ id: "e1", description: "test", evidenceType: "direct_observation", confidence: "high", importance: "supporting" }], nextQuestion: { id: "q1", question: "?", targets: ["x"], reason: "r", expectedInformationValue: "medium", reasoningMode: "other" }, }); expect(parsed.inputClassification.primaryType).toBe("observed_problem"); }); }); // ────────────────────────────────────────────── // Deterministic evaluator scoring tests // ────────────────────────────────────────────── describe("deterministic evaluator scoring", () => { function normalise(text) { return String(text).toLowerCase().replace(/[^\w\s_]/g, " ").replace(/\s+/g, " ").trim(); } function checkPrimaryTypeMatch(actualPrimary, expectedTypes) { if (!actualPrimary || !expectedTypes?.length) return false; const actual = String(actualPrimary).toLowerCase().replace(/\s+/g, "_"); return expectedTypes.some((t) => t.toLowerCase().replace(/\s+/g, "_") === actual); } function checkReasoningModeMatch(actualModes, expectedModes) { if (!actualModes?.length || !expectedModes?.length) return false; const actual = actualModes.map((m) => String(m).toLowerCase().replace(/\s+/g, "_")); const expected = expectedModes.map((m) => String(m).toLowerCase().replace(/\s+/g, "_")); return expected.some((e) => actual.includes(e)); } it("matches primary type when exact", () => { expect(checkPrimaryTypeMatch("observed_problem", ["observed_problem"])).toBe(true); }); it("does not match when primary type differs", () => { expect(checkPrimaryTypeMatch("unexplained_change", ["observed_problem"])).toBe(false); }); it("matches when primary type is in list of expected types", () => { expect(checkPrimaryTypeMatch("observed_problem", ["observed_problem", "fault_report"])).toBe(true); expect(checkPrimaryTypeMatch("unexplained_change", ["observed_problem", "fault_report"])).toBe(false); }); it("matches reasoning mode when present in list", () => { expect(checkReasoningModeMatch(["establish_baseline", "identify_difference"], ["establish_baseline"])).toBe(true); }); it("does not match reasoning mode when absent", () => { expect(checkReasoningModeMatch(["validate_claim"], ["establish_baseline"])).toBe(false); }); it("handles empty lists gracefully", () => { expect(checkPrimaryTypeMatch(null, [])).toBe(false); expect(checkPrimaryTypeMatch("observed_problem", [])).toBe(false); expect(checkReasoningModeMatch([], ["establish_baseline"])).toBe(false); }); it("normalises whitespace in comparison", () => { expect(normalise("hello world")).toBe("hello world"); expect(normalise("Test_With-Symbols!")).toBe("test_with_symbols"); }); }); // ────────────────────────────────────────────── // Paired test case loading // ────────────────────────────────────────────── describe("paired test cases", () => { const pairedTests = [ { id: "p1a", input: "All customers cannot download invoices.", expectedPrimaryTypes: ["observed_problem"], notes: "Universal scope" }, { id: "p1b", input: "Some customers cannot download invoices.", expectedPrimaryTypes: ["observed_problem"], notes: "Subset scope — key difference from p1a" }, { id: "p2a", input: "Complaints increased by 35%.", expectedPrimaryTypes: ["unexplained_change"], notes: "Isolated metric change" }, { id: "p2b", input: "Complaints increased by 35% while production increased by 40%.", expectedPrimaryTypes: ["unexplained_change"], notes: "Context changes significance" }, { id: "p3a", input: "Sales are falling.", expectedPrimaryTypes: ["observed_problem"], notes: "Vague claim" }, { id: "p3b", input: "Sales fell sharply immediately after the price increase.", expectedPrimaryTypes: ["causal_claim"], notes: "Adds temporal anchor and cause" }, { id: "p4a", input: "I think therefore I am.", expectedPrimaryTypes: ["ambiguous_statement"], notes: "Philosophical statement" }, { id: "p4b", input: "I used the phrase 'I think therefore I am' to test whether this system understands ambiguous statements.", expectedPrimaryTypes: ["question"], notes: "Meta-context changes classification" }, ]; it.each(pairedTests)("paired test '%s' loads correctly", (tc) => { expect(tc.id).toBeDefined(); expect(tc.input.length).toBeGreaterThan(0); expect(Array.isArray(tc.expectedPrimaryTypes)).toBe(true); expect(tc.notes.length).toBeGreaterThan(0); }); it("has meaningful differences between paired test A and B inputs", () => { const p1a = pairedTests.find((t) => t.id === "p1a"); const p1b = pairedTests.find((t) => t.id === "p1b"); expect(p1a.input).toContain("All customers"); expect(p1b.input).toContain("Some customers"); }); it("has at least 8 test cases covering different classification types", () => { const coveredTypes = new Set(pairedTests.map((tc) => tc.expectedPrimaryTypes[0])); expect(coveredTypes.size).toBeGreaterThanOrEqual(4); // at least 4 different types }); }); // ────────────────────────────────────────────── // Confidence and importance value validation // ────────────────────────────────────────────── describe("confidence and importance enums", () => { it("has exactly three confidence values: low, medium, high", () => { const validConfidences = ["low", "medium", "high"]; for (const c of validConfidences) { expect(confidenceEnum.safeParse(c).success).toBe(true); } // CONFIDENCE_VALUES should match expect(CONFIDENCE_VALUES).toEqual(["low", "medium", "high"]); }); it("has exactly four importance values", () => { const validImportances = ["incidental", "supporting", "important", "critical"]; for (const imp of validImportances) { expect(evidenceRecordSchema.safeParse({ id: "x", description: "y", evidenceType: "direct_observation", confidence: "high", importance: imp }).success).toBe(true); } }); it("rejects values outside the defined enums", () => { expect(confidenceEnum.safeParse("very_high").success).toBe(false); expect(evidenceRecordSchema.safeParse({ id: "x", description: "y", evidenceType: "direct_observation", confidence: "high", importance: "critical_plus" }).success).toBe(false); }); }); // ────────────────────────────────────────────── // Missing next question test // ────────────────────────────────────────────── describe("missing next question handling", () => { it("schema allows optional nextQuestion for ambiguous inputs", () => { const result = reconstructionV2Schema.safeParse({ inputClassification: { primaryType: "ambiguous_statement", secondaryTypes: [], reasoningModes: [], classificationReason: "Cannot ask meaningful question.", confidence: "low" }, reconstruction: { summary: "Ambiguous philosophical statement detected.", actors: [], systemsOrObjects: [], expectedStates: [], observedStates: [], differences: [], knownTransitions: [], unexplainedTransitions: [], contradictions: [], importantUnknowns: [], plausibleInterpretations: [] }, evidence: [], nextQuestion: undefined, }); expect(result.success).toBe(true); }); it("schema rejects missing required fields", () => { const result = reconstructionV2Schema.safeParse({}); expect(result.success).toBe(false); }); }); // ────────────────────────────────────────────── // Mock evaluation run test // ────────────────────────────────────────────── describe("mock evaluation", () => { function normalise(text) { return String(text).toLowerCase().replace(/[^\w\s_]/g, " ").replace(/\s+/g, " ").trim(); } it("mock provider can generate deterministic v0.2 output", async () => { // Test that the evaluator's mock provider produces valid schema output const mockInput = "All customers cannot download invoices."; // The normaliser should work correctly const normed = normalise(mockInput); expect(normed).toContain("customers"); expect(normed).toContain("invoices"); }); it("mock evaluation logic produces expected classification for 'all' vs 'some'", () => { // Verify the evaluator's mock logic handles the key distinction const allInput = "All customers cannot download invoices."; const someInput = "Some customers cannot download invoices."; const hasAllWord = /\ball\b|\bno one\b|\bevery\b/i.test(allInput); const hasSomeWord = /some\b/i.test(someInput); expect(hasAllWord).toBe(true); expect(hasSomeWord).toBe(true); }); });