feat: add mock-mode docs and additional e2e tests (long investigation, recovery states)

2026-08-05 08:38:44 +01:00
parent 61210c1200
commit 592962325d
4 changed files with 988 additions and 0 deletions
@@ -0,0 +1,255 @@
+/**
+ * Investigation scenarios — shared fixture data for all E2E Playwright tests.
+ *
+ * Each scenario defines:
+ *   - name / centralStatement (the text entered into the scenario textarea)
+ *   - mockMode          (which mock mode to activate: "complete" | "error" | "" )
+ *   - answerSequence    [ { text, expectedHeadingAfterTurn } ]
+ *   - expectedTerminalState (final card heading)
+ *   - screenshots       [ { label, expectedVisible } ]
+ *
+ * The test scripts drive the application through these scenarios using the
+ * Developer-details scenario selector (UI mock mode), or by setting env vars.
+ */
+
+/* ── Shared constants ─────────────────────────────────────── */
+
+const BASE_SCENARIO_TEXT =
+  "Complaints increased by 35% while production increased by 40%.";
+
+/* ── Scenario definitions ─────────────────────────────────── */
+
+/**
+ * Multi-turn investigation with at least 2 turns.
+ * Uses the default sequential-turn mock (no special scenario flag).
+ */
+const happyPathComparison = {
+  name: "Happy path — multi-turn comparison",
+  centralStatement:
+    "Product A has a 4.2 star average rating while Product B averages 4.6 stars across 10,000+ reviews each.",
+  mockMode: "",                     // "" → sequential turns via default mock
+  turnCount: 3,
+  answerSequence: [
+    {
+      text: "Yes, both are standard 5-star scales.",
+      expectedHeadingAfterTurn: "Current investigation",
+    },
+    {
+      text: "The gap persists across verified purchase reviews.",
+      expectedHeadingAfterTurn: "Current investigation",
+    },
+  ],
+  terminalState: null,               // does not reach a terminal state (unknowns remain)
+  screenshots: [
+    { label: "initial-state", afterAction: "before-start" },
+    { label: "after-first-turn", afterAction: "after-answer-1" },
+  ],
+};
+
+/**
+ * Complete investigation — all unknowns resolved.
+ * Uses the "complete" mock mode (jumps to terminal after start).
+ */
+const happyPathComplete = {
+  name: "Happy path — complete investigation",
+  centralStatement: BASE_SCENARIO_TEXT,
+  mockMode: "complete",
+  turnCount: 1,                      // start gives turn 0, first update → final state
+  answerSequence: [
+    {
+      text: "The complaint rate fell from 2.0 per 100 to 1.9 per 100 units.",
+      expectedHeadingAfterTurn: "Investigation complete",
+    },
+  ],
+  terminalState: "Investigation complete",
+  screenshots: [
+    { label: "initial-state", afterAction: "before-start" },
+    { label: "terminal-state", afterAction: "after-answer-1" },
+  ],
+};
+
+/**
+ * Evidence-limit state — unknowns remain but no next question.
+ */
+const evidenceLimit = {
+  name: "Evidence limit (stuck early)",
+  centralStatement:
+    "Should a mid-sized manufacturing company invest in automated quality inspection?",
+  mockMode: "",                      // single-turn default scenario
+  turnCount: 1,
+  answerSequence: [],                // no answers needed — stuck after start
+  terminalState: "Current evidence limit reached",
+  screenshots: [
+    { label: "initial-state", afterAction: "before-start" },
+    { label: "terminal-state", afterAction: "after-start" },
+  ],
+};
+
+/**
+ * Provider error — all calls return a structured provider error.
+ */
+const providerError = {
+  name: "Provider error",
+  centralStatement: BASE_SCENARIO_TEXT,
+  mockMode: "error",
+  turnCount: 0,                      // start call also errors
+  answerSequence: [],
+  terminalState: null,
+  screenshots: [
+    { label: "initial-state", afterAction: "before-start" },
+    { label: "terminal-state", afterAction: "after-start" },
+  ],
+};
+
+/**
+ * Malformed response — the start produces a malformed shape.
+ * (We simulate this via error mode which skips all structured output.)
+ */
+const malformedResponse = {
+  name: "Malformed response",
+  centralStatement: BASE_SCENARIO_TEXT,
+  mockMode: "error",                 // reuses error mode for malformed path
+  turnCount: 0,
+  answerSequence: [],
+  terminalState: null,
+  screenshots: [
+    { label: "initial-state", afterAction: "before-start" },
+    { label: "terminal-state", afterAction: "after-start" },
+  ],
+};
+
+/**
+ * Long investigation — market entry, 5 turns.
+ */
+const longInvestigation = {
+  name: "Long investigation — European market entry",
+  centralStatement:
+    "Should we enter the European market with our SaaS analytics platform?",
+  mockMode: "",                      // sequential turns from scenario data (long array)
+  turnCount: 5,
+  answerSequence: [
+    { text: "The market is valued at approximately €8B and growing.", expectedHeadingAfterTurn: "Current investigation" },
+    { text: "Our platform does not currently support EU data residency.", expectedHeadingAfterTurn: "Current investigation" },
+    { text: "Achieving compliance would take 6 months and $500K engineering investment.", expectedHeadingAfterTurn: "Current investigation" },
+    { text: "Our real-time collaboration feature has no direct European equivalent.", expectedHeadingAfterTurn: "Investigation complete", isLast: true },
+  ],
+  terminalState: "Investigation complete",
+  screenshots: [
+    { label: "initial-state", afterAction: "before-start" },
+    { label: "mid-investigation", afterAction: "after-answer-2" },
+    { label: "terminal-state", afterAction: "after-answer-4" },
+  ],
+};
+
+/**
+ * Contradiction fixture — two opposing recommendations.
+ */
+const contradictionFixture = {
+  name: "Contradiction fixture",
+  centralStatement:
+    "Two consultants provided opposite recommendations about whether to outsource IT operations.",
+  mockMode: "",
+  turnCount: 2,
+  answerSequence: [
+    { text: "Consultant A focused on cost over 2 years; Consultant B focused on quality over 5+ years.", expectedHeadingAfterTurn: "Current investigation" },
+  ],
+  terminalState: null,
+  screenshots: [
+    { label: "initial-state", afterAction: "before-start" },
+    { label: "after-answer-1", afterAction: "after-answer-1" },
+  ],
+};
+
+/**
+ * Diagnosis fixture — churn diagnosis.
+ */
+const diagnosisFixture = {
+  name: "Diagnosis fixture",
+  centralStatement:
+    "Customer churn increased from 2% to 5% monthly over the last quarter.",
+  mockMode: "",
+  turnCount: 1,
+  answerSequence: [],
+  terminalState: null,
+  screenshots: [
+    { label: "initial-state", afterAction: "before-start" },
+    { label: "terminal-state", afterAction: "after-start" },
+  ],
+};
+
+/**
+ * Comparison fixture — product ratings with 3 turns.
+ */
+const comparisonFixture = {
+  name: "Comparison fixture",
+  centralStatement:
+    "Product A has a 4.2 star average rating while Product B averages 4.6 stars across 10,000+ reviews each.",
+  mockMode: "",
+  turnCount: 3,
+  answerSequence: [
+    { text: "Both use the standard 5-star customer review scale.", expectedHeadingAfterTurn: "Current investigation" },
+    { text: "Verified purchase gap remains approximately 0.3 stars.", expectedHeadingAfterTurn: "Current investigation" },
+  ],
+  terminalState: null,
+  screenshots: [
+    { label: "initial-state", afterAction: "before-start" },
+    { label: "after-answer-1", afterAction: "after-answer-1" },
+  ],
+};
+
+/**
+ * Prioritisation fixture — decision investigation.
+ */
+const prioritisationFixture = {
+  name: "Prioritisation fixture (team relocation)",
+  centralStatement:
+    "Should I relocate my engineering team from London to Manchester?",
+  mockMode: "",
+  turnCount: 2,
+  answerSequence: [
+    { text: "Manchester has a growing tech ecosystem with 500+ roles posted monthly.", expectedHeadingAfterTurn: "Current investigation" },
+  ],
+  terminalState: null,
+  screenshots: [
+    { label: "initial-state", afterAction: "before-start" },
+    { label: "after-answer-1", afterAction: "after-answer-1" },
+  ],
+};
+
+/* ── Registry for dynamic test generation ─────────────────── */
+
+export const INVESTIGATION_SCENARIOS = [
+  happyPathComparison,
+  happyPathComplete,
+  evidenceLimit,
+  providerError,
+  malformedResponse,
+  longInvestigation,
+  contradictionFixture,
+  diagnosisFixture,
+  comparisonFixture,
+  prioritisationFixture,
+];
+
+export const SCENARIO_LOOKUP = {};
+INVESTIGATION_SCENARIOS.forEach((s) => {
+  SCENARIO_LOOKUP[s.name] = s;
+});
+
+/**
+ * Helper to get the answer text for a given turn index.
+ * Returns null if there is no answer for that turn.
+ */
+export function getAnswerForTurn(scenario, turnIndex) {
+  return scenario.answerSequence[turnIndex]?.text || null;
+}
+
+/**
+ * Return the list of screenshots in execution order.
+ */
+export function getScreenshotsOrdered(screenshotList) {
+  // Preserve insertion order from scenario definition
+  return screenshotList;
+}
+
+export default INVESTIGATION_SCENARIOS;