test: verify ambiguity handling across domains

docs/v0.6-ambiguity-generalisation.md

@@ -0,0 +1,40 @@
+# v0.6 Ambiguity Generalisation
+
+## Hypothesis
+
+If the selector truly handles unjustified contradiction ties generically, it should return ambiguity across multiple domains without preferring one explanation by wording alone.
+
+## Scenarios
+
+1. Revenue increased by 18%, but cash in the bank fell over the same period.
+2. Customer satisfaction scores increased, but complaints also increased.
+3. Average delivery time decreased by 25%, but order cancellations increased.
+4. Website traffic doubled, but sales remained unchanged.
+5. Production output increased by 30%, but quality defects also increased.
+
+## Observed behaviour
+
+All five fixtures produced the same pattern:
+
+- candidate count: 2
+- selector status: `ambiguous`
+- tie reason: `No justified distinction between leading unknowns.`
+- no explanation was favoured
+- one broad investigation question was produced from the central contradiction
+- neutral label renaming did not collapse ambiguity into a winner
+
+## Repeated failure patterns
+
+None observed across two or more scenarios.
+
+The current ambiguity handling generalised cleanly across the five contradiction fixtures.
+
+## Corrections
+
+No production correction was required in this task.
+
+## Lessons learned
+
+- The current ambiguity path appears domain-agnostic when structure and semantic weights remain intentionally non-discriminating.
+- Central-statement-based tie questions are broad enough to avoid prematurely backing one branch.
+- The most useful regression signal is whether ambiguity survives neutral relabelling, not whether one label sorts ahead of another in display order.

tests/fixtures/ambiguity-generalisation.js

@@ -0,0 +1,198 @@
+import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js";
+
+function buildAmbiguityFixture({
+  key,
+  scenario,
+  summaryLabel,
+  contradictionLabel,
+  observationLabels,
+  unknownLabels,
+  disallowedQuestionTerms,
+}) {
+  const summary = makeNode({
+    id: `${key}-summary`,
+    label: summaryLabel,
+    description: "Summary of the situation from the scenario text",
+    kind: "state",
+    status: "provisional",
+    confidence: "medium",
+  });
+
+  const contradiction = makeNode({
+    id: `${key}-contradiction`,
+    label: contradictionLabel,
+    description: contradictionLabel,
+    kind: "relationship",
+    status: "supported",
+    confidence: "medium",
+  });
+
+  const observations = observationLabels.map((label, index) =>
+    makeNode({
+      id: `${key}-obs-${index + 1}`,
+      label,
+      description: label,
+      kind: "observation",
+      status: "supported",
+      confidence: "high",
+    }),
+  );
+
+  const unknowns = unknownLabels.map((label, index) =>
+    makeNode({
+      id: `${key}-unknown-${index + 1}`,
+      label,
+      description: label,
+      kind: "unknown",
+      status: "unknown",
+      confidence: "high",
+    }),
+  );
+
+  const edges = [
+    ...observations.map((node) =>
+      makeEdge({
+        id: `${node.id}-supports-summary`,
+        fromNodeId: node.id,
+        toNodeId: summary.id,
+        relationship: "supports",
+        description: `${node.label} supports the summary.`,
+      }),
+    ),
+    ...unknowns.map((node) =>
+      makeEdge({
+        id: `${node.id}-depends-summary`,
+        fromNodeId: node.id,
+        toNodeId: summary.id,
+        relationship: "depends_on",
+        description: `${node.label} is an unresolved factor for this situation.`,
+      }),
+    ),
+  ];
+
+  return {
+    key,
+    scenario,
+    disallowedQuestionTerms,
+    graph: makeGraph({
+      centralStatement: scenario,
+      nodes: [summary, contradiction, ...observations, ...unknowns],
+      edges,
+      activeUnknownNodeId: null,
+      resolvedNodeIds: [],
+      currentSummary: `Ambiguity fixture for ${key}`,
+    }),
+  };
+}
+
+export const ambiguityGeneralisationFixtures = [
+  buildAmbiguityFixture({
+    key: "revenue-cash",
+    scenario:
+      "Revenue increased by 18%, but cash in the bank fell over the same period.",
+    summaryLabel: "Revenue rose while cash fell",
+    contradictionLabel:
+      "Contradiction between revenue improvement and lower cash reserves.",
+    observationLabels: [
+      "Revenue increased by 18%.",
+      "Cash in the bank decreased over the same period.",
+    ],
+    unknownLabels: [
+      "Possible explanation for the contradiction from one side of the situation.",
+      "Possible explanation for the contradiction from another side of the situation.",
+    ],
+    disallowedQuestionTerms: [
+      "accounts receivable",
+      "capex",
+      "debt repayments",
+      "working capital",
+    ],
+  }),
+  buildAmbiguityFixture({
+    key: "satisfaction-complaints",
+    scenario:
+      "Customer satisfaction scores increased, but complaints also increased.",
+    summaryLabel: "Satisfaction scores rose while complaints also rose",
+    contradictionLabel:
+      "Contradiction between higher satisfaction scores and higher complaint volume.",
+    observationLabels: [
+      "Customer satisfaction scores increased.",
+      "Complaints increased.",
+    ],
+    unknownLabels: [
+      "Possible explanation for why the positive signal and negative signal moved together.",
+      "Another possible explanation for why the positive signal and negative signal moved together.",
+    ],
+    disallowedQuestionTerms: [
+      "net promoter",
+      "ticket backlog",
+      "call deflection",
+      "support queue",
+    ],
+  }),
+  buildAmbiguityFixture({
+    key: "delivery-cancellations",
+    scenario:
+      "Average delivery time decreased by 25%, but order cancellations increased.",
+    summaryLabel: "Delivery became faster while cancellations increased",
+    contradictionLabel:
+      "Contradiction between faster delivery and more order cancellations.",
+    observationLabels: [
+      "Average delivery time decreased by 25%.",
+      "Order cancellations increased.",
+    ],
+    unknownLabels: [
+      "Possible explanation for why the faster result did not reduce the negative result.",
+      "Another possible explanation for why the faster result did not reduce the negative result.",
+    ],
+    disallowedQuestionTerms: [
+      "fulfilment",
+      "last mile",
+      "warehouse",
+      "routing",
+    ],
+  }),
+  buildAmbiguityFixture({
+    key: "traffic-sales",
+    scenario: "Website traffic doubled, but sales remained unchanged.",
+    summaryLabel: "Website traffic doubled while sales stayed flat",
+    contradictionLabel:
+      "Contradiction between much higher traffic and unchanged sales.",
+    observationLabels: [
+      "Website traffic doubled.",
+      "Sales remained unchanged.",
+    ],
+    unknownLabels: [
+      "Possible explanation for why the stronger signal did not change the outcome.",
+      "Another possible explanation for why the stronger signal did not change the outcome.",
+    ],
+    disallowedQuestionTerms: [
+      "conversion funnel",
+      "campaign attribution",
+      "landing page",
+      "checkout flow",
+    ],
+  }),
+  buildAmbiguityFixture({
+    key: "output-defects",
+    scenario:
+      "Production output increased by 30%, but quality defects also increased.",
+    summaryLabel: "Production output rose while defects also rose",
+    contradictionLabel:
+      "Contradiction between higher output and more quality defects.",
+    observationLabels: [
+      "Production output increased by 30%.",
+      "Quality defects increased.",
+    ],
+    unknownLabels: [
+      "Possible explanation for why the gain came with a worsening result.",
+      "Another possible explanation for why the gain came with a worsening result.",
+    ],
+    disallowedQuestionTerms: [
+      "scrap rate",
+      "throughput",
+      "yield",
+      "root cause",
+    ],
+  }),
+];

tests/graph/ambiguity-generalisation.test.js

@@ -0,0 +1,132 @@
+import { describe, expect, it } from "vitest";
+import {
+  formulateQuestion,
+  formulateTieResolutionQuestion,
+} from "@/lib/graph/question-formulator.js";
+import {
+  explainUnknownSelection,
+  selectActiveUnknownCandidate,
+} from "@/lib/graph/utils.js";
+import { ambiguityGeneralisationFixtures } from "@/tests/fixtures/ambiguity-generalisation.js";
+
+function neutraliseUnknownLabels(graph) {
+  let counter = 0;
+  return {
+    ...graph,
+    nodes: graph.nodes.map((node) => {
+      if (node.kind !== "unknown") return { ...node };
+      counter += 1;
+      return {
+        ...node,
+        label: `Unknown ${String.fromCharCode(64 + counter)}`,
+        description: `Unknown factor ${counter}.`,
+      };
+    }),
+  };
+}
+
+function isSingleQuestion(question) {
+  return (question.match(/\?/g) || []).length === 1;
+}
+
+describe("ambiguity generalisation", () => {
+  it("preserves ambiguity across contradiction scenarios without favouring one explanation", () => {
+    const summary = ambiguityGeneralisationFixtures.map((fixture) => {
+      const explanation = explainUnknownSelection(fixture.graph, []);
+      const selection = selectActiveUnknownCandidate(fixture.graph, []);
+      const neutralExplanation = explainUnknownSelection(
+        neutraliseUnknownLabels(fixture.graph),
+        [],
+      );
+      const tieQuestion = formulateTieResolutionQuestion({
+        graph: fixture.graph,
+      });
+      const representativeUnknown = fixture.graph.nodes.find(
+        (node) => node.kind === "unknown",
+      );
+      const fallbackQuestion = formulateQuestion({
+        node: representativeUnknown,
+        graph: fixture.graph,
+      });
+
+      const lowerQuestion = tieQuestion.question.toLowerCase();
+      for (const term of fixture.disallowedQuestionTerms) {
+        expect(lowerQuestion).not.toContain(term.toLowerCase());
+      }
+
+      expect(explanation.status).toBe("ambiguous");
+      expect(selection.status).toBe("ambiguous");
+      expect(selection.selectedNode).toBeNull();
+      expect(explanation.selectedNodeId).toBeNull();
+      expect(explanation.candidates).toHaveLength(2);
+      expect(explanation.summary.selectedReason).toBe(
+        "No justified distinction between leading unknowns.",
+      );
+      expect(explanation.alphabeticalUsedAsReasoning).toBe(false);
+      expect(neutralExplanation.status).toBe("ambiguous");
+      expect(isSingleQuestion(tieQuestion.question)).toBe(true);
+      expect(tieQuestion.question.toLowerCase()).not.toContain(" and ");
+      expect(tieQuestion.question.toLowerCase()).not.toContain(" or ");
+
+      return {
+        scenario: fixture.scenario,
+        candidateCount: explanation.candidates.length,
+        ambiguityStatus: explanation.status,
+        tieReason: explanation.summary.selectedReason,
+        investigationStrategy: tieQuestion.strategy,
+        question: tieQuestion.question,
+        explanationFavoured: explanation.selectedNodeId !== null,
+      };
+    });
+
+    expect(summary).toMatchInlineSnapshot(`
+      [
+        {
+          "ambiguityStatus": "ambiguous",
+          "candidateCount": 2,
+          "explanationFavoured": false,
+          "investigationStrategy": null,
+          "question": "What changed during the period that could explain why Revenue increased by 18%, but cash in the bank fell over the same period?",
+          "scenario": "Revenue increased by 18%, but cash in the bank fell over the same period.",
+          "tieReason": "No justified distinction between leading unknowns.",
+        },
+        {
+          "ambiguityStatus": "ambiguous",
+          "candidateCount": 2,
+          "explanationFavoured": false,
+          "investigationStrategy": null,
+          "question": "What changed during the period that could explain why Customer satisfaction scores increased, but complaints also increased?",
+          "scenario": "Customer satisfaction scores increased, but complaints also increased.",
+          "tieReason": "No justified distinction between leading unknowns.",
+        },
+        {
+          "ambiguityStatus": "ambiguous",
+          "candidateCount": 2,
+          "explanationFavoured": false,
+          "investigationStrategy": null,
+          "question": "What changed during the period that could explain why Average delivery time decreased by 25%, but order cancellations increased?",
+          "scenario": "Average delivery time decreased by 25%, but order cancellations increased.",
+          "tieReason": "No justified distinction between leading unknowns.",
+        },
+        {
+          "ambiguityStatus": "ambiguous",
+          "candidateCount": 2,
+          "explanationFavoured": false,
+          "investigationStrategy": null,
+          "question": "What changed during the period that could explain why Website traffic doubled, but sales remained unchanged?",
+          "scenario": "Website traffic doubled, but sales remained unchanged.",
+          "tieReason": "No justified distinction between leading unknowns.",
+        },
+        {
+          "ambiguityStatus": "ambiguous",
+          "candidateCount": 2,
+          "explanationFavoured": false,
+          "investigationStrategy": null,
+          "question": "What changed during the period that could explain why Production output increased by 30%, but quality defects also increased?",
+          "scenario": "Production output increased by 30%, but quality defects also increased.",
+          "tieReason": "No justified distinction between leading unknowns.",
+        },
+      ]
+    `);
+  });
+});