test: verify ambiguity handling across domains

2026-08-02 16:17:37 +01:00
parent 51ce356218
commit b84989b96a
3 changed files with 370 additions and 0 deletions
@@ -0,0 +1,198 @@
+import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js";
+
+function buildAmbiguityFixture({
+  key,
+  scenario,
+  summaryLabel,
+  contradictionLabel,
+  observationLabels,
+  unknownLabels,
+  disallowedQuestionTerms,
+}) {
+  const summary = makeNode({
+    id: `${key}-summary`,
+    label: summaryLabel,
+    description: "Summary of the situation from the scenario text",
+    kind: "state",
+    status: "provisional",
+    confidence: "medium",
+  });
+
+  const contradiction = makeNode({
+    id: `${key}-contradiction`,
+    label: contradictionLabel,
+    description: contradictionLabel,
+    kind: "relationship",
+    status: "supported",
+    confidence: "medium",
+  });
+
+  const observations = observationLabels.map((label, index) =>
+    makeNode({
+      id: `${key}-obs-${index + 1}`,
+      label,
+      description: label,
+      kind: "observation",
+      status: "supported",
+      confidence: "high",
+    }),
+  );
+
+  const unknowns = unknownLabels.map((label, index) =>
+    makeNode({
+      id: `${key}-unknown-${index + 1}`,
+      label,
+      description: label,
+      kind: "unknown",
+      status: "unknown",
+      confidence: "high",
+    }),
+  );
+
+  const edges = [
+    ...observations.map((node) =>
+      makeEdge({
+        id: `${node.id}-supports-summary`,
+        fromNodeId: node.id,
+        toNodeId: summary.id,
+        relationship: "supports",
+        description: `${node.label} supports the summary.`,
+      }),
+    ),
+    ...unknowns.map((node) =>
+      makeEdge({
+        id: `${node.id}-depends-summary`,
+        fromNodeId: node.id,
+        toNodeId: summary.id,
+        relationship: "depends_on",
+        description: `${node.label} is an unresolved factor for this situation.`,
+      }),
+    ),
+  ];
+
+  return {
+    key,
+    scenario,
+    disallowedQuestionTerms,
+    graph: makeGraph({
+      centralStatement: scenario,
+      nodes: [summary, contradiction, ...observations, ...unknowns],
+      edges,
+      activeUnknownNodeId: null,
+      resolvedNodeIds: [],
+      currentSummary: `Ambiguity fixture for ${key}`,
+    }),
+  };
+}
+
+export const ambiguityGeneralisationFixtures = [
+  buildAmbiguityFixture({
+    key: "revenue-cash",
+    scenario:
+      "Revenue increased by 18%, but cash in the bank fell over the same period.",
+    summaryLabel: "Revenue rose while cash fell",
+    contradictionLabel:
+      "Contradiction between revenue improvement and lower cash reserves.",
+    observationLabels: [
+      "Revenue increased by 18%.",
+      "Cash in the bank decreased over the same period.",
+    ],
+    unknownLabels: [
+      "Possible explanation for the contradiction from one side of the situation.",
+      "Possible explanation for the contradiction from another side of the situation.",
+    ],
+    disallowedQuestionTerms: [
+      "accounts receivable",
+      "capex",
+      "debt repayments",
+      "working capital",
+    ],
+  }),
+  buildAmbiguityFixture({
+    key: "satisfaction-complaints",
+    scenario:
+      "Customer satisfaction scores increased, but complaints also increased.",
+    summaryLabel: "Satisfaction scores rose while complaints also rose",
+    contradictionLabel:
+      "Contradiction between higher satisfaction scores and higher complaint volume.",
+    observationLabels: [
+      "Customer satisfaction scores increased.",
+      "Complaints increased.",
+    ],
+    unknownLabels: [
+      "Possible explanation for why the positive signal and negative signal moved together.",
+      "Another possible explanation for why the positive signal and negative signal moved together.",
+    ],
+    disallowedQuestionTerms: [
+      "net promoter",
+      "ticket backlog",
+      "call deflection",
+      "support queue",
+    ],
+  }),
+  buildAmbiguityFixture({
+    key: "delivery-cancellations",
+    scenario:
+      "Average delivery time decreased by 25%, but order cancellations increased.",
+    summaryLabel: "Delivery became faster while cancellations increased",
+    contradictionLabel:
+      "Contradiction between faster delivery and more order cancellations.",
+    observationLabels: [
+      "Average delivery time decreased by 25%.",
+      "Order cancellations increased.",
+    ],
+    unknownLabels: [
+      "Possible explanation for why the faster result did not reduce the negative result.",
+      "Another possible explanation for why the faster result did not reduce the negative result.",
+    ],
+    disallowedQuestionTerms: [
+      "fulfilment",
+      "last mile",
+      "warehouse",
+      "routing",
+    ],
+  }),
+  buildAmbiguityFixture({
+    key: "traffic-sales",
+    scenario: "Website traffic doubled, but sales remained unchanged.",
+    summaryLabel: "Website traffic doubled while sales stayed flat",
+    contradictionLabel:
+      "Contradiction between much higher traffic and unchanged sales.",
+    observationLabels: [
+      "Website traffic doubled.",
+      "Sales remained unchanged.",
+    ],
+    unknownLabels: [
+      "Possible explanation for why the stronger signal did not change the outcome.",
+      "Another possible explanation for why the stronger signal did not change the outcome.",
+    ],
+    disallowedQuestionTerms: [
+      "conversion funnel",
+      "campaign attribution",
+      "landing page",
+      "checkout flow",
+    ],
+  }),
+  buildAmbiguityFixture({
+    key: "output-defects",
+    scenario:
+      "Production output increased by 30%, but quality defects also increased.",
+    summaryLabel: "Production output rose while defects also rose",
+    contradictionLabel:
+      "Contradiction between higher output and more quality defects.",
+    observationLabels: [
+      "Production output increased by 30%.",
+      "Quality defects increased.",
+    ],
+    unknownLabels: [
+      "Possible explanation for why the gain came with a worsening result.",
+      "Another possible explanation for why the gain came with a worsening result.",
+    ],
+    disallowedQuestionTerms: [
+      "scrap rate",
+      "throughput",
+      "yield",
+      "root cause",
+    ],
+  }),
+];