diff --git a/docs/v0.6-ambiguity-generalisation.md b/docs/v0.6-ambiguity-generalisation.md new file mode 100644 index 0000000..6ae698e --- /dev/null +++ b/docs/v0.6-ambiguity-generalisation.md @@ -0,0 +1,40 @@ +# v0.6 Ambiguity Generalisation + +## Hypothesis + +If the selector truly handles unjustified contradiction ties generically, it should return ambiguity across multiple domains without preferring one explanation by wording alone. + +## Scenarios + +1. Revenue increased by 18%, but cash in the bank fell over the same period. +2. Customer satisfaction scores increased, but complaints also increased. +3. Average delivery time decreased by 25%, but order cancellations increased. +4. Website traffic doubled, but sales remained unchanged. +5. Production output increased by 30%, but quality defects also increased. + +## Observed behaviour + +All five fixtures produced the same pattern: + +- candidate count: 2 +- selector status: `ambiguous` +- tie reason: `No justified distinction between leading unknowns.` +- no explanation was favoured +- one broad investigation question was produced from the central contradiction +- neutral label renaming did not collapse ambiguity into a winner + +## Repeated failure patterns + +None observed across two or more scenarios. + +The current ambiguity handling generalised cleanly across the five contradiction fixtures. + +## Corrections + +No production correction was required in this task. + +## Lessons learned + +- The current ambiguity path appears domain-agnostic when structure and semantic weights remain intentionally non-discriminating. +- Central-statement-based tie questions are broad enough to avoid prematurely backing one branch. +- The most useful regression signal is whether ambiguity survives neutral relabelling, not whether one label sorts ahead of another in display order. diff --git a/tests/fixtures/ambiguity-generalisation.js b/tests/fixtures/ambiguity-generalisation.js new file mode 100644 index 0000000..af3ce14 --- /dev/null +++ b/tests/fixtures/ambiguity-generalisation.js @@ -0,0 +1,198 @@ +import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js"; + +function buildAmbiguityFixture({ + key, + scenario, + summaryLabel, + contradictionLabel, + observationLabels, + unknownLabels, + disallowedQuestionTerms, +}) { + const summary = makeNode({ + id: `${key}-summary`, + label: summaryLabel, + description: "Summary of the situation from the scenario text", + kind: "state", + status: "provisional", + confidence: "medium", + }); + + const contradiction = makeNode({ + id: `${key}-contradiction`, + label: contradictionLabel, + description: contradictionLabel, + kind: "relationship", + status: "supported", + confidence: "medium", + }); + + const observations = observationLabels.map((label, index) => + makeNode({ + id: `${key}-obs-${index + 1}`, + label, + description: label, + kind: "observation", + status: "supported", + confidence: "high", + }), + ); + + const unknowns = unknownLabels.map((label, index) => + makeNode({ + id: `${key}-unknown-${index + 1}`, + label, + description: label, + kind: "unknown", + status: "unknown", + confidence: "high", + }), + ); + + const edges = [ + ...observations.map((node) => + makeEdge({ + id: `${node.id}-supports-summary`, + fromNodeId: node.id, + toNodeId: summary.id, + relationship: "supports", + description: `${node.label} supports the summary.`, + }), + ), + ...unknowns.map((node) => + makeEdge({ + id: `${node.id}-depends-summary`, + fromNodeId: node.id, + toNodeId: summary.id, + relationship: "depends_on", + description: `${node.label} is an unresolved factor for this situation.`, + }), + ), + ]; + + return { + key, + scenario, + disallowedQuestionTerms, + graph: makeGraph({ + centralStatement: scenario, + nodes: [summary, contradiction, ...observations, ...unknowns], + edges, + activeUnknownNodeId: null, + resolvedNodeIds: [], + currentSummary: `Ambiguity fixture for ${key}`, + }), + }; +} + +export const ambiguityGeneralisationFixtures = [ + buildAmbiguityFixture({ + key: "revenue-cash", + scenario: + "Revenue increased by 18%, but cash in the bank fell over the same period.", + summaryLabel: "Revenue rose while cash fell", + contradictionLabel: + "Contradiction between revenue improvement and lower cash reserves.", + observationLabels: [ + "Revenue increased by 18%.", + "Cash in the bank decreased over the same period.", + ], + unknownLabels: [ + "Possible explanation for the contradiction from one side of the situation.", + "Possible explanation for the contradiction from another side of the situation.", + ], + disallowedQuestionTerms: [ + "accounts receivable", + "capex", + "debt repayments", + "working capital", + ], + }), + buildAmbiguityFixture({ + key: "satisfaction-complaints", + scenario: + "Customer satisfaction scores increased, but complaints also increased.", + summaryLabel: "Satisfaction scores rose while complaints also rose", + contradictionLabel: + "Contradiction between higher satisfaction scores and higher complaint volume.", + observationLabels: [ + "Customer satisfaction scores increased.", + "Complaints increased.", + ], + unknownLabels: [ + "Possible explanation for why the positive signal and negative signal moved together.", + "Another possible explanation for why the positive signal and negative signal moved together.", + ], + disallowedQuestionTerms: [ + "net promoter", + "ticket backlog", + "call deflection", + "support queue", + ], + }), + buildAmbiguityFixture({ + key: "delivery-cancellations", + scenario: + "Average delivery time decreased by 25%, but order cancellations increased.", + summaryLabel: "Delivery became faster while cancellations increased", + contradictionLabel: + "Contradiction between faster delivery and more order cancellations.", + observationLabels: [ + "Average delivery time decreased by 25%.", + "Order cancellations increased.", + ], + unknownLabels: [ + "Possible explanation for why the faster result did not reduce the negative result.", + "Another possible explanation for why the faster result did not reduce the negative result.", + ], + disallowedQuestionTerms: [ + "fulfilment", + "last mile", + "warehouse", + "routing", + ], + }), + buildAmbiguityFixture({ + key: "traffic-sales", + scenario: "Website traffic doubled, but sales remained unchanged.", + summaryLabel: "Website traffic doubled while sales stayed flat", + contradictionLabel: + "Contradiction between much higher traffic and unchanged sales.", + observationLabels: [ + "Website traffic doubled.", + "Sales remained unchanged.", + ], + unknownLabels: [ + "Possible explanation for why the stronger signal did not change the outcome.", + "Another possible explanation for why the stronger signal did not change the outcome.", + ], + disallowedQuestionTerms: [ + "conversion funnel", + "campaign attribution", + "landing page", + "checkout flow", + ], + }), + buildAmbiguityFixture({ + key: "output-defects", + scenario: + "Production output increased by 30%, but quality defects also increased.", + summaryLabel: "Production output rose while defects also rose", + contradictionLabel: + "Contradiction between higher output and more quality defects.", + observationLabels: [ + "Production output increased by 30%.", + "Quality defects increased.", + ], + unknownLabels: [ + "Possible explanation for why the gain came with a worsening result.", + "Another possible explanation for why the gain came with a worsening result.", + ], + disallowedQuestionTerms: [ + "scrap rate", + "throughput", + "yield", + "root cause", + ], + }), +]; diff --git a/tests/graph/ambiguity-generalisation.test.js b/tests/graph/ambiguity-generalisation.test.js new file mode 100644 index 0000000..d31f46b --- /dev/null +++ b/tests/graph/ambiguity-generalisation.test.js @@ -0,0 +1,132 @@ +import { describe, expect, it } from "vitest"; +import { + formulateQuestion, + formulateTieResolutionQuestion, +} from "@/lib/graph/question-formulator.js"; +import { + explainUnknownSelection, + selectActiveUnknownCandidate, +} from "@/lib/graph/utils.js"; +import { ambiguityGeneralisationFixtures } from "@/tests/fixtures/ambiguity-generalisation.js"; + +function neutraliseUnknownLabels(graph) { + let counter = 0; + return { + ...graph, + nodes: graph.nodes.map((node) => { + if (node.kind !== "unknown") return { ...node }; + counter += 1; + return { + ...node, + label: `Unknown ${String.fromCharCode(64 + counter)}`, + description: `Unknown factor ${counter}.`, + }; + }), + }; +} + +function isSingleQuestion(question) { + return (question.match(/\?/g) || []).length === 1; +} + +describe("ambiguity generalisation", () => { + it("preserves ambiguity across contradiction scenarios without favouring one explanation", () => { + const summary = ambiguityGeneralisationFixtures.map((fixture) => { + const explanation = explainUnknownSelection(fixture.graph, []); + const selection = selectActiveUnknownCandidate(fixture.graph, []); + const neutralExplanation = explainUnknownSelection( + neutraliseUnknownLabels(fixture.graph), + [], + ); + const tieQuestion = formulateTieResolutionQuestion({ + graph: fixture.graph, + }); + const representativeUnknown = fixture.graph.nodes.find( + (node) => node.kind === "unknown", + ); + const fallbackQuestion = formulateQuestion({ + node: representativeUnknown, + graph: fixture.graph, + }); + + const lowerQuestion = tieQuestion.question.toLowerCase(); + for (const term of fixture.disallowedQuestionTerms) { + expect(lowerQuestion).not.toContain(term.toLowerCase()); + } + + expect(explanation.status).toBe("ambiguous"); + expect(selection.status).toBe("ambiguous"); + expect(selection.selectedNode).toBeNull(); + expect(explanation.selectedNodeId).toBeNull(); + expect(explanation.candidates).toHaveLength(2); + expect(explanation.summary.selectedReason).toBe( + "No justified distinction between leading unknowns.", + ); + expect(explanation.alphabeticalUsedAsReasoning).toBe(false); + expect(neutralExplanation.status).toBe("ambiguous"); + expect(isSingleQuestion(tieQuestion.question)).toBe(true); + expect(tieQuestion.question.toLowerCase()).not.toContain(" and "); + expect(tieQuestion.question.toLowerCase()).not.toContain(" or "); + + return { + scenario: fixture.scenario, + candidateCount: explanation.candidates.length, + ambiguityStatus: explanation.status, + tieReason: explanation.summary.selectedReason, + investigationStrategy: tieQuestion.strategy, + question: tieQuestion.question, + explanationFavoured: explanation.selectedNodeId !== null, + }; + }); + + expect(summary).toMatchInlineSnapshot(` + [ + { + "ambiguityStatus": "ambiguous", + "candidateCount": 2, + "explanationFavoured": false, + "investigationStrategy": null, + "question": "What changed during the period that could explain why Revenue increased by 18%, but cash in the bank fell over the same period?", + "scenario": "Revenue increased by 18%, but cash in the bank fell over the same period.", + "tieReason": "No justified distinction between leading unknowns.", + }, + { + "ambiguityStatus": "ambiguous", + "candidateCount": 2, + "explanationFavoured": false, + "investigationStrategy": null, + "question": "What changed during the period that could explain why Customer satisfaction scores increased, but complaints also increased?", + "scenario": "Customer satisfaction scores increased, but complaints also increased.", + "tieReason": "No justified distinction between leading unknowns.", + }, + { + "ambiguityStatus": "ambiguous", + "candidateCount": 2, + "explanationFavoured": false, + "investigationStrategy": null, + "question": "What changed during the period that could explain why Average delivery time decreased by 25%, but order cancellations increased?", + "scenario": "Average delivery time decreased by 25%, but order cancellations increased.", + "tieReason": "No justified distinction between leading unknowns.", + }, + { + "ambiguityStatus": "ambiguous", + "candidateCount": 2, + "explanationFavoured": false, + "investigationStrategy": null, + "question": "What changed during the period that could explain why Website traffic doubled, but sales remained unchanged?", + "scenario": "Website traffic doubled, but sales remained unchanged.", + "tieReason": "No justified distinction between leading unknowns.", + }, + { + "ambiguityStatus": "ambiguous", + "candidateCount": 2, + "explanationFavoured": false, + "investigationStrategy": null, + "question": "What changed during the period that could explain why Production output increased by 30%, but quality defects also increased?", + "scenario": "Production output increased by 30%, but quality defects also increased.", + "tieReason": "No justified distinction between leading unknowns.", + }, + ] + `); + }); +});