test: verify ambiguity handling across domains
This commit is contained in:
+198
@@ -0,0 +1,198 @@
|
||||
import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js";
|
||||
|
||||
function buildAmbiguityFixture({
|
||||
key,
|
||||
scenario,
|
||||
summaryLabel,
|
||||
contradictionLabel,
|
||||
observationLabels,
|
||||
unknownLabels,
|
||||
disallowedQuestionTerms,
|
||||
}) {
|
||||
const summary = makeNode({
|
||||
id: `${key}-summary`,
|
||||
label: summaryLabel,
|
||||
description: "Summary of the situation from the scenario text",
|
||||
kind: "state",
|
||||
status: "provisional",
|
||||
confidence: "medium",
|
||||
});
|
||||
|
||||
const contradiction = makeNode({
|
||||
id: `${key}-contradiction`,
|
||||
label: contradictionLabel,
|
||||
description: contradictionLabel,
|
||||
kind: "relationship",
|
||||
status: "supported",
|
||||
confidence: "medium",
|
||||
});
|
||||
|
||||
const observations = observationLabels.map((label, index) =>
|
||||
makeNode({
|
||||
id: `${key}-obs-${index + 1}`,
|
||||
label,
|
||||
description: label,
|
||||
kind: "observation",
|
||||
status: "supported",
|
||||
confidence: "high",
|
||||
}),
|
||||
);
|
||||
|
||||
const unknowns = unknownLabels.map((label, index) =>
|
||||
makeNode({
|
||||
id: `${key}-unknown-${index + 1}`,
|
||||
label,
|
||||
description: label,
|
||||
kind: "unknown",
|
||||
status: "unknown",
|
||||
confidence: "high",
|
||||
}),
|
||||
);
|
||||
|
||||
const edges = [
|
||||
...observations.map((node) =>
|
||||
makeEdge({
|
||||
id: `${node.id}-supports-summary`,
|
||||
fromNodeId: node.id,
|
||||
toNodeId: summary.id,
|
||||
relationship: "supports",
|
||||
description: `${node.label} supports the summary.`,
|
||||
}),
|
||||
),
|
||||
...unknowns.map((node) =>
|
||||
makeEdge({
|
||||
id: `${node.id}-depends-summary`,
|
||||
fromNodeId: node.id,
|
||||
toNodeId: summary.id,
|
||||
relationship: "depends_on",
|
||||
description: `${node.label} is an unresolved factor for this situation.`,
|
||||
}),
|
||||
),
|
||||
];
|
||||
|
||||
return {
|
||||
key,
|
||||
scenario,
|
||||
disallowedQuestionTerms,
|
||||
graph: makeGraph({
|
||||
centralStatement: scenario,
|
||||
nodes: [summary, contradiction, ...observations, ...unknowns],
|
||||
edges,
|
||||
activeUnknownNodeId: null,
|
||||
resolvedNodeIds: [],
|
||||
currentSummary: `Ambiguity fixture for ${key}`,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
export const ambiguityGeneralisationFixtures = [
|
||||
buildAmbiguityFixture({
|
||||
key: "revenue-cash",
|
||||
scenario:
|
||||
"Revenue increased by 18%, but cash in the bank fell over the same period.",
|
||||
summaryLabel: "Revenue rose while cash fell",
|
||||
contradictionLabel:
|
||||
"Contradiction between revenue improvement and lower cash reserves.",
|
||||
observationLabels: [
|
||||
"Revenue increased by 18%.",
|
||||
"Cash in the bank decreased over the same period.",
|
||||
],
|
||||
unknownLabels: [
|
||||
"Possible explanation for the contradiction from one side of the situation.",
|
||||
"Possible explanation for the contradiction from another side of the situation.",
|
||||
],
|
||||
disallowedQuestionTerms: [
|
||||
"accounts receivable",
|
||||
"capex",
|
||||
"debt repayments",
|
||||
"working capital",
|
||||
],
|
||||
}),
|
||||
buildAmbiguityFixture({
|
||||
key: "satisfaction-complaints",
|
||||
scenario:
|
||||
"Customer satisfaction scores increased, but complaints also increased.",
|
||||
summaryLabel: "Satisfaction scores rose while complaints also rose",
|
||||
contradictionLabel:
|
||||
"Contradiction between higher satisfaction scores and higher complaint volume.",
|
||||
observationLabels: [
|
||||
"Customer satisfaction scores increased.",
|
||||
"Complaints increased.",
|
||||
],
|
||||
unknownLabels: [
|
||||
"Possible explanation for why the positive signal and negative signal moved together.",
|
||||
"Another possible explanation for why the positive signal and negative signal moved together.",
|
||||
],
|
||||
disallowedQuestionTerms: [
|
||||
"net promoter",
|
||||
"ticket backlog",
|
||||
"call deflection",
|
||||
"support queue",
|
||||
],
|
||||
}),
|
||||
buildAmbiguityFixture({
|
||||
key: "delivery-cancellations",
|
||||
scenario:
|
||||
"Average delivery time decreased by 25%, but order cancellations increased.",
|
||||
summaryLabel: "Delivery became faster while cancellations increased",
|
||||
contradictionLabel:
|
||||
"Contradiction between faster delivery and more order cancellations.",
|
||||
observationLabels: [
|
||||
"Average delivery time decreased by 25%.",
|
||||
"Order cancellations increased.",
|
||||
],
|
||||
unknownLabels: [
|
||||
"Possible explanation for why the faster result did not reduce the negative result.",
|
||||
"Another possible explanation for why the faster result did not reduce the negative result.",
|
||||
],
|
||||
disallowedQuestionTerms: [
|
||||
"fulfilment",
|
||||
"last mile",
|
||||
"warehouse",
|
||||
"routing",
|
||||
],
|
||||
}),
|
||||
buildAmbiguityFixture({
|
||||
key: "traffic-sales",
|
||||
scenario: "Website traffic doubled, but sales remained unchanged.",
|
||||
summaryLabel: "Website traffic doubled while sales stayed flat",
|
||||
contradictionLabel:
|
||||
"Contradiction between much higher traffic and unchanged sales.",
|
||||
observationLabels: [
|
||||
"Website traffic doubled.",
|
||||
"Sales remained unchanged.",
|
||||
],
|
||||
unknownLabels: [
|
||||
"Possible explanation for why the stronger signal did not change the outcome.",
|
||||
"Another possible explanation for why the stronger signal did not change the outcome.",
|
||||
],
|
||||
disallowedQuestionTerms: [
|
||||
"conversion funnel",
|
||||
"campaign attribution",
|
||||
"landing page",
|
||||
"checkout flow",
|
||||
],
|
||||
}),
|
||||
buildAmbiguityFixture({
|
||||
key: "output-defects",
|
||||
scenario:
|
||||
"Production output increased by 30%, but quality defects also increased.",
|
||||
summaryLabel: "Production output rose while defects also rose",
|
||||
contradictionLabel:
|
||||
"Contradiction between higher output and more quality defects.",
|
||||
observationLabels: [
|
||||
"Production output increased by 30%.",
|
||||
"Quality defects increased.",
|
||||
],
|
||||
unknownLabels: [
|
||||
"Possible explanation for why the gain came with a worsening result.",
|
||||
"Another possible explanation for why the gain came with a worsening result.",
|
||||
],
|
||||
disallowedQuestionTerms: [
|
||||
"scrap rate",
|
||||
"throughput",
|
||||
"yield",
|
||||
"root cause",
|
||||
],
|
||||
}),
|
||||
];
|
||||
@@ -0,0 +1,132 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
formulateQuestion,
|
||||
formulateTieResolutionQuestion,
|
||||
} from "@/lib/graph/question-formulator.js";
|
||||
import {
|
||||
explainUnknownSelection,
|
||||
selectActiveUnknownCandidate,
|
||||
} from "@/lib/graph/utils.js";
|
||||
import { ambiguityGeneralisationFixtures } from "@/tests/fixtures/ambiguity-generalisation.js";
|
||||
|
||||
function neutraliseUnknownLabels(graph) {
|
||||
let counter = 0;
|
||||
return {
|
||||
...graph,
|
||||
nodes: graph.nodes.map((node) => {
|
||||
if (node.kind !== "unknown") return { ...node };
|
||||
counter += 1;
|
||||
return {
|
||||
...node,
|
||||
label: `Unknown ${String.fromCharCode(64 + counter)}`,
|
||||
description: `Unknown factor ${counter}.`,
|
||||
};
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function isSingleQuestion(question) {
|
||||
return (question.match(/\?/g) || []).length === 1;
|
||||
}
|
||||
|
||||
describe("ambiguity generalisation", () => {
|
||||
it("preserves ambiguity across contradiction scenarios without favouring one explanation", () => {
|
||||
const summary = ambiguityGeneralisationFixtures.map((fixture) => {
|
||||
const explanation = explainUnknownSelection(fixture.graph, []);
|
||||
const selection = selectActiveUnknownCandidate(fixture.graph, []);
|
||||
const neutralExplanation = explainUnknownSelection(
|
||||
neutraliseUnknownLabels(fixture.graph),
|
||||
[],
|
||||
);
|
||||
const tieQuestion = formulateTieResolutionQuestion({
|
||||
graph: fixture.graph,
|
||||
});
|
||||
const representativeUnknown = fixture.graph.nodes.find(
|
||||
(node) => node.kind === "unknown",
|
||||
);
|
||||
const fallbackQuestion = formulateQuestion({
|
||||
node: representativeUnknown,
|
||||
graph: fixture.graph,
|
||||
});
|
||||
|
||||
const lowerQuestion = tieQuestion.question.toLowerCase();
|
||||
for (const term of fixture.disallowedQuestionTerms) {
|
||||
expect(lowerQuestion).not.toContain(term.toLowerCase());
|
||||
}
|
||||
|
||||
expect(explanation.status).toBe("ambiguous");
|
||||
expect(selection.status).toBe("ambiguous");
|
||||
expect(selection.selectedNode).toBeNull();
|
||||
expect(explanation.selectedNodeId).toBeNull();
|
||||
expect(explanation.candidates).toHaveLength(2);
|
||||
expect(explanation.summary.selectedReason).toBe(
|
||||
"No justified distinction between leading unknowns.",
|
||||
);
|
||||
expect(explanation.alphabeticalUsedAsReasoning).toBe(false);
|
||||
expect(neutralExplanation.status).toBe("ambiguous");
|
||||
expect(isSingleQuestion(tieQuestion.question)).toBe(true);
|
||||
expect(tieQuestion.question.toLowerCase()).not.toContain(" and ");
|
||||
expect(tieQuestion.question.toLowerCase()).not.toContain(" or ");
|
||||
|
||||
return {
|
||||
scenario: fixture.scenario,
|
||||
candidateCount: explanation.candidates.length,
|
||||
ambiguityStatus: explanation.status,
|
||||
tieReason: explanation.summary.selectedReason,
|
||||
investigationStrategy: tieQuestion.strategy,
|
||||
question: tieQuestion.question,
|
||||
explanationFavoured: explanation.selectedNodeId !== null,
|
||||
};
|
||||
});
|
||||
|
||||
expect(summary).toMatchInlineSnapshot(`
|
||||
[
|
||||
{
|
||||
"ambiguityStatus": "ambiguous",
|
||||
"candidateCount": 2,
|
||||
"explanationFavoured": false,
|
||||
"investigationStrategy": null,
|
||||
"question": "What changed during the period that could explain why Revenue increased by 18%, but cash in the bank fell over the same period?",
|
||||
"scenario": "Revenue increased by 18%, but cash in the bank fell over the same period.",
|
||||
"tieReason": "No justified distinction between leading unknowns.",
|
||||
},
|
||||
{
|
||||
"ambiguityStatus": "ambiguous",
|
||||
"candidateCount": 2,
|
||||
"explanationFavoured": false,
|
||||
"investigationStrategy": null,
|
||||
"question": "What changed during the period that could explain why Customer satisfaction scores increased, but complaints also increased?",
|
||||
"scenario": "Customer satisfaction scores increased, but complaints also increased.",
|
||||
"tieReason": "No justified distinction between leading unknowns.",
|
||||
},
|
||||
{
|
||||
"ambiguityStatus": "ambiguous",
|
||||
"candidateCount": 2,
|
||||
"explanationFavoured": false,
|
||||
"investigationStrategy": null,
|
||||
"question": "What changed during the period that could explain why Average delivery time decreased by 25%, but order cancellations increased?",
|
||||
"scenario": "Average delivery time decreased by 25%, but order cancellations increased.",
|
||||
"tieReason": "No justified distinction between leading unknowns.",
|
||||
},
|
||||
{
|
||||
"ambiguityStatus": "ambiguous",
|
||||
"candidateCount": 2,
|
||||
"explanationFavoured": false,
|
||||
"investigationStrategy": null,
|
||||
"question": "What changed during the period that could explain why Website traffic doubled, but sales remained unchanged?",
|
||||
"scenario": "Website traffic doubled, but sales remained unchanged.",
|
||||
"tieReason": "No justified distinction between leading unknowns.",
|
||||
},
|
||||
{
|
||||
"ambiguityStatus": "ambiguous",
|
||||
"candidateCount": 2,
|
||||
"explanationFavoured": false,
|
||||
"investigationStrategy": null,
|
||||
"question": "What changed during the period that could explain why Production output increased by 30%, but quality defects also increased?",
|
||||
"scenario": "Production output increased by 30%, but quality defects also increased.",
|
||||
"tieReason": "No justified distinction between leading unknowns.",
|
||||
},
|
||||
]
|
||||
`);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user