confidence-engine/tests/graph/question-formulator.test.js

import { describe, expect, it } from "vitest";
import {
  assessUnknownAtomicity,
  formulateQuestion,
  formulateTieResolutionQuestion,
  selectReasoningPattern,
  selectInvestigationStrategy,
} from "@/lib/graph/question-formulator.js";
import { makeEdge, makeGraph, makeNode } from "@/lib/graph/schema.js";

function makeGraphFor(node, extra = {}) {
  return makeGraph({
    centralStatement: extra.centralStatement || "Decision context",
    nodes: [node, ...(extra.nodes || [])],
    edges: extra.edges || [],
    activeUnknownNodeId: node.id,
    resolvedNodeIds: extra.resolvedNodeIds || [],
    currentSummary: "Test summary",
  });
}

describe("formulateQuestion", () => {
  it("atomicity assessment leaves focused unknowns direct and marks broad explanation unknowns composite", () => {
    const atomicUnknown = makeNode({
      id: "n-atomic",
      label: "Complaint rate denominator",
      description:
        "Need the denominator because it directly determines the complaint rate.",
      kind: "unknown",
      status: "unknown",
      confidence: "high",
    });
    const compositeUnknown = makeNode({
      id: "n-composite",
      label:
        "Explanation for why revenue increased by 18%, but cash in the bank fell over the same period",
      description:
        "Need to understand what change or event could explain why these observations differ, because that is needed to investigate their relationship.",
      kind: "unknown",
      status: "unknown",
      confidence: "medium",
    });
    const compositeGraph = makeGraphFor(compositeUnknown, {
      centralStatement:
        "Revenue increased by 18%, but cash in the bank fell over the same period.",
      nodes: [
        makeNode({
          id: "n-revenue-observation",
          label: "Revenue increased by 18%.",
          description: "Revenue increased by 18%.",
          kind: "observation",
          status: "supported",
          confidence: "high",
        }),
        makeNode({
          id: "n-cash-observation",
          label: "Cash in the bank decreased over the same period.",
          description: "Cash in the bank decreased over the same period.",
          kind: "observation",
          status: "supported",
          confidence: "high",
        }),
      ],
    });

    expect(
      assessUnknownAtomicity({
        node: atomicUnknown,
        graph: makeGraphFor(atomicUnknown),
      }).atomicity,
    ).toBe("atomic");
    expect(
      assessUnknownAtomicity({
        node: compositeUnknown,
        graph: compositeGraph,
      }).atomicity,
    ).toBe("composite");
  });

  it("commercial viability plus build decision produces a decision-threshold question", () => {
    const unknown = makeNode({
      id: "n-commercial",
      label: "Uncertainty regarding the commercial value of the product",
      description:
        "Commercial justification remains unclear because the decision depends on it.",
      kind: "unknown",
      status: "unknown",
      confidence: "high",
      parentId: "n-decision",
    });
    const decision = makeNode({
      id: "n-decision",
      label: "Build decision",
      description: "Decision introduced by the answer.",
      kind: "state",
      status: "known",
      confidence: "medium",
      childIds: [unknown.id],
      value: "Deciding whether to build the product",
    });
    const graph = makeGraphFor(unknown, {
      nodes: [decision],
      resolvedNodeIds: [decision.id],
    });

    const result = formulateQuestion({ node: unknown, graph });

    expect(result.strategy).toBe("decision_threshold");
    expect(result.reasoningPattern).toBe("decision");
    expect(result.questionFamily).toBe("decision_threshold");
    expect(result.question).toContain("What outcome");
    expect(result.question.toLowerCase()).toContain("justify");
  });

  it("commercial viability does not produce a pricing-first question", () => {
    const unknown = makeNode({
      id: "n-commercial",
      label: "Commercial viability",
      description:
        "Commercial viability remains unresolved because the decision depends on it.",
      kind: "unknown",
      status: "unknown",
      confidence: "high",
    });
    const graph = makeGraphFor(unknown);

    const result = formulateQuestion({ node: unknown, graph });

    expect(result.question.toLowerCase()).not.toContain("price");
    expect(result.question.toLowerCase()).not.toContain("pricing");
  });

  it("undefined term produces a definition question", () => {
    const unknown = makeNode({
      id: "n-term",
      label: "Success criteria definition",
      description:
        "Need a definition of the term because the team uses it inconsistently.",
      kind: "unknown",
      status: "unknown",
      confidence: "medium",
    });

    const result = formulateQuestion({
      node: unknown,
      graph: makeGraphFor(unknown),
    });

    expect(result.strategy).toBe("definition");
    expect(result.reasoningPattern).toBe("definition");
    expect(result.questionFamily).toBe("definition");
    expect(result.question).toMatch(/^What does /);
  });

  it("unsupported claim produces an evidence question", () => {
    const unknown = makeNode({
      id: "n-claim",
      label: "Demand claim",
      description: "Need evidence because the claim has not been validated.",
      kind: "reported_claim",
      status: "provisional",
      confidence: "low",
    });

    const result = formulateQuestion({
      node: unknown,
      graph: makeGraphFor(unknown),
    });

    expect(result.strategy).toBe("evidence_gathering");
    expect(result.question).toContain("What evidence");
  });

  it("missing previous state produces a baseline question", () => {
    const unknown = makeNode({
      id: "n-baseline",
      label: "Baseline conversion rate",
      description:
        "Need the previous baseline because the change cannot be assessed without it.",
      kind: "unknown",
      status: "unknown",
      confidence: "medium",
    });

    const result = formulateQuestion({
      node: unknown,
      graph: makeGraphFor(unknown),
    });

    expect(result.strategy).toBe("baseline_reconstruction");
    expect(result.question).toContain("What was the comparable state before");
  });

  it("conflicting claim produces a contradiction-resolution question", () => {
    const unknown = makeNode({
      id: "n-conflict",
      label: "Conflicting churn claim",
      description:
        "Need to resolve the inconsistency because the current figures contradict each other.",
      kind: "unknown",
      status: "unknown",
      confidence: "medium",
    });

    const contradiction = makeNode({
      id: "n-contradiction",
      label: "Contradicted report",
      description: "Two sources disagree about churn.",
      kind: "conclusion",
      status: "contradicted",
      confidence: "low",
      childIds: [unknown.id],
    });

    const result = formulateQuestion({
      node: unknown,
      graph: makeGraphFor(unknown, { nodes: [contradiction] }),
    });

    expect(result.strategy).toBe("contradiction_resolution");
    expect(result.reasoningPattern).toBe("contradiction");
    expect(result.question).toContain("resolve the contradiction");
  });

  it("reasoning pattern selection marks commercial validation as decision rather than explanation", () => {
    const unknown = makeNode({
      id: "n-commercial-pattern",
      label:
        "Whether the method addresses a genuine, high-priority problem for a specific audience",
      description:
        "Need to know whether this solves a real problem for a clear audience before continuing development.",
      kind: "unknown",
      status: "unknown",
      confidence: "high",
    });
    const graph = makeGraphFor(unknown, {
      centralStatement:
        "Before investing more, we need to know whether continuing development is commercially justified.",
    });

    const result = selectReasoningPattern({ node: unknown, graph });

    expect(result.pattern).toBe("decision");
  });

  it("comparison scenario selects the comparison pattern", () => {
    const unknown = makeNode({
      id: "n-comparison-pattern",
      label: "How the two observations were measured",
      description:
        "Need evidence about the measure used for each observation, because that could help explain the difference.",
      kind: "unknown",
      status: "unknown",
      confidence: "medium",
    });
    const graph = makeGraphFor(unknown, {
      centralStatement: "Traffic increased, but sales stayed flat.",
      nodes: [
        makeNode({
          id: "n-traffic-observation",
          label: "Traffic increased.",
          description: "Traffic increased.",
          kind: "observation",
          status: "supported",
          confidence: "high",
        }),
        makeNode({
          id: "n-sales-observation",
          label: "Sales stayed flat.",
          description: "Sales stayed flat.",
          kind: "observation",
          status: "supported",
          confidence: "high",
        }),
      ],
    });

    const result = selectReasoningPattern({ node: unknown, graph });

    expect(result.pattern).toBe("comparison");
  });

  it("user-owned constraint ambiguity produces a clarification question rather than an evidence request", () => {
    const unknown = makeNode({
      id: "n-constraint",
      label: "Whether avoiding additional risk is a hard constraint",
      description:
        "Need to know whether avoiding additional risk is a hard constraint or a preference/trade-off.",
      kind: "unknown",
      status: "unknown",
      confidence: "medium",
    });

    const result = formulateQuestion({
      node: unknown,
      graph: makeGraphFor(unknown),
    });

    expect(result.strategy).toBeNull();
    expect(result.question).toBe(
      "Is avoiding additional risk a hard constraint or a preference/trade-off?",
    );
  });

  it("evidence-resolvable competing-cause unknown stays on an evidence route rather than neutral clarification", () => {
    const unknown = makeNode({
      id: "n-delivery-cause",
      label: "Possible causes of the delivery delay",
      description:
        "Need to determine whether staff capacity or supplier lead times are responsible for the delivery delay.",
      kind: "unknown",
      status: "unknown",
      confidence: "medium",
    });

    const result = formulateQuestion({
      node: unknown,
      graph: makeGraphFor(unknown, {
        centralStatement: "Delivery is delayed and the cause is still unknown.",
      }),
    });

    expect(result.reasoningPattern).toBe("diagnosis");
    expect(result.question).toContain("What evidence");
    expect(result.question).not.toContain("What would clarify");
  });

  it("the same unknown can produce different questions when paired with different strategies", () => {
    const thresholdUnknown = makeNode({
      id: "n-threshold-unknown",
      label: "Value threshold",
      description: "Need to resolve the value threshold.",
      kind: "unknown",
      status: "unknown",
      confidence: "high",
    });
    const definitionUnknown = makeNode({
      id: "n-definition-unknown",
      label: "Value term",
      description: "Need to resolve what value term refers to in this context.",
      kind: "unknown",
      status: "unknown",
      confidence: "high",
    });

    const decisionGraph = makeGraphFor(thresholdUnknown, {
      centralStatement: "We are deciding whether to launch this product.",
      nodes: [
        makeNode({
          id: "n-decision",
          label: "Launch decision",
          description: "Decision depends on the value threshold.",
          kind: "state",
          status: "known",
          confidence: "medium",
          childIds: [thresholdUnknown.id],
          value: "Deciding whether to launch the product",
        }),
      ],
    });

    const definitionGraph = makeGraphFor(definitionUnknown, {
      centralStatement:
        "The team uses the term value threshold inconsistently.",
      nodes: [
        makeNode({
          id: "n-definition",
          label: "Definition disagreement about value threshold",
          description:
            "Need a definition of value threshold because the term is used inconsistently before comparing options.",
          kind: "state",
          status: "known",
          confidence: "medium",
          childIds: [definitionUnknown.id],
        }),
      ],
    });

    const decisionResult = formulateQuestion({
      node: thresholdUnknown,
      graph: decisionGraph,
    });
    const definitionResult = formulateQuestion({
      node: definitionUnknown,
      graph: definitionGraph,
    });

    expect(decisionResult.strategy).toBe("decision_threshold");
    expect(definitionResult.strategy).toBe("definition");
    expect(decisionResult.question).not.toBe(definitionResult.question);
  });

  it("strategy selection is deterministic and explainable", () => {
    const unknown = makeNode({
      id: "n-threshold",
      label: "Success threshold",
      description:
        "Need the success threshold because the decision depends on it.",
      kind: "unknown",
      status: "unknown",
      confidence: "high",
    });
    const graph = makeGraphFor(unknown, {
      centralStatement: "We need to decide whether to continue investing.",
    });

    const first = selectInvestigationStrategy({ node: unknown, graph });
    const second = selectInvestigationStrategy({ node: unknown, graph });

    expect(first).toEqual(second);
    expect(first.key).toBe("decision_threshold");
    expect(first.reason).toContain("threshold");
  });

  it("question is singular and answerable", () => {
    const unknown = makeNode({
      id: "n-evidence",
      label: "Evidence of demand",
      description:
        "Need evidence of demand because the decision depends on it.",
      kind: "unknown",
      status: "unknown",
      confidence: "medium",
    });

    const result = formulateQuestion({
      node: unknown,
      graph: makeGraphFor(unknown),
    });

    expect(result.question.match(/\?/g) || []).toHaveLength(1);
    expect(result.question.toLowerCase()).not.toContain(" and ");
  });

  it("awkward uncertainty phrasing is rejected via fallback", () => {
    const unknown = makeNode({
      id: "n-weird",
      label: "Uncertainty regarding service reliability",
      description: "Unknown service reliability.",
      kind: "unknown",
      status: "unknown",
      confidence: "medium",
    });

    const result = formulateQuestion({
      node: unknown,
      graph: makeGraphFor(unknown),
    });

    expect(result.question).not.toContain("How should uncertainty regarding");
    expect(result.question).not.toContain(
      "What would resolve uncertainty regarding",
    );
  });

  it("ambiguous contradiction produces a broad distinguishing question without accounting jargon", () => {
    const unknown = makeNode({
      id: "n-cause-a",
      label: "Cash outflow cause",
      description: "Unclear explanation for the contradiction.",
      kind: "unknown",
      status: "unknown",
      confidence: "high",
    });
    const contradiction = makeNode({
      id: "n-contradiction",
      label: "Divergent movement between revenue and cash",
      description: "Two signals moved in opposite directions.",
      kind: "relationship",
      status: "supported",
      confidence: "medium",
    });
    const revenueObservation = makeNode({
      id: "n-revenue-observation",
      label: "Revenue increased by 18%.",
      description: "Revenue increased by 18%.",
      kind: "observation",
      status: "supported",
      confidence: "high",
    });
    const cashObservation = makeNode({
      id: "n-cash-observation",
      label: "Cash in the bank decreased over the same period.",
      description: "Cash in the bank decreased over the same period.",
      kind: "observation",
      status: "supported",
      confidence: "high",
    });
    const graph = makeGraphFor(unknown, {
      centralStatement:
        "Revenue increased by 18%, but cash in the bank fell over the same period.",
      nodes: [contradiction, revenueObservation, cashObservation],
    });

    const result = formulateTieResolutionQuestion({ graph });

    expect(result.question).toBe(
      "Were these figures measured on the same basis and at the same scale?",
    );
    expect(result.comparabilityStatus).toBe("uncertain");
    expect(result.question.toLowerCase()).not.toMatch(
      /accounts receivable|capex|debt repayments|working capital/,
    );
  });

  it("definition is selected only for genuine definition unknowns", () => {
    const unknown = makeNode({
      id: "n-definition-only",
      label: "Definition of success criteria",
      description: "The term is used inconsistently and needs a definition.",
      kind: "unknown",
      status: "unknown",
      confidence: "medium",
    });

    const result = formulateQuestion({
      node: unknown,
      graph: makeGraphFor(unknown),
    });
    expect(result.strategy).toBe("definition");
  });

  it("an unknown about possible causes does not become a definition question", () => {
    const unknown = makeNode({
      id: "n-causes",
      label: "Possible causes of the divergence",
      description: "Several causes may explain the divergence.",
      kind: "unknown",
      status: "unknown",
      confidence: "medium",
    });

    const result = formulateQuestion({
      node: unknown,
      graph: makeGraphFor(unknown),
    });
    expect(result.reasoningPattern).toBe("diagnosis");
    expect(result.strategy).toBeNull();
    expect(result.question).toBe(
      "What would clarify possible causes of the divergence in this situation?",
    );
  });

  it("malformed punctuation is rejected", () => {
    const unknown = makeNode({
      id: "n-punct",
      label: "Magnitude and nature of cash outflows (operating expenses).",
      description:
        "Magnitude and nature of cash outflows (operating expenses).",
      kind: "unknown",
      status: "unknown",
      confidence: "medium",
    });

    const result = formulateQuestion({
      node: unknown,
      graph: makeGraphFor(unknown),
    });
    expect(result.question).not.toContain("). is true?");
    expect(result.question).toBe(
      "What would clarify magnitude and nature of cash outflows (operating expenses) in this situation?",
    );
  });
});