robbond
a0bcb12792
Core fix: For cases with expectedBehaviours, reasoningQuality.status is now set exclusively from behaviour evaluation results (required behaviour pass/fail). Legacy concept checks remain visible as diagnostic-only metrics and do not influence the authoritative result. Key changes: - Behaviour-based scoring determines reasoning status (passed/failed) instead of legacy concept literal matching - Schema failure correctly forces not_evaluated (no vacuous truth) - Saved live results re-evaluator preserves provenance metadata - Classification tolerance map works bidirectionally for interchangeable types - normalise() treats underscores as word characters, hyphens as spaces Tests: 74 passing across both evaluator test suites - tests/evaluator-behaviour-authoritative.test.mjs (47 tests, new) - tests/evaluator-semantic.test.mjs (27 tests)
612 lines
16 KiB
JSON
612 lines
16 KiB
JSON
[
|
|
{
|
|
"id": "diag-01",
|
|
"input": "We've seen a spike in complaints from our warehouse team this month compared to last month.",
|
|
"expectedPrimaryTypes": [
|
|
"unexplained_change"
|
|
],
|
|
"expectedReasoningModes": [
|
|
"establish_baseline",
|
|
"identify_difference"
|
|
],
|
|
"shouldIdentify": [
|
|
"complaints",
|
|
"warehouse",
|
|
"baseline comparison"
|
|
],
|
|
"shouldNotInfer": [
|
|
"quality issue",
|
|
"staff turnover",
|
|
"training gap"
|
|
],
|
|
"description": "Baseline comparison — change without context. Should NOT jump to conclusions about quality or staff issues.",
|
|
"expectedBehaviours": [
|
|
{
|
|
"id": "b-baseline",
|
|
"description": "Identifies prior state or baseline period",
|
|
"type": "baseline_recognition",
|
|
"acceptedSignals": [
|
|
"baseline",
|
|
"previous period",
|
|
"before comparison",
|
|
"pre-change",
|
|
"prior state"
|
|
],
|
|
"required": true
|
|
},
|
|
{
|
|
"id": "b-nosub",
|
|
"description": "Does NOT assert warehouse quality/staff issues as cause",
|
|
"type": "unsupported_justification",
|
|
"prohibitedSignals": [
|
|
"quality issue",
|
|
"staff turnover",
|
|
"training gap"
|
|
],
|
|
"required": true
|
|
},
|
|
{
|
|
"id": "b-nq1",
|
|
"description": "Asks about baseline detail (absolute numbers, time frame)",
|
|
"type": "next_question_target",
|
|
"acceptedSignals": [
|
|
"baseline",
|
|
"number",
|
|
"period",
|
|
"volume",
|
|
"count",
|
|
"over what period"
|
|
],
|
|
"required": false
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": "diag-02",
|
|
"input": "Some customers reported that the new app crashes when uploading photos.",
|
|
"expectedPrimaryTypes": [
|
|
"observed_problem"
|
|
],
|
|
"expectedReasoningModes": [
|
|
"identify_difference",
|
|
"establish_baseline"
|
|
],
|
|
"shouldIdentify": [
|
|
"app crashes",
|
|
"photo upload",
|
|
"some customers"
|
|
],
|
|
"shouldNotInfer": [
|
|
"all users affected",
|
|
"server-side bug",
|
|
"Android only"
|
|
],
|
|
"description": "Subset modifier — 'some customers' means not universal. Should distinguish from blanket claims.",
|
|
"expectedBehaviours": [
|
|
{
|
|
"id": "b-subset",
|
|
"description": "Recognises subset scope rather than universal claim",
|
|
"type": "subset_recognition",
|
|
"acceptedSignals": [
|
|
"some",
|
|
"subset",
|
|
"partial",
|
|
"not universal",
|
|
"certain users",
|
|
"limited to"
|
|
],
|
|
"required": true
|
|
},
|
|
{
|
|
"id": "b-obv",
|
|
"description": "Acknowledges photo-upload context from the scenario",
|
|
"type": "observation_recognition",
|
|
"acceptedSignals": [
|
|
"photo",
|
|
"upload",
|
|
"crash",
|
|
"app"
|
|
],
|
|
"required": false
|
|
},
|
|
{
|
|
"id": "b-nq2",
|
|
"description": "Asks about which user groups are affected vs unaffected",
|
|
"type": "next_question_target",
|
|
"acceptedSignals": [
|
|
"who",
|
|
"which users",
|
|
"affected group",
|
|
"distinguish",
|
|
"proportion"
|
|
],
|
|
"required": false
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": "diag-03",
|
|
"input": "Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.",
|
|
"expectedPrimaryTypes": [
|
|
"contradiction"
|
|
],
|
|
"expectedReasoningModes": [
|
|
"investigate_contradiction",
|
|
"establish_baseline"
|
|
],
|
|
"shouldIdentify": [
|
|
"sales decline",
|
|
"price increase",
|
|
"revenue increase",
|
|
"CFO report"
|
|
],
|
|
"shouldNotInfer": [
|
|
"price was set too high",
|
|
"competitors gained market share",
|
|
"revenue data is wrong"
|
|
],
|
|
"description": "Apparent contradiction — sales down but revenue up after price change. Distinguishes volume vs value.",
|
|
"expectedBehaviours": [
|
|
{
|
|
"id": "b-metric",
|
|
"description": "Recognises revenue/sales as different metric dimensions",
|
|
"type": "metric_relationship",
|
|
"acceptedSignals": [
|
|
"rate",
|
|
"denominator",
|
|
"comparable scale",
|
|
"volume vs value",
|
|
"per unit",
|
|
"absolute vs relative"
|
|
],
|
|
"required": true
|
|
},
|
|
{
|
|
"id": "b-contra",
|
|
"description": "Identifies the apparent contradiction between sales and revenue signals",
|
|
"type": "contradiction_recognition",
|
|
"acceptedSignals": [
|
|
"contradiction",
|
|
"divergent",
|
|
"opposing",
|
|
"conflicting",
|
|
"conversely",
|
|
"but"
|
|
],
|
|
"required": true
|
|
},
|
|
{
|
|
"id": "b-trans",
|
|
"description": "Acknowledges temporal caution in cause-effect timing",
|
|
"type": "transition_recognition",
|
|
"acceptedSignals": [
|
|
"transition",
|
|
"before to",
|
|
"moved from",
|
|
"after",
|
|
"since"
|
|
],
|
|
"required": false
|
|
},
|
|
{
|
|
"id": "b-nq3",
|
|
"description": "Asks about sales volume and revenue composition breakdown",
|
|
"type": "next_question_target",
|
|
"acceptedSignals": [
|
|
"sales volume",
|
|
"revenue composition",
|
|
"unit price",
|
|
"average",
|
|
"breakdown"
|
|
],
|
|
"required": false
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": "diag-04",
|
|
"input": "We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.",
|
|
"expectedPrimaryTypes": [
|
|
"decision_request"
|
|
],
|
|
"expectedReasoningModes": [
|
|
"decision_support",
|
|
"identify_missing_information"
|
|
],
|
|
"shouldIdentify": [
|
|
"marketplace app",
|
|
"Southeast Asia",
|
|
"competitor gap"
|
|
],
|
|
"shouldNotInfer": [
|
|
"this will definitely succeed",
|
|
"we have the resources",
|
|
"competitors are struggling"
|
|
],
|
|
"description": "Decision request — forward-looking, needs missing info identification.",
|
|
"expectedBehaviours": [
|
|
{
|
|
"id": "b-action",
|
|
"description": "Recognises forward-looking proposed action",
|
|
"type": "proposed_action_recognition",
|
|
"acceptedSignals": [
|
|
"decision_request",
|
|
"desired_outcome",
|
|
"action plan"
|
|
],
|
|
"required": true
|
|
},
|
|
{
|
|
"id": "b-nosub2",
|
|
"description": "Does NOT treat competitor gap as quantified fact",
|
|
"type": "unsupported_justification",
|
|
"prohibitedSignals": [
|
|
"competitor gap",
|
|
"gap confirmed",
|
|
"we lack"
|
|
],
|
|
"required": true
|
|
},
|
|
{
|
|
"id": "b-nq4",
|
|
"description": "Asks about market gap size and scope",
|
|
"type": "next_question_target",
|
|
"acceptedSignals": [
|
|
"gap size",
|
|
"market size",
|
|
"scope",
|
|
"extent",
|
|
"how big"
|
|
],
|
|
"required": false
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": "diag-05",
|
|
"input": "Our production line changed suppliers three months ago but still delivers the same defect rate as before.",
|
|
"expectedPrimaryTypes": [
|
|
"unexplained_change"
|
|
],
|
|
"expectedReasoningModes": [
|
|
"establish_baseline",
|
|
"identify_difference"
|
|
],
|
|
"shouldIdentify": [
|
|
"supplier change",
|
|
"three months ago",
|
|
"same defect rate"
|
|
],
|
|
"shouldNotInfer": [
|
|
"new supplier is worse",
|
|
"old supplier was better",
|
|
"quality process is broken"
|
|
],
|
|
"description": "Unexpected continuity — changed context but no outcome change.",
|
|
"expectedBehaviours": [
|
|
{
|
|
"id": "b-mnorm",
|
|
"description": "Recognises unexpected continuity despite change input",
|
|
"type": "measurement_normalisation",
|
|
"acceptedSignals": [
|
|
"normalise",
|
|
"denominator",
|
|
"rate",
|
|
"comparable scale",
|
|
"per unit"
|
|
],
|
|
"required": true
|
|
},
|
|
{
|
|
"id": "b-timing",
|
|
"description": "Acknowledges timing of the supplier change vs outcome measurement",
|
|
"type": "timing_recognition",
|
|
"acceptedSignals": [
|
|
"after",
|
|
"three months",
|
|
"timeline",
|
|
"time lag",
|
|
"delayed effect"
|
|
],
|
|
"required": false
|
|
},
|
|
{
|
|
"id": "b-nq5",
|
|
"description": "Asks why input change produced no outcome change",
|
|
"type": "next_question_target",
|
|
"acceptedSignals": [
|
|
"why",
|
|
"same rate",
|
|
"defect rate comparison",
|
|
"baseline",
|
|
"period of measurement"
|
|
],
|
|
"required": false
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": "diag-06",
|
|
"input": "From 45% to 62%, the completion rate for our onboarding flow improved significantly.",
|
|
"expectedPrimaryTypes": [
|
|
"unexplained_change"
|
|
],
|
|
"expectedReasoningModes": [
|
|
"establish_baseline",
|
|
"validate_measurement"
|
|
],
|
|
"shouldIdentify": [
|
|
"completion rate",
|
|
"45%",
|
|
"62%",
|
|
"onboarding"
|
|
],
|
|
"shouldNotInfer": [
|
|
"all improvements are due to the redesign",
|
|
"the old flow was bad",
|
|
"users prefer the new design"
|
|
],
|
|
"description": "Quantified improvement — needs context about measurement period and baseline conditions.",
|
|
"expectedBehaviours": [
|
|
{
|
|
"id": "b-baseline2",
|
|
"description": "Recognises quantified improvement needs context for significance",
|
|
"type": "baseline_recognition",
|
|
"acceptedSignals": [
|
|
"baseline",
|
|
"previous period",
|
|
"comparison point",
|
|
"reference",
|
|
"benchmark",
|
|
"pre-change"
|
|
],
|
|
"required": true
|
|
},
|
|
{
|
|
"id": "b-nq6",
|
|
"description": "Asks about timeframe, cohort, and baseline conditions",
|
|
"type": "next_question_target",
|
|
"acceptedSignals": [
|
|
"timeframe",
|
|
"cohort",
|
|
"baseline condition",
|
|
"measurement period",
|
|
"sample size"
|
|
],
|
|
"required": false
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": "diag-07",
|
|
"input": "A user claimed that our pricing model is too complex for small businesses.",
|
|
"expectedPrimaryTypes": [
|
|
"reported_claim"
|
|
],
|
|
"expectedReasoningModes": [
|
|
"validate_claim",
|
|
"identify_difference"
|
|
],
|
|
"shouldIdentify": [
|
|
"pricing complexity",
|
|
"small business",
|
|
"user claim"
|
|
],
|
|
"shouldNotInfer": [
|
|
"the pricing is actually complex",
|
|
"other small businesses agree",
|
|
"we should simplify pricing"
|
|
],
|
|
"description": "Single reported claim — needs validation, not acceptance as fact.",
|
|
"expectedBehaviours": [
|
|
{
|
|
"id": "b-cval",
|
|
"description": "Treats single-user claim as needing corroboration, not acceptance",
|
|
"type": "claim_validation",
|
|
"acceptedSignals": [
|
|
"validate",
|
|
"corroborate",
|
|
"verify",
|
|
"confirm",
|
|
"evidence needed",
|
|
"single user",
|
|
"unverified"
|
|
],
|
|
"required": true
|
|
},
|
|
{
|
|
"id": "b-nq7",
|
|
"description": "Asks for examples or corroboration from other users",
|
|
"type": "next_question_target",
|
|
"acceptedSignals": [
|
|
"examples",
|
|
"corroborate",
|
|
"other users",
|
|
"more examples",
|
|
"survey",
|
|
"feedback"
|
|
],
|
|
"required": false
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": "diag-08",
|
|
"input": "I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?",
|
|
"expectedPrimaryTypes": [
|
|
"ambiguous_statement"
|
|
],
|
|
"expectedReasoningModes": [
|
|
"clarify_meaning"
|
|
],
|
|
"shouldIdentify": [
|
|
"philosophical",
|
|
"ambiguous",
|
|
"meaning clarification"
|
|
],
|
|
"shouldNotInfer": [
|
|
"the phrase was wrong",
|
|
"the colleague is hostile",
|
|
"we should avoid philosophical language"
|
|
],
|
|
"description": "Meta-test — self-referential ambiguous statement. Should trigger clarification mode.",
|
|
"expectedBehaviours": [
|
|
{
|
|
"id": "b-ambig",
|
|
"description": "Recognises the phrase as ambiguous and requiring clarification",
|
|
"type": "ambiguity_recognition",
|
|
"acceptedSignals": [
|
|
"ambiguous",
|
|
"unclear meaning",
|
|
"clarify",
|
|
"interpretation varies",
|
|
"phrase intent"
|
|
],
|
|
"required": true
|
|
},
|
|
{
|
|
"id": "b-nq8",
|
|
"description": "Asks about the phrase intent in meeting context",
|
|
"type": "next_question_target",
|
|
"acceptedSignals": [
|
|
"intent",
|
|
"meaning",
|
|
"context",
|
|
"why said",
|
|
"what meant",
|
|
"phrase intent"
|
|
],
|
|
"required": false
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": "diag-09",
|
|
"input": "After the deployment last week, our complaint volume tripled to 47 cases per day.",
|
|
"expectedPrimaryTypes": [
|
|
"causal_claim"
|
|
],
|
|
"expectedReasoningModes": [
|
|
"investigate_contradiction",
|
|
"establish_baseline"
|
|
],
|
|
"shouldIdentify": [
|
|
"deployment",
|
|
"complaint volume increase",
|
|
"tripled",
|
|
"47 cases"
|
|
],
|
|
"shouldNotInfer": [
|
|
"the deployment caused the complaints",
|
|
"the bug report was insufficient",
|
|
"rollback is needed"
|
|
],
|
|
"description": "Post-event spike — presents correlation as potential causation. Must resist jumping to causal conclusion.",
|
|
"expectedBehaviours": [
|
|
{
|
|
"id": "b-trans2",
|
|
"description": "Distinguishes temporal sequence from causal proof",
|
|
"type": "transition_recognition",
|
|
"acceptedSignals": [
|
|
"transition",
|
|
"before to",
|
|
"after",
|
|
"temporal sequence",
|
|
"coincidence vs cause"
|
|
],
|
|
"required": true
|
|
},
|
|
{
|
|
"id": "b-baseline3",
|
|
"description": "Recognises need for pre-deployment complaint baseline",
|
|
"type": "baseline_recognition",
|
|
"acceptedSignals": [
|
|
"baseline",
|
|
"previous level",
|
|
"before deployment",
|
|
"pre-change",
|
|
"historical"
|
|
],
|
|
"required": true
|
|
},
|
|
{
|
|
"id": "b-nq9",
|
|
"description": "Asks about evidence distinguishing deployment effect from coincidence",
|
|
"type": "next_question_target",
|
|
"acceptedSignals": [
|
|
"coincidence",
|
|
"deployment timing",
|
|
"baseline comparison",
|
|
"other factors",
|
|
"confounders"
|
|
],
|
|
"required": false
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": "diag-10",
|
|
"input": "Some complaints involve production issues, but others say the delivery team is slow.",
|
|
"expectedPrimaryTypes": [
|
|
"observed_problem"
|
|
],
|
|
"expectedReasoningModes": [
|
|
"identify_difference",
|
|
"decompose_aggregate"
|
|
],
|
|
"shouldIdentify": [
|
|
"production issues",
|
|
"delivery speed",
|
|
"complaint types"
|
|
],
|
|
"shouldNotInfer": [
|
|
"production is worse than delivery",
|
|
"the delivery team needs training",
|
|
"both teams are underperforming equally"
|
|
],
|
|
"description": "Paired with diag-01 — distinguishes subset complaints from aggregate claims.",
|
|
"expectedBehaviours": [
|
|
{
|
|
"id": "b-obs2",
|
|
"description": "Decomposes complaints into distinct categories rather than merging",
|
|
"type": "observation_recognition",
|
|
"acceptedSignals": [
|
|
"complaint",
|
|
"production",
|
|
"delivery",
|
|
"categories",
|
|
"types of complaint",
|
|
"decompose"
|
|
],
|
|
"required": true
|
|
},
|
|
{
|
|
"id": "b-metric2",
|
|
"description": "Avoids merging complaint types without quantification",
|
|
"type": "metric_relationship",
|
|
"acceptedSignals": [
|
|
"rate",
|
|
"comparable scale",
|
|
"proportion",
|
|
"percentage",
|
|
"volume vs value"
|
|
],
|
|
"required": false
|
|
},
|
|
{
|
|
"id": "b-nq10",
|
|
"description": "Asks about complaint category proportions (production vs delivery)",
|
|
"type": "next_question_target",
|
|
"acceptedSignals": [
|
|
"proportion",
|
|
"percentage",
|
|
"ratio",
|
|
"how many",
|
|
"which is worse",
|
|
"split"
|
|
],
|
|
"required": false
|
|
}
|
|
]
|
|
}
|
|
] |