robbond
a0bcb12792
Core fix: For cases with expectedBehaviours, reasoningQuality.status is now set exclusively from behaviour evaluation results (required behaviour pass/fail). Legacy concept checks remain visible as diagnostic-only metrics and do not influence the authoritative result. Key changes: - Behaviour-based scoring determines reasoning status (passed/failed) instead of legacy concept literal matching - Schema failure correctly forces not_evaluated (no vacuous truth) - Saved live results re-evaluator preserves provenance metadata - Classification tolerance map works bidirectionally for interchangeable types - normalise() treats underscores as word characters, hyphens as spaces Tests: 74 passing across both evaluator test suites - tests/evaluator-behaviour-authoritative.test.mjs (47 tests, new) - tests/evaluator-semantic.test.mjs (27 tests)
347 lines
18 KiB
JSON
347 lines
18 KiB
JSON
[
|
|
{
|
|
"id": "diag-01",
|
|
"input": "We've seen a spike in complaints from our warehouse team this month compared to last month.",
|
|
"expectedPrimaryTypes": ["unexplained_change"],
|
|
"acceptedPrimaryAlternatives": ["observed_problem", "causal_claim"],
|
|
"expectedReasoningModes": ["establish_baseline", "identify_difference"],
|
|
"shouldIdentify": ["complaints", "warehouse", "baseline comparison"],
|
|
"shouldNotInfer": ["quality issue", "staff turnover", "training gap"],
|
|
"description": "Baseline comparison — change without context. Should NOT jump to conclusions about quality or staff issues.",
|
|
"expectedBehaviours": [
|
|
{
|
|
"id": "diag-01-beh-baseline",
|
|
"description": "Recognises month-to-month baseline comparison",
|
|
"type": "baseline_recognition",
|
|
"acceptedSignals": ["establish_baseline"],
|
|
"required": true,
|
|
"notes": "Model should compare current to prior state or identify the need to do so."
|
|
},
|
|
{
|
|
"id": "diag-01-beh-no-warehouse-quality",
|
|
"description": "Does not assume warehouse quality problems",
|
|
"type": "unsupported_justification",
|
|
"prohibitedSignals": ["quality issue", "staff turnover", "training gap"],
|
|
"required": true,
|
|
"notes": "The model must resist jumping to conclusions about the cause of complaints."
|
|
},
|
|
{
|
|
"id": "diag-01-beh-nq-baseline-detail",
|
|
"description": "Next question should seek baseline detail or complaint breakdown",
|
|
"type": "next_question_target",
|
|
"acceptedSignals": ["baseline", "complaints", "breakdown", "comparison", "previous period", "last month"],
|
|
"required": true,
|
|
"notes": "A useful next question would clarify what changed and by how much."
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": "diag-02",
|
|
"input": "Some customers reported that the new app crashes when uploading photos.",
|
|
"expectedPrimaryTypes": ["observed_problem"],
|
|
"acceptedPrimaryAlternatives": ["reported_claim", "fault_report"],
|
|
"expectedReasoningModes": ["identify_difference", "establish_baseline"],
|
|
"shouldIdentify": ["app crashes", "photo upload", "some customers"],
|
|
"shouldNotInfer": ["all users affected", "server-side bug", "Android only"],
|
|
"description": "Subset modifier — 'some customers' means not universal. Should distinguish from blanket claims.",
|
|
"expectedBehaviours": [
|
|
{
|
|
"id": "diag-02-beh-subset",
|
|
"description": "Recognises only some customers are affected",
|
|
"type": "subset_recognition",
|
|
"acceptedSignals": ["some", "subset", "partial", "certain users", "not universal", "limited to"],
|
|
"required": true,
|
|
"notes": "Model should recognise this is not a blanket claim and investigate what distinguishes affected from unaffected."
|
|
},
|
|
{
|
|
"id": "diag-02-beh-photo-upload",
|
|
"description": "Recognises failure occurs during photo upload",
|
|
"type": "observation_recognition",
|
|
"acceptedSignals": ["photo upload", "uploading photos", "photo upload crash"],
|
|
"required": true,
|
|
"notes": "The specific failure context matters — it isolates the problem to a particular operation."
|
|
},
|
|
{
|
|
"id": "diag-02-beh-nq-distinguish",
|
|
"description": "Next question should distinguish affected from unaffected users or conditions",
|
|
"type": "next_question_target",
|
|
"acceptedSignals": ["affected", "unaffected", "conditions", "users", "who", "what"],
|
|
"required": true,
|
|
"notes": "A useful next question would identify what separates customers who experience the crash from those who do not."
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": "diag-03",
|
|
"input": "Sales fell by 15% last month after we increased prices, but the CFO says revenue is still up 2%.",
|
|
"expectedPrimaryTypes": ["contradiction"],
|
|
"acceptedPrimaryAlternatives": ["observed_problem", "unexplained_change", "causal_claim"],
|
|
"expectedReasoningModes": ["investigate_contradiction", "establish_baseline"],
|
|
"shouldIdentify": ["sales decline", "price increase", "revenue increase", "CFO report"],
|
|
"shouldNotInfer": ["price was set too high", "competitors gained market share", "revenue data is wrong"],
|
|
"description": "Apparent contradiction — sales down but revenue up after price change. Distinguishes volume vs value.",
|
|
"expectedBehaviours": [
|
|
{
|
|
"id": "diag-03-beh-metric-relationship",
|
|
"description": "Recognises sales and revenue are different measures needing normalisation",
|
|
"type": "metric_relationship",
|
|
"acceptedSignals": ["sales", "revenue", "volume", "value", "normalisation", "denominator", "rate"],
|
|
"required": true,
|
|
"notes": "Sales volume and revenue are related but not equivalent — price acts as the bridge between them."
|
|
},
|
|
{
|
|
"id": "diag-03-beh-opposing-metric",
|
|
"description": "Recognises opposing metric movement",
|
|
"type": "contradiction_recognition",
|
|
"acceptedSignals": ["fell", "down", "up 2%", "increased"],
|
|
"required": true,
|
|
"notes": "The opposing directions of sales and revenue are the key signal — not the individual metrics."
|
|
},
|
|
{
|
|
"id": "diag-03-beh-temporal-caution",
|
|
"description": "Recognises price increase is temporally relevant but not proven causal",
|
|
"type": "transition_recognition",
|
|
"acceptedSignals": ["after", "increased prices", "temporally", "correlation", "causation"],
|
|
"required": true,
|
|
"notes": "Temporal sequence alone does not establish causation. The model should flag this distinction."
|
|
},
|
|
{
|
|
"id": "diag-03-beh-nq-metrics",
|
|
"description": "Next question should clarify sales volume, revenue composition or timing",
|
|
"type": "next_question_target",
|
|
"acceptedSignals": ["volume", "revenue", "composition", "timing", "breakdown"],
|
|
"required": true,
|
|
"notes": "A useful next question would distinguish whether the revenue increase comes from existing customers or new ones."
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": "diag-04",
|
|
"input": "We need to launch a marketplace app in Southeast Asia to capture the gap our competitors are exploiting.",
|
|
"expectedPrimaryTypes": ["decision_request"],
|
|
"acceptedPrimaryAlternatives": ["desired_outcome"],
|
|
"expectedReasoningModes": ["decision_support", "identify_missing_information"],
|
|
"shouldIdentify": ["marketplace app", "Southeast Asia", "competitor gap"],
|
|
"shouldNotInfer": ["this will definitely succeed", "we have the resources", "competitors are struggling"],
|
|
"description": "Decision request — forward-looking, needs missing info identification.",
|
|
"expectedBehaviours": [
|
|
{
|
|
"id": "diag-04-beh-proposed-action",
|
|
"description": "Recognises a proposed action or desired outcome",
|
|
"type": "proposed_action_recognition",
|
|
"acceptedSignals": ["need to launch", "we should implement", "launch app"],
|
|
"required": true,
|
|
"notes": "The input is forward-looking and proposes an action — the model should treat it as such."
|
|
},
|
|
{
|
|
"id": "diag-04-beh-competitor-warning",
|
|
"description": "Recognises competitor behaviour is unsupported justification",
|
|
"type": "unsupported_justification",
|
|
"prohibitedSignals": ["will definitely succeed", "we have the resources"],
|
|
"required": true,
|
|
"notes": "The competitor gap is asserted but not quantified — it cannot serve as proof of opportunity."
|
|
},
|
|
{
|
|
"id": "diag-04-beh-nq-market-gap",
|
|
"description": "Next question should clarify the actual market gap or intended outcome",
|
|
"type": "next_question_target",
|
|
"acceptedSignals": ["gap", "demand", "evidence", "market", "outcome"],
|
|
"required": true,
|
|
"notes": "A useful next question would establish what evidence supports the existence and size of the market gap."
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": "diag-05",
|
|
"input": "Our production line changed suppliers three months ago but still delivers the same defect rate as before.",
|
|
"expectedPrimaryTypes": ["unexplained_change"],
|
|
"acceptedPrimaryAlternatives": ["observed_problem"],
|
|
"expectedReasoningModes": ["establish_baseline", "identify_difference"],
|
|
"shouldIdentify": ["supplier change", "three months ago", "same defect rate"],
|
|
"shouldNotInfer": ["new supplier is worse", "old supplier was better", "quality process is broken"],
|
|
"description": "Unexpected continuity — changed context but no outcome change.",
|
|
"expectedBehaviours": [
|
|
{
|
|
"id": "diag-05-beh-continuity",
|
|
"description": "Recognises unexpected continuity: changed input, unchanged output",
|
|
"type": "measurement_normalisation",
|
|
"acceptedSignals": ["same", "unchanged", "still delivers", "continuity"],
|
|
"required": true,
|
|
"notes": "The key signal is that a significant change (supplier) produced no measurable outcome change."
|
|
},
|
|
{
|
|
"id": "diag-05-beh-temporal-anchor",
|
|
"description": "Recognises temporal anchor and stable metric",
|
|
"type": "timing_recognition",
|
|
"acceptedSignals": ["three months ago", "before", "previous"],
|
|
"required": true,
|
|
"notes": "The three-month window is important context — any supplier effect should have manifested by now."
|
|
},
|
|
{
|
|
"id": "diag-05-beh-nq-investigate-why",
|
|
"description": "Next question should investigate why a changed input produced no changed outcome",
|
|
"type": "next_question_target",
|
|
"acceptedSignals": ["why", "difference", "process", "quality process", "supplier"],
|
|
"required": true,
|
|
"notes": "A useful next question would ask whether the defect measurement methodology itself changed."
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": "diag-06",
|
|
"input": "From 45% to 62%, the completion rate for our onboarding flow improved significantly.",
|
|
"expectedPrimaryTypes": ["unexplained_change"],
|
|
"acceptedPrimaryAlternatives": ["observed_problem"],
|
|
"expectedReasoningModes": ["establish_baseline", "validate_measurement"],
|
|
"shouldIdentify": ["completion rate", "45%", "62%", "onboarding"],
|
|
"shouldNotInfer": ["all improvements are due to the redesign", "the old flow was bad", "users prefer the new design"],
|
|
"description": "Quantified improvement — needs context about measurement period and baseline conditions.",
|
|
"expectedBehaviours": [
|
|
{
|
|
"id": "diag-06-beh-quantified",
|
|
"description": "Recognises quantified improvement that needs contextual framing",
|
|
"type": "baseline_recognition",
|
|
"acceptedSignals": ["45%", "62%", "improved", "completion rate"],
|
|
"required": true,
|
|
"notes": "The numbers are only meaningful with baseline conditions, timeframe, and cohort context."
|
|
},
|
|
{
|
|
"id": "diag-06-beh-nq-context",
|
|
"description": "Seeks timeframe, cohort, baseline conditions or measurement consistency",
|
|
"type": "next_question_target",
|
|
"acceptedSignals": ["timeframe", "cohort", "baseline", "measurement", "conditions"],
|
|
"required": true,
|
|
"notes": "A useful next question would establish whether the improvement is due to a redesign or other factor."
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": "diag-07",
|
|
"input": "A user claimed that our pricing model is too complex for small businesses.",
|
|
"expectedPrimaryTypes": ["reported_claim"],
|
|
"acceptedPrimaryAlternatives": ["observed_problem"],
|
|
"expectedReasoningModes": ["validate_claim", "identify_difference"],
|
|
"shouldIdentify": ["pricing complexity", "small business", "user claim"],
|
|
"shouldNotInfer": ["the pricing is actually complex", "other small businesses agree", "we should simplify pricing"],
|
|
"description": "Single reported claim — needs validation, not acceptance as fact.",
|
|
"expectedBehaviours": [
|
|
{
|
|
"id": "diag-07-beh-claim-validation",
|
|
"description": "Treats the user statement as a reported claim requiring validation, not established fact",
|
|
"type": "claim_validation",
|
|
"acceptedSignals": ["claimed", "reported", "validation", "evidence"],
|
|
"required": true,
|
|
"notes": "A single user's opinion should be treated as evidence needing corroboration."
|
|
},
|
|
{
|
|
"id": "diag-07-beh-nq-examples",
|
|
"description": "Seeks examples or evidence of pricing complexity from other users",
|
|
"type": "next_question_target",
|
|
"acceptedSignals": ["examples", "evidence", "other users", "corroborate"],
|
|
"required": true,
|
|
"notes": "A useful next question would ask for additional examples or data points."
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": "diag-08",
|
|
"input": "I used the phrase 'philosophical difference' in a meeting and my colleague said it meant nothing. Is that fair?",
|
|
"expectedPrimaryTypes": ["ambiguous_statement"],
|
|
"acceptedPrimaryAlternatives": ["question"],
|
|
"expectedReasoningModes": ["clarify_meaning"],
|
|
"shouldIdentify": ["philosophical", "ambiguous", "meaning clarification"],
|
|
"shouldNotInfer": ["the phrase was wrong", "the colleague is hostile", "we should avoid philosophical language"],
|
|
"description": "Meta-test — self-referential ambiguous statement. Should trigger clarification mode.",
|
|
"expectedBehaviours": [
|
|
{
|
|
"id": "diag-08-beh-ambiguity",
|
|
"description": "Recognises ambiguity and interpersonal context",
|
|
"type": "ambiguity_recognition",
|
|
"acceptedSignals": ["ambiguous", "meaning", "interpretation", "clarify"],
|
|
"required": true,
|
|
"notes": "The model should flag the self-referential nature of the statement."
|
|
},
|
|
{
|
|
"id": "diag-08-beh-nq-intent",
|
|
"description": "Asks what the phrase was intended to mean in that specific meeting",
|
|
"type": "next_question_target",
|
|
"acceptedSignals": ["meaning", "intent", "phrase", "meeting"],
|
|
"required": true,
|
|
"notes": "A useful next question would ask the speaker what they meant by 'philosophical difference'."
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": "diag-09",
|
|
"input": "After the deployment last week, our complaint volume tripled to 47 cases per day.",
|
|
"expectedPrimaryTypes": ["causal_claim"],
|
|
"acceptedPrimaryAlternatives": ["unexplained_change", "observed_problem"],
|
|
"expectedReasoningModes": ["investigate_contradiction", "establish_baseline"],
|
|
"shouldIdentify": ["deployment", "complaint volume increase", "tripled", "47 cases"],
|
|
"shouldNotInfer": ["the deployment caused the complaints", "the bug report was insufficient", "rollback is needed"],
|
|
"description": "Post-event spike — presents correlation as potential causation. Must resist jumping to causal conclusion.",
|
|
"expectedBehaviours": [
|
|
{
|
|
"id": "diag-09-beh-temporal-sequence",
|
|
"description": "Recognises temporal sequence without assuming causation",
|
|
"type": "transition_recognition",
|
|
"acceptedSignals": ["after", "tripled", "deployment", "correlation", "coincidence"],
|
|
"required": true,
|
|
"notes": "Temporal sequence ≠ causation. The model should flag this distinction explicitly."
|
|
},
|
|
{
|
|
"id": "diag-09-beh-baseline-context",
|
|
"description": "Requires baseline context (what was the volume before?)",
|
|
"type": "baseline_recognition",
|
|
"acceptedSignals": ["before", "previous", "baseline", "normal level"],
|
|
"required": true,
|
|
"notes": "Knowing 'tripled to 47' requires knowing the original value (~16/day) to assess significance."
|
|
},
|
|
{
|
|
"id": "diag-09-beh-nq-evidence",
|
|
"description": "Seeks evidence distinguishing deployment effect from coincidence or another change",
|
|
"type": "next_question_target",
|
|
"acceptedSignals": ["evidence", "coincidence", "change", "deployment", "distinguishing"],
|
|
"required": true,
|
|
"notes": "A useful next question would ask about other changes that occurred around the same time."
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"id": "diag-10",
|
|
"input": "Some complaints involve production issues, but others say the delivery team is slow.",
|
|
"expectedPrimaryTypes": ["observed_problem"],
|
|
"acceptedPrimaryAlternatives": ["reported_claim"],
|
|
"expectedReasoningModes": ["decompose_aggregate", "identify_difference"],
|
|
"shouldIdentify": ["production issues", "delivery speed", "complaint types"],
|
|
"shouldNotInfer": ["production is worse than delivery", "the delivery team needs training", "both teams are underperforming equally"],
|
|
"description": "Paired with diag-01 — distinguishes subset complaints from aggregate claims.",
|
|
"expectedBehaviours": [
|
|
{
|
|
"id": "diag-10-beh-decomposition",
|
|
"description": "Decomposes complaints into at least two categories",
|
|
"type": "observation_recognition",
|
|
"acceptedSignals": ["production", "delivery", "categories", "types", "distinct"],
|
|
"required": true,
|
|
"notes": "The model should recognise these are separate issues that should not be merged."
|
|
},
|
|
{
|
|
"id": "diag-10-beh-no-merging",
|
|
"description": "Recognises production and delivery issues should not be merged without quantification",
|
|
"type": "metric_relationship",
|
|
"acceptedSignals": ["production", "delivery", "comparison", "quantify", "distinguish"],
|
|
"required": true,
|
|
"notes": "Without quantification the two complaint types cannot be compared or prioritised."
|
|
},
|
|
{
|
|
"id": "diag-10-beh-nq-quantify",
|
|
"description": "Next question should quantify or compare complaint categories",
|
|
"type": "next_question_target",
|
|
"acceptedSignals": ["how many", "proportion", "compare", "ratio", "breakdown"],
|
|
"required": true,
|
|
"notes": "A useful next question would ask what proportion of complaints fall into each category."
|
|
}
|
|
]
|
|
}
|
|
]
|