From 6eef97582de2be00db38392a062cc44bc35486a1 Mon Sep 17 00:00:00 2001 From: robbond Date: Tue, 2 Jun 2026 16:44:36 +0100 Subject: [PATCH] feat: use telemetry in guardrails --- .rdb/session-log.jsonl | 6 +++ AGENT_HANDOFF.md | 2 +- PROJECT_STATE.md | 2 +- TASKS.md | 20 ++++++--- src/rdb_discovery/guardrails.py | 68 ++++++++++++++++++++++++++++++- tests/test_guardrails.py | 72 +++++++++++++++++++++++++++++++++ 6 files changed, 160 insertions(+), 10 deletions(-) diff --git a/.rdb/session-log.jsonl b/.rdb/session-log.jsonl index 21e383a..9c67051 100644 --- a/.rdb/session-log.jsonl +++ b/.rdb/session-log.jsonl @@ -1,3 +1,9 @@ {"timestamp": "2026-06-02T14:38:58.532898+00:00", "event_type": "command", "target": "rdb status", "details": {}} {"timestamp": "2026-06-02T14:43:05.281782+00:00", "event_type": "command", "target": "rdb status", "details": {}} {"timestamp": "2026-06-02T14:47:33.841698+00:00", "event_type": "command", "target": "rdb status", "details": {}} +{"timestamp": "2026-06-02T14:49:48.692247+00:00", "event_type": "command", "target": "rdb status", "details": {}} +{"timestamp": "2026-06-02T14:49:51.767548+00:00", "event_type": "command", "target": "rdb next", "details": {}} +{"timestamp": "2026-06-02T14:49:58.009037+00:00", "event_type": "command", "target": "rdb prompt", "details": {}} +{"timestamp": "2026-06-02T15:21:45.848114+00:00", "event_type": "command", "target": "rdb status", "details": {}} +{"timestamp": "2026-06-02T15:40:37.716111+00:00", "event_type": "command", "target": "rdb guardrails", "details": {}} +{"timestamp": "2026-06-02T15:42:28.735379+00:00", "event_type": "command", "target": "rdb status", "details": {}} diff --git a/AGENT_HANDOFF.md b/AGENT_HANDOFF.md index 29fa714..174d4de 100644 --- a/AGENT_HANDOFF.md +++ b/AGENT_HANDOFF.md @@ -6,7 +6,7 @@ TASKS_READY ## Current Task -TASK-013 +TASK-014 ## Instructions For Agent diff --git a/PROJECT_STATE.md b/PROJECT_STATE.md index a3feeb8..bdb5e0f 100644 --- a/PROJECT_STATE.md +++ b/PROJECT_STATE.md @@ -4,7 +4,7 @@ Current Stage: BUILDING Previous Stage: BOOTSTRAP_READY Next Stage: REVIEW_READY -Current Task: TASK-013 +Current Task: TASK-014 Active Branch: main Last Updated: 2026-06-02 diff --git a/TASKS.md b/TASKS.md index 1656cc6..45cc275 100644 --- a/TASKS.md +++ b/TASKS.md @@ -210,7 +210,7 @@ Definition of Done: ## TASK-014 — Integrate telemetry with guardrails -Status: Todo +Status: Done Goal: Use structured telemetry data in guardrail analysis. @@ -221,8 +221,16 @@ Guardrails currently rely on heuristics and markdown files rather than actual ac Acceptance Criteria: -- Read telemetry events -- Detect repeated commands -- Detect repeated reads when available -- Fall back gracefully when telemetry is absent -- Add/update tests +- Read telemetry events ✓ +- Detect repeated commands ✓ +- Detect repeated reads when available ✓ +- Fall back gracefully when telemetry is absent ✓ +- Add/update tests ✓ + +Result: + +Two new guardrail checks added: +- `check_repeated_commands_telemetry` — uses session-log.jsonl to detect repeated CLI commands (>3x) +- `check_repeated_reads_telemetry` — uses session-log.jsonl to detect repeated file reads (>3x) +Both integrate into `run_all_guardrails` alongside existing heuristic checks. +When telemetry data is absent, both return `"ok"` with an informative fallback message instead of failing. diff --git a/src/rdb_discovery/guardrails.py b/src/rdb_discovery/guardrails.py index 1eb3f42..3a7c082 100644 --- a/src/rdb_discovery/guardrails.py +++ b/src/rdb_discovery/guardrails.py @@ -5,6 +5,8 @@ from datetime import datetime, timedelta import re from pathlib import Path +from .telemetry import read_events + def _read_safe(root: Path, relative: str) -> str | None: """Read a file if it exists, otherwise return None.""" @@ -95,6 +97,66 @@ def check_repeated_commands(root: Path) -> dict: } +def check_repeated_commands_telemetry(root: Path) -> dict: + """Detect repeated commands from telemetry events. + + Reads structured session-log.jsonl to find command targets + that have been issued more than 3 times. Falls back gracefully + when the telemetry log is absent. + + Returns a dict with keys: status, details. + """ + events = read_events(root) + if not events: + return {"status": "ok", "details": "No telemetry data available."} + + command_events = [e for e in events if e.event_type == "command"] + if not command_events: + return {"status": "ok", "details": "No command events found in telemetry."} + + counts = Counter(e.target for e in command_events) + repeated = {target: count for target, count in counts.items() if count > 3} + + if not repeated: + return {"status": "ok", "details": "No significantly repeated commands detected in telemetry."} + + items = ", ".join(f"{t} ({c}x)" for t, c in list(repeated.items())[:5]) + return { + "status": "warning", + "details": f"Repeated commands: {items}", + } + + +def check_repeated_reads_telemetry(root: Path) -> dict: + """Detect repeated file reads from telemetry events. + + Reads structured session-log.jsonl to find read/read_file target events + that have been issued more than 3 times. Falls back gracefully + when the telemetry log is absent. + + Returns a dict with keys: status, details. + """ + events = read_events(root) + if not events: + return {"status": "ok", "details": "No telemetry data available."} + + read_events_list = [e for e in events if e.event_type in ("read", "read_file")] + if not read_events_list: + return {"status": "ok", "details": "No file read events found in telemetry."} + + counts = Counter(e.target for e in read_events_list) + repeated = {target: count for target, count in counts.items() if count > 3} + + if not repeated: + return {"status": "ok", "details": "No significantly repeated file reads detected in telemetry."} + + items = ", ".join(f"{t} ({c}x)" for t, c in list(repeated.items())[:5]) + return { + "status": "warning", + "details": f"Repeated file reads: {items}", + } + + def check_no_recent_file_changes(root: Path) -> dict: """Flag if project files have not been modified in a long time (> 48 h). @@ -185,8 +247,10 @@ def check_run_log_updated(root: Path) -> dict: def run_all_guardrails(root: Path) -> dict: """Run all guardrail checks and return combined results.""" checks = { - "Repeated reads": check_repeated_reads(root), - "Repeated commands": check_repeated_commands(root), + "Repeated reads (heuristics)": check_repeated_reads(root), + "Repeated commands (RUN_LOG)": check_repeated_commands(root), + "Repeated commands (telemetry)": check_repeated_commands_telemetry(root), + "Repeated file reads (telemetry)": check_repeated_reads_telemetry(root), "No recent file changes": check_no_recent_file_changes(root), "No test run recorded": check_no_test_run_recorded(root), "Run log updated": check_run_log_updated(root), diff --git a/tests/test_guardrails.py b/tests/test_guardrails.py index 46a13b6..2a6e182 100644 --- a/tests/test_guardrails.py +++ b/tests/test_guardrails.py @@ -7,6 +7,8 @@ import pytest from rdb_discovery.guardrails import ( check_repeated_reads, check_repeated_commands, + check_repeated_commands_telemetry, + check_repeated_reads_telemetry, check_no_recent_file_changes, check_no_test_run_recorded, check_run_log_updated, @@ -79,6 +81,76 @@ def test_check_repeated_commands_warning(tmp_path: Path) -> None: assert result["status"] == "warning" +# -- check_repeated_commands_telemetry -- + +def test_check_repeated_commands_telemetry_no_events(tmp_path: Path) -> None: + """No telemetry events → status ok with fallback message.""" + result = check_repeated_commands_telemetry(tmp_path) + assert result["status"] == "ok" + assert "No telemetry data available" in result["details"] + + +def test_check_repeated_commands_telemetry_warning(tmp_path: Path) -> None: + """Same command repeated >3 times in telemetry → warning.""" + from rdb_discovery.telemetry import record_event + + for _ in range(5): + record_event(tmp_path, "command", "rdb prompt") + result = check_repeated_commands_telemetry(tmp_path) + assert result["status"] == "warning" + assert "rdb prompt" in result["details"] + + +def test_check_repeated_commands_telemetry_no_over_threshold(tmp_path: Path) -> None: + """Same command repeated <=3 times → ok.""" + from rdb_discovery.telemetry import record_event + + for _ in range(3): + record_event(tmp_path, "command", "rdb prompt") + result = check_repeated_commands_telemetry(tmp_path) + assert result["status"] == "ok" + + +# -- check_repeated_reads_telemetry -- + +def test_check_repeated_reads_telemetry_no_events(tmp_path: Path) -> None: + """No telemetry events → status ok with fallback message.""" + result = check_repeated_reads_telemetry(tmp_path) + assert result["status"] == "ok" + assert "No telemetry data available" in result["details"] + + +def test_check_repeated_reads_telemetry_no_read_events(tmp_path: Path) -> None: + """Only command events, no read events → ok with message.""" + from rdb_discovery.telemetry import record_event + + record_event(tmp_path, "command", "rdb prompt") + result = check_repeated_reads_telemetry(tmp_path) + assert result["status"] == "ok" + assert "No file read events found" in result["details"] + + +def test_check_repeated_reads_telemetry_warning(tmp_path: Path) -> None: + """Same file read >3 times in telemetry → warning.""" + from rdb_discovery.telemetry import record_event + + for _ in range(5): + record_event(tmp_path, "read", "README.md") + result = check_repeated_reads_telemetry(tmp_path) + assert result["status"] == "warning" + assert "README.md" in result["details"] + + +def test_check_repeated_reads_telemetry_no_over_threshold(tmp_path: Path) -> None: + """Same file read <=3 times → ok.""" + from rdb_discovery.telemetry import record_event + + for _ in range(3): + record_event(tmp_path, "read", "TASKS.md") + result = check_repeated_reads_telemetry(tmp_path) + assert result["status"] == "ok" + + # -- check_no_recent_file_changes -- def test_check_no_recent_file_changes_ok(tmp_path: Path) -> None: