feat: use telemetry in guardrails

This commit is contained in:
2026-06-02 16:44:36 +01:00
parent 2fcc822394
commit 6eef97582d
6 changed files with 160 additions and 10 deletions
+6
View File
@@ -1,3 +1,9 @@
{"timestamp": "2026-06-02T14:38:58.532898+00:00", "event_type": "command", "target": "rdb status", "details": {}}
{"timestamp": "2026-06-02T14:43:05.281782+00:00", "event_type": "command", "target": "rdb status", "details": {}}
{"timestamp": "2026-06-02T14:47:33.841698+00:00", "event_type": "command", "target": "rdb status", "details": {}}
{"timestamp": "2026-06-02T14:49:48.692247+00:00", "event_type": "command", "target": "rdb status", "details": {}}
{"timestamp": "2026-06-02T14:49:51.767548+00:00", "event_type": "command", "target": "rdb next", "details": {}}
{"timestamp": "2026-06-02T14:49:58.009037+00:00", "event_type": "command", "target": "rdb prompt", "details": {}}
{"timestamp": "2026-06-02T15:21:45.848114+00:00", "event_type": "command", "target": "rdb status", "details": {}}
{"timestamp": "2026-06-02T15:40:37.716111+00:00", "event_type": "command", "target": "rdb guardrails", "details": {}}
{"timestamp": "2026-06-02T15:42:28.735379+00:00", "event_type": "command", "target": "rdb status", "details": {}}
+1 -1
View File
@@ -6,7 +6,7 @@ TASKS_READY
## Current Task
TASK-013
TASK-014
## Instructions For Agent
+1 -1
View File
@@ -4,7 +4,7 @@ Current Stage: BUILDING
Previous Stage: BOOTSTRAP_READY
Next Stage: REVIEW_READY
Current Task: TASK-013
Current Task: TASK-014
Active Branch: main
Last Updated: 2026-06-02
+14 -6
View File
@@ -210,7 +210,7 @@ Definition of Done:
## TASK-014 — Integrate telemetry with guardrails
Status: Todo
Status: Done
Goal:
Use structured telemetry data in guardrail analysis.
@@ -221,8 +221,16 @@ Guardrails currently rely on heuristics and markdown files rather than actual ac
Acceptance Criteria:
- Read telemetry events
- Detect repeated commands
- Detect repeated reads when available
- Fall back gracefully when telemetry is absent
- Add/update tests
- Read telemetry events
- Detect repeated commands
- Detect repeated reads when available
- Fall back gracefully when telemetry is absent
- Add/update tests
Result:
Two new guardrail checks added:
- `check_repeated_commands_telemetry` — uses session-log.jsonl to detect repeated CLI commands (>3x)
- `check_repeated_reads_telemetry` — uses session-log.jsonl to detect repeated file reads (>3x)
Both integrate into `run_all_guardrails` alongside existing heuristic checks.
When telemetry data is absent, both return `"ok"` with an informative fallback message instead of failing.
+66 -2
View File
@@ -5,6 +5,8 @@ from datetime import datetime, timedelta
import re
from pathlib import Path
from .telemetry import read_events
def _read_safe(root: Path, relative: str) -> str | None:
"""Read a file if it exists, otherwise return None."""
@@ -95,6 +97,66 @@ def check_repeated_commands(root: Path) -> dict:
}
def check_repeated_commands_telemetry(root: Path) -> dict:
"""Detect repeated commands from telemetry events.
Reads structured session-log.jsonl to find command targets
that have been issued more than 3 times. Falls back gracefully
when the telemetry log is absent.
Returns a dict with keys: status, details.
"""
events = read_events(root)
if not events:
return {"status": "ok", "details": "No telemetry data available."}
command_events = [e for e in events if e.event_type == "command"]
if not command_events:
return {"status": "ok", "details": "No command events found in telemetry."}
counts = Counter(e.target for e in command_events)
repeated = {target: count for target, count in counts.items() if count > 3}
if not repeated:
return {"status": "ok", "details": "No significantly repeated commands detected in telemetry."}
items = ", ".join(f"{t} ({c}x)" for t, c in list(repeated.items())[:5])
return {
"status": "warning",
"details": f"Repeated commands: {items}",
}
def check_repeated_reads_telemetry(root: Path) -> dict:
"""Detect repeated file reads from telemetry events.
Reads structured session-log.jsonl to find read/read_file target events
that have been issued more than 3 times. Falls back gracefully
when the telemetry log is absent.
Returns a dict with keys: status, details.
"""
events = read_events(root)
if not events:
return {"status": "ok", "details": "No telemetry data available."}
read_events_list = [e for e in events if e.event_type in ("read", "read_file")]
if not read_events_list:
return {"status": "ok", "details": "No file read events found in telemetry."}
counts = Counter(e.target for e in read_events_list)
repeated = {target: count for target, count in counts.items() if count > 3}
if not repeated:
return {"status": "ok", "details": "No significantly repeated file reads detected in telemetry."}
items = ", ".join(f"{t} ({c}x)" for t, c in list(repeated.items())[:5])
return {
"status": "warning",
"details": f"Repeated file reads: {items}",
}
def check_no_recent_file_changes(root: Path) -> dict:
"""Flag if project files have not been modified in a long time (> 48 h).
@@ -185,8 +247,10 @@ def check_run_log_updated(root: Path) -> dict:
def run_all_guardrails(root: Path) -> dict:
"""Run all guardrail checks and return combined results."""
checks = {
"Repeated reads": check_repeated_reads(root),
"Repeated commands": check_repeated_commands(root),
"Repeated reads (heuristics)": check_repeated_reads(root),
"Repeated commands (RUN_LOG)": check_repeated_commands(root),
"Repeated commands (telemetry)": check_repeated_commands_telemetry(root),
"Repeated file reads (telemetry)": check_repeated_reads_telemetry(root),
"No recent file changes": check_no_recent_file_changes(root),
"No test run recorded": check_no_test_run_recorded(root),
"Run log updated": check_run_log_updated(root),
+72
View File
@@ -7,6 +7,8 @@ import pytest
from rdb_discovery.guardrails import (
check_repeated_reads,
check_repeated_commands,
check_repeated_commands_telemetry,
check_repeated_reads_telemetry,
check_no_recent_file_changes,
check_no_test_run_recorded,
check_run_log_updated,
@@ -79,6 +81,76 @@ def test_check_repeated_commands_warning(tmp_path: Path) -> None:
assert result["status"] == "warning"
# -- check_repeated_commands_telemetry --
def test_check_repeated_commands_telemetry_no_events(tmp_path: Path) -> None:
"""No telemetry events → status ok with fallback message."""
result = check_repeated_commands_telemetry(tmp_path)
assert result["status"] == "ok"
assert "No telemetry data available" in result["details"]
def test_check_repeated_commands_telemetry_warning(tmp_path: Path) -> None:
"""Same command repeated >3 times in telemetry → warning."""
from rdb_discovery.telemetry import record_event
for _ in range(5):
record_event(tmp_path, "command", "rdb prompt")
result = check_repeated_commands_telemetry(tmp_path)
assert result["status"] == "warning"
assert "rdb prompt" in result["details"]
def test_check_repeated_commands_telemetry_no_over_threshold(tmp_path: Path) -> None:
"""Same command repeated <=3 times → ok."""
from rdb_discovery.telemetry import record_event
for _ in range(3):
record_event(tmp_path, "command", "rdb prompt")
result = check_repeated_commands_telemetry(tmp_path)
assert result["status"] == "ok"
# -- check_repeated_reads_telemetry --
def test_check_repeated_reads_telemetry_no_events(tmp_path: Path) -> None:
"""No telemetry events → status ok with fallback message."""
result = check_repeated_reads_telemetry(tmp_path)
assert result["status"] == "ok"
assert "No telemetry data available" in result["details"]
def test_check_repeated_reads_telemetry_no_read_events(tmp_path: Path) -> None:
"""Only command events, no read events → ok with message."""
from rdb_discovery.telemetry import record_event
record_event(tmp_path, "command", "rdb prompt")
result = check_repeated_reads_telemetry(tmp_path)
assert result["status"] == "ok"
assert "No file read events found" in result["details"]
def test_check_repeated_reads_telemetry_warning(tmp_path: Path) -> None:
"""Same file read >3 times in telemetry → warning."""
from rdb_discovery.telemetry import record_event
for _ in range(5):
record_event(tmp_path, "read", "README.md")
result = check_repeated_reads_telemetry(tmp_path)
assert result["status"] == "warning"
assert "README.md" in result["details"]
def test_check_repeated_reads_telemetry_no_over_threshold(tmp_path: Path) -> None:
"""Same file read <=3 times → ok."""
from rdb_discovery.telemetry import record_event
for _ in range(3):
record_event(tmp_path, "read", "TASKS.md")
result = check_repeated_reads_telemetry(tmp_path)
assert result["status"] == "ok"
# -- check_no_recent_file_changes --
def test_check_no_recent_file_changes_ok(tmp_path: Path) -> None: