feat: use telemetry in guardrails
This commit is contained in:
@@ -1,3 +1,9 @@
|
||||
{"timestamp": "2026-06-02T14:38:58.532898+00:00", "event_type": "command", "target": "rdb status", "details": {}}
|
||||
{"timestamp": "2026-06-02T14:43:05.281782+00:00", "event_type": "command", "target": "rdb status", "details": {}}
|
||||
{"timestamp": "2026-06-02T14:47:33.841698+00:00", "event_type": "command", "target": "rdb status", "details": {}}
|
||||
{"timestamp": "2026-06-02T14:49:48.692247+00:00", "event_type": "command", "target": "rdb status", "details": {}}
|
||||
{"timestamp": "2026-06-02T14:49:51.767548+00:00", "event_type": "command", "target": "rdb next", "details": {}}
|
||||
{"timestamp": "2026-06-02T14:49:58.009037+00:00", "event_type": "command", "target": "rdb prompt", "details": {}}
|
||||
{"timestamp": "2026-06-02T15:21:45.848114+00:00", "event_type": "command", "target": "rdb status", "details": {}}
|
||||
{"timestamp": "2026-06-02T15:40:37.716111+00:00", "event_type": "command", "target": "rdb guardrails", "details": {}}
|
||||
{"timestamp": "2026-06-02T15:42:28.735379+00:00", "event_type": "command", "target": "rdb status", "details": {}}
|
||||
|
||||
+1
-1
@@ -6,7 +6,7 @@ TASKS_READY
|
||||
|
||||
## Current Task
|
||||
|
||||
TASK-013
|
||||
TASK-014
|
||||
|
||||
## Instructions For Agent
|
||||
|
||||
|
||||
+1
-1
@@ -4,7 +4,7 @@ Current Stage: BUILDING
|
||||
Previous Stage: BOOTSTRAP_READY
|
||||
Next Stage: REVIEW_READY
|
||||
|
||||
Current Task: TASK-013
|
||||
Current Task: TASK-014
|
||||
Active Branch: main
|
||||
|
||||
Last Updated: 2026-06-02
|
||||
|
||||
@@ -210,7 +210,7 @@ Definition of Done:
|
||||
|
||||
## TASK-014 — Integrate telemetry with guardrails
|
||||
|
||||
Status: Todo
|
||||
Status: Done
|
||||
|
||||
Goal:
|
||||
Use structured telemetry data in guardrail analysis.
|
||||
@@ -221,8 +221,16 @@ Guardrails currently rely on heuristics and markdown files rather than actual ac
|
||||
|
||||
Acceptance Criteria:
|
||||
|
||||
- Read telemetry events
|
||||
- Detect repeated commands
|
||||
- Detect repeated reads when available
|
||||
- Fall back gracefully when telemetry is absent
|
||||
- Add/update tests
|
||||
- Read telemetry events ✓
|
||||
- Detect repeated commands ✓
|
||||
- Detect repeated reads when available ✓
|
||||
- Fall back gracefully when telemetry is absent ✓
|
||||
- Add/update tests ✓
|
||||
|
||||
Result:
|
||||
|
||||
Two new guardrail checks added:
|
||||
- `check_repeated_commands_telemetry` — uses session-log.jsonl to detect repeated CLI commands (>3x)
|
||||
- `check_repeated_reads_telemetry` — uses session-log.jsonl to detect repeated file reads (>3x)
|
||||
Both integrate into `run_all_guardrails` alongside existing heuristic checks.
|
||||
When telemetry data is absent, both return `"ok"` with an informative fallback message instead of failing.
|
||||
|
||||
@@ -5,6 +5,8 @@ from datetime import datetime, timedelta
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from .telemetry import read_events
|
||||
|
||||
|
||||
def _read_safe(root: Path, relative: str) -> str | None:
|
||||
"""Read a file if it exists, otherwise return None."""
|
||||
@@ -95,6 +97,66 @@ def check_repeated_commands(root: Path) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def check_repeated_commands_telemetry(root: Path) -> dict:
|
||||
"""Detect repeated commands from telemetry events.
|
||||
|
||||
Reads structured session-log.jsonl to find command targets
|
||||
that have been issued more than 3 times. Falls back gracefully
|
||||
when the telemetry log is absent.
|
||||
|
||||
Returns a dict with keys: status, details.
|
||||
"""
|
||||
events = read_events(root)
|
||||
if not events:
|
||||
return {"status": "ok", "details": "No telemetry data available."}
|
||||
|
||||
command_events = [e for e in events if e.event_type == "command"]
|
||||
if not command_events:
|
||||
return {"status": "ok", "details": "No command events found in telemetry."}
|
||||
|
||||
counts = Counter(e.target for e in command_events)
|
||||
repeated = {target: count for target, count in counts.items() if count > 3}
|
||||
|
||||
if not repeated:
|
||||
return {"status": "ok", "details": "No significantly repeated commands detected in telemetry."}
|
||||
|
||||
items = ", ".join(f"{t} ({c}x)" for t, c in list(repeated.items())[:5])
|
||||
return {
|
||||
"status": "warning",
|
||||
"details": f"Repeated commands: {items}",
|
||||
}
|
||||
|
||||
|
||||
def check_repeated_reads_telemetry(root: Path) -> dict:
|
||||
"""Detect repeated file reads from telemetry events.
|
||||
|
||||
Reads structured session-log.jsonl to find read/read_file target events
|
||||
that have been issued more than 3 times. Falls back gracefully
|
||||
when the telemetry log is absent.
|
||||
|
||||
Returns a dict with keys: status, details.
|
||||
"""
|
||||
events = read_events(root)
|
||||
if not events:
|
||||
return {"status": "ok", "details": "No telemetry data available."}
|
||||
|
||||
read_events_list = [e for e in events if e.event_type in ("read", "read_file")]
|
||||
if not read_events_list:
|
||||
return {"status": "ok", "details": "No file read events found in telemetry."}
|
||||
|
||||
counts = Counter(e.target for e in read_events_list)
|
||||
repeated = {target: count for target, count in counts.items() if count > 3}
|
||||
|
||||
if not repeated:
|
||||
return {"status": "ok", "details": "No significantly repeated file reads detected in telemetry."}
|
||||
|
||||
items = ", ".join(f"{t} ({c}x)" for t, c in list(repeated.items())[:5])
|
||||
return {
|
||||
"status": "warning",
|
||||
"details": f"Repeated file reads: {items}",
|
||||
}
|
||||
|
||||
|
||||
def check_no_recent_file_changes(root: Path) -> dict:
|
||||
"""Flag if project files have not been modified in a long time (> 48 h).
|
||||
|
||||
@@ -185,8 +247,10 @@ def check_run_log_updated(root: Path) -> dict:
|
||||
def run_all_guardrails(root: Path) -> dict:
|
||||
"""Run all guardrail checks and return combined results."""
|
||||
checks = {
|
||||
"Repeated reads": check_repeated_reads(root),
|
||||
"Repeated commands": check_repeated_commands(root),
|
||||
"Repeated reads (heuristics)": check_repeated_reads(root),
|
||||
"Repeated commands (RUN_LOG)": check_repeated_commands(root),
|
||||
"Repeated commands (telemetry)": check_repeated_commands_telemetry(root),
|
||||
"Repeated file reads (telemetry)": check_repeated_reads_telemetry(root),
|
||||
"No recent file changes": check_no_recent_file_changes(root),
|
||||
"No test run recorded": check_no_test_run_recorded(root),
|
||||
"Run log updated": check_run_log_updated(root),
|
||||
|
||||
@@ -7,6 +7,8 @@ import pytest
|
||||
from rdb_discovery.guardrails import (
|
||||
check_repeated_reads,
|
||||
check_repeated_commands,
|
||||
check_repeated_commands_telemetry,
|
||||
check_repeated_reads_telemetry,
|
||||
check_no_recent_file_changes,
|
||||
check_no_test_run_recorded,
|
||||
check_run_log_updated,
|
||||
@@ -79,6 +81,76 @@ def test_check_repeated_commands_warning(tmp_path: Path) -> None:
|
||||
assert result["status"] == "warning"
|
||||
|
||||
|
||||
# -- check_repeated_commands_telemetry --
|
||||
|
||||
def test_check_repeated_commands_telemetry_no_events(tmp_path: Path) -> None:
|
||||
"""No telemetry events → status ok with fallback message."""
|
||||
result = check_repeated_commands_telemetry(tmp_path)
|
||||
assert result["status"] == "ok"
|
||||
assert "No telemetry data available" in result["details"]
|
||||
|
||||
|
||||
def test_check_repeated_commands_telemetry_warning(tmp_path: Path) -> None:
|
||||
"""Same command repeated >3 times in telemetry → warning."""
|
||||
from rdb_discovery.telemetry import record_event
|
||||
|
||||
for _ in range(5):
|
||||
record_event(tmp_path, "command", "rdb prompt")
|
||||
result = check_repeated_commands_telemetry(tmp_path)
|
||||
assert result["status"] == "warning"
|
||||
assert "rdb prompt" in result["details"]
|
||||
|
||||
|
||||
def test_check_repeated_commands_telemetry_no_over_threshold(tmp_path: Path) -> None:
|
||||
"""Same command repeated <=3 times → ok."""
|
||||
from rdb_discovery.telemetry import record_event
|
||||
|
||||
for _ in range(3):
|
||||
record_event(tmp_path, "command", "rdb prompt")
|
||||
result = check_repeated_commands_telemetry(tmp_path)
|
||||
assert result["status"] == "ok"
|
||||
|
||||
|
||||
# -- check_repeated_reads_telemetry --
|
||||
|
||||
def test_check_repeated_reads_telemetry_no_events(tmp_path: Path) -> None:
|
||||
"""No telemetry events → status ok with fallback message."""
|
||||
result = check_repeated_reads_telemetry(tmp_path)
|
||||
assert result["status"] == "ok"
|
||||
assert "No telemetry data available" in result["details"]
|
||||
|
||||
|
||||
def test_check_repeated_reads_telemetry_no_read_events(tmp_path: Path) -> None:
|
||||
"""Only command events, no read events → ok with message."""
|
||||
from rdb_discovery.telemetry import record_event
|
||||
|
||||
record_event(tmp_path, "command", "rdb prompt")
|
||||
result = check_repeated_reads_telemetry(tmp_path)
|
||||
assert result["status"] == "ok"
|
||||
assert "No file read events found" in result["details"]
|
||||
|
||||
|
||||
def test_check_repeated_reads_telemetry_warning(tmp_path: Path) -> None:
|
||||
"""Same file read >3 times in telemetry → warning."""
|
||||
from rdb_discovery.telemetry import record_event
|
||||
|
||||
for _ in range(5):
|
||||
record_event(tmp_path, "read", "README.md")
|
||||
result = check_repeated_reads_telemetry(tmp_path)
|
||||
assert result["status"] == "warning"
|
||||
assert "README.md" in result["details"]
|
||||
|
||||
|
||||
def test_check_repeated_reads_telemetry_no_over_threshold(tmp_path: Path) -> None:
|
||||
"""Same file read <=3 times → ok."""
|
||||
from rdb_discovery.telemetry import record_event
|
||||
|
||||
for _ in range(3):
|
||||
record_event(tmp_path, "read", "TASKS.md")
|
||||
result = check_repeated_reads_telemetry(tmp_path)
|
||||
assert result["status"] == "ok"
|
||||
|
||||
|
||||
# -- check_no_recent_file_changes --
|
||||
|
||||
def test_check_no_recent_file_changes_ok(tmp_path: Path) -> None:
|
||||
|
||||
Reference in New Issue
Block a user