feat: use telemetry in guardrails
This commit is contained in:
@@ -1,3 +1,9 @@
|
|||||||
{"timestamp": "2026-06-02T14:38:58.532898+00:00", "event_type": "command", "target": "rdb status", "details": {}}
|
{"timestamp": "2026-06-02T14:38:58.532898+00:00", "event_type": "command", "target": "rdb status", "details": {}}
|
||||||
{"timestamp": "2026-06-02T14:43:05.281782+00:00", "event_type": "command", "target": "rdb status", "details": {}}
|
{"timestamp": "2026-06-02T14:43:05.281782+00:00", "event_type": "command", "target": "rdb status", "details": {}}
|
||||||
{"timestamp": "2026-06-02T14:47:33.841698+00:00", "event_type": "command", "target": "rdb status", "details": {}}
|
{"timestamp": "2026-06-02T14:47:33.841698+00:00", "event_type": "command", "target": "rdb status", "details": {}}
|
||||||
|
{"timestamp": "2026-06-02T14:49:48.692247+00:00", "event_type": "command", "target": "rdb status", "details": {}}
|
||||||
|
{"timestamp": "2026-06-02T14:49:51.767548+00:00", "event_type": "command", "target": "rdb next", "details": {}}
|
||||||
|
{"timestamp": "2026-06-02T14:49:58.009037+00:00", "event_type": "command", "target": "rdb prompt", "details": {}}
|
||||||
|
{"timestamp": "2026-06-02T15:21:45.848114+00:00", "event_type": "command", "target": "rdb status", "details": {}}
|
||||||
|
{"timestamp": "2026-06-02T15:40:37.716111+00:00", "event_type": "command", "target": "rdb guardrails", "details": {}}
|
||||||
|
{"timestamp": "2026-06-02T15:42:28.735379+00:00", "event_type": "command", "target": "rdb status", "details": {}}
|
||||||
|
|||||||
+1
-1
@@ -6,7 +6,7 @@ TASKS_READY
|
|||||||
|
|
||||||
## Current Task
|
## Current Task
|
||||||
|
|
||||||
TASK-013
|
TASK-014
|
||||||
|
|
||||||
## Instructions For Agent
|
## Instructions For Agent
|
||||||
|
|
||||||
|
|||||||
+1
-1
@@ -4,7 +4,7 @@ Current Stage: BUILDING
|
|||||||
Previous Stage: BOOTSTRAP_READY
|
Previous Stage: BOOTSTRAP_READY
|
||||||
Next Stage: REVIEW_READY
|
Next Stage: REVIEW_READY
|
||||||
|
|
||||||
Current Task: TASK-013
|
Current Task: TASK-014
|
||||||
Active Branch: main
|
Active Branch: main
|
||||||
|
|
||||||
Last Updated: 2026-06-02
|
Last Updated: 2026-06-02
|
||||||
|
|||||||
@@ -210,7 +210,7 @@ Definition of Done:
|
|||||||
|
|
||||||
## TASK-014 — Integrate telemetry with guardrails
|
## TASK-014 — Integrate telemetry with guardrails
|
||||||
|
|
||||||
Status: Todo
|
Status: Done
|
||||||
|
|
||||||
Goal:
|
Goal:
|
||||||
Use structured telemetry data in guardrail analysis.
|
Use structured telemetry data in guardrail analysis.
|
||||||
@@ -221,8 +221,16 @@ Guardrails currently rely on heuristics and markdown files rather than actual ac
|
|||||||
|
|
||||||
Acceptance Criteria:
|
Acceptance Criteria:
|
||||||
|
|
||||||
- Read telemetry events
|
- Read telemetry events ✓
|
||||||
- Detect repeated commands
|
- Detect repeated commands ✓
|
||||||
- Detect repeated reads when available
|
- Detect repeated reads when available ✓
|
||||||
- Fall back gracefully when telemetry is absent
|
- Fall back gracefully when telemetry is absent ✓
|
||||||
- Add/update tests
|
- Add/update tests ✓
|
||||||
|
|
||||||
|
Result:
|
||||||
|
|
||||||
|
Two new guardrail checks added:
|
||||||
|
- `check_repeated_commands_telemetry` — uses session-log.jsonl to detect repeated CLI commands (>3x)
|
||||||
|
- `check_repeated_reads_telemetry` — uses session-log.jsonl to detect repeated file reads (>3x)
|
||||||
|
Both integrate into `run_all_guardrails` alongside existing heuristic checks.
|
||||||
|
When telemetry data is absent, both return `"ok"` with an informative fallback message instead of failing.
|
||||||
|
|||||||
@@ -5,6 +5,8 @@ from datetime import datetime, timedelta
|
|||||||
import re
|
import re
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
|
from .telemetry import read_events
|
||||||
|
|
||||||
|
|
||||||
def _read_safe(root: Path, relative: str) -> str | None:
|
def _read_safe(root: Path, relative: str) -> str | None:
|
||||||
"""Read a file if it exists, otherwise return None."""
|
"""Read a file if it exists, otherwise return None."""
|
||||||
@@ -95,6 +97,66 @@ def check_repeated_commands(root: Path) -> dict:
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def check_repeated_commands_telemetry(root: Path) -> dict:
|
||||||
|
"""Detect repeated commands from telemetry events.
|
||||||
|
|
||||||
|
Reads structured session-log.jsonl to find command targets
|
||||||
|
that have been issued more than 3 times. Falls back gracefully
|
||||||
|
when the telemetry log is absent.
|
||||||
|
|
||||||
|
Returns a dict with keys: status, details.
|
||||||
|
"""
|
||||||
|
events = read_events(root)
|
||||||
|
if not events:
|
||||||
|
return {"status": "ok", "details": "No telemetry data available."}
|
||||||
|
|
||||||
|
command_events = [e for e in events if e.event_type == "command"]
|
||||||
|
if not command_events:
|
||||||
|
return {"status": "ok", "details": "No command events found in telemetry."}
|
||||||
|
|
||||||
|
counts = Counter(e.target for e in command_events)
|
||||||
|
repeated = {target: count for target, count in counts.items() if count > 3}
|
||||||
|
|
||||||
|
if not repeated:
|
||||||
|
return {"status": "ok", "details": "No significantly repeated commands detected in telemetry."}
|
||||||
|
|
||||||
|
items = ", ".join(f"{t} ({c}x)" for t, c in list(repeated.items())[:5])
|
||||||
|
return {
|
||||||
|
"status": "warning",
|
||||||
|
"details": f"Repeated commands: {items}",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def check_repeated_reads_telemetry(root: Path) -> dict:
|
||||||
|
"""Detect repeated file reads from telemetry events.
|
||||||
|
|
||||||
|
Reads structured session-log.jsonl to find read/read_file target events
|
||||||
|
that have been issued more than 3 times. Falls back gracefully
|
||||||
|
when the telemetry log is absent.
|
||||||
|
|
||||||
|
Returns a dict with keys: status, details.
|
||||||
|
"""
|
||||||
|
events = read_events(root)
|
||||||
|
if not events:
|
||||||
|
return {"status": "ok", "details": "No telemetry data available."}
|
||||||
|
|
||||||
|
read_events_list = [e for e in events if e.event_type in ("read", "read_file")]
|
||||||
|
if not read_events_list:
|
||||||
|
return {"status": "ok", "details": "No file read events found in telemetry."}
|
||||||
|
|
||||||
|
counts = Counter(e.target for e in read_events_list)
|
||||||
|
repeated = {target: count for target, count in counts.items() if count > 3}
|
||||||
|
|
||||||
|
if not repeated:
|
||||||
|
return {"status": "ok", "details": "No significantly repeated file reads detected in telemetry."}
|
||||||
|
|
||||||
|
items = ", ".join(f"{t} ({c}x)" for t, c in list(repeated.items())[:5])
|
||||||
|
return {
|
||||||
|
"status": "warning",
|
||||||
|
"details": f"Repeated file reads: {items}",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def check_no_recent_file_changes(root: Path) -> dict:
|
def check_no_recent_file_changes(root: Path) -> dict:
|
||||||
"""Flag if project files have not been modified in a long time (> 48 h).
|
"""Flag if project files have not been modified in a long time (> 48 h).
|
||||||
|
|
||||||
@@ -185,8 +247,10 @@ def check_run_log_updated(root: Path) -> dict:
|
|||||||
def run_all_guardrails(root: Path) -> dict:
|
def run_all_guardrails(root: Path) -> dict:
|
||||||
"""Run all guardrail checks and return combined results."""
|
"""Run all guardrail checks and return combined results."""
|
||||||
checks = {
|
checks = {
|
||||||
"Repeated reads": check_repeated_reads(root),
|
"Repeated reads (heuristics)": check_repeated_reads(root),
|
||||||
"Repeated commands": check_repeated_commands(root),
|
"Repeated commands (RUN_LOG)": check_repeated_commands(root),
|
||||||
|
"Repeated commands (telemetry)": check_repeated_commands_telemetry(root),
|
||||||
|
"Repeated file reads (telemetry)": check_repeated_reads_telemetry(root),
|
||||||
"No recent file changes": check_no_recent_file_changes(root),
|
"No recent file changes": check_no_recent_file_changes(root),
|
||||||
"No test run recorded": check_no_test_run_recorded(root),
|
"No test run recorded": check_no_test_run_recorded(root),
|
||||||
"Run log updated": check_run_log_updated(root),
|
"Run log updated": check_run_log_updated(root),
|
||||||
|
|||||||
@@ -7,6 +7,8 @@ import pytest
|
|||||||
from rdb_discovery.guardrails import (
|
from rdb_discovery.guardrails import (
|
||||||
check_repeated_reads,
|
check_repeated_reads,
|
||||||
check_repeated_commands,
|
check_repeated_commands,
|
||||||
|
check_repeated_commands_telemetry,
|
||||||
|
check_repeated_reads_telemetry,
|
||||||
check_no_recent_file_changes,
|
check_no_recent_file_changes,
|
||||||
check_no_test_run_recorded,
|
check_no_test_run_recorded,
|
||||||
check_run_log_updated,
|
check_run_log_updated,
|
||||||
@@ -79,6 +81,76 @@ def test_check_repeated_commands_warning(tmp_path: Path) -> None:
|
|||||||
assert result["status"] == "warning"
|
assert result["status"] == "warning"
|
||||||
|
|
||||||
|
|
||||||
|
# -- check_repeated_commands_telemetry --
|
||||||
|
|
||||||
|
def test_check_repeated_commands_telemetry_no_events(tmp_path: Path) -> None:
|
||||||
|
"""No telemetry events → status ok with fallback message."""
|
||||||
|
result = check_repeated_commands_telemetry(tmp_path)
|
||||||
|
assert result["status"] == "ok"
|
||||||
|
assert "No telemetry data available" in result["details"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_check_repeated_commands_telemetry_warning(tmp_path: Path) -> None:
|
||||||
|
"""Same command repeated >3 times in telemetry → warning."""
|
||||||
|
from rdb_discovery.telemetry import record_event
|
||||||
|
|
||||||
|
for _ in range(5):
|
||||||
|
record_event(tmp_path, "command", "rdb prompt")
|
||||||
|
result = check_repeated_commands_telemetry(tmp_path)
|
||||||
|
assert result["status"] == "warning"
|
||||||
|
assert "rdb prompt" in result["details"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_check_repeated_commands_telemetry_no_over_threshold(tmp_path: Path) -> None:
|
||||||
|
"""Same command repeated <=3 times → ok."""
|
||||||
|
from rdb_discovery.telemetry import record_event
|
||||||
|
|
||||||
|
for _ in range(3):
|
||||||
|
record_event(tmp_path, "command", "rdb prompt")
|
||||||
|
result = check_repeated_commands_telemetry(tmp_path)
|
||||||
|
assert result["status"] == "ok"
|
||||||
|
|
||||||
|
|
||||||
|
# -- check_repeated_reads_telemetry --
|
||||||
|
|
||||||
|
def test_check_repeated_reads_telemetry_no_events(tmp_path: Path) -> None:
|
||||||
|
"""No telemetry events → status ok with fallback message."""
|
||||||
|
result = check_repeated_reads_telemetry(tmp_path)
|
||||||
|
assert result["status"] == "ok"
|
||||||
|
assert "No telemetry data available" in result["details"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_check_repeated_reads_telemetry_no_read_events(tmp_path: Path) -> None:
|
||||||
|
"""Only command events, no read events → ok with message."""
|
||||||
|
from rdb_discovery.telemetry import record_event
|
||||||
|
|
||||||
|
record_event(tmp_path, "command", "rdb prompt")
|
||||||
|
result = check_repeated_reads_telemetry(tmp_path)
|
||||||
|
assert result["status"] == "ok"
|
||||||
|
assert "No file read events found" in result["details"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_check_repeated_reads_telemetry_warning(tmp_path: Path) -> None:
|
||||||
|
"""Same file read >3 times in telemetry → warning."""
|
||||||
|
from rdb_discovery.telemetry import record_event
|
||||||
|
|
||||||
|
for _ in range(5):
|
||||||
|
record_event(tmp_path, "read", "README.md")
|
||||||
|
result = check_repeated_reads_telemetry(tmp_path)
|
||||||
|
assert result["status"] == "warning"
|
||||||
|
assert "README.md" in result["details"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_check_repeated_reads_telemetry_no_over_threshold(tmp_path: Path) -> None:
|
||||||
|
"""Same file read <=3 times → ok."""
|
||||||
|
from rdb_discovery.telemetry import record_event
|
||||||
|
|
||||||
|
for _ in range(3):
|
||||||
|
record_event(tmp_path, "read", "TASKS.md")
|
||||||
|
result = check_repeated_reads_telemetry(tmp_path)
|
||||||
|
assert result["status"] == "ok"
|
||||||
|
|
||||||
|
|
||||||
# -- check_no_recent_file_changes --
|
# -- check_no_recent_file_changes --
|
||||||
|
|
||||||
def test_check_no_recent_file_changes_ok(tmp_path: Path) -> None:
|
def test_check_no_recent_file_changes_ok(tmp_path: Path) -> None:
|
||||||
|
|||||||
Reference in New Issue
Block a user