feat: improve generated agent prompts
This commit is contained in:
@@ -28,18 +28,23 @@ From the repository root, run:
|
||||
```bash
|
||||
source .venv/bin/activate
|
||||
python -m pytest
|
||||
```
|
||||
|
||||
Do not search the filesystem for pytest.
|
||||
Do not create a new virtual environment unless explicitly asked.
|
||||
|
||||
If pytest is unavailable, run:
|
||||
|
||||
```bash
|
||||
pip install -e '.[dev]'
|
||||
python -m pytest
|
||||
Validation Commands
|
||||
```
|
||||
|
||||
## Validation Commands
|
||||
|
||||
For normal task validation, run:
|
||||
|
||||
```bash
|
||||
python -m pytest
|
||||
rdb status
|
||||
rdb next
|
||||
|
||||
@@ -116,7 +116,7 @@ Acceptance Criteria:
|
||||
|
||||
## TASK-007 — Improve generated agent prompts
|
||||
|
||||
Status: Todo
|
||||
Status: Done
|
||||
|
||||
Goal:
|
||||
Make `rdb prompt` produce smaller, more direct prompts for Claude Code/local LLM agents.
|
||||
@@ -130,21 +130,59 @@ Acceptance Criteria:
|
||||
- Prompt limits scope to one small implementation step
|
||||
- Add/update tests
|
||||
|
||||
## TASK-012 — Agent telemetry
|
||||
## TASK-012 — Agent telemetry foundation
|
||||
|
||||
Status: Todo
|
||||
|
||||
Goal:
|
||||
Capture actual agent actions rather than inferring behaviour
|
||||
from documentation and run logs.
|
||||
Create a minimal telemetry writer for future agent activity tracking.
|
||||
|
||||
Examples:
|
||||
Implementation Gap:
|
||||
There is currently no persistent session log for recording agent activity. Guardrails infer behaviour from markdown and RUN_LOG.md instead of structured telemetry.
|
||||
|
||||
- file reads
|
||||
- file writes
|
||||
- command execution
|
||||
- test execution
|
||||
- git activity
|
||||
Acceptance Criteria:
|
||||
|
||||
Output:
|
||||
.rdb/session-log.jsonl
|
||||
- Add a telemetry module
|
||||
- Create `.rdb/session-log.jsonl` when recording an event
|
||||
- Record JSONL events with:
|
||||
- timestamp
|
||||
- event_type
|
||||
- target
|
||||
- details
|
||||
- Add tests for writing and appending telemetry events
|
||||
- Do not integrate telemetry into every CLI command yet
|
||||
|
||||
## TASK-013 — Log CLI command execution
|
||||
|
||||
Status: Todo
|
||||
|
||||
Goal:
|
||||
Record rdb CLI command execution into telemetry.
|
||||
|
||||
Implementation Gap:
|
||||
Telemetry writer exists, but CLI commands do not yet record command execution events.
|
||||
|
||||
Acceptance Criteria:
|
||||
|
||||
- Record command execution events
|
||||
- Include command name
|
||||
- Include success/failure where practical
|
||||
- Add/update tests
|
||||
|
||||
## TASK-014 — Use telemetry in guardrails
|
||||
|
||||
Status: Todo
|
||||
|
||||
Goal:
|
||||
Make `rdb guardrails` use structured telemetry when available.
|
||||
|
||||
Implementation Gap:
|
||||
Guardrails currently rely on markdown and file mtimes rather than `.rdb/session-log.jsonl`.
|
||||
|
||||
Acceptance Criteria:
|
||||
|
||||
- Read telemetry events when present
|
||||
- Detect repeated reads from telemetry
|
||||
- Detect repeated commands from telemetry
|
||||
- Fall back gracefully if telemetry is missing
|
||||
- Add/update tests
|
||||
|
||||
@@ -10,6 +10,7 @@ STATUS_RE = re.compile(r"^Status:\s*(.+)$", re.MULTILINE)
|
||||
|
||||
GOAL_RE = re.compile(r"^Goal:\s*(.+)", re.MULTILINE)
|
||||
AC_RE = re.compile(r"^- (.+)$", re.MULTILINE)
|
||||
GAP_RE = re.compile(r"^Implementation Gap:\s*(.+)", re.MULTILINE)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -28,6 +29,12 @@ class Task:
|
||||
def ac_lines(self) -> list[str]:
|
||||
return AC_RE.findall(self.body)
|
||||
|
||||
def implementation_gap(self, root: Path) -> str | None:
|
||||
match = GAP_RE.search(self.body)
|
||||
if match and match.group(1).strip():
|
||||
return match.group(1).strip()
|
||||
return None
|
||||
|
||||
|
||||
def read_tasks(root: Path) -> list[Task]:
|
||||
tasks_path = root / "TASKS.md"
|
||||
@@ -92,6 +99,12 @@ CONTEXT_FILES_TO_READ = [
|
||||
"context/agent-guidelines.md",
|
||||
]
|
||||
|
||||
TEST_CMD_SECTION_RE = re.compile(
|
||||
r"^##\s*Test Commands\s*\n([\s\S]*?)(?=^##|\Z)",
|
||||
re.MULTILINE,
|
||||
)
|
||||
BASH_BLOCK_RE = re.compile(r"```bash\n(.*?)```", re.DOTALL)
|
||||
|
||||
|
||||
def _read_file_safe(root: Path, relative: str) -> str:
|
||||
path = root / relative
|
||||
@@ -100,6 +113,55 @@ def _read_file_safe(root: Path, relative: str) -> str:
|
||||
return f"# {relative} — not found"
|
||||
|
||||
|
||||
def _extract_test_commands_from_claude(root: Path) -> list[str]:
|
||||
"""Extract bash test commands from CLAUDE.md Test Commands section."""
|
||||
|
||||
def _looks_like_command(line: str) -> bool:
|
||||
"""Heuristic: keep lines that look like shell commands."""
|
||||
if not line.strip():
|
||||
return False
|
||||
lower = line.lower()
|
||||
# Skip markdown, prose, headings
|
||||
if any(lower.startswith(p) for p in ("# ", "- ", "if ", "do not", "for ", "use ")) or lower in (
|
||||
"validation commands",
|
||||
):
|
||||
return False
|
||||
# Skip lines that look like sentences (contain spaces followed by lowercase words)
|
||||
# but keep things like `source .venv/bin/activate` and `pip install ...`
|
||||
parts = line.split()
|
||||
if len(parts) <= 1:
|
||||
return True
|
||||
# If the first word is a known shell builtin / command prefix, accept it
|
||||
known_prefixes = ("source", "cd", "ls", "cp", "mv", "rm", "mkdir", "echo", "grep",
|
||||
"git", "pip", "python", "pytest", "rdb", "cat", "head", "tail",
|
||||
"sed", "awk", "find", "install")
|
||||
return parts[0].lower() in known_prefixes
|
||||
|
||||
claude_path = root / "CLAUDE.md"
|
||||
if not claude_path.exists():
|
||||
return []
|
||||
|
||||
text = claude_path.read_text(encoding="utf-8")
|
||||
section_match = TEST_CMD_SECTION_RE.search(text)
|
||||
if not section_match:
|
||||
return []
|
||||
|
||||
section_text = section_match.group(1)
|
||||
blocks = BASH_BLOCK_RE.findall(section_text)
|
||||
seen: set[str] = set()
|
||||
commands: list[str] = []
|
||||
for block in blocks:
|
||||
for line in block.strip().splitlines():
|
||||
stripped = line.strip()
|
||||
if not stripped:
|
||||
continue
|
||||
if _looks_like_command(stripped):
|
||||
if stripped not in seen:
|
||||
seen.add(stripped)
|
||||
commands.append(stripped)
|
||||
return commands
|
||||
|
||||
|
||||
def generate_agent_prompt(root: Path) -> str:
|
||||
task = get_next_task(root)
|
||||
if not task:
|
||||
@@ -128,9 +190,20 @@ def generate_agent_prompt(root: Path) -> str:
|
||||
f"Status: {task.status}",
|
||||
"",
|
||||
f"Goal:\n{task.goal(root)}",
|
||||
]
|
||||
|
||||
gap = task.implementation_gap(root)
|
||||
if gap:
|
||||
sections.extend([
|
||||
"",
|
||||
"Implementation Gap:",
|
||||
gap,
|
||||
])
|
||||
|
||||
sections.extend([
|
||||
"",
|
||||
"Acceptance Criteria:",
|
||||
]
|
||||
])
|
||||
|
||||
for line in task.ac_lines():
|
||||
sections.append(f"- {line}")
|
||||
@@ -141,8 +214,25 @@ def generate_agent_prompt(root: Path) -> str:
|
||||
"",
|
||||
"# Constraints",
|
||||
"",
|
||||
"- Implement ONE task only. Do not combine with other tasks.",
|
||||
"- Inspect the codebase first, then edit.",
|
||||
"- Do not repeatedly reread unchanged files.",
|
||||
"- Limit your work to ONE small implementation step only. Do not combine tasks or features.",
|
||||
"- Make the smallest useful change possible.",
|
||||
])
|
||||
|
||||
test_cmds = _extract_test_commands_from_claude(root)
|
||||
if test_cmds:
|
||||
sections.extend([
|
||||
"",
|
||||
"---",
|
||||
"",
|
||||
"# Test Commands (from CLAUDE.md)",
|
||||
"",
|
||||
"Use these existing test commands:",
|
||||
("```bash\n" + "\n".join(test_cmds) + "\n```\n"),
|
||||
])
|
||||
|
||||
sections.extend([
|
||||
"",
|
||||
"---",
|
||||
"",
|
||||
|
||||
+133
-3
@@ -2,7 +2,25 @@ from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from rdb_discovery.tasks import generate_agent_prompt, get_next_task
|
||||
from rdb_discovery.tasks import (
|
||||
_extract_test_commands_from_claude,
|
||||
generate_agent_prompt,
|
||||
get_next_task,
|
||||
)
|
||||
|
||||
|
||||
def _write_claude_with_test_cmds(tmp_path: Path) -> None:
|
||||
"""Write a CLAUDE.md with Test Commands section."""
|
||||
(tmp_path / "CLAUDE.md").write_text(
|
||||
"# Claude Code Instructions\n\n"
|
||||
"## Test Commands\n\n"
|
||||
"Use the existing virtual environment.\n\n"
|
||||
"```bash\n"
|
||||
"source .venv/bin/activate\n"
|
||||
"python -m pytest\n"
|
||||
"```\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
|
||||
def test_generate_agent_prompt_includes_task_info(tmp_path: Path) -> None:
|
||||
@@ -67,7 +85,7 @@ def test_generate_agent_prompt_includes_read_instructions(tmp_path: Path) -> Non
|
||||
assert "context/agent-guidelines.md" in prompt
|
||||
|
||||
|
||||
def test_generate_agent_prompt_includes_one_task_constraint(tmp_path: Path) -> None:
|
||||
def test_generate_agent_prompt_includes_one_step_constraint(tmp_path: Path) -> None:
|
||||
(tmp_path / "TASKS.md").write_text(
|
||||
"# TASKS\n\n"
|
||||
"## TASK-001 — First\nStatus: Done\n\n"
|
||||
@@ -79,7 +97,7 @@ def test_generate_agent_prompt_includes_one_task_constraint(tmp_path: Path) -> N
|
||||
|
||||
prompt = generate_agent_prompt(tmp_path)
|
||||
|
||||
assert "ONE task only" in prompt or "one task only" in prompt
|
||||
assert "ONE small implementation step only" in prompt
|
||||
|
||||
|
||||
def test_generate_agent_prompt_includes_validation(tmp_path: Path) -> None:
|
||||
@@ -129,3 +147,115 @@ def test_generate_agent_prompt_no_tasks_file(tmp_path: Path) -> None:
|
||||
prompt = generate_agent_prompt(tmp_path)
|
||||
|
||||
assert prompt == "No Todo task found."
|
||||
|
||||
|
||||
def test_generate_agent_prompt_includes_implementation_gap(tmp_path: Path) -> None:
|
||||
(tmp_path / "TASKS.md").write_text(
|
||||
"# TASKS\n\n"
|
||||
"## TASK-001 — First\nStatus: Done\n\n"
|
||||
"Goal: x.\n\n"
|
||||
"## TASK-002 — Second\nStatus: Todo\n\n"
|
||||
"Goal: y.\n"
|
||||
"Implementation Gap:\n"
|
||||
"Missing validation handler in cli.py\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
prompt = generate_agent_prompt(tmp_path)
|
||||
|
||||
assert "Implementation Gap:" in prompt
|
||||
assert "Missing validation handler in cli.py" in prompt
|
||||
|
||||
|
||||
def test_generate_agent_prompt_omits_gap_when_missing(tmp_path: Path) -> None:
|
||||
(tmp_path / "TASKS.md").write_text(
|
||||
"# TASKS\n\n"
|
||||
"## TASK-001 — First\nStatus: Done\n\n"
|
||||
"Goal: x.\n\n"
|
||||
"## TASK-002 — Second\nStatus: Todo\n\n"
|
||||
"Goal: y.\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
prompt = generate_agent_prompt(tmp_path)
|
||||
|
||||
assert "Implementation Gap:" not in prompt
|
||||
|
||||
|
||||
def test_generate_agent_prompt_includes_test_commands_from_claude(tmp_path: Path) -> None:
|
||||
(tmp_path / "TASKS.md").write_text(
|
||||
"# TASKS\n\n"
|
||||
"## TASK-001 — First\nStatus: Done\n\n"
|
||||
"## TASK-002 — Second\nStatus: Todo\n\n"
|
||||
"Goal: y.\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
_write_claude_with_test_cmds(tmp_path)
|
||||
|
||||
prompt = generate_agent_prompt(tmp_path)
|
||||
|
||||
assert "Test Commands (from CLAUDE.md)" in prompt
|
||||
assert "source .venv/bin/activate" in prompt
|
||||
assert "python -m pytest" in prompt
|
||||
|
||||
|
||||
def test_generate_agent_prompt_includes_no_reread_constraint(tmp_path: Path) -> None:
|
||||
(tmp_path / "TASKS.md").write_text(
|
||||
"# TASKS\n\n"
|
||||
"## TASK-001 — First\nStatus: Done\n\n"
|
||||
"## TASK-002 — Second\nStatus: Todo\n\n"
|
||||
"Goal: y.\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
prompt = generate_agent_prompt(tmp_path)
|
||||
|
||||
assert "reread" in prompt.lower() or "read" in prompt.lower()
|
||||
|
||||
|
||||
def test_generate_agent_prompt_includes_inspect_first_constraint(tmp_path: Path) -> None:
|
||||
(tmp_path / "TASKS.md").write_text(
|
||||
"# TASKS\n\n"
|
||||
"## TASK-001 — First\nStatus: Done\n\n"
|
||||
"## TASK-002 — Second\nStatus: Todo\n\n"
|
||||
"Goal: y.\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
prompt = generate_agent_prompt(tmp_path)
|
||||
|
||||
assert "Inspect" in prompt or "inspect" in prompt
|
||||
|
||||
|
||||
def test_extract_test_commands_no_claude_file(tmp_path: Path) -> None:
|
||||
commands = _extract_test_commands_from_claude(tmp_path)
|
||||
assert commands == []
|
||||
|
||||
|
||||
def test_extract_test_commands_from_claude(tmp_path: Path) -> None:
|
||||
(tmp_path / "CLAUDE.md").write_text(
|
||||
"# Instructions\n\n"
|
||||
"## Test Commands\n\n"
|
||||
"Run the following:\n\n"
|
||||
"```bash\n"
|
||||
"source .venv/bin/activate\n"
|
||||
"python -m pytest\n"
|
||||
"```\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
commands = _extract_test_commands_from_claude(tmp_path)
|
||||
|
||||
assert any(".venv/bin/activate" in cmd for cmd in commands)
|
||||
assert "python -m pytest" in commands
|
||||
|
||||
|
||||
def test_extract_test_commands_no_section(tmp_path: Path) -> None:
|
||||
(tmp_path / "CLAUDE.md").write_text(
|
||||
"# Instructions\n\n"
|
||||
"No test section here.\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
commands = _extract_test_commands_from_claude(tmp_path)
|
||||
assert commands == []
|
||||
|
||||
Reference in New Issue
Block a user