From 7a1d36c03cca3b2497301d9530cce173a5e72a79 Mon Sep 17 00:00:00 2001 From: robbond Date: Tue, 2 Jun 2026 12:49:55 +0100 Subject: [PATCH] feat: improve generated agent prompts --- CLAUDE.md | 7 +- TASKS.md | 62 +++++++++++++---- src/rdb_discovery/tasks.py | 94 ++++++++++++++++++++++++- tests/test_prompt.py | 136 ++++++++++++++++++++++++++++++++++++- 4 files changed, 281 insertions(+), 18 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index d28a6c4..dec61c5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -28,18 +28,23 @@ From the repository root, run: ```bash source .venv/bin/activate python -m pytest +``` Do not search the filesystem for pytest. Do not create a new virtual environment unless explicitly asked. If pytest is unavailable, run: +```bash pip install -e '.[dev]' python -m pytest -Validation Commands +``` + +## Validation Commands For normal task validation, run: +```bash python -m pytest rdb status rdb next diff --git a/TASKS.md b/TASKS.md index e35b316..a71202b 100644 --- a/TASKS.md +++ b/TASKS.md @@ -116,7 +116,7 @@ Acceptance Criteria: ## TASK-007 — Improve generated agent prompts -Status: Todo +Status: Done Goal: Make `rdb prompt` produce smaller, more direct prompts for Claude Code/local LLM agents. @@ -130,21 +130,59 @@ Acceptance Criteria: - Prompt limits scope to one small implementation step - Add/update tests -## TASK-012 — Agent telemetry +## TASK-012 — Agent telemetry foundation Status: Todo Goal: -Capture actual agent actions rather than inferring behaviour -from documentation and run logs. +Create a minimal telemetry writer for future agent activity tracking. -Examples: +Implementation Gap: +There is currently no persistent session log for recording agent activity. Guardrails infer behaviour from markdown and RUN_LOG.md instead of structured telemetry. -- file reads -- file writes -- command execution -- test execution -- git activity +Acceptance Criteria: -Output: -.rdb/session-log.jsonl +- Add a telemetry module +- Create `.rdb/session-log.jsonl` when recording an event +- Record JSONL events with: + - timestamp + - event_type + - target + - details +- Add tests for writing and appending telemetry events +- Do not integrate telemetry into every CLI command yet + +## TASK-013 — Log CLI command execution + +Status: Todo + +Goal: +Record rdb CLI command execution into telemetry. + +Implementation Gap: +Telemetry writer exists, but CLI commands do not yet record command execution events. + +Acceptance Criteria: + +- Record command execution events +- Include command name +- Include success/failure where practical +- Add/update tests + +## TASK-014 — Use telemetry in guardrails + +Status: Todo + +Goal: +Make `rdb guardrails` use structured telemetry when available. + +Implementation Gap: +Guardrails currently rely on markdown and file mtimes rather than `.rdb/session-log.jsonl`. + +Acceptance Criteria: + +- Read telemetry events when present +- Detect repeated reads from telemetry +- Detect repeated commands from telemetry +- Fall back gracefully if telemetry is missing +- Add/update tests diff --git a/src/rdb_discovery/tasks.py b/src/rdb_discovery/tasks.py index 1d29d0c..438bef8 100644 --- a/src/rdb_discovery/tasks.py +++ b/src/rdb_discovery/tasks.py @@ -10,6 +10,7 @@ STATUS_RE = re.compile(r"^Status:\s*(.+)$", re.MULTILINE) GOAL_RE = re.compile(r"^Goal:\s*(.+)", re.MULTILINE) AC_RE = re.compile(r"^- (.+)$", re.MULTILINE) +GAP_RE = re.compile(r"^Implementation Gap:\s*(.+)", re.MULTILINE) @dataclass @@ -28,6 +29,12 @@ class Task: def ac_lines(self) -> list[str]: return AC_RE.findall(self.body) + def implementation_gap(self, root: Path) -> str | None: + match = GAP_RE.search(self.body) + if match and match.group(1).strip(): + return match.group(1).strip() + return None + def read_tasks(root: Path) -> list[Task]: tasks_path = root / "TASKS.md" @@ -92,6 +99,12 @@ CONTEXT_FILES_TO_READ = [ "context/agent-guidelines.md", ] +TEST_CMD_SECTION_RE = re.compile( + r"^##\s*Test Commands\s*\n([\s\S]*?)(?=^##|\Z)", + re.MULTILINE, +) +BASH_BLOCK_RE = re.compile(r"```bash\n(.*?)```", re.DOTALL) + def _read_file_safe(root: Path, relative: str) -> str: path = root / relative @@ -100,6 +113,55 @@ def _read_file_safe(root: Path, relative: str) -> str: return f"# {relative} — not found" +def _extract_test_commands_from_claude(root: Path) -> list[str]: + """Extract bash test commands from CLAUDE.md Test Commands section.""" + + def _looks_like_command(line: str) -> bool: + """Heuristic: keep lines that look like shell commands.""" + if not line.strip(): + return False + lower = line.lower() + # Skip markdown, prose, headings + if any(lower.startswith(p) for p in ("# ", "- ", "if ", "do not", "for ", "use ")) or lower in ( + "validation commands", + ): + return False + # Skip lines that look like sentences (contain spaces followed by lowercase words) + # but keep things like `source .venv/bin/activate` and `pip install ...` + parts = line.split() + if len(parts) <= 1: + return True + # If the first word is a known shell builtin / command prefix, accept it + known_prefixes = ("source", "cd", "ls", "cp", "mv", "rm", "mkdir", "echo", "grep", + "git", "pip", "python", "pytest", "rdb", "cat", "head", "tail", + "sed", "awk", "find", "install") + return parts[0].lower() in known_prefixes + + claude_path = root / "CLAUDE.md" + if not claude_path.exists(): + return [] + + text = claude_path.read_text(encoding="utf-8") + section_match = TEST_CMD_SECTION_RE.search(text) + if not section_match: + return [] + + section_text = section_match.group(1) + blocks = BASH_BLOCK_RE.findall(section_text) + seen: set[str] = set() + commands: list[str] = [] + for block in blocks: + for line in block.strip().splitlines(): + stripped = line.strip() + if not stripped: + continue + if _looks_like_command(stripped): + if stripped not in seen: + seen.add(stripped) + commands.append(stripped) + return commands + + def generate_agent_prompt(root: Path) -> str: task = get_next_task(root) if not task: @@ -128,9 +190,20 @@ def generate_agent_prompt(root: Path) -> str: f"Status: {task.status}", "", f"Goal:\n{task.goal(root)}", + ] + + gap = task.implementation_gap(root) + if gap: + sections.extend([ + "", + "Implementation Gap:", + gap, + ]) + + sections.extend([ "", "Acceptance Criteria:", - ] + ]) for line in task.ac_lines(): sections.append(f"- {line}") @@ -141,8 +214,25 @@ def generate_agent_prompt(root: Path) -> str: "", "# Constraints", "", - "- Implement ONE task only. Do not combine with other tasks.", + "- Inspect the codebase first, then edit.", + "- Do not repeatedly reread unchanged files.", + "- Limit your work to ONE small implementation step only. Do not combine tasks or features.", "- Make the smallest useful change possible.", + ]) + + test_cmds = _extract_test_commands_from_claude(root) + if test_cmds: + sections.extend([ + "", + "---", + "", + "# Test Commands (from CLAUDE.md)", + "", + "Use these existing test commands:", + ("```bash\n" + "\n".join(test_cmds) + "\n```\n"), + ]) + + sections.extend([ "", "---", "", diff --git a/tests/test_prompt.py b/tests/test_prompt.py index 15d56c8..70c9084 100644 --- a/tests/test_prompt.py +++ b/tests/test_prompt.py @@ -2,7 +2,25 @@ from pathlib import Path import pytest -from rdb_discovery.tasks import generate_agent_prompt, get_next_task +from rdb_discovery.tasks import ( + _extract_test_commands_from_claude, + generate_agent_prompt, + get_next_task, +) + + +def _write_claude_with_test_cmds(tmp_path: Path) -> None: + """Write a CLAUDE.md with Test Commands section.""" + (tmp_path / "CLAUDE.md").write_text( + "# Claude Code Instructions\n\n" + "## Test Commands\n\n" + "Use the existing virtual environment.\n\n" + "```bash\n" + "source .venv/bin/activate\n" + "python -m pytest\n" + "```\n", + encoding="utf-8", + ) def test_generate_agent_prompt_includes_task_info(tmp_path: Path) -> None: @@ -67,7 +85,7 @@ def test_generate_agent_prompt_includes_read_instructions(tmp_path: Path) -> Non assert "context/agent-guidelines.md" in prompt -def test_generate_agent_prompt_includes_one_task_constraint(tmp_path: Path) -> None: +def test_generate_agent_prompt_includes_one_step_constraint(tmp_path: Path) -> None: (tmp_path / "TASKS.md").write_text( "# TASKS\n\n" "## TASK-001 — First\nStatus: Done\n\n" @@ -79,7 +97,7 @@ def test_generate_agent_prompt_includes_one_task_constraint(tmp_path: Path) -> N prompt = generate_agent_prompt(tmp_path) - assert "ONE task only" in prompt or "one task only" in prompt + assert "ONE small implementation step only" in prompt def test_generate_agent_prompt_includes_validation(tmp_path: Path) -> None: @@ -129,3 +147,115 @@ def test_generate_agent_prompt_no_tasks_file(tmp_path: Path) -> None: prompt = generate_agent_prompt(tmp_path) assert prompt == "No Todo task found." + + +def test_generate_agent_prompt_includes_implementation_gap(tmp_path: Path) -> None: + (tmp_path / "TASKS.md").write_text( + "# TASKS\n\n" + "## TASK-001 — First\nStatus: Done\n\n" + "Goal: x.\n\n" + "## TASK-002 — Second\nStatus: Todo\n\n" + "Goal: y.\n" + "Implementation Gap:\n" + "Missing validation handler in cli.py\n", + encoding="utf-8", + ) + + prompt = generate_agent_prompt(tmp_path) + + assert "Implementation Gap:" in prompt + assert "Missing validation handler in cli.py" in prompt + + +def test_generate_agent_prompt_omits_gap_when_missing(tmp_path: Path) -> None: + (tmp_path / "TASKS.md").write_text( + "# TASKS\n\n" + "## TASK-001 — First\nStatus: Done\n\n" + "Goal: x.\n\n" + "## TASK-002 — Second\nStatus: Todo\n\n" + "Goal: y.\n", + encoding="utf-8", + ) + + prompt = generate_agent_prompt(tmp_path) + + assert "Implementation Gap:" not in prompt + + +def test_generate_agent_prompt_includes_test_commands_from_claude(tmp_path: Path) -> None: + (tmp_path / "TASKS.md").write_text( + "# TASKS\n\n" + "## TASK-001 — First\nStatus: Done\n\n" + "## TASK-002 — Second\nStatus: Todo\n\n" + "Goal: y.\n", + encoding="utf-8", + ) + _write_claude_with_test_cmds(tmp_path) + + prompt = generate_agent_prompt(tmp_path) + + assert "Test Commands (from CLAUDE.md)" in prompt + assert "source .venv/bin/activate" in prompt + assert "python -m pytest" in prompt + + +def test_generate_agent_prompt_includes_no_reread_constraint(tmp_path: Path) -> None: + (tmp_path / "TASKS.md").write_text( + "# TASKS\n\n" + "## TASK-001 — First\nStatus: Done\n\n" + "## TASK-002 — Second\nStatus: Todo\n\n" + "Goal: y.\n", + encoding="utf-8", + ) + + prompt = generate_agent_prompt(tmp_path) + + assert "reread" in prompt.lower() or "read" in prompt.lower() + + +def test_generate_agent_prompt_includes_inspect_first_constraint(tmp_path: Path) -> None: + (tmp_path / "TASKS.md").write_text( + "# TASKS\n\n" + "## TASK-001 — First\nStatus: Done\n\n" + "## TASK-002 — Second\nStatus: Todo\n\n" + "Goal: y.\n", + encoding="utf-8", + ) + + prompt = generate_agent_prompt(tmp_path) + + assert "Inspect" in prompt or "inspect" in prompt + + +def test_extract_test_commands_no_claude_file(tmp_path: Path) -> None: + commands = _extract_test_commands_from_claude(tmp_path) + assert commands == [] + + +def test_extract_test_commands_from_claude(tmp_path: Path) -> None: + (tmp_path / "CLAUDE.md").write_text( + "# Instructions\n\n" + "## Test Commands\n\n" + "Run the following:\n\n" + "```bash\n" + "source .venv/bin/activate\n" + "python -m pytest\n" + "```\n", + encoding="utf-8", + ) + + commands = _extract_test_commands_from_claude(tmp_path) + + assert any(".venv/bin/activate" in cmd for cmd in commands) + assert "python -m pytest" in commands + + +def test_extract_test_commands_no_section(tmp_path: Path) -> None: + (tmp_path / "CLAUDE.md").write_text( + "# Instructions\n\n" + "No test section here.\n", + encoding="utf-8", + ) + + commands = _extract_test_commands_from_claude(tmp_path) + assert commands == []