feat: improve generated agent prompts
This commit is contained in:
@@ -28,18 +28,23 @@ From the repository root, run:
|
|||||||
```bash
|
```bash
|
||||||
source .venv/bin/activate
|
source .venv/bin/activate
|
||||||
python -m pytest
|
python -m pytest
|
||||||
|
```
|
||||||
|
|
||||||
Do not search the filesystem for pytest.
|
Do not search the filesystem for pytest.
|
||||||
Do not create a new virtual environment unless explicitly asked.
|
Do not create a new virtual environment unless explicitly asked.
|
||||||
|
|
||||||
If pytest is unavailable, run:
|
If pytest is unavailable, run:
|
||||||
|
|
||||||
|
```bash
|
||||||
pip install -e '.[dev]'
|
pip install -e '.[dev]'
|
||||||
python -m pytest
|
python -m pytest
|
||||||
Validation Commands
|
```
|
||||||
|
|
||||||
|
## Validation Commands
|
||||||
|
|
||||||
For normal task validation, run:
|
For normal task validation, run:
|
||||||
|
|
||||||
|
```bash
|
||||||
python -m pytest
|
python -m pytest
|
||||||
rdb status
|
rdb status
|
||||||
rdb next
|
rdb next
|
||||||
|
|||||||
@@ -116,7 +116,7 @@ Acceptance Criteria:
|
|||||||
|
|
||||||
## TASK-007 — Improve generated agent prompts
|
## TASK-007 — Improve generated agent prompts
|
||||||
|
|
||||||
Status: Todo
|
Status: Done
|
||||||
|
|
||||||
Goal:
|
Goal:
|
||||||
Make `rdb prompt` produce smaller, more direct prompts for Claude Code/local LLM agents.
|
Make `rdb prompt` produce smaller, more direct prompts for Claude Code/local LLM agents.
|
||||||
@@ -130,21 +130,59 @@ Acceptance Criteria:
|
|||||||
- Prompt limits scope to one small implementation step
|
- Prompt limits scope to one small implementation step
|
||||||
- Add/update tests
|
- Add/update tests
|
||||||
|
|
||||||
## TASK-012 — Agent telemetry
|
## TASK-012 — Agent telemetry foundation
|
||||||
|
|
||||||
Status: Todo
|
Status: Todo
|
||||||
|
|
||||||
Goal:
|
Goal:
|
||||||
Capture actual agent actions rather than inferring behaviour
|
Create a minimal telemetry writer for future agent activity tracking.
|
||||||
from documentation and run logs.
|
|
||||||
|
|
||||||
Examples:
|
Implementation Gap:
|
||||||
|
There is currently no persistent session log for recording agent activity. Guardrails infer behaviour from markdown and RUN_LOG.md instead of structured telemetry.
|
||||||
|
|
||||||
- file reads
|
Acceptance Criteria:
|
||||||
- file writes
|
|
||||||
- command execution
|
|
||||||
- test execution
|
|
||||||
- git activity
|
|
||||||
|
|
||||||
Output:
|
- Add a telemetry module
|
||||||
.rdb/session-log.jsonl
|
- Create `.rdb/session-log.jsonl` when recording an event
|
||||||
|
- Record JSONL events with:
|
||||||
|
- timestamp
|
||||||
|
- event_type
|
||||||
|
- target
|
||||||
|
- details
|
||||||
|
- Add tests for writing and appending telemetry events
|
||||||
|
- Do not integrate telemetry into every CLI command yet
|
||||||
|
|
||||||
|
## TASK-013 — Log CLI command execution
|
||||||
|
|
||||||
|
Status: Todo
|
||||||
|
|
||||||
|
Goal:
|
||||||
|
Record rdb CLI command execution into telemetry.
|
||||||
|
|
||||||
|
Implementation Gap:
|
||||||
|
Telemetry writer exists, but CLI commands do not yet record command execution events.
|
||||||
|
|
||||||
|
Acceptance Criteria:
|
||||||
|
|
||||||
|
- Record command execution events
|
||||||
|
- Include command name
|
||||||
|
- Include success/failure where practical
|
||||||
|
- Add/update tests
|
||||||
|
|
||||||
|
## TASK-014 — Use telemetry in guardrails
|
||||||
|
|
||||||
|
Status: Todo
|
||||||
|
|
||||||
|
Goal:
|
||||||
|
Make `rdb guardrails` use structured telemetry when available.
|
||||||
|
|
||||||
|
Implementation Gap:
|
||||||
|
Guardrails currently rely on markdown and file mtimes rather than `.rdb/session-log.jsonl`.
|
||||||
|
|
||||||
|
Acceptance Criteria:
|
||||||
|
|
||||||
|
- Read telemetry events when present
|
||||||
|
- Detect repeated reads from telemetry
|
||||||
|
- Detect repeated commands from telemetry
|
||||||
|
- Fall back gracefully if telemetry is missing
|
||||||
|
- Add/update tests
|
||||||
|
|||||||
@@ -10,6 +10,7 @@ STATUS_RE = re.compile(r"^Status:\s*(.+)$", re.MULTILINE)
|
|||||||
|
|
||||||
GOAL_RE = re.compile(r"^Goal:\s*(.+)", re.MULTILINE)
|
GOAL_RE = re.compile(r"^Goal:\s*(.+)", re.MULTILINE)
|
||||||
AC_RE = re.compile(r"^- (.+)$", re.MULTILINE)
|
AC_RE = re.compile(r"^- (.+)$", re.MULTILINE)
|
||||||
|
GAP_RE = re.compile(r"^Implementation Gap:\s*(.+)", re.MULTILINE)
|
||||||
|
|
||||||
|
|
||||||
@dataclass
|
@dataclass
|
||||||
@@ -28,6 +29,12 @@ class Task:
|
|||||||
def ac_lines(self) -> list[str]:
|
def ac_lines(self) -> list[str]:
|
||||||
return AC_RE.findall(self.body)
|
return AC_RE.findall(self.body)
|
||||||
|
|
||||||
|
def implementation_gap(self, root: Path) -> str | None:
|
||||||
|
match = GAP_RE.search(self.body)
|
||||||
|
if match and match.group(1).strip():
|
||||||
|
return match.group(1).strip()
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
def read_tasks(root: Path) -> list[Task]:
|
def read_tasks(root: Path) -> list[Task]:
|
||||||
tasks_path = root / "TASKS.md"
|
tasks_path = root / "TASKS.md"
|
||||||
@@ -92,6 +99,12 @@ CONTEXT_FILES_TO_READ = [
|
|||||||
"context/agent-guidelines.md",
|
"context/agent-guidelines.md",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
TEST_CMD_SECTION_RE = re.compile(
|
||||||
|
r"^##\s*Test Commands\s*\n([\s\S]*?)(?=^##|\Z)",
|
||||||
|
re.MULTILINE,
|
||||||
|
)
|
||||||
|
BASH_BLOCK_RE = re.compile(r"```bash\n(.*?)```", re.DOTALL)
|
||||||
|
|
||||||
|
|
||||||
def _read_file_safe(root: Path, relative: str) -> str:
|
def _read_file_safe(root: Path, relative: str) -> str:
|
||||||
path = root / relative
|
path = root / relative
|
||||||
@@ -100,6 +113,55 @@ def _read_file_safe(root: Path, relative: str) -> str:
|
|||||||
return f"# {relative} — not found"
|
return f"# {relative} — not found"
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_test_commands_from_claude(root: Path) -> list[str]:
|
||||||
|
"""Extract bash test commands from CLAUDE.md Test Commands section."""
|
||||||
|
|
||||||
|
def _looks_like_command(line: str) -> bool:
|
||||||
|
"""Heuristic: keep lines that look like shell commands."""
|
||||||
|
if not line.strip():
|
||||||
|
return False
|
||||||
|
lower = line.lower()
|
||||||
|
# Skip markdown, prose, headings
|
||||||
|
if any(lower.startswith(p) for p in ("# ", "- ", "if ", "do not", "for ", "use ")) or lower in (
|
||||||
|
"validation commands",
|
||||||
|
):
|
||||||
|
return False
|
||||||
|
# Skip lines that look like sentences (contain spaces followed by lowercase words)
|
||||||
|
# but keep things like `source .venv/bin/activate` and `pip install ...`
|
||||||
|
parts = line.split()
|
||||||
|
if len(parts) <= 1:
|
||||||
|
return True
|
||||||
|
# If the first word is a known shell builtin / command prefix, accept it
|
||||||
|
known_prefixes = ("source", "cd", "ls", "cp", "mv", "rm", "mkdir", "echo", "grep",
|
||||||
|
"git", "pip", "python", "pytest", "rdb", "cat", "head", "tail",
|
||||||
|
"sed", "awk", "find", "install")
|
||||||
|
return parts[0].lower() in known_prefixes
|
||||||
|
|
||||||
|
claude_path = root / "CLAUDE.md"
|
||||||
|
if not claude_path.exists():
|
||||||
|
return []
|
||||||
|
|
||||||
|
text = claude_path.read_text(encoding="utf-8")
|
||||||
|
section_match = TEST_CMD_SECTION_RE.search(text)
|
||||||
|
if not section_match:
|
||||||
|
return []
|
||||||
|
|
||||||
|
section_text = section_match.group(1)
|
||||||
|
blocks = BASH_BLOCK_RE.findall(section_text)
|
||||||
|
seen: set[str] = set()
|
||||||
|
commands: list[str] = []
|
||||||
|
for block in blocks:
|
||||||
|
for line in block.strip().splitlines():
|
||||||
|
stripped = line.strip()
|
||||||
|
if not stripped:
|
||||||
|
continue
|
||||||
|
if _looks_like_command(stripped):
|
||||||
|
if stripped not in seen:
|
||||||
|
seen.add(stripped)
|
||||||
|
commands.append(stripped)
|
||||||
|
return commands
|
||||||
|
|
||||||
|
|
||||||
def generate_agent_prompt(root: Path) -> str:
|
def generate_agent_prompt(root: Path) -> str:
|
||||||
task = get_next_task(root)
|
task = get_next_task(root)
|
||||||
if not task:
|
if not task:
|
||||||
@@ -128,9 +190,20 @@ def generate_agent_prompt(root: Path) -> str:
|
|||||||
f"Status: {task.status}",
|
f"Status: {task.status}",
|
||||||
"",
|
"",
|
||||||
f"Goal:\n{task.goal(root)}",
|
f"Goal:\n{task.goal(root)}",
|
||||||
|
]
|
||||||
|
|
||||||
|
gap = task.implementation_gap(root)
|
||||||
|
if gap:
|
||||||
|
sections.extend([
|
||||||
|
"",
|
||||||
|
"Implementation Gap:",
|
||||||
|
gap,
|
||||||
|
])
|
||||||
|
|
||||||
|
sections.extend([
|
||||||
"",
|
"",
|
||||||
"Acceptance Criteria:",
|
"Acceptance Criteria:",
|
||||||
]
|
])
|
||||||
|
|
||||||
for line in task.ac_lines():
|
for line in task.ac_lines():
|
||||||
sections.append(f"- {line}")
|
sections.append(f"- {line}")
|
||||||
@@ -141,8 +214,25 @@ def generate_agent_prompt(root: Path) -> str:
|
|||||||
"",
|
"",
|
||||||
"# Constraints",
|
"# Constraints",
|
||||||
"",
|
"",
|
||||||
"- Implement ONE task only. Do not combine with other tasks.",
|
"- Inspect the codebase first, then edit.",
|
||||||
|
"- Do not repeatedly reread unchanged files.",
|
||||||
|
"- Limit your work to ONE small implementation step only. Do not combine tasks or features.",
|
||||||
"- Make the smallest useful change possible.",
|
"- Make the smallest useful change possible.",
|
||||||
|
])
|
||||||
|
|
||||||
|
test_cmds = _extract_test_commands_from_claude(root)
|
||||||
|
if test_cmds:
|
||||||
|
sections.extend([
|
||||||
|
"",
|
||||||
|
"---",
|
||||||
|
"",
|
||||||
|
"# Test Commands (from CLAUDE.md)",
|
||||||
|
"",
|
||||||
|
"Use these existing test commands:",
|
||||||
|
("```bash\n" + "\n".join(test_cmds) + "\n```\n"),
|
||||||
|
])
|
||||||
|
|
||||||
|
sections.extend([
|
||||||
"",
|
"",
|
||||||
"---",
|
"---",
|
||||||
"",
|
"",
|
||||||
|
|||||||
+133
-3
@@ -2,7 +2,25 @@ from pathlib import Path
|
|||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
|
|
||||||
from rdb_discovery.tasks import generate_agent_prompt, get_next_task
|
from rdb_discovery.tasks import (
|
||||||
|
_extract_test_commands_from_claude,
|
||||||
|
generate_agent_prompt,
|
||||||
|
get_next_task,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _write_claude_with_test_cmds(tmp_path: Path) -> None:
|
||||||
|
"""Write a CLAUDE.md with Test Commands section."""
|
||||||
|
(tmp_path / "CLAUDE.md").write_text(
|
||||||
|
"# Claude Code Instructions\n\n"
|
||||||
|
"## Test Commands\n\n"
|
||||||
|
"Use the existing virtual environment.\n\n"
|
||||||
|
"```bash\n"
|
||||||
|
"source .venv/bin/activate\n"
|
||||||
|
"python -m pytest\n"
|
||||||
|
"```\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def test_generate_agent_prompt_includes_task_info(tmp_path: Path) -> None:
|
def test_generate_agent_prompt_includes_task_info(tmp_path: Path) -> None:
|
||||||
@@ -67,7 +85,7 @@ def test_generate_agent_prompt_includes_read_instructions(tmp_path: Path) -> Non
|
|||||||
assert "context/agent-guidelines.md" in prompt
|
assert "context/agent-guidelines.md" in prompt
|
||||||
|
|
||||||
|
|
||||||
def test_generate_agent_prompt_includes_one_task_constraint(tmp_path: Path) -> None:
|
def test_generate_agent_prompt_includes_one_step_constraint(tmp_path: Path) -> None:
|
||||||
(tmp_path / "TASKS.md").write_text(
|
(tmp_path / "TASKS.md").write_text(
|
||||||
"# TASKS\n\n"
|
"# TASKS\n\n"
|
||||||
"## TASK-001 — First\nStatus: Done\n\n"
|
"## TASK-001 — First\nStatus: Done\n\n"
|
||||||
@@ -79,7 +97,7 @@ def test_generate_agent_prompt_includes_one_task_constraint(tmp_path: Path) -> N
|
|||||||
|
|
||||||
prompt = generate_agent_prompt(tmp_path)
|
prompt = generate_agent_prompt(tmp_path)
|
||||||
|
|
||||||
assert "ONE task only" in prompt or "one task only" in prompt
|
assert "ONE small implementation step only" in prompt
|
||||||
|
|
||||||
|
|
||||||
def test_generate_agent_prompt_includes_validation(tmp_path: Path) -> None:
|
def test_generate_agent_prompt_includes_validation(tmp_path: Path) -> None:
|
||||||
@@ -129,3 +147,115 @@ def test_generate_agent_prompt_no_tasks_file(tmp_path: Path) -> None:
|
|||||||
prompt = generate_agent_prompt(tmp_path)
|
prompt = generate_agent_prompt(tmp_path)
|
||||||
|
|
||||||
assert prompt == "No Todo task found."
|
assert prompt == "No Todo task found."
|
||||||
|
|
||||||
|
|
||||||
|
def test_generate_agent_prompt_includes_implementation_gap(tmp_path: Path) -> None:
|
||||||
|
(tmp_path / "TASKS.md").write_text(
|
||||||
|
"# TASKS\n\n"
|
||||||
|
"## TASK-001 — First\nStatus: Done\n\n"
|
||||||
|
"Goal: x.\n\n"
|
||||||
|
"## TASK-002 — Second\nStatus: Todo\n\n"
|
||||||
|
"Goal: y.\n"
|
||||||
|
"Implementation Gap:\n"
|
||||||
|
"Missing validation handler in cli.py\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
prompt = generate_agent_prompt(tmp_path)
|
||||||
|
|
||||||
|
assert "Implementation Gap:" in prompt
|
||||||
|
assert "Missing validation handler in cli.py" in prompt
|
||||||
|
|
||||||
|
|
||||||
|
def test_generate_agent_prompt_omits_gap_when_missing(tmp_path: Path) -> None:
|
||||||
|
(tmp_path / "TASKS.md").write_text(
|
||||||
|
"# TASKS\n\n"
|
||||||
|
"## TASK-001 — First\nStatus: Done\n\n"
|
||||||
|
"Goal: x.\n\n"
|
||||||
|
"## TASK-002 — Second\nStatus: Todo\n\n"
|
||||||
|
"Goal: y.\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
prompt = generate_agent_prompt(tmp_path)
|
||||||
|
|
||||||
|
assert "Implementation Gap:" not in prompt
|
||||||
|
|
||||||
|
|
||||||
|
def test_generate_agent_prompt_includes_test_commands_from_claude(tmp_path: Path) -> None:
|
||||||
|
(tmp_path / "TASKS.md").write_text(
|
||||||
|
"# TASKS\n\n"
|
||||||
|
"## TASK-001 — First\nStatus: Done\n\n"
|
||||||
|
"## TASK-002 — Second\nStatus: Todo\n\n"
|
||||||
|
"Goal: y.\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
_write_claude_with_test_cmds(tmp_path)
|
||||||
|
|
||||||
|
prompt = generate_agent_prompt(tmp_path)
|
||||||
|
|
||||||
|
assert "Test Commands (from CLAUDE.md)" in prompt
|
||||||
|
assert "source .venv/bin/activate" in prompt
|
||||||
|
assert "python -m pytest" in prompt
|
||||||
|
|
||||||
|
|
||||||
|
def test_generate_agent_prompt_includes_no_reread_constraint(tmp_path: Path) -> None:
|
||||||
|
(tmp_path / "TASKS.md").write_text(
|
||||||
|
"# TASKS\n\n"
|
||||||
|
"## TASK-001 — First\nStatus: Done\n\n"
|
||||||
|
"## TASK-002 — Second\nStatus: Todo\n\n"
|
||||||
|
"Goal: y.\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
prompt = generate_agent_prompt(tmp_path)
|
||||||
|
|
||||||
|
assert "reread" in prompt.lower() or "read" in prompt.lower()
|
||||||
|
|
||||||
|
|
||||||
|
def test_generate_agent_prompt_includes_inspect_first_constraint(tmp_path: Path) -> None:
|
||||||
|
(tmp_path / "TASKS.md").write_text(
|
||||||
|
"# TASKS\n\n"
|
||||||
|
"## TASK-001 — First\nStatus: Done\n\n"
|
||||||
|
"## TASK-002 — Second\nStatus: Todo\n\n"
|
||||||
|
"Goal: y.\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
prompt = generate_agent_prompt(tmp_path)
|
||||||
|
|
||||||
|
assert "Inspect" in prompt or "inspect" in prompt
|
||||||
|
|
||||||
|
|
||||||
|
def test_extract_test_commands_no_claude_file(tmp_path: Path) -> None:
|
||||||
|
commands = _extract_test_commands_from_claude(tmp_path)
|
||||||
|
assert commands == []
|
||||||
|
|
||||||
|
|
||||||
|
def test_extract_test_commands_from_claude(tmp_path: Path) -> None:
|
||||||
|
(tmp_path / "CLAUDE.md").write_text(
|
||||||
|
"# Instructions\n\n"
|
||||||
|
"## Test Commands\n\n"
|
||||||
|
"Run the following:\n\n"
|
||||||
|
"```bash\n"
|
||||||
|
"source .venv/bin/activate\n"
|
||||||
|
"python -m pytest\n"
|
||||||
|
"```\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
commands = _extract_test_commands_from_claude(tmp_path)
|
||||||
|
|
||||||
|
assert any(".venv/bin/activate" in cmd for cmd in commands)
|
||||||
|
assert "python -m pytest" in commands
|
||||||
|
|
||||||
|
|
||||||
|
def test_extract_test_commands_no_section(tmp_path: Path) -> None:
|
||||||
|
(tmp_path / "CLAUDE.md").write_text(
|
||||||
|
"# Instructions\n\n"
|
||||||
|
"No test section here.\n",
|
||||||
|
encoding="utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
commands = _extract_test_commands_from_claude(tmp_path)
|
||||||
|
assert commands == []
|
||||||
|
|||||||
Reference in New Issue
Block a user