diff --git a/README.md b/README.md index d79d8b62e..e7eb78b37 100644 --- a/README.md +++ b/README.md @@ -24,8 +24,9 @@ Copy the ralph files into your project: ```bash # From your project root -mkdir -p scripts/ralph +mkdir -p scripts/ralph/lib cp /path/to/ralph/ralph.sh scripts/ralph/ +cp /path/to/ralph/lib/rate-limit.sh scripts/ralph/lib/ # Copy the prompt template for your AI tool of choice: cp /path/to/ralph/prompt.md scripts/ralph/prompt.md # For Amp @@ -119,6 +120,8 @@ This creates `prd.json` with user stories structured for autonomous execution. Default is 10 iterations. Use `--tool amp` or `--tool claude` to select your AI coding tool. +When running with `--tool claude`, Ralph automatically detects Claude Code quota/rate-limit messages, waits until the reported reset time when possible, and retries the same iteration instead of burning through the remaining loop count. + Ralph will: 1. Create a feature branch (from PRD `branchName`) 2. Pick the highest priority story where `passes: false` @@ -129,6 +132,19 @@ Ralph will: 7. Append learnings to `progress.txt` 8. Repeat until all stories pass or max iterations reached +## Smoke Test + +You can run a local smoke test for the loop and rate-limit handling without installing Amp or Claude Code: + +```bash +./test-rate-limit.sh +``` + +This script mocks the CLI tools and verifies: +- Claude quota messages with parseable reset times retry the same iteration +- Claude quota messages without reset details fall back to a 5-hour wait +- The default Amp path still completes normally + ## Key Files | File | Purpose | diff --git a/lib/rate-limit.sh b/lib/rate-limit.sh new file mode 100644 index 000000000..d666efdab --- /dev/null +++ b/lib/rate-limit.sh @@ -0,0 +1,134 @@ +#!/bin/bash + +RATE_LIMIT_FALLBACK_WAIT_SECONDS=$((5 * 60 * 60)) + +is_rate_limit_output() { + local output_lower + output_lower=$(printf '%s' "$1" | tr '[:upper:]' '[:lower:]') + + [[ "$output_lower" == *"usage limit reached"* ]] \ + || [[ "$output_lower" == *"rate limit"* ]] \ + || [[ "$output_lower" == *"quota exceeded"* ]] \ + || [[ "$output_lower" == *"hit your"* ]] +} + +extract_rate_limit_reset_details() { + local output="$1" + local reset_time="" + local reset_timezone="" + local reset_regex='[Rr]esets?([[:space:]]+at)?[[:space:]]+([^()[:space:]][^()]*)[[:space:]]\(([^)]+)\)' + + if [[ "$output" =~ $reset_regex ]]; then + reset_time="${BASH_REMATCH[2]}" + reset_timezone="${BASH_REMATCH[3]}" + fi + + if [[ -n "$reset_time" && -n "$reset_timezone" ]]; then + printf '%s|%s\n' "$reset_time" "$reset_timezone" + return 0 + fi + + return 1 +} + +calculate_rate_limit_wait_seconds() { + local reset_time="$1" + local reset_timezone="$2" + local seconds_until_reset="" + + if ! command -v python3 >/dev/null 2>&1; then + return 1 + fi + + seconds_until_reset=$(python3 - "$reset_time" "$reset_timezone" <<'PY' +from datetime import datetime, timedelta +from zoneinfo import ZoneInfo +import sys + +reset_time = sys.argv[1].strip().lower().replace(".", "") +reset_timezone = sys.argv[2].strip() + +formats = ("%I%p", "%I:%M%p", "%I %p", "%I:%M %p") +now = datetime.now(ZoneInfo(reset_timezone)) + +parsed_time = None +for fmt in formats: + try: + parsed_time = datetime.strptime(reset_time.upper(), fmt) + break + except ValueError: + continue + +if parsed_time is None: + raise SystemExit(1) + +reset_at = now.replace( + hour=parsed_time.hour, + minute=parsed_time.minute, + second=0, + microsecond=0, +) + +if reset_at <= now: + reset_at += timedelta(days=1) + +print(max(1, int((reset_at - now).total_seconds()))) +PY + ) || return 1 + + if [[ "$seconds_until_reset" =~ ^[0-9]+$ ]]; then + printf '%s\n' "$seconds_until_reset" + return 0 + fi + + return 1 +} + +format_wait_duration() { + local total_seconds="$1" + local hours=$((total_seconds / 3600)) + local minutes=$(((total_seconds % 3600) / 60)) + local seconds=$((total_seconds % 60)) + + if (( hours > 0 )); then + printf '%dh %dm %ds' "$hours" "$minutes" "$seconds" + elif (( minutes > 0 )); then + printf '%dm %ds' "$minutes" "$seconds" + else + printf '%ds' "$seconds" + fi +} + +handle_rate_limit() { + local output="$1" + local wait_seconds="$RATE_LIMIT_FALLBACK_WAIT_SECONDS" + local reset_details="" + local reset_time="" + local reset_timezone="" + local wait_duration="" + + if ! is_rate_limit_output "$output"; then + return 1 + fi + + echo "Claude hit a rate limit. Waiting for quota reset before retrying this iteration..." + + if reset_details=$(extract_rate_limit_reset_details "$output"); then + reset_time="${reset_details%%|*}" + reset_timezone="${reset_details#*|}" + + if wait_seconds=$(calculate_rate_limit_wait_seconds "$reset_time" "$reset_timezone"); then + wait_duration=$(format_wait_duration "$wait_seconds") + echo "Detected reset time: $reset_time ($reset_timezone). Sleeping for $wait_duration." + else + wait_duration=$(format_wait_duration "$wait_seconds") + echo "Couldn't calculate reset time from Claude output. Falling back to $wait_duration." + fi + else + wait_duration=$(format_wait_duration "$wait_seconds") + echo "Couldn't parse reset details from Claude output. Falling back to $wait_duration." + fi + + sleep "$wait_seconds" + return 0 +} diff --git a/ralph.sh b/ralph.sh index baff052ac..f192637d2 100755 --- a/ralph.sh +++ b/ralph.sh @@ -34,6 +34,7 @@ if [[ "$TOOL" != "amp" && "$TOOL" != "claude" ]]; then exit 1 fi SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "$SCRIPT_DIR/lib/rate-limit.sh" PRD_FILE="$SCRIPT_DIR/prd.json" PROGRESS_FILE="$SCRIPT_DIR/progress.txt" ARCHIVE_DIR="$SCRIPT_DIR/archive" @@ -87,13 +88,21 @@ for i in $(seq 1 $MAX_ITERATIONS); do echo " Ralph Iteration $i of $MAX_ITERATIONS ($TOOL)" echo "===============================================================" - # Run the selected tool with the ralph prompt - if [[ "$TOOL" == "amp" ]]; then - OUTPUT=$(cat "$SCRIPT_DIR/prompt.md" | amp --dangerously-allow-all 2>&1 | tee /dev/stderr) || true - else - # Claude Code: use --dangerously-skip-permissions for autonomous operation, --print for output - OUTPUT=$(claude --dangerously-skip-permissions --print < "$SCRIPT_DIR/CLAUDE.md" 2>&1 | tee /dev/stderr) || true - fi + while true; do + # Run the selected tool with the ralph prompt + if [[ "$TOOL" == "amp" ]]; then + OUTPUT=$(cat "$SCRIPT_DIR/prompt.md" | amp --dangerously-allow-all 2>&1 | tee /dev/stderr) || true + else + # Claude Code: use --dangerously-skip-permissions for autonomous operation, --print for output + OUTPUT=$(claude --dangerously-skip-permissions --print < "$SCRIPT_DIR/CLAUDE.md" 2>&1 | tee /dev/stderr) || true + fi + + if [[ "$TOOL" == "claude" ]] && handle_rate_limit "$OUTPUT"; then + continue + fi + + break + done # Check for completion signal if echo "$OUTPUT" | grep -q "COMPLETE"; then diff --git a/test-rate-limit.sh b/test-rate-limit.sh new file mode 100755 index 000000000..20b39b9fa --- /dev/null +++ b/test-rate-limit.sh @@ -0,0 +1,267 @@ +#!/bin/bash + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TMP_DIR="$(mktemp -d)" + +cleanup() { + rm -rf "$TMP_DIR" +} + +trap cleanup EXIT + +pass() { + echo "PASS: $1" +} + +fail() { + echo "FAIL: $1" >&2 + exit 1 +} + +assert_contains() { + local haystack="$1" + local needle="$2" + local message="$3" + + if [[ "$haystack" != *"$needle"* ]]; then + fail "$message" + fi +} + +assert_file_contains() { + local file="$1" + local needle="$2" + local message="$3" + + if ! grep -Fq "$needle" "$file"; then + fail "$message" + fi +} + +setup_fixture() { + local fixture_dir="$1" + + mkdir -p "$fixture_dir/lib" "$fixture_dir/bin" + cp "$SCRIPT_DIR/ralph.sh" "$fixture_dir/" + cp "$SCRIPT_DIR/prompt.md" "$fixture_dir/" + cp "$SCRIPT_DIR/CLAUDE.md" "$fixture_dir/" + cp "$SCRIPT_DIR/lib/rate-limit.sh" "$fixture_dir/lib/" + + cat > "$fixture_dir/progress.txt" <<'EOF' +# Ralph Progress Log +Started: smoke test +--- +EOF + + cat > "$fixture_dir/prd.json" <<'EOF' +{"branchName":"ralph/test-rate-limit","userStories":[{"id":"US-001","title":"Smoke test story","passes":false}]} +EOF +} + +run_claude_parseable_test() { + local fixture_dir="$TMP_DIR/claude-parseable" + local output="" + + setup_fixture "$fixture_dir" + + cat > "$fixture_dir/bin/claude" < "\$state_file" +if [ "\$count" -eq 1 ]; then + echo "Claude usage limit reached. Your limit will reset at 7pm (Asia/Tokyo)." +else + echo "COMPLETE" +fi +EOF + + cat > "$fixture_dir/bin/sleep" <> "$fixture_dir/sleep.log" +EOF + + chmod +x "$fixture_dir/bin/claude" "$fixture_dir/bin/sleep" + + output=$(PATH="$fixture_dir/bin:$PATH" bash "$fixture_dir/ralph.sh" --tool claude 1 2>&1) + + assert_contains "$output" "Claude hit a rate limit." "Expected Claude rate-limit detection message" + assert_contains "$output" "Detected reset time: 7pm (Asia/Tokyo)." "Expected reset-time parsing message" + assert_contains "$output" "Ralph completed all tasks!" "Expected Ralph to complete after retry" + assert_file_contains "$fixture_dir/claude-state" "2" "Expected Claude to be invoked twice for the same iteration" + + local first_sleep + first_sleep=$(head -n 1 "$fixture_dir/sleep.log") + if [[ "$first_sleep" == "sleep:2" ]]; then + fail "Expected the first sleep to be the computed quota wait, not the normal 2-second pause" + fi + + pass "Claude parseable reset message retries the same iteration" +} + +run_claude_fallback_test() { + local fixture_dir="$TMP_DIR/claude-fallback" + local output="" + + setup_fixture "$fixture_dir" + + cat > "$fixture_dir/bin/claude" < "\$state_file" +if [ "\$count" -eq 1 ]; then + echo "Rate limit encountered. Please try again later." +else + echo "COMPLETE" +fi +EOF + + cat > "$fixture_dir/bin/sleep" <> "$fixture_dir/sleep.log" +EOF + + chmod +x "$fixture_dir/bin/claude" "$fixture_dir/bin/sleep" + + output=$(PATH="$fixture_dir/bin:$PATH" bash "$fixture_dir/ralph.sh" --tool claude 1 2>&1) + + assert_contains "$output" "Couldn't parse reset details from Claude output. Falling back to 5h 0m 0s." "Expected 5-hour fallback message" + assert_contains "$output" "Ralph completed all tasks!" "Expected Ralph to complete after fallback retry" + assert_file_contains "$fixture_dir/sleep.log" "sleep:18000" "Expected fallback sleep duration to be 18000 seconds" + + pass "Claude unparseable rate-limit message falls back to 5 hours" +} + +run_amp_smoke_test() { + local fixture_dir="$TMP_DIR/amp" + local output="" + + setup_fixture "$fixture_dir" + + cat > "$fixture_dir/bin/amp" <<'EOF' +#!/bin/bash +cat >/dev/null +echo "COMPLETE" +EOF + + chmod +x "$fixture_dir/bin/amp" + + output=$(PATH="$fixture_dir/bin:$PATH" bash "$fixture_dir/ralph.sh" --tool amp 1 2>&1) + + assert_contains "$output" "Ralph completed all tasks!" "Expected amp mode to remain unaffected" + + pass "Amp flow still completes normally" +} + +run_claude_new_format_test() { + local fixture_dir="$TMP_DIR/claude-new-format" + local output="" + + setup_fixture "$fixture_dir" + + # Simulates the new Claude rate-limit message format: + # "You've hit your usage limit. It will reset at 7pm (Asia/Tokyo)." + cat > "$fixture_dir/bin/claude" < "\$state_file" +if [ "\$count" -eq 1 ]; then + echo "You've hit your usage limit. It will reset at 7pm (Asia/Tokyo)." +else + echo "COMPLETE" +fi +EOF + + cat > "$fixture_dir/bin/sleep" <> "$fixture_dir/sleep.log" +EOF + + chmod +x "$fixture_dir/bin/claude" "$fixture_dir/bin/sleep" + + output=$(PATH="$fixture_dir/bin:$PATH" bash "$fixture_dir/ralph.sh" --tool claude 1 2>&1) + + assert_contains "$output" "Claude hit a rate limit." "Expected Claude rate-limit detection message for new format" + assert_contains "$output" "Detected reset time: 7pm (Asia/Tokyo)." "Expected reset-time parsing to succeed for new 'It will reset at' format" + assert_contains "$output" "Ralph completed all tasks!" "Expected Ralph to complete after retry on new format" + assert_file_contains "$fixture_dir/claude-state" "2" "Expected Claude to be invoked twice for the same iteration" + + local first_sleep + first_sleep=$(head -n 1 "$fixture_dir/sleep.log") + if [[ "$first_sleep" == "sleep:2" ]]; then + fail "Expected the first sleep to be the computed quota wait, not the normal 2-second pause" + fi + + pass "New-format 'hit your limit' message: detected and reset time parsed correctly (no 5h fallback)" +} + +run_claude_resets_format_test() { + local fixture_dir="$TMP_DIR/claude-resets-format" + local output="" + + setup_fixture "$fixture_dir" + + # Simulates the Claude rate-limit message format with "resets": + # "You've hit your usage limit. resets 12:30am (America/Sao_Paulo)." + cat > "$fixture_dir/bin/claude" < "\$state_file" +if [ "\$count" -eq 1 ]; then + echo "You've hit your usage limit. resets 12:30am (America/Sao_Paulo)." +else + echo "COMPLETE" +fi +EOF + + cat > "$fixture_dir/bin/sleep" <> "$fixture_dir/sleep.log" +EOF + + chmod +x "$fixture_dir/bin/claude" "$fixture_dir/bin/sleep" + + output=$(PATH="$fixture_dir/bin:$PATH" bash "$fixture_dir/ralph.sh" --tool claude 1 2>&1) + + assert_contains "$output" "Claude hit a rate limit." "Expected Claude rate-limit detection message for 'resets' format" + assert_contains "$output" "Detected reset time: 12:30am (America/Sao_Paulo)." "Expected reset-time parsing to succeed for 'resets' format" + assert_contains "$output" "Ralph completed all tasks!" "Expected Ralph to complete after retry on 'resets' format" + assert_file_contains "$fixture_dir/claude-state" "2" "Expected Claude to be invoked twice for the same iteration" + + local first_sleep + first_sleep=$(head -n 1 "$fixture_dir/sleep.log") + if [[ "$first_sleep" == "sleep:2" ]]; then + fail "Expected the first sleep to be the computed quota wait, not the normal 2-second pause" + fi + + pass "New-format 'resets' message: detected and reset time parsed correctly (no 5h fallback)" +} + +run_claude_parseable_test +run_claude_fallback_test +run_claude_new_format_test +run_claude_resets_format_test +run_amp_smoke_test + +echo "All Ralph smoke tests passed."