SERP API Error Handling: 5 Patterns from 90-Day Production

5 Error Patterns

1. Retry with Exponential Backoff

import time
import random

def fetch_with_retry(url, headers, body, max_retries=3):
    for attempt in range(max_retries):
        try:
            r = requests.post(url, headers=headers, json=body, timeout=10)
            r.raise_for_status()
            return r.json()
        except requests.exceptions.Timeout:
            if attempt == max_retries - 1:
                raise
            wait = (2 ** attempt) + random.uniform(0, 1)
            time.sleep(wait)
        except requests.exceptions.HTTPError as e:
            if 400 <= e.response.status_code < 500:
                raise  # 4xx, don't retry
            if e.response.status_code == 429:
                retry_after = int(e.response.headers.get("Retry-After", 1))
                time.sleep(retry_after)
                continue
            if attempt == max_retries - 1:
                raise
            time.sleep(2 ** attempt)

2. Circuit Breaker

class CircuitBreaker:
    def __init__(self, failure_threshold=10, cooldown=30):
        self.failure_count = 0
        self.last_failure_time = 0
        self.failure_threshold = failure_threshold
        self.cooldown = cooldown
        self.state = "CLOSED"  # CLOSED / OPEN / HALF_OPEN

    def call(self, func, *args, **kwargs):
        if self.state == "OPEN":
            if time.time() - self.last_failure_time > self.cooldown:
                self.state = "HALF_OPEN"
            else:
                raise Exception("Circuit open")

        try:
            result = func(*args, **kwargs)
            if self.state == "HALF_OPEN":
                self.state = "CLOSED"
                self.failure_count = 0
            return result
        except Exception:
            self.failure_count += 1
            self.last_failure_time = time.time()
            if self.failure_count >= self.failure_threshold:
                self.state = "OPEN"
            raise

3. Fallback to Alternative

def fetch_serp_with_fallback(query, params):
    try:
        return requests.post(SERPBASE_URL, json=params, headers=headers, timeout=10).json()
    except Exception as e:
        log_warning(f"SerpBase failed: {e}, fallback to cache")
        return get_cached_or_stale(query)  # Degrade to cache

4. Rate Limit Aware (Read 429 Header)

def fetch_serp_rate_aware(url, headers, body, max_retries=3):
    for attempt in range(max_retries):
        try:
            r = requests.post(url, headers=headers, json=body, timeout=10)
            r.raise_for_status()
            return r.json()
        except requests.exceptions.HTTPError as e:
            if e.response.status_code == 429:
                retry_after = int(e.response.headers.get("Retry-After", 1))
                # Use retry-after time wisely, degrade to cache
                if retry_after > 60:
                    log_warning(f"Long rate limit: {retry_after}s, use cache")
                    return get_cached_or_stale(body.get("q"))
                time.sleep(retry_after)
                continue
            raise

5. Graceful Degradation

def fetch_serp_or_fail(query, params, max_age=3600):
    try:
        return requests.post(SERPBASE_URL, json=params, headers=headers, timeout=10).json()
    except Exception:
        # On failure, fall back to old cache
        cached = redis.get(f"serp:fallback:{hash(query)}")
        if cached:
            data = json.loads(cached)
            if (time.time() - data["ts"]) < max_age:
                return data["response"]
        # Cache also expired, return None
        return None

5 Patterns Combined

def robust_fetch(query, params):
    breaker = CircuitBreaker(failure_threshold=5, cooldown=30)

    for attempt in range(3):
        try:
            return breaker.call(
                fetch_serp_with_retry, 
                SERPBASE_URL, headers, params
            )
        except RateLimitError:
            time.sleep(60)
            continue
        except CircuitOpenError:
            # Circuit open → degrade
            return fetch_serp_fallback(query)
        except Exception:
            # Other error → degrade
            return fetch_serp_graceful_degradation(query, params)

90-Day Data

Error Type	Count	Retry Success	Circuit Breaker	Fallback Used
Network jitter	35	35 (100%)	0	0
Vendor 5xx	12	8 (67%)	1	4
QPS exceeded	8	6 (75%)	0	2
4xx client error	3	0 (no retry)	0	0
Total	58	49 (85%)	1	6

85% errors solved by retry, 15% trigger fallback, actual real failures: 0.

4 Key Design Points

1. Don't Retry 4xx

# Wrong: retry on any error
# Right: 4xx (client error) no retry, 5xx / timeout retry

2. Add Jitter to Retry

# Wrong: fixed 1s backoff
# Right: exponential + random jitter, avoid thundering herd
wait = (2 ** attempt) + random.uniform(0, 1)

3. Circuit Breaker Not Too Sensitive

# Wrong: open on 1 failure
# Right: 5-10 consecutive failures before open

4. Fallback Use Stale-While-Revalidate

# Wrong: return None on failure
# Right: return old data + mark stale, refresh next time

Combining with Other Patterns

• retry + circuit breaker: prevent cascade
• fallback + degradation: ensure availability
• rate limit aware: don't waste retry quota
• monitor + alert: long-term pattern observation

Monitoring Alerts

Metric	Threshold	Alert
Failure rate	> 1% / 5 min	Slack
Circuit breaker open count	> 1 / hour	Slack + oncall
Fallback trigger rate	> 5% / 5 min	Slack
Auto-refund rate	> 5% / 1 hour	Slack (vendor issue)

100 free searches: serpbase.dev signup, no card required.