Scraping 100k Pages Per Day With Python asyncio: The Architecture That Actually Works

Most asyncio scraping tutorials show you how to fetch 10 URLs concurrently. They do not show you how to handle backpressure, retry logic, proxy rotation, rate limiting per domain, and persistent queues — the things you need for production.

Here is the architecture I use to scrape 100k pages/day reliably on a $6/month VPS.

Why Naive asyncio Fails at Scale

This is what everyone starts with:

import asyncio, aiohttp

async def fetch_all(urls):
    async with aiohttp.ClientSession() as session:
        tasks = [fetch(session, url) for url in urls]
        return await asyncio.gather(*tasks)

With 10k URLs this will:

• Open 10k simultaneous connections → immediate connection refused / rate limiting
• Consume all available file descriptors
• Hammer target servers and get IP-banned in minutes
• Fail silently if gather hits an exception

The Production Architecture

┌─────────────┐    ┌──────────────┐    ┌──────────────┐
│  URL Queue  │───▶│  Dispatcher  │───▶│  Worker Pool │
│  (asyncio   │    │  (rate limit │    │  (N workers, │
│   Queue)    │    │  per domain) │    │  semaphore)  │
└─────────────┘    └──────────────┘    └──────────────┘
                                              │
                                    ┌─────────▼────────┐
                                    │  Retry Queue     │
                                    │  (exp. backoff)  │
                                    └──────────────────┘

Component 1: Bounded Worker Pool

import asyncio
import aiohttp
from asyncio import Queue, Semaphore
from typing import AsyncGenerator

class ScraperPool:
    def __init__(self, max_workers: int = 50, max_per_domain: int = 2):
        self.semaphore = Semaphore(max_workers)  # Global concurrency limit
        self.domain_semaphores: dict[str, Semaphore] = {}  # Per-domain limits
        self.domain_lock = asyncio.Lock()

    async def get_domain_sem(self, domain: str) -> Semaphore:
        async with self.domain_lock:
            if domain not in self.domain_semaphores:
                self.domain_semaphores[domain] = Semaphore(2)  # Max 2 concurrent/domain
            return self.domain_semaphores[domain]

    async def fetch(self, session: aiohttp.ClientSession, url: str) -> str:
        from urllib.parse import urlparse
        domain = urlparse(url).netloc
        domain_sem = await self.get_domain_sem(domain)

        async with self.semaphore:  # Global limit
            async with domain_sem:  # Per-domain limit
                return await self._fetch_with_retry(session, url)

Component 2: Exponential Backoff Retry

import random
from asyncio import sleep

async def _fetch_with_retry(
    self, 
    session: aiohttp.ClientSession, 
    url: str,
    max_retries: int = 3
) -> str | None:

    for attempt in range(max_retries):
        try:
            timeout = aiohttp.ClientTimeout(total=30)
            async with session.get(url, timeout=timeout) as resp:
                if resp.status == 429:  # Rate limited
                    retry_after = int(resp.headers.get('Retry-After', 60))
                    await sleep(retry_after)
                    continue

                if resp.status in (403, 404, 410):  # Permanent failures
                    return None  # Don't retry

                if resp.status >= 500:  # Transient server errors
                    raise aiohttp.ClientError(f"Server error {resp.status}")

                return await resp.text()

        except (aiohttp.ClientError, asyncio.TimeoutError) as e:
            if attempt == max_retries - 1:
                return None  # Give up after max retries

            # Exponential backoff with jitter
            wait = (2 ** attempt) + random.uniform(0, 1)
            await sleep(wait)

    return None

Component 3: Proxy Rotation

import itertools

class ProxyRotator:
    def __init__(self, proxies: list[str]):
        self._cycle = itertools.cycle(proxies)
        self._failed: set[str] = set()
        self._lock = asyncio.Lock()

    async def get_proxy(self) -> str:
        async with self._lock:
            for _ in range(len(self._failed) + 10):
                proxy = next(self._cycle)
                if proxy not in self._failed:
                    return proxy
        raise RuntimeError("All proxies exhausted")

    async def mark_failed(self, proxy: str):
        async with self._lock:
            self._failed.add(proxy)

    def create_connector(self, proxy: str):
        return aiohttp.TCPConnector(ssl=False)  # + proxy header injection

Component 4: Persistent Queue (Survive Restarts)

import aiosqlite

class PersistentQueue:
    """SQLite-backed queue that survives process restarts."""

    def __init__(self, db_path: str):
        self.db_path = db_path

    async def init(self):
        async with aiosqlite.connect(self.db_path) as db:
            await db.execute("""
                CREATE TABLE IF NOT EXISTS queue (
                    id INTEGER PRIMARY KEY AUTOINCREMENT,
                    url TEXT UNIQUE,
                    status TEXT DEFAULT 'pending',
                    attempts INTEGER DEFAULT 0,
                    created_at DATETIME DEFAULT CURRENT_TIMESTAMP
                )
            """)
            await db.commit()

    async def add_urls(self, urls: list[str]):
        async with aiosqlite.connect(self.db_path) as db:
            await db.executemany(
                "INSERT OR IGNORE INTO queue (url) VALUES (?)",
                [(u,) for u in urls]
            )
            await db.commit()

    async def get_batch(self, size: int = 100) -> list[str]:
        async with aiosqlite.connect(self.db_path) as db:
            # Claim a batch atomically
            await db.execute("""
                UPDATE queue SET status='claimed' 
                WHERE id IN (SELECT id FROM queue WHERE status='pending' LIMIT ?)
            """, (size,))
            await db.commit()

            cursor = await db.execute(
                "SELECT url FROM queue WHERE status='claimed'"
            )
            rows = await cursor.fetchall()
            return [r[0] for r in rows]

Putting It Together: Main Loop

async def main():
    queue = PersistentQueue("scraper.db")
    await queue.init()
    await queue.add_urls(load_urls_from_file("urls.txt"))

    proxies = load_proxies("proxies.txt")
    rotator = ProxyRotator(proxies)
    pool = ScraperPool(max_workers=50)

    connector = aiohttp.TCPConnector(limit=100)
    async with aiohttp.ClientSession(connector=connector) as session:
        while True:
            batch = await queue.get_batch(100)
            if not batch:
                break

            results = await asyncio.gather(
                *[pool.fetch(session, url) for url in batch],
                return_exceptions=True
            )

            # Process results, mark completed/failed
            for url, result in zip(batch, results):
                if isinstance(result, Exception) or result is None:
                    await queue.mark_failed(url)
                else:
                    data = parse_page(result)
                    await save_to_db(data)
                    await queue.mark_done(url)

if __name__ == "__main__":
    asyncio.run(main())

Performance Numbers

With 50 workers, 2 per-domain limit, 30s timeout, residential proxies:

Metric	Value
Pages/hour	~4,200
Pages/day	~100,000
RAM usage	~180MB
Success rate	94%
Cost (proxies)	~$0.003/page

Ready-to-Use Async Scrapers

Building the full retry/backpressure/proxy-rotation stack takes a week. I have packaged production-ready async scrapers:

Async Scraper Toolkit — €29

Includes the full architecture above pre-configured, Playwright async integration for JS sites, domain-aware rate limiting, and persistent SQLite queue.

---

Running asyncio scrapers in production? What concurrency numbers are you hitting before you start getting blocked?