Advanced Web Scraping with Python: Proxies, CAPTCHAs, and Anti-Detection

Basic web scraping breaks when sites deploy anti-bot measures. This guide teaches advanced techniques to scrape reliably at scale using rotating proxies, browser fingerprint management, and CAPTCHA handling strategies.

What You'll Build

A production-grade web scraping system that bypasses common anti-bot protections. You'll learn to rotate IP addresses, manage browser fingerprints, and handle CAPTCHAs while maintaining high success rates.

Why Advanced Scraping Matters

Modern websites deploy sophisticated anti-bot systems:

• IP rate limiting — blocking repeated requests from the same address
• Browser fingerprinting — detecting automation tools
• Behavioral analysis — flagging non-human interaction patterns
• CAPTCHA challenges — blocking suspicious traffic

Full Tutorial

1. Rotating Proxy Infrastructure

# proxy_manager.py
import random
import asyncio
import aiohttp
from dataclasses import dataclass
from typing import Optional

@dataclass
class Proxy:
    host: str
    port: int
    username: str = ""
    password: str = ""
    protocol: str = "http"

    @property
    def url(self) -> str:
        if self.username:
            return f"{self.protocol}://{self.username}:{self.password}@{self.host}:{self.port}"
        return f"{self.protocol}://{self.host}:{self.port}"

class ProxyManager:
    def __init__(self):
        self.proxies: list[Proxy] = []
        self.failed: set[str] = set()

    def add_proxy(self, proxy: Proxy):
        self.proxies.append(proxy)

    def load_from_file(self, filepath: str):
        with open(filepath) as f:
            for line in f:
                line = line.strip()
                if not line:
                    continue
                parts = line.split(":")
                if len(parts) == 2:
                    self.proxies.append(Proxy(host=parts[0], port=int(parts[1])))
                elif len(parts) == 4:
                    self.proxies.append(Proxy(
                        host=parts[0], port=int(parts[1]),
                        username=parts[2], password=parts[3]
                    ))

    def get_random(self) -> Optional[Proxy]:
        available = [p for p in self.proxies if p.url not in self.failed]
        if not available:
            self.failed.clear()
            available = self.proxies
        return random.choice(available) if available else None

    def mark_failed(self, proxy: Proxy):
        self.failed.add(proxy.url)

    async def test_proxy(self, proxy: Proxy, timeout: int = 10) -> bool:
        try:
            async with aiohttp.ClientSession() as session:
                async with session.get(
                    "http://httpbin.org/ip",
                    proxy=proxy.url,
                    timeout=aiohttp.ClientTimeout(total=timeout)
                ) as resp:
                    return resp.status == 200
        except Exception:
            return False

2. Browser Fingerprint Management

# fingerprint_manager.py
import random

FINGERPRINTS = [
    {
        "user_agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
        "viewport": {"width": 1920, "height": 1080},
        "platform": "Win32",
        "languages": ["en-US", "en"]
    },
    {
        "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
        "viewport": {"width": 1440, "height": 900},
        "platform": "MacIntel",
        "languages": ["en-US", "en"]
    },
    {
        "user_agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
        "viewport": {"width": 1920, "height": 1080},
        "platform": "Linux x86_64",
        "languages": ["en-US", "en"]
    }
]

class FingerprintManager:
    def __init__(self):
        self.used = set()

    def get_fingerprint(self) -> dict:
        available = [f for i, f in enumerate(FINGERPRINTS) if i not in self.used]
        if not available:
            self.used.clear()
            available = FINGERPRINTS

        fp = random.choice(available)
        self.used.add(FINGERPRINTS.index(fp))
        return fp

3. Stealth Playwright Configuration

# stealth_scraper.py
from playwright.async_api import async_playwright
from proxy_manager import ProxyManager
from fingerprint_manager import FingerprintManager
import asyncio

class StealthScraper:
    def __init__(self):
        self.proxy_manager = ProxyManager()
        self.fingerprint_manager = FingerprintManager()

    async def create_stealth_page(self):
        fp = self.fingerprint_manager.get_fingerprint()
        proxy = self.proxy_manager.get_random()

        launch_args = {
            "args": [
                "--disable-blink-features=AutomationControlled",
                "--disable-features=IsolateOrigins,site-per-process",
                "--no-sandbox",
            ]
        }

        if proxy:
            launch_args["proxy"] = {"server": proxy.url}

        async with async_playwright() as p:
            browser = await p.chromium.launch(**launch_args)
            context = await browser.new_context(
                user_agent=fp["user_agent"],
                viewport=fp["viewport"],
                locale="en-US",
            )

            # Remove webdriver detection
            await context.add_init_script("""
                Object.defineProperty(navigator, 'webdriver', {
                    get: () => undefined
                });

                window.chrome = {
                    runtime: {}
                };

                Object.defineProperty(navigator, 'plugins', {
                    get: () => [1, 2, 3, 4, 5]
                });

                Object.defineProperty(navigator, 'languages', {
                    get: () => ['en-US', 'en']
                });
            """)

            page = await context.new_page()
            return browser, page

    async def scrape(self, url: str) -> str:
        browser, page = await self.create_stealth_page()
        try:
            await page.goto(url, wait_until="networkidle")
            content = await page.content()
            return content
        finally:
            await browser.close()

4. Human-Like Behavior Simulation

# human_behavior.py
import asyncio
import random

class HumanBehavior:
    @staticmethod
    async def random_delay(min_ms: int = 500, max_ms: int = 2000):
        delay = random.uniform(min_ms / 1000, max_ms / 1000)
        await asyncio.sleep(delay)

    @staticmethod
    async def human_scroll(page):
        scroll_amount = random.randint(300, 800)
        await page.mouse.wheel(0, scroll_amount)
        await HumanBehavior.random_delay(1000, 3000)

    @staticmethod
    async def human_click(page, selector: str):
        element = await page.query_selector(selector)
        if element:
            box = await element.bounding_box()
            if box:
                x = box["x"] + random.uniform(box["width"] * 0.2, box["width"] * 0.8)
                y = box["y"] + random.uniform(box["height"] * 0.2, box["height"] * 0.8)
                await page.mouse.move(x, y, steps=random.randint(5, 15))
                await HumanBehavior.random_delay(100, 300)
                await page.mouse.click(x, y)

    @staticmethod
    async def human_type(page, selector: str, text: str):
        await page.click(selector)
        for char in text:
            await page.keyboard.type(char, delay=random.uniform(50, 150))
            if random.random() < 0.05:
                await asyncio.sleep(random.uniform(0.5, 1.5))

5. CAPTCHA Detection and Handling

# captcha_handler.py
import asyncio
from playwright.async_api import Page

class CaptchaHandler:
    CAPTCHA_SELECTORS = [
        "iframe[src*='captcha']",
        "iframe[src*='recaptcha']",
        "iframe[src*='hcaptcha']",
        "#captcha",
        ".captcha",
        "[data-captcha]"
    ]

    @staticmethod
    async def detect_captcha(page: Page) -> bool:
        for selector in CaptchaHandler.CAPTCHA_SELECTORS:
            element = await page.query_selector(selector)
            if element:
                return True
        return False

    @staticmethod
    async def wait_for_manual_solve(page: Page, timeout: int = 120):
        start = asyncio.get_event_loop().time()
        while asyncio.get_event_loop().time() - start < timeout:
            if not await CaptchaHandler.detect_captcha(page):
                return True
            await asyncio.sleep(2)
        return False

6. Complete Scraping Pipeline

# scraper_pipeline.py
import asyncio
from stealth_scraper import StealthScraper
from captcha_handler import CaptchaHandler
from human_behavior import HumanBehavior

class ScraperPipeline:
    def __init__(self):
        self.stealth = StealthScraper()
        self.results = []

    async def scrape_page(self, url: str) -> dict:
        browser, page = await self.stealth.create_stealth_page()
        try:
            await page.goto(url, wait_until="networkidle")
            await HumanBehavior.random_delay(1000, 3000)

            if await CaptchaHandler.detect_captcha(page):
                print(f"CAPTCHA detected at {url}, waiting...")
                solved = await CaptchaHandler.wait_for_manual_solve(page)
                if not solved:
                    return {"url": url, "error": "CAPTCHA timeout"}

            await HumanBehavior.human_scroll(page)
            await HumanBehavior.random_delay(1000, 2000)

            title = await page.title()
            return {"url": url, "title": title, "status": "success"}
        finally:
            await browser.close()

    async def run(self, urls: list[str], concurrency: int = 3):
        semaphore = asyncio.Semaphore(concurrency)

        async def limited_scrape(url):
            async with semaphore:
                result = await self.scrape_page(url)
                self.results.append(result)

        await asyncio.gather(*[limited_scrape(url) for url in urls])
        return self.results

pipeline = ScraperPipeline()
urls = ["https://example.com/page1", "https://example.com/page2"]
results = asyncio.run(pipeline.run(urls, concurrency=2))
print(results)

Rate Limiting Best Practices

• Add 2-5 second delays between requests to the same domain
• Rotate user agents every 10-20 requests
• Use multiple proxy pools to distribute load
• Monitor response codes and back off on 429 errors
• Respect robots.txt where legally required

Get the Code

Ready to use these tools? Browse our collection of tested, production-ready Python scripts:

🔗 Browse Products: Anna's Digital Products

All products include:

• ✅ Tested and verified code
• ✅ Instant delivery via crypto or card
• ✅ Free updates forever
• ✅ Telegram bot support (@AnnaLilithBot)