Python Web Scraping with httpx and BeautifulSoup4

Web scraping is a core skill for any Python developer. In this tutorial we'll use httpx (a modern HTTP client) and BeautifulSoup4 (the go-to HTML parser) to extract data from any website.

Installation

pip install httpx beautifulsoup4 lxml

Why httpx over requests?

• Built-in HTTP/2 support
• Async-ready (same API for sync and async)
• Better timeout and retry handling
• Type annotations out of the box

Complete Example: Scraping Hacker News

import httpx
from bs4 import BeautifulSoup
import json
from datetime import datetime

BASE_URL = "https://news.ycombinator.com"

def fetch_page(url: str) -> str:
    headers = {
        "User-Agent": (
            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
            "AppleWebKit/537.36 (KHTML, like Gecko) "
            "Chrome/120.0.0.0 Safari/537.36"
        ),
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
        "Accept-Language": "en-US,en;q=0.5",
    }
    with httpx.Client(timeout=15, follow_redirects=True) as client:
        response = client.get(url, headers=headers)
        response.raise_for_status()
        return response.text

def parse_stories(html: str) -> list[dict]:
    soup = BeautifulSoup(html, "lxml")
    stories = []
    for row in soup.select("tr.athing"):
        story_id = row.get("id")
        title_tag = row.select_one("span.titleline > a")
        if not title_tag:
            continue
        title = title_tag.get_text(strip=True)
        link = title_tag.get("href", "")
        subtext = row.find_next_sibling("tr")
        score_tag = subtext.select_one("span.score") if subtext else None
        score = score_tag.get_text(strip=True) if score_tag else "0 points"
        if link.startswith("item?"):
            link = f"{BASE_URL}/{link}"
        stories.append({
            "id": story_id,
            "title": title,
            "url": link,
            "score": score,
        })
    return stories

def scrape_hn_frontpage() -> list[dict]:
    try:
        html = fetch_page(BASE_URL)
        stories = parse_stories(html)
        print(f"Scraped {len(stories)} stories")
        return stories
    except httpx.HTTPStatusError as e:
        print(f"HTTP error {e.response.status_code}: {e.request.url}")
        return []
    except httpx.RequestError as e:
        print(f"Network error: {e}")
        return []

def save_results(data: list[dict], filename: str = "hn_stories.json"):
    with open(filename, "w", encoding="utf-8") as f:
        json.dump({
            "scraped_at": datetime.now().isoformat(),
            "count": len(data),
            "stories": data,
        }, f, indent=2, ensure_ascii=False)
    print(f"Saved {len(data)} stories to {filename}")

if __name__ == "__main__":
    stories = scrape_hn_frontpage()
    if stories:
        for i, s in enumerate(stories[:5], 1):
            print(f"{i}. {s['title']} ({s['score']})")
        save_results(stories)

Error Handling Best Practices

import httpx
from bs4 import BeautifulSoup
import time

def robust_fetch(url: str, retries: int = 3, delay: float = 1.0) -> str | None:
    for attempt in range(retries):
        try:
            with httpx.Client(timeout=10) as client:
                resp = client.get(url, headers={"User-Agent": "Mozilla/5.0"})
                resp.raise_for_status()
                return resp.text
        except httpx.HTTPStatusError as e:
            if e.response.status_code == 429:
                wait = delay * (2 ** attempt)
                print(f"Rate limited. Waiting {wait}s...")
                time.sleep(wait)
            elif e.response.status_code in (403, 404):
                print(f"Permanent error {e.response.status_code}, stopping")
                return None
            else:
                print(f"Attempt {attempt + 1} failed: {e}")
        except httpx.TimeoutException:
            print(f"Timeout on attempt {attempt + 1}")
            time.sleep(delay)
    return None

def safe_extract(soup, selector: str, attr: str = None) -> str:
    tag = soup.select_one(selector)
    if tag is None:
        return ""
    if attr:
        return tag.get(attr, "")
    return tag.get_text(strip=True)

Practical Tips

• Respect robots.txt - check site.com/robots.txt before scraping
• Add delays between requests: time.sleep(1) avoids getting blocked
• Rotate User-Agent strings for large-scale scraping
• Use lxml parser - it is faster than Python's built-in html.parser
• CSS selectors (soup.select()) are more readable than .find_all()
• Store raw HTML first, parse later - easier to debug and re-process

---

Follow me for more Python tips! 🐍