German Yamil

Posted on May 29

Python requests: The Complete HTTP Client Guide

#python #beginners #tutorial #codenewbie

Why requests Over urllib

Python's built-in urllib works but is verbose. requests is what everyone actually uses:

# urllib — verbose
import urllib.request, json
req = urllib.request.Request("https://api.example.com/data",
                              headers={"Authorization": "Bearer token"})
with urllib.request.urlopen(req) as r:
    data = json.loads(r.read().decode())

# requests — clean
import requests
r = requests.get("https://api.example.com/data",
                 headers={"Authorization": "Bearer token"})
data = r.json()

Install: pip install requests

GET Requests

import requests

# Basic GET
r = requests.get("https://jsonplaceholder.typicode.com/posts/1")
print(r.status_code)   # 200
print(r.json())        # {'userId': 1, 'id': 1, 'title': ...}

# With query parameters
r = requests.get(
    "https://jsonplaceholder.typicode.com/posts",
    params={"userId": 1, "_limit": 5}
)
# URL becomes: .../posts?userId=1&_limit=5
posts = r.json()
print(len(posts))  # 5

POST Requests

# Send JSON
r = requests.post(
    "https://jsonplaceholder.typicode.com/posts",
    json={"title": "My Post", "body": "Content here", "userId": 1}
)
print(r.status_code)  # 201
print(r.json())       # {'id': 101, 'title': 'My Post', ...}

# Send form data
r = requests.post(
    "https://example.com/login",
    data={"username": "user", "password": "pass"}
)

# Send raw body
r = requests.post(
    "https://example.com/upload",
    data=b"raw bytes here",
    headers={"Content-Type": "application/octet-stream"}
)

Headers and Authentication

# Custom headers
headers = {
    "Authorization": "Bearer your-token-here",
    "User-Agent": "MyApp/1.0",
    "Accept": "application/json",
}
r = requests.get("https://api.example.com/data", headers=headers)

# Basic auth
r = requests.get(
    "https://api.example.com/protected",
    auth=("username", "password")
)

# API key in header (common pattern)
r = requests.get(
    "https://api.example.com/data",
    headers={"X-API-Key": "your-api-key"}
)

Error Handling

import requests
from requests.exceptions import RequestException, Timeout, ConnectionError

def safe_get(url: str, **kwargs) -> dict | None:
    try:
        r = requests.get(url, timeout=10, **kwargs)
        r.raise_for_status()  # Raises for 4xx, 5xx
        return r.json()
    except Timeout:
        print(f"Request timed out: {url}")
    except ConnectionError:
        print(f"Connection failed: {url}")
    except requests.HTTPError as e:
        print(f"HTTP error {e.response.status_code}: {url}")
    except RequestException as e:
        print(f"Request failed: {e}")
    return None

data = safe_get("https://api.example.com/data")

Sessions: Reuse Connections

Sessions persist headers, cookies, and TCP connections across requests:

import requests

# Without session — new connection each time
r1 = requests.get(url1, headers={"Authorization": "Bearer token"})
r2 = requests.get(url2, headers={"Authorization": "Bearer token"})

# With session — one connection, shared headers
with requests.Session() as s:
    s.headers.update({"Authorization": "Bearer token"})
    r1 = s.get(url1)  # Connection reused
    r2 = s.get(url2)  # Faster

Use sessions when making multiple requests to the same host.

Timeouts

Always set timeouts. Without them, requests can hang forever:

# Timeout after 5 seconds
r = requests.get(url, timeout=5)

# Connect timeout vs read timeout
r = requests.get(url, timeout=(3, 10))
# (connect_timeout, read_timeout)
# Fails if connection takes >3s or response takes >10s

Working with Response Data

r = requests.get("https://jsonplaceholder.typicode.com/posts/1")

r.status_code        # 200
r.ok                 # True (status < 400)
r.headers            # {'Content-Type': 'application/json', ...}
r.text               # Raw string response
r.content            # Raw bytes (for images, files)
r.json()             # Parse JSON → dict/list
r.url                # Final URL (after redirects)
r.elapsed            # Time taken (timedelta)

File Upload

# Upload a file
with open("data.csv", "rb") as f:
    r = requests.post(
        "https://api.example.com/upload",
        files={"file": ("data.csv", f, "text/csv")}
    )

# Upload with additional form fields
with open("image.png", "rb") as f:
    r = requests.post(
        "https://api.example.com/upload",
        files={"image": f},
        data={"title": "My Image"}
    )

Retry Logic

from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry

def create_session_with_retries(
    retries: int = 3,
    backoff_factor: float = 0.3,
) -> requests.Session:
    session = requests.Session()
    retry = Retry(
        total=retries,
        backoff_factor=backoff_factor,
        status_forcelist=[429, 500, 502, 503, 504],
    )
    adapter = HTTPAdapter(max_retries=retry)
    session.mount("http://", adapter)
    session.mount("https://", adapter)
    return session

s = create_session_with_retries()
r = s.get("https://api.example.com/data", timeout=10)

Quick Reference

Task	Code
GET	`requests.get(url)`
POST JSON	`requests.post(url, json=data)`
Headers	`requests.get(url, headers={...})`
Auth	`requests.get(url, auth=(user, pass))`
Params	`requests.get(url, params={...})`
Timeout	`requests.get(url, timeout=10)`
Check status	`r.raise_for_status()`
Parse JSON	`r.json()`

Get the Full Pipeline

This article is part of the Python AI Publishing Pipeline series — a complete system to write, validate, and publish technical ebooks with Python and Claude.

📋 Free checklist: 7 steps to ship a Python ebook — PDF, no email required.

🚀 Full pipeline + source code: germy5.gumroad.com/l/xhxkzz — $9.99, 30-day money-back guarantee.

If this was useful, the ❤️ button helps other developers find it.

Building a Python content pipeline? I sell the complete automation system as a one-time download — Dev.to API, Claude API, launchd, Gumroad. Check it out ($9.99)

DEV Community