Scraping Freelance Platforms: Upwork, Fiverr, and Toptal

Freelance platforms hold valuable market data: pricing trends, in-demand skills, and competitive intelligence. Scraping Upwork, Fiverr, and Toptal can reveal what clients are willing to pay and what services are trending.

Why Scrape Freelance Platforms?

• Pricing research — understand market rates for your skills
• Trend analysis — spot emerging skills before they peak
• Lead generation — find potential clients posting relevant jobs
• Competitive intelligence — see how top freelancers position themselves

Setting Up

pip install requests beautifulsoup4 pandas

Scraping Upwork Job Listings

import requests
from bs4 import BeautifulSoup
from dataclasses import dataclass
import time

@dataclass
class FreelanceJob:
    title: str
    description: str
    budget: str
    skills: list
    platform: str
    url: str

def scrape_upwork_jobs(search_term, api_key, pages=3):
    jobs = []
    for page in range(1, pages + 1):
        url = f"https://www.upwork.com/nx/search/jobs/?q={search_term}&page={page}"
        proxy_url = f"http://api.scraperapi.com?api_key={api_key}&url={url}&render=true"
        response = requests.get(proxy_url, timeout=60)
        soup = BeautifulSoup(response.text, "html.parser")

        job_tiles = soup.find_all("article")
        for tile in job_tiles:
            title_el = tile.find("a")
            desc_el = tile.find("span", attrs={"data-test": "job-description-text"})
            budget_el = tile.find("strong", attrs={"data-test": "budget"})
            skill_els = tile.find_all("span", attrs={"data-test": "token"})

            jobs.append(FreelanceJob(
                title=title_el.get_text(strip=True) if title_el else "",
                description=desc_el.get_text(strip=True)[:300] if desc_el else "",
                budget=budget_el.get_text(strip=True) if budget_el else "N/A",
                skills=[s.get_text(strip=True) for s in skill_els],
                platform="upwork",
                url=title_el["href"] if title_el and title_el.get("href") else ""
            ))
        time.sleep(3)
    return jobs

jobs = scrape_upwork_jobs("python web scraping", "YOUR_KEY")
for job in jobs[:5]:
    print(f"{job.title} - {job.budget}")

Scraping Fiverr Gigs

def scrape_fiverr_gigs(query, api_key, pages=2):
    gigs = []
    for page in range(1, pages + 1):
        url = f"https://www.fiverr.com/search/gigs?query={query}&page={page}"
        proxy_url = f"http://api.scraperapi.com?api_key={api_key}&url={url}&render=true"
        response = requests.get(proxy_url, timeout=60)
        soup = BeautifulSoup(response.text, "html.parser")

        cards = soup.find_all("div", class_=lambda c: c and "gig-card" in c if c else False)
        for card in cards:
            title = card.find("p") or card.find("h3")
            price = card.find("span", class_=lambda c: c and "price" in c.lower() if c else False)
            seller = card.find("span", class_=lambda c: c and "seller" in c.lower() if c else False)

            gigs.append({
                "title": title.get_text(strip=True) if title else "",
                "price": price.get_text(strip=True) if price else "N/A",
                "seller": seller.get_text(strip=True) if seller else "",
                "platform": "fiverr"
            })
        time.sleep(3)
    return gigs

gigs = scrape_fiverr_gigs("web scraping", "YOUR_KEY")
for g in gigs[:5]:
    print(f"{g[title]} - {g[price]}")

Scraping Toptal Profiles

def scrape_toptal(skill, api_key):
    url = f"https://www.toptal.com/developers/{skill}"
    proxy_url = f"http://api.scraperapi.com?api_key={api_key}&url={url}&render=true"
    response = requests.get(proxy_url, timeout=60)
    soup = BeautifulSoup(response.text, "html.parser")

    devs = []
    for profile in soup.find_all("div", class_=lambda c: c and "talent" in c.lower() if c else False):
        name = profile.find("h3")
        title = profile.find("p")
        devs.append({
            "name": name.get_text(strip=True) if name else "",
            "title": title.get_text(strip=True) if title else "",
            "platform": "toptal"
        })
    return devs

Market Analysis

import pandas as pd
from collections import Counter

class FreelanceAnalyzer:
    def __init__(self, api_key):
        self.api_key = api_key
        self.jobs = []
        self.gigs = []

    def collect(self, terms):
        for term in terms:
            self.jobs.extend(scrape_upwork_jobs(term, self.api_key, 2))
            time.sleep(5)
            self.gigs.extend(scrape_fiverr_gigs(term, self.api_key, 2))
            time.sleep(5)

    def trending_skills(self):
        skills = []
        for job in self.jobs:
            skills.extend(job.skills)
        return Counter(skills).most_common(20)

    def price_range(self):
        prices = []
        for gig in self.gigs:
            try:
                p = float(gig["price"].replace("$", "").replace(",", "").strip())
                prices.append(p)
            except ValueError:
                continue
        if prices:
            return {"min": min(prices), "max": max(prices), "avg": sum(prices)/len(prices)}
        return {}

    def export(self, name="freelance"):
        pd.DataFrame([vars(j) for j in self.jobs]).to_csv(f"{name}_jobs.csv", index=False)
        pd.DataFrame(self.gigs).to_csv(f"{name}_gigs.csv", index=False)

analyzer = FreelanceAnalyzer("YOUR_KEY")
analyzer.collect(["python", "web scraping", "data engineering"])
for skill, count in analyzer.trending_skills()[:10]:
    print(f"{skill}: {count}")

Handling Anti-Bot Measures

Freelance platforms use aggressive bot detection:

1. ScraperAPI handles JS rendering and CAPTCHAs for all three platforms
2. ThorData residential proxies are essential since datacenter IPs get blocked instantly
3. Rate limiting — minimum 3-5 seconds between requests
4. Monitor with ScrapeOps to track success rates

Ethical Guidelines

• Respect Terms of Service
• Only scrape publicly visible data
• Use for market research, not spam
• Consider official APIs (Upwork has one)
• Cache results to reduce server load

Conclusion

Scraping freelance platforms reveals what skills are in demand, what clients pay, and how the market shifts. Combine Upwork job data with Fiverr pricing and Toptal talent profiles for comprehensive market intelligence.

Happy scraping!