DEV Community

agenthustler
agenthustler

Posted on

Scraping Kickstarter and Indiegogo for Crowdfunding Trends

#python #tutorial #webdev #programming

Crowdfunding platforms are a window into consumer demand and innovation trends. By scraping Kickstarter and Indiegogo, you can spot what's resonating with backers before these products hit the mainstream market.

What We'll Track

  • Campaign funding amounts and backer counts
  • Category trends (tech, design, games, etc.)
  • Success rates and funding velocity
  • Geographic distribution of projects

Setup 

pip install requests beautifulsoup4 pandas matplotlib
Enter fullscreen mode Exit fullscreen mode

Kickstarter Scraper 

import requests
from bs4 import BeautifulSoup
import json
import time

SCRAPER_API_KEY = "YOUR_KEY"

def scrape_kickstarter_category(category, page=1):
    """Scrape Kickstarter projects from a category page."""
    url = (
        f"https://www.kickstarter.com/discover/advanced"
        f"?category_id={category}&sort=newest&page={page}"
    )
    api_url = (
        f"http://api.scraperapi.com?api_key={SCRAPER_API_KEY}"
        f"&url={url}&render=true"
    )

    response = requests.get(api_url, timeout=60)
    soup = BeautifulSoup(response.text, "html.parser")

    projects = []
    for card in soup.select("[data-project]"):
        data = json.loads(card.get("data-project", "{}"))
        if data:
            projects.append({
                "name": data.get("name", ""),
                "blurb": data.get("blurb", ""),
                "goal": data.get("goal", 0),
                "pledged": data.get("pledged", 0),
                "backers": data.get("backers_count", 0),
                "state": data.get("state", ""),
                "category": data.get("category", {}).get("name", ""),
                "location": data.get("location", {}).get("name", ""),
                "deadline": data.get("deadline", ""),
                "percent_funded": round(
                    data.get("pledged", 0) / max(data.get("goal", 1), 1) * 100, 1
                ),
                "platform": "kickstarter"
            })

    return projects
Enter fullscreen mode Exit fullscreen mode

Indiegogo Scraper 

def scrape_indiegogo_explore(category="tech", page=1):
    """Scrape Indiegogo explore page for projects."""
    url = (
        f"https://www.indiegogo.com/explore/{category}"
        f"?project_type=campaign&sort=trending&page={page}"
    )
    api_url = (
        f"http://api.scraperapi.com?api_key={SCRAPER_API_KEY}"
        f"&url={url}&render=true"
    )

    response = requests.get(api_url, timeout=60)
    soup = BeautifulSoup(response.text, "html.parser")

    projects = []
    for card in soup.select(".discoveryCard"):
        title = card.select_one(".discoveryCard-title")
        raised = card.select_one(".discoveryCard-raised")
        percent = card.select_one(".discoveryCard-percentFunded")

        if title:
            projects.append({
                "name": title.text.strip(),
                "pledged_text": raised.text.strip() if raised else "N/A",
                "percent_text": percent.text.strip() if percent else "N/A",
                "category": category,
                "platform": "indiegogo"
            })

    return projects
Enter fullscreen mode Exit fullscreen mode

Data Collection Pipeline 

KICKSTARTER_CATEGORIES = {
    "tech": 16, "design": 7, "games": 12,
    "film": 11, "music": 14, "fashion": 9,
}

INDIEGOGO_CATEGORIES = ["tech", "design", "film", "music", "health"]

def collect_crowdfunding_data(pages_per_category=3):
    all_projects = []

    for cat_name, cat_id in KICKSTARTER_CATEGORIES.items():
        for page in range(1, pages_per_category + 1):
            print(f"Kickstarter: {cat_name} page {page}")
            projects = scrape_kickstarter_category(cat_id, page)
            all_projects.extend(projects)
            time.sleep(5)

    for category in INDIEGOGO_CATEGORIES:
        for page in range(1, pages_per_category + 1):
            print(f"Indiegogo: {category} page {page}")
            projects = scrape_indiegogo_explore(category, page)
            all_projects.extend(projects)
            time.sleep(5)

    return all_projects
Enter fullscreen mode Exit fullscreen mode

Trend Analysis 

import pandas as pd
import matplotlib.pyplot as plt

def analyze_trends(projects):
    df = pd.DataFrame(projects)
    ks = df[df["platform"] == "kickstarter"]

    print("=== Kickstarter Trends ===\n")
    print(f"Total projects: {len(ks)}")

    print("\nBy Category:")
    cat_stats = ks.groupby("category").agg(
        count=("name", "count"),
        avg_pledged=("pledged", "mean"),
        avg_backers=("backers", "mean"),
        avg_percent=("percent_funded", "mean")
    ).sort_values("avg_pledged", ascending=False)
    print(cat_stats.round(0))

    # Visualization
    fig, axes = plt.subplots(1, 2, figsize=(14, 6))
    cat_stats["avg_pledged"].plot(kind="bar", ax=axes[0], title="Avg Pledged by Category")
    axes[0].set_ylabel("USD")
    cat_stats["count"].plot(kind="bar", ax=axes[1], title="Projects by Category")
    axes[1].set_ylabel("Count")
    plt.tight_layout()
    plt.savefig("crowdfunding_trends.png")
    print("\nChart saved to crowdfunding_trends.png")

projects = collect_crowdfunding_data(pages_per_category=2)
analyze_trends(projects)
Enter fullscreen mode Exit fullscreen mode

Proxy Recommendations

Both Kickstarter and Indiegogo use JavaScript-heavy frontends. ScraperAPI with render=true is essential here. For large-scale collection, ThorData provides reliable residential proxies. Monitor your pipeline health with ScrapeOps.

Conclusion

Crowdfunding data reveals what consumers are willing to pay for before products hit the market. Build this pipeline, run it weekly, and you'll spot trends months ahead of mainstream coverage.

Top comments (0)