How to Build a Competitor Price Intelligence Tool

#python #tutorial #webdev #programming

Price intelligence is one of the most valuable applications of web scraping. E-commerce companies use automated price tracking to stay competitive.

Architecture Overview

Our tool will:

Scrape competitor product pages on a schedule
Store price history in SQLite
Detect price changes and generate alerts

The Price Scraper Class

import requests
from bs4 import BeautifulSoup
import sqlite3

class PriceScraper:
    def __init__(self, db_path="prices.db"):
        self.db_path = db_path
        self.session = requests.Session()
        self.session.headers.update({
            "User-Agent": "Mozilla/5.0"
        })
        self._init_db()

    def _init_db(self):
        conn = sqlite3.connect(self.db_path)
        conn.execute("""CREATE TABLE IF NOT EXISTS prices (
            id INTEGER PRIMARY KEY AUTOINCREMENT,
            product_id TEXT, competitor TEXT,
            product_name TEXT, price REAL,
            url TEXT, scraped_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
        )""")
        conn.commit()
        conn.close()

    def scrape_price(self, url, selectors):
        try:
            response = self.session.get(url, timeout=15)
            soup = BeautifulSoup(response.text, "html.parser")
            name_el = soup.select_one(selectors["name"])
            price_el = soup.select_one(selectors["price"])
            if not price_el:
                return None
            price_text = price_el.get_text(strip=True)
            price = float(price_text.replace("$", "").replace(",", ""))
            name = name_el.get_text(strip=True) if name_el else "Unknown"
            return {"name": name, "price": price}
        except Exception as e:
            print(f"Error: {e}")
            return None

Scaling Tips

For production price intelligence, consider using ScraperAPI to handle anti-bot measures. ThorData provides residential proxies for geo-restricted pricing, and ScrapeOps helps monitor your scraping jobs.