Python Web Scraping for Business Intelligence: Monitor Competitors 24/7 for Free

Your competitors update their pricing and features without telling you. This Python script checks them automatically.

Production-Grade Scraper

import requests
from bs4 import BeautifulSoup
import time, random, csv
from datetime import datetime

class BusinessScraper:
    def __init__(self):
        self.session = requests.Session()
        self.session.headers['User-Agent'] = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64)'

    def get(self, url):
        time.sleep(random.uniform(1, 3))
        try:
            r = self.session.get(url, timeout=10)
            r.raise_for_status()
            return BeautifulSoup(r.content, 'html.parser')
        except Exception as e:
            print(f"Failed {url}: {e}")
            return None

    def extract_prices(self, url):
        soup = self.get(url)
        if not soup:
            return []
        results = []
        for el in soup.find_all(class_=lambda x: x and 'price' in x.lower()):
            text = el.get_text(strip=True)
            if '$' in text or '/mo' in text.lower():
                results.append(text)
        return results

Competitor Monitoring

competitors = [
    {'name': 'Competitor A', 'url': 'https://competitor-a.com/pricing'},
    {'name': 'Competitor B', 'url': 'https://competitor-b.com/pricing'},
]

def daily_monitor():
    scraper = BusinessScraper()
    for comp in competitors:
        prices = scraper.extract_prices(comp['url'])
        timestamp = datetime.now().strftime('%Y-%m-%d %H:%M')
        print(f"{timestamp} | {comp['name']}: {prices}")
        # Save to database for trend analysis

Lead Generation Scraper

def scrape_leads(category, city):
    scraper = BusinessScraper()
    leads = []
    for page in range(1, 6):
        url = f"https://yelp.com/search?find_desc={category}&find_loc={city}&start={page*10}"
        soup = scraper.get(url)
        if not soup:
            break
        for biz in soup.find_all('div', attrs={'data-testid': 'businessName'}):
            name = biz.get_text(strip=True)
            if name:
                leads.append({'name': name, 'category': category, 'city': city})
    return leads

Alert on Price Changes

import sqlite3, smtplib

def check_price_changes(db_path):
    conn = sqlite3.connect(db_path)
    c = conn.cursor()
    c.execute('''SELECT a.company, a.price, b.price as prev_price
                 FROM prices a JOIN prices b ON a.company = b.company
                 WHERE a.date = date("now") AND b.date = date("now", "-1 day")
                 AND a.price != b.price''')
    for row in c.fetchall():
        company, new_price, old_price = row
        print(f"PRICE CHANGE: {company} changed from {old_price} to {new_price}")
        send_alert_email(company, old_price, new_price)

Schedule It

# Run every morning at 7am
0 7 * * * /usr/bin/python3 monitor_competitors.py

Replaces: ScrapeHero ($99/mo), ParseHub ($149/mo), Import.io ($199/mo)

The complete toolkit with 8 scraper templates, proxy rotation, and Excel/Google Sheets export is below.

---

Get 50+ Python automation scripts for $9: Python Business Automation Toolkit