The Small Business Guide to Automated Backup Testing (Before You Need It)

Everyone has backups. Almost nobody tests them. When you need a backup is not the time to discover it doesn't work.

I've seen companies lose years of data because their "backups" were corrupted, incomplete, or couldn't be restored. The backup was running. The files were being created. But nobody ever tried to restore them.

Here's how to set up automated backup testing so you know — not hope — that your backups work.

Why Backups Fail (And You Don't Know It)

Failure 1: The Backup Runs But Saves Nothing

# This cron job runs every night and creates a backup file
0 2 * * * root tar czf /backups/daily.tar.gz /data/

# But if /data/ is empty (mount failed), the backup is 4KB of nothing
# And nobody knows until they try to restore

Failure 2: The Backup Is Incomplete

# This backs up the database while it's running
0 2 * * * root cp /var/lib/mysql/* /backups/mysql/

# But the database is actively writing, so some tables are inconsistent
# The backup file exists, but restoring it gives you corrupted data

Failure 3: The Backup Can't Be Restored

# The backup file is created on the same server as the data
0 2 * * * root tar czf /backups/daily.tar.gz /data/

# When the server dies, the backup dies with it

The Solution: Automated Backup Testing

Step 1: Create Proper Backups

#!/usr/bin/env python3
"""backup.py - Proper backup with verification"""
import subprocess
import os
import hashlib
import json
from datetime import datetime

class BackupSystem:
    def __init__(self, config):
        self.source = config['source']
        self.dest = config['destination']
        self.retention_days = config.get('retention_days', 30)

    def database_backup(self, db_name, db_user, db_password):
        """Proper database backup using pg_dump"""
        timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
        backup_file = f'{self.dest}/{db_name}_{timestamp}.sql.gz'

        # Use pg_dump for consistent backup
        cmd = f'pg_dump -U {db_user} -d {db_name} | gzip > {backup_file}'
        result = subprocess.run(cmd, shell=True, capture_output=True)

        if result.returncode != 0:
            self.alert(f'Database backup FAILED: {result.stderr.decode()}')
            return False

        # Verify the backup is not empty
        size = os.path.getsize(backup_file)
        if size < 100:  # Less than 100 bytes = probably empty
            self.alert(f'Database backup SUSPICIOUS: only {size} bytes')
            return False

        # Calculate checksum for integrity verification
        checksum = self._checksum(backup_file)

        self.log_backup({
            'file': backup_file,
            'size': size,
            'checksum': checksum,
            'timestamp': timestamp,
            'type': 'database'
        })

        return True

    def file_backup(self):
        """File backup with size verification"""
        timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
        backup_file = f'{self.dest}/files_{timestamp}.tar.gz'

        # Create backup
        result = subprocess.run(
            ['tar', 'czf', backup_file, '-C', self.source, '.'],
            capture_output=True
        )

        if result.returncode != 0:
            self.alert(f'File backup FAILED: {result.stderr.decode()}')
            return False

        # Verify backup size is reasonable
        size = os.path.getsize(backup_file)
        source_size = self._dir_size(self.source)

        # Backup should be at least 10% of source (compression)
        if size < source_size * 0.1:
            self.alert(f'Backup SUSPICIOUS: {size} bytes vs source {source_size} bytes')
            return False

        return True

    def _checksum(self, filepath):
        """Calculate SHA256 checksum"""
        with open(filepath, 'rb') as f:
            return hashlib.sha256(f.read()).hexdigest()

    def _dir_size(self, path):
        """Get directory size in bytes"""
        total = 0
        for dirpath, dirnames, filenames in os.walk(path):
            for f in filenames:
                fp = os.path.join(dirpath, f)
                total += os.path.getsize(fp)
        return total

Step 2: Test the Restore (The Important Part)

#!/usr/bin/env python3
"""backup_test.py - Automatically test backup restoration"""
import subprocess
import os
import tempfile
from datetime import datetime

class BackupTester:
    def __init__(self, config):
        self.config = config

    def test_database_restore(self, backup_file):
        """Test restoring a database backup"""
        test_db = f'test_restore_{datetime.now().strftime("%Y%m%d")}'

        try:
            # Create a test database
            subprocess.run(
                ['psql', '-U', self.config['db_user'], '-c', f'CREATE DATABASE {test_db}'],
                check=True, capture_output=True
            )

            # Restore the backup into the test database
            with tempfile.NamedTemporaryFile(suffix='.sql') as tmp:
                # Decompress
                subprocess.run(['gunzip', '-c', backup_file], stdout=tmp, check=True)
                tmp.flush()

                # Restore
                subprocess.run(
                    ['psql', '-U', self.config['db_user'], '-d', test_db, '-f', tmp.name],
                    check=True, capture_output=True
                )

            # Verify: count tables and rows
            result = subprocess.run(
                ['psql', '-U', self.config['db_user'], '-d', test_db, '-c',
                 'SELECT count(*) FROM information_schema.tables WHERE table_schema = \'public\''],
                capture_output=True, text=True
            )

            table_count = int(result.stdout.strip().split('\n')[-2].strip())

            if table_count == 0:
                return {'success': False, 'error': 'No tables in restored backup'}

            return {
                'success': True,
                'tables_restored': table_count,
                'test_db': test_db
            }

        except subprocess.CalledProcessError as e:
            return {'success': False, 'error': str(e)}
        finally:
            # Clean up test database
            subprocess.run(
                ['psql', '-U', self.config['db_user'], '-c', f'DROP DATABASE IF EXISTS {test_db}'],
                capture_output=True
            )

    def test_file_restore(self, backup_file):
        """Test restoring a file backup"""
        with tempfile.TemporaryDirectory() as tmpdir:
            # Extract backup
            result = subprocess.run(
                ['tar', 'xzf', backup_file, '-C', tmpdir],
                capture_output=True
            )

            if result.returncode != 0:
                return {'success': False, 'error': result.stderr.decode()}

            # Count restored files
            file_count = sum(len(files) for _, _, files in os.walk(tmpdir))

            if file_count == 0:
                return {'success': False, 'error': 'No files restored'}

            # Verify a known file exists
            test_file = self.config.get('test_file')
            if test_file:
                if not os.path.exists(os.path.join(tmpdir, test_file)):
                    return {'success': False, 'error': f'Test file {test_file} not found in backup'}

            return {
                'success': True,
                'files_restored': file_count
            }

Step 3: Schedule Everything

#!/bin/bash
# /etc/crontab

# 2 AM: Create backups
0 2 * * * root /opt/backup/backup.py

# 3 AM: Test the most recent backup
0 3 * * * root /opt/backup/backup_test.py

# 4 AM: Copy backups to offsite (S3, another server)
0 4 * * * root /opt/backup/offsite_copy.py

# Weekly: Full restore test (Sunday 5 AM)
0 5 * * 0 root /opt/backup/full_restore_test.py

Step 4: Alert on Failure

def alert(self, message):
    """Send alert if backup or restore test fails"""
    # Slack
    requests.post(SLACK_WEBHOOK, json={
        'text': f'🚨 BACKUP ISSUE: {message}'
    })

    # Email
    self.send_email(
        subject='BACKUP ALERT',
        body=message,
        to=self.config['alert_email']
    )

    # Log
    with open('/var/log/backup_alerts.log', 'a') as f:
        f.write(f'{datetime.now()}: {message}\n')

The Backup Testing Schedule

Frequency	What to Test	How
Daily	Backup created successfully	Size check + checksum
Daily	Database restore works	Restore to test DB, count tables
Daily	File restore works	Extract to temp dir, count files
Weekly	Full restore test	Complete restore to test server
Monthly	Offsite backup works	Download from S3, verify integrity
Quarterly	Disaster recovery drill	Full restore + application test

The 3-2-1 Rule

• 3 copies of your data
• 2 different media types (local disk + cloud)
• 1 copy offsite (different geographic location)

# offsite_copy.py
import boto3

def copy_to_s3(backup_file, bucket):
    s3 = boto3.client('s3')
    s3.upload_file(backup_file, bucket, os.path.basename(backup_file))

    # Verify upload
    response = s3.head_object(Bucket=bucket, Key=os.path.basename(backup_file))
    if response['ContentLength'] != os.path.getsize(backup_file):
        alert('Offsite copy size mismatch!')

The Cost of Not Testing

Scenario	Cost Without Testing	Cost With Testing
Ransomware	$50,000+ recovery	$0 (restore from backup)
Hardware failure	2-7 days downtime	1 hour downtime
Accidental deletion	Permanent data loss	5 minute recovery
Audit requirement	Fail compliance	Pass with documentation

---

Want the complete backup automation toolkit? The Ops Starter Kit includes backup scripts, restore testing templates, offsite copy automation, and alert configurations — everything you need to know your backups actually work.

When was the last time you tested a backup restore?