Python Log Analyzer: Parse 50,000 Lines and Find Errors in Seconds

#python #devops #automation #programming

Python Log Analyzer: Parse 50,000 Lines and Find Errors in Seconds

Your app generates thousands of log lines per day. Finding the 3 critical errors manually takes an hour. This script finds them in seconds.

The Script

import re
from collections import Counter
from pathlib import Path

class LogAnalyzer:
    LOG_PATTERN = re.compile(
        r'(?P<time>\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}) '
        r'(?P<level>\w+) (?P<message>.+)'
    )

    def __init__(self, log_file):
        self.entries = []
        self.parse(log_file)

    def parse(self, log_file):
        with open(log_file) as f:
            for line in f:
                m = self.LOG_PATTERN.match(line.strip())
                if m:
                    self.entries.append(m.groupdict())
        return len(self.entries)

    def errors(self):
        return [e for e in self.entries if e['level'] in ('ERROR', 'CRITICAL')]

    def top_errors(self, n=10):
        msgs = [re.sub(r'\b\d+\b', 'N', e['message'])[:80] for e in self.errors()]
        return Counter(msgs).most_common(n)

    def summary(self):
        levels = Counter(e['level'] for e in self.entries)
        print(f'Total: {len(self.entries)} entries')
        for level, count in levels.most_common():
            print(f'  {level}: {count}')
        print('\nTop errors:')
        for msg, count in self.top_errors():
            print(f'  [{count}x] {msg}')

# Usage
analyzer = LogAnalyzer('/var/log/app.log')
analyzer.summary()

Parse Multiple Files

from pathlib import Path

all_errors = []
for log_file in Path('/var/log/app').glob('*.log'):
    a = LogAnalyzer(str(log_file))
    all_errors.extend(a.errors())
print(f'Total errors: {len(all_errors)}')