How to analyze OpenBSD's httpd access.log with a shell script

#openbsd #httpd

I wanted a shell script to analyze my access logs from httpd. I found this from @ruanbekker, adopted it and modified it for httpd.

You can see

Top requests by IP
Request methods like GET or HEAD
Response codes like 200, 301, 404
Top requested pages like /atom.xml
Top requested pages with a 404 response

You can download the script here

#!/bin/sh

LOGFILE="/var/www/logs/access.log"
RESPONSE_CODE="200"

filters() {
grep $RESPONSE_CODE \
| grep -v "<UNKNOWN>" \
| grep -v "favicon.ico" \
| grep -v "logfile turned over"
}

filter_response_codes()
{
grep -v "<UNKNOWN>" \
| grep -v "logfile turned over" \
| awk '{print $10}'
}

filter_404_response() {
grep "404"
}

ips() {
awk '{print $2}'
}

pages() {
awk '{print $8}'
}

domain() {
awk '{print $1}'
}

methods() {
awk '{print $7}' | cut -d'"' -f2
}

sort_count() {
sort | uniq -c
}

sort_desc() {
sort -rn
}

top_ten() {
head -10
}

sep() {
echo "=================================================="
}

##
# Actions
##
action_request_ips() {
    echo ""
    echo "Top requests from IPs"
    sep
    cat $LOGFILE \
        | filters \
        | ips \
        | sort_count \
        | sort_desc \
        | top_ten
    echo ""
}

action_request_methods() {
    echo ""
    echo "Count requests methods"
    sep
    cat $LOGFILE \
        | filters \
        | methods \
        | sort_count
    echo ""
}

action_pages() {
    echo ""
    echo "Top requested pages"
    sep
    cat $LOGFILE \
        | filters \
        | pages \
        | sort_count \
        | sort_desc \
        | top_ten
    echo ""
}

action_404() {
    echo ""
    echo "Top requests 404"
    sep
    cat $LOGFILE \
        | filter_404_response \
        | pages \
        | sort_count \
        | sort_desc \
        | top_ten
    echo ""
}

action_response_codes() {
    echo ""
    echo "Response code"
    sep
    cat $LOGFILE \
        | filter_response_codes \
        | sort_count \
        | sort_desc
    echo ""
}

action_request_ips
action_request_methods
action_response_codes
action_pages
action_404

Output

$ analyze_access_log

Top requests from IPs
==================================================
13 1.2.3.4
 8 1.2.3.5
 8 1.2.3.6
 4 1.2.3.7
 4 1.2.3.8
 2 1.2.3.9
 2 1.2.3.10
 2 1.2.3.11
 2 1.2.3.12
 2 1.2.3.13


Count requests methods
==================================================
1146 GET
  10 HEAD


Response code
==================================================
1190 200
 792 304
 615 301
  80 404
   8 400
   5 403
   1 405
   1 206
   1 0


Top requested pages
==================================================
 694 /atom.xml
 136 /
  92 /xxx
  46 /yyy
  23 /zzz
  20 /aaa
  17 /eee/fff
  16 /humans.txt
  14 /foo/bar
  12 /something/else


Top requests 404
==================================================
  61 /robots.txt
   5 /css_.php
   2 /admin
   2 /.git/config
   1 /wp-login.php?action=register
   1 /login
   1 /user/register
   1 /index.php?option=com_user%2526task=register
   1 /etc/passwd
   1 /console

You can download the script here

Speedy emails, satisfied customers

Are delayed transactional emails costing you user satisfaction? Postmark delivers your emails almost instantly, keeping your customers happy and connected.