Hourly monitor app'errors from Graylog to Slack

Hi,

I'm using Graylog to store applicative logs.

On error (level:3), app will send an entry to graylog too.

const graylog2 = require('graylog2');

const logger = new graylog2.graylog({
  servers: [{ 'host': '127.0.0.1', port: 12201 }]
});

// (...)

logger.error(msg)

By adding a little cron job shell script, I'm able to collect every hour the last error logs from graylog, then send them to Slack.

This way I'm able to react to applicative issue.

Here is graylogMonitoring.sh:

#!/bin/bash
# tools: apt-get install jq curl
# external secret requirements:
## export GRAY_AUTH=admin:mypassword
## export SLACK_WEBHOOK_ENDPOINT=https://hooks.slack.com/services/JUST/UPDATE/THIS

# to handle exclamation in password 
set +H

# send a slack message if webhook is set
  function slackNotification() {
    # markdown formatting : https://api.slack.com/reference/surfaces/formatting
    SLACK_TEXT_MSG="${1:-Something from logs is important}"
    if [ -z ${SLACK_WEBHOOK_ENDPOINT+x} ]; then
      echo "${SLACK_TEXT_MSG}"
    else
      curl -X POST --data "{\"type\": \"mrkdwn\",\"text\": \"${SLACK_TEXT_MSG}\"}" ${SLACK_WEBHOOK_ENDPOINT} || echo "unable to slack notify"
    fi
  }

# exit if there is no Graylog auth
if [ -z ${GRAY_AUTH+x} ]; then
  exit 0;
fi

# QUERY="*"
# url encoded query "level:3"
QUERY="level%3A3"
# level:3
# Full Graylog API call
GRAY_QUERY="http://localhost:9000/api/search/universal/relative?query=$QUERY&range=3600&fields=message&limit=100&sort=timestamp:desc"

curl --silent -u $GRAY_AUTH \
  -H "X-Requested-By: cli" \
  -H "Accept: application/json" \
  -H "Content-Type: application/json" \
  ${GRAY_QUERY} -o LAST_WARN.json

# use jq to extract message from Graylog json response
cat LAST_WARN.json  |jq --raw-output '.messages[].message | (.timestamp + " | " +  .gl2_remote_ip + " | " + .message) ' > LAST_WARN.log

# send them to slack if result file is not empty
if [ -s "LAST_WARN.log" ]
then
  SLACK_MSG="Log warn sur la période : \n\`\`\`$(cat LAST_WARN.log | tr -d '"')\`\`\`"
  slackNotification "$SLACK_MSG"
fi

---

About script trigger: if you want to be more flexible than linux host cron job which require you to ssh login, edit crontab, ..

then you could map this script to a dedicated webhook via node-hook-action.

Having nodejs on you server,

• install node-hook-action

npm install node-hook-action

• create a webhookServer.js:

const server = require('node-hook-action');
server();

• create a webhook server config.json

{
  "server_config": {
    "host": "0.0.0.0",
    "port": 1502,
    "path": "/webhook",
    "secret": {
      "custom": thisIsASecret
    },
    "directories": {
      "logs": "logs"
    }
  },
  "actions": [
    {
      "headers": {
        "x-action":"logs"
      },
      "events": [
        {
          "event": "custom",
          "action": "bash /var/www/webhooks/graylogMonitoring.sh"
        }
      ]
    }
  ]
}

• start your webhook server using pm2

pm2 start node --name "webhook" -o /tmp/webhook.log -e /tmp/webhook-errors.log --time -- webhookServer.js

Update you reverse proxy (ex. nginx) to make an available webhook endpoint.

Here is an extract of nginx.conf:

location /webhook {
    proxy_pass http://0.0.0.0:1502;
    proxy_http_version 1.1;
    proxy_set_header Upgrade $http_upgrade;
    proxy_set_header Connection 'upgrade';
    proxy_cache_bypass $http_upgrade;

    proxy_set_header Host $host;
    proxy_set_header   X-Real-IP          $remote_addr;
    proxy_set_header   X-Forwarded-Proto  $scheme;
    proxy_set_header   X-Forwarded-For    $proxy_add_x_forwarded_for;
}

Go to the free cronjob service to plan your webhook at regular interval:

And you're done !

Keep an eye on your Slack channel