Get rid of syslog (or a journald log filter in ~100 lines of Python)

We’ve recently switched the host behind the siosm.fr domain and thus decided it was time we dropped syslog logging entirely and use journald only. We used to get weekly log reports sent by mail by logrotate, but we never read them as they were way too big, thus this was useless.

Note: The ‘we’ here refers to PO and I.

Instead, we’ve written a replacement using the Python 3 module that comes with journald.

This simple script is based on the sound principles described by Marcus J. Ranum in: artificial ignorance: how-to guide and The Six Dumbest Ideas in Computer Security (especially #2: Enumerating Badness).

The output is so small, we’ve made it a daily run script, allowing us to catch real issues as soon as they appear.

Here is the full script, with included comments as there is nothing complex here:

#!/usr/bin/env python3

from systemd import journal
from datetime import datetime,timedelta
import re
import smtplib
from email.mime.text import MIMEText

# First, let's define patterns to ignore.

# Those are for matching dynamically named units:
session = re.compile("session-[a-z]?\d+.scope")
sshUnit = re.compile("sshd@[0-9a-f.:]*")

# Those will match the logged message itself:
sshSocketStart = re.compile("(Starting|Stopping) OpenSSH Per-Connection Daemon.*")
sshAcceptPublicKey = re.compile("Accepted publickey for (bob|alice).*")
sshReceivedDisconnect = re.compile("Received disconnect from.*")
logindNewSession = re.compile("New session [a-z]?\d+ of user (bob|alice).*")
sshdSessionClosed = re.compile(".*session closed for user (bob|alice).*")
sessionOpenedRoot = re.compile(".*session opened for user root.*")
suSessionClosedGit = re.compile(".*session opened for user git.*")
anacronNormalExit = re.compile("Normal exit (\d+ jobs run).*")
postfixStatistics = re.compile("statistics:.*")
postfixHostnameDoesNotResolve = re.compile("warning: hostname .* does not resolve to address .*: Name or service not known")

# Open the journal for reading, set log level and go back one day and 10 minutes
j = journal.Reader()
j.log_level(journal.LOG_INFO)
yesterday = datetime.now() - timedelta(days=1, minutes=10)
j.seek_realtime(yesterday)

# We'll store messages in this variable
mailContent = []

# Filter and store output
for entry in j:
    # Special cases for logs without a message
    if 'MESSAGE' not in entry:
        mailContent.append( 'U %s %s[%s]: EMPTY!' % (
            datetime.ctime(entry['__REALTIME_TIMESTAMP']),
            entry['PRIORITY'],
            entry['SYSLOG_IDENTIFIER'],
        ))

    # With systemd unit name
    elif '_SYSTEMD_UNIT' in entry:
        if entry['PRIORITY'] > 4:
            if entry['_SYSTEMD_UNIT'] == "wtcomments.service":
                pass
            elif entry['_SYSTEMD_UNIT'] == "ffsync.service":
                pass
        elif session.match(entry['_SYSTEMD_UNIT']):
            pass
        elif sshUnit.match(entry['_SYSTEMD_UNIT']):
            if sshAcceptPublicKey.match(entry['MESSAGE']):
                pass
            elif sshReceivedDisconnect.match(entry['MESSAGE']):
                pass
        elif entry['_SYSTEMD_UNIT'] == "systemd-logind.service":
            if logindNewSession.match(entry['MESSAGE']):
                pass
        elif entry['_SYSTEMD_UNIT'] == "postfix.service":
            if postfixHostnameDoesNotResolve.match(entry['MESSAGE']):
                pass
        else:
            mailContent.append( 'U %s %s %s %s[%s]: %s' % (
                datetime.ctime(entry['__REALTIME_TIMESTAMP']),
                entry['PRIORITY'],
                entry['_SYSTEMD_UNIT'],
                entry['SYSLOG_IDENTIFIER'],
                entry['_PID'],
                entry['MESSAGE']
            ))

    # With syslog identifier only
    elif entry['SYSLOG_IDENTIFIER'] == "systemd":
        if sshSocketStart.match(entry['MESSAGE']):
            pass
        elif firewalldStart.match(entry['MESSAGE']):
            pass
    elif entry['SYSLOG_IDENTIFIER'] == "sshd":
        if sshdSessionClosed.match(entry['MESSAGE']):
            pass
    elif entry['SYSLOG_IDENTIFIER'] == "sudo":
        if sessionOpenedRoot.match(entry['MESSAGE']):
            pass
    elif entry['SYSLOG_IDENTIFIER'] == "CROND":
        if sessionOpenedRoot.match(entry['MESSAGE']):
            pass
    elif entry['SYSLOG_IDENTIFIER'] == "anacron":
        if anacronNormalExit.match(entry['MESSAGE']):
            pass
    elif entry['SYSLOG_IDENTIFIER'] == "postfix/anvil":
        if postfixStatistics.match(entry['MESSAGE']):
            pass
    elif entry['SYSLOG_IDENTIFIER'] == "su":
        if suSessionClosedGit.match(entry['MESSAGE']):
            pass
    else:
        mailContent.append( 'S %s %s %s: %s' % (
            datetime.ctime(entry['__REALTIME_TIMESTAMP']),
            entry['PRIORITY'],
            entry['SYSLOG_IDENTIFIER'],
            entry['MESSAGE']
        ))

# Send the content in a mail to root
mail = MIMEText('\n'.join(mailContent))
mail['Subject'] = '[example.com] Logs from ' + datetime.ctime(yesterday) + ' to ' + datetime.ctime(datetime.now())
mail['From'] = 'journald@example.com'
mail['To'] = 'root@example.com'
server = smtplib.SMTP('localhost')
server.send_message(mail)
server.quit()