rsyslog filtering (with loggly)

If you’re a bit slow on the uptake, like me … this might help.

Basic logging to Loggly is simple enough –

References : https://www.loggly.com/docs/rsyslog-tls-configuration/ gets you to add in an omfwd action and a template with auth details in …

However, when you also want to mix in sending Apache logs to loggly, and at the same time want to suppress sending some lines ….. life becomes a bit harder.

Here’s what worked for me anyway… replace MAGIC_AUTH_TOKEN_HERE with your loggly auth details.

Place this in /etc/rsyslog.d/loggly.conf.

# Setup disk assisted queues
$WorkDirectory /var/spool/rsyslog # where to place spool files
$ActionQueueFileName fwdRule1     # unique name prefix for spool files
$ActionQueueMaxDiskSpace 1g       # 1gb space limit (use as much as possible)
$ActionQueueSaveOnShutdown on     # save messages to disk on shutdown
$ActionQueueType LinkedList       # run asynchronously
$ActionResumeRetryCount -1        # infinite retries if host is down

#RsyslogGnuTLS
$DefaultNetstreamDriverCAFile /etc/rsyslog.d/keys/ca.d/logs-01.loggly.com_sha12.crt


$ActionSendStreamDriver gtls # use gtls netstream driver
$ActionSendStreamDriverMode 1 # require TLS
$ActionSendStreamDriverAuthMode x509/name # authenticate by hostname
$ActionSendStreamDriverPermittedPeer *.loggly.com

template(name="LogglyFormat" type="string"
string="< %pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [MAGIC_AUTH_TOKEN_HERE tag=\"Syslog\"] %msg%\n"
)


module(load="imfile") 

# Apache file inputs :

input(type="imfile"
    File="/var/log/apache2/access.log"
    Tag="apache-access"
    Severity="info"
    Facility="local7")

input(type="imfile"
    File="/var/log/apache2/error.log"
    Tag="apache-error"
    Severity="error"
    Facility="local7")


# Format for Apache things.
$template LogglyFormatApache,"< %pri%>%protocol-version% %timestamp:::date-rfc3339% %HOSTNAME% %app-name% %procid% %msgid% [MAGIC_AUTH_TOKEN_HERE  tag=\"apache\" ] %msg%\n"

if ( $programname == 'apache-access' ) and not ( $msg contains "/something-to-skip/" ) then {
     action(
        type="omfwd" 
        protocol="tcp" 
        target="logs-01.loggly.com" 
        port="6514" template="LogglyFormatApache" 
        StreamDriver="gtls" 
        StreamDriverMode="1" 
        StreamDriverAuthMode="x509/name" 
        StreamDriverPermittedPeers="*.loggly.com"
    )
    stop
} 

# no further processing of apache-access things 
if ( $programname == 'apache-access') then stop

if ( $programname == 'apache-error' ) then {
         action(
                type="omfwd" 
                protocol="tcp" 
                target="logs-01.loggly.com" 
                port="6514" template="LogglyFormatApache" 
                StreamDriver="gtls" 
                StreamDriverMode="1" 
                StreamDriverAuthMode="x509/name" 
                StreamDriverPermittedPeers="*.loggly.com"
        )
    stop
} 

if ( $programname == 'apache-error') then stop

# Anything else ... sent to loggly.
action(
    type="omfwd" 
    protocol="tcp" 
    target="logs-01.loggly.com" 
    port="6514" template="LogglyFormatApache" 
    StreamDriver="gtls" 
    StreamDriverMode="1" 
    StreamDriverAuthMode="x509/name" 
    StreamDriverPermittedPeers="*.loggly.com"
)