[syslog-ng] New user trying to filter / rewrite apache logs

11 Jul 2017

      Hi all,

reading the docs I got into this config:

source s_apache_access_log
{

file(

"/var/logs/apache2/access_log"

follow-freq(1)

flags(no-parse)

);

};

filter f_apache_access_log
{

match(

        '(.*) (.*) - -
\[[0-9]{2}\/[A-Z][a-z]{2}\/[0-9]{4}:[0-9]{2}:[0-9]{2}:[0-9]{2} -0300\]
\"(.*) (.*) (.*)\" (.*) (.*) \"-\" (.*)'

type("pcre")

flags("store-matches")

);

};

rewrite r_apache_access_log
{

    set("$1", value("DOMAIN")
condition(filter(f_apache_access_log)));

    set("$2", value("IP")
condition(filter(f_apache_access_log)));

    set("$3", value("HTTP_METHOD")
condition(filter(f_apache_access_log)));

    set("$4", value("URI")
condition(filter(f_apache_access_log)));

    set("$6", value("HTTP_STATUS")
condition(filter(f_apache_access_log)));

    set("$7", value("SIZE")
condition(filter(f_apache_access_log)));

    set("$8", value("USER_AGENT")
condition(filter(f_apache_access_log)));

};

destination d_apache_access_log
{

mongodb(

        # https://docs.mongodb.com/manual/reference/connection-string/

persist-name("apache-access-logs")

uri("mongodb://$server_and_port/syslog?wtimeoutMS=60000&socketTimeoutMS=60000&connectTimeoutMS=60000")

collection("logs")

retries(3600)

value-pairs(

            pair("HOST",
"${HOST}")

            pair("SERVICE",
"APACHE")

            pair("DATE",
"${DAY}/${MONTH}/${YEAR}")

            pair("TIME",
"${HOUR}:${MIN}")

            pair("MESSAGE",
"${MESSAGE}")

            pair("DOMAIN",
"${DOMAIN}")

            pair("HTTP_STATUS",
"${HTTP_STATUS}")

            pair("HTTP_METHOD",
"${HTTP_METHOD}")

            pair("USER_AGENT",
"${USER_AGENT}")

            pair("SIZE",
"${SIZE}")

            pair("URI",
"${URI}")

            pair("IP",
"${IP}")

)

);

};

log
{

source(s_apache_access_log);

filter(f_apache_access_log);

rewrite(r_apache_access_log);

destination(d_apache_access_log);

};

but I think something is not ok, I'm not sure this is the right way to do
it.

This log produces an strange behavior:

www.cifa.li 127.0.0.1 - - [11/Jul/2017:09:18:56 -0300] "GET / HTTP/1.1" 200
18652 "http://cifa.li/" "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:54.0)
Gecko/20100101 Firefox/54.0"

but this one doesn't

cifa.li 127.0.0.1 - - [11/Jul/2017:09:18:56 -0300] "GET / HTTP/1.1" 200
18652 "http://cifa.li/" "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:54.0)
Gecko/20100101 Firefox/54.0"

The behavior is (only for subdomains):

DOMAIN: ': www.cifa.li'

corret one

DOMAIN: 'www.cifa.li'

The subdomain seems like it's adding stuff that I didn't (or want) to add.

Am I missing something?

Thanks in advance.

-- 
[ ]'s

Filipe Cifali Stangler