[syslog-ng] Problems to receive multiple/single log lines

[Jorge Pereira]

Hi everyone,

   Question: Is it possible to send multiple/single log lines to the
same destination? In this case, exist some way to differentiate the
receiving of multiple/single lines? Below my client and server setup.

# Client Side
<MY CLIENT CONF>
options {
    threaded(yes);
    flush_lines(0);
    use-dns(no);
    normalize-hostnames(yes);
    keep-hostname(yes);
};

destination d_collector_waf {
    tcp("syslog-server.internal.net"
        port(514)
        keep-alive(on)
        flags(no-parse)
    );
};

source s_modsec_log {
# This file is multiple-line, below the content.
#
# <SAMPLE CONTENT>
#--bfd16c01-H--
#Message: Access denied with code 401 (phase 2). Pattern match
"(?i:(?:[\\;\\|\\`]\\W*?\\bcc|\\b(wget|curl))\\b|\\/cc(?:[\\'\"\\|\\;\\`\\-\\s]|$))"
at ARGS:a. [file
"/usr/local/openresty/nginx/conf/waf/www.happystage.tk.conf"] [line
"177"] [id "950907"] [rev "2"] [msg "System Command Injection"] [data
"Matched Data: wget found within ARGS:a: wget"] [severity "CRITICAL"]
[ver "OWASP_CRS/2.2.9"] [maturity "9"] [accuracy "8"] [tag
"OWASP_CRS/WEB_ATTACK/COMMAND_INJECTION"] [tag "WASCTC/WASC-31"] [tag
"OWASP_TOP_10/A1"] [tag "PCI/6.5.2"]
#Action: Intercepted (phase 2)
#Apache-Handler: IIS
#Stopwatch: 1465000225000663 664069 (- - -)
#Stopwatch2: 1465000225000663 664069; combined=239, p1=138, p2=80,
p3=0, p4=0, p5=20, sr=10, sw=1, l=0, gc=0
#Producer: ModSecurity for nginx (STABLE)/2.8.0
(http://www.modsecurity.org/); 200911012341.
#Server: ModSecurity Standalone
#Engine-Mode: "ENABLED"
#
# </SAMPLE CONTENT>

# The below regex in "multi-line-prefix()" works fine, transform the
inputstream in a single line.
    # below debug output:
    # Incoming log entry;
line='--3215e80c-Z--\x0a\x0a--4b80ac01-A--\x0a[04/Jun/2016:00:31:57
+0000] AiA7Ac8cA0AWAcAcAcA1 at cYc 206.128.156.45 0 127.0.0.1
80\x0a--4b80ac01-B--\x0aGET /WAF-Testing/?a=wget HTTP/1.1\x0ahost:
www.happystage.tk\x0arequest-id:
2016-06-04T00:31:57Z|51d35e61f4|206.128.156.45|EcWqiPItjA\x0aaccept:
*/*\x0auser-agent: curl/7.47.0\x0a\x0a--4b80ac01-H--\x0aMessage:
Access denied with code 401 (phase 2). Pattern match
"(?i:(?:[\\;\\|\\`]\\W*?\\bcc|\\b(wget|curl))\\b|\\/cc(?:[\\\'\"\\|\\;\\`\\-\\s]|$))"
at ARGS:a. [file
"/usr/local/openresty/nginx/conf/waf/www.happystage.tk.conf"] [line
"177"] [id "950907"] [rev "2"] [msg "System Command Injection"] [data
"Matched Data: wget found within ARGS:a: wget"] [severity "CRITICAL"]
[ver "OWASP_CRS/2.2.9"] [maturity "9"] [accuracy "8"] [tag
"OWASP_CRS/WEB_ATTACK/COMMAND_INJECTION"] [tag "WASCTC/WASC-31"] [tag
"OWASP_TOP_10/A1"] [tag "PCI/6.5.2"]\x0aAction: Intercepted (phase
2)\x0aApache-Handler: IIS\x0aStopwatch: 1465000317000372 373530 (- -
-)\x0aStopwatch2: 1465000317000372 373530; combined=221, p1=137,
p2=63, p3=0, p4=0, p5=2'

    file("/usr/local/openresty/nginx/logs/waf/www.mydomain.com"
        follow_freq(1)
        flags(no-parse)
        multi-line-mode(regexp)
        override_program("ng_waf:www.mydomain.com")
        multi-line-prefix("--[a-fA-F0-9]{8}-Z--")
    );
};

source s_access_log {
# The content of this file is a single-line delimited by a "\n"
    file("/usr/local/openresty/nginx/logs/access.log"
         program_override("ng_access")
         follow_freq(1)
         flags(no-parse)
    );
};

log {
    source(s_modsec_log);
    source(s_access_log);
    destination(d_collector);
};
</MY CLIENT CONF>

My setver setup:

<SERVER CONF>
options {
    flush-lines(100);
    log-fifo-size(1000);
    threaded(yes);
    use-dns(no);
    normalize-hostnames(yes);
    keep-hostname(yes);
};

source s_collector {
    tcp(ip(0.0.0.0)
        port(514)
        keep-alive(on)
        flags(no-parse)
    );
};

filter f_nginx_waf {
    match("ng_modsec" value("PROGRAM"));
};

rewrite r_nginx_waf {
subst("ng_modsec:", "", value("PROGRAM"));
};

destination d_nginx_waf {
    file("/var/log/syslog-ng/nginx/waf/${PROGRAM}_log"
         create_dirs(yes) owner("root") group("root") perm(0640)
         dir_perm(0750) flags(no-parse) template("${MSG}")
    );
};

log {
    source(s_collector);
    filter(f_nginx_waf);
    rewrite(r_nginx_waf);
    destination(d_nginx_waf);
    flags(flow-control);
};

filter f_nginx_access {
    match("ng_access" value("PROGRAM"));
};

destination d_nginx_access {
    file("/var/log/syslog-ng/nginx/${PROGRAM}_log"
         create_dirs(yes) owner("root") group("root") perm(0640)
         dir_perm(0750) flags(no-parse) template("${MSG}")
    );
};

log {
    source(s_collector);
    filter(f_nginx_access);
    destination(d_nginx_access);
    flags(flow-control);
};
</SERVER CONF>