Re: [syslog-ng] possible memleak or bad configuration?

15 Mar 2010

      we were running 3.0.4 (self compiled with libdbi for oracle) (same 
problem) and then upgraded to 3.0.5 rhel5 from (directly from the website).

the box itself is a vm on esxi4u1 with centos 5.4 x86_84.

-andy

On 3/13/10 7:03 PM, Martin Holste wrote:
...
The db parser code had a big memory leak in previous 3.1 versions but
was fixed a few months ago; what build are you running?  We process 2
billion logs per day through db parser with no leaks at all using the
build from git commit 9ef6062c1cf72a3f7da880ac245f9ee080bea992.
--Martin
On Sat, Mar 13, 2010 at 2:22 AM, Andreas Sartori
<andreas.sartori@fh-salzburg.ac.at
<mailto:andreas.sartori@fh-salzburg.ac.at>> wrote:
hello,
we have setup a central logging server. currently we are logging
    firewalls
    and  some webserver / mailserver for testing purpose. the memory
    usage on
    the logging server is badly increasing. after 2 days of operation we are
    at 6.8 gb ram usage.
can someone help out, what information do you need to help?
thanks in advance.
-andy
------------
@version:3.0
    #
    # configuration file for syslog-ng, customized for remote logging
    #
options {
            owner("root");
            group("root");
            perm(0600);
            dir_perm(0750);
            create_dirs(yes);
            log_fifo_size(10000);
    };
################################################################################################
    #########################                SOURCES
    ##############################
    ################################################################################################
# Syslog internal logging
    source s_internal { internal(); };
    destination d_syslognglog { file("/var/log/syslog-ng.log"); };
    log { source(s_internal); destination(d_syslognglog); };
# Remote logging
    source s_remote {
            tcp(ip(0.0.0.0) max-connections(20) port(514)
    keep_hostname(yes));
            udp(ip(0.0.0.0) port(514) use_dns(no) log_fetch_limit(500)
    log_iw_size(1000));
    };
################################################################################################
    #########################                FILTER
    ##############################
    ################################################################################################
filter http-official { netmask(xxx.xxx.xxx.47/255.255.255.255
    <http://255.255.255.255>) or
    netmask(xxx.xxx.xxx.48/255.255.255.255 <http://255.255.255.255>) or
    netmask(xxx.xxx.xxx.167/255.255.255.255 <http://255.255.255.255>) or
    netmask(xxx.xxx.xxx.46/255.255.255.255 <http://255.255.255.255>) or
    netmask(xxx.xxx.xxx.52/255.255.255.255 <http://255.255.255.255>) or
    netmask(xxx.xxx.xxx.25/255.255.255.255 <http://255.255.255.255>) or
    netmask(xxx.xxx.xxx.26/255.255.255.255 <http://255.255.255.255>); };
filter mail-proxy-internal { netmask(10.10.9.20/255.255.255.255
    <http://10.10.9.20/255.255.255.255>) and not
    program("perdition"); };
    filter mail-relay-internal { netmask(10.10.9.30/255.255.255.255
    <http://10.10.9.30/255.255.255.255>); };
filter mail-relay-alpha-external-out {
    netmask(xxx.xxx.xxx.59/255.255.255.255 <http://255.255.255.255>) and
    facility(local1); };
    filter mail-relay-beta-external-out {
    netmask(xxx.xxx.xxx.60/255.255.255.255 <http://255.255.255.255>) and
    facility(local1); };
    filter mail-relay-alpha-external-in {
    netmask(xxx.xxx.xxx.59/255.255.255.255 <http://255.255.255.255>) and
    facility(mail); };
    filter mail-relay-beta-external-in {
    netmask(xxx.xxx.xxx.60/255.255.255.255 <http://255.255.255.255>) and
    facility(mail); };
filter mail-proxy-node1-external {
    netmask(xxx.xxx.xxx.18/255.255.255.255 <http://255.255.255.255>)
    and not program("perdition"); };
    filter mail-proxy-node2-external {
    netmask(xxx.xxx.xxx.22/255.255.255.255 <http://255.255.255.255>)
    and not program("perdition"); };
filter vpn { netmask(10.20.40.0/255.255.255.0
    <http://10.20.40.0/255.255.255.0>); };
    filter fw-intern-all { netmask(10.10.20.1/255.255.255.255
    <http://10.10.20.1/255.255.255.255>); };
filter fw-intern-security {
                    netmask(10.10.20.1/255.255.255.255
    <http://10.10.20.1/255.255.255.255>) and
                    match("security" value(".classifier.class")
    type("string"));
    };
filter fw-intern-info {
                    netmask(10.10.20.1/255.255.255.255
    <http://10.10.20.1/255.255.255.255>) and
                    match("informational" value(".classifier.class")
    type("string"));
    };
filter fw-intern-rest {
                    netmask(10.10.20.1/255.255.255.255
    <http://10.10.20.1/255.255.255.255>) and not
                    match("security" value(".classifier.class")
    type("string")) and not
                    match("informational" value(".classifier.class")
    type("string"));
    };
filter fw-extern-all { netmask(10.80.11.20/255.255.255.255
    <http://10.80.11.20/255.255.255.255>); };
filter fw-extern-security {
                    netmask(10.80.11.20/255.255.255.255
    <http://10.80.11.20/255.255.255.255>) and
                    match("security" value(".classifier.class")
    type("string"));
    };
filter fw-extern-info {
                    netmask(10.80.11.20/255.255.255.255
    <http://10.80.11.20/255.255.255.255>) and
                    match("informational" value(".classifier.class")
    type("string"));
    };
filter fw-extern-rest {
                    netmask(10.80.11.20/255.255.255.255
    <http://10.80.11.20/255.255.255.255>) and not
                    match("security" value(".classifier.class")
    type("string")) and not
                    match("informational" value(".classifier.class")
    type("string"));
    };
filter fw-extern-new { netmask(10.80.11.30/255.255.255.255
    <http://10.80.11.30/255.255.255.255>); };
################################################################################################
    #########################                PARSER
    ##############################
    ################################################################################################
parser pattern_db_fwint {
            db_parser(
            file("/etc/syslog-ng/fw-int_patterndb.xml")
            );
    };
parser pattern_db_fwext {
            db_parser(
            file("/etc/syslog-ng/fw-ext_patterndb.xml")
            );
    };
################################################################################################
    #########################             DESTINATIONS
    ##############################
    ################################################################################################
destination http-log { file("/logging/server/web/$HOST"
    template("$MSGONLY\n") template-escape(no) owner("root") group("root")
    perm(0644));  };
destination mail-out {
    file("/logging/server/mail/mail-out_$MONTH.log"); };
    destination mail-in {
    file("/logging/server/mail/mail-in_$MONTH.log"); };
destination vpn {
            file("/logging/network/vpn_$MONTH.log" flush_lines(10));
    };
destination fw-intern-all {
            file("/logging/network/fw-intern_$MONTH.log" flush_lines(10));
    };
destination fw-extern-all {
            file("/logging/network/fw-extern_$MONTH.log" flush_lines(10));
    };
destination fw-extern-new {
            file("/logging/network/fw-new_$MONTH.log" flush_lines(10));
    };
destination dump {
            file("/logging/network/dump.log" template
    ("$R_YEAR-$R_MONTH-$R_DAY
    $R_HOUR:$R_MIN:$R_SEC, $HOST, $FIREWALL_SEQ, $MSGHDR, 0, $FIREWALL_IO,
    $FIREWALL_PROTO, $FIREWALL_SCR_LAN, $FIREWALL_SRC_IP,
    $FIREWALL_SRC_PORT,
    $FIREWALL_DST_LAN, $FIREWALL_DST_IP, $FIREWALL_DST_PORT,
    $FIREWALL_NAT_SRC_IP, $FIREWALL_NAT_DST_IP, $FIREWALL_RULE,
    $FIREWALL_REASON, $FIREWALL_DURATION\n"));
    #       file("/logging/network/dump.log" template ("$MSGHDR\n")
    flush_lines(5));
    };
################################################################################################
    #########################              FINAL-LOGS
    ##############################
    ################################################################################################
##### TO FILE
log { source(s_remote); filter(http-official); destination(http-log); };
    log { source(s_remote); filter(mail-proxy-internal);
    destination(mail-out); };
    log { source(s_remote); filter(mail-relay-internal);
    destination(mail-out); };
    log { source(s_remote); filter(mail-relay-alpha-external-out);
    destination(mail-out); };
    log { source(s_remote); filter(mail-relay-beta-external-out);
    destination(mail-out); };
    log { source(s_remote); filter(mail-proxy-node1-external);
    destination(mail-out); };
    log { source(s_remote); filter(mail-proxy-node2-external);
    destination(mail-out); };
    log { source(s_remote); filter(mail-relay-alpha-external-in);
    destination(mail-in); };
    log { source(s_remote); filter(mail-relay-beta-external-in);
    destination(mail-in); };
    log { source(s_remote); filter(vpn); destination(vpn); };
    log { source(s_remote); filter(fw-intern-all);
    destination(fw-intern-all); };
    log { source(s_remote); filter(fw-extern-new);
    destination(fw-extern-new); };
    log { source(s_remote); filter(fw-extern-all);
    destination(fw-extern-all);
    flags(final); };
______________________________________________________________________________
    Member info: https://lists.balabit.hu/mailman/listinfo/syslog-ng
    Documentation:
    http://www.balabit.com/support/documentation/?product=syslog-ng
    FAQ: http://www.campin.net/syslog-ng/faq.html
______________________________________________________________________________
Member info: https://lists.balabit.hu/mailman/listinfo/syslog-ng
Documentation: http://www.balabit.com/support/documentation/?product=syslog-ng
FAQ: http://www.campin.net/syslog-ng/faq.html
-- 
___________________________________________
FACHHOCHSCHULE SALZBURG GmbH
Salzburg University of Applied Sciences

Andreas Sartori
Systems Engineer
IS - Information Services

Urstein Süd 1 | 5412 Puch/Salzburg | Austria
fon:  +43 (0)50-2211-1655 | fax: -1699
web: www.fh-salzburg.ac.at

Gerichtsstand Salzburg | FN166054y

WELCOME TO YOUR FUTURE!
___________________________________________