The db parser code had a big memory leak in previous 3.1 versions but was fixed a few months ago; what build are you running? We process 2 billion logs per day through db parser with no leaks at all using the build from git commit 9ef6062c1cf72a3f7da880ac245f9ee080bea992. --Martin On Sat, Mar 13, 2010 at 2:22 AM, Andreas Sartori < andreas.sartori@fh-salzburg.ac.at> wrote:
hello,
we have setup a central logging server. currently we are logging firewalls and some webserver / mailserver for testing purpose. the memory usage on the logging server is badly increasing. after 2 days of operation we are at 6.8 gb ram usage.
can someone help out, what information do you need to help?
thanks in advance.
-andy
------------
@version:3.0 # # configuration file for syslog-ng, customized for remote logging #
options { owner("root"); group("root"); perm(0600); dir_perm(0750); create_dirs(yes); log_fifo_size(10000); };
################################################################################################ ######################### SOURCES ##############################
################################################################################################
# Syslog internal logging source s_internal { internal(); }; destination d_syslognglog { file("/var/log/syslog-ng.log"); }; log { source(s_internal); destination(d_syslognglog); };
# Remote logging source s_remote { tcp(ip(0.0.0.0) max-connections(20) port(514) keep_hostname(yes)); udp(ip(0.0.0.0) port(514) use_dns(no) log_fetch_limit(500) log_iw_size(1000)); };
################################################################################################ ######################### FILTER ##############################
################################################################################################
filter http-official { netmask(xxx.xxx.xxx.47/255.255.255.255) or netmask(xxx.xxx.xxx.48/255.255.255.255) or netmask(xxx.xxx.xxx.167/255.255.255.255) or netmask(xxx.xxx.xxx.46/255.255.255.255) or netmask(xxx.xxx.xxx.52/255.255.255.255) or netmask(xxx.xxx.xxx.25/255.255.255.255) or netmask(xxx.xxx.xxx.26/255.255.255.255); };
filter mail-proxy-internal { netmask(10.10.9.20/255.255.255.255) and not program("perdition"); }; filter mail-relay-internal { netmask(10.10.9.30/255.255.255.255); };
filter mail-relay-alpha-external-out { netmask(xxx.xxx.xxx.59/255.255.255.255) and facility(local1); }; filter mail-relay-beta-external-out { netmask(xxx.xxx.xxx.60/255.255.255.255) and facility(local1); }; filter mail-relay-alpha-external-in { netmask(xxx.xxx.xxx.59/255.255.255.255) and facility(mail); }; filter mail-relay-beta-external-in { netmask(xxx.xxx.xxx.60/255.255.255.255) and facility(mail); };
filter mail-proxy-node1-external { netmask(xxx.xxx.xxx.18/255.255.255.255) and not program("perdition"); }; filter mail-proxy-node2-external { netmask(xxx.xxx.xxx.22/255.255.255.255) and not program("perdition"); };
filter vpn { netmask(10.20.40.0/255.255.255.0); }; filter fw-intern-all { netmask(10.10.20.1/255.255.255.255); };
filter fw-intern-security { netmask(10.10.20.1/255.255.255.255) and match("security" value(".classifier.class") type("string")); };
filter fw-intern-info { netmask(10.10.20.1/255.255.255.255) and match("informational" value(".classifier.class") type("string")); };
filter fw-intern-rest { netmask(10.10.20.1/255.255.255.255) and not match("security" value(".classifier.class") type("string")) and not match("informational" value(".classifier.class") type("string")); };
filter fw-extern-all { netmask(10.80.11.20/255.255.255.255); };
filter fw-extern-security { netmask(10.80.11.20/255.255.255.255) and match("security" value(".classifier.class") type("string")); };
filter fw-extern-info { netmask(10.80.11.20/255.255.255.255) and match("informational" value(".classifier.class") type("string")); };
filter fw-extern-rest { netmask(10.80.11.20/255.255.255.255) and not match("security" value(".classifier.class") type("string")) and not match("informational" value(".classifier.class") type("string")); };
filter fw-extern-new { netmask(10.80.11.30/255.255.255.255); };
################################################################################################ ######################### PARSER ##############################
################################################################################################
parser pattern_db_fwint { db_parser( file("/etc/syslog-ng/fw-int_patterndb.xml") ); };
parser pattern_db_fwext { db_parser( file("/etc/syslog-ng/fw-ext_patterndb.xml") ); };
################################################################################################ ######################### DESTINATIONS ##############################
################################################################################################
destination http-log { file("/logging/server/web/$HOST" template("$MSGONLY\n") template-escape(no) owner("root") group("root") perm(0644)); };
destination mail-out { file("/logging/server/mail/mail-out_$MONTH.log"); }; destination mail-in { file("/logging/server/mail/mail-in_$MONTH.log"); };
destination vpn { file("/logging/network/vpn_$MONTH.log" flush_lines(10)); };
destination fw-intern-all { file("/logging/network/fw-intern_$MONTH.log" flush_lines(10)); };
destination fw-extern-all { file("/logging/network/fw-extern_$MONTH.log" flush_lines(10)); };
destination fw-extern-new { file("/logging/network/fw-new_$MONTH.log" flush_lines(10)); };
destination dump { file("/logging/network/dump.log" template ("$R_YEAR-$R_MONTH-$R_DAY $R_HOUR:$R_MIN:$R_SEC, $HOST, $FIREWALL_SEQ, $MSGHDR, 0, $FIREWALL_IO, $FIREWALL_PROTO, $FIREWALL_SCR_LAN, $FIREWALL_SRC_IP, $FIREWALL_SRC_PORT, $FIREWALL_DST_LAN, $FIREWALL_DST_IP, $FIREWALL_DST_PORT, $FIREWALL_NAT_SRC_IP, $FIREWALL_NAT_DST_IP, $FIREWALL_RULE, $FIREWALL_REASON, $FIREWALL_DURATION\n")); # file("/logging/network/dump.log" template ("$MSGHDR\n") flush_lines(5)); };
################################################################################################ ######################### FINAL-LOGS ##############################
################################################################################################
##### TO FILE
log { source(s_remote); filter(http-official); destination(http-log); }; log { source(s_remote); filter(mail-proxy-internal); destination(mail-out); }; log { source(s_remote); filter(mail-relay-internal); destination(mail-out); }; log { source(s_remote); filter(mail-relay-alpha-external-out); destination(mail-out); }; log { source(s_remote); filter(mail-relay-beta-external-out); destination(mail-out); }; log { source(s_remote); filter(mail-proxy-node1-external); destination(mail-out); }; log { source(s_remote); filter(mail-proxy-node2-external); destination(mail-out); }; log { source(s_remote); filter(mail-relay-alpha-external-in); destination(mail-in); }; log { source(s_remote); filter(mail-relay-beta-external-in); destination(mail-in); }; log { source(s_remote); filter(vpn); destination(vpn); }; log { source(s_remote); filter(fw-intern-all); destination(fw-intern-all); }; log { source(s_remote); filter(fw-extern-new); destination(fw-extern-new); }; log { source(s_remote); filter(fw-extern-all); destination(fw-extern-all); flags(final); };
______________________________________________________________________________ Member info: https://lists.balabit.hu/mailman/listinfo/syslog-ng Documentation: http://www.balabit.com/support/documentation/?product=syslog-ng FAQ: http://www.campin.net/syslog-ng/faq.html