The db parser code had a big memory leak in previous 3.1 versions but was fixed a few months ago; what build are you running? We process 2 billion logs per day through db parser with no leaks at all using the build from git commit 9ef6062c1cf72a3f7da880ac245f9ee080bea992.<br>
<br>--Martin<br><br><div class="gmail_quote">On Sat, Mar 13, 2010 at 2:22 AM, Andreas Sartori <span dir="ltr"><<a href="mailto:andreas.sartori@fh-salzburg.ac.at">andreas.sartori@fh-salzburg.ac.at</a>></span> wrote:<br>
<blockquote class="gmail_quote" style="border-left: 1px solid rgb(204, 204, 204); margin: 0pt 0pt 0pt 0.8ex; padding-left: 1ex;">hello,<br>
<br>
<br>
we have setup a central logging server. currently we are logging firewalls<br>
and some webserver / mailserver for testing purpose. the memory usage on<br>
the logging server is badly increasing. after 2 days of operation we are<br>
at 6.8 gb ram usage.<br>
<br>
can someone help out, what information do you need to help?<br>
<br>
thanks in advance.<br>
<br>
-andy<br>
<br>
------------<br>
<br>
@version:3.0<br>
#<br>
# configuration file for syslog-ng, customized for remote logging<br>
#<br>
<br>
options {<br>
owner("root");<br>
group("root");<br>
perm(0600);<br>
dir_perm(0750);<br>
create_dirs(yes);<br>
log_fifo_size(10000);<br>
};<br>
<br>
<br>
<br>
################################################################################################<br>
######################### SOURCES<br>
##############################<br>
################################################################################################<br>
<br>
# Syslog internal logging<br>
source s_internal { internal(); };<br>
destination d_syslognglog { file("/var/log/syslog-ng.log"); };<br>
log { source(s_internal); destination(d_syslognglog); };<br>
<br>
<br>
# Remote logging<br>
source s_remote {<br>
tcp(ip(0.0.0.0) max-connections(20) port(514) keep_hostname(yes));<br>
udp(ip(0.0.0.0) port(514) use_dns(no) log_fetch_limit(500)<br>
log_iw_size(1000));<br>
};<br>
<br>
<br>
################################################################################################<br>
######################### FILTER<br>
##############################<br>
################################################################################################<br>
<br>
filter http-official { netmask(xxx.xxx.xxx.47/<a href="http://255.255.255.255" target="_blank">255.255.255.255</a>) or<br>
netmask(xxx.xxx.xxx.48/<a href="http://255.255.255.255" target="_blank">255.255.255.255</a>) or<br>
netmask(xxx.xxx.xxx.167/<a href="http://255.255.255.255" target="_blank">255.255.255.255</a>) or<br>
netmask(xxx.xxx.xxx.46/<a href="http://255.255.255.255" target="_blank">255.255.255.255</a>) or<br>
netmask(xxx.xxx.xxx.52/<a href="http://255.255.255.255" target="_blank">255.255.255.255</a>) or<br>
netmask(xxx.xxx.xxx.25/<a href="http://255.255.255.255" target="_blank">255.255.255.255</a>) or<br>
netmask(xxx.xxx.xxx.26/<a href="http://255.255.255.255" target="_blank">255.255.255.255</a>); };<br>
<br>
filter mail-proxy-internal { netmask(<a href="http://10.10.9.20/255.255.255.255" target="_blank">10.10.9.20/255.255.255.255</a>) and not<br>
program("perdition"); };<br>
filter mail-relay-internal { netmask(<a href="http://10.10.9.30/255.255.255.255" target="_blank">10.10.9.30/255.255.255.255</a>); };<br>
<br>
filter mail-relay-alpha-external-out {<br>
netmask(xxx.xxx.xxx.59/<a href="http://255.255.255.255" target="_blank">255.255.255.255</a>) and facility(local1); };<br>
filter mail-relay-beta-external-out {<br>
netmask(xxx.xxx.xxx.60/<a href="http://255.255.255.255" target="_blank">255.255.255.255</a>) and facility(local1); };<br>
filter mail-relay-alpha-external-in {<br>
netmask(xxx.xxx.xxx.59/<a href="http://255.255.255.255" target="_blank">255.255.255.255</a>) and facility(mail); };<br>
filter mail-relay-beta-external-in {<br>
netmask(xxx.xxx.xxx.60/<a href="http://255.255.255.255" target="_blank">255.255.255.255</a>) and facility(mail); };<br>
<br>
filter mail-proxy-node1-external { netmask(xxx.xxx.xxx.18/<a href="http://255.255.255.255" target="_blank">255.255.255.255</a>)<br>
and not program("perdition"); };<br>
filter mail-proxy-node2-external { netmask(xxx.xxx.xxx.22/<a href="http://255.255.255.255" target="_blank">255.255.255.255</a>)<br>
and not program("perdition"); };<br>
<br>
filter vpn { netmask(<a href="http://10.20.40.0/255.255.255.0" target="_blank">10.20.40.0/255.255.255.0</a>); };<br>
filter fw-intern-all { netmask(<a href="http://10.10.20.1/255.255.255.255" target="_blank">10.10.20.1/255.255.255.255</a>); };<br>
<br>
filter fw-intern-security {<br>
netmask(<a href="http://10.10.20.1/255.255.255.255" target="_blank">10.10.20.1/255.255.255.255</a>) and<br>
match("security" value(".classifier.class") type("string"));<br>
};<br>
<br>
filter fw-intern-info {<br>
netmask(<a href="http://10.10.20.1/255.255.255.255" target="_blank">10.10.20.1/255.255.255.255</a>) and<br>
match("informational" value(".classifier.class") type("string"));<br>
};<br>
<br>
filter fw-intern-rest {<br>
netmask(<a href="http://10.10.20.1/255.255.255.255" target="_blank">10.10.20.1/255.255.255.255</a>) and not<br>
match("security" value(".classifier.class") type("string")) and not<br>
match("informational" value(".classifier.class") type("string"));<br>
};<br>
<br>
<br>
filter fw-extern-all { netmask(<a href="http://10.80.11.20/255.255.255.255" target="_blank">10.80.11.20/255.255.255.255</a>); };<br>
<br>
filter fw-extern-security {<br>
netmask(<a href="http://10.80.11.20/255.255.255.255" target="_blank">10.80.11.20/255.255.255.255</a>) and<br>
match("security" value(".classifier.class") type("string"));<br>
};<br>
<br>
filter fw-extern-info {<br>
netmask(<a href="http://10.80.11.20/255.255.255.255" target="_blank">10.80.11.20/255.255.255.255</a>) and<br>
match("informational" value(".classifier.class") type("string"));<br>
};<br>
<br>
filter fw-extern-rest {<br>
netmask(<a href="http://10.80.11.20/255.255.255.255" target="_blank">10.80.11.20/255.255.255.255</a>) and not<br>
match("security" value(".classifier.class") type("string")) and not<br>
match("informational" value(".classifier.class") type("string"));<br>
};<br>
<br>
filter fw-extern-new { netmask(<a href="http://10.80.11.30/255.255.255.255" target="_blank">10.80.11.30/255.255.255.255</a>); };<br>
<br>
################################################################################################<br>
######################### PARSER<br>
##############################<br>
################################################################################################<br>
<br>
parser pattern_db_fwint {<br>
db_parser(<br>
file("/etc/syslog-ng/fw-int_patterndb.xml")<br>
);<br>
};<br>
<br>
parser pattern_db_fwext {<br>
db_parser(<br>
file("/etc/syslog-ng/fw-ext_patterndb.xml")<br>
);<br>
};<br>
<br>
################################################################################################<br>
######################### DESTINATIONS<br>
##############################<br>
################################################################################################<br>
<br>
destination http-log { file("/logging/server/web/$HOST"<br>
template("$MSGONLY\n") template-escape(no) owner("root") group("root")<br>
perm(0644)); };<br>
<br>
destination mail-out { file("/logging/server/mail/mail-out_$MONTH.log"); };<br>
destination mail-in { file("/logging/server/mail/mail-in_$MONTH.log"); };<br>
<br>
destination vpn {<br>
file("/logging/network/vpn_$MONTH.log" flush_lines(10));<br>
};<br>
<br>
destination fw-intern-all {<br>
file("/logging/network/fw-intern_$MONTH.log" flush_lines(10));<br>
};<br>
<br>
destination fw-extern-all {<br>
file("/logging/network/fw-extern_$MONTH.log" flush_lines(10));<br>
};<br>
<br>
<br>
destination fw-extern-new {<br>
file("/logging/network/fw-new_$MONTH.log" flush_lines(10));<br>
};<br>
<br>
<br>
destination dump {<br>
file("/logging/network/dump.log" template ("$R_YEAR-$R_MONTH-$R_DAY<br>
$R_HOUR:$R_MIN:$R_SEC, $HOST, $FIREWALL_SEQ, $MSGHDR, 0, $FIREWALL_IO,<br>
$FIREWALL_PROTO, $FIREWALL_SCR_LAN, $FIREWALL_SRC_IP, $FIREWALL_SRC_PORT,<br>
$FIREWALL_DST_LAN, $FIREWALL_DST_IP, $FIREWALL_DST_PORT,<br>
$FIREWALL_NAT_SRC_IP, $FIREWALL_NAT_DST_IP, $FIREWALL_RULE,<br>
$FIREWALL_REASON, $FIREWALL_DURATION\n"));<br>
# file("/logging/network/dump.log" template ("$MSGHDR\n") flush_lines(5));<br>
};<br>
<br>
<br>
<br>
################################################################################################<br>
######################### FINAL-LOGS<br>
##############################<br>
################################################################################################<br>
<br>
##### TO FILE<br>
<br>
log { source(s_remote); filter(http-official); destination(http-log); };<br>
log { source(s_remote); filter(mail-proxy-internal);<br>
destination(mail-out); };<br>
log { source(s_remote); filter(mail-relay-internal);<br>
destination(mail-out); };<br>
log { source(s_remote); filter(mail-relay-alpha-external-out);<br>
destination(mail-out); };<br>
log { source(s_remote); filter(mail-relay-beta-external-out);<br>
destination(mail-out); };<br>
log { source(s_remote); filter(mail-proxy-node1-external);<br>
destination(mail-out); };<br>
log { source(s_remote); filter(mail-proxy-node2-external);<br>
destination(mail-out); };<br>
log { source(s_remote); filter(mail-relay-alpha-external-in);<br>
destination(mail-in); };<br>
log { source(s_remote); filter(mail-relay-beta-external-in);<br>
destination(mail-in); };<br>
log { source(s_remote); filter(vpn); destination(vpn); };<br>
log { source(s_remote); filter(fw-intern-all); destination(fw-intern-all); };<br>
log { source(s_remote); filter(fw-extern-new); destination(fw-extern-new); };<br>
log { source(s_remote); filter(fw-extern-all); destination(fw-extern-all);<br>
flags(final); };<br>
<br>
<br>
<br>
______________________________________________________________________________<br>
Member info: <a href="https://lists.balabit.hu/mailman/listinfo/syslog-ng" target="_blank">https://lists.balabit.hu/mailman/listinfo/syslog-ng</a><br>
Documentation: <a href="http://www.balabit.com/support/documentation/?product=syslog-ng" target="_blank">http://www.balabit.com/support/documentation/?product=syslog-ng</a><br>
FAQ: <a href="http://www.campin.net/syslog-ng/faq.html" target="_blank">http://www.campin.net/syslog-ng/faq.html</a><br>
<br>
</blockquote></div><br>