[syslog-ng] syslog-ng memory usage grows
Jim Hendrick
jrhendri at roadrunner.com
Tue Dec 16 19:24:28 CET 2014
Hi,
I am continuing to test syslog-ng as the parser and shipper into an
elasticsearch cluster.
Right now I have syslog-ng 3.6.1 receiving logs at about ~7000 EPS and
running them through a patterndb parser that splits out the (bluecoat
proxy) fields into key-value pairs, writing them to a redis destination
using format-json.
syslog-ng 3.6.1
Installer-Version: 3.6.1
Revision:
Compile-Date: Dec 9 2014 19:42:20
Available-Modules: dbparser,json-plugin,afuser,affile,afmongodb,tfgeoip,afprog,redis,afstomp,afsql,afsocket,system-source,afamqp,pseudofile,confgen,afsocket-tls,csvparser,basicfuncs,graphite,syslogformat,afsocket-notls,linux-kmsg-format,cryptofuncs
Enable-Debug: off
Enable-GProf: off
Enable-Memtrace: off
Enable-IPv6: on
Enable-Spoof-Source: off
Enable-TCP-Wrapper: off
Enable-Linux-Caps: off
I then use logstash to pull from redis and feed elasticsearch (the
thought being that this would provide a buffer for the messages)
Over the weekend I had syslog-ng crash (unfortunately no core) but now
that I am watching it more closely, I see what appears to be continuous
growth in memory usage (which leads me to suppose this was the cause of
the crash).
I'm not sure what I am asking, other than general advice on:
- performance using patterndb
- performance using redis destination
- advice on debugging where this memory growth is happening
As a rough measure - I have a syslog-ng process that has been running
for less than 3 hours and right now is using 1.52GB of resident memory
(shown by "top")
I am including two config files. One has a single source, no filtering,
single destination. The other splits logs from three upstream syslog-ng
servers thinking this would provide "threading" with multiple patterndb
parsers (I have read that patterndb is single threaded).
Any thoughts on how to figure out where the memory is going?
Or recommendations on whether or not using patterndb in this way is a
particularly dumb idea :-)
(I would rather parse things in syslog-ng, but I *could* do this all
using logstash/grok if this proves too much for patterndb at this load)
Thanks!
Jim
-------------- next part --------------
@version: 3.6
@include "scl.conf"
options {
check_hostname(yes); # check if the hostname contains valid characters
use_dns(no); # do not resolve names for speed
dns_cache(no); # no DNS cache since we do not resolve names
keep_hostname(yes); # keep hostnames to enable related macros
chain_hostnames(no); # do not track / forward syslog forwarder chain
# options related to file and directories
dir_owner("root");
dir_group("adm");
owner("root");
group("adm");
perm(0640);
dir_perm(0750);
create_dirs(yes);
threaded(yes);
};
source s_local {
system();
internal();
};
source s_network {
udp();
};
parser p_bluecoat {
db-parser(file("/usr/local/etc/patterndb.d/bluecoat.xml"));
};
destination d_redis {
redis (
host("localhost")
command("LPUSH", "logstash", "$(format-json proxy_time=${PROXY.TIME} proxy_time_taken=${PROXY.TIME_TAKEN} proxy_c_ip=${PROXY.C_IP} proxy_sc_status=${PROXY.SC_STATUS} proxy_s_action=${PROXY.S_ACTION} proxy_sc_bytes=${PROXY.SC_BYTES} proxy_cs_bytes=${PROXY.CS_BYTES} proxy_cs_method=${PROXY.CS_METHOD} proxy_cs_uri_scheme=${PROXY.CS_URI_SCHEME} proxy_cs_host=${PROXY.CS_HOST} proxy_cs_uri_port=${PROXY.CS_URI_PORT} proxy_cs_uri_path=${PROXY.CS_URI_PATH} proxy_cs_uri_equery=${PROXY.CS_URI_EQUERY} proxy_cs_username=${PROXY.CS_USERNAME} proxy_cs_auth_group=${PROXY.CS_AUTH__GROUP} proxy_s_supplier_name=${PROXY.S_SUPPLIER_NAME} proxy_content_type=${PROXY.CONTENT_TYPE} proxy_referrer=${PROXY.REFERRER} proxy_user_agent=${PROXY.USER_AGENT} proxy_filter_result=${PROXY.FILTER_RESULT} proxy_cs_categories=${PROXY.CS_CATEGORIES} proxy_x_virus_id=${PROXY.X_VIRUS_ID} proxy_s_ip=${PROXY.S_IP})\n")
);
};
destination d_local {
file("/var/log/syslog-ng/$YEAR/$MONTH/$DAY/$HOST_FROM/$HOST_FROM.$FACILITY.$PRIORITY.$YEAR.$MONTH.$DAY");
};
log {
source(s_local);
destination(d_local);
};
log {
source(s_network);
parser(p_bluecoat);
destination(d_redis);
};
-------------- next part --------------
@version: 3.6
@include "scl.conf"
options {
check_hostname(yes); # check if the hostname contains valid characters
use_dns(no); # do not resolve names for speed
dns_cache(no); # no DNS cache since we do not resolve names
keep_hostname(yes); # keep hostnames to enable related macros
chain_hostnames(no); # do not track / forward syslog forwarder chain
# options related to file and directories
dir_owner("root");
dir_group("adm");
owner("root");
group("adm");
perm(0640);
dir_perm(0750);
create_dirs(yes);
threaded(yes);
};
source s_local {
system();
internal();
};
source s_network {
udp();
};
parser p_bluecoat {
db-parser(file("/usr/local/etc/patterndb.d/bluecoat.xml"));
};
filter f_from_pdc {
netmask("10.177.74.55");
};
filter f_from_kdc {
netmask("10.181.73.32");
};
filter f_from_rdc {
netmask("10.185.72.138");
};
destination d_redis {
redis (
host("localhost")
command("LPUSH", "logstash", "$(format-json proxy_time=${PROXY.TIME} proxy_time_taken=${PROXY.TIME_TAKEN} proxy_c_ip=${PROXY.C_IP} proxy_sc_status=${PROXY.SC_STATUS} proxy_s_action=${PROXY.S_ACTION} proxy_sc_bytes=${PROXY.SC_BYTES} proxy_cs_bytes=${PROXY.CS_BYTES} proxy_cs_method=${PROXY.CS_METHOD} proxy_cs_uri_scheme=${PROXY.CS_URI_SCHEME} proxy_cs_host=${PROXY.CS_HOST} proxy_cs_uri_port=${PROXY.CS_URI_PORT} proxy_cs_uri_path=${PROXY.CS_URI_PATH} proxy_cs_uri_equery=${PROXY.CS_URI_EQUERY} proxy_cs_username=${PROXY.CS_USERNAME} proxy_cs_auth_group=${PROXY.CS_AUTH__GROUP} proxy_s_supplier_name=${PROXY.S_SUPPLIER_NAME} proxy_content_type=${PROXY.CONTENT_TYPE} proxy_referrer=${PROXY.REFERRER} proxy_user_agent=${PROXY.USER_AGENT} proxy_filter_result=${PROXY.FILTER_RESULT} proxy_cs_categories=${PROXY.CS_CATEGORIES} proxy_x_virus_id=${PROXY.X_VIRUS_ID} proxy_s_ip=${PROXY.S_IP})\n")
);
};
destination d_local {
file("/var/log/syslog-ng/$YEAR/$MONTH/$DAY/$HOST_FROM/$HOST_FROM.$FACILITY.$PRIORITY.$YEAR.$MONTH.$DAY");
};
log {
source(s_local);
destination(d_local);
};
log {
source(s_network);
filter(f_from_pdc);
parser(p_bluecoat);
destination(d_redis);
flags(final);
};
log {
source(s_network);
filter(f_from_kdc);
parser(p_bluecoat);
destination(d_redis);
flags(final);
};
log {
source(s_network);
parser(p_bluecoat);
destination(d_redis);
};
More information about the syslog-ng
mailing list