Re: [syslog-ng] log failback groups

22 Mar 2010

      Ok, so finally got time to look at this again.
I tried the time_sleep option, and it was a horrible failure. With it 
enabled syslog-ng started losing about 98% of all incoming lines (and 
no, thats not an exaggeration). I'm guessing that time_sleep does not 
play well with udp as thats what incoming data is being sent over.
However I do have the master-slave multi-process thing going and its 
working really good. I was able to put time_sleep on the child processes 
(the one doing regex matches), and it dropped their cpu utilization from 
around 40% to about 20% (master process uses tcp to talk to slave 
processes, so no drops). Another thing is that when I tried using the 
syslog protocol to talk to the child processes, the slaves were 
terminating the connection within seconds of being established. I poked 
and prodded and could not get this to work without constantly dropping 
the connection, so I had to switch back to plain tcp.

Anyway, the attached config is what it looked like when I had all 
regexes run within a single process (the config that was utilizing over 
90% cpu).

Sent: Thursday, March 18, 2010 10:56:16 PM
From: Jan Schaumann <jschauma@netmeister.org>
To: syslog-ng@lists.balabit.hu
Subject: Re: [syslog-ng] log failback groups
...
Martin Holste <mcholste@gmail.com> wrote:
...
How many messages per second is the system attempting to handle?  I'm
very surprised that you're seeing that level of utilization.  In our
setup we've never had a problem pushing up through 30,000 messages per
second written to disk with Syslog-NG in production, and I've pushed
more than 70,000 per second in development.
Could you provide your configuration for these systems (including
sysctls or kernel tunables etc.)?  I've so far not been able to get my
systems to accept and process (without any regex matching) more than
approximagely 25K - 30K UDP messages/s.
-Jan
------------------------------------------------------------------------
______________________________________________________________________________
Member info: https://lists.balabit.hu/mailman/listinfo/syslog-ng
Documentation: http://www.balabit.com/support/documentation/?product=syslog-ng
FAQ: http://www.campin.net/syslog-ng/faq.html
@version: 3.0
# syslog-ng configuration file.
#
# This should behave pretty much like the original syslog on RedHat. But
# it could be configured a lot smarter.
#
# See syslog-ng(8) and syslog-ng.conf(5) for more information.
#

options {
	time_reopen(10);
	use_dns(no);
	use_fqdn(no);
	keep_hostname(yes);
	create_dirs(yes);
	perm(0644);
	dir_perm(0755);
	log_iw_size(10000);
	log_fifo_size(20000);
};

source s_sys {
	file("/proc/kmsg" program-override("kernel"));
	unix-stream ("/dev/log");
	internal();
};
source s_net {
	tcp(ip(0.0.0.0) port(514) max-connections(1000) flags('syslog-protocol'));
	udp(ip(0.0.0.0) port(514));
};

destination d_cons { file("/dev/console"); };
destination d_mesg { file("/var/log/messages"); };
destination d_auth { file("/var/log/secure"); };
destination d_mail { file("/var/log/maillog"); };
destination d_spol { file("/var/log/spooler"); };
destination d_boot { file("/var/log/boot.log"); };
destination d_cron { file("/var/log/cron"); };
destination d_null { file("/dev/null" perm(0666)); };
destination d_syslog { file('/var/log/syslog'); };

filter f_kernel     { facility(kern); };
filter f_default    { level(info..emerg) and
                        not (facility(mail)
                        or facility(authpriv) 
                        or facility(cron)
                        or facility(user)); };
filter f_auth       { facility(authpriv); };
filter f_mail       { facility(mail); };
filter f_emergency  { level(emerg); };
filter f_news       { facility(uucp) or
                        (facility(news) 
                        and level(crit..emerg)); };
filter f_boot   { facility(local7); };
filter f_cron   { facility(cron); };
filter f_user   { facility(user); };
filter f_syslog { facility(syslog); };

log { source(s_sys); filter(f_kernel); destination(d_cons); };
log { source(s_sys); filter(f_default); destination(d_mesg); };
log { source(s_sys); filter(f_auth); destination(d_auth); };
log { source(s_sys); filter(f_mail); destination(d_mail); };
log { source(s_sys); filter(f_emergency); destination(d_cons); destination(d_mesg); };
log { source(s_sys); filter(f_news); destination(d_spol); };
log { source(s_sys); filter(f_boot); destination(d_boot); };
log { source(s_sys); filter(f_cron); destination(d_cron); };
log { source(s_sys); filter(f_user); destination(d_mesg); };
log { source(s_sys); filter(f_syslog); destination(d_syslog); };

# legacy logging format
filter f_usa_app {
	#not level(notice) and
	program('^(?<PBASE>smtad|mtad|mrmad|bbqd|cbqd|mrad|scand)' flags('nobackref','store-matches') type('pcre')) and
	message('^(?<PEXT>\w{4}): (?<TID>\[\d+\]) (?<MSGTAIL>.+)$' flags('nobackref','store-matches') type('pcre'));
};
template t_usa_app {
	template("$PID $TID $DATE $MSGTAIL\n");
};
destination d_usa_app {
	file("/var/log/hosts/$HOST/$PBASE/$PROGRAM.$MONTH$DAY.$PEXT" template(t_usa_app) flush_lines(10) flush_timeout(5000));
};
log { source(s_sys); source(s_net); filter(f_usa_app); destination(d_usa_app); flags('final'); };

# these are apps that get their own log directory & files
filter f_apps {
	program('^(postfix)' flags('store-matches')) or
	program('^(amavis)' flags('store-matches'));
};
destination d_apps {
	file("/var/log/hosts/$HOST/$1/$1.$LEVEL" flush_lines(10) flush_timeout(5000));
};
log { source(s_sys); source(s_net); filter(f_apps); destination(d_apps); flags('final'); };

# vim:ft=syslog-ng:ai:si:ts=4:sw=4:et:

Re: [syslog-ng] log failback groups

Patrick H.