Hi All, I am using syslog-ng OSE & seeing a high CPU usage ? Could you please point out if this could be a config/rules issue or syslog-ng internal issue. CPU usage in sorted order : 0.0 11.9 16.6 22.2 25.0 25.0 25.0 25.0 33.3 33.3 50.0 ------------------------------------------ # syslog-ng --version syslog-ng 3.3.5 Installer-Version: 3.3.5 Revision: Compile-Date: Apr 20 2012 04:47:44 Default-Modules: afsocket,affile,afprog,afuser,basicfuncs,syslogformat,dbparser Available-Modules: syslogformat,dbparser,afuser,basicfuncs,afmongodb,convertfuncs,afprog,dummy,afsocket,confgen,affile,csvparser Enable-Debug: off Enable-GProf: off Enable-Memtrace: off Enable-IPv6: on Enable-Spoof-Source: off Enable-TCP-Wrapper: off Enable-Linux-Caps: off Enable-Pcre: off ----------- Here is the code snippet to send syslog messages to syslogger for(i=0; i<500;i++) { // this is the message format we are following syslog(LOG_INFO, "%s, %s, %d, %s, %s, %s %d, %s", "00:00:18","XYZ",getpid(),"Info","xyz.c", "Func1", 100, "WelCome to Syslog" ); } --------------------- # cat /proc/cpuinfo Processor : ARMv7 Processor rev 10 (v7l) processor : 0 BogoMIPS : 1988.29 processor : 1 BogoMIPS : 1988.29 Features : swp half thumb fastmult vfp edsp neon vfpv3 CPU implementer : 0x41 CPU architecture: 7 CPU variant : 0x2 CPU part : 0xc09 CPU revision : 10 ------------------------------ syslog-ng-confg: @version: 3.3 ###### # options options { # disable the chained hostname format in logs # (default is enabled) chain_hostnames(0); # the time to wait before a died connection is re-established # (default is 60) time_reopen(10); # the time to wait before an idle destination file is closed # (default is 60) time_reap(360); # the number of lines buffered before written to file # you might want to increase this if your disk isn't catching with # all the log messages you get or if you want less disk activity flush_lines(5); flush_timeout(500); # the number of lines fitting in the output queue log_fifo_size(10000); # enable or disable directory creation for destination files create_dirs(yes); # default owner, group, and permissions for log files # (defaults are 0, 0, 0600) #owner(root); group(root); perm(0640); frac_digits(4); # Override the global timestamp format #(set in the global ts_format() parameter) for the specific destination. ts_format(iso); # default owner, group, and permissions for created directories # (defaults are 0, 0, 0700) #dir_owner(root); dir_perm(0755); # enable or disable DNS usage # syslog-ng blocks on DNS queries, so enabling DNS may lead to # a Denial of Service attack # (default is yes) use_dns(no); # maximum length of message in bytes # this is only limited by the program listening on the /dev/log Unix # socket, glibc can handle arbitrary length log messages, but -- for # example -- syslogd accepts only 1024 bytes # (default is 2048) #log_msg_size(2048); # print statistic information only once every hour (it is not really needed) stats_freq(3600); }; ###### # sources # all known message sources source s_all { # message generated by Syslog-NG internal(); # standard Linux log source (this is the default place for the syslog() # function to send logs to) unix-dgram("/dev/log" program_override("mylogger: ")); # messages from the kernel file("/proc/kmsg" program_override("kernel: ")); # use the following line if you want to receive remote UDP logging messages # (this is equivalent to the "-r" syslogd flag) # udp(); }; ######## #templates #template t_default { template("${ISODATE} ${HOST} ${MSGHDR}${MSG}\n"); template_escape(no); }; #template t_logtty { template("${MSG}\n"); template_escape(no); }; template t_restmsg_notempty { template("${PROGRAM} ${date} ${cname}[${processId}] ${type} (${source}:${line}) ${Restmsg}\n"); }; template t_restmsg_empty { template("${PROGRAM} ${FACILITY} ${LEVEL} ${MSG}\n"); }; ###### # destinations destination d_log_notempty { file("/var/my_log" template(t_restmsg_notempty)); }; #destination d_log_notempty { file("/dev/console" template(t_restmsg_notempty)); }; destination d_log_empty { file("/var/my_log" template(t_restmsg_empty)); }; destination authlog { file("/var/log/auth.log"); }; destination debug { file("/var/log/debug"); }; destination messages { file("/var/log/messages"); }; destination _syslog { file("/var/log/syslog"); }; destination cron { file("/var/log/cron.log"); }; destination daemon { file("/var/log/daemon.log"); }; destination lpr { file("/var/log/lpr.log"); }; destination user { file("/var/log/user.log"); }; destination uucp { file("/var/log/uucp.log"); }; destination mail { file("/var/log/mail.log"); }; destination kern { file("/var/log/kern.log"); }; destination mailinfo { file("/var/log/mail.info"); }; destination mailwarn { file("/var/log/mail.warn"); }; destination mailerr { file("/var/log/mail.err"); }; destination newscrit { file("/var/log/news/news.crit"); }; destination newserr { file("/var/log/news/news.err"); }; destination newsnotice { file("/var/log/news/news.notice"); }; ###### # filters # all messages from the auth and authpriv facilities filter f_restmsg_notempty {match(".+" value("Restmsg"));}; filter f_restmsg_empty {not match(".+" value("Restmsg"));}; # all messages except from the auth and authpriv facilities filter f_auth { facility(auth); }; filter f_authpriv { not facility(auth, authpriv); }; #filter f_debug { level(notice) and not facility(auth, authpriv, news, mail); }; filter f_emergency { level(emerg); }; filter f_info { level(info); }; filter f_notice { level(notice); }; filter f_warn { level(warn); }; filter f_crit { level(crit); }; filter f_err { level(err); }; #filter f_syslog { not facility(authpriv, mail ); }; filter f_syslog { program(syslog-ng); }; filter f_cron { facility(cron); }; filter f_daemon { facility(daemon); }; filter f_kern { facility(kern); }; filter f_mail { facility(mail); }; filter f_user { facility(user); and level(error) }; filter f_uucp { facility(uucp); }; filter f_news { facility(news); }; filter f_messages { level(info..warn) and not facility(auth, authpriv, mail, news); }; filter f_all_info { not match("Info", value("type"), type("string")); }; filter f_all_generic{ not match("Generic", value("cname"), type("string")); }; parser p_db {db_parser( file("/etc/syslog-ng/rules/rules.xml") );}; log {source(s_all);parser(p_db);filter(f_restmsg_notempty);destination(d_log_notempty); }; #log { source(s_all); filter(f_authpriv); destination(authlog); }; log { source(s_all); filter(f_syslog); destination(_syslog); }; log { source(s_all); filter(f_cron); destination(cron); }; log { source(s_all); filter(f_daemon); destination(daemon); }; log { source(s_all); filter(f_kern); destination(kern); }; log { source(s_all); filter(f_mail); destination(mail); }; log { source(s_all); filter(f_user); destination(user); }; #log { source(s_all); filter(f_debug); destination(debug); }; ------------------------------------------------------------------------------------- rules.xml: <patterndb version='4' pub_date='2012-02-17'> <ruleset name='root' id='123456678'> <pattern>mylogger</pattern> <rules> <rule provider='test' id='182437592347598' class='system'> <patterns> <pattern>@ESTRING:date:,@ @ESTRING:cname:,@ @ESTRING:processId:,@ @ESTRING:type:,@ @ESTRING:source:,@ @ESTRING:line:,@ @ANYSTRING:Restmsg: @</pattern> </patterns> </rule> </rules> </ruleset> </patterndb> --------------------------------------------------------------------------------------------- Thanks, pattarsatish
off the top of my head I would change # all messages from the auth and authpriv facilities filter f_restmsg_notempty {match(".+" value("Restmsg"));}; filter f_restmsg_empty {not match(".+" value("Restmsg"));}; to # all messages from the auth and authpriv facilities filter f_restmsg_notempty {match("^." value("Restmsg"));}; filter f_restmsg_empty {not match("^." value("Restmsg"));}; and that would save a lot of CPU. Evan. ________________________________________ From: syslog-ng-bounces@lists.balabit.hu [syslog-ng-bounces@lists.balabit.hu] On Behalf Of satish pattar [satish.pattar@gmail.com] Sent: Tuesday, July 31, 2012 12:04 AM To: syslog-ng@lists.balabit.hu Subject: [syslog-ng] Syslog-ng OSE 3.3.5 - High CPU usage Hi All, I am using syslog-ng OSE & seeing a high CPU usage ? Could you please point out if this could be a config/rules issue or syslog-ng internal issue. CPU usage in sorted order : 0.0 11.9 16.6 22.2 25.0 25.0 25.0 25.0 33.3 33.3 50.0 ------------------------------------------ # syslog-ng --version syslog-ng 3.3.5 Installer-Version: 3.3.5 Revision: Compile-Date: Apr 20 2012 04:47:44 Default-Modules: afsocket,affile,afprog,afuser,basicfuncs,syslogformat,dbparser Available-Modules: syslogformat,dbparser,afuser,basicfuncs,afmongodb,convertfuncs,afprog,dummy,afsocket,confgen,affile,csvparser Enable-Debug: off Enable-GProf: off Enable-Memtrace: off Enable-IPv6: on Enable-Spoof-Source: off Enable-TCP-Wrapper: off Enable-Linux-Caps: off Enable-Pcre: off ----------- Here is the code snippet to send syslog messages to syslogger for(i=0; i<500;i++) { // this is the message format we are following syslog(LOG_INFO, "%s, %s, %d, %s, %s, %s %d, %s", "00:00:18","XYZ",getpid(),"Info","xyz.c", "Func1", 100, "WelCome to Syslog" ); } --------------------- # cat /proc/cpuinfo Processor : ARMv7 Processor rev 10 (v7l) processor : 0 BogoMIPS : 1988.29 processor : 1 BogoMIPS : 1988.29 Features : swp half thumb fastmult vfp edsp neon vfpv3 CPU implementer : 0x41 CPU architecture: 7 CPU variant : 0x2 CPU part : 0xc09 CPU revision : 10 ------------------------------ syslog-ng-confg: @version: 3.3 ###### # options options { # disable the chained hostname format in logs # (default is enabled) chain_hostnames(0); # the time to wait before a died connection is re-established # (default is 60) time_reopen(10); # the time to wait before an idle destination file is closed # (default is 60) time_reap(360); # the number of lines buffered before written to file # you might want to increase this if your disk isn't catching with # all the log messages you get or if you want less disk activity flush_lines(5); flush_timeout(500); # the number of lines fitting in the output queue log_fifo_size(10000); # enable or disable directory creation for destination files create_dirs(yes); # default owner, group, and permissions for log files # (defaults are 0, 0, 0600) #owner(root); group(root); perm(0640); frac_digits(4); # Override the global timestamp format #(set in the global ts_format() parameter) for the specific destination. ts_format(iso); # default owner, group, and permissions for created directories # (defaults are 0, 0, 0700) #dir_owner(root); dir_perm(0755); # enable or disable DNS usage # syslog-ng blocks on DNS queries, so enabling DNS may lead to # a Denial of Service attack # (default is yes) use_dns(no); # maximum length of message in bytes # this is only limited by the program listening on the /dev/log Unix # socket, glibc can handle arbitrary length log messages, but -- for # example -- syslogd accepts only 1024 bytes # (default is 2048) #log_msg_size(2048); # print statistic information only once every hour (it is not really needed) stats_freq(3600); }; ###### # sources # all known message sources source s_all { # message generated by Syslog-NG internal(); # standard Linux log source (this is the default place for the syslog() # function to send logs to) unix-dgram("/dev/log" program_override("mylogger: ")); # messages from the kernel file("/proc/kmsg" program_override("kernel: ")); # use the following line if you want to receive remote UDP logging messages # (this is equivalent to the "-r" syslogd flag) # udp(); }; ######## #templates #template t_default { template("${ISODATE} ${HOST} ${MSGHDR}${MSG}\n"); template_escape(no); }; #template t_logtty { template("${MSG}\n"); template_escape(no); }; template t_restmsg_notempty { template("${PROGRAM} ${date} ${cname}[${processId}] ${type} (${source}:${line}) ${Restmsg}\n"); }; template t_restmsg_empty { template("${PROGRAM} ${FACILITY} ${LEVEL} ${MSG}\n"); }; ###### # destinations destination d_log_notempty { file("/var/my_log" template(t_restmsg_notempty)); }; #destination d_log_notempty { file("/dev/console" template(t_restmsg_notempty)); }; destination d_log_empty { file("/var/my_log" template(t_restmsg_empty)); }; destination authlog { file("/var/log/auth.log"); }; destination debug { file("/var/log/debug"); }; destination messages { file("/var/log/messages"); }; destination _syslog { file("/var/log/syslog"); }; destination cron { file("/var/log/cron.log"); }; destination daemon { file("/var/log/daemon.log"); }; destination lpr { file("/var/log/lpr.log"); }; destination user { file("/var/log/user.log"); }; destination uucp { file("/var/log/uucp.log"); }; destination mail { file("/var/log/mail.log"); }; destination kern { file("/var/log/kern.log"); }; destination mailinfo { file("/var/log/mail.info<http://mail.info>"); }; destination mailwarn { file("/var/log/mail.warn"); }; destination mailerr { file("/var/log/mail.err"); }; destination newscrit { file("/var/log/news/news.crit"); }; destination newserr { file("/var/log/news/news.err"); }; destination newsnotice { file("/var/log/news/news.notice"); }; ###### # filters # all messages from the auth and authpriv facilities filter f_restmsg_notempty {match(".+" value("Restmsg"));}; filter f_restmsg_empty {not match(".+" value("Restmsg"));}; # all messages except from the auth and authpriv facilities filter f_auth { facility(auth); }; filter f_authpriv { not facility(auth, authpriv); }; #filter f_debug { level(notice) and not facility(auth, authpriv, news, mail); }; filter f_emergency { level(emerg); }; filter f_info { level(info); }; filter f_notice { level(notice); }; filter f_warn { level(warn); }; filter f_crit { level(crit); }; filter f_err { level(err); }; #filter f_syslog { not facility(authpriv, mail ); }; filter f_syslog { program(syslog-ng); }; filter f_cron { facility(cron); }; filter f_daemon { facility(daemon); }; filter f_kern { facility(kern); }; filter f_mail { facility(mail); }; filter f_user { facility(user); and level(error) }; filter f_uucp { facility(uucp); }; filter f_news { facility(news); }; filter f_messages { level(info..warn) and not facility(auth, authpriv, mail, news); }; filter f_all_info { not match("Info", value("type"), type("string")); }; filter f_all_generic{ not match("Generic", value("cname"), type("string")); }; parser p_db {db_parser( file("/etc/syslog-ng/rules/rules.xml") );}; log {source(s_all);parser(p_db);filter(f_restmsg_notempty);destination(d_log_notempty); }; #log { source(s_all); filter(f_authpriv); destination(authlog); }; log { source(s_all); filter(f_syslog); destination(_syslog); }; log { source(s_all); filter(f_cron); destination(cron); }; log { source(s_all); filter(f_daemon); destination(daemon); }; log { source(s_all); filter(f_kern); destination(kern); }; log { source(s_all); filter(f_mail); destination(mail); }; log { source(s_all); filter(f_user); destination(user); }; #log { source(s_all); filter(f_debug); destination(debug); }; ------------------------------------------------------------------------------------- rules.xml: <patterndb version='4' pub_date='2012-02-17'> <ruleset name='root' id='123456678'> <pattern>mylogger</pattern> <rules> <rule provider='test' id='182437592347598' class='system'> <patterns> <pattern>@ESTRING:date:,@ @ESTRING:cname:,@ @ESTRING:processId:,@ @ESTRING:type:,@ @ESTRING:source:,@ @ESTRING:line:,@ @ANYSTRING:Restmsg: @</pattern> </patterns> </rule> </rules> </ruleset> </patterndb> --------------------------------------------------------------------------------------------- Thanks, pattarsatish
participants (2)
-
Evan Rempel
-
satish pattar