Re: [syslog-ng] Syslog-ng OSE 3.3.5 - High CPU usage

31 Jul 2012

      off the top of my head I would change

# all messages from the auth and authpriv facilities
filter f_restmsg_notempty {match(".+" value("Restmsg"));};
filter f_restmsg_empty {not match(".+" value("Restmsg"));};

to

# all messages from the auth and authpriv facilities
filter f_restmsg_notempty {match("^." value("Restmsg"));};
filter f_restmsg_empty {not match("^." value("Restmsg"));};

and that would save a lot of CPU. 

Evan.

________________________________________
From: syslog-ng-bounces@lists.balabit.hu [syslog-ng-bounces@lists.balabit.hu] On Behalf Of satish pattar [satish.pattar@gmail.com]
Sent: Tuesday, July 31, 2012 12:04 AM
To: syslog-ng@lists.balabit.hu
Subject: [syslog-ng] Syslog-ng OSE 3.3.5 - High CPU usage

Hi All,

I am using syslog-ng OSE & seeing a high CPU usage ? Could you please point out if this could be a config/rules issue or syslog-ng internal issue.

CPU usage in sorted order : 0.0  11.9  16.6 22.2 25.0 25.0 25.0 25.0 33.3 33.3 50.0
------------------------------------------
# syslog-ng --version
syslog-ng 3.3.5
Installer-Version: 3.3.5
Revision:
Compile-Date: Apr 20 2012 04:47:44
Default-Modules: afsocket,affile,afprog,afuser,basicfuncs,syslogformat,dbparser
Available-Modules: syslogformat,dbparser,afuser,basicfuncs,afmongodb,convertfuncs,afprog,dummy,afsocket,confgen,affile,csvparser
Enable-Debug: off
Enable-GProf: off
Enable-Memtrace: off
Enable-IPv6: on
Enable-Spoof-Source: off
Enable-TCP-Wrapper: off
Enable-Linux-Caps: off
Enable-Pcre: off
-----------
Here is the code snippet to send syslog messages to syslogger

for(i=0; i<500;i++)  {
   // this is the message format we are following
                          syslog(LOG_INFO,  "%s, %s, %d, %s, %s, %s %d, %s", "00:00:18","XYZ",getpid(),"Info","xyz.c", "Func1", 100, "WelCome to Syslog"  );
}

---------------------
# cat /proc/cpuinfo
Processor       : ARMv7 Processor rev 10 (v7l)
processor       : 0
BogoMIPS        : 1988.29

processor       : 1
BogoMIPS        : 1988.29

Features        : swp half thumb fastmult vfp edsp neon vfpv3
CPU implementer : 0x41
CPU architecture: 7
CPU variant     : 0x2
CPU part        : 0xc09
CPU revision    : 10

------------------------------

syslog-ng-confg:

@version: 3.3

######
# options

options {
        # disable the chained hostname format in logs
        # (default is enabled)
        chain_hostnames(0);

        # the time to wait before a died connection is re-established
        # (default is 60)
        time_reopen(10);

        # the time to wait before an idle destination file is closed
        # (default is 60)
        time_reap(360);

        # the number of lines buffered before written to file
        # you might want to increase this if your disk isn't catching with
        # all the log messages you get or if you want less disk activity
        flush_lines(5);
        flush_timeout(500);

        # the number of lines fitting in the output queue
        log_fifo_size(10000);

        # enable or disable directory creation for destination files
        create_dirs(yes);

        # default owner, group, and permissions for log files
        # (defaults are 0, 0, 0600)
        #owner(root);
        group(root);
        perm(0640);

        frac_digits(4);

        # Override the global timestamp format
        #(set in the global ts_format() parameter) for the specific destination.
        ts_format(iso);

        # default owner, group, and permissions for created directories
 # (defaults are 0, 0, 0700)
        #dir_owner(root);
        dir_perm(0755);

        # enable or disable DNS usage
        # syslog-ng blocks on DNS queries, so enabling DNS may lead to
        # a Denial of Service attack
        # (default is yes)
        use_dns(no);

        # maximum length of message in bytes
        # this is only limited by the program listening on the /dev/log Unix
        # socket, glibc can handle arbitrary length log messages, but -- for
        # example -- syslogd accepts only 1024 bytes
        # (default is 2048)
        #log_msg_size(2048);

        # print statistic information only once every hour (it is not really needed)
        stats_freq(3600);
};

######
# sources

# all known message sources
source s_all {
        # message generated by Syslog-NG
        internal();

        # standard Linux log source (this is the default place for the syslog()
        # function to send logs to)
        unix-dgram("/dev/log" program_override("mylogger: "));

        # messages from the kernel
        file("/proc/kmsg" program_override("kernel: "));
        # use the following line if you want to receive remote UDP logging messages
        # (this is equivalent to the "-r" syslogd flag)
        # udp();
};

########
#templates
#template t_default  { template("${ISODATE} ${HOST} ${MSGHDR}${MSG}\n"); template_escape(no); };
#template t_logtty   { template("${MSG}\n"); template_escape(no); };

template t_restmsg_notempty { template("${PROGRAM} ${date} ${cname}[${processId}] ${type} (${source}:${line}) ${Restmsg}\n"); };
template t_restmsg_empty    { template("${PROGRAM} ${FACILITY} ${LEVEL} ${MSG}\n"); };

######
# destinations

destination d_log_notempty { file("/var/my_log" template(t_restmsg_notempty)); };
#destination d_log_notempty { file("/dev/console" template(t_restmsg_notempty)); };
destination d_log_empty { file("/var/my_log" template(t_restmsg_empty)); };

destination authlog { file("/var/log/auth.log"); };
destination debug { file("/var/log/debug"); };
destination messages { file("/var/log/messages"); };
destination _syslog { file("/var/log/syslog"); };
destination cron { file("/var/log/cron.log"); };
destination daemon { file("/var/log/daemon.log"); };
destination lpr { file("/var/log/lpr.log"); };
destination user { file("/var/log/user.log"); };
destination uucp { file("/var/log/uucp.log"); };
destination mail { file("/var/log/mail.log"); };
destination kern { file("/var/log/kern.log"); };
destination mailinfo { file("/var/log/mail.info<http://mail.info>"); };
destination mailwarn { file("/var/log/mail.warn"); };
destination mailerr { file("/var/log/mail.err"); };
destination newscrit { file("/var/log/news/news.crit"); };
destination newserr { file("/var/log/news/news.err"); };
destination newsnotice { file("/var/log/news/news.notice"); };

######
# filters

# all messages from the auth and authpriv facilities
filter f_restmsg_notempty {match(".+" value("Restmsg"));};
filter f_restmsg_empty {not match(".+" value("Restmsg"));};

# all messages except from the auth and authpriv facilities

filter f_auth { facility(auth); };
filter f_authpriv { not facility(auth, authpriv); };
#filter f_debug { level(notice) and not facility(auth, authpriv, news, mail); };
filter f_emergency { level(emerg); };
filter f_info { level(info); };
filter f_notice { level(notice); };
filter f_warn { level(warn); };
filter f_crit { level(crit); };
filter f_err { level(err); };
#filter f_syslog { not facility(authpriv, mail ); };
filter f_syslog { program(syslog-ng); };
filter f_cron { facility(cron); };
filter f_daemon { facility(daemon); };
filter f_kern { facility(kern); };
filter f_mail { facility(mail); };
filter f_user { facility(user); and level(error) };
filter f_uucp { facility(uucp); };
filter f_news { facility(news); };
filter f_messages { level(info..warn) and not facility(auth, authpriv, mail, news); };

filter f_all_info {
                not match("Info", value("type"), type("string"));
                };

filter f_all_generic{ not match("Generic", value("cname"), type("string")); };

parser p_db {db_parser(
file("/etc/syslog-ng/rules/rules.xml")
);};

log {source(s_all);parser(p_db);filter(f_restmsg_notempty);destination(d_log_notempty); };
#log { source(s_all); filter(f_authpriv); destination(authlog); };
log { source(s_all); filter(f_syslog); destination(_syslog); };
log { source(s_all); filter(f_cron); destination(cron); };
log { source(s_all); filter(f_daemon); destination(daemon); };
log { source(s_all); filter(f_kern); destination(kern); };
log { source(s_all); filter(f_mail); destination(mail); };
log { source(s_all); filter(f_user); destination(user); };
#log { source(s_all); filter(f_debug); destination(debug); };
-------------------------------------------------------------------------------------

rules.xml:
<patterndb version='4' pub_date='2012-02-17'>
<ruleset name='root' id='123456678'>
<pattern>mylogger</pattern>
<rules>
<rule provider='test' id='182437592347598' class='system'>
<patterns>
<pattern>@ESTRING:date:,@ @ESTRING:cname:,@ @ESTRING:processId:,@ @ESTRING:type:,@ @ESTRING:source:,@ @ESTRING:line:,@ @ANYSTRING:Restmsg: @</pattern>
</patterns>
</rule>
</rules>
</ruleset>
</patterndb>
---------------------------------------------------------------------------------------------

Thanks,
pattarsatish

Re: [syslog-ng] Syslog-ng OSE 3.3.5 - High CPU usage

Evan Rempel