[syslog-ng] strange logging behaviour

2 Feb 2006

      guys

I've setup a central syslog-ng based server on RHEL4 using version 1.6.6.

I have the central server set to log things to 2 destinations. Once to the
regular logfile and again to an archive file.
I open 2 terminals to it and tail -f both /var/log/messages and the
appropriate archive file.
Then I go to one of the clients and do : '/usr/bin/logger test' and watch.

What is weird is this:

/var/log/messages shows 'test' right away but the archive log takes about a
minute to show it.

Then on the client I do: /usr/bin/logger test2

now it doesn't show up on either log

then on the client i do: /usr/bin/logger test3

and all of a sudden all 3 testlogs show up in both logs.

This is not al all consistens though. I can repeat the test and this time
it's the arcive log that shows the log first
and /var/log/messages takes sweet time, or both take time. I understand that
it may take a little to send the stuff over the net
but once it has reached the logserver why isn't it sent to both destinations
at the same time ?

INTERESTING NOTE: If I run the stresstest.sh script that comes with the
syslog-ng source on the client I ALWAYS see ALL logs immediately in both
destinations on the log server. I had noticed before that sending a couple
of logs seems to help. As if syslog-ng decides to send logs in bulk so if
there
is only one tiny log it appears to wait till there are more logs  - then it
catches up. However, sometimes even a small log show up right away. Again -
i can't see consistency. Here is my log server syslog-ng.conf.

###############################################################
# First, set some global options.

options { use_fqdn(no);
          use_dns(yes);
          dns_cache(yes);
          keep_hostname(yes);
          long_hostnames(off);
          sync(1);
          stats(0);
          create_dirs(yes);
          perm(0640);
          dir_perm(0750);
        };

source src { file("/proc/kmsg");
             unix-stream("/dev/log");
             internal();
             udp (ip("10.152.1.67"));
             tcp(ip("0.0.0.0")
                 port(5000)
                 max-connections(1000)
                 keep-alive(yes));
           };

destination authlog { file("/var/log/auth"); };
destination cron { file("/var/log/cron"); };
destination uucp { file("/var/log/uucp"); };
destination mail { file("/var/log/mail"); };
destination messages { file("/var/log/messages"); };
destination console { usertty("root"); };
destination console_all { file("/dev/tty8"); };
destination console_file { file("/var/log/console"); };
destination arch {
file("/usr/local/var/log_archive/$HOST/$YEAR/$MONTH/$DAY/archive"); };

filter f_auth { facility(auth, authpriv); };
filter f_cron { facility(cron); };
filter f_mail { facility(mail); };
filter f_uucp { facility(cron); };
filter f_messages { level(info .. warn) and
                    not facility(auth, authpriv, cron, mail, news); };
filter f_emergency { level(emerg); };
filter f_notcron { not match("cron.minute)"); };
filter f_auth_junk { not ( match("Accepted publickey for nagios from
10.152.48.42") or
                           match("Accepted publickey for nagios from
127.0.0.1") or
                           match("Accepted publickey for oracle from
********") or
                           match("Accepted publickey for oracle from
********") or
                           match("Accepted publickey for stucky from
********") or
                           match("Accepted publickey for cfengine from
**********") or

match("COMMAND=/usr/local/nagios/home/check_duplex") or
                           match("reconnecting to LDAP server.+sleeping") or
                           match("session opened for user nagios") or
                           match("session closed for user nagios") or
                           match("session opened for user root"));
                };
filter f_junk { not ( match("]: AF_INET client connected from")); };

log { source(src); filter(f_auth); filter(f_auth_junk);
destination(authlog); };
log { source(src); filter(f_cron); filter(f_notcron); destination(cron); };
log { source(src); filter(f_mail); destination(mail); };
log { source(src); filter(f_uucp); destination(uucp); };
log { source(src); filter(f_messages); filter(f_junk);
destination(messages); };
log { source(src); filter(f_emergency); destination(console); };
log { source(src); filter(f_emergency); destination(console_file); };
log { source(src); filter(f_auth_junk); filter(f_notcron);
destination(arch); };

and here my syslog-ng.conf from the client:

###############################################################
# First, set some global options.

options { use_fqdn(no);
          use_dns(yes);
          dns_cache(yes);
          keep_hostname(yes);
          long_hostnames(off);
          sync(1);
          stats(0);
          create_dirs(yes);
        };

source src { file("/proc/kmsg");
             unix-stream("/dev/log");
             internal();
           };

destination authlog { file("/var/log/auth"); };
destination cron { file("/var/log/cron"); };
destination kern { file("/var/log/kernel"); };
destination uucp { file("/var/log/uucp"); };
destination mail { file("/var/log/mail"); };
destination messages { file("/var/log/messages"); };
destination console { usertty("root"); };
destination console_all { file("/dev/tty8"); };
destination console_file { file("/var/log/console"); };
destination loghost { tcp("logger" port(5000)); };

filter f_authpriv { facility(auth, authpriv); };
filter f_syslog { not facility(auth, authpriv) and not facility(mail); };
filter f_cron { facility(cron); };
filter f_daemon { facility(daemon); };
filter f_kern { facility(kern); };
filter f_mail { facility(mail); };
filter f_user { facility(user); };
filter f_uucp { facility(cron); };
filter f_messages { level(info .. warn) and not
                    facility(auth, authpriv, cron, daemon, mail); };
filter f_emergency { level(emerg); };

filter f_info { level(info); };
filter f_notice { level(notice); };
filter f_warn { level(warn); };
filter f_crit { level(crit); };
filter f_err { level(err); };

filter f_cnews { level(notice, err, crit) and facility(news); };
filter f_cother { level(debug, info, notice, warn) or facility(daemon,
mail); };

filter f_nagios { not ( match("Accepted publickey for nagios from
*********") or
                        match("COMMAND=/usr/local/nagios/home/check_duplex")
or
                        match("session opened for user nagios") or
                        match("session closed for user nagios"));

log { source(src); filter(f_authpriv); filter(f_nagios);
destination(authlog); };
log { source(src); filter(f_cron); destination(cron); };
log { source(src); filter(f_daemon); destination(messages); };
log { source(src); filter(f_kern); destination(kern); };
log { source(src); filter(f_kern); destination(messages); };
log { source(src); filter(f_mail); destination(mail); };
log { source(src); filter(f_user); destination(messages); };
log { source(src); filter(f_uucp); destination(uucp); };
log { source(src); filter(f_messages); destination(messages); };
log { source(src); filter(f_emergency); destination(console); };
log { source(src); filter(f_cnews); destination(console_all); };
log { source(src); filter(f_cother); destination(console_all); };
log { source(src); filter(f_emergency); destination(console_file); };
log { source(src); filter(f_cnews); destination(console_file); };
log { source(src); filter(f_cother); destination(console_file); };
log { source(src); filter(f_nagios); destination(loghost); };

This may also be a good opportunity to have someone check whether my configs
are sane at all. I'm not 100% sure about them.
I don't really want to log to all kinds of different files, rather I want to
have mail stuff go to mail, cron stuff to cron, auth stuff to auth
and the rest to messages.

help would be appreciated.
--
stucky