- syslog-ng - lists.balabit.hu

Re: [syslog-ng] Filtering duplicate messages
by chris packham 09 Jun '08

09 Jun '08

Updated patch - back to using suppress, fix potential segfault. Implement "Last message repeated N times" functionality. This behaviour can be enabled by adding the new "suppress(<num>)" option to an output configuration e.g. destination tologfile { file("/var/log/messages" template(t) suppress(30)); }; As a log message is added to the queue it is remembered as the last message seen. When a new message is added to the queue it is checked against the last message. If its contents are the same the message is dropped and a counter incremented. A message summary indicating the value of the last message counter and a snippet of the message will be inserted into the log queue if a new message that differs is seen or if the configurable timeout period expires. --- src/cfg-grammar.y | 3 +- src/cfg-lex.l | 3 +- src/logmsg.c | 19 ++++++++++ src/logmsg.h | 2 +- src/logwriter.c | 101 ++++++++++++++++++++++++++++++++++++++++++++++++++++- src/logwriter.h | 5 +++ src/messages.c | 4 +-- 7 files changed, 130 insertions(+), 7 deletions(-) diff --git a/src/cfg-grammar.y b/src/cfg-grammar.y index 39ff172..734daf1 100644 --- a/src/cfg-grammar.y +++ b/src/cfg-grammar.y @@ -62,7 +62,7 @@ gint last_addr_family = AF_INET; %token KW_USER KW_DOOR KW_SUN_STREAMS KW_PROGRAM /* option items */ -%token KW_FSYNC KW_MARK_FREQ KW_STATS_FREQ KW_FLUSH_LINES KW_FLUSH_TIMEOUT KW_LOG_MSG_SIZE KW_FILE_TEMPLATE KW_PROTO_TEMPLATE +%token KW_FSYNC KW_MARK_FREQ KW_STATS_FREQ KW_FLUSH_LINES KW_SUPPRESS KW_FLUSH_TIMEOUT KW_LOG_MSG_SIZE KW_FILE_TEMPLATE KW_PROTO_TEMPLATE %token KW_CHAIN_HOSTNAMES KW_NORMALIZE_HOSTNAMES KW_KEEP_HOSTNAME KW_CHECK_HOSTNAME KW_BAD_HOSTNAME %token KW_KEEP_TIMESTAMP @@ -671,6 +671,7 @@ dest_writer_option : KW_FLAGS '(' dest_writer_options_flags ')' { last_writer_options->options = $3; } | KW_LOG_FIFO_SIZE '(' NUMBER ')' { last_writer_options->fifo_size = $3; } | KW_FLUSH_LINES '(' NUMBER ')' { last_writer_options->flush_lines = $3; } + | KW_SUPPRESS '(' NUMBER ')' { last_writer_options->suppress= $3; } | KW_FLUSH_TIMEOUT '(' NUMBER ')' { last_writer_options->flush_timeout = $3; } | KW_TEMPLATE '(' string ')' { last_writer_options->template = cfg_lookup_template(configuration, $3); diff --git a/src/cfg-lex.l b/src/cfg-lex.l index 5c5f54f..0190278 100644 --- a/src/cfg-lex.l +++ b/src/cfg-lex.l @@ -6,7 +6,7 @@ /*************************************************************************** * - * Copyright (c) 1999 Balzs Scheidler + * Copyright (c) 1999 Balázs Scheidler * Copyright (c) 1999-2007 BalaBit IT Ltd. * * This program is free software; you can redistribute it and/or modify @@ -86,6 +86,7 @@ static struct keyword keywords[] = { { "flush_timeout", KW_FLUSH_TIMEOUT }, { "sync_freq", KW_FLUSH_LINES }, /* obsolete */ { "sync", KW_FLUSH_LINES }, /* obsolete */ + { "suppress", KW_SUPPRESS }, { "fsync", KW_FSYNC }, { "long_hostnames", KW_CHAIN_HOSTNAMES }, { "chain_hostnames", KW_CHAIN_HOSTNAMES }, diff --git a/src/logmsg.c b/src/logmsg.c index 05354b5..9219a01 100644 --- a/src/logmsg.c +++ b/src/logmsg.c @@ -653,6 +653,25 @@ log_msg_new(gchar *msg, gint length, GSockAddr *saddr, guint flags, regex_t *bad } /** + * log_msg_new_internal: + * @prio: message priority (LOG_*) + * @msg: message text + * @flags: parse flags (LP_*) + * + * This function creates a new log message for messages originating + * internally to syslog-ng + **/ +LogMessage * +log_msg_new_internal(gint prio, gchar *msg, guint flags) +{ + gchar *buf = g_strdup_printf("<%d> syslog-ng[%d]: %s\n", prio, getpid(), msg); + LogMessage *self = log_msg_new(buf, strlen(buf), NULL, flags, NULL); + g_free(buf); + + return self; +} + +/** * log_msg_new_mark: * * This function returns a new MARK message. MARK messages have the LF_MARK diff --git a/src/logmsg.h b/src/logmsg.h index 420a711..d780282 100644 --- a/src/logmsg.h +++ b/src/logmsg.h @@ -100,7 +100,7 @@ void log_msg_unref(LogMessage *m); LogMessage *log_msg_new(gchar *msg, gint length, GSockAddr *saddr, guint flags, regex_t *bad_hostname); LogMessage *log_msg_new_mark(void); - +LogMessage *log_msg_new_internal(gint prio, gchar *msg, guint flags); void log_msg_ack_block_inc(LogMessage *m); void log_msg_ack_block_start(LogMessage *m, LMAckFunc func, gpointer user_data); void log_msg_ack_block_end(LogMessage *m); diff --git a/src/logwriter.c b/src/logwriter.c index bb82b43..f831d93 100644 --- a/src/logwriter.c +++ b/src/logwriter.c @@ -179,10 +179,104 @@ log_writer_watch_new(LogWriter *writer, FDWrite *fd) } static void +log_writer_last_msg_release(LogWriter *self) +{ + if(self->last_msg_timerid) + g_source_remove(self->last_msg_timerid); + + if(self->last_msg) + log_msg_unref(self->last_msg); + + self->last_msg = NULL; + self->last_msg_count = 0; + self->last_msg_timerid = 0; +} + +static void +log_writer_last_msg_flush(LogWriter *self) +{ + gchar *msg; + LogMessage *m; + + msg = g_strdup_printf("Last message '%.20s' repeated %d times\n", + self->last_msg->msg.str, + self->last_msg_count); + + m = log_msg_new_internal(self->last_msg->pri, msg, 0); + + g_string_assign(&m->host,self->last_msg->host.str ); + g_string_assign(&m->host_from,self->last_msg->host_from.str ); + g_queue_push_tail(self->queue, m); + g_queue_push_tail(self->queue, GUINT_TO_POINTER(0x80000000)); + g_free(msg); + + log_writer_last_msg_release(self); +} + +static gboolean +last_msg_timer(gpointer pt) +{ + LogWriter *self = (LogWriter *)pt; + + if(self->last_msg_count) + log_writer_last_msg_flush(self); + else + log_writer_last_msg_release(self); + + return FALSE; +} + +static void +log_writer_last_msg_record(LogWriter *self, LogMessage *lm) +{ + if(self->last_msg) + log_msg_unref(self->last_msg); + + log_msg_ref(lm); + self->last_msg = lm; + self->last_msg_count = 0; + self->last_msg_timerid = g_timeout_add(self->options->suppress * 1000, last_msg_timer, self); +} + +static gboolean +log_writer_last_msg_check(LogWriter *self, LogMessage *lm, gint path_flags) +{ + if (self->options->suppress <= 0) + return FALSE; + + if(self->last_msg) + { + if(strcmp(self->last_msg->msg.str, lm->msg.str) == 0 && + strcmp(self->last_msg->host.str, lm->host.str) == 0) + { + if (self->dropped_messages) + (*self->dropped_messages)++; + msg_debug("Dropping duplicate message", + NULL); + self->last_msg_count++; + log_msg_drop(lm, path_flags); + return TRUE; + } + + if (self->last_msg_count) + log_writer_last_msg_flush(self); + else + log_writer_last_msg_release(self); + } + + log_writer_last_msg_record(self, lm); + + return FALSE; +} + +static void log_writer_queue(LogPipe *s, LogMessage *lm, gint path_flags) { LogWriter *self = (LogWriter *) s; - + + if(log_writer_last_msg_check(self, lm, path_flags)) + return; + if ((self->queue->length / 2) == self->options->fifo_size) { /* drop incoming message, we must ack here, otherwise the sender might @@ -415,6 +509,7 @@ log_writer_free(LogPipe *s) log_msg_unref(lm); } g_queue_free(self->queue); + log_writer_last_msg_release(self); g_free(self); } @@ -462,6 +557,9 @@ log_writer_new(guint32 flags, LogPipe *control, LogWriterOptions *options) self->queue = g_queue_new(); self->flags = flags; self->control = control; + self->last_msg = NULL; + self->last_msg_count = 0; + self->last_msg_timerid = 0; return &self->super; } @@ -476,6 +574,7 @@ log_writer_options_defaults(LogWriterOptions *options) options->ts_format = -1; options->zone_offset = -1; options->frac_digits = -1; + options->suppress = 0; } void diff --git a/src/logwriter.h b/src/logwriter.h index 284899b..cf6074b 100644 --- a/src/logwriter.h +++ b/src/logwriter.h @@ -66,6 +66,7 @@ typedef struct _LogWriterOptions gshort ts_format; glong zone_offset; gshort frac_digits; + gint suppress; } LogWriterOptions; typedef struct _LogWriter @@ -79,6 +80,10 @@ typedef struct _LogWriter gint partial_pos; LogPipe *control; LogWriterOptions *options; + LogMessage *last_msg; + guint32 last_msg_count; + guint last_msg_timerid; + } LogWriter; void log_writer_set_options(LogWriter *self, LogPipe *control, LogWriterOptions *options); diff --git a/src/messages.c b/src/messages.c index 6c69f48..206206a 100644 --- a/src/messages.c +++ b/src/messages.c @@ -56,10 +56,8 @@ msg_send_internal_message(int prio, const char *msg) if (G_LIKELY(internal_msg_queue)) { - buf = g_strdup_printf("<%d> syslog-ng[%d]: %s\n", prio, getpid(), msg); - m = log_msg_new(buf, strlen(buf), NULL, LP_INTERNAL | LP_LOCAL, NULL); + m = log_msg_new_internal(prio, msg, LP_INTERNAL | LP_LOCAL); g_queue_push_tail(internal_msg_queue, m); - g_free(buf); } } } -- 1.5.4.5

2 1

syslog_ng and NFS
by Chris Wagner 09 Jun '08

09 Jun '08

We run syslog-ng (both 1.x and now 2.0.9) on FreeBSD writing log files to a NetApp NFS server. At a very low load we start dropping inbound packets (udp, large socket buffers). A quick tcpdump showed a pathological interaction with NFS revolving around how NFS (at least on BSD) deals with APPEND mode. The upshot is that for each write, it flushes the attributes and actually re-reads the partial file block back from the server! A simple change to the source to remove O_APPEND had great results we can handle 10x the number of inbound messages. Our .conf file has every logging host going to a separate file, and with a quick scan of the code it sure looked like syslog-ng won¹t open the same file more than once anyway (I could be wrong here). So, a couple questions (I scanned the last few months of archives but didn¹t see any discussion about this) - what breaks if anything by turning off APPEND, is this interesting to others/how might I get this into future releases ( I can see this being a per-destination option). thanks. -- It is the mark of an instructed mind to rest satisfied with the degree of precision which the nature of a subject admits, and not to seek exactness when only an approximation of the truth is possible. Aristotle

2 3

Re: [syslog-ng] Collecting logs from a chrooted appli
by Gault Stephane 06 Jun '08

06 Jun '08

Hi, > Well, /dev/log shouldn't be a directory. It should be a socket which programs could use by calling openlog/sendlog libc functions. syslog-ng will create the socket for you, you shouldn't do it yourself. I setup syslog-ng.conf with this setings to let create the stream: source src_chroot_appli { unix-stream("/var/appli/dev/log"); }; It works and create the socket. then i config the application log stream path to /dev/log (as the application is chrooted, the root directory is already /var/appli, so logging to /dev/log is ok ) then the configuration to log it in the appli.log file in the /var/log/ outside the chroot : destination appli_log { file("/var/log/appli.log"); }; then log { source(src_chroot_appli); destination(appli_log); }; But nothing is created in the /var/log directory (outside chroot). Btw: syslog-ng version is 1.6.9 Best regards, S.Gautch _________________________________________________________________ Caroline vient de mettre à jour son profil Messenger ! Connectez-vous ! http://login.live.com/login.srf?wa=wsignin1.0&rpsnv=10&ct=1198837564&rver=4…

2 4

Collecting logs from a chrooted appli
by Gault Stephane 05 Jun '08

05 Jun '08

Hi, i'm curretly trying to get the logs of a chrooted application wich is under (/var/appli/) and logging in a file called appli.log (/var/appli/dev/log/appli.log). I tryed to create a /dev/log directory (and a /dev/log stream too, when the file didn't work) where the appli is logging. The application logging is ok, the file or stream work, but syslog-ng can't duplicate and create the appli.log in /var/log/appli.log or send tcp to the remote syslog server as well. In the syslog-ng.conf i Tryed to add a local source : source src_applilog { file("/var/appli/dev/log/appli.log"); }; (or for the stream) source src_applilog { unix-stream("/dev/log"); }; then duplicating the log : destination appli { file("/var/log/appli.log"); }; then statement : log { source(src_applilog); destination(appli); }; I don't want to filter anything in this log, but even if a tryed tu setup a unix-stream in /var/appli/dev/log with the application logging in this stream, nothing get the job done. I must duplicate the log outside the chroot (local then remote with a tcp syslog server listening). Any clue of what am i doing wrong ? Best regards. S.gautch _________________________________________________________________ Votre contact a choisi Hotmail, l'e-mail ultra sécurisé. Créez un compte gratuitement ! http://www.windowslive.fr/hotmail/default.asp

2 1

How many host by relay?
by olivier "madmax"rolland 05 Jun '08

05 Jun '08

Hello, I would like to know how many hosts can be relayed by a syslog-ng relay with the 1.6.8 version of syslog-ng. For the moment I use a relay for 200 hosts but at least I will collect logs of 3000 hosts. How many relay do you think I will need? Thank for your help. __________________________________________________ Do You Yahoo!? En finir avec le spam? Yahoo! Mail vous offre la meilleure protection possible contre les messages non sollicités http://mail.yahoo.fr Yahoo! Mail

2 1

log route problem
by Marcin Orlowski 05 Jun '08

05 Jun '08

Hi, I setup client with mostly default settings. To log to remote server I added (remote server listens on 514 as by default): log { destination(d_remote); flags(catchall); }; this used to work pretty ok. So now I wanted to log xferlog too. Set the source: source s_ftp_xfer { file("/var/log/proftpd/xferlog" follow_freq(2) flags(no-parse)); }; destination d_remote_515 { tcp('x.x.x.x' port(515)); }; log { source(s_ftp_xfer); destination(d_remote_515); }; Port 515 is intentional here for other reasons. On the server: source s_xferlog {tcp( ip(0.0.0.0) port(515)); }; template t_ftp { template("$MSG\n"); template_escape(no); }; destination df_ftp_xfer { ("/<PATH>/xferlog" template(t_ftp)); }; filter f_proftpd { program('^proftpd$'); }; log { source(s_xferlog); filter( f_proftpd ); destination(df_ftp_xfer); flags(final); }; My problem is that everytime xferlog entry arrives to the server it's being added to xferlog as expected, but it also goes into syslog and console. I am stuck to find out why. There's no other rule that touches s_xferlog on the server. Any hints appreciated. Marcin

2 1

Linking syslog-ng statically error
by Andrej Lobovski 05 Jun '08

05 Jun '08

Hello. While dynamic linking works, for some reason I need static version. Well when I try it I get output like this: checking for a BSD-compatible install... /usr/bin/install -c checking whether build environment is sane... yes checking for gawk... no checking for mawk... mawk checking whether make sets $(MAKE)... yes checking for gcc... gcc checking for C compiler default output file name... a.out checking whether the C compiler works... yes checking whether we are cross compiling... no checking for suffix of executables... checking for suffix of object files... o checking whether we are using the GNU C compiler... yes checking whether gcc accepts -g... yes checking for gcc option to accept ISO C89... none needed checking for style of include used by make... GNU checking dependency style of gcc... gcc3 checking for gcc option to accept ANSI C... none needed checking for bison... bison -y checking for flex... flex checking lex output file root... lex.yy checking lex library... -lfl checking whether yytext is a pointer... yes checking whether make sets $(MAKE)... (cached) yes checking for ranlib... ranlib checking how to enable static linking for certain libraries... GNU or Solaris checking how to run the C preprocessor... gcc -E checking for grep that handles long lines and -e... /bin/grep checking for egrep... /bin/grep -E checking for ANSI C header files... yes checking for sys/types.h... yes checking for sys/stat.h... yes checking for stdlib.h... yes checking for string.h... yes checking for memory.h... yes checking for strings.h... yes checking for inttypes.h... yes checking for stdint.h... yes checking for unistd.h... yes checking dmalloc.h usability... no checking dmalloc.h presence... no checking for dmalloc.h... no checking for strings.h... (cached) yes checking openssl/ssl.h usability... no checking openssl/ssl.h presence... no checking for openssl/ssl.h... no checking getopt.h usability... yes checking getopt.h presence... yes checking for getopt.h... yes checking stropts.h usability... yes checking stropts.h presence... yes checking for stropts.h... yes checking sys/strlog.h usability... no checking sys/strlog.h presence... no checking for sys/strlog.h... no checking door.h usability... no checking door.h presence... no checking for door.h... no checking tcpd.h usability... no checking tcpd.h presence... no checking for tcpd.h... no checking whether struct tm is in sys/time.h or time.h... time.h checking for struct tm.tm_gmtoff... yes checking for I_CONSLOG... no checking for O_LARGEFILE... yes checking for struct sockaddr_storage... yes checking for struct sockaddr_in6... yes checking for door_create in -ldoor... no checking for socket in -lsocket... no checking for nanosleep in -lrt... yes checking for gethostbyname in -lnsl... yes checking for regexec in -lregex... no checking for res_init in -lresolv... no checking for strdup... yes checking for strtol... yes checking for strtoll... yes checking for strtoimax... yes checking for inet_aton... yes checking for inet_ntoa... yes checking for getopt_long... yes checking for getaddrinfo... yes checking for getutent... yes checking for TCP wrapper library... checking for LIBNET... yes checking whether to enable Sun STREAMS support... no checking whether to enable Sun door support... no checking whether to enable TCP wrapper support... no checking whether to enable spoof source support... yes checking whether to enable IPv6 support... yes checking for pkg-config... /usr/bin/pkg-config checking pkg-config is at least version 0.9.0... yes checking for GLIB... yes checking for EVTLOG... yes checking for static GLib libraries... yes checking sanity checking Glib headers... yes configure: creating ./config.status config.status: creating dist.conf config.status: creating Makefile config.status: creating syslog-ng.spec.bb config.status: creating src/Makefile config.status: creating doc/Makefile config.status: creating doc/docvars.xml config.status: creating contrib/Makefile config.status: creating tests/Makefile config.status: creating solbuild/Makefile config.status: creating tgzbuild/Makefile config.status: creating tests/unit/Makefile config.status: creating tests/functional/Makefile config.status: creating tests/loggen/Makefile config.status: creating config.h config.status: config.h is unchanged config.status: executing depfiles commands and the make command gives me macros.c: In function ÔÇśresult_appendÔÇÖ: macros.c:136: warning: pointer targets in passing argument 2 of ÔÇśg_string_append_lenÔÇÖ differ in signedness macros.c: In function ÔÇślog_macro_expandÔÇÖ: macros.c:148: warning: pointer targets in passing argument 2 of ÔÇśresult_appendÔÇÖ differ in signedness macros.c:212: warning: pointer targets in passing argument 2 of ÔÇśresult_appendÔÇÖ differ in signedness macros.c:232: warning: pointer targets in passing argument 2 of ÔÇśresult_appendÔÇÖ differ in signedness macros.c:240: warning: pointer targets in passing argument 2 of ÔÇśresult_appendÔÇÖ differ in signedness macros.c:258: warning: pointer targets in passing argument 2 of ÔÇśresult_appendÔÇÖ differ in signedness macros.c:403: warning: pointer targets in passing argument 2 of ÔÇśresult_appendÔÇÖ differ in signedness macros.c:422: warning: pointer targets in passing argument 2 of ÔÇśresult_appendÔÇÖ differ in signedness macros.c:440: warning: pointer targets in passing argument 2 of ÔÇśresult_appendÔÇÖ differ in signedness logmsg.c: In function ÔÇślog_msg_parseÔÇÖ: logmsg.c:151: warning: pointer targets in assignment differ in signedness logmsg.c:216: warning: pointer targets in passing argument 2 of ÔÇśg_string_assign_lenÔÇÖ differ in signedness logmsg.c:224: warning: pointer targets in assignment differ in signedness logmsg.c:242: warning: pointer targets in passing argument 1 of ÔÇśstrlenÔÇÖ differ in signedness logmsg.c:283: warning: pointer targets in passing argument 2 of ÔÇśg_string_assign_lenÔÇÖ differ in signedness logmsg.c:309: warning: pointer targets in passing argument 2 of ÔÇśg_string_assign_lenÔÇÖ differ in signedness logmsg.c:335: warning: pointer targets in passing argument 2 of ÔÇśg_string_assign_lenÔÇÖ differ in signedness logmsg.c:394: warning: pointer targets in assignment differ in signedness logmsg.c:398: warning: pointer targets in assignment differ in signedness logmsg.c:429: warning: pointer targets in assignment differ in signedness logmsg.c:463: warning: pointer targets in assignment differ in signedness logmsg.c:474: warning: pointer targets in assignment differ in signedness logmsg.c:486: warning: pointer targets in assignment differ in signedness logmsg.c:499: warning: pointer targets in assignment differ in signedness logmsg.c:520: warning: pointer targets in assignment differ in signedness logmsg.c:525: warning: pointer targets in assignment differ in signedness logmsg.c:530: warning: pointer targets in passing argument 2 of ÔÇśg_string_assign_lenÔÇÖ differ in signedness main.o: In function `setup_creds': /root/syslog-za-oddat/MYKGXX90/rpmbuild/SOURCES/syslog-ng-2.0.8/src/main.c:462: warning: Using 'initgroups' in statically linked applications requires at runtime the shared libraries from the glibc version used for linking libsyslog-ng.a(misc.o): In function `resolve_group': /root/syslog-za-oddat/MYKGXX90/rpmbuild/SOURCES/syslog-ng-2.0.8/src/misc.c:274: warning: Using 'getgrnam' in statically linked applications requires at runtime the shared libraries from the glibc version used for linking libsyslog-ng.a(misc.o): In function `resolve_user': /root/syslog-za-oddat/MYKGXX90/rpmbuild/SOURCES/syslog-ng-2.0.8/src/misc.c:254: warning: Using 'getpwnam' in statically linked applications requires at runtime the shared libraries from the glibc version used for linking /usr/lib/gcc/i486-linux-gnu/4.1.2/../../../../lib/libglib-2.0.a(gutils.o): In function `g_get_any_init_do': (.text+0x141e): warning: Using 'getpwuid' in statically linked applications requires at runtime the shared libraries from the glibc version used for linking /usr/lib/gcc/i486-linux-gnu/4.1.2/../../../../lib/libglib-2.0.a(gutils.o): In function `g_get_any_init_do': (.text+0x1411): warning: Using 'setpwent' in statically linked applications requires at runtime the shared libraries from the glibc version used for linking /usr/lib/gcc/i486-linux-gnu/4.1.2/../../../../lib/libglib-2.0.a(gutils.o): In function `g_get_any_init_do': (.text+0x1426): warning: Using 'endpwent' in statically linked applications requires at runtime the shared libraries from the glibc version used for linking /usr/lib/gcc/i486-linux-gnu/4.1.2/../../../../lib/libglib-2.0.a(gutils.o): In function `g_get_any_init_do': (.text+0x115a): warning: Using 'getpwnam_r' in statically linked applications requires at runtime the shared libraries from the glibc version used for linking /usr/lib/gcc/i486-linux-gnu/4.1.2/../../../../lib/libglib-2.0.a(gutils.o): In function `g_get_any_init_do': (.text+0x1198): warning: Using 'getpwuid_r' in statically linked applications requires at runtime the shared libraries from the glibc version used for linking libsyslog-ng.a(afinet.o): In function `afinet_resolve_name': /root/syslog-za-oddat/MYKGXX90/rpmbuild/SOURCES/syslog-ng-2.0.8/src/afinet.c:103: warning: Using 'getaddrinfo' in statically linked applications requires at runtime the shared libraries from the glibc version used for linking libsyslog-ng.a(misc.o): In function `resolve_hostname': /root/syslog-za-oddat/MYKGXX90/rpmbuild/SOURCES/syslog-ng-2.0.8/src/misc.c:165: warning: Using 'gethostbyaddr' in statically linked applications requires at runtime the shared libraries from the glibc version used for linking libsyslog-ng.a(misc.o): In function `getlonghostname': /root/syslog-za-oddat/MYKGXX90/rpmbuild/SOURCES/syslog-ng-2.0.8/src/misc.c:70: warning: Using 'gethostbyname' in statically linked applications requires at runtime the shared libraries from the glibc version used for linking /usr/lib/gcc/i486-linux-gnu/4.1.2/../../../../lib/libnet.a(libnet_resolve.o): In function `libnet_name2addr6': (.text+0x453): warning: Using 'gethostbyname2' in statically linked applications requires at runtime the shared libraries from the glibc version used for linking libsyslog-ng.a(afinet.o): In function `afinet_set_port': /root/syslog-za-oddat/MYKGXX90/rpmbuild/SOURCES/syslog-ng-2.0.8/src/afinet.c:58: warning: Using 'getservbyname' in statically linked applications requires at runtime the shared libraries from the glibc version used for linking /usr/lib/gcc/i486-linux-gnu/4.1.2/../../../../lib/libglib-2.0.a(gtimer.o): In function `g_timer_elapsed': (.text+0x220): undefined reference to `clock_gettime' /usr/lib/gcc/i486-linux-gnu/4.1.2/../../../../lib/libglib-2.0.a(gtimer.o): In function `g_timer_continue': (.text+0x2a4): undefined reference to `clock_gettime' /usr/lib/gcc/i486-linux-gnu/4.1.2/../../../../lib/libglib-2.0.a(gtimer.o): In function `g_timer_reset': (.text+0x34a): undefined reference to `clock_gettime' /usr/lib/gcc/i486-linux-gnu/4.1.2/../../../../lib/libglib-2.0.a(gtimer.o): In function `g_timer_stop': (.text+0x391): undefined reference to `clock_gettime' /usr/lib/gcc/i486-linux-gnu/4.1.2/../../../../lib/libglib-2.0.a(gtimer.o): In function `g_timer_start': (.text+0x3de): undefined reference to `clock_gettime' /usr/lib/gcc/i486-linux-gnu/4.1.2/../../../../lib/libglib-2.0.a(gtimer.o):(.text+0x49f): more undefined references to `clock_gettime' follow collect2: ld returned 1 exit status make[2]: *** [syslog-ng] Error 1 make[1]: *** [all-recursive] Error 1 make: *** [all] Error 2 I googled for this problem and found nothing. :( Please help! _________________________________________________________________ Connect to the next generation of MSN Messenger http://imagine-msn.com/messenger/launch80/default.aspx?locale=en-us&source=…

2 1

Flow-control behaviour and Cached lines timestamping, plus a suspend error in 2.1beta1
by Behal, Pavel 05 Jun '08

05 Jun '08

Dear Bazsi, I have found two "features" and one bug I would like to discuss a little: 1) We use the syslog-ng 2.1alpha1 as a root system syslog. The problem is, we have had two destinations setup. First destination was the traditional write of system messages to the file like "/var/log/messages" etc. The second destination path was to the TCP stream to the central syslog server. And the problem is, we have used the flow-control flag on the central TCP path. But, when the central TCP syslog-ng becomes unavailable, after all the buffers filled up, the local system syslog-ng stopped processing any system messages. It did not write anything through the local file destinations until the TCP communication was re-established. Is this a correct behaviour? We have found, that to allow smooth processing of local system sources, we can not use flow-control flag on any destination paths it used.. 2) When messages are in syslog-ng buffer, they do not contain any timestamp. We have realized, that the timestamp is added at the time the syslog-ng flushes all the buffers to the disk. This is not very good in case you need to debug the messages ages and all of them have same and non-related timestamp. Is it possible to modify the logic of syslog-ng to timestamp the messages as soon as possible - even before the buffer, at the collection time? We have found it important for log integrity reasons. 3) Directly in the 2.1beta1 there has been introduced some error with the "suspend writing a destination file when an I/O error ..." patch. During our tests we have found it continously thinks the destination is unavailable and writes strange messages into to the internal log: May 29 15:00:02 europa syslog-ng[16798]: I/O error occurred while writing; fd='7', error='Connection timed out (110)' May 29 15:00:02 europa syslog-ng[16798]: Connection broken; time_reopen='60' May 29 15:00:02 europa syslog-ng[16798]: Suspending write operation because of an I/O error; fd='7', time_reopen='60' the lsof -p 16798: COMMAND PID USER FD TYPE DEVICE SIZE NODE NAME syslog-ng 16798 root cwd DIR 253,0 4096 2 / syslog-ng 16798 root rtd DIR 253,0 4096 2 / syslog-ng 16798 root txt REG 253,0 114584 671841 /sbin/syslog-ng syslog-ng 16798 root mem REG 253,0 106397 820372 /lib/ld-2.3.4.so syslog-ng 16798 root mem REG 253,0 1454546 820373 /lib/tls/libc-2.3.4.so syslog-ng 16798 root mem REG 253,0 28504 2919694 /usr/lib/libwrap.so.0.7.6 syslog-ng 16798 root mem REG 253,0 93985 820376 /lib/tls/libpthread-2.3.4.so syslog-ng 16798 root mem REG 253,0 47671 820386 /lib/tls/librt-2.3.4.so syslog-ng 16798 root mem REG 253,0 505200 2930064 /usr/lib/libglib-2.0.so.0.400.7 syslog-ng 16798 root mem REG 253,0 95148 820384 /lib/libnsl-2.3.4.so syslog-ng 16798 root mem REG 253,0 36639 2920956 /usr/local/lib/libevtlog.so.0.0.0 syslog-ng 16798 root 0r CHR 1,3 1619 /dev/null syslog-ng 16798 root 1w CHR 1,3 1619 /dev/null syslog-ng 16798 root 2w CHR 1,3 1619 /dev/null syslog-ng 16798 root 3r REG 0,3 0 4026531850 /proc/kmsg syslog-ng 16798 root 4u unix 0xc1be7ba0 351802 /dev/log syslog-ng 16798 root 5u IPv4 351804 TCP europa.training.hp.com:37205->europa.training.hp.com:5140 (ESTABLISHED) syslog-ng 16798 root 6w REG 253,0 4802 1655155 /var/log/messages syslog-ng 16798 root 8w REG 253,0 525 1655159 /var/log/boot.log This happens during standard system syslog processing on RH ELinux 4 and Solaris 9. You have the syslog-ng confs attached. Could you pleae check it? Best Regards, Pavel

3 3

Re: [syslog-ng] file() sourcing application logs
by Alan Edmonds 04 Jun '08

04 Jun '08

We are doing something similar; we use tomcat and apache. For apache, something like this (we are using 2.0): CustomLog "|/usr/bin/logger -p local1.info -t apache-access " access-format Then on syslog-ng I use the "tag" to point it to the right log file. For tomcat, that is tougher; stack dumps are a curse to syslog. There is a hook in the Catalina startup scripts to create a named pipe for gc.out and tomcat.out. The it spawns a logger -p local2.info < gc.out-named-pipe thingy to send to syslog-ng. I can send more details if you need them. Alan T-Mobile International UK Limited Company Registered Number: 3951860 Registered Office Address: Hatfield Business Park, Hatfield, Hertfordshire, AL10 9BW Registered in England and Wales NOTICE AND DISCLAIMER This email (including attachments) is confidential. If you are not the intended recipient, notify the sender immediately, delete this email from your system and do not disclose or use for any purpose

2 1

file() sourcing application logs
by Chris Pratt 04 Jun '08

04 Jun '08

Hi syslog-ng, Been all over google and still not found what I need so I'm hoping to get some help here... I have syslog-ng capturing and duplicating all of my *normal* /var/logs. I still need to capture some application specific logs that are not using syslog at all currently but dumping to text in application log directories. I wanted to use the file() source but I am not getting the results I wanted. I have tried both of these options to get these files logged: # See http://www.syslog.org/syslog-ng/v2/#id2535001 # tail -f [file log] | logger -p local#.info # # See http://www.balabit.com/dl/html/syslog-ng-admin-guide_en.html/ch09s01.html # source s_tail { file("/var/log/apache/access.log" follow_freq(1) flags(no-parse)); }; but I am running into two issues: 1.) Either above way of sourcing the files results in each line in my application log showing up as a separate syslog event even though it is a single application event that wraps lines in its own application log. For example: {Heap before GC invocations=1087 (full 0): par new generation total 307200K, used 219907K [0x5af10000, 0x73f10000, 0x73f10000) eden space 204800K, 100% used [0x5af10000, 0x67710000, 0x67710000) from space 102400K, 14% used [0x6db10000, 0x6e9d0d38, 0x73f10000) to space 102400K, 0% used [0x67710000, 0x67710000, 0x6db10000) concurrent mark-sweep generation total 991232K, used 116200K [0x73f10000, 0xb0710000, 0xb0710000) concurrent-mark-sweep perm gen total 71680K, used 46017K [0xb0710000, 0xb4d10000, 0xb4d10000) 27877.615: [GC 27877.615: [ParNew Desired survivor size 94371840 bytes, new threshold 15 (max 15) - age 1: 32802608 bytes, 32802608 total - age 2: 604752 bytes, 33407360 total - age 3: 40968 bytes, 33448328 total - age 4: 50728 bytes, 33499056 total - age 5: 54952 bytes, 33554008 total - age 6: 36096 bytes, 33590104 total - age 7: 47128 bytes, 33637232 total - age 8: 44048 bytes, 33681280 total - age 9: 40976 bytes, 33722256 total - age 10: 53912 bytes, 33776168 total - age 11: 63680 bytes, 33839848 total - age 12: 90944 bytes, 33930792 total - age 13: 33560 bytes, 33964352 total - age 14: 60552 bytes, 34024904 total - age 15: 59328 bytes, 34084232 total : 219907K->33808K(307200K), 0.0730690 secs] 336108K->150064K(1298432K), 0.0732270 secs] Heap after GC invocations=1088 (full 0): par new generation total 307200K, used 33808K [0x5af10000, 0x73f10000, 0x73f10000) eden space 204800K, 0% used [0x5af10000, 0x5af10000, 0x67710000) from space 102400K, 33% used [0x67710000, 0x69814218, 0x6db10000) to space 102400K, 0% used [0x6db10000, 0x6db10000, 0x73f10000) concurrent mark-sweep generation total 991232K, used 116256K [0x73f10000, 0xb0710000, 0xb0710000) concurrent-mark-sweep perm gen total 71680K, used 46017K [0xb0710000, 0xb4d10000, 0xb4d10000) } becomes something like... 2008-06-03T14:27:40-04:00 s_catalinag@tomcat {Heap before GC invocations=1087 (full 0): 2008-06-03T14:27:40-04:00 s_catalinag@tomcat par new generation total 307200K, used 219907K [0x5af10000, 0x73f10000, 0x73f10000) 2008-06-03T14:27:40-04:00 s_catalinag@tomcat eden space 204800K, 100% used [0x5af10000, 0x67710000, 0x67710000) 2008-06-03T14:27:40-04:00 s_catalinag@tomcat from space 102400K, 14% used [0x6db10000, 0x6e9d0d38, 0x73f10000) 2008-06-03T14:27:40-04:00 s_catalinag@tomcat to space 102400K, 0% used [0x67710000, 0x67710000, 0x6db10000) 2008-06-03T14:27:40-04:00 s_catalinag@tomcat concurrent mark-sweep generation total 991232K, used 116200K [0x73f10000, 0xb0710000, 0xb0710000) 2008-06-03T14:27:40-04:00 s_catalinag@tomcat concurrent-mark-sweep perm gen total 71680K, used 46017K [0xb0710000, 0xb4d10000, 0xb4d10000) 2008-06-03T14:27:40-04:00 s_catalinag@tomcat 27877.615: [GC 27877.615: [ParNew 2008-06-03T14:27:40-04:00 s_catalinag@tomcat Desired survivor size 94371840 bytes, new threshold 15 (max 15) 2008-06-03T14:27:40-04:00 s_catalinag@tomcat - age 1: 32802608 bytes, 32802608 total 2008-06-03T14:27:40-04:00 s_catalinag@tomcat - age 2: 604752 bytes, 33407360 total 2008-06-03T14:27:40-04:00 s_catalinag@tomcat - age 3: 40968 bytes, 33448328 total 2008-06-03T14:27:40-04:00 s_catalinag@tomcat - age 4: 50728 bytes, 33499056 total 2008-06-03T14:27:40-04:00 s_catalinag@tomcat - age 5: 54952 bytes, 33554008 total 2008-06-03T14:27:40-04:00 s_catalinag@tomcat - age 6: 36096 bytes, 33590104 total 2008-06-03T14:27:40-04:00 s_catalinag@tomcat - age 7: 47128 bytes, 33637232 total 2008-06-03T14:27:40-04:00 s_catalinag@tomcat - age 8: 44048 bytes, 33681280 total 2008-06-03T14:27:40-04:00 s_catalinag@tomcat - age 9: 40976 bytes, 33722256 total 2008-06-03T14:27:40-04:00 s_catalinag@tomcat - age 10: 53912 bytes, 33776168 total 2008-06-03T14:27:40-04:00 s_catalinag@tomcat - age 11: 63680 bytes, 33839848 total 2008-06-03T14:27:40-04:00 s_catalinag@tomcat - age 12: 90944 bytes, 33930792 total 2008-06-03T14:27:41-04:00 s_catalinag@tomcat - age 13: 33560 bytes, 33964352 total 2008-06-03T14:27:41-04:00 s_catalinag@tomcat - age 14: 60552 bytes, 34024904 total 2008-06-03T14:27:41-04:00 s_catalinag@tomcat - age 15: 59328 bytes, 34084232 total 2008-06-03T14:27:41-04:00 s_catalinag@tomcat : 219907K->33808K(307200K), 0.0730690 secs] 336108K->150064K(1298432K), 0.0732270 secs] 2008-06-03T14:27:41-04:00 s_catalinag@tomcat Heap after GC invocations=1088 (full 0): 2008-06-03T14:27:41-04:00 s_catalinag@tomcat par new generation total 307200K, used 33808K [0x5af10000, 0x73f10000, 0x73f10000) 2008-06-03T14:27:41-04:00 s_catalinag@tomcat eden space 204800K, 0% used [0x5af10000, 0x5af10000, 0x67710000) 2008-06-03T14:27:41-04:00 s_catalinag@tomcat from space 102400K, 33% used [0x67710000, 0x69814218, 0x6db10000) 2008-06-03T14:27:41-04:00 s_catalinag@tomcat to space 102400K, 0% used [0x6db10000, 0x6db10000, 0x73f10000) 2008-06-03T14:27:41-04:00 s_catalinag@tomcat concurrent mark-sweep generation total 991232K, used 116256K [0x73f10000, 0xb0710000, 0xb0710000) 2008-06-03T14:27:41-04:00 s_catalinag@tomcat concurrent-mark-sweep perm gen total 71680K, used 46017K [0xb0710000, 0xb4d10000, 0xb4d10000) 2008-06-03T14:27:41-04:00 s_catalinag@tomcat } 2.) I also have a couple of log files that use the date in their name but from what I can tell macros only work with destinations, not sources? I'd like to do something like: #source s_tomcat { file("/usr/local/server/apache-tomcat-5.5.20/logs/tomcat.${YEAR}-${MONTH}-${DAY}.log"); }; to accommodate this. Anyone have any similar app log setup and a solution to these issues? TIA, Chris

1 0