- syslog-ng - lists.balabit.hu

syslog-ng-crypto.so is versioned again
by Peter Czanik 20 Aug '11

20 Aug '11

Hello, This was once fixed in http://git.balabit.hu/?p=bazsi/syslog-ng-3.3.git;a=commitdiff;h=dbb63b6f2d8… but now I see it again. Verified both on FreeBSD 8.1 and openSUSE factory, versioned libraries are installed under lib/syslog/ Caught, as usual, by the openSUSE post build checks... Bye, -- Peter Czanik (CzP) <czanik(a)balabit.hu> BalaBit IT Security / syslog-ng upstream http://czanik.blogs.balabit.com/

2 1

[PATCH] affile: Implement size based filename rotation.
by Gergely Nagy 20 Aug '11

20 Aug '11

Based on a patch made by Sergei Zhirikov for syslog-ng 3.1, this patch implements a new size-limit() option for file based destinations. The syntax is as follows: destination d_messages { file("/var/log/messages" size-limit(104857600)); }; destination d_syslog { file("/var/log/syslog" size-limit(10485760 "/var/log/syslog.${SEQNUM}")); }; That is, size-limit() takes a file size limit (in bytes), and optionally a filename template to rotate to. If no filename template is specified, it will use the template used for the file itself, and append "-${UNIXTIME}.${SEQNUM}". The file is rotated after it reaches the specified size limit, therefore, it will usually be a message bigger than requested. And while rotating, if the filename to rotate to already exists, it will be overwritten. It can also be configured globally (only the limit, not the template) using the file-size-limit() global option. Signed-off-by: Sergei Zhirikov <sfzhi(a)yahoo.com> Signed-off-by: Gergely Nagy <algernon(a)balabit.hu> --- lib/cfg-grammar.y | 2 + lib/cfg-parser.c | 1 + lib/cfg.c | 1 + lib/cfg.h | 1 + lib/logproto.c | 15 +++- lib/logproto.h | 14 ++++- modules/affile/affile-grammar.ym | 3 + modules/affile/affile-parser.c | 1 + modules/affile/affile.c | 137 ++++++++++++++++++++++++++++++++++---- modules/affile/affile.h | 3 + 10 files changed, 160 insertions(+), 18 deletions(-) diff --git a/lib/cfg-grammar.y b/lib/cfg-grammar.y index 4748de4..ab4fd83 100644 --- a/lib/cfg-grammar.y +++ b/lib/cfg-grammar.y @@ -213,6 +213,7 @@ %token KW_OWNER 10250 %token KW_GROUP 10251 %token KW_PERM 10252 +%token KW_FILE_SIZE_LIMIT 10253 %token KW_DIR_OWNER 10260 %token KW_DIR_GROUP 10261 @@ -653,6 +654,7 @@ options_item | KW_RECV_TIME_ZONE '(' string ')' { configuration->recv_time_zone = g_strdup($3); free($3); } | KW_SEND_TIME_ZONE '(' string ')' { configuration->template_options.time_zone[LTZ_SEND] = g_strdup($3); free($3); } | KW_LOCAL_TIME_ZONE '(' string ')' { configuration->template_options.time_zone[LTZ_LOCAL] = g_strdup($3); free($3); } + | KW_FILE_SIZE_LIMIT '(' LL_NUMBER ')' { configuration->file_size_limit = $3; } ; /* START_RULES */ diff --git a/lib/cfg-parser.c b/lib/cfg-parser.c index 589206a..2763a4c 100644 --- a/lib/cfg-parser.c +++ b/lib/cfg-parser.c @@ -101,6 +101,7 @@ static CfgLexerKeyword main_keywords[] = { { "default_priority", KW_DEFAULT_LEVEL, 0x0300 }, { "default_facility", KW_DEFAULT_FACILITY, 0x0300 }, { "threaded", KW_THREADED, 0x0303 }, + { "file_size_limit", KW_FILE_SIZE_LIMIT, 0x0303 }, { "value", KW_VALUE, 0x0300 }, diff --git a/lib/cfg.c b/lib/cfg.c index 006f416..896eed3 100644 --- a/lib/cfg.c +++ b/lib/cfg.c @@ -339,6 +339,7 @@ cfg_new(gint version) self->file_uid = 0; self->file_gid = 0; self->file_perm = 0600; + self->file_size_limit = 0; self->dir_uid = 0; self->dir_gid = 0; self->dir_perm = 0700; diff --git a/lib/cfg.h b/lib/cfg.h index 3083a51..2b18316 100644 --- a/lib/cfg.h +++ b/lib/cfg.h @@ -88,6 +88,7 @@ struct _GlobalConfig gint file_uid; gint file_gid; gint file_perm; + off_t file_size_limit; gint dir_uid; gint dir_gid; diff --git a/lib/logproto.c b/lib/logproto.c index 92476ac..70286d5 100644 --- a/lib/logproto.c +++ b/lib/logproto.c @@ -241,6 +241,8 @@ typedef struct _LogProtoFileWriter gint buf_count; gint fd; gint sum_len; + LogProtoFileWriterCallback callback; + void *context; struct iovec buffer[0]; } LogProtoFileWriter; @@ -257,6 +259,7 @@ log_proto_file_writer_flush(LogProto *s) { LogProtoFileWriter *self = (LogProtoFileWriter *)s; gint rc, i, i0, sum, ofs; + off_t pos; /* we might be called from log_writer_deinit() without having a buffer at all */ @@ -265,8 +268,11 @@ log_proto_file_writer_flush(LogProto *s) /* lseek() is used instead of O_APPEND, as on NFS O_APPEND performs * poorly, as reported on the mailing list 2008/05/29 */ + pos = lseek(self->fd, 0, SEEK_END); + + if (self->callback) + self->callback(self->fd, pos, self->context); - lseek(self->fd, 0, SEEK_END); rc = writev(self->fd, self->buffer, self->buf_count); if (rc < 0) @@ -414,7 +420,8 @@ log_proto_file_writer_prepare(LogProto *s, gint *fd, GIOCondition *cond) } LogProto * -log_proto_file_writer_new(LogTransport *transport, gint flush_lines) +log_proto_file_writer_new_with_callback(LogTransport *transport, gint flush_lines, + LogProtoFileWriterCallback callback, void *context) { if (flush_lines == 0) /* the flush-lines option has not been specified, use a default value */ @@ -428,6 +435,8 @@ log_proto_file_writer_new(LogTransport *transport, gint flush_lines) self->fd = transport->fd; self->buf_size = flush_lines; + self->callback = callback; + self->context = context; self->super.prepare = log_proto_file_writer_prepare; self->super.post = log_proto_file_writer_post; self->super.flush = log_proto_file_writer_flush; @@ -436,8 +445,6 @@ log_proto_file_writer_new(LogTransport *transport, gint flush_lines) return &self->super; } - - typedef struct _LogProtoBufferedServerState { /* NOTE: that if you add/remove structure members you have to update diff --git a/lib/logproto.h b/lib/logproto.h index 443de16..a7cf200 100644 --- a/lib/logproto.h +++ b/lib/logproto.h @@ -36,6 +36,7 @@ typedef struct _LogProtoFileReader LogProtoFileReader; typedef enum { LPS_SUCCESS, + LPS_RETRY, LPS_ERROR, LPS_EOF, } LogProtoStatus; @@ -152,7 +153,18 @@ LogProto *log_proto_dgram_server_new(LogTransport *transport, gint max_msg_size, */ LogProto *log_proto_text_server_new(LogTransport *transport, gint max_msg_size, guint flags); -LogProto *log_proto_file_writer_new(LogTransport *transport, gint flush_lines); +/* LogProtoFileWriter + */ +typedef void (*LogProtoFileWriterCallback)(gint fd, off_t size, void *context); +LogProto *log_proto_file_writer_new_with_callback(LogTransport *transport, gint flush_lines, + LogProtoFileWriterCallback callback, + void *context); + +static inline LogProto * +log_proto_file_writer_new(LogTransport *transport, gint flush_lines) +{ + return log_proto_file_writer_new_with_callback(transport, flush_lines, NULL, NULL); +} /* * LogProtoTextClient diff --git a/modules/affile/affile-grammar.ym b/modules/affile/affile-grammar.ym index 5915e5f..34c8e9b 100644 --- a/modules/affile/affile-grammar.ym +++ b/modules/affile/affile-grammar.ym @@ -61,6 +61,7 @@ extern LogDriver *last_driver; %token KW_FSYNC %token KW_FOLLOW_FREQ %token KW_OVERWRITE_IF_OLDER +%token KW_SIZE_LIMIT %type <ptr> source_affile %type <ptr> source_affile_params @@ -164,6 +165,8 @@ dest_affile_option | KW_OVERWRITE_IF_OLDER '(' LL_NUMBER ')' { affile_dd_set_overwrite_if_older(last_driver, $3); } | KW_FSYNC '(' yesno ')' { affile_dd_set_fsync(last_driver, $3); } | KW_LOCAL_TIME_ZONE '(' string ')' { affile_dd_set_local_time_zone(last_driver, $3); free($3); } + | KW_SIZE_LIMIT '(' LL_NUMBER ')' { affile_dd_set_file_size_limit(last_driver, $3, NULL); } + | KW_SIZE_LIMIT '(' LL_NUMBER string ')'{ affile_dd_set_file_size_limit(last_driver, $3, $4); free($4); } ; dest_afpipe_params diff --git a/modules/affile/affile-parser.c b/modules/affile/affile-parser.c index 1e0bb5f..cf25e42 100644 --- a/modules/affile/affile-parser.c +++ b/modules/affile/affile-parser.c @@ -38,6 +38,7 @@ static CfgLexerKeyword affile_keywords[] = { { "remove_if_older", KW_OVERWRITE_IF_OLDER, 0, KWS_OBSOLETE, "overwrite_if_older" }, { "overwrite_if_older", KW_OVERWRITE_IF_OLDER }, { "follow_freq", KW_FOLLOW_FREQ, }, + { "size_limit", KW_SIZE_LIMIT }, { NULL } }; diff --git a/modules/affile/affile.c b/modules/affile/affile.c index 168cc1b..1e1403a 100644 --- a/modules/affile/affile.c +++ b/modules/affile/affile.c @@ -472,6 +472,12 @@ struct _AFFileDestWriter time_t last_open_stamp; time_t time_reopen; struct iv_timer reap_timer; + struct + { + LogMessage *current_msg; + GString *filename; + gint32 seq; + } cb_context; gboolean reopen_pending, queue_pending; }; @@ -514,9 +520,77 @@ affile_dw_reap(gpointer s) } static gboolean +affile_dw_open_file(AFFileDestWriter *self, int *fd) +{ + int flags; + + if (self->owner->flags & AFFILE_PIPE) + flags = O_RDWR | O_NOCTTY | O_NONBLOCK | O_LARGEFILE; + else + flags = O_WRONLY | O_CREAT | O_NOCTTY | O_NONBLOCK | O_LARGEFILE; + + return affile_open_file(self->filename, flags, + self->owner->file_uid, self->owner->file_gid, self->owner->file_perm, + self->owner->dir_uid, self->owner->dir_gid, self->owner->dir_perm, + !!(self->owner->flags & AFFILE_CREATE_DIRS), FALSE, !!(self->owner->flags & AFFILE_PIPE), fd); +} + +static void +affile_dw_writer_callback(gint fd, off_t size, void *context) +{ + AFFileDestWriter *self = (AFFileDestWriter *) context; + int reopen_fd; + + if (self->owner->size_limit <= 0) + return; + + if ((size > 0) && (size >= self->owner->size_limit)) + { + g_string_truncate(self->cb_context.filename, 0); + log_template_format(self->owner->size_fn_format, self->cb_context.current_msg, + NULL, LTZ_LOCAL, self->cb_context.seq, NULL, + self->cb_context.filename); + + if (rename(self->filename, self->cb_context.filename->str) == 0) + { + if (affile_dw_open_file(self, &reopen_fd)) + { + if (dup2(reopen_fd, fd) < 0) + { + msg_error("Error switching to new log file", + evt_tag_str("old_filename", self->filename), + evt_tag_str("new_filename", self->cb_context.filename->str), + evt_tag_errno(EVT_TAG_OSERROR, errno), + NULL); + } + else + self->cb_context.seq++; + close(reopen_fd); + } + else + { + msg_error("Error opening file for writing", + evt_tag_str("old_filename", self->filename), + evt_tag_str("new_filename", self->cb_context.filename->str), + evt_tag_errno(EVT_TAG_OSERROR, errno), + NULL); + } + } + else + { + msg_error("Error renaming overgrown file", + evt_tag_str("old_filename", self->filename), + evt_tag_str("new_filanem", self->cb_context.filename->str), + evt_tag_errno(EVT_TAG_OSERROR, errno), + NULL); + } + } +} + +static gboolean affile_dw_reopen(AFFileDestWriter *self) { - int fd, flags; + int fd; struct stat st; self->last_open_stamp = self->last_msg_stamp; @@ -531,16 +605,7 @@ affile_dw_reopen(AFFileDestWriter *self) unlink(self->filename); } - if (self->owner->flags & AFFILE_PIPE) - flags = O_RDWR | O_NOCTTY | O_NONBLOCK | O_LARGEFILE; - else - flags = O_WRONLY | O_CREAT | O_NOCTTY | O_NONBLOCK | O_LARGEFILE; - - - if (affile_open_file(self->filename, flags, - self->owner->file_uid, self->owner->file_gid, self->owner->file_perm, - self->owner->dir_uid, self->owner->dir_gid, self->owner->dir_perm, - !!(self->owner->flags & AFFILE_CREATE_DIRS), FALSE, !!(self->owner->flags & AFFILE_PIPE), &fd)) + if (affile_dw_open_file(self, &fd)) { guint write_flags; @@ -550,7 +615,9 @@ affile_dw_reopen(AFFileDestWriter *self) log_writer_reopen(self->writer, self->owner->flags & AFFILE_PIPE ? log_proto_text_client_new(log_transport_plain_new(fd, write_flags)) - : log_proto_file_writer_new(log_transport_plain_new(fd, write_flags), self->owner->writer_options.flush_lines)); + : log_proto_file_writer_new_with_callback(log_transport_plain_new(fd, write_flags), + self->owner->writer_options.flush_lines, + affile_dw_writer_callback, self)); main_loop_call((void * (*)(void *)) affile_dw_arm_reaper, self, TRUE); } @@ -635,6 +702,8 @@ affile_dw_queue(LogPipe *s, LogMessage *lm, const LogPathOptions *path_options, if (self->last_open_stamp == 0) self->last_open_stamp = self->last_msg_stamp; + self->cb_context.current_msg = lm; + if (!log_writer_opened((LogWriter *) self->writer) && !self->reopen_pending && (self->last_open_stamp < self->last_msg_stamp - self->time_reopen)) @@ -667,7 +736,8 @@ static void affile_dw_free(LogPipe *s) { AFFileDestWriter *self = (AFFileDestWriter *) s; - + + g_string_free(self->cb_context.filename, TRUE); log_pipe_unref(self->writer); self->writer = NULL; g_free(self->filename); @@ -697,6 +767,8 @@ affile_dw_new(AFFileDestDriver *owner, const gchar *filename) /* we have to take care about freeing filename later. This avoids a move of the filename. */ self->filename = g_strdup(filename); + self->cb_context.filename = g_string_sized_new(1024); + self->cb_context.seq = 1; g_static_mutex_init(&self->lock); return self; } @@ -737,6 +809,33 @@ affile_dd_set_file_perm(LogDriver *s, gint file_perm) self->file_perm = file_perm; } +void +affile_dd_set_file_size_limit(LogDriver *s, off_t file_size_limit, const gchar *template) +{ + AFFileDestDriver *self = (AFFileDestDriver *) s; + + self->size_limit = file_size_limit; + + if (!self->size_fn_format) + self->size_fn_format = log_template_new(configuration, NULL); + + if (template) + log_template_compile(self->size_fn_format, template, NULL); + else + { + if (self->filename_template) + { + gchar *fn = g_strdup_printf ("%s-${UNIXTIME}.${SEQNUM}", self->filename_template->template); + log_template_compile(self->size_fn_format, fn, NULL); + g_free (fn); + } + else + { + msg_error("Error setting size-rotated filename template", NULL); + } + } +} + void affile_dd_set_dir_uid(LogDriver *s, const gchar *dir_uid) { @@ -889,6 +988,16 @@ affile_dd_init(LogPipe *s) self->dir_perm = cfg->dir_perm; if (self->time_reap == -1) self->time_reap = cfg->time_reap; + if (self->size_limit == -1) + { + gchar *fn = g_strdup_printf("%s-${UNIXTIME}.${SEQNUM}", + self->filename_template->template); + + self->size_limit = cfg->file_size_limit; + self->size_fn_format = log_template_new(cfg, NULL); + log_template_compile(self->size_fn_format, fn, NULL); + g_free (fn); + } log_writer_options_init(&self->writer_options, cfg, 0); log_template_options_init(&self->template_fname_options, cfg); @@ -1147,6 +1256,7 @@ affile_dd_free(LogPipe *s) g_assert(self->single_writer == NULL && self->writer_hash == NULL); log_template_options_destroy(&self->template_fname_options); + log_template_unref(self->size_fn_format); log_template_unref(self->filename_template); log_writer_options_destroy(&self->writer_options); log_dest_driver_free(s); @@ -1175,6 +1285,7 @@ affile_dd_new(gchar *filename, guint32 flags) self->flags |= AFFILE_NO_EXPAND; } self->time_reap = -1; + self->size_limit = -1; log_template_options_defaults(&self->template_fname_options); g_static_mutex_init(&self->lock); return &self->super.super; diff --git a/modules/affile/affile.h b/modules/affile/affile.h index f418e0d..e961fed 100644 --- a/modules/affile/affile.h +++ b/modules/affile/affile.h @@ -75,6 +75,8 @@ typedef struct _AFFileDestDriver gint overwrite_if_older; gboolean use_time_recvd; gint time_reap; + off_t size_limit; + LogTemplate *size_fn_format; } AFFileDestDriver; LogDriver *affile_dd_new(gchar *filename, guint32 flags); @@ -91,5 +93,6 @@ void affile_dd_set_create_dirs(LogDriver *s, gboolean create_dirs); void affile_dd_set_fsync(LogDriver *s, gboolean enable); void affile_dd_set_overwrite_if_older(LogDriver *s, gint overwrite_if_older); void affile_dd_set_local_time_zone(LogDriver *s, const gchar *local_time_zone); +void affile_dd_set_file_size_limit(LogDriver *s, off_t file_size_limit, const gchar *template); #endif -- 1.7.0.4

3 3

snmptrapd to syslog-ng 3.1
by Smart, Dan 20 Aug '11

20 Aug '11

My interest is in network device syslog and traps. I'm trying to receive traps, and then process them in Simple Event Correlator (SEC). I've got SEC working fine with standard remote syslog. After reading everything I could find, I found a discussion from 2008 about losing the source hostname when sending the trap to syslog. I'm trying the source program method, and eliminating multi-line traps. As I understand that syslog-ng is looking for Standard Out from the program, I specified -f in snmptrapd to stop forking, and -Lo to send output to standard output. I'm getting nothing in my d_debug file. Any suggestions? There is also a web page with a filter and rewrite recipe for traps. Not sure why I need this if I am sending the trap directly to SEC. See https://lists.balabit.hu/pipermail/syslog-ng/2008-November/012200.html And http://bazsi.blogs.balabit.com/2008/11/syslog-ng-3-0-and-snmp-traps/ -=Dan=- ========= syslog-ng.conf ================= # options { long_hostnames(off); use_dns(yes); use_fqdn(no); keep_hostname(yes); owner("root"); group("adm"); perm(0640); stats_freq(0); bad_hostname("^gconfd$"); ts_format(iso); flush_lines(100); log_fetch_limit(100); log_fifo_size(2048); dir_perm(0755); }; source s_program { program("/usr/sbin/snmptrapd -a -f -Lo --disableAuthorization=yes", flags(no-multi-line)); }; destination d_debug { file("/var/log/syslog-ng-debug" owner(root) group(root) perm(0600) dir_perm(0700) create_dirs(yes)); }; destination d_sec { program("/usr/local/bin/sec -input=\"-\" -conf=/usr/local/etc/sec.conf" flags(no-multi-line) ); }; log { source(s_program); destination(d_sec); destination(d_debug); flags(flow-control); };

2 6

Store syslog occurrence frequency instead of adding all of them to the DB
by Marcos Tang 19 Aug '11

19 Aug '11

Hi, I am using Syslog-NG which stores the Syslog to a remote MySQL server for central logging. Now, I am running into a situation a lot of records inside the MySQL database is "almost" exactly the same, just except the timestamp is different. For example, I get "file system is full" every minute for a host and this message repeats itself periodically. In order to better manage the MySQL database, do you have any suggestions how I can reduce the number of overall records inside the DB, but add the occurrence frequency instead? For example, syslog message "file system is full" occurs every minutes on the system. There will be 24 x 60 "file system is full" records on my MySQL DB after 1 day. If no one fix it, I will get 10 x 24 x 60 "file system is full" records for 10 hosts having the same problem for 1 day. Can those records being "processed" some how and when I search the MySQL DB, I only see ONE record list the following only? Total occurrenceMessage content ===================== 14,400File system is full If I can do that, it will definitely reduce a log of space of my DB and speed up the query. It seems this is a database problem as well. Please let me know your idea at anytime. Regards, Marcos

5 7

openSUSE 3.3beta2 packages, nice article, poll
by Peter Czanik 19 Aug '11

19 Aug '11

Hello, I just sent out the syslog-ng newsletter a few days ago, but there are already fresh news. But first of all: If you got 1 minute, please fill our syslog-ng performance survey at https://www.surveymonkey.com/s/6ZQDVH6 There was a new release of syslog-ng OSE 3.3 last weekend, it's now at beta2. FreeBSD ports was the first to add support for it, and now here are the openSUSE packages for testing: http://download.opensuse.org/repositories/home:/czanik:/syslog-ng33/ or if more interested in package sources, then check https://build.opensuse.org/project/show?project=home%3Aczanik%3Asyslog-ng33 where these packages were built. Systemd support is being reworked in openSUSE, so please stick to sysvinit (which is the default) when testing these packages. There is one known problem affecting functionality: mongodb support is broken. It's already fixed in git, so it will be part of the next release, or if there is a request for it, I can add the patch to the beta2 package. I just found a nice introductory article about syslog-ng at Linux Journal, which describes the basics and also some more interesting topics and provides good explanations. You can read it at http://www.linuxjournal.com/content/creating-centralized-syslog-server It was written on Debian, but most information should apply to any Linux distribution. Bye, -- Peter Czanik (CzP) <czanik(a)balabit.hu> BalaBit IT Security / syslog-ng upstream http://czanik.blogs.balabit.com/

2 1

mongodb stats problem
by Peter Czanik 17 Aug '11

17 Aug '11

Hello, I just ran into this problem while checking Google Alerts about syslog-ng: http://www.quora.com/Error-starting-syslog-ng-with-mongo-module Is it addressed with the latest patch in git, or it's something new? Bye, -- Peter Czanik (CzP) <czanik(a)balabit.hu> BalaBit IT Security / syslog-ng upstream http://czanik.blogs.balabit.com/

2 1

syslog-ng Insider - August 2011
by Peter Czanik 17 Aug '11

17 Aug '11

Dear syslog-ng users, This is the 5th issue of the syslog-ng Insider, a monthly newsletter that brings you syslog-ng related news. Your feedback and news tips about the next issue is welcome at documentation(a)balabit.com <mailto:documentation@balabit.com> FEATURED NEWS Please test 3.3 beta2! -------------------- Version 3.3 is a major step in syslog-ng's development, as it is now fully multi-threaded, and is also the first release with considerable amount of community developed features. A new beta was released this weekend! It received many changes and fixes since the beta1 release. According to internal testing, it works nice and stable, but we already received some external problem reports. So your feedback is very valuable, especially if you could test it in real world situations with logs and configurations we could never imagine ourselves. As threading is a major new feature, which is not enabled by default, please try it by adding “threaded(yes)” to your options in syslog-ng.conf Please download sources from http://www.balabit.com/downloads/files?path=/syslog-ng/sources/3.3.0beta2 ! For a list of changes and fixes check the announcment at http://lists.balabit.hu/pipermail/syslog-ng-announce/2011-August/000117.html If you use FreeBSD, ports is already updated to beta2: http://www.freshports.org/sysutils/syslog-ng3-devel/ Documentation is also available: http://www.balabit.com/sites/default/files/documents/syslog-ng-ose-3.3-guid… Windows in focus ---------------- The syslog-ng application is often used in a Windows environment. There is now an ongoing blog series about syslog clients for Windows. BalaBit also released some patterns for Windows. The patterns are available from http://czanik.blogs.balabit.com/2011/07/patterns-for-windows-server-2008/ Blogs about syslog clients: http://czanik.blogs.balabit.com/2011/07/eventlog-to-syslog/ http://czanik.blogs.balabit.com/2011/07/snare/ http://czanik.blogs.balabit.com/2011/07/some-more-applications-to-forward-w… To be continued... Performance ----------- Both syslog-ng OSE and PE are now multi threaded, which brings performance to extreme. On the test machine HDD and multiple gigabit Ethernet lines were the limiting factor, not syslog-ng. The test were done using PE, but OSE performance should be similar: http://pzolee.blogs.balabit.com/2011/07/do-you-want-to-process-800-000-mess… BalaBit call the features and technologies aimed compliance at a high performance HSRL (High Spead Reliable Logging). POLL Please take a minute to answer three syslog-ng related questions at https://www.surveymonkey.com/s/6ZQDVH6 OTHER SHORT NEWS * A few more words on CVE-2011-1951: http://bazsi.blogs.balabit.com/2011/07/on-cve-2011-1951-bug-or-security-iss… * Logs as a movie: http://algernon.blogs.balabit.com/2011/06/twelve-days-of-log/ * ELSA updated: http://ossectools.blogspot.com/2011/07/elsa-vmware-appliance-available.html NEW RELEASES * syslog-ng OSE 3.3 beta2 was released: http://lists.balabit.hu/pipermail/syslog-ng-announce/2011-August/000117.html ARCHIVE http://insider.blogs.balabit.com/ -- Peter Czanik (CzP) <czanik(a)balabit.hu> BalaBit IT Security / syslog-ng upstream http://czanik.blogs.balabit.com/

1 0

internal() messages looping back on PostgreSQL restart
by Aaron 17 Aug '11

17 Aug '11

I have syslog-ng logging to a remote PostgreSQL database and to a local log file as well. If at any point the Postgres database restarts and syslog-ng happens to attempt to log to the database for the brief period that Postgres is unavailable, my local messages are flooded with internal() looping back messages similar to this: syslog-ng[31587]: syslog-ng internal() messages are looping back, preventing loop by suppressing further messages; recurse_count='3' syslog-ng[31587]: syslog-ng internal() messages are looping back, preventing loop by suppressing further messages; recurse_count='3' syslog-ng[31587]: syslog-ng internal() messages are looping back, preventing loop by suppressing further messages; recurse_count='2' My guess is that syslog-ng panics when it is unable to connect to the PostgreSQL database. The problem is that syslog-ng never re-establishes the connection, and writes several thousands of the above messages to the local log file each second, which can become a little problematic resource-wise. In order to make syslog-ng behave again, I need to manually restart the service, otherwise it would continue to flood my local messages and never reconnect to the database. I thought it might be a problem with my configuration, and I tried using the time_reopen() option but that seemed to have no effect. Is this perhaps a bug? Syslog-ng version: 3.2.4

2 1

[PATCH] afmongodb: Add stats locking.
by Gergely Nagy 17 Aug '11

17 Aug '11

Lock / unlock the stats when registering the counters. Reported-by: Costa Farber <costaf(a)wix.com> Signed-off-by: Gergely Nagy <algernon(a)balabit.hu> --- modules/afmongodb/afmongodb.c | 2 ++ 1 files changed, 2 insertions(+), 0 deletions(-) diff --git a/modules/afmongodb/afmongodb.c b/modules/afmongodb/afmongodb.c index f82f10e..9e155ed 100644 --- a/modules/afmongodb/afmongodb.c +++ b/modules/afmongodb/afmongodb.c @@ -439,12 +439,14 @@ afmongodb_dd_init(LogPipe *s) self->queue = log_dest_driver_acquire_queue(&self->super, afmongodb_dd_format_persist_name(self)); + stats_lock(); stats_register_counter(0, SCS_MONGODB | SCS_DESTINATION, self->super.super.id, afmongodb_dd_format_stats_instance(self), SC_TYPE_STORED, &self->stored_messages); stats_register_counter(0, SCS_MONGODB | SCS_DESTINATION, self->super.super.id, afmongodb_dd_format_stats_instance(self), SC_TYPE_DROPPED, &self->dropped_messages); + stats_unlock(); log_queue_set_counters(self->queue, self->stored_messages, self->dropped_messages); afmongodb_dd_start_thread(self); -- 1.7.0.4

2 1

Frontend viewing logfiles
by Volker Lieder 17 Aug '11

17 Aug '11

Hi there, we have installed syslog-ng-pe with file encryption. Is there a webfrontend where we can search through the encrypted logfiles? e.g i have to set the cert-key-file in a config and then i can search the logfiles via webfrontend. Any ideas? Regards, Volker Lieder -- Mit freundlichen Grüßen/Best regards Volker Lieder System Administrator mail volker(a)its-lieder.de ITS-Lieder Kleinlindenerstrasse 3 35398 Giessen / Germany http://www.its-lieder.de UST-ID: DE260836372 ========================================================== Jegliche Stellungnahmen und Meinungen dieser E-Mail sind alleine die des Autors und nicht notwendigerweise die der Firma. Falls erforderlich, können Sie eine gesonderte schriftliche Bestätigung anfordern. Any views or opinions presented in this email are solely those of the author and do not necessarily represent those of the company. If verification is required please request a hard-copy version.

3 4