Based on a patch made by Sergei Zhirikov for syslog-ng 3.1, this patch implements a new size-limit() option for file based destinations. The syntax is as follows: destination d_messages { file("/var/log/messages" size-limit(104857600)); }; destination d_syslog { file("/var/log/syslog" size-limit(10485760 "/var/log/syslog.${SEQNUM}")); }; That is, size-limit() takes a file size limit (in bytes), and optionally a filename template to rotate to. If no filename template is specified, it will use the template used for the file itself, and append "-${UNIXTIME}.${SEQNUM}". The file is rotated after it reaches the specified size limit, therefore, it will usually be a message bigger than requested. And while rotating, if the filename to rotate to already exists, it will be overwritten. It can also be configured globally (only the limit, not the template) using the file-size-limit() global option. Signed-off-by: Sergei Zhirikov <sfzhi@yahoo.com> Signed-off-by: Gergely Nagy <algernon@balabit.hu> --- lib/cfg-grammar.y | 2 + lib/cfg-parser.c | 1 + lib/cfg.c | 1 + lib/cfg.h | 1 + lib/logproto.c | 15 +++- lib/logproto.h | 14 ++++- modules/affile/affile-grammar.ym | 3 + modules/affile/affile-parser.c | 1 + modules/affile/affile.c | 137 ++++++++++++++++++++++++++++++++++---- modules/affile/affile.h | 3 + 10 files changed, 160 insertions(+), 18 deletions(-) diff --git a/lib/cfg-grammar.y b/lib/cfg-grammar.y index 4748de4..ab4fd83 100644 --- a/lib/cfg-grammar.y +++ b/lib/cfg-grammar.y @@ -213,6 +213,7 @@ %token KW_OWNER 10250 %token KW_GROUP 10251 %token KW_PERM 10252 +%token KW_FILE_SIZE_LIMIT 10253 %token KW_DIR_OWNER 10260 %token KW_DIR_GROUP 10261 @@ -653,6 +654,7 @@ options_item | KW_RECV_TIME_ZONE '(' string ')' { configuration->recv_time_zone = g_strdup($3); free($3); } | KW_SEND_TIME_ZONE '(' string ')' { configuration->template_options.time_zone[LTZ_SEND] = g_strdup($3); free($3); } | KW_LOCAL_TIME_ZONE '(' string ')' { configuration->template_options.time_zone[LTZ_LOCAL] = g_strdup($3); free($3); } + | KW_FILE_SIZE_LIMIT '(' LL_NUMBER ')' { configuration->file_size_limit = $3; } ; /* START_RULES */ diff --git a/lib/cfg-parser.c b/lib/cfg-parser.c index 589206a..2763a4c 100644 --- a/lib/cfg-parser.c +++ b/lib/cfg-parser.c @@ -101,6 +101,7 @@ static CfgLexerKeyword main_keywords[] = { { "default_priority", KW_DEFAULT_LEVEL, 0x0300 }, { "default_facility", KW_DEFAULT_FACILITY, 0x0300 }, { "threaded", KW_THREADED, 0x0303 }, + { "file_size_limit", KW_FILE_SIZE_LIMIT, 0x0303 }, { "value", KW_VALUE, 0x0300 }, diff --git a/lib/cfg.c b/lib/cfg.c index 006f416..896eed3 100644 --- a/lib/cfg.c +++ b/lib/cfg.c @@ -339,6 +339,7 @@ cfg_new(gint version) self->file_uid = 0; self->file_gid = 0; self->file_perm = 0600; + self->file_size_limit = 0; self->dir_uid = 0; self->dir_gid = 0; self->dir_perm = 0700; diff --git a/lib/cfg.h b/lib/cfg.h index 3083a51..2b18316 100644 --- a/lib/cfg.h +++ b/lib/cfg.h @@ -88,6 +88,7 @@ struct _GlobalConfig gint file_uid; gint file_gid; gint file_perm; + off_t file_size_limit; gint dir_uid; gint dir_gid; diff --git a/lib/logproto.c b/lib/logproto.c index 92476ac..70286d5 100644 --- a/lib/logproto.c +++ b/lib/logproto.c @@ -241,6 +241,8 @@ typedef struct _LogProtoFileWriter gint buf_count; gint fd; gint sum_len; + LogProtoFileWriterCallback callback; + void *context; struct iovec buffer[0]; } LogProtoFileWriter; @@ -257,6 +259,7 @@ log_proto_file_writer_flush(LogProto *s) { LogProtoFileWriter *self = (LogProtoFileWriter *)s; gint rc, i, i0, sum, ofs; + off_t pos; /* we might be called from log_writer_deinit() without having a buffer at all */ @@ -265,8 +268,11 @@ log_proto_file_writer_flush(LogProto *s) /* lseek() is used instead of O_APPEND, as on NFS O_APPEND performs * poorly, as reported on the mailing list 2008/05/29 */ + pos = lseek(self->fd, 0, SEEK_END); + + if (self->callback) + self->callback(self->fd, pos, self->context); - lseek(self->fd, 0, SEEK_END); rc = writev(self->fd, self->buffer, self->buf_count); if (rc < 0) @@ -414,7 +420,8 @@ log_proto_file_writer_prepare(LogProto *s, gint *fd, GIOCondition *cond) } LogProto * -log_proto_file_writer_new(LogTransport *transport, gint flush_lines) +log_proto_file_writer_new_with_callback(LogTransport *transport, gint flush_lines, + LogProtoFileWriterCallback callback, void *context) { if (flush_lines == 0) /* the flush-lines option has not been specified, use a default value */ @@ -428,6 +435,8 @@ log_proto_file_writer_new(LogTransport *transport, gint flush_lines) self->fd = transport->fd; self->buf_size = flush_lines; + self->callback = callback; + self->context = context; self->super.prepare = log_proto_file_writer_prepare; self->super.post = log_proto_file_writer_post; self->super.flush = log_proto_file_writer_flush; @@ -436,8 +445,6 @@ log_proto_file_writer_new(LogTransport *transport, gint flush_lines) return &self->super; } - - typedef struct _LogProtoBufferedServerState { /* NOTE: that if you add/remove structure members you have to update diff --git a/lib/logproto.h b/lib/logproto.h index 443de16..a7cf200 100644 --- a/lib/logproto.h +++ b/lib/logproto.h @@ -36,6 +36,7 @@ typedef struct _LogProtoFileReader LogProtoFileReader; typedef enum { LPS_SUCCESS, + LPS_RETRY, LPS_ERROR, LPS_EOF, } LogProtoStatus; @@ -152,7 +153,18 @@ LogProto *log_proto_dgram_server_new(LogTransport *transport, gint max_msg_size, */ LogProto *log_proto_text_server_new(LogTransport *transport, gint max_msg_size, guint flags); -LogProto *log_proto_file_writer_new(LogTransport *transport, gint flush_lines); +/* LogProtoFileWriter + */ +typedef void (*LogProtoFileWriterCallback)(gint fd, off_t size, void *context); +LogProto *log_proto_file_writer_new_with_callback(LogTransport *transport, gint flush_lines, + LogProtoFileWriterCallback callback, + void *context); + +static inline LogProto * +log_proto_file_writer_new(LogTransport *transport, gint flush_lines) +{ + return log_proto_file_writer_new_with_callback(transport, flush_lines, NULL, NULL); +} /* * LogProtoTextClient diff --git a/modules/affile/affile-grammar.ym b/modules/affile/affile-grammar.ym index 5915e5f..34c8e9b 100644 --- a/modules/affile/affile-grammar.ym +++ b/modules/affile/affile-grammar.ym @@ -61,6 +61,7 @@ extern LogDriver *last_driver; %token KW_FSYNC %token KW_FOLLOW_FREQ %token KW_OVERWRITE_IF_OLDER +%token KW_SIZE_LIMIT %type <ptr> source_affile %type <ptr> source_affile_params @@ -164,6 +165,8 @@ dest_affile_option | KW_OVERWRITE_IF_OLDER '(' LL_NUMBER ')' { affile_dd_set_overwrite_if_older(last_driver, $3); } | KW_FSYNC '(' yesno ')' { affile_dd_set_fsync(last_driver, $3); } | KW_LOCAL_TIME_ZONE '(' string ')' { affile_dd_set_local_time_zone(last_driver, $3); free($3); } + | KW_SIZE_LIMIT '(' LL_NUMBER ')' { affile_dd_set_file_size_limit(last_driver, $3, NULL); } + | KW_SIZE_LIMIT '(' LL_NUMBER string ')'{ affile_dd_set_file_size_limit(last_driver, $3, $4); free($4); } ; dest_afpipe_params diff --git a/modules/affile/affile-parser.c b/modules/affile/affile-parser.c index 1e0bb5f..cf25e42 100644 --- a/modules/affile/affile-parser.c +++ b/modules/affile/affile-parser.c @@ -38,6 +38,7 @@ static CfgLexerKeyword affile_keywords[] = { { "remove_if_older", KW_OVERWRITE_IF_OLDER, 0, KWS_OBSOLETE, "overwrite_if_older" }, { "overwrite_if_older", KW_OVERWRITE_IF_OLDER }, { "follow_freq", KW_FOLLOW_FREQ, }, + { "size_limit", KW_SIZE_LIMIT }, { NULL } }; diff --git a/modules/affile/affile.c b/modules/affile/affile.c index 168cc1b..1e1403a 100644 --- a/modules/affile/affile.c +++ b/modules/affile/affile.c @@ -472,6 +472,12 @@ struct _AFFileDestWriter time_t last_open_stamp; time_t time_reopen; struct iv_timer reap_timer; + struct + { + LogMessage *current_msg; + GString *filename; + gint32 seq; + } cb_context; gboolean reopen_pending, queue_pending; }; @@ -514,9 +520,77 @@ affile_dw_reap(gpointer s) } static gboolean +affile_dw_open_file(AFFileDestWriter *self, int *fd) +{ + int flags; + + if (self->owner->flags & AFFILE_PIPE) + flags = O_RDWR | O_NOCTTY | O_NONBLOCK | O_LARGEFILE; + else + flags = O_WRONLY | O_CREAT | O_NOCTTY | O_NONBLOCK | O_LARGEFILE; + + return affile_open_file(self->filename, flags, + self->owner->file_uid, self->owner->file_gid, self->owner->file_perm, + self->owner->dir_uid, self->owner->dir_gid, self->owner->dir_perm, + !!(self->owner->flags & AFFILE_CREATE_DIRS), FALSE, !!(self->owner->flags & AFFILE_PIPE), fd); +} + +static void +affile_dw_writer_callback(gint fd, off_t size, void *context) +{ + AFFileDestWriter *self = (AFFileDestWriter *) context; + int reopen_fd; + + if (self->owner->size_limit <= 0) + return; + + if ((size > 0) && (size >= self->owner->size_limit)) + { + g_string_truncate(self->cb_context.filename, 0); + log_template_format(self->owner->size_fn_format, self->cb_context.current_msg, + NULL, LTZ_LOCAL, self->cb_context.seq, NULL, + self->cb_context.filename); + + if (rename(self->filename, self->cb_context.filename->str) == 0) + { + if (affile_dw_open_file(self, &reopen_fd)) + { + if (dup2(reopen_fd, fd) < 0) + { + msg_error("Error switching to new log file", + evt_tag_str("old_filename", self->filename), + evt_tag_str("new_filename", self->cb_context.filename->str), + evt_tag_errno(EVT_TAG_OSERROR, errno), + NULL); + } + else + self->cb_context.seq++; + close(reopen_fd); + } + else + { + msg_error("Error opening file for writing", + evt_tag_str("old_filename", self->filename), + evt_tag_str("new_filename", self->cb_context.filename->str), + evt_tag_errno(EVT_TAG_OSERROR, errno), + NULL); + } + } + else + { + msg_error("Error renaming overgrown file", + evt_tag_str("old_filename", self->filename), + evt_tag_str("new_filanem", self->cb_context.filename->str), + evt_tag_errno(EVT_TAG_OSERROR, errno), + NULL); + } + } +} + +static gboolean affile_dw_reopen(AFFileDestWriter *self) { - int fd, flags; + int fd; struct stat st; self->last_open_stamp = self->last_msg_stamp; @@ -531,16 +605,7 @@ affile_dw_reopen(AFFileDestWriter *self) unlink(self->filename); } - if (self->owner->flags & AFFILE_PIPE) - flags = O_RDWR | O_NOCTTY | O_NONBLOCK | O_LARGEFILE; - else - flags = O_WRONLY | O_CREAT | O_NOCTTY | O_NONBLOCK | O_LARGEFILE; - - - if (affile_open_file(self->filename, flags, - self->owner->file_uid, self->owner->file_gid, self->owner->file_perm, - self->owner->dir_uid, self->owner->dir_gid, self->owner->dir_perm, - !!(self->owner->flags & AFFILE_CREATE_DIRS), FALSE, !!(self->owner->flags & AFFILE_PIPE), &fd)) + if (affile_dw_open_file(self, &fd)) { guint write_flags; @@ -550,7 +615,9 @@ affile_dw_reopen(AFFileDestWriter *self) log_writer_reopen(self->writer, self->owner->flags & AFFILE_PIPE ? log_proto_text_client_new(log_transport_plain_new(fd, write_flags)) - : log_proto_file_writer_new(log_transport_plain_new(fd, write_flags), self->owner->writer_options.flush_lines)); + : log_proto_file_writer_new_with_callback(log_transport_plain_new(fd, write_flags), + self->owner->writer_options.flush_lines, + affile_dw_writer_callback, self)); main_loop_call((void * (*)(void *)) affile_dw_arm_reaper, self, TRUE); } @@ -635,6 +702,8 @@ affile_dw_queue(LogPipe *s, LogMessage *lm, const LogPathOptions *path_options, if (self->last_open_stamp == 0) self->last_open_stamp = self->last_msg_stamp; + self->cb_context.current_msg = lm; + if (!log_writer_opened((LogWriter *) self->writer) && !self->reopen_pending && (self->last_open_stamp < self->last_msg_stamp - self->time_reopen)) @@ -667,7 +736,8 @@ static void affile_dw_free(LogPipe *s) { AFFileDestWriter *self = (AFFileDestWriter *) s; - + + g_string_free(self->cb_context.filename, TRUE); log_pipe_unref(self->writer); self->writer = NULL; g_free(self->filename); @@ -697,6 +767,8 @@ affile_dw_new(AFFileDestDriver *owner, const gchar *filename) /* we have to take care about freeing filename later. This avoids a move of the filename. */ self->filename = g_strdup(filename); + self->cb_context.filename = g_string_sized_new(1024); + self->cb_context.seq = 1; g_static_mutex_init(&self->lock); return self; } @@ -737,6 +809,33 @@ affile_dd_set_file_perm(LogDriver *s, gint file_perm) self->file_perm = file_perm; } +void +affile_dd_set_file_size_limit(LogDriver *s, off_t file_size_limit, const gchar *template) +{ + AFFileDestDriver *self = (AFFileDestDriver *) s; + + self->size_limit = file_size_limit; + + if (!self->size_fn_format) + self->size_fn_format = log_template_new(configuration, NULL); + + if (template) + log_template_compile(self->size_fn_format, template, NULL); + else + { + if (self->filename_template) + { + gchar *fn = g_strdup_printf ("%s-${UNIXTIME}.${SEQNUM}", self->filename_template->template); + log_template_compile(self->size_fn_format, fn, NULL); + g_free (fn); + } + else + { + msg_error("Error setting size-rotated filename template", NULL); + } + } +} + void affile_dd_set_dir_uid(LogDriver *s, const gchar *dir_uid) { @@ -889,6 +988,16 @@ affile_dd_init(LogPipe *s) self->dir_perm = cfg->dir_perm; if (self->time_reap == -1) self->time_reap = cfg->time_reap; + if (self->size_limit == -1) + { + gchar *fn = g_strdup_printf("%s-${UNIXTIME}.${SEQNUM}", + self->filename_template->template); + + self->size_limit = cfg->file_size_limit; + self->size_fn_format = log_template_new(cfg, NULL); + log_template_compile(self->size_fn_format, fn, NULL); + g_free (fn); + } log_writer_options_init(&self->writer_options, cfg, 0); log_template_options_init(&self->template_fname_options, cfg); @@ -1147,6 +1256,7 @@ affile_dd_free(LogPipe *s) g_assert(self->single_writer == NULL && self->writer_hash == NULL); log_template_options_destroy(&self->template_fname_options); + log_template_unref(self->size_fn_format); log_template_unref(self->filename_template); log_writer_options_destroy(&self->writer_options); log_dest_driver_free(s); @@ -1175,6 +1285,7 @@ affile_dd_new(gchar *filename, guint32 flags) self->flags |= AFFILE_NO_EXPAND; } self->time_reap = -1; + self->size_limit = -1; log_template_options_defaults(&self->template_fname_options); g_static_mutex_init(&self->lock); return &self->super.super; diff --git a/modules/affile/affile.h b/modules/affile/affile.h index f418e0d..e961fed 100644 --- a/modules/affile/affile.h +++ b/modules/affile/affile.h @@ -75,6 +75,8 @@ typedef struct _AFFileDestDriver gint overwrite_if_older; gboolean use_time_recvd; gint time_reap; + off_t size_limit; + LogTemplate *size_fn_format; } AFFileDestDriver; LogDriver *affile_dd_new(gchar *filename, guint32 flags); @@ -91,5 +93,6 @@ void affile_dd_set_create_dirs(LogDriver *s, gboolean create_dirs); void affile_dd_set_fsync(LogDriver *s, gboolean enable); void affile_dd_set_overwrite_if_older(LogDriver *s, gint overwrite_if_older); void affile_dd_set_local_time_zone(LogDriver *s, const gchar *local_time_zone); +void affile_dd_set_file_size_limit(LogDriver *s, off_t file_size_limit, const gchar *template); #endif -- 1.7.0.4