[syslog-ng] PATCH: Log file size limit (was: Logfile Rotation)
Sergei Zhirikov
sfzhi at yahoo.com
Mon Aug 31 22:19:45 CEST 2009
Balazs Scheidler wrote:
>>
>> Personally I prefer to arrange log rotation based on the file size
>> rather than some fixed time interval, so I have added that feature
>> to syslog-ng. If anyone is interested I can explain in more details
>> and post the patch.
>
> Please do. That's what open source is about.
>
The following patch introduces two new configuration options:
'file_size_limit' to be used inside global 'options' and 'size_limit' to
be used inside 'file' destination. Each option specifies log file size
limit in bytes. If the global option is set to a value greater than zero
it applies to all 'file' destinations. A particular file destination can
remove the limit by setting it to zero. For example:
# set the global file size limit
options { file_size_limit(123456); };
# set a different size limit for a particular file destination
destination log1 { file("/var/log/log1.log" size_limit(456789)); };
# remove size limit for a particular file destination
# (only useful if there is a global size limit set)
destination log2 { file("/var/log/log2.log" size_limit(0)); };
The file size is checked after writing each log message and if the file
has grown up to or above the size limit the file is renamed and a new
empty file is created to continue logging to. The name format the
"overgrown" log file is renamed to is "<p>-<s>.<m>-<r>", where <p> - the
full path and name of the original log file, <s> - current time in UNIX
format (seconds since Jan 1, 1970), <m> - fractional part of the current
time (microseconds, 6 digits), <r> - a random number (10 digits).
The intended use is to have incrond or another similar mechanism to
detect when there is a new "renamed" log file and to process it in
whatever way necessary (gzip it, parse it, send it my email, etc.).
*********** BEGIN PATCH ***********
diff -U5 -rb syslog-ng-3.0.4-orig/src/affile.c syslog-ng-3.0.4/src/affile.c
--- syslog-ng-3.0.4-orig/src/affile.c 2009-07-31 11:41:20.000000000 +0200
+++ syslog-ng-3.0.4/src/affile.c 2009-08-17 11:02:41.000000000 +0200
@@ -428,14 +428,72 @@
{
return log_queue_get_length(((LogWriter *) self->writer)->queue) == 0;
}
static gboolean
+affile_dw_open_file(AFFileDestWriter *self, int *fd)
+{
+ return affile_open_file(self->filename->str, (self->owner->flags & AFFILE_PIPE)?
+ (O_RDWR | O_NOCTTY | O_NONBLOCK | O_LARGEFILE):
+ (O_WRONLY | O_CREAT | O_NOCTTY | O_NONBLOCK | O_LARGEFILE),
+ self->owner->file_uid, self->owner->file_gid, self->owner->file_perm,
+ self->owner->dir_uid, self->owner->dir_gid, self->owner->dir_perm,
+ !!(self->owner->flags & AFFILE_CREATE_DIRS), FALSE, !!(self->owner->flags & AFFILE_PIPE), fd);
+}
+
+static void
+affile_dw_transport_callback(gint fd, void *context)
+{
+ static const size_t extra_length = 32;
+ AFFileDestWriter *self = (AFFileDestWriter *) context;
+ off_t size = lseek(fd, 0, SEEK_CUR);
+ GTimeVal time;
+ GString *name;
+ int reopen_fd;
+ if ((size > 0) && (size >= self->owner->size_limit))
+ {
+ /* TODO: use g_file_read_link() */
+ g_get_current_time(&time);
+ name = g_string_sized_new(self->filename->len + extra_length);
+ g_string_printf(name, "%s-%lu.%06lu-%010lu", self->filename->str, (gulong)time.tv_sec, (gulong)time.tv_usec, (gulong)g_random_int());
+ if (rename(self->filename->str, name->str) == 0)
+ {
+ if (affile_dw_open_file(self, &reopen_fd))
+ {
+ if (dup2(reopen_fd, fd) < 0)
+ {
+ msg_error("Error swithing to new log file",
+ evt_tag_str("filename", self->filename->str),
+ evt_tag_errno(EVT_TAG_OSERROR, errno),
+ NULL);
+ }
+ close(reopen_fd);
+ }
+ else
+ {
+ msg_error("Error opening file for writing",
+ evt_tag_str("filename", self->filename->str),
+ evt_tag_errno(EVT_TAG_OSERROR, errno),
+ NULL);
+ }
+ }
+ else
+ {
+ msg_error("Error renaming overgrown file",
+ evt_tag_str("filename", self->filename->str),
+ evt_tag_errno(EVT_TAG_OSERROR, errno),
+ NULL);
+ }
+ g_string_free(name, TRUE);
+ }
+}
+
+static gboolean
affile_dw_init(LogPipe *s)
{
AFFileDestWriter *self = (AFFileDestWriter *) s;
- int fd, flags;
+ int fd;
struct stat st;
GlobalConfig *cfg = log_pipe_get_config(s);
if (cfg)
self->time_reopen = cfg->time_reopen;
@@ -454,20 +512,12 @@
evt_tag_int("overwrite_if_older", self->owner->overwrite_if_older),
NULL);
unlink(self->filename->str);
}
- if (self->owner->flags & AFFILE_PIPE)
- flags = O_RDWR | O_NOCTTY | O_NONBLOCK | O_LARGEFILE;
- else
- flags = O_WRONLY | O_CREAT | O_NOCTTY | O_NONBLOCK | O_LARGEFILE;
-
self->last_open_stamp = time(NULL);
- if (affile_open_file(self->filename->str, flags,
- self->owner->file_uid, self->owner->file_gid, self->owner->file_perm,
- self->owner->dir_uid, self->owner->dir_gid, self->owner->dir_perm,
- !!(self->owner->flags & AFFILE_CREATE_DIRS), FALSE, !!(self->owner->flags & AFFILE_PIPE), &fd))
+ if (affile_dw_open_file(self, &fd))
{
guint write_flags;
if (!self->writer)
{
@@ -482,11 +532,11 @@
self->writer = NULL;
close(fd);
return FALSE;
}
write_flags = ((self->owner->flags & AFFILE_FSYNC) ? LTF_FSYNC : 0) | LTF_APPEND;
- log_writer_reopen(self->writer, log_proto_plain_new_client(log_transport_plain_new(fd, write_flags)));
+ log_writer_reopen(self->writer, log_proto_plain_new_client(((self->owner->size_limit > 0) && !(self->owner->flags & AFFILE_PIPE))? log_transport_plain_new_with_callback(fd, write_flags, affile_dw_transport_callback, self): log_transport_plain_new(fd, write_flags)));
}
else
{
msg_error("Error opening file for writing",
evt_tag_str("filename", self->filename->str),
@@ -678,10 +728,18 @@
AFFileDestDriver *self = (AFFileDestDriver *) s;
self->local_time_zone = g_strdup(local_time_zone);
}
+void
+affile_dd_set_file_size_limit(LogDriver *s, off_t file_size_limit)
+{
+ AFFileDestDriver *self = (AFFileDestDriver *) s;
+
+ self->size_limit = file_size_limit;
+}
+
static inline gchar *
affile_dd_format_persist_name(AFFileDestDriver *self)
{
static gchar persist_name[1024];
@@ -766,10 +824,12 @@
self->dir_gid = cfg->dir_gid;
if (self->dir_perm == (mode_t) -1)
self->dir_perm = cfg->dir_perm;
if (self->time_reap == -1)
self->time_reap = cfg->time_reap;
+ if (self->size_limit == -1)
+ self->size_limit = cfg->file_size_limit;
self->use_time_recvd = cfg->use_time_recvd;
if (self->local_time_zone_info)
time_zone_info_free(self->local_time_zone_info);
@@ -973,7 +1033,8 @@
if (strchr(filename, '$') == NULL)
{
self->flags |= AFFILE_NO_EXPAND;
}
self->time_reap = -1;
+ self->size_limit = -1;
return &self->super;
}
diff -U5 -rb syslog-ng-3.0.4-orig/src/affile.h syslog-ng-3.0.4/src/affile.h
--- syslog-ng-3.0.4-orig/src/affile.h 2009-04-30 12:22:53.000000000 +0200
+++ syslog-ng-3.0.4/src/affile.h 2009-08-17 10:10:39.000000000 +0200
@@ -72,10 +72,11 @@
gint overwrite_if_older;
gboolean use_time_recvd;
gint time_reap;
guint reap_timer;
+ off_t size_limit;
} AFFileDestDriver;
LogDriver *affile_dd_new(gchar *filename, guint32 flags);
void affile_dd_set_compress(LogDriver *s, gboolean compress);
@@ -88,7 +89,8 @@
void affile_dd_set_dir_perm(LogDriver *s, mode_t dir_perm);
void affile_dd_set_create_dirs(LogDriver *s, gboolean create_dirs);
void affile_dd_set_fsync(LogDriver *s, gboolean enable);
void affile_dd_set_overwrite_if_older(LogDriver *s, gint overwrite_if_older);
void affile_dd_set_local_time_zone(LogDriver *s, const gchar *local_time_zone);
+void affile_dd_set_file_size_limit(LogDriver *s, off_t file_size_limit);
#endif
diff -U5 -rb syslog-ng-3.0.4-orig/src/cfg-grammar.y syslog-ng-3.0.4/src/cfg-grammar.y
--- syslog-ng-3.0.4-orig/src/cfg-grammar.y 2009-08-05 12:34:18.000000000 +0200
+++ syslog-ng-3.0.4/src/cfg-grammar.y 2009-08-17 10:10:39.000000000 +0200
@@ -154,10 +154,11 @@
%token KW_DIR_OWNER KW_DIR_GROUP KW_DIR_PERM
%token KW_TEMPLATE KW_TEMPLATE_ESCAPE
%token KW_FOLLOW_FREQ
%token KW_OVERWRITE_IF_OLDER
%token KW_DEFAULT_FACILITY KW_DEFAULT_LEVEL
+%token KW_FILE_SIZE_LIMIT KW_SIZE_LIMIT
/* socket related options */
%token KW_KEEP_ALIVE KW_MAX_CONNECTIONS
%token KW_LOCALIP KW_IP KW_LOCALPORT KW_PORT KW_DESTPORT
%token KW_IP_TTL KW_SO_BROADCAST KW_IP_TOS KW_SO_SNDBUF KW_SO_RCVBUF KW_SO_KEEPALIVE KW_SPOOF_SOURCE
@@ -799,10 +800,11 @@
| KW_DIR_PERM '(' LL_NUMBER ')' { affile_dd_set_dir_perm(last_driver, $3); }
| KW_CREATE_DIRS '(' yesno ')' { affile_dd_set_create_dirs(last_driver, $3); }
| KW_OVERWRITE_IF_OLDER '(' LL_NUMBER ')' { affile_dd_set_overwrite_if_older(last_driver, $3); }
| KW_FSYNC '(' yesno ')' { affile_dd_set_fsync(last_driver, $3); }
| KW_LOCAL_TIME_ZONE '(' string ')' { affile_dd_set_local_time_zone(last_driver, $3); free($3); }
+ | KW_SIZE_LIMIT '(' LL_NUMBER ')' { affile_dd_set_file_size_limit(last_driver, $3); }
;
dest_afpipe
: KW_PIPE '(' dest_afpipe_params ')' { $$ = $3; }
;
@@ -1141,10 +1143,11 @@
| KW_FILE_TEMPLATE '(' string ')' { configuration->file_template_name = g_strdup($3); free($3); }
| KW_PROTO_TEMPLATE '(' string ')' { configuration->proto_template_name = g_strdup($3); free($3); }
| KW_RECV_TIME_ZONE '(' string ')' { configuration->recv_time_zone = g_strdup($3); free($3); }
| KW_SEND_TIME_ZONE '(' string ')' { configuration->send_time_zone = g_strdup($3); free($3); }
| KW_LOCAL_TIME_ZONE '(' string ')' { configuration->local_time_zone = g_strdup($3); free($3); }
+ | KW_FILE_SIZE_LIMIT '(' LL_NUMBER ')' { configuration->file_size_limit = $3; }
;
/* BEGIN MARK: tls */
tls_options
: tls_option tls_options
diff -U5 -rb syslog-ng-3.0.4-orig/src/cfg-lex.l syslog-ng-3.0.4/src/cfg-lex.l
--- syslog-ng-3.0.4-orig/src/cfg-lex.l 2009-05-06 11:20:22.000000000 +0200
+++ syslog-ng-3.0.4/src/cfg-lex.l 2009-08-17 10:10:39.000000000 +0200
@@ -126,10 +126,12 @@
{ "recv_time_zone", KW_RECV_TIME_ZONE },
{ "send_time_zone", KW_SEND_TIME_ZONE },
{ "local_time_zone", KW_LOCAL_TIME_ZONE },
{ "use_time_recvd", KW_USE_TIME_RECVD, KWS_OBSOLETE, "Use R_ or S_ prefixed macros in templates" },
{ "use_fqdn", KW_USE_FQDN },
+ { "size_limit", KW_SIZE_LIMIT },
+ { "file_size_limit", KW_FILE_SIZE_LIMIT },
{ "use_dns", KW_USE_DNS },
{ "gc_threshold", KW_GC_BUSY_THRESHOLD },
{ "gc_busy_threshold", KW_GC_BUSY_THRESHOLD },
{ "gc_idle_threshold", KW_GC_IDLE_THRESHOLD },
{ "time_reopen", KW_TIME_REOPEN },
diff -U5 -rb syslog-ng-3.0.4-orig/src/cfg.c syslog-ng-3.0.4/src/cfg.c
--- syslog-ng-3.0.4-orig/src/cfg.c 2009-04-30 12:00:54.000000000 +0200
+++ syslog-ng-3.0.4/src/cfg.c 2009-08-17 10:10:39.000000000 +0200
@@ -298,10 +298,11 @@
self->follow_freq = -1;
self->file_uid = 0;
self->file_gid = 0;
self->file_perm = 0600;
+ self->file_size_limit = 0;
self->dir_uid = 0;
self->dir_gid = 0;
self->dir_perm = 0700;
self->use_dns_cache = 1;
diff -U5 -rb syslog-ng-3.0.4-orig/src/cfg.h syslog-ng-3.0.4/src/cfg.h
--- syslog-ng-3.0.4-orig/src/cfg.h 2009-06-03 13:08:27.000000000 +0200
+++ syslog-ng-3.0.4/src/cfg.h 2009-08-17 10:10:39.000000000 +0200
@@ -78,10 +78,11 @@
gint follow_freq;
gboolean create_dirs;
uid_t file_uid;
gid_t file_gid;
mode_t file_perm;
+ off_t file_size_limit;
uid_t dir_uid;
gid_t dir_gid;
mode_t dir_perm;
diff -U5 -rb syslog-ng-3.0.4-orig/src/logtransport.c syslog-ng-3.0.4/src/logtransport.c
--- syslog-ng-3.0.4-orig/src/logtransport.c 2009-04-30 10:53:22.000000000 +0200
+++ syslog-ng-3.0.4/src/logtransport.c 2009-08-17 11:01:33.000000000 +0200
@@ -53,10 +53,12 @@
typedef struct _LogTransportPlain LogTransportPlain;
struct _LogTransportPlain
{
LogTransport super;
+ LogTransportCallback callback;
+ void *context;
};
static gssize
log_transport_plain_read_method(LogTransport *s, gpointer buf, gsize buflen, GSockAddr **sa)
{
@@ -138,24 +140,32 @@
alarm_cancel();
if (self->super.flags & LTF_FSYNC)
fsync(self->super.fd);
}
while (rc == -1 && errno == EINTR);
+ if ((self->callback != NULL) && (rc > 0))
+ self->callback(self->super.fd, self->context);
return rc;
}
LogTransport *
-log_transport_plain_new(gint fd, guint flags)
+log_transport_plain_new_with_callback(gint fd, guint flags, LogTransportCallback callback, void *context)
{
LogTransportPlain *self = g_new0(LogTransportPlain, 1);
self->super.fd = fd;
self->super.cond = 0;
self->super.flags = flags;
self->super.read = log_transport_plain_read_method;
self->super.write = log_transport_plain_write_method;
self->super.free_fn = log_transport_free_method;
+ self->callback = callback;
+ self->context = context;
return &self->super;
}
-
+LogTransport *
+log_transport_plain_new(gint fd, guint flags)
+{
+ return log_transport_plain_new_with_callback(fd, flags, NULL, NULL);
+}
diff -U5 -rb syslog-ng-3.0.4-orig/src/logtransport.h syslog-ng-3.0.4/src/logtransport.h
--- syslog-ng-3.0.4-orig/src/logtransport.h 2009-04-30 10:46:55.000000000 +0200
+++ syslog-ng-3.0.4/src/logtransport.h 2009-08-17 11:00:58.000000000 +0200
@@ -57,10 +57,12 @@
log_transport_read(LogTransport *self, gpointer buf, gsize count, GSockAddr **sa)
{
return self->read(self, buf, count, sa);
}
+typedef void (*LogTransportCallback)(gint fd, void *context);
+LogTransport *log_transport_plain_new_with_callback(gint fd, guint flags, LogTransportCallback callback, void *context);
LogTransport *log_transport_plain_new(gint fd, guint flags);
void log_transport_free(LogTransport *s);
void log_transport_free_method(LogTransport *s);
************ END PATCH ************
--
Kind regards,
Sergei.
More information about the syslog-ng
mailing list