[syslog-ng] PATCH: Log file size limit (was: Logfile Rotation)

31 Aug 2009

Balazs Scheidler wrote:
...
...
Personally I prefer to arrange log rotation based on the file size 
rather than some fixed time interval, so I have added that feature 
to syslog-ng. If anyone is interested I can explain in more details
 and post the patch.
Please do. That's what open source is about.
The following patch introduces two new configuration options:
'file_size_limit' to be used inside global 'options' and 'size_limit' to
be used inside 'file' destination. Each option specifies log file size
limit in bytes. If the global option is set to a value greater than zero
it applies to all 'file' destinations. A particular file destination can
remove the limit by setting it to zero. For example:

# set the global file size limit
options { file_size_limit(123456); };

# set a different size limit for a particular file destination
destination log1 { file("/var/log/log1.log" size_limit(456789)); };

# remove size limit for a particular file destination
# (only useful if there is a global size limit set)
destination log2 { file("/var/log/log2.log" size_limit(0)); };

The file size is checked after writing each log message and if the file
has grown up to or above the size limit the file is renamed and a new
empty file is created to continue logging to. The name format the
"overgrown" log file is renamed to is "<p>-<s>.<m>-<r>", where <p> - the
full path and name of the original log file, <s> - current time in UNIX
format (seconds since Jan 1, 1970), <m> - fractional part of the current
time (microseconds, 6 digits), <r> - a random number (10 digits).

The intended use is to have incrond or another similar mechanism to
detect when there is a new "renamed" log file and to process it in
whatever way necessary (gzip it, parse it, send it my email, etc.).

*********** BEGIN PATCH ***********
diff -U5 -rb syslog-ng-3.0.4-orig/src/affile.c syslog-ng-3.0.4/src/affile.c

--- syslog-ng-3.0.4-orig/src/affile.c	2009-07-31 11:41:20.000000000 +0200
+++ syslog-ng-3.0.4/src/affile.c	2009-08-17 11:02:41.000000000 +0200
@@ -428,14 +428,72 @@
 {
   return log_queue_get_length(((LogWriter *) self->writer)->queue) == 0;
 }
 
 static gboolean
+affile_dw_open_file(AFFileDestWriter *self, int *fd)
+{
+  return affile_open_file(self->filename->str, (self->owner->flags & AFFILE_PIPE)?
+                            (O_RDWR | O_NOCTTY | O_NONBLOCK | O_LARGEFILE):
+                            (O_WRONLY | O_CREAT | O_NOCTTY | O_NONBLOCK | O_LARGEFILE),
+                          self->owner->file_uid, self->owner->file_gid, self->owner->file_perm,
+                          self->owner->dir_uid, self->owner->dir_gid, self->owner->dir_perm,
+                          !!(self->owner->flags & AFFILE_CREATE_DIRS), FALSE, !!(self->owner->flags & AFFILE_PIPE), fd);
+}
+
+static void
+affile_dw_transport_callback(gint fd, void *context)
+{
+  static const size_t extra_length = 32;
+  AFFileDestWriter *self = (AFFileDestWriter *) context;
+  off_t size = lseek(fd, 0, SEEK_CUR);
+  GTimeVal time;
+  GString *name;
+  int reopen_fd;
+  if ((size > 0) && (size >= self->owner->size_limit))
+    {
+      /* TODO: use g_file_read_link() */
+      g_get_current_time(&time);
+      name = g_string_sized_new(self->filename->len + extra_length);
+      g_string_printf(name, "%s-%lu.%06lu-%010lu", self->filename->str, (gulong)time.tv_sec, (gulong)time.tv_usec, (gulong)g_random_int());
+      if (rename(self->filename->str, name->str) == 0)
+        {
+          if (affile_dw_open_file(self, &reopen_fd))
+            {
+              if (dup2(reopen_fd, fd) < 0)
+                {
+                  msg_error("Error swithing to new log file",
+                            evt_tag_str("filename", self->filename->str),
+                            evt_tag_errno(EVT_TAG_OSERROR, errno),
+                            NULL);
+                }
+              close(reopen_fd);
+            }
+          else
+            {
+              msg_error("Error opening file for writing",
+                        evt_tag_str("filename", self->filename->str),
+                        evt_tag_errno(EVT_TAG_OSERROR, errno),
+                        NULL);
+            }
+        }
+      else
+        {
+          msg_error("Error renaming overgrown file",
+                    evt_tag_str("filename", self->filename->str),
+                    evt_tag_errno(EVT_TAG_OSERROR, errno),
+                    NULL);
+        }
+      g_string_free(name, TRUE);
+    }
+}
+
+static gboolean
 affile_dw_init(LogPipe *s)
 {
   AFFileDestWriter *self = (AFFileDestWriter *) s;
-  int fd, flags;
+  int fd;
   struct stat st;
   GlobalConfig *cfg = log_pipe_get_config(s);
 
   if (cfg)
     self->time_reopen = cfg->time_reopen;
@@ -454,20 +512,12 @@
                  evt_tag_int("overwrite_if_older", self->owner->overwrite_if_older),
                  NULL);
       unlink(self->filename->str);
     }
 
-  if (self->owner->flags & AFFILE_PIPE)
-    flags = O_RDWR | O_NOCTTY | O_NONBLOCK | O_LARGEFILE;
-  else
-    flags = O_WRONLY | O_CREAT | O_NOCTTY | O_NONBLOCK | O_LARGEFILE;
-
   self->last_open_stamp = time(NULL);
-  if (affile_open_file(self->filename->str, flags, 
-                       self->owner->file_uid, self->owner->file_gid, self->owner->file_perm, 
-                       self->owner->dir_uid, self->owner->dir_gid, self->owner->dir_perm, 
-                       !!(self->owner->flags & AFFILE_CREATE_DIRS), FALSE, !!(self->owner->flags & AFFILE_PIPE), &fd))
+  if (affile_dw_open_file(self, &fd))
     {
       guint write_flags;
       
       if (!self->writer)
         {
@@ -482,11 +532,11 @@
           self->writer = NULL;
           close(fd);
           return FALSE;
         }
       write_flags = ((self->owner->flags & AFFILE_FSYNC) ? LTF_FSYNC : 0) | LTF_APPEND;
-      log_writer_reopen(self->writer, log_proto_plain_new_client(log_transport_plain_new(fd, write_flags)));
+      log_writer_reopen(self->writer, log_proto_plain_new_client(((self->owner->size_limit > 0) && !(self->owner->flags & AFFILE_PIPE))? log_transport_plain_new_with_callback(fd, write_flags, affile_dw_transport_callback, self): log_transport_plain_new(fd, write_flags)));
     }
   else
     {
       msg_error("Error opening file for writing",
                 evt_tag_str("filename", self->filename->str),
@@ -678,10 +728,18 @@
   AFFileDestDriver *self = (AFFileDestDriver *) s;
 
   self->local_time_zone = g_strdup(local_time_zone);
 }
 
+void
+affile_dd_set_file_size_limit(LogDriver *s, off_t file_size_limit)
+{
+  AFFileDestDriver *self = (AFFileDestDriver *) s;
+
+  self->size_limit = file_size_limit;
+}
+
 static inline gchar *
 affile_dd_format_persist_name(AFFileDestDriver *self)
 {
   static gchar persist_name[1024];
 
@@ -766,10 +824,12 @@
     self->dir_gid = cfg->dir_gid;
   if (self->dir_perm == (mode_t) -1)
     self->dir_perm = cfg->dir_perm;
   if (self->time_reap == -1)
     self->time_reap = cfg->time_reap;
+  if (self->size_limit == -1)
+    self->size_limit = cfg->file_size_limit;
   
   self->use_time_recvd = cfg->use_time_recvd;
 
   if (self->local_time_zone_info)
     time_zone_info_free(self->local_time_zone_info);
@@ -973,7 +1033,8 @@
   if (strchr(filename, '$') == NULL)
     {
       self->flags |= AFFILE_NO_EXPAND;
     }
   self->time_reap = -1;
+  self->size_limit = -1;
   return &self->super;
 }
diff -U5 -rb syslog-ng-3.0.4-orig/src/affile.h syslog-ng-3.0.4/src/affile.h
--- syslog-ng-3.0.4-orig/src/affile.h	2009-04-30 12:22:53.000000000 +0200
+++ syslog-ng-3.0.4/src/affile.h	2009-08-17 10:10:39.000000000 +0200
@@ -72,10 +72,11 @@
     
   gint overwrite_if_older;
   gboolean use_time_recvd;
   gint time_reap;
   guint reap_timer;
+  off_t size_limit;
 } AFFileDestDriver;
 
 LogDriver *affile_dd_new(gchar *filename, guint32 flags);
 
 void affile_dd_set_compress(LogDriver *s, gboolean compress);
@@ -88,7 +89,8 @@
 void affile_dd_set_dir_perm(LogDriver *s, mode_t dir_perm);
 void affile_dd_set_create_dirs(LogDriver *s, gboolean create_dirs);
 void affile_dd_set_fsync(LogDriver *s, gboolean enable);
 void affile_dd_set_overwrite_if_older(LogDriver *s, gint overwrite_if_older);
 void affile_dd_set_local_time_zone(LogDriver *s, const gchar *local_time_zone);
+void affile_dd_set_file_size_limit(LogDriver *s, off_t file_size_limit);
 
 #endif
diff -U5 -rb syslog-ng-3.0.4-orig/src/cfg-grammar.y syslog-ng-3.0.4/src/cfg-grammar.y
--- syslog-ng-3.0.4-orig/src/cfg-grammar.y	2009-08-05 12:34:18.000000000 +0200
+++ syslog-ng-3.0.4/src/cfg-grammar.y	2009-08-17 10:10:39.000000000 +0200
@@ -154,10 +154,11 @@
 %token KW_DIR_OWNER KW_DIR_GROUP KW_DIR_PERM 
 %token KW_TEMPLATE KW_TEMPLATE_ESCAPE
 %token KW_FOLLOW_FREQ
 %token KW_OVERWRITE_IF_OLDER
 %token KW_DEFAULT_FACILITY KW_DEFAULT_LEVEL
+%token KW_FILE_SIZE_LIMIT KW_SIZE_LIMIT
 
 /* socket related options */
 %token KW_KEEP_ALIVE KW_MAX_CONNECTIONS
 %token KW_LOCALIP KW_IP KW_LOCALPORT KW_PORT KW_DESTPORT
 %token KW_IP_TTL KW_SO_BROADCAST KW_IP_TOS KW_SO_SNDBUF KW_SO_RCVBUF KW_SO_KEEPALIVE KW_SPOOF_SOURCE
@@ -799,10 +800,11 @@
 	| KW_DIR_PERM '(' LL_NUMBER ')'		{ affile_dd_set_dir_perm(last_driver, $3); }
 	| KW_CREATE_DIRS '(' yesno ')'		{ affile_dd_set_create_dirs(last_driver, $3); }
 	| KW_OVERWRITE_IF_OLDER '(' LL_NUMBER ')'	{ affile_dd_set_overwrite_if_older(last_driver, $3); }
 	| KW_FSYNC '(' yesno ')'		{ affile_dd_set_fsync(last_driver, $3); }
 	| KW_LOCAL_TIME_ZONE '(' string ')'     { affile_dd_set_local_time_zone(last_driver, $3); free($3); }
+	| KW_SIZE_LIMIT '(' LL_NUMBER ')'	{ affile_dd_set_file_size_limit(last_driver, $3); }
 	;
 
 dest_afpipe
 	: KW_PIPE '(' dest_afpipe_params ')'    { $$ = $3; }
 	;
@@ -1141,10 +1143,11 @@
 	| KW_FILE_TEMPLATE '(' string ')'	{ configuration->file_template_name = g_strdup($3); free($3); }
 	| KW_PROTO_TEMPLATE '(' string ')'	{ configuration->proto_template_name = g_strdup($3); free($3); }
 	| KW_RECV_TIME_ZONE '(' string ')'      { configuration->recv_time_zone = g_strdup($3); free($3); }
 	| KW_SEND_TIME_ZONE '(' string ')'      { configuration->send_time_zone = g_strdup($3); free($3); }
 	| KW_LOCAL_TIME_ZONE '(' string ')'     { configuration->local_time_zone = g_strdup($3); free($3); }
+	| KW_FILE_SIZE_LIMIT '(' LL_NUMBER ')'  { configuration->file_size_limit = $3; }
 	;
 
 /* BEGIN MARK: tls */
 tls_options
 	: tls_option tls_options
diff -U5 -rb syslog-ng-3.0.4-orig/src/cfg-lex.l syslog-ng-3.0.4/src/cfg-lex.l
--- syslog-ng-3.0.4-orig/src/cfg-lex.l	2009-05-06 11:20:22.000000000 +0200
+++ syslog-ng-3.0.4/src/cfg-lex.l	2009-08-17 10:10:39.000000000 +0200
@@ -126,10 +126,12 @@
         { "recv_time_zone",	KW_RECV_TIME_ZONE },
         { "send_time_zone",	KW_SEND_TIME_ZONE },
         { "local_time_zone",    KW_LOCAL_TIME_ZONE },
         { "use_time_recvd",	KW_USE_TIME_RECVD, KWS_OBSOLETE, "Use R_ or S_ prefixed macros in templates" },
         { "use_fqdn",           KW_USE_FQDN },
+        { "size_limit",        KW_SIZE_LIMIT },
+        { "file_size_limit",   KW_FILE_SIZE_LIMIT },
 	{ "use_dns",		KW_USE_DNS },
   	{ "gc_threshold",	KW_GC_BUSY_THRESHOLD },
   	{ "gc_busy_threshold",	KW_GC_BUSY_THRESHOLD },
   	{ "gc_idle_threshold",	KW_GC_IDLE_THRESHOLD },
  	{ "time_reopen",	KW_TIME_REOPEN },
diff -U5 -rb syslog-ng-3.0.4-orig/src/cfg.c syslog-ng-3.0.4/src/cfg.c
--- syslog-ng-3.0.4-orig/src/cfg.c	2009-04-30 12:00:54.000000000 +0200
+++ syslog-ng-3.0.4/src/cfg.c	2009-08-17 10:10:39.000000000 +0200
@@ -298,10 +298,11 @@
 
   self->follow_freq = -1;
   self->file_uid = 0;
   self->file_gid = 0;
   self->file_perm = 0600;
+  self->file_size_limit = 0;
   self->dir_uid = 0;
   self->dir_gid = 0;
   self->dir_perm = 0700;
 
   self->use_dns_cache = 1;
diff -U5 -rb syslog-ng-3.0.4-orig/src/cfg.h syslog-ng-3.0.4/src/cfg.h
--- syslog-ng-3.0.4-orig/src/cfg.h	2009-06-03 13:08:27.000000000 +0200
+++ syslog-ng-3.0.4/src/cfg.h	2009-08-17 10:10:39.000000000 +0200
@@ -78,10 +78,11 @@
   gint follow_freq;
   gboolean create_dirs;
   uid_t file_uid;
   gid_t file_gid;
   mode_t file_perm;
+  off_t file_size_limit;
   
   uid_t dir_uid;
   gid_t dir_gid;
   mode_t dir_perm;
 
diff -U5 -rb syslog-ng-3.0.4-orig/src/logtransport.c syslog-ng-3.0.4/src/logtransport.c
--- syslog-ng-3.0.4-orig/src/logtransport.c	2009-04-30 10:53:22.000000000 +0200
+++ syslog-ng-3.0.4/src/logtransport.c	2009-08-17 11:01:33.000000000 +0200
@@ -53,10 +53,12 @@
 typedef struct _LogTransportPlain LogTransportPlain;
 
 struct _LogTransportPlain
 {
   LogTransport super;
+  LogTransportCallback callback;
+  void *context;
 };
 
 static gssize
 log_transport_plain_read_method(LogTransport *s, gpointer buf, gsize buflen, GSockAddr **sa)
 {
@@ -138,24 +140,32 @@
         alarm_cancel();
       if (self->super.flags & LTF_FSYNC)
         fsync(self->super.fd);
     }
   while (rc == -1 && errno == EINTR);
+  if ((self->callback != NULL) && (rc > 0))
+    self->callback(self->super.fd, self->context);
   return rc;
 }
 
 
 LogTransport *
-log_transport_plain_new(gint fd, guint flags)
+log_transport_plain_new_with_callback(gint fd, guint flags, LogTransportCallback callback, void *context)
 {
   LogTransportPlain *self = g_new0(LogTransportPlain, 1);
   
   self->super.fd = fd;
   self->super.cond = 0;
   self->super.flags = flags;
   self->super.read = log_transport_plain_read_method;
   self->super.write = log_transport_plain_write_method;
   self->super.free_fn = log_transport_free_method;
+  self->callback = callback;
+  self->context = context;
   return &self->super;
 }
 
-
+LogTransport *
+log_transport_plain_new(gint fd, guint flags)
+{
+  return log_transport_plain_new_with_callback(fd, flags, NULL, NULL);
+}
diff -U5 -rb syslog-ng-3.0.4-orig/src/logtransport.h syslog-ng-3.0.4/src/logtransport.h
--- syslog-ng-3.0.4-orig/src/logtransport.h	2009-04-30 10:46:55.000000000 +0200
+++ syslog-ng-3.0.4/src/logtransport.h	2009-08-17 11:00:58.000000000 +0200
@@ -57,10 +57,12 @@
 log_transport_read(LogTransport *self, gpointer buf, gsize count, GSockAddr **sa)
 {
   return self->read(self, buf, count, sa);
 }
 
+typedef void (*LogTransportCallback)(gint fd, void *context);
+LogTransport *log_transport_plain_new_with_callback(gint fd, guint flags, LogTransportCallback callback, void *context);
 LogTransport *log_transport_plain_new(gint fd, guint flags);
 void log_transport_free(LogTransport *s);
 void log_transport_free_method(LogTransport *s);

************ END PATCH ************
--
Kind regards,
Sergei.

    

[syslog-ng] PATCH: Log file size limit (was: Logfile Rotation)

Sergei Zhirikov