[syslog-ng] Regex subpattern macros [1.9.9]

[Balazs Scheidler]

On Tue, 2006-03-07 at 18:18 +0000, Mel Collins wrote:
>  Hi,
>  I'm using syslog-ng's match filter to extract parts of messages for use in 
> parts of the destination; specifically in the log file name and template 
> sections.
>  The messages sent to syslog-ng for this purpose are all of the format: 
> "<filename> <message>" (plus whatever sl-ng prepends).
> 
>  .conf extract:
> filter f_logsplitter { match ("^.*?: ([^ ]*) (.*)$"); };
> destination d_logfiles {
> 	file ("/our/logs/$1"
> 		template ("$2\n")
> 		...
> etc.
> 
>  Eg. For the message "root: foo/bar.log some text", the log 
> file /our/logs/foo/bar.log gets "some text" appended to it.
> 
>  Now this seems to work fine for low traffic, but if I send 10k messages in a 
> tight loop - either locally or remotely via TCP - $2 contains the wrong 
> string for many/most of them, eg:
> 07/03/2006 16:56:34 SEEK #3
> 07/03/2006 16:56:35 SEEK #3
> 07/03/2006 16:56:35 SEEK #3
> 07/03/2006 16:56:35 SEEK #7
> 07/03/2006 16:56:35 SEEK #7
> 07/03/2006 16:56:35 SEEK #7
> 07/03/2006 16:56:35 SEEK #7
> 07/03/2006 16:56:35 SEEK #8
> 07/03/2006 16:56:35 SEEK #11
> 07/03/2006 16:56:35 SEEK #11
> 07/03/2006 16:56:35 SEEK #11
>  ...where the #number should be consecutive.
>  If I use $MSGONLY, or such, the correct message line is used.
> 
>  It seems to me like the subpattern macros - $1, $2, etc - aren't maintained 
> on a per-message basis, but are global..?
>  Given that they aren't documented (that I can find), are the presence of $1, 
> etc. an unintended side-effect of using a regex? Am I doing something I 
> shouldn't by using them?
>  Or did I find me a bug? ;o)

You probably found a bug. I assummed that processing of a message is
always finished before starting a new one, e.g. the pattern matches are
global.

This is however not true when the message is queued in which case the
scenario that you described happens, the global match space is
overwritten. Let me fix that buglet...

Can you check if this compile-tested patch fixes it?

M  src/filter.c
M  src/filter.h
M  src/logmsg.h
M  src/macros.c

* modified files

--- orig/src/filter.c
+++ mod/src/filter.c
@@ -30,11 +30,8 @@
 #include <string.h>


-gchar *re_matches[RE_MAX_MATCHES];
-
 static void log_filter_rule_free(LogFilterRule *self);

-
 gboolean
 log_filter_rule_eval(LogFilterRule *self, LogMessage *msg)
 {
@@ -304,7 +301,7 @@ filter_re_compile(const char *re, regex_
 }

 static gboolean
-filter_re_eval(FilterRE *self, gchar *str)
+filter_re_eval(FilterRE *self, LogMessage *msg, gchar *str)
 {
   regmatch_t matches[RE_MAX_MATCHES];
   gboolean rc;
@@ -312,8 +309,9 @@ filter_re_eval(FilterRE *self, gchar *st

   for (i = 0; i < RE_MAX_MATCHES; i++)
     {
-      g_free(re_matches[i]);
-      re_matches[i] = NULL;
+      if (msg->re_matches[i])
+        g_free(msg->re_matches[i]);
+      msg->re_matches[i] = NULL;
     }
   rc = !regexec(&self->regex, str, RE_MAX_MATCHES, matches, 0);
   if (rc)
@@ -321,9 +319,9 @@ filter_re_eval(FilterRE *self, gchar *st
       for (i = 0; i < RE_MAX_MATCHES && matches[i].rm_so != -1; i++)
         {
           gint length = matches[i].rm_eo - matches[i].rm_so;
-          re_matches[i] = g_malloc(length + 1);
-          memcpy(re_matches[i], &str[matches[i].rm_so], length);
-          re_matches[i][length] = 0;
+          msg->re_matches[i] = g_malloc(length + 1);
+          memcpy(msg->re_matches[i], &str[matches[i].rm_so], length);
+          msg->re_matches[i][length] = 0;
         }
     }
   return rc ^ self->super.comp;
@@ -341,7 +339,7 @@ filter_re_free(FilterExprNode *s)
 static gboolean
 filter_prog_eval(FilterExprNode *s, LogMessage *msg)
 {
-  return filter_re_eval((FilterRE *) s, msg->program->str);
+  return filter_re_eval((FilterRE *) s, msg, msg->program->str);
 }

 FilterExprNode *
@@ -363,7 +361,7 @@ filter_prog_new(gchar *prog)
 static gboolean
 filter_host_eval(FilterExprNode *s, LogMessage *msg)
 {
-  return filter_re_eval((FilterRE *) s, msg->host->str);
+  return filter_re_eval((FilterRE *) s, msg, msg->host->str);
 }

 FilterExprNode *
@@ -385,7 +383,7 @@ filter_host_new(gchar *host)
 static gboolean
 filter_match_eval(FilterExprNode *s, LogMessage *msg)
 {
-  return filter_re_eval((FilterRE *) s, msg->msg->str);
+  return filter_re_eval((FilterRE *) s, msg, msg->msg->str);
 }

 FilterExprNode *


--- orig/src/filter.h
+++ mod/src/filter.h
@@ -31,10 +31,6 @@
 struct _LogFilterRule;
 struct _GlobalConfig;

-/* regex substitutions from the last match */
-#define RE_MAX_MATCHES 10
-extern gchar *re_matches[RE_MAX_MATCHES];
-
 typedef struct _FilterExprNode
 {
   gboolean comp;


--- orig/src/logmsg.h
+++ mod/src/logmsg.h
@@ -72,6 +72,8 @@ typedef struct _LogStamp

 void log_stamp_format(LogStamp *stamp, GString *target, gint ts_format, glong zone_offset, gint frac_digits);

+#define RE_MAX_MATCHES 10
+
 typedef struct _LogMessage
 {
   guint ref_cnt;
@@ -87,6 +89,7 @@ typedef struct _LogMessage
   LogStamp stamp;
   LogStamp recvd;
   GString *date, *host, *host_from, *program, *msg;
+  gchar *re_matches[RE_MAX_MATCHES];
 } LogMessage;

 LogMessage *log_msg_ref(LogMessage *m);


--- orig/src/macros.c
+++ mod/src/macros.c
@@ -142,8 +142,8 @@ log_macro_expand(GString *result, gint i
     {
       gint ndx = id - M_MATCH_REF_OFS;
       /* match reference */
-      if (re_matches[ndx])
-        result_append(result, re_matches[ndx], strlen(re_matches[ndx]), !!(flags & MF_ESCAPE_RESULT));
+      if (msg->re_matches[ndx])
+        result_append(result, msg->re_matches[ndx], strlen(msg->re_matches[ndx]), !!(flags & MF_ESCAPE_RESULT));

       return TRUE;
     }





-- 
Bazsi