On Tue, 2006-03-07 at 18:18 +0000, Mel Collins wrote:
Hi, I'm using syslog-ng's match filter to extract parts of messages for use in parts of the destination; specifically in the log file name and template sections. The messages sent to syslog-ng for this purpose are all of the format: "<filename> <message>" (plus whatever sl-ng prepends).
.conf extract: filter f_logsplitter { match ("^.*?: ([^ ]*) (.*)$"); }; destination d_logfiles { file ("/our/logs/$1" template ("$2\n") ... etc.
Eg. For the message "root: foo/bar.log some text", the log file /our/logs/foo/bar.log gets "some text" appended to it.
Now this seems to work fine for low traffic, but if I send 10k messages in a tight loop - either locally or remotely via TCP - $2 contains the wrong string for many/most of them, eg: 07/03/2006 16:56:34 SEEK #3 07/03/2006 16:56:35 SEEK #3 07/03/2006 16:56:35 SEEK #3 07/03/2006 16:56:35 SEEK #7 07/03/2006 16:56:35 SEEK #7 07/03/2006 16:56:35 SEEK #7 07/03/2006 16:56:35 SEEK #7 07/03/2006 16:56:35 SEEK #8 07/03/2006 16:56:35 SEEK #11 07/03/2006 16:56:35 SEEK #11 07/03/2006 16:56:35 SEEK #11 ...where the #number should be consecutive. If I use $MSGONLY, or such, the correct message line is used.
It seems to me like the subpattern macros - $1, $2, etc - aren't maintained on a per-message basis, but are global..? Given that they aren't documented (that I can find), are the presence of $1, etc. an unintended side-effect of using a regex? Am I doing something I shouldn't by using them? Or did I find me a bug? ;o)
You probably found a bug. I assummed that processing of a message is always finished before starting a new one, e.g. the pattern matches are global. This is however not true when the message is queued in which case the scenario that you described happens, the global match space is overwritten. Let me fix that buglet... Can you check if this compile-tested patch fixes it? M src/filter.c M src/filter.h M src/logmsg.h M src/macros.c * modified files --- orig/src/filter.c +++ mod/src/filter.c @@ -30,11 +30,8 @@ #include <string.h> -gchar *re_matches[RE_MAX_MATCHES]; - static void log_filter_rule_free(LogFilterRule *self); - gboolean log_filter_rule_eval(LogFilterRule *self, LogMessage *msg) { @@ -304,7 +301,7 @@ filter_re_compile(const char *re, regex_ } static gboolean -filter_re_eval(FilterRE *self, gchar *str) +filter_re_eval(FilterRE *self, LogMessage *msg, gchar *str) { regmatch_t matches[RE_MAX_MATCHES]; gboolean rc; @@ -312,8 +309,9 @@ filter_re_eval(FilterRE *self, gchar *st for (i = 0; i < RE_MAX_MATCHES; i++) { - g_free(re_matches[i]); - re_matches[i] = NULL; + if (msg->re_matches[i]) + g_free(msg->re_matches[i]); + msg->re_matches[i] = NULL; } rc = !regexec(&self->regex, str, RE_MAX_MATCHES, matches, 0); if (rc) @@ -321,9 +319,9 @@ filter_re_eval(FilterRE *self, gchar *st for (i = 0; i < RE_MAX_MATCHES && matches[i].rm_so != -1; i++) { gint length = matches[i].rm_eo - matches[i].rm_so; - re_matches[i] = g_malloc(length + 1); - memcpy(re_matches[i], &str[matches[i].rm_so], length); - re_matches[i][length] = 0; + msg->re_matches[i] = g_malloc(length + 1); + memcpy(msg->re_matches[i], &str[matches[i].rm_so], length); + msg->re_matches[i][length] = 0; } } return rc ^ self->super.comp; @@ -341,7 +339,7 @@ filter_re_free(FilterExprNode *s) static gboolean filter_prog_eval(FilterExprNode *s, LogMessage *msg) { - return filter_re_eval((FilterRE *) s, msg->program->str); + return filter_re_eval((FilterRE *) s, msg, msg->program->str); } FilterExprNode * @@ -363,7 +361,7 @@ filter_prog_new(gchar *prog) static gboolean filter_host_eval(FilterExprNode *s, LogMessage *msg) { - return filter_re_eval((FilterRE *) s, msg->host->str); + return filter_re_eval((FilterRE *) s, msg, msg->host->str); } FilterExprNode * @@ -385,7 +383,7 @@ filter_host_new(gchar *host) static gboolean filter_match_eval(FilterExprNode *s, LogMessage *msg) { - return filter_re_eval((FilterRE *) s, msg->msg->str); + return filter_re_eval((FilterRE *) s, msg, msg->msg->str); } FilterExprNode * --- orig/src/filter.h +++ mod/src/filter.h @@ -31,10 +31,6 @@ struct _LogFilterRule; struct _GlobalConfig; -/* regex substitutions from the last match */ -#define RE_MAX_MATCHES 10 -extern gchar *re_matches[RE_MAX_MATCHES]; - typedef struct _FilterExprNode { gboolean comp; --- orig/src/logmsg.h +++ mod/src/logmsg.h @@ -72,6 +72,8 @@ typedef struct _LogStamp void log_stamp_format(LogStamp *stamp, GString *target, gint ts_format, glong zone_offset, gint frac_digits); +#define RE_MAX_MATCHES 10 + typedef struct _LogMessage { guint ref_cnt; @@ -87,6 +89,7 @@ typedef struct _LogMessage LogStamp stamp; LogStamp recvd; GString *date, *host, *host_from, *program, *msg; + gchar *re_matches[RE_MAX_MATCHES]; } LogMessage; LogMessage *log_msg_ref(LogMessage *m); --- orig/src/macros.c +++ mod/src/macros.c @@ -142,8 +142,8 @@ log_macro_expand(GString *result, gint i { gint ndx = id - M_MATCH_REF_OFS; /* match reference */ - if (re_matches[ndx]) - result_append(result, re_matches[ndx], strlen(re_matches[ndx]), !!(flags & MF_ESCAPE_RESULT)); + if (msg->re_matches[ndx]) + result_append(result, msg->re_matches[ndx], strlen(msg->re_matches[ndx]), !!(flags & MF_ESCAPE_RESULT)); return TRUE; } -- Bazsi