This is the implementation of the key rewriting feature, along with a couple of other enhancements that were required for the functionality. The first change is that the key() statement now works very differently: instead of being a wrapper around pair(), it is now a glob-accepting include statement instead (with keeping the name). As such, it no longer guarantees that a given key will appear in the final set, as excludes still have a chance to override it. key() and exclude() patterns are now matched in order, and the last one wins. The only way to guarantee a key into a set, in spite of any exclude() rules, is via pair(). The key-value pairs specified by pair() are still inserted into the set afterwards. The key rewriting itself is done via the new rekey() statement, to be used under the key() statement, along these lines: value-pairs( key(".cee.*" rekey(shift(4) add-prefix("Events") replace("Events.some_field" "a_new_name") ) ) ) Similarly, the same thing can be done with format-json (without the line breaks): $(format-json --key .cee.* --rekey --shift 4 --add-prefix Events --replace Events.some_field=a_new_name) The transformations listed under rekey() are applied to each key matched by the glob specified for the outer key() statement, and the transformations are applied in the order they're listed in the configuration. This patch implements the following translations: shift($AMOUNT), which simply shifts the key $AMOUNT characters right; add-prefix($PREFIX) which does as the name suggests; and replace($PREFIX, $NEW_PREFIX), which replaces a substring at the start of the key, with another. The replace() transformation does not support replacing anything other than a prefix. Signed-off-by: Gergely Nagy <algernon@balabit.hu> --- lib/Makefile.am | 4 +- lib/cfg-grammar.y | 34 ++++- lib/cfg-parser.c | 4 + lib/value-pairs.c | 238 ++++++++++++++++++++++++---- lib/value-pairs.h | 6 +- lib/vptransform.c | 269 ++++++++++++++++++++++++++++++++ lib/vptransform.h | 42 +++++ modules/afmongodb/afmongodb-grammar.ym | 3 +- modules/afmongodb/afmongodb.c | 12 +- tests/unit/test_value_pairs.c | 47 ++++-- 10 files changed, 598 insertions(+), 61 deletions(-) create mode 100644 lib/vptransform.c create mode 100644 lib/vptransform.h diff --git a/lib/Makefile.am b/lib/Makefile.am index 7faf241..ef8403b 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -79,7 +79,8 @@ pkginclude_HEADERS = \ tlscontext.h \ tlstransport.h \ utils.h \ - value-pairs.h + value-pairs.h \ + vptransform.h # this is intentionally formatted so conflicts are less likely to arise. one name in every line. libsyslog_ng_crypto_la_SOURCES = \ @@ -146,6 +147,7 @@ libsyslog_ng_la_SOURCES = \ timeutils.c \ utils.c \ value-pairs.c \ + vptransform.c \ \ cfg-lex.l \ cfg-grammar.y \ diff --git a/lib/cfg-grammar.y b/lib/cfg-grammar.y index 46c267d..e8b32b1 100644 --- a/lib/cfg-grammar.y +++ b/lib/cfg-grammar.y @@ -285,6 +285,10 @@ extern struct _LogDriver *last_driver; %token KW_PAIR 10503 %token KW_KEY 10504 %token KW_SCOPE 10505 +%token KW_SHIFT 10506 +%token KW_REKEY 10507 +%token KW_ADD_PREFIX 10508 +%token KW_REPLACE 10509 /* END_DECLS */ @@ -301,6 +305,7 @@ extern struct _LogDriver *last_driver; #include "logparser.h" #include "logrewrite.h" #include "value-pairs.h" +#include "vptransform.h" #include "filter-expr-parser.h" #include "rewrite-expr-parser.h" #include "block-ref-parser.h" @@ -331,6 +336,7 @@ GList *last_parser_expr; FilterExprNode *last_filter_expr; CfgArgs *last_block_args; ValuePairs *last_value_pairs; +ValuePairsTransformSet *last_vp_transset; } @@ -881,13 +887,16 @@ vp_options vp_option : KW_PAIR '(' string ':' string ')' { value_pairs_add_pair(last_value_pairs, configuration, $3, $5); free($3); free($5); } | KW_PAIR '(' string string ')' { value_pairs_add_pair(last_value_pairs, configuration, $3, $4); free($3); free($4); } - | KW_KEY '(' string ')' { - gchar *k = g_strconcat("$", $3, NULL); - value_pairs_add_pair(last_value_pairs, configuration, $3, k); - g_free(k); - free($3); - } - | KW_EXCLUDE '(' string ')' { value_pairs_add_exclude_glob(last_value_pairs, $3); free($3); } + | KW_KEY '(' string KW_REKEY '(' + { + last_vp_transset = value_pairs_transform_set_new($3); + value_pairs_add_glob_pattern(last_value_pairs, $3, TRUE); + free($3); + } + vp_rekey_options + ')' { value_pairs_add_transforms(last_value_pairs, last_vp_transset); } ')' + | KW_KEY '(' string ')' { value_pairs_add_glob_pattern(last_value_pairs, $3, TRUE); free($3); } + | KW_EXCLUDE '(' string ')' { value_pairs_add_glob_pattern(last_value_pairs, $3, FALSE); free($3); } | KW_SCOPE '(' vp_scope_list ')' ; @@ -896,6 +905,17 @@ vp_scope_list | ; +vp_rekey_options + : vp_rekey_option vp_rekey_options + | + ; + +vp_rekey_option + : KW_SHIFT '(' LL_NUMBER ')' { value_pairs_transform_set_add_func(last_vp_transset, value_pairs_new_transform_shift($3)); } + | KW_ADD_PREFIX '(' string ')' { value_pairs_transform_set_add_func(last_vp_transset, value_pairs_new_transform_add_prefix($3)); free($3); } + | KW_REPLACE '(' string string ')' { value_pairs_transform_set_add_func(last_vp_transset, value_pairs_new_transform_replace($3, $4)); free($3); free($4); } + ; + /* END_RULES */ diff --git a/lib/cfg-parser.c b/lib/cfg-parser.c index 589206a..bd5c76d 100644 --- a/lib/cfg-parser.c +++ b/lib/cfg-parser.c @@ -60,6 +60,10 @@ static CfgLexerKeyword main_keywords[] = { { "pair", KW_PAIR, 0x0303 }, { "key", KW_KEY, 0x0303 }, { "scope", KW_SCOPE, 0x0303 }, + { "rekey", KW_REKEY, 0x0303 }, + { "shift", KW_SHIFT, 0x0303 }, + { "add_prefix", KW_ADD_PREFIX, 0x0303 }, + { "replace", KW_REPLACE, 0x0303 }, /* option items */ { "flags", KW_FLAGS }, diff --git a/lib/value-pairs.c b/lib/value-pairs.c index c58d294..7f230f7 100644 --- a/lib/value-pairs.c +++ b/lib/value-pairs.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002-2011 BalaBit IT Ltd, Budapest, Hungary + * Copyright (c) 2011 BalaBit IT Ltd, Budapest, Hungary * Copyright (c) 2011 Gergely Nagy <algernon@balabit.hu> * * This library is free software; you can redistribute it and/or @@ -23,23 +23,31 @@ */ #include "value-pairs.h" +#include "vptransform.h" #include "logmsg.h" #include "templates.h" #include "cfg-parser.h" #include "misc.h" #include "scratch-buffers.h" +#include <stdlib.h> #include <string.h> +typedef struct +{ + GPatternSpec *pattern; + gboolean include; +} VPPatternSpec; struct _ValuePairs { - GPatternSpec **excludes; + VPPatternSpec **patterns; GHashTable *vpairs; + GList *transforms; /* guint32 as CfgFlagHandler only supports 32 bit integers */ guint32 scopes; - guint32 exclude_size; + guint32 patterns_size; }; typedef enum @@ -121,13 +129,20 @@ value_pairs_add_scope(ValuePairs *vp, const gchar *scope) } void -value_pairs_add_exclude_glob(ValuePairs *vp, const gchar *pattern) +value_pairs_add_glob_pattern(ValuePairs *vp, const gchar *pattern, + gboolean include) { gint i; + VPPatternSpec *p; + + i = vp->patterns_size++; + vp->patterns = g_renew(VPPatternSpec *, vp->patterns, vp->patterns_size); + + p = g_new(VPPatternSpec, 1); + p->pattern = g_pattern_spec_new(pattern); + p->include = include; - i = vp->exclude_size++; - vp->excludes = g_renew(GPatternSpec *, vp->excludes, vp->exclude_size); - vp->excludes[i] = g_pattern_spec_new(pattern); + vp->patterns[i] = p; } void @@ -140,10 +155,31 @@ value_pairs_add_pair(ValuePairs *vp, GlobalConfig *cfg, const gchar *key, const g_hash_table_insert(vp->vpairs, g_strdup(key), t); } +static gchar * +vp_transform_apply (ValuePairs *vp, gchar *key) +{ + GList *l; + gchar *ckey = key; + + if (!vp->transforms) + return ckey; + + l = vp->transforms; + while (l) + { + ValuePairsTransformSet *t = (ValuePairsTransformSet *)l->data; + ckey = value_pairs_transform_set_apply(t, ckey); + l = g_list_next (l); + } + + return ckey; +} + /* runs over the name-value pairs requested by the user (e.g. with value_pairs_add_pair) */ static void vp_pairs_foreach(gpointer key, gpointer value, gpointer user_data) { + ValuePairs *vp = ((gpointer *)user_data)[0]; LogMessage *msg = ((gpointer *)user_data)[2]; gint32 seq_num = GPOINTER_TO_INT (((gpointer *)user_data)[3]); GHashTable *scope_set = ((gpointer *)user_data)[5]; @@ -158,7 +194,7 @@ vp_pairs_foreach(gpointer key, gpointer value, gpointer user_data) return; } - g_hash_table_insert(scope_set, key, sb_string(sb)->str); + g_hash_table_insert(scope_set, vp_transform_apply (vp, key), sb_string(sb)->str); g_string_steal(sb_string(sb)); scratch_buffer_release(sb); } @@ -172,20 +208,22 @@ vp_msg_nvpairs_foreach(NVHandle handle, gchar *name, ValuePairs *vp = ((gpointer *)user_data)[0]; GHashTable *scope_set = ((gpointer *)user_data)[5]; gint j; + gboolean inc = FALSE; + + for (j = 0; j < vp->patterns_size; j++) + { + if (g_pattern_match_string(vp->patterns[j]->pattern, name)) + inc = vp->patterns[j]->include; + } /* NOTE: dot-nv-pairs include SDATA too */ if (((name[0] == '.' && (vp->scopes & VPS_DOT_NV_PAIRS)) || (name[0] != '.' && (vp->scopes & VPS_NV_PAIRS)) || - (log_msg_is_handle_sdata(handle) && (vp->scopes & VPS_SDATA)))) + (log_msg_is_handle_sdata(handle) && (vp->scopes & VPS_SDATA))) || + inc) { - for (j = 0; j < vp->exclude_size; j++) - { - if (g_pattern_match_string(vp->excludes[j], name)) - return FALSE; - } - /* NOTE: the key is a borrowed reference in the hash, and value is freed */ - g_hash_table_insert(scope_set, name, g_strndup(value, value_len)); + g_hash_table_insert(scope_set, vp_transform_apply(vp, name), g_strndup(value, value_len)); } return FALSE; @@ -203,10 +241,10 @@ vp_merge_set(ValuePairs *vp, LogMessage *msg, gint32 seq_num, ValuePairSpec *set gint j; gboolean exclude = FALSE; - for (j = 0; j < vp->exclude_size; j++) + for (j = 0; j < vp->patterns_size; j++) { - if (g_pattern_match_string(vp->excludes[j], set[i].name)) - exclude = TRUE; + if (g_pattern_match_string(vp->patterns[j]->pattern, set[i].name)) + exclude = !vp->patterns[j]->include; } if (exclude) @@ -233,7 +271,7 @@ vp_merge_set(ValuePairs *vp, LogMessage *msg, gint32 seq_num, ValuePairSpec *set if (!sb_string(sb)->str[0]) continue; - g_hash_table_insert(dest, set[i].name, sb_string(sb)->str); + g_hash_table_insert(dest, vp_transform_apply(vp, set[i].name), sb_string(sb)->str); g_string_steal(sb_string(sb)); } scratch_buffer_release(sb); @@ -246,7 +284,8 @@ value_pairs_foreach (ValuePairs *vp, VPForeachFunc func, gpointer args[] = { vp, func, msg, GINT_TO_POINTER (seq_num), user_data, NULL }; GHashTable *scope_set; - scope_set = g_hash_table_new_full(g_str_hash, g_str_equal, NULL, + scope_set = g_hash_table_new_full(g_str_hash, g_str_equal, + (GDestroyNotify) g_free, (GDestroyNotify) g_free); args[5] = scope_set; @@ -254,7 +293,8 @@ value_pairs_foreach (ValuePairs *vp, VPForeachFunc func, /* * Build up the base set */ - if (vp->scopes & (VPS_NV_PAIRS + VPS_DOT_NV_PAIRS + VPS_SDATA)) + if (vp->scopes & (VPS_NV_PAIRS + VPS_DOT_NV_PAIRS + VPS_SDATA) || + vp->patterns_size > 0) nv_table_foreach(msg->payload, logmsg_registry, (NVTableForeachFunc) vp_msg_nvpairs_foreach, args); @@ -351,15 +391,48 @@ void value_pairs_free (ValuePairs *vp) { gint i; + GList *l; g_hash_table_destroy(vp->vpairs); - for (i = 0; i < vp->exclude_size; i++) - g_pattern_spec_free(vp->excludes[i]); - g_free(vp->excludes); + for (i = 0; i < vp->patterns_size; i++) + { + g_pattern_spec_free(vp->patterns[i]->pattern); + g_free(vp->patterns[i]); + } + g_free(vp->patterns); + + l = vp->transforms; + while (l) + { + value_pairs_transform_set_free((ValuePairsTransformSet *)l->data); + + l = g_list_delete_link (l, l); + } + g_free(vp); } +void +value_pairs_add_transforms(ValuePairs *vp, gpointer *vpts) +{ + vp->transforms = g_list_append(vp->transforms, vpts); +} + +static void +vp_cmdline_parse_rekey_finish (gpointer data) +{ + gpointer *args = (gpointer *) data; + ValuePairs *vp = (ValuePairs *) args[1]; + ValuePairsTransformSet *vpts = (ValuePairsTransformSet *) args[2]; + + if (vpts) + value_pairs_add_transforms (vp, args[2]); + args[2] = NULL; + g_free(args[3]); + args[3] = NULL; +} + /* parse a value-pair specification from a command-line like environment */ static gboolean vp_cmdline_parse_scope(const gchar *option_name, const gchar *value, @@ -368,6 +441,8 @@ vp_cmdline_parse_scope(const gchar *option_name, const gchar *value, gpointer *args = (gpointer *) data; ValuePairs *vp = (ValuePairs *) args[1]; + vp_cmdline_parse_rekey_finish (data); + if (!value_pairs_add_scope (vp, value)) { g_set_error (error, G_OPTION_ERROR, G_OPTION_ERROR_BAD_VALUE, @@ -384,7 +459,8 @@ vp_cmdline_parse_exclude(const gchar *option_name, const gchar *value, gpointer *args = (gpointer *) data; ValuePairs *vp = (ValuePairs *) args[1]; - value_pairs_add_exclude_glob(vp, value); + vp_cmdline_parse_rekey_finish (data); + value_pairs_add_glob_pattern(vp, value, FALSE); return TRUE; } @@ -394,12 +470,10 @@ vp_cmdline_parse_key(const gchar *option_name, const gchar *value, { gpointer *args = (gpointer *) data; ValuePairs *vp = (ValuePairs *) args[1]; - GlobalConfig *cfg = (GlobalConfig *) args[0]; - gchar *k = g_strconcat ("$", value, NULL); - - value_pairs_add_pair(vp, cfg, value, k); - g_free (k); + vp_cmdline_parse_rekey_finish (data); + value_pairs_add_glob_pattern(vp, value, TRUE); + args[3] = g_strdup(value); return TRUE; } @@ -412,6 +486,7 @@ vp_cmdline_parse_pair (const gchar *option_name, const gchar *value, GlobalConfig *cfg = (GlobalConfig *) args[0]; gchar **kv; + vp_cmdline_parse_rekey_finish (data); if (!g_strstr_len (value, strlen (value), "=")) { g_set_error (error, G_OPTION_ERROR, G_OPTION_ERROR_BAD_VALUE, @@ -429,6 +504,92 @@ vp_cmdline_parse_pair (const gchar *option_name, const gchar *value, return TRUE; } +static gboolean +vp_cmdline_parse_rekey (const gchar *option_name, const gchar *value, + gpointer data, GError **error) +{ + gpointer *args = (gpointer *) data; + ValuePairsTransformSet *vpts = (ValuePairsTransformSet *) args[2]; + gchar *key = (gchar *) args[3]; + + vpts = value_pairs_transform_set_new (key); + vp_cmdline_parse_rekey_finish (data); + + args[2] = vpts; + return TRUE; +} + +static gboolean +vp_cmdline_parse_rekey_replace (const gchar *option_name, const gchar *value, + gpointer data, GError **error) +{ + gpointer *args = (gpointer *) data; + ValuePairsTransformSet *vpts = (ValuePairsTransformSet *) args[2]; + gchar **kv; + + if (!vpts) + { + g_set_error (error, G_OPTION_ERROR, G_OPTION_ERROR_FAILED, + "Error parsing value-pairs: --replace used without --rekey"); + return FALSE; + } + + if (!g_strstr_len (value, strlen (value), "=")) + { + g_set_error (error, G_OPTION_ERROR, G_OPTION_ERROR_BAD_VALUE, + "Error parsing value-pairs' rekey replace construct"); + return FALSE; + } + + kv = g_strsplit(value, "=", 2); + value_pairs_transform_set_add_func + (vpts, value_pairs_new_transform_replace (kv[0], kv[1])); + + g_free (kv[0]); + g_free (kv[1]); + g_free (kv); + + return TRUE; +} + +static gboolean +vp_cmdline_parse_rekey_add_prefix (const gchar *option_name, const gchar *value, + gpointer data, GError **error) +{ + gpointer *args = (gpointer *) data; + ValuePairsTransformSet *vpts = (ValuePairsTransformSet *) args[2]; + + if (!vpts) + { + g_set_error (error, G_OPTION_ERROR, G_OPTION_ERROR_FAILED, + "Error parsing value-pairs: --add-prefix used without --rekey"); + return FALSE; + } + + value_pairs_transform_set_add_func + (vpts, value_pairs_new_transform_add_prefix (value)); + return TRUE; +} + +static gboolean +vp_cmdline_parse_rekey_shift (const gchar *option_name, const gchar *value, + gpointer data, GError **error) +{ + gpointer *args = (gpointer *) data; + ValuePairsTransformSet *vpts = (ValuePairsTransformSet *) args[2]; + + if (!vpts) + { + g_set_error (error, G_OPTION_ERROR, G_OPTION_ERROR_FAILED, + "Error parsing value-pairs: --shift used without --rekey"); + return FALSE; + } + + value_pairs_transform_set_add_func + (vpts, value_pairs_new_transform_shift (atoi (value))); + return TRUE; +} + ValuePairs * value_pairs_new_from_cmdline (GlobalConfig *cfg, gint argc, gchar **argv, @@ -436,6 +597,7 @@ value_pairs_new_from_cmdline (GlobalConfig *cfg, { ValuePairs *vp; GOptionContext *ctx; + GOptionEntry vp_options[] = { { "scope", 's', 0, G_OPTION_ARG_CALLBACK, vp_cmdline_parse_scope, NULL, NULL }, @@ -445,16 +607,26 @@ value_pairs_new_from_cmdline (GlobalConfig *cfg, NULL, NULL }, { "pair", 'p', 0, G_OPTION_ARG_CALLBACK, vp_cmdline_parse_pair, NULL, NULL }, + { "rekey", 'r', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, + vp_cmdline_parse_rekey, NULL, NULL }, + { "shift", 'S', 0, G_OPTION_ARG_CALLBACK, vp_cmdline_parse_rekey_shift, + NULL, NULL }, + { "add-prefix", 'A', 0, G_OPTION_ARG_CALLBACK, vp_cmdline_parse_rekey_add_prefix, + NULL, NULL }, + { "replace", 'R', 0, G_OPTION_ARG_CALLBACK, vp_cmdline_parse_rekey_replace, + NULL, NULL }, { G_OPTION_REMAINING, 0, 0, G_OPTION_ARG_CALLBACK, vp_cmdline_parse_pair, NULL, NULL }, { NULL } }; GOptionGroup *og; - gpointer user_data_args[2]; + gpointer user_data_args[4]; vp = value_pairs_new(); user_data_args[0] = cfg; user_data_args[1] = vp; + user_data_args[2] = NULL; + user_data_args[3] = NULL; ctx = g_option_context_new ("value-pairs"); og = g_option_group_new (NULL, NULL, NULL, user_data_args, NULL); @@ -463,11 +635,13 @@ value_pairs_new_from_cmdline (GlobalConfig *cfg, if (!g_option_context_parse (ctx, &argc, &argv, error)) { + g_free(user_data_args[3]); value_pairs_free (vp); g_option_context_free (ctx); return NULL; } g_option_context_free (ctx); + vp_cmdline_parse_rekey_finish (user_data_args); return vp; } diff --git a/lib/value-pairs.h b/lib/value-pairs.h index da26763..9b77266 100644 --- a/lib/value-pairs.h +++ b/lib/value-pairs.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2002-2011 BalaBit IT Ltd, Budapest, Hungary + * Copyright (c) 2011 BalaBit IT Ltd, Budapest, Hungary * Copyright (c) 2011 Gergely Nagy <algernon@balabit.hu> * * This library is free software; you can redistribute it and/or @@ -32,9 +32,11 @@ typedef struct _ValuePairs ValuePairs; typedef gboolean (*VPForeachFunc)(const gchar *name, const gchar *value, gpointer user_data); gboolean value_pairs_add_scope(ValuePairs *vp, const gchar *scope); -void value_pairs_add_exclude_glob(ValuePairs *vp, const gchar *pattern); +void value_pairs_add_glob_pattern(ValuePairs *vp, const gchar *pattern, gboolean include); void value_pairs_add_pair(ValuePairs *vp, GlobalConfig *cfg, const gchar *key, const gchar *value); +void value_pairs_add_transforms(ValuePairs *vp, gpointer *vpts); + void value_pairs_foreach(ValuePairs *vp, VPForeachFunc func, LogMessage *msg, gint32 seq_num, gpointer user_data); diff --git a/lib/vptransform.c b/lib/vptransform.c new file mode 100644 index 0000000..c371c91 --- /dev/null +++ b/lib/vptransform.c @@ -0,0 +1,269 @@ +/* + * Copyright (c) 2011 BalaBit IT Ltd, Budapest, Hungary + * Copyright (c) 2011 Gergely Nagy <algernon@balabit.hu> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As an additional exemption you are allowed to compile & link against the + * OpenSSL libraries as published by the OpenSSL project. See the file + * COPYING for details. + * + */ + +#include "value-pairs.h" +#include "vptransform.h" +#include "templates.h" +#include "cfg-parser.h" +#include "misc.h" +#include "scratch-buffers.h" + +#include <string.h> + +typedef gchar *(*VPTransFunc)(ValuePairsTransform *t, gchar *name, + ScratchBuffer *sb); +typedef void (*VPTransDestroyFunc)(ValuePairsTransform *t); + +struct _ValuePairsTransformSet +{ + GPatternSpec *pattern; + + GList *transforms; +}; + +struct _ValuePairsTransform +{ + VPTransFunc transform; + VPTransDestroyFunc destroy; +}; + +typedef struct +{ + ValuePairsTransform super; + + gint amount; +} VPTransShift; + +typedef struct +{ + ValuePairsTransform super; + + gchar *prefix; +} VPTransAddPrefix; + +typedef struct +{ + ValuePairsTransform super; + + gchar *old_prefix; + gchar *new_prefix; + gint new_prefix_len; + gint old_prefix_len; +} VPTransReplace; + +static void +vp_trans_init(ValuePairsTransform *t, + VPTransFunc trans, + VPTransDestroyFunc dest) +{ + if (!t) + return; + + t->transform = trans; + t->destroy = dest; +} + +static inline void +value_pairs_transform_free(ValuePairsTransform *t) +{ + if (t->destroy) + t->destroy(t); + g_free(t); +} + +static inline gchar * +value_pairs_transform_apply(ValuePairsTransform *t, gchar *key, + ScratchBuffer *sb) +{ + return t->transform(t, key, sb); +} + +/* add_prefix() */ + +static gchar * +vp_trans_add_prefix(ValuePairsTransform *t, gchar *name, ScratchBuffer *sb) +{ + VPTransAddPrefix *self = (VPTransAddPrefix *)t; + + g_string_assign(sb_string(sb), self->prefix); + g_string_append(sb_string(sb), name); + return NULL; +} + +static void +vp_trans_add_prefix_destroy(ValuePairsTransform *t) +{ + VPTransAddPrefix *self = (VPTransAddPrefix *)t; + + g_free(self->prefix); +} + +ValuePairsTransform * +value_pairs_new_transform_add_prefix (const gchar *prefix) +{ + VPTransAddPrefix *vpt; + + vpt = g_new(VPTransAddPrefix, 1); + vp_trans_init((ValuePairsTransform *)vpt, + vp_trans_add_prefix, + vp_trans_add_prefix_destroy); + vpt->prefix = g_strdup(prefix); + + return (ValuePairsTransform *)vpt; +} + +/* shift() */ + +static gchar * +vp_trans_shift(ValuePairsTransform *t, gchar* name, ScratchBuffer *sb) +{ + VPTransShift *self = (VPTransShift *)t; + + if (self->amount <= 0) + return name; + return name + self->amount; +} + +ValuePairsTransform * +value_pairs_new_transform_shift (gint amount) +{ + VPTransShift *vpt; + + vpt = g_new(VPTransShift, 1); + vp_trans_init((ValuePairsTransform *)vpt, + vp_trans_shift, NULL); + + vpt->amount = amount; + + return (ValuePairsTransform *)vpt; +} + +/* replace() */ + +static gchar * +vp_trans_replace(ValuePairsTransform *t, gchar *name, ScratchBuffer *sb) +{ + VPTransReplace *self = (VPTransReplace *)t; + + if (strncmp(self->old_prefix, name, self->old_prefix_len) != 0) + return name; + + g_string_append_len(sb_string(sb), self->new_prefix, self->new_prefix_len); + g_string_append(sb_string(sb), name + self->old_prefix_len); + return NULL; +} + +static void +vp_trans_replace_destroy(ValuePairsTransform *t) +{ + VPTransReplace *self = (VPTransReplace *)t; + + g_free(self->old_prefix); + g_free(self->new_prefix); +} + +ValuePairsTransform * +value_pairs_new_transform_replace(const gchar *prefix, const gchar *new_prefix) +{ + VPTransReplace *vpt; + + vpt = g_new(VPTransReplace, 1); + vp_trans_init((ValuePairsTransform *)vpt, + vp_trans_replace, + vp_trans_replace_destroy); + + vpt->old_prefix = g_strdup(prefix); + vpt->old_prefix_len = strlen(prefix); + vpt->new_prefix = g_strdup(new_prefix); + vpt->new_prefix_len = strlen(vpt->new_prefix); + + return (ValuePairsTransform *)vpt; +} + +/* + * ValuePairsTransformSet + */ + +ValuePairsTransformSet * +value_pairs_transform_set_new(const gchar *glob) +{ + ValuePairsTransformSet *vpts; + + vpts = g_new(ValuePairsTransformSet, 1); + vpts->transforms = NULL; + vpts->pattern = g_pattern_spec_new(glob); + + return vpts; +} + +void +value_pairs_transform_set_add_func(ValuePairsTransformSet *vpts, + ValuePairsTransform *vpt) +{ + vpts->transforms = g_list_append(vpts->transforms, vpt); +} + +void +value_pairs_transform_set_free(ValuePairsTransformSet *vpts) +{ + GList *l; + + l = vpts->transforms; + while (l) + { + value_pairs_transform_free((ValuePairsTransform *)l->data); + l = g_list_delete_link(l, l); + } + g_pattern_spec_free(vpts->pattern); + g_free(vpts); +} + +gchar * +value_pairs_transform_set_apply(ValuePairsTransformSet *vpts, gchar *key) +{ + if (g_pattern_match_string(vpts->pattern, key)) + { + gchar *current_key = key, *new_key = NULL; + GList *l; + ScratchBuffer *sb; + + sb = scratch_buffer_acquire (); + + l = vpts->transforms; + while (l) + { + current_key = value_pairs_transform_apply((ValuePairsTransform *)l->data, + current_key, sb); + if (!current_key) + current_key = sb_string(sb)->str; + l = l->next; + } + + new_key = g_strdup(current_key); + scratch_buffer_release (sb); + + return new_key; + } + return g_strdup(key); +} diff --git a/lib/vptransform.h b/lib/vptransform.h new file mode 100644 index 0000000..764a8c4 --- /dev/null +++ b/lib/vptransform.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2011 BalaBit IT Ltd, Budapest, Hungary + * Copyright (c) 2011 Gergely Nagy <algernon@balabit.hu> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As an additional exemption you are allowed to compile & link against the + * OpenSSL libraries as published by the OpenSSL project. See the file + * COPYING for details. + * + */ + +#ifndef VPTRANSFORM_INCLUDED +#define VPTRANSFORM_INCLUDED 1 + +#include "value-pairs.h" + +typedef struct _ValuePairsTransform ValuePairsTransform; +typedef struct _ValuePairsTransformSet ValuePairsTransformSet; + +ValuePairsTransform *value_pairs_new_transform_add_prefix (const gchar *prefix); +ValuePairsTransform *value_pairs_new_transform_shift (gint amount); +ValuePairsTransform *value_pairs_new_transform_replace(const gchar *prefix, const gchar *new_prefix); + +ValuePairsTransformSet *value_pairs_transform_set_new(const gchar *glob); +void value_pairs_transform_set_add_func(ValuePairsTransformSet *vpts, ValuePairsTransform *vpt); +void value_pairs_transform_set_free(ValuePairsTransformSet *vpts); +gchar *value_pairs_transform_set_apply(ValuePairsTransformSet *vpts, gchar *key); + +#endif diff --git a/modules/afmongodb/afmongodb-grammar.ym b/modules/afmongodb/afmongodb-grammar.ym index 7da664c..36d69c4 100644 --- a/modules/afmongodb/afmongodb-grammar.ym +++ b/modules/afmongodb/afmongodb-grammar.ym @@ -32,10 +32,11 @@ #include "cfg-parser.h" #include "afmongodb-grammar.h" #include "plugin.h" +#include "vptransform.h" extern LogDriver *last_driver; extern ValuePairs *last_value_pairs; - +extern ValuePairsTransformSet *last_vp_transset; } %name-prefix "afmongodb_" diff --git a/modules/afmongodb/afmongodb.c b/modules/afmongodb/afmongodb.c index 9f75bf3..0314481 100644 --- a/modules/afmongodb/afmongodb.c +++ b/modules/afmongodb/afmongodb.c @@ -424,12 +424,12 @@ afmongodb_dd_init(LogPipe *s) self->vp = value_pairs_new(); value_pairs_add_scope(self->vp, "selected-macros"); value_pairs_add_scope(self->vp, "nv-pairs"); - value_pairs_add_exclude_glob(self->vp, "R_*"); - value_pairs_add_exclude_glob(self->vp, "S_*"); - value_pairs_add_exclude_glob(self->vp, "HOST_FROM"); - value_pairs_add_exclude_glob(self->vp, "LEGACY_MSGHDR"); - value_pairs_add_exclude_glob(self->vp, "MSG"); - value_pairs_add_exclude_glob(self->vp, "SDATA"); + value_pairs_add_glob_pattern(self->vp, "R_*", FALSE); + value_pairs_add_glob_pattern(self->vp, "S_*", FALSE); + value_pairs_add_glob_pattern(self->vp, "HOST_FROM", FALSE); + value_pairs_add_glob_pattern(self->vp, "LEGACY_MSGHDR", FALSE); + value_pairs_add_glob_pattern(self->vp, "MSG", FALSE); + value_pairs_add_glob_pattern(self->vp, "SDATA", FALSE); } msg_verbose("Initializing MongoDB destination", diff --git a/tests/unit/test_value_pairs.c b/tests/unit/test_value_pairs.c index 8855c91..9cd6140 100644 --- a/tests/unit/test_value_pairs.c +++ b/tests/unit/test_value_pairs.c @@ -1,4 +1,5 @@ #include "value-pairs.h" +#include "vptransform.h" #include "logmsg.h" #include "apphook.h" #include "cfg.h" @@ -46,7 +47,7 @@ create_message(void) } void -testcase(const gchar *scope, const gchar *exclude, const gchar *expected) +testcase(const gchar *scope, const gchar *exclude, const gchar *expected, GPtrArray *transformers) { ValuePairs *vp; GList *vp_keys_list = NULL; @@ -60,9 +61,19 @@ testcase(const gchar *scope, const gchar *exclude, const gchar *expected) vp = value_pairs_new(); value_pairs_add_scope(vp, scope); if (exclude) - value_pairs_add_exclude_glob(vp, exclude); + value_pairs_add_glob_pattern(vp, exclude, FALSE); value_pairs_add_pair(vp, configuration, "test.key", "$MESSAGE"); + if (transformers) + { + gint i; + ValuePairsTransformSet *vpts = value_pairs_transform_set_new("*"); + + for (i = 0; i < transformers->len; i++) + value_pairs_transform_set_add_func(vpts, g_ptr_array_index(transformers, i)); + value_pairs_add_transforms(vp, (gpointer *)vpts); + } + args[0] = &vp_keys_list; args[1] = &test_key_found; value_pairs_foreach(vp, vp_keys_foreach, msg, 11, args); @@ -88,6 +99,8 @@ testcase(const gchar *scope, const gchar *exclude, const gchar *expected) int main(int argc, char *argv[]) { + GPtrArray *transformers; + app_startup(); putenv("TZ=MET-1METDST"); tzset(); @@ -98,26 +111,36 @@ main(int argc, char *argv[]) msg_format_options_init(&parse_options, configuration); parse_options.flags |= LP_SYSLOG_PROTOCOL; - testcase("rfc3164", NULL, "DATE,FACILITY,HOST,MESSAGE,PID,PRIORITY,PROGRAM"); - testcase("rfc5424", NULL, "DATE,FACILITY,HOST,MESSAGE,MSGID,PID,PRIORITY,PROGRAM"); - testcase("selected-macros", NULL, "DATE,FACILITY,HOST,MESSAGE,PID,PRIORITY,PROGRAM,SEQNUM,SOURCEIP,TAGS"); + testcase("rfc3164", NULL, "DATE,FACILITY,HOST,MESSAGE,PID,PRIORITY,PROGRAM", NULL); + testcase("rfc5424", NULL, "DATE,FACILITY,HOST,MESSAGE,MSGID,PID,PRIORITY,PROGRAM", NULL); + testcase("selected-macros", NULL, "DATE,FACILITY,HOST,MESSAGE,PID,PRIORITY,PROGRAM,SEQNUM,SOURCEIP,TAGS", NULL); - testcase("nv-pairs", NULL, "HOST,MESSAGE,MSGID,PID,PROGRAM"); - testcase("dot-nv-pairs", NULL, ".SDATA.EventData@18372.4.Data,.SDATA.Keywords@18372.4.Keyword,.SDATA.meta.sequenceId,.SDATA.meta.sysUpTime,.SDATA.origin.ip"); + testcase("nv-pairs", NULL, "HOST,MESSAGE,MSGID,PID,PROGRAM", NULL); + testcase("dot-nv-pairs", NULL, ".SDATA.EventData@18372.4.Data,.SDATA.Keywords@18372.4.Keyword,.SDATA.meta.sequenceId,.SDATA.meta.sysUpTime,.SDATA.origin.ip", NULL); - testcase("sdata", NULL, ".SDATA.EventData@18372.4.Data,.SDATA.Keywords@18372.4.Keyword,.SDATA.meta.sequenceId,.SDATA.meta.sysUpTime,.SDATA.origin.ip"); + testcase("sdata", NULL, ".SDATA.EventData@18372.4.Data,.SDATA.Keywords@18372.4.Keyword,.SDATA.meta.sequenceId,.SDATA.meta.sysUpTime,.SDATA.origin.ip", NULL); - testcase("all-nv-pairs", NULL, ".SDATA.EventData@18372.4.Data,.SDATA.Keywords@18372.4.Keyword,.SDATA.meta.sequenceId,.SDATA.meta.sysUpTime,.SDATA.origin.ip,HOST,MESSAGE,MSGID,PID,PROGRAM"); + testcase("all-nv-pairs", NULL, ".SDATA.EventData@18372.4.Data,.SDATA.Keywords@18372.4.Keyword,.SDATA.meta.sequenceId,.SDATA.meta.sysUpTime,.SDATA.origin.ip,HOST,MESSAGE,MSGID,PID,PROGRAM", NULL); - testcase("everything", NULL, ".SDATA.EventData@18372.4.Data,.SDATA.Keywords@18372.4.Keyword,.SDATA.meta.sequenceId,.SDATA.meta.sysUpTime,.SDATA.origin.ip,BSDTAG,C_DATE,C_DAY,C_FULLDATE,C_HOUR,C_ISODATE,C_MIN,C_MONTH,C_MONTH_ABBREV,C_MONTH_NAME,C_MONTH_WEEK,C_SEC,C_STAMP,C_TZ,C_TZOFFSET,C_UNIXTIME,C_WEEK,C_WEEKDAY,C_WEEK_DAY,C_WEEK_DAY_ABBREV,C_WEEK_DAY_NAME,C_YEAR,C_YEAR_DAY,DATE,DAY,FACILITY,FACILITY_NUM,FULLDATE,HOST,HOUR,ISODATE,LEVEL,LEVEL_NUM,LOGHOST,MESSAGE,MIN,MONTH,MONTH_ABBREV,MONTH_NAME,MONTH_WEEK,MSG,MSGHDR,MSGID,PID,PRI,PRIORITY,PROGRAM,R_DATE,R_DAY,R_FULLDATE,R_HOUR,R_ISODATE,R_MIN,R_MONTH,R_MONTH_ABBREV,R_MONTH_NAME,R_MONTH_WEEK,R_SEC,R_STAMP,R_TZ,R_TZOFFSET,R_UNIXTIME,R_WEEK,R_WEEKDAY,R_WEEK_DAY,R_WEEK_DAY_ABBREV,R_WEEK_DAY_NAME,R_YEAR,R_YEAR_DAY,SDATA,SEC,SEQNUM,SOURCEIP,STAMP,S_DATE,S_DAY,S_FULLDATE,S_HOUR,S_ISODATE,S_MIN,S_MONTH,S_MONTH_ABBREV,S_MONTH_NAME,S_MONTH_WEEK,S_SEC,S_STAMP,S_TZ,S_TZOFFSET,S_UNIXTIME,S_WEEK,S_WEEKDAY,S_WEEK_DAY,S_WEEK_DAY_ABBREV, S_WEEK_DAY_NAME,S_YEAR,S_YEAR_DAY,TAG,TAGS,TZ,TZOFFSET,UNIXTIME,WEEK,WEEKDAY,WEEK_DAY,WEEK_DAY_ABBREV,WEEK_DAY_NAME,YEAR,YEAR_DAY"); + testcase("everything", NULL, ".SDATA.EventData@18372.4.Data,.SDATA.Keywords@18372.4.Keyword,.SDATA.meta.sequenceId,.SDATA.meta.sysUpTime,.SDATA.origin.ip,BSDTAG,C_DATE,C_DAY,C_FULLDATE,C_HOUR,C_ISODATE,C_MIN,C_MONTH,C_MONTH_ABBREV,C_MONTH_NAME,C_MONTH_WEEK,C_SEC,C_STAMP,C_TZ,C_TZOFFSET,C_UNIXTIME,C_WEEK,C_WEEKDAY,C_WEEK_DAY,C_WEEK_DAY_ABBREV,C_WEEK_DAY_NAME,C_YEAR,C_YEAR_DAY,DATE,DAY,FACILITY,FACILITY_NUM,FULLDATE,HOST,HOUR,ISODATE,LEVEL,LEVEL_NUM,LOGHOST,MESSAGE,MIN,MONTH,MONTH_ABBREV,MONTH_NAME,MONTH_WEEK,MSG,MSGHDR,MSGID,PID,PRI,PRIORITY,PROGRAM,R_DATE,R_DAY,R_FULLDATE,R_HOUR,R_ISODATE,R_MIN,R_MONTH,R_MONTH_ABBREV,R_MONTH_NAME,R_MONTH_WEEK,R_SEC,R_STAMP,R_TZ,R_TZOFFSET,R_UNIXTIME,R_WEEK,R_WEEKDAY,R_WEEK_DAY,R_WEEK_DAY_ABBREV,R_WEEK_DAY_NAME,R_YEAR,R_YEAR_DAY,SDATA,SEC,SEQNUM,SOURCEIP,STAMP,S_DATE,S_DAY,S_FULLDATE,S_HOUR,S_ISODATE,S_MIN,S_MONTH,S_MONTH_ABBREV,S_MONTH_NAME,S_MONTH_WEEK,S_SEC,S_STAMP,S_TZ,S_TZOFFSET,S_UNIXTIME,S_WEEK,S_WEEKDAY,S_WEEK_DAY,S_WEEK_DAY_ABBREV, S_WEEK_DAY_NAME,S_YEAR,S_YEAR_DAY,TAG,TAGS,TZ,TZOFFSET,UNIXTIME,WEEK,WEEKDAY,WEEK_DAY,WEEK_DAY_ABBREV,WEEK_DAY_NAME,YEAR,YEAR_DAY", NULL); - testcase("nv-pairs", ".SDATA.*", "HOST,MESSAGE,MSGID,PID,PROGRAM"); + testcase("nv-pairs", ".SDATA.*", "HOST,MESSAGE,MSGID,PID,PROGRAM", NULL); /* tests that the exclude patterns do not affect explicitly added * keys. The testcase function adds a "test.key" and then checks if * it is indeed present. Even if it would be excluded it still has * to be in the result set. */ - testcase("rfc3164", "test.*", "DATE,FACILITY,HOST,MESSAGE,PID,PRIORITY,PROGRAM"); + testcase("rfc3164", "test.*", "DATE,FACILITY,HOST,MESSAGE,PID,PRIORITY,PROGRAM", NULL); + + /* test the value-pair transformators */ + transformers = g_ptr_array_new(); + g_ptr_array_add(transformers, value_pairs_new_transform_add_prefix("__")); + g_ptr_array_add(transformers, value_pairs_new_transform_shift(2)); + g_ptr_array_add(transformers, value_pairs_new_transform_replace("C_", "CC_")); + + testcase("everything", NULL, ".SDATA.EventData@18372.4.Data,.SDATA.Keywords@18372.4.Keyword,.SDATA.meta.sequenceId,.SDATA.meta.sysUpTime,.SDATA.origin.ip,BSDTAG,CC_DATE,CC_DAY,CC_FULLDATE,CC_HOUR,CC_ISODATE,CC_MIN,CC_MONTH,CC_MONTH_ABBREV,CC_MONTH_NAME,CC_MONTH_WEEK,CC_SEC,CC_STAMP,CC_TZ,CC_TZOFFSET,CC_UNIXTIME,CC_WEEK,CC_WEEKDAY,CC_WEEK_DAY,CC_WEEK_DAY_ABBREV,CC_WEEK_DAY_NAME,CC_YEAR,CC_YEAR_DAY,DATE,DAY,FACILITY,FACILITY_NUM,FULLDATE,HOST,HOUR,ISODATE,LEVEL,LEVEL_NUM,LOGHOST,MESSAGE,MIN,MONTH,MONTH_ABBREV,MONTH_NAME,MONTH_WEEK,MSG,MSGHDR,MSGID,PID,PRI,PRIORITY,PROGRAM,R_DATE,R_DAY,R_FULLDATE,R_HOUR,R_ISODATE,R_MIN,R_MONTH,R_MONTH_ABBREV,R_MONTH_NAME,R_MONTH_WEEK,R_SEC,R_STAMP,R_TZ,R_TZOFFSET,R_UNIXTIME,R_WEEK,R_WEEKDAY,R_WEEK_DAY,R_WEEK_DAY_ABBREV,R_WEEK_DAY_NAME,R_YEAR,R_YEAR_DAY,SDATA,SEC,SEQNUM,SOURCEIP,STAMP,S_DATE,S_DAY,S_FULLDATE,S_HOUR,S_ISODATE,S_MIN,S_MONTH,S_MONTH_ABBREV,S_MONTH_NAME,S_MONTH_WEEK,S_SEC,S_STAMP,S_TZ,S_TZOFFSET,S_UNIXTIME,S_WEEK,S_WEEKDAY,S_WEEK_ DAY,S_WEEK_DAY_ABBREV,S_WEEK_DAY_NAME,S_YEAR,S_YEAR_DAY,TAG,TAGS,TZ,TZOFFSET,UNIXTIME,WEEK,WEEKDAY,WEEK_DAY,WEEK_DAY_ABBREV,WEEK_DAY_NAME,YEAR,YEAR_DAY", transformers); + + g_ptr_array_free(transformers, FALSE); app_shutdown(); if (success) -- 1.7.7.3