The patches that will follow this overview implement type-hinting support into LogTemplate and value-pairs(), then extend the mongodb destination and format-json to use these hints. In the end, we'll do things like: mongodb(value-pairs(scope("selected-macros", "nv-pairs") pair("DATE", datetime("$UNIXTIME")) pair("PID", int("$PID")) pair(".bool", boolean("true")))); And this will use the appropriate types when inserting into mongodb. The $(format-json) template function works similarly: template("$(format-json --scope selected-macros,nv-pairs DATE=datetime($UNIXTIME) PID=int($PID) .bool=boolean(TRUE))\n"); I tried to make the code as performant as possible, there are no extra allocations performed, and in case there is no casting to perform, it's just an extra integer comparsion. It can be tweaked further, if so need be, but for now, I'm happy with the results. Nevertheless, there are a couple of things I'm not entirely satisfied with (namely, abusing the ScratchBuffers to store the type hint on the first byte; or the way the value-pairs CLI interface parses type hints), and I will fix them in the coming days. While using this isn't the most convenient thing ever, and setting default types is not possible from within drivers (technically, it is, but it's awkward), it's a step in the right direction. Once nvpairs gains proper type support, that'll make things a whole lot sweeter! This - and any future updates - are available from my feature/3.4/templates/cast-typehints branch.
This implements very basic support for type hinting template strings, where those hints will be available to value-pairs too, allowing any user of either LogTemplate or value-pairs to handle different types of values than plain strings. How they handle it, is up to the callers, this patch just lays the groundwork that makes it possible to pass these hints around. There are a few not terribly nice solutions here, namely that the type information is encoded into the first byte of the value we store while building the value-pairs set: we do this to avoid having to allocate memory separately: we'll just reuse a byte from a ScratchBuffer. This, however, is done behind the scenes, the callback functions will get the type and value information separately. Signed-off-by: Gergely Nagy <algernon@balabit.hu> --- lib/cfg-grammar.y | 41 ++++++++++++++++++++++--- lib/cfg-lexer.h | 5 +++ lib/filter-expr-grammar.ym | 18 ++++++++--- lib/templates.c | 28 +++++++++++++++++ lib/templates.h | 13 ++++++++ lib/value-pairs.c | 67 +++++++++++++++++++++++++++++++++-------- lib/value-pairs.h | 8 +++-- modules/afamqp/afamqp.c | 3 +- modules/afmongodb/afmongodb.c | 2 +- modules/json/format-json.c | 3 +- tests/unit/test_value_pairs.c | 6 ++-- 11 files changed, 164 insertions(+), 30 deletions(-) diff --git a/lib/cfg-grammar.y b/lib/cfg-grammar.y index f45d486..a275aa6 100644 --- a/lib/cfg-grammar.y +++ b/lib/cfg-grammar.y @@ -370,6 +370,8 @@ ValuePairsTransformSet *last_vp_transset; %type <ptr> dest_item %type <ptr> dest_plugin +%type <tpl> template_content + %type <ptr> filter_content %type <ptr> parser_content @@ -709,12 +711,23 @@ template_items | ; +/* START_RULES */ + +template_content + : string { $$.str = $1; $$.typehint = NULL; } + | LL_IDENTIFIER '(' string ')' { $$.str = $3; $$.typehint = $1; } + ; + +/* END_RULES */ + template_item - : KW_TEMPLATE '(' string ')' { + : KW_TEMPLATE '(' template_content ')' { GError *error = NULL; - CHECK_ERROR(log_template_compile(last_template, $3, &error), @3, "Error compiling template (%s)", error->message); - free($3); + CHECK_ERROR(log_template_compile(last_template, $3.str, &error), @3, "Error compiling template (%s)", error->message); + free($3.str); + CHECK_ERROR(log_template_set_type_hint(last_template, $3.typehint, &error), @3, "Error setting the template type-hint (%s)", error->message); + g_free($3.typehint); } | KW_TEMPLATE_ESCAPE '(' yesno ')' { log_template_set_escape(last_template, $3); } ; @@ -1065,8 +1078,26 @@ vp_options ; vp_option - : KW_PAIR '(' string ':' string ')' { value_pairs_add_pair(last_value_pairs, configuration, $3, $5); free($3); free($5); } - | KW_PAIR '(' string string ')' { value_pairs_add_pair(last_value_pairs, configuration, $3, $4); free($3); free($4); } + : KW_PAIR '(' string ':' template_content ')' + { + GError *error = NULL; + + CHECK_ERROR(value_pairs_add_pair_with_type(last_value_pairs, configuration, $3, $5.typehint, $5.str, &error), + @5, "Error processing value-pair (%s)", error->message); + free($3); + g_free($5.typehint); + free($5.str); + } + | KW_PAIR '(' string template_content ')' + { + GError *error = NULL; + + CHECK_ERROR(value_pairs_add_pair_with_type(last_value_pairs, configuration, $3, $4.typehint, $4.str, &error), + @4, "Error processing value-pair (%s)", error->message); + free($3); + g_free($4.typehint); + free($4.str); + } | KW_KEY '(' string KW_REKEY '(' { last_vp_transset = value_pairs_transform_set_new($3); diff --git a/lib/cfg-lexer.h b/lib/cfg-lexer.h index 03c8a61..2030b66 100644 --- a/lib/cfg-lexer.h +++ b/lib/cfg-lexer.h @@ -65,6 +65,11 @@ typedef struct YYSTYPE char *cptr; void *ptr; gpointer node; + struct + { + gchar *str; + gchar *typehint; + } tpl; }; } YYSTYPE; #define YYSTYPE_IS_TRIVIAL 1 diff --git a/lib/filter-expr-grammar.ym b/lib/filter-expr-grammar.ym index 1efa651..0a61df2 100644 --- a/lib/filter-expr-grammar.ym +++ b/lib/filter-expr-grammar.ym @@ -174,7 +174,7 @@ filter_simple_expr } $$ = node; } - | string operator string + | template_content operator template_content { LogTemplate *left, *right; GError *error = NULL; @@ -182,13 +182,21 @@ filter_simple_expr left = log_template_new(configuration, NULL); right = log_template_new(configuration, NULL); - success_left = log_template_compile(left, $1, &error); - success_right = log_template_compile(right, $3, &error); - free($1); - free($3); + success_left = log_template_compile(left, $1.str, &error); + success_right = log_template_compile(right, $3.str, &error); + free($1.str); + free($3.str); CHECK_ERROR(success_left, @1, "error compiling template: %s", error->message); CHECK_ERROR(success_right, @3, "error compiling template: %s", error->message); + CHECK_ERROR(log_template_set_type_hint(left, $1.typehint, &error), @1, "error setting type hint on template: %s", error->message); + CHECK_ERROR(log_template_set_type_hint(right, $3.typehint, &error), @3, "error setting type hint on template: %s", error->message); + + CHECK_ERROR(left->type_hint == right->type_hint, @$, "template types (%s <-> %s) do not match", $1.typehint, $3.typehint); + + g_free($1.typehint); + g_free($3.typehint); + $$ = fop_cmp_new(left, right, $2); } ; diff --git a/lib/templates.c b/lib/templates.c index f62f8b8..9be8110 100644 --- a/lib/templates.c +++ b/lib/templates.c @@ -886,6 +886,34 @@ parse_msg_ref(gchar **p, gint *msg_ref) } gboolean +log_template_set_type_hint(LogTemplate *self, const gchar *hint, GError **error) +{ + if (hint == NULL) + return TRUE; + + if (strcmp(hint, "string") == 0) + self->type_hint = TEMPLATE_TYPE_STRING; + else if (strcmp(hint, "int32") == 0 || strcmp(hint, "int") == 0) + self->type_hint = TEMPLATE_TYPE_INT32; + else if (strcmp(hint, "int64") == 0) + self->type_hint = TEMPLATE_TYPE_INT64; + else if (strcmp(hint, "datetime") == 0) + self->type_hint = TEMPLATE_TYPE_DATETIME; + else if (strcmp(hint, "boolean") == 0) + self->type_hint = TEMPLATE_TYPE_BOOLEAN; + else if (strcmp(hint, "default") == 0) + self->type_hint = TEMPLATE_TYPE_DEFAULT; + else + { + g_set_error(error, LOG_TEMPLATE_ERROR, LOG_TEMPLATE_ERROR_TYPE, + "%s", hint); + return FALSE; + } + + return TRUE; +} + +gboolean log_template_compile(LogTemplate *self, const gchar *template, GError **error) { gchar *start, *p; diff --git a/lib/templates.h b/lib/templates.h index ed5b688..f3ad226 100644 --- a/lib/templates.h +++ b/lib/templates.h @@ -40,8 +40,19 @@ enum LogTemplateError { LOG_TEMPLATE_ERROR_FAILED, LOG_TEMPLATE_ERROR_COMPILE, + LOG_TEMPLATE_ERROR_TYPE, }; +typedef enum +{ + TEMPLATE_TYPE_STRING, + TEMPLATE_TYPE_BOOLEAN, + TEMPLATE_TYPE_INT32, + TEMPLATE_TYPE_INT64, + TEMPLATE_TYPE_DATETIME, + TEMPLATE_TYPE_DEFAULT, +} LogTemplateType; + /* structure that represents an expandable syslog-ng template */ typedef struct _LogTemplate { @@ -54,6 +65,7 @@ typedef struct _LogTemplate GlobalConfig *cfg; GStaticMutex arg_lock; GPtrArray *arg_bufs; + LogTemplateType type_hint; } LogTemplate; /* template expansion options that can be influenced by the user and @@ -189,6 +201,7 @@ void tf_simple_func_free_state(gpointer state); void log_template_set_escape(LogTemplate *self, gboolean enable); gboolean log_template_compile(LogTemplate *self, const gchar *template, GError **error); +gboolean log_template_set_type_hint(LogTemplate *self, const gchar *hint, GError **error); void log_template_format(LogTemplate *self, LogMessage *lm, LogTemplateOptions *opts, gint tz, gint32 seq_num, const gchar *context_id, GString *result); void log_template_append_format(LogTemplate *self, LogMessage *lm, LogTemplateOptions *opts, gint tz, gint32 seq_num, const gchar *context_id, GString *result); void log_template_append_format_with_context(LogTemplate *self, LogMessage **messages, gint num_messages, LogTemplateOptions *opts, gint tz, gint32 seq_num, const gchar *context_id, GString *result); diff --git a/lib/value-pairs.c b/lib/value-pairs.c index ee22b73..da2e391 100644 --- a/lib/value-pairs.c +++ b/lib/value-pairs.c @@ -151,16 +151,30 @@ value_pairs_add_glob_pattern(ValuePairs *vp, const gchar *pattern, vp->patterns[i] = p; } -void -value_pairs_add_pair(ValuePairs *vp, GlobalConfig *cfg, const gchar *key, const gchar *value) +gboolean +value_pairs_add_pair_with_type(ValuePairs *vp, GlobalConfig *cfg, + const gchar *key, const gchar *type, + const gchar *value, GError **error) { VPPairConf *p = g_new(VPPairConf, 1); p->name = g_strdup(key); p->template = log_template_new(cfg, NULL); - log_template_compile(p->template, value, NULL); + log_template_compile(p->template, value, error); + if (error && *error) + return FALSE; + log_template_set_type_hint(p->template, type, error); + if (error && *error) + return FALSE; g_ptr_array_add(vp->vpairs, p); + return TRUE; +} + +void +value_pairs_add_pair(ValuePairs *vp, GlobalConfig *cfg, const gchar *key, const gchar *value) +{ + value_pairs_add_pair_with_type(vp, cfg, key, NULL, value, NULL); } static gchar * @@ -185,6 +199,13 @@ vp_transform_apply (ValuePairs *vp, gchar *key) return ckey; } +static GString * +vp_pairs_typehint(GString *str, LogTemplateType type_hint) +{ + g_string_append_printf(str, "%c", type_hint); + return str; +} + /* runs over the name-value pairs requested by the user (e.g. with value_pairs_add_pair) */ static void vp_pairs_foreach(gpointer data, gpointer user_data) @@ -196,10 +217,11 @@ vp_pairs_foreach(gpointer data, gpointer user_data) ScratchBuffer *sb = scratch_buffer_acquire(); VPPairConf *vpc = (VPPairConf *)data; - log_template_format((LogTemplate *)vpc->template, msg, NULL, LTZ_LOCAL, - seq_num, NULL, sb_string(sb)); + vp_pairs_typehint(sb_string(sb), vpc->template->type_hint); + log_template_append_format((LogTemplate *)vpc->template, msg, NULL, LTZ_LOCAL, + seq_num, NULL, sb_string(sb)); - if (!sb_string(sb)->str[0]) + if (sb_string(sb)->len == 0) { scratch_buffer_release(sb); return; @@ -234,8 +256,13 @@ vp_msg_nvpairs_foreach(NVHandle handle, gchar *name, (log_msg_is_handle_sdata(handle) && (vp->scopes & VPS_SDATA))) || inc) { + gchar *v = g_malloc(value_len + 2); + /* NOTE: the key is a borrowed reference in the hash, and value is freed */ - g_tree_insert(scope_set, vp_transform_apply(vp, name), g_strndup(value, value_len)); + v[0] = 0; + strncpy(v + 1, value, value_len); + v[value_len + 1] = 0; + g_tree_insert(scope_set, vp_transform_apply(vp, name), v); } return FALSE; @@ -265,6 +292,7 @@ vp_merge_set(ValuePairs *vp, LogMessage *msg, gint32 seq_num, ValuePairSpec *set switch (set[i].type) { case VPT_MACRO: + vp_pairs_typehint(sb_string(sb), TEMPLATE_TYPE_STRING); log_macro_expand(sb_string(sb), set[i].id, FALSE, NULL, LTZ_LOCAL, seq_num, NULL, msg); break; case VPT_NVPAIR: @@ -272,6 +300,7 @@ vp_merge_set(ValuePairs *vp, LogMessage *msg, gint32 seq_num, ValuePairSpec *set const gchar *nv; gssize len; + vp_pairs_typehint(sb_string(sb), TEMPLATE_TYPE_STRING); nv = log_msg_get_value(msg, (NVHandle) set[i].id, &len); g_string_append_len(sb_string(sb), nv, len); break; @@ -280,7 +309,7 @@ vp_merge_set(ValuePairs *vp, LogMessage *msg, gint32 seq_num, ValuePairSpec *set g_assert_not_reached(); } - if (!sb_string(sb)->str[0]) + if (sb_string(sb)->len == 0) continue; g_tree_insert(dest, vp_transform_apply(vp, set[i].name), sb_string(sb)->str); @@ -289,12 +318,22 @@ vp_merge_set(ValuePairs *vp, LogMessage *msg, gint32 seq_num, ValuePairSpec *set scratch_buffer_release(sb); } +static gboolean +vp_foreach_helper (const gchar *name, const gchar *hinted_value, gpointer data) +{ + VPForeachFunc func = ((gpointer *)data)[0]; + gpointer user_data = ((gpointer *)data)[1]; + + return func(name, (LogTemplateType)hinted_value[0], hinted_value + 1, user_data); +} + void value_pairs_foreach_sorted (ValuePairs *vp, VPForeachFunc func, GCompareDataFunc compare_func, LogMessage *msg, gint32 seq_num, gpointer user_data) { gpointer args[] = { vp, func, msg, GINT_TO_POINTER (seq_num), user_data, NULL }; + gpointer helper_args[] = { func, user_data }; GTree *scope_set; scope_set = g_tree_new_full((GCompareDataFunc)compare_func, NULL, @@ -326,7 +365,7 @@ value_pairs_foreach_sorted (ValuePairs *vp, VPForeachFunc func, g_ptr_array_foreach(vp->vpairs, (GFunc)vp_pairs_foreach, args); /* Aaand we run it through the callback! */ - g_tree_foreach(scope_set, (GTraverseFunc)func, user_data); + g_tree_foreach(scope_set, (GTraverseFunc)vp_foreach_helper, helper_args); g_tree_destroy(scope_set); } @@ -481,7 +520,7 @@ vp_walker_name_split(vp_walk_stack_t **stack, vp_walk_state_t *state, } static gboolean -value_pairs_walker(const gchar *name, const gchar *value, +value_pairs_walker(const gchar *name, LogTemplateType type, const gchar *value, gpointer user_data) { vp_walk_state_t *state = (vp_walk_state_t *)user_data; @@ -494,10 +533,14 @@ value_pairs_walker(const gchar *name, const gchar *value, key = vp_walker_name_split (&st, state, name); if (st) - result = state->process_value(key, st->prefix, value, &st->data, + result = state->process_value(key, st->prefix, + type, value, + &st->data, state->user_data); else - result = state->process_value(key, NULL, value, NULL, + result = state->process_value(key, NULL, + type, value, + NULL, state->user_data); g_free(key); diff --git a/lib/value-pairs.h b/lib/value-pairs.h index a6c9f34..0438d53 100644 --- a/lib/value-pairs.h +++ b/lib/value-pairs.h @@ -27,12 +27,13 @@ #include "syslog-ng.h" #include "nvtable.h" +#include "templates.h" typedef struct _ValuePairs ValuePairs; -typedef gboolean (*VPForeachFunc)(const gchar *name, const gchar *value, gpointer user_data); +typedef gboolean (*VPForeachFunc)(const gchar *name, LogTemplateType type, const gchar *value, gpointer user_data); typedef gboolean (*VPWalkValueCallbackFunc)(const gchar *name, const gchar *prefix, - const gchar *value, + LogTemplateType type, const gchar *value, gpointer *prefix_data, gpointer user_data); typedef gboolean (*VPWalkCallbackFunc)(const gchar *name, const gchar *prefix, gpointer *prefix_data, @@ -42,6 +43,9 @@ typedef gboolean (*VPWalkCallbackFunc)(const gchar *name, gboolean value_pairs_add_scope(ValuePairs *vp, const gchar *scope); void value_pairs_add_glob_pattern(ValuePairs *vp, const gchar *pattern, gboolean include); void value_pairs_add_pair(ValuePairs *vp, GlobalConfig *cfg, const gchar *key, const gchar *value); +gboolean value_pairs_add_pair_with_type(ValuePairs *vp, GlobalConfig *cfg, + const gchar *key, const gchar *type, + const gchar *value, GError **error); void value_pairs_add_transforms(ValuePairs *vp, gpointer vpts); diff --git a/modules/afamqp/afamqp.c b/modules/afamqp/afamqp.c index 70fd3e8..fb3a333 100644 --- a/modules/afamqp/afamqp.c +++ b/modules/afamqp/afamqp.c @@ -374,7 +374,8 @@ afamqp_dd_connect(AMQPDestDriver *self, gboolean reconnect) */ static gboolean -afamqp_vp_foreach(const gchar *name, const gchar *value, +afamqp_vp_foreach(const gchar *name, + LogTemplateType type, const gchar *value, gpointer user_data) { amqp_table_entry_t **entries = (amqp_table_entry_t **) ((gpointer *)user_data)[0]; diff --git a/modules/afmongodb/afmongodb.c b/modules/afmongodb/afmongodb.c index 9779097..5230d8f 100644 --- a/modules/afmongodb/afmongodb.c +++ b/modules/afmongodb/afmongodb.c @@ -359,7 +359,7 @@ afmongodb_vp_obj_end(const gchar *name, static gboolean afmongodb_vp_process_value(const gchar *name, const gchar *prefix, - const gchar *value, + LogTemplateType type, const gchar *value, gpointer *prefix_data, gpointer user_data) { bson *o; diff --git a/modules/json/format-json.c b/modules/json/format-json.c index d654c2e..4b79701 100644 --- a/modules/json/format-json.c +++ b/modules/json/format-json.c @@ -163,7 +163,8 @@ tf_json_obj_end(const gchar *name, } static gboolean -tf_json_value(const gchar *name, const gchar *prefix, const gchar *value, +tf_json_value(const gchar *name, const gchar *prefix, + LogTemplateType type, const gchar *value, gpointer *prefix_data, gpointer user_data) { json_state_t *state = (json_state_t *)user_data; diff --git a/tests/unit/test_value_pairs.c b/tests/unit/test_value_pairs.c index ca4ba32..3fe09de 100644 --- a/tests/unit/test_value_pairs.c +++ b/tests/unit/test_value_pairs.c @@ -10,7 +10,7 @@ gboolean success = TRUE; gboolean -vp_keys_foreach(const gchar *name, const gchar *value, gpointer user_data) +vp_keys_foreach(const gchar *name, LogTemplateType type, const gchar *value, gpointer user_data) { gpointer *args = (gpointer *) user_data; GList **keys = (GList **) args[0]; @@ -123,7 +123,7 @@ main(int argc, char *argv[]) testcase("all-nv-pairs", NULL, ".SDATA.EventData@18372.4.Data,.SDATA.Keywords@18372.4.Keyword,.SDATA.meta.sequenceId,.SDATA.meta.sysUpTime,.SDATA.origin.ip,HOST,MESSAGE,MSGID,PID,PROGRAM", NULL); - testcase("everything", NULL, ".SDATA.EventData@18372.4.Data,.SDATA.Keywords@18372.4.Keyword,.SDATA.meta.sequenceId,.SDATA.meta.sysUpTime,.SDATA.origin.ip,AMPM,BSDTAG,C_DATE,C_DAY,C_FULLDATE,C_HOUR,C_ISODATE,C_MIN,C_MONTH,C_MONTH_ABBREV,C_MONTH_NAME,C_MONTH_WEEK,C_SEC,C_STAMP,C_TZ,C_TZOFFSET,C_UNIXTIME,C_WEEK,C_WEEKDAY,C_WEEK_DAY,C_WEEK_DAY_ABBREV,C_WEEK_DAY_NAME,C_YEAR,C_YEAR_DAY,DATE,DAY,FACILITY,FACILITY_NUM,FULLDATE,HOST,HOUR,HOUR12,ISODATE,LEVEL,LEVEL_NUM,LOGHOST,MESSAGE,MIN,MONTH,MONTH_ABBREV,MONTH_NAME,MONTH_WEEK,MSEC,MSG,MSGHDR,MSGID,PID,PRI,PRIORITY,PROGRAM,R_AMPM,R_DATE,R_DAY,R_FULLDATE,R_HOUR,R_HOUR12,R_ISODATE,R_MIN,R_MONTH,R_MONTH_ABBREV,R_MONTH_NAME,R_MONTH_WEEK,R_MSEC,R_SEC,R_STAMP,R_TZ,R_TZOFFSET,R_UNIXTIME,R_USEC,R_WEEK,R_WEEKDAY,R_WEEK_DAY,R_WEEK_DAY_ABBREV,R_WEEK_DAY_NAME,R_YEAR,R_YEAR_DAY,SDATA,SEC,SEQNUM,SOURCEIP,STAMP,SYSUPTIME,S_AMPM,S_DATE,S_DAY,S_FULLDATE,S_HOUR,S_HOUR12,S_ISODATE,S_MIN,S_MONTH,S_MONTH_ABBREV,S_MONTH_NAME,S_MONTH_WEEK,S_MSEC,S_SEC,S _STAMP,S _TZ,S_TZOFFSET,S_UNIXTIME,S_USEC,S_WEEK,S_WEEKDAY,S_WEEK_DAY,S_WEEK_DAY_ABBREV,S_WEEK_DAY_NAME,S_YEAR,S_YEAR_DAY,TAG,TAGS,TZ,TZOFFSET,UNIXTIME,USEC,WEEK,WEEKDAY,WEEK_DAY,WEEK_DAY_ABBREV,WEEK_DAY_NAME,YEAR,YEAR_DAY", NULL); + testcase("everything", NULL, ".SDATA.EventData@18372.4.Data,.SDATA.Keywords@18372.4.Keyword,.SDATA.meta.sequenceId,.SDATA.meta.sysUpTime,.SDATA.origin.ip,AMPM,BSDTAG,CONTEXT_ID,C_DATE,C_DAY,C_FULLDATE,C_HOUR,C_ISODATE,C_MIN,C_MONTH,C_MONTH_ABBREV,C_MONTH_NAME,C_MONTH_WEEK,C_SEC,C_STAMP,C_TZ,C_TZOFFSET,C_UNIXTIME,C_WEEK,C_WEEKDAY,C_WEEK_DAY,C_WEEK_DAY_ABBREV,C_WEEK_DAY_NAME,C_YEAR,C_YEAR_DAY,DATE,DAY,FACILITY,FACILITY_NUM,FULLDATE,HOST,HOUR,HOUR12,ISODATE,LEVEL,LEVEL_NUM,LOGHOST,MESSAGE,MIN,MONTH,MONTH_ABBREV,MONTH_NAME,MONTH_WEEK,MSEC,MSG,MSGHDR,MSGID,PID,PRI,PRIORITY,PROGRAM,R_AMPM,R_DATE,R_DAY,R_FULLDATE,R_HOUR,R_HOUR12,R_ISODATE,R_MIN,R_MONTH,R_MONTH_ABBREV,R_MONTH_NAME,R_MONTH_WEEK,R_MSEC,R_SEC,R_STAMP,R_TZ,R_TZOFFSET,R_UNIXTIME,R_USEC,R_WEEK,R_WEEKDAY,R_WEEK_DAY,R_WEEK_DAY_ABBREV,R_WEEK_DAY_NAME,R_YEAR,R_YEAR_DAY,SDATA,SEC,SEQNUM,SOURCEIP,STAMP,SYSUPTIME,S_AMPM,S_DATE,S_DAY,S_FULLDATE,S_HOUR,S_HOUR12,S_ISODATE,S_MIN,S_MONTH,S_MONTH_ABBREV,S_MONTH_NAME,S_MONTH_WEEK,S_M SEC,S_SE C,S_STAMP,S_TZ,S_TZOFFSET,S_UNIXTIME,S_USEC,S_WEEK,S_WEEKDAY,S_WEEK_DAY,S_WEEK_DAY_ABBREV,S_WEEK_DAY_NAME,S_YEAR,S_YEAR_DAY,TAG,TAGS,TZ,TZOFFSET,UNIXTIME,USEC,WEEK,WEEKDAY,WEEK_DAY,WEEK_DAY_ABBREV,WEEK_DAY_NAME,YEAR,YEAR_DAY", NULL); testcase("nv-pairs", ".SDATA.*", "HOST,MESSAGE,MSGID,PID,PROGRAM", NULL); @@ -139,7 +139,7 @@ main(int argc, char *argv[]) g_ptr_array_add(transformers, value_pairs_new_transform_shift(2)); g_ptr_array_add(transformers, value_pairs_new_transform_replace("C_", "CC_")); - testcase("everything", NULL, ".SDATA.EventData@18372.4.Data,.SDATA.Keywords@18372.4.Keyword,.SDATA.meta.sequenceId,.SDATA.meta.sysUpTime,.SDATA.origin.ip,AMPM,BSDTAG,CC_DATE,CC_DAY,CC_FULLDATE,CC_HOUR,CC_ISODATE,CC_MIN,CC_MONTH,CC_MONTH_ABBREV,CC_MONTH_NAME,CC_MONTH_WEEK,CC_SEC,CC_STAMP,CC_TZ,CC_TZOFFSET,CC_UNIXTIME,CC_WEEK,CC_WEEKDAY,CC_WEEK_DAY,CC_WEEK_DAY_ABBREV,CC_WEEK_DAY_NAME,CC_YEAR,CC_YEAR_DAY,DATE,DAY,FACILITY,FACILITY_NUM,FULLDATE,HOST,HOUR,HOUR12,ISODATE,LEVEL,LEVEL_NUM,LOGHOST,MESSAGE,MIN,MONTH,MONTH_ABBREV,MONTH_NAME,MONTH_WEEK,MSEC,MSG,MSGHDR,MSGID,PID,PRI,PRIORITY,PROGRAM,R_AMPM,R_DATE,R_DAY,R_FULLDATE,R_HOUR,R_HOUR12,R_ISODATE,R_MIN,R_MONTH,R_MONTH_ABBREV,R_MONTH_NAME,R_MONTH_WEEK,R_MSEC,R_SEC,R_STAMP,R_TZ,R_TZOFFSET,R_UNIXTIME,R_USEC,R_WEEK,R_WEEKDAY,R_WEEK_DAY,R_WEEK_DAY_ABBREV,R_WEEK_DAY_NAME,R_YEAR,R_YEAR_DAY,SDATA,SEC,SEQNUM,SOURCEIP,STAMP,SYSUPTIME,S_AMPM,S_DATE,S_DAY,S_FULLDATE,S_HOUR,S_HOUR12,S_ISODATE,S_MIN,S_MONTH,S_MONTH_ABBREV,S_MONTH_NAME,S_MON TH_WEEK, S_MSEC,S_SEC,S_STAMP,S_TZ,S_TZOFFSET,S_UNIXTIME,S_USEC,S_WEEK,S_WEEKDAY,S_WEEK_DAY,S_WEEK_DAY_ABBREV,S_WEEK_DAY_NAME,S_YEAR,S_YEAR_DAY,TAG,TAGS,TZ,TZOFFSET,UNIXTIME,USEC,WEEK,WEEKDAY,WEEK_DAY,WEEK_DAY_ABBREV,WEEK_DAY_NAME,YEAR,YEAR_DAY", transformers); + testcase("everything", NULL, ".SDATA.EventData@18372.4.Data,.SDATA.Keywords@18372.4.Keyword,.SDATA.meta.sequenceId,.SDATA.meta.sysUpTime,.SDATA.origin.ip,AMPM,BSDTAG,CC_DATE,CC_DAY,CC_FULLDATE,CC_HOUR,CC_ISODATE,CC_MIN,CC_MONTH,CC_MONTH_ABBREV,CC_MONTH_NAME,CC_MONTH_WEEK,CC_SEC,CC_STAMP,CC_TZ,CC_TZOFFSET,CC_UNIXTIME,CC_WEEK,CC_WEEKDAY,CC_WEEK_DAY,CC_WEEK_DAY_ABBREV,CC_WEEK_DAY_NAME,CC_YEAR,CC_YEAR_DAY,CONTEXT_ID,DATE,DAY,FACILITY,FACILITY_NUM,FULLDATE,HOST,HOUR,HOUR12,ISODATE,LEVEL,LEVEL_NUM,LOGHOST,MESSAGE,MIN,MONTH,MONTH_ABBREV,MONTH_NAME,MONTH_WEEK,MSEC,MSG,MSGHDR,MSGID,PID,PRI,PRIORITY,PROGRAM,R_AMPM,R_DATE,R_DAY,R_FULLDATE,R_HOUR,R_HOUR12,R_ISODATE,R_MIN,R_MONTH,R_MONTH_ABBREV,R_MONTH_NAME,R_MONTH_WEEK,R_MSEC,R_SEC,R_STAMP,R_TZ,R_TZOFFSET,R_UNIXTIME,R_USEC,R_WEEK,R_WEEKDAY,R_WEEK_DAY,R_WEEK_DAY_ABBREV,R_WEEK_DAY_NAME,R_YEAR,R_YEAR_DAY,SDATA,SEC,SEQNUM,SOURCEIP,STAMP,SYSUPTIME,S_AMPM,S_DATE,S_DAY,S_FULLDATE,S_HOUR,S_HOUR12,S_ISODATE,S_MIN,S_MONTH,S_MONTH_ABBREV,S_MONTH _NAME,S_ MONTH_WEEK,S_MSEC,S_SEC,S_STAMP,S_TZ,S_TZOFFSET,S_UNIXTIME,S_USEC,S_WEEK,S_WEEKDAY,S_WEEK_DAY,S_WEEK_DAY_ABBREV,S_WEEK_DAY_NAME,S_YEAR,S_YEAR_DAY,TAG,TAGS,TZ,TZOFFSET,UNIXTIME,USEC,WEEK,WEEKDAY,WEEK_DAY,WEEK_DAY_ABBREV,WEEK_DAY_NAME,YEAR,YEAR_DAY", transformers); g_ptr_array_free(transformers, TRUE); app_shutdown(); -- 1.7.10.4
----- Original message -----
This implements very basic support for type hinting template strings, where those hints will be available to value-pairs too, allowing any user of either LogTemplate or value-pairs to handle different types of values than plain strings. How they handle it, is up to the callers, this patch just lays the groundwork that makes it possible to pass these hints around.
There are a few not terribly nice solutions here, namely that the type information is encoded into the first byte of the value we store while building the value-pairs set: we do this to avoid having to allocate memory separately: we'll just reuse a byte from a ScratchBuffer. This, however, is done behind the scenes, the callback functions will get the type and value information separately.
Signed-off-by: Gergely Nagy <algernon@balabit.hu> --- lib/cfg-grammar.y | 41 ++++++++++++++++++++++--- lib/cfg-lexer.h | 5 +++ lib/filter-expr-grammar.ym | 18 ++++++++--- lib/templates.c | 28 +++++++++++++++++ lib/templates.h | 13 ++++++++ lib/value-pairs.c | 67 +++++++++++++++++++++++++++++++++-------- lib/value-pairs.h | 8 +++-- modules/afamqp/afamqp.c | 3 +- modules/afmongodb/afmongodb.c | 2 +- modules/json/format-json.c | 3 +- tests/unit/test_value_pairs.c | 6 ++-- 11 files changed, 164 insertions(+), 30 deletions(-)
diff --git a/lib/cfg-grammar.y b/lib/cfg-grammar.y index f45d486..a275aa6 100644 --- a/lib/cfg-grammar.y +++ b/lib/cfg-grammar.y @@ -370,6 +370,8 @@ ValuePairsTransformSet *last_vp_transset; %type <ptr> dest_item %type <ptr> dest_plugin +%type <tpl> template_content + %type <ptr> filter_content %type <ptr> parser_content @@ -709,12 +711,23 @@ template_items | ; +/* START_RULES */ + +template_content + : string { $$.str = $1; $$.typehint = NULL; } + | LL_IDENTIFIER '(' string ')' { $$.str = $3; $$.typehint = $1; } + ; +
can't we move template compilation here somehow? e.g. return logtemplate as a ptr?
+/* END_RULES */ + template_item - : KW_TEMPLATE '(' string ')' { + : KW_TEMPLATE '(' template_content ')' { GError *error = NULL; - CHECK_ERROR(log_template_compile(last_template, $3, &error), @3, "Error compiling template (%s)", error->message); - free($3); + CHECK_ERROR(log_template_compile(last_template, $3.str, &error), @3, "Error compiling template (%s)", error->message); + free($3.str); + CHECK_ERROR(log_template_set_type_hint(last_template, $3.typehint, &error), @3, "Error setting the template type-hint (%s)", error->message); + g_free($3.typehint); }
that way this code would be there
| KW_TEMPLATE_ESCAPE '(' yesno ')' { log_template_set_escape(last_template, $3); } ; @@ -1065,8 +1078,26 @@ vp_options ; vp_option - : KW_PAIR '(' string ':' string ')' { value_pairs_add_pair(last_value_pairs, configuration, $3, $5); free($3); free($5); } - | KW_PAIR '(' string string ')' { value_pairs_add_pair(last_value_pairs, configuration, $3, $4); free($3); free($4); } + : KW_PAIR '(' string ':' template_content ')' + { + GError *error = NULL; + + CHECK_ERROR(value_pairs_add_pair_with_type(last_value_pairs, configuration, $3, $5.typehint, $5.str, &error), + @5, "Error processing value-pair (%s)", error->message); + free($3); + g_free($5.typehint); + free($5.str); + } + | KW_PAIR '(' string template_content ')' + { + GError *error = NULL; + + CHECK_ERROR(value_pairs_add_pair_with_type(last_value_pairs, configuration, $3, $4.typehint, $4.str, &error), + @4, "Error processing value-pair (%s)", error->message); + free($3); + g_free($4.typehint); + free($4.str); + }
and these dupes would be gone.
| KW_KEY '(' string KW_REKEY '(' { last_vp_transset = value_pairs_transform_set_new($3); diff --git a/lib/cfg-lexer.h b/lib/cfg-lexer.h index 03c8a61..2030b66 100644 --- a/lib/cfg-lexer.h +++ b/lib/cfg-lexer.h @@ -65,6 +65,11 @@ typedef struct YYSTYPE char *cptr; void *ptr; gpointer node; + struct + { + gchar *str; + gchar *typehint; + } tpl; }; } YYSTYPE; #define YYSTYPE_IS_TRIVIAL 1 diff --git a/lib/filter-expr-grammar.ym b/lib/filter-expr-grammar.ym index 1efa651..0a61df2 100644 --- a/lib/filter-expr-grammar.ym +++ b/lib/filter-expr-grammar.ym @@ -174,7 +174,7 @@ filter_simple_expr } $$ = node; } - | string operator string + | template_content operator template_content { LogTemplate *left, *right; GError *error = NULL; @@ -182,13 +182,21 @@ filter_simple_expr left = log_template_new(configuration, NULL); right = log_template_new(configuration, NULL); - success_left = log_template_compile(left, $1, &error); - success_right = log_template_compile(right, $3, &error); - free($1); - free($3); + success_left = log_template_compile(left, $1.str, &error); + success_right = log_template_compile(right, $3.str, &error); + free($1.str); + free($3.str); CHECK_ERROR(success_left, @1, "error compiling template: %s", error->message); CHECK_ERROR(success_right, @3, "error compiling template: %s", error->message); + CHECK_ERROR(log_template_set_type_hint(left, $1.typehint, &error), @1, "error setting type hint on template: %s", error->message); + CHECK_ERROR(log_template_set_type_hint(right, $3.typehint, &error), @3, "error setting type hint on template: %s", error->message); + + CHECK_ERROR(left->type_hint == right->type_hint, @$, "template types (%s <-> %s) do not match", $1.typehint, $3.typehint); + + g_free($1.typehint); + g_free($3.typehint); + $$ = fop_cmp_new(left, right, $2);
most of these would be gone too.
} ; diff --git a/lib/templates.c b/lib/templates.c index f62f8b8..9be8110 100644 --- a/lib/templates.c +++ b/lib/templates.c @@ -886,6 +886,34 @@ parse_msg_ref(gchar **p, gint *msg_ref) } gboolean +log_template_set_type_hint(LogTemplate *self, const gchar *hint, GError **error) +{ + if (hint == NULL) + return TRUE; + + if (strcmp(hint, "string") == 0) + self->type_hint = TEMPLATE_TYPE_STRING; + else if (strcmp(hint, "int32") == 0 || strcmp(hint, "int") == 0) + self->type_hint = TEMPLATE_TYPE_INT32; + else if (strcmp(hint, "int64") == 0) + self->type_hint = TEMPLATE_TYPE_INT64; + else if (strcmp(hint, "datetime") == 0) + self->type_hint = TEMPLATE_TYPE_DATETIME; + else if (strcmp(hint, "boolean") == 0) + self->type_hint = TEMPLATE_TYPE_BOOLEAN; + else if (strcmp(hint, "default") == 0) + self->type_hint = TEMPLATE_TYPE_DEFAULT; + else + { + g_set_error(error, LOG_TEMPLATE_ERROR, LOG_TEMPLATE_ERROR_TYPE, + "%s", hint); + return FALSE; + } + + return TRUE; +}
I'm thinking about whether it'd make sense to configure this at runtime, and whether the list of supported values could be destination specific.
+ +gboolean log_template_compile(LogTemplate *self, const gchar *template, GError **error) { gchar *start, *p; diff --git a/lib/templates.h b/lib/templates.h index ed5b688..f3ad226 100644 --- a/lib/templates.h +++ b/lib/templates.h @@ -40,8 +40,19 @@ enum LogTemplateError { LOG_TEMPLATE_ERROR_FAILED, LOG_TEMPLATE_ERROR_COMPILE, + LOG_TEMPLATE_ERROR_TYPE, }; +typedef enum +{ + TEMPLATE_TYPE_STRING, + TEMPLATE_TYPE_BOOLEAN, + TEMPLATE_TYPE_INT32, + TEMPLATE_TYPE_INT64, + TEMPLATE_TYPE_DATETIME, + TEMPLATE_TYPE_DEFAULT, +} LogTemplateType; +
default is somehow an exception here.
/* structure that represents an expandable syslog-ng template */ typedef struct _LogTemplate { @@ -54,6 +65,7 @@ typedef struct _LogTemplate GlobalConfig *cfg; GStaticMutex arg_lock; GPtrArray *arg_bufs; + LogTemplateType type_hint; } LogTemplate; /* template expansion options that can be influenced by the user and @@ -189,6 +201,7 @@ void tf_simple_func_free_state(gpointer state); void log_template_set_escape(LogTemplate *self, gboolean enable); gboolean log_template_compile(LogTemplate *self, const gchar *template, GError **error); +gboolean log_template_set_type_hint(LogTemplate *self, const gchar *hint, GError **error); void log_template_format(LogTemplate *self, LogMessage *lm, LogTemplateOptions *opts, gint tz, gint32 seq_num, const gchar *context_id, GString *result); void log_template_append_format(LogTemplate *self, LogMessage *lm, LogTemplateOptions *opts, gint tz, gint32 seq_num, const gchar *context_id, GString *result); void log_template_append_format_with_context(LogTemplate *self, LogMessage **messages, gint num_messages, LogTemplateOptions *opts, gint tz, gint32 seq_num, const gchar *context_id, GString *result); diff --git a/lib/value-pairs.c b/lib/value-pairs.c index ee22b73..da2e391 100644 --- a/lib/value-pairs.c +++ b/lib/value-pairs.c @@ -151,16 +151,30 @@ value_pairs_add_glob_pattern(ValuePairs *vp, const gchar *pattern, vp->patterns[i] = p; } -void -value_pairs_add_pair(ValuePairs *vp, GlobalConfig *cfg, const gchar *key, const gchar *value) +gboolean +value_pairs_add_pair_with_type(ValuePairs *vp, GlobalConfig *cfg, + const gchar *key, const gchar *type, + const gchar *value, GError **error) { VPPairConf *p = g_new(VPPairConf, 1); p->name = g_strdup(key); p->template = log_template_new(cfg, NULL); - log_template_compile(p->template, value, NULL); + log_template_compile(p->template, value, error); + if (error && *error) + return FALSE; + log_template_set_type_hint(p->template, type, error); + if (error && *error) + return FALSE; g_ptr_array_add(vp->vpairs, p); + return TRUE; +} + +void +value_pairs_add_pair(ValuePairs *vp, GlobalConfig *cfg, const gchar *key, const gchar *value) +{ + value_pairs_add_pair_with_type(vp, cfg, key, NULL, value, NULL); } static gchar * @@ -185,6 +199,13 @@ vp_transform_apply (ValuePairs *vp, gchar *key) return ckey; } +static GString * +vp_pairs_typehint(GString *str, LogTemplateType type_hint) +{ + g_string_append_printf(str, "%c", type_hint); + return str; +} + /* runs over the name-value pairs requested by the user (e.g. with value_pairs_add_pair) */ static void vp_pairs_foreach(gpointer data, gpointer user_data) @@ -196,10 +217,11 @@ vp_pairs_foreach(gpointer data, gpointer user_data) ScratchBuffer *sb = scratch_buffer_acquire(); VPPairConf *vpc = (VPPairConf *)data; - log_template_format((LogTemplate *)vpc->template, msg, NULL, LTZ_LOCAL, - seq_num, NULL, sb_string(sb)); + vp_pairs_typehint(sb_string(sb), vpc->template->type_hint); + log_template_append_format((LogTemplate *)vpc->template, msg, NULL, LTZ_LOCAL, + seq_num, NULL, sb_string(sb)); - if (!sb_string(sb)->str[0]) + if (sb_string(sb)->len == 0) { scratch_buffer_release(sb); return; @@ -234,8 +256,13 @@ vp_msg_nvpairs_foreach(NVHandle handle, gchar *name, (log_msg_is_handle_sdata(handle) && (vp->scopes & VPS_SDATA))) || inc) { + gchar *v = g_malloc(value_len + 2); + /* NOTE: the key is a borrowed reference in the hash, and value is freed */ - g_tree_insert(scope_set, vp_transform_apply(vp, name), g_strndup(value, value_len)); + v[0] = 0; + strncpy(v + 1, value, value_len); + v[value_len + 1] = 0; + g_tree_insert(scope_set, vp_transform_apply(vp, name), v); } return FALSE; @@ -265,6 +292,7 @@ vp_merge_set(ValuePairs *vp, LogMessage *msg, gint32 seq_num, ValuePairSpec *set switch (set[i].type) { case VPT_MACRO: + vp_pairs_typehint(sb_string(sb), TEMPLATE_TYPE_STRING); log_macro_expand(sb_string(sb), set[i].id, FALSE, NULL, LTZ_LOCAL, seq_num, NULL, msg); break; case VPT_NVPAIR: @@ -272,6 +300,7 @@ vp_merge_set(ValuePairs *vp, LogMessage *msg, gint32 seq_num, ValuePairSpec *set const gchar *nv; gssize len; + vp_pairs_typehint(sb_string(sb), TEMPLATE_TYPE_STRING); nv = log_msg_get_value(msg, (NVHandle) set[i].id, &len); g_string_append_len(sb_string(sb), nv, len); break; @@ -280,7 +309,7 @@ vp_merge_set(ValuePairs *vp, LogMessage *msg, gint32 seq_num, ValuePairSpec *set g_assert_not_reached(); } - if (!sb_string(sb)->str[0]) + if (sb_string(sb)->len == 0) continue; g_tree_insert(dest, vp_transform_apply(vp, set[i].name), sb_string(sb)->str); @@ -289,12 +318,22 @@ vp_merge_set(ValuePairs *vp, LogMessage *msg, gint32 seq_num, ValuePairSpec *set scratch_buffer_release(sb); } +static gboolean +vp_foreach_helper (const gchar *name, const gchar *hinted_value, gpointer data) +{ + VPForeachFunc func = ((gpointer *)data)[0]; + gpointer user_data = ((gpointer *)data)[1]; + + return func(name, (LogTemplateType)hinted_value[0], hinted_value + 1, user_data); +} + void value_pairs_foreach_sorted (ValuePairs *vp, VPForeachFunc func, GCompareDataFunc compare_func, LogMessage *msg, gint32 seq_num, gpointer user_data) { gpointer args[] = { vp, func, msg, GINT_TO_POINTER (seq_num), user_data, NULL }; + gpointer helper_args[] = { func, user_data }; GTree *scope_set; scope_set = g_tree_new_full((GCompareDataFunc)compare_func, NULL, @@ -326,7 +365,7 @@ value_pairs_foreach_sorted (ValuePairs *vp, VPForeachFunc func, g_ptr_array_foreach(vp->vpairs, (GFunc)vp_pairs_foreach, args); /* Aaand we run it through the callback! */ - g_tree_foreach(scope_set, (GTraverseFunc)func, user_data); + g_tree_foreach(scope_set, (GTraverseFunc)vp_foreach_helper, helper_args); g_tree_destroy(scope_set); } @@ -481,7 +520,7 @@ vp_walker_name_split(vp_walk_stack_t **stack, vp_walk_state_t *state, } static gboolean -value_pairs_walker(const gchar *name, const gchar *value, +value_pairs_walker(const gchar *name, LogTemplateType type, const gchar *value, gpointer user_data) { vp_walk_state_t *state = (vp_walk_state_t *)user_data; @@ -494,10 +533,14 @@ value_pairs_walker(const gchar *name, const gchar *value, key = vp_walker_name_split (&st, state, name); if (st) - result = state->process_value(key, st->prefix, value, &st->data, + result = state->process_value(key, st->prefix, + type, value, + &st->data, state->user_data); else - result = state->process_value(key, NULL, value, NULL, + result = state->process_value(key, NULL, + type, value, + NULL, state->user_data); g_free(key); diff --git a/lib/value-pairs.h b/lib/value-pairs.h index a6c9f34..0438d53 100644 --- a/lib/value-pairs.h +++ b/lib/value-pairs.h @@ -27,12 +27,13 @@ #include "syslog-ng.h" #include "nvtable.h" +#include "templates.h" typedef struct _ValuePairs ValuePairs; -typedef gboolean (*VPForeachFunc)(const gchar *name, const gchar *value, gpointer user_data); +typedef gboolean (*VPForeachFunc)(const gchar *name, LogTemplateType type, const gchar *value, gpointer user_data); typedef gboolean (*VPWalkValueCallbackFunc)(const gchar *name, const gchar *prefix, - const gchar *value, + LogTemplateType type, const gchar *value, gpointer *prefix_data, gpointer user_data); typedef gboolean (*VPWalkCallbackFunc)(const gchar *name, const gchar *prefix, gpointer *prefix_data, @@ -42,6 +43,9 @@ typedef gboolean (*VPWalkCallbackFunc)(const gchar *name, gboolean value_pairs_add_scope(ValuePairs *vp, const gchar *scope); void value_pairs_add_glob_pattern(ValuePairs *vp, const gchar *pattern, gboolean include); void value_pairs_add_pair(ValuePairs *vp, GlobalConfig *cfg, const gchar *key, const gchar *value); +gboolean value_pairs_add_pair_with_type(ValuePairs *vp, GlobalConfig *cfg, + const gchar *key, const gchar *type, + const gchar *value, GError **error); void value_pairs_add_transforms(ValuePairs *vp, gpointer vpts); diff --git a/modules/afamqp/afamqp.c b/modules/afamqp/afamqp.c index 70fd3e8..fb3a333 100644 --- a/modules/afamqp/afamqp.c +++ b/modules/afamqp/afamqp.c @@ -374,7 +374,8 @@ afamqp_dd_connect(AMQPDestDriver *self, gboolean reconnect) */ static gboolean -afamqp_vp_foreach(const gchar *name, const gchar *value, +afamqp_vp_foreach(const gchar *name, + LogTemplateType type, const gchar *value, gpointer user_data) { amqp_table_entry_t **entries = (amqp_table_entry_t **) ((gpointer *)user_data)[0]; diff --git a/modules/afmongodb/afmongodb.c b/modules/afmongodb/afmongodb.c index 9779097..5230d8f 100644 --- a/modules/afmongodb/afmongodb.c +++ b/modules/afmongodb/afmongodb.c @@ -359,7 +359,7 @@ afmongodb_vp_obj_end(const gchar *name, static gboolean afmongodb_vp_process_value(const gchar *name, const gchar *prefix, - const gchar *value, + LogTemplateType type, const gchar *value, gpointer *prefix_data, gpointer user_data) { bson *o; diff --git a/modules/json/format-json.c b/modules/json/format-json.c index d654c2e..4b79701 100644 --- a/modules/json/format-json.c +++ b/modules/json/format-json.c @@ -163,7 +163,8 @@ tf_json_obj_end(const gchar *name, } static gboolean -tf_json_value(const gchar *name, const gchar *prefix, const gchar *value, +tf_json_value(const gchar *name, const gchar *prefix, + LogTemplateType type, const gchar *value, gpointer *prefix_data, gpointer user_data) { json_state_t *state = (json_state_t *)user_data; diff --git a/tests/unit/test_value_pairs.c b/tests/unit/test_value_pairs.c index ca4ba32..3fe09de 100644 --- a/tests/unit/test_value_pairs.c +++ b/tests/unit/test_value_pairs.c @@ -10,7 +10,7 @@ gboolean success = TRUE; gboolean -vp_keys_foreach(const gchar *name, const gchar *value, gpointer user_data) +vp_keys_foreach(const gchar *name, LogTemplateType type, const gchar *value, gpointer user_data) { gpointer *args = (gpointer *) user_data; GList **keys = (GList **) args[0]; @@ -123,7 +123,7 @@ main(int argc, char *argv[]) testcase("all-nv-pairs", NULL, ".SDATA.EventData@18372.4.Data,.SDATA.Keywords@18372.4.Keyword,.SDATA.meta.sequenceId,.SDATA.meta.sysUpTime,.SDATA.origin.ip,HOST,MESSAGE,MSGID,PID,PROGRAM", NULL); - testcase("everything", NULL, ".SDATA.EventData@18372.4.Data,.SDATA.Keywords@18372.4.Keyword,.SDATA.meta.sequenceId,.SDATA.meta.sysUpTime,.SDATA.origin.ip,AMPM,BSDTAG,C_DATE,C_DAY,C_FULLDATE,C_HOUR,C_ISODATE,C_MIN,C_MONTH,C_MONTH_ABBREV,C_MONTH_NAME,C_MONTH_WEEK,C_SEC,C_STAMP,C_TZ,C_T ZOFFSET,C_UNIXTIME,C_WEEK,C_WEEKDAY,C_WEEK_DAY,C_WEEK_DAY_ABBREV,C_WEEK_DAY_NAME,C_YEAR,C_YEAR_DAY,DATE,DAY,FACILITY,FACILITY_NUM,FULLDATE,HOST,HOUR,HOUR12,ISODATE,LEVEL,LEVEL_NUM,LOGHOST,MESSAGE,MIN,MONTH,MONTH_ABBREV,MONTH_NAME,MONTH_WEEK,MSEC,MSG,MSGHDR ,MSGID,PID,PRI,PRIORITY,PROGRAM,R_AMPM,R_DATE,R_DAY,R_FULLDATE,R_HOUR,R_HOUR12,R_ISODATE,R_MIN,R_MONTH,R_MONTH_ABBREV,R_MONTH_NAME,R_MONTH_WEEK,R_MSEC,R_SEC,R_STAMP,R_TZ,R_TZOFFSET,R_UNIXTIME,R_USEC,R_WEEK,R_WEEKDAY,R_WEEK_DAY,R_WEEK_DAY_ABBREV,R_WEEK_DAY_ NAME,R_YEAR,R_YEAR_DAY,SDATA,SEC,SEQNUM,SOURCEIP,STAMP,SYSUPTIME,S_AMPM,S_DATE,S_DAY,S_FULLDATE,S_HOUR,S_HOUR12,S_ISODATE,S_MIN,S_MONTH,S_MONTH_ABBREV,S_MONTH_NAME,S_MONTH_WEEK,S_MSEC,S_SEC,S _STAMP,S _TZ,S_TZOFFSET,S_UNIXTIME,S_USEC,S_WEEK,S_WEEKDAY,S_WEEK_DAY,S_WEEK_DAY_ABBREV,S_WEEK_DAY_NAME,S_YEAR,S_YEAR_DAY,TAG,TAGS,TZ,TZOFFSET,UNIXTIME,USEC,WEEK,WEEKDAY,WEEK_DAY,WEEK_DAY_ABBREV,WEEK_DAY_NAME,YEAR,YEAR_DAY", NULL); + testcase("everything", NULL, ".SDATA.EventData@18372.4.Data,.SDATA.Keywords@18372.4.Keyword,.SDATA.meta.sequenceId,.SDATA.meta.sysUpTime,.SDATA.origin.ip,AMPM,BSDTAG,CONTEXT_ID,C_DATE,C_DAY,C_FULLDATE,C_HOUR,C_ISODATE,C_MIN,C_MONTH,C_MONTH_ABBREV,C_MONTH_NAME,C_MONTH_WEEK,C_SEC,C_STA MP,C_TZ,C_TZOFFSET,C_UNIXTIME,C_WEEK,C_WEEKDAY,C_WEEK_DAY,C_WEEK_DAY_ABBREV,C_WEEK_DAY_NAME,C_YEAR,C_YEAR_DAY,DATE,DAY,FACILITY,FACILITY_NUM,FULLDATE,HOST,HOUR,HOUR12,ISODATE,LEVEL,LEVEL_NUM,LOGHOST,MESSAGE,MIN,MONTH,MONTH_ABBREV,MONTH_NAME,MONTH_WEEK,MSEC ,MSG,MSGHDR,MSGID,PID,PRI,PRIORITY,PROGRAM,R_AMPM,R_DATE,R_DAY,R_FULLDATE,R_HOUR,R_HOUR12,R_ISODATE,R_MIN,R_MONTH,R_MONTH_ABBREV,R_MONTH_NAME,R_MONTH_WEEK,R_MSEC,R_SEC,R_STAMP,R_TZ,R_TZOFFSET,R_UNIXTIME,R_USEC,R_WEEK,R_WEEKDAY,R_WEEK_DAY,R_WEEK_DAY_ABBREV, R_WEEK_DAY_NAME,R_YEAR,R_YEAR_DAY,SDATA,SEC,SEQNUM,SOURCEIP,STAMP,SYSUPTIME,S_AMPM,S_DATE,S_DAY,S_FULLDATE,S_HOUR,S_HOUR12,S_ISODATE,S_MIN,S_MONTH,S_MONTH_ABBREV,S_MONTH_NAME,S_MONTH_WEEK,S_M SEC,S_SE C,S_STAMP,S_TZ,S_TZOFFSET,S_UNIXTIME,S_USEC,S_WEEK,S_WEEKDAY,S_WEEK_DAY,S_WEEK_DAY_ABBREV,S_WEEK_DAY_NAME,S_YEAR,S_YEAR_DAY,TAG,TAGS,TZ,TZOFFSET,UNIXTIME,USEC,WEEK,WEEKDAY,WEEK_DAY,WEEK_DAY_ABBREV,WEEK_DAY_NAME,YEAR,YEAR_DAY", NULL); testcase("nv-pairs", ".SDATA.*", "HOST,MESSAGE,MSGID,PID,PROGRAM", NULL); @@ -139,7 +139,7 @@ main(int argc, char *argv[]) g_ptr_array_add(transformers, value_pairs_new_transform_shift(2)); g_ptr_array_add(transformers, value_pairs_new_transform_replace("C_", "CC_")); - testcase("everything", NULL, ".SDATA.EventData@18372.4.Data,.SDATA.Keywords@18372.4.Keyword,.SDATA.meta.sequenceId,.SDATA.meta.sysUpTime,.SDATA.origin.ip,AMPM,BSDTAG,CC_DATE,CC_DAY,CC_FULLDATE,CC_HOUR,CC_ISODATE,CC_MIN,CC_MONTH,CC_MONTH_ABBREV,CC_MONTH_NAME,CC_MONTH_WEEK,CC_SEC,CC_ST AMP,CC_TZ,CC_TZOFFSET,CC_UNIXTIME,CC_WEEK,CC_WEEKDAY,CC_WEEK_DAY,CC_WEEK_DAY_ABBREV,CC_WEEK_DAY_NAME,CC_YEAR,CC_YEAR_DAY,DATE,DAY,FACILITY,FACILITY_NUM,FULLDATE,HOST,HOUR,HOUR12,ISODATE,LEVEL,LEVEL_NUM,LOGHOST,MESSAGE,MIN,MONTH,MONTH_ABBREV,MONTH_NAME,MONT H_WEEK,MSEC,MSG,MSGHDR,MSGID,PID,PRI,PRIORITY,PROGRAM,R_AMPM,R_DATE,R_DAY,R_FULLDATE,R_HOUR,R_HOUR12,R_ISODATE,R_MIN,R_MONTH,R_MONTH_ABBREV,R_MONTH_NAME,R_MONTH_WEEK,R_MSEC,R_SEC,R_STAMP,R_TZ,R_TZOFFSET,R_UNIXTIME,R_USEC,R_WEEK,R_WEEKDAY,R_WEEK_DAY,R_WEEK_ DAY_ABBREV,R_WEEK_DAY_NAME,R_YEAR,R_YEAR_DAY,SDATA,SEC,SEQNUM,SOURCEIP,STAMP,SYSUPTIME,S_AMPM,S_DATE,S_DAY,S_FULLDATE,S_HOUR,S_HOUR12,S_ISODATE,S_MIN,S_MONTH,S_MONTH_ABBREV,S_MONTH_NAME,S_MON TH_WEEK, S_MSEC,S_SEC,S_STAMP,S_TZ,S_TZOFFSET,S_UNIXTIME,S_USEC,S_WEEK,S_WEEKDAY,S_WEEK_DAY,S_WEEK_DAY_ABBREV,S_WEEK_DAY_NAME,S_YEAR,S_YEAR_DAY,TAG,TAGS,TZ,TZOFFSET,UNIXTIME,USEC,WEEK,WEEKDAY,WEEK_DAY,WEEK_DAY_ABBREV,WEEK_DAY_NAME,YEAR,YEAR_DAY", transformers); + testcase("everything", NULL, ".SDATA.EventData@18372.4.Data,.SDATA.Keywords@18372.4.Keyword,.SDATA.meta.sequenceId,.SDATA.meta.sysUpTime,.SDATA.origin.ip,AMPM,BSDTAG,CC_DATE,CC_DAY,CC_FULLDATE,CC_HOUR,CC_ISODATE,CC_MIN,CC_MONTH,CC_MONTH_ABBREV,CC_MONTH_NAME,CC_MONTH_WEEK,CC_SEC,CC_ST AMP,CC_TZ,CC_TZOFFSET,CC_UNIXTIME,CC_WEEK,CC_WEEKDAY,CC_WEEK_DAY,CC_WEEK_DAY_ABBREV,CC_WEEK_DAY_NAME,CC_YEAR,CC_YEAR_DAY,CONTEXT_ID,DATE,DAY,FACILITY,FACILITY_NUM,FULLDATE,HOST,HOUR,HOUR12,ISODATE,LEVEL,LEVEL_NUM,LOGHOST,MESSAGE,MIN,MONTH,MONTH_ABBREV,MONT H_NAME,MONTH_WEEK,MSEC,MSG,MSGHDR,MSGID,PID,PRI,PRIORITY,PROGRAM,R_AMPM,R_DATE,R_DAY,R_FULLDATE,R_HOUR,R_HOUR12,R_ISODATE,R_MIN,R_MONTH,R_MONTH_ABBREV,R_MONTH_NAME,R_MONTH_WEEK,R_MSEC,R_SEC,R_STAMP,R_TZ,R_TZOFFSET,R_UNIXTIME,R_USEC,R_WEEK,R_WEEKDAY,R_WEEK_ DAY,R_WEEK_DAY_ABBREV,R_WEEK_DAY_NAME,R_YEAR,R_YEAR_DAY,SDATA,SEC,SEQNUM,SOURCEIP,STAMP,SYSUPTIME,S_AMPM,S_DATE,S_DAY,S_FULLDATE,S_HOUR,S_HOUR12,S_ISODATE,S_MIN,S_MONTH,S_MONTH_ABBREV,S_MONTH _NAME,S_ MONTH_WEEK,S_MSEC,S_SEC,S_STAMP,S_TZ,S_TZOFFSET,S_UNIXTIME,S_USEC,S_WEEK,S_WEEKDAY,S_WEEK_DAY,S_WEEK_DAY_ABBREV,S_WEEK_DAY_NAME,S_YEAR,S_YEAR_DAY,TAG,TAGS,TZ,TZOFFSET,UNIXTIME,USEC,WEEK,WEEKDAY,WEEK_DAY,WEEK_DAY_ABBREV,WEEK_DAY_NAME,YEAR,YEAR_DAY", transformers); g_ptr_array_free(transformers, TRUE); app_shutdown(); -- 1.7.10.4
______________________________________________________________________________ Member info: https://lists.balabit.hu/mailman/listinfo/syslog-ng Documentation: http://www.balabit.com/support/documentation/?product=syslog-ng FAQ: http://www.balabit.com/wiki/syslog-ng-faq
Balazs Scheidler <bazsi77@gmail.com> writes:
+/* START_RULES */ + +template_content + : string { $$.str = $1; $$.typehint = NULL; } + | LL_IDENTIFIER '(' string ')' { $$.str = $3; $$.typehint = $1; } + ; +
can't we move template compilation here somehow? e.g. return logtemplate as a ptr?
We probably can. I originally had it written in a very different way, and forgot to simplify it further.
+log_template_set_type_hint(LogTemplate *self, const gchar *hint, GError **error) +{ + if (hint == NULL) + return TRUE; + + if (strcmp(hint, "string") == 0) + self->type_hint = TEMPLATE_TYPE_STRING; + else if (strcmp(hint, "int32") == 0 || strcmp(hint, "int") == 0) + self->type_hint = TEMPLATE_TYPE_INT32; + else if (strcmp(hint, "int64") == 0) + self->type_hint = TEMPLATE_TYPE_INT64; + else if (strcmp(hint, "datetime") == 0) + self->type_hint = TEMPLATE_TYPE_DATETIME; + else if (strcmp(hint, "boolean") == 0) + self->type_hint = TEMPLATE_TYPE_BOOLEAN; + else if (strcmp(hint, "default") == 0) + self->type_hint = TEMPLATE_TYPE_DEFAULT; + else + { + g_set_error(error, LOG_TEMPLATE_ERROR, LOG_TEMPLATE_ERROR_TYPE, + "%s", hint); + return FALSE; + } + + return TRUE; +}
I'm thinking about whether it'd make sense to configure this at runtime, and whether the list of supported values could be destination specific.
I think we should have a centrally governed list, both because that's simpler, and because there's only so many times our current drivers can potentially support. I don't really see a need to make it extensible at this point. If the need arises, we can rework the code, until then, I don't really see the point.
+typedef enum +{ + TEMPLATE_TYPE_STRING, + TEMPLATE_TYPE_BOOLEAN, + TEMPLATE_TYPE_INT32, + TEMPLATE_TYPE_INT64, + TEMPLATE_TYPE_DATETIME, + TEMPLATE_TYPE_DEFAULT, +} LogTemplateType; +
default is somehow an exception here.
Mhm. That's not really a type, indeed. Perhaps it should be lifted out, and implemented separately from the type hinting system? -- |8]
Balazs Scheidler <bazsi77@gmail.com> writes:
+template_content + : string { $$.str = $1; $$.typehint = NULL; } + | LL_IDENTIFIER '(' string ')' { $$.str = $3; $$.typehint = $1; } + ; +
can't we move template compilation here somehow? e.g. return logtemplate as a ptr?
We can. Turns out that the reason I did it this way, was due to the filter expressions, but there type-hinting makes little sense, so I updated the patch in such a way that template_content returns a LogTemplate as <ptr>, and filter expressions don't support type hinting anymore. This did make the code a lot nicer, thanks! I pushed the new code to feature/3.4/templates/cast-typehints for now, as this is something I'll likely be poking at during the next few days, and would need a througher review. So I'm not pushing it to merge-queue/3.4 yet. (The updated branch also adds a set of unit tests to guard the hinting & type casting logic) -- |8]
Since we have type hints available, lets use them! This adds support for 32- and 64-bit integers, datetime and boolean types (along with the already existing string type, of course) for the mongodb destination. Signed-off-by: Gergely Nagy <algernon@balabit.hu> --- modules/afmongodb/afmongodb.c | 46 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/modules/afmongodb/afmongodb.c b/modules/afmongodb/afmongodb.c index 5230d8f..ea9c002 100644 --- a/modules/afmongodb/afmongodb.c +++ b/modules/afmongodb/afmongodb.c @@ -22,6 +22,7 @@ */ #include <time.h> +#include <stdlib.h> #include "afmongodb.h" #include "afmongodb-parser.h" @@ -369,7 +370,50 @@ afmongodb_vp_process_value(const gchar *name, const gchar *prefix, else o = (bson *)user_data; - bson_append_string (o, name, value, -1); + switch (type) + { + case TEMPLATE_TYPE_BOOLEAN: + { + if (value[0] == 'T' || value[0] == 't' || value[0] == '1') + bson_append_boolean (o, name, TRUE); + else + bson_append_boolean (o, name, FALSE); + break; + } + case TEMPLATE_TYPE_INT32: + { + gchar *endptr; + gint32 i = (gint32)strtoul(value, &endptr, 10); + + if (endptr[0] == '\0') + bson_append_int32 (o, name, i); + + break; + } + case TEMPLATE_TYPE_INT64: + { + gchar *endptr; + gint64 i = (gint64)strtoul(value, &endptr, 10); + + if (endptr[0] == '\0') + bson_append_int64 (o, name, i); + + break; + } + case TEMPLATE_TYPE_DATETIME: + { + gchar *endptr; + gint64 i = (gint64)strtoul(value, &endptr, 10) * 1000; + + if (endptr[0] == '\0') + bson_append_utc_datetime (o, name, i); + + break; + } + default: + bson_append_string (o, name, value, -1); + break; + } return FALSE; } -- 1.7.10.4
----- Original message -----
Since we have type hints available, lets use them! This adds support for 32- and 64-bit integers, datetime and boolean types (along with the already existing string type, of course) for the mongodb destination.
Signed-off-by: Gergely Nagy <algernon@balabit.hu> --- modules/afmongodb/afmongodb.c | 46 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-)
diff --git a/modules/afmongodb/afmongodb.c b/modules/afmongodb/afmongodb.c index 5230d8f..ea9c002 100644 --- a/modules/afmongodb/afmongodb.c +++ b/modules/afmongodb/afmongodb.c @@ -22,6 +22,7 @@ */ #include <time.h> +#include <stdlib.h> #include "afmongodb.h" #include "afmongodb-parser.h" @@ -369,7 +370,50 @@ afmongodb_vp_process_value(const gchar *name, const gchar *prefix, else o = (bson *)user_data; - bson_append_string (o, name, value, -1); + switch (type) + { + case TEMPLATE_TYPE_BOOLEAN: + { + if (value[0] == 'T' || value[0] == 't' || value[0] == '1') + bson_append_boolean (o, name, TRUE); + else + bson_append_boolean (o, name, FALSE); + break; + } + case TEMPLATE_TYPE_INT32: + { + gchar *endptr; + gint32 i = (gint32)strtoul(value, &endptr, 10); + + if (endptr[0] == '\0') + bson_append_int32 (o, name, i); + + break; + }
are you sure it's a good idea to drop data for invalid formats? i'd have added as strings in this case. this applies to all cases.
+ case TEMPLATE_TYPE_INT64: + { + gchar *endptr; + gint64 i = (gint64)strtoul(value, &endptr, 10);
strtoll?
+ + if (endptr[0] == '\0') + bson_append_int64 (o, name, i); + + break; + } + case TEMPLATE_TYPE_DATETIME: + { + gchar *endptr; + gint64 i = (gint64)strtoul(value, &endptr, 10) * 1000; + + if (endptr[0] == '\0') + bson_append_utc_datetime (o, name, i); + + break; + }
we might need to support a format with broken-down timestamp. an iso stamp might easily be formatted using a template, and the log message may carry such date in its payload.
+ default: + bson_append_string (o, name, value, -1); + break; + } return FALSE; } -- 1.7.10.4
______________________________________________________________________________ Member info: https://lists.balabit.hu/mailman/listinfo/syslog-ng Documentation: http://www.balabit.com/support/documentation/?product=syslog-ng FAQ: http://www.balabit.com/wiki/syslog-ng-faq
Balazs Scheidler <bazsi77@gmail.com> writes:
+ case TEMPLATE_TYPE_INT32: + { + gchar *endptr; + gint32 i = (gint32)strtoul(value, &endptr, 10); + + if (endptr[0] == '\0') + bson_append_int32 (o, name, i); + + break; + }
are you sure it's a good idea to drop data for invalid formats? i'd have added as strings in this case. this applies to all cases.
I'm not convinced, but the reason behind dropping was that if the user requested an int type, putting in string is not what one would expect. Probably it would be better to propagate some kind of error to higher layers, and report it back, though.
+ case TEMPLATE_TYPE_DATETIME: + { + gchar *endptr; + gint64 i = (gint64)strtoul(value, &endptr, 10) * 1000; + + if (endptr[0] == '\0') + bson_append_utc_datetime (o, name, i); + + break; + }
we might need to support a format with broken-down timestamp. an iso stamp might easily be formatted using a template, and the log message may carry such date in its payload.
It's entirely up to drivers how to handle dates, imo. MongoDB only supports a single datetime format: a 64bit int (microseconds since the epoch). So for this driver, $UNIXTIME is the closest thing. SQL destinations, or other formats that support different datetime representation (eg, $(format-edn), which I will submit soonish) will likely end up doing something very different to what the MongoDB driver is doing. -- |8]
Gergely Nagy <algernon@balabit.hu> writes:
Balazs Scheidler <bazsi77@gmail.com> writes:
+ case TEMPLATE_TYPE_INT32: + { + gchar *endptr; + gint32 i = (gint32)strtoul(value, &endptr, 10); + + if (endptr[0] == '\0') + bson_append_int32 (o, name, i); + + break; + }
are you sure it's a good idea to drop data for invalid formats? i'd have added as strings in this case. this applies to all cases.
I'm not convinced, but the reason behind dropping was that if the user requested an int type, putting in string is not what one would expect.
Probably it would be better to propagate some kind of error to higher layers, and report it back, though.
After talking about this in-house, the way these kind of things should be handled is as follows: We'd have a global (overridable per-destination) setting called type-hint(strict|weak|fallback). If set to strict (default), then any typecasting error should result in the message being discarded, and an appropriate error logged. If set to weak, then any typecasting error should result in the value beging discarded (but the rest of the message not), with a warning produced aswell. If set to fallback, then any typecasting error should result in the value getting stored as string instead, with a warning produced aswell. Any of these can be prefixed with "silent-", which turns the error/warning off.
+ case TEMPLATE_TYPE_DATETIME: + { + gchar *endptr; + gint64 i = (gint64)strtoul(value, &endptr, 10) * 1000; + + if (endptr[0] == '\0') + bson_append_utc_datetime (o, name, i); + + break; + }
we might need to support a format with broken-down timestamp. an iso stamp might easily be formatted using a template, and the log message may carry such date in its payload.
I moved the parsing & casting stuff to lib/type-hint.[ch], which makes it possible to extend type_cast_to_datetime_int() so that it recognises not only $UNIXTIME, but other formats aswell. This way, anything that uses these helper functions (only mongodb for now), will support them all automatically. Some drivers (format-json, for example) do not use the helpers though. -- |8]
Unfortunately, format-json does not use the yacc grammar to parse its options, so we have to implement our own: this is what value_pairs_parse_type() does, conveniently hidden, so that value_pairs_new_from_cmdline() will do the right thing, and annotate the constructed templates with type hints. The assumption made here is that if a template string has '(' and a ')' after, then it is a type-hinted template. This may be refined later to be stricter (for example, to require the closing parens to be at the end). And since we can construct type-hinted templates from the command-line, we can teach $(format-json) to handle these types too! Signed-off-by: Gergely Nagy <algernon@balabit.hu> --- lib/value-pairs.c | 32 ++++++++++++++++++++++++++++++-- modules/json/format-json.c | 35 ++++++++++++++++++++++++++++++----- modules/json/tests/test_json.c | 12 ++++++++++++ 3 files changed, 72 insertions(+), 7 deletions(-) diff --git a/lib/value-pairs.c b/lib/value-pairs.c index da2e391..a8511fd 100644 --- a/lib/value-pairs.c +++ b/lib/value-pairs.c @@ -151,6 +151,32 @@ value_pairs_add_glob_pattern(ValuePairs *vp, const gchar *pattern, vp->patterns[i] = p; } +static void +value_pairs_parse_type(gchar *spec, gchar **value, gchar **type) +{ + char *sp, *ep; + + *type = NULL; + + sp = strchr(spec, '('); + if (sp == NULL) + { + *value = spec; + return; + } + ep = strchr(sp, ')'); + if (ep == NULL) + { + *value = spec; + return; + } + + *value = sp + 1; + *type = spec; + sp[0] = '\0'; + ep[0] = '\0'; +} + gboolean value_pairs_add_pair_with_type(ValuePairs *vp, GlobalConfig *cfg, const gchar *key, const gchar *type, @@ -770,7 +796,7 @@ vp_cmdline_parse_pair (const gchar *option_name, const gchar *value, gpointer *args = (gpointer *) data; ValuePairs *vp = (ValuePairs *) args[1]; GlobalConfig *cfg = (GlobalConfig *) args[0]; - gchar **kv; + gchar **kv, *v, *t; vp_cmdline_parse_rekey_finish (data); if (!g_strstr_len (value, strlen (value), "=")) @@ -781,7 +807,9 @@ vp_cmdline_parse_pair (const gchar *option_name, const gchar *value, } kv = g_strsplit(value, "=", 2); - value_pairs_add_pair (vp, cfg, kv[0], kv[1]); + value_pairs_parse_type(kv[1], &v, &t); + + value_pairs_add_pair_with_type(vp, cfg, kv[0], t, v, NULL); g_free (kv[0]); g_free (kv[1]); diff --git a/modules/json/format-json.c b/modules/json/format-json.c index 4b79701..cbdb513 100644 --- a/modules/json/format-json.c +++ b/modules/json/format-json.c @@ -172,11 +172,36 @@ tf_json_value(const gchar *name, const gchar *prefix, if (state->need_comma) g_string_append_c(state->buffer, ','); - g_string_append_c(state->buffer, '"'); - g_string_append_escaped(state->buffer, name); - g_string_append(state->buffer, "\":\""); - g_string_append_escaped(state->buffer, value); - g_string_append_c(state->buffer, '"'); + switch (type) + { + case TEMPLATE_TYPE_STRING: + case TEMPLATE_TYPE_DATETIME: + default: + g_string_append_c(state->buffer, '"'); + g_string_append_escaped(state->buffer, name); + g_string_append(state->buffer, "\":\""); + g_string_append_escaped(state->buffer, value); + g_string_append_c(state->buffer, '"'); + break; + case TEMPLATE_TYPE_INT32: + case TEMPLATE_TYPE_INT64: + g_string_append_c(state->buffer, '"'); + g_string_append_escaped(state->buffer, name); + g_string_append(state->buffer, "\":"); + g_string_append(state->buffer, value); + break; + case TEMPLATE_TYPE_BOOLEAN: + { + g_string_append_c(state->buffer, '"'); + g_string_append_escaped(state->buffer, name); + g_string_append(state->buffer, "\":"); + if (value[0] == 't' || value[0] == 'T' || value[0] == '1') + g_string_append(state->buffer, "true"); + else + g_string_append(state->buffer, "false"); + break; + } + } state->need_comma = TRUE; diff --git a/modules/json/tests/test_json.c b/modules/json/tests/test_json.c index 8087c3e..ada48b6 100644 --- a/modules/json/tests/test_json.c +++ b/modules/json/tests/test_json.c @@ -23,6 +23,17 @@ test_format_json_rekey(void) "{\"_msg\":{\"text\":\"dotted\"}}"); } +void +test_format_json_with_type_hints(void) +{ + assert_template_format("$(format-json i32=int32(1234))", + "{\"i32\":1234}"); + assert_template_format("$(format-json \"i=ifoo(\")", + "{\"i\":\"ifoo(\"}"); + assert_template_format("$(format-json b=boolean(TRUE))", + "{\"b\":true}"); +} + int main(int argc G_GNUC_UNUSED, char *argv[] G_GNUC_UNUSED) { @@ -34,6 +45,7 @@ main(int argc G_GNUC_UNUSED, char *argv[] G_GNUC_UNUSED) test_format_json(); test_format_json_rekey(); + test_format_json_with_type_hints(); deinit_template_tests(); app_shutdown(); -- 1.7.10.4
participants (2)
-
Balazs Scheidler
-
Gergely Nagy