With this patch, $(format-json) becomes able to act on type hints, and store ints as ints and so on and so forth. Forthermore, the on-error() setting is implemented for format-json aswell. Together with the afmongodb patch, this fixes #6. Signed-off-by: Gergely Nagy <algernon@balabit.hu> --- modules/json/format-json.c | 101 ++++++++++++++++++++++++++++++++++------ modules/json/tests/test_json.c | 47 +++++++++++++++++++ 2 files changed, 133 insertions(+), 15 deletions(-) diff --git a/modules/json/format-json.c b/modules/json/format-json.c index edb1608..06673f1 100644 --- a/modules/json/format-json.c +++ b/modules/json/format-json.c @@ -1,7 +1,7 @@ /* - * Copyright (c) 2011-2012 BalaBit IT Ltd, Budapest, Hungary + * Copyright (c) 2011-2013 BalaBit IT Ltd, Budapest, Hungary * Copyright (c) 2011 Balint Kovacs <blint@balabit.hu> - * Copyright (c) 2011-2012 Gergely Nagy <algernon@balabit.hu> + * Copyright (c) 2011-2013 Gergely Nagy <algernon@balabit.hu> * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 as published @@ -29,11 +29,13 @@ #include "cfg.h" #include "value-pairs.h" #include "vptransform.h" +#include "syslog-ng.h" typedef struct _TFJsonState { TFSimpleFuncState super; ValuePairs *vp; + gint tc_strictness; } TFJsonState; static gboolean @@ -48,6 +50,8 @@ tf_json_prepare(LogTemplateFunction *self, gpointer s, LogTemplate *parent, if (!state->vp) return FALSE; + state->tc_strictness = parent->cfg->type_cast_strictness; + /* Always replace a leading dot with an underscore. */ vpts = value_pairs_transform_set_new(".*"); value_pairs_transform_set_add_func(vpts, value_pairs_new_transform_replace(".", "_")); @@ -60,6 +64,7 @@ typedef struct { gboolean need_comma; GString *buffer; + gint tc_strictness; } json_state_t; static inline void @@ -163,37 +168,97 @@ tf_json_obj_end(const gchar *name, } static gboolean -tf_json_value(const gchar *name, const gchar *prefix, - TypeHint type, const gchar *value, - gpointer *prefix_data, gpointer user_data) +tf_json_append_value(const gchar *name, const gchar *value, + json_state_t *state, gboolean quoted) { - json_state_t *state = (json_state_t *)user_data; - if (state->need_comma) g_string_append_c(state->buffer, ','); g_string_append_c(state->buffer, '"'); g_string_append_escaped(state->buffer, name); - g_string_append(state->buffer, "\":\""); + + if (quoted) + g_string_append(state->buffer, "\":\""); + else + g_string_append(state->buffer, "\":"); + g_string_append_escaped(state->buffer, value); - g_string_append_c(state->buffer, '"'); + + if (quoted) + g_string_append_c(state->buffer, '"'); + + return TRUE; +} + +static gboolean +tf_json_value(const gchar *name, const gchar *prefix, + TypeHint type, const gchar *value, + gpointer *prefix_data, gpointer user_data) +{ + json_state_t *state = (json_state_t *)user_data; + gint tc_strictness = state->tc_strictness; + + switch (type) + { + case TYPE_HINT_STRING: + case TYPE_HINT_DATETIME: + default: + tf_json_append_value(name, value, state, TRUE); + break; + case TYPE_HINT_LITERAL: + tf_json_append_value(name, value, state, FALSE); + break; + case TYPE_HINT_INT32: + case TYPE_HINT_INT64: + case TYPE_HINT_BOOLEAN: + { + gint32 i32; + gint64 i64; + gboolean b; + gboolean r = FALSE, fail = FALSE; + const gchar *v = value; + + if (type == TYPE_HINT_INT32 && + (fail = !type_cast_to_int32(value, &i32 , NULL)) == TRUE) + r = type_cast_drop_helper(tc_strictness, value, "int32"); + else if (type == TYPE_HINT_INT64 && + (fail = !type_cast_to_int64(value, &i64 , NULL)) == TRUE) + r = type_cast_drop_helper(tc_strictness, value, "int64"); + else if (type == TYPE_HINT_BOOLEAN) + { + if ((fail = !type_cast_to_boolean(value, &b , NULL)) == TRUE) + r = type_cast_drop_helper(tc_strictness, value, "boolean"); + else + v = b ? "true" : "false"; + } + + if (fail && + !(tc_strictness & TYPE_CAST_FALLBACK_TO_STRING)) + return r; + + tf_json_append_value(name, v, state, fail); + break; + } + } state->need_comma = TRUE; return FALSE; } -static void -tf_json_append(GString *result, ValuePairs *vp, LogMessage *msg) +static gboolean +tf_json_append(GString *result, ValuePairs *vp, + gint strictness, LogMessage *msg) { json_state_t state; state.need_comma = FALSE; state.buffer = result; + state.tc_strictness = strictness; - value_pairs_walk(vp, - tf_json_obj_start, tf_json_value, tf_json_obj_end, - msg, 0, &state); + return value_pairs_walk(vp, + tf_json_obj_start, tf_json_value, tf_json_obj_end, + msg, 0, &state); } static void @@ -202,9 +267,15 @@ tf_json_call(LogTemplateFunction *self, gpointer s, { TFJsonState *state = (TFJsonState *)s; gint i; + gboolean r = TRUE; + gsize orig_size = result->len; for (i = 0; i < args->num_messages; i++) - tf_json_append(result, state->vp, args->messages[i]); + r &= tf_json_append(result, state->vp, + state->tc_strictness, args->messages[i]); + + if (!r && state->tc_strictness & TYPE_CAST_DROP_MESSAGE) + g_string_set_size(result, orig_size); } static void diff --git a/modules/json/tests/test_json.c b/modules/json/tests/test_json.c index 8087c3e..4061c4c 100644 --- a/modules/json/tests/test_json.c +++ b/modules/json/tests/test_json.c @@ -23,6 +23,51 @@ test_format_json_rekey(void) "{\"_msg\":{\"text\":\"dotted\"}}"); } +void +test_format_json_with_type_hints(void) +{ + assert_template_format("$(format-json i32=int32(1234))", + "{\"i32\":1234}"); + assert_template_format("$(format-json \"i=ifoo(\")", + "{\"i\":\"ifoo(\"}"); + assert_template_format("$(format-json b=boolean(TRUE))", + "{\"b\":true}"); +} + +void +test_format_json_strictness(void) +{ + configuration->type_cast_strictness = TYPE_CAST_DROP_MESSAGE; + assert_template_format("$(format-json x=y bad=boolean(blah) foo=bar)", + ""); + assert_template_format("$(format-json x=y bad=int32(blah) foo=bar)", + ""); + assert_template_format("$(format-json x=y bad=int64(blah) foo=bar)", + ""); + + configuration->type_cast_strictness = TYPE_CAST_DROP_PROPERTY; + assert_template_format("$(format-json x=y bad=boolean(blah) foo=bar)", + "{\"x\":\"y\",\"foo\":\"bar\"}"); + assert_template_format("$(format-json x=y bad=boolean(blah))", + "{\"x\":\"y\"}"); + assert_template_format("$(format-json x=y bad=int32(blah))", + "{\"x\":\"y\"}"); + assert_template_format("$(format-json x=y bad=int64(blah))", + "{\"x\":\"y\"}"); + + configuration->type_cast_strictness = TYPE_CAST_FALLBACK_TO_STRING; + + assert_template_format("$(format-json x=y bad=boolean(blah) foo=bar)", + "{\"x\":\"y\",\"foo\":\"bar\",\"bad\":\"blah\"}"); + assert_template_format("$(format-json x=y bad=boolean(blah))", + "{\"x\":\"y\",\"bad\":\"blah\"}"); + assert_template_format("$(format-json x=y bad=int32(blah))", + "{\"x\":\"y\",\"bad\":\"blah\"}"); + assert_template_format("$(format-json x=y bad=int64(blah))", + "{\"x\":\"y\",\"bad\":\"blah\"}"); + +} + int main(int argc G_GNUC_UNUSED, char *argv[] G_GNUC_UNUSED) { @@ -34,6 +79,8 @@ main(int argc G_GNUC_UNUSED, char *argv[] G_GNUC_UNUSED) test_format_json(); test_format_json_rekey(); + test_format_json_with_type_hints(); + test_format_json_strictness(); deinit_template_tests(); app_shutdown(); -- 1.7.10.4