[syslog-ng] [PATCH] tfjson: $(format_json) template function implementation.
Balazs Scheidler
bazsi at balabit.hu
Sun Feb 6 21:07:40 CET 2011
Hi,
Since this is a self-contained module, I'm willing to include it in 3.2
(and certainly in 3.3), but I'd like the value-pairs() to be finalized
that Algernon is working on. The only reason is that I wouldn't want to
handle possible incompatible changes if the interface will change with
value-pairs.
On Mon, 2011-01-31 at 18:29 -0800, Matthew Hall wrote:
> Implements a $(format_json) template function, largely based on the
> work of Balint Kovacs <blint at balabit.hu>, built on top of the json-c
> library.
>
> The syntax of the template function is as follows:
>
> $(format_json [--select <GLOB>] [--exclude <GLOB>]
> [--skip-builtins]
> [MACRO_NAMES...] KEY=VALUE)
>
> There can be multiple selects, excludes, macro names and key=value
> pairs. The VALUE part of the key=value pair can contain template
> macros.
>
> Macro names must be listed without the "$" sign, however.
>
> If --skip-builtins is specified, all built-in variables will be
> excluded.
>
> As an example, this is a valid argument list (without the line
> breaks):
>
> $(format_json --select .classification.* --select useracct.*
> --exclude *.*id HOST my_own_key='$SOURCEIP ($HOST)')
>
> This will include every key that begins with ".classification." or
> "useracct.", but will exclude any, that has a dot, and ends with
> "id". The latter keys will be excluded even if they match any of the
> select patterns.
>
> Furthermore, the $HOST macro will be included (with the key HOST),
> along with a my_own_key key, whose value will be evaluated as a
> template, thus, will contain the source IP and the HOST (again, for
> demo purposes).
>
> Signed-off-by: Gergely Nagy <algernon at balabit.hu>
> (cherry picked from commit 9ab6b381433d657b59eb1d19f994f9328fca9547)
>
> Conflicts:
>
> configure.in
> modules/Makefile.am
>
> removed mongodb references from configure.in and modules/Makefile.am; resolved conflicts there
> Signed-off-by: Matthew Hall <mhall at mhcomputing.net>
> ---
> configure.in | 27 ++++-
> modules/Makefile.am | 3 +-
> modules/tfjson/Makefile.am | 11 ++
> modules/tfjson/tfjson.c | 305 ++++++++++++++++++++++++++++++++++++++++++++
> 4 files changed, 344 insertions(+), 2 deletions(-)
> create mode 100644 modules/tfjson/Makefile.am
> create mode 100644 modules/tfjson/tfjson.c
>
> diff --git /tfjson-base/configure.in /syslog-ng-3.2/configure.in
> new file mode 100644
> index 0c6cef2..08d6a6b 100644
> --- /tfjson-base/configure.in
> +++ /syslog-ng-3.2/configure.in
> @@ -25,6 +25,8 @@ EVTLOG_MIN_VERSION="0.2"
> OPENSSL_MIN_VERSION="0.9.8"
> LIBDBI_MIN_VERSION="0.8.0"
> PCRE_MIN_VERSION="7.3"
> +IVYKIS_MIN_VERSION="0.18"
> +JSON_C_MIN_VERSION="0.9"
>
> dnl ***************************************************************************
> dnl Initial setup
> @@ -144,6 +146,10 @@ AC_ARG_ENABLE(gcov,
> ,,enable_gcov="no")
>
>
> +AC_ARG_ENABLE(json,
> + [ --enable-json Enable support for JSON template formatting (default: auto)]
> + ,,enable_json="auto")
> +
> dnl ***************************************************************************
> dnl Checks for programs.
> AC_PROG_CC
> @@ -493,6 +499,23 @@ if test "x$linking_mode" != "xdynamic" -a "x$blb_cv_static_glib" = "xno"; then
> fi
>
> dnl ***************************************************************************
> +dnl json-glib headers/libraries
> +dnl ***************************************************************************
> +if test "x$enable_json" = "xyes" -o "x$enable_json" = "xauto"; then
> + PKG_CHECK_MODULES(JSON, json >= $JSON_C_MIN_VERSION,,JSON_LIBS="")
> + if test -z "$JSON_LIBS"; then
> + if test "x$enable_json" = "xyes"; then
> + AC_MSG_ERROR(Cannot find json >= $JSON_C_MIN_VERSION.)
> + else
> + AC_MSG_WARN(Cannot find json >= $JSON_C_MIN_VERSION.)
> + fi
> + enable_json="no"
> + else
> + enable_json="yes"
> + fi
> +fi
> +
> +dnl ***************************************************************************
> dnl pcre headers/libraries
> dnl ***************************************************************************
>
> @@ -750,6 +773,7 @@ AM_CONDITIONAL(ENABLE_SSL, [test "$enable_ssl" = "yes"])
> AM_CONDITIONAL(ENABLE_SQL, [test "$enable_sql" = "yes"])
> AM_CONDITIONAL(ENABLE_SUN_STREAMS, [test "$enable_sun_streams" = "yes"])
> AM_CONDITIONAL(ENABLE_PACCT, [test "$enable_pacct" = "yes"])
> +AM_CONDITIONAL(ENABLE_JSON, [test "$enable_json" = "yes"])
>
> # substitution into manual pages
> expanded_sysconfdir=[`patheval $sysconfdir | sed -e 's/-/\\\\-/g'`]
> @@ -803,6 +827,7 @@ AC_OUTPUT(dist.conf
> modules/pacctformat/Makefile
> modules/basicfuncs/Makefile
> modules/convertfuncs/Makefile
> + modules/tfjson/Makefile
> scripts/Makefile
> scripts/update-patterndb
> doc/Makefile
> @@ -843,5 +868,5 @@ echo " Linux capability support : ${enable_linux_caps:=no}"
> echo " PCRE support : ${enable_pcre:=no}"
> echo " Env wrapper support : ${enable_env_wrapper:=no}"
> echo " PACCT module (EXPERIMENTAL) : ${enable_pacct:=no}"
> -
> +echo " JSON support (module) : ${enable_json:=no}"
>
> diff --git /tfjson-base/modules/Makefile.am /syslog-ng-3.2/modules/Makefile.am
> new file mode 100644
> index cfc721e..16e38df 100644
> --- /tfjson-base/modules/Makefile.am
> +++ /syslog-ng-3.2/modules/Makefile.am
> @@ -1 +1,2 @@
> -SUBDIRS = afsocket afsql afstreams affile afprog afuser csvparser confgen syslogformat pacctformat basicfuncs convertfuncs dbparser dummy
> +SUBDIRS = afsocket afsql afstreams affile afprog afuser csvparser confgen syslogformat pacctformat basicfuncs convertfuncs dbparser dummy \
> + tfjson
> diff --git /tfjson-base/modules/tfjson/Makefile.am /syslog-ng-3.2/modules/tfjson/Makefile.am
> new file mode 100644
> index 0000000..8b30f83
> --- /dev/null
> +++ /syslog-ng-3.2/modules/tfjson/Makefile.am
> @@ -0,0 +1,11 @@
> +moduledir = @moduledir@
> +export top_srcdir
> +
> +if ENABLE_JSON
> +AM_CPPFLAGS = -I$(top_srcdir)/lib -I../../lib @JSON_CFLAGS@
> +module_LTLIBRARIES = libtfjson.la
> +
> +libtfjson_la_SOURCES = tfjson.c
> +libtfjson_la_LIBADD = ../../lib/libsyslog-ng.la @JSON_LIBS@
> +libtfjson_la_LDFLAGS = -avoid-version
> +endif
> diff --git /tfjson-base/modules/tfjson/tfjson.c /syslog-ng-3.2/modules/tfjson/tfjson.c
> new file mode 100644
> index 0000000..e57dce3
> --- /dev/null
> +++ /syslog-ng-3.2/modules/tfjson/tfjson.c
> @@ -0,0 +1,305 @@
> +/*
> + * Copyright (c) 2002-2011 BalaBit IT Ltd, Budapest, Hungary
> + * Copyright (c) 2011 Balint Kovacs <blint at balabit.hu>
> + * Copyright (c) 2011 Gergely Nagy <algernon at balabit.hu>
> + *
> + * This program is free software; you can redistribute it and/or modify it
> + * under the terms of the GNU General Public License version 2 as published
> + * by the Free Software Foundation, or (at your option) any later version.
> + *
> + * This library is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with this library; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
> + *
> + */
> +
> +#include "plugin.h"
> +#include "templates.h"
> +#include "filter.h"
> +#include "filter-expr-parser.h"
> +#include "cfg.h"
> +
> +#include <json.h>
> +
> +typedef struct
> +{
> + GPatternSpec **select_patterns;
> + GPatternSpec **exclude_patterns;
> +
> + GHashTable *template_values;
> +
> + gboolean skip_builtins;
> +
> + /* Temporary variables. */
> + GString *res;
> +} TFJsonState;
> +
> +static void
> +tf_json_free_state (TFJsonState *state)
> +{
> + gint i = 0;
> +
> + if (state->select_patterns)
> + {
> + while (state->select_patterns[i])
> + {
> + g_pattern_spec_free(state->select_patterns[i]);
> + i++;
> + }
> + g_free (state->select_patterns);
> + }
> + if (state->exclude_patterns)
> + {
> + i = 0;
> + while (state->exclude_patterns[i])
> + {
> + g_pattern_spec_free(state->exclude_patterns[i]);
> + i++;
> + }
> + g_free (state->exclude_patterns);
> + }
> +
> + g_hash_table_destroy(state->template_values);
> + if (state->res)
> + g_string_free(state->res, TRUE);
> + g_free(state);
> +}
> +
> +static gboolean
> +tf_json_prepare(LogTemplateFunction *self, LogTemplate *parent,
> + gint argc, gchar *argv[],
> + gpointer *state, GDestroyNotify *state_destroy,
> + GError **error)
> +{
> + TFJsonState *jstate = g_new0(TFJsonState, 1);
> + GOptionContext *ctx;
> + GError *parse_error = NULL;
> + gchar **selects = NULL;
> + gchar **excludes = NULL;
> +
> + GOptionEntry format_options[] =
> + {
> + { "select", 's', 0, G_OPTION_ARG_STRING_ARRAY, &selects,
> + "Specify which name-value pairs are included in the result. The parameter is a shell-like glob pattern.",
> + "<glob expression>" },
> + { "exclude", 'x', 0, G_OPTION_ARG_STRING_ARRAY, &excludes,
> + "Specify which name-value pairs to exclude from the result. The parameter is a shell-like glob pattern.",
> + "<glob expression>" },
> + { "skip-builtins", 'D', 0, G_OPTION_ARG_NONE, &jstate->skip_builtins,
> + "Skip built-in variables, like $HOST and the rest.", NULL },
> + { NULL }
> + };
> +
> + gchar **ar;
> + gint arc = argc + 1;
> + gint i;
> +
> + ar = g_new(gchar *, arc + 1);
> + for (i = 0; i < argc; i++)
> + ar[i + 1] = g_strdup(argv[i]);
> + ar[0] = g_strdup("format-json");
> + ar[argc + 1] = NULL;
> +
> + jstate->template_values =
> + g_hash_table_new_full(g_str_hash, g_str_equal, g_free,
> + (GDestroyNotify) log_template_unref);
> + ctx = g_option_context_new("format-json");
> + msg_add_option_group(ctx);
> + g_option_context_add_main_entries(ctx, format_options, NULL);
> + if (!g_option_context_parse(ctx, &arc, &ar, &parse_error))
> + {
> + /*
> + msg_error ("Error parsing function arguments",
> + evt_tag_str("error", parse_error ?
> + parse_error->message : "Invalid arguments"));
> + */
> + g_set_error(error, LOG_TEMPLATE_ERROR, LOG_TEMPLATE_ERROR_COMPILE,
> + "Error parsing function arguments");
> + g_option_context_free(ctx);
> + tf_json_free_state(jstate);
> + g_strfreev(ar);
> + return FALSE;
> + }
> + g_option_context_free(ctx);
> +
> + if (selects == NULL)
> + {
> + jstate->select_patterns = g_new0(GPatternSpec *, 2);
> + jstate->select_patterns[0] = g_pattern_spec_new("*");
> + }
> + else
> + {
> + gint n = g_strv_length (selects);
> + jstate->select_patterns = g_new0(GPatternSpec *, n + 1);
> +
> + for (i = 0; i < n; i++)
> + jstate->select_patterns[i] = g_pattern_spec_new(selects[i]);
> + }
> +
> + if (excludes != NULL)
> + {
> + gint n = g_strv_length (excludes);
> + jstate->exclude_patterns = g_new0(GPatternSpec *, n + 1);
> +
> + for (i = 0; i < n; i++)
> + jstate->exclude_patterns[i] = g_pattern_spec_new(excludes[i]);
> + }
> +
> + for (i = 1; i < arc; i++)
> + {
> + gchar *key = NULL;
> + LogTemplate *t = NULL;
> +
> + if (g_strstr_len(ar[i], strlen (ar[i]), "="))
> + {
> + gchar **kv = g_strsplit(ar[i], "=", 2);
> +
> + key = g_strdup(kv[0]);
> + t = log_template_new(parent->cfg, NULL, kv[1]);
> + g_free(kv[0]);
> + g_free(kv[1]);
> + g_free(kv);
> + }
> + else
> + {
> + gchar *m = g_strconcat("$", ar[i], NULL);
> +
> + key = g_strdup(ar[i]);
> + t = log_template_new(parent->cfg, NULL, m);
> + g_free(m);
> + }
> + g_hash_table_insert(jstate->template_values, key, t);
> + }
> +
> + jstate->res = g_string_sized_new(256);
> +
> + *state = jstate;
> + *state_destroy = (GDestroyNotify) tf_json_free_state;
> +
> + g_strfreev(ar);
> + return TRUE;
> +}
> +
> +static gboolean
> +tf_json_foreach(NVHandle handle, const gchar *name, const gchar *value,
> + gssize length, gpointer user_data)
> +{
> + gpointer *args = user_data;
> + json_object *root = args[0];
> + TFJsonState *jstate = args[1];
> + json_object *this;
> + gboolean match = FALSE;
> + gint i = 0;
> +
> + if (jstate->skip_builtins && handle < LM_V_MAX)
> + return FALSE;
> +
> + while (jstate->select_patterns[i])
> + {
> + if (g_pattern_match_string(jstate->select_patterns[i], name))
> + {
> + match = TRUE;
> + break;
> + }
> + i++;
> + }
> +
> + i = 0;
> + while (jstate->exclude_patterns &&
> + jstate->exclude_patterns[i])
> + {
> + if (g_pattern_match_string(jstate->exclude_patterns[i], name))
> + {
> + match = FALSE;
> + break;
> + }
> + i++;
> + }
> +
> + if (match)
> + {
> + this = json_object_new_string_len(value, length);
> + json_object_object_add(root, name, this);
> + }
> +
> + return FALSE;
> +}
> +
> +static void
> +tf_json_eval (LogTemplateFunction *self, gpointer state, GPtrArray *arg_bufs,
> + LogMessage **messages, gint num_messages, LogTemplateOptions *opts,
> + gint tz, gint seq_num)
> +{
> + return;
> +}
> +
> +static json_object *
> +tf_json_format_message(TFJsonState *jstate, LogMessage *msg)
> +{
> + json_object *root;
> + gpointer args[] = { NULL, jstate };
> +
> + root = json_object_new_object();
> + args[0] = root;
> + nv_table_foreach(msg->payload, logmsg_registry, tf_json_foreach,
> + args);
> +
> + return root;
> +}
> +
> +static void
> +tf_json_format_template(gpointer key, gpointer value, gpointer user_data)
> +{
> + json_object *this;
> + gpointer *args = user_data;
> + LogMessage *msg = (LogMessage *)args[1];
> + TFJsonState *jstate = (TFJsonState *)args[2];
> +
> + g_string_truncate(jstate->res, 0);
> + log_template_format((LogTemplate *)value, msg, NULL, LTZ_LOCAL, 0, jstate->res);
> +
> + this = json_object_new_string(jstate->res->str);
> + json_object_object_add((json_object *)args[0], (gchar *)key, this);
> +}
> +
> +static void
> +tf_json_call(LogTemplateFunction *self, gpointer state, GPtrArray *arg_bufs,
> + LogMessage **messages, gint num_messages, LogTemplateOptions *opts,
> + gint tz, gint seq_num, GString *result)
> +{
> + gint i;
> + TFJsonState *jstate = (TFJsonState *)state;
> +
> + for (i = 0; i < num_messages; i++)
> + {
> + LogMessage *msg = messages[i];
> + json_object *json;
> + gpointer args[] = { NULL, msg, jstate };
> +
> + json = tf_json_format_message(jstate, msg);
> + args[0] = json;
> + g_hash_table_foreach(jstate->template_values, tf_json_format_template,
> + args);
> + g_string_append(result, json_object_to_json_string (json));
> + json_object_put(json);
> + }
> +}
> +
> +TEMPLATE_FUNCTION(tf_json, tf_json_prepare, tf_json_eval, tf_json_call, NULL);
> +
> +static Plugin builtin_tmpl_func_plugins[] =
> + {
> + TEMPLATE_FUNCTION_PLUGIN(tf_json, "format_json"),
> + };
> +
> +gboolean
> +tfjson_module_init(GlobalConfig *cfg, CfgArgs *args)
> +{
> + plugin_register(cfg, builtin_tmpl_func_plugins, G_N_ELEMENTS(builtin_tmpl_func_plugins));
> + return TRUE;
> +}
--
Bazsi
More information about the syslog-ng
mailing list