[syslog-ng] [PATCH] tfjson: $(format_json) template function implementation.

Implements a $(format_json) template function, largely based on the work of Balint Kovacs <blint@balabit.hu>, built on top of the json-c library. The syntax of the template function is as follows: $(format_json [--select <GLOB>] [--exclude <GLOB>] [--skip-builtins] [MACRO_NAMES...] KEY=VALUE) There can be multiple selects, excludes, macro names and key=value pairs. The VALUE part of the key=value pair can contain template macros. Macro names must be listed without the "$" sign, however. If --skip-builtins is specified, all built-in variables will be excluded. As an example, this is a valid argument list (without the line breaks): $(format_json --select .classification.* --select useracct.* --exclude *.*id HOST my_own_key='$SOURCEIP ($HOST)') This will include every key that begins with ".classification." or "useracct.", but will exclude any, that has a dot, and ends with "id". The latter keys will be excluded even if they match any of the select patterns. Furthermore, the $HOST macro will be included (with the key HOST), along with a my_own_key key, whose value will be evaluated as a template, thus, will contain the source IP and the HOST (again, for demo purposes). Signed-off-by: Gergely Nagy <algernon@balabit.hu> (cherry picked from commit 9ab6b381433d657b59eb1d19f994f9328fca9547) Conflicts: configure.in modules/Makefile.am removed mongodb references from configure.in and modules/Makefile.am; resolved conflicts there Signed-off-by: Matthew Hall <mhall@mhcomputing.net> --- configure.in | 27 ++++- modules/Makefile.am | 3 +- modules/tfjson/Makefile.am | 11 ++ modules/tfjson/tfjson.c | 305 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 344 insertions(+), 2 deletions(-) create mode 100644 modules/tfjson/Makefile.am create mode 100644 modules/tfjson/tfjson.c diff --git /tfjson-base/configure.in /syslog-ng-3.2/configure.in new file mode 100644 index 0c6cef2..08d6a6b 100644 --- /tfjson-base/configure.in +++ /syslog-ng-3.2/configure.in @@ -25,6 +25,8 @@ EVTLOG_MIN_VERSION="0.2" OPENSSL_MIN_VERSION="0.9.8" LIBDBI_MIN_VERSION="0.8.0" PCRE_MIN_VERSION="7.3" +IVYKIS_MIN_VERSION="0.18" +JSON_C_MIN_VERSION="0.9" dnl *************************************************************************** dnl Initial setup @@ -144,6 +146,10 @@ AC_ARG_ENABLE(gcov, ,,enable_gcov="no") +AC_ARG_ENABLE(json, + [ --enable-json Enable support for JSON template formatting (default: auto)] + ,,enable_json="auto") + dnl *************************************************************************** dnl Checks for programs. AC_PROG_CC @@ -493,6 +499,23 @@ if test "x$linking_mode" != "xdynamic" -a "x$blb_cv_static_glib" = "xno"; then fi dnl *************************************************************************** +dnl json-glib headers/libraries +dnl *************************************************************************** +if test "x$enable_json" = "xyes" -o "x$enable_json" = "xauto"; then + PKG_CHECK_MODULES(JSON, json >= $JSON_C_MIN_VERSION,,JSON_LIBS="") + if test -z "$JSON_LIBS"; then + if test "x$enable_json" = "xyes"; then + AC_MSG_ERROR(Cannot find json >= $JSON_C_MIN_VERSION.) + else + AC_MSG_WARN(Cannot find json >= $JSON_C_MIN_VERSION.) + fi + enable_json="no" + else + enable_json="yes" + fi +fi + +dnl *************************************************************************** dnl pcre headers/libraries dnl *************************************************************************** @@ -750,6 +773,7 @@ AM_CONDITIONAL(ENABLE_SSL, [test "$enable_ssl" = "yes"]) AM_CONDITIONAL(ENABLE_SQL, [test "$enable_sql" = "yes"]) AM_CONDITIONAL(ENABLE_SUN_STREAMS, [test "$enable_sun_streams" = "yes"]) AM_CONDITIONAL(ENABLE_PACCT, [test "$enable_pacct" = "yes"]) +AM_CONDITIONAL(ENABLE_JSON, [test "$enable_json" = "yes"]) # substitution into manual pages expanded_sysconfdir=[`patheval $sysconfdir | sed -e 's/-/\\\\-/g'`] @@ -803,6 +827,7 @@ AC_OUTPUT(dist.conf modules/pacctformat/Makefile modules/basicfuncs/Makefile modules/convertfuncs/Makefile + modules/tfjson/Makefile scripts/Makefile scripts/update-patterndb doc/Makefile @@ -843,5 +868,5 @@ echo " Linux capability support : ${enable_linux_caps:=no}" echo " PCRE support : ${enable_pcre:=no}" echo " Env wrapper support : ${enable_env_wrapper:=no}" echo " PACCT module (EXPERIMENTAL) : ${enable_pacct:=no}" - +echo " JSON support (module) : ${enable_json:=no}" diff --git /tfjson-base/modules/Makefile.am /syslog-ng-3.2/modules/Makefile.am new file mode 100644 index cfc721e..16e38df 100644 --- /tfjson-base/modules/Makefile.am +++ /syslog-ng-3.2/modules/Makefile.am @@ -1 +1,2 @@ -SUBDIRS = afsocket afsql afstreams affile afprog afuser csvparser confgen syslogformat pacctformat basicfuncs convertfuncs dbparser dummy +SUBDIRS = afsocket afsql afstreams affile afprog afuser csvparser confgen syslogformat pacctformat basicfuncs convertfuncs dbparser dummy \ + tfjson diff --git /tfjson-base/modules/tfjson/Makefile.am /syslog-ng-3.2/modules/tfjson/Makefile.am new file mode 100644 index 0000000..8b30f83 --- /dev/null +++ /syslog-ng-3.2/modules/tfjson/Makefile.am @@ -0,0 +1,11 @@ +moduledir = @moduledir@ +export top_srcdir + +if ENABLE_JSON +AM_CPPFLAGS = -I$(top_srcdir)/lib -I../../lib @JSON_CFLAGS@ +module_LTLIBRARIES = libtfjson.la + +libtfjson_la_SOURCES = tfjson.c +libtfjson_la_LIBADD = ../../lib/libsyslog-ng.la @JSON_LIBS@ +libtfjson_la_LDFLAGS = -avoid-version +endif diff --git /tfjson-base/modules/tfjson/tfjson.c /syslog-ng-3.2/modules/tfjson/tfjson.c new file mode 100644 index 0000000..e57dce3 --- /dev/null +++ /syslog-ng-3.2/modules/tfjson/tfjson.c @@ -0,0 +1,305 @@ +/* + * Copyright (c) 2002-2011 BalaBit IT Ltd, Budapest, Hungary + * Copyright (c) 2011 Balint Kovacs <blint@balabit.hu> + * Copyright (c) 2011 Gergely Nagy <algernon@balabit.hu> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + */ + +#include "plugin.h" +#include "templates.h" +#include "filter.h" +#include "filter-expr-parser.h" +#include "cfg.h" + +#include <json.h> + +typedef struct +{ + GPatternSpec **select_patterns; + GPatternSpec **exclude_patterns; + + GHashTable *template_values; + + gboolean skip_builtins; + + /* Temporary variables. */ + GString *res; +} TFJsonState; + +static void +tf_json_free_state (TFJsonState *state) +{ + gint i = 0; + + if (state->select_patterns) + { + while (state->select_patterns[i]) + { + g_pattern_spec_free(state->select_patterns[i]); + i++; + } + g_free (state->select_patterns); + } + if (state->exclude_patterns) + { + i = 0; + while (state->exclude_patterns[i]) + { + g_pattern_spec_free(state->exclude_patterns[i]); + i++; + } + g_free (state->exclude_patterns); + } + + g_hash_table_destroy(state->template_values); + if (state->res) + g_string_free(state->res, TRUE); + g_free(state); +} + +static gboolean +tf_json_prepare(LogTemplateFunction *self, LogTemplate *parent, + gint argc, gchar *argv[], + gpointer *state, GDestroyNotify *state_destroy, + GError **error) +{ + TFJsonState *jstate = g_new0(TFJsonState, 1); + GOptionContext *ctx; + GError *parse_error = NULL; + gchar **selects = NULL; + gchar **excludes = NULL; + + GOptionEntry format_options[] = + { + { "select", 's', 0, G_OPTION_ARG_STRING_ARRAY, &selects, + "Specify which name-value pairs are included in the result. The parameter is a shell-like glob pattern.", + "<glob expression>" }, + { "exclude", 'x', 0, G_OPTION_ARG_STRING_ARRAY, &excludes, + "Specify which name-value pairs to exclude from the result. The parameter is a shell-like glob pattern.", + "<glob expression>" }, + { "skip-builtins", 'D', 0, G_OPTION_ARG_NONE, &jstate->skip_builtins, + "Skip built-in variables, like $HOST and the rest.", NULL }, + { NULL } + }; + + gchar **ar; + gint arc = argc + 1; + gint i; + + ar = g_new(gchar *, arc + 1); + for (i = 0; i < argc; i++) + ar[i + 1] = g_strdup(argv[i]); + ar[0] = g_strdup("format-json"); + ar[argc + 1] = NULL; + + jstate->template_values = + g_hash_table_new_full(g_str_hash, g_str_equal, g_free, + (GDestroyNotify) log_template_unref); + ctx = g_option_context_new("format-json"); + msg_add_option_group(ctx); + g_option_context_add_main_entries(ctx, format_options, NULL); + if (!g_option_context_parse(ctx, &arc, &ar, &parse_error)) + { + /* + msg_error ("Error parsing function arguments", + evt_tag_str("error", parse_error ? + parse_error->message : "Invalid arguments")); + */ + g_set_error(error, LOG_TEMPLATE_ERROR, LOG_TEMPLATE_ERROR_COMPILE, + "Error parsing function arguments"); + g_option_context_free(ctx); + tf_json_free_state(jstate); + g_strfreev(ar); + return FALSE; + } + g_option_context_free(ctx); + + if (selects == NULL) + { + jstate->select_patterns = g_new0(GPatternSpec *, 2); + jstate->select_patterns[0] = g_pattern_spec_new("*"); + } + else + { + gint n = g_strv_length (selects); + jstate->select_patterns = g_new0(GPatternSpec *, n + 1); + + for (i = 0; i < n; i++) + jstate->select_patterns[i] = g_pattern_spec_new(selects[i]); + } + + if (excludes != NULL) + { + gint n = g_strv_length (excludes); + jstate->exclude_patterns = g_new0(GPatternSpec *, n + 1); + + for (i = 0; i < n; i++) + jstate->exclude_patterns[i] = g_pattern_spec_new(excludes[i]); + } + + for (i = 1; i < arc; i++) + { + gchar *key = NULL; + LogTemplate *t = NULL; + + if (g_strstr_len(ar[i], strlen (ar[i]), "=")) + { + gchar **kv = g_strsplit(ar[i], "=", 2); + + key = g_strdup(kv[0]); + t = log_template_new(parent->cfg, NULL, kv[1]); + g_free(kv[0]); + g_free(kv[1]); + g_free(kv); + } + else + { + gchar *m = g_strconcat("$", ar[i], NULL); + + key = g_strdup(ar[i]); + t = log_template_new(parent->cfg, NULL, m); + g_free(m); + } + g_hash_table_insert(jstate->template_values, key, t); + } + + jstate->res = g_string_sized_new(256); + + *state = jstate; + *state_destroy = (GDestroyNotify) tf_json_free_state; + + g_strfreev(ar); + return TRUE; +} + +static gboolean +tf_json_foreach(NVHandle handle, const gchar *name, const gchar *value, + gssize length, gpointer user_data) +{ + gpointer *args = user_data; + json_object *root = args[0]; + TFJsonState *jstate = args[1]; + json_object *this; + gboolean match = FALSE; + gint i = 0; + + if (jstate->skip_builtins && handle < LM_V_MAX) + return FALSE; + + while (jstate->select_patterns[i]) + { + if (g_pattern_match_string(jstate->select_patterns[i], name)) + { + match = TRUE; + break; + } + i++; + } + + i = 0; + while (jstate->exclude_patterns && + jstate->exclude_patterns[i]) + { + if (g_pattern_match_string(jstate->exclude_patterns[i], name)) + { + match = FALSE; + break; + } + i++; + } + + if (match) + { + this = json_object_new_string_len(value, length); + json_object_object_add(root, name, this); + } + + return FALSE; +} + +static void +tf_json_eval (LogTemplateFunction *self, gpointer state, GPtrArray *arg_bufs, + LogMessage **messages, gint num_messages, LogTemplateOptions *opts, + gint tz, gint seq_num) +{ + return; +} + +static json_object * +tf_json_format_message(TFJsonState *jstate, LogMessage *msg) +{ + json_object *root; + gpointer args[] = { NULL, jstate }; + + root = json_object_new_object(); + args[0] = root; + nv_table_foreach(msg->payload, logmsg_registry, tf_json_foreach, + args); + + return root; +} + +static void +tf_json_format_template(gpointer key, gpointer value, gpointer user_data) +{ + json_object *this; + gpointer *args = user_data; + LogMessage *msg = (LogMessage *)args[1]; + TFJsonState *jstate = (TFJsonState *)args[2]; + + g_string_truncate(jstate->res, 0); + log_template_format((LogTemplate *)value, msg, NULL, LTZ_LOCAL, 0, jstate->res); + + this = json_object_new_string(jstate->res->str); + json_object_object_add((json_object *)args[0], (gchar *)key, this); +} + +static void +tf_json_call(LogTemplateFunction *self, gpointer state, GPtrArray *arg_bufs, + LogMessage **messages, gint num_messages, LogTemplateOptions *opts, + gint tz, gint seq_num, GString *result) +{ + gint i; + TFJsonState *jstate = (TFJsonState *)state; + + for (i = 0; i < num_messages; i++) + { + LogMessage *msg = messages[i]; + json_object *json; + gpointer args[] = { NULL, msg, jstate }; + + json = tf_json_format_message(jstate, msg); + args[0] = json; + g_hash_table_foreach(jstate->template_values, tf_json_format_template, + args); + g_string_append(result, json_object_to_json_string (json)); + json_object_put(json); + } +} + +TEMPLATE_FUNCTION(tf_json, tf_json_prepare, tf_json_eval, tf_json_call, NULL); + +static Plugin builtin_tmpl_func_plugins[] = + { + TEMPLATE_FUNCTION_PLUGIN(tf_json, "format_json"), + }; + +gboolean +tfjson_module_init(GlobalConfig *cfg, CfgArgs *args) +{ + plugin_register(cfg, builtin_tmpl_func_plugins, G_N_ELEMENTS(builtin_tmpl_func_plugins)); + return TRUE; +} -- 1.7.0.4