[syslog-ng] [PATCH] tfjson: $(format_json) template function implementation.

Matthew Hall mhall at mhcomputing.net
Tue Feb 1 03:29:18 CET 2011


Implements a $(format_json) template function, largely based on the
work of Balint Kovacs <blint at balabit.hu>, built on top of the json-c
library.

The syntax of the template function is as follows:

  $(format_json [--select <GLOB>] [--exclude <GLOB>]
                [--skip-builtins]
                [MACRO_NAMES...] KEY=VALUE)

There can be multiple selects, excludes, macro names and key=value
pairs. The VALUE part of the key=value pair can contain template
macros.

Macro names must be listed without the "$" sign, however.

If --skip-builtins is specified, all built-in variables will be
excluded.

As an example, this is a valid argument list (without the line
breaks):

  $(format_json --select .classification.* --select useracct.*
                --exclude *.*id HOST my_own_key='$SOURCEIP ($HOST)')

This will include every key that begins with ".classification." or
"useracct.", but will exclude any, that has a dot, and ends with
"id". The latter keys will be excluded even if they match any of the
select patterns.

Furthermore, the $HOST macro will be included (with the key HOST),
along with a my_own_key key, whose value will be evaluated as a
template, thus, will contain the source IP and the HOST (again, for
demo purposes).

Signed-off-by: Gergely Nagy <algernon at balabit.hu>
(cherry picked from commit 9ab6b381433d657b59eb1d19f994f9328fca9547)

Conflicts:

	configure.in
	modules/Makefile.am

removed mongodb references from configure.in and modules/Makefile.am; resolved conflicts there
Signed-off-by: Matthew Hall <mhall at mhcomputing.net>
---
 configure.in               |   27 ++++-
 modules/Makefile.am        |    3 +-
 modules/tfjson/Makefile.am |   11 ++
 modules/tfjson/tfjson.c    |  305 ++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 344 insertions(+), 2 deletions(-)
 create mode 100644 modules/tfjson/Makefile.am
 create mode 100644 modules/tfjson/tfjson.c

diff --git /tfjson-base/configure.in /syslog-ng-3.2/configure.in
new file mode 100644
index 0c6cef2..08d6a6b 100644
--- /tfjson-base/configure.in
+++ /syslog-ng-3.2/configure.in
@@ -25,6 +25,8 @@ EVTLOG_MIN_VERSION="0.2"
 OPENSSL_MIN_VERSION="0.9.8"
 LIBDBI_MIN_VERSION="0.8.0"
 PCRE_MIN_VERSION="7.3"
+IVYKIS_MIN_VERSION="0.18"
+JSON_C_MIN_VERSION="0.9"
 
 dnl ***************************************************************************
 dnl Initial setup
@@ -144,6 +146,10 @@ AC_ARG_ENABLE(gcov,
               ,,enable_gcov="no")
 
 
+AC_ARG_ENABLE(json,
+              [  --enable-json           Enable support for JSON template formatting (default: auto)]
+              ,,enable_json="auto")
+
 dnl ***************************************************************************
 dnl Checks for programs.
 AC_PROG_CC
@@ -493,6 +499,23 @@ if test "x$linking_mode" != "xdynamic" -a "x$blb_cv_static_glib" = "xno"; then
 fi
 
 dnl ***************************************************************************
+dnl json-glib headers/libraries
+dnl ***************************************************************************
+if test "x$enable_json" = "xyes" -o "x$enable_json" = "xauto"; then
+   PKG_CHECK_MODULES(JSON, json >= $JSON_C_MIN_VERSION,,JSON_LIBS="")
+   if test -z "$JSON_LIBS"; then
+      if test "x$enable_json" = "xyes"; then
+      	 AC_MSG_ERROR(Cannot find json >= $JSON_C_MIN_VERSION.)
+      else
+	 AC_MSG_WARN(Cannot find json >= $JSON_C_MIN_VERSION.)
+      fi
+      enable_json="no"
+   else
+      enable_json="yes"
+   fi
+fi
+
+dnl ***************************************************************************
 dnl pcre headers/libraries
 dnl ***************************************************************************
 
@@ -750,6 +773,7 @@ AM_CONDITIONAL(ENABLE_SSL, [test "$enable_ssl" = "yes"])
 AM_CONDITIONAL(ENABLE_SQL, [test "$enable_sql" = "yes"])
 AM_CONDITIONAL(ENABLE_SUN_STREAMS, [test "$enable_sun_streams" = "yes"])
 AM_CONDITIONAL(ENABLE_PACCT, [test "$enable_pacct" = "yes"])
+AM_CONDITIONAL(ENABLE_JSON, [test "$enable_json" = "yes"])
 
 # substitution into manual pages
 expanded_sysconfdir=[`patheval $sysconfdir | sed -e 's/-/\\\\-/g'`]
@@ -803,6 +827,7 @@ AC_OUTPUT(dist.conf
           modules/pacctformat/Makefile
           modules/basicfuncs/Makefile
           modules/convertfuncs/Makefile
+	  modules/tfjson/Makefile
 	  scripts/Makefile
 	  scripts/update-patterndb
 	  doc/Makefile
@@ -843,5 +868,5 @@ echo "  Linux capability support    : ${enable_linux_caps:=no}"
 echo "  PCRE support                : ${enable_pcre:=no}"
 echo "  Env wrapper support         : ${enable_env_wrapper:=no}"
 echo "  PACCT module (EXPERIMENTAL) : ${enable_pacct:=no}"
-
+echo "  JSON support (module)       : ${enable_json:=no}"
 
diff --git /tfjson-base/modules/Makefile.am /syslog-ng-3.2/modules/Makefile.am
new file mode 100644
index cfc721e..16e38df 100644
--- /tfjson-base/modules/Makefile.am
+++ /syslog-ng-3.2/modules/Makefile.am
@@ -1 +1,2 @@
-SUBDIRS = afsocket afsql afstreams affile afprog afuser csvparser confgen syslogformat pacctformat basicfuncs convertfuncs dbparser dummy
+SUBDIRS = afsocket afsql afstreams affile afprog afuser csvparser confgen syslogformat pacctformat basicfuncs convertfuncs dbparser dummy \
+	tfjson
diff --git /tfjson-base/modules/tfjson/Makefile.am /syslog-ng-3.2/modules/tfjson/Makefile.am
new file mode 100644
index 0000000..8b30f83
--- /dev/null
+++ /syslog-ng-3.2/modules/tfjson/Makefile.am
@@ -0,0 +1,11 @@
+moduledir = @moduledir@
+export top_srcdir
+
+if ENABLE_JSON
+AM_CPPFLAGS = -I$(top_srcdir)/lib -I../../lib @JSON_CFLAGS@
+module_LTLIBRARIES = libtfjson.la
+
+libtfjson_la_SOURCES = tfjson.c
+libtfjson_la_LIBADD = ../../lib/libsyslog-ng.la @JSON_LIBS@
+libtfjson_la_LDFLAGS = -avoid-version
+endif
diff --git /tfjson-base/modules/tfjson/tfjson.c /syslog-ng-3.2/modules/tfjson/tfjson.c
new file mode 100644
index 0000000..e57dce3
--- /dev/null
+++ /syslog-ng-3.2/modules/tfjson/tfjson.c
@@ -0,0 +1,305 @@
+/*
+ * Copyright (c) 2002-2011 BalaBit IT Ltd, Budapest, Hungary
+ * Copyright (c) 2011 Balint Kovacs <blint at balabit.hu>
+ * Copyright (c) 2011 Gergely Nagy <algernon at balabit.hu>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
+ *
+ */
+
+#include "plugin.h"
+#include "templates.h"
+#include "filter.h"
+#include "filter-expr-parser.h"
+#include "cfg.h"
+
+#include <json.h>
+
+typedef struct
+{
+  GPatternSpec **select_patterns;
+  GPatternSpec **exclude_patterns;
+
+  GHashTable *template_values;
+
+  gboolean skip_builtins;
+
+  /* Temporary variables. */
+  GString *res;
+} TFJsonState;
+
+static void
+tf_json_free_state (TFJsonState *state)
+{
+  gint i = 0;
+
+  if (state->select_patterns)
+    {
+      while (state->select_patterns[i])
+	{
+	  g_pattern_spec_free(state->select_patterns[i]);
+	  i++;
+	}
+      g_free (state->select_patterns);
+    }
+  if (state->exclude_patterns)
+    {
+      i = 0;
+      while (state->exclude_patterns[i])
+	{
+	  g_pattern_spec_free(state->exclude_patterns[i]);
+	  i++;
+	}
+      g_free (state->exclude_patterns);
+    }
+
+  g_hash_table_destroy(state->template_values);
+  if (state->res)
+    g_string_free(state->res, TRUE);
+  g_free(state);
+}
+
+static gboolean
+tf_json_prepare(LogTemplateFunction *self, LogTemplate *parent,
+		gint argc, gchar *argv[],
+		gpointer *state, GDestroyNotify *state_destroy,
+		GError **error)
+{
+  TFJsonState *jstate = g_new0(TFJsonState, 1);
+  GOptionContext *ctx;
+  GError *parse_error = NULL;
+  gchar **selects = NULL;
+  gchar **excludes = NULL;
+
+  GOptionEntry format_options[] =
+    {
+      { "select", 's', 0, G_OPTION_ARG_STRING_ARRAY, &selects,
+	"Specify which name-value pairs are included in the result. The parameter is a shell-like glob pattern.",
+	"<glob expression>" },
+      { "exclude", 'x', 0, G_OPTION_ARG_STRING_ARRAY, &excludes,
+	"Specify which name-value pairs to exclude from the result. The parameter is a shell-like glob pattern.",
+	"<glob expression>" },
+      { "skip-builtins", 'D', 0, G_OPTION_ARG_NONE, &jstate->skip_builtins,
+	"Skip built-in variables, like $HOST and the rest.", NULL },
+      { NULL }
+    };
+
+  gchar **ar;
+  gint arc = argc + 1;
+  gint i;
+
+  ar = g_new(gchar *, arc + 1);
+  for (i = 0; i < argc; i++)
+    ar[i + 1] = g_strdup(argv[i]);
+  ar[0] = g_strdup("format-json");
+  ar[argc + 1] = NULL;
+
+  jstate->template_values =
+    g_hash_table_new_full(g_str_hash, g_str_equal, g_free,
+			  (GDestroyNotify) log_template_unref);
+  ctx = g_option_context_new("format-json");
+  msg_add_option_group(ctx);
+  g_option_context_add_main_entries(ctx, format_options, NULL);
+  if (!g_option_context_parse(ctx, &arc, &ar, &parse_error))
+    {
+      /*
+      msg_error ("Error parsing function arguments",
+		 evt_tag_str("error", parse_error ?
+		 parse_error->message : "Invalid arguments"));
+      */
+      g_set_error(error, LOG_TEMPLATE_ERROR, LOG_TEMPLATE_ERROR_COMPILE,
+		  "Error parsing function arguments");
+      g_option_context_free(ctx);
+      tf_json_free_state(jstate);
+      g_strfreev(ar);
+      return FALSE;
+    }
+  g_option_context_free(ctx);
+
+  if (selects == NULL)
+    {
+      jstate->select_patterns = g_new0(GPatternSpec *, 2);
+      jstate->select_patterns[0] = g_pattern_spec_new("*");
+    }
+  else
+    {
+      gint n = g_strv_length (selects);
+      jstate->select_patterns = g_new0(GPatternSpec *, n + 1);
+
+      for (i = 0; i < n; i++)
+	jstate->select_patterns[i] = g_pattern_spec_new(selects[i]);
+    }
+
+  if (excludes != NULL)
+    {
+      gint n = g_strv_length (excludes);
+      jstate->exclude_patterns = g_new0(GPatternSpec *, n + 1);
+
+      for (i = 0; i < n; i++)
+	jstate->exclude_patterns[i] = g_pattern_spec_new(excludes[i]);
+    }
+
+  for (i = 1; i < arc; i++)
+    {
+      gchar *key = NULL;
+      LogTemplate *t = NULL;
+
+      if (g_strstr_len(ar[i], strlen (ar[i]), "="))
+	{
+	  gchar **kv = g_strsplit(ar[i], "=", 2);
+
+	  key = g_strdup(kv[0]);
+	  t = log_template_new(parent->cfg, NULL, kv[1]);
+	  g_free(kv[0]);
+	  g_free(kv[1]);
+	  g_free(kv);
+	}
+      else
+	{
+	  gchar *m = g_strconcat("$", ar[i], NULL);
+
+	  key = g_strdup(ar[i]);
+	  t = log_template_new(parent->cfg, NULL, m);
+	  g_free(m);
+	}
+      g_hash_table_insert(jstate->template_values, key, t);
+    }
+
+  jstate->res = g_string_sized_new(256);
+
+  *state = jstate;
+  *state_destroy = (GDestroyNotify) tf_json_free_state;
+
+  g_strfreev(ar);
+  return TRUE;
+}
+
+static gboolean
+tf_json_foreach(NVHandle handle, const gchar *name, const gchar *value,
+		gssize length, gpointer user_data)
+{
+  gpointer *args = user_data;
+  json_object *root = args[0];
+  TFJsonState *jstate = args[1];
+  json_object *this;
+  gboolean match = FALSE;
+  gint i = 0;
+
+  if (jstate->skip_builtins && handle < LM_V_MAX)
+    return FALSE;
+
+  while (jstate->select_patterns[i])
+    {
+      if (g_pattern_match_string(jstate->select_patterns[i], name))
+	{
+	  match = TRUE;
+	  break;
+	}
+      i++;
+    }
+
+  i = 0;
+  while (jstate->exclude_patterns &&
+	 jstate->exclude_patterns[i])
+    {
+      if (g_pattern_match_string(jstate->exclude_patterns[i], name))
+	{
+	  match = FALSE;
+	  break;
+	}
+      i++;
+    }
+
+  if (match)
+    {
+      this = json_object_new_string_len(value, length);
+      json_object_object_add(root, name, this);
+    }
+
+  return FALSE;
+}
+
+static void
+tf_json_eval (LogTemplateFunction *self, gpointer state, GPtrArray *arg_bufs,
+	      LogMessage **messages, gint num_messages, LogTemplateOptions *opts,
+	      gint tz, gint seq_num)
+{
+  return;
+}
+
+static json_object *
+tf_json_format_message(TFJsonState *jstate, LogMessage *msg)
+{
+  json_object *root;
+  gpointer args[] = { NULL, jstate };
+
+  root = json_object_new_object();
+  args[0] = root;
+  nv_table_foreach(msg->payload, logmsg_registry, tf_json_foreach,
+		   args);
+
+  return root;
+}
+
+static void
+tf_json_format_template(gpointer key, gpointer value, gpointer user_data)
+{
+  json_object *this;
+  gpointer *args = user_data;
+  LogMessage *msg = (LogMessage *)args[1];
+  TFJsonState *jstate = (TFJsonState *)args[2];
+
+  g_string_truncate(jstate->res, 0);
+  log_template_format((LogTemplate *)value, msg, NULL, LTZ_LOCAL, 0, jstate->res);
+
+  this = json_object_new_string(jstate->res->str);
+  json_object_object_add((json_object *)args[0], (gchar *)key, this);
+}
+
+static void
+tf_json_call(LogTemplateFunction *self, gpointer state, GPtrArray *arg_bufs,
+	     LogMessage **messages, gint num_messages, LogTemplateOptions *opts,
+	     gint tz, gint seq_num, GString *result)
+{
+  gint i;
+  TFJsonState *jstate = (TFJsonState *)state;
+
+  for (i = 0; i < num_messages; i++)
+    {
+      LogMessage *msg = messages[i];
+      json_object *json;
+      gpointer args[] = { NULL, msg, jstate };
+
+      json = tf_json_format_message(jstate, msg);
+      args[0] = json;
+      g_hash_table_foreach(jstate->template_values, tf_json_format_template,
+			   args);
+      g_string_append(result, json_object_to_json_string (json));
+      json_object_put(json);
+    }
+}
+
+TEMPLATE_FUNCTION(tf_json, tf_json_prepare, tf_json_eval, tf_json_call, NULL);
+
+static Plugin builtin_tmpl_func_plugins[] =
+  {
+    TEMPLATE_FUNCTION_PLUGIN(tf_json, "format_json"),
+  };
+
+gboolean
+tfjson_module_init(GlobalConfig *cfg, CfgArgs *args)
+{
+  plugin_register(cfg, builtin_tmpl_func_plugins, G_N_ELEMENTS(builtin_tmpl_func_plugins));
+  return TRUE;
+}
-- 
1.7.0.4



More information about the syslog-ng mailing list