Implements a $(format_json) template function, largely based on the
work of Balint Kovacs <blint(a)balabit.hu>, built on top of the json-c
library.
The syntax of the template function is as follows:
$(format_json [--select <GLOB>] [--exclude <GLOB>]
[--skip-builtins]
[MACRO_NAMES...] KEY=VALUE)
There can be multiple selects, excludes, macro names and key=value
pairs. The VALUE part of the key=value pair can contain template
macros.
Macro names must be listed without the "$" sign, however.
If --skip-builtins is specified, all built-in variables will be
excluded.
As an example, this is a valid argument list (without the line
breaks):
$(format_json --select .classification.* --select useracct.*
--exclude *.*id HOST my_own_key='$SOURCEIP ($HOST)')
This will include every key that begins with ".classification." or
"useracct.", but will exclude any, that has a dot, and ends with
"id". The latter keys will be excluded even if they match any of the
select patterns.
Furthermore, the $HOST macro will be included (with the key HOST),
along with a my_own_key key, whose value will be evaluated as a
template, thus, will contain the source IP and the HOST (again, for
demo purposes).
Signed-off-by: Gergely Nagy <algernon(a)balabit.hu>
(cherry picked from commit 9ab6b381433d657b59eb1d19f994f9328fca9547)
Conflicts:
configure.in
modules/Makefile.am
removed mongodb references from configure.in and modules/Makefile.am; resolved conflicts there
Signed-off-by: Matthew Hall <mhall(a)mhcomputing.net>
---
configure.in | 27 ++++-
modules/Makefile.am | 3 +-
modules/tfjson/Makefile.am | 11 ++
modules/tfjson/tfjson.c | 305 ++++++++++++++++++++++++++++++++++++++++++++
4 files changed, 344 insertions(+), 2 deletions(-)
create mode 100644 modules/tfjson/Makefile.am
create mode 100644 modules/tfjson/tfjson.c
diff --git /tfjson-base/configure.in /syslog-ng-3.2/configure.in
new file mode 100644
index 0c6cef2..08d6a6b 100644
--- /tfjson-base/configure.in
+++ /syslog-ng-3.2/configure.in
@@ -25,6 +25,8 @@ EVTLOG_MIN_VERSION="0.2"
OPENSSL_MIN_VERSION="0.9.8"
LIBDBI_MIN_VERSION="0.8.0"
PCRE_MIN_VERSION="7.3"
+IVYKIS_MIN_VERSION="0.18"
+JSON_C_MIN_VERSION="0.9"
dnl ***************************************************************************
dnl Initial setup
@@ -144,6 +146,10 @@ AC_ARG_ENABLE(gcov,
,,enable_gcov="no")
+AC_ARG_ENABLE(json,
+ [ --enable-json Enable support for JSON template formatting (default: auto)]
+ ,,enable_json="auto")
+
dnl ***************************************************************************
dnl Checks for programs.
AC_PROG_CC
@@ -493,6 +499,23 @@ if test "x$linking_mode" != "xdynamic" -a "x$blb_cv_static_glib" = "xno"; then
fi
dnl ***************************************************************************
+dnl json-glib headers/libraries
+dnl ***************************************************************************
+if test "x$enable_json" = "xyes" -o "x$enable_json" = "xauto"; then
+ PKG_CHECK_MODULES(JSON, json >= $JSON_C_MIN_VERSION,,JSON_LIBS="")
+ if test -z "$JSON_LIBS"; then
+ if test "x$enable_json" = "xyes"; then
+ AC_MSG_ERROR(Cannot find json >= $JSON_C_MIN_VERSION.)
+ else
+ AC_MSG_WARN(Cannot find json >= $JSON_C_MIN_VERSION.)
+ fi
+ enable_json="no"
+ else
+ enable_json="yes"
+ fi
+fi
+
+dnl ***************************************************************************
dnl pcre headers/libraries
dnl ***************************************************************************
@@ -750,6 +773,7 @@ AM_CONDITIONAL(ENABLE_SSL, [test "$enable_ssl" = "yes"])
AM_CONDITIONAL(ENABLE_SQL, [test "$enable_sql" = "yes"])
AM_CONDITIONAL(ENABLE_SUN_STREAMS, [test "$enable_sun_streams" = "yes"])
AM_CONDITIONAL(ENABLE_PACCT, [test "$enable_pacct" = "yes"])
+AM_CONDITIONAL(ENABLE_JSON, [test "$enable_json" = "yes"])
# substitution into manual pages
expanded_sysconfdir=[`patheval $sysconfdir | sed -e 's/-/\\\\-/g'`]
@@ -803,6 +827,7 @@ AC_OUTPUT(dist.conf
modules/pacctformat/Makefile
modules/basicfuncs/Makefile
modules/convertfuncs/Makefile
+ modules/tfjson/Makefile
scripts/Makefile
scripts/update-patterndb
doc/Makefile
@@ -843,5 +868,5 @@ echo " Linux capability support : ${enable_linux_caps:=no}"
echo " PCRE support : ${enable_pcre:=no}"
echo " Env wrapper support : ${enable_env_wrapper:=no}"
echo " PACCT module (EXPERIMENTAL) : ${enable_pacct:=no}"
-
+echo " JSON support (module) : ${enable_json:=no}"
diff --git /tfjson-base/modules/Makefile.am /syslog-ng-3.2/modules/Makefile.am
new file mode 100644
index cfc721e..16e38df 100644
--- /tfjson-base/modules/Makefile.am
+++ /syslog-ng-3.2/modules/Makefile.am
@@ -1 +1,2 @@
-SUBDIRS = afsocket afsql afstreams affile afprog afuser csvparser confgen syslogformat pacctformat basicfuncs convertfuncs dbparser dummy
+SUBDIRS = afsocket afsql afstreams affile afprog afuser csvparser confgen syslogformat pacctformat basicfuncs convertfuncs dbparser dummy \
+ tfjson
diff --git /tfjson-base/modules/tfjson/Makefile.am /syslog-ng-3.2/modules/tfjson/Makefile.am
new file mode 100644
index 0000000..8b30f83
--- /dev/null
+++ /syslog-ng-3.2/modules/tfjson/Makefile.am
@@ -0,0 +1,11 @@
+moduledir = @moduledir@
+export top_srcdir
+
+if ENABLE_JSON
+AM_CPPFLAGS = -I$(top_srcdir)/lib -I../../lib @JSON_CFLAGS@
+module_LTLIBRARIES = libtfjson.la
+
+libtfjson_la_SOURCES = tfjson.c
+libtfjson_la_LIBADD = ../../lib/libsyslog-ng.la @JSON_LIBS@
+libtfjson_la_LDFLAGS = -avoid-version
+endif
diff --git /tfjson-base/modules/tfjson/tfjson.c /syslog-ng-3.2/modules/tfjson/tfjson.c
new file mode 100644
index 0000000..e57dce3
--- /dev/null
+++ /syslog-ng-3.2/modules/tfjson/tfjson.c
@@ -0,0 +1,305 @@
+/*
+ * Copyright (c) 2002-2011 BalaBit IT Ltd, Budapest, Hungary
+ * Copyright (c) 2011 Balint Kovacs <blint(a)balabit.hu>
+ * Copyright (c) 2011 Gergely Nagy <algernon(a)balabit.hu>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ */
+
+#include "plugin.h"
+#include "templates.h"
+#include "filter.h"
+#include "filter-expr-parser.h"
+#include "cfg.h"
+
+#include <json.h>
+
+typedef struct
+{
+ GPatternSpec **select_patterns;
+ GPatternSpec **exclude_patterns;
+
+ GHashTable *template_values;
+
+ gboolean skip_builtins;
+
+ /* Temporary variables. */
+ GString *res;
+} TFJsonState;
+
+static void
+tf_json_free_state (TFJsonState *state)
+{
+ gint i = 0;
+
+ if (state->select_patterns)
+ {
+ while (state->select_patterns[i])
+ {
+ g_pattern_spec_free(state->select_patterns[i]);
+ i++;
+ }
+ g_free (state->select_patterns);
+ }
+ if (state->exclude_patterns)
+ {
+ i = 0;
+ while (state->exclude_patterns[i])
+ {
+ g_pattern_spec_free(state->exclude_patterns[i]);
+ i++;
+ }
+ g_free (state->exclude_patterns);
+ }
+
+ g_hash_table_destroy(state->template_values);
+ if (state->res)
+ g_string_free(state->res, TRUE);
+ g_free(state);
+}
+
+static gboolean
+tf_json_prepare(LogTemplateFunction *self, LogTemplate *parent,
+ gint argc, gchar *argv[],
+ gpointer *state, GDestroyNotify *state_destroy,
+ GError **error)
+{
+ TFJsonState *jstate = g_new0(TFJsonState, 1);
+ GOptionContext *ctx;
+ GError *parse_error = NULL;
+ gchar **selects = NULL;
+ gchar **excludes = NULL;
+
+ GOptionEntry format_options[] =
+ {
+ { "select", 's', 0, G_OPTION_ARG_STRING_ARRAY, &selects,
+ "Specify which name-value pairs are included in the result. The parameter is a shell-like glob pattern.",
+ "<glob expression>" },
+ { "exclude", 'x', 0, G_OPTION_ARG_STRING_ARRAY, &excludes,
+ "Specify which name-value pairs to exclude from the result. The parameter is a shell-like glob pattern.",
+ "<glob expression>" },
+ { "skip-builtins", 'D', 0, G_OPTION_ARG_NONE, &jstate->skip_builtins,
+ "Skip built-in variables, like $HOST and the rest.", NULL },
+ { NULL }
+ };
+
+ gchar **ar;
+ gint arc = argc + 1;
+ gint i;
+
+ ar = g_new(gchar *, arc + 1);
+ for (i = 0; i < argc; i++)
+ ar[i + 1] = g_strdup(argv[i]);
+ ar[0] = g_strdup("format-json");
+ ar[argc + 1] = NULL;
+
+ jstate->template_values =
+ g_hash_table_new_full(g_str_hash, g_str_equal, g_free,
+ (GDestroyNotify) log_template_unref);
+ ctx = g_option_context_new("format-json");
+ msg_add_option_group(ctx);
+ g_option_context_add_main_entries(ctx, format_options, NULL);
+ if (!g_option_context_parse(ctx, &arc, &ar, &parse_error))
+ {
+ /*
+ msg_error ("Error parsing function arguments",
+ evt_tag_str("error", parse_error ?
+ parse_error->message : "Invalid arguments"));
+ */
+ g_set_error(error, LOG_TEMPLATE_ERROR, LOG_TEMPLATE_ERROR_COMPILE,
+ "Error parsing function arguments");
+ g_option_context_free(ctx);
+ tf_json_free_state(jstate);
+ g_strfreev(ar);
+ return FALSE;
+ }
+ g_option_context_free(ctx);
+
+ if (selects == NULL)
+ {
+ jstate->select_patterns = g_new0(GPatternSpec *, 2);
+ jstate->select_patterns[0] = g_pattern_spec_new("*");
+ }
+ else
+ {
+ gint n = g_strv_length (selects);
+ jstate->select_patterns = g_new0(GPatternSpec *, n + 1);
+
+ for (i = 0; i < n; i++)
+ jstate->select_patterns[i] = g_pattern_spec_new(selects[i]);
+ }
+
+ if (excludes != NULL)
+ {
+ gint n = g_strv_length (excludes);
+ jstate->exclude_patterns = g_new0(GPatternSpec *, n + 1);
+
+ for (i = 0; i < n; i++)
+ jstate->exclude_patterns[i] = g_pattern_spec_new(excludes[i]);
+ }
+
+ for (i = 1; i < arc; i++)
+ {
+ gchar *key = NULL;
+ LogTemplate *t = NULL;
+
+ if (g_strstr_len(ar[i], strlen (ar[i]), "="))
+ {
+ gchar **kv = g_strsplit(ar[i], "=", 2);
+
+ key = g_strdup(kv[0]);
+ t = log_template_new(parent->cfg, NULL, kv[1]);
+ g_free(kv[0]);
+ g_free(kv[1]);
+ g_free(kv);
+ }
+ else
+ {
+ gchar *m = g_strconcat("$", ar[i], NULL);
+
+ key = g_strdup(ar[i]);
+ t = log_template_new(parent->cfg, NULL, m);
+ g_free(m);
+ }
+ g_hash_table_insert(jstate->template_values, key, t);
+ }
+
+ jstate->res = g_string_sized_new(256);
+
+ *state = jstate;
+ *state_destroy = (GDestroyNotify) tf_json_free_state;
+
+ g_strfreev(ar);
+ return TRUE;
+}
+
+static gboolean
+tf_json_foreach(NVHandle handle, const gchar *name, const gchar *value,
+ gssize length, gpointer user_data)
+{
+ gpointer *args = user_data;
+ json_object *root = args[0];
+ TFJsonState *jstate = args[1];
+ json_object *this;
+ gboolean match = FALSE;
+ gint i = 0;
+
+ if (jstate->skip_builtins && handle < LM_V_MAX)
+ return FALSE;
+
+ while (jstate->select_patterns[i])
+ {
+ if (g_pattern_match_string(jstate->select_patterns[i], name))
+ {
+ match = TRUE;
+ break;
+ }
+ i++;
+ }
+
+ i = 0;
+ while (jstate->exclude_patterns &&
+ jstate->exclude_patterns[i])
+ {
+ if (g_pattern_match_string(jstate->exclude_patterns[i], name))
+ {
+ match = FALSE;
+ break;
+ }
+ i++;
+ }
+
+ if (match)
+ {
+ this = json_object_new_string_len(value, length);
+ json_object_object_add(root, name, this);
+ }
+
+ return FALSE;
+}
+
+static void
+tf_json_eval (LogTemplateFunction *self, gpointer state, GPtrArray *arg_bufs,
+ LogMessage **messages, gint num_messages, LogTemplateOptions *opts,
+ gint tz, gint seq_num)
+{
+ return;
+}
+
+static json_object *
+tf_json_format_message(TFJsonState *jstate, LogMessage *msg)
+{
+ json_object *root;
+ gpointer args[] = { NULL, jstate };
+
+ root = json_object_new_object();
+ args[0] = root;
+ nv_table_foreach(msg->payload, logmsg_registry, tf_json_foreach,
+ args);
+
+ return root;
+}
+
+static void
+tf_json_format_template(gpointer key, gpointer value, gpointer user_data)
+{
+ json_object *this;
+ gpointer *args = user_data;
+ LogMessage *msg = (LogMessage *)args[1];
+ TFJsonState *jstate = (TFJsonState *)args[2];
+
+ g_string_truncate(jstate->res, 0);
+ log_template_format((LogTemplate *)value, msg, NULL, LTZ_LOCAL, 0, jstate->res);
+
+ this = json_object_new_string(jstate->res->str);
+ json_object_object_add((json_object *)args[0], (gchar *)key, this);
+}
+
+static void
+tf_json_call(LogTemplateFunction *self, gpointer state, GPtrArray *arg_bufs,
+ LogMessage **messages, gint num_messages, LogTemplateOptions *opts,
+ gint tz, gint seq_num, GString *result)
+{
+ gint i;
+ TFJsonState *jstate = (TFJsonState *)state;
+
+ for (i = 0; i < num_messages; i++)
+ {
+ LogMessage *msg = messages[i];
+ json_object *json;
+ gpointer args[] = { NULL, msg, jstate };
+
+ json = tf_json_format_message(jstate, msg);
+ args[0] = json;
+ g_hash_table_foreach(jstate->template_values, tf_json_format_template,
+ args);
+ g_string_append(result, json_object_to_json_string (json));
+ json_object_put(json);
+ }
+}
+
+TEMPLATE_FUNCTION(tf_json, tf_json_prepare, tf_json_eval, tf_json_call, NULL);
+
+static Plugin builtin_tmpl_func_plugins[] =
+ {
+ TEMPLATE_FUNCTION_PLUGIN(tf_json, "format_json"),
+ };
+
+gboolean
+tfjson_module_init(GlobalConfig *cfg, CfgArgs *args)
+{
+ plugin_register(cfg, builtin_tmpl_func_plugins, G_N_ELEMENTS(builtin_tmpl_func_plugins));
+ return TRUE;
+}
--
1.7.0.4