[syslog-ng] [PATCH] jsonparser: New parser module for JSON parsing.
Gergely Nagy
algernon at balabit.hu
Fri Oct 28 16:04:08 CEST 2011
The new parser supports json-c only, it does not work with
json-glib. For this reason, the configure script was modified a
little, so that it uses --with-json only. When json-c is found or
selected, then both the parser and the template function will be
built. If json-glib is found or selected, then only the template
function will be available.
The summary after configure now lists the determined settings.
The new parser is - so far - fairly simple: it parses a JSON with
json-c (int, double and string values are supported so far, booleans,
arrays and sub-objects aren't yet), and sets the appropriate
name-value pair in the LogMessage.
Users can configure a prefix to prepend before each key.
Usage is as simple as:
source s_json { tcp(port(21514) flags(no-parse)); };
destination d_json {
file("/tmp/test.json"
template("$(format-json --scope dot-nv-pairs)\n"));
};
parser p_json { json-parser (prefix(".json.")); };
log { source(s_json); parser(p_json); destination(d_json); };
Signed-off-by: Gergely Nagy <algernon at balabit.hu>
---
configure.in | 34 ++++---
modules/Makefile.am | 2 +-
modules/jsonparser/Makefile.am | 22 ++++
modules/jsonparser/jsonparser-grammar.ym | 86 ++++++++++++++++
modules/jsonparser/jsonparser-parser.c | 49 +++++++++
modules/jsonparser/jsonparser-parser.h | 34 +++++++
modules/jsonparser/jsonparser-plugin.c | 53 ++++++++++
modules/jsonparser/jsonparser.c | 158 ++++++++++++++++++++++++++++++
modules/jsonparser/jsonparser.h | 33 ++++++
modules/tfjson/Makefile.am | 2 +-
10 files changed, 455 insertions(+), 18 deletions(-)
create mode 100644 modules/jsonparser/Makefile.am
create mode 100644 modules/jsonparser/jsonparser-grammar.ym
create mode 100644 modules/jsonparser/jsonparser-parser.c
create mode 100644 modules/jsonparser/jsonparser-parser.h
create mode 100644 modules/jsonparser/jsonparser-plugin.c
create mode 100644 modules/jsonparser/jsonparser.c
create mode 100644 modules/jsonparser/jsonparser.h
diff --git a/configure.in b/configure.in
index 0d97bde..e9b0148 100644
--- a/configure.in
+++ b/configure.in
@@ -178,10 +178,6 @@ AC_ARG_WITH(json,
Use the JSON implementation specified]
,,with_json="auto")
-AC_ARG_ENABLE(json,
- [ --enable-json Enable support for JSON template formatting (default: auto)]
- ,,enable_json="auto")
-
AC_ARG_ENABLE(systemd,
[ --enable-systemd Enable systemd support (default: auto)]
,,enable_systemd="auto")
@@ -607,8 +603,12 @@ fi
dnl ***************************************************************************
dnl json headers/libraries
dnl ***************************************************************************
-PKG_CHECK_MODULES(JSON_C, json >= $JSON_C_MIN_VERSION,, JSON_C_LIBS="")
-PKG_CHECK_MODULES(JSON_GLIB, json-glib-1.0 >= $JSON_GLIB_MIN_VERSION,, JSON_GLIB_LIBS="")
+if test "x$with_json" = "xauto" || test "x$with_json" = "json-c"; then
+ PKG_CHECK_MODULES(JSON_C, json >= $JSON_C_MIN_VERSION,, JSON_C_LIBS="")
+fi
+if test "x$with_json" = "xauto" || test "x$with_json" = "json-glib"; then
+ PKG_CHECK_MODULES(JSON_GLIB, json-glib-1.0 >= $JSON_GLIB_MIN_VERSION,, JSON_GLIB_LIBS="")
+fi
dnl ***************************************************************************
dnl pcre headers/libraries
@@ -887,14 +887,14 @@ elif test "x$with_json" = "xjson-c"; then
AC_MSG_ERROR([Cannot find json-c version >= $JSON_C_MIN_VERSION: is pkg-config in path?])
fi
-if test "x$enable_json" = "xauto"; then
- if test "x$with_json" = "xno"; then
- enable_json="no"
- else
- enable_json="yes"
- fi
-elif test "x$enable_json" = "xyes" -a "x$with_json" = "xno"; then
- AC_MSG_ERROR([Cannot find json-c version >= $JSON_C_MIN_VERSION or json-glib-1.0 >= $JSON_GLIB_MIN_VERSION: is pkg-config in path?])
+if test "x$with_json" = "xjson-c"; then
+ enable_json_parse="yes"
+ enable_json_format="yes"
+fi
+
+if test "x$with_json" = "xjson-glib"; then
+ enable_json_parse="no"
+ enable_json_format="yes"
fi
if test "x$enable_systemd" = "xauto"; then
@@ -1077,7 +1077,8 @@ AM_CONDITIONAL(ENABLE_SQL, [test "$enable_sql" = "yes"])
AM_CONDITIONAL(ENABLE_SUN_STREAMS, [test "$enable_sun_streams" = "yes"])
AM_CONDITIONAL(ENABLE_PACCT, [test "$enable_pacct" = "yes"])
AM_CONDITIONAL(ENABLE_MONGODB, [test "$enable_mongodb" = "yes"])
-AM_CONDITIONAL(ENABLE_JSON, [test "$enable_json" = "yes"])
+AM_CONDITIONAL(ENABLE_JSON_FORMAT, [test "$enable_json_format" = "yes"])
+AM_CONDITIONAL(ENABLE_JSON_PARSE, [test "$enable_json_parse" = "yes"])
AM_CONDITIONAL(WITH_LIBSYSTEMD, [test "$with_libsystemd" = "yes"])
# substitution into manual pages
@@ -1145,6 +1146,7 @@ AC_OUTPUT(dist.conf
modules/basicfuncs/Makefile
modules/convertfuncs/Makefile
modules/tfjson/Makefile
+ modules/jsonparser/Makefile
scripts/Makefile
scripts/update-patterndb
doc/Makefile
@@ -1194,6 +1196,6 @@ echo " SSL support (module) : ${enable_ssl:=no}"
echo " SQL support (module) : ${enable_sql:=no}"
echo " PACCT module (EXPERIMENTAL) : ${enable_pacct:=no}"
echo " MongoDB destination (module): ${enable_mongodb:=no}"
-echo " JSON support (module) : ${enable_json:=no} (using ${with_json})"
+echo " JSON support (module) : parser=${enable_json_parse:=no}, formatter=${enable_json_format:=no} (using ${with_json})"
diff --git a/modules/Makefile.am b/modules/Makefile.am
index 46de2b8..9f67920 100644
--- a/modules/Makefile.am
+++ b/modules/Makefile.am
@@ -1 +1 @@
-SUBDIRS = afsocket afsql afstreams affile afprog afuser afmongodb csvparser confgen syslogformat pacctformat basicfuncs convertfuncs dbparser tfjson dummy
+SUBDIRS = afsocket afsql afstreams affile afprog afuser afmongodb csvparser confgen syslogformat pacctformat basicfuncs convertfuncs dbparser tfjson jsonparser dummy
diff --git a/modules/jsonparser/Makefile.am b/modules/jsonparser/Makefile.am
new file mode 100644
index 0000000..0925f98
--- /dev/null
+++ b/modules/jsonparser/Makefile.am
@@ -0,0 +1,22 @@
+moduledir = @moduledir@
+AM_CPPFLAGS = -I$(top_srcdir)/lib -I../../lib
+export top_srcdir
+
+if ENABLE_JSON_PARSE
+module_LTLIBRARIES := libjsonparser.la
+libjsonparser_la_SOURCES = \
+ jsonparser.c jsonparser.h \
+ jsonparser-grammar.y \
+ jsonparser-parser.c jsonparser-parser.h \
+ jsonparser-plugin.c
+
+libjsonparser_la_CPPFLAGS = $(AM_CPPFLAGS)
+libjsonparser_la_CFLAGS = $(JSON_CFLAGS)
+libjsonparser_la_LIBADD = $(MODULE_DEPS_LIBS)
+libjsonparser_la_LDFLAGS = $(MODULE_LDFLAGS) $(JSON_LIBS)
+
+BUILT_SOURCES = jsonparser-grammar.y jsonparser-grammar.c jsonparser-grammar.h
+EXTRA_DIST = $(BUILT_SOURCES) jsonparser-grammar.ym
+endif
+
+include $(top_srcdir)/build/lex-rules.am
diff --git a/modules/jsonparser/jsonparser-grammar.ym b/modules/jsonparser/jsonparser-grammar.ym
new file mode 100644
index 0000000..2fe2f3c
--- /dev/null
+++ b/modules/jsonparser/jsonparser-grammar.ym
@@ -0,0 +1,86 @@
+/*
+ * Copyright (c) 2011 BalaBit IT Ltd, Budapest, Hungary
+ * Copyright (c) 2011 Gergely Nagy <algernon at balabit.hu>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * As an additional exemption you are allowed to compile & link against the
+ * OpenSSL libraries as published by the OpenSSL project. See the file
+ * COPYING for details.
+ */
+
+%code top {
+#include "jsonparser-parser.h"
+
+}
+
+
+%code {
+
+#include "jsonparser.h"
+#include "cfg-parser.h"
+#include "jsonparser-grammar.h"
+#include "syslog-names.h"
+#include "messages.h"
+
+extern LogParser *last_parser;
+
+}
+
+%name-prefix "jsonparser_"
+
+/* this parameter is needed in order to instruct bison to use a complete
+ * argument list for yylex/yyerror */
+
+%lex-param {CfgLexer *lexer}
+%parse-param {CfgLexer *lexer}
+%parse-param {LogParser **instance}
+%parse-param {gpointer arg}
+
+/* INCLUDE_DECLS */
+
+%token KW_JSON_PARSER
+%token KW_PREFIX
+
+%type <ptr> parser_expr_json
+
+%%
+
+start
+ : LL_CONTEXT_PARSER parser_expr_json { YYACCEPT; }
+ ;
+
+
+parser_expr_json
+ : KW_JSON_PARSER '('
+ {
+ last_parser = *instance = (LogParser *) log_json_parser_new();
+ }
+ parser_json_opts
+ ')' { $$ = last_parser; }
+ ;
+
+parser_json_opts
+ : parser_json_opt parser_json_opts
+ |
+ ;
+
+parser_json_opt
+ : KW_PREFIX '(' string ')' { log_json_parser_set_prefix(last_parser, $3); free ($3); }
+ |
+ ;
+
+/* INCLUDE_RULES */
+
+%%
diff --git a/modules/jsonparser/jsonparser-parser.c b/modules/jsonparser/jsonparser-parser.c
new file mode 100644
index 0000000..a7fe1a6
--- /dev/null
+++ b/modules/jsonparser/jsonparser-parser.c
@@ -0,0 +1,49 @@
+/*
+ * Copyright (c) 2011 BalaBit IT Ltd, Budapest, Hungary
+ * Copyright (c) 2011 Gergely Nagy <algernon at balabit.hu>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * As an additional exemption you are allowed to compile & link against the
+ * OpenSSL libraries as published by the OpenSSL project. See the file
+ * COPYING for details.
+ */
+
+#include "jsonparser.h"
+#include "cfg-parser.h"
+#include "jsonparser-grammar.h"
+
+extern int jsonparser_debug;
+
+int jsonparser_parse(CfgLexer *lexer, LogParser **instance, gpointer arg);
+
+static CfgLexerKeyword jsonparser_keywords[] =
+{
+ { "json_parser", KW_JSON_PARSER, },
+ { "prefix", KW_PREFIX, },
+ { NULL }
+};
+
+CfgParser jsonparser_parser =
+{
+#if ENABLE_DEBUG
+ .debug_flag = &jsonparser_debug,
+#endif
+ .name = "jsonparser",
+ .keywords = jsonparser_keywords,
+ .parse = (gint (*)(CfgLexer *, gpointer *, gpointer)) jsonparser_parse,
+ .cleanup = (void (*)(gpointer)) log_pipe_unref,
+};
+
+CFG_PARSER_IMPLEMENT_LEXER_BINDING(jsonparser_, LogParser **)
diff --git a/modules/jsonparser/jsonparser-parser.h b/modules/jsonparser/jsonparser-parser.h
new file mode 100644
index 0000000..22ceb84
--- /dev/null
+++ b/modules/jsonparser/jsonparser-parser.h
@@ -0,0 +1,34 @@
+/*
+ * Copyright (c) 2011 BalaBit IT Ltd, Budapest, Hungary
+ * Copyright (c) 2011 Gergely Nagy <algernon at balabit.hu>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * As an additional exemption you are allowed to compile & link against the
+ * OpenSSL libraries as published by the OpenSSL project. See the file
+ * COPYING for details.
+ */
+
+#ifndef JSONPARSER_PARSER_H_INCLUDED
+#define JSONPARSER_PARSER_H_INCLUDED
+
+#include "cfg-parser.h"
+#include "cfg-lexer.h"
+#include "logparser.h"
+
+extern CfgParser jsonparser_parser;
+
+CFG_PARSER_DECLARE_LEXER_BINDING(jsonparser_, LogParser **)
+
+#endif
diff --git a/modules/jsonparser/jsonparser-plugin.c b/modules/jsonparser/jsonparser-plugin.c
new file mode 100644
index 0000000..15c4c91
--- /dev/null
+++ b/modules/jsonparser/jsonparser-plugin.c
@@ -0,0 +1,53 @@
+/*
+ * Copyright (c) 2011 BalaBit IT Ltd, Budapest, Hungary
+ * Copyright (c) 2011 Gergely Nagy <algernon at balabit.hu>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * As an additional exemption you are allowed to compile & link against the
+ * OpenSSL libraries as published by the OpenSSL project. See the file
+ * COPYING for details.
+ */
+
+#include "cfg-parser.h"
+#include "plugin.h"
+#include "jsonparser.h"
+
+extern CfgParser jsonparser_parser;
+
+static Plugin jsonparser_plugins[] =
+{
+ {
+ .type = LL_CONTEXT_PARSER,
+ .name = "json-parser",
+ .parser = &jsonparser_parser,
+ },
+};
+
+gboolean
+jsonparser_module_init(GlobalConfig *cfg, CfgArgs *args)
+{
+ plugin_register(cfg, jsonparser_plugins, G_N_ELEMENTS(jsonparser_plugins));
+ return TRUE;
+}
+
+const ModuleInfo module_info =
+{
+ .canonical_name = "jsonparser",
+ .version = VERSION,
+ .description = "The jsonparser module provides JSON parsing support for syslog-ng.",
+ .core_revision = SOURCE_REVISION,
+ .plugins = jsonparser_plugins,
+ .plugins_len = G_N_ELEMENTS(jsonparser_plugins),
+};
diff --git a/modules/jsonparser/jsonparser.c b/modules/jsonparser/jsonparser.c
new file mode 100644
index 0000000..f42766d
--- /dev/null
+++ b/modules/jsonparser/jsonparser.c
@@ -0,0 +1,158 @@
+/*
+ * Copyright (c) 2011 BalaBit IT Ltd, Budapest, Hungary
+ * Copyright (c) 2011 Gergely Nagy <algernon at balabit.hu>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * As an additional exemption you are allowed to compile & link against the
+ * OpenSSL libraries as published by the OpenSSL project. See the file
+ * COPYING for details.
+ */
+
+#include "jsonparser.h"
+#include "logparser.h"
+
+#include <string.h>
+
+#include <json.h>
+#include <json_object_private.h>
+
+struct _LogJSONParser
+{
+ LogParser super;
+ gchar *prefix;
+
+ json_tokener *tokener;
+ struct
+ {
+ GString *key;
+ GString *value;
+ } serialized;
+};
+
+void
+log_json_parser_set_prefix (LogParser *p, const gchar *prefix)
+{
+ LogJSONParser *self = (LogJSONParser *)p;
+
+ g_free (self->prefix);
+ self->prefix = g_strdup (prefix);
+}
+
+static gboolean
+log_json_parser_process (LogParser *s, LogMessage *msg, const gchar *input)
+{
+ LogJSONParser *self = (LogJSONParser *) s;
+ struct json_object *jso;
+ struct json_object_iter itr;
+
+ json_tokener_reset (self->tokener);
+ jso = json_tokener_parse_ex (self->tokener, input, strlen (input));
+
+ json_object_object_foreachC (jso, itr)
+ {
+ gboolean parsed = FALSE;
+
+ switch (json_object_get_type (itr.val))
+ {
+ case json_type_boolean:
+ msg_info ("JSON parser does not support boolean types yet, skipping",
+ evt_tag_str ("key", itr.key), NULL);
+ break;
+ case json_type_double:
+ parsed = TRUE;
+ g_string_printf (self->serialized.value, "%f",
+ json_object_get_double (itr.val));
+ break;
+ case json_type_int:
+ parsed = TRUE;
+ g_string_printf (self->serialized.value, "%i",
+ json_object_get_int (itr.val));
+ break;
+ case json_type_string:
+ parsed = TRUE;
+ g_string_assign (self->serialized.value,
+ json_object_get_string (itr.val));
+ break;
+ case json_type_object:
+ case json_type_array:
+ msg_error ("JSON parser does not support objects and arrays yet, "
+ "skipping",
+ evt_tag_str ("key", itr.key), NULL);
+ break;
+ default:
+ msg_error ("JSON parser encountered an unknown type, skipping",
+ evt_tag_str ("key", itr.key), NULL);
+ break;
+ }
+
+ if (parsed)
+ {
+ if (self->prefix)
+ g_string_printf (self->serialized.key, "%s%s",
+ self->prefix, itr.key);
+ else
+ g_string_assign (self->serialized.key, itr.key);
+ log_msg_set_value (msg,
+ log_msg_get_value_handle (self->serialized.key->str),
+ self->serialized.value->str,
+ self->serialized.value->len);
+ }
+ }
+ json_object_put (jso);
+
+ return TRUE;
+}
+
+static LogPipe *
+log_json_parser_clone (LogProcessPipe *s)
+{
+ LogJSONParser *self = (LogJSONParser *) s;
+ LogJSONParser *cloned;
+
+ cloned = (LogJSONParser *) log_json_parser_new ();
+ log_json_parser_set_prefix ((LogParser *)cloned, self->prefix);
+
+ return &cloned->super.super.super;
+}
+
+static void
+log_json_parser_free (LogPipe *s)
+{
+ LogJSONParser *self = (LogJSONParser *)s;
+
+ g_free (self->prefix);
+ g_string_free (self->serialized.key, TRUE);
+ g_string_free (self->serialized.value, TRUE);
+
+ json_tokener_free (self->tokener);
+ log_parser_free_method (s);
+}
+
+LogJSONParser *
+log_json_parser_new (void)
+{
+ LogJSONParser *self = g_new0 (LogJSONParser, 1);
+
+ log_parser_init_instance (&self->super);
+ self->super.super.super.free_fn = log_json_parser_free;
+ self->super.super.clone = log_json_parser_clone;
+ self->super.process = log_json_parser_process;
+
+ self->tokener = json_tokener_new ();
+ self->serialized.key = g_string_new (NULL);
+ self->serialized.value = g_string_new (NULL);
+
+ return self;
+}
diff --git a/modules/jsonparser/jsonparser.h b/modules/jsonparser/jsonparser.h
new file mode 100644
index 0000000..3f6796a
--- /dev/null
+++ b/modules/jsonparser/jsonparser.h
@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 2011 BalaBit IT Ltd, Budapest, Hungary
+ * Copyright (c) 2011 Gergely Nagy <algernon at balabit.hu>
+ *
+ * This program is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 as published
+ * by the Free Software Foundation, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
+ *
+ * As an additional exemption you are allowed to compile & link against the
+ * OpenSSL libraries as published by the OpenSSL project. See the file
+ * COPYING for details.
+ */
+
+#ifndef JSONPARSER_H_INCLUDED
+#define JSONPARSER_H_INCLUDED
+
+#include "logparser.h"
+
+typedef struct _LogJSONParser LogJSONParser;
+
+void log_json_parser_set_prefix(LogParser *p, const gchar *prefix);
+LogJSONParser *log_json_parser_new(void);
+
+#endif
diff --git a/modules/tfjson/Makefile.am b/modules/tfjson/Makefile.am
index 5d80564..2387366 100644
--- a/modules/tfjson/Makefile.am
+++ b/modules/tfjson/Makefile.am
@@ -1,7 +1,7 @@
moduledir = @moduledir@
export top_srcdir
-if ENABLE_JSON
+if ENABLE_JSON_FORMAT
AM_CPPFLAGS = -I$(top_srcdir)/lib -I../../lib $(JSON_CFLAGS)
module_LTLIBRARIES = libtfjson.la
--
1.7.7.1
More information about the syslog-ng
mailing list