The new parser supports json-c only, it does not work with json-glib. For this reason, the configure script was modified a little, so that it uses --with-json only. When json-c is found or selected, then both the parser and the template function will be built. If json-glib is found or selected, then only the template function will be available. The summary after configure now lists the determined settings. The new parser is - so far - fairly simple: it parses a JSON with json-c (int, double and string values are supported so far, booleans, arrays and sub-objects aren't yet), and sets the appropriate name-value pair in the LogMessage. Users can configure a prefix to prepend before each key. Usage is as simple as: source s_json { tcp(port(21514) flags(no-parse)); }; destination d_json { file("/tmp/test.json" template("$(format-json --scope dot-nv-pairs)\n")); }; parser p_json { json-parser (prefix(".json.")); }; log { source(s_json); parser(p_json); destination(d_json); }; Signed-off-by: Gergely Nagy <algernon@balabit.hu> --- configure.in | 34 ++++--- modules/Makefile.am | 2 +- modules/jsonparser/Makefile.am | 22 ++++ modules/jsonparser/jsonparser-grammar.ym | 86 ++++++++++++++++ modules/jsonparser/jsonparser-parser.c | 49 +++++++++ modules/jsonparser/jsonparser-parser.h | 34 +++++++ modules/jsonparser/jsonparser-plugin.c | 53 ++++++++++ modules/jsonparser/jsonparser.c | 158 ++++++++++++++++++++++++++++++ modules/jsonparser/jsonparser.h | 33 ++++++ modules/tfjson/Makefile.am | 2 +- 10 files changed, 455 insertions(+), 18 deletions(-) create mode 100644 modules/jsonparser/Makefile.am create mode 100644 modules/jsonparser/jsonparser-grammar.ym create mode 100644 modules/jsonparser/jsonparser-parser.c create mode 100644 modules/jsonparser/jsonparser-parser.h create mode 100644 modules/jsonparser/jsonparser-plugin.c create mode 100644 modules/jsonparser/jsonparser.c create mode 100644 modules/jsonparser/jsonparser.h diff --git a/configure.in b/configure.in index 0d97bde..e9b0148 100644 --- a/configure.in +++ b/configure.in @@ -178,10 +178,6 @@ AC_ARG_WITH(json, Use the JSON implementation specified] ,,with_json="auto") -AC_ARG_ENABLE(json, - [ --enable-json Enable support for JSON template formatting (default: auto)] - ,,enable_json="auto") - AC_ARG_ENABLE(systemd, [ --enable-systemd Enable systemd support (default: auto)] ,,enable_systemd="auto") @@ -607,8 +603,12 @@ fi dnl *************************************************************************** dnl json headers/libraries dnl *************************************************************************** -PKG_CHECK_MODULES(JSON_C, json >= $JSON_C_MIN_VERSION,, JSON_C_LIBS="") -PKG_CHECK_MODULES(JSON_GLIB, json-glib-1.0 >= $JSON_GLIB_MIN_VERSION,, JSON_GLIB_LIBS="") +if test "x$with_json" = "xauto" || test "x$with_json" = "json-c"; then + PKG_CHECK_MODULES(JSON_C, json >= $JSON_C_MIN_VERSION,, JSON_C_LIBS="") +fi +if test "x$with_json" = "xauto" || test "x$with_json" = "json-glib"; then + PKG_CHECK_MODULES(JSON_GLIB, json-glib-1.0 >= $JSON_GLIB_MIN_VERSION,, JSON_GLIB_LIBS="") +fi dnl *************************************************************************** dnl pcre headers/libraries @@ -887,14 +887,14 @@ elif test "x$with_json" = "xjson-c"; then AC_MSG_ERROR([Cannot find json-c version >= $JSON_C_MIN_VERSION: is pkg-config in path?]) fi -if test "x$enable_json" = "xauto"; then - if test "x$with_json" = "xno"; then - enable_json="no" - else - enable_json="yes" - fi -elif test "x$enable_json" = "xyes" -a "x$with_json" = "xno"; then - AC_MSG_ERROR([Cannot find json-c version >= $JSON_C_MIN_VERSION or json-glib-1.0 >= $JSON_GLIB_MIN_VERSION: is pkg-config in path?]) +if test "x$with_json" = "xjson-c"; then + enable_json_parse="yes" + enable_json_format="yes" +fi + +if test "x$with_json" = "xjson-glib"; then + enable_json_parse="no" + enable_json_format="yes" fi if test "x$enable_systemd" = "xauto"; then @@ -1077,7 +1077,8 @@ AM_CONDITIONAL(ENABLE_SQL, [test "$enable_sql" = "yes"]) AM_CONDITIONAL(ENABLE_SUN_STREAMS, [test "$enable_sun_streams" = "yes"]) AM_CONDITIONAL(ENABLE_PACCT, [test "$enable_pacct" = "yes"]) AM_CONDITIONAL(ENABLE_MONGODB, [test "$enable_mongodb" = "yes"]) -AM_CONDITIONAL(ENABLE_JSON, [test "$enable_json" = "yes"]) +AM_CONDITIONAL(ENABLE_JSON_FORMAT, [test "$enable_json_format" = "yes"]) +AM_CONDITIONAL(ENABLE_JSON_PARSE, [test "$enable_json_parse" = "yes"]) AM_CONDITIONAL(WITH_LIBSYSTEMD, [test "$with_libsystemd" = "yes"]) # substitution into manual pages @@ -1145,6 +1146,7 @@ AC_OUTPUT(dist.conf modules/basicfuncs/Makefile modules/convertfuncs/Makefile modules/tfjson/Makefile + modules/jsonparser/Makefile scripts/Makefile scripts/update-patterndb doc/Makefile @@ -1194,6 +1196,6 @@ echo " SSL support (module) : ${enable_ssl:=no}" echo " SQL support (module) : ${enable_sql:=no}" echo " PACCT module (EXPERIMENTAL) : ${enable_pacct:=no}" echo " MongoDB destination (module): ${enable_mongodb:=no}" -echo " JSON support (module) : ${enable_json:=no} (using ${with_json})" +echo " JSON support (module) : parser=${enable_json_parse:=no}, formatter=${enable_json_format:=no} (using ${with_json})" diff --git a/modules/Makefile.am b/modules/Makefile.am index 46de2b8..9f67920 100644 --- a/modules/Makefile.am +++ b/modules/Makefile.am @@ -1 +1 @@ -SUBDIRS = afsocket afsql afstreams affile afprog afuser afmongodb csvparser confgen syslogformat pacctformat basicfuncs convertfuncs dbparser tfjson dummy +SUBDIRS = afsocket afsql afstreams affile afprog afuser afmongodb csvparser confgen syslogformat pacctformat basicfuncs convertfuncs dbparser tfjson jsonparser dummy diff --git a/modules/jsonparser/Makefile.am b/modules/jsonparser/Makefile.am new file mode 100644 index 0000000..0925f98 --- /dev/null +++ b/modules/jsonparser/Makefile.am @@ -0,0 +1,22 @@ +moduledir = @moduledir@ +AM_CPPFLAGS = -I$(top_srcdir)/lib -I../../lib +export top_srcdir + +if ENABLE_JSON_PARSE +module_LTLIBRARIES := libjsonparser.la +libjsonparser_la_SOURCES = \ + jsonparser.c jsonparser.h \ + jsonparser-grammar.y \ + jsonparser-parser.c jsonparser-parser.h \ + jsonparser-plugin.c + +libjsonparser_la_CPPFLAGS = $(AM_CPPFLAGS) +libjsonparser_la_CFLAGS = $(JSON_CFLAGS) +libjsonparser_la_LIBADD = $(MODULE_DEPS_LIBS) +libjsonparser_la_LDFLAGS = $(MODULE_LDFLAGS) $(JSON_LIBS) + +BUILT_SOURCES = jsonparser-grammar.y jsonparser-grammar.c jsonparser-grammar.h +EXTRA_DIST = $(BUILT_SOURCES) jsonparser-grammar.ym +endif + +include $(top_srcdir)/build/lex-rules.am diff --git a/modules/jsonparser/jsonparser-grammar.ym b/modules/jsonparser/jsonparser-grammar.ym new file mode 100644 index 0000000..2fe2f3c --- /dev/null +++ b/modules/jsonparser/jsonparser-grammar.ym @@ -0,0 +1,86 @@ +/* + * Copyright (c) 2011 BalaBit IT Ltd, Budapest, Hungary + * Copyright (c) 2011 Gergely Nagy <algernon@balabit.hu> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As an additional exemption you are allowed to compile & link against the + * OpenSSL libraries as published by the OpenSSL project. See the file + * COPYING for details. + */ + +%code top { +#include "jsonparser-parser.h" + +} + + +%code { + +#include "jsonparser.h" +#include "cfg-parser.h" +#include "jsonparser-grammar.h" +#include "syslog-names.h" +#include "messages.h" + +extern LogParser *last_parser; + +} + +%name-prefix "jsonparser_" + +/* this parameter is needed in order to instruct bison to use a complete + * argument list for yylex/yyerror */ + +%lex-param {CfgLexer *lexer} +%parse-param {CfgLexer *lexer} +%parse-param {LogParser **instance} +%parse-param {gpointer arg} + +/* INCLUDE_DECLS */ + +%token KW_JSON_PARSER +%token KW_PREFIX + +%type <ptr> parser_expr_json + +%% + +start + : LL_CONTEXT_PARSER parser_expr_json { YYACCEPT; } + ; + + +parser_expr_json + : KW_JSON_PARSER '(' + { + last_parser = *instance = (LogParser *) log_json_parser_new(); + } + parser_json_opts + ')' { $$ = last_parser; } + ; + +parser_json_opts + : parser_json_opt parser_json_opts + | + ; + +parser_json_opt + : KW_PREFIX '(' string ')' { log_json_parser_set_prefix(last_parser, $3); free ($3); } + | + ; + +/* INCLUDE_RULES */ + +%% diff --git a/modules/jsonparser/jsonparser-parser.c b/modules/jsonparser/jsonparser-parser.c new file mode 100644 index 0000000..a7fe1a6 --- /dev/null +++ b/modules/jsonparser/jsonparser-parser.c @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2011 BalaBit IT Ltd, Budapest, Hungary + * Copyright (c) 2011 Gergely Nagy <algernon@balabit.hu> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As an additional exemption you are allowed to compile & link against the + * OpenSSL libraries as published by the OpenSSL project. See the file + * COPYING for details. + */ + +#include "jsonparser.h" +#include "cfg-parser.h" +#include "jsonparser-grammar.h" + +extern int jsonparser_debug; + +int jsonparser_parse(CfgLexer *lexer, LogParser **instance, gpointer arg); + +static CfgLexerKeyword jsonparser_keywords[] = +{ + { "json_parser", KW_JSON_PARSER, }, + { "prefix", KW_PREFIX, }, + { NULL } +}; + +CfgParser jsonparser_parser = +{ +#if ENABLE_DEBUG + .debug_flag = &jsonparser_debug, +#endif + .name = "jsonparser", + .keywords = jsonparser_keywords, + .parse = (gint (*)(CfgLexer *, gpointer *, gpointer)) jsonparser_parse, + .cleanup = (void (*)(gpointer)) log_pipe_unref, +}; + +CFG_PARSER_IMPLEMENT_LEXER_BINDING(jsonparser_, LogParser **) diff --git a/modules/jsonparser/jsonparser-parser.h b/modules/jsonparser/jsonparser-parser.h new file mode 100644 index 0000000..22ceb84 --- /dev/null +++ b/modules/jsonparser/jsonparser-parser.h @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2011 BalaBit IT Ltd, Budapest, Hungary + * Copyright (c) 2011 Gergely Nagy <algernon@balabit.hu> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As an additional exemption you are allowed to compile & link against the + * OpenSSL libraries as published by the OpenSSL project. See the file + * COPYING for details. + */ + +#ifndef JSONPARSER_PARSER_H_INCLUDED +#define JSONPARSER_PARSER_H_INCLUDED + +#include "cfg-parser.h" +#include "cfg-lexer.h" +#include "logparser.h" + +extern CfgParser jsonparser_parser; + +CFG_PARSER_DECLARE_LEXER_BINDING(jsonparser_, LogParser **) + +#endif diff --git a/modules/jsonparser/jsonparser-plugin.c b/modules/jsonparser/jsonparser-plugin.c new file mode 100644 index 0000000..15c4c91 --- /dev/null +++ b/modules/jsonparser/jsonparser-plugin.c @@ -0,0 +1,53 @@ +/* + * Copyright (c) 2011 BalaBit IT Ltd, Budapest, Hungary + * Copyright (c) 2011 Gergely Nagy <algernon@balabit.hu> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As an additional exemption you are allowed to compile & link against the + * OpenSSL libraries as published by the OpenSSL project. See the file + * COPYING for details. + */ + +#include "cfg-parser.h" +#include "plugin.h" +#include "jsonparser.h" + +extern CfgParser jsonparser_parser; + +static Plugin jsonparser_plugins[] = +{ + { + .type = LL_CONTEXT_PARSER, + .name = "json-parser", + .parser = &jsonparser_parser, + }, +}; + +gboolean +jsonparser_module_init(GlobalConfig *cfg, CfgArgs *args) +{ + plugin_register(cfg, jsonparser_plugins, G_N_ELEMENTS(jsonparser_plugins)); + return TRUE; +} + +const ModuleInfo module_info = +{ + .canonical_name = "jsonparser", + .version = VERSION, + .description = "The jsonparser module provides JSON parsing support for syslog-ng.", + .core_revision = SOURCE_REVISION, + .plugins = jsonparser_plugins, + .plugins_len = G_N_ELEMENTS(jsonparser_plugins), +}; diff --git a/modules/jsonparser/jsonparser.c b/modules/jsonparser/jsonparser.c new file mode 100644 index 0000000..f42766d --- /dev/null +++ b/modules/jsonparser/jsonparser.c @@ -0,0 +1,158 @@ +/* + * Copyright (c) 2011 BalaBit IT Ltd, Budapest, Hungary + * Copyright (c) 2011 Gergely Nagy <algernon@balabit.hu> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As an additional exemption you are allowed to compile & link against the + * OpenSSL libraries as published by the OpenSSL project. See the file + * COPYING for details. + */ + +#include "jsonparser.h" +#include "logparser.h" + +#include <string.h> + +#include <json.h> +#include <json_object_private.h> + +struct _LogJSONParser +{ + LogParser super; + gchar *prefix; + + json_tokener *tokener; + struct + { + GString *key; + GString *value; + } serialized; +}; + +void +log_json_parser_set_prefix (LogParser *p, const gchar *prefix) +{ + LogJSONParser *self = (LogJSONParser *)p; + + g_free (self->prefix); + self->prefix = g_strdup (prefix); +} + +static gboolean +log_json_parser_process (LogParser *s, LogMessage *msg, const gchar *input) +{ + LogJSONParser *self = (LogJSONParser *) s; + struct json_object *jso; + struct json_object_iter itr; + + json_tokener_reset (self->tokener); + jso = json_tokener_parse_ex (self->tokener, input, strlen (input)); + + json_object_object_foreachC (jso, itr) + { + gboolean parsed = FALSE; + + switch (json_object_get_type (itr.val)) + { + case json_type_boolean: + msg_info ("JSON parser does not support boolean types yet, skipping", + evt_tag_str ("key", itr.key), NULL); + break; + case json_type_double: + parsed = TRUE; + g_string_printf (self->serialized.value, "%f", + json_object_get_double (itr.val)); + break; + case json_type_int: + parsed = TRUE; + g_string_printf (self->serialized.value, "%i", + json_object_get_int (itr.val)); + break; + case json_type_string: + parsed = TRUE; + g_string_assign (self->serialized.value, + json_object_get_string (itr.val)); + break; + case json_type_object: + case json_type_array: + msg_error ("JSON parser does not support objects and arrays yet, " + "skipping", + evt_tag_str ("key", itr.key), NULL); + break; + default: + msg_error ("JSON parser encountered an unknown type, skipping", + evt_tag_str ("key", itr.key), NULL); + break; + } + + if (parsed) + { + if (self->prefix) + g_string_printf (self->serialized.key, "%s%s", + self->prefix, itr.key); + else + g_string_assign (self->serialized.key, itr.key); + log_msg_set_value (msg, + log_msg_get_value_handle (self->serialized.key->str), + self->serialized.value->str, + self->serialized.value->len); + } + } + json_object_put (jso); + + return TRUE; +} + +static LogPipe * +log_json_parser_clone (LogProcessPipe *s) +{ + LogJSONParser *self = (LogJSONParser *) s; + LogJSONParser *cloned; + + cloned = (LogJSONParser *) log_json_parser_new (); + log_json_parser_set_prefix ((LogParser *)cloned, self->prefix); + + return &cloned->super.super.super; +} + +static void +log_json_parser_free (LogPipe *s) +{ + LogJSONParser *self = (LogJSONParser *)s; + + g_free (self->prefix); + g_string_free (self->serialized.key, TRUE); + g_string_free (self->serialized.value, TRUE); + + json_tokener_free (self->tokener); + log_parser_free_method (s); +} + +LogJSONParser * +log_json_parser_new (void) +{ + LogJSONParser *self = g_new0 (LogJSONParser, 1); + + log_parser_init_instance (&self->super); + self->super.super.super.free_fn = log_json_parser_free; + self->super.super.clone = log_json_parser_clone; + self->super.process = log_json_parser_process; + + self->tokener = json_tokener_new (); + self->serialized.key = g_string_new (NULL); + self->serialized.value = g_string_new (NULL); + + return self; +} diff --git a/modules/jsonparser/jsonparser.h b/modules/jsonparser/jsonparser.h new file mode 100644 index 0000000..3f6796a --- /dev/null +++ b/modules/jsonparser/jsonparser.h @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2011 BalaBit IT Ltd, Budapest, Hungary + * Copyright (c) 2011 Gergely Nagy <algernon@balabit.hu> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As an additional exemption you are allowed to compile & link against the + * OpenSSL libraries as published by the OpenSSL project. See the file + * COPYING for details. + */ + +#ifndef JSONPARSER_H_INCLUDED +#define JSONPARSER_H_INCLUDED + +#include "logparser.h" + +typedef struct _LogJSONParser LogJSONParser; + +void log_json_parser_set_prefix(LogParser *p, const gchar *prefix); +LogJSONParser *log_json_parser_new(void); + +#endif diff --git a/modules/tfjson/Makefile.am b/modules/tfjson/Makefile.am index 5d80564..2387366 100644 --- a/modules/tfjson/Makefile.am +++ b/modules/tfjson/Makefile.am @@ -1,7 +1,7 @@ moduledir = @moduledir@ export top_srcdir -if ENABLE_JSON +if ENABLE_JSON_FORMAT AM_CPPFLAGS = -I$(top_srcdir)/lib -I../../lib $(JSON_CFLAGS) module_LTLIBRARIES = libtfjson.la -- 1.7.7.1