- syslog-ng - lists.balabit.hu

[PATCH] afsmtp: New destination driver, to send mails via SMTP.
by Gergely Nagy 05 Feb '12

05 Feb '12

This introduces the afsmtp destination driver, updated to the latest 3.4 conventions, and with all the threading bugs that were also present in afmongodb too, fixed. The driver itself uses libESMTP to send mail, and has the following config syntax: destination d_smtp { smtp( host("localhost") port(25) subject("ALERT! IMPORTANT EVENT FROM ${HOST}") header("X-Severity", "${SEVERITY}") from("no-reply(a)${HOST}.dummy") to("syslog-alerts(a)my.example.domain") bcc("big-brother(a)my.example.domain") cc("it(a)my.example.domain") reply-to("admin(a)my.example.domain") ); }; Signed-off-by: Gergely Nagy <algernon(a)balabit.hu> --- configure.in | 41 ++- modules/Makefile.am | 2 +- modules/afsmtp/Makefile.am | 19 + modules/afsmtp/afsmtp-grammar.ym | 101 ++++++ modules/afsmtp/afsmtp-parser.c | 58 +++ modules/afsmtp/afsmtp-parser.h | 35 ++ modules/afsmtp/afsmtp.c | 728 ++++++++++++++++++++++++++++++++++++++ modules/afsmtp/afsmtp.h | 52 +++ 8 files changed, 1034 insertions(+), 2 deletions(-) create mode 100644 modules/afsmtp/Makefile.am create mode 100644 modules/afsmtp/afsmtp-grammar.ym create mode 100644 modules/afsmtp/afsmtp-parser.c create mode 100644 modules/afsmtp/afsmtp-parser.h create mode 100644 modules/afsmtp/afsmtp.c create mode 100644 modules/afsmtp/afsmtp.h diff --git a/configure.in b/configure.in index 0a3d730..2fab7ac 100644 --- a/configure.in +++ b/configure.in @@ -174,6 +174,10 @@ AC_ARG_WITH(json, Use the JSON implementation specified] ,,with_json="auto") +AC_ARG_WITH(libesmtp, + AC_HELP_STRING([--with-libesmtp=DIR], + [use libesmtp library from (prefix) directory DIR]),,) + AC_ARG_ENABLE(systemd, [ --enable-systemd Enable systemd support (default: auto)] ,,enable_systemd="auto") @@ -793,6 +797,37 @@ if test "x$with_libmongo_client" = "xno"; then fi dnl *************************************************************************** +dnl libesmtp headers/libraries +dnl *************************************************************************** +if test "x$enable_smtp" != "xno" && test "x$with_libesmtp" != "no"; then + libesmtp="yes" + if test "x$with_libesmtp" != "yes" && test "x$with_libesmtp" != "x"; then + CPPFLAGS_SAVE="$CPPFLAGS" + LDFLAGS_SAVE="$LDFLAGS" + CPPFLAGS="$CPPFLAGS -I$with_libesmtp/include" + LDFLAGS="$LDFLAGS -L$with_libesmtp/lib" + AC_CHECK_HEADER(libesmtp.h, [LIBESMTP_CFLAGS="-I$with_libesmtp/include" + LIBESMTP_LIBS="-L$with_libesmtp/lib"], [libesmtp=no]) + CPPFLAGS="$CPPFLAGS_SAVE" + LDFLAGS="$LDFLAGS_SAVE" + else + AC_MSG_CHECKING(for libESMTP) + if libesmtp-config --version >/dev/null 2>&1; then + AC_MSG_RESULT(yes) + LIBESMTP_CFLAGS="`libesmtp-config --cflags`" + LIBESMTP_LIBS="`libesmtp-config --libs`" + else + AC_MSG_RESULT(no) + libesmtp=no + fi + fi + if test "x$enable_smtp" = "xyes" && test "x$libesmtp" = "xno"; then + AC_MSG_ERROR(libESMTP not found) + fi + enable_smtp=$libesmtp +fi + +dnl *************************************************************************** dnl misc features to be enabled dnl *************************************************************************** @@ -1065,6 +1100,7 @@ AM_CONDITIONAL(ENABLE_SQL, [test "$enable_sql" = "yes"]) AM_CONDITIONAL(ENABLE_SUN_STREAMS, [test "$enable_sun_streams" = "yes"]) AM_CONDITIONAL(ENABLE_PACCT, [test "$enable_pacct" = "yes"]) AM_CONDITIONAL(ENABLE_MONGODB, [test "$enable_mongodb" = "yes"]) +AM_CONDITIONAL(ENABLE_SMTP, [test "$enable_smtp" = "yes"]) AM_CONDITIONAL(ENABLE_JSON_FORMAT, [test "$enable_json_format" = "yes"]) AM_CONDITIONAL(ENABLE_JSON_PARSE, [test "$enable_json_parse" = "yes"]) AM_CONDITIONAL(WITH_LIBSYSTEMD, [test "$with_libsystemd" = "yes"]) @@ -1098,6 +1134,8 @@ AC_SUBST(LIBDBI_CFLAGS) AC_SUBST(LIBMONGO_LIBS) AC_SUBST(LIBMONGO_CFLAGS) AC_SUBST(LIBMONGO_SUBDIRS) +AC_SUBST(LIBESMTP_CFLAGS) +AC_SUBST(LIBESMTP_LIBS) AC_SUBST(JSON_LIBS) AC_SUBST(JSON_CFLAGS) AC_SUBST(IVYKIS_SUBDIRS) @@ -1125,6 +1163,7 @@ AC_OUTPUT(dist.conf modules/afprog/Makefile modules/afuser/Makefile modules/afmongodb/Makefile + modules/afsmtp/Makefile modules/dbparser/Makefile modules/dbparser/tests/Makefile modules/csvparser/Makefile @@ -1183,5 +1222,5 @@ echo " SQL support (module) : ${enable_sql:=no}" echo " PACCT module (EXPERIMENTAL) : ${enable_pacct:=no}" echo " MongoDB destination (module): ${enable_mongodb:=no}" echo " JSON support (module) : parser=${enable_json_parse:=no}, formatter=${enable_json_format:=no} (using ${with_json})" - +echo " SMTP support (module) : ${enable_smtp:=no}" diff --git a/modules/Makefile.am b/modules/Makefile.am index 0d0594b..340df40 100644 --- a/modules/Makefile.am +++ b/modules/Makefile.am @@ -1 +1 @@ -SUBDIRS = afsocket afsql afstreams affile afprog afuser afmongodb csvparser confgen syslogformat pacctformat basicfuncs dbparser tfjson jsonparser dummy +SUBDIRS = afsocket afsql afstreams affile afprog afuser afmongodb afsmtp csvparser confgen syslogformat pacctformat basicfuncs dbparser tfjson jsonparser dummy diff --git a/modules/afsmtp/Makefile.am b/modules/afsmtp/Makefile.am new file mode 100644 index 0000000..790aad7 --- /dev/null +++ b/modules/afsmtp/Makefile.am @@ -0,0 +1,19 @@ +moduledir = @moduledir@ +AM_CPPFLAGS = -I$(top_srcdir)/lib -I../../lib +module_LTLIBRARIES = libafsmtp.la + +export top_srcdir + +if ENABLE_SMTP + +libafsmtp_la_CFLAGS = $(LIBESMTP_CFLAGS) $(IVYKIS_CFLAGS) $(EVENTLOG_CFLAGS) $(SYSLOG_NG_CFLAGS) +libafsmtp_la_SOURCES = afsmtp-grammar.y afsmtp.c afsmtp.h afsmtp-parser.c afsmtp-parser.h +libafsmtp_la_LIBADD = $(LIBESMTP_LIBS) $(IVYKIS_LIBS) $(EVENTLOG_LIBS) $(SYSLOG_NG_LIBS) +libafsmtp_la_LDFLAGS = -avoid-version + +endif + +BUILT_SOURCES = afsmtp-grammar.y afsmtp-grammar.c afsmtp-grammar.h +EXTRA_DIST = $(BUILT_SOURCES) afsmtp-grammar.ym + +include $(top_srcdir)/build/lex-rules.am diff --git a/modules/afsmtp/afsmtp-grammar.ym b/modules/afsmtp/afsmtp-grammar.ym new file mode 100644 index 0000000..eb6eecf --- /dev/null +++ b/modules/afsmtp/afsmtp-grammar.ym @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2011-2012 BalaBit IT Ltd, Budapest, Hungary + * Copyright (c) 2011-2012 Gergely Nagy <algernon(a)balabit.hu> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As an additional exemption you are allowed to compile & link against the + * OpenSSL libraries as published by the OpenSSL project. See the file + * COPYING for details. + * + */ + +%code requires { + +#include "afsmtp-parser.h" + +} + +%code { + +#include "cfg-parser.h" +#include "afsmtp-grammar.h" +#include "plugin.h" + +extern LogDriver *last_driver; + +} + +%name-prefix "afsmtp_" +%lex-param {CfgLexer *lexer} +%parse-param {CfgLexer *lexer} +%parse-param {LogDriver **instance} +%parse-param {gpointer arg} + +/* INCLUDE_DECLS */ + +%token KW_SMTP +%token KW_SUBJECT +%token KW_FROM +%token KW_TO +%token KW_BODY +%token KW_HEADER +%token KW_CC +%token KW_BCC +%token KW_SENDER +%token KW_REPLY_TO + +%% + +start + : LL_CONTEXT_DESTINATION KW_SMTP + { + last_driver = *instance = afsmtp_dd_new(); + } + '(' afsmtp_options ')' { YYACCEPT; } + ; + +afsmtp_options + : afsmtp_option afsmtp_options + | + ; + +afsmtp_option + : KW_HOST '(' string ')' { afsmtp_dd_set_host(last_driver, $3); free($3); } + | KW_PORT '(' LL_NUMBER ')' { afsmtp_dd_set_port(last_driver, $3); } + | KW_SUBJECT '(' string ')' { afsmtp_dd_set_subject(last_driver, $3); free($3); } + | KW_BODY '(' string ')' { afsmtp_dd_set_body(last_driver, $3); free($3); } + | KW_HEADER '(' string string ')' { + afsmtp_dd_add_header(last_driver, $3, $4); + free($3); free($4); + } + + | KW_FROM '(' string ')' { afsmtp_dd_set_from(last_driver, $3, $3); free($3); } + | KW_FROM '(' string string ')' { afsmtp_dd_set_from(last_driver, $3, $4); free($3); free($4); } + | KW_SENDER '(' string ')' { afsmtp_dd_set_from(last_driver, $3, $3); free($3); } + | KW_SENDER '(' string string ')' { afsmtp_dd_set_from(last_driver, $3, $4); free($3); free($4); } + | KW_TO '(' string ')' { afsmtp_dd_add_rcpt(last_driver, AFSMTP_RCPT_TYPE_TO, $3, $3); free($3); } + | KW_TO '(' string string ')' { afsmtp_dd_add_rcpt(last_driver, AFSMTP_RCPT_TYPE_TO, $3, $4); free($3); free($4); } + | KW_CC '(' string ')' { afsmtp_dd_add_rcpt(last_driver, AFSMTP_RCPT_TYPE_CC, $3, $3); free($3); } + | KW_CC '(' string string ')' { afsmtp_dd_add_rcpt(last_driver, AFSMTP_RCPT_TYPE_CC, $3, $4); free($3); free($4); } + | KW_BCC '(' string ')' { afsmtp_dd_add_rcpt(last_driver, AFSMTP_RCPT_TYPE_BCC, $3, $3); free($3); } + | KW_BCC '(' string string ')' { afsmtp_dd_add_rcpt(last_driver, AFSMTP_RCPT_TYPE_BCC, $3, $4); free($3); free($4); } + | KW_REPLY_TO '(' string ')' { afsmtp_dd_add_rcpt(last_driver, AFSMTP_RCPT_TYPE_REPLY_TO, $3, $3); free($3); } + | KW_REPLY_TO '(' string string ')' { afsmtp_dd_add_rcpt(last_driver, AFSMTP_RCPT_TYPE_REPLY_TO, $3, $4); free($3); free($4); } + | dest_driver_option + ; + +/* INCLUDE_RULES */ + +%% diff --git a/modules/afsmtp/afsmtp-parser.c b/modules/afsmtp/afsmtp-parser.c new file mode 100644 index 0000000..f190e2f --- /dev/null +++ b/modules/afsmtp/afsmtp-parser.c @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2011-2012 BalaBit IT Ltd, Budapest, Hungary + * Copyright (c) 2011-2012 Gergely Nagy <algernon(a)balabit.hu> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As an additional exemption you are allowed to compile & link against the + * OpenSSL libraries as published by the OpenSSL project. See the file + * COPYING for details. + * + */ + +#include "afsmtp.h" +#include "cfg-parser.h" +#include "afsmtp-grammar.h" + +extern int afsmtp_debug; +int afsmtp_parse(CfgLexer *lexer, LogDriver **instance); + +static CfgLexerKeyword afsmtp_keywords[] = { + { "smtp", KW_SMTP }, + { "host", KW_HOST }, + { "port", KW_PORT }, + { "subject", KW_SUBJECT }, + { "from", KW_FROM }, + { "to", KW_TO }, + { "cc", KW_CC }, + { "bcc", KW_BCC }, + { "reply_to", KW_REPLY_TO }, + { "sender", KW_SENDER }, + { "body", KW_BODY }, + { "header", KW_HEADER }, + { NULL } +}; + +CfgParser afsmtp_parser = +{ +#if ENABLE_DEBUG + .debug_flag = &afsmtp_debug, +#endif + .name = "afsmtp", + .keywords = afsmtp_keywords, + .parse = (int (*)(CfgLexer *lexer, gpointer *instance, gpointer)) afsmtp_parse, + .cleanup = (void (*)(gpointer)) log_pipe_unref, +}; + +CFG_PARSER_IMPLEMENT_LEXER_BINDING(afsmtp_, LogDriver **) diff --git a/modules/afsmtp/afsmtp-parser.h b/modules/afsmtp/afsmtp-parser.h new file mode 100644 index 0000000..c178db0 --- /dev/null +++ b/modules/afsmtp/afsmtp-parser.h @@ -0,0 +1,35 @@ +/* + * Copyright (c) 2011-2012 BalaBit IT Ltd, Budapest, Hungary + * Copyright (c) 2011-2012 Gergely Nagy <algernon(a)balabit.hu> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As an additional exemption you are allowed to compile & link against the + * OpenSSL libraries as published by the OpenSSL project. See the file + * COPYING for details. + * + */ + +#ifndef AFSMTP_PARSER_H_INCLUDED +#define AFSMTP_PARSER_H_INCLUDED + +#include "cfg-parser.h" +#include "cfg-lexer.h" +#include "afsmtp.h" + +extern CfgParser afsmtp_parser; + +CFG_PARSER_DECLARE_LEXER_BINDING(afsmtp_, LogDriver **) + +#endif diff --git a/modules/afsmtp/afsmtp.c b/modules/afsmtp/afsmtp.c new file mode 100644 index 0000000..bd8adfe --- /dev/null +++ b/modules/afsmtp/afsmtp.c @@ -0,0 +1,728 @@ +/* + * Copyright (c) 2011-2012 BalaBit IT Ltd, Budapest, Hungary + * Copyright (c) 2011-2012 Gergely Nagy <algernon(a)balabit.hu> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As an additional exemption you are allowed to compile & link against the + * OpenSSL libraries as published by the OpenSSL project. See the file + * COPYING for details. + * + */ + +#include <signal.h> + +#include "afsmtp.h" +#include "afsmtp-parser.h" +#include "plugin.h" +#include "messages.h" +#include "misc.h" +#include "stats.h" +#include "logqueue.h" + +#include <libesmtp.h> + +#ifndef SCS_SMTP +#define SCS_SMTP 0 +#endif + +typedef struct +{ + gchar *name; + gchar *template; + LogTemplate *value; +} AFSMTPHeader; + +typedef struct +{ + gchar *phrase; + gchar *address; + afsmtp_rcpt_type_t type; +} AFSMTPRecipient; + +typedef struct +{ + LogDestDriver super; + + /* Shared between main/writer; only read by the writer, never + written */ + gchar *host; + gint port; + + gchar *subject; + AFSMTPRecipient *mail_from; + GList *rcpt_tos; + GList *headers; + gchar *body; + + time_t time_reopen; + + StatsCounterItem *dropped_messages; + StatsCounterItem *stored_messages; + + LogTemplate *subject_tmpl; + LogTemplate *body_tmpl; + + /* Thread related stuff; shared */ + GThread *writer_thread; + GMutex *queue_mutex; + GMutex *suspend_mutex; + GCond *writer_thread_wakeup_cond; + + gboolean writer_thread_terminate; + gboolean writer_thread_suspended; + GTimeVal writer_thread_suspend_target; + + LogQueue *queue; + + /* Writer-only stuff */ + gint32 seq_num; + GString *str; +} AFSMTPDriver; + +static gchar * +afsmtp_wash_string (gchar *str) +{ + gint i; + + for (i = 0; i < strlen (str); i++) + if (str[i] == '\n' || + str[i] == '\r') + str[i] = ' '; + + return str; +} + +/* + * Configuration + */ + +void +afsmtp_dd_set_host(LogDriver *d, const gchar *host) +{ + AFSMTPDriver *self = (AFSMTPDriver *)d; + + g_free(self->host); + self->host = g_strdup (host); +} + +void +afsmtp_dd_set_port(LogDriver *d, gint port) +{ + AFSMTPDriver *self = (AFSMTPDriver *)d; + + self->port = (int)port; +} + +void +afsmtp_dd_set_subject(LogDriver *d, const gchar *subject) +{ + AFSMTPDriver *self = (AFSMTPDriver *)d; + + g_free(self->subject); + self->subject = g_strdup(subject); +} + +void +afsmtp_dd_set_from(LogDriver *d, const gchar *phrase, const gchar *mbox) +{ + AFSMTPDriver *self = (AFSMTPDriver *)d; + + g_free(self->mail_from->phrase); + g_free(self->mail_from->address); + self->mail_from->phrase = afsmtp_wash_string(g_strdup(phrase)); + self->mail_from->address = afsmtp_wash_string(g_strdup(mbox)); +} + +void +afsmtp_dd_add_rcpt(LogDriver *d, afsmtp_rcpt_type_t type, const gchar *phrase, + const gchar *mbox) +{ + AFSMTPDriver *self = (AFSMTPDriver *)d; + AFSMTPRecipient *rcpt; + + rcpt = g_new0(AFSMTPRecipient, 1); + rcpt->phrase = afsmtp_wash_string(g_strdup(phrase)); + rcpt->address = afsmtp_wash_string(g_strdup(mbox)); + rcpt->type = type; + + self->rcpt_tos = g_list_append(self->rcpt_tos, rcpt); +} + +void +afsmtp_dd_set_body(LogDriver *d, const gchar *body) +{ + AFSMTPDriver *self = (AFSMTPDriver *)d; + + g_free(self->body); + self->body = g_strdup(body); +} + +gboolean +afsmtp_dd_add_header(LogDriver *d, const gchar *header, const gchar *value) +{ + AFSMTPDriver *self = (AFSMTPDriver *)d; + AFSMTPHeader *h; + + if (!g_ascii_strcasecmp(header, "to") || + !g_ascii_strcasecmp(header, "cc") || + !g_ascii_strcasecmp(header, "bcc") || + !g_ascii_strcasecmp(header, "from") || + !g_ascii_strcasecmp(header, "sender") || + !g_ascii_strcasecmp(header, "reply-to") || + !g_ascii_strcasecmp(header, "date")) + return FALSE; + + h = g_new0(AFSMTPHeader, 1); + h->name = g_strdup(header); + h->template = g_strdup(value); + + self->headers = g_list_append(self->headers, h); + + return TRUE; +} + +/* + * Utilities + */ +void +ignore_sigpipe (void) +{ + struct sigaction sa; + + sa.sa_handler = SIG_IGN; + sigemptyset(&sa.sa_mask); + sa.sa_flags = 0; + sigaction(SIGPIPE, &sa, NULL); +} + +static gchar * +afsmtp_dd_format_stats_instance(AFSMTPDriver *self) +{ + static gchar persist_name[1024]; + + g_snprintf(persist_name, sizeof(persist_name), + "smtp,%s,%u", self->host, self->port); + return persist_name; +} + +static void +afsmtp_dd_suspend(AFSMTPDriver *self) +{ + self->writer_thread_suspended = TRUE; + g_get_current_time(&self->writer_thread_suspend_target); + g_time_val_add(&self->writer_thread_suspend_target, + self->time_reopen * 1000000); +} + +/* + * Worker thread + */ +static void +afsmtp_dd_msg_add_recipient(AFSMTPRecipient *rcpt, smtp_message_t message) +{ + gchar *hdr; + + smtp_add_recipient(message, rcpt->address); + + switch (rcpt->type) + { + case AFSMTP_RCPT_TYPE_TO: + hdr = "To"; + break; + case AFSMTP_RCPT_TYPE_CC: + hdr = "Cc"; + break; + case AFSMTP_RCPT_TYPE_REPLY_TO: + hdr = "Reply-To"; + break; + default: + return; + } + smtp_set_header(message, hdr, rcpt->phrase, rcpt->address); + smtp_set_header_option(message, hdr, Hdr_OVERRIDE, 1); +} + +static void +afsmtp_dd_msg_add_header(AFSMTPHeader *hdr, gpointer user_data) +{ + AFSMTPDriver *self = ((gpointer *)user_data)[0]; + LogMessage *msg = ((gpointer *)user_data)[1]; + smtp_message_t message = ((gpointer *)user_data)[2]; + + log_template_format(hdr->value, msg, NULL, LTZ_SEND, self->seq_num, NULL, self->str); + + smtp_set_header(message, hdr->name, afsmtp_wash_string (self->str->str), NULL); + smtp_set_header_option(message, hdr->name, Hdr_OVERRIDE, 1); +} + +static void +afsmtp_dd_log_rcpt_status(smtp_recipient_t rcpt, const char *mailbox, + gpointer user_data) +{ + const smtp_status_t *status; + + status = smtp_recipient_status(rcpt); + msg_debug("SMTP recipient result", + evt_tag_str("recipient", mailbox), + evt_tag_int("code", status->code), + evt_tag_str("text", status->text), + NULL); +} + +static void +afsmtp_dd_cb_event(smtp_session_t session, int event, AFSMTPDriver *self) +{ + switch (event) + { + case SMTP_EV_CONNECT: + msg_verbose("Connected to SMTP server", + evt_tag_str("host", self->host), + evt_tag_int("port", self->port), + NULL); + break; + case SMTP_EV_MAILSTATUS: + case SMTP_EV_RCPTSTATUS: + case SMTP_EV_MESSAGEDATA: + case SMTP_EV_MESSAGESENT: + /* Ignore */ + break; + case SMTP_EV_DISCONNECT: + msg_verbose("Disconnected from SMTP server", + evt_tag_str("host", self->host), + evt_tag_int("port", self->port), + NULL); + break; + default: + msg_verbose("Unknown SMTP event", + evt_tag_int("event_id", event), + NULL); + break; + } +} + +static void +afsmtp_dd_cb_monitor(const gchar *buf, gint buflen, gint writing, + AFSMTPDriver *self) +{ + gchar fmt[32]; + + g_snprintf(fmt, sizeof(fmt), "%%.%us", buflen); + + switch (writing) + { + case SMTP_CB_READING: + msg_debug ("SMTP Session: SERVER", + evt_tag_printf("message", fmt, buf), + NULL); + break; + case SMTP_CB_WRITING: + msg_debug("SMTP Session: CLIENT", + evt_tag_printf("message", fmt, buf), + NULL); + break; + case SMTP_CB_HEADERS: + msg_debug("SMTP Session: HEADERS", + evt_tag_printf("data", fmt, buf), + NULL); + break; + } +} + +static gboolean +afsmtp_worker_insert(AFSMTPDriver *self) +{ + gboolean success; + LogMessage *msg; + LogPathOptions path_options = LOG_PATH_OPTIONS_INIT; + smtp_session_t session; + smtp_message_t message; + gpointer args[] = { self, NULL, NULL }; + + g_mutex_lock(self->queue_mutex); + log_queue_reset_parallel_push(self->queue); + success = log_queue_pop_head(self->queue, &msg, &path_options, FALSE, FALSE); + g_mutex_unlock(self->queue_mutex); + if (!success) + return TRUE; + + msg_set_context(msg); + + session = smtp_create_session(); + message = smtp_add_message(session); + + g_string_printf(self->str, "%s:%d", self->host, self->port); + smtp_set_server(session, self->str->str); + + smtp_set_eventcb(session, (smtp_eventcb_t)afsmtp_dd_cb_event, (void *)self); + smtp_set_monitorcb(session, (smtp_monitorcb_t)afsmtp_dd_cb_monitor, + (void *)self, 1); + + smtp_set_reverse_path(message, self->mail_from->address); + + /* Defaults */ + smtp_set_header(message, "To", NULL, NULL); + smtp_set_header(message, "From", NULL, NULL); + + log_template_format(self->subject_tmpl, msg, NULL, LTZ_SEND, + self->seq_num, NULL, self->str); + smtp_set_header(message, "Subject", afsmtp_wash_string(self->str->str)); + smtp_set_header_option(message, "Subject", Hdr_OVERRIDE, 1); + + /* Add recipients */ + g_list_foreach(self->rcpt_tos, (GFunc)afsmtp_dd_msg_add_recipient, message); + + /* Add custom header (overrides anything set before, or in the + body). */ + args[1] = msg; + args[2] = message; + g_list_foreach(self->headers, (GFunc)afsmtp_dd_msg_add_header, args); + + /* Set the body. + * + * We add a header to the body, otherwise libesmtp will not + * recognise headers, and will append them to the end of the body. + */ + g_string_assign(self->str, "X-Mailer: syslog-ng " VERSION "\r\n\r\n"); + log_template_append_format(self->body_tmpl, msg, NULL, LTZ_SEND, + self->seq_num, NULL, self->str); + smtp_set_message_str(message, self->str->str); + + if (!smtp_start_session(session)) + { + gchar error[1024]; + smtp_strerror(smtp_errno(), error, sizeof (error) - 1); + + msg_error("SMTP server error, suspending", + evt_tag_str("error", error), + evt_tag_int("time_reopen", self->time_reopen), + NULL); + success = FALSE; + } + else + { + const smtp_status_t *status = smtp_message_transfer_status(message); + msg_debug("SMTP result", + evt_tag_int("code", status->code), + evt_tag_str("text", status->text), + NULL); + smtp_enumerate_recipients(message, afsmtp_dd_log_rcpt_status, NULL); + } + smtp_destroy_session(session); + + msg_set_context(NULL); + + if (success) + { + stats_counter_inc(self->stored_messages); + step_sequence_number(&self->seq_num); + log_msg_ack(msg, &path_options); + log_msg_unref(msg); + } + else + { + g_mutex_lock(self->queue_mutex); + log_queue_push_head(self->queue, msg, &path_options); + g_mutex_unlock(self->queue_mutex); + } + + return success; +} + +static gpointer +afsmtp_worker_thread(gpointer arg) +{ + AFSMTPDriver *self = (AFSMTPDriver *)arg; + + msg_debug("Worker thread started", + evt_tag_str("driver", self->super.super.id), + NULL); + + self->str = g_string_sized_new(1024); + + ignore_sigpipe(); + + while (!self->writer_thread_terminate) + { + g_mutex_lock(self->suspend_mutex); + if (self->writer_thread_suspended) + { + g_cond_timed_wait(self->writer_thread_wakeup_cond, + self->suspend_mutex, + &self->writer_thread_suspend_target); + self->writer_thread_suspended = FALSE; + g_mutex_unlock(self->suspend_mutex); + } + else + { + g_mutex_unlock(self->suspend_mutex); + + g_mutex_lock(self->queue_mutex); + if (log_queue_get_length(self->queue) == 0) + { + g_cond_wait(self->writer_thread_wakeup_cond, self->queue_mutex); + } + g_mutex_unlock(self->queue_mutex); + } + + if (self->writer_thread_terminate) + break; + + if (!afsmtp_worker_insert (self)) + { + afsmtp_dd_suspend(self); + } + } + + g_string_free(self->str, TRUE); + + msg_debug("Worker thread finished", + evt_tag_str("driver", self->super.super.id), + NULL); + + return NULL; +} + +/* + * Main thread + */ + +static void +afsmtp_dd_start_thread(AFSMTPDriver *self) +{ + self->writer_thread = create_worker_thread(afsmtp_worker_thread, self, TRUE, NULL); +} + +static void +afsmtp_dd_stop_thread(AFSMTPDriver *self) +{ + self->writer_thread_terminate = TRUE; + g_mutex_lock(self->queue_mutex); + g_cond_signal(self->writer_thread_wakeup_cond); + g_mutex_unlock(self->queue_mutex); + g_thread_join(self->writer_thread); +} + +static void +afsmtp_dd_init_header(AFSMTPHeader *hdr, GlobalConfig *cfg) +{ + if (!hdr->value) + { + hdr->value = log_template_new(cfg, hdr->name); + log_template_compile(hdr->value, hdr->template, NULL); + } +} + +static gboolean +afsmtp_dd_init(LogPipe *s) +{ + AFSMTPDriver *self = (AFSMTPDriver *)s; + GlobalConfig *cfg = log_pipe_get_config(s); + + if (cfg) + self->time_reopen = cfg->time_reopen; + + msg_verbose("Initializing SMTP destination", + evt_tag_str("host", self->host), + evt_tag_int("port", self->port), + NULL); + + self->queue = log_dest_driver_acquire_queue(&self->super, afsmtp_dd_format_stats_instance(self)); + + g_list_foreach(self->headers, (GFunc)afsmtp_dd_init_header, cfg); + if (!self->subject_tmpl) + { + self->subject_tmpl = log_template_new(cfg, "subject"); + log_template_compile(self->subject_tmpl, self->subject, NULL); + } + if (!self->body_tmpl) + { + self->body_tmpl = log_template_new(cfg, "body"); + log_template_compile(self->body_tmpl, self->body, NULL); + } + + stats_lock(); + stats_register_counter(0, SCS_SMTP | SCS_DESTINATION, self->super.super.id, + afsmtp_dd_format_stats_instance(self), + SC_TYPE_STORED, &self->stored_messages); + stats_register_counter(0, SCS_SMTP | SCS_DESTINATION, self->super.super.id, + afsmtp_dd_format_stats_instance(self), + SC_TYPE_DROPPED, &self->dropped_messages); + stats_unlock(); + + afsmtp_dd_start_thread(self); + + return TRUE; +} + +static gboolean +afsmtp_dd_deinit(LogPipe *s) +{ + AFSMTPDriver *self = (AFSMTPDriver *)s; + + afsmtp_dd_stop_thread(self); + + stats_lock(); + stats_unregister_counter(SCS_SMTP | SCS_DESTINATION, self->super.super.id, + afsmtp_dd_format_stats_instance(self), + SC_TYPE_STORED, &self->stored_messages); + stats_unregister_counter(SCS_SMTP | SCS_DESTINATION, self->super.super.id, + afsmtp_dd_format_stats_instance(self), + SC_TYPE_DROPPED, &self->dropped_messages); + stats_unlock(); + + return TRUE; +} + +static void +afsmtp_dd_free(LogPipe *d) +{ + AFSMTPDriver *self = (AFSMTPDriver *)d; + GList *l; + + g_mutex_free(self->suspend_mutex); + g_mutex_free(self->queue_mutex); + g_cond_free(self->writer_thread_wakeup_cond); + + if (self->queue) + log_queue_unref(self->queue); + + g_free(self->host); + g_free(self->mail_from->phrase); + g_free(self->mail_from->address); + g_free(self->mail_from); + log_template_unref(self->subject_tmpl); + log_template_unref(self->body_tmpl); + g_free(self->body); + g_free(self->subject); + g_string_free(self->str, TRUE); + + l = self->rcpt_tos; + while (l) + { + AFSMTPRecipient *rcpt = (AFSMTPRecipient *)l->data; + g_free(rcpt->address); + g_free(rcpt->phrase); + g_free(rcpt); + l = g_list_delete_link(l, l); + } + + l = self->headers; + while (l) + { + AFSMTPHeader *hdr = (AFSMTPHeader *)l->data; + g_free(hdr->name); + g_free(hdr->template); + log_template_unref(hdr->value); + g_free(hdr); + l = g_list_delete_link(l, l); + } + + log_dest_driver_free(d); +} + +static void +afsmtp_dd_queue_notify(gpointer s) +{ + AFSMTPDriver *self = (AFSMTPDriver *)s; + + g_mutex_lock(self->queue_mutex); + g_cond_signal(self->writer_thread_wakeup_cond); + log_queue_reset_parallel_push(self->queue); + g_mutex_unlock(self->queue_mutex); +} + +static void +afsmtp_dd_queue(LogPipe *s, LogMessage *msg, + const LogPathOptions *path_options, gpointer user_data) +{ + AFSMTPDriver *self = (AFSMTPDriver *)s; + gboolean queue_was_empty; + LogPathOptions local_options; + + if (!path_options->flow_control_requested) + path_options = log_msg_break_ack(msg, path_options, &local_options); + + g_mutex_lock(self->queue_mutex); + queue_was_empty = log_queue_get_length(self->queue) == 0; + g_mutex_unlock(self->queue_mutex); + + log_msg_add_ack(msg, path_options); + log_queue_push_tail(self->queue, log_msg_ref(msg), path_options); + + g_mutex_lock(self->suspend_mutex); + if (queue_was_empty && !self->writer_thread_suspended) + { + g_mutex_lock(self->queue_mutex); + log_queue_set_parallel_push(self->queue, 1, afsmtp_dd_queue_notify, self, NULL); + g_mutex_unlock(self->queue_mutex); + } + g_mutex_unlock(self->suspend_mutex); + log_dest_driver_queue_method(s, msg, path_options, user_data); +} + +/* + * Plugin glue. + */ + +LogDriver * +afsmtp_dd_new(void) +{ + AFSMTPDriver *self = g_new0(AFSMTPDriver, 1); + + log_dest_driver_init_instance(&self->super); + self->super.super.super.init = afsmtp_dd_init; + self->super.super.super.deinit = afsmtp_dd_deinit; + self->super.super.super.queue = afsmtp_dd_queue; + self->super.super.super.free_fn = afsmtp_dd_free; + + afsmtp_dd_set_host((LogDriver *)self, "127.0.0.1"); + afsmtp_dd_set_port((LogDriver *)self, 25); + + self->mail_from = g_new0(AFSMTPRecipient, 1); + + init_sequence_number(&self->seq_num); + + self->writer_thread_wakeup_cond = g_cond_new(); + self->suspend_mutex = g_mutex_new(); + self->queue_mutex = g_mutex_new(); + + return (LogDriver *)self; +} + +extern CfgParser afsmtp_dd_parser; + +static Plugin afsmtp_plugin = +{ + .type = LL_CONTEXT_DESTINATION, + .name = "smtp", + .parser = &afsmtp_parser, +}; + +gboolean +afsmtp_module_init(GlobalConfig *cfg, CfgArgs *args) +{ + plugin_register(cfg, &afsmtp_plugin, 1); + return TRUE; +} + +const ModuleInfo module_info = +{ + .canonical_name = "afsmtp", + .version = VERSION, + .description = "The afsmtp module provides SMTP destination support for syslog-ng.", + .core_revision = SOURCE_REVISION, + .plugins = &afsmtp_plugin, + .plugins_len = 1, +}; diff --git a/modules/afsmtp/afsmtp.h b/modules/afsmtp/afsmtp.h new file mode 100644 index 0000000..75c70c5 --- /dev/null +++ b/modules/afsmtp/afsmtp.h @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2011-2012 BalaBit IT Ltd, Budapest, Hungary + * Copyright (c) 2011-2012 Gergely Nagy <algernon(a)balabit.hu> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + * As an additional exemption you are allowed to compile & link against the + * OpenSSL libraries as published by the OpenSSL project. See the file + * COPYING for details. + * + */ + +#ifndef AFSMTP_H_INCLUDED +#define AFSMTP_H_INCLUDED + +#include "driver.h" + +typedef enum + { + AFSMTP_RCPT_TYPE_NONE, + AFSMTP_RCPT_TYPE_TO, + AFSMTP_RCPT_TYPE_CC, + AFSMTP_RCPT_TYPE_BCC, + AFSMTP_RCPT_TYPE_REPLY_TO, + AFSMTP_RCPT_TYPE_SENDER, + } afsmtp_rcpt_type_t; + +LogDriver *afsmtp_dd_new(void); + +void afsmtp_dd_set_host(LogDriver *d, const gchar *host); +void afsmtp_dd_set_port(LogDriver *d, gint port); + +void afsmtp_dd_set_subject(LogDriver *d, const gchar *subject); +void afsmtp_dd_set_from(LogDriver *d, const gchar *phrase, const gchar *mbox); +void afsmtp_dd_add_rcpt(LogDriver *d, afsmtp_rcpt_type_t type, + const gchar *phrase, const gchar *mbox); +void afsmtp_dd_set_body(LogDriver *d, const gchar *body); +gboolean afsmtp_dd_add_header(LogDriver *d, const gchar *header, + const gchar *value); + +#endif -- 1.7.8.3

2 2

[PATCH (3.4) 0/2]: tfuuid: New template function to generate UUIDs.
by Gergely Nagy 03 Feb '12

03 Feb '12

Following this mail, two patches will be sent: one that moves the uuid implementation from patternize to lib/uuid.c, and extends it to be able to use libuuid if OpenSSL was not found. And another, that implements a very simple template function, that can be used to generate UUIDs. It uses the wrapper introduced in the first patch. I'm not sure tfuuid warrants an entirely separate module, though: it's so trivial it could easily fit into basicfuncs, for example. If that's more desirable, I'll send an updated version that puts it there, instead of a separate module. -- |8]

1 2

Format-json scope
by Vámos Balázs 03 Feb '12

03 Feb '12

Hi, Version: syslog-ng ose 3.3.3 Error message in log: error in template: $(format-json --scope soft-macros) Config: destination d_json{ file("/var/log/messages.json" template("$(format-json --scope soft-macros)\n") ); }; The 'all-macros' and 'selected-macros' are working. Thanks, Balazs

2 4

[PATCH (3.4)] tfuuid: New template function to generate UUIDs.
by Gergely Nagy 03 Feb '12

03 Feb '12

This patch adds a new template function: $(UUID) (also available as $(uuid)), which can be used to generate Universally Unique IDentifiers. The use case for this is to add an UUID to each message as soon as they're received, so that they can be identified later, accross destinations. This allows me to pre-generate an ID for messages going into mongodb, and send those messages to a different destination too, with the same ID included. Use it like this: rewrite r_add_uuid { set("$(UUID)" value("UUID")); }; log default { source(s_network); rewrite(r_add_uuid); destination(d_mongo); destination(d_program); }; The module uses the widely available libuuid library to generate the UUIDs. Signed-off-by: Gergely Nagy <algernon(a)balabit.hu> --- configure.in | 16 ++++++++++- modules/Makefile.am | 2 +- modules/tfuuid/Makefile.am | 11 ++++++++ modules/tfuuid/tfuuid.c | 62 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 89 insertions(+), 2 deletions(-) create mode 100644 modules/tfuuid/Makefile.am create mode 100644 modules/tfuuid/tfuuid.c diff --git a/configure.in b/configure.in index 734d703..83b81ac 100644 --- a/configure.in +++ b/configure.in @@ -182,6 +182,10 @@ AC_ARG_ENABLE(systemd, [ --enable-systemd Enable systemd support (default: auto)] ,,enable_systemd="auto") +AC_ARG_ENABLE(uuid, + [ --enable-uuid Enable UUID support (default: auto)] + ,,enable_uuid="auto") + AC_ARG_WITH(compile-date, [ --without-compile-date Do not include the compile date in the binary] ,wcmp_date="${withval}", wcmp_date="yes") @@ -930,6 +934,14 @@ if test "x$enable_systemd" = "xyes"; then fi fi +if test "x$enable_uuid" = "xyes" || test "x$enable_uuid" = "xauto"; then + PKG_CHECK_MODULES(UUID, uuid, with_uuid="yes", with_uuid="no") + if test "x$with_uuid" = "xno" && test "x$enable_uuid" = "xyes"; then + AC_MSG_ERROR([Could not find libuuid, and uuid support was explicitly enabled.]) + fi + enable_uuid="$with_uuid" +fi + dnl *************************************************************************** dnl default modules to be loaded dnl *************************************************************************** @@ -1087,6 +1099,7 @@ AM_CONDITIONAL(ENABLE_PACCT, [test "$enable_pacct" = "yes"]) AM_CONDITIONAL(ENABLE_MONGODB, [test "$enable_mongodb" = "yes"]) AM_CONDITIONAL(ENABLE_JSON_FORMAT, [test "$enable_json_format" = "yes"]) AM_CONDITIONAL(ENABLE_JSON_PARSE, [test "$enable_json_parse" = "yes"]) +AM_CONDITIONAL(ENABLE_UUID, [test "$enable_uuid" = "yes"]) AM_CONDITIONAL(WITH_LIBSYSTEMD, [test "$with_libsystemd" = "yes"]) # substitution into manual pages @@ -1153,6 +1166,7 @@ AC_OUTPUT(dist.conf modules/pacctformat/Makefile modules/basicfuncs/Makefile modules/tfjson/Makefile + modules/tfuuid/Makefile modules/jsonparser/Makefile scripts/Makefile scripts/update-patterndb @@ -1204,5 +1218,5 @@ echo " SQL support (module) : ${enable_sql:=no}" echo " PACCT module (EXPERIMENTAL) : ${enable_pacct:=no}" echo " MongoDB destination (module): ${enable_mongodb:=no}" echo " JSON support (module) : parser=${enable_json_parse:=no}, formatter=${enable_json_format:=no} (using ${with_json})" - +echo " UUID support (module) : ${enable_uuid:=no}" diff --git a/modules/Makefile.am b/modules/Makefile.am index 0d0594b..72c005f 100644 --- a/modules/Makefile.am +++ b/modules/Makefile.am @@ -1 +1 @@ -SUBDIRS = afsocket afsql afstreams affile afprog afuser afmongodb csvparser confgen syslogformat pacctformat basicfuncs dbparser tfjson jsonparser dummy +SUBDIRS = afsocket afsql afstreams affile afprog afuser afmongodb csvparser confgen syslogformat pacctformat basicfuncs dbparser tfjson tfuuid jsonparser dummy diff --git a/modules/tfuuid/Makefile.am b/modules/tfuuid/Makefile.am new file mode 100644 index 0000000..9d96fe1 --- /dev/null +++ b/modules/tfuuid/Makefile.am @@ -0,0 +1,11 @@ +moduledir = @moduledir@ +export top_srcdir + +#if ENABLE_UUID +AM_CPPFLAGS = -I$(top_srcdir)/lib -I../../lib $(UUID_CFLAGS) +module_LTLIBRARIES = libtfuuid.la + +libtfuuid_la_SOURCES = tfuuid.c +libtfuuid_la_LIBADD = $(MODULE_DEPS_LIBS) $(UUID_LIBS) +libtfuuid_la_LDFLAGS = $(MODULE_LDFLAGS) +#endif diff --git a/modules/tfuuid/tfuuid.c b/modules/tfuuid/tfuuid.c new file mode 100644 index 0000000..c426cb8 --- /dev/null +++ b/modules/tfuuid/tfuuid.c @@ -0,0 +1,62 @@ +/* + * Copyright (c) 2012 BalaBit IT Ltd, Budapest, Hungary + * Copyright (c) 2012 Gergely Nagy <algernon(a)balabit.hu> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "plugin.h" +#include "templates.h" +#include "cfg.h" + +#include "config.h" + +#include <uuid/uuid.h> + +static void +tf_uuid(LogMessage *msg, gint argc, GString *argv[], GString *result) +{ + uuid_t uuid; + gint i; + + uuid_generate (uuid); + + for (i = 0; i < sizeof (uuid); i++) + g_string_append_printf (result, "%02x", uuid[i]); +} + +TEMPLATE_FUNCTION_SIMPLE(tf_uuid); + +static Plugin tfuuid_plugins[] = + { + TEMPLATE_FUNCTION_PLUGIN(tf_uuid, "UUID"), + TEMPLATE_FUNCTION_PLUGIN(tf_uuid, "uuid"), + }; + +gboolean +tfuuid_module_init(GlobalConfig *cfg, CfgArgs *args) +{ + plugin_register(cfg, tfuuid_plugins, G_N_ELEMENTS(tfuuid_plugins)); + return TRUE; +} + +const ModuleInfo module_info = +{ + .canonical_name = "tfuuid", + .version = VERSION, + .description = "The tfuuid module provides a template function to generate UUIDs.", + .core_revision = SOURCE_REVISION, + .plugins = tfuuid_plugins, + .plugins_len = G_N_ELEMENTS(tfuuid_plugins), +}; -- 1.7.7.3

2 4

[PATCH (3.4)] value-pairs: Do not require --rekey in the command-line parser.
by Gergely Nagy 03 Feb '12

03 Feb '12

The --rekey in the commandline parser is not needed, the various transformation functions can work just fine without it, and do the setup themselves, if so need be. This patch gets rid of --rekey. Signed-off-by: Gergely Nagy <algernon(a)balabit.hu> --- lib/value-pairs.c | 44 ++++++++++++++++++++++++++------------------ 1 files changed, 26 insertions(+), 18 deletions(-) diff --git a/lib/value-pairs.c b/lib/value-pairs.c index 6ecd810..59380f5 100644 --- a/lib/value-pairs.c +++ b/lib/value-pairs.c @@ -1,6 +1,6 @@ /* - * Copyright (c) 2011 BalaBit IT Ltd, Budapest, Hungary - * Copyright (c) 2011 Gergely Nagy <algernon(a)balabit.hu> + * Copyright (c) 2011-2012 BalaBit IT Ltd, Budapest, Hungary + * Copyright (c) 2011-2012 Gergely Nagy <algernon(a)balabit.hu> * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -506,33 +506,39 @@ vp_cmdline_parse_pair (const gchar *option_name, const gchar *value, return TRUE; } -static gboolean -vp_cmdline_parse_rekey (const gchar *option_name, const gchar *value, - gpointer data, GError **error) +static ValuePairsTransformSet * +vp_cmdline_rekey_verify (gchar *key, ValuePairsTransformSet *vpts, + gpointer data) { - gpointer *args = (gpointer *) data; - ValuePairsTransformSet *vpts = (ValuePairsTransformSet *) args[2]; - gchar *key = (gchar *) args[3]; + gpointer *args = (gpointer *)data; - vpts = value_pairs_transform_set_new (key); - vp_cmdline_parse_rekey_finish (data); - - args[2] = vpts; - return TRUE; + if (!vpts) + { + if (!key) + return NULL; + vpts = value_pairs_transform_set_new (key); + vp_cmdline_parse_rekey_finish (data); + args[2] = vpts; + return vpts; + } + return vpts; } + static gboolean vp_cmdline_parse_rekey_replace (const gchar *option_name, const gchar *value, gpointer data, GError **error) { gpointer *args = (gpointer *) data; ValuePairsTransformSet *vpts = (ValuePairsTransformSet *) args[2]; + gchar *key = (gchar *) args[3]; gchar **kv; + vpts = vp_cmdline_rekey_verify (key, vpts, data); if (!vpts) { g_set_error (error, G_OPTION_ERROR, G_OPTION_ERROR_FAILED, - "Error parsing value-pairs: --replace used without --rekey"); + "Error parsing value-pairs: --replace used without --key"); return FALSE; } @@ -560,11 +566,13 @@ vp_cmdline_parse_rekey_add_prefix (const gchar *option_name, const gchar *value, { gpointer *args = (gpointer *) data; ValuePairsTransformSet *vpts = (ValuePairsTransformSet *) args[2]; + gchar *key = (gchar *) args[3]; + vpts = vp_cmdline_rekey_verify (key, vpts, data); if (!vpts) { g_set_error (error, G_OPTION_ERROR, G_OPTION_ERROR_FAILED, - "Error parsing value-pairs: --add-prefix used without --rekey"); + "Error parsing value-pairs: --add-prefix used without --key"); return FALSE; } @@ -579,11 +587,13 @@ vp_cmdline_parse_rekey_shift (const gchar *option_name, const gchar *value, { gpointer *args = (gpointer *) data; ValuePairsTransformSet *vpts = (ValuePairsTransformSet *) args[2]; + gchar *key = (gchar *) args[3]; + vpts = vp_cmdline_rekey_verify (key, vpts, data); if (!vpts) { g_set_error (error, G_OPTION_ERROR, G_OPTION_ERROR_FAILED, - "Error parsing value-pairs: --shift used without --rekey"); + "Error parsing value-pairs: --shift used without --key"); return FALSE; } @@ -609,8 +619,6 @@ value_pairs_new_from_cmdline (GlobalConfig *cfg, NULL, NULL }, { "pair", 'p', 0, G_OPTION_ARG_CALLBACK, vp_cmdline_parse_pair, NULL, NULL }, - { "rekey", 'r', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK, - vp_cmdline_parse_rekey, NULL, NULL }, { "shift", 'S', 0, G_OPTION_ARG_CALLBACK, vp_cmdline_parse_rekey_shift, NULL, NULL }, { "add-prefix", 'A', 0, G_OPTION_ARG_CALLBACK, vp_cmdline_parse_rekey_add_prefix, -- 1.7.8.3

2 2

[PATCH] afmongodb: Fix a deadlock at deinit time.
by Gergely Nagy 03 Feb '12

03 Feb '12

There is a race condition between the writer thread and stop_threads (via deinit): if deinit sends the signal just at the wrong time, then the worker thread willl never wake up, as the main thread is already waiting for a join to finish. The fix is to lock the queue_mutex before signalling, that eliminates the race. This patch does just that. Signed-off-by: Gergely Nagy <algernon(a)balabit.hu> --- modules/afmongodb/afmongodb.c | 6 ++++-- 1 files changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/afmongodb/afmongodb.c b/modules/afmongodb/afmongodb.c index 42c140e..dade669 100644 --- a/modules/afmongodb/afmongodb.c +++ b/modules/afmongodb/afmongodb.c @@ -1,6 +1,6 @@ /* - * Copyright (c) 2002-2011 BalaBit IT Ltd, Budapest, Hungary - * Copyright (c) 2010-2011 Gergely Nagy <algernon(a)balabit.hu> + * Copyright (c) 2010-2012 BalaBit IT Ltd, Budapest, Hungary + * Copyright (c) 2010-2012 Gergely Nagy <algernon(a)balabit.hu> * * This program is free software; you can redistribute it and/or modify it * under the terms of the GNU General Public License version 2 as published @@ -403,7 +403,9 @@ static void afmongodb_dd_stop_thread (MongoDBDestDriver *self) { self->writer_thread_terminate = TRUE; + g_mutex_lock(self->queue_mutex); g_cond_signal(self->writer_thread_wakeup_cond); + g_mutex_unlock(self->queue_mutex); g_thread_join(self->writer_thread); } -- 1.7.8.3

2 1

Bazsi's blog: syslog-ng flexibility improvements
by Balazs Scheidler 03 Feb '12

03 Feb '12

syslog-ng flexibility improvements syslog-ng is often referred as a very flexible application when it comes to processing logs. Over the years however, I began to feel that some things are a bit more difficult to achieve in the configuration language than it should be. For instance it is sometimes too rigid when you need a combination of parsers (patterndb with db-parser) and rewrite rules to achieve the goal you wanted. Parsers and rewrite rules are distinct part of the configuration, it is not possible to combine them into a single functionality. Also, declaring objects first and then referencing them later, makes the configuration easy to read, however sometimes it is quite cumbersome, when you only need to invert the result of an already existing filter. To solve this situation, I’ve set out to implement an idea I had on mind for some time now. It is quite difficult to describe the feature in clear and concise words, as it is a combination of various changes that together makes syslog-ng configuration more flexible and easier to use, without sacrificing readability. Curious? Please read on. In-line objects Perhaps the simplest of all features is that you can now define the contents of a given object right on the spot, without having to use a separate statement. For example, earlier you had to write: log { source(s_local); filter(f_postfix); destination(d_postfix); }; Sometimes, f_postfix filter is only used once and is trivial. This can now be written as: log { source(s_local); filter { program("^postfix/"); }; destination(d_postfix); }; Furthermore both the source() and destination() options can be written in-line, you simply use braces instead of parentheses. The same functionality applies to everything: sources, destinations, filters, parsers and rewrite rules. Junctions A limited form of junctions has been supported since syslog-ng 3.0 in the form of “embedded log statements”, which has been generalized now. Within syslog-ng, when a message is received it is dispatched to a log processing path or pipeline, which carries out the task at hand. A junction is a point in the log processing path where the processing is performed on multiple independent branches, each doing its own specific thing with the message. The limited functionality in 3.0 only allowed the processing tree to split (or fork) into independent branches, each of the branches was a “sink”, where processing also ended. Configuration example: log { source(s_all); filter(f); log { filter(f1); destination(d1); }; log { filter(f2); destination(d2); }; }; This sample forks the processing path into two branches starting with the “log” keyword within the top-level log statement. The first branch evaluates the filter f1 and the writes matching messages to the d1 destination, effectively sending all messages that match (f AND f1) to d1. Likewise, d2 receives all messages that match (f AND f2). The limitation of the embedded log statement concept was simple: it could only be listed at the very end of a log statement, and the end-result of the branches couldn’t be processed further. Effectively the message at the end of each branch “fell off”. Junctions on the other hand makes it possible to do things to messages once the branches converge to the same point again. Repeating the sample above, it is now possible to write: log { source(s_all); filter(f); junction { log { filter(f1); destination(d1); }; log { filter(f2); destination(d2); }; }; destination(d_all); }; The new thing is that you can now add processing after the branches finish their processing. A bit more useful example would be: log { source(s_apache_files); source(s_syslog); junction { log { filter(f_apache_files); rewrite(r_apache_remove_file_header); parser(p_apache); flags(final); }; log { filter(f_apache_syslog); parser(p_apache); flags(final); }; }; destination(d_files); }; This example does an alternative processing of incoming logs based on where the message came from. Everything is a log expression This feature is probably the most complicated, however provides very nice properties and expressiveness to the configuration. From now on, not just the well known log statement allows the specification of log processing rules, but all the objects in the syslog-ng configuration file can use the same expressive power. It is now possible to use embedded log statements, junctions and in-line object definitions within source, destination, filter, rewrite and parser definitions. Huh, you could ask: what does it bring to me as a benefit? Well, until now, objects of different types were separate entities, connected using log statements, with this change a source can also specify a rewrite rule and that combination used as a log source in a log statement. For instance, a usual source definition looked like this: source s_apache { file("/var/log/apache/error.log"); }; If you wanted to process this log file in a specific way, you needed to define the accompanying processing rules (parsers and rewrite expressions) and combine them in a log statement. But how about this: source s_apache { log { source { file("/var/log/apache/error.log"); }; parser(p_apache_parser); }; }; }; log { source(s_apache); ... }; Can you see? The s_apache source used a file source and the reference of a specific parser and all messages read from the apache error log file would be processed by that parser. The log statement is just as simple as if s_apache would be a “normal” source definition. This feature allows pairing the essential log preprocessing functionality very close to the source itself, making it very easy to write and read the log statements. As an added bonus, it becomes very easy to distribute application specific source & parser definitions as an SCL configuration snippet. Where? This stuff is available in the syslog-ng 3.4 git tree, on master. It passes the included regression test, so it is at least dogfoodable. The nice thing about the implementation is that it only slightly increased the code size, but brought a lot of new features. If you have trouble getting the code from git, let me know, I’m willing to create an alpha release, so that it becomes easier to play with it. Feedback I see a lot of potential in this functionality, however my examples may have not been the best ones. I would really appreciate any kind of feedback, please be sure to send those to the syslog-ng mailing list or post me as a private email.

4 13

Patch for tests/loggen.c (bind src addr and full SendQ sleep)
by Russell Tait Milne 03 Feb '12

03 Feb '12

A colleague of mine has patched the loggen utility to allow for: - binding to a particular source address - adding a usleep() when the SendQ is full This patch was applied and tested on syslog-ng-3.3.3 found in the FreeBSD ports tree. I apologize in advance if this is not the proper way to submit a patch. (See attached patch) Best, Russ This communication, including any attachments, does not necessarily represent official policy of Seccuris Inc. Please see http://www.seccuris.com/Contact-PrivacyPolicy.htm for further details about Seccuris Inc.'s Privacy Policy. If you have received this communication in error, please notify Seccuris Inc. at info(a)seccuris.com or at 1-866-644-8442.

2 3

Dumping syslog-ng log files to mysql db
by Anup Shetty 02 Feb '12

02 Feb '12

Hello, With recent addition of MYSQL DB to my logging infrastructure, would I be able to dump my previous logs collected from the by syslog-ng on the database? If I configure the source to be the local log file that has logs collected from devices over the network and the destination to be the DB, would I be loosing the source IP address when I log these to the DB and instead get the source of the message to be localhost or 127.0.0.1 instead of the original source? -- Thanks and regards, Anup

2 1

parsing what gets into syslog from windows machines.
by greg whynott 02 Feb '12

02 Feb '12

Hello, We have all our windows servers/controllers send system messages (using snare) to our central log server which is a linux machine running syslog-ng. They are very verbose, multi line messages. Is there a way to have syslog-ng trim these down before the are wrote out to the log file? Or perhaps you know of another method I should be looking at? Thanks very much. here is a typical windows server message: Feb 1 16:08:31 10.111.1.11 MSWinEventLog 1 Security 5043398 Wed Feb 01 16:08:29 2012 4624 Microsoft-Windows-Security-Auditing NT AUTHORITY\ANONYMOUS LOGON N/A Success Audit AD02.domain.com Logon An account was successfully logged on. Subject: Security ID: S-1-0-0 Account Name: - Account Domain: - Logon ID: 0x0 Logon Type: 3 New Logon: Security ID: S-1-5-7 Account Name: ANONYMOUS LOGON Account Domain: NT AUTHORITY Logon ID: 0x45ab55d Logon GUID: {00000000-0000-0000-0000-000000000000} Process Information: Process ID: 0x0 Process Name: - Network Information: Workstation Name: AC010 Source Network Address: 10.101.2.44 Source Port: 60711 Detailed Authentication Information: Logon Process: NtLmSsp Authentication Package: NTLM Transited Services: - Package Name (NTLM only): NTLM V1 Key Length: 128 This event is generated when a logon session is created. It is generated on the computer that was accessed. The subject fields indicate the account on the local system which requested the logon. This is most commonly a service such as the Server service, or a local process such as Winlogon.exe or Services.exe. The logon type field indicates the kind of logon that occurred. The most common types are 2 (interactive) and 3 (network). The New Logon fields indicate the account for whom the new logon was created, i.e. the account that was logged on. The network fields indicate where a remote logon request originated. Workstation name is not always available and may be left blank in some cases. The authentication information fields provide detailed information about this specific logon request. - Logon GUID is a unique identifier that can be used to correlate this event with a KDC event. - Transited services indicate which intermediate services have participated in this logon request. - Package name indicates which sub-protocol was used among the NTLM protocols. - Key length indicates the length of the generated session key. This will be 0 if no session key was requested. 5040030 I'd like to parse out the redundant data which is in every similar message, several times a second... the ultimate goal would be for that message to be trimmed down to: Feb 1 16:08:31 10.111.1.11 5043398 4624 Microsoft-Windows-Security-Auditing NT AUTHORITY\ANONYMOUS LOGON N/A Success Audit AD02.domain.com Logon Workstation Name: AC110 Source Network Address: 10.101.2.110 Source Port: 60711 or similar... I can define aregx to do this, but I am not sure where the best place would be to intercept and parse this before it is wrote out, or if syslog-ng has the parts to allow this to happen. thanks for your time, greg

3 3