The indented multiline class is derived from LogProtoTextServer, the main difference is in determining what constitutes a full line: while the text server considers a newline as a terminator, the indented multiline server has stricter requirements: the newline must be followed by another line that does not start with whitespace. Signed-off-by: Gergely Nagy <algernon@balabit.hu> --- lib/Makefile.am | 2 + lib/logproto-builtins.c | 3 + lib/logproto-indented-multiline-server.c | 90 ++++++++++++++++++++++++++++++ lib/logproto-indented-multiline-server.h | 44 +++++++++++++++ lib/logproto-text-server.c | 37 +++++++----- lib/logproto-text-server.h | 8 +++ tests/unit/test_logproto.c | 45 +++++++++++++++ 7 files changed, 216 insertions(+), 13 deletions(-) create mode 100644 lib/logproto-indented-multiline-server.c create mode 100644 lib/logproto-indented-multiline-server.h diff --git a/lib/Makefile.am b/lib/Makefile.am index 65de69a..406cbd3 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -54,6 +54,7 @@ pkginclude_HEADERS = \ logproto-framed-server.h \ logproto-text-client.h \ logproto-text-server.h \ + logproto-indented-multiline-server.h \ logproto-record-server.h \ logproto-builtins.h \ logproto.h \ @@ -137,6 +138,7 @@ libsyslog_ng_la_SOURCES = \ logproto-framed-server.c \ logproto-text-client.c \ logproto-text-server.c \ + logproto-indented-multiline-server.c \ logproto-record-server.c \ logproto-builtins.c \ logqueue.c \ diff --git a/lib/logproto-builtins.c b/lib/logproto-builtins.c index bbe23ce..c1d674c 100644 --- a/lib/logproto-builtins.c +++ b/lib/logproto-builtins.c @@ -24,6 +24,7 @@ #include "logproto-dgram-server.h" #include "logproto-text-client.h" #include "logproto-text-server.h" +#include "logproto-indented-multiline-server.h" #include "logproto-framed-client.h" #include "logproto-framed-server.h" #include "plugin.h" @@ -35,6 +36,7 @@ DEFINE_LOG_PROTO_SERVER(log_proto_dgram); DEFINE_LOG_PROTO_CLIENT(log_proto_text); DEFINE_LOG_PROTO_SERVER(log_proto_text); +DEFINE_LOG_PROTO_SERVER(log_proto_indented_multiline); DEFINE_LOG_PROTO_CLIENT(log_proto_framed); DEFINE_LOG_PROTO_SERVER(log_proto_framed); @@ -43,6 +45,7 @@ static Plugin framed_server_plugins[] = LOG_PROTO_SERVER_PLUGIN(log_proto_dgram, "dgram"), LOG_PROTO_CLIENT_PLUGIN(log_proto_text, "text"), LOG_PROTO_SERVER_PLUGIN(log_proto_text, "text"), + LOG_PROTO_SERVER_PLUGIN(log_proto_indented_multiline, "indented-multiline"), LOG_PROTO_CLIENT_PLUGIN(log_proto_framed, "framed"), LOG_PROTO_SERVER_PLUGIN(log_proto_framed, "framed"), }; diff --git a/lib/logproto-indented-multiline-server.c b/lib/logproto-indented-multiline-server.c new file mode 100644 index 0000000..e3b52d3 --- /dev/null +++ b/lib/logproto-indented-multiline-server.c @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2012 BalaBit IT Ltd, Budapest, Hungary + * Copyright (c) 2012 Gergely Nagy <algernon@balabit.hu> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ +#include "logproto-indented-multiline-server.h" +#include "messages.h" + +#include <stdio.h> + +static gboolean +log_proto_indented_multiline_server_line_is_complete(LogProtoTextServer *self, + LogProtoBufferedServerState *state, + const guchar **eol) +{ + gsize pos; + + if (!*eol) + return FALSE; + + /* If we have eol, find more eols until we get one where the next + char is non-whitespace. */ + + pos = *eol - self->super.buffer; + + while (pos < state->pending_buffer_end - 1) + { + if (self->super.buffer[pos] == '\n') + *eol = self->super.buffer + pos; + else + { + pos++; + continue; + } + + if (self->super.buffer[pos + 1] != ' ' && + self->super.buffer[pos + 1] != '\t') + return TRUE; + + pos++; + } + + return FALSE; +} + +static void +log_proto_indented_multiline_server_line_flush (LogProtoTextServer *self, + LogProtoBufferedServerState *state, + const guchar **msg, gsize *msg_len, + const guchar *buffer_start, gsize buffer_bytes) +{ + *msg = buffer_start; + *msg_len = buffer_bytes; + if ((*msg)[buffer_bytes - 1] == '\n') + (*msg_len)--; + + state->pending_buffer_pos = state->pending_buffer_end; +} + +void +log_proto_indented_multiline_server_init(LogProtoIMultiLineServer *self, + LogTransport *transport, + const LogProtoServerOptions *options) +{ + log_proto_text_server_init(&self->super, transport, options); + self->super.is_line_complete = log_proto_indented_multiline_server_line_is_complete; + self->super.line_flush = log_proto_indented_multiline_server_line_flush; +} + +LogProtoServer * +log_proto_indented_multiline_server_new(LogTransport *transport, const LogProtoServerOptions *options) +{ + LogProtoIMultiLineServer *self = g_new0(LogProtoIMultiLineServer, 1); + + log_proto_indented_multiline_server_init(self, transport, options); + return &self->super.super.super; +} diff --git a/lib/logproto-indented-multiline-server.h b/lib/logproto-indented-multiline-server.h new file mode 100644 index 0000000..260d9b1 --- /dev/null +++ b/lib/logproto-indented-multiline-server.h @@ -0,0 +1,44 @@ +/* + * Copyright (c) 2012 BalaBit IT Ltd, Budapest, Hungary + * Copyright (c) 2012 Gergely Nagy <algernon@balabit.hu> + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + * + */ +#ifndef LOGPROTO_INDENTED_MULTILINE_SERVER_INCLUDED +#define LOGPROTO_INDENTED_MULTILINE_SERVER_INCLUDED + +#include "logproto-text-server.h" + +typedef struct _LogProtoIMultiLineServer LogProtoIMultiLineServer; +struct _LogProtoIMultiLineServer +{ + LogProtoTextServer super; +}; + +/* LogProtoIMultiLineServer + * + * This class processes indented multiline text files/streams. Each + * record consists of one line that starts with non-whitespace, with + * zero or more lines starting with whitespace. A record is terminated + * when we reach a line that starts with non-whitespace, or EOF. + */ +LogProtoServer *log_proto_indented_multiline_server_new(LogTransport *transport, + const LogProtoServerOptions *options); +void log_proto_indented_multiline_server_init(LogProtoIMultiLineServer *self, + LogTransport *transport, + const LogProtoServerOptions *options); + +#endif diff --git a/lib/logproto-text-server.c b/lib/logproto-text-server.c index ed1ed90..600c06d 100644 --- a/lib/logproto-text-server.c +++ b/lib/logproto-text-server.c @@ -185,16 +185,25 @@ log_proto_text_server_get_raw_size_of_buffer(LogProtoTextServer *self, const guc } } +static gboolean +log_proto_text_server_line_is_complete(LogProtoTextServer *self, + LogProtoBufferedServerState *state, + const guchar **eol) +{ + return !!(*eol); +} + +static void +log_proto_text_server_line_flush (LogProtoTextServer *self, + LogProtoBufferedServerState *state, + const guchar **msg, gsize *msg_len, + const guchar *buffer_start, gsize buffer_bytes) +{ + *msg = buffer_start; + *msg_len = buffer_bytes; + state->pending_buffer_pos = state->pending_buffer_end; +} -/** - * log_proto_text_server_fetch_from_buf: - * @self: LogReader instance - * @saddr: socket address to be assigned to new messages (consumed!) - * @flush: whether to flush the input buffer - * @msg_counter: the number of messages processed in the current poll iteration - * - * Returns TRUE if a message was found in the buffer, FALSE if we need to read again. - **/ static gboolean log_proto_text_server_fetch_from_buf(LogProtoBufferedServer *s, const guchar *buffer_start, gsize buffer_bytes, const guchar **msg, gsize *msg_len, gboolean flush_the_rest) { @@ -209,9 +218,7 @@ log_proto_text_server_fetch_from_buf(LogProtoBufferedServer *s, const guchar *bu * we are set to packet terminating mode or the connection is to * be teared down and we have partial data in our buffer. */ - *msg = buffer_start; - *msg_len = buffer_bytes; - state->pending_buffer_pos = state->pending_buffer_end; + self->line_flush (self, state, msg, msg_len, buffer_start, buffer_bytes); goto success; } @@ -235,7 +242,7 @@ log_proto_text_server_fetch_from_buf(LogProtoBufferedServer *s, const guchar *bu *msg = buffer_start; goto success; } - else if (!eol) + else if (!self->is_line_complete(self, state, &eol)) { gsize raw_split_size; @@ -328,6 +335,10 @@ log_proto_text_server_init(LogProtoTextServer *self, LogTransport *transport, co self->super.super.prepare = log_proto_text_server_prepare; self->super.super.free_fn = log_proto_text_server_free; self->super.fetch_from_buf = log_proto_text_server_fetch_from_buf; + + self->is_line_complete = log_proto_text_server_line_is_complete; + self->line_flush = log_proto_text_server_line_flush; + self->super.stream_based = TRUE; self->reverse_convert = (GIConv) -1; } diff --git a/lib/logproto-text-server.h b/lib/logproto-text-server.h index 720d2a6..012db89 100644 --- a/lib/logproto-text-server.h +++ b/lib/logproto-text-server.h @@ -34,6 +34,14 @@ struct _LogProtoTextServer gchar *reverse_buffer; gsize reverse_buffer_len; gint convert_scale; + + gboolean (*is_line_complete)(LogProtoTextServer *self, + LogProtoBufferedServerState *state, + const guchar **eol); + void (*line_flush)(LogProtoTextServer *self, + LogProtoBufferedServerState *state, + const guchar **msg, gsize *msg_len, + const guchar *buffer_start, gsize buffer_bytes); }; /* LogProtoTextServer diff --git a/tests/unit/test_logproto.c b/tests/unit/test_logproto.c index 2feec3d..51063f5 100644 --- a/tests/unit/test_logproto.c +++ b/tests/unit/test_logproto.c @@ -4,6 +4,7 @@ #include "logproto-text-server.h" #include "logproto-framed-server.h" #include "logproto-dgram-server.h" +#include "logproto-indented-multiline-server.h" #include "logproto-record-server.h" #include "apphook.h" @@ -507,6 +508,49 @@ test_log_proto_text_server(void) } /**************************************************************************************** + * LogProtoIMultiLineServer + ****************************************************************************************/ +static void +test_log_proto_indented_multiline_server_base(gboolean input_is_stream) +{ + LogProtoServer *proto; + + log_proto_testcase_begin("test_log_proto_indented_multiline_server_base"); + proto_server_options.max_msg_size = 32; + + proto = log_proto_indented_multiline_server_new( + /* 32 bytes max line length */ + (input_is_stream ? log_transport_mock_stream_new : log_transport_mock_records_new)( + "01234567\n" + /* line too long */ + //"0123456789ABCDEF0123456789ABCDEF01234567\n", -1, + /* multi-line */ + "0\n 1=2\n 3=4\nEND\n", -1, + + LTM_EOF), + get_inited_proto_server_options()); + + assert_proto_server_fetch(proto, "01234567", -1); + + /* input split due to an oversized input line */ + //assert_proto_server_fetch(proto, "0123456789ABCDEF0123456789ABCDEF", -1); + //assert_proto_server_fetch(proto, "01234567", -1); + + assert_proto_server_fetch(proto, "0\n 1=2\n 3=4", -1); + assert_proto_server_fetch(proto, "END", -1); + + log_proto_server_free(proto); + log_proto_testcase_end(); +} + +static void +test_log_proto_indented_multiline_server(void) +{ + test_log_proto_indented_multiline_server_base(FALSE); + test_log_proto_indented_multiline_server_base(TRUE); +} + +/**************************************************************************************** * LogProtoDGramServer ****************************************************************************************/ @@ -883,6 +927,7 @@ test_log_proto(void) test_log_proto_base(); test_log_proto_record_server(); test_log_proto_text_server(); + test_log_proto_indented_multiline_server(); test_log_proto_dgram_server(); test_log_proto_framed_server(); } -- 1.7.10.4