- syslog-ng - lists.balabit.hu

[PATCH (3.5) 3/3] template escaping changed
by Juhász Viktor 23 Aug '13

23 Aug '13

template_escaping: template and correlation index escaping changed. correlation index escaping introduced, because it's a correct use-case, that somebody want to use @ character after template, in this case use '@@' after the template e.g.: ${USERNAME}(a)gmail.com caused bad behaviour and there wasn't any possible solution to reach the correct behaviour from now the correct usage is: ${USERNAME}@@gmail.com The template $ escaping changed from '\$' to '$$'\ because it was very confusing that the lexer could process '\' and the template_compile processed '\' too. >From now, the correct usage if you want to use $ in the output string is: $$HELLO instead of \$HELLO Signed-off-by: Juhasz Viktor <jviktor(a)balabit.hu> --- lib/templates.c | 38 +++++++++++++++++++++++++++--------- tests/unit/test_template_compile.c | 38 +++++++++++++++++++++++++++++++++--- 2 files changed, 64 insertions(+), 12 deletions(-) diff --git a/lib/templates.c b/lib/templates.c index 6cd90ee..a34667a 100644 --- a/lib/templates.c +++ b/lib/templates.c @@ -871,19 +871,33 @@ log_template_add_func_elem(LogTemplate *self, GString *text, gint argc, gchar *a } static void -parse_msg_ref(gchar **p, gint *msg_ref) +parse_msg_ref(gchar *template_string, gchar **p, gint *msg_ref) { *msg_ref = 0; if ((**p) == '@') { (*p)++; - /* syntax: ${name}@1 to denote the log message index in the correllation state */ - while ((**p) >= '0' && (**p) <= '9') + if ((**p) >= '0' && (**p) <= '9') { - (*msg_ref) += (*msg_ref) * 10 + ((**p) - '0'); - (*p)++; + /* syntax: ${name}@1 to denote the log message index in the correllation state */ + while ((**p) >= '0' && (**p) <= '9') + { + (*msg_ref) += (*msg_ref) * 10 + ((**p) - '0'); + (*p)++; + } + *msg_ref += 1; + } + else + { + if ((**p) != '@') + { + msg_warning("Non-numeric correlation state ID found, assuming a literal '@' character. To avoid confusion when using a literal '@' after a macro or template function, write '@@' in the templ + evt_tag_str("Template", template_string), + NULL); + (*p)--; + } + *msg_ref = 0; } - *msg_ref += 1; } } @@ -978,7 +992,7 @@ log_template_process_braced_template(LogTemplate *self, gchar **p, GString *text return FALSE; } } - parse_msg_ref(p, &msg_ref); + parse_msg_ref(self->template, p, &msg_ref); log_template_add_elem(self, start, macro_len, default_value, text, msg_ref); return TRUE; } @@ -1091,7 +1105,7 @@ log_template_process_template_function(LogTemplate *self, gchar **p, GString *te goto error; } (*p)++; - parse_msg_ref(p, &msg_ref); + parse_msg_ref(self->template, p, &msg_ref); if (!log_template_add_func_elem(self, text, strv->len - 1, (gchar **) strv->pdata, msg_ref, error)) { goto error; @@ -1180,7 +1194,13 @@ log_template_process_character(LogTemplate *self, gchar **p, GString **text, GEr } if (**p == '\\') { - (*p)++; + if (cfg_is_config_version_older(self->cfg, 0x0305)) + { + msg_warning("Template escaping changed in syslog-ng-ose 3.5. Use '$$' to specify a literal dollar sign instead of '\\$' and remove the escaping of the backslash character when you upgrade your c + evt_tag_str("Template", self->template), + NULL); + (*p)++; + } } if (**p) { diff --git a/tests/unit/test_template_compile.c b/tests/unit/test_template_compile.c index 23fed4f..f73f653 100644 --- a/tests/unit/test_template_compile.c +++ b/tests/unit/test_template_compile.c @@ -10,6 +10,7 @@ #define SIMPLE_MACRO_UNBRACE_ADD_TEXT "$MESSAGE test value" #define SIMPLE_MACRO_MSGREF "${MESSAGE}@1" #define SIMPLE_MACRO_INVALID_REF "${MESSAGE}(a)gmail.com" +#define SIMPLE_MACRO_ESCAPED_REF "${MESSAGE}@@12" #define SIMPLE_MACRO_ADD_TEXT "${MESSAGE}test value" #define ESCAPED_CHAR "Test \\$STRING" #define VALID_SUBST "${MESSAGE:-default value}" @@ -20,6 +21,7 @@ #define TRICKY_VALUE_5 "$" #define TRICKY_VALUE_6 "${MESSAGE:-}" #define TRICKY_VALUE_7 "${}" +#define TRICKY_VALUE_8 "\\tsomething ${MESSAGE}" #define INVALID_MACRO "${MESSAGE" #define INVALID_SUBST "${MESSAGE:1}" @@ -36,6 +38,9 @@ #define SIMPLE_UNKNOWN_FUNCTION "$(unknown function)" +#define CURRENT_VERSION 0x0305 +#define PREVIOUS_VERSION 0x0304 + static void hello(LogMessage *msg, int argc, GString *argv[], GString *result) { @@ -147,17 +152,32 @@ test_template_compile_macro() assert_template_compile(SIMPLE_MACRO_INVALID_REF); element = (LogTemplateElem *)template->compiled_template->data; - fill_expected_template_element(expected_element, text = "", default_value = NULL, macro = M_MESSAGE, type = LTE_MACRO, msg_ref = 1); + fill_expected_template_element(expected_element, text = "", default_value = NULL, macro = M_MESSAGE, type = LTE_MACRO, msg_ref = 0); + assert_macro_element(element[0], expected_element); + element = (LogTemplateElem *)template->compiled_template->next->data; + fill_expected_template_element(expected_element, text = "@gmail.com", default_value = NULL, macro = M_NONE, type = LTE_MACRO, msg_ref = 0); + assert_macro_element(element[0], expected_element); + + assert_template_compile(SIMPLE_MACRO_ESCAPED_REF); + element = (LogTemplateElem *)template->compiled_template->data; + fill_expected_template_element(expected_element, text = "", default_value = NULL, macro = M_MESSAGE, type = LTE_MACRO, msg_ref = 0); assert_macro_element(element[0], expected_element); element = (LogTemplateElem *)template->compiled_template->next->data; - fill_expected_template_element(expected_element, text = "gmail.com", default_value = NULL, macro = M_NONE, type = LTE_MACRO, msg_ref = 0); + fill_expected_template_element(expected_element, text = "@12", default_value = NULL, macro = M_NONE, type = LTE_MACRO, msg_ref = 0); assert_macro_element(element[0], expected_element); + cfg_set_version(template->cfg, PREVIOUS_VERSION); assert_template_compile(ESCAPED_CHAR); element = (LogTemplateElem *)template->compiled_template->data; fill_expected_template_element(expected_element, text = "Test $STRING", default_value = NULL, macro = M_NONE, type = LTE_MACRO, msg_ref = 0); assert_macro_element(element[0], expected_element); + cfg_set_version(template->cfg, CURRENT_VERSION); + assert_template_compile(ESCAPED_CHAR); + element = (LogTemplateElem *)template->compiled_template->data; + fill_expected_template_element(expected_element, text = "Test \\", default_value = NULL, value_handle=log_msg_get_value_handle("STRING"), type = LTE_VALUE, msg_ref = 0); + assert_value_element(element[0], expected_element); + assert_template_compile(VALID_SUBST); element = (LogTemplateElem *)template->compiled_template->data; fill_expected_template_element(expected_element, text = "", default_value = "default value", macro = M_MESSAGE, type = LTE_MACRO, msg_ref = 0); @@ -178,11 +198,18 @@ test_template_compile_macro() fill_expected_template_element(expected_element, text = "${VALUE_NAME}", default_value = NULL, macro = M_NONE, type = LTE_MACRO, msg_ref = 0); assert_macro_element(element[0], expected_element); + cfg_set_version(template->cfg, PREVIOUS_VERSION); assert_template_compile(TRICKY_VALUE_4); element = (LogTemplateElem *)template->compiled_template->data; fill_expected_template_element(expected_element, text = "", default_value = NULL, macro = M_NONE, type = LTE_MACRO, msg_ref = 0); assert_macro_element(element[0], expected_element); + cfg_set_version(template->cfg, CURRENT_VERSION); + assert_template_compile(TRICKY_VALUE_4); + element = (LogTemplateElem *)template->compiled_template->data; + fill_expected_template_element(expected_element, text = "\\", default_value = NULL, macro = M_NONE, type = LTE_MACRO, msg_ref = 0); + assert_macro_element(element[0], expected_element); + assert_template_compile(TRICKY_VALUE_5); element = (LogTemplateElem *)template->compiled_template->data; fill_expected_template_element(expected_element, text = "$", default_value = NULL, macro = M_NONE, type = LTE_MACRO, msg_ref = 0); @@ -193,6 +220,11 @@ test_template_compile_macro() fill_expected_template_element(expected_element, text = "", default_value = "", macro = M_MESSAGE, type = LTE_MACRO, msg_ref = 0); assert_macro_element(element[0], expected_element); + assert_template_compile(TRICKY_VALUE_8); + element = (LogTemplateElem *)template->compiled_template->data; + fill_expected_template_element(expected_element, text = "\\tsomething ", default_value = NULL, macro = M_MESSAGE, type = LTE_MACRO, msg_ref = 0); + assert_macro_element(element[0], expected_element); + log_template_unref(template); } @@ -303,7 +335,7 @@ test_template_compile_negativ_tests() int main(int argc, char **argv) { msg_init(FALSE); - configuration = cfg_new(0x305); + configuration = cfg_new(CURRENT_VERSION); log_msg_registry_init(); log_template_global_init(); --

1 0

[PATCH (3.5) 2/3] log_template_compile function refactorized
by Juhász Viktor 23 Aug '13

23 Aug '13

templates: log_template_compile refactored. It was a monster function. It was very hard to maintain and develop, so before the template escaping changes it was useful to refactor this huge function. Signed-off-by: Juhasz Viktor <jviktor(a)balabit.hu> --- lib/templates.c | 554 +++++++++++++++++++++++++++++++------------------------ 1 file changed, 315 insertions(+), 239 deletions(-) diff --git a/lib/templates.c b/lib/templates.c index e084c1a..6cd90ee 100644 --- a/lib/templates.c +++ b/lib/templates.c @@ -887,15 +887,314 @@ parse_msg_ref(gchar **p, gint *msg_ref) } } +static void +log_template_fill_compile_error(GError **error, const gchar *error_info, gint error_pos) +{ + g_set_error(error, LOG_TEMPLATE_ERROR, LOG_TEMPLATE_ERROR_COMPILE, "%s, error_pos='%d'", error_info, error_pos); +} + +static void +log_template_append_and_increment(gchar **p, GString *text) +{ + g_string_append_c(text, **p); + (*p)++; +} + +static gint +log_template_get_macro_length(gchar *start, gchar *end, gchar **token) +{ + gint result = 0; + gchar *colon = memchr(start, ':', end - start - 1); + if (colon) + { + result = colon - start; + *token = colon < end ? colon + 1 : NULL; + } + else + { + result = end - start - 1; + *token = NULL; + } + return result; +} + + +static gchar * +log_template_parse_default_value(gchar *macro_end, gchar *token) +{ + g_assert(token); + if (*token != '-') + { + return NULL; + } + return g_strndup(token + 1, macro_end - token - 2); +} + +#define IS_MACRO_NAME(c) (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c == '_') || (c >= '0' && c <= '9') + +#define STEP_BY_TRUE(p, x) while(x) p++; + +static void +log_template_add_elem(LogTemplate *self, gchar *start, gint macro_len, gchar *default_value, GString *text, gint msg_ref) +{ + gint macro = log_macro_lookup(start, macro_len); + if (macro == M_NONE) + { + log_template_add_value_elem(self, start, macro_len, text, default_value, msg_ref); + } + else + { + log_template_add_macro_elem(self, macro, text, default_value, msg_ref); + } +} + +static gboolean +log_template_process_braced_template(LogTemplate *self, gchar **p, GString *text, GError **error) +{ + gint macro_len; + gint msg_ref = 0; + gchar *default_value = NULL; + gchar *start; + gchar *token; + + (*p)++; + start = *p; + *p = strchr(*p, '}'); + + if (!*p) + { + log_template_fill_compile_error(error, "Invalid macro, '}' is missing", strlen(self->template)); + return FALSE; + } + + (*p)++; + macro_len = log_template_get_macro_length(start, *p, &token); + if (token) + { + default_value = log_template_parse_default_value(*p, token); + if (!default_value) + { + log_template_fill_compile_error(error, "Unknown substitution function", token - self->template); + return FALSE; + } + } + parse_msg_ref(p, &msg_ref); + log_template_add_elem(self, start, macro_len, default_value, text, msg_ref); + return TRUE; +} + +static gboolean +log_template_get_unquoted_part(gchar **p, GPtrArray *strv, gint *parens, gchar *quote, GString *arg_buf) +{ + gboolean result = TRUE; + if (**p == '\\') + { + (*p)++; + } + else if (**p == '(') + { + (*parens)++; + } + else if (**p == ')') + { + (*parens)--; + } + else if (**p == '"' || **p == '\'') + { + *quote = **p; + if (*parens == 1) + { + /* skip the quote in top-level and don't skip in expressions enclosed in parens */ + (*p)++; + result = FALSE; + } + } + else if (*parens == 1 && (**p == ' ' || **p == '\t')) + { + g_ptr_array_add(strv, g_strndup(arg_buf->str, arg_buf->len)); + g_string_truncate(arg_buf, 0); + while (**p && (**p == ' ' || **p == '\t')) + (*p)++; + result = FALSE; + } + if (*parens == 0) + { + result = FALSE; + } + return result; +} + +static gboolean +log_template_get_quoted_part(gchar **p, gint parens, gchar *quote) +{ + gboolean result = TRUE; + if (*quote == **p) + { + /* end of string */ + *quote = 0; + if (parens == 1) + { + (*p)++; + result = FALSE; + } + } + return result; +} + +static void +log_template_parse_template_function(gchar **p, GPtrArray *strv) +{ + gchar quote = 0; + gint parens = 1; + GString *arg_buf; + arg_buf = g_string_sized_new(32); + (*p)++; + gboolean add_char_to_string; + /* skip white space */ + while (**p && **p == ' ') + (*p)++; + + while (**p && parens != 0) + { + add_char_to_string = TRUE; + if (!quote) + { + add_char_to_string = log_template_get_unquoted_part(p, strv, &parens, &quote, arg_buf); + } + else + { + add_char_to_string = log_template_get_quoted_part(p, parens, &quote); + } + if (**p && add_char_to_string) + { + log_template_append_and_increment(p, arg_buf); + } + } + + if (arg_buf->len) + g_ptr_array_add(strv, g_strndup(arg_buf->str, arg_buf->len)); + g_ptr_array_add(strv, NULL); + g_string_free(arg_buf, TRUE); +} + +static gboolean +log_template_process_template_function(LogTemplate *self, gchar **p, GString *text, GError **error) +{ + GPtrArray *strv = g_ptr_array_new(); + gint msg_ref; + + log_template_parse_template_function(p, strv); + + if (**p != ')') + { + log_template_fill_compile_error(error, "Invalid template function reference, missing function name or inbalanced '('", *p - self->template); + goto error; + } + (*p)++; + parse_msg_ref(p, &msg_ref); + if (!log_template_add_func_elem(self, text, strv->len - 1, (gchar **) strv->pdata, msg_ref, error)) + { + goto error; + } + g_ptr_array_free(strv, FALSE); + return TRUE; +error: + g_strfreev((gchar **)strv->pdata); + g_ptr_array_free(strv, FALSE); + return FALSE; +} + +static void +log_template_process_unbraced_template(LogTemplate *self, gchar **p, GString *text) +{ + gchar *start = *p; + gint macro_len; + do + { + (*p)++; + } + while (IS_MACRO_NAME(**p)); + macro_len = *p - start; + log_template_add_elem(self, start, macro_len, NULL, text, 0); +} + +static gboolean +log_template_process_value(LogTemplate *self, gchar **p, GString **text, GError **error) +{ + gboolean finished = FALSE; + (*p)++; + /* macro reference */ + if (**p == '{') + { + if (!log_template_process_braced_template(self, p, *text, error)) + { + return FALSE; + } + finished = TRUE; + } + /* template function */ + else if (**p == '(') + { + if (!log_template_process_template_function(self, p, *text, error)) + { + return FALSE; + } + finished = TRUE; + } + /* unbraced macro */ + else if (IS_MACRO_NAME(**p)) + { + log_template_process_unbraced_template(self, p, *text); + finished = TRUE; + } + /* escaped value with dollar */ + else + { + if (**p != '$') + { + g_string_append_c(*text, '$'); + } + if (**p) + { + log_template_append_and_increment(p, *text); + } + } + if (finished) + { + if (*text) + g_string_free(*text, TRUE); + *text = NULL; + } + return TRUE; +} + +gboolean +log_template_process_character(LogTemplate *self, gchar **p, GString **text, GError **error) +{ + if (*text == NULL) + *text = g_string_sized_new(32); + + if (**p == '$') + { + return log_template_process_value(self, p, text, error); + } + if (**p == '\\') + { + (*p)++; + } + if (**p) + { + log_template_append_and_increment(p, *text); + } + return TRUE; +} + gboolean log_template_compile(LogTemplate *self, const gchar *template, GError **error) { - gchar *start, *p; - guint last_macro = M_NONE; + gchar *p; GString *last_text = NULL; - gchar *error_info; - gint error_pos = 0; - + g_return_val_if_fail(error == NULL || *error == NULL, FALSE); log_template_reset_compiled(self); @@ -903,250 +1202,27 @@ log_template_compile(LogTemplate *self, const gchar *template, GError **error) g_free(self->template); self->template = g_strdup(template); p = self->template; - + while (*p) { - if (last_text == NULL) - last_text = g_string_sized_new(32); - if (*p == '\\') + if (!log_template_process_character(self, &p, &last_text, error)) { - p++; - if (*p) - { - g_string_append_c(last_text, *p); - p++; - } - } - else if (*p == '$') - { - gboolean finished = FALSE; - p++; - /* macro reference */ - if (*p == '{') - { - gchar *colon, *fncode; - gint macro_len; - gchar *default_value = NULL; - gint msg_ref = 0; - - p++; - start = p; - while (*p && *p != '}') - p++; - - if (!*p) - { - error_pos = p - self->template; - error_info = "Invalid macro, '}' is missing"; - goto error; - } - - p++; - - colon = memchr(start, ':', p - start - 1); - if (colon) - { - macro_len = colon - start; - fncode = colon < p ? colon + 1 : NULL; - } - else - { - macro_len = p - start - 1; - fncode = NULL; - } - - if (fncode) - { - if (*fncode == '-') - { - default_value = g_strndup(fncode + 1, p - fncode - 2); - } - else - { - error_pos = fncode - self->template; - error_info = "Unknown substitution function"; - goto error; - } - } - parse_msg_ref(&p, &msg_ref); - last_macro = log_macro_lookup(start, macro_len); - if (last_macro == M_NONE) - { - /* this was not a known macro, take it as a "value" reference */ - log_template_add_value_elem(self, start, macro_len, last_text, default_value, msg_ref); - } - else - { - log_template_add_macro_elem(self, last_macro, last_text, default_value, msg_ref); - } - finished = TRUE; - } - /* template function */ - else if (*p == '(') - { - gchar quote = 0; - gint parens = 1; - GPtrArray *strv; - GString *arg_buf; - gint msg_ref; - - strv = g_ptr_array_new(); - arg_buf = g_string_sized_new(32); - - p++; - start = p; - /* skip white space */ - while (*p && *p == ' ') - p++; - - g_string_truncate(arg_buf, 0); - while (*p) - { - if (!quote) - { - if (*p == '\\') - { - p++; - } - else if (*p == '"' || *p == '\'') - { - quote = *p; - if (parens == 1) - { - /* skip the quote in top-level and don't skip in expressions enclosed in parens */ - p++; - continue; - } - } - else if (parens == 1 && (*p == ' ' || *p == '\t')) - { - g_ptr_array_add(strv, g_strndup(arg_buf->str, arg_buf->len)); - g_string_truncate(arg_buf, 0); - while (*p && (*p == ' ' || *p == '\t')) - p++; - continue; - } - else if (*p == '(') - { - parens++; - } - else if (*p == ')') - { - parens--; - if (parens == 0) - break; - } - } - else - { - if (quote == *p) - { - /* end of string */ - quote = 0; - if (parens == 1) - { - p++; - continue; - } - } - } - if (*p) - { - g_string_append_c(arg_buf, *p); - p++; - } - } - - if (arg_buf->len) - g_ptr_array_add(strv, g_strndup(arg_buf->str, arg_buf->len)); - g_ptr_array_add(strv, NULL); - if (*p != ')') - { - g_string_free(arg_buf, TRUE); - g_ptr_array_foreach(strv, (GFunc) g_free, NULL); - g_ptr_array_free(strv, TRUE); - - error_pos = p - self->template; - error_info = "Invalid template function reference, missing function name or inbalanced '('"; - goto error; - } - p++; - parse_msg_ref(&p, &msg_ref); - if (!log_template_add_func_elem(self, last_text, strv->len - 1, (gchar **) strv->pdata, msg_ref, error)) - { - g_ptr_array_foreach(strv, (GFunc) g_free, NULL); - g_ptr_array_free(strv, TRUE); - g_string_free(arg_buf, TRUE); - goto error_set; - } - g_ptr_array_free(strv, FALSE); - g_string_free(arg_buf, TRUE); - finished = TRUE; - } - else if ((*p >= 'A' && *p <= 'Z') || (*p >= 'a' && *p <= 'z') || (*p == '_') || (*p >= '0' && *p <= '9')) - { - start = p; - do - { - p++; - } - while ((*p >= 'A' && *p <= 'Z') || (*p >= 'a' && *p <= 'z') || (*p == '_') || (*p >= '0' && *p <= '9')); - last_macro = log_macro_lookup(start, p - start); - if (last_macro == M_NONE) - { - /* this was not a known macro, take it as a "value" reference */ - log_template_add_value_elem(self, start, p-start, last_text, NULL, 0); - } - else - { - log_template_add_macro_elem(self, last_macro, last_text, NULL, 0); - } - finished = TRUE; - } - else if (*p == '$') - { - g_string_append_c(last_text, '$'); - p++; - } - else - { - g_string_append_c(last_text, '$'); - g_string_append_c(last_text, *p); - p++; - } - if (finished) - { - if (last_text) - g_string_free(last_text, TRUE); - last_macro = M_NONE; - last_text = NULL; - } - } - else - { - g_string_append_c(last_text, *p); - p++; + log_template_reset_compiled(self); + if (!last_text) + last_text = g_string_sized_new(0); + g_string_sprintf(last_text, "error in template: %s", self->template); + log_template_add_macro_elem(self, M_NONE, last_text, NULL, 0); + g_string_free(last_text, TRUE); + return FALSE; } } - if (last_macro != M_NONE || last_text) + if (last_text) { - log_template_add_macro_elem(self, last_macro, last_text, NULL, 0); + log_template_add_macro_elem(self, M_NONE, last_text, NULL, 0); g_string_free(last_text, TRUE); } self->compiled_template = g_list_reverse(self->compiled_template); return TRUE; - - error: - g_set_error(error, LOG_TEMPLATE_ERROR, LOG_TEMPLATE_ERROR_COMPILE, "%s, error_pos='%d'", error_info, error_pos); - - error_set: - log_template_reset_compiled(self); - if (!last_text) - last_text = g_string_sized_new(0); - g_string_sprintf(last_text, "error in template: %s", self->template); - log_template_add_macro_elem(self, M_NONE, last_text, NULL, 0); - g_string_free(last_text, TRUE); - return FALSE; } --

1 0

syslogng-web: a simple and fast log viewer
by Peter Czanik 23 Aug '13

23 Aug '13

Hello, It's always nice to learn about projects building on syslog-ng. "syslogng-web" is one of them, a simple and fast log viewer for syslog-ng logs collected into mongodb. You can read more about it at https://czanik.blogs.balabit.com/2013/08/syslog-ng-web-a-simple-and-fast-lo… If you know about any other projects with syslog-ng at their core, please let me know so I can check it out and spread the word about it! Bye, -- Peter Czanik (CzP) <czanik(a)balabit.hu> BalaBit IT Security / syslog-ng upstream http://czanik.blogs.balabit.com/

1 0

[Bug 245] New: syslog-ng 3.4.3 / 3.4.2 add bogus characters to /var/log/ messages
by bugzilla＠bugzilla.balabit.com 17 Aug '13

17 Aug '13

https://bugzilla.balabit.com/show_bug.cgi?id=245 Summary: syslog-ng 3.4.3 / 3.4.2 add bogus characters to /var/log/messages Product: syslog-ng Version: 3.4.x Platform: PC OS/Version: Linux Status: NEW Severity: normal Priority: unspecified Component: syslog-ng AssignedTo: bazsi(a)balabit.hu ReportedBy: himbeere(a)meine-oma.de Type of the Report: bug Estimated Hours: 0.0 Hello. Suddenly my syslog-ng adds bogus characters to /var/log/messages like this: Aug 16 21:40:56 hn2 syslog-ng[2009]: syslog-ng shutting down; version='3.4.2' Aug 16 21:41:44 hn2 syslog-ng[1994]: syslog-ng starting up; version='3.4.3' Aug 16 21:41:44 hn2 syslog-ng[1994]: Internal error, alarm_set() called while an alarm is still active; Aug 16 21:41:44 ^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@^@ kernel: [ 0.000000] Initializing cgroup subsys cpuset Aug 16 21:41:44 hn2 kernel: [ 0.000000] Initializing cgroup subsys cpu Aug 16 21:41:44 hn2 kernel: [ 0.000000] Initializing cgroup subsys cpuacct Aug 16 21:41:44 hn2 kernel: [ 0.000000] Linux version 3.10.7 (root@hn2) (gcc version 4.6.3 (Gentoo 4.6.3 p1.13, pie-0.5.2) ) #2 SMP Fri Aug 16 21:17:31 CEST 2013 This leads to an unreadable file. Well i can open the file with vi and then it looks okay. But with less it looks like this: 00000000 41 75 67 20 31 36 20 32 31 3a 32 37 3a 34 36 20 |Aug 16 21:27:46 | 00000010 68 6e 32 20 73 79 73 6c 6f 67 2d 6e 67 5b 32 38 |hn2 syslog-ng[28| 00000020 38 31 5d 3a 20 73 79 73 6c 6f 67 2d 6e 67 20 73 |81]: syslog-ng s| 00000030 74 61 72 74 69 6e 67 20 75 70 3b 20 76 65 72 73 |tarting up; vers| 00000040 69 6f 6e 3d 27 33 2e 34 2e 32 27 0a 41 75 67 20 |ion='3.4.2'.Aug | 00000050 31 36 20 32 31 3a 32 38 3a 30 33 20 68 6e 32 20 |16 21:28:03 hn2 | 00000060 73 68 75 74 64 6f 77 6e 5b 32 38 39 35 5d 3a 20 |shutdown[2895]: | 00000070 73 68 75 74 74 69 6e 67 20 64 6f 77 6e 20 66 6f |shutting down fo| 00000080 72 20 73 79 73 74 65 6d 20 72 65 62 6f 6f 74 0a |r system reboot.| 00000090 41 75 67 20 31 36 20 32 31 3a 32 38 3a 30 33 20 |Aug 16 21:28:03 | 000000a0 68 6e 32 20 69 6e 69 74 3a 20 53 77 69 74 63 68 |hn2 init: Switch| 000000b0 69 6e 67 20 74 6f 20 72 75 6e 6c 65 76 65 6c 3a |ing to runlevel:| 000000c0 20 36 0a 41 75 67 20 31 36 20 32 31 3a 32 38 3a | 6.Aug 16 21:28:| Is there any solution to this? Thanks and best regards t. -- Configure bugmail: https://bugzilla.balabit.com/userprefs.cgi?tab=email ------- You are receiving this mail because: ------- You are watching all bug changes.

1 2

Cisco ASA logging to syslog-ng and a weird extra-characters in new-lines
by Finnur Orn Gudmundsson 16 Aug '13

16 Aug '13

Hi all, I have a syslog-ng server: rpm -qi syslog-ng Name : syslog-ng Relocations: (not relocatable) Version : 3.2.5 Vendor: Fedora Project Release : 3.el6 Build Date: Sun 15 Jan 2012 07:49:04 PM GMT Install Date: Wed 06 Mar 2013 03:56:17 PM GMT Build Host: x86-14.phx2.fedoraproject.org Group : System Environment/Daemons Source RPM: syslog-ng-3.2.5-3.el6.src.rpm Size : 1594638 License: GPLv2+ Signature : RSA/8, Sun 15 Jan 2012 08:55:15 PM GMT, Key ID 3b49df2a0608b895 The syslog-ng package installed is built and distributed by Fedora EPEL but I was hoping I could post here. I replaced a old syslog-ng 2.x server to this one and almost everything works as expected (there is always this one thing that does not work after a migration:). There is a Cisco ASA box logging to this server. For some records I get this extra space in a new-line like this: Aug 16 15:35:56 10.X.X.X %ASA-6-302021: Teardown ICMP connection for faddr 10.X.X.X/9483 gaddr 10.X.X.X/0 laddr 10.X.X.X/0 Aug 16 15:35:56 10.X.X.X %ASA-5-304001: 10.X.X.X Accessed URL 46.X.X.X:/ < Aug 16 15:35:56 10.X.X.X %ASA-5-304001: 10.X.X.X Accessed URL 92.X.X.X:http://sphotos-g.ak.fbcdn.net/hphotos-X/XxX/11X_221X18801_1X554_n.… Aug 16 15:35:56 10.X.X.X %ASA-6-305011: Built dynamic TCP translation from inside:10.X.X.X/38697 to outside:194.X.X.X/38697 (the < char is in a new-line) And another one: Aug 16 15:35:56 10.Z.Z.Z %ASA-6-302013: Built outbound TCP connection 1809896329 for outside:54.Z.Z.Z/80 (54.Z.Z.Z/80) to inside:10.Z.Z.Z/38684 (194.Z.Z.Z/38684) Aug 16 15:35:56 10.Z.Z.Z %ASA-5-304001: 10.Z.Z.Z Accessed URL 10.Z.Z.Z:/ 1 Aug 16 15:35:56 10.Z.Z.Z %ASA-6-302014: Teardown TCP connection 1809896328 for dmz-lb-int:10.Z.Z.Z/58759 to inside:10.Z.Z.Z/80 duration 0:00:00 bytes 162 TCP FINs (the single 1 char is in a new-line). Does someone know if this is a known bug ? I am aware that I am running a somewhat old version. Bgrds, Finnzi

1 1

insider 2013-08: New releases; GSoC updates; Dedicated OSE developer; Observe, Hack, Make 2013;
by Peter Czanik 15 Aug '13

15 Aug '13

Dear syslog-ng users, This is the 26th issue of the syslog-ng Insider, a monthly newsletter that brings you syslog-ng related news. FEATURED NEWS syslog-ng 3.4.3 and 3.3.11 are released --------------------------------------- Both the current (3.4) and previous (3.3) stable syslog-ng releases received maintenance updates. Sources are available from git or as tgz from http://www.balabit.com/network-security/syslog-ng/opensource-logging-system… For third-party binary packages for various Linux and UNIX platforms, visithttp://www.balabit.com/network-security/syslog-ng/opensource-logging-s… for 3^rd party binary packages for various distributions and UNIX variants. syslog-ng Agent for Windows 5.0 ------------------------------- There is a major new version of syslog-ng Agent available, version 5.0. Under the hood it builds now on the same code base as the UNIX version of syslog-ng. The new highlights include support for the Reliable Log Transfer Protocol (RLTP), flow control and support for the Windows Server 2012 and Windows 8 platforms. Long-time syslog-ng contributor Gergely Nagy now working full-time on syslog-ng Open Source Edition ----------------------------- If you've been following the mailing list, the community around syslog-ng or the commits in git you most certainly recognize the name of Gergely Nagy. He's been working at BalaBit for several years and we're happy to announce that from now on his dedicated role is to work on syslog-ng OSE full-time. You can read his thoughts about the change on his blog: https://algernon.blogs.balabit.com/2013/07/onto-greener-open-pastures/ syslog-ng GSoC: it's half-time! ------------------------------- The coding for Google Summer of Code is at half-time. Our students passed half-time evaluations and are working hard on a faster MySQL destination and on a brand new redis destination. You can follow their progress from their related blogs: http://petrovicsgyula.blogspot.com/ http://tichygsoc.blogspot.hu/ There is also a plan to provide test packages for those really adventurous in the coming weeks :-) syslog-ng JSON HowTo -------------------- JSON is gaining popularity, not just in Web 2.0 but in all fields of IT. As it's an easy way of storing and transmitting name value pairs, JSON emitting and parsing is now part of syslog-ng. The HowTo describing how it works in syslog-ng is available at http://asylum.madhouse-project.org/blog/2013/07/29/json-howto/ syslog-ng developers at Observe, Hack, Make 2013 ------------------------------------------------ Laser harp, Wikileaks, lockpicking, NSA, SIM card exploiting, LHC, counter-cryptanalysis, 24 pull requests, neutrino detectors, open source Bach, weird lights, quadrocopters, electric music cranked up to 11, workshops, lectures, discussions, and a lot more! We've been Observing, Hacking and Making! You can read more about it at https://athos.blogs.balabit.com/2013/08/observed-hacked-made/ SHORT NEWS: * CzP presenting log management and syslog-ng at Libre Software Meeting in Brussels: https://czanik.blogs.balabit.com/2013/08/czp-libre-software-meeting/ * What is Pseudonymous Data? An update on how upcoming EU regulations affect logging: https://jluby.blogs.balabit.com/2013/07/18/what-is-pseudonymous-data-in-the… NEW RELEASES * syslog-ng 3.3.11: https://lists.balabit.hu/pipermail/syslog-ng-announce/2013-June/000163.html * syslog-ng 3.4.3: https://lists.balabit.hu/pipermail/syslog-ng-announce/2013-August/000164.ht… * syslog-ng Agent 5.0.1: https://lists.balabit.hu/pipermail/syslog-ng-announce/2013-May/000158.html * syslog-ng PE 4.0.6: https://lists.balabit.hu/pipermail/syslog-ng-announce/2013-August/000165.ht… * syslog-ng PE 4.2.4: https://lists.balabit.hu/pipermail/syslog-ng-announce/2013-June/000160.html Your feedback and news tips about the next issue is welcome at documentation(a)balabit.com <mailto:documentation@balabit.com> To read this newsletter on-line, visit: http://insider.blogs.balabit.com/ -- Peter Czanik (CzP) <czanik(a)balabit.hu> BalaBit IT Security / syslog-ng upstream http://czanik.blogs.balabit.com/

1 0

syslog-ng 3.4.3 has been released
by devel＠balabit.hu 13 Aug '13

13 Aug '13

------------------------------------------------------------------------------ PACKAGE : syslog-ng VERSION : 3.4.3 SUMMARY : new stable release DATE : Aug 13, 2013 ------------------------------------------------------------------------------ DESCRIPTION: A new stable version of syslog-ng Open Source Edition (3.4.3) has been released. For latest fixes in the 3.4.x feature branch you are recommended to upgrade to this version. CHANGES: 3.4.3 Tue, 13 Aug 2013 13:00:00 +0100 This is the second bug-fix release for the 3.4.x series, collecting significant fixes over the 3.4.2 release. Upgrading to this release is highly recommended. Bugfixes ======== * The pacct module has been fixed, and now compiles correctly. * Updated the configure.in script, and a few other places to fix issues with recent autotools. * Fixed an infinite loop when template expansion resulted in a zero-length string. * A considerable number of minor issues surrounding the build system have been eliminated. * The template_escape() setting is now respected by template functions that create templates themselves (such as the conditional functions) * Restore support for defaultless block{} arguments. [#230] * RFC5424 NILVALUE is now accepted in timestamps [#238] * The JSON parser will throw an error when the input is not an object (but, for example, a string) instead of crashing. * The program() source has been fixed, and now does not throw a syntax error at config parsing time, but works correctly again. * In case resolving a hostname fails, the IP address does not get truncated anymore. [#240 & #242] * A race condition in certain filter operations have been eliminated. Features ======== * The JSON parser now accepts generic parser related options, such as template(), so one can easily set the input for the parser. Credits: ======== syslog-ng is developed as a community project, and as such it relies on volunteers to do the work necessarily to produce syslog-ng. Reporting bugs, testing changes, writing code or simply providing feedback are all important contributions, so please if you are a user of syslog-ng, contribute. These people have helped in this release: Andras Tim <tia(a)balabit.hu> Anton Koldaev <koldaevav(a)gmail.com> Attila Szalay <sasa(a)balabit.hu> Balazs Scheidler <bazsi(a)balabit.hu> Gergely Nagy <algernon(a)balabit.hu> Martin <bmartin(a)lavabit.com> Michael Sterrett <mr_bones_(a)gentoo.org> Vincent Brillault <spam(a)lerya.net> DOWNLOAD: You can download the source or binary packages from: http://www.balabit.com/network-security/syslog-ng/opensource-logging-system… The documentation of the syslog-ng Open Source Edition is available in The syslog-ng Open Source Edition Administrator's Guide at http://www.balabit.com/support/documentation/

3 2

[Bug 238] New: syslog-protocol: allow NILVALUE for TIMESTAMP as specified in rfc5424
by bugzilla＠bugzilla.balabit.com 13 Aug '13

13 Aug '13

https://bugzilla.balabit.com/show_bug.cgi?id=238 Summary: syslog-protocol: allow NILVALUE for TIMESTAMP as specified in rfc5424 Product: syslog-ng Version: 3.4.x Platform: PC OS/Version: Linux Status: NEW Severity: normal Priority: unspecified Component: syslog-ng AssignedTo: bazsi(a)balabit.hu ReportedBy: spam(a)lerya.net Type of the Report: --- Estimated Hours: 0.0 RFC5424 states: - section 6: """ TIMESTAMP = NILVALUE / FULL-DATE "T" FULL-TIME """ - section 6.2.3 "" A syslog application MUST use the NILVALUE as TIMESTAMP if the syslog application is incapable of obtaining system time. "" But the 3.4 version of syslog-ng does not allow such a NILVALUE: https://github.com/balabit/syslog-ng-3.4/blob/master/modules/syslogformat/s… For example, the following message does not work: <6>1 - - netlog - - - [ 775.895618] /bin/ping[2936] UDP 10.0.2.15:33285 -> 192.168.0.1:53 (uid=0) Whereas this one works: <6>1 2003-10-11T22:14:15.003Z - netlog - - - [ 775.895618] /bin/ping[2936] UDP 10.0.2.15:33285 -> 192.168.0.1:53 (uid=0) -- Configure bugmail: https://bugzilla.balabit.com/userprefs.cgi?tab=email ------- You are receiving this mail because: ------- You are watching all bug changes.

1 13

[Bug 240] New: host() in filters fail to match if using use_dns(yes)
by bugzilla＠bugzilla.balabit.com 13 Aug '13

13 Aug '13

https://bugzilla.balabit.com/show_bug.cgi?id=240 Summary: host() in filters fail to match if using use_dns(yes) Product: syslog-ng Version: 3.4.x Platform: PC OS/Version: Linux Status: NEW Severity: major Priority: unspecified Component: syslog-ng AssignedTo: bazsi(a)balabit.hu ReportedBy: bmartin(a)lavabit.com Type of the Report: bug Estimated Hours: 0.0 After upgrading from version 3.2.5 to 3.4.2 a filter using host() did not work anymore. The filter looks something like this: filter f_filter { host("172.12.0.1") or host("172.12.0.2"); }; Messages which would match this filter never reached their destination as they were used to with 3.2.5. I figured out that $HOST contained only the string '172'. My assumption is that host() matches against $HOST. In turn I set use_dns(no) and things started working again and now $HOST contains the whole IP address. used global options: options { threaded(yes); chain_hostnames(no); use_dns(no); stats_freq(43200); mark_freq(3600); }; -- Configure bugmail: https://bugzilla.balabit.com/userprefs.cgi?tab=email ------- You are receiving this mail because: ------- You are watching all bug changes.

1 4

[Bug 242] New: IP address stripped to first number with use_fqdn off
by bugzilla＠bugzilla.balabit.com 13 Aug '13

13 Aug '13

https://bugzilla.balabit.com/show_bug.cgi?id=242 Summary: IP address stripped to first number with use_fqdn off Product: syslog-ng Version: 3.4.x Platform: PC OS/Version: Linux Status: NEW Severity: normal Priority: unspecified Component: syslog-ng AssignedTo: bazsi(a)balabit.hu ReportedBy: list0570(a)paradise.net.nz Type of the Report: bug Estimated Hours: 0.0 transport("udp") ip("0.0.0.0") port(514) ip-protocol(4) keep_timestamp(no) keep_hostname(no) # no = rewrite hostname use_dns(yes) # yes, no, persist_only use_fqdn(no) Regardless of what the use_dns setting is, if the host name is an IP address A.B.C.D, use_fqdn is no, and /etc/hosts has no name and the DNS lookup fails, then only "A" is logged as the host name. This is not useful at all - if a name lookup fails for an IP address, you have to keep the IP address and not truncate it to something useless. syslog-ng-3.4.2-1.5.1.i586 from openSUSE 12.3 -- Configure bugmail: https://bugzilla.balabit.com/userprefs.cgi?tab=email ------- You are receiving this mail because: ------- You are watching all bug changes.

1 2