- syslog-ng - lists.balabit.hu

Program stop/start bug in 3.3.x
by Martin Holste 22 Dec '11

22 Dec '11

I'm getting a weird bug with the program destination in which on shutdown it sends TERM and then sends a start: <snip> Sending destination program a TERM signal; cmdline='perl /usr/local/elsa/node/elsa.pl -c /etc/elsa_node.conf', child_pid='21421' Starting destination program; cmdline='perl /usr/local/elsa/node/elsa.pl -c /etc/elsa_node.conf' </snip> 3.2.4 is fine.

2 1

[PATCH (3.4)] afsql: Add support for specifying custom SQL backend options.
by Gergely Nagy 22 Dec '11

22 Dec '11

When one is using an SQL backend that supports a set of options that syslog-ng does not set itself, yet, one does want these options set, there was previously no way to accomplish that without changing the source. This patch adds a new option: dbd-option(OPTION_NAME VALUE) (where OPTION_NAME is always a string, and value can be either a string, or a number). The options specified this way will always be set with libdbi whenever we (re)connect to the database. A simple use-case: destination d_slow_sql_test { sql(type("null") dbd-option("null.sleep.connect" 1) dbd-option("null.sleep.query" 5) columns("dummy") values("1") ); }; Signed-off-by: Gergely Nagy <algernon(a)balabit.hu> --- modules/afsql/afsql-grammar.ym | 3 +++ modules/afsql/afsql-parser.c | 2 ++ modules/afsql/afsql.c | 40 ++++++++++++++++++++++++++++++++++++++++ modules/afsql/afsql.h | 4 ++++ 4 files changed, 49 insertions(+), 0 deletions(-) diff --git a/modules/afsql/afsql-grammar.ym b/modules/afsql/afsql-grammar.ym index 580fb80..d21a3bf 100644 --- a/modules/afsql/afsql-grammar.ym +++ b/modules/afsql/afsql-grammar.ym @@ -54,6 +54,7 @@ extern LogDriver *last_driver; %token KW_DEFAULT %token KW_RETRIES +%token KW_DBD_OPTION %type <ptr> dest_afsql %type <ptr> dest_afsql_params @@ -100,6 +101,8 @@ dest_afsql_option free($3); } | KW_USERNAME '(' string ')' { afsql_dd_set_user(last_driver, $3); free($3); } + | KW_DBD_OPTION '(' string LL_NUMBER ')' { afsql_dd_add_dbd_option_numeric(last_driver, $3, $4); free($3); } + | KW_DBD_OPTION '(' string string ')' { afsql_dd_add_dbd_option(last_driver, $3, $4); free($3); free($4); } | KW_PASSWORD '(' string ')' { afsql_dd_set_password(last_driver, $3); free($3); } | KW_DATABASE '(' string ')' { afsql_dd_set_database(last_driver, $3); free($3); } | KW_TABLE '(' string ')' { afsql_dd_set_table(last_driver, $3); free($3); } diff --git a/modules/afsql/afsql-parser.c b/modules/afsql/afsql-parser.c index 877389a..291e495 100644 --- a/modules/afsql/afsql-parser.c +++ b/modules/afsql/afsql-parser.c @@ -55,6 +55,8 @@ static CfgLexerKeyword afsql_keywords[] = { { "flush_lines", KW_FLUSH_LINES }, { "flush_timeout", KW_FLUSH_TIMEOUT }, { "flags", KW_FLAGS }, + + { "dbd_option", KW_DBD_OPTION }, { NULL } }; diff --git a/modules/afsql/afsql.c b/modules/afsql/afsql.c index b3fec86..a327cdd 100644 --- a/modules/afsql/afsql.c +++ b/modules/afsql/afsql.c @@ -103,6 +103,9 @@ typedef struct _AFSqlDestDriver StatsCounterItem *dropped_messages; StatsCounterItem *stored_messages; + GHashTable *dbd_options; + GHashTable *dbd_options_numeric; + /* shared by the main/db thread */ GThread *db_thread; GMutex *db_thread_mutex; @@ -127,6 +130,22 @@ static const char *s_freetds = "freetds"; #define MAX_FAILED_ATTEMPTS 3 void +afsql_dd_add_dbd_option(LogDriver *s, const gchar *name, const gchar *value) +{ + AFSqlDestDriver *self = (AFSqlDestDriver *) s; + + g_hash_table_insert(self->dbd_options, g_strdup(name), g_strdup(value)); +} + +void +afsql_dd_add_dbd_option_numeric(LogDriver *s, const gchar *name, gint value) +{ + AFSqlDestDriver *self = (AFSqlDestDriver *) s; + + g_hash_table_insert(self->dbd_options_numeric, g_strdup(name), GINT_TO_POINTER(value)); +} + +void afsql_dd_set_type(LogDriver *s, const gchar *type) { AFSqlDestDriver *self = (AFSqlDestDriver *) s; @@ -608,6 +627,19 @@ afsql_dd_disconnect(AFSqlDestDriver *self) g_hash_table_remove_all(self->validated_tables); } +static void +afsql_dd_set_dbd_opt(gpointer key, gpointer value, gpointer user_data) +{ + dbi_conn_set_option((dbi_conn)user_data, (gchar *)key, (gchar *)value); +} + +static void +afsql_dd_set_dbd_opt_numeric(gpointer key, gpointer value, gpointer user_data) +{ + dbi_conn_set_option_numeric((dbi_conn)user_data, (gchar *)key, + GPOINTER_TO_INT(value)); +} + /** * afsql_dd_insert_db: * @@ -645,6 +677,10 @@ afsql_dd_insert_db(AFSqlDestDriver *self) dbi_conn_set_option(self->dbi_ctx, "sqlite_dbdir", ""); dbi_conn_set_option(self->dbi_ctx, "sqlite3_dbdir", ""); + /* Set user-specified options */ + g_hash_table_foreach(self->dbd_options, afsql_dd_set_dbd_opt, self->dbi_ctx); + g_hash_table_foreach(self->dbd_options_numeric, afsql_dd_set_dbd_opt_numeric, self->dbi_ctx); + if (dbi_conn_connect(self->dbi_ctx) < 0) { const gchar *dbi_error; @@ -1200,6 +1236,8 @@ afsql_dd_free(LogPipe *s) string_list_free(self->values); log_template_unref(self->table); g_hash_table_destroy(self->validated_tables); + g_hash_table_destroy(self->dbd_options); + g_hash_table_destroy(self->dbd_options_numeric); if(self->session_statements) string_list_free(self->session_statements); log_dest_driver_free(s); @@ -1235,6 +1273,8 @@ afsql_dd_new(void) self->num_retries = MAX_FAILED_ATTEMPTS; self->validated_tables = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, NULL); + self->dbd_options = g_hash_table_new_full(g_str_hash, g_str_equal, g_free, g_free); + self->dbd_options_numeric = g_hash_table_new_full(g_str_hash, g_int_equal, g_free, NULL); log_template_options_defaults(&self->template_options); init_sequence_number(&self->seq_num); diff --git a/modules/afsql/afsql.h b/modules/afsql/afsql.h index ee2ae68..ce2e18a 100644 --- a/modules/afsql/afsql.h +++ b/modules/afsql/afsql.h @@ -56,6 +56,8 @@ void afsql_dd_set_flags(LogDriver *s, gint flags); LogDriver *afsql_dd_new(); gint afsql_dd_lookup_flag(const gchar *flag); void afsql_dd_set_retries(LogDriver *s, gint num_retries); +void afsql_dd_add_dbd_option(LogDriver *s, const gchar *name, const gchar *value); +void afsql_dd_add_dbd_option_numeric(LogDriver *s, const gchar *name, gint value); #else @@ -69,6 +71,8 @@ void afsql_dd_set_retries(LogDriver *s, gint num_retries); #define afsql_dd_set_columns(s, c) #define afsql_dd_set_values(s, v) #define afsql_dd_set_null_value(s, v) +#define afsql_dd_add_dbd_option(s, n, v) +#define afsql_dd_add_dbd_option_numeric(s, n, v) #define afsql_dd_new() 0 -- 1.7.7.3

2 4

Minimum Syslog Message Length
by vincent＠ragosta.net 22 Dec '11

22 Dec '11

Is it possible to specify a restriction on the minimum length of a syslog message? We often receive small, malformed messages that we do not want to transmit. Thanks, Vincent

3 3

trap divide error
by Dmitry Gilev 22 Dec '11

22 Dec '11

Hi, I've started to get the following messages since upgrade to 3.3.3 (from 3.0.7: >From dmesg logs: syslog-ng[30744] trap divide error ip:7fa0f8ed5a56 sp:7fa0f64423a0 error:0 in libsyslog-ng-3.3.3.so[7fa0f8ea3000+7b000] syslog-ng[30798] trap divide error ip:7fa0f8ed5a56 sp:7fa0df5fb3a0 error:0 in libsyslog-ng-3.3.3.so[7fa0f8ea3000+7b000] syslog-ng[31043] trap divide error ip:7fa0f8ed5a56 sp:7fa0d2bfa3a0 error:0 in libsyslog-ng-3.3.3.so[7fa0f8ea3000+7b000] syslog-ng[31059] trap divide error ip:7fa0f8ed5a56 sp:7fa0e6bfa3a0 error:0 in libsyslog-ng-3.3.3.so[7fa0f8ea3000+7b000] syslog-ng[31087] trap divide error ip:7fa0f8ed5a56 sp:7fa0de1f93a0 error:0 in libsyslog-ng-3.3.3.so[7fa0f8ea3000+7b000] syslog-ng[31145] trap divide error ip:7fa0f8ed5a56 sp:7fa0f64423a0 error:0 in libsyslog-ng-3.3.3.so[7fa0f8ea3000+7b000] syslog-ng[32733] trap divide error ip:7fa0f8ed5a56 sp:7fa0f64423a0 error:0 in libsyslog-ng-3.3.3.so[7fa0f8ea3000+7b000] syslog-ng[1404] trap divide error ip:7fa0f8ed5a56 sp:7fa0f6e433a0 error:0 in libsyslog-ng-3.3.3.so[7fa0f8ea3000+7b000] syslog-ng[1531] trap divide error ip:7fa0f8ed5a56 sp:7fa0f64423a0 error:0 in libsyslog-ng-3.3.3.so[7fa0f8ea3000+7b000] syslog-ng[1688] trap divide error ip:7fa0f8ed5a56 sp:7fa0e7ffc3a0 error:0 in libsyslog-ng-3.3.3.so[7fa0f8ea3000+7b000] syslog-ng[2013] trap divide error ip:7fa0f8ed5a56 sp:7fa0f4c2c3a0 error:0 in libsyslog-ng-3.3.3.so[7fa0f8ea3000+7b000] syslog-ng[2019] trap divide error ip:7fa0f8ed5a56 sp:7fa0f6e433a0 error:0 in libsyslog-ng-3.3.3.so[7fa0f8ea3000+7b000] Does anybody know what is wrong here? Is it known bug? # syslog-ng -V syslog-ng 3.3.3 Installer-Version: 3.3.3 Revision: ssh+git://bazsi@git.balabit//var/scm/git/syslog-ng/syslog-ng-ose--mainline--3.3#master#d199a1980be6b23fe24189e86a882812288e292c Compile-Date: Dec 20 2011 15:17:13 Default-Modules: affile,afprog,afsocket,afuser,basicfuncs,csvparser,dbparser,syslogformat Available-Modules: basicfuncs,confgen,affile,csvparser,dbparser,afsocket,dummy,afsocket-notls,syslogformat,afprog,convertfuncs,afuser Enable-Debug: off Enable-GProf: off Enable-Memtrace: off Enable-IPv6: on Enable-Spoof-Source: on Enable-TCP-Wrapper: off Enable-Linux-Caps: off Enable-Pcre: off # ldd libsyslog-ng-3.3.3.so linux-vdso.so.1 => (0x00007fffc41fc000) libnsl.so.1 => /lib64/libnsl.so.1 (0x00007f42f3d64000) libgmodule-2.0.so.0 => /lib64/libgmodule-2.0.so.0 (0x00007f42f3b61000) libgthread-2.0.so.0 => /lib64/libgthread-2.0.so.0 (0x00007f42f395c000) librt.so.1 => /lib64/librt.so.1 (0x00007f42f3754000) libglib-2.0.so.0 => /lib64/libglib-2.0.so.0 (0x00007f42f346e000) libevtlog.so.0 => /lib64/libevtlog.so.0 (0x00007f42f3269000) libdl.so.2 => /lib64/libdl.so.2 (0x00007f42f3065000) libpthread.so.0 => /lib64/libpthread.so.0 (0x00007f42f2e49000) libc.so.6 => /lib64/libc.so.6 (0x00007f42f2aa7000) /lib64/ld-linux-x86-64.so.2 (0x00000035d0200000) # uname -a Linux xxx.xxx.com 2.6.32-220.el6.x86_64 #1 SMP Tue Dec 6 19:48:22 GMT 2011 x86_64 x86_64 x86_64 GNU/Linux (CentOS 6.2) Thanks! Dmitry

3 4

patterndb & not describing the entire log message
by Christophe Brocas 22 Dec '11

22 Dec '11

Hello As I said before in other message, I currently work with patterndb to write a pattern file describing Windows log messages related to file system. These messages are sent to Syslog-NG through Snare software. Example : Aug 12 15:08:48 10.0.0.1 MSWinEventLog 1 Security 541 ven. août 12 15:08:46 201 4656 Microsoft-Windows-Security-Auditing D00000001\accout1 N/A Success Audit hostname.d00000001.subdom1.dom1.fr Système de fichiers Un handle vers un objet a été demandé. Sujet : ID de sécurité : S-1-5-21-514083418-1667312537-733111063-1179 Nom du compte : account1 Domaine du compte : D00000001 ID d’ouverture de session : 0x8e90858 Objet : Serveur de l’objet : Security Type d’objet : File Nom de l’objet : E:\xxx\yyyy\filename.doc.doc ID du handle : 0x2b0 Informations sur le processus : ID du processus : 0xeb0 Nom du processus : C:\Windows\System32\inetsrv\w3wp.exe Informations sur la demande d’accès : ID de la transaction : {00000000-0000-0000-0000-000000000000} Accès : DELETE SYNCHRONIZE ReadAttributes Masque d’accès : 0x110080 Privilèges utilisés pour les vérifications d’accès : - Nombre de SID restreints : 0 540 Question : That kind of messages are extremely long and some areas seem empty in my example but may be not in the future. Is there a way with pattern to not describe the entire message but only the relevant part for us (the equivalent of wildcard in regexp) ? Thanks Christophe ***************************************************** "Le contenu de ce courriel et ses éventuelles pièces jointes sont confidentiels. Ils s'adressent exclusivement à la personne destinataire. Si cet envoi ne vous est pas destiné, ou si vous l'avez reçu par erreur, et afin de ne pas violer le secret des correspondances, vous ne devez pas le transmettre à d'autres personnes ni le reproduire. Merci de le renvoyer à l'émetteur et de le détruire. Attention : L'organisme de l'émetteur du message ne pourra être tenu responsable de l'altération du présent courriel. Il appartient au destinataire de vérifier que les messages et pièces jointes reçus ne contiennent pas de virus. Les opinions contenues dans ce courriel et ses éventuelles pièces jointes sont celles de l'émetteur. Elles ne reflètent pas la position de l'organisme sauf s'il en est disposé autrement dans le présent courriel." ******************************************************

4 7

Message parameter substitution
by Anup Shetty 21 Dec '11

21 Dec '11

I am new to syslog-ng and would like some help on the pattern matching and the substitution option. Currently the requirement is to substitute a parameter in the message with a random value in order to anonymize it. *For example:* Dec 31 23:13:25 servername sshd[25218]: Failed keyboard-interactive/pam for *user1* from 10.x.x.x port 47325 ssh2 If I create a pattern database for this message and pick out the username using the string and substitute it user1 to say anon1, will I be able to store the original-substituted value pair for this user and use it repeatedly? Would I be able to do it for all the subsequent logs? To be more clear, an example substitution process that must happen as the logs arrive and the patterns are matched. log with user1 arrives and is substituted by anon1 log with user2 arrives and is substituted by anon2 again log with user1 arrives and is again substituted by anon1 log with user3 arrives and is substituted by anon3 again log with user2 arrives and is again substituted by anon2 . . . . This is required so that once the usernames are substituted for attaining anonymity, there must be a way to reverse them for audit purposes. -- Thanks and regards, AS

3 5

Can I put all the parsed fields into a destination template?
by VÁMOS Balázs 21 Dec '11

21 Dec '11

Hi, I would like to forward the parsed fields to LOGalyze in an unstructured way. Is it possible to create a serialized object (JSON or something like that) after parsing message with patterndb? -- VÁMOS Balázs LOGalyze

4 4

syslog-ng 3.3.1 quits at reload
by Michael Hocke 21 Dec '11

21 Dec '11

-----BEGIN PGP SIGNED MESSAGE----- Hi there, I am having a problem with syslog-ng 3.3.1. Once in a while syslog-ng quits with a QUIT signal after it was asked to reload the configuration through a HUP signal. I have two instances of it running in separate Solaris 10 zones. Both are extremely busy and both crashed once within the last two weeks. By looking at the code in lib/mainloop.c I found the place where syslog-ng quits. It is within main_loop_reload_config_apply caused by a corrupt old configuration. Here is the backtrace with dbx: - ------< SNIP >------- (dbx) where current thread: t@1 =>[1] _kill(0x347f, 0x3, 0x0, 0xfef557d2, 0x80ef41c, 0xfed6217c), at 0xfeb7af85 [2] main_loop_reload_config_apply(0x8047920, 0xfefad000, 0x8047908, 0xfef707f9, 0x80ef368, 0xfef92934, 0x0, 0x0, 0xfef706c9), at 0xfef70595 [3] main_loop_io_worker_sync_call(0x0, 0x8047920, 0x0, 0x0, 0x0, 0x0), at 0xfef70742 [4] iv_signal_event(0xfefad000, 0x8047960, 0x400, 0x796c746e, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0), at 0xfef8d958 [5] iv_event_raw_got_event(0xfefad018, 0x8047db0, 0xfe9, 0xfefaa468, 0x2, 0x1), at 0xfef8d54a [6] iv_main(0xfefad180, 0xfef9291e, 0x804c2e8, 0x0, 0x804c7b0, 0x8047e1c), at 0xfef8ac82 [7] main_loop_run(0x8049cd7, 0x8047e44, 0x8047e48, 0x8047e2c, 0x8047e30, 0x8049390), at 0xfef71259 [8] main(0x1, 0x8047e60, 0x8047e70), at 0x80490fa - ------< SNIP >------- I used to run 3.0.6 before without a problem. The configuration of the server that crashed last night is pretty vanilla: - ------< SNIP >------- @version: 3.3 @include "scl.conf" options { time_reopen (10); chain_hostnames (no); use_dns (no); use_fqdn (no); create_dirs (no); keep_hostname (yes); perm (0644); }; # syslog-ng internal logging source s_local { internal (); }; # local syslogging only source s_sys { system (); }; destination d_sysmsg { file ("/dev/sysmsg"); }; destination d_local { file ("/var/log/syslog-ng.log"); }; destination d_messages { file ("/var/adm/messages"); }; destination d_mail { file ("/var/log/mail.log"); }; destination d_daemon { file ("/var/log/daemon.log"); }; destination d_auth { file ("/var/log/auth.log"); }; destination d_cron { file ("/var/log/cron.log"); }; destination d_user { file ("/var/log/user.log"); }; destination d_lpr { file ("/var/log/lpr.log"); }; destination d_news { file ("/var/log/news.log"); }; destination d_uucp { file ("/var/log/uucp.log"); }; destination d_local0 { file ("/var/log/ipfilter.log"); }; destination d_local1 { file ("/var/log/tcpwrapper.log"); }; destination d_local2 { file ("/var/log/tripwire.log"); }; destination d_local3 { file ("/var/log/local3.log"); }; destination d_local4 { file ("/var/log/local4.log"); }; destination d_local5 { file ("/var/log/local5.log"); }; destination d_local6 { file ("/var/log/local6.log"); }; destination d_local7 { file ("/var/log/local7.log"); }; destination d_snmptraps { file ("/usr/local/var/log/snmptrapd.${R_YEAR}${R_MONTH}${R_DAY}${R_HOUR}.log" group ("snmptrap") perm (416)); }; # this syslog config send a copy of every message to a Splunk server destination d_archive { udp ("x.x.x.x"); }; filter f_sysmsg { level (notice) and facility (kern); }; filter f_messages { facility (kern); }; filter f_mail { facility (mail); }; filter f_daemon { facility (daemon); }; filter f_auth { facility (auth); }; filter f_cron { facility (cron); }; filter f_user { facility (user); }; filter f_lpr { facility (lpr); }; filter f_news { facility (news); }; filter f_uucp { facility (uucp); }; filter f_local0 { facility (local0); }; filter f_local1 { facility (local1); }; filter f_local2 { facility (local2); }; filter f_local3 { facility (local3); }; filter f_local4 { facility (local4); }; filter f_local5 { facility (local5); }; filter f_local6 { facility (local6); }; filter f_local7 { facility (local7); }; log { source (s_local); destination (d_local); }; log { source (s_sys); filter (f_sysmsg); destination (d_sysmsg); destination (d_archive); }; log { source (s_sys); filter (f_messages); destination (d_messages); destination (d_archive); }; log { source (s_sys); filter (f_mail); destination (d_mail); destination (d_archive); }; log { source (s_sys); filter (f_daemon); destination (d_daemon); destination (d_archive); }; log { source (s_sys); filter (f_auth); destination (d_auth); destination (d_archive); }; log { source (s_sys); filter (f_cron); destination (d_cron); destination (d_archive); }; log { source (s_sys); filter (f_user); destination (d_snmptraps); }; log { source (s_sys); filter (f_lpr); destination (d_lpr); destination (d_archive); }; log { source (s_sys); filter (f_news); destination (d_news); destination (d_archive); }; log { source (s_sys); filter (f_uucp); destination (d_uucp); destination (d_archive); }; log { source (s_sys); filter (f_user); destination (d_snmptraps); }; log { source (s_sys); filter (f_lpr); destination (d_lpr); destination (d_archive); }; log { source (s_sys); filter (f_news); destination (d_news); destination (d_archive); }; log { source (s_sys); filter (f_uucp); destination (d_uucp); destination (d_archive); }; log { source (s_sys); filter (f_local0); destination (d_local0); destination (d_archive); }; log { source (s_sys); filter (f_local1); destination (d_local1); destination (d_archive); }; log { source (s_sys); filter (f_local2); destination (d_local2); destination (d_archive); }; log { source (s_sys); filter (f_local3); destination (d_local3); destination (d_archive); }; log { source (s_sys); filter (f_local4); destination (d_local4); destination (d_archive); }; log { source (s_sys); filter (f_local5); destination (d_local5); destination (d_archive); }; log { source (s_sys); filter (f_local6); destination (d_local6); destination (d_archive); }; log { source (s_sys); filter (f_local7); destination (d_local7); destination (d_archive); }; - ------< SNIP >------- Anybody else having this problem? Any hope we can nail this down? - - Michael -----BEGIN PGP SIGNATURE----- Version: PGP Desktop 10.0.3 (Build 1) Charset: us-ascii wsBVAwUBTrmaT5bfnpCg64TVAQE3JAf+KCFVzUL/MGWTgMqRLowtYRDhT8NtThnL p4bRqDqggXjJR8eQhY8dRKyi7zSN+T59WnGunCA2WD98hX5+bUWtyR3yVs5GH4KK XplVrpoPzJNOQFnPrjnrg0VGahO83XMO0nIcS5HdgId8hsGgKKknkWosDTFl97J+ IPxQeqoVuj92DT9MSm00z6SLrX9XtD4dK762zcQ2Tsvz9j9KW21jiWo9sUor6Lpe Dh4Mle0V/dGJOBi7j7UiQOSQCRikl20zTZtId7wsFqsDVGSFlOKTvW+l3XLok0KU ufVnI46N8mrnUWFqDU5SQ952gelIsIAdj6UUwKioZQMafeZvvRLwFA== =9NxF -----END PGP SIGNATURE-----

3 7

Syslog-ng Message Statistics
by vincent＠ragosta.net 21 Dec '11

21 Dec '11

Some of our older systems are utilizing syslog-ng 1.6.12 and I am interested in gathering log counts for each unique source that is reporting. I don't believe there is any mechanism in this version of syslog-ng for performing such analysis, correct? Thanks.

2 1

syslog-ng remote MSSQL databases
by Bely Silva Junior 21 Dec '11

21 Dec '11

dear Srs I need configure syslog-ng insert data on remote MSSQL database, it's possible ? my conf: destination d_mssql { sql(type(mssql) host("ipaddres") port("1433") username("syslogng") password("password") database("syslogng") table("logs") columns("datetime varchar(16)","host varchar(32)" ,"program varchar(32)","pid varchar(8)","message varchar(4096)") values("$R_DATE", "$HOST", "$PROGRAM", "$PID", "$MSGONLY") indexes() ); }; I had error when restart syslog-ng say : syntax erro -- Bely Silva Junior

2 1

Invalid frame header messages
by Mr. Jan Walter 21 Dec '11

21 Dec '11

Hi everyone, I compiled version 3.3.3 and tried getting a client and server to talk based on the documentation on the balabit site, specifically the documentation for OSE version 3.3. The server config file: -- source remotelogs { syslog( port(2072) transport(tcp) ); } ; destination r_log { file("/var/log/remotelogs/all"); }; log{ source(remotelogs); destination(r_log); } ; -- And the client sending: -- # remote logging test log # @version: 3.3 source s_kern { file("/proc/kmsg"); }; source s_local { unix-stream("/dev/log"); internal(); }; destination d_file {file("/var/log/messages"); }; destination d_secure { file("/var/log/secure");}; # set the filters filter f_messages { not facility(auth, authpriv);}; filter f_authpriv { facility(auth, authpriv); }; destination loghost { tcp(10.195.105.214 port(2072) ); }; log { source(s_kern) ; source(s_local) ; filter(f_messages) ; destination(loghost) ; } ; -- There are some other log directives there for local files, I'll post them if people want to see them So when this runs, I get on the server: Dec 7 11:18:05 xxxxxxxxxx syslog-ng[4188]: Syslog connection closed; fd='21', client='AF_INET(10.203.54.9:54837)', local='AF_INET(0.0.0.0:2072)' And the client: Dec 7 11:19:04 xxxxxxxxxxxx syslog-ng[1735]: Syslog connection established; fd='20', server='AF_INET(10.195.105.214:2072)', local='AF_INET(0.0.0.0:0)' I read the email thread from 2009, and tried the vanilla configs - any ideas? Thanks, Jan

3 4

syslog-ng 3.3.3 pipe source does not parse host
by Evan Rempel 21 Dec '11

21 Dec '11

Try as I might, I can not get a pipe source to maintain the hostname. Syslog-ng includes the hostname as the $MSGHDR (or the $PROGRAM). my config # ----------------------------------- source patterndb { pipe("/var/log/syslog.pipes/patterndb" log_fetch_limit(500) log_iw_size(100000) ); }; source int { internal(); }; template t_standardfile { template("$S_ISODATE $FULLHOST $FACILITY.$LEVEL $MSGHDR][$PROGRAM][$MESSAGE\n"); template_escape(no); }; destination d_var_patterndb { file("/var/log/patterndb.$R_YEAR$R_MONTH$R_DAY.000000" owner("root") group("syslogs") perm(0640) template(t_standardfile)); }; log { source(patterndb); destination(d_var_patterndb); }; # ----------------------------------- data I write tothe pipe # ----------------------------------- <12>2011-11-25T00:00:30-08:00 somehost.uvic.ca mmfs: Fri Nov 25 00:00:29.618 2011: Accepted and connected to 172.20.102.38 hermes0080 <c0n350> <12>2011-11-25T00:00:30-08:00 somehost.uvic.ca mmfs: Fri Nov 25 00:00:29.620 2011: Connecting to 172.20.107.23 nestor0167 <c0n200> <12>2011-11-25T00:00:30-08:00 somehost.uvic.ca mmfs: Fri Nov 25 00:00:29.621 2011: Connected to 172.20.107.23 nestor0167 <c0n200> # ----------------------------------- The output file # ----------------------------------- 2011-11-25T00:00:30-08:00 patterndb(a)catamount.comp.uvic.ca user.warning somehost.uvic.ca ][somehost.uvic.ca][mmfs: Fri Nov 25 00:00:29.618 2011: Accepted and connected to 172.20.102.38 hermes0080 <c0n350> 2011-11-25T00:00:30-08:00 patterndb(a)catamount.comp.uvic.ca user.warning somehost.uvic.ca ][somehost.uvic.ca][mmfs: Fri Nov 25 00:00:29.620 2011: Connecting to 172.20.107.23 nestor0167 <c0n200> 2011-11-25T00:00:30-08:00 patterndb(a)catamount.comp.uvic.ca user.warning somehost.uvic.ca ][somehost.uvic.ca][mmfs: Fri Nov 25 00:00:29.621 2011: Connected to 172.20.107.23 nestor0167 <c0n200> # ----------------------------------- Am I missing something silly with the 3.3 config syntax (Up until now I have only used up to 3.0) Evan.

2 1

[Bug 153] New: default year when no year is present should be current year
by bugzilla＠bugzilla.balabit.com 21 Dec '11

21 Dec '11

https://bugzilla.balabit.com/show_bug.cgi?id=153 Summary: default year when no year is present should be current year Product: syslog-ng Version: 3.3.x Platform: PC OS/Version: Linux Status: NEW Severity: major Priority: unspecified Component: syslog-ng AssignedTo: bazsi(a)balabit.hu ReportedBy: erempel(a)uvic.ca Type of the Report: --- Estimated Hours: 0.0 When parsing a source that has a month and day but no year, the S_YEAR macro does not default to R_YEAR. It seems to be defaulting to R_YEAR+1 What is the intention when there is no year in the source? To test, use historicatl syslog or rsyslog to create a syslog file, then use that file as a source (I was doing a "cat filename > /smoe/pipe" and using a pipe source) and write a file that has an ISODate in it. The year is 2012. -- Configure bugmail: https://bugzilla.balabit.com/userprefs.cgi?tab=email ------- You are receiving this mail because: ------- You are watching all bug changes.

1 1

syslog-ng 3.3.3 and multiple incarnations of the same parser
by Michael Hocke 21 Dec '11

21 Dec '11

-----BEGIN PGP SIGNED MESSAGE----- Hi guys, I ran into another problem with 3.3.3. It seems that I can use a cvs-parser only in one and only one log path. If I reuse it in multiple log paths the columns of the cvs parser are empty. I downgraded to 3.0.9 and that resolved my problem. Here is an example config: - --------< SNIP >---------- source s_remote { udp () }; parser p_hostname_site { csv-parser (columns ("REMOTEHOST.SITE", "REMOTEHOST.NAME") delimiters ("-") template ("${HOST}")); }; filter f_local0 { facility (local0); }; filter f_local1 { facility (local1); }; destination d_cacl { file ("/usr/local/var/log/current/${REMOTEHOST.SITE:-other}/cisco-acl.${S_YEAR}${S_MONTH}${S_DAY}${S_HOUR}.log" template ("$S_DATE ${REMOTEHOST.NAME:-none} $MSGHDR$MSG\n") }; destination d_caps { file ("/usr/local/var/log/current/${REMOTEHOST.SITE:-other}/cisco-aps.${S_YEAR}${S_MONTH}${S_DAY}${S_HOUR}.log" template ("$S_DATE ${REMOTEHOST.NAME:-none} $MSGHDR$MSG\n") }; log (source (s_remote); filter (f_local0); parser (p_hostname_site); destination (d_cacl); }; log (source (s_remote); filter (f_local1); parser (p_hostname_site); destination (d_caps); }; - --------< SNIP >---------- The incoming logs for local0 would look like this: Dec 2 16:00:00 florence-VPN-FLORENCE %ASA-6-305011: Built dynamic TCP translation from inside-vlan765 Dec 2 16:00:00 florence-VPN-FLORENCE %ASA-6-302013: Built outbound TCP connection 100466601 for outside-vlan757 Under 3.3.3 it would produce the following logs: /usr/local/var/log/current/other/cisco-acl.2011120216.log: Dec 2 16:00:00 none %ASA-6-305011: Built dynamic TCP translation from inside-vlan765 Dec 2 16:00:00 none %ASA-6-302013: Built outbound TCP connection 100466601 for outside-vlan757 The same configuration with 3.0.9 produces the following: /usr/local/var/log/current/florence/cisco-acl.2011120216.log: Dec 2 16:00:00 VPN-FLORENCE %ASA-6-305011: Built dynamic TCP translation from inside-vlan765 Dec 2 16:00:00 VPN-FLORENCE %ASA-6-302013: Built outbound TCP connection 100466601 for outside-vlan757 If I remove the parser from one of the log paths 3.3.3 would spit out the same output as 3.0.9 but of course, the second log path would need its own parser to make it work. I am unable to check syslog-ng 3.1 or 3.2. - - Michael -----BEGIN PGP SIGNATURE----- Version: PGP Desktop 10.0.3 (Build 1) Charset: us-ascii wsBVAwUBTtjvwJbfnpCg64TVAQFmvwf9EZ3wShhBxIXOGXWiSJguF07HCgw99Wxu amUrvAAeI9nZSj4wvpQCZNLrpPJ2Q7/DCSKTunzopHNJVwMQQvFSwzaAcezNNt4Y ogmTAx+TeyMc4rYuknpG9Khz6WXFTM8rH4imjNziDSxSwkqZrCQDnmYM4B2pxfqo BNZbcRKifbo9mts0FbeD9hpBiYNxt+k9pXPtqUD193ihUWcnNF4NG3hgt/1RXB7E 7v79A6swf30PXSLpdJn8fATptKVrfzzvXlW0ncNolSUyKK7XaY4kgvjsnFba++CS eTil+xkINNnuUz2FDYfEDziTYFeezO/lhSrhE8BiDaRt1Gx10XfbDQ== =2Idy -----END PGP SIGNATURE-----

4 5

[PATCH] afprog: Terminate the child process on deinit.
by Gergely Nagy 21 Dec '11

21 Dec '11

When reloading the configuration, terminate the child process of a program destination, for the following reasons: it will be restarted anyway by afprog_dd_reopen(), so there's no harm done. But if dash is used as /bin/sh, simply closing the stdin of the process will not make it stop. That, in turn, results in us starting the same program over and over again on each SIGHUP. So instead of doing this, send a SIGTERM to the child on deinit, too. Reported-by: Thomas Wollner <tw(a)wollner-net.de> Signed-off-by: Gergely Nagy <algernon(a)balabit.hu> --- modules/afprog/afprog.c | 10 ++++++++++ 1 files changed, 10 insertions(+), 0 deletions(-) diff --git a/modules/afprog/afprog.c b/modules/afprog/afprog.c index 6ce74b0..264e85b 100644 --- a/modules/afprog/afprog.c +++ b/modules/afprog/afprog.c @@ -328,6 +328,16 @@ afprogram_dd_deinit(LogPipe *s) { AFProgramDestDriver *self = (AFProgramDestDriver *) s; + if (self->pid != -1) + { + msg_verbose("Sending destination program a TERM signal", + evt_tag_str("cmdline", self->cmdline->str), + evt_tag_int("child_pid", self->pid), + NULL); + kill(self->pid, SIGTERM); + self->pid = -1; + } + if (self->writer) log_pipe_deinit(self->writer); -- 1.7.7.3

2 1

patterndb - user defined parsers
by Evan Rempel 19 Dec '11

19 Dec '11

It would be useful to permit users to define parsers in the patterndb. For example, in our environment, by policy we user a special set and order of characters of our administrators log into hosts and administer them. It would be useful to define a parser of @SYSADMIN@ that would match only our sysadmin accounts. We could then use this parser in the patterndb to take some action such as sending a message to the administrators about the event. Another example would be to create parser for @LOCALIP@ that matches my organizaions IP space. That way a set of rules can be defined using @LOCALIP@ for some kind of alerting, and then any organization could redifine the @LOCALIP@ and use all of the goodness that some third party had created for monitoring logs like an intrusion protection system. Current parsers can be described as QSTRING - match opening char - while not closing char, keep looking ESTRING - while not end string, keep looking NUMBER - while digit keep looking So it seems that general parsers could be constructed with two styles of matching, and then concatenating the together. 1. While in set of characters [some list of characters] 2. While not in set of characters [some list of characters] I would call these INSET to match 1 or more of a set of characters, unless a #-# were specified, then a minimum to a maximum would be required. OUTSET to match 1 or more of anything except the characters, unless a #-# were specified, then a minimum to a maximum would be required. (perhaps a count of + or * could be used to specify 1 or more and 0 or more respectively) and then limit the count of such occurrences so that you could build the @IPv4@ parser as @INSET::123456789*1@@INSET::0123456789:0-2@.@INSET::123456789:1@@INSET::0123456789:0-2@.@INSET::123456789:1@@INSET::0123456789:0-2@.@INSET::123456789:1@@INSET::0123456789:0-2@ and @NUMBER@ would be @INSET::123456789:1@@INSET::0123456789@ @FLOAT@ would be @INSET::0123456789.@ Then a user could make <parser name="THOUSAND">@INSET::,:0-1@@INSET::0123456789:3@</parser> <parser name="MONEY">$@INSET::123456789:1-3@@THOUSAND:::*@.@INSET::0123456789:2@ This is kind of like inventing regular expressions :-( I'm not sure how well this fits into the radix tree matching structure, but I wanted to start this discussion. Given the MONEY example, I think it is obvious that there needs to be a way to specify repeating groups of "something" Let the discussion begin!

4 9

Re: [syslog-ng] Log only one host
by tokie＠tiscali.it 19 Dec '11

19 Dec '11

Hi, I wrong to write. filter is before to destination, but don't work!! Can you help me?? I would like log every host in one different file. Tks Tokie Message: 4 Date: Tue, 13 Dec 2011 20:47:21 +0100 From: Szalay Attila <sasa(a)balabit.hu> Subject: Re: [syslog-ng] Log only one host To: "tokie(a)tiscali.it" <tokie(a)tiscali.it>, Syslog-ng users' and developers' mailing list <syslog-ng(a)lists.balabit.hu> Message-ID: <1323805641.9268.6.camel(a)mochrul.balabit> Content-Type: text/plain; charset="UTF-8" Hi All! On Tue, 2011-12-13 at 15:39 +0100, tokie(a)tiscali.it wrote: > log { > source(udp); > > destination (d_fw); > filter(f_fw); > }; The log path is a sequence. If you first write the logs into the destination the filter will do nothing. You should change the destination and the filter statement. -- Attila Szalay Support (L3) Team Leader e-mail: attila.szalay(a)balabit.com phone: +36 1 398 6707 BalaBit IT Security www.balabit.com H-1117 Budapest, Aliz street 2. This Communication is Confidential. We only send and receive email on the basis of the term set out at http://www. balabit.com/disclaimer/. ------------------------------ _______________________________________________ syslog-ng maillist - syslog-ng(a)lists.balabit.hu https://lists.balabit. hu/mailman/listinfo/syslog-ng End of syslog-ng Digest, Vol 80, Issue 19 ***************************************** E' nata indoona: chiama, videochiama e messaggia Gratis. Scarica indoona per iPhone, Android e PC: http://www.indoona.com/

1 0

Feature Request - patterndb match set
by Evan Rempel 18 Dec '11

18 Dec '11

I have come across some odd lines that really can't be matched/parsed by the patterndb 2011-11-25T10:49:21-08:00 mmfs@hermes0022.westgrid.uvic.ca/localhost/hermes0022/xcat2.westgrid.uvic.ca local2.info mmfs: Module Size Used by 2011-11-25T10:49:21-08:00 mmfs@hermes0022.westgrid.uvic.ca/localhost/hermes0022/xcat2.westgrid.uvic.ca local2.info mmfs: mmfs26 1945576 0 2011-11-25T10:49:21-08:00 mmfs@hermes0022.westgrid.uvic.ca/localhost/hermes0022/xcat2.westgrid.uvic.ca local2.info mmfs: mmfslinux 326280 1 mmfs26 2011-11-25T10:49:21-08:00 mmfs@hermes0022.westgrid.uvic.ca/localhost/hermes0022/xcat2.westgrid.uvic.ca local2.info mmfs: tracedev 67148 2 mmf I would like to match these and parse out the number. The catch is that the number is right justified which means that there is a variable number of spaces before the number. I am open to suggestions about how to make a paterndb pattern to match this and parse the number into a tag/value pair. Failing that I would propose that a @SET@ parser. @SET:name:character set@ This will match a sequence of characters that contain any of, and only those characters listed by "character set" This will allow matches of arbitrary length separators such as spaces or hyphens or other cases that can not yet be handled. Comments? Evan

5 6

[PATCH] [dbparser] SET parser
by Balint Kovacs 18 Dec '11

18 Dec '11

Hi, based on Evan's idea, please find a patch attached, that implements the SET parser that matches any number of characters specified as its parameters. Balint commit ad5da5958104a1dcf5e2492370cbb7037815c3ae Author: Balint Kovacs <blint(a)balabit.hu> Date: Sun Nov 27 18:40:28 2011 +0100 [dbparser] Implemented SET parser The SET parser matches any characters specified in its parameter list without further checks. This is a generic solution for the need of matching and arbitrary number of whitespaces in messages with padding for fields Reported-By: Evan Rempel <erempel(a)uvic.ca> Signed-Off-By: Balint Kovacs <blint(a)balabit.hu> diff --git a/modules/dbparser/radix.c b/modules/dbparser/radix.c index 6afa9a4..8c43709 100644 --- a/modules/dbparser/radix.c +++ b/modules/dbparser/radix.c @@ -116,6 +116,21 @@ r_parser_anystring(guint8 *str, gint *len, const gchar *param, gpointer state, R } gboolean +r_parser_set(guint8 *str, gint *len, const gchar *param, gpointer state, RParserMatch *match) +{ + *len = 0; + + while (param && strchr(param, str[*len])) + (*len)++; + + if (*len > 0) + { + return TRUE; + } + return FALSE; +} + +gboolean r_parser_ipv4(guint8 *str, gint *len, const gchar *param, gpointer state, RParserMatch *match) { gint dots = 0; @@ -379,6 +394,21 @@ r_new_pnode(guint8 *key) parser_node->parse = r_parser_anystring; parser_node->type = RPT_ANYSTRING; } + else if (strcmp(params[0], "SET") == 0) + { + if (params_len == 3) + { + parser_node->parse = r_parser_estring_c; + parser_node->type = RPT_ESTRING; + } + else + { + g_free(parser_node); + msg_error("Missing SET parser parameters", + evt_tag_str("type", params[0]), NULL); + parser_node = NULL; + } + } else if (g_str_has_prefix(params[0], "QSTRING")) { if (params_len == 3) diff --git a/modules/dbparser/radix.h b/modules/dbparser/radix.h index 9082419..9756dfe 100644 --- a/modules/dbparser/radix.h +++ b/modules/dbparser/radix.h @@ -40,7 +40,8 @@ enum RPT_ANYSTRING, RPT_IPV6, RPT_IP, - RPT_FLOAT + RPT_FLOAT, + RPT_SET }; typedef struct _RParserMatch @@ -124,6 +125,8 @@ r_parser_type_name(guint8 type) return "IP"; case RPT_FLOAT: return "FLOAT"; + case RPT_SET: + return "SET"; default: return "UNKNOWN"; }

2 2

[Bug 149] New: patterndb action message does not do variable substitution
by bugzilla＠bugzilla.balabit.com 17 Dec '11

17 Dec '11

https://bugzilla.balabit.com/show_bug.cgi?id=149 Summary: patterndb action message does not do variable substitution Product: syslog-ng Version: 3.3.x Platform: PC OS/Version: Linux Status: NEW Severity: normal Priority: unspecified Component: syslog-ng AssignedTo: bazsi(a)balabit.hu ReportedBy: erempel(a)uvic.ca Type of the Report: --- Estimated Hours: 0.0 When using a simple message action in a patterndb rule the message can not be populated with any variables. I am not using any correlation, I am just performing the message action every time I match the rule pattern. <rule id="463e34dd-f946-4f25-9cd2-9c00e25eba62" class="cluster" provider="UVic"> <patterns> <pattern>@ESTRING:: @@ESTRING:: @@NUMBER::@ @NUMBER@:@NUMBER@:@NUMBER@.@NUMBER@ @NUMBER@: VERBS RDMA library libibverbs.so failed to load: libibverbs.so: cannot open shared object file: No such file or directory.</pattern> </patterns> <actions> <action> <message> <values> <value name="MESSAGE">$.classifier.rule_id</value> <value name="PROGRAM">patterndb</value> </values> </message> </action> </actions> </rule> results in the verbatim log line of 2011-11-28T10:12:21-08:00 int(a)catamount.comp.uvic.ca kern.emerg patterndb: $.classifier.rule_id The log line should be 2011-11-28T10:12:21-08:00 int(a)catamount.comp.uvic.ca kern.emerg patterndb: 463e34dd-f946-4f25-9cd2-9c00e25eba62 -- Configure bugmail: https://bugzilla.balabit.com/userprefs.cgi?tab=email ------- You are receiving this mail because: ------- You are watching all bug changes.

4 15

Syslog daemon restart fails on AIX 7.1 in central server
by Rakesh Rajasekharan 16 Dec '11

16 Dec '11

Hi, I am faced with a rather strange issue with syslog 3.0.7 version. The daemon does not start at all when I try to run it in the central server mode. I though do not get any error messages . But, I noticed that after a reboot of the machine. Everything works fine. Can someone please suggest of any possible areas to check. Thanks, Rakesh

2 2

web interface
by Bely Silva Junior 15 Dec '11

15 Dec '11

Hi, I see phpsyslog-ng was deprecated, another option what would be a choice for this ? logzilla is a good idea? -- Bely Silva Junior

4 5

Problem instert data MSSQL
by Bely Silva Junior 14 Dec '11

14 Dec '11

hi i have problem when i try insert data in mssql server, when i try run script, just 2 or 3 lines insert ok, but , insert stop . I use pipe, my confs. destination d_mssql { pipe("/var/mssql.syslog-ng.pipe" template( "INSERT INTO logs (host, facility, priority, level, tag, datetime, program, msg ) VALUES ( '$HOST','$FACILITY','$PRIORITY','$LEVEL','$TAG','$YEAR-$MONTH-$DAY $HOUR:$MIN:$SEC','$PROGRAM','$MSG' );\n" ) template-escape(yes)); template-escape(yes)); }; log { source(s_all); destination(d_mssql); }; my script: if [ -e /var/mssql.syslog-ng.pipe ]; then while [ -e /var/mssql.syslog-ng.pipe ] do #isql -v MSSQL "syslogng" "h0hYgbDB6Q" < /var/mssql.syslog-ng.pipe > /dev/null isql -v MSSQL "syslogng" "h0hYgbDB6Q" < /var/mssql.syslog-ng.pipe done else mkfifo /var/mssql.syslog-ng.pipe fi when i try run script: ./syslog2mssql.sh +---------------------------------------+ | Connected! | | | | sql-statement | | help [tablename] | | quit | | | +---------------------------------------+ SQL> SQLRowCount returns 1 SQL> +------------+ | | +------------+ +------------+ SQLRowCount returns 0 SQL> SQLRowCount returns -1 SQL> SQLRowCount returns 1 SQL> [ISQL]ERROR: Could not SQLPrepare SQL> [ISQL]ERROR: Could not SQLPrepare./syslog2mssql.sh +---------------------------------------+ | Connected! | | | | sql-statement | | help [tablename] | | quit | | | +---------------------------------------+ SQL> SQLRowCount returns 1 SQL> +------------+ | | +------------+ +------------+ SQLRowCount returns 0 SQL> SQLRowCount returns -1 SQL> SQLRowCount returns 1 SQL> [ISQL]ERROR: Could not SQLPrepare SQL> [ISQL]ERROR: Could not SQLPrepare someone have idea for this ? -- Bely Silva Junior

2 1

syslog-ng article in BSDmag, not just for BSD users
by Peter Czanik 14 Dec '11

14 Dec '11

Hello, Robert Fekete, our technical writer wrote a very nice article for BSDmag about syslog-ng with the title: "Extracting Useful Information From Log Messages". The following topics are addressed: • The message model behind syslog messages • The importance of metadata • How to process, recognize, and correlate log messages using syslog-ng If you want just this single article, you can download it from http://www.balabit.com/support/documentation/BSD_12_2011%20-%20BalaBit_sing… Other way I'd recommend to download the complete December issue of BSDmag from http://bsdmag.org/magazine/1781-rolling-your-own-kernel Bye, -- Peter Czanik (CzP)<czanik(a)balabit.hu> BalaBit IT Security / syslog-ng upstream http://czanik.blogs.balabit.com/

1 0

GUI Web Interface for Syslog NG
by Kaushal Shriyan 14 Dec '11

14 Dec '11

Hi I happen to see that http://code.google.com/p/php-syslog-ng/ Php-syslog-ng is now known as LogZilla <http://www.logzilla.pro/>. Same owner, better code :-). But to my surprise,LogZilla is available for 30 days evaluation period. is there a free version available or a equivalent GUI Web Interface for Syslog NG Regards Kaushal

4 3