- syslog-ng - lists.balabit.hu

[PATCH (3.4)] json: The test should not depend on the environment's timezone
by Gergely Nagy 13 Oct '12

13 Oct '12

The $(format-json) test was failing when a timezone was set in the environment that did not match the tests expectations. To fix this, explicitly set the timezone to UTC. Signed-off-by: Gergely Nagy <algernon(a)balabit.hu> --- modules/json/tests/test_json.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/modules/json/tests/test_json.c b/modules/json/tests/test_json.c index f188b7f..15d038b 100644 --- a/modules/json/tests/test_json.c +++ b/modules/json/tests/test_json.c @@ -7,7 +7,7 @@ test_format_json(void) { assert_template_format("$(format-json MSG=$MSG)", "{\"MSG\":\"árvíztűrőtükörfúrógép\"}"); assert_template_format_with_context("$(format-json MSG=$MSG)", "{\"MSG\":\"árvíztűrőtükörfúrógép\"}{\"MSG\":\"árvíztűrőtükörfúrógép\"}"); - assert_template_format("$(format-json --scope rfc3164)", "{\"PROGRAM\":\"syslog-ng\",\"PRIORITY\":\"err\",\"PID\":\"23323\",\"MESSAGE\":\"árvíztűrőtükörfúrógép\",\"HOST\":\"bzorp\",\"FACILITY\":\"local3\",\"DATE\":\"Feb 11 19:58:35\"}"); + assert_template_format("$(format-json --scope rfc3164)", "{\"PROGRAM\":\"syslog-ng\",\"PRIORITY\":\"err\",\"PID\":\"23323\",\"MESSAGE\":\"árvíztűrőtükörfúrógép\",\"HOST\":\"bzorp\",\"FACILITY\":\"local3\",\"DATE\":\"Feb 11 18:58:35\"}"); assert_template_format("$(format-json msg.text=$MSG msg.id=42 host=bzorp)", "{\"msg\":{\"text\":\"árvíztűrőtükörfúrógép\",\"id\":\"42\"},\"host\":\"bzorp\"}"); assert_template_format("$(format-json msg.text.str=$MSG msg.text.len=42 msg.id=42 host=bzorp)", "{\"msg\":{\"text\":{\"str\":\"árvíztűrőtükörfúrógép\",\"len\":\"42\"},\"id\":\"42\"},\"host\":\"bzorp\"}"); @@ -17,6 +17,8 @@ int main(int argc G_GNUC_UNUSED, char *argv[] G_GNUC_UNUSED) { app_startup(); + putenv("TZ=UTC"); + tzset(); init_template_tests(); plugin_load_module("json-plugin", configuration, NULL); -- 1.7.10.4

1 0

[PATCH (3.4)] json-format: Unbreak $(format-json)
by Gergely Nagy 13 Oct '12

13 Oct '12

The $(format-json) template function was misbehaving under certain conditions, namely where we had a sub-object, a simple value, and another object starting right after: in this case, no comma was placed between the simple value and the name of the new sub-object. The fix changes how comma placement is done completely: it is no longer a mix of prefix_data & user_data magic, but all state is kept in the user data, with the _obj_start(), _obj_end() and _value() functions setting it as appropriate: - When starting a new object, if we need a comma, we'll put one, and turn the comma flag off (the first item within an object won't need a comma). - When ending an object, we never add a comma, but the next item will need the comma flag set. - When adding a value, if the comma flag is set, we add one, and in the end, set the flag ourselves, as the next one will need a comma too. This is much simpler than the previous hackery, and also results in correct JSON in every situation. Signed-off-by: Gergely Nagy <algernon(a)balabit.hu> --- modules/json/format-json.c | 27 +++++---------------------- modules/json/tests/test_json.c | 2 ++ 2 files changed, 7 insertions(+), 22 deletions(-) diff --git a/modules/json/format-json.c b/modules/json/format-json.c index 4f0f01c..e17bcb0 100644 --- a/modules/json/format-json.c +++ b/modules/json/format-json.c @@ -118,14 +118,8 @@ tf_json_obj_start(const gchar *name, gpointer user_data) { json_state_t *state = (json_state_t *)user_data; - gboolean need_comma = FALSE; - if (prefix_data) - need_comma = GPOINTER_TO_INT(*prefix_data); - else - need_comma = state->need_comma; - - if (need_comma) + if (state->need_comma) g_string_append_c(state->buffer, ','); if (name) @@ -133,13 +127,11 @@ tf_json_obj_start(const gchar *name, g_string_append_c(state->buffer, '"'); g_string_append_escaped(state->buffer, name); g_string_append(state->buffer, "\":{"); - state->need_comma = TRUE; } else g_string_append_c(state->buffer, '{'); - if (prefix_data) - *prefix_data=GINT_TO_POINTER(0); + state->need_comma = FALSE; return FALSE; } @@ -152,11 +144,10 @@ tf_json_obj_end(const gchar *name, { json_state_t *state = (json_state_t *)user_data; - if (prev_data) - *prev_data = GINT_TO_POINTER(1); - g_string_append_c(state->buffer, '}'); + state->need_comma = TRUE; + return FALSE; } @@ -164,18 +155,10 @@ static gboolean tf_json_value(const gchar *name, const gchar *prefix, const gchar *value, gpointer *prefix_data, gpointer user_data) { - gboolean need_comma = FALSE; json_state_t *state = (json_state_t *)user_data; - if (prefix_data) - need_comma = GPOINTER_TO_INT(*prefix_data); - else - need_comma = state->need_comma; - - if (need_comma) + if (state->need_comma) g_string_append_c(state->buffer, ','); - else if (prefix_data) - *prefix_data = GINT_TO_POINTER(1); g_string_append_c(state->buffer, '"'); g_string_append_escaped(state->buffer, name); diff --git a/modules/json/tests/test_json.c b/modules/json/tests/test_json.c index f188b7f..2a88c63 100644 --- a/modules/json/tests/test_json.c +++ b/modules/json/tests/test_json.c @@ -11,6 +11,8 @@ test_format_json(void) assert_template_format("$(format-json msg.text=$MSG msg.id=42 host=bzorp)", "{\"msg\":{\"text\":\"árvíztűrőtükörfúrógép\",\"id\":\"42\"},\"host\":\"bzorp\"}"); assert_template_format("$(format-json msg.text.str=$MSG msg.text.len=42 msg.id=42 host=bzorp)", "{\"msg\":{\"text\":{\"str\":\"árvíztűrőtükörfúrógép\",\"len\":\"42\"},\"id\":\"42\"},\"host\":\"bzorp\"}"); + assert_template_format("$(format-json kernel.SUBSYSTEM=pci kernel.DEVICE.type=pci kernel.DEVICE.name=0000:02:00.0 MSGID=801 MESSAGE=test)", + "{\"kernel\":{\"SUBSYSTEM\":\"pci\",\"DEVICE\":{\"type\":\"pci\",\"name\":\"0000:02:00.0\"}},\"MSGID\":\"801\",\"MESSAGE\":\"test\"}"); } int -- 1.7.10.4

1 0

[RFC (3.4) 0/5]: multiline & linux 3.5+ /dev/kmsg support
by Gergely Nagy 13 Oct '12

13 Oct '12

The next few patches gradually introduce two - in my opinion - interesting features for syslog-ng 3.4: support for one kind of multiline messages, where continuation lines start with whitespace; and support for the /dev/kmsg format introduced in linux 3.5. Neither implementation is complete yet, there are subtle issues with both (see later), but they're at a stage where taking them to a test drive would be useful, and code review wouldn't hurt either. The patches are also available on the feature/3.4/indented-multiline[1] branch of my git repository[2]. The branch may get rebased in the future, I do not make any promise of keeping it fast-forwardable. [1]: https://github.com/algernon/syslog-ng/commits/feature/3.4/indented-multiline [2]: git://github.com/algernon/syslog-ng.git linux 3.5 /dev/kmsg =================== The linux 3.5+ /dev/kmsg support is the easier to test: you only need a 3.5+ kernel, and when using the system() source, things will magically work! This includes parsing the relative timestamps in kmsg messages and also parsing any additional key-value pairs. This means that if we had an input line like this: ,---- | 6,802,65338577;ATL1E 0000:02:00.0: eth0: NIC Link is Up <100 Mbps Full Duplex> | SUBSYSTEM=pci | DEVICE=+pci:0000:02:00.0 `---- Then we'll get back something like this: ,---- | { | "FILE_NAME" : "/dev/kmsg", | "MSGID" : "802", | "HOST" : "luthien", | "DATE" : "Oct 13 16:53:44", | "FACILITY" : "kern", | "SOURCEIP" : "127.0.0.1", | "TAGS" : ".source.s_kmsg", | "kernel" : { | "SUBSYSTEM" : "pci", | "DEVICE" : { | "name" : "0000:02:00.0", | "type" : "pci" | } | }, | "PRIORITY" : "info", | "HOST_FROM" : "luthien", | "MESSAGE" : "ATL1E 0000:02:00.0: eth0: NIC Link is Up <100 Mbps Full Duplex>", | "PROGRAM" : "kernel" | } `---- For the above, the following template was used: $(format-json --scope selected-macros --scope nv-pairs --key .kernel.* --shift 1) (Mind you, program_override() is currently broken in 3.4, but that is unrelated to the /dev/kmsg support) Any name-value pairs the kernel supplies will be put into variables of the same name, prefixed with ".kernel." (so SUBSYSTEM becomes .kernel.SUBSYSTEM), and the value of DEVICE= will be further parsed, based on rules written down in the appropriate kernel docs. Indented multiline ================== The indented multiline support enhances the file() and tcp() sources with the ability to read multiline records (if the indented-multiline flag is set), where continuation lines start with whitespace. Such output is produced by the $(indent-multi-line) template function. Usage: source s_ml { tcp(port(12345) flags(indented-multiline, no-parse)); }; Then, you can test it like this: (cat <<EOF This is the first line - second - third EOF ) | nc localhost 12345 Using a JSON output, this would result in: {"MESSAGE":"This is the first line\n - second\n - third"} A record can be terminated in two ways: by another line that does not begin with whitespace, or by reacing EOF. This means that if netcat didn't close the connection, we wouldn't get the test record until another line arrives. Unfortunately, there's not much I can do about this limitation, but if anyone has a good idea how to work around or fix it, please let me know! Known issues ============ The /dev/kmsg handling is not reload/restart-safe: if syslog-ng gets reloaded or restarted, it will read the whole lot of /dev/kmsg over again, for example. There are possibly other subtle issues too, but all of those will be ironed out before I submit the patches for merging. The indented-multiline support has a more serious limitation: if data needs to be split (which happens in some circumstances when the incoming data is bigger than the internal message size (8192 by default)), it is not correctly reassembled, making indented-multiline less than useful for some tasks.

1 5

[Bug 202] New: pdbtool dump does not recognize the -p option
by bugzilla＠bugzilla.balabit.com 13 Oct '12

13 Oct '12

https://bugzilla.balabit.com/show_bug.cgi?id=202 Summary: pdbtool dump does not recognize the -p option Product: syslog-ng Version: 3.3.x Platform: PC OS/Version: Linux Status: NEW Severity: normal Priority: unspecified Component: syslog-ng AssignedTo: bazsi(a)balabit.hu ReportedBy: erempel(a)uvic.ca Type of the Report: --- Estimated Hours: 0.0 Man page states dump [options] Display the RADIX tree built from the patterns. This shows how are the patterns represented in syslog-ng and it might also help to track down pattern-matching problems. The dump utility can dump the tree used for matching the PROGRAM or the MSG parts. --debug or -d Enable debug/diagnostic messages on stderr. --pdb or -p Name of the pattern database file to use. --program or -P Displays the RADIX tree built from the patterns belonging to the $PROGRAM application. --program-tree or -T Display the $PROGRAM tree. --verbose or -v Enable verbose messages on stderr. Example and sample output: But the command does localhost% pdbtool dump -P fscheck -p patterndb.xml Error parsing command line arguments: Unknown option -p localhost% pdbtool dump -P fscheck --pdb patterndb.xml Error parsing command line arguments: Unknown option --pdb -- Configure bugmail: https://bugzilla.balabit.com/userprefs.cgi?tab=email ------- You are receiving this mail because: ------- You are watching all bug changes.

1 2

[Bug 201] New: system() does not work on FreeBSD: /dev/ klog is pretty much ignored
by bugzilla＠bugzilla.balabit.com 13 Oct '12

13 Oct '12

https://bugzilla.balabit.com/show_bug.cgi?id=201 Summary: system() does not work on FreeBSD: /dev/klog is pretty much ignored Product: syslog-ng Version: 3.3.x Platform: All OS/Version: FreeBSD Status: ASSIGNED Severity: normal Priority: unspecified Component: syslog-ng AssignedTo: algernon(a)balabit.hu ReportedBy: algernon(a)balabit.hu Type of the Report: --- Estimated Hours: 0.0 While debugging another problem, I found a couple of issues with system() on FreeBSD: * follow-freq(0) doesn't appear to work, so /dev/klog gets ignored outright * Even with follow-freq(1), we never read from /dev/klog, because stat() will report its size to be 0, so we never end up reading from it. The second is easy to fix, adding an || !S_ISREG(st.st_mode) to the appropriate condition in log_reader_io_follow_file() does the right thing. -- Configure bugmail: https://bugzilla.balabit.com/userprefs.cgi?tab=email ------- You are receiving this mail because: ------- You are watching all bug changes.

1 4

[Bug 200] New: logging does not work on FreeBSD
by bugzilla＠bugzilla.balabit.com 13 Oct '12

13 Oct '12

https://bugzilla.balabit.com/show_bug.cgi?id=200 Summary: logging does not work on FreeBSD Product: syslog-ng Version: 3.4.x Platform: PC OS/Version: FreeBSD Status: NEW Severity: normal Priority: unspecified Component: syslog-ng AssignedTo: bazsi(a)balabit.hu ReportedBy: czanik(a)balabit.hu Type of the Report: --- Estimated Hours: 0.0 Using the latest syslog-ng 3.4 git, with Algernon sandbox patches applied (to be able to generate a tar.gz dist file), logging does not seem to work. Without command line arguments, nothing gets logged, except for version information at startup. When using "-vdF", logs coming from the network are logged. Tested on FreeBSD 9.0 with a slightly modified sysutils/syslog-ng-rc port (removed utmp magic from ports Makefile). -- Configure bugmail: https://bugzilla.balabit.com/userprefs.cgi?tab=email ------- You are receiving this mail because: ------- You are watching all bug changes.

1 6

cisco rewrite code
by Evan Rempel 12 Oct '12

12 Oct '12

Does anyone have a pre build set of patterns/rewrite rule to rewrite all cisco logs into something that is a little more compliant? We are trying to use a master pattern database to identify/classify messages, but the cisco logs don't have usable "program names" so the pattern database can't even get started :-( Thanks for any pointers. -- Evan

4 3

[PATCH (3.4)] afsocket: Use the "dgram" proto for dgram sockets
by Gergely Nagy 11 Oct '12

11 Oct '12

When applying a transport to a socket, don't just set the transport, but also set logproto_name properly: "dgram" for SOCK_DGRAM sockets, "text" for everything else. If this is not done, and our socket is SOCK_DGRAM, we'd use the text proto, which after the refactoring, would look for a terminating new line, instead of datagram boundary - and it wouldn't find it, which will result in partial messages accumulating in a buffer, to eventually be sent out as garbage. This patch simply makes it so that SOCK_DGRAM will use the "dgram" logproto, which does the right thing, and knows about datagram boundaries. Reported-by: Peter Czanik <czanik(a)balabit.hu> Signed-off-by: Gergely Nagy <algernon(a)balabit.hu> --- modules/afsocket/afunix.c | 8 ++++++-- 1 files changed, 6 insertions(+), 2 deletions(-) diff --git a/modules/afsocket/afunix.c b/modules/afsocket/afunix.c index 1a1465f..8849927 100644 --- a/modules/afsocket/afunix.c +++ b/modules/afsocket/afunix.c @@ -172,12 +172,16 @@ afunix_sd_apply_transport(AFSocketSourceDriver *s) if (!self->super.bind_addr) self->super.bind_addr = g_sockaddr_unix_new(self->filename); + self->super.logproto_name = "text"; + if (self->super.sock_type == SOCK_DGRAM) - afsocket_sd_set_transport(&self->super.super.super, "unix-dgram"); + { + afsocket_sd_set_transport(&self->super.super.super, "unix-dgram"); + self->super.logproto_name = "dgram"; + } else afsocket_sd_set_transport(&self->super.super.super, "unix-stream"); - self->super.logproto_name = "text"; return TRUE; } -- 1.7.9

1 0

syslog-ng Insider - October 2012
by Peter Czanik 11 Oct '12

11 Oct '12

Dear syslog-ng users, This is the 17th issue of the syslog-ng Insider, a monthly newsletter that brings you syslog-ng related news. Your feedback and news tips about the next issue is welcome at documentation(a)balabit.com <mailto:documentation@balabit.com> FEATURED NEWS New AMQP destination driver --------------------------- An important milestone in syslog-ng's history is the AMQP destination driver, which is the first complete module contributed by a third party, and was not originated from within BalaBit! The destination allows syslog-ng to publish messages (using various exchange types) to an AMQP server such as RabbitMQ using an intuitive method: sending the headers selected by value-pairs() as headers, thereby solving the problem of how to serialize the payload: don't. For more information about the driver, see the announcement at: https://lists.balabit.hu/pipermail/syslog-ng/2012-October/019483.html A comparison of PE diskbuffer and the AMQP driver: http://algernon.blogs.balabit.com/2012/10/syslog-ng-amqp-and-disk-buffers/ Vote for syslog-ng Store Box to become the Best Computer Forensics Tool ------------------------------------------------------------------ We’re proud to announce we entered our syslog-ng Store Box appliance into the SC Awards US for the Best Computer Forensic Tool. Voting is now open until Oct 26. (requires registration or subscription to SC Magazine) so we kindly ask you visit https://www.bigpulse.com/m20086/intro to vote for us. We appreciate your help. syslog-ng 3.3.7RC2 is released ------------------------------ Version 3.3.7 is scheduled to arrive at the end of October, and a second release candidate was released for it (called version 3.3.6.91). It has many fixes for FreeBSD, so it should compile with the bundled ivykis on FreeBSD 8 and 9. Patches to compile it on FreeBSD 7 are included in the updated sysutils/syslog-ng-rc port: http://www.freshports.org/sysutils/syslog-ng-rc/ Packages for all supported or development versions of openSUSE are also available: https://build.opensuse.org/project/show?project=home%3Aczanik%3Asyslog-ng33 Announcement for 3.3.6.91: https://lists.balabit.hu/pipermail/syslog-ng/2012-October/019480.html OTHER SHORT NEWS * syslog-ng will be presented at two conferences in October: http://fedoraproject.org/wiki/FUDCon:Paris_2012 and http://conference.opensuse.org/ NEW RELEASES: * syslog-ng OSE 3.3.6.91: https://lists.balabit.hu/pipermail/syslog-ng/2012- <https://lists.balabit.hu/pipermail/syslog-ng/2012-October/019480.html>October/019480.html <https://lists.balabit.hu/pipermail/syslog-ng/2012-October/019480.html> ARCHIVE http://insider.blogs.balabit.com/ -- Peter Czanik (CzP) <czanik(a)balabit.hu> BalaBit IT Security / syslog-ng upstream http://czanik.blogs.balabit.com/

1 0

challenges with SDATA
by Chris Johnson 09 Oct '12

09 Oct '12

I have successfully written parsers for unstructured logs, but would like to be able to leverage SDATA. Based on what I have researched so far, I have built the following skeleton conf file for writing to mongodb #####Filter##### filter f_session_close {message("RT_FLOW_SESSION_CLOSE");}; #####Destinations##### destination d_mongodb_close { mongodb( collection("close") value-pairs( scope(rfc5424 sdata) ) ); }; #####Log##### log { source(s_network); filter(f_session_close); destination(d_mongodb_close); }; With this in mind, I get the following logs with only the rfc5424 scope: "DATE" : "Oct 5 20:40:21", "FACILITY" : "user", "HOST" : "192.168.199.2", "MESSAGE" : "2012-10-05T20:40:20.444 SRX100 RT_FLOW - RT_FLOW_SESSION_CREATE [junos(a)2636.1.1.1.2.36 source-address=\"192.168.199.207\" source-port=\"55625\" destination-address=\"130.57.4.24\" destination-port=\"80\" service-name=\"junos-http\" nat-source-address=\"50.193.12.149\" nat-source-port=\"28178\" nat-destination-address=\"130.57.4.24\" nat-destination-port=\"80\" src-nat-rule-name=\"source-nat-rule\" dst-nat-rule-name=\"None\" protocol-id=\"6\" policy-name=\"trust-to-untrust\" source-zone-name=\"trust\" destination-zone-name=\"untrust\" session-id-32=\"61189\" username=\"N/A\" roles=\"N/A\" packet-incoming-interface=\"vlan.0\"]", "PRIORITY" : "info", "PROGRAM" : "1", "_id" : ObjectId("506fa825817f52de65000001") When using the SDATA scope, I only get an object id. None of the structured data is ever parsed. Is there something special I need to do to parse the structured portion of the messge? Chris

2 1