[syslog-ng] Fwd: syslog-ng 3.7.3 crashing randomly
Juhász, Viktor
viktor.juhasz at balabit.com
Tue Oct 25 12:44:37 UTC 2016
Hi Soumyadip,
This symptoms are very similar to the problem solved which is solved by PR:
https://github.com/balabit/syslog-ng/pull/1183
It is a known bug the next 3.8 release will contain the fix.
BR,
Viktor Juhász
On Tue, Oct 25, 2016 at 1:01 PM, Soumyadip Das Mahapatra <
soumyadip.bt at gmail.com> wrote:
> Hi,
>
> We have installed syslog-ng 3.7.3 on RHEL 6.6 from COPR repo:
>
> From dmesg:
>
> [Thu Sep 1 02:24:08 2016] syslog-ng[10374]: segfault at 1a116c10 ip
> 000000001a116c10 sp 00007f26eebfa928 error 15
> [Sun Sep 4 20:10:47 2016] syslog-ng[27797]: segfault at 0 ip (null) sp
> 00007f998b5fb928 error 14 in syslog-ng[400000+3000]
> [Mon Sep 5 19:04:15 2016] syslog-ng[12727]: segfault at 36aaa42080 ip
> 00000039b8618df0 sp 00007f996fffc878 error 7 in
> libglib-2.0.so.0.2800.8[39b8600000+115000]
> [Tue Sep 6 03:01:21 2016] syslog-ng[3827]: segfault at 15cb09d0 ip
> 0000000015cb09d0 sp 00007f99aebfa928 error 15
> [Fri Sep 9 05:03:41 2016] syslog-ng[28275] general protection
> ip:39b8618df0 sp:7f29c57f8878 error:0 in libglib-2.0.so.0.2800.8[39b860
> 0000+115000]
> [Sun Sep 11 00:48:01 2016] syslog-ng[14479]: segfault at 0 ip (null) sp
> 00007f2a03caf928 error 14 in syslog-ng[400000+3000]
> [Sun Sep 11 10:15:37 2016] syslog-ng[2012]: segfault at 0 ip
> 00000036aaa3cc1c sp 00007feed2bfa880 error 6 in
> libsyslog-ng-3.7.so.0.0.0[36aaa00000+a2000]
>
> Core 1:
>
> (gdb) bt full
> #0 0x00000039b8618df0 in g_atomic_int_add () from /lib64/libglib-2.0.so.0
> No symbol table info available.
> #1 0x00000036aaa3cbe4 in stats_counter_add (self=0xabe68ea0, thread_id=5)
> at lib/stats/stats-counter.h:39
> No locals.
> #2 log_queue_fifo_move_input_unlocked (self=0xabe68ea0, thread_id=5)
> at lib/logqueue-fifo.c:193
> queue_len = 1755882337
> #3 0x00000036aaa3cd6e in log_queue_fifo_move_input (user_data=0xabe68ea0)
> at lib/logqueue-fifo.c:215
> self = 0xabe68ea0
> thread_id = 5
> __PRETTY_FUNCTION__ = "log_queue_fifo_move_input"
> #4 0x00000036aaa456ca in main_loop_worker_invoke_batch_callbacks ()
> at lib/mainloop-worker.c:270
> cb = 0xabe690e0
> lh = 0xabe690e0
> lh2 = 0xabe690e0
> #5 0x00000036aaa6fa2a in iv_work_thread_do_work (_thr=0x9d6bd530) at
> iv_work.c:118
> work = 0x6fb160
> thr = 0x9d6bd530
> pool = 0x6e48c0
> last_seq = 1477269410
> #6 0x00000036aaa6ed4a in iv_run_tasks (st=0x7f29b44d3c00) at iv_task.c:48
> t = <value optimized out>
> tasks = {next = 0x7f29c57f89a0, prev = 0x7f29c57f89a0}
> #7 0x00000036aaa7100c in iv_main () at iv_main_posix.c:106
> to = {tv_sec = 10, tv_nsec = 0}
> st = 0x7f29b44d3c00
> #8 0x00000036aaa6f841 in iv_work_thread (_thr=0x9d6bd530) at iv_work.c:200
> thr = 0x9d6bd530
> pool = 0x6e48c0
> #9 0x00000036aaa71a1f in iv_thread_handler (_thr=0x75242f90)
> at iv_thread_posix.c:142
> __clframe = {__cancel_routine = 0x36aaa71a80
> <iv_thread_cleanup_handler>,
> __cancel_arg = 0x75242f90, __do_it = 1, __cancel_type = 0}
> thr = 0x75242f90
> #10 0x00000039b6e079d1 in start_thread () from /lib64/libpthread.so.0
> No symbol table info available.
> #11 0x00000039b6ae88fd in clone () from /lib64/libc.so.6
> No symbol table info available.
>
> Core 2:
>
> (gdb) bt full
> #0 0x0000000000000000 in ?? ()
> No symbol table info available.
> #1 0x00000036aaa456ca in main_loop_worker_invoke_batch_callbacks ()
> at lib/mainloop-worker.c:270
> cb = 0x87e96b10
> lh = 0x87e96b10
> lh2 = 0x0
> #2 0x00000036aaa6fa2a in iv_work_thread_do_work (_thr=0x7034ffd0) at
> iv_work.c:118
> work = 0x6fb390
> thr = 0x7034ffd0
> pool = 0x6e48b0
> last_seq = 1323426114
> #3 0x00000036aaa6ed4a in iv_run_tasks (st=0x7f29d4360200) at iv_task.c:48
> t = <value optimized out>
> tasks = {next = 0x7f2a03caf9a0, prev = 0x7f2a03caf9a0}
> #4 0x00000036aaa7100c in iv_main () at iv_main_posix.c:106
> to = {tv_sec = 10, tv_nsec = 0}
> st = 0x7f29d4360200
> #5 0x00000036aaa6f841 in iv_work_thread (_thr=0x7034ffd0) at iv_work.c:200
> thr = 0x7034ffd0
> pool = 0x6e48b0
> #6 0x00000036aaa71a1f in iv_thread_handler (_thr=0x426b490)
> at iv_thread_posix.c:142
> __clframe = {__cancel_routine = 0x36aaa71a80
> <iv_thread_cleanup_handler>,
> __cancel_arg = 0x426b490, __do_it = 1, __cancel_type = 0}
> thr = 0x426b490
> #7 0x00000039b6e079d1 in start_thread () from /lib64/libpthread.so.0
> No symbol table info available.
> #8 0x00000039b6ae88fd in clone () from /lib64/libc.so.6
> No symbol table info available.
>
> So looks like segfault is happening at random part of the code.
>
> # cat /etc/sysconfig/syslog-ng
> #---
> # Syslog-ng command line options
> # See syslog-ng(8) for more details
> #---
> SYSLOGNG_PID="/var/run/syslog-ng.pid"
> SYSLOGNG_OPTIONS="-p $SYSLOGNG_PID --fd-limit 30000"
> SYSLOGNG_COMPAT_PID="/var/run/syslogd.pid"
>
>
> # cat /etc/syslog-ng/syslog-ng.conf
>
> @version:3.6
> options {
> flush_lines (0);
> time_reopen (10);
> log_fifo_size (20000);
> use_dns (yes);
> use_fqdn (yes);
> create_dirs (yes);
> dir_group("wheel");
> dir_owner("nobody");
> dir_perm(0755);
> owner("nobody");
> group("nobody");
> perm(0644);
> threaded(yes);
> keep_hostname (yes);
> chain_hostnames(yes);
> bad_hostname("[^[:print:]]");
> dns_cache(yes);
> dns_cache_expire(300);
> dns_cache_expire_failed(30);
> dns_cache_size(1000);
> stats_freq(3600);
> flush_lines(0);
> };
> source s_sys {
> file ("/proc/kmsg" program_override("kernel: "));
> unix-stream ("/dev/log");
> # udp(ip(0.0.0.0) port(514));
> };
> source s_tcp {
> network(transport("tcp") port(514) so_rcvbuf(8388608)
> max-connections(200) log-iw-size(20000));
> };
> source s_udp {
> network(transport("udp") port(514) so_rcvbuf(8388608)
> log-iw-size(20000));
> };
> source s_net {
> network(transport("udp") port(515) so_rcvbuf(8388608)
> log-iw-size(20000) tags("source_net"));
> network(transport("tcp") port(515) so_rcvbuf(8388608)
> max-connections(200) log-iw-size(20000) tags("source_net"));
> };
> source s_windows {
> network(transport("tcp") port(516) so_rcvbuf(8388608)
> max-connections(200) log-iw-size(20000) tags("windows"));
> };
> destination d_cons { file("/dev/console"); };
> destination d_mesg { file("/var/log/messages"); };
> destination d_auth { file("/var/log/secure"); };
> destination d_mail { file("/var/log/maillog" flush_lines(10)); };
> destination d_spol { file("/var/log/spooler"); };
> destination d_boot { file("/var/log/boot.log"); };
> destination d_cron { file("/var/log/cron"); };
> destination d_kern { file("/var/log/kern"); };
> destination d_mlal { usertty("*"); };
> destination d_userx { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/user.log");
> };
> destination d_kernx { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/kern.log");
> };
> destination d_mailx { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/mail.log");
> };
> destination d_mp { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/mp.log"); };
> destination d_daemonx { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/daemon.log");
> };
> destination d_authx { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/auth.log");
> };
> destination d_lprx { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/lpr.log"); };
> destination d_cronx { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/cron.log");
> };
> destination d_messagesx { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/messages.log");
> };
> destination d_networkx { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/network.log");
> };
> destination d_firewallx { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/firewall.log");
> };
> destination d_local0 { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/sp.log"); };
> destination d_local1 { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/app.log");
> };
> destination d_localx { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/local.log");
> };
> destination d_local6 { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/httpd.log");
> };
> destination d_dnslogs { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/dnslogs.log");
> };
> destination d_logstash {
> tcp("127.0.0.1" port(5140)
> template("$R_ISODATE <$FACILITY_NUM.$LEVEL_NUM> $HOST $MSGHDR
> $MESSAGE\n")
> frac_digits(3) template-escape(no));
> pipe("/var/log/logstash-pipe"
> template("$R_ISODATE <$FACILITY_NUM.$LEVEL_NUM> $HOST $MSGHDR
> $MESSAGE\n")
> frac_digits(3) template-escape(no));
> };
> destination d_userauth {
> tcp("abc.xyz.com" port(13456));
> };
> destination d_otto {
> tcp("abc.xyz.com" port(13456));
> tcp("abc.xyz.com" port(13456));
> };
> destination d_notifier {
> tcp("abc.xyz.com" port(6000));
> tcp("10.255.xx.yy" port(1248));
> };
> destination d_graylog {
> syslog("abc.xyz.com" transport("tcp") port(9514));
> pipe("/var/log/graylog-pipe" ts-format(iso));
> };
> destination d_gtsadfeed {
> syslog("abc.xyz.com" transport("udp") port(514));
> };
> filter f_kernel { facility(kern); };
> filter f_default { level(info..emerg)
> and not (facility(mail)
> or facility(authpriv)
> or facility(cron))
> and not message("Syslog connection
> (accepted|closed)"); };
> filter f_auth { facility(auth,authpriv); };
> filter f_mail { facility(mail); };
> filter f_emergency { level(emerg); };
> filter f_news { facility(uucp) or
> (facility(news)
> and level(crit..emerg)); };
> filter f_boot { facility(local7); };
> filter f_cron { facility(cron,solaris-cron); };
> filter f_user { facility(user); };
> filter f_daemon { facility(daemon); };
> filter f_lpr { facility(lpr); };
> filter f_dnslogs { program("mydns_logger"); };
> filter f_local0 { facility(local0); };
> filter f_local1 { facility(local1) and not
> program("mydns_logger"); };
> filter f_local2 { facility(local2); };
> filter f_local3 { facility(local3); };
> filter f_local4 { facility(local4); };
> filter f_local5 { facility(local5) or tags("source_net"); }; #
> local5 is network, so is port 515
> filter f_local6 { facility(local6); };
> filter f_local7 { facility(local7); };
> filter f_alert { level(alert); };
> filter f_crit { level(crit); };
> filter f_err { level(err); };
> filter f_warn { level(warn); };
> filter f_notice { level(notice); };
> filter f_info { level(info); };
> filter f_debug { level(debug); };
> filter f_nonet { not facility(local4) and not
> facility(local5) and not tags("source_net"); };
> filter f_nocf3 { not program("cf3"); };
> filter f_nodhcpd { not program("dhcpd"); };
> filter f_windows { tags("windows"); };
> filter f_snmptrapd { program("snmptrapd"); };
> filter f_userauth { not facility(local3); };
> filter f_notifier_filter { not match("ASA-4-302015|ASA-4-3020
> 13|TRAFFIC|permitted|Deny|Denied|denied", value("MESSAGE")); };
> filter f_gtsadfeed_filter { host("*-adc*.abc.xyz.com" type(glob)); };
> rewrite f_rewrite_name { set("$FULLHOST_FROM", value("HOST")
> condition(not tags("windows")
> and not match("REMOTELOG",
> value("MESSAGE")))); };
> rewrite r_rewrite_name_windows { set("$FULLHOST_FROM", value("HOST")
> condition(not match("REMOTELOG",
> value("MESSAGE")))); };
> rewrite r_snmptrapd { subst("^([^ ]+) (.*)", "${2}",
> value("MESSAGE"));
> set("${1}", value("HOST")); };
> log { source(s_sys); filter(f_kernel); destination(d_kern); };
> log { source(s_sys); filter(f_default); destination(d_mesg); };
> log { source(s_sys); filter(f_auth); destination(d_auth); };
> log { source(s_sys); filter(f_mail); destination(d_mail); };
> log { source(s_sys); filter(f_emergency); destination(d_mlal); };
> log { source(s_sys); filter(f_news); destination(d_spol); };
> log { source(s_sys); filter(f_boot); destination(d_boot); };
> log { source(s_sys); filter(f_cron); destination(d_cron); };
> log {
> source(s_sys);
> filter(f_snmptrapd);
> rewrite(r_snmptrapd);
> destination(d_networkx);
> destination(d_logstash);
> log {
> filter(f_notifier_filter);
> destination(d_notifier);
> };
> };
> log {
> source(s_net);
> source(s_udp);
> source(s_tcp);
> source(s_sys);
> source(s_windows);
> log {
> filter(f_userauth);
> destination(d_userauth);
> destination(d_otto);
> };
> log {
> rewrite(f_rewrite_name);
> log {
> destination(d_logstash);
> };
> log {
> filter(f_nocf3);
> filter(f_nodhcpd);
> destination(d_graylog);
> };
> log {
> filter(f_user);
> destination(d_userx);
> };
> log {
> filter(f_kernel);
> destination(d_kernx);
> };
> log {
> filter(f_mail);
> destination(d_mailx);
> };
> log {
> filter(f_daemon);
> destination(d_daemonx);
> };
> log {
> filter(f_auth);
> destination(d_authx);
> };
> log {
> filter(f_lpr);
> destination(d_lprx);
> };
> log {
> filter(f_cron);
> destination(d_cronx);
> };
> log {
> filter(f_local0);
> destination(d_local0);
> };
> log {
> filter(f_local1);
> destination(d_local1);
> };
> log {
> filter(f_local2);
> destination(d_authx);
> };
> log {
> filter(f_dnslogs);
> destination(d_dnslogs);
> };
> log {
> filter(f_local4);
> destination(d_firewallx);
> log {
> filter(f_notifier_filter);
> destination(d_notifier);
> };
> };
> log {
> filter(f_local5);
> destination(d_networkx);
> log {
> filter(f_notifier_filter);
> destination(d_notifier);
> };
> };
> log {
> filter(f_local6);
> destination(d_local6);
> };
> log {
> filter(f_local7);
> destination(d_mp);
> };
> log {
> filter(f_windows);
> filter(f_gtsadfeed_filter);
> rewrite(r_rewrite_name_windows);
> destination(d_gtsadfeed);
> };
> };
> };
>
>
> Could you please advise?
>
> Regards,
> Soumyadip
>
>
> ____________________________________________________________
> __________________
> Member info: https://lists.balabit.hu/mailman/listinfo/syslog-ng
> Documentation: http://www.balabit.com/support/documentation/?
> product=syslog-ng
> FAQ: http://www.balabit.com/wiki/syslog-ng-faq
>
>
>
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.balabit.hu/pipermail/syslog-ng/attachments/20161025/36f9e3ac/attachment-0001.html>
More information about the syslog-ng
mailing list