[syslog-ng] Fwd: syslog-ng 3.7.3 crashing randomly

25 Oct 2016

      Hi,

We have installed syslog-ng 3.7.3 on RHEL 6.6 from COPR repo:
...
From dmesg:
[Thu Sep  1 02:24:08 2016] syslog-ng[10374]: segfault at 1a116c10 ip
000000001a116c10 sp 00007f26eebfa928 error 15
[Sun Sep  4 20:10:47 2016] syslog-ng[27797]: segfault at 0 ip (null) sp
00007f998b5fb928 error 14 in syslog-ng[400000+3000]
[Mon Sep  5 19:04:15 2016] syslog-ng[12727]: segfault at 36aaa42080 ip
00000039b8618df0 sp 00007f996fffc878 error 7 in libglib-2.0.so.0.2800.8[
39b8600000+115000]
[Tue Sep  6 03:01:21 2016] syslog-ng[3827]: segfault at 15cb09d0 ip
0000000015cb09d0 sp 00007f99aebfa928 error 15
[Fri Sep  9 05:03:41 2016] syslog-ng[28275] general protection
ip:39b8618df0 sp:7f29c57f8878 error:0 in libglib-2.0.so.0.2800.8[
39b8600000+115000]
[Sun Sep 11 00:48:01 2016] syslog-ng[14479]: segfault at 0 ip (null) sp
00007f2a03caf928 error 14 in syslog-ng[400000+3000]
[Sun Sep 11 10:15:37 2016] syslog-ng[2012]: segfault at 0 ip
00000036aaa3cc1c sp 00007feed2bfa880 error 6 in libsyslog-ng-3.7.so.0.0.0[
36aaa00000+a2000]

Core 1:

(gdb) bt full
#0  0x00000039b8618df0 in g_atomic_int_add () from /lib64/libglib-2.0.so.0
No symbol table info available.
#1  0x00000036aaa3cbe4 in stats_counter_add (self=0xabe68ea0, thread_id=5)
    at lib/stats/stats-counter.h:39
No locals.
#2  log_queue_fifo_move_input_unlocked (self=0xabe68ea0, thread_id=5)
    at lib/logqueue-fifo.c:193
        queue_len = 1755882337
#3  0x00000036aaa3cd6e in log_queue_fifo_move_input (user_data=0xabe68ea0)
    at lib/logqueue-fifo.c:215
        self = 0xabe68ea0
        thread_id = 5
        __PRETTY_FUNCTION__ = "log_queue_fifo_move_input"
#4  0x00000036aaa456ca in main_loop_worker_invoke_batch_callbacks ()
    at lib/mainloop-worker.c:270
        cb = 0xabe690e0
        lh = 0xabe690e0
        lh2 = 0xabe690e0
#5  0x00000036aaa6fa2a in iv_work_thread_do_work (_thr=0x9d6bd530) at
iv_work.c:118
        work = 0x6fb160
        thr = 0x9d6bd530
        pool = 0x6e48c0
        last_seq = 1477269410
#6  0x00000036aaa6ed4a in iv_run_tasks (st=0x7f29b44d3c00) at iv_task.c:48
        t = <value optimized out>
        tasks = {next = 0x7f29c57f89a0, prev = 0x7f29c57f89a0}
#7  0x00000036aaa7100c in iv_main () at iv_main_posix.c:106
        to = {tv_sec = 10, tv_nsec = 0}
        st = 0x7f29b44d3c00
#8  0x00000036aaa6f841 in iv_work_thread (_thr=0x9d6bd530) at iv_work.c:200
        thr = 0x9d6bd530
        pool = 0x6e48c0
#9  0x00000036aaa71a1f in iv_thread_handler (_thr=0x75242f90)
    at iv_thread_posix.c:142
        __clframe = {__cancel_routine = 0x36aaa71a80
<iv_thread_cleanup_handler>,
          __cancel_arg = 0x75242f90, __do_it = 1, __cancel_type = 0}
        thr = 0x75242f90
#10 0x00000039b6e079d1 in start_thread () from /lib64/libpthread.so.0
No symbol table info available.
#11 0x00000039b6ae88fd in clone () from /lib64/libc.so.6
No symbol table info available.

Core 2:

(gdb) bt full
#0  0x0000000000000000 in ?? ()
No symbol table info available.
#1  0x00000036aaa456ca in main_loop_worker_invoke_batch_callbacks ()
    at lib/mainloop-worker.c:270
        cb = 0x87e96b10
        lh = 0x87e96b10
        lh2 = 0x0
#2  0x00000036aaa6fa2a in iv_work_thread_do_work (_thr=0x7034ffd0) at
iv_work.c:118
        work = 0x6fb390
        thr = 0x7034ffd0
        pool = 0x6e48b0
        last_seq = 1323426114
#3  0x00000036aaa6ed4a in iv_run_tasks (st=0x7f29d4360200) at iv_task.c:48
        t = <value optimized out>
        tasks = {next = 0x7f2a03caf9a0, prev = 0x7f2a03caf9a0}
#4  0x00000036aaa7100c in iv_main () at iv_main_posix.c:106
        to = {tv_sec = 10, tv_nsec = 0}
        st = 0x7f29d4360200
#5  0x00000036aaa6f841 in iv_work_thread (_thr=0x7034ffd0) at iv_work.c:200
        thr = 0x7034ffd0
        pool = 0x6e48b0
#6  0x00000036aaa71a1f in iv_thread_handler (_thr=0x426b490)
    at iv_thread_posix.c:142
        __clframe = {__cancel_routine = 0x36aaa71a80
<iv_thread_cleanup_handler>,
          __cancel_arg = 0x426b490, __do_it = 1, __cancel_type = 0}
        thr = 0x426b490
#7  0x00000039b6e079d1 in start_thread () from /lib64/libpthread.so.0
No symbol table info available.
#8  0x00000039b6ae88fd in clone () from /lib64/libc.so.6
No symbol table info available.

So looks like segfault is happening at random part of the code.

# cat /etc/sysconfig/syslog-ng
#---
# Syslog-ng command line options
# See syslog-ng(8) for more details
#---
SYSLOGNG_PID="/var/run/syslog-ng.pid"
SYSLOGNG_OPTIONS="-p $SYSLOGNG_PID --fd-limit 30000"
SYSLOGNG_COMPAT_PID="/var/run/syslogd.pid"

# cat /etc/syslog-ng/syslog-ng.conf

@version:3.6
options {
    flush_lines (0);
    time_reopen (10);
    log_fifo_size (20000);
    use_dns (yes);
    use_fqdn (yes);
    create_dirs (yes);
    dir_group("wheel");
    dir_owner("nobody");
    dir_perm(0755);
    owner("nobody");
    group("nobody");
    perm(0644);
    threaded(yes);
    keep_hostname (yes);
    chain_hostnames(yes);
    bad_hostname("[^[:print:]]");
    dns_cache(yes);
    dns_cache_expire(300);
    dns_cache_expire_failed(30);
    dns_cache_size(1000);
    stats_freq(3600);
    flush_lines(0);
};
source s_sys {
    file ("/proc/kmsg" program_override("kernel: "));
    unix-stream ("/dev/log");
    # udp(ip(0.0.0.0) port(514));
};
source s_tcp {
    network(transport("tcp") port(514) so_rcvbuf(8388608)
max-connections(200) log-iw-size(20000));
};
source s_udp {
    network(transport("udp") port(514) so_rcvbuf(8388608)
log-iw-size(20000));
};
source s_net {
    network(transport("udp") port(515) so_rcvbuf(8388608)
log-iw-size(20000) tags("source_net"));
    network(transport("tcp") port(515) so_rcvbuf(8388608)
max-connections(200) log-iw-size(20000) tags("source_net"));
};
source s_windows {
    network(transport("tcp") port(516) so_rcvbuf(8388608)
max-connections(200) log-iw-size(20000) tags("windows"));
};
destination d_cons { file("/dev/console"); };
destination d_mesg { file("/var/log/messages"); };
destination d_auth { file("/var/log/secure"); };
destination d_mail { file("/var/log/maillog" flush_lines(10)); };
destination d_spol { file("/var/log/spooler"); };
destination d_boot { file("/var/log/boot.log"); };
destination d_cron { file("/var/log/cron"); };
destination d_kern { file("/var/log/kern"); };
destination d_mlal { usertty("*"); };
destination d_userx { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/user.log"); };
destination d_kernx { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/kern.log"); };
destination d_mailx { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/mail.log"); };
destination d_mp { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/mp.log"); };
destination d_daemonx { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/daemon.log");
};
destination d_authx { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/auth.log"); };
destination d_lprx { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/lpr.log"); };
destination d_cronx { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/cron.log"); };
destination d_messagesx {
file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/messages.log");
};
destination d_networkx { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/network.log");
};
destination d_firewallx {
file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/firewall.log");
};
destination d_local0 { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/sp.log"); };
destination d_local1 { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/app.log"); };
destination d_localx { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/local.log");
};
destination d_local6 { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/httpd.log");
};
destination d_dnslogs { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/dnslogs.log");
};
destination d_logstash {
    tcp("127.0.0.1" port(5140)
        template("$R_ISODATE <$FACILITY_NUM.$LEVEL_NUM> $HOST $MSGHDR
$MESSAGE\n")
        frac_digits(3) template-escape(no));
    pipe("/var/log/logstash-pipe"
        template("$R_ISODATE <$FACILITY_NUM.$LEVEL_NUM> $HOST $MSGHDR
$MESSAGE\n")
        frac_digits(3) template-escape(no));
};
destination d_userauth {
    tcp("abc.xyz.com" port(13456));
};
destination d_otto {
    tcp("abc.xyz.com" port(13456));
    tcp("abc.xyz.com" port(13456));
};
destination d_notifier {
    tcp("abc.xyz.com" port(6000));
    tcp("10.255.xx.yy" port(1248));
};
destination d_graylog {
    syslog("abc.xyz.com" transport("tcp") port(9514));
    pipe("/var/log/graylog-pipe" ts-format(iso));
};
destination d_gtsadfeed {
    syslog("abc.xyz.com" transport("udp") port(514));
};
filter f_kernel             { facility(kern); };
filter f_default            { level(info..emerg)
                              and not (facility(mail)
                                  or facility(authpriv)
                                  or facility(cron))
                              and not message("Syslog connection
(accepted|closed)"); };
filter f_auth               { facility(auth,authpriv); };
filter f_mail               { facility(mail); };
filter f_emergency          { level(emerg); };
filter f_news               { facility(uucp) or
                              (facility(news)
                                  and level(crit..emerg)); };
filter f_boot               { facility(local7); };
filter f_cron               { facility(cron,solaris-cron); };
filter f_user               { facility(user); };
filter f_daemon             { facility(daemon); };
filter f_lpr                { facility(lpr); };
filter f_dnslogs            { program("mydns_logger"); };
filter f_local0             { facility(local0); };
filter f_local1             { facility(local1) and not
program("mydns_logger"); };
filter f_local2             { facility(local2); };
filter f_local3             { facility(local3); };
filter f_local4             { facility(local4); };
filter f_local5             { facility(local5) or tags("source_net"); }; #
local5 is network, so is port 515
filter f_local6             { facility(local6); };
filter f_local7             { facility(local7); };
filter f_alert              { level(alert); };
filter f_crit               { level(crit); };
filter f_err                { level(err); };
filter f_warn               { level(warn); };
filter f_notice             { level(notice); };
filter f_info               { level(info); };
filter f_debug              { level(debug); };
filter f_nonet              { not facility(local4) and not facility(local5)
and not tags("source_net"); };
filter f_nocf3              { not program("cf3"); };
filter f_nodhcpd            { not program("dhcpd"); };
filter f_windows            { tags("windows"); };
filter f_snmptrapd          { program("snmptrapd"); };
filter f_userauth           { not facility(local3); };
filter f_notifier_filter    { not match("ASA-4-302015|ASA-4-
302013|TRAFFIC|permitted|Deny|Denied|denied", value("MESSAGE")); };
filter f_gtsadfeed_filter    { host("*-adc*.abc.xyz.com" type(glob)); };
rewrite f_rewrite_name      { set("$FULLHOST_FROM", value("HOST")
                              condition(not tags("windows")
                                        and not match("REMOTELOG",
value("MESSAGE")))); };
rewrite r_rewrite_name_windows { set("$FULLHOST_FROM", value("HOST")
                                 condition(not match("REMOTELOG",
value("MESSAGE")))); };
rewrite r_snmptrapd         { subst("^([^ ]+) (.*)", "${2}",
value("MESSAGE"));
                              set("${1}", value("HOST")); };
log { source(s_sys); filter(f_kernel); destination(d_kern); };
log { source(s_sys); filter(f_default); destination(d_mesg); };
log { source(s_sys); filter(f_auth); destination(d_auth); };
log { source(s_sys); filter(f_mail); destination(d_mail); };
log { source(s_sys); filter(f_emergency); destination(d_mlal); };
log { source(s_sys); filter(f_news); destination(d_spol); };
log { source(s_sys); filter(f_boot); destination(d_boot); };
log { source(s_sys); filter(f_cron); destination(d_cron); };
log {
    source(s_sys);
    filter(f_snmptrapd);
    rewrite(r_snmptrapd);
    destination(d_networkx);
    destination(d_logstash);
    log {
    filter(f_notifier_filter);
    destination(d_notifier);
    };
};
log {
    source(s_net);
    source(s_udp);
    source(s_tcp);
    source(s_sys);
    source(s_windows);
    log {
        filter(f_userauth);
        destination(d_userauth);
        destination(d_otto);
    };
    log {
        rewrite(f_rewrite_name);
        log {
            destination(d_logstash);
        };
        log {
            filter(f_nocf3);
            filter(f_nodhcpd);
            destination(d_graylog);
        };
        log {
            filter(f_user);
            destination(d_userx);
        };
        log {
            filter(f_kernel);
            destination(d_kernx);
        };
        log {
            filter(f_mail);
            destination(d_mailx);
        };
        log {
            filter(f_daemon);
            destination(d_daemonx);
        };
        log {
            filter(f_auth);
            destination(d_authx);
        };
        log {
            filter(f_lpr);
            destination(d_lprx);
        };
        log {
            filter(f_cron);
            destination(d_cronx);
        };
        log {
            filter(f_local0);
            destination(d_local0);
        };
        log {
            filter(f_local1);
            destination(d_local1);
        };
        log {
            filter(f_local2);
            destination(d_authx);
        };
        log {
            filter(f_dnslogs);
            destination(d_dnslogs);
        };
        log {
            filter(f_local4);
            destination(d_firewallx);
            log {
                filter(f_notifier_filter);
                destination(d_notifier);
            };
        };
        log {
            filter(f_local5);
            destination(d_networkx);
            log {
                filter(f_notifier_filter);
                destination(d_notifier);
            };
        };
        log {
            filter(f_local6);
            destination(d_local6);
        };
        log {
            filter(f_local7);
            destination(d_mp);
        };
        log {
            filter(f_windows);
            filter(f_gtsadfeed_filter);
            rewrite(r_rewrite_name_windows);
            destination(d_gtsadfeed);
        };
    };
};

Could you please advise?

Regards,
Soumyadip

[syslog-ng] Fwd: syslog-ng 3.7.3 crashing randomly

Soumyadip Das Mahapatra