Hi,

We have installed syslog-ng 3.7.3 on RHEL 6.6 from COPR repo: 

From dmesg: 

[Thu Sep  1 02:24:08 2016] syslog-ng[10374]: segfault at 1a116c10 ip 000000001a116c10 sp 00007f26eebfa928 error 15

[Sun Sep  4 20:10:47 2016] syslog-ng[27797]: segfault at 0 ip (null) sp 00007f998b5fb928 error 14 in syslog-ng[400000+3000]

[Mon Sep  5 19:04:15 2016] syslog-ng[12727]: segfault at 36aaa42080 ip 00000039b8618df0 sp 00007f996fffc878 error 7 in libglib-2.0.so.0.2800.8[39b8600000+115000]

[Tue Sep  6 03:01:21 2016] syslog-ng[3827]: segfault at 15cb09d0 ip 0000000015cb09d0 sp 00007f99aebfa928 error 15

[Fri Sep  9 05:03:41 2016] syslog-ng[28275] general protection ip:39b8618df0 sp:7f29c57f8878 error:0 in libglib-2.0.so.0.2800.8[39b8600000+115000]

[Sun Sep 11 00:48:01 2016] syslog-ng[14479]: segfault at 0 ip (null) sp 00007f2a03caf928 error 14 in syslog-ng[400000+3000]

[Sun Sep 11 10:15:37 2016] syslog-ng[2012]: segfault at 0 ip 00000036aaa3cc1c sp 00007feed2bfa880 error 6 in libsyslog-ng-3.7.so.0.0.0[36aaa00000+a2000]

Core 1:

(gdb) bt full

#0  0x00000039b8618df0 in g_atomic_int_add () from /lib64/libglib-2.0.so.0

No symbol table info available.

#1  0x00000036aaa3cbe4 in stats_counter_add (self=0xabe68ea0, thread_id=5)

    at lib/stats/stats-counter.h:39

No locals.

#2  log_queue_fifo_move_input_unlocked (self=0xabe68ea0, thread_id=5)

    at lib/logqueue-fifo.c:193

        queue_len = 1755882337

#3  0x00000036aaa3cd6e in log_queue_fifo_move_input (user_data=0xabe68ea0)

    at lib/logqueue-fifo.c:215

        self = 0xabe68ea0

        thread_id = 5

        __PRETTY_FUNCTION__ = "log_queue_fifo_move_input"

#4  0x00000036aaa456ca in main_loop_worker_invoke_batch_callbacks ()

    at lib/mainloop-worker.c:270

        cb = 0xabe690e0

        lh = 0xabe690e0

        lh2 = 0xabe690e0

#5  0x00000036aaa6fa2a in iv_work_thread_do_work (_thr=0x9d6bd530) at iv_work.c:118

        work = 0x6fb160

        thr = 0x9d6bd530

        pool = 0x6e48c0

        last_seq = 1477269410

#6  0x00000036aaa6ed4a in iv_run_tasks (st=0x7f29b44d3c00) at iv_task.c:48

        t = <value optimized out>

        tasks = {next = 0x7f29c57f89a0, prev = 0x7f29c57f89a0}

#7  0x00000036aaa7100c in iv_main () at iv_main_posix.c:106

        to = {tv_sec = 10, tv_nsec = 0}

        st = 0x7f29b44d3c00

#8  0x00000036aaa6f841 in iv_work_thread (_thr=0x9d6bd530) at iv_work.c:200

        thr = 0x9d6bd530

        pool = 0x6e48c0

#9  0x00000036aaa71a1f in iv_thread_handler (_thr=0x75242f90)

    at iv_thread_posix.c:142

        __clframe = {__cancel_routine = 0x36aaa71a80 <iv_thread_cleanup_handler>,

          __cancel_arg = 0x75242f90, __do_it = 1, __cancel_type = 0}

        thr = 0x75242f90

#10 0x00000039b6e079d1 in start_thread () from /lib64/libpthread.so.0

No symbol table info available.

#11 0x00000039b6ae88fd in clone () from /lib64/libc.so.6

No symbol table info available.

Core 2:

(gdb) bt full

#0  0x0000000000000000 in ?? ()

No symbol table info available.

#1  0x00000036aaa456ca in main_loop_worker_invoke_batch_callbacks ()

    at lib/mainloop-worker.c:270

        cb = 0x87e96b10

        lh = 0x87e96b10

        lh2 = 0x0

#2  0x00000036aaa6fa2a in iv_work_thread_do_work (_thr=0x7034ffd0) at iv_work.c:118

        work = 0x6fb390

        thr = 0x7034ffd0

        pool = 0x6e48b0

        last_seq = 1323426114

#3  0x00000036aaa6ed4a in iv_run_tasks (st=0x7f29d4360200) at iv_task.c:48

        t = <value optimized out>

        tasks = {next = 0x7f2a03caf9a0, prev = 0x7f2a03caf9a0}

#4  0x00000036aaa7100c in iv_main () at iv_main_posix.c:106

        to = {tv_sec = 10, tv_nsec = 0}

        st = 0x7f29d4360200

#5  0x00000036aaa6f841 in iv_work_thread (_thr=0x7034ffd0) at iv_work.c:200

        thr = 0x7034ffd0

        pool = 0x6e48b0

#6  0x00000036aaa71a1f in iv_thread_handler (_thr=0x426b490)

    at iv_thread_posix.c:142

        __clframe = {__cancel_routine = 0x36aaa71a80 <iv_thread_cleanup_handler>,

          __cancel_arg = 0x426b490, __do_it = 1, __cancel_type = 0}

        thr = 0x426b490

#7  0x00000039b6e079d1 in start_thread () from /lib64/libpthread.so.0

No symbol table info available.

#8  0x00000039b6ae88fd in clone () from /lib64/libc.so.6

No symbol table info available.

So looks like segfault is happening at random part of the code.

# cat /etc/sysconfig/syslog-ng

#---

# Syslog-ng command line options

# See syslog-ng(8) for more details

#---

SYSLOGNG_PID="/var/run/syslog-ng.pid"

SYSLOGNG_OPTIONS="-p $SYSLOGNG_PID --fd-limit 30000"

SYSLOGNG_COMPAT_PID="/var/run/syslogd.pid"


# cat /etc/syslog-ng/syslog-ng.conf

@version:3.6

options {

    flush_lines (0);

    time_reopen (10);

    log_fifo_size (20000);

    use_dns (yes);

    use_fqdn (yes);

    create_dirs (yes);

    dir_group("wheel");

    dir_owner("nobody");

    dir_perm(0755);

    owner("nobody");

    group("nobody");

    perm(0644);

    threaded(yes);

    keep_hostname (yes);

    chain_hostnames(yes);

    bad_hostname("[^[:print:]]");

    dns_cache(yes);

    dns_cache_expire(300);

    dns_cache_expire_failed(30);

    dns_cache_size(1000);

    stats_freq(3600);

    flush_lines(0);

};

source s_sys {

    file ("/proc/kmsg" program_override("kernel: "));

    unix-stream ("/dev/log");

    # udp(ip(0.0.0.0) port(514));

};

source s_tcp {

    network(transport("tcp") port(514) so_rcvbuf(8388608) max-connections(200) log-iw-size(20000));

};

source s_udp {

    network(transport("udp") port(514) so_rcvbuf(8388608) log-iw-size(20000));

};

source s_net {

    network(transport("udp") port(515) so_rcvbuf(8388608) log-iw-size(20000) tags("source_net"));

    network(transport("tcp") port(515) so_rcvbuf(8388608) max-connections(200) log-iw-size(20000) tags("source_net"));

};

source s_windows {

    network(transport("tcp") port(516) so_rcvbuf(8388608) max-connections(200) log-iw-size(20000) tags("windows"));

};

destination d_cons { file("/dev/console"); };

destination d_mesg { file("/var/log/messages"); };

destination d_auth { file("/var/log/secure"); };

destination d_mail { file("/var/log/maillog" flush_lines(10)); };

destination d_spol { file("/var/log/spooler"); };

destination d_boot { file("/var/log/boot.log"); };

destination d_cron { file("/var/log/cron"); };

destination d_kern { file("/var/log/kern"); };

destination d_mlal { usertty("*"); };

destination d_userx { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/user.log"); };

destination d_kernx { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/kern.log"); };

destination d_mailx { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/mail.log"); };

destination d_mp { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/mp.log"); };

destination d_daemonx { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/daemon.log"); };

destination d_authx { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/auth.log"); };

destination d_lprx { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/lpr.log"); };

destination d_cronx { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/cron.log"); };

destination d_messagesx { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/messages.log"); };

destination d_networkx { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/network.log"); };

destination d_firewallx { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/firewall.log"); };

destination d_local0 { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/sp.log"); };

destination d_local1 { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/app.log"); };

destination d_localx { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/local.log"); };

destination d_local6 { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/httpd.log"); };

destination d_dnslogs { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/dnslogs.log"); };

destination d_logstash {

    tcp("127.0.0.1" port(5140)

        template("$R_ISODATE <$FACILITY_NUM.$LEVEL_NUM> $HOST $MSGHDR $MESSAGE\n")

        frac_digits(3) template-escape(no));

    pipe("/var/log/logstash-pipe"

        template("$R_ISODATE <$FACILITY_NUM.$LEVEL_NUM> $HOST $MSGHDR $MESSAGE\n")

        frac_digits(3) template-escape(no));

};

destination d_userauth {

    tcp("abc.xyz.com" port(13456));

};

destination d_otto {

    tcp("abc.xyz.com" port(13456));

    tcp("abc.xyz.com" port(13456));

};

destination d_notifier {

    tcp("abc.xyz.com" port(6000));

    tcp("10.255.xx.yy" port(1248));

};

destination d_graylog {

    syslog("abc.xyz.com" transport("tcp") port(9514));

    pipe("/var/log/graylog-pipe" ts-format(iso));

};

destination d_gtsadfeed {

    syslog("abc.xyz.com" transport("udp") port(514));

};

filter f_kernel             { facility(kern); };

filter f_default            { level(info..emerg)

                              and not (facility(mail)

                                  or facility(authpriv)

                                  or facility(cron))

                              and not message("Syslog connection (accepted|closed)"); };

filter f_auth               { facility(auth,authpriv); };

filter f_mail               { facility(mail); };

filter f_emergency          { level(emerg); };

filter f_news               { facility(uucp) or

                              (facility(news)

                                  and level(crit..emerg)); };

filter f_boot               { facility(local7); };

filter f_cron               { facility(cron,solaris-cron); };

filter f_user               { facility(user); };

filter f_daemon             { facility(daemon); };

filter f_lpr                { facility(lpr); };

filter f_dnslogs            { program("mydns_logger"); };

filter f_local0             { facility(local0); };

filter f_local1             { facility(local1) and not program("mydns_logger"); };

filter f_local2             { facility(local2); };

filter f_local3             { facility(local3); };

filter f_local4             { facility(local4); };

filter f_local5             { facility(local5) or tags("source_net"); }; # local5 is network, so is port 515

filter f_local6             { facility(local6); };

filter f_local7             { facility(local7); };

filter f_alert              { level(alert); };

filter f_crit               { level(crit); };

filter f_err                { level(err); };

filter f_warn               { level(warn); };

filter f_notice             { level(notice); };

filter f_info               { level(info); };

filter f_debug              { level(debug); };

filter f_nonet              { not facility(local4) and not facility(local5) and not tags("source_net"); };

filter f_nocf3              { not program("cf3"); };

filter f_nodhcpd            { not program("dhcpd"); };

filter f_windows            { tags("windows"); };

filter f_snmptrapd          { program("snmptrapd"); };

filter f_userauth           { not facility(local3); };

filter f_notifier_filter    { not match("ASA-4-302015|ASA-4-302013|TRAFFIC|permitted|Deny|Denied|denied", value("MESSAGE")); };

filter f_gtsadfeed_filter    { host("*-adc*.abc.xyz.com" type(glob)); };

rewrite f_rewrite_name      { set("$FULLHOST_FROM", value("HOST")

                              condition(not tags("windows")

                                        and not match("REMOTELOG", value("MESSAGE")))); };

rewrite r_rewrite_name_windows { set("$FULLHOST_FROM", value("HOST")

                                 condition(not match("REMOTELOG", value("MESSAGE")))); };

rewrite r_snmptrapd         { subst("^([^ ]+) (.*)", "${2}", value("MESSAGE"));

                              set("${1}", value("HOST")); };

log { source(s_sys); filter(f_kernel); destination(d_kern); };

log { source(s_sys); filter(f_default); destination(d_mesg); };

log { source(s_sys); filter(f_auth); destination(d_auth); };

log { source(s_sys); filter(f_mail); destination(d_mail); };

log { source(s_sys); filter(f_emergency); destination(d_mlal); };

log { source(s_sys); filter(f_news); destination(d_spol); };

log { source(s_sys); filter(f_boot); destination(d_boot); };

log { source(s_sys); filter(f_cron); destination(d_cron); };

log {

    source(s_sys);

    filter(f_snmptrapd);

    rewrite(r_snmptrapd);

    destination(d_networkx);

    destination(d_logstash);

    log {

    filter(f_notifier_filter);

    destination(d_notifier);

    };

};

log {

    source(s_net);

    source(s_udp);

    source(s_tcp);

    source(s_sys);

    source(s_windows);

    log {

        filter(f_userauth);

        destination(d_userauth);

        destination(d_otto);

    };

    log {

        rewrite(f_rewrite_name);

        log {

            destination(d_logstash);

        };

        log {

            filter(f_nocf3);

            filter(f_nodhcpd);

            destination(d_graylog);

        };

        log {

            filter(f_user);

            destination(d_userx);

        };

        log {

            filter(f_kernel);

            destination(d_kernx);

        };

        log {

            filter(f_mail);

            destination(d_mailx);

        };

        log {

            filter(f_daemon);

            destination(d_daemonx);

        };

        log {

            filter(f_auth);

            destination(d_authx);

        };

        log {

            filter(f_lpr);

            destination(d_lprx);

        };

        log {

            filter(f_cron);

            destination(d_cronx);

        };

        log {

            filter(f_local0);

            destination(d_local0);

        };

        log {

            filter(f_local1);

            destination(d_local1);

        };

        log {

            filter(f_local2);

            destination(d_authx);

        };

        log {

            filter(f_dnslogs);

            destination(d_dnslogs);

        };

        log {

            filter(f_local4);

            destination(d_firewallx);

            log {

                filter(f_notifier_filter);

                destination(d_notifier);

            };

        };

        log {

            filter(f_local5);

            destination(d_networkx);

            log {

                filter(f_notifier_filter);

                destination(d_notifier);

            };

        };

        log {

            filter(f_local6);

            destination(d_local6);

        };

        log {

            filter(f_local7);

            destination(d_mp);

        };

        log {

            filter(f_windows);

            filter(f_gtsadfeed_filter);

            rewrite(r_rewrite_name_windows);

            destination(d_gtsadfeed);

        };

    };

};

Could you please advise?

Regards,

Soumyadip

______________________________________________________________________________
Member info: https://lists.balabit.hu/mailman/listinfo/syslog-ng
Documentation: http://www.balabit.com/support/documentation/?product=syslog-ng
FAQ: http://www.balabit.com/wiki/syslog-ng-faq