[syslog-ng] Fwd: syslog-ng 3.7.3 crashing randomly
Soumyadip Das Mahapatra
soumyadip.bt at gmail.com
Tue Oct 25 11:01:43 UTC 2016
Hi,
We have installed syslog-ng 3.7.3 on RHEL 6.6 from COPR repo:
>From dmesg:
[Thu Sep 1 02:24:08 2016] syslog-ng[10374]: segfault at 1a116c10 ip
000000001a116c10 sp 00007f26eebfa928 error 15
[Sun Sep 4 20:10:47 2016] syslog-ng[27797]: segfault at 0 ip (null) sp
00007f998b5fb928 error 14 in syslog-ng[400000+3000]
[Mon Sep 5 19:04:15 2016] syslog-ng[12727]: segfault at 36aaa42080 ip
00000039b8618df0 sp 00007f996fffc878 error 7 in libglib-2.0.so.0.2800.8[
39b8600000+115000]
[Tue Sep 6 03:01:21 2016] syslog-ng[3827]: segfault at 15cb09d0 ip
0000000015cb09d0 sp 00007f99aebfa928 error 15
[Fri Sep 9 05:03:41 2016] syslog-ng[28275] general protection
ip:39b8618df0 sp:7f29c57f8878 error:0 in libglib-2.0.so.0.2800.8[
39b8600000+115000]
[Sun Sep 11 00:48:01 2016] syslog-ng[14479]: segfault at 0 ip (null) sp
00007f2a03caf928 error 14 in syslog-ng[400000+3000]
[Sun Sep 11 10:15:37 2016] syslog-ng[2012]: segfault at 0 ip
00000036aaa3cc1c sp 00007feed2bfa880 error 6 in libsyslog-ng-3.7.so.0.0.0[
36aaa00000+a2000]
Core 1:
(gdb) bt full
#0 0x00000039b8618df0 in g_atomic_int_add () from /lib64/libglib-2.0.so.0
No symbol table info available.
#1 0x00000036aaa3cbe4 in stats_counter_add (self=0xabe68ea0, thread_id=5)
at lib/stats/stats-counter.h:39
No locals.
#2 log_queue_fifo_move_input_unlocked (self=0xabe68ea0, thread_id=5)
at lib/logqueue-fifo.c:193
queue_len = 1755882337
#3 0x00000036aaa3cd6e in log_queue_fifo_move_input (user_data=0xabe68ea0)
at lib/logqueue-fifo.c:215
self = 0xabe68ea0
thread_id = 5
__PRETTY_FUNCTION__ = "log_queue_fifo_move_input"
#4 0x00000036aaa456ca in main_loop_worker_invoke_batch_callbacks ()
at lib/mainloop-worker.c:270
cb = 0xabe690e0
lh = 0xabe690e0
lh2 = 0xabe690e0
#5 0x00000036aaa6fa2a in iv_work_thread_do_work (_thr=0x9d6bd530) at
iv_work.c:118
work = 0x6fb160
thr = 0x9d6bd530
pool = 0x6e48c0
last_seq = 1477269410
#6 0x00000036aaa6ed4a in iv_run_tasks (st=0x7f29b44d3c00) at iv_task.c:48
t = <value optimized out>
tasks = {next = 0x7f29c57f89a0, prev = 0x7f29c57f89a0}
#7 0x00000036aaa7100c in iv_main () at iv_main_posix.c:106
to = {tv_sec = 10, tv_nsec = 0}
st = 0x7f29b44d3c00
#8 0x00000036aaa6f841 in iv_work_thread (_thr=0x9d6bd530) at iv_work.c:200
thr = 0x9d6bd530
pool = 0x6e48c0
#9 0x00000036aaa71a1f in iv_thread_handler (_thr=0x75242f90)
at iv_thread_posix.c:142
__clframe = {__cancel_routine = 0x36aaa71a80
<iv_thread_cleanup_handler>,
__cancel_arg = 0x75242f90, __do_it = 1, __cancel_type = 0}
thr = 0x75242f90
#10 0x00000039b6e079d1 in start_thread () from /lib64/libpthread.so.0
No symbol table info available.
#11 0x00000039b6ae88fd in clone () from /lib64/libc.so.6
No symbol table info available.
Core 2:
(gdb) bt full
#0 0x0000000000000000 in ?? ()
No symbol table info available.
#1 0x00000036aaa456ca in main_loop_worker_invoke_batch_callbacks ()
at lib/mainloop-worker.c:270
cb = 0x87e96b10
lh = 0x87e96b10
lh2 = 0x0
#2 0x00000036aaa6fa2a in iv_work_thread_do_work (_thr=0x7034ffd0) at
iv_work.c:118
work = 0x6fb390
thr = 0x7034ffd0
pool = 0x6e48b0
last_seq = 1323426114
#3 0x00000036aaa6ed4a in iv_run_tasks (st=0x7f29d4360200) at iv_task.c:48
t = <value optimized out>
tasks = {next = 0x7f2a03caf9a0, prev = 0x7f2a03caf9a0}
#4 0x00000036aaa7100c in iv_main () at iv_main_posix.c:106
to = {tv_sec = 10, tv_nsec = 0}
st = 0x7f29d4360200
#5 0x00000036aaa6f841 in iv_work_thread (_thr=0x7034ffd0) at iv_work.c:200
thr = 0x7034ffd0
pool = 0x6e48b0
#6 0x00000036aaa71a1f in iv_thread_handler (_thr=0x426b490)
at iv_thread_posix.c:142
__clframe = {__cancel_routine = 0x36aaa71a80
<iv_thread_cleanup_handler>,
__cancel_arg = 0x426b490, __do_it = 1, __cancel_type = 0}
thr = 0x426b490
#7 0x00000039b6e079d1 in start_thread () from /lib64/libpthread.so.0
No symbol table info available.
#8 0x00000039b6ae88fd in clone () from /lib64/libc.so.6
No symbol table info available.
So looks like segfault is happening at random part of the code.
# cat /etc/sysconfig/syslog-ng
#---
# Syslog-ng command line options
# See syslog-ng(8) for more details
#---
SYSLOGNG_PID="/var/run/syslog-ng.pid"
SYSLOGNG_OPTIONS="-p $SYSLOGNG_PID --fd-limit 30000"
SYSLOGNG_COMPAT_PID="/var/run/syslogd.pid"
# cat /etc/syslog-ng/syslog-ng.conf
@version:3.6
options {
flush_lines (0);
time_reopen (10);
log_fifo_size (20000);
use_dns (yes);
use_fqdn (yes);
create_dirs (yes);
dir_group("wheel");
dir_owner("nobody");
dir_perm(0755);
owner("nobody");
group("nobody");
perm(0644);
threaded(yes);
keep_hostname (yes);
chain_hostnames(yes);
bad_hostname("[^[:print:]]");
dns_cache(yes);
dns_cache_expire(300);
dns_cache_expire_failed(30);
dns_cache_size(1000);
stats_freq(3600);
flush_lines(0);
};
source s_sys {
file ("/proc/kmsg" program_override("kernel: "));
unix-stream ("/dev/log");
# udp(ip(0.0.0.0) port(514));
};
source s_tcp {
network(transport("tcp") port(514) so_rcvbuf(8388608)
max-connections(200) log-iw-size(20000));
};
source s_udp {
network(transport("udp") port(514) so_rcvbuf(8388608)
log-iw-size(20000));
};
source s_net {
network(transport("udp") port(515) so_rcvbuf(8388608)
log-iw-size(20000) tags("source_net"));
network(transport("tcp") port(515) so_rcvbuf(8388608)
max-connections(200) log-iw-size(20000) tags("source_net"));
};
source s_windows {
network(transport("tcp") port(516) so_rcvbuf(8388608)
max-connections(200) log-iw-size(20000) tags("windows"));
};
destination d_cons { file("/dev/console"); };
destination d_mesg { file("/var/log/messages"); };
destination d_auth { file("/var/log/secure"); };
destination d_mail { file("/var/log/maillog" flush_lines(10)); };
destination d_spol { file("/var/log/spooler"); };
destination d_boot { file("/var/log/boot.log"); };
destination d_cron { file("/var/log/cron"); };
destination d_kern { file("/var/log/kern"); };
destination d_mlal { usertty("*"); };
destination d_userx { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/user.log"); };
destination d_kernx { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/kern.log"); };
destination d_mailx { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/mail.log"); };
destination d_mp { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/mp.log"); };
destination d_daemonx { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/daemon.log");
};
destination d_authx { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/auth.log"); };
destination d_lprx { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/lpr.log"); };
destination d_cronx { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/cron.log"); };
destination d_messagesx {
file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/messages.log");
};
destination d_networkx { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/network.log");
};
destination d_firewallx {
file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/firewall.log");
};
destination d_local0 { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/sp.log"); };
destination d_local1 { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/app.log"); };
destination d_localx { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/local.log");
};
destination d_local6 { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/httpd.log");
};
destination d_dnslogs { file("/u1/syslog/$HOST/$YEAR$MONTH$DAY/dnslogs.log");
};
destination d_logstash {
tcp("127.0.0.1" port(5140)
template("$R_ISODATE <$FACILITY_NUM.$LEVEL_NUM> $HOST $MSGHDR
$MESSAGE\n")
frac_digits(3) template-escape(no));
pipe("/var/log/logstash-pipe"
template("$R_ISODATE <$FACILITY_NUM.$LEVEL_NUM> $HOST $MSGHDR
$MESSAGE\n")
frac_digits(3) template-escape(no));
};
destination d_userauth {
tcp("abc.xyz.com" port(13456));
};
destination d_otto {
tcp("abc.xyz.com" port(13456));
tcp("abc.xyz.com" port(13456));
};
destination d_notifier {
tcp("abc.xyz.com" port(6000));
tcp("10.255.xx.yy" port(1248));
};
destination d_graylog {
syslog("abc.xyz.com" transport("tcp") port(9514));
pipe("/var/log/graylog-pipe" ts-format(iso));
};
destination d_gtsadfeed {
syslog("abc.xyz.com" transport("udp") port(514));
};
filter f_kernel { facility(kern); };
filter f_default { level(info..emerg)
and not (facility(mail)
or facility(authpriv)
or facility(cron))
and not message("Syslog connection
(accepted|closed)"); };
filter f_auth { facility(auth,authpriv); };
filter f_mail { facility(mail); };
filter f_emergency { level(emerg); };
filter f_news { facility(uucp) or
(facility(news)
and level(crit..emerg)); };
filter f_boot { facility(local7); };
filter f_cron { facility(cron,solaris-cron); };
filter f_user { facility(user); };
filter f_daemon { facility(daemon); };
filter f_lpr { facility(lpr); };
filter f_dnslogs { program("mydns_logger"); };
filter f_local0 { facility(local0); };
filter f_local1 { facility(local1) and not
program("mydns_logger"); };
filter f_local2 { facility(local2); };
filter f_local3 { facility(local3); };
filter f_local4 { facility(local4); };
filter f_local5 { facility(local5) or tags("source_net"); }; #
local5 is network, so is port 515
filter f_local6 { facility(local6); };
filter f_local7 { facility(local7); };
filter f_alert { level(alert); };
filter f_crit { level(crit); };
filter f_err { level(err); };
filter f_warn { level(warn); };
filter f_notice { level(notice); };
filter f_info { level(info); };
filter f_debug { level(debug); };
filter f_nonet { not facility(local4) and not facility(local5)
and not tags("source_net"); };
filter f_nocf3 { not program("cf3"); };
filter f_nodhcpd { not program("dhcpd"); };
filter f_windows { tags("windows"); };
filter f_snmptrapd { program("snmptrapd"); };
filter f_userauth { not facility(local3); };
filter f_notifier_filter { not match("ASA-4-302015|ASA-4-
302013|TRAFFIC|permitted|Deny|Denied|denied", value("MESSAGE")); };
filter f_gtsadfeed_filter { host("*-adc*.abc.xyz.com" type(glob)); };
rewrite f_rewrite_name { set("$FULLHOST_FROM", value("HOST")
condition(not tags("windows")
and not match("REMOTELOG",
value("MESSAGE")))); };
rewrite r_rewrite_name_windows { set("$FULLHOST_FROM", value("HOST")
condition(not match("REMOTELOG",
value("MESSAGE")))); };
rewrite r_snmptrapd { subst("^([^ ]+) (.*)", "${2}",
value("MESSAGE"));
set("${1}", value("HOST")); };
log { source(s_sys); filter(f_kernel); destination(d_kern); };
log { source(s_sys); filter(f_default); destination(d_mesg); };
log { source(s_sys); filter(f_auth); destination(d_auth); };
log { source(s_sys); filter(f_mail); destination(d_mail); };
log { source(s_sys); filter(f_emergency); destination(d_mlal); };
log { source(s_sys); filter(f_news); destination(d_spol); };
log { source(s_sys); filter(f_boot); destination(d_boot); };
log { source(s_sys); filter(f_cron); destination(d_cron); };
log {
source(s_sys);
filter(f_snmptrapd);
rewrite(r_snmptrapd);
destination(d_networkx);
destination(d_logstash);
log {
filter(f_notifier_filter);
destination(d_notifier);
};
};
log {
source(s_net);
source(s_udp);
source(s_tcp);
source(s_sys);
source(s_windows);
log {
filter(f_userauth);
destination(d_userauth);
destination(d_otto);
};
log {
rewrite(f_rewrite_name);
log {
destination(d_logstash);
};
log {
filter(f_nocf3);
filter(f_nodhcpd);
destination(d_graylog);
};
log {
filter(f_user);
destination(d_userx);
};
log {
filter(f_kernel);
destination(d_kernx);
};
log {
filter(f_mail);
destination(d_mailx);
};
log {
filter(f_daemon);
destination(d_daemonx);
};
log {
filter(f_auth);
destination(d_authx);
};
log {
filter(f_lpr);
destination(d_lprx);
};
log {
filter(f_cron);
destination(d_cronx);
};
log {
filter(f_local0);
destination(d_local0);
};
log {
filter(f_local1);
destination(d_local1);
};
log {
filter(f_local2);
destination(d_authx);
};
log {
filter(f_dnslogs);
destination(d_dnslogs);
};
log {
filter(f_local4);
destination(d_firewallx);
log {
filter(f_notifier_filter);
destination(d_notifier);
};
};
log {
filter(f_local5);
destination(d_networkx);
log {
filter(f_notifier_filter);
destination(d_notifier);
};
};
log {
filter(f_local6);
destination(d_local6);
};
log {
filter(f_local7);
destination(d_mp);
};
log {
filter(f_windows);
filter(f_gtsadfeed_filter);
rewrite(r_rewrite_name_windows);
destination(d_gtsadfeed);
};
};
};
Could you please advise?
Regards,
Soumyadip
-------------- next part --------------
An HTML attachment was scrubbed...
URL: <http://lists.balabit.hu/pipermail/syslog-ng/attachments/20161025/69914da6/attachment-0001.html>
More information about the syslog-ng
mailing list