I am running syslog-ng 3.4.7 as distributed by ELSA. I have a problem
whereby nxlog agents (v2.9.1347) making a new TCP connection to the
syslog-ng server repeatedly show lines like this in nxlog.log:
2015-09-28 14:28:25 INFO reconnecting in 1 seconds
Many connections are working, but it especially has a problem with new
agents coming online. Sometimes when I restart syslog-ng it helps, but
not always.
Other than the standard ELSA config, I am simply also writing to a
filesystem destination. If I remove the elsa.pl destination, I still
have problems.
The problem isn't necessarily related to just nxlog agents. I also have
a Cisco proxy that is using TCP to send logs, and that is also getting
RSTs sent to it.
I tried v3.6.3 and that didn't seem to help.
A packet capture shows the SYN from the agent, a SYN ACK, another SYN,
then a FIN *prior* to the data packet being received. Immediately after
the data packet, the log server sends a RST. As a result, I have a bunch
of TIME_WAITs open.
No host firewall is running.
I am currently only bursting to about 350 EPS and I have syslog-ng
running in the foreground looking for dropped packets like so (there
haven't been any): /usr/local/syslog-ng-3.4.7/sbin/syslog-ng -Ftvd 2>&1
| grep "dropping messages"
Any assistance would be greatly appreciated!
Currently loaded syslog-ng.conf:
# cat /usr/local/syslog-ng/etc/syslog-ng.conf
@version: 3.2
options {
log_fifo_size(30000);
#threaded(yes); # enable if using Syslog-NG 3.3.x
use_dns (no); # IMPORTANT! A busy syslog server can become
unresponsive if DNS lookups fail, so we disable them.
use_fqdn (no);
keep_hostname (yes);
};
source s_network {
# no-multi-line means parse message as-is, newlines included, to
allow for multi-line messages
tcp(flags(no-multi-line));
udp(flags(no-multi-line));
};
source s_internal {
internal();
};
source s_realtime {
pipe("/data/elsa/tmp/realtime" flags(expect-hostname));
};
source s_import {
pipe("/data/elsa/tmp/import" flags(expect-hostname
syslog-protocol));
};
source s_ops {
unix-dgram("/data/elsa/tmp/ops" flags(expect-hostname
syslog-protocol no-multi-line));
};
parser p_db {
db-parser(file("/usr/local/elsa/node/conf/merged.xml"));
};
filter f_rewrite_cisco_program { match('^(%[A-Z\_]+\-\d\-[0-9A-Z\_]+):
([^\n]+)' value("MSGONLY") type("pcre") flags("store-matches"
"nobackref")); };
filter f_rewrite_cisco_program_2 {
match('^[\*\.]?(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d{1,2}\s\d{1,2}:\d{1,2}:\d{1,2}(?:\.\d+)?(?:
[A-Z]{3})?: (%[^:]+): ([^\n]+)' value("MSGONLY") type("pcre")
flags("store-matches" "nobackref")); };
filter f_rewrite_cisco_program_3 { match('^\d+[ywdh]\d+[ywdh]: (%[^:]+):
([^\n]+)' value("MSGONLY") type("pcre") flags("store-matches"
"nobackref")); };
filter f_rewrite_cisco_program_4 { match('^\d{6}:
[\*\.]?(?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+\d{1,2}\s\d{1,2}:\d{1,2}:\d{1,2}(?:\.\d+)?(?:
[A-Z]{3})?: (%[^:]+): ([^\n]+)' value("MSGONLY") type("pcre")
flags("store-matches" "nobackref")); };
filter f_unclassified { "${.classifier.class}" eq "unknown" };
rewrite r_unparsed { set("98", value("MSGID")); };
rewrite r_cisco_program {
set("$1", value("PROGRAM")
condition(filter(f_rewrite_cisco_program) or
filter(f_rewrite_cisco_program_2) or filter(f_rewrite_cisco_program_3)
or filter(f_rewrite_cisco_program_4)));
set("$2", value("MESSAGE")
condition(filter(f_rewrite_cisco_program) or
filter(f_rewrite_cisco_program_2) or filter(f_rewrite_cisco_program_3)
or filter(f_rewrite_cisco_program_4)));
};
rewrite r_snare {
subst("MSWinEventLog.+(Security|Application|System).+", "$1",
value("PROGRAM") flags(global)); };
# URL encode any literal pipes so the tab rewriter doesn't introduce
ambiguity
rewrite r_from_pipes { subst('\|', "%7C", value("MESSAGE") flags(global)
condition(program("bro_*" type(glob)))); };
#rewrite r_from_pipes { subst('\|', "%7C", value("MESSAGE")
flags(global)); };
rewrite r_pipes { subst("\t", "|", value("MESSAGE") flags(global)); };
rewrite r_host { set("$SOURCEIP", value("HOST")); };
rewrite r_extracted_host { set("$pdb_extracted_sourceip", value("HOST")
condition("$pdb_extracted_sourceip" != "")); };
rewrite r_extracted_timestamp {
set("$S_UNIXTIME", value("SOURCE"));
set("$pdb_extracted_timestamp", value("SOURCE")
condition("$pdb_extracted_timestamp" != ""));
};
template t_db_parsed {
template("$R_UNIXTIME\t$HOST\t$PROGRAM\t${.classifier.class}\t$MSGONLY\t${i0}\t${i1}\t${i2}\t${i3}\t${i4}\t${i5}\t${s0}\t${s1}\t${s2}\t${s3}\t${s4}\t${s5}\n");
};
template t_db_parsed_import {
template("$MSGID\t$SOURCE\t$HOST\t$PROGRAM\t${.classifier.class}\t$MSGONLY\t${i0}\t${i1}\t${i2}\t${i3}\t${i4}\t${i5}\t${s0}\t${s1}\t${s2}\t${s3}\t${s4}\t${s5}\n");
};
template t_ops {
template("$S_UNIXTIME\t$HOST\t$PROGRAM\t$MSGID\t$MSGONLY\t${.SDATA.elsa@32473.line_number}\t${.SDATA.elsa@32473.pid}\t${.SDATA.elsa@32473.client}\t${.SDATA.elsa@32473.qid}\t\t\t${.SDATA.elsa@32473.priority}\t${.SDATA.elsa@32473.file}\t${.SDATA.elsa@32473.method}\t${.SDATA.elsa@32473.hostname}\t\t\n");
};
template t_unparsed { template("Unparsed host=$HOST program=$PROGRAM
msg=$MSGONLY\n"); };
destination d_elsa { program("perl /usr/local/elsa/node/elsa.pl -c
/etc/elsa_node.conf" template(t_db_parsed)); };
destination d_elsa_import { program("perl /usr/local/elsa/node/elsa.pl
-c /etc/elsa_node.conf -f __IMPORT__" template(t_db_parsed_import)); };
destination d_elsa_ops { program("perl /usr/local/elsa/node/elsa.pl -c
/etc/elsa_node.conf -f __OPS__" template(t_ops)); };
destination d_unclassified { unix-dgram("/data/elsa/tmp/ops"
flags(syslog-protocol) template(t_unparsed)); };
destination d_debug { unix-dgram("/data/elsa/tmp/debug"
template(t_db_parsed)); };
# Custom destination -Mike
destination d_hosts {
file("/data/logs/$HOST/$HOST.log");
};
include /etc/elsa_syslog-ng.conf;
#log {
# source(s_ops);
# destination(d_elsa_ops);
# flags(final);
#};
#log {
# source(s_network);
# source(s_realtime);
# rewrite(r_host);
# rewrite(r_cisco_program);
# rewrite(r_snare);
# rewrite(r_from_pipes);
# rewrite(r_pipes);
# parser(p_db);
# rewrite(r_extracted_host);
# ###FILTER_UNPARSED###log { filter(f_unclassified);
rewrite(r_unparsed); destination(d_unclassified); flags(final); };
# log { destination(d_elsa); };
# log { destination(d_debug); };
#};
#log {
# source(s_import);
# rewrite(r_cisco_program);
# rewrite(r_snare);
# rewrite(r_from_pipes);
# rewrite(r_pipes);
# parser(p_db);
# rewrite(r_extracted_host);
# rewrite(r_extracted_timestamp);
# destination(d_elsa_import);
#};
# cat /etc/elsa_syslog-ng.conf
##################################
#### Global Options #########
##################################
options {
check_hostname(yes);
create_dirs(yes);
dir_group(log_analyst);
dir_owner(root);
dir_perm(0550);
group(log_analyst);
owner(root);
perm(0640);
# Let's not be vulnerable to DNS attacks and general problems
dns_cache(no);
use_dns(no);
# Keep the original hostname (good for relay servers)
keep_hostname(yes);
# Keep original timestamp (not received time) - this could be forged, so
use caution during forensincs examinations
keep_timestamp(yes);
};
##################################
########### Sources #########
##################################
# Source for local system logs
source s_local {
# message generated by Syslog-NG
internal();
# standard Linux log source (this is the default place for the syslog()
# function to send logs to)
unix-stream("/dev/log");
# messages from the kernel
file("/proc/kmsg" program_override("kernel: "));
};
##################################
######### Destinations ##########
##################################
# This is the directory structure that will be automatically created as
logs come in
destination d_hosts {
file("/data/logs/$HOST/$HOST.log");
};