[syslog-ng] problems with multiline log messages on ubuntu18

[Jerry Scharf]

Hi,

We have been struggling with problems with multiple log messages being 
broken apart. Our apps can send random things to the logger including 
entire stack traces. We have a classifier that sends unknown lines to 
their own log file, so this makes a complete mess. This was with the 
distro apt package.

So I went to building 3.25.1 by hand. If I build it with nothing on the 
configure line, it processes the multiline log messages just fine but if 
I build it with the ubuntu18 debian rules, it breaks the same way. I 
narrowed it down to if --eneable-systemd is set it breaks.

Here are some relevant things.

DISTRIB_DESCRIPTION="Ubuntu 18.04.5 LTS"

config cmd

../configure --prefix=/usr/local --sysconfdir=/etc/syslog-ng 
--enable-dynamic-linking --enable-json --enable-manpages --enable-python 
--enable-java=no --enable-mongodb=no --enable-legacy-mongodb-options=no 
SOURCE_REVISION=3.25.1-ir1 --enable-systemd=no

attaching the conf and the file classifier (60...)

thanks,

jerry



-------------- next part --------------
@version: 3.25
@include "scl.conf"

options {
	chain_hostnames(off);
	time_reopen(10);
	time_reap(360);
	flush_lines(0);
	log_fifo_size(2048);
	owner("syslog");
	group("adm");
	perm(0640);
	create_dirs(yes);
	dir_perm(0755);
	use_dns(no);
	use_fqdn(no);
	stats_freq(0);
};

########################
# Sources
########################
source s_all {
	internal();
	unix-dgram("/run/systemd/journal/dev-log");
	unix-stream("/run/systemd/journal/dev-log");
	system();
	file("/proc/kmsg" program-override("kernel"));
};

###
# Note we are inverting the priorities here and making the Flashblade configurations
# under /etc/syslog-ng/conf.d have higher priority than the defaults.
###
@include "/etc/syslog-ng/conf.d/*.conf"


########################
# Millisecond date template for syslog
########################
template template_millisecond_format {
    template("${R_DATE} ${HOST} ${MSGHDR}${MSG}\n");
    template_escape(no);
};

########################
# Destinations
########################
destination d_auth { file("/var/log/auth.log" owner("syslog") group("adm") perm(0640)); };
destination d_syslog { file("/var/log/syslog" owner("syslog") group("adm") perm(0640) frac_digits(3) template(template_millisecond_format)); };
destination d_cron { file("/var/log/cron.log"); };
destination d_daemon { file("/var/log/daemon.log"); };
destination d_kern { file("/var/log/kern.log" owner("syslog") group("adm") perm(0640)); };
destination d_lpr { file("/var/log/lpr.log"); };
destination d_mail { file("/var/log/mail.log" owner("syslog") group("adm") perm(0640)); };
destination d_mail_err { file("/var/log/mail.err" owner("syslog") group("adm") perm(0640)); };
destination d_user { file("/var/log/user.log"); };
destination d_uucp { file("/var/log/uucp.log"); };
destination d_news_crit { file("/var/log/news/news.crit" owner("syslog") group("adm") perm(0640)); };
destination d_news_err { file("/var/log/news/news.err" owner("syslog") group("adm") perm(0640)); };
destination d_news_notice { file("/var/log/news/news.notice" owner("syslog") group("adm") perm(0640)); };
destination d_xconsole { pipe("/dev/xconsole"); };
destination d_snmpd { file("/var/log/purity/snmpd.log" owner("syslog") group("adm") perm(0640)); };

########################
# Filters
########################
filter f_auth { facility(auth, authpriv); };
filter f_syslog { not facility(auth, authpriv, local6) and not program(ccm); };
filter f_kern { facility(kern); };
filter f_mail { facility(mail); };
filter f_news { facility(news); };
filter f_notice { level(notice .. emerg); };
filter f_err { level(err .. emerg); };
filter f_crit { level(crit .. emerg); };
filter f_xconsole { facility(daemon, mail) or level(warning .. debug) or (facility(news) and level(err)); };
filter f_snmpd { facility(local0) and program("snmpd"); };

########################
# Log paths
########################
log { source(s_all); filter(f_auth); destination(d_auth); };
log { source(s_all); filter(f_syslog); destination(d_syslog); };
log { source(s_all); filter(f_kern); destination(d_kern); };
log { source(s_all); filter(f_mail); destination(d_mail); };
log { source(s_all); filter(f_mail); filter(f_err); destination(d_mail_err); };
log { source(s_all); filter(f_news); filter(f_crit); destination(d_news_crit); };
log { source(s_all); filter(f_news); filter(f_err); destination(d_news_err); };
log { source(s_all); filter(f_news); filter(f_notice); destination(d_news_notice); };
log { source(s_all); filter(f_xconsole); destination(d_xconsole); };
log { source(s_all); filter(f_snmpd); destination(d_snmpd); };

-------------- next part --------------
#
# Iridium Log Redirection Rules
#

define(`su_rules',dnl $1=SU_NUM $2=SU_IP
`destination d_Su$1_Main, { file("/ssd/rsyslog/su$1.log" template(t_IridiumFileFormatDiag)); };
destination d_Su$1_System, {
    file("/ssd/rsyslog/su$1_system.log" template(t_IridiumFileFormatExt) frac-digits(6));
};
destination d_Su$1_Separate, {
    file("/ssd/rsyslog/su$1_${PROGRAM}.log" template(t_IridiumFileFormatExt) frac-digits(6));
};'

# Write messages from storage unit $1 Iridium applications to su$1 log.
filter f_fromhost_ip_matches_Su$1 { netmask($2/32); };
filter f_Su$1_MainProgs { program('^epcq') or
                      program('^su_main') or
                      program('^su_reset') or
                      program('^upgrade') };

filter f_Su$1_SystemProgs { program('^asyncio') or
                      program('^cron') or
                      program('^CRON') or
                      program('^failsafe') or
                      program('^kernel') or
                      program('^login') or
                      program('^ntpdate') or
                      program('^rsyslogd') or
                      program('^sshd') or
                      program('^su$') };

filter f_Su$1_OtherProgs {
    not filter(f_Su$1_SystemProgs);
    not filter(f_Su$1_MainProgs);
};

log {
    source(s_all); filter(f_fromhost_ip_matches_Su$1);
    log { filter(f_Su$1_MainProgs); destination(d_Su$1_Main); };
    log { filter(f_Su$1_SystemProgs); destination(d_Su$1_System); };
    log { filter(f_Su$1_OtherProgs); destination(d_Su$1_Separate); };
    flags(final);
};
)dnl

su_rules(0,172.16.0.3)
su_rules(1,172.16.0.4)
su_rules(2,172.16.0.5)
su_rules(3,172.16.0.6)
su_rules(4,172.16.0.7)
su_rules(5,172.16.0.8)
su_rules(6,172.16.0.9)
su_rules(7,172.16.0.10)dnl

################
# Destinations
################
destination d_HostChassis { file("/ssd/rsyslog/chassis.log" template(t_IridiumFileFormatHost) frac-digits(6)); };
destination d_HostChassisExt {
    file("/ssd/rsyslog/chassis.log" template(t_IridiumFileFormatExt) frac-digits(6) persist-name("chassis_ext")); };
destination d_HostPlatform { file("/ssd/rsyslog/platform.log" template(t_IridiumFileFormatHost) frac-digits(6)); };
destination d_HostPlatformStructured {
    file("/ssd/rsyslog/platform-structured.log", template(t_IridiumFileFormatStructured) frac-digits(6) persist-name("structured_ext")); };
destination d_HostCongo { file("/ssd/rsyslog/congo.log" template(t_IridiumFileFormatHost) frac-digits(6)); };
destination d_HostNfs { file("/ssd/rsyslog/nfs.log" template(t_IridiumFileFormatDiag)); };
destination d_HostMiddleware { file("/ssd/rsyslog/middleware.log" template(t_IridiumFileFormatHost) frac-digits(6)); };
destination d_HostSmb { file("/ssd/rsyslog/smb.log" template(t_IridiumFileFormatHost) frac-digits(6)); };
destination d_HostSystem { file("/ssd/rsyslog/system.log" template(t_IridiumFileFormatHost) frac-digits(6) persist-name("fb_system_log")); };
destination d_HostSeparate { file("/ssd/rsyslog/${PROGRAM}.log" template(t_IridiumFileFormatHost) frac-digits(6)); };
destination d_HostDirectoryService { file("/ssd/rsyslog/directory_service.log" template(t_IridiumFileFormatHost) frac-digits(6)); };
destination d_HostDHCP { file("/ssd/rsyslog/dhcp.log" template(t_IridiumFileFormatHost) frac-digits(6)); };
destination d_HostHttp { file("/ssd/rsyslog/http.log" template(t_IridiumFileFormatHost) frac-digits(6)); };
destination d_HourlyDiagnostics { file("/ssd/rsyslog/hourlydiagnostics.log" template(t_IridiumFileFormatNoHost) frac-digits(6)); };
destination d_FrequentDiagnostics { file("/ssd/rsyslog/frequentdiagnostics.log" template(t_IridiumFileFormatNoHost) frac-digits(6)); };
destination d_Dotserver { file("/ssd/rsyslog/dotserver.log" template(t_IridiumFileFormatHost) frac-digits(6)); };

# Write all chassis-tagged messages from external hosts (not localhost or SU) to
# chassis log. These would normally consist of chassis-tagged messages from
# external Iridium tools.
filter f_not_from_127_0_0_1 { not netmask('127.0.0.1/32'); };
log {
    source(s_all);
    filter(f_not_from_127_0_0_1);
    destination(d_HostChassisExt);
    flags(final);
};

# Write chassis-tagged messages originating from local admin tools to chassis log.
filter f_IridiumChassisApp {
    program("^chassis$") or
    program("^python3.4");  # admin tools use python3.4. TODO: don't use python3.4 and fix this
};
log {
    source(s_all);
    filter(f_IridiumChassisApp);
    destination(d_HostChassis);
    flags(final);
};

# Write platform structured logs to platform-structured log.
filter f_IridiumPlatformStructuredApp { program('^platform_structured'); };
log {
    source(s_all);
    filter(f_IridiumPlatformStructuredApp);
    destination(d_HostPlatformStructured);
    flags(final);
};

# Write messages from local Iridium platform applications to platform log.
filter f_IridiumPlatformApps {
    program("^aiohttp") or
    program("^alertd") or
    program("^su_boot") or
    program("^irinit") or
    program("^inuk") or
    program("^ldap_proxy") or  # the ldap proxy is a subsection of inuk run on the blades underneath nfs
    program("^netconf") or
    program("^jsonrpcserver") or
    program("^pure") or
    program("^etcd") or
    program("^hald") or
    program("^phonehome") or
    program("^sh.command") or  #  we need to ban python sh
    program("^switch") or
    program("^purity.alert") or
    program("^fastrak-logger");
};
log {
    source(s_all);
    filter(f_IridiumPlatformApps);
    destination(d_HostPlatform);
    flags(final);
};

# Write messages from local Iridium congo applications to congo log.
filter f_IridiumCongoApp {
    program("^congo_blade") or
    program("^congo_fiom");
};
log {
    source(s_all);
    filter(f_IridiumCongoApp);
    destination(d_HostCongo);
    flags(final);
};

# Write messages from local Iridium NFS applications to nfs log.
filter f_IridiumNFSApps {
    program("^nfs") or
    program("^null_mount") or
    program("^format_sled_ir") or
    program("^epcq") or
    program("^load_aes_key") or
    program("^load_iridium_key");
};
log {
    source(s_all);
    filter(f_IridiumNFSApps);
    destination(d_HostNfs);
    flags(final);
};

# Write messages from local Iridium SMB applications to smb log.
filter f_IridiumSMBApps {
    program("^smbd") or
    program("^nmdb") or
    program("^winbindd") or
    program("^net") or
    program("^ctdbd");
};
log {
    source(s_all);
    filter(f_IridiumSMBApps);
    destination(d_HostSmb);
    flags(final);
};

# Write messages from local public rest applications to middleware log.
filter f_IridiumMiddlewareApps {
    program("^public-rest") or
    program("^middleware") or
    program("^purity.audit");
};
log {
    source(s_all);
    filter(f_IridiumMiddlewareApps);
    destination(d_HostMiddleware);
    flags(final);
};

# Write messages related to dhcp (including zero touch provisioning) to dhcp log.
filter f_IridiumDHCPApps {
    program("^6tunnel") or
    program("^dhclient") or
    program("^dhcpdump");
};
log {
    source(s_all);
    filter(f_IridiumDHCPApps);
    destination(d_HostDHCP);
    flags(final);
};

# Write messages from local HTTP software stack to http log.
filter f_IridiumHTTPApps {
    program("^http");
};
log {
    source(s_all);
    filter(f_IridiumHTTPApps);
    destination(d_HostHttp);
    flags(final);
};

# Write messages from specific local Linux applications to system log.
filter f_LinuxApps {
    program('(^asyncio)|(^chage)|(^cron)|(^CRON)|(^dbus)|(^failsafe)') or
    program('(^groupadd)|(^groupdel)|(^groupmod)|(^history)|(^kdump)|(^kernel)') or
    program('(^lldp)|(^login)|(^mcelog)|(^named)|(^ntpd)|(^ntpd_intres)') or
    program('(^ntpdate)|(^passwd)|(^postfix)|(^postmulti)|(^rsyslog)|(^syslog-ng)' ) or
    program('(^smartd)|(^sqlalchemy)|(^sshd)|(^sudo)|(^systemd)|(^statd)') or
    program('(^supervisord)|(^upstart)|(^urllib)|(^useradd)|(^userdel)|(^usermod)') or
    program('^su$');
};
log {
    source(s_all);
    filter(f_LinuxApps);
    destination(d_HostSystem);
    flags(final);
};

#Write messages from ds_pam and auth_helper to directory_service log.
filter f_DirectoryServiceApps {
    program("^ds");
};
log {
    source(s_all);
    filter(f_DirectoryServiceApps);
    destination(d_HostDirectoryService);
    flags(final);
};

# Write messages to hourly diagnostics log
filter f_HourlyDiagnostics {
    message("^hourlydiagnostics");
};
log {
    source(s_all);
    filter(f_HourlyDiagnostics);
    destination(d_HourlyDiagnostics);
    flags(final);
};

# Write messages to frequent diagnostics log
filter f_FrequentDiagnostics {
    message("^frequentdiagnostics");
};
log {
    source(s_all);
    filter(f_FrequentDiagnostics);
    destination(d_FrequentDiagnostics);
    flags(final);
};

# Write messages to dotserver log.  These messages are coming in with malformed messages where
# program name is parsed as INF, ERR or DBG.
filter f_Dotserver {
    message("^dotserver");
};
log {
    source(s_all);
    filter(f_Dotserver);
    destination(d_Dotserver);
    flags(final);
};

# Write messages from all other sources to their own log files. If additional
# log files appear in the future, we will notice them, and if necessary divert
# them to one of the other files.
log {
    source(s_all);
    destination(d_HostSeparate);
    flags(final);
};