Re: [syslog-ng] [Bug 128] template() broken in latest 3.3 git

19 Jul 2011

      On 14 Jul 2011, bugzilla@bugzilla.balabit.com told this:
...
https://bugzilla.balabit.com/show_bug.cgi?id=128
Balazs Scheidler <bazsi@balabit.hu> changed:
What    |Removed                     |Added
----------------------------------------------------------------------------
                 CC|                            |bazsi@balabit.hu
         Resolution|                            |FIXED
             Status|ASSIGNED                    |RESOLVED
--- Comment #3 from Balazs Scheidler <bazsi@balabit.hu>  2011-07-14 12:29:48 ---
integrated, thanks.
This is seemingly still broken :( (and so are my fingers, sorry for the
last email with a subject line describing this and completely different
content).

My syslog-ng-3.3 config is derived directly from a syslog-ng-3.0 config
saving only an adjustment of tcp() and udp(). The config is attached.
syslog-ng-3.0 emits things looking like this:

Jul 18 14:52:17 fold info: p6IDqFJu004668: Milter add: header: X-DCC-URT-Metrics: spindle 1060; Body=many Fuz1=many Fuz2=many
Jul 18 14:52:17 fold info: p6IDqFJu004668: to=<nix@esperi.org.uk>, delay=00:00:01, xdelay=00:00:00, mailer=local, pri=144021, dsn=2.0.0, stat=Sent
Jul 18 14:52:38 fold notice: ruleset=check_relay, arg1=[41.201.240.140], arg2=41.201.240.140, relay=[41.201.240.140], reject=550 5.7.1 Fix reverse DNS for 41.201.240.140
Jul 18 14:52:56 fold info: p6IDqtS6004673: from=<linux-kernel-owner@vger.kernel.org>, size=2651, class=-60, nrcpts=1, msgid=<20110718135243.GA5349@suse.de>, bodytype=8BITMIME, proto=ESMTP, daemon=MTA, relay=vger.kernel.org [209.132.180.67]
Jul 18 14:52:56 fold info: p6IDqtS6004673: Milter add: header: X-DCC-URT-Metrics: spindle 1060; Body=1 Fuz1=1 Fuz2=1

i.e. ordinary-looking log messages, slightly adjusted.

syslog-ng-3.3 from git head as of yesterday (including the attempted
fix) emits this:

priority_templatepriority_templatepriority_templatepriority_template[...]

endlessly: one 'priority_template' per message, no linefeeds.

It looks like

template priority_template
 {
  template("$S_DATE $HOST $PRIORITY: $MESSAGE\n");
  template_escape(no);
 };

destination by-facility
 {
  file("/var/log/by-facility/$FACILITY" template(priority_template) suppress(30));
 };

is emitting the *words* 'priority_template' into the file.

Full syslog-ng.conf attached, sorry that it's so ugly and verbose, I
want a proper lisp-style macro system. :) or perhaps I can just generate
it from m4... probably there is a really neat way of doing what I'm
doing in here in about six lines, but I haven't revised it much since
the syslog-ng-2.x days and I couldn't find a neater approach then.

@version: 3.3
#
# syslog config for primary log host
#
# Nix  <nix@esperi.org.uk>
#

#
# Options
#

options
 {
  keep_hostname(no);
  create_dirs(1);
  perm(0644);
  dir_perm(0755);
  log_fifo_size(1000);
  use_dns(yes);
  dns_cache(yes);
  use_fqdn(no);
  flush_lines(100);
  flush_timeout(1000);
  stats_freq(3600);
  chain_hostnames(yes);
 };

#
# Sources
#

source local
 {
  unix-stream("/dev/log" flags(no-multi-line));
  internal();
  file("/proc/kmsg" program_override("kernel: ") flags(kernel no-multi-line));
 };

source remote
 {
  tcp(ip(0.0.0.0) port(601) max-connections(50) log_fetch_limit(100) log_iw_size(5192) flags(no-multi-line));
  udp(ip(0.0.0.0) port(514) log_fetch_limit(100) log_iw_size(1100) flags(no-multi-line));
 };

source netconsole
 {
  udp(ip(0.0.0.0) port(24183) log_fetch_limit(100) log_iw_size(1100) flags(no-parse));
 };

#
# Templates
#

# A template containing the priority, for by-facility log messages

template priority_template
 {
  template("$S_DATE $HOST $PRIORITY: $MESSAGE\n");
  template_escape(no);
 };

# A bare template, for netconsole messages.
template bare
 {
  template("$MSG\n");
  template_escape(no);
 };

# A template containing the facility and priority, for by-priority
# messages (which are logged to this-or-greater files, so many
# priorities are intermixed)

template facility_template
 {
  template("$S_DATE $HOST $FACILITY $PRIORITY: $MESSAGE\n");
  template_escape(no);
 };

# An `original syslogd' message, for programs to parse

template machine_readable_template
 {
  template("$S_DATE $HOST: $MESSAGE\n");
  template_escape(no);
 };

#
# Destinations
#

# We have the following divisions of logfiles:
# by-facility:
#  logs with priority in the template, above debugging priority [archived]
# by-facility/machine-readable:
#  as by-facility, but without the different template [not archived]
# by-facility/machine-readable/{blah}.debug:
#  as machine-readable, debug-priority alone
#  debug-priority stuff alone.
# by-priority:
#  logs with facility in the template, including higher priority levels,
#  except for debug, which logs debug priority alone [not archived]
# by-priority/machine-readable:
#  as by-priority, but without the different template [not archived]

destination by-facility
 {
  file("/var/log/by-facility/$FACILITY" template(priority_template) suppress(30));
 };

destination by-facility-spam
 {
  file("/var/log/by-facility/spam" template(priority_template) suppress(30));
 };

destination by-facility-machine-readable
 {
  file("/var/log/by-facility/machine-readable/$FACILITY"
       template(machine_readable_template)
       suppress(30));
 };

destination by-facility-machine-readable-spam
 {
  file("/var/log/by-facility/machine-readable/spam"
       template(machine_readable_template)
       suppress(30));
 };

destination by-facility-fallback
 {
  file("/var/log/by-facility/null" template(priority_template) suppress(30));
 };

destination by-facility-machine-readable-fallback
 {
  file("/var/log/by-facility/machine-readable/null"
       template(machine_readable_template)
       suppress(30));
 };

destination by-facility-machine-readable-debugging
 {
  file("/var/log/by-facility/machine-readable/$FACILITY.debug"
       template(machine_readable_template)
       suppress(30));
 };

destination by-priority-emerg
 {
  file("/var/log/by-priority/emerg" template(facility_template) suppress(30));
 };

destination by-priority-alert
 {
  file("/var/log/by-priority/alert" template(facility_template) suppress(30));
 };

destination by-priority-crit
 {
  file("/var/log/by-priority/crit" template(facility_template) suppress(30));
 };

destination by-priority-err
 {
  file("/var/log/by-priority/err" template(facility_template) suppress(30));
 };

destination by-priority-warning
 {
  file("/var/log/by-priority/warning" template(facility_template) suppress(30));
 };

destination by-priority-notice
 {
  file("/var/log/by-priority/notice" template(facility_template) suppress(30));
 };

destination by-priority-info
 {
  file("/var/log/by-priority/info" template(facility_template) suppress(30));
 };

destination by-priority-debug
 {
  file("/var/log/by-priority/debug" template(facility_template) suppress(30));
 };

destination by-priority-fallback
 {
  file("/var/log/by-priority/null" template(facility_template) suppress(30));
 };

destination by-priority-machine-readable-emerg
 {
  file("/var/log/by-priority/machine-readable/emerg"
       template(machine_readable_template)
       suppress(30));
 };

destination by-priority-machine-readable-alert
 {
  file("/var/log/by-priority/machine-readable/alert"
       template(machine_readable_template)
       suppress(30));
 };

destination by-priority-machine-readable-crit
 {
  file("/var/log/by-priority/machine-readable/crit"
       template(machine_readable_template)
       suppress(30));
 };

destination by-priority-machine-readable-err
 {
  file("/var/log/by-priority/machine-readable/err"
       template(machine_readable_template)
       suppress(30));
 };

destination by-priority-machine-readable-warning
 {
  file("/var/log/by-priority/machine-readable/warning"
       template(machine_readable_template)
       suppress(30));
 };

destination by-priority-machine-readable-notice
 {
  file("/var/log/by-priority/machine-readable/notice"
       template(machine_readable_template)
       suppress(30));
 };

destination by-priority-machine-readable-info
 {
  file("/var/log/by-priority/machine-readable/info"
       template(machine_readable_template)
       suppress(30));
 };

destination by-priority-machine-readable-debug
 {
  file("/var/log/by-priority/machine-readable/debug"
       template(machine_readable_template)
       suppress(30));
 };

destination by-priority-machine-readable-fallback
 {
  file("/var/log/by-priority/machine-readable/null"
       template(machine_readable_template)
       suppress(30));
 };

destination external-firewall-warnings
 {
  file("/var/log/firewall"
       template(machine_readable_template)
       suppress(30));
 };

destination root-and-nix
 {
  usertty(root);
  usertty(nix);
 };

destination netconsole
 {
  file("/var/log/$HOST-netconsole.log"
       template(bare));
 };

#
# Filters.
#
# Filter stuff equal to or greater than priority (foo).
#

filter emerg { priority(emerg); };
filter alert { priority(alert .. emerg); };
filter crit { priority(crit .. emerg); };
filter err { priority(err .. emerg); };
filter warning { priority(warning .. emerg); };
filter notice { priority(notice .. emerg); };
filter info { priority(info .. emerg); };

# Filter out non-debugging stuff and debugging-only stuff
filter not-debug { not(priority(debug)); };
filter debug-only { priority(debug); };

# Filter out stuff which is likely to be spam-notice-related
filter mail-notice-only { priority(notice) and facility(mail); };

# External-firewall-related stuff (for dshield reports)

filter external-firewall { facility(local1) or (facility(kern) and match(" packet denied IN=adsl " value("MESSAGE"))); };

#
# Loggers.
#

log
 {
  source(local);
  source(remote);
  filter(not-debug);
  destination(by-facility);
  destination(by-facility-machine-readable);
 };

log
 {
  source(local);
  source(remote);
  filter(debug-only);
  destination(by-facility-machine-readable-debugging);
 };

log
 {
  source(local);
  source(remote);
  filter(mail-notice-only);
  destination(by-facility-spam);
  destination(by-facility-machine-readable-spam);
 };

log
 {
  source(local);
  source(remote);
  filter(emerg);
  destination(by-priority-emerg);
  destination(by-priority-machine-readable-emerg);
  destination(root-and-nix);
 };

log
 {
  source(local);
  source(remote);
  filter(alert);
  destination(by-priority-alert);
  destination(by-priority-machine-readable-alert);
 };

log
 {
  source(local);
  source(remote);
  filter(crit);
  destination(by-priority-crit);
  destination(by-priority-machine-readable-crit);
 };

log
 {
  source(local);
  source(remote);
  filter(err);
  destination(by-priority-err);
  destination(by-priority-machine-readable-err);
 };

log
 {
  source(local);
  source(remote);
  filter(warning);
  destination(by-priority-warning);
  destination(by-priority-machine-readable-warning);
 };

log
 {
  source(local);
  source(remote);
  filter(notice);
  destination(by-priority-notice);
  destination(by-priority-machine-readable-notice);
 };

log
 {
  source(local);
  source(remote);
  filter(info);
  destination(by-priority-info);
  destination(by-priority-machine-readable-info);
 };

log
 {
  source(local);
  source(remote);
  filter(debug-only);
  destination(by-priority-debug);
  destination(by-priority-machine-readable-debug);
 };

log
 {
  source(local);
  source(remote);
  filter(external-firewall);
  destination(external-firewall-warnings);
 };

log
 {
  source(local);
  source(remote);
  destination(by-facility-fallback);
  destination(by-facility-machine-readable-fallback);
  destination(by-priority-fallback);
  destination(by-priority-machine-readable-fallback);
  flags(fallback);
 };

log
 {
  source(netconsole);
  destination(netconsole);
 };