New subject: syslog-ng 2.0rc1 memory usage

21 Jul 2006

      Configs included..
Similar behavior noticed on my RHEL4 instance.
Both are running 2.0rc1

[Solaris 9 Host]

   PID USERNAME THR PRI NICE  SIZE   RES STATE    TIME    CPU COMMAND
 29935 root       1  59    0  221M  220M sleep   16:49  2.38% syslog-ng.2.0rc

# cat syslog-ng.conf
options {
  sync(0);
  time_reopen(10);
  log_fifo_size(8192);
  flush_lines(4096);
  long_hostnames(off);
  use_dns(yes);
  use_fqdn(no);
  create_dirs(no);
  keep_hostname(yes);
  owner(syslogng);
  group(syslogng);
};

source local {
  sun-streams("/dev/log" door("/etc/.syslog_door"));
  internal();
};

source network {
  udp();
  tcp();
};

#
# standard destinations for local standard system messages
#
destination authlog {
  file("/var/log/auth.log");
};
destination syslog {
  file("/var/log/syslog");
};
destination kern {
  file("/var/log/kern.log");
};
destination maillog {
  file("/var/log/maillog");
};

destination messages {
  file("/var/log/messages" perm(0644));
};

#
# console destination
#
destination console {
  file("/dev/sysmsg");
};

#
# filters for standard local system messages which come
# in on non-local facilities
#
filter f_authpriv {
  facility(auth) ;
};
filter f_syslog {
  not facility(auth) and not facility(mail);
};
filter f_kern {
  facility(kern);
};
filter f_mail {
  facility(mail);
};

#
# catch the rest
#
filter f_messages {
  level(info .. warn) and not facility(auth);
};

#
# filters for various emergency level messages
#
filter f_emergency {
  level(emerg);
};

#
# log emergency level messages out to console
#
log {
  source(local);
  filter(f_emergency);
  destination(console);
};

#
# log messages from local machine
#
log {
  source(local);
  filter(f_authpriv);
  destination(authlog);
};
log {
  source(local);
  filter(f_syslog);
  destination(syslog);
};
log {
  source(local);
  filter(f_kern);
  destination(kern);
};
log {
  source(local);
  filter(f_mail);
  destination(maillog);
};

#
# catch the rest of the messages
#
log {
  source(local);
  source(network);
  filter(f_messages);
  destination(messages);
  flags(flow-control);
};

destination hosts {
  file("/var/log/messages" owner(syslogng) group(syslogng) perm(0644)
dir_perm(0700));
};
#
# logs all incoming messages from network source to the sorted
# destination
#
log {
  source(network);
  destination(hosts);
};

destination d_sec {
  program(
    "/opt/sec/2.3.3/sec.pl -input=\"-\" -conf=/opt/sec/etc/sec.conf"
    template("$MONTH$DAY $UNIXTIME $HOST $MSG\n")
  );
};
filter f_mysql {
  host("XXXXXXXX") or host("XXXXXXXX");
};
log {
  source(network);
  filter(f_mysql);
  destination(d_sec);
};

[RHEL 4 Update 3 Host]

  PID USER      PR  NI  VIRT  RES  SHR S %CPU %MEM    TIME+  COMMAND
16414 root      15   0 54064  50m 1068 S  1.3  1.2   3:51.60 syslog-ng

# cat syslog-ng.conf
###############################################################################
# options - global

options {
  log_fifo_size(8192);
  create_dirs(no); #default
  chain_hostnames(no); #default yes
  normalize_hostnames(yes); #default requires 1.9.9
  keep_hostname(yes); #default no
  owner(root);
  group(root);
  perm(0600);
  use_time_recvd(yes); #default no
};

###############################################################################
# sources

source abs_udp514 {
  udp(ip(0.0.0.0) port(514));
};

source abs_localhost {
  unix-stream("/dev/log");
  internal();
};

###############################################################################
# filters

filter abf_messages {
  level(debug)
  and not (facility(mail,auth,authpriv,cron,user));
};
filter abf_auth {
  facility(auth,authpriv);
};
filter abf_mail {
  facility(mail);
};
filter abf_cron {
  facility(cron);
};
filter abf_spooler {
  facility(uucp)
  or (facility(news) and level(crit));
};
filter abf_boot {
  facility(local7);
};
filter abf_daemon {
  facility(daemon);
};
filter abf_user {
  facility(user);
};
filter abf_ignore {
  not (host("2006"));
};
filter abf_postfix {
  program ("postfix/");
};

###############################################################################
# destinations

destination abd_messages {
  file(
    "/var/log/messages"
    group(bb)
    perm(0640)
  );
};
destination abd_auth {
  file("/var/log/auth.log");
};
destination abd_mail {
  file(
    "/var/log/mail.log"
    group(bb)
    perm(0640)
  );
};
destination abd_cron {
  file("/var/log/cron.log");
};
destination abd_spooler {
  file("/var/log/spooler.log");
};
destination abd_boot {
  file("/var/log/boot.log");
};
destination abd_daemon {
  file("/var/log/daemon.log");
};
destination abd_user {
  file("/var/log/user.log");
};
destination abd_sec {
  program(
    "/opt/sec/2.3.3/sec.pl -input=\"-\" -conf=/opt/sec/etc/sec.conf"
    template("$MONTH$DAY $UNIXTIME $HOST $MSG\n")
  );
};
destination abd_localhost {
  file("/var/log/localhost.log");
};

###############################################################################
# logs

log {
  source(abs_udp514);
  filter(abf_messages);
  destination(abd_messages);
};
log {
  source(abs_udp514);
  filter(abf_auth);
  destination(abd_auth);
};
log {
  source(abs_udp514);
  filter(abf_mail);
  destination(abd_mail);
};
log {
  source(abs_udp514);
  filter(abf_cron);
  destination(abd_cron);
};
log {
  source(abs_udp514);
  filter(abf_spooler);
  destination(abd_spooler);
};
log {
  source(abs_udp514);
  filter(abf_boot);
  destination(abd_boot);
};
log {
  source(abs_udp514);
  filter(abf_daemon);
  destination(abd_daemon);
};
log {
  source(abs_udp514);
  filter(abf_user);
  destination(abd_user);
};
log {
  source(abs_udp514);
  filter(abf_postfix);
  destination(abd_sec);
};
log {
  source(abs_localhost);
  destination(abd_localhost);
};

------------------------------

Message: 2
Date: Fri, 21 Jul 2006 11:16:27 +0200
From: Balazs Scheidler <bazsi@balabit.hu>
Subject: Re: [syslog-ng] syslog-ng 2.0rc1 memory usage
To: Syslog-ng users' and developers' mailing list
	<syslog-ng@lists.balabit.hu>
Message-ID: <1153473387.6682.6.camel@bzorp.balabit>
Content-Type: text/plain

On Thu, 2006-07-20 at 19:47 -0500, Martin, David M wrote:
...
Anyone running 2.0rc1 on Solaris 9 (sparc)? 
If so, what memory requirements are you seeing for the syslog-ng 
process?
My install handles ~ 700 messages/sec (all UDP) and over the course of 
a day syslog-ng will gradually consume nearly all available free 
memory (I've seen it up to 1GB).
(syslog-ng is bounced by logrotate daily, so the process repeats
itself)
I was able to substantially decrease the rate of consumption by adding 
the 'flush_lines" global (syslog-ng process now at 91MB and growing 
... slowly).
This definitely seems to be some kind of memory leak. Can you post a
configuration file that reproduces the problem?

Hopefully it is a leak in syslog-ng itself and not in one of the system
libraries.

--
Bazsi

------------------------------

_______________________________________________
syslog-ng maillist  -  syslog-ng@lists.balabit.hu
https://lists.balabit.hu/mailman/listinfo/syslog-ng

End of syslog-ng Digest, Vol 15, Issue 15
*****************************************

The information transmitted (including attachments) is
covered by the Electronic Communications Privacy Act,
18 U.S.C. 2510-2521, is intended only for the person(s) or
entity/entities to which it is addressed and may contain
confidential and/or privileged material. Any review,
retransmission, dissemination or other use of, or taking
of any action in reliance upon, this information by persons
or entities other than the intended recipient(s) is prohibited.
If you received this in error, please contact the sender and
delete the material from any computer.

Re: Re: [syslog-ng] syslog-ng 2.0rc1 memory usage

Martin, David M

Balazs Scheidler

tags

participants (2)