[syslog-ng]DNS Caching
=?iso-8859-1?Q?Ga=EBl?= Roualland
gael.roualland@oleane.net
Tue, 20 Feb 2001 13:50:59 +0100
This is a multi-part message in MIME format.
--------------A972B474EDD98CFDF43AB03C
Content-Type: text/plain; charset=iso-8859-1
Content-Transfer-Encoding: 8bit
Hello,
We've been using syslog-ng here for a while as a remote loghost for
several high-volume servers and the loghost would stay at a high load
and often lose some log lines.
We have name resolution enabled and it turned out to be one of the big
performance hit. Using a local DNS cache together with nscd and name
service files tuning made things a lot better but syslog-ng was still
spending lots of time doing reverse name resolution.
So I tried to add a simple DNS caching code to syslog-ng itself, and it
really improved the overall performance : the syslog-ng process now uses
between 10 to 20% of cpu time, whereas it used to be around 100%. As
this is an issue that could affect many users, it might be a good idea
to have this feature in syslog-ng (maybe as an option ?).
The attached patch is my implementation of it. The algorithm used is
quite simple (flat fixed-size hash, with refresh and aging) and should
be sufficient regarding the fact that a loghost will probably always
lookup the same set of hosts.
I'd be glad to know what you think of it and if it can be intregated in
the main release (it might need some work to fit nicer than now).
Gaël.
--
Gaël Roualland -+- gael.roualland@oleane.net
--------------A972B474EDD98CFDF43AB03C
Content-Type: text/plain; charset=iso-8859-1;
name="syslog-ng-1.4.10.nscache.diff"
Content-Transfer-Encoding: 8bit
Content-Disposition: inline;
filename="syslog-ng-1.4.10.nscache.diff"
diff -ruN syslog-ng-1.4.10/src/Makefile.am syslog-ng-1.4.10.nscache/src/Makefile.am
--- syslog-ng-1.4.10/src/Makefile.am Wed Nov 29 12:04:29 2000
+++ syslog-ng-1.4.10.nscache/src/Makefile.am Mon Feb 19 19:11:59 2001
@@ -65,6 +65,7 @@
afuser.c \
afstreams.c \
afprogram.c \
+ nscache.c \
utils.c \
syslog-names.c
diff -ruN syslog-ng-1.4.10/src/main.c syslog-ng-1.4.10.nscache/src/main.c
--- syslog-ng-1.4.10/src/main.c Wed Nov 29 12:04:29 2000
+++ syslog-ng-1.4.10.nscache/src/main.c Mon Feb 19 19:17:56 2001
@@ -29,6 +29,7 @@
#include "format.h"
#include "werror.h"
#include "queue.h"
+#include "nscache.h"
#include <sys/types.h>
#include <sys/wait.h>
@@ -144,7 +145,8 @@
signal(SIGHUP, sig_hup);
signal(SIGTERM, sig_term);
signal(SIGCHLD, sig_child);
-
+ nscache_init();
+
while (io_iter(&backend->super)) {
if (exit_main_loop)
break;
@@ -156,6 +158,10 @@
}
notice("SIGHUP received, restarting syslog-ng\n");
+
+ nscache_flush();
+ nscache_init();
+
restarting = 1;
backend->newconfig = make_syslog_config(cfgfilename, &backend->super);
diff -ruN syslog-ng-1.4.10/src/nscache.c syslog-ng-1.4.10.nscache/src/nscache.c
--- syslog-ng-1.4.10/src/nscache.c Thu Jan 1 01:00:00 1970
+++ syslog-ng-1.4.10.nscache/src/nscache.c Mon Feb 19 19:12:09 2001
@@ -0,0 +1,134 @@
+#include <netdb.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+#include "nscache.h"
+
+/* Simple NS cache, using a flat hash table (NOT thread safe).
+ * This is designed to be used with a fixed set of ip adresses.
+ * Inspired from similar caches in ippl, iplog...
+ * Gaël Roualland <gael.roualland@iname.com>, 2001. */
+
+#define NSCACHE_SIZE 1000
+#define NSCACHE_EXPIRE 3600
+#undef NSCACHE_DEBUG
+
+struct nsentry {
+ struct in_addr addr;
+ char * name;
+ time_t used;
+ time_t created;
+};
+
+static struct nsentry * nscache = NULL;
+
+void nscache_init() {
+ int i;
+
+ if (nscache != NULL)
+ nscache_flush();
+
+ if ((nscache = malloc(NSCACHE_SIZE * sizeof(struct nsentry))) == NULL)
+ return;
+
+ for (i = 0; i < NSCACHE_SIZE; i++) {
+ nscache[i].addr.s_addr = INADDR_NONE;
+ nscache[i].name = NULL;
+ nscache[i].created = -1;
+ nscache[i].used = -1;
+ }
+}
+
+void nscache_flush() {
+ int i;
+
+ if (nscache == NULL)
+ return;
+
+ for (i = 0; i < NSCACHE_SIZE; i++) {
+ if (nscache[i].name != NULL)
+ free(nscache[i].name);
+ }
+
+ free(nscache);
+ nscache = NULL;
+}
+
+static char * nsresolve(struct in_addr addr) {
+ struct hostent *host;
+
+ if ((host = gethostbyaddr((char *) &addr,
+ sizeof(struct in_addr),
+ AF_INET)) != NULL &&
+ host->h_name != NULL)
+ return strdup(host->h_name);
+ return NULL;
+}
+
+char *nscache_lookup(struct in_addr in) {
+ int h, k, old, new;
+#ifdef NSCACHE_DEBUG
+ int iter = 0;
+#endif
+ time_t now = time(NULL);
+ time_t oldest = now;
+
+ k = h = ntohl(in.s_addr) % (NSCACHE_SIZE - 1) + 1;
+ old = new = -1;
+
+ do {
+ if (nscache[k].created > 0 && nscache[k].addr.s_addr == in.s_addr) { /* found it! */
+ if (now - nscache[k].created >= NSCACHE_EXPIRE) {
+ /* entry has expired, update it */
+#ifdef NSCACHE_DEBUG
+ fprintf(stderr, "updating entry for %s...\n", inet_ntoa(in));
+#endif
+ if (nscache[k].name != NULL)
+ free (nscache[k].name);
+ nscache[k].name = nsresolve(in);
+ nscache[k].created = now;
+ }
+ nscache[k].used = now;
+#ifdef NSCACHE_DEBUG
+ fprintf(stderr, "FOUND! %s <=> %s (%d iters)...\n", inet_ntoa(in),
+ nscache[k].name, iter);
+#endif
+ return nscache[k].name;
+ }
+ if (nscache[k].created < 0 && new < 0)
+ new = k; /* use this to save new entry if not found */
+ else {
+ if (nscache[k].used <= oldest) {
+ old = k;
+ oldest = nscache[k].used;
+ }
+ }
+ k = (h + k) % NSCACHE_SIZE;
+#ifdef NSCACHE_DEBUG
+ iter++;
+#endif
+ } while (k > 0);
+
+ /* not found, add to cache */
+ if (new < 0) {
+ /* no new entries ? use oldest. */
+ if (old < 0)
+ old = 0; /* shouldn't happen.. */
+#ifdef NSCACHE_DEBUG
+ fprintf(stderr, "NOT FOUND! Replacing %s..\n", inet_ntoa(nscache[old].addr));
+#endif
+ if (nscache[old].name != NULL)
+ free(nscache[old].name);
+ new = old;
+ }
+#ifdef NSCACHE_DEBUG
+ fprintf(stderr, "NOT FOUND! %s... (adding as %d)\n", inet_ntoa(in), new);
+#endif
+ nscache[new].name = nsresolve(in);
+ nscache[new].created = nscache[new].used = now;
+ nscache[new].addr.s_addr = in.s_addr;
+ return nscache[new].name;
+}
diff -ruN syslog-ng-1.4.10/src/nscache.h syslog-ng-1.4.10.nscache/src/nscache.h
--- syslog-ng-1.4.10/src/nscache.h Thu Jan 1 01:00:00 1970
+++ syslog-ng-1.4.10.nscache/src/nscache.h Mon Feb 19 19:12:11 2001
@@ -0,0 +1,8 @@
+#ifndef NSCACHE_H
+#define NSCACHE_H
+
+void nscache_init();
+void nscache_flush();
+char * nscache_lookup(struct in_addr in);
+
+#endif
diff -ruN syslog-ng-1.4.10/src/sources.c syslog-ng-1.4.10.nscache/src/sources.c
--- syslog-ng-1.4.10/src/sources.c Wed Nov 29 12:04:29 2000
+++ syslog-ng-1.4.10.nscache/src/sources.c Mon Feb 19 19:14:20 2001
@@ -40,7 +40,7 @@
#undef CLASS_DEFINE
#include "sources.c.x"
-
+#include "nscache.h"
/* CLASS:
(class
(name log_reader)
@@ -155,15 +155,13 @@
if (a && a->super.isa == &inet_address_info_class) {
CAST(inet_address_info, inet_addr, a);
- struct hostent *hp;
char *hname, *p;
- hp = usedns ? gethostbyaddr((char *) &(inet_addr->sa.sin_addr), sizeof(struct in_addr), AF_INET) : NULL;
- if (!hp) {
+ hname = usedns ? nscache_lookup(inet_addr->sa.sin_addr) : NULL;
+ if (!hname) {
return ol_string_use(inet_addr->ip);
}
else {
- hname = hp->h_name;
if (!usefqdn) {
p = strchr(hname, '.');
if (p) *p = 0;
--------------A972B474EDD98CFDF43AB03C--