Hello, We've been using syslog-ng here for a while as a remote loghost for several high-volume servers and the loghost would stay at a high load and often lose some log lines. We have name resolution enabled and it turned out to be one of the big performance hit. Using a local DNS cache together with nscd and name service files tuning made things a lot better but syslog-ng was still spending lots of time doing reverse name resolution. So I tried to add a simple DNS caching code to syslog-ng itself, and it really improved the overall performance : the syslog-ng process now uses between 10 to 20% of cpu time, whereas it used to be around 100%. As this is an issue that could affect many users, it might be a good idea to have this feature in syslog-ng (maybe as an option ?). The attached patch is my implementation of it. The algorithm used is quite simple (flat fixed-size hash, with refresh and aging) and should be sufficient regarding the fact that a loghost will probably always lookup the same set of hosts. I'd be glad to know what you think of it and if it can be intregated in the main release (it might need some work to fit nicer than now). Gaël. -- Gaël Roualland -+- gael.roualland@oleane.net diff -ruN syslog-ng-1.4.10/src/Makefile.am syslog-ng-1.4.10.nscache/src/Makefile.am --- syslog-ng-1.4.10/src/Makefile.am Wed Nov 29 12:04:29 2000 +++ syslog-ng-1.4.10.nscache/src/Makefile.am Mon Feb 19 19:11:59 2001 @@ -65,6 +65,7 @@ afuser.c \ afstreams.c \ afprogram.c \ + nscache.c \ utils.c \ syslog-names.c diff -ruN syslog-ng-1.4.10/src/main.c syslog-ng-1.4.10.nscache/src/main.c --- syslog-ng-1.4.10/src/main.c Wed Nov 29 12:04:29 2000 +++ syslog-ng-1.4.10.nscache/src/main.c Mon Feb 19 19:17:56 2001 @@ -29,6 +29,7 @@ #include "format.h" #include "werror.h" #include "queue.h" +#include "nscache.h" #include <sys/types.h> #include <sys/wait.h> @@ -144,7 +145,8 @@ signal(SIGHUP, sig_hup); signal(SIGTERM, sig_term); signal(SIGCHLD, sig_child); - + nscache_init(); + while (io_iter(&backend->super)) { if (exit_main_loop) break; @@ -156,6 +158,10 @@ } notice("SIGHUP received, restarting syslog-ng\n"); + + nscache_flush(); + nscache_init(); + restarting = 1; backend->newconfig = make_syslog_config(cfgfilename, &backend->super); diff -ruN syslog-ng-1.4.10/src/nscache.c syslog-ng-1.4.10.nscache/src/nscache.c --- syslog-ng-1.4.10/src/nscache.c Thu Jan 1 01:00:00 1970 +++ syslog-ng-1.4.10.nscache/src/nscache.c Mon Feb 19 19:12:09 2001 @@ -0,0 +1,134 @@ +#include <netdb.h> +#include <netinet/in.h> +#include <arpa/inet.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include "nscache.h" + +/* Simple NS cache, using a flat hash table (NOT thread safe). + * This is designed to be used with a fixed set of ip adresses. + * Inspired from similar caches in ippl, iplog... + * Gaël Roualland <gael.roualland@iname.com>, 2001. */ + +#define NSCACHE_SIZE 1000 +#define NSCACHE_EXPIRE 3600 +#undef NSCACHE_DEBUG + +struct nsentry { + struct in_addr addr; + char * name; + time_t used; + time_t created; +}; + +static struct nsentry * nscache = NULL; + +void nscache_init() { + int i; + + if (nscache != NULL) + nscache_flush(); + + if ((nscache = malloc(NSCACHE_SIZE * sizeof(struct nsentry))) == NULL) + return; + + for (i = 0; i < NSCACHE_SIZE; i++) { + nscache[i].addr.s_addr = INADDR_NONE; + nscache[i].name = NULL; + nscache[i].created = -1; + nscache[i].used = -1; + } +} + +void nscache_flush() { + int i; + + if (nscache == NULL) + return; + + for (i = 0; i < NSCACHE_SIZE; i++) { + if (nscache[i].name != NULL) + free(nscache[i].name); + } + + free(nscache); + nscache = NULL; +} + +static char * nsresolve(struct in_addr addr) { + struct hostent *host; + + if ((host = gethostbyaddr((char *) &addr, + sizeof(struct in_addr), + AF_INET)) != NULL && + host->h_name != NULL) + return strdup(host->h_name); + return NULL; +} + +char *nscache_lookup(struct in_addr in) { + int h, k, old, new; +#ifdef NSCACHE_DEBUG + int iter = 0; +#endif + time_t now = time(NULL); + time_t oldest = now; + + k = h = ntohl(in.s_addr) % (NSCACHE_SIZE - 1) + 1; + old = new = -1; + + do { + if (nscache[k].created > 0 && nscache[k].addr.s_addr == in.s_addr) { /* found it! */ + if (now - nscache[k].created >= NSCACHE_EXPIRE) { + /* entry has expired, update it */ +#ifdef NSCACHE_DEBUG + fprintf(stderr, "updating entry for %s...\n", inet_ntoa(in)); +#endif + if (nscache[k].name != NULL) + free (nscache[k].name); + nscache[k].name = nsresolve(in); + nscache[k].created = now; + } + nscache[k].used = now; +#ifdef NSCACHE_DEBUG + fprintf(stderr, "FOUND! %s <=> %s (%d iters)...\n", inet_ntoa(in), + nscache[k].name, iter); +#endif + return nscache[k].name; + } + if (nscache[k].created < 0 && new < 0) + new = k; /* use this to save new entry if not found */ + else { + if (nscache[k].used <= oldest) { + old = k; + oldest = nscache[k].used; + } + } + k = (h + k) % NSCACHE_SIZE; +#ifdef NSCACHE_DEBUG + iter++; +#endif + } while (k > 0); + + /* not found, add to cache */ + if (new < 0) { + /* no new entries ? use oldest. */ + if (old < 0) + old = 0; /* shouldn't happen.. */ +#ifdef NSCACHE_DEBUG + fprintf(stderr, "NOT FOUND! Replacing %s..\n", inet_ntoa(nscache[old].addr)); +#endif + if (nscache[old].name != NULL) + free(nscache[old].name); + new = old; + } +#ifdef NSCACHE_DEBUG + fprintf(stderr, "NOT FOUND! %s... (adding as %d)\n", inet_ntoa(in), new); +#endif + nscache[new].name = nsresolve(in); + nscache[new].created = nscache[new].used = now; + nscache[new].addr.s_addr = in.s_addr; + return nscache[new].name; +} diff -ruN syslog-ng-1.4.10/src/nscache.h syslog-ng-1.4.10.nscache/src/nscache.h --- syslog-ng-1.4.10/src/nscache.h Thu Jan 1 01:00:00 1970 +++ syslog-ng-1.4.10.nscache/src/nscache.h Mon Feb 19 19:12:11 2001 @@ -0,0 +1,8 @@ +#ifndef NSCACHE_H +#define NSCACHE_H + +void nscache_init(); +void nscache_flush(); +char * nscache_lookup(struct in_addr in); + +#endif diff -ruN syslog-ng-1.4.10/src/sources.c syslog-ng-1.4.10.nscache/src/sources.c --- syslog-ng-1.4.10/src/sources.c Wed Nov 29 12:04:29 2000 +++ syslog-ng-1.4.10.nscache/src/sources.c Mon Feb 19 19:14:20 2001 @@ -40,7 +40,7 @@ #undef CLASS_DEFINE #include "sources.c.x" - +#include "nscache.h" /* CLASS: (class (name log_reader) @@ -155,15 +155,13 @@ if (a && a->super.isa == &inet_address_info_class) { CAST(inet_address_info, inet_addr, a); - struct hostent *hp; char *hname, *p; - hp = usedns ? gethostbyaddr((char *) &(inet_addr->sa.sin_addr), sizeof(struct in_addr), AF_INET) : NULL; - if (!hp) { + hname = usedns ? nscache_lookup(inet_addr->sa.sin_addr) : NULL; + if (!hname) { return ol_string_use(inet_addr->ip); } else { - hname = hp->h_name; if (!usefqdn) { p = strchr(hname, '.'); if (p) *p = 0;