[syslog-ng]replacing part of prog name with hostname

Balazs Scheidler bazsi@balabit.hu
Mon, 6 Jan 2003 13:48:14 +0100


On Mon, Jan 06, 2003 at 10:24:27AM +0100, Balazs Scheidler wrote:
> On Sat, Jan 04, 2003 at 12:50:30PM -0800, Nate Campi wrote:
> > On Sat, Jan 04, 2003 at 02:55:51PM +0300, Borzenkov Andrey wrote:
> > > > It knows to actually shift the message over one place to the right and
> > > > stick the value of the $FULLHOST_FROM macro in there. Even if I tried
> > > > templating out the message on my own syslog-ng will still think that
> > > > "ctlds" or "last" isn't part of the message and it'll get lost.
> > > 
> > > Better is to implement source templates. This way you can precisely describe
> > > input line, so if you know your source never appends host name, you just
> > > omit this from template. Something like
> > > 
> > > source s_stream { unix-stream("/dev/log" max-connections(10)); template(DATE
> > > PROG[PID]:... );}; 
> > 
> > You missed the fact that before you ever get around to templating, part
> > of the program name is *already* lost. It's too late for that.
> 
> he meant 'source templates' to specify how to parse messages. while that
> would be interesting it is less than trivial.
> 
> I'm trying to hack a bad_hostname() feature right now.

I'm finished, Nate can you test if this patch is good for your problem?

Usage: new global option named bad_hostname(), expects a regular expression
which should match all bad hostnames:

options { bad_hostname("^ctld$"); };

It is currently a global option, and I don't think it will become a
per-source option in 1.5.x.

I only tested it on Linux.

Index: ChangeLog
===================================================================
RCS file: /var/cvs/syslog-ng/syslog-ng/ChangeLog,v
retrieving revision 1.67
diff -u -r1.67 ChangeLog
--- ChangeLog	5 Dec 2002 16:23:50 -0000	1.67
+++ ChangeLog	6 Jan 2003 12:42:54 -0000
@@ -1,3 +1,19 @@
+2003-01-06  Balazs Scheidler  <bazsi@balabit.balabit>
+
+	* af*.c: updated to call make_log_reader according to the latest
+	interface change
+
+	* src/log.c (make_log_info): expect a new argument (bad_hostname),
+	(parse_log_msg): check if the hostname matches bad_hostname, and
+	if it does do not interpret it as a hostname
+
+	* src/sources.c (make_log_reader): new argument, a regular
+	expression which matches bad hostnames
+
+2002-12-18  Balazs Scheidler  <bazsi@balabit.balabit>
+
+	* configure.in: bumped version number to 1.5.24
+
 2002-12-05  Balazs Scheidler  <bazsi@balabit.balabit>
 
 	* src/afinet.c (inet_address_setip): check addr if it is NULL
Index: src/affile.c
===================================================================
RCS file: /var/cvs/syslog-ng/syslog-ng/src/affile.c,v
retrieving revision 1.56
diff -u -r1.56 affile.c
--- src/affile.c	30 Oct 2002 19:28:11 -0000	1.56
+++ src/affile.c	6 Jan 2003 12:42:54 -0000
@@ -162,7 +162,7 @@
 	if (do_open_file(self->name, flags, -1, -1, -1, -1, -1, -1, 0, &fd)) {
 		lseek(fd, 0, SEEK_END);
 		self->src = io_read(make_io_fd(cfg->backend, fd, ol_string_use(self->name)), 
-			make_log_reader(0, self->prefix, cfg->log_msg_size, self->pad_size, cfg->check_hostname ? LF_CHECK_HOSTNAME : 0, c), 
+			make_log_reader(0, self->prefix, cfg->log_msg_size, self->pad_size, cfg->check_hostname ? LF_CHECK_HOSTNAME : 0, cfg->bad_hostname, c), 
 			NULL);
 		self->res = REMEMBER_RESOURCE(cfg->resources, &self->src->super.super);
 		return ST_OK | ST_GOON;
Index: src/afinet.c
===================================================================
RCS file: /var/cvs/syslog-ng/syslog-ng/src/afinet.c,v
retrieving revision 1.21
diff -u -r1.21 afinet.c
--- src/afinet.c	5 Dec 2002 16:23:50 -0000	1.21
+++ src/afinet.c	6 Jan 2003 12:42:54 -0000
@@ -89,13 +89,13 @@
 		notice("AF_INET client connected from %S, port %i\n", 
 		       inet->ip, inet->port);
 		io_read(self->client, 
-			make_log_reader(0, NULL, cfg->log_msg_size, 0, cfg->check_hostname ? LF_CHECK_HOSTNAME : 0, c), 
+			make_log_reader(0, NULL, cfg->log_msg_size, 0, cfg->check_hostname ? LF_CHECK_HOSTNAME : 0, cfg->bad_hostname, c), 
 			make_afsocket_source_close_callback(self));
 	}
 	else {
 		/* SOCK_DGRAM */
 		io_read(self->client, 
-			make_log_reader(1, NULL, cfg->log_msg_size, 0, cfg->check_hostname ? LF_CHECK_HOSTNAME : 0, c), 
+			make_log_reader(1, NULL, cfg->log_msg_size, 0, cfg->check_hostname ? LF_CHECK_HOSTNAME : 0, cfg->bad_hostname, c), 
 			make_afsocket_source_close_callback(self));
 	}
 	
Index: src/afstreams.c
===================================================================
RCS file: /var/cvs/syslog-ng/syslog-ng/src/afstreams.c,v
retrieving revision 1.14
diff -u -r1.14 afstreams.c
--- src/afstreams.c	21 Aug 2002 14:03:50 -0000	1.14
+++ src/afstreams.c	6 Jan 2003 12:42:54 -0000
@@ -77,6 +77,7 @@
      (name stream_fd)
      (super nonblocking_fd)
      (vars
+       (bad_hostname special-struct regex_t #f regfree)
        (pipe object log_handler)))
 */
 
@@ -145,7 +146,7 @@
 			
 			length = eol - bol;
 			if (length) {
-				li = make_log_info(length, bol, NULL, 0);
+				li = make_log_info(length, bol, NULL, 0, NULL);
 				li->pri = pri;
 				HANDLE_LOG(self->pipe, li);
 			}
@@ -160,6 +161,7 @@
 
 struct nonblocking_fd *io_stream_get(struct io_backend *backend,
 				     int fd, 
+				     UINT8 *hostname_re,
 				     struct log_handler *pipe)
 {
 	NEW(stream_fd, f);
@@ -169,6 +171,10 @@
 	f->super.read = stream_read_callback;
 	f->super.want_read = 1;
 	f->pipe = pipe;
+	if (hostname_re == NULL)
+		regcomp(&self->bad_hostname, "^$", REG_NOSUB | REG_EXTENDED);
+	else
+		regcomp(&self->bad_hostname, hostname_re, REG_NOSUB | REG_EXTENDED);
 
 	return &f->super;
 }
@@ -220,7 +226,7 @@
 			close(fd);
 			return ST_FAIL | ST_QUIT;
 		}
-		self->stream_fd = io_stream_get(cfg->backend, fd, c);
+		self->stream_fd = io_stream_get(cfg->backend, fd, cfg->bad_hostname, c);
 
 		REMEMBER_RESOURCE(cfg->resources, &self->stream_fd->super);
 
Index: src/afunix.c
===================================================================
RCS file: /var/cvs/syslog-ng/syslog-ng/src/afunix.c,v
retrieving revision 1.24
diff -u -r1.24 afunix.c
--- src/afunix.c	4 Sep 2002 14:52:25 -0000	1.24
+++ src/afunix.c	6 Jan 2003 12:42:54 -0000
@@ -51,7 +51,7 @@
 	CAST(afsocket_source_connection, self, c);
 
 	io_read(self->client, 
-		make_log_reader(!!(self->owner->flags & AFSOCKET_DGRAM), NULL, cfg->log_msg_size, 0, cfg->check_hostname ? LF_CHECK_HOSTNAME : 0, c), 
+		make_log_reader(!!(self->owner->flags & AFSOCKET_DGRAM), NULL, cfg->log_msg_size, 0, cfg->check_hostname ? LF_CHECK_HOSTNAME : 0, cfg->bad_hostname, c), 
 		make_afsocket_source_close_callback(self));
 	
 	return ST_OK | ST_GOON;
Index: src/center.c
===================================================================
RCS file: /var/cvs/syslog-ng/syslog-ng/src/center.c,v
retrieving revision 1.19
diff -u -r1.19 center.c
--- src/center.c	25 Aug 2001 13:11:48 -0000	1.19
+++ src/center.c	6 Jan 2003 12:42:54 -0000
@@ -154,6 +154,7 @@
 			}
 			
 		next_connection:
+			;
 		}
 	}
 
Index: src/cfg-grammar.y
===================================================================
RCS file: /var/cvs/syslog-ng/syslog-ng/src/cfg-grammar.y,v
retrieving revision 1.56
diff -u -r1.56 cfg-grammar.y
--- src/cfg-grammar.y	21 Aug 2002 14:03:50 -0000	1.56
+++ src/cfg-grammar.y	6 Jan 2003 12:42:54 -0000
@@ -75,7 +75,7 @@
 
 /* option items */
 %token KW_FLAGS KW_CATCHALL KW_FALLBACK KW_FINAL
-%token KW_FSYNC KW_MARK_FREQ KW_SYNC_FREQ KW_STATS_FREQ KW_CHAIN_HOSTNAMES KW_KEEP_HOSTNAME KW_CHECK_HOSTNAME
+%token KW_FSYNC KW_MARK_FREQ KW_SYNC_FREQ KW_STATS_FREQ KW_CHAIN_HOSTNAMES KW_KEEP_HOSTNAME KW_CHECK_HOSTNAME KW_BAD_HOSTNAME
 %token KW_LOG_FIFO_SIZE KW_LOG_MSG_SIZE
 %token KW_TIME_REOPEN KW_TIME_REAP KW_USE_TIME_RECVD
 %token KW_USE_DNS KW_USE_FQDN KW_GC_BUSY_THRESHOLD 
@@ -575,6 +575,7 @@
 	| KW_CHAIN_HOSTNAMES '(' yesno ')'	{ configuration->chain_hostnames = $3; }
 	| KW_KEEP_HOSTNAME '(' yesno ')'	{ configuration->keep_hostname = $3; }
 	| KW_CHECK_HOSTNAME '(' yesno ')'	{ configuration->check_hostname = $3; }
+	| KW_BAD_HOSTNAME '(' STRING ')'	{ cfg_set_bad_hostname($3); }
 	| KW_USE_TIME_RECVD '(' yesno ')'	{ configuration->use_time_recvd = $3; }
 	| KW_USE_FQDN '(' yesno ')'		{ configuration->use_fqdn = $3; };
 	| KW_USE_DNS '(' yesno ')'		{ configuration->use_dns = $3; };
Index: src/cfg-lex.l
===================================================================
RCS file: /var/cvs/syslog-ng/syslog-ng/src/cfg-lex.l,v
retrieving revision 1.25
diff -u -r1.25 cfg-lex.l
--- src/cfg-lex.l	21 Aug 2002 14:03:50 -0000	1.25
+++ src/cfg-lex.l	6 Jan 2003 12:42:54 -0000
@@ -61,6 +61,7 @@
         { "use_fqdn",           KW_USE_FQDN },
 	{ "use_dns",		KW_USE_DNS },
 	{ "check_hostname",	KW_CHECK_HOSTNAME },
+	{ "bad_hostname",	KW_BAD_HOSTNAME },
   	{ "gc_threshold",	KW_GC_BUSY_THRESHOLD },
   	{ "gc_busy_threshold",	KW_GC_BUSY_THRESHOLD },
   	{ "gc_idle_threshold",	KW_GC_IDLE_THRESHOLD },
Index: src/cfgfile.c
===================================================================
RCS file: /var/cvs/syslog-ng/syslog-ng/src/cfgfile.c,v
retrieving revision 1.39
diff -u -r1.39 cfgfile.c
--- src/cfgfile.c	26 Apr 2002 09:43:54 -0000	1.39
+++ src/cfgfile.c	6 Jan 2003 12:42:54 -0000
@@ -125,6 +125,11 @@
 	configuration->dir_perm = perm;
 }
 
+void cfg_set_bad_hostname(char *bad_hostname)
+{
+	configuration->bad_hostname = bad_hostname;
+}
+
 struct persistent_info *
 make_persistent_info(struct ol_string *name, 
 		     struct ol_object *o, 
@@ -320,6 +325,7 @@
 	self->dns_cache_expire = 3600;
 	self->dns_cache_expire_failed = 60;
 	self->log_msg_size = 2048;
+	self->bad_hostname = NULL;
 	if ((cfg = fopen(name, "r")) != NULL) {
 		lex_init(cfg);
 		res = yyparse();
Index: src/cfgfile.h
===================================================================
RCS file: /var/cvs/syslog-ng/syslog-ng/src/cfgfile.h,v
retrieving revision 1.25
diff -u -r1.25 cfgfile.h
--- src/cfgfile.h	18 Oct 2002 12:31:08 -0000	1.25
+++ src/cfgfile.h	6 Jan 2003 12:42:54 -0000
@@ -57,6 +58,7 @@
 	(use_fqdn simple UINT32)
 	(use_dns simple UINT32)
 	(check_hostname simple UINT32)
+	(bad_hostname pointer UINT8)
 	(create_dirs simple UINT32)
 	(uid simple int)
 	(gid simple int)
@@ -129,10 +131,7 @@
 void cfg_set_dir_owner(char *uid);
 void cfg_set_dir_group(char *gid);
 void cfg_set_dir_perm(int perm);
-
-
-
-
+void cfg_set_bad_hostname(char *bad_hostname);
 
 
 struct syslog_config *make_syslog_config(const char *name, struct io_backend *backend);
Index: src/filters.c
===================================================================
RCS file: /var/cvs/syslog-ng/syslog-ng/src/filters.c,v
retrieving revision 1.16
diff -u -r1.16 filters.c
--- src/filters.c	4 Feb 2002 16:07:50 -0000	1.16
+++ src/filters.c	6 Jan 2003 12:42:54 -0000
@@ -34,6 +34,7 @@
 #include <sys/socket.h>
 #include <netinet/in.h>
 #include <arpa/inet.h>
+#include <string.h>
 
 #define CLASS_DEFINE
 #include "filters.h.x"
Index: src/log.c
===================================================================
RCS file: /var/cvs/syslog-ng/syslog-ng/src/log.c,v
retrieving revision 1.28
diff -u -r1.28 log.c
--- src/log.c	18 Oct 2002 12:31:08 -0000	1.28
+++ src/log.c	6 Jan 2003 12:42:54 -0000
@@ -41,7 +41,7 @@
 static char aix_fwd_string[] = "Message forwarded from ";
 static char repeat_msg_string[] = "last message repeated";
 
-static void parse_log_msg(struct log_info *lm, UINT32 length, UINT8 *data, UINT8 *prefix)
+static void parse_log_msg(struct log_info *lm, UINT32 length, UINT8 *data, UINT8 *prefix, regex_t *hostname_re)
 {
 	unsigned char *src;
 	unsigned int left, pri, oldleft;
@@ -153,12 +153,15 @@
 		else {
 			/* If we haven't already found the original hostname,
 			   look for it now. */
+			char hostname_buf[256];
+			int dst;
 
 		        oldsrc = src;	
 			oldleft = left;
 
+			dst = 0;
 			while (left && *src != ' ' && *src != ':' 
-			       && *src != '[') {
+			       && *src != '[' && dst < sizeof(hostname_buf) - 1) {
 				if (lm->flags & LF_CHECK_HOSTNAME &&
 				    !((*src >= 'A' && *src <= 'Z') ||
 				      (*src >= 'a' && *src <= 'z') ||
@@ -168,11 +171,13 @@
 				      *src == '@' || *src == '/')) {
 					break;
 				}
+				hostname_buf[dst++] = *src;
 		 		src++;
 				left--;
 			}
-
-			if (left && *src == ' ') {
+			hostname_buf[dst] = 0;
+			if (left && *src == ' ' &&
+			    (!hostname_re || regexec(hostname_re, hostname_buf, 0, NULL, 0))) {
 		  		/* This was a hostname. It came from a
 				   syslog-ng, since syslogd doesn't send
 				   hostnames. It's even better then the one
@@ -263,13 +268,13 @@
 	}
 }
 
-struct log_info *make_log_info(UINT32 length, UINT8 *msg, UINT8 *prefix, UINT32 flags)
+struct log_info *make_log_info(UINT32 length, UINT8 *msg, UINT8 *prefix, UINT32 flags, regex_t *hostname_re)
 {
 	struct log_info *self;
 
 	NEW_SPACE(self);
 	self->flags = flags & LF_USER_FLAGS;
-  	parse_log_msg(self, length, msg, prefix);
+  	parse_log_msg(self, length, msg, prefix, hostname_re);
 	self->use_cnt = 1;
   	self->recvd = time(NULL);
 	return self;
Index: src/log.h
===================================================================
RCS file: /var/cvs/syslog-ng/syslog-ng/src/log.h,v
retrieving revision 1.18
diff -u -r1.18 log.h
--- src/log.h	21 Aug 2002 14:03:50 -0000	1.18
+++ src/log.h	6 Jan 2003 12:42:54 -0000
@@ -30,6 +30,7 @@
 #include "io.h"
 
 #include <sys/time.h>
+#include <regex.h>
 
 struct syslog_config;
 struct persistent_config;
@@ -89,7 +90,7 @@
 struct log_info *log_info_use(struct log_info *msg);
 void log_info_free(struct log_info *msg);
 
-struct log_info *make_log_info(UINT32 length, UINT8 *data, UINT8 *prefix, UINT32 flags);
+struct log_info *make_log_info(UINT32 length, UINT8 *data, UINT8 *prefix, UINT32 flags, regex_t *badhostname_re);
 struct log_info *make_internal_message(UINT32 pri, UINT32 length, UINT8 *data);
 struct log_info *make_mark_message(void);
 
Index: src/sources.c
===================================================================
RCS file: /var/cvs/syslog-ng/syslog-ng/src/sources.c,v
retrieving revision 1.36
diff -u -r1.36 sources.c
--- src/sources.c	28 Oct 2002 08:33:30 -0000	1.36
+++ src/sources.c	6 Jan 2003 12:42:54 -0000
@@ -42,6 +42,7 @@
 
 #include "sources.c.x"
 #include "nscache.h"
+
 /* CLASS:
      (class
        (name log_reader)
@@ -54,6 +55,7 @@
 	 (max_log_line simple UINT32)
 	 (pad_size simple UINT32)
 	 (msg_flags simple UINT32)
+	 (bad_hostname special-struct regex_t #f regfree)
          (next object log_handler)))
 */
 
@@ -65,7 +67,7 @@
 {
 	struct log_info *logmsg;
 
-	logmsg = make_log_info(length, data, self->prefix, self->msg_flags);
+	logmsg = make_log_info(length, data, self->prefix, self->msg_flags, &self->bad_hostname);
 	if (addrlen) {
 		logmsg->saddr = sockaddr2address_info(addrlen, addr);
 	}
@@ -155,6 +157,7 @@
 		UINT32 max_log_line,
 		UINT32 pad_size,
 		UINT32 msg_flags,
+		UINT8 *hostname_re,
 		struct log_handler *next)
 {
 	NEW(log_reader, self);
@@ -167,7 +170,10 @@
 	self->pad_size = pad_size;
 	self->msg_flags = msg_flags;
 	self->buffer = ol_space_alloc(self->max_log_line);
-	
+	if (hostname_re == NULL)
+		regcomp(&self->bad_hostname, "^$", REG_NOSUB | REG_EXTENDED);
+	else
+		regcomp(&self->bad_hostname, hostname_re, REG_NOSUB | REG_EXTENDED);
 	return &self->super;
 }
 
Index: src/sources.h
===================================================================
RCS file: /var/cvs/syslog-ng/syslog-ng/src/sources.h,v
retrieving revision 1.16
diff -u -r1.16 sources.h
--- src/sources.h	21 Aug 2002 14:03:50 -0000	1.16
+++ src/sources.h	6 Jan 2003 12:42:54 -0000
@@ -66,6 +66,7 @@
 		UINT32 max_log_line,
 		UINT32 pad_size,
 		UINT32 msg_flags,
+		UINT8 *bad_hostname,
 		struct log_handler *next);
 
 struct log_source_group *make_source_group(const char *name, struct log_source_driver *drvs);

-- 
Bazsi
PGP info: KeyID 9AF8D0A9 Fingerprint CD27 CFB0 802C 0944 9CFD 804E C82C 8EB1