Re: [syslog-ng]replacing part of prog name with hostname

6 Jan 2003

On Mon, Jan 06, 2003 at 10:24:27AM +0100, Balazs Scheidler wrote:
...
On Sat, Jan 04, 2003 at 12:50:30PM -0800, Nate Campi wrote:
...
On Sat, Jan 04, 2003 at 02:55:51PM +0300, Borzenkov Andrey wrote:
...
...
It knows to actually shift the message over one place to the right and
stick the value of the $FULLHOST_FROM macro in there. Even if I tried
templating out the message on my own syslog-ng will still think that
"ctlds" or "last" isn't part of the message and it'll get lost.
Better is to implement source templates. This way you can precisely describe
input line, so if you know your source never appends host name, you just
omit this from template. Something like
source s_stream { unix-stream("/dev/log" max-connections(10)); template(DATE
PROG[PID]:... );};
You missed the fact that before you ever get around to templating, part
of the program name is *already* lost. It's too late for that.
he meant 'source templates' to specify how to parse messages. while that
would be interesting it is less than trivial.
I'm trying to hack a bad_hostname() feature right now.
I'm finished, Nate can you test if this patch is good for your problem?

Usage: new global option named bad_hostname(), expects a regular expression
which should match all bad hostnames:

options { bad_hostname("^ctld$"); };

It is currently a global option, and I don't think it will become a
per-source option in 1.5.x.

I only tested it on Linux.

Index: ChangeLog
===================================================================
RCS file: /var/cvs/syslog-ng/syslog-ng/ChangeLog,v
retrieving revision 1.67
diff -u -r1.67 ChangeLog

--- ChangeLog	5 Dec 2002 16:23:50 -0000	1.67
+++ ChangeLog	6 Jan 2003 12:42:54 -0000
@@ -1,3 +1,19 @@
+2003-01-06  Balazs Scheidler  <bazsi@balabit.balabit>
+
+	* af*.c: updated to call make_log_reader according to the latest
+	interface change
+
+	* src/log.c (make_log_info): expect a new argument (bad_hostname),
+	(parse_log_msg): check if the hostname matches bad_hostname, and
+	if it does do not interpret it as a hostname
+
+	* src/sources.c (make_log_reader): new argument, a regular
+	expression which matches bad hostnames
+
+2002-12-18  Balazs Scheidler  <bazsi@balabit.balabit>
+
+	* configure.in: bumped version number to 1.5.24
+
 2002-12-05  Balazs Scheidler  <bazsi@balabit.balabit>
 
 	* src/afinet.c (inet_address_setip): check addr if it is NULL
Index: src/affile.c
===================================================================
RCS file: /var/cvs/syslog-ng/syslog-ng/src/affile.c,v
retrieving revision 1.56
diff -u -r1.56 affile.c
--- src/affile.c	30 Oct 2002 19:28:11 -0000	1.56
+++ src/affile.c	6 Jan 2003 12:42:54 -0000
@@ -162,7 +162,7 @@
 	if (do_open_file(self->name, flags, -1, -1, -1, -1, -1, -1, 0, &fd)) {
 		lseek(fd, 0, SEEK_END);
 		self->src = io_read(make_io_fd(cfg->backend, fd, ol_string_use(self->name)), 
-			make_log_reader(0, self->prefix, cfg->log_msg_size, self->pad_size, cfg->check_hostname ? LF_CHECK_HOSTNAME : 0, c), 
+			make_log_reader(0, self->prefix, cfg->log_msg_size, self->pad_size, cfg->check_hostname ? LF_CHECK_HOSTNAME : 0, cfg->bad_hostname, c), 
 			NULL);
 		self->res = REMEMBER_RESOURCE(cfg->resources, &self->src->super.super);
 		return ST_OK | ST_GOON;
Index: src/afinet.c
===================================================================
RCS file: /var/cvs/syslog-ng/syslog-ng/src/afinet.c,v
retrieving revision 1.21
diff -u -r1.21 afinet.c
--- src/afinet.c	5 Dec 2002 16:23:50 -0000	1.21
+++ src/afinet.c	6 Jan 2003 12:42:54 -0000
@@ -89,13 +89,13 @@
 		notice("AF_INET client connected from %S, port %i\n", 
 		       inet->ip, inet->port);
 		io_read(self->client, 
-			make_log_reader(0, NULL, cfg->log_msg_size, 0, cfg->check_hostname ? LF_CHECK_HOSTNAME : 0, c), 
+			make_log_reader(0, NULL, cfg->log_msg_size, 0, cfg->check_hostname ? LF_CHECK_HOSTNAME : 0, cfg->bad_hostname, c), 
 			make_afsocket_source_close_callback(self));
 	}
 	else {
 		/* SOCK_DGRAM */
 		io_read(self->client, 
-			make_log_reader(1, NULL, cfg->log_msg_size, 0, cfg->check_hostname ? LF_CHECK_HOSTNAME : 0, c), 
+			make_log_reader(1, NULL, cfg->log_msg_size, 0, cfg->check_hostname ? LF_CHECK_HOSTNAME : 0, cfg->bad_hostname, c), 
 			make_afsocket_source_close_callback(self));
 	}
 	
Index: src/afstreams.c
===================================================================
RCS file: /var/cvs/syslog-ng/syslog-ng/src/afstreams.c,v
retrieving revision 1.14
diff -u -r1.14 afstreams.c
--- src/afstreams.c	21 Aug 2002 14:03:50 -0000	1.14
+++ src/afstreams.c	6 Jan 2003 12:42:54 -0000
@@ -77,6 +77,7 @@
      (name stream_fd)
      (super nonblocking_fd)
      (vars
+       (bad_hostname special-struct regex_t #f regfree)
        (pipe object log_handler)))
 */
 
@@ -145,7 +146,7 @@
 			
 			length = eol - bol;
 			if (length) {
-				li = make_log_info(length, bol, NULL, 0);
+				li = make_log_info(length, bol, NULL, 0, NULL);
 				li->pri = pri;
 				HANDLE_LOG(self->pipe, li);
 			}
@@ -160,6 +161,7 @@
 
 struct nonblocking_fd *io_stream_get(struct io_backend *backend,
 				     int fd, 
+				     UINT8 *hostname_re,
 				     struct log_handler *pipe)
 {
 	NEW(stream_fd, f);
@@ -169,6 +171,10 @@
 	f->super.read = stream_read_callback;
 	f->super.want_read = 1;
 	f->pipe = pipe;
+	if (hostname_re == NULL)
+		regcomp(&self->bad_hostname, "^$", REG_NOSUB | REG_EXTENDED);
+	else
+		regcomp(&self->bad_hostname, hostname_re, REG_NOSUB | REG_EXTENDED);
 
 	return &f->super;
 }
@@ -220,7 +226,7 @@
 			close(fd);
 			return ST_FAIL | ST_QUIT;
 		}
-		self->stream_fd = io_stream_get(cfg->backend, fd, c);
+		self->stream_fd = io_stream_get(cfg->backend, fd, cfg->bad_hostname, c);
 
 		REMEMBER_RESOURCE(cfg->resources, &self->stream_fd->super);
 
Index: src/afunix.c
===================================================================
RCS file: /var/cvs/syslog-ng/syslog-ng/src/afunix.c,v
retrieving revision 1.24
diff -u -r1.24 afunix.c
--- src/afunix.c	4 Sep 2002 14:52:25 -0000	1.24
+++ src/afunix.c	6 Jan 2003 12:42:54 -0000
@@ -51,7 +51,7 @@
 	CAST(afsocket_source_connection, self, c);
 
 	io_read(self->client, 
-		make_log_reader(!!(self->owner->flags & AFSOCKET_DGRAM), NULL, cfg->log_msg_size, 0, cfg->check_hostname ? LF_CHECK_HOSTNAME : 0, c), 
+		make_log_reader(!!(self->owner->flags & AFSOCKET_DGRAM), NULL, cfg->log_msg_size, 0, cfg->check_hostname ? LF_CHECK_HOSTNAME : 0, cfg->bad_hostname, c), 
 		make_afsocket_source_close_callback(self));
 	
 	return ST_OK | ST_GOON;
Index: src/center.c
===================================================================
RCS file: /var/cvs/syslog-ng/syslog-ng/src/center.c,v
retrieving revision 1.19
diff -u -r1.19 center.c
--- src/center.c	25 Aug 2001 13:11:48 -0000	1.19
+++ src/center.c	6 Jan 2003 12:42:54 -0000
@@ -154,6 +154,7 @@
 			}
 			
 		next_connection:
+			;
 		}
 	}
 
Index: src/cfg-grammar.y
===================================================================
RCS file: /var/cvs/syslog-ng/syslog-ng/src/cfg-grammar.y,v
retrieving revision 1.56
diff -u -r1.56 cfg-grammar.y
--- src/cfg-grammar.y	21 Aug 2002 14:03:50 -0000	1.56
+++ src/cfg-grammar.y	6 Jan 2003 12:42:54 -0000
@@ -75,7 +75,7 @@
 
 /* option items */
 %token KW_FLAGS KW_CATCHALL KW_FALLBACK KW_FINAL
-%token KW_FSYNC KW_MARK_FREQ KW_SYNC_FREQ KW_STATS_FREQ KW_CHAIN_HOSTNAMES KW_KEEP_HOSTNAME KW_CHECK_HOSTNAME
+%token KW_FSYNC KW_MARK_FREQ KW_SYNC_FREQ KW_STATS_FREQ KW_CHAIN_HOSTNAMES KW_KEEP_HOSTNAME KW_CHECK_HOSTNAME KW_BAD_HOSTNAME
 %token KW_LOG_FIFO_SIZE KW_LOG_MSG_SIZE
 %token KW_TIME_REOPEN KW_TIME_REAP KW_USE_TIME_RECVD
 %token KW_USE_DNS KW_USE_FQDN KW_GC_BUSY_THRESHOLD 
@@ -575,6 +575,7 @@
 	| KW_CHAIN_HOSTNAMES '(' yesno ')'	{ configuration->chain_hostnames = $3; }
 	| KW_KEEP_HOSTNAME '(' yesno ')'	{ configuration->keep_hostname = $3; }
 	| KW_CHECK_HOSTNAME '(' yesno ')'	{ configuration->check_hostname = $3; }
+	| KW_BAD_HOSTNAME '(' STRING ')'	{ cfg_set_bad_hostname($3); }
 	| KW_USE_TIME_RECVD '(' yesno ')'	{ configuration->use_time_recvd = $3; }
 	| KW_USE_FQDN '(' yesno ')'		{ configuration->use_fqdn = $3; };
 	| KW_USE_DNS '(' yesno ')'		{ configuration->use_dns = $3; };
Index: src/cfg-lex.l
===================================================================
RCS file: /var/cvs/syslog-ng/syslog-ng/src/cfg-lex.l,v
retrieving revision 1.25
diff -u -r1.25 cfg-lex.l
--- src/cfg-lex.l	21 Aug 2002 14:03:50 -0000	1.25
+++ src/cfg-lex.l	6 Jan 2003 12:42:54 -0000
@@ -61,6 +61,7 @@
         { "use_fqdn",           KW_USE_FQDN },
 	{ "use_dns",		KW_USE_DNS },
 	{ "check_hostname",	KW_CHECK_HOSTNAME },
+	{ "bad_hostname",	KW_BAD_HOSTNAME },
   	{ "gc_threshold",	KW_GC_BUSY_THRESHOLD },
   	{ "gc_busy_threshold",	KW_GC_BUSY_THRESHOLD },
   	{ "gc_idle_threshold",	KW_GC_IDLE_THRESHOLD },
Index: src/cfgfile.c
===================================================================
RCS file: /var/cvs/syslog-ng/syslog-ng/src/cfgfile.c,v
retrieving revision 1.39
diff -u -r1.39 cfgfile.c
--- src/cfgfile.c	26 Apr 2002 09:43:54 -0000	1.39
+++ src/cfgfile.c	6 Jan 2003 12:42:54 -0000
@@ -125,6 +125,11 @@
 	configuration->dir_perm = perm;
 }
 
+void cfg_set_bad_hostname(char *bad_hostname)
+{
+	configuration->bad_hostname = bad_hostname;
+}
+
 struct persistent_info *
 make_persistent_info(struct ol_string *name, 
 		     struct ol_object *o, 
@@ -320,6 +325,7 @@
 	self->dns_cache_expire = 3600;
 	self->dns_cache_expire_failed = 60;
 	self->log_msg_size = 2048;
+	self->bad_hostname = NULL;
 	if ((cfg = fopen(name, "r")) != NULL) {
 		lex_init(cfg);
 		res = yyparse();
Index: src/cfgfile.h
===================================================================
RCS file: /var/cvs/syslog-ng/syslog-ng/src/cfgfile.h,v
retrieving revision 1.25
diff -u -r1.25 cfgfile.h
--- src/cfgfile.h	18 Oct 2002 12:31:08 -0000	1.25
+++ src/cfgfile.h	6 Jan 2003 12:42:54 -0000
@@ -57,6 +58,7 @@
 	(use_fqdn simple UINT32)
 	(use_dns simple UINT32)
 	(check_hostname simple UINT32)
+	(bad_hostname pointer UINT8)
 	(create_dirs simple UINT32)
 	(uid simple int)
 	(gid simple int)
@@ -129,10 +131,7 @@
 void cfg_set_dir_owner(char *uid);
 void cfg_set_dir_group(char *gid);
 void cfg_set_dir_perm(int perm);
-
-
-
-
+void cfg_set_bad_hostname(char *bad_hostname);
 
 
 struct syslog_config *make_syslog_config(const char *name, struct io_backend *backend);
Index: src/filters.c
===================================================================
RCS file: /var/cvs/syslog-ng/syslog-ng/src/filters.c,v
retrieving revision 1.16
diff -u -r1.16 filters.c
--- src/filters.c	4 Feb 2002 16:07:50 -0000	1.16
+++ src/filters.c	6 Jan 2003 12:42:54 -0000
@@ -34,6 +34,7 @@
 #include <sys/socket.h>
 #include <netinet/in.h>
 #include <arpa/inet.h>
+#include <string.h>
 
 #define CLASS_DEFINE
 #include "filters.h.x"
Index: src/log.c
===================================================================
RCS file: /var/cvs/syslog-ng/syslog-ng/src/log.c,v
retrieving revision 1.28
diff -u -r1.28 log.c
--- src/log.c	18 Oct 2002 12:31:08 -0000	1.28
+++ src/log.c	6 Jan 2003 12:42:54 -0000
@@ -41,7 +41,7 @@
 static char aix_fwd_string[] = "Message forwarded from ";
 static char repeat_msg_string[] = "last message repeated";
 
-static void parse_log_msg(struct log_info *lm, UINT32 length, UINT8 *data, UINT8 *prefix)
+static void parse_log_msg(struct log_info *lm, UINT32 length, UINT8 *data, UINT8 *prefix, regex_t *hostname_re)
 {
 	unsigned char *src;
 	unsigned int left, pri, oldleft;
@@ -153,12 +153,15 @@
 		else {
 			/* If we haven't already found the original hostname,
 			   look for it now. */
+			char hostname_buf[256];
+			int dst;
 
 		        oldsrc = src;	
 			oldleft = left;
 
+			dst = 0;
 			while (left && *src != ' ' && *src != ':' 
-			       && *src != '[') {
+			       && *src != '[' && dst < sizeof(hostname_buf) - 1) {
 				if (lm->flags & LF_CHECK_HOSTNAME &&
 				    !((*src >= 'A' && *src <= 'Z') ||
 				      (*src >= 'a' && *src <= 'z') ||
@@ -168,11 +171,13 @@
 				      *src == '@' || *src == '/')) {
 					break;
 				}
+				hostname_buf[dst++] = *src;
 		 		src++;
 				left--;
 			}
-
-			if (left && *src == ' ') {
+			hostname_buf[dst] = 0;
+			if (left && *src == ' ' &&
+			    (!hostname_re || regexec(hostname_re, hostname_buf, 0, NULL, 0))) {
 		  		/* This was a hostname. It came from a
 				   syslog-ng, since syslogd doesn't send
 				   hostnames. It's even better then the one
@@ -263,13 +268,13 @@
 	}
 }
 
-struct log_info *make_log_info(UINT32 length, UINT8 *msg, UINT8 *prefix, UINT32 flags)
+struct log_info *make_log_info(UINT32 length, UINT8 *msg, UINT8 *prefix, UINT32 flags, regex_t *hostname_re)
 {
 	struct log_info *self;
 
 	NEW_SPACE(self);
 	self->flags = flags & LF_USER_FLAGS;
-  	parse_log_msg(self, length, msg, prefix);
+  	parse_log_msg(self, length, msg, prefix, hostname_re);
 	self->use_cnt = 1;
   	self->recvd = time(NULL);
 	return self;
Index: src/log.h
===================================================================
RCS file: /var/cvs/syslog-ng/syslog-ng/src/log.h,v
retrieving revision 1.18
diff -u -r1.18 log.h
--- src/log.h	21 Aug 2002 14:03:50 -0000	1.18
+++ src/log.h	6 Jan 2003 12:42:54 -0000
@@ -30,6 +30,7 @@
 #include "io.h"
 
 #include <sys/time.h>
+#include <regex.h>
 
 struct syslog_config;
 struct persistent_config;
@@ -89,7 +90,7 @@
 struct log_info *log_info_use(struct log_info *msg);
 void log_info_free(struct log_info *msg);
 
-struct log_info *make_log_info(UINT32 length, UINT8 *data, UINT8 *prefix, UINT32 flags);
+struct log_info *make_log_info(UINT32 length, UINT8 *data, UINT8 *prefix, UINT32 flags, regex_t *badhostname_re);
 struct log_info *make_internal_message(UINT32 pri, UINT32 length, UINT8 *data);
 struct log_info *make_mark_message(void);
 
Index: src/sources.c
===================================================================
RCS file: /var/cvs/syslog-ng/syslog-ng/src/sources.c,v
retrieving revision 1.36
diff -u -r1.36 sources.c
--- src/sources.c	28 Oct 2002 08:33:30 -0000	1.36
+++ src/sources.c	6 Jan 2003 12:42:54 -0000
@@ -42,6 +42,7 @@
 
 #include "sources.c.x"
 #include "nscache.h"
+
 /* CLASS:
      (class
        (name log_reader)
@@ -54,6 +55,7 @@
 	 (max_log_line simple UINT32)
 	 (pad_size simple UINT32)
 	 (msg_flags simple UINT32)
+	 (bad_hostname special-struct regex_t #f regfree)
          (next object log_handler)))
 */
 
@@ -65,7 +67,7 @@
 {
 	struct log_info *logmsg;
 
-	logmsg = make_log_info(length, data, self->prefix, self->msg_flags);
+	logmsg = make_log_info(length, data, self->prefix, self->msg_flags, &self->bad_hostname);
 	if (addrlen) {
 		logmsg->saddr = sockaddr2address_info(addrlen, addr);
 	}
@@ -155,6 +157,7 @@
 		UINT32 max_log_line,
 		UINT32 pad_size,
 		UINT32 msg_flags,
+		UINT8 *hostname_re,
 		struct log_handler *next)
 {
 	NEW(log_reader, self);
@@ -167,7 +170,10 @@
 	self->pad_size = pad_size;
 	self->msg_flags = msg_flags;
 	self->buffer = ol_space_alloc(self->max_log_line);
-	
+	if (hostname_re == NULL)
+		regcomp(&self->bad_hostname, "^$", REG_NOSUB | REG_EXTENDED);
+	else
+		regcomp(&self->bad_hostname, hostname_re, REG_NOSUB | REG_EXTENDED);
 	return &self->super;
 }
 
Index: src/sources.h
===================================================================
RCS file: /var/cvs/syslog-ng/syslog-ng/src/sources.h,v
retrieving revision 1.16
diff -u -r1.16 sources.h
--- src/sources.h	21 Aug 2002 14:03:50 -0000	1.16
+++ src/sources.h	6 Jan 2003 12:42:54 -0000
@@ -66,6 +66,7 @@
 		UINT32 max_log_line,
 		UINT32 pad_size,
 		UINT32 msg_flags,
+		UINT8 *bad_hostname,
 		struct log_handler *next);
 
 struct log_source_group *make_source_group(const char *name, struct log_source_driver *drvs);

-- 
Bazsi
PGP info: KeyID 9AF8D0A9 Fingerprint CD27 CFB0 802C 0944 9CFD 804E C82C 8EB1

    