[syslog-ng]performance test questions
Achim Gsell
syslog-ng@lists.balabit.hu
Tue, 25 Feb 2003 23:24:38 +0100
--Boundary-00=_m0+W+S4bSeOu9aH
Content-Type: text/plain;
charset="iso-8859-1"
Content-Transfer-Encoding: 7bit
Content-Disposition: inline
Hi
On Monday 24 February 2003 13:03, Roberto Nibali wrote:
> > I'm not really surprised the macro expansion has performance problems.
> > When I wrote that code only 6-8 macros existed, I've checked my table and
> > it contains 51 entries. All looked up sequentially. Very bad performance
> > wise.
>
> Indeed.
>
> > Attached you'll find a patch which changes this to a faster algorithm. I
> > have also tested it.
>
> I'll give it a spin.
You can also test & try the attached "macro.c" which uses a hash function
generated with gperf. In my tests the binary search made the macro expansion
(4 macros in the template) about 2.5 times faster and the hash function (and
some other minor changes) about 3 times.
Achim
--Boundary-00=_m0+W+S4bSeOu9aH
Content-Type: text/x-csrc;
charset="iso-8859-1";
name="macros.c"
Content-Transfer-Encoding: quoted-printable
Content-Disposition: attachment; filename="macros.c"
/***************************************************************************
*
* Copyright (c) 1999 Bal=E1zs Scheidler
* Copyright (c) 1999-2001 BalaBit IT Ltd.
*=20
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program; if not, write to the Free Software
* Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
*
* Inspired by nsyslog, originally written by Darren Reed.
*
* $Id: macros.c,v 1.1 2003/01/31 14:26:48 bazsi Exp $
*
**************************************************************************=
*/
/*
* Several patches to the macro expansion function were contributed
* by Gert Menke.
*
* Portions Copyright (c) 2002 by Gert Menke
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. Neither the name of the author nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS `AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPO=
SE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTI=
AL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRI=
CT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
/*
* The patch to add the "template" and "template_escape" options to all
* destination drivers was contributed by Achim Gsell.
*/
#include <string.h>
#include <arpa/inet.h>
#include <stdio.h>
#include <assert.h>
#include "format.h"
#include "cfgfile.h"
#include "macros.h"
#include "syslog-names.h"
#define MAX_MACRO_ARGS 32
#define MAX_EXPANDED_MACRO 2048
#define M_FACILITY 0
#define M_LEVEL 1
#define M_TAG 2
#define M_DATE 3
#define M_FULLDATE 4
#define M_ISODATE 5
#define M_YEAR 6
#define M_MONTH 7
#define M_DAY 8
#define M_HOUR 9
#define M_MIN 10
#define M_SEC 11
#define M_WEEKDAY 12
#define M_TZOFFSET 13
#define M_TZ 14
#define M_UNIXTIME 15
#define M_DATE_RECVD 16
#define M_FULLDATE_RECVD 17
#define M_ISODATE_RECVD 18
#define M_YEAR_RECVD 19
#define M_MONTH_RECVD 20
#define M_DAY_RECVD 21
#define M_HOUR_RECVD 22
#define M_MIN_RECVD 23
#define M_SEC_RECVD 24
#define M_WEEKDAY_RECVD 25
#define M_TZOFFSET_RECVD 26
#define M_TZ_RECVD 27
#define M_UNIXTIME_RECVD 28
#define M_DATE_STAMP 30
#define M_FULLDATE_STAMP 31
#define M_ISODATE_STAMP 32
#define M_YEAR_STAMP 33
#define M_MONTH_STAMP 34
#define M_DAY_STAMP 35
#define M_HOUR_STAMP 36
#define M_MIN_STAMP 37
#define M_SEC_STAMP 38
#define M_WEEKDAY_STAMP 39
#define M_TZOFFSET_STAMP 40
#define M_TZ_STAMP 41
#define M_UNIXTIME_STAMP 42
#define M_FULLHOST 43
#define M_HOST 44
#define M_FULLHOST_FROM 45
#define M_HOST_FROM 46
#define M_PROGRAM 47
#define M_MESSAGE 48
#define M_SOURCE_IP 49
static int
append_string(char **dest, int left, char *str, int length, int escape)
{
if (!escape) {
int l;
l =3D MIN(length, left - 1);
memcpy(*dest, str, l);
return l;
}
else {
char *dest_ptr =3D *dest;
=20
for (; length && (left > 2); length--, left--) {
if (*str =3D=3D '\'' || *str =3D=3D '\"' || *str =3D=3D '\\') { /* '" */
*dest_ptr++ =3D '\\';
left--;
}
*dest_ptr++ =3D *str++;
}
return (dest_ptr - *dest);
}
}
static void
expand_macro(struct syslog_config *cfg, int id, int escape, char **dest, un=
signed int *left, struct log_info *msg)
{
int length =3D 0;
=09
switch (id) {
case M_FACILITY: {
/* facility */
char *n =3D syslog_lookup_value(msg->pri & LOG_FACMASK, sl_facilities);
if (n) {
length =3D append_string(dest, *left, n, strlen(n), 0);
}
else {
length =3D snprintf(*dest, *left, "%x", (msg->pri & LOG_FACMASK) >> 3);
}
break;
}
case M_LEVEL: {
/* level */
char *n =3D syslog_lookup_value(msg->pri & LOG_PRIMASK, sl_levels);
if (n) {
length =3D append_string(dest, *left, n, strlen(n), 0);
}
else {
/* should never happen */
length =3D snprintf(*dest, *left, "%d", msg->pri & LOG_PRIMASK);
}
break;
}
case M_TAG: {
length =3D snprintf(*dest, *left, "%02x", msg->pri);
break;
}
case M_SOURCE_IP: {
char *ip;
=09
if (msg->saddr) {
CAST(inet_address_info, addr, msg->saddr);
=09
ip =3D inet_ntoa(addr->sa.sin_addr);
}
else {
ip =3D "127.0.0.1";
}
length =3D append_string(dest, *left, ip, strlen(ip), escape);
break;
}
case M_FULLHOST_FROM:
case M_FULLHOST: {
struct ol_string *host =3D (id =3D=3D M_FULLHOST ? msg->host : msg->host_=
from);
/* full hostname */
length =3D append_string(dest, *left, host->data, host->length, escape);
break;
}
case M_HOST_FROM:
case M_HOST: {
/* host */
struct ol_string *host =3D (id =3D=3D M_HOST ? msg->host : msg->host_from=
);
UINT8 *p1;
UINT8 *p2;
int remaining;
=09
p1 =3D memchr(host->data, '@', host->length);
if (p1)=20
p1++;=20
else=20
p1 =3D host->data;
remaining =3D host->length - (p1 - host->data);
p2 =3D memchr(p1, '/', remaining);
if (p2) {
length =3D MIN((unsigned int) (p2 - p1), *left);
}
else {
length =3D MIN(*left, (unsigned int) (host->length - (p1 - host->data)));
}
length =3D append_string(dest, *left, p1, length, escape);
break;
}
case M_PROGRAM: {
/* program */
if (msg->program) {
length =3D append_string(dest, *left, msg->program->data, msg->program->=
length, escape);
}
break;
}
case M_FULLDATE_RECVD:
case M_ISODATE_RECVD:
case M_WEEKDAY_RECVD:
case M_DATE_RECVD:
case M_YEAR_RECVD:
case M_MONTH_RECVD:
case M_DAY_RECVD:
case M_HOUR_RECVD:
case M_MIN_RECVD:
case M_SEC_RECVD:
case M_TZOFFSET_RECVD:
case M_TZ_RECVD:
case M_UNIXTIME_RECVD:
case M_FULLDATE_STAMP:
case M_ISODATE_STAMP:
case M_WEEKDAY_STAMP:
case M_DATE_STAMP:
case M_YEAR_STAMP:
case M_MONTH_STAMP:
case M_DAY_STAMP:
case M_HOUR_STAMP:
case M_MIN_STAMP:
case M_SEC_STAMP:
case M_TZOFFSET_STAMP:
case M_TZ_STAMP:
case M_UNIXTIME_STAMP:
case M_FULLDATE:
case M_ISODATE:
case M_WEEKDAY:
case M_DATE:
case M_YEAR:
case M_MONTH:
case M_DAY:=20
case M_HOUR:
case M_MIN:
case M_SEC:
case M_TZOFFSET:
case M_TZ:
case M_UNIXTIME: {
/* year, month, day */
struct tm *tm;
time_t unixtime;
switch(id) {
case M_FULLDATE_RECVD:
case M_ISODATE_RECVD:=20
case M_WEEKDAY_RECVD:=20
case M_DATE_RECVD: =20
case M_YEAR_RECVD: =20
case M_MONTH_RECVD: =20
case M_DAY_RECVD: =20
case M_HOUR_RECVD: =20
case M_MIN_RECVD: =20
case M_SEC_RECVD: =20
case M_TZOFFSET_RECVD:
case M_TZ_RECVD:
case M_UNIXTIME_RECVD: =20
unixtime =3D msg->recvd;
break;
case M_FULLDATE_STAMP:
case M_ISODATE_STAMP:=20
case M_WEEKDAY_STAMP:=20
case M_DATE_STAMP: =20
case M_YEAR_STAMP: =20
case M_MONTH_STAMP: =20
case M_DAY_STAMP: =20
case M_HOUR_STAMP: =20
case M_MIN_STAMP: =20
case M_SEC_STAMP: =20
case M_TZOFFSET_STAMP:
case M_TZ_STAMP:
case M_UNIXTIME_STAMP: =20
unixtime =3D msg->stamp;
break;
default:
if (cfg->use_time_recvd)
unixtime =3D msg->recvd;
else
unixtime =3D msg->stamp;
break;
}
=09
tm =3D localtime(&unixtime);
switch (id) {
case M_WEEKDAY:
case M_WEEKDAY_RECVD:
case M_WEEKDAY_STAMP:
length =3D strftime(*dest, *left - 1, "%a", tm); =09
break;
case M_YEAR:
case M_YEAR_RECVD:
case M_YEAR_STAMP:
length =3D snprintf(*dest, *left, "%04d", tm->tm_year + 1900);
break;
case M_MONTH:
case M_MONTH_RECVD:
case M_MONTH_STAMP:
length =3D snprintf(*dest, *left, "%02d", tm->tm_mon + 1);
break;
case M_DAY:
case M_DAY_RECVD:
case M_DAY_STAMP: =20
length =3D snprintf(*dest, *left, "%02d", tm->tm_mday);
break;
case M_HOUR:
case M_HOUR_RECVD:
case M_HOUR_STAMP:
length =3D snprintf(*dest, *left, "%02d", tm->tm_hour);
break;
case M_MIN:
case M_MIN_RECVD:
case M_MIN_STAMP:
length =3D snprintf(*dest, *left, "%02d", tm->tm_min);
break;
case M_SEC:
case M_SEC_RECVD:
case M_SEC_STAMP:
length =3D snprintf(*dest, *left, "%02d", tm->tm_sec);
break;
case M_ISODATE:
case M_ISODATE_RECVD:
case M_ISODATE_STAMP:
length =3D strftime(*dest, *left - 1, "%Y-%m-%dT%H:%M:%S%z=
", tm);
break;
case M_FULLDATE:
case M_FULLDATE_RECVD:
case M_FULLDATE_STAMP:
length =3D strftime(*dest, *left - 1, "%Y %h %e %H:%M:%S",=
tm);
break;
case M_DATE:
case M_DATE_RECVD:
case M_DATE_STAMP:
length =3D strftime(*dest, *left - 1, "%h %e %H:%M:%S", tm=
);
break;
case M_TZOFFSET:
case M_TZOFFSET_RECVD:
case M_TZOFFSET_STAMP:
length =3D strftime(*dest, *left -1, "%z", tm);
break;
case M_TZ:
case M_TZ_RECVD:
case M_TZ_STAMP:
length =3D strftime(*dest, *left -1, "%Z", tm);
break;
case M_UNIXTIME:
case M_UNIXTIME_RECVD:
case M_UNIXTIME_STAMP:
length =3D snprintf(*dest, *left, "%ld", (long) unixtime);
break;
}
break;
}
case M_MESSAGE: {
/* message */
length =3D append_string(dest, *left, msg->msg->data, msg->msg->length, e=
scape);
break;
}
default:
break;
}
if (length < 0 || (unsigned int) length > *left)=20
length =3D *left;
*left -=3D length;
*dest +=3D length;
}
/* C code produced by gperf version 2.7.2 */
/* Command-line: gperf -t --compare-strncmp --duplicates --lookup-fn-name=
=3Dfind_macro macros_hash.gperf */
struct macro_def { char *name; int id; int len; };
#define TOTAL_KEYWORDS 51
#define MIN_WORD_LENGTH 2
#define MAX_WORD_LENGTH 13
#define MIN_HASH_VALUE 4
#define MAX_HASH_VALUE 112
/* maximum key range =3D 109, duplicates =3D 4 */
#ifdef __GNUC__
__inline
#else
#ifdef __cplusplus
inline
#endif
#endif
static unsigned int
hash (str, len)
register const char *str;
register unsigned int len;
{
static unsigned char asso_values[] =3D
{
113, 113, 113, 113, 113, 113, 113, 113, 113, 113,
113, 113, 113, 113, 113, 113, 113, 113, 113, 113,
113, 113, 113, 113, 113, 113, 113, 113, 113, 113,
113, 113, 113, 113, 113, 113, 113, 113, 113, 113,
113, 113, 113, 113, 113, 113, 113, 113, 113, 113,
113, 113, 113, 113, 113, 113, 113, 113, 113, 113,
113, 113, 113, 113, 113, 113, 113, 6, 20, 25,
35, 25, 0, 5, 113, 113, 5, 0, 55, 113,
5, 113, 0, 10, 52, 30, 113, 0, 113, 46,
10, 113, 113, 113, 113, 113, 113, 113, 113, 113,
113, 113, 113, 113, 113, 113, 113, 113, 113, 113,
113, 113, 113, 113, 113, 113, 113, 113, 113, 113,
113, 113, 113, 113, 113, 113, 113, 113, 113, 113,
113, 113, 113, 113, 113, 113, 113, 113, 113, 113,
113, 113, 113, 113, 113, 113, 113, 113, 113, 113,
113, 113, 113, 113, 113, 113, 113, 113, 113, 113,
113, 113, 113, 113, 113, 113, 113, 113, 113, 113,
113, 113, 113, 113, 113, 113, 113, 113, 113, 113,
113, 113, 113, 113, 113, 113, 113, 113, 113, 113,
113, 113, 113, 113, 113, 113, 113, 113, 113, 113,
113, 113, 113, 113, 113, 113, 113, 113, 113, 113,
113, 113, 113, 113, 113, 113, 113, 113, 113, 113,
113, 113, 113, 113, 113, 113, 113, 113, 113, 113,
113, 113, 113, 113, 113, 113, 113, 113, 113, 113,
113, 113, 113, 113, 113, 113, 113, 113, 113, 113,
113, 113, 113, 113, 113, 113
};
return len + asso_values[(unsigned char)str[len - 1]] + asso_values[(unsi=
gned char)str[0]];
}
#ifdef __GNUC__
__inline
#endif
struct macro_def *
find_macro (str, len)
register const char *str;
register unsigned int len;
{
static struct macro_def wordlist[] =3D
{
{"HOUR", M_HOUR},
{"MONTH", M_MONTH},
{"R_YEAR", M_YEAR_RECVD},
{"R_HOUR", M_HOUR_RECVD},
{"R_MONTH", M_MONTH_RECVD},
{"HOST_FROM", M_HOST_FROM},
{"R_SEC", M_SEC_RECVD},
{"PROGRAM", M_PROGRAM},
{"R_TZ", M_TZ_RECVD},
{"LEVEL", M_LEVEL},
{"S_YEAR", M_YEAR_STAMP},
{"S_HOUR", M_HOUR_STAMP},
{"S_MONTH", M_MONTH_STAMP},
{"SEC", M_SEC},
{"S_SEC", M_SEC_STAMP},
{"SOURCEIP", M_SOURCE_IP},
{"S_TZ", M_TZ_STAMP},
{"MSG", M_MESSAGE},
{"R_DATE", M_DATE_RECVD},
{"MESSAGE", M_MESSAGE},
{"R_ISODATE", M_ISODATE_RECVD},
{"R_FULLDATE", M_FULLDATE_RECVD},
{"R_UNIXTIME", M_UNIXTIME_RECVD},
{"ISODATE", M_ISODATE},
{"S_DATE", M_DATE_STAMP},
{"S_ISODATE", M_ISODATE_STAMP},
{"S_FULLDATE", M_FULLDATE_STAMP},
{"S_UNIXTIME", M_UNIXTIME_STAMP},
{"FULLHOST_FROM", M_FULLHOST_FROM},
{"DATE", M_DATE},
{"YEAR", M_YEAR},
{"R_DAY", M_DAY_RECVD},
{"WEEKDAY", M_WEEKDAY},
{"R_WEEKDAY", M_WEEKDAY_RECVD},
{"HOST", M_HOST},
{"MIN", M_MIN},
{"PRIORITY", M_LEVEL},
{"R_MIN", M_MIN_RECVD},
{"S_DAY", M_DAY_STAMP},
{"R_TZOFFSET", M_TZOFFSET_RECVD},
{"UNIXTIME", M_UNIXTIME},
{"TZ", M_TZ},
{"S_WEEKDAY", M_WEEKDAY_STAMP},
{"FULLDATE", M_FULLDATE},
{"DAY", M_DAY},
{"S_MIN", M_MIN_STAMP},
{"S_TZOFFSET", M_TZOFFSET_STAMP},
{"TAG", M_TAG},
{"FACILITY", M_FACILITY},
{"FULLHOST", M_FULLHOST},
{"TZOFFSET", M_TZOFFSET}
};
static short lookup[] =3D
{
-1, -1, -1, -1, 0, 1, -81, 4, -1, 5,
-1, 6, 7, -1, 8, 9, -77, 12, -1, 13,
-1, 14, -1, 15, 16, -41, -2, -1, 17, -49,
-2, 18, 19, -1, 20, -90, -1, 23, -30, -2,
-1, 24, -1, -1, 25, -98, -25, -2, 28, 29,
30, 31, -1, 32, -1, 33, 34, -1, 35, 36,
37, 38, 39, 40, 41, 42, -1, -1, 43, 44,
45, -1, 46, -1, -1, -1, -1, -1, -1, -1,
47, -1, -1, -1, -1, -1, -1, -1, -1, 48,
-1, -1, -1, -1, -1, 49, -1, -1, -1, -1,
-1, -1, -1, -1, -1, -1, -1, -1, -1, -1,
-1, -1, 50
};
if (len <=3D MAX_WORD_LENGTH && len >=3D MIN_WORD_LENGTH)
{
register int key =3D hash (str, len);
if (key <=3D MAX_HASH_VALUE && key >=3D 0)
{
register int index =3D lookup[key];
if (index >=3D 0)
{
register const char *s =3D wordlist[index].name;
if (*str =3D=3D *s && !strncmp (str + 1, s + 1, len - 1) && s=
[len] =3D=3D '\0')
return &wordlist[index];
}
else if (index < -TOTAL_KEYWORDS)
{
register int offset =3D - 1 - TOTAL_KEYWORDS - index;
register struct macro_def *wordptr =3D &wordlist[TOTAL_KEYWOR=
DS + lookup[offset]];
register struct macro_def *wordendptr =3D wordptr + -lookup[o=
ffset + 1];
while (wordptr < wordendptr)
{
register const char *s =3D wordptr->name;
if (*str =3D=3D *s && !strncmp (str + 1, s + 1, len - 1) =
&& s[len] =3D=3D '\0')
return wordptr;
wordptr++;
}
}
}
}
return 0;
}
struct ol_string *
expand_macros(struct syslog_config *cfg, struct ol_string *template, int te=
mplate_escape, struct log_info *msg)
{
char format[cfg->log_msg_size + 1];
char *format_ptr =3D format;
unsigned int left =3D sizeof(format) - 1;
char *template_ptr =3D template->data;
char *template_end_ptr =3D template_ptr + template->length;=20
while (left && (template_ptr < template_end_ptr)) {
if (*template_ptr =3D=3D '$') {
/* beginning of a macro */
struct macro_def *m;
char *begin_ptr =3D ++template_ptr;
while ( ((*template_ptr >=3D 'A') && (*template_ptr <=3D 'Z')) || (*temp=
late_ptr =3D=3D '_') ) {
template_ptr++;
}
=09
if ( (m =3D find_macro(begin_ptr, template_ptr-begin_ptr)) !=3D NULL) {
expand_macro(cfg, m->id, template_escape, &format_ptr, &left, msg);
}
}
else {
*format_ptr++ =3D *template_ptr++;
left--;
}
}
*format_ptr =3D '\0';
return c_format_cstring("%z", format);
}
--Boundary-00=_m0+W+S4bSeOu9aH--