[syslog-ng] Missing hosts

8 Jul 2009

      Hi guys,

I have a problem that needs guru magic.  When I first setup syslog-ng I 
had too many host/log entries for the system I was using to handle.  I 
had tens of thousands of entries and each day created a 10G database 
table.  I remedied this logging certain hosts to flat files and 
separating host and network equipment into different databases.  I left 
my original catch all DB in place for comparison, to verify that all 
hosts were being logged properly.  When I do queries I find that a 
number of hosts are in the catch all but not in the individual 
databases.  Many of the missing hosts should be caught by the netmask 
entries but aren't.

options {
    sync (0);
    time_reopen (10);
    log_fifo_size (10000);
    time_sleep(50);
    log_fetch_limit(100);
    chain_hostnames(no);
    long_hostnames (off);
    keep_hostname(no);
    use_dns (yes);
    dns_cache(yes);
    use_fqdn (no);
    create_dirs (no);
    keep_hostname (yes);
};

source s_sys {
    file ("/proc/kmsg" log_prefix("kernel: "));
    unix-stream ("/dev/log");
    internal();
};

source s_everything {
    file ("/proc/kmsg" log_prefix("kernel: "));
    unix-stream ("/dev/log");
    internal();
    udp();
};

destination d_database { pipe("/tmp/mysql.pipe" template("INSERT INTO 
logs (host, facility, priority, level, tag, datetime, program, msg) 
VALUES ( '$HOST', '$FACILITY', '$PRIORITY', '$LEVEL', '$TAG', 
'$YEAR-$MONTH-$DAY $HOUR:$MIN:$SEC', '$PROGRAM', '$MSG' );\n") 
template-escape(yes)); };

destination d_networkdb { pipe("/tmp/mysql-network.pipe" 
template("INSERT INTO logs (host, facility, priority, level, tag, 
datetime, program, msg) VALUES ( '$HOST', '$FACILITY', '$PRIORITY', 
'$LEVEL', '$TAG', '$YEAR-$MONTH-$DAY $HOUR:$MIN:$SEC', '$PROGRAM', 
'$MSG' );\n") template-escape(yes)); };

destination d_hostsdb { pipe("/tmp/mysql-hosts.pipe" template("INSERT 
INTO logs (host, facility, priority, level, tag, datetime, program, msg) 
VALUES ( '$HOST', '$FACILITY', '$PRIORITY', '$LEVEL', '$TAG', 
'$YEAR-$MONTH-$DAY $HOUR:$MIN:$SEC', '$PROGRAM', '$MSG' );\n") 
template-escape(yes)); };

filter f_nofw            { not host("fwhost"); };

filter f_hosts            { netmask("10.16.5.0/255.255.255.0") or
                                  netmask("10.16.6.0/255.255.255.0") or
                                  host("hostA") or
                                  host("hostB") or
                                  host("hostC");
                                };
filter f_network        { not host("fwhost") and (
                                  host("host1") or
                                  host("host2") or
                                  host("host3") or
                                  netmask("10.16.57.0/255.255.255.0") or
                                  netmask("10.16.36.0/255.255.255.0") or
                                  netmask("10.16.120.0/255.255.255.0") or
                                  netmask("10.16.217.0/255.255.255.0")
                                  );
                                };

log { source(s_everything); filter(f_hosts); destination(d_hostsdb); };
log { source(s_everything); filter(f_network); destination(d_networkdb); };
log { source(s_everything); filter(f_nofw); destination(d_database); };

[syslog-ng] Missing hosts

Christopher Bland