On Wed, 2009-07-08 at 13:24 -0400, Christopher Bland wrote:
Hi guys,
I have a problem that needs guru magic. When I first setup syslog-ng I had too many host/log entries for the system I was using to handle. I had tens of thousands of entries and each day created a 10G database table. I remedied this logging certain hosts to flat files and separating host and network equipment into different databases. I left my original catch all DB in place for comparison, to verify that all hosts were being logged properly. When I do queries I find that a number of hosts are in the catch all but not in the individual databases. Many of the missing hosts should be caught by the netmask entries but aren't.
Although I didn't try to comprehend your filters completely, but here are some tips: 1) netmask() is filtering against the last-hop relay, that actually sent the message to syslog-ng, not the contents of the "HOST" field in the message 2) your "catch-all" database does not exactly negate the filter of the first two statements 3) you could enable --verbose --debug and have syslog-ng log to your terminal (redirecting syslog-ng to your terminal is quite important in this case as debug may generate recursive messages if you are not using syslog-ng 3.0), then you should see messages that help you debug your filter statements: This message is printed to aid filter evaluation: msg_debug("Filter node evaluation result", evt_tag_str("filter_result", res ? "match" : "not-match"), evt_tag_str("filter_type", self->type), NULL); msg_debug("Filter rule evaluation result", evt_tag_str("filter_result", res ? "match" : "not-match"), evt_tag_str("filter_rule", self->super.name), NULL);
options { sync (0); time_reopen (10); log_fifo_size (10000); time_sleep(50); log_fetch_limit(100); chain_hostnames(no); long_hostnames (off); keep_hostname(no); use_dns (yes); dns_cache(yes); use_fqdn (no); create_dirs (no); keep_hostname (yes); };
source s_sys { file ("/proc/kmsg" log_prefix("kernel: ")); unix-stream ("/dev/log"); internal(); };
source s_everything { file ("/proc/kmsg" log_prefix("kernel: ")); unix-stream ("/dev/log"); internal(); udp(); };
destination d_database { pipe("/tmp/mysql.pipe" template("INSERT INTO logs (host, facility, priority, level, tag, datetime, program, msg) VALUES ( '$HOST', '$FACILITY', '$PRIORITY', '$LEVEL', '$TAG', '$YEAR-$MONTH-$DAY $HOUR:$MIN:$SEC', '$PROGRAM', '$MSG' );\n") template-escape(yes)); };
destination d_networkdb { pipe("/tmp/mysql-network.pipe" template("INSERT INTO logs (host, facility, priority, level, tag, datetime, program, msg) VALUES ( '$HOST', '$FACILITY', '$PRIORITY', '$LEVEL', '$TAG', '$YEAR-$MONTH-$DAY $HOUR:$MIN:$SEC', '$PROGRAM', '$MSG' );\n") template-escape(yes)); };
destination d_hostsdb { pipe("/tmp/mysql-hosts.pipe" template("INSERT INTO logs (host, facility, priority, level, tag, datetime, program, msg) VALUES ( '$HOST', '$FACILITY', '$PRIORITY', '$LEVEL', '$TAG', '$YEAR-$MONTH-$DAY $HOUR:$MIN:$SEC', '$PROGRAM', '$MSG' );\n") template-escape(yes)); };
filter f_nofw { not host("fwhost"); };
filter f_hosts { netmask("10.16.5.0/255.255.255.0") or netmask("10.16.6.0/255.255.255.0") or host("hostA") or host("hostB") or host("hostC"); }; filter f_network { not host("fwhost") and ( host("host1") or host("host2") or host("host3") or netmask("10.16.57.0/255.255.255.0") or netmask("10.16.36.0/255.255.255.0") or netmask("10.16.120.0/255.255.255.0") or netmask("10.16.217.0/255.255.255.0") ); };
log { source(s_everything); filter(f_hosts); destination(d_hostsdb); }; log { source(s_everything); filter(f_network); destination(d_networkdb); }; log { source(s_everything); filter(f_nofw); destination(d_database); };
______________________________________________________________________________ Member info: https://lists.balabit.hu/mailman/listinfo/syslog-ng Documentation: http://www.balabit.com/support/documentation/?product=syslog-ng FAQ: http://www.campin.net/syslog-ng/faq.html
-- Bazsi