[syslog-ng] Losing to much remote sent logs

2 Mar 2012

      Hello there,
I've started playing around with syslog-ng 3.3.4 ose a few days ago but I'm still experiencing some trouble. First of all we want to use syslog-ng to send all of our logs via udp to a central syslog server. This includes of course syslogs, apache logs and custom generated applogs. These logs are generated from 400 clients and produces a minimum of 300 mio. log lines a day.
The problem is really simple: I'm losing log lines :P Most of the time everything goes well but when the logs are peaking high 1-5% logs are getting lost.
Last night the stats of the server and a client said 0 drops but when I counted the lines I found lost lines. The server has 24g ram & 8 cores and I can rule out a network problem for sure.

So now to my questions, has anyone else an idea where I can tweak my cfg or where I have to look to find more clues? Is tcp the only way to get around it?
I've attached my syslog server cfg. The so_rcvbuf buffer is the same size as the os net.core.rmem settings. And as described in the various balabit blog posts I played around with log_fetch_limit and flush_lines already.

syslog-ng.conf:
@version: 3.3

options {
    threaded(yes);
    owner("root");
    group("root");
    perm(0660);

    dir_owner("root");
    dir_group("root");
    dir_perm(0770);
    create_dirs(yes);

    stats_freq(600);
    stats_level(2);
    chain_hostnames(yes);
    normalize_hostnames(yes);
    check_hostname(yes);

    dns_cache(yes);
    dns_cache_size(16384);
    dns_cache_expire(3600);
    dns_cache_expire_failed(60);

    log_msg_size(16384);
    log_fifo_size(100000);

    use_fqdn(yes);
#disabled 4 debugging
#    flush_lines(200);
};

source s_src {
        unix-dgram("/dev/log");
        internal();
        file("/proc/kmsg" program_override("kernel"));
};

source s_net {
udp(
        log_fetch_limit(400)
        so_rcvbuf(51200000)
        keep_hostname(yes)
        keep_timestamp(no)
        ip("10.8.4.10")
        port(514)
);
tcp(
        so_rcvbuf(51200000)
        so_keepalive(yes)
        keep_hostname(no)
        keep_timestamp(no)
        ip("10.8.4.10")
        port(514)

);
syslog();
};

filter f_syslog {
     not program(access.log) and
     not program(error.log) and
     not program(beetle.log) and
     not program(edge.log);

};

filter f_apache {
    program(access.log) or
    program(error.log);
};

filter f_applogs {
    program(beetle.log)
    or program(edge.log);
};

template t_plain {
    template("$MSG\n"); template_escape(no);
};

destination d_messages { file("/var/log/messages"); };
destination d_remote { file("/log/syslog/${R_YEAR}/${R_MONTH}/${R_DAY}/$HOST"); };
destination d_apache { file("/log/apache/${R_YEAR}/${R_MONTH}/${R_DAY}/$HOST/$PROGRAM" template(t_plain)); };
destination d_applogs { file("/log/applogs/${R_YEAR}/${R_MONTH}/${R_DAY}/$HOST/$PROGRAM" template(t_plain)); };

log {
    source(s_src);
    destination(d_messages);
};

log {
    source(s_net);
    filter(f_syslog);
    destination(d_remote);
};

log {
    source(s_net);
    filter(f_apache);
    destination(d_apache);
};

log {
    source(s_net);
    filter(f_applogs);
    destination(d_applogs);
};

Thanks
Daniel Neubacher