[syslog-ng] How to name dir or logfile with the time by the message timestamp itself

22 May 2009


      Hi,

    I'm now have a syslog-ng to collect the syslog of 2 sites, 1 is in China
and another is in US, the host *vip-syslog* is in China, and US send the the
messages to *vip-syslog* through a special tunnel, (for reliability, I can
build a proxy with disk buffering to relay the syslog messages). Now *
vip-syslog* can recevied the messages and works well, but I still suffer a
problem.

    This is the syslog-ng.conf:

#cat /etc/syslog-ng/syslog-ng.conf
#
# configuration file for syslog-ng, customized for remote logging
#

source s_internal { internal(); };
destination d_syslognglog { file("/var/log/syslog-ng.log"); };
log { source(s_internal); destination(d_syslognglog); };

options {
        ts_format(rfc3164);
        chain_hostnames(no);
        use_dns(yes);
        dns_cache(yes);
        dns_cache_hosts(/etc/syslog-ng/hosts);
        use_fqdn(no);
        perm(0644);
        dir_perm(0700);
        flush_lines(128);
        flush_timeout(500);
        log_msg_size(16384);
        use_time_recvd(no);
        # recv_time_zone(-07:00);
        # send_time_zone(-07:00);
};

# ---------------------------------------------------------------------
# Local sources, filters and destinations are commented out
# If you want to replace sysklogd simply uncomment the following
# parts and disable sysklogd
#
# Local sources
#
source s_local {
        unix-dgram("/dev/log");
        file("/proc/kmsg" log_prefix("kernel:"));
};
#
# ---------------------------------------------------------------------
# Local filters
#
filter f_messages { level(info..emerg); };
filter f_secure { facility(authpriv); };
filter f_mail { facility(mail); };
filter f_cron { facility(cron); };
filter f_emerg { level(emerg); };
filter f_spooler { level(crit..emerg) and facility(uucp, news); };
filter f_local7 { facility(local7); };
#
# ---------------------------------------------------------------------
# Local destinations
#
destination d_messages { file("/var/log/messages"); };
destination d_secure { file("/var/log/secure"); };
destination d_maillog { file("/var/log/maillog"); };
destination d_cron { file("/var/log/cron"); };
destination d_console { usertty("root"); };
destination d_spooler { file("/var/log/spooler"); };
destination d_bootlog { file("/var/log/boot.log"); };
#
# ---------------------------------------------------------------------
# Local logs - order DOES matter !
#
log { source(s_local); filter(f_emerg); destination(d_console); };
log { source(s_local); filter(f_secure); destination(d_secure);
flags(final); };
log { source(s_local); filter(f_mail); destination(d_maillog); flags(final);
};
log { source(s_local); filter(f_cron); destination(d_cron); flags(final); };
log { source(s_local); filter(f_spooler); destination(d_spooler); };
log { source(s_local); filter(f_local7); destination(d_bootlog); };
log { source(s_local); filter(f_messages); destination(d_messages); };


# ---------------------------------------------------------------------
# Remote logging
#
# Remote sources
#
source s_remote {
        tcp(ip(0.0.0.0) port(514));
        udp(ip(0.0.0.0) port(514));
};

# ---------------------------------------------------------------------
# Remote destinations
#
template t_message {
        # template("$STAMP $HOST $PROGRAM[$PID] $MSG\n");
template_escape(no);
        template("$S_STAMP $HOST $MSG\n"); template_escape(no);
};

template t_apache {
        template("$MSGONLY\n"); template_escape(no);
};

# ---------------------------------
# Global site with US timezone:
destination d_en_alarm {
        file("/var/log/syslog-ng/$S_YEAR-$S_MONTH-$S_DAY/$HOST/alarm"
owner("root") group("root") perm(0640) dir_perm(0750) create_dirs(yes)
time_zone(-07:00) template(t_message));
    # program("sec.pl")
};

destination d_en_necessary {
        file("/var/log/syslog-ng/$S_YEAR-$S_MONTH-$S_DAY/$HOST/necessary"
owner("root") group("root") perm(0640) dir_perm(0750) create_dirs(yes)
time_zone(-07:00) template(t_message));
};

destination d_en_cookie_log {
        file("/var/log/syslog-ng/$S_YEAR-$S_MONTH-$S_DAY/$HOST/cookie_log"
owner("root") group("root") perm(0640) dir_perm(0750) create_dirs(yes)
time_zone(-07:00) template(t_apache));
};

# ---------------------------------
# China site with Beijing timezone:
destination d_cn_alarm {
        file("/var/log/syslog-ng/$S_YEAR-$S_MONTH-$S_DAY/$HOST/alarm"
owner("root") group("root") perm(0640) dir_perm(0750) create_dirs(yes)
template(t_message));
};

destination d_cn_necessary {
        file("/var/log/syslog-ng/$S_YEAR-$S_MONTH-$S_DAY/$HOST/necessary"
owner("root") group("root") perm(0640) dir_perm(0750) create_dirs(yes)
template(t_message));
};

destination d_cn_cookie_log {
        file("/var/log/syslog-ng/$S_YEAR-$S_MONTH-$S_DAY/$HOST/cookie_log"
owner("root") group("root") perm(0640) dir_perm(0750) create_dirs(yes)
template(t_apache));
};

destination d_null { file("/dev/null"); };

destination d_sec {
        program("/etc/syslog-ng/sec.pl -input=\"-\"
-conf=/etc/syslog-ng/sec.conf");
};

# ---------------------------------------------------------------------
# Remote filters
#
filter f_en_host { host("(^hz|^us|^hk)_"); };
#
# Used by data warehouse(dw):
filter f_cn_local1 { facility(local1); };
filter f_en_local2 { facility(local2); };
filter f_en_dw { filter(f_en_host) and filter(f_en_local2); };
filter f_cn_dw { not filter(f_en_host) and filter(f_cn_local1); };

# Used by system admins and security admins:
filter f_en_sys { filter(f_en_host) and not filter(f_en_local2); };
filter f_cn_sys { not filter(f_en_host) and not filter(f_cn_local1); };

filter f_necessary {
        match("necessary")
        or match("important");
        ......
};
filter f_alarm {
        match("(?i)(error|fail)");
        ......
};

......

# ---------------------------------------------------------------------
# Remote log path
#
log { source(s_remote); filter(f_en_sys); filter(f_necessary);
destination(d_en_necessary); };
log { source(s_remote); filter(f_cn_sys); filter(f_necessary);
destination(d_cn_necessary); };
log { source(s_remote); filter(f_en_sys); filter(f_alarm);
destination(d_en_alarm); };
# log { source(s_remote); filter(f_en_sys); filter(f_necessary);
destination(d_en_alarm); };
#       --> This will not generate duplicated message!
log { source(s_remote); filter(f_cn_sys); filter(f_alarm);
destination(d_cn_alarm); };
log { source(s_remote); filter(f_cn_dw); destination(d_cn_cookie_log); };
log { source(s_remote); filter(f_en_dw); destination(d_en_cookie_log); };
log { source(s_remote); destination(d_null); };


    NOW as you can see, I want the message to be logged to the dir:
/var/log/syslog-ng/$S_YEAR-$S_MONTH-$S_DAY/$HOST/

    The logs of China is OK, but the US is not since a timezone translating
must be performed. Now the message timestamp in the logfile is right, but
the dir name is not, for example:
#head /var/log/syslog-ng/2009-05-22/us_search63/alarm
May 21 10:50:27 us_search63 kernel: isupdate[7157]: segfault at
0000000000000010 rip 000000000041adad rsp 00007fffaa3cf560 error 4
May 21 18:09:55 us_search63 snmpd[26753]: /etc/snmp/snmpd.conf: line 433:
Error: WARNING: This output format is being deprecated - Please use the
'extend' directive instead
May 21 18:09:55 us_search63 snmpd[26753]: /etc/snmp/snmpd.conf: line 434:
Error: WARNING: This output format is being deprecated - Please use the
'extend' directive instead
May 21 18:09:55 us_search63 snmpd[26753]: /etc/snmp/snmpd.conf: line 435:
Error: WARNING: This output format is being deprecated - Please use the
'extend' directive instead
May 21 18:09:56 us_search63 snmpd[26753]: /etc/snmp/snmpd.conf: line 436:
Error: WARNING: This output format is being deprecated - Please use the
'extend' directive instead

    As you can see, the dir of 2009-05-22 has the records of May 21, which
is not what I want.

    I have tried the MACROS of $DAY, $S_DAY and $R_DAY, seems can not solve
this problem.

    Any suggestions?

    Thanks.

[syslog-ng] How to name dir or logfile with the time by the message timestamp itself

Roc Zhou