On Thu, 2006-01-19 at 13:49 -0600, Paul Krizak wrote:
I've been getting a constant stream of these messages in my syslog-ng log, as much as 70MB worth per day:
Jan 19 13:43:39 i-dl385 syslog-ng[25668]: Duplicate stats counter; counter='file(/var/log/logs/$R_YEAR/$R_MONTH/$R_DAY/$HOST-messages.log)' Jan 19 13:43:40 i-dl385 syslog-ng[25668]: Duplicate stats counter; counter='file(/var/log/logs/$R_YEAR/$R_MONTH/$R_DAY/$HOST-messages.log)' Jan 19 13:43:40 i-dl385 syslog-ng[25668]: Duplicate stats counter; counter='file(/var/log/logs/$R_YEAR/$R_MONTH/$R_DAY/$HOST-messages.log)'
What does this mean? Are stats getting collected properly? Is there a bug in the stats stuff?
Oops, this seems to be a bug in that the stats counter is registered for the template filename and not the expanded filename. This is not a real problem though, as file destinations are not flow controlled, thus they never drop messages. It's ugly nevertheless. The patch below adds a way to disable stats counting for specific destinations and to instruct the stats subsystem to share a counter. Don't be afraid of the patch size, it is not as big as it seems :) It should be available in tomorrow's snapshot. BTW: what are the experiences with 1.9.8, does it work otherwise fine? --- orig/src/affile.c +++ mod/src/affile.c @@ -367,18 +367,6 @@ affile_dd_set_create_dirs(LogDriver *s, self->flags &= ~AFFILE_CREATE_DIRS; } -static const gchar * -affile_dd_format_stats_name(AFFileDestDriver *self) -{ - static gchar stats_name[256]; - - g_snprintf(stats_name, sizeof(stats_name), - "file(%s)", self->filename_template->template->str); - - return stats_name; -} - - static time_t reap_now = 0; static gboolean @@ -439,7 +427,7 @@ affile_dd_init(LogPipe *s, GlobalConfig self->time_reap = cfg->time_reap; self->use_time_recvd = cfg->use_time_recvd; - log_writer_options_init(&self->writer_options, cfg, FALSE, affile_dd_format_stats_name(self)); + log_writer_options_init(&self->writer_options, cfg, LWOF_NO_STATS, NULL); self->cfg = cfg; if ((self->flags & AFFILE_NO_EXPAND) == 0) --- orig/src/afprog.c +++ mod/src/afprog.c @@ -70,7 +70,7 @@ afprogram_dd_init(LogPipe *s, GlobalConf int msg_pipe[2]; if (cfg) - log_writer_options_init(&self->writer_options, cfg, FALSE, afprogram_dd_format_stats_name(self)); + log_writer_options_init(&self->writer_options, cfg, 0, afprogram_dd_format_stats_name(self)); msg_verbose("Starting destination program", evt_tag_str("cmdline", self->cmdline->str), --- orig/src/afsocket.c +++ mod/src/afsocket.c @@ -722,7 +722,7 @@ afsocket_dd_init(LogPipe *s, GlobalConfi if (!self->writer) { - log_writer_options_init(&self->writer_options, cfg, !!(self->flags & AFSOCKET_PROTO_RFC3164), afsocket_dd_format_stats_name(self)); + log_writer_options_init(&self->writer_options, cfg, (self->flags & AFSOCKET_PROTO_RFC3164) ? LWOF_FIXED_STAMP : 0, afsocket_dd_format_stats_name(self)); /* NOTE: we open our writer with no fd, so we can send messages down there * even while the connection is not established */ --- orig/src/logwriter.c +++ mod/src/logwriter.c @@ -377,8 +377,8 @@ log_writer_init(LogPipe *s, GlobalConfig { LogWriter *self = (LogWriter *) s; - if (!self->dropped_messages) - stats_register_counter(SC_TYPE_DROPPED, self->options->stats_name, &self->dropped_messages); + if ((self->options->flags & LWOF_NO_STATS) == 0 && !self->dropped_messages) + stats_register_counter(SC_TYPE_DROPPED, self->options->stats_name, &self->dropped_messages, !!(self->options->flags & LWOF_SHARE_STATS)); return TRUE; } @@ -477,8 +477,9 @@ log_writer_options_set_template_escape(L } void -log_writer_options_init(LogWriterOptions *options, GlobalConfig *cfg, gboolean fixed_stamp, const gchar *stats_name) +log_writer_options_init(LogWriterOptions *options, GlobalConfig *cfg, guint32 flags, const gchar *stats_name) { + options->flags = flags; if (options->fifo_size == -1) options->fifo_size = cfg->log_fifo_size; if (options->use_time_recvd == -1) @@ -498,7 +499,7 @@ log_writer_options_init(LogWriterOptions options->flush_lines = options->fifo_size - 1; } - if (!fixed_stamp) + if ((flags & LWOF_FIXED_STAMP) == 0) { if (options->keep_timestamp == -1) options->keep_timestamp = cfg->keep_timestamp; --- orig/src/logwriter.h +++ mod/src/logwriter.h @@ -33,14 +33,23 @@ #define LW_FORMAT_FILE 0x0002 #define LW_FORMAT_PROTO 0x0004 -/* writer options */ +/* writer options (set by the user) */ #define LWO_TMPL_ESCAPE 0x0001 +/* writer flags */ +#define LWOF_FIXED_STAMP 0x0001 +/* we don't want to have a dropped counter for this writer */ +#define LWOF_NO_STATS 0x0002 +/* several writers use the same counter */ +#define LWOF_SHARE_STATS 0x0004 + typedef struct _LogWriterOptions { gchar *stats_name; - /* bitmask of LWO_* */ + /* bitmask of LWO_* (set by the user) */ guint32 options; + /* bitmask of LWOF_* (set by the driver) */ + guint32 flags; /* maximum number of entries */ gint fifo_size; @@ -78,6 +87,6 @@ gboolean log_writer_reopen(LogPipe *s, F void log_writer_options_set_template_escape(LogWriterOptions *options, gboolean enable); void log_writer_options_defaults(LogWriterOptions *options); -void log_writer_options_init(LogWriterOptions *options, GlobalConfig *cfg, gboolean fixed_stamps, const gchar *stats_name); +void log_writer_options_init(LogWriterOptions *options, GlobalConfig *cfg, guint32 flags, const gchar *stats_name); #endif --- orig/src/stats.c +++ mod/src/stats.c @@ -28,6 +28,7 @@ typedef struct _StatsCounter { + guint ref_cnt; StatsCounterType type; gchar *name; guint32 counter; @@ -47,10 +48,25 @@ stats_find_counter(const gchar *counter_ return g_list_find_custom(counters, counter_name, (GCompareFunc) stats_cmp_name); } +/** + * stats_register_counter: + * @type: counter type + * @counter_name: name to identify this stats counter + * @counter: returned pointer to the counter + * @shared: whether multiple sources will use the same counter + * + * This fuction registers a general purpose counter. Whenever multiple + * objects touch the same counter all of these should register the counter + * with the same name, specifying TRUE for the value of permit_dup, + * internally the stats subsystem counts the number of users of the same + * counter in this case, thus the counter will only be freed when all of + * these uses are unregistered. + **/ void -stats_register_counter(StatsCounterType type, const gchar *counter_name, guint32 **counter) +stats_register_counter(StatsCounterType type, const gchar *counter_name, guint32 **counter, gboolean shared) { StatsCounter *sc; + GList *l; /* FIXME: we should use a separate name-space for all different types, * however we only have a single type so far */ @@ -59,12 +75,21 @@ stats_register_counter(StatsCounterType *counter = NULL; if (!counter_name) return; - if (stats_find_counter(counter_name)) + if ((l = stats_find_counter(counter_name))) { - msg_notice("Duplicate stats counter", - evt_tag_str("counter", counter_name), - NULL); - *counter = NULL; + if (!shared) + { + msg_notice("Duplicate stats counter", + evt_tag_str("counter", counter_name), + NULL); + *counter = NULL; + } + else + { + sc = (StatsCounter *) l->data; + sc->ref_cnt++; + *counter = &sc->counter; + } return; } @@ -73,6 +98,7 @@ stats_register_counter(StatsCounterType sc->type = type; sc->name = g_strdup(counter_name); sc->counter = 0; + sc->ref_cnt = 1; *counter = &sc->counter; counters = g_list_prepend(counters, sc); } @@ -101,10 +127,13 @@ stats_unregister_counter(const gchar *co evt_tag_str("counter", counter_name), NULL); } - - counters = g_list_delete_link(counters, l); - g_free(sc->name); - g_free(sc); + sc->ref_cnt--; + if (sc->ref_cnt == 0) + { + counters = g_list_delete_link(counters, l); + g_free(sc->name); + g_free(sc); + } *counter = NULL; } --- orig/src/stats.h +++ mod/src/stats.h @@ -33,7 +33,7 @@ typedef enum } StatsCounterType; void stats_generate_log(void); -void stats_register_counter(StatsCounterType type, const gchar *counter_name, guint32 **counter); +void stats_register_counter(StatsCounterType type, const gchar *counter_name, guint32 **counter, gboolean shared); void stats_unregister_counter(const gchar *name, guint32 **counter); #endif -- Bazsi