[syslog-ng] duplicate stats counter?
Balazs Scheidler
bazsi at balabit.hu
Fri Jan 20 11:07:22 CET 2006
On Thu, 2006-01-19 at 13:49 -0600, Paul Krizak wrote:
> I've been getting a constant stream of these messages in my syslog-ng
> log, as much as 70MB worth per day:
>
> Jan 19 13:43:39 i-dl385 syslog-ng[25668]: Duplicate stats counter;
> counter='file(/var/log/logs/$R_YEAR/$R_MONTH/$R_DAY/$HOST-messages.log)'
> Jan 19 13:43:40 i-dl385 syslog-ng[25668]: Duplicate stats counter;
> counter='file(/var/log/logs/$R_YEAR/$R_MONTH/$R_DAY/$HOST-messages.log)'
> Jan 19 13:43:40 i-dl385 syslog-ng[25668]: Duplicate stats counter;
> counter='file(/var/log/logs/$R_YEAR/$R_MONTH/$R_DAY/$HOST-messages.log)'
>
> What does this mean? Are stats getting collected properly? Is there a
> bug in the stats stuff?
>
Oops, this seems to be a bug in that the stats counter is registered for
the template filename and not the expanded filename. This is not a real
problem though, as file destinations are not flow controlled, thus they
never drop messages.
It's ugly nevertheless. The patch below adds a way to disable stats
counting for specific destinations and to instruct the stats subsystem
to share a counter.
Don't be afraid of the patch size, it is not as big as it seems :)
It should be available in tomorrow's snapshot.
BTW: what are the experiences with 1.9.8, does it work otherwise fine?
--- orig/src/affile.c
+++ mod/src/affile.c
@@ -367,18 +367,6 @@ affile_dd_set_create_dirs(LogDriver *s,
self->flags &= ~AFFILE_CREATE_DIRS;
}
-static const gchar *
-affile_dd_format_stats_name(AFFileDestDriver *self)
-{
- static gchar stats_name[256];
-
- g_snprintf(stats_name, sizeof(stats_name),
- "file(%s)", self->filename_template->template->str);
-
- return stats_name;
-}
-
-
static time_t reap_now = 0;
static gboolean
@@ -439,7 +427,7 @@ affile_dd_init(LogPipe *s, GlobalConfig
self->time_reap = cfg->time_reap;
self->use_time_recvd = cfg->use_time_recvd;
- log_writer_options_init(&self->writer_options, cfg, FALSE, affile_dd_format_stats_name(self));
+ log_writer_options_init(&self->writer_options, cfg, LWOF_NO_STATS, NULL);
self->cfg = cfg;
if ((self->flags & AFFILE_NO_EXPAND) == 0)
--- orig/src/afprog.c
+++ mod/src/afprog.c
@@ -70,7 +70,7 @@ afprogram_dd_init(LogPipe *s, GlobalConf
int msg_pipe[2];
if (cfg)
- log_writer_options_init(&self->writer_options, cfg, FALSE, afprogram_dd_format_stats_name(self));
+ log_writer_options_init(&self->writer_options, cfg, 0, afprogram_dd_format_stats_name(self));
msg_verbose("Starting destination program",
evt_tag_str("cmdline", self->cmdline->str),
--- orig/src/afsocket.c
+++ mod/src/afsocket.c
@@ -722,7 +722,7 @@ afsocket_dd_init(LogPipe *s, GlobalConfi
if (!self->writer)
{
- log_writer_options_init(&self->writer_options, cfg, !!(self->flags & AFSOCKET_PROTO_RFC3164), afsocket_dd_format_stats_name(self));
+ log_writer_options_init(&self->writer_options, cfg, (self->flags & AFSOCKET_PROTO_RFC3164) ? LWOF_FIXED_STAMP : 0, afsocket_dd_format_stats_name(self));
/* NOTE: we open our writer with no fd, so we can send messages down there
* even while the connection is not established */
--- orig/src/logwriter.c
+++ mod/src/logwriter.c
@@ -377,8 +377,8 @@ log_writer_init(LogPipe *s, GlobalConfig
{
LogWriter *self = (LogWriter *) s;
- if (!self->dropped_messages)
- stats_register_counter(SC_TYPE_DROPPED, self->options->stats_name, &self->dropped_messages);
+ if ((self->options->flags & LWOF_NO_STATS) == 0 && !self->dropped_messages)
+ stats_register_counter(SC_TYPE_DROPPED, self->options->stats_name, &self->dropped_messages, !!(self->options->flags & LWOF_SHARE_STATS));
return TRUE;
}
@@ -477,8 +477,9 @@ log_writer_options_set_template_escape(L
}
void
-log_writer_options_init(LogWriterOptions *options, GlobalConfig *cfg, gboolean fixed_stamp, const gchar *stats_name)
+log_writer_options_init(LogWriterOptions *options, GlobalConfig *cfg, guint32 flags, const gchar *stats_name)
{
+ options->flags = flags;
if (options->fifo_size == -1)
options->fifo_size = cfg->log_fifo_size;
if (options->use_time_recvd == -1)
@@ -498,7 +499,7 @@ log_writer_options_init(LogWriterOptions
options->flush_lines = options->fifo_size - 1;
}
- if (!fixed_stamp)
+ if ((flags & LWOF_FIXED_STAMP) == 0)
{
if (options->keep_timestamp == -1)
options->keep_timestamp = cfg->keep_timestamp;
--- orig/src/logwriter.h
+++ mod/src/logwriter.h
@@ -33,14 +33,23 @@
#define LW_FORMAT_FILE 0x0002
#define LW_FORMAT_PROTO 0x0004
-/* writer options */
+/* writer options (set by the user) */
#define LWO_TMPL_ESCAPE 0x0001
+/* writer flags */
+#define LWOF_FIXED_STAMP 0x0001
+/* we don't want to have a dropped counter for this writer */
+#define LWOF_NO_STATS 0x0002
+/* several writers use the same counter */
+#define LWOF_SHARE_STATS 0x0004
+
typedef struct _LogWriterOptions
{
gchar *stats_name;
- /* bitmask of LWO_* */
+ /* bitmask of LWO_* (set by the user) */
guint32 options;
+ /* bitmask of LWOF_* (set by the driver) */
+ guint32 flags;
/* maximum number of entries */
gint fifo_size;
@@ -78,6 +87,6 @@ gboolean log_writer_reopen(LogPipe *s, F
void log_writer_options_set_template_escape(LogWriterOptions *options, gboolean enable);
void log_writer_options_defaults(LogWriterOptions *options);
-void log_writer_options_init(LogWriterOptions *options, GlobalConfig *cfg, gboolean fixed_stamps, const gchar *stats_name);
+void log_writer_options_init(LogWriterOptions *options, GlobalConfig *cfg, guint32 flags, const gchar *stats_name);
#endif
--- orig/src/stats.c
+++ mod/src/stats.c
@@ -28,6 +28,7 @@
typedef struct _StatsCounter
{
+ guint ref_cnt;
StatsCounterType type;
gchar *name;
guint32 counter;
@@ -47,10 +48,25 @@ stats_find_counter(const gchar *counter_
return g_list_find_custom(counters, counter_name, (GCompareFunc) stats_cmp_name);
}
+/**
+ * stats_register_counter:
+ * @type: counter type
+ * @counter_name: name to identify this stats counter
+ * @counter: returned pointer to the counter
+ * @shared: whether multiple sources will use the same counter
+ *
+ * This fuction registers a general purpose counter. Whenever multiple
+ * objects touch the same counter all of these should register the counter
+ * with the same name, specifying TRUE for the value of permit_dup,
+ * internally the stats subsystem counts the number of users of the same
+ * counter in this case, thus the counter will only be freed when all of
+ * these uses are unregistered.
+ **/
void
-stats_register_counter(StatsCounterType type, const gchar *counter_name, guint32 **counter)
+stats_register_counter(StatsCounterType type, const gchar *counter_name, guint32 **counter, gboolean shared)
{
StatsCounter *sc;
+ GList *l;
/* FIXME: we should use a separate name-space for all different types,
* however we only have a single type so far */
@@ -59,12 +75,21 @@ stats_register_counter(StatsCounterType
*counter = NULL;
if (!counter_name)
return;
- if (stats_find_counter(counter_name))
+ if ((l = stats_find_counter(counter_name)))
{
- msg_notice("Duplicate stats counter",
- evt_tag_str("counter", counter_name),
- NULL);
- *counter = NULL;
+ if (!shared)
+ {
+ msg_notice("Duplicate stats counter",
+ evt_tag_str("counter", counter_name),
+ NULL);
+ *counter = NULL;
+ }
+ else
+ {
+ sc = (StatsCounter *) l->data;
+ sc->ref_cnt++;
+ *counter = &sc->counter;
+ }
return;
}
@@ -73,6 +98,7 @@ stats_register_counter(StatsCounterType
sc->type = type;
sc->name = g_strdup(counter_name);
sc->counter = 0;
+ sc->ref_cnt = 1;
*counter = &sc->counter;
counters = g_list_prepend(counters, sc);
}
@@ -101,10 +127,13 @@ stats_unregister_counter(const gchar *co
evt_tag_str("counter", counter_name),
NULL);
}
-
- counters = g_list_delete_link(counters, l);
- g_free(sc->name);
- g_free(sc);
+ sc->ref_cnt--;
+ if (sc->ref_cnt == 0)
+ {
+ counters = g_list_delete_link(counters, l);
+ g_free(sc->name);
+ g_free(sc);
+ }
*counter = NULL;
}
--- orig/src/stats.h
+++ mod/src/stats.h
@@ -33,7 +33,7 @@ typedef enum
} StatsCounterType;
void stats_generate_log(void);
-void stats_register_counter(StatsCounterType type, const gchar *counter_name, guint32 **counter);
+void stats_register_counter(StatsCounterType type, const gchar *counter_name, guint32 **counter, gboolean shared);
void stats_unregister_counter(const gchar *name, guint32 **counter);
#endif
--
Bazsi
More information about the syslog-ng
mailing list