[syslog-ng] [PATCH] [pdbtool] debug-id option

[Balazs Scheidler]

Hi,

The patch has been line-breaked by your mailer and has become corrupt.
Can you repost without linebreaks?

Otherwise it looks ok, and I'd like to apply it.

Thanks.

On Wed, 2011-08-17 at 15:38 +0200, Balint Kovacs wrote:
> Hi,
> 
> yes, that'd be much more useful indeed, also because pdbtool test sets 
> the return code on failures, so it's better for using it in scripts. I 
> would avoid using -p as an option tough, as it was used in earlier 
> versions of pdbtool test for specifying the patterndb xml to be tested, 
> so I used -r. The debug info is only printed for non-matching patterns - 
> without giving it much thought, this seemed appropriate, but can easily 
> be changed.
> 
> Balint
> 
> 
> commit 1a1ccc8db55b502ea6f4c363a710fe442a3a228d
> Author: Balint Kovacs <blint at balabit.hu>
> Date:   Wed Aug 17 15:00:42 2011 +0200
> 
>      [pdbtool] rule-id, debug and color-out options for pdbtool test
> 
>      Added options to test a specific rule against its example message
>      with debug and colorizing.
> 
>      Signed-off-by: Balint Kovacs <blint at balabit.hu>
> 
> diff --git a/modules/dbparser/pdbtool.c b/modules/dbparser/pdbtool.c
> index 64ccd17..1c5d818 100644
> --- a/modules/dbparser/pdbtool.c
> +++ b/modules/dbparser/pdbtool.c
> @@ -641,6 +641,7 @@ static GOptionEntry match_options[] =
>   };
> 
>   static gboolean test_validate = FALSE;
> +static gchar *test_ruleid = NULL;
> 
>   static gboolean
>   pdbtool_test_value(LogMessage *msg, const gchar *name, const gchar 
> *test_value)
> @@ -676,6 +677,7 @@ pdbtool_test(int argc, char *argv[])
>     gboolean failed_to_load = FALSE;
>     gboolean failed_to_match = FALSE;
>     gboolean failed_to_validate = FALSE;
> +  gboolean failed_to_find_id = TRUE;
> 
>     for (arg_pos = 1; arg_pos < argc; arg_pos++)
>       {
> @@ -711,6 +713,18 @@ pdbtool_test(int argc, char *argv[])
> 
>             if (example->message && example->program)
>               {
> +
> +              if (test_ruleid)
> +                {
> +                  if (strcmp(example->rule->rule_id, test_ruleid) != 0)
> +                    {
> +                      examples = g_list_delete_link(examples, examples);
> +                      continue;
> +                    }
> +                  else
> +                    failed_to_find_id = FALSE;
> +                }
> +
>                 msg = log_msg_new_empty();
>                 log_msg_set_value(msg, LM_V_MESSAGE, example->message, 
> strlen(example->message));
>                 if (example->program && example->program[0])
> @@ -719,7 +733,13 @@ pdbtool_test(int argc, char *argv[])
>                 printf("Testing message program='%s' message='%s'\n", 
> example->program, example->message);
>                 pattern_db_process(patterndb, msg);
> 
> -              pdbtool_test_value(msg, ".classifier.rule_id", 
> example->rule->rule_id);
> +              if (!pdbtool_test_value(msg, ".classifier.rule_id", 
> example->rule->rule_id) && debug_pattern)
> +                {
> +                  match_message = example->message;
> +                  match_program = example->program;
> +                  patterndb_file = argv[arg_pos];
> +                  pdbtool_match(0, NULL);
> +                }
> 
>                 for (i = 0; example->values && i < example->values->len; 
> i++)
>                   {
> @@ -739,6 +759,11 @@ pdbtool_test(int argc, char *argv[])
>       return 1;
>     if (failed_to_match)
>       return 2;
> +  if (failed_to_find_id)
> +    {
> +      printf("Could not find the specified ID, or the defined rule 
> doesn't have an example message.\n");
> +      return 3;
> +    }
>     return 0;
>   }
> 
> @@ -746,6 +771,12 @@ static GOptionEntry test_options[] =
>   {
>     { "validate", 0, 0, G_OPTION_ARG_NONE, &test_validate,
>       "Validate the pdb file against the xsd (requires xmllint from 
> libxml2)", NULL },
> +  { "rule-id", 'r', 0, G_OPTION_ARG_STRING, &test_ruleid,
> +    "Rule ID of the patterndb rule to be tested against its example", 
> NULL },
> +  { "debug", 'D', 0, G_OPTION_ARG_NONE, &debug_pattern,
> +    "Print debuging information on non-matching patterns", NULL },
> +  { "color-out", 'c', 0, G_OPTION_ARG_NONE, &color_out,
> +    "Color terminal output", NULL },
>     { NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL }
>   };
> 
> 
> 
> 
> On 08/17/2011 09:24 AM, Balazs Scheidler wrote:
> > Hi,
> >
> > Looks nice, however it'd probably make more sense to do this directly in
> > pdbtool test, wouldn't it?
> >
> > e.g.
> >
> > $ pdbtool test -p<id>  --debug --color-out
> >
> > On Mon, 2011-08-15 at 11:35 +0200, Balint Kovacs wrote:
> >> Hi,
> >>
> >> While working with `pdbtool test`, I found that it's quite uncomfortable
> >> to find problems with non-matching messages, as it only displays the ID
> >> of the erroneous pattern. Right now you would need to copy the example
> >> message and program name, and pass it to `pdbtool match` as arguments to
> >> find out what's the exact issue and get a nice colorized output pointing
> >> to the problematic part of the pattern.
> >>
> >> To make that easier, I've done a small enhancement to pdbtool, an option
> >> to do a full, colorized debug output on a given rule by only supplying
> >> its ID. This would look something like (coloring lost in email):
> >>
> >> blint at lyra:~/blah/syslog-ng-ose-mainline-3.4$ ./bin/pdbtool debug-id -p
> >> /var/tmp/patterndb/system-bind.xml -r "b57a384f-c8be-41e9-bc10-735695dc63e7"
> >> Pattern matching part:
> >> unexpected RCODE (REFUSED) resolving
> >> @QSTRING:.dict.arpa=hushmail.com/AAAA/IN@:@QSTRING:.dict.src=203.197.12.30 at deliberately
> >> freaked up test message 53
> >> Matching part:
> >> unexpected RCODE (REFUSED) resolving 'hushmail.com/AAAA/IN': 203.197.12.30#
> >> Values:
> >> MESSAGE=unexpected RCODE (REFUSED) resolving 'hushmail.com/AAAA/IN':
> >> 203.197.12.30#deliberately freaked up test message 53
> >> PROGRAM=named
> >> .classifier.class=unknown
> >>
> >> The patch is against 3.4, but should apply to 3.3 as well.
> >>
> >> Balint
> >>
> >> commit 93ca04700f8706643fedea51936af02daa314766
> >> Author: Balint Kovacs<blint at balabit.hu>
> >> Date:   Mon Aug 15 11:07:50 2011 +0200
> >>
> >>       [pdbtool] Implemented debug-id option
> >>
> >>       Added a new option to pdbtool to test a specific rule against its
> >>       example message with colorizing debug turned on.
> >>
> >>       Signed-off-by: Balint Kovacs<blint at balabit.hu>
> >>
> >> diff --git a/modules/dbparser/pdbtool.c b/modules/dbparser/pdbtool.c
> >> index 64ccd17..f64012d 100644
> >> --- a/modules/dbparser/pdbtool.c
> >> +++ b/modules/dbparser/pdbtool.c
> >> @@ -640,6 +640,75 @@ static GOptionEntry match_options[] =
> >>      { NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL }
> >>    };
> >>
> >> +static gchar *rule_id = NULL;
> >> +
> >> +static gboolean
> >> +pdbtool_debug_id(int argc, char *argv[])
> >> +{
> >> +  PatternDB *patterndb;
> >> +  PDBExample *example;
> >> +  GList *examples = NULL;
> >> +  gboolean id_is_found = FALSE;
> >> +
> >> +  debug_pattern = TRUE;
> >> +  debug_pattern_parse = FALSE;
> >> +  color_out = TRUE;
> >> +  colors = full_colors;
> >> +
> >> +  if (!rule_id)
> >> +    {
> >> +      printf("Please specify a rule ID to be tested against its example message!\n");
> >> +      return FALSE;
> >> +    }
> >> +
> >> +  patterndb = pattern_db_new();
> >> +  if (!pdb_rule_set_load(patterndb->ruleset, configuration, patterndb_file,&examples))
> >> +    {
> >> +      pattern_db_free(patterndb);
> >> +      return FALSE;
> >> +    }
> >> +
> >> +    while (examples)
> >> +      {
> >> +        example = examples->data;
> >> +
> >> +        if (strcmp(example->rule->rule_id, rule_id) != 0)
> >> +          {
> >> +            examples = g_list_delete_link(examples, examples);
> >> +            continue;
> >> +          }
> >> +
> >> +        id_is_found = TRUE;
> >> +
> >> +        if (example->message&&   example->program)
> >> +          {
> >> +            match_message = example->message;
> >> +            match_program = example->   program;
> >> +            pdbtool_match(argc, argv);
> >> +          }
> >> +        examples = g_list_delete_link(examples, examples);
> >> +      }
> >> +
> >> +  pattern_db_free(patterndb);
> >> +
> >> +  if (!id_is_found)
> >> +    {
> >> +      printf("Could not find a corresponding ID in the patterndb file or the rule does not have an example message.\n");
> >> +      return FALSE;
> >> +    }
> >> +
> >> +  return TRUE;
> >> +}
> >> +
> >> +static GOptionEntry debug_id_options[] =
> >> +{
> >> +  { "pdb",       'p', 0, G_OPTION_ARG_STRING,&patterndb_file,
> >> +    "Name of the patterndb file", "<patterndb_file>" },
> >> +  { "ruleid",    'r', 0, G_OPTION_ARG_STRING,&rule_id,
> >> +    "ID of the patterndb rule to debug", "<rule_id>" },
> >> +  { NULL, 0, 0, G_OPTION_ARG_NONE, NULL, NULL }
> >> +};
> >> +
> >>    static gboolean test_validate = FALSE;
> >>
> >>    static gboolean
> >> @@ -934,6 +1003,7 @@ static struct
> >>      { "dump", dump_options, "Dump pattern datebase tree", pdbtool_dump },
> >>      { "merge", merge_options, "Merge pattern databases", pdbtool_merge },
> >>      { "test", test_options, "Test pattern databases", pdbtool_test },
> >> +  { "debug-id", debug_id_options, "Test pattern databases", pdbtool_debug_id },
> >>      { "patternize", patternize_options, "Create a pattern database from logs", pdbtool_patternize },
> >>      { NULL, NULL },
> >>    };
> >>
> >>
-- 
Bazsi