TkGnats/Gnats general questions

[Robert Lupton the Good]

Leonardo.Araujo writes:
 > 5. Can we create customized reports?

I've been meaning to post this for a while.

Because you can connect to the gnatsd from other programmes, you can
create any report that you feel like in e.g. perl.

Here's my current report generator.  The only local customisation that
I know of is that the -a (active) flag allows locally defined states
such as "needstest".



                                R

#!/usr/bin/perl
# -*- perl -*-
#
# Prepare a summary report for gnats (needs perl5)
#
# Robert Lupton, rhl@princeton.edu
#
require "getopts.pl";

$gnatsd_host = "localhost"; 
$gnatsd_port = 1529;
$one_month = 365.25/12*24*3600; # number of seconds in a month
$PR = -1;                       # desired PR

if(!&Getopts('haAc:C:g:i:l:nop:P:r:s:SUt:Tv')) {
    &syntax;
    exit 1;
}

if($opt_h) {
    &syntax;
    exit 1;
}

if($opt_a) {
    $active = 1;
}
if($opt_A) {
    $show_all = 1;
}
if($opt_c) {
    $sel_category = $opt_c;
}
if($opt_C) {
    $sel_class = $opt_C;
}
if($opt_g) {
    $opt_g =~ /^([^:]*)(:(.*))?$/;
    $gnatsd_host = $1;
    if($3) {
        $gnatsd_port = $3;
    }
}
if($opt_i) {
    $sel_sid = $opt_i;
}
if($opt_l) {
    $linelen = $opt_l;
}
if($opt_n) {
    $output_num = 1;
}
if($opt_o) {
    $open = 1;
}
if($opt_P) {
    $opt_P =~ /^([0-9]+)(:(.*))?$/;
    if(!$1) {
        die "Please specify a numerical PR id, not $opt_P\n";
    }
    $PR = $1;
    if(defined($3)) {
        $PR_regexp = $3;
    }
}
if($opt_p) {
    $sel_priority = $opt_p;
}
if($opt_r) {
    $gnats_db = $opt_r;
}
if($opt_S) {
    $summary = 1;
}
if($opt_s) {
    $sel_severity = $opt_s;
}
if($opt_t) {
    $new_interval = $opt_t;
}
if($opt_T) {
    $summary_by_times = 1;
}
if($opt_U) {
    $summary_by_user = 1;
}
if($opt_v) {
    $verbose = 1;
}

if($new_interval) {
    $summary_by_times = 1;
} else {
    $new_interval = 1.0;        # Number of months for which PRs are `new'
}

if(!$summary && !$summary_by_user && !$summary_by_times) {
    $print = 1;
}
if($summary + $summary_by_user + $summary_by_times > 1) {
    die "You mayn't specify more than one of -S, -T (or -t), and -U\n";
}

if($summary_by_times) { 
   eval "use Time::Local";
   if($EVAL_ERROR) {
       die "You cannot use -T unless module Time::Local is available\n";
   }

   if($active) {
       warn "It doesn't make sense to use -a with -T; ignoring\n";
       $active = 0;
   }
}

use Socket;
use IO::Handle;

$iaddr = inet_aton($gnatsd_host) || die "No such host: $gnatsd_host\n";
$paddr = sockaddr_in($gnatsd_port, $iaddr);
socket(GNATSD, PF_INET, SOCK_STREAM, getprotobyname("tcp")) ||
    die "Opening gnatsd port: $!\n";
connect(GNATSD, $paddr) || die "Connecting to gnatsd: $!\n";
GNATSD->autoflush(1);

$_ = <GNATSD>;
if($verbose) {
    warn "Reply: $_";
}

print GNATSD "CHDB $gnats_db\n";

$_ = <GNATSD>;
if($verbose) {
    warn "Reply: $_";
}

if(/^480/) {
    die "Invalid database name: $gnats_db\n";
}

if($PR >= 0) {                  # simply return a PR
    if(defined($PR_regexp)) {
        print GNATSD "FULL $PR\n";
    } else {
        $PR_regexp = "";
        print GNATSD "QURY $PR\n";
    }

    $_ = <GNATSD>;
    if($verbose) {
        warn "Reply: $_";
    }
    
    if(!/^220/) {
        die "Invalid PR: $PR\n";
    }

    while(<GNATSD>) {
        s/[\n\r]*$//;

        if(/^\.$/) {
            last;
        } elsif($PR_regexp) {
            if(!/$PR_regexp/io) {
                next;
            }
            if($PR_regexp !~ /\|/) {
                s/^>\S+:\s*//;
            }
        }

        print "$_\n";
    }

    print GNATSD "QUIT\n";
    close(GNATSD);
    exit(0);
}

print GNATSD "LCAT\n";

$_ = <GNATSD>;
if($verbose) {
    warn "Reply: $_";
}

while(<GNATSD>) {
    chop; tr/\r//d;

    if(/^\.$/) {
        last;
    } elsif(/^\s*#/) { 
        next;
    }
    ($category) = split(":", $_);
    ($category) = unpack("A8", $category);
    $categories{$category}++;
}
#
# We need to map severity etc. from ints to strings
#
%severities = ( "1", "critical",  "2", "serious",  "3", "non-critical");
%priorities = ( "1", "high",      "2", "medium",   "3", "low");

foreach $arr ("classes", "states", "submitters") {
    if($arr eq "classes") {
        print GNATSD "LCLA\n";
    } elsif($arr eq "states") {
        print GNATSD "LSTA\n";
    } elsif($arr eq "submitters") {
        print GNATSD "LSUB\n";
    } else {
        die "Unknown field to list: $arr\n";
    }
   
    $_ = <GNATSD>;
    if($verbose) {
        warn "Reply: $_";
    }
    
    $i = 1;
    while(<GNATSD>) {
        chop; tr/\r//d;

        if(/^\.$/) {
            last;
        } elsif(/^\s*($|#)/) {
            next;
        }
        ($name) = split(/:/);
        if($verbose) {
            warn "\$${arr}{$i} = $name\n";
        }
        eval "\$${arr}{$i} = \"$name\"";
        $i++;
    }
}
#
# Find desired PRs
#
$gnatsd_sep = "!";
$opts = "";

if($active) {
    if($open) {
        warn "You cannot specify both -a[ctive] and -o[pen]; ignoring -o\n";
    }

    $states = "open|assigned|analyzed|feedback|needstest";
} elsif($open) {
    $states = "open";
}
if($states ne "") {
    $opts .= "STAT $states"; $opts .= $gnatsd_sep;
}

if($sel_category) {
    foreach (split("\\|", $sel_category)) {
        if(!defined($categories{$_})) {
            die "$_ is not a valid category\n";
        }
    }
    $opts .= "CATG $sel_category"; $opts .= $gnatsd_sep;
}
if($sel_class) {
    foreach $c (split("\\|", $sel_class)) {
        if(!grep(/^$c$/, values %classes)) {
            die "$c is not a valid class: " .
                join(" ", values %classes) . "\n";
        }
    }
    $opts .= "CLSS $sel_class"; $opts .= $gnatsd_sep;
}
if($sel_sid) {
    foreach $c (split("\\|", $sel_sid)) {
        if(!grep(/^$c$/, values %submitters)) {
            die "$c is not a valid submitter-id: " .
                join(" ", values %submitters) . "\n";
        }
    }
    $opts .= "SUBM $sel_sid"; $opts .= $gnatsd_sep;
}
if($sel_priority) {
    $opts .= "PRIO $sel_priority"; $opts .= $gnatsd_sep;
}
if($sel_severity) {
    $opts .= "SVTY $sel_severity"; $opts .= $gnatsd_sep;
}

if($verbose) {
    warn "query-pr $opts\n";
}

$opts .= "SQL2"; $opts .= $gnatsd_sep;
foreach (split($gnatsd_sep, $opts)) {
    if($verbose) {
        warn "GNATSD $_\n";
    }
    print GNATSD "$_\n";
    
    $_ = <GNATSD>;
    if($verbose) {
        warn "Reply: $_";
    }
    
    if(/^440 /) {
        $no_prs = 1;
    }
}

while(!$no_prs && ($_ = <GNATSD>)) {
    chop; tr/\r//d;

    if(/^\.$/) {
        last;
    }

    ($num, $cat, $synopsis, $conf, $severity, $priority,
     $resp, $state, $class, $submitter, $arrival_date, $orig,
     $release, $last_modified, $closed_date) = split(/\s*\|/, $_);
    
    $severity = $severities{$severity};
    $priority = $priorities{$priority};
    $state = $states{$state};
    $class = $classes{$class};

    if(0) {                     # RHL
        $nmax++;
        if($nmax < 0) { 
            next;
        } elsif($nmax > 3) {
            last; 
        }       
        print "RHL$_\n";
    }

    push(@PR, "$num|$cat|$resp|$state|$severity|$priority|$synopsis" .
         "|$arrival_date|$last_modified|$closed_date");
    #
    # Prepare summary information
    #
    $cats{$cat}++;
    $summary{$cat,$state}++;

    if($summary_by_times) {
        $time = time();
        $sum_age{$cat,$state} += $time - &time_in_sec($arrival_date);
        
        if($time - &time_in_sec($arrival_date) < $new_interval*$one_month) {
            $summary{$cat,opened_new}++;
        }
        
        if($state =~ /closed|suspended|mistaken|duplicate/ &&
           $closed_date eq "") {
            $closed_date = "1900-01-01 00:00:00"; # old gnats didn't have
                                                  # closed_date
        }

        if($closed_date ne "") {
            if($time - &time_in_sec($closed_date) < $new_interval*$one_month){
                $summary{$cat,closed_new}++;
            }
        }
    }
}

print GNATSD "QUIT\n";
close(GNATSD);

undef %severities; undef %priorities; undef %states; undef %classes;
#
# Output whatever is desired
#
($wday, $mon, $day, $time, $tz, $year) = split(" ", `date`);

if($summary) {
    $title = sprintf("Summary of %sPRs by State%s",
                     ($active ? "Active " : ""),
                     ($summary == 1 ? " and Category" : ""));
    printf("%-68s$day $mon $year\n",$title);
    &print_selection_criteria;
    print "\n";
    #
    # make column headings
    #
    $fmt = "%-15s  %-6s";
    $hdr1 = sprintf($fmt, "Category", "Open");
    $hdr2 = sprintf($fmt, "--------", "----");
    $states{"open"}++;

    if(!$open) {
        $dfmt = "%-6s %-6s %-6s %-6s";
        $fmt .= $dfmt;
        $hdr1 .= sprintf($dfmt, "Assign", "Analyz", "Needts", "Feedbk");
        $hdr2 .= sprintf($dfmt, "------", "------", "------", "------");
        $states{assigned}++; $states{analyzed}++;
        $states{needstest}++;$states{feedback}++;
    }
    
    if(!$active) {
        $dfmt = " %-6s %-6s %-6s %-6s";
        $fmt .= $dfmt;
        $hdr1 .= sprintf($dfmt, "Closed", "Suspnd", "duplct", "mistak");
        $hdr2 .= sprintf($dfmt, "------", "------", "------", "------");
        $states{closed}++; $states{suspended}++;
        $states{duplicate}++; $states{mistaken}++;
    }
    $fmt .= "\n";
    print "$hdr1\n"; print "$hdr2\n";
    #
    # Print the summary
    #
    foreach $c (sort(keys %cats), "total") {
        if($c eq "total") {
            print "\n";
            foreach $s (keys %states) {
                $npr{$s} = $total{$s};

                if(!$npr{$s}) { $npr{$s} = "_"; }
            }
        } else {
            foreach $s (keys %states) {
                $npr{$s} = $summary{$c,$s};
                $total{$s} += $npr{$s};
                if(!$npr{$s}) { $npr{$s} = "_"; }
            }
        }
            
        printf($fmt, pack("A15", $c), $npr{"open"},
               $npr{assigned}, $npr{analyzed}, $npr{needstest}, $npr{feedback},
               $npr{closed}, $npr{suspended}, $npr{duplicate}, $npr{mistaken});

    }
}
if($summary_by_user) {
    @PR = sort by_responsible @PR;

    $fmt = "%-15s  %-6s";
    $hdr1 = sprintf($fmt, "User", "Open");
    $hdr2 = sprintf($fmt, "----", "----");
    $states{"open"}++;

    if(!$open) {
        $dfmt = "%-6s %-6s %-6s %-6s";
        $fmt .= $dfmt;
        $hdr1 .= sprintf($dfmt, "Assign", "Analyz", "Needts", "Feedbk");
        $hdr2 .= sprintf($dfmt, "------", "------", "------", "------");
        $states{assigned}++; $states{analyzed}++; 
        $states{needstest}++;$states{feedback}++;
    }
    
    if(!$active) {
        $dfmt = " %-6s %-6s %-6s %-6s";
        $fmt .= $dfmt;
        $hdr1 .= sprintf($dfmt, "Closed", "Suspnd", "duplct", "mistak");
        $hdr2 .= sprintf($dfmt, "------", "------", "------", "------");
        $states{closed}++; $states{suspended}++; 
        $states{duplicate}++; $states{mistaken}++;
    }
    $fmt .= "\n";

    if($active) {
        $type = "active ";
    } elsif($open) {
        $type = "open ";
    } else {
        $type = "";
    }

    printf "%-68s $day $mon $year\n",
    "The following ${type}Problem Reports (PRs) are currently in GNATS:";

    &print_selection_criteria;
    print "\n";
        
    print "$hdr1\n"; print "$hdr2\n";

    $old_resp = "";
    undef %total; undef %npr;
    foreach (@PR, "total") {
        if($_ eq "total") {
            &print_summary_row;

            print "\n";
            foreach $s (keys %states) {
                $npr{$s} = $total{$s};
            }
            $old_resp = $_;
        } else {
            ($num, $cat, $resp, $state) = split(/\|/);
        }

        if($resp ne $old_resp) {
            &print_summary_row;
        }

        $npr{$state}++;
        $total{$state}++;
    }
}

if($summary_by_times) {
    $title = "Summary of PRs by Category and History",
    printf("%-68s$day $mon $year\n",$title);
    &print_selection_criteria;

    print "\n";
    print "Opened/Closed refer to PRs changing state within $new_interval " .
        "month" . ($new_interval == 1.0 ? "" : "s") . "\n";
    
    print "Numbers in (parentheses) are the mean age of the PR, in months\n";
    print "\n";
    #
    # make column headings
    #
    $fmt = "%-15s  %-8s %-8s    %-12s %-12s %-12s";
    $hdr1 = sprintf($fmt, 
                    "Category", "Opened", "Closed", "Open", "Accepted", 
"Testing");
    $hdr2 = sprintf($fmt, 
                    "--------", "------", "------", "----", "--------", 
"-------");
    $fmt .= "\n";
    print "$hdr1\n"; print "$hdr2\n";
    #
    # Generate summaries
    #
    $states{opened_new}++; $states{closed_new}++;

    $states{"open"}++; 
    $states{assigned}++; $states{analyzed}++; 
    $states{needstest}++;$states{feedback}++;
    $states{closed}++; $states{suspended}++; 
    $states{duplicate}++; $states{mistaken}++;
    #
    # Print the summary
    #
    foreach $c (sort(keys %cats), "total") {
        if($c eq "total") {
            print "\n";
            foreach $s (keys %states) {
                $npr{$s} = $total{$s};
                $age{$s} = $total_age{$s};
            }
        } else {
            foreach $s (keys %states) {
                $npr{$s} = $summary{$c,$s};
                $age{$s} = $sum_age{$c,$s};
                $total{$s} += $npr{$s};
                $total_age{$s} += $age{$s};
            }
        }
        $npr{active} = $npr{assigned} + $npr{analyzed};
        $age{active} = $age{assigned} + $age{analyzed};
        $npr{testing} = $npr{needstest} + $npr{feedback};
        $age{testing} = $age{needstest} + $age{feedback};

        foreach $s ("open", "active", "testing") {
            if($npr{$s} > 0) {
                $mean = $age{$s}/$npr{$s};
                $mean /= $one_month; # convert to months
                $npr{$s} =
                    sprintf("%-2d %6s", $npr{$s}, sprintf("(%.1f)", $mean))
            }
        }

        $show_cat = 0;          # print a line for this category?
        if($show_all) {
            if($c !~ /^test/) {
                $show_cat++;
            }
        } else {
            foreach $s (opened_new, closed_new, "open", active, testing) {
                if($npr{$s}) {
                    $show_cat++;
                    last;
                }
            }
        }

        foreach $s (keys %npr) {
            if(!$npr{$s}) {
                $npr{$s} = "_"; 
            }
        }

        if($show_cat) {
            printf($fmt, pack("A15", $c),
                   $npr{opened_new}, $npr{closed_new},
                   $npr{"open"}, $npr{active}, $npr{testing});
        }
    }
}

if($print) {
    @PR = sort suitably @PR;

    if($output_num) {
        $numstr = sprintf("%-4s ", "PR#");
    } else {
        $numstr = "";
    }

    $fmt = "%-9s%-9s%-10s%-10s%-9s%-11s%s\n";

    &print_selection_criteria;
    printf($numstr . $fmt,
           "category", "Respon.", "State", "Severity", 
           "Priority", "Synopsis");
    if($output_num) {
        $numstr = sprintf("%-4s ", "----");
    }
    printf($numstr . $fmt,
           "--------", "-------", "-----", "--------",
           "--------", "--------");

    foreach (@PR) {
        ($num, $cat, $respon, $state, $severity, $priority, $synop) = 
            split(/\|/, $_);
        $line = sprintf($fmt, pack("A8", $cat), pack("A8", $respon), 
                        pack("A8", $state), pack("A8", $severity),
                        $priority, $synop);
        
        if($output_num) {
            $line = sprintf("%-4s ", $num) . $line;
        }
        
        if($linelen && length($line) > $linelen) {
            $line = pack("A$linelen", $line);
            $line =~ s/.$/\$/;
            $line .= "\n";
        }
        print "$line";
    }
}

###############################################################################
#
# Print the selection criteria
#
sub print_selection_criteria
{
    if($sel_category || $sel_class || $sel_priority ||
       $sel_severity || $sel_sid) {
        printf "Selected by: " .
            join(" ", ($sel_category, $sel_priority, $sel_severity,
                       $sel_class, $sel_sid)) . "\n";
    }
}

###############################################################################
#
# Sort a PR by appropriate fields
#
sub suitably
{
    local($num_a, $cat_a,
          $resp_a, $state_a, $severity_a, $priority_a) = split(/\|/, $a);
    local($num_b, $cat_b,
          $resp_b, $state_b, $severity_b, $priority_b) = split(/\|/, $b);
    %rank = (
             "critical",  0,
             "serious",   1,
             "non-criti", 2,

             "high",      0,
             "medium",    1,
             "low",       2,

             "open",      0,
             "assigned",  1,
             "analyzed",  2,
             "needstest", 3,
             "feedback",  4,
             "suspended", 5,
             "closed",    6,
             'duplicate', 7,
             'mistaken',  8,
             );

    if(0) {
        print "\$rank{$severity_a} <=> \$rank{$severity_b} ";
        print "$rank{$severity_a} <=> $rank{$severity_b}\n";
    }

    $cmp = $rank{$severity_a} <=> $rank{$severity_b};
    if($cmp) {
        return($cmp);
    }

    $cmp = $rank{$priority_a} <=> $rank{$priority_b};
    if($cmp) {
        return($cmp);
    }

    $cmp = $rank{$state_a} <=> $rank{$state_b};
    if($cmp) {
        return($cmp);
    }

    $cmp = $cat_a cmp $cat_b;
    if($cmp) {
        return($cmp);
    }

    $cmp = $resp_a cmp $resp_b;
    if($cmp) {
        return($cmp);
    }

    return $cmp;
}

###############################################################################
#
# Print a summary row
#
sub print_summary_row
{
    if($old_resp ne "") {
        foreach $s (keys %states) {
            if(!$npr{$s}) { $npr{$s} = "_"; }
        }
        
        printf($fmt, pack("A15", "$old_resp"), $npr{"open"},
               $npr{assigned}, $npr{analyzed}, $npr{needstest},
               $npr{feedback}, $npr{closed}, $npr{suspended},
               $npr{duplicate}, $npr{mistaken});
    }
    
    undef %npr;
    $old_resp = $resp;
}

###############################################################################
#
# Sort PRs by responsible
#
sub by_responsible
{
    local($num_a, $cat_a, $resp_a) = split(/\|/, $a);
    local($num_b, $cat_b, $resp_b) = split(/\|/, $b);
    
    return $resp_a cmp $resp_b;
}

###############################################################################
#
# Covert a formatted time from gnatsd back into Unix time
sub
time_in_sec
{
    local($gnats_time) = @_;
    local($date, $time) = split(" ", $gnats_time);

    local($year, $mon, $mday) = split("-", $date);
    $year -= 1900;
    $mon -= 1;
    local($hour, $min, $sec) = split(":", $time);

    return(&timegm($sec, $min, $hour, $mday, $mon, $year));
}

###############################################################################

sub syntax
{
   print <<"EOT";
Generate a report, extracting information by connecting to gnatsd
Usage:
    make-report [options]
Options:
    -h          Print this message
    -a          Only report active PRs 
                        (open|assigned|analyzed|feedback|needstest)
    -A          Report summaries for all categories in the database
    -c list     Only process PRs with category <list> (e.g. "psp|frames")
    -C list     Only process PRs with class <list> (e.g. "sw-bug|mistaken")
    -g host[:port] Connect to gnatsd at this host (default: localhost)
                and port (default 1529)
    -i list     Only process PRs with subitter-id <list> (e.g. 
"allowed|longTerm")
    -l len      Maximum length of output line
    -n          Include PR numbers in reports
    -o          Only report open PRs
    -p list     Only process PRs with priority <list> (e.g. "high|medium")
    -P #[:expr] Return the text of PR #.  If expr is specified, it's a
                regular expression that must match lines that'll be printed;
                if it contains no |s, the field name will be removed
    -r dir      Gnats root or a database name; by default root is "/u/gnats"
                (Can be used with -g to specify a database name)
    -s list     Only process PRs with severity <list> (e.g. "critical|serious")
    -S          Print a summary of states for each category
    -t nmonth   Interval for which PRs are `new'; in months. Implies -T
    -T          Breakdown states by times, and how many PRs have opened/closed
    -U          Print a summary of states for each responsible user
    -v          Be chatty

You may not use -a with -T or -t as you'll miss PRs that were opened
and closed during the interval you're analysing.

E.g.
    make-report -s critical -a -r SDSS -U
    make-report -s critical -r SDSS -T -t 3
    make-report -r SDSS -l 80 -n -a -c "frames|psp" -s "critical"
    make-report -r SDSS -P 2013:Synopsis
    make-report -r SDSS -P "2013:Category|Synopsis"
    
EOT
}