freeswan/testing/utils/regress-summarize-results.pl

   1 #!/usr/bin/perl
   2
   3 #
   4 # this program processes a bunch of directories routed at
   5 # $REGRESSRESULTS. Each one is examined for a file "status"
   6 # the result is an HTML table with the directory name as
   7 # left columns (it is the implied test name), and the status
   8 # on the right.
   9 #
  10 # if the test status is negative, then the results are a hotlink
  11 # to that directory's output.
  12 #
  13 # The test names are links to the file "description.txt" if it
  14 # exists.
  15 #
  16
  17 require 'ctime.pl';
  18
  19 # colours are RRGGBB
  20 $failedcolour="#990000";
  21 $succeedcolour="#009900";
  22 $missingcolour="#000099";
  23
  24 $failed=0;
  25 $passed=0;
  26 $missed=0;
  27 $total=0;
  28
  29 @faillist=();
  30
  31 sub htmlize_test {
  32   local($testname)=@_;
  33
  34   if(-f "$testname/description.txt") {
  35     print HTMLFILE "<TR><TD><A HREF=\"$testname/description.txt\">$testname</A></TD>\n";
  36   } else {
  37     print HTMLFILE "<TR><TD>$testname</TD>\n";
  38   }
  39
  40   if(open(STATUS,"$testname/status")) {
  41     $total++;
  42     chop($result=<STATUS>);
  43     if($result =~ /(Yes|True|1|Succeed|Passed)/i) {
  44       $result=1;
  45       $link="<FONT COLOR=\"$succeedcolour\">passed</FONT>";
  46       $passed++;
  47     } else {
  48       $result=0;
  49       $link="<FONT COLOR=\"$failedcolour\">FAILED</FONT>";
  50       if(-d "$testname/OUTPUT") {
  51         $output="$testname/OUTPUT";
  52         $link="<A HREF=\"$output\">$link</A>";
  53       }
  54       push(@faillist, $testname);
  55       $failed++;
  56     }
  57     close(STATUS);
  58   } else {
  59     $link="<FONT COLOR=\"$missingcolour\">missing</FONT>";
  60     if(-d "$testname/OUTPUT") {
  61       $output="$testname/OUTPUT";
  62       $link="<A HREF=\"$output\">$link</A>";
  63     }
  64     $missed++;
  65   }
  66
  67   print HTMLFILE "<TD>$link</TD>";
  68
  69   if(-f "$testname/regress.txt") {
  70     open(PROBREPORT, "$testname/regress.txt") || die "$testname/regress.txt: $!\n";
  71     chop($prnum=<PROBREPORT>);
  72     close(PROBREPORT);
  73     print "<TD><A HREF=\"http://gnats.freeswan.org/bugs/gnatsweb.pl?database=freeswan&cmd=view+audit-trail&pr=$prnum\">PR#$prnum</A></TD>";
  74
  75   } elsif(-f "$testname/goal.txt") {
  76     open(GOALREQ, "$testname/goal.txt") || die "$testname/regress.txt: $!\n";
  77     chop($goalnum=<GOALREQ>);
  78     close(GOALREQ);
  79
  80     $goalnum=sprintf("%03d", $goalnum);
  81     print "<TD><A HREF=\"http://www.freeswan.org/freeswan_snaps/CURRENT-SNAP/klips/doc/klipsNGreq/requirements/$goalnum\">Requirement $goalnum</A></TD>";
  82
  83   } elsif(-f "$testname/exploit.txt") {
  84     open(EXPLOIT, "$testname/exploit.txt") || die "$testname/exploit.txt: $!\n";
  85     chop($url=<EXPLOIT>);
  86     close(EXPLOIT);
  87
  88   } else {
  89     # test not categorized, output nothing.
  90   }
  91
  92   print HTMLFILE "</TR>\n";
  93 }
  94
  95 # the test names are sorted.
  96
  97 $REGRESSRESULTS=$ENV{'REGRESSRESULTS'};
  98
  99 if(defined($ARGV[0])) {
 100   $REGRESSRESULTS=$ARGV[0];
 101 }
 102
 103 if( ! -d $REGRESSRESULTS ) {
 104   die "No such directory $REGRESSRESULTS.";
 105 }
 106
 107 chdir($REGRESSRESULTS);
 108
 109 opendir(TESTS,".") || die "opendir $REGRESSRESULTS: $!\n";
 110 @tests=readdir(TESTS);
 111 closedir(TESTS);
 112
 113 @testnames=sort @tests;
 114
 115 #
 116 # make pass through the tests, categorizing them.
 117 #
 118 @regresstests=();
 119 @goaltests=();
 120 @exploittests=();
 121 foreach $testname (@testnames) {
 122   if(-f "$testname/regress.txt") {
 123     push(@regresstests,$testname);
 124   } elsif(-f "$testname/goal.txt") {
 125     push(@goaltests, $testname);
 126   } elsif(-f "$testname/exploit.txt") {
 127     push(@exploittests, $testname);
 128   } else {
 129     push(@regresstests,$testname);
 130   }
 131 }
 132
 133 if(open(DATE, "datestamp")) {
 134   chop($timestamp=<DATE>);
 135   close(DATE);
 136   $runtime=&ctime($timestamp);
 137 } else {
 138   $runtime="an unknown time";
 139 }
 140 $hostname=`uname -n`;
 141
 142 open(HTMLFILE, ">testresults.html") || die "Can not open testresults.html: $!\n";
 143
 144 print HTMLFILE "<!DOCTYPE HTML PUBLIC \"-//W3C//DTD HTML 3.2 Final//EN\">\n";
 145 print HTMLFILE "<HTML>  <HEAD>\n";
 146 print HTMLFILE "<TITLE>FreeSWAN nightly testing results for $runtime</TITLE>\n";
 147 print HTMLFILE "</HEAD>  <BODY>\n";
 148 print HTMLFILE "<H1>FreeSWAN nightly testing results for $runtime on $hostname</H1>\n";
 149 print HTMLFILE "<TABLE>\n";
 150
 151 print HTMLFILE "<TR COLSPAN=3>Regression tests</TR>\n";
 152 print HTMLFILE "<TR><TH>Test name</TH><TH>Result</TH><TH>Detail</TH></TR>\n";
 153
 154 foreach $testname (@regresstests) {
 155   next if($testname =~ /^\./);
 156   next unless(-d $testname);
 157
 158   &htmlize_test($testname);
 159 }
 160
 161 print HTMLFILE "<TR COLSPAN=3>Goal tests</TR>\n";
 162 print HTMLFILE "<TR><TH>Test name</TH><TH>Result</TH><TH>Detail</TH></TR>\n";
 163
 164 foreach $testname (@goaltests) {
 165   next if($testname =~ /^\./);
 166   next unless(-d $testname);
 167
 168   &htmlize_test($testname);
 169 }
 170
 171 print HTMLFILE "<TR COLSPAN=3>Exploits</TR>\n";
 172 print HTMLFILE "<TR><TH>Test name</TH><TH>Result</TH><TH>Detail</TH></TR>\n";
 173
 174 foreach $testname (@exploittests) {
 175   next if($testname =~ /^\./);
 176   next unless(-d $testname);
 177
 178   &htmlize_test($testname);
 179 }
 180
 181 print HTMLFILE "</TABLE>  \n";
 182 print HTMLFILE "\n<BR><PRE>TOTAL tests: $total   PASSED: $passed   FAILED: $failed   MISSED: $missed  SUCCESS RATE: %".sprintf("%2.1d",(($passed*100)/$total))."</PRE><BR>\n";
 183 print HTMLFILE "<A HREF=\"stdout.txt\">stdout</A><BR>\n";
 184 print HTMLFILE "<A HREF=\"stderr.txt\">stderr</A><BR>\n";
 185 print HTMLFILE "</BODY></HTML>\n";
 186 close(HTMLFILE);
 187
 188 open(FAILLIST, ">faillist.txt") || die "failed to write to faillist.txt: $!\n";
 189 print FAILLIST join('\n', @faillist)."\n";
 190 close(FAILLIST);
 191
 192
 193
 194