test/gen_coverage.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2018, The Android Open Source Project
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16
  17 import argparse
  18 import datetime
  19 import logging
  20 import json
  21 import os
  22 import shutil
  23 import subprocess
  24 import sys
  25 import webbrowser
  26
  27 from run_host_unit_tests import *
  28 """
  29 This script is used to generate code coverage results host supported libraries.
  30 The script by default will generate an html report that summarizes the coverage
  31 results of the specified tests. The results can also be browsed to provide a
  32 report of which lines have been traveled upon execution of the binary.
  33
  34 NOTE: Code that is compiled out or hidden by a #DEFINE will be listed as
  35 having been executed 0 times, thus reducing overall coverage.
  36
  37 The steps in order to add coverage support to a new library and its
  38 corrisponding host test are as follows.
  39
  40 1. Add "clang_file_coverage" (defined in //build/Android.bp) as a default to the
  41    source library(s) you want statistics for.
  42    NOTE: Forgoing this default will cause no coverage data to be generated for
  43          the source files in the library.
  44
  45 2. Add "clang_coverage_bin" as a default to the host supported test binary that
  46    excercises the libraries that you covered in step 1.
  47    NOTE: Forgetting to add this will cause there to be *NO* coverage data
  48          generated when the binary is run.
  49
  50 3. Add the host test binary name and the files/directories you want coverage
  51    statistics for to the COVERAGE_TESTS variable defined below. You may add
  52    individual filenames or a directory to be tested.
  53    NOTE: Avoid using a / at the beginning of a covered_files entry as this
  54          breaks how the coverage generator resolves filenames.
  55
  56 TODO: Support generating XML data and printing results to standard out.
  57 """
  58
  59 COVERAGE_TESTS = [
  60     {
  61         "test_name": "net_test_avrcp",
  62         "covered_files": [
  63             "system/bt/profile/avrcp",
  64         ],
  65     },
  66     {
  67         "test_name": "bluetooth_test_sdp",
  68         "covered_files": [
  69             "system/bt/profile/sdp",
  70         ],
  71     },
  72     {
  73         "test_name":
  74         "test-vendor_test_host",
  75         "covered_files": [
  76             "system/bt/vendor_libs/test_vendor_lib/include",
  77             "system/bt/vendor_libs/test_vendor_lib/src",
  78         ],
  79     },
  80     {
  81         "test_name": "rootcanal-packets_test_host",
  82         "covered_files": [
  83             "system/bt/vendor_libs/test_vendor_lib/packets",
  84         ],
  85     },
  86     {
  87         "test_name": "bluetooth_test_common",
  88         "covered_files": [
  89             "system/bt/common",
  90         ],
  91     },
  92 ]
  93
  94 WORKING_DIR = '/tmp/coverage'
  95 SOONG_UI_BASH = 'build/soong/soong_ui.bash'
  96 LLVM_DIR = 'prebuilts/clang/host/linux-x86/clang-r353983b/bin'
  97 LLVM_MERGE = LLVM_DIR + '/llvm-profdata'
  98 LLVM_COV = LLVM_DIR + '/llvm-cov'
  99
 100
 101 def write_root_html_head(f):
 102     # Write the header part of the root html file. This was pulled from the
 103     # page source of one of the generated html files.
 104     f.write("<!doctype html><html><head>" \
 105       "<meta name='viewport' content='width=device-width,initial-scale=1'><met" \
 106       "a charset='UTF-8'><link rel='stylesheet' type='text/css' href='style.cs" \
 107       "s'></head><body><h2>Coverage Report</h2><h4>Created: " +
 108       str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M')) +
 109       "</h4><p>Click <a href='http://clang.llvm.org/docs/SourceBasedCodeCovera" \
 110       "ge.html#interpreting-reports'>here</a> for information about interpreti" \
 111       "ng this report.</p><div class='centered'><table><tr><td class='column-e" \
 112       "ntry-bold'>Filename</td><td class='column-entry-bold'>Function Coverage" \
 113       "</td><td class='column-entry-bold'>Instantiation Coverage</td><td class" \
 114       "='column-entry-bold'>Line Coverage</td><td class='column-entry-bold'>Re" \
 115       "gion Coverage</td></tr>"
 116     )
 117
 118
 119 def write_root_html_column(f, covered, count):
 120     percent = covered * 100.0 / count
 121     value = "%.2f%% (%d/%d) " % (percent, covered, count)
 122     color = 'column-entry-yellow'
 123     if percent == 100:
 124         color = 'column-entry-green'
 125     if percent < 80.0:
 126         color = 'column-entry-red'
 127     f.write("<td class=\'" + color + "\'><pre>" + value + "</pre></td>")
 128
 129
 130 def write_root_html_rows(f, tests):
 131     totals = {
 132         "functions": {
 133             "covered": 0,
 134             "count": 0
 135         },
 136         "instantiations": {
 137             "covered": 0,
 138             "count": 0
 139         },
 140         "lines": {
 141             "covered": 0,
 142             "count": 0
 143         },
 144         "regions": {
 145             "covered": 0,
 146             "count": 0
 147         }
 148     }
 149
 150     # Write the tests with their coverage summaries.
 151     for test in tests:
 152         test_name = test['test_name']
 153         covered_files = test['covered_files']
 154         json_results = generate_coverage_json(test)
 155         test_totals = json_results['data'][0]['totals']
 156
 157         f.write("<tr class='light-row'><td><pre><a href=\'" +
 158                 os.path.join(test_name, "index.html") + "\'>" + test_name +
 159                 "</a></pre></td>")
 160         for field_name in ['functions', 'instantiations', 'lines', 'regions']:
 161             field = test_totals[field_name]
 162             totals[field_name]['covered'] += field['covered']
 163             totals[field_name]['count'] += field['count']
 164             write_root_html_column(f, field['covered'], field['count'])
 165         f.write("</tr>")
 166
 167     #Write the totals row.
 168     f.write("<tr class='light-row-bold'><td><pre>Totals</a></pre></td>")
 169     for field_name in ['functions', 'instantiations', 'lines', 'regions']:
 170         field = totals[field_name]
 171         write_root_html_column(f, field['covered'], field['count'])
 172     f.write("</tr>")
 173
 174
 175 def write_root_html_tail(f):
 176     # Pulled from the generated html coverage report.
 177     f.write("</table></div><h5>Generated by llvm-cov -- llvm version 7.0.2svn<" \
 178       "/h5></body></html>")
 179
 180
 181 def generate_root_html(tests):
 182     # Copy the css file from one of the coverage reports.
 183     source_file = os.path.join(
 184         os.path.join(WORKING_DIR, tests[0]['test_name']), "style.css")
 185     dest_file = os.path.join(WORKING_DIR, "style.css")
 186     shutil.copy2(source_file, dest_file)
 187
 188     # Write the root index.html file that sumarizes all the tests.
 189     f = open(os.path.join(WORKING_DIR, "index.html"), "w")
 190     write_root_html_head(f)
 191     write_root_html_rows(f, tests)
 192     write_root_html_tail(f)
 193
 194
 195 def get_profraw_for_test(test_name):
 196     test_root = get_native_test_root_or_die()
 197     test_cmd = os.path.join(os.path.join(test_root, test_name), test_name)
 198     if not os.path.isfile(test_cmd):
 199         logging.error('The test ' + test_name +
 200                       ' does not exist, please compile first')
 201         sys.exit(1)
 202
 203     profraw_file_name = test_name + ".profraw"
 204     profraw_path = os.path.join(WORKING_DIR,
 205                                 os.path.join(test_name, profraw_file_name))
 206     llvm_env_var = "LLVM_PROFILE_FILE=\"" + profraw_path + "\""
 207
 208     test_cmd = llvm_env_var + " " + test_cmd
 209     logging.info('Generating profraw data for ' + test_name)
 210     logging.debug('cmd: ' + test_cmd)
 211     if subprocess.call(test_cmd, shell=True) != 0:
 212         logging.error(
 213             'Test ' + test_name +
 214             ' failed. Please fix the test before generating coverage.')
 215         sys.exit(1)
 216
 217     if not os.path.isfile(profraw_path):
 218         logging.error(
 219             'Generating the profraw file failed. Did you remember to add the proper compiler flags to your build?'
 220         )
 221         sys.exit(1)
 222
 223     return profraw_file_name
 224
 225
 226 def merge_profraw_data(test_name):
 227     cmd = []
 228     cmd.append(os.path.join(get_android_root_or_die(), LLVM_MERGE + " merge "))
 229
 230     test_working_dir = os.path.join(WORKING_DIR, test_name)
 231     cmd.append(os.path.join(test_working_dir, test_name + ".profraw"))
 232     profdata_file = os.path.join(test_working_dir, test_name + ".profdata")
 233
 234     cmd.append('-o ' + profdata_file)
 235     logging.info('Combining profraw files into profdata for ' + test_name)
 236     logging.debug('cmd: ' + " ".join(cmd))
 237     if subprocess.call(" ".join(cmd), shell=True) != 0:
 238         logging.error('Failed to merge profraw files for ' + test_name)
 239         sys.exit(1)
 240
 241
 242 def generate_coverage_html(test):
 243     COVERAGE_ROOT = '/proc/self/cwd'
 244
 245     test_name = test['test_name']
 246     file_list = test['covered_files']
 247
 248     test_working_dir = os.path.join(WORKING_DIR, test_name)
 249     test_profdata_file = os.path.join(test_working_dir, test_name + ".profdata")
 250
 251     cmd = [
 252         os.path.join(get_android_root_or_die(), LLVM_COV), "show",
 253         "-format=html", "-summary-only", "-show-line-counts-or-regions",
 254         "-show-instantiation-summary", "-instr-profile=" + test_profdata_file,
 255         "-path-equivalence=\"" + COVERAGE_ROOT + "\",\"" +
 256         get_android_root_or_die() + "\"", "-output-dir=" + test_working_dir
 257     ]
 258
 259     # We have to have one object file not as an argument otherwise we can't specify source files.
 260     test_cmd = os.path.join(
 261         os.path.join(get_native_test_root_or_die(), test_name), test_name)
 262     cmd.append(test_cmd)
 263
 264     # Filter out the specific files we want coverage for
 265     for filename in file_list:
 266         cmd.append(os.path.join(get_android_root_or_die(), filename))
 267
 268     logging.info('Generating coverage report for ' + test['test_name'])
 269     logging.debug('cmd: ' + " ".join(cmd))
 270     if subprocess.call(" ".join(cmd), shell=True) != 0:
 271         logging.error('Failed to generate coverage for ' + test['test_name'])
 272         sys.exit(1)
 273
 274
 275 def generate_coverage_json(test):
 276     COVERAGE_ROOT = '/proc/self/cwd'
 277     test_name = test['test_name']
 278     file_list = test['covered_files']
 279
 280     test_working_dir = os.path.join(WORKING_DIR, test_name)
 281     test_profdata_file = os.path.join(test_working_dir, test_name + ".profdata")
 282
 283     cmd = [
 284         os.path.join(get_android_root_or_die(), LLVM_COV),
 285         "export",
 286         "-summary-only",
 287         "-show-region-summary",
 288         "-instr-profile=" + test_profdata_file,
 289         "-path-equivalence=\"" + COVERAGE_ROOT + "\",\"" +
 290         get_android_root_or_die() + "\"",
 291     ]
 292
 293     test_cmd = os.path.join(
 294         os.path.join(get_native_test_root_or_die(), test_name), test_name)
 295     cmd.append(test_cmd)
 296
 297     # Filter out the specific files we want coverage for
 298     for filename in file_list:
 299         cmd.append(os.path.join(get_android_root_or_die(), filename))
 300
 301     logging.info('Generating coverage json for ' + test['test_name'])
 302     logging.debug('cmd: ' + " ".join(cmd))
 303
 304     json_str = subprocess.check_output(" ".join(cmd), shell=True)
 305     return json.loads(json_str)
 306
 307
 308 def write_json_summary(test):
 309     test_name = test['test_name']
 310     test_working_dir = os.path.join(WORKING_DIR, test_name)
 311     test_json_summary_file = os.path.join(test_working_dir, test_name + '.json')
 312     logging.debug('Writing json summary file: ' + test_json_summary_file)
 313     json_file = open(test_json_summary_file, 'w')
 314     json.dump(generate_coverage_json(test), json_file)
 315     json_file.close()
 316
 317
 318 def list_tests():
 319     for test in COVERAGE_TESTS:
 320         print "Test Name: " + test['test_name']
 321         print "Covered Files: "
 322         for covered_file in test['covered_files']:
 323             print "  " + covered_file
 324         print
 325
 326
 327 def main():
 328     parser = argparse.ArgumentParser(
 329         description='Generate code coverage for enabled tests.')
 330     parser.add_argument(
 331         '-l',
 332         '--list-tests',
 333         action='store_true',
 334         dest='list_tests',
 335         help='List all the available tests to be run as well as covered files.')
 336     parser.add_argument(
 337       '-a', '--all',
 338       action='store_true',
 339       help='Runs all available tests and prints their outputs. If no tests ' \
 340            'are specified via the -t option all tests will be run.')
 341     parser.add_argument(
 342       '-t', '--test',
 343       dest='tests',
 344       action='append',
 345       type=str,
 346       metavar='TESTNAME',
 347       default=[],
 348       help='Specifies a test to be run. Multiple tests can be specified by ' \
 349            'using this option multiple times. ' \
 350            'Example: \"gen_coverage.py -t test1 -t test2\"')
 351     parser.add_argument(
 352       '-o', '--output',
 353       type=str,
 354       metavar='DIRECTORY',
 355       default='/tmp/coverage',
 356       help='Specifies the directory to store all files. The directory will be ' \
 357            'created if it does not exist. Default is \"/tmp/coverage\"')
 358     parser.add_argument(
 359         '-s',
 360         '--skip-html',
 361         dest='skip_html',
 362         action='store_true',
 363         help='Skip opening up the results of the coverage report in a browser.')
 364     parser.add_argument(
 365         '-j',
 366         '--json-file',
 367         dest='json_file',
 368         action='store_true',
 369         help='Write out summary results to json file in test directory.')
 370
 371     logging.basicConfig(
 372         stream=sys.stderr,
 373         level=logging.DEBUG,
 374         format='%(levelname)s %(message)s')
 375     logging.addLevelName(
 376         logging.DEBUG,
 377         "[\033[1;34m%s\033[0m]" % logging.getLevelName(logging.DEBUG))
 378     logging.addLevelName(
 379         logging.INFO,
 380         "[\033[1;34m%s\033[0m]" % logging.getLevelName(logging.INFO))
 381     logging.addLevelName(
 382         logging.WARNING,
 383         "[\033[1;31m%s\033[0m]" % logging.getLevelName(logging.WARNING))
 384     logging.addLevelName(
 385         logging.ERROR,
 386         "[\033[1;31m%s\033[0m]" % logging.getLevelName(logging.ERROR))
 387
 388     args = parser.parse_args()
 389     logging.debug("Args: " + str(args))
 390
 391     # Set the working directory
 392     global WORKING_DIR
 393     WORKING_DIR = os.path.abspath(args.output)
 394     logging.debug("Working Dir: " + WORKING_DIR)
 395
 396     # Print out the list of tests then exit
 397     if args.list_tests:
 398         list_tests()
 399         sys.exit(0)
 400
 401     # Check to see if a test was specified and if so only generate coverage for
 402     # that test.
 403     if len(args.tests) == 0:
 404         args.all = True
 405
 406     tests_to_run = []
 407     for test in COVERAGE_TESTS:
 408         if args.all or test['test_name'] in args.tests:
 409             tests_to_run.append(test)
 410         if test['test_name'] in args.tests:
 411             args.tests.remove(test['test_name'])
 412
 413     # Error if a test was specified but doesn't exist.
 414     if len(args.tests) != 0:
 415         for test_name in args.tests:
 416             logging.error('\"' + test_name +
 417                           '\" was not found in the list of available tests.')
 418         sys.exit(1)
 419
 420     # Generate the info for the tests
 421     for test in tests_to_run:
 422         logging.info('Getting coverage for ' + test['test_name'])
 423         get_profraw_for_test(test['test_name'])
 424         merge_profraw_data(test['test_name'])
 425         if args.json_file:
 426             write_json_summary(test)
 427         generate_coverage_html(test)
 428
 429     # Generate the root index.html page that sumarizes all of the coverage reports.
 430     generate_root_html(tests_to_run)
 431
 432     # Open the results in a browser.
 433     if not args.skip_html:
 434         webbrowser.open('file://' + os.path.join(WORKING_DIR, 'index.html'))
 435
 436
 437 if __name__ == '__main__':
 438     main()