3 # Copyright 2018, The Android Open Source Project
5 # Licensed under the Apache License, Version 2.0 (the "License");
6 # you may not use this file except in compliance with the License.
7 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
27 from run_host_unit_tests import *
30 This script is used to generate code coverage results host supported libraries.
31 The script by default will generate an html report that summarizes the coverage
32 results of the specified tests. The results can also be browsed to provide a
33 report of which lines have been traveled upon execution of the binary.
35 NOTE: Code that is compiled out or hidden by a #DEFINE will be listed as
36 having been executed 0 times, thus reducing overall coverage.
38 The steps in order to add coverage support to a new library and its
39 corrisponding host test are as follows.
41 1. Add "clang_file_coverage" (defined in //build/Android.bp) as a default to the
42 source library(s) you want statistics for.
43 NOTE: Forgoing this default will cause no coverage data to be generated for
44 the source files in the library.
46 2. Add "clang_coverage_bin" as a default to the host supported test binary that
47 excercises the libraries that you covered in step 1.
48 NOTE: Forgetting to add this will cause there to be *NO* coverage data
49 generated when the binary is run.
51 3. Add the host test binary name and the files/directories you want coverage
52 statistics for to the COVERAGE_TESTS variable defined below. You may add
53 individual filenames or a directory to be tested.
54 NOTE: Avoid using a / at the beginning of a covered_files entry as this
55 breaks how the coverage generator resolves filenames.
57 TODO: Support generating XML data and printing results to standard out.
62 "test_name": "net_test_avrcp",
64 "system/bt/profile/avrcp",
67 "test_name": "bluetooth_test_sdp",
69 "system/bt/profile/sdp",
72 "test_name": "test-vendor_test_host",
74 "system/bt/vendor_libs/test_vendor_lib/include",
75 "system/bt/vendor_libs/test_vendor_lib/src",
78 "test_name": "rootcanal-packets_test_host",
80 "system/bt/vendor_libs/test_vendor_lib/packets",
83 "test_name": "bluetooth_test_common",
90 WORKING_DIR = '/tmp/coverage'
91 SOONG_UI_BASH = 'build/soong/soong_ui.bash'
92 LLVM_DIR = 'prebuilts/clang/host/linux-x86/clang-r353983b/bin'
93 LLVM_MERGE = LLVM_DIR + '/llvm-profdata'
94 LLVM_COV = LLVM_DIR + '/llvm-cov'
96 def write_root_html_head(f):
97 # Write the header part of the root html file. This was pulled from the
98 # page source of one of the generated html files.
99 f.write("<!doctype html><html><head>" \
100 "<meta name='viewport' content='width=device-width,initial-scale=1'><met" \
101 "a charset='UTF-8'><link rel='stylesheet' type='text/css' href='style.cs" \
102 "s'></head><body><h2>Coverage Report</h2><h4>Created: " +
103 str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M')) +
104 "</h4><p>Click <a href='http://clang.llvm.org/docs/SourceBasedCodeCovera" \
105 "ge.html#interpreting-reports'>here</a> for information about interpreti" \
106 "ng this report.</p><div class='centered'><table><tr><td class='column-e" \
107 "ntry-bold'>Filename</td><td class='column-entry-bold'>Function Coverage" \
108 "</td><td class='column-entry-bold'>Instantiation Coverage</td><td class" \
109 "='column-entry-bold'>Line Coverage</td><td class='column-entry-bold'>Re" \
110 "gion Coverage</td></tr>"
114 def write_root_html_column(f, covered, count):
115 percent = covered * 100.0 / count
116 value = "%.2f%% (%d/%d) " % (percent, covered, count)
117 color = 'column-entry-yellow'
119 color = 'column-entry-green'
121 color = 'column-entry-red'
122 f.write("<td class=\'" + color + "\'><pre>" + value + "</pre></td>")
125 def write_root_html_rows(f, tests):
145 # Write the tests with their coverage summaries.
147 test_name = test['test_name']
148 covered_files = test['covered_files']
149 json_results = generate_coverage_json(test)
150 test_totals = json_results['data'][0]['totals']
152 f.write("<tr class='light-row'><td><pre><a href=\'" +
153 os.path.join(test_name, "index.html") + "\'>" + test_name +
155 for field_name in ['functions', 'instantiations', 'lines', 'regions']:
156 field = test_totals[field_name]
157 totals[field_name]['covered'] += field['covered']
158 totals[field_name]['count'] += field['count']
159 write_root_html_column(f, field['covered'], field['count'])
162 #Write the totals row.
163 f.write("<tr class='light-row-bold'><td><pre>Totals</a></pre></td>")
164 for field_name in ['functions', 'instantiations', 'lines', 'regions']:
165 field = totals[field_name]
166 write_root_html_column(f, field['covered'], field['count'])
170 def write_root_html_tail(f):
171 # Pulled from the generated html coverage report.
172 f.write("</table></div><h5>Generated by llvm-cov -- llvm version 7.0.2svn<" \
173 "/h5></body></html>")
176 def generate_root_html(tests):
177 # Copy the css file from one of the coverage reports.
178 source_file = os.path.join(os.path.join(WORKING_DIR, tests[0]['test_name']), "style.css")
179 dest_file = os.path.join(WORKING_DIR, "style.css")
180 shutil.copy2(source_file, dest_file)
182 # Write the root index.html file that sumarizes all the tests.
183 f = open(os.path.join(WORKING_DIR, "index.html"), "w")
184 write_root_html_head(f)
185 write_root_html_rows(f, tests)
186 write_root_html_tail(f)
189 def get_profraw_for_test(test_name):
190 test_root = get_native_test_root_or_die()
191 test_cmd = os.path.join(os.path.join(test_root, test_name), test_name)
192 if not os.path.isfile(test_cmd):
193 logging.error('The test ' + test_name + ' does not exist, please compile first')
196 profraw_file_name = test_name + ".profraw"
197 profraw_path = os.path.join(WORKING_DIR, os.path.join(test_name, profraw_file_name))
198 llvm_env_var = "LLVM_PROFILE_FILE=\"" + profraw_path + "\""
200 test_cmd = llvm_env_var + " " + test_cmd
201 logging.info('Generating profraw data for ' + test_name)
202 logging.debug('cmd: ' + test_cmd)
203 if subprocess.call(test_cmd, shell=True) != 0:
204 logging.error('Test ' + test_name + ' failed. Please fix the test before generating coverage.')
207 if not os.path.isfile(profraw_path):
208 logging.error('Generating the profraw file failed. Did you remember to add the proper compiler flags to your build?')
211 return profraw_file_name
214 def merge_profraw_data(test_name):
216 cmd.append(os.path.join(get_android_root_or_die(), LLVM_MERGE + " merge "))
218 test_working_dir = os.path.join(WORKING_DIR, test_name);
219 cmd.append(os.path.join(test_working_dir, test_name + ".profraw"))
220 profdata_file = os.path.join(test_working_dir, test_name + ".profdata")
222 cmd.append('-o ' + profdata_file)
223 logging.info('Combining profraw files into profdata for ' + test_name)
224 logging.debug('cmd: ' + " ".join(cmd))
225 if subprocess.call(" ".join(cmd), shell=True) != 0:
226 logging.error('Failed to merge profraw files for ' + test_name)
230 def generate_coverage_html(test):
231 COVERAGE_ROOT = '/proc/self/cwd'
233 test_name = test['test_name']
234 file_list = test['covered_files']
236 test_working_dir = os.path.join(WORKING_DIR, test_name)
237 test_profdata_file = os.path.join(test_working_dir, test_name + ".profdata")
240 os.path.join(get_android_root_or_die(), LLVM_COV),
244 "-show-line-counts-or-regions",
245 "-show-instantiation-summary",
246 "-instr-profile=" + test_profdata_file,
247 "-path-equivalence=\"" + COVERAGE_ROOT + "\",\"" +
248 get_android_root_or_die() + "\"",
249 "-output-dir=" + test_working_dir
252 # We have to have one object file not as an argument otherwise we can't specify source files.
253 test_cmd = os.path.join(os.path.join(get_native_test_root_or_die(), test_name), test_name)
256 # Filter out the specific files we want coverage for
257 for filename in file_list:
258 cmd.append(os.path.join(get_android_root_or_die(), filename))
260 logging.info('Generating coverage report for ' + test['test_name'])
261 logging.debug('cmd: ' + " ".join(cmd))
262 if subprocess.call(" ".join(cmd), shell=True) != 0:
263 logging.error('Failed to generate coverage for ' + test['test_name'])
267 def generate_coverage_json(test):
268 COVERAGE_ROOT = '/proc/self/cwd'
269 test_name = test['test_name']
270 file_list = test['covered_files']
272 test_working_dir = os.path.join(WORKING_DIR, test_name)
273 test_profdata_file = os.path.join(test_working_dir, test_name + ".profdata")
276 os.path.join(get_android_root_or_die(), LLVM_COV),
279 "-show-region-summary",
280 "-instr-profile=" + test_profdata_file,
281 "-path-equivalence=\"" + COVERAGE_ROOT + "\",\"" + get_android_root_or_die() + "\"",
284 test_cmd = os.path.join(os.path.join(get_native_test_root_or_die(), test_name), test_name)
287 # Filter out the specific files we want coverage for
288 for filename in file_list:
289 cmd.append(os.path.join(get_android_root_or_die(), filename))
291 logging.info('Generating coverage json for ' + test['test_name'])
292 logging.debug('cmd: ' + " ".join(cmd))
294 json_str = subprocess.check_output(" ".join(cmd), shell=True)
295 return json.loads(json_str)
298 def write_json_summary(test):
299 test_name = test['test_name']
300 test_working_dir = os.path.join(WORKING_DIR, test_name)
301 test_json_summary_file = os.path.join(test_working_dir, test_name + '.json')
302 logging.debug('Writing json summary file: ' + test_json_summary_file)
303 json_file = open(test_json_summary_file, 'w')
304 json.dump(generate_coverage_json(test), json_file)
309 for test in COVERAGE_TESTS:
310 print "Test Name: " + test['test_name']
311 print "Covered Files: "
312 for covered_file in test['covered_files']:
313 print " " + covered_file
318 parser = argparse.ArgumentParser(description='Generate code coverage for enabled tests.')
320 '-l', '--list-tests',
323 help='List all the available tests to be run as well as covered files.')
327 help='Runs all available tests and prints their outputs. If no tests ' \
328 'are specified via the -t option all tests will be run.')
336 help='Specifies a test to be run. Multiple tests can be specified by ' \
337 'using this option multiple times. ' \
338 'Example: \"gen_coverage.py -t test1 -t test2\"')
343 default='/tmp/coverage',
344 help='Specifies the directory to store all files. The directory will be ' \
345 'created if it does not exist. Default is \"/tmp/coverage\"')
350 help='Skip opening up the results of the coverage report in a browser.')
355 help='Write out summary results to json file in test directory.')
357 logging.basicConfig(stream=sys.stderr, level=logging.DEBUG, format='%(levelname)s %(message)s')
358 logging.addLevelName(logging.DEBUG, "[\033[1;34m%s\033[0m]" % logging.getLevelName(logging.DEBUG))
359 logging.addLevelName(logging.INFO, "[\033[1;34m%s\033[0m]" % logging.getLevelName(logging.INFO))
360 logging.addLevelName(logging.WARNING, "[\033[1;31m%s\033[0m]" % logging.getLevelName(logging.WARNING))
361 logging.addLevelName(logging.ERROR, "[\033[1;31m%s\033[0m]" % logging.getLevelName(logging.ERROR))
363 args = parser.parse_args()
364 logging.debug("Args: " + str(args))
366 # Set the working directory
368 WORKING_DIR = os.path.abspath(args.output)
369 logging.debug("Working Dir: " + WORKING_DIR)
371 # Print out the list of tests then exit
376 # Check to see if a test was specified and if so only generate coverage for
378 if len(args.tests) == 0:
382 for test in COVERAGE_TESTS:
383 if args.all or test['test_name'] in args.tests:
384 tests_to_run.append(test)
385 if test['test_name'] in args.tests:
386 args.tests.remove(test['test_name'])
388 # Error if a test was specified but doesn't exist.
389 if len(args.tests) != 0:
390 for test_name in args.tests:
391 logging.error('\"' + test_name + '\" was not found in the list of available tests.')
394 # Generate the info for the tests
395 for test in tests_to_run:
396 logging.info('Getting coverage for ' + test['test_name'])
397 get_profraw_for_test(test['test_name'])
398 merge_profraw_data(test['test_name'])
400 write_json_summary(test)
401 generate_coverage_html(test)
403 # Generate the root index.html page that sumarizes all of the coverage reports.
404 generate_root_html(tests_to_run)
406 # Open the results in a browser.
407 if not args.skip_html:
408 webbrowser.open('file://' + os.path.join(WORKING_DIR, 'index.html'))
411 if __name__ == '__main__':