simpleperf/runtest/runtest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (C) 2015 The Android Open Source Project
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #      http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17 """Simpleperf runtest runner: run simpleperf runtests on host or on device.
  18
  19 For a simpleperf runtest like one_function test, it contains following steps:
  20 1. Run simpleperf record command to record simpleperf_runtest_one_function's
  21    running samples, which is generated in perf.data.
  22 2. Run simpleperf report command to parse perf.data, generate perf.report.
  23 4. Parse perf.report and see if it matches expectation.
  24
  25 The information of all runtests is stored in runtest.conf.
  26 """
  27
  28 import os
  29 import os.path
  30 import re
  31 import subprocess
  32 import sys
  33 import xml.etree.ElementTree as ET
  34
  35
  36 class CallTreeNode(object):
  37
  38   def __init__(self, name):
  39     self.name = name
  40     self.children = []
  41
  42   def add_child(self, child):
  43     self.children.append(child)
  44
  45   def __str__(self):
  46     return 'CallTreeNode:\n' + '\n'.join(self._dump(1))
  47
  48   def _dump(self, indent):
  49     indent_str = '  ' * indent
  50     strs = [indent_str + self.name]
  51     for child in self.children:
  52       strs.extend(child._dump(indent + 1))
  53     return strs
  54
  55
  56 class Symbol(object):
  57
  58   def __init__(self, name, comm, overhead, children_overhead):
  59     self.name = name
  60     self.comm = comm
  61     self.overhead = overhead
  62     # children_overhead is the overhead sum of this symbol and functions
  63     # called by this symbol.
  64     self.children_overhead = children_overhead
  65     self.call_tree = None
  66
  67   def set_call_tree(self, call_tree):
  68     self.call_tree = call_tree
  69
  70   def __str__(self):
  71     strs = []
  72     strs.append('Symbol name=%s comm=%s overhead=%f children_overhead=%f' % (
  73         self.name, self.comm, self.overhead, self.children_overhead))
  74     if self.call_tree:
  75       strs.append('\t%s' % self.call_tree)
  76     return '\n'.join(strs)
  77
  78
  79 class SymbolOverheadRequirement(object):
  80
  81   def __init__(self, symbol_name=None, comm=None, min_overhead=None,
  82                max_overhead=None):
  83     self.symbol_name = symbol_name
  84     self.comm = comm
  85     self.min_overhead = min_overhead
  86     self.max_overhead = max_overhead
  87
  88   def __str__(self):
  89     strs = []
  90     strs.append('SymbolOverheadRequirement')
  91     if self.symbol_name is not None:
  92       strs.append('symbol_name=%s' % self.symbol_name)
  93     if self.comm is not None:
  94       strs.append('comm=%s' % self.comm)
  95     if self.min_overhead is not None:
  96       strs.append('min_overhead=%f' % self.min_overhead)
  97     if self.max_overhead is not None:
  98       strs.append('max_overhead=%f' % self.max_overhead)
  99     return ' '.join(strs)
 100
 101   def is_match(self, symbol):
 102     if self.symbol_name is not None:
 103       if self.symbol_name != symbol.name:
 104         return False
 105     if self.comm is not None:
 106       if self.comm != symbol.comm:
 107         return False
 108     return True
 109
 110   def check_overhead(self, overhead):
 111     if self.min_overhead is not None:
 112       if self.min_overhead > overhead:
 113         return False
 114     if self.max_overhead is not None:
 115       if self.max_overhead < overhead:
 116         return False
 117     return True
 118
 119
 120 class SymbolRelationRequirement(object):
 121
 122   def __init__(self, symbol_name, comm=None):
 123     self.symbol_name = symbol_name
 124     self.comm = comm
 125     self.children = []
 126
 127   def add_child(self, child):
 128     self.children.append(child)
 129
 130   def __str__(self):
 131     return 'SymbolRelationRequirement:\n' + '\n'.join(self._dump(1))
 132
 133   def _dump(self, indent):
 134     indent_str = '  ' * indent
 135     strs = [indent_str + self.symbol_name +
 136             (' ' + self.comm if self.comm else '')]
 137     for child in self.children:
 138       strs.extend(child._dump(indent + 1))
 139     return strs
 140
 141   def is_match(self, symbol):
 142     if symbol.name != self.symbol_name:
 143       return False
 144     if self.comm is not None:
 145       if symbol.comm != self.comm:
 146         return False
 147     return True
 148
 149   def check_relation(self, call_tree):
 150     if not call_tree:
 151       return False
 152     if self.symbol_name != call_tree.name:
 153       return False
 154     for child in self.children:
 155       child_matched = False
 156       for node in call_tree.children:
 157         if child.check_relation(node):
 158           child_matched = True
 159           break
 160       if not child_matched:
 161         return False
 162     return True
 163
 164
 165 class Test(object):
 166
 167   def __init__(
 168           self,
 169           test_name,
 170           executable_name,
 171           report_options,
 172           symbol_overhead_requirements,
 173           symbol_children_overhead_requirements,
 174           symbol_relation_requirements):
 175     self.test_name = test_name
 176     self.executable_name = executable_name
 177     self.report_options = report_options
 178     self.symbol_overhead_requirements = symbol_overhead_requirements
 179     self.symbol_children_overhead_requirements = (
 180         symbol_children_overhead_requirements)
 181     self.symbol_relation_requirements = symbol_relation_requirements
 182
 183   def __str__(self):
 184     strs = []
 185     strs.append('Test test_name=%s' % self.test_name)
 186     strs.append('\texecutable_name=%s' % self.executable_name)
 187     strs.append('\treport_options=%s' % (' '.join(self.report_options)))
 188     strs.append('\tsymbol_overhead_requirements:')
 189     for req in self.symbol_overhead_requirements:
 190       strs.append('\t\t%s' % req)
 191     strs.append('\tsymbol_children_overhead_requirements:')
 192     for req in self.symbol_children_overhead_requirements:
 193       strs.append('\t\t%s' % req)
 194     strs.append('\tsymbol_relation_requirements:')
 195     for req in self.symbol_relation_requirements:
 196       strs.append('\t\t%s' % req)
 197     return '\n'.join(strs)
 198
 199
 200 def load_config_file(config_file):
 201   tests = []
 202   tree = ET.parse(config_file)
 203   root = tree.getroot()
 204   assert root.tag == 'runtests'
 205   for test in root:
 206     assert test.tag == 'test'
 207     test_name = test.attrib['name']
 208     executable_name = None
 209     report_options = []
 210     symbol_overhead_requirements = []
 211     symbol_children_overhead_requirements = []
 212     symbol_relation_requirements = []
 213     for test_item in test:
 214       if test_item.tag == 'executable':
 215         executable_name = test_item.attrib['name']
 216       elif test_item.tag == 'report':
 217         report_options = test_item.attrib['option'].split()
 218       elif (test_item.tag == 'symbol_overhead' or
 219               test_item.tag == 'symbol_children_overhead'):
 220         for symbol_item in test_item:
 221           assert symbol_item.tag == 'symbol'
 222           symbol_name = None
 223           if 'name' in symbol_item.attrib:
 224             symbol_name = symbol_item.attrib['name']
 225           comm = None
 226           if 'comm' in symbol_item.attrib:
 227             comm = symbol_item.attrib['comm']
 228           overhead_min = None
 229           if 'min' in symbol_item.attrib:
 230             overhead_min = float(symbol_item.attrib['min'])
 231           overhead_max = None
 232           if 'max' in symbol_item.attrib:
 233             overhead_max = float(symbol_item.attrib['max'])
 234
 235           if test_item.tag == 'symbol_overhead':
 236             symbol_overhead_requirements.append(
 237                 SymbolOverheadRequirement(
 238                     symbol_name,
 239                     comm,
 240                     overhead_min,
 241                     overhead_max)
 242             )
 243           else:
 244             symbol_children_overhead_requirements.append(
 245                 SymbolOverheadRequirement(
 246                     symbol_name,
 247                     comm,
 248                     overhead_min,
 249                     overhead_max))
 250       elif test_item.tag == 'symbol_callgraph_relation':
 251         for symbol_item in test_item:
 252           req = load_symbol_relation_requirement(symbol_item)
 253           symbol_relation_requirements.append(req)
 254
 255     tests.append(
 256         Test(
 257             test_name,
 258             executable_name,
 259             report_options,
 260             symbol_overhead_requirements,
 261             symbol_children_overhead_requirements,
 262             symbol_relation_requirements))
 263   return tests
 264
 265
 266 def load_symbol_relation_requirement(symbol_item):
 267   symbol_name = symbol_item.attrib['name']
 268   comm = None
 269   if 'comm' in symbol_item.attrib:
 270     comm = symbol_item.attrib['comm']
 271   req = SymbolRelationRequirement(symbol_name, comm)
 272   for item in symbol_item:
 273     child_req = load_symbol_relation_requirement(item)
 274     req.add_child(child_req)
 275   return req
 276
 277
 278 class Runner(object):
 279
 280   def __init__(self, target, perf_path):
 281     self.target = target
 282     self.is32 = target.endswith('32')
 283     self.perf_path = perf_path
 284     self.use_callgraph = False
 285     self.sampler = 'cpu-cycles'
 286
 287   def record(self, test_executable_name, record_file, additional_options=[]):
 288     call_args = [self.perf_path, 'record']
 289     call_args += ['--duration', '2']
 290     call_args += ['-e', '%s:u' % self.sampler]
 291     if self.use_callgraph:
 292       call_args += ['-f', '1000', '-g']
 293     call_args += ['-o', record_file]
 294     call_args += additional_options
 295     test_executable_name += '32' if self.is32 else '64'
 296     call_args += [test_executable_name]
 297     self._call(call_args)
 298
 299   def report(self, record_file, report_file, additional_options=[]):
 300     call_args = [self.perf_path, 'report']
 301     call_args += ['-i', record_file]
 302     if self.use_callgraph:
 303       call_args += ['-g', 'callee']
 304     call_args += additional_options
 305     self._call(call_args, report_file)
 306
 307   def _call(self, args, output_file=None):
 308     pass
 309
 310
 311 class HostRunner(Runner):
 312
 313   """Run perf test on host."""
 314
 315   def __init__(self, target):
 316     perf_path = 'simpleperf32' if target.endswith('32') else 'simpleperf'
 317     super(HostRunner, self).__init__(target, perf_path)
 318
 319   def _call(self, args, output_file=None):
 320     output_fh = None
 321     if output_file is not None:
 322       output_fh = open(output_file, 'w')
 323     subprocess.check_call(args, stdout=output_fh)
 324     if output_fh is not None:
 325       output_fh.close()
 326
 327
 328 class DeviceRunner(Runner):
 329
 330   """Run perf test on device."""
 331
 332   def __init__(self, target):
 333     self.tmpdir = '/data/local/tmp/'
 334     perf_path = 'simpleperf32' if target.endswith('32') else 'simpleperf'
 335     super(DeviceRunner, self).__init__(target, self.tmpdir + perf_path)
 336     self._download(os.environ['OUT'] + '/system/xbin/' + perf_path, self.tmpdir)
 337     lib = 'lib' if self.is32 else 'lib64'
 338     self._download(os.environ['OUT'] + '/system/' + lib + '/libsimpleperf_inplace_sampler.so',
 339                    self.tmpdir)
 340
 341   def _call(self, args, output_file=None):
 342     output_fh = None
 343     if output_file is not None:
 344       output_fh = open(output_file, 'w')
 345     args_with_adb = ['adb', 'shell']
 346     args_with_adb.append('export LD_LIBRARY_PATH=' + self.tmpdir + ' && ' + ' '.join(args))
 347     subprocess.check_call(args_with_adb, stdout=output_fh)
 348     if output_fh is not None:
 349       output_fh.close()
 350
 351   def _download(self, file, to_dir):
 352     args = ['adb', 'push', file, to_dir]
 353     subprocess.check_call(args)
 354
 355   def record(self, test_executable_name, record_file, additional_options=[]):
 356     self._download(os.environ['OUT'] + '/system/bin/' + test_executable_name +
 357                    ('32' if self.is32 else '64'), self.tmpdir)
 358     super(DeviceRunner, self).record(self.tmpdir + test_executable_name,
 359                                      self.tmpdir + record_file,
 360                                      additional_options)
 361
 362   def report(self, record_file, report_file, additional_options=[]):
 363     super(DeviceRunner, self).report(self.tmpdir + record_file,
 364                                      report_file,
 365                                      additional_options)
 366
 367 class ReportAnalyzer(object):
 368
 369   """Check if perf.report matches expectation in Configuration."""
 370
 371   def _read_report_file(self, report_file, has_callgraph):
 372     fh = open(report_file, 'r')
 373     lines = fh.readlines()
 374     fh.close()
 375
 376     lines = [x.rstrip() for x in lines]
 377     blank_line_index = -1
 378     for i in range(len(lines)):
 379       if not lines[i]:
 380         blank_line_index = i
 381     assert blank_line_index != -1
 382     assert blank_line_index + 1 < len(lines)
 383     title_line = lines[blank_line_index + 1]
 384     report_item_lines = lines[blank_line_index + 2:]
 385
 386     if has_callgraph:
 387       assert re.search(r'^Children\s+Self\s+Command.+Symbol$', title_line)
 388     else:
 389       assert re.search(r'^Overhead\s+Command.+Symbol$', title_line)
 390
 391     return self._parse_report_items(report_item_lines, has_callgraph)
 392
 393   def _parse_report_items(self, lines, has_callgraph):
 394     symbols = []
 395     cur_symbol = None
 396     call_tree_stack = {}
 397     vertical_columns = []
 398     last_node = None
 399     last_depth = -1
 400
 401     for line in lines:
 402       if not line:
 403         continue
 404       if not line[0].isspace():
 405         if has_callgraph:
 406           m = re.search(r'^([\d\.]+)%\s+([\d\.]+)%\s+(\S+).*\s+(\S+)$', line)
 407           children_overhead = float(m.group(1))
 408           overhead = float(m.group(2))
 409           comm = m.group(3)
 410           symbol_name = m.group(4)
 411           cur_symbol = Symbol(symbol_name, comm, overhead, children_overhead)
 412           symbols.append(cur_symbol)
 413         else:
 414           m = re.search(r'^([\d\.]+)%\s+(\S+).*\s+(\S+)$', line)
 415           overhead = float(m.group(1))
 416           comm = m.group(2)
 417           symbol_name = m.group(3)
 418           cur_symbol = Symbol(symbol_name, comm, overhead, 0)
 419           symbols.append(cur_symbol)
 420         # Each report item can have different column depths.
 421         vertical_columns = []
 422       else:
 423         for i in range(len(line)):
 424           if line[i] == '|':
 425             if not vertical_columns or vertical_columns[-1] < i:
 426               vertical_columns.append(i)
 427
 428         if not line.strip('| \t'):
 429           continue
 430         if line.find('-') == -1:
 431           function_name = line.strip('| \t')
 432           node = CallTreeNode(function_name)
 433           last_node.add_child(node)
 434           last_node = node
 435           call_tree_stack[last_depth] = node
 436         else:
 437           pos = line.find('-')
 438           depth = -1
 439           for i in range(len(vertical_columns)):
 440             if pos >= vertical_columns[i]:
 441               depth = i
 442           assert depth != -1
 443
 444           line = line.strip('|- \t')
 445           m = re.search(r'^[\d\.]+%[-\s]+(.+)$', line)
 446           if m:
 447             function_name = m.group(1)
 448           else:
 449             function_name = line
 450
 451           node = CallTreeNode(function_name)
 452           if depth == 0:
 453             cur_symbol.set_call_tree(node)
 454
 455           else:
 456             call_tree_stack[depth - 1].add_child(node)
 457           call_tree_stack[depth] = node
 458           last_node = node
 459           last_depth = depth
 460
 461     return symbols
 462
 463   def check_report_file(self, test, report_file, has_callgraph):
 464     symbols = self._read_report_file(report_file, has_callgraph)
 465     if not self._check_symbol_overhead_requirements(test, symbols):
 466       return False
 467     if has_callgraph:
 468       if not self._check_symbol_children_overhead_requirements(test, symbols):
 469         return False
 470       if not self._check_symbol_relation_requirements(test, symbols):
 471         return False
 472     return True
 473
 474   def _check_symbol_overhead_requirements(self, test, symbols):
 475     result = True
 476     matched = [False] * len(test.symbol_overhead_requirements)
 477     matched_overhead = [0] * len(test.symbol_overhead_requirements)
 478     for symbol in symbols:
 479       for i in range(len(test.symbol_overhead_requirements)):
 480         req = test.symbol_overhead_requirements[i]
 481         if req.is_match(symbol):
 482           matched[i] = True
 483           matched_overhead[i] += symbol.overhead
 484     for i in range(len(matched)):
 485       if not matched[i]:
 486         print 'requirement (%s) has no matched symbol in test %s' % (
 487             test.symbol_overhead_requirements[i], test)
 488         result = False
 489       else:
 490         fulfilled = req.check_overhead(matched_overhead[i])
 491         if not fulfilled:
 492           print "Symbol (%s) doesn't match requirement (%s) in test %s" % (
 493               symbol, req, test)
 494           result = False
 495     return result
 496
 497   def _check_symbol_children_overhead_requirements(self, test, symbols):
 498     result = True
 499     matched = [False] * len(test.symbol_children_overhead_requirements)
 500     for symbol in symbols:
 501       for i in range(len(test.symbol_children_overhead_requirements)):
 502         req = test.symbol_children_overhead_requirements[i]
 503         if req.is_match(symbol):
 504           matched[i] = True
 505           fulfilled = req.check_overhead(symbol.children_overhead)
 506           if not fulfilled:
 507             print "Symbol (%s) doesn't match requirement (%s) in test %s" % (
 508                 symbol, req, test)
 509             result = False
 510     for i in range(len(matched)):
 511       if not matched[i]:
 512         print 'requirement (%s) has no matched symbol in test %s' % (
 513             test.symbol_children_overhead_requirements[i], test)
 514         result = False
 515     return result
 516
 517   def _check_symbol_relation_requirements(self, test, symbols):
 518     result = True
 519     matched = [False] * len(test.symbol_relation_requirements)
 520     for symbol in symbols:
 521       for i in range(len(test.symbol_relation_requirements)):
 522         req = test.symbol_relation_requirements[i]
 523         if req.is_match(symbol):
 524           matched[i] = True
 525           fulfilled = req.check_relation(symbol.call_tree)
 526           if not fulfilled:
 527             print "Symbol (%s) doesn't match requirement (%s) in test %s" % (
 528                 symbol, req, test)
 529             result = False
 530     for i in range(len(matched)):
 531       if not matched[i]:
 532         print 'requirement (%s) has no matched symbol in test %s' % (
 533             test.symbol_relation_requirements[i], test)
 534         result = False
 535     return result
 536
 537
 538 def build_runner(target, use_callgraph, sampler):
 539   if target == 'host32' and use_callgraph:
 540     print "Current 64bit linux host doesn't support `simpleperf32 record -g`"
 541     return None
 542   if target.startswith('host'):
 543     runner = HostRunner(target)
 544   else:
 545     runner = DeviceRunner(target)
 546   runner.use_callgraph = use_callgraph
 547   runner.sampler = sampler
 548   return runner
 549
 550
 551 def test_with_runner(runner, tests):
 552   report_analyzer = ReportAnalyzer()
 553   for test in tests:
 554     runner.record(test.executable_name, 'perf.data')
 555     if runner.sampler == 'inplace-sampler':
 556       # TODO: fix this when inplace-sampler actually works.
 557       runner.report('perf.data', 'perf.report')
 558       symbols = report_analyzer._read_report_file('perf.report', runner.use_callgraph)
 559       result = False
 560       if len(symbols) == 1 and symbols[0].name.find('FakeFunction()') != -1:
 561         result = True
 562     else:
 563       runner.report('perf.data', 'perf.report', additional_options = test.report_options)
 564       result = report_analyzer.check_report_file(test, 'perf.report', runner.use_callgraph)
 565     str = 'test %s on %s ' % (test.test_name, runner.target)
 566     if runner.use_callgraph:
 567       str += 'with call graph '
 568     str += 'using %s ' % runner.sampler
 569     str += ' Succeeded' if result else 'Failed'
 570     print str
 571     if not result:
 572       exit(1)
 573
 574
 575 def runtest(target_options, use_callgraph_options, sampler_options, selected_tests):
 576   tests = load_config_file(os.path.dirname(os.path.realpath(__file__)) + \
 577                            '/runtest.conf')
 578   if selected_tests is not None:
 579     new_tests = []
 580     for test in tests:
 581       if test.test_name in selected_tests:
 582         new_tests.append(test)
 583     tests = new_tests
 584   for target in target_options:
 585     for use_callgraph in use_callgraph_options:
 586       for sampler in sampler_options:
 587         runner = build_runner(target, use_callgraph, sampler)
 588         if runner is not None:
 589           test_with_runner(runner, tests)
 590
 591
 592 def main():
 593   target_options = ['host64', 'host32', 'device64', 'device32']
 594   use_callgraph_options = [False, True]
 595   sampler_options = ['cpu-cycles', 'inplace-sampler']
 596   selected_tests = None
 597   i = 1
 598   while i < len(sys.argv):
 599     if sys.argv[i] == '--host':
 600       target_options = ['host64', 'host32']
 601     elif sys.argv[i] == '--device':
 602       target_options = ['device64', 'device32']
 603     elif sys.argv[i] == '--normal':
 604       use_callgraph_options = [False]
 605     elif sys.argv[i] == '--callgraph':
 606       use_callgraph_options = [True]
 607     elif sys.argv[i] == '--no-inplace-sampler':
 608       sampler_options = ['cpu-cycles']
 609     elif sys.argv[i] == '--inplace-sampler':
 610       sampler_options = ['inplace-sampler']
 611     elif sys.argv[i] == '--test':
 612       if i < len(sys.argv):
 613         i += 1
 614         for test in sys.argv[i].split(','):
 615           if selected_tests is None:
 616             selected_tests = {}
 617           selected_tests[test] = True
 618     i += 1
 619   runtest(target_options, use_callgraph_options, sampler_options, selected_tests)
 620
 621 if __name__ == '__main__':
 622   main()