simpleperf/runtest/runtest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (C) 2015 The Android Open Source Project
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #      http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17 """Simpleperf runtest runner: run simpleperf runtests on host or on device.
  18
  19 For a simpleperf runtest like one_function test, it contains following steps:
  20 1. Run simpleperf record command to record simpleperf_runtest_one_function's
  21    running samples, which is generated in perf.data.
  22 2. Run simpleperf report command to parse perf.data, generate perf.report.
  23 4. Parse perf.report and see if it matches expectation.
  24
  25 The information of all runtests is stored in runtest.conf.
  26 """
  27
  28 import re
  29 import subprocess
  30 import sys
  31 import xml.etree.ElementTree as ET
  32
  33
  34 class CallTreeNode(object):
  35
  36   def __init__(self, name):
  37     self.name = name
  38     self.children = []
  39
  40   def add_child(self, child):
  41     self.children.append(child)
  42
  43   def __str__(self):
  44     return 'CallTreeNode:\n' + '\n'.join(self._dump(1))
  45
  46   def _dump(self, indent):
  47     indent_str = '  ' * indent
  48     strs = [indent_str + self.name]
  49     for child in self.children:
  50       strs.extend(child._dump(indent + 1))
  51     return strs
  52
  53
  54 class Symbol(object):
  55
  56   def __init__(self, name, comm, overhead, children_overhead):
  57     self.name = name
  58     self.comm = comm
  59     self.overhead = overhead
  60     # children_overhead is the overhead sum of this symbol and functions
  61     # called by this symbol.
  62     self.children_overhead = children_overhead
  63     self.call_tree = None
  64
  65   def set_call_tree(self, call_tree):
  66     self.call_tree = call_tree
  67
  68   def __str__(self):
  69     strs = []
  70     strs.append('Symbol name=%s comm=%s overhead=%f children_overhead=%f' % (
  71         self.name, self.comm, self.overhead, self.children_overhead))
  72     if self.call_tree:
  73       strs.append('\t%s' % self.call_tree)
  74     return '\n'.join(strs)
  75
  76
  77 class SymbolOverheadRequirement(object):
  78
  79   def __init__(self, symbol_name=None, comm=None, min_overhead=None,
  80                max_overhead=None):
  81     self.symbol_name = symbol_name
  82     self.comm = comm
  83     self.min_overhead = min_overhead
  84     self.max_overhead = max_overhead
  85
  86   def __str__(self):
  87     strs = []
  88     strs.append('SymbolOverheadRequirement')
  89     if self.symbol_name is not None:
  90       strs.append('symbol_name=%s' % self.symbol_name)
  91     if self.comm is not None:
  92       strs.append('comm=%s' % self.comm)
  93     if self.min_overhead is not None:
  94       strs.append('min_overhead=%f' % self.min_overhead)
  95     if self.max_overhead is not None:
  96       strs.append('max_overhead=%f' % self.max_overhead)
  97     return ' '.join(strs)
  98
  99   def is_match(self, symbol):
 100     if self.symbol_name is not None:
 101       if self.symbol_name != symbol.name:
 102         return False
 103     if self.comm is not None:
 104       if self.comm != symbol.comm:
 105         return False
 106     return True
 107
 108   def check_overhead(self, overhead):
 109     if self.min_overhead is not None:
 110       if self.min_overhead > overhead:
 111         return False
 112     if self.max_overhead is not None:
 113       if self.max_overhead < overhead:
 114         return False
 115     return True
 116
 117
 118 class SymbolRelationRequirement(object):
 119
 120   def __init__(self, symbol_name, comm=None):
 121     self.symbol_name = symbol_name
 122     self.comm = comm
 123     self.children = []
 124
 125   def add_child(self, child):
 126     self.children.append(child)
 127
 128   def __str__(self):
 129     return 'SymbolRelationRequirement:\n' + '\n'.join(self._dump(1))
 130
 131   def _dump(self, indent):
 132     indent_str = '  ' * indent
 133     strs = [indent_str + self.symbol_name +
 134             (' ' + self.comm if self.comm else '')]
 135     for child in self.children:
 136       strs.extend(child._dump(indent + 1))
 137     return strs
 138
 139   def is_match(self, symbol):
 140     if symbol.name != self.symbol_name:
 141       return False
 142     if self.comm is not None:
 143       if symbol.comm != self.comm:
 144         return False
 145     return True
 146
 147   def check_relation(self, call_tree):
 148     if not call_tree:
 149       return False
 150     if self.symbol_name != call_tree.name:
 151       return False
 152     for child in self.children:
 153       child_matched = False
 154       for node in call_tree.children:
 155         if child.check_relation(node):
 156           child_matched = True
 157           break
 158       if not child_matched:
 159         return False
 160     return True
 161
 162
 163 class Test(object):
 164
 165   def __init__(
 166           self,
 167           test_name,
 168           executable_name,
 169           report_options,
 170           symbol_overhead_requirements,
 171           symbol_children_overhead_requirements,
 172           symbol_relation_requirements):
 173     self.test_name = test_name
 174     self.executable_name = executable_name
 175     self.report_options = report_options
 176     self.symbol_overhead_requirements = symbol_overhead_requirements
 177     self.symbol_children_overhead_requirements = (
 178         symbol_children_overhead_requirements)
 179     self.symbol_relation_requirements = symbol_relation_requirements
 180
 181   def __str__(self):
 182     strs = []
 183     strs.append('Test test_name=%s' % self.test_name)
 184     strs.append('\texecutable_name=%s' % self.executable_name)
 185     strs.append('\treport_options=%s' % (' '.join(self.report_options)))
 186     strs.append('\tsymbol_overhead_requirements:')
 187     for req in self.symbol_overhead_requirements:
 188       strs.append('\t\t%s' % req)
 189     strs.append('\tsymbol_children_overhead_requirements:')
 190     for req in self.symbol_children_overhead_requirements:
 191       strs.append('\t\t%s' % req)
 192     strs.append('\tsymbol_relation_requirements:')
 193     for req in self.symbol_relation_requirements:
 194       strs.append('\t\t%s' % req)
 195     return '\n'.join(strs)
 196
 197
 198 def load_config_file(config_file):
 199   tests = []
 200   tree = ET.parse(config_file)
 201   root = tree.getroot()
 202   assert root.tag == 'runtests'
 203   for test in root:
 204     assert test.tag == 'test'
 205     test_name = test.attrib['name']
 206     executable_name = None
 207     report_options = []
 208     symbol_overhead_requirements = []
 209     symbol_children_overhead_requirements = []
 210     symbol_relation_requirements = []
 211     for test_item in test:
 212       if test_item.tag == 'executable':
 213         executable_name = test_item.attrib['name']
 214       elif test_item.tag == 'report':
 215         report_options = test_item.attrib['option'].split()
 216       elif (test_item.tag == 'symbol_overhead' or
 217               test_item.tag == 'symbol_children_overhead'):
 218         for symbol_item in test_item:
 219           assert symbol_item.tag == 'symbol'
 220           symbol_name = None
 221           if 'name' in symbol_item.attrib:
 222             symbol_name = symbol_item.attrib['name']
 223           comm = None
 224           if 'comm' in symbol_item.attrib:
 225             comm = symbol_item.attrib['comm']
 226           overhead_min = None
 227           if 'min' in symbol_item.attrib:
 228             overhead_min = float(symbol_item.attrib['min'])
 229           overhead_max = None
 230           if 'max' in symbol_item.attrib:
 231             overhead_max = float(symbol_item.attrib['max'])
 232
 233           if test_item.tag == 'symbol_overhead':
 234             symbol_overhead_requirements.append(
 235                 SymbolOverheadRequirement(
 236                     symbol_name,
 237                     comm,
 238                     overhead_min,
 239                     overhead_max)
 240             )
 241           else:
 242             symbol_children_overhead_requirements.append(
 243                 SymbolOverheadRequirement(
 244                     symbol_name,
 245                     comm,
 246                     overhead_min,
 247                     overhead_max))
 248       elif test_item.tag == 'symbol_callgraph_relation':
 249         for symbol_item in test_item:
 250           req = load_symbol_relation_requirement(symbol_item)
 251           symbol_relation_requirements.append(req)
 252
 253     tests.append(
 254         Test(
 255             test_name,
 256             executable_name,
 257             report_options,
 258             symbol_overhead_requirements,
 259             symbol_children_overhead_requirements,
 260             symbol_relation_requirements))
 261   return tests
 262
 263
 264 def load_symbol_relation_requirement(symbol_item):
 265   symbol_name = symbol_item.attrib['name']
 266   comm = None
 267   if 'comm' in symbol_item.attrib:
 268     comm = symbol_item.attrib['comm']
 269   req = SymbolRelationRequirement(symbol_name, comm)
 270   for item in symbol_item:
 271     child_req = load_symbol_relation_requirement(item)
 272     req.add_child(child_req)
 273   return req
 274
 275
 276 class Runner(object):
 277
 278   def __init__(self, perf_path):
 279     self.perf_path = perf_path
 280
 281   def record(self, test_executable_name, record_file, additional_options=[]):
 282     call_args = [self.perf_path,
 283                  'record'] + additional_options + ['-e',
 284                                                    'cpu-cycles:u',
 285                                                    '-o',
 286                                                    record_file,
 287                                                    test_executable_name]
 288     self._call(call_args)
 289
 290   def report(self, record_file, report_file, additional_options=[]):
 291     call_args = [self.perf_path,
 292                  'report'] + additional_options + ['-i',
 293                                                    record_file]
 294     self._call(call_args, report_file)
 295
 296   def _call(self, args, output_file=None):
 297     pass
 298
 299
 300 class HostRunner(Runner):
 301
 302   """Run perf test on host."""
 303
 304   def _call(self, args, output_file=None):
 305     output_fh = None
 306     if output_file is not None:
 307       output_fh = open(output_file, 'w')
 308     subprocess.check_call(args, stdout=output_fh)
 309     if output_fh is not None:
 310       output_fh.close()
 311
 312
 313 class DeviceRunner(Runner):
 314
 315   """Run perf test on device."""
 316
 317   def _call(self, args, output_file=None):
 318     output_fh = None
 319     if output_file is not None:
 320       output_fh = open(output_file, 'w')
 321     args_with_adb = ['adb', 'shell']
 322     args_with_adb.extend(args)
 323     subprocess.check_call(args_with_adb, stdout=output_fh)
 324     if output_fh is not None:
 325       output_fh.close()
 326
 327
 328 class ReportAnalyzer(object):
 329
 330   """Check if perf.report matches expectation in Configuration."""
 331
 332   def _read_report_file(self, report_file, has_callgraph):
 333     fh = open(report_file, 'r')
 334     lines = fh.readlines()
 335     fh.close()
 336
 337     lines = [x.rstrip() for x in lines]
 338     blank_line_index = -1
 339     for i in range(len(lines)):
 340       if not lines[i]:
 341         blank_line_index = i
 342     assert blank_line_index != -1
 343     assert blank_line_index + 1 < len(lines)
 344     title_line = lines[blank_line_index + 1]
 345     report_item_lines = lines[blank_line_index + 2:]
 346
 347     if has_callgraph:
 348       assert re.search(r'^Children\s+Self\s+Command.+Symbol$', title_line)
 349     else:
 350       assert re.search(r'^Overhead\s+Command.+Symbol$', title_line)
 351
 352     return self._parse_report_items(report_item_lines, has_callgraph)
 353
 354   def _parse_report_items(self, lines, has_callgraph):
 355     symbols = []
 356     cur_symbol = None
 357     call_tree_stack = {}
 358     vertical_columns = []
 359     last_node = None
 360     last_depth = -1
 361
 362     for line in lines:
 363       if not line:
 364         continue
 365       if not line[0].isspace():
 366         if has_callgraph:
 367           m = re.search(r'^([\d\.]+)%\s+([\d\.]+)%\s+(\S+).*\s+(\S+)$', line)
 368           children_overhead = float(m.group(1))
 369           overhead = float(m.group(2))
 370           comm = m.group(3)
 371           symbol_name = m.group(4)
 372           cur_symbol = Symbol(symbol_name, comm, overhead, children_overhead)
 373           symbols.append(cur_symbol)
 374         else:
 375           m = re.search(r'^([\d\.]+)%\s+(\S+).*\s+(\S+)$', line)
 376           overhead = float(m.group(1))
 377           comm = m.group(2)
 378           symbol_name = m.group(3)
 379           cur_symbol = Symbol(symbol_name, comm, overhead, 0)
 380           symbols.append(cur_symbol)
 381         # Each report item can have different column depths.
 382         vertical_columns = []
 383       else:
 384         for i in range(len(line)):
 385           if line[i] == '|':
 386             if not vertical_columns or vertical_columns[-1] < i:
 387               vertical_columns.append(i)
 388
 389         if not line.strip('| \t'):
 390           continue
 391         if line.find('-') == -1:
 392           function_name = line.strip('| \t')
 393           node = CallTreeNode(function_name)
 394           last_node.add_child(node)
 395           last_node = node
 396           call_tree_stack[last_depth] = node
 397         else:
 398           pos = line.find('-')
 399           depth = -1
 400           for i in range(len(vertical_columns)):
 401             if pos >= vertical_columns[i]:
 402               depth = i
 403           assert depth != -1
 404
 405           line = line.strip('|- \t')
 406           m = re.search(r'^[\d\.]+%[-\s]+(.+)$', line)
 407           if m:
 408             function_name = m.group(1)
 409           else:
 410             function_name = line
 411
 412           node = CallTreeNode(function_name)
 413           if depth == 0:
 414             cur_symbol.set_call_tree(node)
 415
 416           else:
 417             call_tree_stack[depth - 1].add_child(node)
 418           call_tree_stack[depth] = node
 419           last_node = node
 420           last_depth = depth
 421
 422     return symbols
 423
 424   def check_report_file(self, test, report_file, has_callgraph):
 425     symbols = self._read_report_file(report_file, has_callgraph)
 426     if not self._check_symbol_overhead_requirements(test, symbols):
 427       return False
 428     if has_callgraph:
 429       if not self._check_symbol_children_overhead_requirements(test, symbols):
 430         return False
 431       if not self._check_symbol_relation_requirements(test, symbols):
 432         return False
 433     return True
 434
 435   def _check_symbol_overhead_requirements(self, test, symbols):
 436     result = True
 437     matched = [False] * len(test.symbol_overhead_requirements)
 438     matched_overhead = [0] * len(test.symbol_overhead_requirements)
 439     for symbol in symbols:
 440       for i in range(len(test.symbol_overhead_requirements)):
 441         req = test.symbol_overhead_requirements[i]
 442         if req.is_match(symbol):
 443           matched[i] = True
 444           matched_overhead[i] += symbol.overhead
 445     for i in range(len(matched)):
 446       if not matched[i]:
 447         print 'requirement (%s) has no matched symbol in test %s' % (
 448             test.symbol_overhead_requirements[i], test)
 449         result = False
 450       else:
 451         fulfilled = req.check_overhead(matched_overhead[i])
 452         if not fulfilled:
 453           print "Symbol (%s) doesn't match requirement (%s) in test %s" % (
 454               symbol, req, test)
 455           result = False
 456     return result
 457
 458   def _check_symbol_children_overhead_requirements(self, test, symbols):
 459     result = True
 460     matched = [False] * len(test.symbol_children_overhead_requirements)
 461     for symbol in symbols:
 462       for i in range(len(test.symbol_children_overhead_requirements)):
 463         req = test.symbol_children_overhead_requirements[i]
 464         if req.is_match(symbol):
 465           matched[i] = True
 466           fulfilled = req.check_overhead(symbol.children_overhead)
 467           if not fulfilled:
 468             print "Symbol (%s) doesn't match requirement (%s) in test %s" % (
 469                 symbol, req, test)
 470             result = False
 471     for i in range(len(matched)):
 472       if not matched[i]:
 473         print 'requirement (%s) has no matched symbol in test %s' % (
 474             test.symbol_children_overhead_requirements[i], test)
 475         result = False
 476     return result
 477
 478   def _check_symbol_relation_requirements(self, test, symbols):
 479     result = True
 480     matched = [False] * len(test.symbol_relation_requirements)
 481     for symbol in symbols:
 482       for i in range(len(test.symbol_relation_requirements)):
 483         req = test.symbol_relation_requirements[i]
 484         if req.is_match(symbol):
 485           matched[i] = True
 486           fulfilled = req.check_relation(symbol.call_tree)
 487           if not fulfilled:
 488             print "Symbol (%s) doesn't match requirement (%s) in test %s" % (
 489                 symbol, req, test)
 490             result = False
 491     for i in range(len(matched)):
 492       if not matched[i]:
 493         print 'requirement (%s) has no matched symbol in test %s' % (
 494             test.symbol_relation_requirements[i], test)
 495         result = False
 496     return result
 497
 498
 499 def runtest(host, device, normal, callgraph, selected_tests):
 500   tests = load_config_file('runtest.conf')
 501   host_runner = HostRunner('simpleperf')
 502   device_runner = DeviceRunner('simpleperf')
 503   report_analyzer = ReportAnalyzer()
 504   for test in tests:
 505     if selected_tests is not None:
 506       if test.test_name not in selected_tests:
 507         continue
 508     if host and normal:
 509       host_runner.record(test.executable_name, 'perf.data')
 510       host_runner.report('perf.data', 'perf.report',
 511                          additional_options = test.report_options)
 512       result = report_analyzer.check_report_file(
 513           test, 'perf.report', False)
 514       print 'test %s on host %s' % (
 515           test.test_name, 'Succeeded' if result else 'Failed')
 516       if not result:
 517         exit(1)
 518
 519     if device and normal:
 520       device_runner.record(test.executable_name, '/data/perf.data')
 521       device_runner.report('/data/perf.data', 'perf.report',
 522                            additional_options = test.report_options)
 523       result = report_analyzer.check_report_file(test, 'perf.report', False)
 524       print 'test %s on device %s' % (
 525           test.test_name, 'Succeeded' if result else 'Failed')
 526       if not result:
 527         exit(1)
 528
 529     if host and callgraph:
 530       host_runner.record(
 531           test.executable_name,
 532           'perf_g.data',
 533           additional_options=['-g'])
 534       host_runner.report(
 535           'perf_g.data',
 536           'perf_g.report',
 537           additional_options=['-g'] + test.report_options)
 538       result = report_analyzer.check_report_file(test, 'perf_g.report', True)
 539       print 'call-graph test %s on host %s' % (
 540           test.test_name, 'Succeeded' if result else 'Failed')
 541       if not result:
 542         exit(1)
 543
 544     if device and callgraph:
 545       device_runner.record(
 546           test.executable_name,
 547           '/data/perf_g.data',
 548           additional_options=['-g'])
 549       device_runner.report(
 550           '/data/perf_g.data',
 551           'perf_g.report',
 552           additional_options=['-g'] + test.report_options)
 553       result = report_analyzer.check_report_file(test, 'perf_g.report', True)
 554       print 'call-graph test %s on device %s' % (
 555           test.test_name, 'Succeeded' if result else 'Failed')
 556       if not result:
 557         exit(1)
 558
 559 def main():
 560   host = True
 561   device = True
 562   normal = True
 563   callgraph = True
 564   selected_tests = None
 565   i = 1
 566   while i < len(sys.argv):
 567     if sys.argv[i] == '--host':
 568       host = True
 569       device = False
 570     elif sys.argv[i] == '--device':
 571       host = False
 572       device = True
 573     elif sys.argv[i] == '--normal':
 574       normal = True
 575       callgraph = False
 576     elif sys.argv[i] == '--callgraph':
 577       normal = False
 578       callgraph = True
 579     elif sys.argv[i] == '--test':
 580       if i < len(sys.argv):
 581         i += 1
 582         for test in sys.argv[i].split(','):
 583           if selected_tests is None:
 584             selected_tests = {}
 585           selected_tests[test] = True
 586     i += 1
 587   runtest(host, device, normal, callgraph, selected_tests)
 588
 589 if __name__ == '__main__':
 590   main()