simpleperf/runtest/runtest.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright (C) 2015 The Android Open Source Project
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #      http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17 """Simpleperf runtest runner: run simpleperf runtests on host or on device.
  18
  19 For a simpleperf runtest like one_function test, it contains following steps:
  20 1. Run simpleperf record command to record simpleperf_runtest_one_function's
  21    running samples, which is generated in perf.data.
  22 2. Run simpleperf report command to parse perf.data, generate perf.report.
  23 4. Parse perf.report and see if it matches expectation.
  24
  25 The information of all runtests is stored in runtest.conf.
  26 """
  27
  28 import re
  29 import subprocess
  30 import xml.etree.ElementTree as ET
  31
  32
  33 class CallTreeNode(object):
  34
  35   def __init__(self, name):
  36     self.name = name
  37     self.children = []
  38
  39   def add_child(self, child):
  40     self.children.append(child)
  41
  42   def __str__(self):
  43     return 'CallTreeNode:\n' + '\n'.join(self._dump(1))
  44
  45   def _dump(self, indent):
  46     indent_str = '  ' * indent
  47     strs = [indent_str + self.name]
  48     for child in self.children:
  49       strs.extend(child._dump(indent + 1))
  50     return strs
  51
  52
  53 class Symbol(object):
  54
  55   def __init__(self, name, comm, overhead, children_overhead):
  56     self.name = name
  57     self.comm = comm
  58     self.overhead = overhead
  59     # children_overhead is the overhead sum of this symbol and functions
  60     # called by this symbol.
  61     self.children_overhead = children_overhead
  62     self.call_tree = None
  63
  64   def set_call_tree(self, call_tree):
  65     self.call_tree = call_tree
  66
  67   def __str__(self):
  68     strs = []
  69     strs.append('Symbol name=%s comm=%s overhead=%f children_overhead=%f' % (
  70         self.name, self.comm, self.overhead, self.children_overhead))
  71     if self.call_tree:
  72       strs.append('\t%s' % self.call_tree)
  73     return '\n'.join(strs)
  74
  75
  76 class SymbolOverheadRequirement(object):
  77
  78   def __init__(self, symbol_name, comm=None, min_overhead=None,
  79                max_overhead=None):
  80     self.symbol_name = symbol_name
  81     self.comm = comm
  82     self.min_overhead = min_overhead
  83     self.max_overhead = max_overhead
  84
  85   def __str__(self):
  86     strs = []
  87     strs.append('SymbolOverheadRequirement symbol_name=%s' %
  88                 self.symbol_name)
  89     if self.comm is not None:
  90       strs.append('comm=%s' % self.comm)
  91     if self.min_overhead is not None:
  92       strs.append('min_overhead=%f' % self.min_overhead)
  93     if self.max_overhead is not None:
  94       strs.append('max_overhead=%f' % self.max_overhead)
  95     return ' '.join(strs)
  96
  97   def is_match(self, symbol):
  98     if symbol.name != self.symbol_name:
  99       return False
 100     if self.comm is not None:
 101       if self.comm != symbol.comm:
 102         return False
 103     return True
 104
 105   def check_overhead(self, overhead):
 106     if self.min_overhead is not None:
 107       if self.min_overhead > overhead:
 108         return False
 109     if self.max_overhead is not None:
 110       if self.max_overhead < overhead:
 111         return False
 112     return True
 113
 114
 115 class SymbolRelationRequirement(object):
 116
 117   def __init__(self, symbol_name, comm=None):
 118     self.symbol_name = symbol_name
 119     self.comm = comm
 120     self.children = []
 121
 122   def add_child(self, child):
 123     self.children.append(child)
 124
 125   def __str__(self):
 126     return 'SymbolRelationRequirement:\n' + '\n'.join(self._dump(1))
 127
 128   def _dump(self, indent):
 129     indent_str = '  ' * indent
 130     strs = [indent_str + self.symbol_name +
 131             (' ' + self.comm if self.comm else '')]
 132     for child in self.children:
 133       strs.extend(child._dump(indent + 1))
 134     return strs
 135
 136   def is_match(self, symbol):
 137     if symbol.name != self.symbol_name:
 138       return False
 139     if self.comm is not None:
 140       if symbol.comm != self.comm:
 141         return False
 142     return True
 143
 144   def check_relation(self, call_tree):
 145     if not call_tree:
 146       return False
 147     if self.symbol_name != call_tree.name:
 148       return False
 149     for child in self.children:
 150       child_matched = False
 151       for node in call_tree.children:
 152         if child.check_relation(node):
 153           child_matched = True
 154           break
 155       if not child_matched:
 156         return False
 157     return True
 158
 159
 160 class Test(object):
 161
 162   def __init__(
 163           self,
 164           test_name,
 165           executable_name,
 166           symbol_overhead_requirements,
 167           symbol_children_overhead_requirements,
 168           symbol_relation_requirements):
 169     self.test_name = test_name
 170     self.executable_name = executable_name
 171     self.symbol_overhead_requirements = symbol_overhead_requirements
 172     self.symbol_children_overhead_requirements = (
 173         symbol_children_overhead_requirements)
 174     self.symbol_relation_requirements = symbol_relation_requirements
 175
 176   def __str__(self):
 177     strs = []
 178     strs.append('Test test_name=%s' % self.test_name)
 179     strs.append('\texecutable_name=%s' % self.executable_name)
 180     strs.append('\tsymbol_overhead_requirements:')
 181     for req in self.symbol_overhead_requirements:
 182       strs.append('\t\t%s' % req)
 183     strs.append('\tsymbol_children_overhead_requirements:')
 184     for req in self.symbol_children_overhead_requirements:
 185       strs.append('\t\t%s' % req)
 186     strs.append('\tsymbol_relation_requirements:')
 187     for req in self.symbol_relation_requirements:
 188       strs.append('\t\t%s' % req)
 189     return '\n'.join(strs)
 190
 191
 192 def load_config_file(config_file):
 193   tests = []
 194   tree = ET.parse(config_file)
 195   root = tree.getroot()
 196   assert root.tag == 'runtests'
 197   for test in root:
 198     assert test.tag == 'test'
 199     test_name = test.attrib['name']
 200     executable_name = None
 201     symbol_overhead_requirements = []
 202     symbol_children_overhead_requirements = []
 203     symbol_relation_requirements = []
 204     for test_item in test:
 205       if test_item.tag == 'executable':
 206         executable_name = test_item.attrib['name']
 207       elif (test_item.tag == 'symbol_overhead' or
 208               test_item.tag == 'symbol_children_overhead'):
 209         for symbol_item in test_item:
 210           assert symbol_item.tag == 'symbol'
 211           symbol_name = symbol_item.attrib['name']
 212           comm = None
 213           if 'comm' in symbol_item.attrib:
 214             comm = symbol_item.attrib['comm']
 215           overhead_min = None
 216           if 'min' in symbol_item.attrib:
 217             overhead_min = float(symbol_item.attrib['min'])
 218           overhead_max = None
 219           if 'max' in symbol_item.attrib:
 220             overhead_max = float(symbol_item.attrib['max'])
 221
 222           if test_item.tag == 'symbol_overhead':
 223             symbol_overhead_requirements.append(
 224                 SymbolOverheadRequirement(
 225                     symbol_name,
 226                     comm,
 227                     overhead_min,
 228                     overhead_max)
 229             )
 230           else:
 231             symbol_children_overhead_requirements.append(
 232                 SymbolOverheadRequirement(
 233                     symbol_name,
 234                     comm,
 235                     overhead_min,
 236                     overhead_max))
 237       elif test_item.tag == 'symbol_callgraph_relation':
 238         for symbol_item in test_item:
 239           req = load_symbol_relation_requirement(symbol_item)
 240           symbol_relation_requirements.append(req)
 241
 242     tests.append(
 243         Test(
 244             test_name,
 245             executable_name,
 246             symbol_overhead_requirements,
 247             symbol_children_overhead_requirements,
 248             symbol_relation_requirements))
 249   return tests
 250
 251
 252 def load_symbol_relation_requirement(symbol_item):
 253   symbol_name = symbol_item.attrib['name']
 254   comm = None
 255   if 'comm' in symbol_item.attrib:
 256     comm = symbol_item.attrib['comm']
 257   req = SymbolRelationRequirement(symbol_name, comm)
 258   for item in symbol_item:
 259     child_req = load_symbol_relation_requirement(item)
 260     req.add_child(child_req)
 261   return req
 262
 263
 264 class Runner(object):
 265
 266   def __init__(self, perf_path):
 267     self.perf_path = perf_path
 268
 269   def record(self, test_executable_name, record_file, additional_options=[]):
 270     call_args = [self.perf_path,
 271                  'record'] + additional_options + ['-e',
 272                                                    'cpu-cycles:u',
 273                                                    '-o',
 274                                                    record_file,
 275                                                    test_executable_name]
 276     self._call(call_args)
 277
 278   def report(self, record_file, report_file, additional_options=[]):
 279     call_args = [self.perf_path,
 280                  'report'] + additional_options + ['-i',
 281                                                    record_file]
 282     self._call(call_args, report_file)
 283
 284   def _call(self, args, output_file=None):
 285     pass
 286
 287
 288 class HostRunner(Runner):
 289
 290   """Run perf test on host."""
 291
 292   def _call(self, args, output_file=None):
 293     output_fh = None
 294     if output_file is not None:
 295       output_fh = open(output_file, 'w')
 296     subprocess.check_call(args, stdout=output_fh)
 297     if output_fh is not None:
 298       output_fh.close()
 299
 300
 301 class DeviceRunner(Runner):
 302
 303   """Run perf test on device."""
 304
 305   def _call(self, args, output_file=None):
 306     output_fh = None
 307     if output_file is not None:
 308       output_fh = open(output_file, 'w')
 309     args_with_adb = ['adb', 'shell']
 310     args_with_adb.extend(args)
 311     subprocess.check_call(args_with_adb, stdout=output_fh)
 312     if output_fh is not None:
 313       output_fh.close()
 314
 315
 316 class ReportAnalyzer(object):
 317
 318   """Check if perf.report matches expectation in Configuration."""
 319
 320   def _read_report_file(self, report_file, has_callgraph):
 321     fh = open(report_file, 'r')
 322     lines = fh.readlines()
 323     fh.close()
 324
 325     lines = [x.rstrip() for x in lines]
 326     blank_line_index = -1
 327     for i in range(len(lines)):
 328       if not lines[i]:
 329         blank_line_index = i
 330     assert blank_line_index != -1
 331     assert blank_line_index + 1 < len(lines)
 332     title_line = lines[blank_line_index + 1]
 333     report_item_lines = lines[blank_line_index + 2:]
 334
 335     if has_callgraph:
 336       assert re.search(r'^Children\s+Self\s+Command.+Symbol$', title_line)
 337     else:
 338       assert re.search(r'^Overhead\s+Command.+Symbol$', title_line)
 339
 340     return self._parse_report_items(report_item_lines, has_callgraph)
 341
 342   def _parse_report_items(self, lines, has_callgraph):
 343     symbols = []
 344     cur_symbol = None
 345     call_tree_stack = {}
 346     vertical_columns = []
 347     last_node = None
 348     last_depth = -1
 349
 350     for line in lines:
 351       if not line:
 352         continue
 353       if not line[0].isspace():
 354         if has_callgraph:
 355           m = re.search(r'^([\d\.]+)%\s+([\d\.]+)%\s+(\S+).*\s+(\S+)$', line)
 356           children_overhead = float(m.group(1))
 357           overhead = float(m.group(2))
 358           comm = m.group(3)
 359           symbol_name = m.group(4)
 360           cur_symbol = Symbol(symbol_name, comm, overhead, children_overhead)
 361           symbols.append(cur_symbol)
 362         else:
 363           m = re.search(r'^([\d\.]+)%\s+(\S+).*\s+(\S+)$', line)
 364           overhead = float(m.group(1))
 365           comm = m.group(2)
 366           symbol_name = m.group(3)
 367           cur_symbol = Symbol(symbol_name, comm, overhead, 0)
 368           symbols.append(cur_symbol)
 369         # Each report item can have different column depths.
 370         vertical_columns = []
 371       else:
 372         for i in range(len(line)):
 373           if line[i] == '|':
 374             if not vertical_columns or vertical_columns[-1] < i:
 375               vertical_columns.append(i)
 376
 377         if not line.strip('| \t'):
 378           continue
 379         if line.find('-') == -1:
 380           function_name = line.strip('| \t')
 381           node = CallTreeNode(function_name)
 382           last_node.add_child(node)
 383           last_node = node
 384           call_tree_stack[last_depth] = node
 385         else:
 386           pos = line.find('-')
 387           depth = -1
 388           for i in range(len(vertical_columns)):
 389             if pos >= vertical_columns[i]:
 390               depth = i
 391           assert depth != -1
 392
 393           line = line.strip('|- \t')
 394           m = re.search(r'^[\d\.]+%[-\s]+(.+)$', line)
 395           if m:
 396             function_name = m.group(1)
 397           else:
 398             function_name = line
 399
 400           node = CallTreeNode(function_name)
 401           if depth == 0:
 402             cur_symbol.set_call_tree(node)
 403
 404           else:
 405             call_tree_stack[depth - 1].add_child(node)
 406           call_tree_stack[depth] = node
 407           last_node = node
 408           last_depth = depth
 409
 410     return symbols
 411
 412   def check_report_file(self, test, report_file, has_callgraph):
 413     symbols = self._read_report_file(report_file, has_callgraph)
 414     if not self._check_symbol_overhead_requirements(test, symbols):
 415       return False
 416     if has_callgraph:
 417       if not self._check_symbol_children_overhead_requirements(test, symbols):
 418         return False
 419       if not self._check_symbol_relation_requirements(test, symbols):
 420         return False
 421     return True
 422
 423   def _check_symbol_overhead_requirements(self, test, symbols):
 424     result = True
 425     matched = [False] * len(test.symbol_overhead_requirements)
 426     for symbol in symbols:
 427       for i in range(len(test.symbol_overhead_requirements)):
 428         req = test.symbol_overhead_requirements[i]
 429         if req.is_match(symbol):
 430           matched[i] = True
 431           fulfilled = req.check_overhead(symbol.overhead)
 432           if not fulfilled:
 433             print "Symbol (%s) doesn't match requirement (%s) in test %s" % (
 434                 symbol, req, test)
 435             result = False
 436     for i in range(len(matched)):
 437       if not matched[i]:
 438         print 'requirement (%s) has no matched symbol in test %s' % (
 439             test.symbol_overhead_requirements[i], test)
 440         result = False
 441     return result
 442
 443   def _check_symbol_children_overhead_requirements(self, test, symbols):
 444     result = True
 445     matched = [False] * len(test.symbol_children_overhead_requirements)
 446     for symbol in symbols:
 447       for i in range(len(test.symbol_children_overhead_requirements)):
 448         req = test.symbol_children_overhead_requirements[i]
 449         if req.is_match(symbol):
 450           matched[i] = True
 451           fulfilled = req.check_overhead(symbol.children_overhead)
 452           if not fulfilled:
 453             print "Symbol (%s) doesn't match requirement (%s) in test %s" % (
 454                 symbol, req, test)
 455             result = False
 456     for i in range(len(matched)):
 457       if not matched[i]:
 458         print 'requirement (%s) has no matched symbol in test %s' % (
 459             test.symbol_children_overhead_requirements[i], test)
 460         result = False
 461     return result
 462
 463   def _check_symbol_relation_requirements(self, test, symbols):
 464     result = True
 465     matched = [False] * len(test.symbol_relation_requirements)
 466     for symbol in symbols:
 467       for i in range(len(test.symbol_relation_requirements)):
 468         req = test.symbol_relation_requirements[i]
 469         if req.is_match(symbol):
 470           matched[i] = True
 471           fulfilled = req.check_relation(symbol.call_tree)
 472           if not fulfilled:
 473             print "Symbol (%s) doesn't match requirement (%s) in test %s" % (
 474                 symbol, req, test)
 475             result = False
 476     for i in range(len(matched)):
 477       if not matched[i]:
 478         print 'requirement (%s) has no matched symbol in test %s' % (
 479             test.symbol_relation_requirements[i], test)
 480         result = False
 481     return result
 482
 483
 484 def main():
 485   tests = load_config_file('runtest.conf')
 486   host_runner = HostRunner('simpleperf')
 487   device_runner = DeviceRunner('simpleperf')
 488   report_analyzer = ReportAnalyzer()
 489   for test in tests:
 490     host_runner.record(test.executable_name, 'perf.data')
 491     host_runner.report('perf.data', 'perf.report')
 492     result = report_analyzer.check_report_file(
 493         test, 'perf.report', False)
 494     print 'test %s on host %s' % (
 495         test.test_name, 'Succeeded' if result else 'Failed')
 496     if not result:
 497       exit(1)
 498     device_runner.record(test.executable_name, '/data/perf.data')
 499     device_runner.report('/data/perf.data', 'perf.report')
 500     result = report_analyzer.check_report_file(test, 'perf.report', False)
 501     print 'test %s on device %s' % (
 502         test.test_name, 'Succeeded' if result else 'Failed')
 503     if not result:
 504       exit(1)
 505
 506     host_runner.record(
 507         test.executable_name,
 508         'perf_g.data',
 509         additional_options=['-g'])
 510     host_runner.report(
 511         'perf_g.data',
 512         'perf_g.report',
 513         additional_options=['-g'])
 514     result = report_analyzer.check_report_file(test, 'perf_g.report', True)
 515     print 'call-graph test %s on host %s' % (
 516         test.test_name, 'Succeeded' if result else 'Failed')
 517     if not result:
 518       exit(1)
 519
 520     device_runner.record(
 521         test.executable_name,
 522         '/data/perf_g.data',
 523         additional_options=['-g'])
 524     device_runner.report(
 525         '/data/perf_g.data',
 526         'perf_g.report',
 527         additional_options=['-g'])
 528     result = report_analyzer.check_report_file(test, 'perf_g.report', True)
 529     print 'call-graph test %s on device %s' % (
 530         test.test_name, 'Succeeded' if result else 'Failed')
 531     if not result:
 532       exit(1)
 533
 534
 535 if __name__ == '__main__':
 536   main()