From 16dde31614b73822fba3606345e463703558bb92 Mon Sep 17 00:00:00 2001 From: Yabin Cui Date: Thu, 16 Aug 2018 16:40:35 -0700 Subject: [PATCH] python: run tests for both python and python3. This is to reduce failures like https://github.com/android-ndk/ndk/issues/740. Also fix some places for python3. Bug: none Test: run test.py. Change-Id: I751ccbc9940efa8727ec7b19710d452896049762 --- simpleperf/scripts/app_profiler.py | 9 ++--- simpleperf/scripts/debug_unwind_reporter.py | 4 +-- simpleperf/scripts/pprof_proto_generator.py | 8 ++--- simpleperf/scripts/report.py | 1 + simpleperf/scripts/simpleperf_report_lib.py | 2 +- simpleperf/scripts/test.py | 51 +++++++++++++++++++++-------- simpleperf/scripts/utils.py | 4 +-- 7 files changed, 52 insertions(+), 27 deletions(-) diff --git a/simpleperf/scripts/app_profiler.py b/simpleperf/scripts/app_profiler.py index b7d41154..66615af8 100755 --- a/simpleperf/scripts/app_profiler.py +++ b/simpleperf/scripts/app_profiler.py @@ -29,8 +29,8 @@ import subprocess import sys import time -from utils import get_script_dir, log_debug, log_info, log_exit, get_target_binary_path, extant_dir -from utils import AdbHelper, ReadElf, remove +from utils import AdbHelper, bytes_to_str, extant_dir, get_script_dir, get_target_binary_path +from utils import log_debug, log_info, log_exit, ReadElf, remove, str_to_bytes NATIVE_LIBS_DIR_ON_DEVICE = '/data/local/tmp/native_libs/' @@ -121,7 +121,7 @@ class NativeLibDownloader(object): if os.path.exists(self.build_id_list_file): with open(self.build_id_list_file, 'rb') as fh: for line in fh.readlines(): - line = line.strip() + line = bytes_to_str(line).strip() items = line.split('=') if len(items) == 2: self.device_build_id_map[items[0]] = items[1] @@ -141,7 +141,8 @@ class NativeLibDownloader(object): # Push new build_id_list on device. with open(self.build_id_list_file, 'wb') as fh: for build_id in self.host_build_id_map: - fh.write('%s=%s\n' % (build_id, self.host_build_id_map[build_id].name)) + s = str_to_bytes('%s=%s\n' % (build_id, self.host_build_id_map[build_id].name)) + fh.write(s) self.adb.check_run(['push', self.build_id_list_file, self.dir_on_device + self.build_id_list_file]) os.remove(self.build_id_list_file) diff --git a/simpleperf/scripts/debug_unwind_reporter.py b/simpleperf/scripts/debug_unwind_reporter.py index 72853e9c..b60dea60 100755 --- a/simpleperf/scripts/debug_unwind_reporter.py +++ b/simpleperf/scripts/debug_unwind_reporter.py @@ -44,8 +44,7 @@ import copy import re import subprocess -from utils import log_exit, log_fatal -from utils import get_host_binary_path +from utils import bytes_to_str, log_exit, log_fatal, get_host_binary_path class MapEntry(object): @@ -381,6 +380,7 @@ def build_unwinding_result_report(args): proc = subprocess.Popen([simpleperf_path, 'dump', args.record_file[0]], stdout=subprocess.PIPE) (stdoutdata, _) = proc.communicate() + stdoutdata = bytes_to_str(stdoutdata) if 'debug_unwind = true' not in stdoutdata: log_exit("Can't parse unwinding result. Because " + "%s was not generated by the debug-unwind cmd." % args.record_file[0]) diff --git a/simpleperf/scripts/pprof_proto_generator.py b/simpleperf/scripts/pprof_proto_generator.py index b236c550..d0b4da95 100755 --- a/simpleperf/scripts/pprof_proto_generator.py +++ b/simpleperf/scripts/pprof_proto_generator.py @@ -30,8 +30,8 @@ import os import os.path from simpleperf_report_lib import ReportLib -from utils import log_info, log_exit -from utils import Addr2Nearestline, extant_dir, find_tool_path, flatten_arg_list +from utils import Addr2Nearestline, bytes_to_str, extant_dir, find_tool_path, flatten_arg_list +from utils import log_info, log_exit, str_to_bytes try: import profile_pb2 except ImportError: @@ -40,13 +40,13 @@ except ImportError: def load_pprof_profile(filename): profile = profile_pb2.Profile() with open(filename, "rb") as f: - profile.ParseFromString(f.read()) + profile.ParseFromString(bytes_to_str(f.read())) return profile def store_pprof_profile(filename, profile): with open(filename, 'wb') as f: - f.write(profile.SerializeToString()) + f.write(str_to_bytes(profile.SerializeToString())) class PprofProfilePrinter(object): diff --git a/simpleperf/scripts/report.py b/simpleperf/scripts/report.py index 76f56ff1..ae99e25f 100755 --- a/simpleperf/scripts/report.py +++ b/simpleperf/scripts/report.py @@ -302,6 +302,7 @@ def get_simpleperf_report_help_msg(): args = [simpleperf_path, 'report', '-h'] proc = subprocess.Popen(args, stdout=subprocess.PIPE) (stdoutdata, _) = proc.communicate() + stdoutdata = bytes_to_str(stdoutdata) return stdoutdata[stdoutdata.find('\n') + 1:] diff --git a/simpleperf/scripts/simpleperf_report_lib.py b/simpleperf/scripts/simpleperf_report_lib.py index d2bc2d39..ab45b67a 100644 --- a/simpleperf/scripts/simpleperf_report_lib.py +++ b/simpleperf/scripts/simpleperf_report_lib.py @@ -105,7 +105,7 @@ class TracingFieldFormatStruct(ct.Structure): if self.elem_count > 1 and self.elem_size == 1 and self.is_signed == 0: # The field is a string. length = 0 - while length < self.elem_count and data[self.offset + length] != '\x00': + while length < self.elem_count and bytes_to_str(data[self.offset + length]) != '\x00': length += 1 return bytes_to_str(data[self.offset : self.offset + length]) unpack_key = self._unpack_key_dict.get(self.elem_size) diff --git a/simpleperf/scripts/test.py b/simpleperf/scripts/test.py index 2eb6cc73..a3ef42fc 100755 --- a/simpleperf/scripts/test.py +++ b/simpleperf/scripts/test.py @@ -53,8 +53,8 @@ from app_profiler import NativeLibDownloader from binary_cache_builder import BinaryCacheBuilder from simpleperf_report_lib import ReportLib from utils import log_exit, log_info, log_fatal -from utils import AdbHelper, Addr2Nearestline, find_tool_path, get_script_dir, is_windows, Objdump -from utils import ReadElf, remove, SourceFileSearcher +from utils import AdbHelper, Addr2Nearestline, bytes_to_str, find_tool_path, get_script_dir +from utils import is_python3, is_windows, Objdump, ReadElf, remove, SourceFileSearcher try: # pylint: disable=unused-import @@ -114,6 +114,7 @@ class TestBase(unittest.TestCase): else: subproc = subprocess.Popen(args, stdout=subprocess.PIPE, shell=use_shell) (output_data, _) = subproc.communicate() + output_data = bytes_to_str(output_data) returncode = subproc.returncode except OSError: returncode = None @@ -1206,7 +1207,7 @@ class TestNativeLibDownloader(unittest.TestCase): downloader.collect_native_libs_on_device() self.assertEqual(len(downloader.device_build_id_map), 0) - lib_list = downloader.host_build_id_map.items() + lib_list = list(downloader.host_build_id_map.items()) for sync_count in [0, 1, 2]: build_id_map = {} for i in range(sync_count): @@ -1228,8 +1229,8 @@ class TestNativeLibDownloader(unittest.TestCase): if sync_count == 1: adb.run(['pull', '/data/local/tmp/native_libs/build_id_list', 'build_id_list']) with open('build_id_list', 'rb') as fh: - self.assertEqual(fh.read(), '{}={}\n'.format(lib_list[0][0], - lib_list[0][1].name)) + self.assertEqual(bytes_to_str(fh.read()), + '{}={}\n'.format(lib_list[0][0], lib_list[0][1].name)) remove('build_id_list') adb.run(['shell', 'rm', '-rf', '/data/local/tmp/native_libs']) @@ -1276,16 +1277,27 @@ def get_all_tests(): tests = [] for name, value in globals().items(): if isinstance(value, type) and issubclass(value, unittest.TestCase): - test_methods = [x for x, y in inspect.getmembers(value) - if isinstance(y, types.UnboundMethodType) and x.startswith('test')] - for method in test_methods: - tests.append(name + '.' + method) + for member_name, member in inspect.getmembers(value): + if isinstance(member, (types.MethodType, types.FunctionType)): + if member_name.startswith('test'): + tests.append(name + '.' + member_name) return sorted(tests) + +def run_tests(tests): + os.chdir(get_script_dir()) + build_testdata() + log_info('Run tests with python%d\n%s' % (3 if is_python3() else 2, '\n'.join(tests))) + argv = [sys.argv[0]] + tests + unittest.main(argv=argv, failfast=True, verbosity=2, exit=False) + + def main(): parser = argparse.ArgumentParser(description='Test simpleperf scripts') parser.add_argument('--list-tests', action='store_true', help='List all tests.') parser.add_argument('--test-from', nargs=1, help='Run left tests from the selected test.') + parser.add_argument('--python-version', choices=['2', '3', 'both'], default='both', help=""" + Run tests on which python versions.""") parser.add_argument('pattern', nargs='*', help='Run tests matching the selected pattern.') args = parser.parse_args() tests = get_all_tests() @@ -1309,14 +1321,25 @@ def main(): if not tests: log_exit('No tests are matched.') - os.chdir(get_script_dir()) - build_testdata() if AdbHelper().get_android_version() < 7: log_info("Skip tests on Android version < N.") sys.exit(0) - log_info('Run tests %s' % ('\n'.join(tests))) - argv = [sys.argv[0]] + tests - unittest.main(argv=argv, failfast=True, verbosity=2) + + if args.python_version == 'both': + python_versions = [2, 3] + else: + python_versions = [int(args.python_version)] + current_version = 3 if is_python3() else 2 + for version in python_versions: + if version != current_version: + argv = ['python3' if version == 3 else 'python'] + argv.append(os.path.join(get_script_dir(), 'test.py')) + argv += sys.argv[1:] + argv += ['--python-version', str(version)] + subprocess.check_call(argv) + else: + run_tests(tests) + if __name__ == '__main__': main() diff --git a/simpleperf/scripts/utils.py b/simpleperf/scripts/utils.py index 7ae6cc98..20ea5c5c 100644 --- a/simpleperf/scripts/utils.py +++ b/simpleperf/scripts/utils.py @@ -78,7 +78,7 @@ def str_to_bytes(str_value): return str_value.encode('utf-8') def bytes_to_str(bytes_value): - if not is_python3(): + if not is_python3() or not bytes_value: return bytes_value return bytes_value.decode('utf-8') @@ -239,10 +239,10 @@ class AdbHelper(object): else: subproc = subprocess.Popen(adb_args, stdout=subprocess.PIPE) (stdoutdata, _) = subproc.communicate() + stdoutdata = bytes_to_str(stdoutdata) returncode = subproc.returncode result = (returncode == 0) if stdoutdata and adb_args[1] != 'push' and adb_args[1] != 'pull': - stdoutdata = bytes_to_str(stdoutdata) if log_output: log_debug(stdoutdata) log_debug('run adb cmd: %s [result %s]' % (adb_args, result)) -- 2.11.0