From 15d3fef0da03a8548fd25ec75381e86eea5c6882 Mon Sep 17 00:00:00 2001 From: Christopher Ferris Date: Fri, 24 Feb 2017 12:26:48 -0800 Subject: [PATCH] Refactor the kernel update scripts. The previous versions of the scripts did a lot of redundant changes and were hard to follow. I rewrote most of update_all.py so that it's clear about what's going on. I updated clean_header.py to change the cleanupFile function so that there is no magic about where the destination file is going to wind up. Now the caller specifies the final location. I updated utils.py so that if you are trying to do an update in one location, but your lunch target is from another location, it causes an error. Bug: 35726570 Change-Id: Ic5a44d90c2774a627eecde34c0c403bc925a497c Test: Ran the updater and verified it works properly. Test: Verified that doing an update in one tree to another tree Test: fails. --- libc/kernel/tools/clean_header.py | 74 ++++++++----------- libc/kernel/tools/update_all.py | 148 ++++++++++++++++---------------------- libc/kernel/tools/utils.py | 8 ++- 3 files changed, 95 insertions(+), 135 deletions(-) diff --git a/libc/kernel/tools/clean_header.py b/libc/kernel/tools/clean_header.py index 99f4c7f58..f474f745e 100755 --- a/libc/kernel/tools/clean_header.py +++ b/libc/kernel/tools/clean_header.py @@ -79,55 +79,33 @@ def print_error(no_update, msg): sys.stderr.write("warning: " + msg) -def cleanupFile(dst_dir, src_dir, rel_path, no_update = True): +def cleanupFile(dst_file, src_file, rel_path, no_update = True): """reads an original header and perform the cleanup operation on it this functions returns the destination path and the clean header as a single string""" - # check the header path - full_path = os.path.join(src_dir, rel_path) - - if not os.path.exists(full_path): - print_error(no_update, "file does not exist: '%s'\n" % full_path) + # Check the header path + if not os.path.exists(src_file): + print_error(no_update, "'%s' does not exist\n" % src_file) return None, None - if not os.path.isfile(full_path): - print_error(no_update, "path is not a file: '%s'\n" % full_path) + if not os.path.isfile(src_file): + print_error(no_update, "'%s' is not a file\n" % src_file) return None, None - # convert into destination path, extracting architecture if needed - # and the corresponding list of known static functions - # + # Extract the architecture if found. arch = None statics = kernel_known_generic_statics - m = re.match(r"asm-([\w\d_\+\.\-]+)(/.*)", rel_path) - if m and m.group(1) != 'generic': - dst_path = "arch-%s/asm/%s" % m.groups() - arch = m.group(1) - statics = statics.union(kernel_known_statics.get(arch, set())) - else: - # process headers under the uapi directory - # note the "asm" level has been explicitly added in the original - # kernel header tree for architectural-dependent uapi headers - m_uapi = re.match(r"(uapi)/([\w\d_\+\.\-]+)(/.*)", rel_path) - if m_uapi: - dst_path = rel_path - m_uapi_arch = re.match(r"asm-([\w\d_\+\.\-]+)", m_uapi.group(2)) - if m_uapi_arch and m_uapi_arch.group(1) != 'generic': - arch = m_uapi_arch.group(1) - statics = statics.union(kernel_known_statics.get(arch, set())) - # common headers (ie non-asm and non-uapi) - else: - dst_path = os.path.join("android", rel_path) + m = re.search(r"(^|/)asm-([\w\d_\+\.\-]+)/.*", rel_path) + if m and m.group(2) != 'generic': + arch = m.group(2) + statics = statics.union(kernel_known_statics.get(arch, set())) - dst_path = os.path.join(dst_dir, dst_path) - - # now, let's parse the file - # + # Now, let's parse the file. parser = cpp.BlockParser() - blocks = parser.parseFile(full_path) + blocks = parser.parseFile(src_file) if not parser.parsed: - print_error(no_update, "can't parse '%s'%" % full_path) - return None, None + print_error(no_update, "Can't parse '%s'" % src_file) + return None macros = kernel_known_macros.copy() if arch and arch in kernel_default_arch_macros: @@ -145,7 +123,7 @@ def cleanupFile(dst_dir, src_dir, rel_path, no_update = True): out = StringOutput() out.write(kernel_disclaimer) blocks.writeWithWarning(out, kernel_warning, 4) - return dst_path, out.get() + return out.get() if __name__ == "__main__": @@ -194,22 +172,26 @@ if __name__ == "__main__": if no_update: for path in args: - dst_path, newdata = cleanupFile(dst_dir, src_dir, path) - print newdata + dst_file = os.path.join(dst_dir, path) + src_file = os.path.join(src_dir, path) + new_data = cleanupFile(dst_file, src_file, path) + print new_data sys.exit(0) - # now let's update our files. + # Now let's update our files. b = BatchFileUpdater() for path in args: - dst_path, newdata = cleanupFile(dst_dir, src_dir, path, no_update) - if not dst_path: + dst_file = os.path.join(dst_dir, path) + src_file = os.path.join(src_dir, path) + new_data = cleanupFile(dst_file, src_file, path, no_update) + if not new_data: continue - b.readFile(dst_path) - r = b.editFile(dst_path, newdata) + b.readFile(path) + r = b.editFile(path, new_data) if r == 0: r = "unchanged" elif r == 1: @@ -217,7 +199,7 @@ if __name__ == "__main__": else: r = "added" - print "cleaning: %-*s -> %-*s (%s)" % (35, path, 35, dst_path, r) + print "cleaning: %-*s -> %-*s (%s)" % (35, path, 35, path, r) b.updateGitFiles() diff --git a/libc/kernel/tools/update_all.py b/libc/kernel/tools/update_all.py index 5031168ce..e5a07f8c2 100755 --- a/libc/kernel/tools/update_all.py +++ b/libc/kernel/tools/update_all.py @@ -24,105 +24,77 @@ def usage(): """ % { "progname" : os.path.basename(sys.argv[0]) } sys.exit(0) +def processFiles(updater, original_dir, modified_dir, src_rel_dir, update_rel_dir): + # Delete the old headers before updating to the new headers. + update_dir = os.path.join(get_kernel_dir(), update_rel_dir) + shutil.rmtree(update_dir) + os.mkdir(update_dir, 0755) + + src_dir = os.path.normpath(os.path.join(original_dir, src_rel_dir)) + src_dir_len = len(src_dir) + 1 + mod_src_dir = os.path.join(modified_dir, src_rel_dir) + update_dir = os.path.join(get_kernel_dir(), update_rel_dir) + + kernel_dir = get_kernel_dir() + for root, _, files in os.walk(src_dir): + for file in sorted(files): + _, ext = os.path.splitext(file) + if ext != ".h": + continue + src_file = os.path.normpath(os.path.join(root, file)) + rel_path = src_file[src_dir_len:] + # Check to see if there is a modified header to use instead. + if os.path.exists(os.path.join(mod_src_dir, rel_path)): + src_file = os.path.join(mod_src_dir, rel_path) + src_str = os.path.join("", src_rel_dir, rel_path) + else: + src_str = os.path.join("", src_rel_dir, rel_path) + dst_file = os.path.join(update_dir, rel_path) + new_data = clean_header.cleanupFile(dst_file, src_file, rel_path) + if not new_data: + continue + updater.readFile(dst_file) + ret_val = updater.editFile(dst_file, new_data) + if ret_val == 0: + state = "unchanged" + elif ret_val == 1: + state = "edited" + else: + state = "added" + update_path = os.path.join(update_rel_dir, rel_path) + print "cleaning %s -> %s (%s)" % (src_str, update_path, state) + try: optlist, args = getopt.getopt(sys.argv[1:], '') except: - # unrecognized option + # Unrecognized option sys.stderr.write("error: unrecognized option\n") usage() if len(optlist) > 0 or len(args) > 2: usage() -modified_dir = get_kernel_headers_modified_dir() -if len(args) == 1 or len(args) == 2: +if len(args) > 0: original_dir = args[0] - if not os.path.isdir(original_dir): - panic("Not a directory: %s\n" % original_dir) - - if len(args) == 2: - modified_dir = args[1] - if not os.path.isdir(modified_dir): - panic("Not a directory: %s\n" % modified_dir) else: original_dir = get_kernel_headers_original_dir() - if not os.path.isdir(original_dir): - panic("Missing directory, please specify one through command-line: %s\n" % original_dir) + +if len(args) > 1: + modified_dir = args[1] +else: + modified_dir = get_kernel_headers_modified_dir() + +if not os.path.isdir(original_dir): + panic("The kernel directory %s is not a directory\n" % original_dir) if not os.path.isdir(modified_dir): - modified_dir = None - -# Find all source files in 'original'. -sources = dict() -original_dir = os.path.normpath(original_dir) -original_dir_len = len(original_dir) + 1 -for root, _, files in os.walk(original_dir): - for file in files: - _, ext = os.path.splitext(file) - if ext == ".h": - rel_path = os.path.normpath(os.path.join(root, file)) - rel_path = rel_path[original_dir_len:] - # Check to see if there is a modified header to use instead. - if modified_dir and os.path.exists(os.path.join(modified_dir, rel_path)): - sources[rel_path] = False - else: - sources[rel_path] = True - - -b = BatchFileUpdater() - -kernel_dir = get_kernel_dir() -for arch in kernel_archs: - b.readDir(os.path.join(kernel_dir, "arch-%s" % arch)) - -b.readDir(os.path.join(kernel_dir, "android")) - -# Delete the old uapi headers before updating to handle headers that -# get moved/deleted. -uapi_dir = os.path.join(get_kernel_dir(), "uapi") -shutil.rmtree(uapi_dir) -os.mkdir(uapi_dir, 0755) - -oldlen = 120 -android_root_len = len(get_android_root()) + 1 -for rel_path in sorted(sources): - if sources[rel_path]: - src_dir = original_dir - src_str = "/" - else: - src_dir = modified_dir - src_str = "/" - dst_path, newdata = clean_header.cleanupFile(kernel_dir, src_dir, rel_path) - if not dst_path: - continue - - dst_path = os.path.join(kernel_dir, dst_path) - b.readFile(dst_path) - r = b.editFile(dst_path, newdata) - if r == 0: - state = "unchanged" - elif r == 1: - state = "edited" - else: - state = "added" - - # dst_path is guaranteed to include android root. - rel_dst_path = dst_path[android_root_len:] - str = "cleaning: %-*s -> %-*s (%s)" % (35, src_str + rel_path, 35, rel_dst_path, state) - if sys.stdout.isatty(): - print "%-*s" % (oldlen, str), - if (r == 0): - print "\r", - else: - print "\n", - oldlen = 0 - else: - print str - - oldlen = len(str) - -print "%-*s" % (oldlen, "Done!") - -b.updateGitFiles() - -sys.exit(0) + panic("The kernel modified directory %s is not a directory\n" % modified_dir) + +updater = BatchFileUpdater() +# Process the original uapi headers first. +processFiles(updater, original_dir, modified_dir, "uapi", "uapi"), + +# Now process the special files. +processFiles(updater, original_dir, modified_dir, "scsi", os.path.join("android", "scsi")) + +updater.updateGitFiles() diff --git a/libc/kernel/tools/utils.py b/libc/kernel/tools/utils.py index e2cc9ce10..1b06b1b48 100644 --- a/libc/kernel/tools/utils.py +++ b/libc/kernel/tools/utils.py @@ -31,8 +31,14 @@ def get_kernel_dir(): def get_android_root(): if "ANDROID_BUILD_TOP" in os.environ: + # Verify that the current directory is in the root. + # If not, then print an error. + cwd = os.getcwd() + root = os.environ["ANDROID_BUILD_TOP"] + if len(cwd) < len(root) or not root == cwd[:len(root)]: + panic("Not in android tree pointed at by ANDROID_BUILD_TOP (%s)\n" % root) return os.environ["ANDROID_BUILD_TOP"] - panic("Unable to find root of tree, did you forget to lunch a target?") + panic("Unable to find root of tree, did you forget to lunch a target?\n") class StringOutput: -- 2.11.0