From 08800fd905e70faf01d9392d00ff3f49d99097b7 Mon Sep 17 00:00:00 2001 From: Ying Wang Date: Thu, 3 Mar 2016 20:57:21 -0800 Subject: [PATCH] Speed up lunch/tapas/etc. shell utility functions. 1. Combined ~10 calls to the make build system to only one. We added a phony target "dump-many-vars" to the build system to dump "=" pairs. We then store the pairs as shell variables. With this cache get_build_var/get_abs_build_var can just return the shell variables instead of querying the build system. 2. Prune .git when we search for AndroidProduct.mks. In internal source tree lunch time was reduced from ~15s to ~1.5s. Bug: 27429759 Change-Id: I24e88598f6fab598ef26518885fd5e86e71a772d --- core/dumpvar.mk | 89 +++++++++++++++++--------- core/envsetup.mk | 2 + core/product.mk | 13 +++- envsetup.sh | 189 ++++++++++++++++++++++++++++++++++++++----------------- 4 files changed, 202 insertions(+), 91 deletions(-) diff --git a/core/dumpvar.mk b/core/dumpvar.mk index cfb031ff9..36fd08a7c 100644 --- a/core/dumpvar.mk +++ b/core/dumpvar.mk @@ -1,3 +1,35 @@ + +# List of variables we want to print in the build banner. +print_build_config_vars := \ + PLATFORM_VERSION_CODENAME \ + PLATFORM_VERSION \ + TARGET_PRODUCT \ + TARGET_BUILD_VARIANT \ + TARGET_BUILD_TYPE \ + TARGET_BUILD_APPS \ + TARGET_ARCH \ + TARGET_ARCH_VARIANT \ + TARGET_CPU_VARIANT \ + TARGET_2ND_ARCH \ + TARGET_2ND_ARCH_VARIANT \ + TARGET_2ND_CPU_VARIANT \ + HOST_ARCH \ + HOST_2ND_ARCH \ + HOST_OS \ + HOST_OS_EXTRA \ + HOST_CROSS_OS \ + HOST_CROSS_ARCH \ + HOST_CROSS_2ND_ARCH \ + HOST_BUILD_TYPE \ + BUILD_ID \ + OUT_DIR + +ifeq ($(TARGET_BUILD_PDK),true) +print_build_config_vars += \ + TARGET_BUILD_PDK \ + PDK_FUSION_PLATFORM_ZIP +endif + # --------------------------------------------------------------- # the setpath shell function in envsetup.sh uses this to figure out # what to add to the path given the config we have chosen. @@ -59,38 +91,37 @@ ifneq ($(dumpvar_goals),report_config) PRINT_BUILD_CONFIG:= endif -endif # CALLED_FROM_SETUP +ifneq ($(filter report_config,$(DUMP_MANY_VARS)),) +# Construct the shell commands that print the config banner. +report_config_sh := echo '============================================'; +report_config_sh += $(foreach v,$(print_build_config_vars),echo '$v=$($(v))';) +report_config_sh += echo '============================================'; +endif +# Dump mulitple variables to "=" pairs, one per line. +# The output may be executed as bash script. +# Input variables: +# DUMP_MANY_VARS: the list of variable names. +# DUMP_VAR_PREFIX: an optional prefix of the variable name added to the output. +# DUMP_MANY_ABS_VARS: the list of abs variable names. +# DUMP_ABS_VAR_PREFIX: an optional prefix of the abs variable name added to the output. +.PHONY: dump-many-vars +dump-many-vars : + @$(foreach v, $(filter-out report_config, $(DUMP_MANY_VARS)),\ + echo "$(DUMP_VAR_PREFIX)$(v)='$($(v))'";) +ifneq ($(filter report_config, $(DUMP_MANY_VARS)),) + @# Construct a special variable for report_config. + @# Escape \` to defer the execution of report_config_sh to preserve the line breaks. + @echo "$(DUMP_VAR_PREFIX)report_config=\`$(report_config_sh)\`" +endif + @$(foreach v, $(sort $(DUMP_MANY_ABS_VARS)),\ + echo "$(DUMP_ABS_VAR_PREFIX)$(v)='$(PWD)/$($(v))'";) + +endif # CALLED_FROM_SETUP ifneq ($(PRINT_BUILD_CONFIG),) -HOST_OS_EXTRA:=$(shell python -c "import platform; print(platform.platform())") $(info ============================================) -$(info PLATFORM_VERSION_CODENAME=$(PLATFORM_VERSION_CODENAME)) -$(info PLATFORM_VERSION=$(PLATFORM_VERSION)) -$(info TARGET_PRODUCT=$(TARGET_PRODUCT)) -$(info TARGET_BUILD_VARIANT=$(TARGET_BUILD_VARIANT)) -$(info TARGET_BUILD_TYPE=$(TARGET_BUILD_TYPE)) -$(info TARGET_BUILD_APPS=$(TARGET_BUILD_APPS)) -$(info TARGET_ARCH=$(TARGET_ARCH)) -$(info TARGET_ARCH_VARIANT=$(TARGET_ARCH_VARIANT)) -$(info TARGET_CPU_VARIANT=$(TARGET_CPU_VARIANT)) -$(info TARGET_2ND_ARCH=$(TARGET_2ND_ARCH)) -$(info TARGET_2ND_ARCH_VARIANT=$(TARGET_2ND_ARCH_VARIANT)) -$(info TARGET_2ND_CPU_VARIANT=$(TARGET_2ND_CPU_VARIANT)) -$(info HOST_ARCH=$(HOST_ARCH)) -$(info HOST_2ND_ARCH=$(HOST_2ND_ARCH)) -$(info HOST_OS=$(HOST_OS)) -$(info HOST_OS_EXTRA=$(HOST_OS_EXTRA)) -$(info HOST_CROSS_OS=$(HOST_CROSS_OS)) -$(info HOST_CROSS_ARCH=$(HOST_CROSS_ARCH)) -$(info HOST_CROSS_2ND_ARCH=$(HOST_CROSS_2ND_ARCH)) -$(info HOST_BUILD_TYPE=$(HOST_BUILD_TYPE)) -$(info BUILD_ID=$(BUILD_ID)) -$(info OUT_DIR=$(OUT_DIR)) -ifeq ($(TARGET_BUILD_PDK),true) -$(info TARGET_BUILD_PDK=$(TARGET_BUILD_PDK)) -$(info PDK_FUSION_PLATFORM_ZIP=$(PDK_FUSION_PLATFORM_ZIP)) -endif - +$(foreach v, $(print_build_config_vars),\ + $(info $v=$($(v)))) $(info ============================================) endif diff --git a/core/envsetup.mk b/core/envsetup.mk index 4456809f7..e8fa6a76a 100644 --- a/core/envsetup.mk +++ b/core/envsetup.mk @@ -54,6 +54,8 @@ ifneq (,$(findstring Macintosh,$(UNAME))) HOST_OS := darwin endif +HOST_OS_EXTRA:=$(shell python -c "import platform; print(platform.platform())") + # BUILD_OS is the real host doing the build. BUILD_OS := $(HOST_OS) diff --git a/core/product.mk b/core/product.mk index 4d35704f6..7043cffa3 100644 --- a/core/product.mk +++ b/core/product.mk @@ -23,14 +23,21 @@ # and the .mk suffix) of the product makefile, ":" can be # omitted. +# Search for AndroidProducts.mks in the given dir. +# $(1): the path to the dir +define _search-android-products-files-in-dir +$(sort $(shell test -d $(1) && find -L $(1) \ + -maxdepth 6 \ + -name .git -prune \ + -o -name AndroidProducts.mk -print)) +endef + # # Returns the list of all AndroidProducts.mk files. # $(call ) isn't necessary. # define _find-android-products-files -$(sort $(shell test -d device && find -L device -maxdepth 6 -name AndroidProducts.mk)) \ - $(sort $(shell test -d vendor && find -L vendor -maxdepth 6 -name AndroidProducts.mk)) \ - $(sort $(shell test -d product && find -L product -maxdepth 6 -name AndroidProducts.mk)) \ +$(foreach d, device vendor product,$(call _search-android-products-files-in-dir,$(d))) \ $(SRC_TARGET_DIR)/product/AndroidProducts.mk endef diff --git a/envsetup.sh b/envsetup.sh index 9489d21e0..2aac383b5 100644 --- a/envsetup.sh +++ b/envsetup.sh @@ -37,9 +37,62 @@ EOF echo $A } +# Get all the build variables needed by this script in a single call to the build system. +function build_build_var_cache() +{ + T=$(gettop) + # Grep out the variable names from the script. + cached_vars=`cat $T/build/envsetup.sh | tr '()' ' ' | awk '{for(i=1;i<=NF;i++) if($i~/get_build_var/) print $(i+1)}' | sort -u | tr '\n' ' '` + cached_abs_vars=`cat $T/build/envsetup.sh | tr '()' ' ' | awk '{for(i=1;i<=NF;i++) if($i~/get_abs_build_var/) print $(i+1)}' | sort -u | tr '\n' ' '` + # Call the build system to dump the "=" pairs as a shell script. + build_dicts_script=`\cd $T; CALLED_FROM_SETUP=true BUILD_SYSTEM=build/core \ + command make --no-print-directory -f build/core/config.mk \ + dump-many-vars \ + DUMP_MANY_VARS="$cached_vars" \ + DUMP_MANY_ABS_VARS="$cached_abs_vars" \ + DUMP_VAR_PREFIX="var_cache_" \ + DUMP_ABS_VAR_PREFIX="abs_var_cache_"` + local ret=$? + if [ $ret -ne 0 ] + then + unset build_dicts_script + return $ret + fi + # Excute the script to store the "=" pairs as shell variables. + eval "$build_dicts_script" + unset build_dicts_script + ret=$? + if [ $ret -ne 0 ] + then + return $ret + fi + BUILD_VAR_CACHE_READY="true" +} + +# Delete the build cache, so that we can still call into the build system +# to get build variables not listed in this script. +function destroy_build_var_cache() +{ + unset BUILD_VAR_CACHE_READY + for v in $cached_vars; do + unset var_cache_$v + done + unset cached_vars + for v in $cached_abs_vars; do + unset abs_var_cache_$v + done + unset cached_abs_vars +} + # Get the value of a build variable as an absolute path. function get_abs_build_var() { + if [ "$BUILD_VAR_CACHE_READY" = "true" ] + then + eval echo \"\${abs_var_cache_$1}\" + return + fi + T=$(gettop) if [ ! "$T" ]; then echo "Couldn't locate the top of the tree. Try setting TOP." >&2 @@ -52,6 +105,12 @@ function get_abs_build_var() # Get the exact value of a build variable. function get_build_var() { + if [ "$BUILD_VAR_CACHE_READY" = "true" ] + then + eval echo \"\${var_cache_$1}\" + return + fi + T=$(gettop) if [ ! "$T" ]; then echo "Couldn't locate the top of the tree. Try setting TOP." >&2 @@ -321,7 +380,9 @@ function choosetype() fi done + build_build_var_cache set_stuff_for_environment + destroy_build_var_cache } # @@ -338,6 +399,7 @@ function chooseproduct() default_value=aosp_arm fi + export TARGET_BUILD_APPS= export TARGET_PRODUCT= local ANSWER while [ -z "$TARGET_PRODUCT" ] @@ -365,7 +427,9 @@ function chooseproduct() fi done + build_build_var_cache set_stuff_for_environment + destroy_build_var_cache } function choosevariant() @@ -428,8 +492,10 @@ function choosecombo() choosevariant $3 echo + build_build_var_cache set_stuff_for_environment printconfig + destroy_build_var_cache } # Clear this variable. It will be built up again when the vendorsetup.sh @@ -511,16 +577,6 @@ function lunch() export TARGET_BUILD_APPS= - local product=$(echo -n $selection | sed -e "s/-.*$//") - check_product $product - if [ $? -ne 0 ] - then - echo - echo "** Don't have a product spec for: '$product'" - echo "** Do you have the right repo manifest?" - product= - fi - local variant=$(echo -n $selection | sed -e "s/^[^\-]*-//") check_variant $variant if [ $? -ne 0 ] @@ -531,6 +587,18 @@ function lunch() variant= fi + local product=$(echo -n $selection | sed -e "s/-.*$//") + TARGET_PRODUCT=$product \ + TARGET_BUILD_VARIANT=$variant \ + build_build_var_cache + if [ $? -ne 0 ] + then + echo + echo "** Don't have a product spec for: '$product'" + echo "** Do you have the right repo manifest?" + product= + fi + if [ -z "$product" -o -z "$variant" ] then echo @@ -545,6 +613,7 @@ function lunch() set_stuff_for_environment printconfig + destroy_build_var_cache } # Tab completion for lunch. @@ -607,8 +676,10 @@ function tapas() export TARGET_BUILD_TYPE=release export TARGET_BUILD_APPS=$apps + build_build_var_cache set_stuff_for_environment printconfig + destroy_build_var_cache } function gettop @@ -878,18 +949,18 @@ function qpid() { append='$' shift elif [ "$1" = "--help" -o "$1" = "-h" ]; then - echo "usage: qpid [[--exact] " - return 255 - fi + echo "usage: qpid [[--exact] " + return 255 + fi local EXE="$1" if [ "$EXE" ] ; then - qpid | \grep "$prepend$EXE$append" - else - adb shell ps \ - | tr -d '\r' \ - | sed -e 1d -e 's/^[^ ]* *\([0-9]*\).* \([^ ]*\)$/\1 \2/' - fi + qpid | \grep "$prepend$EXE$append" + else + adb shell ps \ + | tr -d '\r' \ + | sed -e 1d -e 's/^[^ ]* *\([0-9]*\).* \([^ ]*\)$/\1 \2/' + fi } function pid() @@ -910,7 +981,7 @@ function pid() echo "$PID" else echo "usage: pid [--exact] " - return 255 + return 255 fi } @@ -923,25 +994,25 @@ function pid() function coredump_setup() { - echo "Getting root..."; - adb root; - adb wait-for-device; + echo "Getting root..."; + adb root; + adb wait-for-device; - echo "Remounting root partition read-write..."; - adb shell mount -w -o remount -t rootfs rootfs; - sleep 1; - adb wait-for-device; - adb shell mkdir -p /cores; - adb shell mount -t tmpfs tmpfs /cores; - adb shell chmod 0777 /cores; + echo "Remounting root partition read-write..."; + adb shell mount -w -o remount -t rootfs rootfs; + sleep 1; + adb wait-for-device; + adb shell mkdir -p /cores; + adb shell mount -t tmpfs tmpfs /cores; + adb shell chmod 0777 /cores; - echo "Granting SELinux permission to dump in /cores..."; - adb shell restorecon -R /cores; + echo "Granting SELinux permission to dump in /cores..."; + adb shell restorecon -R /cores; - echo "Set core pattern."; - adb shell 'echo /cores/core.%p > /proc/sys/kernel/core_pattern'; + echo "Set core pattern."; + adb shell 'echo /cores/core.%p > /proc/sys/kernel/core_pattern'; - echo "Done." + echo "Done." } # coredump_enable - enable core dumps for the specified process @@ -952,13 +1023,13 @@ function coredump_setup() function coredump_enable() { - local PID=$1; - if [ -z "$PID" ]; then - printf "Expecting a PID!\n"; - return; - fi; - echo "Setting core limit for $PID to infinite..."; - adb shell prlimit $PID 4 -1 -1 + local PID=$1; + if [ -z "$PID" ]; then + printf "Expecting a PID!\n"; + return; + fi; + echo "Setting core limit for $PID to infinite..."; + adb shell prlimit $PID 4 -1 -1 } # core - send SIGV and pull the core for process @@ -969,28 +1040,28 @@ function coredump_enable() function core() { - local PID=$1; + local PID=$1; - if [ -z "$PID" ]; then - printf "Expecting a PID!\n"; - return; - fi; + if [ -z "$PID" ]; then + printf "Expecting a PID!\n"; + return; + fi; - local CORENAME=core.$PID; - local COREPATH=/cores/$CORENAME; - local SIG=SEGV; + local CORENAME=core.$PID; + local COREPATH=/cores/$CORENAME; + local SIG=SEGV; - coredump_enable $1; + coredump_enable $1; - local done=0; - while [ $(adb shell "[ -d /proc/$PID ] && echo -n yes") ]; do - printf "\tSending SIG%s to %d...\n" $SIG $PID; - adb shell kill -$SIG $PID; - sleep 1; - done; + local done=0; + while [ $(adb shell "[ -d /proc/$PID ] && echo -n yes") ]; do + printf "\tSending SIG%s to %d...\n" $SIG $PID; + adb shell kill -$SIG $PID; + sleep 1; + done; - adb shell "while [ ! -f $COREPATH ] ; do echo waiting for $COREPATH to be generated; sleep 1; done" - echo "Done: core is under $COREPATH on device."; + adb shell "while [ ! -f $COREPATH ] ; do echo waiting for $COREPATH to be generated; sleep 1; done" + echo "Done: core is under $COREPATH on device."; } # systemstack - dump the current stack trace of all threads in the system process -- 2.11.0