From b5a77b91f3b52ff7de5bb3be2fb8c420f6968cb6 Mon Sep 17 00:00:00 2001 From: Shubham Ajmera Date: Wed, 7 Dec 2016 19:39:29 +0000 Subject: [PATCH] ART Test Runner The testrunner runs the ART run-tests by simply invoking the script. It fetches the list of eligible tests from art/test directory, and list of disabled tests from art/test/knownfailures.json. It runs the tests by invoking art/test/run-test script and parses it output to check if the test passed or failed. Before invoking the script, first build all the tests dependencies by building 'test-art-host-run-test-dependencies' for host tests, 'test-art-target-run-test-dependencies' for target tests, and 'test-art-run-test-dependencies' for building dependencies for both. There are various options to invoke the script which are: -t: Either the test name as in art/test or the test name including the variant information. Eg, "-t 001-HelloWorld", "-t test-art-host-run-test-debug-prebuild-optimizing-relocate-ntrace-cms-checkjni-picimage-npictest-ndebuggable-001-HelloWorld32" -j: Number of thread workers to be used. Eg - "-j64" --dry-run: Instead of running the test name, just print its name. --verbose -b To build the dependencies before running the test. To specify any specific variants for the test, use --<>. For eg, for compiler type as optimizing, use --optimizing. In the end, the script will print the failed and skipped tests if any. Changes in the Android.run-test.mk 1) Test targets like "test-art-host-run-test-debug-prebuild-jit-relocate-ntrace-cms-checkjni-picimage-npictest-ndebuggable-055-enum-performance64" 2) New targets - test-art-host-run-test-dependencies and test-art-target-run-test-dependencies have been added. The targets build dependencies for host and target respectively. 3) Old targets like test-art-host-run-test-optimizing, test-art-host-run-test build the dependencies using the above targets and then invokes the testrunner to run the tests. Test: No tests Change-Id: I1823aa1cbfe3637a3c6ae8eb5843709fa77cee09 --- test/Android.run-test.mk | 1181 +++++------------------------------------ test/knownfailures.json | 359 +++++++++++++ test/testrunner/env.py | 174 ++++++ test/testrunner/testrunner.py | 794 +++++++++++++++++++++++++++ 4 files changed, 1469 insertions(+), 1039 deletions(-) create mode 100644 test/knownfailures.json create mode 100644 test/testrunner/env.py create mode 100755 test/testrunner/testrunner.py diff --git a/test/Android.run-test.mk b/test/Android.run-test.mk index 742353da4..770fc2110 100644 --- a/test/Android.run-test.mk +++ b/test/Android.run-test.mk @@ -14,7 +14,6 @@ # LOCAL_PATH := $(call my-dir) - include art/build/Android.common_test.mk # List of all tests of the form 003-omnibus-opcodes. @@ -107,689 +106,52 @@ TEST_ART_RUN_TEST_BUILD_RULES := # General rules to build and run a run-test. TARGET_TYPES := host target -PREBUILD_TYPES := -ifeq ($(ART_TEST_RUN_TEST_PREBUILD),true) - PREBUILD_TYPES += prebuild -endif -ifeq ($(ART_TEST_RUN_TEST_NO_PREBUILD),true) - PREBUILD_TYPES += no-prebuild -endif -ifeq ($(ART_TEST_RUN_TEST_NO_DEX2OAT),true) - PREBUILD_TYPES += no-dex2oat -endif -COMPILER_TYPES := -ifeq ($(ART_TEST_INTERPRETER_ACCESS_CHECKS),true) - COMPILER_TYPES += interp-ac -endif -ifeq ($(ART_TEST_INTERPRETER),true) - COMPILER_TYPES += interpreter -endif -ifeq ($(ART_TEST_JIT),true) - COMPILER_TYPES += jit -endif -OPTIMIZING_COMPILER_TYPES := -ifeq ($(ART_TEST_OPTIMIZING),true) - COMPILER_TYPES += optimizing - OPTIMIZING_COMPILER_TYPES += optimizing -endif -ifeq ($(ART_TEST_OPTIMIZING_GRAPH_COLOR),true) - COMPILER_TYPES += regalloc_gc - OPTIMIZING_COMPILER_TYPES += regalloc_gc -endif -RELOCATE_TYPES := no-relocate -ifeq ($(ART_TEST_RUN_TEST_RELOCATE),true) - RELOCATE_TYPES += relocate -endif -ifeq ($(ART_TEST_RUN_TEST_RELOCATE_NO_PATCHOAT),true) - RELOCATE_TYPES += relocate-npatchoat -endif -TRACE_TYPES := ntrace -ifeq ($(ART_TEST_TRACE),true) - TRACE_TYPES += trace -endif -ifeq ($(ART_TEST_TRACE_STREAM),true) - TRACE_TYPES += stream -endif -GC_TYPES := cms -ifeq ($(ART_TEST_GC_STRESS),true) - GC_TYPES += gcstress -endif -ifeq ($(ART_TEST_GC_VERIFY),true) - GC_TYPES += gcverify -endif -JNI_TYPES := checkjni -ifeq ($(ART_TEST_JNI_FORCECOPY),true) - JNI_TYPES += forcecopy -endif -ifeq ($(ART_TEST_RUN_TEST_IMAGE),true) -IMAGE_TYPES := picimage -endif -ifeq ($(ART_TEST_RUN_TEST_NO_IMAGE),true) - IMAGE_TYPES += no-image -endif -ifeq ($(ART_TEST_RUN_TEST_MULTI_IMAGE),true) - IMAGE_TYPES := multipicimage -endif -ifeq ($(ART_TEST_NPIC_IMAGE),true) - IMAGE_TYPES += npicimage - ifeq ($(ART_TEST_RUN_TEST_MULTI_IMAGE),true) - IMAGE_TYPES := multinpicimage - endif -endif -PICTEST_TYPES := npictest -ifeq ($(ART_TEST_PIC_TEST),true) - PICTEST_TYPES += pictest -endif -RUN_TYPES := -ifeq ($(ART_TEST_RUN_TEST_DEBUG),true) - RUN_TYPES += debug -endif -ifeq ($(ART_TEST_RUN_TEST_NDEBUG),true) - RUN_TYPES += ndebug -endif -DEBUGGABLE_TYPES := ndebuggable -ifeq ($(ART_TEST_RUN_TEST_DEBUGGABLE),true) -DEBUGGABLE_TYPES += debuggable -endif -ADDRESS_SIZES_TARGET := $(ART_PHONY_TEST_TARGET_SUFFIX) -ADDRESS_SIZES_HOST := $(ART_PHONY_TEST_HOST_SUFFIX) -ifeq ($(ART_TEST_RUN_TEST_2ND_ARCH),true) - ADDRESS_SIZES_TARGET += $(2ND_ART_PHONY_TEST_TARGET_SUFFIX) - ADDRESS_SIZES_HOST += $(2ND_ART_PHONY_TEST_HOST_SUFFIX) -endif -ALL_ADDRESS_SIZES := 64 32 - -# List all run test names with number arguments agreeing with the comment above. -define all-run-test-names - $(foreach target, $(1), \ - $(foreach run-type, $(2), \ - $(foreach prebuild, $(3), \ - $(foreach compiler, $(4), \ - $(foreach relocate, $(5), \ - $(foreach trace, $(6), \ - $(foreach gc, $(7), \ - $(foreach jni, $(8), \ - $(foreach image, $(9), \ - $(foreach pictest, $(10), \ - $(foreach debuggable, $(11), \ - $(foreach test, $(12), \ - $(foreach address_size, $(13), \ - test-art-$(target)-run-test-$(run-type)-$(prebuild)-$(compiler)-$(relocate)-$(trace)-$(gc)-$(jni)-$(image)-$(pictest)-$(debuggable)-$(test)$(address_size) \ - ))))))))))))) -endef # all-run-test-names - -# To generate a full list or tests: -# $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES),$(COMPILER_TYPES), \ -# $(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES),$(IMAGE_TYPES), \ -# $(PICTEST_TYPES),$(DEBUGGABLE_TYPES),$(TEST_ART_RUN_TESTS),$(ALL_ADDRESS_SIZES)) - -# Convert's a rule name to the form used in variables, e.g. no-relocate to NO_RELOCATE -define name-to-var -$(shell echo $(1) | tr '[:lower:]' '[:upper:]' | tr '-' '_') -endef # name-to-var - -# Disable 153-reference-stress temporarily until a fix arrives. b/33389022. -# Disable 080-oom-fragmentation due to flakes. b/33795328 -# Disable 497-inlining-and-class-loader and 542-unresolved-access-check until -# they are rewritten. These tests use a broken class loader that tries to -# register a dex file that's already registered with a different loader. -# b/34193123 -ART_TEST_RUN_TEST_SKIP += \ - 153-reference-stress \ - 080-oom-fragmentation \ - 497-inlining-and-class-loader \ - 542-unresolved-access-check - -ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \ - $(COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ - $(IMAGE_TYPES), $(PICTEST_TYPES), $(DEBUGGABLE_TYPES), $(ART_TEST_RUN_TEST_SKIP), $(ALL_ADDRESS_SIZES)) - - -# Disable 149-suspend-all-stress, its output is flaky (b/28988206). -# Disable 577-profile-foreign-dex (b/27454772). -TEST_ART_BROKEN_ALL_TARGET_TESTS := \ - 149-suspend-all-stress \ - 577-profile-foreign-dex \ - -ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \ - $(COMPILER_TYPES), $(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ - $(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES), $(TEST_ART_BROKEN_ALL_TARGET_TESTS), \ - $(ALL_ADDRESS_SIZES)) - -TEST_ART_BROKEN_ALL_TARGET_TESTS := - -# Tests that are timing sensitive and flaky on heavily loaded systems. -TEST_ART_TIMING_SENSITIVE_RUN_TESTS := \ - 002-sleep \ - 053-wait-some \ - 055-enum-performance \ - 133-static-invoke-super - -# disable timing sensitive tests on "dist" builds. -ifdef dist_goal - ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \ - $(COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ - $(IMAGE_TYPES), $(PICTEST_TYPES), $(DEBUGGABLE_TYPES), $(TEST_ART_TIMING_SENSITIVE_RUN_TESTS), $(ALL_ADDRESS_SIZES)) -endif - -# 147-stripped-dex-fallback isn't supported on device because --strip-dex -# requires the zip command. -# 569-checker-pattern-replacement tests behaviour present only on host. -TEST_ART_BROKEN_TARGET_TESTS := \ - 147-stripped-dex-fallback \ - 569-checker-pattern-replacement - -ifneq (,$(filter target,$(TARGET_TYPES))) - ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,target,$(RUN_TYPES),$(PREBUILD_TYPES), \ - $(COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ - $(IMAGE_TYPES), $(PICTEST_TYPES), $(DEBUGGABLE_TYPES), $(TEST_ART_BROKEN_TARGET_TESTS), $(ALL_ADDRESS_SIZES)) -endif - -TEST_ART_BROKEN_TARGET_TESTS := - -# Tests that require python3. -TEST_ART_PYTHON3_DEPENDENCY_RUN_TESTS := \ - 960-default-smali \ - 961-default-iface-resolution-gen \ - 964-default-iface-init-gen \ - 968-default-partial-compile-gen \ - 969-iface-super \ - 970-iface-super-resolution-gen \ - 971-iface-super - -# Check if we have python3 to run our tests. -ifeq ($(wildcard /usr/bin/python3),) - $(warning "No python3 found. Disabling tests: $(TEST_ART_PYTHON3_DEPENDENCY_RUN_TESTS)") - - # Currently disable tests requiring python3 when it is not installed. - ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \ - $(COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ - $(IMAGE_TYPES), $(PICTEST_TYPES), $(DEBUGGABLE_TYPES), $(TEST_ART_PYTHON3_DEPENDENCY_RUN_TESTS), $(ALL_ADDRESS_SIZES)) -endif - -TEST_ART_TIMING_SENSITIVE_RUN_TESTS := - -# Note 116-nodex2oat is not broken per-se it just doesn't (and isn't meant to) work with --prebuild. -TEST_ART_BROKEN_PREBUILD_RUN_TESTS := \ - 116-nodex2oat \ - 118-noimage-dex2oat \ - 134-nodex2oat-nofallback - -ifneq (,$(filter prebuild,$(PREBUILD_TYPES))) - ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),prebuild, \ - $(COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ - $(IMAGE_TYPES), $(PICTEST_TYPES), $(DEBUGGABLE_TYPES), $(TEST_ART_BROKEN_PREBUILD_RUN_TESTS), $(ALL_ADDRESS_SIZES)) -endif - -TEST_ART_BROKEN_PREBUILD_RUN_TESTS := - -# 554-jit-profile-file is disabled because it needs a primary oat file to know what it should save. -# 529 and 555: b/27784033 -TEST_ART_BROKEN_NO_PREBUILD_TESTS := \ - 117-nopatchoat \ - 147-stripped-dex-fallback \ - 554-jit-profile-file \ - 529-checker-unresolved \ - 555-checker-regression-x86const \ - 608-checker-unresolved-lse - -ifneq (,$(filter no-prebuild,$(PREBUILD_TYPES))) - ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),no-prebuild, \ - $(COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ - $(IMAGE_TYPES), $(PICTEST_TYPES), $(DEBUGGABLE_TYPES), $(TEST_ART_BROKEN_NO_PREBUILD_TESTS), $(ALL_ADDRESS_SIZES)) -endif - -TEST_ART_BROKEN_NO_PREBUILD_TESTS := - -# Note 117-nopatchoat is not broken per-se it just doesn't work (and isn't meant to) without -# --prebuild --relocate -TEST_ART_BROKEN_NO_RELOCATE_TESTS := \ - 117-nopatchoat \ - 118-noimage-dex2oat \ - 119-noimage-patchoat \ - 554-jit-profile-file - -ifneq (,$(filter no-relocate,$(RELOCATE_TYPES))) - ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \ - $(COMPILER_TYPES), no-relocate,$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ - $(IMAGE_TYPES), $(PICTEST_TYPES), $(DEBUGGABLE_TYPES), $(TEST_ART_BROKEN_NO_RELOCATE_TESTS), $(ALL_ADDRESS_SIZES)) -endif - -TEST_ART_BROKEN_NO_RELOCATE_TESTS := - -# Temporarily disable some broken tests when forcing access checks in interpreter b/22414682 -# 629 requires compilation. -TEST_ART_BROKEN_INTERPRETER_ACCESS_CHECK_TESTS := \ - 137-cfi \ - 629-vdex-speed - -ifneq (,$(filter interp-ac,$(COMPILER_TYPES))) - ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \ - interp-ac,$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ - $(IMAGE_TYPES), $(PICTEST_TYPES), $(DEBUGGABLE_TYPES), $(TEST_ART_BROKEN_INTERPRETER_ACCESS_CHECK_TESTS), $(ALL_ADDRESS_SIZES)) -endif - -TEST_ART_BROKEN_INTERPRETER_ACCESS_CHECK_TESTS := - -# Tests that are broken with GC stress. -# * 137-cfi needs to unwind a second forked process. We're using a primitive sleep to wait till we -# hope the second process got into the expected state. The slowness of gcstress makes this bad. -# * 908-gc-start-finish expects GCs only to be run at clear points. The reduced heap size makes -# this non-deterministic. Same for 913. -# * 961-default-iface-resolution-gen and 964-default-iface-init-genare very long tests that often -# will take more than the timeout to run when gcstress is enabled. This is because gcstress -# slows down allocations significantly which these tests do a lot. -TEST_ART_BROKEN_GCSTRESS_RUN_TESTS := \ - 137-cfi \ - 154-gc-loop \ - 908-gc-start-finish \ - 913-heaps \ - 961-default-iface-resolution-gen \ - 964-default-iface-init-gen - -ifneq (,$(filter gcstress,$(GC_TYPES))) - ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \ - $(COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),gcstress,$(JNI_TYPES), \ - $(IMAGE_TYPES), $(PICTEST_TYPES), $(DEBUGGABLE_TYPES), $(TEST_ART_BROKEN_GCSTRESS_RUN_TESTS), $(ALL_ADDRESS_SIZES)) -endif - -TEST_ART_BROKEN_GCSTRESS_RUN_TESTS := - -# 115-native-bridge setup is complicated. Need to implement it correctly for the target. -ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,target,$(RUN_TYPES),$(PREBUILD_TYPES),$(COMPILER_TYPES), \ - $(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES),$(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES), 115-native-bridge, \ - $(ALL_ADDRESS_SIZES)) - -# 130-hprof dumps the heap and runs hprof-conv to check whether the file is somewhat readable. This -# is only possible on the host. -# TODO: Turn off all the other combinations, this is more about testing actual ART code. A gtest is -# very hard to write here, as (for a complete test) JDWP must be set up. -ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,target,$(RUN_TYPES),$(PREBUILD_TYPES), \ - $(COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES),$(IMAGE_TYPES), \ - $(PICTEST_TYPES),$(DEBUGGABLE_TYPES),130-hprof,$(ALL_ADDRESS_SIZES)) - -# 131 is an old test. The functionality has been implemented at an earlier stage and is checked -# in tests 138. Blacklisted for debug builds since these builds have duplicate classes checks which -# punt to interpreter. -ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),debug,$(PREBUILD_TYPES), \ - $(COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES),$(IMAGE_TYPES), \ - $(PICTEST_TYPES),$(DEBUGGABLE_TYPES),131-structural-change,$(ALL_ADDRESS_SIZES)) - -# 138-duplicate-classes-check. Turned on for debug builds since debug builds have duplicate classes -# checks enabled, b/2133391. -ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),ndebug,$(PREBUILD_TYPES), \ - $(COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES),$(IMAGE_TYPES), \ - $(PICTEST_TYPES),$(DEBUGGABLE_TYPES),138-duplicate-classes-check,$(ALL_ADDRESS_SIZES)) - -# All these tests check that we have sane behavior if we don't have a patchoat or dex2oat. -# Therefore we shouldn't run them in situations where we actually don't have these since they -# explicitly test for them. These all also assume we have an image. -# 147-stripped-dex-fallback is disabled because it requires --prebuild. -# 554-jit-profile-file is disabled because it needs a primary oat file to know what it should save. -# 629-vdex-speed requires compiled code. -TEST_ART_BROKEN_FALLBACK_RUN_TESTS := \ - 116-nodex2oat \ - 117-nopatchoat \ - 118-noimage-dex2oat \ - 119-noimage-patchoat \ - 137-cfi \ - 138-duplicate-classes-check2 \ - 147-stripped-dex-fallback \ - 554-jit-profile-file \ - 629-vdex-speed - -# This test fails without an image. -# 018, 961, 964, 968 often time out. b/34369284 -TEST_ART_BROKEN_NO_IMAGE_RUN_TESTS := \ - 137-cfi \ - 138-duplicate-classes-check \ - 018-stack-overflow \ - 961-default-iface-resolution-gen \ - 964-default-iface-init \ - 968-default-partial-compile-gen \ - -ifneq (,$(filter no-dex2oat,$(PREBUILD_TYPES))) - ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),no-dex2oat, \ - $(COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES),$(IMAGE_TYPES), \ - $(PICTEST_TYPES),$(DEBUGGABLE_TYPES), $(TEST_ART_BROKEN_FALLBACK_RUN_TESTS),$(ALL_ADDRESS_SIZES)) -endif - - -ifneq (,$(filter no-image,$(IMAGE_TYPES))) - ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \ - $(COMPILER_TYPES), $(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES),no-image, \ - $(PICTEST_TYPES), $(DEBUGGABLE_TYPES), $(TEST_ART_BROKEN_FALLBACK_RUN_TESTS),$(ALL_ADDRESS_SIZES)) - ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \ - $(COMPILER_TYPES), $(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES),no-image, \ - $(PICTEST_TYPES), $(DEBUGGABLE_TYPES), $(TEST_ART_BROKEN_NO_IMAGE_RUN_TESTS),$(ALL_ADDRESS_SIZES)) -endif - -ifneq (,$(filter relocate-npatchoat,$(RELOCATE_TYPES))) - ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \ - $(COMPILER_TYPES), relocate-npatchoat,$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ - $(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES), $(TEST_ART_BROKEN_FALLBACK_RUN_TESTS),$(ALL_ADDRESS_SIZES)) -endif - -TEST_ART_BROKEN_FALLBACK_RUN_TESTS := - -# 137: -# This test unrolls and expects managed frames, but tracing means we run the interpreter. -# 802 and 570-checker-osr: -# This test dynamically enables tracing to force a deoptimization. This makes the test meaningless -# when already tracing, and writes an error message that we do not want to check for. -# 130 occasional timeout b/32383962. -# 629 requires compilation. -TEST_ART_BROKEN_TRACING_RUN_TESTS := \ - 087-gc-after-link \ - 130-hprof \ - 137-cfi \ - 141-class-unload \ - 570-checker-osr \ - 629-vdex-speed \ - 802-deoptimization - -ifneq (,$(filter trace stream,$(TRACE_TYPES))) - ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \ - $(COMPILER_TYPES),$(RELOCATE_TYPES),trace stream,$(GC_TYPES),$(JNI_TYPES),$(IMAGE_TYPES), \ - $(PICTEST_TYPES),$(DEBUGGABLE_TYPES), $(TEST_ART_BROKEN_TRACING_RUN_TESTS),$(ALL_ADDRESS_SIZES)) -endif - -TEST_ART_BROKEN_TRACING_RUN_TESTS := - -# These tests expect JIT compilation, which is suppressed when tracing. -TEST_ART_BROKEN_JIT_TRACING_RUN_TESTS := \ - 604-hot-static-interface \ - 612-jit-dex-cache \ - 613-inlining-dex-cache \ - 616-cha \ - 626-set-resolved-string \ - -ifneq (,$(filter trace stream,$(TRACE_TYPES))) - ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \ - jit,$(RELOCATE_TYPES),trace stream,$(GC_TYPES),$(JNI_TYPES),$(IMAGE_TYPES), \ - $(PICTEST_TYPES),$(DEBUGGABLE_TYPES), $(TEST_ART_BROKEN_JIT_TRACING_RUN_TESTS),$(ALL_ADDRESS_SIZES)) -endif - -TEST_ART_BROKEN_JIT_TRACING_RUN_TESTS := - -# Known broken tests for the interpreter. -# CFI unwinding expects managed frames. -# 629 requires compilation. -TEST_ART_BROKEN_INTERPRETER_RUN_TESTS := \ - 137-cfi \ - 554-jit-profile-file \ - 629-vdex-speed - -ifneq (,$(filter interpreter,$(COMPILER_TYPES))) - ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \ - interpreter,$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ - $(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES),$(TEST_ART_BROKEN_INTERPRETER_RUN_TESTS),$(ALL_ADDRESS_SIZES)) -endif - -TEST_ART_BROKEN_INTERPRETER_RUN_TESTS := - -# Known broken tests for the JIT. -# CFI unwinding expects managed frames, and the test does not iterate enough to even compile. JIT -# also uses Generic JNI instead of the JNI compiler. -# Test 906 iterates the heap filtering with different options. No instances should be created -# between those runs to be able to have precise checks. -# Test 902 hits races with the JIT compiler. b/32821077 -# Test 629 requires compilation. -# Test 914, 915, 917, & 919 are very sensitive to the exact state of the stack, -# including the jit-inserted runtime frames. This causes them to be somewhat -# flaky as JIT tests. This should be fixed once b/33630159 or b/33616143 are -# resolved but until then just disable them. Test 916 already checks this -# feature for JIT use cases in a way that is resilient to the jit frames. -# 912: b/34655682 -TEST_ART_BROKEN_JIT_RUN_TESTS := \ - 137-cfi \ - 629-vdex-speed \ - 902-hello-transformation \ - 904-object-allocation \ - 906-iterate-heap \ - 912-classes \ - 914-hello-obsolescence \ - 915-obsolete-2 \ - 917-fields-transformation \ - 919-obsolete-fields \ - 926-multi-obsolescence \ - -ifneq (,$(filter jit,$(COMPILER_TYPES))) - ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \ - jit,$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ - $(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES),$(TEST_ART_BROKEN_JIT_RUN_TESTS),$(ALL_ADDRESS_SIZES)) -endif - -TEST_ART_BROKEN_JIT_RUN_TESTS := - -# Known broken tests for the graph coloring register allocator. -# These tests were based on the linear scan allocator, which makes different decisions than -# the graph coloring allocator. (These attempt to test for code quality, not correctness.) -TEST_ART_BROKEN_OPTIMIZING_GRAPH_COLOR := \ - 570-checker-select \ - 484-checker-register-hints - -ifneq (,$(filter regalloc_gc,$(COMPILER_TYPES))) - ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \ - regalloc_gc,$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ - $(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES), \ - $(TEST_ART_BROKEN_OPTIMIZING_GRAPH_COLOR),$(ALL_ADDRESS_SIZES)) -endif - -# Known broken tests for the mips32 optimizing compiler backend. -TEST_ART_BROKEN_OPTIMIZING_MIPS_RUN_TESTS := \ - -ifeq (mips,$(TARGET_ARCH)) - ifneq (,$(filter $(OPTIMIZING_COMPILER_TYPES),$(COMPILER_TYPES))) - ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,target,$(RUN_TYPES),$(PREBUILD_TYPES), \ - $(OPTIMIZING_COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ - $(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES), \ - $(TEST_ART_BROKEN_OPTIMIZING_MIPS_RUN_TESTS),$(ALL_ADDRESS_SIZES)) - endif -endif - -TEST_ART_BROKEN_OPTIMIZING_MIPS_RUN_TESTS := - -# Known broken tests for the mips64 optimizing compiler backend. -TEST_ART_BROKEN_OPTIMIZING_MIPS64_RUN_TESTS := \ - -ifeq (mips64,$(TARGET_ARCH)) - ifneq (,$(filter $(OPTIMIZING_COMPILER_TYPES),$(COMPILER_TYPES))) - ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,target,$(RUN_TYPES),$(PREBUILD_TYPES), \ - $(OPTIMIZING_COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ - $(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES), \ - $(TEST_ART_BROKEN_OPTIMIZING_MIPS64_RUN_TESTS),$(ALL_ADDRESS_SIZES)) - endif -endif - -TEST_ART_BROKEN_OPTIMIZING_MIPS64_RUN_TESTS := - -# Tests that should fail when the optimizing compiler compiles them non-debuggable. -TEST_ART_BROKEN_OPTIMIZING_NONDEBUGGABLE_RUN_TESTS := \ - 454-get-vreg \ - 457-regs \ - 602-deoptimizeable - -ifneq (,$(filter $(OPTIMIZING_COMPILER_TYPES),$(COMPILER_TYPES))) - ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \ - $(OPTIMIZING_COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ - $(IMAGE_TYPES),$(PICTEST_TYPES),ndebuggable,$(TEST_ART_BROKEN_OPTIMIZING_NONDEBUGGABLE_RUN_TESTS),$(ALL_ADDRESS_SIZES)) -endif - -TEST_ART_BROKEN_OPTIMIZING_NONDEBUGGABLE_RUN_TESTS := - -# Tests that should fail when the optimizing compiler compiles them debuggable. -TEST_ART_BROKEN_OPTIMIZING_DEBUGGABLE_RUN_TESTS := \ - -ifneq (,$(filter $(OPTIMIZING_COMPILER_TYPES),$(COMPILER_TYPES))) - ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \ - $(OPTIMIZING_COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ - $(IMAGE_TYPES),$(PICTEST_TYPES),debuggable,$(TEST_ART_BROKEN_OPTIMIZING_DEBUGGABLE_RUN_TESTS),$(ALL_ADDRESS_SIZES)) -endif - -TEST_ART_BROKEN_OPTIMIZING_DEBUGGABLE_RUN_TESTS := - -# Tests that should fail in the read barrier configuration with the interpreter. -TEST_ART_BROKEN_INTERPRETER_READ_BARRIER_RUN_TESTS := - -# Tests that should fail in the read barrier configuration with the Optimizing compiler (AOT). -TEST_ART_BROKEN_OPTIMIZING_READ_BARRIER_RUN_TESTS := - -# Tests that should fail in the read barrier configuration with JIT (Optimizing compiler). -TEST_ART_BROKEN_JIT_READ_BARRIER_RUN_TESTS := - -# Tests failing in non-Baker read barrier configurations with the Optimizing compiler (AOT). -# 537: Expects an array copy to be intrinsified, but calling-on-slowpath intrinsics are not yet -# handled in non-Baker read barrier configurations. -TEST_ART_BROKEN_OPTIMIZING_NON_BAKER_READ_BARRIER_RUN_TESTS := \ - 537-checker-arraycopy - -# Tests failing in non-Baker read barrier configurations with JIT (Optimizing compiler). -# 537: Expects an array copy to be intrinsified, but calling-on-slowpath intrinsics are not yet -# handled in non-Baker read barrier configurations. -TEST_ART_BROKEN_JIT_NON_BAKER_READ_BARRIER_RUN_TESTS := \ - 537-checker-arraycopy - -ifeq ($(ART_USE_READ_BARRIER),true) - ifneq (,$(filter interpreter,$(COMPILER_TYPES))) - ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES), \ - $(PREBUILD_TYPES),interpreter,$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES), \ - $(JNI_TYPES),$(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES), \ - $(TEST_ART_BROKEN_INTERPRETER_READ_BARRIER_RUN_TESTS),$(ALL_ADDRESS_SIZES)) - endif - - ifneq (,$(filter $(OPTIMIZING_COMPILER_TYPES),$(COMPILER_TYPES))) - ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES), \ - $(PREBUILD_TYPES),$(OPTIMIZING_COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES), \ - $(GC_TYPES),$(JNI_TYPES),$(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES), \ - $(TEST_ART_BROKEN_OPTIMIZING_READ_BARRIER_RUN_TESTS),$(ALL_ADDRESS_SIZES)) - ifneq ($(ART_READ_BARRIER_TYPE),BAKER) - ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES), \ - $(PREBUILD_TYPES),$(OPTIMIZING_COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES), \ - $(GC_TYPES),$(JNI_TYPES),$(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES), \ - $(TEST_ART_BROKEN_OPTIMIZING_NON_BAKER_READ_BARRIER_RUN_TESTS),$(ALL_ADDRESS_SIZES)) - endif - endif - ifneq (,$(filter jit,$(COMPILER_TYPES))) - ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES), \ - $(PREBUILD_TYPES),jit,$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES), \ - $(JNI_TYPES),$(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES), \ - $(TEST_ART_BROKEN_JIT_READ_BARRIER_RUN_TESTS),$(ALL_ADDRESS_SIZES)) - ifneq ($(ART_READ_BARRIER_TYPE),BAKER) - ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES), \ - $(PREBUILD_TYPES),jit,$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES), \ - $(JNI_TYPES),$(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES), \ - $(TEST_ART_BROKEN_JIT_NON_BAKER_READ_BARRIER_RUN_TESTS),$(ALL_ADDRESS_SIZES)) - endif - endif -endif +PREBUILD_TYPES := prebuild +PREBUILD_TYPES += no-prebuild +PREBUILD_TYPES += no-dex2oat -TEST_ART_BROKEN_OPTIMIZING_READ_BARRIER_RUN_TESTS := -TEST_ART_BROKEN_JIT_READ_BARRIER_RUN_TESTS := +COMPILER_TYPES := interp-ac +COMPILER_TYPES += interpreter +COMPILER_TYPES += jit +COMPILER_TYPES += optimizing +COMPILER_TYPES += regalloc_gc -TEST_ART_BROKEN_NPIC_RUN_TESTS := 596-app-images -ifneq (,$(filter npictest,$(PICTEST_TYPES))) - ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \ - ${COMPILER_TYPES},$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ - $(IMAGE_TYPES),npictest,$(DEBUGGABLE_TYPES),$(TEST_ART_BROKEN_NPIC_RUN_TESTS),$(ALL_ADDRESS_SIZES)) -endif +RELOCATE_TYPES := relocate +RELOCATE_TYPES += no-relocate +RELOCATE_TYPES += relocate-npatchoat -# Tests that should fail in the heap poisoning configuration with the Optimizing compiler. -# 055: Exceeds run time limits due to heap poisoning instrumentation (on ARM and ARM64 devices). -TEST_ART_BROKEN_OPTIMIZING_HEAP_POISONING_RUN_TESTS := \ - 055-enum-performance +TRACE_TYPES := ntrace +TRACE_TYPES += trace +TRACE_TYPES += stream -ifeq ($(ART_HEAP_POISONING),true) - ifneq (,$(filter $(OPTIMIZING_COMPILER_TYPES),$(COMPILER_TYPES))) - ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES), \ - $(PREBUILD_TYPES),$(OPTIMIZING_COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ - $(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES), \ - $(TEST_ART_BROKEN_OPTIMIZING_HEAP_POISONING_RUN_TESTS),$(ALL_ADDRESS_SIZES)) - endif -endif +GC_TYPES := cms +GC_TYPES += gcstress +GC_TYPES += gcverify -TEST_ART_BROKEN_OPTIMIZING_HEAP_POISONING_RUN_TESTS := +JNI_TYPES := checkjni +JNI_TYPES += forcecopy -# 909: Tests that check semantics for a non-debuggable app. -# 137: relies on AOT code and debuggable makes us JIT always. -TEST_ART_BROKEN_DEBUGGABLE_RUN_TESTS := \ - 137-cfi \ - 909-attach-agent \ +IMAGE_TYPES := picimage +IMAGE_TYPES += no-image +IMAGE_TYPES += multipicimage +IMAGE_TYPES += npicimage +IMAGE_TYPES += multinpicimage -ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES),$(PREBUILD_TYPES), \ - $(COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ - $(IMAGE_TYPES),$(PICTEST_TYPES),debuggable,$(TEST_ART_BROKEN_DEBUGGABLE_RUN_TESTS),$(ALL_ADDRESS_SIZES)) +PICTEST_TYPES := npictest +PICTEST_TYPES += pictest -TEST_ART_BROKEN_DEBUGGABLE_RUN_TESTS := +RUN_TYPES := debug +RUN_TYPES += ndebug -# Tests incompatible with bisection bug search. Sorted by incompatibility reason. -# 000 through 595 do not compile anything. 089 tests a build failure. 018 through 137 -# run dalvikvm more than once. 115 and 088 assume they are always compiled. -# 055 tests performance which is degraded during bisecting. -TEST_ART_INCOMPATIBLE_BISECTION_SEARCH_RUN_TESTS := \ - 000-nop \ - 134-nodex2oat-nofallback \ - 147-stripped-dex-fallback \ - 595-profile-saving \ - \ - 089-many-methods \ - \ - 018-stack-overflow \ - 116-nodex2oat \ - 117-nopatchoat \ - 118-noimage-dex2oat \ - 119-noimage-patchoat \ - 126-miranda-multidex \ - 137-cfi \ - \ - 115-native-bridge \ - 088-monitor-verification \ - \ - 055-enum-performance +DEBUGGABLE_TYPES := ndebuggable +DEBUGGABLE_TYPES += debuggable -ifeq ($(ART_TEST_BISECTION),true) - ART_TEST_KNOWN_BROKEN += $(call all-run-test-names,$(TARGET_TYPES),$(RUN_TYPES), \ - $(PREBUILD_TYPES),$(OPTIMIZING_COMPILER_TYPES),$(RELOCATE_TYPES),$(TRACE_TYPES),$(GC_TYPES),$(JNI_TYPES), \ - $(IMAGE_TYPES),$(PICTEST_TYPES),$(DEBUGGABLE_TYPES), \ - $(TEST_ART_INCOMPATIBLE_BISECTION_SEARCH_RUN_TESTS),$(ALL_ADDRESS_SIZES)) -endif +ALL_ADDRESS_SIZES := 64 32 -# Clear variables ahead of appending to them when defining tests. -$(foreach target, $(TARGET_TYPES), $(eval ART_RUN_TEST_$(call name-to-var,$(target))_RULES :=)) -$(foreach target, $(TARGET_TYPES), \ - $(foreach prebuild, $(PREBUILD_TYPES), \ - $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(prebuild))_RULES :=))) -$(foreach target, $(TARGET_TYPES), \ - $(foreach compiler, $(COMPILER_TYPES), \ - $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(compiler))_RULES :=))) -$(foreach target, $(TARGET_TYPES), \ - $(foreach relocate, $(RELOCATE_TYPES), \ - $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(relocate))_RULES :=))) -$(foreach target, $(TARGET_TYPES), \ - $(foreach trace, $(TRACE_TYPES), \ - $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(trace))_RULES :=))) -$(foreach target, $(TARGET_TYPES), \ - $(foreach gc, $(GC_TYPES), \ - $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(gc))_RULES :=))) -$(foreach target, $(TARGET_TYPES), \ - $(foreach jni, $(JNI_TYPES), \ - $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(jni))_RULES :=))) -$(foreach target, $(TARGET_TYPES), \ - $(foreach image, $(IMAGE_TYPES), \ - $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(image))_RULES :=))) -$(foreach target, $(TARGET_TYPES), \ - $(foreach test, $(TEST_ART_RUN_TESTS), \ - $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(test))_RULES :=))) -$(foreach target, $(TARGET_TYPES), \ - $(foreach address_size, $(ALL_ADDRESS_SIZES), \ - $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(address_size))_RULES :=))) -$(foreach target, $(TARGET_TYPES), \ - $(foreach run_type, $(RUN_TYPES), \ - $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(run_type))_RULES :=))) -$(foreach target, $(TARGET_TYPES), \ - $(foreach debuggable_type, $(DEBUGGABLE_TYPES), \ - $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(debuggable_type))_RULES :=))) +define name-to-var +$(shell echo $(1) | tr '[:lower:]' '[:upper:]' | tr '-' '_') +endef # name-to-var # We need dex2oat and dalvikvm on the target as well as the core images (all images as we sync # only once). @@ -863,405 +225,148 @@ ART_TEST_HOST_RUN_TEST_DEPENDENCIES += \ endif -# Create a rule to build and run a tests following the form: -# test-art-{1: host or target}-run-test-{2: debug ndebug}-{3: prebuild no-prebuild no-dex2oat}- -# {4: interpreter optimizing jit interp-ac}- -# {5: relocate nrelocate relocate-npatchoat}- -# {6: trace or ntrace}-{7: gcstress gcverify cms}-{8: forcecopy checkjni jni}- -# {9: no-image image picimage}-{10: pictest npictest}- -# {11: ndebuggable debuggable}-{12: test name}{13: 32 or 64} -define define-test-art-run-test - run_test_options := - prereq_rule := - test_groups := - uc_host_or_target := - jack_classpath := - ifeq ($(ART_TEST_WITH_STRACE),true) - run_test_options += --strace - endif - ifeq ($(ART_TEST_RUN_TEST_ALWAYS_CLEAN),true) - run_test_options += --always-clean - endif - ifeq ($(ART_TEST_BISECTION),true) - run_test_options += --bisection-search - endif - ifeq ($(1),host) - uc_host_or_target := HOST - test_groups := ART_RUN_TEST_HOST_RULES - run_test_options += --host - prereq_rule := $(ART_TEST_HOST_RUN_TEST_DEPENDENCIES) $(HOST_JACK_CLASSPATH_DEPENDENCIES) - jack_classpath := $(HOST_JACK_CLASSPATH) - else - ifeq ($(1),target) - uc_host_or_target := TARGET - test_groups := ART_RUN_TEST_TARGET_RULES - prereq_rule := test-art-target-sync $(TARGET_JACK_CLASSPATH_DEPENDENCIES) - jack_classpath := $(TARGET_JACK_CLASSPATH) - else - $$(error found $(1) expected $(TARGET_TYPES)) - endif - endif - ifeq ($(2),debug) - test_groups += ART_RUN_TEST_$$(uc_host_or_target)_DEBUG_RULES - else - ifeq ($(2),ndebug) - test_groups += ART_RUN_TEST_$$(uc_host_or_target)_RELEASE_RULES - run_test_options += -O - else - $$(error found $(2) expected $(RUN_TYPES)) - endif - endif - ifeq ($(3),prebuild) - test_groups += ART_RUN_TEST_$$(uc_host_or_target)_PREBUILD_RULES - run_test_options += --prebuild - else - ifeq ($(3),no-prebuild) - test_groups += ART_RUN_TEST_$$(uc_host_or_target)_NO_PREBUILD_RULES - run_test_options += --no-prebuild - else - ifeq ($(3),no-dex2oat) - test_groups += ART_RUN_TEST_$$(uc_host_or_target)_NO_DEX2OAT_RULES - run_test_options += --no-prebuild --no-dex2oat - else - $$(error found $(3) expected $(PREBUILD_TYPES)) - endif - endif - endif - ifeq ($(4),optimizing) - test_groups += ART_RUN_TEST_$$(uc_host_or_target)_OPTIMIZING_RULES - run_test_options += --optimizing - else ifeq ($(4),regalloc_gc) - test_groups += ART_RUN_TEST_$$(uc_host_or_target)_OPTIMIZING_GRAPH_COLOR_RULES - run_test_options += --optimizing -Xcompiler-option --register-allocation-strategy=graph-color - else - ifeq ($(4),interpreter) - test_groups += ART_RUN_TEST_$$(uc_host_or_target)_INTERPRETER_RULES - run_test_options += --interpreter - else ifeq ($(4),interp-ac) - test_groups += ART_RUN_TEST_$$(uc_host_or_target)_INTERPRETER_ACCESS_CHECKS_RULES - run_test_options += --interpreter --verify-soft-fail - else - ifeq ($(4),jit) - test_groups += ART_RUN_TEST_$$(uc_host_or_target)_JIT_RULES - run_test_options += --jit - else - $$(error found $(4) expected $(COMPILER_TYPES)) - endif - endif - endif +host_prereq_rules := $(ART_TEST_HOST_RUN_TEST_DEPENDENCIES) +host_prereq_rules += $(HOST_JACK_CLASSPATH_DEPENDENCIES) +host_prereq_rules += $(HOST_CORE_IMAGE_OUT) +host_prereq_rules += $(TEST_ART_RUN_TEST_DEPENDENCIES) +host_prereq_rules += $(HOST_OUT_EXECUTABLES)/hprof-conv - ifeq ($(5),relocate) - test_groups += ART_RUN_TEST_$$(uc_host_or_target)_RELOCATE_RULES - run_test_options += --relocate - else - ifeq ($(5),no-relocate) - test_groups += ART_RUN_TEST_$$(uc_host_or_target)_NO_RELOCATE_RULES - run_test_options += --no-relocate - else - ifeq ($(5),relocate-npatchoat) - test_groups += ART_RUN_TEST_$$(uc_host_or_target)_RELOCATE_NO_PATCHOAT_RULES - run_test_options += --relocate --no-patchoat - else - $$(error found $(5) expected $(RELOCATE_TYPES)) - endif - endif - endif - ifeq ($(6),trace) - test_groups += ART_RUN_TEST_$$(uc_host_or_target)_TRACE_RULES - run_test_options += --trace - else - ifeq ($(6),ntrace) - test_groups += ART_RUN_TEST_$$(uc_host_or_target)_NO_TRACE_RULES - else - ifeq ($(6),stream) - # Group streaming under normal tracing rules. - test_groups += ART_RUN_TEST_$$(uc_host_or_target)_TRACE_RULES - run_test_options += --trace --stream - else - $$(error found $(6) expected $(TRACE_TYPES)) - endif - endif - endif - ifeq ($(7),gcverify) - test_groups += ART_RUN_TEST_$$(uc_host_or_target)_GCVERIFY_RULES - run_test_options += --gcverify - else - ifeq ($(7),gcstress) - test_groups += ART_RUN_TEST_$$(uc_host_or_target)_GCSTRESS_RULES - run_test_options += --gcstress - else - ifeq ($(7),cms) - test_groups += ART_RUN_TEST_$$(uc_host_or_target)_CMS_RULES - else - $$(error found $(7) expected $(GC_TYPES)) - endif - endif - endif - ifeq ($(8),forcecopy) - test_groups += ART_RUN_TEST_$$(uc_host_or_target)_FORCECOPY_RULES - run_test_options += --runtime-option -Xjniopts:forcecopy - ifneq ($$(ART_TEST_JNI_FORCECOPY),true) - skip_test := true - endif - else - ifeq ($(8),checkjni) - test_groups += ART_RUN_TEST_$$(uc_host_or_target)_CHECKJNI_RULES - run_test_options += --runtime-option -Xcheck:jni - else - ifeq ($(8),jni) - test_groups += ART_RUN_TEST_$$(uc_host_or_target)_JNI_RULES - else - $$(error found $(8) expected $(JNI_TYPES)) - endif - endif - endif - image_suffix := $(4) - ifeq ($(4),regalloc_gc) - # Graph coloring tests share the image_suffix with optimizing tests. - image_suffix := optimizing +target_prereq_rules := $(TARGET_JACK_CLASSPATH_DEPENDENCIES) +target_prereq_rules += test-art-target-sync + +define core-image-dependencies + ifeq ($(2),no-image) + $(1)_prereq_rules += $($(call name-to-var,$(1))_CORE_IMAGE_$(3)_pic_$(4)) else - ifeq ($(4),jit) - # JIT tests share the image_suffix with interpreter tests. - image_suffix := interpreter - endif - endif - ifeq ($(9),no-image) - test_groups += ART_RUN_TEST_$$(uc_host_or_target)_NO_IMAGE_RULES - run_test_options += --no-image - # Add the core dependency. This is required for pre-building. - # Use the PIC image, as it is the default in run-test, to match dependencies. - ifeq ($(1),host) - prereq_rule += $$(HOST_CORE_IMAGE_$$(image_suffix)_pic_$(13)) + ifeq ($(2),npicimage) + $(1)_prereq_rules += $($(call name-to-var,$(1))_CORE_IMAGE_$(3)_no-pic_$(4)) else - prereq_rule += $$(TARGET_CORE_IMAGE_$$(image_suffix)_pic_$(13)) - endif - else - ifeq ($(9),npicimage) - test_groups += ART_RUN_TEST_$$(uc_host_or_target)_IMAGE_RULES - run_test_options += --npic-image - # Add the core dependency. - ifeq ($(1),host) - prereq_rule += $$(HOST_CORE_IMAGE_$$(image_suffix)_no-pic_$(13)) + ifeq ($(2),picimage) + $(1)_prereq_rules += $($(call name-to-var,$(1))_CORE_IMAGE_$(3)_pic_$(4)) else - prereq_rule += $$(TARGET_CORE_IMAGE_$$(image_suffix)_no-pic_$(13)) - endif - else - ifeq ($(9),picimage) - test_groups += ART_RUN_TEST_$$(uc_host_or_target)_PICIMAGE_RULES - ifeq ($(1),host) - prereq_rule += $$(HOST_CORE_IMAGE_$$(image_suffix)_pic_$(13)) + ifeq ($(2),multinpicimage) + $(1)_prereq_rules += $($(call name-to-var,$(1))_CORE_IMAGE_$(3)_no-pic_multi_$(4)) else - prereq_rule += $$(TARGET_CORE_IMAGE_$$(image_suffix)_pic_$(13)) - endif - else - ifeq ($(9),multinpicimage) - test_groups += ART_RUN_TEST_$$(uc_host_or_target)_IMAGE_RULES - run_test_options += --npic-image --multi-image - ifeq ($(1),host) - prereq_rule += $$(HOST_CORE_IMAGE_$$(image_suffix)_no-pic_multi_$(13)) - else - prereq_rule += $$(TARGET_CORE_IMAGE_$$(image_suffix)_no-pic_multi_$(13)) - endif - else - ifeq ($(9),multipicimage) - test_groups += ART_RUN_TEST_$$(uc_host_or_target)_PICIMAGE_RULES - run_test_options += --multi-image - ifeq ($(1),host) - prereq_rule += $$(HOST_CORE_IMAGE_$$(image_suffix)_pic_multi_$(13)) - else - prereq_rule += $$(TARGET_CORE_IMAGE_$$(image_suffix)_pic_multi_$(13)) - endif - else - $$(error found $(9) expected $(IMAGE_TYPES)) + ifeq ($(2),multipicimage) + $(1)_prereq_rules += $($(call name-to-var,$(1))_CORE_IMAGE_$(3)_pic_multi_$(4)) endif endif endif endif endif - ifeq ($(10),pictest) - run_test_options += --pic-test - else - ifeq ($(10),npictest) - # Nothing to be done. - else - $$(error found $(10) expected $(PICTEST_TYPES)) - endif - endif - ifeq ($(11),debuggable) - test_groups += ART_RUN_TEST_$$(uc_host_or_target)_DEBUGGABLE_RULES - run_test_options += --debuggable - else - ifeq ($(11),ndebuggable) - test_groups += ART_RUN_TEST_$$(uc_host_or_target)_NONDEBUGGABLE_RULES - # Nothing to be done. - else - $$(error found $(11) expected $(DEBUGGABLE_TYPES)) - endif - endif - # $(12) is the test name. - test_groups += ART_RUN_TEST_$$(uc_host_or_target)_$(call name-to-var,$(12))_RULES - ifeq ($(13),64) - test_groups += ART_RUN_TEST_$$(uc_host_or_target)_64_RULES - run_test_options += --64 - else - ifeq ($(13),32) - test_groups += ART_RUN_TEST_$$(uc_host_or_target)_32_RULES - else - $$(error found $(13) expected $(ALL_ADDRESS_SIZES)) - endif - endif - # Override of host instruction-set-features. Required to test advanced x86 intrinsics. The - # conditionals aren't really correct, they will fail to do the right thing on a 32-bit only - # host. However, this isn't common enough to worry here and make the conditions complicated. - ifneq ($(DEX2OAT_HOST_INSTRUCTION_SET_FEATURES),) - ifeq ($(13),64) - run_test_options += --instruction-set-features $(DEX2OAT_HOST_INSTRUCTION_SET_FEATURES) - endif - endif - ifneq ($($(HOST_2ND_ARCH_VAR_PREFIX)DEX2OAT_HOST_INSTRUCTION_SET_FEATURES),) - ifeq ($(13),32) - run_test_options += --instruction-set-features $($(HOST_2ND_ARCH_VAR_PREFIX)DEX2OAT_HOST_INSTRUCTION_SET_FEATURES) - endif - endif - run_test_rule_name := test-art-$(1)-run-test-$(2)-$(3)-$(4)-$(5)-$(6)-$(7)-$(8)-$(9)-$(10)-$(11)-$(12)$(13) - run_test_options := --output-path $(ART_HOST_TEST_DIR)/run-test-output/$$(run_test_rule_name) \ - $$(run_test_options) - ifneq ($(ART_TEST_ANDROID_ROOT),) - run_test_options := --android-root $(ART_TEST_ANDROID_ROOT) $$(run_test_options) - endif - ifeq ($(ART_TEST_QUIET),true) - run_test_options += --quiet - endif -$$(run_test_rule_name): PRIVATE_RUN_TEST_OPTIONS := $$(run_test_options) -$$(run_test_rule_name): PRIVATE_JACK_CLASSPATH := $$(jack_classpath) -.PHONY: $$(run_test_rule_name) -$$(run_test_rule_name): $(TEST_ART_RUN_TEST_DEPENDENCIES) $(HOST_OUT_EXECUTABLES)/hprof-conv $$(prereq_rule) | $(TEST_ART_RUN_TEST_ORDERONLY_DEPENDENCIES) - $(hide) $$(call ART_TEST_SKIP,$$@) && \ - DX=$(abspath $(DX)) \ - JASMIN=$(abspath $(HOST_OUT_EXECUTABLES)/jasmin) \ - SMALI=$(abspath $(HOST_OUT_EXECUTABLES)/smali) \ - DXMERGER=$(abspath $(HOST_OUT_EXECUTABLES)/dexmerger) \ - JACK_VERSION=$(JACK_DEFAULT_VERSION) \ - JACK=$(abspath $(JACK)) \ - JACK_VERSION=$(JACK_DEFAULT_VERSION) \ - JACK_CLASSPATH=$$(PRIVATE_JACK_CLASSPATH) \ - art/test/run-test $$(PRIVATE_RUN_TEST_OPTIONS) $(12) \ - && $$(call ART_TEST_PASSED,$$@) || $$(call ART_TEST_FAILED,$$@) - $$(hide) (echo $(MAKECMDGOALS) | grep -q $$@ && \ - echo "run-test run as top-level target, removing test directory $(ART_HOST_TEST_DIR)" && \ - rm -r $(ART_HOST_TEST_DIR)) || true - - $$(foreach test_group,$$(test_groups), $$(eval $$(value test_group) += $$(run_test_rule_name))) - - # Clear locally defined variables. - uc_host_or_target := - test_groups := - run_test_options := - run_test_rule_name := - prereq_rule := - jack_classpath := -endef # define-test-art-run-test +endef +# Generate list of dependencies required for given target - HOST or TARGET, IMAGE_TYPE, +# COMPILER_TYPE and ADDRESS_SIZE. $(foreach target, $(TARGET_TYPES), \ - $(foreach test, $(TEST_ART_RUN_TESTS), \ - $(foreach run_type, $(RUN_TYPES), \ - $(foreach address_size, $(ADDRESS_SIZES_$(call name-to-var,$(target))), \ - $(foreach prebuild, $(PREBUILD_TYPES), \ - $(foreach compiler, $(COMPILER_TYPES), \ - $(foreach relocate, $(RELOCATE_TYPES), \ - $(foreach trace, $(TRACE_TYPES), \ - $(foreach gc, $(GC_TYPES), \ - $(foreach jni, $(JNI_TYPES), \ - $(foreach image, $(IMAGE_TYPES), \ - $(foreach pictest, $(PICTEST_TYPES), \ - $(foreach debuggable, $(DEBUGGABLE_TYPES), \ - $(eval $(call define-test-art-run-test,$(target),$(run_type),$(prebuild),$(compiler),$(relocate),$(trace),$(gc),$(jni),$(image),$(pictest),$(debuggable),$(test),$(address_size))) \ - ))))))))))))) -define-test-art-run-test := - -# Define a phony rule whose purpose is to test its prerequisites. -# $(1): host or target -# $(2): list of prerequisites + $(foreach image, $(IMAGE_TYPES), \ + $(foreach compiler, $(COMPILER_TYPES), \ + $(foreach address_size, $(ALL_ADDRESS_SIZES), $(eval \ + $(call core-image-dependencies,$(target),$(image),$(compiler),$(address_size))))))) + +test-art-host-run-test-dependencies : $(host_prereq_rules) +test-art-target-run-test-dependencies : $(target_prereq_rules) +test-art-run-test-dependencies : test-art-host-run-test-dependencies test-art-target-run-test-dependencies + +# Create a rule to build and run a test group of the following form: +# test-art-{1: host target}-run-test-{2: prebuild no-prebuild no-dex2oat \ +# interp-ac interpreter jit optimizing regalloc_gc relocate no-relocate \ +# relocate-npatchoat ntrace trace stream cms gcstress gcverify checkjni \ +# forcecopy picimage no-image multinpicimage npictest pictest debug ndebug \ +# ndebuggable debuggable} +# +# test-art-{1: host target}-run-test{2: 32 64} define define-test-art-run-test-group -.PHONY: $(1) -$(1): $(2) - $(hide) $$(call ART_TEST_PREREQ_FINISHED,$$@) - + ifneq (,$(filter $(2),$(ALL_ADDRESS_SIZES))) + build_target := test-art-$(1)-run-test$(2) + else + build_target := test-art-$(1)-run-test-$(2) + endif + # Python testrunner support multi-threaded test run, however, to invoke the script, one + # has to provide the concurrency level for the run. To provide that value, the concurrency + # level of the make invocation is used. The make invocation information is extracted through + # `ps` and subsequently the number of threads is determined. + kati_pid = $(shell echo $$PPID) + make_pid = $$(shell ps -o ppid= -p $$(kati_pid)) + grep_build_targets_from_ps_keyword := -e + grep_build_targets_from_ps_keyword += $$(shell echo $(MAKECMDGOALS) | sed -e "s/\s\+/ -e /g") + ps_data := $$(shell ps aux | grep $$(make_pid) | grep $$(grep_build_targets_from_ps_keyword) ) + concurrency := $$(shell [[ "$$(ps_data)" =~ -[a-zA-Z]*j[[:space:]]*[0-9]+ ]] && [[ $$$$BASH_REMATCH =~ [0-9]+ ]] && echo $$$$BASH_REMATCH || echo 1) + + .PHONY: $$(build_target) + $$(build_target) : args := --$(1) --$(2) --verbose -j$$(concurrency) + $$(build_target) : test-art-$(1)-run-test-dependencies + ./art/test/testrunner/testrunner.py $$(args) + build_target := + concurrency := + ps_data := + kati_pid := + make_pid := + grep_build_targets_from_ps_keyword := endef # define-test-art-run-test-group - -$(foreach target, $(TARGET_TYPES), $(eval \ - $(call define-test-art-run-test-group,test-art-$(target)-run-test,$(ART_RUN_TEST_$(call name-to-var,$(target))_RULES)))) $(foreach target, $(TARGET_TYPES), \ $(foreach prebuild, $(PREBUILD_TYPES), $(eval \ - $(call define-test-art-run-test-group,test-art-$(target)-run-test-$(prebuild),$(ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(prebuild))_RULES))))) + $(call define-test-art-run-test-group,$(target),$(prebuild))))) $(foreach target, $(TARGET_TYPES), \ $(foreach run-type, $(RUN_TYPES), $(eval \ - $(call define-test-art-run-test-group,test-art-$(target)-run-test-$(run-type),$(ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(run-type))_RULES))))) + $(call define-test-art-run-test-group,$(target),$(run-type))))) $(foreach target, $(TARGET_TYPES), \ $(foreach compiler, $(COMPILER_TYPES), $(eval \ - $(call define-test-art-run-test-group,test-art-$(target)-run-test-$(compiler),$(ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(compiler))_RULES))))) + $(call define-test-art-run-test-group,$(target),$(compiler))))) $(foreach target, $(TARGET_TYPES), \ $(foreach relocate, $(RELOCATE_TYPES), $(eval \ - $(call define-test-art-run-test-group,test-art-$(target)-run-test-$(relocate),$(ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(relocate))_RULES))))) + $(call define-test-art-run-test-group,$(target),$(relocate))))) $(foreach target, $(TARGET_TYPES), \ $(foreach trace, $(TRACE_TYPES), $(eval \ - $(call define-test-art-run-test-group,test-art-$(target)-run-test-$(trace),$(ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(trace))_RULES))))) + $(call define-test-art-run-test-group,$(target),$(trace))))) $(foreach target, $(TARGET_TYPES), \ $(foreach gc, $(GC_TYPES), $(eval \ - $(call define-test-art-run-test-group,test-art-$(target)-run-test-$(gc),$(ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(gc))_RULES))))) + $(call define-test-art-run-test-group,$(target),$(gc))))) $(foreach target, $(TARGET_TYPES), \ $(foreach jni, $(JNI_TYPES), $(eval \ - $(call define-test-art-run-test-group,test-art-$(target)-run-test-$(jni),$(ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(jni))_RULES))))) + $(call define-test-art-run-test-group,$(target),$(jni))))) $(foreach target, $(TARGET_TYPES), \ $(foreach debuggable, $(DEBUGGABLE_TYPES), $(eval \ - $(call define-test-art-run-test-group,test-art-$(target)-run-test-$(debuggable),$(ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(debuggable))_RULES))))) + $(call define-test-art-run-test-group,$(target),$(debuggable))))) $(foreach target, $(TARGET_TYPES), \ $(foreach image, $(IMAGE_TYPES), $(eval \ - $(call define-test-art-run-test-group,test-art-$(target)-run-test-$(image),$(ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(image))_RULES))))) -$(foreach target, $(TARGET_TYPES), \ - $(foreach test, $(TEST_ART_RUN_TESTS), $(eval \ - $(call define-test-art-run-test-group,test-art-$(target)-run-test-$(test),$(ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(test))_RULES))))) + $(call define-test-art-run-test-group,$(target),$(image))))) $(foreach target, $(TARGET_TYPES), \ $(foreach address_size, $(ADDRESS_SIZES_$(call name-to-var,$(target))), $(eval \ - $(call define-test-art-run-test-group,test-art-$(target)-run-test$(address_size),$(ART_RUN_TEST_$(call name-to-var,$(target))_$(address_size)_RULES))))) + $(call define-test-art-run-test-group,$(target),$(address_size))))) + +# Create a rule to build and run a test group of the following form: +# test-art-{1: host target}-run-test +define define-test-art-host-or-target-run-test-group + build_target := test-art-$(1)-run-test + .PHONY: $$(build_target) + # Python testrunner support multi-threaded test run, however, to invoke the script, one + # has to provide the concurrency level for the run. To provide that value, the concurrency + # level of the make invocation is used. The make invocation information is extracted through + # `ps` and subsequently the number of threads is determined. + kati_pid = $(shell echo $$PPID) + make_pid = $$(shell ps -o ppid= -p $$(kati_pid)) + grep_build_targets_from_ps_keyword := -e + grep_build_targets_from_ps_keyword += $$(shell echo $(MAKECMDGOALS) | sed -e "s/\s\+/ -e /g") + ps_data := $$(shell ps aux | grep $$(make_pid) | grep $$(grep_build_targets_from_ps_keyword)) + concurrency := $$(shell [[ "$$(ps_data)" =~ -[a-zA-Z]*j[[:space:]]*[0-9]+ ]] && [[ $$$$BASH_REMATCH =~ [0-9]+ ]] && echo $$$$BASH_REMATCH || echo 1) + + $$(build_target) : args := --$(1) --verbose -j$$(concurrency) + $$(build_target) : test-art-$(1)-run-test-dependencies + ./art/test/testrunner/testrunner.py $$(args) + build_target := + args := + grep_build_targets_from_ps_keyword := + ps_data := + concurrency := + kati_pid := + make_pid := +endef # define-test-art-host-or-target-run-test-group + +$(foreach target, $(TARGET_TYPES), $(eval \ + $(call define-test-art-host-or-target-run-test-group,$(target)))) -# Clear variables now we're finished with them. -$(foreach target, $(TARGET_TYPES), $(eval ART_RUN_TEST_$(call name-to-var,$(target))_RULES :=)) -$(foreach target, $(TARGET_TYPES), \ - $(foreach prebuild, $(PREBUILD_TYPES), \ - $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(prebuild))_RULES :=))) -$(foreach target, $(TARGET_TYPES), \ - $(foreach compiler, $(COMPILER_TYPES), \ - $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(compiler))_RULES :=))) -$(foreach target, $(TARGET_TYPES), \ - $(foreach relocate, $(RELOCATE_TYPES), \ - $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(relocate))_RULES :=))) -$(foreach target, $(TARGET_TYPES), \ - $(foreach trace, $(TRACE_TYPES), \ - $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(trace))_RULES :=))) -$(foreach target, $(TARGET_TYPES), \ - $(foreach gc, $(GC_TYPES), \ - $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(gc))_RULES :=))) -$(foreach target, $(TARGET_TYPES), \ - $(foreach jni, $(JNI_TYPES), \ - $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(jni))_RULES :=))) -$(foreach target, $(TARGET_TYPES), \ - $(foreach debuggable, $(DEBUGGABLE_TYPES), \ - $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(debuggable))_RULES :=))) -$(foreach target, $(TARGET_TYPES), \ - $(foreach image, $(IMAGE_TYPES), \ - $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(image))_RULES :=))) -$(foreach target, $(TARGET_TYPES), \ - $(foreach test, $(TEST_ART_RUN_TESTS), \ - $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(test))_RULES :=))) -$(foreach target, $(TARGET_TYPES), \ - $(foreach address_size, $(ALL_ADDRESS_SIZES), \ - $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(address_size))_RULES :=))) -$(foreach target, $(TARGET_TYPES), \ - $(foreach run_type, $(RUN_TYPES), \ - $(eval ART_RUN_TEST_$(call name-to-var,$(target))_$(call name-to-var,$(run_type))_RULES :=))) -define-test-art-run-test-group := TARGET_TYPES := PREBUILD_TYPES := COMPILER_TYPES := @@ -1270,10 +375,8 @@ TRACE_TYPES := GC_TYPES := JNI_TYPES := IMAGE_TYPES := -ADDRESS_SIZES_TARGET := -ADDRESS_SIZES_HOST := -ALL_ADDRESS_SIZES := +PICTEST_TYPES := RUN_TYPES := DEBUGGABLE_TYPES := - +ALL_ADDRESS_SIZES := LOCAL_PATH := diff --git a/test/knownfailures.json b/test/knownfailures.json new file mode 100644 index 000000000..3cfff4345 --- /dev/null +++ b/test/knownfailures.json @@ -0,0 +1,359 @@ +[ + { + "test": "153-reference-stress", + "description": ["Disable 153-reference-stress temporarily until a fix", + "arrives."], + "bug": "http://b/33389022" + }, + { + "test": "080-oom-fragmentation", + "description": "Disable 080-oom-fragmentation due to flakes.", + "bug": "http://b/33795328" + }, + { + "tests": ["497-inlining-and-class-loader", + "542-unresolved-access-check"], + "description": ["Disable 497-inlining-and-class-loader and ", + "542-unresolved-access-check until they are rewritten.", + "These tests use a broken class loader that tries to", + "register a dex file that's already registered with a", + "different loader."], + "bug": "http://b/34193123" + }, + { + "test": "149-suspend-all-stress", + "description": "Disable 149-suspend-all-stress, its output is flaky", + "bug": "http://b/28988206" + }, + { + "test": "577-profile-foreign-dex", + "description": "Disable 577-profile-foreign-dex", + "bug": "http://b/27454772" + }, + { + "tests": ["002-sleep", + "053-wait-some", + "055-enum-performance", + "133-static-invoke-super"], + "description": ["Tests that are timing sensitive and flaky on heavily", + "loaded systems."] + }, + { + "test": "147-stripped-dex-fallback", + "variant": "target", + "description": ["147-stripped-dex-fallback isn't supported on device", + "because --strip-dex requires the zip command."] + }, + { + "test": "569-checker-pattern-replacement", + "variant": "target", + "description": ["569-checker-pattern-replacement tests behaviour", + "present only on host."] + }, + { + "tests": ["116-nodex2oat", + "118-noimage-dex2oat", + "134-nodex2oat-nofallback"], + "variant": "prebuild", + "description": ["Note 116-nodex2oat is not broken per-se it just", + "doesn't (and isn't meant to) work with --prebuild."] + }, + { + "test": "554-jit-profile-file", + "variant": "no-prebuild | interpreter", + "description": ["554-jit-profile-file is disabled because it needs a", + "primary oat file to know what it should save."] + }, + { + "tests": ["529-checker-unresolved", "555-checker-regression-x86const"], + "variant": "no-prebuild", + "bug": "http://b/27784033" + }, + { + "tests": ["117-nopatchoat", + "147-stripped-dex-fallback", + "608-checker-unresolved-lse"], + "variant": "no-prebuild" + }, + { + "tests": ["117-nopatchoat", + "118-noimage-dex2oat", + "119-noimage-patchoat", + "554-jit-profile-file"], + "variant": "no-relocate", + "description": ["117-nopatchoat is not broken per-se it just doesn't", + "work (and isn't meant to) without --prebuild", + "--relocate"] + }, + { + "test": "137-cfi", + "variant": "interp-ac", + "description": ["Temporarily disable some broken tests when forcing", + "access checks in interpreter"], + "bug": "http://b/22414682" + }, + { + "test" : "629-vdex-speed", + "variant": "interp-ac | no-dex2oat | interpreter | jit", + "description": "629 requires compilation." + }, + { + "test": "137-cfi", + "variant": "gcstress", + "description": ["137-cfi needs to unwind a second forked process. We're", + "using a primitive sleep to wait till we hope the", + "second process got into the expected state. The", + "slowness of gcstress makes this bad."] + }, + { + "tests": ["908-gc-start-finish", + "913-heaps"], + "variant": "gcstress", + "description": ["908-gc-start-finish expects GCs only to be run at", + "clear points. The reduced heap size makes this", + "non-deterministic. Same for 913."] + }, + { + "test": "961-default-iface-resolution-gen", + "variant": "gcstress", + "description": ["961-default-iface-resolution-gen and", + "964-default-iface-init-genare very long tests that", + "often will take more than the timeout to run when", + "gcstress is enabled. This is because gcstress slows", + "down allocations significantly which these tests do a", + "lot."] + }, + { + "tests": ["964-default-iface-init-gen", + "154-gc-loop"], + "variant": "gcstress" + }, + { + "test": "115-native-bridge", + "variant": "target", + "description": ["115-native-bridge setup is complicated. Need to", + "implement it correctly for the target."] + }, + { + "test": "130-hprof", + "variant": "target", + "desription": ["130-hprof dumps the heap and runs hprof-conv to check", + "whether the file is somewhat readable. Thi is only", + "possible on the host. TODO: Turn off all the other", + "combinations, this is more about testing actual ART", + "code. A gtest is very hard to write here, as (for a", + "complete test) JDWP must be set up."] + }, + { + "test": "131-structural-change", + "variant": "debug", + "description": ["131 is an old test. The functionality has been", + "implemented at an earlier stage and is checked", + "in tests 138. Blacklisted for debug builds since", + "these builds have duplicate classes checks which", + "punt to interpreter"] + }, + { + "test": "138-duplicate-classes-check", + "variant": "ndebug", + "description": ["Turned on for debug builds since debug builds have", + "duplicate classes checks enabled"], + "bug": "http://b/2133391" + }, + { + "test": "147-stripped-dex-fallback", + "variant": "no-dex2oat | no-image", + "description": ["147-stripped-dex-fallback is disabled because it", + "requires --prebuild."] + }, + { + "test": "554-jit-profile-file", + "variant": "no-dex2oat | no-image | relocate-npatchoat", + "description": ["554-jit-profile-file is disabled because it needs a", + "primary oat file to know what it should save."] + }, + { + "tests": ["116-nodex2oat", + "117-nopatchoat", + "118-noimage-dex2oat", + "119-noimage-patchoat", + "137-cfi", + "138-duplicate-classes-check2"], + "variant": "no-dex2oat | no-image", + "description": ["All these tests check that we have sane behavior if we", + "don't have a patchoat or dex2oat. Therefore we", + "shouldn't run them in situations where we actually", + "don't have these since they explicitly test for them.", + "These all also assume we have an image."] + }, + { + "tests": ["137-cfi", + "138-duplicate-classes-check", + "018-stack-overflow", + "961-default-iface-resolution-gen", + "964-default-iface-init"], + "variant": "no-image", + "description": ["This test fails without an image. 018, 961, 964 often", + "time out."], + "bug": "http://b/34369284" + }, + { + "test": "137-cfi", + "description": ["This test unrolls and expects managed frames, but", + "tracing means we run the interpreter."], + "variant": "trace | stream" + }, + { + "tests": ["802-deoptimization", + "570-checker-osr"], + "description": ["This test dynamically enables tracing to force a", + "deoptimization. This makes the test meaningless", + "when already tracing, and writes an error message", + "that we do not want to check for."], + "variant": "trace | stream" + }, + { + "test": "130-hprof", + "description": "130 occasional timeout", + "bug": "http://b/32383962", + "variant": "trace | stream" + }, + { + "tests": ["087-gc-after-link", + "141-class-unload"], + "variant": "trace | stream" + }, + { + "tests": ["604-hot-static-interface", + "612-jit-dex-cache", + "613-inlining-dex-cache", + "616-cha", + "626-set-resolved-string"], + "variant": "trace | stream", + "description": ["These tests expect JIT compilation, which is", + "suppressed when tracing."] + }, + { + "test": "137-cfi", + "description": ["CFI unwinding expects managed frames, and the test", + "does not iterate enough to even compile. JIT also", + "uses Generic JNI instead of the JNI compiler."], + "variant": "interpreter | jit" + }, + { + "test": "902-hello-transformation", + "description": "Test 902 hits races with the JIT compiler.", + "bug": "http://b/32821077", + "variant": "jit" + }, + { + "test": "906-iterate-heap", + "description": ["Test 906 iterates the heap filtering with different", + "options. No instances should be created between those", + "runs to be able to have precise checks."], + "variant": "jit" + }, + { + "tests": ["914-hello-obsolescence", + "915-obsolete-2", + "917-fields-transformation", + "919-obsolete-fields"], + "description": ["Test 914, 915, 917, & 919 are very sensitive to the", + "exact state of the stack, including the jit-inserted", + "runtime frames. This causes them to be somewhat flaky", + "as JIT tests. This should be fixed once b/33630159 or", + "b/33616143 are resolved but until then just disable", + "them. Test 916 already checks this feature for JIT", + "use cases in a way that is resilient to the jit", + "frames."], + "variant": "jit" + }, + { + "tests": ["926-multi-obsolescence", + "904-object-allocation"], + "variant": "jit" + }, + { + "test": "912-classes", + "variant": "jit", + "bug": "http://b/34655682" + }, + { + "tests": ["570-checker-select", + "484-checker-register-hints"], + "description": ["These tests were based on the linear scan allocator,", + "which makes different decisions than the graph", + "coloring allocator. (These attempt to test for code", + "quality, not correctness.)"], + "variant": "regalloc_gc" + }, + { + "tests": ["454-get-vreg", + "457-regs", + "602-deoptimizeable"], + "description": ["Tests that should fail when the optimizing compiler ", + "compiles them non-debuggable."], + "variant": "optimizing & ndebuggable | regalloc_gc & ndebuggable" + }, + { + "test": "596-app-images", + "variant": "npictest" + }, + { + "test": "055-enum-performance", + "variant": "optimizing | regalloc_gc", + "description": ["055: Exceeds run time limits due to heap poisoning ", + "instrumentation (on ARM and ARM64 devices)."] + }, + { + "test": "909-attach-agent", + "variant": "debuggable", + "description": "Tests that check semantics for a non-debuggable app." + }, + { + "tests": ["000-nop", + "134-nodex2oat-nofallback", + "147-stripped-dex-fallback", + "595-profile-saving"], + "description": "The doesn't compile anything", + "env_vars": {"ART_TEST_BISECTION": "true"}, + "variant": "optimizing | regalloc_gc" + }, + { + "tests": "089-many-methods", + "description": "The test tests a build failure", + "env_vars": {"ART_TEST_BISECTION": "true"}, + "variant": "optimizing | regalloc_gc" + }, + { + "tests": ["018-stack-overflow", + "116-nodex2oat", + "117-nopatchoat", + "118-noimage-dex2oat", + "119-noimage-patchoat", + "126-miranda-multidex", + "137-cfi"], + "description": "The test run dalvikvm more than once.", + "env_vars": {"ART_TEST_BISECTION": "true"}, + "variant": "optimizing | regalloc_gc" + }, + { + "tests": ["115-native-bridge", + "088-monitor-verification"], + "description": "The test assume they are always compiled.", + "env_vars": {"ART_TEST_BISECTION": "true"}, + "variant": "optimizing | regalloc_gc" + }, + { + "test": "055-enum-performance", + "description": ["The test tests performance which degrades during", + "bisecting."], + "env_vars": {"ART_TEST_BISECTION": "true"}, + "variant": "optimizing | regalloc_gc" + }, + { + "test": "537-checker-arraycopy", + "env_vars": {"ART_USE_READ_BARRIER": "true"}, + "variant": "interpreter | optimizing | regalloc_gc | jit" + } +] diff --git a/test/testrunner/env.py b/test/testrunner/env.py new file mode 100644 index 000000000..e015d7427 --- /dev/null +++ b/test/testrunner/env.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python +# +# Copyright 2017, The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import tempfile +import subprocess + +env = dict(os.environ) + +def getEnvBoolean(var, default): + val = env.get(var) + if val: + if val == "True" or val == "true": + return True + if val == "False" or val == "false": + return False + return default + +def get_build_var(var_name): + # The command is taken from build/envsetup.sh to fetch build variables. + command = ("CALLED_FROM_SETUP=true BUILD_SYSTEM=build/core " + "make --no-print-directory -C \"%s\" -f build/core/config.mk " + "dumpvar-%s") % (ANDROID_BUILD_TOP, var_name) + config = subprocess.Popen(command, stdout=subprocess.PIPE, + shell=True).communicate()[0] + return config.strip() + +def get_env(key): + return env.get(key) + +ANDROID_BUILD_TOP = env.get('ANDROID_BUILD_TOP', os.getcwd()) + +# Directory used for temporary test files on the host. +ART_HOST_TEST_DIR = tempfile.mkdtemp(prefix = 'test-art-') + +# Keep going after encountering a test failure? +ART_TEST_KEEP_GOING = getEnvBoolean('ART_TEST_KEEP_GOING', True) + +# Do you want all tests, even those that are time consuming? +ART_TEST_FULL = getEnvBoolean('ART_TEST_FULL', False) + +# Do you want interpreter tests run? +ART_TEST_INTERPRETER = getEnvBoolean('ART_TEST_INTERPRETER', ART_TEST_FULL) +ART_TEST_INTERPRETER_ACCESS_CHECKS = getEnvBoolean('ART_TEST_INTERPRETER_ACCESS_CHECKS', + ART_TEST_FULL) + +# Do you want JIT tests run? +ART_TEST_JIT = getEnvBoolean('ART_TEST_JIT', ART_TEST_FULL) + +# Do you want optimizing compiler tests run? +ART_TEST_OPTIMIZING = getEnvBoolean('ART_TEST_OPTIMIZING', True) + +# Do you want to test the optimizing compiler with graph coloring register allocation? +ART_TEST_OPTIMIZING_GRAPH_COLOR = getEnvBoolean('ART_TEST_OPTIMIZING_GRAPH_COLOR', ART_TEST_FULL) + +# Do we want to test a non-PIC-compiled core image? +ART_TEST_NPIC_IMAGE = getEnvBoolean('ART_TEST_NPIC_IMAGE', ART_TEST_FULL) + +# Do we want to test PIC-compiled tests ("apps")? +ART_TEST_PIC_TEST = getEnvBoolean('ART_TEST_PIC_TEST', ART_TEST_FULL) +# Do you want tracing tests run? +ART_TEST_TRACE = getEnvBoolean('ART_TEST_TRACE', ART_TEST_FULL) + +# Do you want tracing tests (streaming mode) run? +ART_TEST_TRACE_STREAM = getEnvBoolean('ART_TEST_TRACE_STREAM', ART_TEST_FULL) + +# Do you want tests with GC verification enabled run? +ART_TEST_GC_VERIFY = getEnvBoolean('ART_TEST_GC_VERIFY', ART_TEST_FULL) + +# Do you want tests with the GC stress mode enabled run? +ART_TEST_GC_STRESS = getEnvBoolean('ART_TEST_GC_STRESS', ART_TEST_FULL) + +# Do you want tests with the JNI forcecopy mode enabled run? +ART_TEST_JNI_FORCECOPY = getEnvBoolean('ART_TEST_JNI_FORCECOPY', ART_TEST_FULL) + +# Do you want run-tests with relocation disabled run? +ART_TEST_RUN_TEST_RELOCATE = getEnvBoolean('ART_TEST_RUN_TEST_RELOCATE', ART_TEST_FULL) + +# Do you want run-tests with prebuilding? +ART_TEST_RUN_TEST_PREBUILD = getEnvBoolean('ART_TEST_RUN_TEST_PREBUILD', True) + +# Do you want run-tests with no prebuilding enabled run? +ART_TEST_RUN_TEST_NO_PREBUILD = getEnvBoolean('ART_TEST_RUN_TEST_NO_PREBUILD', ART_TEST_FULL) + +# Do you want run-tests with a pregenerated core.art? +ART_TEST_RUN_TEST_IMAGE = getEnvBoolean('ART_TEST_RUN_TEST_IMAGE', True) + +# Do you want run-tests without a pregenerated core.art? +ART_TEST_RUN_TEST_NO_IMAGE = getEnvBoolean('ART_TEST_RUN_TEST_NO_IMAGE', ART_TEST_FULL) + +# Do you want run-tests with relocation enabled but patchoat failing? +ART_TEST_RUN_TEST_RELOCATE_NO_PATCHOAT = getEnvBoolean('ART_TEST_RUN_TEST_RELOCATE_NO_PATCHOAT', + ART_TEST_FULL) + +# Do you want run-tests without a dex2oat? +ART_TEST_RUN_TEST_NO_DEX2OAT = getEnvBoolean('ART_TEST_RUN_TEST_NO_DEX2OAT', ART_TEST_FULL) + +# Do you want run-tests with libartd.so? +ART_TEST_RUN_TEST_DEBUG = getEnvBoolean('ART_TEST_RUN_TEST_DEBUG', True) + +# Do you want run-tests with libart.so? +ART_TEST_RUN_TEST_NDEBUG = getEnvBoolean('ART_TEST_RUN_TEST_NDEBUG', ART_TEST_FULL) + +# Do you want failed tests to have their artifacts cleaned up? +ART_TEST_RUN_TEST_ALWAYS_CLEAN = getEnvBoolean('ART_TEST_RUN_TEST_ALWAYS_CLEAN', True) + +# Do you want run-tests with the --debuggable flag +ART_TEST_RUN_TEST_DEBUGGABLE = getEnvBoolean('ART_TEST_RUN_TEST_DEBUGGABLE', ART_TEST_FULL) + +# Do you want to test multi-part boot-image functionality? +ART_TEST_RUN_TEST_MULTI_IMAGE = getEnvBoolean('ART_TEST_RUN_TEST_MULTI_IMAGE', ART_TEST_FULL) + +ART_TEST_DEBUG_GC = getEnvBoolean('ART_TEST_DEBUG_GC', False) + +ART_TEST_BISECTION = getEnvBoolean('ART_TEST_BISECTION', False) + +DEX2OAT_HOST_INSTRUCTION_SET_FEATURES = env.get('DEX2OAT_HOST_INSTRUCTION_SET_FEATURES') + +# Do you want run-tests with the host/target's second arch? +ART_TEST_RUN_TEST_2ND_ARCH = getEnvBoolean('ART_TEST_RUN_TEST_2ND_ARCH', True) + +HOST_2ND_ARCH_PREFIX = get_build_var('HOST_2ND_ARCH_PREFIX') +HOST_2ND_ARCH_PREFIX_DEX2OAT_HOST_INSTRUCTION_SET_FEATURES = env.get( + HOST_2ND_ARCH_PREFIX + 'DEX2OAT_HOST_INSTRUCTION_SET_FEATURES') + +ART_TEST_ANDROID_ROOT = env.get('ART_TEST_ANDROID_ROOT') + +ART_TEST_WITH_STRACE = getEnvBoolean('ART_TEST_DEBUG_GC', False) + +TARGET_2ND_ARCH = get_build_var('TARGET_2ND_ARCH') +TARGET_ARCH = get_build_var('TARGET_ARCH') +if TARGET_2ND_ARCH: + if "64" in TARGET_ARCH: + ART_PHONY_TEST_TARGET_SUFFIX = "64" + _2ND_ART_PHONY_TEST_TARGET_SUFFIX = "32" + else: + ART_PHONY_TEST_TARGET_SUFFIX = "32" + _2ND_ART_PHONY_TEST_TARGET_SUFFIX = "" +else: + if "64" in TARGET_ARCH: + ART_PHONY_TEST_TARGET_SUFFIX = "64" + _2ND_ART_PHONY_TEST_TARGET_SUFFIX = "" + else: + ART_PHONY_TEST_TARGET_SUFFIX = "32" + _2ND_ART_PHONY_TEST_TARGET_SUFFIX = "" + +HOST_PREFER_32_BIT = get_build_var('HOST_PREFER_32_BIT') +if HOST_PREFER_32_BIT == "true": + ART_PHONY_TEST_HOST_SUFFIX = "32" + _2ND_ART_PHONY_TEST_HOST_SUFFIX = "" +else: + ART_PHONY_TEST_HOST_SUFFIX = "64" + _2ND_ART_PHONY_TEST_HOST_SUFFIX = "32" + +HOST_OUT_EXECUTABLES = ('%s/%s') % (ANDROID_BUILD_TOP, + get_build_var("HOST_OUT_EXECUTABLES")) +os.environ['JACK'] = HOST_OUT_EXECUTABLES + '/jack' +os.environ['DX'] = HOST_OUT_EXECUTABLES + '/dx' +os.environ['SMALI'] = HOST_OUT_EXECUTABLES + '/smali' +os.environ['JASMIN'] = HOST_OUT_EXECUTABLES + '/jasmin' +os.environ['DXMERGER'] = HOST_OUT_EXECUTABLES + '/dexmerger' diff --git a/test/testrunner/testrunner.py b/test/testrunner/testrunner.py new file mode 100755 index 000000000..f237268f6 --- /dev/null +++ b/test/testrunner/testrunner.py @@ -0,0 +1,794 @@ +#!/usr/bin/env python +# +# Copyright 2017, The Android Open Source Project +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""ART Run-Test TestRunner + +The testrunner runs the ART run-tests by simply invoking the script. +It fetches the list of eligible tests from art/test directory, and list of +disabled tests from art/test/knownfailures.json. It runs the tests by +invoking art/test/run-test script and checks the exit value to decide if the +test passed or failed. + +Before invoking the script, first build all the tests dependencies. +There are two major build targets for building target and host tests +dependencies: +1) test-art-host-run-test +2) test-art-target-run-test + +There are various options to invoke the script which are: +-t: Either the test name as in art/test or the test name including the variant + information. Eg, "-t 001-HelloWorld", + "-t test-art-host-run-test-debug-prebuild-optimizing-relocate-ntrace-cms-checkjni-picimage-npictest-ndebuggable-001-HelloWorld32" +-j: Number of thread workers to be used. Eg - "-j64" +--dry-run: Instead of running the test name, just print its name. +--verbose +-b / --build-dependencies: to build the dependencies before running the test + +To specify any specific variants for the test, use --<>. +For eg, for compiler type as optimizing, use --optimizing. + + +In the end, the script will print the failed and skipped tests if any. + +""" +import fnmatch +import itertools +import json +from optparse import OptionParser +import os +import re +import subprocess +import sys +import threading +import time + +import env + +TARGET_TYPES = set() +RUN_TYPES = set() +PREBUILD_TYPES = set() +COMPILER_TYPES = set() +RELOCATE_TYPES = set() +TRACE_TYPES = set() +GC_TYPES = set() +JNI_TYPES = set() +IMAGE_TYPES = set() +PICTEST_TYPES = set() +DEBUGGABLE_TYPES = set() +ADDRESS_SIZES = set() +OPTIMIZING_COMPILER_TYPES = set() +ADDRESS_SIZES_TARGET = {'host': set(), 'target': set()} + +# DISABLED_TEST_CONTAINER holds information about the disabled tests. It is a map +# that has key as the test name (like 001-HelloWorld), and value as set of +# variants that the test is disabled for. +DISABLED_TEST_CONTAINER = {} + +# The Dict contains the list of all possible variants for a given type. For example, +# for key TARGET, the value would be target and host. The list is used to parse +# the test name given as the argument to run. +VARIANT_TYPE_DICT = {} + +# The set contains all the variants of each time. +TOTAL_VARIANTS_SET = set() + +# The colors are used in the output. When a test passes, COLOR_PASS is used, +# and so on. +COLOR_ERROR = '\033[91m' +COLOR_PASS = '\033[92m' +COLOR_SKIP = '\033[93m' +COLOR_NORMAL = '\033[0m' + +# The mutex object is used by the threads for exclusive access of test_count +# to make any changes in its value. +test_count_mutex = threading.Lock() +# The set contains the list of all the possible run tests that are in art/test +# directory. +RUN_TEST_SET = set() +# The semaphore object is used by the testrunner to limit the number of +# threads to the user requested concurrency value. +semaphore = threading.Semaphore(1) +# The mutex object is used to provide exclusive access to a thread to print +# its output. +print_mutex = threading.Lock() +failed_tests = [] +skipped_tests = [] + +# Flags +n_thread = 1 +test_count = 0 +total_test_count = 0 +verbose = False +last_print_length = 0 +dry_run = False +build = False +gdb = False +gdb_arg = '' +stop_testrunner = False + +def gather_test_info(): + """The method gathers test information about the test to be run which includes + generating the list of total tests from the art/test directory and the list + of disabled test. It also maps various variants to types. + """ + global TOTAL_VARIANTS_SET + global DISABLED_TEST_CONTAINER + # TODO: Avoid duplication of the variant names in different lists. + VARIANT_TYPE_DICT['pictest'] = {'pictest', 'npictest'} + VARIANT_TYPE_DICT['run'] = {'ndebug', 'debug'} + VARIANT_TYPE_DICT['target'] = {'target', 'host'} + VARIANT_TYPE_DICT['trace'] = {'trace', 'ntrace', 'stream'} + VARIANT_TYPE_DICT['image'] = {'picimage', 'no-image', 'npicimage', + 'multinpicimage', 'multipicimage'} + VARIANT_TYPE_DICT['debuggable'] = {'ndebuggable', 'debuggable'} + VARIANT_TYPE_DICT['gc'] = {'gcstress', 'gcverify', 'cms'} + VARIANT_TYPE_DICT['prebuild'] = {'no-prebuild', 'no-dex2oat', 'prebuild'} + VARIANT_TYPE_DICT['relocate'] = {'relocate-npatchoat', 'relocate', 'no-relocate'} + VARIANT_TYPE_DICT['jni'] = {'jni', 'forcecopy', 'checkjni'} + VARIANT_TYPE_DICT['address_sizes'] = {'64', '32'} + VARIANT_TYPE_DICT['compiler'] = {'interp-ac', 'interpreter', 'jit', 'optimizing', + 'regalloc_gc'} + + for v_type in VARIANT_TYPE_DICT: + TOTAL_VARIANTS_SET = TOTAL_VARIANTS_SET.union(VARIANT_TYPE_DICT.get(v_type)) + + test_dir = env.ANDROID_BUILD_TOP + '/art/test' + for f in os.listdir(test_dir): + if fnmatch.fnmatch(f, '[0-9]*'): + RUN_TEST_SET.add(f) + DISABLED_TEST_CONTAINER = get_disabled_test_info() + + +def setup_test_env(): + """The method sets default value for the various variants of the tests if they + are already not set. + """ + if env.ART_TEST_BISECTION: + env.ART_TEST_RUN_TEST_NO_PREBUILD = True + env.ART_TEST_RUN_TEST_PREBUILD = False + # Bisection search writes to standard output. + env.ART_TEST_QUIET = False + + if not TARGET_TYPES: + TARGET_TYPES.add('host') + TARGET_TYPES.add('target') + + if not PREBUILD_TYPES: + if env.ART_TEST_RUN_TEST_PREBUILD: + PREBUILD_TYPES.add('prebuild') + if env.ART_TEST_RUN_TEST_NO_PREBUILD: + PREBUILD_TYPES.add('no-prebuild') + if env.ART_TEST_RUN_TEST_NO_DEX2OAT: + PREBUILD_TYPES.add('no-dex2oat') + + if not COMPILER_TYPES: + if env.ART_TEST_INTERPRETER_ACCESS_CHECKS: + COMPILER_TYPES.add('interp-ac') + if env.ART_TEST_INTERPRETER: + COMPILER_TYPES.add('interpreter') + if env.ART_TEST_JIT: + COMPILER_TYPES.add('jit') + + if env.ART_TEST_OPTIMIZING: + COMPILER_TYPES.add('optimizing') + OPTIMIZING_COMPILER_TYPES.add('optimizing') + if env.ART_TEST_OPTIMIZING_GRAPH_COLOR: + COMPILER_TYPES.add('regalloc_gc') + OPTIMIZING_COMPILER_TYPES.add('regalloc_gc') + + if not RELOCATE_TYPES: + RELOCATE_TYPES.add('no-relocate') + if env.ART_TEST_RUN_TEST_RELOCATE: + RELOCATE_TYPES.add('relocate') + if env.ART_TEST_RUN_TEST_RELOCATE_NO_PATCHOAT: + RELOCATE_TYPES.add('relocate-npatchoat') + + if not TRACE_TYPES: + TRACE_TYPES.add('ntrace') + if env.ART_TEST_TRACE: + TRACE_TYPES.add('trace') + if env.ART_TEST_TRACE_STREAM: + TRACE_TYPES.add('stream') + + if not GC_TYPES: + GC_TYPES.add('cms') + if env.ART_TEST_GC_STRESS: + GC_TYPES.add('gcstress') + if env.ART_TEST_GC_VERIFY: + GC_TYPES.add('gcverify') + + if not JNI_TYPES: + JNI_TYPES.add('checkjni') + if env.ART_TEST_JNI_FORCECOPY: + JNI_TYPES.add('forcecopy') + + if not IMAGE_TYPES: + IMAGE_TYPES.add('picimage') + if env.ART_TEST_RUN_TEST_NO_IMAGE: + IMAGE_TYPES.add('no-image') + if env.ART_TEST_RUN_TEST_MULTI_IMAGE: + IMAGE_TYPES.add('multipicimage') + if env.ART_TEST_NPIC_IMAGE: + IMAGE_TYPES.add('npicimage') + if env.ART_TEST_RUN_TEST_MULTI_IMAGE: + IMAGE_TYPES.add('multinpicimage') + + if not PICTEST_TYPES: + PICTEST_TYPES.add('npictest') + if env.ART_TEST_PIC_TEST: + PICTEST_TYPES.add('pictest') + + if not RUN_TYPES: + if env.ART_TEST_RUN_TEST_DEBUG: + RUN_TYPES.add('debug') + if env.ART_TEST_RUN_TEST_NDEBUG: + RUN_TYPES.add('ndebug') + + if not DEBUGGABLE_TYPES: + DEBUGGABLE_TYPES.add('ndebuggable') + + if env.ART_TEST_RUN_TEST_DEBUGGABLE: + DEBUGGABLE_TYPES.add('debuggable') + + if not ADDRESS_SIZES: + ADDRESS_SIZES_TARGET['target'].add(env.ART_PHONY_TEST_TARGET_SUFFIX) + ADDRESS_SIZES_TARGET['host'].add(env.ART_PHONY_TEST_HOST_SUFFIX) + if env.ART_TEST_RUN_TEST_2ND_ARCH: + ADDRESS_SIZES_TARGET['host'].add(env._2ND_ART_PHONY_TEST_HOST_SUFFIX) + ADDRESS_SIZES_TARGET['target'].add(env._2ND_ART_PHONY_TEST_TARGET_SUFFIX) + else: + ADDRESS_SIZES_TARGET['host'] = ADDRESS_SIZES_TARGET['host'].union(ADDRESS_SIZES) + ADDRESS_SIZES_TARGET['target'] = ADDRESS_SIZES_TARGET['target'].union(ADDRESS_SIZES) + + global semaphore + semaphore = threading.Semaphore(n_thread) + + +def run_tests(tests): + """Creates thread workers to run the tests. + + The method generates command and thread worker to run the tests. Depending on + the user input for the number of threads to be used, the method uses a + semaphore object to keep a count in control for the thread workers. When a new + worker is created, it acquires the semaphore object, and when the number of + workers reaches the maximum allowed concurrency, the method wait for an + existing thread worker to release the semaphore object. Worker releases the + semaphore object when they finish printing the output. + + Args: + tests: The set of tests to be run. + """ + options_all = '' + global total_test_count + total_test_count = len(tests) + total_test_count *= len(RUN_TYPES) + total_test_count *= len(PREBUILD_TYPES) + total_test_count *= len(RELOCATE_TYPES) + total_test_count *= len(TRACE_TYPES) + total_test_count *= len(GC_TYPES) + total_test_count *= len(JNI_TYPES) + total_test_count *= len(IMAGE_TYPES) + total_test_count *= len(PICTEST_TYPES) + total_test_count *= len(DEBUGGABLE_TYPES) + total_test_count *= len(COMPILER_TYPES) + target_address_combinations = 0 + for target in TARGET_TYPES: + for address_size in ADDRESS_SIZES_TARGET[target]: + target_address_combinations += 1 + total_test_count *= target_address_combinations + + if env.ART_TEST_WITH_STRACE: + options_all += ' --strace' + + if env.ART_TEST_RUN_TEST_ALWAYS_CLEAN: + options_all += ' --always-clean' + + if env.ART_TEST_BISECTION: + options_all += ' --bisection-search' + + if env.ART_TEST_ANDROID_ROOT: + options_all += ' --android-root ' + env.ART_TEST_ARDROID_ROOT + + if gdb: + options_all += ' --gdb' + if gdb_arg: + options_all += ' --gdb-arg ' + gdb_arg + + config = itertools.product(tests, TARGET_TYPES, RUN_TYPES, PREBUILD_TYPES, + COMPILER_TYPES, RELOCATE_TYPES, TRACE_TYPES, + GC_TYPES, JNI_TYPES, IMAGE_TYPES, PICTEST_TYPES, + DEBUGGABLE_TYPES) + + for test, target, run, prebuild, compiler, relocate, trace, gc, \ + jni, image, pictest, debuggable in config: + for address_size in ADDRESS_SIZES_TARGET[target]: + if stop_testrunner: + # When ART_TEST_KEEP_GOING is set to false, then as soon as a test + # fails, stop_testrunner is set to True. When this happens, the method + # stops creating any any thread and wait for all the exising threads + # to end. + while threading.active_count() > 2: + time.sleep(0.1) + return + test_name = 'test-art-' + test_name += target + '-run-test-' + test_name += run + '-' + test_name += prebuild + '-' + test_name += compiler + '-' + test_name += relocate + '-' + test_name += trace + '-' + test_name += gc + '-' + test_name += jni + '-' + test_name += image + '-' + test_name += pictest + '-' + test_name += debuggable + '-' + test_name += test + test_name += address_size + + variant_set = {target, run, prebuild, compiler, relocate, trace, gc, jni, + image, pictest, debuggable, address_size} + + options_test = options_all + + if target == 'host': + options_test += ' --host' + + if run == 'ndebug': + options_test += ' -O' + + if prebuild == 'prebuild': + options_test += ' --prebuild' + elif prebuild == 'no-prebuild': + options_test += ' --no-prebuild' + elif prebuild == 'no-dex2oat': + options_test += ' --no-prebuild --no-dex2oat' + + if compiler == 'optimizing': + options_test += ' --optimizing' + elif compiler == 'regalloc_gc': + options_test += ' --optimizing -Xcompiler-option --register-allocation-strategy=graph-color' + elif compiler == 'interpreter': + options_test += ' --interpreter' + elif compiler == 'interp-ac': + options_test += ' --interpreter --verify-soft-fail' + elif compiler == 'jit': + options_test += ' --jit' + + if relocate == 'relocate': + options_test += ' --relocate' + elif relocate == 'no-relocate': + options_test += ' --no-relocate' + elif relocate == 'relocate-npatchoat': + options_test += ' --relocate --no-patchoat' + + if trace == 'trace': + options_test += ' --trace' + elif trace == 'stream': + options_test += ' --trace --stream' + + if gc == 'gcverify': + options_test += ' --gcverify' + elif gc == 'gcstress': + options_test += ' --gcstress' + + if jni == 'forcecopy': + options_test += ' --runtime-option -Xjniopts:forcecopy' + elif jni == 'checkjni': + options_test += ' --runtime-option -Xcheck:jni' + + if image == 'no-image': + options_test += ' --no-image' + elif image == 'npicimage': + options_test += ' --npic-image' + elif image == 'multinpicimage': + options_test += ' --npic-image --multi-image' + elif image == 'multipicimage': + options_test += ' --multi-image' + + if pictest == 'pictest': + options_test += ' --pic-test' + + if debuggable == 'debuggable': + options_test += ' --debuggable' + + if address_size == '64': + options_test += ' --64' + + if env.DEX2OAT_HOST_INSTRUCTION_SET_FEATURES: + options_test += ' --instruction-set-features' + env.DEX2OAT_HOST_INSTRUCTION_SET_FEATURES + + elif address_size == '32': + if env.HOST_2ND_ARCH_PREFIX_DEX2OAT_HOST_INSTRUCTION_SET_FEATURES: + options_test += ' --instruction-set-features ' + \ + env.HOST_2ND_ARCH_PREFIX_DEX2OAT_HOST_INSTRUCTION_SET_FEATURES + + options_test = (' --output-path %s/run-test-output/%s') % ( + env.ART_HOST_TEST_DIR, test_name) + options_test + + run_test_sh = env.ANDROID_BUILD_TOP + '/art/test/run-test' + command = run_test_sh + ' ' + options_test + ' ' + test + + semaphore.acquire() + worker = threading.Thread(target=run_test, args=(command, test, variant_set, test_name)) + worker.daemon = True + worker.start() + + while threading.active_count() > 2: + time.sleep(0.1) + + +def run_test(command, test, test_variant, test_name): + """Runs the test. + + It invokes art/test/run-test script to run the test. The output of the script + is checked, and if it ends with "Succeeded!", it assumes that the tests + passed, otherwise, put it in the list of failed test. Before actually running + the test, it also checks if the test is placed in the list of disabled tests, + and if yes, it skips running it, and adds the test in the list of skipped + tests. The method uses print_text method to actually print the output. After + successfully running and capturing the output for the test, it releases the + semaphore object. + + Args: + command: The command to be used to invoke the script + test: The name of the test without the variant information. + test_variant: The set of variant for the test. + test_name: The name of the test along with the variants. + """ + global last_print_length + global test_count + global stop_testrunner + if is_test_disabled(test, test_variant): + test_skipped = True + else: + test_skipped = False + proc = subprocess.Popen(command.split(), stderr=subprocess.PIPE) + script_output = proc.stderr.read().strip() + test_passed = not proc.wait() + + # If verbose is set to True, every test information is printed on a new line. + # If not, the information is printed on the same line overriding the + # previous test output. + if not verbose: + suffix = '\r' + prefix = ' ' * last_print_length + '\r' + else: + suffix = '\n' + prefix = '' + test_count_mutex.acquire() + test_count += 1 + percent = (test_count * 100) / total_test_count + out = '[ ' + str(percent) + '% ' + str(test_count) + '/' + str(total_test_count) + ' ] ' + test_count_mutex.release() + out += test_name + ' ' + if not test_skipped: + if test_passed: + out += COLOR_PASS + 'PASS' + COLOR_NORMAL + else: + out += COLOR_ERROR + 'FAIL' + COLOR_NORMAL + failed_tests.append(test_name) + if verbose: + out += '\n' + command + '\n' + script_output + if not env.ART_TEST_KEEP_GOING: + stop_testrunner = True + elif not dry_run: + out += COLOR_SKIP + 'SKIP' + COLOR_NORMAL + skipped_tests.append(test_name) + last_print_length = len(out) + print_mutex.acquire() + print_text(prefix + out + suffix) + print_mutex.release() + semaphore.release() + + +def get_disabled_test_info(): + """Generate set of known failures. + + It parses the art/test/knownfailures.json file to generate the list of + disabled tests. + + Returns: + The method returns a dict of tests mapped to the variants list + for which the test should not be run. + """ + known_failures_file = env.ANDROID_BUILD_TOP + '/art/test/knownfailures.json' + with open(known_failures_file) as known_failures_json: + known_failures_info = json.loads(known_failures_json.read()) + + disabled_test_info = {} + for failure in known_failures_info: + tests = failure.get('test') + if tests: + tests = [tests] + else: + tests = failure.get('tests', []) + variants = parse_variants(failure.get('variant')) + env_vars = failure.get('env_vars') + if check_env_vars(env_vars): + for test in tests: + if test in disabled_test_info: + disabled_test_info[test] = disabled_test_info[test].union(variants) + else: + disabled_test_info[test] = variants + return disabled_test_info + + +def check_env_vars(env_vars): + """Checks if the env variables are set as required to run the test. + + Returns: + True if all the env variables are set as required, otherwise False. + """ + + if not env_vars: + return True + for key in env_vars: + if env.get_env(key) != env_vars.get(key): + return False + return True + + +def is_test_disabled(test, variant_set): + """Checks if the test along with the variant_set is disabled. + + Args: + test: The name of the test as in art/test directory. + variant_set: Variants to be used for the test. + Returns: + True, if the test is disabled. + """ + if dry_run: + return True + variants_list = DISABLED_TEST_CONTAINER.get(test, {}) + for variants in variants_list: + variants_present = True + for variant in variants: + if variant not in variant_set: + variants_present = False + break + if variants_present: + return True + return False + + +def parse_variants(variants): + """Parse variants fetched from art/test/knownfailures.json. + """ + if not variants: + variants = '' + for variant in TOTAL_VARIANTS_SET: + variants += variant + variants += '|' + variants = variants[:-1] + variant_list = set() + or_variants = variants.split('|') + for or_variant in or_variants: + and_variants = or_variant.split('&') + variant = set() + for and_variant in and_variants: + and_variant = and_variant.strip() + variant.add(and_variant) + variant_list.add(frozenset(variant)) + return variant_list + + +def print_text(output): + sys.stdout.write(output) + sys.stdout.flush() + + +def print_analysis(): + if not verbose: + print_text(' ' * last_print_length + '\r') + if skipped_tests: + print_text(COLOR_SKIP + 'SKIPPED TESTS' + COLOR_NORMAL + '\n') + for test in skipped_tests: + print_text(test + '\n') + print_text('\n') + + if failed_tests: + print_text(COLOR_ERROR + 'FAILED TESTS' + COLOR_NORMAL + '\n') + for test in failed_tests: + print_text(test + '\n') + + +def parse_test_name(test_name): + """Parses the testname provided by the user. + It supports two types of test_name: + 1) Like 001-HelloWorld. In this case, it will just verify if the test actually + exists and if it does, it returns the testname. + 2) Like test-art-host-run-test-debug-prebuild-interpreter-no-relocate-ntrace-cms-checkjni-picimage-npictest-ndebuggable-001-HelloWorld32 + In this case, it will parse all the variants and check if they are placed + correctly. If yes, it will set the various VARIANT_TYPES to use the + variants required to run the test. Again, it returns the test_name + without the variant information like 001-HelloWorld. + """ + if test_name in RUN_TEST_SET: + return {test_name} + + regex = '^test-art-' + regex += '(' + '|'.join(VARIANT_TYPE_DICT['target']) + ')-' + regex += 'run-test-' + regex += '(' + '|'.join(VARIANT_TYPE_DICT['run']) + ')-' + regex += '(' + '|'.join(VARIANT_TYPE_DICT['prebuild']) + ')-' + regex += '(' + '|'.join(VARIANT_TYPE_DICT['compiler']) + ')-' + regex += '(' + '|'.join(VARIANT_TYPE_DICT['relocate']) + ')-' + regex += '(' + '|'.join(VARIANT_TYPE_DICT['trace']) + ')-' + regex += '(' + '|'.join(VARIANT_TYPE_DICT['gc']) + ')-' + regex += '(' + '|'.join(VARIANT_TYPE_DICT['jni']) + ')-' + regex += '(' + '|'.join(VARIANT_TYPE_DICT['image']) + ')-' + regex += '(' + '|'.join(VARIANT_TYPE_DICT['pictest']) + ')-' + regex += '(' + '|'.join(VARIANT_TYPE_DICT['debuggable']) + ')-' + regex += '(' + '|'.join(RUN_TEST_SET) + ')' + regex += '(' + '|'.join(VARIANT_TYPE_DICT['address_sizes']) + ')$' + match = re.match(regex, test_name) + if match: + TARGET_TYPES.add(match.group(1)) + RUN_TYPES.add(match.group(2)) + PREBUILD_TYPES.add(match.group(3)) + COMPILER_TYPES.add(match.group(4)) + RELOCATE_TYPES.add(match.group(5)) + TRACE_TYPES.add(match.group(6)) + GC_TYPES.add(match.group(7)) + JNI_TYPES.add(match.group(8)) + IMAGE_TYPES.add(match.group(9)) + PICTEST_TYPES.add(match.group(10)) + DEBUGGABLE_TYPES.add(match.group(11)) + ADDRESS_SIZES.add(match.group(13)) + return {match.group(12)} + + +def parse_option(): + global verbose + global dry_run + global n_thread + global build + global gdb + global gdb_arg + + parser = OptionParser() + parser.add_option('-t', '--test', dest='test', help='name of the test') + parser.add_option('-j', type='int', dest='n_thread') + for variant in TOTAL_VARIANTS_SET: + flag = '--' + variant + flag_dest = variant.replace('-', '_') + if variant == '32' or variant == '64': + flag_dest = 'n' + flag_dest + parser.add_option(flag, action='store_true', dest=flag_dest) + parser.add_option('--verbose', '-v', action='store_true', dest='verbose') + parser.add_option('--dry-run', action='store_true', dest='dry_run') + parser.add_option('-b', '--build-dependencies', action='store_true', dest='build') + parser.add_option('--gdb', action='store_true', dest='gdb') + parser.add_option('--gdb-arg', dest='gdb_arg') + + options = parser.parse_args()[0] + test = '' + if options.test: + test = parse_test_name(options.test) + if options.pictest: + PICTEST_TYPES.add('pictest') + if options.ndebug: + RUN_TYPES.add('ndebug') + if options.interp_ac: + COMPILER_TYPES.add('interp-ac') + if options.picimage: + IMAGE_TYPES.add('picimage') + if options.n64: + ADDRESS_SIZES.add('64') + if options.interpreter: + COMPILER_TYPES.add('interpreter') + if options.jni: + JNI_TYPES.add('jni') + if options.relocate_npatchoat: + RELOCATE_TYPES.add('relocate-npatchoat') + if options.no_prebuild: + PREBUILD_TYPES.add('no-prebuild') + if options.npictest: + PICTEST_TYPES.add('npictest') + if options.no_dex2oat: + PREBUILD_TYPES.add('no-dex2oat') + if options.jit: + COMPILER_TYPES.add('jit') + if options.relocate: + RELOCATE_TYPES.add('relocate') + if options.ndebuggable: + DEBUGGABLE_TYPES.add('ndebuggable') + if options.no_image: + IMAGE_TYPES.add('no-image') + if options.optimizing: + COMPILER_TYPES.add('optimizing') + if options.trace: + TRACE_TYPES.add('trace') + if options.gcstress: + GC_TYPES.add('gcstress') + if options.no_relocate: + RELOCATE_TYPES.add('no-relocate') + if options.target: + TARGET_TYPES.add('target') + if options.forcecopy: + JNI_TYPES.add('forcecopy') + if options.n32: + ADDRESS_SIZES.add('32') + if options.host: + TARGET_TYPES.add('host') + if options.gcverify: + GC_TYPES.add('gcverify') + if options.debuggable: + DEBUGGABLE_TYPES.add('debuggable') + if options.prebuild: + PREBUILD_TYPES.add('prebuild') + if options.debug: + RUN_TYPES.add('debug') + if options.checkjni: + JNI_TYPES.add('checkjni') + if options.ntrace: + TRACE_TYPES.add('ntrace') + if options.cms: + GC_TYPES.add('cms') + if options.npicimage: + IMAGE_TYPES.add('npicimage') + if options.multinpicimage: + IMAGE_TYPES.add('multinpicimage') + if options.multipicimage: + IMAGE_TYPES.add('multipicimage') + if options.verbose: + verbose = True + if options.n_thread: + n_thread = max(1, options.n_thread) + if options.dry_run: + dry_run = True + verbose = True + if options.build: + build = True + if options.gdb: + n_thread = 1 + gdb = True + if options.gdb_arg: + gdb_arg = options.gdb_arg + + return test + +def main(): + gather_test_info() + user_requested_test = parse_option() + setup_test_env() + if build: + build_targets = '' + if 'host' in TARGET_TYPES: + build_targets += 'test-art-host-run-test-dependencies' + if 'target' in TARGET_TYPES: + build_targets += 'test-art-target-run-test-dependencies' + build_command = 'make -j' + str(n_thread) + ' ' + build_targets + if subprocess.call(build_command.split()): + sys.exit(1) + if user_requested_test: + test_runner_thread = threading.Thread(target=run_tests, args=(user_requested_test,)) + else: + test_runner_thread = threading.Thread(target=run_tests, args=(RUN_TEST_SET,)) + test_runner_thread.daemon = True + try: + test_runner_thread.start() + while threading.active_count() > 1: + time.sleep(0.1) + print_analysis() + if failed_tests: + sys.exit(1) + sys.exit(0) + except KeyboardInterrupt: + print_analysis() + +if __name__ == '__main__': + main() -- 2.11.0