Add interface fast path to art_quick_check_cast for X86_64

X86_64 CC ritzperf results from perf:
art_quick_check_cast: 0.44% -> 0.76%
artIsAssignableFromCode: 1.78% -> 0.11%

Added stub test.

Bug: 32577579

Test: test-art-host

Change-Id: I5ed5675c4674fac8eed8826eb50527f4876e5f07

diff --git a/runtime/arch/stub_test.cc b/runtime/arch/stub_test.cc
index 4638c3f..c151f00 100644 (file)
--- a/runtime/arch/stub_test.cc
+++ b/runtime/arch/stub_test.cc
@@ -819,34 +819,60 @@ TEST_F(StubTest, CheckCast) {
   ScopedObjectAccess soa(self);
   // garbage is created during ClassLinker::Init
 
-  StackHandleScope<2> hs(soa.Self());
+  StackHandleScope<4> hs(soa.Self());
   Handle<mirror::Class> c(
       hs.NewHandle(class_linker_->FindSystemClass(soa.Self(), "[Ljava/lang/Object;")));
   Handle<mirror::Class> c2(
       hs.NewHandle(class_linker_->FindSystemClass(soa.Self(), "[Ljava/lang/String;")));
+  Handle<mirror::Class> list(
+      hs.NewHandle(class_linker_->FindSystemClass(soa.Self(), "[Ljava/util/List;")));
+  Handle<mirror::Class> array_list(
+      hs.NewHandle(class_linker_->FindSystemClass(soa.Self(), "[Ljava/util/ArrayList;")));
 
   EXPECT_FALSE(self->IsExceptionPending());
 
-  Invoke3(reinterpret_cast<size_t>(c.Get()), reinterpret_cast<size_t>(c.Get()), 0U,
-          art_quick_check_cast, self);
-
+  Invoke3(reinterpret_cast<size_t>(c.Get()),
+          reinterpret_cast<size_t>(c.Get()),
+          0U,
+          art_quick_check_cast,
+          self);
   EXPECT_FALSE(self->IsExceptionPending());
 
-  Invoke3(reinterpret_cast<size_t>(c2.Get()), reinterpret_cast<size_t>(c2.Get()), 0U,
-          art_quick_check_cast, self);
-
+  Invoke3(reinterpret_cast<size_t>(c2.Get()),
+          reinterpret_cast<size_t>(c2.Get()),
+          0U,
+          art_quick_check_cast,
+          self);
   EXPECT_FALSE(self->IsExceptionPending());
 
-  Invoke3(reinterpret_cast<size_t>(c.Get()), reinterpret_cast<size_t>(c2.Get()), 0U,
-          art_quick_check_cast, self);
-
+  Invoke3(reinterpret_cast<size_t>(c.Get()),
+          reinterpret_cast<size_t>(c2.Get()),
+          0U,
+          art_quick_check_cast,
+          self);
   EXPECT_FALSE(self->IsExceptionPending());
 
-  // TODO: Make the following work. But that would require correct managed frames.
+  Invoke3(reinterpret_cast<size_t>(list.Get()),
+          reinterpret_cast<size_t>(array_list.Get()),
+          0U,
+          art_quick_check_cast,
+          self);
+  EXPECT_FALSE(self->IsExceptionPending());
 
-  Invoke3(reinterpret_cast<size_t>(c2.Get()), reinterpret_cast<size_t>(c.Get()), 0U,
-          art_quick_check_cast, self);
+  Invoke3(reinterpret_cast<size_t>(list.Get()),
+          reinterpret_cast<size_t>(c2.Get()),
+          0U,
+          art_quick_check_cast,
+          self);
+  EXPECT_TRUE(self->IsExceptionPending());
+  self->ClearException();
 
+  // TODO: Make the following work. But that would require correct managed frames.
+  Invoke3(reinterpret_cast<size_t>(c2.Get()),
+          reinterpret_cast<size_t>(c.Get()),
+          0U,
+          art_quick_check_cast,
+          self);
   EXPECT_TRUE(self->IsExceptionPending());
   self->ClearException();
 

diff --git a/runtime/arch/x86_64/quick_entrypoints_x86_64.S b/runtime/arch/x86_64/quick_entrypoints_x86_64.S
index afa1c0f..49e1b56 100644 (file)
--- a/runtime/arch/x86_64/quick_entrypoints_x86_64.S
+++ b/runtime/arch/x86_64/quick_entrypoints_x86_64.S
@@ -1481,6 +1481,32 @@ DEFINE_FUNCTION art_quick_unlock_object_no_inline
 END_FUNCTION art_quick_unlock_object_no_inline
 
 DEFINE_FUNCTION art_quick_check_cast
+    testl LITERAL(ACCESS_FLAGS_CLASS_IS_INTERFACE), MIRROR_CLASS_ACCESS_FLAGS_OFFSET(%rdi)
+    jz .Lnot_interface
+
+    // There are no read barriers since the iftable is immutable. There can be false negatives for
+    // the read barrier case if classes in the IfTable are in the from-space. In the case where
+    // we do not find a matching interface we call into artIsAssignableFromCode which will have
+    // read barriers.
+    movl MIRROR_CLASS_IF_TABLE_OFFSET(%rsi), %ecx
+    UNPOISON_HEAP_REF %ecx
+    testl %ecx, %ecx
+    jz .Lnot_interface
+    movl MIRROR_ARRAY_LENGTH_OFFSET(%rcx), %r8d
+.Lstart_loop:
+    // Re-poison before comparing to prevent rare possible false positives. This is done inside
+    // the loop since heap poisoning is only for testing builds.
+    POISON_HEAP_REF %edi
+    cmpl MIRROR_OBJECT_ARRAY_DATA_OFFSET(%rcx), %edi
+    je .Lreturn  // Return if same class.
+    UNPOISON_HEAP_REF %edi
+    // Go to next interface.
+    add LITERAL(COMPRESSED_REFERENCE_SIZE * 2), %rcx
+    sub LITERAL(2), %r8
+    jnz .Lstart_loop
+
+.Lnot_interface:
+    // We could check the super classes here but that is usually already checked in the caller.
     PUSH rdi                          // Save args for exc
     PUSH rsi
     subq LITERAL(8), %rsp             // Alignment padding.
@@ -1493,6 +1519,7 @@ DEFINE_FUNCTION art_quick_check_cast
     addq LITERAL(24), %rsp            // pop arguments
     CFI_ADJUST_CFA_OFFSET(-24)
 
+.Lreturn:
     ret
 
     CFI_ADJUST_CFA_OFFSET(24 + 4 * 8)  // Reset unwind info so following code unwinds.

diff --git a/runtime/asm_support.h b/runtime/asm_support.h
index cd8815b..1e5e127 100644 (file)
--- a/runtime/asm_support.h
+++ b/runtime/asm_support.h
@@ -172,6 +172,9 @@ ADD_TEST_EQ(size_t(MIRROR_OBJECT_HEADER_SIZE), sizeof(art::mirror::Object))
 #define MIRROR_CLASS_COMPONENT_TYPE_OFFSET (4 + MIRROR_OBJECT_HEADER_SIZE)
 ADD_TEST_EQ(MIRROR_CLASS_COMPONENT_TYPE_OFFSET,
             art::mirror::Class::ComponentTypeOffset().Int32Value())
+#define MIRROR_CLASS_IF_TABLE_OFFSET (12 + MIRROR_OBJECT_HEADER_SIZE)
+ADD_TEST_EQ(MIRROR_CLASS_IF_TABLE_OFFSET,
+            art::mirror::Class::IfTableOffset().Int32Value())
 #define MIRROR_CLASS_ACCESS_FLAGS_OFFSET (64 + MIRROR_OBJECT_HEADER_SIZE)
 ADD_TEST_EQ(MIRROR_CLASS_ACCESS_FLAGS_OFFSET,
             art::mirror::Class::AccessFlagsOffset().Int32Value())

diff --git a/runtime/generated/asm_support_gen.h b/runtime/generated/asm_support_gen.h
index 03f5bf6..6c189b0 100644 (file)
--- a/runtime/generated/asm_support_gen.h
+++ b/runtime/generated/asm_support_gen.h
@@ -52,6 +52,8 @@ DEFINE_CHECK_EQ(static_cast<int32_t>(MIRROR_OBJECT_LOCK_WORD_OFFSET), (static_ca
 DEFINE_CHECK_EQ(static_cast<uint32_t>(MIRROR_CLASS_STATUS_INITIALIZED), (static_cast<uint32_t>((art::mirror::Class::kStatusInitialized))))
 #define ACCESS_FLAGS_CLASS_IS_FINALIZABLE 0x80000000
 DEFINE_CHECK_EQ(static_cast<uint32_t>(ACCESS_FLAGS_CLASS_IS_FINALIZABLE), (static_cast<uint32_t>((art::kAccClassIsFinalizable))))
+#define ACCESS_FLAGS_CLASS_IS_INTERFACE 0x200
+DEFINE_CHECK_EQ(static_cast<uint32_t>(ACCESS_FLAGS_CLASS_IS_INTERFACE), (static_cast<uint32_t>((art::kAccInterface))))
 #define ACCESS_FLAGS_CLASS_IS_FINALIZABLE_BIT 0x1f
 DEFINE_CHECK_EQ(static_cast<uint32_t>(ACCESS_FLAGS_CLASS_IS_FINALIZABLE_BIT), (static_cast<uint32_t>((art::MostSignificantBit(art::kAccClassIsFinalizable)))))
 #define ART_METHOD_DEX_CACHE_METHODS_OFFSET_32 20

diff --git a/runtime/mirror/class-inl.h b/runtime/mirror/class-inl.h
index 9992a9e..bbdb2af 100644 (file)
--- a/runtime/mirror/class-inl.h
+++ b/runtime/mirror/class-inl.h
@@ -526,8 +526,7 @@ inline ArtMethod* Class::FindVirtualMethodForVirtualOrInterface(ArtMethod* metho
 template<VerifyObjectFlags kVerifyFlags,
          ReadBarrierOption kReadBarrierOption>
 inline IfTable* Class::GetIfTable() {
-  return GetFieldObject<IfTable, kVerifyFlags, kReadBarrierOption>(
-      OFFSET_OF_OBJECT_MEMBER(Class, iftable_));
+  return GetFieldObject<IfTable, kVerifyFlags, kReadBarrierOption>(IfTableOffset());
 }
 
 inline int32_t Class::GetIfTableCount() {
@@ -539,7 +538,7 @@ inline int32_t Class::GetIfTableCount() {
 }
 
 inline void Class::SetIfTable(ObjPtr<IfTable> new_iftable) {
-  SetFieldObject<false>(OFFSET_OF_OBJECT_MEMBER(Class, iftable_), new_iftable);
+  SetFieldObject<false>(IfTableOffset(), new_iftable);
 }
 
 inline LengthPrefixedArray<ArtField>* Class::GetIFieldsPtr() {

diff --git a/runtime/mirror/class.h b/runtime/mirror/class.h
index 5793795..f115e6c 100644 (file)
--- a/runtime/mirror/class.h
+++ b/runtime/mirror/class.h
@@ -680,6 +680,10 @@ class MANAGED Class FINAL : public Object {
     return MemberOffset(OFFSETOF_MEMBER(Class, dex_cache_));
   }
 
+  static MemberOffset IfTableOffset() {
+    return MemberOffset(OFFSETOF_MEMBER(Class, iftable_));
+  }
+
   enum {
     kDumpClassFullDetail = 1,
     kDumpClassClassLoader = (1 << 1),

diff --git a/tools/cpp-define-generator/constant_class.def b/tools/cpp-define-generator/constant_class.def
index 58372f9..f46cd33 100644 (file)
--- a/tools/cpp-define-generator/constant_class.def
+++ b/tools/cpp-define-generator/constant_class.def
@@ -25,6 +25,7 @@
 
 DEFINE_FLAG_OFFSET(MIRROR_CLASS, STATUS_INITIALIZED,       art::mirror::Class::kStatusInitialized)
 DEFINE_FLAG_OFFSET(ACCESS_FLAGS, CLASS_IS_FINALIZABLE,     art::kAccClassIsFinalizable)
+DEFINE_FLAG_OFFSET(ACCESS_FLAGS, CLASS_IS_INTERFACE,       art::kAccInterface)
 // TODO: We should really have a BitPosition which also checks it's a power of 2.
 DEFINE_FLAG_OFFSET(ACCESS_FLAGS, CLASS_IS_FINALIZABLE_BIT, art::MostSignificantBit(art::kAccClassIsFinalizable))