From 1aea3510b8dd0c512cec61c91c5ef1f1e5d53d64 Mon Sep 17 00:00:00 2001 From: Vladimir Marko Date: Thu, 8 Dec 2016 11:39:42 +0000 Subject: [PATCH] Hash-based DexCache field array. Test: m test-art-host, both AOT and interpreter Test: m test-art-target, both AOT and interpreter Test: m valgrind-test-art-host Bug: 30627598 Change-Id: If992f091aadd862d17b09928d21659573dd285a0 --- compiler/driver/compiler_driver_test.cc | 5 +- compiler/image_writer.cc | 20 +++-- compiler/optimizing/inliner.cc | 50 +++++------ compiler/optimizing/inliner.h | 11 +-- oatdump/oatdump.cc | 10 +-- patchoat/patchoat.cc | 13 +-- runtime/base/arena_allocator.cc | 124 +++++++++++++++++++++++----- runtime/base/arena_allocator.h | 55 +++++++----- runtime/base/scoped_arena_allocator.h | 11 +-- runtime/class_linker-inl.h | 15 +++- runtime/class_linker.cc | 72 ++++++++++++---- runtime/class_linker.h | 9 +- runtime/gc/space/image_space.cc | 13 +-- runtime/image.cc | 2 +- runtime/linear_alloc.cc | 5 ++ runtime/linear_alloc.h | 1 + runtime/mirror/dex_cache-inl.h | 76 +++++++++++++++-- runtime/mirror/dex_cache.cc | 66 +++++++++++---- runtime/mirror/dex_cache.h | 121 +++++++++++++++++++++++++-- runtime/mirror/dex_cache_test.cc | 3 +- runtime/mirror/field.cc | 12 ++- runtime/native/dalvik_system_VMRuntime.cc | 3 +- runtime/oat.h | 2 +- runtime/quick/inline_method_analyser.cc | 15 ++-- runtime/utils/dex_cache_arrays_layout-inl.h | 25 ++++-- runtime/utils/dex_cache_arrays_layout.h | 6 +- 26 files changed, 568 insertions(+), 177 deletions(-) diff --git a/compiler/driver/compiler_driver_test.cc b/compiler/driver/compiler_driver_test.cc index 562f97b3a..35aa1eef2 100644 --- a/compiler/driver/compiler_driver_test.cc +++ b/compiler/driver/compiler_driver_test.cc @@ -133,9 +133,10 @@ TEST_F(CompilerDriverTest, DISABLED_LARGE_CompileDexLibCore) { << " " << dex.GetMethodDeclaringClassDescriptor(dex.GetMethodId(i)) << " " << dex.GetMethodName(dex.GetMethodId(i)); } - EXPECT_EQ(dex.NumFieldIds(), dex_cache->NumResolvedFields()); + EXPECT_TRUE(dex_cache->StaticArtFieldSize() == dex_cache->NumResolvedFields() + || dex.NumFieldIds() == dex_cache->NumResolvedFields()); for (size_t i = 0; i < dex_cache->NumResolvedFields(); i++) { - ArtField* field = cl->GetResolvedField(i, dex_cache); + ArtField* field = dex_cache->GetResolvedField(i, cl->GetImagePointerSize()); EXPECT_TRUE(field != nullptr) << "field_idx=" << i << " " << dex.GetFieldDeclaringClassDescriptor(dex.GetFieldId(i)) << " " << dex.GetFieldName(dex.GetFieldId(i)); diff --git a/compiler/image_writer.cc b/compiler/image_writer.cc index 65d82ed98..a4a1fd36c 100644 --- a/compiler/image_writer.cc +++ b/compiler/image_writer.cc @@ -968,11 +968,12 @@ void ImageWriter::PruneNonImageClasses() { << Class::PrettyClass(declaring_class) << " not in class linker table"; } } - ArtField** resolved_fields = dex_cache->GetResolvedFields(); + mirror::FieldDexCacheType* resolved_fields = dex_cache->GetResolvedFields(); for (size_t i = 0; i < dex_cache->NumResolvedFields(); i++) { - ArtField* field = mirror::DexCache::GetElementPtrSize(resolved_fields, i, target_ptr_size_); + auto pair = mirror::DexCache::GetNativePairPtrSize(resolved_fields, i, target_ptr_size_); + ArtField* field = pair.object; if (field != nullptr && !KeepClass(field->GetDeclaringClass().Ptr())) { - dex_cache->SetResolvedField(i, nullptr, target_ptr_size_); + dex_cache->ClearResolvedField(pair.index, target_ptr_size_); } } // Clean the dex field. It might have been populated during the initialization phase, but @@ -1596,7 +1597,7 @@ void ImageWriter::CalculateNewObjectOffsets() { break; } case kBinDexCacheArray: - bin_offset = RoundUp(bin_offset, DexCacheArraysLayout::Alignment()); + bin_offset = RoundUp(bin_offset, DexCacheArraysLayout::Alignment(target_ptr_size_)); break; case kBinImTable: case kBinIMTConflictTable: { @@ -2236,16 +2237,17 @@ void ImageWriter::FixupDexCache(mirror::DexCache* orig_dex_cache, mirror::DexCache::SetElementPtrSize(copy_methods, i, copy, target_ptr_size_); } } - ArtField** orig_fields = orig_dex_cache->GetResolvedFields(); + mirror::FieldDexCacheType* orig_fields = orig_dex_cache->GetResolvedFields(); if (orig_fields != nullptr) { copy_dex_cache->SetFieldPtrWithSize(mirror::DexCache::ResolvedFieldsOffset(), NativeLocationInImage(orig_fields), PointerSize::k64); - ArtField** copy_fields = NativeCopyLocation(orig_fields, orig_dex_cache); + mirror::FieldDexCacheType* copy_fields = NativeCopyLocation(orig_fields, orig_dex_cache); for (size_t i = 0, num = orig_dex_cache->NumResolvedFields(); i != num; ++i) { - ArtField* orig = mirror::DexCache::GetElementPtrSize(orig_fields, i, target_ptr_size_); - ArtField* copy = NativeLocationInImage(orig); - mirror::DexCache::SetElementPtrSize(copy_fields, i, copy, target_ptr_size_); + mirror::FieldDexCachePair orig = + mirror::DexCache::GetNativePairPtrSize(orig_fields, i, target_ptr_size_); + mirror::FieldDexCachePair copy(NativeLocationInImage(orig.object), orig.index); + mirror::DexCache::SetNativePairPtrSize(copy_fields, i, copy, target_ptr_size_); } } mirror::MethodTypeDexCacheType* orig_method_types = orig_dex_cache->GetResolvedMethodTypes(); diff --git a/compiler/optimizing/inliner.cc b/compiler/optimizing/inliner.cc index 0b96005a1..ff7dc895d 100644 --- a/compiler/optimizing/inliner.cc +++ b/compiler/optimizing/inliner.cc @@ -1064,9 +1064,8 @@ bool HInliner::TryPatternSubstitution(HInvoke* invoke_instruction, // TODO: Needs null check. return false; } - Handle dex_cache(handles_->NewHandle(resolved_method->GetDexCache())); HInstruction* obj = GetInvokeInputForArgVRegIndex(invoke_instruction, data.object_arg); - HInstanceFieldGet* iget = CreateInstanceFieldGet(dex_cache, data.field_idx, obj); + HInstanceFieldGet* iget = CreateInstanceFieldGet(data.field_idx, resolved_method, obj); DCHECK_EQ(iget->GetFieldOffset().Uint32Value(), data.field_offset); DCHECK_EQ(iget->IsVolatile() ? 1u : 0u, data.is_volatile); invoke_instruction->GetBlock()->InsertInstructionBefore(iget, invoke_instruction); @@ -1079,10 +1078,9 @@ bool HInliner::TryPatternSubstitution(HInvoke* invoke_instruction, // TODO: Needs null check. return false; } - Handle dex_cache(handles_->NewHandle(resolved_method->GetDexCache())); HInstruction* obj = GetInvokeInputForArgVRegIndex(invoke_instruction, data.object_arg); HInstruction* value = GetInvokeInputForArgVRegIndex(invoke_instruction, data.src_arg); - HInstanceFieldSet* iput = CreateInstanceFieldSet(dex_cache, data.field_idx, obj, value); + HInstanceFieldSet* iput = CreateInstanceFieldSet(data.field_idx, resolved_method, obj, value); DCHECK_EQ(iput->GetFieldOffset().Uint32Value(), data.field_offset); DCHECK_EQ(iput->IsVolatile() ? 1u : 0u, data.is_volatile); invoke_instruction->GetBlock()->InsertInstructionBefore(iput, invoke_instruction); @@ -1116,24 +1114,19 @@ bool HInliner::TryPatternSubstitution(HInvoke* invoke_instruction, [](uint16_t index) { return index != DexFile::kDexNoIndex16; })); // Create HInstanceFieldSet for each IPUT that stores non-zero data. - Handle dex_cache; HInstruction* obj = GetInvokeInputForArgVRegIndex(invoke_instruction, /* this */ 0u); bool needs_constructor_barrier = false; for (size_t i = 0; i != number_of_iputs; ++i) { HInstruction* value = GetInvokeInputForArgVRegIndex(invoke_instruction, iput_args[i]); if (!value->IsConstant() || !value->AsConstant()->IsZeroBitPattern()) { - if (dex_cache.GetReference() == nullptr) { - dex_cache = handles_->NewHandle(resolved_method->GetDexCache()); - } uint16_t field_index = iput_field_indexes[i]; - HInstanceFieldSet* iput = CreateInstanceFieldSet(dex_cache, field_index, obj, value); + bool is_final; + HInstanceFieldSet* iput = + CreateInstanceFieldSet(field_index, resolved_method, obj, value, &is_final); invoke_instruction->GetBlock()->InsertInstructionBefore(iput, invoke_instruction); // Check whether the field is final. If it is, we need to add a barrier. - PointerSize pointer_size = InstructionSetPointerSize(codegen_->GetInstructionSet()); - ArtField* resolved_field = dex_cache->GetResolvedField(field_index, pointer_size); - DCHECK(resolved_field != nullptr); - if (resolved_field->IsFinal()) { + if (is_final) { needs_constructor_barrier = true; } } @@ -1152,12 +1145,13 @@ bool HInliner::TryPatternSubstitution(HInvoke* invoke_instruction, return true; } -HInstanceFieldGet* HInliner::CreateInstanceFieldGet(Handle dex_cache, - uint32_t field_index, +HInstanceFieldGet* HInliner::CreateInstanceFieldGet(uint32_t field_index, + ArtMethod* referrer, HInstruction* obj) REQUIRES_SHARED(Locks::mutator_lock_) { - PointerSize pointer_size = InstructionSetPointerSize(codegen_->GetInstructionSet()); - ArtField* resolved_field = dex_cache->GetResolvedField(field_index, pointer_size); + ClassLinker* class_linker = Runtime::Current()->GetClassLinker(); + ArtField* resolved_field = + class_linker->LookupResolvedField(field_index, referrer, /* is_static */ false); DCHECK(resolved_field != nullptr); HInstanceFieldGet* iget = new (graph_->GetArena()) HInstanceFieldGet( obj, @@ -1167,12 +1161,13 @@ HInstanceFieldGet* HInliner::CreateInstanceFieldGet(Handle dex resolved_field->IsVolatile(), field_index, resolved_field->GetDeclaringClass()->GetDexClassDefIndex(), - *dex_cache->GetDexFile(), + *referrer->GetDexFile(), // Read barrier generates a runtime call in slow path and we need a valid // dex pc for the associated stack map. 0 is bogus but valid. Bug: 26854537. /* dex_pc */ 0); if (iget->GetType() == Primitive::kPrimNot) { // Use the same dex_cache that we used for field lookup as the hint_dex_cache. + Handle dex_cache = handles_->NewHandle(referrer->GetDexCache()); ReferenceTypePropagation rtp(graph_, outer_compilation_unit_.GetClassLoader(), dex_cache, @@ -1183,14 +1178,21 @@ HInstanceFieldGet* HInliner::CreateInstanceFieldGet(Handle dex return iget; } -HInstanceFieldSet* HInliner::CreateInstanceFieldSet(Handle dex_cache, - uint32_t field_index, +HInstanceFieldSet* HInliner::CreateInstanceFieldSet(uint32_t field_index, + ArtMethod* referrer, HInstruction* obj, - HInstruction* value) + HInstruction* value, + bool* is_final) REQUIRES_SHARED(Locks::mutator_lock_) { - PointerSize pointer_size = InstructionSetPointerSize(codegen_->GetInstructionSet()); - ArtField* resolved_field = dex_cache->GetResolvedField(field_index, pointer_size); + ClassLinker* class_linker = Runtime::Current()->GetClassLinker(); + ArtField* resolved_field = + class_linker->LookupResolvedField(field_index, referrer, /* is_static */ false); DCHECK(resolved_field != nullptr); + if (is_final != nullptr) { + // This information is needed only for constructors. + DCHECK(referrer->IsConstructor()); + *is_final = resolved_field->IsFinal(); + } HInstanceFieldSet* iput = new (graph_->GetArena()) HInstanceFieldSet( obj, value, @@ -1200,7 +1202,7 @@ HInstanceFieldSet* HInliner::CreateInstanceFieldSet(Handle dex resolved_field->IsVolatile(), field_index, resolved_field->GetDeclaringClass()->GetDexClassDefIndex(), - *dex_cache->GetDexFile(), + *referrer->GetDexFile(), // Read barrier generates a runtime call in slow path and we need a valid // dex pc for the associated stack map. 0 is bogus but valid. Bug: 26854537. /* dex_pc */ 0); diff --git a/compiler/optimizing/inliner.h b/compiler/optimizing/inliner.h index 75d025ae4..df98a1cf2 100644 --- a/compiler/optimizing/inliner.h +++ b/compiler/optimizing/inliner.h @@ -97,14 +97,15 @@ class HInliner : public HOptimization { REQUIRES_SHARED(Locks::mutator_lock_); // Create a new HInstanceFieldGet. - HInstanceFieldGet* CreateInstanceFieldGet(Handle dex_cache, - uint32_t field_index, + HInstanceFieldGet* CreateInstanceFieldGet(uint32_t field_index, + ArtMethod* referrer, HInstruction* obj); // Create a new HInstanceFieldSet. - HInstanceFieldSet* CreateInstanceFieldSet(Handle dex_cache, - uint32_t field_index, + HInstanceFieldSet* CreateInstanceFieldSet(uint32_t field_index, + ArtMethod* referrer, HInstruction* obj, - HInstruction* value); + HInstruction* value, + bool* is_final = nullptr); // Try to inline the target of a monomorphic call. If successful, the code // in the graph will look like: diff --git a/oatdump/oatdump.cc b/oatdump/oatdump.cc index becb827d2..e7670230e 100644 --- a/oatdump/oatdump.cc +++ b/oatdump/oatdump.cc @@ -2210,13 +2210,13 @@ class ImageDumper { ScopedIndentation indent2(&state->vios_); auto* resolved_fields = dex_cache->GetResolvedFields(); for (size_t i = 0, length = dex_cache->NumResolvedFields(); i < length; ++i) { - auto* elem = mirror::DexCache::GetElementPtrSize( - resolved_fields, i, image_pointer_size); + auto* elem = mirror::DexCache::GetNativePairPtrSize( + resolved_fields, i, image_pointer_size).object; size_t run = 0; for (size_t j = i + 1; - j != length && elem == mirror::DexCache::GetElementPtrSize(resolved_fields, - j, - image_pointer_size); + j != length && + elem == mirror::DexCache::GetNativePairPtrSize( + resolved_fields, j, image_pointer_size).object; ++j) { ++run; } diff --git a/patchoat/patchoat.cc b/patchoat/patchoat.cc index 18a667015..dfaae7d86 100644 --- a/patchoat/patchoat.cc +++ b/patchoat/patchoat.cc @@ -534,17 +534,18 @@ void PatchOat::PatchDexFileArrays(mirror::ObjectArray* img_roots mirror::DexCache::SetElementPtrSize(copy_methods, j, copy, pointer_size); } } - ArtField** orig_fields = orig_dex_cache->GetResolvedFields(); - ArtField** relocated_fields = RelocatedAddressOfPointer(orig_fields); + mirror::FieldDexCacheType* orig_fields = orig_dex_cache->GetResolvedFields(); + mirror::FieldDexCacheType* relocated_fields = RelocatedAddressOfPointer(orig_fields); copy_dex_cache->SetField64( mirror::DexCache::ResolvedFieldsOffset(), static_cast(reinterpret_cast(relocated_fields))); if (orig_fields != nullptr) { - ArtField** copy_fields = RelocatedCopyOf(orig_fields); + mirror::FieldDexCacheType* copy_fields = RelocatedCopyOf(orig_fields); for (size_t j = 0, num = orig_dex_cache->NumResolvedFields(); j != num; ++j) { - ArtField* orig = mirror::DexCache::GetElementPtrSize(orig_fields, j, pointer_size); - ArtField* copy = RelocatedAddressOfPointer(orig); - mirror::DexCache::SetElementPtrSize(copy_fields, j, copy, pointer_size); + mirror::FieldDexCachePair orig = + mirror::DexCache::GetNativePairPtrSize(orig_fields, j, pointer_size); + mirror::FieldDexCachePair copy(RelocatedAddressOfPointer(orig.object), orig.index); + mirror::DexCache::SetNativePairPtrSize(copy_fields, j, copy, pointer_size); } } mirror::MethodTypeDexCacheType* orig_method_types = orig_dex_cache->GetResolvedMethodTypes(); diff --git a/runtime/base/arena_allocator.cc b/runtime/base/arena_allocator.cc index db433194d..5aede3833 100644 --- a/runtime/base/arena_allocator.cc +++ b/runtime/base/arena_allocator.cc @@ -15,6 +15,7 @@ */ #include +#include #include #include @@ -27,7 +28,7 @@ namespace art { -static constexpr size_t kMemoryToolRedZoneBytes = 8; +constexpr size_t kMemoryToolRedZoneBytes = 8; constexpr size_t Arena::kDefaultSize; template @@ -168,23 +169,75 @@ void ArenaAllocatorMemoryTool::DoMakeInaccessible(void* ptr, size_t size) { Arena::Arena() : bytes_allocated_(0), next_(nullptr) { } +class MallocArena FINAL : public Arena { + public: + explicit MallocArena(size_t size = Arena::kDefaultSize); + virtual ~MallocArena(); + private: + static constexpr size_t RequiredOverallocation() { + return (alignof(std::max_align_t) < ArenaAllocator::kArenaAlignment) + ? ArenaAllocator::kArenaAlignment - alignof(std::max_align_t) + : 0u; + } + + uint8_t* unaligned_memory_; +}; + MallocArena::MallocArena(size_t size) { - memory_ = reinterpret_cast(calloc(1, size)); - CHECK(memory_ != nullptr); // Abort on OOM. - DCHECK_ALIGNED(memory_, ArenaAllocator::kAlignment); + // We need to guarantee kArenaAlignment aligned allocation for the new arena. + // TODO: Use std::aligned_alloc() when it becomes available with C++17. + constexpr size_t overallocation = RequiredOverallocation(); + unaligned_memory_ = reinterpret_cast(calloc(1, size + overallocation)); + CHECK(unaligned_memory_ != nullptr); // Abort on OOM. + DCHECK_ALIGNED(unaligned_memory_, alignof(std::max_align_t)); + if (overallocation == 0u) { + memory_ = unaligned_memory_; + } else { + memory_ = AlignUp(unaligned_memory_, ArenaAllocator::kArenaAlignment); + if (UNLIKELY(RUNNING_ON_MEMORY_TOOL > 0)) { + size_t head = memory_ - unaligned_memory_; + size_t tail = overallocation - head; + MEMORY_TOOL_MAKE_NOACCESS(unaligned_memory_, head); + MEMORY_TOOL_MAKE_NOACCESS(memory_ + size, tail); + } + } + DCHECK_ALIGNED(memory_, ArenaAllocator::kArenaAlignment); size_ = size; } MallocArena::~MallocArena() { - free(reinterpret_cast(memory_)); + constexpr size_t overallocation = RequiredOverallocation(); + if (overallocation != 0u && UNLIKELY(RUNNING_ON_MEMORY_TOOL > 0)) { + size_t head = memory_ - unaligned_memory_; + size_t tail = overallocation - head; + MEMORY_TOOL_MAKE_UNDEFINED(unaligned_memory_, head); + MEMORY_TOOL_MAKE_UNDEFINED(memory_ + size_, tail); + } + free(reinterpret_cast(unaligned_memory_)); } +class MemMapArena FINAL : public Arena { + public: + MemMapArena(size_t size, bool low_4gb, const char* name); + virtual ~MemMapArena(); + void Release() OVERRIDE; + + private: + std::unique_ptr map_; +}; + MemMapArena::MemMapArena(size_t size, bool low_4gb, const char* name) { + // Round up to a full page as that's the smallest unit of allocation for mmap() + // and we want to be able to use all memory that we actually allocate. + size = RoundUp(size, kPageSize); std::string error_msg; map_.reset(MemMap::MapAnonymous( name, nullptr, size, PROT_READ | PROT_WRITE, low_4gb, false, &error_msg)); CHECK(map_.get() != nullptr) << error_msg; memory_ = map_->Begin(); + static_assert(ArenaAllocator::kArenaAlignment <= kPageSize, + "Arena should not need stronger alignment than kPageSize."); + DCHECK_ALIGNED(memory_, ArenaAllocator::kArenaAlignment); size_ = map_->Size(); } @@ -332,21 +385,32 @@ void* ArenaAllocator::AllocWithMemoryTool(size_t bytes, ArenaAllocKind kind) { ArenaAllocatorStats::RecordAlloc(rounded_bytes, kind); uint8_t* ret; if (UNLIKELY(rounded_bytes > static_cast(end_ - ptr_))) { - ret = AllocFromNewArena(rounded_bytes); - uint8_t* noaccess_begin = ret + bytes; - uint8_t* noaccess_end; - if (ret == arena_head_->Begin()) { - DCHECK(ptr_ - rounded_bytes == ret); - noaccess_end = end_; - } else { - // We're still using the old arena but `ret` comes from a new one just after it. - DCHECK(arena_head_->next_ != nullptr); - DCHECK(ret == arena_head_->next_->Begin()); - DCHECK_EQ(rounded_bytes, arena_head_->next_->GetBytesAllocated()); - noaccess_end = arena_head_->next_->End(); - } - MEMORY_TOOL_MAKE_NOACCESS(noaccess_begin, noaccess_end - noaccess_begin); + ret = AllocFromNewArenaWithMemoryTool(rounded_bytes); + } else { + ret = ptr_; + ptr_ += rounded_bytes; + } + MEMORY_TOOL_MAKE_DEFINED(ret, bytes); + // Check that the memory is already zeroed out. + DCHECK(std::all_of(ret, ret + bytes, [](uint8_t val) { return val == 0u; })); + return ret; +} + +void* ArenaAllocator::AllocWithMemoryToolAlign16(size_t bytes, ArenaAllocKind kind) { + // We mark all memory for a newly retrieved arena as inaccessible and then + // mark only the actually allocated memory as defined. That leaves red zones + // and padding between allocations marked as inaccessible. + size_t rounded_bytes = bytes + kMemoryToolRedZoneBytes; + DCHECK_ALIGNED(rounded_bytes, 8); // `bytes` is 16-byte aligned, red zone is 8-byte aligned. + uintptr_t padding = + ((reinterpret_cast(ptr_) + 15u) & 15u) - reinterpret_cast(ptr_); + ArenaAllocatorStats::RecordAlloc(rounded_bytes, kind); + uint8_t* ret; + if (UNLIKELY(padding + rounded_bytes > static_cast(end_ - ptr_))) { + static_assert(kArenaAlignment >= 16, "Expecting sufficient alignment for new Arena."); + ret = AllocFromNewArenaWithMemoryTool(rounded_bytes); } else { + ptr_ += padding; // Leave padding inaccessible. ret = ptr_; ptr_ += rounded_bytes; } @@ -386,6 +450,24 @@ uint8_t* ArenaAllocator::AllocFromNewArena(size_t bytes) { return new_arena->Begin(); } +uint8_t* ArenaAllocator::AllocFromNewArenaWithMemoryTool(size_t bytes) { + uint8_t* ret = AllocFromNewArena(bytes); + uint8_t* noaccess_begin = ret + bytes; + uint8_t* noaccess_end; + if (ret == arena_head_->Begin()) { + DCHECK(ptr_ - bytes == ret); + noaccess_end = end_; + } else { + // We're still using the old arena but `ret` comes from a new one just after it. + DCHECK(arena_head_->next_ != nullptr); + DCHECK(ret == arena_head_->next_->Begin()); + DCHECK_EQ(bytes, arena_head_->next_->GetBytesAllocated()); + noaccess_end = arena_head_->next_->End(); + } + MEMORY_TOOL_MAKE_NOACCESS(noaccess_begin, noaccess_end - noaccess_begin); + return ret; +} + bool ArenaAllocator::Contains(const void* ptr) const { if (ptr >= begin_ && ptr < end_) { return true; @@ -398,7 +480,9 @@ bool ArenaAllocator::Contains(const void* ptr) const { return false; } -MemStats::MemStats(const char* name, const ArenaAllocatorStats* stats, const Arena* first_arena, +MemStats::MemStats(const char* name, + const ArenaAllocatorStats* stats, + const Arena* first_arena, ssize_t lost_bytes_adjustment) : name_(name), stats_(stats), diff --git a/runtime/base/arena_allocator.h b/runtime/base/arena_allocator.h index f92fbea15..c39429ce0 100644 --- a/runtime/base/arena_allocator.h +++ b/runtime/base/arena_allocator.h @@ -34,7 +34,6 @@ class ArenaPool; class ArenaAllocator; class ArenaStack; class ScopedArenaAllocator; -class MemMap; class MemStats; template @@ -244,22 +243,6 @@ class Arena { DISALLOW_COPY_AND_ASSIGN(Arena); }; -class MallocArena FINAL : public Arena { - public: - explicit MallocArena(size_t size = Arena::kDefaultSize); - virtual ~MallocArena(); -}; - -class MemMapArena FINAL : public Arena { - public: - MemMapArena(size_t size, bool low_4gb, const char* name); - virtual ~MemMapArena(); - void Release() OVERRIDE; - - private: - std::unique_ptr map_; -}; - class ArenaPool { public: explicit ArenaPool(bool use_malloc = true, @@ -319,8 +302,31 @@ class ArenaAllocator return ret; } + // Returns zeroed memory. + void* AllocAlign16(size_t bytes, ArenaAllocKind kind = kArenaAllocMisc) ALWAYS_INLINE { + // It is an error to request 16-byte aligned allocation of unaligned size. + DCHECK_ALIGNED(bytes, 16); + if (UNLIKELY(IsRunningOnMemoryTool())) { + return AllocWithMemoryToolAlign16(bytes, kind); + } + uintptr_t padding = + ((reinterpret_cast(ptr_) + 15u) & 15u) - reinterpret_cast(ptr_); + ArenaAllocatorStats::RecordAlloc(bytes, kind); + if (UNLIKELY(padding + bytes > static_cast(end_ - ptr_))) { + static_assert(kArenaAlignment >= 16, "Expecting sufficient alignment for new Arena."); + return AllocFromNewArena(bytes); + } + ptr_ += padding; + uint8_t* ret = ptr_; + DCHECK_ALIGNED(ret, 16); + ptr_ += bytes; + return ret; + } + // Realloc never frees the input pointer, it is the caller's job to do this if necessary. - void* Realloc(void* ptr, size_t ptr_size, size_t new_size, + void* Realloc(void* ptr, + size_t ptr_size, + size_t new_size, ArenaAllocKind kind = kArenaAllocMisc) ALWAYS_INLINE { DCHECK_GE(new_size, ptr_size); DCHECK_EQ(ptr == nullptr, ptr_size == 0u); @@ -371,12 +377,17 @@ class ArenaAllocator bool Contains(const void* ptr) const; - static constexpr size_t kAlignment = 8; + // The alignment guaranteed for individual allocations. + static constexpr size_t kAlignment = 8u; + + // The alignment required for the whole Arena rather than individual allocations. + static constexpr size_t kArenaAlignment = 16u; private: void* AllocWithMemoryTool(size_t bytes, ArenaAllocKind kind); + void* AllocWithMemoryToolAlign16(size_t bytes, ArenaAllocKind kind); uint8_t* AllocFromNewArena(size_t bytes); - + uint8_t* AllocFromNewArenaWithMemoryTool(size_t bytes); void UpdateBytesAllocated(); @@ -396,7 +407,9 @@ class ArenaAllocator class MemStats { public: - MemStats(const char* name, const ArenaAllocatorStats* stats, const Arena* first_arena, + MemStats(const char* name, + const ArenaAllocatorStats* stats, + const Arena* first_arena, ssize_t lost_bytes_adjustment = 0); void Dump(std::ostream& os) const; diff --git a/runtime/base/scoped_arena_allocator.h b/runtime/base/scoped_arena_allocator.h index 55044b34e..1a0eb5ea0 100644 --- a/runtime/base/scoped_arena_allocator.h +++ b/runtime/base/scoped_arena_allocator.h @@ -39,8 +39,6 @@ enum class ArenaFreeTag : uint8_t { kFree, }; -static constexpr size_t kArenaAlignment = 8; - // Holds a list of Arenas for use by ScopedArenaAllocator stack. // The memory is returned to the ArenaPool when the ArenaStack is destroyed. class ArenaStack : private DebugStackRefCounter, private ArenaAllocatorMemoryTool { @@ -67,6 +65,9 @@ class ArenaStack : private DebugStackRefCounter, private ArenaAllocatorMemoryToo return *(reinterpret_cast(ptr) - 1); } + // The alignment guaranteed for individual allocations. + static constexpr size_t kAlignment = 8u; + private: struct Peak; struct Current; @@ -89,8 +90,8 @@ class ArenaStack : private DebugStackRefCounter, private ArenaAllocatorMemoryToo if (UNLIKELY(IsRunningOnMemoryTool())) { return AllocWithMemoryTool(bytes, kind); } - // Add kArenaAlignment for the free or used tag. Required to preserve alignment. - size_t rounded_bytes = RoundUp(bytes + (kIsDebugBuild ? kArenaAlignment : 0u), kArenaAlignment); + // Add kAlignment for the free or used tag. Required to preserve alignment. + size_t rounded_bytes = RoundUp(bytes + (kIsDebugBuild ? kAlignment : 0u), kAlignment); uint8_t* ptr = top_ptr_; if (UNLIKELY(static_cast(top_end_ - ptr) < rounded_bytes)) { ptr = AllocateFromNextArena(rounded_bytes); @@ -98,7 +99,7 @@ class ArenaStack : private DebugStackRefCounter, private ArenaAllocatorMemoryToo CurrentStats()->RecordAlloc(bytes, kind); top_ptr_ = ptr + rounded_bytes; if (kIsDebugBuild) { - ptr += kArenaAlignment; + ptr += kAlignment; ArenaTagForAllocation(ptr) = ArenaFreeTag::kUsed; } return ptr; diff --git a/runtime/class_linker-inl.h b/runtime/class_linker-inl.h index bd510ca0e..9ddc6cf0a 100644 --- a/runtime/class_linker-inl.h +++ b/runtime/class_linker-inl.h @@ -161,9 +161,15 @@ inline ArtMethod* ClassLinker::ResolveMethod(Thread* self, return resolved_method; } -inline ArtField* ClassLinker::GetResolvedField(uint32_t field_idx, - ObjPtr dex_cache) { - return dex_cache->GetResolvedField(field_idx, image_pointer_size_); +inline ArtField* ClassLinker::LookupResolvedField(uint32_t field_idx, + ArtMethod* referrer, + bool is_static) { + ObjPtr dex_cache = referrer->GetDexCache(); + ArtField* field = dex_cache->GetResolvedField(field_idx, image_pointer_size_); + if (field == nullptr) { + field = LookupResolvedField(field_idx, dex_cache, referrer->GetClassLoader(), is_static); + } + return field; } inline ArtField* ClassLinker::ResolveField(uint32_t field_idx, @@ -171,7 +177,8 @@ inline ArtField* ClassLinker::ResolveField(uint32_t field_idx, bool is_static) { Thread::PoisonObjectPointersIfDebug(); ObjPtr declaring_class = referrer->GetDeclaringClass(); - ArtField* resolved_field = GetResolvedField(field_idx, referrer->GetDexCache()); + ArtField* resolved_field = + referrer->GetDexCache()->GetResolvedField(field_idx, image_pointer_size_); if (UNLIKELY(resolved_field == nullptr)) { StackHandleScope<2> hs(Thread::Current()); Handle dex_cache(hs.NewHandle(referrer->GetDexCache())); diff --git a/runtime/class_linker.cc b/runtime/class_linker.cc index b611aa213..ab2b39588 100644 --- a/runtime/class_linker.cc +++ b/runtime/class_linker.cc @@ -1282,7 +1282,10 @@ bool ClassLinker::UpdateAppImageClassLoadersAndDexCaches( num_types = dex_file->NumTypeIds(); } const size_t num_methods = dex_file->NumMethodIds(); - const size_t num_fields = dex_file->NumFieldIds(); + size_t num_fields = mirror::DexCache::kDexCacheFieldCacheSize; + if (dex_file->NumFieldIds() < num_fields) { + num_fields = dex_file->NumFieldIds(); + } size_t num_method_types = mirror::DexCache::kDexCacheMethodTypeCacheSize; if (dex_file->NumProtoIds() < num_method_types) { num_method_types = dex_file->NumProtoIds(); @@ -1326,17 +1329,22 @@ bool ClassLinker::UpdateAppImageClassLoadersAndDexCaches( dex_cache->SetResolvedMethods(methods); } if (num_fields != 0u) { - ArtField** const fields = - reinterpret_cast(raw_arrays + layout.FieldsOffset()); - for (size_t j = 0; kIsDebugBuild && j < num_fields; ++j) { - DCHECK(fields[j] == nullptr); + mirror::FieldDexCacheType* const image_resolved_fields = dex_cache->GetResolvedFields(); + mirror::FieldDexCacheType* const fields = + reinterpret_cast(raw_arrays + layout.FieldsOffset()); + for (size_t j = 0; j < num_fields; ++j) { + DCHECK_EQ(mirror::DexCache::GetNativePairPtrSize(fields, j, image_pointer_size_).index, + 0u); + DCHECK(mirror::DexCache::GetNativePairPtrSize(fields, j, image_pointer_size_).object == + nullptr); + mirror::DexCache::SetNativePairPtrSize( + fields, + j, + mirror::DexCache::GetNativePairPtrSize(image_resolved_fields, + j, + image_pointer_size_), + image_pointer_size_); } - CopyNonNull(dex_cache->GetResolvedFields(), - num_fields, - fields, - [] (const ArtField* field) { - return field == nullptr; - }); dex_cache->SetResolvedFields(fields); } if (num_method_types != 0u) { @@ -8260,6 +8268,43 @@ ArtMethod* ClassLinker::ResolveMethodWithoutInvokeType(const DexFile& dex_file, return resolved; } +ArtField* ClassLinker::LookupResolvedField(uint32_t field_idx, + ObjPtr dex_cache, + ObjPtr class_loader, + bool is_static) { + const DexFile& dex_file = *dex_cache->GetDexFile(); + const DexFile::FieldId& field_id = dex_file.GetFieldId(field_idx); + ObjPtr klass = dex_cache->GetResolvedType(field_id.class_idx_); + if (klass == nullptr) { + klass = LookupResolvedType(dex_file, field_id.class_idx_, dex_cache, class_loader); + } + if (klass == nullptr) { + // The class has not been resolved yet, so the field is also unresolved. + return nullptr; + } + DCHECK(klass->IsResolved()); + Thread* self = is_static ? Thread::Current() : nullptr; + + // First try to find a field declared directly by `klass` by the field index. + ArtField* resolved_field = is_static + ? mirror::Class::FindStaticField(self, klass, dex_cache, field_idx) + : klass->FindInstanceField(dex_cache, field_idx); + + if (resolved_field == nullptr) { + // If not found in `klass` by field index, search the class hierarchy using the name and type. + const char* name = dex_file.GetFieldName(field_id); + const char* type = dex_file.GetFieldTypeDescriptor(field_id); + resolved_field = is_static + ? mirror::Class::FindStaticField(self, klass, name, type) + : klass->FindInstanceField(name, type); + } + + if (resolved_field != nullptr) { + dex_cache->SetResolvedField(field_idx, resolved_field, image_pointer_size_); + } + return resolved_field; +} + ArtField* ClassLinker::ResolveField(const DexFile& dex_file, uint32_t field_idx, Handle dex_cache, @@ -8320,9 +8365,8 @@ ArtField* ClassLinker::ResolveFieldJLS(const DexFile& dex_file, return nullptr; } - StringPiece name(dex_file.StringDataByIdx(field_id.name_idx_)); - StringPiece type(dex_file.StringDataByIdx( - dex_file.GetTypeId(field_id.type_idx_).descriptor_idx_)); + StringPiece name(dex_file.GetFieldName(field_id)); + StringPiece type(dex_file.GetFieldTypeDescriptor(field_id)); resolved = mirror::Class::FindField(self, klass, name, type); if (resolved != nullptr) { dex_cache->SetResolvedField(field_idx, resolved, image_pointer_size_); diff --git a/runtime/class_linker.h b/runtime/class_linker.h index a5d26c7a8..6254acbb9 100644 --- a/runtime/class_linker.h +++ b/runtime/class_linker.h @@ -333,7 +333,7 @@ class ClassLinker { REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(!Locks::dex_lock_, !Roles::uninterruptible_); - ArtField* GetResolvedField(uint32_t field_idx, ObjPtr dex_cache) + ArtField* LookupResolvedField(uint32_t field_idx, ArtMethod* referrer, bool is_static) REQUIRES_SHARED(Locks::mutator_lock_); ArtField* ResolveField(uint32_t field_idx, ArtMethod* referrer, bool is_static) REQUIRES_SHARED(Locks::mutator_lock_) @@ -842,6 +842,13 @@ class ClassLinker { REQUIRES(!Locks::classlinker_classes_lock_) REQUIRES_SHARED(Locks::mutator_lock_); + // Find a field by its field index. + ArtField* LookupResolvedField(uint32_t field_idx, + ObjPtr dex_cache, + ObjPtr class_loader, + bool is_static) + REQUIRES_SHARED(Locks::mutator_lock_); + void RegisterDexFileLocked(const DexFile& dex_file, ObjPtr dex_cache, ObjPtr class_loader) diff --git a/runtime/gc/space/image_space.cc b/runtime/gc/space/image_space.cc index 010ef1156..568f8d62a 100644 --- a/runtime/gc/space/image_space.cc +++ b/runtime/gc/space/image_space.cc @@ -1259,17 +1259,18 @@ class ImageSpaceLoader { } } } - ArtField** fields = dex_cache->GetResolvedFields(); + mirror::FieldDexCacheType* fields = dex_cache->GetResolvedFields(); if (fields != nullptr) { - ArtField** new_fields = fixup_adapter.ForwardObject(fields); + mirror::FieldDexCacheType* new_fields = fixup_adapter.ForwardObject(fields); if (fields != new_fields) { dex_cache->SetResolvedFields(new_fields); } for (size_t j = 0, num = dex_cache->NumResolvedFields(); j != num; ++j) { - ArtField* orig = mirror::DexCache::GetElementPtrSize(new_fields, j, pointer_size); - ArtField* copy = fixup_adapter.ForwardObject(orig); - if (orig != copy) { - mirror::DexCache::SetElementPtrSize(new_fields, j, copy, pointer_size); + mirror::FieldDexCachePair orig = + mirror::DexCache::GetNativePairPtrSize(new_fields, j, pointer_size); + mirror::FieldDexCachePair copy(fixup_adapter.ForwardObject(orig.object), orig.index); + if (orig.object != copy.object) { + mirror::DexCache::SetNativePairPtrSize(new_fields, j, copy, pointer_size); } } } diff --git a/runtime/image.cc b/runtime/image.cc index 88f28f3ea..5fbb7a6f6 100644 --- a/runtime/image.cc +++ b/runtime/image.cc @@ -25,7 +25,7 @@ namespace art { const uint8_t ImageHeader::kImageMagic[] = { 'a', 'r', 't', '\n' }; -const uint8_t ImageHeader::kImageVersion[] = { '0', '4', '0', '\0' }; // Integer.valueOf intrinsic +const uint8_t ImageHeader::kImageVersion[] = { '0', '4', '1', '\0' }; // hash-based DexCache fields ImageHeader::ImageHeader(uint32_t image_begin, uint32_t image_size, diff --git a/runtime/linear_alloc.cc b/runtime/linear_alloc.cc index f91b0ed9e..e9db9b8b4 100644 --- a/runtime/linear_alloc.cc +++ b/runtime/linear_alloc.cc @@ -33,6 +33,11 @@ void* LinearAlloc::Alloc(Thread* self, size_t size) { return allocator_.Alloc(size); } +void* LinearAlloc::AllocAlign16(Thread* self, size_t size) { + MutexLock mu(self, lock_); + return allocator_.AllocAlign16(size); +} + size_t LinearAlloc::GetUsedMemory() const { MutexLock mu(Thread::Current(), lock_); return allocator_.BytesUsed(); diff --git a/runtime/linear_alloc.h b/runtime/linear_alloc.h index df7f17dd7..384b2e324 100644 --- a/runtime/linear_alloc.h +++ b/runtime/linear_alloc.h @@ -29,6 +29,7 @@ class LinearAlloc { explicit LinearAlloc(ArenaPool* pool); void* Alloc(Thread* self, size_t size) REQUIRES(!lock_); + void* AllocAlign16(Thread* self, size_t size) REQUIRES(!lock_); // Realloc never frees the input pointer, it is the caller's job to do this if necessary. void* Realloc(Thread* self, void* ptr, size_t old_size, size_t new_size) REQUIRES(!lock_); diff --git a/runtime/mirror/dex_cache-inl.h b/runtime/mirror/dex_cache-inl.h index 29bf6a024..582ecb23e 100644 --- a/runtime/mirror/dex_cache-inl.h +++ b/runtime/mirror/dex_cache-inl.h @@ -24,6 +24,7 @@ #include "base/casts.h" #include "base/enums.h" #include "base/logging.h" +#include "dex_file.h" #include "gc_root.h" #include "mirror/class.h" #include "mirror/call_site.h" @@ -36,6 +37,15 @@ namespace art { namespace mirror { +template +inline void NativeDexCachePair::Initialize(std::atomic>* dex_cache, + PointerSize pointer_size) { + NativeDexCachePair first_elem; + first_elem.object = nullptr; + first_elem.index = InvalidIndexForSlot(0); + DexCache::SetNativePairPtrSize(dex_cache, 0, first_elem, pointer_size); +} + inline uint32_t DexCache::ClassSize(PointerSize pointer_size) { uint32_t vtable_entries = Object::kVTableLength + 5; return Class::ComputeClassSize(true, vtable_entries, 0, 0, 0, 0, 0, pointer_size); @@ -164,20 +174,36 @@ inline CallSite* DexCache::SetResolvedCallSite(uint32_t call_site_idx, CallSite* } } +inline uint32_t DexCache::FieldSlotIndex(uint32_t field_idx) { + DCHECK_LT(field_idx, GetDexFile()->NumFieldIds()); + const uint32_t slot_idx = field_idx % kDexCacheFieldCacheSize; + DCHECK_LT(slot_idx, NumResolvedFields()); + return slot_idx; +} + inline ArtField* DexCache::GetResolvedField(uint32_t field_idx, PointerSize ptr_size) { DCHECK_EQ(Runtime::Current()->GetClassLinker()->GetImagePointerSize(), ptr_size); - DCHECK_LT(field_idx, NumResolvedFields()); // NOTE: Unchecked, i.e. not throwing AIOOB. - ArtField* field = GetElementPtrSize(GetResolvedFields(), field_idx, ptr_size); - if (field == nullptr || field->GetDeclaringClass()->IsErroneous()) { - return nullptr; - } - return field; + auto pair = GetNativePairPtrSize(GetResolvedFields(), FieldSlotIndex(field_idx), ptr_size); + return pair.GetObjectForIndex(field_idx); } inline void DexCache::SetResolvedField(uint32_t field_idx, ArtField* field, PointerSize ptr_size) { DCHECK_EQ(Runtime::Current()->GetClassLinker()->GetImagePointerSize(), ptr_size); - DCHECK_LT(field_idx, NumResolvedFields()); // NOTE: Unchecked, i.e. not throwing AIOOB. - SetElementPtrSize(GetResolvedFields(), field_idx, field, ptr_size); + DCHECK(field != nullptr); + FieldDexCachePair pair(field, field_idx); + SetNativePairPtrSize(GetResolvedFields(), FieldSlotIndex(field_idx), pair, ptr_size); +} + +inline void DexCache::ClearResolvedField(uint32_t field_idx, PointerSize ptr_size) { + DCHECK_EQ(Runtime::Current()->GetClassLinker()->GetImagePointerSize(), ptr_size); + uint32_t slot_idx = FieldSlotIndex(field_idx); + auto* resolved_fields = GetResolvedFields(); + // This is racy but should only be called from the single-threaded ImageWriter. + DCHECK(Runtime::Current()->IsAotCompiler()); + if (GetNativePairPtrSize(resolved_fields, slot_idx, ptr_size).index == field_idx) { + FieldDexCachePair cleared(nullptr, FieldDexCachePair::InvalidIndexForSlot(slot_idx)); + SetNativePairPtrSize(resolved_fields, slot_idx, cleared, ptr_size); + } } inline ArtMethod* DexCache::GetResolvedMethod(uint32_t method_idx, PointerSize ptr_size) { @@ -225,6 +251,40 @@ inline void DexCache::SetElementPtrSize(PtrType* ptr_array, } } +template +NativeDexCachePair DexCache::GetNativePairPtrSize(std::atomic>* pair_array, + size_t idx, + PointerSize ptr_size) { + if (ptr_size == PointerSize::k64) { + auto* array = reinterpret_cast*>(pair_array); + ConversionPair64 value = AtomicLoadRelaxed16B(&array[idx]); + return NativeDexCachePair(reinterpret_cast64(value.first), + dchecked_integral_cast(value.second)); + } else { + auto* array = reinterpret_cast*>(pair_array); + ConversionPair32 value = array[idx].load(std::memory_order_relaxed); + return NativeDexCachePair(reinterpret_cast(value.first), value.second); + } +} + +template +void DexCache::SetNativePairPtrSize(std::atomic>* pair_array, + size_t idx, + NativeDexCachePair pair, + PointerSize ptr_size) { + if (ptr_size == PointerSize::k64) { + auto* array = reinterpret_cast*>(pair_array); + ConversionPair64 v(reinterpret_cast64(pair.object), pair.index); + AtomicStoreRelease16B(&array[idx], v); + } else { + auto* array = reinterpret_cast*>(pair_array); + ConversionPair32 v( + dchecked_integral_cast(reinterpret_cast(pair.object)), + dchecked_integral_cast(pair.index)); + array[idx].store(v, std::memory_order_release); + } +} + template diff --git a/runtime/mirror/dex_cache.cc b/runtime/mirror/dex_cache.cc index 1b8b3913b..c95d92e34 100644 --- a/runtime/mirror/dex_cache.cc +++ b/runtime/mirror/dex_cache.cc @@ -52,8 +52,12 @@ void DexCache::InitializeDexCache(Thread* self, dex_file->NumTypeIds() != 0u || dex_file->NumMethodIds() != 0u || dex_file->NumFieldIds() != 0u) { + static_assert(ArenaAllocator::kAlignment == 8, "Expecting arena alignment of 8."); + DCHECK(layout.Alignment() == 8u || layout.Alignment() == 16u); // Zero-initialized. - raw_arrays = reinterpret_cast(linear_alloc->Alloc(self, layout.Size())); + raw_arrays = (layout.Alignment() == 16u) + ? reinterpret_cast(linear_alloc->AllocAlign16(self, layout.Size())) + : reinterpret_cast(linear_alloc->Alloc(self, layout.Size())); } mirror::StringDexCacheType* strings = (dex_file->NumStringIds() == 0u) ? nullptr : @@ -62,17 +66,21 @@ void DexCache::InitializeDexCache(Thread* self, reinterpret_cast(raw_arrays + layout.TypesOffset()); ArtMethod** methods = (dex_file->NumMethodIds() == 0u) ? nullptr : reinterpret_cast(raw_arrays + layout.MethodsOffset()); - ArtField** fields = (dex_file->NumFieldIds() == 0u) ? nullptr : - reinterpret_cast(raw_arrays + layout.FieldsOffset()); + mirror::FieldDexCacheType* fields = (dex_file->NumFieldIds() == 0u) ? nullptr : + reinterpret_cast(raw_arrays + layout.FieldsOffset()); - size_t num_strings = mirror::DexCache::kDexCacheStringCacheSize; + size_t num_strings = kDexCacheStringCacheSize; if (dex_file->NumStringIds() < num_strings) { num_strings = dex_file->NumStringIds(); } - size_t num_types = mirror::DexCache::kDexCacheTypeCacheSize; + size_t num_types = kDexCacheTypeCacheSize; if (dex_file->NumTypeIds() < num_types) { num_types = dex_file->NumTypeIds(); } + size_t num_fields = kDexCacheFieldCacheSize; + if (dex_file->NumFieldIds() < num_fields) { + num_fields = dex_file->NumFieldIds(); + } // Note that we allocate the method type dex caches regardless of this flag, // and we make sure here that they're not used by the runtime. This is in the @@ -80,17 +88,17 @@ void DexCache::InitializeDexCache(Thread* self, // // If this needs to be mitigated in a production system running this code, // DexCache::kDexCacheMethodTypeCacheSize can be set to zero. - mirror::MethodTypeDexCacheType* method_types = nullptr; + MethodTypeDexCacheType* method_types = nullptr; size_t num_method_types = 0; - if (dex_file->NumProtoIds() < mirror::DexCache::kDexCacheMethodTypeCacheSize) { + if (dex_file->NumProtoIds() < kDexCacheMethodTypeCacheSize) { num_method_types = dex_file->NumProtoIds(); } else { - num_method_types = mirror::DexCache::kDexCacheMethodTypeCacheSize; + num_method_types = kDexCacheMethodTypeCacheSize; } if (num_method_types > 0) { - method_types = reinterpret_cast( + method_types = reinterpret_cast( raw_arrays + layout.MethodTypesOffset()); } @@ -98,13 +106,13 @@ void DexCache::InitializeDexCache(Thread* self, ? nullptr : reinterpret_cast*>(raw_arrays + layout.CallSitesOffset()); - DCHECK_ALIGNED(raw_arrays, alignof(mirror::StringDexCacheType)) << + DCHECK_ALIGNED(raw_arrays, alignof(StringDexCacheType)) << "Expected raw_arrays to align to StringDexCacheType."; - DCHECK_ALIGNED(layout.StringsOffset(), alignof(mirror::StringDexCacheType)) << + DCHECK_ALIGNED(layout.StringsOffset(), alignof(StringDexCacheType)) << "Expected StringsOffset() to align to StringDexCacheType."; - DCHECK_ALIGNED(strings, alignof(mirror::StringDexCacheType)) << + DCHECK_ALIGNED(strings, alignof(StringDexCacheType)) << "Expected strings to align to StringDexCacheType."; - static_assert(alignof(mirror::StringDexCacheType) == 8u, + static_assert(alignof(StringDexCacheType) == 8u, "Expected StringDexCacheType to have align of 8."); if (kIsDebugBuild) { // Sanity check to make sure all the dex cache arrays are empty. b/28992179 @@ -117,10 +125,11 @@ void DexCache::InitializeDexCache(Thread* self, CHECK(types[i].load(std::memory_order_relaxed).object.IsNull()); } for (size_t i = 0; i < dex_file->NumMethodIds(); ++i) { - CHECK(mirror::DexCache::GetElementPtrSize(methods, i, image_pointer_size) == nullptr); + CHECK(GetElementPtrSize(methods, i, image_pointer_size) == nullptr); } - for (size_t i = 0; i < dex_file->NumFieldIds(); ++i) { - CHECK(mirror::DexCache::GetElementPtrSize(fields, i, image_pointer_size) == nullptr); + for (size_t i = 0; i < num_fields; ++i) { + CHECK_EQ(GetNativePairPtrSize(fields, i, image_pointer_size).index, 0u); + CHECK(GetNativePairPtrSize(fields, i, image_pointer_size).object == nullptr); } for (size_t i = 0; i < num_method_types; ++i) { CHECK_EQ(method_types[i].load(std::memory_order_relaxed).index, 0u); @@ -136,6 +145,9 @@ void DexCache::InitializeDexCache(Thread* self, if (types != nullptr) { mirror::TypeDexCachePair::Initialize(types); } + if (fields != nullptr) { + mirror::FieldDexCachePair::Initialize(fields, image_pointer_size); + } if (method_types != nullptr) { mirror::MethodTypeDexCachePair::Initialize(method_types); } @@ -148,7 +160,7 @@ void DexCache::InitializeDexCache(Thread* self, methods, dex_file->NumMethodIds(), fields, - dex_file->NumFieldIds(), + num_fields, method_types, num_method_types, call_sites, @@ -164,7 +176,7 @@ void DexCache::Init(const DexFile* dex_file, uint32_t num_resolved_types, ArtMethod** resolved_methods, uint32_t num_resolved_methods, - ArtField** resolved_fields, + FieldDexCacheType* resolved_fields, uint32_t num_resolved_fields, MethodTypeDexCacheType* resolved_method_types, uint32_t num_resolved_method_types, @@ -218,5 +230,23 @@ void DexCache::SetLocation(ObjPtr location) { SetFieldObject(OFFSET_OF_OBJECT_MEMBER(DexCache, location_), location); } +#if !defined(__aarch64__) && !defined(__x86_64__) +static pthread_mutex_t dex_cache_slow_atomic_mutex = PTHREAD_MUTEX_INITIALIZER; + +DexCache::ConversionPair64 DexCache::AtomicLoadRelaxed16B(std::atomic* target) { + pthread_mutex_lock(&dex_cache_slow_atomic_mutex); + DexCache::ConversionPair64 value = *reinterpret_cast(target); + pthread_mutex_unlock(&dex_cache_slow_atomic_mutex); + return value; +} + +void DexCache::AtomicStoreRelease16B(std::atomic* target, + ConversionPair64 value) { + pthread_mutex_lock(&dex_cache_slow_atomic_mutex); + *reinterpret_cast(target) = value; + pthread_mutex_unlock(&dex_cache_slow_atomic_mutex); +} +#endif + } // namespace mirror } // namespace art diff --git a/runtime/mirror/dex_cache.h b/runtime/mirror/dex_cache.h index 057919806..35707ef4e 100644 --- a/runtime/mirror/dex_cache.h +++ b/runtime/mirror/dex_cache.h @@ -91,12 +91,44 @@ template struct PACKED(8) DexCachePair { } }; +template struct PACKED(2 * __SIZEOF_POINTER__) NativeDexCachePair { + T* object; + size_t index; + // This is similar to DexCachePair except that we're storing a native pointer + // instead of a GC root. See DexCachePair for the details. + NativeDexCachePair(T* object, uint32_t index) + : object(object), + index(index) {} + NativeDexCachePair() : object(nullptr), index(0u) { } + NativeDexCachePair(const NativeDexCachePair&) = default; + NativeDexCachePair& operator=(const NativeDexCachePair&) = default; + + static void Initialize(std::atomic>* dex_cache, PointerSize pointer_size); + + static uint32_t InvalidIndexForSlot(uint32_t slot) { + // Since the cache size is a power of two, 0 will always map to slot 0. + // Use 1 for slot 0 and 0 for all other slots. + return (slot == 0) ? 1u : 0u; + } + + T* GetObjectForIndex(uint32_t idx) REQUIRES_SHARED(Locks::mutator_lock_) { + if (idx != index) { + return nullptr; + } + DCHECK(object != nullptr); + return object; + } +}; + using TypeDexCachePair = DexCachePair; using TypeDexCacheType = std::atomic; using StringDexCachePair = DexCachePair; using StringDexCacheType = std::atomic; +using FieldDexCachePair = NativeDexCachePair; +using FieldDexCacheType = std::atomic; + using MethodTypeDexCachePair = DexCachePair; using MethodTypeDexCacheType = std::atomic; @@ -116,6 +148,11 @@ class MANAGED DexCache FINAL : public Object { static_assert(IsPowerOfTwo(kDexCacheStringCacheSize), "String dex cache size is not a power of 2."); + // Size of field dex cache. Needs to be a power of 2 for entrypoint assumptions to hold. + static constexpr size_t kDexCacheFieldCacheSize = 1024; + static_assert(IsPowerOfTwo(kDexCacheFieldCacheSize), + "Field dex cache size is not a power of 2."); + // Size of method type dex cache. Needs to be a power of 2 for entrypoint assumptions // to hold. static constexpr size_t kDexCacheMethodTypeCacheSize = 1024; @@ -130,6 +167,10 @@ class MANAGED DexCache FINAL : public Object { return kDexCacheStringCacheSize; } + static constexpr size_t StaticArtFieldSize() { + return kDexCacheFieldCacheSize; + } + static constexpr size_t StaticMethodTypeSize() { return kDexCacheMethodTypeCacheSize; } @@ -255,6 +296,8 @@ class MANAGED DexCache FINAL : public Object { // Pointer sized variant, used for patching. ALWAYS_INLINE void SetResolvedField(uint32_t idx, ArtField* field, PointerSize ptr_size) REQUIRES_SHARED(Locks::mutator_lock_); + ALWAYS_INLINE void ClearResolvedField(uint32_t idx, PointerSize ptr_size) + REQUIRES_SHARED(Locks::mutator_lock_); MethodType* GetResolvedMethodType(uint32_t proto_idx) REQUIRES_SHARED(Locks::mutator_lock_); @@ -299,11 +342,11 @@ class MANAGED DexCache FINAL : public Object { SetFieldPtr(ResolvedMethodsOffset(), resolved_methods); } - ArtField** GetResolvedFields() ALWAYS_INLINE REQUIRES_SHARED(Locks::mutator_lock_) { - return GetFieldPtr(ResolvedFieldsOffset()); + FieldDexCacheType* GetResolvedFields() ALWAYS_INLINE REQUIRES_SHARED(Locks::mutator_lock_) { + return GetFieldPtr(ResolvedFieldsOffset()); } - void SetResolvedFields(ArtField** resolved_fields) + void SetResolvedFields(FieldDexCacheType* resolved_fields) ALWAYS_INLINE REQUIRES_SHARED(Locks::mutator_lock_) { SetFieldPtr(ResolvedFieldsOffset(), resolved_fields); @@ -376,6 +419,17 @@ class MANAGED DexCache FINAL : public Object { template static void SetElementPtrSize(PtrType* ptr_array, size_t idx, PtrType ptr, PointerSize ptr_size); + template + static NativeDexCachePair GetNativePairPtrSize(std::atomic>* pair_array, + size_t idx, + PointerSize ptr_size); + + template + static void SetNativePairPtrSize(std::atomic>* pair_array, + size_t idx, + NativeDexCachePair pair, + PointerSize ptr_size); + private: void Init(const DexFile* dex_file, ObjPtr location, @@ -385,7 +439,7 @@ class MANAGED DexCache FINAL : public Object { uint32_t num_resolved_types, ArtMethod** resolved_methods, uint32_t num_resolved_methods, - ArtField** resolved_fields, + FieldDexCacheType* resolved_fields, uint32_t num_resolved_fields, MethodTypeDexCacheType* resolved_method_types, uint32_t num_resolved_method_types, @@ -394,8 +448,22 @@ class MANAGED DexCache FINAL : public Object { PointerSize pointer_size) REQUIRES_SHARED(Locks::mutator_lock_); + // std::pair<> is not trivially copyable and as such it is unsuitable for atomic operations, + // so we use a custom pair class for loading and storing the NativeDexCachePair<>. + template + struct PACKED(2 * sizeof(IntType)) ConversionPair { + ConversionPair(IntType f, IntType s) : first(f), second(s) { } + ConversionPair(const ConversionPair&) = default; + ConversionPair& operator=(const ConversionPair&) = default; + IntType first; + IntType second; + }; + using ConversionPair32 = ConversionPair; + using ConversionPair64 = ConversionPair; + uint32_t StringSlotIndex(dex::StringIndex string_idx) REQUIRES_SHARED(Locks::mutator_lock_); uint32_t TypeSlotIndex(dex::TypeIndex type_idx) REQUIRES_SHARED(Locks::mutator_lock_); + uint32_t FieldSlotIndex(uint32_t field_idx) REQUIRES_SHARED(Locks::mutator_lock_); uint32_t MethodTypeSlotIndex(uint32_t proto_idx) REQUIRES_SHARED(Locks::mutator_lock_); // Visit instance fields of the dex cache as well as its associated arrays. @@ -406,12 +474,55 @@ class MANAGED DexCache FINAL : public Object { void VisitReferences(ObjPtr klass, const Visitor& visitor) REQUIRES_SHARED(Locks::mutator_lock_) REQUIRES(Locks::heap_bitmap_lock_); + // Due to lack of 16-byte atomics support, we use hand-crafted routines. +#if defined(__aarch64__) + // 16-byte atomics are supported on aarch64. + ALWAYS_INLINE static ConversionPair64 AtomicLoadRelaxed16B( + std::atomic* target) { + return target->load(std::memory_order_relaxed); + } + + ALWAYS_INLINE static void AtomicStoreRelease16B( + std::atomic* target, ConversionPair64 value) { + target->store(value, std::memory_order_release); + } +#elif defined(__x86_64__) + ALWAYS_INLINE static ConversionPair64 AtomicLoadRelaxed16B( + std::atomic* target) { + uint64_t first, second; + __asm__ __volatile__( + "lock cmpxchg16b (%2)" + : "=&a"(first), "=&d"(second) + : "r"(target), "a"(0), "d"(0), "b"(0), "c"(0) + : "cc"); + return ConversionPair64(first, second); + } + + ALWAYS_INLINE static void AtomicStoreRelease16B( + std::atomic* target, ConversionPair64 value) { + uint64_t first, second; + __asm__ __volatile__ ( + "movq (%2), %%rax\n\t" + "movq 8(%2), %%rdx\n\t" + "1:\n\t" + "lock cmpxchg16b (%2)\n\t" + "jnz 1b" + : "=&a"(first), "=&d"(second) + : "r"(target), "b"(value.first), "c"(value.second) + : "cc"); + } +#else + static ConversionPair64 AtomicLoadRelaxed16B(std::atomic* target); + static void AtomicStoreRelease16B(std::atomic* target, ConversionPair64 value); +#endif + HeapReference dex_; HeapReference location_; uint64_t dex_file_; // const DexFile* uint64_t resolved_call_sites_; // GcRoot* array with num_resolved_call_sites_ // elements. - uint64_t resolved_fields_; // ArtField*, array with num_resolved_fields_ elements. + uint64_t resolved_fields_; // std::atomic*, array with + // num_resolved_fields_ elements. uint64_t resolved_method_types_; // std::atomic* array with // num_resolved_method_types_ elements. uint64_t resolved_methods_; // ArtMethod*, array with num_resolved_methods_ elements. diff --git a/runtime/mirror/dex_cache_test.cc b/runtime/mirror/dex_cache_test.cc index ef0aaaaa7..71a47f66a 100644 --- a/runtime/mirror/dex_cache_test.cc +++ b/runtime/mirror/dex_cache_test.cc @@ -54,7 +54,8 @@ TEST_F(DexCacheTest, Open) { EXPECT_TRUE(dex_cache->StaticTypeSize() == dex_cache->NumResolvedTypes() || java_lang_dex_file_->NumTypeIds() == dex_cache->NumResolvedTypes()); EXPECT_EQ(java_lang_dex_file_->NumMethodIds(), dex_cache->NumResolvedMethods()); - EXPECT_EQ(java_lang_dex_file_->NumFieldIds(), dex_cache->NumResolvedFields()); + EXPECT_TRUE(dex_cache->StaticArtFieldSize() == dex_cache->NumResolvedFields() + || java_lang_dex_file_->NumFieldIds() == dex_cache->NumResolvedFields()); EXPECT_TRUE(dex_cache->StaticMethodTypeSize() == dex_cache->NumResolvedMethodTypes() || java_lang_dex_file_->NumProtoIds() == dex_cache->NumResolvedMethodTypes()); } diff --git a/runtime/mirror/field.cc b/runtime/mirror/field.cc index f6b64897f..54034c2bb 100644 --- a/runtime/mirror/field.cc +++ b/runtime/mirror/field.cc @@ -68,8 +68,16 @@ ArtField* Field::GetArtField() { } } mirror::DexCache* const dex_cache = declaring_class->GetDexCache(); - ArtField* const art_field = dex_cache->GetResolvedField(GetDexFieldIndex(), kRuntimePointerSize); - CHECK(art_field != nullptr); + ArtField* art_field = dex_cache->GetResolvedField(GetDexFieldIndex(), kRuntimePointerSize); + if (UNLIKELY(art_field == nullptr)) { + if (IsStatic()) { + art_field = declaring_class->FindDeclaredStaticField(dex_cache, GetDexFieldIndex()); + } else { + art_field = declaring_class->FindInstanceField(dex_cache, GetDexFieldIndex()); + } + CHECK(art_field != nullptr); + dex_cache->SetResolvedField(GetDexFieldIndex(), art_field, kRuntimePointerSize); + } CHECK_EQ(declaring_class, art_field->GetDeclaringClass()); return art_field; } diff --git a/runtime/native/dalvik_system_VMRuntime.cc b/runtime/native/dalvik_system_VMRuntime.cc index 9b707f8e1..d81c13df1 100644 --- a/runtime/native/dalvik_system_VMRuntime.cc +++ b/runtime/native/dalvik_system_VMRuntime.cc @@ -444,6 +444,7 @@ static void PreloadDexCachesStatsFilled(DexCacheStats* filled) if (!kPreloadDexCachesCollectStats) { return; } + // TODO: Update for hash-based DexCache arrays. ClassLinker* const class_linker = Runtime::Current()->GetClassLinker(); Thread* const self = Thread::Current(); for (const DexFile* dex_file : class_linker->GetBootClassPath()) { @@ -463,7 +464,7 @@ static void PreloadDexCachesStatsFilled(DexCacheStats* filled) } } for (size_t j = 0; j < dex_cache->NumResolvedFields(); j++) { - ArtField* field = class_linker->GetResolvedField(j, dex_cache); + ArtField* field = dex_cache->GetResolvedField(j, class_linker->GetImagePointerSize()); if (field != nullptr) { filled->num_fields++; } diff --git a/runtime/oat.h b/runtime/oat.h index 1544121ed..df4310764 100644 --- a/runtime/oat.h +++ b/runtime/oat.h @@ -32,7 +32,7 @@ class InstructionSetFeatures; class PACKED(4) OatHeader { public: static constexpr uint8_t kOatMagic[] = { 'o', 'a', 't', '\n' }; - static constexpr uint8_t kOatVersion[] = { '1', '1', '4', '\0' }; // hash-based DexCache types. + static constexpr uint8_t kOatVersion[] = { '1', '1', '5', '\0' }; // hash-based DexCache fields static constexpr const char* kImageLocationKey = "image-location"; static constexpr const char* kDex2OatCmdLineKey = "dex2oat-cmdline"; diff --git a/runtime/quick/inline_method_analyser.cc b/runtime/quick/inline_method_analyser.cc index b009b4719..334707046 100644 --- a/runtime/quick/inline_method_analyser.cc +++ b/runtime/quick/inline_method_analyser.cc @@ -215,9 +215,8 @@ bool RecordConstructorIPut(ArtMethod* method, REQUIRES_SHARED(Locks::mutator_lock_) { DCHECK(IsInstructionIPut(new_iput->Opcode())); uint32_t field_index = new_iput->VRegC_22c(); - PointerSize pointer_size = Runtime::Current()->GetClassLinker()->GetImagePointerSize(); - mirror::DexCache* dex_cache = method->GetDexCache(); - ArtField* field = dex_cache->GetResolvedField(field_index, pointer_size); + ClassLinker* class_linker = Runtime::Current()->GetClassLinker(); + ArtField* field = class_linker->LookupResolvedField(field_index, method, /* is_static */ false); if (UNLIKELY(field == nullptr)) { return false; } @@ -227,7 +226,9 @@ bool RecordConstructorIPut(ArtMethod* method, if (iputs[old_pos].field_index == DexFile::kDexNoIndex16) { break; } - ArtField* f = dex_cache->GetResolvedField(iputs[old_pos].field_index, pointer_size); + ArtField* f = class_linker->LookupResolvedField(iputs[old_pos].field_index, + method, + /* is_static */ false); DCHECK(f != nullptr); if (f == field) { auto back_it = std::copy(iputs + old_pos + 1, iputs + arraysize(iputs), iputs + old_pos); @@ -732,9 +733,9 @@ bool InlineMethodAnalyser::ComputeSpecialAccessorInfo(ArtMethod* method, if (method == nullptr) { return false; } - mirror::DexCache* dex_cache = method->GetDexCache(); - PointerSize pointer_size = Runtime::Current()->GetClassLinker()->GetImagePointerSize(); - ArtField* field = dex_cache->GetResolvedField(field_idx, pointer_size); + ObjPtr dex_cache = method->GetDexCache(); + ClassLinker* class_linker = Runtime::Current()->GetClassLinker(); + ArtField* field = class_linker->LookupResolvedField(field_idx, method, /* is_static */ false); if (field == nullptr || field->IsStatic()) { return false; } diff --git a/runtime/utils/dex_cache_arrays_layout-inl.h b/runtime/utils/dex_cache_arrays_layout-inl.h index f9a140535..95904af01 100644 --- a/runtime/utils/dex_cache_arrays_layout-inl.h +++ b/runtime/utils/dex_cache_arrays_layout-inl.h @@ -51,7 +51,11 @@ inline DexCacheArraysLayout::DexCacheArraysLayout(PointerSize pointer_size, cons : DexCacheArraysLayout(pointer_size, dex_file->GetHeader(), dex_file->NumCallSiteIds()) { } -constexpr size_t DexCacheArraysLayout::Alignment() { +inline size_t DexCacheArraysLayout::Alignment() const { + return Alignment(pointer_size_); +} + +inline constexpr size_t DexCacheArraysLayout::Alignment(PointerSize pointer_size) { // mirror::Type/String/MethodTypeDexCacheType alignment is 8, // i.e. higher than or equal to the pointer alignment. static_assert(alignof(mirror::TypeDexCacheType) == 8, @@ -60,8 +64,8 @@ constexpr size_t DexCacheArraysLayout::Alignment() { "Expecting alignof(StringDexCacheType) == 8"); static_assert(alignof(mirror::MethodTypeDexCacheType) == 8, "Expecting alignof(MethodTypeDexCacheType) == 8"); - // This is the same as alignof(MethodTypeDexCacheType). - return alignof(mirror::StringDexCacheType); + // This is the same as alignof(FieldDexCacheType) for the given pointer size. + return 2u * static_cast(pointer_size); } template @@ -100,8 +104,8 @@ inline size_t DexCacheArraysLayout::MethodsAlignment() const { } inline size_t DexCacheArraysLayout::StringOffset(uint32_t string_idx) const { - return strings_offset_ + ElementOffset(PointerSize::k64, - string_idx % mirror::DexCache::kDexCacheStringCacheSize); + uint32_t string_hash = string_idx % mirror::DexCache::kDexCacheStringCacheSize; + return strings_offset_ + ElementOffset(PointerSize::k64, string_hash); } inline size_t DexCacheArraysLayout::StringsSize(size_t num_elements) const { @@ -119,15 +123,20 @@ inline size_t DexCacheArraysLayout::StringsAlignment() const { } inline size_t DexCacheArraysLayout::FieldOffset(uint32_t field_idx) const { - return fields_offset_ + ElementOffset(pointer_size_, field_idx); + uint32_t field_hash = field_idx % mirror::DexCache::kDexCacheFieldCacheSize; + return fields_offset_ + 2u * static_cast(pointer_size_) * field_hash; } inline size_t DexCacheArraysLayout::FieldsSize(size_t num_elements) const { - return ArraySize(pointer_size_, num_elements); + size_t cache_size = mirror::DexCache::kDexCacheFieldCacheSize; + if (num_elements < cache_size) { + cache_size = num_elements; + } + return 2u * static_cast(pointer_size_) * num_elements; } inline size_t DexCacheArraysLayout::FieldsAlignment() const { - return static_cast(pointer_size_); + return 2u * static_cast(pointer_size_); } inline size_t DexCacheArraysLayout::MethodTypesSize(size_t num_elements) const { diff --git a/runtime/utils/dex_cache_arrays_layout.h b/runtime/utils/dex_cache_arrays_layout.h index ed677ed3f..377a3749a 100644 --- a/runtime/utils/dex_cache_arrays_layout.h +++ b/runtime/utils/dex_cache_arrays_layout.h @@ -57,7 +57,9 @@ class DexCacheArraysLayout { return size_; } - static constexpr size_t Alignment(); + size_t Alignment() const; + + static constexpr size_t Alignment(PointerSize pointer_size); size_t TypesOffset() const { return types_offset_; @@ -125,8 +127,6 @@ class DexCacheArraysLayout { const size_t call_sites_offset_; const size_t size_; - static size_t Alignment(PointerSize pointer_size); - static size_t ElementOffset(PointerSize element_size, uint32_t idx); static size_t ArraySize(PointerSize element_size, uint32_t num_elements); -- 2.11.0