From: Vladimir Marko Date: Tue, 31 Mar 2015 20:49:49 +0000 (+0100) Subject: Refactor RelativePatcher out of OatWriter. X-Git-Tag: android-x86-7.1-r1~889^2~1642^2~2 X-Git-Url: http://git.osdn.net/view?a=commitdiff_plain;h=b163bb742a099c1808907b513ae39068b63b1692;p=android-x86%2Fart.git Refactor RelativePatcher out of OatWriter. Move the relative patcher classes to compiler/linker/ and compiler/linker// . Refactor them to avoid OatWriter dependency so that they can be unit tested. Add tests for x86 and x86-64. Change-Id: I1b42baa9fc431378e4cce1399bec590c5b5a409f --- diff --git a/build/Android.gtest.mk b/build/Android.gtest.mk index 1a4c30c20..58a3290e0 100644 --- a/build/Android.gtest.mk +++ b/build/Android.gtest.mk @@ -193,6 +193,8 @@ COMPILER_GTEST_COMMON_SRC_FILES := \ compiler/elf_writer_test.cc \ compiler/image_test.cc \ compiler/jni/jni_compiler_test.cc \ + compiler/linker/x86/relative_patcher_x86_test.cc \ + compiler/linker/x86_64/relative_patcher_x86_64_test.cc \ compiler/oat_test.cc \ compiler/optimizing/bounds_check_elimination_test.cc \ compiler/optimizing/codegen_test.cc \ diff --git a/compiler/Android.mk b/compiler/Android.mk index 904f117a5..eaea031b6 100644 --- a/compiler/Android.mk +++ b/compiler/Android.mk @@ -79,6 +79,13 @@ LIBART_COMPILER_SRC_FILES := \ driver/compiler_driver.cc \ driver/compiler_options.cc \ driver/dex_compilation_unit.cc \ + linker/relative_patcher.cc \ + linker/arm/relative_patcher_arm_base.cc \ + linker/arm/relative_patcher_thumb2.cc \ + linker/arm64/relative_patcher_arm64.cc \ + linker/x86/relative_patcher_x86_base.cc \ + linker/x86/relative_patcher_x86.cc \ + linker/x86_64/relative_patcher_x86_64.cc \ jit/jit_compiler.cc \ jni/quick/arm/calling_convention_arm.cc \ jni/quick/arm64/calling_convention_arm64.cc \ diff --git a/compiler/driver/compiler_options.cc b/compiler/driver/compiler_options.cc index e436f52db..fc00c926b 100644 --- a/compiler/driver/compiler_options.cc +++ b/compiler/driver/compiler_options.cc @@ -42,6 +42,11 @@ CompilerOptions::CompilerOptions() init_failure_output_(nullptr) { } +CompilerOptions::~CompilerOptions() { + // The destructor looks empty but it destroys a PassManagerOptions object. We keep it here + // because we don't want to include the PassManagerOptions definition from the header file. +} + CompilerOptions::CompilerOptions(CompilerFilter compiler_filter, size_t huge_method_threshold, size_t large_method_threshold, diff --git a/compiler/driver/compiler_options.h b/compiler/driver/compiler_options.h index d06ec278a..f7ea385e1 100644 --- a/compiler/driver/compiler_options.h +++ b/compiler/driver/compiler_options.h @@ -53,6 +53,7 @@ class CompilerOptions FINAL { static const bool kDefaultIncludePatchInformation = false; CompilerOptions(); + ~CompilerOptions(); CompilerOptions(CompilerFilter compiler_filter, size_t huge_method_threshold, diff --git a/compiler/linker/arm/relative_patcher_arm_base.cc b/compiler/linker/arm/relative_patcher_arm_base.cc new file mode 100644 index 000000000..ecbbd09c8 --- /dev/null +++ b/compiler/linker/arm/relative_patcher_arm_base.cc @@ -0,0 +1,163 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "linker/arm/relative_patcher_arm_base.h" + +#include "compiled_method.h" +#include "oat.h" +#include "output_stream.h" + +namespace art { +namespace linker { + +uint32_t ArmBaseRelativePatcher::ReserveSpace(uint32_t offset, + const CompiledMethod* compiled_method) { + return ReserveSpaceInternal(offset, compiled_method, 0u); +} + +uint32_t ArmBaseRelativePatcher::WriteThunks(OutputStream* out, uint32_t offset) { + if (current_thunk_to_write_ == thunk_locations_.size()) { + return offset; + } + uint32_t aligned_offset = CompiledMethod::AlignCode(offset, instruction_set_); + if (UNLIKELY(aligned_offset == thunk_locations_[current_thunk_to_write_])) { + ++current_thunk_to_write_; + uint32_t aligned_code_delta = aligned_offset - offset; + if (aligned_code_delta != 0u && !WriteCodeAlignment(out, aligned_code_delta)) { + return 0u; + } + if (UNLIKELY(!WriteRelCallThunk(out, ArrayRef(thunk_code_)))) { + return 0u; + } + uint32_t thunk_end_offset = aligned_offset + thunk_code_.size(); + // Align after writing chunk, see the ReserveSpace() above. + offset = CompiledMethod::AlignCode(thunk_end_offset, instruction_set_); + aligned_code_delta = offset - thunk_end_offset; + if (aligned_code_delta != 0u && !WriteCodeAlignment(out, aligned_code_delta)) { + return 0u; + } + } + return offset; +} + +ArmBaseRelativePatcher::ArmBaseRelativePatcher(RelativePatcherTargetProvider* provider, + InstructionSet instruction_set, + std::vector thunk_code, + uint32_t max_positive_displacement, + uint32_t max_negative_displacement) + : provider_(provider), instruction_set_(instruction_set), thunk_code_(thunk_code), + max_positive_displacement_(max_positive_displacement), + max_negative_displacement_(max_negative_displacement), + thunk_locations_(), current_thunk_to_write_(0u), unprocessed_patches_() { +} + +uint32_t ArmBaseRelativePatcher::ReserveSpaceInternal(uint32_t offset, + const CompiledMethod* compiled_method, + uint32_t max_extra_space) { + // NOTE: The final thunk can be reserved from InitCodeMethodVisitor::EndClass() while it + // may be written early by WriteCodeMethodVisitor::VisitMethod() for a deduplicated chunk + // of code. To avoid any alignment discrepancies for the final chunk, we always align the + // offset after reserving of writing any chunk. + if (UNLIKELY(compiled_method == nullptr)) { + uint32_t aligned_offset = CompiledMethod::AlignCode(offset, instruction_set_); + bool needs_thunk = ReserveSpaceProcessPatches(aligned_offset); + if (needs_thunk) { + thunk_locations_.push_back(aligned_offset); + offset = CompiledMethod::AlignCode(aligned_offset + thunk_code_.size(), instruction_set_); + } + return offset; + } + DCHECK(compiled_method->GetQuickCode() != nullptr); + uint32_t quick_code_size = compiled_method->GetQuickCode()->size(); + uint32_t quick_code_offset = compiled_method->AlignCode(offset) + sizeof(OatQuickMethodHeader); + uint32_t next_aligned_offset = compiled_method->AlignCode(quick_code_offset + quick_code_size); + // Adjust for extra space required by the subclass. + next_aligned_offset = compiled_method->AlignCode(next_aligned_offset + max_extra_space); + if (!unprocessed_patches_.empty() && + next_aligned_offset - unprocessed_patches_.front().second > max_positive_displacement_) { + bool needs_thunk = ReserveSpaceProcessPatches(next_aligned_offset); + if (needs_thunk) { + // A single thunk will cover all pending patches. + unprocessed_patches_.clear(); + uint32_t thunk_location = compiled_method->AlignCode(offset); + thunk_locations_.push_back(thunk_location); + offset = CompiledMethod::AlignCode(thunk_location + thunk_code_.size(), instruction_set_); + } + } + for (const LinkerPatch& patch : compiled_method->GetPatches()) { + if (patch.Type() == kLinkerPatchCallRelative) { + unprocessed_patches_.emplace_back(patch.TargetMethod(), + quick_code_offset + patch.LiteralOffset()); + } + } + return offset; +} + +uint32_t ArmBaseRelativePatcher::CalculateDisplacement(uint32_t patch_offset, + uint32_t target_offset) { + // Unsigned arithmetic with its well-defined overflow behavior is just fine here. + uint32_t displacement = target_offset - patch_offset; + // NOTE: With unsigned arithmetic we do mean to use && rather than || below. + if (displacement > max_positive_displacement_ && displacement < -max_negative_displacement_) { + // Unwritten thunks have higher offsets, check if it's within range. + DCHECK(current_thunk_to_write_ == thunk_locations_.size() || + thunk_locations_[current_thunk_to_write_] > patch_offset); + if (current_thunk_to_write_ != thunk_locations_.size() && + thunk_locations_[current_thunk_to_write_] - patch_offset < max_positive_displacement_) { + displacement = thunk_locations_[current_thunk_to_write_] - patch_offset; + } else { + // We must have a previous thunk then. + DCHECK_NE(current_thunk_to_write_, 0u); + DCHECK_LT(thunk_locations_[current_thunk_to_write_ - 1], patch_offset); + displacement = thunk_locations_[current_thunk_to_write_ - 1] - patch_offset; + DCHECK(displacement >= -max_negative_displacement_); + } + } + return displacement; +} + +bool ArmBaseRelativePatcher::ReserveSpaceProcessPatches(uint32_t next_aligned_offset) { + // Process as many patches as possible, stop only on unresolved targets or calls too far back. + while (!unprocessed_patches_.empty()) { + uint32_t patch_offset = unprocessed_patches_.front().second; + auto result = provider_->FindMethodOffset(unprocessed_patches_.front().first); + if (!result.first) { + // If still unresolved, check if we have a thunk within range. + DCHECK(thunk_locations_.empty() || thunk_locations_.back() <= patch_offset); + if (thunk_locations_.empty() || + patch_offset - thunk_locations_.back() > max_negative_displacement_) { + return next_aligned_offset - patch_offset > max_positive_displacement_; + } + } else if (result.second >= patch_offset) { + DCHECK_LE(result.second - patch_offset, max_positive_displacement_); + } else { + // When calling back, check if we have a thunk that's closer than the actual target. + uint32_t target_offset = + (thunk_locations_.empty() || result.second > thunk_locations_.back()) + ? result.second + : thunk_locations_.back(); + DCHECK_GT(patch_offset, target_offset); + if (patch_offset - target_offset > max_negative_displacement_) { + return true; + } + } + unprocessed_patches_.pop_front(); + } + return false; +} + +} // namespace linker +} // namespace art diff --git a/compiler/linker/arm/relative_patcher_arm_base.h b/compiler/linker/arm/relative_patcher_arm_base.h new file mode 100644 index 000000000..a88d25bde --- /dev/null +++ b/compiler/linker/arm/relative_patcher_arm_base.h @@ -0,0 +1,63 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_COMPILER_LINKER_ARM_RELATIVE_PATCHER_ARM_BASE_H_ +#define ART_COMPILER_LINKER_ARM_RELATIVE_PATCHER_ARM_BASE_H_ + +#include + +#include "linker/relative_patcher.h" +#include "method_reference.h" + +namespace art { +namespace linker { + +class ArmBaseRelativePatcher : public RelativePatcher { + public: + uint32_t ReserveSpace(uint32_t offset, const CompiledMethod* compiled_method) OVERRIDE; + uint32_t WriteThunks(OutputStream* out, uint32_t offset) OVERRIDE; + + protected: + ArmBaseRelativePatcher(RelativePatcherTargetProvider* provider, + InstructionSet instruction_set, std::vector thunk_code, + uint32_t max_positive_displacement, uint32_t max_negative_displacement); + + uint32_t ReserveSpaceInternal(uint32_t offset, const CompiledMethod* compiled_method, + uint32_t max_extra_space); + uint32_t CalculateDisplacement(uint32_t patch_offset, uint32_t target_offset); + + private: + bool ReserveSpaceProcessPatches(uint32_t next_aligned_offset); + + RelativePatcherTargetProvider* const provider_; + const InstructionSet instruction_set_; + const std::vector thunk_code_; + const uint32_t max_positive_displacement_; + const uint32_t max_negative_displacement_; + std::vector thunk_locations_; + size_t current_thunk_to_write_; + + // ReserveSpace() tracks unprocessed patches. + typedef std::pair UnprocessedPatch; + std::deque unprocessed_patches_; + + DISALLOW_COPY_AND_ASSIGN(ArmBaseRelativePatcher); +}; + +} // namespace linker +} // namespace art + +#endif // ART_COMPILER_LINKER_ARM_RELATIVE_PATCHER_ARM_BASE_H_ diff --git a/compiler/linker/arm/relative_patcher_thumb2.cc b/compiler/linker/arm/relative_patcher_thumb2.cc new file mode 100644 index 000000000..426774309 --- /dev/null +++ b/compiler/linker/arm/relative_patcher_thumb2.cc @@ -0,0 +1,84 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "linker/arm/relative_patcher_thumb2.h" + +#include "compiled_method.h" +#include "mirror/art_method.h" +#include "utils/arm/assembler_thumb2.h" + +namespace art { +namespace linker { + +Thumb2RelativePatcher::Thumb2RelativePatcher(RelativePatcherTargetProvider* provider) + : ArmBaseRelativePatcher(provider, kThumb2, CompileThunkCode(), + kMaxPositiveDisplacement, kMaxNegativeDisplacement) { +} + +void Thumb2RelativePatcher::PatchCall(std::vector* code, uint32_t literal_offset, + uint32_t patch_offset, uint32_t target_offset) { + DCHECK_LE(literal_offset + 4u, code->size()); + DCHECK_EQ(literal_offset & 1u, 0u); + DCHECK_EQ(patch_offset & 1u, 0u); + DCHECK_EQ(target_offset & 1u, 1u); // Thumb2 mode bit. + uint32_t displacement = CalculateDisplacement(patch_offset, target_offset & ~1u); + displacement -= kPcDisplacement; // The base PC is at the end of the 4-byte patch. + DCHECK_EQ(displacement & 1u, 0u); + DCHECK((displacement >> 24) == 0u || (displacement >> 24) == 255u); // 25-bit signed. + uint32_t signbit = (displacement >> 31) & 0x1; + uint32_t i1 = (displacement >> 23) & 0x1; + uint32_t i2 = (displacement >> 22) & 0x1; + uint32_t imm10 = (displacement >> 12) & 0x03ff; + uint32_t imm11 = (displacement >> 1) & 0x07ff; + uint32_t j1 = i1 ^ (signbit ^ 1); + uint32_t j2 = i2 ^ (signbit ^ 1); + uint32_t value = (signbit << 26) | (j1 << 13) | (j2 << 11) | (imm10 << 16) | imm11; + value |= 0xf000d000; // BL + + uint8_t* addr = &(*code)[literal_offset]; + // Check that we're just overwriting an existing BL. + DCHECK_EQ(addr[1] & 0xf8, 0xf0); + DCHECK_EQ(addr[3] & 0xd0, 0xd0); + // Write the new BL. + addr[0] = (value >> 16) & 0xff; + addr[1] = (value >> 24) & 0xff; + addr[2] = (value >> 0) & 0xff; + addr[3] = (value >> 8) & 0xff; +} + +void Thumb2RelativePatcher::PatchDexCacheReference(std::vector* code ATTRIBUTE_UNUSED, + const LinkerPatch& patch ATTRIBUTE_UNUSED, + uint32_t patch_offset ATTRIBUTE_UNUSED, + uint32_t target_offset ATTRIBUTE_UNUSED) { + LOG(FATAL) << "Unexpected relative dex cache array patch."; +} + +std::vector Thumb2RelativePatcher::CompileThunkCode() { + // The thunk just uses the entry point in the ArtMethod. This works even for calls + // to the generic JNI and interpreter trampolines. + arm::Thumb2Assembler assembler; + assembler.LoadFromOffset( + arm::kLoadWord, arm::PC, arm::R0, + mirror::ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize).Int32Value()); + assembler.bkpt(0); + std::vector thunk_code(assembler.CodeSize()); + MemoryRegion code(thunk_code.data(), thunk_code.size()); + assembler.FinalizeInstructions(code); + return thunk_code; +} + +} // namespace linker +} // namespace art diff --git a/compiler/linker/arm/relative_patcher_thumb2.h b/compiler/linker/arm/relative_patcher_thumb2.h new file mode 100644 index 000000000..561130305 --- /dev/null +++ b/compiler/linker/arm/relative_patcher_thumb2.h @@ -0,0 +1,52 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_COMPILER_LINKER_ARM_RELATIVE_PATCHER_THUMB2_H_ +#define ART_COMPILER_LINKER_ARM_RELATIVE_PATCHER_THUMB2_H_ + +#include "linker/arm/relative_patcher_arm_base.h" + +namespace art { +namespace linker { + +class Thumb2RelativePatcher FINAL : public ArmBaseRelativePatcher { + public: + explicit Thumb2RelativePatcher(RelativePatcherTargetProvider* provider); + + void PatchCall(std::vector* code, uint32_t literal_offset, + uint32_t patch_offset, uint32_t target_offset) OVERRIDE; + void PatchDexCacheReference(std::vector* code, const LinkerPatch& patch, + uint32_t patch_offset, uint32_t target_offset) OVERRIDE; + + private: + static std::vector CompileThunkCode(); + + // PC displacement from patch location; Thumb2 PC is always at instruction address + 4. + static constexpr int32_t kPcDisplacement = 4; + + // Maximum positive and negative displacement measured from the patch location. + // (Signed 25 bit displacement with the last bit 0 has range [-2^24, 2^24-2] measured from + // the Thumb2 PC pointing right after the BL, i.e. 4 bytes later than the patch location.) + static constexpr uint32_t kMaxPositiveDisplacement = (1u << 24) - 2 + kPcDisplacement; + static constexpr uint32_t kMaxNegativeDisplacement = (1u << 24) - kPcDisplacement; + + DISALLOW_COPY_AND_ASSIGN(Thumb2RelativePatcher); +}; + +} // namespace linker +} // namespace art + +#endif // ART_COMPILER_LINKER_ARM_RELATIVE_PATCHER_THUMB2_H_ diff --git a/compiler/linker/arm64/relative_patcher_arm64.cc b/compiler/linker/arm64/relative_patcher_arm64.cc new file mode 100644 index 000000000..bac300118 --- /dev/null +++ b/compiler/linker/arm64/relative_patcher_arm64.cc @@ -0,0 +1,284 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "linker/arm64/relative_patcher_arm64.h" + +#include "arch/arm64/instruction_set_features_arm64.h" +#include "compiled_method.h" +#include "driver/compiler_driver.h" +#include "mirror/art_method.h" +#include "utils/arm64/assembler_arm64.h" +#include "oat.h" +#include "output_stream.h" + +namespace art { +namespace linker { + +Arm64RelativePatcher::Arm64RelativePatcher(RelativePatcherTargetProvider* provider, + const Arm64InstructionSetFeatures* features) + : ArmBaseRelativePatcher(provider, kArm64, CompileThunkCode(), + kMaxPositiveDisplacement, kMaxNegativeDisplacement), + fix_cortex_a53_843419_(features->NeedFixCortexA53_843419()), + reserved_adrp_thunks_(0u), + processed_adrp_thunks_(0u) { + if (fix_cortex_a53_843419_) { + adrp_thunk_locations_.reserve(16u); + current_method_thunks_.reserve(16u * kAdrpThunkSize); + } +} + +uint32_t Arm64RelativePatcher::ReserveSpace(uint32_t offset, + const CompiledMethod* compiled_method) { + if (!fix_cortex_a53_843419_) { + DCHECK(adrp_thunk_locations_.empty()); + return ReserveSpaceInternal(offset, compiled_method, 0u); + } + + // Add thunks for previous method if any. + if (reserved_adrp_thunks_ != adrp_thunk_locations_.size()) { + size_t num_adrp_thunks = adrp_thunk_locations_.size() - reserved_adrp_thunks_; + offset = CompiledMethod::AlignCode(offset, kArm64) + kAdrpThunkSize * num_adrp_thunks; + reserved_adrp_thunks_ = adrp_thunk_locations_.size(); + } + + // Count the number of ADRP insns as the upper bound on the number of thunks needed + // and use it to reserve space for other linker patches. + size_t num_adrp = 0u; + if (LIKELY(compiled_method != nullptr)) { + for (const LinkerPatch& patch : compiled_method->GetPatches()) { + if (patch.Type() == kLinkerPatchDexCacheArray && + patch.LiteralOffset() == patch.PcInsnOffset()) { // ADRP patch + ++num_adrp; + } + } + } + offset = ReserveSpaceInternal(offset, compiled_method, kAdrpThunkSize * num_adrp); + if (num_adrp == 0u) { + return offset; + } + + // Now that we have the actual offset where the code will be placed, locate the ADRP insns + // that actually require the thunk. + uint32_t quick_code_offset = compiled_method->AlignCode(offset) + sizeof(OatQuickMethodHeader); + ArrayRef code(*compiled_method->GetQuickCode()); + uint32_t thunk_offset = compiled_method->AlignCode(quick_code_offset + code.size()); + DCHECK(compiled_method != nullptr); + for (const LinkerPatch& patch : compiled_method->GetPatches()) { + if (patch.Type() == kLinkerPatchDexCacheArray && + patch.LiteralOffset() == patch.PcInsnOffset()) { // ADRP patch + uint32_t patch_offset = quick_code_offset + patch.LiteralOffset(); + if (NeedsErratum843419Thunk(code, patch.LiteralOffset(), patch_offset)) { + adrp_thunk_locations_.emplace_back(patch_offset, thunk_offset); + thunk_offset += kAdrpThunkSize; + } + } + } + return offset; +} + +uint32_t Arm64RelativePatcher::WriteThunks(OutputStream* out, uint32_t offset) { + if (fix_cortex_a53_843419_) { + if (!current_method_thunks_.empty()) { + uint32_t aligned_offset = CompiledMethod::AlignCode(offset, kArm64); + if (kIsDebugBuild) { + CHECK(IsAligned(current_method_thunks_.size())); + size_t num_thunks = current_method_thunks_.size() / kAdrpThunkSize; + CHECK_LE(num_thunks, processed_adrp_thunks_); + for (size_t i = 0u; i != num_thunks; ++i) { + const auto& entry = adrp_thunk_locations_[processed_adrp_thunks_ - num_thunks + i]; + CHECK_EQ(entry.second, aligned_offset + i * kAdrpThunkSize); + } + } + uint32_t aligned_code_delta = aligned_offset - offset; + if (aligned_code_delta != 0u && !WriteCodeAlignment(out, aligned_code_delta)) { + return 0u; + } + if (!WriteMiscThunk(out, ArrayRef(current_method_thunks_))) { + return 0u; + } + offset = aligned_offset + current_method_thunks_.size(); + current_method_thunks_.clear(); + } + } + return ArmBaseRelativePatcher::WriteThunks(out, offset); +} + +void Arm64RelativePatcher::PatchCall(std::vector* code, uint32_t literal_offset, + uint32_t patch_offset, uint32_t target_offset) { + DCHECK_LE(literal_offset + 4u, code->size()); + DCHECK_EQ(literal_offset & 3u, 0u); + DCHECK_EQ(patch_offset & 3u, 0u); + DCHECK_EQ(target_offset & 3u, 0u); + uint32_t displacement = CalculateDisplacement(patch_offset, target_offset & ~1u); + DCHECK_EQ(displacement & 3u, 0u); + DCHECK((displacement >> 27) == 0u || (displacement >> 27) == 31u); // 28-bit signed. + uint32_t insn = (displacement & 0x0fffffffu) >> 2; + insn |= 0x94000000; // BL + + // Check that we're just overwriting an existing BL. + DCHECK_EQ(GetInsn(code, literal_offset) & 0xfc000000u, 0x94000000u); + // Write the new BL. + SetInsn(code, literal_offset, insn); +} + +void Arm64RelativePatcher::PatchDexCacheReference(std::vector* code, + const LinkerPatch& patch, + uint32_t patch_offset, + uint32_t target_offset) { + DCHECK_EQ(patch_offset & 3u, 0u); + DCHECK_EQ(target_offset & 3u, 0u); + uint32_t literal_offset = patch.LiteralOffset(); + uint32_t insn = GetInsn(code, literal_offset); + uint32_t pc_insn_offset = patch.PcInsnOffset(); + uint32_t disp = target_offset - ((patch_offset - literal_offset + pc_insn_offset) & ~0xfffu); + if (literal_offset == pc_insn_offset) { + // Check it's an ADRP with imm == 0 (unset). + DCHECK_EQ((insn & 0xffffffe0u), 0x90000000u) + << literal_offset << ", " << pc_insn_offset << ", 0x" << std::hex << insn; + if (fix_cortex_a53_843419_ && processed_adrp_thunks_ != adrp_thunk_locations_.size() && + adrp_thunk_locations_[processed_adrp_thunks_].first == patch_offset) { + DCHECK(NeedsErratum843419Thunk(ArrayRef(*code), + literal_offset, patch_offset)); + uint32_t thunk_offset = adrp_thunk_locations_[processed_adrp_thunks_].second; + uint32_t adrp_disp = target_offset - (thunk_offset & ~0xfffu); + uint32_t adrp = PatchAdrp(insn, adrp_disp); + + uint32_t out_disp = thunk_offset - patch_offset; + DCHECK_EQ(out_disp & 3u, 0u); + DCHECK((out_disp >> 27) == 0u || (out_disp >> 27) == 31u); // 28-bit signed. + insn = (out_disp & 0x0fffffffu) >> 2; + insn |= 0x14000000; // B + + uint32_t back_disp = -out_disp; + DCHECK_EQ(back_disp & 3u, 0u); + DCHECK((back_disp >> 27) == 0u || (back_disp >> 27) == 31u); // 28-bit signed. + uint32_t b_back = (back_disp & 0x0fffffffu) >> 2; + b_back |= 0x14000000; // B + size_t thunks_code_offset = current_method_thunks_.size(); + current_method_thunks_.resize(thunks_code_offset + kAdrpThunkSize); + SetInsn(¤t_method_thunks_, thunks_code_offset, adrp); + SetInsn(¤t_method_thunks_, thunks_code_offset + 4u, b_back); + static_assert(kAdrpThunkSize == 2 * 4u, "thunk has 2 instructions"); + + processed_adrp_thunks_ += 1u; + } else { + insn = PatchAdrp(insn, disp); + } + // Write the new ADRP (or B to the erratum 843419 thunk). + SetInsn(code, literal_offset, insn); + } else { + DCHECK_EQ(insn & 0xfffffc00, 0xb9400000); // LDR 32-bit with imm12 == 0 (unset). + if (kIsDebugBuild) { + uint32_t adrp = GetInsn(code, pc_insn_offset); + if ((adrp & 0x9f000000u) != 0x90000000u) { + CHECK(fix_cortex_a53_843419_); + CHECK_EQ(adrp & 0xfc000000u, 0x14000000u); // B + CHECK(IsAligned(current_method_thunks_.size())); + size_t num_thunks = current_method_thunks_.size() / kAdrpThunkSize; + CHECK_LE(num_thunks, processed_adrp_thunks_); + uint32_t b_offset = patch_offset - literal_offset + pc_insn_offset; + for (size_t i = processed_adrp_thunks_ - num_thunks; ; ++i) { + CHECK_NE(i, processed_adrp_thunks_); + if (adrp_thunk_locations_[i].first == b_offset) { + size_t idx = num_thunks - (processed_adrp_thunks_ - i); + adrp = GetInsn(¤t_method_thunks_, idx * kAdrpThunkSize); + break; + } + } + } + CHECK_EQ(adrp & 0x9f00001fu, // Check that pc_insn_offset points + 0x90000000 | ((insn >> 5) & 0x1fu)); // to ADRP with matching register. + } + uint32_t imm12 = (disp & 0xfffu) >> 2; + insn = (insn & ~(0xfffu << 10)) | (imm12 << 10); + SetInsn(code, literal_offset, insn); + } +} + +std::vector Arm64RelativePatcher::CompileThunkCode() { + // The thunk just uses the entry point in the ArtMethod. This works even for calls + // to the generic JNI and interpreter trampolines. + arm64::Arm64Assembler assembler; + Offset offset(mirror::ArtMethod::EntryPointFromQuickCompiledCodeOffset( + kArm64PointerSize).Int32Value()); + assembler.JumpTo(ManagedRegister(arm64::X0), offset, ManagedRegister(arm64::IP0)); + // Ensure we emit the literal pool. + assembler.EmitSlowPaths(); + std::vector thunk_code(assembler.CodeSize()); + MemoryRegion code(thunk_code.data(), thunk_code.size()); + assembler.FinalizeInstructions(code); + return thunk_code; +} + +uint32_t Arm64RelativePatcher::PatchAdrp(uint32_t adrp, uint32_t disp) { + return (adrp & 0x9f00001fu) | // Clear offset bits, keep ADRP with destination reg. + // Bottom 12 bits are ignored, the next 2 lowest bits are encoded in bits 29-30. + ((disp & 0x00003000u) << (29 - 12)) | + // The next 16 bits are encoded in bits 5-22. + ((disp & 0xffffc000u) >> (12 + 2 - 5)) | + // Since the target_offset is based on the beginning of the oat file and the + // image space precedes the oat file, the target_offset into image space will + // be negative yet passed as uint32_t. Therefore we limit the displacement + // to +-2GiB (rather than the maximim +-4GiB) and determine the sign bit from + // the highest bit of the displacement. This is encoded in bit 23. + ((disp & 0x80000000u) >> (31 - 23)); +} + +bool Arm64RelativePatcher::NeedsErratum843419Thunk(ArrayRef code, + uint32_t literal_offset, + uint32_t patch_offset) { + DCHECK_EQ(patch_offset & 0x3u, 0u); + if ((patch_offset & 0xff8) == 0xff8) { // ...ff8 or ...ffc + uint32_t adrp = GetInsn(code, literal_offset); + DCHECK_EQ(adrp & 0xff000000, 0x90000000); + // TODO: Improve the check. For now, we're just checking if the next insn is + // the LDR using the result of the ADRP, otherwise we implement the workaround. + uint32_t next_insn = GetInsn(code, literal_offset + 4u); + bool ok = (next_insn & 0xffc00000) == 0xb9400000 && // LDR , [, #pimm] + (((next_insn >> 5) ^ adrp) & 0x1f) == 0; // == ADRP destination reg + return !ok; + } + return false; +} + +void Arm64RelativePatcher::SetInsn(std::vector* code, uint32_t offset, uint32_t value) { + DCHECK_LE(offset + 4u, code->size()); + DCHECK_EQ(offset & 3u, 0u); + uint8_t* addr = &(*code)[offset]; + addr[0] = (value >> 0) & 0xff; + addr[1] = (value >> 8) & 0xff; + addr[2] = (value >> 16) & 0xff; + addr[3] = (value >> 24) & 0xff; +} + +uint32_t Arm64RelativePatcher::GetInsn(ArrayRef code, uint32_t offset) { + DCHECK_LE(offset + 4u, code.size()); + DCHECK_EQ(offset & 3u, 0u); + const uint8_t* addr = &code[offset]; + return + (static_cast(addr[0]) << 0) + + (static_cast(addr[1]) << 8) + + (static_cast(addr[2]) << 16)+ + (static_cast(addr[3]) << 24); +} + +template +uint32_t Arm64RelativePatcher::GetInsn(std::vector* code, uint32_t offset) { + return GetInsn(ArrayRef(*code), offset); +} + +} // namespace linker +} // namespace art diff --git a/compiler/linker/arm64/relative_patcher_arm64.h b/compiler/linker/arm64/relative_patcher_arm64.h new file mode 100644 index 000000000..f8c6802f4 --- /dev/null +++ b/compiler/linker/arm64/relative_patcher_arm64.h @@ -0,0 +1,72 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_COMPILER_LINKER_ARM64_RELATIVE_PATCHER_ARM64_H_ +#define ART_COMPILER_LINKER_ARM64_RELATIVE_PATCHER_ARM64_H_ + +#include "linker/arm/relative_patcher_arm_base.h" +#include "utils/array_ref.h" + +namespace art { +namespace linker { + +class Arm64RelativePatcher FINAL : public ArmBaseRelativePatcher { + public: + Arm64RelativePatcher(RelativePatcherTargetProvider* provider, + const Arm64InstructionSetFeatures* features); + + uint32_t ReserveSpace(uint32_t offset, const CompiledMethod* compiled_method) OVERRIDE; + uint32_t WriteThunks(OutputStream* out, uint32_t offset) OVERRIDE; + void PatchCall(std::vector* code, uint32_t literal_offset, + uint32_t patch_offset, uint32_t target_offset) OVERRIDE; + void PatchDexCacheReference(std::vector* code, const LinkerPatch& patch, + uint32_t patch_offset, uint32_t target_offset) OVERRIDE; + + private: + static std::vector CompileThunkCode(); + static uint32_t PatchAdrp(uint32_t adrp, uint32_t disp); + + static bool NeedsErratum843419Thunk(ArrayRef code, uint32_t literal_offset, + uint32_t patch_offset); + void SetInsn(std::vector* code, uint32_t offset, uint32_t value); + static uint32_t GetInsn(ArrayRef code, uint32_t offset); + + template + static uint32_t GetInsn(std::vector* code, uint32_t offset); + + // Maximum positive and negative displacement measured from the patch location. + // (Signed 28 bit displacement with the last bit 0 has range [-2^27, 2^27-4] measured from + // the ARM64 PC pointing to the BL.) + static constexpr uint32_t kMaxPositiveDisplacement = (1u << 27) - 4u; + static constexpr uint32_t kMaxNegativeDisplacement = (1u << 27); + + // The ADRP thunk for erratum 843419 is 2 instructions, i.e. 8 bytes. + static constexpr uint32_t kAdrpThunkSize = 8u; + + const bool fix_cortex_a53_843419_; + // Map original patch_offset to thunk offset. + std::vector> adrp_thunk_locations_; + size_t reserved_adrp_thunks_; + size_t processed_adrp_thunks_; + std::vector current_method_thunks_; + + DISALLOW_COPY_AND_ASSIGN(Arm64RelativePatcher); +}; + +} // namespace linker +} // namespace art + +#endif // ART_COMPILER_LINKER_ARM64_RELATIVE_PATCHER_ARM64_H_ diff --git a/compiler/linker/relative_patcher.cc b/compiler/linker/relative_patcher.cc new file mode 100644 index 000000000..ed82ac294 --- /dev/null +++ b/compiler/linker/relative_patcher.cc @@ -0,0 +1,113 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "linker/relative_patcher.h" + +#include "linker/arm/relative_patcher_thumb2.h" +#include "linker/arm64/relative_patcher_arm64.h" +#include "linker/x86/relative_patcher_x86.h" +#include "linker/x86_64/relative_patcher_x86_64.h" +#include "output_stream.h" + +namespace art { +namespace linker { + +std::unique_ptr RelativePatcher::Create( + InstructionSet instruction_set, const InstructionSetFeatures* features, + RelativePatcherTargetProvider* provider) { + class RelativePatcherNone FINAL : public RelativePatcher { + public: + RelativePatcherNone() { } + + uint32_t ReserveSpace(uint32_t offset, + const CompiledMethod* compiled_method ATTRIBUTE_UNUSED) OVERRIDE { + return offset; // No space reserved; no patches expected. + } + + uint32_t WriteThunks(OutputStream* out ATTRIBUTE_UNUSED, uint32_t offset) OVERRIDE { + return offset; // No thunks added; no patches expected. + } + + void PatchCall(std::vector* code ATTRIBUTE_UNUSED, + uint32_t literal_offset ATTRIBUTE_UNUSED, + uint32_t patch_offset ATTRIBUTE_UNUSED, + uint32_t target_offset ATTRIBUTE_UNUSED) OVERRIDE { + LOG(FATAL) << "Unexpected relative call patch."; + } + + virtual void PatchDexCacheReference(std::vector* code ATTRIBUTE_UNUSED, + const LinkerPatch& patch ATTRIBUTE_UNUSED, + uint32_t patch_offset ATTRIBUTE_UNUSED, + uint32_t target_offset ATTRIBUTE_UNUSED) { + LOG(FATAL) << "Unexpected relative dex cache array patch."; + } + + private: + DISALLOW_COPY_AND_ASSIGN(RelativePatcherNone); + }; + + switch (instruction_set) { + case kX86: + return std::unique_ptr(new X86RelativePatcher()); + break; + case kX86_64: + return std::unique_ptr(new X86_64RelativePatcher()); + break; + case kArm: + // Fall through: we generate Thumb2 code for "arm". + case kThumb2: + return std::unique_ptr(new Thumb2RelativePatcher(provider)); + break; + case kArm64: + return std::unique_ptr( + new Arm64RelativePatcher(provider, features->AsArm64InstructionSetFeatures())); + break; + default: + return std::unique_ptr(new RelativePatcherNone); + break; + } +} + +bool RelativePatcher::WriteCodeAlignment(OutputStream* out, uint32_t aligned_code_delta) { + static const uint8_t kPadding[] = { + 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u + }; + DCHECK_LE(aligned_code_delta, sizeof(kPadding)); + if (UNLIKELY(!out->WriteFully(kPadding, aligned_code_delta))) { + return false; + } + size_code_alignment_ += aligned_code_delta; + return true; +} + +bool RelativePatcher::WriteRelCallThunk(OutputStream* out, const ArrayRef& thunk) { + if (UNLIKELY(!out->WriteFully(thunk.data(), thunk.size()))) { + return false; + } + size_relative_call_thunks_ += thunk.size(); + return true; +} + +bool RelativePatcher::WriteMiscThunk(OutputStream* out, const ArrayRef& thunk) { + if (UNLIKELY(!out->WriteFully(thunk.data(), thunk.size()))) { + return false; + } + size_misc_thunks_ += thunk.size(); + return true; +} + +} // namespace linker +} // namespace art diff --git a/compiler/linker/relative_patcher.h b/compiler/linker/relative_patcher.h new file mode 100644 index 000000000..dbb18cc91 --- /dev/null +++ b/compiler/linker/relative_patcher.h @@ -0,0 +1,123 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_COMPILER_LINKER_RELATIVE_PATCHER_H_ +#define ART_COMPILER_LINKER_RELATIVE_PATCHER_H_ + +#include + +#include "arch/instruction_set.h" +#include "arch/instruction_set_features.h" +#include "base/macros.h" +#include "method_reference.h" +#include "utils/array_ref.h" + +namespace art { + +class CompiledMethod; +class LinkerPatch; +class OutputStream; + +namespace linker { + +/** + * @class RelativePatcherTargetProvider + * @brief Interface for providing method offsets for relative call targets. + */ +class RelativePatcherTargetProvider { + public: + /** + * Find the offset of the target method of a relative call if known. + * + * The process of assigning target method offsets includes calls to the relative patcher's + * ReserveSpace() which in turn can use FindMethodOffset() to determine if a method already + * has an offset assigned and, if so, what's that offset. If the offset has not yet been + * assigned or if it's too far for the particular architecture's relative call, + * ReserveSpace() may need to allocate space for a special dispatch thunk. + * + * @param ref the target method of the relative call. + * @return true in the first element of the pair if the method was found, false otherwise; + * if found, the second element specifies the offset. + */ + virtual std::pair FindMethodOffset(MethodReference ref) = 0; + + protected: + virtual ~RelativePatcherTargetProvider() { } +}; + +/** + * @class RelativePatcher + * @brief Interface for architecture-specific link-time patching of PC-relative references. + */ +class RelativePatcher { + public: + static std::unique_ptr Create( + InstructionSet instruction_set, const InstructionSetFeatures* features, + RelativePatcherTargetProvider* provider); + + virtual ~RelativePatcher() { } + + uint32_t CodeAlignmentSize() const { + return size_code_alignment_; + } + + uint32_t RelativeCallThunksSize() const { + return size_relative_call_thunks_; + } + + uint32_t MiscThunksSize() const { + return size_misc_thunks_; + } + + // Reserve space for relative call thunks if needed, return adjusted offset. After all methods + // of a class have been processed it's called one last time with compiled_method == nullptr. + virtual uint32_t ReserveSpace(uint32_t offset, const CompiledMethod* compiled_method) = 0; + + // Write relative call thunks if needed, return adjusted offset. + virtual uint32_t WriteThunks(OutputStream* out, uint32_t offset) = 0; + + // Patch method code. The input displacement is relative to the patched location, + // the patcher may need to adjust it if the correct base is different. + virtual void PatchCall(std::vector* code, uint32_t literal_offset, + uint32_t patch_offset, uint32_t target_offset) = 0; + + // Patch a reference to a dex cache location. + virtual void PatchDexCacheReference(std::vector* code, const LinkerPatch& patch, + uint32_t patch_offset, uint32_t target_offset) = 0; + + protected: + RelativePatcher() + : size_code_alignment_(0u), + size_relative_call_thunks_(0u), + size_misc_thunks_(0u) { + } + + bool WriteCodeAlignment(OutputStream* out, uint32_t aligned_code_delta); + bool WriteRelCallThunk(OutputStream* out, const ArrayRef& thunk); + bool WriteMiscThunk(OutputStream* out, const ArrayRef& thunk); + + private: + uint32_t size_code_alignment_; + uint32_t size_relative_call_thunks_; + uint32_t size_misc_thunks_; + + DISALLOW_COPY_AND_ASSIGN(RelativePatcher); +}; + +} // namespace linker +} // namespace art + +#endif // ART_COMPILER_LINKER_RELATIVE_PATCHER_H_ diff --git a/compiler/linker/relative_patcher_test.h b/compiler/linker/relative_patcher_test.h new file mode 100644 index 000000000..515355be0 --- /dev/null +++ b/compiler/linker/relative_patcher_test.h @@ -0,0 +1,237 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_COMPILER_LINKER_RELATIVE_PATCHER_TEST_H_ +#define ART_COMPILER_LINKER_RELATIVE_PATCHER_TEST_H_ + +#include "arch/instruction_set.h" +#include "arch/instruction_set_features.h" +#include "base/macros.h" +#include "compiled_method.h" +#include "dex/quick/dex_file_to_method_inliner_map.h" +#include "dex/verification_results.h" +#include "driver/compiler_driver.h" +#include "driver/compiler_options.h" +#include "globals.h" +#include "gtest/gtest.h" +#include "linker/relative_patcher.h" +#include "method_reference.h" +#include "oat.h" +#include "utils/array_ref.h" +#include "vector_output_stream.h" + +namespace art { +namespace linker { + +// Base class providing infrastructure for architecture-specific tests. +class RelativePatcherTest : public testing::Test { + protected: + RelativePatcherTest(InstructionSet instruction_set, const std::string& variant) + : compiler_options_(), + verification_results_(&compiler_options_), + inliner_map_(), + driver_(&compiler_options_, &verification_results_, &inliner_map_, + Compiler::kQuick, instruction_set, nullptr, + false, nullptr, nullptr, 1u, + false, false, "", nullptr, -1, ""), + error_msg_(), + instruction_set_(instruction_set), + features_(InstructionSetFeatures::FromVariant(instruction_set, variant, &error_msg_)), + method_offset_map_(), + patcher_(RelativePatcher::Create(instruction_set, features_.get(), &method_offset_map_)), + dex_cache_arrays_begin_(0u), + compiled_method_refs_(), + compiled_methods_(), + patched_code_(), + output_(), + out_("test output stream", &output_) { + CHECK(error_msg_.empty()) << instruction_set << "/" << variant; + patched_code_.reserve(16 * KB); + } + + MethodReference MethodRef(uint32_t method_idx) { + return MethodReference(nullptr, method_idx); + } + + void AddCompiledMethod(MethodReference method_ref, + const ArrayRef& code, + const ArrayRef& patches) { + compiled_method_refs_.push_back(method_ref); + compiled_methods_.emplace_back(new CompiledMethod( + &driver_, instruction_set_, code, + 0u, 0u, 0u, nullptr, ArrayRef(), ArrayRef(), + ArrayRef(), ArrayRef(), + patches)); + } + + void Link() { + // Reserve space. + static_assert(kTrampolineOffset == 0u, "Unexpected trampoline offset."); + uint32_t offset = kTrampolineSize; + size_t idx = 0u; + for (auto& compiled_method : compiled_methods_) { + offset = patcher_->ReserveSpace(offset, compiled_method.get()); + + uint32_t aligned_offset = compiled_method->AlignCode(offset); + uint32_t aligned_code_delta = aligned_offset - offset; + offset += aligned_code_delta; + + offset += sizeof(OatQuickMethodHeader); + uint32_t quick_code_offset = offset + compiled_method->CodeDelta(); + const auto& code = *compiled_method->GetQuickCode(); + offset += code.size(); + + method_offset_map_.map.Put(compiled_method_refs_[idx], quick_code_offset); + ++idx; + } + offset = patcher_->ReserveSpace(offset, nullptr); + uint32_t output_size = offset; + output_.reserve(output_size); + + // Write data. + DCHECK(output_.empty()); + uint8_t dummy_trampoline[kTrampolineSize]; + memset(dummy_trampoline, 0, sizeof(dummy_trampoline)); + out_.WriteFully(dummy_trampoline, kTrampolineSize); + offset = kTrampolineSize; + static const uint8_t kPadding[] = { + 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u, 0u + }; + uint8_t dummy_header[sizeof(OatQuickMethodHeader)]; + memset(dummy_header, 0, sizeof(dummy_header)); + for (auto& compiled_method : compiled_methods_) { + offset = patcher_->WriteThunks(&out_, offset); + + uint32_t aligned_offset = compiled_method->AlignCode(offset); + uint32_t aligned_code_delta = aligned_offset - offset; + CHECK_LE(aligned_code_delta, sizeof(kPadding)); + out_.WriteFully(kPadding, aligned_code_delta); + offset += aligned_code_delta; + + out_.WriteFully(dummy_header, sizeof(OatQuickMethodHeader)); + offset += sizeof(OatQuickMethodHeader); + ArrayRef code(*compiled_method->GetQuickCode()); + if (!compiled_method->GetPatches().empty()) { + patched_code_.assign(code.begin(), code.end()); + code = ArrayRef(patched_code_); + for (const LinkerPatch& patch : compiled_method->GetPatches()) { + if (patch.Type() == kLinkerPatchCallRelative) { + auto result = method_offset_map_.FindMethodOffset(patch.TargetMethod()); + uint32_t target_offset = + result.first ? result.second : kTrampolineOffset + compiled_method->CodeDelta(); + patcher_->PatchCall(&patched_code_, patch.LiteralOffset(), + offset + patch.LiteralOffset(), target_offset); + } else if (patch.Type() == kLinkerPatchDexCacheArray) { + uint32_t target_offset = dex_cache_arrays_begin_ + patch.TargetDexCacheElementOffset(); + patcher_->PatchDexCacheReference(&patched_code_, patch, + offset + patch.LiteralOffset(), target_offset); + } else { + LOG(FATAL) << "Bad patch type."; + } + } + } + out_.WriteFully(&code[0], code.size()); + offset += code.size(); + } + offset = patcher_->WriteThunks(&out_, offset); + CHECK_EQ(offset, output_size); + CHECK_EQ(output_.size(), output_size); + } + + bool CheckLinkedMethod(MethodReference method_ref, const ArrayRef& expected_code) { + // Sanity check: original code size must match linked_code.size(). + size_t idx = 0u; + for (auto ref : compiled_method_refs_) { + if (ref.dex_file == method_ref.dex_file && + ref.dex_method_index == method_ref.dex_method_index) { + break; + } + ++idx; + } + CHECK_NE(idx, compiled_method_refs_.size()); + CHECK_EQ(compiled_methods_[idx]->GetQuickCode()->size(), expected_code.size()); + + auto result = method_offset_map_.FindMethodOffset(method_ref); + CHECK(result.first); // Must have been linked. + size_t offset = result.second; + CHECK_LT(offset, output_.size()); + CHECK_LE(offset + expected_code.size(), output_.size()); + ArrayRef linked_code(&output_[offset], expected_code.size()); + if (linked_code == expected_code) { + return true; + } + // Log failure info. + std::ostringstream expected_hex; + std::ostringstream linked_hex; + std::ostringstream diff_indicator; + static const char digits[] = "0123456789abcdef"; + bool found_diff = false; + for (size_t i = 0; i != expected_code.size(); ++i) { + expected_hex << " " << digits[expected_code[i] >> 4] << digits[expected_code[i] & 0xf]; + linked_hex << " " << digits[linked_code[i] >> 4] << digits[linked_code[i] & 0xf]; + diff_indicator << " "; + if (!found_diff) { + found_diff = (expected_code[i] != linked_code[i]); + diff_indicator << (found_diff ? "^^" : " "); + } + } + CHECK(found_diff); + LOG(ERROR) << "diff expected_code linked_code"; + LOG(ERROR) << "<" << expected_hex.str(); + LOG(ERROR) << ">" << linked_hex.str(); + LOG(ERROR) << " " << diff_indicator.str(); + return false; + } + + // Map method reference to assinged offset. + // Wrap the map in a class implementing linker::RelativePatcherTargetProvider. + class MethodOffsetMap FINAL : public linker::RelativePatcherTargetProvider { + public: + std::pair FindMethodOffset(MethodReference ref) OVERRIDE { + auto it = map.find(ref); + if (it == map.end()) { + return std::pair(false, 0u); + } else { + return std::pair(true, it->second); + } + } + SafeMap map; + }; + + static const uint32_t kTrampolineSize = 4u; + static const uint32_t kTrampolineOffset = 0u; + + CompilerOptions compiler_options_; + VerificationResults verification_results_; + DexFileToMethodInlinerMap inliner_map_; + CompilerDriver driver_; // Needed for constructing CompiledMethod. + std::string error_msg_; + InstructionSet instruction_set_; + std::unique_ptr features_; + MethodOffsetMap method_offset_map_; + std::unique_ptr patcher_; + uint32_t dex_cache_arrays_begin_; + std::vector compiled_method_refs_; + std::vector> compiled_methods_; + std::vector patched_code_; + std::vector output_; + VectorOutputStream out_; +}; + +} // namespace linker +} // namespace art + +#endif // ART_COMPILER_LINKER_RELATIVE_PATCHER_TEST_H_ diff --git a/compiler/linker/x86/relative_patcher_x86.cc b/compiler/linker/x86/relative_patcher_x86.cc new file mode 100644 index 000000000..246cf11da --- /dev/null +++ b/compiler/linker/x86/relative_patcher_x86.cc @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "linker/x86/relative_patcher_x86.h" + +namespace art { +namespace linker { + +void X86RelativePatcher::PatchDexCacheReference(std::vector* code ATTRIBUTE_UNUSED, + const LinkerPatch& patch ATTRIBUTE_UNUSED, + uint32_t patch_offset ATTRIBUTE_UNUSED, + uint32_t target_offset ATTRIBUTE_UNUSED) { + LOG(FATAL) << "Unexpected relative dex cache array patch."; +} + +} // namespace linker +} // namespace art diff --git a/compiler/linker/x86/relative_patcher_x86.h b/compiler/linker/x86/relative_patcher_x86.h new file mode 100644 index 000000000..0c881f00b --- /dev/null +++ b/compiler/linker/x86/relative_patcher_x86.h @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_COMPILER_LINKER_X86_RELATIVE_PATCHER_X86_H_ +#define ART_COMPILER_LINKER_X86_RELATIVE_PATCHER_X86_H_ + +#include "linker/x86/relative_patcher_x86_base.h" + +namespace art { +namespace linker { + +class X86RelativePatcher FINAL : public X86BaseRelativePatcher { + public: + X86RelativePatcher() { } + + void PatchDexCacheReference(std::vector* code, const LinkerPatch& patch, + uint32_t patch_offset, uint32_t target_offset) OVERRIDE; +}; + +} // namespace linker +} // namespace art + +#endif // ART_COMPILER_LINKER_X86_RELATIVE_PATCHER_X86_H_ diff --git a/compiler/linker/x86/relative_patcher_x86_base.cc b/compiler/linker/x86/relative_patcher_x86_base.cc new file mode 100644 index 000000000..3108ca7b8 --- /dev/null +++ b/compiler/linker/x86/relative_patcher_x86_base.cc @@ -0,0 +1,43 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "linker/x86/relative_patcher_x86_base.h" + +namespace art { +namespace linker { + +uint32_t X86BaseRelativePatcher::ReserveSpace( + uint32_t offset, const CompiledMethod* compiled_method ATTRIBUTE_UNUSED) { + return offset; // No space reserved; no limit on relative call distance. +} + +uint32_t X86BaseRelativePatcher::WriteThunks(OutputStream* out ATTRIBUTE_UNUSED, uint32_t offset) { + return offset; // No thunks added; no limit on relative call distance. +} + +void X86BaseRelativePatcher::PatchCall(std::vector* code, uint32_t literal_offset, + uint32_t patch_offset, uint32_t target_offset) { + DCHECK_LE(literal_offset + 4u, code->size()); + // Unsigned arithmetic with its well-defined overflow behavior is just fine here. + uint32_t displacement = target_offset - patch_offset; + displacement -= kPcDisplacement; // The base PC is at the end of the 4-byte patch. + + typedef __attribute__((__aligned__(1))) int32_t unaligned_int32_t; + reinterpret_cast(&(*code)[literal_offset])[0] = displacement; +} + +} // namespace linker +} // namespace art diff --git a/compiler/linker/x86/relative_patcher_x86_base.h b/compiler/linker/x86/relative_patcher_x86_base.h new file mode 100644 index 000000000..9e1fc0c71 --- /dev/null +++ b/compiler/linker/x86/relative_patcher_x86_base.h @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_COMPILER_LINKER_X86_RELATIVE_PATCHER_X86_BASE_H_ +#define ART_COMPILER_LINKER_X86_RELATIVE_PATCHER_X86_BASE_H_ + +#include "linker/relative_patcher.h" + +namespace art { +namespace linker { + +class X86BaseRelativePatcher : public RelativePatcher { + public: + uint32_t ReserveSpace(uint32_t offset, + const CompiledMethod* compiled_method ATTRIBUTE_UNUSED) OVERRIDE; + uint32_t WriteThunks(OutputStream* out ATTRIBUTE_UNUSED, uint32_t offset) OVERRIDE; + void PatchCall(std::vector* code, uint32_t literal_offset, + uint32_t patch_offset, uint32_t target_offset) OVERRIDE; + + protected: + X86BaseRelativePatcher() { } + + // PC displacement from patch location; the base address of x86/x86-64 relative + // calls and x86-64 RIP-relative addressing is the PC of the next instruction and + // the patch location is 4 bytes earlier. + static constexpr int32_t kPcDisplacement = 4; + + private: + DISALLOW_COPY_AND_ASSIGN(X86BaseRelativePatcher); +}; + +} // namespace linker +} // namespace art + +#endif // ART_COMPILER_LINKER_X86_RELATIVE_PATCHER_X86_BASE_H_ diff --git a/compiler/linker/x86/relative_patcher_x86_test.cc b/compiler/linker/x86/relative_patcher_x86_test.cc new file mode 100644 index 000000000..f471c835f --- /dev/null +++ b/compiler/linker/x86/relative_patcher_x86_test.cc @@ -0,0 +1,90 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "linker/relative_patcher_test.h" +#include "linker/x86/relative_patcher_x86.h" + +namespace art { +namespace linker { + +class X86RelativePatcherTest : public RelativePatcherTest { + public: + X86RelativePatcherTest() : RelativePatcherTest(kX86, "default") { } + + protected: + static const uint8_t kCallRawCode[]; + static const ArrayRef kCallCode; +}; + +const uint8_t X86RelativePatcherTest::kCallRawCode[] = { + 0xe8, 0x00, 0x01, 0x00, 0x00 +}; + +const ArrayRef X86RelativePatcherTest::kCallCode(kCallRawCode); + +TEST_F(X86RelativePatcherTest, CallSelf) { + LinkerPatch patches[] = { + LinkerPatch::RelativeCodePatch(kCallCode.size() - 4u, nullptr, 0u), + }; + AddCompiledMethod(MethodRef(0u), kCallCode, ArrayRef(patches)); + Link(); + + static const uint8_t expected_code[] = { + 0xe8, 0xfb, 0xff, 0xff, 0xff + }; + EXPECT_TRUE(CheckLinkedMethod(MethodRef(0), ArrayRef(expected_code))); +} + +TEST_F(X86RelativePatcherTest, CallOther) { + static constexpr uint32_t kMethod1Offset = 0x12345678; + method_offset_map_.map.Put(MethodRef(1), kMethod1Offset); + LinkerPatch patches[] = { + LinkerPatch::RelativeCodePatch(kCallCode.size() - 4u, nullptr, 1u), + }; + AddCompiledMethod(MethodRef(0u), kCallCode, ArrayRef(patches)); + Link(); + + auto result = method_offset_map_.FindMethodOffset(MethodRef(0)); + ASSERT_TRUE(result.first); + uint32_t diff = kMethod1Offset - (result.second + kCallCode.size()); + static const uint8_t expected_code[] = { + 0xe8, + static_cast(diff), static_cast(diff >> 8), + static_cast(diff >> 16), static_cast(diff >> 24) + }; + EXPECT_TRUE(CheckLinkedMethod(MethodRef(0), ArrayRef(expected_code))); +} + +TEST_F(X86RelativePatcherTest, CallTrampoline) { + LinkerPatch patches[] = { + LinkerPatch::RelativeCodePatch(kCallCode.size() - 4u, nullptr, 1u), + }; + AddCompiledMethod(MethodRef(0u), kCallCode, ArrayRef(patches)); + Link(); + + auto result = method_offset_map_.FindMethodOffset(MethodRef(0)); + ASSERT_TRUE(result.first); + uint32_t diff = kTrampolineOffset - (result.second + kCallCode.size()); + static const uint8_t expected_code[] = { + 0xe8, + static_cast(diff), static_cast(diff >> 8), + static_cast(diff >> 16), static_cast(diff >> 24) + }; + EXPECT_TRUE(CheckLinkedMethod(MethodRef(0), ArrayRef(expected_code))); +} + +} // namespace linker +} // namespace art diff --git a/compiler/linker/x86_64/relative_patcher_x86_64.cc b/compiler/linker/x86_64/relative_patcher_x86_64.cc new file mode 100644 index 000000000..598f3ac4a --- /dev/null +++ b/compiler/linker/x86_64/relative_patcher_x86_64.cc @@ -0,0 +1,37 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "linker/x86_64/relative_patcher_x86_64.h" + +#include "compiled_method.h" + +namespace art { +namespace linker { + +void X86_64RelativePatcher::PatchDexCacheReference(std::vector* code, + const LinkerPatch& patch, + uint32_t patch_offset, uint32_t target_offset) { + DCHECK_LE(patch.LiteralOffset() + 4u, code->size()); + // Unsigned arithmetic with its well-defined overflow behavior is just fine here. + uint32_t displacement = target_offset - patch_offset; + displacement -= kPcDisplacement; // The base PC is at the end of the 4-byte patch. + + typedef __attribute__((__aligned__(1))) int32_t unaligned_int32_t; + reinterpret_cast(&(*code)[patch.LiteralOffset()])[0] = displacement; +} + +} // namespace linker +} // namespace art diff --git a/compiler/linker/x86_64/relative_patcher_x86_64.h b/compiler/linker/x86_64/relative_patcher_x86_64.h new file mode 100644 index 000000000..af687b4a2 --- /dev/null +++ b/compiler/linker/x86_64/relative_patcher_x86_64.h @@ -0,0 +1,36 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef ART_COMPILER_LINKER_X86_64_RELATIVE_PATCHER_X86_64_H_ +#define ART_COMPILER_LINKER_X86_64_RELATIVE_PATCHER_X86_64_H_ + +#include "linker/x86/relative_patcher_x86_base.h" + +namespace art { +namespace linker { + +class X86_64RelativePatcher FINAL : public X86BaseRelativePatcher { + public: + X86_64RelativePatcher() { } + + void PatchDexCacheReference(std::vector* code, const LinkerPatch& patch, + uint32_t patch_offset, uint32_t target_offset) OVERRIDE; +}; + +} // namespace linker +} // namespace art + +#endif // ART_COMPILER_LINKER_X86_64_RELATIVE_PATCHER_X86_64_H_ diff --git a/compiler/linker/x86_64/relative_patcher_x86_64_test.cc b/compiler/linker/x86_64/relative_patcher_x86_64_test.cc new file mode 100644 index 000000000..7ca2fa14e --- /dev/null +++ b/compiler/linker/x86_64/relative_patcher_x86_64_test.cc @@ -0,0 +1,121 @@ +/* + * Copyright (C) 2015 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "linker/relative_patcher_test.h" +#include "linker/x86_64/relative_patcher_x86_64.h" + +namespace art { +namespace linker { + +class X86_64RelativePatcherTest : public RelativePatcherTest { + public: + X86_64RelativePatcherTest() : RelativePatcherTest(kX86_64, "default") { } + + protected: + static const uint8_t kCallRawCode[]; + static const ArrayRef kCallCode; + static const uint8_t kDexCacheLoadRawCode[]; + static const ArrayRef kDexCacheLoadCode; +}; + +const uint8_t X86_64RelativePatcherTest::kCallRawCode[] = { + 0xe8, 0x00, 0x01, 0x00, 0x00 +}; + +const ArrayRef X86_64RelativePatcherTest::kCallCode(kCallRawCode); + +const uint8_t X86_64RelativePatcherTest::kDexCacheLoadRawCode[] = { + 0x8b, 0x05, // mov eax, [rip + ] + 0x00, 0x01, 0x00, 0x00 +}; + +const ArrayRef X86_64RelativePatcherTest::kDexCacheLoadCode( + kDexCacheLoadRawCode); + +TEST_F(X86_64RelativePatcherTest, CallSelf) { + LinkerPatch patches[] = { + LinkerPatch::RelativeCodePatch(kCallCode.size() - 4u, nullptr, 0u), + }; + AddCompiledMethod(MethodRef(0u), kCallCode, ArrayRef(patches)); + Link(); + + static const uint8_t expected_code[] = { + 0xe8, 0xfb, 0xff, 0xff, 0xff + }; + EXPECT_TRUE(CheckLinkedMethod(MethodRef(0), ArrayRef(expected_code))); +} + +TEST_F(X86_64RelativePatcherTest, CallOther) { + static constexpr uint32_t kMethod1Offset = 0x12345678; + method_offset_map_.map.Put(MethodRef(1), kMethod1Offset); + LinkerPatch patches[] = { + LinkerPatch::RelativeCodePatch(kCallCode.size() - 4u, nullptr, 1u), + }; + AddCompiledMethod(MethodRef(0u), kCallCode, ArrayRef(patches)); + Link(); + + auto result = method_offset_map_.FindMethodOffset(MethodRef(0)); + ASSERT_TRUE(result.first); + uint32_t diff = kMethod1Offset - (result.second + kCallCode.size()); + static const uint8_t expected_code[] = { + 0xe8, + static_cast(diff), static_cast(diff >> 8), + static_cast(diff >> 16), static_cast(diff >> 24) + }; + EXPECT_TRUE(CheckLinkedMethod(MethodRef(0), ArrayRef(expected_code))); +} + +TEST_F(X86_64RelativePatcherTest, CallTrampoline) { + LinkerPatch patches[] = { + LinkerPatch::RelativeCodePatch(kCallCode.size() - 4u, nullptr, 1u), + }; + AddCompiledMethod(MethodRef(0u), kCallCode, ArrayRef(patches)); + Link(); + + auto result = method_offset_map_.FindMethodOffset(MethodRef(0)); + ASSERT_TRUE(result.first); + uint32_t diff = kTrampolineOffset - (result.second + kCallCode.size()); + static const uint8_t expected_code[] = { + 0xe8, + static_cast(diff), static_cast(diff >> 8), + static_cast(diff >> 16), static_cast(diff >> 24) + }; + EXPECT_TRUE(CheckLinkedMethod(MethodRef(0), ArrayRef(expected_code))); +} + +TEST_F(X86_64RelativePatcherTest, DexCacheReference) { + dex_cache_arrays_begin_ = 0x12345678; + constexpr size_t kElementOffset = 0x1234; + LinkerPatch patches[] = { + LinkerPatch::DexCacheArrayPatch(kDexCacheLoadCode.size() - 4u, nullptr, 0u, kElementOffset), + }; + AddCompiledMethod(MethodRef(0u), kDexCacheLoadCode, ArrayRef(patches)); + Link(); + + auto result = method_offset_map_.FindMethodOffset(MethodRef(0)); + ASSERT_TRUE(result.first); + uint32_t diff = + dex_cache_arrays_begin_ + kElementOffset - (result.second + kDexCacheLoadCode.size()); + static const uint8_t expected_code[] = { + 0x8b, 0x05, + static_cast(diff), static_cast(diff >> 8), + static_cast(diff >> 16), static_cast(diff >> 24) + }; + EXPECT_TRUE(CheckLinkedMethod(MethodRef(0), ArrayRef(expected_code))); +} + +} // namespace linker +} // namespace art diff --git a/compiler/oat_writer.cc b/compiler/oat_writer.cc index 2c3f98240..d2a0f4fec 100644 --- a/compiler/oat_writer.cc +++ b/compiler/oat_writer.cc @@ -32,6 +32,7 @@ #include "driver/compiler_options.h" #include "gc/space/space.h" #include "image_writer.h" +#include "linker/relative_patcher.h" #include "mirror/art_method-inl.h" #include "mirror/array.h" #include "mirror/class_loader.h" @@ -41,632 +42,10 @@ #include "safe_map.h" #include "scoped_thread_state_change.h" #include "handle_scope-inl.h" -#include "utils/arm/assembler_thumb2.h" -#include "utils/arm64/assembler_arm64.h" #include "verifier/method_verifier.h" namespace art { -class OatWriter::RelativePatcher { - public: - virtual ~RelativePatcher() { } - - // Reserve space for relative call thunks if needed, return adjusted offset. After all methods - // of a class have been processed it's called one last time with compiled_method == nullptr. - virtual uint32_t ReserveSpace(uint32_t offset, const CompiledMethod* compiled_method) = 0; - - // Write relative call thunks if needed, return adjusted offset. - virtual uint32_t WriteThunks(OutputStream* out, uint32_t offset) = 0; - - // Patch method code. The input displacement is relative to the patched location, - // the patcher may need to adjust it if the correct base is different. - virtual void PatchCall(std::vector* code, uint32_t literal_offset, - uint32_t patch_offset, uint32_t target_offset) = 0; - - // Patch a reference to a dex cache location. - virtual void PatchDexCacheReference(std::vector* code, const LinkerPatch& patch, - uint32_t patch_offset, uint32_t target_offset) = 0; - - protected: - RelativePatcher() { } - - private: - DISALLOW_COPY_AND_ASSIGN(RelativePatcher); -}; - -class OatWriter::NoRelativePatcher FINAL : public RelativePatcher { - public: - NoRelativePatcher() { } - - uint32_t ReserveSpace(uint32_t offset, - const CompiledMethod* compiled_method ATTRIBUTE_UNUSED) OVERRIDE { - return offset; // No space reserved; no patches expected. - } - - uint32_t WriteThunks(OutputStream* out ATTRIBUTE_UNUSED, uint32_t offset) OVERRIDE { - return offset; // No thunks added; no patches expected. - } - - void PatchCall(std::vector* code ATTRIBUTE_UNUSED, - uint32_t literal_offset ATTRIBUTE_UNUSED, - uint32_t patch_offset ATTRIBUTE_UNUSED, - uint32_t target_offset ATTRIBUTE_UNUSED) OVERRIDE { - LOG(FATAL) << "Unexpected relative call patch."; - } - - virtual void PatchDexCacheReference(std::vector* code ATTRIBUTE_UNUSED, - const LinkerPatch& patch ATTRIBUTE_UNUSED, - uint32_t patch_offset ATTRIBUTE_UNUSED, - uint32_t target_offset ATTRIBUTE_UNUSED) { - LOG(FATAL) << "Unexpected relative dex cache array patch."; - } - - private: - DISALLOW_COPY_AND_ASSIGN(NoRelativePatcher); -}; - -class OatWriter::X86BaseRelativePatcher : public RelativePatcher { - public: - X86BaseRelativePatcher() { } - - uint32_t ReserveSpace(uint32_t offset, - const CompiledMethod* compiled_method ATTRIBUTE_UNUSED) OVERRIDE { - return offset; // No space reserved; no limit on relative call distance. - } - - uint32_t WriteThunks(OutputStream* out ATTRIBUTE_UNUSED, uint32_t offset) OVERRIDE { - return offset; // No thunks added; no limit on relative call distance. - } - - void PatchCall(std::vector* code, uint32_t literal_offset, - uint32_t patch_offset, uint32_t target_offset) OVERRIDE { - DCHECK_LE(literal_offset + 4u, code->size()); - // Unsigned arithmetic with its well-defined overflow behavior is just fine here. - uint32_t displacement = target_offset - patch_offset; - displacement -= kPcDisplacement; // The base PC is at the end of the 4-byte patch. - - typedef __attribute__((__aligned__(1))) int32_t unaligned_int32_t; - reinterpret_cast(&(*code)[literal_offset])[0] = displacement; - } - - protected: - // PC displacement from patch location; the base address of x86/x86-64 relative - // calls and x86-64 RIP-relative addressing is the PC of the next instruction and - // the patch location is 4 bytes earlier. - static constexpr int32_t kPcDisplacement = 4; - - private: - DISALLOW_COPY_AND_ASSIGN(X86BaseRelativePatcher); -}; - -class OatWriter::X86RelativePatcher FINAL : public X86BaseRelativePatcher { - public: - X86RelativePatcher() { } - - virtual void PatchDexCacheReference(std::vector* code ATTRIBUTE_UNUSED, - const LinkerPatch& patch ATTRIBUTE_UNUSED, - uint32_t patch_offset ATTRIBUTE_UNUSED, - uint32_t target_offset ATTRIBUTE_UNUSED) { - LOG(FATAL) << "Unexpected relative dex cache array patch."; - } -}; - -class OatWriter::X86_64RelativePatcher FINAL : public X86BaseRelativePatcher { - public: - X86_64RelativePatcher() { } - - virtual void PatchDexCacheReference(std::vector* code, const LinkerPatch& patch, - uint32_t patch_offset, uint32_t target_offset) { - DCHECK_LE(patch.LiteralOffset() + 4u, code->size()); - // Unsigned arithmetic with its well-defined overflow behavior is just fine here. - uint32_t displacement = target_offset - patch_offset; - displacement -= kPcDisplacement; // The base PC is at the end of the 4-byte patch. - - typedef __attribute__((__aligned__(1))) int32_t unaligned_int32_t; - reinterpret_cast(&(*code)[patch.LiteralOffset()])[0] = displacement; - } -}; - -class OatWriter::ArmBaseRelativePatcher : public RelativePatcher { - public: - ArmBaseRelativePatcher(OatWriter* writer, - InstructionSet instruction_set, std::vector thunk_code, - uint32_t max_positive_displacement, uint32_t max_negative_displacement) - : writer_(writer), instruction_set_(instruction_set), thunk_code_(thunk_code), - max_positive_displacement_(max_positive_displacement), - max_negative_displacement_(max_negative_displacement), - thunk_locations_(), current_thunk_to_write_(0u), unprocessed_patches_() { - } - - uint32_t ReserveSpace(uint32_t offset, const CompiledMethod* compiled_method) OVERRIDE { - return ReserveSpaceInternal(offset, compiled_method, 0u); - } - - uint32_t WriteThunks(OutputStream* out, uint32_t offset) OVERRIDE { - if (current_thunk_to_write_ == thunk_locations_.size()) { - return offset; - } - uint32_t aligned_offset = CompiledMethod::AlignCode(offset, instruction_set_); - if (UNLIKELY(aligned_offset == thunk_locations_[current_thunk_to_write_])) { - ++current_thunk_to_write_; - uint32_t aligned_code_delta = aligned_offset - offset; - if (aligned_code_delta != 0u && !writer_->WriteCodeAlignment(out, aligned_code_delta)) { - return 0u; - } - if (!out->WriteFully(thunk_code_.data(), thunk_code_.size())) { - return 0u; - } - writer_->size_relative_call_thunks_ += thunk_code_.size(); - uint32_t thunk_end_offset = aligned_offset + thunk_code_.size(); - // Align after writing chunk, see the ReserveSpace() above. - offset = CompiledMethod::AlignCode(thunk_end_offset, instruction_set_); - aligned_code_delta = offset - thunk_end_offset; - if (aligned_code_delta != 0u && !writer_->WriteCodeAlignment(out, aligned_code_delta)) { - return 0u; - } - } - return offset; - } - - protected: - uint32_t ReserveSpaceInternal(uint32_t offset, const CompiledMethod* compiled_method, - uint32_t max_extra_space) { - // NOTE: The final thunk can be reserved from InitCodeMethodVisitor::EndClass() while it - // may be written early by WriteCodeMethodVisitor::VisitMethod() for a deduplicated chunk - // of code. To avoid any alignment discrepancies for the final chunk, we always align the - // offset after reserving of writing any chunk. - if (UNLIKELY(compiled_method == nullptr)) { - uint32_t aligned_offset = CompiledMethod::AlignCode(offset, instruction_set_); - bool needs_thunk = ReserveSpaceProcessPatches(aligned_offset); - if (needs_thunk) { - thunk_locations_.push_back(aligned_offset); - offset = CompiledMethod::AlignCode(aligned_offset + thunk_code_.size(), instruction_set_); - } - return offset; - } - DCHECK(compiled_method->GetQuickCode() != nullptr); - uint32_t quick_code_size = compiled_method->GetQuickCode()->size(); - uint32_t quick_code_offset = compiled_method->AlignCode(offset) + sizeof(OatQuickMethodHeader); - uint32_t next_aligned_offset = compiled_method->AlignCode(quick_code_offset + quick_code_size); - // Adjust for extra space required by the subclass. - next_aligned_offset = compiled_method->AlignCode(next_aligned_offset + max_extra_space); - if (!unprocessed_patches_.empty() && - next_aligned_offset - unprocessed_patches_.front().second > max_positive_displacement_) { - bool needs_thunk = ReserveSpaceProcessPatches(next_aligned_offset); - if (needs_thunk) { - // A single thunk will cover all pending patches. - unprocessed_patches_.clear(); - uint32_t thunk_location = compiled_method->AlignCode(offset); - thunk_locations_.push_back(thunk_location); - offset = CompiledMethod::AlignCode(thunk_location + thunk_code_.size(), instruction_set_); - } - } - for (const LinkerPatch& patch : compiled_method->GetPatches()) { - if (patch.Type() == kLinkerPatchCallRelative) { - unprocessed_patches_.emplace_back(patch.TargetMethod(), - quick_code_offset + patch.LiteralOffset()); - } - } - return offset; - } - - uint32_t CalculateDisplacement(uint32_t patch_offset, uint32_t target_offset) { - // Unsigned arithmetic with its well-defined overflow behavior is just fine here. - uint32_t displacement = target_offset - patch_offset; - // NOTE: With unsigned arithmetic we do mean to use && rather than || below. - if (displacement > max_positive_displacement_ && displacement < -max_negative_displacement_) { - // Unwritten thunks have higher offsets, check if it's within range. - DCHECK(current_thunk_to_write_ == thunk_locations_.size() || - thunk_locations_[current_thunk_to_write_] > patch_offset); - if (current_thunk_to_write_ != thunk_locations_.size() && - thunk_locations_[current_thunk_to_write_] - patch_offset < max_positive_displacement_) { - displacement = thunk_locations_[current_thunk_to_write_] - patch_offset; - } else { - // We must have a previous thunk then. - DCHECK_NE(current_thunk_to_write_, 0u); - DCHECK_LT(thunk_locations_[current_thunk_to_write_ - 1], patch_offset); - displacement = thunk_locations_[current_thunk_to_write_ - 1] - patch_offset; - DCHECK(displacement >= -max_negative_displacement_); - } - } - return displacement; - } - - OatWriter* Writer() const { - return writer_; - } - - private: - bool ReserveSpaceProcessPatches(uint32_t next_aligned_offset) { - // Process as many patches as possible, stop only on unresolved targets or calls too far back. - while (!unprocessed_patches_.empty()) { - uint32_t patch_offset = unprocessed_patches_.front().second; - auto it = writer_->method_offset_map_.find(unprocessed_patches_.front().first); - if (it == writer_->method_offset_map_.end()) { - // If still unresolved, check if we have a thunk within range. - DCHECK(thunk_locations_.empty() || thunk_locations_.back() <= patch_offset); - if (thunk_locations_.empty() || - patch_offset - thunk_locations_.back() > max_negative_displacement_) { - return next_aligned_offset - patch_offset > max_positive_displacement_; - } - } else if (it->second >= patch_offset) { - DCHECK_LE(it->second - patch_offset, max_positive_displacement_); - } else { - // When calling back, check if we have a thunk that's closer than the actual target. - uint32_t target_offset = (thunk_locations_.empty() || it->second > thunk_locations_.back()) - ? it->second - : thunk_locations_.back(); - DCHECK_GT(patch_offset, target_offset); - if (patch_offset - target_offset > max_negative_displacement_) { - return true; - } - } - unprocessed_patches_.pop_front(); - } - return false; - } - - OatWriter* const writer_; - const InstructionSet instruction_set_; - const std::vector thunk_code_; - const uint32_t max_positive_displacement_; - const uint32_t max_negative_displacement_; - std::vector thunk_locations_; - size_t current_thunk_to_write_; - - // ReserveSpace() tracks unprocessed patches. - typedef std::pair UnprocessedPatch; - std::deque unprocessed_patches_; - - DISALLOW_COPY_AND_ASSIGN(ArmBaseRelativePatcher); -}; - -class OatWriter::Thumb2RelativePatcher FINAL : public ArmBaseRelativePatcher { - public: - explicit Thumb2RelativePatcher(OatWriter* writer) - : ArmBaseRelativePatcher(writer, kThumb2, CompileThunkCode(), - kMaxPositiveDisplacement, kMaxNegativeDisplacement) { - } - - void PatchCall(std::vector* code, uint32_t literal_offset, - uint32_t patch_offset, uint32_t target_offset) OVERRIDE { - DCHECK_LE(literal_offset + 4u, code->size()); - DCHECK_EQ(literal_offset & 1u, 0u); - DCHECK_EQ(patch_offset & 1u, 0u); - DCHECK_EQ(target_offset & 1u, 1u); // Thumb2 mode bit. - uint32_t displacement = CalculateDisplacement(patch_offset, target_offset & ~1u); - displacement -= kPcDisplacement; // The base PC is at the end of the 4-byte patch. - DCHECK_EQ(displacement & 1u, 0u); - DCHECK((displacement >> 24) == 0u || (displacement >> 24) == 255u); // 25-bit signed. - uint32_t signbit = (displacement >> 31) & 0x1; - uint32_t i1 = (displacement >> 23) & 0x1; - uint32_t i2 = (displacement >> 22) & 0x1; - uint32_t imm10 = (displacement >> 12) & 0x03ff; - uint32_t imm11 = (displacement >> 1) & 0x07ff; - uint32_t j1 = i1 ^ (signbit ^ 1); - uint32_t j2 = i2 ^ (signbit ^ 1); - uint32_t value = (signbit << 26) | (j1 << 13) | (j2 << 11) | (imm10 << 16) | imm11; - value |= 0xf000d000; // BL - - uint8_t* addr = &(*code)[literal_offset]; - // Check that we're just overwriting an existing BL. - DCHECK_EQ(addr[1] & 0xf8, 0xf0); - DCHECK_EQ(addr[3] & 0xd0, 0xd0); - // Write the new BL. - addr[0] = (value >> 16) & 0xff; - addr[1] = (value >> 24) & 0xff; - addr[2] = (value >> 0) & 0xff; - addr[3] = (value >> 8) & 0xff; - } - - virtual void PatchDexCacheReference(std::vector* code ATTRIBUTE_UNUSED, - const LinkerPatch& patch ATTRIBUTE_UNUSED, - uint32_t patch_offset ATTRIBUTE_UNUSED, - uint32_t target_offset ATTRIBUTE_UNUSED) { - LOG(FATAL) << "Unexpected relative dex cache array patch."; - } - - private: - static std::vector CompileThunkCode() { - // The thunk just uses the entry point in the ArtMethod. This works even for calls - // to the generic JNI and interpreter trampolines. - arm::Thumb2Assembler assembler; - assembler.LoadFromOffset( - arm::kLoadWord, arm::PC, arm::R0, - mirror::ArtMethod::EntryPointFromQuickCompiledCodeOffset(kArmPointerSize).Int32Value()); - assembler.bkpt(0); - std::vector thunk_code(assembler.CodeSize()); - MemoryRegion code(thunk_code.data(), thunk_code.size()); - assembler.FinalizeInstructions(code); - return thunk_code; - } - - // PC displacement from patch location; Thumb2 PC is always at instruction address + 4. - static constexpr int32_t kPcDisplacement = 4; - - // Maximum positive and negative displacement measured from the patch location. - // (Signed 25 bit displacement with the last bit 0 has range [-2^24, 2^24-2] measured from - // the Thumb2 PC pointing right after the BL, i.e. 4 bytes later than the patch location.) - static constexpr uint32_t kMaxPositiveDisplacement = (1u << 24) - 2 + kPcDisplacement; - static constexpr uint32_t kMaxNegativeDisplacement = (1u << 24) - kPcDisplacement; - - DISALLOW_COPY_AND_ASSIGN(Thumb2RelativePatcher); -}; - -class OatWriter::Arm64RelativePatcher FINAL : public ArmBaseRelativePatcher { - public: - explicit Arm64RelativePatcher(OatWriter* writer) - : ArmBaseRelativePatcher(writer, kArm64, CompileThunkCode(), - kMaxPositiveDisplacement, kMaxNegativeDisplacement), - fix_cortex_a53_843419_(writer->compiler_driver_->GetInstructionSetFeatures() - ->AsArm64InstructionSetFeatures()->NeedFixCortexA53_835769()), - reserved_adrp_thunks_(0u), - processed_adrp_thunks_(0u) { - if (fix_cortex_a53_843419_) { - adrp_thunk_locations_.reserve(16u); - current_method_thunks_.reserve(16u * kAdrpThunkSize); - } - } - - uint32_t ReserveSpace(uint32_t offset, const CompiledMethod* compiled_method) OVERRIDE { - if (!fix_cortex_a53_843419_) { - DCHECK(adrp_thunk_locations_.empty()); - return ReserveSpaceInternal(offset, compiled_method, 0u); - } - - // Add thunks for previous method if any. - if (reserved_adrp_thunks_ != adrp_thunk_locations_.size()) { - size_t num_adrp_thunks = adrp_thunk_locations_.size() - reserved_adrp_thunks_; - offset = CompiledMethod::AlignCode(offset, kArm64) + kAdrpThunkSize * num_adrp_thunks; - reserved_adrp_thunks_ = adrp_thunk_locations_.size(); - } - - // Count the number of ADRP insns as the upper bound on the number of thunks needed - // and use it to reserve space for other linker patches. - size_t num_adrp = 0u; - if (LIKELY(compiled_method != nullptr)) { - for (const LinkerPatch& patch : compiled_method->GetPatches()) { - if (patch.Type() == kLinkerPatchDexCacheArray && - patch.LiteralOffset() == patch.PcInsnOffset()) { // ADRP patch - ++num_adrp; - } - } - } - offset = ReserveSpaceInternal(offset, compiled_method, kAdrpThunkSize * num_adrp); - if (num_adrp == 0u) { - return offset; - } - - // Now that we have the actual offset where the code will be placed, locate the ADRP insns - // that actually require the thunk. - uint32_t quick_code_offset = compiled_method->AlignCode(offset) + sizeof(OatQuickMethodHeader); - ArrayRef code(*compiled_method->GetQuickCode()); - uint32_t thunk_offset = compiled_method->AlignCode(quick_code_offset + code.size()); - DCHECK(compiled_method != nullptr); - for (const LinkerPatch& patch : compiled_method->GetPatches()) { - if (patch.Type() == kLinkerPatchDexCacheArray && - patch.LiteralOffset() == patch.PcInsnOffset()) { // ADRP patch - uint32_t patch_offset = quick_code_offset + patch.LiteralOffset(); - if (NeedsErratum843419Thunk(code, patch.LiteralOffset(), patch_offset)) { - adrp_thunk_locations_.emplace_back(patch_offset, thunk_offset); - thunk_offset += kAdrpThunkSize; - } - } - } - return offset; - } - - uint32_t WriteThunks(OutputStream* out, uint32_t offset) OVERRIDE { - if (fix_cortex_a53_843419_) { - if (!current_method_thunks_.empty()) { - uint32_t aligned_offset = CompiledMethod::AlignCode(offset, kArm64); - if (kIsDebugBuild) { - CHECK(IsAligned(current_method_thunks_.size())); - size_t num_thunks = current_method_thunks_.size() / kAdrpThunkSize; - CHECK_LE(num_thunks, processed_adrp_thunks_); - for (size_t i = 0u; i != num_thunks; ++i) { - const auto& entry = adrp_thunk_locations_[processed_adrp_thunks_ - num_thunks + i]; - CHECK_EQ(entry.second, aligned_offset + i * kAdrpThunkSize); - } - } - uint32_t aligned_code_delta = aligned_offset - offset; - if (aligned_code_delta != 0u && !Writer()->WriteCodeAlignment(out, aligned_code_delta)) { - return 0u; - } - if (!out->WriteFully(¤t_method_thunks_[0], current_method_thunks_.size())) { - return 0u; - } - Writer()->size_misc_thunks_ += current_method_thunks_.size(); - offset = aligned_offset + current_method_thunks_.size(); - current_method_thunks_.clear(); - } - } - return ArmBaseRelativePatcher::WriteThunks(out, offset); - } - - void PatchCall(std::vector* code, uint32_t literal_offset, - uint32_t patch_offset, uint32_t target_offset) OVERRIDE { - DCHECK_LE(literal_offset + 4u, code->size()); - DCHECK_EQ(literal_offset & 3u, 0u); - DCHECK_EQ(patch_offset & 3u, 0u); - DCHECK_EQ(target_offset & 3u, 0u); - uint32_t displacement = CalculateDisplacement(patch_offset, target_offset & ~1u); - DCHECK_EQ(displacement & 3u, 0u); - DCHECK((displacement >> 27) == 0u || (displacement >> 27) == 31u); // 28-bit signed. - uint32_t insn = (displacement & 0x0fffffffu) >> 2; - insn |= 0x94000000; // BL - - // Check that we're just overwriting an existing BL. - DCHECK_EQ(GetInsn(code, literal_offset) & 0xfc000000u, 0x94000000u); - // Write the new BL. - SetInsn(code, literal_offset, insn); - } - - virtual void PatchDexCacheReference(std::vector* code ATTRIBUTE_UNUSED, - const LinkerPatch& patch ATTRIBUTE_UNUSED, - uint32_t patch_offset ATTRIBUTE_UNUSED, - uint32_t target_offset ATTRIBUTE_UNUSED) { - DCHECK_EQ(patch_offset & 3u, 0u); - DCHECK_EQ(target_offset & 3u, 0u); - uint32_t literal_offset = patch.LiteralOffset(); - uint32_t insn = GetInsn(code, literal_offset); - uint32_t pc_insn_offset = patch.PcInsnOffset(); - uint32_t disp = target_offset - ((patch_offset - literal_offset + pc_insn_offset) & ~0xfffu); - if (literal_offset == pc_insn_offset) { - // Check it's an ADRP with imm == 0 (unset). - DCHECK_EQ((insn & 0xffffffe0u), 0x90000000u) - << literal_offset << ", " << pc_insn_offset << ", 0x" << std::hex << insn; - if (fix_cortex_a53_843419_ && processed_adrp_thunks_ != adrp_thunk_locations_.size() && - adrp_thunk_locations_[processed_adrp_thunks_].first == patch_offset) { - DCHECK(NeedsErratum843419Thunk(ArrayRef(*code), - literal_offset, patch_offset)); - uint32_t thunk_offset = adrp_thunk_locations_[processed_adrp_thunks_].second; - uint32_t adrp_disp = target_offset - (thunk_offset & ~0xfffu); - uint32_t adrp = PatchAdrp(insn, adrp_disp); - - uint32_t out_disp = thunk_offset - patch_offset; - DCHECK_EQ(out_disp & 3u, 0u); - DCHECK((out_disp >> 27) == 0u || (out_disp >> 27) == 31u); // 28-bit signed. - insn = (out_disp & 0x0fffffffu) >> 2; - insn |= 0x14000000; // B - - uint32_t back_disp = -out_disp; - DCHECK_EQ(back_disp & 3u, 0u); - DCHECK((back_disp >> 27) == 0u || (back_disp >> 27) == 31u); // 28-bit signed. - uint32_t b_back = (back_disp & 0x0fffffffu) >> 2; - b_back |= 0x14000000; // B - size_t thunks_code_offset = current_method_thunks_.size(); - current_method_thunks_.resize(thunks_code_offset + kAdrpThunkSize); - SetInsn(¤t_method_thunks_, thunks_code_offset, adrp); - SetInsn(¤t_method_thunks_, thunks_code_offset + 4u, b_back); - static_assert(kAdrpThunkSize == 2 * 4u, "thunk has 2 instructions"); - - processed_adrp_thunks_ += 1u; - } else { - insn = PatchAdrp(insn, disp); - } - // Write the new ADRP (or B to the erratum 843419 thunk). - SetInsn(code, literal_offset, insn); - } else { - DCHECK_EQ(insn & 0xfffffc00, 0xb9400000); // LDR 32-bit with imm12 == 0 (unset). - if (kIsDebugBuild) { - uint32_t adrp = GetInsn(code, pc_insn_offset); - if ((adrp & 0x9f000000u) != 0x90000000u) { - CHECK(fix_cortex_a53_843419_); - CHECK_EQ(adrp & 0xfc000000u, 0x14000000u); // B - CHECK(IsAligned(current_method_thunks_.size())); - size_t num_thunks = current_method_thunks_.size() / kAdrpThunkSize; - CHECK_LE(num_thunks, processed_adrp_thunks_); - uint32_t b_offset = patch_offset - literal_offset + pc_insn_offset; - for (size_t i = processed_adrp_thunks_ - num_thunks; ; ++i) { - CHECK_NE(i, processed_adrp_thunks_); - if (adrp_thunk_locations_[i].first == b_offset) { - size_t idx = num_thunks - (processed_adrp_thunks_ - i); - adrp = GetInsn(¤t_method_thunks_, idx * kAdrpThunkSize); - break; - } - } - } - CHECK_EQ(adrp & 0x9f00001fu, // Check that pc_insn_offset points - 0x90000000 | ((insn >> 5) & 0x1fu)); // to ADRP with matching register. - } - uint32_t imm12 = (disp & 0xfffu) >> 2; - insn = (insn & ~(0xfffu << 10)) | (imm12 << 10); - SetInsn(code, literal_offset, insn); - } - } - - private: - static uint32_t PatchAdrp(uint32_t adrp, uint32_t disp) { - return (adrp & 0x9f00001fu) | // Clear offset bits, keep ADRP with destination reg. - // Bottom 12 bits are ignored, the next 2 lowest bits are encoded in bits 29-30. - ((disp & 0x00003000u) << (29 - 12)) | - // The next 16 bits are encoded in bits 5-22. - ((disp & 0xffffc000u) >> (12 + 2 - 5)) | - // Since the target_offset is based on the beginning of the oat file and the - // image space precedes the oat file, the target_offset into image space will - // be negative yet passed as uint32_t. Therefore we limit the displacement - // to +-2GiB (rather than the maximim +-4GiB) and determine the sign bit from - // the highest bit of the displacement. This is encoded in bit 23. - ((disp & 0x80000000u) >> (31 - 23)); - } - - static std::vector CompileThunkCode() { - // The thunk just uses the entry point in the ArtMethod. This works even for calls - // to the generic JNI and interpreter trampolines. - arm64::Arm64Assembler assembler; - Offset offset(mirror::ArtMethod::EntryPointFromQuickCompiledCodeOffset( - kArm64PointerSize).Int32Value()); - assembler.JumpTo(ManagedRegister(arm64::X0), offset, ManagedRegister(arm64::IP0)); - // Ensure we emit the literal pool. - assembler.EmitSlowPaths(); - std::vector thunk_code(assembler.CodeSize()); - MemoryRegion code(thunk_code.data(), thunk_code.size()); - assembler.FinalizeInstructions(code); - return thunk_code; - } - - static bool NeedsErratum843419Thunk(ArrayRef code, uint32_t literal_offset, - uint32_t patch_offset) { - DCHECK_EQ(patch_offset & 0x3u, 0u); - if ((patch_offset & 0xff8) == 0xff8) { // ...ff8 or ...ffc - uint32_t adrp = GetInsn(code, literal_offset); - DCHECK_EQ(adrp & 0xff000000, 0x90000000); - // TODO: Improve the check. For now, we're just checking if the next insn is - // the LDR using the result of the ADRP, otherwise we implement the workaround. - uint32_t next_insn = GetInsn(code, literal_offset + 4u); - bool ok = (next_insn & 0xffc00000) == 0xb9400000 && // LDR , [, #pimm] - (((next_insn >> 5) ^ adrp) & 0x1f) == 0; // == ADRP destination reg - return !ok; - } - return false; - } - - static uint32_t GetInsn(ArrayRef code, uint32_t offset) { - DCHECK_LE(offset + 4u, code.size()); - DCHECK_EQ(offset & 3u, 0u); - const uint8_t* addr = &code[offset]; - return - (static_cast(addr[0]) << 0) + - (static_cast(addr[1]) << 8) + - (static_cast(addr[2]) << 16)+ - (static_cast(addr[3]) << 24); - } - - template - static uint32_t GetInsn(std::vector* code, uint32_t offset) { - return GetInsn(ArrayRef(*code), offset); - } - - void SetInsn(std::vector* code, uint32_t offset, uint32_t value) { - DCHECK_LE(offset + 4u, code->size()); - DCHECK_EQ(offset & 3u, 0u); - uint8_t* addr = &(*code)[offset]; - addr[0] = (value >> 0) & 0xff; - addr[1] = (value >> 8) & 0xff; - addr[2] = (value >> 16) & 0xff; - addr[3] = (value >> 24) & 0xff; - } - - // Maximum positive and negative displacement measured from the patch location. - // (Signed 28 bit displacement with the last bit 0 has range [-2^27, 2^27-4] measured from - // the ARM64 PC pointing to the BL.) - static constexpr uint32_t kMaxPositiveDisplacement = (1u << 27) - 4u; - static constexpr uint32_t kMaxNegativeDisplacement = (1u << 27); - - // The ADRP thunk for erratum 843419 is 2 instructions, i.e. 8 bytes. - static constexpr uint32_t kAdrpThunkSize = 8u; - - const bool fix_cortex_a53_843419_; - // Map original patch_offset to thunk offset. - std::vector> adrp_thunk_locations_; - size_t reserved_adrp_thunks_; - size_t processed_adrp_thunks_; - std::vector current_method_thunks_; - - DISALLOW_COPY_AND_ASSIGN(Arm64RelativePatcher); -}; - #define DCHECK_OFFSET() \ DCHECK_EQ(static_cast(file_offset + relative_offset), out->Seek(0, kSeekCurrent)) \ << "file_offset=" << file_offset << " relative_offset=" << relative_offset @@ -727,25 +106,10 @@ OatWriter::OatWriter(const std::vector& dex_files, method_offset_map_() { CHECK(key_value_store != nullptr); - switch (compiler_driver_->GetInstructionSet()) { - case kX86: - relative_patcher_.reset(new X86RelativePatcher); - break; - case kX86_64: - relative_patcher_.reset(new X86_64RelativePatcher); - break; - case kArm: - // Fall through: we generate Thumb2 code for "arm". - case kThumb2: - relative_patcher_.reset(new Thumb2RelativePatcher(this)); - break; - case kArm64: - relative_patcher_.reset(new Arm64RelativePatcher(this)); - break; - default: - relative_patcher_.reset(new NoRelativePatcher); - break; - } + InstructionSet instruction_set = compiler_driver_->GetInstructionSet(); + const InstructionSetFeatures* features = compiler_driver_->GetInstructionSetFeatures(); + relative_patcher_ = linker::RelativePatcher::Create(instruction_set, features, + &method_offset_map_); size_t offset; { @@ -1029,15 +393,15 @@ class OatWriter::InitCodeMethodVisitor : public OatDexMethodVisitor { } MethodReference method_ref(dex_file_, it.GetMemberIndex()); - auto method_lb = writer_->method_offset_map_.lower_bound(method_ref); - if (method_lb != writer_->method_offset_map_.end() && - !writer_->method_offset_map_.key_comp()(method_ref, method_lb->first)) { + auto method_lb = writer_->method_offset_map_.map.lower_bound(method_ref); + if (method_lb != writer_->method_offset_map_.map.end() && + !writer_->method_offset_map_.map.key_comp()(method_ref, method_lb->first)) { // TODO: Should this be a hard failure? LOG(WARNING) << "Multiple definitions of " << PrettyMethod(method_ref.dex_method_index, *method_ref.dex_file) << ((method_lb->second != quick_code_offset) ? "; OFFSET MISMATCH" : ""); } else { - writer_->method_offset_map_.PutBefore(method_lb, method_ref, quick_code_offset); + writer_->method_offset_map_.map.PutBefore(method_lb, method_ref, quick_code_offset); } // Update quick method header. @@ -1424,9 +788,9 @@ class OatWriter::WriteCodeMethodVisitor : public OatDexMethodVisitor { } uint32_t GetTargetOffset(const LinkerPatch& patch) SHARED_LOCKS_REQUIRED(Locks::mutator_lock_) { - auto target_it = writer_->method_offset_map_.find(patch.TargetMethod()); + auto target_it = writer_->method_offset_map_.map.find(patch.TargetMethod()); uint32_t target_offset = - (target_it != writer_->method_offset_map_.end()) ? target_it->second : 0u; + (target_it != writer_->method_offset_map_.map.end()) ? target_it->second : 0u; // If there's no compiled code, point to the correct trampoline. if (UNLIKELY(target_offset == 0)) { mirror::ArtMethod* target = GetTargetMethod(patch); @@ -1965,6 +1329,10 @@ size_t OatWriter::WriteCodeDexFiles(OutputStream* out, #undef VISIT + size_code_alignment_ += relative_patcher_->CodeAlignmentSize(); + size_relative_call_thunks_ += relative_patcher_->RelativeCallThunksSize(); + size_misc_thunks_ += relative_patcher_->MiscThunksSize(); + return relative_offset; } @@ -1980,6 +1348,15 @@ bool OatWriter::WriteCodeAlignment(OutputStream* out, uint32_t aligned_code_delt return true; } +std::pair OatWriter::MethodOffsetMap::FindMethodOffset(MethodReference ref) { + auto it = map.find(ref); + if (it == map.end()) { + return std::pair(false, 0u); + } else { + return std::pair(true, it->second); + } +} + OatWriter::OatDexFile::OatDexFile(size_t offset, const DexFile& dex_file) { offset_ = offset; const std::string& location(dex_file.GetLocation()); diff --git a/compiler/oat_writer.h b/compiler/oat_writer.h index 895163441..0cddae7bd 100644 --- a/compiler/oat_writer.h +++ b/compiler/oat_writer.h @@ -21,6 +21,7 @@ #include #include +#include "linker/relative_patcher.h" // For linker::RelativePatcherTargetProvider. #include "mem_map.h" #include "method_reference.h" #include "oat.h" @@ -133,6 +134,10 @@ class OatWriter { return method_info_; } + const CompilerDriver* GetCompilerDriver() { + return compiler_driver_; + } + private: // The DataAccess classes are helper classes that provide access to members related to // a given map, i.e. GC map, mapping table or vmap table. By abstracting these away @@ -327,21 +332,19 @@ class OatWriter { uint32_t size_oat_class_method_bitmaps_; uint32_t size_oat_class_method_offsets_; - class RelativePatcher; - class NoRelativePatcher; - class X86BaseRelativePatcher; - class X86RelativePatcher; - class X86_64RelativePatcher; - class ArmBaseRelativePatcher; - class Thumb2RelativePatcher; - class Arm64RelativePatcher; - - std::unique_ptr relative_patcher_; + std::unique_ptr relative_patcher_; // The locations of absolute patches relative to the start of the executable section. std::vector absolute_patch_locations_; - SafeMap method_offset_map_; + // Map method reference to assigned offset. + // Wrap the map in a class implementing linker::RelativePatcherTargetProvider. + class MethodOffsetMap FINAL : public linker::RelativePatcherTargetProvider { + public: + std::pair FindMethodOffset(MethodReference ref) OVERRIDE; + SafeMap map; + }; + MethodOffsetMap method_offset_map_; DISALLOW_COPY_AND_ASSIGN(OatWriter); };