From 8158f28b6689314213eb4dbbe14166073be71f7e Mon Sep 17 00:00:00 2001 From: Alexandre Rames Date: Fri, 7 Aug 2015 10:26:17 +0100 Subject: [PATCH] Ensure coherency of call kinds for LocationSummary. The coherency is enforced with checks added in the `InvokeRuntime` helper, that we now also use on x86 and x86_64. Change-Id: I8cb92b042f25dc3c5fd390e9c61a45b477d081f4 --- compiler/optimizing/code_generator.h | 2 + compiler/optimizing/code_generator_arm.cc | 14 +++ compiler/optimizing/code_generator_arm64.cc | 14 +++ compiler/optimizing/code_generator_mips64.cc | 14 +++ compiler/optimizing/code_generator_x86.cc | 143 ++++++++++++++++++++------- compiler/optimizing/code_generator_x86.h | 6 ++ compiler/optimizing/code_generator_x86_64.cc | 133 ++++++++++++++++--------- compiler/optimizing/code_generator_x86_64.h | 6 ++ compiler/optimizing/register_allocator.cc | 2 +- 9 files changed, 251 insertions(+), 83 deletions(-) diff --git a/compiler/optimizing/code_generator.h b/compiler/optimizing/code_generator.h index eb63b4988..540da1c86 100644 --- a/compiler/optimizing/code_generator.h +++ b/compiler/optimizing/code_generator.h @@ -98,6 +98,8 @@ class SlowPathCode : public ArenaObject { return saved_fpu_stack_offsets_[reg]; } + virtual bool IsFatal() const { return false; } + virtual const char* GetDescription() const = 0; protected: diff --git a/compiler/optimizing/code_generator_arm.cc b/compiler/optimizing/code_generator_arm.cc index b0a4ce2b6..d89d2b2dd 100644 --- a/compiler/optimizing/code_generator_arm.cc +++ b/compiler/optimizing/code_generator_arm.cc @@ -69,6 +69,8 @@ class NullCheckSlowPathARM : public SlowPathCodeARM { QUICK_ENTRY_POINT(pThrowNullPointer), instruction_, instruction_->GetDexPc(), this); } + bool IsFatal() const OVERRIDE { return true; } + const char* GetDescription() const OVERRIDE { return "NullCheckSlowPathARM"; } private: @@ -87,6 +89,8 @@ class DivZeroCheckSlowPathARM : public SlowPathCodeARM { QUICK_ENTRY_POINT(pThrowDivZero), instruction_, instruction_->GetDexPc(), this); } + bool IsFatal() const OVERRIDE { return true; } + const char* GetDescription() const OVERRIDE { return "DivZeroCheckSlowPathARM"; } private: @@ -161,6 +165,8 @@ class BoundsCheckSlowPathARM : public SlowPathCodeARM { QUICK_ENTRY_POINT(pThrowArrayBounds), instruction_, instruction_->GetDexPc(), this); } + bool IsFatal() const OVERRIDE { return true; } + const char* GetDescription() const OVERRIDE { return "BoundsCheckSlowPathARM"; } private: @@ -947,6 +953,14 @@ void CodeGeneratorARM::InvokeRuntime(int32_t entry_point_offset, HInstruction* instruction, uint32_t dex_pc, SlowPathCode* slow_path) { + // Ensure that the call kind indication given to the register allocator is + // coherent with the runtime call generated. + if (slow_path == nullptr) { + DCHECK(instruction->GetLocations()->WillCall()); + } else { + DCHECK(instruction->GetLocations()->OnlyCallsOnSlowPath() || slow_path->IsFatal()); + } + __ LoadFromOffset(kLoadWord, LR, TR, entry_point_offset); __ blx(LR); RecordPcInfo(instruction, dex_pc, slow_path); diff --git a/compiler/optimizing/code_generator_arm64.cc b/compiler/optimizing/code_generator_arm64.cc index bbde7e8ef..7fab5cfca 100644 --- a/compiler/optimizing/code_generator_arm64.cc +++ b/compiler/optimizing/code_generator_arm64.cc @@ -212,6 +212,8 @@ class BoundsCheckSlowPathARM64 : public SlowPathCodeARM64 { CheckEntrypointTypes(); } + bool IsFatal() const OVERRIDE { return true; } + const char* GetDescription() const OVERRIDE { return "BoundsCheckSlowPathARM64"; } private: @@ -234,6 +236,8 @@ class DivZeroCheckSlowPathARM64 : public SlowPathCodeARM64 { CheckEntrypointTypes(); } + bool IsFatal() const OVERRIDE { return true; } + const char* GetDescription() const OVERRIDE { return "DivZeroCheckSlowPathARM64"; } private: @@ -344,6 +348,8 @@ class NullCheckSlowPathARM64 : public SlowPathCodeARM64 { CheckEntrypointTypes(); } + bool IsFatal() const OVERRIDE { return true; } + const char* GetDescription() const OVERRIDE { return "NullCheckSlowPathARM64"; } private: @@ -1097,6 +1103,14 @@ void CodeGeneratorARM64::InvokeRuntime(int32_t entry_point_offset, HInstruction* instruction, uint32_t dex_pc, SlowPathCode* slow_path) { + // Ensure that the call kind indication given to the register allocator is + // coherent with the runtime call generated. + if (slow_path == nullptr) { + DCHECK(instruction->GetLocations()->WillCall()); + } else { + DCHECK(instruction->GetLocations()->OnlyCallsOnSlowPath() || slow_path->IsFatal()); + } + BlockPoolsScope block_pools(GetVIXLAssembler()); __ Ldr(lr, MemOperand(tr, entry_point_offset)); __ Blr(lr); diff --git a/compiler/optimizing/code_generator_mips64.cc b/compiler/optimizing/code_generator_mips64.cc index a5bad654c..b6d67de18 100644 --- a/compiler/optimizing/code_generator_mips64.cc +++ b/compiler/optimizing/code_generator_mips64.cc @@ -138,6 +138,8 @@ class BoundsCheckSlowPathMIPS64 : public SlowPathCodeMIPS64 { CheckEntrypointTypes(); } + bool IsFatal() const OVERRIDE { return true; } + const char* GetDescription() const OVERRIDE { return "BoundsCheckSlowPathMIPS64"; } private: @@ -162,6 +164,8 @@ class DivZeroCheckSlowPathMIPS64 : public SlowPathCodeMIPS64 { CheckEntrypointTypes(); } + bool IsFatal() const OVERRIDE { return true; } + const char* GetDescription() const OVERRIDE { return "DivZeroCheckSlowPathMIPS64"; } private: @@ -278,6 +282,8 @@ class NullCheckSlowPathMIPS64 : public SlowPathCodeMIPS64 { CheckEntrypointTypes(); } + bool IsFatal() const OVERRIDE { return true; } + const char* GetDescription() const OVERRIDE { return "NullCheckSlowPathMIPS64"; } private: @@ -971,6 +977,14 @@ void CodeGeneratorMIPS64::InvokeRuntime(int32_t entry_point_offset, HInstruction* instruction, uint32_t dex_pc, SlowPathCode* slow_path) { + // Ensure that the call kind indication given to the register allocator is + // coherent with the runtime call generated. + if (slow_path == nullptr) { + DCHECK(instruction->GetLocations()->WillCall()); + } else { + DCHECK(instruction->GetLocations()->OnlyCallsOnSlowPath() || slow_path->IsFatal()); + } + // TODO: anything related to T9/GP/GOT/PIC/.so's? __ LoadFromOffset(kLoadDoubleword, T9, TR, entry_point_offset); __ Jalr(T9); diff --git a/compiler/optimizing/code_generator_x86.cc b/compiler/optimizing/code_generator_x86.cc index fdef06b11..5ffab3319 100644 --- a/compiler/optimizing/code_generator_x86.cc +++ b/compiler/optimizing/code_generator_x86.cc @@ -45,17 +45,23 @@ static constexpr int kC2ConditionMask = 0x400; static constexpr int kFakeReturnRegister = Register(8); #define __ down_cast(codegen->GetAssembler())-> +#define QUICK_ENTRY_POINT(x) Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, x)) class NullCheckSlowPathX86 : public SlowPathCodeX86 { public: explicit NullCheckSlowPathX86(HNullCheck* instruction) : instruction_(instruction) {} void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + CodeGeneratorX86* x86_codegen = down_cast(codegen); __ Bind(GetEntryLabel()); - __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pThrowNullPointer))); - RecordPcInfo(codegen, instruction_, instruction_->GetDexPc()); + x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pThrowNullPointer), + instruction_, + instruction_->GetDexPc(), + this); } + bool IsFatal() const OVERRIDE { return true; } + const char* GetDescription() const OVERRIDE { return "NullCheckSlowPathX86"; } private: @@ -68,11 +74,16 @@ class DivZeroCheckSlowPathX86 : public SlowPathCodeX86 { explicit DivZeroCheckSlowPathX86(HDivZeroCheck* instruction) : instruction_(instruction) {} void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + CodeGeneratorX86* x86_codegen = down_cast(codegen); __ Bind(GetEntryLabel()); - __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pThrowDivZero))); - RecordPcInfo(codegen, instruction_, instruction_->GetDexPc()); + x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pThrowDivZero), + instruction_, + instruction_->GetDexPc(), + this); } + bool IsFatal() const OVERRIDE { return true; } + const char* GetDescription() const OVERRIDE { return "DivZeroCheckSlowPathX86"; } private: @@ -124,10 +135,14 @@ class BoundsCheckSlowPathX86 : public SlowPathCodeX86 { length_location_, Location::RegisterLocation(calling_convention.GetRegisterAt(1)), Primitive::kPrimInt); - __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pThrowArrayBounds))); - RecordPcInfo(codegen, instruction_, instruction_->GetDexPc()); + x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pThrowArrayBounds), + instruction_, + instruction_->GetDexPc(), + this); } + bool IsFatal() const OVERRIDE { return true; } + const char* GetDescription() const OVERRIDE { return "BoundsCheckSlowPathX86"; } private: @@ -147,8 +162,10 @@ class SuspendCheckSlowPathX86 : public SlowPathCodeX86 { CodeGeneratorX86* x86_codegen = down_cast(codegen); __ Bind(GetEntryLabel()); SaveLiveRegisters(codegen, instruction_->GetLocations()); - __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pTestSuspend))); - RecordPcInfo(codegen, instruction_, instruction_->GetDexPc()); + x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pTestSuspend), + instruction_, + instruction_->GetDexPc(), + this); RestoreLiveRegisters(codegen, instruction_->GetLocations()); if (successor_ == nullptr) { __ jmp(GetReturnLabel()); @@ -190,8 +207,10 @@ class LoadStringSlowPathX86 : public SlowPathCodeX86 { InvokeRuntimeCallingConvention calling_convention; __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction_->GetStringIndex())); - __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pResolveString))); - RecordPcInfo(codegen, instruction_, instruction_->GetDexPc()); + x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pResolveString), + instruction_, + instruction_->GetDexPc(), + this); x86_codegen->Move32(locations->Out(), Location::RegisterLocation(EAX)); RestoreLiveRegisters(codegen, locations); @@ -224,10 +243,9 @@ class LoadClassSlowPathX86 : public SlowPathCodeX86 { InvokeRuntimeCallingConvention calling_convention; __ movl(calling_convention.GetRegisterAt(0), Immediate(cls_->GetTypeIndex())); - __ fs()->call(Address::Absolute(do_clinit_ - ? QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pInitializeStaticStorage) - : QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pInitializeType))); - RecordPcInfo(codegen, at_, dex_pc_); + x86_codegen->InvokeRuntime(do_clinit_ ? QUICK_ENTRY_POINT(pInitializeStaticStorage) + : QUICK_ENTRY_POINT(pInitializeType), + at_, dex_pc_, this); // Move the class to the desired location. Location out = locations->Out(); @@ -291,11 +309,16 @@ class TypeCheckSlowPathX86 : public SlowPathCodeX86 { Primitive::kPrimNot); if (instruction_->IsInstanceOf()) { - __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, - pInstanceofNonTrivial))); + x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pInstanceofNonTrivial), + instruction_, + instruction_->GetDexPc(), + this); } else { DCHECK(instruction_->IsCheckCast()); - __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pCheckCast))); + x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pCheckCast), + instruction_, + instruction_->GetDexPc(), + this); } RecordPcInfo(codegen, instruction_, dex_pc_); @@ -324,9 +347,13 @@ class DeoptimizationSlowPathX86 : public SlowPathCodeX86 { : instruction_(instruction) {} void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + CodeGeneratorX86* x86_codegen = down_cast(codegen); __ Bind(GetEntryLabel()); SaveLiveRegisters(codegen, instruction_->GetLocations()); - __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pDeoptimize))); + x86_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pDeoptimize), + instruction_, + instruction_->GetDexPc(), + this); // No need to restore live registers. DCHECK(instruction_->IsDeoptimize()); HDeoptimize* deoptimize = instruction_->AsDeoptimize(); @@ -398,6 +425,27 @@ size_t CodeGeneratorX86::RestoreFloatingPointRegister(size_t stack_index, uint32 return GetFloatingPointSpillSlotSize(); } +void CodeGeneratorX86::InvokeRuntime(Address entry_point, + HInstruction* instruction, + uint32_t dex_pc, + SlowPathCode* slow_path) { + // Ensure that the call kind indication given to the register allocator is + // coherent with the runtime call generated. + if (slow_path == nullptr) { + DCHECK(instruction->GetLocations()->WillCall()); + } else { + DCHECK(instruction->GetLocations()->OnlyCallsOnSlowPath() || slow_path->IsFatal()); + } + + __ fs()->call(entry_point); + RecordPcInfo(instruction, dex_pc, slow_path); + DCHECK(instruction->IsSuspendCheck() + || instruction->IsBoundsCheck() + || instruction->IsNullCheck() + || instruction->IsDivZeroCheck() + || !IsLeafMethod()); +} + CodeGeneratorX86::CodeGeneratorX86(HGraph* graph, const X86InstructionSetFeatures& isa_features, const CompilerOptions& compiler_options) @@ -2015,14 +2063,18 @@ void InstructionCodeGeneratorX86::VisitTypeConversion(HTypeConversion* conversio case Primitive::kPrimFloat: // Processing a Dex `float-to-long' instruction. - __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pF2l))); - codegen_->RecordPcInfo(conversion, conversion->GetDexPc()); + codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pF2l), + conversion, + conversion->GetDexPc(), + nullptr); break; case Primitive::kPrimDouble: // Processing a Dex `double-to-long' instruction. - __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pD2l))); - codegen_->RecordPcInfo(conversion, conversion->GetDexPc()); + codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pD2l), + conversion, + conversion->GetDexPc(), + nullptr); break; default: @@ -2779,9 +2831,15 @@ void InstructionCodeGeneratorX86::GenerateDivRemIntegral(HBinaryOperation* instr DCHECK_EQ(EDX, out.AsRegisterPairHigh()); if (is_div) { - __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pLdiv))); + codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pLdiv), + instruction, + instruction->GetDexPc(), + nullptr); } else { - __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pLmod))); + codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pLmod), + instruction, + instruction->GetDexPc(), + nullptr); } uint32_t dex_pc = is_div ? instruction->AsDiv()->GetDexPc() @@ -3233,9 +3291,11 @@ void InstructionCodeGeneratorX86::VisitNewInstance(HNewInstance* instruction) { __ movl(calling_convention.GetRegisterAt(0), Immediate(instruction->GetTypeIndex())); // Note: if heap poisoning is enabled, the entry point takes cares // of poisoning the reference. - __ fs()->call(Address::Absolute(GetThreadOffset(instruction->GetEntrypoint()))); - - codegen_->RecordPcInfo(instruction, instruction->GetDexPc()); + codegen_->InvokeRuntime( + Address::Absolute(GetThreadOffset(instruction->GetEntrypoint())), + instruction, + instruction->GetDexPc(), + nullptr); DCHECK(!codegen_->IsLeafMethod()); } @@ -3255,9 +3315,11 @@ void InstructionCodeGeneratorX86::VisitNewArray(HNewArray* instruction) { // Note: if heap poisoning is enabled, the entry point takes cares // of poisoning the reference. - __ fs()->call(Address::Absolute(GetThreadOffset(instruction->GetEntrypoint()))); - - codegen_->RecordPcInfo(instruction, instruction->GetDexPc()); + codegen_->InvokeRuntime( + Address::Absolute(GetThreadOffset(instruction->GetEntrypoint())), + instruction, + instruction->GetDexPc(), + nullptr); DCHECK(!codegen_->IsLeafMethod()); } @@ -4160,8 +4222,10 @@ void InstructionCodeGeneratorX86::VisitArraySet(HArraySet* instruction) { DCHECK(!codegen_->IsLeafMethod()); // Note: if heap poisoning is enabled, pAputObject takes cares // of poisoning the reference. - __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pAputObject))); - codegen_->RecordPcInfo(instruction, instruction->GetDexPc()); + codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pAputObject), + instruction, + instruction->GetDexPc(), + nullptr); } break; } @@ -4723,8 +4787,10 @@ void LocationsBuilderX86::VisitThrow(HThrow* instruction) { } void InstructionCodeGeneratorX86::VisitThrow(HThrow* instruction) { - __ fs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pDeliverException))); - codegen_->RecordPcInfo(instruction, instruction->GetDexPc()); + codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pDeliverException), + instruction, + instruction->GetDexPc(), + nullptr); } void LocationsBuilderX86::VisitInstanceOf(HInstanceOf* instruction) { @@ -4835,10 +4901,11 @@ void LocationsBuilderX86::VisitMonitorOperation(HMonitorOperation* instruction) } void InstructionCodeGeneratorX86::VisitMonitorOperation(HMonitorOperation* instruction) { - __ fs()->call(Address::Absolute(instruction->IsEnter() - ? QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pLockObject) - : QUICK_ENTRYPOINT_OFFSET(kX86WordSize, pUnlockObject))); - codegen_->RecordPcInfo(instruction, instruction->GetDexPc()); + codegen_->InvokeRuntime(instruction->IsEnter() ? QUICK_ENTRY_POINT(pLockObject) + : QUICK_ENTRY_POINT(pUnlockObject), + instruction, + instruction->GetDexPc(), + nullptr); } void LocationsBuilderX86::VisitAnd(HAnd* instruction) { HandleBitwiseOperation(instruction); } diff --git a/compiler/optimizing/code_generator_x86.h b/compiler/optimizing/code_generator_x86.h index 65d6e0a6c..2e3d4d4bf 100644 --- a/compiler/optimizing/code_generator_x86.h +++ b/compiler/optimizing/code_generator_x86.h @@ -232,6 +232,12 @@ class CodeGeneratorX86 : public CodeGenerator { size_t SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) OVERRIDE; size_t RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) OVERRIDE; + // Generate code to invoke a runtime entry point. + void InvokeRuntime(Address entry_point, + HInstruction* instruction, + uint32_t dex_pc, + SlowPathCode* slow_path); + size_t GetWordSize() const OVERRIDE { return kX86WordSize; } diff --git a/compiler/optimizing/code_generator_x86_64.cc b/compiler/optimizing/code_generator_x86_64.cc index 4fe93f99b..2b5fcbd71 100644 --- a/compiler/optimizing/code_generator_x86_64.cc +++ b/compiler/optimizing/code_generator_x86_64.cc @@ -47,18 +47,23 @@ static constexpr FloatRegister kFpuCalleeSaves[] = { XMM12, XMM13, XMM14, XMM15 static constexpr int kC2ConditionMask = 0x400; #define __ down_cast(codegen->GetAssembler())-> +#define QUICK_ENTRY_POINT(x) Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, x), true) class NullCheckSlowPathX86_64 : public SlowPathCodeX86_64 { public: explicit NullCheckSlowPathX86_64(HNullCheck* instruction) : instruction_(instruction) {} void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + CodeGeneratorX86_64* x64_codegen = down_cast(codegen); __ Bind(GetEntryLabel()); - __ gs()->call( - Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pThrowNullPointer), true)); - RecordPcInfo(codegen, instruction_, instruction_->GetDexPc()); + x64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pThrowNullPointer), + instruction_, + instruction_->GetDexPc(), + this); } + bool IsFatal() const OVERRIDE { return true; } + const char* GetDescription() const OVERRIDE { return "NullCheckSlowPathX86_64"; } private: @@ -71,12 +76,16 @@ class DivZeroCheckSlowPathX86_64 : public SlowPathCodeX86_64 { explicit DivZeroCheckSlowPathX86_64(HDivZeroCheck* instruction) : instruction_(instruction) {} void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + CodeGeneratorX86_64* x64_codegen = down_cast(codegen); __ Bind(GetEntryLabel()); - __ gs()->call( - Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pThrowDivZero), true)); - RecordPcInfo(codegen, instruction_, instruction_->GetDexPc()); + x64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pThrowDivZero), + instruction_, + instruction_->GetDexPc(), + this); } + bool IsFatal() const OVERRIDE { return true; } + const char* GetDescription() const OVERRIDE { return "DivZeroCheckSlowPathX86_64"; } private: @@ -127,8 +136,10 @@ class SuspendCheckSlowPathX86_64 : public SlowPathCodeX86_64 { CodeGeneratorX86_64* x64_codegen = down_cast(codegen); __ Bind(GetEntryLabel()); SaveLiveRegisters(codegen, instruction_->GetLocations()); - __ gs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pTestSuspend), true)); - RecordPcInfo(codegen, instruction_, instruction_->GetDexPc()); + x64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pTestSuspend), + instruction_, + instruction_->GetDexPc(), + this); RestoreLiveRegisters(codegen, instruction_->GetLocations()); if (successor_ == nullptr) { __ jmp(GetReturnLabel()); @@ -166,6 +177,7 @@ class BoundsCheckSlowPathX86_64 : public SlowPathCodeX86_64 { length_location_(length_location) {} void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + CodeGeneratorX86_64* x64_codegen = down_cast(codegen); __ Bind(GetEntryLabel()); // We're moving two locations to locations that could overlap, so we need a parallel // move resolver. @@ -177,11 +189,12 @@ class BoundsCheckSlowPathX86_64 : public SlowPathCodeX86_64 { length_location_, Location::RegisterLocation(calling_convention.GetRegisterAt(1)), Primitive::kPrimInt); - __ gs()->call(Address::Absolute( - QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pThrowArrayBounds), true)); - RecordPcInfo(codegen, instruction_, instruction_->GetDexPc()); + x64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pThrowArrayBounds), + instruction_, instruction_->GetDexPc(), this); } + bool IsFatal() const OVERRIDE { return true; } + const char* GetDescription() const OVERRIDE { return "BoundsCheckSlowPathX86_64"; } private: @@ -211,10 +224,9 @@ class LoadClassSlowPathX86_64 : public SlowPathCodeX86_64 { InvokeRuntimeCallingConvention calling_convention; __ movl(CpuRegister(calling_convention.GetRegisterAt(0)), Immediate(cls_->GetTypeIndex())); - __ gs()->call(Address::Absolute((do_clinit_ - ? QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pInitializeStaticStorage) - : QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pInitializeType)), true)); - RecordPcInfo(codegen, at_, dex_pc_); + x64_codegen->InvokeRuntime(do_clinit_ ? QUICK_ENTRY_POINT(pInitializeStaticStorage) + : QUICK_ENTRY_POINT(pInitializeType), + at_, dex_pc_, this); Location out = locations->Out(); // Move the class to the desired location. @@ -261,9 +273,10 @@ class LoadStringSlowPathX86_64 : public SlowPathCodeX86_64 { InvokeRuntimeCallingConvention calling_convention; __ movl(CpuRegister(calling_convention.GetRegisterAt(0)), Immediate(instruction_->GetStringIndex())); - __ gs()->call(Address::Absolute( - QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pResolveString), true)); - RecordPcInfo(codegen, instruction_, instruction_->GetDexPc()); + x64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pResolveString), + instruction_, + instruction_->GetDexPc(), + this); x64_codegen->Move(locations->Out(), Location::RegisterLocation(RAX)); RestoreLiveRegisters(codegen, locations); __ jmp(GetExitLabel()); @@ -309,14 +322,17 @@ class TypeCheckSlowPathX86_64 : public SlowPathCodeX86_64 { Primitive::kPrimNot); if (instruction_->IsInstanceOf()) { - __ gs()->call( - Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pInstanceofNonTrivial), true)); + x64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pInstanceofNonTrivial), + instruction_, + dex_pc_, + this); } else { DCHECK(instruction_->IsCheckCast()); - __ gs()->call( - Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pCheckCast), true)); + x64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pCheckCast), + instruction_, + dex_pc_, + this); } - RecordPcInfo(codegen, instruction_, dex_pc_); if (instruction_->IsInstanceOf()) { x64_codegen->Move(locations->Out(), Location::RegisterLocation(RAX)); @@ -343,14 +359,15 @@ class DeoptimizationSlowPathX86_64 : public SlowPathCodeX86_64 { : instruction_(instruction) {} void EmitNativeCode(CodeGenerator* codegen) OVERRIDE { + CodeGeneratorX86_64* x64_codegen = down_cast(codegen); __ Bind(GetEntryLabel()); SaveLiveRegisters(codegen, instruction_->GetLocations()); - __ gs()->call( - Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pDeoptimize), true)); DCHECK(instruction_->IsDeoptimize()); HDeoptimize* deoptimize = instruction_->AsDeoptimize(); - uint32_t dex_pc = deoptimize->GetDexPc(); - codegen->RecordPcInfo(instruction_, dex_pc, this); + x64_codegen->InvokeRuntime(QUICK_ENTRY_POINT(pDeoptimize), + deoptimize, + deoptimize->GetDexPc(), + this); } const char* GetDescription() const OVERRIDE { return "DeoptimizationSlowPathX86_64"; } @@ -463,6 +480,27 @@ size_t CodeGeneratorX86_64::RestoreFloatingPointRegister(size_t stack_index, uin return kX86_64WordSize; } +void CodeGeneratorX86_64::InvokeRuntime(Address entry_point, + HInstruction* instruction, + uint32_t dex_pc, + SlowPathCode* slow_path) { + // Ensure that the call kind indication given to the register allocator is + // coherent with the runtime call generated. + if (slow_path == nullptr) { + DCHECK(instruction->GetLocations()->WillCall()); + } else { + DCHECK(instruction->GetLocations()->OnlyCallsOnSlowPath() || slow_path->IsFatal()); + } + + __ gs()->call(entry_point); + RecordPcInfo(instruction, dex_pc, slow_path); + DCHECK(instruction->IsSuspendCheck() + || instruction->IsBoundsCheck() + || instruction->IsNullCheck() + || instruction->IsDivZeroCheck() + || !IsLeafMethod()); +} + static constexpr int kNumberOfCpuRegisterPairs = 0; // Use a fake return address register to mimic Quick. static constexpr Register kFakeReturnRegister = Register(kLastCpuRegister + 1); @@ -3292,11 +3330,14 @@ void InstructionCodeGeneratorX86_64::VisitNewInstance(HNewInstance* instruction) instruction->GetTypeIndex()); // Note: if heap poisoning is enabled, the entry point takes cares // of poisoning the reference. - __ gs()->call( - Address::Absolute(GetThreadOffset(instruction->GetEntrypoint()), true)); + + codegen_->InvokeRuntime( + Address::Absolute(GetThreadOffset(instruction->GetEntrypoint()), true), + instruction, + instruction->GetDexPc(), + nullptr); DCHECK(!codegen_->IsLeafMethod()); - codegen_->RecordPcInfo(instruction, instruction->GetDexPc()); } void LocationsBuilderX86_64::VisitNewArray(HNewArray* instruction) { @@ -3316,11 +3357,13 @@ void InstructionCodeGeneratorX86_64::VisitNewArray(HNewArray* instruction) { // Note: if heap poisoning is enabled, the entry point takes cares // of poisoning the reference. - __ gs()->call( - Address::Absolute(GetThreadOffset(instruction->GetEntrypoint()), true)); + codegen_->InvokeRuntime( + Address::Absolute(GetThreadOffset(instruction->GetEntrypoint()), true), + instruction, + instruction->GetDexPc(), + nullptr); DCHECK(!codegen_->IsLeafMethod()); - codegen_->RecordPcInfo(instruction, instruction->GetDexPc()); } void LocationsBuilderX86_64::VisitParameterValue(HParameterValue* instruction) { @@ -4007,10 +4050,11 @@ void InstructionCodeGeneratorX86_64::VisitArraySet(HArraySet* instruction) { DCHECK_EQ(value_type, Primitive::kPrimNot); // Note: if heap poisoning is enabled, pAputObject takes cares // of poisoning the reference. - __ gs()->call(Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pAputObject), - true)); + codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pAputObject), + instruction, + instruction->GetDexPc(), + nullptr); DCHECK(!codegen_->IsLeafMethod()); - codegen_->RecordPcInfo(instruction, instruction->GetDexPc()); } break; } @@ -4557,9 +4601,10 @@ void LocationsBuilderX86_64::VisitThrow(HThrow* instruction) { } void InstructionCodeGeneratorX86_64::VisitThrow(HThrow* instruction) { - __ gs()->call( - Address::Absolute(QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pDeliverException), true)); - codegen_->RecordPcInfo(instruction, instruction->GetDexPc()); + codegen_->InvokeRuntime(QUICK_ENTRY_POINT(pDeliverException), + instruction, + instruction->GetDexPc(), + nullptr); } void LocationsBuilderX86_64::VisitInstanceOf(HInstanceOf* instruction) { @@ -4669,11 +4714,11 @@ void LocationsBuilderX86_64::VisitMonitorOperation(HMonitorOperation* instructio } void InstructionCodeGeneratorX86_64::VisitMonitorOperation(HMonitorOperation* instruction) { - __ gs()->call(Address::Absolute(instruction->IsEnter() - ? QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pLockObject) - : QUICK_ENTRYPOINT_OFFSET(kX86_64WordSize, pUnlockObject), - true)); - codegen_->RecordPcInfo(instruction, instruction->GetDexPc()); + codegen_->InvokeRuntime(instruction->IsEnter() ? QUICK_ENTRY_POINT(pLockObject) + : QUICK_ENTRY_POINT(pUnlockObject), + instruction, + instruction->GetDexPc(), + nullptr); } void LocationsBuilderX86_64::VisitAnd(HAnd* instruction) { HandleBitwiseOperation(instruction); } diff --git a/compiler/optimizing/code_generator_x86_64.h b/compiler/optimizing/code_generator_x86_64.h index 4b90381f0..3b3915f2a 100644 --- a/compiler/optimizing/code_generator_x86_64.h +++ b/compiler/optimizing/code_generator_x86_64.h @@ -232,6 +232,12 @@ class CodeGeneratorX86_64 : public CodeGenerator { size_t SaveFloatingPointRegister(size_t stack_index, uint32_t reg_id) OVERRIDE; size_t RestoreFloatingPointRegister(size_t stack_index, uint32_t reg_id) OVERRIDE; + // Generate code to invoke a runtime entry point. + void InvokeRuntime(Address entry_point, + HInstruction* instruction, + uint32_t dex_pc, + SlowPathCode* slow_path); + size_t GetWordSize() const OVERRIDE { return kX86_64WordSize; } diff --git a/compiler/optimizing/register_allocator.cc b/compiler/optimizing/register_allocator.cc index 72ddabe55..de625301d 100644 --- a/compiler/optimizing/register_allocator.cc +++ b/compiler/optimizing/register_allocator.cc @@ -248,7 +248,7 @@ void RegisterAllocator::ProcessInstruction(HInstruction* instruction) { bool core_register = (instruction->GetType() != Primitive::kPrimDouble) && (instruction->GetType() != Primitive::kPrimFloat); - if (locations->CanCall()) { + if (locations->NeedsSafepoint()) { if (codegen_->IsLeafMethod()) { // TODO: We do this here because we do not want the suspend check to artificially // create live registers. We should find another place, but this is currently the -- 2.11.0