1 //===-- llvm/lib/Target/AMDGPU/AMDGPUCallLowering.cpp - Call lowering -----===//
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
7 //===----------------------------------------------------------------------===//
10 /// This file implements the lowering of LLVM calls to machine code calls for
13 //===----------------------------------------------------------------------===//
15 #include "AMDGPUCallLowering.h"
17 #include "AMDGPUISelLowering.h"
18 #include "AMDGPUSubtarget.h"
19 #include "SIISelLowering.h"
20 #include "SIMachineFunctionInfo.h"
21 #include "SIRegisterInfo.h"
22 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
23 #include "llvm/CodeGen/Analysis.h"
24 #include "llvm/CodeGen/CallingConvLower.h"
25 #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h"
26 #include "llvm/CodeGen/MachineInstrBuilder.h"
27 #include "llvm/Support/LowLevelTypeImpl.h"
33 struct OutgoingArgHandler : public CallLowering::ValueHandler {
34 OutgoingArgHandler(MachineIRBuilder &MIRBuilder, MachineRegisterInfo &MRI,
35 MachineInstrBuilder MIB, CCAssignFn *AssignFn)
36 : ValueHandler(MIRBuilder, MRI, AssignFn), MIB(MIB) {}
38 MachineInstrBuilder MIB;
40 Register getStackAddress(uint64_t Size, int64_t Offset,
41 MachinePointerInfo &MPO) override {
42 llvm_unreachable("not implemented");
45 void assignValueToAddress(Register ValVReg, Register Addr, uint64_t Size,
46 MachinePointerInfo &MPO, CCValAssign &VA) override {
47 llvm_unreachable("not implemented");
50 void assignValueToReg(Register ValVReg, Register PhysReg,
51 CCValAssign &VA) override {
53 MIRBuilder.buildCopy(PhysReg, ValVReg);
56 bool assignArg(unsigned ValNo, MVT ValVT, MVT LocVT,
57 CCValAssign::LocInfo LocInfo,
58 const CallLowering::ArgInfo &Info,
59 CCState &State) override {
60 return AssignFn(ValNo, ValVT, LocVT, LocInfo, Info.Flags, State);
66 AMDGPUCallLowering::AMDGPUCallLowering(const AMDGPUTargetLowering &TLI)
67 : CallLowering(&TLI) {
70 bool AMDGPUCallLowering::lowerReturn(MachineIRBuilder &MIRBuilder,
72 ArrayRef<Register> VRegs) const {
74 MachineFunction &MF = MIRBuilder.getMF();
75 MachineRegisterInfo &MRI = MF.getRegInfo();
76 SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
77 MFI->setIfReturnsVoid(!Val);
80 MIRBuilder.buildInstr(AMDGPU::S_ENDPGM).addImm(0);
84 Register VReg = VRegs[0];
86 const Function &F = MF.getFunction();
87 auto &DL = F.getParent()->getDataLayout();
88 if (!AMDGPU::isShader(F.getCallingConv()))
92 const AMDGPUTargetLowering &TLI = *getTLI<AMDGPUTargetLowering>();
93 SmallVector<EVT, 4> SplitVTs;
94 SmallVector<uint64_t, 4> Offsets;
95 ArgInfo OrigArg{VReg, Val->getType()};
96 setArgFlags(OrigArg, AttributeList::ReturnIndex, DL, F);
97 ComputeValueVTs(TLI, DL, OrigArg.Ty, SplitVTs, &Offsets, 0);
99 SmallVector<ArgInfo, 8> SplitArgs;
100 CCAssignFn *AssignFn = CCAssignFnForReturn(F.getCallingConv(), false);
101 for (unsigned i = 0, e = Offsets.size(); i != e; ++i) {
102 Type *SplitTy = SplitVTs[i].getTypeForEVT(F.getContext());
103 SplitArgs.push_back({VRegs[i], SplitTy, OrigArg.Flags, OrigArg.IsFixed});
105 auto RetInstr = MIRBuilder.buildInstrNoInsert(AMDGPU::SI_RETURN_TO_EPILOG);
106 OutgoingArgHandler Handler(MIRBuilder, MRI, RetInstr, AssignFn);
107 if (!handleAssignments(MIRBuilder, SplitArgs, Handler))
109 MIRBuilder.insertInstr(RetInstr);
114 Register AMDGPUCallLowering::lowerParameterPtr(MachineIRBuilder &MIRBuilder,
116 uint64_t Offset) const {
118 MachineFunction &MF = MIRBuilder.getMF();
119 const SIMachineFunctionInfo *MFI = MF.getInfo<SIMachineFunctionInfo>();
120 MachineRegisterInfo &MRI = MF.getRegInfo();
121 const Function &F = MF.getFunction();
122 const DataLayout &DL = F.getParent()->getDataLayout();
123 PointerType *PtrTy = PointerType::get(ParamTy, AMDGPUAS::CONSTANT_ADDRESS);
124 LLT PtrType = getLLTForType(*PtrTy, DL);
125 Register DstReg = MRI.createGenericVirtualRegister(PtrType);
126 Register KernArgSegmentPtr =
127 MFI->getPreloadedReg(AMDGPUFunctionArgInfo::KERNARG_SEGMENT_PTR);
128 Register KernArgSegmentVReg = MRI.getLiveInVirtReg(KernArgSegmentPtr);
130 Register OffsetReg = MRI.createGenericVirtualRegister(LLT::scalar(64));
131 MIRBuilder.buildConstant(OffsetReg, Offset);
133 MIRBuilder.buildGEP(DstReg, KernArgSegmentVReg, OffsetReg);
138 void AMDGPUCallLowering::lowerParameter(MachineIRBuilder &MIRBuilder,
139 Type *ParamTy, uint64_t Offset,
141 Register DstReg) const {
142 MachineFunction &MF = MIRBuilder.getMF();
143 const Function &F = MF.getFunction();
144 const DataLayout &DL = F.getParent()->getDataLayout();
145 PointerType *PtrTy = PointerType::get(ParamTy, AMDGPUAS::CONSTANT_ADDRESS);
146 MachinePointerInfo PtrInfo(UndefValue::get(PtrTy));
147 unsigned TypeSize = DL.getTypeStoreSize(ParamTy);
148 Register PtrReg = lowerParameterPtr(MIRBuilder, ParamTy, Offset);
150 MachineMemOperand *MMO =
151 MF.getMachineMemOperand(PtrInfo, MachineMemOperand::MOLoad |
152 MachineMemOperand::MONonTemporal |
153 MachineMemOperand::MOInvariant,
156 MIRBuilder.buildLoad(DstReg, PtrReg, *MMO);
159 static Register findFirstFreeSGPR(CCState &CCInfo) {
160 unsigned NumSGPRs = AMDGPU::SGPR_32RegClass.getNumRegs();
161 for (unsigned Reg = 0; Reg < NumSGPRs; ++Reg) {
162 if (!CCInfo.isAllocated(AMDGPU::SGPR0 + Reg)) {
163 return AMDGPU::SGPR0 + Reg;
166 llvm_unreachable("Cannot allocate sgpr");
169 static void allocateSystemSGPRs(CCState &CCInfo,
171 SIMachineFunctionInfo &Info,
172 CallingConv::ID CallConv,
174 if (Info.hasPrivateSegmentWaveByteOffset()) {
175 // Scratch wave offset passed in system SGPR.
176 unsigned PrivateSegmentWaveByteOffsetReg;
179 PrivateSegmentWaveByteOffsetReg =
180 Info.getPrivateSegmentWaveByteOffsetSystemSGPR();
182 // This is true if the scratch wave byte offset doesn't have a fixed
184 if (PrivateSegmentWaveByteOffsetReg == AMDGPU::NoRegister) {
185 PrivateSegmentWaveByteOffsetReg = findFirstFreeSGPR(CCInfo);
186 Info.setPrivateSegmentWaveByteOffset(PrivateSegmentWaveByteOffsetReg);
189 PrivateSegmentWaveByteOffsetReg = Info.addPrivateSegmentWaveByteOffset();
191 MF.addLiveIn(PrivateSegmentWaveByteOffsetReg, &AMDGPU::SGPR_32RegClass);
192 CCInfo.AllocateReg(PrivateSegmentWaveByteOffsetReg);
196 bool AMDGPUCallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder,
198 ArrayRef<Register> VRegs) const {
199 // AMDGPU_GS and AMDGP_HS are not supported yet.
200 if (F.getCallingConv() == CallingConv::AMDGPU_GS ||
201 F.getCallingConv() == CallingConv::AMDGPU_HS)
204 MachineFunction &MF = MIRBuilder.getMF();
205 const GCNSubtarget *Subtarget = &MF.getSubtarget<GCNSubtarget>();
206 MachineRegisterInfo &MRI = MF.getRegInfo();
207 SIMachineFunctionInfo *Info = MF.getInfo<SIMachineFunctionInfo>();
208 const SIRegisterInfo *TRI = MF.getSubtarget<GCNSubtarget>().getRegisterInfo();
209 const DataLayout &DL = F.getParent()->getDataLayout();
211 bool IsShader = AMDGPU::isShader(F.getCallingConv());
213 SmallVector<CCValAssign, 16> ArgLocs;
214 CCState CCInfo(F.getCallingConv(), F.isVarArg(), MF, ArgLocs, F.getContext());
216 // FIXME: How should these inputs interact with inreg / custom SGPR inputs?
217 if (Info->hasPrivateSegmentBuffer()) {
218 Register PrivateSegmentBufferReg = Info->addPrivateSegmentBuffer(*TRI);
219 MF.addLiveIn(PrivateSegmentBufferReg, &AMDGPU::SReg_128RegClass);
220 CCInfo.AllocateReg(PrivateSegmentBufferReg);
223 if (Info->hasDispatchPtr()) {
224 Register DispatchPtrReg = Info->addDispatchPtr(*TRI);
225 // FIXME: Need to add reg as live-in
226 CCInfo.AllocateReg(DispatchPtrReg);
229 if (Info->hasQueuePtr()) {
230 Register QueuePtrReg = Info->addQueuePtr(*TRI);
231 // FIXME: Need to add reg as live-in
232 CCInfo.AllocateReg(QueuePtrReg);
235 if (Info->hasKernargSegmentPtr()) {
236 Register InputPtrReg = Info->addKernargSegmentPtr(*TRI);
237 const LLT P2 = LLT::pointer(AMDGPUAS::CONSTANT_ADDRESS, 64);
238 Register VReg = MRI.createGenericVirtualRegister(P2);
239 MRI.addLiveIn(InputPtrReg, VReg);
240 MIRBuilder.getMBB().addLiveIn(InputPtrReg);
241 MIRBuilder.buildCopy(VReg, InputPtrReg);
242 CCInfo.AllocateReg(InputPtrReg);
245 if (Info->hasDispatchID()) {
246 unsigned DispatchIDReg = Info->addDispatchID(*TRI);
247 // FIXME: Need to add reg as live-in
248 CCInfo.AllocateReg(DispatchIDReg);
251 if (Info->hasFlatScratchInit()) {
252 unsigned FlatScratchInitReg = Info->addFlatScratchInit(*TRI);
253 // FIXME: Need to add reg as live-in
254 CCInfo.AllocateReg(FlatScratchInitReg);
257 // The infrastructure for normal calling convention lowering is essentially
258 // useless for kernels. We want to avoid any kind of legalization or argument
260 if (F.getCallingConv() == CallingConv::AMDGPU_KERNEL) {
262 const unsigned KernArgBaseAlign = 16;
263 const unsigned BaseOffset = Subtarget->getExplicitKernelArgOffset(F);
264 uint64_t ExplicitArgOffset = 0;
266 // TODO: Align down to dword alignment and extract bits for extending loads.
267 for (auto &Arg : F.args()) {
268 Type *ArgTy = Arg.getType();
269 unsigned AllocSize = DL.getTypeAllocSize(ArgTy);
273 unsigned ABIAlign = DL.getABITypeAlignment(ArgTy);
275 uint64_t ArgOffset = alignTo(ExplicitArgOffset, ABIAlign) + BaseOffset;
276 ExplicitArgOffset = alignTo(ExplicitArgOffset, ABIAlign) + AllocSize;
278 unsigned Align = MinAlign(KernArgBaseAlign, ArgOffset);
279 ArgOffset = alignTo(ArgOffset, DL.getABITypeAlignment(ArgTy));
280 lowerParameter(MIRBuilder, ArgTy, ArgOffset, Align, VRegs[i]);
284 allocateSystemSGPRs(CCInfo, MF, *Info, F.getCallingConv(), IsShader);
288 unsigned NumArgs = F.arg_size();
289 Function::const_arg_iterator CurOrigArg = F.arg_begin();
290 const AMDGPUTargetLowering &TLI = *getTLI<AMDGPUTargetLowering>();
291 unsigned PSInputNum = 0;
292 BitVector Skipped(NumArgs);
293 for (unsigned i = 0; i != NumArgs; ++i, ++CurOrigArg) {
294 EVT ValEVT = TLI.getValueType(DL, CurOrigArg->getType());
296 // We can only hanlde simple value types at the moment.
297 ISD::ArgFlagsTy Flags;
298 ArgInfo OrigArg{VRegs[i], CurOrigArg->getType()};
299 setArgFlags(OrigArg, i + 1, DL, F);
300 Flags.setOrigAlign(DL.getABITypeAlignment(CurOrigArg->getType()));
302 if (F.getCallingConv() == CallingConv::AMDGPU_PS &&
303 !OrigArg.Flags.isInReg() && !OrigArg.Flags.isByVal() &&
305 if (CurOrigArg->use_empty() && !Info->isPSInputAllocated(PSInputNum)) {
311 Info->markPSInputAllocated(PSInputNum);
312 if (!CurOrigArg->use_empty())
313 Info->markPSInputEnabled(PSInputNum);
318 CCAssignFn *AssignFn = CCAssignFnForCall(F.getCallingConv(),
321 if (ValEVT.isVector()) {
322 EVT ElemVT = ValEVT.getVectorElementType();
323 if (!ValEVT.isSimple())
325 MVT ValVT = ElemVT.getSimpleVT();
326 bool Res = AssignFn(i, ValVT, ValVT, CCValAssign::Full,
327 OrigArg.Flags, CCInfo);
331 MVT ValVT = ValEVT.getSimpleVT();
332 if (!ValEVT.isSimple())
335 AssignFn(i, ValVT, ValVT, CCValAssign::Full, OrigArg.Flags, CCInfo);
337 // Fail if we don't know how to handle this type.
343 Function::const_arg_iterator Arg = F.arg_begin();
345 if (F.getCallingConv() == CallingConv::AMDGPU_VS ||
346 F.getCallingConv() == CallingConv::AMDGPU_PS) {
347 for (unsigned i = 0, OrigArgIdx = 0;
348 OrigArgIdx != NumArgs && i != ArgLocs.size(); ++Arg, ++OrigArgIdx) {
349 if (Skipped.test(OrigArgIdx))
351 CCValAssign &VA = ArgLocs[i++];
352 MRI.addLiveIn(VA.getLocReg(), VRegs[OrigArgIdx]);
353 MIRBuilder.getMBB().addLiveIn(VA.getLocReg());
354 MIRBuilder.buildCopy(VRegs[OrigArgIdx], VA.getLocReg());
357 allocateSystemSGPRs(CCInfo, MF, *Info, F.getCallingConv(), IsShader);