1 ; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=tahiti -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,SI,SIVI %s
2 ; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=fiji -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,VI,GFX89,SIVI %s
3 ; RUN: llc -amdgpu-scalarize-global-loads=false -march=amdgcn -mcpu=gfx900 -mattr=-flat-for-global -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefixes=GCN,GFX9,GFX89 %s
5 ; GCN-LABEL: {{^}}fmul_f16
6 ; GCN: buffer_load_ushort v[[A_F16:[0-9]+]]
7 ; GCN: buffer_load_ushort v[[B_F16:[0-9]+]]
8 ; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]]
9 ; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]]
10 ; SI: v_mul_f32_e32 v[[R_F32:[0-9]+]], v[[A_F32]], v[[B_F32]]
11 ; SI: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]]
12 ; GFX89: v_mul_f16_e32 v[[R_F16:[0-9]+]], v[[A_F16]], v[[B_F16]]
13 ; GCN: buffer_store_short v[[R_F16]]
15 define amdgpu_kernel void @fmul_f16(
16 half addrspace(1)* %r,
17 half addrspace(1)* %a,
18 half addrspace(1)* %b) {
20 %a.val = load half, half addrspace(1)* %a
21 %b.val = load half, half addrspace(1)* %b
22 %r.val = fmul half %a.val, %b.val
23 store half %r.val, half addrspace(1)* %r
27 ; GCN-LABEL: {{^}}fmul_f16_imm_a
28 ; GCN: buffer_load_ushort v[[B_F16:[0-9]+]]
29 ; SI: v_cvt_f32_f16_e32 v[[B_F32:[0-9]+]], v[[B_F16]]
30 ; SI: v_mul_f32_e32 v[[R_F32:[0-9]+]], 0x40400000, v[[B_F32]]
31 ; SI: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]]
32 ; GFX89: v_mul_f16_e32 v[[R_F16:[0-9]+]], 0x4200, v[[B_F16]]
33 ; GCN: buffer_store_short v[[R_F16]]
35 define amdgpu_kernel void @fmul_f16_imm_a(
36 half addrspace(1)* %r,
37 half addrspace(1)* %b) {
39 %b.val = load half, half addrspace(1)* %b
40 %r.val = fmul half 3.0, %b.val
41 store half %r.val, half addrspace(1)* %r
45 ; GCN-LABEL: {{^}}fmul_f16_imm_b
46 ; GCN: buffer_load_ushort v[[A_F16:[0-9]+]]
47 ; SI: v_cvt_f32_f16_e32 v[[A_F32:[0-9]+]], v[[A_F16]]
48 ; SI: v_mul_f32_e32 v[[R_F32:[0-9]+]], 4.0, v[[A_F32]]
49 ; SI: v_cvt_f16_f32_e32 v[[R_F16:[0-9]+]], v[[R_F32]]
51 ; GFX89: v_mul_f16_e32 v[[R_F16:[0-9]+]], 4.0, v[[A_F16]]
52 ; GCN: buffer_store_short v[[R_F16]]
54 define amdgpu_kernel void @fmul_f16_imm_b(
55 half addrspace(1)* %r,
56 half addrspace(1)* %a) {
58 %a.val = load half, half addrspace(1)* %a
59 %r.val = fmul half %a.val, 4.0
60 store half %r.val, half addrspace(1)* %r
64 ; GCN-LABEL: {{^}}fmul_v2f16:
65 ; GCN: buffer_load_dword v[[A_V2_F16:[0-9]+]]
66 ; GCN: buffer_load_dword v[[B_V2_F16:[0-9]+]]
68 ; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]]
69 ; SI: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]]
70 ; SI: v_cvt_f32_f16_e32 v[[B_F32_0:[0-9]+]], v[[B_V2_F16]]
71 ; SI: v_lshrrev_b32_e32 v[[B_F16_1:[0-9]+]], 16, v[[B_V2_F16]]
72 ; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]]
73 ; SI: v_cvt_f32_f16_e32 v[[B_F32_1:[0-9]+]], v[[B_F16_1]]
74 ; SI: v_mul_f32_e32 v[[R_F32_0:[0-9]+]], v[[A_F32_0]], v[[B_F32_0]]
75 ; SI: v_mul_f32_e32 v[[R_F32_1:[0-9]+]], v[[A_F32_1]], v[[B_F32_1]]
76 ; SI-DAG: v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[R_F32_0]]
77 ; SI-DAG: v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[R_F32_1]]
78 ; SI-DAG: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]]
79 ; SI: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_0]], v[[R_F16_HI]]
81 ; VI-DAG: v_mul_f16_e32 v[[R_F16_LO:[0-9]+]], v[[A_V2_F16]], v[[B_V2_F16]]
82 ; VI-DAG: v_mul_f16_sdwa v[[R_F16_HI:[0-9]+]], v[[A_V2_F16]], v[[B_V2_F16]] dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
83 ; VI: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_LO]], v[[R_F16_HI]]
85 ; GFX9: v_pk_mul_f16 v[[R_V2_F16:[0-9]+]], v[[A_V2_F16]], v[[B_V2_F16]]
87 ; GCN: buffer_store_dword v[[R_V2_F16]]
89 define amdgpu_kernel void @fmul_v2f16(
90 <2 x half> addrspace(1)* %r,
91 <2 x half> addrspace(1)* %a,
92 <2 x half> addrspace(1)* %b) {
94 %a.val = load <2 x half>, <2 x half> addrspace(1)* %a
95 %b.val = load <2 x half>, <2 x half> addrspace(1)* %b
96 %r.val = fmul <2 x half> %a.val, %b.val
97 store <2 x half> %r.val, <2 x half> addrspace(1)* %r
101 ; GCN-LABEL: {{^}}fmul_v2f16_imm_a:
102 ; GCN-DAG: buffer_load_dword v[[B_V2_F16:[0-9]+]]
103 ; SI: v_cvt_f32_f16_e32 v[[B_F32_0:[0-9]+]], v[[B_V2_F16]]
104 ; SI: v_lshrrev_b32_e32 v[[B_F16_1:[0-9]+]], 16, v[[B_V2_F16]]
105 ; SI: v_cvt_f32_f16_e32 v[[B_F32_1:[0-9]+]], v[[B_F16_1]]
106 ; SI: v_mul_f32_e32 v[[R_F32_0:[0-9]+]], 0x40400000, v[[B_F32_0]]
107 ; SI: v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[R_F32_0]]
108 ; SI: v_mul_f32_e32 v[[R_F32_1:[0-9]+]], 4.0, v[[B_F32_1]]
109 ; SI: v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[R_F32_1]]
112 ; VI-DAG: v_mov_b32_e32 v[[CONST4:[0-9]+]], 0x4400
113 ; VI-DAG: v_mul_f16_sdwa v[[R_F16_HI:[0-9]+]], v[[B_V2_F16]], v[[CONST4]] dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
114 ; VI-DAG: v_mul_f16_e32 v[[R_F16_0:[0-9]+]], 0x4200, v[[B_V2_F16]]
116 ; GFX9: s_mov_b32 [[K:s[0-9]+]], 0x44004200
117 ; GFX9: v_pk_mul_f16 v[[R_V2_F16:[0-9]+]], v[[B_V2_F16]], [[K]]
119 ; SI-DAG: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]]
120 ; SIVI: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_0]], v[[R_F16_HI]]
122 ; GCN: buffer_store_dword v[[R_V2_F16]]
124 define amdgpu_kernel void @fmul_v2f16_imm_a(
125 <2 x half> addrspace(1)* %r,
126 <2 x half> addrspace(1)* %b) {
128 %b.val = load <2 x half>, <2 x half> addrspace(1)* %b
129 %r.val = fmul <2 x half> <half 3.0, half 4.0>, %b.val
130 store <2 x half> %r.val, <2 x half> addrspace(1)* %r
134 ; GCN-LABEL: {{^}}fmul_v2f16_imm_b:
135 ; GCN-DAG: buffer_load_dword v[[A_V2_F16:[0-9]+]]
136 ; SI: v_cvt_f32_f16_e32 v[[A_F32_0:[0-9]+]], v[[A_V2_F16]]
137 ; SI: v_lshrrev_b32_e32 v[[A_F16_1:[0-9]+]], 16, v[[A_V2_F16]]
138 ; SI: v_cvt_f32_f16_e32 v[[A_F32_1:[0-9]+]], v[[A_F16_1]]
139 ; SI: v_mul_f32_e32 v[[R_F32_0:[0-9]+]], 4.0, v[[A_F32_0]]
140 ; SI: v_cvt_f16_f32_e32 v[[R_F16_0:[0-9]+]], v[[R_F32_0]]
141 ; SI: v_mul_f32_e32 v[[R_F32_1:[0-9]+]], 0x40400000, v[[A_F32_1]]
142 ; SI: v_cvt_f16_f32_e32 v[[R_F16_1:[0-9]+]], v[[R_F32_1]]
144 ; VI-DAG: v_mov_b32_e32 v[[CONST3:[0-9]+]], 0x4200
145 ; VI-DAG: v_mul_f16_sdwa v[[R_F16_HI:[0-9]+]], v[[A_V2_F16]], v[[CONST3]] dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
146 ; VI-DAG: v_mul_f16_e32 v[[R_F16_0:[0-9]+]], 4.0, v[[A_V2_F16]]
148 ; GFX9: s_mov_b32 [[K:s[0-9]+]], 0x42004400
149 ; GFX9: v_pk_mul_f16 v[[R_V2_F16:[0-9]+]], v[[A_V2_F16]], [[K]]
151 ; SI-DAG: v_lshlrev_b32_e32 v[[R_F16_HI:[0-9]+]], 16, v[[R_F16_1]]
152 ; SIVI: v_or_b32_e32 v[[R_V2_F16:[0-9]+]], v[[R_F16_0]], v[[R_F16_HI]]
154 ; GCN: buffer_store_dword v[[R_V2_F16]]
156 define amdgpu_kernel void @fmul_v2f16_imm_b(
157 <2 x half> addrspace(1)* %r,
158 <2 x half> addrspace(1)* %a) {
160 %a.val = load <2 x half>, <2 x half> addrspace(1)* %a
161 %r.val = fmul <2 x half> %a.val, <half 4.0, half 3.0>
162 store <2 x half> %r.val, <2 x half> addrspace(1)* %r
166 ; GCN-LABEL: {{^}}fmul_v4f16:
167 ; GFX89: buffer_load_dwordx2 v{{\[}}[[A_LO:[0-9]+]]:[[A_HI:[0-9]+]]{{\]}}
168 ; GFX89: buffer_load_dwordx2 v{{\[}}[[B_LO:[0-9]+]]:[[B_HI:[0-9]+]]{{\]}}
170 ; GFX9-DAG: v_pk_mul_f16 v[[MUL_LO:[0-9]+]], v[[A_LO]], v[[B_LO]]
171 ; GFX9-DAG: v_pk_mul_f16 v[[MUL_HI:[0-9]+]], v[[A_HI]], v[[B_HI]]
172 ; GFX9: buffer_store_dwordx2 v{{\[}}[[MUL_LO]]:[[MUL_HI]]{{\]}}
180 define amdgpu_kernel void @fmul_v4f16(
181 <4 x half> addrspace(1)* %r,
182 <4 x half> addrspace(1)* %a,
183 <4 x half> addrspace(1)* %b) {
185 %a.val = load <4 x half>, <4 x half> addrspace(1)* %a
186 %b.val = load <4 x half>, <4 x half> addrspace(1)* %b
187 %r.val = fmul <4 x half> %a.val, %b.val
188 store <4 x half> %r.val, <4 x half> addrspace(1)* %r
192 ; GCN-LABEL: {{^}}fmul_v4f16_imm_a:
193 ; GFX89-DAG: buffer_load_dwordx2 v{{\[}}[[A_LO:[0-9]+]]:[[A_HI:[0-9]+]]{{\]}}
194 ; GFX9-DAG: s_mov_b32 [[K1:s[0-9]+]], 0x44004200
195 ; GFX9-DAG: s_mov_b32 [[K0:s[0-9]+]], 0x40004800
197 ; GFX9-DAG: v_pk_mul_f16 v[[MUL_LO:[0-9]+]], v[[A_LO]], [[K0]]
198 ; GFX9-DAG: v_pk_mul_f16 v[[MUL_HI:[0-9]+]], v[[A_HI]], [[K1]]
199 ; GFX9: buffer_store_dwordx2 v{{\[}}[[MUL_LO]]:[[MUL_HI]]{{\]}}
201 ; VI-DAG: v_mov_b32_e32 [[K4:v[0-9]+]], 0x4400
203 ; VI-DAG: v_mul_f16_sdwa v[[MUL_HI_HI:[0-9]+]], v[[A_HI]], [[K4]] dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD
204 ; VI-DAG: v_mul_f16_e32 v[[MUL_HI_LO:[0-9]+]], 0x4200, v[[A_HI]]
205 ; VI-DAG: v_add_f16_sdwa v[[MUL_LO_HI:[0-9]+]], v[[A_LO]], v[[A_LO]] dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
206 ; VI-DAG: v_mul_f16_e32 v[[MUL_LO_LO:[0-9]+]], 0x4800, v[[A_LO]]
208 ; VI-DAG: v_or_b32_e32 v[[OR0:[0-9]+]], v[[MUL_LO_LO]], v[[MUL_LO_HI]]
209 ; VI-DAG: v_or_b32_e32 v[[OR1:[0-9]+]], v[[MUL_HI_LO]], v[[MUL_HI_HI]]
211 ; VI: buffer_store_dwordx2 v{{\[}}[[OR0]]:[[OR1]]{{\]}}
212 define amdgpu_kernel void @fmul_v4f16_imm_a(
213 <4 x half> addrspace(1)* %r,
214 <4 x half> addrspace(1)* %b) {
216 %b.val = load <4 x half>, <4 x half> addrspace(1)* %b
217 %r.val = fmul <4 x half> <half 8.0, half 2.0, half 3.0, half 4.0>, %b.val
218 store <4 x half> %r.val, <4 x half> addrspace(1)* %r