1 //===-- X86InstrAVX512.td - AVX512 Instruction Set ---------*- tablegen -*-===//
3 // The LLVM Compiler Infrastructure
5 // This file is distributed under the University of Illinois Open Source
6 // License. See LICENSE.TXT for details.
8 //===----------------------------------------------------------------------===//
10 // This file describes the X86 AVX512 instruction set, defining the
11 // instructions, and properties of the instructions which are needed for code
12 // generation, machine code emission, and analysis.
14 //===----------------------------------------------------------------------===//
16 // Group template arguments that can be derived from the vector type (EltNum x
17 // EltVT). These are things like the register class for the writemask, etc.
18 // The idea is to pass one of these as the template argument rather than the
19 // individual arguments.
20 // The template is also used for scalar types, in this case numelts is 1.
21 class X86VectorVTInfo<int numelts, ValueType eltvt, RegisterClass rc,
23 RegisterClass RC = rc;
24 ValueType EltVT = eltvt;
25 int NumElts = numelts;
27 // Corresponding mask register class.
28 RegisterClass KRC = !cast<RegisterClass>("VK" # NumElts);
30 // Corresponding write-mask register class.
31 RegisterClass KRCWM = !cast<RegisterClass>("VK" # NumElts # "WM");
34 ValueType KVT = !cast<ValueType>("v" # NumElts # "i1");
36 // Suffix used in the instruction mnemonic.
37 string Suffix = suffix;
39 // VTName is a string name for vector VT. For vector types it will be
40 // v # NumElts # EltVT, so for vector of 8 elements of i32 it will be v8i32
41 // It is a little bit complex for scalar types, where NumElts = 1.
42 // In this case we build v4f32 or v2f64
43 string VTName = "v" # !if (!eq (NumElts, 1),
44 !if (!eq (EltVT.Size, 32), 4,
45 !if (!eq (EltVT.Size, 64), 2, NumElts)), NumElts) # EltVT;
48 ValueType VT = !cast<ValueType>(VTName);
50 string EltTypeName = !cast<string>(EltVT);
51 // Size of the element type in bits, e.g. 32 for v16i32.
52 string EltSizeName = !subst("i", "", !subst("f", "", EltTypeName));
53 int EltSize = EltVT.Size;
55 // "i" for integer types and "f" for floating-point types
56 string TypeVariantName = !subst(EltSizeName, "", EltTypeName);
58 // Size of RC in bits, e.g. 512 for VR512.
61 // The corresponding memory operand, e.g. i512mem for VR512.
62 X86MemOperand MemOp = !cast<X86MemOperand>(TypeVariantName # Size # "mem");
63 X86MemOperand ScalarMemOp = !cast<X86MemOperand>(EltVT # "mem");
64 // FP scalar memory operand for intrinsics - ssmem/sdmem.
65 Operand IntScalarMemOp = !if (!eq (EltTypeName, "f32"), !cast<Operand>("ssmem"),
66 !if (!eq (EltTypeName, "f64"), !cast<Operand>("sdmem"), ?));
69 // Note: For 128/256-bit integer VT we choose loadv2i64/loadv4i64
70 // due to load promotion during legalization
71 PatFrag LdFrag = !cast<PatFrag>("load" #
72 !if (!eq (TypeVariantName, "i"),
73 !if (!eq (Size, 128), "v2i64",
74 !if (!eq (Size, 256), "v4i64",
75 !if (!eq (Size, 512), "v8i64",
78 PatFrag AlignedLdFrag = !cast<PatFrag>("alignedload" #
79 !if (!eq (TypeVariantName, "i"),
80 !if (!eq (Size, 128), "v2i64",
81 !if (!eq (Size, 256), "v4i64",
82 !if (!eq (Size, 512), "v8i64",
85 PatFrag ScalarLdFrag = !cast<PatFrag>("load" # EltVT);
87 ComplexPattern ScalarIntMemCPat = !if (!eq (EltTypeName, "f32"),
88 !cast<ComplexPattern>("sse_load_f32"),
89 !if (!eq (EltTypeName, "f64"),
90 !cast<ComplexPattern>("sse_load_f64"),
93 // The corresponding float type, e.g. v16f32 for v16i32
94 // Note: For EltSize < 32, FloatVT is illegal and TableGen
95 // fails to compile, so we choose FloatVT = VT
96 ValueType FloatVT = !cast<ValueType>(
97 !if (!eq (!srl(EltSize,5),0),
99 !if (!eq(TypeVariantName, "i"),
100 "v" # NumElts # "f" # EltSize,
103 ValueType IntVT = !cast<ValueType>(
104 !if (!eq (!srl(EltSize,5),0),
106 !if (!eq(TypeVariantName, "f"),
107 "v" # NumElts # "i" # EltSize,
109 // The string to specify embedded broadcast in assembly.
110 string BroadcastStr = "{1to" # NumElts # "}";
112 // 8-bit compressed displacement tuple/subvector format. This is only
113 // defined for NumElts <= 8.
114 CD8VForm CD8TupleForm = !if (!eq (!srl(NumElts, 4), 0),
115 !cast<CD8VForm>("CD8VT" # NumElts), ?);
117 SubRegIndex SubRegIdx = !if (!eq (Size, 128), sub_xmm,
118 !if (!eq (Size, 256), sub_ymm, ?));
120 Domain ExeDomain = !if (!eq (EltTypeName, "f32"), SSEPackedSingle,
121 !if (!eq (EltTypeName, "f64"), SSEPackedDouble,
124 RegisterClass FRC = !if (!eq (EltTypeName, "f32"), FR32X, FR64X);
126 // A vector tye of the same width with element type i64. This is used to
127 // create patterns for logic ops.
128 ValueType i64VT = !cast<ValueType>("v" # !srl(Size, 6) # "i64");
130 // A vector type of the same width with element type i32. This is used to
131 // create the canonical constant zero node ImmAllZerosV.
132 ValueType i32VT = !cast<ValueType>("v" # !srl(Size, 5) # "i32");
133 dag ImmAllZerosV = (VT (bitconvert (i32VT immAllZerosV)));
135 string ZSuffix = !if (!eq (Size, 128), "Z128",
136 !if (!eq (Size, 256), "Z256", "Z"));
139 def v64i8_info : X86VectorVTInfo<64, i8, VR512, "b">;
140 def v32i16_info : X86VectorVTInfo<32, i16, VR512, "w">;
141 def v16i32_info : X86VectorVTInfo<16, i32, VR512, "d">;
142 def v8i64_info : X86VectorVTInfo<8, i64, VR512, "q">;
143 def v16f32_info : X86VectorVTInfo<16, f32, VR512, "ps">;
144 def v8f64_info : X86VectorVTInfo<8, f64, VR512, "pd">;
146 // "x" in v32i8x_info means RC = VR256X
147 def v32i8x_info : X86VectorVTInfo<32, i8, VR256X, "b">;
148 def v16i16x_info : X86VectorVTInfo<16, i16, VR256X, "w">;
149 def v8i32x_info : X86VectorVTInfo<8, i32, VR256X, "d">;
150 def v4i64x_info : X86VectorVTInfo<4, i64, VR256X, "q">;
151 def v8f32x_info : X86VectorVTInfo<8, f32, VR256X, "ps">;
152 def v4f64x_info : X86VectorVTInfo<4, f64, VR256X, "pd">;
154 def v16i8x_info : X86VectorVTInfo<16, i8, VR128X, "b">;
155 def v8i16x_info : X86VectorVTInfo<8, i16, VR128X, "w">;
156 def v4i32x_info : X86VectorVTInfo<4, i32, VR128X, "d">;
157 def v2i64x_info : X86VectorVTInfo<2, i64, VR128X, "q">;
158 def v4f32x_info : X86VectorVTInfo<4, f32, VR128X, "ps">;
159 def v2f64x_info : X86VectorVTInfo<2, f64, VR128X, "pd">;
161 // We map scalar types to the smallest (128-bit) vector type
162 // with the appropriate element type. This allows to use the same masking logic.
163 def i32x_info : X86VectorVTInfo<1, i32, GR32, "si">;
164 def i64x_info : X86VectorVTInfo<1, i64, GR64, "sq">;
165 def f32x_info : X86VectorVTInfo<1, f32, VR128X, "ss">;
166 def f64x_info : X86VectorVTInfo<1, f64, VR128X, "sd">;
168 class AVX512VLVectorVTInfo<X86VectorVTInfo i512, X86VectorVTInfo i256,
169 X86VectorVTInfo i128> {
170 X86VectorVTInfo info512 = i512;
171 X86VectorVTInfo info256 = i256;
172 X86VectorVTInfo info128 = i128;
175 def avx512vl_i8_info : AVX512VLVectorVTInfo<v64i8_info, v32i8x_info,
177 def avx512vl_i16_info : AVX512VLVectorVTInfo<v32i16_info, v16i16x_info,
179 def avx512vl_i32_info : AVX512VLVectorVTInfo<v16i32_info, v8i32x_info,
181 def avx512vl_i64_info : AVX512VLVectorVTInfo<v8i64_info, v4i64x_info,
183 def avx512vl_f32_info : AVX512VLVectorVTInfo<v16f32_info, v8f32x_info,
185 def avx512vl_f64_info : AVX512VLVectorVTInfo<v8f64_info, v4f64x_info,
188 class X86KVectorVTInfo<RegisterClass _krc, RegisterClass _krcwm,
190 RegisterClass KRC = _krc;
191 RegisterClass KRCWM = _krcwm;
195 def v1i1_info : X86KVectorVTInfo<VK1, VK1WM, v1i1>;
196 def v2i1_info : X86KVectorVTInfo<VK2, VK2WM, v2i1>;
197 def v4i1_info : X86KVectorVTInfo<VK4, VK4WM, v4i1>;
198 def v8i1_info : X86KVectorVTInfo<VK8, VK8WM, v8i1>;
199 def v16i1_info : X86KVectorVTInfo<VK16, VK16WM, v16i1>;
200 def v32i1_info : X86KVectorVTInfo<VK32, VK32WM, v32i1>;
201 def v64i1_info : X86KVectorVTInfo<VK64, VK64WM, v64i1>;
203 // This multiclass generates the masking variants from the non-masking
204 // variant. It only provides the assembly pieces for the masking variants.
205 // It assumes custom ISel patterns for masking which can be provided as
206 // template arguments.
207 multiclass AVX512_maskable_custom<bits<8> O, Format F,
209 dag Ins, dag MaskingIns, dag ZeroMaskingIns,
211 string AttSrcAsm, string IntelSrcAsm,
213 list<dag> MaskingPattern,
214 list<dag> ZeroMaskingPattern,
215 string MaskingConstraint = "",
216 bit IsCommutable = 0,
217 bit IsKCommutable = 0> {
218 let isCommutable = IsCommutable in
219 def NAME: AVX512<O, F, Outs, Ins,
220 OpcodeStr#"\t{"#AttSrcAsm#", $dst|"#
221 "$dst, "#IntelSrcAsm#"}",
224 // Prefer over VMOV*rrk Pat<>
225 let isCommutable = IsKCommutable in
226 def NAME#k: AVX512<O, F, Outs, MaskingIns,
227 OpcodeStr#"\t{"#AttSrcAsm#", $dst {${mask}}|"#
228 "$dst {${mask}}, "#IntelSrcAsm#"}",
231 // In case of the 3src subclass this is overridden with a let.
232 string Constraints = MaskingConstraint;
235 // Zero mask does not add any restrictions to commute operands transformation.
236 // So, it is Ok to use IsCommutable instead of IsKCommutable.
237 let isCommutable = IsCommutable in // Prefer over VMOV*rrkz Pat<>
238 def NAME#kz: AVX512<O, F, Outs, ZeroMaskingIns,
239 OpcodeStr#"\t{"#AttSrcAsm#", $dst {${mask}} {z}|"#
240 "$dst {${mask}} {z}, "#IntelSrcAsm#"}",
246 // Common base class of AVX512_maskable and AVX512_maskable_3src.
247 multiclass AVX512_maskable_common<bits<8> O, Format F, X86VectorVTInfo _,
249 dag Ins, dag MaskingIns, dag ZeroMaskingIns,
251 string AttSrcAsm, string IntelSrcAsm,
252 dag RHS, dag MaskingRHS,
253 SDNode Select = vselect,
254 string MaskingConstraint = "",
255 bit IsCommutable = 0,
256 bit IsKCommutable = 0> :
257 AVX512_maskable_custom<O, F, Outs, Ins, MaskingIns, ZeroMaskingIns, OpcodeStr,
258 AttSrcAsm, IntelSrcAsm,
259 [(set _.RC:$dst, RHS)],
260 [(set _.RC:$dst, MaskingRHS)],
262 (Select _.KRCWM:$mask, RHS, _.ImmAllZerosV))],
263 MaskingConstraint, IsCommutable,
266 // This multiclass generates the unconditional/non-masking, the masking and
267 // the zero-masking variant of the vector instruction. In the masking case, the
268 // perserved vector elements come from a new dummy input operand tied to $dst.
269 // This version uses a separate dag for non-masking and masking.
270 multiclass AVX512_maskable_split<bits<8> O, Format F, X86VectorVTInfo _,
271 dag Outs, dag Ins, string OpcodeStr,
272 string AttSrcAsm, string IntelSrcAsm,
273 dag RHS, dag MaskRHS,
274 bit IsCommutable = 0, bit IsKCommutable = 0,
275 SDNode Select = vselect> :
276 AVX512_maskable_custom<O, F, Outs, Ins,
277 !con((ins _.RC:$src0, _.KRCWM:$mask), Ins),
278 !con((ins _.KRCWM:$mask), Ins),
279 OpcodeStr, AttSrcAsm, IntelSrcAsm,
280 [(set _.RC:$dst, RHS)],
282 (Select _.KRCWM:$mask, MaskRHS, _.RC:$src0))],
284 (Select _.KRCWM:$mask, MaskRHS, _.ImmAllZerosV))],
285 "$src0 = $dst", IsCommutable, IsKCommutable>;
287 // This multiclass generates the unconditional/non-masking, the masking and
288 // the zero-masking variant of the vector instruction. In the masking case, the
289 // perserved vector elements come from a new dummy input operand tied to $dst.
290 multiclass AVX512_maskable<bits<8> O, Format F, X86VectorVTInfo _,
291 dag Outs, dag Ins, string OpcodeStr,
292 string AttSrcAsm, string IntelSrcAsm,
294 bit IsCommutable = 0, bit IsKCommutable = 0,
295 SDNode Select = vselect> :
296 AVX512_maskable_common<O, F, _, Outs, Ins,
297 !con((ins _.RC:$src0, _.KRCWM:$mask), Ins),
298 !con((ins _.KRCWM:$mask), Ins),
299 OpcodeStr, AttSrcAsm, IntelSrcAsm, RHS,
300 (Select _.KRCWM:$mask, RHS, _.RC:$src0),
301 Select, "$src0 = $dst", IsCommutable, IsKCommutable>;
303 // This multiclass generates the unconditional/non-masking, the masking and
304 // the zero-masking variant of the scalar instruction.
305 multiclass AVX512_maskable_scalar<bits<8> O, Format F, X86VectorVTInfo _,
306 dag Outs, dag Ins, string OpcodeStr,
307 string AttSrcAsm, string IntelSrcAsm,
309 bit IsCommutable = 0> :
310 AVX512_maskable<O, F, _, Outs, Ins, OpcodeStr, AttSrcAsm, IntelSrcAsm,
311 RHS, IsCommutable, 0, X86selects>;
313 // Similar to AVX512_maskable but in this case one of the source operands
314 // ($src1) is already tied to $dst so we just use that for the preserved
315 // vector elements. NOTE that the NonTiedIns (the ins dag) should exclude
317 multiclass AVX512_maskable_3src<bits<8> O, Format F, X86VectorVTInfo _,
318 dag Outs, dag NonTiedIns, string OpcodeStr,
319 string AttSrcAsm, string IntelSrcAsm,
321 bit IsCommutable = 0,
322 bit IsKCommutable = 0,
323 SDNode Select = vselect,
325 AVX512_maskable_common<O, F, _, Outs,
326 !con((ins _.RC:$src1), NonTiedIns),
327 !con((ins _.RC:$src1, _.KRCWM:$mask), NonTiedIns),
328 !con((ins _.RC:$src1, _.KRCWM:$mask), NonTiedIns),
329 OpcodeStr, AttSrcAsm, IntelSrcAsm,
330 !if(MaskOnly, (null_frag), RHS),
331 (Select _.KRCWM:$mask, RHS, _.RC:$src1),
332 Select, "", IsCommutable, IsKCommutable>;
334 // Similar to AVX512_maskable_3src but in this case the input VT for the tied
335 // operand differs from the output VT. This requires a bitconvert on
336 // the preserved vector going into the vselect.
337 // NOTE: The unmasked pattern is disabled.
338 multiclass AVX512_maskable_3src_cast<bits<8> O, Format F, X86VectorVTInfo OutVT,
339 X86VectorVTInfo InVT,
340 dag Outs, dag NonTiedIns, string OpcodeStr,
341 string AttSrcAsm, string IntelSrcAsm,
342 dag RHS, bit IsCommutable = 0> :
343 AVX512_maskable_common<O, F, OutVT, Outs,
344 !con((ins InVT.RC:$src1), NonTiedIns),
345 !con((ins InVT.RC:$src1, InVT.KRCWM:$mask), NonTiedIns),
346 !con((ins InVT.RC:$src1, InVT.KRCWM:$mask), NonTiedIns),
347 OpcodeStr, AttSrcAsm, IntelSrcAsm, (null_frag),
348 (vselect InVT.KRCWM:$mask, RHS,
349 (bitconvert InVT.RC:$src1)),
350 vselect, "", IsCommutable>;
352 multiclass AVX512_maskable_3src_scalar<bits<8> O, Format F, X86VectorVTInfo _,
353 dag Outs, dag NonTiedIns, string OpcodeStr,
354 string AttSrcAsm, string IntelSrcAsm,
356 bit IsCommutable = 0,
357 bit IsKCommutable = 0,
359 AVX512_maskable_3src<O, F, _, Outs, NonTiedIns, OpcodeStr, AttSrcAsm,
360 IntelSrcAsm, RHS, IsCommutable, IsKCommutable,
361 X86selects, MaskOnly>;
363 multiclass AVX512_maskable_in_asm<bits<8> O, Format F, X86VectorVTInfo _,
366 string AttSrcAsm, string IntelSrcAsm,
368 AVX512_maskable_custom<O, F, Outs, Ins,
369 !con((ins _.RC:$src0, _.KRCWM:$mask), Ins),
370 !con((ins _.KRCWM:$mask), Ins),
371 OpcodeStr, AttSrcAsm, IntelSrcAsm, Pattern, [], [],
374 multiclass AVX512_maskable_3src_in_asm<bits<8> O, Format F, X86VectorVTInfo _,
375 dag Outs, dag NonTiedIns,
377 string AttSrcAsm, string IntelSrcAsm,
379 AVX512_maskable_custom<O, F, Outs,
380 !con((ins _.RC:$src1), NonTiedIns),
381 !con((ins _.RC:$src1, _.KRCWM:$mask), NonTiedIns),
382 !con((ins _.RC:$src1, _.KRCWM:$mask), NonTiedIns),
383 OpcodeStr, AttSrcAsm, IntelSrcAsm, Pattern, [], [],
386 // Instruction with mask that puts result in mask register,
387 // like "compare" and "vptest"
388 multiclass AVX512_maskable_custom_cmp<bits<8> O, Format F,
390 dag Ins, dag MaskingIns,
392 string AttSrcAsm, string IntelSrcAsm,
394 list<dag> MaskingPattern,
395 bit IsCommutable = 0> {
396 let isCommutable = IsCommutable in
397 def NAME: AVX512<O, F, Outs, Ins,
398 OpcodeStr#"\t{"#AttSrcAsm#", $dst|"#
399 "$dst, "#IntelSrcAsm#"}",
402 def NAME#k: AVX512<O, F, Outs, MaskingIns,
403 OpcodeStr#"\t{"#AttSrcAsm#", $dst {${mask}}|"#
404 "$dst {${mask}}, "#IntelSrcAsm#"}",
405 MaskingPattern>, EVEX_K;
408 multiclass AVX512_maskable_common_cmp<bits<8> O, Format F, X86VectorVTInfo _,
410 dag Ins, dag MaskingIns,
412 string AttSrcAsm, string IntelSrcAsm,
413 dag RHS, dag MaskingRHS,
414 bit IsCommutable = 0> :
415 AVX512_maskable_custom_cmp<O, F, Outs, Ins, MaskingIns, OpcodeStr,
416 AttSrcAsm, IntelSrcAsm,
417 [(set _.KRC:$dst, RHS)],
418 [(set _.KRC:$dst, MaskingRHS)], IsCommutable>;
420 multiclass AVX512_maskable_cmp<bits<8> O, Format F, X86VectorVTInfo _,
421 dag Outs, dag Ins, string OpcodeStr,
422 string AttSrcAsm, string IntelSrcAsm,
423 dag RHS, bit IsCommutable = 0> :
424 AVX512_maskable_common_cmp<O, F, _, Outs, Ins,
425 !con((ins _.KRCWM:$mask), Ins),
426 OpcodeStr, AttSrcAsm, IntelSrcAsm, RHS,
427 (and _.KRCWM:$mask, RHS), IsCommutable>;
429 multiclass AVX512_maskable_cmp_alt<bits<8> O, Format F, X86VectorVTInfo _,
430 dag Outs, dag Ins, string OpcodeStr,
431 string AttSrcAsm, string IntelSrcAsm> :
432 AVX512_maskable_custom_cmp<O, F, Outs,
433 Ins, !con((ins _.KRCWM:$mask),Ins), OpcodeStr,
434 AttSrcAsm, IntelSrcAsm, [], []>;
436 // This multiclass generates the unconditional/non-masking, the masking and
437 // the zero-masking variant of the vector instruction. In the masking case, the
438 // perserved vector elements come from a new dummy input operand tied to $dst.
439 multiclass AVX512_maskable_logic<bits<8> O, Format F, X86VectorVTInfo _,
440 dag Outs, dag Ins, string OpcodeStr,
441 string AttSrcAsm, string IntelSrcAsm,
442 dag RHS, dag MaskedRHS,
443 bit IsCommutable = 0, SDNode Select = vselect> :
444 AVX512_maskable_custom<O, F, Outs, Ins,
445 !con((ins _.RC:$src0, _.KRCWM:$mask), Ins),
446 !con((ins _.KRCWM:$mask), Ins),
447 OpcodeStr, AttSrcAsm, IntelSrcAsm,
448 [(set _.RC:$dst, RHS)],
450 (Select _.KRCWM:$mask, MaskedRHS, _.RC:$src0))],
452 (Select _.KRCWM:$mask, MaskedRHS,
454 "$src0 = $dst", IsCommutable>;
457 // Alias instruction that maps zero vector to pxor / xorp* for AVX-512.
458 // This is expanded by ExpandPostRAPseudos to an xorps / vxorps, and then
459 // swizzled by ExecutionDomainFix to pxor.
460 // We set canFoldAsLoad because this can be converted to a constant-pool
461 // load of an all-zeros value if folding it would be beneficial.
462 let isReMaterializable = 1, isAsCheapAsAMove = 1, canFoldAsLoad = 1,
463 isPseudo = 1, Predicates = [HasAVX512], SchedRW = [WriteZero] in {
464 def AVX512_512_SET0 : I<0, Pseudo, (outs VR512:$dst), (ins), "",
465 [(set VR512:$dst, (v16i32 immAllZerosV))]>;
466 def AVX512_512_SETALLONES : I<0, Pseudo, (outs VR512:$dst), (ins), "",
467 [(set VR512:$dst, (v16i32 immAllOnesV))]>;
470 // Alias instructions that allow VPTERNLOG to be used with a mask to create
471 // a mix of all ones and all zeros elements. This is done this way to force
472 // the same register to be used as input for all three sources.
473 let isPseudo = 1, Predicates = [HasAVX512], SchedRW = [WriteVecALU] in {
474 def AVX512_512_SEXT_MASK_32 : I<0, Pseudo, (outs VR512:$dst),
475 (ins VK16WM:$mask), "",
476 [(set VR512:$dst, (vselect (v16i1 VK16WM:$mask),
477 (v16i32 immAllOnesV),
478 (v16i32 immAllZerosV)))]>;
479 def AVX512_512_SEXT_MASK_64 : I<0, Pseudo, (outs VR512:$dst),
480 (ins VK8WM:$mask), "",
481 [(set VR512:$dst, (vselect (v8i1 VK8WM:$mask),
482 (bc_v8i64 (v16i32 immAllOnesV)),
483 (bc_v8i64 (v16i32 immAllZerosV))))]>;
486 let isReMaterializable = 1, isAsCheapAsAMove = 1, canFoldAsLoad = 1,
487 isPseudo = 1, Predicates = [HasAVX512], SchedRW = [WriteZero] in {
488 def AVX512_128_SET0 : I<0, Pseudo, (outs VR128X:$dst), (ins), "",
489 [(set VR128X:$dst, (v4i32 immAllZerosV))]>;
490 def AVX512_256_SET0 : I<0, Pseudo, (outs VR256X:$dst), (ins), "",
491 [(set VR256X:$dst, (v8i32 immAllZerosV))]>;
494 // Alias instructions that map fld0 to xorps for sse or vxorps for avx.
495 // This is expanded by ExpandPostRAPseudos.
496 let isReMaterializable = 1, isAsCheapAsAMove = 1, canFoldAsLoad = 1,
497 isPseudo = 1, SchedRW = [WriteZero], Predicates = [HasAVX512] in {
498 def AVX512_FsFLD0SS : I<0, Pseudo, (outs FR32X:$dst), (ins), "",
499 [(set FR32X:$dst, fp32imm0)]>;
500 def AVX512_FsFLD0SD : I<0, Pseudo, (outs FR64X:$dst), (ins), "",
501 [(set FR64X:$dst, fpimm0)]>;
504 //===----------------------------------------------------------------------===//
505 // AVX-512 - VECTOR INSERT
508 // Supports two different pattern operators for mask and unmasked ops. Allows
509 // null_frag to be passed for one.
510 multiclass vinsert_for_size_split<int Opcode, X86VectorVTInfo From,
512 SDPatternOperator vinsert_insert,
513 SDPatternOperator vinsert_for_mask,
514 X86FoldableSchedWrite sched> {
515 let hasSideEffects = 0, ExeDomain = To.ExeDomain in {
516 defm rr : AVX512_maskable_split<Opcode, MRMSrcReg, To, (outs To.RC:$dst),
517 (ins To.RC:$src1, From.RC:$src2, u8imm:$src3),
518 "vinsert" # From.EltTypeName # "x" # From.NumElts,
519 "$src3, $src2, $src1", "$src1, $src2, $src3",
520 (vinsert_insert:$src3 (To.VT To.RC:$src1),
521 (From.VT From.RC:$src2),
523 (vinsert_for_mask:$src3 (To.VT To.RC:$src1),
524 (From.VT From.RC:$src2),
526 AVX512AIi8Base, EVEX_4V, Sched<[sched]>;
528 defm rm : AVX512_maskable_split<Opcode, MRMSrcMem, To, (outs To.RC:$dst),
529 (ins To.RC:$src1, From.MemOp:$src2, u8imm:$src3),
530 "vinsert" # From.EltTypeName # "x" # From.NumElts,
531 "$src3, $src2, $src1", "$src1, $src2, $src3",
532 (vinsert_insert:$src3 (To.VT To.RC:$src1),
533 (From.VT (bitconvert (From.LdFrag addr:$src2))),
535 (vinsert_for_mask:$src3 (To.VT To.RC:$src1),
536 (From.VT (bitconvert (From.LdFrag addr:$src2))),
537 (iPTR imm))>, AVX512AIi8Base, EVEX_4V,
538 EVEX_CD8<From.EltSize, From.CD8TupleForm>,
539 Sched<[sched.Folded, ReadAfterLd]>;
543 // Passes the same pattern operator for masked and unmasked ops.
544 multiclass vinsert_for_size<int Opcode, X86VectorVTInfo From,
546 SDPatternOperator vinsert_insert,
547 X86FoldableSchedWrite sched> :
548 vinsert_for_size_split<Opcode, From, To, vinsert_insert, vinsert_insert, sched>;
550 multiclass vinsert_for_size_lowering<string InstrStr, X86VectorVTInfo From,
551 X86VectorVTInfo To, PatFrag vinsert_insert,
552 SDNodeXForm INSERT_get_vinsert_imm , list<Predicate> p> {
553 let Predicates = p in {
554 def : Pat<(vinsert_insert:$ins
555 (To.VT To.RC:$src1), (From.VT From.RC:$src2), (iPTR imm)),
556 (To.VT (!cast<Instruction>(InstrStr#"rr")
557 To.RC:$src1, From.RC:$src2,
558 (INSERT_get_vinsert_imm To.RC:$ins)))>;
560 def : Pat<(vinsert_insert:$ins
562 (From.VT (bitconvert (From.LdFrag addr:$src2))),
564 (To.VT (!cast<Instruction>(InstrStr#"rm")
565 To.RC:$src1, addr:$src2,
566 (INSERT_get_vinsert_imm To.RC:$ins)))>;
570 multiclass vinsert_for_type<ValueType EltVT32, int Opcode128,
571 ValueType EltVT64, int Opcode256,
572 X86FoldableSchedWrite sched> {
574 let Predicates = [HasVLX] in
575 defm NAME # "32x4Z256" : vinsert_for_size<Opcode128,
576 X86VectorVTInfo< 4, EltVT32, VR128X>,
577 X86VectorVTInfo< 8, EltVT32, VR256X>,
578 vinsert128_insert, sched>, EVEX_V256;
580 defm NAME # "32x4Z" : vinsert_for_size<Opcode128,
581 X86VectorVTInfo< 4, EltVT32, VR128X>,
582 X86VectorVTInfo<16, EltVT32, VR512>,
583 vinsert128_insert, sched>, EVEX_V512;
585 defm NAME # "64x4Z" : vinsert_for_size<Opcode256,
586 X86VectorVTInfo< 4, EltVT64, VR256X>,
587 X86VectorVTInfo< 8, EltVT64, VR512>,
588 vinsert256_insert, sched>, VEX_W, EVEX_V512;
590 // Even with DQI we'd like to only use these instructions for masking.
591 let Predicates = [HasVLX, HasDQI] in
592 defm NAME # "64x2Z256" : vinsert_for_size_split<Opcode128,
593 X86VectorVTInfo< 2, EltVT64, VR128X>,
594 X86VectorVTInfo< 4, EltVT64, VR256X>,
595 null_frag, vinsert128_insert, sched>,
598 // Even with DQI we'd like to only use these instructions for masking.
599 let Predicates = [HasDQI] in {
600 defm NAME # "64x2Z" : vinsert_for_size_split<Opcode128,
601 X86VectorVTInfo< 2, EltVT64, VR128X>,
602 X86VectorVTInfo< 8, EltVT64, VR512>,
603 null_frag, vinsert128_insert, sched>,
606 defm NAME # "32x8Z" : vinsert_for_size_split<Opcode256,
607 X86VectorVTInfo< 8, EltVT32, VR256X>,
608 X86VectorVTInfo<16, EltVT32, VR512>,
609 null_frag, vinsert256_insert, sched>,
614 // FIXME: Is there a better scheduler class for VINSERTF/VINSERTI?
615 defm VINSERTF : vinsert_for_type<f32, 0x18, f64, 0x1a, WriteFShuffle256>;
616 defm VINSERTI : vinsert_for_type<i32, 0x38, i64, 0x3a, WriteShuffle256>;
618 // Codegen pattern with the alternative types,
619 // Even with AVX512DQ we'll still use these for unmasked operations.
620 defm : vinsert_for_size_lowering<"VINSERTF32x4Z256", v2f64x_info, v4f64x_info,
621 vinsert128_insert, INSERT_get_vinsert128_imm, [HasVLX]>;
622 defm : vinsert_for_size_lowering<"VINSERTI32x4Z256", v2i64x_info, v4i64x_info,
623 vinsert128_insert, INSERT_get_vinsert128_imm, [HasVLX]>;
625 defm : vinsert_for_size_lowering<"VINSERTF32x4Z", v2f64x_info, v8f64_info,
626 vinsert128_insert, INSERT_get_vinsert128_imm, [HasAVX512]>;
627 defm : vinsert_for_size_lowering<"VINSERTI32x4Z", v2i64x_info, v8i64_info,
628 vinsert128_insert, INSERT_get_vinsert128_imm, [HasAVX512]>;
630 defm : vinsert_for_size_lowering<"VINSERTF64x4Z", v8f32x_info, v16f32_info,
631 vinsert256_insert, INSERT_get_vinsert256_imm, [HasAVX512]>;
632 defm : vinsert_for_size_lowering<"VINSERTI64x4Z", v8i32x_info, v16i32_info,
633 vinsert256_insert, INSERT_get_vinsert256_imm, [HasAVX512]>;
635 // Codegen pattern with the alternative types insert VEC128 into VEC256
636 defm : vinsert_for_size_lowering<"VINSERTI32x4Z256", v8i16x_info, v16i16x_info,
637 vinsert128_insert, INSERT_get_vinsert128_imm, [HasVLX]>;
638 defm : vinsert_for_size_lowering<"VINSERTI32x4Z256", v16i8x_info, v32i8x_info,
639 vinsert128_insert, INSERT_get_vinsert128_imm, [HasVLX]>;
640 // Codegen pattern with the alternative types insert VEC128 into VEC512
641 defm : vinsert_for_size_lowering<"VINSERTI32x4Z", v8i16x_info, v32i16_info,
642 vinsert128_insert, INSERT_get_vinsert128_imm, [HasAVX512]>;
643 defm : vinsert_for_size_lowering<"VINSERTI32x4Z", v16i8x_info, v64i8_info,
644 vinsert128_insert, INSERT_get_vinsert128_imm, [HasAVX512]>;
645 // Codegen pattern with the alternative types insert VEC256 into VEC512
646 defm : vinsert_for_size_lowering<"VINSERTI64x4Z", v16i16x_info, v32i16_info,
647 vinsert256_insert, INSERT_get_vinsert256_imm, [HasAVX512]>;
648 defm : vinsert_for_size_lowering<"VINSERTI64x4Z", v32i8x_info, v64i8_info,
649 vinsert256_insert, INSERT_get_vinsert256_imm, [HasAVX512]>;
652 multiclass vinsert_for_mask_cast<string InstrStr, X86VectorVTInfo From,
653 X86VectorVTInfo To, X86VectorVTInfo Cast,
654 PatFrag vinsert_insert,
655 SDNodeXForm INSERT_get_vinsert_imm,
657 let Predicates = p in {
659 (vselect Cast.KRCWM:$mask,
661 (vinsert_insert:$ins (To.VT To.RC:$src1),
662 (From.VT From.RC:$src2),
665 (!cast<Instruction>(InstrStr#"rrk")
666 Cast.RC:$src0, Cast.KRCWM:$mask, To.RC:$src1, From.RC:$src2,
667 (INSERT_get_vinsert_imm To.RC:$ins))>;
669 (vselect Cast.KRCWM:$mask,
671 (vinsert_insert:$ins (To.VT To.RC:$src1),
674 (From.LdFrag addr:$src2))),
677 (!cast<Instruction>(InstrStr#"rmk")
678 Cast.RC:$src0, Cast.KRCWM:$mask, To.RC:$src1, addr:$src2,
679 (INSERT_get_vinsert_imm To.RC:$ins))>;
682 (vselect Cast.KRCWM:$mask,
684 (vinsert_insert:$ins (To.VT To.RC:$src1),
685 (From.VT From.RC:$src2),
688 (!cast<Instruction>(InstrStr#"rrkz")
689 Cast.KRCWM:$mask, To.RC:$src1, From.RC:$src2,
690 (INSERT_get_vinsert_imm To.RC:$ins))>;
692 (vselect Cast.KRCWM:$mask,
694 (vinsert_insert:$ins (To.VT To.RC:$src1),
697 (From.LdFrag addr:$src2))),
700 (!cast<Instruction>(InstrStr#"rmkz")
701 Cast.KRCWM:$mask, To.RC:$src1, addr:$src2,
702 (INSERT_get_vinsert_imm To.RC:$ins))>;
706 defm : vinsert_for_mask_cast<"VINSERTF32x4Z256", v2f64x_info, v4f64x_info,
707 v8f32x_info, vinsert128_insert,
708 INSERT_get_vinsert128_imm, [HasVLX]>;
709 defm : vinsert_for_mask_cast<"VINSERTF64x2Z256", v4f32x_info, v8f32x_info,
710 v4f64x_info, vinsert128_insert,
711 INSERT_get_vinsert128_imm, [HasDQI, HasVLX]>;
713 defm : vinsert_for_mask_cast<"VINSERTI32x4Z256", v2i64x_info, v4i64x_info,
714 v8i32x_info, vinsert128_insert,
715 INSERT_get_vinsert128_imm, [HasVLX]>;
716 defm : vinsert_for_mask_cast<"VINSERTI32x4Z256", v8i16x_info, v16i16x_info,
717 v8i32x_info, vinsert128_insert,
718 INSERT_get_vinsert128_imm, [HasVLX]>;
719 defm : vinsert_for_mask_cast<"VINSERTI32x4Z256", v16i8x_info, v32i8x_info,
720 v8i32x_info, vinsert128_insert,
721 INSERT_get_vinsert128_imm, [HasVLX]>;
722 defm : vinsert_for_mask_cast<"VINSERTF64x2Z256", v4i32x_info, v8i32x_info,
723 v4i64x_info, vinsert128_insert,
724 INSERT_get_vinsert128_imm, [HasDQI, HasVLX]>;
725 defm : vinsert_for_mask_cast<"VINSERTF64x2Z256", v8i16x_info, v16i16x_info,
726 v4i64x_info, vinsert128_insert,
727 INSERT_get_vinsert128_imm, [HasDQI, HasVLX]>;
728 defm : vinsert_for_mask_cast<"VINSERTF64x2Z256", v16i8x_info, v32i8x_info,
729 v4i64x_info, vinsert128_insert,
730 INSERT_get_vinsert128_imm, [HasDQI, HasVLX]>;
732 defm : vinsert_for_mask_cast<"VINSERTF32x4Z", v2f64x_info, v8f64_info,
733 v16f32_info, vinsert128_insert,
734 INSERT_get_vinsert128_imm, [HasAVX512]>;
735 defm : vinsert_for_mask_cast<"VINSERTF64x2Z", v4f32x_info, v16f32_info,
736 v8f64_info, vinsert128_insert,
737 INSERT_get_vinsert128_imm, [HasDQI]>;
739 defm : vinsert_for_mask_cast<"VINSERTI32x4Z", v2i64x_info, v8i64_info,
740 v16i32_info, vinsert128_insert,
741 INSERT_get_vinsert128_imm, [HasAVX512]>;
742 defm : vinsert_for_mask_cast<"VINSERTI32x4Z", v8i16x_info, v32i16_info,
743 v16i32_info, vinsert128_insert,
744 INSERT_get_vinsert128_imm, [HasAVX512]>;
745 defm : vinsert_for_mask_cast<"VINSERTI32x4Z", v16i8x_info, v64i8_info,
746 v16i32_info, vinsert128_insert,
747 INSERT_get_vinsert128_imm, [HasAVX512]>;
748 defm : vinsert_for_mask_cast<"VINSERTI64x2Z", v4i32x_info, v16i32_info,
749 v8i64_info, vinsert128_insert,
750 INSERT_get_vinsert128_imm, [HasDQI]>;
751 defm : vinsert_for_mask_cast<"VINSERTI64x2Z", v8i16x_info, v32i16_info,
752 v8i64_info, vinsert128_insert,
753 INSERT_get_vinsert128_imm, [HasDQI]>;
754 defm : vinsert_for_mask_cast<"VINSERTI64x2Z", v16i8x_info, v64i8_info,
755 v8i64_info, vinsert128_insert,
756 INSERT_get_vinsert128_imm, [HasDQI]>;
758 defm : vinsert_for_mask_cast<"VINSERTF32x8Z", v4f64x_info, v8f64_info,
759 v16f32_info, vinsert256_insert,
760 INSERT_get_vinsert256_imm, [HasDQI]>;
761 defm : vinsert_for_mask_cast<"VINSERTF64x4Z", v8f32x_info, v16f32_info,
762 v8f64_info, vinsert256_insert,
763 INSERT_get_vinsert256_imm, [HasAVX512]>;
765 defm : vinsert_for_mask_cast<"VINSERTI32x8Z", v4i64x_info, v8i64_info,
766 v16i32_info, vinsert256_insert,
767 INSERT_get_vinsert256_imm, [HasDQI]>;
768 defm : vinsert_for_mask_cast<"VINSERTI32x8Z", v16i16x_info, v32i16_info,
769 v16i32_info, vinsert256_insert,
770 INSERT_get_vinsert256_imm, [HasDQI]>;
771 defm : vinsert_for_mask_cast<"VINSERTI32x8Z", v32i8x_info, v64i8_info,
772 v16i32_info, vinsert256_insert,
773 INSERT_get_vinsert256_imm, [HasDQI]>;
774 defm : vinsert_for_mask_cast<"VINSERTI64x4Z", v8i32x_info, v16i32_info,
775 v8i64_info, vinsert256_insert,
776 INSERT_get_vinsert256_imm, [HasAVX512]>;
777 defm : vinsert_for_mask_cast<"VINSERTI64x4Z", v16i16x_info, v32i16_info,
778 v8i64_info, vinsert256_insert,
779 INSERT_get_vinsert256_imm, [HasAVX512]>;
780 defm : vinsert_for_mask_cast<"VINSERTI64x4Z", v32i8x_info, v64i8_info,
781 v8i64_info, vinsert256_insert,
782 INSERT_get_vinsert256_imm, [HasAVX512]>;
784 // vinsertps - insert f32 to XMM
785 let ExeDomain = SSEPackedSingle in {
786 def VINSERTPSZrr : AVX512AIi8<0x21, MRMSrcReg, (outs VR128X:$dst),
787 (ins VR128X:$src1, VR128X:$src2, u8imm:$src3),
788 "vinsertps\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
789 [(set VR128X:$dst, (X86insertps VR128X:$src1, VR128X:$src2, imm:$src3))]>,
790 EVEX_4V, Sched<[SchedWriteFShuffle.XMM]>;
791 def VINSERTPSZrm: AVX512AIi8<0x21, MRMSrcMem, (outs VR128X:$dst),
792 (ins VR128X:$src1, f32mem:$src2, u8imm:$src3),
793 "vinsertps\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
794 [(set VR128X:$dst, (X86insertps VR128X:$src1,
795 (v4f32 (scalar_to_vector (loadf32 addr:$src2))),
797 EVEX_4V, EVEX_CD8<32, CD8VT1>,
798 Sched<[SchedWriteFShuffle.XMM.Folded, ReadAfterLd]>;
801 //===----------------------------------------------------------------------===//
802 // AVX-512 VECTOR EXTRACT
805 // Supports two different pattern operators for mask and unmasked ops. Allows
806 // null_frag to be passed for one.
807 multiclass vextract_for_size_split<int Opcode,
808 X86VectorVTInfo From, X86VectorVTInfo To,
809 SDPatternOperator vextract_extract,
810 SDPatternOperator vextract_for_mask,
811 SchedWrite SchedRR, SchedWrite SchedMR> {
813 let hasSideEffects = 0, ExeDomain = To.ExeDomain in {
814 defm rr : AVX512_maskable_split<Opcode, MRMDestReg, To, (outs To.RC:$dst),
815 (ins From.RC:$src1, u8imm:$idx),
816 "vextract" # To.EltTypeName # "x" # To.NumElts,
817 "$idx, $src1", "$src1, $idx",
818 (vextract_extract:$idx (From.VT From.RC:$src1), (iPTR imm)),
819 (vextract_for_mask:$idx (From.VT From.RC:$src1), (iPTR imm))>,
820 AVX512AIi8Base, EVEX, Sched<[SchedRR]>;
822 def mr : AVX512AIi8<Opcode, MRMDestMem, (outs),
823 (ins To.MemOp:$dst, From.RC:$src1, u8imm:$idx),
824 "vextract" # To.EltTypeName # "x" # To.NumElts #
825 "\t{$idx, $src1, $dst|$dst, $src1, $idx}",
826 [(store (To.VT (vextract_extract:$idx
827 (From.VT From.RC:$src1), (iPTR imm))),
831 let mayStore = 1, hasSideEffects = 0 in
832 def mrk : AVX512AIi8<Opcode, MRMDestMem, (outs),
833 (ins To.MemOp:$dst, To.KRCWM:$mask,
834 From.RC:$src1, u8imm:$idx),
835 "vextract" # To.EltTypeName # "x" # To.NumElts #
836 "\t{$idx, $src1, $dst {${mask}}|"
837 "$dst {${mask}}, $src1, $idx}", []>,
838 EVEX_K, EVEX, Sched<[SchedMR]>, NotMemoryFoldable;
842 // Passes the same pattern operator for masked and unmasked ops.
843 multiclass vextract_for_size<int Opcode, X86VectorVTInfo From,
845 SDPatternOperator vextract_extract,
846 SchedWrite SchedRR, SchedWrite SchedMR> :
847 vextract_for_size_split<Opcode, From, To, vextract_extract, vextract_extract, SchedRR, SchedMR>;
849 // Codegen pattern for the alternative types
850 multiclass vextract_for_size_lowering<string InstrStr, X86VectorVTInfo From,
851 X86VectorVTInfo To, PatFrag vextract_extract,
852 SDNodeXForm EXTRACT_get_vextract_imm, list<Predicate> p> {
853 let Predicates = p in {
854 def : Pat<(vextract_extract:$ext (From.VT From.RC:$src1), (iPTR imm)),
855 (To.VT (!cast<Instruction>(InstrStr#"rr")
857 (EXTRACT_get_vextract_imm To.RC:$ext)))>;
858 def : Pat<(store (To.VT (vextract_extract:$ext (From.VT From.RC:$src1),
859 (iPTR imm))), addr:$dst),
860 (!cast<Instruction>(InstrStr#"mr") addr:$dst, From.RC:$src1,
861 (EXTRACT_get_vextract_imm To.RC:$ext))>;
865 multiclass vextract_for_type<ValueType EltVT32, int Opcode128,
866 ValueType EltVT64, int Opcode256,
867 SchedWrite SchedRR, SchedWrite SchedMR> {
868 let Predicates = [HasAVX512] in {
869 defm NAME # "32x4Z" : vextract_for_size<Opcode128,
870 X86VectorVTInfo<16, EltVT32, VR512>,
871 X86VectorVTInfo< 4, EltVT32, VR128X>,
872 vextract128_extract, SchedRR, SchedMR>,
873 EVEX_V512, EVEX_CD8<32, CD8VT4>;
874 defm NAME # "64x4Z" : vextract_for_size<Opcode256,
875 X86VectorVTInfo< 8, EltVT64, VR512>,
876 X86VectorVTInfo< 4, EltVT64, VR256X>,
877 vextract256_extract, SchedRR, SchedMR>,
878 VEX_W, EVEX_V512, EVEX_CD8<64, CD8VT4>;
880 let Predicates = [HasVLX] in
881 defm NAME # "32x4Z256" : vextract_for_size<Opcode128,
882 X86VectorVTInfo< 8, EltVT32, VR256X>,
883 X86VectorVTInfo< 4, EltVT32, VR128X>,
884 vextract128_extract, SchedRR, SchedMR>,
885 EVEX_V256, EVEX_CD8<32, CD8VT4>;
887 // Even with DQI we'd like to only use these instructions for masking.
888 let Predicates = [HasVLX, HasDQI] in
889 defm NAME # "64x2Z256" : vextract_for_size_split<Opcode128,
890 X86VectorVTInfo< 4, EltVT64, VR256X>,
891 X86VectorVTInfo< 2, EltVT64, VR128X>,
892 null_frag, vextract128_extract, SchedRR, SchedMR>,
893 VEX_W1X, EVEX_V256, EVEX_CD8<64, CD8VT2>;
895 // Even with DQI we'd like to only use these instructions for masking.
896 let Predicates = [HasDQI] in {
897 defm NAME # "64x2Z" : vextract_for_size_split<Opcode128,
898 X86VectorVTInfo< 8, EltVT64, VR512>,
899 X86VectorVTInfo< 2, EltVT64, VR128X>,
900 null_frag, vextract128_extract, SchedRR, SchedMR>,
901 VEX_W, EVEX_V512, EVEX_CD8<64, CD8VT2>;
902 defm NAME # "32x8Z" : vextract_for_size_split<Opcode256,
903 X86VectorVTInfo<16, EltVT32, VR512>,
904 X86VectorVTInfo< 8, EltVT32, VR256X>,
905 null_frag, vextract256_extract, SchedRR, SchedMR>,
906 EVEX_V512, EVEX_CD8<32, CD8VT8>;
910 // TODO - replace WriteFStore/WriteVecStore with X86SchedWriteMoveLSWidths types.
911 defm VEXTRACTF : vextract_for_type<f32, 0x19, f64, 0x1b, WriteFShuffle256, WriteFStore>;
912 defm VEXTRACTI : vextract_for_type<i32, 0x39, i64, 0x3b, WriteShuffle256, WriteVecStore>;
914 // extract_subvector codegen patterns with the alternative types.
915 // Even with AVX512DQ we'll still use these for unmasked operations.
916 defm : vextract_for_size_lowering<"VEXTRACTF32x4Z", v8f64_info, v2f64x_info,
917 vextract128_extract, EXTRACT_get_vextract128_imm, [HasAVX512]>;
918 defm : vextract_for_size_lowering<"VEXTRACTI32x4Z", v8i64_info, v2i64x_info,
919 vextract128_extract, EXTRACT_get_vextract128_imm, [HasAVX512]>;
921 defm : vextract_for_size_lowering<"VEXTRACTF64x4Z", v16f32_info, v8f32x_info,
922 vextract256_extract, EXTRACT_get_vextract256_imm, [HasAVX512]>;
923 defm : vextract_for_size_lowering<"VEXTRACTI64x4Z", v16i32_info, v8i32x_info,
924 vextract256_extract, EXTRACT_get_vextract256_imm, [HasAVX512]>;
926 defm : vextract_for_size_lowering<"VEXTRACTF32x4Z256", v4f64x_info, v2f64x_info,
927 vextract128_extract, EXTRACT_get_vextract128_imm, [HasVLX]>;
928 defm : vextract_for_size_lowering<"VEXTRACTI32x4Z256", v4i64x_info, v2i64x_info,
929 vextract128_extract, EXTRACT_get_vextract128_imm, [HasVLX]>;
931 // Codegen pattern with the alternative types extract VEC128 from VEC256
932 defm : vextract_for_size_lowering<"VEXTRACTI32x4Z256", v16i16x_info, v8i16x_info,
933 vextract128_extract, EXTRACT_get_vextract128_imm, [HasVLX]>;
934 defm : vextract_for_size_lowering<"VEXTRACTI32x4Z256", v32i8x_info, v16i8x_info,
935 vextract128_extract, EXTRACT_get_vextract128_imm, [HasVLX]>;
937 // Codegen pattern with the alternative types extract VEC128 from VEC512
938 defm : vextract_for_size_lowering<"VEXTRACTI32x4Z", v32i16_info, v8i16x_info,
939 vextract128_extract, EXTRACT_get_vextract128_imm, [HasAVX512]>;
940 defm : vextract_for_size_lowering<"VEXTRACTI32x4Z", v64i8_info, v16i8x_info,
941 vextract128_extract, EXTRACT_get_vextract128_imm, [HasAVX512]>;
942 // Codegen pattern with the alternative types extract VEC256 from VEC512
943 defm : vextract_for_size_lowering<"VEXTRACTI64x4Z", v32i16_info, v16i16x_info,
944 vextract256_extract, EXTRACT_get_vextract256_imm, [HasAVX512]>;
945 defm : vextract_for_size_lowering<"VEXTRACTI64x4Z", v64i8_info, v32i8x_info,
946 vextract256_extract, EXTRACT_get_vextract256_imm, [HasAVX512]>;
949 // A 128-bit extract from bits [255:128] of a 512-bit vector should use a
950 // smaller extract to enable EVEX->VEX.
951 let Predicates = [NoVLX] in {
952 def : Pat<(v2i64 (extract_subvector (v8i64 VR512:$src), (iPTR 2))),
953 (v2i64 (VEXTRACTI128rr
954 (v4i64 (EXTRACT_SUBREG (v8i64 VR512:$src), sub_ymm)),
956 def : Pat<(v2f64 (extract_subvector (v8f64 VR512:$src), (iPTR 2))),
957 (v2f64 (VEXTRACTF128rr
958 (v4f64 (EXTRACT_SUBREG (v8f64 VR512:$src), sub_ymm)),
960 def : Pat<(v4i32 (extract_subvector (v16i32 VR512:$src), (iPTR 4))),
961 (v4i32 (VEXTRACTI128rr
962 (v8i32 (EXTRACT_SUBREG (v16i32 VR512:$src), sub_ymm)),
964 def : Pat<(v4f32 (extract_subvector (v16f32 VR512:$src), (iPTR 4))),
965 (v4f32 (VEXTRACTF128rr
966 (v8f32 (EXTRACT_SUBREG (v16f32 VR512:$src), sub_ymm)),
968 def : Pat<(v8i16 (extract_subvector (v32i16 VR512:$src), (iPTR 8))),
969 (v8i16 (VEXTRACTI128rr
970 (v16i16 (EXTRACT_SUBREG (v32i16 VR512:$src), sub_ymm)),
972 def : Pat<(v16i8 (extract_subvector (v64i8 VR512:$src), (iPTR 16))),
973 (v16i8 (VEXTRACTI128rr
974 (v32i8 (EXTRACT_SUBREG (v64i8 VR512:$src), sub_ymm)),
978 // A 128-bit extract from bits [255:128] of a 512-bit vector should use a
979 // smaller extract to enable EVEX->VEX.
980 let Predicates = [HasVLX] in {
981 def : Pat<(v2i64 (extract_subvector (v8i64 VR512:$src), (iPTR 2))),
982 (v2i64 (VEXTRACTI32x4Z256rr
983 (v4i64 (EXTRACT_SUBREG (v8i64 VR512:$src), sub_ymm)),
985 def : Pat<(v2f64 (extract_subvector (v8f64 VR512:$src), (iPTR 2))),
986 (v2f64 (VEXTRACTF32x4Z256rr
987 (v4f64 (EXTRACT_SUBREG (v8f64 VR512:$src), sub_ymm)),
989 def : Pat<(v4i32 (extract_subvector (v16i32 VR512:$src), (iPTR 4))),
990 (v4i32 (VEXTRACTI32x4Z256rr
991 (v8i32 (EXTRACT_SUBREG (v16i32 VR512:$src), sub_ymm)),
993 def : Pat<(v4f32 (extract_subvector (v16f32 VR512:$src), (iPTR 4))),
994 (v4f32 (VEXTRACTF32x4Z256rr
995 (v8f32 (EXTRACT_SUBREG (v16f32 VR512:$src), sub_ymm)),
997 def : Pat<(v8i16 (extract_subvector (v32i16 VR512:$src), (iPTR 8))),
998 (v8i16 (VEXTRACTI32x4Z256rr
999 (v16i16 (EXTRACT_SUBREG (v32i16 VR512:$src), sub_ymm)),
1001 def : Pat<(v16i8 (extract_subvector (v64i8 VR512:$src), (iPTR 16))),
1002 (v16i8 (VEXTRACTI32x4Z256rr
1003 (v32i8 (EXTRACT_SUBREG (v64i8 VR512:$src), sub_ymm)),
1008 // Additional patterns for handling a bitcast between the vselect and the
1009 // extract_subvector.
1010 multiclass vextract_for_mask_cast<string InstrStr, X86VectorVTInfo From,
1011 X86VectorVTInfo To, X86VectorVTInfo Cast,
1012 PatFrag vextract_extract,
1013 SDNodeXForm EXTRACT_get_vextract_imm,
1014 list<Predicate> p> {
1015 let Predicates = p in {
1016 def : Pat<(Cast.VT (vselect Cast.KRCWM:$mask,
1018 (To.VT (vextract_extract:$ext
1019 (From.VT From.RC:$src), (iPTR imm)))),
1021 (Cast.VT (!cast<Instruction>(InstrStr#"rrk")
1022 Cast.RC:$src0, Cast.KRCWM:$mask, From.RC:$src,
1023 (EXTRACT_get_vextract_imm To.RC:$ext)))>;
1025 def : Pat<(Cast.VT (vselect Cast.KRCWM:$mask,
1027 (To.VT (vextract_extract:$ext
1028 (From.VT From.RC:$src), (iPTR imm)))),
1029 Cast.ImmAllZerosV)),
1030 (Cast.VT (!cast<Instruction>(InstrStr#"rrkz")
1031 Cast.KRCWM:$mask, From.RC:$src,
1032 (EXTRACT_get_vextract_imm To.RC:$ext)))>;
1036 defm : vextract_for_mask_cast<"VEXTRACTF32x4Z256", v4f64x_info, v2f64x_info,
1037 v4f32x_info, vextract128_extract,
1038 EXTRACT_get_vextract128_imm, [HasVLX]>;
1039 defm : vextract_for_mask_cast<"VEXTRACTF64x2Z256", v8f32x_info, v4f32x_info,
1040 v2f64x_info, vextract128_extract,
1041 EXTRACT_get_vextract128_imm, [HasDQI, HasVLX]>;
1043 defm : vextract_for_mask_cast<"VEXTRACTI32x4Z256", v4i64x_info, v2i64x_info,
1044 v4i32x_info, vextract128_extract,
1045 EXTRACT_get_vextract128_imm, [HasVLX]>;
1046 defm : vextract_for_mask_cast<"VEXTRACTI32x4Z256", v16i16x_info, v8i16x_info,
1047 v4i32x_info, vextract128_extract,
1048 EXTRACT_get_vextract128_imm, [HasVLX]>;
1049 defm : vextract_for_mask_cast<"VEXTRACTI32x4Z256", v32i8x_info, v16i8x_info,
1050 v4i32x_info, vextract128_extract,
1051 EXTRACT_get_vextract128_imm, [HasVLX]>;
1052 defm : vextract_for_mask_cast<"VEXTRACTI64x2Z256", v8i32x_info, v4i32x_info,
1053 v2i64x_info, vextract128_extract,
1054 EXTRACT_get_vextract128_imm, [HasDQI, HasVLX]>;
1055 defm : vextract_for_mask_cast<"VEXTRACTI64x2Z256", v16i16x_info, v8i16x_info,
1056 v2i64x_info, vextract128_extract,
1057 EXTRACT_get_vextract128_imm, [HasDQI, HasVLX]>;
1058 defm : vextract_for_mask_cast<"VEXTRACTI64x2Z256", v32i8x_info, v16i8x_info,
1059 v2i64x_info, vextract128_extract,
1060 EXTRACT_get_vextract128_imm, [HasDQI, HasVLX]>;
1062 defm : vextract_for_mask_cast<"VEXTRACTF32x4Z", v8f64_info, v2f64x_info,
1063 v4f32x_info, vextract128_extract,
1064 EXTRACT_get_vextract128_imm, [HasAVX512]>;
1065 defm : vextract_for_mask_cast<"VEXTRACTF64x2Z", v16f32_info, v4f32x_info,
1066 v2f64x_info, vextract128_extract,
1067 EXTRACT_get_vextract128_imm, [HasDQI]>;
1069 defm : vextract_for_mask_cast<"VEXTRACTI32x4Z", v8i64_info, v2i64x_info,
1070 v4i32x_info, vextract128_extract,
1071 EXTRACT_get_vextract128_imm, [HasAVX512]>;
1072 defm : vextract_for_mask_cast<"VEXTRACTI32x4Z", v32i16_info, v8i16x_info,
1073 v4i32x_info, vextract128_extract,
1074 EXTRACT_get_vextract128_imm, [HasAVX512]>;
1075 defm : vextract_for_mask_cast<"VEXTRACTI32x4Z", v64i8_info, v16i8x_info,
1076 v4i32x_info, vextract128_extract,
1077 EXTRACT_get_vextract128_imm, [HasAVX512]>;
1078 defm : vextract_for_mask_cast<"VEXTRACTI64x2Z", v16i32_info, v4i32x_info,
1079 v2i64x_info, vextract128_extract,
1080 EXTRACT_get_vextract128_imm, [HasDQI]>;
1081 defm : vextract_for_mask_cast<"VEXTRACTI64x2Z", v32i16_info, v8i16x_info,
1082 v2i64x_info, vextract128_extract,
1083 EXTRACT_get_vextract128_imm, [HasDQI]>;
1084 defm : vextract_for_mask_cast<"VEXTRACTI64x2Z", v64i8_info, v16i8x_info,
1085 v2i64x_info, vextract128_extract,
1086 EXTRACT_get_vextract128_imm, [HasDQI]>;
1088 defm : vextract_for_mask_cast<"VEXTRACTF32x8Z", v8f64_info, v4f64x_info,
1089 v8f32x_info, vextract256_extract,
1090 EXTRACT_get_vextract256_imm, [HasDQI]>;
1091 defm : vextract_for_mask_cast<"VEXTRACTF64x4Z", v16f32_info, v8f32x_info,
1092 v4f64x_info, vextract256_extract,
1093 EXTRACT_get_vextract256_imm, [HasAVX512]>;
1095 defm : vextract_for_mask_cast<"VEXTRACTI32x8Z", v8i64_info, v4i64x_info,
1096 v8i32x_info, vextract256_extract,
1097 EXTRACT_get_vextract256_imm, [HasDQI]>;
1098 defm : vextract_for_mask_cast<"VEXTRACTI32x8Z", v32i16_info, v16i16x_info,
1099 v8i32x_info, vextract256_extract,
1100 EXTRACT_get_vextract256_imm, [HasDQI]>;
1101 defm : vextract_for_mask_cast<"VEXTRACTI32x8Z", v64i8_info, v32i8x_info,
1102 v8i32x_info, vextract256_extract,
1103 EXTRACT_get_vextract256_imm, [HasDQI]>;
1104 defm : vextract_for_mask_cast<"VEXTRACTI64x4Z", v16i32_info, v8i32x_info,
1105 v4i64x_info, vextract256_extract,
1106 EXTRACT_get_vextract256_imm, [HasAVX512]>;
1107 defm : vextract_for_mask_cast<"VEXTRACTI64x4Z", v32i16_info, v16i16x_info,
1108 v4i64x_info, vextract256_extract,
1109 EXTRACT_get_vextract256_imm, [HasAVX512]>;
1110 defm : vextract_for_mask_cast<"VEXTRACTI64x4Z", v64i8_info, v32i8x_info,
1111 v4i64x_info, vextract256_extract,
1112 EXTRACT_get_vextract256_imm, [HasAVX512]>;
1114 // vextractps - extract 32 bits from XMM
1115 def VEXTRACTPSZrr : AVX512AIi8<0x17, MRMDestReg, (outs GR32:$dst),
1116 (ins VR128X:$src1, u8imm:$src2),
1117 "vextractps\t{$src2, $src1, $dst|$dst, $src1, $src2}",
1118 [(set GR32:$dst, (extractelt (bc_v4i32 (v4f32 VR128X:$src1)), imm:$src2))]>,
1119 EVEX, VEX_WIG, Sched<[WriteVecExtract]>;
1121 def VEXTRACTPSZmr : AVX512AIi8<0x17, MRMDestMem, (outs),
1122 (ins f32mem:$dst, VR128X:$src1, u8imm:$src2),
1123 "vextractps\t{$src2, $src1, $dst|$dst, $src1, $src2}",
1124 [(store (extractelt (bc_v4i32 (v4f32 VR128X:$src1)), imm:$src2),
1126 EVEX, VEX_WIG, EVEX_CD8<32, CD8VT1>, Sched<[WriteVecExtractSt]>;
1128 //===---------------------------------------------------------------------===//
1129 // AVX-512 BROADCAST
1131 // broadcast with a scalar argument.
1132 multiclass avx512_broadcast_scalar<bits<8> opc, string OpcodeStr,
1134 X86VectorVTInfo DestInfo, X86VectorVTInfo SrcInfo> {
1135 def : Pat<(DestInfo.VT (X86VBroadcast SrcInfo.FRC:$src)),
1136 (!cast<Instruction>(Name#DestInfo.ZSuffix#r)
1137 (COPY_TO_REGCLASS SrcInfo.FRC:$src, SrcInfo.RC))>;
1138 def : Pat<(DestInfo.VT (vselect DestInfo.KRCWM:$mask,
1139 (X86VBroadcast SrcInfo.FRC:$src),
1140 DestInfo.RC:$src0)),
1141 (!cast<Instruction>(Name#DestInfo.ZSuffix#rk)
1142 DestInfo.RC:$src0, DestInfo.KRCWM:$mask,
1143 (COPY_TO_REGCLASS SrcInfo.FRC:$src, SrcInfo.RC))>;
1144 def : Pat<(DestInfo.VT (vselect DestInfo.KRCWM:$mask,
1145 (X86VBroadcast SrcInfo.FRC:$src),
1146 DestInfo.ImmAllZerosV)),
1147 (!cast<Instruction>(Name#DestInfo.ZSuffix#rkz)
1148 DestInfo.KRCWM:$mask, (COPY_TO_REGCLASS SrcInfo.FRC:$src, SrcInfo.RC))>;
1151 // Split version to allow mask and broadcast node to be different types. This
1152 // helps support the 32x2 broadcasts.
1153 multiclass avx512_broadcast_rm_split<bits<8> opc, string OpcodeStr,
1155 SchedWrite SchedRR, SchedWrite SchedRM,
1156 X86VectorVTInfo MaskInfo,
1157 X86VectorVTInfo DestInfo,
1158 X86VectorVTInfo SrcInfo,
1159 SDPatternOperator UnmaskedOp = X86VBroadcast> {
1160 let ExeDomain = DestInfo.ExeDomain, hasSideEffects = 0 in {
1161 defm r : AVX512_maskable_split<opc, MRMSrcReg, MaskInfo,
1162 (outs MaskInfo.RC:$dst),
1163 (ins SrcInfo.RC:$src), OpcodeStr, "$src", "$src",
1167 (UnmaskedOp (SrcInfo.VT SrcInfo.RC:$src))))),
1171 (X86VBroadcast (SrcInfo.VT SrcInfo.RC:$src)))))>,
1172 T8PD, EVEX, Sched<[SchedRR]>;
1174 defm m : AVX512_maskable_split<opc, MRMSrcMem, MaskInfo,
1175 (outs MaskInfo.RC:$dst),
1176 (ins SrcInfo.ScalarMemOp:$src), OpcodeStr, "$src", "$src",
1179 (DestInfo.VT (UnmaskedOp
1180 (SrcInfo.ScalarLdFrag addr:$src))))),
1183 (DestInfo.VT (X86VBroadcast
1184 (SrcInfo.ScalarLdFrag addr:$src)))))>,
1185 T8PD, EVEX, EVEX_CD8<SrcInfo.EltSize, CD8VT1>,
1189 def : Pat<(MaskInfo.VT
1191 (DestInfo.VT (UnmaskedOp
1192 (SrcInfo.VT (scalar_to_vector
1193 (SrcInfo.ScalarLdFrag addr:$src))))))),
1194 (!cast<Instruction>(Name#MaskInfo.ZSuffix#m) addr:$src)>;
1195 def : Pat<(MaskInfo.VT (vselect MaskInfo.KRCWM:$mask,
1199 (SrcInfo.VT (scalar_to_vector
1200 (SrcInfo.ScalarLdFrag addr:$src)))))),
1201 MaskInfo.RC:$src0)),
1202 (!cast<Instruction>(Name#DestInfo.ZSuffix#mk)
1203 MaskInfo.RC:$src0, MaskInfo.KRCWM:$mask, addr:$src)>;
1204 def : Pat<(MaskInfo.VT (vselect MaskInfo.KRCWM:$mask,
1208 (SrcInfo.VT (scalar_to_vector
1209 (SrcInfo.ScalarLdFrag addr:$src)))))),
1210 MaskInfo.ImmAllZerosV)),
1211 (!cast<Instruction>(Name#MaskInfo.ZSuffix#mkz)
1212 MaskInfo.KRCWM:$mask, addr:$src)>;
1215 // Helper class to force mask and broadcast result to same type.
1216 multiclass avx512_broadcast_rm<bits<8> opc, string OpcodeStr, string Name,
1217 SchedWrite SchedRR, SchedWrite SchedRM,
1218 X86VectorVTInfo DestInfo,
1219 X86VectorVTInfo SrcInfo> :
1220 avx512_broadcast_rm_split<opc, OpcodeStr, Name, SchedRR, SchedRM,
1221 DestInfo, DestInfo, SrcInfo>;
1223 multiclass avx512_fp_broadcast_sd<bits<8> opc, string OpcodeStr,
1224 AVX512VLVectorVTInfo _> {
1225 let Predicates = [HasAVX512] in {
1226 defm Z : avx512_broadcast_rm<opc, OpcodeStr, NAME, WriteFShuffle256,
1227 WriteFShuffle256Ld, _.info512, _.info128>,
1228 avx512_broadcast_scalar<opc, OpcodeStr, NAME, _.info512,
1233 let Predicates = [HasVLX] in {
1234 defm Z256 : avx512_broadcast_rm<opc, OpcodeStr, NAME, WriteFShuffle256,
1235 WriteFShuffle256Ld, _.info256, _.info128>,
1236 avx512_broadcast_scalar<opc, OpcodeStr, NAME, _.info256,
1242 multiclass avx512_fp_broadcast_ss<bits<8> opc, string OpcodeStr,
1243 AVX512VLVectorVTInfo _> {
1244 let Predicates = [HasAVX512] in {
1245 defm Z : avx512_broadcast_rm<opc, OpcodeStr, NAME, WriteFShuffle256,
1246 WriteFShuffle256Ld, _.info512, _.info128>,
1247 avx512_broadcast_scalar<opc, OpcodeStr, NAME, _.info512,
1252 let Predicates = [HasVLX] in {
1253 defm Z256 : avx512_broadcast_rm<opc, OpcodeStr, NAME, WriteFShuffle256,
1254 WriteFShuffle256Ld, _.info256, _.info128>,
1255 avx512_broadcast_scalar<opc, OpcodeStr, NAME, _.info256,
1258 defm Z128 : avx512_broadcast_rm<opc, OpcodeStr, NAME, WriteFShuffle256,
1259 WriteFShuffle256Ld, _.info128, _.info128>,
1260 avx512_broadcast_scalar<opc, OpcodeStr, NAME, _.info128,
1265 defm VBROADCASTSS : avx512_fp_broadcast_ss<0x18, "vbroadcastss",
1267 defm VBROADCASTSD : avx512_fp_broadcast_sd<0x19, "vbroadcastsd",
1268 avx512vl_f64_info>, VEX_W1X;
1270 multiclass avx512_int_broadcast_reg<bits<8> opc, SchedWrite SchedRR,
1271 X86VectorVTInfo _, SDPatternOperator OpNode,
1272 RegisterClass SrcRC> {
1273 let ExeDomain = _.ExeDomain in
1274 defm r : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
1276 "vpbroadcast"##_.Suffix, "$src", "$src",
1277 (_.VT (OpNode SrcRC:$src))>, T8PD, EVEX,
1281 multiclass avx512_int_broadcastbw_reg<bits<8> opc, string Name, SchedWrite SchedRR,
1282 X86VectorVTInfo _, SDPatternOperator OpNode,
1283 RegisterClass SrcRC, SubRegIndex Subreg> {
1284 let hasSideEffects = 0, ExeDomain = _.ExeDomain in
1285 defm r : AVX512_maskable_custom<opc, MRMSrcReg,
1286 (outs _.RC:$dst), (ins GR32:$src),
1287 !con((ins _.RC:$src0, _.KRCWM:$mask), (ins GR32:$src)),
1288 !con((ins _.KRCWM:$mask), (ins GR32:$src)),
1289 "vpbroadcast"##_.Suffix, "$src", "$src", [], [], [],
1290 "$src0 = $dst">, T8PD, EVEX, Sched<[SchedRR]>;
1292 def : Pat <(_.VT (OpNode SrcRC:$src)),
1293 (!cast<Instruction>(Name#r)
1294 (i32 (INSERT_SUBREG (i32 (IMPLICIT_DEF)), SrcRC:$src, Subreg)))>;
1296 def : Pat <(vselect _.KRCWM:$mask, (_.VT (OpNode SrcRC:$src)), _.RC:$src0),
1297 (!cast<Instruction>(Name#rk) _.RC:$src0, _.KRCWM:$mask,
1298 (i32 (INSERT_SUBREG (i32 (IMPLICIT_DEF)), SrcRC:$src, Subreg)))>;
1300 def : Pat <(vselect _.KRCWM:$mask, (_.VT (OpNode SrcRC:$src)), _.ImmAllZerosV),
1301 (!cast<Instruction>(Name#rkz) _.KRCWM:$mask,
1302 (i32 (INSERT_SUBREG (i32 (IMPLICIT_DEF)), SrcRC:$src, Subreg)))>;
1305 multiclass avx512_int_broadcastbw_reg_vl<bits<8> opc, string Name,
1306 AVX512VLVectorVTInfo _, SDPatternOperator OpNode,
1307 RegisterClass SrcRC, SubRegIndex Subreg, Predicate prd> {
1308 let Predicates = [prd] in
1309 defm Z : avx512_int_broadcastbw_reg<opc, Name#Z, WriteShuffle256, _.info512,
1310 OpNode, SrcRC, Subreg>, EVEX_V512;
1311 let Predicates = [prd, HasVLX] in {
1312 defm Z256 : avx512_int_broadcastbw_reg<opc, Name#Z256, WriteShuffle256,
1313 _.info256, OpNode, SrcRC, Subreg>, EVEX_V256;
1314 defm Z128 : avx512_int_broadcastbw_reg<opc, Name#Z128, WriteShuffle,
1315 _.info128, OpNode, SrcRC, Subreg>, EVEX_V128;
1319 multiclass avx512_int_broadcast_reg_vl<bits<8> opc, AVX512VLVectorVTInfo _,
1320 SDPatternOperator OpNode,
1321 RegisterClass SrcRC, Predicate prd> {
1322 let Predicates = [prd] in
1323 defm Z : avx512_int_broadcast_reg<opc, WriteShuffle256, _.info512, OpNode,
1325 let Predicates = [prd, HasVLX] in {
1326 defm Z256 : avx512_int_broadcast_reg<opc, WriteShuffle256, _.info256, OpNode,
1328 defm Z128 : avx512_int_broadcast_reg<opc, WriteShuffle, _.info128, OpNode,
1333 defm VPBROADCASTBr : avx512_int_broadcastbw_reg_vl<0x7A, "VPBROADCASTBr",
1334 avx512vl_i8_info, X86VBroadcast, GR8, sub_8bit, HasBWI>;
1335 defm VPBROADCASTWr : avx512_int_broadcastbw_reg_vl<0x7B, "VPBROADCASTWr",
1336 avx512vl_i16_info, X86VBroadcast, GR16, sub_16bit,
1338 defm VPBROADCASTDr : avx512_int_broadcast_reg_vl<0x7C, avx512vl_i32_info,
1339 X86VBroadcast, GR32, HasAVX512>;
1340 defm VPBROADCASTQr : avx512_int_broadcast_reg_vl<0x7C, avx512vl_i64_info,
1341 X86VBroadcast, GR64, HasAVX512>, VEX_W;
1343 // Provide aliases for broadcast from the same register class that
1344 // automatically does the extract.
1345 multiclass avx512_int_broadcast_rm_lowering<string Name,
1346 X86VectorVTInfo DestInfo,
1347 X86VectorVTInfo SrcInfo> {
1348 def : Pat<(DestInfo.VT (X86VBroadcast (SrcInfo.VT SrcInfo.RC:$src))),
1349 (!cast<Instruction>(Name#DestInfo.ZSuffix#"r")
1350 (EXTRACT_SUBREG (SrcInfo.VT SrcInfo.RC:$src), sub_xmm))>;
1353 multiclass avx512_int_broadcast_rm_vl<bits<8> opc, string OpcodeStr,
1354 AVX512VLVectorVTInfo _, Predicate prd> {
1355 let Predicates = [prd] in {
1356 defm Z : avx512_broadcast_rm<opc, OpcodeStr, NAME, WriteShuffle256,
1357 WriteShuffle256Ld, _.info512, _.info128>,
1358 avx512_int_broadcast_rm_lowering<NAME, _.info512, _.info256>,
1360 // Defined separately to avoid redefinition.
1361 defm Z_Alt : avx512_int_broadcast_rm_lowering<NAME, _.info512, _.info512>;
1363 let Predicates = [prd, HasVLX] in {
1364 defm Z256 : avx512_broadcast_rm<opc, OpcodeStr, NAME, WriteShuffle256,
1365 WriteShuffle256Ld, _.info256, _.info128>,
1366 avx512_int_broadcast_rm_lowering<NAME, _.info256, _.info256>,
1368 defm Z128 : avx512_broadcast_rm<opc, OpcodeStr, NAME, WriteShuffle,
1369 WriteShuffleXLd, _.info128, _.info128>,
1374 defm VPBROADCASTB : avx512_int_broadcast_rm_vl<0x78, "vpbroadcastb",
1375 avx512vl_i8_info, HasBWI>;
1376 defm VPBROADCASTW : avx512_int_broadcast_rm_vl<0x79, "vpbroadcastw",
1377 avx512vl_i16_info, HasBWI>;
1378 defm VPBROADCASTD : avx512_int_broadcast_rm_vl<0x58, "vpbroadcastd",
1379 avx512vl_i32_info, HasAVX512>;
1380 defm VPBROADCASTQ : avx512_int_broadcast_rm_vl<0x59, "vpbroadcastq",
1381 avx512vl_i64_info, HasAVX512>, VEX_W1X;
1383 multiclass avx512_subvec_broadcast_rm<bits<8> opc, string OpcodeStr,
1384 X86VectorVTInfo _Dst, X86VectorVTInfo _Src> {
1385 defm rm : AVX512_maskable<opc, MRMSrcMem, _Dst, (outs _Dst.RC:$dst),
1386 (ins _Src.MemOp:$src), OpcodeStr, "$src", "$src",
1387 (_Dst.VT (X86SubVBroadcast
1388 (_Src.VT (bitconvert (_Src.LdFrag addr:$src)))))>,
1389 Sched<[SchedWriteShuffle.YMM.Folded]>,
1393 // This should be used for the AVX512DQ broadcast instructions. It disables
1394 // the unmasked patterns so that we only use the DQ instructions when masking
1396 multiclass avx512_subvec_broadcast_rm_dq<bits<8> opc, string OpcodeStr,
1397 X86VectorVTInfo _Dst, X86VectorVTInfo _Src> {
1398 let hasSideEffects = 0, mayLoad = 1 in
1399 defm rm : AVX512_maskable_split<opc, MRMSrcMem, _Dst, (outs _Dst.RC:$dst),
1400 (ins _Src.MemOp:$src), OpcodeStr, "$src", "$src",
1402 (_Dst.VT (X86SubVBroadcast
1403 (_Src.VT (bitconvert (_Src.LdFrag addr:$src)))))>,
1404 Sched<[SchedWriteShuffle.YMM.Folded]>,
1408 let Predicates = [HasAVX512] in {
1409 // 32-bit targets will fail to load a i64 directly but can use ZEXT_LOAD.
1410 def : Pat<(v8i64 (X86VBroadcast (v8i64 (X86vzload addr:$src)))),
1411 (VPBROADCASTQZm addr:$src)>;
1414 let Predicates = [HasVLX] in {
1415 // 32-bit targets will fail to load a i64 directly but can use ZEXT_LOAD.
1416 def : Pat<(v2i64 (X86VBroadcast (v2i64 (X86vzload addr:$src)))),
1417 (VPBROADCASTQZ128m addr:$src)>;
1418 def : Pat<(v4i64 (X86VBroadcast (v4i64 (X86vzload addr:$src)))),
1419 (VPBROADCASTQZ256m addr:$src)>;
1421 let Predicates = [HasVLX, HasBWI] in {
1422 // loadi16 is tricky to fold, because !isTypeDesirableForOp, justifiably.
1423 // This means we'll encounter truncated i32 loads; match that here.
1424 def : Pat<(v8i16 (X86VBroadcast (i16 (trunc (i32 (load addr:$src)))))),
1425 (VPBROADCASTWZ128m addr:$src)>;
1426 def : Pat<(v16i16 (X86VBroadcast (i16 (trunc (i32 (load addr:$src)))))),
1427 (VPBROADCASTWZ256m addr:$src)>;
1428 def : Pat<(v8i16 (X86VBroadcast
1429 (i16 (trunc (i32 (zextloadi16 addr:$src)))))),
1430 (VPBROADCASTWZ128m addr:$src)>;
1431 def : Pat<(v16i16 (X86VBroadcast
1432 (i16 (trunc (i32 (zextloadi16 addr:$src)))))),
1433 (VPBROADCASTWZ256m addr:$src)>;
1436 //===----------------------------------------------------------------------===//
1437 // AVX-512 BROADCAST SUBVECTORS
1440 defm VBROADCASTI32X4 : avx512_subvec_broadcast_rm<0x5a, "vbroadcasti32x4",
1441 v16i32_info, v4i32x_info>,
1442 EVEX_V512, EVEX_CD8<32, CD8VT4>;
1443 defm VBROADCASTF32X4 : avx512_subvec_broadcast_rm<0x1a, "vbroadcastf32x4",
1444 v16f32_info, v4f32x_info>,
1445 EVEX_V512, EVEX_CD8<32, CD8VT4>;
1446 defm VBROADCASTI64X4 : avx512_subvec_broadcast_rm<0x5b, "vbroadcasti64x4",
1447 v8i64_info, v4i64x_info>, VEX_W,
1448 EVEX_V512, EVEX_CD8<64, CD8VT4>;
1449 defm VBROADCASTF64X4 : avx512_subvec_broadcast_rm<0x1b, "vbroadcastf64x4",
1450 v8f64_info, v4f64x_info>, VEX_W,
1451 EVEX_V512, EVEX_CD8<64, CD8VT4>;
1453 let Predicates = [HasAVX512] in {
1454 def : Pat<(v16f32 (X86SubVBroadcast (loadv8f32 addr:$src))),
1455 (VBROADCASTF64X4rm addr:$src)>;
1456 def : Pat<(v16i32 (X86SubVBroadcast (bc_v8i32 (loadv4i64 addr:$src)))),
1457 (VBROADCASTI64X4rm addr:$src)>;
1458 def : Pat<(v32i16 (X86SubVBroadcast (bc_v16i16 (loadv4i64 addr:$src)))),
1459 (VBROADCASTI64X4rm addr:$src)>;
1460 def : Pat<(v64i8 (X86SubVBroadcast (bc_v32i8 (loadv4i64 addr:$src)))),
1461 (VBROADCASTI64X4rm addr:$src)>;
1463 // Provide fallback in case the load node that is used in the patterns above
1464 // is used by additional users, which prevents the pattern selection.
1465 def : Pat<(v8f64 (X86SubVBroadcast (v4f64 VR256X:$src))),
1466 (VINSERTF64x4Zrr (INSERT_SUBREG (v8f64 (IMPLICIT_DEF)), VR256X:$src, sub_ymm),
1467 (v4f64 VR256X:$src), 1)>;
1468 def : Pat<(v16f32 (X86SubVBroadcast (v8f32 VR256X:$src))),
1469 (VINSERTF64x4Zrr (INSERT_SUBREG (v16f32 (IMPLICIT_DEF)), VR256X:$src, sub_ymm),
1470 (v8f32 VR256X:$src), 1)>;
1471 def : Pat<(v8i64 (X86SubVBroadcast (v4i64 VR256X:$src))),
1472 (VINSERTI64x4Zrr (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR256X:$src, sub_ymm),
1473 (v4i64 VR256X:$src), 1)>;
1474 def : Pat<(v16i32 (X86SubVBroadcast (v8i32 VR256X:$src))),
1475 (VINSERTI64x4Zrr (INSERT_SUBREG (v16i32 (IMPLICIT_DEF)), VR256X:$src, sub_ymm),
1476 (v8i32 VR256X:$src), 1)>;
1477 def : Pat<(v32i16 (X86SubVBroadcast (v16i16 VR256X:$src))),
1478 (VINSERTI64x4Zrr (INSERT_SUBREG (v32i16 (IMPLICIT_DEF)), VR256X:$src, sub_ymm),
1479 (v16i16 VR256X:$src), 1)>;
1480 def : Pat<(v64i8 (X86SubVBroadcast (v32i8 VR256X:$src))),
1481 (VINSERTI64x4Zrr (INSERT_SUBREG (v64i8 (IMPLICIT_DEF)), VR256X:$src, sub_ymm),
1482 (v32i8 VR256X:$src), 1)>;
1484 def : Pat<(v8f64 (X86SubVBroadcast (loadv2f64 addr:$src))),
1485 (VBROADCASTF32X4rm addr:$src)>;
1486 def : Pat<(v8i64 (X86SubVBroadcast (loadv2i64 addr:$src))),
1487 (VBROADCASTI32X4rm addr:$src)>;
1488 def : Pat<(v32i16 (X86SubVBroadcast (bc_v8i16 (loadv2i64 addr:$src)))),
1489 (VBROADCASTI32X4rm addr:$src)>;
1490 def : Pat<(v64i8 (X86SubVBroadcast (bc_v16i8 (loadv2i64 addr:$src)))),
1491 (VBROADCASTI32X4rm addr:$src)>;
1493 // Patterns for selects of bitcasted operations.
1494 def : Pat<(vselect VK16WM:$mask,
1495 (bc_v16f32 (v8f64 (X86SubVBroadcast (loadv2f64 addr:$src)))),
1496 (bc_v16f32 (v16i32 immAllZerosV))),
1497 (VBROADCASTF32X4rmkz VK16WM:$mask, addr:$src)>;
1498 def : Pat<(vselect VK16WM:$mask,
1499 (bc_v16f32 (v8f64 (X86SubVBroadcast (loadv2f64 addr:$src)))),
1501 (VBROADCASTF32X4rmk VR512:$src0, VK16WM:$mask, addr:$src)>;
1502 def : Pat<(vselect VK16WM:$mask,
1503 (bc_v16i32 (v8i64 (X86SubVBroadcast (loadv2i64 addr:$src)))),
1504 (v16i32 immAllZerosV)),
1505 (VBROADCASTI32X4rmkz VK16WM:$mask, addr:$src)>;
1506 def : Pat<(vselect VK16WM:$mask,
1507 (bc_v16i32 (v8i64 (X86SubVBroadcast (loadv2i64 addr:$src)))),
1509 (VBROADCASTI32X4rmk VR512:$src0, VK16WM:$mask, addr:$src)>;
1511 def : Pat<(vselect VK8WM:$mask,
1512 (bc_v8f64 (v16f32 (X86SubVBroadcast (loadv8f32 addr:$src)))),
1513 (bc_v8f64 (v16i32 immAllZerosV))),
1514 (VBROADCASTF64X4rmkz VK8WM:$mask, addr:$src)>;
1515 def : Pat<(vselect VK8WM:$mask,
1516 (bc_v8f64 (v16f32 (X86SubVBroadcast (loadv8f32 addr:$src)))),
1518 (VBROADCASTF64X4rmk VR512:$src0, VK8WM:$mask, addr:$src)>;
1519 def : Pat<(vselect VK8WM:$mask,
1520 (bc_v8i64 (v16i32 (X86SubVBroadcast (bc_v8i32 (loadv4i64 addr:$src))))),
1521 (bc_v8i64 (v16i32 immAllZerosV))),
1522 (VBROADCASTI64X4rmkz VK8WM:$mask, addr:$src)>;
1523 def : Pat<(vselect VK8WM:$mask,
1524 (bc_v8i64 (v16i32 (X86SubVBroadcast (bc_v8i32 (loadv4i64 addr:$src))))),
1526 (VBROADCASTI64X4rmk VR512:$src0, VK8WM:$mask, addr:$src)>;
1529 let Predicates = [HasVLX] in {
1530 defm VBROADCASTI32X4Z256 : avx512_subvec_broadcast_rm<0x5a, "vbroadcasti32x4",
1531 v8i32x_info, v4i32x_info>,
1532 EVEX_V256, EVEX_CD8<32, CD8VT4>;
1533 defm VBROADCASTF32X4Z256 : avx512_subvec_broadcast_rm<0x1a, "vbroadcastf32x4",
1534 v8f32x_info, v4f32x_info>,
1535 EVEX_V256, EVEX_CD8<32, CD8VT4>;
1537 def : Pat<(v4f64 (X86SubVBroadcast (loadv2f64 addr:$src))),
1538 (VBROADCASTF32X4Z256rm addr:$src)>;
1539 def : Pat<(v4i64 (X86SubVBroadcast (loadv2i64 addr:$src))),
1540 (VBROADCASTI32X4Z256rm addr:$src)>;
1541 def : Pat<(v16i16 (X86SubVBroadcast (bc_v8i16 (loadv2i64 addr:$src)))),
1542 (VBROADCASTI32X4Z256rm addr:$src)>;
1543 def : Pat<(v32i8 (X86SubVBroadcast (bc_v16i8 (loadv2i64 addr:$src)))),
1544 (VBROADCASTI32X4Z256rm addr:$src)>;
1546 // Patterns for selects of bitcasted operations.
1547 def : Pat<(vselect VK8WM:$mask,
1548 (bc_v8f32 (v4f64 (X86SubVBroadcast (loadv2f64 addr:$src)))),
1549 (bc_v8f32 (v8i32 immAllZerosV))),
1550 (VBROADCASTF32X4Z256rmkz VK8WM:$mask, addr:$src)>;
1551 def : Pat<(vselect VK8WM:$mask,
1552 (bc_v8f32 (v4f64 (X86SubVBroadcast (loadv2f64 addr:$src)))),
1554 (VBROADCASTF32X4Z256rmk VR256X:$src0, VK8WM:$mask, addr:$src)>;
1555 def : Pat<(vselect VK8WM:$mask,
1556 (bc_v8i32 (v4i64 (X86SubVBroadcast (loadv2i64 addr:$src)))),
1557 (v8i32 immAllZerosV)),
1558 (VBROADCASTI32X4Z256rmkz VK8WM:$mask, addr:$src)>;
1559 def : Pat<(vselect VK8WM:$mask,
1560 (bc_v8i32 (v4i64 (X86SubVBroadcast (loadv2i64 addr:$src)))),
1562 (VBROADCASTI32X4Z256rmk VR256X:$src0, VK8WM:$mask, addr:$src)>;
1565 // Provide fallback in case the load node that is used in the patterns above
1566 // is used by additional users, which prevents the pattern selection.
1567 def : Pat<(v4f64 (X86SubVBroadcast (v2f64 VR128X:$src))),
1568 (VINSERTF32x4Z256rr (INSERT_SUBREG (v4f64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
1569 (v2f64 VR128X:$src), 1)>;
1570 def : Pat<(v8f32 (X86SubVBroadcast (v4f32 VR128X:$src))),
1571 (VINSERTF32x4Z256rr (INSERT_SUBREG (v8f32 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
1572 (v4f32 VR128X:$src), 1)>;
1573 def : Pat<(v4i64 (X86SubVBroadcast (v2i64 VR128X:$src))),
1574 (VINSERTI32x4Z256rr (INSERT_SUBREG (v4i64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
1575 (v2i64 VR128X:$src), 1)>;
1576 def : Pat<(v8i32 (X86SubVBroadcast (v4i32 VR128X:$src))),
1577 (VINSERTI32x4Z256rr (INSERT_SUBREG (v8i32 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
1578 (v4i32 VR128X:$src), 1)>;
1579 def : Pat<(v16i16 (X86SubVBroadcast (v8i16 VR128X:$src))),
1580 (VINSERTI32x4Z256rr (INSERT_SUBREG (v16i16 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
1581 (v8i16 VR128X:$src), 1)>;
1582 def : Pat<(v32i8 (X86SubVBroadcast (v16i8 VR128X:$src))),
1583 (VINSERTI32x4Z256rr (INSERT_SUBREG (v32i8 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
1584 (v16i8 VR128X:$src), 1)>;
1587 let Predicates = [HasVLX, HasDQI] in {
1588 defm VBROADCASTI64X2Z128 : avx512_subvec_broadcast_rm_dq<0x5a, "vbroadcasti64x2",
1589 v4i64x_info, v2i64x_info>, VEX_W1X,
1590 EVEX_V256, EVEX_CD8<64, CD8VT2>;
1591 defm VBROADCASTF64X2Z128 : avx512_subvec_broadcast_rm_dq<0x1a, "vbroadcastf64x2",
1592 v4f64x_info, v2f64x_info>, VEX_W1X,
1593 EVEX_V256, EVEX_CD8<64, CD8VT2>;
1595 // Patterns for selects of bitcasted operations.
1596 def : Pat<(vselect VK4WM:$mask,
1597 (bc_v4f64 (v8f32 (X86SubVBroadcast (loadv4f32 addr:$src)))),
1598 (bc_v4f64 (v8i32 immAllZerosV))),
1599 (VBROADCASTF64X2Z128rmkz VK4WM:$mask, addr:$src)>;
1600 def : Pat<(vselect VK4WM:$mask,
1601 (bc_v4f64 (v8f32 (X86SubVBroadcast (loadv4f32 addr:$src)))),
1603 (VBROADCASTF64X2Z128rmk VR256X:$src0, VK4WM:$mask, addr:$src)>;
1604 def : Pat<(vselect VK4WM:$mask,
1605 (bc_v4i64 (v8i32 (X86SubVBroadcast (bc_v4i32 (loadv2i64 addr:$src))))),
1606 (bc_v4i64 (v8i32 immAllZerosV))),
1607 (VBROADCASTI64X2Z128rmkz VK4WM:$mask, addr:$src)>;
1608 def : Pat<(vselect VK4WM:$mask,
1609 (bc_v4i64 (v8i32 (X86SubVBroadcast (bc_v4i32 (loadv2i64 addr:$src))))),
1611 (VBROADCASTI64X2Z128rmk VR256X:$src0, VK4WM:$mask, addr:$src)>;
1614 let Predicates = [HasDQI] in {
1615 defm VBROADCASTI64X2 : avx512_subvec_broadcast_rm_dq<0x5a, "vbroadcasti64x2",
1616 v8i64_info, v2i64x_info>, VEX_W,
1617 EVEX_V512, EVEX_CD8<64, CD8VT2>;
1618 defm VBROADCASTI32X8 : avx512_subvec_broadcast_rm_dq<0x5b, "vbroadcasti32x8",
1619 v16i32_info, v8i32x_info>,
1620 EVEX_V512, EVEX_CD8<32, CD8VT8>;
1621 defm VBROADCASTF64X2 : avx512_subvec_broadcast_rm_dq<0x1a, "vbroadcastf64x2",
1622 v8f64_info, v2f64x_info>, VEX_W,
1623 EVEX_V512, EVEX_CD8<64, CD8VT2>;
1624 defm VBROADCASTF32X8 : avx512_subvec_broadcast_rm_dq<0x1b, "vbroadcastf32x8",
1625 v16f32_info, v8f32x_info>,
1626 EVEX_V512, EVEX_CD8<32, CD8VT8>;
1628 // Patterns for selects of bitcasted operations.
1629 def : Pat<(vselect VK16WM:$mask,
1630 (bc_v16f32 (v8f64 (X86SubVBroadcast (loadv4f64 addr:$src)))),
1631 (bc_v16f32 (v16i32 immAllZerosV))),
1632 (VBROADCASTF32X8rmkz VK16WM:$mask, addr:$src)>;
1633 def : Pat<(vselect VK16WM:$mask,
1634 (bc_v16f32 (v8f64 (X86SubVBroadcast (loadv4f64 addr:$src)))),
1636 (VBROADCASTF32X8rmk VR512:$src0, VK16WM:$mask, addr:$src)>;
1637 def : Pat<(vselect VK16WM:$mask,
1638 (bc_v16i32 (v8i64 (X86SubVBroadcast (loadv4i64 addr:$src)))),
1639 (v16i32 immAllZerosV)),
1640 (VBROADCASTI32X8rmkz VK16WM:$mask, addr:$src)>;
1641 def : Pat<(vselect VK16WM:$mask,
1642 (bc_v16i32 (v8i64 (X86SubVBroadcast (loadv4i64 addr:$src)))),
1644 (VBROADCASTI32X8rmk VR512:$src0, VK16WM:$mask, addr:$src)>;
1646 def : Pat<(vselect VK8WM:$mask,
1647 (bc_v8f64 (v16f32 (X86SubVBroadcast (loadv4f32 addr:$src)))),
1648 (bc_v8f64 (v16i32 immAllZerosV))),
1649 (VBROADCASTF64X2rmkz VK8WM:$mask, addr:$src)>;
1650 def : Pat<(vselect VK8WM:$mask,
1651 (bc_v8f64 (v16f32 (X86SubVBroadcast (loadv4f32 addr:$src)))),
1653 (VBROADCASTF64X2rmk VR512:$src0, VK8WM:$mask, addr:$src)>;
1654 def : Pat<(vselect VK8WM:$mask,
1655 (bc_v8i64 (v16i32 (X86SubVBroadcast (bc_v4i32 (loadv2i64 addr:$src))))),
1656 (bc_v8i64 (v16i32 immAllZerosV))),
1657 (VBROADCASTI64X2rmkz VK8WM:$mask, addr:$src)>;
1658 def : Pat<(vselect VK8WM:$mask,
1659 (bc_v8i64 (v16i32 (X86SubVBroadcast (bc_v4i32 (loadv2i64 addr:$src))))),
1661 (VBROADCASTI64X2rmk VR512:$src0, VK8WM:$mask, addr:$src)>;
1664 multiclass avx512_common_broadcast_32x2<bits<8> opc, string OpcodeStr,
1665 AVX512VLVectorVTInfo _Dst, AVX512VLVectorVTInfo _Src> {
1666 let Predicates = [HasDQI] in
1667 defm Z : avx512_broadcast_rm_split<opc, OpcodeStr, NAME, WriteShuffle256,
1668 WriteShuffle256Ld, _Dst.info512,
1669 _Src.info512, _Src.info128, null_frag>,
1671 let Predicates = [HasDQI, HasVLX] in
1672 defm Z256 : avx512_broadcast_rm_split<opc, OpcodeStr, NAME, WriteShuffle256,
1673 WriteShuffle256Ld, _Dst.info256,
1674 _Src.info256, _Src.info128, null_frag>,
1678 multiclass avx512_common_broadcast_i32x2<bits<8> opc, string OpcodeStr,
1679 AVX512VLVectorVTInfo _Dst, AVX512VLVectorVTInfo _Src> :
1680 avx512_common_broadcast_32x2<opc, OpcodeStr, _Dst, _Src> {
1682 let Predicates = [HasDQI, HasVLX] in
1683 defm Z128 : avx512_broadcast_rm_split<opc, OpcodeStr, NAME, WriteShuffle,
1684 WriteShuffleXLd, _Dst.info128,
1685 _Src.info128, _Src.info128, null_frag>,
1689 defm VBROADCASTI32X2 : avx512_common_broadcast_i32x2<0x59, "vbroadcasti32x2",
1690 avx512vl_i32_info, avx512vl_i64_info>;
1691 defm VBROADCASTF32X2 : avx512_common_broadcast_32x2<0x19, "vbroadcastf32x2",
1692 avx512vl_f32_info, avx512vl_f64_info>;
1694 let Predicates = [HasVLX] in {
1695 def : Pat<(v8f32 (X86VBroadcast (v8f32 VR256X:$src))),
1696 (VBROADCASTSSZ256r (EXTRACT_SUBREG (v8f32 VR256X:$src), sub_xmm))>;
1697 def : Pat<(v4f64 (X86VBroadcast (v4f64 VR256X:$src))),
1698 (VBROADCASTSDZ256r (EXTRACT_SUBREG (v4f64 VR256X:$src), sub_xmm))>;
1701 def : Pat<(v16f32 (X86VBroadcast (v16f32 VR512:$src))),
1702 (VBROADCASTSSZr (EXTRACT_SUBREG (v16f32 VR512:$src), sub_xmm))>;
1703 def : Pat<(v16f32 (X86VBroadcast (v8f32 VR256X:$src))),
1704 (VBROADCASTSSZr (EXTRACT_SUBREG (v8f32 VR256X:$src), sub_xmm))>;
1706 def : Pat<(v8f64 (X86VBroadcast (v8f64 VR512:$src))),
1707 (VBROADCASTSDZr (EXTRACT_SUBREG (v8f64 VR512:$src), sub_xmm))>;
1708 def : Pat<(v8f64 (X86VBroadcast (v4f64 VR256X:$src))),
1709 (VBROADCASTSDZr (EXTRACT_SUBREG (v4f64 VR256X:$src), sub_xmm))>;
1711 //===----------------------------------------------------------------------===//
1712 // AVX-512 BROADCAST MASK TO VECTOR REGISTER
1714 multiclass avx512_mask_broadcastm<bits<8> opc, string OpcodeStr,
1715 X86VectorVTInfo _, RegisterClass KRC> {
1716 def rr : AVX512XS8I<opc, MRMSrcReg, (outs _.RC:$dst), (ins KRC:$src),
1717 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
1718 [(set _.RC:$dst, (_.VT (X86VBroadcastm KRC:$src)))]>,
1719 EVEX, Sched<[WriteShuffle]>;
1722 multiclass avx512_mask_broadcast<bits<8> opc, string OpcodeStr,
1723 AVX512VLVectorVTInfo VTInfo, RegisterClass KRC> {
1724 let Predicates = [HasCDI] in
1725 defm Z : avx512_mask_broadcastm<opc, OpcodeStr, VTInfo.info512, KRC>, EVEX_V512;
1726 let Predicates = [HasCDI, HasVLX] in {
1727 defm Z256 : avx512_mask_broadcastm<opc, OpcodeStr, VTInfo.info256, KRC>, EVEX_V256;
1728 defm Z128 : avx512_mask_broadcastm<opc, OpcodeStr, VTInfo.info128, KRC>, EVEX_V128;
1732 defm VPBROADCASTMW2D : avx512_mask_broadcast<0x3A, "vpbroadcastmw2d",
1733 avx512vl_i32_info, VK16>;
1734 defm VPBROADCASTMB2Q : avx512_mask_broadcast<0x2A, "vpbroadcastmb2q",
1735 avx512vl_i64_info, VK8>, VEX_W;
1737 //===----------------------------------------------------------------------===//
1738 // -- VPERMI2 - 3 source operands form --
1739 multiclass avx512_perm_i<bits<8> opc, string OpcodeStr,
1740 X86FoldableSchedWrite sched,
1741 X86VectorVTInfo _, X86VectorVTInfo IdxVT> {
1742 let Constraints = "$src1 = $dst", ExeDomain = _.ExeDomain,
1743 hasSideEffects = 0 in {
1744 defm rr: AVX512_maskable_3src_cast<opc, MRMSrcReg, _, IdxVT, (outs _.RC:$dst),
1745 (ins _.RC:$src2, _.RC:$src3),
1746 OpcodeStr, "$src3, $src2", "$src2, $src3",
1747 (_.VT (X86VPermt2 _.RC:$src2, IdxVT.RC:$src1, _.RC:$src3)), 1>,
1748 EVEX_4V, AVX5128IBase, Sched<[sched]>;
1751 defm rm: AVX512_maskable_3src_cast<opc, MRMSrcMem, _, IdxVT, (outs _.RC:$dst),
1752 (ins _.RC:$src2, _.MemOp:$src3),
1753 OpcodeStr, "$src3, $src2", "$src2, $src3",
1754 (_.VT (X86VPermt2 _.RC:$src2, IdxVT.RC:$src1,
1755 (_.VT (bitconvert (_.LdFrag addr:$src3))))), 1>,
1756 EVEX_4V, AVX5128IBase, Sched<[sched.Folded, ReadAfterLd]>;
1760 multiclass avx512_perm_i_mb<bits<8> opc, string OpcodeStr,
1761 X86FoldableSchedWrite sched,
1762 X86VectorVTInfo _, X86VectorVTInfo IdxVT> {
1763 let Constraints = "$src1 = $dst", ExeDomain = _.ExeDomain,
1764 hasSideEffects = 0, mayLoad = 1 in
1765 defm rmb: AVX512_maskable_3src_cast<opc, MRMSrcMem, _, IdxVT, (outs _.RC:$dst),
1766 (ins _.RC:$src2, _.ScalarMemOp:$src3),
1767 OpcodeStr, !strconcat("${src3}", _.BroadcastStr,", $src2"),
1768 !strconcat("$src2, ${src3}", _.BroadcastStr ),
1769 (_.VT (X86VPermt2 _.RC:$src2,
1770 IdxVT.RC:$src1,(_.VT (X86VBroadcast (_.ScalarLdFrag addr:$src3))))), 1>,
1771 AVX5128IBase, EVEX_4V, EVEX_B,
1772 Sched<[sched.Folded, ReadAfterLd]>;
1775 multiclass avx512_perm_i_sizes<bits<8> opc, string OpcodeStr,
1776 X86FoldableSchedWrite sched,
1777 AVX512VLVectorVTInfo VTInfo,
1778 AVX512VLVectorVTInfo ShuffleMask> {
1779 defm NAME: avx512_perm_i<opc, OpcodeStr, sched, VTInfo.info512,
1780 ShuffleMask.info512>,
1781 avx512_perm_i_mb<opc, OpcodeStr, sched, VTInfo.info512,
1782 ShuffleMask.info512>, EVEX_V512;
1783 let Predicates = [HasVLX] in {
1784 defm NAME#128: avx512_perm_i<opc, OpcodeStr, sched, VTInfo.info128,
1785 ShuffleMask.info128>,
1786 avx512_perm_i_mb<opc, OpcodeStr, sched, VTInfo.info128,
1787 ShuffleMask.info128>, EVEX_V128;
1788 defm NAME#256: avx512_perm_i<opc, OpcodeStr, sched, VTInfo.info256,
1789 ShuffleMask.info256>,
1790 avx512_perm_i_mb<opc, OpcodeStr, sched, VTInfo.info256,
1791 ShuffleMask.info256>, EVEX_V256;
1795 multiclass avx512_perm_i_sizes_bw<bits<8> opc, string OpcodeStr,
1796 X86FoldableSchedWrite sched,
1797 AVX512VLVectorVTInfo VTInfo,
1798 AVX512VLVectorVTInfo Idx,
1800 let Predicates = [Prd] in
1801 defm NAME: avx512_perm_i<opc, OpcodeStr, sched, VTInfo.info512,
1802 Idx.info512>, EVEX_V512;
1803 let Predicates = [Prd, HasVLX] in {
1804 defm NAME#128: avx512_perm_i<opc, OpcodeStr, sched, VTInfo.info128,
1805 Idx.info128>, EVEX_V128;
1806 defm NAME#256: avx512_perm_i<opc, OpcodeStr, sched, VTInfo.info256,
1807 Idx.info256>, EVEX_V256;
1811 defm VPERMI2D : avx512_perm_i_sizes<0x76, "vpermi2d", WriteVarShuffle256,
1812 avx512vl_i32_info, avx512vl_i32_info>, EVEX_CD8<32, CD8VF>;
1813 defm VPERMI2Q : avx512_perm_i_sizes<0x76, "vpermi2q", WriteVarShuffle256,
1814 avx512vl_i64_info, avx512vl_i64_info>, VEX_W, EVEX_CD8<64, CD8VF>;
1815 defm VPERMI2W : avx512_perm_i_sizes_bw<0x75, "vpermi2w", WriteVarShuffle256,
1816 avx512vl_i16_info, avx512vl_i16_info, HasBWI>,
1817 VEX_W, EVEX_CD8<16, CD8VF>;
1818 defm VPERMI2B : avx512_perm_i_sizes_bw<0x75, "vpermi2b", WriteVarShuffle256,
1819 avx512vl_i8_info, avx512vl_i8_info, HasVBMI>,
1821 defm VPERMI2PS : avx512_perm_i_sizes<0x77, "vpermi2ps", WriteFVarShuffle256,
1822 avx512vl_f32_info, avx512vl_i32_info>, EVEX_CD8<32, CD8VF>;
1823 defm VPERMI2PD : avx512_perm_i_sizes<0x77, "vpermi2pd", WriteFVarShuffle256,
1824 avx512vl_f64_info, avx512vl_i64_info>, VEX_W, EVEX_CD8<64, CD8VF>;
1826 // Extra patterns to deal with extra bitcasts due to passthru and index being
1827 // different types on the fp versions.
1828 multiclass avx512_perm_i_lowering<string InstrStr, X86VectorVTInfo _,
1829 X86VectorVTInfo IdxVT,
1830 X86VectorVTInfo CastVT> {
1831 def : Pat<(_.VT (vselect _.KRCWM:$mask,
1832 (X86VPermt2 (_.VT _.RC:$src2),
1833 (IdxVT.VT (bitconvert (CastVT.VT _.RC:$src1))), _.RC:$src3),
1834 (_.VT (bitconvert (CastVT.VT _.RC:$src1))))),
1835 (!cast<Instruction>(InstrStr#"rrk") _.RC:$src1, _.KRCWM:$mask,
1836 _.RC:$src2, _.RC:$src3)>;
1837 def : Pat<(_.VT (vselect _.KRCWM:$mask,
1838 (X86VPermt2 _.RC:$src2,
1839 (IdxVT.VT (bitconvert (CastVT.VT _.RC:$src1))),
1840 (_.LdFrag addr:$src3)),
1841 (_.VT (bitconvert (CastVT.VT _.RC:$src1))))),
1842 (!cast<Instruction>(InstrStr#"rmk") _.RC:$src1, _.KRCWM:$mask,
1843 _.RC:$src2, addr:$src3)>;
1844 def : Pat<(_.VT (vselect _.KRCWM:$mask,
1845 (X86VPermt2 _.RC:$src2,
1846 (IdxVT.VT (bitconvert (CastVT.VT _.RC:$src1))),
1847 (X86VBroadcast (_.ScalarLdFrag addr:$src3))),
1848 (_.VT (bitconvert (CastVT.VT _.RC:$src1))))),
1849 (!cast<Instruction>(InstrStr#"rmbk") _.RC:$src1, _.KRCWM:$mask,
1850 _.RC:$src2, addr:$src3)>;
1853 // TODO: Should we add more casts? The vXi64 case is common due to ABI.
1854 defm : avx512_perm_i_lowering<"VPERMI2PS", v16f32_info, v16i32_info, v8i64_info>;
1855 defm : avx512_perm_i_lowering<"VPERMI2PS256", v8f32x_info, v8i32x_info, v4i64x_info>;
1856 defm : avx512_perm_i_lowering<"VPERMI2PS128", v4f32x_info, v4i32x_info, v2i64x_info>;
1859 multiclass avx512_perm_t<bits<8> opc, string OpcodeStr,
1860 X86FoldableSchedWrite sched,
1861 X86VectorVTInfo _, X86VectorVTInfo IdxVT> {
1862 let Constraints = "$src1 = $dst", ExeDomain = _.ExeDomain in {
1863 defm rr: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
1864 (ins IdxVT.RC:$src2, _.RC:$src3),
1865 OpcodeStr, "$src3, $src2", "$src2, $src3",
1866 (_.VT (X86VPermt2 _.RC:$src1, IdxVT.RC:$src2, _.RC:$src3)), 1>,
1867 EVEX_4V, AVX5128IBase, Sched<[sched]>;
1869 defm rm: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
1870 (ins IdxVT.RC:$src2, _.MemOp:$src3),
1871 OpcodeStr, "$src3, $src2", "$src2, $src3",
1872 (_.VT (X86VPermt2 _.RC:$src1, IdxVT.RC:$src2,
1873 (bitconvert (_.LdFrag addr:$src3)))), 1>,
1874 EVEX_4V, AVX5128IBase, Sched<[sched.Folded, ReadAfterLd]>;
1877 multiclass avx512_perm_t_mb<bits<8> opc, string OpcodeStr,
1878 X86FoldableSchedWrite sched,
1879 X86VectorVTInfo _, X86VectorVTInfo IdxVT> {
1880 let Constraints = "$src1 = $dst", ExeDomain = _.ExeDomain in
1881 defm rmb: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
1882 (ins IdxVT.RC:$src2, _.ScalarMemOp:$src3),
1883 OpcodeStr, !strconcat("${src3}", _.BroadcastStr,", $src2"),
1884 !strconcat("$src2, ${src3}", _.BroadcastStr ),
1885 (_.VT (X86VPermt2 _.RC:$src1,
1886 IdxVT.RC:$src2,(_.VT (X86VBroadcast (_.ScalarLdFrag addr:$src3))))), 1>,
1887 AVX5128IBase, EVEX_4V, EVEX_B,
1888 Sched<[sched.Folded, ReadAfterLd]>;
1891 multiclass avx512_perm_t_sizes<bits<8> opc, string OpcodeStr,
1892 X86FoldableSchedWrite sched,
1893 AVX512VLVectorVTInfo VTInfo,
1894 AVX512VLVectorVTInfo ShuffleMask> {
1895 defm NAME: avx512_perm_t<opc, OpcodeStr, sched, VTInfo.info512,
1896 ShuffleMask.info512>,
1897 avx512_perm_t_mb<opc, OpcodeStr, sched, VTInfo.info512,
1898 ShuffleMask.info512>, EVEX_V512;
1899 let Predicates = [HasVLX] in {
1900 defm NAME#128: avx512_perm_t<opc, OpcodeStr, sched, VTInfo.info128,
1901 ShuffleMask.info128>,
1902 avx512_perm_t_mb<opc, OpcodeStr, sched, VTInfo.info128,
1903 ShuffleMask.info128>, EVEX_V128;
1904 defm NAME#256: avx512_perm_t<opc, OpcodeStr, sched, VTInfo.info256,
1905 ShuffleMask.info256>,
1906 avx512_perm_t_mb<opc, OpcodeStr, sched, VTInfo.info256,
1907 ShuffleMask.info256>, EVEX_V256;
1911 multiclass avx512_perm_t_sizes_bw<bits<8> opc, string OpcodeStr,
1912 X86FoldableSchedWrite sched,
1913 AVX512VLVectorVTInfo VTInfo,
1914 AVX512VLVectorVTInfo Idx, Predicate Prd> {
1915 let Predicates = [Prd] in
1916 defm NAME: avx512_perm_t<opc, OpcodeStr, sched, VTInfo.info512,
1917 Idx.info512>, EVEX_V512;
1918 let Predicates = [Prd, HasVLX] in {
1919 defm NAME#128: avx512_perm_t<opc, OpcodeStr, sched, VTInfo.info128,
1920 Idx.info128>, EVEX_V128;
1921 defm NAME#256: avx512_perm_t<opc, OpcodeStr, sched, VTInfo.info256,
1922 Idx.info256>, EVEX_V256;
1926 defm VPERMT2D : avx512_perm_t_sizes<0x7E, "vpermt2d", WriteVarShuffle256,
1927 avx512vl_i32_info, avx512vl_i32_info>, EVEX_CD8<32, CD8VF>;
1928 defm VPERMT2Q : avx512_perm_t_sizes<0x7E, "vpermt2q", WriteVarShuffle256,
1929 avx512vl_i64_info, avx512vl_i64_info>, VEX_W, EVEX_CD8<64, CD8VF>;
1930 defm VPERMT2W : avx512_perm_t_sizes_bw<0x7D, "vpermt2w", WriteVarShuffle256,
1931 avx512vl_i16_info, avx512vl_i16_info, HasBWI>,
1932 VEX_W, EVEX_CD8<16, CD8VF>;
1933 defm VPERMT2B : avx512_perm_t_sizes_bw<0x7D, "vpermt2b", WriteVarShuffle256,
1934 avx512vl_i8_info, avx512vl_i8_info, HasVBMI>,
1936 defm VPERMT2PS : avx512_perm_t_sizes<0x7F, "vpermt2ps", WriteFVarShuffle256,
1937 avx512vl_f32_info, avx512vl_i32_info>, EVEX_CD8<32, CD8VF>;
1938 defm VPERMT2PD : avx512_perm_t_sizes<0x7F, "vpermt2pd", WriteFVarShuffle256,
1939 avx512vl_f64_info, avx512vl_i64_info>, VEX_W, EVEX_CD8<64, CD8VF>;
1941 //===----------------------------------------------------------------------===//
1942 // AVX-512 - BLEND using mask
1945 multiclass WriteFVarBlendask<bits<8> opc, string OpcodeStr,
1946 X86FoldableSchedWrite sched, X86VectorVTInfo _> {
1947 let ExeDomain = _.ExeDomain, hasSideEffects = 0 in {
1948 def rr : AVX5128I<opc, MRMSrcReg, (outs _.RC:$dst),
1949 (ins _.RC:$src1, _.RC:$src2),
1950 !strconcat(OpcodeStr,
1951 "\t{$src2, $src1, ${dst}|${dst}, $src1, $src2}"), []>,
1952 EVEX_4V, Sched<[sched]>;
1953 def rrk : AVX5128I<opc, MRMSrcReg, (outs _.RC:$dst),
1954 (ins _.KRCWM:$mask, _.RC:$src1, _.RC:$src2),
1955 !strconcat(OpcodeStr,
1956 "\t{$src2, $src1, ${dst} {${mask}}|${dst} {${mask}}, $src1, $src2}"),
1957 []>, EVEX_4V, EVEX_K, Sched<[sched]>;
1958 def rrkz : AVX5128I<opc, MRMSrcReg, (outs _.RC:$dst),
1959 (ins _.KRCWM:$mask, _.RC:$src1, _.RC:$src2),
1960 !strconcat(OpcodeStr,
1961 "\t{$src2, $src1, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src1, $src2}"),
1962 []>, EVEX_4V, EVEX_KZ, Sched<[sched]>, NotMemoryFoldable;
1963 let mayLoad = 1 in {
1964 def rm : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst),
1965 (ins _.RC:$src1, _.MemOp:$src2),
1966 !strconcat(OpcodeStr,
1967 "\t{$src2, $src1, ${dst}|${dst}, $src1, $src2}"),
1968 []>, EVEX_4V, EVEX_CD8<_.EltSize, CD8VF>,
1969 Sched<[sched.Folded, ReadAfterLd]>;
1970 def rmk : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst),
1971 (ins _.KRCWM:$mask, _.RC:$src1, _.MemOp:$src2),
1972 !strconcat(OpcodeStr,
1973 "\t{$src2, $src1, ${dst} {${mask}}|${dst} {${mask}}, $src1, $src2}"),
1974 []>, EVEX_4V, EVEX_K, EVEX_CD8<_.EltSize, CD8VF>,
1975 Sched<[sched.Folded, ReadAfterLd]>;
1976 def rmkz : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst),
1977 (ins _.KRCWM:$mask, _.RC:$src1, _.MemOp:$src2),
1978 !strconcat(OpcodeStr,
1979 "\t{$src2, $src1, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src1, $src2}"),
1980 []>, EVEX_4V, EVEX_KZ, EVEX_CD8<_.EltSize, CD8VF>,
1981 Sched<[sched.Folded, ReadAfterLd]>, NotMemoryFoldable;
1985 multiclass WriteFVarBlendask_rmb<bits<8> opc, string OpcodeStr,
1986 X86FoldableSchedWrite sched, X86VectorVTInfo _> {
1987 let mayLoad = 1, hasSideEffects = 0 in {
1988 def rmbk : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst),
1989 (ins _.KRCWM:$mask, _.RC:$src1, _.ScalarMemOp:$src2),
1990 !strconcat(OpcodeStr,
1991 "\t{${src2}", _.BroadcastStr, ", $src1, $dst {${mask}}|",
1992 "$dst {${mask}}, $src1, ${src2}", _.BroadcastStr, "}"), []>,
1993 EVEX_4V, EVEX_K, EVEX_B, EVEX_CD8<_.EltSize, CD8VF>,
1994 Sched<[sched.Folded, ReadAfterLd]>;
1996 def rmbkz : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst),
1997 (ins _.KRCWM:$mask, _.RC:$src1, _.ScalarMemOp:$src2),
1998 !strconcat(OpcodeStr,
1999 "\t{${src2}", _.BroadcastStr, ", $src1, $dst {${mask}} {z}|",
2000 "$dst {${mask}} {z}, $src1, ${src2}", _.BroadcastStr, "}"), []>,
2001 EVEX_4V, EVEX_KZ, EVEX_B, EVEX_CD8<_.EltSize, CD8VF>,
2002 Sched<[sched.Folded, ReadAfterLd]>, NotMemoryFoldable;
2004 def rmb : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst),
2005 (ins _.RC:$src1, _.ScalarMemOp:$src2),
2006 !strconcat(OpcodeStr,
2007 "\t{${src2}", _.BroadcastStr, ", $src1, $dst|",
2008 "$dst, $src1, ${src2}", _.BroadcastStr, "}"), []>,
2009 EVEX_4V, EVEX_B, EVEX_CD8<_.EltSize, CD8VF>,
2010 Sched<[sched.Folded, ReadAfterLd]>;
2014 multiclass blendmask_dq<bits<8> opc, string OpcodeStr, X86SchedWriteWidths sched,
2015 AVX512VLVectorVTInfo VTInfo> {
2016 defm Z : WriteFVarBlendask<opc, OpcodeStr, sched.ZMM, VTInfo.info512>,
2017 WriteFVarBlendask_rmb<opc, OpcodeStr, sched.ZMM, VTInfo.info512>,
2020 let Predicates = [HasVLX] in {
2021 defm Z256 : WriteFVarBlendask<opc, OpcodeStr, sched.YMM, VTInfo.info256>,
2022 WriteFVarBlendask_rmb<opc, OpcodeStr, sched.YMM, VTInfo.info256>,
2024 defm Z128 : WriteFVarBlendask<opc, OpcodeStr, sched.XMM, VTInfo.info128>,
2025 WriteFVarBlendask_rmb<opc, OpcodeStr, sched.XMM, VTInfo.info128>,
2030 multiclass blendmask_bw<bits<8> opc, string OpcodeStr, X86SchedWriteWidths sched,
2031 AVX512VLVectorVTInfo VTInfo> {
2032 let Predicates = [HasBWI] in
2033 defm Z : WriteFVarBlendask<opc, OpcodeStr, sched.ZMM, VTInfo.info512>,
2036 let Predicates = [HasBWI, HasVLX] in {
2037 defm Z256 : WriteFVarBlendask<opc, OpcodeStr, sched.YMM, VTInfo.info256>,
2039 defm Z128 : WriteFVarBlendask<opc, OpcodeStr, sched.XMM, VTInfo.info128>,
2044 defm VBLENDMPS : blendmask_dq<0x65, "vblendmps", SchedWriteFVarBlend,
2046 defm VBLENDMPD : blendmask_dq<0x65, "vblendmpd", SchedWriteFVarBlend,
2047 avx512vl_f64_info>, VEX_W;
2048 defm VPBLENDMD : blendmask_dq<0x64, "vpblendmd", SchedWriteVarBlend,
2050 defm VPBLENDMQ : blendmask_dq<0x64, "vpblendmq", SchedWriteVarBlend,
2051 avx512vl_i64_info>, VEX_W;
2052 defm VPBLENDMB : blendmask_bw<0x66, "vpblendmb", SchedWriteVarBlend,
2054 defm VPBLENDMW : blendmask_bw<0x66, "vpblendmw", SchedWriteVarBlend,
2055 avx512vl_i16_info>, VEX_W;
2057 //===----------------------------------------------------------------------===//
2058 // Compare Instructions
2059 //===----------------------------------------------------------------------===//
2061 // avx512_cmp_scalar - AVX512 CMPSS and CMPSD
2063 multiclass avx512_cmp_scalar<X86VectorVTInfo _, SDNode OpNode, SDNode OpNodeRnd,
2064 X86FoldableSchedWrite sched> {
2065 defm rr_Int : AVX512_maskable_cmp<0xC2, MRMSrcReg, _,
2067 (ins _.RC:$src1, _.RC:$src2, AVXCC:$cc),
2068 "vcmp${cc}"#_.Suffix,
2069 "$src2, $src1", "$src1, $src2",
2070 (OpNode (_.VT _.RC:$src1),
2072 imm:$cc)>, EVEX_4V, Sched<[sched]>;
2074 defm rm_Int : AVX512_maskable_cmp<0xC2, MRMSrcMem, _,
2076 (ins _.RC:$src1, _.IntScalarMemOp:$src2, AVXCC:$cc),
2077 "vcmp${cc}"#_.Suffix,
2078 "$src2, $src1", "$src1, $src2",
2079 (OpNode (_.VT _.RC:$src1), _.ScalarIntMemCPat:$src2,
2080 imm:$cc)>, EVEX_4V, EVEX_CD8<_.EltSize, CD8VT1>,
2081 Sched<[sched.Folded, ReadAfterLd]>;
2083 defm rrb_Int : AVX512_maskable_cmp<0xC2, MRMSrcReg, _,
2085 (ins _.RC:$src1, _.RC:$src2, AVXCC:$cc),
2086 "vcmp${cc}"#_.Suffix,
2087 "{sae}, $src2, $src1", "$src1, $src2, {sae}",
2088 (OpNodeRnd (_.VT _.RC:$src1),
2091 (i32 FROUND_NO_EXC))>,
2092 EVEX_4V, EVEX_B, Sched<[sched]>;
2093 // Accept explicit immediate argument form instead of comparison code.
2094 let isAsmParserOnly = 1, hasSideEffects = 0 in {
2095 defm rri_alt : AVX512_maskable_cmp_alt<0xC2, MRMSrcReg, _,
2097 (ins _.RC:$src1, _.RC:$src2, u8imm:$cc),
2099 "$cc, $src2, $src1", "$src1, $src2, $cc">, EVEX_4V,
2100 Sched<[sched]>, NotMemoryFoldable;
2102 defm rmi_alt : AVX512_maskable_cmp_alt<0xC2, MRMSrcMem, _,
2104 (ins _.RC:$src1, _.ScalarMemOp:$src2, u8imm:$cc),
2106 "$cc, $src2, $src1", "$src1, $src2, $cc">,
2107 EVEX_4V, EVEX_CD8<_.EltSize, CD8VT1>,
2108 Sched<[sched.Folded, ReadAfterLd]>, NotMemoryFoldable;
2110 defm rrb_alt : AVX512_maskable_cmp_alt<0xC2, MRMSrcReg, _,
2112 (ins _.RC:$src1, _.RC:$src2, u8imm:$cc),
2114 "$cc, {sae}, $src2, $src1","$src1, $src2, {sae}, $cc">,
2115 EVEX_4V, EVEX_B, Sched<[sched]>, NotMemoryFoldable;
2116 }// let isAsmParserOnly = 1, hasSideEffects = 0
2118 let isCodeGenOnly = 1 in {
2119 let isCommutable = 1 in
2120 def rr : AVX512Ii8<0xC2, MRMSrcReg,
2121 (outs _.KRC:$dst), (ins _.FRC:$src1, _.FRC:$src2, AVXCC:$cc),
2122 !strconcat("vcmp${cc}", _.Suffix,
2123 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
2124 [(set _.KRC:$dst, (OpNode _.FRC:$src1,
2127 EVEX_4V, Sched<[sched]>;
2128 def rm : AVX512Ii8<0xC2, MRMSrcMem,
2130 (ins _.FRC:$src1, _.ScalarMemOp:$src2, AVXCC:$cc),
2131 !strconcat("vcmp${cc}", _.Suffix,
2132 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
2133 [(set _.KRC:$dst, (OpNode _.FRC:$src1,
2134 (_.ScalarLdFrag addr:$src2),
2136 EVEX_4V, EVEX_CD8<_.EltSize, CD8VT1>,
2137 Sched<[sched.Folded, ReadAfterLd]>;
2141 let Predicates = [HasAVX512] in {
2142 let ExeDomain = SSEPackedSingle in
2143 defm VCMPSSZ : avx512_cmp_scalar<f32x_info, X86cmpms, X86cmpmsRnd,
2144 SchedWriteFCmp.Scl>, AVX512XSIi8Base;
2145 let ExeDomain = SSEPackedDouble in
2146 defm VCMPSDZ : avx512_cmp_scalar<f64x_info, X86cmpms, X86cmpmsRnd,
2147 SchedWriteFCmp.Scl>, AVX512XDIi8Base, VEX_W;
2150 multiclass avx512_icmp_packed<bits<8> opc, string OpcodeStr, PatFrag OpNode,
2151 X86FoldableSchedWrite sched, X86VectorVTInfo _,
2153 let isCommutable = IsCommutable in
2154 def rr : AVX512BI<opc, MRMSrcReg,
2155 (outs _.KRC:$dst), (ins _.RC:$src1, _.RC:$src2),
2156 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
2157 [(set _.KRC:$dst, (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2)))]>,
2158 EVEX_4V, Sched<[sched]>;
2159 def rm : AVX512BI<opc, MRMSrcMem,
2160 (outs _.KRC:$dst), (ins _.RC:$src1, _.MemOp:$src2),
2161 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
2162 [(set _.KRC:$dst, (OpNode (_.VT _.RC:$src1),
2163 (_.VT (bitconvert (_.LdFrag addr:$src2)))))]>,
2164 EVEX_4V, Sched<[sched.Folded, ReadAfterLd]>;
2165 let isCommutable = IsCommutable in
2166 def rrk : AVX512BI<opc, MRMSrcReg,
2167 (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1, _.RC:$src2),
2168 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst {${mask}}|",
2169 "$dst {${mask}}, $src1, $src2}"),
2170 [(set _.KRC:$dst, (and _.KRCWM:$mask,
2171 (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2))))]>,
2172 EVEX_4V, EVEX_K, Sched<[sched]>;
2173 def rmk : AVX512BI<opc, MRMSrcMem,
2174 (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1, _.MemOp:$src2),
2175 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst {${mask}}|",
2176 "$dst {${mask}}, $src1, $src2}"),
2177 [(set _.KRC:$dst, (and _.KRCWM:$mask,
2178 (OpNode (_.VT _.RC:$src1),
2180 (_.LdFrag addr:$src2))))))]>,
2181 EVEX_4V, EVEX_K, Sched<[sched.Folded, ReadAfterLd]>;
2184 multiclass avx512_icmp_packed_rmb<bits<8> opc, string OpcodeStr, PatFrag OpNode,
2185 X86FoldableSchedWrite sched, X86VectorVTInfo _,
2187 avx512_icmp_packed<opc, OpcodeStr, OpNode, sched, _, IsCommutable> {
2188 def rmb : AVX512BI<opc, MRMSrcMem,
2189 (outs _.KRC:$dst), (ins _.RC:$src1, _.ScalarMemOp:$src2),
2190 !strconcat(OpcodeStr, "\t{${src2}", _.BroadcastStr, ", $src1, $dst",
2191 "|$dst, $src1, ${src2}", _.BroadcastStr, "}"),
2192 [(set _.KRC:$dst, (OpNode (_.VT _.RC:$src1),
2193 (X86VBroadcast (_.ScalarLdFrag addr:$src2))))]>,
2194 EVEX_4V, EVEX_B, Sched<[sched.Folded, ReadAfterLd]>;
2195 def rmbk : AVX512BI<opc, MRMSrcMem,
2196 (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1,
2197 _.ScalarMemOp:$src2),
2198 !strconcat(OpcodeStr,
2199 "\t{${src2}", _.BroadcastStr, ", $src1, $dst {${mask}}|",
2200 "$dst {${mask}}, $src1, ${src2}", _.BroadcastStr, "}"),
2201 [(set _.KRC:$dst, (and _.KRCWM:$mask,
2202 (OpNode (_.VT _.RC:$src1),
2204 (_.ScalarLdFrag addr:$src2)))))]>,
2205 EVEX_4V, EVEX_K, EVEX_B,
2206 Sched<[sched.Folded, ReadAfterLd]>;
2209 multiclass avx512_icmp_packed_vl<bits<8> opc, string OpcodeStr, PatFrag OpNode,
2210 X86SchedWriteWidths sched,
2211 AVX512VLVectorVTInfo VTInfo, Predicate prd,
2212 bit IsCommutable = 0> {
2213 let Predicates = [prd] in
2214 defm Z : avx512_icmp_packed<opc, OpcodeStr, OpNode, sched.ZMM,
2215 VTInfo.info512, IsCommutable>, EVEX_V512;
2217 let Predicates = [prd, HasVLX] in {
2218 defm Z256 : avx512_icmp_packed<opc, OpcodeStr, OpNode, sched.YMM,
2219 VTInfo.info256, IsCommutable>, EVEX_V256;
2220 defm Z128 : avx512_icmp_packed<opc, OpcodeStr, OpNode, sched.XMM,
2221 VTInfo.info128, IsCommutable>, EVEX_V128;
2225 multiclass avx512_icmp_packed_rmb_vl<bits<8> opc, string OpcodeStr,
2226 PatFrag OpNode, X86SchedWriteWidths sched,
2227 AVX512VLVectorVTInfo VTInfo,
2228 Predicate prd, bit IsCommutable = 0> {
2229 let Predicates = [prd] in
2230 defm Z : avx512_icmp_packed_rmb<opc, OpcodeStr, OpNode, sched.ZMM,
2231 VTInfo.info512, IsCommutable>, EVEX_V512;
2233 let Predicates = [prd, HasVLX] in {
2234 defm Z256 : avx512_icmp_packed_rmb<opc, OpcodeStr, OpNode, sched.YMM,
2235 VTInfo.info256, IsCommutable>, EVEX_V256;
2236 defm Z128 : avx512_icmp_packed_rmb<opc, OpcodeStr, OpNode, sched.XMM,
2237 VTInfo.info128, IsCommutable>, EVEX_V128;
2241 // This fragment treats X86cmpm as commutable to help match loads in both
2242 // operands for PCMPEQ.
2243 def X86setcc_commute : SDNode<"ISD::SETCC", SDTSetCC, [SDNPCommutative]>;
2244 def X86pcmpeqm_c : PatFrag<(ops node:$src1, node:$src2),
2245 (X86setcc_commute node:$src1, node:$src2, SETEQ)>;
2246 def X86pcmpgtm : PatFrag<(ops node:$src1, node:$src2),
2247 (setcc node:$src1, node:$src2, SETGT)>;
2249 // AddedComplexity is needed because the explicit SETEQ/SETGT CondCode doesn't
2250 // increase the pattern complexity the way an immediate would.
2251 let AddedComplexity = 2 in {
2252 // FIXME: Is there a better scheduler class for VPCMP?
2253 defm VPCMPEQB : avx512_icmp_packed_vl<0x74, "vpcmpeqb", X86pcmpeqm_c,
2254 SchedWriteVecALU, avx512vl_i8_info, HasBWI, 1>,
2255 EVEX_CD8<8, CD8VF>, VEX_WIG;
2257 defm VPCMPEQW : avx512_icmp_packed_vl<0x75, "vpcmpeqw", X86pcmpeqm_c,
2258 SchedWriteVecALU, avx512vl_i16_info, HasBWI, 1>,
2259 EVEX_CD8<16, CD8VF>, VEX_WIG;
2261 defm VPCMPEQD : avx512_icmp_packed_rmb_vl<0x76, "vpcmpeqd", X86pcmpeqm_c,
2262 SchedWriteVecALU, avx512vl_i32_info, HasAVX512, 1>,
2263 EVEX_CD8<32, CD8VF>;
2265 defm VPCMPEQQ : avx512_icmp_packed_rmb_vl<0x29, "vpcmpeqq", X86pcmpeqm_c,
2266 SchedWriteVecALU, avx512vl_i64_info, HasAVX512, 1>,
2267 T8PD, VEX_W, EVEX_CD8<64, CD8VF>;
2269 defm VPCMPGTB : avx512_icmp_packed_vl<0x64, "vpcmpgtb", X86pcmpgtm,
2270 SchedWriteVecALU, avx512vl_i8_info, HasBWI>,
2271 EVEX_CD8<8, CD8VF>, VEX_WIG;
2273 defm VPCMPGTW : avx512_icmp_packed_vl<0x65, "vpcmpgtw", X86pcmpgtm,
2274 SchedWriteVecALU, avx512vl_i16_info, HasBWI>,
2275 EVEX_CD8<16, CD8VF>, VEX_WIG;
2277 defm VPCMPGTD : avx512_icmp_packed_rmb_vl<0x66, "vpcmpgtd", X86pcmpgtm,
2278 SchedWriteVecALU, avx512vl_i32_info, HasAVX512>,
2279 EVEX_CD8<32, CD8VF>;
2281 defm VPCMPGTQ : avx512_icmp_packed_rmb_vl<0x37, "vpcmpgtq", X86pcmpgtm,
2282 SchedWriteVecALU, avx512vl_i64_info, HasAVX512>,
2283 T8PD, VEX_W, EVEX_CD8<64, CD8VF>;
2286 multiclass avx512_icmp_cc<bits<8> opc, string Suffix, PatFrag Frag,
2287 PatFrag CommFrag, X86FoldableSchedWrite sched,
2288 X86VectorVTInfo _, string Name> {
2289 let isCommutable = 1 in
2290 def rri : AVX512AIi8<opc, MRMSrcReg,
2291 (outs _.KRC:$dst), (ins _.RC:$src1, _.RC:$src2, AVX512ICC:$cc),
2292 !strconcat("vpcmp${cc}", Suffix,
2293 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
2294 [(set _.KRC:$dst, (_.KVT (Frag:$cc (_.VT _.RC:$src1),
2297 EVEX_4V, Sched<[sched]>;
2298 def rmi : AVX512AIi8<opc, MRMSrcMem,
2299 (outs _.KRC:$dst), (ins _.RC:$src1, _.MemOp:$src2, AVX512ICC:$cc),
2300 !strconcat("vpcmp${cc}", Suffix,
2301 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
2302 [(set _.KRC:$dst, (_.KVT
2305 (_.VT (bitconvert (_.LdFrag addr:$src2))),
2307 EVEX_4V, Sched<[sched.Folded, ReadAfterLd]>;
2308 let isCommutable = 1 in
2309 def rrik : AVX512AIi8<opc, MRMSrcReg,
2310 (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1, _.RC:$src2,
2312 !strconcat("vpcmp${cc}", Suffix,
2313 "\t{$src2, $src1, $dst {${mask}}|",
2314 "$dst {${mask}}, $src1, $src2}"),
2315 [(set _.KRC:$dst, (and _.KRCWM:$mask,
2316 (_.KVT (Frag:$cc (_.VT _.RC:$src1),
2319 EVEX_4V, EVEX_K, Sched<[sched]>;
2320 def rmik : AVX512AIi8<opc, MRMSrcMem,
2321 (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1, _.MemOp:$src2,
2323 !strconcat("vpcmp${cc}", Suffix,
2324 "\t{$src2, $src1, $dst {${mask}}|",
2325 "$dst {${mask}}, $src1, $src2}"),
2326 [(set _.KRC:$dst, (and _.KRCWM:$mask,
2331 (_.LdFrag addr:$src2))),
2333 EVEX_4V, EVEX_K, Sched<[sched.Folded, ReadAfterLd]>;
2335 // Accept explicit immediate argument form instead of comparison code.
2336 let isAsmParserOnly = 1, hasSideEffects = 0 in {
2337 def rri_alt : AVX512AIi8<opc, MRMSrcReg,
2338 (outs _.KRC:$dst), (ins _.RC:$src1, _.RC:$src2, u8imm:$cc),
2339 !strconcat("vpcmp", Suffix, "\t{$cc, $src2, $src1, $dst|",
2340 "$dst, $src1, $src2, $cc}"), []>,
2341 EVEX_4V, Sched<[sched]>, NotMemoryFoldable;
2343 def rmi_alt : AVX512AIi8<opc, MRMSrcMem,
2344 (outs _.KRC:$dst), (ins _.RC:$src1, _.MemOp:$src2, u8imm:$cc),
2345 !strconcat("vpcmp", Suffix, "\t{$cc, $src2, $src1, $dst|",
2346 "$dst, $src1, $src2, $cc}"), []>,
2347 EVEX_4V, Sched<[sched.Folded, ReadAfterLd]>, NotMemoryFoldable;
2348 def rrik_alt : AVX512AIi8<opc, MRMSrcReg,
2349 (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1, _.RC:$src2,
2351 !strconcat("vpcmp", Suffix,
2352 "\t{$cc, $src2, $src1, $dst {${mask}}|",
2353 "$dst {${mask}}, $src1, $src2, $cc}"), []>,
2354 EVEX_4V, EVEX_K, Sched<[sched]>, NotMemoryFoldable;
2356 def rmik_alt : AVX512AIi8<opc, MRMSrcMem,
2357 (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1, _.MemOp:$src2,
2359 !strconcat("vpcmp", Suffix,
2360 "\t{$cc, $src2, $src1, $dst {${mask}}|",
2361 "$dst {${mask}}, $src1, $src2, $cc}"), []>,
2362 EVEX_4V, EVEX_K, Sched<[sched.Folded, ReadAfterLd]>,
2366 def : Pat<(_.KVT (CommFrag:$cc (bitconvert (_.LdFrag addr:$src2)),
2367 (_.VT _.RC:$src1), cond)),
2368 (!cast<Instruction>(Name#_.ZSuffix#"rmi")
2369 _.RC:$src1, addr:$src2, (CommFrag.OperandTransform $cc))>;
2371 def : Pat<(and _.KRCWM:$mask,
2372 (_.KVT (CommFrag:$cc (bitconvert (_.LdFrag addr:$src2)),
2373 (_.VT _.RC:$src1), cond))),
2374 (!cast<Instruction>(Name#_.ZSuffix#"rmik")
2375 _.KRCWM:$mask, _.RC:$src1, addr:$src2,
2376 (CommFrag.OperandTransform $cc))>;
2379 multiclass avx512_icmp_cc_rmb<bits<8> opc, string Suffix, PatFrag Frag,
2380 PatFrag CommFrag, X86FoldableSchedWrite sched,
2381 X86VectorVTInfo _, string Name> :
2382 avx512_icmp_cc<opc, Suffix, Frag, CommFrag, sched, _, Name> {
2383 def rmib : AVX512AIi8<opc, MRMSrcMem,
2384 (outs _.KRC:$dst), (ins _.RC:$src1, _.ScalarMemOp:$src2,
2386 !strconcat("vpcmp${cc}", Suffix,
2387 "\t{${src2}", _.BroadcastStr, ", $src1, $dst|",
2388 "$dst, $src1, ${src2}", _.BroadcastStr, "}"),
2389 [(set _.KRC:$dst, (_.KVT (Frag:$cc
2392 (_.ScalarLdFrag addr:$src2)),
2394 EVEX_4V, EVEX_B, Sched<[sched.Folded, ReadAfterLd]>;
2395 def rmibk : AVX512AIi8<opc, MRMSrcMem,
2396 (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1,
2397 _.ScalarMemOp:$src2, AVX512ICC:$cc),
2398 !strconcat("vpcmp${cc}", Suffix,
2399 "\t{${src2}", _.BroadcastStr, ", $src1, $dst {${mask}}|",
2400 "$dst {${mask}}, $src1, ${src2}", _.BroadcastStr, "}"),
2401 [(set _.KRC:$dst, (and _.KRCWM:$mask,
2405 (_.ScalarLdFrag addr:$src2)),
2407 EVEX_4V, EVEX_K, EVEX_B, Sched<[sched.Folded, ReadAfterLd]>;
2409 // Accept explicit immediate argument form instead of comparison code.
2410 let isAsmParserOnly = 1, hasSideEffects = 0, mayLoad = 1 in {
2411 def rmib_alt : AVX512AIi8<opc, MRMSrcMem,
2412 (outs _.KRC:$dst), (ins _.RC:$src1, _.ScalarMemOp:$src2,
2414 !strconcat("vpcmp", Suffix,
2415 "\t{$cc, ${src2}", _.BroadcastStr, ", $src1, $dst|",
2416 "$dst, $src1, ${src2}", _.BroadcastStr, ", $cc}"), []>,
2417 EVEX_4V, EVEX_B, Sched<[sched.Folded, ReadAfterLd]>,
2419 def rmibk_alt : AVX512AIi8<opc, MRMSrcMem,
2420 (outs _.KRC:$dst), (ins _.KRCWM:$mask, _.RC:$src1,
2421 _.ScalarMemOp:$src2, u8imm:$cc),
2422 !strconcat("vpcmp", Suffix,
2423 "\t{$cc, ${src2}", _.BroadcastStr, ", $src1, $dst {${mask}}|",
2424 "$dst {${mask}}, $src1, ${src2}", _.BroadcastStr, ", $cc}"), []>,
2425 EVEX_4V, EVEX_K, EVEX_B, Sched<[sched.Folded, ReadAfterLd]>,
2429 def : Pat<(_.KVT (CommFrag:$cc (X86VBroadcast (_.ScalarLdFrag addr:$src2)),
2430 (_.VT _.RC:$src1), cond)),
2431 (!cast<Instruction>(Name#_.ZSuffix#"rmib")
2432 _.RC:$src1, addr:$src2, (CommFrag.OperandTransform $cc))>;
2434 def : Pat<(and _.KRCWM:$mask,
2435 (_.KVT (CommFrag:$cc (X86VBroadcast
2436 (_.ScalarLdFrag addr:$src2)),
2437 (_.VT _.RC:$src1), cond))),
2438 (!cast<Instruction>(Name#_.ZSuffix#"rmibk")
2439 _.KRCWM:$mask, _.RC:$src1, addr:$src2,
2440 (CommFrag.OperandTransform $cc))>;
2443 multiclass avx512_icmp_cc_vl<bits<8> opc, string Suffix, PatFrag Frag,
2444 PatFrag CommFrag, X86SchedWriteWidths sched,
2445 AVX512VLVectorVTInfo VTInfo, Predicate prd> {
2446 let Predicates = [prd] in
2447 defm Z : avx512_icmp_cc<opc, Suffix, Frag, CommFrag, sched.ZMM,
2448 VTInfo.info512, NAME>, EVEX_V512;
2450 let Predicates = [prd, HasVLX] in {
2451 defm Z256 : avx512_icmp_cc<opc, Suffix, Frag, CommFrag, sched.YMM,
2452 VTInfo.info256, NAME>, EVEX_V256;
2453 defm Z128 : avx512_icmp_cc<opc, Suffix, Frag, CommFrag, sched.XMM,
2454 VTInfo.info128, NAME>, EVEX_V128;
2458 multiclass avx512_icmp_cc_rmb_vl<bits<8> opc, string Suffix, PatFrag Frag,
2459 PatFrag CommFrag, X86SchedWriteWidths sched,
2460 AVX512VLVectorVTInfo VTInfo, Predicate prd> {
2461 let Predicates = [prd] in
2462 defm Z : avx512_icmp_cc_rmb<opc, Suffix, Frag, CommFrag, sched.ZMM,
2463 VTInfo.info512, NAME>, EVEX_V512;
2465 let Predicates = [prd, HasVLX] in {
2466 defm Z256 : avx512_icmp_cc_rmb<opc, Suffix, Frag, CommFrag, sched.YMM,
2467 VTInfo.info256, NAME>, EVEX_V256;
2468 defm Z128 : avx512_icmp_cc_rmb<opc, Suffix, Frag, CommFrag, sched.XMM,
2469 VTInfo.info128, NAME>, EVEX_V128;
2473 def X86pcmpm_imm : SDNodeXForm<setcc, [{
2474 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get();
2475 uint8_t SSECC = X86::getVPCMPImmForCond(CC);
2476 return getI8Imm(SSECC, SDLoc(N));
2479 // Swapped operand version of the above.
2480 def X86pcmpm_imm_commute : SDNodeXForm<setcc, [{
2481 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get();
2482 uint8_t SSECC = X86::getVPCMPImmForCond(CC);
2483 SSECC = X86::getSwappedVPCMPImm(SSECC);
2484 return getI8Imm(SSECC, SDLoc(N));
2487 def X86pcmpm : PatFrag<(ops node:$src1, node:$src2, node:$cc),
2488 (setcc node:$src1, node:$src2, node:$cc), [{
2489 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get();
2490 return !ISD::isUnsignedIntSetCC(CC);
2493 // Same as above, but commutes immediate. Use for load folding.
2494 def X86pcmpm_commute : PatFrag<(ops node:$src1, node:$src2, node:$cc),
2495 (setcc node:$src1, node:$src2, node:$cc), [{
2496 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get();
2497 return !ISD::isUnsignedIntSetCC(CC);
2498 }], X86pcmpm_imm_commute>;
2500 def X86pcmpum : PatFrag<(ops node:$src1, node:$src2, node:$cc),
2501 (setcc node:$src1, node:$src2, node:$cc), [{
2502 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get();
2503 return ISD::isUnsignedIntSetCC(CC);
2506 // Same as above, but commutes immediate. Use for load folding.
2507 def X86pcmpum_commute : PatFrag<(ops node:$src1, node:$src2, node:$cc),
2508 (setcc node:$src1, node:$src2, node:$cc), [{
2509 ISD::CondCode CC = cast<CondCodeSDNode>(N->getOperand(2))->get();
2510 return ISD::isUnsignedIntSetCC(CC);
2511 }], X86pcmpm_imm_commute>;
2513 // FIXME: Is there a better scheduler class for VPCMP/VPCMPU?
2514 defm VPCMPB : avx512_icmp_cc_vl<0x3F, "b", X86pcmpm, X86pcmpm_commute,
2515 SchedWriteVecALU, avx512vl_i8_info, HasBWI>,
2517 defm VPCMPUB : avx512_icmp_cc_vl<0x3E, "ub", X86pcmpum, X86pcmpum_commute,
2518 SchedWriteVecALU, avx512vl_i8_info, HasBWI>,
2521 defm VPCMPW : avx512_icmp_cc_vl<0x3F, "w", X86pcmpm, X86pcmpm_commute,
2522 SchedWriteVecALU, avx512vl_i16_info, HasBWI>,
2523 VEX_W, EVEX_CD8<16, CD8VF>;
2524 defm VPCMPUW : avx512_icmp_cc_vl<0x3E, "uw", X86pcmpum, X86pcmpum_commute,
2525 SchedWriteVecALU, avx512vl_i16_info, HasBWI>,
2526 VEX_W, EVEX_CD8<16, CD8VF>;
2528 defm VPCMPD : avx512_icmp_cc_rmb_vl<0x1F, "d", X86pcmpm, X86pcmpm_commute,
2529 SchedWriteVecALU, avx512vl_i32_info,
2530 HasAVX512>, EVEX_CD8<32, CD8VF>;
2531 defm VPCMPUD : avx512_icmp_cc_rmb_vl<0x1E, "ud", X86pcmpum, X86pcmpum_commute,
2532 SchedWriteVecALU, avx512vl_i32_info,
2533 HasAVX512>, EVEX_CD8<32, CD8VF>;
2535 defm VPCMPQ : avx512_icmp_cc_rmb_vl<0x1F, "q", X86pcmpm, X86pcmpm_commute,
2536 SchedWriteVecALU, avx512vl_i64_info,
2537 HasAVX512>, VEX_W, EVEX_CD8<64, CD8VF>;
2538 defm VPCMPUQ : avx512_icmp_cc_rmb_vl<0x1E, "uq", X86pcmpum, X86pcmpum_commute,
2539 SchedWriteVecALU, avx512vl_i64_info,
2540 HasAVX512>, VEX_W, EVEX_CD8<64, CD8VF>;
2542 multiclass avx512_vcmp_common<X86FoldableSchedWrite sched, X86VectorVTInfo _,
2544 defm rri : AVX512_maskable_cmp<0xC2, MRMSrcReg, _,
2545 (outs _.KRC:$dst), (ins _.RC:$src1, _.RC:$src2,AVXCC:$cc),
2546 "vcmp${cc}"#_.Suffix,
2547 "$src2, $src1", "$src1, $src2",
2548 (X86cmpm (_.VT _.RC:$src1),
2553 defm rmi : AVX512_maskable_cmp<0xC2, MRMSrcMem, _,
2554 (outs _.KRC:$dst),(ins _.RC:$src1, _.MemOp:$src2, AVXCC:$cc),
2555 "vcmp${cc}"#_.Suffix,
2556 "$src2, $src1", "$src1, $src2",
2557 (X86cmpm (_.VT _.RC:$src1),
2558 (_.VT (bitconvert (_.LdFrag addr:$src2))),
2560 Sched<[sched.Folded, ReadAfterLd]>;
2562 defm rmbi : AVX512_maskable_cmp<0xC2, MRMSrcMem, _,
2564 (ins _.RC:$src1, _.ScalarMemOp:$src2, AVXCC:$cc),
2565 "vcmp${cc}"#_.Suffix,
2566 "${src2}"##_.BroadcastStr##", $src1",
2567 "$src1, ${src2}"##_.BroadcastStr,
2568 (X86cmpm (_.VT _.RC:$src1),
2569 (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src2))),
2571 EVEX_B, Sched<[sched.Folded, ReadAfterLd]>;
2572 // Accept explicit immediate argument form instead of comparison code.
2573 let isAsmParserOnly = 1, hasSideEffects = 0 in {
2574 defm rri_alt : AVX512_maskable_cmp_alt<0xC2, MRMSrcReg, _,
2576 (ins _.RC:$src1, _.RC:$src2, u8imm:$cc),
2578 "$cc, $src2, $src1", "$src1, $src2, $cc">,
2579 Sched<[sched]>, NotMemoryFoldable;
2581 let mayLoad = 1 in {
2582 defm rmi_alt : AVX512_maskable_cmp_alt<0xC2, MRMSrcMem, _,
2584 (ins _.RC:$src1, _.MemOp:$src2, u8imm:$cc),
2586 "$cc, $src2, $src1", "$src1, $src2, $cc">,
2587 Sched<[sched.Folded, ReadAfterLd]>,
2590 defm rmbi_alt : AVX512_maskable_cmp_alt<0xC2, MRMSrcMem, _,
2592 (ins _.RC:$src1, _.ScalarMemOp:$src2, u8imm:$cc),
2594 "$cc, ${src2}"##_.BroadcastStr##", $src1",
2595 "$src1, ${src2}"##_.BroadcastStr##", $cc">,
2596 EVEX_B, Sched<[sched.Folded, ReadAfterLd]>,
2601 // Patterns for selecting with loads in other operand.
2602 def : Pat<(X86cmpm (_.LdFrag addr:$src2), (_.VT _.RC:$src1),
2603 CommutableCMPCC:$cc),
2604 (!cast<Instruction>(Name#_.ZSuffix#"rmi") _.RC:$src1, addr:$src2,
2607 def : Pat<(and _.KRCWM:$mask, (X86cmpm (_.LdFrag addr:$src2),
2609 CommutableCMPCC:$cc)),
2610 (!cast<Instruction>(Name#_.ZSuffix#"rmik") _.KRCWM:$mask,
2611 _.RC:$src1, addr:$src2,
2614 def : Pat<(X86cmpm (X86VBroadcast (_.ScalarLdFrag addr:$src2)),
2615 (_.VT _.RC:$src1), CommutableCMPCC:$cc),
2616 (!cast<Instruction>(Name#_.ZSuffix#"rmbi") _.RC:$src1, addr:$src2,
2619 def : Pat<(and _.KRCWM:$mask, (X86cmpm (X86VBroadcast
2620 (_.ScalarLdFrag addr:$src2)),
2622 CommutableCMPCC:$cc)),
2623 (!cast<Instruction>(Name#_.ZSuffix#"rmbik") _.KRCWM:$mask,
2624 _.RC:$src1, addr:$src2,
2628 multiclass avx512_vcmp_sae<X86FoldableSchedWrite sched, X86VectorVTInfo _> {
2629 // comparison code form (VCMP[EQ/LT/LE/...]
2630 defm rrib : AVX512_maskable_cmp<0xC2, MRMSrcReg, _,
2631 (outs _.KRC:$dst),(ins _.RC:$src1, _.RC:$src2, AVXCC:$cc),
2632 "vcmp${cc}"#_.Suffix,
2633 "{sae}, $src2, $src1", "$src1, $src2, {sae}",
2634 (X86cmpmRnd (_.VT _.RC:$src1),
2637 (i32 FROUND_NO_EXC))>,
2638 EVEX_B, Sched<[sched]>;
2640 let isAsmParserOnly = 1, hasSideEffects = 0 in {
2641 defm rrib_alt : AVX512_maskable_cmp_alt<0xC2, MRMSrcReg, _,
2643 (ins _.RC:$src1, _.RC:$src2, u8imm:$cc),
2645 "$cc, {sae}, $src2, $src1",
2646 "$src1, $src2, {sae}, $cc">,
2647 EVEX_B, Sched<[sched]>, NotMemoryFoldable;
2651 multiclass avx512_vcmp<X86SchedWriteWidths sched, AVX512VLVectorVTInfo _> {
2652 let Predicates = [HasAVX512] in {
2653 defm Z : avx512_vcmp_common<sched.ZMM, _.info512, NAME>,
2654 avx512_vcmp_sae<sched.ZMM, _.info512>, EVEX_V512;
2657 let Predicates = [HasAVX512,HasVLX] in {
2658 defm Z128 : avx512_vcmp_common<sched.XMM, _.info128, NAME>, EVEX_V128;
2659 defm Z256 : avx512_vcmp_common<sched.YMM, _.info256, NAME>, EVEX_V256;
2663 defm VCMPPD : avx512_vcmp<SchedWriteFCmp, avx512vl_f64_info>,
2664 AVX512PDIi8Base, EVEX_4V, EVEX_CD8<64, CD8VF>, VEX_W;
2665 defm VCMPPS : avx512_vcmp<SchedWriteFCmp, avx512vl_f32_info>,
2666 AVX512PSIi8Base, EVEX_4V, EVEX_CD8<32, CD8VF>;
2668 // Patterns to select fp compares with load as first operand.
2669 let Predicates = [HasAVX512] in {
2670 def : Pat<(v1i1 (X86cmpms (loadf64 addr:$src2), FR64X:$src1,
2671 CommutableCMPCC:$cc)),
2672 (VCMPSDZrm FR64X:$src1, addr:$src2, imm:$cc)>;
2674 def : Pat<(v1i1 (X86cmpms (loadf32 addr:$src2), FR32X:$src1,
2675 CommutableCMPCC:$cc)),
2676 (VCMPSSZrm FR32X:$src1, addr:$src2, imm:$cc)>;
2679 // ----------------------------------------------------------------
2681 //handle fpclass instruction mask = op(reg_scalar,imm)
2682 // op(mem_scalar,imm)
2683 multiclass avx512_scalar_fpclass<bits<8> opc, string OpcodeStr, SDNode OpNode,
2684 X86FoldableSchedWrite sched, X86VectorVTInfo _,
2686 let Predicates = [prd], ExeDomain = _.ExeDomain in {
2687 def rr : AVX512<opc, MRMSrcReg, (outs _.KRC:$dst),
2688 (ins _.RC:$src1, i32u8imm:$src2),
2689 OpcodeStr##_.Suffix#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
2690 [(set _.KRC:$dst,(OpNode (_.VT _.RC:$src1),
2691 (i32 imm:$src2)))]>,
2693 def rrk : AVX512<opc, MRMSrcReg, (outs _.KRC:$dst),
2694 (ins _.KRCWM:$mask, _.RC:$src1, i32u8imm:$src2),
2695 OpcodeStr##_.Suffix#
2696 "\t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}",
2697 [(set _.KRC:$dst,(and _.KRCWM:$mask,
2698 (OpNode (_.VT _.RC:$src1),
2699 (i32 imm:$src2))))]>,
2700 EVEX_K, Sched<[sched]>;
2701 def rm : AVX512<opc, MRMSrcMem, (outs _.KRC:$dst),
2702 (ins _.IntScalarMemOp:$src1, i32u8imm:$src2),
2703 OpcodeStr##_.Suffix##
2704 "\t{$src2, $src1, $dst|$dst, $src1, $src2}",
2706 (OpNode _.ScalarIntMemCPat:$src1,
2707 (i32 imm:$src2)))]>,
2708 Sched<[sched.Folded, ReadAfterLd]>;
2709 def rmk : AVX512<opc, MRMSrcMem, (outs _.KRC:$dst),
2710 (ins _.KRCWM:$mask, _.IntScalarMemOp:$src1, i32u8imm:$src2),
2711 OpcodeStr##_.Suffix##
2712 "\t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}",
2713 [(set _.KRC:$dst,(and _.KRCWM:$mask,
2714 (OpNode _.ScalarIntMemCPat:$src1,
2715 (i32 imm:$src2))))]>,
2716 EVEX_K, Sched<[sched.Folded, ReadAfterLd]>;
2720 //handle fpclass instruction mask = fpclass(reg_vec, reg_vec, imm)
2721 // fpclass(reg_vec, mem_vec, imm)
2722 // fpclass(reg_vec, broadcast(eltVt), imm)
2723 multiclass avx512_vector_fpclass<bits<8> opc, string OpcodeStr, SDNode OpNode,
2724 X86FoldableSchedWrite sched, X86VectorVTInfo _,
2725 string mem, string broadcast>{
2726 let ExeDomain = _.ExeDomain in {
2727 def rr : AVX512<opc, MRMSrcReg, (outs _.KRC:$dst),
2728 (ins _.RC:$src1, i32u8imm:$src2),
2729 OpcodeStr##_.Suffix#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
2730 [(set _.KRC:$dst,(OpNode (_.VT _.RC:$src1),
2731 (i32 imm:$src2)))]>,
2733 def rrk : AVX512<opc, MRMSrcReg, (outs _.KRC:$dst),
2734 (ins _.KRCWM:$mask, _.RC:$src1, i32u8imm:$src2),
2735 OpcodeStr##_.Suffix#
2736 "\t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}",
2737 [(set _.KRC:$dst,(and _.KRCWM:$mask,
2738 (OpNode (_.VT _.RC:$src1),
2739 (i32 imm:$src2))))]>,
2740 EVEX_K, Sched<[sched]>;
2741 def rm : AVX512<opc, MRMSrcMem, (outs _.KRC:$dst),
2742 (ins _.MemOp:$src1, i32u8imm:$src2),
2743 OpcodeStr##_.Suffix##mem#
2744 "\t{$src2, $src1, $dst|$dst, $src1, $src2}",
2745 [(set _.KRC:$dst,(OpNode
2746 (_.VT (bitconvert (_.LdFrag addr:$src1))),
2747 (i32 imm:$src2)))]>,
2748 Sched<[sched.Folded, ReadAfterLd]>;
2749 def rmk : AVX512<opc, MRMSrcMem, (outs _.KRC:$dst),
2750 (ins _.KRCWM:$mask, _.MemOp:$src1, i32u8imm:$src2),
2751 OpcodeStr##_.Suffix##mem#
2752 "\t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}",
2753 [(set _.KRC:$dst, (and _.KRCWM:$mask, (OpNode
2754 (_.VT (bitconvert (_.LdFrag addr:$src1))),
2755 (i32 imm:$src2))))]>,
2756 EVEX_K, Sched<[sched.Folded, ReadAfterLd]>;
2757 def rmb : AVX512<opc, MRMSrcMem, (outs _.KRC:$dst),
2758 (ins _.ScalarMemOp:$src1, i32u8imm:$src2),
2759 OpcodeStr##_.Suffix##broadcast##"\t{$src2, ${src1}"##
2760 _.BroadcastStr##", $dst|$dst, ${src1}"
2761 ##_.BroadcastStr##", $src2}",
2762 [(set _.KRC:$dst,(OpNode
2763 (_.VT (X86VBroadcast
2764 (_.ScalarLdFrag addr:$src1))),
2765 (i32 imm:$src2)))]>,
2766 EVEX_B, Sched<[sched.Folded, ReadAfterLd]>;
2767 def rmbk : AVX512<opc, MRMSrcMem, (outs _.KRC:$dst),
2768 (ins _.KRCWM:$mask, _.ScalarMemOp:$src1, i32u8imm:$src2),
2769 OpcodeStr##_.Suffix##broadcast##"\t{$src2, ${src1}"##
2770 _.BroadcastStr##", $dst {${mask}}|$dst {${mask}}, ${src1}"##
2771 _.BroadcastStr##", $src2}",
2772 [(set _.KRC:$dst,(and _.KRCWM:$mask, (OpNode
2773 (_.VT (X86VBroadcast
2774 (_.ScalarLdFrag addr:$src1))),
2775 (i32 imm:$src2))))]>,
2776 EVEX_B, EVEX_K, Sched<[sched.Folded, ReadAfterLd]>;
2780 multiclass avx512_vector_fpclass_all<string OpcodeStr, AVX512VLVectorVTInfo _,
2781 bits<8> opc, SDNode OpNode,
2782 X86SchedWriteWidths sched, Predicate prd,
2784 let Predicates = [prd] in {
2785 defm Z : avx512_vector_fpclass<opc, OpcodeStr, OpNode, sched.ZMM,
2786 _.info512, "{z}", broadcast>, EVEX_V512;
2788 let Predicates = [prd, HasVLX] in {
2789 defm Z128 : avx512_vector_fpclass<opc, OpcodeStr, OpNode, sched.XMM,
2790 _.info128, "{x}", broadcast>, EVEX_V128;
2791 defm Z256 : avx512_vector_fpclass<opc, OpcodeStr, OpNode, sched.YMM,
2792 _.info256, "{y}", broadcast>, EVEX_V256;
2796 multiclass avx512_fp_fpclass_all<string OpcodeStr, bits<8> opcVec,
2797 bits<8> opcScalar, SDNode VecOpNode,
2798 SDNode ScalarOpNode, X86SchedWriteWidths sched,
2800 defm PS : avx512_vector_fpclass_all<OpcodeStr, avx512vl_f32_info, opcVec,
2801 VecOpNode, sched, prd, "{l}">,
2802 EVEX_CD8<32, CD8VF>;
2803 defm PD : avx512_vector_fpclass_all<OpcodeStr, avx512vl_f64_info, opcVec,
2804 VecOpNode, sched, prd, "{q}">,
2805 EVEX_CD8<64, CD8VF> , VEX_W;
2806 defm SSZ : avx512_scalar_fpclass<opcScalar, OpcodeStr, ScalarOpNode,
2807 sched.Scl, f32x_info, prd>,
2808 EVEX_CD8<32, CD8VT1>;
2809 defm SDZ : avx512_scalar_fpclass<opcScalar, OpcodeStr, ScalarOpNode,
2810 sched.Scl, f64x_info, prd>,
2811 EVEX_CD8<64, CD8VT1>, VEX_W;
2814 defm VFPCLASS : avx512_fp_fpclass_all<"vfpclass", 0x66, 0x67, X86Vfpclass,
2815 X86Vfpclasss, SchedWriteFCmp, HasDQI>,
2816 AVX512AIi8Base, EVEX;
2818 //-----------------------------------------------------------------
2819 // Mask register copy, including
2820 // - copy between mask registers
2821 // - load/store mask registers
2822 // - copy from GPR to mask register and vice versa
2824 multiclass avx512_mask_mov<bits<8> opc_kk, bits<8> opc_km, bits<8> opc_mk,
2825 string OpcodeStr, RegisterClass KRC,
2826 ValueType vvt, X86MemOperand x86memop> {
2827 let isMoveReg = 1, hasSideEffects = 0, SchedRW = [WriteMove] in
2828 def kk : I<opc_kk, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src),
2829 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), []>,
2831 def km : I<opc_km, MRMSrcMem, (outs KRC:$dst), (ins x86memop:$src),
2832 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
2833 [(set KRC:$dst, (vvt (load addr:$src)))]>,
2835 def mk : I<opc_mk, MRMDestMem, (outs), (ins x86memop:$dst, KRC:$src),
2836 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
2837 [(store KRC:$src, addr:$dst)]>,
2838 Sched<[WriteStore]>;
2841 multiclass avx512_mask_mov_gpr<bits<8> opc_kr, bits<8> opc_rk,
2843 RegisterClass KRC, RegisterClass GRC> {
2844 let hasSideEffects = 0 in {
2845 def kr : I<opc_kr, MRMSrcReg, (outs KRC:$dst), (ins GRC:$src),
2846 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), []>,
2848 def rk : I<opc_rk, MRMSrcReg, (outs GRC:$dst), (ins KRC:$src),
2849 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), []>,
2854 let Predicates = [HasDQI] in
2855 defm KMOVB : avx512_mask_mov<0x90, 0x90, 0x91, "kmovb", VK8, v8i1, i8mem>,
2856 avx512_mask_mov_gpr<0x92, 0x93, "kmovb", VK8, GR32>,
2859 let Predicates = [HasAVX512] in
2860 defm KMOVW : avx512_mask_mov<0x90, 0x90, 0x91, "kmovw", VK16, v16i1, i16mem>,
2861 avx512_mask_mov_gpr<0x92, 0x93, "kmovw", VK16, GR32>,
2864 let Predicates = [HasBWI] in {
2865 defm KMOVD : avx512_mask_mov<0x90, 0x90, 0x91, "kmovd", VK32, v32i1,i32mem>,
2867 defm KMOVD : avx512_mask_mov_gpr<0x92, 0x93, "kmovd", VK32, GR32>,
2869 defm KMOVQ : avx512_mask_mov<0x90, 0x90, 0x91, "kmovq", VK64, v64i1, i64mem>,
2871 defm KMOVQ : avx512_mask_mov_gpr<0x92, 0x93, "kmovq", VK64, GR64>,
2875 // GR from/to mask register
2876 def : Pat<(v16i1 (bitconvert (i16 GR16:$src))),
2877 (COPY_TO_REGCLASS (i32 (INSERT_SUBREG (IMPLICIT_DEF), GR16:$src, sub_16bit)), VK16)>;
2878 def : Pat<(i16 (bitconvert (v16i1 VK16:$src))),
2879 (EXTRACT_SUBREG (i32 (COPY_TO_REGCLASS VK16:$src, GR32)), sub_16bit)>;
2881 def : Pat<(v8i1 (bitconvert (i8 GR8:$src))),
2882 (COPY_TO_REGCLASS (i32 (INSERT_SUBREG (IMPLICIT_DEF), GR8:$src, sub_8bit)), VK8)>;
2883 def : Pat<(i8 (bitconvert (v8i1 VK8:$src))),
2884 (EXTRACT_SUBREG (i32 (COPY_TO_REGCLASS VK8:$src, GR32)), sub_8bit)>;
2886 def : Pat<(i32 (zext (i16 (bitconvert (v16i1 VK16:$src))))),
2887 (KMOVWrk VK16:$src)>;
2888 def : Pat<(i32 (anyext (i16 (bitconvert (v16i1 VK16:$src))))),
2889 (COPY_TO_REGCLASS VK16:$src, GR32)>;
2891 def : Pat<(i32 (zext (i8 (bitconvert (v8i1 VK8:$src))))),
2892 (KMOVBrk VK8:$src)>, Requires<[HasDQI]>;
2893 def : Pat<(i32 (anyext (i8 (bitconvert (v8i1 VK8:$src))))),
2894 (COPY_TO_REGCLASS VK8:$src, GR32)>;
2896 def : Pat<(v32i1 (bitconvert (i32 GR32:$src))),
2897 (COPY_TO_REGCLASS GR32:$src, VK32)>;
2898 def : Pat<(i32 (bitconvert (v32i1 VK32:$src))),
2899 (COPY_TO_REGCLASS VK32:$src, GR32)>;
2900 def : Pat<(v64i1 (bitconvert (i64 GR64:$src))),
2901 (COPY_TO_REGCLASS GR64:$src, VK64)>;
2902 def : Pat<(i64 (bitconvert (v64i1 VK64:$src))),
2903 (COPY_TO_REGCLASS VK64:$src, GR64)>;
2906 let Predicates = [HasDQI] in {
2907 def : Pat<(store VK1:$src, addr:$dst),
2908 (KMOVBmk addr:$dst, (COPY_TO_REGCLASS VK1:$src, VK8))>;
2910 def : Pat<(v1i1 (load addr:$src)),
2911 (COPY_TO_REGCLASS (KMOVBkm addr:$src), VK1)>;
2912 def : Pat<(v2i1 (load addr:$src)),
2913 (COPY_TO_REGCLASS (KMOVBkm addr:$src), VK2)>;
2914 def : Pat<(v4i1 (load addr:$src)),
2915 (COPY_TO_REGCLASS (KMOVBkm addr:$src), VK4)>;
2918 let Predicates = [HasAVX512] in {
2919 def : Pat<(v8i1 (bitconvert (i8 (load addr:$src)))),
2920 (COPY_TO_REGCLASS (MOVZX32rm8 addr:$src), VK8)>;
2923 let Predicates = [HasAVX512] in {
2924 multiclass operation_gpr_mask_copy_lowering<RegisterClass maskRC, ValueType maskVT> {
2925 def : Pat<(maskVT (scalar_to_vector GR32:$src)),
2926 (COPY_TO_REGCLASS GR32:$src, maskRC)>;
2928 def : Pat<(maskVT (scalar_to_vector GR8:$src)),
2929 (COPY_TO_REGCLASS (INSERT_SUBREG (i32 (IMPLICIT_DEF)), GR8:$src, sub_8bit), maskRC)>;
2932 defm : operation_gpr_mask_copy_lowering<VK1, v1i1>;
2933 defm : operation_gpr_mask_copy_lowering<VK2, v2i1>;
2934 defm : operation_gpr_mask_copy_lowering<VK4, v4i1>;
2935 defm : operation_gpr_mask_copy_lowering<VK8, v8i1>;
2936 defm : operation_gpr_mask_copy_lowering<VK16, v16i1>;
2937 defm : operation_gpr_mask_copy_lowering<VK32, v32i1>;
2938 defm : operation_gpr_mask_copy_lowering<VK64, v64i1>;
2940 def : Pat<(insert_subvector (v16i1 immAllZerosV),
2941 (v1i1 (scalar_to_vector GR8:$src)), (iPTR 0)),
2944 (INSERT_SUBREG (i32 (IMPLICIT_DEF)), GR8:$src, sub_8bit),
2948 // Mask unary operation
2950 multiclass avx512_mask_unop<bits<8> opc, string OpcodeStr,
2951 RegisterClass KRC, SDPatternOperator OpNode,
2952 X86FoldableSchedWrite sched, Predicate prd> {
2953 let Predicates = [prd] in
2954 def rr : I<opc, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src),
2955 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
2956 [(set KRC:$dst, (OpNode KRC:$src))]>,
2960 multiclass avx512_mask_unop_all<bits<8> opc, string OpcodeStr,
2961 SDPatternOperator OpNode,
2962 X86FoldableSchedWrite sched> {
2963 defm B : avx512_mask_unop<opc, !strconcat(OpcodeStr, "b"), VK8, OpNode,
2964 sched, HasDQI>, VEX, PD;
2965 defm W : avx512_mask_unop<opc, !strconcat(OpcodeStr, "w"), VK16, OpNode,
2966 sched, HasAVX512>, VEX, PS;
2967 defm D : avx512_mask_unop<opc, !strconcat(OpcodeStr, "d"), VK32, OpNode,
2968 sched, HasBWI>, VEX, PD, VEX_W;
2969 defm Q : avx512_mask_unop<opc, !strconcat(OpcodeStr, "q"), VK64, OpNode,
2970 sched, HasBWI>, VEX, PS, VEX_W;
2973 // TODO - do we need a X86SchedWriteWidths::KMASK type?
2974 defm KNOT : avx512_mask_unop_all<0x44, "knot", vnot, SchedWriteVecLogic.XMM>;
2976 // KNL does not support KMOVB, 8-bit mask is promoted to 16-bit
2977 let Predicates = [HasAVX512, NoDQI] in
2978 def : Pat<(vnot VK8:$src),
2979 (COPY_TO_REGCLASS (KNOTWrr (COPY_TO_REGCLASS VK8:$src, VK16)), VK8)>;
2981 def : Pat<(vnot VK4:$src),
2982 (COPY_TO_REGCLASS (KNOTWrr (COPY_TO_REGCLASS VK4:$src, VK16)), VK4)>;
2983 def : Pat<(vnot VK2:$src),
2984 (COPY_TO_REGCLASS (KNOTWrr (COPY_TO_REGCLASS VK2:$src, VK16)), VK2)>;
2986 // Mask binary operation
2987 // - KAND, KANDN, KOR, KXNOR, KXOR
2988 multiclass avx512_mask_binop<bits<8> opc, string OpcodeStr,
2989 RegisterClass KRC, SDPatternOperator OpNode,
2990 X86FoldableSchedWrite sched, Predicate prd,
2992 let Predicates = [prd], isCommutable = IsCommutable in
2993 def rr : I<opc, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src1, KRC:$src2),
2994 !strconcat(OpcodeStr,
2995 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
2996 [(set KRC:$dst, (OpNode KRC:$src1, KRC:$src2))]>,
3000 multiclass avx512_mask_binop_all<bits<8> opc, string OpcodeStr,
3001 SDPatternOperator OpNode,
3002 X86FoldableSchedWrite sched, bit IsCommutable,
3003 Predicate prdW = HasAVX512> {
3004 defm B : avx512_mask_binop<opc, !strconcat(OpcodeStr, "b"), VK8, OpNode,
3005 sched, HasDQI, IsCommutable>, VEX_4V, VEX_L, PD;
3006 defm W : avx512_mask_binop<opc, !strconcat(OpcodeStr, "w"), VK16, OpNode,
3007 sched, prdW, IsCommutable>, VEX_4V, VEX_L, PS;
3008 defm D : avx512_mask_binop<opc, !strconcat(OpcodeStr, "d"), VK32, OpNode,
3009 sched, HasBWI, IsCommutable>, VEX_4V, VEX_L, VEX_W, PD;
3010 defm Q : avx512_mask_binop<opc, !strconcat(OpcodeStr, "q"), VK64, OpNode,
3011 sched, HasBWI, IsCommutable>, VEX_4V, VEX_L, VEX_W, PS;
3014 def andn : PatFrag<(ops node:$i0, node:$i1), (and (not node:$i0), node:$i1)>;
3015 def xnor : PatFrag<(ops node:$i0, node:$i1), (not (xor node:$i0, node:$i1))>;
3016 // These nodes use 'vnot' instead of 'not' to support vectors.
3017 def vandn : PatFrag<(ops node:$i0, node:$i1), (and (vnot node:$i0), node:$i1)>;
3018 def vxnor : PatFrag<(ops node:$i0, node:$i1), (vnot (xor node:$i0, node:$i1))>;
3020 // TODO - do we need a X86SchedWriteWidths::KMASK type?
3021 defm KAND : avx512_mask_binop_all<0x41, "kand", and, SchedWriteVecLogic.XMM, 1>;
3022 defm KOR : avx512_mask_binop_all<0x45, "kor", or, SchedWriteVecLogic.XMM, 1>;
3023 defm KXNOR : avx512_mask_binop_all<0x46, "kxnor", vxnor, SchedWriteVecLogic.XMM, 1>;
3024 defm KXOR : avx512_mask_binop_all<0x47, "kxor", xor, SchedWriteVecLogic.XMM, 1>;
3025 defm KANDN : avx512_mask_binop_all<0x42, "kandn", vandn, SchedWriteVecLogic.XMM, 0>;
3026 defm KADD : avx512_mask_binop_all<0x4A, "kadd", X86kadd, SchedWriteVecLogic.XMM, 1, HasDQI>;
3028 multiclass avx512_binop_pat<SDPatternOperator VOpNode, SDPatternOperator OpNode,
3030 // With AVX512F, 8-bit mask is promoted to 16-bit mask,
3031 // for the DQI set, this type is legal and KxxxB instruction is used
3032 let Predicates = [NoDQI] in
3033 def : Pat<(VOpNode VK8:$src1, VK8:$src2),
3035 (Inst (COPY_TO_REGCLASS VK8:$src1, VK16),
3036 (COPY_TO_REGCLASS VK8:$src2, VK16)), VK8)>;
3038 // All types smaller than 8 bits require conversion anyway
3039 def : Pat<(OpNode VK1:$src1, VK1:$src2),
3040 (COPY_TO_REGCLASS (Inst
3041 (COPY_TO_REGCLASS VK1:$src1, VK16),
3042 (COPY_TO_REGCLASS VK1:$src2, VK16)), VK1)>;
3043 def : Pat<(VOpNode VK2:$src1, VK2:$src2),
3044 (COPY_TO_REGCLASS (Inst
3045 (COPY_TO_REGCLASS VK2:$src1, VK16),
3046 (COPY_TO_REGCLASS VK2:$src2, VK16)), VK1)>;
3047 def : Pat<(VOpNode VK4:$src1, VK4:$src2),
3048 (COPY_TO_REGCLASS (Inst
3049 (COPY_TO_REGCLASS VK4:$src1, VK16),
3050 (COPY_TO_REGCLASS VK4:$src2, VK16)), VK1)>;
3053 defm : avx512_binop_pat<and, and, KANDWrr>;
3054 defm : avx512_binop_pat<vandn, andn, KANDNWrr>;
3055 defm : avx512_binop_pat<or, or, KORWrr>;
3056 defm : avx512_binop_pat<vxnor, xnor, KXNORWrr>;
3057 defm : avx512_binop_pat<xor, xor, KXORWrr>;
3060 multiclass avx512_mask_unpck<string Suffix,RegisterClass KRC, ValueType VT,
3061 RegisterClass KRCSrc, X86FoldableSchedWrite sched,
3063 let Predicates = [prd] in {
3064 let hasSideEffects = 0 in
3065 def rr : I<0x4b, MRMSrcReg, (outs KRC:$dst),
3066 (ins KRC:$src1, KRC:$src2),
3067 "kunpck"#Suffix#"\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>,
3068 VEX_4V, VEX_L, Sched<[sched]>;
3070 def : Pat<(VT (concat_vectors KRCSrc:$src1, KRCSrc:$src2)),
3071 (!cast<Instruction>(NAME##rr)
3072 (COPY_TO_REGCLASS KRCSrc:$src2, KRC),
3073 (COPY_TO_REGCLASS KRCSrc:$src1, KRC))>;
3077 defm KUNPCKBW : avx512_mask_unpck<"bw", VK16, v16i1, VK8, WriteShuffle, HasAVX512>, PD;
3078 defm KUNPCKWD : avx512_mask_unpck<"wd", VK32, v32i1, VK16, WriteShuffle, HasBWI>, PS;
3079 defm KUNPCKDQ : avx512_mask_unpck<"dq", VK64, v64i1, VK32, WriteShuffle, HasBWI>, PS, VEX_W;
3082 multiclass avx512_mask_testop<bits<8> opc, string OpcodeStr, RegisterClass KRC,
3083 SDNode OpNode, X86FoldableSchedWrite sched,
3085 let Predicates = [prd], Defs = [EFLAGS] in
3086 def rr : I<opc, MRMSrcReg, (outs), (ins KRC:$src1, KRC:$src2),
3087 !strconcat(OpcodeStr, "\t{$src2, $src1|$src1, $src2}"),
3088 [(set EFLAGS, (OpNode KRC:$src1, KRC:$src2))]>,
3092 multiclass avx512_mask_testop_w<bits<8> opc, string OpcodeStr, SDNode OpNode,
3093 X86FoldableSchedWrite sched,
3094 Predicate prdW = HasAVX512> {
3095 defm B : avx512_mask_testop<opc, OpcodeStr#"b", VK8, OpNode, sched, HasDQI>,
3097 defm W : avx512_mask_testop<opc, OpcodeStr#"w", VK16, OpNode, sched, prdW>,
3099 defm Q : avx512_mask_testop<opc, OpcodeStr#"q", VK64, OpNode, sched, HasBWI>,
3101 defm D : avx512_mask_testop<opc, OpcodeStr#"d", VK32, OpNode, sched, HasBWI>,
3105 // TODO - do we need a X86SchedWriteWidths::KMASK type?
3106 defm KORTEST : avx512_mask_testop_w<0x98, "kortest", X86kortest, SchedWriteVecLogic.XMM>;
3107 defm KTEST : avx512_mask_testop_w<0x99, "ktest", X86ktest, SchedWriteVecLogic.XMM, HasDQI>;
3110 multiclass avx512_mask_shiftop<bits<8> opc, string OpcodeStr, RegisterClass KRC,
3111 SDNode OpNode, X86FoldableSchedWrite sched> {
3112 let Predicates = [HasAVX512] in
3113 def ri : Ii8<opc, MRMSrcReg, (outs KRC:$dst), (ins KRC:$src, u8imm:$imm),
3114 !strconcat(OpcodeStr,
3115 "\t{$imm, $src, $dst|$dst, $src, $imm}"),
3116 [(set KRC:$dst, (OpNode KRC:$src, (i8 imm:$imm)))]>,
3120 multiclass avx512_mask_shiftop_w<bits<8> opc1, bits<8> opc2, string OpcodeStr,
3121 SDNode OpNode, X86FoldableSchedWrite sched> {
3122 defm W : avx512_mask_shiftop<opc1, !strconcat(OpcodeStr, "w"), VK16, OpNode,
3123 sched>, VEX, TAPD, VEX_W;
3124 let Predicates = [HasDQI] in
3125 defm B : avx512_mask_shiftop<opc1, !strconcat(OpcodeStr, "b"), VK8, OpNode,
3127 let Predicates = [HasBWI] in {
3128 defm Q : avx512_mask_shiftop<opc2, !strconcat(OpcodeStr, "q"), VK64, OpNode,
3129 sched>, VEX, TAPD, VEX_W;
3130 defm D : avx512_mask_shiftop<opc2, !strconcat(OpcodeStr, "d"), VK32, OpNode,
3135 defm KSHIFTL : avx512_mask_shiftop_w<0x32, 0x33, "kshiftl", X86kshiftl, WriteShuffle>;
3136 defm KSHIFTR : avx512_mask_shiftop_w<0x30, 0x31, "kshiftr", X86kshiftr, WriteShuffle>;
3138 // Patterns for comparing 128/256-bit integer vectors using 512-bit instruction.
3139 multiclass axv512_icmp_packed_no_vlx_lowering<PatFrag Frag, string InstStr,
3140 X86VectorVTInfo Narrow,
3141 X86VectorVTInfo Wide> {
3142 def : Pat<(Narrow.KVT (Frag (Narrow.VT Narrow.RC:$src1),
3143 (Narrow.VT Narrow.RC:$src2))),
3145 (!cast<Instruction>(InstStr#"Zrr")
3146 (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src1, Narrow.SubRegIdx)),
3147 (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src2, Narrow.SubRegIdx))),
3150 def : Pat<(Narrow.KVT (and Narrow.KRC:$mask,
3151 (Frag (Narrow.VT Narrow.RC:$src1),
3152 (Narrow.VT Narrow.RC:$src2)))),
3154 (!cast<Instruction>(InstStr#"Zrrk")
3155 (COPY_TO_REGCLASS Narrow.KRC:$mask, Wide.KRC),
3156 (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src1, Narrow.SubRegIdx)),
3157 (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src2, Narrow.SubRegIdx))),
3161 // Patterns for comparing 128/256-bit integer vectors using 512-bit instruction.
3162 multiclass axv512_icmp_packed_cc_no_vlx_lowering<PatFrag Frag,
3164 X86VectorVTInfo Narrow,
3165 X86VectorVTInfo Wide> {
3166 def : Pat<(Narrow.KVT (Frag:$cc (Narrow.VT Narrow.RC:$src1),
3167 (Narrow.VT Narrow.RC:$src2), cond)),
3169 (!cast<Instruction>(InstStr##Zrri)
3170 (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src1, Narrow.SubRegIdx)),
3171 (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src2, Narrow.SubRegIdx)),
3172 (Frag.OperandTransform $cc)), Narrow.KRC)>;
3174 def : Pat<(Narrow.KVT (and Narrow.KRC:$mask,
3175 (Narrow.KVT (Frag:$cc (Narrow.VT Narrow.RC:$src1),
3176 (Narrow.VT Narrow.RC:$src2),
3178 (COPY_TO_REGCLASS (!cast<Instruction>(InstStr##Zrrik)
3179 (COPY_TO_REGCLASS Narrow.KRC:$mask, Wide.KRC),
3180 (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src1, Narrow.SubRegIdx)),
3181 (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src2, Narrow.SubRegIdx)),
3182 (Frag.OperandTransform $cc)), Narrow.KRC)>;
3185 // Same as above, but for fp types which don't use PatFrags.
3186 multiclass axv512_cmp_packed_cc_no_vlx_lowering<SDNode OpNode, string InstStr,
3187 X86VectorVTInfo Narrow,
3188 X86VectorVTInfo Wide> {
3189 def : Pat<(Narrow.KVT (OpNode (Narrow.VT Narrow.RC:$src1),
3190 (Narrow.VT Narrow.RC:$src2), imm:$cc)),
3192 (!cast<Instruction>(InstStr##Zrri)
3193 (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src1, Narrow.SubRegIdx)),
3194 (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src2, Narrow.SubRegIdx)),
3195 imm:$cc), Narrow.KRC)>;
3197 def : Pat<(Narrow.KVT (and Narrow.KRC:$mask,
3198 (OpNode (Narrow.VT Narrow.RC:$src1),
3199 (Narrow.VT Narrow.RC:$src2), imm:$cc))),
3200 (COPY_TO_REGCLASS (!cast<Instruction>(InstStr##Zrrik)
3201 (COPY_TO_REGCLASS Narrow.KRC:$mask, Wide.KRC),
3202 (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src1, Narrow.SubRegIdx)),
3203 (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src2, Narrow.SubRegIdx)),
3204 imm:$cc), Narrow.KRC)>;
3207 let Predicates = [HasAVX512, NoVLX] in {
3208 // AddedComplexity is needed because the explicit SETEQ/SETGT CondCode doesn't
3209 // increase the pattern complexity the way an immediate would.
3210 let AddedComplexity = 2 in {
3211 defm : axv512_icmp_packed_no_vlx_lowering<X86pcmpgtm, "VPCMPGTD", v8i32x_info, v16i32_info>;
3212 defm : axv512_icmp_packed_no_vlx_lowering<X86pcmpeqm_c, "VPCMPEQD", v8i32x_info, v16i32_info>;
3214 defm : axv512_icmp_packed_no_vlx_lowering<X86pcmpgtm, "VPCMPGTD", v4i32x_info, v16i32_info>;
3215 defm : axv512_icmp_packed_no_vlx_lowering<X86pcmpeqm_c, "VPCMPEQD", v4i32x_info, v16i32_info>;
3217 defm : axv512_icmp_packed_no_vlx_lowering<X86pcmpgtm, "VPCMPGTQ", v4i64x_info, v8i64_info>;
3218 defm : axv512_icmp_packed_no_vlx_lowering<X86pcmpeqm_c, "VPCMPEQQ", v4i64x_info, v8i64_info>;
3220 defm : axv512_icmp_packed_no_vlx_lowering<X86pcmpgtm, "VPCMPGTQ", v2i64x_info, v8i64_info>;
3221 defm : axv512_icmp_packed_no_vlx_lowering<X86pcmpeqm_c, "VPCMPEQQ", v2i64x_info, v8i64_info>;
3224 defm : axv512_icmp_packed_cc_no_vlx_lowering<X86pcmpm, "VPCMPD", v8i32x_info, v16i32_info>;
3225 defm : axv512_icmp_packed_cc_no_vlx_lowering<X86pcmpum, "VPCMPUD", v8i32x_info, v16i32_info>;
3227 defm : axv512_icmp_packed_cc_no_vlx_lowering<X86pcmpm, "VPCMPD", v4i32x_info, v16i32_info>;
3228 defm : axv512_icmp_packed_cc_no_vlx_lowering<X86pcmpum, "VPCMPUD", v4i32x_info, v16i32_info>;
3230 defm : axv512_icmp_packed_cc_no_vlx_lowering<X86pcmpm, "VPCMPQ", v4i64x_info, v8i64_info>;
3231 defm : axv512_icmp_packed_cc_no_vlx_lowering<X86pcmpum, "VPCMPUQ", v4i64x_info, v8i64_info>;
3233 defm : axv512_icmp_packed_cc_no_vlx_lowering<X86pcmpm, "VPCMPQ", v2i64x_info, v8i64_info>;
3234 defm : axv512_icmp_packed_cc_no_vlx_lowering<X86pcmpum, "VPCMPUQ", v2i64x_info, v8i64_info>;
3236 defm : axv512_cmp_packed_cc_no_vlx_lowering<X86cmpm, "VCMPPS", v8f32x_info, v16f32_info>;
3237 defm : axv512_cmp_packed_cc_no_vlx_lowering<X86cmpm, "VCMPPS", v4f32x_info, v16f32_info>;
3238 defm : axv512_cmp_packed_cc_no_vlx_lowering<X86cmpm, "VCMPPD", v4f64x_info, v8f64_info>;
3239 defm : axv512_cmp_packed_cc_no_vlx_lowering<X86cmpm, "VCMPPD", v2f64x_info, v8f64_info>;
3242 let Predicates = [HasBWI, NoVLX] in {
3243 // AddedComplexity is needed because the explicit SETEQ/SETGT CondCode doesn't
3244 // increase the pattern complexity the way an immediate would.
3245 let AddedComplexity = 2 in {
3246 defm : axv512_icmp_packed_no_vlx_lowering<X86pcmpgtm, "VPCMPGTB", v32i8x_info, v64i8_info>;
3247 defm : axv512_icmp_packed_no_vlx_lowering<X86pcmpeqm_c, "VPCMPEQB", v32i8x_info, v64i8_info>;
3249 defm : axv512_icmp_packed_no_vlx_lowering<X86pcmpgtm, "VPCMPGTB", v16i8x_info, v64i8_info>;
3250 defm : axv512_icmp_packed_no_vlx_lowering<X86pcmpeqm_c, "VPCMPEQB", v16i8x_info, v64i8_info>;
3252 defm : axv512_icmp_packed_no_vlx_lowering<X86pcmpgtm, "VPCMPGTW", v16i16x_info, v32i16_info>;
3253 defm : axv512_icmp_packed_no_vlx_lowering<X86pcmpeqm_c, "VPCMPEQW", v16i16x_info, v32i16_info>;
3255 defm : axv512_icmp_packed_no_vlx_lowering<X86pcmpgtm, "VPCMPGTW", v8i16x_info, v32i16_info>;
3256 defm : axv512_icmp_packed_no_vlx_lowering<X86pcmpeqm_c, "VPCMPEQW", v8i16x_info, v32i16_info>;
3259 defm : axv512_icmp_packed_cc_no_vlx_lowering<X86pcmpm, "VPCMPB", v32i8x_info, v64i8_info>;
3260 defm : axv512_icmp_packed_cc_no_vlx_lowering<X86pcmpum, "VPCMPUB", v32i8x_info, v64i8_info>;
3262 defm : axv512_icmp_packed_cc_no_vlx_lowering<X86pcmpm, "VPCMPB", v16i8x_info, v64i8_info>;
3263 defm : axv512_icmp_packed_cc_no_vlx_lowering<X86pcmpum, "VPCMPUB", v16i8x_info, v64i8_info>;
3265 defm : axv512_icmp_packed_cc_no_vlx_lowering<X86pcmpm, "VPCMPW", v16i16x_info, v32i16_info>;
3266 defm : axv512_icmp_packed_cc_no_vlx_lowering<X86pcmpum, "VPCMPUW", v16i16x_info, v32i16_info>;
3268 defm : axv512_icmp_packed_cc_no_vlx_lowering<X86pcmpm, "VPCMPW", v8i16x_info, v32i16_info>;
3269 defm : axv512_icmp_packed_cc_no_vlx_lowering<X86pcmpum, "VPCMPUW", v8i16x_info, v32i16_info>;
3272 // Mask setting all 0s or 1s
3273 multiclass avx512_mask_setop<RegisterClass KRC, ValueType VT, PatFrag Val> {
3274 let Predicates = [HasAVX512] in
3275 let isReMaterializable = 1, isAsCheapAsAMove = 1, isPseudo = 1,
3276 SchedRW = [WriteZero] in
3277 def #NAME# : I<0, Pseudo, (outs KRC:$dst), (ins), "",
3278 [(set KRC:$dst, (VT Val))]>;
3281 multiclass avx512_mask_setop_w<PatFrag Val> {
3282 defm W : avx512_mask_setop<VK16, v16i1, Val>;
3283 defm D : avx512_mask_setop<VK32, v32i1, Val>;
3284 defm Q : avx512_mask_setop<VK64, v64i1, Val>;
3287 defm KSET0 : avx512_mask_setop_w<immAllZerosV>;
3288 defm KSET1 : avx512_mask_setop_w<immAllOnesV>;
3290 // With AVX-512 only, 8-bit mask is promoted to 16-bit mask.
3291 let Predicates = [HasAVX512] in {
3292 def : Pat<(v8i1 immAllZerosV), (COPY_TO_REGCLASS (KSET0W), VK8)>;
3293 def : Pat<(v4i1 immAllZerosV), (COPY_TO_REGCLASS (KSET0W), VK4)>;
3294 def : Pat<(v2i1 immAllZerosV), (COPY_TO_REGCLASS (KSET0W), VK2)>;
3295 def : Pat<(v1i1 immAllZerosV), (COPY_TO_REGCLASS (KSET0W), VK1)>;
3296 def : Pat<(v8i1 immAllOnesV), (COPY_TO_REGCLASS (KSET1W), VK8)>;
3297 def : Pat<(v4i1 immAllOnesV), (COPY_TO_REGCLASS (KSET1W), VK4)>;
3298 def : Pat<(v2i1 immAllOnesV), (COPY_TO_REGCLASS (KSET1W), VK2)>;
3299 def : Pat<(v1i1 immAllOnesV), (COPY_TO_REGCLASS (KSET1W), VK1)>;
3302 // Patterns for kmask insert_subvector/extract_subvector to/from index=0
3303 multiclass operation_subvector_mask_lowering<RegisterClass subRC, ValueType subVT,
3304 RegisterClass RC, ValueType VT> {
3305 def : Pat<(subVT (extract_subvector (VT RC:$src), (iPTR 0))),
3306 (subVT (COPY_TO_REGCLASS RC:$src, subRC))>;
3308 def : Pat<(VT (insert_subvector undef, subRC:$src, (iPTR 0))),
3309 (VT (COPY_TO_REGCLASS subRC:$src, RC))>;
3311 defm : operation_subvector_mask_lowering<VK1, v1i1, VK2, v2i1>;
3312 defm : operation_subvector_mask_lowering<VK1, v1i1, VK4, v4i1>;
3313 defm : operation_subvector_mask_lowering<VK1, v1i1, VK8, v8i1>;
3314 defm : operation_subvector_mask_lowering<VK1, v1i1, VK16, v16i1>;
3315 defm : operation_subvector_mask_lowering<VK1, v1i1, VK32, v32i1>;
3316 defm : operation_subvector_mask_lowering<VK1, v1i1, VK64, v64i1>;
3318 defm : operation_subvector_mask_lowering<VK2, v2i1, VK4, v4i1>;
3319 defm : operation_subvector_mask_lowering<VK2, v2i1, VK8, v8i1>;
3320 defm : operation_subvector_mask_lowering<VK2, v2i1, VK16, v16i1>;
3321 defm : operation_subvector_mask_lowering<VK2, v2i1, VK32, v32i1>;
3322 defm : operation_subvector_mask_lowering<VK2, v2i1, VK64, v64i1>;
3324 defm : operation_subvector_mask_lowering<VK4, v4i1, VK8, v8i1>;
3325 defm : operation_subvector_mask_lowering<VK4, v4i1, VK16, v16i1>;
3326 defm : operation_subvector_mask_lowering<VK4, v4i1, VK32, v32i1>;
3327 defm : operation_subvector_mask_lowering<VK4, v4i1, VK64, v64i1>;
3329 defm : operation_subvector_mask_lowering<VK8, v8i1, VK16, v16i1>;
3330 defm : operation_subvector_mask_lowering<VK8, v8i1, VK32, v32i1>;
3331 defm : operation_subvector_mask_lowering<VK8, v8i1, VK64, v64i1>;
3333 defm : operation_subvector_mask_lowering<VK16, v16i1, VK32, v32i1>;
3334 defm : operation_subvector_mask_lowering<VK16, v16i1, VK64, v64i1>;
3336 defm : operation_subvector_mask_lowering<VK32, v32i1, VK64, v64i1>;
3338 //===----------------------------------------------------------------------===//
3339 // AVX-512 - Aligned and unaligned load and store
3342 multiclass avx512_load<bits<8> opc, string OpcodeStr, string Name,
3343 X86VectorVTInfo _, PatFrag ld_frag, PatFrag mload,
3344 X86SchedWriteMoveLS Sched, string EVEX2VEXOvrd,
3345 bit NoRMPattern = 0,
3346 SDPatternOperator SelectOprr = vselect> {
3347 let hasSideEffects = 0 in {
3348 let isMoveReg = 1 in
3349 def rr : AVX512PI<opc, MRMSrcReg, (outs _.RC:$dst), (ins _.RC:$src),
3350 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"), [],
3351 _.ExeDomain>, EVEX, Sched<[Sched.RR]>,
3352 EVEX2VEXOverride<EVEX2VEXOvrd#"rr">;
3353 def rrkz : AVX512PI<opc, MRMSrcReg, (outs _.RC:$dst),
3354 (ins _.KRCWM:$mask, _.RC:$src),
3355 !strconcat(OpcodeStr, "\t{$src, ${dst} {${mask}} {z}|",
3356 "${dst} {${mask}} {z}, $src}"),
3357 [(set _.RC:$dst, (_.VT (SelectOprr _.KRCWM:$mask,
3359 _.ImmAllZerosV)))], _.ExeDomain>,
3360 EVEX, EVEX_KZ, Sched<[Sched.RR]>;
3362 let mayLoad = 1, canFoldAsLoad = 1, isReMaterializable = 1 in
3363 def rm : AVX512PI<opc, MRMSrcMem, (outs _.RC:$dst), (ins _.MemOp:$src),
3364 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
3365 !if(NoRMPattern, [],
3367 (_.VT (bitconvert (ld_frag addr:$src))))]),
3368 _.ExeDomain>, EVEX, Sched<[Sched.RM]>,
3369 EVEX2VEXOverride<EVEX2VEXOvrd#"rm">;
3371 let Constraints = "$src0 = $dst", isConvertibleToThreeAddress = 1 in {
3372 def rrk : AVX512PI<opc, MRMSrcReg, (outs _.RC:$dst),
3373 (ins _.RC:$src0, _.KRCWM:$mask, _.RC:$src1),
3374 !strconcat(OpcodeStr, "\t{$src1, ${dst} {${mask}}|",
3375 "${dst} {${mask}}, $src1}"),
3376 [(set _.RC:$dst, (_.VT (SelectOprr _.KRCWM:$mask,
3378 (_.VT _.RC:$src0))))], _.ExeDomain>,
3379 EVEX, EVEX_K, Sched<[Sched.RR]>;
3380 def rmk : AVX512PI<opc, MRMSrcMem, (outs _.RC:$dst),
3381 (ins _.RC:$src0, _.KRCWM:$mask, _.MemOp:$src1),
3382 !strconcat(OpcodeStr, "\t{$src1, ${dst} {${mask}}|",
3383 "${dst} {${mask}}, $src1}"),
3384 [(set _.RC:$dst, (_.VT
3385 (vselect _.KRCWM:$mask,
3386 (_.VT (bitconvert (ld_frag addr:$src1))),
3387 (_.VT _.RC:$src0))))], _.ExeDomain>,
3388 EVEX, EVEX_K, Sched<[Sched.RM]>;
3390 def rmkz : AVX512PI<opc, MRMSrcMem, (outs _.RC:$dst),
3391 (ins _.KRCWM:$mask, _.MemOp:$src),
3392 OpcodeStr #"\t{$src, ${dst} {${mask}} {z}|"#
3393 "${dst} {${mask}} {z}, $src}",
3394 [(set _.RC:$dst, (_.VT (vselect _.KRCWM:$mask,
3395 (_.VT (bitconvert (ld_frag addr:$src))), _.ImmAllZerosV)))],
3396 _.ExeDomain>, EVEX, EVEX_KZ, Sched<[Sched.RM]>;
3398 def : Pat<(_.VT (mload addr:$ptr, _.KRCWM:$mask, undef)),
3399 (!cast<Instruction>(Name#_.ZSuffix##rmkz) _.KRCWM:$mask, addr:$ptr)>;
3401 def : Pat<(_.VT (mload addr:$ptr, _.KRCWM:$mask, _.ImmAllZerosV)),
3402 (!cast<Instruction>(Name#_.ZSuffix##rmkz) _.KRCWM:$mask, addr:$ptr)>;
3404 def : Pat<(_.VT (mload addr:$ptr, _.KRCWM:$mask, (_.VT _.RC:$src0))),
3405 (!cast<Instruction>(Name#_.ZSuffix##rmk) _.RC:$src0,
3406 _.KRCWM:$mask, addr:$ptr)>;
3409 multiclass avx512_alignedload_vl<bits<8> opc, string OpcodeStr,
3410 AVX512VLVectorVTInfo _, Predicate prd,
3411 X86SchedWriteMoveLSWidths Sched,
3412 string EVEX2VEXOvrd, bit NoRMPattern = 0> {
3413 let Predicates = [prd] in
3414 defm Z : avx512_load<opc, OpcodeStr, NAME, _.info512,
3415 _.info512.AlignedLdFrag, masked_load_aligned512,
3416 Sched.ZMM, "", NoRMPattern>, EVEX_V512;
3418 let Predicates = [prd, HasVLX] in {
3419 defm Z256 : avx512_load<opc, OpcodeStr, NAME, _.info256,
3420 _.info256.AlignedLdFrag, masked_load_aligned256,
3421 Sched.YMM, EVEX2VEXOvrd#"Y", NoRMPattern>, EVEX_V256;
3422 defm Z128 : avx512_load<opc, OpcodeStr, NAME, _.info128,
3423 _.info128.AlignedLdFrag, masked_load_aligned128,
3424 Sched.XMM, EVEX2VEXOvrd, NoRMPattern>, EVEX_V128;
3428 multiclass avx512_load_vl<bits<8> opc, string OpcodeStr,
3429 AVX512VLVectorVTInfo _, Predicate prd,
3430 X86SchedWriteMoveLSWidths Sched,
3431 string EVEX2VEXOvrd, bit NoRMPattern = 0,
3432 SDPatternOperator SelectOprr = vselect> {
3433 let Predicates = [prd] in
3434 defm Z : avx512_load<opc, OpcodeStr, NAME, _.info512, _.info512.LdFrag,
3435 masked_load_unaligned, Sched.ZMM, "",
3436 NoRMPattern, SelectOprr>, EVEX_V512;
3438 let Predicates = [prd, HasVLX] in {
3439 defm Z256 : avx512_load<opc, OpcodeStr, NAME, _.info256, _.info256.LdFrag,
3440 masked_load_unaligned, Sched.YMM, EVEX2VEXOvrd#"Y",
3441 NoRMPattern, SelectOprr>, EVEX_V256;
3442 defm Z128 : avx512_load<opc, OpcodeStr, NAME, _.info128, _.info128.LdFrag,
3443 masked_load_unaligned, Sched.XMM, EVEX2VEXOvrd,
3444 NoRMPattern, SelectOprr>, EVEX_V128;
3448 multiclass avx512_store<bits<8> opc, string OpcodeStr, string BaseName,
3449 X86VectorVTInfo _, PatFrag st_frag, PatFrag mstore,
3450 X86SchedWriteMoveLS Sched, string EVEX2VEXOvrd,
3451 bit NoMRPattern = 0> {
3452 let hasSideEffects = 0, isCodeGenOnly = 1, ForceDisassemble = 1 in {
3453 let isMoveReg = 1 in
3454 def rr_REV : AVX512PI<opc, MRMDestReg, (outs _.RC:$dst), (ins _.RC:$src),
3455 OpcodeStr # "\t{$src, $dst|$dst, $src}",
3456 [], _.ExeDomain>, EVEX,
3457 FoldGenData<BaseName#_.ZSuffix#rr>, Sched<[Sched.RR]>,
3458 EVEX2VEXOverride<EVEX2VEXOvrd#"rr_REV">;
3459 def rrk_REV : AVX512PI<opc, MRMDestReg, (outs _.RC:$dst),
3460 (ins _.KRCWM:$mask, _.RC:$src),
3461 OpcodeStr # "\t{$src, ${dst} {${mask}}|"#
3462 "${dst} {${mask}}, $src}",
3463 [], _.ExeDomain>, EVEX, EVEX_K,
3464 FoldGenData<BaseName#_.ZSuffix#rrk>,
3466 def rrkz_REV : AVX512PI<opc, MRMDestReg, (outs _.RC:$dst),
3467 (ins _.KRCWM:$mask, _.RC:$src),
3468 OpcodeStr # "\t{$src, ${dst} {${mask}} {z}|" #
3469 "${dst} {${mask}} {z}, $src}",
3470 [], _.ExeDomain>, EVEX, EVEX_KZ,
3471 FoldGenData<BaseName#_.ZSuffix#rrkz>,
3475 let hasSideEffects = 0, mayStore = 1 in
3476 def mr : AVX512PI<opc, MRMDestMem, (outs), (ins _.MemOp:$dst, _.RC:$src),
3477 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
3478 !if(NoMRPattern, [],
3479 [(st_frag (_.VT _.RC:$src), addr:$dst)]),
3480 _.ExeDomain>, EVEX, Sched<[Sched.MR]>,
3481 EVEX2VEXOverride<EVEX2VEXOvrd#"mr">;
3482 def mrk : AVX512PI<opc, MRMDestMem, (outs),
3483 (ins _.MemOp:$dst, _.KRCWM:$mask, _.RC:$src),
3484 OpcodeStr # "\t{$src, ${dst} {${mask}}|${dst} {${mask}}, $src}",
3485 [], _.ExeDomain>, EVEX, EVEX_K, Sched<[Sched.MR]>,
3488 def: Pat<(mstore addr:$ptr, _.KRCWM:$mask, (_.VT _.RC:$src)),
3489 (!cast<Instruction>(BaseName#_.ZSuffix#mrk) addr:$ptr,
3490 _.KRCWM:$mask, _.RC:$src)>;
3492 def : InstAlias<OpcodeStr#".s\t{$src, $dst|$dst, $src}",
3493 (!cast<Instruction>(BaseName#_.ZSuffix#"rr_REV")
3494 _.RC:$dst, _.RC:$src), 0>;
3495 def : InstAlias<OpcodeStr#".s\t{$src, ${dst} {${mask}}|${dst} {${mask}}, $src}",
3496 (!cast<Instruction>(BaseName#_.ZSuffix#"rrk_REV")
3497 _.RC:$dst, _.KRCWM:$mask, _.RC:$src), 0>;
3498 def : InstAlias<OpcodeStr#".s\t{$src, ${dst} {${mask}} {z}|${dst} {${mask}} {z}, $src}",
3499 (!cast<Instruction>(BaseName#_.ZSuffix#"rrkz_REV")
3500 _.RC:$dst, _.KRCWM:$mask, _.RC:$src), 0>;
3503 multiclass avx512_store_vl< bits<8> opc, string OpcodeStr,
3504 AVX512VLVectorVTInfo _, Predicate prd,
3505 X86SchedWriteMoveLSWidths Sched,
3506 string EVEX2VEXOvrd, bit NoMRPattern = 0> {
3507 let Predicates = [prd] in
3508 defm Z : avx512_store<opc, OpcodeStr, NAME, _.info512, store,
3509 masked_store_unaligned, Sched.ZMM, "",
3510 NoMRPattern>, EVEX_V512;
3511 let Predicates = [prd, HasVLX] in {
3512 defm Z256 : avx512_store<opc, OpcodeStr, NAME, _.info256, store,
3513 masked_store_unaligned, Sched.YMM,
3514 EVEX2VEXOvrd#"Y", NoMRPattern>, EVEX_V256;
3515 defm Z128 : avx512_store<opc, OpcodeStr, NAME, _.info128, store,
3516 masked_store_unaligned, Sched.XMM, EVEX2VEXOvrd,
3517 NoMRPattern>, EVEX_V128;
3521 multiclass avx512_alignedstore_vl<bits<8> opc, string OpcodeStr,
3522 AVX512VLVectorVTInfo _, Predicate prd,
3523 X86SchedWriteMoveLSWidths Sched,
3524 string EVEX2VEXOvrd, bit NoMRPattern = 0> {
3525 let Predicates = [prd] in
3526 defm Z : avx512_store<opc, OpcodeStr, NAME, _.info512, alignedstore,
3527 masked_store_aligned512, Sched.ZMM, "",
3528 NoMRPattern>, EVEX_V512;
3530 let Predicates = [prd, HasVLX] in {
3531 defm Z256 : avx512_store<opc, OpcodeStr, NAME, _.info256, alignedstore,
3532 masked_store_aligned256, Sched.YMM,
3533 EVEX2VEXOvrd#"Y", NoMRPattern>, EVEX_V256;
3534 defm Z128 : avx512_store<opc, OpcodeStr, NAME, _.info128, alignedstore,
3535 masked_store_aligned128, Sched.XMM, EVEX2VEXOvrd,
3536 NoMRPattern>, EVEX_V128;
3540 defm VMOVAPS : avx512_alignedload_vl<0x28, "vmovaps", avx512vl_f32_info,
3541 HasAVX512, SchedWriteFMoveLS, "VMOVAPS">,
3542 avx512_alignedstore_vl<0x29, "vmovaps", avx512vl_f32_info,
3543 HasAVX512, SchedWriteFMoveLS, "VMOVAPS">,
3544 PS, EVEX_CD8<32, CD8VF>;
3546 defm VMOVAPD : avx512_alignedload_vl<0x28, "vmovapd", avx512vl_f64_info,
3547 HasAVX512, SchedWriteFMoveLS, "VMOVAPD">,
3548 avx512_alignedstore_vl<0x29, "vmovapd", avx512vl_f64_info,
3549 HasAVX512, SchedWriteFMoveLS, "VMOVAPD">,
3550 PD, VEX_W, EVEX_CD8<64, CD8VF>;
3552 defm VMOVUPS : avx512_load_vl<0x10, "vmovups", avx512vl_f32_info, HasAVX512,
3553 SchedWriteFMoveLS, "VMOVUPS", 0, null_frag>,
3554 avx512_store_vl<0x11, "vmovups", avx512vl_f32_info, HasAVX512,
3555 SchedWriteFMoveLS, "VMOVUPS">,
3556 PS, EVEX_CD8<32, CD8VF>;
3558 defm VMOVUPD : avx512_load_vl<0x10, "vmovupd", avx512vl_f64_info, HasAVX512,
3559 SchedWriteFMoveLS, "VMOVUPD", 0, null_frag>,
3560 avx512_store_vl<0x11, "vmovupd", avx512vl_f64_info, HasAVX512,
3561 SchedWriteFMoveLS, "VMOVUPD">,
3562 PD, VEX_W, EVEX_CD8<64, CD8VF>;
3564 defm VMOVDQA32 : avx512_alignedload_vl<0x6F, "vmovdqa32", avx512vl_i32_info,
3565 HasAVX512, SchedWriteVecMoveLS,
3567 avx512_alignedstore_vl<0x7F, "vmovdqa32", avx512vl_i32_info,
3568 HasAVX512, SchedWriteVecMoveLS,
3570 PD, EVEX_CD8<32, CD8VF>;
3572 defm VMOVDQA64 : avx512_alignedload_vl<0x6F, "vmovdqa64", avx512vl_i64_info,
3573 HasAVX512, SchedWriteVecMoveLS,
3575 avx512_alignedstore_vl<0x7F, "vmovdqa64", avx512vl_i64_info,
3576 HasAVX512, SchedWriteVecMoveLS,
3578 PD, VEX_W, EVEX_CD8<64, CD8VF>;
3580 defm VMOVDQU8 : avx512_load_vl<0x6F, "vmovdqu8", avx512vl_i8_info, HasBWI,
3581 SchedWriteVecMoveLS, "VMOVDQU", 1>,
3582 avx512_store_vl<0x7F, "vmovdqu8", avx512vl_i8_info, HasBWI,
3583 SchedWriteVecMoveLS, "VMOVDQU", 1>,
3584 XD, EVEX_CD8<8, CD8VF>;
3586 defm VMOVDQU16 : avx512_load_vl<0x6F, "vmovdqu16", avx512vl_i16_info, HasBWI,
3587 SchedWriteVecMoveLS, "VMOVDQU", 1>,
3588 avx512_store_vl<0x7F, "vmovdqu16", avx512vl_i16_info, HasBWI,
3589 SchedWriteVecMoveLS, "VMOVDQU", 1>,
3590 XD, VEX_W, EVEX_CD8<16, CD8VF>;
3592 defm VMOVDQU32 : avx512_load_vl<0x6F, "vmovdqu32", avx512vl_i32_info, HasAVX512,
3593 SchedWriteVecMoveLS, "VMOVDQU", 1, null_frag>,
3594 avx512_store_vl<0x7F, "vmovdqu32", avx512vl_i32_info, HasAVX512,
3595 SchedWriteVecMoveLS, "VMOVDQU", 1>,
3596 XS, EVEX_CD8<32, CD8VF>;
3598 defm VMOVDQU64 : avx512_load_vl<0x6F, "vmovdqu64", avx512vl_i64_info, HasAVX512,
3599 SchedWriteVecMoveLS, "VMOVDQU", 0, null_frag>,
3600 avx512_store_vl<0x7F, "vmovdqu64", avx512vl_i64_info, HasAVX512,
3601 SchedWriteVecMoveLS, "VMOVDQU">,
3602 XS, VEX_W, EVEX_CD8<64, CD8VF>;
3604 // Special instructions to help with spilling when we don't have VLX. We need
3605 // to load or store from a ZMM register instead. These are converted in
3606 // expandPostRAPseudos.
3607 let isReMaterializable = 1, canFoldAsLoad = 1,
3608 isPseudo = 1, mayLoad = 1, hasSideEffects = 0 in {
3609 def VMOVAPSZ128rm_NOVLX : I<0, Pseudo, (outs VR128X:$dst), (ins f128mem:$src),
3610 "", []>, Sched<[WriteFLoadX]>;
3611 def VMOVAPSZ256rm_NOVLX : I<0, Pseudo, (outs VR256X:$dst), (ins f256mem:$src),
3612 "", []>, Sched<[WriteFLoadY]>;
3613 def VMOVUPSZ128rm_NOVLX : I<0, Pseudo, (outs VR128X:$dst), (ins f128mem:$src),
3614 "", []>, Sched<[WriteFLoadX]>;
3615 def VMOVUPSZ256rm_NOVLX : I<0, Pseudo, (outs VR256X:$dst), (ins f256mem:$src),
3616 "", []>, Sched<[WriteFLoadY]>;
3619 let isPseudo = 1, mayStore = 1, hasSideEffects = 0 in {
3620 def VMOVAPSZ128mr_NOVLX : I<0, Pseudo, (outs), (ins f128mem:$dst, VR128X:$src),
3621 "", []>, Sched<[WriteFStoreX]>;
3622 def VMOVAPSZ256mr_NOVLX : I<0, Pseudo, (outs), (ins f256mem:$dst, VR256X:$src),
3623 "", []>, Sched<[WriteFStoreY]>;
3624 def VMOVUPSZ128mr_NOVLX : I<0, Pseudo, (outs), (ins f128mem:$dst, VR128X:$src),
3625 "", []>, Sched<[WriteFStoreX]>;
3626 def VMOVUPSZ256mr_NOVLX : I<0, Pseudo, (outs), (ins f256mem:$dst, VR256X:$src),
3627 "", []>, Sched<[WriteFStoreY]>;
3630 def : Pat<(v8i64 (vselect VK8WM:$mask, (bc_v8i64 (v16i32 immAllZerosV)),
3631 (v8i64 VR512:$src))),
3632 (VMOVDQA64Zrrkz (COPY_TO_REGCLASS (KNOTWrr (COPY_TO_REGCLASS VK8:$mask, VK16)),
3635 def : Pat<(v16i32 (vselect VK16WM:$mask, (v16i32 immAllZerosV),
3636 (v16i32 VR512:$src))),
3637 (VMOVDQA32Zrrkz (KNOTWrr VK16WM:$mask), VR512:$src)>;
3639 // These patterns exist to prevent the above patterns from introducing a second
3640 // mask inversion when one already exists.
3641 def : Pat<(v8i64 (vselect (xor VK8:$mask, (v8i1 immAllOnesV)),
3642 (bc_v8i64 (v16i32 immAllZerosV)),
3643 (v8i64 VR512:$src))),
3644 (VMOVDQA64Zrrkz VK8:$mask, VR512:$src)>;
3645 def : Pat<(v16i32 (vselect (xor VK16:$mask, (v16i1 immAllOnesV)),
3646 (v16i32 immAllZerosV),
3647 (v16i32 VR512:$src))),
3648 (VMOVDQA32Zrrkz VK16WM:$mask, VR512:$src)>;
3650 multiclass mask_move_lowering<string InstrStr, X86VectorVTInfo Narrow,
3651 X86VectorVTInfo Wide> {
3652 def : Pat<(Narrow.VT (vselect (Narrow.KVT Narrow.KRCWM:$mask),
3653 Narrow.RC:$src1, Narrow.RC:$src0)),
3656 (!cast<Instruction>(InstrStr#"rrk")
3657 (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src0, Narrow.SubRegIdx)),
3658 (COPY_TO_REGCLASS Narrow.KRCWM:$mask, Wide.KRCWM),
3659 (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src1, Narrow.SubRegIdx)))),
3662 def : Pat<(Narrow.VT (vselect (Narrow.KVT Narrow.KRCWM:$mask),
3663 Narrow.RC:$src1, Narrow.ImmAllZerosV)),
3666 (!cast<Instruction>(InstrStr#"rrkz")
3667 (COPY_TO_REGCLASS Narrow.KRCWM:$mask, Wide.KRCWM),
3668 (Wide.VT (INSERT_SUBREG (IMPLICIT_DEF), Narrow.RC:$src1, Narrow.SubRegIdx)))),
3672 // Patterns for handling v8i1 selects of 256-bit vectors when VLX isn't
3673 // available. Use a 512-bit operation and extract.
3674 let Predicates = [HasAVX512, NoVLX] in {
3675 defm : mask_move_lowering<"VMOVAPSZ", v4f32x_info, v16f32_info>;
3676 defm : mask_move_lowering<"VMOVDQA32Z", v4i32x_info, v16i32_info>;
3677 defm : mask_move_lowering<"VMOVAPSZ", v8f32x_info, v16f32_info>;
3678 defm : mask_move_lowering<"VMOVDQA32Z", v8i32x_info, v16i32_info>;
3680 defm : mask_move_lowering<"VMOVAPDZ", v2f64x_info, v8f64_info>;
3681 defm : mask_move_lowering<"VMOVDQA64Z", v2i64x_info, v8i64_info>;
3682 defm : mask_move_lowering<"VMOVAPDZ", v4f64x_info, v8f64_info>;
3683 defm : mask_move_lowering<"VMOVDQA64Z", v4i64x_info, v8i64_info>;
3686 let Predicates = [HasBWI, NoVLX] in {
3687 defm : mask_move_lowering<"VMOVDQU8Z", v16i8x_info, v64i8_info>;
3688 defm : mask_move_lowering<"VMOVDQU8Z", v32i8x_info, v64i8_info>;
3690 defm : mask_move_lowering<"VMOVDQU16Z", v8i16x_info, v32i16_info>;
3691 defm : mask_move_lowering<"VMOVDQU16Z", v16i16x_info, v32i16_info>;
3694 let Predicates = [HasAVX512] in {
3696 def : Pat<(alignedstore (v16i32 VR512:$src), addr:$dst),
3697 (VMOVDQA64Zmr addr:$dst, VR512:$src)>;
3698 def : Pat<(alignedstore (v32i16 VR512:$src), addr:$dst),
3699 (VMOVDQA64Zmr addr:$dst, VR512:$src)>;
3700 def : Pat<(alignedstore (v64i8 VR512:$src), addr:$dst),
3701 (VMOVDQA64Zmr addr:$dst, VR512:$src)>;
3702 def : Pat<(store (v16i32 VR512:$src), addr:$dst),
3703 (VMOVDQU64Zmr addr:$dst, VR512:$src)>;
3704 def : Pat<(store (v32i16 VR512:$src), addr:$dst),
3705 (VMOVDQU64Zmr addr:$dst, VR512:$src)>;
3706 def : Pat<(store (v64i8 VR512:$src), addr:$dst),
3707 (VMOVDQU64Zmr addr:$dst, VR512:$src)>;
3710 let Predicates = [HasVLX] in {
3712 def : Pat<(alignedstore (v4i32 VR128X:$src), addr:$dst),
3713 (VMOVDQA64Z128mr addr:$dst, VR128X:$src)>;
3714 def : Pat<(alignedstore (v8i16 VR128X:$src), addr:$dst),
3715 (VMOVDQA64Z128mr addr:$dst, VR128X:$src)>;
3716 def : Pat<(alignedstore (v16i8 VR128X:$src), addr:$dst),
3717 (VMOVDQA64Z128mr addr:$dst, VR128X:$src)>;
3718 def : Pat<(store (v4i32 VR128X:$src), addr:$dst),
3719 (VMOVDQU64Z128mr addr:$dst, VR128X:$src)>;
3720 def : Pat<(store (v8i16 VR128X:$src), addr:$dst),
3721 (VMOVDQU64Z128mr addr:$dst, VR128X:$src)>;
3722 def : Pat<(store (v16i8 VR128X:$src), addr:$dst),
3723 (VMOVDQU64Z128mr addr:$dst, VR128X:$src)>;
3726 def : Pat<(alignedstore (v8i32 VR256X:$src), addr:$dst),
3727 (VMOVDQA64Z256mr addr:$dst, VR256X:$src)>;
3728 def : Pat<(alignedstore (v16i16 VR256X:$src), addr:$dst),
3729 (VMOVDQA64Z256mr addr:$dst, VR256X:$src)>;
3730 def : Pat<(alignedstore (v32i8 VR256X:$src), addr:$dst),
3731 (VMOVDQA64Z256mr addr:$dst, VR256X:$src)>;
3732 def : Pat<(store (v8i32 VR256X:$src), addr:$dst),
3733 (VMOVDQU64Z256mr addr:$dst, VR256X:$src)>;
3734 def : Pat<(store (v16i16 VR256X:$src), addr:$dst),
3735 (VMOVDQU64Z256mr addr:$dst, VR256X:$src)>;
3736 def : Pat<(store (v32i8 VR256X:$src), addr:$dst),
3737 (VMOVDQU64Z256mr addr:$dst, VR256X:$src)>;
3740 multiclass masked_move_for_extract<string InstrStr, X86VectorVTInfo From,
3741 X86VectorVTInfo To, X86VectorVTInfo Cast> {
3742 def : Pat<(Cast.VT (vselect Cast.KRCWM:$mask,
3744 (To.VT (extract_subvector
3745 (From.VT From.RC:$src), (iPTR 0)))),
3747 (Cast.VT (!cast<Instruction>(InstrStr#"rrk")
3748 Cast.RC:$src0, Cast.KRCWM:$mask,
3749 (EXTRACT_SUBREG From.RC:$src, To.SubRegIdx)))>;
3751 def : Pat<(Cast.VT (vselect Cast.KRCWM:$mask,
3753 (To.VT (extract_subvector
3754 (From.VT From.RC:$src), (iPTR 0)))),
3755 Cast.ImmAllZerosV)),
3756 (Cast.VT (!cast<Instruction>(InstrStr#"rrkz")
3758 (EXTRACT_SUBREG From.RC:$src, To.SubRegIdx)))>;
3762 let Predicates = [HasVLX] in {
3763 // A masked extract from the first 128-bits of a 256-bit vector can be
3764 // implemented with masked move.
3765 defm : masked_move_for_extract<"VMOVDQA64Z128", v4i64x_info, v2i64x_info, v2i64x_info>;
3766 defm : masked_move_for_extract<"VMOVDQA64Z128", v8i32x_info, v4i32x_info, v2i64x_info>;
3767 defm : masked_move_for_extract<"VMOVDQA64Z128", v16i16x_info, v8i16x_info, v2i64x_info>;
3768 defm : masked_move_for_extract<"VMOVDQA64Z128", v32i8x_info, v16i8x_info, v2i64x_info>;
3769 defm : masked_move_for_extract<"VMOVDQA32Z128", v4i64x_info, v2i64x_info, v4i32x_info>;
3770 defm : masked_move_for_extract<"VMOVDQA32Z128", v8i32x_info, v4i32x_info, v4i32x_info>;
3771 defm : masked_move_for_extract<"VMOVDQA32Z128", v16i16x_info, v8i16x_info, v4i32x_info>;
3772 defm : masked_move_for_extract<"VMOVDQA32Z128", v32i8x_info, v16i8x_info, v4i32x_info>;
3773 defm : masked_move_for_extract<"VMOVAPDZ128", v4f64x_info, v2f64x_info, v2f64x_info>;
3774 defm : masked_move_for_extract<"VMOVAPDZ128", v8f32x_info, v4f32x_info, v2f64x_info>;
3775 defm : masked_move_for_extract<"VMOVAPSZ128", v4f64x_info, v2f64x_info, v4f32x_info>;
3776 defm : masked_move_for_extract<"VMOVAPSZ128", v8f32x_info, v4f32x_info, v4f32x_info>;
3778 // A masked extract from the first 128-bits of a 512-bit vector can be
3779 // implemented with masked move.
3780 defm : masked_move_for_extract<"VMOVDQA64Z128", v8i64_info, v2i64x_info, v2i64x_info>;
3781 defm : masked_move_for_extract<"VMOVDQA64Z128", v16i32_info, v4i32x_info, v2i64x_info>;
3782 defm : masked_move_for_extract<"VMOVDQA64Z128", v32i16_info, v8i16x_info, v2i64x_info>;
3783 defm : masked_move_for_extract<"VMOVDQA64Z128", v64i8_info, v16i8x_info, v2i64x_info>;
3784 defm : masked_move_for_extract<"VMOVDQA32Z128", v8i64_info, v2i64x_info, v4i32x_info>;
3785 defm : masked_move_for_extract<"VMOVDQA32Z128", v16i32_info, v4i32x_info, v4i32x_info>;
3786 defm : masked_move_for_extract<"VMOVDQA32Z128", v32i16_info, v8i16x_info, v4i32x_info>;
3787 defm : masked_move_for_extract<"VMOVDQA32Z128", v64i8_info, v16i8x_info, v4i32x_info>;
3788 defm : masked_move_for_extract<"VMOVAPDZ128", v8f64_info, v2f64x_info, v2f64x_info>;
3789 defm : masked_move_for_extract<"VMOVAPDZ128", v16f32_info, v4f32x_info, v2f64x_info>;
3790 defm : masked_move_for_extract<"VMOVAPSZ128", v8f64_info, v2f64x_info, v4f32x_info>;
3791 defm : masked_move_for_extract<"VMOVAPSZ128", v16f32_info, v4f32x_info, v4f32x_info>;
3793 // A masked extract from the first 256-bits of a 512-bit vector can be
3794 // implemented with masked move.
3795 defm : masked_move_for_extract<"VMOVDQA64Z256", v8i64_info, v4i64x_info, v4i64x_info>;
3796 defm : masked_move_for_extract<"VMOVDQA64Z256", v16i32_info, v8i32x_info, v4i64x_info>;
3797 defm : masked_move_for_extract<"VMOVDQA64Z256", v32i16_info, v16i16x_info, v4i64x_info>;
3798 defm : masked_move_for_extract<"VMOVDQA64Z256", v64i8_info, v32i8x_info, v4i64x_info>;
3799 defm : masked_move_for_extract<"VMOVDQA32Z256", v8i64_info, v4i64x_info, v8i32x_info>;
3800 defm : masked_move_for_extract<"VMOVDQA32Z256", v16i32_info, v8i32x_info, v8i32x_info>;
3801 defm : masked_move_for_extract<"VMOVDQA32Z256", v32i16_info, v16i16x_info, v8i32x_info>;
3802 defm : masked_move_for_extract<"VMOVDQA32Z256", v64i8_info, v32i8x_info, v8i32x_info>;
3803 defm : masked_move_for_extract<"VMOVAPDZ256", v8f64_info, v4f64x_info, v4f64x_info>;
3804 defm : masked_move_for_extract<"VMOVAPDZ256", v16f32_info, v8f32x_info, v4f64x_info>;
3805 defm : masked_move_for_extract<"VMOVAPSZ256", v8f64_info, v4f64x_info, v8f32x_info>;
3806 defm : masked_move_for_extract<"VMOVAPSZ256", v16f32_info, v8f32x_info, v8f32x_info>;
3809 // Move Int Doubleword to Packed Double Int
3811 let ExeDomain = SSEPackedInt in {
3812 def VMOVDI2PDIZrr : AVX512BI<0x6E, MRMSrcReg, (outs VR128X:$dst), (ins GR32:$src),
3813 "vmovd\t{$src, $dst|$dst, $src}",
3815 (v4i32 (scalar_to_vector GR32:$src)))]>,
3816 EVEX, Sched<[WriteVecMoveFromGpr]>;
3817 def VMOVDI2PDIZrm : AVX512BI<0x6E, MRMSrcMem, (outs VR128X:$dst), (ins i32mem:$src),
3818 "vmovd\t{$src, $dst|$dst, $src}",
3820 (v4i32 (scalar_to_vector (loadi32 addr:$src))))]>,
3821 EVEX, EVEX_CD8<32, CD8VT1>, Sched<[WriteVecLoad]>;
3822 def VMOV64toPQIZrr : AVX512BI<0x6E, MRMSrcReg, (outs VR128X:$dst), (ins GR64:$src),
3823 "vmovq\t{$src, $dst|$dst, $src}",
3825 (v2i64 (scalar_to_vector GR64:$src)))]>,
3826 EVEX, VEX_W, Sched<[WriteVecMoveFromGpr]>;
3827 let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0, mayLoad = 1 in
3828 def VMOV64toPQIZrm : AVX512BI<0x6E, MRMSrcMem, (outs VR128X:$dst),
3830 "vmovq\t{$src, $dst|$dst, $src}", []>,
3831 EVEX, VEX_W, EVEX_CD8<64, CD8VT1>, Sched<[WriteVecLoad]>;
3832 let isCodeGenOnly = 1 in {
3833 def VMOV64toSDZrr : AVX512BI<0x6E, MRMSrcReg, (outs FR64X:$dst), (ins GR64:$src),
3834 "vmovq\t{$src, $dst|$dst, $src}",
3835 [(set FR64X:$dst, (bitconvert GR64:$src))]>,
3836 EVEX, VEX_W, Sched<[WriteVecMoveFromGpr]>;
3837 def VMOV64toSDZrm : AVX512XSI<0x7E, MRMSrcMem, (outs FR64X:$dst), (ins i64mem:$src),
3838 "vmovq\t{$src, $dst|$dst, $src}",
3839 [(set FR64X:$dst, (bitconvert (loadi64 addr:$src)))]>,
3840 EVEX, VEX_W, EVEX_CD8<8, CD8VT8>, Sched<[WriteVecLoad]>;
3841 def VMOVSDto64Zrr : AVX512BI<0x7E, MRMDestReg, (outs GR64:$dst), (ins FR64X:$src),
3842 "vmovq\t{$src, $dst|$dst, $src}",
3843 [(set GR64:$dst, (bitconvert FR64X:$src))]>,
3844 EVEX, VEX_W, Sched<[WriteVecMoveFromGpr]>;
3845 def VMOVSDto64Zmr : AVX512BI<0x7E, MRMDestMem, (outs), (ins i64mem:$dst, FR64X:$src),
3846 "vmovq\t{$src, $dst|$dst, $src}",
3847 [(store (i64 (bitconvert FR64X:$src)), addr:$dst)]>,
3848 EVEX, VEX_W, Sched<[WriteVecStore]>,
3849 EVEX_CD8<64, CD8VT1>;
3851 } // ExeDomain = SSEPackedInt
3853 // Move Int Doubleword to Single Scalar
3855 let ExeDomain = SSEPackedInt, isCodeGenOnly = 1 in {
3856 def VMOVDI2SSZrr : AVX512BI<0x6E, MRMSrcReg, (outs FR32X:$dst), (ins GR32:$src),
3857 "vmovd\t{$src, $dst|$dst, $src}",
3858 [(set FR32X:$dst, (bitconvert GR32:$src))]>,
3859 EVEX, Sched<[WriteVecMoveFromGpr]>;
3861 def VMOVDI2SSZrm : AVX512BI<0x6E, MRMSrcMem, (outs FR32X:$dst), (ins i32mem:$src),
3862 "vmovd\t{$src, $dst|$dst, $src}",
3863 [(set FR32X:$dst, (bitconvert (loadi32 addr:$src)))]>,
3864 EVEX, EVEX_CD8<32, CD8VT1>, Sched<[WriteVecLoad]>;
3865 } // ExeDomain = SSEPackedInt, isCodeGenOnly = 1
3867 // Move doubleword from xmm register to r/m32
3869 let ExeDomain = SSEPackedInt in {
3870 def VMOVPDI2DIZrr : AVX512BI<0x7E, MRMDestReg, (outs GR32:$dst), (ins VR128X:$src),
3871 "vmovd\t{$src, $dst|$dst, $src}",
3872 [(set GR32:$dst, (extractelt (v4i32 VR128X:$src),
3874 EVEX, Sched<[WriteVecMoveToGpr]>;
3875 def VMOVPDI2DIZmr : AVX512BI<0x7E, MRMDestMem, (outs),
3876 (ins i32mem:$dst, VR128X:$src),
3877 "vmovd\t{$src, $dst|$dst, $src}",
3878 [(store (i32 (extractelt (v4i32 VR128X:$src),
3879 (iPTR 0))), addr:$dst)]>,
3880 EVEX, EVEX_CD8<32, CD8VT1>, Sched<[WriteVecStore]>;
3881 } // ExeDomain = SSEPackedInt
3883 // Move quadword from xmm1 register to r/m64
3885 let ExeDomain = SSEPackedInt in {
3886 def VMOVPQIto64Zrr : I<0x7E, MRMDestReg, (outs GR64:$dst), (ins VR128X:$src),
3887 "vmovq\t{$src, $dst|$dst, $src}",
3888 [(set GR64:$dst, (extractelt (v2i64 VR128X:$src),
3890 PD, EVEX, VEX_W, Sched<[WriteVecMoveToGpr]>,
3891 Requires<[HasAVX512]>;
3893 let isCodeGenOnly = 1, ForceDisassemble = 1, hasSideEffects = 0, mayStore = 1 in
3894 def VMOVPQIto64Zmr : I<0x7E, MRMDestMem, (outs), (ins i64mem:$dst, VR128X:$src),
3895 "vmovq\t{$src, $dst|$dst, $src}", []>, PD,
3896 EVEX, VEX_W, Sched<[WriteVecStore]>,
3897 Requires<[HasAVX512, In64BitMode]>;
3899 def VMOVPQI2QIZmr : I<0xD6, MRMDestMem, (outs),
3900 (ins i64mem:$dst, VR128X:$src),
3901 "vmovq\t{$src, $dst|$dst, $src}",
3902 [(store (extractelt (v2i64 VR128X:$src), (iPTR 0)),
3904 EVEX, PD, VEX_W, EVEX_CD8<64, CD8VT1>,
3905 Sched<[WriteVecStore]>, Requires<[HasAVX512]>;
3907 let hasSideEffects = 0, isCodeGenOnly = 1, ForceDisassemble = 1 in
3908 def VMOVPQI2QIZrr : AVX512BI<0xD6, MRMDestReg, (outs VR128X:$dst),
3910 "vmovq\t{$src, $dst|$dst, $src}", []>,
3911 EVEX, VEX_W, Sched<[SchedWriteVecLogic.XMM]>;
3912 } // ExeDomain = SSEPackedInt
3914 def : InstAlias<"vmovq.s\t{$src, $dst|$dst, $src}",
3915 (VMOVPQI2QIZrr VR128X:$dst, VR128X:$src), 0>;
3917 // Move Scalar Single to Double Int
3919 let ExeDomain = SSEPackedInt, isCodeGenOnly = 1 in {
3920 def VMOVSS2DIZrr : AVX512BI<0x7E, MRMDestReg, (outs GR32:$dst),
3922 "vmovd\t{$src, $dst|$dst, $src}",
3923 [(set GR32:$dst, (bitconvert FR32X:$src))]>,
3924 EVEX, Sched<[WriteVecMoveToGpr]>;
3925 def VMOVSS2DIZmr : AVX512BI<0x7E, MRMDestMem, (outs),
3926 (ins i32mem:$dst, FR32X:$src),
3927 "vmovd\t{$src, $dst|$dst, $src}",
3928 [(store (i32 (bitconvert FR32X:$src)), addr:$dst)]>,
3929 EVEX, EVEX_CD8<32, CD8VT1>, Sched<[WriteVecStore]>;
3930 } // ExeDomain = SSEPackedInt, isCodeGenOnly = 1
3932 // Move Quadword Int to Packed Quadword Int
3934 let ExeDomain = SSEPackedInt in {
3935 def VMOVQI2PQIZrm : AVX512XSI<0x7E, MRMSrcMem, (outs VR128X:$dst),
3937 "vmovq\t{$src, $dst|$dst, $src}",
3939 (v2i64 (scalar_to_vector (loadi64 addr:$src))))]>,
3940 EVEX, VEX_W, EVEX_CD8<8, CD8VT8>, Sched<[WriteVecLoad]>;
3941 } // ExeDomain = SSEPackedInt
3943 // Allow "vmovd" but print "vmovq".
3944 def : InstAlias<"vmovd\t{$src, $dst|$dst, $src}",
3945 (VMOV64toPQIZrr VR128X:$dst, GR64:$src), 0>;
3946 def : InstAlias<"vmovd\t{$src, $dst|$dst, $src}",
3947 (VMOVPQIto64Zrr GR64:$dst, VR128X:$src), 0>;
3949 //===----------------------------------------------------------------------===//
3950 // AVX-512 MOVSS, MOVSD
3951 //===----------------------------------------------------------------------===//
3953 multiclass avx512_move_scalar<string asm, SDNode OpNode,
3954 X86VectorVTInfo _> {
3955 def rr : AVX512PI<0x10, MRMSrcReg, (outs _.RC:$dst),
3956 (ins _.RC:$src1, _.RC:$src2),
3957 !strconcat(asm, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
3958 [(set _.RC:$dst, (_.VT (OpNode _.RC:$src1, _.RC:$src2)))],
3959 _.ExeDomain>, EVEX_4V, Sched<[SchedWriteFShuffle.XMM]>;
3960 def rrkz : AVX512PI<0x10, MRMSrcReg, (outs _.RC:$dst),
3961 (ins _.KRCWM:$mask, _.RC:$src1, _.RC:$src2),
3962 !strconcat(asm, "\t{$src2, $src1, $dst {${mask}} {z}|",
3963 "$dst {${mask}} {z}, $src1, $src2}"),
3964 [(set _.RC:$dst, (_.VT (X86selects _.KRCWM:$mask,
3965 (_.VT (OpNode _.RC:$src1, _.RC:$src2)),
3967 _.ExeDomain>, EVEX_4V, EVEX_KZ, Sched<[SchedWriteFShuffle.XMM]>;
3968 let Constraints = "$src0 = $dst" in
3969 def rrk : AVX512PI<0x10, MRMSrcReg, (outs _.RC:$dst),
3970 (ins _.RC:$src0, _.KRCWM:$mask, _.RC:$src1, _.RC:$src2),
3971 !strconcat(asm, "\t{$src2, $src1, $dst {${mask}}|",
3972 "$dst {${mask}}, $src1, $src2}"),
3973 [(set _.RC:$dst, (_.VT (X86selects _.KRCWM:$mask,
3974 (_.VT (OpNode _.RC:$src1, _.RC:$src2)),
3975 (_.VT _.RC:$src0))))],
3976 _.ExeDomain>, EVEX_4V, EVEX_K, Sched<[SchedWriteFShuffle.XMM]>;
3977 let canFoldAsLoad = 1, isReMaterializable = 1 in
3978 def rm : AVX512PI<0x10, MRMSrcMem, (outs _.FRC:$dst), (ins _.ScalarMemOp:$src),
3979 !strconcat(asm, "\t{$src, $dst|$dst, $src}"),
3980 [(set _.FRC:$dst, (_.ScalarLdFrag addr:$src))],
3981 _.ExeDomain>, EVEX, Sched<[WriteFLoad]>;
3982 let mayLoad = 1, hasSideEffects = 0 in {
3983 let Constraints = "$src0 = $dst" in
3984 def rmk : AVX512PI<0x10, MRMSrcMem, (outs _.RC:$dst),
3985 (ins _.RC:$src0, _.KRCWM:$mask, _.ScalarMemOp:$src),
3986 !strconcat(asm, "\t{$src, $dst {${mask}}|",
3987 "$dst {${mask}}, $src}"),
3988 [], _.ExeDomain>, EVEX, EVEX_K, Sched<[WriteFLoad]>;
3989 def rmkz : AVX512PI<0x10, MRMSrcMem, (outs _.RC:$dst),
3990 (ins _.KRCWM:$mask, _.ScalarMemOp:$src),
3991 !strconcat(asm, "\t{$src, $dst {${mask}} {z}|",
3992 "$dst {${mask}} {z}, $src}"),
3993 [], _.ExeDomain>, EVEX, EVEX_KZ, Sched<[WriteFLoad]>;
3995 def mr: AVX512PI<0x11, MRMDestMem, (outs), (ins _.ScalarMemOp:$dst, _.FRC:$src),
3996 !strconcat(asm, "\t{$src, $dst|$dst, $src}"),
3997 [(store _.FRC:$src, addr:$dst)], _.ExeDomain>,
3998 EVEX, Sched<[WriteFStore]>;
3999 let mayStore = 1, hasSideEffects = 0 in
4000 def mrk: AVX512PI<0x11, MRMDestMem, (outs),
4001 (ins _.ScalarMemOp:$dst, VK1WM:$mask, _.FRC:$src),
4002 !strconcat(asm, "\t{$src, $dst {${mask}}|$dst {${mask}}, $src}"),
4003 [], _.ExeDomain>, EVEX, EVEX_K, Sched<[WriteFStore]>,
4007 defm VMOVSSZ : avx512_move_scalar<"vmovss", X86Movss, f32x_info>,
4008 VEX_LIG, XS, EVEX_CD8<32, CD8VT1>;
4010 defm VMOVSDZ : avx512_move_scalar<"vmovsd", X86Movsd, f64x_info>,
4011 VEX_LIG, XD, VEX_W, EVEX_CD8<64, CD8VT1>;
4014 multiclass avx512_move_scalar_lowering<string InstrStr, SDNode OpNode,
4015 PatLeaf ZeroFP, X86VectorVTInfo _> {
4017 def : Pat<(_.VT (OpNode _.RC:$src0,
4018 (_.VT (scalar_to_vector
4019 (_.EltVT (X86selects VK1WM:$mask,
4020 (_.EltVT _.FRC:$src1),
4021 (_.EltVT _.FRC:$src2))))))),
4022 (!cast<Instruction>(InstrStr#rrk)
4023 (COPY_TO_REGCLASS _.FRC:$src2, _.RC),
4026 (COPY_TO_REGCLASS _.FRC:$src1, _.RC))>;
4028 def : Pat<(_.VT (OpNode _.RC:$src0,
4029 (_.VT (scalar_to_vector
4030 (_.EltVT (X86selects VK1WM:$mask,
4031 (_.EltVT _.FRC:$src1),
4032 (_.EltVT ZeroFP))))))),
4033 (!cast<Instruction>(InstrStr#rrkz)
4036 (COPY_TO_REGCLASS _.FRC:$src1, _.RC))>;
4039 multiclass avx512_store_scalar_lowering<string InstrStr, AVX512VLVectorVTInfo _,
4040 dag Mask, RegisterClass MaskRC> {
4042 def : Pat<(masked_store addr:$dst, Mask,
4043 (_.info512.VT (insert_subvector undef,
4044 (_.info128.VT _.info128.RC:$src),
4046 (!cast<Instruction>(InstrStr#mrk) addr:$dst,
4047 (COPY_TO_REGCLASS MaskRC:$mask, VK1WM),
4048 (COPY_TO_REGCLASS _.info128.RC:$src, _.info128.FRC))>;
4052 multiclass avx512_store_scalar_lowering_subreg<string InstrStr,
4053 AVX512VLVectorVTInfo _,
4054 dag Mask, RegisterClass MaskRC,
4055 SubRegIndex subreg> {
4057 def : Pat<(masked_store addr:$dst, Mask,
4058 (_.info512.VT (insert_subvector undef,
4059 (_.info128.VT _.info128.RC:$src),
4061 (!cast<Instruction>(InstrStr#mrk) addr:$dst,
4062 (COPY_TO_REGCLASS (i32 (INSERT_SUBREG (IMPLICIT_DEF), MaskRC:$mask, subreg)), VK1WM),
4063 (COPY_TO_REGCLASS _.info128.RC:$src, _.info128.FRC))>;
4067 // This matches the more recent codegen from clang that avoids emitting a 512
4068 // bit masked store directly. Codegen will widen 128-bit masked store to 512
4069 // bits on AVX512F only targets.
4070 multiclass avx512_store_scalar_lowering_subreg2<string InstrStr,
4071 AVX512VLVectorVTInfo _,
4072 dag Mask512, dag Mask128,
4073 RegisterClass MaskRC,
4074 SubRegIndex subreg> {
4077 def : Pat<(masked_store addr:$dst, Mask512,
4078 (_.info512.VT (insert_subvector undef,
4079 (_.info128.VT _.info128.RC:$src),
4081 (!cast<Instruction>(InstrStr#mrk) addr:$dst,
4082 (COPY_TO_REGCLASS (i32 (INSERT_SUBREG (IMPLICIT_DEF), MaskRC:$mask, subreg)), VK1WM),
4083 (COPY_TO_REGCLASS _.info128.RC:$src, _.info128.FRC))>;
4085 // AVX512VL pattern.
4086 def : Pat<(masked_store addr:$dst, Mask128, (_.info128.VT _.info128.RC:$src)),
4087 (!cast<Instruction>(InstrStr#mrk) addr:$dst,
4088 (COPY_TO_REGCLASS (i32 (INSERT_SUBREG (IMPLICIT_DEF), MaskRC:$mask, subreg)), VK1WM),
4089 (COPY_TO_REGCLASS _.info128.RC:$src, _.info128.FRC))>;
4092 multiclass avx512_load_scalar_lowering<string InstrStr, AVX512VLVectorVTInfo _,
4093 dag Mask, RegisterClass MaskRC> {
4095 def : Pat<(_.info128.VT (extract_subvector
4096 (_.info512.VT (masked_load addr:$srcAddr, Mask,
4097 (_.info512.VT (bitconvert
4098 (v16i32 immAllZerosV))))),
4100 (!cast<Instruction>(InstrStr#rmkz)
4101 (COPY_TO_REGCLASS MaskRC:$mask, VK1WM),
4104 def : Pat<(_.info128.VT (extract_subvector
4105 (_.info512.VT (masked_load addr:$srcAddr, Mask,
4106 (_.info512.VT (insert_subvector undef,
4107 (_.info128.VT (X86vzmovl _.info128.RC:$src)),
4110 (!cast<Instruction>(InstrStr#rmk) _.info128.RC:$src,
4111 (COPY_TO_REGCLASS MaskRC:$mask, VK1WM),
4116 multiclass avx512_load_scalar_lowering_subreg<string InstrStr,
4117 AVX512VLVectorVTInfo _,
4118 dag Mask, RegisterClass MaskRC,
4119 SubRegIndex subreg> {
4121 def : Pat<(_.info128.VT (extract_subvector
4122 (_.info512.VT (masked_load addr:$srcAddr, Mask,
4123 (_.info512.VT (bitconvert
4124 (v16i32 immAllZerosV))))),
4126 (!cast<Instruction>(InstrStr#rmkz)
4127 (COPY_TO_REGCLASS (i32 (INSERT_SUBREG (IMPLICIT_DEF), MaskRC:$mask, subreg)), VK1WM),
4130 def : Pat<(_.info128.VT (extract_subvector
4131 (_.info512.VT (masked_load addr:$srcAddr, Mask,
4132 (_.info512.VT (insert_subvector undef,
4133 (_.info128.VT (X86vzmovl _.info128.RC:$src)),
4136 (!cast<Instruction>(InstrStr#rmk) _.info128.RC:$src,
4137 (COPY_TO_REGCLASS (i32 (INSERT_SUBREG (IMPLICIT_DEF), MaskRC:$mask, subreg)), VK1WM),
4142 // This matches the more recent codegen from clang that avoids emitting a 512
4143 // bit masked load directly. Codegen will widen 128-bit masked load to 512
4144 // bits on AVX512F only targets.
4145 multiclass avx512_load_scalar_lowering_subreg2<string InstrStr,
4146 AVX512VLVectorVTInfo _,
4147 dag Mask512, dag Mask128,
4148 RegisterClass MaskRC,
4149 SubRegIndex subreg> {
4150 // AVX512F patterns.
4151 def : Pat<(_.info128.VT (extract_subvector
4152 (_.info512.VT (masked_load addr:$srcAddr, Mask512,
4153 (_.info512.VT (bitconvert
4154 (v16i32 immAllZerosV))))),
4156 (!cast<Instruction>(InstrStr#rmkz)
4157 (COPY_TO_REGCLASS (i32 (INSERT_SUBREG (IMPLICIT_DEF), MaskRC:$mask, subreg)), VK1WM),
4160 def : Pat<(_.info128.VT (extract_subvector
4161 (_.info512.VT (masked_load addr:$srcAddr, Mask512,
4162 (_.info512.VT (insert_subvector undef,
4163 (_.info128.VT (X86vzmovl _.info128.RC:$src)),
4166 (!cast<Instruction>(InstrStr#rmk) _.info128.RC:$src,
4167 (COPY_TO_REGCLASS (i32 (INSERT_SUBREG (IMPLICIT_DEF), MaskRC:$mask, subreg)), VK1WM),
4170 // AVX512Vl patterns.
4171 def : Pat<(_.info128.VT (masked_load addr:$srcAddr, Mask128,
4172 (_.info128.VT (bitconvert (v4i32 immAllZerosV))))),
4173 (!cast<Instruction>(InstrStr#rmkz)
4174 (COPY_TO_REGCLASS (i32 (INSERT_SUBREG (IMPLICIT_DEF), MaskRC:$mask, subreg)), VK1WM),
4177 def : Pat<(_.info128.VT (masked_load addr:$srcAddr, Mask128,
4178 (_.info128.VT (X86vzmovl _.info128.RC:$src)))),
4179 (!cast<Instruction>(InstrStr#rmk) _.info128.RC:$src,
4180 (COPY_TO_REGCLASS (i32 (INSERT_SUBREG (IMPLICIT_DEF), MaskRC:$mask, subreg)), VK1WM),
4184 defm : avx512_move_scalar_lowering<"VMOVSSZ", X86Movss, fp32imm0, v4f32x_info>;
4185 defm : avx512_move_scalar_lowering<"VMOVSDZ", X86Movsd, fp64imm0, v2f64x_info>;
4187 defm : avx512_store_scalar_lowering<"VMOVSSZ", avx512vl_f32_info,
4188 (v16i1 (bitconvert (i16 (trunc (and GR32:$mask, (i32 1)))))), GR32>;
4189 defm : avx512_store_scalar_lowering_subreg<"VMOVSSZ", avx512vl_f32_info,
4190 (v16i1 (bitconvert (i16 (and GR16:$mask, (i16 1))))), GR16, sub_16bit>;
4191 defm : avx512_store_scalar_lowering_subreg<"VMOVSDZ", avx512vl_f64_info,
4192 (v8i1 (bitconvert (i8 (and GR8:$mask, (i8 1))))), GR8, sub_8bit>;
4194 defm : avx512_store_scalar_lowering_subreg2<"VMOVSSZ", avx512vl_f32_info,
4195 (v16i1 (insert_subvector
4196 (v16i1 immAllZerosV),
4197 (v4i1 (extract_subvector
4198 (v8i1 (bitconvert (and GR8:$mask, (i8 1)))),
4201 (v4i1 (extract_subvector
4202 (v8i1 (bitconvert (and GR8:$mask, (i8 1)))),
4203 (iPTR 0))), GR8, sub_8bit>;
4204 defm : avx512_store_scalar_lowering_subreg2<"VMOVSDZ", avx512vl_f64_info,
4209 (v16i1 immAllZerosV),
4210 (v2i1 (extract_subvector
4211 (v8i1 (bitconvert (i8 (and GR8:$mask, (i8 1))))),
4215 (v2i1 (extract_subvector
4216 (v8i1 (bitconvert (i8 (and GR8:$mask, (i8 1))))),
4217 (iPTR 0))), GR8, sub_8bit>;
4219 defm : avx512_load_scalar_lowering<"VMOVSSZ", avx512vl_f32_info,
4220 (v16i1 (bitconvert (i16 (trunc (and GR32:$mask, (i32 1)))))), GR32>;
4221 defm : avx512_load_scalar_lowering_subreg<"VMOVSSZ", avx512vl_f32_info,
4222 (v16i1 (bitconvert (i16 (and GR16:$mask, (i16 1))))), GR16, sub_16bit>;
4223 defm : avx512_load_scalar_lowering_subreg<"VMOVSDZ", avx512vl_f64_info,
4224 (v8i1 (bitconvert (i8 (and GR8:$mask, (i8 1))))), GR8, sub_8bit>;
4226 defm : avx512_load_scalar_lowering_subreg2<"VMOVSSZ", avx512vl_f32_info,
4227 (v16i1 (insert_subvector
4228 (v16i1 immAllZerosV),
4229 (v4i1 (extract_subvector
4230 (v8i1 (bitconvert (and GR8:$mask, (i8 1)))),
4233 (v4i1 (extract_subvector
4234 (v8i1 (bitconvert (and GR8:$mask, (i8 1)))),
4235 (iPTR 0))), GR8, sub_8bit>;
4236 defm : avx512_load_scalar_lowering_subreg2<"VMOVSDZ", avx512vl_f64_info,
4241 (v16i1 immAllZerosV),
4242 (v2i1 (extract_subvector
4243 (v8i1 (bitconvert (i8 (and GR8:$mask, (i8 1))))),
4247 (v2i1 (extract_subvector
4248 (v8i1 (bitconvert (i8 (and GR8:$mask, (i8 1))))),
4249 (iPTR 0))), GR8, sub_8bit>;
4251 def : Pat<(f32 (X86selects (scalar_to_vector GR8:$mask),
4252 (f32 FR32X:$src1), (f32 FR32X:$src2))),
4254 (VMOVSSZrrk (COPY_TO_REGCLASS FR32X:$src2, VR128X),
4255 (COPY_TO_REGCLASS (i32 (INSERT_SUBREG (IMPLICIT_DEF),
4256 GR8:$mask, sub_8bit)), VK1WM),
4257 (v4f32 (IMPLICIT_DEF)), (COPY_TO_REGCLASS FR32X:$src1, VR128X)),
4260 def : Pat<(f32 (X86selects VK1WM:$mask, (f32 FR32X:$src1), (f32 FR32X:$src2))),
4261 (COPY_TO_REGCLASS (VMOVSSZrrk (COPY_TO_REGCLASS FR32X:$src2, VR128X),
4262 VK1WM:$mask, (v4f32 (IMPLICIT_DEF)),
4263 (COPY_TO_REGCLASS FR32X:$src1, VR128X)), FR32X)>;
4265 def : Pat<(f64 (X86selects (scalar_to_vector GR8:$mask),
4266 (f64 FR64X:$src1), (f64 FR64X:$src2))),
4268 (VMOVSDZrrk (COPY_TO_REGCLASS FR64X:$src2, VR128X),
4269 (COPY_TO_REGCLASS (i32 (INSERT_SUBREG (IMPLICIT_DEF),
4270 GR8:$mask, sub_8bit)), VK1WM),
4271 (v2f64 (IMPLICIT_DEF)), (COPY_TO_REGCLASS FR64X:$src1, VR128X)),
4274 def : Pat<(f64 (X86selects VK1WM:$mask, (f64 FR64X:$src1), (f64 FR64X:$src2))),
4275 (COPY_TO_REGCLASS (VMOVSDZrrk (COPY_TO_REGCLASS FR64X:$src2, VR128X),
4276 VK1WM:$mask, (v2f64 (IMPLICIT_DEF)),
4277 (COPY_TO_REGCLASS FR64X:$src1, VR128X)), FR64X)>;
4279 let hasSideEffects = 0, isCodeGenOnly = 1, ForceDisassemble = 1 in {
4280 def VMOVSSZrr_REV: AVX512<0x11, MRMDestReg, (outs VR128X:$dst),
4281 (ins VR128X:$src1, VR128X:$src2),
4282 "vmovss\t{$src2, $src1, $dst|$dst, $src1, $src2}",
4283 []>, XS, EVEX_4V, VEX_LIG,
4284 FoldGenData<"VMOVSSZrr">,
4285 Sched<[SchedWriteFShuffle.XMM]>;
4287 let Constraints = "$src0 = $dst" in
4288 def VMOVSSZrrk_REV: AVX512<0x11, MRMDestReg, (outs VR128X:$dst),
4289 (ins f32x_info.RC:$src0, f32x_info.KRCWM:$mask,
4290 VR128X:$src1, VR128X:$src2),
4291 "vmovss\t{$src2, $src1, $dst {${mask}}|"#
4292 "$dst {${mask}}, $src1, $src2}",
4293 []>, EVEX_K, XS, EVEX_4V, VEX_LIG,
4294 FoldGenData<"VMOVSSZrrk">,
4295 Sched<[SchedWriteFShuffle.XMM]>;
4297 def VMOVSSZrrkz_REV: AVX512<0x11, MRMDestReg, (outs VR128X:$dst),
4298 (ins f32x_info.KRCWM:$mask, VR128X:$src1, VR128X:$src2),
4299 "vmovss\t{$src2, $src1, $dst {${mask}} {z}|"#
4300 "$dst {${mask}} {z}, $src1, $src2}",
4301 []>, EVEX_KZ, XS, EVEX_4V, VEX_LIG,
4302 FoldGenData<"VMOVSSZrrkz">,
4303 Sched<[SchedWriteFShuffle.XMM]>;
4305 def VMOVSDZrr_REV: AVX512<0x11, MRMDestReg, (outs VR128X:$dst),
4306 (ins VR128X:$src1, VR128X:$src2),
4307 "vmovsd\t{$src2, $src1, $dst|$dst, $src1, $src2}",
4308 []>, XD, EVEX_4V, VEX_LIG, VEX_W,
4309 FoldGenData<"VMOVSDZrr">,
4310 Sched<[SchedWriteFShuffle.XMM]>;
4312 let Constraints = "$src0 = $dst" in
4313 def VMOVSDZrrk_REV: AVX512<0x11, MRMDestReg, (outs VR128X:$dst),
4314 (ins f64x_info.RC:$src0, f64x_info.KRCWM:$mask,
4315 VR128X:$src1, VR128X:$src2),
4316 "vmovsd\t{$src2, $src1, $dst {${mask}}|"#
4317 "$dst {${mask}}, $src1, $src2}",
4318 []>, EVEX_K, XD, EVEX_4V, VEX_LIG,
4319 VEX_W, FoldGenData<"VMOVSDZrrk">,
4320 Sched<[SchedWriteFShuffle.XMM]>;
4322 def VMOVSDZrrkz_REV: AVX512<0x11, MRMDestReg, (outs VR128X:$dst),
4323 (ins f64x_info.KRCWM:$mask, VR128X:$src1,
4325 "vmovsd\t{$src2, $src1, $dst {${mask}} {z}|"#
4326 "$dst {${mask}} {z}, $src1, $src2}",
4327 []>, EVEX_KZ, XD, EVEX_4V, VEX_LIG,
4328 VEX_W, FoldGenData<"VMOVSDZrrkz">,
4329 Sched<[SchedWriteFShuffle.XMM]>;
4332 def : InstAlias<"vmovss.s\t{$src2, $src1, $dst|$dst, $src1, $src2}",
4333 (VMOVSSZrr_REV VR128X:$dst, VR128X:$src1, VR128X:$src2), 0>;
4334 def : InstAlias<"vmovss.s\t{$src2, $src1, $dst {${mask}}|"#
4335 "$dst {${mask}}, $src1, $src2}",
4336 (VMOVSSZrrk_REV VR128X:$dst, VK1WM:$mask,
4337 VR128X:$src1, VR128X:$src2), 0>;
4338 def : InstAlias<"vmovss.s\t{$src2, $src1, $dst {${mask}} {z}|"#
4339 "$dst {${mask}} {z}, $src1, $src2}",
4340 (VMOVSSZrrkz_REV VR128X:$dst, VK1WM:$mask,
4341 VR128X:$src1, VR128X:$src2), 0>;
4342 def : InstAlias<"vmovsd.s\t{$src2, $src1, $dst|$dst, $src1, $src2}",
4343 (VMOVSDZrr_REV VR128X:$dst, VR128X:$src1, VR128X:$src2), 0>;
4344 def : InstAlias<"vmovsd.s\t{$src2, $src1, $dst {${mask}}|"#
4345 "$dst {${mask}}, $src1, $src2}",
4346 (VMOVSDZrrk_REV VR128X:$dst, VK1WM:$mask,
4347 VR128X:$src1, VR128X:$src2), 0>;
4348 def : InstAlias<"vmovsd.s\t{$src2, $src1, $dst {${mask}} {z}|"#
4349 "$dst {${mask}} {z}, $src1, $src2}",
4350 (VMOVSDZrrkz_REV VR128X:$dst, VK1WM:$mask,
4351 VR128X:$src1, VR128X:$src2), 0>;
4353 let Predicates = [HasAVX512] in {
4354 let AddedComplexity = 15 in {
4355 def : Pat<(v4f32 (X86vzmovl (v4f32 VR128X:$src))),
4356 (VMOVSSZrr (v4f32 (AVX512_128_SET0)), VR128X:$src)>;
4357 def : Pat<(v4i32 (X86vzmovl (v4i32 VR128X:$src))),
4358 (VMOVSSZrr (v4i32 (AVX512_128_SET0)), VR128X:$src)>;
4359 def : Pat<(v2f64 (X86vzmovl (v2f64 (scalar_to_vector FR64X:$src)))),
4360 (VMOVSDZrr (v2f64 (AVX512_128_SET0)),
4361 (COPY_TO_REGCLASS FR64X:$src, VR128X))>;
4364 // Move low f32 and clear high bits.
4365 def : Pat<(v8f32 (X86vzmovl (v8f32 VR256X:$src))),
4366 (SUBREG_TO_REG (i32 0),
4367 (VMOVSSZrr (v4f32 (AVX512_128_SET0)),
4368 (EXTRACT_SUBREG (v8f32 VR256X:$src), sub_xmm)), sub_xmm)>;
4369 def : Pat<(v8i32 (X86vzmovl (v8i32 VR256X:$src))),
4370 (SUBREG_TO_REG (i32 0),
4371 (VMOVSSZrr (v4i32 (AVX512_128_SET0)),
4372 (EXTRACT_SUBREG (v8i32 VR256X:$src), sub_xmm)), sub_xmm)>;
4373 def : Pat<(v16f32 (X86vzmovl (v16f32 VR512:$src))),
4374 (SUBREG_TO_REG (i32 0),
4375 (VMOVSSZrr (v4f32 (AVX512_128_SET0)),
4376 (EXTRACT_SUBREG (v16f32 VR512:$src), sub_xmm)), sub_xmm)>;
4377 def : Pat<(v16i32 (X86vzmovl (v16i32 VR512:$src))),
4378 (SUBREG_TO_REG (i32 0),
4379 (VMOVSSZrr (v4i32 (AVX512_128_SET0)),
4380 (EXTRACT_SUBREG (v16i32 VR512:$src), sub_xmm)), sub_xmm)>;
4382 let AddedComplexity = 20 in {
4383 // MOVSSrm zeros the high parts of the register; represent this
4384 // with SUBREG_TO_REG. The AVX versions also write: DST[255:128] <- 0
4385 def : Pat<(v4f32 (X86vzmovl (v4f32 (scalar_to_vector (loadf32 addr:$src))))),
4386 (COPY_TO_REGCLASS (VMOVSSZrm addr:$src), VR128X)>;
4387 def : Pat<(v4f32 (scalar_to_vector (loadf32 addr:$src))),
4388 (COPY_TO_REGCLASS (VMOVSSZrm addr:$src), VR128X)>;
4389 def : Pat<(v4f32 (X86vzmovl (loadv4f32 addr:$src))),
4390 (COPY_TO_REGCLASS (VMOVSSZrm addr:$src), VR128X)>;
4391 def : Pat<(v4f32 (X86vzload addr:$src)),
4392 (COPY_TO_REGCLASS (VMOVSSZrm addr:$src), VR128X)>;
4394 // MOVSDrm zeros the high parts of the register; represent this
4395 // with SUBREG_TO_REG. The AVX versions also write: DST[255:128] <- 0
4396 def : Pat<(v2f64 (X86vzmovl (v2f64 (scalar_to_vector (loadf64 addr:$src))))),
4397 (COPY_TO_REGCLASS (VMOVSDZrm addr:$src), VR128X)>;
4398 def : Pat<(v2f64 (scalar_to_vector (loadf64 addr:$src))),
4399 (COPY_TO_REGCLASS (VMOVSDZrm addr:$src), VR128X)>;
4400 def : Pat<(v2f64 (X86vzmovl (loadv2f64 addr:$src))),
4401 (COPY_TO_REGCLASS (VMOVSDZrm addr:$src), VR128X)>;
4402 def : Pat<(v2f64 (X86vzmovl (bc_v2f64 (loadv4f32 addr:$src)))),
4403 (COPY_TO_REGCLASS (VMOVSDZrm addr:$src), VR128X)>;
4404 def : Pat<(v2f64 (X86vzload addr:$src)),
4405 (COPY_TO_REGCLASS (VMOVSDZrm addr:$src), VR128X)>;
4407 // Represent the same patterns above but in the form they appear for
4409 def : Pat<(v8i32 (X86vzmovl (insert_subvector undef,
4410 (v4i32 (scalar_to_vector (loadi32 addr:$src))), (iPTR 0)))),
4411 (SUBREG_TO_REG (i32 0), (VMOVDI2PDIZrm addr:$src), sub_xmm)>;
4412 def : Pat<(v8f32 (X86vzmovl (insert_subvector undef,
4413 (v4f32 (scalar_to_vector (loadf32 addr:$src))), (iPTR 0)))),
4414 (SUBREG_TO_REG (i32 0), (VMOVSSZrm addr:$src), sub_xmm)>;
4415 def : Pat<(v8f32 (X86vzload addr:$src)),
4416 (SUBREG_TO_REG (i32 0), (VMOVSSZrm addr:$src), sub_xmm)>;
4417 def : Pat<(v4f64 (X86vzmovl (insert_subvector undef,
4418 (v2f64 (scalar_to_vector (loadf64 addr:$src))), (iPTR 0)))),
4419 (SUBREG_TO_REG (i32 0), (VMOVSDZrm addr:$src), sub_xmm)>;
4420 def : Pat<(v4f64 (X86vzload addr:$src)),
4421 (SUBREG_TO_REG (i32 0), (VMOVSDZrm addr:$src), sub_xmm)>;
4423 // Represent the same patterns above but in the form they appear for
4425 def : Pat<(v16i32 (X86vzmovl (insert_subvector undef,
4426 (v4i32 (scalar_to_vector (loadi32 addr:$src))), (iPTR 0)))),
4427 (SUBREG_TO_REG (i32 0), (VMOVDI2PDIZrm addr:$src), sub_xmm)>;
4428 def : Pat<(v16f32 (X86vzmovl (insert_subvector undef,
4429 (v4f32 (scalar_to_vector (loadf32 addr:$src))), (iPTR 0)))),
4430 (SUBREG_TO_REG (i32 0), (VMOVSSZrm addr:$src), sub_xmm)>;
4431 def : Pat<(v16f32 (X86vzload addr:$src)),
4432 (SUBREG_TO_REG (i32 0), (VMOVSSZrm addr:$src), sub_xmm)>;
4433 def : Pat<(v8f64 (X86vzmovl (insert_subvector undef,
4434 (v2f64 (scalar_to_vector (loadf64 addr:$src))), (iPTR 0)))),
4435 (SUBREG_TO_REG (i32 0), (VMOVSDZrm addr:$src), sub_xmm)>;
4436 def : Pat<(v8f64 (X86vzload addr:$src)),
4437 (SUBREG_TO_REG (i32 0), (VMOVSDZrm addr:$src), sub_xmm)>;
4439 def : Pat<(v4i64 (X86vzmovl (insert_subvector undef,
4440 (v2i64 (scalar_to_vector (loadi64 addr:$src))), (iPTR 0)))),
4441 (SUBREG_TO_REG (i64 0), (VMOVQI2PQIZrm addr:$src), sub_xmm)>;
4443 // Move low f64 and clear high bits.
4444 def : Pat<(v4f64 (X86vzmovl (v4f64 VR256X:$src))),
4445 (SUBREG_TO_REG (i32 0),
4446 (VMOVSDZrr (v2f64 (AVX512_128_SET0)),
4447 (EXTRACT_SUBREG (v4f64 VR256X:$src), sub_xmm)), sub_xmm)>;
4448 def : Pat<(v8f64 (X86vzmovl (v8f64 VR512:$src))),
4449 (SUBREG_TO_REG (i32 0),
4450 (VMOVSDZrr (v2f64 (AVX512_128_SET0)),
4451 (EXTRACT_SUBREG (v8f64 VR512:$src), sub_xmm)), sub_xmm)>;
4453 def : Pat<(v4i64 (X86vzmovl (v4i64 VR256X:$src))),
4454 (SUBREG_TO_REG (i32 0), (VMOVSDZrr (v2i64 (AVX512_128_SET0)),
4455 (EXTRACT_SUBREG (v4i64 VR256X:$src), sub_xmm)), sub_xmm)>;
4456 def : Pat<(v8i64 (X86vzmovl (v8i64 VR512:$src))),
4457 (SUBREG_TO_REG (i32 0), (VMOVSDZrr (v2i64 (AVX512_128_SET0)),
4458 (EXTRACT_SUBREG (v8i64 VR512:$src), sub_xmm)), sub_xmm)>;
4460 // Extract and store.
4461 def : Pat<(store (f32 (extractelt (v4f32 VR128X:$src), (iPTR 0))),
4463 (VMOVSSZmr addr:$dst, (COPY_TO_REGCLASS (v4f32 VR128X:$src), FR32X))>;
4465 // Shuffle with VMOVSS
4466 def : Pat<(v4i32 (X86Movss VR128X:$src1, VR128X:$src2)),
4467 (VMOVSSZrr (v4i32 VR128X:$src1), VR128X:$src2)>;
4469 def : Pat<(v4f32 (X86Movss VR128X:$src1, (scalar_to_vector FR32X:$src2))),
4470 (VMOVSSZrr VR128X:$src1,
4471 (COPY_TO_REGCLASS FR32X:$src2, VR128X))>;
4473 // Shuffle with VMOVSD
4474 def : Pat<(v2i64 (X86Movsd VR128X:$src1, VR128X:$src2)),
4475 (VMOVSDZrr VR128X:$src1, VR128X:$src2)>;
4477 def : Pat<(v2f64 (X86Movsd VR128X:$src1, (scalar_to_vector FR64X:$src2))),
4478 (VMOVSDZrr VR128X:$src1, (COPY_TO_REGCLASS FR64X:$src2, VR128X))>;
4480 def : Pat<(v2f64 (X86Movlpd VR128X:$src1, VR128X:$src2)),
4481 (VMOVSDZrr VR128X:$src1, VR128X:$src2)>;
4482 def : Pat<(v4f32 (X86Movlps VR128X:$src1, VR128X:$src2)),
4483 (VMOVSDZrr VR128X:$src1, VR128X:$src2)>;
4486 let ExeDomain = SSEPackedInt, SchedRW = [SchedWriteVecLogic.XMM] in {
4487 let AddedComplexity = 15 in
4488 def VMOVZPQILo2PQIZrr : AVX512XSI<0x7E, MRMSrcReg, (outs VR128X:$dst),
4490 "vmovq\t{$src, $dst|$dst, $src}",
4491 [(set VR128X:$dst, (v2i64 (X86vzmovl
4492 (v2i64 VR128X:$src))))]>,
4496 let Predicates = [HasAVX512] in {
4497 let AddedComplexity = 15 in {
4498 def : Pat<(v4i32 (X86vzmovl (v4i32 (scalar_to_vector GR32:$src)))),
4499 (VMOVDI2PDIZrr GR32:$src)>;
4501 def : Pat<(v2i64 (X86vzmovl (v2i64 (scalar_to_vector GR64:$src)))),
4502 (VMOV64toPQIZrr GR64:$src)>;
4504 def : Pat<(v4i64 (X86vzmovl (insert_subvector undef,
4505 (v2i64 (scalar_to_vector GR64:$src)),(iPTR 0)))),
4506 (SUBREG_TO_REG (i64 0), (VMOV64toPQIZrr GR64:$src), sub_xmm)>;
4508 def : Pat<(v8i64 (X86vzmovl (insert_subvector undef,
4509 (v2i64 (scalar_to_vector GR64:$src)),(iPTR 0)))),
4510 (SUBREG_TO_REG (i64 0), (VMOV64toPQIZrr GR64:$src), sub_xmm)>;
4512 // AVX 128-bit movd/movq instruction write zeros in the high 128-bit part.
4513 let AddedComplexity = 20 in {
4514 def : Pat<(v2i64 (X86vzmovl (v2i64 (scalar_to_vector (zextloadi64i32 addr:$src))))),
4515 (VMOVDI2PDIZrm addr:$src)>;
4516 def : Pat<(v4i32 (X86vzmovl (v4i32 (scalar_to_vector (loadi32 addr:$src))))),
4517 (VMOVDI2PDIZrm addr:$src)>;
4518 def : Pat<(v4i32 (X86vzmovl (bc_v4i32 (loadv4f32 addr:$src)))),
4519 (VMOVDI2PDIZrm addr:$src)>;
4520 def : Pat<(v4i32 (X86vzmovl (bc_v4i32 (loadv2i64 addr:$src)))),
4521 (VMOVDI2PDIZrm addr:$src)>;
4522 def : Pat<(v4i32 (X86vzload addr:$src)),
4523 (VMOVDI2PDIZrm addr:$src)>;
4524 def : Pat<(v8i32 (X86vzload addr:$src)),
4525 (SUBREG_TO_REG (i32 0), (VMOVDI2PDIZrm addr:$src), sub_xmm)>;
4526 def : Pat<(v2i64 (X86vzmovl (loadv2i64 addr:$src))),
4527 (VMOVQI2PQIZrm addr:$src)>;
4528 def : Pat<(v2f64 (X86vzmovl (v2f64 VR128X:$src))),
4529 (VMOVZPQILo2PQIZrr VR128X:$src)>;
4530 def : Pat<(v2i64 (X86vzload addr:$src)),
4531 (VMOVQI2PQIZrm addr:$src)>;
4532 def : Pat<(v4i64 (X86vzload addr:$src)),
4533 (SUBREG_TO_REG (i64 0), (VMOVQI2PQIZrm addr:$src), sub_xmm)>;
4536 // Use regular 128-bit instructions to match 256-bit scalar_to_vec+zext.
4537 def : Pat<(v8i32 (X86vzmovl (insert_subvector undef,
4538 (v4i32 (scalar_to_vector GR32:$src)),(iPTR 0)))),
4539 (SUBREG_TO_REG (i32 0), (VMOVDI2PDIZrr GR32:$src), sub_xmm)>;
4540 def : Pat<(v16i32 (X86vzmovl (insert_subvector undef,
4541 (v4i32 (scalar_to_vector GR32:$src)),(iPTR 0)))),
4542 (SUBREG_TO_REG (i32 0), (VMOVDI2PDIZrr GR32:$src), sub_xmm)>;
4544 // Use regular 128-bit instructions to match 512-bit scalar_to_vec+zext.
4545 def : Pat<(v16i32 (X86vzload addr:$src)),
4546 (SUBREG_TO_REG (i32 0), (VMOVDI2PDIZrm addr:$src), sub_xmm)>;
4547 def : Pat<(v8i64 (X86vzload addr:$src)),
4548 (SUBREG_TO_REG (i64 0), (VMOVQI2PQIZrm addr:$src), sub_xmm)>;
4551 //===----------------------------------------------------------------------===//
4552 // AVX-512 - Non-temporals
4553 //===----------------------------------------------------------------------===//
4555 def VMOVNTDQAZrm : AVX512PI<0x2A, MRMSrcMem, (outs VR512:$dst),
4556 (ins i512mem:$src), "vmovntdqa\t{$src, $dst|$dst, $src}",
4557 [], SSEPackedInt>, Sched<[SchedWriteVecMoveLS.ZMM.RM]>,
4558 EVEX, T8PD, EVEX_V512, EVEX_CD8<64, CD8VF>;
4560 let Predicates = [HasVLX] in {
4561 def VMOVNTDQAZ256rm : AVX512PI<0x2A, MRMSrcMem, (outs VR256X:$dst),
4563 "vmovntdqa\t{$src, $dst|$dst, $src}",
4564 [], SSEPackedInt>, Sched<[SchedWriteVecMoveLS.YMM.RM]>,
4565 EVEX, T8PD, EVEX_V256, EVEX_CD8<64, CD8VF>;
4567 def VMOVNTDQAZ128rm : AVX512PI<0x2A, MRMSrcMem, (outs VR128X:$dst),
4569 "vmovntdqa\t{$src, $dst|$dst, $src}",
4570 [], SSEPackedInt>, Sched<[SchedWriteVecMoveLS.XMM.RM]>,
4571 EVEX, T8PD, EVEX_V128, EVEX_CD8<64, CD8VF>;
4574 multiclass avx512_movnt<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
4575 X86SchedWriteMoveLS Sched,
4576 PatFrag st_frag = alignednontemporalstore> {
4577 let SchedRW = [Sched.MR], AddedComplexity = 400 in
4578 def mr : AVX512PI<opc, MRMDestMem, (outs), (ins _.MemOp:$dst, _.RC:$src),
4579 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
4580 [(st_frag (_.VT _.RC:$src), addr:$dst)],
4581 _.ExeDomain>, EVEX, EVEX_CD8<_.EltSize, CD8VF>;
4584 multiclass avx512_movnt_vl<bits<8> opc, string OpcodeStr,
4585 AVX512VLVectorVTInfo VTInfo,
4586 X86SchedWriteMoveLSWidths Sched> {
4587 let Predicates = [HasAVX512] in
4588 defm Z : avx512_movnt<opc, OpcodeStr, VTInfo.info512, Sched.ZMM>, EVEX_V512;
4590 let Predicates = [HasAVX512, HasVLX] in {
4591 defm Z256 : avx512_movnt<opc, OpcodeStr, VTInfo.info256, Sched.YMM>, EVEX_V256;
4592 defm Z128 : avx512_movnt<opc, OpcodeStr, VTInfo.info128, Sched.XMM>, EVEX_V128;
4596 defm VMOVNTDQ : avx512_movnt_vl<0xE7, "vmovntdq", avx512vl_i64_info,
4597 SchedWriteVecMoveLSNT>, PD;
4598 defm VMOVNTPD : avx512_movnt_vl<0x2B, "vmovntpd", avx512vl_f64_info,
4599 SchedWriteFMoveLSNT>, PD, VEX_W;
4600 defm VMOVNTPS : avx512_movnt_vl<0x2B, "vmovntps", avx512vl_f32_info,
4601 SchedWriteFMoveLSNT>, PS;
4603 let Predicates = [HasAVX512], AddedComplexity = 400 in {
4604 def : Pat<(alignednontemporalstore (v16i32 VR512:$src), addr:$dst),
4605 (VMOVNTDQZmr addr:$dst, VR512:$src)>;
4606 def : Pat<(alignednontemporalstore (v32i16 VR512:$src), addr:$dst),
4607 (VMOVNTDQZmr addr:$dst, VR512:$src)>;
4608 def : Pat<(alignednontemporalstore (v64i8 VR512:$src), addr:$dst),
4609 (VMOVNTDQZmr addr:$dst, VR512:$src)>;
4611 def : Pat<(v8f64 (alignednontemporalload addr:$src)),
4612 (VMOVNTDQAZrm addr:$src)>;
4613 def : Pat<(v16f32 (alignednontemporalload addr:$src)),
4614 (VMOVNTDQAZrm addr:$src)>;
4615 def : Pat<(v8i64 (alignednontemporalload addr:$src)),
4616 (VMOVNTDQAZrm addr:$src)>;
4619 let Predicates = [HasVLX], AddedComplexity = 400 in {
4620 def : Pat<(alignednontemporalstore (v8i32 VR256X:$src), addr:$dst),
4621 (VMOVNTDQZ256mr addr:$dst, VR256X:$src)>;
4622 def : Pat<(alignednontemporalstore (v16i16 VR256X:$src), addr:$dst),
4623 (VMOVNTDQZ256mr addr:$dst, VR256X:$src)>;
4624 def : Pat<(alignednontemporalstore (v32i8 VR256X:$src), addr:$dst),
4625 (VMOVNTDQZ256mr addr:$dst, VR256X:$src)>;
4627 def : Pat<(v4f64 (alignednontemporalload addr:$src)),
4628 (VMOVNTDQAZ256rm addr:$src)>;
4629 def : Pat<(v8f32 (alignednontemporalload addr:$src)),
4630 (VMOVNTDQAZ256rm addr:$src)>;
4631 def : Pat<(v4i64 (alignednontemporalload addr:$src)),
4632 (VMOVNTDQAZ256rm addr:$src)>;
4634 def : Pat<(alignednontemporalstore (v4i32 VR128X:$src), addr:$dst),
4635 (VMOVNTDQZ128mr addr:$dst, VR128X:$src)>;
4636 def : Pat<(alignednontemporalstore (v8i16 VR128X:$src), addr:$dst),
4637 (VMOVNTDQZ128mr addr:$dst, VR128X:$src)>;
4638 def : Pat<(alignednontemporalstore (v16i8 VR128X:$src), addr:$dst),
4639 (VMOVNTDQZ128mr addr:$dst, VR128X:$src)>;
4641 def : Pat<(v2f64 (alignednontemporalload addr:$src)),
4642 (VMOVNTDQAZ128rm addr:$src)>;
4643 def : Pat<(v4f32 (alignednontemporalload addr:$src)),
4644 (VMOVNTDQAZ128rm addr:$src)>;
4645 def : Pat<(v2i64 (alignednontemporalload addr:$src)),
4646 (VMOVNTDQAZ128rm addr:$src)>;
4649 //===----------------------------------------------------------------------===//
4650 // AVX-512 - Integer arithmetic
4652 multiclass avx512_binop_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
4653 X86VectorVTInfo _, X86FoldableSchedWrite sched,
4654 bit IsCommutable = 0> {
4655 defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
4656 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,
4657 "$src2, $src1", "$src1, $src2",
4658 (_.VT (OpNode _.RC:$src1, _.RC:$src2)),
4659 IsCommutable>, AVX512BIBase, EVEX_4V,
4662 defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
4663 (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr,
4664 "$src2, $src1", "$src1, $src2",
4665 (_.VT (OpNode _.RC:$src1,
4666 (bitconvert (_.LdFrag addr:$src2))))>,
4667 AVX512BIBase, EVEX_4V,
4668 Sched<[sched.Folded, ReadAfterLd]>;
4671 multiclass avx512_binop_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode,
4672 X86VectorVTInfo _, X86FoldableSchedWrite sched,
4673 bit IsCommutable = 0> :
4674 avx512_binop_rm<opc, OpcodeStr, OpNode, _, sched, IsCommutable> {
4675 defm rmb : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
4676 (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr,
4677 "${src2}"##_.BroadcastStr##", $src1",
4678 "$src1, ${src2}"##_.BroadcastStr,
4679 (_.VT (OpNode _.RC:$src1,
4681 (_.ScalarLdFrag addr:$src2))))>,
4682 AVX512BIBase, EVEX_4V, EVEX_B,
4683 Sched<[sched.Folded, ReadAfterLd]>;
4686 multiclass avx512_binop_rm_vl<bits<8> opc, string OpcodeStr, SDNode OpNode,
4687 AVX512VLVectorVTInfo VTInfo,
4688 X86SchedWriteWidths sched, Predicate prd,
4689 bit IsCommutable = 0> {
4690 let Predicates = [prd] in
4691 defm Z : avx512_binop_rm<opc, OpcodeStr, OpNode, VTInfo.info512, sched.ZMM,
4692 IsCommutable>, EVEX_V512;
4694 let Predicates = [prd, HasVLX] in {
4695 defm Z256 : avx512_binop_rm<opc, OpcodeStr, OpNode, VTInfo.info256,
4696 sched.YMM, IsCommutable>, EVEX_V256;
4697 defm Z128 : avx512_binop_rm<opc, OpcodeStr, OpNode, VTInfo.info128,
4698 sched.XMM, IsCommutable>, EVEX_V128;
4702 multiclass avx512_binop_rmb_vl<bits<8> opc, string OpcodeStr, SDNode OpNode,
4703 AVX512VLVectorVTInfo VTInfo,
4704 X86SchedWriteWidths sched, Predicate prd,
4705 bit IsCommutable = 0> {
4706 let Predicates = [prd] in
4707 defm Z : avx512_binop_rmb<opc, OpcodeStr, OpNode, VTInfo.info512, sched.ZMM,
4708 IsCommutable>, EVEX_V512;
4710 let Predicates = [prd, HasVLX] in {
4711 defm Z256 : avx512_binop_rmb<opc, OpcodeStr, OpNode, VTInfo.info256,
4712 sched.YMM, IsCommutable>, EVEX_V256;
4713 defm Z128 : avx512_binop_rmb<opc, OpcodeStr, OpNode, VTInfo.info128,
4714 sched.XMM, IsCommutable>, EVEX_V128;
4718 multiclass avx512_binop_rm_vl_q<bits<8> opc, string OpcodeStr, SDNode OpNode,
4719 X86SchedWriteWidths sched, Predicate prd,
4720 bit IsCommutable = 0> {
4721 defm NAME : avx512_binop_rmb_vl<opc, OpcodeStr, OpNode, avx512vl_i64_info,
4722 sched, prd, IsCommutable>,
4723 VEX_W, EVEX_CD8<64, CD8VF>;
4726 multiclass avx512_binop_rm_vl_d<bits<8> opc, string OpcodeStr, SDNode OpNode,
4727 X86SchedWriteWidths sched, Predicate prd,
4728 bit IsCommutable = 0> {
4729 defm NAME : avx512_binop_rmb_vl<opc, OpcodeStr, OpNode, avx512vl_i32_info,
4730 sched, prd, IsCommutable>, EVEX_CD8<32, CD8VF>;
4733 multiclass avx512_binop_rm_vl_w<bits<8> opc, string OpcodeStr, SDNode OpNode,
4734 X86SchedWriteWidths sched, Predicate prd,
4735 bit IsCommutable = 0> {
4736 defm NAME : avx512_binop_rm_vl<opc, OpcodeStr, OpNode, avx512vl_i16_info,
4737 sched, prd, IsCommutable>, EVEX_CD8<16, CD8VF>,
4741 multiclass avx512_binop_rm_vl_b<bits<8> opc, string OpcodeStr, SDNode OpNode,
4742 X86SchedWriteWidths sched, Predicate prd,
4743 bit IsCommutable = 0> {
4744 defm NAME : avx512_binop_rm_vl<opc, OpcodeStr, OpNode, avx512vl_i8_info,
4745 sched, prd, IsCommutable>, EVEX_CD8<8, CD8VF>,
4749 multiclass avx512_binop_rm_vl_dq<bits<8> opc_d, bits<8> opc_q, string OpcodeStr,
4750 SDNode OpNode, X86SchedWriteWidths sched,
4751 Predicate prd, bit IsCommutable = 0> {
4752 defm Q : avx512_binop_rm_vl_q<opc_q, OpcodeStr#"q", OpNode, sched, prd,
4755 defm D : avx512_binop_rm_vl_d<opc_d, OpcodeStr#"d", OpNode, sched, prd,
4759 multiclass avx512_binop_rm_vl_bw<bits<8> opc_b, bits<8> opc_w, string OpcodeStr,
4760 SDNode OpNode, X86SchedWriteWidths sched,
4761 Predicate prd, bit IsCommutable = 0> {
4762 defm W : avx512_binop_rm_vl_w<opc_w, OpcodeStr#"w", OpNode, sched, prd,
4765 defm B : avx512_binop_rm_vl_b<opc_b, OpcodeStr#"b", OpNode, sched, prd,
4769 multiclass avx512_binop_rm_vl_all<bits<8> opc_b, bits<8> opc_w,
4770 bits<8> opc_d, bits<8> opc_q,
4771 string OpcodeStr, SDNode OpNode,
4772 X86SchedWriteWidths sched,
4773 bit IsCommutable = 0> {
4774 defm NAME : avx512_binop_rm_vl_dq<opc_d, opc_q, OpcodeStr, OpNode,
4775 sched, HasAVX512, IsCommutable>,
4776 avx512_binop_rm_vl_bw<opc_b, opc_w, OpcodeStr, OpNode,
4777 sched, HasBWI, IsCommutable>;
4780 multiclass avx512_binop_rm2<bits<8> opc, string OpcodeStr,
4781 X86FoldableSchedWrite sched,
4782 SDNode OpNode,X86VectorVTInfo _Src,
4783 X86VectorVTInfo _Dst, X86VectorVTInfo _Brdct,
4784 bit IsCommutable = 0> {
4785 defm rr : AVX512_maskable<opc, MRMSrcReg, _Dst, (outs _Dst.RC:$dst),
4786 (ins _Src.RC:$src1, _Src.RC:$src2), OpcodeStr,
4787 "$src2, $src1","$src1, $src2",
4789 (_Src.VT _Src.RC:$src1),
4790 (_Src.VT _Src.RC:$src2))),
4792 AVX512BIBase, EVEX_4V, Sched<[sched]>;
4793 defm rm : AVX512_maskable<opc, MRMSrcMem, _Dst, (outs _Dst.RC:$dst),
4794 (ins _Src.RC:$src1, _Src.MemOp:$src2), OpcodeStr,
4795 "$src2, $src1", "$src1, $src2",
4796 (_Dst.VT (OpNode (_Src.VT _Src.RC:$src1),
4797 (bitconvert (_Src.LdFrag addr:$src2))))>,
4798 AVX512BIBase, EVEX_4V,
4799 Sched<[sched.Folded, ReadAfterLd]>;
4801 defm rmb : AVX512_maskable<opc, MRMSrcMem, _Dst, (outs _Dst.RC:$dst),
4802 (ins _Src.RC:$src1, _Brdct.ScalarMemOp:$src2),
4804 "${src2}"##_Brdct.BroadcastStr##", $src1",
4805 "$src1, ${src2}"##_Brdct.BroadcastStr,
4806 (_Dst.VT (OpNode (_Src.VT _Src.RC:$src1), (bitconvert
4807 (_Brdct.VT (X86VBroadcast
4808 (_Brdct.ScalarLdFrag addr:$src2))))))>,
4809 AVX512BIBase, EVEX_4V, EVEX_B,
4810 Sched<[sched.Folded, ReadAfterLd]>;
4813 defm VPADD : avx512_binop_rm_vl_all<0xFC, 0xFD, 0xFE, 0xD4, "vpadd", add,
4814 SchedWriteVecALU, 1>;
4815 defm VPSUB : avx512_binop_rm_vl_all<0xF8, 0xF9, 0xFA, 0xFB, "vpsub", sub,
4816 SchedWriteVecALU, 0>;
4817 defm VPADDS : avx512_binop_rm_vl_bw<0xEC, 0xED, "vpadds", X86adds,
4818 SchedWriteVecALU, HasBWI, 1>;
4819 defm VPSUBS : avx512_binop_rm_vl_bw<0xE8, 0xE9, "vpsubs", X86subs,
4820 SchedWriteVecALU, HasBWI, 0>;
4821 defm VPADDUS : avx512_binop_rm_vl_bw<0xDC, 0xDD, "vpaddus", X86addus,
4822 SchedWriteVecALU, HasBWI, 1>;
4823 defm VPSUBUS : avx512_binop_rm_vl_bw<0xD8, 0xD9, "vpsubus", X86subus,
4824 SchedWriteVecALU, HasBWI, 0>;
4825 defm VPMULLD : avx512_binop_rm_vl_d<0x40, "vpmulld", mul,
4826 SchedWritePMULLD, HasAVX512, 1>, T8PD;
4827 defm VPMULLW : avx512_binop_rm_vl_w<0xD5, "vpmullw", mul,
4828 SchedWriteVecIMul, HasBWI, 1>;
4829 defm VPMULLQ : avx512_binop_rm_vl_q<0x40, "vpmullq", mul,
4830 SchedWriteVecIMul, HasDQI, 1>, T8PD,
4831 NotEVEX2VEXConvertible;
4832 defm VPMULHW : avx512_binop_rm_vl_w<0xE5, "vpmulhw", mulhs, SchedWriteVecIMul,
4834 defm VPMULHUW : avx512_binop_rm_vl_w<0xE4, "vpmulhuw", mulhu, SchedWriteVecIMul,
4836 defm VPMULHRSW : avx512_binop_rm_vl_w<0x0B, "vpmulhrsw", X86mulhrs,
4837 SchedWriteVecIMul, HasBWI, 1>, T8PD;
4838 defm VPAVG : avx512_binop_rm_vl_bw<0xE0, 0xE3, "vpavg", X86avg,
4839 SchedWriteVecALU, HasBWI, 1>;
4840 defm VPMULDQ : avx512_binop_rm_vl_q<0x28, "vpmuldq", X86pmuldq,
4841 SchedWriteVecIMul, HasAVX512, 1>, T8PD;
4842 defm VPMULUDQ : avx512_binop_rm_vl_q<0xF4, "vpmuludq", X86pmuludq,
4843 SchedWriteVecIMul, HasAVX512, 1>;
4845 multiclass avx512_binop_all<bits<8> opc, string OpcodeStr,
4846 X86SchedWriteWidths sched,
4847 AVX512VLVectorVTInfo _SrcVTInfo,
4848 AVX512VLVectorVTInfo _DstVTInfo,
4849 SDNode OpNode, Predicate prd, bit IsCommutable = 0> {
4850 let Predicates = [prd] in
4851 defm NAME#Z : avx512_binop_rm2<opc, OpcodeStr, sched.ZMM, OpNode,
4852 _SrcVTInfo.info512, _DstVTInfo.info512,
4853 v8i64_info, IsCommutable>,
4854 EVEX_V512, EVEX_CD8<64, CD8VF>, VEX_W;
4855 let Predicates = [HasVLX, prd] in {
4856 defm NAME#Z256 : avx512_binop_rm2<opc, OpcodeStr, sched.YMM, OpNode,
4857 _SrcVTInfo.info256, _DstVTInfo.info256,
4858 v4i64x_info, IsCommutable>,
4859 EVEX_V256, EVEX_CD8<64, CD8VF>, VEX_W;
4860 defm NAME#Z128 : avx512_binop_rm2<opc, OpcodeStr, sched.XMM, OpNode,
4861 _SrcVTInfo.info128, _DstVTInfo.info128,
4862 v2i64x_info, IsCommutable>,
4863 EVEX_V128, EVEX_CD8<64, CD8VF>, VEX_W;
4867 defm VPMULTISHIFTQB : avx512_binop_all<0x83, "vpmultishiftqb", SchedWriteVecALU,
4868 avx512vl_i8_info, avx512vl_i8_info,
4869 X86multishift, HasVBMI, 0>, T8PD;
4871 multiclass avx512_packs_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode,
4872 X86VectorVTInfo _Src, X86VectorVTInfo _Dst,
4873 X86FoldableSchedWrite sched> {
4874 defm rmb : AVX512_maskable<opc, MRMSrcMem, _Dst, (outs _Dst.RC:$dst),
4875 (ins _Src.RC:$src1, _Src.ScalarMemOp:$src2),
4877 "${src2}"##_Src.BroadcastStr##", $src1",
4878 "$src1, ${src2}"##_Src.BroadcastStr,
4879 (_Dst.VT (OpNode (_Src.VT _Src.RC:$src1), (bitconvert
4880 (_Src.VT (X86VBroadcast
4881 (_Src.ScalarLdFrag addr:$src2))))))>,
4882 EVEX_4V, EVEX_B, EVEX_CD8<_Src.EltSize, CD8VF>,
4883 Sched<[sched.Folded, ReadAfterLd]>;
4886 multiclass avx512_packs_rm<bits<8> opc, string OpcodeStr,
4887 SDNode OpNode,X86VectorVTInfo _Src,
4888 X86VectorVTInfo _Dst, X86FoldableSchedWrite sched,
4889 bit IsCommutable = 0> {
4890 defm rr : AVX512_maskable<opc, MRMSrcReg, _Dst, (outs _Dst.RC:$dst),
4891 (ins _Src.RC:$src1, _Src.RC:$src2), OpcodeStr,
4892 "$src2, $src1","$src1, $src2",
4894 (_Src.VT _Src.RC:$src1),
4895 (_Src.VT _Src.RC:$src2))),
4897 EVEX_CD8<_Src.EltSize, CD8VF>, EVEX_4V, Sched<[sched]>;
4898 defm rm : AVX512_maskable<opc, MRMSrcMem, _Dst, (outs _Dst.RC:$dst),
4899 (ins _Src.RC:$src1, _Src.MemOp:$src2), OpcodeStr,
4900 "$src2, $src1", "$src1, $src2",
4901 (_Dst.VT (OpNode (_Src.VT _Src.RC:$src1),
4902 (bitconvert (_Src.LdFrag addr:$src2))))>,
4903 EVEX_4V, EVEX_CD8<_Src.EltSize, CD8VF>,
4904 Sched<[sched.Folded, ReadAfterLd]>;
4907 multiclass avx512_packs_all_i32_i16<bits<8> opc, string OpcodeStr,
4909 let Predicates = [HasBWI] in
4910 defm NAME#Z : avx512_packs_rm<opc, OpcodeStr, OpNode, v16i32_info,
4911 v32i16_info, SchedWriteShuffle.ZMM>,
4912 avx512_packs_rmb<opc, OpcodeStr, OpNode, v16i32_info,
4913 v32i16_info, SchedWriteShuffle.ZMM>, EVEX_V512;
4914 let Predicates = [HasBWI, HasVLX] in {
4915 defm NAME#Z256 : avx512_packs_rm<opc, OpcodeStr, OpNode, v8i32x_info,
4916 v16i16x_info, SchedWriteShuffle.YMM>,
4917 avx512_packs_rmb<opc, OpcodeStr, OpNode, v8i32x_info,
4918 v16i16x_info, SchedWriteShuffle.YMM>,
4920 defm NAME#Z128 : avx512_packs_rm<opc, OpcodeStr, OpNode, v4i32x_info,
4921 v8i16x_info, SchedWriteShuffle.XMM>,
4922 avx512_packs_rmb<opc, OpcodeStr, OpNode, v4i32x_info,
4923 v8i16x_info, SchedWriteShuffle.XMM>,
4927 multiclass avx512_packs_all_i16_i8<bits<8> opc, string OpcodeStr,
4929 let Predicates = [HasBWI] in
4930 defm NAME#Z : avx512_packs_rm<opc, OpcodeStr, OpNode, v32i16_info, v64i8_info,
4931 SchedWriteShuffle.ZMM>, EVEX_V512, VEX_WIG;
4932 let Predicates = [HasBWI, HasVLX] in {
4933 defm NAME#Z256 : avx512_packs_rm<opc, OpcodeStr, OpNode, v16i16x_info,
4934 v32i8x_info, SchedWriteShuffle.YMM>,
4936 defm NAME#Z128 : avx512_packs_rm<opc, OpcodeStr, OpNode, v8i16x_info,
4937 v16i8x_info, SchedWriteShuffle.XMM>,
4942 multiclass avx512_vpmadd<bits<8> opc, string OpcodeStr,
4943 SDNode OpNode, AVX512VLVectorVTInfo _Src,
4944 AVX512VLVectorVTInfo _Dst, bit IsCommutable = 0> {
4945 let Predicates = [HasBWI] in
4946 defm NAME#Z : avx512_packs_rm<opc, OpcodeStr, OpNode, _Src.info512,
4947 _Dst.info512, SchedWriteVecIMul.ZMM,
4948 IsCommutable>, EVEX_V512;
4949 let Predicates = [HasBWI, HasVLX] in {
4950 defm NAME#Z256 : avx512_packs_rm<opc, OpcodeStr, OpNode, _Src.info256,
4951 _Dst.info256, SchedWriteVecIMul.YMM,
4952 IsCommutable>, EVEX_V256;
4953 defm NAME#Z128 : avx512_packs_rm<opc, OpcodeStr, OpNode, _Src.info128,
4954 _Dst.info128, SchedWriteVecIMul.XMM,
4955 IsCommutable>, EVEX_V128;
4959 defm VPACKSSDW : avx512_packs_all_i32_i16<0x6B, "vpackssdw", X86Packss>, AVX512BIBase;
4960 defm VPACKUSDW : avx512_packs_all_i32_i16<0x2b, "vpackusdw", X86Packus>, AVX5128IBase;
4961 defm VPACKSSWB : avx512_packs_all_i16_i8 <0x63, "vpacksswb", X86Packss>, AVX512BIBase;
4962 defm VPACKUSWB : avx512_packs_all_i16_i8 <0x67, "vpackuswb", X86Packus>, AVX512BIBase;
4964 defm VPMADDUBSW : avx512_vpmadd<0x04, "vpmaddubsw", X86vpmaddubsw,
4965 avx512vl_i8_info, avx512vl_i16_info>, AVX512BIBase, T8PD, VEX_WIG;
4966 defm VPMADDWD : avx512_vpmadd<0xF5, "vpmaddwd", X86vpmaddwd,
4967 avx512vl_i16_info, avx512vl_i32_info, 1>, AVX512BIBase, VEX_WIG;
4969 defm VPMAXSB : avx512_binop_rm_vl_b<0x3C, "vpmaxsb", smax,
4970 SchedWriteVecALU, HasBWI, 1>, T8PD;
4971 defm VPMAXSW : avx512_binop_rm_vl_w<0xEE, "vpmaxsw", smax,
4972 SchedWriteVecALU, HasBWI, 1>;
4973 defm VPMAXSD : avx512_binop_rm_vl_d<0x3D, "vpmaxsd", smax,
4974 SchedWriteVecALU, HasAVX512, 1>, T8PD;
4975 defm VPMAXSQ : avx512_binop_rm_vl_q<0x3D, "vpmaxsq", smax,
4976 SchedWriteVecALU, HasAVX512, 1>, T8PD,
4977 NotEVEX2VEXConvertible;
4979 defm VPMAXUB : avx512_binop_rm_vl_b<0xDE, "vpmaxub", umax,
4980 SchedWriteVecALU, HasBWI, 1>;
4981 defm VPMAXUW : avx512_binop_rm_vl_w<0x3E, "vpmaxuw", umax,
4982 SchedWriteVecALU, HasBWI, 1>, T8PD;
4983 defm VPMAXUD : avx512_binop_rm_vl_d<0x3F, "vpmaxud", umax,
4984 SchedWriteVecALU, HasAVX512, 1>, T8PD;
4985 defm VPMAXUQ : avx512_binop_rm_vl_q<0x3F, "vpmaxuq", umax,
4986 SchedWriteVecALU, HasAVX512, 1>, T8PD,
4987 NotEVEX2VEXConvertible;
4989 defm VPMINSB : avx512_binop_rm_vl_b<0x38, "vpminsb", smin,
4990 SchedWriteVecALU, HasBWI, 1>, T8PD;
4991 defm VPMINSW : avx512_binop_rm_vl_w<0xEA, "vpminsw", smin,
4992 SchedWriteVecALU, HasBWI, 1>;
4993 defm VPMINSD : avx512_binop_rm_vl_d<0x39, "vpminsd", smin,
4994 SchedWriteVecALU, HasAVX512, 1>, T8PD;
4995 defm VPMINSQ : avx512_binop_rm_vl_q<0x39, "vpminsq", smin,
4996 SchedWriteVecALU, HasAVX512, 1>, T8PD,
4997 NotEVEX2VEXConvertible;
4999 defm VPMINUB : avx512_binop_rm_vl_b<0xDA, "vpminub", umin,
5000 SchedWriteVecALU, HasBWI, 1>;
5001 defm VPMINUW : avx512_binop_rm_vl_w<0x3A, "vpminuw", umin,
5002 SchedWriteVecALU, HasBWI, 1>, T8PD;
5003 defm VPMINUD : avx512_binop_rm_vl_d<0x3B, "vpminud", umin,
5004 SchedWriteVecALU, HasAVX512, 1>, T8PD;
5005 defm VPMINUQ : avx512_binop_rm_vl_q<0x3B, "vpminuq", umin,
5006 SchedWriteVecALU, HasAVX512, 1>, T8PD,
5007 NotEVEX2VEXConvertible;
5009 // PMULLQ: Use 512bit version to implement 128/256 bit in case NoVLX.
5010 let Predicates = [HasDQI, NoVLX] in {
5011 def : Pat<(v4i64 (mul (v4i64 VR256X:$src1), (v4i64 VR256X:$src2))),
5014 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR256X:$src1, sub_ymm),
5015 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR256X:$src2, sub_ymm)),
5018 def : Pat<(v2i64 (mul (v2i64 VR128X:$src1), (v2i64 VR128X:$src2))),
5021 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR128X:$src1, sub_xmm),
5022 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR128X:$src2, sub_xmm)),
5026 // PMULLQ: Use 512bit version to implement 128/256 bit in case NoVLX.
5027 let Predicates = [HasDQI, NoVLX] in {
5028 def : Pat<(v4i64 (mul (v4i64 VR256X:$src1), (v4i64 VR256X:$src2))),
5031 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR256X:$src1, sub_ymm),
5032 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR256X:$src2, sub_ymm)),
5035 def : Pat<(v2i64 (mul (v2i64 VR128X:$src1), (v2i64 VR128X:$src2))),
5038 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR128X:$src1, sub_xmm),
5039 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR128X:$src2, sub_xmm)),
5043 multiclass avx512_min_max_lowering<Instruction Instr, SDNode OpNode> {
5044 def : Pat<(v4i64 (OpNode VR256X:$src1, VR256X:$src2)),
5047 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR256X:$src1, sub_ymm),
5048 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR256X:$src2, sub_ymm)),
5051 def : Pat<(v2i64 (OpNode VR128X:$src1, VR128X:$src2)),
5054 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR128X:$src1, sub_xmm),
5055 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR128X:$src2, sub_xmm)),
5059 let Predicates = [HasAVX512, NoVLX] in {
5060 defm : avx512_min_max_lowering<VPMAXUQZrr, umax>;
5061 defm : avx512_min_max_lowering<VPMINUQZrr, umin>;
5062 defm : avx512_min_max_lowering<VPMAXSQZrr, smax>;
5063 defm : avx512_min_max_lowering<VPMINSQZrr, smin>;
5066 //===----------------------------------------------------------------------===//
5067 // AVX-512 Logical Instructions
5068 //===----------------------------------------------------------------------===//
5070 // OpNodeMsk is the OpNode to use when element size is important. OpNode will
5071 // be set to null_frag for 32-bit elements.
5072 multiclass avx512_logic_rm<bits<8> opc, string OpcodeStr,
5073 SDPatternOperator OpNode,
5074 SDNode OpNodeMsk, X86FoldableSchedWrite sched,
5075 X86VectorVTInfo _, bit IsCommutable = 0> {
5076 let hasSideEffects = 0 in
5077 defm rr : AVX512_maskable_logic<opc, MRMSrcReg, _, (outs _.RC:$dst),
5078 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,
5079 "$src2, $src1", "$src1, $src2",
5080 (_.i64VT (OpNode (bitconvert (_.VT _.RC:$src1)),
5081 (bitconvert (_.VT _.RC:$src2)))),
5082 (_.VT (bitconvert (_.i64VT (OpNodeMsk _.RC:$src1,
5084 IsCommutable>, AVX512BIBase, EVEX_4V,
5087 let hasSideEffects = 0, mayLoad = 1 in
5088 defm rm : AVX512_maskable_logic<opc, MRMSrcMem, _, (outs _.RC:$dst),
5089 (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr,
5090 "$src2, $src1", "$src1, $src2",
5091 (_.i64VT (OpNode (bitconvert (_.VT _.RC:$src1)),
5092 (bitconvert (_.LdFrag addr:$src2)))),
5093 (_.VT (bitconvert (_.i64VT (OpNodeMsk _.RC:$src1,
5094 (bitconvert (_.LdFrag addr:$src2))))))>,
5095 AVX512BIBase, EVEX_4V,
5096 Sched<[sched.Folded, ReadAfterLd]>;
5099 // OpNodeMsk is the OpNode to use where element size is important. So use
5100 // for all of the broadcast patterns.
5101 multiclass avx512_logic_rmb<bits<8> opc, string OpcodeStr,
5102 SDPatternOperator OpNode,
5103 SDNode OpNodeMsk, X86FoldableSchedWrite sched, X86VectorVTInfo _,
5104 bit IsCommutable = 0> :
5105 avx512_logic_rm<opc, OpcodeStr, OpNode, OpNodeMsk, sched, _,
5107 defm rmb : AVX512_maskable_logic<opc, MRMSrcMem, _, (outs _.RC:$dst),
5108 (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr,
5109 "${src2}"##_.BroadcastStr##", $src1",
5110 "$src1, ${src2}"##_.BroadcastStr,
5111 (_.i64VT (OpNodeMsk _.RC:$src1,
5113 (_.VT (X86VBroadcast
5114 (_.ScalarLdFrag addr:$src2)))))),
5115 (_.VT (bitconvert (_.i64VT (OpNodeMsk _.RC:$src1,
5117 (_.VT (X86VBroadcast
5118 (_.ScalarLdFrag addr:$src2))))))))>,
5119 AVX512BIBase, EVEX_4V, EVEX_B,
5120 Sched<[sched.Folded, ReadAfterLd]>;
5123 multiclass avx512_logic_rmb_vl<bits<8> opc, string OpcodeStr,
5124 SDPatternOperator OpNode,
5125 SDNode OpNodeMsk, X86SchedWriteWidths sched,
5126 AVX512VLVectorVTInfo VTInfo,
5127 bit IsCommutable = 0> {
5128 let Predicates = [HasAVX512] in
5129 defm Z : avx512_logic_rmb<opc, OpcodeStr, OpNode, OpNodeMsk, sched.ZMM,
5130 VTInfo.info512, IsCommutable>, EVEX_V512;
5132 let Predicates = [HasAVX512, HasVLX] in {
5133 defm Z256 : avx512_logic_rmb<opc, OpcodeStr, OpNode, OpNodeMsk, sched.YMM,
5134 VTInfo.info256, IsCommutable>, EVEX_V256;
5135 defm Z128 : avx512_logic_rmb<opc, OpcodeStr, OpNode, OpNodeMsk, sched.XMM,
5136 VTInfo.info128, IsCommutable>, EVEX_V128;
5140 multiclass avx512_logic_rm_vl_dq<bits<8> opc_d, bits<8> opc_q, string OpcodeStr,
5141 SDNode OpNode, X86SchedWriteWidths sched,
5142 bit IsCommutable = 0> {
5143 defm Q : avx512_logic_rmb_vl<opc_q, OpcodeStr#"q", OpNode, OpNode, sched,
5144 avx512vl_i64_info, IsCommutable>,
5145 VEX_W, EVEX_CD8<64, CD8VF>;
5146 defm D : avx512_logic_rmb_vl<opc_d, OpcodeStr#"d", null_frag, OpNode, sched,
5147 avx512vl_i32_info, IsCommutable>,
5148 EVEX_CD8<32, CD8VF>;
5151 defm VPAND : avx512_logic_rm_vl_dq<0xDB, 0xDB, "vpand", and,
5152 SchedWriteVecLogic, 1>;
5153 defm VPOR : avx512_logic_rm_vl_dq<0xEB, 0xEB, "vpor", or,
5154 SchedWriteVecLogic, 1>;
5155 defm VPXOR : avx512_logic_rm_vl_dq<0xEF, 0xEF, "vpxor", xor,
5156 SchedWriteVecLogic, 1>;
5157 defm VPANDN : avx512_logic_rm_vl_dq<0xDF, 0xDF, "vpandn", X86andnp,
5158 SchedWriteVecLogic>;
5160 //===----------------------------------------------------------------------===//
5161 // AVX-512 FP arithmetic
5162 //===----------------------------------------------------------------------===//
5164 multiclass avx512_fp_scalar<bits<8> opc, string OpcodeStr,X86VectorVTInfo _,
5165 SDNode OpNode, SDNode VecNode,
5166 X86FoldableSchedWrite sched, bit IsCommutable> {
5167 let ExeDomain = _.ExeDomain in {
5168 defm rr_Int : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
5169 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,
5170 "$src2, $src1", "$src1, $src2",
5171 (_.VT (VecNode _.RC:$src1, _.RC:$src2,
5172 (i32 FROUND_CURRENT)))>,
5175 defm rm_Int : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
5176 (ins _.RC:$src1, _.IntScalarMemOp:$src2), OpcodeStr,
5177 "$src2, $src1", "$src1, $src2",
5178 (_.VT (VecNode _.RC:$src1,
5179 _.ScalarIntMemCPat:$src2,
5180 (i32 FROUND_CURRENT)))>,
5181 Sched<[sched.Folded, ReadAfterLd]>;
5182 let isCodeGenOnly = 1, Predicates = [HasAVX512] in {
5183 def rr : I< opc, MRMSrcReg, (outs _.FRC:$dst),
5184 (ins _.FRC:$src1, _.FRC:$src2),
5185 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
5186 [(set _.FRC:$dst, (OpNode _.FRC:$src1, _.FRC:$src2))]>,
5188 let isCommutable = IsCommutable;
5190 def rm : I< opc, MRMSrcMem, (outs _.FRC:$dst),
5191 (ins _.FRC:$src1, _.ScalarMemOp:$src2),
5192 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
5193 [(set _.FRC:$dst, (OpNode _.FRC:$src1,
5194 (_.ScalarLdFrag addr:$src2)))]>,
5195 Sched<[sched.Folded, ReadAfterLd]>;
5200 multiclass avx512_fp_scalar_round<bits<8> opc, string OpcodeStr,X86VectorVTInfo _,
5201 SDNode VecNode, X86FoldableSchedWrite sched,
5202 bit IsCommutable = 0> {
5203 let ExeDomain = _.ExeDomain in
5204 defm rrb_Int : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
5205 (ins _.RC:$src1, _.RC:$src2, AVX512RC:$rc), OpcodeStr,
5206 "$rc, $src2, $src1", "$src1, $src2, $rc",
5207 (VecNode (_.VT _.RC:$src1), (_.VT _.RC:$src2),
5208 (i32 imm:$rc)), IsCommutable>,
5209 EVEX_B, EVEX_RC, Sched<[sched]>;
5211 multiclass avx512_fp_scalar_sae<bits<8> opc, string OpcodeStr,X86VectorVTInfo _,
5212 SDNode OpNode, SDNode VecNode, SDNode SaeNode,
5213 X86FoldableSchedWrite sched, bit IsCommutable> {
5214 let ExeDomain = _.ExeDomain in {
5215 defm rr_Int : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
5216 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,
5217 "$src2, $src1", "$src1, $src2",
5218 (_.VT (VecNode _.RC:$src1, _.RC:$src2))>,
5221 defm rm_Int : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
5222 (ins _.RC:$src1, _.IntScalarMemOp:$src2), OpcodeStr,
5223 "$src2, $src1", "$src1, $src2",
5224 (_.VT (VecNode _.RC:$src1,
5225 _.ScalarIntMemCPat:$src2))>,
5226 Sched<[sched.Folded, ReadAfterLd]>;
5228 let isCodeGenOnly = 1, Predicates = [HasAVX512] in {
5229 def rr : I< opc, MRMSrcReg, (outs _.FRC:$dst),
5230 (ins _.FRC:$src1, _.FRC:$src2),
5231 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
5232 [(set _.FRC:$dst, (OpNode _.FRC:$src1, _.FRC:$src2))]>,
5234 let isCommutable = IsCommutable;
5236 def rm : I< opc, MRMSrcMem, (outs _.FRC:$dst),
5237 (ins _.FRC:$src1, _.ScalarMemOp:$src2),
5238 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
5239 [(set _.FRC:$dst, (OpNode _.FRC:$src1,
5240 (_.ScalarLdFrag addr:$src2)))]>,
5241 Sched<[sched.Folded, ReadAfterLd]>;
5244 defm rrb_Int : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
5245 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,
5246 "{sae}, $src2, $src1", "$src1, $src2, {sae}",
5247 (SaeNode (_.VT _.RC:$src1), (_.VT _.RC:$src2),
5248 (i32 FROUND_NO_EXC))>, EVEX_B,
5253 multiclass avx512_binop_s_round<bits<8> opc, string OpcodeStr, SDNode OpNode,
5254 SDNode VecNode, X86SchedWriteSizes sched,
5256 defm SSZ : avx512_fp_scalar<opc, OpcodeStr#"ss", f32x_info, OpNode, VecNode,
5257 sched.PS.Scl, IsCommutable>,
5258 avx512_fp_scalar_round<opc, OpcodeStr#"ss", f32x_info, VecNode,
5259 sched.PS.Scl, IsCommutable>,
5260 XS, EVEX_4V, VEX_LIG, EVEX_CD8<32, CD8VT1>;
5261 defm SDZ : avx512_fp_scalar<opc, OpcodeStr#"sd", f64x_info, OpNode, VecNode,
5262 sched.PD.Scl, IsCommutable>,
5263 avx512_fp_scalar_round<opc, OpcodeStr#"sd", f64x_info, VecNode,
5264 sched.PD.Scl, IsCommutable>,
5265 XD, VEX_W, EVEX_4V, VEX_LIG, EVEX_CD8<64, CD8VT1>;
5268 multiclass avx512_binop_s_sae<bits<8> opc, string OpcodeStr, SDNode OpNode,
5269 SDNode VecNode, SDNode SaeNode,
5270 X86SchedWriteSizes sched, bit IsCommutable> {
5271 defm SSZ : avx512_fp_scalar_sae<opc, OpcodeStr#"ss", f32x_info, OpNode,
5272 VecNode, SaeNode, sched.PS.Scl, IsCommutable>,
5273 XS, EVEX_4V, VEX_LIG, EVEX_CD8<32, CD8VT1>;
5274 defm SDZ : avx512_fp_scalar_sae<opc, OpcodeStr#"sd", f64x_info, OpNode,
5275 VecNode, SaeNode, sched.PD.Scl, IsCommutable>,
5276 XD, VEX_W, EVEX_4V, VEX_LIG, EVEX_CD8<64, CD8VT1>;
5278 defm VADD : avx512_binop_s_round<0x58, "vadd", fadd, X86faddRnds,
5279 SchedWriteFAddSizes, 1>;
5280 defm VMUL : avx512_binop_s_round<0x59, "vmul", fmul, X86fmulRnds,
5281 SchedWriteFMulSizes, 1>;
5282 defm VSUB : avx512_binop_s_round<0x5C, "vsub", fsub, X86fsubRnds,
5283 SchedWriteFAddSizes, 0>;
5284 defm VDIV : avx512_binop_s_round<0x5E, "vdiv", fdiv, X86fdivRnds,
5285 SchedWriteFDivSizes, 0>;
5286 defm VMIN : avx512_binop_s_sae<0x5D, "vmin", X86fmin, X86fmins, X86fminRnds,
5287 SchedWriteFCmpSizes, 0>;
5288 defm VMAX : avx512_binop_s_sae<0x5F, "vmax", X86fmax, X86fmaxs, X86fmaxRnds,
5289 SchedWriteFCmpSizes, 0>;
5291 // MIN/MAX nodes are commutable under "unsafe-fp-math". In this case we use
5292 // X86fminc and X86fmaxc instead of X86fmin and X86fmax
5293 multiclass avx512_comutable_binop_s<bits<8> opc, string OpcodeStr,
5294 X86VectorVTInfo _, SDNode OpNode,
5295 X86FoldableSchedWrite sched> {
5296 let isCodeGenOnly = 1, Predicates = [HasAVX512], ExeDomain = _.ExeDomain in {
5297 def rr : I< opc, MRMSrcReg, (outs _.FRC:$dst),
5298 (ins _.FRC:$src1, _.FRC:$src2),
5299 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
5300 [(set _.FRC:$dst, (OpNode _.FRC:$src1, _.FRC:$src2))]>,
5302 let isCommutable = 1;
5304 def rm : I< opc, MRMSrcMem, (outs _.FRC:$dst),
5305 (ins _.FRC:$src1, _.ScalarMemOp:$src2),
5306 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
5307 [(set _.FRC:$dst, (OpNode _.FRC:$src1,
5308 (_.ScalarLdFrag addr:$src2)))]>,
5309 Sched<[sched.Folded, ReadAfterLd]>;
5312 defm VMINCSSZ : avx512_comutable_binop_s<0x5D, "vminss", f32x_info, X86fminc,
5313 SchedWriteFCmp.Scl>, XS, EVEX_4V,
5314 VEX_LIG, EVEX_CD8<32, CD8VT1>;
5316 defm VMINCSDZ : avx512_comutable_binop_s<0x5D, "vminsd", f64x_info, X86fminc,
5317 SchedWriteFCmp.Scl>, XD, VEX_W, EVEX_4V,
5318 VEX_LIG, EVEX_CD8<64, CD8VT1>;
5320 defm VMAXCSSZ : avx512_comutable_binop_s<0x5F, "vmaxss", f32x_info, X86fmaxc,
5321 SchedWriteFCmp.Scl>, XS, EVEX_4V,
5322 VEX_LIG, EVEX_CD8<32, CD8VT1>;
5324 defm VMAXCSDZ : avx512_comutable_binop_s<0x5F, "vmaxsd", f64x_info, X86fmaxc,
5325 SchedWriteFCmp.Scl>, XD, VEX_W, EVEX_4V,
5326 VEX_LIG, EVEX_CD8<64, CD8VT1>;
5328 multiclass avx512_fp_packed<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode,
5329 X86VectorVTInfo _, X86FoldableSchedWrite sched,
5331 let ExeDomain = _.ExeDomain, hasSideEffects = 0 in {
5332 defm rr: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
5333 (ins _.RC:$src1, _.RC:$src2), OpcodeStr##_.Suffix,
5334 "$src2, $src1", "$src1, $src2",
5335 (_.VT (OpNode _.RC:$src1, _.RC:$src2)), IsCommutable>,
5336 EVEX_4V, Sched<[sched]>;
5337 let mayLoad = 1 in {
5338 defm rm: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
5339 (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr##_.Suffix,
5340 "$src2, $src1", "$src1, $src2",
5341 (OpNode _.RC:$src1, (_.LdFrag addr:$src2))>,
5342 EVEX_4V, Sched<[sched.Folded, ReadAfterLd]>;
5343 defm rmb: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
5344 (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr##_.Suffix,
5345 "${src2}"##_.BroadcastStr##", $src1",
5346 "$src1, ${src2}"##_.BroadcastStr,
5347 (OpNode _.RC:$src1, (_.VT (X86VBroadcast
5348 (_.ScalarLdFrag addr:$src2))))>,
5350 Sched<[sched.Folded, ReadAfterLd]>;
5355 multiclass avx512_fp_round_packed<bits<8> opc, string OpcodeStr,
5356 SDPatternOperator OpNodeRnd,
5357 X86FoldableSchedWrite sched, X86VectorVTInfo _> {
5358 let ExeDomain = _.ExeDomain in
5359 defm rrb: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
5360 (ins _.RC:$src1, _.RC:$src2, AVX512RC:$rc), OpcodeStr##_.Suffix,
5361 "$rc, $src2, $src1", "$src1, $src2, $rc",
5362 (_.VT (OpNodeRnd _.RC:$src1, _.RC:$src2, (i32 imm:$rc)))>,
5363 EVEX_4V, EVEX_B, EVEX_RC, Sched<[sched]>;
5366 multiclass avx512_fp_sae_packed<bits<8> opc, string OpcodeStr,
5367 SDPatternOperator OpNodeRnd,
5368 X86FoldableSchedWrite sched, X86VectorVTInfo _> {
5369 let ExeDomain = _.ExeDomain in
5370 defm rrb: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
5371 (ins _.RC:$src1, _.RC:$src2), OpcodeStr##_.Suffix,
5372 "{sae}, $src2, $src1", "$src1, $src2, {sae}",
5373 (_.VT (OpNodeRnd _.RC:$src1, _.RC:$src2, (i32 FROUND_NO_EXC)))>,
5374 EVEX_4V, EVEX_B, Sched<[sched]>;
5377 multiclass avx512_fp_binop_p<bits<8> opc, string OpcodeStr, SDPatternOperator OpNode,
5378 Predicate prd, X86SchedWriteSizes sched,
5379 bit IsCommutable = 0> {
5380 let Predicates = [prd] in {
5381 defm PSZ : avx512_fp_packed<opc, OpcodeStr, OpNode, v16f32_info,
5382 sched.PS.ZMM, IsCommutable>, EVEX_V512, PS,
5383 EVEX_CD8<32, CD8VF>;
5384 defm PDZ : avx512_fp_packed<opc, OpcodeStr, OpNode, v8f64_info,
5385 sched.PD.ZMM, IsCommutable>, EVEX_V512, PD, VEX_W,
5386 EVEX_CD8<64, CD8VF>;
5389 // Define only if AVX512VL feature is present.
5390 let Predicates = [prd, HasVLX] in {
5391 defm PSZ128 : avx512_fp_packed<opc, OpcodeStr, OpNode, v4f32x_info,
5392 sched.PS.XMM, IsCommutable>, EVEX_V128, PS,
5393 EVEX_CD8<32, CD8VF>;
5394 defm PSZ256 : avx512_fp_packed<opc, OpcodeStr, OpNode, v8f32x_info,
5395 sched.PS.YMM, IsCommutable>, EVEX_V256, PS,
5396 EVEX_CD8<32, CD8VF>;
5397 defm PDZ128 : avx512_fp_packed<opc, OpcodeStr, OpNode, v2f64x_info,
5398 sched.PD.XMM, IsCommutable>, EVEX_V128, PD, VEX_W,
5399 EVEX_CD8<64, CD8VF>;
5400 defm PDZ256 : avx512_fp_packed<opc, OpcodeStr, OpNode, v4f64x_info,
5401 sched.PD.YMM, IsCommutable>, EVEX_V256, PD, VEX_W,
5402 EVEX_CD8<64, CD8VF>;
5406 multiclass avx512_fp_binop_p_round<bits<8> opc, string OpcodeStr, SDNode OpNodeRnd,
5407 X86SchedWriteSizes sched> {
5408 defm PSZ : avx512_fp_round_packed<opc, OpcodeStr, OpNodeRnd, sched.PS.ZMM,
5410 EVEX_V512, PS, EVEX_CD8<32, CD8VF>;
5411 defm PDZ : avx512_fp_round_packed<opc, OpcodeStr, OpNodeRnd, sched.PD.ZMM,
5413 EVEX_V512, PD, VEX_W,EVEX_CD8<64, CD8VF>;
5416 multiclass avx512_fp_binop_p_sae<bits<8> opc, string OpcodeStr, SDNode OpNodeRnd,
5417 X86SchedWriteSizes sched> {
5418 defm PSZ : avx512_fp_sae_packed<opc, OpcodeStr, OpNodeRnd, sched.PS.ZMM,
5420 EVEX_V512, PS, EVEX_CD8<32, CD8VF>;
5421 defm PDZ : avx512_fp_sae_packed<opc, OpcodeStr, OpNodeRnd, sched.PD.ZMM,
5423 EVEX_V512, PD, VEX_W,EVEX_CD8<64, CD8VF>;
5426 defm VADD : avx512_fp_binop_p<0x58, "vadd", fadd, HasAVX512,
5427 SchedWriteFAddSizes, 1>,
5428 avx512_fp_binop_p_round<0x58, "vadd", X86faddRnd, SchedWriteFAddSizes>;
5429 defm VMUL : avx512_fp_binop_p<0x59, "vmul", fmul, HasAVX512,
5430 SchedWriteFMulSizes, 1>,
5431 avx512_fp_binop_p_round<0x59, "vmul", X86fmulRnd, SchedWriteFMulSizes>;
5432 defm VSUB : avx512_fp_binop_p<0x5C, "vsub", fsub, HasAVX512,
5433 SchedWriteFAddSizes>,
5434 avx512_fp_binop_p_round<0x5C, "vsub", X86fsubRnd, SchedWriteFAddSizes>;
5435 defm VDIV : avx512_fp_binop_p<0x5E, "vdiv", fdiv, HasAVX512,
5436 SchedWriteFDivSizes>,
5437 avx512_fp_binop_p_round<0x5E, "vdiv", X86fdivRnd, SchedWriteFDivSizes>;
5438 defm VMIN : avx512_fp_binop_p<0x5D, "vmin", X86fmin, HasAVX512,
5439 SchedWriteFCmpSizes, 0>,
5440 avx512_fp_binop_p_sae<0x5D, "vmin", X86fminRnd, SchedWriteFCmpSizes>;
5441 defm VMAX : avx512_fp_binop_p<0x5F, "vmax", X86fmax, HasAVX512,
5442 SchedWriteFCmpSizes, 0>,
5443 avx512_fp_binop_p_sae<0x5F, "vmax", X86fmaxRnd, SchedWriteFCmpSizes>;
5444 let isCodeGenOnly = 1 in {
5445 defm VMINC : avx512_fp_binop_p<0x5D, "vmin", X86fminc, HasAVX512,
5446 SchedWriteFCmpSizes, 1>;
5447 defm VMAXC : avx512_fp_binop_p<0x5F, "vmax", X86fmaxc, HasAVX512,
5448 SchedWriteFCmpSizes, 1>;
5450 defm VAND : avx512_fp_binop_p<0x54, "vand", null_frag, HasDQI,
5451 SchedWriteFLogicSizes, 1>;
5452 defm VANDN : avx512_fp_binop_p<0x55, "vandn", null_frag, HasDQI,
5453 SchedWriteFLogicSizes, 0>;
5454 defm VOR : avx512_fp_binop_p<0x56, "vor", null_frag, HasDQI,
5455 SchedWriteFLogicSizes, 1>;
5456 defm VXOR : avx512_fp_binop_p<0x57, "vxor", null_frag, HasDQI,
5457 SchedWriteFLogicSizes, 1>;
5459 // Patterns catch floating point selects with bitcasted integer logic ops.
5460 multiclass avx512_fp_logical_lowering<string InstrStr, SDNode OpNode,
5461 X86VectorVTInfo _, Predicate prd> {
5462 let Predicates = [prd] in {
5463 // Masked register-register logical operations.
5464 def : Pat<(_.VT (vselect _.KRCWM:$mask,
5465 (bitconvert (_.i64VT (OpNode _.RC:$src1, _.RC:$src2))),
5467 (!cast<Instruction>(InstrStr#rrk) _.RC:$src0, _.KRCWM:$mask,
5468 _.RC:$src1, _.RC:$src2)>;
5469 def : Pat<(_.VT (vselect _.KRCWM:$mask,
5470 (bitconvert (_.i64VT (OpNode _.RC:$src1, _.RC:$src2))),
5472 (!cast<Instruction>(InstrStr#rrkz) _.KRCWM:$mask, _.RC:$src1,
5474 // Masked register-memory logical operations.
5475 def : Pat<(_.VT (vselect _.KRCWM:$mask,
5476 (bitconvert (_.i64VT (OpNode _.RC:$src1,
5477 (load addr:$src2)))),
5479 (!cast<Instruction>(InstrStr#rmk) _.RC:$src0, _.KRCWM:$mask,
5480 _.RC:$src1, addr:$src2)>;
5481 def : Pat<(_.VT (vselect _.KRCWM:$mask,
5482 (bitconvert (_.i64VT (OpNode _.RC:$src1, (load addr:$src2)))),
5484 (!cast<Instruction>(InstrStr#rmkz) _.KRCWM:$mask, _.RC:$src1,
5486 // Register-broadcast logical operations.
5487 def : Pat<(_.i64VT (OpNode _.RC:$src1,
5488 (bitconvert (_.VT (X86VBroadcast
5489 (_.ScalarLdFrag addr:$src2)))))),
5490 (!cast<Instruction>(InstrStr#rmb) _.RC:$src1, addr:$src2)>;
5491 def : Pat<(_.VT (vselect _.KRCWM:$mask,
5493 (_.i64VT (OpNode _.RC:$src1,
5496 (_.ScalarLdFrag addr:$src2))))))),
5498 (!cast<Instruction>(InstrStr#rmbk) _.RC:$src0, _.KRCWM:$mask,
5499 _.RC:$src1, addr:$src2)>;
5500 def : Pat<(_.VT (vselect _.KRCWM:$mask,
5502 (_.i64VT (OpNode _.RC:$src1,
5505 (_.ScalarLdFrag addr:$src2))))))),
5507 (!cast<Instruction>(InstrStr#rmbkz) _.KRCWM:$mask,
5508 _.RC:$src1, addr:$src2)>;
5512 multiclass avx512_fp_logical_lowering_sizes<string InstrStr, SDNode OpNode> {
5513 defm : avx512_fp_logical_lowering<InstrStr#DZ128, OpNode, v4f32x_info, HasVLX>;
5514 defm : avx512_fp_logical_lowering<InstrStr#QZ128, OpNode, v2f64x_info, HasVLX>;
5515 defm : avx512_fp_logical_lowering<InstrStr#DZ256, OpNode, v8f32x_info, HasVLX>;
5516 defm : avx512_fp_logical_lowering<InstrStr#QZ256, OpNode, v4f64x_info, HasVLX>;
5517 defm : avx512_fp_logical_lowering<InstrStr#DZ, OpNode, v16f32_info, HasAVX512>;
5518 defm : avx512_fp_logical_lowering<InstrStr#QZ, OpNode, v8f64_info, HasAVX512>;
5521 defm : avx512_fp_logical_lowering_sizes<"VPAND", and>;
5522 defm : avx512_fp_logical_lowering_sizes<"VPOR", or>;
5523 defm : avx512_fp_logical_lowering_sizes<"VPXOR", xor>;
5524 defm : avx512_fp_logical_lowering_sizes<"VPANDN", X86andnp>;
5526 let Predicates = [HasVLX,HasDQI] in {
5527 // Use packed logical operations for scalar ops.
5528 def : Pat<(f64 (X86fand FR64X:$src1, FR64X:$src2)),
5529 (COPY_TO_REGCLASS (VANDPDZ128rr
5530 (COPY_TO_REGCLASS FR64X:$src1, VR128X),
5531 (COPY_TO_REGCLASS FR64X:$src2, VR128X)), FR64X)>;
5532 def : Pat<(f64 (X86for FR64X:$src1, FR64X:$src2)),
5533 (COPY_TO_REGCLASS (VORPDZ128rr
5534 (COPY_TO_REGCLASS FR64X:$src1, VR128X),
5535 (COPY_TO_REGCLASS FR64X:$src2, VR128X)), FR64X)>;
5536 def : Pat<(f64 (X86fxor FR64X:$src1, FR64X:$src2)),
5537 (COPY_TO_REGCLASS (VXORPDZ128rr
5538 (COPY_TO_REGCLASS FR64X:$src1, VR128X),
5539 (COPY_TO_REGCLASS FR64X:$src2, VR128X)), FR64X)>;
5540 def : Pat<(f64 (X86fandn FR64X:$src1, FR64X:$src2)),
5541 (COPY_TO_REGCLASS (VANDNPDZ128rr
5542 (COPY_TO_REGCLASS FR64X:$src1, VR128X),
5543 (COPY_TO_REGCLASS FR64X:$src2, VR128X)), FR64X)>;
5545 def : Pat<(f32 (X86fand FR32X:$src1, FR32X:$src2)),
5546 (COPY_TO_REGCLASS (VANDPSZ128rr
5547 (COPY_TO_REGCLASS FR32X:$src1, VR128X),
5548 (COPY_TO_REGCLASS FR32X:$src2, VR128X)), FR32X)>;
5549 def : Pat<(f32 (X86for FR32X:$src1, FR32X:$src2)),
5550 (COPY_TO_REGCLASS (VORPSZ128rr
5551 (COPY_TO_REGCLASS FR32X:$src1, VR128X),
5552 (COPY_TO_REGCLASS FR32X:$src2, VR128X)), FR32X)>;
5553 def : Pat<(f32 (X86fxor FR32X:$src1, FR32X:$src2)),
5554 (COPY_TO_REGCLASS (VXORPSZ128rr
5555 (COPY_TO_REGCLASS FR32X:$src1, VR128X),
5556 (COPY_TO_REGCLASS FR32X:$src2, VR128X)), FR32X)>;
5557 def : Pat<(f32 (X86fandn FR32X:$src1, FR32X:$src2)),
5558 (COPY_TO_REGCLASS (VANDNPSZ128rr
5559 (COPY_TO_REGCLASS FR32X:$src1, VR128X),
5560 (COPY_TO_REGCLASS FR32X:$src2, VR128X)), FR32X)>;
5563 multiclass avx512_fp_scalef_p<bits<8> opc, string OpcodeStr, SDNode OpNode,
5564 X86FoldableSchedWrite sched, X86VectorVTInfo _> {
5565 let ExeDomain = _.ExeDomain in {
5566 defm rr: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
5567 (ins _.RC:$src1, _.RC:$src2), OpcodeStr##_.Suffix,
5568 "$src2, $src1", "$src1, $src2",
5569 (_.VT (OpNode _.RC:$src1, _.RC:$src2, (i32 FROUND_CURRENT)))>,
5570 EVEX_4V, Sched<[sched]>;
5571 defm rm: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
5572 (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr##_.Suffix,
5573 "$src2, $src1", "$src1, $src2",
5574 (OpNode _.RC:$src1, (_.LdFrag addr:$src2), (i32 FROUND_CURRENT))>,
5575 EVEX_4V, Sched<[sched.Folded, ReadAfterLd]>;
5576 defm rmb: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
5577 (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr##_.Suffix,
5578 "${src2}"##_.BroadcastStr##", $src1",
5579 "$src1, ${src2}"##_.BroadcastStr,
5580 (OpNode _.RC:$src1, (_.VT (X86VBroadcast
5581 (_.ScalarLdFrag addr:$src2))),
5582 (i32 FROUND_CURRENT))>,
5583 EVEX_4V, EVEX_B, Sched<[sched.Folded, ReadAfterLd]>;
5587 multiclass avx512_fp_scalef_scalar<bits<8> opc, string OpcodeStr, SDNode OpNode,
5588 X86FoldableSchedWrite sched, X86VectorVTInfo _> {
5589 let ExeDomain = _.ExeDomain in {
5590 defm rr: AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
5591 (ins _.RC:$src1, _.RC:$src2), OpcodeStr##_.Suffix,
5592 "$src2, $src1", "$src1, $src2",
5593 (_.VT (OpNode _.RC:$src1, _.RC:$src2, (i32 FROUND_CURRENT)))>,
5595 defm rm: AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
5596 (ins _.RC:$src1, _.IntScalarMemOp:$src2), OpcodeStr##_.Suffix,
5597 "$src2, $src1", "$src1, $src2",
5598 (OpNode _.RC:$src1, _.ScalarIntMemCPat:$src2,
5599 (i32 FROUND_CURRENT))>,
5600 Sched<[sched.Folded, ReadAfterLd]>;
5604 multiclass avx512_fp_scalef_all<bits<8> opc, bits<8> opcScaler, string OpcodeStr,
5605 SDNode OpNode, SDNode OpNodeScal,
5606 X86SchedWriteWidths sched> {
5607 defm PSZ : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, sched.ZMM, v16f32_info>,
5608 avx512_fp_round_packed<opc, OpcodeStr, OpNode, sched.ZMM, v16f32_info>,
5609 EVEX_V512, EVEX_CD8<32, CD8VF>;
5610 defm PDZ : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, sched.ZMM, v8f64_info>,
5611 avx512_fp_round_packed<opc, OpcodeStr, OpNode, sched.ZMM, v8f64_info>,
5612 EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>;
5613 defm SSZ : avx512_fp_scalef_scalar<opcScaler, OpcodeStr, OpNodeScal, sched.Scl, f32x_info>,
5614 avx512_fp_scalar_round<opcScaler, OpcodeStr##"ss", f32x_info, OpNodeScal, sched.Scl>,
5615 EVEX_4V,EVEX_CD8<32, CD8VT1>;
5616 defm SDZ : avx512_fp_scalef_scalar<opcScaler, OpcodeStr, OpNodeScal, sched.Scl, f64x_info>,
5617 avx512_fp_scalar_round<opcScaler, OpcodeStr##"sd", f64x_info, OpNodeScal, sched.Scl>,
5618 EVEX_4V, EVEX_CD8<64, CD8VT1>, VEX_W;
5620 // Define only if AVX512VL feature is present.
5621 let Predicates = [HasVLX] in {
5622 defm PSZ128 : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, sched.XMM, v4f32x_info>,
5623 EVEX_V128, EVEX_CD8<32, CD8VF>;
5624 defm PSZ256 : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, sched.YMM, v8f32x_info>,
5625 EVEX_V256, EVEX_CD8<32, CD8VF>;
5626 defm PDZ128 : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, sched.XMM, v2f64x_info>,
5627 EVEX_V128, VEX_W, EVEX_CD8<64, CD8VF>;
5628 defm PDZ256 : avx512_fp_scalef_p<opc, OpcodeStr, OpNode, sched.YMM, v4f64x_info>,
5629 EVEX_V256, VEX_W, EVEX_CD8<64, CD8VF>;
5632 defm VSCALEF : avx512_fp_scalef_all<0x2C, 0x2D, "vscalef", X86scalef, X86scalefs,
5633 SchedWriteFAdd>, T8PD, NotEVEX2VEXConvertible;
5635 //===----------------------------------------------------------------------===//
5636 // AVX-512 VPTESTM instructions
5637 //===----------------------------------------------------------------------===//
5639 multiclass avx512_vptest<bits<8> opc, string OpcodeStr, PatFrag OpNode,
5640 X86FoldableSchedWrite sched, X86VectorVTInfo _,
5642 let ExeDomain = _.ExeDomain in {
5643 let isCommutable = 1 in
5644 defm rr : AVX512_maskable_cmp<opc, MRMSrcReg, _, (outs _.KRC:$dst),
5645 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,
5646 "$src2, $src1", "$src1, $src2",
5647 (OpNode (bitconvert (_.i64VT (and _.RC:$src1, _.RC:$src2))),
5649 EVEX_4V, Sched<[sched]>;
5650 defm rm : AVX512_maskable_cmp<opc, MRMSrcMem, _, (outs _.KRC:$dst),
5651 (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr,
5652 "$src2, $src1", "$src1, $src2",
5654 (_.i64VT (and _.RC:$src1,
5655 (bitconvert (_.LdFrag addr:$src2))))),
5657 EVEX_4V, EVEX_CD8<_.EltSize, CD8VF>,
5658 Sched<[sched.Folded, ReadAfterLd]>;
5661 // Patterns for compare with 0 that just use the same source twice.
5662 def : Pat<(_.KVT (OpNode _.RC:$src, _.ImmAllZerosV)),
5663 (_.KVT (!cast<Instruction>(Name # _.ZSuffix # "rr")
5664 _.RC:$src, _.RC:$src))>;
5666 def : Pat<(_.KVT (and _.KRC:$mask, (OpNode _.RC:$src, _.ImmAllZerosV))),
5667 (_.KVT (!cast<Instruction>(Name # _.ZSuffix # "rrk")
5668 _.KRC:$mask, _.RC:$src, _.RC:$src))>;
5671 multiclass avx512_vptest_mb<bits<8> opc, string OpcodeStr, PatFrag OpNode,
5672 X86FoldableSchedWrite sched, X86VectorVTInfo _> {
5673 let ExeDomain = _.ExeDomain in
5674 defm rmb : AVX512_maskable_cmp<opc, MRMSrcMem, _, (outs _.KRC:$dst),
5675 (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr,
5676 "${src2}"##_.BroadcastStr##", $src1",
5677 "$src1, ${src2}"##_.BroadcastStr,
5678 (OpNode (and _.RC:$src1,
5680 (_.ScalarLdFrag addr:$src2))),
5682 EVEX_B, EVEX_4V, EVEX_CD8<_.EltSize, CD8VF>,
5683 Sched<[sched.Folded, ReadAfterLd]>;
5686 // Use 512bit version to implement 128/256 bit in case NoVLX.
5687 multiclass avx512_vptest_lowering<PatFrag OpNode, X86VectorVTInfo ExtendInfo,
5688 X86VectorVTInfo _, string Name> {
5689 def : Pat<(_.KVT (OpNode (bitconvert (_.i64VT (and _.RC:$src1, _.RC:$src2))),
5691 (_.KVT (COPY_TO_REGCLASS
5692 (!cast<Instruction>(Name # "Zrr")
5693 (INSERT_SUBREG (ExtendInfo.VT (IMPLICIT_DEF)),
5694 _.RC:$src1, _.SubRegIdx),
5695 (INSERT_SUBREG (ExtendInfo.VT (IMPLICIT_DEF)),
5696 _.RC:$src2, _.SubRegIdx)),
5699 def : Pat<(_.KVT (and _.KRC:$mask,
5700 (OpNode (bitconvert (_.i64VT (and _.RC:$src1, _.RC:$src2))),
5703 (!cast<Instruction>(Name # "Zrrk")
5704 (COPY_TO_REGCLASS _.KRC:$mask, ExtendInfo.KRC),
5705 (INSERT_SUBREG (ExtendInfo.VT (IMPLICIT_DEF)),
5706 _.RC:$src1, _.SubRegIdx),
5707 (INSERT_SUBREG (ExtendInfo.VT (IMPLICIT_DEF)),
5708 _.RC:$src2, _.SubRegIdx)),
5711 def : Pat<(_.KVT (OpNode _.RC:$src, _.ImmAllZerosV)),
5712 (_.KVT (COPY_TO_REGCLASS
5713 (!cast<Instruction>(Name # "Zrr")
5714 (INSERT_SUBREG (ExtendInfo.VT (IMPLICIT_DEF)),
5715 _.RC:$src, _.SubRegIdx),
5716 (INSERT_SUBREG (ExtendInfo.VT (IMPLICIT_DEF)),
5717 _.RC:$src, _.SubRegIdx)),
5720 def : Pat<(_.KVT (and _.KRC:$mask, (OpNode _.RC:$src, _.ImmAllZerosV))),
5722 (!cast<Instruction>(Name # "Zrrk")
5723 (COPY_TO_REGCLASS _.KRC:$mask, ExtendInfo.KRC),
5724 (INSERT_SUBREG (ExtendInfo.VT (IMPLICIT_DEF)),
5725 _.RC:$src, _.SubRegIdx),
5726 (INSERT_SUBREG (ExtendInfo.VT (IMPLICIT_DEF)),
5727 _.RC:$src, _.SubRegIdx)),
5731 multiclass avx512_vptest_dq_sizes<bits<8> opc, string OpcodeStr, PatFrag OpNode,
5732 X86SchedWriteWidths sched, AVX512VLVectorVTInfo _> {
5733 let Predicates = [HasAVX512] in
5734 defm Z : avx512_vptest<opc, OpcodeStr, OpNode, sched.ZMM, _.info512, NAME>,
5735 avx512_vptest_mb<opc, OpcodeStr, OpNode, sched.ZMM, _.info512>, EVEX_V512;
5737 let Predicates = [HasAVX512, HasVLX] in {
5738 defm Z256 : avx512_vptest<opc, OpcodeStr, OpNode, sched.YMM, _.info256, NAME>,
5739 avx512_vptest_mb<opc, OpcodeStr, OpNode, sched.YMM, _.info256>, EVEX_V256;
5740 defm Z128 : avx512_vptest<opc, OpcodeStr, OpNode, sched.XMM, _.info128, NAME>,
5741 avx512_vptest_mb<opc, OpcodeStr, OpNode, sched.XMM, _.info128>, EVEX_V128;
5743 let Predicates = [HasAVX512, NoVLX] in {
5744 defm Z256_Alt : avx512_vptest_lowering< OpNode, _.info512, _.info256, NAME>;
5745 defm Z128_Alt : avx512_vptest_lowering< OpNode, _.info512, _.info128, NAME>;
5749 multiclass avx512_vptest_dq<bits<8> opc, string OpcodeStr, PatFrag OpNode,
5750 X86SchedWriteWidths sched> {
5751 defm D : avx512_vptest_dq_sizes<opc, OpcodeStr#"d", OpNode, sched,
5753 defm Q : avx512_vptest_dq_sizes<opc, OpcodeStr#"q", OpNode, sched,
5754 avx512vl_i64_info>, VEX_W;
5757 multiclass avx512_vptest_wb<bits<8> opc, string OpcodeStr,
5758 PatFrag OpNode, X86SchedWriteWidths sched> {
5759 let Predicates = [HasBWI] in {
5760 defm WZ: avx512_vptest<opc, OpcodeStr#"w", OpNode, sched.ZMM,
5761 v32i16_info, NAME#"W">, EVEX_V512, VEX_W;
5762 defm BZ: avx512_vptest<opc, OpcodeStr#"b", OpNode, sched.ZMM,
5763 v64i8_info, NAME#"B">, EVEX_V512;
5765 let Predicates = [HasVLX, HasBWI] in {
5767 defm WZ256: avx512_vptest<opc, OpcodeStr#"w", OpNode, sched.YMM,
5768 v16i16x_info, NAME#"W">, EVEX_V256, VEX_W;
5769 defm WZ128: avx512_vptest<opc, OpcodeStr#"w", OpNode, sched.XMM,
5770 v8i16x_info, NAME#"W">, EVEX_V128, VEX_W;
5771 defm BZ256: avx512_vptest<opc, OpcodeStr#"b", OpNode, sched.YMM,
5772 v32i8x_info, NAME#"B">, EVEX_V256;
5773 defm BZ128: avx512_vptest<opc, OpcodeStr#"b", OpNode, sched.XMM,
5774 v16i8x_info, NAME#"B">, EVEX_V128;
5777 let Predicates = [HasAVX512, NoVLX] in {
5778 defm BZ256_Alt : avx512_vptest_lowering<OpNode, v64i8_info, v32i8x_info, NAME#"B">;
5779 defm BZ128_Alt : avx512_vptest_lowering<OpNode, v64i8_info, v16i8x_info, NAME#"B">;
5780 defm WZ256_Alt : avx512_vptest_lowering<OpNode, v32i16_info, v16i16x_info, NAME#"W">;
5781 defm WZ128_Alt : avx512_vptest_lowering<OpNode, v32i16_info, v8i16x_info, NAME#"W">;
5785 // These patterns are used to match vptestm/vptestnm. We don't treat pcmpeqm
5786 // as commutable here because we already canonicalized all zeros vectors to the
5787 // RHS during lowering.
5788 def X86pcmpeqm : PatFrag<(ops node:$src1, node:$src2),
5789 (setcc node:$src1, node:$src2, SETEQ)>;
5790 def X86pcmpnem : PatFrag<(ops node:$src1, node:$src2),
5791 (setcc node:$src1, node:$src2, SETNE)>;
5793 multiclass avx512_vptest_all_forms<bits<8> opc_wb, bits<8> opc_dq, string OpcodeStr,
5794 PatFrag OpNode, X86SchedWriteWidths sched> :
5795 avx512_vptest_wb<opc_wb, OpcodeStr, OpNode, sched>,
5796 avx512_vptest_dq<opc_dq, OpcodeStr, OpNode, sched>;
5798 defm VPTESTM : avx512_vptest_all_forms<0x26, 0x27, "vptestm", X86pcmpnem,
5799 SchedWriteVecLogic>, T8PD;
5800 defm VPTESTNM : avx512_vptest_all_forms<0x26, 0x27, "vptestnm", X86pcmpeqm,
5801 SchedWriteVecLogic>, T8XS;
5803 //===----------------------------------------------------------------------===//
5804 // AVX-512 Shift instructions
5805 //===----------------------------------------------------------------------===//
5807 multiclass avx512_shift_rmi<bits<8> opc, Format ImmFormR, Format ImmFormM,
5808 string OpcodeStr, SDNode OpNode,
5809 X86FoldableSchedWrite sched, X86VectorVTInfo _> {
5810 let ExeDomain = _.ExeDomain in {
5811 defm ri : AVX512_maskable<opc, ImmFormR, _, (outs _.RC:$dst),
5812 (ins _.RC:$src1, u8imm:$src2), OpcodeStr,
5813 "$src2, $src1", "$src1, $src2",
5814 (_.VT (OpNode _.RC:$src1, (i8 imm:$src2)))>,
5816 defm mi : AVX512_maskable<opc, ImmFormM, _, (outs _.RC:$dst),
5817 (ins _.MemOp:$src1, u8imm:$src2), OpcodeStr,
5818 "$src2, $src1", "$src1, $src2",
5819 (_.VT (OpNode (_.VT (bitconvert (_.LdFrag addr:$src1))),
5821 Sched<[sched.Folded]>;
5825 multiclass avx512_shift_rmbi<bits<8> opc, Format ImmFormM,
5826 string OpcodeStr, SDNode OpNode,
5827 X86FoldableSchedWrite sched, X86VectorVTInfo _> {
5828 let ExeDomain = _.ExeDomain in
5829 defm mbi : AVX512_maskable<opc, ImmFormM, _, (outs _.RC:$dst),
5830 (ins _.ScalarMemOp:$src1, u8imm:$src2), OpcodeStr,
5831 "$src2, ${src1}"##_.BroadcastStr, "${src1}"##_.BroadcastStr##", $src2",
5832 (_.VT (OpNode (X86VBroadcast (_.ScalarLdFrag addr:$src1)), (i8 imm:$src2)))>,
5833 EVEX_B, Sched<[sched.Folded]>;
5836 multiclass avx512_shift_rrm<bits<8> opc, string OpcodeStr, SDNode OpNode,
5837 X86FoldableSchedWrite sched, ValueType SrcVT,
5838 PatFrag bc_frag, X86VectorVTInfo _> {
5839 // src2 is always 128-bit
5840 let ExeDomain = _.ExeDomain in {
5841 defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
5842 (ins _.RC:$src1, VR128X:$src2), OpcodeStr,
5843 "$src2, $src1", "$src1, $src2",
5844 (_.VT (OpNode _.RC:$src1, (SrcVT VR128X:$src2)))>,
5845 AVX512BIBase, EVEX_4V, Sched<[sched]>;
5846 defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
5847 (ins _.RC:$src1, i128mem:$src2), OpcodeStr,
5848 "$src2, $src1", "$src1, $src2",
5849 (_.VT (OpNode _.RC:$src1, (bc_frag (loadv2i64 addr:$src2))))>,
5851 EVEX_4V, Sched<[sched.Folded, ReadAfterLd]>;
5855 multiclass avx512_shift_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode,
5856 X86SchedWriteWidths sched, ValueType SrcVT,
5857 PatFrag bc_frag, AVX512VLVectorVTInfo VTInfo,
5859 let Predicates = [prd] in
5860 defm Z : avx512_shift_rrm<opc, OpcodeStr, OpNode, sched.ZMM, SrcVT,
5861 bc_frag, VTInfo.info512>, EVEX_V512,
5862 EVEX_CD8<VTInfo.info512.EltSize, CD8VQ> ;
5863 let Predicates = [prd, HasVLX] in {
5864 defm Z256 : avx512_shift_rrm<opc, OpcodeStr, OpNode, sched.YMM, SrcVT,
5865 bc_frag, VTInfo.info256>, EVEX_V256,
5866 EVEX_CD8<VTInfo.info256.EltSize, CD8VH>;
5867 defm Z128 : avx512_shift_rrm<opc, OpcodeStr, OpNode, sched.XMM, SrcVT,
5868 bc_frag, VTInfo.info128>, EVEX_V128,
5869 EVEX_CD8<VTInfo.info128.EltSize, CD8VF>;
5873 multiclass avx512_shift_types<bits<8> opcd, bits<8> opcq, bits<8> opcw,
5874 string OpcodeStr, SDNode OpNode,
5875 X86SchedWriteWidths sched,
5876 bit NotEVEX2VEXConvertibleQ = 0> {
5877 defm D : avx512_shift_sizes<opcd, OpcodeStr#"d", OpNode, sched, v4i32,
5878 bc_v4i32, avx512vl_i32_info, HasAVX512>;
5879 let notEVEX2VEXConvertible = NotEVEX2VEXConvertibleQ in
5880 defm Q : avx512_shift_sizes<opcq, OpcodeStr#"q", OpNode, sched, v2i64,
5881 bc_v2i64, avx512vl_i64_info, HasAVX512>, VEX_W;
5882 defm W : avx512_shift_sizes<opcw, OpcodeStr#"w", OpNode, sched, v8i16,
5883 bc_v2i64, avx512vl_i16_info, HasBWI>;
5886 multiclass avx512_shift_rmi_sizes<bits<8> opc, Format ImmFormR, Format ImmFormM,
5887 string OpcodeStr, SDNode OpNode,
5888 X86SchedWriteWidths sched,
5889 AVX512VLVectorVTInfo VTInfo> {
5890 let Predicates = [HasAVX512] in
5891 defm Z: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode,
5892 sched.ZMM, VTInfo.info512>,
5893 avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode, sched.ZMM,
5894 VTInfo.info512>, EVEX_V512;
5895 let Predicates = [HasAVX512, HasVLX] in {
5896 defm Z256: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode,
5897 sched.YMM, VTInfo.info256>,
5898 avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode, sched.YMM,
5899 VTInfo.info256>, EVEX_V256;
5900 defm Z128: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode,
5901 sched.XMM, VTInfo.info128>,
5902 avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode, sched.XMM,
5903 VTInfo.info128>, EVEX_V128;
5907 multiclass avx512_shift_rmi_w<bits<8> opcw, Format ImmFormR, Format ImmFormM,
5908 string OpcodeStr, SDNode OpNode,
5909 X86SchedWriteWidths sched> {
5910 let Predicates = [HasBWI] in
5911 defm WZ: avx512_shift_rmi<opcw, ImmFormR, ImmFormM, OpcodeStr, OpNode,
5912 sched.ZMM, v32i16_info>, EVEX_V512, VEX_WIG;
5913 let Predicates = [HasVLX, HasBWI] in {
5914 defm WZ256: avx512_shift_rmi<opcw, ImmFormR, ImmFormM, OpcodeStr, OpNode,
5915 sched.YMM, v16i16x_info>, EVEX_V256, VEX_WIG;
5916 defm WZ128: avx512_shift_rmi<opcw, ImmFormR, ImmFormM, OpcodeStr, OpNode,
5917 sched.XMM, v8i16x_info>, EVEX_V128, VEX_WIG;
5921 multiclass avx512_shift_rmi_dq<bits<8> opcd, bits<8> opcq,
5922 Format ImmFormR, Format ImmFormM,
5923 string OpcodeStr, SDNode OpNode,
5924 X86SchedWriteWidths sched,
5925 bit NotEVEX2VEXConvertibleQ = 0> {
5926 defm D: avx512_shift_rmi_sizes<opcd, ImmFormR, ImmFormM, OpcodeStr#"d", OpNode,
5927 sched, avx512vl_i32_info>, EVEX_CD8<32, CD8VF>;
5928 let notEVEX2VEXConvertible = NotEVEX2VEXConvertibleQ in
5929 defm Q: avx512_shift_rmi_sizes<opcq, ImmFormR, ImmFormM, OpcodeStr#"q", OpNode,
5930 sched, avx512vl_i64_info>, EVEX_CD8<64, CD8VF>, VEX_W;
5933 defm VPSRL : avx512_shift_rmi_dq<0x72, 0x73, MRM2r, MRM2m, "vpsrl", X86vsrli,
5934 SchedWriteVecShiftImm>,
5935 avx512_shift_rmi_w<0x71, MRM2r, MRM2m, "vpsrlw", X86vsrli,
5936 SchedWriteVecShiftImm>, AVX512BIi8Base, EVEX_4V;
5938 defm VPSLL : avx512_shift_rmi_dq<0x72, 0x73, MRM6r, MRM6m, "vpsll", X86vshli,
5939 SchedWriteVecShiftImm>,
5940 avx512_shift_rmi_w<0x71, MRM6r, MRM6m, "vpsllw", X86vshli,
5941 SchedWriteVecShiftImm>, AVX512BIi8Base, EVEX_4V;
5943 defm VPSRA : avx512_shift_rmi_dq<0x72, 0x72, MRM4r, MRM4m, "vpsra", X86vsrai,
5944 SchedWriteVecShiftImm, 1>,
5945 avx512_shift_rmi_w<0x71, MRM4r, MRM4m, "vpsraw", X86vsrai,
5946 SchedWriteVecShiftImm>, AVX512BIi8Base, EVEX_4V;
5948 defm VPROR : avx512_shift_rmi_dq<0x72, 0x72, MRM0r, MRM0m, "vpror", X86vrotri,
5949 SchedWriteVecShiftImm>, AVX512BIi8Base, EVEX_4V;
5950 defm VPROL : avx512_shift_rmi_dq<0x72, 0x72, MRM1r, MRM1m, "vprol", X86vrotli,
5951 SchedWriteVecShiftImm>, AVX512BIi8Base, EVEX_4V;
5953 defm VPSLL : avx512_shift_types<0xF2, 0xF3, 0xF1, "vpsll", X86vshl,
5954 SchedWriteVecShift>;
5955 defm VPSRA : avx512_shift_types<0xE2, 0xE2, 0xE1, "vpsra", X86vsra,
5956 SchedWriteVecShift, 1>;
5957 defm VPSRL : avx512_shift_types<0xD2, 0xD3, 0xD1, "vpsrl", X86vsrl,
5958 SchedWriteVecShift>;
5960 // Use 512bit VPSRA/VPSRAI version to implement v2i64/v4i64 in case NoVLX.
5961 let Predicates = [HasAVX512, NoVLX] in {
5962 def : Pat<(v4i64 (X86vsra (v4i64 VR256X:$src1), (v2i64 VR128X:$src2))),
5963 (EXTRACT_SUBREG (v8i64
5965 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)),
5966 VR128X:$src2)), sub_ymm)>;
5968 def : Pat<(v2i64 (X86vsra (v2i64 VR128X:$src1), (v2i64 VR128X:$src2))),
5969 (EXTRACT_SUBREG (v8i64
5971 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src1, sub_xmm)),
5972 VR128X:$src2)), sub_xmm)>;
5974 def : Pat<(v4i64 (X86vsrai (v4i64 VR256X:$src1), (i8 imm:$src2))),
5975 (EXTRACT_SUBREG (v8i64
5977 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)),
5978 imm:$src2)), sub_ymm)>;
5980 def : Pat<(v2i64 (X86vsrai (v2i64 VR128X:$src1), (i8 imm:$src2))),
5981 (EXTRACT_SUBREG (v8i64
5983 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src1, sub_xmm)),
5984 imm:$src2)), sub_xmm)>;
5987 //===-------------------------------------------------------------------===//
5988 // Variable Bit Shifts
5989 //===-------------------------------------------------------------------===//
5991 multiclass avx512_var_shift<bits<8> opc, string OpcodeStr, SDNode OpNode,
5992 X86FoldableSchedWrite sched, X86VectorVTInfo _> {
5993 let ExeDomain = _.ExeDomain in {
5994 defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
5995 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,
5996 "$src2, $src1", "$src1, $src2",
5997 (_.VT (OpNode _.RC:$src1, (_.VT _.RC:$src2)))>,
5998 AVX5128IBase, EVEX_4V, Sched<[sched]>;
5999 defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
6000 (ins _.RC:$src1, _.MemOp:$src2), OpcodeStr,
6001 "$src2, $src1", "$src1, $src2",
6002 (_.VT (OpNode _.RC:$src1,
6003 (_.VT (bitconvert (_.LdFrag addr:$src2)))))>,
6004 AVX5128IBase, EVEX_4V, EVEX_CD8<_.EltSize, CD8VF>,
6005 Sched<[sched.Folded, ReadAfterLd]>;
6009 multiclass avx512_var_shift_mb<bits<8> opc, string OpcodeStr, SDNode OpNode,
6010 X86FoldableSchedWrite sched, X86VectorVTInfo _> {
6011 let ExeDomain = _.ExeDomain in
6012 defm rmb : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
6013 (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr,
6014 "${src2}"##_.BroadcastStr##", $src1",
6015 "$src1, ${src2}"##_.BroadcastStr,
6016 (_.VT (OpNode _.RC:$src1, (_.VT (X86VBroadcast
6017 (_.ScalarLdFrag addr:$src2)))))>,
6018 AVX5128IBase, EVEX_B, EVEX_4V, EVEX_CD8<_.EltSize, CD8VF>,
6019 Sched<[sched.Folded, ReadAfterLd]>;
6022 multiclass avx512_var_shift_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode,
6023 X86SchedWriteWidths sched, AVX512VLVectorVTInfo _> {
6024 let Predicates = [HasAVX512] in
6025 defm Z : avx512_var_shift<opc, OpcodeStr, OpNode, sched.ZMM, _.info512>,
6026 avx512_var_shift_mb<opc, OpcodeStr, OpNode, sched.ZMM, _.info512>, EVEX_V512;
6028 let Predicates = [HasAVX512, HasVLX] in {
6029 defm Z256 : avx512_var_shift<opc, OpcodeStr, OpNode, sched.YMM, _.info256>,
6030 avx512_var_shift_mb<opc, OpcodeStr, OpNode, sched.YMM, _.info256>, EVEX_V256;
6031 defm Z128 : avx512_var_shift<opc, OpcodeStr, OpNode, sched.XMM, _.info128>,
6032 avx512_var_shift_mb<opc, OpcodeStr, OpNode, sched.XMM, _.info128>, EVEX_V128;
6036 multiclass avx512_var_shift_types<bits<8> opc, string OpcodeStr,
6037 SDNode OpNode, X86SchedWriteWidths sched> {
6038 defm D : avx512_var_shift_sizes<opc, OpcodeStr#"d", OpNode, sched,
6040 defm Q : avx512_var_shift_sizes<opc, OpcodeStr#"q", OpNode, sched,
6041 avx512vl_i64_info>, VEX_W;
6044 // Use 512bit version to implement 128/256 bit in case NoVLX.
6045 multiclass avx512_var_shift_lowering<AVX512VLVectorVTInfo _, string OpcodeStr,
6046 SDNode OpNode, list<Predicate> p> {
6047 let Predicates = p in {
6048 def : Pat<(_.info256.VT (OpNode (_.info256.VT _.info256.RC:$src1),
6049 (_.info256.VT _.info256.RC:$src2))),
6051 (!cast<Instruction>(OpcodeStr#"Zrr")
6052 (INSERT_SUBREG (_.info512.VT (IMPLICIT_DEF)), VR256X:$src1, sub_ymm),
6053 (INSERT_SUBREG (_.info512.VT (IMPLICIT_DEF)), VR256X:$src2, sub_ymm)),
6056 def : Pat<(_.info128.VT (OpNode (_.info128.VT _.info128.RC:$src1),
6057 (_.info128.VT _.info128.RC:$src2))),
6059 (!cast<Instruction>(OpcodeStr#"Zrr")
6060 (INSERT_SUBREG (_.info512.VT (IMPLICIT_DEF)), VR128X:$src1, sub_xmm),
6061 (INSERT_SUBREG (_.info512.VT (IMPLICIT_DEF)), VR128X:$src2, sub_xmm)),
6065 multiclass avx512_var_shift_w<bits<8> opc, string OpcodeStr,
6066 SDNode OpNode, X86SchedWriteWidths sched> {
6067 let Predicates = [HasBWI] in
6068 defm WZ: avx512_var_shift<opc, OpcodeStr, OpNode, sched.ZMM, v32i16_info>,
6070 let Predicates = [HasVLX, HasBWI] in {
6072 defm WZ256: avx512_var_shift<opc, OpcodeStr, OpNode, sched.YMM, v16i16x_info>,
6074 defm WZ128: avx512_var_shift<opc, OpcodeStr, OpNode, sched.XMM, v8i16x_info>,
6079 defm VPSLLV : avx512_var_shift_types<0x47, "vpsllv", shl, SchedWriteVarVecShift>,
6080 avx512_var_shift_w<0x12, "vpsllvw", shl, SchedWriteVarVecShift>;
6082 defm VPSRAV : avx512_var_shift_types<0x46, "vpsrav", sra, SchedWriteVarVecShift>,
6083 avx512_var_shift_w<0x11, "vpsravw", sra, SchedWriteVarVecShift>;
6085 defm VPSRLV : avx512_var_shift_types<0x45, "vpsrlv", srl, SchedWriteVarVecShift>,
6086 avx512_var_shift_w<0x10, "vpsrlvw", srl, SchedWriteVarVecShift>;
6088 defm VPRORV : avx512_var_shift_types<0x14, "vprorv", rotr, SchedWriteVarVecShift>;
6089 defm VPROLV : avx512_var_shift_types<0x15, "vprolv", rotl, SchedWriteVarVecShift>;
6091 defm : avx512_var_shift_lowering<avx512vl_i64_info, "VPSRAVQ", sra, [HasAVX512, NoVLX]>;
6092 defm : avx512_var_shift_lowering<avx512vl_i16_info, "VPSLLVW", shl, [HasBWI, NoVLX]>;
6093 defm : avx512_var_shift_lowering<avx512vl_i16_info, "VPSRAVW", sra, [HasBWI, NoVLX]>;
6094 defm : avx512_var_shift_lowering<avx512vl_i16_info, "VPSRLVW", srl, [HasBWI, NoVLX]>;
6096 // Special handing for handling VPSRAV intrinsics.
6097 multiclass avx512_var_shift_int_lowering<string InstrStr, X86VectorVTInfo _,
6098 list<Predicate> p> {
6099 let Predicates = p in {
6100 def : Pat<(_.VT (X86vsrav _.RC:$src1, _.RC:$src2)),
6101 (!cast<Instruction>(InstrStr#_.ZSuffix#rr) _.RC:$src1,
6103 def : Pat<(_.VT (X86vsrav _.RC:$src1, (bitconvert (_.LdFrag addr:$src2)))),
6104 (!cast<Instruction>(InstrStr#_.ZSuffix##rm)
6105 _.RC:$src1, addr:$src2)>;
6106 def : Pat<(_.VT (vselect _.KRCWM:$mask,
6107 (X86vsrav _.RC:$src1, _.RC:$src2), _.RC:$src0)),
6108 (!cast<Instruction>(InstrStr#_.ZSuffix#rrk) _.RC:$src0,
6109 _.KRC:$mask, _.RC:$src1, _.RC:$src2)>;
6110 def : Pat<(_.VT (vselect _.KRCWM:$mask,
6111 (X86vsrav _.RC:$src1, (bitconvert (_.LdFrag addr:$src2))),
6113 (!cast<Instruction>(InstrStr#_.ZSuffix##rmk) _.RC:$src0,
6114 _.KRC:$mask, _.RC:$src1, addr:$src2)>;
6115 def : Pat<(_.VT (vselect _.KRCWM:$mask,
6116 (X86vsrav _.RC:$src1, _.RC:$src2), _.ImmAllZerosV)),
6117 (!cast<Instruction>(InstrStr#_.ZSuffix#rrkz) _.KRC:$mask,
6118 _.RC:$src1, _.RC:$src2)>;
6119 def : Pat<(_.VT (vselect _.KRCWM:$mask,
6120 (X86vsrav _.RC:$src1, (bitconvert (_.LdFrag addr:$src2))),
6122 (!cast<Instruction>(InstrStr#_.ZSuffix##rmkz) _.KRC:$mask,
6123 _.RC:$src1, addr:$src2)>;
6127 multiclass avx512_var_shift_int_lowering_mb<string InstrStr, X86VectorVTInfo _,
6128 list<Predicate> p> :
6129 avx512_var_shift_int_lowering<InstrStr, _, p> {
6130 let Predicates = p in {
6131 def : Pat<(_.VT (X86vsrav _.RC:$src1,
6132 (X86VBroadcast (_.ScalarLdFrag addr:$src2)))),
6133 (!cast<Instruction>(InstrStr#_.ZSuffix##rmb)
6134 _.RC:$src1, addr:$src2)>;
6135 def : Pat<(_.VT (vselect _.KRCWM:$mask,
6136 (X86vsrav _.RC:$src1,
6137 (X86VBroadcast (_.ScalarLdFrag addr:$src2))),
6139 (!cast<Instruction>(InstrStr#_.ZSuffix##rmbk) _.RC:$src0,
6140 _.KRC:$mask, _.RC:$src1, addr:$src2)>;
6141 def : Pat<(_.VT (vselect _.KRCWM:$mask,
6142 (X86vsrav _.RC:$src1,
6143 (X86VBroadcast (_.ScalarLdFrag addr:$src2))),
6145 (!cast<Instruction>(InstrStr#_.ZSuffix##rmbkz) _.KRC:$mask,
6146 _.RC:$src1, addr:$src2)>;
6150 defm : avx512_var_shift_int_lowering<"VPSRAVW", v8i16x_info, [HasVLX, HasBWI]>;
6151 defm : avx512_var_shift_int_lowering<"VPSRAVW", v16i16x_info, [HasVLX, HasBWI]>;
6152 defm : avx512_var_shift_int_lowering<"VPSRAVW", v32i16_info, [HasBWI]>;
6153 defm : avx512_var_shift_int_lowering_mb<"VPSRAVD", v4i32x_info, [HasVLX]>;
6154 defm : avx512_var_shift_int_lowering_mb<"VPSRAVD", v8i32x_info, [HasVLX]>;
6155 defm : avx512_var_shift_int_lowering_mb<"VPSRAVD", v16i32_info, [HasAVX512]>;
6156 defm : avx512_var_shift_int_lowering_mb<"VPSRAVQ", v2i64x_info, [HasVLX]>;
6157 defm : avx512_var_shift_int_lowering_mb<"VPSRAVQ", v4i64x_info, [HasVLX]>;
6158 defm : avx512_var_shift_int_lowering_mb<"VPSRAVQ", v8i64_info, [HasAVX512]>;
6160 // Use 512bit VPROL/VPROLI version to implement v2i64/v4i64 + v4i32/v8i32 in case NoVLX.
6161 let Predicates = [HasAVX512, NoVLX] in {
6162 def : Pat<(v2i64 (rotl (v2i64 VR128X:$src1), (v2i64 VR128X:$src2))),
6163 (EXTRACT_SUBREG (v8i64
6165 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src1, sub_xmm)),
6166 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src2, sub_xmm)))),
6168 def : Pat<(v4i64 (rotl (v4i64 VR256X:$src1), (v4i64 VR256X:$src2))),
6169 (EXTRACT_SUBREG (v8i64
6171 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)),
6172 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src2, sub_ymm)))),
6175 def : Pat<(v4i32 (rotl (v4i32 VR128X:$src1), (v4i32 VR128X:$src2))),
6176 (EXTRACT_SUBREG (v16i32
6178 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src1, sub_xmm)),
6179 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src2, sub_xmm)))),
6181 def : Pat<(v8i32 (rotl (v8i32 VR256X:$src1), (v8i32 VR256X:$src2))),
6182 (EXTRACT_SUBREG (v16i32
6184 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)),
6185 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src2, sub_ymm)))),
6188 def : Pat<(v2i64 (X86vrotli (v2i64 VR128X:$src1), (i8 imm:$src2))),
6189 (EXTRACT_SUBREG (v8i64
6191 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src1, sub_xmm)),
6192 imm:$src2)), sub_xmm)>;
6193 def : Pat<(v4i64 (X86vrotli (v4i64 VR256X:$src1), (i8 imm:$src2))),
6194 (EXTRACT_SUBREG (v8i64
6196 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)),
6197 imm:$src2)), sub_ymm)>;
6199 def : Pat<(v4i32 (X86vrotli (v4i32 VR128X:$src1), (i8 imm:$src2))),
6200 (EXTRACT_SUBREG (v16i32
6202 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src1, sub_xmm)),
6203 imm:$src2)), sub_xmm)>;
6204 def : Pat<(v8i32 (X86vrotli (v8i32 VR256X:$src1), (i8 imm:$src2))),
6205 (EXTRACT_SUBREG (v16i32
6207 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)),
6208 imm:$src2)), sub_ymm)>;
6211 // Use 512bit VPROR/VPRORI version to implement v2i64/v4i64 + v4i32/v8i32 in case NoVLX.
6212 let Predicates = [HasAVX512, NoVLX] in {
6213 def : Pat<(v2i64 (rotr (v2i64 VR128X:$src1), (v2i64 VR128X:$src2))),
6214 (EXTRACT_SUBREG (v8i64
6216 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src1, sub_xmm)),
6217 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src2, sub_xmm)))),
6219 def : Pat<(v4i64 (rotr (v4i64 VR256X:$src1), (v4i64 VR256X:$src2))),
6220 (EXTRACT_SUBREG (v8i64
6222 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)),
6223 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src2, sub_ymm)))),
6226 def : Pat<(v4i32 (rotr (v4i32 VR128X:$src1), (v4i32 VR128X:$src2))),
6227 (EXTRACT_SUBREG (v16i32
6229 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src1, sub_xmm)),
6230 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src2, sub_xmm)))),
6232 def : Pat<(v8i32 (rotr (v8i32 VR256X:$src1), (v8i32 VR256X:$src2))),
6233 (EXTRACT_SUBREG (v16i32
6235 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)),
6236 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src2, sub_ymm)))),
6239 def : Pat<(v2i64 (X86vrotri (v2i64 VR128X:$src1), (i8 imm:$src2))),
6240 (EXTRACT_SUBREG (v8i64
6242 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src1, sub_xmm)),
6243 imm:$src2)), sub_xmm)>;
6244 def : Pat<(v4i64 (X86vrotri (v4i64 VR256X:$src1), (i8 imm:$src2))),
6245 (EXTRACT_SUBREG (v8i64
6247 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)),
6248 imm:$src2)), sub_ymm)>;
6250 def : Pat<(v4i32 (X86vrotri (v4i32 VR128X:$src1), (i8 imm:$src2))),
6251 (EXTRACT_SUBREG (v16i32
6253 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR128X:$src1, sub_xmm)),
6254 imm:$src2)), sub_xmm)>;
6255 def : Pat<(v8i32 (X86vrotri (v8i32 VR256X:$src1), (i8 imm:$src2))),
6256 (EXTRACT_SUBREG (v16i32
6258 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF), VR256X:$src1, sub_ymm)),
6259 imm:$src2)), sub_ymm)>;
6262 //===-------------------------------------------------------------------===//
6263 // 1-src variable permutation VPERMW/D/Q
6264 //===-------------------------------------------------------------------===//
6266 multiclass avx512_vperm_dq_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode,
6267 X86FoldableSchedWrite sched, AVX512VLVectorVTInfo _> {
6268 let Predicates = [HasAVX512] in
6269 defm Z : avx512_var_shift<opc, OpcodeStr, OpNode, sched, _.info512>,
6270 avx512_var_shift_mb<opc, OpcodeStr, OpNode, sched, _.info512>, EVEX_V512;
6272 let Predicates = [HasAVX512, HasVLX] in
6273 defm Z256 : avx512_var_shift<opc, OpcodeStr, OpNode, sched, _.info256>,
6274 avx512_var_shift_mb<opc, OpcodeStr, OpNode, sched, _.info256>, EVEX_V256;
6277 multiclass avx512_vpermi_dq_sizes<bits<8> opc, Format ImmFormR, Format ImmFormM,
6278 string OpcodeStr, SDNode OpNode,
6279 X86FoldableSchedWrite sched, AVX512VLVectorVTInfo VTInfo> {
6280 let Predicates = [HasAVX512] in
6281 defm Z: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode,
6282 sched, VTInfo.info512>,
6283 avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode,
6284 sched, VTInfo.info512>, EVEX_V512;
6285 let Predicates = [HasAVX512, HasVLX] in
6286 defm Z256: avx512_shift_rmi<opc, ImmFormR, ImmFormM, OpcodeStr, OpNode,
6287 sched, VTInfo.info256>,
6288 avx512_shift_rmbi<opc, ImmFormM, OpcodeStr, OpNode,
6289 sched, VTInfo.info256>, EVEX_V256;
6292 multiclass avx512_vperm_bw<bits<8> opc, string OpcodeStr,
6293 Predicate prd, SDNode OpNode,
6294 X86FoldableSchedWrite sched, AVX512VLVectorVTInfo _> {
6295 let Predicates = [prd] in
6296 defm Z: avx512_var_shift<opc, OpcodeStr, OpNode, sched, _.info512>,
6298 let Predicates = [HasVLX, prd] in {
6299 defm Z256: avx512_var_shift<opc, OpcodeStr, OpNode, sched, _.info256>,
6301 defm Z128: avx512_var_shift<opc, OpcodeStr, OpNode, sched, _.info128>,
6306 defm VPERMW : avx512_vperm_bw<0x8D, "vpermw", HasBWI, X86VPermv,
6307 WriteVarShuffle256, avx512vl_i16_info>, VEX_W;
6308 defm VPERMB : avx512_vperm_bw<0x8D, "vpermb", HasVBMI, X86VPermv,
6309 WriteVarShuffle256, avx512vl_i8_info>;
6311 defm VPERMD : avx512_vperm_dq_sizes<0x36, "vpermd", X86VPermv,
6312 WriteVarShuffle256, avx512vl_i32_info>;
6313 defm VPERMQ : avx512_vperm_dq_sizes<0x36, "vpermq", X86VPermv,
6314 WriteVarShuffle256, avx512vl_i64_info>, VEX_W;
6315 defm VPERMPS : avx512_vperm_dq_sizes<0x16, "vpermps", X86VPermv,
6316 WriteFVarShuffle256, avx512vl_f32_info>;
6317 defm VPERMPD : avx512_vperm_dq_sizes<0x16, "vpermpd", X86VPermv,
6318 WriteFVarShuffle256, avx512vl_f64_info>, VEX_W;
6320 defm VPERMQ : avx512_vpermi_dq_sizes<0x00, MRMSrcReg, MRMSrcMem, "vpermq",
6321 X86VPermi, WriteShuffle256, avx512vl_i64_info>,
6322 EVEX, AVX512AIi8Base, EVEX_CD8<64, CD8VF>, VEX_W;
6323 defm VPERMPD : avx512_vpermi_dq_sizes<0x01, MRMSrcReg, MRMSrcMem, "vpermpd",
6324 X86VPermi, WriteFShuffle256, avx512vl_f64_info>,
6325 EVEX, AVX512AIi8Base, EVEX_CD8<64, CD8VF>, VEX_W;
6327 //===----------------------------------------------------------------------===//
6328 // AVX-512 - VPERMIL
6329 //===----------------------------------------------------------------------===//
6331 multiclass avx512_permil_vec<bits<8> OpcVar, string OpcodeStr, SDNode OpNode,
6332 X86FoldableSchedWrite sched, X86VectorVTInfo _,
6333 X86VectorVTInfo Ctrl> {
6334 defm rr: AVX512_maskable<OpcVar, MRMSrcReg, _, (outs _.RC:$dst),
6335 (ins _.RC:$src1, Ctrl.RC:$src2), OpcodeStr,
6336 "$src2, $src1", "$src1, $src2",
6337 (_.VT (OpNode _.RC:$src1,
6338 (Ctrl.VT Ctrl.RC:$src2)))>,
6339 T8PD, EVEX_4V, Sched<[sched]>;
6340 defm rm: AVX512_maskable<OpcVar, MRMSrcMem, _, (outs _.RC:$dst),
6341 (ins _.RC:$src1, Ctrl.MemOp:$src2), OpcodeStr,
6342 "$src2, $src1", "$src1, $src2",
6345 (Ctrl.VT (bitconvert(Ctrl.LdFrag addr:$src2)))))>,
6346 T8PD, EVEX_4V, EVEX_CD8<_.EltSize, CD8VF>,
6347 Sched<[sched.Folded, ReadAfterLd]>;
6348 defm rmb: AVX512_maskable<OpcVar, MRMSrcMem, _, (outs _.RC:$dst),
6349 (ins _.RC:$src1, _.ScalarMemOp:$src2), OpcodeStr,
6350 "${src2}"##_.BroadcastStr##", $src1",
6351 "$src1, ${src2}"##_.BroadcastStr,
6354 (Ctrl.VT (X86VBroadcast
6355 (Ctrl.ScalarLdFrag addr:$src2)))))>,
6356 T8PD, EVEX_4V, EVEX_B, EVEX_CD8<_.EltSize, CD8VF>,
6357 Sched<[sched.Folded, ReadAfterLd]>;
6360 multiclass avx512_permil_vec_common<string OpcodeStr, bits<8> OpcVar,
6361 X86SchedWriteWidths sched,
6362 AVX512VLVectorVTInfo _,
6363 AVX512VLVectorVTInfo Ctrl> {
6364 let Predicates = [HasAVX512] in {
6365 defm Z : avx512_permil_vec<OpcVar, OpcodeStr, X86VPermilpv, sched.ZMM,
6366 _.info512, Ctrl.info512>, EVEX_V512;
6368 let Predicates = [HasAVX512, HasVLX] in {
6369 defm Z128 : avx512_permil_vec<OpcVar, OpcodeStr, X86VPermilpv, sched.XMM,
6370 _.info128, Ctrl.info128>, EVEX_V128;
6371 defm Z256 : avx512_permil_vec<OpcVar, OpcodeStr, X86VPermilpv, sched.YMM,
6372 _.info256, Ctrl.info256>, EVEX_V256;
6376 multiclass avx512_permil<string OpcodeStr, bits<8> OpcImm, bits<8> OpcVar,
6377 AVX512VLVectorVTInfo _, AVX512VLVectorVTInfo Ctrl>{
6378 defm NAME: avx512_permil_vec_common<OpcodeStr, OpcVar, SchedWriteFVarShuffle,
6380 defm NAME: avx512_shift_rmi_sizes<OpcImm, MRMSrcReg, MRMSrcMem, OpcodeStr,
6381 X86VPermilpi, SchedWriteFShuffle, _>,
6382 EVEX, AVX512AIi8Base, EVEX_CD8<_.info128.EltSize, CD8VF>;
6385 let ExeDomain = SSEPackedSingle in
6386 defm VPERMILPS : avx512_permil<"vpermilps", 0x04, 0x0C, avx512vl_f32_info,
6388 let ExeDomain = SSEPackedDouble in
6389 defm VPERMILPD : avx512_permil<"vpermilpd", 0x05, 0x0D, avx512vl_f64_info,
6390 avx512vl_i64_info>, VEX_W1X;
6392 //===----------------------------------------------------------------------===//
6393 // AVX-512 - VPSHUFD, VPSHUFLW, VPSHUFHW
6394 //===----------------------------------------------------------------------===//
6396 defm VPSHUFD : avx512_shift_rmi_sizes<0x70, MRMSrcReg, MRMSrcMem, "vpshufd",
6397 X86PShufd, SchedWriteShuffle, avx512vl_i32_info>,
6398 EVEX, AVX512BIi8Base, EVEX_CD8<32, CD8VF>;
6399 defm VPSHUFH : avx512_shift_rmi_w<0x70, MRMSrcReg, MRMSrcMem, "vpshufhw",
6400 X86PShufhw, SchedWriteShuffle>,
6401 EVEX, AVX512XSIi8Base;
6402 defm VPSHUFL : avx512_shift_rmi_w<0x70, MRMSrcReg, MRMSrcMem, "vpshuflw",
6403 X86PShuflw, SchedWriteShuffle>,
6404 EVEX, AVX512XDIi8Base;
6406 //===----------------------------------------------------------------------===//
6407 // AVX-512 - VPSHUFB
6408 //===----------------------------------------------------------------------===//
6410 multiclass avx512_pshufb_sizes<bits<8> opc, string OpcodeStr, SDNode OpNode,
6411 X86SchedWriteWidths sched> {
6412 let Predicates = [HasBWI] in
6413 defm Z: avx512_var_shift<opc, OpcodeStr, OpNode, sched.ZMM, v64i8_info>,
6416 let Predicates = [HasVLX, HasBWI] in {
6417 defm Z256: avx512_var_shift<opc, OpcodeStr, OpNode, sched.YMM, v32i8x_info>,
6419 defm Z128: avx512_var_shift<opc, OpcodeStr, OpNode, sched.XMM, v16i8x_info>,
6424 defm VPSHUFB: avx512_pshufb_sizes<0x00, "vpshufb", X86pshufb,
6425 SchedWriteVarShuffle>, VEX_WIG;
6427 //===----------------------------------------------------------------------===//
6428 // Move Low to High and High to Low packed FP Instructions
6429 //===----------------------------------------------------------------------===//
6431 def VMOVLHPSZrr : AVX512PSI<0x16, MRMSrcReg, (outs VR128X:$dst),
6432 (ins VR128X:$src1, VR128X:$src2),
6433 "vmovlhps\t{$src2, $src1, $dst|$dst, $src1, $src2}",
6434 [(set VR128X:$dst, (v4f32 (X86Movlhps VR128X:$src1, VR128X:$src2)))]>,
6435 Sched<[SchedWriteFShuffle.XMM]>, EVEX_4V;
6436 def VMOVHLPSZrr : AVX512PSI<0x12, MRMSrcReg, (outs VR128X:$dst),
6437 (ins VR128X:$src1, VR128X:$src2),
6438 "vmovhlps\t{$src2, $src1, $dst|$dst, $src1, $src2}",
6439 [(set VR128X:$dst, (v4f32 (X86Movhlps VR128X:$src1, VR128X:$src2)))]>,
6440 Sched<[SchedWriteFShuffle.XMM]>, EVEX_4V, NotMemoryFoldable;
6442 //===----------------------------------------------------------------------===//
6443 // VMOVHPS/PD VMOVLPS Instructions
6444 // All patterns was taken from SSS implementation.
6445 //===----------------------------------------------------------------------===//
6447 multiclass avx512_mov_hilo_packed<bits<8> opc, string OpcodeStr, SDNode OpNode,
6448 X86VectorVTInfo _> {
6449 let ExeDomain = _.ExeDomain in
6450 def rm : AVX512<opc, MRMSrcMem, (outs _.RC:$dst),
6451 (ins _.RC:$src1, f64mem:$src2),
6452 !strconcat(OpcodeStr,
6453 "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
6457 (v2f64 (scalar_to_vector (loadf64 addr:$src2)))))))]>,
6458 Sched<[SchedWriteFShuffle.XMM.Folded, ReadAfterLd]>, EVEX_4V;
6461 defm VMOVHPSZ128 : avx512_mov_hilo_packed<0x16, "vmovhps", X86Movlhps,
6462 v4f32x_info>, EVEX_CD8<32, CD8VT2>, PS;
6463 defm VMOVHPDZ128 : avx512_mov_hilo_packed<0x16, "vmovhpd", X86Unpckl,
6464 v2f64x_info>, EVEX_CD8<64, CD8VT1>, PD, VEX_W;
6465 defm VMOVLPSZ128 : avx512_mov_hilo_packed<0x12, "vmovlps", X86Movlps,
6466 v4f32x_info>, EVEX_CD8<32, CD8VT2>, PS;
6467 defm VMOVLPDZ128 : avx512_mov_hilo_packed<0x12, "vmovlpd", X86Movlpd,
6468 v2f64x_info>, EVEX_CD8<64, CD8VT1>, PD, VEX_W;
6470 let Predicates = [HasAVX512] in {
6472 def : Pat<(X86Movlhps VR128X:$src1,
6473 (bc_v4f32 (v2i64 (scalar_to_vector (loadi64 addr:$src2))))),
6474 (VMOVHPSZ128rm VR128X:$src1, addr:$src2)>;
6475 def : Pat<(X86Movlhps VR128X:$src1,
6476 (bc_v4f32 (v2i64 (X86vzload addr:$src2)))),
6477 (VMOVHPSZ128rm VR128X:$src1, addr:$src2)>;
6479 def : Pat<(v2f64 (X86Unpckl VR128X:$src1,
6480 (bc_v2f64 (v2i64 (scalar_to_vector (loadi64 addr:$src2)))))),
6481 (VMOVHPDZ128rm VR128X:$src1, addr:$src2)>;
6483 def : Pat<(v4f32 (X86Movlps VR128X:$src1, (load addr:$src2))),
6484 (VMOVLPSZ128rm VR128X:$src1, addr:$src2)>;
6486 def : Pat<(v2f64 (X86Movlpd VR128X:$src1, (load addr:$src2))),
6487 (VMOVLPDZ128rm VR128X:$src1, addr:$src2)>;
6488 def : Pat<(v2f64 (X86Movsd VR128X:$src1,
6489 (v2f64 (scalar_to_vector (loadf64 addr:$src2))))),
6490 (VMOVLPDZ128rm VR128X:$src1, addr:$src2)>;
6493 let SchedRW = [WriteFStore] in {
6494 def VMOVHPSZ128mr : AVX512PSI<0x17, MRMDestMem, (outs),
6495 (ins f64mem:$dst, VR128X:$src),
6496 "vmovhps\t{$src, $dst|$dst, $src}",
6497 [(store (f64 (extractelt
6498 (X86Unpckh (bc_v2f64 (v4f32 VR128X:$src)),
6499 (bc_v2f64 (v4f32 VR128X:$src))),
6500 (iPTR 0))), addr:$dst)]>,
6501 EVEX, EVEX_CD8<32, CD8VT2>;
6502 def VMOVHPDZ128mr : AVX512PDI<0x17, MRMDestMem, (outs),
6503 (ins f64mem:$dst, VR128X:$src),
6504 "vmovhpd\t{$src, $dst|$dst, $src}",
6505 [(store (f64 (extractelt
6506 (v2f64 (X86Unpckh VR128X:$src, VR128X:$src)),
6507 (iPTR 0))), addr:$dst)]>,
6508 EVEX, EVEX_CD8<64, CD8VT1>, VEX_W;
6509 def VMOVLPSZ128mr : AVX512PSI<0x13, MRMDestMem, (outs),
6510 (ins f64mem:$dst, VR128X:$src),
6511 "vmovlps\t{$src, $dst|$dst, $src}",
6512 [(store (f64 (extractelt (bc_v2f64 (v4f32 VR128X:$src)),
6513 (iPTR 0))), addr:$dst)]>,
6514 EVEX, EVEX_CD8<32, CD8VT2>;
6515 def VMOVLPDZ128mr : AVX512PDI<0x13, MRMDestMem, (outs),
6516 (ins f64mem:$dst, VR128X:$src),
6517 "vmovlpd\t{$src, $dst|$dst, $src}",
6518 [(store (f64 (extractelt (v2f64 VR128X:$src),
6519 (iPTR 0))), addr:$dst)]>,
6520 EVEX, EVEX_CD8<64, CD8VT1>, VEX_W;
6523 let Predicates = [HasAVX512] in {
6525 def : Pat<(store (f64 (extractelt
6526 (v2f64 (X86VPermilpi VR128X:$src, (i8 1))),
6527 (iPTR 0))), addr:$dst),
6528 (VMOVHPDZ128mr addr:$dst, VR128X:$src)>;
6530 def : Pat<(store (v4f32 (X86Movlps (load addr:$src1), VR128X:$src2)),
6532 (VMOVLPSZ128mr addr:$src1, VR128X:$src2)>;
6534 def : Pat<(store (v2f64 (X86Movlpd (load addr:$src1), VR128X:$src2)),
6536 (VMOVLPDZ128mr addr:$src1, VR128X:$src2)>;
6538 //===----------------------------------------------------------------------===//
6539 // FMA - Fused Multiply Operations
6542 multiclass avx512_fma3p_213_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
6543 X86FoldableSchedWrite sched,
6544 X86VectorVTInfo _, string Suff> {
6545 let Constraints = "$src1 = $dst", ExeDomain = _.ExeDomain, hasSideEffects = 0 in {
6546 defm r: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
6547 (ins _.RC:$src2, _.RC:$src3),
6548 OpcodeStr, "$src3, $src2", "$src2, $src3",
6549 (_.VT (OpNode _.RC:$src2, _.RC:$src1, _.RC:$src3)), 1, 1>,
6550 AVX512FMA3Base, Sched<[sched]>;
6552 defm m: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
6553 (ins _.RC:$src2, _.MemOp:$src3),
6554 OpcodeStr, "$src3, $src2", "$src2, $src3",
6555 (_.VT (OpNode _.RC:$src2, _.RC:$src1, (_.LdFrag addr:$src3))), 1, 0>,
6556 AVX512FMA3Base, Sched<[sched.Folded, ReadAfterLd]>;
6558 defm mb: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
6559 (ins _.RC:$src2, _.ScalarMemOp:$src3),
6560 OpcodeStr, !strconcat("${src3}", _.BroadcastStr,", $src2"),
6561 !strconcat("$src2, ${src3}", _.BroadcastStr ),
6563 _.RC:$src1,(_.VT (X86VBroadcast (_.ScalarLdFrag addr:$src3)))), 1, 0>,
6564 AVX512FMA3Base, EVEX_B, Sched<[sched.Folded, ReadAfterLd]>;
6568 multiclass avx512_fma3_213_round<bits<8> opc, string OpcodeStr, SDNode OpNode,
6569 X86FoldableSchedWrite sched,
6570 X86VectorVTInfo _, string Suff> {
6571 let Constraints = "$src1 = $dst", ExeDomain = _.ExeDomain, hasSideEffects = 0 in
6572 defm rb: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
6573 (ins _.RC:$src2, _.RC:$src3, AVX512RC:$rc),
6574 OpcodeStr, "$rc, $src3, $src2", "$src2, $src3, $rc",
6575 (_.VT ( OpNode _.RC:$src2, _.RC:$src1, _.RC:$src3, (i32 imm:$rc))), 1, 1>,
6576 AVX512FMA3Base, EVEX_B, EVEX_RC, Sched<[sched]>;
6579 multiclass avx512_fma3p_213_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
6580 SDNode OpNodeRnd, X86SchedWriteWidths sched,
6581 AVX512VLVectorVTInfo _, string Suff> {
6582 let Predicates = [HasAVX512] in {
6583 defm Z : avx512_fma3p_213_rm<opc, OpcodeStr, OpNode, sched.ZMM,
6585 avx512_fma3_213_round<opc, OpcodeStr, OpNodeRnd, sched.ZMM,
6587 EVEX_V512, EVEX_CD8<_.info512.EltSize, CD8VF>;
6589 let Predicates = [HasVLX, HasAVX512] in {
6590 defm Z256 : avx512_fma3p_213_rm<opc, OpcodeStr, OpNode, sched.YMM,
6592 EVEX_V256, EVEX_CD8<_.info256.EltSize, CD8VF>;
6593 defm Z128 : avx512_fma3p_213_rm<opc, OpcodeStr, OpNode, sched.XMM,
6595 EVEX_V128, EVEX_CD8<_.info128.EltSize, CD8VF>;
6599 multiclass avx512_fma3p_213_f<bits<8> opc, string OpcodeStr, SDNode OpNode,
6601 defm PS : avx512_fma3p_213_common<opc, OpcodeStr#"ps", OpNode, OpNodeRnd,
6602 SchedWriteFMA, avx512vl_f32_info, "PS">;
6603 defm PD : avx512_fma3p_213_common<opc, OpcodeStr#"pd", OpNode, OpNodeRnd,
6604 SchedWriteFMA, avx512vl_f64_info, "PD">,
6608 defm VFMADD213 : avx512_fma3p_213_f<0xA8, "vfmadd213", X86Fmadd, X86FmaddRnd>;
6609 defm VFMSUB213 : avx512_fma3p_213_f<0xAA, "vfmsub213", X86Fmsub, X86FmsubRnd>;
6610 defm VFMADDSUB213 : avx512_fma3p_213_f<0xA6, "vfmaddsub213", X86Fmaddsub, X86FmaddsubRnd>;
6611 defm VFMSUBADD213 : avx512_fma3p_213_f<0xA7, "vfmsubadd213", X86Fmsubadd, X86FmsubaddRnd>;
6612 defm VFNMADD213 : avx512_fma3p_213_f<0xAC, "vfnmadd213", X86Fnmadd, X86FnmaddRnd>;
6613 defm VFNMSUB213 : avx512_fma3p_213_f<0xAE, "vfnmsub213", X86Fnmsub, X86FnmsubRnd>;
6616 multiclass avx512_fma3p_231_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
6617 X86FoldableSchedWrite sched,
6618 X86VectorVTInfo _, string Suff> {
6619 let Constraints = "$src1 = $dst", ExeDomain = _.ExeDomain, hasSideEffects = 0 in {
6620 defm r: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
6621 (ins _.RC:$src2, _.RC:$src3),
6622 OpcodeStr, "$src3, $src2", "$src2, $src3",
6623 (_.VT (OpNode _.RC:$src2, _.RC:$src3, _.RC:$src1)), 1, 1,
6624 vselect, 1>, AVX512FMA3Base, Sched<[sched]>;
6626 defm m: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
6627 (ins _.RC:$src2, _.MemOp:$src3),
6628 OpcodeStr, "$src3, $src2", "$src2, $src3",
6629 (_.VT (OpNode _.RC:$src2, (_.LdFrag addr:$src3), _.RC:$src1)), 1, 0>,
6630 AVX512FMA3Base, Sched<[sched.Folded, ReadAfterLd]>;
6632 defm mb: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
6633 (ins _.RC:$src2, _.ScalarMemOp:$src3),
6634 OpcodeStr, "${src3}"##_.BroadcastStr##", $src2",
6635 "$src2, ${src3}"##_.BroadcastStr,
6636 (_.VT (OpNode _.RC:$src2,
6637 (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src3))),
6638 _.RC:$src1)), 1, 0>, AVX512FMA3Base, EVEX_B,
6639 Sched<[sched.Folded, ReadAfterLd]>;
6643 multiclass avx512_fma3_231_round<bits<8> opc, string OpcodeStr, SDNode OpNode,
6644 X86FoldableSchedWrite sched,
6645 X86VectorVTInfo _, string Suff> {
6646 let Constraints = "$src1 = $dst", ExeDomain = _.ExeDomain, hasSideEffects = 0 in
6647 defm rb: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
6648 (ins _.RC:$src2, _.RC:$src3, AVX512RC:$rc),
6649 OpcodeStr, "$rc, $src3, $src2", "$src2, $src3, $rc",
6650 (_.VT ( OpNode _.RC:$src2, _.RC:$src3, _.RC:$src1, (i32 imm:$rc))),
6652 AVX512FMA3Base, EVEX_B, EVEX_RC, Sched<[sched]>;
6655 multiclass avx512_fma3p_231_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
6656 SDNode OpNodeRnd, X86SchedWriteWidths sched,
6657 AVX512VLVectorVTInfo _, string Suff> {
6658 let Predicates = [HasAVX512] in {
6659 defm Z : avx512_fma3p_231_rm<opc, OpcodeStr, OpNode, sched.ZMM,
6661 avx512_fma3_231_round<opc, OpcodeStr, OpNodeRnd, sched.ZMM,
6663 EVEX_V512, EVEX_CD8<_.info512.EltSize, CD8VF>;
6665 let Predicates = [HasVLX, HasAVX512] in {
6666 defm Z256 : avx512_fma3p_231_rm<opc, OpcodeStr, OpNode, sched.YMM,
6668 EVEX_V256, EVEX_CD8<_.info256.EltSize, CD8VF>;
6669 defm Z128 : avx512_fma3p_231_rm<opc, OpcodeStr, OpNode, sched.XMM,
6671 EVEX_V128, EVEX_CD8<_.info128.EltSize, CD8VF>;
6675 multiclass avx512_fma3p_231_f<bits<8> opc, string OpcodeStr, SDNode OpNode,
6676 SDNode OpNodeRnd > {
6677 defm PS : avx512_fma3p_231_common<opc, OpcodeStr#"ps", OpNode, OpNodeRnd,
6678 SchedWriteFMA, avx512vl_f32_info, "PS">;
6679 defm PD : avx512_fma3p_231_common<opc, OpcodeStr#"pd", OpNode, OpNodeRnd,
6680 SchedWriteFMA, avx512vl_f64_info, "PD">,
6684 defm VFMADD231 : avx512_fma3p_231_f<0xB8, "vfmadd231", X86Fmadd, X86FmaddRnd>;
6685 defm VFMSUB231 : avx512_fma3p_231_f<0xBA, "vfmsub231", X86Fmsub, X86FmsubRnd>;
6686 defm VFMADDSUB231 : avx512_fma3p_231_f<0xB6, "vfmaddsub231", X86Fmaddsub, X86FmaddsubRnd>;
6687 defm VFMSUBADD231 : avx512_fma3p_231_f<0xB7, "vfmsubadd231", X86Fmsubadd, X86FmsubaddRnd>;
6688 defm VFNMADD231 : avx512_fma3p_231_f<0xBC, "vfnmadd231", X86Fnmadd, X86FnmaddRnd>;
6689 defm VFNMSUB231 : avx512_fma3p_231_f<0xBE, "vfnmsub231", X86Fnmsub, X86FnmsubRnd>;
6691 multiclass avx512_fma3p_132_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
6692 X86FoldableSchedWrite sched,
6693 X86VectorVTInfo _, string Suff> {
6694 let Constraints = "$src1 = $dst", ExeDomain = _.ExeDomain, hasSideEffects = 0 in {
6695 defm r: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
6696 (ins _.RC:$src2, _.RC:$src3),
6697 OpcodeStr, "$src3, $src2", "$src2, $src3",
6698 (_.VT (OpNode _.RC:$src1, _.RC:$src3, _.RC:$src2)), 1, 1, vselect, 1>,
6699 AVX512FMA3Base, Sched<[sched]>;
6701 // Pattern is 312 order so that the load is in a different place from the
6702 // 213 and 231 patterns this helps tablegen's duplicate pattern detection.
6703 defm m: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
6704 (ins _.RC:$src2, _.MemOp:$src3),
6705 OpcodeStr, "$src3, $src2", "$src2, $src3",
6706 (_.VT (OpNode (_.LdFrag addr:$src3), _.RC:$src1, _.RC:$src2)), 1, 0>,
6707 AVX512FMA3Base, Sched<[sched.Folded, ReadAfterLd]>;
6709 // Pattern is 312 order so that the load is in a different place from the
6710 // 213 and 231 patterns this helps tablegen's duplicate pattern detection.
6711 defm mb: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
6712 (ins _.RC:$src2, _.ScalarMemOp:$src3),
6713 OpcodeStr, "${src3}"##_.BroadcastStr##", $src2",
6714 "$src2, ${src3}"##_.BroadcastStr,
6715 (_.VT (OpNode (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src3))),
6716 _.RC:$src1, _.RC:$src2)), 1, 0>,
6717 AVX512FMA3Base, EVEX_B, Sched<[sched.Folded, ReadAfterLd]>;
6721 multiclass avx512_fma3_132_round<bits<8> opc, string OpcodeStr, SDNode OpNode,
6722 X86FoldableSchedWrite sched,
6723 X86VectorVTInfo _, string Suff> {
6724 let Constraints = "$src1 = $dst", ExeDomain = _.ExeDomain, hasSideEffects = 0 in
6725 defm rb: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
6726 (ins _.RC:$src2, _.RC:$src3, AVX512RC:$rc),
6727 OpcodeStr, "$rc, $src3, $src2", "$src2, $src3, $rc",
6728 (_.VT ( OpNode _.RC:$src1, _.RC:$src3, _.RC:$src2, (i32 imm:$rc))),
6730 AVX512FMA3Base, EVEX_B, EVEX_RC, Sched<[sched]>;
6733 multiclass avx512_fma3p_132_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
6734 SDNode OpNodeRnd, X86SchedWriteWidths sched,
6735 AVX512VLVectorVTInfo _, string Suff> {
6736 let Predicates = [HasAVX512] in {
6737 defm Z : avx512_fma3p_132_rm<opc, OpcodeStr, OpNode, sched.ZMM,
6739 avx512_fma3_132_round<opc, OpcodeStr, OpNodeRnd, sched.ZMM,
6741 EVEX_V512, EVEX_CD8<_.info512.EltSize, CD8VF>;
6743 let Predicates = [HasVLX, HasAVX512] in {
6744 defm Z256 : avx512_fma3p_132_rm<opc, OpcodeStr, OpNode, sched.YMM,
6746 EVEX_V256, EVEX_CD8<_.info256.EltSize, CD8VF>;
6747 defm Z128 : avx512_fma3p_132_rm<opc, OpcodeStr, OpNode, sched.XMM,
6749 EVEX_V128, EVEX_CD8<_.info128.EltSize, CD8VF>;
6753 multiclass avx512_fma3p_132_f<bits<8> opc, string OpcodeStr, SDNode OpNode,
6754 SDNode OpNodeRnd > {
6755 defm PS : avx512_fma3p_132_common<opc, OpcodeStr#"ps", OpNode, OpNodeRnd,
6756 SchedWriteFMA, avx512vl_f32_info, "PS">;
6757 defm PD : avx512_fma3p_132_common<opc, OpcodeStr#"pd", OpNode, OpNodeRnd,
6758 SchedWriteFMA, avx512vl_f64_info, "PD">,
6762 defm VFMADD132 : avx512_fma3p_132_f<0x98, "vfmadd132", X86Fmadd, X86FmaddRnd>;
6763 defm VFMSUB132 : avx512_fma3p_132_f<0x9A, "vfmsub132", X86Fmsub, X86FmsubRnd>;
6764 defm VFMADDSUB132 : avx512_fma3p_132_f<0x96, "vfmaddsub132", X86Fmaddsub, X86FmaddsubRnd>;
6765 defm VFMSUBADD132 : avx512_fma3p_132_f<0x97, "vfmsubadd132", X86Fmsubadd, X86FmsubaddRnd>;
6766 defm VFNMADD132 : avx512_fma3p_132_f<0x9C, "vfnmadd132", X86Fnmadd, X86FnmaddRnd>;
6767 defm VFNMSUB132 : avx512_fma3p_132_f<0x9E, "vfnmsub132", X86Fnmsub, X86FnmsubRnd>;
6770 multiclass avx512_fma3s_common<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
6771 dag RHS_VEC_r, dag RHS_VEC_m, dag RHS_VEC_rb,
6772 dag RHS_r, dag RHS_m, bit MaskOnlyReg> {
6773 let Constraints = "$src1 = $dst", hasSideEffects = 0 in {
6774 defm r_Int: AVX512_maskable_3src_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
6775 (ins _.RC:$src2, _.RC:$src3), OpcodeStr,
6776 "$src3, $src2", "$src2, $src3", RHS_VEC_r, 1, 1>,
6777 AVX512FMA3Base, Sched<[SchedWriteFMA.Scl]>;
6779 defm m_Int: AVX512_maskable_3src_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
6780 (ins _.RC:$src2, _.IntScalarMemOp:$src3), OpcodeStr,
6781 "$src3, $src2", "$src2, $src3", RHS_VEC_m, 1, 1>,
6782 AVX512FMA3Base, Sched<[SchedWriteFMA.Scl.Folded, ReadAfterLd]>;
6784 defm rb_Int: AVX512_maskable_3src_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
6785 (ins _.RC:$src2, _.RC:$src3, AVX512RC:$rc),
6786 OpcodeStr, "$rc, $src3, $src2", "$src2, $src3, $rc", RHS_VEC_rb, 1, 1>,
6787 AVX512FMA3Base, EVEX_B, EVEX_RC, Sched<[SchedWriteFMA.Scl]>;
6789 let isCodeGenOnly = 1, isCommutable = 1 in {
6790 def r : AVX512FMA3S<opc, MRMSrcReg, (outs _.FRC:$dst),
6791 (ins _.FRC:$src1, _.FRC:$src2, _.FRC:$src3),
6792 !strconcat(OpcodeStr,
6793 "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
6794 !if(MaskOnlyReg, [], [RHS_r])>, Sched<[SchedWriteFMA.Scl]>;
6795 def m : AVX512FMA3S<opc, MRMSrcMem, (outs _.FRC:$dst),
6796 (ins _.FRC:$src1, _.FRC:$src2, _.ScalarMemOp:$src3),
6797 !strconcat(OpcodeStr,
6798 "\t{$src3, $src2, $dst|$dst, $src2, $src3}"),
6799 [RHS_m]>, Sched<[SchedWriteFMA.Scl.Folded, ReadAfterLd]>;
6800 }// isCodeGenOnly = 1
6801 }// Constraints = "$src1 = $dst"
6804 multiclass avx512_fma3s_all<bits<8> opc213, bits<8> opc231, bits<8> opc132,
6805 string OpcodeStr, SDNode OpNode, SDNode OpNodes1,
6806 SDNode OpNodeRnds1, SDNode OpNodes3,
6807 SDNode OpNodeRnds3, X86VectorVTInfo _,
6809 let ExeDomain = _.ExeDomain in {
6810 defm NAME#213#SUFF#Z: avx512_fma3s_common<opc213, OpcodeStr#"213"#_.Suffix, _,
6811 // Operands for intrinsic are in 123 order to preserve passthu
6813 (_.VT (OpNodes1 _.RC:$src1, _.RC:$src2, _.RC:$src3)),
6814 (_.VT (OpNodes1 _.RC:$src1, _.RC:$src2,
6815 _.ScalarIntMemCPat:$src3)),
6816 (_.VT (OpNodeRnds1 _.RC:$src1, _.RC:$src2, _.RC:$src3,
6818 (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src2, _.FRC:$src1,
6820 (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src2, _.FRC:$src1,
6821 (_.ScalarLdFrag addr:$src3)))), 0>;
6823 defm NAME#231#SUFF#Z: avx512_fma3s_common<opc231, OpcodeStr#"231"#_.Suffix, _,
6824 (_.VT (OpNodes3 _.RC:$src2, _.RC:$src3, _.RC:$src1)),
6825 (_.VT (OpNodes3 _.RC:$src2, _.ScalarIntMemCPat:$src3,
6827 (_.VT ( OpNodeRnds3 _.RC:$src2, _.RC:$src3, _.RC:$src1,
6829 (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src2, _.FRC:$src3,
6831 (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src2,
6832 (_.ScalarLdFrag addr:$src3), _.FRC:$src1))), 1>;
6834 // One pattern is 312 order so that the load is in a different place from the
6835 // 213 and 231 patterns this helps tablegen's duplicate pattern detection.
6836 defm NAME#132#SUFF#Z: avx512_fma3s_common<opc132, OpcodeStr#"132"#_.Suffix, _,
6838 (_.VT (OpNodes1 _.RC:$src1, _.ScalarIntMemCPat:$src3,
6841 (set _.FRC:$dst, (_.EltVT (OpNode _.FRC:$src1, _.FRC:$src3,
6843 (set _.FRC:$dst, (_.EltVT (OpNode (_.ScalarLdFrag addr:$src3),
6844 _.FRC:$src1, _.FRC:$src2))), 1>;
6848 multiclass avx512_fma3s<bits<8> opc213, bits<8> opc231, bits<8> opc132,
6849 string OpcodeStr, SDNode OpNode, SDNode OpNodes1,
6850 SDNode OpNodeRnds1, SDNode OpNodes3,
6851 SDNode OpNodeRnds3> {
6852 let Predicates = [HasAVX512] in {
6853 defm NAME : avx512_fma3s_all<opc213, opc231, opc132, OpcodeStr, OpNode,
6854 OpNodes1, OpNodeRnds1, OpNodes3, OpNodeRnds3,
6856 EVEX_CD8<32, CD8VT1>, VEX_LIG;
6857 defm NAME : avx512_fma3s_all<opc213, opc231, opc132, OpcodeStr, OpNode,
6858 OpNodes1, OpNodeRnds1, OpNodes3, OpNodeRnds3,
6860 EVEX_CD8<64, CD8VT1>, VEX_LIG, VEX_W;
6864 defm VFMADD : avx512_fma3s<0xA9, 0xB9, 0x99, "vfmadd", X86Fmadd, X86Fmadds1,
6865 X86FmaddRnds1, X86Fmadds3, X86FmaddRnds3>;
6866 defm VFMSUB : avx512_fma3s<0xAB, 0xBB, 0x9B, "vfmsub", X86Fmsub, X86Fmsubs1,
6867 X86FmsubRnds1, X86Fmsubs3, X86FmsubRnds3>;
6868 defm VFNMADD : avx512_fma3s<0xAD, 0xBD, 0x9D, "vfnmadd", X86Fnmadd, X86Fnmadds1,
6869 X86FnmaddRnds1, X86Fnmadds3, X86FnmaddRnds3>;
6870 defm VFNMSUB : avx512_fma3s<0xAF, 0xBF, 0x9F, "vfnmsub", X86Fnmsub, X86Fnmsubs1,
6871 X86FnmsubRnds1, X86Fnmsubs3, X86FnmsubRnds3>;
6873 multiclass avx512_scalar_fma_patterns<SDNode Op, string Prefix, string Suffix,
6874 SDNode Move, X86VectorVTInfo _,
6876 let Predicates = [HasAVX512] in {
6877 def : Pat<(_.VT (Move (_.VT VR128X:$src1), (_.VT (scalar_to_vector
6879 (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0))),
6881 (!cast<I>(Prefix#"213"#Suffix#"Zr_Int")
6882 VR128X:$src1, (COPY_TO_REGCLASS _.FRC:$src2, VR128X),
6883 (COPY_TO_REGCLASS _.FRC:$src3, VR128X))>;
6885 def : Pat<(_.VT (Move (_.VT VR128X:$src1), (_.VT (scalar_to_vector
6886 (X86selects VK1WM:$mask,
6888 (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0))),
6890 (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0)))))))),
6891 (!cast<I>(Prefix#"213"#Suffix#"Zr_Intk")
6892 VR128X:$src1, VK1WM:$mask,
6893 (COPY_TO_REGCLASS _.FRC:$src2, VR128X),
6894 (COPY_TO_REGCLASS _.FRC:$src3, VR128X))>;
6896 def : Pat<(_.VT (Move (_.VT VR128X:$src1), (_.VT (scalar_to_vector
6897 (X86selects VK1WM:$mask,
6898 (Op _.FRC:$src2, _.FRC:$src3,
6899 (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0)))),
6900 (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0)))))))),
6901 (!cast<I>(Prefix#"231"#Suffix#"Zr_Intk")
6902 VR128X:$src1, VK1WM:$mask,
6903 (COPY_TO_REGCLASS _.FRC:$src2, VR128X),
6904 (COPY_TO_REGCLASS _.FRC:$src3, VR128X))>;
6906 def : Pat<(_.VT (Move (_.VT VR128X:$src1), (_.VT (scalar_to_vector
6907 (X86selects VK1WM:$mask,
6909 (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0))),
6911 (_.EltVT ZeroFP)))))),
6912 (!cast<I>(Prefix#"213"#Suffix#"Zr_Intkz")
6913 VR128X:$src1, VK1WM:$mask,
6914 (COPY_TO_REGCLASS _.FRC:$src2, VR128X),
6915 (COPY_TO_REGCLASS _.FRC:$src3, VR128X))>;
6919 defm : avx512_scalar_fma_patterns<X86Fmadd, "VFMADD", "SS", X86Movss,
6920 v4f32x_info, fp32imm0>;
6921 defm : avx512_scalar_fma_patterns<X86Fmsub, "VFMSUB", "SS", X86Movss,
6922 v4f32x_info, fp32imm0>;
6923 defm : avx512_scalar_fma_patterns<X86Fnmadd, "VFNMADD", "SS", X86Movss,
6924 v4f32x_info, fp32imm0>;
6925 defm : avx512_scalar_fma_patterns<X86Fnmsub, "VFNMSUB", "SS", X86Movss,
6926 v4f32x_info, fp32imm0>;
6928 defm : avx512_scalar_fma_patterns<X86Fmadd, "VFMADD", "SD", X86Movsd,
6929 v2f64x_info, fp64imm0>;
6930 defm : avx512_scalar_fma_patterns<X86Fmsub, "VFMSUB", "SD", X86Movsd,
6931 v2f64x_info, fp64imm0>;
6932 defm : avx512_scalar_fma_patterns<X86Fnmadd, "VFNMADD", "SD", X86Movsd,
6933 v2f64x_info, fp64imm0>;
6934 defm : avx512_scalar_fma_patterns<X86Fnmsub, "VFNMSUB", "SD", X86Movsd,
6935 v2f64x_info, fp64imm0>;
6937 //===----------------------------------------------------------------------===//
6938 // AVX-512 Packed Multiply of Unsigned 52-bit Integers and Add the Low 52-bit IFMA
6939 //===----------------------------------------------------------------------===//
6940 let Constraints = "$src1 = $dst" in {
6941 multiclass avx512_pmadd52_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
6942 X86FoldableSchedWrite sched, X86VectorVTInfo _> {
6943 // NOTE: The SDNode have the multiply operands first with the add last.
6944 // This enables commuted load patterns to be autogenerated by tablegen.
6945 let ExeDomain = _.ExeDomain in {
6946 defm r: AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
6947 (ins _.RC:$src2, _.RC:$src3),
6948 OpcodeStr, "$src3, $src2", "$src2, $src3",
6949 (_.VT (OpNode _.RC:$src2, _.RC:$src3, _.RC:$src1)), 1, 1>,
6950 AVX512FMA3Base, Sched<[sched]>;
6952 defm m: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
6953 (ins _.RC:$src2, _.MemOp:$src3),
6954 OpcodeStr, "$src3, $src2", "$src2, $src3",
6955 (_.VT (OpNode _.RC:$src2, (_.LdFrag addr:$src3), _.RC:$src1))>,
6956 AVX512FMA3Base, Sched<[sched.Folded, ReadAfterLd]>;
6958 defm mb: AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
6959 (ins _.RC:$src2, _.ScalarMemOp:$src3),
6960 OpcodeStr, !strconcat("${src3}", _.BroadcastStr,", $src2"),
6961 !strconcat("$src2, ${src3}", _.BroadcastStr ),
6963 (_.VT (X86VBroadcast (_.ScalarLdFrag addr:$src3))),
6965 AVX512FMA3Base, EVEX_B, Sched<[sched.Folded, ReadAfterLd]>;
6968 } // Constraints = "$src1 = $dst"
6970 multiclass avx512_pmadd52_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
6971 X86SchedWriteWidths sched, AVX512VLVectorVTInfo _> {
6972 let Predicates = [HasIFMA] in {
6973 defm Z : avx512_pmadd52_rm<opc, OpcodeStr, OpNode, sched.ZMM, _.info512>,
6974 EVEX_V512, EVEX_CD8<_.info512.EltSize, CD8VF>;
6976 let Predicates = [HasVLX, HasIFMA] in {
6977 defm Z256 : avx512_pmadd52_rm<opc, OpcodeStr, OpNode, sched.YMM, _.info256>,
6978 EVEX_V256, EVEX_CD8<_.info256.EltSize, CD8VF>;
6979 defm Z128 : avx512_pmadd52_rm<opc, OpcodeStr, OpNode, sched.XMM, _.info128>,
6980 EVEX_V128, EVEX_CD8<_.info128.EltSize, CD8VF>;
6984 defm VPMADD52LUQ : avx512_pmadd52_common<0xb4, "vpmadd52luq", x86vpmadd52l,
6985 SchedWriteVecIMul, avx512vl_i64_info>,
6987 defm VPMADD52HUQ : avx512_pmadd52_common<0xb5, "vpmadd52huq", x86vpmadd52h,
6988 SchedWriteVecIMul, avx512vl_i64_info>,
6991 //===----------------------------------------------------------------------===//
6992 // AVX-512 Scalar convert from sign integer to float/double
6993 //===----------------------------------------------------------------------===//
6995 multiclass avx512_vcvtsi<bits<8> opc, SDNode OpNode, X86FoldableSchedWrite sched,
6996 RegisterClass SrcRC, X86VectorVTInfo DstVT,
6997 X86MemOperand x86memop, PatFrag ld_frag, string asm> {
6998 let hasSideEffects = 0 in {
6999 def rr : SI<opc, MRMSrcReg, (outs DstVT.FRC:$dst),
7000 (ins DstVT.FRC:$src1, SrcRC:$src),
7001 !strconcat(asm,"\t{$src, $src1, $dst|$dst, $src1, $src}"), []>,
7002 EVEX_4V, Sched<[sched]>;
7004 def rm : SI<opc, MRMSrcMem, (outs DstVT.FRC:$dst),
7005 (ins DstVT.FRC:$src1, x86memop:$src),
7006 !strconcat(asm,"\t{$src, $src1, $dst|$dst, $src1, $src}"), []>,
7007 EVEX_4V, Sched<[sched.Folded, ReadAfterLd]>;
7008 } // hasSideEffects = 0
7009 let isCodeGenOnly = 1 in {
7010 def rr_Int : SI<opc, MRMSrcReg, (outs DstVT.RC:$dst),
7011 (ins DstVT.RC:$src1, SrcRC:$src2),
7012 !strconcat(asm,"\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
7013 [(set DstVT.RC:$dst,
7014 (OpNode (DstVT.VT DstVT.RC:$src1),
7016 (i32 FROUND_CURRENT)))]>,
7017 EVEX_4V, Sched<[sched]>;
7019 def rm_Int : SI<opc, MRMSrcMem, (outs DstVT.RC:$dst),
7020 (ins DstVT.RC:$src1, x86memop:$src2),
7021 !strconcat(asm,"\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
7022 [(set DstVT.RC:$dst,
7023 (OpNode (DstVT.VT DstVT.RC:$src1),
7024 (ld_frag addr:$src2),
7025 (i32 FROUND_CURRENT)))]>,
7026 EVEX_4V, Sched<[sched.Folded, ReadAfterLd]>;
7027 }//isCodeGenOnly = 1
7030 multiclass avx512_vcvtsi_round<bits<8> opc, SDNode OpNode,
7031 X86FoldableSchedWrite sched, RegisterClass SrcRC,
7032 X86VectorVTInfo DstVT, string asm> {
7033 def rrb_Int : SI<opc, MRMSrcReg, (outs DstVT.RC:$dst),
7034 (ins DstVT.RC:$src1, SrcRC:$src2, AVX512RC:$rc),
7036 "\t{$src2, $rc, $src1, $dst|$dst, $src1, $rc, $src2}"),
7037 [(set DstVT.RC:$dst,
7038 (OpNode (DstVT.VT DstVT.RC:$src1),
7041 EVEX_4V, EVEX_B, EVEX_RC, Sched<[sched]>;
7044 multiclass avx512_vcvtsi_common<bits<8> opc, SDNode OpNode,
7045 X86FoldableSchedWrite sched,
7046 RegisterClass SrcRC, X86VectorVTInfo DstVT,
7047 X86MemOperand x86memop, PatFrag ld_frag, string asm> {
7048 defm NAME : avx512_vcvtsi_round<opc, OpNode, sched, SrcRC, DstVT, asm>,
7049 avx512_vcvtsi<opc, OpNode, sched, SrcRC, DstVT, x86memop,
7050 ld_frag, asm>, VEX_LIG;
7053 let Predicates = [HasAVX512] in {
7054 defm VCVTSI2SSZ : avx512_vcvtsi_common<0x2A, X86SintToFpRnd, WriteCvtI2SS, GR32,
7055 v4f32x_info, i32mem, loadi32, "cvtsi2ss{l}">,
7056 XS, EVEX_CD8<32, CD8VT1>;
7057 defm VCVTSI642SSZ: avx512_vcvtsi_common<0x2A, X86SintToFpRnd, WriteCvtI2SS, GR64,
7058 v4f32x_info, i64mem, loadi64, "cvtsi2ss{q}">,
7059 XS, VEX_W, EVEX_CD8<64, CD8VT1>;
7060 defm VCVTSI2SDZ : avx512_vcvtsi_common<0x2A, X86SintToFpRnd, WriteCvtI2SD, GR32,
7061 v2f64x_info, i32mem, loadi32, "cvtsi2sd{l}">,
7062 XD, EVEX_CD8<32, CD8VT1>;
7063 defm VCVTSI642SDZ: avx512_vcvtsi_common<0x2A, X86SintToFpRnd, WriteCvtI2SD, GR64,
7064 v2f64x_info, i64mem, loadi64, "cvtsi2sd{q}">,
7065 XD, VEX_W, EVEX_CD8<64, CD8VT1>;
7067 def : InstAlias<"vcvtsi2ss\t{$src, $src1, $dst|$dst, $src1, $src}",
7068 (VCVTSI2SSZrm FR64X:$dst, FR64X:$src1, i32mem:$src), 0, "att">;
7069 def : InstAlias<"vcvtsi2sd\t{$src, $src1, $dst|$dst, $src1, $src}",
7070 (VCVTSI2SDZrm FR64X:$dst, FR64X:$src1, i32mem:$src), 0, "att">;
7072 def : Pat<(f32 (sint_to_fp (loadi32 addr:$src))),
7073 (VCVTSI2SSZrm (f32 (IMPLICIT_DEF)), addr:$src)>;
7074 def : Pat<(f32 (sint_to_fp (loadi64 addr:$src))),
7075 (VCVTSI642SSZrm (f32 (IMPLICIT_DEF)), addr:$src)>;
7076 def : Pat<(f64 (sint_to_fp (loadi32 addr:$src))),
7077 (VCVTSI2SDZrm (f64 (IMPLICIT_DEF)), addr:$src)>;
7078 def : Pat<(f64 (sint_to_fp (loadi64 addr:$src))),
7079 (VCVTSI642SDZrm (f64 (IMPLICIT_DEF)), addr:$src)>;
7081 def : Pat<(f32 (sint_to_fp GR32:$src)),
7082 (VCVTSI2SSZrr (f32 (IMPLICIT_DEF)), GR32:$src)>;
7083 def : Pat<(f32 (sint_to_fp GR64:$src)),
7084 (VCVTSI642SSZrr (f32 (IMPLICIT_DEF)), GR64:$src)>;
7085 def : Pat<(f64 (sint_to_fp GR32:$src)),
7086 (VCVTSI2SDZrr (f64 (IMPLICIT_DEF)), GR32:$src)>;
7087 def : Pat<(f64 (sint_to_fp GR64:$src)),
7088 (VCVTSI642SDZrr (f64 (IMPLICIT_DEF)), GR64:$src)>;
7090 defm VCVTUSI2SSZ : avx512_vcvtsi_common<0x7B, X86UintToFpRnd, WriteCvtI2SS, GR32,
7091 v4f32x_info, i32mem, loadi32,
7092 "cvtusi2ss{l}">, XS, EVEX_CD8<32, CD8VT1>;
7093 defm VCVTUSI642SSZ : avx512_vcvtsi_common<0x7B, X86UintToFpRnd, WriteCvtI2SS, GR64,
7094 v4f32x_info, i64mem, loadi64, "cvtusi2ss{q}">,
7095 XS, VEX_W, EVEX_CD8<64, CD8VT1>;
7096 defm VCVTUSI2SDZ : avx512_vcvtsi<0x7B, X86UintToFpRnd, WriteCvtI2SD, GR32, v2f64x_info,
7097 i32mem, loadi32, "cvtusi2sd{l}">,
7098 XD, VEX_LIG, EVEX_CD8<32, CD8VT1>;
7099 defm VCVTUSI642SDZ : avx512_vcvtsi_common<0x7B, X86UintToFpRnd, WriteCvtI2SD, GR64,
7100 v2f64x_info, i64mem, loadi64, "cvtusi2sd{q}">,
7101 XD, VEX_W, EVEX_CD8<64, CD8VT1>;
7103 def : InstAlias<"vcvtusi2ss\t{$src, $src1, $dst|$dst, $src1, $src}",
7104 (VCVTUSI2SSZrm FR64X:$dst, FR64X:$src1, i32mem:$src), 0, "att">;
7105 def : InstAlias<"vcvtusi2sd\t{$src, $src1, $dst|$dst, $src1, $src}",
7106 (VCVTUSI2SDZrm FR64X:$dst, FR64X:$src1, i32mem:$src), 0, "att">;
7108 def : Pat<(f32 (uint_to_fp (loadi32 addr:$src))),
7109 (VCVTUSI2SSZrm (f32 (IMPLICIT_DEF)), addr:$src)>;
7110 def : Pat<(f32 (uint_to_fp (loadi64 addr:$src))),
7111 (VCVTUSI642SSZrm (f32 (IMPLICIT_DEF)), addr:$src)>;
7112 def : Pat<(f64 (uint_to_fp (loadi32 addr:$src))),
7113 (VCVTUSI2SDZrm (f64 (IMPLICIT_DEF)), addr:$src)>;
7114 def : Pat<(f64 (uint_to_fp (loadi64 addr:$src))),
7115 (VCVTUSI642SDZrm (f64 (IMPLICIT_DEF)), addr:$src)>;
7117 def : Pat<(f32 (uint_to_fp GR32:$src)),
7118 (VCVTUSI2SSZrr (f32 (IMPLICIT_DEF)), GR32:$src)>;
7119 def : Pat<(f32 (uint_to_fp GR64:$src)),
7120 (VCVTUSI642SSZrr (f32 (IMPLICIT_DEF)), GR64:$src)>;
7121 def : Pat<(f64 (uint_to_fp GR32:$src)),
7122 (VCVTUSI2SDZrr (f64 (IMPLICIT_DEF)), GR32:$src)>;
7123 def : Pat<(f64 (uint_to_fp GR64:$src)),
7124 (VCVTUSI642SDZrr (f64 (IMPLICIT_DEF)), GR64:$src)>;
7127 //===----------------------------------------------------------------------===//
7128 // AVX-512 Scalar convert from float/double to integer
7129 //===----------------------------------------------------------------------===//
7131 multiclass avx512_cvt_s_int_round<bits<8> opc, X86VectorVTInfo SrcVT,
7132 X86VectorVTInfo DstVT, SDNode OpNode,
7133 X86FoldableSchedWrite sched, string asm,
7135 bit CodeGenOnly = 1> {
7136 let Predicates = [HasAVX512] in {
7137 def rr_Int : SI<opc, MRMSrcReg, (outs DstVT.RC:$dst), (ins SrcVT.RC:$src),
7138 !strconcat(asm,"\t{$src, $dst|$dst, $src}"),
7139 [(set DstVT.RC:$dst, (OpNode (SrcVT.VT SrcVT.RC:$src),(i32 FROUND_CURRENT)))]>,
7140 EVEX, VEX_LIG, Sched<[sched]>;
7141 def rrb_Int : SI<opc, MRMSrcReg, (outs DstVT.RC:$dst), (ins SrcVT.RC:$src, AVX512RC:$rc),
7142 !strconcat(asm,"\t{$rc, $src, $dst|$dst, $src, $rc}"),
7143 [(set DstVT.RC:$dst, (OpNode (SrcVT.VT SrcVT.RC:$src),(i32 imm:$rc)))]>,
7144 EVEX, VEX_LIG, EVEX_B, EVEX_RC,
7146 let isCodeGenOnly = CodeGenOnly, ForceDisassemble = CodeGenOnly in
7147 def rm_Int : SI<opc, MRMSrcMem, (outs DstVT.RC:$dst), (ins SrcVT.IntScalarMemOp:$src),
7148 !strconcat(asm,"\t{$src, $dst|$dst, $src}"),
7149 [(set DstVT.RC:$dst, (OpNode
7150 (SrcVT.VT SrcVT.ScalarIntMemCPat:$src),
7151 (i32 FROUND_CURRENT)))]>,
7152 EVEX, VEX_LIG, Sched<[sched.Folded, ReadAfterLd]>;
7154 def : InstAlias<"v" # asm # aliasStr # "\t{$src, $dst|$dst, $src}",
7155 (!cast<Instruction>(NAME # "rr_Int") DstVT.RC:$dst, SrcVT.RC:$src), 0, "att">;
7156 def : InstAlias<"v" # asm # aliasStr # "\t{$rc, $src, $dst|$dst, $src, $rc}",
7157 (!cast<Instruction>(NAME # "rrb_Int") DstVT.RC:$dst, SrcVT.RC:$src, AVX512RC:$rc), 0, "att">;
7158 } // Predicates = [HasAVX512]
7161 multiclass avx512_cvt_s_int_round_aliases<bits<8> opc, X86VectorVTInfo SrcVT,
7162 X86VectorVTInfo DstVT, SDNode OpNode,
7163 X86FoldableSchedWrite sched, string asm,
7165 avx512_cvt_s_int_round<opc, SrcVT, DstVT, OpNode, sched, asm, aliasStr, 0> {
7166 let Predicates = [HasAVX512] in {
7167 def : InstAlias<"v" # asm # aliasStr # "\t{$src, $dst|$dst, $src}",
7168 (!cast<Instruction>(NAME # "rm_Int") DstVT.RC:$dst,
7169 SrcVT.IntScalarMemOp:$src), 0, "att">;
7170 } // Predicates = [HasAVX512]
7173 // Convert float/double to signed/unsigned int 32/64
7174 defm VCVTSS2SIZ: avx512_cvt_s_int_round<0x2D, f32x_info, i32x_info,
7175 X86cvts2si, WriteCvtSS2I, "cvtss2si", "{l}">,
7176 XS, EVEX_CD8<32, CD8VT1>;
7177 defm VCVTSS2SI64Z: avx512_cvt_s_int_round<0x2D, f32x_info, i64x_info,
7178 X86cvts2si, WriteCvtSS2I, "cvtss2si", "{q}">,
7179 XS, VEX_W, EVEX_CD8<32, CD8VT1>;
7180 defm VCVTSS2USIZ: avx512_cvt_s_int_round_aliases<0x79, f32x_info, i32x_info,
7181 X86cvts2usi, WriteCvtSS2I, "cvtss2usi", "{l}">,
7182 XS, EVEX_CD8<32, CD8VT1>;
7183 defm VCVTSS2USI64Z: avx512_cvt_s_int_round_aliases<0x79, f32x_info, i64x_info,
7184 X86cvts2usi, WriteCvtSS2I, "cvtss2usi", "{q}">,
7185 XS, VEX_W, EVEX_CD8<32, CD8VT1>;
7186 defm VCVTSD2SIZ: avx512_cvt_s_int_round<0x2D, f64x_info, i32x_info,
7187 X86cvts2si, WriteCvtSD2I, "cvtsd2si", "{l}">,
7188 XD, EVEX_CD8<64, CD8VT1>;
7189 defm VCVTSD2SI64Z: avx512_cvt_s_int_round<0x2D, f64x_info, i64x_info,
7190 X86cvts2si, WriteCvtSD2I, "cvtsd2si", "{q}">,
7191 XD, VEX_W, EVEX_CD8<64, CD8VT1>;
7192 defm VCVTSD2USIZ: avx512_cvt_s_int_round_aliases<0x79, f64x_info, i32x_info,
7193 X86cvts2usi, WriteCvtSD2I, "cvtsd2usi", "{l}">,
7194 XD, EVEX_CD8<64, CD8VT1>;
7195 defm VCVTSD2USI64Z: avx512_cvt_s_int_round_aliases<0x79, f64x_info, i64x_info,
7196 X86cvts2usi, WriteCvtSD2I, "cvtsd2usi", "{q}">,
7197 XD, VEX_W, EVEX_CD8<64, CD8VT1>;
7199 // The SSE version of these instructions are disabled for AVX512.
7200 // Therefore, the SSE intrinsics are mapped to the AVX512 instructions.
7201 let Predicates = [HasAVX512] in {
7202 def : Pat<(i32 (int_x86_sse_cvtss2si (v4f32 VR128X:$src))),
7203 (VCVTSS2SIZrr_Int VR128X:$src)>;
7204 def : Pat<(i32 (int_x86_sse_cvtss2si sse_load_f32:$src)),
7205 (VCVTSS2SIZrm_Int sse_load_f32:$src)>;
7206 def : Pat<(i64 (int_x86_sse_cvtss2si64 (v4f32 VR128X:$src))),
7207 (VCVTSS2SI64Zrr_Int VR128X:$src)>;
7208 def : Pat<(i64 (int_x86_sse_cvtss2si64 sse_load_f32:$src)),
7209 (VCVTSS2SI64Zrm_Int sse_load_f32:$src)>;
7210 def : Pat<(i32 (int_x86_sse2_cvtsd2si (v2f64 VR128X:$src))),
7211 (VCVTSD2SIZrr_Int VR128X:$src)>;
7212 def : Pat<(i32 (int_x86_sse2_cvtsd2si sse_load_f64:$src)),
7213 (VCVTSD2SIZrm_Int sse_load_f64:$src)>;
7214 def : Pat<(i64 (int_x86_sse2_cvtsd2si64 (v2f64 VR128X:$src))),
7215 (VCVTSD2SI64Zrr_Int VR128X:$src)>;
7216 def : Pat<(i64 (int_x86_sse2_cvtsd2si64 sse_load_f64:$src)),
7217 (VCVTSD2SI64Zrm_Int sse_load_f64:$src)>;
7220 // Patterns used for matching vcvtsi2s{s,d} intrinsic sequences from clang
7221 // which produce unnecessary vmovs{s,d} instructions
7222 let Predicates = [HasAVX512] in {
7223 def : Pat<(v4f32 (X86Movss
7224 (v4f32 VR128X:$dst),
7225 (v4f32 (scalar_to_vector (f32 (sint_to_fp GR64:$src)))))),
7226 (VCVTSI642SSZrr_Int VR128X:$dst, GR64:$src)>;
7228 def : Pat<(v4f32 (X86Movss
7229 (v4f32 VR128X:$dst),
7230 (v4f32 (scalar_to_vector (f32 (sint_to_fp (loadi64 addr:$src))))))),
7231 (VCVTSI642SSZrm_Int VR128X:$dst, addr:$src)>;
7233 def : Pat<(v4f32 (X86Movss
7234 (v4f32 VR128X:$dst),
7235 (v4f32 (scalar_to_vector (f32 (sint_to_fp GR32:$src)))))),
7236 (VCVTSI2SSZrr_Int VR128X:$dst, GR32:$src)>;
7238 def : Pat<(v4f32 (X86Movss
7239 (v4f32 VR128X:$dst),
7240 (v4f32 (scalar_to_vector (f32 (sint_to_fp (loadi32 addr:$src))))))),
7241 (VCVTSI2SSZrm_Int VR128X:$dst, addr:$src)>;
7243 def : Pat<(v2f64 (X86Movsd
7244 (v2f64 VR128X:$dst),
7245 (v2f64 (scalar_to_vector (f64 (sint_to_fp GR64:$src)))))),
7246 (VCVTSI642SDZrr_Int VR128X:$dst, GR64:$src)>;
7248 def : Pat<(v2f64 (X86Movsd
7249 (v2f64 VR128X:$dst),
7250 (v2f64 (scalar_to_vector (f64 (sint_to_fp (loadi64 addr:$src))))))),
7251 (VCVTSI642SDZrm_Int VR128X:$dst, addr:$src)>;
7253 def : Pat<(v2f64 (X86Movsd
7254 (v2f64 VR128X:$dst),
7255 (v2f64 (scalar_to_vector (f64 (sint_to_fp GR32:$src)))))),
7256 (VCVTSI2SDZrr_Int VR128X:$dst, GR32:$src)>;
7258 def : Pat<(v2f64 (X86Movsd
7259 (v2f64 VR128X:$dst),
7260 (v2f64 (scalar_to_vector (f64 (sint_to_fp (loadi32 addr:$src))))))),
7261 (VCVTSI2SDZrm_Int VR128X:$dst, addr:$src)>;
7263 def : Pat<(v4f32 (X86Movss
7264 (v4f32 VR128X:$dst),
7265 (v4f32 (scalar_to_vector (f32 (uint_to_fp GR64:$src)))))),
7266 (VCVTUSI642SSZrr_Int VR128X:$dst, GR64:$src)>;
7268 def : Pat<(v4f32 (X86Movss
7269 (v4f32 VR128X:$dst),
7270 (v4f32 (scalar_to_vector (f32 (uint_to_fp (loadi64 addr:$src))))))),
7271 (VCVTUSI642SSZrm_Int VR128X:$dst, addr:$src)>;
7273 def : Pat<(v4f32 (X86Movss
7274 (v4f32 VR128X:$dst),
7275 (v4f32 (scalar_to_vector (f32 (uint_to_fp GR32:$src)))))),
7276 (VCVTUSI2SSZrr_Int VR128X:$dst, GR32:$src)>;
7278 def : Pat<(v4f32 (X86Movss
7279 (v4f32 VR128X:$dst),
7280 (v4f32 (scalar_to_vector (f32 (uint_to_fp (loadi32 addr:$src))))))),
7281 (VCVTUSI2SSZrm_Int VR128X:$dst, addr:$src)>;
7283 def : Pat<(v2f64 (X86Movsd
7284 (v2f64 VR128X:$dst),
7285 (v2f64 (scalar_to_vector (f64 (uint_to_fp GR64:$src)))))),
7286 (VCVTUSI642SDZrr_Int VR128X:$dst, GR64:$src)>;
7288 def : Pat<(v2f64 (X86Movsd
7289 (v2f64 VR128X:$dst),
7290 (v2f64 (scalar_to_vector (f64 (uint_to_fp (loadi64 addr:$src))))))),
7291 (VCVTUSI642SDZrm_Int VR128X:$dst, addr:$src)>;
7293 def : Pat<(v2f64 (X86Movsd
7294 (v2f64 VR128X:$dst),
7295 (v2f64 (scalar_to_vector (f64 (uint_to_fp GR32:$src)))))),
7296 (VCVTUSI2SDZrr_Int VR128X:$dst, GR32:$src)>;
7298 def : Pat<(v2f64 (X86Movsd
7299 (v2f64 VR128X:$dst),
7300 (v2f64 (scalar_to_vector (f64 (uint_to_fp (loadi32 addr:$src))))))),
7301 (VCVTUSI2SDZrm_Int VR128X:$dst, addr:$src)>;
7302 } // Predicates = [HasAVX512]
7304 // Convert float/double to signed/unsigned int 32/64 with truncation
7305 multiclass avx512_cvt_s_all<bits<8> opc, string asm, X86VectorVTInfo _SrcRC,
7306 X86VectorVTInfo _DstRC, SDNode OpNode,
7307 SDNode OpNodeRnd, X86FoldableSchedWrite sched,
7308 string aliasStr, bit CodeGenOnly = 1>{
7309 let Predicates = [HasAVX512] in {
7310 let isCodeGenOnly = 1 in {
7311 def rr : AVX512<opc, MRMSrcReg, (outs _DstRC.RC:$dst), (ins _SrcRC.FRC:$src),
7312 !strconcat(asm,"\t{$src, $dst|$dst, $src}"),
7313 [(set _DstRC.RC:$dst, (OpNode _SrcRC.FRC:$src))]>,
7314 EVEX, Sched<[sched]>;
7315 def rm : AVX512<opc, MRMSrcMem, (outs _DstRC.RC:$dst), (ins _SrcRC.ScalarMemOp:$src),
7316 !strconcat(asm,"\t{$src, $dst|$dst, $src}"),
7317 [(set _DstRC.RC:$dst, (OpNode (_SrcRC.ScalarLdFrag addr:$src)))]>,
7318 EVEX, Sched<[sched.Folded, ReadAfterLd]>;
7321 def rr_Int : AVX512<opc, MRMSrcReg, (outs _DstRC.RC:$dst), (ins _SrcRC.RC:$src),
7322 !strconcat(asm,"\t{$src, $dst|$dst, $src}"),
7323 [(set _DstRC.RC:$dst, (OpNodeRnd (_SrcRC.VT _SrcRC.RC:$src),
7324 (i32 FROUND_CURRENT)))]>,
7325 EVEX, VEX_LIG, Sched<[sched]>;
7326 def rrb_Int : AVX512<opc, MRMSrcReg, (outs _DstRC.RC:$dst), (ins _SrcRC.RC:$src),
7327 !strconcat(asm,"\t{{sae}, $src, $dst|$dst, $src, {sae}}"),
7328 [(set _DstRC.RC:$dst, (OpNodeRnd (_SrcRC.VT _SrcRC.RC:$src),
7329 (i32 FROUND_NO_EXC)))]>,
7330 EVEX,VEX_LIG , EVEX_B, Sched<[sched]>;
7331 let isCodeGenOnly = CodeGenOnly, ForceDisassemble = CodeGenOnly in
7332 def rm_Int : AVX512<opc, MRMSrcMem, (outs _DstRC.RC:$dst),
7333 (ins _SrcRC.IntScalarMemOp:$src),
7334 !strconcat(asm,"\t{$src, $dst|$dst, $src}"),
7335 [(set _DstRC.RC:$dst, (OpNodeRnd
7336 (_SrcRC.VT _SrcRC.ScalarIntMemCPat:$src),
7337 (i32 FROUND_CURRENT)))]>,
7338 EVEX, VEX_LIG, Sched<[sched.Folded, ReadAfterLd]>;
7340 def : InstAlias<asm # aliasStr # "\t{$src, $dst|$dst, $src}",
7341 (!cast<Instruction>(NAME # "rr_Int") _DstRC.RC:$dst, _SrcRC.RC:$src), 0, "att">;
7342 def : InstAlias<asm # aliasStr # "\t{{sae}, $src, $dst|$dst, $src, {sae}}",
7343 (!cast<Instruction>(NAME # "rrb_Int") _DstRC.RC:$dst, _SrcRC.RC:$src), 0, "att">;
7347 multiclass avx512_cvt_s_all_unsigned<bits<8> opc, string asm,
7348 X86VectorVTInfo _SrcRC,
7349 X86VectorVTInfo _DstRC, SDNode OpNode,
7350 SDNode OpNodeRnd, X86FoldableSchedWrite sched,
7352 avx512_cvt_s_all<opc, asm, _SrcRC, _DstRC, OpNode, OpNodeRnd, sched,
7354 let Predicates = [HasAVX512] in {
7355 def : InstAlias<asm # aliasStr # "\t{$src, $dst|$dst, $src}",
7356 (!cast<Instruction>(NAME # "rm_Int") _DstRC.RC:$dst,
7357 _SrcRC.IntScalarMemOp:$src), 0, "att">;
7361 defm VCVTTSS2SIZ: avx512_cvt_s_all<0x2C, "vcvttss2si", f32x_info, i32x_info,
7362 fp_to_sint, X86cvtts2IntRnd, WriteCvtSS2I, "{l}">,
7363 XS, EVEX_CD8<32, CD8VT1>;
7364 defm VCVTTSS2SI64Z: avx512_cvt_s_all<0x2C, "vcvttss2si", f32x_info, i64x_info,
7365 fp_to_sint, X86cvtts2IntRnd, WriteCvtSS2I, "{q}">,
7366 VEX_W, XS, EVEX_CD8<32, CD8VT1>;
7367 defm VCVTTSD2SIZ: avx512_cvt_s_all<0x2C, "vcvttsd2si", f64x_info, i32x_info,
7368 fp_to_sint, X86cvtts2IntRnd, WriteCvtSD2I, "{l}">,
7369 XD, EVEX_CD8<64, CD8VT1>;
7370 defm VCVTTSD2SI64Z: avx512_cvt_s_all<0x2C, "vcvttsd2si", f64x_info, i64x_info,
7371 fp_to_sint, X86cvtts2IntRnd, WriteCvtSD2I, "{q}">,
7372 VEX_W, XD, EVEX_CD8<64, CD8VT1>;
7374 defm VCVTTSS2USIZ: avx512_cvt_s_all_unsigned<0x78, "vcvttss2usi", f32x_info, i32x_info,
7375 fp_to_uint, X86cvtts2UIntRnd, WriteCvtSS2I, "{l}">,
7376 XS, EVEX_CD8<32, CD8VT1>;
7377 defm VCVTTSS2USI64Z: avx512_cvt_s_all_unsigned<0x78, "vcvttss2usi", f32x_info, i64x_info,
7378 fp_to_uint, X86cvtts2UIntRnd, WriteCvtSS2I, "{q}">,
7379 XS,VEX_W, EVEX_CD8<32, CD8VT1>;
7380 defm VCVTTSD2USIZ: avx512_cvt_s_all_unsigned<0x78, "vcvttsd2usi", f64x_info, i32x_info,
7381 fp_to_uint, X86cvtts2UIntRnd, WriteCvtSD2I, "{l}">,
7382 XD, EVEX_CD8<64, CD8VT1>;
7383 defm VCVTTSD2USI64Z: avx512_cvt_s_all_unsigned<0x78, "vcvttsd2usi", f64x_info, i64x_info,
7384 fp_to_uint, X86cvtts2UIntRnd, WriteCvtSD2I, "{q}">,
7385 XD, VEX_W, EVEX_CD8<64, CD8VT1>;
7387 let Predicates = [HasAVX512] in {
7388 def : Pat<(i32 (int_x86_sse_cvttss2si (v4f32 VR128X:$src))),
7389 (VCVTTSS2SIZrr_Int VR128X:$src)>;
7390 def : Pat<(i32 (int_x86_sse_cvttss2si sse_load_f32:$src)),
7391 (VCVTTSS2SIZrm_Int ssmem:$src)>;
7392 def : Pat<(i64 (int_x86_sse_cvttss2si64 (v4f32 VR128X:$src))),
7393 (VCVTTSS2SI64Zrr_Int VR128X:$src)>;
7394 def : Pat<(i64 (int_x86_sse_cvttss2si64 sse_load_f32:$src)),
7395 (VCVTTSS2SI64Zrm_Int ssmem:$src)>;
7396 def : Pat<(i32 (int_x86_sse2_cvttsd2si (v2f64 VR128X:$src))),
7397 (VCVTTSD2SIZrr_Int VR128X:$src)>;
7398 def : Pat<(i32 (int_x86_sse2_cvttsd2si sse_load_f64:$src)),
7399 (VCVTTSD2SIZrm_Int sdmem:$src)>;
7400 def : Pat<(i64 (int_x86_sse2_cvttsd2si64 (v2f64 VR128X:$src))),
7401 (VCVTTSD2SI64Zrr_Int VR128X:$src)>;
7402 def : Pat<(i64 (int_x86_sse2_cvttsd2si64 sse_load_f64:$src)),
7403 (VCVTTSD2SI64Zrm_Int sdmem:$src)>;
7406 //===----------------------------------------------------------------------===//
7407 // AVX-512 Convert form float to double and back
7408 //===----------------------------------------------------------------------===//
7410 multiclass avx512_cvt_fp_scalar<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
7411 X86VectorVTInfo _Src, SDNode OpNode,
7412 X86FoldableSchedWrite sched> {
7413 defm rr_Int : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
7414 (ins _.RC:$src1, _Src.RC:$src2), OpcodeStr,
7415 "$src2, $src1", "$src1, $src2",
7416 (_.VT (OpNode (_.VT _.RC:$src1),
7417 (_Src.VT _Src.RC:$src2),
7418 (i32 FROUND_CURRENT)))>,
7419 EVEX_4V, VEX_LIG, Sched<[sched]>;
7420 defm rm_Int : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
7421 (ins _.RC:$src1, _Src.IntScalarMemOp:$src2), OpcodeStr,
7422 "$src2, $src1", "$src1, $src2",
7423 (_.VT (OpNode (_.VT _.RC:$src1),
7424 (_Src.VT _Src.ScalarIntMemCPat:$src2),
7425 (i32 FROUND_CURRENT)))>,
7427 Sched<[sched.Folded, ReadAfterLd]>;
7429 let isCodeGenOnly = 1, hasSideEffects = 0 in {
7430 def rr : I<opc, MRMSrcReg, (outs _.FRC:$dst),
7431 (ins _.FRC:$src1, _Src.FRC:$src2),
7432 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>,
7433 EVEX_4V, VEX_LIG, Sched<[sched]>;
7435 def rm : I<opc, MRMSrcMem, (outs _.FRC:$dst),
7436 (ins _.FRC:$src1, _Src.ScalarMemOp:$src2),
7437 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>,
7438 EVEX_4V, VEX_LIG, Sched<[sched.Folded, ReadAfterLd]>;
7442 // Scalar Coversion with SAE - suppress all exceptions
7443 multiclass avx512_cvt_fp_sae_scalar<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
7444 X86VectorVTInfo _Src, SDNode OpNodeRnd,
7445 X86FoldableSchedWrite sched> {
7446 defm rrb_Int : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
7447 (ins _.RC:$src1, _Src.RC:$src2), OpcodeStr,
7448 "{sae}, $src2, $src1", "$src1, $src2, {sae}",
7449 (_.VT (OpNodeRnd (_.VT _.RC:$src1),
7450 (_Src.VT _Src.RC:$src2),
7451 (i32 FROUND_NO_EXC)))>,
7452 EVEX_4V, VEX_LIG, EVEX_B, Sched<[sched]>;
7455 // Scalar Conversion with rounding control (RC)
7456 multiclass avx512_cvt_fp_rc_scalar<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
7457 X86VectorVTInfo _Src, SDNode OpNodeRnd,
7458 X86FoldableSchedWrite sched> {
7459 defm rrb_Int : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
7460 (ins _.RC:$src1, _Src.RC:$src2, AVX512RC:$rc), OpcodeStr,
7461 "$rc, $src2, $src1", "$src1, $src2, $rc",
7462 (_.VT (OpNodeRnd (_.VT _.RC:$src1),
7463 (_Src.VT _Src.RC:$src2), (i32 imm:$rc)))>,
7464 EVEX_4V, VEX_LIG, Sched<[sched]>,
7467 multiclass avx512_cvt_fp_scalar_sd2ss<bits<8> opc, string OpcodeStr,
7468 SDNode OpNodeRnd, X86FoldableSchedWrite sched,
7469 X86VectorVTInfo _src, X86VectorVTInfo _dst> {
7470 let Predicates = [HasAVX512] in {
7471 defm Z : avx512_cvt_fp_scalar<opc, OpcodeStr, _dst, _src, OpNodeRnd, sched>,
7472 avx512_cvt_fp_rc_scalar<opc, OpcodeStr, _dst, _src,
7473 OpNodeRnd, sched>, VEX_W, EVEX_CD8<64, CD8VT1>, XD;
7477 multiclass avx512_cvt_fp_scalar_ss2sd<bits<8> opc, string OpcodeStr, SDNode OpNodeRnd,
7478 X86FoldableSchedWrite sched,
7479 X86VectorVTInfo _src, X86VectorVTInfo _dst> {
7480 let Predicates = [HasAVX512] in {
7481 defm Z : avx512_cvt_fp_scalar<opc, OpcodeStr, _dst, _src, OpNodeRnd, sched>,
7482 avx512_cvt_fp_sae_scalar<opc, OpcodeStr, _dst, _src, OpNodeRnd, sched>,
7483 EVEX_CD8<32, CD8VT1>, XS;
7486 defm VCVTSD2SS : avx512_cvt_fp_scalar_sd2ss<0x5A, "vcvtsd2ss",
7487 X86froundRnd, WriteCvtSD2SS, f64x_info,
7489 defm VCVTSS2SD : avx512_cvt_fp_scalar_ss2sd<0x5A, "vcvtss2sd",
7490 X86fpextRnd, WriteCvtSS2SD, f32x_info,
7493 def : Pat<(f64 (fpextend FR32X:$src)),
7494 (VCVTSS2SDZrr (f64 (IMPLICIT_DEF)), FR32X:$src)>,
7495 Requires<[HasAVX512]>;
7496 def : Pat<(f64 (fpextend (loadf32 addr:$src))),
7497 (VCVTSS2SDZrm (f64 (IMPLICIT_DEF)), addr:$src)>,
7498 Requires<[HasAVX512, OptForSize]>;
7500 def : Pat<(f64 (extloadf32 addr:$src)),
7501 (VCVTSS2SDZrm (f64 (IMPLICIT_DEF)), addr:$src)>,
7502 Requires<[HasAVX512, OptForSize]>;
7504 def : Pat<(f64 (extloadf32 addr:$src)),
7505 (VCVTSS2SDZrr (f64 (IMPLICIT_DEF)), (VMOVSSZrm addr:$src))>,
7506 Requires<[HasAVX512, OptForSpeed]>;
7508 def : Pat<(f32 (fpround FR64X:$src)),
7509 (VCVTSD2SSZrr (f32 (IMPLICIT_DEF)), FR64X:$src)>,
7510 Requires<[HasAVX512]>;
7512 def : Pat<(v4f32 (X86Movss
7513 (v4f32 VR128X:$dst),
7514 (v4f32 (scalar_to_vector
7515 (f32 (fpround (f64 (extractelt VR128X:$src, (iPTR 0))))))))),
7516 (VCVTSD2SSZrr_Int VR128X:$dst, VR128X:$src)>,
7517 Requires<[HasAVX512]>;
7519 def : Pat<(v2f64 (X86Movsd
7520 (v2f64 VR128X:$dst),
7521 (v2f64 (scalar_to_vector
7522 (f64 (fpextend (f32 (extractelt VR128X:$src, (iPTR 0))))))))),
7523 (VCVTSS2SDZrr_Int VR128X:$dst, VR128X:$src)>,
7524 Requires<[HasAVX512]>;
7526 //===----------------------------------------------------------------------===//
7527 // AVX-512 Vector convert from signed/unsigned integer to float/double
7528 // and from float/double to signed/unsigned integer
7529 //===----------------------------------------------------------------------===//
7531 multiclass avx512_vcvt_fp<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
7532 X86VectorVTInfo _Src, SDNode OpNode,
7533 X86FoldableSchedWrite sched,
7534 string Broadcast = _.BroadcastStr,
7535 string Alias = "", X86MemOperand MemOp = _Src.MemOp> {
7537 defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
7538 (ins _Src.RC:$src), OpcodeStr, "$src", "$src",
7539 (_.VT (OpNode (_Src.VT _Src.RC:$src)))>,
7540 EVEX, Sched<[sched]>;
7542 defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
7543 (ins MemOp:$src), OpcodeStr#Alias, "$src", "$src",
7544 (_.VT (OpNode (_Src.VT
7545 (bitconvert (_Src.LdFrag addr:$src)))))>,
7546 EVEX, Sched<[sched.Folded]>;
7548 defm rmb : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
7549 (ins _Src.ScalarMemOp:$src), OpcodeStr,
7550 "${src}"##Broadcast, "${src}"##Broadcast,
7551 (_.VT (OpNode (_Src.VT
7552 (X86VBroadcast (_Src.ScalarLdFrag addr:$src)))
7554 Sched<[sched.Folded]>;
7556 // Coversion with SAE - suppress all exceptions
7557 multiclass avx512_vcvt_fp_sae<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
7558 X86VectorVTInfo _Src, SDNode OpNodeRnd,
7559 X86FoldableSchedWrite sched> {
7560 defm rrb : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
7561 (ins _Src.RC:$src), OpcodeStr,
7562 "{sae}, $src", "$src, {sae}",
7563 (_.VT (OpNodeRnd (_Src.VT _Src.RC:$src),
7564 (i32 FROUND_NO_EXC)))>,
7565 EVEX, EVEX_B, Sched<[sched]>;
7568 // Conversion with rounding control (RC)
7569 multiclass avx512_vcvt_fp_rc<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
7570 X86VectorVTInfo _Src, SDNode OpNodeRnd,
7571 X86FoldableSchedWrite sched> {
7572 defm rrb : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
7573 (ins _Src.RC:$src, AVX512RC:$rc), OpcodeStr,
7574 "$rc, $src", "$src, $rc",
7575 (_.VT (OpNodeRnd (_Src.VT _Src.RC:$src), (i32 imm:$rc)))>,
7576 EVEX, EVEX_B, EVEX_RC, Sched<[sched]>;
7579 // Extend Float to Double
7580 multiclass avx512_cvtps2pd<bits<8> opc, string OpcodeStr,
7581 X86SchedWriteWidths sched> {
7582 let Predicates = [HasAVX512] in {
7583 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f64_info, v8f32x_info,
7584 fpextend, sched.ZMM>,
7585 avx512_vcvt_fp_sae<opc, OpcodeStr, v8f64_info, v8f32x_info,
7586 X86vfpextRnd, sched.ZMM>, EVEX_V512;
7588 let Predicates = [HasVLX] in {
7589 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2f64x_info, v4f32x_info,
7590 X86vfpext, sched.XMM, "{1to2}", "", f64mem>, EVEX_V128;
7591 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f64x_info, v4f32x_info, fpextend,
7592 sched.YMM>, EVEX_V256;
7596 // Truncate Double to Float
7597 multiclass avx512_cvtpd2ps<bits<8> opc, string OpcodeStr, X86SchedWriteWidths sched> {
7598 let Predicates = [HasAVX512] in {
7599 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f32x_info, v8f64_info, fpround, sched.ZMM>,
7600 avx512_vcvt_fp_rc<opc, OpcodeStr, v8f32x_info, v8f64_info,
7601 X86vfproundRnd, sched.ZMM>, EVEX_V512;
7603 let Predicates = [HasVLX] in {
7604 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v2f64x_info,
7605 X86vfpround, sched.XMM, "{1to2}", "{x}">, EVEX_V128;
7606 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v4f64x_info, fpround,
7607 sched.YMM, "{1to4}", "{y}">, EVEX_V256;
7609 def : InstAlias<OpcodeStr##"x\t{$src, $dst|$dst, $src}",
7610 (!cast<Instruction>(NAME # "Z128rr") VR128X:$dst, VR128X:$src), 0>;
7611 def : InstAlias<OpcodeStr##"x\t{$src, $dst|$dst, $src}",
7612 (!cast<Instruction>(NAME # "Z128rm") VR128X:$dst, f128mem:$src), 0, "intel">;
7613 def : InstAlias<OpcodeStr##"y\t{$src, $dst|$dst, $src}",
7614 (!cast<Instruction>(NAME # "Z256rr") VR128X:$dst, VR256X:$src), 0>;
7615 def : InstAlias<OpcodeStr##"y\t{$src, $dst|$dst, $src}",
7616 (!cast<Instruction>(NAME # "Z256rm") VR128X:$dst, f256mem:$src), 0, "intel">;
7620 defm VCVTPD2PS : avx512_cvtpd2ps<0x5A, "vcvtpd2ps", SchedWriteCvtPD2PS>,
7621 VEX_W, PD, EVEX_CD8<64, CD8VF>;
7622 defm VCVTPS2PD : avx512_cvtps2pd<0x5A, "vcvtps2pd", SchedWriteCvtPS2PD>,
7623 PS, EVEX_CD8<32, CD8VH>;
7625 def : Pat<(v8f64 (extloadv8f32 addr:$src)),
7626 (VCVTPS2PDZrm addr:$src)>;
7628 let Predicates = [HasVLX] in {
7629 let AddedComplexity = 15 in {
7630 def : Pat<(X86vzmovl (v2f64 (bitconvert
7631 (v4f32 (X86vfpround (v2f64 VR128X:$src)))))),
7632 (VCVTPD2PSZ128rr VR128X:$src)>;
7633 def : Pat<(X86vzmovl (v2f64 (bitconvert
7634 (v4f32 (X86vfpround (loadv2f64 addr:$src)))))),
7635 (VCVTPD2PSZ128rm addr:$src)>;
7637 def : Pat<(v2f64 (extloadv2f32 addr:$src)),
7638 (VCVTPS2PDZ128rm addr:$src)>;
7639 def : Pat<(v4f64 (extloadv4f32 addr:$src)),
7640 (VCVTPS2PDZ256rm addr:$src)>;
7643 // Convert Signed/Unsigned Doubleword to Double
7644 multiclass avx512_cvtdq2pd<bits<8> opc, string OpcodeStr, SDNode OpNode,
7645 SDNode OpNode128, X86SchedWriteWidths sched> {
7646 // No rounding in this op
7647 let Predicates = [HasAVX512] in
7648 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f64_info, v8i32x_info, OpNode,
7649 sched.ZMM>, EVEX_V512;
7651 let Predicates = [HasVLX] in {
7652 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2f64x_info, v4i32x_info,
7653 OpNode128, sched.XMM, "{1to2}", "", i64mem>, EVEX_V128;
7654 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f64x_info, v4i32x_info, OpNode,
7655 sched.YMM>, EVEX_V256;
7659 // Convert Signed/Unsigned Doubleword to Float
7660 multiclass avx512_cvtdq2ps<bits<8> opc, string OpcodeStr, SDNode OpNode,
7661 SDNode OpNodeRnd, X86SchedWriteWidths sched> {
7662 let Predicates = [HasAVX512] in
7663 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v16f32_info, v16i32_info, OpNode,
7665 avx512_vcvt_fp_rc<opc, OpcodeStr, v16f32_info, v16i32_info,
7666 OpNodeRnd, sched.ZMM>, EVEX_V512;
7668 let Predicates = [HasVLX] in {
7669 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v4i32x_info, OpNode,
7670 sched.XMM>, EVEX_V128;
7671 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v8f32x_info, v8i32x_info, OpNode,
7672 sched.YMM>, EVEX_V256;
7676 // Convert Float to Signed/Unsigned Doubleword with truncation
7677 multiclass avx512_cvttps2dq<bits<8> opc, string OpcodeStr, SDNode OpNode,
7678 SDNode OpNodeRnd, X86SchedWriteWidths sched> {
7679 let Predicates = [HasAVX512] in {
7680 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v16i32_info, v16f32_info, OpNode,
7682 avx512_vcvt_fp_sae<opc, OpcodeStr, v16i32_info, v16f32_info,
7683 OpNodeRnd, sched.ZMM>, EVEX_V512;
7685 let Predicates = [HasVLX] in {
7686 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v4f32x_info, OpNode,
7687 sched.XMM>, EVEX_V128;
7688 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v8i32x_info, v8f32x_info, OpNode,
7689 sched.YMM>, EVEX_V256;
7693 // Convert Float to Signed/Unsigned Doubleword
7694 multiclass avx512_cvtps2dq<bits<8> opc, string OpcodeStr, SDNode OpNode,
7695 SDNode OpNodeRnd, X86SchedWriteWidths sched> {
7696 let Predicates = [HasAVX512] in {
7697 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v16i32_info, v16f32_info, OpNode,
7699 avx512_vcvt_fp_rc<opc, OpcodeStr, v16i32_info, v16f32_info,
7700 OpNodeRnd, sched.ZMM>, EVEX_V512;
7702 let Predicates = [HasVLX] in {
7703 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v4f32x_info, OpNode,
7704 sched.XMM>, EVEX_V128;
7705 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v8i32x_info, v8f32x_info, OpNode,
7706 sched.YMM>, EVEX_V256;
7710 // Convert Double to Signed/Unsigned Doubleword with truncation
7711 multiclass avx512_cvttpd2dq<bits<8> opc, string OpcodeStr, SDNode OpNode,
7712 SDNode OpNodeRnd, X86SchedWriteWidths sched> {
7713 let Predicates = [HasAVX512] in {
7714 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i32x_info, v8f64_info, OpNode,
7716 avx512_vcvt_fp_sae<opc, OpcodeStr, v8i32x_info, v8f64_info,
7717 OpNodeRnd, sched.ZMM>, EVEX_V512;
7719 let Predicates = [HasVLX] in {
7720 // we need "x"/"y" suffixes in order to distinguish between 128 and 256
7721 // memory forms of these instructions in Asm Parser. They have the same
7722 // dest type - 'v4i32x_info'. We also specify the broadcast string explicitly
7723 // due to the same reason.
7724 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v2f64x_info,
7725 OpNode, sched.XMM, "{1to2}", "{x}">, EVEX_V128;
7726 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v4f64x_info, OpNode,
7727 sched.YMM, "{1to4}", "{y}">, EVEX_V256;
7729 def : InstAlias<OpcodeStr##"x\t{$src, $dst|$dst, $src}",
7730 (!cast<Instruction>(NAME # "Z128rr") VR128X:$dst, VR128X:$src), 0>;
7731 def : InstAlias<OpcodeStr##"x\t{$src, $dst|$dst, $src}",
7732 (!cast<Instruction>(NAME # "Z128rm") VR128X:$dst, i128mem:$src), 0, "intel">;
7733 def : InstAlias<OpcodeStr##"y\t{$src, $dst|$dst, $src}",
7734 (!cast<Instruction>(NAME # "Z256rr") VR128X:$dst, VR256X:$src), 0>;
7735 def : InstAlias<OpcodeStr##"y\t{$src, $dst|$dst, $src}",
7736 (!cast<Instruction>(NAME # "Z256rm") VR128X:$dst, i256mem:$src), 0, "intel">;
7740 // Convert Double to Signed/Unsigned Doubleword
7741 multiclass avx512_cvtpd2dq<bits<8> opc, string OpcodeStr, SDNode OpNode,
7742 SDNode OpNodeRnd, X86SchedWriteWidths sched> {
7743 let Predicates = [HasAVX512] in {
7744 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i32x_info, v8f64_info, OpNode,
7746 avx512_vcvt_fp_rc<opc, OpcodeStr, v8i32x_info, v8f64_info,
7747 OpNodeRnd, sched.ZMM>, EVEX_V512;
7749 let Predicates = [HasVLX] in {
7750 // we need "x"/"y" suffixes in order to distinguish between 128 and 256
7751 // memory forms of these instructions in Asm Parcer. They have the same
7752 // dest type - 'v4i32x_info'. We also specify the broadcast string explicitly
7753 // due to the same reason.
7754 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v2f64x_info, OpNode,
7755 sched.XMM, "{1to2}", "{x}">, EVEX_V128;
7756 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i32x_info, v4f64x_info, OpNode,
7757 sched.YMM, "{1to4}", "{y}">, EVEX_V256;
7759 def : InstAlias<OpcodeStr##"x\t{$src, $dst|$dst, $src}",
7760 (!cast<Instruction>(NAME # "Z128rr") VR128X:$dst, VR128X:$src), 0>;
7761 def : InstAlias<OpcodeStr##"x\t{$src, $dst|$dst, $src}",
7762 (!cast<Instruction>(NAME # "Z128rm") VR128X:$dst, f128mem:$src), 0, "intel">;
7763 def : InstAlias<OpcodeStr##"y\t{$src, $dst|$dst, $src}",
7764 (!cast<Instruction>(NAME # "Z256rr") VR128X:$dst, VR256X:$src), 0>;
7765 def : InstAlias<OpcodeStr##"y\t{$src, $dst|$dst, $src}",
7766 (!cast<Instruction>(NAME # "Z256rm") VR128X:$dst, f256mem:$src), 0, "intel">;
7770 // Convert Double to Signed/Unsigned Quardword
7771 multiclass avx512_cvtpd2qq<bits<8> opc, string OpcodeStr, SDNode OpNode,
7772 SDNode OpNodeRnd, X86SchedWriteWidths sched> {
7773 let Predicates = [HasDQI] in {
7774 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i64_info, v8f64_info, OpNode,
7776 avx512_vcvt_fp_rc<opc, OpcodeStr, v8i64_info, v8f64_info,
7777 OpNodeRnd, sched.ZMM>, EVEX_V512;
7779 let Predicates = [HasDQI, HasVLX] in {
7780 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2i64x_info, v2f64x_info, OpNode,
7781 sched.XMM>, EVEX_V128;
7782 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i64x_info, v4f64x_info, OpNode,
7783 sched.YMM>, EVEX_V256;
7787 // Convert Double to Signed/Unsigned Quardword with truncation
7788 multiclass avx512_cvttpd2qq<bits<8> opc, string OpcodeStr, SDNode OpNode,
7789 SDNode OpNodeRnd, X86SchedWriteWidths sched> {
7790 let Predicates = [HasDQI] in {
7791 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i64_info, v8f64_info, OpNode,
7793 avx512_vcvt_fp_sae<opc, OpcodeStr, v8i64_info, v8f64_info,
7794 OpNodeRnd, sched.ZMM>, EVEX_V512;
7796 let Predicates = [HasDQI, HasVLX] in {
7797 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2i64x_info, v2f64x_info, OpNode,
7798 sched.XMM>, EVEX_V128;
7799 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i64x_info, v4f64x_info, OpNode,
7800 sched.YMM>, EVEX_V256;
7804 // Convert Signed/Unsigned Quardword to Double
7805 multiclass avx512_cvtqq2pd<bits<8> opc, string OpcodeStr, SDNode OpNode,
7806 SDNode OpNodeRnd, X86SchedWriteWidths sched> {
7807 let Predicates = [HasDQI] in {
7808 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f64_info, v8i64_info, OpNode,
7810 avx512_vcvt_fp_rc<opc, OpcodeStr, v8f64_info, v8i64_info,
7811 OpNodeRnd, sched.ZMM>, EVEX_V512;
7813 let Predicates = [HasDQI, HasVLX] in {
7814 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2f64x_info, v2i64x_info, OpNode,
7815 sched.XMM>, EVEX_V128, NotEVEX2VEXConvertible;
7816 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f64x_info, v4i64x_info, OpNode,
7817 sched.YMM>, EVEX_V256, NotEVEX2VEXConvertible;
7821 // Convert Float to Signed/Unsigned Quardword
7822 multiclass avx512_cvtps2qq<bits<8> opc, string OpcodeStr, SDNode OpNode,
7823 SDNode OpNodeRnd, X86SchedWriteWidths sched> {
7824 let Predicates = [HasDQI] in {
7825 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i64_info, v8f32x_info, OpNode,
7827 avx512_vcvt_fp_rc<opc, OpcodeStr, v8i64_info, v8f32x_info,
7828 OpNodeRnd, sched.ZMM>, EVEX_V512;
7830 let Predicates = [HasDQI, HasVLX] in {
7831 // Explicitly specified broadcast string, since we take only 2 elements
7832 // from v4f32x_info source
7833 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2i64x_info, v4f32x_info, OpNode,
7834 sched.XMM, "{1to2}", "", f64mem>, EVEX_V128;
7835 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i64x_info, v4f32x_info, OpNode,
7836 sched.YMM>, EVEX_V256;
7840 // Convert Float to Signed/Unsigned Quardword with truncation
7841 multiclass avx512_cvttps2qq<bits<8> opc, string OpcodeStr, SDNode OpNode,
7842 SDNode OpNodeRnd, X86SchedWriteWidths sched> {
7843 let Predicates = [HasDQI] in {
7844 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8i64_info, v8f32x_info, OpNode, sched.ZMM>,
7845 avx512_vcvt_fp_sae<opc, OpcodeStr, v8i64_info, v8f32x_info,
7846 OpNodeRnd, sched.ZMM>, EVEX_V512;
7848 let Predicates = [HasDQI, HasVLX] in {
7849 // Explicitly specified broadcast string, since we take only 2 elements
7850 // from v4f32x_info source
7851 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v2i64x_info, v4f32x_info, OpNode,
7852 sched.XMM, "{1to2}", "", f64mem>, EVEX_V128;
7853 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4i64x_info, v4f32x_info, OpNode,
7854 sched.YMM>, EVEX_V256;
7858 // Convert Signed/Unsigned Quardword to Float
7859 multiclass avx512_cvtqq2ps<bits<8> opc, string OpcodeStr, SDNode OpNode,
7860 SDNode OpNode128, SDNode OpNodeRnd,
7861 X86SchedWriteWidths sched> {
7862 let Predicates = [HasDQI] in {
7863 defm Z : avx512_vcvt_fp<opc, OpcodeStr, v8f32x_info, v8i64_info, OpNode,
7865 avx512_vcvt_fp_rc<opc, OpcodeStr, v8f32x_info, v8i64_info,
7866 OpNodeRnd, sched.ZMM>, EVEX_V512;
7868 let Predicates = [HasDQI, HasVLX] in {
7869 // we need "x"/"y" suffixes in order to distinguish between 128 and 256
7870 // memory forms of these instructions in Asm Parcer. They have the same
7871 // dest type - 'v4i32x_info'. We also specify the broadcast string explicitly
7872 // due to the same reason.
7873 defm Z128 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v2i64x_info, OpNode128,
7874 sched.XMM, "{1to2}", "{x}">, EVEX_V128,
7875 NotEVEX2VEXConvertible;
7876 defm Z256 : avx512_vcvt_fp<opc, OpcodeStr, v4f32x_info, v4i64x_info, OpNode,
7877 sched.YMM, "{1to4}", "{y}">, EVEX_V256,
7878 NotEVEX2VEXConvertible;
7880 def : InstAlias<OpcodeStr##"x\t{$src, $dst|$dst, $src}",
7881 (!cast<Instruction>(NAME # "Z128rr") VR128X:$dst, VR128X:$src), 0>;
7882 def : InstAlias<OpcodeStr##"x\t{$src, $dst|$dst, $src}",
7883 (!cast<Instruction>(NAME # "Z128rm") VR128X:$dst, i128mem:$src), 0, "intel">;
7884 def : InstAlias<OpcodeStr##"y\t{$src, $dst|$dst, $src}",
7885 (!cast<Instruction>(NAME # "Z256rr") VR128X:$dst, VR256X:$src), 0>;
7886 def : InstAlias<OpcodeStr##"y\t{$src, $dst|$dst, $src}",
7887 (!cast<Instruction>(NAME # "Z256rm") VR128X:$dst, i256mem:$src), 0, "intel">;
7891 defm VCVTDQ2PD : avx512_cvtdq2pd<0xE6, "vcvtdq2pd", sint_to_fp, X86VSintToFP,
7892 SchedWriteCvtDQ2PD>, XS, EVEX_CD8<32, CD8VH>;
7894 defm VCVTDQ2PS : avx512_cvtdq2ps<0x5B, "vcvtdq2ps", sint_to_fp,
7895 X86VSintToFpRnd, SchedWriteCvtDQ2PS>,
7896 PS, EVEX_CD8<32, CD8VF>;
7898 defm VCVTTPS2DQ : avx512_cvttps2dq<0x5B, "vcvttps2dq", X86cvttp2si,
7899 X86cvttp2siRnd, SchedWriteCvtPS2DQ>,
7900 XS, EVEX_CD8<32, CD8VF>;
7902 defm VCVTTPD2DQ : avx512_cvttpd2dq<0xE6, "vcvttpd2dq", X86cvttp2si,
7903 X86cvttp2siRnd, SchedWriteCvtPD2DQ>,
7904 PD, VEX_W, EVEX_CD8<64, CD8VF>;
7906 defm VCVTTPS2UDQ : avx512_cvttps2dq<0x78, "vcvttps2udq", X86cvttp2ui,
7907 X86cvttp2uiRnd, SchedWriteCvtPS2DQ>, PS,
7908 EVEX_CD8<32, CD8VF>;
7910 defm VCVTTPD2UDQ : avx512_cvttpd2dq<0x78, "vcvttpd2udq", X86cvttp2ui,
7911 X86cvttp2uiRnd, SchedWriteCvtPD2DQ>,
7912 PS, VEX_W, EVEX_CD8<64, CD8VF>;
7914 defm VCVTUDQ2PD : avx512_cvtdq2pd<0x7A, "vcvtudq2pd", uint_to_fp,
7915 X86VUintToFP, SchedWriteCvtDQ2PD>, XS,
7916 EVEX_CD8<32, CD8VH>;
7918 defm VCVTUDQ2PS : avx512_cvtdq2ps<0x7A, "vcvtudq2ps", uint_to_fp,
7919 X86VUintToFpRnd, SchedWriteCvtDQ2PS>, XD,
7920 EVEX_CD8<32, CD8VF>;
7922 defm VCVTPS2DQ : avx512_cvtps2dq<0x5B, "vcvtps2dq", X86cvtp2Int,
7923 X86cvtp2IntRnd, SchedWriteCvtPS2DQ>, PD,
7924 EVEX_CD8<32, CD8VF>;
7926 defm VCVTPD2DQ : avx512_cvtpd2dq<0xE6, "vcvtpd2dq", X86cvtp2Int,
7927 X86cvtp2IntRnd, SchedWriteCvtPD2DQ>, XD,
7928 VEX_W, EVEX_CD8<64, CD8VF>;
7930 defm VCVTPS2UDQ : avx512_cvtps2dq<0x79, "vcvtps2udq", X86cvtp2UInt,
7931 X86cvtp2UIntRnd, SchedWriteCvtPS2DQ>,
7932 PS, EVEX_CD8<32, CD8VF>;
7934 defm VCVTPD2UDQ : avx512_cvtpd2dq<0x79, "vcvtpd2udq", X86cvtp2UInt,
7935 X86cvtp2UIntRnd, SchedWriteCvtPD2DQ>, VEX_W,
7936 PS, EVEX_CD8<64, CD8VF>;
7938 defm VCVTPD2QQ : avx512_cvtpd2qq<0x7B, "vcvtpd2qq", X86cvtp2Int,
7939 X86cvtp2IntRnd, SchedWriteCvtPD2DQ>, VEX_W,
7940 PD, EVEX_CD8<64, CD8VF>;
7942 defm VCVTPS2QQ : avx512_cvtps2qq<0x7B, "vcvtps2qq", X86cvtp2Int,
7943 X86cvtp2IntRnd, SchedWriteCvtPS2DQ>, PD,
7944 EVEX_CD8<32, CD8VH>;
7946 defm VCVTPD2UQQ : avx512_cvtpd2qq<0x79, "vcvtpd2uqq", X86cvtp2UInt,
7947 X86cvtp2UIntRnd, SchedWriteCvtPD2DQ>, VEX_W,
7948 PD, EVEX_CD8<64, CD8VF>;
7950 defm VCVTPS2UQQ : avx512_cvtps2qq<0x79, "vcvtps2uqq", X86cvtp2UInt,
7951 X86cvtp2UIntRnd, SchedWriteCvtPS2DQ>, PD,
7952 EVEX_CD8<32, CD8VH>;
7954 defm VCVTTPD2QQ : avx512_cvttpd2qq<0x7A, "vcvttpd2qq", X86cvttp2si,
7955 X86cvttp2siRnd, SchedWriteCvtPD2DQ>, VEX_W,
7956 PD, EVEX_CD8<64, CD8VF>;
7958 defm VCVTTPS2QQ : avx512_cvttps2qq<0x7A, "vcvttps2qq", X86cvttp2si,
7959 X86cvttp2siRnd, SchedWriteCvtPS2DQ>, PD,
7960 EVEX_CD8<32, CD8VH>;
7962 defm VCVTTPD2UQQ : avx512_cvttpd2qq<0x78, "vcvttpd2uqq", X86cvttp2ui,
7963 X86cvttp2uiRnd, SchedWriteCvtPD2DQ>, VEX_W,
7964 PD, EVEX_CD8<64, CD8VF>;
7966 defm VCVTTPS2UQQ : avx512_cvttps2qq<0x78, "vcvttps2uqq", X86cvttp2ui,
7967 X86cvttp2uiRnd, SchedWriteCvtPS2DQ>, PD,
7968 EVEX_CD8<32, CD8VH>;
7970 defm VCVTQQ2PD : avx512_cvtqq2pd<0xE6, "vcvtqq2pd", sint_to_fp,
7971 X86VSintToFpRnd, SchedWriteCvtDQ2PD>, VEX_W, XS,
7972 EVEX_CD8<64, CD8VF>;
7974 defm VCVTUQQ2PD : avx512_cvtqq2pd<0x7A, "vcvtuqq2pd", uint_to_fp,
7975 X86VUintToFpRnd, SchedWriteCvtDQ2PD>, VEX_W, XS,
7976 EVEX_CD8<64, CD8VF>;
7978 defm VCVTQQ2PS : avx512_cvtqq2ps<0x5B, "vcvtqq2ps", sint_to_fp, X86VSintToFP,
7979 X86VSintToFpRnd, SchedWriteCvtDQ2PS>, VEX_W, PS,
7980 EVEX_CD8<64, CD8VF>;
7982 defm VCVTUQQ2PS : avx512_cvtqq2ps<0x7A, "vcvtuqq2ps", uint_to_fp, X86VUintToFP,
7983 X86VUintToFpRnd, SchedWriteCvtDQ2PS>, VEX_W, XD,
7984 EVEX_CD8<64, CD8VF>;
7986 let Predicates = [HasAVX512] in {
7987 def : Pat<(v16i32 (fp_to_sint (v16f32 VR512:$src))),
7988 (VCVTTPS2DQZrr VR512:$src)>;
7989 def : Pat<(v16i32 (fp_to_sint (loadv16f32 addr:$src))),
7990 (VCVTTPS2DQZrm addr:$src)>;
7992 def : Pat<(v16i32 (fp_to_uint (v16f32 VR512:$src))),
7993 (VCVTTPS2UDQZrr VR512:$src)>;
7994 def : Pat<(v16i32 (fp_to_uint (loadv16f32 addr:$src))),
7995 (VCVTTPS2UDQZrm addr:$src)>;
7997 def : Pat<(v8i32 (fp_to_sint (v8f64 VR512:$src))),
7998 (VCVTTPD2DQZrr VR512:$src)>;
7999 def : Pat<(v8i32 (fp_to_sint (loadv8f64 addr:$src))),
8000 (VCVTTPD2DQZrm addr:$src)>;
8002 def : Pat<(v8i32 (fp_to_uint (v8f64 VR512:$src))),
8003 (VCVTTPD2UDQZrr VR512:$src)>;
8004 def : Pat<(v8i32 (fp_to_uint (loadv8f64 addr:$src))),
8005 (VCVTTPD2UDQZrm addr:$src)>;
8008 let Predicates = [HasVLX] in {
8009 def : Pat<(v4i32 (fp_to_sint (v4f32 VR128X:$src))),
8010 (VCVTTPS2DQZ128rr VR128X:$src)>;
8011 def : Pat<(v4i32 (fp_to_sint (loadv4f32 addr:$src))),
8012 (VCVTTPS2DQZ128rm addr:$src)>;
8014 def : Pat<(v4i32 (fp_to_uint (v4f32 VR128X:$src))),
8015 (VCVTTPS2UDQZ128rr VR128X:$src)>;
8016 def : Pat<(v4i32 (fp_to_uint (loadv4f32 addr:$src))),
8017 (VCVTTPS2UDQZ128rm addr:$src)>;
8019 def : Pat<(v8i32 (fp_to_sint (v8f32 VR256X:$src))),
8020 (VCVTTPS2DQZ256rr VR256X:$src)>;
8021 def : Pat<(v8i32 (fp_to_sint (loadv8f32 addr:$src))),
8022 (VCVTTPS2DQZ256rm addr:$src)>;
8024 def : Pat<(v8i32 (fp_to_uint (v8f32 VR256X:$src))),
8025 (VCVTTPS2UDQZ256rr VR256X:$src)>;
8026 def : Pat<(v8i32 (fp_to_uint (loadv8f32 addr:$src))),
8027 (VCVTTPS2UDQZ256rm addr:$src)>;
8029 def : Pat<(v4i32 (fp_to_sint (v4f64 VR256X:$src))),
8030 (VCVTTPD2DQZ256rr VR256X:$src)>;
8031 def : Pat<(v4i32 (fp_to_sint (loadv4f64 addr:$src))),
8032 (VCVTTPD2DQZ256rm addr:$src)>;
8034 def : Pat<(v4i32 (fp_to_uint (v4f64 VR256X:$src))),
8035 (VCVTTPD2UDQZ256rr VR256X:$src)>;
8036 def : Pat<(v4i32 (fp_to_uint (loadv4f64 addr:$src))),
8037 (VCVTTPD2UDQZ256rm addr:$src)>;
8040 let Predicates = [HasDQI] in {
8041 def : Pat<(v8i64 (fp_to_sint (v8f32 VR256X:$src))),
8042 (VCVTTPS2QQZrr VR256X:$src)>;
8043 def : Pat<(v8i64 (fp_to_sint (loadv8f32 addr:$src))),
8044 (VCVTTPS2QQZrm addr:$src)>;
8046 def : Pat<(v8i64 (fp_to_uint (v8f32 VR256X:$src))),
8047 (VCVTTPS2UQQZrr VR256X:$src)>;
8048 def : Pat<(v8i64 (fp_to_uint (loadv8f32 addr:$src))),
8049 (VCVTTPS2UQQZrm addr:$src)>;
8051 def : Pat<(v8i64 (fp_to_sint (v8f64 VR512:$src))),
8052 (VCVTTPD2QQZrr VR512:$src)>;
8053 def : Pat<(v8i64 (fp_to_sint (loadv8f64 addr:$src))),
8054 (VCVTTPD2QQZrm addr:$src)>;
8056 def : Pat<(v8i64 (fp_to_uint (v8f64 VR512:$src))),
8057 (VCVTTPD2UQQZrr VR512:$src)>;
8058 def : Pat<(v8i64 (fp_to_uint (loadv8f64 addr:$src))),
8059 (VCVTTPD2UQQZrm addr:$src)>;
8062 let Predicates = [HasDQI, HasVLX] in {
8063 def : Pat<(v4i64 (fp_to_sint (v4f32 VR128X:$src))),
8064 (VCVTTPS2QQZ256rr VR128X:$src)>;
8065 def : Pat<(v4i64 (fp_to_sint (loadv4f32 addr:$src))),
8066 (VCVTTPS2QQZ256rm addr:$src)>;
8068 def : Pat<(v4i64 (fp_to_uint (v4f32 VR128X:$src))),
8069 (VCVTTPS2UQQZ256rr VR128X:$src)>;
8070 def : Pat<(v4i64 (fp_to_uint (loadv4f32 addr:$src))),
8071 (VCVTTPS2UQQZ256rm addr:$src)>;
8073 def : Pat<(v2i64 (fp_to_sint (v2f64 VR128X:$src))),
8074 (VCVTTPD2QQZ128rr VR128X:$src)>;
8075 def : Pat<(v2i64 (fp_to_sint (loadv2f64 addr:$src))),
8076 (VCVTTPD2QQZ128rm addr:$src)>;
8078 def : Pat<(v2i64 (fp_to_uint (v2f64 VR128X:$src))),
8079 (VCVTTPD2UQQZ128rr VR128X:$src)>;
8080 def : Pat<(v2i64 (fp_to_uint (loadv2f64 addr:$src))),
8081 (VCVTTPD2UQQZ128rm addr:$src)>;
8083 def : Pat<(v4i64 (fp_to_sint (v4f64 VR256X:$src))),
8084 (VCVTTPD2QQZ256rr VR256X:$src)>;
8085 def : Pat<(v4i64 (fp_to_sint (loadv4f64 addr:$src))),
8086 (VCVTTPD2QQZ256rm addr:$src)>;
8088 def : Pat<(v4i64 (fp_to_uint (v4f64 VR256X:$src))),
8089 (VCVTTPD2UQQZ256rr VR256X:$src)>;
8090 def : Pat<(v4i64 (fp_to_uint (loadv4f64 addr:$src))),
8091 (VCVTTPD2UQQZ256rm addr:$src)>;
8094 let Predicates = [HasAVX512, NoVLX] in {
8095 def : Pat<(v8i32 (fp_to_uint (v8f32 VR256X:$src1))),
8096 (EXTRACT_SUBREG (v16i32 (VCVTTPS2UDQZrr
8097 (v16f32 (INSERT_SUBREG (IMPLICIT_DEF),
8098 VR256X:$src1, sub_ymm)))), sub_ymm)>;
8100 def : Pat<(v4i32 (fp_to_uint (v4f32 VR128X:$src1))),
8101 (EXTRACT_SUBREG (v16i32 (VCVTTPS2UDQZrr
8102 (v16f32 (INSERT_SUBREG (IMPLICIT_DEF),
8103 VR128X:$src1, sub_xmm)))), sub_xmm)>;
8105 def : Pat<(v4i32 (fp_to_uint (v4f64 VR256X:$src1))),
8106 (EXTRACT_SUBREG (v8i32 (VCVTTPD2UDQZrr
8107 (v8f64 (INSERT_SUBREG (IMPLICIT_DEF),
8108 VR256X:$src1, sub_ymm)))), sub_xmm)>;
8110 def : Pat<(v8f32 (uint_to_fp (v8i32 VR256X:$src1))),
8111 (EXTRACT_SUBREG (v16f32 (VCVTUDQ2PSZrr
8112 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF),
8113 VR256X:$src1, sub_ymm)))), sub_ymm)>;
8115 def : Pat<(v4f32 (uint_to_fp (v4i32 VR128X:$src1))),
8116 (EXTRACT_SUBREG (v16f32 (VCVTUDQ2PSZrr
8117 (v16i32 (INSERT_SUBREG (IMPLICIT_DEF),
8118 VR128X:$src1, sub_xmm)))), sub_xmm)>;
8120 def : Pat<(v4f64 (uint_to_fp (v4i32 VR128X:$src1))),
8121 (EXTRACT_SUBREG (v8f64 (VCVTUDQ2PDZrr
8122 (v8i32 (INSERT_SUBREG (IMPLICIT_DEF),
8123 VR128X:$src1, sub_xmm)))), sub_ymm)>;
8125 def : Pat<(v2f64 (X86VUintToFP (v4i32 VR128X:$src1))),
8126 (EXTRACT_SUBREG (v8f64 (VCVTUDQ2PDZrr
8127 (v8i32 (INSERT_SUBREG (IMPLICIT_DEF),
8128 VR128X:$src1, sub_xmm)))), sub_xmm)>;
8131 let Predicates = [HasAVX512, HasVLX] in {
8132 let AddedComplexity = 15 in {
8133 def : Pat<(X86vzmovl (v2i64 (bitconvert
8134 (v4i32 (X86cvtp2Int (v2f64 VR128X:$src)))))),
8135 (VCVTPD2DQZ128rr VR128X:$src)>;
8136 def : Pat<(X86vzmovl (v2i64 (bitconvert
8137 (v4i32 (X86cvtp2Int (loadv2f64 addr:$src)))))),
8138 (VCVTPD2DQZ128rm addr:$src)>;
8139 def : Pat<(X86vzmovl (v2i64 (bitconvert
8140 (v4i32 (X86cvtp2UInt (v2f64 VR128X:$src)))))),
8141 (VCVTPD2UDQZ128rr VR128X:$src)>;
8142 def : Pat<(X86vzmovl (v2i64 (bitconvert
8143 (v4i32 (X86cvttp2si (v2f64 VR128X:$src)))))),
8144 (VCVTTPD2DQZ128rr VR128X:$src)>;
8145 def : Pat<(X86vzmovl (v2i64 (bitconvert
8146 (v4i32 (X86cvttp2si (loadv2f64 addr:$src)))))),
8147 (VCVTTPD2DQZ128rm addr:$src)>;
8148 def : Pat<(X86vzmovl (v2i64 (bitconvert
8149 (v4i32 (X86cvttp2ui (v2f64 VR128X:$src)))))),
8150 (VCVTTPD2UDQZ128rr VR128X:$src)>;
8153 def : Pat<(v2f64 (X86VSintToFP (bc_v4i32 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))),
8154 (VCVTDQ2PDZ128rm addr:$src)>;
8155 def : Pat<(v2f64 (X86VSintToFP (bc_v4i32 (v2i64 (X86vzload addr:$src))))),
8156 (VCVTDQ2PDZ128rm addr:$src)>;
8158 def : Pat<(v2f64 (X86VUintToFP (bc_v4i32 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))),
8159 (VCVTUDQ2PDZ128rm addr:$src)>;
8160 def : Pat<(v2f64 (X86VUintToFP (bc_v4i32 (v2i64 (X86vzload addr:$src))))),
8161 (VCVTUDQ2PDZ128rm addr:$src)>;
8164 let Predicates = [HasAVX512] in {
8165 def : Pat<(v8f32 (fpround (loadv8f64 addr:$src))),
8166 (VCVTPD2PSZrm addr:$src)>;
8167 def : Pat<(v8f64 (extloadv8f32 addr:$src)),
8168 (VCVTPS2PDZrm addr:$src)>;
8171 let Predicates = [HasDQI, HasVLX] in {
8172 let AddedComplexity = 15 in {
8173 def : Pat<(X86vzmovl (v2f64 (bitconvert
8174 (v4f32 (X86VSintToFP (v2i64 VR128X:$src)))))),
8175 (VCVTQQ2PSZ128rr VR128X:$src)>;
8176 def : Pat<(X86vzmovl (v2f64 (bitconvert
8177 (v4f32 (X86VUintToFP (v2i64 VR128X:$src)))))),
8178 (VCVTUQQ2PSZ128rr VR128X:$src)>;
8182 let Predicates = [HasDQI, NoVLX] in {
8183 def : Pat<(v2i64 (fp_to_sint (v2f64 VR128X:$src1))),
8184 (EXTRACT_SUBREG (v8i64 (VCVTTPD2QQZrr
8185 (v8f64 (INSERT_SUBREG (IMPLICIT_DEF),
8186 VR128X:$src1, sub_xmm)))), sub_xmm)>;
8188 def : Pat<(v4i64 (fp_to_sint (v4f32 VR128X:$src1))),
8189 (EXTRACT_SUBREG (v8i64 (VCVTTPS2QQZrr
8190 (v8f32 (INSERT_SUBREG (IMPLICIT_DEF),
8191 VR128X:$src1, sub_xmm)))), sub_ymm)>;
8193 def : Pat<(v4i64 (fp_to_sint (v4f64 VR256X:$src1))),
8194 (EXTRACT_SUBREG (v8i64 (VCVTTPD2QQZrr
8195 (v8f64 (INSERT_SUBREG (IMPLICIT_DEF),
8196 VR256X:$src1, sub_ymm)))), sub_ymm)>;
8198 def : Pat<(v2i64 (fp_to_uint (v2f64 VR128X:$src1))),
8199 (EXTRACT_SUBREG (v8i64 (VCVTTPD2UQQZrr
8200 (v8f64 (INSERT_SUBREG (IMPLICIT_DEF),
8201 VR128X:$src1, sub_xmm)))), sub_xmm)>;
8203 def : Pat<(v4i64 (fp_to_uint (v4f32 VR128X:$src1))),
8204 (EXTRACT_SUBREG (v8i64 (VCVTTPS2UQQZrr
8205 (v8f32 (INSERT_SUBREG (IMPLICIT_DEF),
8206 VR128X:$src1, sub_xmm)))), sub_ymm)>;
8208 def : Pat<(v4i64 (fp_to_uint (v4f64 VR256X:$src1))),
8209 (EXTRACT_SUBREG (v8i64 (VCVTTPD2UQQZrr
8210 (v8f64 (INSERT_SUBREG (IMPLICIT_DEF),
8211 VR256X:$src1, sub_ymm)))), sub_ymm)>;
8213 def : Pat<(v4f32 (sint_to_fp (v4i64 VR256X:$src1))),
8214 (EXTRACT_SUBREG (v8f32 (VCVTQQ2PSZrr
8215 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF),
8216 VR256X:$src1, sub_ymm)))), sub_xmm)>;
8218 def : Pat<(v2f64 (sint_to_fp (v2i64 VR128X:$src1))),
8219 (EXTRACT_SUBREG (v8f64 (VCVTQQ2PDZrr
8220 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF),
8221 VR128X:$src1, sub_xmm)))), sub_xmm)>;
8223 def : Pat<(v4f64 (sint_to_fp (v4i64 VR256X:$src1))),
8224 (EXTRACT_SUBREG (v8f64 (VCVTQQ2PDZrr
8225 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF),
8226 VR256X:$src1, sub_ymm)))), sub_ymm)>;
8228 def : Pat<(v4f32 (uint_to_fp (v4i64 VR256X:$src1))),
8229 (EXTRACT_SUBREG (v8f32 (VCVTUQQ2PSZrr
8230 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF),
8231 VR256X:$src1, sub_ymm)))), sub_xmm)>;
8233 def : Pat<(v2f64 (uint_to_fp (v2i64 VR128X:$src1))),
8234 (EXTRACT_SUBREG (v8f64 (VCVTUQQ2PDZrr
8235 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF),
8236 VR128X:$src1, sub_xmm)))), sub_xmm)>;
8238 def : Pat<(v4f64 (uint_to_fp (v4i64 VR256X:$src1))),
8239 (EXTRACT_SUBREG (v8f64 (VCVTUQQ2PDZrr
8240 (v8i64 (INSERT_SUBREG (IMPLICIT_DEF),
8241 VR256X:$src1, sub_ymm)))), sub_ymm)>;
8244 //===----------------------------------------------------------------------===//
8245 // Half precision conversion instructions
8246 //===----------------------------------------------------------------------===//
8248 multiclass avx512_cvtph2ps<X86VectorVTInfo _dest, X86VectorVTInfo _src,
8249 X86MemOperand x86memop, PatFrag ld_frag,
8250 X86FoldableSchedWrite sched> {
8251 defm rr : AVX512_maskable<0x13, MRMSrcReg, _dest ,(outs _dest.RC:$dst),
8252 (ins _src.RC:$src), "vcvtph2ps", "$src", "$src",
8253 (X86cvtph2ps (_src.VT _src.RC:$src))>,
8254 T8PD, Sched<[sched]>;
8255 defm rm : AVX512_maskable<0x13, MRMSrcMem, _dest, (outs _dest.RC:$dst),
8256 (ins x86memop:$src), "vcvtph2ps", "$src", "$src",
8257 (X86cvtph2ps (_src.VT
8259 (ld_frag addr:$src))))>,
8260 T8PD, Sched<[sched.Folded]>;
8263 multiclass avx512_cvtph2ps_sae<X86VectorVTInfo _dest, X86VectorVTInfo _src,
8264 X86FoldableSchedWrite sched> {
8265 defm rrb : AVX512_maskable<0x13, MRMSrcReg, _dest, (outs _dest.RC:$dst),
8266 (ins _src.RC:$src), "vcvtph2ps",
8267 "{sae}, $src", "$src, {sae}",
8268 (X86cvtph2psRnd (_src.VT _src.RC:$src),
8269 (i32 FROUND_NO_EXC))>,
8270 T8PD, EVEX_B, Sched<[sched]>;
8273 let Predicates = [HasAVX512] in
8274 defm VCVTPH2PSZ : avx512_cvtph2ps<v16f32_info, v16i16x_info, f256mem, loadv4i64,
8276 avx512_cvtph2ps_sae<v16f32_info, v16i16x_info, WriteCvtPH2PSZ>,
8277 EVEX, EVEX_V512, EVEX_CD8<32, CD8VH>;
8279 let Predicates = [HasVLX] in {
8280 defm VCVTPH2PSZ256 : avx512_cvtph2ps<v8f32x_info, v8i16x_info, f128mem,
8281 loadv2i64, WriteCvtPH2PSY>, EVEX, EVEX_V256,
8282 EVEX_CD8<32, CD8VH>;
8283 defm VCVTPH2PSZ128 : avx512_cvtph2ps<v4f32x_info, v8i16x_info, f64mem,
8284 loadv2i64, WriteCvtPH2PS>, EVEX, EVEX_V128,
8285 EVEX_CD8<32, CD8VH>;
8287 // Pattern match vcvtph2ps of a scalar i64 load.
8288 def : Pat<(v4f32 (X86cvtph2ps (v8i16 (vzmovl_v2i64 addr:$src)))),
8289 (VCVTPH2PSZ128rm addr:$src)>;
8290 def : Pat<(v4f32 (X86cvtph2ps (v8i16 (vzload_v2i64 addr:$src)))),
8291 (VCVTPH2PSZ128rm addr:$src)>;
8292 def : Pat<(v4f32 (X86cvtph2ps (v8i16 (bitconvert
8293 (v2i64 (scalar_to_vector (loadi64 addr:$src))))))),
8294 (VCVTPH2PSZ128rm addr:$src)>;
8297 multiclass avx512_cvtps2ph<X86VectorVTInfo _dest, X86VectorVTInfo _src,
8298 X86MemOperand x86memop, SchedWrite RR, SchedWrite MR> {
8299 defm rr : AVX512_maskable<0x1D, MRMDestReg, _dest ,(outs _dest.RC:$dst),
8300 (ins _src.RC:$src1, i32u8imm:$src2),
8301 "vcvtps2ph", "$src2, $src1", "$src1, $src2",
8302 (X86cvtps2ph (_src.VT _src.RC:$src1),
8303 (i32 imm:$src2)), 0, 0>,
8304 AVX512AIi8Base, Sched<[RR]>;
8305 let hasSideEffects = 0, mayStore = 1 in {
8306 def mr : AVX512AIi8<0x1D, MRMDestMem, (outs),
8307 (ins x86memop:$dst, _src.RC:$src1, i32u8imm:$src2),
8308 "vcvtps2ph\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>,
8310 def mrk : AVX512AIi8<0x1D, MRMDestMem, (outs),
8311 (ins x86memop:$dst, _dest.KRCWM:$mask, _src.RC:$src1, i32u8imm:$src2),
8312 "vcvtps2ph\t{$src2, $src1, $dst {${mask}}|$dst {${mask}}, $src1, $src2}", []>,
8313 EVEX_K, Sched<[MR]>, NotMemoryFoldable;
8317 multiclass avx512_cvtps2ph_sae<X86VectorVTInfo _dest, X86VectorVTInfo _src,
8319 let hasSideEffects = 0 in
8320 defm rrb : AVX512_maskable_in_asm<0x1D, MRMDestReg, _dest,
8321 (outs _dest.RC:$dst),
8322 (ins _src.RC:$src1, i32u8imm:$src2),
8323 "vcvtps2ph", "$src2, {sae}, $src1", "$src1, {sae}, $src2", []>,
8324 EVEX_B, AVX512AIi8Base, Sched<[Sched]>;
8327 let Predicates = [HasAVX512] in {
8328 defm VCVTPS2PHZ : avx512_cvtps2ph<v16i16x_info, v16f32_info, f256mem,
8329 WriteCvtPS2PHZ, WriteCvtPS2PHZSt>,
8330 avx512_cvtps2ph_sae<v16i16x_info, v16f32_info, WriteCvtPS2PHZ>,
8331 EVEX, EVEX_V512, EVEX_CD8<32, CD8VH>;
8332 let Predicates = [HasVLX] in {
8333 defm VCVTPS2PHZ256 : avx512_cvtps2ph<v8i16x_info, v8f32x_info, f128mem,
8334 WriteCvtPS2PHY, WriteCvtPS2PHYSt>,
8335 EVEX, EVEX_V256, EVEX_CD8<32, CD8VH>;
8336 defm VCVTPS2PHZ128 : avx512_cvtps2ph<v8i16x_info, v4f32x_info, f64mem,
8337 WriteCvtPS2PH, WriteCvtPS2PHSt>,
8338 EVEX, EVEX_V128, EVEX_CD8<32, CD8VH>;
8341 def : Pat<(store (f64 (extractelt
8342 (bc_v2f64 (v8i16 (X86cvtps2ph VR128X:$src1, i32:$src2))),
8343 (iPTR 0))), addr:$dst),
8344 (VCVTPS2PHZ128mr addr:$dst, VR128X:$src1, imm:$src2)>;
8345 def : Pat<(store (i64 (extractelt
8346 (bc_v2i64 (v8i16 (X86cvtps2ph VR128X:$src1, i32:$src2))),
8347 (iPTR 0))), addr:$dst),
8348 (VCVTPS2PHZ128mr addr:$dst, VR128X:$src1, imm:$src2)>;
8349 def : Pat<(store (v8i16 (X86cvtps2ph VR256X:$src1, i32:$src2)), addr:$dst),
8350 (VCVTPS2PHZ256mr addr:$dst, VR256X:$src1, imm:$src2)>;
8351 def : Pat<(store (v16i16 (X86cvtps2ph VR512:$src1, i32:$src2)), addr:$dst),
8352 (VCVTPS2PHZmr addr:$dst, VR512:$src1, imm:$src2)>;
8355 // Patterns for matching conversions from float to half-float and vice versa.
8356 let Predicates = [HasVLX] in {
8357 // Use MXCSR.RC for rounding instead of explicitly specifying the default
8358 // rounding mode (Nearest-Even, encoded as 0). Both are equivalent in the
8359 // configurations we support (the default). However, falling back to MXCSR is
8360 // more consistent with other instructions, which are always controlled by it.
8361 // It's encoded as 0b100.
8362 def : Pat<(fp_to_f16 FR32X:$src),
8363 (i16 (EXTRACT_SUBREG (VMOVPDI2DIZrr (VCVTPS2PHZ128rr
8364 (COPY_TO_REGCLASS FR32X:$src, VR128X), 4)), sub_16bit))>;
8366 def : Pat<(f16_to_fp GR16:$src),
8367 (f32 (COPY_TO_REGCLASS (VCVTPH2PSZ128rr
8368 (COPY_TO_REGCLASS (MOVSX32rr16 GR16:$src), VR128X)), FR32X)) >;
8370 def : Pat<(f16_to_fp (i16 (fp_to_f16 FR32X:$src))),
8371 (f32 (COPY_TO_REGCLASS (VCVTPH2PSZ128rr
8372 (VCVTPS2PHZ128rr (COPY_TO_REGCLASS FR32X:$src, VR128X), 4)), FR32X)) >;
8375 // Unordered/Ordered scalar fp compare with Sea and set EFLAGS
8376 multiclass avx512_ord_cmp_sae<bits<8> opc, X86VectorVTInfo _,
8377 string OpcodeStr, X86FoldableSchedWrite sched> {
8378 let hasSideEffects = 0 in
8379 def rrb: AVX512<opc, MRMSrcReg, (outs), (ins _.RC:$src1, _.RC:$src2),
8380 !strconcat(OpcodeStr, "\t{{sae}, $src2, $src1|$src1, $src2, {sae}}"), []>,
8381 EVEX, EVEX_B, VEX_LIG, EVEX_V128, Sched<[sched]>;
8384 let Defs = [EFLAGS], Predicates = [HasAVX512] in {
8385 defm VUCOMISSZ : avx512_ord_cmp_sae<0x2E, v4f32x_info, "vucomiss", WriteFCom>,
8386 AVX512PSIi8Base, EVEX_CD8<32, CD8VT1>;
8387 defm VUCOMISDZ : avx512_ord_cmp_sae<0x2E, v2f64x_info, "vucomisd", WriteFCom>,
8388 AVX512PDIi8Base, VEX_W, EVEX_CD8<64, CD8VT1>;
8389 defm VCOMISSZ : avx512_ord_cmp_sae<0x2F, v4f32x_info, "vcomiss", WriteFCom>,
8390 AVX512PSIi8Base, EVEX_CD8<32, CD8VT1>;
8391 defm VCOMISDZ : avx512_ord_cmp_sae<0x2F, v2f64x_info, "vcomisd", WriteFCom>,
8392 AVX512PDIi8Base, VEX_W, EVEX_CD8<64, CD8VT1>;
8395 let Defs = [EFLAGS], Predicates = [HasAVX512] in {
8396 defm VUCOMISSZ : sse12_ord_cmp<0x2E, FR32X, X86cmp, f32, f32mem, loadf32,
8397 "ucomiss", WriteFCom>, PS, EVEX, VEX_LIG,
8398 EVEX_CD8<32, CD8VT1>;
8399 defm VUCOMISDZ : sse12_ord_cmp<0x2E, FR64X, X86cmp, f64, f64mem, loadf64,
8400 "ucomisd", WriteFCom>, PD, EVEX,
8401 VEX_LIG, VEX_W, EVEX_CD8<64, CD8VT1>;
8402 let Pattern = []<dag> in {
8403 defm VCOMISSZ : sse12_ord_cmp<0x2F, FR32X, undef, f32, f32mem, loadf32,
8404 "comiss", WriteFCom>, PS, EVEX, VEX_LIG,
8405 EVEX_CD8<32, CD8VT1>;
8406 defm VCOMISDZ : sse12_ord_cmp<0x2F, FR64X, undef, f64, f64mem, loadf64,
8407 "comisd", WriteFCom>, PD, EVEX,
8408 VEX_LIG, VEX_W, EVEX_CD8<64, CD8VT1>;
8410 let isCodeGenOnly = 1 in {
8411 defm VUCOMISSZ : sse12_ord_cmp_int<0x2E, VR128X, X86ucomi, v4f32, ssmem,
8412 sse_load_f32, "ucomiss", WriteFCom>, PS, EVEX, VEX_LIG,
8413 EVEX_CD8<32, CD8VT1>;
8414 defm VUCOMISDZ : sse12_ord_cmp_int<0x2E, VR128X, X86ucomi, v2f64, sdmem,
8415 sse_load_f64, "ucomisd", WriteFCom>, PD, EVEX,
8416 VEX_LIG, VEX_W, EVEX_CD8<64, CD8VT1>;
8418 defm VCOMISSZ : sse12_ord_cmp_int<0x2F, VR128X, X86comi, v4f32, ssmem,
8419 sse_load_f32, "comiss", WriteFCom>, PS, EVEX, VEX_LIG,
8420 EVEX_CD8<32, CD8VT1>;
8421 defm VCOMISDZ : sse12_ord_cmp_int<0x2F, VR128X, X86comi, v2f64, sdmem,
8422 sse_load_f64, "comisd", WriteFCom>, PD, EVEX,
8423 VEX_LIG, VEX_W, EVEX_CD8<64, CD8VT1>;
8427 /// avx512_fp14_s rcp14ss, rcp14sd, rsqrt14ss, rsqrt14sd
8428 multiclass avx512_fp14_s<bits<8> opc, string OpcodeStr, SDNode OpNode,
8429 X86FoldableSchedWrite sched, X86VectorVTInfo _> {
8430 let Predicates = [HasAVX512], ExeDomain = _.ExeDomain in {
8431 defm rr : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
8432 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,
8433 "$src2, $src1", "$src1, $src2",
8434 (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2))>,
8435 EVEX_4V, Sched<[sched]>;
8436 defm rm : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
8437 (ins _.RC:$src1, _.IntScalarMemOp:$src2), OpcodeStr,
8438 "$src2, $src1", "$src1, $src2",
8439 (OpNode (_.VT _.RC:$src1),
8440 _.ScalarIntMemCPat:$src2)>, EVEX_4V,
8441 Sched<[sched.Folded, ReadAfterLd]>;
8445 defm VRCP14SSZ : avx512_fp14_s<0x4D, "vrcp14ss", X86rcp14s, SchedWriteFRcp.Scl,
8446 f32x_info>, EVEX_CD8<32, CD8VT1>,
8448 defm VRCP14SDZ : avx512_fp14_s<0x4D, "vrcp14sd", X86rcp14s, SchedWriteFRcp.Scl,
8449 f64x_info>, VEX_W, EVEX_CD8<64, CD8VT1>,
8451 defm VRSQRT14SSZ : avx512_fp14_s<0x4F, "vrsqrt14ss", X86rsqrt14s,
8452 SchedWriteFRsqrt.Scl, f32x_info>,
8453 EVEX_CD8<32, CD8VT1>, T8PD;
8454 defm VRSQRT14SDZ : avx512_fp14_s<0x4F, "vrsqrt14sd", X86rsqrt14s,
8455 SchedWriteFRsqrt.Scl, f64x_info>, VEX_W,
8456 EVEX_CD8<64, CD8VT1>, T8PD;
8458 /// avx512_fp14_p rcp14ps, rcp14pd, rsqrt14ps, rsqrt14pd
8459 multiclass avx512_fp14_p<bits<8> opc, string OpcodeStr, SDNode OpNode,
8460 X86FoldableSchedWrite sched, X86VectorVTInfo _> {
8461 let ExeDomain = _.ExeDomain in {
8462 defm r: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
8463 (ins _.RC:$src), OpcodeStr, "$src", "$src",
8464 (_.FloatVT (OpNode _.RC:$src))>, EVEX, T8PD,
8466 defm m: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
8467 (ins _.MemOp:$src), OpcodeStr, "$src", "$src",
8469 (bitconvert (_.LdFrag addr:$src))))>, EVEX, T8PD,
8470 Sched<[sched.Folded, ReadAfterLd]>;
8471 defm mb: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
8472 (ins _.ScalarMemOp:$src), OpcodeStr,
8473 "${src}"##_.BroadcastStr, "${src}"##_.BroadcastStr,
8475 (X86VBroadcast (_.ScalarLdFrag addr:$src))))>,
8476 EVEX, T8PD, EVEX_B, Sched<[sched.Folded, ReadAfterLd]>;
8480 multiclass avx512_fp14_p_vl_all<bits<8> opc, string OpcodeStr, SDNode OpNode,
8481 X86SchedWriteWidths sched> {
8482 defm PSZ : avx512_fp14_p<opc, !strconcat(OpcodeStr, "ps"), OpNode, sched.ZMM,
8483 v16f32_info>, EVEX_V512, EVEX_CD8<32, CD8VF>;
8484 defm PDZ : avx512_fp14_p<opc, !strconcat(OpcodeStr, "pd"), OpNode, sched.ZMM,
8485 v8f64_info>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>;
8487 // Define only if AVX512VL feature is present.
8488 let Predicates = [HasVLX] in {
8489 defm PSZ128 : avx512_fp14_p<opc, !strconcat(OpcodeStr, "ps"),
8490 OpNode, sched.XMM, v4f32x_info>,
8491 EVEX_V128, EVEX_CD8<32, CD8VF>;
8492 defm PSZ256 : avx512_fp14_p<opc, !strconcat(OpcodeStr, "ps"),
8493 OpNode, sched.YMM, v8f32x_info>,
8494 EVEX_V256, EVEX_CD8<32, CD8VF>;
8495 defm PDZ128 : avx512_fp14_p<opc, !strconcat(OpcodeStr, "pd"),
8496 OpNode, sched.XMM, v2f64x_info>,
8497 EVEX_V128, VEX_W, EVEX_CD8<64, CD8VF>;
8498 defm PDZ256 : avx512_fp14_p<opc, !strconcat(OpcodeStr, "pd"),
8499 OpNode, sched.YMM, v4f64x_info>,
8500 EVEX_V256, VEX_W, EVEX_CD8<64, CD8VF>;
8504 defm VRSQRT14 : avx512_fp14_p_vl_all<0x4E, "vrsqrt14", X86rsqrt14, SchedWriteFRsqrt>;
8505 defm VRCP14 : avx512_fp14_p_vl_all<0x4C, "vrcp14", X86rcp14, SchedWriteFRcp>;
8507 /// avx512_fp28_s rcp28ss, rcp28sd, rsqrt28ss, rsqrt28sd
8508 multiclass avx512_fp28_s<bits<8> opc, string OpcodeStr,X86VectorVTInfo _,
8509 SDNode OpNode, X86FoldableSchedWrite sched> {
8510 let ExeDomain = _.ExeDomain in {
8511 defm r : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
8512 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,
8513 "$src2, $src1", "$src1, $src2",
8514 (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2),
8515 (i32 FROUND_CURRENT))>,
8518 defm rb : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
8519 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,
8520 "{sae}, $src2, $src1", "$src1, $src2, {sae}",
8521 (OpNode (_.VT _.RC:$src1), (_.VT _.RC:$src2),
8522 (i32 FROUND_NO_EXC))>, EVEX_B,
8525 defm m : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
8526 (ins _.RC:$src1, _.IntScalarMemOp:$src2), OpcodeStr,
8527 "$src2, $src1", "$src1, $src2",
8528 (OpNode (_.VT _.RC:$src1), _.ScalarIntMemCPat:$src2,
8529 (i32 FROUND_CURRENT))>,
8530 Sched<[sched.Folded, ReadAfterLd]>;
8534 multiclass avx512_eri_s<bits<8> opc, string OpcodeStr, SDNode OpNode,
8535 X86FoldableSchedWrite sched> {
8536 defm SSZ : avx512_fp28_s<opc, OpcodeStr#"ss", f32x_info, OpNode, sched>,
8537 EVEX_CD8<32, CD8VT1>;
8538 defm SDZ : avx512_fp28_s<opc, OpcodeStr#"sd", f64x_info, OpNode, sched>,
8539 EVEX_CD8<64, CD8VT1>, VEX_W;
8542 let Predicates = [HasERI] in {
8543 defm VRCP28 : avx512_eri_s<0xCB, "vrcp28", X86rcp28s, SchedWriteFRcp.Scl>,
8545 defm VRSQRT28 : avx512_eri_s<0xCD, "vrsqrt28", X86rsqrt28s,
8546 SchedWriteFRsqrt.Scl>, T8PD, EVEX_4V;
8549 defm VGETEXP : avx512_eri_s<0x43, "vgetexp", X86fgetexpRnds,
8550 SchedWriteFRnd.Scl>, T8PD, EVEX_4V;
8551 /// avx512_fp28_p rcp28ps, rcp28pd, rsqrt28ps, rsqrt28pd
8553 multiclass avx512_fp28_p<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
8554 SDNode OpNode, X86FoldableSchedWrite sched> {
8555 let ExeDomain = _.ExeDomain in {
8556 defm r : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
8557 (ins _.RC:$src), OpcodeStr, "$src", "$src",
8558 (OpNode (_.VT _.RC:$src), (i32 FROUND_CURRENT))>,
8561 defm m : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
8562 (ins _.MemOp:$src), OpcodeStr, "$src", "$src",
8564 (bitconvert (_.LdFrag addr:$src))),
8565 (i32 FROUND_CURRENT))>,
8566 Sched<[sched.Folded, ReadAfterLd]>;
8568 defm mb : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
8569 (ins _.ScalarMemOp:$src), OpcodeStr,
8570 "${src}"##_.BroadcastStr, "${src}"##_.BroadcastStr,
8572 (X86VBroadcast (_.ScalarLdFrag addr:$src))),
8573 (i32 FROUND_CURRENT))>, EVEX_B,
8574 Sched<[sched.Folded, ReadAfterLd]>;
8577 multiclass avx512_fp28_p_round<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
8578 SDNode OpNode, X86FoldableSchedWrite sched> {
8579 let ExeDomain = _.ExeDomain in
8580 defm rb : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
8581 (ins _.RC:$src), OpcodeStr,
8582 "{sae}, $src", "$src, {sae}",
8583 (OpNode (_.VT _.RC:$src), (i32 FROUND_NO_EXC))>,
8584 EVEX_B, Sched<[sched]>;
8587 multiclass avx512_eri<bits<8> opc, string OpcodeStr, SDNode OpNode,
8588 X86SchedWriteWidths sched> {
8589 defm PSZ : avx512_fp28_p<opc, OpcodeStr#"ps", v16f32_info, OpNode, sched.ZMM>,
8590 avx512_fp28_p_round<opc, OpcodeStr#"ps", v16f32_info, OpNode, sched.ZMM>,
8591 T8PD, EVEX_V512, EVEX_CD8<32, CD8VF>;
8592 defm PDZ : avx512_fp28_p<opc, OpcodeStr#"pd", v8f64_info, OpNode, sched.ZMM>,
8593 avx512_fp28_p_round<opc, OpcodeStr#"pd", v8f64_info, OpNode, sched.ZMM>,
8594 T8PD, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VF>;
8597 multiclass avx512_fp_unaryop_packed<bits<8> opc, string OpcodeStr,
8598 SDNode OpNode, X86SchedWriteWidths sched> {
8599 // Define only if AVX512VL feature is present.
8600 let Predicates = [HasVLX] in {
8601 defm PSZ128 : avx512_fp28_p<opc, OpcodeStr#"ps", v4f32x_info, OpNode, sched.XMM>,
8602 EVEX_V128, T8PD, EVEX_CD8<32, CD8VF>;
8603 defm PSZ256 : avx512_fp28_p<opc, OpcodeStr#"ps", v8f32x_info, OpNode, sched.YMM>,
8604 EVEX_V256, T8PD, EVEX_CD8<32, CD8VF>;
8605 defm PDZ128 : avx512_fp28_p<opc, OpcodeStr#"pd", v2f64x_info, OpNode, sched.XMM>,
8606 EVEX_V128, VEX_W, T8PD, EVEX_CD8<64, CD8VF>;
8607 defm PDZ256 : avx512_fp28_p<opc, OpcodeStr#"pd", v4f64x_info, OpNode, sched.YMM>,
8608 EVEX_V256, VEX_W, T8PD, EVEX_CD8<64, CD8VF>;
8612 let Predicates = [HasERI] in {
8613 defm VRSQRT28 : avx512_eri<0xCC, "vrsqrt28", X86rsqrt28, SchedWriteFRsqrt>, EVEX;
8614 defm VRCP28 : avx512_eri<0xCA, "vrcp28", X86rcp28, SchedWriteFRcp>, EVEX;
8615 defm VEXP2 : avx512_eri<0xC8, "vexp2", X86exp2, SchedWriteFAdd>, EVEX;
8617 defm VGETEXP : avx512_eri<0x42, "vgetexp", X86fgetexpRnd, SchedWriteFRnd>,
8618 avx512_fp_unaryop_packed<0x42, "vgetexp", X86fgetexpRnd,
8619 SchedWriteFRnd>, EVEX;
8621 multiclass avx512_sqrt_packed_round<bits<8> opc, string OpcodeStr,
8622 X86FoldableSchedWrite sched, X86VectorVTInfo _>{
8623 let ExeDomain = _.ExeDomain in
8624 defm rb: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
8625 (ins _.RC:$src, AVX512RC:$rc), OpcodeStr, "$rc, $src", "$src, $rc",
8626 (_.VT (X86fsqrtRnd _.RC:$src, (i32 imm:$rc)))>,
8627 EVEX, EVEX_B, EVEX_RC, Sched<[sched]>;
8630 multiclass avx512_sqrt_packed<bits<8> opc, string OpcodeStr,
8631 X86FoldableSchedWrite sched, X86VectorVTInfo _>{
8632 let ExeDomain = _.ExeDomain in {
8633 defm r: AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
8634 (ins _.RC:$src), OpcodeStr, "$src", "$src",
8635 (_.FloatVT (fsqrt _.RC:$src))>, EVEX,
8637 defm m: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
8638 (ins _.MemOp:$src), OpcodeStr, "$src", "$src",
8640 (bitconvert (_.LdFrag addr:$src))))>, EVEX,
8641 Sched<[sched.Folded, ReadAfterLd]>;
8642 defm mb: AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
8643 (ins _.ScalarMemOp:$src), OpcodeStr,
8644 "${src}"##_.BroadcastStr, "${src}"##_.BroadcastStr,
8646 (X86VBroadcast (_.ScalarLdFrag addr:$src))))>,
8647 EVEX, EVEX_B, Sched<[sched.Folded, ReadAfterLd]>;
8651 multiclass avx512_sqrt_packed_all<bits<8> opc, string OpcodeStr,
8652 X86SchedWriteSizes sched> {
8653 defm PSZ : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "ps"),
8654 sched.PS.ZMM, v16f32_info>,
8655 EVEX_V512, PS, EVEX_CD8<32, CD8VF>;
8656 defm PDZ : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "pd"),
8657 sched.PD.ZMM, v8f64_info>,
8658 EVEX_V512, VEX_W, PD, EVEX_CD8<64, CD8VF>;
8659 // Define only if AVX512VL feature is present.
8660 let Predicates = [HasVLX] in {
8661 defm PSZ128 : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "ps"),
8662 sched.PS.XMM, v4f32x_info>,
8663 EVEX_V128, PS, EVEX_CD8<32, CD8VF>;
8664 defm PSZ256 : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "ps"),
8665 sched.PS.YMM, v8f32x_info>,
8666 EVEX_V256, PS, EVEX_CD8<32, CD8VF>;
8667 defm PDZ128 : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "pd"),
8668 sched.PD.XMM, v2f64x_info>,
8669 EVEX_V128, VEX_W, PD, EVEX_CD8<64, CD8VF>;
8670 defm PDZ256 : avx512_sqrt_packed<opc, !strconcat(OpcodeStr, "pd"),
8671 sched.PD.YMM, v4f64x_info>,
8672 EVEX_V256, VEX_W, PD, EVEX_CD8<64, CD8VF>;
8676 multiclass avx512_sqrt_packed_all_round<bits<8> opc, string OpcodeStr,
8677 X86SchedWriteSizes sched> {
8678 defm PSZ : avx512_sqrt_packed_round<opc, !strconcat(OpcodeStr, "ps"),
8679 sched.PS.ZMM, v16f32_info>,
8680 EVEX_V512, PS, EVEX_CD8<32, CD8VF>;
8681 defm PDZ : avx512_sqrt_packed_round<opc, !strconcat(OpcodeStr, "pd"),
8682 sched.PD.ZMM, v8f64_info>,
8683 EVEX_V512, VEX_W, PD, EVEX_CD8<64, CD8VF>;
8686 multiclass avx512_sqrt_scalar<bits<8> opc, string OpcodeStr, X86FoldableSchedWrite sched,
8687 X86VectorVTInfo _, string Name> {
8688 let ExeDomain = _.ExeDomain in {
8689 defm r_Int : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
8690 (ins _.RC:$src1, _.RC:$src2), OpcodeStr,
8691 "$src2, $src1", "$src1, $src2",
8692 (X86fsqrtRnds (_.VT _.RC:$src1),
8694 (i32 FROUND_CURRENT))>,
8696 defm m_Int : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
8697 (ins _.RC:$src1, _.IntScalarMemOp:$src2), OpcodeStr,
8698 "$src2, $src1", "$src1, $src2",
8699 (X86fsqrtRnds (_.VT _.RC:$src1),
8700 _.ScalarIntMemCPat:$src2,
8701 (i32 FROUND_CURRENT))>,
8702 Sched<[sched.Folded, ReadAfterLd]>;
8703 defm rb_Int : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
8704 (ins _.RC:$src1, _.RC:$src2, AVX512RC:$rc), OpcodeStr,
8705 "$rc, $src2, $src1", "$src1, $src2, $rc",
8706 (X86fsqrtRnds (_.VT _.RC:$src1),
8709 EVEX_B, EVEX_RC, Sched<[sched]>;
8711 let isCodeGenOnly = 1, hasSideEffects = 0, Predicates=[HasAVX512] in {
8712 def r : I<opc, MRMSrcReg, (outs _.FRC:$dst),
8713 (ins _.FRC:$src1, _.FRC:$src2),
8714 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>,
8717 def m : I<opc, MRMSrcMem, (outs _.FRC:$dst),
8718 (ins _.FRC:$src1, _.ScalarMemOp:$src2),
8719 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>,
8720 Sched<[sched.Folded, ReadAfterLd]>;
8724 let Predicates = [HasAVX512] in {
8725 def : Pat<(_.EltVT (fsqrt _.FRC:$src)),
8726 (!cast<Instruction>(Name#Zr)
8727 (_.EltVT (IMPLICIT_DEF)), _.FRC:$src)>;
8730 let Predicates = [HasAVX512, OptForSize] in {
8731 def : Pat<(_.EltVT (fsqrt (load addr:$src))),
8732 (!cast<Instruction>(Name#Zm)
8733 (_.EltVT (IMPLICIT_DEF)), addr:$src)>;
8737 multiclass avx512_sqrt_scalar_all<bits<8> opc, string OpcodeStr,
8738 X86SchedWriteSizes sched> {
8739 defm SSZ : avx512_sqrt_scalar<opc, OpcodeStr#"ss", sched.PS.Scl, f32x_info, NAME#"SS">,
8740 EVEX_CD8<32, CD8VT1>, EVEX_4V, XS;
8741 defm SDZ : avx512_sqrt_scalar<opc, OpcodeStr#"sd", sched.PD.Scl, f64x_info, NAME#"SD">,
8742 EVEX_CD8<64, CD8VT1>, EVEX_4V, XD, VEX_W;
8745 defm VSQRT : avx512_sqrt_packed_all<0x51, "vsqrt", SchedWriteFSqrtSizes>,
8746 avx512_sqrt_packed_all_round<0x51, "vsqrt", SchedWriteFSqrtSizes>;
8748 defm VSQRT : avx512_sqrt_scalar_all<0x51, "vsqrt", SchedWriteFSqrtSizes>, VEX_LIG;
8750 multiclass avx512_rndscale_scalar<bits<8> opc, string OpcodeStr,
8751 X86FoldableSchedWrite sched, X86VectorVTInfo _> {
8752 let ExeDomain = _.ExeDomain in {
8753 defm r_Int : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
8754 (ins _.RC:$src1, _.RC:$src2, i32u8imm:$src3), OpcodeStr,
8755 "$src3, $src2, $src1", "$src1, $src2, $src3",
8756 (_.VT (X86RndScales (_.VT _.RC:$src1), (_.VT _.RC:$src2),
8760 defm rb_Int : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
8761 (ins _.RC:$src1, _.RC:$src2, i32u8imm:$src3), OpcodeStr,
8762 "$src3, {sae}, $src2, $src1", "$src1, $src2, {sae}, $src3",
8763 (_.VT (X86RndScalesRnd (_.VT _.RC:$src1), (_.VT _.RC:$src2),
8764 (i32 imm:$src3), (i32 FROUND_NO_EXC)))>, EVEX_B,
8767 defm m_Int : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
8768 (ins _.RC:$src1, _.IntScalarMemOp:$src2, i32u8imm:$src3),
8770 "$src3, $src2, $src1", "$src1, $src2, $src3",
8771 (_.VT (X86RndScales _.RC:$src1,
8772 _.ScalarIntMemCPat:$src2, (i32 imm:$src3)))>,
8773 Sched<[sched.Folded, ReadAfterLd]>;
8775 let isCodeGenOnly = 1, hasSideEffects = 0, Predicates = [HasAVX512] in {
8776 def r : I<opc, MRMSrcReg, (outs _.FRC:$dst),
8777 (ins _.FRC:$src1, _.FRC:$src2, i32u8imm:$src3),
8778 OpcodeStr#"\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
8779 []>, Sched<[sched]>;
8782 def m : I<opc, MRMSrcMem, (outs _.FRC:$dst),
8783 (ins _.FRC:$src1, _.ScalarMemOp:$src2, i32u8imm:$src3),
8784 OpcodeStr#"\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
8785 []>, Sched<[sched.Folded, ReadAfterLd]>;
8789 let Predicates = [HasAVX512] in {
8790 def : Pat<(ffloor _.FRC:$src),
8791 (_.EltVT (!cast<Instruction>(NAME##r) (_.EltVT (IMPLICIT_DEF)),
8792 _.FRC:$src, (i32 0x9)))>;
8793 def : Pat<(fceil _.FRC:$src),
8794 (_.EltVT (!cast<Instruction>(NAME##r) (_.EltVT (IMPLICIT_DEF)),
8795 _.FRC:$src, (i32 0xa)))>;
8796 def : Pat<(ftrunc _.FRC:$src),
8797 (_.EltVT (!cast<Instruction>(NAME##r) (_.EltVT (IMPLICIT_DEF)),
8798 _.FRC:$src, (i32 0xb)))>;
8799 def : Pat<(frint _.FRC:$src),
8800 (_.EltVT (!cast<Instruction>(NAME##r) (_.EltVT (IMPLICIT_DEF)),
8801 _.FRC:$src, (i32 0x4)))>;
8802 def : Pat<(fnearbyint _.FRC:$src),
8803 (_.EltVT (!cast<Instruction>(NAME##r) (_.EltVT (IMPLICIT_DEF)),
8804 _.FRC:$src, (i32 0xc)))>;
8807 let Predicates = [HasAVX512, OptForSize] in {
8808 def : Pat<(ffloor (_.ScalarLdFrag addr:$src)),
8809 (_.EltVT (!cast<Instruction>(NAME##m) (_.EltVT (IMPLICIT_DEF)),
8810 addr:$src, (i32 0x9)))>;
8811 def : Pat<(fceil (_.ScalarLdFrag addr:$src)),
8812 (_.EltVT (!cast<Instruction>(NAME##m) (_.EltVT (IMPLICIT_DEF)),
8813 addr:$src, (i32 0xa)))>;
8814 def : Pat<(ftrunc (_.ScalarLdFrag addr:$src)),
8815 (_.EltVT (!cast<Instruction>(NAME##m) (_.EltVT (IMPLICIT_DEF)),
8816 addr:$src, (i32 0xb)))>;
8817 def : Pat<(frint (_.ScalarLdFrag addr:$src)),
8818 (_.EltVT (!cast<Instruction>(NAME##m) (_.EltVT (IMPLICIT_DEF)),
8819 addr:$src, (i32 0x4)))>;
8820 def : Pat<(fnearbyint (_.ScalarLdFrag addr:$src)),
8821 (_.EltVT (!cast<Instruction>(NAME##m) (_.EltVT (IMPLICIT_DEF)),
8822 addr:$src, (i32 0xc)))>;
8826 defm VRNDSCALESSZ : avx512_rndscale_scalar<0x0A, "vrndscaless",
8827 SchedWriteFRnd.Scl, f32x_info>,
8828 AVX512AIi8Base, EVEX_4V,
8829 EVEX_CD8<32, CD8VT1>;
8831 defm VRNDSCALESDZ : avx512_rndscale_scalar<0x0B, "vrndscalesd",
8832 SchedWriteFRnd.Scl, f64x_info>,
8833 VEX_W, AVX512AIi8Base, EVEX_4V,
8834 EVEX_CD8<64, CD8VT1>;
8836 multiclass avx512_masked_scalar<SDNode OpNode, string OpcPrefix, SDNode Move,
8837 dag Mask, X86VectorVTInfo _, PatLeaf ZeroFP,
8838 dag OutMask, Predicate BasePredicate> {
8839 let Predicates = [BasePredicate] in {
8840 def : Pat<(Move _.VT:$src1, (scalar_to_vector (X86selects Mask,
8841 (OpNode (extractelt _.VT:$src2, (iPTR 0))),
8842 (extractelt _.VT:$dst, (iPTR 0))))),
8843 (!cast<Instruction>("V"#OpcPrefix#r_Intk)
8844 _.VT:$dst, OutMask, _.VT:$src2, _.VT:$src1)>;
8846 def : Pat<(Move _.VT:$src1, (scalar_to_vector (X86selects Mask,
8847 (OpNode (extractelt _.VT:$src2, (iPTR 0))),
8849 (!cast<Instruction>("V"#OpcPrefix#r_Intkz)
8850 OutMask, _.VT:$src2, _.VT:$src1)>;
8854 defm : avx512_masked_scalar<fsqrt, "SQRTSSZ", X86Movss,
8855 (v1i1 (scalar_to_vector (i8 (trunc (i32 GR32:$mask))))), v4f32x_info,
8856 fp32imm0, (COPY_TO_REGCLASS $mask, VK1WM), HasAVX512>;
8857 defm : avx512_masked_scalar<fsqrt, "SQRTSDZ", X86Movsd,
8858 (v1i1 (scalar_to_vector (i8 (trunc (i32 GR32:$mask))))), v2f64x_info,
8859 fp64imm0, (COPY_TO_REGCLASS $mask, VK1WM), HasAVX512>;
8861 multiclass avx512_masked_scalar_imm<SDNode OpNode, string OpcPrefix, SDNode Move,
8862 dag Mask, X86VectorVTInfo _, PatLeaf ZeroFP,
8863 bits<8> ImmV, dag OutMask,
8864 Predicate BasePredicate> {
8865 let Predicates = [BasePredicate] in {
8866 def : Pat<(Move _.VT:$src1, (scalar_to_vector (X86selects Mask,
8867 (OpNode (extractelt _.VT:$src2, (iPTR 0))),
8868 (extractelt _.VT:$dst, (iPTR 0))))),
8869 (!cast<Instruction>("V"#OpcPrefix#Zr_Intk)
8870 _.VT:$dst, OutMask, _.VT:$src1, _.VT:$src2, (i32 ImmV))>;
8872 def : Pat<(Move _.VT:$src1, (scalar_to_vector (X86selects Mask,
8873 (OpNode (extractelt _.VT:$src2, (iPTR 0))), ZeroFP))),
8874 (!cast<Instruction>("V"#OpcPrefix#Zr_Intkz)
8875 OutMask, _.VT:$src1, _.VT:$src2, (i32 ImmV))>;
8879 defm : avx512_masked_scalar_imm<ffloor, "RNDSCALESS", X86Movss,
8880 (v1i1 (scalar_to_vector GR32:$mask)),
8881 v4f32x_info, fp32imm0, 0x01,
8882 (COPY_TO_REGCLASS $mask, VK1WM), HasAVX512>;
8883 defm : avx512_masked_scalar_imm<ffloor, "RNDSCALESS", X86Movss,
8884 (v1i1 (scalar_to_vector GR8:$mask)),
8885 v4f32x_info, fp32imm0, 0x01,
8886 (COPY_TO_REGCLASS $mask, VK1WM), HasAVX512>;
8887 defm : avx512_masked_scalar_imm<fceil, "RNDSCALESS", X86Movss,
8888 (v1i1 (scalar_to_vector GR32:$mask)),
8889 v4f32x_info, fp32imm0, 0x02,
8890 (COPY_TO_REGCLASS $mask, VK1WM), HasAVX512>;
8891 defm : avx512_masked_scalar_imm<fceil, "RNDSCALESS", X86Movss,
8892 (v1i1 (scalar_to_vector GR8:$mask)),
8893 v4f32x_info, fp32imm0, 0x02,
8894 (COPY_TO_REGCLASS $mask, VK1WM), HasAVX512>;
8895 defm : avx512_masked_scalar_imm<ffloor, "RNDSCALESD", X86Movsd,
8896 (v1i1 (scalar_to_vector GR32:$mask)),
8897 v2f64x_info, fp64imm0, 0x01,
8898 (COPY_TO_REGCLASS $mask, VK1WM), HasAVX512>;
8899 defm : avx512_masked_scalar_imm<ffloor, "RNDSCALESD", X86Movsd,
8900 (v1i1 (scalar_to_vector GR8:$mask)),
8901 v2f64x_info, fp64imm0, 0x01,
8902 (COPY_TO_REGCLASS $mask, VK1WM), HasAVX512>;
8903 defm : avx512_masked_scalar_imm<fceil, "RNDSCALESD", X86Movsd,
8904 (v1i1 (scalar_to_vector GR32:$mask)),
8905 v2f64x_info, fp64imm0, 0x02,
8906 (COPY_TO_REGCLASS $mask, VK1WM), HasAVX512>;
8907 defm : avx512_masked_scalar_imm<fceil, "RNDSCALESD", X86Movsd,
8908 (v1i1 (scalar_to_vector GR8:$mask)),
8909 v2f64x_info, fp64imm0, 0x02,
8910 (COPY_TO_REGCLASS $mask, VK1WM), HasAVX512>;
8913 //-------------------------------------------------
8914 // Integer truncate and extend operations
8915 //-------------------------------------------------
8917 multiclass avx512_trunc_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
8918 X86FoldableSchedWrite sched, X86VectorVTInfo SrcInfo,
8919 X86VectorVTInfo DestInfo, X86MemOperand x86memop> {
8920 let ExeDomain = DestInfo.ExeDomain in
8921 defm rr : AVX512_maskable<opc, MRMDestReg, DestInfo, (outs DestInfo.RC:$dst),
8922 (ins SrcInfo.RC:$src1), OpcodeStr ,"$src1", "$src1",
8923 (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src1)))>,
8924 EVEX, T8XS, Sched<[sched]>;
8926 let mayStore = 1, hasSideEffects = 0, ExeDomain = DestInfo.ExeDomain in {
8927 def mr : AVX512XS8I<opc, MRMDestMem, (outs),
8928 (ins x86memop:$dst, SrcInfo.RC:$src),
8929 OpcodeStr # "\t{$src, $dst|$dst, $src}", []>,
8930 EVEX, Sched<[sched.Folded]>;
8932 def mrk : AVX512XS8I<opc, MRMDestMem, (outs),
8933 (ins x86memop:$dst, SrcInfo.KRCWM:$mask, SrcInfo.RC:$src),
8934 OpcodeStr # "\t{$src, $dst {${mask}}|$dst {${mask}}, $src}", []>,
8935 EVEX, EVEX_K, Sched<[sched.Folded]>, NotMemoryFoldable;
8936 }//mayStore = 1, hasSideEffects = 0
8939 multiclass avx512_trunc_mr_lowering<X86VectorVTInfo SrcInfo,
8940 X86VectorVTInfo DestInfo,
8941 PatFrag truncFrag, PatFrag mtruncFrag,
8944 def : Pat<(truncFrag (SrcInfo.VT SrcInfo.RC:$src), addr:$dst),
8945 (!cast<Instruction>(Name#SrcInfo.ZSuffix##mr)
8946 addr:$dst, SrcInfo.RC:$src)>;
8948 def : Pat<(mtruncFrag addr:$dst, SrcInfo.KRCWM:$mask,
8949 (SrcInfo.VT SrcInfo.RC:$src)),
8950 (!cast<Instruction>(Name#SrcInfo.ZSuffix##mrk)
8951 addr:$dst, SrcInfo.KRCWM:$mask, SrcInfo.RC:$src)>;
8954 multiclass avx512_trunc<bits<8> opc, string OpcodeStr, SDNode OpNode128,
8955 SDNode OpNode256, SDNode OpNode512, X86FoldableSchedWrite sched,
8956 AVX512VLVectorVTInfo VTSrcInfo,
8957 X86VectorVTInfo DestInfoZ128,
8958 X86VectorVTInfo DestInfoZ256, X86VectorVTInfo DestInfoZ,
8959 X86MemOperand x86memopZ128, X86MemOperand x86memopZ256,
8960 X86MemOperand x86memopZ, PatFrag truncFrag,
8961 PatFrag mtruncFrag, Predicate prd = HasAVX512>{
8963 let Predicates = [HasVLX, prd] in {
8964 defm Z128: avx512_trunc_common<opc, OpcodeStr, OpNode128, sched,
8965 VTSrcInfo.info128, DestInfoZ128, x86memopZ128>,
8966 avx512_trunc_mr_lowering<VTSrcInfo.info128, DestInfoZ128,
8967 truncFrag, mtruncFrag, NAME>, EVEX_V128;
8969 defm Z256: avx512_trunc_common<opc, OpcodeStr, OpNode256, sched,
8970 VTSrcInfo.info256, DestInfoZ256, x86memopZ256>,
8971 avx512_trunc_mr_lowering<VTSrcInfo.info256, DestInfoZ256,
8972 truncFrag, mtruncFrag, NAME>, EVEX_V256;
8974 let Predicates = [prd] in
8975 defm Z: avx512_trunc_common<opc, OpcodeStr, OpNode512, sched,
8976 VTSrcInfo.info512, DestInfoZ, x86memopZ>,
8977 avx512_trunc_mr_lowering<VTSrcInfo.info512, DestInfoZ,
8978 truncFrag, mtruncFrag, NAME>, EVEX_V512;
8981 multiclass avx512_trunc_qb<bits<8> opc, string OpcodeStr, SDNode OpNode,
8982 X86FoldableSchedWrite sched, PatFrag StoreNode,
8983 PatFrag MaskedStoreNode, SDNode InVecNode = OpNode> {
8984 defm NAME: avx512_trunc<opc, OpcodeStr, InVecNode, InVecNode, InVecNode, sched,
8985 avx512vl_i64_info, v16i8x_info, v16i8x_info,
8986 v16i8x_info, i16mem, i32mem, i64mem, StoreNode,
8987 MaskedStoreNode>, EVEX_CD8<8, CD8VO>;
8990 multiclass avx512_trunc_qw<bits<8> opc, string OpcodeStr, SDNode OpNode,
8991 X86FoldableSchedWrite sched, PatFrag StoreNode,
8992 PatFrag MaskedStoreNode, SDNode InVecNode = OpNode> {
8993 defm NAME: avx512_trunc<opc, OpcodeStr, InVecNode, InVecNode, OpNode, sched,
8994 avx512vl_i64_info, v8i16x_info, v8i16x_info,
8995 v8i16x_info, i32mem, i64mem, i128mem, StoreNode,
8996 MaskedStoreNode>, EVEX_CD8<16, CD8VQ>;
8999 multiclass avx512_trunc_qd<bits<8> opc, string OpcodeStr, SDNode OpNode,
9000 X86FoldableSchedWrite sched, PatFrag StoreNode,
9001 PatFrag MaskedStoreNode, SDNode InVecNode = OpNode> {
9002 defm NAME: avx512_trunc<opc, OpcodeStr, InVecNode, OpNode, OpNode, sched,
9003 avx512vl_i64_info, v4i32x_info, v4i32x_info,
9004 v8i32x_info, i64mem, i128mem, i256mem, StoreNode,
9005 MaskedStoreNode>, EVEX_CD8<32, CD8VH>;
9008 multiclass avx512_trunc_db<bits<8> opc, string OpcodeStr, SDNode OpNode,
9009 X86FoldableSchedWrite sched, PatFrag StoreNode,
9010 PatFrag MaskedStoreNode, SDNode InVecNode = OpNode> {
9011 defm NAME: avx512_trunc<opc, OpcodeStr, InVecNode, InVecNode, OpNode, sched,
9012 avx512vl_i32_info, v16i8x_info, v16i8x_info,
9013 v16i8x_info, i32mem, i64mem, i128mem, StoreNode,
9014 MaskedStoreNode>, EVEX_CD8<8, CD8VQ>;
9017 multiclass avx512_trunc_dw<bits<8> opc, string OpcodeStr, SDNode OpNode,
9018 X86FoldableSchedWrite sched, PatFrag StoreNode,
9019 PatFrag MaskedStoreNode, SDNode InVecNode = OpNode> {
9020 defm NAME: avx512_trunc<opc, OpcodeStr, InVecNode, OpNode, OpNode, sched,
9021 avx512vl_i32_info, v8i16x_info, v8i16x_info,
9022 v16i16x_info, i64mem, i128mem, i256mem, StoreNode,
9023 MaskedStoreNode>, EVEX_CD8<16, CD8VH>;
9026 multiclass avx512_trunc_wb<bits<8> opc, string OpcodeStr, SDNode OpNode,
9027 X86FoldableSchedWrite sched, PatFrag StoreNode,
9028 PatFrag MaskedStoreNode, SDNode InVecNode = OpNode> {
9029 defm NAME: avx512_trunc<opc, OpcodeStr, InVecNode, OpNode, OpNode,
9030 sched, avx512vl_i16_info, v16i8x_info, v16i8x_info,
9031 v32i8x_info, i64mem, i128mem, i256mem, StoreNode,
9032 MaskedStoreNode, HasBWI>, EVEX_CD8<16, CD8VH>;
9035 defm VPMOVQB : avx512_trunc_qb<0x32, "vpmovqb", trunc, WriteShuffle256,
9036 truncstorevi8, masked_truncstorevi8, X86vtrunc>;
9037 defm VPMOVSQB : avx512_trunc_qb<0x22, "vpmovsqb", X86vtruncs, WriteShuffle256,
9038 truncstore_s_vi8, masked_truncstore_s_vi8>;
9039 defm VPMOVUSQB : avx512_trunc_qb<0x12, "vpmovusqb", X86vtruncus, WriteShuffle256,
9040 truncstore_us_vi8, masked_truncstore_us_vi8>;
9042 defm VPMOVQW : avx512_trunc_qw<0x34, "vpmovqw", trunc, WriteShuffle256,
9043 truncstorevi16, masked_truncstorevi16, X86vtrunc>;
9044 defm VPMOVSQW : avx512_trunc_qw<0x24, "vpmovsqw", X86vtruncs, WriteShuffle256,
9045 truncstore_s_vi16, masked_truncstore_s_vi16>;
9046 defm VPMOVUSQW : avx512_trunc_qw<0x14, "vpmovusqw", X86vtruncus, WriteShuffle256,
9047 truncstore_us_vi16, masked_truncstore_us_vi16>;
9049 defm VPMOVQD : avx512_trunc_qd<0x35, "vpmovqd", trunc, WriteShuffle256,
9050 truncstorevi32, masked_truncstorevi32, X86vtrunc>;
9051 defm VPMOVSQD : avx512_trunc_qd<0x25, "vpmovsqd", X86vtruncs, WriteShuffle256,
9052 truncstore_s_vi32, masked_truncstore_s_vi32>;
9053 defm VPMOVUSQD : avx512_trunc_qd<0x15, "vpmovusqd", X86vtruncus, WriteShuffle256,
9054 truncstore_us_vi32, masked_truncstore_us_vi32>;
9056 defm VPMOVDB : avx512_trunc_db<0x31, "vpmovdb", trunc, WriteShuffle256,
9057 truncstorevi8, masked_truncstorevi8, X86vtrunc>;
9058 defm VPMOVSDB : avx512_trunc_db<0x21, "vpmovsdb", X86vtruncs, WriteShuffle256,
9059 truncstore_s_vi8, masked_truncstore_s_vi8>;
9060 defm VPMOVUSDB : avx512_trunc_db<0x11, "vpmovusdb", X86vtruncus, WriteShuffle256,
9061 truncstore_us_vi8, masked_truncstore_us_vi8>;
9063 defm VPMOVDW : avx512_trunc_dw<0x33, "vpmovdw", trunc, WriteShuffle256,
9064 truncstorevi16, masked_truncstorevi16, X86vtrunc>;
9065 defm VPMOVSDW : avx512_trunc_dw<0x23, "vpmovsdw", X86vtruncs, WriteShuffle256,
9066 truncstore_s_vi16, masked_truncstore_s_vi16>;
9067 defm VPMOVUSDW : avx512_trunc_dw<0x13, "vpmovusdw", X86vtruncus, WriteShuffle256,
9068 truncstore_us_vi16, masked_truncstore_us_vi16>;
9070 defm VPMOVWB : avx512_trunc_wb<0x30, "vpmovwb", trunc, WriteShuffle256,
9071 truncstorevi8, masked_truncstorevi8, X86vtrunc>;
9072 defm VPMOVSWB : avx512_trunc_wb<0x20, "vpmovswb", X86vtruncs, WriteShuffle256,
9073 truncstore_s_vi8, masked_truncstore_s_vi8>;
9074 defm VPMOVUSWB : avx512_trunc_wb<0x10, "vpmovuswb", X86vtruncus, WriteShuffle256,
9075 truncstore_us_vi8, masked_truncstore_us_vi8>;
9077 let Predicates = [HasAVX512, NoVLX] in {
9078 def: Pat<(v8i16 (trunc (v8i32 VR256X:$src))),
9079 (v8i16 (EXTRACT_SUBREG
9080 (v16i16 (VPMOVDWZrr (v16i32 (INSERT_SUBREG (IMPLICIT_DEF),
9081 VR256X:$src, sub_ymm)))), sub_xmm))>;
9082 def: Pat<(v4i32 (trunc (v4i64 VR256X:$src))),
9083 (v4i32 (EXTRACT_SUBREG
9084 (v8i32 (VPMOVQDZrr (v8i64 (INSERT_SUBREG (IMPLICIT_DEF),
9085 VR256X:$src, sub_ymm)))), sub_xmm))>;
9088 let Predicates = [HasBWI, NoVLX] in {
9089 def: Pat<(v16i8 (trunc (v16i16 VR256X:$src))),
9090 (v16i8 (EXTRACT_SUBREG (VPMOVWBZrr (v32i16 (INSERT_SUBREG (IMPLICIT_DEF),
9091 VR256X:$src, sub_ymm))), sub_xmm))>;
9094 multiclass WriteShuffle256_common<bits<8> opc, string OpcodeStr, X86FoldableSchedWrite sched,
9095 X86VectorVTInfo DestInfo, X86VectorVTInfo SrcInfo,
9096 X86MemOperand x86memop, PatFrag LdFrag, SDNode OpNode>{
9097 let ExeDomain = DestInfo.ExeDomain in {
9098 defm rr : AVX512_maskable<opc, MRMSrcReg, DestInfo, (outs DestInfo.RC:$dst),
9099 (ins SrcInfo.RC:$src), OpcodeStr ,"$src", "$src",
9100 (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src)))>,
9101 EVEX, Sched<[sched]>;
9103 defm rm : AVX512_maskable<opc, MRMSrcMem, DestInfo, (outs DestInfo.RC:$dst),
9104 (ins x86memop:$src), OpcodeStr ,"$src", "$src",
9105 (DestInfo.VT (LdFrag addr:$src))>,
9106 EVEX, Sched<[sched.Folded]>;
9110 multiclass WriteShuffle256_BW<bits<8> opc, string OpcodeStr,
9111 SDNode OpNode, SDNode InVecNode, string ExtTy,
9112 X86FoldableSchedWrite sched, PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi8")> {
9113 let Predicates = [HasVLX, HasBWI] in {
9114 defm Z128: WriteShuffle256_common<opc, OpcodeStr, sched, v8i16x_info,
9115 v16i8x_info, i64mem, LdFrag, InVecNode>,
9116 EVEX_CD8<8, CD8VH>, T8PD, EVEX_V128, VEX_WIG;
9118 defm Z256: WriteShuffle256_common<opc, OpcodeStr, sched, v16i16x_info,
9119 v16i8x_info, i128mem, LdFrag, OpNode>,
9120 EVEX_CD8<8, CD8VH>, T8PD, EVEX_V256, VEX_WIG;
9122 let Predicates = [HasBWI] in {
9123 defm Z : WriteShuffle256_common<opc, OpcodeStr, sched, v32i16_info,
9124 v32i8x_info, i256mem, LdFrag, OpNode>,
9125 EVEX_CD8<8, CD8VH>, T8PD, EVEX_V512, VEX_WIG;
9129 multiclass WriteShuffle256_BD<bits<8> opc, string OpcodeStr,
9130 SDNode OpNode, SDNode InVecNode, string ExtTy,
9131 X86FoldableSchedWrite sched, PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi8")> {
9132 let Predicates = [HasVLX, HasAVX512] in {
9133 defm Z128: WriteShuffle256_common<opc, OpcodeStr, sched, v4i32x_info,
9134 v16i8x_info, i32mem, LdFrag, InVecNode>,
9135 EVEX_CD8<8, CD8VQ>, T8PD, EVEX_V128, VEX_WIG;
9137 defm Z256: WriteShuffle256_common<opc, OpcodeStr, sched, v8i32x_info,
9138 v16i8x_info, i64mem, LdFrag, OpNode>,
9139 EVEX_CD8<8, CD8VQ>, T8PD, EVEX_V256, VEX_WIG;
9141 let Predicates = [HasAVX512] in {
9142 defm Z : WriteShuffle256_common<opc, OpcodeStr, sched, v16i32_info,
9143 v16i8x_info, i128mem, LdFrag, OpNode>,
9144 EVEX_CD8<8, CD8VQ>, T8PD, EVEX_V512, VEX_WIG;
9148 multiclass WriteShuffle256_BQ<bits<8> opc, string OpcodeStr,
9149 SDNode OpNode, SDNode InVecNode, string ExtTy,
9150 X86FoldableSchedWrite sched, PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi8")> {
9151 let Predicates = [HasVLX, HasAVX512] in {
9152 defm Z128: WriteShuffle256_common<opc, OpcodeStr, sched, v2i64x_info,
9153 v16i8x_info, i16mem, LdFrag, InVecNode>,
9154 EVEX_CD8<8, CD8VO>, T8PD, EVEX_V128, VEX_WIG;
9156 defm Z256: WriteShuffle256_common<opc, OpcodeStr, sched, v4i64x_info,
9157 v16i8x_info, i32mem, LdFrag, OpNode>,
9158 EVEX_CD8<8, CD8VO>, T8PD, EVEX_V256, VEX_WIG;
9160 let Predicates = [HasAVX512] in {
9161 defm Z : WriteShuffle256_common<opc, OpcodeStr, sched, v8i64_info,
9162 v16i8x_info, i64mem, LdFrag, OpNode>,
9163 EVEX_CD8<8, CD8VO>, T8PD, EVEX_V512, VEX_WIG;
9167 multiclass WriteShuffle256_WD<bits<8> opc, string OpcodeStr,
9168 SDNode OpNode, SDNode InVecNode, string ExtTy,
9169 X86FoldableSchedWrite sched, PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi16")> {
9170 let Predicates = [HasVLX, HasAVX512] in {
9171 defm Z128: WriteShuffle256_common<opc, OpcodeStr, sched, v4i32x_info,
9172 v8i16x_info, i64mem, LdFrag, InVecNode>,
9173 EVEX_CD8<16, CD8VH>, T8PD, EVEX_V128, VEX_WIG;
9175 defm Z256: WriteShuffle256_common<opc, OpcodeStr, sched, v8i32x_info,
9176 v8i16x_info, i128mem, LdFrag, OpNode>,
9177 EVEX_CD8<16, CD8VH>, T8PD, EVEX_V256, VEX_WIG;
9179 let Predicates = [HasAVX512] in {
9180 defm Z : WriteShuffle256_common<opc, OpcodeStr, sched, v16i32_info,
9181 v16i16x_info, i256mem, LdFrag, OpNode>,
9182 EVEX_CD8<16, CD8VH>, T8PD, EVEX_V512, VEX_WIG;
9186 multiclass WriteShuffle256_WQ<bits<8> opc, string OpcodeStr,
9187 SDNode OpNode, SDNode InVecNode, string ExtTy,
9188 X86FoldableSchedWrite sched, PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi16")> {
9189 let Predicates = [HasVLX, HasAVX512] in {
9190 defm Z128: WriteShuffle256_common<opc, OpcodeStr, sched, v2i64x_info,
9191 v8i16x_info, i32mem, LdFrag, InVecNode>,
9192 EVEX_CD8<16, CD8VQ>, T8PD, EVEX_V128, VEX_WIG;
9194 defm Z256: WriteShuffle256_common<opc, OpcodeStr, sched, v4i64x_info,
9195 v8i16x_info, i64mem, LdFrag, OpNode>,
9196 EVEX_CD8<16, CD8VQ>, T8PD, EVEX_V256, VEX_WIG;
9198 let Predicates = [HasAVX512] in {
9199 defm Z : WriteShuffle256_common<opc, OpcodeStr, sched, v8i64_info,
9200 v8i16x_info, i128mem, LdFrag, OpNode>,
9201 EVEX_CD8<16, CD8VQ>, T8PD, EVEX_V512, VEX_WIG;
9205 multiclass WriteShuffle256_DQ<bits<8> opc, string OpcodeStr,
9206 SDNode OpNode, SDNode InVecNode, string ExtTy,
9207 X86FoldableSchedWrite sched, PatFrag LdFrag = !cast<PatFrag>(ExtTy#"extloadvi32")> {
9209 let Predicates = [HasVLX, HasAVX512] in {
9210 defm Z128: WriteShuffle256_common<opc, OpcodeStr, sched, v2i64x_info,
9211 v4i32x_info, i64mem, LdFrag, InVecNode>,
9212 EVEX_CD8<32, CD8VH>, T8PD, EVEX_V128;
9214 defm Z256: WriteShuffle256_common<opc, OpcodeStr, sched, v4i64x_info,
9215 v4i32x_info, i128mem, LdFrag, OpNode>,
9216 EVEX_CD8<32, CD8VH>, T8PD, EVEX_V256;
9218 let Predicates = [HasAVX512] in {
9219 defm Z : WriteShuffle256_common<opc, OpcodeStr, sched, v8i64_info,
9220 v8i32x_info, i256mem, LdFrag, OpNode>,
9221 EVEX_CD8<32, CD8VH>, T8PD, EVEX_V512;
9225 defm VPMOVZXBW : WriteShuffle256_BW<0x30, "vpmovzxbw", X86vzext, zext_invec, "z", WriteShuffle256>;
9226 defm VPMOVZXBD : WriteShuffle256_BD<0x31, "vpmovzxbd", X86vzext, zext_invec, "z", WriteShuffle256>;
9227 defm VPMOVZXBQ : WriteShuffle256_BQ<0x32, "vpmovzxbq", X86vzext, zext_invec, "z", WriteShuffle256>;
9228 defm VPMOVZXWD : WriteShuffle256_WD<0x33, "vpmovzxwd", X86vzext, zext_invec, "z", WriteShuffle256>;
9229 defm VPMOVZXWQ : WriteShuffle256_WQ<0x34, "vpmovzxwq", X86vzext, zext_invec, "z", WriteShuffle256>;
9230 defm VPMOVZXDQ : WriteShuffle256_DQ<0x35, "vpmovzxdq", X86vzext, zext_invec, "z", WriteShuffle256>;
9232 defm VPMOVSXBW: WriteShuffle256_BW<0x20, "vpmovsxbw", X86vsext, sext_invec, "s", WriteShuffle256>;
9233 defm VPMOVSXBD: WriteShuffle256_BD<0x21, "vpmovsxbd", X86vsext, sext_invec, "s", WriteShuffle256>;
9234 defm VPMOVSXBQ: WriteShuffle256_BQ<0x22, "vpmovsxbq", X86vsext, sext_invec, "s", WriteShuffle256>;
9235 defm VPMOVSXWD: WriteShuffle256_WD<0x23, "vpmovsxwd", X86vsext, sext_invec, "s", WriteShuffle256>;
9236 defm VPMOVSXWQ: WriteShuffle256_WQ<0x24, "vpmovsxwq", X86vsext, sext_invec, "s", WriteShuffle256>;
9237 defm VPMOVSXDQ: WriteShuffle256_DQ<0x25, "vpmovsxdq", X86vsext, sext_invec, "s", WriteShuffle256>;
9240 multiclass AVX512_pmovx_patterns<string OpcPrefix, SDNode ExtOp,
9243 let Predicates = [HasVLX, HasBWI] in {
9244 def : Pat<(v8i16 (InVecOp (bc_v16i8 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))),
9245 (!cast<I>(OpcPrefix#BWZ128rm) addr:$src)>;
9246 def : Pat<(v8i16 (InVecOp (bc_v16i8 (v2f64 (scalar_to_vector (loadf64 addr:$src)))))),
9247 (!cast<I>(OpcPrefix#BWZ128rm) addr:$src)>;
9248 def : Pat<(v8i16 (InVecOp (v16i8 (vzmovl_v2i64 addr:$src)))),
9249 (!cast<I>(OpcPrefix#BWZ128rm) addr:$src)>;
9250 def : Pat<(v8i16 (InVecOp (v16i8 (vzload_v2i64 addr:$src)))),
9251 (!cast<I>(OpcPrefix#BWZ128rm) addr:$src)>;
9252 def : Pat<(v8i16 (InVecOp (bc_v16i8 (loadv2i64 addr:$src)))),
9253 (!cast<I>(OpcPrefix#BWZ128rm) addr:$src)>;
9255 let Predicates = [HasVLX] in {
9256 def : Pat<(v4i32 (InVecOp (bc_v16i8 (v4i32 (scalar_to_vector (loadi32 addr:$src)))))),
9257 (!cast<I>(OpcPrefix#BDZ128rm) addr:$src)>;
9258 def : Pat<(v4i32 (InVecOp (v16i8 (vzmovl_v4i32 addr:$src)))),
9259 (!cast<I>(OpcPrefix#BDZ128rm) addr:$src)>;
9260 def : Pat<(v4i32 (InVecOp (v16i8 (vzload_v2i64 addr:$src)))),
9261 (!cast<I>(OpcPrefix#BDZ128rm) addr:$src)>;
9262 def : Pat<(v4i32 (InVecOp (bc_v16i8 (loadv2i64 addr:$src)))),
9263 (!cast<I>(OpcPrefix#BDZ128rm) addr:$src)>;
9265 def : Pat<(v2i64 (InVecOp (bc_v16i8 (v4i32 (scalar_to_vector (extloadi32i16 addr:$src)))))),
9266 (!cast<I>(OpcPrefix#BQZ128rm) addr:$src)>;
9267 def : Pat<(v2i64 (InVecOp (v16i8 (vzmovl_v4i32 addr:$src)))),
9268 (!cast<I>(OpcPrefix#BQZ128rm) addr:$src)>;
9269 def : Pat<(v2i64 (InVecOp (v16i8 (vzload_v2i64 addr:$src)))),
9270 (!cast<I>(OpcPrefix#BQZ128rm) addr:$src)>;
9271 def : Pat<(v2i64 (InVecOp (bc_v16i8 (loadv2i64 addr:$src)))),
9272 (!cast<I>(OpcPrefix#BQZ128rm) addr:$src)>;
9274 def : Pat<(v4i32 (InVecOp (bc_v8i16 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))),
9275 (!cast<I>(OpcPrefix#WDZ128rm) addr:$src)>;
9276 def : Pat<(v4i32 (InVecOp (bc_v8i16 (v2f64 (scalar_to_vector (loadf64 addr:$src)))))),
9277 (!cast<I>(OpcPrefix#WDZ128rm) addr:$src)>;
9278 def : Pat<(v4i32 (InVecOp (v8i16 (vzmovl_v2i64 addr:$src)))),
9279 (!cast<I>(OpcPrefix#WDZ128rm) addr:$src)>;
9280 def : Pat<(v4i32 (InVecOp (v8i16 (vzload_v2i64 addr:$src)))),
9281 (!cast<I>(OpcPrefix#WDZ128rm) addr:$src)>;
9282 def : Pat<(v4i32 (InVecOp (bc_v8i16 (loadv2i64 addr:$src)))),
9283 (!cast<I>(OpcPrefix#WDZ128rm) addr:$src)>;
9285 def : Pat<(v2i64 (InVecOp (bc_v8i16 (v4i32 (scalar_to_vector (loadi32 addr:$src)))))),
9286 (!cast<I>(OpcPrefix#WQZ128rm) addr:$src)>;
9287 def : Pat<(v2i64 (InVecOp (v8i16 (vzmovl_v4i32 addr:$src)))),
9288 (!cast<I>(OpcPrefix#WQZ128rm) addr:$src)>;
9289 def : Pat<(v2i64 (InVecOp (v8i16 (vzload_v2i64 addr:$src)))),
9290 (!cast<I>(OpcPrefix#WQZ128rm) addr:$src)>;
9291 def : Pat<(v2i64 (InVecOp (bc_v8i16 (loadv2i64 addr:$src)))),
9292 (!cast<I>(OpcPrefix#WQZ128rm) addr:$src)>;
9294 def : Pat<(v2i64 (InVecOp (bc_v4i32 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))),
9295 (!cast<I>(OpcPrefix#DQZ128rm) addr:$src)>;
9296 def : Pat<(v2i64 (InVecOp (bc_v4i32 (v2f64 (scalar_to_vector (loadf64 addr:$src)))))),
9297 (!cast<I>(OpcPrefix#DQZ128rm) addr:$src)>;
9298 def : Pat<(v2i64 (InVecOp (v4i32 (vzmovl_v2i64 addr:$src)))),
9299 (!cast<I>(OpcPrefix#DQZ128rm) addr:$src)>;
9300 def : Pat<(v2i64 (InVecOp (v4i32 (vzload_v2i64 addr:$src)))),
9301 (!cast<I>(OpcPrefix#DQZ128rm) addr:$src)>;
9302 def : Pat<(v2i64 (InVecOp (bc_v4i32 (loadv2i64 addr:$src)))),
9303 (!cast<I>(OpcPrefix#DQZ128rm) addr:$src)>;
9306 let Predicates = [HasVLX, HasBWI] in {
9307 def : Pat<(v16i16 (ExtOp (bc_v16i8 (loadv2i64 addr:$src)))),
9308 (!cast<I>(OpcPrefix#BWZ256rm) addr:$src)>;
9309 def : Pat<(v16i16 (ExtOp (v16i8 (vzmovl_v2i64 addr:$src)))),
9310 (!cast<I>(OpcPrefix#BWZ256rm) addr:$src)>;
9311 def : Pat<(v16i16 (ExtOp (v16i8 (vzload_v2i64 addr:$src)))),
9312 (!cast<I>(OpcPrefix#BWZ256rm) addr:$src)>;
9314 let Predicates = [HasVLX] in {
9315 def : Pat<(v8i32 (ExtOp (bc_v16i8 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))),
9316 (!cast<I>(OpcPrefix#BDZ256rm) addr:$src)>;
9317 def : Pat<(v8i32 (ExtOp (v16i8 (vzmovl_v2i64 addr:$src)))),
9318 (!cast<I>(OpcPrefix#BDZ256rm) addr:$src)>;
9319 def : Pat<(v8i32 (ExtOp (v16i8 (vzload_v2i64 addr:$src)))),
9320 (!cast<I>(OpcPrefix#BDZ256rm) addr:$src)>;
9321 def : Pat<(v8i32 (ExtOp (bc_v16i8 (loadv2i64 addr:$src)))),
9322 (!cast<I>(OpcPrefix#BDZ256rm) addr:$src)>;
9324 def : Pat<(v4i64 (ExtOp (bc_v16i8 (v4i32 (scalar_to_vector (loadi32 addr:$src)))))),
9325 (!cast<I>(OpcPrefix#BQZ256rm) addr:$src)>;
9326 def : Pat<(v4i64 (ExtOp (v16i8 (vzmovl_v4i32 addr:$src)))),
9327 (!cast<I>(OpcPrefix#BQZ256rm) addr:$src)>;
9328 def : Pat<(v4i64 (ExtOp (v16i8 (vzload_v2i64 addr:$src)))),
9329 (!cast<I>(OpcPrefix#BQZ256rm) addr:$src)>;
9330 def : Pat<(v4i64 (ExtOp (bc_v16i8 (loadv2i64 addr:$src)))),
9331 (!cast<I>(OpcPrefix#BQZ256rm) addr:$src)>;
9333 def : Pat<(v8i32 (ExtOp (bc_v8i16 (loadv2i64 addr:$src)))),
9334 (!cast<I>(OpcPrefix#WDZ256rm) addr:$src)>;
9335 def : Pat<(v8i32 (ExtOp (v8i16 (vzmovl_v2i64 addr:$src)))),
9336 (!cast<I>(OpcPrefix#WDZ256rm) addr:$src)>;
9337 def : Pat<(v8i32 (ExtOp (v8i16 (vzload_v2i64 addr:$src)))),
9338 (!cast<I>(OpcPrefix#WDZ256rm) addr:$src)>;
9340 def : Pat<(v4i64 (ExtOp (bc_v8i16 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))),
9341 (!cast<I>(OpcPrefix#WQZ256rm) addr:$src)>;
9342 def : Pat<(v4i64 (ExtOp (v8i16 (vzmovl_v2i64 addr:$src)))),
9343 (!cast<I>(OpcPrefix#WQZ256rm) addr:$src)>;
9344 def : Pat<(v4i64 (ExtOp (v8i16 (vzload_v2i64 addr:$src)))),
9345 (!cast<I>(OpcPrefix#WQZ256rm) addr:$src)>;
9346 def : Pat<(v4i64 (ExtOp (bc_v8i16 (loadv2i64 addr:$src)))),
9347 (!cast<I>(OpcPrefix#WQZ256rm) addr:$src)>;
9349 def : Pat<(v4i64 (ExtOp (bc_v4i32 (loadv2i64 addr:$src)))),
9350 (!cast<I>(OpcPrefix#DQZ256rm) addr:$src)>;
9351 def : Pat<(v4i64 (ExtOp (v4i32 (vzmovl_v2i64 addr:$src)))),
9352 (!cast<I>(OpcPrefix#DQZ256rm) addr:$src)>;
9353 def : Pat<(v4i64 (ExtOp (v4i32 (vzload_v2i64 addr:$src)))),
9354 (!cast<I>(OpcPrefix#DQZ256rm) addr:$src)>;
9357 let Predicates = [HasBWI] in {
9358 def : Pat<(v32i16 (ExtOp (bc_v32i8 (loadv4i64 addr:$src)))),
9359 (!cast<I>(OpcPrefix#BWZrm) addr:$src)>;
9361 let Predicates = [HasAVX512] in {
9362 def : Pat<(v16i32 (ExtOp (bc_v16i8 (loadv2i64 addr:$src)))),
9363 (!cast<I>(OpcPrefix#BDZrm) addr:$src)>;
9365 def : Pat<(v8i64 (ExtOp (bc_v16i8 (v2i64 (scalar_to_vector (loadi64 addr:$src)))))),
9366 (!cast<I>(OpcPrefix#BQZrm) addr:$src)>;
9367 def : Pat<(v8i64 (ExtOp (bc_v16i8 (loadv2i64 addr:$src)))),
9368 (!cast<I>(OpcPrefix#BQZrm) addr:$src)>;
9370 def : Pat<(v16i32 (ExtOp (bc_v16i16 (loadv4i64 addr:$src)))),
9371 (!cast<I>(OpcPrefix#WDZrm) addr:$src)>;
9373 def : Pat<(v8i64 (ExtOp (bc_v8i16 (loadv2i64 addr:$src)))),
9374 (!cast<I>(OpcPrefix#WQZrm) addr:$src)>;
9376 def : Pat<(v8i64 (ExtOp (bc_v8i32 (loadv4i64 addr:$src)))),
9377 (!cast<I>(OpcPrefix#DQZrm) addr:$src)>;
9381 defm : AVX512_pmovx_patterns<"VPMOVSX", X86vsext, sext_invec>;
9382 defm : AVX512_pmovx_patterns<"VPMOVZX", X86vzext, zext_invec>;
9384 //===----------------------------------------------------------------------===//
9385 // GATHER - SCATTER Operations
9387 // FIXME: Improve scheduling of gather/scatter instructions.
9388 multiclass avx512_gather<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
9389 X86MemOperand memop, PatFrag GatherNode,
9390 RegisterClass MaskRC = _.KRCWM> {
9391 let Constraints = "@earlyclobber $dst, $src1 = $dst, $mask = $mask_wb",
9392 ExeDomain = _.ExeDomain in
9393 def rm : AVX5128I<opc, MRMSrcMem, (outs _.RC:$dst, MaskRC:$mask_wb),
9394 (ins _.RC:$src1, MaskRC:$mask, memop:$src2),
9395 !strconcat(OpcodeStr#_.Suffix,
9396 "\t{$src2, ${dst} {${mask}}|${dst} {${mask}}, $src2}"),
9397 [(set _.RC:$dst, MaskRC:$mask_wb,
9398 (GatherNode (_.VT _.RC:$src1), MaskRC:$mask,
9399 vectoraddr:$src2))]>, EVEX, EVEX_K,
9400 EVEX_CD8<_.EltSize, CD8VT1>, Sched<[WriteLoad]>;
9403 multiclass avx512_gather_q_pd<bits<8> dopc, bits<8> qopc,
9404 AVX512VLVectorVTInfo _, string OpcodeStr, string SUFF> {
9405 defm NAME##D##SUFF##Z: avx512_gather<dopc, OpcodeStr##"d", _.info512,
9406 vy512xmem, mgatherv8i32>, EVEX_V512, VEX_W;
9407 defm NAME##Q##SUFF##Z: avx512_gather<qopc, OpcodeStr##"q", _.info512,
9408 vz512mem, mgatherv8i64>, EVEX_V512, VEX_W;
9409 let Predicates = [HasVLX] in {
9410 defm NAME##D##SUFF##Z256: avx512_gather<dopc, OpcodeStr##"d", _.info256,
9411 vx256xmem, mgatherv4i32>, EVEX_V256, VEX_W;
9412 defm NAME##Q##SUFF##Z256: avx512_gather<qopc, OpcodeStr##"q", _.info256,
9413 vy256xmem, mgatherv4i64>, EVEX_V256, VEX_W;
9414 defm NAME##D##SUFF##Z128: avx512_gather<dopc, OpcodeStr##"d", _.info128,
9415 vx128xmem, mgatherv4i32>, EVEX_V128, VEX_W;
9416 defm NAME##Q##SUFF##Z128: avx512_gather<qopc, OpcodeStr##"q", _.info128,
9417 vx128xmem, mgatherv2i64>, EVEX_V128, VEX_W;
9421 multiclass avx512_gather_d_ps<bits<8> dopc, bits<8> qopc,
9422 AVX512VLVectorVTInfo _, string OpcodeStr, string SUFF> {
9423 defm NAME##D##SUFF##Z: avx512_gather<dopc, OpcodeStr##"d", _.info512, vz512mem,
9424 mgatherv16i32>, EVEX_V512;
9425 defm NAME##Q##SUFF##Z: avx512_gather<qopc, OpcodeStr##"q", _.info256, vz256mem,
9426 mgatherv8i64>, EVEX_V512;
9427 let Predicates = [HasVLX] in {
9428 defm NAME##D##SUFF##Z256: avx512_gather<dopc, OpcodeStr##"d", _.info256,
9429 vy256xmem, mgatherv8i32>, EVEX_V256;
9430 defm NAME##Q##SUFF##Z256: avx512_gather<qopc, OpcodeStr##"q", _.info128,
9431 vy128xmem, mgatherv4i64>, EVEX_V256;
9432 defm NAME##D##SUFF##Z128: avx512_gather<dopc, OpcodeStr##"d", _.info128,
9433 vx128xmem, mgatherv4i32>, EVEX_V128;
9434 defm NAME##Q##SUFF##Z128: avx512_gather<qopc, OpcodeStr##"q", _.info128,
9435 vx64xmem, mgatherv2i64, VK2WM>,
9441 defm VGATHER : avx512_gather_q_pd<0x92, 0x93, avx512vl_f64_info, "vgather", "PD">,
9442 avx512_gather_d_ps<0x92, 0x93, avx512vl_f32_info, "vgather", "PS">;
9444 defm VPGATHER : avx512_gather_q_pd<0x90, 0x91, avx512vl_i64_info, "vpgather", "Q">,
9445 avx512_gather_d_ps<0x90, 0x91, avx512vl_i32_info, "vpgather", "D">;
9447 multiclass avx512_scatter<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
9448 X86MemOperand memop, PatFrag ScatterNode,
9449 RegisterClass MaskRC = _.KRCWM> {
9451 let mayStore = 1, Constraints = "$mask = $mask_wb", ExeDomain = _.ExeDomain in
9453 def mr : AVX5128I<opc, MRMDestMem, (outs MaskRC:$mask_wb),
9454 (ins memop:$dst, MaskRC:$mask, _.RC:$src),
9455 !strconcat(OpcodeStr#_.Suffix,
9456 "\t{$src, ${dst} {${mask}}|${dst} {${mask}}, $src}"),
9457 [(set MaskRC:$mask_wb, (ScatterNode (_.VT _.RC:$src),
9458 MaskRC:$mask, vectoraddr:$dst))]>,
9459 EVEX, EVEX_K, EVEX_CD8<_.EltSize, CD8VT1>,
9460 Sched<[WriteStore]>;
9463 multiclass avx512_scatter_q_pd<bits<8> dopc, bits<8> qopc,
9464 AVX512VLVectorVTInfo _, string OpcodeStr, string SUFF> {
9465 defm NAME##D##SUFF##Z: avx512_scatter<dopc, OpcodeStr##"d", _.info512,
9466 vy512xmem, mscatterv8i32>, EVEX_V512, VEX_W;
9467 defm NAME##Q##SUFF##Z: avx512_scatter<qopc, OpcodeStr##"q", _.info512,
9468 vz512mem, mscatterv8i64>, EVEX_V512, VEX_W;
9469 let Predicates = [HasVLX] in {
9470 defm NAME##D##SUFF##Z256: avx512_scatter<dopc, OpcodeStr##"d", _.info256,
9471 vx256xmem, mscatterv4i32>, EVEX_V256, VEX_W;
9472 defm NAME##Q##SUFF##Z256: avx512_scatter<qopc, OpcodeStr##"q", _.info256,
9473 vy256xmem, mscatterv4i64>, EVEX_V256, VEX_W;
9474 defm NAME##D##SUFF##Z128: avx512_scatter<dopc, OpcodeStr##"d", _.info128,
9475 vx128xmem, mscatterv4i32>, EVEX_V128, VEX_W;
9476 defm NAME##Q##SUFF##Z128: avx512_scatter<qopc, OpcodeStr##"q", _.info128,
9477 vx128xmem, mscatterv2i64>, EVEX_V128, VEX_W;
9481 multiclass avx512_scatter_d_ps<bits<8> dopc, bits<8> qopc,
9482 AVX512VLVectorVTInfo _, string OpcodeStr, string SUFF> {
9483 defm NAME##D##SUFF##Z: avx512_scatter<dopc, OpcodeStr##"d", _.info512, vz512mem,
9484 mscatterv16i32>, EVEX_V512;
9485 defm NAME##Q##SUFF##Z: avx512_scatter<qopc, OpcodeStr##"q", _.info256, vz256mem,
9486 mscatterv8i64>, EVEX_V512;
9487 let Predicates = [HasVLX] in {
9488 defm NAME##D##SUFF##Z256: avx512_scatter<dopc, OpcodeStr##"d", _.info256,
9489 vy256xmem, mscatterv8i32>, EVEX_V256;
9490 defm NAME##Q##SUFF##Z256: avx512_scatter<qopc, OpcodeStr##"q", _.info128,
9491 vy128xmem, mscatterv4i64>, EVEX_V256;
9492 defm NAME##D##SUFF##Z128: avx512_scatter<dopc, OpcodeStr##"d", _.info128,
9493 vx128xmem, mscatterv4i32>, EVEX_V128;
9494 defm NAME##Q##SUFF##Z128: avx512_scatter<qopc, OpcodeStr##"q", _.info128,
9495 vx64xmem, mscatterv2i64, VK2WM>,
9500 defm VSCATTER : avx512_scatter_q_pd<0xA2, 0xA3, avx512vl_f64_info, "vscatter", "PD">,
9501 avx512_scatter_d_ps<0xA2, 0xA3, avx512vl_f32_info, "vscatter", "PS">;
9503 defm VPSCATTER : avx512_scatter_q_pd<0xA0, 0xA1, avx512vl_i64_info, "vpscatter", "Q">,
9504 avx512_scatter_d_ps<0xA0, 0xA1, avx512vl_i32_info, "vpscatter", "D">;
9507 multiclass avx512_gather_scatter_prefetch<bits<8> opc, Format F, string OpcodeStr,
9508 RegisterClass KRC, X86MemOperand memop> {
9509 let Predicates = [HasPFI], hasSideEffects = 1 in
9510 def m : AVX5128I<opc, F, (outs), (ins KRC:$mask, memop:$src),
9511 !strconcat(OpcodeStr, "\t{$src {${mask}}|{${mask}}, $src}"), []>,
9512 EVEX, EVEX_K, Sched<[WriteLoad]>;
9515 defm VGATHERPF0DPS: avx512_gather_scatter_prefetch<0xC6, MRM1m, "vgatherpf0dps",
9516 VK16WM, vz512mem>, EVEX_V512, EVEX_CD8<32, CD8VT1>;
9518 defm VGATHERPF0QPS: avx512_gather_scatter_prefetch<0xC7, MRM1m, "vgatherpf0qps",
9519 VK8WM, vz256mem>, EVEX_V512, EVEX_CD8<64, CD8VT1>;
9521 defm VGATHERPF0DPD: avx512_gather_scatter_prefetch<0xC6, MRM1m, "vgatherpf0dpd",
9522 VK8WM, vy512xmem>, EVEX_V512, VEX_W, EVEX_CD8<32, CD8VT1>;
9524 defm VGATHERPF0QPD: avx512_gather_scatter_prefetch<0xC7, MRM1m, "vgatherpf0qpd",
9525 VK8WM, vz512mem>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>;
9527 defm VGATHERPF1DPS: avx512_gather_scatter_prefetch<0xC6, MRM2m, "vgatherpf1dps",
9528 VK16WM, vz512mem>, EVEX_V512, EVEX_CD8<32, CD8VT1>;
9530 defm VGATHERPF1QPS: avx512_gather_scatter_prefetch<0xC7, MRM2m, "vgatherpf1qps",
9531 VK8WM, vz256mem>, EVEX_V512, EVEX_CD8<64, CD8VT1>;
9533 defm VGATHERPF1DPD: avx512_gather_scatter_prefetch<0xC6, MRM2m, "vgatherpf1dpd",
9534 VK8WM, vy512xmem>, EVEX_V512, VEX_W, EVEX_CD8<32, CD8VT1>;
9536 defm VGATHERPF1QPD: avx512_gather_scatter_prefetch<0xC7, MRM2m, "vgatherpf1qpd",
9537 VK8WM, vz512mem>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>;
9539 defm VSCATTERPF0DPS: avx512_gather_scatter_prefetch<0xC6, MRM5m, "vscatterpf0dps",
9540 VK16WM, vz512mem>, EVEX_V512, EVEX_CD8<32, CD8VT1>;
9542 defm VSCATTERPF0QPS: avx512_gather_scatter_prefetch<0xC7, MRM5m, "vscatterpf0qps",
9543 VK8WM, vz256mem>, EVEX_V512, EVEX_CD8<64, CD8VT1>;
9545 defm VSCATTERPF0DPD: avx512_gather_scatter_prefetch<0xC6, MRM5m, "vscatterpf0dpd",
9546 VK8WM, vy512xmem>, EVEX_V512, VEX_W, EVEX_CD8<32, CD8VT1>;
9548 defm VSCATTERPF0QPD: avx512_gather_scatter_prefetch<0xC7, MRM5m, "vscatterpf0qpd",
9549 VK8WM, vz512mem>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>;
9551 defm VSCATTERPF1DPS: avx512_gather_scatter_prefetch<0xC6, MRM6m, "vscatterpf1dps",
9552 VK16WM, vz512mem>, EVEX_V512, EVEX_CD8<32, CD8VT1>;
9554 defm VSCATTERPF1QPS: avx512_gather_scatter_prefetch<0xC7, MRM6m, "vscatterpf1qps",
9555 VK8WM, vz256mem>, EVEX_V512, EVEX_CD8<64, CD8VT1>;
9557 defm VSCATTERPF1DPD: avx512_gather_scatter_prefetch<0xC6, MRM6m, "vscatterpf1dpd",
9558 VK8WM, vy512xmem>, EVEX_V512, VEX_W, EVEX_CD8<32, CD8VT1>;
9560 defm VSCATTERPF1QPD: avx512_gather_scatter_prefetch<0xC7, MRM6m, "vscatterpf1qpd",
9561 VK8WM, vz512mem>, EVEX_V512, VEX_W, EVEX_CD8<64, CD8VT1>;
9563 multiclass cvt_by_vec_width<bits<8> opc, X86VectorVTInfo Vec, string OpcodeStr > {
9564 def rr : AVX512XS8I<opc, MRMSrcReg, (outs Vec.RC:$dst), (ins Vec.KRC:$src),
9565 !strconcat(OpcodeStr##Vec.Suffix, "\t{$src, $dst|$dst, $src}"),
9566 [(set Vec.RC:$dst, (Vec.VT (sext Vec.KRC:$src)))]>,
9567 EVEX, Sched<[WriteMove]>; // TODO - WriteVecTrunc?
9570 multiclass cvt_mask_by_elt_width<bits<8> opc, AVX512VLVectorVTInfo VTInfo,
9571 string OpcodeStr, Predicate prd> {
9572 let Predicates = [prd] in
9573 defm Z : cvt_by_vec_width<opc, VTInfo.info512, OpcodeStr>, EVEX_V512;
9575 let Predicates = [prd, HasVLX] in {
9576 defm Z256 : cvt_by_vec_width<opc, VTInfo.info256, OpcodeStr>, EVEX_V256;
9577 defm Z128 : cvt_by_vec_width<opc, VTInfo.info128, OpcodeStr>, EVEX_V128;
9581 defm VPMOVM2B : cvt_mask_by_elt_width<0x28, avx512vl_i8_info, "vpmovm2" , HasBWI>;
9582 defm VPMOVM2W : cvt_mask_by_elt_width<0x28, avx512vl_i16_info, "vpmovm2", HasBWI> , VEX_W;
9583 defm VPMOVM2D : cvt_mask_by_elt_width<0x38, avx512vl_i32_info, "vpmovm2", HasDQI>;
9584 defm VPMOVM2Q : cvt_mask_by_elt_width<0x38, avx512vl_i64_info, "vpmovm2", HasDQI> , VEX_W;
9586 multiclass convert_vector_to_mask_common<bits<8> opc, X86VectorVTInfo _, string OpcodeStr > {
9587 def rr : AVX512XS8I<opc, MRMSrcReg, (outs _.KRC:$dst), (ins _.RC:$src),
9588 !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
9589 [(set _.KRC:$dst, (X86pcmpgtm _.ImmAllZerosV, (_.VT _.RC:$src)))]>,
9590 EVEX, Sched<[WriteMove]>;
9593 // Use 512bit version to implement 128/256 bit in case NoVLX.
9594 multiclass convert_vector_to_mask_lowering<X86VectorVTInfo ExtendInfo,
9598 def : Pat<(_.KVT (X86pcmpgtm _.ImmAllZerosV, (_.VT _.RC:$src))),
9599 (_.KVT (COPY_TO_REGCLASS
9600 (!cast<Instruction>(Name#"Zrr")
9601 (INSERT_SUBREG (ExtendInfo.VT (IMPLICIT_DEF)),
9602 _.RC:$src, _.SubRegIdx)),
9606 multiclass avx512_convert_vector_to_mask<bits<8> opc, string OpcodeStr,
9607 AVX512VLVectorVTInfo VTInfo, Predicate prd> {
9608 let Predicates = [prd] in
9609 defm Z : convert_vector_to_mask_common <opc, VTInfo.info512, OpcodeStr>,
9612 let Predicates = [prd, HasVLX] in {
9613 defm Z256 : convert_vector_to_mask_common<opc, VTInfo.info256, OpcodeStr>,
9615 defm Z128 : convert_vector_to_mask_common<opc, VTInfo.info128, OpcodeStr>,
9618 let Predicates = [prd, NoVLX] in {
9619 defm Z256_Alt : convert_vector_to_mask_lowering<VTInfo.info512, VTInfo.info256, NAME>;
9620 defm Z128_Alt : convert_vector_to_mask_lowering<VTInfo.info512, VTInfo.info128, NAME>;
9624 defm VPMOVB2M : avx512_convert_vector_to_mask<0x29, "vpmovb2m",
9625 avx512vl_i8_info, HasBWI>;
9626 defm VPMOVW2M : avx512_convert_vector_to_mask<0x29, "vpmovw2m",
9627 avx512vl_i16_info, HasBWI>, VEX_W;
9628 defm VPMOVD2M : avx512_convert_vector_to_mask<0x39, "vpmovd2m",
9629 avx512vl_i32_info, HasDQI>;
9630 defm VPMOVQ2M : avx512_convert_vector_to_mask<0x39, "vpmovq2m",
9631 avx512vl_i64_info, HasDQI>, VEX_W;
9633 // Patterns for handling sext from a mask register to v16i8/v16i16 when DQI
9634 // is available, but BWI is not. We can't handle this in lowering because
9635 // a target independent DAG combine likes to combine sext and trunc.
9636 let Predicates = [HasDQI, NoBWI] in {
9637 def : Pat<(v16i8 (sext (v16i1 VK16:$src))),
9638 (VPMOVDBZrr (v16i32 (VPMOVM2DZrr VK16:$src)))>;
9639 def : Pat<(v16i16 (sext (v16i1 VK16:$src))),
9640 (VPMOVDWZrr (v16i32 (VPMOVM2DZrr VK16:$src)))>;
9643 //===----------------------------------------------------------------------===//
9644 // AVX-512 - COMPRESS and EXPAND
9647 multiclass compress_by_vec_width_common<bits<8> opc, X86VectorVTInfo _,
9648 string OpcodeStr, X86FoldableSchedWrite sched> {
9649 defm rr : AVX512_maskable<opc, MRMDestReg, _, (outs _.RC:$dst),
9650 (ins _.RC:$src1), OpcodeStr, "$src1", "$src1",
9651 (_.VT (X86compress _.RC:$src1))>, AVX5128IBase,
9654 let mayStore = 1, hasSideEffects = 0 in
9655 def mr : AVX5128I<opc, MRMDestMem, (outs),
9656 (ins _.MemOp:$dst, _.RC:$src),
9657 OpcodeStr # "\t{$src, $dst|$dst, $src}",
9658 []>, EVEX_CD8<_.EltSize, CD8VT1>,
9659 Sched<[sched.Folded]>;
9661 def mrk : AVX5128I<opc, MRMDestMem, (outs),
9662 (ins _.MemOp:$dst, _.KRCWM:$mask, _.RC:$src),
9663 OpcodeStr # "\t{$src, $dst {${mask}}|$dst {${mask}}, $src}",
9665 EVEX_K, EVEX_CD8<_.EltSize, CD8VT1>,
9666 Sched<[sched.Folded]>;
9669 multiclass compress_by_vec_width_lowering<X86VectorVTInfo _, string Name> {
9670 def : Pat<(X86mCompressingStore addr:$dst, _.KRCWM:$mask,
9672 (!cast<Instruction>(Name#_.ZSuffix##mrk)
9673 addr:$dst, _.KRCWM:$mask, _.RC:$src)>;
9676 multiclass compress_by_elt_width<bits<8> opc, string OpcodeStr,
9677 X86FoldableSchedWrite sched,
9678 AVX512VLVectorVTInfo VTInfo,
9679 Predicate Pred = HasAVX512> {
9680 let Predicates = [Pred] in
9681 defm Z : compress_by_vec_width_common<opc, VTInfo.info512, OpcodeStr, sched>,
9682 compress_by_vec_width_lowering<VTInfo.info512, NAME>, EVEX_V512;
9684 let Predicates = [Pred, HasVLX] in {
9685 defm Z256 : compress_by_vec_width_common<opc, VTInfo.info256, OpcodeStr, sched>,
9686 compress_by_vec_width_lowering<VTInfo.info256, NAME>, EVEX_V256;
9687 defm Z128 : compress_by_vec_width_common<opc, VTInfo.info128, OpcodeStr, sched>,
9688 compress_by_vec_width_lowering<VTInfo.info128, NAME>, EVEX_V128;
9692 // FIXME: Is there a better scheduler class for VPCOMPRESS?
9693 defm VPCOMPRESSD : compress_by_elt_width <0x8B, "vpcompressd", WriteVarShuffle256,
9694 avx512vl_i32_info>, EVEX, NotMemoryFoldable;
9695 defm VPCOMPRESSQ : compress_by_elt_width <0x8B, "vpcompressq", WriteVarShuffle256,
9696 avx512vl_i64_info>, EVEX, VEX_W, NotMemoryFoldable;
9697 defm VCOMPRESSPS : compress_by_elt_width <0x8A, "vcompressps", WriteVarShuffle256,
9698 avx512vl_f32_info>, EVEX, NotMemoryFoldable;
9699 defm VCOMPRESSPD : compress_by_elt_width <0x8A, "vcompresspd", WriteVarShuffle256,
9700 avx512vl_f64_info>, EVEX, VEX_W, NotMemoryFoldable;
9703 multiclass expand_by_vec_width<bits<8> opc, X86VectorVTInfo _,
9704 string OpcodeStr, X86FoldableSchedWrite sched> {
9705 defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
9706 (ins _.RC:$src1), OpcodeStr, "$src1", "$src1",
9707 (_.VT (X86expand _.RC:$src1))>, AVX5128IBase,
9710 defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
9711 (ins _.MemOp:$src1), OpcodeStr, "$src1", "$src1",
9712 (_.VT (X86expand (_.VT (bitconvert
9713 (_.LdFrag addr:$src1)))))>,
9714 AVX5128IBase, EVEX_CD8<_.EltSize, CD8VT1>,
9715 Sched<[sched.Folded, ReadAfterLd]>;
9718 multiclass expand_by_vec_width_lowering<X86VectorVTInfo _, string Name> {
9720 def : Pat<(_.VT (X86mExpandingLoad addr:$src, _.KRCWM:$mask, undef)),
9721 (!cast<Instruction>(Name#_.ZSuffix##rmkz)
9722 _.KRCWM:$mask, addr:$src)>;
9724 def : Pat<(_.VT (X86mExpandingLoad addr:$src, _.KRCWM:$mask, _.ImmAllZerosV)),
9725 (!cast<Instruction>(Name#_.ZSuffix##rmkz)
9726 _.KRCWM:$mask, addr:$src)>;
9728 def : Pat<(_.VT (X86mExpandingLoad addr:$src, _.KRCWM:$mask,
9729 (_.VT _.RC:$src0))),
9730 (!cast<Instruction>(Name#_.ZSuffix##rmk)
9731 _.RC:$src0, _.KRCWM:$mask, addr:$src)>;
9734 multiclass expand_by_elt_width<bits<8> opc, string OpcodeStr,
9735 X86FoldableSchedWrite sched,
9736 AVX512VLVectorVTInfo VTInfo,
9737 Predicate Pred = HasAVX512> {
9738 let Predicates = [Pred] in
9739 defm Z : expand_by_vec_width<opc, VTInfo.info512, OpcodeStr, sched>,
9740 expand_by_vec_width_lowering<VTInfo.info512, NAME>, EVEX_V512;
9742 let Predicates = [Pred, HasVLX] in {
9743 defm Z256 : expand_by_vec_width<opc, VTInfo.info256, OpcodeStr, sched>,
9744 expand_by_vec_width_lowering<VTInfo.info256, NAME>, EVEX_V256;
9745 defm Z128 : expand_by_vec_width<opc, VTInfo.info128, OpcodeStr, sched>,
9746 expand_by_vec_width_lowering<VTInfo.info128, NAME>, EVEX_V128;
9750 // FIXME: Is there a better scheduler class for VPEXPAND?
9751 defm VPEXPANDD : expand_by_elt_width <0x89, "vpexpandd", WriteVarShuffle256,
9752 avx512vl_i32_info>, EVEX;
9753 defm VPEXPANDQ : expand_by_elt_width <0x89, "vpexpandq", WriteVarShuffle256,
9754 avx512vl_i64_info>, EVEX, VEX_W;
9755 defm VEXPANDPS : expand_by_elt_width <0x88, "vexpandps", WriteVarShuffle256,
9756 avx512vl_f32_info>, EVEX;
9757 defm VEXPANDPD : expand_by_elt_width <0x88, "vexpandpd", WriteVarShuffle256,
9758 avx512vl_f64_info>, EVEX, VEX_W;
9760 //handle instruction reg_vec1 = op(reg_vec,imm)
9762 // op(broadcast(eltVt),imm)
9763 //all instruction created with FROUND_CURRENT
9764 multiclass avx512_unary_fp_packed_imm<bits<8> opc, string OpcodeStr, SDNode OpNode,
9765 X86FoldableSchedWrite sched, X86VectorVTInfo _> {
9766 let ExeDomain = _.ExeDomain in {
9767 defm rri : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
9768 (ins _.RC:$src1, i32u8imm:$src2),
9769 OpcodeStr##_.Suffix, "$src2, $src1", "$src1, $src2",
9770 (OpNode (_.VT _.RC:$src1),
9771 (i32 imm:$src2))>, Sched<[sched]>;
9772 defm rmi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
9773 (ins _.MemOp:$src1, i32u8imm:$src2),
9774 OpcodeStr##_.Suffix, "$src2, $src1", "$src1, $src2",
9775 (OpNode (_.VT (bitconvert (_.LdFrag addr:$src1))),
9777 Sched<[sched.Folded, ReadAfterLd]>;
9778 defm rmbi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
9779 (ins _.ScalarMemOp:$src1, i32u8imm:$src2),
9780 OpcodeStr##_.Suffix, "$src2, ${src1}"##_.BroadcastStr,
9781 "${src1}"##_.BroadcastStr##", $src2",
9782 (OpNode (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src1))),
9783 (i32 imm:$src2))>, EVEX_B,
9784 Sched<[sched.Folded, ReadAfterLd]>;
9788 //handle instruction reg_vec1 = op(reg_vec2,reg_vec3,imm),{sae}
9789 multiclass avx512_unary_fp_sae_packed_imm<bits<8> opc, string OpcodeStr,
9790 SDNode OpNode, X86FoldableSchedWrite sched,
9791 X86VectorVTInfo _> {
9792 let ExeDomain = _.ExeDomain in
9793 defm rrib : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
9794 (ins _.RC:$src1, i32u8imm:$src2),
9795 OpcodeStr##_.Suffix, "$src2, {sae}, $src1",
9796 "$src1, {sae}, $src2",
9797 (OpNode (_.VT _.RC:$src1),
9799 (i32 FROUND_NO_EXC))>,
9800 EVEX_B, Sched<[sched]>;
9803 multiclass avx512_common_unary_fp_sae_packed_imm<string OpcodeStr,
9804 AVX512VLVectorVTInfo _, bits<8> opc, SDNode OpNode,
9805 SDNode OpNodeRnd, X86SchedWriteWidths sched, Predicate prd>{
9806 let Predicates = [prd] in {
9807 defm Z : avx512_unary_fp_packed_imm<opc, OpcodeStr, OpNode, sched.ZMM,
9809 avx512_unary_fp_sae_packed_imm<opc, OpcodeStr, OpNodeRnd,
9810 sched.ZMM, _.info512>, EVEX_V512;
9812 let Predicates = [prd, HasVLX] in {
9813 defm Z128 : avx512_unary_fp_packed_imm<opc, OpcodeStr, OpNode, sched.XMM,
9814 _.info128>, EVEX_V128;
9815 defm Z256 : avx512_unary_fp_packed_imm<opc, OpcodeStr, OpNode, sched.YMM,
9816 _.info256>, EVEX_V256;
9820 //handle instruction reg_vec1 = op(reg_vec2,reg_vec3,imm)
9821 // op(reg_vec2,mem_vec,imm)
9822 // op(reg_vec2,broadcast(eltVt),imm)
9823 //all instruction created with FROUND_CURRENT
9824 multiclass avx512_fp_packed_imm<bits<8> opc, string OpcodeStr, SDNode OpNode,
9825 X86FoldableSchedWrite sched, X86VectorVTInfo _>{
9826 let ExeDomain = _.ExeDomain in {
9827 defm rri : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
9828 (ins _.RC:$src1, _.RC:$src2, i32u8imm:$src3),
9829 OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",
9830 (OpNode (_.VT _.RC:$src1),
9834 defm rmi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
9835 (ins _.RC:$src1, _.MemOp:$src2, i32u8imm:$src3),
9836 OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",
9837 (OpNode (_.VT _.RC:$src1),
9838 (_.VT (bitconvert (_.LdFrag addr:$src2))),
9840 Sched<[sched.Folded, ReadAfterLd]>;
9841 defm rmbi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
9842 (ins _.RC:$src1, _.ScalarMemOp:$src2, i32u8imm:$src3),
9843 OpcodeStr, "$src3, ${src2}"##_.BroadcastStr##", $src1",
9844 "$src1, ${src2}"##_.BroadcastStr##", $src3",
9845 (OpNode (_.VT _.RC:$src1),
9846 (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src2))),
9847 (i32 imm:$src3))>, EVEX_B,
9848 Sched<[sched.Folded, ReadAfterLd]>;
9852 //handle instruction reg_vec1 = op(reg_vec2,reg_vec3,imm)
9853 // op(reg_vec2,mem_vec,imm)
9854 multiclass avx512_3Op_rm_imm8<bits<8> opc, string OpcodeStr, SDNode OpNode,
9855 X86FoldableSchedWrite sched, X86VectorVTInfo DestInfo,
9856 X86VectorVTInfo SrcInfo>{
9857 let ExeDomain = DestInfo.ExeDomain in {
9858 defm rri : AVX512_maskable<opc, MRMSrcReg, DestInfo, (outs DestInfo.RC:$dst),
9859 (ins SrcInfo.RC:$src1, SrcInfo.RC:$src2, u8imm:$src3),
9860 OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",
9861 (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src1),
9862 (SrcInfo.VT SrcInfo.RC:$src2),
9865 defm rmi : AVX512_maskable<opc, MRMSrcMem, DestInfo, (outs DestInfo.RC:$dst),
9866 (ins SrcInfo.RC:$src1, SrcInfo.MemOp:$src2, u8imm:$src3),
9867 OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",
9868 (DestInfo.VT (OpNode (SrcInfo.VT SrcInfo.RC:$src1),
9869 (SrcInfo.VT (bitconvert
9870 (SrcInfo.LdFrag addr:$src2))),
9872 Sched<[sched.Folded, ReadAfterLd]>;
9876 //handle instruction reg_vec1 = op(reg_vec2,reg_vec3,imm)
9877 // op(reg_vec2,mem_vec,imm)
9878 // op(reg_vec2,broadcast(eltVt),imm)
9879 multiclass avx512_3Op_imm8<bits<8> opc, string OpcodeStr, SDNode OpNode,
9880 X86FoldableSchedWrite sched, X86VectorVTInfo _>:
9881 avx512_3Op_rm_imm8<opc, OpcodeStr, OpNode, sched, _, _>{
9883 let ExeDomain = _.ExeDomain in
9884 defm rmbi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
9885 (ins _.RC:$src1, _.ScalarMemOp:$src2, u8imm:$src3),
9886 OpcodeStr, "$src3, ${src2}"##_.BroadcastStr##", $src1",
9887 "$src1, ${src2}"##_.BroadcastStr##", $src3",
9888 (OpNode (_.VT _.RC:$src1),
9889 (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src2))),
9890 (i8 imm:$src3))>, EVEX_B,
9891 Sched<[sched.Folded, ReadAfterLd]>;
9894 //handle scalar instruction reg_vec1 = op(reg_vec2,reg_vec3,imm)
9895 // op(reg_vec2,mem_scalar,imm)
9896 multiclass avx512_fp_scalar_imm<bits<8> opc, string OpcodeStr, SDNode OpNode,
9897 X86FoldableSchedWrite sched, X86VectorVTInfo _> {
9898 let ExeDomain = _.ExeDomain in {
9899 defm rri : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
9900 (ins _.RC:$src1, _.RC:$src2, i32u8imm:$src3),
9901 OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",
9902 (OpNode (_.VT _.RC:$src1),
9906 defm rmi : AVX512_maskable_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
9907 (ins _.RC:$src1, _.ScalarMemOp:$src2, i32u8imm:$src3),
9908 OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",
9909 (OpNode (_.VT _.RC:$src1),
9910 (_.VT (scalar_to_vector
9911 (_.ScalarLdFrag addr:$src2))),
9913 Sched<[sched.Folded, ReadAfterLd]>;
9917 //handle instruction reg_vec1 = op(reg_vec2,reg_vec3,imm),{sae}
9918 multiclass avx512_fp_sae_packed_imm<bits<8> opc, string OpcodeStr,
9919 SDNode OpNode, X86FoldableSchedWrite sched,
9920 X86VectorVTInfo _> {
9921 let ExeDomain = _.ExeDomain in
9922 defm rrib : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
9923 (ins _.RC:$src1, _.RC:$src2, i32u8imm:$src3),
9924 OpcodeStr, "$src3, {sae}, $src2, $src1",
9925 "$src1, $src2, {sae}, $src3",
9926 (OpNode (_.VT _.RC:$src1),
9929 (i32 FROUND_NO_EXC))>,
9930 EVEX_B, Sched<[sched]>;
9933 //handle scalar instruction reg_vec1 = op(reg_vec2,reg_vec3,imm),{sae}
9934 multiclass avx512_fp_sae_scalar_imm<bits<8> opc, string OpcodeStr, SDNode OpNode,
9935 X86FoldableSchedWrite sched, X86VectorVTInfo _> {
9936 let ExeDomain = _.ExeDomain in
9937 defm NAME#rrib : AVX512_maskable_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
9938 (ins _.RC:$src1, _.RC:$src2, i32u8imm:$src3),
9939 OpcodeStr, "$src3, {sae}, $src2, $src1",
9940 "$src1, $src2, {sae}, $src3",
9941 (OpNode (_.VT _.RC:$src1),
9944 (i32 FROUND_NO_EXC))>,
9945 EVEX_B, Sched<[sched]>;
9948 multiclass avx512_common_fp_sae_packed_imm<string OpcodeStr,
9949 AVX512VLVectorVTInfo _, bits<8> opc, SDNode OpNode,
9950 SDNode OpNodeRnd, X86SchedWriteWidths sched, Predicate prd>{
9951 let Predicates = [prd] in {
9952 defm Z : avx512_fp_packed_imm<opc, OpcodeStr, OpNode, sched.ZMM, _.info512>,
9953 avx512_fp_sae_packed_imm<opc, OpcodeStr, OpNodeRnd, sched.ZMM, _.info512>,
9957 let Predicates = [prd, HasVLX] in {
9958 defm Z128 : avx512_fp_packed_imm<opc, OpcodeStr, OpNode, sched.XMM, _.info128>,
9960 defm Z256 : avx512_fp_packed_imm<opc, OpcodeStr, OpNode, sched.YMM, _.info256>,
9965 multiclass avx512_common_3Op_rm_imm8<bits<8> opc, SDNode OpNode, string OpStr,
9966 X86SchedWriteWidths sched, AVX512VLVectorVTInfo DestInfo,
9967 AVX512VLVectorVTInfo SrcInfo, Predicate Pred = HasBWI> {
9968 let Predicates = [Pred] in {
9969 defm Z : avx512_3Op_rm_imm8<opc, OpStr, OpNode, sched.ZMM, DestInfo.info512,
9970 SrcInfo.info512>, EVEX_V512, AVX512AIi8Base, EVEX_4V;
9972 let Predicates = [Pred, HasVLX] in {
9973 defm Z128 : avx512_3Op_rm_imm8<opc, OpStr, OpNode, sched.XMM, DestInfo.info128,
9974 SrcInfo.info128>, EVEX_V128, AVX512AIi8Base, EVEX_4V;
9975 defm Z256 : avx512_3Op_rm_imm8<opc, OpStr, OpNode, sched.YMM, DestInfo.info256,
9976 SrcInfo.info256>, EVEX_V256, AVX512AIi8Base, EVEX_4V;
9980 multiclass avx512_common_3Op_imm8<string OpcodeStr, AVX512VLVectorVTInfo _,
9981 bits<8> opc, SDNode OpNode, X86SchedWriteWidths sched,
9982 Predicate Pred = HasAVX512> {
9983 let Predicates = [Pred] in {
9984 defm Z : avx512_3Op_imm8<opc, OpcodeStr, OpNode, sched.ZMM, _.info512>,
9987 let Predicates = [Pred, HasVLX] in {
9988 defm Z128 : avx512_3Op_imm8<opc, OpcodeStr, OpNode, sched.XMM, _.info128>,
9990 defm Z256 : avx512_3Op_imm8<opc, OpcodeStr, OpNode, sched.YMM, _.info256>,
9995 multiclass avx512_common_fp_sae_scalar_imm<string OpcodeStr,
9996 X86VectorVTInfo _, bits<8> opc, SDNode OpNode,
9997 SDNode OpNodeRnd, X86SchedWriteWidths sched, Predicate prd> {
9998 let Predicates = [prd] in {
9999 defm Z : avx512_fp_scalar_imm<opc, OpcodeStr, OpNode, sched.XMM, _>,
10000 avx512_fp_sae_scalar_imm<opc, OpcodeStr, OpNodeRnd, sched.XMM, _>;
10004 multiclass avx512_common_unary_fp_sae_packed_imm_all<string OpcodeStr,
10005 bits<8> opcPs, bits<8> opcPd, SDNode OpNode,
10006 SDNode OpNodeRnd, X86SchedWriteWidths sched, Predicate prd>{
10007 defm PS : avx512_common_unary_fp_sae_packed_imm<OpcodeStr, avx512vl_f32_info,
10008 opcPs, OpNode, OpNodeRnd, sched, prd>,
10009 EVEX_CD8<32, CD8VF>;
10010 defm PD : avx512_common_unary_fp_sae_packed_imm<OpcodeStr, avx512vl_f64_info,
10011 opcPd, OpNode, OpNodeRnd, sched, prd>,
10012 EVEX_CD8<64, CD8VF>, VEX_W;
10015 defm VREDUCE : avx512_common_unary_fp_sae_packed_imm_all<"vreduce", 0x56, 0x56,
10016 X86VReduce, X86VReduceRnd, SchedWriteFRnd, HasDQI>,
10017 AVX512AIi8Base, EVEX;
10018 defm VRNDSCALE : avx512_common_unary_fp_sae_packed_imm_all<"vrndscale", 0x08, 0x09,
10019 X86VRndScale, X86VRndScaleRnd, SchedWriteFRnd, HasAVX512>,
10020 AVX512AIi8Base, EVEX;
10021 defm VGETMANT : avx512_common_unary_fp_sae_packed_imm_all<"vgetmant", 0x26, 0x26,
10022 X86VGetMant, X86VGetMantRnd, SchedWriteFRnd, HasAVX512>,
10023 AVX512AIi8Base, EVEX;
10025 defm VRANGEPD : avx512_common_fp_sae_packed_imm<"vrangepd", avx512vl_f64_info,
10026 0x50, X86VRange, X86VRangeRnd,
10027 SchedWriteFAdd, HasDQI>,
10028 AVX512AIi8Base, EVEX_4V, EVEX_CD8<64, CD8VF>, VEX_W;
10029 defm VRANGEPS : avx512_common_fp_sae_packed_imm<"vrangeps", avx512vl_f32_info,
10030 0x50, X86VRange, X86VRangeRnd,
10031 SchedWriteFAdd, HasDQI>,
10032 AVX512AIi8Base, EVEX_4V, EVEX_CD8<32, CD8VF>;
10034 defm VRANGESD: avx512_common_fp_sae_scalar_imm<"vrangesd",
10035 f64x_info, 0x51, X86Ranges, X86RangesRnd, SchedWriteFAdd, HasDQI>,
10036 AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<64, CD8VT1>, VEX_W;
10037 defm VRANGESS: avx512_common_fp_sae_scalar_imm<"vrangess", f32x_info,
10038 0x51, X86Ranges, X86RangesRnd, SchedWriteFAdd, HasDQI>,
10039 AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<32, CD8VT1>;
10041 defm VREDUCESD: avx512_common_fp_sae_scalar_imm<"vreducesd", f64x_info,
10042 0x57, X86Reduces, X86ReducesRnd, SchedWriteFRnd, HasDQI>,
10043 AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<64, CD8VT1>, VEX_W;
10044 defm VREDUCESS: avx512_common_fp_sae_scalar_imm<"vreducess", f32x_info,
10045 0x57, X86Reduces, X86ReducesRnd, SchedWriteFRnd, HasDQI>,
10046 AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<32, CD8VT1>;
10048 defm VGETMANTSD: avx512_common_fp_sae_scalar_imm<"vgetmantsd", f64x_info,
10049 0x27, X86GetMants, X86GetMantsRnd, SchedWriteFRnd, HasAVX512>,
10050 AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<64, CD8VT1>, VEX_W;
10051 defm VGETMANTSS: avx512_common_fp_sae_scalar_imm<"vgetmantss", f32x_info,
10052 0x27, X86GetMants, X86GetMantsRnd, SchedWriteFRnd, HasAVX512>,
10053 AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<32, CD8VT1>;
10055 let Predicates = [HasAVX512] in {
10056 def : Pat<(v16f32 (ffloor VR512:$src)),
10057 (VRNDSCALEPSZrri VR512:$src, (i32 0x9))>;
10058 def : Pat<(v16f32 (vselect VK16WM:$mask, (ffloor VR512:$src), VR512:$dst)),
10059 (VRNDSCALEPSZrrik VR512:$dst, VK16WM:$mask, VR512:$src, (i32 0x9))>;
10060 def : Pat<(v16f32 (vselect VK16WM:$mask, (ffloor VR512:$src), v16f32_info.ImmAllZerosV)),
10061 (VRNDSCALEPSZrrikz VK16WM:$mask, VR512:$src, (i32 0x9))>;
10062 def : Pat<(v16f32 (fnearbyint VR512:$src)),
10063 (VRNDSCALEPSZrri VR512:$src, (i32 0xC))>;
10064 def : Pat<(v16f32 (fceil VR512:$src)),
10065 (VRNDSCALEPSZrri VR512:$src, (i32 0xA))>;
10066 def : Pat<(v16f32 (vselect VK16WM:$mask, (fceil VR512:$src), VR512:$dst)),
10067 (VRNDSCALEPSZrrik VR512:$dst, VK16WM:$mask, VR512:$src, (i32 0xA))>;
10068 def : Pat<(v16f32 (vselect VK16WM:$mask, (fceil VR512:$src), v16f32_info.ImmAllZerosV)),
10069 (VRNDSCALEPSZrrikz VK16WM:$mask, VR512:$src, (i32 0xA))>;
10070 def : Pat<(v16f32 (frint VR512:$src)),
10071 (VRNDSCALEPSZrri VR512:$src, (i32 0x4))>;
10072 def : Pat<(v16f32 (ftrunc VR512:$src)),
10073 (VRNDSCALEPSZrri VR512:$src, (i32 0xB))>;
10075 def : Pat<(v16f32 (ffloor (loadv16f32 addr:$src))),
10076 (VRNDSCALEPSZrmi addr:$src, (i32 0x9))>;
10077 def : Pat<(v16f32 (fnearbyint (loadv16f32 addr:$src))),
10078 (VRNDSCALEPSZrmi addr:$src, (i32 0xC))>;
10079 def : Pat<(v16f32 (fceil (loadv16f32 addr:$src))),
10080 (VRNDSCALEPSZrmi addr:$src, (i32 0xA))>;
10081 def : Pat<(v16f32 (frint (loadv16f32 addr:$src))),
10082 (VRNDSCALEPSZrmi addr:$src, (i32 0x4))>;
10083 def : Pat<(v16f32 (ftrunc (loadv16f32 addr:$src))),
10084 (VRNDSCALEPSZrmi addr:$src, (i32 0xB))>;
10086 def : Pat<(v8f64 (ffloor VR512:$src)),
10087 (VRNDSCALEPDZrri VR512:$src, (i32 0x9))>;
10088 def : Pat<(v8f64 (vselect VK8WM:$mask, (ffloor VR512:$src), VR512:$dst)),
10089 (VRNDSCALEPDZrrik VR512:$dst, VK8WM:$mask, VR512:$src, (i32 0x9))>;
10090 def : Pat<(v8f64 (vselect VK8WM:$mask, (ffloor VR512:$src), v8f64_info.ImmAllZerosV)),
10091 (VRNDSCALEPDZrrikz VK8WM:$mask, VR512:$src, (i32 0x9))>;
10092 def : Pat<(v8f64 (fnearbyint VR512:$src)),
10093 (VRNDSCALEPDZrri VR512:$src, (i32 0xC))>;
10094 def : Pat<(v8f64 (fceil VR512:$src)),
10095 (VRNDSCALEPDZrri VR512:$src, (i32 0xA))>;
10096 def : Pat<(v8f64 (vselect VK8WM:$mask, (fceil VR512:$src), VR512:$dst)),
10097 (VRNDSCALEPDZrrik VR512:$dst, VK8WM:$mask, VR512:$src, (i32 0xA))>;
10098 def : Pat<(v8f64 (vselect VK8WM:$mask, (fceil VR512:$src), v8f64_info.ImmAllZerosV)),
10099 (VRNDSCALEPDZrrikz VK8WM:$mask, VR512:$src, (i32 0xA))>;
10100 def : Pat<(v8f64 (frint VR512:$src)),
10101 (VRNDSCALEPDZrri VR512:$src, (i32 0x4))>;
10102 def : Pat<(v8f64 (ftrunc VR512:$src)),
10103 (VRNDSCALEPDZrri VR512:$src, (i32 0xB))>;
10105 def : Pat<(v8f64 (ffloor (loadv8f64 addr:$src))),
10106 (VRNDSCALEPDZrmi addr:$src, (i32 0x9))>;
10107 def : Pat<(v8f64 (fnearbyint (loadv8f64 addr:$src))),
10108 (VRNDSCALEPDZrmi addr:$src, (i32 0xC))>;
10109 def : Pat<(v8f64 (fceil (loadv8f64 addr:$src))),
10110 (VRNDSCALEPDZrmi addr:$src, (i32 0xA))>;
10111 def : Pat<(v8f64 (frint (loadv8f64 addr:$src))),
10112 (VRNDSCALEPDZrmi addr:$src, (i32 0x4))>;
10113 def : Pat<(v8f64 (ftrunc (loadv8f64 addr:$src))),
10114 (VRNDSCALEPDZrmi addr:$src, (i32 0xB))>;
10117 let Predicates = [HasVLX] in {
10118 def : Pat<(v4f32 (ffloor VR128X:$src)),
10119 (VRNDSCALEPSZ128rri VR128X:$src, (i32 0x9))>;
10120 def : Pat<(v4f32 (vselect VK4WM:$mask, (ffloor VR128X:$src), VR128X:$dst)),
10121 (VRNDSCALEPSZ128rrik VR128X:$dst, VK4WM:$mask, VR128X:$src, (i32 0x9))>;
10122 def : Pat<(v4f32 (vselect VK4WM:$mask, (ffloor VR128X:$src), v4f32x_info.ImmAllZerosV)),
10123 (VRNDSCALEPSZ128rrikz VK4WM:$mask, VR128X:$src, (i32 0x9))>;
10124 def : Pat<(v4f32 (fnearbyint VR128X:$src)),
10125 (VRNDSCALEPSZ128rri VR128X:$src, (i32 0xC))>;
10126 def : Pat<(v4f32 (fceil VR128X:$src)),
10127 (VRNDSCALEPSZ128rri VR128X:$src, (i32 0xA))>;
10128 def : Pat<(v4f32 (vselect VK4WM:$mask, (fceil VR128X:$src), VR128X:$dst)),
10129 (VRNDSCALEPSZ128rrik VR128X:$dst, VK4WM:$mask, VR128X:$src, (i32 0xA))>;
10130 def : Pat<(v4f32 (vselect VK4WM:$mask, (fceil VR128X:$src), v4f32x_info.ImmAllZerosV)),
10131 (VRNDSCALEPSZ128rrikz VK4WM:$mask, VR128X:$src, (i32 0xA))>;
10132 def : Pat<(v4f32 (frint VR128X:$src)),
10133 (VRNDSCALEPSZ128rri VR128X:$src, (i32 0x4))>;
10134 def : Pat<(v4f32 (ftrunc VR128X:$src)),
10135 (VRNDSCALEPSZ128rri VR128X:$src, (i32 0xB))>;
10137 def : Pat<(v4f32 (ffloor (loadv4f32 addr:$src))),
10138 (VRNDSCALEPSZ128rmi addr:$src, (i32 0x9))>;
10139 def : Pat<(v4f32 (fnearbyint (loadv4f32 addr:$src))),
10140 (VRNDSCALEPSZ128rmi addr:$src, (i32 0xC))>;
10141 def : Pat<(v4f32 (fceil (loadv4f32 addr:$src))),
10142 (VRNDSCALEPSZ128rmi addr:$src, (i32 0xA))>;
10143 def : Pat<(v4f32 (frint (loadv4f32 addr:$src))),
10144 (VRNDSCALEPSZ128rmi addr:$src, (i32 0x4))>;
10145 def : Pat<(v4f32 (ftrunc (loadv4f32 addr:$src))),
10146 (VRNDSCALEPSZ128rmi addr:$src, (i32 0xB))>;
10148 def : Pat<(v2f64 (ffloor VR128X:$src)),
10149 (VRNDSCALEPDZ128rri VR128X:$src, (i32 0x9))>;
10150 def : Pat<(v2f64 (vselect VK2WM:$mask, (ffloor VR128X:$src), VR128X:$dst)),
10151 (VRNDSCALEPDZ128rrik VR128X:$dst, VK2WM:$mask, VR128X:$src, (i32 0x9))>;
10152 def : Pat<(v2f64 (vselect VK2WM:$mask, (ffloor VR128X:$src), v2f64x_info.ImmAllZerosV)),
10153 (VRNDSCALEPDZ128rrikz VK2WM:$mask, VR128X:$src, (i32 0x9))>;
10154 def : Pat<(v2f64 (fnearbyint VR128X:$src)),
10155 (VRNDSCALEPDZ128rri VR128X:$src, (i32 0xC))>;
10156 def : Pat<(v2f64 (fceil VR128X:$src)),
10157 (VRNDSCALEPDZ128rri VR128X:$src, (i32 0xA))>;
10158 def : Pat<(v2f64 (vselect VK2WM:$mask, (fceil VR128X:$src), VR128X:$dst)),
10159 (VRNDSCALEPDZ128rrik VR128X:$dst, VK2WM:$mask, VR128X:$src, (i32 0xA))>;
10160 def : Pat<(v2f64 (vselect VK2WM:$mask, (fceil VR128X:$src), v2f64x_info.ImmAllZerosV)),
10161 (VRNDSCALEPDZ128rrikz VK2WM:$mask, VR128X:$src, (i32 0xA))>;
10162 def : Pat<(v2f64 (frint VR128X:$src)),
10163 (VRNDSCALEPDZ128rri VR128X:$src, (i32 0x4))>;
10164 def : Pat<(v2f64 (ftrunc VR128X:$src)),
10165 (VRNDSCALEPDZ128rri VR128X:$src, (i32 0xB))>;
10167 def : Pat<(v2f64 (ffloor (loadv2f64 addr:$src))),
10168 (VRNDSCALEPDZ128rmi addr:$src, (i32 0x9))>;
10169 def : Pat<(v2f64 (fnearbyint (loadv2f64 addr:$src))),
10170 (VRNDSCALEPDZ128rmi addr:$src, (i32 0xC))>;
10171 def : Pat<(v2f64 (fceil (loadv2f64 addr:$src))),
10172 (VRNDSCALEPDZ128rmi addr:$src, (i32 0xA))>;
10173 def : Pat<(v2f64 (frint (loadv2f64 addr:$src))),
10174 (VRNDSCALEPDZ128rmi addr:$src, (i32 0x4))>;
10175 def : Pat<(v2f64 (ftrunc (loadv2f64 addr:$src))),
10176 (VRNDSCALEPDZ128rmi addr:$src, (i32 0xB))>;
10178 def : Pat<(v8f32 (ffloor VR256X:$src)),
10179 (VRNDSCALEPSZ256rri VR256X:$src, (i32 0x9))>;
10180 def : Pat<(v8f32 (vselect VK8WM:$mask, (ffloor VR256X:$src), VR256X:$dst)),
10181 (VRNDSCALEPSZ256rrik VR256X:$dst, VK8WM:$mask, VR256X:$src, (i32 0x9))>;
10182 def : Pat<(v8f32 (vselect VK8WM:$mask, (ffloor VR256X:$src), v8f32x_info.ImmAllZerosV)),
10183 (VRNDSCALEPSZ256rrikz VK8WM:$mask, VR256X:$src, (i32 0x9))>;
10184 def : Pat<(v8f32 (fnearbyint VR256X:$src)),
10185 (VRNDSCALEPSZ256rri VR256X:$src, (i32 0xC))>;
10186 def : Pat<(v8f32 (fceil VR256X:$src)),
10187 (VRNDSCALEPSZ256rri VR256X:$src, (i32 0xA))>;
10188 def : Pat<(v8f32 (vselect VK8WM:$mask, (fceil VR256X:$src), VR256X:$dst)),
10189 (VRNDSCALEPSZ256rrik VR256X:$dst, VK8WM:$mask, VR256X:$src, (i32 0xA))>;
10190 def : Pat<(v8f32 (vselect VK8WM:$mask, (fceil VR256X:$src), v8f32x_info.ImmAllZerosV)),
10191 (VRNDSCALEPSZ256rrikz VK8WM:$mask, VR256X:$src, (i32 0xA))>;
10192 def : Pat<(v8f32 (frint VR256X:$src)),
10193 (VRNDSCALEPSZ256rri VR256X:$src, (i32 0x4))>;
10194 def : Pat<(v8f32 (ftrunc VR256X:$src)),
10195 (VRNDSCALEPSZ256rri VR256X:$src, (i32 0xB))>;
10197 def : Pat<(v8f32 (ffloor (loadv8f32 addr:$src))),
10198 (VRNDSCALEPSZ256rmi addr:$src, (i32 0x9))>;
10199 def : Pat<(v8f32 (fnearbyint (loadv8f32 addr:$src))),
10200 (VRNDSCALEPSZ256rmi addr:$src, (i32 0xC))>;
10201 def : Pat<(v8f32 (fceil (loadv8f32 addr:$src))),
10202 (VRNDSCALEPSZ256rmi addr:$src, (i32 0xA))>;
10203 def : Pat<(v8f32 (frint (loadv8f32 addr:$src))),
10204 (VRNDSCALEPSZ256rmi addr:$src, (i32 0x4))>;
10205 def : Pat<(v8f32 (ftrunc (loadv8f32 addr:$src))),
10206 (VRNDSCALEPSZ256rmi addr:$src, (i32 0xB))>;
10208 def : Pat<(v4f64 (ffloor VR256X:$src)),
10209 (VRNDSCALEPDZ256rri VR256X:$src, (i32 0x9))>;
10210 def : Pat<(v4f64 (vselect VK4WM:$mask, (ffloor VR256X:$src), VR256X:$dst)),
10211 (VRNDSCALEPDZ256rrik VR256X:$dst, VK4WM:$mask, VR256X:$src, (i32 0x9))>;
10212 def : Pat<(v4f64 (vselect VK4WM:$mask, (ffloor VR256X:$src), v4f64x_info.ImmAllZerosV)),
10213 (VRNDSCALEPDZ256rrikz VK4WM:$mask, VR256X:$src, (i32 0x9))>;
10214 def : Pat<(v4f64 (fnearbyint VR256X:$src)),
10215 (VRNDSCALEPDZ256rri VR256X:$src, (i32 0xC))>;
10216 def : Pat<(v4f64 (fceil VR256X:$src)),
10217 (VRNDSCALEPDZ256rri VR256X:$src, (i32 0xA))>;
10218 def : Pat<(v4f64 (vselect VK4WM:$mask, (fceil VR256X:$src), VR256X:$dst)),
10219 (VRNDSCALEPDZ256rrik VR256X:$dst, VK4WM:$mask, VR256X:$src, (i32 0xA))>;
10220 def : Pat<(v4f64 (vselect VK4WM:$mask, (fceil VR256X:$src), v4f64x_info.ImmAllZerosV)),
10221 (VRNDSCALEPDZ256rrikz VK4WM:$mask, VR256X:$src, (i32 0xA))>;
10222 def : Pat<(v4f64 (frint VR256X:$src)),
10223 (VRNDSCALEPDZ256rri VR256X:$src, (i32 0x4))>;
10224 def : Pat<(v4f64 (ftrunc VR256X:$src)),
10225 (VRNDSCALEPDZ256rri VR256X:$src, (i32 0xB))>;
10227 def : Pat<(v4f64 (ffloor (loadv4f64 addr:$src))),
10228 (VRNDSCALEPDZ256rmi addr:$src, (i32 0x9))>;
10229 def : Pat<(v4f64 (fnearbyint (loadv4f64 addr:$src))),
10230 (VRNDSCALEPDZ256rmi addr:$src, (i32 0xC))>;
10231 def : Pat<(v4f64 (fceil (loadv4f64 addr:$src))),
10232 (VRNDSCALEPDZ256rmi addr:$src, (i32 0xA))>;
10233 def : Pat<(v4f64 (frint (loadv4f64 addr:$src))),
10234 (VRNDSCALEPDZ256rmi addr:$src, (i32 0x4))>;
10235 def : Pat<(v4f64 (ftrunc (loadv4f64 addr:$src))),
10236 (VRNDSCALEPDZ256rmi addr:$src, (i32 0xB))>;
10239 multiclass avx512_shuff_packed_128_common<bits<8> opc, string OpcodeStr,
10240 X86FoldableSchedWrite sched,
10242 X86VectorVTInfo CastInfo,
10243 string EVEX2VEXOvrd> {
10244 let ExeDomain = _.ExeDomain in {
10245 defm rri : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
10246 (ins _.RC:$src1, _.RC:$src2, u8imm:$src3),
10247 OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",
10249 (CastInfo.VT (X86Shuf128 _.RC:$src1, _.RC:$src2,
10250 (i8 imm:$src3)))))>,
10251 Sched<[sched]>, EVEX2VEXOverride<EVEX2VEXOvrd#"rr">;
10252 defm rmi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
10253 (ins _.RC:$src1, _.MemOp:$src2, u8imm:$src3),
10254 OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",
10257 (CastInfo.VT (X86Shuf128 _.RC:$src1,
10258 (bitconvert (_.LdFrag addr:$src2)),
10259 (i8 imm:$src3)))))>,
10260 Sched<[sched.Folded, ReadAfterLd]>,
10261 EVEX2VEXOverride<EVEX2VEXOvrd#"rm">;
10262 defm rmbi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
10263 (ins _.RC:$src1, _.ScalarMemOp:$src2, u8imm:$src3),
10264 OpcodeStr, "$src3, ${src2}"##_.BroadcastStr##", $src1",
10265 "$src1, ${src2}"##_.BroadcastStr##", $src3",
10269 (X86Shuf128 _.RC:$src1,
10270 (X86VBroadcast (_.ScalarLdFrag addr:$src2)),
10271 (i8 imm:$src3)))))>, EVEX_B,
10272 Sched<[sched.Folded, ReadAfterLd]>;
10276 multiclass avx512_shuff_packed_128<string OpcodeStr, X86FoldableSchedWrite sched,
10277 AVX512VLVectorVTInfo _,
10278 AVX512VLVectorVTInfo CastInfo, bits<8> opc,
10279 string EVEX2VEXOvrd>{
10280 let Predicates = [HasAVX512] in
10281 defm Z : avx512_shuff_packed_128_common<opc, OpcodeStr, sched,
10282 _.info512, CastInfo.info512, "">, EVEX_V512;
10284 let Predicates = [HasAVX512, HasVLX] in
10285 defm Z256 : avx512_shuff_packed_128_common<opc, OpcodeStr, sched,
10286 _.info256, CastInfo.info256,
10287 EVEX2VEXOvrd>, EVEX_V256;
10290 defm VSHUFF32X4 : avx512_shuff_packed_128<"vshuff32x4", WriteFShuffle256,
10291 avx512vl_f32_info, avx512vl_f64_info, 0x23, "VPERM2F128">, AVX512AIi8Base, EVEX_4V, EVEX_CD8<32, CD8VF>;
10292 defm VSHUFF64X2 : avx512_shuff_packed_128<"vshuff64x2", WriteFShuffle256,
10293 avx512vl_f64_info, avx512vl_f64_info, 0x23, "VPERM2F128">, AVX512AIi8Base, EVEX_4V, EVEX_CD8<64, CD8VF>, VEX_W;
10294 defm VSHUFI32X4 : avx512_shuff_packed_128<"vshufi32x4", WriteFShuffle256,
10295 avx512vl_i32_info, avx512vl_i64_info, 0x43, "VPERM2I128">, AVX512AIi8Base, EVEX_4V, EVEX_CD8<32, CD8VF>;
10296 defm VSHUFI64X2 : avx512_shuff_packed_128<"vshufi64x2", WriteFShuffle256,
10297 avx512vl_i64_info, avx512vl_i64_info, 0x43, "VPERM2I128">, AVX512AIi8Base, EVEX_4V, EVEX_CD8<64, CD8VF>, VEX_W;
10299 let Predicates = [HasAVX512] in {
10300 // Provide fallback in case the load node that is used in the broadcast
10301 // patterns above is used by additional users, which prevents the pattern
10303 def : Pat<(v8f64 (X86SubVBroadcast (v2f64 VR128X:$src))),
10304 (VSHUFF64X2Zrri (INSERT_SUBREG (v8f64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
10305 (INSERT_SUBREG (v8f64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
10307 def : Pat<(v8i64 (X86SubVBroadcast (v2i64 VR128X:$src))),
10308 (VSHUFI64X2Zrri (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
10309 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
10312 def : Pat<(v16f32 (X86SubVBroadcast (v4f32 VR128X:$src))),
10313 (VSHUFF32X4Zrri (INSERT_SUBREG (v16f32 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
10314 (INSERT_SUBREG (v16f32 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
10316 def : Pat<(v16i32 (X86SubVBroadcast (v4i32 VR128X:$src))),
10317 (VSHUFI32X4Zrri (INSERT_SUBREG (v16i32 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
10318 (INSERT_SUBREG (v16i32 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
10321 def : Pat<(v32i16 (X86SubVBroadcast (v8i16 VR128X:$src))),
10322 (VSHUFI32X4Zrri (INSERT_SUBREG (v32i16 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
10323 (INSERT_SUBREG (v32i16 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
10326 def : Pat<(v64i8 (X86SubVBroadcast (v16i8 VR128X:$src))),
10327 (VSHUFI32X4Zrri (INSERT_SUBREG (v64i8 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
10328 (INSERT_SUBREG (v64i8 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
10332 multiclass avx512_valign<bits<8> opc, string OpcodeStr,
10333 X86FoldableSchedWrite sched, X86VectorVTInfo _>{
10334 // NOTE: EVEX2VEXOverride changed back to Unset for 256-bit at the
10335 // instantiation of this class.
10336 let ExeDomain = _.ExeDomain in {
10337 defm rri : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
10338 (ins _.RC:$src1, _.RC:$src2, u8imm:$src3),
10339 OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",
10340 (_.VT (X86VAlign _.RC:$src1, _.RC:$src2, (i8 imm:$src3)))>,
10341 Sched<[sched]>, EVEX2VEXOverride<"VPALIGNRrri">;
10342 defm rmi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
10343 (ins _.RC:$src1, _.MemOp:$src2, u8imm:$src3),
10344 OpcodeStr, "$src3, $src2, $src1", "$src1, $src2, $src3",
10345 (_.VT (X86VAlign _.RC:$src1,
10346 (bitconvert (_.LdFrag addr:$src2)),
10348 Sched<[sched.Folded, ReadAfterLd]>,
10349 EVEX2VEXOverride<"VPALIGNRrmi">;
10351 defm rmbi : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
10352 (ins _.RC:$src1, _.ScalarMemOp:$src2, u8imm:$src3),
10353 OpcodeStr, "$src3, ${src2}"##_.BroadcastStr##", $src1",
10354 "$src1, ${src2}"##_.BroadcastStr##", $src3",
10355 (X86VAlign _.RC:$src1,
10356 (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src2))),
10357 (i8 imm:$src3))>, EVEX_B,
10358 Sched<[sched.Folded, ReadAfterLd]>;
10362 multiclass avx512_valign_common<string OpcodeStr, X86SchedWriteWidths sched,
10363 AVX512VLVectorVTInfo _> {
10364 let Predicates = [HasAVX512] in {
10365 defm Z : avx512_valign<0x03, OpcodeStr, sched.ZMM, _.info512>,
10366 AVX512AIi8Base, EVEX_4V, EVEX_V512;
10368 let Predicates = [HasAVX512, HasVLX] in {
10369 defm Z128 : avx512_valign<0x03, OpcodeStr, sched.XMM, _.info128>,
10370 AVX512AIi8Base, EVEX_4V, EVEX_V128;
10371 // We can't really override the 256-bit version so change it back to unset.
10372 let EVEX2VEXOverride = ? in
10373 defm Z256 : avx512_valign<0x03, OpcodeStr, sched.YMM, _.info256>,
10374 AVX512AIi8Base, EVEX_4V, EVEX_V256;
10378 defm VALIGND: avx512_valign_common<"valignd", SchedWriteShuffle,
10379 avx512vl_i32_info>, EVEX_CD8<32, CD8VF>;
10380 defm VALIGNQ: avx512_valign_common<"valignq", SchedWriteShuffle,
10381 avx512vl_i64_info>, EVEX_CD8<64, CD8VF>,
10384 defm VPALIGNR: avx512_common_3Op_rm_imm8<0x0F, X86PAlignr, "vpalignr",
10385 SchedWriteShuffle, avx512vl_i8_info,
10386 avx512vl_i8_info>, EVEX_CD8<8, CD8VF>;
10388 // Fragments to help convert valignq into masked valignd. Or valignq/valignd
10390 def ValignqImm32XForm : SDNodeXForm<imm, [{
10391 return getI8Imm(N->getZExtValue() * 2, SDLoc(N));
10393 def ValignqImm8XForm : SDNodeXForm<imm, [{
10394 return getI8Imm(N->getZExtValue() * 8, SDLoc(N));
10396 def ValigndImm8XForm : SDNodeXForm<imm, [{
10397 return getI8Imm(N->getZExtValue() * 4, SDLoc(N));
10400 multiclass avx512_vpalign_mask_lowering<string OpcodeStr, SDNode OpNode,
10401 X86VectorVTInfo From, X86VectorVTInfo To,
10402 SDNodeXForm ImmXForm> {
10403 def : Pat<(To.VT (vselect To.KRCWM:$mask,
10405 (From.VT (OpNode From.RC:$src1, From.RC:$src2,
10408 (!cast<Instruction>(OpcodeStr#"rrik") To.RC:$src0, To.KRCWM:$mask,
10409 To.RC:$src1, To.RC:$src2,
10410 (ImmXForm imm:$src3))>;
10412 def : Pat<(To.VT (vselect To.KRCWM:$mask,
10414 (From.VT (OpNode From.RC:$src1, From.RC:$src2,
10417 (!cast<Instruction>(OpcodeStr#"rrikz") To.KRCWM:$mask,
10418 To.RC:$src1, To.RC:$src2,
10419 (ImmXForm imm:$src3))>;
10421 def : Pat<(To.VT (vselect To.KRCWM:$mask,
10423 (From.VT (OpNode From.RC:$src1,
10424 (bitconvert (To.LdFrag addr:$src2)),
10427 (!cast<Instruction>(OpcodeStr#"rmik") To.RC:$src0, To.KRCWM:$mask,
10428 To.RC:$src1, addr:$src2,
10429 (ImmXForm imm:$src3))>;
10431 def : Pat<(To.VT (vselect To.KRCWM:$mask,
10433 (From.VT (OpNode From.RC:$src1,
10434 (bitconvert (To.LdFrag addr:$src2)),
10437 (!cast<Instruction>(OpcodeStr#"rmikz") To.KRCWM:$mask,
10438 To.RC:$src1, addr:$src2,
10439 (ImmXForm imm:$src3))>;
10442 multiclass avx512_vpalign_mask_lowering_mb<string OpcodeStr, SDNode OpNode,
10443 X86VectorVTInfo From,
10444 X86VectorVTInfo To,
10445 SDNodeXForm ImmXForm> :
10446 avx512_vpalign_mask_lowering<OpcodeStr, OpNode, From, To, ImmXForm> {
10447 def : Pat<(From.VT (OpNode From.RC:$src1,
10448 (bitconvert (To.VT (X86VBroadcast
10449 (To.ScalarLdFrag addr:$src2)))),
10451 (!cast<Instruction>(OpcodeStr#"rmbi") To.RC:$src1, addr:$src2,
10452 (ImmXForm imm:$src3))>;
10454 def : Pat<(To.VT (vselect To.KRCWM:$mask,
10456 (From.VT (OpNode From.RC:$src1,
10458 (To.VT (X86VBroadcast
10459 (To.ScalarLdFrag addr:$src2)))),
10462 (!cast<Instruction>(OpcodeStr#"rmbik") To.RC:$src0, To.KRCWM:$mask,
10463 To.RC:$src1, addr:$src2,
10464 (ImmXForm imm:$src3))>;
10466 def : Pat<(To.VT (vselect To.KRCWM:$mask,
10468 (From.VT (OpNode From.RC:$src1,
10470 (To.VT (X86VBroadcast
10471 (To.ScalarLdFrag addr:$src2)))),
10474 (!cast<Instruction>(OpcodeStr#"rmbikz") To.KRCWM:$mask,
10475 To.RC:$src1, addr:$src2,
10476 (ImmXForm imm:$src3))>;
10479 let Predicates = [HasAVX512] in {
10480 // For 512-bit we lower to the widest element type we can. So we only need
10481 // to handle converting valignq to valignd.
10482 defm : avx512_vpalign_mask_lowering_mb<"VALIGNDZ", X86VAlign, v8i64_info,
10483 v16i32_info, ValignqImm32XForm>;
10486 let Predicates = [HasVLX] in {
10487 // For 128-bit we lower to the widest element type we can. So we only need
10488 // to handle converting valignq to valignd.
10489 defm : avx512_vpalign_mask_lowering_mb<"VALIGNDZ128", X86VAlign, v2i64x_info,
10490 v4i32x_info, ValignqImm32XForm>;
10491 // For 256-bit we lower to the widest element type we can. So we only need
10492 // to handle converting valignq to valignd.
10493 defm : avx512_vpalign_mask_lowering_mb<"VALIGNDZ256", X86VAlign, v4i64x_info,
10494 v8i32x_info, ValignqImm32XForm>;
10497 let Predicates = [HasVLX, HasBWI] in {
10498 // We can turn 128 and 256 bit VALIGND/VALIGNQ into VPALIGNR.
10499 defm : avx512_vpalign_mask_lowering<"VPALIGNRZ128", X86VAlign, v2i64x_info,
10500 v16i8x_info, ValignqImm8XForm>;
10501 defm : avx512_vpalign_mask_lowering<"VPALIGNRZ128", X86VAlign, v4i32x_info,
10502 v16i8x_info, ValigndImm8XForm>;
10505 defm VDBPSADBW: avx512_common_3Op_rm_imm8<0x42, X86dbpsadbw, "vdbpsadbw",
10506 SchedWritePSADBW, avx512vl_i16_info, avx512vl_i8_info>,
10507 EVEX_CD8<8, CD8VF>, NotEVEX2VEXConvertible;
10509 multiclass avx512_unary_rm<bits<8> opc, string OpcodeStr, SDNode OpNode,
10510 X86FoldableSchedWrite sched, X86VectorVTInfo _> {
10511 let ExeDomain = _.ExeDomain in {
10512 defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
10513 (ins _.RC:$src1), OpcodeStr,
10515 (_.VT (OpNode _.RC:$src1))>, EVEX, AVX5128IBase,
10518 defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
10519 (ins _.MemOp:$src1), OpcodeStr,
10521 (_.VT (OpNode (bitconvert (_.LdFrag addr:$src1))))>,
10522 EVEX, AVX5128IBase, EVEX_CD8<_.EltSize, CD8VF>,
10523 Sched<[sched.Folded]>;
10527 multiclass avx512_unary_rmb<bits<8> opc, string OpcodeStr, SDNode OpNode,
10528 X86FoldableSchedWrite sched, X86VectorVTInfo _> :
10529 avx512_unary_rm<opc, OpcodeStr, OpNode, sched, _> {
10530 defm rmb : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
10531 (ins _.ScalarMemOp:$src1), OpcodeStr,
10532 "${src1}"##_.BroadcastStr,
10533 "${src1}"##_.BroadcastStr,
10534 (_.VT (OpNode (X86VBroadcast
10535 (_.ScalarLdFrag addr:$src1))))>,
10536 EVEX, AVX5128IBase, EVEX_B, EVEX_CD8<_.EltSize, CD8VF>,
10537 Sched<[sched.Folded]>;
10540 multiclass avx512_unary_rm_vl<bits<8> opc, string OpcodeStr, SDNode OpNode,
10541 X86SchedWriteWidths sched,
10542 AVX512VLVectorVTInfo VTInfo, Predicate prd> {
10543 let Predicates = [prd] in
10544 defm Z : avx512_unary_rm<opc, OpcodeStr, OpNode, sched.ZMM, VTInfo.info512>,
10547 let Predicates = [prd, HasVLX] in {
10548 defm Z256 : avx512_unary_rm<opc, OpcodeStr, OpNode, sched.YMM, VTInfo.info256>,
10550 defm Z128 : avx512_unary_rm<opc, OpcodeStr, OpNode, sched.XMM, VTInfo.info128>,
10555 multiclass avx512_unary_rmb_vl<bits<8> opc, string OpcodeStr, SDNode OpNode,
10556 X86SchedWriteWidths sched, AVX512VLVectorVTInfo VTInfo,
10558 let Predicates = [prd] in
10559 defm Z : avx512_unary_rmb<opc, OpcodeStr, OpNode, sched.ZMM, VTInfo.info512>,
10562 let Predicates = [prd, HasVLX] in {
10563 defm Z256 : avx512_unary_rmb<opc, OpcodeStr, OpNode, sched.YMM, VTInfo.info256>,
10565 defm Z128 : avx512_unary_rmb<opc, OpcodeStr, OpNode, sched.XMM, VTInfo.info128>,
10570 multiclass avx512_unary_rm_vl_dq<bits<8> opc_d, bits<8> opc_q, string OpcodeStr,
10571 SDNode OpNode, X86SchedWriteWidths sched,
10573 defm Q : avx512_unary_rmb_vl<opc_q, OpcodeStr#"q", OpNode, sched,
10574 avx512vl_i64_info, prd>, VEX_W;
10575 defm D : avx512_unary_rmb_vl<opc_d, OpcodeStr#"d", OpNode, sched,
10576 avx512vl_i32_info, prd>;
10579 multiclass avx512_unary_rm_vl_bw<bits<8> opc_b, bits<8> opc_w, string OpcodeStr,
10580 SDNode OpNode, X86SchedWriteWidths sched,
10582 defm W : avx512_unary_rm_vl<opc_w, OpcodeStr#"w", OpNode, sched,
10583 avx512vl_i16_info, prd>, VEX_WIG;
10584 defm B : avx512_unary_rm_vl<opc_b, OpcodeStr#"b", OpNode, sched,
10585 avx512vl_i8_info, prd>, VEX_WIG;
10588 multiclass avx512_unary_rm_vl_all<bits<8> opc_b, bits<8> opc_w,
10589 bits<8> opc_d, bits<8> opc_q,
10590 string OpcodeStr, SDNode OpNode,
10591 X86SchedWriteWidths sched> {
10592 defm NAME : avx512_unary_rm_vl_dq<opc_d, opc_q, OpcodeStr, OpNode, sched,
10594 avx512_unary_rm_vl_bw<opc_b, opc_w, OpcodeStr, OpNode, sched,
10598 defm VPABS : avx512_unary_rm_vl_all<0x1C, 0x1D, 0x1E, 0x1F, "vpabs", abs,
10601 // VPABS: Use 512bit version to implement 128/256 bit in case NoVLX.
10602 let Predicates = [HasAVX512, NoVLX] in {
10603 def : Pat<(v4i64 (abs VR256X:$src)),
10606 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR256X:$src, sub_ymm)),
10608 def : Pat<(v2i64 (abs VR128X:$src)),
10611 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm)),
10615 // Use 512bit version to implement 128/256 bit.
10616 multiclass avx512_unary_lowering<string InstrStr, SDNode OpNode,
10617 AVX512VLVectorVTInfo _, Predicate prd> {
10618 let Predicates = [prd, NoVLX] in {
10619 def : Pat<(_.info256.VT(OpNode _.info256.RC:$src1)),
10621 (!cast<Instruction>(InstrStr # "Zrr")
10622 (INSERT_SUBREG(_.info512.VT(IMPLICIT_DEF)),
10623 _.info256.RC:$src1,
10624 _.info256.SubRegIdx)),
10625 _.info256.SubRegIdx)>;
10627 def : Pat<(_.info128.VT(OpNode _.info128.RC:$src1)),
10629 (!cast<Instruction>(InstrStr # "Zrr")
10630 (INSERT_SUBREG(_.info512.VT(IMPLICIT_DEF)),
10631 _.info128.RC:$src1,
10632 _.info128.SubRegIdx)),
10633 _.info128.SubRegIdx)>;
10637 defm VPLZCNT : avx512_unary_rm_vl_dq<0x44, 0x44, "vplzcnt", ctlz,
10638 SchedWriteVecIMul, HasCDI>;
10640 // FIXME: Is there a better scheduler class for VPCONFLICT?
10641 defm VPCONFLICT : avx512_unary_rm_vl_dq<0xC4, 0xC4, "vpconflict", X86Conflict,
10642 SchedWriteVecALU, HasCDI>;
10644 // VPLZCNT: Use 512bit version to implement 128/256 bit in case NoVLX.
10645 defm : avx512_unary_lowering<"VPLZCNTQ", ctlz, avx512vl_i64_info, HasCDI>;
10646 defm : avx512_unary_lowering<"VPLZCNTD", ctlz, avx512vl_i32_info, HasCDI>;
10648 //===---------------------------------------------------------------------===//
10649 // Counts number of ones - VPOPCNTD and VPOPCNTQ
10650 //===---------------------------------------------------------------------===//
10652 // FIXME: Is there a better scheduler class for VPOPCNTD/VPOPCNTQ?
10653 defm VPOPCNT : avx512_unary_rm_vl_dq<0x55, 0x55, "vpopcnt", ctpop,
10654 SchedWriteVecALU, HasVPOPCNTDQ>;
10656 defm : avx512_unary_lowering<"VPOPCNTQ", ctpop, avx512vl_i64_info, HasVPOPCNTDQ>;
10657 defm : avx512_unary_lowering<"VPOPCNTD", ctpop, avx512vl_i32_info, HasVPOPCNTDQ>;
10659 //===---------------------------------------------------------------------===//
10660 // Replicate Single FP - MOVSHDUP and MOVSLDUP
10661 //===---------------------------------------------------------------------===//
10663 multiclass avx512_replicate<bits<8> opc, string OpcodeStr, SDNode OpNode,
10664 X86SchedWriteWidths sched> {
10665 defm NAME: avx512_unary_rm_vl<opc, OpcodeStr, OpNode, sched,
10666 avx512vl_f32_info, HasAVX512>, XS;
10669 defm VMOVSHDUP : avx512_replicate<0x16, "vmovshdup", X86Movshdup,
10670 SchedWriteFShuffle>;
10671 defm VMOVSLDUP : avx512_replicate<0x12, "vmovsldup", X86Movsldup,
10672 SchedWriteFShuffle>;
10674 //===----------------------------------------------------------------------===//
10675 // AVX-512 - MOVDDUP
10676 //===----------------------------------------------------------------------===//
10678 multiclass avx512_movddup_128<bits<8> opc, string OpcodeStr, SDNode OpNode,
10679 X86FoldableSchedWrite sched, X86VectorVTInfo _> {
10680 let ExeDomain = _.ExeDomain in {
10681 defm rr : AVX512_maskable<opc, MRMSrcReg, _, (outs _.RC:$dst),
10682 (ins _.RC:$src), OpcodeStr, "$src", "$src",
10683 (_.VT (OpNode (_.VT _.RC:$src)))>, EVEX,
10685 defm rm : AVX512_maskable<opc, MRMSrcMem, _, (outs _.RC:$dst),
10686 (ins _.ScalarMemOp:$src), OpcodeStr, "$src", "$src",
10687 (_.VT (OpNode (_.VT (scalar_to_vector
10688 (_.ScalarLdFrag addr:$src)))))>,
10689 EVEX, EVEX_CD8<_.EltSize, CD8VH>,
10690 Sched<[sched.Folded]>;
10694 multiclass avx512_movddup_common<bits<8> opc, string OpcodeStr, SDNode OpNode,
10695 X86SchedWriteWidths sched, AVX512VLVectorVTInfo VTInfo> {
10696 defm Z : avx512_unary_rm<opc, OpcodeStr, X86Movddup, sched.ZMM,
10697 VTInfo.info512>, EVEX_V512;
10699 let Predicates = [HasAVX512, HasVLX] in {
10700 defm Z256 : avx512_unary_rm<opc, OpcodeStr, X86Movddup, sched.YMM,
10701 VTInfo.info256>, EVEX_V256;
10702 defm Z128 : avx512_movddup_128<opc, OpcodeStr, X86VBroadcast, sched.XMM,
10703 VTInfo.info128>, EVEX_V128;
10707 multiclass avx512_movddup<bits<8> opc, string OpcodeStr, SDNode OpNode,
10708 X86SchedWriteWidths sched> {
10709 defm NAME: avx512_movddup_common<opc, OpcodeStr, OpNode, sched,
10710 avx512vl_f64_info>, XD, VEX_W;
10713 defm VMOVDDUP : avx512_movddup<0x12, "vmovddup", X86Movddup, SchedWriteFShuffle>;
10715 let Predicates = [HasVLX] in {
10716 def : Pat<(v2f64 (X86VBroadcast (loadf64 addr:$src))),
10717 (VMOVDDUPZ128rm addr:$src)>;
10718 def : Pat<(v2f64 (X86VBroadcast f64:$src)),
10719 (VMOVDDUPZ128rr (COPY_TO_REGCLASS FR64X:$src, VR128X))>;
10720 def : Pat<(v2f64 (X86VBroadcast (loadv2f64 addr:$src))),
10721 (VMOVDDUPZ128rm addr:$src)>;
10723 def : Pat<(vselect (v2i1 VK2WM:$mask), (v2f64 (X86VBroadcast f64:$src)),
10724 (v2f64 VR128X:$src0)),
10725 (VMOVDDUPZ128rrk VR128X:$src0, VK2WM:$mask,
10726 (COPY_TO_REGCLASS FR64X:$src, VR128X))>;
10727 def : Pat<(vselect (v2i1 VK2WM:$mask), (v2f64 (X86VBroadcast f64:$src)),
10728 (bitconvert (v4i32 immAllZerosV))),
10729 (VMOVDDUPZ128rrkz VK2WM:$mask, (COPY_TO_REGCLASS FR64X:$src, VR128X))>;
10731 def : Pat<(vselect (v2i1 VK2WM:$mask), (v2f64 (X86VBroadcast (loadf64 addr:$src))),
10732 (v2f64 VR128X:$src0)),
10733 (VMOVDDUPZ128rmk VR128X:$src0, VK2WM:$mask, addr:$src)>;
10734 def : Pat<(vselect (v2i1 VK2WM:$mask), (v2f64 (X86VBroadcast (loadf64 addr:$src))),
10735 (bitconvert (v4i32 immAllZerosV))),
10736 (VMOVDDUPZ128rmkz VK2WM:$mask, addr:$src)>;
10738 def : Pat<(vselect (v2i1 VK2WM:$mask), (v2f64 (X86VBroadcast (loadv2f64 addr:$src))),
10739 (v2f64 VR128X:$src0)),
10740 (VMOVDDUPZ128rmk VR128X:$src0, VK2WM:$mask, addr:$src)>;
10741 def : Pat<(vselect (v2i1 VK2WM:$mask), (v2f64 (X86VBroadcast (loadv2f64 addr:$src))),
10742 (bitconvert (v4i32 immAllZerosV))),
10743 (VMOVDDUPZ128rmkz VK2WM:$mask, addr:$src)>;
10746 //===----------------------------------------------------------------------===//
10747 // AVX-512 - Unpack Instructions
10748 //===----------------------------------------------------------------------===//
10750 defm VUNPCKH : avx512_fp_binop_p<0x15, "vunpckh", X86Unpckh, HasAVX512,
10751 SchedWriteFShuffleSizes>;
10752 defm VUNPCKL : avx512_fp_binop_p<0x14, "vunpckl", X86Unpckl, HasAVX512,
10753 SchedWriteFShuffleSizes>;
10755 defm VPUNPCKLBW : avx512_binop_rm_vl_b<0x60, "vpunpcklbw", X86Unpckl,
10756 SchedWriteShuffle, HasBWI>;
10757 defm VPUNPCKHBW : avx512_binop_rm_vl_b<0x68, "vpunpckhbw", X86Unpckh,
10758 SchedWriteShuffle, HasBWI>;
10759 defm VPUNPCKLWD : avx512_binop_rm_vl_w<0x61, "vpunpcklwd", X86Unpckl,
10760 SchedWriteShuffle, HasBWI>;
10761 defm VPUNPCKHWD : avx512_binop_rm_vl_w<0x69, "vpunpckhwd", X86Unpckh,
10762 SchedWriteShuffle, HasBWI>;
10764 defm VPUNPCKLDQ : avx512_binop_rm_vl_d<0x62, "vpunpckldq", X86Unpckl,
10765 SchedWriteShuffle, HasAVX512>;
10766 defm VPUNPCKHDQ : avx512_binop_rm_vl_d<0x6A, "vpunpckhdq", X86Unpckh,
10767 SchedWriteShuffle, HasAVX512>;
10768 defm VPUNPCKLQDQ : avx512_binop_rm_vl_q<0x6C, "vpunpcklqdq", X86Unpckl,
10769 SchedWriteShuffle, HasAVX512>;
10770 defm VPUNPCKHQDQ : avx512_binop_rm_vl_q<0x6D, "vpunpckhqdq", X86Unpckh,
10771 SchedWriteShuffle, HasAVX512>;
10773 //===----------------------------------------------------------------------===//
10774 // AVX-512 - Extract & Insert Integer Instructions
10775 //===----------------------------------------------------------------------===//
10777 multiclass avx512_extract_elt_bw_m<bits<8> opc, string OpcodeStr, SDNode OpNode,
10778 X86VectorVTInfo _> {
10779 def mr : AVX512Ii8<opc, MRMDestMem, (outs),
10780 (ins _.ScalarMemOp:$dst, _.RC:$src1, u8imm:$src2),
10781 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
10782 [(store (_.EltVT (trunc (OpNode (_.VT _.RC:$src1), imm:$src2))),
10784 EVEX, EVEX_CD8<_.EltSize, CD8VT1>, Sched<[WriteVecExtractSt]>;
10787 multiclass avx512_extract_elt_b<string OpcodeStr, X86VectorVTInfo _> {
10788 let Predicates = [HasBWI] in {
10789 def rr : AVX512Ii8<0x14, MRMDestReg, (outs GR32orGR64:$dst),
10790 (ins _.RC:$src1, u8imm:$src2),
10791 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
10792 [(set GR32orGR64:$dst,
10793 (X86pextrb (_.VT _.RC:$src1), imm:$src2))]>,
10794 EVEX, TAPD, Sched<[WriteVecExtract]>;
10796 defm NAME : avx512_extract_elt_bw_m<0x14, OpcodeStr, X86pextrb, _>, TAPD;
10800 multiclass avx512_extract_elt_w<string OpcodeStr, X86VectorVTInfo _> {
10801 let Predicates = [HasBWI] in {
10802 def rr : AVX512Ii8<0xC5, MRMSrcReg, (outs GR32orGR64:$dst),
10803 (ins _.RC:$src1, u8imm:$src2),
10804 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
10805 [(set GR32orGR64:$dst,
10806 (X86pextrw (_.VT _.RC:$src1), imm:$src2))]>,
10807 EVEX, PD, Sched<[WriteVecExtract]>;
10809 let hasSideEffects = 0, isCodeGenOnly = 1, ForceDisassemble = 1 in
10810 def rr_REV : AVX512Ii8<0x15, MRMDestReg, (outs GR32orGR64:$dst),
10811 (ins _.RC:$src1, u8imm:$src2),
10812 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}", []>,
10813 EVEX, TAPD, FoldGenData<NAME#rr>,
10814 Sched<[WriteVecExtract]>;
10816 defm NAME : avx512_extract_elt_bw_m<0x15, OpcodeStr, X86pextrw, _>, TAPD;
10820 multiclass avx512_extract_elt_dq<string OpcodeStr, X86VectorVTInfo _,
10821 RegisterClass GRC> {
10822 let Predicates = [HasDQI] in {
10823 def rr : AVX512Ii8<0x16, MRMDestReg, (outs GRC:$dst),
10824 (ins _.RC:$src1, u8imm:$src2),
10825 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
10827 (extractelt (_.VT _.RC:$src1), imm:$src2))]>,
10828 EVEX, TAPD, Sched<[WriteVecExtract]>;
10830 def mr : AVX512Ii8<0x16, MRMDestMem, (outs),
10831 (ins _.ScalarMemOp:$dst, _.RC:$src1, u8imm:$src2),
10832 OpcodeStr#"\t{$src2, $src1, $dst|$dst, $src1, $src2}",
10833 [(store (extractelt (_.VT _.RC:$src1),
10834 imm:$src2),addr:$dst)]>,
10835 EVEX, EVEX_CD8<_.EltSize, CD8VT1>, TAPD,
10836 Sched<[WriteVecExtractSt]>;
10840 defm VPEXTRBZ : avx512_extract_elt_b<"vpextrb", v16i8x_info>, VEX_WIG;
10841 defm VPEXTRWZ : avx512_extract_elt_w<"vpextrw", v8i16x_info>, VEX_WIG;
10842 defm VPEXTRDZ : avx512_extract_elt_dq<"vpextrd", v4i32x_info, GR32>;
10843 defm VPEXTRQZ : avx512_extract_elt_dq<"vpextrq", v2i64x_info, GR64>, VEX_W;
10845 multiclass avx512_insert_elt_m<bits<8> opc, string OpcodeStr, SDNode OpNode,
10846 X86VectorVTInfo _, PatFrag LdFrag> {
10847 def rm : AVX512Ii8<opc, MRMSrcMem, (outs _.RC:$dst),
10848 (ins _.RC:$src1, _.ScalarMemOp:$src2, u8imm:$src3),
10849 OpcodeStr#"\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
10851 (_.VT (OpNode _.RC:$src1, (LdFrag addr:$src2), imm:$src3)))]>,
10852 EVEX_4V, EVEX_CD8<_.EltSize, CD8VT1>, Sched<[WriteVecInsertLd, ReadAfterLd]>;
10855 multiclass avx512_insert_elt_bw<bits<8> opc, string OpcodeStr, SDNode OpNode,
10856 X86VectorVTInfo _, PatFrag LdFrag> {
10857 let Predicates = [HasBWI] in {
10858 def rr : AVX512Ii8<opc, MRMSrcReg, (outs _.RC:$dst),
10859 (ins _.RC:$src1, GR32orGR64:$src2, u8imm:$src3),
10860 OpcodeStr#"\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
10862 (OpNode _.RC:$src1, GR32orGR64:$src2, imm:$src3))]>, EVEX_4V,
10863 Sched<[WriteVecInsert]>;
10865 defm NAME : avx512_insert_elt_m<opc, OpcodeStr, OpNode, _, LdFrag>;
10869 multiclass avx512_insert_elt_dq<bits<8> opc, string OpcodeStr,
10870 X86VectorVTInfo _, RegisterClass GRC> {
10871 let Predicates = [HasDQI] in {
10872 def rr : AVX512Ii8<opc, MRMSrcReg, (outs _.RC:$dst),
10873 (ins _.RC:$src1, GRC:$src2, u8imm:$src3),
10874 OpcodeStr#"\t{$src3, $src2, $src1, $dst|$dst, $src1, $src2, $src3}",
10876 (_.VT (insertelt _.RC:$src1, GRC:$src2, imm:$src3)))]>,
10877 EVEX_4V, TAPD, Sched<[WriteVecInsert]>;
10879 defm NAME : avx512_insert_elt_m<opc, OpcodeStr, insertelt, _,
10880 _.ScalarLdFrag>, TAPD;
10884 defm VPINSRBZ : avx512_insert_elt_bw<0x20, "vpinsrb", X86pinsrb, v16i8x_info,
10885 extloadi8>, TAPD, VEX_WIG;
10886 defm VPINSRWZ : avx512_insert_elt_bw<0xC4, "vpinsrw", X86pinsrw, v8i16x_info,
10887 extloadi16>, PD, VEX_WIG;
10888 defm VPINSRDZ : avx512_insert_elt_dq<0x22, "vpinsrd", v4i32x_info, GR32>;
10889 defm VPINSRQZ : avx512_insert_elt_dq<0x22, "vpinsrq", v2i64x_info, GR64>, VEX_W;
10891 //===----------------------------------------------------------------------===//
10892 // VSHUFPS - VSHUFPD Operations
10893 //===----------------------------------------------------------------------===//
10895 multiclass avx512_shufp<string OpcodeStr, AVX512VLVectorVTInfo VTInfo_I,
10896 AVX512VLVectorVTInfo VTInfo_FP>{
10897 defm NAME: avx512_common_3Op_imm8<OpcodeStr, VTInfo_FP, 0xC6, X86Shufp,
10898 SchedWriteFShuffle>,
10899 EVEX_CD8<VTInfo_FP.info512.EltSize, CD8VF>,
10900 AVX512AIi8Base, EVEX_4V;
10903 defm VSHUFPS: avx512_shufp<"vshufps", avx512vl_i32_info, avx512vl_f32_info>, PS;
10904 defm VSHUFPD: avx512_shufp<"vshufpd", avx512vl_i64_info, avx512vl_f64_info>, PD, VEX_W;
10906 //===----------------------------------------------------------------------===//
10907 // AVX-512 - Byte shift Left/Right
10908 //===----------------------------------------------------------------------===//
10910 // FIXME: The SSE/AVX names are PSLLDQri etc. - should we add the i here as well?
10911 multiclass avx512_shift_packed<bits<8> opc, SDNode OpNode, Format MRMr,
10912 Format MRMm, string OpcodeStr,
10913 X86FoldableSchedWrite sched, X86VectorVTInfo _>{
10914 def rr : AVX512<opc, MRMr,
10915 (outs _.RC:$dst), (ins _.RC:$src1, u8imm:$src2),
10916 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
10917 [(set _.RC:$dst,(_.VT (OpNode _.RC:$src1, (i8 imm:$src2))))]>,
10919 def rm : AVX512<opc, MRMm,
10920 (outs _.RC:$dst), (ins _.MemOp:$src1, u8imm:$src2),
10921 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
10922 [(set _.RC:$dst,(_.VT (OpNode
10923 (_.VT (bitconvert (_.LdFrag addr:$src1))),
10924 (i8 imm:$src2))))]>,
10925 Sched<[sched.Folded, ReadAfterLd]>;
10928 multiclass avx512_shift_packed_all<bits<8> opc, SDNode OpNode, Format MRMr,
10929 Format MRMm, string OpcodeStr,
10930 X86SchedWriteWidths sched, Predicate prd>{
10931 let Predicates = [prd] in
10932 defm Z : avx512_shift_packed<opc, OpNode, MRMr, MRMm, OpcodeStr,
10933 sched.ZMM, v64i8_info>, EVEX_V512;
10934 let Predicates = [prd, HasVLX] in {
10935 defm Z256 : avx512_shift_packed<opc, OpNode, MRMr, MRMm, OpcodeStr,
10936 sched.YMM, v32i8x_info>, EVEX_V256;
10937 defm Z128 : avx512_shift_packed<opc, OpNode, MRMr, MRMm, OpcodeStr,
10938 sched.XMM, v16i8x_info>, EVEX_V128;
10941 defm VPSLLDQ : avx512_shift_packed_all<0x73, X86vshldq, MRM7r, MRM7m, "vpslldq",
10942 SchedWriteShuffle, HasBWI>,
10943 AVX512PDIi8Base, EVEX_4V, VEX_WIG;
10944 defm VPSRLDQ : avx512_shift_packed_all<0x73, X86vshrdq, MRM3r, MRM3m, "vpsrldq",
10945 SchedWriteShuffle, HasBWI>,
10946 AVX512PDIi8Base, EVEX_4V, VEX_WIG;
10948 multiclass avx512_psadbw_packed<bits<8> opc, SDNode OpNode,
10949 string OpcodeStr, X86FoldableSchedWrite sched,
10950 X86VectorVTInfo _dst, X86VectorVTInfo _src> {
10951 def rr : AVX512BI<opc, MRMSrcReg,
10952 (outs _dst.RC:$dst), (ins _src.RC:$src1, _src.RC:$src2),
10953 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
10954 [(set _dst.RC:$dst,(_dst.VT
10955 (OpNode (_src.VT _src.RC:$src1),
10956 (_src.VT _src.RC:$src2))))]>,
10958 def rm : AVX512BI<opc, MRMSrcMem,
10959 (outs _dst.RC:$dst), (ins _src.RC:$src1, _src.MemOp:$src2),
10960 !strconcat(OpcodeStr, "\t{$src2, $src1, $dst|$dst, $src1, $src2}"),
10961 [(set _dst.RC:$dst,(_dst.VT
10962 (OpNode (_src.VT _src.RC:$src1),
10963 (_src.VT (bitconvert
10964 (_src.LdFrag addr:$src2))))))]>,
10965 Sched<[sched.Folded, ReadAfterLd]>;
10968 multiclass avx512_psadbw_packed_all<bits<8> opc, SDNode OpNode,
10969 string OpcodeStr, X86SchedWriteWidths sched,
10971 let Predicates = [prd] in
10972 defm Z : avx512_psadbw_packed<opc, OpNode, OpcodeStr, sched.ZMM,
10973 v8i64_info, v64i8_info>, EVEX_V512;
10974 let Predicates = [prd, HasVLX] in {
10975 defm Z256 : avx512_psadbw_packed<opc, OpNode, OpcodeStr, sched.YMM,
10976 v4i64x_info, v32i8x_info>, EVEX_V256;
10977 defm Z128 : avx512_psadbw_packed<opc, OpNode, OpcodeStr, sched.XMM,
10978 v2i64x_info, v16i8x_info>, EVEX_V128;
10982 defm VPSADBW : avx512_psadbw_packed_all<0xf6, X86psadbw, "vpsadbw",
10983 SchedWritePSADBW, HasBWI>, EVEX_4V, VEX_WIG;
10985 // Transforms to swizzle an immediate to enable better matching when
10986 // memory operand isn't in the right place.
10987 def VPTERNLOG321_imm8 : SDNodeXForm<imm, [{
10988 // Convert a VPTERNLOG immediate by swapping operand 0 and operand 2.
10989 uint8_t Imm = N->getZExtValue();
10990 // Swap bits 1/4 and 3/6.
10991 uint8_t NewImm = Imm & 0xa5;
10992 if (Imm & 0x02) NewImm |= 0x10;
10993 if (Imm & 0x10) NewImm |= 0x02;
10994 if (Imm & 0x08) NewImm |= 0x40;
10995 if (Imm & 0x40) NewImm |= 0x08;
10996 return getI8Imm(NewImm, SDLoc(N));
10998 def VPTERNLOG213_imm8 : SDNodeXForm<imm, [{
10999 // Convert a VPTERNLOG immediate by swapping operand 1 and operand 2.
11000 uint8_t Imm = N->getZExtValue();
11001 // Swap bits 2/4 and 3/5.
11002 uint8_t NewImm = Imm & 0xc3;
11003 if (Imm & 0x04) NewImm |= 0x10;
11004 if (Imm & 0x10) NewImm |= 0x04;
11005 if (Imm & 0x08) NewImm |= 0x20;
11006 if (Imm & 0x20) NewImm |= 0x08;
11007 return getI8Imm(NewImm, SDLoc(N));
11009 def VPTERNLOG132_imm8 : SDNodeXForm<imm, [{
11010 // Convert a VPTERNLOG immediate by swapping operand 1 and operand 2.
11011 uint8_t Imm = N->getZExtValue();
11012 // Swap bits 1/2 and 5/6.
11013 uint8_t NewImm = Imm & 0x99;
11014 if (Imm & 0x02) NewImm |= 0x04;
11015 if (Imm & 0x04) NewImm |= 0x02;
11016 if (Imm & 0x20) NewImm |= 0x40;
11017 if (Imm & 0x40) NewImm |= 0x20;
11018 return getI8Imm(NewImm, SDLoc(N));
11020 def VPTERNLOG231_imm8 : SDNodeXForm<imm, [{
11021 // Convert a VPTERNLOG immediate by moving operand 1 to the end.
11022 uint8_t Imm = N->getZExtValue();
11023 // Move bits 1->2, 2->4, 3->6, 4->1, 5->3, 6->5
11024 uint8_t NewImm = Imm & 0x81;
11025 if (Imm & 0x02) NewImm |= 0x04;
11026 if (Imm & 0x04) NewImm |= 0x10;
11027 if (Imm & 0x08) NewImm |= 0x40;
11028 if (Imm & 0x10) NewImm |= 0x02;
11029 if (Imm & 0x20) NewImm |= 0x08;
11030 if (Imm & 0x40) NewImm |= 0x20;
11031 return getI8Imm(NewImm, SDLoc(N));
11033 def VPTERNLOG312_imm8 : SDNodeXForm<imm, [{
11034 // Convert a VPTERNLOG immediate by moving operand 2 to the beginning.
11035 uint8_t Imm = N->getZExtValue();
11036 // Move bits 1->4, 2->1, 3->5, 4->2, 5->6, 6->3
11037 uint8_t NewImm = Imm & 0x81;
11038 if (Imm & 0x02) NewImm |= 0x10;
11039 if (Imm & 0x04) NewImm |= 0x02;
11040 if (Imm & 0x08) NewImm |= 0x20;
11041 if (Imm & 0x10) NewImm |= 0x04;
11042 if (Imm & 0x20) NewImm |= 0x40;
11043 if (Imm & 0x40) NewImm |= 0x08;
11044 return getI8Imm(NewImm, SDLoc(N));
11047 multiclass avx512_ternlog<bits<8> opc, string OpcodeStr, SDNode OpNode,
11048 X86FoldableSchedWrite sched, X86VectorVTInfo _,
11050 let Constraints = "$src1 = $dst", ExeDomain = _.ExeDomain in {
11051 defm rri : AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
11052 (ins _.RC:$src2, _.RC:$src3, u8imm:$src4),
11053 OpcodeStr, "$src4, $src3, $src2", "$src2, $src3, $src4",
11054 (OpNode (_.VT _.RC:$src1),
11057 (i8 imm:$src4)), 1, 1>,
11058 AVX512AIi8Base, EVEX_4V, Sched<[sched]>;
11059 defm rmi : AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
11060 (ins _.RC:$src2, _.MemOp:$src3, u8imm:$src4),
11061 OpcodeStr, "$src4, $src3, $src2", "$src2, $src3, $src4",
11062 (OpNode (_.VT _.RC:$src1),
11064 (_.VT (bitconvert (_.LdFrag addr:$src3))),
11065 (i8 imm:$src4)), 1, 0>,
11066 AVX512AIi8Base, EVEX_4V, EVEX_CD8<_.EltSize, CD8VF>,
11067 Sched<[sched.Folded, ReadAfterLd]>;
11068 defm rmbi : AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
11069 (ins _.RC:$src2, _.ScalarMemOp:$src3, u8imm:$src4),
11070 OpcodeStr, "$src4, ${src3}"##_.BroadcastStr##", $src2",
11071 "$src2, ${src3}"##_.BroadcastStr##", $src4",
11072 (OpNode (_.VT _.RC:$src1),
11074 (_.VT (X86VBroadcast(_.ScalarLdFrag addr:$src3))),
11075 (i8 imm:$src4)), 1, 0>, EVEX_B,
11076 AVX512AIi8Base, EVEX_4V, EVEX_CD8<_.EltSize, CD8VF>,
11077 Sched<[sched.Folded, ReadAfterLd]>;
11078 }// Constraints = "$src1 = $dst"
11080 // Additional patterns for matching passthru operand in other positions.
11081 def : Pat<(_.VT (vselect _.KRCWM:$mask,
11082 (OpNode _.RC:$src3, _.RC:$src2, _.RC:$src1, (i8 imm:$src4)),
11084 (!cast<Instruction>(Name#_.ZSuffix#rrik) _.RC:$src1, _.KRCWM:$mask,
11085 _.RC:$src2, _.RC:$src3, (VPTERNLOG321_imm8 imm:$src4))>;
11086 def : Pat<(_.VT (vselect _.KRCWM:$mask,
11087 (OpNode _.RC:$src2, _.RC:$src1, _.RC:$src3, (i8 imm:$src4)),
11089 (!cast<Instruction>(Name#_.ZSuffix#rrik) _.RC:$src1, _.KRCWM:$mask,
11090 _.RC:$src2, _.RC:$src3, (VPTERNLOG213_imm8 imm:$src4))>;
11092 // Additional patterns for matching loads in other positions.
11093 def : Pat<(_.VT (OpNode (bitconvert (_.LdFrag addr:$src3)),
11094 _.RC:$src2, _.RC:$src1, (i8 imm:$src4))),
11095 (!cast<Instruction>(Name#_.ZSuffix#rmi) _.RC:$src1, _.RC:$src2,
11096 addr:$src3, (VPTERNLOG321_imm8 imm:$src4))>;
11097 def : Pat<(_.VT (OpNode _.RC:$src1,
11098 (bitconvert (_.LdFrag addr:$src3)),
11099 _.RC:$src2, (i8 imm:$src4))),
11100 (!cast<Instruction>(Name#_.ZSuffix#rmi) _.RC:$src1, _.RC:$src2,
11101 addr:$src3, (VPTERNLOG132_imm8 imm:$src4))>;
11103 // Additional patterns for matching zero masking with loads in other
11105 def : Pat<(_.VT (vselect _.KRCWM:$mask,
11106 (OpNode (bitconvert (_.LdFrag addr:$src3)),
11107 _.RC:$src2, _.RC:$src1, (i8 imm:$src4)),
11109 (!cast<Instruction>(Name#_.ZSuffix#rmikz) _.RC:$src1, _.KRCWM:$mask,
11110 _.RC:$src2, addr:$src3, (VPTERNLOG321_imm8 imm:$src4))>;
11111 def : Pat<(_.VT (vselect _.KRCWM:$mask,
11112 (OpNode _.RC:$src1, (bitconvert (_.LdFrag addr:$src3)),
11113 _.RC:$src2, (i8 imm:$src4)),
11115 (!cast<Instruction>(Name#_.ZSuffix#rmikz) _.RC:$src1, _.KRCWM:$mask,
11116 _.RC:$src2, addr:$src3, (VPTERNLOG132_imm8 imm:$src4))>;
11118 // Additional patterns for matching masked loads with different
11120 def : Pat<(_.VT (vselect _.KRCWM:$mask,
11121 (OpNode _.RC:$src1, (bitconvert (_.LdFrag addr:$src3)),
11122 _.RC:$src2, (i8 imm:$src4)),
11124 (!cast<Instruction>(Name#_.ZSuffix#rmik) _.RC:$src1, _.KRCWM:$mask,
11125 _.RC:$src2, addr:$src3, (VPTERNLOG132_imm8 imm:$src4))>;
11126 def : Pat<(_.VT (vselect _.KRCWM:$mask,
11127 (OpNode (bitconvert (_.LdFrag addr:$src3)),
11128 _.RC:$src2, _.RC:$src1, (i8 imm:$src4)),
11130 (!cast<Instruction>(Name#_.ZSuffix#rmik) _.RC:$src1, _.KRCWM:$mask,
11131 _.RC:$src2, addr:$src3, (VPTERNLOG321_imm8 imm:$src4))>;
11132 def : Pat<(_.VT (vselect _.KRCWM:$mask,
11133 (OpNode _.RC:$src2, _.RC:$src1,
11134 (bitconvert (_.LdFrag addr:$src3)), (i8 imm:$src4)),
11136 (!cast<Instruction>(Name#_.ZSuffix#rmik) _.RC:$src1, _.KRCWM:$mask,
11137 _.RC:$src2, addr:$src3, (VPTERNLOG213_imm8 imm:$src4))>;
11138 def : Pat<(_.VT (vselect _.KRCWM:$mask,
11139 (OpNode _.RC:$src2, (bitconvert (_.LdFrag addr:$src3)),
11140 _.RC:$src1, (i8 imm:$src4)),
11142 (!cast<Instruction>(Name#_.ZSuffix#rmik) _.RC:$src1, _.KRCWM:$mask,
11143 _.RC:$src2, addr:$src3, (VPTERNLOG231_imm8 imm:$src4))>;
11144 def : Pat<(_.VT (vselect _.KRCWM:$mask,
11145 (OpNode (bitconvert (_.LdFrag addr:$src3)),
11146 _.RC:$src1, _.RC:$src2, (i8 imm:$src4)),
11148 (!cast<Instruction>(Name#_.ZSuffix#rmik) _.RC:$src1, _.KRCWM:$mask,
11149 _.RC:$src2, addr:$src3, (VPTERNLOG312_imm8 imm:$src4))>;
11151 // Additional patterns for matching broadcasts in other positions.
11152 def : Pat<(_.VT (OpNode (X86VBroadcast (_.ScalarLdFrag addr:$src3)),
11153 _.RC:$src2, _.RC:$src1, (i8 imm:$src4))),
11154 (!cast<Instruction>(Name#_.ZSuffix#rmbi) _.RC:$src1, _.RC:$src2,
11155 addr:$src3, (VPTERNLOG321_imm8 imm:$src4))>;
11156 def : Pat<(_.VT (OpNode _.RC:$src1,
11157 (X86VBroadcast (_.ScalarLdFrag addr:$src3)),
11158 _.RC:$src2, (i8 imm:$src4))),
11159 (!cast<Instruction>(Name#_.ZSuffix#rmbi) _.RC:$src1, _.RC:$src2,
11160 addr:$src3, (VPTERNLOG132_imm8 imm:$src4))>;
11162 // Additional patterns for matching zero masking with broadcasts in other
11164 def : Pat<(_.VT (vselect _.KRCWM:$mask,
11165 (OpNode (X86VBroadcast (_.ScalarLdFrag addr:$src3)),
11166 _.RC:$src2, _.RC:$src1, (i8 imm:$src4)),
11168 (!cast<Instruction>(Name#_.ZSuffix#rmbikz) _.RC:$src1,
11169 _.KRCWM:$mask, _.RC:$src2, addr:$src3,
11170 (VPTERNLOG321_imm8 imm:$src4))>;
11171 def : Pat<(_.VT (vselect _.KRCWM:$mask,
11172 (OpNode _.RC:$src1,
11173 (X86VBroadcast (_.ScalarLdFrag addr:$src3)),
11174 _.RC:$src2, (i8 imm:$src4)),
11176 (!cast<Instruction>(Name#_.ZSuffix#rmbikz) _.RC:$src1,
11177 _.KRCWM:$mask, _.RC:$src2, addr:$src3,
11178 (VPTERNLOG132_imm8 imm:$src4))>;
11180 // Additional patterns for matching masked broadcasts with different
11182 def : Pat<(_.VT (vselect _.KRCWM:$mask,
11183 (OpNode _.RC:$src1,
11184 (X86VBroadcast (_.ScalarLdFrag addr:$src3)),
11185 _.RC:$src2, (i8 imm:$src4)),
11187 (!cast<Instruction>(Name#_.ZSuffix#rmbik) _.RC:$src1, _.KRCWM:$mask,
11188 _.RC:$src2, addr:$src3, (VPTERNLOG132_imm8 imm:$src4))>;
11189 def : Pat<(_.VT (vselect _.KRCWM:$mask,
11190 (OpNode (X86VBroadcast (_.ScalarLdFrag addr:$src3)),
11191 _.RC:$src2, _.RC:$src1, (i8 imm:$src4)),
11193 (!cast<Instruction>(Name#_.ZSuffix#rmbik) _.RC:$src1, _.KRCWM:$mask,
11194 _.RC:$src2, addr:$src3, (VPTERNLOG321_imm8 imm:$src4))>;
11195 def : Pat<(_.VT (vselect _.KRCWM:$mask,
11196 (OpNode _.RC:$src2, _.RC:$src1,
11197 (X86VBroadcast (_.ScalarLdFrag addr:$src3)),
11198 (i8 imm:$src4)), _.RC:$src1)),
11199 (!cast<Instruction>(Name#_.ZSuffix#rmbik) _.RC:$src1, _.KRCWM:$mask,
11200 _.RC:$src2, addr:$src3, (VPTERNLOG213_imm8 imm:$src4))>;
11201 def : Pat<(_.VT (vselect _.KRCWM:$mask,
11202 (OpNode _.RC:$src2,
11203 (X86VBroadcast (_.ScalarLdFrag addr:$src3)),
11204 _.RC:$src1, (i8 imm:$src4)),
11206 (!cast<Instruction>(Name#_.ZSuffix#rmbik) _.RC:$src1, _.KRCWM:$mask,
11207 _.RC:$src2, addr:$src3, (VPTERNLOG231_imm8 imm:$src4))>;
11208 def : Pat<(_.VT (vselect _.KRCWM:$mask,
11209 (OpNode (X86VBroadcast (_.ScalarLdFrag addr:$src3)),
11210 _.RC:$src1, _.RC:$src2, (i8 imm:$src4)),
11212 (!cast<Instruction>(Name#_.ZSuffix#rmbik) _.RC:$src1, _.KRCWM:$mask,
11213 _.RC:$src2, addr:$src3, (VPTERNLOG312_imm8 imm:$src4))>;
11216 multiclass avx512_common_ternlog<string OpcodeStr, X86SchedWriteWidths sched,
11217 AVX512VLVectorVTInfo _> {
11218 let Predicates = [HasAVX512] in
11219 defm Z : avx512_ternlog<0x25, OpcodeStr, X86vpternlog, sched.ZMM,
11220 _.info512, NAME>, EVEX_V512;
11221 let Predicates = [HasAVX512, HasVLX] in {
11222 defm Z128 : avx512_ternlog<0x25, OpcodeStr, X86vpternlog, sched.XMM,
11223 _.info128, NAME>, EVEX_V128;
11224 defm Z256 : avx512_ternlog<0x25, OpcodeStr, X86vpternlog, sched.YMM,
11225 _.info256, NAME>, EVEX_V256;
11229 defm VPTERNLOGD : avx512_common_ternlog<"vpternlogd", SchedWriteVecALU,
11230 avx512vl_i32_info>;
11231 defm VPTERNLOGQ : avx512_common_ternlog<"vpternlogq", SchedWriteVecALU,
11232 avx512vl_i64_info>, VEX_W;
11234 // Patterns to implement vnot using vpternlog instead of creating all ones
11235 // using pcmpeq or vpternlog and then xoring with that. The value 15 is chosen
11236 // so that the result is only dependent on src0. But we use the same source
11237 // for all operands to prevent a false dependency.
11238 // TODO: We should maybe have a more generalized algorithm for folding to
11240 let Predicates = [HasAVX512] in {
11241 def : Pat<(v8i64 (xor VR512:$src, (bc_v8i64 (v16i32 immAllOnesV)))),
11242 (VPTERNLOGQZrri VR512:$src, VR512:$src, VR512:$src, (i8 15))>;
11245 let Predicates = [HasAVX512, NoVLX] in {
11246 def : Pat<(v2i64 (xor VR128X:$src, (bc_v2i64 (v4i32 immAllOnesV)))),
11249 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
11250 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
11251 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR128X:$src, sub_xmm),
11252 (i8 15)), sub_xmm)>;
11253 def : Pat<(v4i64 (xor VR256X:$src, (bc_v4i64 (v8i32 immAllOnesV)))),
11256 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR256X:$src, sub_ymm),
11257 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR256X:$src, sub_ymm),
11258 (INSERT_SUBREG (v8i64 (IMPLICIT_DEF)), VR256X:$src, sub_ymm),
11259 (i8 15)), sub_ymm)>;
11262 let Predicates = [HasVLX] in {
11263 def : Pat<(v2i64 (xor VR128X:$src, (bc_v2i64 (v4i32 immAllOnesV)))),
11264 (VPTERNLOGQZ128rri VR128X:$src, VR128X:$src, VR128X:$src, (i8 15))>;
11265 def : Pat<(v4i64 (xor VR256X:$src, (bc_v4i64 (v8i32 immAllOnesV)))),
11266 (VPTERNLOGQZ256rri VR256X:$src, VR256X:$src, VR256X:$src, (i8 15))>;
11269 //===----------------------------------------------------------------------===//
11270 // AVX-512 - FixupImm
11271 //===----------------------------------------------------------------------===//
11273 multiclass avx512_fixupimm_packed<bits<8> opc, string OpcodeStr, SDNode OpNode,
11274 X86FoldableSchedWrite sched, X86VectorVTInfo _>{
11275 let Constraints = "$src1 = $dst", ExeDomain = _.ExeDomain in {
11276 defm rri : AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
11277 (ins _.RC:$src2, _.RC:$src3, i32u8imm:$src4),
11278 OpcodeStr##_.Suffix, "$src4, $src3, $src2", "$src2, $src3, $src4",
11279 (OpNode (_.VT _.RC:$src1),
11281 (_.IntVT _.RC:$src3),
11283 (i32 FROUND_CURRENT))>, Sched<[sched]>;
11284 defm rmi : AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
11285 (ins _.RC:$src2, _.MemOp:$src3, i32u8imm:$src4),
11286 OpcodeStr##_.Suffix, "$src4, $src3, $src2", "$src2, $src3, $src4",
11287 (OpNode (_.VT _.RC:$src1),
11289 (_.IntVT (bitconvert (_.LdFrag addr:$src3))),
11291 (i32 FROUND_CURRENT))>,
11292 Sched<[sched.Folded, ReadAfterLd]>;
11293 defm rmbi : AVX512_maskable_3src<opc, MRMSrcMem, _, (outs _.RC:$dst),
11294 (ins _.RC:$src2, _.ScalarMemOp:$src3, i32u8imm:$src4),
11295 OpcodeStr##_.Suffix, "$src4, ${src3}"##_.BroadcastStr##", $src2",
11296 "$src2, ${src3}"##_.BroadcastStr##", $src4",
11297 (OpNode (_.VT _.RC:$src1),
11299 (_.IntVT (X86VBroadcast(_.ScalarLdFrag addr:$src3))),
11301 (i32 FROUND_CURRENT))>,
11302 EVEX_B, Sched<[sched.Folded, ReadAfterLd]>;
11303 } // Constraints = "$src1 = $dst"
11306 multiclass avx512_fixupimm_packed_sae<bits<8> opc, string OpcodeStr,
11307 SDNode OpNode, X86FoldableSchedWrite sched,
11308 X86VectorVTInfo _>{
11309 let Constraints = "$src1 = $dst", ExeDomain = _.ExeDomain in {
11310 defm rrib : AVX512_maskable_3src<opc, MRMSrcReg, _, (outs _.RC:$dst),
11311 (ins _.RC:$src2, _.RC:$src3, i32u8imm:$src4),
11312 OpcodeStr##_.Suffix, "$src4, {sae}, $src3, $src2",
11313 "$src2, $src3, {sae}, $src4",
11314 (OpNode (_.VT _.RC:$src1),
11316 (_.IntVT _.RC:$src3),
11318 (i32 FROUND_NO_EXC))>,
11319 EVEX_B, Sched<[sched]>;
11323 multiclass avx512_fixupimm_scalar<bits<8> opc, string OpcodeStr, SDNode OpNode,
11324 X86FoldableSchedWrite sched, X86VectorVTInfo _,
11325 X86VectorVTInfo _src3VT> {
11326 let Constraints = "$src1 = $dst" , Predicates = [HasAVX512],
11327 ExeDomain = _.ExeDomain in {
11328 defm rri : AVX512_maskable_3src_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
11329 (ins _.RC:$src2, _.RC:$src3, i32u8imm:$src4),
11330 OpcodeStr##_.Suffix, "$src4, $src3, $src2", "$src2, $src3, $src4",
11331 (OpNode (_.VT _.RC:$src1),
11333 (_src3VT.VT _src3VT.RC:$src3),
11335 (i32 FROUND_CURRENT))>, Sched<[sched]>;
11336 defm rrib : AVX512_maskable_3src_scalar<opc, MRMSrcReg, _, (outs _.RC:$dst),
11337 (ins _.RC:$src2, _.RC:$src3, i32u8imm:$src4),
11338 OpcodeStr##_.Suffix, "$src4, {sae}, $src3, $src2",
11339 "$src2, $src3, {sae}, $src4",
11340 (OpNode (_.VT _.RC:$src1),
11342 (_src3VT.VT _src3VT.RC:$src3),
11344 (i32 FROUND_NO_EXC))>,
11345 EVEX_B, Sched<[sched.Folded, ReadAfterLd]>;
11346 defm rmi : AVX512_maskable_3src_scalar<opc, MRMSrcMem, _, (outs _.RC:$dst),
11347 (ins _.RC:$src2, _.ScalarMemOp:$src3, i32u8imm:$src4),
11348 OpcodeStr##_.Suffix, "$src4, $src3, $src2", "$src2, $src3, $src4",
11349 (OpNode (_.VT _.RC:$src1),
11351 (_src3VT.VT (scalar_to_vector
11352 (_src3VT.ScalarLdFrag addr:$src3))),
11354 (i32 FROUND_CURRENT))>,
11355 Sched<[sched.Folded, ReadAfterLd]>;
11359 multiclass avx512_fixupimm_packed_all<X86SchedWriteWidths sched,
11360 AVX512VLVectorVTInfo _Vec> {
11361 let Predicates = [HasAVX512] in
11362 defm Z : avx512_fixupimm_packed<0x54, "vfixupimm", X86VFixupimm, sched.ZMM,
11364 avx512_fixupimm_packed_sae<0x54, "vfixupimm", X86VFixupimm, sched.ZMM,
11365 _Vec.info512>, AVX512AIi8Base, EVEX_4V, EVEX_V512;
11366 let Predicates = [HasAVX512, HasVLX] in {
11367 defm Z128 : avx512_fixupimm_packed<0x54, "vfixupimm", X86VFixupimm, sched.XMM,
11368 _Vec.info128>, AVX512AIi8Base, EVEX_4V, EVEX_V128;
11369 defm Z256 : avx512_fixupimm_packed<0x54, "vfixupimm", X86VFixupimm, sched.YMM,
11370 _Vec.info256>, AVX512AIi8Base, EVEX_4V, EVEX_V256;
11374 defm VFIXUPIMMSSZ : avx512_fixupimm_scalar<0x55, "vfixupimm", X86VFixupimmScalar,
11375 SchedWriteFAdd.Scl, f32x_info, v4i32x_info>,
11376 AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<32, CD8VT1>;
11377 defm VFIXUPIMMSDZ : avx512_fixupimm_scalar<0x55, "vfixupimm", X86VFixupimmScalar,
11378 SchedWriteFAdd.Scl, f64x_info, v2i64x_info>,
11379 AVX512AIi8Base, VEX_LIG, EVEX_4V, EVEX_CD8<64, CD8VT1>, VEX_W;
11380 defm VFIXUPIMMPS : avx512_fixupimm_packed_all<SchedWriteFAdd, avx512vl_f32_info>,
11381 EVEX_CD8<32, CD8VF>;
11382 defm VFIXUPIMMPD : avx512_fixupimm_packed_all<SchedWriteFAdd, avx512vl_f64_info>,
11383 EVEX_CD8<64, CD8VF>, VEX_W;
11385 // Patterns used to select SSE scalar fp arithmetic instructions from
11388 // (1) a scalar fp operation followed by a blend
11390 // The effect is that the backend no longer emits unnecessary vector
11391 // insert instructions immediately after SSE scalar fp instructions
11392 // like addss or mulss.
11394 // For example, given the following code:
11395 // __m128 foo(__m128 A, __m128 B) {
11400 // Previously we generated:
11401 // addss %xmm0, %xmm1
11402 // movss %xmm1, %xmm0
11404 // We now generate:
11405 // addss %xmm1, %xmm0
11407 // (2) a vector packed single/double fp operation followed by a vector insert
11409 // The effect is that the backend converts the packed fp instruction
11410 // followed by a vector insert into a single SSE scalar fp instruction.
11412 // For example, given the following code:
11413 // __m128 foo(__m128 A, __m128 B) {
11414 // __m128 C = A + B;
11415 // return (__m128) {c[0], a[1], a[2], a[3]};
11418 // Previously we generated:
11419 // addps %xmm0, %xmm1
11420 // movss %xmm1, %xmm0
11422 // We now generate:
11423 // addss %xmm1, %xmm0
11425 // TODO: Some canonicalization in lowering would simplify the number of
11426 // patterns we have to try to match.
11427 multiclass AVX512_scalar_math_fp_patterns<SDNode Op, string OpcPrefix, SDNode MoveNode,
11428 X86VectorVTInfo _, PatLeaf ZeroFP> {
11429 let Predicates = [HasAVX512] in {
11430 // extracted scalar math op with insert via movss
11431 def : Pat<(_.VT (MoveNode (_.VT VR128X:$dst), (_.VT (scalar_to_vector
11432 (Op (_.EltVT (extractelt (_.VT VR128X:$dst), (iPTR 0))),
11434 (!cast<I>("V"#OpcPrefix#Zrr_Int) _.VT:$dst,
11435 (COPY_TO_REGCLASS _.FRC:$src, VR128X))>;
11437 // vector math op with insert via movss
11438 def : Pat<(_.VT (MoveNode (_.VT VR128X:$dst),
11439 (Op (_.VT VR128X:$dst), (_.VT VR128X:$src)))),
11440 (!cast<I>("V"#OpcPrefix#Zrr_Int) _.VT:$dst, _.VT:$src)>;
11442 // extracted masked scalar math op with insert via movss
11443 def : Pat<(MoveNode (_.VT VR128X:$src1),
11445 (X86selects VK1WM:$mask,
11446 (Op (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0))),
11449 (!cast<I>("V"#OpcPrefix#Zrr_Intk) (COPY_TO_REGCLASS _.FRC:$src0, VR128X),
11450 VK1WM:$mask, _.VT:$src1,
11451 (COPY_TO_REGCLASS _.FRC:$src2, VR128X))>;
11453 // extracted masked scalar math op with insert via movss
11454 def : Pat<(MoveNode (_.VT VR128X:$src1),
11456 (X86selects VK1WM:$mask,
11457 (Op (_.EltVT (extractelt (_.VT VR128X:$src1), (iPTR 0))),
11458 _.FRC:$src2), (_.EltVT ZeroFP)))),
11459 (!cast<I>("V"#OpcPrefix#Zrr_Intkz)
11460 VK1WM:$mask, _.VT:$src1,
11461 (COPY_TO_REGCLASS _.FRC:$src2, VR128X))>;
11465 defm : AVX512_scalar_math_fp_patterns<fadd, "ADDSS", X86Movss, v4f32x_info, fp32imm0>;
11466 defm : AVX512_scalar_math_fp_patterns<fsub, "SUBSS", X86Movss, v4f32x_info, fp32imm0>;
11467 defm : AVX512_scalar_math_fp_patterns<fmul, "MULSS", X86Movss, v4f32x_info, fp32imm0>;
11468 defm : AVX512_scalar_math_fp_patterns<fdiv, "DIVSS", X86Movss, v4f32x_info, fp32imm0>;
11470 defm : AVX512_scalar_math_fp_patterns<fadd, "ADDSD", X86Movsd, v2f64x_info, fp64imm0>;
11471 defm : AVX512_scalar_math_fp_patterns<fsub, "SUBSD", X86Movsd, v2f64x_info, fp64imm0>;
11472 defm : AVX512_scalar_math_fp_patterns<fmul, "MULSD", X86Movsd, v2f64x_info, fp64imm0>;
11473 defm : AVX512_scalar_math_fp_patterns<fdiv, "DIVSD", X86Movsd, v2f64x_info, fp64imm0>;
11476 //===----------------------------------------------------------------------===//
11477 // AES instructions
11478 //===----------------------------------------------------------------------===//
11480 multiclass avx512_vaes<bits<8> Op, string OpStr, string IntPrefix> {
11481 let Predicates = [HasVLX, HasVAES] in {
11482 defm Z128 : AESI_binop_rm_int<Op, OpStr,
11483 !cast<Intrinsic>(IntPrefix),
11484 loadv2i64, 0, VR128X, i128mem>,
11485 EVEX_4V, EVEX_CD8<64, CD8VF>, EVEX_V128, VEX_WIG;
11486 defm Z256 : AESI_binop_rm_int<Op, OpStr,
11487 !cast<Intrinsic>(IntPrefix##"_256"),
11488 loadv4i64, 0, VR256X, i256mem>,
11489 EVEX_4V, EVEX_CD8<64, CD8VF>, EVEX_V256, VEX_WIG;
11491 let Predicates = [HasAVX512, HasVAES] in
11492 defm Z : AESI_binop_rm_int<Op, OpStr,
11493 !cast<Intrinsic>(IntPrefix##"_512"),
11494 loadv8i64, 0, VR512, i512mem>,
11495 EVEX_4V, EVEX_CD8<64, CD8VF>, EVEX_V512, VEX_WIG;
11498 defm VAESENC : avx512_vaes<0xDC, "vaesenc", "int_x86_aesni_aesenc">;
11499 defm VAESENCLAST : avx512_vaes<0xDD, "vaesenclast", "int_x86_aesni_aesenclast">;
11500 defm VAESDEC : avx512_vaes<0xDE, "vaesdec", "int_x86_aesni_aesdec">;
11501 defm VAESDECLAST : avx512_vaes<0xDF, "vaesdeclast", "int_x86_aesni_aesdeclast">;
11503 //===----------------------------------------------------------------------===//
11504 // PCLMUL instructions - Carry less multiplication
11505 //===----------------------------------------------------------------------===//
11507 let Predicates = [HasAVX512, HasVPCLMULQDQ] in
11508 defm VPCLMULQDQZ : vpclmulqdq<VR512, i512mem, loadv8i64, int_x86_pclmulqdq_512>,
11509 EVEX_4V, EVEX_V512, EVEX_CD8<64, CD8VF>, VEX_WIG;
11511 let Predicates = [HasVLX, HasVPCLMULQDQ] in {
11512 defm VPCLMULQDQZ128 : vpclmulqdq<VR128X, i128mem, loadv2i64, int_x86_pclmulqdq>,
11513 EVEX_4V, EVEX_V128, EVEX_CD8<64, CD8VF>, VEX_WIG;
11515 defm VPCLMULQDQZ256: vpclmulqdq<VR256X, i256mem, loadv4i64,
11516 int_x86_pclmulqdq_256>, EVEX_4V, EVEX_V256,
11517 EVEX_CD8<64, CD8VF>, VEX_WIG;
11521 defm : vpclmulqdq_aliases<"VPCLMULQDQZ", VR512, i512mem>;
11522 defm : vpclmulqdq_aliases<"VPCLMULQDQZ128", VR128X, i128mem>;
11523 defm : vpclmulqdq_aliases<"VPCLMULQDQZ256", VR256X, i256mem>;
11525 //===----------------------------------------------------------------------===//
11527 //===----------------------------------------------------------------------===//
11529 multiclass VBMI2_shift_var_rm<bits<8> Op, string OpStr, SDNode OpNode,
11530 X86FoldableSchedWrite sched, X86VectorVTInfo VTI> {
11531 let Constraints = "$src1 = $dst",
11532 ExeDomain = VTI.ExeDomain in {
11533 defm r: AVX512_maskable_3src<Op, MRMSrcReg, VTI, (outs VTI.RC:$dst),
11534 (ins VTI.RC:$src2, VTI.RC:$src3), OpStr,
11535 "$src3, $src2", "$src2, $src3",
11536 (VTI.VT (OpNode VTI.RC:$src1, VTI.RC:$src2, VTI.RC:$src3))>,
11537 AVX512FMA3Base, Sched<[sched]>;
11538 defm m: AVX512_maskable_3src<Op, MRMSrcMem, VTI, (outs VTI.RC:$dst),
11539 (ins VTI.RC:$src2, VTI.MemOp:$src3), OpStr,
11540 "$src3, $src2", "$src2, $src3",
11541 (VTI.VT (OpNode VTI.RC:$src1, VTI.RC:$src2,
11542 (VTI.VT (bitconvert (VTI.LdFrag addr:$src3)))))>,
11544 Sched<[sched.Folded, ReadAfterLd]>;
11548 multiclass VBMI2_shift_var_rmb<bits<8> Op, string OpStr, SDNode OpNode,
11549 X86FoldableSchedWrite sched, X86VectorVTInfo VTI>
11550 : VBMI2_shift_var_rm<Op, OpStr, OpNode, sched, VTI> {
11551 let Constraints = "$src1 = $dst",
11552 ExeDomain = VTI.ExeDomain in
11553 defm mb: AVX512_maskable_3src<Op, MRMSrcMem, VTI, (outs VTI.RC:$dst),
11554 (ins VTI.RC:$src2, VTI.ScalarMemOp:$src3), OpStr,
11555 "${src3}"##VTI.BroadcastStr##", $src2",
11556 "$src2, ${src3}"##VTI.BroadcastStr,
11557 (OpNode VTI.RC:$src1, VTI.RC:$src2,
11558 (VTI.VT (X86VBroadcast (VTI.ScalarLdFrag addr:$src3))))>,
11559 AVX512FMA3Base, EVEX_B,
11560 Sched<[sched.Folded, ReadAfterLd]>;
11563 multiclass VBMI2_shift_var_rm_common<bits<8> Op, string OpStr, SDNode OpNode,
11564 X86SchedWriteWidths sched, AVX512VLVectorVTInfo VTI> {
11565 let Predicates = [HasVBMI2] in
11566 defm Z : VBMI2_shift_var_rm<Op, OpStr, OpNode, sched.ZMM, VTI.info512>,
11568 let Predicates = [HasVBMI2, HasVLX] in {
11569 defm Z256 : VBMI2_shift_var_rm<Op, OpStr, OpNode, sched.YMM, VTI.info256>,
11571 defm Z128 : VBMI2_shift_var_rm<Op, OpStr, OpNode, sched.XMM, VTI.info128>,
11576 multiclass VBMI2_shift_var_rmb_common<bits<8> Op, string OpStr, SDNode OpNode,
11577 X86SchedWriteWidths sched, AVX512VLVectorVTInfo VTI> {
11578 let Predicates = [HasVBMI2] in
11579 defm Z : VBMI2_shift_var_rmb<Op, OpStr, OpNode, sched.ZMM, VTI.info512>,
11581 let Predicates = [HasVBMI2, HasVLX] in {
11582 defm Z256 : VBMI2_shift_var_rmb<Op, OpStr, OpNode, sched.YMM, VTI.info256>,
11584 defm Z128 : VBMI2_shift_var_rmb<Op, OpStr, OpNode, sched.XMM, VTI.info128>,
11588 multiclass VBMI2_shift_var<bits<8> wOp, bits<8> dqOp, string Prefix,
11589 SDNode OpNode, X86SchedWriteWidths sched> {
11590 defm W : VBMI2_shift_var_rm_common<wOp, Prefix##"w", OpNode, sched,
11591 avx512vl_i16_info>, VEX_W, EVEX_CD8<16, CD8VF>;
11592 defm D : VBMI2_shift_var_rmb_common<dqOp, Prefix##"d", OpNode, sched,
11593 avx512vl_i32_info>, EVEX_CD8<32, CD8VF>;
11594 defm Q : VBMI2_shift_var_rmb_common<dqOp, Prefix##"q", OpNode, sched,
11595 avx512vl_i64_info>, VEX_W, EVEX_CD8<64, CD8VF>;
11598 multiclass VBMI2_shift_imm<bits<8> wOp, bits<8> dqOp, string Prefix,
11599 SDNode OpNode, X86SchedWriteWidths sched> {
11600 defm W : avx512_common_3Op_rm_imm8<wOp, OpNode, Prefix##"w", sched,
11601 avx512vl_i16_info, avx512vl_i16_info, HasVBMI2>,
11602 VEX_W, EVEX_CD8<16, CD8VF>;
11603 defm D : avx512_common_3Op_imm8<Prefix##"d", avx512vl_i32_info, dqOp,
11604 OpNode, sched, HasVBMI2>, AVX512AIi8Base, EVEX_4V, EVEX_CD8<32, CD8VF>;
11605 defm Q : avx512_common_3Op_imm8<Prefix##"q", avx512vl_i64_info, dqOp, OpNode,
11606 sched, HasVBMI2>, AVX512AIi8Base, EVEX_4V, EVEX_CD8<64, CD8VF>, VEX_W;
11610 defm VPSHLDV : VBMI2_shift_var<0x70, 0x71, "vpshldv", X86VShldv, SchedWriteVecIMul>;
11611 defm VPSHRDV : VBMI2_shift_var<0x72, 0x73, "vpshrdv", X86VShrdv, SchedWriteVecIMul>;
11612 defm VPSHLD : VBMI2_shift_imm<0x70, 0x71, "vpshld", X86VShld, SchedWriteVecIMul>;
11613 defm VPSHRD : VBMI2_shift_imm<0x72, 0x73, "vpshrd", X86VShrd, SchedWriteVecIMul>;
11616 defm VPCOMPRESSB : compress_by_elt_width<0x63, "vpcompressb", WriteVarShuffle256,
11617 avx512vl_i8_info, HasVBMI2>, EVEX,
11619 defm VPCOMPRESSW : compress_by_elt_width <0x63, "vpcompressw", WriteVarShuffle256,
11620 avx512vl_i16_info, HasVBMI2>, EVEX, VEX_W,
11623 defm VPEXPANDB : expand_by_elt_width <0x62, "vpexpandb", WriteVarShuffle256,
11624 avx512vl_i8_info, HasVBMI2>, EVEX;
11625 defm VPEXPANDW : expand_by_elt_width <0x62, "vpexpandw", WriteVarShuffle256,
11626 avx512vl_i16_info, HasVBMI2>, EVEX, VEX_W;
11628 //===----------------------------------------------------------------------===//
11630 //===----------------------------------------------------------------------===//
11632 let Constraints = "$src1 = $dst" in
11633 multiclass VNNI_rmb<bits<8> Op, string OpStr, SDNode OpNode,
11634 X86FoldableSchedWrite sched, X86VectorVTInfo VTI> {
11635 defm r : AVX512_maskable_3src<Op, MRMSrcReg, VTI, (outs VTI.RC:$dst),
11636 (ins VTI.RC:$src2, VTI.RC:$src3), OpStr,
11637 "$src3, $src2", "$src2, $src3",
11638 (VTI.VT (OpNode VTI.RC:$src1,
11639 VTI.RC:$src2, VTI.RC:$src3))>,
11640 EVEX_4V, T8PD, Sched<[sched]>;
11641 defm m : AVX512_maskable_3src<Op, MRMSrcMem, VTI, (outs VTI.RC:$dst),
11642 (ins VTI.RC:$src2, VTI.MemOp:$src3), OpStr,
11643 "$src3, $src2", "$src2, $src3",
11644 (VTI.VT (OpNode VTI.RC:$src1, VTI.RC:$src2,
11645 (VTI.VT (bitconvert
11646 (VTI.LdFrag addr:$src3)))))>,
11647 EVEX_4V, EVEX_CD8<32, CD8VF>, T8PD,
11648 Sched<[sched.Folded, ReadAfterLd]>;
11649 defm mb : AVX512_maskable_3src<Op, MRMSrcMem, VTI, (outs VTI.RC:$dst),
11650 (ins VTI.RC:$src2, VTI.ScalarMemOp:$src3),
11651 OpStr, "${src3}"##VTI.BroadcastStr##", $src2",
11652 "$src2, ${src3}"##VTI.BroadcastStr,
11653 (OpNode VTI.RC:$src1, VTI.RC:$src2,
11654 (VTI.VT (X86VBroadcast
11655 (VTI.ScalarLdFrag addr:$src3))))>,
11656 EVEX_4V, EVEX_CD8<32, CD8VF>, EVEX_B,
11657 T8PD, Sched<[sched.Folded, ReadAfterLd]>;
11660 multiclass VNNI_common<bits<8> Op, string OpStr, SDNode OpNode,
11661 X86SchedWriteWidths sched> {
11662 let Predicates = [HasVNNI] in
11663 defm Z : VNNI_rmb<Op, OpStr, OpNode, sched.ZMM, v16i32_info>, EVEX_V512;
11664 let Predicates = [HasVNNI, HasVLX] in {
11665 defm Z256 : VNNI_rmb<Op, OpStr, OpNode, sched.YMM, v8i32x_info>, EVEX_V256;
11666 defm Z128 : VNNI_rmb<Op, OpStr, OpNode, sched.XMM, v4i32x_info>, EVEX_V128;
11670 // FIXME: Is there a better scheduler class for VPDP?
11671 defm VPDPBUSD : VNNI_common<0x50, "vpdpbusd", X86Vpdpbusd, SchedWriteVecIMul>;
11672 defm VPDPBUSDS : VNNI_common<0x51, "vpdpbusds", X86Vpdpbusds, SchedWriteVecIMul>;
11673 defm VPDPWSSD : VNNI_common<0x52, "vpdpwssd", X86Vpdpwssd, SchedWriteVecIMul>;
11674 defm VPDPWSSDS : VNNI_common<0x53, "vpdpwssds", X86Vpdpwssds, SchedWriteVecIMul>;
11676 //===----------------------------------------------------------------------===//
11678 //===----------------------------------------------------------------------===//
11680 // FIXME: Is there a better scheduler class for VPOPCNTB/VPOPCNTW?
11681 defm VPOPCNTB : avx512_unary_rm_vl<0x54, "vpopcntb", ctpop, SchedWriteVecALU,
11682 avx512vl_i8_info, HasBITALG>;
11683 defm VPOPCNTW : avx512_unary_rm_vl<0x54, "vpopcntw", ctpop, SchedWriteVecALU,
11684 avx512vl_i16_info, HasBITALG>, VEX_W;
11686 defm : avx512_unary_lowering<"VPOPCNTB", ctpop, avx512vl_i8_info, HasBITALG>;
11687 defm : avx512_unary_lowering<"VPOPCNTW", ctpop, avx512vl_i16_info, HasBITALG>;
11689 multiclass VPSHUFBITQMB_rm<X86FoldableSchedWrite sched, X86VectorVTInfo VTI> {
11690 defm rr : AVX512_maskable_cmp<0x8F, MRMSrcReg, VTI, (outs VTI.KRC:$dst),
11691 (ins VTI.RC:$src1, VTI.RC:$src2),
11693 "$src2, $src1", "$src1, $src2",
11694 (X86Vpshufbitqmb (VTI.VT VTI.RC:$src1),
11695 (VTI.VT VTI.RC:$src2))>, EVEX_4V, T8PD,
11697 defm rm : AVX512_maskable_cmp<0x8F, MRMSrcMem, VTI, (outs VTI.KRC:$dst),
11698 (ins VTI.RC:$src1, VTI.MemOp:$src2),
11700 "$src2, $src1", "$src1, $src2",
11701 (X86Vpshufbitqmb (VTI.VT VTI.RC:$src1),
11702 (VTI.VT (bitconvert (VTI.LdFrag addr:$src2))))>,
11703 EVEX_4V, EVEX_CD8<8, CD8VF>, T8PD,
11704 Sched<[sched.Folded, ReadAfterLd]>;
11707 multiclass VPSHUFBITQMB_common<X86SchedWriteWidths sched, AVX512VLVectorVTInfo VTI> {
11708 let Predicates = [HasBITALG] in
11709 defm Z : VPSHUFBITQMB_rm<sched.ZMM, VTI.info512>, EVEX_V512;
11710 let Predicates = [HasBITALG, HasVLX] in {
11711 defm Z256 : VPSHUFBITQMB_rm<sched.YMM, VTI.info256>, EVEX_V256;
11712 defm Z128 : VPSHUFBITQMB_rm<sched.XMM, VTI.info128>, EVEX_V128;
11716 // FIXME: Is there a better scheduler class for VPSHUFBITQMB?
11717 defm VPSHUFBITQMB : VPSHUFBITQMB_common<SchedWriteVecIMul, avx512vl_i8_info>;
11719 //===----------------------------------------------------------------------===//
11721 //===----------------------------------------------------------------------===//
11723 multiclass GF2P8MULB_avx512_common<bits<8> Op, string OpStr, SDNode OpNode,
11724 X86SchedWriteWidths sched> {
11725 let Predicates = [HasGFNI, HasAVX512, HasBWI] in
11726 defm Z : avx512_binop_rm<Op, OpStr, OpNode, v64i8_info, sched.ZMM, 1>,
11728 let Predicates = [HasGFNI, HasVLX, HasBWI] in {
11729 defm Z256 : avx512_binop_rm<Op, OpStr, OpNode, v32i8x_info, sched.YMM, 1>,
11731 defm Z128 : avx512_binop_rm<Op, OpStr, OpNode, v16i8x_info, sched.XMM, 1>,
11736 defm VGF2P8MULB : GF2P8MULB_avx512_common<0xCF, "vgf2p8mulb", X86GF2P8mulb,
11738 EVEX_CD8<8, CD8VF>, T8PD;
11740 multiclass GF2P8AFFINE_avx512_rmb_imm<bits<8> Op, string OpStr, SDNode OpNode,
11741 X86FoldableSchedWrite sched, X86VectorVTInfo VTI,
11742 X86VectorVTInfo BcstVTI>
11743 : avx512_3Op_rm_imm8<Op, OpStr, OpNode, sched, VTI, VTI> {
11744 let ExeDomain = VTI.ExeDomain in
11745 defm rmbi : AVX512_maskable<Op, MRMSrcMem, VTI, (outs VTI.RC:$dst),
11746 (ins VTI.RC:$src1, VTI.ScalarMemOp:$src2, u8imm:$src3),
11747 OpStr, "$src3, ${src2}"##BcstVTI.BroadcastStr##", $src1",
11748 "$src1, ${src2}"##BcstVTI.BroadcastStr##", $src3",
11749 (OpNode (VTI.VT VTI.RC:$src1),
11750 (bitconvert (BcstVTI.VT (X86VBroadcast (loadi64 addr:$src2)))),
11751 (i8 imm:$src3))>, EVEX_B,
11752 Sched<[sched.Folded, ReadAfterLd]>;
11755 multiclass GF2P8AFFINE_avx512_common<bits<8> Op, string OpStr, SDNode OpNode,
11756 X86SchedWriteWidths sched> {
11757 let Predicates = [HasGFNI, HasAVX512, HasBWI] in
11758 defm Z : GF2P8AFFINE_avx512_rmb_imm<Op, OpStr, OpNode, sched.ZMM,
11759 v64i8_info, v8i64_info>, EVEX_V512;
11760 let Predicates = [HasGFNI, HasVLX, HasBWI] in {
11761 defm Z256 : GF2P8AFFINE_avx512_rmb_imm<Op, OpStr, OpNode, sched.YMM,
11762 v32i8x_info, v4i64x_info>, EVEX_V256;
11763 defm Z128 : GF2P8AFFINE_avx512_rmb_imm<Op, OpStr, OpNode, sched.XMM,
11764 v16i8x_info, v2i64x_info>, EVEX_V128;
11768 defm VGF2P8AFFINEINVQB : GF2P8AFFINE_avx512_common<0xCF, "vgf2p8affineinvqb",
11769 X86GF2P8affineinvqb, SchedWriteVecIMul>,
11770 EVEX_4V, EVEX_CD8<8, CD8VF>, VEX_W, AVX512AIi8Base;
11771 defm VGF2P8AFFINEQB : GF2P8AFFINE_avx512_common<0xCE, "vgf2p8affineqb",
11772 X86GF2P8affineqb, SchedWriteVecIMul>,
11773 EVEX_4V, EVEX_CD8<8, CD8VF>, VEX_W, AVX512AIi8Base;
11776 //===----------------------------------------------------------------------===//
11778 //===----------------------------------------------------------------------===//
11780 let hasSideEffects = 0, mayLoad = 1, ExeDomain = SSEPackedSingle,
11781 Constraints = "$src1 = $dst" in {
11782 defm V4FMADDPSrm : AVX512_maskable_3src_in_asm<0x9A, MRMSrcMem, v16f32_info,
11783 (outs VR512:$dst), (ins VR512:$src2, f128mem:$src3),
11784 "v4fmaddps", "$src3, $src2", "$src2, $src3",
11785 []>, EVEX_V512, EVEX_4V, T8XD, EVEX_CD8<32, CD8VQ>,
11786 Sched<[SchedWriteFMA.ZMM.Folded]>;
11788 defm V4FNMADDPSrm : AVX512_maskable_3src_in_asm<0xAA, MRMSrcMem, v16f32_info,
11789 (outs VR512:$dst), (ins VR512:$src2, f128mem:$src3),
11790 "v4fnmaddps", "$src3, $src2", "$src2, $src3",
11791 []>, EVEX_V512, EVEX_4V, T8XD, EVEX_CD8<32, CD8VQ>,
11792 Sched<[SchedWriteFMA.ZMM.Folded]>;
11794 defm V4FMADDSSrm : AVX512_maskable_3src_in_asm<0x9B, MRMSrcMem, f32x_info,
11795 (outs VR128X:$dst), (ins VR128X:$src2, f128mem:$src3),
11796 "v4fmaddss", "$src3, $src2", "$src2, $src3",
11797 []>, EVEX_V128, EVEX_4V, T8XD, EVEX_CD8<32, CD8VF>,
11798 Sched<[SchedWriteFMA.Scl.Folded]>;
11800 defm V4FNMADDSSrm : AVX512_maskable_3src_in_asm<0xAB, MRMSrcMem, f32x_info,
11801 (outs VR128X:$dst), (ins VR128X:$src2, f128mem:$src3),
11802 "v4fnmaddss", "$src3, $src2", "$src2, $src3",
11803 []>, EVEX_V128, EVEX_4V, T8XD, EVEX_CD8<32, CD8VF>,
11804 Sched<[SchedWriteFMA.Scl.Folded]>;
11807 //===----------------------------------------------------------------------===//
11809 //===----------------------------------------------------------------------===//
11811 let hasSideEffects = 0, mayLoad = 1, ExeDomain = SSEPackedInt,
11812 Constraints = "$src1 = $dst" in {
11813 defm VP4DPWSSDrm : AVX512_maskable_3src_in_asm<0x52, MRMSrcMem, v16i32_info,
11814 (outs VR512:$dst), (ins VR512:$src2, f128mem:$src3),
11815 "vp4dpwssd", "$src3, $src2", "$src2, $src3",
11816 []>, EVEX_V512, EVEX_4V, T8XD, EVEX_CD8<32, CD8VQ>,
11817 Sched<[SchedWriteFMA.ZMM.Folded]>;
11819 defm VP4DPWSSDSrm : AVX512_maskable_3src_in_asm<0x53, MRMSrcMem, v16i32_info,
11820 (outs VR512:$dst), (ins VR512:$src2, f128mem:$src3),
11821 "vp4dpwssds", "$src3, $src2", "$src2, $src3",
11822 []>, EVEX_V512, EVEX_4V, T8XD, EVEX_CD8<32, CD8VQ>,
11823 Sched<[SchedWriteFMA.ZMM.Folded]>;