--- /dev/null
+//=== lib/CodeGen/GlobalISel/AMDGPUPreLegalizerCombiner.cpp ---------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass does combining of machine instructions at the generic MI level,
+// before the legalizer.
+//
+//===----------------------------------------------------------------------===//
+
+#include "AMDGPUTargetMachine.h"
+#include "llvm/CodeGen/GlobalISel/Combiner.h"
+#include "llvm/CodeGen/GlobalISel/CombinerHelper.h"
+#include "llvm/CodeGen/GlobalISel/CombinerInfo.h"
+#include "llvm/CodeGen/GlobalISel/GISelKnownBits.h"
+#include "llvm/CodeGen/GlobalISel/MIPatternMatch.h"
+#include "llvm/CodeGen/MachineDominators.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/TargetPassConfig.h"
+#include "llvm/Support/Debug.h"
+
+#define DEBUG_TYPE "amdgpu-prelegalizer-combiner"
+
+using namespace llvm;
+using namespace MIPatternMatch;
+
+#define AMDGPUPRELEGALIZERCOMBINERHELPER_GENCOMBINERHELPER_DEPS
+#include "AMDGPUGenGICombiner.inc"
+#undef AMDGPUPRELEGALIZERCOMBINERHELPER_GENCOMBINERHELPER_DEPS
+
+namespace {
+#define AMDGPUPRELEGALIZERCOMBINERHELPER_GENCOMBINERHELPER_H
+#include "AMDGPUGenGICombiner.inc"
+#undef AMDGPUPRELEGALIZERCOMBINERHELPER_GENCOMBINERHELPER_H
+
+class AMDGPUPreLegalizerCombinerInfo : public CombinerInfo {
+ GISelKnownBits *KB;
+ MachineDominatorTree *MDT;
+
+public:
+ AMDGPUGenPreLegalizerCombinerHelper Generated;
+
+ AMDGPUPreLegalizerCombinerInfo(bool EnableOpt, bool OptSize, bool MinSize,
+ GISelKnownBits *KB, MachineDominatorTree *MDT)
+ : CombinerInfo(/*AllowIllegalOps*/ true, /*ShouldLegalizeIllegal*/ false,
+ /*LegalizerInfo*/ nullptr, EnableOpt, OptSize, MinSize),
+ KB(KB), MDT(MDT) {
+ if (!Generated.parseCommandLineOption())
+ report_fatal_error("Invalid rule identifier");
+ }
+
+ virtual bool combine(GISelChangeObserver &Observer, MachineInstr &MI,
+ MachineIRBuilder &B) const override;
+};
+
+bool AMDGPUPreLegalizerCombinerInfo::combine(GISelChangeObserver &Observer,
+ MachineInstr &MI,
+ MachineIRBuilder &B) const {
+ CombinerHelper Helper(Observer, B, KB, MDT);
+
+ if (Generated.tryCombineAll(Observer, MI, B, Helper))
+ return true;
+
+ switch (MI.getOpcode()) {
+ case TargetOpcode::G_CONCAT_VECTORS:
+ return Helper.tryCombineConcatVectors(MI);
+ case TargetOpcode::G_SHUFFLE_VECTOR:
+ return Helper.tryCombineShuffleVector(MI);
+ }
+
+ return false;
+}
+
+#define AMDGPUPRELEGALIZERCOMBINERHELPER_GENCOMBINERHELPER_CPP
+#include "AMDGPUGenGICombiner.inc"
+#undef AMDGPUPRELEGALIZERCOMBINERHELPER_GENCOMBINERHELPER_CPP
+
+// Pass boilerplate
+// ================
+
+class AMDGPUPreLegalizerCombiner : public MachineFunctionPass {
+public:
+ static char ID;
+
+ AMDGPUPreLegalizerCombiner(bool IsOptNone = false);
+
+ StringRef getPassName() const override { return "AMDGPUPreLegalizerCombiner"; }
+
+ bool runOnMachineFunction(MachineFunction &MF) override;
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override;
+private:
+ bool IsOptNone;
+};
+} // end anonymous namespace
+
+void AMDGPUPreLegalizerCombiner::getAnalysisUsage(AnalysisUsage &AU) const {
+ AU.addRequired<TargetPassConfig>();
+ AU.setPreservesCFG();
+ getSelectionDAGFallbackAnalysisUsage(AU);
+ AU.addRequired<GISelKnownBitsAnalysis>();
+ AU.addPreserved<GISelKnownBitsAnalysis>();
+ if (!IsOptNone) {
+ AU.addRequired<MachineDominatorTree>();
+ AU.addPreserved<MachineDominatorTree>();
+ }
+ MachineFunctionPass::getAnalysisUsage(AU);
+}
+
+AMDGPUPreLegalizerCombiner::AMDGPUPreLegalizerCombiner(bool IsOptNone)
+ : MachineFunctionPass(ID), IsOptNone(IsOptNone) {
+ initializeAMDGPUPreLegalizerCombinerPass(*PassRegistry::getPassRegistry());
+}
+
+bool AMDGPUPreLegalizerCombiner::runOnMachineFunction(MachineFunction &MF) {
+ if (MF.getProperties().hasProperty(
+ MachineFunctionProperties::Property::FailedISel))
+ return false;
+ auto *TPC = &getAnalysis<TargetPassConfig>();
+ const Function &F = MF.getFunction();
+ bool EnableOpt =
+ MF.getTarget().getOptLevel() != CodeGenOpt::None && !skipFunction(F);
+ GISelKnownBits *KB = &getAnalysis<GISelKnownBitsAnalysis>().get(MF);
+ MachineDominatorTree *MDT =
+ IsOptNone ? nullptr : &getAnalysis<MachineDominatorTree>();
+ AMDGPUPreLegalizerCombinerInfo PCInfo(EnableOpt, F.hasOptSize(),
+ F.hasMinSize(), KB, MDT);
+ Combiner C(PCInfo, TPC);
+ return C.combineMachineInstrs(MF, /*CSEInfo*/ nullptr);
+}
+
+char AMDGPUPreLegalizerCombiner::ID = 0;
+INITIALIZE_PASS_BEGIN(AMDGPUPreLegalizerCombiner, DEBUG_TYPE,
+ "Combine AMDGPU machine instrs before legalization",
+ false, false)
+INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
+INITIALIZE_PASS_DEPENDENCY(GISelKnownBitsAnalysis)
+INITIALIZE_PASS_END(AMDGPUPreLegalizerCombiner, DEBUG_TYPE,
+ "Combine AMDGPU machine instrs before legalization", false,
+ false)
+
+namespace llvm {
+FunctionPass *createAMDGPUPreLegalizeCombiner(bool IsOptNone) {
+ return new AMDGPUPreLegalizerCombiner(IsOptNone);
+}
+} // end namespace llvm
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
-;
; GFX9-LABEL: lds_atomic_dec_ret_i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
-;
; GFX9-LABEL: lds_atomic_dec_ret_i32_offset:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
; VI-NEXT: v_mov_b32_e32 v1, s0
; VI-NEXT: ds_dec_rtn_u32 v0, v1, v0
; VI-NEXT: s_endpgm
-;
; GFX9-LABEL: lds_atomic_dec_noret_i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s0, s[0:1], 0x24
; VI-NEXT: v_mov_b32_e32 v1, s0
; VI-NEXT: ds_dec_rtn_u32 v0, v1, v0
; VI-NEXT: s_endpgm
-;
; GFX9-LABEL: lds_atomic_dec_noret_i32_offset:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s0, s[0:1], 0x24
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
-;
; GFX9-LABEL: global_atomic_dec_ret_i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
-;
; GFX9-LABEL: global_atomic_dec_ret_i32_offset:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: flat_atomic_dec v0, v[0:1], v2 glc
; VI-NEXT: s_endpgm
-;
; GFX9-LABEL: global_atomic_dec_noret_i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: flat_atomic_dec v0, v[0:1], v2 glc
; VI-NEXT: s_endpgm
-;
; GFX9-LABEL: global_atomic_dec_noret_i32_offset:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_store_dword v[2:3], v0
; VI-NEXT: s_endpgm
-;
; GFX9-LABEL: global_atomic_dec_ret_i32_offset_addr64:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: flat_atomic_dec v0, v[0:1], v2 glc
; VI-NEXT: s_endpgm
-;
; GFX9-LABEL: global_atomic_dec_noret_i32_offset_addr64:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
-;
; GFX9-LABEL: flat_atomic_dec_ret_i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_store_dword v[0:1], v2
; VI-NEXT: s_endpgm
-;
; GFX9-LABEL: flat_atomic_dec_ret_i32_offset:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: flat_atomic_dec v0, v[0:1], v2 glc
; VI-NEXT: s_endpgm
-;
; GFX9-LABEL: flat_atomic_dec_noret_i32:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: flat_atomic_dec v0, v[0:1], v2 glc
; VI-NEXT: s_endpgm
-;
; GFX9-LABEL: flat_atomic_dec_noret_i32_offset:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_store_dword v[2:3], v0
; VI-NEXT: s_endpgm
-;
; GFX9-LABEL: flat_atomic_dec_ret_i32_offset_addr64:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; VI-NEXT: v_addc_u32_e32 v1, vcc, 0, v1, vcc
; VI-NEXT: flat_atomic_dec v0, v[0:1], v2 glc
; VI-NEXT: s_endpgm
-;
; GFX9-LABEL: flat_atomic_dec_noret_i32_offset_addr64:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_endpgm
-;
; GFX9-LABEL: flat_atomic_dec_ret_i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_endpgm
-;
; GFX9-LABEL: flat_atomic_dec_ret_i64_offset:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
; VI-NEXT: s_endpgm
-;
; GFX9-LABEL: flat_atomic_dec_noret_i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
; VI-NEXT: s_endpgm
-;
; GFX9-LABEL: flat_atomic_dec_noret_i64_offset:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_endpgm
-;
; GFX9-LABEL: flat_atomic_dec_ret_i64_offset_addr64:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; VI-NEXT: v_mov_b32_e32 v3, 0
; VI-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
; VI-NEXT: s_endpgm
-;
; GFX9-LABEL: flat_atomic_dec_noret_i64_offset_addr64:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_ashrrev_i32_e32 v1, 31, v0
define amdgpu_kernel void @atomic_dec_shl_base_lds_0(i32 addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 {
; CI-LABEL: atomic_dec_shl_base_lds_0:
; CI: ; %bb.0:
-; CI-NEXT: v_mul_lo_u32 v1, 4, v0
+; CI-NEXT: v_mul_lo_u32 v5, 4, v0
; CI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
-; CI-NEXT: v_add_i32_e32 v2, vcc, 2, v0
+; CI-NEXT: v_mov_b32_e32 v6, 9
; CI-NEXT: s_mov_b32 m0, -1
-; CI-NEXT: v_add_i32_e32 v0, vcc, 0, v1
-; CI-NEXT: v_add_i32_e32 v0, vcc, 8, v0
-; CI-NEXT: v_mov_b32_e32 v1, 9
-; CI-NEXT: ds_dec_rtn_u32 v3, v0, v1
+; CI-NEXT: v_add_i32_e32 v5, vcc, 0, v5
+; CI-NEXT: v_add_i32_e32 v5, vcc, 8, v5
+; CI-NEXT: ds_dec_rtn_u32 v5, v5, v6
+; CI-NEXT: v_add_i32_e32 v4, vcc, 2, v0
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v0, s2
+; CI-NEXT: v_mov_b32_e32 v3, s1
; CI-NEXT: v_mov_b32_e32 v1, s3
-; CI-NEXT: flat_store_dword v[0:1], v2
-; CI-NEXT: v_mov_b32_e32 v0, s0
-; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: flat_store_dword v[0:1], v3
+; CI-NEXT: v_mov_b32_e32 v2, s0
+; CI-NEXT: flat_store_dword v[0:1], v4
+; CI-NEXT: flat_store_dword v[2:3], v5
; CI-NEXT: s_endpgm
;
; VI-LABEL: atomic_dec_shl_base_lds_0:
; VI: ; %bb.0:
-; VI-NEXT: v_mul_lo_u32 v1, 4, v0
+; VI-NEXT: v_mul_lo_u32 v5, 4, v0
; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
-; VI-NEXT: v_add_u32_e32 v2, vcc, 2, v0
+; VI-NEXT: v_mov_b32_e32 v6, 9
; VI-NEXT: s_mov_b32 m0, -1
-; VI-NEXT: v_add_u32_e32 v0, vcc, 0, v1
-; VI-NEXT: v_add_u32_e32 v0, vcc, 8, v0
-; VI-NEXT: v_mov_b32_e32 v1, 9
-; VI-NEXT: ds_dec_rtn_u32 v3, v0, v1
+; VI-NEXT: v_add_u32_e32 v5, vcc, 0, v5
+; VI-NEXT: v_add_u32_e32 v5, vcc, 8, v5
+; VI-NEXT: ds_dec_rtn_u32 v5, v5, v6
+; VI-NEXT: v_add_u32_e32 v4, vcc, 2, v0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_mov_b32_e32 v1, s3
-; VI-NEXT: flat_store_dword v[0:1], v2
-; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: flat_store_dword v[0:1], v3
+; VI-NEXT: v_mov_b32_e32 v2, s0
+; VI-NEXT: flat_store_dword v[0:1], v4
+; VI-NEXT: flat_store_dword v[2:3], v5
; VI-NEXT: s_endpgm
-;
; GFX9-LABEL: atomic_dec_shl_base_lds_0:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_mul_lo_u32 v1, 4, v0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_endpgm
-;
; GFX9-LABEL: lds_atomic_dec_ret_i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_endpgm
-;
; GFX9-LABEL: lds_atomic_dec_ret_i64_offset:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1]
; VI-NEXT: s_endpgm
-;
; GFX9-LABEL: lds_atomic_dec_noret_i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s0, s[0:1], 0x24
; VI-NEXT: v_mov_b32_e32 v2, s0
; VI-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1]
; VI-NEXT: s_endpgm
-;
; GFX9-LABEL: lds_atomic_dec_noret_i64_offset:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dword s0, s[0:1], 0x24
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_endpgm
-;
; GFX9-LABEL: global_atomic_dec_ret_i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_endpgm
-;
; GFX9-LABEL: global_atomic_dec_ret_i64_offset:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
; VI-NEXT: s_endpgm
-;
; GFX9-LABEL: global_atomic_dec_noret_i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; VI-NEXT: v_mov_b32_e32 v1, s1
; VI-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
; VI-NEXT: s_endpgm
-;
; GFX9-LABEL: global_atomic_dec_noret_i64_offset:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x24
; VI-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
; VI-NEXT: s_endpgm
-;
; GFX9-LABEL: global_atomic_dec_ret_i64_offset_addr64:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; VI-NEXT: v_mov_b32_e32 v3, 0
; VI-NEXT: flat_atomic_dec_x2 v[0:1], v[0:1], v[2:3] glc
; VI-NEXT: s_endpgm
-;
; GFX9-LABEL: global_atomic_dec_noret_i64_offset_addr64:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_ashrrev_i32_e32 v1, 31, v0
define amdgpu_kernel void @atomic_dec_shl_base_lds_0_i64(i64 addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 {
; CI-LABEL: atomic_dec_shl_base_lds_0_i64:
; CI: ; %bb.0:
-; CI-NEXT: v_mul_lo_u32 v1, 8, v0
+; CI-NEXT: v_mul_lo_u32 v7, 8, v0
; CI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
-; CI-NEXT: v_add_i32_e32 v4, vcc, 2, v0
-; CI-NEXT: s_mov_b32 m0, -1
-; CI-NEXT: v_add_i32_e32 v0, vcc, 0, v1
-; CI-NEXT: v_add_i32_e32 v2, vcc, 16, v0
+; CI-NEXT: v_add_i32_e32 v6, vcc, 2, v0
; CI-NEXT: v_mov_b32_e32 v0, 9
+; CI-NEXT: v_add_i32_e32 v7, vcc, 0, v7
; CI-NEXT: v_mov_b32_e32 v1, 0
-; CI-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1]
+; CI-NEXT: v_add_i32_e32 v7, vcc, 16, v7
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: ds_dec_rtn_u64 v[0:1], v7, v[0:1]
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v2, s2
+; CI-NEXT: v_mov_b32_e32 v5, s1
; CI-NEXT: v_mov_b32_e32 v3, s3
-; CI-NEXT: flat_store_dword v[2:3], v4
-; CI-NEXT: v_mov_b32_e32 v3, s1
-; CI-NEXT: v_mov_b32_e32 v2, s0
-; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; CI-NEXT: v_mov_b32_e32 v4, s0
+; CI-NEXT: flat_store_dword v[2:3], v6
+; CI-NEXT: flat_store_dwordx2 v[4:5], v[0:1]
; CI-NEXT: s_endpgm
;
; VI-LABEL: atomic_dec_shl_base_lds_0_i64:
; VI: ; %bb.0:
-; VI-NEXT: v_mul_lo_u32 v1, 8, v0
+; VI-NEXT: v_mul_lo_u32 v7, 8, v0
; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
-; VI-NEXT: v_add_u32_e32 v4, vcc, 2, v0
-; VI-NEXT: s_mov_b32 m0, -1
-; VI-NEXT: v_add_u32_e32 v0, vcc, 0, v1
-; VI-NEXT: v_add_u32_e32 v2, vcc, 16, v0
+; VI-NEXT: v_add_u32_e32 v6, vcc, 2, v0
; VI-NEXT: v_mov_b32_e32 v0, 9
+; VI-NEXT: v_add_u32_e32 v7, vcc, 0, v7
; VI-NEXT: v_mov_b32_e32 v1, 0
-; VI-NEXT: ds_dec_rtn_u64 v[0:1], v2, v[0:1]
+; VI-NEXT: v_add_u32_e32 v7, vcc, 16, v7
+; VI-NEXT: s_mov_b32 m0, -1
+; VI-NEXT: ds_dec_rtn_u64 v[0:1], v7, v[0:1]
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v5, s1
; VI-NEXT: v_mov_b32_e32 v3, s3
-; VI-NEXT: flat_store_dword v[2:3], v4
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: v_mov_b32_e32 v4, s0
+; VI-NEXT: flat_store_dword v[2:3], v6
+; VI-NEXT: flat_store_dwordx2 v[4:5], v[0:1]
; VI-NEXT: s_endpgm
-;
; GFX9-LABEL: atomic_dec_shl_base_lds_0_i64:
; GFX9: ; %bb.0:
; GFX9-NEXT: v_mul_lo_u32 v3, 8, v0
define amdgpu_kernel void @atomic_inc_shl_base_lds_0_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 {
; CI-LABEL: atomic_inc_shl_base_lds_0_i32:
; CI: ; %bb.0:
-; CI-NEXT: v_mul_lo_u32 v1, 4, v0
+; CI-NEXT: v_mul_lo_u32 v5, 4, v0
; CI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
-; CI-NEXT: v_add_i32_e32 v2, vcc, 2, v0
+; CI-NEXT: v_mov_b32_e32 v6, 9
; CI-NEXT: s_mov_b32 m0, -1
-; CI-NEXT: v_add_i32_e32 v0, vcc, 0, v1
-; CI-NEXT: v_add_i32_e32 v0, vcc, 8, v0
-; CI-NEXT: v_mov_b32_e32 v1, 9
-; CI-NEXT: ds_inc_rtn_u32 v3, v0, v1
+; CI-NEXT: v_add_i32_e32 v5, vcc, 0, v5
+; CI-NEXT: v_add_i32_e32 v5, vcc, 8, v5
+; CI-NEXT: ds_inc_rtn_u32 v5, v5, v6
+; CI-NEXT: v_add_i32_e32 v4, vcc, 2, v0
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v0, s2
+; CI-NEXT: v_mov_b32_e32 v3, s1
; CI-NEXT: v_mov_b32_e32 v1, s3
-; CI-NEXT: flat_store_dword v[0:1], v2
-; CI-NEXT: v_mov_b32_e32 v0, s0
-; CI-NEXT: v_mov_b32_e32 v1, s1
-; CI-NEXT: flat_store_dword v[0:1], v3
+; CI-NEXT: v_mov_b32_e32 v2, s0
+; CI-NEXT: flat_store_dword v[0:1], v4
+; CI-NEXT: flat_store_dword v[2:3], v5
; CI-NEXT: s_endpgm
;
; VI-LABEL: atomic_inc_shl_base_lds_0_i32:
; VI: ; %bb.0:
-; VI-NEXT: v_mul_lo_u32 v1, 4, v0
+; VI-NEXT: v_mul_lo_u32 v5, 4, v0
; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
-; VI-NEXT: v_add_u32_e32 v2, vcc, 2, v0
+; VI-NEXT: v_mov_b32_e32 v6, 9
; VI-NEXT: s_mov_b32 m0, -1
-; VI-NEXT: v_add_u32_e32 v0, vcc, 0, v1
-; VI-NEXT: v_add_u32_e32 v0, vcc, 8, v0
-; VI-NEXT: v_mov_b32_e32 v1, 9
-; VI-NEXT: ds_inc_rtn_u32 v3, v0, v1
+; VI-NEXT: v_add_u32_e32 v5, vcc, 0, v5
+; VI-NEXT: v_add_u32_e32 v5, vcc, 8, v5
+; VI-NEXT: ds_inc_rtn_u32 v5, v5, v6
+; VI-NEXT: v_add_u32_e32 v4, vcc, 2, v0
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v0, s2
+; VI-NEXT: v_mov_b32_e32 v3, s1
; VI-NEXT: v_mov_b32_e32 v1, s3
-; VI-NEXT: flat_store_dword v[0:1], v2
-; VI-NEXT: v_mov_b32_e32 v0, s0
-; VI-NEXT: v_mov_b32_e32 v1, s1
-; VI-NEXT: flat_store_dword v[0:1], v3
+; VI-NEXT: v_mov_b32_e32 v2, s0
+; VI-NEXT: flat_store_dword v[0:1], v4
+; VI-NEXT: flat_store_dword v[2:3], v5
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_inc_shl_base_lds_0_i32:
define amdgpu_kernel void @atomic_inc_shl_base_lds_0_i64(i64 addrspace(1)* %out, i32 addrspace(1)* %add_use) #0 {
; CI-LABEL: atomic_inc_shl_base_lds_0_i64:
; CI: ; %bb.0:
-; CI-NEXT: v_mul_lo_u32 v1, 8, v0
+; CI-NEXT: v_mul_lo_u32 v7, 8, v0
; CI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x9
-; CI-NEXT: v_add_i32_e32 v4, vcc, 2, v0
-; CI-NEXT: s_mov_b32 m0, -1
-; CI-NEXT: v_add_i32_e32 v0, vcc, 0, v1
-; CI-NEXT: v_add_i32_e32 v2, vcc, 16, v0
+; CI-NEXT: v_add_i32_e32 v6, vcc, 2, v0
; CI-NEXT: v_mov_b32_e32 v0, 9
+; CI-NEXT: v_add_i32_e32 v7, vcc, 0, v7
; CI-NEXT: v_mov_b32_e32 v1, 0
-; CI-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1]
+; CI-NEXT: v_add_i32_e32 v7, vcc, 16, v7
+; CI-NEXT: s_mov_b32 m0, -1
+; CI-NEXT: ds_inc_rtn_u64 v[0:1], v7, v[0:1]
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_mov_b32_e32 v2, s2
+; CI-NEXT: v_mov_b32_e32 v5, s1
; CI-NEXT: v_mov_b32_e32 v3, s3
-; CI-NEXT: flat_store_dword v[2:3], v4
-; CI-NEXT: v_mov_b32_e32 v3, s1
-; CI-NEXT: v_mov_b32_e32 v2, s0
-; CI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; CI-NEXT: v_mov_b32_e32 v4, s0
+; CI-NEXT: flat_store_dword v[2:3], v6
+; CI-NEXT: flat_store_dwordx2 v[4:5], v[0:1]
; CI-NEXT: s_endpgm
;
; VI-LABEL: atomic_inc_shl_base_lds_0_i64:
; VI: ; %bb.0:
-; VI-NEXT: v_mul_lo_u32 v1, 8, v0
+; VI-NEXT: v_mul_lo_u32 v7, 8, v0
; VI-NEXT: s_load_dwordx4 s[0:3], s[0:1], 0x24
-; VI-NEXT: v_add_u32_e32 v4, vcc, 2, v0
-; VI-NEXT: s_mov_b32 m0, -1
-; VI-NEXT: v_add_u32_e32 v0, vcc, 0, v1
-; VI-NEXT: v_add_u32_e32 v2, vcc, 16, v0
+; VI-NEXT: v_add_u32_e32 v6, vcc, 2, v0
; VI-NEXT: v_mov_b32_e32 v0, 9
+; VI-NEXT: v_add_u32_e32 v7, vcc, 0, v7
; VI-NEXT: v_mov_b32_e32 v1, 0
-; VI-NEXT: ds_inc_rtn_u64 v[0:1], v2, v[0:1]
+; VI-NEXT: v_add_u32_e32 v7, vcc, 16, v7
+; VI-NEXT: s_mov_b32 m0, -1
+; VI-NEXT: ds_inc_rtn_u64 v[0:1], v7, v[0:1]
; VI-NEXT: s_waitcnt lgkmcnt(0)
; VI-NEXT: v_mov_b32_e32 v2, s2
+; VI-NEXT: v_mov_b32_e32 v5, s1
; VI-NEXT: v_mov_b32_e32 v3, s3
-; VI-NEXT: flat_store_dword v[2:3], v4
-; VI-NEXT: v_mov_b32_e32 v3, s1
-; VI-NEXT: v_mov_b32_e32 v2, s0
-; VI-NEXT: flat_store_dwordx2 v[2:3], v[0:1]
+; VI-NEXT: v_mov_b32_e32 v4, s0
+; VI-NEXT: flat_store_dword v[2:3], v6
+; VI-NEXT: flat_store_dwordx2 v[4:5], v[0:1]
; VI-NEXT: s_endpgm
;
; GFX9-LABEL: atomic_inc_shl_base_lds_0_i64: