drivers/infiniband/hw/hns/hns_roce_hem.c

   1 /*
   2  * Copyright (c) 2016 Hisilicon Limited.
   3  * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
   4  *
   5  * This software is available to you under a choice of one of two
   6  * licenses.  You may choose to be licensed under the terms of the GNU
   7  * General Public License (GPL) Version 2, available from the file
   8  * COPYING in the main directory of this source tree, or the
   9  * OpenIB.org BSD license below:
  10  *
  11  *     Redistribution and use in source and binary forms, with or
  12  *     without modification, are permitted provided that the following
  13  *     conditions are met:
  14  *
  15  *      - Redistributions of source code must retain the above
  16  *        copyright notice, this list of conditions and the following
  17  *        disclaimer.
  18  *
  19  *      - Redistributions in binary form must reproduce the above
  20  *        copyright notice, this list of conditions and the following
  21  *        disclaimer in the documentation and/or other materials
  22  *        provided with the distribution.
  23  *
  24  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31  * SOFTWARE.
  32  */
  33
  34 #include <linux/platform_device.h>
  35 #include "hns_roce_device.h"
  36 #include "hns_roce_hem.h"
  37 #include "hns_roce_common.h"
  38
  39 #define DMA_ADDR_T_SHIFT                12
  40 #define BT_BA_SHIFT                     32
  41
  42 bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type)
  43 {
  44         if ((hr_dev->caps.qpc_hop_num && type == HEM_TYPE_QPC) ||
  45             (hr_dev->caps.mpt_hop_num && type == HEM_TYPE_MTPT) ||
  46             (hr_dev->caps.cqc_hop_num && type == HEM_TYPE_CQC) ||
  47             (hr_dev->caps.srqc_hop_num && type == HEM_TYPE_SRQC) ||
  48             (hr_dev->caps.sccc_hop_num && type == HEM_TYPE_SCCC) ||
  49             (hr_dev->caps.qpc_timer_hop_num && type == HEM_TYPE_QPC_TIMER) ||
  50             (hr_dev->caps.cqc_timer_hop_num && type == HEM_TYPE_CQC_TIMER) ||
  51             (hr_dev->caps.cqe_hop_num && type == HEM_TYPE_CQE) ||
  52             (hr_dev->caps.mtt_hop_num && type == HEM_TYPE_MTT) ||
  53             (hr_dev->caps.srqwqe_hop_num && type == HEM_TYPE_SRQWQE) ||
  54             (hr_dev->caps.idx_hop_num && type == HEM_TYPE_IDX))
  55                 return true;
  56
  57         return false;
  58 }
  59 EXPORT_SYMBOL_GPL(hns_roce_check_whether_mhop);
  60
  61 static bool hns_roce_check_hem_null(struct hns_roce_hem **hem, u64 start_idx,
  62                             u32 bt_chunk_num)
  63 {
  64         int i;
  65
  66         for (i = 0; i < bt_chunk_num; i++)
  67                 if (hem[start_idx + i])
  68                         return false;
  69
  70         return true;
  71 }
  72
  73 static bool hns_roce_check_bt_null(u64 **bt, u64 start_idx, u32 bt_chunk_num)
  74 {
  75         int i;
  76
  77         for (i = 0; i < bt_chunk_num; i++)
  78                 if (bt[start_idx + i])
  79                         return false;
  80
  81         return true;
  82 }
  83
  84 static int hns_roce_get_bt_num(u32 table_type, u32 hop_num)
  85 {
  86         if (check_whether_bt_num_3(table_type, hop_num))
  87                 return 3;
  88         else if (check_whether_bt_num_2(table_type, hop_num))
  89                 return 2;
  90         else if (check_whether_bt_num_1(table_type, hop_num))
  91                 return 1;
  92         else
  93                 return 0;
  94 }
  95
  96 int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev,
  97                            struct hns_roce_hem_table *table, unsigned long *obj,
  98                            struct hns_roce_hem_mhop *mhop)
  99 {
 100         struct device *dev = hr_dev->dev;
 101         u32 chunk_ba_num;
 102         u32 table_idx;
 103         u32 bt_num;
 104         u32 chunk_size;
 105
 106         switch (table->type) {
 107         case HEM_TYPE_QPC:
 108                 mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz
 109                                              + PAGE_SHIFT);
 110                 mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_ba_pg_sz
 111                                              + PAGE_SHIFT);
 112                 mhop->ba_l0_num = hr_dev->caps.qpc_bt_num;
 113                 mhop->hop_num = hr_dev->caps.qpc_hop_num;
 114                 break;
 115         case HEM_TYPE_MTPT:
 116                 mhop->buf_chunk_size = 1 << (hr_dev->caps.mpt_buf_pg_sz
 117                                              + PAGE_SHIFT);
 118                 mhop->bt_chunk_size = 1 << (hr_dev->caps.mpt_ba_pg_sz
 119                                              + PAGE_SHIFT);
 120                 mhop->ba_l0_num = hr_dev->caps.mpt_bt_num;
 121                 mhop->hop_num = hr_dev->caps.mpt_hop_num;
 122                 break;
 123         case HEM_TYPE_CQC:
 124                 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_buf_pg_sz
 125                                              + PAGE_SHIFT);
 126                 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_ba_pg_sz
 127                                             + PAGE_SHIFT);
 128                 mhop->ba_l0_num = hr_dev->caps.cqc_bt_num;
 129                 mhop->hop_num = hr_dev->caps.cqc_hop_num;
 130                 break;
 131         case HEM_TYPE_SCCC:
 132                 mhop->buf_chunk_size = 1 << (hr_dev->caps.sccc_buf_pg_sz
 133                                              + PAGE_SHIFT);
 134                 mhop->bt_chunk_size = 1 << (hr_dev->caps.sccc_ba_pg_sz
 135                                             + PAGE_SHIFT);
 136                 mhop->ba_l0_num = hr_dev->caps.sccc_bt_num;
 137                 mhop->hop_num = hr_dev->caps.sccc_hop_num;
 138                 break;
 139         case HEM_TYPE_QPC_TIMER:
 140                 mhop->buf_chunk_size = 1 << (hr_dev->caps.qpc_timer_buf_pg_sz
 141                                              + PAGE_SHIFT);
 142                 mhop->bt_chunk_size = 1 << (hr_dev->caps.qpc_timer_ba_pg_sz
 143                                             + PAGE_SHIFT);
 144                 mhop->ba_l0_num = hr_dev->caps.qpc_timer_bt_num;
 145                 mhop->hop_num = hr_dev->caps.qpc_timer_hop_num;
 146                 break;
 147         case HEM_TYPE_CQC_TIMER:
 148                 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqc_timer_buf_pg_sz
 149                                              + PAGE_SHIFT);
 150                 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqc_timer_ba_pg_sz
 151                                             + PAGE_SHIFT);
 152                 mhop->ba_l0_num = hr_dev->caps.cqc_timer_bt_num;
 153                 mhop->hop_num = hr_dev->caps.cqc_timer_hop_num;
 154                 break;
 155         case HEM_TYPE_SRQC:
 156                 mhop->buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz
 157                                              + PAGE_SHIFT);
 158                 mhop->bt_chunk_size = 1 << (hr_dev->caps.srqc_ba_pg_sz
 159                                              + PAGE_SHIFT);
 160                 mhop->ba_l0_num = hr_dev->caps.srqc_bt_num;
 161                 mhop->hop_num = hr_dev->caps.srqc_hop_num;
 162                 break;
 163         case HEM_TYPE_MTT:
 164                 mhop->buf_chunk_size = 1 << (hr_dev->caps.mtt_buf_pg_sz
 165                                              + PAGE_SHIFT);
 166                 mhop->bt_chunk_size = 1 << (hr_dev->caps.mtt_ba_pg_sz
 167                                              + PAGE_SHIFT);
 168                 mhop->ba_l0_num = mhop->bt_chunk_size / 8;
 169                 mhop->hop_num = hr_dev->caps.mtt_hop_num;
 170                 break;
 171         case HEM_TYPE_CQE:
 172                 mhop->buf_chunk_size = 1 << (hr_dev->caps.cqe_buf_pg_sz
 173                                              + PAGE_SHIFT);
 174                 mhop->bt_chunk_size = 1 << (hr_dev->caps.cqe_ba_pg_sz
 175                                              + PAGE_SHIFT);
 176                 mhop->ba_l0_num = mhop->bt_chunk_size / 8;
 177                 mhop->hop_num = hr_dev->caps.cqe_hop_num;
 178                 break;
 179         case HEM_TYPE_SRQWQE:
 180                 mhop->buf_chunk_size = 1 << (hr_dev->caps.srqwqe_buf_pg_sz
 181                                             + PAGE_SHIFT);
 182                 mhop->bt_chunk_size = 1 << (hr_dev->caps.srqwqe_ba_pg_sz
 183                                             + PAGE_SHIFT);
 184                 mhop->ba_l0_num = mhop->bt_chunk_size / 8;
 185                 mhop->hop_num = hr_dev->caps.srqwqe_hop_num;
 186                 break;
 187         case HEM_TYPE_IDX:
 188                 mhop->buf_chunk_size = 1 << (hr_dev->caps.idx_buf_pg_sz
 189                                        + PAGE_SHIFT);
 190                 mhop->bt_chunk_size = 1 << (hr_dev->caps.idx_ba_pg_sz
 191                                        + PAGE_SHIFT);
 192                 mhop->ba_l0_num = mhop->bt_chunk_size / 8;
 193                 mhop->hop_num = hr_dev->caps.idx_hop_num;
 194                 break;
 195         default:
 196                 dev_err(dev, "Table %d not support multi-hop addressing!\n",
 197                          table->type);
 198                 return -EINVAL;
 199         }
 200
 201         if (!obj)
 202                 return 0;
 203
 204         /*
 205          * QPC/MTPT/CQC/SRQC/SCCC alloc hem for buffer pages.
 206          * MTT/CQE alloc hem for bt pages.
 207          */
 208         bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num);
 209         chunk_ba_num = mhop->bt_chunk_size / 8;
 210         chunk_size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size :
 211                               mhop->bt_chunk_size;
 212         table_idx = (*obj & (table->num_obj - 1)) /
 213                      (chunk_size / table->obj_size);
 214         switch (bt_num) {
 215         case 3:
 216                 mhop->l2_idx = table_idx & (chunk_ba_num - 1);
 217                 mhop->l1_idx = table_idx / chunk_ba_num & (chunk_ba_num - 1);
 218                 mhop->l0_idx = (table_idx / chunk_ba_num) / chunk_ba_num;
 219                 break;
 220         case 2:
 221                 mhop->l1_idx = table_idx & (chunk_ba_num - 1);
 222                 mhop->l0_idx = table_idx / chunk_ba_num;
 223                 break;
 224         case 1:
 225                 mhop->l0_idx = table_idx;
 226                 break;
 227         default:
 228                 dev_err(dev, "Table %d not support hop_num = %d!\n",
 229                              table->type, mhop->hop_num);
 230                 return -EINVAL;
 231         }
 232         if (mhop->l0_idx >= mhop->ba_l0_num)
 233                 mhop->l0_idx %= mhop->ba_l0_num;
 234
 235         return 0;
 236 }
 237 EXPORT_SYMBOL_GPL(hns_roce_calc_hem_mhop);
 238
 239 static struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev,
 240                                                int npages,
 241                                                unsigned long hem_alloc_size,
 242                                                gfp_t gfp_mask)
 243 {
 244         struct hns_roce_hem_chunk *chunk = NULL;
 245         struct hns_roce_hem *hem;
 246         struct scatterlist *mem;
 247         int order;
 248         void *buf;
 249
 250         WARN_ON(gfp_mask & __GFP_HIGHMEM);
 251
 252         hem = kmalloc(sizeof(*hem),
 253                       gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
 254         if (!hem)
 255                 return NULL;
 256
 257         hem->refcount = 0;
 258         INIT_LIST_HEAD(&hem->chunk_list);
 259
 260         order = get_order(hem_alloc_size);
 261
 262         while (npages > 0) {
 263                 if (!chunk) {
 264                         chunk = kmalloc(sizeof(*chunk),
 265                                 gfp_mask & ~(__GFP_HIGHMEM | __GFP_NOWARN));
 266                         if (!chunk)
 267                                 goto fail;
 268
 269                         sg_init_table(chunk->mem, HNS_ROCE_HEM_CHUNK_LEN);
 270                         chunk->npages = 0;
 271                         chunk->nsg = 0;
 272                         memset(chunk->buf, 0, sizeof(chunk->buf));
 273                         list_add_tail(&chunk->list, &hem->chunk_list);
 274                 }
 275
 276                 while (1 << order > npages)
 277                         --order;
 278
 279                 /*
 280                  * Alloc memory one time. If failed, don't alloc small block
 281                  * memory, directly return fail.
 282                  */
 283                 mem = &chunk->mem[chunk->npages];
 284                 buf = dma_alloc_coherent(hr_dev->dev, PAGE_SIZE << order,
 285                                 &sg_dma_address(mem), gfp_mask);
 286                 if (!buf)
 287                         goto fail;
 288
 289                 chunk->buf[chunk->npages] = buf;
 290                 sg_dma_len(mem) = PAGE_SIZE << order;
 291
 292                 ++chunk->npages;
 293                 ++chunk->nsg;
 294                 npages -= 1 << order;
 295         }
 296
 297         return hem;
 298
 299 fail:
 300         hns_roce_free_hem(hr_dev, hem);
 301         return NULL;
 302 }
 303
 304 void hns_roce_free_hem(struct hns_roce_dev *hr_dev, struct hns_roce_hem *hem)
 305 {
 306         struct hns_roce_hem_chunk *chunk, *tmp;
 307         int i;
 308
 309         if (!hem)
 310                 return;
 311
 312         list_for_each_entry_safe(chunk, tmp, &hem->chunk_list, list) {
 313                 for (i = 0; i < chunk->npages; ++i)
 314                         dma_free_coherent(hr_dev->dev,
 315                                    sg_dma_len(&chunk->mem[i]),
 316                                    chunk->buf[i],
 317                                    sg_dma_address(&chunk->mem[i]));
 318                 kfree(chunk);
 319         }
 320
 321         kfree(hem);
 322 }
 323
 324 static int hns_roce_set_hem(struct hns_roce_dev *hr_dev,
 325                             struct hns_roce_hem_table *table, unsigned long obj)
 326 {
 327         spinlock_t *lock = &hr_dev->bt_cmd_lock;
 328         struct device *dev = hr_dev->dev;
 329         unsigned long end = 0;
 330         unsigned long flags;
 331         struct hns_roce_hem_iter iter;
 332         void __iomem *bt_cmd;
 333         u32 bt_cmd_h_val = 0;
 334         u32 bt_cmd_val[2];
 335         u32 bt_cmd_l = 0;
 336         u64 bt_ba = 0;
 337         int ret = 0;
 338
 339         /* Find the HEM(Hardware Entry Memory) entry */
 340         unsigned long i = (obj & (table->num_obj - 1)) /
 341                           (table->table_chunk_size / table->obj_size);
 342
 343         switch (table->type) {
 344         case HEM_TYPE_QPC:
 345                 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
 346                                ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_QPC);
 347                 break;
 348         case HEM_TYPE_MTPT:
 349                 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
 350                                ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S,
 351                                HEM_TYPE_MTPT);
 352                 break;
 353         case HEM_TYPE_CQC:
 354                 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
 355                                ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S, HEM_TYPE_CQC);
 356                 break;
 357         case HEM_TYPE_SRQC:
 358                 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_M,
 359                                ROCEE_BT_CMD_H_ROCEE_BT_CMD_MDF_S,
 360                                HEM_TYPE_SRQC);
 361                 break;
 362         default:
 363                 return ret;
 364         }
 365         roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_M,
 366                        ROCEE_BT_CMD_H_ROCEE_BT_CMD_IN_MDF_S, obj);
 367         roce_set_bit(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_S, 0);
 368         roce_set_bit(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_HW_SYNS_S, 1);
 369
 370         /* Currently iter only a chunk */
 371         for (hns_roce_hem_first(table->hem[i], &iter);
 372              !hns_roce_hem_last(&iter); hns_roce_hem_next(&iter)) {
 373                 bt_ba = hns_roce_hem_addr(&iter) >> DMA_ADDR_T_SHIFT;
 374
 375                 spin_lock_irqsave(lock, flags);
 376
 377                 bt_cmd = hr_dev->reg_base + ROCEE_BT_CMD_H_REG;
 378
 379                 end = msecs_to_jiffies(HW_SYNC_TIMEOUT_MSECS) + jiffies;
 380                 while (1) {
 381                         if (readl(bt_cmd) >> BT_CMD_SYNC_SHIFT) {
 382                                 if (!(time_before(jiffies, end))) {
 383                                         dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n");
 384                                         spin_unlock_irqrestore(lock, flags);
 385                                         return -EBUSY;
 386                                 }
 387                         } else {
 388                                 break;
 389                         }
 390                         mdelay(HW_SYNC_SLEEP_TIME_INTERVAL);
 391                 }
 392
 393                 bt_cmd_l = (u32)bt_ba;
 394                 roce_set_field(bt_cmd_h_val, ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_M,
 395                                ROCEE_BT_CMD_H_ROCEE_BT_CMD_BA_H_S,
 396                                bt_ba >> BT_BA_SHIFT);
 397
 398                 bt_cmd_val[0] = bt_cmd_l;
 399                 bt_cmd_val[1] = bt_cmd_h_val;
 400                 hns_roce_write64_k(bt_cmd_val,
 401                                    hr_dev->reg_base + ROCEE_BT_CMD_L_REG);
 402                 spin_unlock_irqrestore(lock, flags);
 403         }
 404
 405         return ret;
 406 }
 407
 408 static int hns_roce_table_mhop_get(struct hns_roce_dev *hr_dev,
 409                                    struct hns_roce_hem_table *table,
 410                                    unsigned long obj)
 411 {
 412         struct device *dev = hr_dev->dev;
 413         struct hns_roce_hem_mhop mhop;
 414         struct hns_roce_hem_iter iter;
 415         u32 buf_chunk_size;
 416         u32 bt_chunk_size;
 417         u32 chunk_ba_num;
 418         u32 hop_num;
 419         u32 size;
 420         u32 bt_num;
 421         u64 hem_idx;
 422         u64 bt_l1_idx = 0;
 423         u64 bt_l0_idx = 0;
 424         u64 bt_ba;
 425         unsigned long mhop_obj = obj;
 426         int bt_l1_allocated = 0;
 427         int bt_l0_allocated = 0;
 428         int step_idx;
 429         int ret;
 430
 431         ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);
 432         if (ret)
 433                 return ret;
 434
 435         buf_chunk_size = mhop.buf_chunk_size;
 436         bt_chunk_size = mhop.bt_chunk_size;
 437         hop_num = mhop.hop_num;
 438         chunk_ba_num = bt_chunk_size / 8;
 439
 440         bt_num = hns_roce_get_bt_num(table->type, hop_num);
 441         switch (bt_num) {
 442         case 3:
 443                 hem_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num +
 444                           mhop.l1_idx * chunk_ba_num + mhop.l2_idx;
 445                 bt_l1_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;
 446                 bt_l0_idx = mhop.l0_idx;
 447                 break;
 448         case 2:
 449                 hem_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;
 450                 bt_l0_idx = mhop.l0_idx;
 451                 break;
 452         case 1:
 453                 hem_idx = mhop.l0_idx;
 454                 break;
 455         default:
 456                 dev_err(dev, "Table %d not support hop_num = %d!\n",
 457                              table->type, hop_num);
 458                 return -EINVAL;
 459         }
 460
 461         mutex_lock(&table->mutex);
 462
 463         if (table->hem[hem_idx]) {
 464                 ++table->hem[hem_idx]->refcount;
 465                 goto out;
 466         }
 467
 468         /* alloc L1 BA's chunk */
 469         if ((check_whether_bt_num_3(table->type, hop_num) ||
 470                 check_whether_bt_num_2(table->type, hop_num)) &&
 471                 !table->bt_l0[bt_l0_idx]) {
 472                 table->bt_l0[bt_l0_idx] = dma_alloc_coherent(dev, bt_chunk_size,
 473                                             &(table->bt_l0_dma_addr[bt_l0_idx]),
 474                                             GFP_KERNEL);
 475                 if (!table->bt_l0[bt_l0_idx]) {
 476                         ret = -ENOMEM;
 477                         goto out;
 478                 }
 479                 bt_l0_allocated = 1;
 480
 481                 /* set base address to hardware */
 482                 if (table->type < HEM_TYPE_MTT) {
 483                         step_idx = 0;
 484                         if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) {
 485                                 ret = -ENODEV;
 486                                 dev_err(dev, "set HEM base address to HW failed!\n");
 487                                 goto err_dma_alloc_l1;
 488                         }
 489                 }
 490         }
 491
 492         /* alloc L2 BA's chunk */
 493         if (check_whether_bt_num_3(table->type, hop_num) &&
 494             !table->bt_l1[bt_l1_idx])  {
 495                 table->bt_l1[bt_l1_idx] = dma_alloc_coherent(dev, bt_chunk_size,
 496                                             &(table->bt_l1_dma_addr[bt_l1_idx]),
 497                                             GFP_KERNEL);
 498                 if (!table->bt_l1[bt_l1_idx]) {
 499                         ret = -ENOMEM;
 500                         goto err_dma_alloc_l1;
 501                 }
 502                 bt_l1_allocated = 1;
 503                 *(table->bt_l0[bt_l0_idx] + mhop.l1_idx) =
 504                                                table->bt_l1_dma_addr[bt_l1_idx];
 505
 506                 /* set base address to hardware */
 507                 step_idx = 1;
 508                 if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) {
 509                         ret = -ENODEV;
 510                         dev_err(dev, "set HEM base address to HW failed!\n");
 511                         goto err_alloc_hem_buf;
 512                 }
 513         }
 514
 515         /*
 516          * alloc buffer space chunk for QPC/MTPT/CQC/SRQC/SCCC.
 517          * alloc bt space chunk for MTT/CQE.
 518          */
 519         size = table->type < HEM_TYPE_MTT ? buf_chunk_size : bt_chunk_size;
 520         table->hem[hem_idx] = hns_roce_alloc_hem(hr_dev,
 521                                                 size >> PAGE_SHIFT,
 522                                                 size,
 523                                                 (table->lowmem ? GFP_KERNEL :
 524                                                 GFP_HIGHUSER) | __GFP_NOWARN);
 525         if (!table->hem[hem_idx]) {
 526                 ret = -ENOMEM;
 527                 goto err_alloc_hem_buf;
 528         }
 529
 530         hns_roce_hem_first(table->hem[hem_idx], &iter);
 531         bt_ba = hns_roce_hem_addr(&iter);
 532
 533         if (table->type < HEM_TYPE_MTT) {
 534                 if (hop_num == 2) {
 535                         *(table->bt_l1[bt_l1_idx] + mhop.l2_idx) = bt_ba;
 536                         step_idx = 2;
 537                 } else if (hop_num == 1) {
 538                         *(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = bt_ba;
 539                         step_idx = 1;
 540                 } else if (hop_num == HNS_ROCE_HOP_NUM_0) {
 541                         step_idx = 0;
 542                 } else {
 543                         ret = -EINVAL;
 544                         goto err_dma_alloc_l1;
 545                 }
 546
 547                 /* set HEM base address to hardware */
 548                 if (hr_dev->hw->set_hem(hr_dev, table, obj, step_idx)) {
 549                         ret = -ENODEV;
 550                         dev_err(dev, "set HEM base address to HW failed!\n");
 551                         goto err_alloc_hem_buf;
 552                 }
 553         } else if (hop_num == 2) {
 554                 *(table->bt_l0[bt_l0_idx] + mhop.l1_idx) = bt_ba;
 555         }
 556
 557         ++table->hem[hem_idx]->refcount;
 558         goto out;
 559
 560 err_alloc_hem_buf:
 561         if (bt_l1_allocated) {
 562                 dma_free_coherent(dev, bt_chunk_size, table->bt_l1[bt_l1_idx],
 563                                   table->bt_l1_dma_addr[bt_l1_idx]);
 564                 table->bt_l1[bt_l1_idx] = NULL;
 565         }
 566
 567 err_dma_alloc_l1:
 568         if (bt_l0_allocated) {
 569                 dma_free_coherent(dev, bt_chunk_size, table->bt_l0[bt_l0_idx],
 570                                   table->bt_l0_dma_addr[bt_l0_idx]);
 571                 table->bt_l0[bt_l0_idx] = NULL;
 572         }
 573
 574 out:
 575         mutex_unlock(&table->mutex);
 576         return ret;
 577 }
 578
 579 int hns_roce_table_get(struct hns_roce_dev *hr_dev,
 580                        struct hns_roce_hem_table *table, unsigned long obj)
 581 {
 582         struct device *dev = hr_dev->dev;
 583         int ret = 0;
 584         unsigned long i;
 585
 586         if (hns_roce_check_whether_mhop(hr_dev, table->type))
 587                 return hns_roce_table_mhop_get(hr_dev, table, obj);
 588
 589         i = (obj & (table->num_obj - 1)) / (table->table_chunk_size /
 590              table->obj_size);
 591
 592         mutex_lock(&table->mutex);
 593
 594         if (table->hem[i]) {
 595                 ++table->hem[i]->refcount;
 596                 goto out;
 597         }
 598
 599         table->hem[i] = hns_roce_alloc_hem(hr_dev,
 600                                        table->table_chunk_size >> PAGE_SHIFT,
 601                                        table->table_chunk_size,
 602                                        (table->lowmem ? GFP_KERNEL :
 603                                         GFP_HIGHUSER) | __GFP_NOWARN);
 604         if (!table->hem[i]) {
 605                 ret = -ENOMEM;
 606                 goto out;
 607         }
 608
 609         /* Set HEM base address(128K/page, pa) to Hardware */
 610         if (hns_roce_set_hem(hr_dev, table, obj)) {
 611                 hns_roce_free_hem(hr_dev, table->hem[i]);
 612                 table->hem[i] = NULL;
 613                 ret = -ENODEV;
 614                 dev_err(dev, "set HEM base address to HW failed.\n");
 615                 goto out;
 616         }
 617
 618         ++table->hem[i]->refcount;
 619 out:
 620         mutex_unlock(&table->mutex);
 621         return ret;
 622 }
 623 EXPORT_SYMBOL_GPL(hns_roce_table_get);
 624
 625 static void hns_roce_table_mhop_put(struct hns_roce_dev *hr_dev,
 626                                     struct hns_roce_hem_table *table,
 627                                     unsigned long obj,
 628                                     int check_refcount)
 629 {
 630         struct device *dev = hr_dev->dev;
 631         struct hns_roce_hem_mhop mhop;
 632         unsigned long mhop_obj = obj;
 633         u32 bt_chunk_size;
 634         u32 chunk_ba_num;
 635         u32 hop_num;
 636         u32 start_idx;
 637         u32 bt_num;
 638         u64 hem_idx;
 639         u64 bt_l1_idx = 0;
 640         int ret;
 641
 642         ret = hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);
 643         if (ret)
 644                 return;
 645
 646         bt_chunk_size = mhop.bt_chunk_size;
 647         hop_num = mhop.hop_num;
 648         chunk_ba_num = bt_chunk_size / 8;
 649
 650         bt_num = hns_roce_get_bt_num(table->type, hop_num);
 651         switch (bt_num) {
 652         case 3:
 653                 hem_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num +
 654                           mhop.l1_idx * chunk_ba_num + mhop.l2_idx;
 655                 bt_l1_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;
 656                 break;
 657         case 2:
 658                 hem_idx = mhop.l0_idx * chunk_ba_num + mhop.l1_idx;
 659                 break;
 660         case 1:
 661                 hem_idx = mhop.l0_idx;
 662                 break;
 663         default:
 664                 dev_err(dev, "Table %d not support hop_num = %d!\n",
 665                              table->type, hop_num);
 666                 return;
 667         }
 668
 669         mutex_lock(&table->mutex);
 670
 671         if (check_refcount && (--table->hem[hem_idx]->refcount > 0)) {
 672                 mutex_unlock(&table->mutex);
 673                 return;
 674         }
 675
 676         if (table->type < HEM_TYPE_MTT && hop_num == 1) {
 677                 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1))
 678                         dev_warn(dev, "Clear HEM base address failed.\n");
 679         } else if (table->type < HEM_TYPE_MTT && hop_num == 2) {
 680                 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 2))
 681                         dev_warn(dev, "Clear HEM base address failed.\n");
 682         } else if (table->type < HEM_TYPE_MTT &&
 683                    hop_num == HNS_ROCE_HOP_NUM_0) {
 684                 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0))
 685                         dev_warn(dev, "Clear HEM base address failed.\n");
 686         }
 687
 688         /*
 689          * free buffer space chunk for QPC/MTPT/CQC/SRQC/SCCC.
 690          * free bt space chunk for MTT/CQE.
 691          */
 692         hns_roce_free_hem(hr_dev, table->hem[hem_idx]);
 693         table->hem[hem_idx] = NULL;
 694
 695         if (check_whether_bt_num_2(table->type, hop_num)) {
 696                 start_idx = mhop.l0_idx * chunk_ba_num;
 697                 if (hns_roce_check_hem_null(table->hem, start_idx,
 698                                             chunk_ba_num)) {
 699                         if (table->type < HEM_TYPE_MTT &&
 700                             hr_dev->hw->clear_hem(hr_dev, table, obj, 0))
 701                                 dev_warn(dev, "Clear HEM base address failed.\n");
 702
 703                         dma_free_coherent(dev, bt_chunk_size,
 704                                           table->bt_l0[mhop.l0_idx],
 705                                           table->bt_l0_dma_addr[mhop.l0_idx]);
 706                         table->bt_l0[mhop.l0_idx] = NULL;
 707                 }
 708         } else if (check_whether_bt_num_3(table->type, hop_num)) {
 709                 start_idx = mhop.l0_idx * chunk_ba_num * chunk_ba_num +
 710                             mhop.l1_idx * chunk_ba_num;
 711                 if (hns_roce_check_hem_null(table->hem, start_idx,
 712                                             chunk_ba_num)) {
 713                         if (hr_dev->hw->clear_hem(hr_dev, table, obj, 1))
 714                                 dev_warn(dev, "Clear HEM base address failed.\n");
 715
 716                         dma_free_coherent(dev, bt_chunk_size,
 717                                           table->bt_l1[bt_l1_idx],
 718                                           table->bt_l1_dma_addr[bt_l1_idx]);
 719                         table->bt_l1[bt_l1_idx] = NULL;
 720
 721                         start_idx = mhop.l0_idx * chunk_ba_num;
 722                         if (hns_roce_check_bt_null(table->bt_l1, start_idx,
 723                                                    chunk_ba_num)) {
 724                                 if (hr_dev->hw->clear_hem(hr_dev, table, obj,
 725                                                           0))
 726                                         dev_warn(dev, "Clear HEM base address failed.\n");
 727
 728                                 dma_free_coherent(dev, bt_chunk_size,
 729                                             table->bt_l0[mhop.l0_idx],
 730                                             table->bt_l0_dma_addr[mhop.l0_idx]);
 731                                 table->bt_l0[mhop.l0_idx] = NULL;
 732                         }
 733                 }
 734         }
 735
 736         mutex_unlock(&table->mutex);
 737 }
 738
 739 void hns_roce_table_put(struct hns_roce_dev *hr_dev,
 740                         struct hns_roce_hem_table *table, unsigned long obj)
 741 {
 742         struct device *dev = hr_dev->dev;
 743         unsigned long i;
 744
 745         if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
 746                 hns_roce_table_mhop_put(hr_dev, table, obj, 1);
 747                 return;
 748         }
 749
 750         i = (obj & (table->num_obj - 1)) /
 751             (table->table_chunk_size / table->obj_size);
 752
 753         mutex_lock(&table->mutex);
 754
 755         if (--table->hem[i]->refcount == 0) {
 756                 /* Clear HEM base address */
 757                 if (hr_dev->hw->clear_hem(hr_dev, table, obj, 0))
 758                         dev_warn(dev, "Clear HEM base address failed.\n");
 759
 760                 hns_roce_free_hem(hr_dev, table->hem[i]);
 761                 table->hem[i] = NULL;
 762         }
 763
 764         mutex_unlock(&table->mutex);
 765 }
 766 EXPORT_SYMBOL_GPL(hns_roce_table_put);
 767
 768 void *hns_roce_table_find(struct hns_roce_dev *hr_dev,
 769                           struct hns_roce_hem_table *table,
 770                           unsigned long obj, dma_addr_t *dma_handle)
 771 {
 772         struct hns_roce_hem_chunk *chunk;
 773         struct hns_roce_hem_mhop mhop;
 774         struct hns_roce_hem *hem;
 775         void *addr = NULL;
 776         unsigned long mhop_obj = obj;
 777         unsigned long obj_per_chunk;
 778         unsigned long idx_offset;
 779         int offset, dma_offset;
 780         int length;
 781         int i, j;
 782         u32 hem_idx = 0;
 783
 784         if (!table->lowmem)
 785                 return NULL;
 786
 787         mutex_lock(&table->mutex);
 788
 789         if (!hns_roce_check_whether_mhop(hr_dev, table->type)) {
 790                 obj_per_chunk = table->table_chunk_size / table->obj_size;
 791                 hem = table->hem[(obj & (table->num_obj - 1)) / obj_per_chunk];
 792                 idx_offset = (obj & (table->num_obj - 1)) % obj_per_chunk;
 793                 dma_offset = offset = idx_offset * table->obj_size;
 794         } else {
 795                 u32 seg_size = 64; /* 8 bytes per BA and 8 BA per segment */
 796
 797                 hns_roce_calc_hem_mhop(hr_dev, table, &mhop_obj, &mhop);
 798                 /* mtt mhop */
 799                 i = mhop.l0_idx;
 800                 j = mhop.l1_idx;
 801                 if (mhop.hop_num == 2)
 802                         hem_idx = i * (mhop.bt_chunk_size / 8) + j;
 803                 else if (mhop.hop_num == 1 ||
 804                          mhop.hop_num == HNS_ROCE_HOP_NUM_0)
 805                         hem_idx = i;
 806
 807                 hem = table->hem[hem_idx];
 808                 dma_offset = offset = (obj & (table->num_obj - 1)) * seg_size %
 809                                        mhop.bt_chunk_size;
 810                 if (mhop.hop_num == 2)
 811                         dma_offset = offset = 0;
 812         }
 813
 814         if (!hem)
 815                 goto out;
 816
 817         list_for_each_entry(chunk, &hem->chunk_list, list) {
 818                 for (i = 0; i < chunk->npages; ++i) {
 819                         length = sg_dma_len(&chunk->mem[i]);
 820                         if (dma_handle && dma_offset >= 0) {
 821                                 if (length > (u32)dma_offset)
 822                                         *dma_handle = sg_dma_address(
 823                                                 &chunk->mem[i]) + dma_offset;
 824                                 dma_offset -= length;
 825                         }
 826
 827                         if (length > (u32)offset) {
 828                                 addr = chunk->buf[i] + offset;
 829                                 goto out;
 830                         }
 831                         offset -= length;
 832                 }
 833         }
 834
 835 out:
 836         mutex_unlock(&table->mutex);
 837         return addr;
 838 }
 839 EXPORT_SYMBOL_GPL(hns_roce_table_find);
 840
 841 int hns_roce_table_get_range(struct hns_roce_dev *hr_dev,
 842                              struct hns_roce_hem_table *table,
 843                              unsigned long start, unsigned long end)
 844 {
 845         struct hns_roce_hem_mhop mhop;
 846         unsigned long inc = table->table_chunk_size / table->obj_size;
 847         unsigned long i;
 848         int ret;
 849
 850         if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
 851                 hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop);
 852                 inc = mhop.bt_chunk_size / table->obj_size;
 853         }
 854
 855         /* Allocate MTT entry memory according to chunk(128K) */
 856         for (i = start; i <= end; i += inc) {
 857                 ret = hns_roce_table_get(hr_dev, table, i);
 858                 if (ret)
 859                         goto fail;
 860         }
 861
 862         return 0;
 863
 864 fail:
 865         while (i > start) {
 866                 i -= inc;
 867                 hns_roce_table_put(hr_dev, table, i);
 868         }
 869         return ret;
 870 }
 871
 872 void hns_roce_table_put_range(struct hns_roce_dev *hr_dev,
 873                               struct hns_roce_hem_table *table,
 874                               unsigned long start, unsigned long end)
 875 {
 876         struct hns_roce_hem_mhop mhop;
 877         unsigned long inc = table->table_chunk_size / table->obj_size;
 878         unsigned long i;
 879
 880         if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
 881                 hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop);
 882                 inc = mhop.bt_chunk_size / table->obj_size;
 883         }
 884
 885         for (i = start; i <= end; i += inc)
 886                 hns_roce_table_put(hr_dev, table, i);
 887 }
 888
 889 int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev,
 890                             struct hns_roce_hem_table *table, u32 type,
 891                             unsigned long obj_size, unsigned long nobj,
 892                             int use_lowmem)
 893 {
 894         struct device *dev = hr_dev->dev;
 895         unsigned long obj_per_chunk;
 896         unsigned long num_hem;
 897
 898         if (!hns_roce_check_whether_mhop(hr_dev, type)) {
 899                 table->table_chunk_size = hr_dev->caps.chunk_sz;
 900                 obj_per_chunk = table->table_chunk_size / obj_size;
 901                 num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk;
 902
 903                 table->hem = kcalloc(num_hem, sizeof(*table->hem), GFP_KERNEL);
 904                 if (!table->hem)
 905                         return -ENOMEM;
 906         } else {
 907                 unsigned long buf_chunk_size;
 908                 unsigned long bt_chunk_size;
 909                 unsigned long bt_chunk_num;
 910                 unsigned long num_bt_l0 = 0;
 911                 u32 hop_num;
 912
 913                 switch (type) {
 914                 case HEM_TYPE_QPC:
 915                         buf_chunk_size = 1 << (hr_dev->caps.qpc_buf_pg_sz
 916                                         + PAGE_SHIFT);
 917                         bt_chunk_size = 1 << (hr_dev->caps.qpc_ba_pg_sz
 918                                         + PAGE_SHIFT);
 919                         num_bt_l0 = hr_dev->caps.qpc_bt_num;
 920                         hop_num = hr_dev->caps.qpc_hop_num;
 921                         break;
 922                 case HEM_TYPE_MTPT:
 923                         buf_chunk_size = 1 << (hr_dev->caps.mpt_buf_pg_sz
 924                                         + PAGE_SHIFT);
 925                         bt_chunk_size = 1 << (hr_dev->caps.mpt_ba_pg_sz
 926                                         + PAGE_SHIFT);
 927                         num_bt_l0 = hr_dev->caps.mpt_bt_num;
 928                         hop_num = hr_dev->caps.mpt_hop_num;
 929                         break;
 930                 case HEM_TYPE_CQC:
 931                         buf_chunk_size = 1 << (hr_dev->caps.cqc_buf_pg_sz
 932                                         + PAGE_SHIFT);
 933                         bt_chunk_size = 1 << (hr_dev->caps.cqc_ba_pg_sz
 934                                         + PAGE_SHIFT);
 935                         num_bt_l0 = hr_dev->caps.cqc_bt_num;
 936                         hop_num = hr_dev->caps.cqc_hop_num;
 937                         break;
 938                 case HEM_TYPE_SCCC:
 939                         buf_chunk_size = 1 << (hr_dev->caps.sccc_buf_pg_sz
 940                                         + PAGE_SHIFT);
 941                         bt_chunk_size = 1 << (hr_dev->caps.sccc_ba_pg_sz
 942                                         + PAGE_SHIFT);
 943                         num_bt_l0 = hr_dev->caps.sccc_bt_num;
 944                         hop_num = hr_dev->caps.sccc_hop_num;
 945                         break;
 946                 case HEM_TYPE_QPC_TIMER:
 947                         buf_chunk_size = 1 << (hr_dev->caps.qpc_timer_buf_pg_sz
 948                                         + PAGE_SHIFT);
 949                         bt_chunk_size = 1 << (hr_dev->caps.qpc_timer_ba_pg_sz
 950                                         + PAGE_SHIFT);
 951                         num_bt_l0 = hr_dev->caps.qpc_timer_bt_num;
 952                         hop_num = hr_dev->caps.qpc_timer_hop_num;
 953                         break;
 954                 case HEM_TYPE_CQC_TIMER:
 955                         buf_chunk_size = 1 << (hr_dev->caps.cqc_timer_buf_pg_sz
 956                                         + PAGE_SHIFT);
 957                         bt_chunk_size = 1 << (hr_dev->caps.cqc_timer_ba_pg_sz
 958                                         + PAGE_SHIFT);
 959                         num_bt_l0 = hr_dev->caps.cqc_timer_bt_num;
 960                         hop_num = hr_dev->caps.cqc_timer_hop_num;
 961                         break;
 962                 case HEM_TYPE_SRQC:
 963                         buf_chunk_size = 1 << (hr_dev->caps.srqc_buf_pg_sz
 964                                         + PAGE_SHIFT);
 965                         bt_chunk_size = 1 << (hr_dev->caps.srqc_ba_pg_sz
 966                                         + PAGE_SHIFT);
 967                         num_bt_l0 = hr_dev->caps.srqc_bt_num;
 968                         hop_num = hr_dev->caps.srqc_hop_num;
 969                         break;
 970                 case HEM_TYPE_MTT:
 971                         buf_chunk_size = 1 << (hr_dev->caps.mtt_ba_pg_sz
 972                                         + PAGE_SHIFT);
 973                         bt_chunk_size = buf_chunk_size;
 974                         hop_num = hr_dev->caps.mtt_hop_num;
 975                         break;
 976                 case HEM_TYPE_CQE:
 977                         buf_chunk_size = 1 << (hr_dev->caps.cqe_ba_pg_sz
 978                                         + PAGE_SHIFT);
 979                         bt_chunk_size = buf_chunk_size;
 980                         hop_num = hr_dev->caps.cqe_hop_num;
 981                         break;
 982                 case HEM_TYPE_SRQWQE:
 983                         buf_chunk_size = 1 << (hr_dev->caps.srqwqe_ba_pg_sz
 984                                         + PAGE_SHIFT);
 985                         bt_chunk_size = buf_chunk_size;
 986                         hop_num = hr_dev->caps.srqwqe_hop_num;
 987                         break;
 988                 case HEM_TYPE_IDX:
 989                         buf_chunk_size = 1 << (hr_dev->caps.idx_ba_pg_sz
 990                                         + PAGE_SHIFT);
 991                         bt_chunk_size = buf_chunk_size;
 992                         hop_num = hr_dev->caps.idx_hop_num;
 993                         break;
 994                 default:
 995                         dev_err(dev,
 996                           "Table %d not support to init hem table here!\n",
 997                           type);
 998                         return -EINVAL;
 999                 }
1000                 obj_per_chunk = buf_chunk_size / obj_size;
1001                 num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk;
1002                 bt_chunk_num = bt_chunk_size / 8;
1003                 if (type >= HEM_TYPE_MTT)
1004                         num_bt_l0 = bt_chunk_num;
1005
1006                 table->hem = kcalloc(num_hem, sizeof(*table->hem),
1007                                          GFP_KERNEL);
1008                 if (!table->hem)
1009                         goto err_kcalloc_hem_buf;
1010
1011                 if (check_whether_bt_num_3(type, hop_num)) {
1012                         unsigned long num_bt_l1;
1013
1014                         num_bt_l1 = (num_hem + bt_chunk_num - 1) /
1015                                              bt_chunk_num;
1016                         table->bt_l1 = kcalloc(num_bt_l1,
1017                                                sizeof(*table->bt_l1),
1018                                                GFP_KERNEL);
1019                         if (!table->bt_l1)
1020                                 goto err_kcalloc_bt_l1;
1021
1022                         table->bt_l1_dma_addr = kcalloc(num_bt_l1,
1023                                                  sizeof(*table->bt_l1_dma_addr),
1024                                                  GFP_KERNEL);
1025
1026                         if (!table->bt_l1_dma_addr)
1027                                 goto err_kcalloc_l1_dma;
1028                 }
1029
1030                 if (check_whether_bt_num_2(type, hop_num) ||
1031                         check_whether_bt_num_3(type, hop_num)) {
1032                         table->bt_l0 = kcalloc(num_bt_l0, sizeof(*table->bt_l0),
1033                                                GFP_KERNEL);
1034                         if (!table->bt_l0)
1035                                 goto err_kcalloc_bt_l0;
1036
1037                         table->bt_l0_dma_addr = kcalloc(num_bt_l0,
1038                                                  sizeof(*table->bt_l0_dma_addr),
1039                                                  GFP_KERNEL);
1040                         if (!table->bt_l0_dma_addr)
1041                                 goto err_kcalloc_l0_dma;
1042                 }
1043         }
1044
1045         table->type = type;
1046         table->num_hem = num_hem;
1047         table->num_obj = nobj;
1048         table->obj_size = obj_size;
1049         table->lowmem = use_lowmem;
1050         mutex_init(&table->mutex);
1051
1052         return 0;
1053
1054 err_kcalloc_l0_dma:
1055         kfree(table->bt_l0);
1056         table->bt_l0 = NULL;
1057
1058 err_kcalloc_bt_l0:
1059         kfree(table->bt_l1_dma_addr);
1060         table->bt_l1_dma_addr = NULL;
1061
1062 err_kcalloc_l1_dma:
1063         kfree(table->bt_l1);
1064         table->bt_l1 = NULL;
1065
1066 err_kcalloc_bt_l1:
1067         kfree(table->hem);
1068         table->hem = NULL;
1069
1070 err_kcalloc_hem_buf:
1071         return -ENOMEM;
1072 }
1073
1074 static void hns_roce_cleanup_mhop_hem_table(struct hns_roce_dev *hr_dev,
1075                                             struct hns_roce_hem_table *table)
1076 {
1077         struct hns_roce_hem_mhop mhop;
1078         u32 buf_chunk_size;
1079         int i;
1080         u64 obj;
1081
1082         hns_roce_calc_hem_mhop(hr_dev, table, NULL, &mhop);
1083         buf_chunk_size = table->type < HEM_TYPE_MTT ? mhop.buf_chunk_size :
1084                                         mhop.bt_chunk_size;
1085
1086         for (i = 0; i < table->num_hem; ++i) {
1087                 obj = i * buf_chunk_size / table->obj_size;
1088                 if (table->hem[i])
1089                         hns_roce_table_mhop_put(hr_dev, table, obj, 0);
1090         }
1091
1092         kfree(table->hem);
1093         table->hem = NULL;
1094         kfree(table->bt_l1);
1095         table->bt_l1 = NULL;
1096         kfree(table->bt_l1_dma_addr);
1097         table->bt_l1_dma_addr = NULL;
1098         kfree(table->bt_l0);
1099         table->bt_l0 = NULL;
1100         kfree(table->bt_l0_dma_addr);
1101         table->bt_l0_dma_addr = NULL;
1102 }
1103
1104 void hns_roce_cleanup_hem_table(struct hns_roce_dev *hr_dev,
1105                                 struct hns_roce_hem_table *table)
1106 {
1107         struct device *dev = hr_dev->dev;
1108         unsigned long i;
1109
1110         if (hns_roce_check_whether_mhop(hr_dev, table->type)) {
1111                 hns_roce_cleanup_mhop_hem_table(hr_dev, table);
1112                 return;
1113         }
1114
1115         for (i = 0; i < table->num_hem; ++i)
1116                 if (table->hem[i]) {
1117                         if (hr_dev->hw->clear_hem(hr_dev, table,
1118                             i * table->table_chunk_size / table->obj_size, 0))
1119                                 dev_err(dev, "Clear HEM base address failed.\n");
1120
1121                         hns_roce_free_hem(hr_dev, table->hem[i]);
1122                 }
1123
1124         kfree(table->hem);
1125 }
1126
1127 void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev)
1128 {
1129         if ((hr_dev->caps.num_idx_segs))
1130                 hns_roce_cleanup_hem_table(hr_dev,
1131                                            &hr_dev->mr_table.mtt_idx_table);
1132         if (hr_dev->caps.num_srqwqe_segs)
1133                 hns_roce_cleanup_hem_table(hr_dev,
1134                                            &hr_dev->mr_table.mtt_srqwqe_table);
1135         if (hr_dev->caps.srqc_entry_sz)
1136                 hns_roce_cleanup_hem_table(hr_dev,
1137                                            &hr_dev->srq_table.table);
1138         hns_roce_cleanup_hem_table(hr_dev, &hr_dev->cq_table.table);
1139         if (hr_dev->caps.qpc_timer_entry_sz)
1140                 hns_roce_cleanup_hem_table(hr_dev,
1141                                            &hr_dev->qpc_timer_table);
1142         if (hr_dev->caps.cqc_timer_entry_sz)
1143                 hns_roce_cleanup_hem_table(hr_dev,
1144                                            &hr_dev->cqc_timer_table);
1145         if (hr_dev->caps.sccc_entry_sz)
1146                 hns_roce_cleanup_hem_table(hr_dev,
1147                                            &hr_dev->qp_table.sccc_table);
1148         if (hr_dev->caps.trrl_entry_sz)
1149                 hns_roce_cleanup_hem_table(hr_dev,
1150                                            &hr_dev->qp_table.trrl_table);
1151         hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.irrl_table);
1152         hns_roce_cleanup_hem_table(hr_dev, &hr_dev->qp_table.qp_table);
1153         hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtpt_table);
1154         if (hns_roce_check_whether_mhop(hr_dev, HEM_TYPE_CQE))
1155                 hns_roce_cleanup_hem_table(hr_dev,
1156                                            &hr_dev->mr_table.mtt_cqe_table);
1157         hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtt_table);
1158 }