Merge branch 'odp_rework' into rdma.git for-next

Jason Gunthorpe says:

====================
In order to hoist the interval tree code out of the drivers and into the
mmu_notifiers it is necessary for the drivers to not use the interval tree
for other things.

This series replaces the interval tree with an xarray and along the way
re-aligns all the locking to use a sensible SRCU model where the 'update'
step is done by modifying an xarray.

The result is overall much simpler and with less locking in the critical
path. Many functions were reworked for clarity and small details like
using 'imr' to refer to the implicit MR make the entire code flow here
more readable.

This also squashes at least two race bugs on its own, and quite possibily
more that haven't been identified.
====================

Merge conflicts with the odp statistics patch resolved.

* branch 'odp_rework':
  RDMA/odp: Remove broken debugging call to invalidate_range
  RDMA/mlx5: Do not race with mlx5_ib_invalidate_range during create and destroy
  RDMA/mlx5: Do not store implicit children in the odp_mkeys xarray
  RDMA/mlx5: Rework implicit ODP destroy
  RDMA/mlx5: Avoid double lookups on the pagefault path
  RDMA/mlx5: Reduce locking in implicit_mr_get_data()
  RDMA/mlx5: Use an xarray for the children of an implicit ODP
  RDMA/mlx5: Split implicit handling from pagefault_mr
  RDMA/mlx5: Set the HW IOVA of the child MRs to their place in the tree
  RDMA/mlx5: Lift implicit_mr_alloc() into the two routines that call it
  RDMA/mlx5: Rework implicit_mr_get_data
  RDMA/mlx5: Delete struct mlx5_priv->mkey_table
  RDMA/mlx5: Use a dedicated mkey xarray for ODP
  RDMA/mlx5: Split sig_err MR data into its own xarray
  RDMA/mlx5: Use SRCU properly in ODP prefetch

Signed-off-by: Jason Gunthorpe <jgg@mellanox.com>

diff --cc drivers/infiniband/hw/mlx5/main.c
Simple merge

diff --cc drivers/infiniband/hw/mlx5/mlx5_ib.h
index 3a97c2d,f61d400..5b4c575
--- 1/drivers/infiniband/hw/mlx5/mlx5_ib.h
--- 2/drivers/infiniband/hw/mlx5/mlx5_ib.h
+++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h
@@@ -616,12 -617,16 +615,18 @@@ struct mlx5_ib_mr 
        u64                     data_iova;
        u64                     pi_iova;
  
-       atomic_t                num_leaf_free;
-       wait_queue_head_t       q_leaf_free;
-       struct mlx5_async_work  cb_work;
-       atomic_t                num_pending_prefetch;
+       /* For ODP and implicit */
+       atomic_t                num_deferred_work;
+       struct xarray           implicit_children;
+       union {
+               struct rcu_head rcu;
+               struct list_head elm;
+               struct work_struct work;
+       } odp_destroy;
 +      struct ib_odp_counters  odp_stats;
 +      bool                    is_odp_implicit;
+ 
+       struct mlx5_async_work  cb_work;
  };
  
  static inline bool is_odp_mr(struct mlx5_ib_mr *mr)

diff --cc drivers/infiniband/hw/mlx5/mr.c
Simple merge

diff --cc drivers/infiniband/hw/mlx5/odp.c
index b332117,bcfc098..45ee40c
--- 1/drivers/infiniband/hw/mlx5/odp.c
--- 2/drivers/infiniband/hw/mlx5/odp.c
+++ b/drivers/infiniband/hw/mlx5/odp.c
@@@ -563,21 -482,44 +487,45 @@@ struct mlx5_ib_mr *mlx5_ib_alloc_implic
        if (IS_ERR(umem_odp))
                return ERR_CAST(umem_odp);
  
-       imr = implicit_mr_alloc(&pd->ibpd, umem_odp, 1, access_flags);
+       imr = mlx5_mr_cache_alloc(dev, MLX5_IMR_KSM_CACHE_ENTRY);
        if (IS_ERR(imr)) {
-               ib_umem_odp_release(umem_odp);
-               return ERR_CAST(imr);
+               err = PTR_ERR(imr);
+               goto out_umem;
        }
  
+       imr->ibmr.pd = &pd->ibpd;
+       imr->access_flags = access_flags;
+       imr->mmkey.iova = 0;
+       imr->umem = &umem_odp->umem;
+       imr->ibmr.lkey = imr->mmkey.key;
+       imr->ibmr.rkey = imr->mmkey.key;
        imr->umem = &umem_odp->umem;
-       init_waitqueue_head(&imr->q_leaf_free);
-       atomic_set(&imr->num_leaf_free, 0);
-       atomic_set(&imr->num_pending_prefetch, 0);
-       smp_store_release(&imr->live, 1);
- 
 +      imr->is_odp_implicit = true;
+       atomic_set(&imr->num_deferred_work, 0);
+       xa_init(&imr->implicit_children);
+ 
+       err = mlx5_ib_update_xlt(imr, 0,
+                                mlx5_imr_ksm_entries,
+                                MLX5_KSM_PAGE_SHIFT,
+                                MLX5_IB_UPD_XLT_INDIRECT |
+                                MLX5_IB_UPD_XLT_ZAP |
+                                MLX5_IB_UPD_XLT_ENABLE);
+       if (err)
+               goto out_mr;
+ 
+       err = xa_err(xa_store(&dev->odp_mkeys, mlx5_base_mkey(imr->mmkey.key),
+                             &imr->mmkey, GFP_KERNEL));
+       if (err)
+               goto out_mr;
  
+       mlx5_ib_dbg(dev, "key %x mr %p\n", imr->mmkey.key, imr);
        return imr;
+ out_mr:
+       mlx5_ib_err(dev, "Failed to register MKEY %d\n", err);
+       mlx5_mr_cache_free(dev, imr);
+ out_umem:
+       ib_umem_odp_release(umem_odp);
+       return ERR_PTR(err);
  }
  
  void mlx5_ib_free_implicit_mr(struct mlx5_ib_mr *imr)