sbitmap: add __sbitmap_queue_get_batch()

The block layer tag allocation batching still calls into sbitmap to get
each tag, but we can improve on that. Add __sbitmap_queue_get_batch(),
which returns a mask of tags all at once, along with an offset for
those tags.

An example return would be 0xff, where bits 0..7 are set, with
tag_offset == 128. The valid tags in this case would be 128..135.

A batch is specific to an individual sbitmap_map, hence it cannot be
larger than that. The requested number of tags is automatically reduced
to the max that can be satisfied with a single map.

On failure, 0 is returned. Caller should fall back to single tag
allocation at that point/

Signed-off-by: Jens Axboe <axboe@kernel.dk>

diff --git a/include/linux/sbitmap.h b/include/linux/sbitmap.h
index 2713e68..e30b560 100644 (file)
--- a/include/linux/sbitmap.h
+++ b/include/linux/sbitmap.h
@@ -427,6 +427,19 @@ void sbitmap_queue_resize(struct sbitmap_queue *sbq, unsigned int depth);
 int __sbitmap_queue_get(struct sbitmap_queue *sbq);
 
 /**
+ * __sbitmap_queue_get_batch() - Try to allocate a batch of free bits
+ * @sbq: Bitmap queue to allocate from.
+ * @nr_tags: number of tags requested
+ * @offset: offset to add to returned bits
+ *
+ * Return: Mask of allocated tags, 0 if none are found. Each tag allocated is
+ * a bit in the mask returned, and the caller must add @offset to the value to
+ * get the absolute tag value.
+ */
+unsigned long __sbitmap_queue_get_batch(struct sbitmap_queue *sbq, int nr_tags,
+                                       unsigned int *offset);
+
+/**
  * __sbitmap_queue_get_shallow() - Try to allocate a free bit from a &struct
  * sbitmap_queue, limiting the depth used from each word, with preemption
  * already disabled.

diff --git a/lib/sbitmap.c b/lib/sbitmap.c
index b25db9b..f398e0a 100644 (file)
--- a/lib/sbitmap.c
+++ b/lib/sbitmap.c
@@ -489,6 +489,57 @@ int __sbitmap_queue_get(struct sbitmap_queue *sbq)
 }
 EXPORT_SYMBOL_GPL(__sbitmap_queue_get);
 
+unsigned long __sbitmap_queue_get_batch(struct sbitmap_queue *sbq, int nr_tags,
+                                       unsigned int *offset)
+{
+       struct sbitmap *sb = &sbq->sb;
+       unsigned int hint, depth;
+       unsigned long index, nr;
+       int i;
+
+       if (unlikely(sb->round_robin))
+               return 0;
+
+       depth = READ_ONCE(sb->depth);
+       hint = update_alloc_hint_before_get(sb, depth);
+
+       index = SB_NR_TO_INDEX(sb, hint);
+
+       for (i = 0; i < sb->map_nr; i++) {
+               struct sbitmap_word *map = &sb->map[index];
+               unsigned long get_mask;
+
+               sbitmap_deferred_clear(map);
+               if (map->word == (1UL << (map->depth - 1)) - 1)
+                       continue;
+
+               nr = find_first_zero_bit(&map->word, map->depth);
+               if (nr + nr_tags <= map->depth) {
+                       atomic_long_t *ptr = (atomic_long_t *) &map->word;
+                       int map_tags = min_t(int, nr_tags, map->depth);
+                       unsigned long val, ret;
+
+                       get_mask = ((1UL << map_tags) - 1) << nr;
+                       do {
+                               val = READ_ONCE(map->word);
+                               ret = atomic_long_cmpxchg(ptr, val, get_mask | val);
+                       } while (ret != val);
+                       get_mask = (get_mask & ~ret) >> nr;
+                       if (get_mask) {
+                               *offset = nr + (index << sb->shift);
+                               update_alloc_hint_after_get(sb, depth, hint,
+                                                       *offset + map_tags - 1);
+                               return get_mask;
+                       }
+               }
+               /* Jump to next index. */
+               if (++index >= sb->map_nr)
+                       index = 0;
+       }
+
+       return 0;
+}
+
 int __sbitmap_queue_get_shallow(struct sbitmap_queue *sbq,
                                unsigned int shallow_depth)
 {