bitops: use try_cmpxchg in set_mask_bits and bit_clear_unless

Use try_cmpxchg instead of cmpxchg (*ptr, old, new) == old in
set_mask_bits and bit_clear_unless.  x86 CMPXCHG instruction returns
success in ZF flag, so this change saves a compare after cmpxchg (and
related move instruction in front of cmpxchg).

Also, try_cmpxchg implicitly assigns old *ptr value to "old" when cmpxchg
fails, enabling further code simplifications.

Link: https://lkml.kernel.org/r/20220822143851.3290-1-ubizjak@gmail.com
Signed-off-by: Uros Bizjak <ubizjak@gmail.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/include/linux/bitops.h b/include/linux/bitops.h
index 3b89c64..fb24a51 100644 (file)
--- a/include/linux/bitops.h
+++ b/include/linux/bitops.h
@@ -328,10 +328,10 @@ static __always_inline void __assign_bit(long nr, volatile unsigned long *addr,
        const typeof(*(ptr)) mask__ = (mask), bits__ = (bits);  \
        typeof(*(ptr)) old__, new__;                            \
                                                                \
+       old__ = READ_ONCE(*(ptr));                              \
        do {                                                    \
-               old__ = READ_ONCE(*(ptr));                      \
                new__ = (old__ & ~mask__) | bits__;             \
-       } while (cmpxchg(ptr, old__, new__) != old__);          \
+       } while (!try_cmpxchg(ptr, &old__, new__));             \
                                                                \
        old__;                                                  \
 })
@@ -343,11 +343,12 @@ static __always_inline void __assign_bit(long nr, volatile unsigned long *addr,
        const typeof(*(ptr)) clear__ = (clear), test__ = (test);\
        typeof(*(ptr)) old__, new__;                            \
                                                                \
+       old__ = READ_ONCE(*(ptr));                              \
        do {                                                    \
-               old__ = READ_ONCE(*(ptr));                      \
+               if (old__ & test__)                             \
+                       break;                                  \
                new__ = old__ & ~clear__;                       \
-       } while (!(old__ & test__) &&                           \
-                cmpxchg(ptr, old__, new__) != old__);          \
+       } while (!try_cmpxchg(ptr, &old__, new__));             \
                                                                \
        !(old__ & test__);                                      \
 })