arch/aarch64/atomic_arch.h

   1 #define a_ctz_64 a_ctz_64
   2 static inline int a_ctz_64(uint64_t x)
   3 {
   4         __asm__(
   5                 "       rbit %0, %1\n"
   6                 "       clz %0, %0\n"
   7                 : "=r"(x) : "r"(x));
   8         return x;
   9 }
  10
  11 #define a_barrier a_barrier
  12 static inline void a_barrier()
  13 {
  14         __asm__ __volatile__("dmb ish");
  15 }
  16
  17 #define a_cas_p a_cas_p
  18 static inline void *a_cas_p(volatile void *p, void *t, void *s)
  19 {
  20         void *old;
  21         __asm__ __volatile__(
  22                 "       dmb ish\n"
  23                 "1:     ldxr %0,%3\n"
  24                 "       cmp %0,%1\n"
  25                 "       b.ne 1f\n"
  26                 "       stxr %w0,%2,%3\n"
  27                 "       cbnz %w0,1b\n"
  28                 "       mov %0,%1\n"
  29                 "1:     dmb ish\n"
  30                 : "=&r"(old)
  31                 : "r"(t), "r"(s), "Q"(*(long*)p)
  32                 : "memory", "cc");
  33         return old;
  34 }
  35
  36 #define a_cas a_cas
  37 static inline int a_cas(volatile int *p, int t, int s)
  38 {
  39         int old;
  40         __asm__ __volatile__(
  41                 "       dmb ish\n"
  42                 "1:     ldxr %w0,%3\n"
  43                 "       cmp %w0,%w1\n"
  44                 "       b.ne 1f\n"
  45                 "       stxr %w0,%w2,%3\n"
  46                 "       cbnz %w0,1b\n"
  47                 "       mov %w0,%w1\n"
  48                 "1:     dmb ish\n"
  49                 : "=&r"(old)
  50                 : "r"(t), "r"(s), "Q"(*p)
  51                 : "memory", "cc");
  52         return old;
  53 }
  54
  55 #define a_swap a_swap
  56 static inline int a_swap(volatile int *x, int v)
  57 {
  58         int old, tmp;
  59         __asm__ __volatile__(
  60                 "       dmb ish\n"
  61                 "1:     ldxr %w0,%3\n"
  62                 "       stxr %w1,%w2,%3\n"
  63                 "       cbnz %w1,1b\n"
  64                 "       dmb ish\n"
  65                 : "=&r"(old), "=&r"(tmp)
  66                 : "r"(v), "Q"(*x)
  67                 : "memory", "cc" );
  68         return old;
  69 }
  70
  71 #define a_fetch_add a_fetch_add
  72 static inline int a_fetch_add(volatile int *x, int v)
  73 {
  74         int old, tmp;
  75         __asm__ __volatile__(
  76                 "       dmb ish\n"
  77                 "1:     ldxr %w0,%3\n"
  78                 "       add %w0,%w0,%w2\n"
  79                 "       stxr %w1,%w0,%3\n"
  80                 "       cbnz %w1,1b\n"
  81                 "       dmb ish\n"
  82                 : "=&r"(old), "=&r"(tmp)
  83                 : "r"(v), "Q"(*x)
  84                 : "memory", "cc" );
  85         return old-v;
  86 }
  87
  88 #define a_inc a_inc
  89 static inline void a_inc(volatile int *x)
  90 {
  91         int tmp, tmp2;
  92         __asm__ __volatile__(
  93                 "       dmb ish\n"
  94                 "1:     ldxr %w0,%2\n"
  95                 "       add %w0,%w0,#1\n"
  96                 "       stxr %w1,%w0,%2\n"
  97                 "       cbnz %w1,1b\n"
  98                 "       dmb ish\n"
  99                 : "=&r"(tmp), "=&r"(tmp2)
 100                 : "Q"(*x)
 101                 : "memory", "cc" );
 102 }
 103
 104 #define a_dec a_dec
 105 static inline void a_dec(volatile int *x)
 106 {
 107         int tmp, tmp2;
 108         __asm__ __volatile__(
 109                 "       dmb ish\n"
 110                 "1:     ldxr %w0,%2\n"
 111                 "       sub %w0,%w0,#1\n"
 112                 "       stxr %w1,%w0,%2\n"
 113                 "       cbnz %w1,1b\n"
 114                 "       dmb ish\n"
 115                 : "=&r"(tmp), "=&r"(tmp2)
 116                 : "Q"(*x)
 117                 : "memory", "cc" );
 118 }
 119
 120 #define a_and_64 a_and_64
 121 static inline void a_and_64(volatile uint64_t *p, uint64_t v)
 122 {
 123         int tmp, tmp2;
 124         __asm__ __volatile__(
 125                 "       dmb ish\n"
 126                 "1:     ldxr %0,%3\n"
 127                 "       and %0,%0,%2\n"
 128                 "       stxr %w1,%0,%3\n"
 129                 "       cbnz %w1,1b\n"
 130                 "       dmb ish\n"
 131                 : "=&r"(tmp), "=&r"(tmp2)
 132                 : "r"(v), "Q"(*p)
 133                 : "memory", "cc" );
 134 }
 135
 136 #define a_and a_and
 137 static inline void a_and(volatile int *p, int v)
 138 {
 139         int tmp, tmp2;
 140         __asm__ __volatile__(
 141                 "       dmb ish\n"
 142                 "1:     ldxr %w0,%3\n"
 143                 "       and %w0,%w0,%w2\n"
 144                 "       stxr %w1,%w0,%3\n"
 145                 "       cbnz %w1,1b\n"
 146                 "       dmb ish\n"
 147                 : "=&r"(tmp), "=&r"(tmp2)
 148                 : "r"(v), "Q"(*p)
 149                 : "memory", "cc" );
 150 }
 151
 152 #define a_or_64 a_or_64
 153 static inline void a_or_64(volatile uint64_t *p, uint64_t v)
 154 {
 155         int tmp, tmp2;
 156         __asm__ __volatile__(
 157                 "       dmb ish\n"
 158                 "1:     ldxr %0,%3\n"
 159                 "       orr %0,%0,%2\n"
 160                 "       stxr %w1,%0,%3\n"
 161                 "       cbnz %w1,1b\n"
 162                 "       dmb ish\n"
 163                 : "=&r"(tmp), "=&r"(tmp2)
 164                 : "r"(v), "Q"(*p)
 165                 : "memory", "cc" );
 166 }
 167
 168 #define a_or_l a_or_l
 169 static inline void a_or_l(volatile void *p, long v)
 170 {
 171         return a_or_64(p, v);
 172 }
 173
 174 #define a_or a_or
 175 static inline void a_or(volatile int *p, int v)
 176 {
 177         int tmp, tmp2;
 178         __asm__ __volatile__(
 179                 "       dmb ish\n"
 180                 "1:     ldxr %w0,%3\n"
 181                 "       orr %w0,%w0,%w2\n"
 182                 "       stxr %w1,%w0,%3\n"
 183                 "       cbnz %w1,1b\n"
 184                 "       dmb ish\n"
 185                 : "=&r"(tmp), "=&r"(tmp2)
 186                 : "r"(v), "Q"(*p)
 187                 : "memory", "cc" );
 188 }
 189
 190 #define a_store a_store
 191 static inline void a_store(volatile int *p, int x)
 192 {
 193         __asm__ __volatile__(
 194                 "       dmb ish\n"
 195                 "       str %w1,%0\n"
 196                 "       dmb ish\n"
 197                 : "=m"(*p)
 198                 : "r"(x)
 199                 : "memory", "cc" );
 200 }
 201
 202 #define a_spin a_barrier