libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h

   1 /* Copyright (C) 2002-2004, 2006-2008, 2009 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, see
  17    <http://www.gnu.org/licenses/>.  */
  18
  19 #ifndef _LOWLEVELLOCK_H
  20 #define _LOWLEVELLOCK_H 1
  21
  22 #ifndef __ASSEMBLER__
  23 # include <time.h>
  24 # include <sys/param.h>
  25 # include <bits/pthreadtypes.h>
  26 # include <bits/kernel-features.h>
  27 # include <tcb-offsets.h>
  28
  29 # ifndef LOCK_INSTR
  30 #  ifdef UP
  31 #   define LOCK_INSTR   /* nothing */
  32 #  else
  33 #   define LOCK_INSTR "lock;"
  34 #  endif
  35 # endif
  36 #else
  37 # ifndef LOCK
  38 #  ifdef UP
  39 #   define LOCK
  40 #  else
  41 #   define LOCK lock
  42 #  endif
  43 # endif
  44 #endif
  45
  46 #define FUTEX_WAIT              0
  47 #define FUTEX_WAKE              1
  48 #define FUTEX_CMP_REQUEUE       4
  49 #define FUTEX_WAKE_OP           5
  50 #define FUTEX_LOCK_PI           6
  51 #define FUTEX_UNLOCK_PI         7
  52 #define FUTEX_TRYLOCK_PI        8
  53 #define FUTEX_WAIT_BITSET       9
  54 #define FUTEX_WAKE_BITSET       10
  55 #define FUTEX_WAIT_REQUEUE_PI   11
  56 #define FUTEX_CMP_REQUEUE_PI    12
  57 #define FUTEX_PRIVATE_FLAG      128
  58 #define FUTEX_CLOCK_REALTIME    256
  59
  60 #define FUTEX_BITSET_MATCH_ANY  0xffffffff
  61
  62 #define FUTEX_OP_CLEAR_WAKE_IF_GT_ONE   ((4 << 24) | 1)
  63
  64 /* Values for 'private' parameter of locking macros.  Yes, the
  65    definition seems to be backwards.  But it is not.  The bit will be
  66    reversed before passing to the system call.  */
  67 #define LLL_PRIVATE     0
  68 #define LLL_SHARED      FUTEX_PRIVATE_FLAG
  69
  70 #ifndef __ASSEMBLER__
  71
  72 #if !defined NOT_IN_libc || defined IS_IN_rtld
  73 /* In libc.so or ld.so all futexes are private.  */
  74 # ifdef __ASSUME_PRIVATE_FUTEX
  75 #  define __lll_private_flag(fl, private) \
  76   ((fl) | FUTEX_PRIVATE_FLAG)
  77 # else
  78 #  define __lll_private_flag(fl, private) \
  79   ((fl) | THREAD_GETMEM (THREAD_SELF, header.private_futex))
  80 # endif
  81 #else
  82 # ifdef __ASSUME_PRIVATE_FUTEX
  83 #  define __lll_private_flag(fl, private) \
  84   (((fl) | FUTEX_PRIVATE_FLAG) ^ (private))
  85 # else
  86 #  define __lll_private_flag(fl, private) \
  87   (__builtin_constant_p (private)                                             \
  88    ? ((private) == 0                                                          \
  89       ? ((fl) | THREAD_GETMEM (THREAD_SELF, header.private_futex))            \
  90       : (fl))                                                                 \
  91    : ({ unsigned int __fl = ((private) ^ FUTEX_PRIVATE_FLAG);                 \
  92         __asm__ ("andl %%fs:%P1, %0" : "+r" (__fl)                                    \
  93              : "i" (offsetof (struct pthread, header.private_futex)));        \
  94         __fl | (fl); }))
  95 # endif
  96 #endif
  97
  98 /* Initializer for lock.  */
  99 #define LLL_LOCK_INITIALIZER            (0)
 100 #define LLL_LOCK_INITIALIZER_LOCKED     (1)
 101 #define LLL_LOCK_INITIALIZER_WAITERS    (2)
 102
 103 /* Delay in spinlock loop.  */
 104 #define BUSY_WAIT_NOP     __asm__ ("rep; nop")
 105
 106
 107 #define LLL_STUB_UNWIND_INFO_START \
 108         ".section       .eh_frame,\"a\",@progbits\n"            \
 109 "7:\t"  ".long  9f-8f   # Length of Common Information Entry\n" \
 110 "8:\t"  ".long  0x0     # CIE Identifier Tag\n\t"               \
 111         ".byte  0x1     # CIE Version\n\t"                      \
 112         ".ascii \"zR\\0\"       # CIE Augmentation\n\t"         \
 113         ".uleb128 0x1   # CIE Code Alignment Factor\n\t"        \
 114         ".sleb128 -8    # CIE Data Alignment Factor\n\t"        \
 115         ".byte  0x10    # CIE RA Column\n\t"                    \
 116         ".uleb128 0x1   # Augmentation size\n\t"                \
 117         ".byte  0x1b    # FDE Encoding (pcrel sdata4)\n\t"      \
 118         ".byte  0x12    # DW_CFA_def_cfa_sf\n\t"                \
 119         ".uleb128 0x7\n\t"                                      \
 120         ".sleb128 16\n\t"                                       \
 121         ".align 8\n"                                            \
 122 "9:\t"  ".long  23f-10f # FDE Length\n"                         \
 123 "10:\t" ".long  10b-7b  # FDE CIE offset\n\t"                   \
 124         ".long  1b-.    # FDE initial location\n\t"             \
 125         ".long  6b-1b   # FDE address range\n\t"                \
 126         ".uleb128 0x0   # Augmentation size\n\t"                \
 127         ".byte  0x16    # DW_CFA_val_expression\n\t"            \
 128         ".uleb128 0x10\n\t"                                     \
 129         ".uleb128 12f-11f\n"                                    \
 130 "11:\t" ".byte  0x80    # DW_OP_breg16\n\t"                     \
 131         ".sleb128 4b-1b\n"
 132 #define LLL_STUB_UNWIND_INFO_END \
 133         ".byte  0x16    # DW_CFA_val_expression\n\t"            \
 134         ".uleb128 0x10\n\t"                                     \
 135         ".uleb128 14f-13f\n"                                    \
 136 "13:\t" ".byte  0x80    # DW_OP_breg16\n\t"                     \
 137         ".sleb128 4b-2b\n"                                      \
 138 "14:\t" ".byte  0x40 + (3b-2b) # DW_CFA_advance_loc\n\t"        \
 139         ".byte  0x0e    # DW_CFA_def_cfa_offset\n\t"            \
 140         ".uleb128 0\n\t"                                        \
 141         ".byte  0x16    # DW_CFA_val_expression\n\t"            \
 142         ".uleb128 0x10\n\t"                                     \
 143         ".uleb128 16f-15f\n"                                    \
 144 "15:\t" ".byte  0x80    # DW_OP_breg16\n\t"                     \
 145         ".sleb128 4b-3b\n"                                      \
 146 "16:\t" ".byte  0x40 + (4b-3b-1) # DW_CFA_advance_loc\n\t"      \
 147         ".byte  0x0e    # DW_CFA_def_cfa_offset\n\t"            \
 148         ".uleb128 128\n\t"                                      \
 149         ".byte  0x16    # DW_CFA_val_expression\n\t"            \
 150         ".uleb128 0x10\n\t"                                     \
 151         ".uleb128 20f-17f\n"                                    \
 152 "17:\t" ".byte  0x80    # DW_OP_breg16\n\t"                     \
 153         ".sleb128 19f-18f\n\t"                                  \
 154         ".byte  0x0d    # DW_OP_const4s\n"                      \
 155 "18:\t" ".4byte 4b-.\n\t"                                       \
 156         ".byte  0x1c    # DW_OP_minus\n\t"                      \
 157         ".byte  0x0d    # DW_OP_const4s\n"                      \
 158 "19:\t" ".4byte 24f-.\n\t"                                      \
 159         ".byte  0x22    # DW_OP_plus\n"                         \
 160 "20:\t" ".byte  0x40 + (5b-4b+1) # DW_CFA_advance_loc\n\t"      \
 161         ".byte  0x13    # DW_CFA_def_cfa_offset_sf\n\t"         \
 162         ".sleb128 16\n\t"                                       \
 163         ".byte  0x16    # DW_CFA_val_expression\n\t"            \
 164         ".uleb128 0x10\n\t"                                     \
 165         ".uleb128 22f-21f\n"                                    \
 166 "21:\t" ".byte  0x80    # DW_OP_breg16\n\t"                     \
 167         ".sleb128 4b-5b\n"                                      \
 168 "22:\t" ".align 8\n"                                            \
 169 "23:\t" ".previous\n"
 170
 171 /* Unwind info for
 172    1: leaq ..., %rdi
 173    2: subq $128, %rsp
 174    3: callq ...
 175    4: addq $128, %rsp
 176    5: jmp 24f
 177    6:
 178    snippet.  */
 179 #define LLL_STUB_UNWIND_INFO_5 \
 180 LLL_STUB_UNWIND_INFO_START                                      \
 181 "12:\t" ".byte  0x40 + (2b-1b) # DW_CFA_advance_loc\n\t"        \
 182 LLL_STUB_UNWIND_INFO_END
 183
 184 /* Unwind info for
 185    1: leaq ..., %rdi
 186    0: movq ..., %rdx
 187    2: subq $128, %rsp
 188    3: callq ...
 189    4: addq $128, %rsp
 190    5: jmp 24f
 191    6:
 192    snippet.  */
 193 #define LLL_STUB_UNWIND_INFO_6 \
 194 LLL_STUB_UNWIND_INFO_START                                      \
 195 "12:\t" ".byte  0x40 + (0b-1b) # DW_CFA_advance_loc\n\t"        \
 196         ".byte  0x16    # DW_CFA_val_expression\n\t"            \
 197         ".uleb128 0x10\n\t"                                     \
 198         ".uleb128 26f-25f\n"                                    \
 199 "25:\t" ".byte  0x80    # DW_OP_breg16\n\t"                     \
 200         ".sleb128 4b-0b\n"                                      \
 201 "26:\t" ".byte  0x40 + (2b-0b) # DW_CFA_advance_loc\n\t"        \
 202 LLL_STUB_UNWIND_INFO_END
 203
 204
 205 #define lll_futex_wait(futex, val, private) \
 206   lll_futex_timed_wait(futex, val, NULL, private)
 207
 208
 209 #define lll_futex_timed_wait(futex, val, timeout, private) \
 210   ({                                                                          \
 211     register const struct timespec *__to __asm__ ("r10") = timeout;           \
 212     int __status;                                                             \
 213     register __typeof (val) _val __asm__ ("edx") = (val);                             \
 214     __asm__ __volatile__ ("syscall"                                                   \
 215                       : "=a" (__status)                                       \
 216                       : "0" (SYS_futex), "D" (futex),                         \
 217                         "S" (__lll_private_flag (FUTEX_WAIT, private)),       \
 218                         "d" (_val), "r" (__to)                                \
 219                       : "memory", "cc", "r11", "cx");                         \
 220     __status;                                                                 \
 221   })
 222
 223
 224 #define lll_futex_wake(futex, nr, private) \
 225   do {                                                                        \
 226     int __ignore;                                                             \
 227     register __typeof (nr) _nr __asm__ ("edx") = (nr);                        \
 228     __asm__ __volatile__ ("syscall"                                                   \
 229                       : "=a" (__ignore)                                       \
 230                       : "0" (SYS_futex), "D" (futex),                         \
 231                         "S" (__lll_private_flag (FUTEX_WAKE, private)),       \
 232                         "d" (_nr)                                             \
 233                       : "memory", "cc", "r10", "r11", "cx");                  \
 234   } while (0)
 235
 236
 237 /* NB: in the lll_trylock macro we simply return the value in %eax
 238    after the cmpxchg instruction.  In case the operation succeded this
 239    value is zero.  In case the operation failed, the cmpxchg instruction
 240    has loaded the current value of the memory work which is guaranteed
 241    to be nonzero.  */
 242 #if defined NOT_IN_libc || defined UP
 243 # define __lll_trylock_asm LOCK_INSTR "cmpxchgl %2, %1"
 244 #else
 245 # define __lll_trylock_asm "cmpl $0, __libc_multiple_threads(%%rip)\n\t"      \
 246                            "je 0f\n\t"                                        \
 247                            "lock; cmpxchgl %2, %1\n\t"                        \
 248                            "jmp 1f\n\t"                                       \
 249                            "0:\tcmpxchgl %2, %1\n\t"                          \
 250                            "1:"
 251 #endif
 252
 253 #define lll_trylock(futex) \
 254   ({ int ret;                                                                 \
 255      __asm__ __volatile__ (__lll_trylock_asm                                  \
 256                        : "=a" (ret), "=m" (futex)                             \
 257                        : "r" (LLL_LOCK_INITIALIZER_LOCKED), "m" (futex),      \
 258                          "0" (LLL_LOCK_INITIALIZER)                           \
 259                        : "memory");                                           \
 260      ret; })
 261
 262 #define lll_robust_trylock(futex, id) \
 263   ({ int ret;                                                                 \
 264      __asm__ __volatile__ (LOCK_INSTR "cmpxchgl %2, %1"                       \
 265                        : "=a" (ret), "=m" (futex)                             \
 266                        : "r" (id), "m" (futex), "0" (LLL_LOCK_INITIALIZER)    \
 267                        : "memory");                                           \
 268      ret; })
 269
 270 #define lll_cond_trylock(futex) \
 271   ({ int ret;                                                                 \
 272      __asm__ __volatile__ (LOCK_INSTR "cmpxchgl %2, %1"                       \
 273                        : "=a" (ret), "=m" (futex)                             \
 274                        : "r" (LLL_LOCK_INITIALIZER_WAITERS),                  \
 275                          "m" (futex), "0" (LLL_LOCK_INITIALIZER)              \
 276                        : "memory");                                           \
 277      ret; })
 278
 279 #if defined NOT_IN_libc || defined UP
 280 # define __lll_lock_asm_start LOCK_INSTR "cmpxchgl %4, %2\n\t"                \
 281                               "jnz 1f\n\t"
 282 #else
 283 # define __lll_lock_asm_start "cmpl $0, __libc_multiple_threads(%%rip)\n\t"   \
 284                               "je 0f\n\t"                                     \
 285                               "lock; cmpxchgl %4, %2\n\t"                     \
 286                               "jnz 1f\n\t"                                    \
 287                               "jmp 24f\n"                                     \
 288                               "0:\tcmpxchgl %4, %2\n\t"                       \
 289                               "jnz 1f\n\t"
 290 #endif
 291
 292 #define lll_lock(futex, private) \
 293   (void)                                                                      \
 294     ({ int ignore1, ignore2, ignore3;                                         \
 295        if (__builtin_constant_p (private) && (private) == LLL_PRIVATE)        \
 296          __asm__ __volatile__ (__lll_lock_asm_start                                   \
 297                            ".subsection 1\n\t"                                \
 298                            ".type _L_lock_%=, @function\n"                    \
 299                            "_L_lock_%=:\n"                                    \
 300                            "1:\tleaq %2, %%rdi\n"                             \
 301                            "2:\tsubq $128, %%rsp\n"                           \
 302                            "3:\tcallq __lll_lock_wait_private\n"              \
 303                            "4:\taddq $128, %%rsp\n"                           \
 304                            "5:\tjmp 24f\n"                                    \
 305                            "6:\t.size _L_lock_%=, 6b-1b\n\t"                  \
 306                            ".previous\n"                                      \
 307                            LLL_STUB_UNWIND_INFO_5                             \
 308                            "24:"                                              \
 309                            : "=S" (ignore1), "=&D" (ignore2), "=m" (futex),   \
 310                              "=a" (ignore3)                                   \
 311                            : "0" (1), "m" (futex), "3" (0)                    \
 312                            : "cx", "r11", "cc", "memory");                    \
 313        else                                                                   \
 314          __asm__ __volatile__ (__lll_lock_asm_start                                   \
 315                            ".subsection 1\n\t"                                \
 316                            ".type _L_lock_%=, @function\n"                    \
 317                            "_L_lock_%=:\n"                                    \
 318                            "1:\tleaq %2, %%rdi\n"                             \
 319                            "2:\tsubq $128, %%rsp\n"                           \
 320                            "3:\tcallq __lll_lock_wait\n"                      \
 321                            "4:\taddq $128, %%rsp\n"                           \
 322                            "5:\tjmp 24f\n"                                    \
 323                            "6:\t.size _L_lock_%=, 6b-1b\n\t"                  \
 324                            ".previous\n"                                      \
 325                            LLL_STUB_UNWIND_INFO_5                             \
 326                            "24:"                                              \
 327                            : "=S" (ignore1), "=D" (ignore2), "=m" (futex),    \
 328                              "=a" (ignore3)                                   \
 329                            : "1" (1), "m" (futex), "3" (0), "0" (private)     \
 330                            : "cx", "r11", "cc", "memory");                    \
 331     })                                                                        \
 332
 333 #define lll_robust_lock(futex, id, private) \
 334   ({ int __ret, ignore1, ignore2;                                             \
 335     __asm__ __volatile__ (LOCK_INSTR "cmpxchgl %4, %2\n\t"                            \
 336                       "jnz 1f\n\t"                                            \
 337                       ".subsection 1\n\t"                                     \
 338                       ".type _L_robust_lock_%=, @function\n"                  \
 339                       "_L_robust_lock_%=:\n"                                  \
 340                       "1:\tleaq %2, %%rdi\n"                                  \
 341                       "2:\tsubq $128, %%rsp\n"                                \
 342                       "3:\tcallq __lll_robust_lock_wait\n"                    \
 343                       "4:\taddq $128, %%rsp\n"                                \
 344                       "5:\tjmp 24f\n"                                         \
 345                       "6:\t.size _L_robust_lock_%=, 6b-1b\n\t"                \
 346                       ".previous\n"                                           \
 347                       LLL_STUB_UNWIND_INFO_5                                  \
 348                       "24:"                                                   \
 349                       : "=S" (ignore1), "=D" (ignore2), "=m" (futex),         \
 350                         "=a" (__ret)                                          \
 351                       : "1" (id), "m" (futex), "3" (0), "0" (private)         \
 352                       : "cx", "r11", "cc", "memory");                         \
 353     __ret; })
 354
 355 #define lll_cond_lock(futex, private) \
 356   (void)                                                                      \
 357     ({ int ignore1, ignore2, ignore3;                                         \
 358        __asm__ __volatile__ (LOCK_INSTR "cmpxchgl %4, %2\n\t"                 \
 359                          "jnz 1f\n\t"                                         \
 360                          ".subsection 1\n\t"                                  \
 361                          ".type _L_cond_lock_%=, @function\n"                 \
 362                          "_L_cond_lock_%=:\n"                                 \
 363                          "1:\tleaq %2, %%rdi\n"                               \
 364                          "2:\tsubq $128, %%rsp\n"                             \
 365                          "3:\tcallq __lll_lock_wait\n"                        \
 366                          "4:\taddq $128, %%rsp\n"                             \
 367                          "5:\tjmp 24f\n"                                      \
 368                          "6:\t.size _L_cond_lock_%=, 6b-1b\n\t"               \
 369                          ".previous\n"                                        \
 370                          LLL_STUB_UNWIND_INFO_5                               \
 371                          "24:"                                                \
 372                          : "=S" (ignore1), "=D" (ignore2), "=m" (futex),      \
 373                            "=a" (ignore3)                                     \
 374                          : "1" (2), "m" (futex), "3" (0), "0" (private)       \
 375                          : "cx", "r11", "cc", "memory");                      \
 376     })
 377
 378 #define lll_robust_cond_lock(futex, id, private) \
 379   ({ int __ret, ignore1, ignore2;                                             \
 380     __asm__ __volatile__ (LOCK_INSTR "cmpxchgl %4, %2\n\t"                            \
 381                       "jnz 1f\n\t"                                            \
 382                       ".subsection 1\n\t"                                     \
 383                       ".type _L_robust_cond_lock_%=, @function\n"             \
 384                       "_L_robust_cond_lock_%=:\n"                             \
 385                       "1:\tleaq %2, %%rdi\n"                                  \
 386                       "2:\tsubq $128, %%rsp\n"                                \
 387                       "3:\tcallq __lll_robust_lock_wait\n"                    \
 388                       "4:\taddq $128, %%rsp\n"                                \
 389                       "5:\tjmp 24f\n"                                         \
 390                       "6:\t.size _L_robust_cond_lock_%=, 6b-1b\n\t"           \
 391                       ".previous\n"                                           \
 392                       LLL_STUB_UNWIND_INFO_5                                  \
 393                       "24:"                                                   \
 394                       : "=S" (ignore1), "=D" (ignore2), "=m" (futex),         \
 395                         "=a" (__ret)                                          \
 396                       : "1" (id | FUTEX_WAITERS), "m" (futex), "3" (0),       \
 397                         "0" (private)                                         \
 398                       : "cx", "r11", "cc", "memory");                         \
 399     __ret; })
 400
 401 #define lll_timedlock(futex, timeout, private) \
 402   ({ int __ret, ignore1, ignore2, ignore3;                                    \
 403      __asm__ __volatile__ (LOCK_INSTR "cmpxchgl %1, %4\n\t"                           \
 404                        "jnz 1f\n\t"                                           \
 405                        ".subsection 1\n\t"                                    \
 406                        ".type _L_timedlock_%=, @function\n"                   \
 407                        "_L_timedlock_%=:\n"                                   \
 408                        "1:\tleaq %4, %%rdi\n"                                 \
 409                        "0:\tmovq %8, %%rdx\n"                                 \
 410                        "2:\tsubq $128, %%rsp\n"                               \
 411                        "3:\tcallq __lll_timedlock_wait\n"                     \
 412                        "4:\taddq $128, %%rsp\n"                               \
 413                        "5:\tjmp 24f\n"                                        \
 414                        "6:\t.size _L_timedlock_%=, 6b-1b\n\t"                 \
 415                        ".previous\n"                                          \
 416                        LLL_STUB_UNWIND_INFO_6                                 \
 417                        "24:"                                                  \
 418                        : "=a" (__ret), "=D" (ignore1), "=S" (ignore2),        \
 419                          "=&d" (ignore3), "=m" (futex)                        \
 420                        : "0" (0), "1" (1), "m" (futex), "m" (timeout),        \
 421                          "2" (private)                                        \
 422                        : "memory", "cx", "cc", "r10", "r11");                 \
 423      __ret; })
 424
 425 #define lll_robust_timedlock(futex, timeout, id, private) \
 426   ({ int __ret, ignore1, ignore2, ignore3;                                    \
 427      __asm__ __volatile__ (LOCK_INSTR "cmpxchgl %1, %4\n\t"                           \
 428                        "jnz 1f\n\t"                                           \
 429                        ".subsection 1\n\t"                                    \
 430                        ".type _L_robust_timedlock_%=, @function\n"            \
 431                        "_L_robust_timedlock_%=:\n"                            \
 432                        "1:\tleaq %4, %%rdi\n"                                 \
 433                        "0:\tmovq %8, %%rdx\n"                                 \
 434                        "2:\tsubq $128, %%rsp\n"                               \
 435                        "3:\tcallq __lll_robust_timedlock_wait\n"              \
 436                        "4:\taddq $128, %%rsp\n"                               \
 437                        "5:\tjmp 24f\n"                                        \
 438                        "6:\t.size _L_robust_timedlock_%=, 6b-1b\n\t"          \
 439                        ".previous\n"                                          \
 440                        LLL_STUB_UNWIND_INFO_6                                 \
 441                        "24:"                                                  \
 442                        : "=a" (__ret), "=D" (ignore1), "=S" (ignore2),       \
 443                          "=&d" (ignore3), "=m" (futex)                        \
 444                        : "0" (0), "1" (id), "m" (futex), "m" (timeout),       \
 445                          "2" (private)                                        \
 446                        : "memory", "cx", "cc", "r10", "r11");                 \
 447      __ret; })
 448
 449 #if defined NOT_IN_libc || defined UP
 450 # define __lll_unlock_asm_start LOCK_INSTR "decl %0\n\t"                      \
 451                                 "jne 1f\n\t"
 452 #else
 453 # define __lll_unlock_asm_start "cmpl $0, __libc_multiple_threads(%%rip)\n\t" \
 454                                 "je 0f\n\t"                                   \
 455                                 "lock; decl %0\n\t"                           \
 456                                 "jne 1f\n\t"                                  \
 457                                 "jmp 24f\n\t"                                 \
 458                                 "0:\tdecl %0\n\t"                             \
 459                                 "jne 1f\n\t"
 460 #endif
 461
 462 #define lll_unlock(futex, private) \
 463   (void)                                                                      \
 464     ({ int ignore;                                                            \
 465        if (__builtin_constant_p (private) && (private) == LLL_PRIVATE)        \
 466          __asm__ __volatile__ (__lll_unlock_asm_start                         \
 467                            ".subsection 1\n\t"                                \
 468                            ".type _L_unlock_%=, @function\n"                  \
 469                            "_L_unlock_%=:\n"                                  \
 470                            "1:\tleaq %0, %%rdi\n"                             \
 471                            "2:\tsubq $128, %%rsp\n"                           \
 472                            "3:\tcallq __lll_unlock_wake_private\n"            \
 473                            "4:\taddq $128, %%rsp\n"                           \
 474                            "5:\tjmp 24f\n"                                    \
 475                            "6:\t.size _L_unlock_%=, 6b-1b\n\t"                \
 476                            ".previous\n"                                      \
 477                            LLL_STUB_UNWIND_INFO_5                             \
 478                            "24:"                                              \
 479                            : "=m" (futex), "=&D" (ignore)                     \
 480                            : "m" (futex)                                      \
 481                            : "ax", "cx", "r11", "cc", "memory");              \
 482        else                                                                   \
 483          __asm__ __volatile__ (__lll_unlock_asm_start                         \
 484                            ".subsection 1\n\t"                                \
 485                            ".type _L_unlock_%=, @function\n"                  \
 486                            "_L_unlock_%=:\n"                                  \
 487                            "1:\tleaq %0, %%rdi\n"                             \
 488                            "2:\tsubq $128, %%rsp\n"                           \
 489                            "3:\tcallq __lll_unlock_wake\n"                    \
 490                            "4:\taddq $128, %%rsp\n"                           \
 491                            "5:\tjmp 24f\n"                                    \
 492                            "6:\t.size _L_unlock_%=, 6b-1b\n\t"                \
 493                            ".previous\n"                                      \
 494                            LLL_STUB_UNWIND_INFO_5                             \
 495                            "24:"                                              \
 496                            : "=m" (futex), "=&D" (ignore)                     \
 497                            : "m" (futex), "S" (private)                       \
 498                            : "ax", "cx", "r11", "cc", "memory");              \
 499     })
 500
 501 #define lll_robust_unlock(futex, private) \
 502   do                                                                          \
 503     {                                                                         \
 504       int ignore;                                                             \
 505       __asm__ __volatile__ (LOCK_INSTR "andl %2, %0\n\t"                              \
 506                         "jne 1f\n\t"                                          \
 507                         ".subsection 1\n\t"                                   \
 508                         ".type _L_robust_unlock_%=, @function\n"              \
 509                         "_L_robust_unlock_%=:\n"                              \
 510                         "1:\tleaq %0, %%rdi\n"                                \
 511                         "2:\tsubq $128, %%rsp\n"                              \
 512                         "3:\tcallq __lll_unlock_wake\n"                       \
 513                         "4:\taddq $128, %%rsp\n"                              \
 514                         "5:\tjmp 24f\n"                                       \
 515                         "6:\t.size _L_robust_unlock_%=, 6b-1b\n\t"            \
 516                         ".previous\n"                                         \
 517                         LLL_STUB_UNWIND_INFO_5                                \
 518                         "24:"                                                 \
 519                         : "=m" (futex), "=&D" (ignore)                        \
 520                         : "i" (FUTEX_WAITERS), "m" (futex),                   \
 521                           "S" (private)                                       \
 522                         : "ax", "cx", "r11", "cc", "memory");                 \
 523     }                                                                         \
 524   while (0)
 525
 526 #define lll_robust_dead(futex, private) \
 527   do                                                                          \
 528     {                                                                         \
 529       int ignore;                                                             \
 530       __asm__ __volatile__ (LOCK_INSTR "orl %3, (%2)\n\t"                             \
 531                         "syscall"                                             \
 532                         : "=m" (futex), "=a" (ignore)                         \
 533                         : "D" (&(futex)), "i" (FUTEX_OWNER_DIED),             \
 534                           "S" (__lll_private_flag (FUTEX_WAKE, private)),     \
 535                           "1" (__NR_futex), "d" (1)                           \
 536                         : "cx", "r11", "cc", "memory");                       \
 537     }                                                                         \
 538   while (0)
 539
 540 /* Returns non-zero if error happened, zero if success.  */
 541 #define lll_futex_requeue(ftx, nr_wake, nr_move, mutex, val, private) \
 542   ({ int __res;                                                               \
 543      register int __nr_move __asm__ ("r10") = nr_move;                        \
 544      register void *__mutex __asm__ ("r8") = mutex;                           \
 545      register int __val __asm__ ("r9") = val;                                 \
 546      __asm__ __volatile__ ("syscall"                                          \
 547                        : "=a" (__res)                                         \
 548                        : "0" (__NR_futex), "D" ((void *) ftx),                \
 549                          "S" (__lll_private_flag (FUTEX_CMP_REQUEUE,          \
 550                                                   private)), "d" (nr_wake),   \
 551                          "r" (__nr_move), "r" (__mutex), "r" (__val)          \
 552                        : "cx", "r11", "cc", "memory");                        \
 553      __res < 0; })
 554
 555 #define lll_islocked(futex) \
 556   (futex != LLL_LOCK_INITIALIZER)
 557
 558
 559 /* The kernel notifies a process with uses CLONE_CLEARTID via futex
 560    wakeup when the clone terminates.  The memory location contains the
 561    thread ID while the clone is running and is reset to zero
 562    afterwards.
 563
 564    The macro parameter must not have any side effect.  */
 565 #define lll_wait_tid(tid) \
 566   do {                                                                        \
 567     int __ignore;                                                             \
 568     register __typeof (tid) _tid __asm__ ("edx") = (tid);                             \
 569     if (_tid != 0)                                                            \
 570       __asm__ __volatile__ ("xorq %%r10, %%r10\n\t"                                   \
 571                         "1:\tmovq %2, %%rax\n\t"                              \
 572                         "syscall\n\t"                                         \
 573                         "cmpl $0, (%%rdi)\n\t"                                \
 574                         "jne 1b"                                              \
 575                         : "=&a" (__ignore)                                    \
 576                         : "S" (FUTEX_WAIT), "i" (SYS_futex), "D" (&tid),      \
 577                           "d" (_tid)                                          \
 578                         : "memory", "cc", "r10", "r11", "cx");                \
 579   } while (0)
 580
 581 extern int __lll_timedwait_tid (int *tid, const struct timespec *abstime)
 582      attribute_hidden;
 583 #define lll_timedwait_tid(tid, abstime) \
 584   ({                                                                          \
 585     int __ret = 0;                                                            \
 586     if (tid != 0)                                                             \
 587       {                                                                       \
 588         if (abstime->tv_nsec < 0 || abstime->tv_nsec >= 1000000000)           \
 589           __ret = EINVAL;                                                     \
 590         else                                                                  \
 591           __ret = __lll_timedwait_tid (&tid, abstime);                \
 592       }                                                                       \
 593     __ret; })
 594
 595 #endif  /* !__ASSEMBLER__ */
 596
 597 #endif  /* lowlevellock.h */