libpthread/nptl/sysdeps/unix/sysv/linux/x86_64/lowlevellock.h

   1 /* Copyright (C) 2002-2004, 2006-2008, 2009 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3    Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, write to the Free
  17    Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA
  18    02111-1307 USA.  */
  19
  20 #ifndef _LOWLEVELLOCK_H
  21 #define _LOWLEVELLOCK_H 1
  22
  23 #ifndef __ASSEMBLER__
  24 # include <time.h>
  25 # include <sys/param.h>
  26 # include <bits/pthreadtypes.h>
  27 # include <bits/kernel-features.h>
  28 # include <tcb-offsets.h>
  29
  30 # ifndef LOCK_INSTR
  31 #  ifdef UP
  32 #   define LOCK_INSTR   /* nothing */
  33 #  else
  34 #   define LOCK_INSTR "lock;"
  35 #  endif
  36 # endif
  37 #else
  38 # ifndef LOCK
  39 #  ifdef UP
  40 #   define LOCK
  41 #  else
  42 #   define LOCK lock
  43 #  endif
  44 # endif
  45 #endif
  46
  47 #define FUTEX_WAIT              0
  48 #define FUTEX_WAKE              1
  49 #define FUTEX_CMP_REQUEUE       4
  50 #define FUTEX_WAKE_OP           5
  51 #define FUTEX_LOCK_PI           6
  52 #define FUTEX_UNLOCK_PI         7
  53 #define FUTEX_TRYLOCK_PI        8
  54 #define FUTEX_WAIT_BITSET       9
  55 #define FUTEX_WAKE_BITSET       10
  56 #define FUTEX_WAIT_REQUEUE_PI   11
  57 #define FUTEX_CMP_REQUEUE_PI    12
  58 #define FUTEX_PRIVATE_FLAG      128
  59 #define FUTEX_CLOCK_REALTIME    256
  60
  61 #define FUTEX_BITSET_MATCH_ANY  0xffffffff
  62
  63 #define FUTEX_OP_CLEAR_WAKE_IF_GT_ONE   ((4 << 24) | 1)
  64
  65 /* Values for 'private' parameter of locking macros.  Yes, the
  66    definition seems to be backwards.  But it is not.  The bit will be
  67    reversed before passing to the system call.  */
  68 #define LLL_PRIVATE     0
  69 #define LLL_SHARED      FUTEX_PRIVATE_FLAG
  70
  71 #ifndef __ASSEMBLER__
  72
  73 #if !defined NOT_IN_libc || defined IS_IN_rtld
  74 /* In libc.so or ld.so all futexes are private.  */
  75 # ifdef __ASSUME_PRIVATE_FUTEX
  76 #  define __lll_private_flag(fl, private) \
  77   ((fl) | FUTEX_PRIVATE_FLAG)
  78 # else
  79 #  define __lll_private_flag(fl, private) \
  80   ((fl) | THREAD_GETMEM (THREAD_SELF, header.private_futex))
  81 # endif
  82 #else
  83 # ifdef __ASSUME_PRIVATE_FUTEX
  84 #  define __lll_private_flag(fl, private) \
  85   (((fl) | FUTEX_PRIVATE_FLAG) ^ (private))
  86 # else
  87 #  define __lll_private_flag(fl, private) \
  88   (__builtin_constant_p (private)                                             \
  89    ? ((private) == 0                                                          \
  90       ? ((fl) | THREAD_GETMEM (THREAD_SELF, header.private_futex))            \
  91       : (fl))                                                                 \
  92    : ({ unsigned int __fl = ((private) ^ FUTEX_PRIVATE_FLAG);                 \
  93         __asm__ ("andl %%fs:%P1, %0" : "+r" (__fl)                                    \
  94              : "i" (offsetof (struct pthread, header.private_futex)));        \
  95         __fl | (fl); }))
  96 # endif
  97 #endif
  98
  99 /* Initializer for lock.  */
 100 #define LLL_LOCK_INITIALIZER            (0)
 101 #define LLL_LOCK_INITIALIZER_LOCKED     (1)
 102 #define LLL_LOCK_INITIALIZER_WAITERS    (2)
 103
 104 /* Delay in spinlock loop.  */
 105 #define BUSY_WAIT_NOP     __asm__ ("rep; nop")
 106
 107
 108 #define LLL_STUB_UNWIND_INFO_START \
 109         ".section       .eh_frame,\"a\",@progbits\n"            \
 110 "7:\t"  ".long  9f-8f   # Length of Common Information Entry\n" \
 111 "8:\t"  ".long  0x0     # CIE Identifier Tag\n\t"               \
 112         ".byte  0x1     # CIE Version\n\t"                      \
 113         ".ascii \"zR\\0\"       # CIE Augmentation\n\t"         \
 114         ".uleb128 0x1   # CIE Code Alignment Factor\n\t"        \
 115         ".sleb128 -8    # CIE Data Alignment Factor\n\t"        \
 116         ".byte  0x10    # CIE RA Column\n\t"                    \
 117         ".uleb128 0x1   # Augmentation size\n\t"                \
 118         ".byte  0x1b    # FDE Encoding (pcrel sdata4)\n\t"      \
 119         ".byte  0x12    # DW_CFA_def_cfa_sf\n\t"                \
 120         ".uleb128 0x7\n\t"                                      \
 121         ".sleb128 16\n\t"                                       \
 122         ".align 8\n"                                            \
 123 "9:\t"  ".long  23f-10f # FDE Length\n"                         \
 124 "10:\t" ".long  10b-7b  # FDE CIE offset\n\t"                   \
 125         ".long  1b-.    # FDE initial location\n\t"             \
 126         ".long  6b-1b   # FDE address range\n\t"                \
 127         ".uleb128 0x0   # Augmentation size\n\t"                \
 128         ".byte  0x16    # DW_CFA_val_expression\n\t"            \
 129         ".uleb128 0x10\n\t"                                     \
 130         ".uleb128 12f-11f\n"                                    \
 131 "11:\t" ".byte  0x80    # DW_OP_breg16\n\t"                     \
 132         ".sleb128 4b-1b\n"
 133 #define LLL_STUB_UNWIND_INFO_END \
 134         ".byte  0x16    # DW_CFA_val_expression\n\t"            \
 135         ".uleb128 0x10\n\t"                                     \
 136         ".uleb128 14f-13f\n"                                    \
 137 "13:\t" ".byte  0x80    # DW_OP_breg16\n\t"                     \
 138         ".sleb128 4b-2b\n"                                      \
 139 "14:\t" ".byte  0x40 + (3b-2b) # DW_CFA_advance_loc\n\t"        \
 140         ".byte  0x0e    # DW_CFA_def_cfa_offset\n\t"            \
 141         ".uleb128 0\n\t"                                        \
 142         ".byte  0x16    # DW_CFA_val_expression\n\t"            \
 143         ".uleb128 0x10\n\t"                                     \
 144         ".uleb128 16f-15f\n"                                    \
 145 "15:\t" ".byte  0x80    # DW_OP_breg16\n\t"                     \
 146         ".sleb128 4b-3b\n"                                      \
 147 "16:\t" ".byte  0x40 + (4b-3b-1) # DW_CFA_advance_loc\n\t"      \
 148         ".byte  0x0e    # DW_CFA_def_cfa_offset\n\t"            \
 149         ".uleb128 128\n\t"                                      \
 150         ".byte  0x16    # DW_CFA_val_expression\n\t"            \
 151         ".uleb128 0x10\n\t"                                     \
 152         ".uleb128 20f-17f\n"                                    \
 153 "17:\t" ".byte  0x80    # DW_OP_breg16\n\t"                     \
 154         ".sleb128 19f-18f\n\t"                                  \
 155         ".byte  0x0d    # DW_OP_const4s\n"                      \
 156 "18:\t" ".4byte 4b-.\n\t"                                       \
 157         ".byte  0x1c    # DW_OP_minus\n\t"                      \
 158         ".byte  0x0d    # DW_OP_const4s\n"                      \
 159 "19:\t" ".4byte 24f-.\n\t"                                      \
 160         ".byte  0x22    # DW_OP_plus\n"                         \
 161 "20:\t" ".byte  0x40 + (5b-4b+1) # DW_CFA_advance_loc\n\t"      \
 162         ".byte  0x13    # DW_CFA_def_cfa_offset_sf\n\t"         \
 163         ".sleb128 16\n\t"                                       \
 164         ".byte  0x16    # DW_CFA_val_expression\n\t"            \
 165         ".uleb128 0x10\n\t"                                     \
 166         ".uleb128 22f-21f\n"                                    \
 167 "21:\t" ".byte  0x80    # DW_OP_breg16\n\t"                     \
 168         ".sleb128 4b-5b\n"                                      \
 169 "22:\t" ".align 8\n"                                            \
 170 "23:\t" ".previous\n"
 171
 172 /* Unwind info for
 173    1: leaq ..., %rdi
 174    2: subq $128, %rsp
 175    3: callq ...
 176    4: addq $128, %rsp
 177    5: jmp 24f
 178    6:
 179    snippet.  */
 180 #define LLL_STUB_UNWIND_INFO_5 \
 181 LLL_STUB_UNWIND_INFO_START                                      \
 182 "12:\t" ".byte  0x40 + (2b-1b) # DW_CFA_advance_loc\n\t"        \
 183 LLL_STUB_UNWIND_INFO_END
 184
 185 /* Unwind info for
 186    1: leaq ..., %rdi
 187    0: movq ..., %rdx
 188    2: subq $128, %rsp
 189    3: callq ...
 190    4: addq $128, %rsp
 191    5: jmp 24f
 192    6:
 193    snippet.  */
 194 #define LLL_STUB_UNWIND_INFO_6 \
 195 LLL_STUB_UNWIND_INFO_START                                      \
 196 "12:\t" ".byte  0x40 + (0b-1b) # DW_CFA_advance_loc\n\t"        \
 197         ".byte  0x16    # DW_CFA_val_expression\n\t"            \
 198         ".uleb128 0x10\n\t"                                     \
 199         ".uleb128 26f-25f\n"                                    \
 200 "25:\t" ".byte  0x80    # DW_OP_breg16\n\t"                     \
 201         ".sleb128 4b-0b\n"                                      \
 202 "26:\t" ".byte  0x40 + (2b-0b) # DW_CFA_advance_loc\n\t"        \
 203 LLL_STUB_UNWIND_INFO_END
 204
 205
 206 #define lll_futex_wait(futex, val, private) \
 207   lll_futex_timed_wait(futex, val, NULL, private)
 208
 209
 210 #define lll_futex_timed_wait(futex, val, timeout, private) \
 211   ({                                                                          \
 212     register const struct timespec *__to __asm__ ("r10") = timeout;           \
 213     int __status;                                                             \
 214     register __typeof (val) _val __asm__ ("edx") = (val);                             \
 215     __asm__ __volatile__ ("syscall"                                                   \
 216                       : "=a" (__status)                                       \
 217                       : "0" (SYS_futex), "D" (futex),                         \
 218                         "S" (__lll_private_flag (FUTEX_WAIT, private)),       \
 219                         "d" (_val), "r" (__to)                                \
 220                       : "memory", "cc", "r11", "cx");                         \
 221     __status;                                                                 \
 222   })
 223
 224
 225 #define lll_futex_wake(futex, nr, private) \
 226   do {                                                                        \
 227     int __ignore;                                                             \
 228     register __typeof (nr) _nr __asm__ ("edx") = (nr);                        \
 229     __asm__ __volatile__ ("syscall"                                                   \
 230                       : "=a" (__ignore)                                       \
 231                       : "0" (SYS_futex), "D" (futex),                         \
 232                         "S" (__lll_private_flag (FUTEX_WAKE, private)),       \
 233                         "d" (_nr)                                             \
 234                       : "memory", "cc", "r10", "r11", "cx");                  \
 235   } while (0)
 236
 237
 238 /* NB: in the lll_trylock macro we simply return the value in %eax
 239    after the cmpxchg instruction.  In case the operation succeded this
 240    value is zero.  In case the operation failed, the cmpxchg instruction
 241    has loaded the current value of the memory work which is guaranteed
 242    to be nonzero.  */
 243 #if defined NOT_IN_libc || defined UP
 244 # define __lll_trylock_asm LOCK_INSTR "cmpxchgl %2, %1"
 245 #else
 246 # define __lll_trylock_asm "cmpl $0, __libc_multiple_threads(%%rip)\n\t"      \
 247                            "je 0f\n\t"                                        \
 248                            "lock; cmpxchgl %2, %1\n\t"                        \
 249                            "jmp 1f\n\t"                                       \
 250                            "0:\tcmpxchgl %2, %1\n\t"                          \
 251                            "1:"
 252 #endif
 253
 254 #define lll_trylock(futex) \
 255   ({ int ret;                                                                 \
 256      __asm__ __volatile__ (__lll_trylock_asm                                  \
 257                        : "=a" (ret), "=m" (futex)                             \
 258                        : "r" (LLL_LOCK_INITIALIZER_LOCKED), "m" (futex),      \
 259                          "0" (LLL_LOCK_INITIALIZER)                           \
 260                        : "memory");                                           \
 261      ret; })
 262
 263 #define lll_robust_trylock(futex, id) \
 264   ({ int ret;                                                                 \
 265      __asm__ __volatile__ (LOCK_INSTR "cmpxchgl %2, %1"                       \
 266                        : "=a" (ret), "=m" (futex)                             \
 267                        : "r" (id), "m" (futex), "0" (LLL_LOCK_INITIALIZER)    \
 268                        : "memory");                                           \
 269      ret; })
 270
 271 #define lll_cond_trylock(futex) \
 272   ({ int ret;                                                                 \
 273      __asm__ __volatile__ (LOCK_INSTR "cmpxchgl %2, %1"                       \
 274                        : "=a" (ret), "=m" (futex)                             \
 275                        : "r" (LLL_LOCK_INITIALIZER_WAITERS),                  \
 276                          "m" (futex), "0" (LLL_LOCK_INITIALIZER)              \
 277                        : "memory");                                           \
 278      ret; })
 279
 280 #if defined NOT_IN_libc || defined UP
 281 # define __lll_lock_asm_start LOCK_INSTR "cmpxchgl %4, %2\n\t"                \
 282                               "jnz 1f\n\t"
 283 #else
 284 # define __lll_lock_asm_start "cmpl $0, __libc_multiple_threads(%%rip)\n\t"   \
 285                               "je 0f\n\t"                                     \
 286                               "lock; cmpxchgl %4, %2\n\t"                     \
 287                               "jnz 1f\n\t"                                    \
 288                               "jmp 24f\n"                                     \
 289                               "0:\tcmpxchgl %4, %2\n\t"                       \
 290                               "jnz 1f\n\t"
 291 #endif
 292
 293 #define lll_lock(futex, private) \
 294   (void)                                                                      \
 295     ({ int ignore1, ignore2, ignore3;                                         \
 296        if (__builtin_constant_p (private) && (private) == LLL_PRIVATE)        \
 297          __asm__ __volatile__ (__lll_lock_asm_start                                   \
 298                            ".subsection 1\n\t"                                \
 299                            ".type _L_lock_%=, @function\n"                    \
 300                            "_L_lock_%=:\n"                                    \
 301                            "1:\tleaq %2, %%rdi\n"                             \
 302                            "2:\tsubq $128, %%rsp\n"                           \
 303                            "3:\tcallq __lll_lock_wait_private\n"              \
 304                            "4:\taddq $128, %%rsp\n"                           \
 305                            "5:\tjmp 24f\n"                                    \
 306                            "6:\t.size _L_lock_%=, 6b-1b\n\t"                  \
 307                            ".previous\n"                                      \
 308                            LLL_STUB_UNWIND_INFO_5                             \
 309                            "24:"                                              \
 310                            : "=S" (ignore1), "=&D" (ignore2), "=m" (futex),   \
 311                              "=a" (ignore3)                                   \
 312                            : "0" (1), "m" (futex), "3" (0)                    \
 313                            : "cx", "r11", "cc", "memory");                    \
 314        else                                                                   \
 315          __asm__ __volatile__ (__lll_lock_asm_start                                   \
 316                            ".subsection 1\n\t"                                \
 317                            ".type _L_lock_%=, @function\n"                    \
 318                            "_L_lock_%=:\n"                                    \
 319                            "1:\tleaq %2, %%rdi\n"                             \
 320                            "2:\tsubq $128, %%rsp\n"                           \
 321                            "3:\tcallq __lll_lock_wait\n"                      \
 322                            "4:\taddq $128, %%rsp\n"                           \
 323                            "5:\tjmp 24f\n"                                    \
 324                            "6:\t.size _L_lock_%=, 6b-1b\n\t"                  \
 325                            ".previous\n"                                      \
 326                            LLL_STUB_UNWIND_INFO_5                             \
 327                            "24:"                                              \
 328                            : "=S" (ignore1), "=D" (ignore2), "=m" (futex),    \
 329                              "=a" (ignore3)                                   \
 330                            : "1" (1), "m" (futex), "3" (0), "0" (private)     \
 331                            : "cx", "r11", "cc", "memory");                    \
 332     })                                                                        \
 333
 334 #define lll_robust_lock(futex, id, private) \
 335   ({ int result, ignore1, ignore2;                                            \
 336     __asm__ __volatile__ (LOCK_INSTR "cmpxchgl %4, %2\n\t"                            \
 337                       "jnz 1f\n\t"                                            \
 338                       ".subsection 1\n\t"                                     \
 339                       ".type _L_robust_lock_%=, @function\n"                  \
 340                       "_L_robust_lock_%=:\n"                                  \
 341                       "1:\tleaq %2, %%rdi\n"                                  \
 342                       "2:\tsubq $128, %%rsp\n"                                \
 343                       "3:\tcallq __lll_robust_lock_wait\n"                    \
 344                       "4:\taddq $128, %%rsp\n"                                \
 345                       "5:\tjmp 24f\n"                                         \
 346                       "6:\t.size _L_robust_lock_%=, 6b-1b\n\t"                \
 347                       ".previous\n"                                           \
 348                       LLL_STUB_UNWIND_INFO_5                                  \
 349                       "24:"                                                   \
 350                       : "=S" (ignore1), "=D" (ignore2), "=m" (futex),         \
 351                         "=a" (result)                                         \
 352                       : "1" (id), "m" (futex), "3" (0), "0" (private)         \
 353                       : "cx", "r11", "cc", "memory");                         \
 354     result; })
 355
 356 #define lll_cond_lock(futex, private) \
 357   (void)                                                                      \
 358     ({ int ignore1, ignore2, ignore3;                                         \
 359        __asm__ __volatile__ (LOCK_INSTR "cmpxchgl %4, %2\n\t"                 \
 360                          "jnz 1f\n\t"                                         \
 361                          ".subsection 1\n\t"                                  \
 362                          ".type _L_cond_lock_%=, @function\n"                 \
 363                          "_L_cond_lock_%=:\n"                                 \
 364                          "1:\tleaq %2, %%rdi\n"                               \
 365                          "2:\tsubq $128, %%rsp\n"                             \
 366                          "3:\tcallq __lll_lock_wait\n"                        \
 367                          "4:\taddq $128, %%rsp\n"                             \
 368                          "5:\tjmp 24f\n"                                      \
 369                          "6:\t.size _L_cond_lock_%=, 6b-1b\n\t"               \
 370                          ".previous\n"                                        \
 371                          LLL_STUB_UNWIND_INFO_5                               \
 372                          "24:"                                                \
 373                          : "=S" (ignore1), "=D" (ignore2), "=m" (futex),      \
 374                            "=a" (ignore3)                                     \
 375                          : "1" (2), "m" (futex), "3" (0), "0" (private)       \
 376                          : "cx", "r11", "cc", "memory");                      \
 377     })
 378
 379 #define lll_robust_cond_lock(futex, id, private) \
 380   ({ int result, ignore1, ignore2;                                            \
 381     __asm__ __volatile__ (LOCK_INSTR "cmpxchgl %4, %2\n\t"                            \
 382                       "jnz 1f\n\t"                                            \
 383                       ".subsection 1\n\t"                                     \
 384                       ".type _L_robust_cond_lock_%=, @function\n"             \
 385                       "_L_robust_cond_lock_%=:\n"                             \
 386                       "1:\tleaq %2, %%rdi\n"                                  \
 387                       "2:\tsubq $128, %%rsp\n"                                \
 388                       "3:\tcallq __lll_robust_lock_wait\n"                    \
 389                       "4:\taddq $128, %%rsp\n"                                \
 390                       "5:\tjmp 24f\n"                                         \
 391                       "6:\t.size _L_robust_cond_lock_%=, 6b-1b\n\t"           \
 392                       ".previous\n"                                           \
 393                       LLL_STUB_UNWIND_INFO_5                                  \
 394                       "24:"                                                   \
 395                       : "=S" (ignore1), "=D" (ignore2), "=m" (futex),         \
 396                         "=a" (result)                                         \
 397                       : "1" (id | FUTEX_WAITERS), "m" (futex), "3" (0),       \
 398                         "0" (private)                                         \
 399                       : "cx", "r11", "cc", "memory");                         \
 400     result; })
 401
 402 #define lll_timedlock(futex, timeout, private) \
 403   ({ int result, ignore1, ignore2, ignore3;                                   \
 404      __asm__ __volatile__ (LOCK_INSTR "cmpxchgl %1, %4\n\t"                           \
 405                        "jnz 1f\n\t"                                           \
 406                        ".subsection 1\n\t"                                    \
 407                        ".type _L_timedlock_%=, @function\n"                   \
 408                        "_L_timedlock_%=:\n"                                   \
 409                        "1:\tleaq %4, %%rdi\n"                                 \
 410                        "0:\tmovq %8, %%rdx\n"                                 \
 411                        "2:\tsubq $128, %%rsp\n"                               \
 412                        "3:\tcallq __lll_timedlock_wait\n"                     \
 413                        "4:\taddq $128, %%rsp\n"                               \
 414                        "5:\tjmp 24f\n"                                        \
 415                        "6:\t.size _L_timedlock_%=, 6b-1b\n\t"                 \
 416                        ".previous\n"                                          \
 417                        LLL_STUB_UNWIND_INFO_6                                 \
 418                        "24:"                                                  \
 419                        : "=a" (result), "=D" (ignore1), "=S" (ignore2),       \
 420                          "=&d" (ignore3), "=m" (futex)                        \
 421                        : "0" (0), "1" (1), "m" (futex), "m" (timeout),        \
 422                          "2" (private)                                        \
 423                        : "memory", "cx", "cc", "r10", "r11");                 \
 424      result; })
 425
 426 #define lll_robust_timedlock(futex, timeout, id, private) \
 427   ({ int result, ignore1, ignore2, ignore3;                                   \
 428      __asm__ __volatile__ (LOCK_INSTR "cmpxchgl %1, %4\n\t"                           \
 429                        "jnz 1f\n\t"                                           \
 430                        ".subsection 1\n\t"                                    \
 431                        ".type _L_robust_timedlock_%=, @function\n"            \
 432                        "_L_robust_timedlock_%=:\n"                            \
 433                        "1:\tleaq %4, %%rdi\n"                                 \
 434                        "0:\tmovq %8, %%rdx\n"                                 \
 435                        "2:\tsubq $128, %%rsp\n"                               \
 436                        "3:\tcallq __lll_robust_timedlock_wait\n"              \
 437                        "4:\taddq $128, %%rsp\n"                               \
 438                        "5:\tjmp 24f\n"                                        \
 439                        "6:\t.size _L_robust_timedlock_%=, 6b-1b\n\t"          \
 440                        ".previous\n"                                          \
 441                        LLL_STUB_UNWIND_INFO_6                                 \
 442                        "24:"                                                  \
 443                        : "=a" (result), "=D" (ignore1), "=S" (ignore2),       \
 444                          "=&d" (ignore3), "=m" (futex)                        \
 445                        : "0" (0), "1" (id), "m" (futex), "m" (timeout),       \
 446                          "2" (private)                                        \
 447                        : "memory", "cx", "cc", "r10", "r11");                 \
 448      result; })
 449
 450 #if defined NOT_IN_libc || defined UP
 451 # define __lll_unlock_asm_start LOCK_INSTR "decl %0\n\t"                      \
 452                                 "jne 1f\n\t"
 453 #else
 454 # define __lll_unlock_asm_start "cmpl $0, __libc_multiple_threads(%%rip)\n\t" \
 455                                 "je 0f\n\t"                                   \
 456                                 "lock; decl %0\n\t"                           \
 457                                 "jne 1f\n\t"                                  \
 458                                 "jmp 24f\n\t"                                 \
 459                                 "0:\tdecl %0\n\t"                             \
 460                                 "jne 1f\n\t"
 461 #endif
 462
 463 #define lll_unlock(futex, private) \
 464   (void)                                                                      \
 465     ({ int ignore;                                                            \
 466        if (__builtin_constant_p (private) && (private) == LLL_PRIVATE)        \
 467          __asm__ __volatile__ (__lll_unlock_asm_start                         \
 468                            ".subsection 1\n\t"                                \
 469                            ".type _L_unlock_%=, @function\n"                  \
 470                            "_L_unlock_%=:\n"                                  \
 471                            "1:\tleaq %0, %%rdi\n"                             \
 472                            "2:\tsubq $128, %%rsp\n"                           \
 473                            "3:\tcallq __lll_unlock_wake_private\n"            \
 474                            "4:\taddq $128, %%rsp\n"                           \
 475                            "5:\tjmp 24f\n"                                    \
 476                            "6:\t.size _L_unlock_%=, 6b-1b\n\t"                \
 477                            ".previous\n"                                      \
 478                            LLL_STUB_UNWIND_INFO_5                             \
 479                            "24:"                                              \
 480                            : "=m" (futex), "=&D" (ignore)                     \
 481                            : "m" (futex)                                      \
 482                            : "ax", "cx", "r11", "cc", "memory");              \
 483        else                                                                   \
 484          __asm__ __volatile__ (__lll_unlock_asm_start                         \
 485                            ".subsection 1\n\t"                                \
 486                            ".type _L_unlock_%=, @function\n"                  \
 487                            "_L_unlock_%=:\n"                                  \
 488                            "1:\tleaq %0, %%rdi\n"                             \
 489                            "2:\tsubq $128, %%rsp\n"                           \
 490                            "3:\tcallq __lll_unlock_wake\n"                    \
 491                            "4:\taddq $128, %%rsp\n"                           \
 492                            "5:\tjmp 24f\n"                                    \
 493                            "6:\t.size _L_unlock_%=, 6b-1b\n\t"                \
 494                            ".previous\n"                                      \
 495                            LLL_STUB_UNWIND_INFO_5                             \
 496                            "24:"                                              \
 497                            : "=m" (futex), "=&D" (ignore)                     \
 498                            : "m" (futex), "S" (private)                       \
 499                            : "ax", "cx", "r11", "cc", "memory");              \
 500     })
 501
 502 #define lll_robust_unlock(futex, private) \
 503   do                                                                          \
 504     {                                                                         \
 505       int ignore;                                                             \
 506       __asm__ __volatile__ (LOCK_INSTR "andl %2, %0\n\t"                              \
 507                         "jne 1f\n\t"                                          \
 508                         ".subsection 1\n\t"                                   \
 509                         ".type _L_robust_unlock_%=, @function\n"              \
 510                         "_L_robust_unlock_%=:\n"                              \
 511                         "1:\tleaq %0, %%rdi\n"                                \
 512                         "2:\tsubq $128, %%rsp\n"                              \
 513                         "3:\tcallq __lll_unlock_wake\n"                       \
 514                         "4:\taddq $128, %%rsp\n"                              \
 515                         "5:\tjmp 24f\n"                                       \
 516                         "6:\t.size _L_robust_unlock_%=, 6b-1b\n\t"            \
 517                         ".previous\n"                                         \
 518                         LLL_STUB_UNWIND_INFO_5                                \
 519                         "24:"                                                 \
 520                         : "=m" (futex), "=&D" (ignore)                        \
 521                         : "i" (FUTEX_WAITERS), "m" (futex),                   \
 522                           "S" (private)                                       \
 523                         : "ax", "cx", "r11", "cc", "memory");                 \
 524     }                                                                         \
 525   while (0)
 526
 527 #define lll_robust_dead(futex, private) \
 528   do                                                                          \
 529     {                                                                         \
 530       int ignore;                                                             \
 531       __asm__ __volatile__ (LOCK_INSTR "orl %3, (%2)\n\t"                             \
 532                         "syscall"                                             \
 533                         : "=m" (futex), "=a" (ignore)                         \
 534                         : "D" (&(futex)), "i" (FUTEX_OWNER_DIED),             \
 535                           "S" (__lll_private_flag (FUTEX_WAKE, private)),     \
 536                           "1" (__NR_futex), "d" (1)                           \
 537                         : "cx", "r11", "cc", "memory");                       \
 538     }                                                                         \
 539   while (0)
 540
 541 /* Returns non-zero if error happened, zero if success.  */
 542 #define lll_futex_requeue(ftx, nr_wake, nr_move, mutex, val, private) \
 543   ({ int __res;                                                               \
 544      register int __nr_move __asm__ ("r10") = nr_move;                        \
 545      register void *__mutex __asm__ ("r8") = mutex;                           \
 546      register int __val __asm__ ("r9") = val;                                 \
 547      __asm__ __volatile__ ("syscall"                                          \
 548                        : "=a" (__res)                                         \
 549                        : "0" (__NR_futex), "D" ((void *) ftx),                \
 550                          "S" (__lll_private_flag (FUTEX_CMP_REQUEUE,          \
 551                                                   private)), "d" (nr_wake),   \
 552                          "r" (__nr_move), "r" (__mutex), "r" (__val)          \
 553                        : "cx", "r11", "cc", "memory");                        \
 554      __res < 0; })
 555
 556 #define lll_islocked(futex) \
 557   (futex != LLL_LOCK_INITIALIZER)
 558
 559
 560 /* The kernel notifies a process with uses CLONE_CLEARTID via futex
 561    wakeup when the clone terminates.  The memory location contains the
 562    thread ID while the clone is running and is reset to zero
 563    afterwards.
 564
 565    The macro parameter must not have any side effect.  */
 566 #define lll_wait_tid(tid) \
 567   do {                                                                        \
 568     int __ignore;                                                             \
 569     register __typeof (tid) _tid __asm__ ("edx") = (tid);                             \
 570     if (_tid != 0)                                                            \
 571       __asm__ __volatile__ ("xorq %%r10, %%r10\n\t"                                   \
 572                         "1:\tmovq %2, %%rax\n\t"                              \
 573                         "syscall\n\t"                                         \
 574                         "cmpl $0, (%%rdi)\n\t"                                \
 575                         "jne 1b"                                              \
 576                         : "=&a" (__ignore)                                    \
 577                         : "S" (FUTEX_WAIT), "i" (SYS_futex), "D" (&tid),      \
 578                           "d" (_tid)                                          \
 579                         : "memory", "cc", "r10", "r11", "cx");                \
 580   } while (0)
 581
 582 extern int __lll_timedwait_tid (int *tid, const struct timespec *abstime)
 583      attribute_hidden;
 584 #define lll_timedwait_tid(tid, abstime) \
 585   ({                                                                          \
 586     int __result = 0;                                                         \
 587     if (tid != 0)                                                             \
 588       {                                                                       \
 589         if (abstime->tv_nsec < 0 || abstime->tv_nsec >= 1000000000)           \
 590           __result = EINVAL;                                                  \
 591         else                                                                  \
 592           __result = __lll_timedwait_tid (&tid, abstime);                     \
 593       }                                                                       \
 594     __result; })
 595
 596 #endif  /* !__ASSEMBLER__ */
 597
 598 #endif  /* lowlevellock.h */