This is the mail archive of the
libc-alpha@sourceware.org
mailing list for the glibc project.
Re: [PATCH 05/14] Add lock elision to rwlocks
- From: "Carlos O'Donell" <carlos at redhat dot com>
- To: Roland McGrath <roland at hack dot frob dot com>, "Joseph S. Myers" <joseph at codesourcery dot com>, Andreas Jaeger <aj at suse dot com>
- Cc: Andi Kleen <andi at firstfloor dot org>, Torvald Riegel <triegel at redhat dot com>, Rich Felker <dalias at aerifal dot cx>, libc-alpha at sourceware dot org, Andi Kleen <ak at linux dot jf dot intel dot com>
- Date: Fri, 28 Jun 2013 13:24:56 -0400
- Subject: Re: [PATCH 05/14] Add lock elision to rwlocks
- References: <1372398717-16530-1-git-send-email-andi at firstfloor dot org> <1372398717-16530-6-git-send-email-andi at firstfloor dot org> <51CD3C79 dot 2040906 at redhat dot com>
On 06/28/2013 03:34 AM, Carlos O'Donell wrote:
> I need some consensus on this issue.
>
> In POSIX Issue 7:
> http://pubs.opengroup.org/onlinepubs/9699919799/functions/pthread_rwlock_trywrlock.html
>
> The language for pthread_rlock_wrlock and pthread_rwlock_trywrlock says
> that it "may" deadlock, or it "may" also return EDEADLK.
>
> My reading of this is that Torvald's guideline is incorrect:
> http://sourceware.org/glibc/wiki/LockElisionGuide#Compability_with_pthreads_rwlock
>
> In that wrlock/trywrlock are not *required* to have a deadlock on relock
> semantic by the standard.
>
> The same soft language of "may deadlock" is used for rdlock and thus the same
> flexibility is granted there.
>
> Therefore this patch #5 falls within the semantic guarantees of the standard
> and therefore can be considered correct.
>
> Comments?
Roland,
Do you have any comments on my rationale above?
That POSIX doesn't strictly require the wrlock to deadlock on relock,
simply that it may do so, but that is implementation defined.
Eliding a rwlock causes relock not to deadlock immediately.
Cheers,
Carlos.
>> 2013-06-24 Andi Kleen <ak@linux.intel.com>
>> Hongjiu Lu <hongjiu.lu@intel.com>
>>
>> * sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S: dito.
>> * sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S: dito
>> * sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S: dito.
>> * sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S: dito.
>> * sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S: dito.
>> * sysdeps/unix/sysv/linux/internaltypes.h (struct pthread_condattr):
>> (struct pthread_rwlockattr): Support elision flags in internal rwlock.
>> * sysdeps/unix/sysv/linux/lowlevelrwlock.sym: Add RW_ELISION.
>> * sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h: Add elision flags
>> * sysdeps/unix/sysv/linux/x86/pthread_rwlock_tryrdlock.c: New file
>> * sysdeps/unix/sysv/linux/x86/pthread_rwlock_trywrlock.c: dito.
>> * sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S: Handle elision.
>> * sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S: dito.
>> * sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S: dito.
>> * sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S: dito.
>> * sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S: dito.
>> ---
>> nptl/pthread_rwlock_tryrdlock.c | 2 +
>> nptl/pthread_rwlock_trywrlock.c | 2 +
>> .../sysv/linux/i386/i486/pthread_rwlock_rdlock.S | 65 ++++++++++++++++++++++
>> .../linux/i386/i486/pthread_rwlock_timedrdlock.S | 60 ++++++++++++++++++++
>> .../linux/i386/i486/pthread_rwlock_timedwrlock.S | 46 ++++++++++++++-
>> .../sysv/linux/i386/i486/pthread_rwlock_unlock.S | 19 +++++++
>> .../sysv/linux/i386/i486/pthread_rwlock_wrlock.S | 52 +++++++++++++++++
>> nptl/sysdeps/unix/sysv/linux/internaltypes.h | 16 +++++-
>> nptl/sysdeps/unix/sysv/linux/lowlevelrwlock.sym | 5 ++
>> .../unix/sysv/linux/x86/bits/pthreadtypes.h | 6 +-
>> .../unix/sysv/linux/x86/pthread_rwlock_tryrdlock.c | 52 +++++++++++++++++
>> .../unix/sysv/linux/x86/pthread_rwlock_trywrlock.c | 51 +++++++++++++++++
>> .../unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S | 48 ++++++++++++++++
>> .../sysv/linux/x86_64/pthread_rwlock_timedrdlock.S | 48 ++++++++++++++++
>> .../sysv/linux/x86_64/pthread_rwlock_timedwrlock.S | 32 +++++++++++
>> .../unix/sysv/linux/x86_64/pthread_rwlock_unlock.S | 17 +++++-
>> .../unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S | 31 +++++++++++
>> 17 files changed, 546 insertions(+), 6 deletions(-)
>> create mode 100644 nptl/sysdeps/unix/sysv/linux/x86/pthread_rwlock_tryrdlock.c
>> create mode 100644 nptl/sysdeps/unix/sysv/linux/x86/pthread_rwlock_trywrlock.c
>>
>> diff --git a/nptl/pthread_rwlock_tryrdlock.c b/nptl/pthread_rwlock_tryrdlock.c
>> index 935ac87..896809b 100644
>> --- a/nptl/pthread_rwlock_tryrdlock.c
>> +++ b/nptl/pthread_rwlock_tryrdlock.c
>> @@ -45,4 +45,6 @@ __pthread_rwlock_tryrdlock (pthread_rwlock_t *rwlock)
>>
>> return result;
>> }
>> +#ifndef __pthread_rwlock_tryrdlock
>> strong_alias (__pthread_rwlock_tryrdlock, pthread_rwlock_tryrdlock)
>> +#endif
>> diff --git a/nptl/pthread_rwlock_trywrlock.c b/nptl/pthread_rwlock_trywrlock.c
>> index 01754ae..6d8a8c4 100644
>> --- a/nptl/pthread_rwlock_trywrlock.c
>> +++ b/nptl/pthread_rwlock_trywrlock.c
>> @@ -38,4 +38,6 @@ __pthread_rwlock_trywrlock (pthread_rwlock_t *rwlock)
>>
>> return result;
>> }
>> +#ifndef __pthread_rwlock_trywrlock
>> strong_alias (__pthread_rwlock_trywrlock, pthread_rwlock_trywrlock)
>> +#endif
>> diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S
>> index 6c46ba6..4151b03 100644
>> --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S
>> +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S
>> @@ -21,9 +21,16 @@
>> #include <lowlevelrwlock.h>
>> #include <pthread-errnos.h>
>> #include <kernel-features.h>
>> +#include <hle.h>
>>
>> #include <stap-probe.h>
>>
>> +#ifdef PIC
>> +#define MO(x) x##@GOTOFF(%edx)
>> +#else
>> +#define MO(x) x
>> +#endif
>> +
>> .text
>>
>> .globl __pthread_rwlock_rdlock
>> @@ -43,6 +50,62 @@ __pthread_rwlock_rdlock:
>>
>> LIBC_PROBE (rdlock_entry, 1, %ebx)
>>
>> +#ifdef PIC
>> + SETUP_PIC_REG(dx)
>> + addl $_GLOBAL_OFFSET_TABLE_,%edx
>> +#endif
>> +
>> + cmpb $0,RW_ELISION(%ebx)
>> + js not_elided_rdlock
>> + jnz 2f
>> + /* zero: use default */
>> +
>> + cmpl $0,MO(__rwlock_rtm_enabled)
>> + jz not_elided_rdlock
>> +
>> +2:
>> + mov MO(__rwlock_rtm_read_retries),%ecx
>> +
>> +try_trans_rdlock:
>> + XBEGIN abort_rdlock
>> +
>> + /* Lock writer/reader free? */
>> + cmpl $0,WRITER(%ebx)
>> + jnz 1f
>> + cmpl $0,NR_READERS(%ebx)
>> + jnz 1f
>> +
>> + /* Lock is free. Run with transaction */
>> + xor %eax,%eax
>> +
>> + pop %ebx
>> + cfi_adjust_cfa_offset(-4)
>> + pop %esi
>> + cfi_adjust_cfa_offset(-4)
>> + ret
>> +
>> + /* Lock is not free. Run */
>> +1: XABORT 0xff
>> + jmp not_elided_rdlock
>> +
>> + /* Abort happened. */
>> +abort_rdlock:
>> + testl $_XABORT_CONFLICT,%eax
>> + jz not_elided_rdlock
>> +
>> + /* For a reader that aborts due a conflict retry speculation
>> + a limited number of times. This way when some reader aborts
>> + because the reader count is written the other readers will
>> + still elide, at the cost of retrying the speculation. */
>> +
>> + dec %ecx
>> + jnz try_trans_rdlock
>> +
>> + /* Otherwise we just fall back directly to the lock.
>> + Here's the place to add more adaptation. */
>> +
>> +not_elided_rdlock:
>> +
>> /* Get the lock. */
>> movl $1, %edx
>> xorl %eax, %eax
>> @@ -188,5 +251,7 @@ __pthread_rwlock_rdlock:
>> cfi_endproc
>> .size __pthread_rwlock_rdlock,.-__pthread_rwlock_rdlock
>>
>> +#ifndef __pthread_rwlock_rdlock
>> strong_alias (__pthread_rwlock_rdlock, pthread_rwlock_rdlock)
>> hidden_def (__pthread_rwlock_rdlock)
>> +#endif
>> diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S
>> index 1908f6f..e710c49 100644
>> --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S
>> +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S
>> @@ -21,7 +21,13 @@
>> #include <lowlevelrwlock.h>
>> #include <pthread-errnos.h>
>> #include <kernel-features.h>
>> +#include <hle.h>
>>
>> +#ifdef PIC
>> +#define MO(x) x##@GOTOFF(%edx)
>> +#else
>> +#define MO(x) x
>> +#endif
>>
>> .text
>>
>> @@ -48,6 +54,59 @@ pthread_rwlock_timedrdlock:
>> movl 28(%esp), %ebp
>> movl 32(%esp), %edi
>>
>> +#ifdef PIC
>> + SETUP_PIC_REG(dx)
>> + addl $_GLOBAL_OFFSET_TABLE_,%edx
>> +#endif
>> +
>> + cmpb $0,RW_ELISION(%ebp)
>> + js not_elided_trdlock
>> + jnz 2f
>> + /* zero: use default */
>> +
>> + cmpl $0,MO(__rwlock_rtm_enabled)
>> + jz not_elided_trdlock
>> +
>> +2:
>> + mov MO(__rwlock_rtm_read_retries),%ecx
>> +
>> +try_trans_trdlock:
>> + XBEGIN abort_trdlock
>> +
>> + /* Lock writer/reader free? */
>> + cmpl $0,WRITER(%ebp)
>> + jnz 1f
>> + cmpl $0,NR_READERS(%ebp)
>> + jnz 1f
>> +
>> + /* Lock is free. Run with transaction */
>> + xor %eax,%eax
>> + jmp 77f
>> +
>> +
>> + /* Lock is not free. Run */
>> +1: XABORT 0xff
>> + jmp not_elided_trdlock
>> +
>> + /* Abort happened. */
>> +abort_trdlock:
>> + testl $_XABORT_CONFLICT,%eax
>> + jz not_elided_trdlock
>> +
>> + /* For a reader that aborts due a conflict retry speculation
>> + a limited number of times. This way when some reader aborts
>> + because the reader count is written the other readers will
>> + still elide, at the cost of retrying the speculation. */
>> +
>> + dec %ecx
>> + jnz try_trans_trdlock
>> +
>> + /* Otherwise we just fall back directly to the lock.
>> + Here's the place to add more adaptation. */
>> +
>> +not_elided_trdlock:
>> +
>> +
>> /* Get the lock. */
>> movl $1, %edx
>> xorl %eax, %eax
>> @@ -158,6 +217,7 @@ pthread_rwlock_timedrdlock:
>>
>> 7: movl %edx, %eax
>>
>> +77:
>> addl $8, %esp
>> cfi_adjust_cfa_offset(-8)
>> popl %ebp
>> diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S
>> index e0fc809..c23f270 100644
>> --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S
>> +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S
>> @@ -21,7 +21,13 @@
>> #include <lowlevelrwlock.h>
>> #include <pthread-errnos.h>
>> #include <kernel-features.h>
>> +#include <hle.h>
>>
>> +#ifdef PIC
>> +#define MO(x) x##@GOTOFF(%edx)
>> +#else
>> +#define MO(x) x
>> +#endif
>>
>> .text
>>
>> @@ -48,6 +54,44 @@ pthread_rwlock_timedwrlock:
>> movl 28(%esp), %ebp
>> movl 32(%esp), %edi
>>
>> +#ifdef PIC
>> + SETUP_PIC_REG(dx)
>> + addl $_GLOBAL_OFFSET_TABLE_,%edx
>> +#endif
>> + cmpb $0,RW_ELISION(%ebp)
>> + js not_elided_twrlock
>> + jnz try_trans_wrlock
>> + /* zero: use default */
>> +
>> + cmpl $0,MO(__rwlock_rtm_enabled)
>> + jz not_elided_twrlock
>> +
>> +try_trans_wrlock:
>> + XBEGIN abort_twrlock
>> +
>> + /* Lock writer free? */
>> + cmpl $0,WRITER(%ebp)
>> + jnz 1f
>> + cmpl $0,NR_READERS(%ebp)
>> + jnz 1f
>> +
>> + /* Lock is free. Run with transaction */
>> + xor %eax,%eax
>> +
>> + jmp 77f
>> +
>> + /* Lock is not free. Run */
>> +1: XABORT 0xff
>> + jmp not_elided_twrlock
>> +
>> + /* Abort happened. */
>> +abort_twrlock:
>> +
>> + /* Otherwise we just fall back directly to the lock.
>> + Here's the place to add more adaptation. */
>> +
>> +not_elided_twrlock:
>> +
>> /* Get the lock. */
>> movl $1, %edx
>> xorl %eax, %eax
>> @@ -156,7 +200,7 @@ pthread_rwlock_timedwrlock:
>>
>> 7: movl %edx, %eax
>>
>> - addl $8, %esp
>> +77: addl $8, %esp
>> cfi_adjust_cfa_offset(-8)
>> popl %ebp
>> cfi_adjust_cfa_offset(-4)
>> diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S
>> index 708e31c..9b384bb 100644
>> --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S
>> +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S
>> @@ -20,6 +20,7 @@
>> #include <lowlevellock.h>
>> #include <lowlevelrwlock.h>
>> #include <kernel-features.h>
>> +#include <hle.h>
>>
>>
>> .text
>> @@ -38,6 +39,24 @@ __pthread_rwlock_unlock:
>>
>> movl 12(%esp), %edi
>>
>> + /* Is lock free? */
>> + cmpl $0,WRITER(%edi)
>> + jnz 1f
>> + cmpl $0,NR_READERS(%edi)
>> + jnz 1f
>> +
>> + /* Looks free. Assume transaction.
>> + If you crash here you unlocked a free lock. */
>> + XEND
>> + xor %eax,%eax
>> +
>> + pop %edi
>> + cfi_adjust_cfa_offset(-4)
>> + pop %ebx
>> + cfi_adjust_cfa_offset(-4)
>> + ret
>> +
>> +1:
>> /* Get the lock. */
>> movl $1, %edx
>> xorl %eax, %eax
>> diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S
>> index 6ea17f7..e0c3428 100644
>> --- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S
>> +++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S
>> @@ -21,9 +21,16 @@
>> #include <lowlevelrwlock.h>
>> #include <pthread-errnos.h>
>> #include <kernel-features.h>
>> +#include <hle.h>
>>
>> #include <stap-probe.h>
>>
>> +#ifdef PIC
>> +#define MO(x) x@GOTOFF(%edx)
>> +#else
>> +#define MO(x) x
>> +#endif
>> +
>> .text
>>
>> .globl __pthread_rwlock_wrlock
>> @@ -43,6 +50,49 @@ __pthread_rwlock_wrlock:
>>
>> LIBC_PROBE (wrlock_entry, 1, %ebx)
>>
>> +#ifdef PIC
>> + SETUP_PIC_REG(dx)
>> + addl $_GLOBAL_OFFSET_TABLE_,%edx
>> +#endif
>> + cmpb $0,RW_ELISION(%ebx)
>> + js not_elided_wrlock
>> + jnz try_trans_wrlock
>> + /* zero: use default */
>> +
>> + cmpl $0,MO(__rwlock_rtm_enabled)
>> + jz not_elided_wrlock
>> +
>> +try_trans_wrlock:
>> + XBEGIN abort_wrlock
>> +
>> + /* Lock writer free? */
>> + /* Ignore readers because we don't need them */
>> + cmpl $0,WRITER(%ebx)
>> + jnz 1f
>> + cmpl $0,NR_READERS(%ebx)
>> + jnz 1f
>> +
>> + /* Lock is free. Run with transaction */
>> + xor %eax,%eax
>> +
>> + pop %ebx
>> + cfi_adjust_cfa_offset(-4)
>> + pop %esi
>> + cfi_adjust_cfa_offset(-4)
>> + ret
>> +
>> + /* Lock is not free. Run */
>> +1: XABORT 0xff
>> + jmp not_elided_wrlock
>> +
>> + /* Abort happened. */
>> +abort_wrlock:
>> +
>> + /* Otherwise we just fall back directly to the lock.
>> + Here's the place to add more adaptation. */
>> +
>> +not_elided_wrlock:
>> +
>> /* Get the lock. */
>> movl $1, %edx
>> xorl %eax, %eax
>> @@ -179,5 +229,7 @@ __pthread_rwlock_wrlock:
>> cfi_endproc
>> .size __pthread_rwlock_wrlock,.-__pthread_rwlock_wrlock
>>
>> +#ifndef __pthread_rwlock_wrlock
>> strong_alias (__pthread_rwlock_wrlock, pthread_rwlock_wrlock)
>> hidden_def (__pthread_rwlock_wrlock)
>> +#endif
>> diff --git a/nptl/sysdeps/unix/sysv/linux/internaltypes.h b/nptl/sysdeps/unix/sysv/linux/internaltypes.h
>> index 699a618..70aaa40 100644
>> --- a/nptl/sysdeps/unix/sysv/linux/internaltypes.h
>> +++ b/nptl/sysdeps/unix/sysv/linux/internaltypes.h
>> @@ -20,7 +20,7 @@
>> #define _INTERNALTYPES_H 1
>>
>> #include <stdint.h>
>> -
>> +#include <endian.h>
>>
>> struct pthread_attr
>> {
>> @@ -86,7 +86,19 @@ struct pthread_condattr
>> struct pthread_rwlockattr
>> {
>> int lockkind;
>> - int pshared;
>> + /* This used to be an int pshared. We keep the pshared at the
>> + same location.
>> + PTHREAD_PROCESS_SHARED and PTHREAD_PROCESS_PRIVATE, both fit into one
>> + byte. */
>> +#if __BYTE_ORDER == __LITTLE_ENDIAN
>> + short pshared;
>> + char rw_elision;
>> + char pad;
>> +#else
>> + char rw_elision;
>> + char pad;
>> + short pshared;
>> +#endif
>> };
>>
>>
>> diff --git a/nptl/sysdeps/unix/sysv/linux/lowlevelrwlock.sym b/nptl/sysdeps/unix/sysv/linux/lowlevelrwlock.sym
>> index f50b25b..6d9d201 100644
>> --- a/nptl/sysdeps/unix/sysv/linux/lowlevelrwlock.sym
>> +++ b/nptl/sysdeps/unix/sysv/linux/lowlevelrwlock.sym
>> @@ -3,6 +3,10 @@
>> #include <bits/pthreadtypes.h>
>> #include <bits/wordsize.h>
>>
>> +#ifndef __PTHREAD_RWLOCK_ELISION
>> +#define __rw_elision __shared
>> +#endif
>> +
>> --
>>
>> MUTEX offsetof (pthread_rwlock_t, __data.__lock)
>> @@ -14,3 +18,4 @@ WRITERS_QUEUED offsetof (pthread_rwlock_t, __data.__nr_writers_queued)
>> FLAGS offsetof (pthread_rwlock_t, __data.__flags)
>> WRITER offsetof (pthread_rwlock_t, __data.__writer)
>> PSHARED offsetof (pthread_rwlock_t, __data.__shared)
>> +RW_ELISION offsetof (pthread_rwlock_t, __data.__rw_elision)
>> diff --git a/nptl/sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h
>> index 1852e07..c3325d9 100644
>> --- a/nptl/sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h
>> +++ b/nptl/sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h
>> @@ -183,12 +183,13 @@ typedef union
>> unsigned int __nr_writers_queued;
>> int __writer;
>> int __shared;
>> - unsigned long int __pad1;
>> + long __rw_elision;
>> unsigned long int __pad2;
>> /* FLAGS must stay at this position in the structure to maintain
>> binary compatibility. */
>> unsigned int __flags;
>> # define __PTHREAD_RWLOCK_INT_FLAGS_SHARED 1
>> +# define __PTHREAD_RWLOCK_ELISION 1
>> } __data;
>> # else
>> struct
>> @@ -203,8 +204,9 @@ typedef union
>> binary compatibility. */
>> unsigned char __flags;
>> unsigned char __shared;
>> - unsigned char __pad1;
>> + char __rw_elision;
>> unsigned char __pad2;
>> +# define __PTHREAD_RWLOCK_ELISION 2
>> int __writer;
>> } __data;
>> # endif
>> diff --git a/nptl/sysdeps/unix/sysv/linux/x86/pthread_rwlock_tryrdlock.c b/nptl/sysdeps/unix/sysv/linux/x86/pthread_rwlock_tryrdlock.c
>> new file mode 100644
>> index 0000000..f18295c
>> --- /dev/null
>> +++ b/nptl/sysdeps/unix/sysv/linux/x86/pthread_rwlock_tryrdlock.c
>> @@ -0,0 +1,52 @@
>> +/* pthread_rwlock_tryrdlock: Lock elision version of pthreads rwlock_tryrdlock.
>> + Copyright (C) 2013 Free Software Foundation, Inc.
>> + This file is part of the GNU C Library.
>> +
>> + The GNU C Library is free software; you can redistribute it and/or
>> + modify it under the terms of the GNU Lesser General Public
>> + License as published by the Free Software Foundation; either
>> + version 2.1 of the License, or (at your option) any later version.
>> +
>> + The GNU C Library is distributed in the hope that it will be useful,
>> + but WITHOUT ANY WARRANTY; without even the implied warranty of
>> + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
>> + Lesser General Public License for more details.
>> +
>> + You should have received a copy of the GNU Lesser General Public
>> + License along with the GNU C Library; if not, see
>> + <http://www.gnu.org/licenses/>. */
>> +#include <pthread.h>
>> +#include <pthreadP.h>
>> +#include <hle.h>
>> +#include <elision-conf.h>
>> +#include "init-arch.h"
>> +
>> +#define __pthread_rwlock_tryrdlock __full_pthread_rwlock_tryrdlock
>> +#include <nptl/pthread_rwlock_tryrdlock.c>
>> +#undef __pthread_rwlock_tryrdlock
>> +
>> +int
>> +__pthread_rwlock_tryrdlock(pthread_rwlock_t *rwlock)
>> +{
>> + unsigned status;
>> +
>> + if (rwlock->__data.__rw_elision > 0
>> + || (rwlock->__data.__rw_elision == 0 && __rwlock_rtm_enabled))
>> + {
>> + if ((status = _xbegin()) == _XBEGIN_STARTED)
>> + {
>> + if (rwlock->__data.__writer == 0
>> + && rwlock->__data.__nr_readers == 0)
>> + return 0;
>> + /* Lock was busy. Fall back to normal locking.
>> + Could also _xend here but xabort with 0xff code
>> + is more visible in the profiler. */
>> + _xabort (_ABORT_LOCK_BUSY);
>> + }
>> + /* Aborts come here */
>> + }
>> +
>> + return __full_pthread_rwlock_tryrdlock (rwlock);
>> +}
>> +
>> +strong_alias(__pthread_rwlock_tryrdlock, pthread_rwlock_tryrdlock);
>> diff --git a/nptl/sysdeps/unix/sysv/linux/x86/pthread_rwlock_trywrlock.c b/nptl/sysdeps/unix/sysv/linux/x86/pthread_rwlock_trywrlock.c
>> new file mode 100644
>> index 0000000..0f5cd3c
>> --- /dev/null
>> +++ b/nptl/sysdeps/unix/sysv/linux/x86/pthread_rwlock_trywrlock.c
>> @@ -0,0 +1,51 @@
>> +/* pthread_rwlock_trywrlock: Lock elision version of pthreads rwlock trywrlock.
>> + Copyright (C) 2013 Free Software Foundation, Inc.
>> + This file is part of the GNU C Library.
>> +
>> + The GNU C Library is free software; you can redistribute it and/or
>> + modify it under the terms of the GNU Lesser General Public
>> + License as published by the Free Software Foundation; either
>> + version 2.1 of the License, or (at your option) any later version.
>> +
>> + The GNU C Library is distributed in the hope that it will be useful,
>> + but WITHOUT ANY WARRANTY; without even the implied warranty of
>> + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
>> + Lesser General Public License for more details.
>> +
>> + You should have received a copy of the GNU Lesser General Public
>> + License along with the GNU C Library; if not, see
>> + <http://www.gnu.org/licenses/>. */
>> +#include <pthread.h>
>> +#include <pthreadP.h>
>> +#include <hle.h>
>> +#include <elision-conf.h>
>> +#include "init-arch.h"
>> +
>> +#define __pthread_rwlock_trywrlock __full_pthread_rwlock_trywrlock
>> +#include <nptl/pthread_rwlock_trywrlock.c>
>> +#undef __pthread_rwlock_trywrlock
>> +
>> +int
>> +__pthread_rwlock_trywrlock(pthread_rwlock_t *rwlock)
>> +{
>> + unsigned status;
>> +
>> + if ((rwlock->__data.__rw_elision == 0 && __rwlock_rtm_enabled)
>> + || rwlock->__data.__rw_elision > 0)
>> + {
>> + _xabort (_ABORT_NESTED_TRYLOCK);
>> + if ((status = _xbegin()) == _XBEGIN_STARTED)
>> + {
>> + if (rwlock->__data.__writer == 0
>> + && rwlock->__data.__nr_readers == 0)
>> + return 0;
>> + /* Lock was busy. Fall back to normal locking.
>> + Could also _xend here but xabort with 0xff code
>> + is more visible in the profiler. */
>> + _xabort (_ABORT_LOCK_BUSY);
>> + }
>> + }
>> +
>> + return __full_pthread_rwlock_trywrlock (rwlock);
>> +}
>> +strong_alias(__pthread_rwlock_trywrlock, pthread_rwlock_trywrlock);
>> diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S
>> index 7681818..3b335bc 100644
>> --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S
>> +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S
>> @@ -22,6 +22,7 @@
>> #include <pthread-errnos.h>
>> #include <kernel-features.h>
>> #include <stap-probe.h>
>> +#include <hle.h>
>>
>> .text
>>
>> @@ -33,6 +34,51 @@ __pthread_rwlock_rdlock:
>>
>> LIBC_PROBE (rdlock_entry, 1, %rdi)
>>
>> + cmpq $0,RW_ELISION(%rdi)
>> + js not_elided_rdlock
>> + jnz 2f
>> + /* zero: use default */
>> +
>> + cmpl $0,__rwlock_rtm_enabled(%rip)
>> + jz not_elided_rdlock
>> +
>> +2:
>> + mov __rwlock_rtm_read_retries(%rip),%esi
>> +
>> +try_trans_rdlock:
>> + XBEGIN abort_rdlock
>> +
>> + /* Lock reader/writer free? */
>> + cmpl $0,WRITER(%rdi)
>> + jnz 1f
>> + cmpl $0,NR_READERS(%rdi)
>> + jnz 1f
>> +
>> + /* Lock is free. Run with transaction */
>> + xor %eax,%eax
>> + ret
>> +
>> + /* Lock is not free. Run */
>> +1: XABORT 0xff
>> + jmp not_elided_rdlock
>> +
>> + /* Abort happened. */
>> +abort_rdlock:
>> + testl $_XABORT_CONFLICT,%eax
>> + jz not_elided_rdlock
>> +
>> + /* For a reader that aborts due a conflict retry speculation
>> + a limited number of times. This way when some reader aborts
>> + because the reader count is written the other readers will
>> + still elide, at the cost of retrying the speculation. */
>> +
>> + dec %esi
>> + jnz try_trans_rdlock
>> +
>> + /* Otherwise we just fall back directly to the lock.
>> + Here's the place to add more adaptation. */
>> +
>> +not_elided_rdlock:
>> xorq %r10, %r10
>>
>> /* Get the lock. */
>> @@ -173,5 +219,7 @@ __pthread_rwlock_rdlock:
>> cfi_endproc
>> .size __pthread_rwlock_rdlock,.-__pthread_rwlock_rdlock
>>
>> +#ifndef __pthread_rwlock_rdlock
>> strong_alias (__pthread_rwlock_rdlock, pthread_rwlock_rdlock)
>> hidden_def (__pthread_rwlock_rdlock)
>> +#endif
>> diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S
>> index 57fe1e9..2bc9037 100644
>> --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S
>> +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S
>> @@ -21,6 +21,7 @@
>> #include <lowlevelrwlock.h>
>> #include <pthread-errnos.h>
>> #include <kernel-features.h>
>> +#include <hle.h>
>>
>> .text
>>
>> @@ -29,6 +30,53 @@
>> .align 16
>> pthread_rwlock_timedrdlock:
>> cfi_startproc
>> +
>> + cmpq $0,RW_ELISION(%rdi)
>> + js not_elided_timedrdlock
>> + jnz 2f
>> + /* zero: use default */
>> +
>> + cmpl $0,__rwlock_rtm_enabled(%rip)
>> + jz not_elided_timedrdlock
>> +
>> +2:
>> + mov __rwlock_rtm_read_retries(%rip),%ecx
>> +
>> +try_trans_timedrdlock:
>> + XBEGIN abort_timedrdlock
>> +
>> + /* Lock reader/writer free? */
>> + cmpl $0,WRITER(%rdi)
>> + jnz 1f
>> + cmpl $0,NR_READERS(%rdi)
>> + jnz 1f
>> +
>> + /* Lock is free. Run with transaction */
>> + xor %eax,%eax
>> + ret
>> +
>> + /* Lock is not free. Run */
>> +1: XABORT 0xff
>> + jmp not_elided_timedrdlock
>> +
>> + /* Abort happened. */
>> +abort_timedrdlock:
>> + testl $_XABORT_CONFLICT,%eax
>> + jz not_elided_timedrdlock
>> +
>> + /* For a reader that aborts due a conflict retry speculation
>> + a limited number of times. This way when some reader aborts
>> + because the reader count is written the other readers will
>> + still elide, at the cost of retrying the speculation. */
>> +
>> + dec %ecx
>> + jnz try_trans_timedrdlock
>> +
>> + /* Otherwise we just fall back directly to the lock.
>> + Here's the place to add more adaptation. */
>> +
>> +not_elided_timedrdlock:
>> +
>> pushq %r12
>> cfi_adjust_cfa_offset(8)
>> cfi_rel_offset(%r12, 0)
>> diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S
>> index 391be17..701c07a 100644
>> --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S
>> +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S
>> @@ -21,6 +21,7 @@
>> #include <lowlevelrwlock.h>
>> #include <pthread-errnos.h>
>> #include <kernel-features.h>
>> +#include <hle.h>
>>
>> .text
>>
>> @@ -29,6 +30,37 @@
>> .align 16
>> pthread_rwlock_timedwrlock:
>> cfi_startproc
>> +
>> + cmpq $0,RW_ELISION(%rdi)
>> + js not_elided_timedwrlock
>> + jnz 2f
>> + /* zero: use default */
>> +
>> + cmpl $0,__rwlock_rtm_enabled(%rip)
>> + jz not_elided_timedwrlock
>> +
>> +2:
>> + XBEGIN abort
>> +
>> + /* Lock free? */
>> + cmpl $0,NR_READERS(%rdi)
>> + jnz 1f
>> + cmpl $0,WRITER(%rdi)
>> + jnz 1f
>> +
>> + /* Lock is free. Run with transaction */
>> + xor %rax, %rax
>> + ret
>> +
>> + /* Lock is not free. Run */
>> +1: XABORT 0xff
>> + jmp not_elided_timedwrlock
>> +
>> + /* Abort happened. */
>> +abort:
>> + /* No retries here */
>> +
>> +not_elided_timedwrlock:
>> pushq %r12
>> cfi_adjust_cfa_offset(8)
>> cfi_rel_offset(%r12, 0)
>> diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S
>> index 86dda05..f1b9516 100644
>> --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S
>> +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S
>> @@ -20,7 +20,7 @@
>> #include <lowlevellock.h>
>> #include <lowlevelrwlock.h>
>> #include <kernel-features.h>
>> -
>> +#include <hle.h>
>>
>> .text
>>
>> @@ -29,6 +29,21 @@
>> .align 16
>> __pthread_rwlock_unlock:
>> cfi_startproc
>> +
>> + /* Is lock free? */
>> + cmpl $0,WRITER(%rdi)
>> + jnz 1f
>> + cmpl $0,NR_READERS(%rdi)
>> + jnz 1f
>> +
>> + /* Looks free. Assume transaction.
>> + If you crash here you unlocked a free lock. */
>> + XEND
>> + xor %rax,%rax
>> + ret
>> +
>> +1:
>> +
>> /* Get the lock. */
>> movl $1, %esi
>> xorl %eax, %eax
>> diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S
>> index 734bee3..28d1b4a 100644
>> --- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S
>> +++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S
>> @@ -22,6 +22,7 @@
>> #include <pthread-errnos.h>
>> #include <kernel-features.h>
>> #include <stap-probe.h>
>> +#include <hle.h>
>>
>> .text
>>
>> @@ -33,6 +34,34 @@ __pthread_rwlock_wrlock:
>>
>> LIBC_PROBE (wrlock_entry, 1, %rdi)
>>
>> + cmpq $0,RW_ELISION(%rdi)
>> + js not_elided_wrlock
>> + jnz 2f
>> + /* zero: use default */
>> +
>> + cmpl $0,__rwlock_rtm_enabled(%rip)
>> + jz not_elided_wrlock
>> +
>> +2:
>> + XBEGIN abort_wrlock
>> +
>> + /* Lock free? */
>> + cmpl $0,WRITER(%rdi)
>> + jnz 1f
>> + cmpl $0,NR_READERS(%rdi)
>> + jnz 1f
>> +
>> + /* Lock is free. Run with transaction */
>> + xor %eax,%eax
>> + ret
>> +
>> + /* Lock is not free. End transaction */
>> +1: XABORT 0xff
>> + jmp not_elided_wrlock
>> +
>> + /* Abort happened. */
>> +abort_wrlock:
>> +not_elided_wrlock:
>> xorq %r10, %r10
>>
>> /* Get the lock. */
>> @@ -161,5 +190,7 @@ __pthread_rwlock_wrlock:
>> cfi_endproc
>> .size __pthread_rwlock_wrlock,.-__pthread_rwlock_wrlock
>>
>> +#ifndef __pthread_rwlock_wrlock
>> strong_alias (__pthread_rwlock_wrlock, pthread_rwlock_wrlock)
>> hidden_def (__pthread_rwlock_wrlock)
>> +#endif
>>
>