This is the mail archive of the
libc-alpha@sourceware.org
mailing list for the glibc project.
[PATCH 11/15] Add lock elision to rwlocks
- From: Andi Kleen <andi at firstfloor dot org>
- To: libc-alpha at sourceware dot org
- Cc: hjl dot tools at gmail dot com,Andi Kleen <ak at linux dot intel dot com>
- Date: Sat, 22 Dec 2012 19:02:38 -0800
- Subject: [PATCH 11/15] Add lock elision to rwlocks
- References: <1356231762-23777-1-git-send-email-andi@firstfloor.org>
From: Andi Kleen <ak@linux.intel.com>
This code is very similar to the main pthread mutex code, but we have
to implement part of it in assembler. IFUNC is used to avoid any
fast path overhead on non RTM systems for the lock functions (but not
unlock)
The patch kit is large because there are so many combinations
of reader and writer locks, but it is fairly repetive.
The basic elision algorithm is very similar to normal mutexes:
a wrapper checks if the lock is free (both reader and writer)
We have to check both reader and writer because separating them
would require putting state on different cache lines.
The algorithm is very basic currently and may be tuned
more in the future.
No adaptation at this point, the elision will not automatically
disable itself. The programmer or user can do it explicitely though.
New lock type flags are exposed to force enable/disable elision
for a specific rwlocks.
We use some padding in the internal rwlock as a elision flag.
This is 8 bytes on 64bits currently, but could be made smaller later
without compatibility problems (or to share some bytes with a
adaptation count)
2012-12-22 Andi Kleen <ak@linux.intel.com>
Hongjiu Lu <hongjiu.lu@intel.com>
* nptl/pthread_rwlock_init.c (__pthread_rwlock_init): Support elision.
* nptl/pthread_rwlockattr_setkind_np.c (pthread_rwlockattr_setkind_np): Support elision.
* nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S: New file.
* nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S: dito.
* nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S: dito.
* nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S: dito.
* nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S: dito.
* nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/Makefile: New file.
* nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/pthread_rwlock_rdlock-nortm.S: dito.
* nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/pthread_rwlock_rdlock-rtm.S: dito.
* nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/pthread_rwlock_timedrdlock-nortm.S: dito.
* nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/pthread_rwlock_timedrdlock-rtm.S:
* nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/pthread_rwlock_timedwrlock-nortm.S:
* nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/pthread_rwlock_timedwrlock-rtm.S:
* nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/pthread_rwlock_wrlock-nortm.S:
* nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/pthread_rwlock_wrlock-rtm.S:
* nptl/sysdeps/unix/sysv/linux/internaltypes.h (struct pthread_condattr):
(struct pthread_rwlockattr): Support elision flags in internal rwlock.
* nptl/sysdeps/unix/sysv/linux/lowlevelrwlock.sym: Add ELIDED.
* nptl/sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h: Add elision flags
* nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_rwlock_rdlock.c: New file.
* nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_rwlock_timedrdlock.c: dito.
* nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_rwlock_timedwrlock.c: dito.
* nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_rwlock_tryrdlock.c: dito.
* nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_rwlock_trywrlock.c: dito.
* nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_rwlock_wrlock.c: dito.
* nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/Makefile: New file
* nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/pthread_rwlock_rdlock-nortm.S: dito.
* nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/pthread_rwlock_rdlock-rtm.S: dito.
* nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/pthread_rwlock_timedrdlock-nortm.S: dito.
* nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/pthread_rwlock_timedrdlock-rtm.S: dito.
* nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/pthread_rwlock_timedwrlock-nortm.S: dito.
* nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/pthread_rwlock_timedwrlock-rtm.S: dito.
* nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/pthread_rwlock_wrlock-nortm.S: dito.
* nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/pthread_rwlock_wrlock-rtm.S: dito.
* nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S: dito.
* nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S: dito.
* nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S: dito.
* nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S: dito.
* nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S: dito
---
nptl/pthread_rwlock_init.c | 4 +
nptl/pthread_rwlockattr_setkind_np.c | 12 +++-
.../sysv/linux/i386/i486/pthread_rwlock_rdlock.S | 85 ++++++++++++++++++++
.../linux/i386/i486/pthread_rwlock_timedrdlock.S | 79 ++++++++++++++++++
.../linux/i386/i486/pthread_rwlock_timedwrlock.S | 61 ++++++++++++++-
.../sysv/linux/i386/i486/pthread_rwlock_unlock.S | 19 +++++
.../sysv/linux/i386/i486/pthread_rwlock_wrlock.S | 70 ++++++++++++++++
.../unix/sysv/linux/i386/i686/multiarch/Makefile | 4 +
.../i686/multiarch/pthread_rwlock_rdlock-nortm.S | 2 +
.../i686/multiarch/pthread_rwlock_rdlock-rtm.S | 3 +
.../multiarch/pthread_rwlock_timedrdlock-nortm.S | 2 +
.../multiarch/pthread_rwlock_timedrdlock-rtm.S | 3 +
.../multiarch/pthread_rwlock_timedwrlock-nortm.S | 2 +
.../multiarch/pthread_rwlock_timedwrlock-rtm.S | 3 +
.../i686/multiarch/pthread_rwlock_wrlock-nortm.S | 2 +
.../i686/multiarch/pthread_rwlock_wrlock-rtm.S | 3 +
nptl/sysdeps/unix/sysv/linux/internaltypes.h | 4 +-
nptl/sysdeps/unix/sysv/linux/lowlevelrwlock.sym | 5 +
.../unix/sysv/linux/x86/bits/pthreadtypes.h | 6 +-
.../linux/x86/multiarch/pthread_rwlock_rdlock.c | 10 +++
.../x86/multiarch/pthread_rwlock_timedrdlock.c | 8 ++
.../x86/multiarch/pthread_rwlock_timedwrlock.c | 8 ++
.../linux/x86/multiarch/pthread_rwlock_tryrdlock.c | 56 +++++++++++++
.../linux/x86/multiarch/pthread_rwlock_trywrlock.c | 55 +++++++++++++
.../linux/x86/multiarch/pthread_rwlock_wrlock.c | 10 +++
.../unix/sysv/linux/x86_64/multiarch/Makefile | 4 +
.../x86_64/multiarch/pthread_rwlock_rdlock-nortm.S | 2 +
.../x86_64/multiarch/pthread_rwlock_rdlock-rtm.S | 3 +
.../multiarch/pthread_rwlock_timedrdlock-nortm.S | 2 +
.../multiarch/pthread_rwlock_timedrdlock-rtm.S | 3 +
.../multiarch/pthread_rwlock_timedwrlock-nortm.S | 2 +
.../multiarch/pthread_rwlock_timedwrlock-rtm.S | 3 +
.../x86_64/multiarch/pthread_rwlock_wrlock-nortm.S | 2 +
.../x86_64/multiarch/pthread_rwlock_wrlock-rtm.S | 3 +
.../unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S | 71 ++++++++++++++++
.../sysv/linux/x86_64/pthread_rwlock_timedrdlock.S | 74 +++++++++++++++++
.../sysv/linux/x86_64/pthread_rwlock_timedwrlock.S | 51 ++++++++++++
.../unix/sysv/linux/x86_64/pthread_rwlock_unlock.S | 17 ++++-
.../unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S | 46 +++++++++++
39 files changed, 793 insertions(+), 6 deletions(-)
create mode 100644 nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/Makefile
create mode 100644 nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/pthread_rwlock_rdlock-nortm.S
create mode 100644 nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/pthread_rwlock_rdlock-rtm.S
create mode 100644 nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/pthread_rwlock_timedrdlock-nortm.S
create mode 100644 nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/pthread_rwlock_timedrdlock-rtm.S
create mode 100644 nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/pthread_rwlock_timedwrlock-nortm.S
create mode 100644 nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/pthread_rwlock_timedwrlock-rtm.S
create mode 100644 nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/pthread_rwlock_wrlock-nortm.S
create mode 100644 nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/pthread_rwlock_wrlock-rtm.S
create mode 100644 nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_rwlock_rdlock.c
create mode 100644 nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_rwlock_timedrdlock.c
create mode 100644 nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_rwlock_timedwrlock.c
create mode 100644 nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_rwlock_tryrdlock.c
create mode 100644 nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_rwlock_trywrlock.c
create mode 100644 nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_rwlock_wrlock.c
create mode 100644 nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/Makefile
create mode 100644 nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/pthread_rwlock_rdlock-nortm.S
create mode 100644 nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/pthread_rwlock_rdlock-rtm.S
create mode 100644 nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/pthread_rwlock_timedrdlock-nortm.S
create mode 100644 nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/pthread_rwlock_timedrdlock-rtm.S
create mode 100644 nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/pthread_rwlock_timedwrlock-nortm.S
create mode 100644 nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/pthread_rwlock_timedwrlock-rtm.S
create mode 100644 nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/pthread_rwlock_wrlock-nortm.S
create mode 100644 nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/pthread_rwlock_wrlock-rtm.S
diff --git a/nptl/pthread_rwlock_init.c b/nptl/pthread_rwlock_init.c
index a84c18b..d115765 100644
--- a/nptl/pthread_rwlock_init.c
+++ b/nptl/pthread_rwlock_init.c
@@ -68,6 +68,10 @@ __pthread_rwlock_init (rwlock, attr)
header.private_futex));
#endif
+#ifdef __PTHREAD_RWLOCK_ELIDING
+ rwlock->__data.__eliding = iattr->eliding;
+#endif
+
return 0;
}
strong_alias (__pthread_rwlock_init, pthread_rwlock_init)
diff --git a/nptl/pthread_rwlockattr_setkind_np.c b/nptl/pthread_rwlockattr_setkind_np.c
index 111207e..78bcca8 100644
--- a/nptl/pthread_rwlockattr_setkind_np.c
+++ b/nptl/pthread_rwlockattr_setkind_np.c
@@ -27,12 +27,22 @@ pthread_rwlockattr_setkind_np (attr, pref)
{
struct pthread_rwlockattr *iattr;
+ iattr = (struct pthread_rwlockattr *) attr;
+
+ iattr->eliding = 0;
+ if (pref & (PTHREAD_RWLOCK_ELISION_NP|PTHREAD_RWLOCK_NO_ELISION_NP))
+ {
+ if (pref & PTHREAD_RWLOCK_ELISION_NP)
+ iattr->eliding = 1;
+ if (pref & PTHREAD_RWLOCK_NO_ELISION_NP)
+ iattr->eliding = -1;
+ pref &= ~(PTHREAD_RWLOCK_ELISION_NP|PTHREAD_RWLOCK_NO_ELISION_NP);
+ }
if (pref != PTHREAD_RWLOCK_PREFER_READER_NP
&& pref != PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP
&& __builtin_expect (pref != PTHREAD_RWLOCK_PREFER_WRITER_NP, 0))
return EINVAL;
- iattr = (struct pthread_rwlockattr *) attr;
iattr->lockkind = pref;
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S
index e9487f9..c98dd06 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S
@@ -21,9 +21,16 @@
#include <lowlevelrwlock.h>
#include <pthread-errnos.h>
#include <kernel-features.h>
+#include <hle.h>
#include <stap-probe.h>
+#ifdef PIC
+#define MO(x) x##@GOTOFF(%edx)
+#else
+#define MO(x) x
+#endif
+
.text
.globl __pthread_rwlock_rdlock
@@ -43,6 +50,82 @@ __pthread_rwlock_rdlock:
LIBC_PROBE (rdlock_entry, 1, %ebx)
+#ifdef ENABLE_ELISION
+#ifdef PIC
+ SETUP_PIC_REG(dx)
+ addl $_GLOBAL_OFFSET_TABLE_,%edx
+#endif
+ cmpb $0,ELIDING(%ebx)
+ js not_elided_rdlock
+ jnz 2f
+ /* zero: use default */
+
+ cmpl $0,MO(__rwlock_rtm_enabled)
+ jz not_elided_rdlock
+
+2:
+ mov MO(__rwlock_rtm_read_retries),%ecx
+
+try_trans_rdlock:
+ XBEGIN abort_rdlock
+
+ /* Lock writer/reader free? */
+ cmpl $0,WRITER(%ebx)
+ jnz 1f
+ cmpl $0,NR_READERS(%ebx)
+ jnz 1f
+
+ /* Lock is free. Run with transaction */
+ xor %eax,%eax
+
+ pop %ebx
+ cfi_adjust_cfa_offset(-4)
+ pop %esi
+ cfi_adjust_cfa_offset(-4)
+ ret
+
+ /* Lock is not free. Run */
+1: XABORT 0xff
+ jmp not_elided_rdlock
+
+ /* Abort happened. */
+abort_rdlock:
+ cmpl $0,MO(__tsx_abort_hook)
+ jz 1f
+ push %ecx
+ cfi_adjust_cfa_offset(4)
+ push %eax
+ cfi_adjust_cfa_offset(4)
+ call *MO(__tsx_abort_hook)
+ pop %eax
+ cfi_adjust_cfa_offset(-4)
+ pop %ecx
+ xor %esi, %esi /* needed? */
+ movl 12(%esp), %ebx
+#ifdef PIC
+ SETUP_PIC_REG(dx)
+ addl $_GLOBAL_OFFSET_TABLE_,%edx
+#endif
+
+1:
+
+ testl $_XABORT_CONFLICT,%eax
+ jz not_elided_rdlock
+
+ /* For a reader that aborts due a conflict retry speculation
+ a limited number of times. This way when some reader aborts
+ because the reader count is written the other readers will
+ still elide, at the cost of retrying the speculation. */
+
+ dec %ecx
+ jnz try_trans_rdlock
+
+ /* Otherwise we just fall back directly to the lock.
+ Here's the place to add more adaptation. */
+
+not_elided_rdlock:
+#endif
+
/* Get the lock. */
movl $1, %edx
xorl %eax, %eax
@@ -188,5 +271,7 @@ __pthread_rwlock_rdlock:
cfi_endproc
.size __pthread_rwlock_rdlock,.-__pthread_rwlock_rdlock
+#ifndef __pthread_rwlock_rdlock
strong_alias (__pthread_rwlock_rdlock, pthread_rwlock_rdlock)
hidden_def (__pthread_rwlock_rdlock)
+#endif
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S
index 2dc1f0d..b52acf0 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S
@@ -21,7 +21,13 @@
#include <lowlevelrwlock.h>
#include <pthread-errnos.h>
#include <kernel-features.h>
+#include <hle.h>
+#ifdef PIC
+#define MO(x) x##@GOTOFF(%edx)
+#else
+#define MO(x) x
+#endif
.text
@@ -48,6 +54,78 @@ pthread_rwlock_timedrdlock:
movl 28(%esp), %ebp
movl 32(%esp), %edi
+#ifdef ENABLE_ELISION
+#ifdef PIC
+ SETUP_PIC_REG(dx)
+ addl $_GLOBAL_OFFSET_TABLE_,%edx
+#endif
+
+ cmpb $0,ELIDING(%ebp)
+ js not_elided_trdlock
+ jnz 2f
+ /* zero: use default */
+
+ cmpl $0,MO(__rwlock_rtm_enabled)
+ jz not_elided_trdlock
+2:
+ mov MO(__rwlock_rtm_read_retries),%ecx
+
+try_trans_trdlock:
+ XBEGIN abort_trdlock
+
+ /* Lock writer/reader free? */
+ cmpl $0,WRITER(%ebp)
+ jnz 1f
+ cmpl $0,NR_READERS(%ebp)
+ jnz 1f
+
+ /* Lock is free. Run with transaction */
+ xor %eax,%eax
+ jmp 77f
+
+
+ /* Lock is not free. Run */
+1: XABORT 0xff
+ jmp not_elided_trdlock
+
+ /* Abort happened. */
+abort_trdlock:
+ cmpl $0,MO(__tsx_abort_hook)
+ jz 1f
+ push %ecx
+ cfi_adjust_cfa_offset(4)
+ push %eax
+ cfi_adjust_cfa_offset(4)
+ call *MO(__tsx_abort_hook)
+ pop %eax
+ cfi_adjust_cfa_offset(-4)
+ pop %ecx
+ movl 32(%esp), %edi
+#ifdef PIC
+ SETUP_PIC_REG(dx)
+ addl $_GLOBAL_OFFSET_TABLE_,%edx
+#endif
+
+1:
+
+ testl $_XABORT_CONFLICT,%eax
+ jz not_elided_trdlock
+
+ /* For a reader that aborts due a conflict retry speculation
+ a limited number of times. This way when some reader aborts
+ because the reader count is written the other readers will
+ still elide, at the cost of retrying the speculation. */
+
+ dec %ecx
+ jnz try_trans_trdlock
+
+ /* Otherwise we just fall back directly to the lock.
+ Here's the place to add more adaptation. */
+
+not_elided_trdlock:
+#endif
+
+
/* Get the lock. */
movl $1, %edx
xorl %eax, %eax
@@ -158,6 +236,7 @@ pthread_rwlock_timedrdlock:
7: movl %edx, %eax
+77:
addl $8, %esp
cfi_adjust_cfa_offset(-8)
popl %ebp
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S
index 132ab06..af47698 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S
@@ -21,7 +21,13 @@
#include <lowlevelrwlock.h>
#include <pthread-errnos.h>
#include <kernel-features.h>
+#include <hle.h>
+#ifdef PIC
+#define MO(x) x##@GOTOFF(%edx)
+#else
+#define MO(x) x
+#endif
.text
@@ -48,6 +54,59 @@ pthread_rwlock_timedwrlock:
movl 28(%esp), %ebp
movl 32(%esp), %edi
+#ifdef ENABLE_ELISION
+#ifdef PIC
+ SETUP_PIC_REG(dx)
+ addl $_GLOBAL_OFFSET_TABLE_,%edx
+#endif
+ cmpb $0,ELIDING(%ebp)
+ js not_elided_twrlock
+ jnz try_trans_wrlock
+ /* zero: use default */
+
+ cmpl $0,MO(__rwlock_rtm_enabled)
+ jz not_elided_twrlock
+
+try_trans_wrlock:
+ XBEGIN abort_twrlock
+
+ /* Lock writer free? */
+ cmpl $0,WRITER(%ebp)
+ jnz 1f
+ cmpl $0,NR_READERS(%ebp)
+ jnz 1f
+
+ /* Lock is free. Run with transaction */
+ xor %eax,%eax
+
+ jmp 77f
+
+ /* Lock is not free. Run */
+1: XABORT 0xff
+ jmp not_elided_twrlock
+
+ /* Abort happened. */
+abort_twrlock:
+ cmpl $0,MO(__tsx_abort_hook)
+ jz 1f
+ push %eax
+ cfi_adjust_cfa_offset(4)
+ call *MO(__tsx_abort_hook)
+ pop %eax
+ cfi_adjust_cfa_offset(-4)
+ mov 32(%esp), %edi
+#ifdef PIC
+ SETUP_PIC_REG(bx)
+ addl $_GLOBAL_OFFSET_TABLE_,%edx
+#endif
+
+1:
+ /* Otherwise we just fall back directly to the lock.
+ Here's the place to add more adaptation. */
+
+not_elided_twrlock:
+#endif
+
/* Get the lock. */
movl $1, %edx
xorl %eax, %eax
@@ -156,7 +215,7 @@ pthread_rwlock_timedwrlock:
7: movl %edx, %eax
- addl $8, %esp
+77: addl $8, %esp
cfi_adjust_cfa_offset(-8)
popl %ebp
cfi_adjust_cfa_offset(-4)
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S
index 0d0e514..c05c4d1 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S
@@ -20,6 +20,7 @@
#include <lowlevellock.h>
#include <lowlevelrwlock.h>
#include <kernel-features.h>
+#include <hle.h>
.text
@@ -38,6 +39,24 @@ __pthread_rwlock_unlock:
movl 12(%esp), %edi
+ /* Is lock free? */
+ cmpl $0,WRITER(%edi)
+ jnz 1f
+ cmpl $0,NR_READERS(%edi)
+ jnz 1f
+
+ /* Looks free. Assume transaction.
+ If you crash here you unlocked a free lock. */
+ XEND
+ xor %eax,%eax
+
+ pop %edi
+ cfi_adjust_cfa_offset(-4)
+ pop %ebx
+ cfi_adjust_cfa_offset(-4)
+ ret
+
+1:
/* Get the lock. */
movl $1, %edx
xorl %eax, %eax
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S
index 1e539b0..43fc4e0 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S
@@ -21,9 +21,16 @@
#include <lowlevelrwlock.h>
#include <pthread-errnos.h>
#include <kernel-features.h>
+#include <hle.h>
#include <stap-probe.h>
+#ifdef PIC
+#define MO(x) x@GOTOFF(%edx)
+#else
+#define MO(x) x
+#endif
+
.text
.globl __pthread_rwlock_wrlock
@@ -43,6 +50,67 @@ __pthread_rwlock_wrlock:
LIBC_PROBE (wrlock_entry, 1, %ebx)
+#ifdef ENABLE_ELISION
+#ifdef PIC
+ SETUP_PIC_REG(dx)
+ addl $_GLOBAL_OFFSET_TABLE_,%edx
+#endif
+
+ cmpb $0,ELIDING(%ebx)
+ js not_elided_wrlock
+ jnz try_trans_wrlock
+ /* zero: use default */
+
+ cmpl $0,MO(__rwlock_rtm_enabled)
+ jz not_elided_wrlock
+
+try_trans_wrlock:
+ XBEGIN abort_wrlock
+
+ /* Lock writer free? */
+ /* Ignore readers because we don't need them */
+ cmpl $0,WRITER(%ebx)
+ jnz 1f
+ cmpl $0,NR_READERS(%ebx)
+ jnz 1f
+
+ /* Lock is free. Run with transaction */
+ xor %eax,%eax
+
+ pop %ebx
+ cfi_adjust_cfa_offset(-4)
+ pop %esi
+ cfi_adjust_cfa_offset(-4)
+ ret
+
+ /* Lock is not free. Run */
+1: XABORT 0xff
+ jmp not_elided_wrlock
+
+ /* Abort happened. */
+abort_wrlock:
+ cmpl $0,MO(__tsx_abort_hook)
+ jz 1f
+ push %eax
+ cfi_adjust_cfa_offset(4)
+ call *MO(__tsx_abort_hook)
+ pop %eax
+ cfi_adjust_cfa_offset(-4)
+ xorl %esi, %esi /* needed? */
+ movl 12(%esp), %ebx
+#ifdef PIC
+ SETUP_PIC_REG(dx)
+ addl $_GLOBAL_OFFSET_TABLE_,%edx
+#endif
+
+1:
+
+ /* Otherwise we just fall back directly to the lock.
+ Here's the place to add more adaptation. */
+
+not_elided_wrlock:
+#endif
+
/* Get the lock. */
movl $1, %edx
xorl %eax, %eax
@@ -179,5 +247,7 @@ __pthread_rwlock_wrlock:
cfi_endproc
.size __pthread_rwlock_wrlock,.-__pthread_rwlock_wrlock
+#ifndef __pthread_rwlock_wrlock
strong_alias (__pthread_rwlock_wrlock, pthread_rwlock_wrlock)
hidden_def (__pthread_rwlock_wrlock)
+#endif
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/Makefile b/nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/Makefile
new file mode 100644
index 0000000..31d145b
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/Makefile
@@ -0,0 +1,4 @@
+libpthread-sysdep_routines += pthread_rwlock_rdlock-nortm pthread_rwlock_rdlock-rtm
+libpthread-sysdep_routines += pthread_rwlock_wrlock-nortm pthread_rwlock_wrlock-rtm
+libpthread-sysdep_routines += pthread_rwlock_timedrdlock-nortm pthread_rwlock_timedrdlock-rtm
+libpthread-sysdep_routines += pthread_rwlock_timedwrlock-nortm pthread_rwlock_timedwrlock-rtm
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/pthread_rwlock_rdlock-nortm.S b/nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/pthread_rwlock_rdlock-nortm.S
new file mode 100644
index 0000000..bbabec6
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/pthread_rwlock_rdlock-nortm.S
@@ -0,0 +1,2 @@
+#define __pthread_rwlock_rdlock __pthread_rwlock_rdlock_nortm
+#include "sysdeps/unix/sysv/linux/i386/i686/pthread_rwlock_rdlock.S"
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/pthread_rwlock_rdlock-rtm.S b/nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/pthread_rwlock_rdlock-rtm.S
new file mode 100644
index 0000000..7860e1b
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/pthread_rwlock_rdlock-rtm.S
@@ -0,0 +1,3 @@
+#define ENABLE_ELISION 1
+#define __pthread_rwlock_rdlock __pthread_rwlock_rdlock_rtm
+#include "sysdeps/unix/sysv/linux/i386/i686/pthread_rwlock_rdlock.S"
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/pthread_rwlock_timedrdlock-nortm.S b/nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/pthread_rwlock_timedrdlock-nortm.S
new file mode 100644
index 0000000..17f5491
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/pthread_rwlock_timedrdlock-nortm.S
@@ -0,0 +1,2 @@
+#define pthread_rwlock_timedrdlock __pthread_rwlock_timedrdlock_nortm
+#include "sysdeps/unix/sysv/linux/i386/i686/pthread_rwlock_timedrdlock.S"
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/pthread_rwlock_timedrdlock-rtm.S b/nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/pthread_rwlock_timedrdlock-rtm.S
new file mode 100644
index 0000000..05f60bb
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/pthread_rwlock_timedrdlock-rtm.S
@@ -0,0 +1,3 @@
+#define ENABLE_ELISION 1
+#define pthread_rwlock_timedrdlock __pthread_rwlock_timedrdlock_rtm
+#include "sysdeps/unix/sysv/linux/i386/i686/pthread_rwlock_timedrdlock.S"
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/pthread_rwlock_timedwrlock-nortm.S b/nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/pthread_rwlock_timedwrlock-nortm.S
new file mode 100644
index 0000000..ab64c2a
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/pthread_rwlock_timedwrlock-nortm.S
@@ -0,0 +1,2 @@
+#define pthread_rwlock_timedwrlock __pthread_rwlock_timedwrlock_nortm
+#include "sysdeps/unix/sysv/linux/i386/i686/pthread_rwlock_timedwrlock.S"
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/pthread_rwlock_timedwrlock-rtm.S b/nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/pthread_rwlock_timedwrlock-rtm.S
new file mode 100644
index 0000000..494b04a
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/pthread_rwlock_timedwrlock-rtm.S
@@ -0,0 +1,3 @@
+#define ENABLE_ELISION 1
+#define pthread_rwlock_timedwrlock __pthread_rwlock_timedwrlock_rtm
+#include "sysdeps/unix/sysv/linux/i386/i686/pthread_rwlock_timedwrlock.S"
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/pthread_rwlock_wrlock-nortm.S b/nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/pthread_rwlock_wrlock-nortm.S
new file mode 100644
index 0000000..81b75bd
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/pthread_rwlock_wrlock-nortm.S
@@ -0,0 +1,2 @@
+#define __pthread_rwlock_wrlock __pthread_rwlock_wrlock_nortm
+#include "sysdeps/unix/sysv/linux/i386/i686/pthread_rwlock_wrlock.S"
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/pthread_rwlock_wrlock-rtm.S b/nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/pthread_rwlock_wrlock-rtm.S
new file mode 100644
index 0000000..31869c1
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i686/multiarch/pthread_rwlock_wrlock-rtm.S
@@ -0,0 +1,3 @@
+#define ENABLE_ELISION 1
+#define __pthread_rwlock_wrlock __pthread_rwlock_wrlock_rtm
+#include "sysdeps/unix/sysv/linux/i386/i686/pthread_rwlock_wrlock.S"
diff --git a/nptl/sysdeps/unix/sysv/linux/internaltypes.h b/nptl/sysdeps/unix/sysv/linux/internaltypes.h
index ff2d231..0a85509 100644
--- a/nptl/sysdeps/unix/sysv/linux/internaltypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/internaltypes.h
@@ -86,7 +86,9 @@ struct pthread_condattr
struct pthread_rwlockattr
{
int lockkind;
- int pshared;
+ short pshared;
+ char eliding;
+ char pad;
};
diff --git a/nptl/sysdeps/unix/sysv/linux/lowlevelrwlock.sym b/nptl/sysdeps/unix/sysv/linux/lowlevelrwlock.sym
index f50b25b..fcbcd5a 100644
--- a/nptl/sysdeps/unix/sysv/linux/lowlevelrwlock.sym
+++ b/nptl/sysdeps/unix/sysv/linux/lowlevelrwlock.sym
@@ -3,6 +3,10 @@
#include <bits/pthreadtypes.h>
#include <bits/wordsize.h>
+#ifndef __PTHREAD_RWLOCK_ELIDING
+#define __eliding __shared
+#endif
+
--
MUTEX offsetof (pthread_rwlock_t, __data.__lock)
@@ -14,3 +18,4 @@ WRITERS_QUEUED offsetof (pthread_rwlock_t, __data.__nr_writers_queued)
FLAGS offsetof (pthread_rwlock_t, __data.__flags)
WRITER offsetof (pthread_rwlock_t, __data.__writer)
PSHARED offsetof (pthread_rwlock_t, __data.__shared)
+ELIDING offsetof (pthread_rwlock_t, __data.__eliding)
diff --git a/nptl/sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h
index 10fab9b..199668f 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h
@@ -183,12 +183,13 @@ typedef union
unsigned int __nr_writers_queued;
int __writer;
int __shared;
- unsigned long int __pad1;
+ unsigned long __eliding;
unsigned long int __pad2;
/* FLAGS must stay at this position in the structure to maintain
binary compatibility. */
unsigned int __flags;
# define __PTHREAD_RWLOCK_INT_FLAGS_SHARED 1
+# define __PTHREAD_RWLOCK_ELIDING 1
} __data;
# else
struct
@@ -203,8 +204,9 @@ typedef union
binary compatibility. */
unsigned char __flags;
unsigned char __shared;
- unsigned char __pad1;
+ unsigned char __eliding;
unsigned char __pad2;
+# define __PTHREAD_RWLOCK_ELIDING 2
int __writer;
} __data;
# endif
diff --git a/nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_rwlock_rdlock.c b/nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_rwlock_rdlock.c
new file mode 100644
index 0000000..54732e3
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_rwlock_rdlock.c
@@ -0,0 +1,10 @@
+#include "pthreadP.h"
+#include "init-arch.h"
+extern typeof(__pthread_rwlock_rdlock) __pthread_rwlock_rdlock_rtm;
+extern typeof(__pthread_rwlock_rdlock) __pthread_rwlock_rdlock_nortm;
+libm_ifunc(__pthread_rwlock_rdlock,
+ (HAS_RTM
+ ? __pthread_rwlock_rdlock_rtm
+ : __pthread_rwlock_rdlock_nortm))
+strong_alias (__pthread_rwlock_rdlock, pthread_rwlock_rdlock)
+libc_ifunc_hidden_def (__pthread_rwlock_rdlock)
diff --git a/nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_rwlock_timedrdlock.c b/nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_rwlock_timedrdlock.c
new file mode 100644
index 0000000..e1cd95d
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_rwlock_timedrdlock.c
@@ -0,0 +1,8 @@
+#include "pthreadP.h"
+#include "init-arch.h"
+extern typeof(pthread_rwlock_timedrdlock) __pthread_rwlock_timedrdlock_rtm;
+extern typeof(pthread_rwlock_timedrdlock) __pthread_rwlock_timedrdlock_nortm;
+libm_ifunc(pthread_rwlock_timedrdlock,
+ (HAS_RTM
+ ? __pthread_rwlock_timedrdlock_rtm
+ : __pthread_rwlock_timedrdlock_nortm))
diff --git a/nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_rwlock_timedwrlock.c b/nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_rwlock_timedwrlock.c
new file mode 100644
index 0000000..952a23d
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_rwlock_timedwrlock.c
@@ -0,0 +1,8 @@
+#include "pthreadP.h"
+#include "init-arch.h"
+extern typeof(pthread_rwlock_timedwrlock) __pthread_rwlock_timedwrlock_rtm;
+extern typeof(pthread_rwlock_timedwrlock) __pthread_rwlock_timedwrlock_nortm;
+libm_ifunc(pthread_rwlock_timedwrlock,
+ (HAS_RTM
+ ? __pthread_rwlock_timedwrlock_rtm
+ : __pthread_rwlock_timedwrlock_nortm))
diff --git a/nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_rwlock_tryrdlock.c b/nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_rwlock_tryrdlock.c
new file mode 100644
index 0000000..706a670
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_rwlock_tryrdlock.c
@@ -0,0 +1,56 @@
+/* pthread_rwlock_tryrdlock: Lock eliding version of pthreads rwlock_tryrdlock.
+ Copyright (C) 2012 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+#include <pthread.h>
+#include <pthreadP.h>
+#include <hle.h>
+#include "elision-conf.h"
+#include "init-arch.h"
+
+#define __pthread_rwlock_tryrdlock __full_pthread_rwlock_tryrdlock
+#include <nptl/pthread_rwlock_tryrdlock.c>
+#undef __pthread_rwlock_tryrdlock
+
+int
+__pthread_rwlock_tryrdlock_rtm(pthread_rwlock_t *rwlock)
+{
+ unsigned status;
+
+ if (__rwlock_rtm_enabled != 0)
+ {
+ if ((status = _xbegin()) == _XBEGIN_STARTED)
+ {
+ if (rwlock->__data.__writer == 0
+ && rwlock->__data.__nr_readers == 0)
+ return 0;
+ /* Lock was busy. Fall back to normal locking.
+ Could also _xend here but xabort with 0xff code
+ is more visible in the profiler. */
+ _xabort (0xff);
+ }
+ /* Aborts come here */
+ if (__tsx_abort_hook)
+ __tsx_abort_hook (status);
+ }
+
+ return __full_pthread_rwlock_tryrdlock (rwlock);
+}
+
+libm_ifunc(__pthread_rwlock_tryrdlock,
+ (HAS_RTM
+ ? __pthread_rwlock_tryrdlock_rtm
+ : __full_pthread_rwlock_tryrdlock))
diff --git a/nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_rwlock_trywrlock.c b/nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_rwlock_trywrlock.c
new file mode 100644
index 0000000..7e6abc5
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_rwlock_trywrlock.c
@@ -0,0 +1,55 @@
+/* pthread_rwlock_trywrlock: Lock eliding version of pthreads rwlock trywrlock.
+ Copyright (C) 2012 Free Software Foundation, Inc.
+ This file is part of the GNU C Library.
+
+ The GNU C Library is free software; you can redistribute it and/or
+ modify it under the terms of the GNU Lesser General Public
+ License as published by the Free Software Foundation; either
+ version 2.1 of the License, or (at your option) any later version.
+
+ The GNU C Library is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ Lesser General Public License for more details.
+
+ You should have received a copy of the GNU Lesser General Public
+ License along with the GNU C Library; if not, see
+ <http://www.gnu.org/licenses/>. */
+#include <pthread.h>
+#include <pthreadP.h>
+#include <hle.h>
+#include "elision-conf.h"
+#include "init-arch.h"
+
+#define __pthread_rwlock_trywrlock __full_pthread_rwlock_trywrlock
+#include <nptl/pthread_rwlock_trywrlock.c>
+#undef __pthread_rwlock_trywrlock
+
+int
+__pthread_rwlock_trywrlock_rtm(pthread_rwlock_t *rwlock)
+{
+ unsigned status;
+
+ if (__rwlock_rtm_enabled != 0)
+ {
+ if ((status = _xbegin()) == _XBEGIN_STARTED)
+ {
+ if (rwlock->__data.__writer == 0
+ && rwlock->__data.__nr_readers == 0)
+ return 0;
+ /* Lock was busy. Fall back to normal locking.
+ Could also _xend here but xabort with 0xff code
+ is more visible in the profiler. */
+ _xabort (0xff);
+ }
+ if (__tsx_abort_hook)
+ __tsx_abort_hook (status);
+ }
+
+ return __full_pthread_rwlock_trywrlock (rwlock);
+}
+
+libm_ifunc(__pthread_rwlock_trywrlock,
+ (HAS_RTM
+ ? __pthread_rwlock_trywrlock_rtm
+ : __full_pthread_rwlock_trywrlock))
diff --git a/nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_rwlock_wrlock.c b/nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_rwlock_wrlock.c
new file mode 100644
index 0000000..076e72a
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/x86/multiarch/pthread_rwlock_wrlock.c
@@ -0,0 +1,10 @@
+#include "pthreadP.h"
+#include "init-arch.h"
+extern typeof(__pthread_rwlock_wrlock) __pthread_rwlock_wrlock_rtm;
+extern typeof(__pthread_rwlock_wrlock) __pthread_rwlock_wrlock_nortm;
+libm_ifunc(__pthread_rwlock_wrlock,
+ (HAS_RTM
+ ? __pthread_rwlock_wrlock_rtm
+ : __pthread_rwlock_wrlock_nortm))
+strong_alias (__pthread_rwlock_wrlock, pthread_rwlock_wrlock)
+libc_ifunc_hidden_def (__pthread_rwlock_wrlock)
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/Makefile b/nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/Makefile
new file mode 100644
index 0000000..31d145b
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/Makefile
@@ -0,0 +1,4 @@
+libpthread-sysdep_routines += pthread_rwlock_rdlock-nortm pthread_rwlock_rdlock-rtm
+libpthread-sysdep_routines += pthread_rwlock_wrlock-nortm pthread_rwlock_wrlock-rtm
+libpthread-sysdep_routines += pthread_rwlock_timedrdlock-nortm pthread_rwlock_timedrdlock-rtm
+libpthread-sysdep_routines += pthread_rwlock_timedwrlock-nortm pthread_rwlock_timedwrlock-rtm
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/pthread_rwlock_rdlock-nortm.S b/nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/pthread_rwlock_rdlock-nortm.S
new file mode 100644
index 0000000..0d509ad
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/pthread_rwlock_rdlock-nortm.S
@@ -0,0 +1,2 @@
+#define __pthread_rwlock_rdlock __pthread_rwlock_rdlock_nortm
+#include "sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S"
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/pthread_rwlock_rdlock-rtm.S b/nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/pthread_rwlock_rdlock-rtm.S
new file mode 100644
index 0000000..0e2cee5
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/pthread_rwlock_rdlock-rtm.S
@@ -0,0 +1,3 @@
+#define ENABLE_ELISION 1
+#define __pthread_rwlock_rdlock __pthread_rwlock_rdlock_rtm
+#include "sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S"
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/pthread_rwlock_timedrdlock-nortm.S b/nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/pthread_rwlock_timedrdlock-nortm.S
new file mode 100644
index 0000000..5e86737
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/pthread_rwlock_timedrdlock-nortm.S
@@ -0,0 +1,2 @@
+#define pthread_rwlock_timedrdlock __pthread_rwlock_timedrdlock_nortm
+#include "sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S"
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/pthread_rwlock_timedrdlock-rtm.S b/nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/pthread_rwlock_timedrdlock-rtm.S
new file mode 100644
index 0000000..59ceef4
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/pthread_rwlock_timedrdlock-rtm.S
@@ -0,0 +1,3 @@
+#define ENABLE_ELISION 1
+#define pthread_rwlock_timedrdlock __pthread_rwlock_timedrdlock_rtm
+#include "sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S"
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/pthread_rwlock_timedwrlock-nortm.S b/nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/pthread_rwlock_timedwrlock-nortm.S
new file mode 100644
index 0000000..a487986
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/pthread_rwlock_timedwrlock-nortm.S
@@ -0,0 +1,2 @@
+#define pthread_rwlock_timedwrlock __pthread_rwlock_timedwrlock_nortm
+#include "sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S"
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/pthread_rwlock_timedwrlock-rtm.S b/nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/pthread_rwlock_timedwrlock-rtm.S
new file mode 100644
index 0000000..83988f1
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/pthread_rwlock_timedwrlock-rtm.S
@@ -0,0 +1,3 @@
+#define ENABLE_ELISION 1
+#define pthread_rwlock_timedwrlock __pthread_rwlock_timedwrlock_rtm
+#include "sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S"
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/pthread_rwlock_wrlock-nortm.S b/nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/pthread_rwlock_wrlock-nortm.S
new file mode 100644
index 0000000..019bc70
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/pthread_rwlock_wrlock-nortm.S
@@ -0,0 +1,2 @@
+#define __pthread_rwlock_wrlock __pthread_rwlock_wrlock_nortm
+#include "sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S"
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/pthread_rwlock_wrlock-rtm.S b/nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/pthread_rwlock_wrlock-rtm.S
new file mode 100644
index 0000000..268e14b
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/multiarch/pthread_rwlock_wrlock-rtm.S
@@ -0,0 +1,3 @@
+#define ENABLE_ELISION 1
+#define __pthread_rwlock_wrlock __pthread_rwlock_wrlock_rtm
+#include "sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S"
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S
index 264ba58..0ea6774 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S
@@ -22,6 +22,7 @@
#include <pthread-errnos.h>
#include <kernel-features.h>
#include <stap-probe.h>
+#include <hle.h>
.text
@@ -33,6 +34,74 @@ __pthread_rwlock_rdlock:
LIBC_PROBE (rdlock_entry, 1, %rdi)
+#ifdef ENABLE_ELISION
+ cmpq $0,ELIDING(%rdi)
+ js not_elided_rdlock
+ jnz 2f
+ /* zero: use default */
+
+ cmpl $0,__rwlock_rtm_enabled(%rip)
+ jz not_elided_rdlock
+
+2:
+ mov __rwlock_rtm_read_retries(%rip),%esi
+
+try_trans_rdlock:
+ XBEGIN abort_rdlock
+
+ /* Lock reader/writer free? */
+ cmpl $0,WRITER(%rdi)
+ jnz 1f
+ cmpl $0,NR_READERS(%rdi)
+ jnz 1f
+
+ /* Lock is free. Run with transaction */
+ xor %eax,%eax
+ ret
+
+ /* Lock is not free. Run */
+1: XABORT 0xff
+ jmp not_elided_rdlock
+
+ /* Abort happened. */
+abort_rdlock:
+ /* Call the abort hook */
+ cmpq $0,__tsx_abort_hook(%rip)
+ jz 1f
+
+ push %rdi
+ cfi_adjust_cfa_offset(8)
+ push %rsi
+ cfi_adjust_cfa_offset(8)
+ push %rax
+ cfi_adjust_cfa_offset(8)
+ mov %eax,%edi
+ call *__tsx_abort_hook(%rip)
+ pop %rax
+ cfi_adjust_cfa_offset(-8)
+ pop %rsi
+ cfi_adjust_cfa_offset(-8)
+ pop %rdi
+ cfi_adjust_cfa_offset(-8)
+
+1:
+
+ testl $_XABORT_CONFLICT,%eax
+ jz not_elided_rdlock
+
+ /* For a reader that aborts due a conflict retry speculation
+ a limited number of times. This way when some reader aborts
+ because the reader count is written the other readers will
+ still elide, at the cost of retrying the speculation. */
+
+ dec %esi
+ jnz try_trans_rdlock
+
+ /* Otherwise we just fall back directly to the lock.
+ Here's the place to add more adaptation. */
+
+not_elided_rdlock:
+#endif
xorq %r10, %r10
/* Get the lock. */
@@ -173,5 +242,7 @@ __pthread_rwlock_rdlock:
cfi_endproc
.size __pthread_rwlock_rdlock,.-__pthread_rwlock_rdlock
+#ifndef __pthread_rwlock_rdlock
strong_alias (__pthread_rwlock_rdlock, pthread_rwlock_rdlock)
hidden_def (__pthread_rwlock_rdlock)
+#endif
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S
index f60530e..18faf57 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S
@@ -21,6 +21,7 @@
#include <lowlevelrwlock.h>
#include <pthread-errnos.h>
#include <kernel-features.h>
+#include <hle.h>
.text
@@ -29,6 +30,79 @@
.align 16
pthread_rwlock_timedrdlock:
cfi_startproc
+
+#ifdef ENABLE_ELISION
+ cmpq $0,ELIDING(%rdi)
+ js not_elided_timedrdlock
+ jnz 2f
+ /* zero: use default */
+
+ cmpl $0,__rwlock_rtm_enabled(%rip)
+ jz not_elided_timedrdlock
+
+2:
+ mov __rwlock_rtm_read_retries(%rip),%ecx
+
+try_trans_timedrdlock:
+ XBEGIN abort_timedrdlock
+
+ /* Lock reader/writer free? */
+ cmpl $0,WRITER(%rdi)
+ jnz 1f
+ cmpl $0,NR_READERS(%rdi)
+ jnz 1f
+
+ /* Lock is free. Run with transaction */
+ xor %eax,%eax
+ ret
+
+ /* Lock is not free. Run */
+1: XABORT 0xff
+ jmp not_elided_timedrdlock
+
+ /* Abort happened. */
+abort_timedrdlock:
+ /* Call the abort hook */
+ cmpq $0,__tsx_abort_hook(%rip)
+ jz 1f
+ push %rcx
+ cfi_adjust_cfa_offset(8)
+ push %rdi
+ cfi_adjust_cfa_offset(8)
+ push %rsi
+ cfi_adjust_cfa_offset(8)
+ push %rax
+ cfi_adjust_cfa_offset(8)
+ mov %eax,%edi
+ call *__tsx_abort_hook(%rip)
+ pop %rax
+ cfi_adjust_cfa_offset(-8)
+ pop %rsi
+ cfi_adjust_cfa_offset(-8)
+ pop %rdi
+ cfi_adjust_cfa_offset(-8)
+ pop %rcx
+ cfi_adjust_cfa_offset(-8)
+
+1:
+
+ testl $_XABORT_CONFLICT,%eax
+ jz not_elided_timedrdlock
+
+ /* For a reader that aborts due a conflict retry speculation
+ a limited number of times. This way when some reader aborts
+ because the reader count is written the other readers will
+ still elide, at the cost of retrying the speculation. */
+
+ dec %ecx
+ jnz try_trans_timedrdlock
+
+ /* Otherwise we just fall back directly to the lock.
+ Here's the place to add more adaptation. */
+
+not_elided_timedrdlock:
+#endif
+
pushq %r12
cfi_adjust_cfa_offset(8)
cfi_rel_offset(%r12, 0)
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S
index 7870733..41f49c8 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S
@@ -22,6 +22,7 @@
#include <lowlevelrwlock.h>
#include <pthread-errnos.h>
#include <kernel-features.h>
+#include <hle.h>
.text
@@ -30,6 +31,56 @@
.align 16
pthread_rwlock_timedwrlock:
cfi_startproc
+
+#ifdef ENABLE_ELISION
+ cmpq $0,ELIDING(%rdi)
+ js not_elided_timedwrlock
+ jnz 2f
+ /* zero: use default */
+
+ cmpl $0,__rwlock_rtm_enabled(%rip)
+ jz not_elided_timedwrlock
+
+2:
+ XBEGIN abort
+
+ /* Lock free? */
+ cmpl $0,NR_READERS(%rdi)
+ jnz 1f
+ cmpl $0,WRITER(%rdi)
+ jnz 1f
+
+ /* Lock is free. Run with transaction */
+ xor %rax, %rax
+ ret
+
+ /* Lock is not free. Run */
+1: XABORT 0xff
+ jmp not_elided_timedwrlock
+
+ /* Abort happened. */
+abort:
+
+ /* Call abort hook */
+ cmpq $0,__tsx_abort_hook(%rip)
+ jz 1f
+ push %rdi
+ cfi_adjust_cfa_offset(8)
+ push %rsi
+ cfi_adjust_cfa_offset(8)
+ mov %eax,%edi
+ call *__tsx_abort_hook(%rip)
+ pop %rsi
+ cfi_adjust_cfa_offset(-8)
+ pop %rdi
+ cfi_adjust_cfa_offset(-8)
+
+1:
+
+ /* No retries here */
+
+not_elided_timedwrlock:
+#endif
pushq %r12
cfi_adjust_cfa_offset(8)
cfi_rel_offset(%r12, 0)
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S
index e971529..3a240d1 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S
@@ -20,7 +20,7 @@
#include <lowlevellock.h>
#include <lowlevelrwlock.h>
#include <kernel-features.h>
-
+#include <hle.h>
.text
@@ -29,6 +29,21 @@
.align 16
__pthread_rwlock_unlock:
cfi_startproc
+
+ /* Is lock free? */
+ cmpl $0,WRITER(%rdi)
+ jnz 1f
+ cmpl $0,NR_READERS(%rdi)
+ jnz 1f
+
+ /* Looks free. Assume transaction.
+ If you crash here you unlocked a free lock. */
+ XEND
+ xor %rax,%rax
+ ret
+
+1:
+
/* Get the lock. */
movl $1, %esi
xorl %eax, %eax
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S
index ff5392c..5c8a5df 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S
@@ -22,6 +22,7 @@
#include <pthread-errnos.h>
#include <kernel-features.h>
#include <stap-probe.h>
+#include <hle.h>
.text
@@ -33,6 +34,49 @@ __pthread_rwlock_wrlock:
LIBC_PROBE (wrlock_entry, 1, %rdi)
+#ifdef ENABLE_ELISION
+ cmpq $0,ELIDING(%rdi)
+ js not_elided_wrlock
+ jnz 2f
+ /* zero: use default */
+
+ cmpl $0,__rwlock_rtm_enabled(%rip)
+ jz not_elided_wrlock
+
+2:
+ XBEGIN abort_wrlock
+
+ /* Lock free? */
+ cmpl $0,WRITER(%rdi)
+ jnz 1f
+ cmpl $0,NR_READERS(%rdi)
+ jnz 1f
+
+ /* Lock is free. Run with transaction */
+ xor %eax,%eax
+ ret
+
+ /* Lock is not free. End transaction */
+1: XABORT 0xff
+ jmp not_elided_wrlock
+
+ /* Abort happened. */
+abort_wrlock:
+
+ /* Call the abort hook */
+ cmpq $0,__tsx_abort_hook(%rip)
+ jz 1f
+ push %rdi
+ cfi_adjust_cfa_offset(8)
+ mov %eax,%edi
+ call *__tsx_abort_hook(%rip)
+ pop %rdi
+ cfi_adjust_cfa_offset(-8)
+
+1:
+
+not_elided_wrlock:
+#endif
xorq %r10, %r10
/* Get the lock. */
@@ -161,5 +205,7 @@ __pthread_rwlock_wrlock:
cfi_endproc
.size __pthread_rwlock_wrlock,.-__pthread_rwlock_wrlock
+#ifndef __pthread_rwlock_wrlock
strong_alias (__pthread_rwlock_wrlock, pthread_rwlock_wrlock)
hidden_def (__pthread_rwlock_wrlock)
+#endif
--
1.7.7.6