This is the mail archive of the libc-alpha@sourceware.org mailing list for the glibc project.


Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]
Other format: [Raw text]

[PATCH 05/14] Add lock elision to rwlocks


From: Andi Kleen <ak@linux.intel.com>

This code is very similar to the main pthread mutex code, but we have
to implement part of it in assembler.

The patch kit is large because there are so many combinations
of reader and writer locks, but it is fairly repetive.

The basic elision algorithm is very similar to normal mutexes:
a wrapper checks if the lock is free (both reader and writer)
We have to check both reader and writer because separating them
would require putting state on different cache lines.

The algorithm is very basic currently and may be tuned
more in the future.

No adaptation at this point, the elision will not automatically
disable itself. The programmer or user can do it explicitely though.

We use some padding in the internal rwlock as a elision flag.
This is 8 bytes on 64bits currently, but could be made smaller later
without compatibility problems (or to share some bytes with a
adaptation count). 32bit uses one byte.

2013-06-24  Andi Kleen <ak@linux.intel.com>
            Hongjiu Lu <hongjiu.lu@intel.com>

	* sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S: dito.
	* sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S: dito
	* sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S: dito.
	* sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S: dito.
	* sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S: dito.
	* sysdeps/unix/sysv/linux/internaltypes.h (struct pthread_condattr):
	(struct pthread_rwlockattr): Support elision flags in internal rwlock.
	* sysdeps/unix/sysv/linux/lowlevelrwlock.sym: Add RW_ELISION.
	* sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h: Add elision flags
	* sysdeps/unix/sysv/linux/x86/pthread_rwlock_tryrdlock.c: New file
	* sysdeps/unix/sysv/linux/x86/pthread_rwlock_trywrlock.c: dito.
	* sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S: Handle elision.
	* sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S: dito.
	* sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S: dito.
	* sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S: dito.
	* sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S: dito.
---
 nptl/pthread_rwlock_tryrdlock.c                    |  2 +
 nptl/pthread_rwlock_trywrlock.c                    |  2 +
 .../sysv/linux/i386/i486/pthread_rwlock_rdlock.S   | 65 ++++++++++++++++++++++
 .../linux/i386/i486/pthread_rwlock_timedrdlock.S   | 60 ++++++++++++++++++++
 .../linux/i386/i486/pthread_rwlock_timedwrlock.S   | 46 ++++++++++++++-
 .../sysv/linux/i386/i486/pthread_rwlock_unlock.S   | 19 +++++++
 .../sysv/linux/i386/i486/pthread_rwlock_wrlock.S   | 52 +++++++++++++++++
 nptl/sysdeps/unix/sysv/linux/internaltypes.h       | 16 +++++-
 nptl/sysdeps/unix/sysv/linux/lowlevelrwlock.sym    |  5 ++
 .../unix/sysv/linux/x86/bits/pthreadtypes.h        |  6 +-
 .../unix/sysv/linux/x86/pthread_rwlock_tryrdlock.c | 52 +++++++++++++++++
 .../unix/sysv/linux/x86/pthread_rwlock_trywrlock.c | 51 +++++++++++++++++
 .../unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S | 48 ++++++++++++++++
 .../sysv/linux/x86_64/pthread_rwlock_timedrdlock.S | 48 ++++++++++++++++
 .../sysv/linux/x86_64/pthread_rwlock_timedwrlock.S | 32 +++++++++++
 .../unix/sysv/linux/x86_64/pthread_rwlock_unlock.S | 17 +++++-
 .../unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S | 31 +++++++++++
 17 files changed, 546 insertions(+), 6 deletions(-)
 create mode 100644 nptl/sysdeps/unix/sysv/linux/x86/pthread_rwlock_tryrdlock.c
 create mode 100644 nptl/sysdeps/unix/sysv/linux/x86/pthread_rwlock_trywrlock.c

diff --git a/nptl/pthread_rwlock_tryrdlock.c b/nptl/pthread_rwlock_tryrdlock.c
index 935ac87..896809b 100644
--- a/nptl/pthread_rwlock_tryrdlock.c
+++ b/nptl/pthread_rwlock_tryrdlock.c
@@ -45,4 +45,6 @@ __pthread_rwlock_tryrdlock (pthread_rwlock_t *rwlock)
 
   return result;
 }
+#ifndef __pthread_rwlock_tryrdlock
 strong_alias (__pthread_rwlock_tryrdlock, pthread_rwlock_tryrdlock)
+#endif
diff --git a/nptl/pthread_rwlock_trywrlock.c b/nptl/pthread_rwlock_trywrlock.c
index 01754ae..6d8a8c4 100644
--- a/nptl/pthread_rwlock_trywrlock.c
+++ b/nptl/pthread_rwlock_trywrlock.c
@@ -38,4 +38,6 @@ __pthread_rwlock_trywrlock (pthread_rwlock_t *rwlock)
 
   return result;
 }
+#ifndef __pthread_rwlock_trywrlock
 strong_alias (__pthread_rwlock_trywrlock, pthread_rwlock_trywrlock)
+#endif
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S
index 6c46ba6..4151b03 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_rdlock.S
@@ -21,9 +21,16 @@
 #include <lowlevelrwlock.h>
 #include <pthread-errnos.h>
 #include <kernel-features.h>
+#include <hle.h>
 
 #include <stap-probe.h>
 
+#ifdef PIC
+#define MO(x) x##@GOTOFF(%edx)
+#else
+#define MO(x) x
+#endif
+
 	.text
 
 	.globl	__pthread_rwlock_rdlock
@@ -43,6 +50,62 @@ __pthread_rwlock_rdlock:
 
 	LIBC_PROBE (rdlock_entry, 1, %ebx)
 
+#ifdef PIC
+	SETUP_PIC_REG(dx)
+	addl $_GLOBAL_OFFSET_TABLE_,%edx
+#endif
+
+	cmpb  $0,RW_ELISION(%ebx)
+	js    not_elided_rdlock
+	jnz   2f
+	/* zero: use default  */
+
+	cmpl	$0,MO(__rwlock_rtm_enabled)
+	jz    not_elided_rdlock
+
+2:
+	mov   MO(__rwlock_rtm_read_retries),%ecx
+
+try_trans_rdlock:
+	XBEGIN abort_rdlock
+
+	/* Lock writer/reader free?  */
+	cmpl  $0,WRITER(%ebx)
+	jnz   1f
+	cmpl  $0,NR_READERS(%ebx)
+	jnz   1f
+
+	/* Lock is free. Run with transaction  */
+	xor   %eax,%eax
+
+	pop   %ebx
+	cfi_adjust_cfa_offset(-4)
+	pop   %esi
+	cfi_adjust_cfa_offset(-4)
+	ret
+
+	/* Lock is not free. Run  */
+1:	XABORT 0xff
+	jmp not_elided_rdlock
+
+	/* Abort happened.  */
+abort_rdlock:
+	testl $_XABORT_CONFLICT,%eax
+	jz    not_elided_rdlock
+
+	/* For a reader that aborts due a conflict retry speculation
+	   a limited number of times. This way when some reader aborts
+	   because the reader count is written the other readers will
+	   still elide, at the cost of retrying the speculation.  */
+
+	dec   %ecx
+	jnz   try_trans_rdlock
+
+	/* Otherwise we just fall back directly to the lock.
+	   Here's the place to add more adaptation.  */
+
+not_elided_rdlock:
+
 	/* Get the lock.  */
 	movl	$1, %edx
 	xorl	%eax, %eax
@@ -188,5 +251,7 @@ __pthread_rwlock_rdlock:
 	cfi_endproc
 	.size	__pthread_rwlock_rdlock,.-__pthread_rwlock_rdlock
 
+#ifndef __pthread_rwlock_rdlock
 strong_alias (__pthread_rwlock_rdlock, pthread_rwlock_rdlock)
 hidden_def (__pthread_rwlock_rdlock)
+#endif
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S
index 1908f6f..e710c49 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedrdlock.S
@@ -21,7 +21,13 @@
 #include <lowlevelrwlock.h>
 #include <pthread-errnos.h>
 #include <kernel-features.h>
+#include <hle.h>
 
+#ifdef PIC
+#define MO(x) x##@GOTOFF(%edx)
+#else
+#define MO(x) x
+#endif
 
 	.text
 
@@ -48,6 +54,59 @@ pthread_rwlock_timedrdlock:
 	movl	28(%esp), %ebp
 	movl	32(%esp), %edi
 
+#ifdef PIC
+	SETUP_PIC_REG(dx)
+	addl $_GLOBAL_OFFSET_TABLE_,%edx
+#endif
+
+	cmpb  $0,RW_ELISION(%ebp)
+	js    not_elided_trdlock
+	jnz   2f
+	/* zero: use default  */
+
+	cmpl  $0,MO(__rwlock_rtm_enabled)
+	jz    not_elided_trdlock
+
+2:
+	mov   MO(__rwlock_rtm_read_retries),%ecx
+
+try_trans_trdlock:
+	XBEGIN abort_trdlock
+
+	/* Lock writer/reader free?  */
+	cmpl  $0,WRITER(%ebp)
+	jnz   1f
+	cmpl  $0,NR_READERS(%ebp)
+	jnz   1f
+
+	/* Lock is free. Run with transaction  */
+	xor   %eax,%eax
+	jmp   77f
+
+
+	/* Lock is not free. Run  */
+1:	XABORT 0xff
+	jmp not_elided_trdlock
+
+	/* Abort happened.  */
+abort_trdlock:
+	testl $_XABORT_CONFLICT,%eax
+	jz    not_elided_trdlock
+
+	/* For a reader that aborts due a conflict retry speculation
+	   a limited number of times. This way when some reader aborts
+	   because the reader count is written the other readers will
+	   still elide, at the cost of retrying the speculation.  */
+
+	dec   %ecx
+	jnz   try_trans_trdlock
+
+	/* Otherwise we just fall back directly to the lock.
+	   Here's the place to add more adaptation.  */
+
+not_elided_trdlock:
+
+
 	/* Get the lock.  */
 	movl	$1, %edx
 	xorl	%eax, %eax
@@ -158,6 +217,7 @@ pthread_rwlock_timedrdlock:
 
 7:	movl	%edx, %eax
 
+77:
 	addl	$8, %esp
 	cfi_adjust_cfa_offset(-8)
 	popl	%ebp
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S
index e0fc809..c23f270 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_timedwrlock.S
@@ -21,7 +21,13 @@
 #include <lowlevelrwlock.h>
 #include <pthread-errnos.h>
 #include <kernel-features.h>
+#include <hle.h>
 
+#ifdef PIC
+#define MO(x) x##@GOTOFF(%edx)
+#else
+#define MO(x) x
+#endif
 
 	.text
 
@@ -48,6 +54,44 @@ pthread_rwlock_timedwrlock:
 	movl	28(%esp), %ebp
 	movl	32(%esp), %edi
 
+#ifdef PIC
+	SETUP_PIC_REG(dx)
+	addl $_GLOBAL_OFFSET_TABLE_,%edx
+#endif
+	cmpb  $0,RW_ELISION(%ebp)
+	js    not_elided_twrlock
+	jnz   try_trans_wrlock
+	/* zero: use default  */
+
+	cmpl	$0,MO(__rwlock_rtm_enabled)
+	jz	not_elided_twrlock
+
+try_trans_wrlock:
+	XBEGIN abort_twrlock
+
+	/* Lock writer free?  */
+	cmpl  $0,WRITER(%ebp)
+	jnz   1f
+	cmpl  $0,NR_READERS(%ebp)
+	jnz   1f
+
+	/* Lock is free. Run with transaction  */
+	xor   %eax,%eax
+
+	jmp   77f
+
+	/* Lock is not free. Run  */
+1:	XABORT 0xff
+	jmp   not_elided_twrlock
+
+	/* Abort happened.  */
+abort_twrlock:
+
+	/* Otherwise we just fall back directly to the lock.
+	   Here's the place to add more adaptation.  */
+
+not_elided_twrlock:
+
 	/* Get the lock.  */
 	movl	$1, %edx
 	xorl	%eax, %eax
@@ -156,7 +200,7 @@ pthread_rwlock_timedwrlock:
 
 7:	movl	%edx, %eax
 
-	addl	$8, %esp
+77:	addl	$8, %esp
 	cfi_adjust_cfa_offset(-8)
 	popl	%ebp
 	cfi_adjust_cfa_offset(-4)
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S
index 708e31c..9b384bb 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_unlock.S
@@ -20,6 +20,7 @@
 #include <lowlevellock.h>
 #include <lowlevelrwlock.h>
 #include <kernel-features.h>
+#include <hle.h>
 
 
 	.text
@@ -38,6 +39,24 @@ __pthread_rwlock_unlock:
 
 	movl	12(%esp), %edi
 
+	/* Is lock free?  */
+	cmpl $0,WRITER(%edi)
+	jnz  1f
+	cmpl $0,NR_READERS(%edi)
+	jnz  1f
+
+	/* Looks free. Assume transaction.
+	   If you crash here you unlocked a free lock.  */
+	XEND
+	xor  %eax,%eax
+
+	pop	%edi
+	cfi_adjust_cfa_offset(-4)
+	pop	%ebx
+	cfi_adjust_cfa_offset(-4)
+	ret
+
+1:
 	/* Get the lock.  */
 	movl	$1, %edx
 	xorl	%eax, %eax
diff --git a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S
index 6ea17f7..e0c3428 100644
--- a/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_rwlock_wrlock.S
@@ -21,9 +21,16 @@
 #include <lowlevelrwlock.h>
 #include <pthread-errnos.h>
 #include <kernel-features.h>
+#include <hle.h>
 
 #include <stap-probe.h>
 
+#ifdef PIC
+#define MO(x) x@GOTOFF(%edx)
+#else
+#define MO(x) x
+#endif
+
 	.text
 
 	.globl	__pthread_rwlock_wrlock
@@ -43,6 +50,49 @@ __pthread_rwlock_wrlock:
 
 	LIBC_PROBE (wrlock_entry, 1, %ebx)
 
+#ifdef PIC
+	SETUP_PIC_REG(dx)
+	addl $_GLOBAL_OFFSET_TABLE_,%edx
+#endif
+	cmpb  $0,RW_ELISION(%ebx)
+	js    not_elided_wrlock
+	jnz   try_trans_wrlock
+	/* zero: use default  */
+
+	cmpl    $0,MO(__rwlock_rtm_enabled)
+	jz      not_elided_wrlock
+
+try_trans_wrlock:
+	XBEGIN abort_wrlock
+
+	/* Lock writer free?  */
+	/* Ignore readers because we don't need them  */
+	cmpl  $0,WRITER(%ebx)
+	jnz   1f
+	cmpl  $0,NR_READERS(%ebx)
+	jnz   1f
+
+	/* Lock is free. Run with transaction  */
+	xor   %eax,%eax
+
+	pop   %ebx
+	cfi_adjust_cfa_offset(-4)
+	pop   %esi
+	cfi_adjust_cfa_offset(-4)
+	ret
+
+	/* Lock is not free. Run  */
+1:	XABORT 0xff
+	jmp   not_elided_wrlock
+
+	/* Abort happened.  */
+abort_wrlock:
+
+	/* Otherwise we just fall back directly to the lock.
+	   Here's the place to add more adaptation.  */
+
+not_elided_wrlock:
+
 	/* Get the lock.  */
 	movl	$1, %edx
 	xorl	%eax, %eax
@@ -179,5 +229,7 @@ __pthread_rwlock_wrlock:
 	cfi_endproc
 	.size	__pthread_rwlock_wrlock,.-__pthread_rwlock_wrlock
 
+#ifndef __pthread_rwlock_wrlock
 strong_alias (__pthread_rwlock_wrlock, pthread_rwlock_wrlock)
 hidden_def (__pthread_rwlock_wrlock)
+#endif
diff --git a/nptl/sysdeps/unix/sysv/linux/internaltypes.h b/nptl/sysdeps/unix/sysv/linux/internaltypes.h
index 699a618..70aaa40 100644
--- a/nptl/sysdeps/unix/sysv/linux/internaltypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/internaltypes.h
@@ -20,7 +20,7 @@
 #define _INTERNALTYPES_H	1
 
 #include <stdint.h>
-
+#include <endian.h>
 
 struct pthread_attr
 {
@@ -86,7 +86,19 @@ struct pthread_condattr
 struct pthread_rwlockattr
 {
   int lockkind;
-  int pshared;
+  /* This used to be an int pshared. We keep the pshared at the
+     same location.
+     PTHREAD_PROCESS_SHARED and PTHREAD_PROCESS_PRIVATE, both fit into one
+     byte.  */
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+  short pshared;
+  char  rw_elision;
+  char  pad;
+#else
+  char  rw_elision;
+  char  pad;
+  short pshared;
+#endif
 };
 
 
diff --git a/nptl/sysdeps/unix/sysv/linux/lowlevelrwlock.sym b/nptl/sysdeps/unix/sysv/linux/lowlevelrwlock.sym
index f50b25b..6d9d201 100644
--- a/nptl/sysdeps/unix/sysv/linux/lowlevelrwlock.sym
+++ b/nptl/sysdeps/unix/sysv/linux/lowlevelrwlock.sym
@@ -3,6 +3,10 @@
 #include <bits/pthreadtypes.h>
 #include <bits/wordsize.h>
 
+#ifndef __PTHREAD_RWLOCK_ELISION
+#define __rw_elision __shared
+#endif
+
 --
 
 MUTEX		offsetof (pthread_rwlock_t, __data.__lock)
@@ -14,3 +18,4 @@ WRITERS_QUEUED	offsetof (pthread_rwlock_t, __data.__nr_writers_queued)
 FLAGS		offsetof (pthread_rwlock_t, __data.__flags)
 WRITER		offsetof (pthread_rwlock_t, __data.__writer)
 PSHARED		offsetof (pthread_rwlock_t, __data.__shared)
+RW_ELISION		offsetof (pthread_rwlock_t, __data.__rw_elision)
diff --git a/nptl/sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h b/nptl/sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h
index 1852e07..c3325d9 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h
+++ b/nptl/sysdeps/unix/sysv/linux/x86/bits/pthreadtypes.h
@@ -183,12 +183,13 @@ typedef union
     unsigned int __nr_writers_queued;
     int __writer;
     int __shared;
-    unsigned long int __pad1;
+    long __rw_elision;
     unsigned long int __pad2;
     /* FLAGS must stay at this position in the structure to maintain
        binary compatibility.  */
     unsigned int __flags;
 # define __PTHREAD_RWLOCK_INT_FLAGS_SHARED	1
+# define __PTHREAD_RWLOCK_ELISION		1
   } __data;
 # else
   struct
@@ -203,8 +204,9 @@ typedef union
        binary compatibility.  */
     unsigned char __flags;
     unsigned char __shared;
-    unsigned char __pad1;
+    char __rw_elision;
     unsigned char __pad2;
+# define __PTHREAD_RWLOCK_ELISION		2
     int __writer;
   } __data;
 # endif
diff --git a/nptl/sysdeps/unix/sysv/linux/x86/pthread_rwlock_tryrdlock.c b/nptl/sysdeps/unix/sysv/linux/x86/pthread_rwlock_tryrdlock.c
new file mode 100644
index 0000000..f18295c
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/x86/pthread_rwlock_tryrdlock.c
@@ -0,0 +1,52 @@
+/* pthread_rwlock_tryrdlock: Lock elision version of pthreads rwlock_tryrdlock.
+   Copyright (C) 2013 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+#include <pthread.h>
+#include <pthreadP.h>
+#include <hle.h>
+#include <elision-conf.h>
+#include "init-arch.h"
+
+#define __pthread_rwlock_tryrdlock __full_pthread_rwlock_tryrdlock
+#include <nptl/pthread_rwlock_tryrdlock.c>
+#undef __pthread_rwlock_tryrdlock
+
+int
+__pthread_rwlock_tryrdlock(pthread_rwlock_t *rwlock)
+{
+  unsigned status;
+
+  if (rwlock->__data.__rw_elision > 0
+      || (rwlock->__data.__rw_elision == 0 && __rwlock_rtm_enabled))
+    {
+      if ((status = _xbegin()) == _XBEGIN_STARTED)
+	{
+	  if (rwlock->__data.__writer == 0
+	      && rwlock->__data.__nr_readers == 0)
+	    return 0;
+	  /* Lock was busy. Fall back to normal locking.
+	     Could also _xend here but xabort with 0xff code
+	     is more visible in the profiler.  */
+	  _xabort (_ABORT_LOCK_BUSY);
+	}
+      /* Aborts come here  */
+    }
+
+  return __full_pthread_rwlock_tryrdlock (rwlock);
+}
+
+strong_alias(__pthread_rwlock_tryrdlock, pthread_rwlock_tryrdlock);
diff --git a/nptl/sysdeps/unix/sysv/linux/x86/pthread_rwlock_trywrlock.c b/nptl/sysdeps/unix/sysv/linux/x86/pthread_rwlock_trywrlock.c
new file mode 100644
index 0000000..0f5cd3c
--- /dev/null
+++ b/nptl/sysdeps/unix/sysv/linux/x86/pthread_rwlock_trywrlock.c
@@ -0,0 +1,51 @@
+/* pthread_rwlock_trywrlock: Lock elision version of pthreads rwlock trywrlock.
+   Copyright (C) 2013 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public
+   License as published by the Free Software Foundation; either
+   version 2.1 of the License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; if not, see
+   <http://www.gnu.org/licenses/>.  */
+#include <pthread.h>
+#include <pthreadP.h>
+#include <hle.h>
+#include <elision-conf.h>
+#include "init-arch.h"
+
+#define __pthread_rwlock_trywrlock __full_pthread_rwlock_trywrlock
+#include <nptl/pthread_rwlock_trywrlock.c>
+#undef __pthread_rwlock_trywrlock
+
+int
+__pthread_rwlock_trywrlock(pthread_rwlock_t *rwlock)
+{
+  unsigned status;
+
+  if ((rwlock->__data.__rw_elision == 0 && __rwlock_rtm_enabled)
+       || rwlock->__data.__rw_elision > 0)
+    {
+      _xabort (_ABORT_NESTED_TRYLOCK);
+      if ((status = _xbegin()) == _XBEGIN_STARTED)
+	{
+	  if (rwlock->__data.__writer == 0
+	      && rwlock->__data.__nr_readers == 0)
+	    return 0;
+	  /* Lock was busy. Fall back to normal locking.
+	     Could also _xend here but xabort with 0xff code
+	     is more visible in the profiler.  */
+	  _xabort (_ABORT_LOCK_BUSY);
+	}
+    }
+
+  return __full_pthread_rwlock_trywrlock (rwlock);
+}
+strong_alias(__pthread_rwlock_trywrlock, pthread_rwlock_trywrlock);
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S
index 7681818..3b335bc 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_rdlock.S
@@ -22,6 +22,7 @@
 #include <pthread-errnos.h>
 #include <kernel-features.h>
 #include <stap-probe.h>
+#include <hle.h>
 
 	.text
 
@@ -33,6 +34,51 @@ __pthread_rwlock_rdlock:
 
 	LIBC_PROBE (rdlock_entry, 1, %rdi)
 
+	cmpq  $0,RW_ELISION(%rdi)
+	js    not_elided_rdlock
+	jnz   2f
+	/* zero: use default  */
+
+	cmpl  $0,__rwlock_rtm_enabled(%rip)
+	jz    not_elided_rdlock
+
+2:
+	mov   __rwlock_rtm_read_retries(%rip),%esi
+
+try_trans_rdlock:
+	XBEGIN abort_rdlock
+
+	/* Lock reader/writer free?  */
+	cmpl  $0,WRITER(%rdi)
+	jnz   1f
+	cmpl  $0,NR_READERS(%rdi)
+	jnz   1f
+
+	/* Lock is free. Run with transaction  */
+	xor   %eax,%eax
+	ret
+
+	/* Lock is not free. Run  */
+1:	XABORT 0xff
+	jmp not_elided_rdlock
+
+	/* Abort happened.  */
+abort_rdlock:
+	testl $_XABORT_CONFLICT,%eax
+	jz    not_elided_rdlock
+
+	/* For a reader that aborts due a conflict retry speculation
+	   a limited number of times. This way when some reader aborts
+	   because the reader count is written the other readers will
+	   still elide, at the cost of retrying the speculation.  */
+
+	dec   %esi
+	jnz   try_trans_rdlock
+
+	/* Otherwise we just fall back directly to the lock.
+	   Here's the place to add more adaptation.  */
+
+not_elided_rdlock:
 	xorq	%r10, %r10
 
 	/* Get the lock.  */
@@ -173,5 +219,7 @@ __pthread_rwlock_rdlock:
 	cfi_endproc
 	.size	__pthread_rwlock_rdlock,.-__pthread_rwlock_rdlock
 
+#ifndef __pthread_rwlock_rdlock
 strong_alias (__pthread_rwlock_rdlock, pthread_rwlock_rdlock)
 hidden_def (__pthread_rwlock_rdlock)
+#endif
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S
index 57fe1e9..2bc9037 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedrdlock.S
@@ -21,6 +21,7 @@
 #include <lowlevelrwlock.h>
 #include <pthread-errnos.h>
 #include <kernel-features.h>
+#include <hle.h>
 
 	.text
 
@@ -29,6 +30,53 @@
 	.align	16
 pthread_rwlock_timedrdlock:
 	cfi_startproc
+
+	cmpq  $0,RW_ELISION(%rdi)
+	js    not_elided_timedrdlock
+	jnz   2f
+	/* zero: use default  */
+
+	cmpl  $0,__rwlock_rtm_enabled(%rip)
+	jz    not_elided_timedrdlock
+
+2:	
+	mov   __rwlock_rtm_read_retries(%rip),%ecx
+
+try_trans_timedrdlock:
+	XBEGIN abort_timedrdlock
+
+	/* Lock reader/writer free?  */
+	cmpl  $0,WRITER(%rdi)
+	jnz   1f
+	cmpl  $0,NR_READERS(%rdi)
+	jnz   1f
+
+	/* Lock is free. Run with transaction  */
+	xor   %eax,%eax
+	ret
+
+	/* Lock is not free. Run  */
+1:	XABORT 0xff
+	jmp not_elided_timedrdlock
+
+	/* Abort happened.  */
+abort_timedrdlock:
+	testl $_XABORT_CONFLICT,%eax
+	jz    not_elided_timedrdlock
+
+	/* For a reader that aborts due a conflict retry speculation
+	   a limited number of times. This way when some reader aborts
+	   because the reader count is written the other readers will
+	   still elide, at the cost of retrying the speculation.  */
+
+	dec   %ecx
+	jnz   try_trans_timedrdlock
+
+	/* Otherwise we just fall back directly to the lock.
+	   Here's the place to add more adaptation.  */
+
+not_elided_timedrdlock:
+
 	pushq	%r12
 	cfi_adjust_cfa_offset(8)
 	cfi_rel_offset(%r12, 0)
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S
index 391be17..701c07a 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_timedwrlock.S
@@ -21,6 +21,7 @@
 #include <lowlevelrwlock.h>
 #include <pthread-errnos.h>
 #include <kernel-features.h>
+#include <hle.h>
 
 	.text
 
@@ -29,6 +30,37 @@
 	.align	16
 pthread_rwlock_timedwrlock:
 	cfi_startproc
+
+	cmpq  $0,RW_ELISION(%rdi)
+	js    not_elided_timedwrlock
+	jnz   2f
+	/* zero: use default  */
+
+	cmpl  $0,__rwlock_rtm_enabled(%rip)
+	jz    not_elided_timedwrlock
+
+2:
+	XBEGIN abort
+
+	/* Lock free?  */
+	cmpl  $0,NR_READERS(%rdi)
+	jnz   1f
+	cmpl  $0,WRITER(%rdi)
+	jnz   1f
+
+	/* Lock is free. Run with transaction  */
+	xor   %rax, %rax
+	ret
+
+	/* Lock is not free. Run  */
+1:	XABORT 0xff
+	jmp   not_elided_timedwrlock
+
+	/* Abort happened.  */
+abort:
+	/* No retries here  */
+
+not_elided_timedwrlock:
 	pushq	%r12
 	cfi_adjust_cfa_offset(8)
 	cfi_rel_offset(%r12, 0)
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S
index 86dda05..f1b9516 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_unlock.S
@@ -20,7 +20,7 @@
 #include <lowlevellock.h>
 #include <lowlevelrwlock.h>
 #include <kernel-features.h>
-
+#include <hle.h>
 
 	.text
 
@@ -29,6 +29,21 @@
 	.align	16
 __pthread_rwlock_unlock:
 	cfi_startproc
+
+	/* Is lock free?  */
+	cmpl $0,WRITER(%rdi)
+	jnz  1f
+	cmpl $0,NR_READERS(%rdi)
+	jnz  1f
+
+	/* Looks free. Assume transaction.
+	   If you crash here you unlocked a free lock.  */
+	XEND
+	xor  %rax,%rax
+	ret
+
+1:
+
 	/* Get the lock.  */
 	movl	$1, %esi
 	xorl	%eax, %eax
diff --git a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S
index 734bee3..28d1b4a 100644
--- a/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S
+++ b/nptl/sysdeps/unix/sysv/linux/x86_64/pthread_rwlock_wrlock.S
@@ -22,6 +22,7 @@
 #include <pthread-errnos.h>
 #include <kernel-features.h>
 #include <stap-probe.h>
+#include <hle.h>
 
 	.text
 
@@ -33,6 +34,34 @@ __pthread_rwlock_wrlock:
 
 	LIBC_PROBE (wrlock_entry, 1, %rdi)
 
+	cmpq  $0,RW_ELISION(%rdi)
+	js    not_elided_wrlock
+	jnz   2f
+	/* zero: use default  */
+
+	cmpl  $0,__rwlock_rtm_enabled(%rip)
+	jz    not_elided_wrlock
+
+2:
+	XBEGIN abort_wrlock
+
+	/* Lock free?  */
+	cmpl  $0,WRITER(%rdi)
+	jnz   1f
+	cmpl  $0,NR_READERS(%rdi)
+	jnz   1f
+
+	/* Lock is free. Run with transaction  */
+	xor   %eax,%eax
+	ret
+
+	/* Lock is not free. End transaction  */
+1:	XABORT 0xff
+	jmp   not_elided_wrlock
+
+	/* Abort happened.  */
+abort_wrlock:
+not_elided_wrlock:
 	xorq	%r10, %r10
 
 	/* Get the lock.  */
@@ -161,5 +190,7 @@ __pthread_rwlock_wrlock:
 	cfi_endproc
 	.size	__pthread_rwlock_wrlock,.-__pthread_rwlock_wrlock
 
+#ifndef __pthread_rwlock_wrlock
 strong_alias (__pthread_rwlock_wrlock, pthread_rwlock_wrlock)
 hidden_def (__pthread_rwlock_wrlock)
+#endif
-- 
1.8.1.4


Index Nav: [Date Index] [Subject Index] [Author Index] [Thread Index]
Message Nav: [Date Prev] [Date Next] [Thread Prev] [Thread Next]