This is the mail archive of the
glibc-cvs@sourceware.org
mailing list for the glibc project.
GNU C Library master sources branch master updated. glibc-2.17-241-g8313cb9
- From: siddhesh at sourceware dot org
- To: glibc-cvs at sourceware dot org
- Date: 18 Feb 2013 10:38:10 -0000
- Subject: GNU C Library master sources branch master updated. glibc-2.17-241-g8313cb9
This is an automated email from the git hooks/post-receive script. It was
generated because a ref change was pushed to the repository containing
the project "GNU C Library master sources".
The branch, master has been updated
via 8313cb997d2da2465c8560d3164358a68ea1e9ad (commit)
from f78b5caa6ece23ce86f6cabac8edf3ecd6850473 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
- Log -----------------------------------------------------------------
http://sources.redhat.com/git/gitweb.cgi?p=glibc.git;a=commitdiff;h=8313cb997d2da2465c8560d3164358a68ea1e9ad
commit 8313cb997d2da2465c8560d3164358a68ea1e9ad
Author: Siddhesh Poyarekar <siddhesh@redhat.com>
Date: Mon Feb 18 16:07:10 2013 +0530
FUTEX_*_REQUEUE_PI support for non-x86 code
Add FUTEX_*_REQUEUE_PI support for the default C code and also add
implementations for s-390 and ppc.
diff --git a/NEWS b/NEWS
index 769ae61..814fe37 100644
--- a/NEWS
+++ b/NEWS
@@ -9,9 +9,9 @@ Version 2.18
* The following bugs are resolved with this release:
- 11561, 13951, 14142, 14200, 14317, 14327, 14496, 14964, 14981, 14982,
- 14985, 14994, 14996, 15003, 15006, 15020, 15023, 15036, 15054, 15062,
- 15078.
+ 11561, 13951, 14142, 14200, 14317, 14327, 14496, 14920, 14964, 14981,
+ 14982, 14985, 14994, 14996, 15003, 15006, 15020, 15023, 15036, 15054,
+ 15062, 15078.
Version 2.17
diff --git a/nptl/ChangeLog b/nptl/ChangeLog
index bcc1660..8fedcf4 100644
--- a/nptl/ChangeLog
+++ b/nptl/ChangeLog
@@ -1,3 +1,33 @@
+2013-02-18 Siddhesh Poyarekar <siddhesh@redhat.com>
+
+ [BZ #14920]
+ * pthreadP.h (USE_REQUEUE_PI): New macro to check if mutex is
+ PI-aware.
+ * pthread_cond_broadcast.c (__pthread_cond_broadcast): Use
+ PI-aware futex operations if available and mutex is PI-aware.
+ * pthread_cond_signal.c (__pthread_cond_signal): Likewise.
+ * nptl/pthread_cond_timedwait.c (__pthread_cond_timedwait):
+ Likewise.
+ * pthread_cond_wait.c (__condvar_cleanup): Adjust lock if
+ cancellation occurred just after futex returned successfully
+ from a PI operation with the mutex held.
+ (__pthread_cond_wait): Use PI-aware futex operations if
+ available and mutex is PI-aware.
+ * sysdeps/unix/sysv/linux/powerpc/lowlevellock.h
+ (FUTEX_WAIT_REQUEUE_PI): Define.
+ (FUTEX_CMP_REQUEUE_PI): Likewise.
+ (lll_futex_wait_requeue_pi): Likewise.
+ (lll_futex_timed_wait_requeue_pi): Likewise.
+ (lll_futex_cmp_requeue_pi): Likewise.
+ * nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h
+ (FUTEX_WAIT_REQUEUE_PI): Define.
+ (FUTEX_CMP_REQUEUE_PI): Likewise.
+ (lll_futex_wait_requeue_pi): Likewise.
+ (lll_futex_timed_wait_requeue_pi): Likewise.
+ (lll_futex_cmp_requeue_pi): Likewise.
+ * sysdeps/unix/sysv/linux/kernel-features.h: Define
+ __ASSUME_REQUEUE_PI for Linux version higher than 2.6.31.
+
2013-02-04 Andreas Schwab <schwab@suse.de>
[BZ #14142]
diff --git a/nptl/pthreadP.h b/nptl/pthreadP.h
index 993a79e..d08b219 100644
--- a/nptl/pthreadP.h
+++ b/nptl/pthreadP.h
@@ -577,4 +577,16 @@ extern void __wait_lookup_done (void) attribute_hidden;
# define PTHREAD_STATIC_FN_REQUIRE(name) __asm (".globl " #name);
#endif
+/* Test if the mutex is suitable for the FUTEX_WAIT_REQUEUE_PI operation. */
+#if (defined lll_futex_wait_requeue_pi \
+ && defined __ASSUME_REQUEUE_PI)
+# define USE_REQUEUE_PI(mut) \
+ ((mut) && (mut) != (void *) ~0l \
+ && (((mut)->__data.__kind \
+ & (PTHREAD_MUTEX_PRIO_INHERIT_NP | PTHREAD_MUTEX_ROBUST_NORMAL_NP)) \
+ == PTHREAD_MUTEX_PRIO_INHERIT_NP))
+#else
+# define USE_REQUEUE_PI(mut) 0
+#endif
+
#endif /* pthreadP.h */
diff --git a/nptl/pthread_cond_broadcast.c b/nptl/pthread_cond_broadcast.c
index 968ee03..0702ec0 100644
--- a/nptl/pthread_cond_broadcast.c
+++ b/nptl/pthread_cond_broadcast.c
@@ -53,34 +53,37 @@ __pthread_cond_broadcast (cond)
/* We are done. */
lll_unlock (cond->__data.__lock, pshared);
- /* Do not use requeue for pshared condvars. */
- if (cond->__data.__mutex == (void *) ~0l)
- goto wake_all;
-
/* Wake everybody. */
pthread_mutex_t *mut = (pthread_mutex_t *) cond->__data.__mutex;
- /* XXX: Kernel so far doesn't support requeue to PI futex. */
- /* XXX: Kernel so far can only requeue to the same type of futex,
- in this case private (we don't requeue for pshared condvars). */
- if (__builtin_expect (mut->__data.__kind
- & (PTHREAD_MUTEX_PRIO_INHERIT_NP
- | PTHREAD_MUTEX_PSHARED_BIT), 0))
+ /* Do not use requeue for pshared condvars. */
+ if (mut == (void *) ~0l
+ || PTHREAD_MUTEX_PSHARED (mut) & PTHREAD_MUTEX_PSHARED_BIT)
goto wake_all;
- /* lll_futex_requeue returns 0 for success and non-zero
- for errors. */
- if (__builtin_expect (lll_futex_requeue (&cond->__data.__futex, 1,
- INT_MAX, &mut->__data.__lock,
- futex_val, LLL_PRIVATE), 0))
+#if (defined lll_futex_cmp_requeue_pi \
+ && defined __ASSUME_REQUEUE_PI)
+ int pi_flag = PTHREAD_MUTEX_PRIO_INHERIT_NP | PTHREAD_MUTEX_ROBUST_NP;
+ pi_flag &= mut->__data.__kind;
+
+ if (pi_flag == PTHREAD_MUTEX_PRIO_INHERIT_NP)
{
- /* The requeue functionality is not available. */
- wake_all:
- lll_futex_wake (&cond->__data.__futex, INT_MAX, pshared);
+ if (lll_futex_cmp_requeue_pi (&cond->__data.__futex, 1, INT_MAX,
+ &mut->__data.__lock, futex_val,
+ LLL_PRIVATE) == 0)
+ return 0;
}
-
- /* That's all. */
- return 0;
+ else
+#endif
+ /* lll_futex_requeue returns 0 for success and non-zero
+ for errors. */
+ if (!__builtin_expect (lll_futex_requeue (&cond->__data.__futex, 1,
+ INT_MAX, &mut->__data.__lock,
+ futex_val, LLL_PRIVATE), 0))
+ return 0;
+
+wake_all:
+ lll_futex_wake (&cond->__data.__futex, INT_MAX, pshared);
}
/* We are done. */
diff --git a/nptl/pthread_cond_signal.c b/nptl/pthread_cond_signal.c
index 908a2ac..102d0b3 100644
--- a/nptl/pthread_cond_signal.c
+++ b/nptl/pthread_cond_signal.c
@@ -47,12 +47,35 @@ __pthread_cond_signal (cond)
++cond->__data.__wakeup_seq;
++cond->__data.__futex;
- /* Wake one. */
- if (! __builtin_expect (lll_futex_wake_unlock (&cond->__data.__futex, 1,
- 1, &cond->__data.__lock,
- pshared), 0))
- return 0;
+#if (defined lll_futex_cmp_requeue_pi \
+ && defined __ASSUME_REQUEUE_PI)
+ int pi_flag = PTHREAD_MUTEX_PRIO_INHERIT_NP | PTHREAD_MUTEX_ROBUST_NP;
+ pthread_mutex_t *mut = cond->__data.__mutex;
+ /* Do not use requeue for pshared condvars. */
+ if (mut != (void *) ~0l)
+ pi_flag &= mut->__data.__kind;
+
+ if (__builtin_expect (pi_flag == PTHREAD_MUTEX_PRIO_INHERIT_NP, 0)
+ /* This can only really fail with a ENOSYS, since nobody can modify
+ futex while we have the cond_lock. */
+ && lll_futex_cmp_requeue_pi (&cond->__data.__futex, 1, 0,
+ &mut->__data.__lock,
+ cond->__data.__futex, pshared) == 0)
+ {
+ lll_unlock (cond->__data.__lock, pshared);
+ return 0;
+ }
+ else
+#endif
+ /* Wake one. */
+ if (! __builtin_expect (lll_futex_wake_unlock (&cond->__data.__futex,
+ 1, 1,
+ &cond->__data.__lock,
+ pshared), 0))
+ return 0;
+
+ /* Fallback if neither of them work. */
lll_futex_wake (&cond->__data.__futex, 1, pshared);
}
diff --git a/nptl/pthread_cond_timedwait.c b/nptl/pthread_cond_timedwait.c
index 0f52bd8..0a2d092 100644
--- a/nptl/pthread_cond_timedwait.c
+++ b/nptl/pthread_cond_timedwait.c
@@ -64,6 +64,11 @@ __pthread_cond_timedwait (cond, mutex, abstime)
int pshared = (cond->__data.__mutex == (void *) ~0l)
? LLL_SHARED : LLL_PRIVATE;
+#if (defined lll_futex_timed_wait_requeue_pi \
+ && defined __ASSUME_REQUEUE_PI)
+ int pi_flag = 0;
+#endif
+
/* Make sure we are alone. */
lll_lock (cond->__data.__lock, pshared);
@@ -155,17 +160,46 @@ __pthread_cond_timedwait (cond, mutex, abstime)
/* Enable asynchronous cancellation. Required by the standard. */
cbuffer.oldtype = __pthread_enable_asynccancel ();
+/* REQUEUE_PI was implemented after FUTEX_CLOCK_REALTIME, so it is sufficient
+ to check just the former. */
+#if (defined lll_futex_timed_wait_requeue_pi \
+ && defined __ASSUME_REQUEUE_PI)
+ /* If pi_flag remained 1 then it means that we had the lock and the mutex
+ but a spurious waker raced ahead of us. Give back the mutex before
+ going into wait again. */
+ if (pi_flag)
+ {
+ __pthread_mutex_cond_lock_adjust (mutex);
+ __pthread_mutex_unlock_usercnt (mutex, 0);
+ }
+ pi_flag = USE_REQUEUE_PI (mutex);
+
+ if (pi_flag)
+ {
+ unsigned int clockbit = (cond->__data.__nwaiters & 1
+ ? 0 : FUTEX_CLOCK_REALTIME);
+ err = lll_futex_timed_wait_requeue_pi (&cond->__data.__futex,
+ futex_val, abstime, clockbit,
+ &mutex->__data.__lock,
+ pshared);
+ pi_flag = (err == 0);
+ }
+ else
+#endif
+
+ {
#if (!defined __ASSUME_FUTEX_CLOCK_REALTIME \
|| !defined lll_futex_timed_wait_bitset)
- /* Wait until woken by signal or broadcast. */
- err = lll_futex_timed_wait (&cond->__data.__futex,
- futex_val, &rt, pshared);
+ /* Wait until woken by signal or broadcast. */
+ err = lll_futex_timed_wait (&cond->__data.__futex,
+ futex_val, &rt, pshared);
#else
- unsigned int clockbit = (cond->__data.__nwaiters & 1
- ? 0 : FUTEX_CLOCK_REALTIME);
- err = lll_futex_timed_wait_bitset (&cond->__data.__futex, futex_val,
- abstime, clockbit, pshared);
+ unsigned int clockbit = (cond->__data.__nwaiters & 1
+ ? 0 : FUTEX_CLOCK_REALTIME);
+ err = lll_futex_timed_wait_bitset (&cond->__data.__futex, futex_val,
+ abstime, clockbit, pshared);
#endif
+ }
/* Disable asynchronous cancellation. */
__pthread_disable_asynccancel (cbuffer.oldtype);
@@ -217,7 +251,16 @@ __pthread_cond_timedwait (cond, mutex, abstime)
__pthread_cleanup_pop (&buffer, 0);
/* Get the mutex before returning. */
- err = __pthread_mutex_cond_lock (mutex);
+#if (defined lll_futex_timed_wait_requeue_pi \
+ && defined __ASSUME_REQUEUE_PI)
+ if (pi_flag)
+ {
+ __pthread_mutex_cond_lock_adjust (mutex);
+ err = 0;
+ }
+ else
+#endif
+ err = __pthread_mutex_cond_lock (mutex);
return err ?: result;
}
diff --git a/nptl/pthread_cond_wait.c b/nptl/pthread_cond_wait.c
index 0ae320c..01d42d7 100644
--- a/nptl/pthread_cond_wait.c
+++ b/nptl/pthread_cond_wait.c
@@ -26,7 +26,6 @@
#include <shlib-compat.h>
#include <stap-probe.h>
-
struct _condvar_cleanup_buffer
{
int oldtype;
@@ -85,8 +84,15 @@ __condvar_cleanup (void *arg)
lll_futex_wake (&cbuffer->cond->__data.__futex, INT_MAX, pshared);
/* Get the mutex before returning unless asynchronous cancellation
- is in effect. */
- __pthread_mutex_cond_lock (cbuffer->mutex);
+ is in effect. We don't try to get the mutex if we already own it. */
+ if (!(USE_REQUEUE_PI (cbuffer->mutex))
+ || ((cbuffer->mutex->__data.__lock & FUTEX_TID_MASK)
+ != THREAD_GETMEM (THREAD_SELF, tid)))
+ {
+ __pthread_mutex_cond_lock (cbuffer->mutex);
+ }
+ else
+ __pthread_mutex_cond_lock_adjust (cbuffer->mutex);
}
@@ -101,6 +107,11 @@ __pthread_cond_wait (cond, mutex)
int pshared = (cond->__data.__mutex == (void *) ~0l)
? LLL_SHARED : LLL_PRIVATE;
+#if (defined lll_futex_wait_requeue_pi \
+ && defined __ASSUME_REQUEUE_PI)
+ int pi_flag = 0;
+#endif
+
LIBC_PROBE (cond_wait, 2, cond, mutex);
/* Make sure we are alone. */
@@ -144,15 +155,36 @@ __pthread_cond_wait (cond, mutex)
do
{
unsigned int futex_val = cond->__data.__futex;
-
/* Prepare to wait. Release the condvar futex. */
lll_unlock (cond->__data.__lock, pshared);
/* Enable asynchronous cancellation. Required by the standard. */
cbuffer.oldtype = __pthread_enable_asynccancel ();
- /* Wait until woken by signal or broadcast. */
- lll_futex_wait (&cond->__data.__futex, futex_val, pshared);
+#if (defined lll_futex_wait_requeue_pi \
+ && defined __ASSUME_REQUEUE_PI)
+ /* If pi_flag remained 1 then it means that we had the lock and the mutex
+ but a spurious waker raced ahead of us. Give back the mutex before
+ going into wait again. */
+ if (pi_flag)
+ {
+ __pthread_mutex_cond_lock_adjust (mutex);
+ __pthread_mutex_unlock_usercnt (mutex, 0);
+ }
+ pi_flag = USE_REQUEUE_PI (mutex);
+
+ if (pi_flag)
+ {
+ err = lll_futex_wait_requeue_pi (&cond->__data.__futex,
+ futex_val, &mutex->__data.__lock,
+ pshared);
+
+ pi_flag = (err == 0);
+ }
+ else
+#endif
+ /* Wait until woken by signal or broadcast. */
+ lll_futex_wait (&cond->__data.__futex, futex_val, pshared);
/* Disable asynchronous cancellation. */
__pthread_disable_asynccancel (cbuffer.oldtype);
@@ -189,8 +221,17 @@ __pthread_cond_wait (cond, mutex)
/* The cancellation handling is back to normal, remove the handler. */
__pthread_cleanup_pop (&buffer, 0);
- /* Get the mutex before returning. */
- return __pthread_mutex_cond_lock (mutex);
+ /* Get the mutex before returning. Not needed for PI. */
+#if (defined lll_futex_wait_requeue_pi \
+ && defined __ASSUME_REQUEUE_PI)
+ if (pi_flag)
+ {
+ __pthread_mutex_cond_lock_adjust (mutex);
+ return 0;
+ }
+ else
+#endif
+ return __pthread_mutex_cond_lock (mutex);
}
versioned_symbol (libpthread, __pthread_cond_wait, pthread_cond_wait,
diff --git a/nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h
index b4b1fd4..f33f703 100644
--- a/nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/powerpc/lowlevellock.h
@@ -39,6 +39,8 @@
#define FUTEX_TRYLOCK_PI 8
#define FUTEX_WAIT_BITSET 9
#define FUTEX_WAKE_BITSET 10
+#define FUTEX_WAIT_REQUEUE_PI 11
+#define FUTEX_CMP_REQUEUE_PI 12
#define FUTEX_PRIVATE_FLAG 128
#define FUTEX_CLOCK_REALTIME 256
@@ -149,6 +151,34 @@
INTERNAL_SYSCALL_ERROR_P (__ret, __err); \
})
+/* Priority Inheritance support. */
+#define lll_futex_wait_requeue_pi(futexp, val, mutex, private) \
+ lll_futex_timed_wait_requeue_pi (futexp, val, NULL, 0, mutex, private)
+
+#define lll_futex_timed_wait_requeue_pi(futexp, val, timespec, clockbit, \
+ mutex, private) \
+ ({ \
+ INTERNAL_SYSCALL_DECL (__err); \
+ long int __ret; \
+ int __op = FUTEX_WAIT_REQUEUE_PI | clockbit; \
+ \
+ __ret = INTERNAL_SYSCALL (futex, __err, 5, (futexp), \
+ __lll_private_flag (__op, private), \
+ (val), (timespec), mutex); \
+ INTERNAL_SYSCALL_ERROR_P (__ret, __err) ? -__ret : __ret; \
+ })
+
+#define lll_futex_cmp_requeue_pi(futexp, nr_wake, nr_move, mutex, val, priv) \
+ ({ \
+ INTERNAL_SYSCALL_DECL (__err); \
+ long int __ret; \
+ \
+ __ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp), \
+ __lll_private_flag (FUTEX_CMP_REQUEUE_PI, priv),\
+ (nr_wake), (nr_move), (mutex), (val)); \
+ INTERNAL_SYSCALL_ERROR_P (__ret, __err); \
+ })
+
#ifdef UP
# define __lll_acq_instr ""
diff --git a/nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h b/nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h
index a0163d6..3dab05e 100644
--- a/nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h
+++ b/nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h
@@ -37,6 +37,8 @@
#define FUTEX_TRYLOCK_PI 8
#define FUTEX_WAIT_BITSET 9
#define FUTEX_WAKE_BITSET 10
+#define FUTEX_WAIT_REQUEUE_PI 11
+#define FUTEX_CMP_REQUEUE_PI 12
#define FUTEX_PRIVATE_FLAG 128
#define FUTEX_CLOCK_REALTIME 256
@@ -141,6 +143,32 @@
INTERNAL_SYSCALL_ERROR_P (__ret, __err); \
})
+/* Priority Inheritance support. */
+#define lll_futex_wait_requeue_pi(futexp, val, mutex, private) \
+ lll_futex_timed_wait_requeue_pi (futexp, val, NULL, 0, mutex, private)
+
+#define lll_futex_timed_wait_requeue_pi(futexp, val, timespec, clockbit, \
+ mutex, private) \
+ ({ \
+ INTERNAL_SYSCALL_DECL (__err); \
+ int __op = FUTEX_WAIT_REQUEUE_PI | clockbit; \
+ \
+ INTERNAL_SYSCALL (futex, __err, 5, (futexp), \
+ __lll_private_flag (__op, private), \
+ (val), (timespec), mutex); \
+ })
+
+#define lll_futex_cmp_requeue_pi(futexp, nr_wake, nr_move, mutex, val, priv) \
+ ({ \
+ INTERNAL_SYSCALL_DECL (__err); \
+ long int __ret; \
+ \
+ __ret = INTERNAL_SYSCALL (futex, __err, 6, (futexp), \
+ __lll_private_flag (FUTEX_CMP_REQUEUE_PI, priv),\
+ (nr_wake), (nr_move), (mutex), (val)); \
+ INTERNAL_SYSCALL_ERROR_P (__ret, __err); \
+ })
+
#define lll_compare_and_swap(futex, oldval, newval, operation) \
do { \
__typeof (futex) __futex = (futex); \
diff --git a/sysdeps/unix/sysv/linux/kernel-features.h b/sysdeps/unix/sysv/linux/kernel-features.h
index 21eef43..8fdff7e 100644
--- a/sysdeps/unix/sysv/linux/kernel-features.h
+++ b/sysdeps/unix/sysv/linux/kernel-features.h
@@ -187,6 +187,11 @@
# define __ASSUME_PWRITEV 1
#endif
+/* Support for FUTEX_*_REQUEUE_PI was added in 2.6.31. */
+#if __LINUX_KERNEL_VERSION >= 0x02061f
+# define __ASSUME_REQUEUE_PI 1
+#endif
+
/* Support for F_GETOWN_EX was introduced in 2.6.32. */
#if __LINUX_KERNEL_VERSION >= 0x020620
# define __ASSUME_F_GETOWN_EX 1
-----------------------------------------------------------------------
Summary of changes:
NEWS | 6 +-
nptl/ChangeLog | 30 ++++++++++
nptl/pthreadP.h | 12 ++++
nptl/pthread_cond_broadcast.c | 45 ++++++++-------
nptl/pthread_cond_signal.c | 33 +++++++++--
nptl/pthread_cond_timedwait.c | 59 +++++++++++++++++---
nptl/pthread_cond_wait.c | 57 ++++++++++++++++---
.../sysdeps/unix/sysv/linux/powerpc/lowlevellock.h | 30 ++++++++++
nptl/sysdeps/unix/sysv/linux/s390/lowlevellock.h | 28 +++++++++
sysdeps/unix/sysv/linux/kernel-features.h | 5 ++
10 files changed, 260 insertions(+), 45 deletions(-)
hooks/post-receive
--
GNU C Library master sources