This is the mail archive of the
libc-alpha@sourceware.org
mailing list for the glibc project.
[PATCH 2/2] Requeue-PI support for x86 arch
- From: Dinakar Guniguntala <dino at in dot ibm dot com>
- To: libc-alpha at sources dot redhat dot com
- Cc: Ulrich Drepper <drepper at redhat dot com>, Darren Hart <dvhltc at us dot ibm dot com>, tglx at linutronix dot de
- Date: Tue, 17 Nov 2009 19:08:05 +0530
- Subject: [PATCH 2/2] Requeue-PI support for x86 arch
- References: <20091117133341.GC6042@in.ibm.com>
- Reply-to: dino at in dot ibm dot com
The following patch adds requeue PI support for the x86 arch. This closely
resembles the x86_64 requeue PI support in the current git tree. I have
tested this with 2.6.32-rc6 which should have all of the futex fixes from
Darren Hart and Thomas Gleixner.
The patch is against latest glibc git (2009-Nov-17)
-Dinakar
2009-11-17 Dinakar Guniguntala <dino@in.ibm.com>
* sysdeps/unix/sysv/linux/i386/i486/lowlevellock.h: Define
FUTEX_WAIT_REQUEUE_PI and FUTEX_CMP_REQUEUE_PI.
* sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S: If mutex
is a PI mutex, then use FUTEX_CMP_REQUEUE_PI.
* sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S: Likewise.
* sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S: If mutex
is a PI mutex, then use FUTEX_WAIT_REQUEUE_PI.
* sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S: Likewise.
diff -X ignore -Nurp glibc-cfi.mod/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S glibc-requeue-pi.mod/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S
--- glibc-cfi.mod/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S 2009-11-10 06:04:23.000000000 -0500
+++ glibc-requeue-pi.mod/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_broadcast.S 2009-11-13 10:00:26.000000000 -0500
@@ -1,4 +1,4 @@
-/* Copyright (C) 2002, 2003, 2004, 2006, 2007 Free Software Foundation, Inc.
+/* Copyright (C) 2002-2004, 2006-2007, 2009 Free Software Foundation, Inc.
This file is part of the GNU C Library.
Contributed by Ulrich Drepper <drepper@redhat.com>, 2002.
@@ -90,12 +90,14 @@ __pthread_cond_broadcast:
8: cmpl $-1, %edi
je 9f
- /* XXX: The kernel so far doesn't support requeue to PI futex. */
- /* XXX: The kernel only supports FUTEX_CMP_REQUEUE to the same
- type of futex (private resp. shared). */
- testl $(PI_BIT | PS_BIT), MUTEX_KIND(%edi)
+ /* Do not use requeue for pshared condvars. */
+ testl $PS_BIT, MUTEX_KIND(%edi)
jne 9f
+ /* Requeue to a PI mutex if the PI bit is set. */
+ testl $PI_BIT, MUTEX_KIND(%edi)
+ jne 81f
+
/* Wake up all threads. */
#ifdef __ASSUME_PRIVATE_FUTEX
movl $(FUTEX_CMP_REQUEUE|FUTEX_PRIVATE_FLAG), %ecx
@@ -140,6 +142,24 @@ __pthread_cond_broadcast:
cfi_offset(%esi, -12)
cfi_offset(%edi, -16)
cfi_offset(%ebp, -20)
+
+81: movl $(FUTEX_CMP_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
+ movl $SYS_futex, %eax
+ movl $0x7fffffff, %esi
+ movl $1, %edx
+ /* Get the address of the futex involved. */
+# if MUTEX_FUTEX != 0
+ addl $MUTEX_FUTEX, %edi
+# endif
+ int $0x80
+
+ /* For any kind of error, which mainly is EAGAIN, we try again
+ with WAKE. The general test also covers running on old
+ kernels. */
+ cmpl $0xfffff001, %eax
+ jb 10b
+ jmp 9f
+
.align 16
/* Unlock. */
4: LOCK
diff -X ignore -Nurp glibc-cfi.mod/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S glibc-requeue-pi.mod/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S
--- glibc-cfi.mod/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S 2009-11-10 06:04:23.000000000 -0500
+++ glibc-requeue-pi.mod/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_signal.S 2009-11-13 10:00:26.000000000 -0500
@@ -22,6 +22,7 @@
#include <lowlevellock.h>
#include <lowlevelcond.h>
#include <kernel-features.h>
+#include <pthread-pi-defines.h>
#include <pthread-errnos.h>
@@ -85,7 +86,13 @@ __pthread_cond_signal:
#endif
cmpl $-1, dep_mutex-cond_futex(%ebx)
sete %cl
- subl $1, %ecx
+ je 8f
+
+ movl dep_mutex-cond_futex(%ebx), %edx
+ testl $PI_BIT, MUTEX_KIND(%edx)
+ jne 9f
+
+8: subl $1, %ecx
#ifdef __ASSUME_PRIVATE_FUTEX
andl $FUTEX_PRIVATE_FLAG, %ecx
#else
@@ -125,8 +132,34 @@ __pthread_cond_signal:
cfi_offset(%ebx, -8)
cfi_offset(%edi, -12)
-7: /* %ecx should be either FUTEX_WAKE_OP or
- FUTEX_WAKE_OP|FUTEX_PRIVATE_FLAG from the previous syscall. */
+9: movl $(FUTEX_CMP_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
+ movl $SYS_futex, %eax
+ movl $1, %edx
+ xorl %esi, %esi
+ movl dep_mutex-cond_futex(%ebx), %edi
+ movl (%ebx), %ebp
+ /* FIXME: Until Ingo fixes 4G/4G vDSO, 6 arg syscalls are broken for
+ sysenter.
+ ENTER_KERNEL */
+ int $0x80
+ popl %ebp
+ popl %esi
+
+ leal -cond_futex(%ebx), %edi
+
+ /* For any kind of error, we try again with WAKE.
+ The general test also covers running on old kernels. */
+ cmpl $-4095, %eax
+ jb 4f
+
+7:
+#ifdef __ASSUME_PRIVATE_FUTEX
+ andl $FUTEX_PRIVATE_FLAG, %ecx
+#else
+ andl %gs:PRIVATE_FUTEX, %ecx
+#endif
+ orl $FUTEX_WAKE, %ecx
+
xorl $(FUTEX_WAKE ^ FUTEX_WAKE_OP), %ecx
movl $SYS_futex, %eax
/* %edx should be 1 already from $FUTEX_WAKE_OP syscall.
diff -X ignore -Nurp glibc-cfi.mod/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S glibc-requeue-pi.mod/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S
--- glibc-cfi.mod/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S 2009-11-10 10:35:18.000000000 -0500
+++ glibc-requeue-pi.mod/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_timedwait.S 2009-11-13 10:00:27.000000000 -0500
@@ -22,6 +22,7 @@
#include <lowlevellock.h>
#include <lowlevelcond.h>
#include <pthread-errnos.h>
+#include <pthread-pi-defines.h>
#include <kernel-features.h>
@@ -94,7 +95,7 @@ __pthread_cond_timedwait:
addl $1, cond_futex(%ebx)
addl $(1 << nwaiters_shift), cond_nwaiters(%ebx)
-#define FRAME_SIZE 24
+#define FRAME_SIZE 32
subl $FRAME_SIZE, %esp
cfi_adjust_cfa_offset(FRAME_SIZE)
@@ -106,8 +107,10 @@ __pthread_cond_timedwait:
movl %edx, 16(%esp)
movl %eax, 20(%esp)
+ /* Reset the pi-requeued flag. */
+8: movl $0, 24(%esp)
/* Get the current time. */
-8: movl %ebx, %edx
+ movl %ebx, %edx
#ifdef __NR_clock_gettime
/* Get the clock number. */
movl cond_nwaiters(%ebx), %ebx
@@ -157,6 +160,7 @@ __pthread_cond_timedwait:
movl %edx, 8(%esp)
movl cond_futex(%ebx), %edi
+ movl %edi, 28(%esp)
/* Unlock. */
LOCK
@@ -171,13 +175,47 @@ __pthread_cond_timedwait:
4: call __pthread_enable_asynccancel
movl %eax, (%esp)
- leal 4(%esp), %esi
#if FUTEX_PRIVATE_FLAG > 255
xorl %ecx, %ecx
#endif
cmpl $-1, dep_mutex(%ebx)
sete %cl
- subl $1, %ecx
+ je 18f
+
+ movl dep_mutex(%ebx), %edi
+ /* Requeue to a PI mutex if the PI bit is set. */
+ testl $PI_BIT, MUTEX_KIND(%edi)
+ je 18f
+
+ movl $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
+ /* The following only works like this because we only support
+ two clocks, represented using a single bit. */
+ testl $1, cond_nwaiters(%ebx)
+ /* XXX Need to implement using sete instead of a jump. */
+ jne 44f
+ orl $FUTEX_CLOCK_REALTIME, %ecx
+
+ /* Requeue-PI uses absolute timeout */
+44: leal (%ebp), %esi
+ movl 28(%esp), %edx
+ addl $cond_futex, %ebx
+ movl $SYS_futex, %eax
+ ENTER_KERNEL
+ subl $cond_futex, %ebx
+ movl %eax, %esi
+ /* Set the pi-requeued flag only if the kernel has returned 0. The
+ kernel does not hold the mutex on ETIMEDOUT or any other error. */
+ cmpl $0, %eax
+ sete 24(%esp)
+ je 19f
+
+ /* Normal and PI futexes dont mix. Use normal futex functions only
+ if the kernel does not support the PI futex functions. */
+ cmpl $-ENOSYS, %eax
+ jne 19f
+ xorl %ecx, %ecx
+
+18: subl $1, %ecx
#ifdef __ASSUME_PRIVATE_FUTEX
andl $FUTEX_PRIVATE_FLAG, %ecx
#else
@@ -186,7 +224,8 @@ __pthread_cond_timedwait:
#if FUTEX_WAIT != 0
addl $FUTEX_WAIT, %ecx
#endif
- movl %edi, %edx
+ leal 4(%esp), %esi
+ movl 28(%esp), %edx
addl $cond_futex, %ebx
.Ladd_cond_futex:
movl $SYS_futex, %eax
@@ -195,7 +234,7 @@ __pthread_cond_timedwait:
.Lsub_cond_futex:
movl %eax, %esi
- movl (%esp), %eax
+19: movl (%esp), %eax
call __pthread_disable_asynccancel
.LcleanupEND:
@@ -283,10 +322,16 @@ __pthread_cond_timedwait:
#endif
jne 10f
+11: xorl %eax, %eax
+ /* With requeue_pi, the mutex lock is held in the kernel. */
+ movl 24(%esp), %ecx
+ testl %ecx, %ecx
+ jnz 26f
+
/* Remove cancellation handler. */
-11: movl 24+FRAME_SIZE(%esp), %eax
+ movl 24+FRAME_SIZE(%esp), %eax
call __pthread_mutex_cond_lock
- addl $FRAME_SIZE, %esp
+26: addl $FRAME_SIZE, %esp
cfi_adjust_cfa_offset(-FRAME_SIZE);
/* We return the result of the mutex_lock operation if it failed. */
diff -X ignore -Nurp glibc-cfi.mod/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S glibc-requeue-pi.mod/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S
--- glibc-cfi.mod/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S 2009-11-12 02:09:47.000000000 -0500
+++ glibc-requeue-pi.mod/nptl/sysdeps/unix/sysv/linux/i386/i486/pthread_cond_wait.S 2009-11-13 10:00:28.000000000 -0500
@@ -22,6 +22,8 @@
#include <lowlevellock.h>
#include <lowlevelcond.h>
#include <tcb-offsets.h>
+#include <pthread-errnos.h>
+#include <pthread-pi-defines.h>
#include <kernel-features.h>
@@ -43,18 +45,21 @@ __pthread_cond_wait:
cfi_lsda(DW_EH_PE_udata4, .LexceptSTART)
#endif
+ pushl %ebp
+ cfi_adjust_cfa_offset(4)
pushl %edi
cfi_adjust_cfa_offset(4)
pushl %esi
cfi_adjust_cfa_offset(4)
pushl %ebx
cfi_adjust_cfa_offset(4)
- cfi_offset(%edi, -8)
- cfi_offset(%esi, -12)
- cfi_offset(%ebx, -16)
+ cfi_offset(%ebp, -8)
+ cfi_offset(%edi, -12)
+ cfi_offset(%esi, -16)
+ cfi_offset(%ebx, -20)
xorl %esi, %esi
- movl 16(%esp), %ebx
+ movl 20(%esp), %ebx
/* Get internal lock. */
movl $1, %edx
@@ -70,7 +75,7 @@ __pthread_cond_wait:
/* Store the reference to the mutex. If there is already a
different value in there this is a bad user bug. */
2: cmpl $-1, dep_mutex(%ebx)
- movl 20(%esp), %eax
+ movl 24(%esp), %eax
je 15f
movl %eax, dep_mutex(%ebx)
@@ -86,7 +91,7 @@ __pthread_cond_wait:
addl $1, cond_futex(%ebx)
addl $(1 << nwaiters_shift), cond_nwaiters(%ebx)
-#define FRAME_SIZE 16
+#define FRAME_SIZE 20
subl $FRAME_SIZE, %esp
cfi_adjust_cfa_offset(FRAME_SIZE)
@@ -98,8 +103,10 @@ __pthread_cond_wait:
movl %edx, 8(%esp)
movl %eax, 12(%esp)
-8: movl cond_futex(%ebx), %edi
-
+ /* Reset the pi-requeued flag. */
+8: movl $0, 16(%esp)
+ movl cond_futex(%ebx), %ebp
+
/* Unlock. */
LOCK
#if cond_lock == 0
@@ -113,12 +120,36 @@ __pthread_cond_wait:
4: call __pthread_enable_asynccancel
movl %eax, (%esp)
-#if FUTEX_PRIVATE_FLAG > 255
xorl %ecx, %ecx
-#endif
cmpl $-1, dep_mutex(%ebx)
sete %cl
- subl $1, %ecx
+ je 18f
+
+ movl dep_mutex(%ebx), %edi
+ /* Requeue to a PI mutex if the PI bit is set. */
+ testl $PI_BIT, MUTEX_KIND(%edi)
+ je 18f
+
+ movl $(FUTEX_WAIT_REQUEUE_PI|FUTEX_PRIVATE_FLAG), %ecx
+ movl %ebp, %edx
+ xorl %esi, %esi
+ addl $cond_futex, %ebx
+ movl $SYS_futex, %eax
+ ENTER_KERNEL
+ subl $cond_futex, %ebx
+ /* Set the pi-requeued flag only if the kernel has returned 0. The
+ kernel does not hold the mutex on error. */
+ cmpl $0, %eax
+ sete 16(%esp)
+ je 19f
+
+ /* Normal and PI futexes dont mix. Use normal futex functions only
+ if the kernel does not support the PI futex functions. */
+ cmpl $-ENOSYS, %eax
+ jne 19f
+ xorl %ecx, %ecx
+
+18: subl $1, %ecx
#ifdef __ASSUME_PRIVATE_FUTEX
andl $FUTEX_PRIVATE_FLAG, %ecx
#else
@@ -127,7 +158,7 @@ __pthread_cond_wait:
#if FUTEX_WAIT != 0
addl $FUTEX_WAIT, %ecx
#endif
- movl %edi, %edx
+ movl %ebp, %edx
addl $cond_futex, %ebx
.Ladd_cond_futex:
movl $SYS_futex, %eax
@@ -135,7 +166,7 @@ __pthread_cond_wait:
subl $cond_futex, %ebx
.Lsub_cond_futex:
- movl (%esp), %eax
+19: movl (%esp), %eax
call __pthread_disable_asynccancel
.LcleanupEND:
@@ -211,9 +242,15 @@ __pthread_cond_wait:
#endif
jne 10f
-11: movl 20+FRAME_SIZE(%esp), %eax
+ /* With requeue_pi, the mutex lock is held in the kernel. */
+11: xorl %eax, %eax
+ movl 16(%esp), %ecx
+ testl %ecx, %ecx
+ jnz 20f
+
+ movl 24+FRAME_SIZE(%esp), %eax
call __pthread_mutex_cond_lock
- addl $FRAME_SIZE, %esp
+20: addl $FRAME_SIZE, %esp
cfi_adjust_cfa_offset(-FRAME_SIZE);
14: popl %ebx
@@ -225,14 +262,18 @@ __pthread_cond_wait:
popl %edi
cfi_adjust_cfa_offset(-4)
cfi_restore(%edi)
+ popl %ebp
+ cfi_adjust_cfa_offset(-4)
+ cfi_restore(%ebp)
/* We return the result of the mutex_lock operation. */
ret
- cfi_adjust_cfa_offset(3 * 4 + FRAME_SIZE)
- cfi_offset(%edi, -8)
- cfi_offset(%esi, -12)
- cfi_offset(%ebx, -16)
+ cfi_adjust_cfa_offset(4 * 4 + FRAME_SIZE)
+ cfi_offset(%ebp, -8)
+ cfi_offset(%edi, -12)
+ cfi_offset(%esi, -16)
+ cfi_offset(%ebx, -20)
/* Initial locking failed. */
1:
diff -X ignore -Nurp glibc-cfi.mod/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h glibc-requeue-pi.mod/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h
--- glibc-cfi.mod/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h 2009-11-10 06:04:23.000000000 -0500
+++ glibc-requeue-pi.mod/nptl/sysdeps/unix/sysv/linux/i386/lowlevellock.h 2009-11-12 01:47:48.000000000 -0500
@@ -54,6 +54,8 @@
#define FUTEX_TRYLOCK_PI 8
#define FUTEX_WAIT_BITSET 9
#define FUTEX_WAKE_BITSET 10
+#define FUTEX_WAIT_REQUEUE_PI 11
+#define FUTEX_CMP_REQUEUE_PI 12
#define FUTEX_PRIVATE_FLAG 128
#define FUTEX_CLOCK_REALTIME 256