From: Florian Weimer via Libc-alpha <libc-alpha@sourceware.org>
To: libc-alpha@sourceware.org
Subject: [PATCH v4 32/37] nptl: pthread_mutex_lock, pthread_mutex_unock single-threaded optimization
Date: Fri, 16 Apr 2021 11:24:00 +0200 [thread overview]
Message-ID: <85feebd8764b7b1ad0332ee0cc422765d38fe6c8.1618564630.git.fweimer@redhat.com> (raw)
In-Reply-To: <cover.1618564630.git.fweimer@redhat.com>
This is optimization is similar in spirit to the SINGLE_THREAD_P check
in the malloc implementation. Doing this in generic code allows us
to prioritize those cases which are likely to occur in single-threaded
programs (normal and recursive mutexes).
Reviewed-by: Adhemerval Zanella <adhemerval.zanella@linaro.org>
---
nptl/pthread_mutex_cond_lock.c | 1 +
nptl/pthread_mutex_lock.c | 25 ++++++++++++++++++++++---
nptl/pthread_mutex_unlock.c | 17 ++++++++++++++++-
3 files changed, 39 insertions(+), 4 deletions(-)
diff --git a/nptl/pthread_mutex_cond_lock.c b/nptl/pthread_mutex_cond_lock.c
index 2f0771302f..3386bd689b 100644
--- a/nptl/pthread_mutex_cond_lock.c
+++ b/nptl/pthread_mutex_cond_lock.c
@@ -2,6 +2,7 @@
#define LLL_MUTEX_LOCK(mutex) \
lll_cond_lock ((mutex)->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex))
+#define LLL_MUTEX_LOCK_OPTIMIZED(mutex) LLL_MUTEX_LOCK (mutex)
/* Not actually elided so far. Needed? */
#define LLL_MUTEX_LOCK_ELISION(mutex) \
diff --git a/nptl/pthread_mutex_lock.c b/nptl/pthread_mutex_lock.c
index f0de7b7fd6..8649a92ffb 100644
--- a/nptl/pthread_mutex_lock.c
+++ b/nptl/pthread_mutex_lock.c
@@ -30,8 +30,27 @@
/* Some of the following definitions differ when pthread_mutex_cond_lock.c
includes this file. */
#ifndef LLL_MUTEX_LOCK
-# define LLL_MUTEX_LOCK(mutex) \
+/* lll_lock with single-thread optimization. */
+static inline void
+lll_mutex_lock_optimized (pthread_mutex_t *mutex)
+{
+ /* The single-threaded optimization is only valid for private
+ mutexes. For process-shared mutexes, the mutex could be in a
+ shared mapping, so synchronization with another process is needed
+ even without any threads. If the lock is already marked as
+ acquired, POSIX requires that pthread_mutex_lock deadlocks for
+ normal mutexes, so skip the optimization in that case as
+ well. */
+ int private = PTHREAD_MUTEX_PSHARED (mutex);
+ if (private == LLL_PRIVATE && SINGLE_THREAD_P && mutex->__data.__lock == 0)
+ mutex->__data.__lock = 1;
+ else
+ lll_lock (mutex->__data.__lock, private);
+}
+
+# define LLL_MUTEX_LOCK(mutex) \
lll_lock ((mutex)->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex))
+# define LLL_MUTEX_LOCK_OPTIMIZED(mutex) lll_mutex_lock_optimized (mutex)
# define LLL_MUTEX_TRYLOCK(mutex) \
lll_trylock ((mutex)->__data.__lock)
# define LLL_ROBUST_MUTEX_LOCK_MODIFIER 0
@@ -64,7 +83,7 @@ __pthread_mutex_lock (pthread_mutex_t *mutex)
FORCE_ELISION (mutex, goto elision);
simple:
/* Normal mutex. */
- LLL_MUTEX_LOCK (mutex);
+ LLL_MUTEX_LOCK_OPTIMIZED (mutex);
assert (mutex->__data.__owner == 0);
}
#if ENABLE_ELISION_SUPPORT
@@ -99,7 +118,7 @@ __pthread_mutex_lock (pthread_mutex_t *mutex)
}
/* We have to get the mutex. */
- LLL_MUTEX_LOCK (mutex);
+ LLL_MUTEX_LOCK_OPTIMIZED (mutex);
assert (mutex->__data.__owner == 0);
mutex->__data.__count = 1;
diff --git a/nptl/pthread_mutex_unlock.c b/nptl/pthread_mutex_unlock.c
index 3b5ccdacf9..655093ee9a 100644
--- a/nptl/pthread_mutex_unlock.c
+++ b/nptl/pthread_mutex_unlock.c
@@ -28,6 +28,21 @@ static int
__pthread_mutex_unlock_full (pthread_mutex_t *mutex, int decr)
__attribute_noinline__;
+/* lll_lock with single-thread optimization. */
+static inline void
+lll_mutex_unlock_optimized (pthread_mutex_t *mutex)
+{
+ /* The single-threaded optimization is only valid for private
+ mutexes. For process-shared mutexes, the mutex could be in a
+ shared mapping, so synchronization with another process is needed
+ even without any threads. */
+ int private = PTHREAD_MUTEX_PSHARED (mutex);
+ if (private == LLL_PRIVATE && SINGLE_THREAD_P)
+ mutex->__data.__lock = 0;
+ else
+ lll_unlock (mutex->__data.__lock, private);
+}
+
int
attribute_hidden
__pthread_mutex_unlock_usercnt (pthread_mutex_t *mutex, int decr)
@@ -51,7 +66,7 @@ __pthread_mutex_unlock_usercnt (pthread_mutex_t *mutex, int decr)
--mutex->__data.__nusers;
/* Unlock. */
- lll_unlock (mutex->__data.__lock, PTHREAD_MUTEX_PSHARED (mutex));
+ lll_mutex_unlock_optimized (mutex);
LIBC_PROBE (mutex_release, 1, mutex);
--
2.30.2
next prev parent reply other threads:[~2021-04-16 9:23 UTC|newest]
Thread overview: 43+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-04-16 9:20 [PATCH v4 00/37] libpthread: Remove NPTL forwarders Florian Weimer via Libc-alpha
2021-04-16 9:20 ` [PATCH v4 01/37] nptl: Move pthread_mutex_consistent into libc Florian Weimer via Libc-alpha
2021-04-16 9:20 ` [PATCH v4 02/37] nptl: Move __pthread_cleanup_routine " Florian Weimer via Libc-alpha
2021-04-16 9:20 ` [PATCH v4 03/37] nptl: Move legacy unwinding implementation " Florian Weimer via Libc-alpha
2021-04-16 9:20 ` [PATCH v4 04/37] nptl: Move legacy cancelation handling into libc as compat symbols Florian Weimer via Libc-alpha
2021-04-16 9:20 ` [PATCH v4 05/37] nptl: Remove longjmp, siglongjmp from libpthread Florian Weimer via Libc-alpha
2021-04-16 9:20 ` [PATCH v4 06/37] x86: Restore compile-time check for shadow stack pointer in longjmp Florian Weimer via Libc-alpha
2021-04-16 9:20 ` [PATCH v4 07/37] nptl: Move __pthread_cleanup_upto into libc Florian Weimer via Libc-alpha
2021-04-16 9:20 ` [PATCH v4 08/37] nptl: Move pthread_once and __pthread_once " Florian Weimer via Libc-alpha
2021-04-16 9:21 ` [PATCH v4 09/37] nptl: Move __pthread_unwind_next " Florian Weimer via Libc-alpha
2021-04-16 9:21 ` [PATCH v4 10/37] csu: Move calling main out of __libc_start_main_impl Florian Weimer via Libc-alpha
2021-04-16 9:21 ` [PATCH v4 11/37] nptl: Move internal __nptl_nthreads variable into libc Florian Weimer via Libc-alpha
2021-04-16 9:21 ` [PATCH v4 12/37] nptl_db: Introduce DB_MAIN_ARRAY_VARIABLE Florian Weimer via Libc-alpha
2021-04-16 9:21 ` [PATCH v4 13/37] nptl: Move __pthread_keys global variable into libc Florian Weimer via Libc-alpha
2021-04-16 9:21 ` [PATCH v4 14/37] nptl: Move __nptl_deallocate_tsd " Florian Weimer via Libc-alpha
2021-04-16 9:21 ` [PATCH v4 15/37] nptl: Move pthread_exit " Florian Weimer via Libc-alpha
2021-04-16 9:22 ` [PATCH v4 16/37] nptl: Move pthread_setcancelstate " Florian Weimer via Libc-alpha
2021-04-16 9:22 ` [PATCH v4 17/37] nptl: Move pthread_setcanceltype " Florian Weimer via Libc-alpha
2021-04-16 9:22 ` [PATCH v4 18/37] nptl: Invoke the set_robust_list system call directly in fork Florian Weimer via Libc-alpha
2021-04-16 9:22 ` [PATCH v4 19/37] dlfcn: Failures after dlmopen should not terminate process [BZ #24772] Florian Weimer via Libc-alpha
2021-04-16 9:22 ` [PATCH v4 20/37] dlfcn: dlerror needs to call free from the base namespace [BZ #24773] Florian Weimer via Libc-alpha
2021-04-16 9:22 ` [PATCH v4 21/37] Remove pthread_key_create-related internals from libc-lock.h Florian Weimer via Libc-alpha
2021-04-16 9:22 ` [PATCH v4 22/37] elf: Introduce __tls_init_tp for second-phase TCB initialization Florian Weimer via Libc-alpha
2021-04-16 9:22 ` [PATCH v4 23/37] nptl: Move part of TCB initialization from libpthread to __tls_init_tp Florian Weimer via Libc-alpha
2021-04-16 9:22 ` [PATCH v4 24/37] nptl: Move pthread_key_create, __pthread_key_create into libc Florian Weimer via Libc-alpha
2021-04-16 9:23 ` [PATCH v4 25/37] nptl: Move pthread_getspecific, __pthread_getspecific " Florian Weimer via Libc-alpha
2021-04-16 9:23 ` [PATCH v4 26/37] nptl: Move pthread_setspecific, __pthread_setspecific " Florian Weimer via Libc-alpha
2021-04-16 9:23 ` [PATCH v4 27/37] nptl: Move pthread_key_delete " Florian Weimer via Libc-alpha
2021-04-16 9:23 ` [PATCH v4 28/37] nptl: Move rwlock functions with forwarders " Florian Weimer via Libc-alpha
2021-04-16 9:23 ` [PATCH v4 29/37] nptl: Move the internal thread priority protection symbols " Florian Weimer via Libc-alpha
2021-04-16 9:23 ` [PATCH v4 30/37] pthread: Introduce __pthread_early_init Florian Weimer via Libc-alpha
2021-04-16 9:23 ` [PATCH v4 31/37] nptl: Move internal symbol __mutex_aconf into libc Florian Weimer via Libc-alpha
2021-04-16 9:24 ` Florian Weimer via Libc-alpha [this message]
2021-04-16 9:24 ` [PATCH v4 33/37] x86: Remove low-level lock optimization Florian Weimer via Libc-alpha
2021-04-16 9:24 ` [PATCH v4 34/37] nptl: Move core mutex functions into libc Florian Weimer via Libc-alpha
2021-04-16 9:24 ` [PATCH v4 35/37] nptl: Move core condition variable " Florian Weimer via Libc-alpha
2021-04-16 9:24 ` [PATCH v4 36/37] nptl: Move setxid broadcast implementation " Florian Weimer via Libc-alpha
2021-04-16 9:24 ` [PATCH v4 37/37] nptl: Remove remnants of the libc/libpthread forwarder interface Florian Weimer via Libc-alpha
2021-04-21 11:47 ` [PATCH v4 00/37] libpthread: Remove NPTL forwarders Florian Weimer via Libc-alpha
2021-04-21 13:42 ` H.J. Lu via Libc-alpha
2021-04-22 7:35 ` Szabolcs Nagy via Libc-alpha
2021-04-22 7:58 ` Florian Weimer via Libc-alpha
2021-04-22 8:56 ` Szabolcs Nagy via Libc-alpha
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
List information: https://www.gnu.org/software/libc/involved.html
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=85feebd8764b7b1ad0332ee0cc422765d38fe6c8.1618564630.git.fweimer@redhat.com \
--to=libc-alpha@sourceware.org \
--cc=fweimer@redhat.com \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
Be sure your reply has a Subject: header at the top and a blank line
before the message body.
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for read-only IMAP folder(s) and NNTP newsgroup(s).