From 4750cf4202c29a895639b89099a7bdbe9ae422b6 Mon Sep 17 00:00:00 2001
From: Rich Felker <dalias@aerifal.cx>
Date: Tue, 24 Apr 2012 16:32:23 -0400
Subject: ditch the priority inheritance locks; use malloc's version of lock

i did some testing trying to switch malloc to use the new internal
lock with priority inheritance, and my malloc contention test got
20-100 times slower. if priority inheritance futexes are this slow,
it's simply too high a price to pay for avoiding priority inversion.
maybe we can consider them somewhere down the road once the kernel
folks get their act together on this (and perferably don't link it to
glibc's inefficient lock API)...

as such, i've switch __lock to use malloc's implementation of
lightweight locks, and updated all the users of the code to use an
array with a waiter count for their locks. this should give optimal
performance in the vast majority of cases, and it's simple.

malloc is still using its own internal copy of the lock code because
it seems to yield measurably better performance with -O3 when it's
inlined (20% or more difference in the contention stress test).
---
 src/thread/__lock.c         | 26 +++-----------------------
 src/thread/pthread_atfork.c | 10 +++++-----
 2 files changed, 8 insertions(+), 28 deletions(-)

(limited to 'src/thread')

diff --git a/src/thread/__lock.c b/src/thread/__lock.c
index 5ba5dc5e..2f345ae7 100644
--- a/src/thread/__lock.c
+++ b/src/thread/__lock.c
@@ -1,32 +1,12 @@
 #include "pthread_impl.h"
 
-void __lock_2(volatile int *l)
-{
-	if (!__syscall(SYS_futex, l, FUTEX_LOCK_PI, 0, 0))
-		return;
-	int old, tid = __pthread_self()->tid|INT_MIN;
-	while ((old = a_cas(l, 0, tid))) {
-		a_cas(l, old, old|INT_MIN);
-		__syscall(SYS_futex, l, FUTEX_WAIT, old|INT_MIN, 0);
-	}
-}
-
 void __lock(volatile int *l)
 {
-	if (a_cas(l, 0, __pthread_self()->tid)) __lock_2(l);
-}
-
-void __unlock_2(volatile int *l)
-{
-	if (__syscall(SYS_futex, l, FUTEX_UNLOCK_PI)) {
-		*l = 0;
-		__syscall(SYS_futex, l, FUTEX_WAKE, 1);
-	}
+	while (a_swap(l, 1)) __wait(l, l+1, 1, 1);
 }
 
 void __unlock(volatile int *l)
 {
-	int old = *l;
-	if (!(old & INT_MIN) && a_cas(l, old, 0)==old) return;
-	__unlock_2(l);
+	a_store(l, 0);
+	if (l[1]) __wake(l, 1, 1);
 }
diff --git a/src/thread/pthread_atfork.c b/src/thread/pthread_atfork.c
index a7a82016..95fce207 100644
--- a/src/thread/pthread_atfork.c
+++ b/src/thread/pthread_atfork.c
@@ -8,14 +8,14 @@ static struct atfork_funcs {
 	struct atfork_funcs *prev, *next;
 } *funcs;
 
-static int lock;
+static int lock[2];
 
 void __fork_handler(int who)
 {
 	struct atfork_funcs *p;
 	if (!funcs) return;
 	if (who < 0) {
-		LOCK(&lock);
+		LOCK(lock);
 		for (p=funcs; p; p = p->next) {
 			if (p->prepare) p->prepare();
 			funcs = p;
@@ -26,7 +26,7 @@ void __fork_handler(int who)
 			else if (who && p->child) p->child();
 			funcs = p;
 		}
-		UNLOCK(&lock);
+		UNLOCK(lock);
 	}
 }
 
@@ -35,7 +35,7 @@ int pthread_atfork(void (*prepare)(void), void (*parent)(void), void (*child)(vo
 	struct atfork_funcs *new = malloc(sizeof *new);
 	if (!new) return -1;
 
-	LOCK(&lock);
+	LOCK(lock);
 	new->next = funcs;
 	new->prev = 0;
 	new->prepare = prepare;
@@ -43,6 +43,6 @@ int pthread_atfork(void (*prepare)(void), void (*parent)(void), void (*child)(vo
 	new->child = child;
 	if (funcs) funcs->prev = new;
 	funcs = new;
-	UNLOCK(&lock);
+	UNLOCK(lock);
 	return 0;
 }
-- 
cgit v1.2.1