Merge pull request #119 from janciesko/lock_API_extension

Add support for recursive locks with external lock allocation

Author: janciesko

include/qt_atomics.h

@@ -144,13 +144,6 @@ extern pthread_mutexattr_t _fastlock_attr;
     && defined(QTHREAD_ATOMIC_INCR)                           \
     && !defined(QTHREAD_MUTEX_INCREMENT)
 
-typedef union qt_spin_trylock_s {
-    aligned_t u;
-    struct {
-        haligned_t ticket;
-        haligned_t users;
-    } s;
-} Q_ALIGNED(QTHREAD_ALIGNMENT_ALIGNED_T) qt_spin_trylock_t;
 
 # define QTHREAD_TRYLOCK_TYPE qt_spin_trylock_t
 # define QTHREAD_TRYLOCK_INIT(x)     { (x).u = 0; }

include/qthread/qthread.h

@@ -456,6 +456,28 @@ int       qthread_queue_destroy(qthread_queue_t q);
  * (full, no waiters) state at any one time.
  */
 
+#define QTHREAD_SPINLOCK_IS_RECURSIVE (-1)
+#define QTHREAD_SPINLOCK_IS_NOT_RECURSIVE (-2)
+
+typedef union qt_spin_trylock_s {
+    aligned_t u;
+    struct {
+        haligned_t ticket;
+        haligned_t users;
+    } s;
+} Q_ALIGNED(QTHREAD_ALIGNMENT_ALIGNED_T) qt_spin_trylock_t;
+
+typedef struct {
+    int64_t s;
+    int64_t count;
+} qthread_spinlock_state_t;
+
+typedef struct {
+    qt_spin_trylock_t lock;
+    qthread_spinlock_state_t state;
+} qthread_spinlock_t;
+
+
 /* This function is just to assist with debugging; it returns 1 if the address
  * is full, and 0 if the address is empty */
 int qthread_feb_status(const aligned_t *addr);
@@ -587,6 +609,21 @@ int qthread_lock(const aligned_t *a);
 int qthread_unlock(const aligned_t *a);
 int qthread_trylock(const aligned_t *a);
 
+int qthread_spinlock_init(qthread_spinlock_t *a, const bool is_recursive);
+int qthread_spinlock_destroy(qthread_spinlock_t *a);
+int qthread_spinlock_lock(qthread_spinlock_t *a);
+int qthread_spinlock_unlock(qthread_spinlock_t *a);
+int qthread_spinlock_trylock(qthread_spinlock_t *a);
+
+int qthread_spinlocks_init(qthread_spinlock_t *a, const bool is_recursive); 
+int qthread_spinlocks_destroy(qthread_spinlock_t *a);
+
+#define QTHREAD_SPINLOCK_IS_RECURSIVE (-1)
+#define QTHREAD_SPINLOCK_IS_NOT_RECURSIVE (-2)
+
+#define QTHREAD_MUTEX_INITIALIZER  {{.s={0,0}},{QTHREAD_SPINLOCK_IS_NOT_RECURSIVE,0}}
+#define QTHREAD_RECURSIVE_MUTEX_INITIALIZER {{.s={0,0}},{QTHREAD_SPINLOCK_IS_RECURSIVE,0}}
+
 /* functions to implement spinlock-based locking/unlocking 
  * if qthread_lock_init(adr) is called, subsequent locking over adr 
  * uses spin locking instead of FEBs. Support recursive locking.

src/locks.c

@@ -18,16 +18,7 @@
 
 #define SPINLOCK_IS_RECURSIVE (-1)
 #define SPINLOCK_IS_NOT_RECURSIVE (-2)
-                              
-typedef struct {
-    int64_t s;
-    int64_t count;
-} qthread_spinlock_state_t;
 
-typedef struct {
-    qt_spin_trylock_t lock;
-    qthread_spinlock_state_t state;
-} qthread_spinlock_t;
 
 #define QTHREAD_CHOOSE_STRIPE2(addr) (qt_hash64((uint64_t)(uintptr_t)addr) & (QTHREAD_LOCKING_STRIPES - 1))
 #define LOCKBIN(key) QTHREAD_CHOOSE_STRIPE2(key)
@@ -66,16 +57,31 @@ INTERNAL int lock_hashmap_remove(const aligned_t * key) {
     return QTHREAD_OPFAIL;
 }
 
-INTERNAL bool qthread_is_spin_lock(const aligned_t * a) {
+INTERNAL bool is_spin_lock_hashed(const aligned_t * a) {
     return (NULL != lock_hashmap_get(a));
 }
 
-INTERNAL int qthread_spinlock_initialize() {
+INTERNAL int spinlocks_initialize() {
     qthread_spinlock_buckets = NULL;
     return QTHREAD_SUCCESS;
 }
 
-INTERNAL int qthread_spinlock_init(const aligned_t * a, const bool is_recursive) {
+INTERNAL int spinlocks_finalize() {
+    if(qthread_spinlock_buckets){
+        for (unsigned i = 0; i < QTHREAD_LOCKING_STRIPES; i++) {
+            assert(qthread_spinlock_buckets[i]);
+            qt_hash_destroy_deallocate(qthread_spinlock_buckets[i],
+                            (qt_hash_deallocator_fn)
+                            qthread_spinlock_destroy_fn);
+        }
+        qt_free(qthread_spinlock_buckets);
+    }
+    return QTHREAD_SUCCESS;
+}
+
+/* locks over addresses using internal hashmaps */
+
+INTERNAL int spinlock_init_hashed(const aligned_t * a, const bool is_recursive) {
     uint_fast8_t need_sync = 1;
 
     if(!qthread_spinlock_buckets){
@@ -87,7 +93,7 @@ INTERNAL int qthread_spinlock_init(const aligned_t * a, const bool is_recursive)
         }
     }
 
-    if (!qthread_is_spin_lock(a)) {
+    if (!is_spin_lock_hashed(a)) {
         qthread_spinlock_t * l = qt_malloc (sizeof(qthread_spinlock_t));
         assert(l);
         l->state.s = is_recursive ? SPINLOCK_IS_RECURSIVE : SPINLOCK_IS_NOT_RECURSIVE;
@@ -99,25 +105,98 @@ INTERNAL int qthread_spinlock_init(const aligned_t * a, const bool is_recursive)
     return QTHREAD_OPFAIL;
 }
 
-INTERNAL int qthread_spinlock_destroy(const aligned_t * a) {
+INTERNAL int spinlock_destroy_hashed(const aligned_t * a) {
     return lock_hashmap_remove(a);
 }
 
-INTERNAL int qthread_spinlock_finalize() {
-    if(qthread_spinlock_buckets){
-        for (unsigned i = 0; i < QTHREAD_LOCKING_STRIPES; i++) {
-            assert(qthread_spinlock_buckets[i]);
-            qt_hash_destroy_deallocate(qthread_spinlock_buckets[i],
-                            (qt_hash_deallocator_fn)
-                            qthread_spinlock_destroy_fn);
+INTERNAL int spinlock_lock_hashed(const aligned_t * a) {
+    qthread_spinlock_t * l = lock_hashmap_get(a);
+    if (l != NULL) {
+        if (l->state.s >= SPINLOCK_IS_RECURSIVE) {
+            if(l->state.s == qthread_readstate(CURRENT_UNIQUE_WORKER)){ // Reentrant
+                ++l->state.count;
+                MACHINE_FENCE;
+            }else {
+                QTHREAD_TRYLOCK_LOCK(&l->lock);
+                l->state.s = qthread_readstate(CURRENT_UNIQUE_WORKER);
+                ++l->state.count;
+                MACHINE_FENCE;
+            }     
+        } else {
+            QTHREAD_TRYLOCK_LOCK(&l->lock);
         }
-        qt_free(qthread_spinlock_buckets);
+        return QTHREAD_SUCCESS;
     }
-    return QTHREAD_SUCCESS;
+    return QTHREAD_OPFAIL;
 }
 
-INTERNAL int qthread_spinlock_lock(const aligned_t * a) {
+INTERNAL int spinlock_trylock_hashed(const aligned_t * a) {
     qthread_spinlock_t * l = lock_hashmap_get(a);
+    if (l != NULL) {
+        if (l->state.s >= SPINLOCK_IS_RECURSIVE) {
+            if(l->state.s == qthread_readstate(CURRENT_UNIQUE_WORKER)){ // Reentrant
+                ++l->state.count;
+                MACHINE_FENCE;
+            }else {
+                if(QTHREAD_TRYLOCK_TRY(&l->lock)) {
+                    l->state.s = qthread_readstate(CURRENT_UNIQUE_WORKER);
+                    ++l->state.count;
+                    MACHINE_FENCE;
+                } else {
+                    return QTHREAD_OPFAIL;
+                }
+            }     
+        } else {
+            return QTHREAD_TRYLOCK_TRY(&l->lock);
+            
+        }
+        return QTHREAD_SUCCESS;
+    }
+    return QTHREAD_OPFAIL;
+}
+
+INTERNAL int spinlock_unlock_hashed(const aligned_t * a) {
+    qthread_spinlock_t * l = lock_hashmap_get(a);
+    if (l != NULL) {
+        if (l->state.s >= SPINLOCK_IS_RECURSIVE) {
+            if(l->state.s == qthread_readstate(CURRENT_UNIQUE_WORKER)){
+                --l->state.count;
+                if(!l->state.count) {
+                    l->state.s = SPINLOCK_IS_RECURSIVE; // Reset
+                    MACHINE_FENCE;
+                    QTHREAD_TRYLOCK_UNLOCK(&l->lock); 
+                }
+            }else {
+                if(l->state.count)
+                    return QTHREAD_OPFAIL;
+            }
+        } else {
+            QTHREAD_TRYLOCK_UNLOCK(&l->lock);
+        }
+        return QTHREAD_SUCCESS;
+    }
+    return QTHREAD_OPFAIL;
+}
+
+/* locks over lock types externally allocated */
+
+INTERNAL int spinlock_init(qthread_spinlock_t * a, const bool is_recursive) {
+   if (is_recursive) {
+        const qthread_spinlock_t init_mutex = QTHREAD_RECURSIVE_MUTEX_INITIALIZER;
+        memcpy(a, &init_mutex, sizeof(qthread_spinlock_t));
+    } else {
+        const qthread_spinlock_t init_mutex = QTHREAD_MUTEX_INITIALIZER;
+        memcpy(a, &init_mutex, sizeof(qthread_spinlock_t));
+    }
+    return QTHREAD_SUCCESS;
+}
+
+INTERNAL int spinlock_destroy(qthread_spinlock_t * a) {
+    return QTHREAD_SUCCESS;
+}
+
+INTERNAL int spinlock_lock(qthread_spinlock_t * a) {
+    qthread_spinlock_t * l = a;
     if (l != NULL) {
         if (l->state.s >= SPINLOCK_IS_RECURSIVE) {
             if(l->state.s == qthread_readstate(CURRENT_UNIQUE_WORKER)){ // Reentrant
@@ -137,8 +216,8 @@ INTERNAL int qthread_spinlock_lock(const aligned_t * a) {
     return QTHREAD_OPFAIL;
 }
 
-INTERNAL int qthread_spinlock_trylock(const aligned_t * a) {
-    qthread_spinlock_t * l = lock_hashmap_get(a);
+INTERNAL int spinlock_trylock(qthread_spinlock_t * a) {
+    qthread_spinlock_t * l = a;
     if (l != NULL) {
         if (l->state.s >= SPINLOCK_IS_RECURSIVE) {
             if(l->state.s == qthread_readstate(CURRENT_UNIQUE_WORKER)){ // Reentrant
@@ -162,8 +241,8 @@ INTERNAL int qthread_spinlock_trylock(const aligned_t * a) {
     return QTHREAD_OPFAIL;
 }
 
-INTERNAL int qthread_spinlock_unlock(const aligned_t * a) {
-    qthread_spinlock_t * l = lock_hashmap_get(a);
+INTERNAL int spinlock_unlock(qthread_spinlock_t * a) {
+    qthread_spinlock_t * l = a;
     if (l != NULL) {
         if (l->state.s >= SPINLOCK_IS_RECURSIVE) {
             if(l->state.s == qthread_readstate(CURRENT_UNIQUE_WORKER)){
@@ -189,39 +268,64 @@ INTERNAL int qthread_spinlock_unlock(const aligned_t * a) {
 
 int API_FUNC qthread_lock_init(const aligned_t *a, const bool is_recursive)
 {                      /*{{{ */
-    return qthread_spinlock_init(a, is_recursive);
+    return spinlock_init_hashed(a, is_recursive);
 }                      /*}}} */
 
 int API_FUNC qthread_lock_destroy(aligned_t *a)
 {                      /*{{{ */
-    if (!qthread_is_spin_lock(a)) {
+    if (!is_spin_lock_hashed(a)) {
         return QTHREAD_SUCCESS;
     }
-    return qthread_spinlock_destroy(a);
+    return spinlock_destroy_hashed(a);
 }                      /*}}} */
 
 int API_FUNC qthread_lock(const aligned_t *a)
 {                      /*{{{ */
-    if (!qthread_is_spin_lock(a)) {
+    if (!is_spin_lock_hashed(a)) {
         return qthread_readFE(NULL, a);
     }
-    return qthread_spinlock_lock(a);
+    return spinlock_lock_hashed(a);
 }                      /*}}} */
 
 int API_FUNC qthread_trylock(const aligned_t *a)
 {                      /*{{{ */
-    if (!qthread_is_spin_lock(a)){
+    if (!is_spin_lock_hashed(a)){
         return qthread_readFE_nb(NULL, a);
     }
-    return qthread_spinlock_trylock(a);
+    return spinlock_trylock_hashed(a);
 }                      /*}}} */
 
 int API_FUNC qthread_unlock(const aligned_t *a)
 {                      /*{{{ */
-    if (!qthread_is_spin_lock(a)) {
+    if (!is_spin_lock_hashed(a)) {
         return qthread_fill(a);
     }
-    return qthread_spinlock_unlock(a);
+    return spinlock_unlock_hashed(a);
+}                      /*}}} */
+
+int API_FUNC qthread_spinlock_init(qthread_spinlock_t *a, const bool is_recursive)
+{                      /*{{{ */
+    return spinlock_init(a, is_recursive);
+}                      /*}}} */
+
+int API_FUNC qthread_spinlock_destroy(qthread_spinlock_t *a)
+{                      /*{{{ */
+    return spinlock_destroy(a);
+}                      /*}}} */
+
+int API_FUNC qthread_spinlock_lock(qthread_spinlock_t *a)
+{                      /*{{{ */
+    return spinlock_lock(a);
+}                      /*}}} */
+
+int API_FUNC qthread_spinlock_trylock(qthread_spinlock_t *a)
+{                      /*{{{ */
+    return spinlock_trylock(a);
+}                      /*}}} */
+
+int API_FUNC qthread_spinlock_unlock(qthread_spinlock_t *a)
+{                      /*{{{ */
+    return spinlock_unlock(a);
 }                      /*}}} */
 
 #undef qt_hash_t

src/qthread.c

@@ -132,9 +132,8 @@ int GUARD_PAGES = 1;
 #define GUARD_PAGES 0
 #endif
 
-//extern qt_hash * qthread_spinlock_buckets;
-extern int INTERNAL qthread_spinlock_finalize();
-extern int INTERNAL qthread_spinlock_initialize();
+extern int INTERNAL spinlocks_finalize();
+extern int INTERNAL spinlocks_initialize();
 
 /* Internal Prototypes */
 #ifdef QTHREAD_MAKECONTEXT_SPLIT
@@ -1016,7 +1015,7 @@ int API_FUNC qthread_initialize(void)
         qthread_debug(SHEPHERD_DETAILS, "shepherd %i set up (%p)\n", i, &qlib->shepherds[i]);
     }
 
-    qthread_spinlock_initialize();
+    spinlocks_initialize();
 
     qthread_debug(SHEPHERD_DETAILS, "done setting up shepherds.\n");
 
@@ -1574,7 +1573,7 @@ void API_FUNC qthread_finalize(void)
     qthread_debug(CORE_DETAILS, "freeing shep0's threadqueue\n");
     qt_threadqueue_free(shep0->ready);
 
-    qthread_spinlock_finalize();
+    spinlocks_finalize();
 
     qthread_debug(CORE_DETAILS, "calling cleanup functions\n");
     while (qt_cleanup_funcs != NULL) {