Could use GCC atomics for older versions of GCC.

This should help greatly with performance on older systems such
as CentOS 7 and GCC 4.8.  Though, such folks really should update
to newer compilers.  Folks running version of GCC earlier than 4.7
will still pay a rather significant performance penalty, as they
still implement atomics with a global mutex.

3 files changed, 221 insertions, 53 deletions

diff --git a/src/core/defs.h b/src/core/defs.h
index fb9c7447..f53c1d27 100644
--- a/src/core/defs.h
+++ b/src/core/defs.h
@@ -31,11 +31,11 @@
 
 // These types are common but have names shared with user space.
 // Internal code should use these names when possible.
-typedef nng_msg           nni_msg;
-typedef nng_sockaddr      nni_sockaddr;
-typedef nng_url           nni_url;
-typedef nng_iov           nni_iov;
-typedef nng_aio           nni_aio;
+typedef nng_msg      nni_msg;
+typedef nng_sockaddr nni_sockaddr;
+typedef nng_url      nni_url;
+typedef nng_iov      nni_iov;
+typedef nng_aio      nni_aio;
 
 // These are our own names.
 typedef struct nni_socket   nni_sock;
@@ -44,10 +44,10 @@ typedef struct nni_dialer   nni_dialer;
 typedef struct nni_listener nni_listener;
 typedef struct nni_pipe     nni_pipe;
 
-typedef struct nni_sp_tran           nni_sp_tran;
-typedef struct nni_sp_dialer_ops     nni_sp_dialer_ops;
-typedef struct nni_sp_listener_ops   nni_sp_listener_ops;
-typedef struct nni_sp_pipe_ops       nni_sp_pipe_ops;
+typedef struct nni_sp_tran         nni_sp_tran;
+typedef struct nni_sp_dialer_ops   nni_sp_dialer_ops;
+typedef struct nni_sp_listener_ops nni_sp_listener_ops;
+typedef struct nni_sp_pipe_ops     nni_sp_pipe_ops;
 
 typedef struct nni_proto_ctx_ops  nni_proto_ctx_ops;
 typedef struct nni_proto_sock_ops nni_proto_sock_ops;
@@ -75,51 +75,51 @@ typedef void (*nni_cb)(void *);
 #define NNI_ALLOC_STRUCTS(s, n) nni_zalloc(sizeof(*s) * n)
 #define NNI_FREE_STRUCTS(s, n) nni_free(s, sizeof(*s) * n)
 
-#define NNI_PUT16(ptr, u)                                    \
-	do {                                                 \
-		(ptr)[0] = (uint8_t)(((uint16_t)(u)) >> 8u); \
-		(ptr)[1] = (uint8_t)((uint16_t)(u));         \
+#define NNI_PUT16(ptr, u)                                      \
+	do {                                                   \
+		(ptr)[0] = (uint8_t) (((uint16_t) (u)) >> 8u); \
+		(ptr)[1] = (uint8_t) ((uint16_t) (u));         \
 	} while (0)
 
-#define NNI_PUT32(ptr, u)                                     \
-	do {                                                  \
-		(ptr)[0] = (uint8_t)(((uint32_t)(u)) >> 24u); \
-		(ptr)[1] = (uint8_t)(((uint32_t)(u)) >> 16u); \
-		(ptr)[2] = (uint8_t)(((uint32_t)(u)) >> 8u);  \
-		(ptr)[3] = (uint8_t)((uint32_t)(u));          \
+#define NNI_PUT32(ptr, u)                                       \
+	do {                                                    \
+		(ptr)[0] = (uint8_t) (((uint32_t) (u)) >> 24u); \
+		(ptr)[1] = (uint8_t) (((uint32_t) (u)) >> 16u); \
+		(ptr)[2] = (uint8_t) (((uint32_t) (u)) >> 8u);  \
+		(ptr)[3] = (uint8_t) ((uint32_t) (u));          \
 	} while (0)
 
-#define NNI_PUT64(ptr, u)                                     \
-	do {                                                  \
-		(ptr)[0] = (uint8_t)(((uint64_t)(u)) >> 56u); \
-		(ptr)[1] = (uint8_t)(((uint64_t)(u)) >> 48u); \
-		(ptr)[2] = (uint8_t)(((uint64_t)(u)) >> 40u); \
-		(ptr)[3] = (uint8_t)(((uint64_t)(u)) >> 32u); \
-		(ptr)[4] = (uint8_t)(((uint64_t)(u)) >> 24u); \
-		(ptr)[5] = (uint8_t)(((uint64_t)(u)) >> 16u); \
-		(ptr)[6] = (uint8_t)(((uint64_t)(u)) >> 8u);  \
-		(ptr)[7] = (uint8_t)((uint64_t)(u));          \
+#define NNI_PUT64(ptr, u)                                       \
+	do {                                                    \
+		(ptr)[0] = (uint8_t) (((uint64_t) (u)) >> 56u); \
+		(ptr)[1] = (uint8_t) (((uint64_t) (u)) >> 48u); \
+		(ptr)[2] = (uint8_t) (((uint64_t) (u)) >> 40u); \
+		(ptr)[3] = (uint8_t) (((uint64_t) (u)) >> 32u); \
+		(ptr)[4] = (uint8_t) (((uint64_t) (u)) >> 24u); \
+		(ptr)[5] = (uint8_t) (((uint64_t) (u)) >> 16u); \
+		(ptr)[6] = (uint8_t) (((uint64_t) (u)) >> 8u);  \
+		(ptr)[7] = (uint8_t) ((uint64_t) (u));          \
 	} while (0)
 
-#define NNI_GET16(ptr, v)                             \
-	v = (((uint16_t)((uint8_t)(ptr)[0])) << 8u) + \
-	    (((uint16_t)(uint8_t)(ptr)[1]))
-
-#define NNI_GET32(ptr, v)                              \
-	v = (((uint32_t)((uint8_t)(ptr)[0])) << 24u) + \
-	    (((uint32_t)((uint8_t)(ptr)[1])) << 16u) + \
-	    (((uint32_t)((uint8_t)(ptr)[2])) << 8u) +  \
-	    (((uint32_t)(uint8_t)(ptr)[3]))
-
-#define NNI_GET64(ptr, v)                              \
-	v = (((uint64_t)((uint8_t)(ptr)[0])) << 56u) + \
-	    (((uint64_t)((uint8_t)(ptr)[1])) << 48u) + \
-	    (((uint64_t)((uint8_t)(ptr)[2])) << 40u) + \
-	    (((uint64_t)((uint8_t)(ptr)[3])) << 32u) + \
-	    (((uint64_t)((uint8_t)(ptr)[4])) << 24u) + \
-	    (((uint64_t)((uint8_t)(ptr)[5])) << 16u) + \
-	    (((uint64_t)((uint8_t)(ptr)[6])) << 8u) +  \
-	    (((uint64_t)(uint8_t)(ptr)[7]))
+#define NNI_GET16(ptr, v)                               \
+	v = (((uint16_t) ((uint8_t) (ptr)[0])) << 8u) + \
+	    (((uint16_t) (uint8_t) (ptr)[1]))
+
+#define NNI_GET32(ptr, v)                                \
+	v = (((uint32_t) ((uint8_t) (ptr)[0])) << 24u) + \
+	    (((uint32_t) ((uint8_t) (ptr)[1])) << 16u) + \
+	    (((uint32_t) ((uint8_t) (ptr)[2])) << 8u) +  \
+	    (((uint32_t) (uint8_t) (ptr)[3]))
+
+#define NNI_GET64(ptr, v)                                \
+	v = (((uint64_t) ((uint8_t) (ptr)[0])) << 56u) + \
+	    (((uint64_t) ((uint8_t) (ptr)[1])) << 48u) + \
+	    (((uint64_t) ((uint8_t) (ptr)[2])) << 40u) + \
+	    (((uint64_t) ((uint8_t) (ptr)[3])) << 32u) + \
+	    (((uint64_t) ((uint8_t) (ptr)[4])) << 24u) + \
+	    (((uint64_t) ((uint8_t) (ptr)[5])) << 16u) + \
+	    (((uint64_t) ((uint8_t) (ptr)[6])) << 8u) +  \
+	    (((uint64_t) (uint8_t) (ptr)[7]))
 
 // This increments a pointer a fixed number of byte cells.
 #define NNI_INCPTR(ptr, n) ((ptr) = (void *) ((char *) (ptr) + (n)))
@@ -171,4 +171,11 @@ typedef nni_type nni_opt_type;
 #define NNI_EXPIRE_BATCH 100
 #endif
 
+#if __GNUC__ > 3
+// NNI_GCC_VERSION is used to indicate a GNU version.  It is used
+// to trigger certain cases like atomics that might be compiler specific.
+#define NNI_GCC_VERSION \
+	(__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__)
+#endif
+
 #endif // CORE_DEFS_H
diff --git a/src/platform/posix/posix_atomic.c b/src/platform/posix/posix_atomic.c
index b75bbfd7..0f904ae1 100644
--- a/src/platform/posix/posix_atomic.c
+++ b/src/platform/posix/posix_atomic.c
@@ -81,7 +81,7 @@ nni_atomic_get(nni_atomic_int *v)
 void
 nni_atomic_set(nni_atomic_int *v, int i)
 {
-	 atomic_store(&v->v, i);
+	atomic_store(&v->v, i);
 }
 
 void *
@@ -190,6 +190,159 @@ nni_atomic_cas64(nni_atomic_u64 *v, uint64_t comp, uint64_t new)
 	return (atomic_compare_exchange_strong(&v->v, &cv, nv));
 }
 
+#elif NNI_GCC_VERSION >= 40700 || \
+    defined(__clang__) // we have "new" GCC __atomic builtins
+bool
+nni_atomic_flag_test_and_set(nni_atomic_flag *f)
+{
+	return (__atomic_test_and_set(&f->f, __ATOMIC_SEQ_CST));
+}
+
+void
+nni_atomic_flag_reset(nni_atomic_flag *f)
+{
+	__atomic_clear(&f->f, __ATOMIC_SEQ_CST);
+}
+
+void
+nni_atomic_set_bool(nni_atomic_bool *b, bool n)
+{
+	__atomic_store_n(&b->b, n, __ATOMIC_SEQ_CST);
+}
+
+bool
+nni_atomic_get_bool(nni_atomic_bool *b)
+{
+	return (__atomic_load_n(&b->b, __ATOMIC_SEQ_CST));
+}
+
+bool
+nni_atomic_swap_bool(nni_atomic_bool *b, bool n)
+{
+	return (__atomic_exchange_n(&b->b, n, __ATOMIC_SEQ_CST));
+}
+
+void
+nni_atomic_init_bool(nni_atomic_bool *b)
+{
+	__atomic_store_n(&b->b, false, __ATOMIC_SEQ_CST);
+}
+
+void
+nni_atomic_add64(nni_atomic_u64 *v, uint64_t bump)
+{
+	__atomic_add_fetch(&v->v, bump, __ATOMIC_RELAXED);
+}
+
+void
+nni_atomic_sub64(nni_atomic_u64 *v, uint64_t bump)
+{
+	__atomic_sub_fetch(&v->v, bump, __ATOMIC_RELAXED);
+}
+
+uint64_t
+nni_atomic_get64(nni_atomic_u64 *v)
+{
+	return (__atomic_load_n(&v->v, __ATOMIC_SEQ_CST));
+}
+
+void
+nni_atomic_set64(nni_atomic_u64 *v, uint64_t u)
+{
+	__atomic_store_n(&v->v, u, __ATOMIC_SEQ_CST);
+}
+
+uint64_t
+nni_atomic_swap64(nni_atomic_u64 *v, uint64_t u)
+{
+	return (__atomic_exchange_n(&v->v, u, __ATOMIC_SEQ_CST));
+}
+
+void
+nni_atomic_init64(nni_atomic_u64 *v)
+{
+	__atomic_store_n(&v->v, 0, __ATOMIC_SEQ_CST);
+}
+
+void
+nni_atomic_inc64(nni_atomic_u64 *v)
+{
+	__atomic_add_fetch(&v->v, 1, __ATOMIC_SEQ_CST);
+}
+
+uint64_t
+nni_atomic_dec64_nv(nni_atomic_u64 *v)
+{
+	return (__atomic_sub_fetch(&v->v, 1, __ATOMIC_SEQ_CST));
+}
+
+bool
+nni_atomic_cas64(nni_atomic_u64 *v, uint64_t comp, uint64_t new)
+{
+
+	return (__atomic_compare_exchange_n(
+	    &v->v, &comp, new, false, __ATOMIC_SEQ_CST, __ATOMIC_SEQ_CST));
+}
+
+void
+nni_atomic_init(nni_atomic_int *v)
+{
+	__atomic_store_n(&v->v, 0, __ATOMIC_SEQ_CST);
+}
+
+void
+nni_atomic_add(nni_atomic_int *v, int bump)
+{
+	__atomic_add_fetch(&v->v, bump, __ATOMIC_RELAXED);
+}
+
+void
+nni_atomic_sub(nni_atomic_int *v, int bump)
+{
+	__atomic_sub_fetch(&v->v, bump, __ATOMIC_RELAXED);
+}
+
+void
+nni_atomic_set(nni_atomic_int *v, int val)
+{
+	__atomic_store_n(&v->v, val, __ATOMIC_SEQ_CST);
+}
+
+int
+nni_atomic_get(nni_atomic_int *v)
+{
+	return (__atomic_load_n(&v->v, __ATOMIC_SEQ_CST));
+}
+void
+nni_atomic_inc(nni_atomic_int *v)
+{
+	__atomic_add_fetch(&v->v, 1, __ATOMIC_SEQ_CST);
+}
+
+void
+nni_atomic_dec(nni_atomic_int *v)
+{
+	__atomic_sub_fetch(&v->v, 1, __ATOMIC_SEQ_CST);
+}
+
+int
+nni_atomic_dec_nv(nni_atomic_int *v)
+{
+	return (__atomic_sub_fetch(&v->v, 1, __ATOMIC_SEQ_CST));
+}
+
+void *
+nni_atomic_get_ptr(nni_atomic_ptr *v)
+{
+	return (__atomic_load_n(&v->v, __ATOMIC_SEQ_CST));
+}
+
+void
+nni_atomic_set_ptr(nni_atomic_ptr *v, void *p)
+{
+	__atomic_store_n(&v->v, p, __ATOMIC_SEQ_CST);
+}
+
 #else
 
 #include <pthread.h>
diff --git a/src/platform/posix/posix_impl.h b/src/platform/posix/posix_impl.h
index 0ee7cb38..afbdce97 100644
--- a/src/platform/posix/posix_impl.h
+++ b/src/platform/posix/posix_impl.h
@@ -58,13 +58,19 @@ struct nni_plat_mtx {
 	pthread_mutex_t mtx;
 };
 
-#define NNI_MTX_INITIALIZER { PTHREAD_MUTEX_INITIALIZER }
+#define NNI_MTX_INITIALIZER               \
+	{                                 \
+		PTHREAD_MUTEX_INITIALIZER \
+	}
 
 struct nni_rwlock {
 	pthread_rwlock_t rwl;
 };
 
-#define NNI_RWLOCK_INITIALIZER { PTHREAD_RWLOCK_INITIALIZER }
+#define NNI_RWLOCK_INITIALIZER             \
+	{                                  \
+		PTHREAD_RWLOCK_INITIALIZER \
+	}
 
 // No static form of CV initialization because of the need to use
 // attributes to set the clock type.
@@ -76,7 +82,10 @@ struct nni_plat_cv {
 // NOTE: condition variables initialized with this should *NOT*
 // be used with nni_cv_until -- the clock attributes are not passed
 // and the wake-up times will not be correct.
-#define NNI_CV_INITIALIZER(mxp) { .mtx = mxp, .cv = PTHREAD_COND_INITIALIZER }
+#define NNI_CV_INITIALIZER(mxp)                            \
+	{                                                  \
+		.mtx = mxp, .cv = PTHREAD_COND_INITIALIZER \
+	}
 
 struct nni_plat_thr {
 	pthread_t tid;
@@ -91,7 +100,6 @@ struct nni_plat_flock {
 #define NNG_PLATFORM_DIR_SEP "/"
 
 #ifdef NNG_HAVE_STDATOMIC
-
 #include <stdatomic.h>
 
 struct nni_atomic_flag {