refactor initialization/finalization

Applications must now call nng_init(), but they can supply
a set of parameters optionally.  The code is now safe for
multiple libraries to do this concurrently, meaning nng_fini
no longer can race against another instance starting up.

The nni_init checks on all public APIs are removed now.

7 files changed, 51 insertions, 70 deletions

diff --git a/src/platform/posix/posix_impl.h b/src/platform/posix/posix_impl.h
index afbdce97..f8d8965e 100644
--- a/src/platform/posix/posix_impl.h
+++ b/src/platform/posix/posix_impl.h
@@ -147,9 +147,9 @@ struct nni_atomic_ptr {
 
 #endif
 
-extern int  nni_posix_pollq_sysinit(void);
+extern int  nni_posix_pollq_sysinit(nng_init_params *);
 extern void nni_posix_pollq_sysfini(void);
-extern int  nni_posix_resolv_sysinit(void);
+extern int  nni_posix_resolv_sysinit(nng_init_params *);
 extern void nni_posix_resolv_sysfini(void);
 
 #endif // PLATFORM_POSIX_IMPL_H
diff --git a/src/platform/posix/posix_pollq_epoll.c b/src/platform/posix/posix_pollq_epoll.c
index dde331d5..a5c8d8c9 100644
--- a/src/platform/posix/posix_pollq_epoll.c
+++ b/src/platform/posix/posix_pollq_epoll.c
@@ -1,5 +1,5 @@
 //
-// Copyright 2020 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2024 Staysail Systems, Inc. <info@staysail.tech>
 // Copyright 2018 Capitar IT Group BV <info@capitar.com>
 // Copyright 2018 Liam Staskawicz <liam@stask.net>
 //
@@ -65,7 +65,7 @@ struct nni_posix_pfd {
 	nni_posix_pollq *pq;
 	int              fd;
 	nni_posix_pfd_cb cb;
-	void *           arg;
+	void            *arg;
 	bool             closed;
 	bool             closing;
 	bool             reap;
@@ -80,8 +80,8 @@ static nni_posix_pollq nni_posix_global_pollq;
 int
 nni_posix_pfd_init(nni_posix_pfd **pfdp, int fd)
 {
-	nni_posix_pfd *    pfd;
-	nni_posix_pollq *  pq;
+	nni_posix_pfd     *pfd;
+	nni_posix_pollq   *pq;
 	struct epoll_event ev;
 	int                rv;
 
@@ -173,7 +173,7 @@ nni_posix_pfd_close(nni_posix_pfd *pfd)
 {
 	nni_mtx_lock(&pfd->mtx);
 	if (!pfd->closing) {
-		nni_posix_pollq *  pq = pfd->pq;
+		nni_posix_pollq   *pq = pfd->pq;
 		struct epoll_event ev; // Not actually used.
 		pfd->closing = true;
 
@@ -243,7 +243,7 @@ nni_posix_pollq_reap(nni_posix_pollq *pq)
 static void
 nni_posix_poll_thr(void *arg)
 {
-	nni_posix_pollq *  pq = arg;
+	nni_posix_pollq   *pq = arg;
 	struct epoll_event events[NNI_MAX_EPOLL_EVENTS];
 
 	for (;;) {
@@ -271,9 +271,9 @@ nni_posix_poll_thr(void *arg)
 				}
 				reap = true;
 			} else {
-				nni_posix_pfd *  pfd = ev->data.ptr;
+				nni_posix_pfd   *pfd = ev->data.ptr;
 				nni_posix_pfd_cb cb;
-				void *           cbarg;
+				void            *cbarg;
 				unsigned         mask;
 
 				mask = ev->events &
@@ -395,8 +395,9 @@ nni_posix_pollq_create(nni_posix_pollq *pq)
 }
 
 int
-nni_posix_pollq_sysinit(void)
+nni_posix_pollq_sysinit(nng_init_params *params)
 {
+	NNI_ARG_UNUSED(params);
 	return (nni_posix_pollq_create(&nni_posix_global_pollq));
 }
 
diff --git a/src/platform/posix/posix_pollq_kqueue.c b/src/platform/posix/posix_pollq_kqueue.c
index dda81be4..ae1b2f47 100644
--- a/src/platform/posix/posix_pollq_kqueue.c
+++ b/src/platform/posix/posix_pollq_kqueue.c
@@ -1,5 +1,5 @@
 //
-// Copyright 2020 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2024 Staysail Systems, Inc. <info@staysail.tech>
 // Copyright 2018 Capitar IT Group BV <info@capitar.com>
 // Copyright 2018 Liam Staskawicz <liam@stask.net>
 //
@@ -9,6 +9,7 @@
 // found online at https://opensource.org/licenses/MIT.
 //
 
+#include "core/defs.h"
 #ifdef NNG_HAVE_KQUEUE
 
 #include <errno.h>
@@ -41,7 +42,7 @@ struct nni_posix_pfd {
 	nni_list_node    node; // linkage into the reap list
 	nni_posix_pollq *pq;   // associated pollq
 	int              fd;   // file descriptor to poll
-	void *           arg;  // user data
+	void            *arg;  // user data
 	nni_posix_pfd_cb cb;   // user callback on event
 	bool             closed;
 	unsigned         events;
@@ -57,7 +58,7 @@ static nni_posix_pollq nni_posix_global_pollq;
 int
 nni_posix_pfd_init(nni_posix_pfd **pfdp, int fd)
 {
-	nni_posix_pfd *  pf;
+	nni_posix_pfd   *pf;
 	nni_posix_pollq *pq;
 	struct kevent    ev[2];
 	unsigned         flags = EV_ADD | EV_DISABLE | EV_CLEAR;
@@ -234,9 +235,9 @@ nni_posix_poll_thr(void *arg)
 	for (;;) {
 		int              n;
 		struct kevent    evs[NNI_MAX_KQUEUE_EVENTS];
-		nni_posix_pfd *  pf;
+		nni_posix_pfd   *pf;
 		nni_posix_pfd_cb cb;
-		void *           cbarg;
+		void            *cbarg;
 		unsigned         revents;
 
 		nni_mtx_lock(&pq->mtx);
@@ -335,8 +336,9 @@ nni_posix_pollq_create(nni_posix_pollq *pq)
 }
 
 int
-nni_posix_pollq_sysinit(void)
+nni_posix_pollq_sysinit(nng_init_params *params)
 {
+	NNI_ARG_UNUSED(params);
 	return (nni_posix_pollq_create(&nni_posix_global_pollq));
 }
 
diff --git a/src/platform/posix/posix_pollq_poll.c b/src/platform/posix/posix_pollq_poll.c
index f6f81703..302f97ca 100644
--- a/src/platform/posix/posix_pollq_poll.c
+++ b/src/platform/posix/posix_pollq_poll.c
@@ -1,5 +1,5 @@
 //
-// Copyright 2020 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2024 Staysail Systems, Inc. <info@staysail.tech>
 // Copyright 2018 Capitar IT Group BV <info@capitar.com>
 //
 // This software is supplied under the terms of the MIT License, a
@@ -8,6 +8,7 @@
 // found online at https://opensource.org/licenses/MIT.
 //
 
+#include "core/defs.h"
 #include "core/nng_impl.h"
 #include "platform/posix/posix_pollq.h"
 
@@ -63,7 +64,7 @@ struct nni_posix_pfd {
 	nni_mtx          mtx;
 	unsigned         events;
 	nni_posix_pfd_cb cb;
-	void *           arg;
+	void            *arg;
 };
 
 static nni_posix_pollq nni_posix_global_pollq;
@@ -71,7 +72,7 @@ static nni_posix_pollq nni_posix_global_pollq;
 int
 nni_posix_pfd_init(nni_posix_pfd **pfdp, int fd)
 {
-	nni_posix_pfd *  pfd;
+	nni_posix_pfd   *pfd;
 	nni_posix_pollq *pq = &nni_posix_global_pollq;
 
 	// Set this is as soon as possible (narrow the close-exec race as
@@ -186,8 +187,8 @@ nni_posix_poll_thr(void *arg)
 {
 	nni_posix_pollq *pq     = arg;
 	int              nalloc = 0;
-	struct pollfd *  fds    = NULL;
-	nni_posix_pfd ** pfds   = NULL;
+	struct pollfd   *fds    = NULL;
+	nni_posix_pfd  **pfds   = NULL;
 
 	for (;;) {
 		int            nfds;
@@ -277,7 +278,7 @@ nni_posix_poll_thr(void *arg)
 		for (int i = 1; i < nfds; i++) {
 			if ((events = fds[i].revents) != 0) {
 				nni_posix_pfd_cb cb;
-				void *           arg;
+				void            *arg;
 
 				pfd = pfds[i];
 
@@ -336,8 +337,9 @@ nni_posix_pollq_create(nni_posix_pollq *pq)
 }
 
 int
-nni_posix_pollq_sysinit(void)
+nni_posix_pollq_sysinit(nng_init_params *params)
 {
+	NNI_ARG_UNUSED(params);
 	return (nni_posix_pollq_create(&nni_posix_global_pollq));
 }
 
diff --git a/src/platform/posix/posix_pollq_port.c b/src/platform/posix/posix_pollq_port.c
index 7ce96f68..23b4f074 100644
--- a/src/platform/posix/posix_pollq_port.c
+++ b/src/platform/posix/posix_pollq_port.c
@@ -42,7 +42,7 @@ struct nni_posix_pfd {
 	bool             closed;
 	bool             closing;
 	nni_posix_pfd_cb cb;
-	void *           data;
+	void            *data;
 };
 
 // single global instance for now
@@ -52,7 +52,7 @@ int
 nni_posix_pfd_init(nni_posix_pfd **pfdp, int fd)
 {
 	nni_posix_pollq *pq;
-	nni_posix_pfd *  pfd;
+	nni_posix_pfd   *pfd;
 
 	pq = &nni_posix_global_pollq;
 
@@ -153,10 +153,10 @@ nni_posix_poll_thr(void *arg)
 	for (;;) {
 		nni_posix_pollq *pq = arg;
 		port_event_t     ev[NNI_MAX_PORTEV];
-		nni_posix_pfd *  pfd;
+		nni_posix_pfd   *pfd;
 		unsigned         events;
 		nni_posix_pfd_cb cb;
-		void *           arg;
+		void            *arg;
 		unsigned         n;
 
 		n = 1; // wake us even on just one event
@@ -246,8 +246,9 @@ nni_posix_pfd_set_cb(nni_posix_pfd *pfd, nni_posix_pfd_cb cb, void *arg)
 }
 
 int
-nni_posix_pollq_sysinit(void)
+nni_posix_pollq_sysinit(nng_init_params *params)
 {
+	NNI_ARG_UNUSED(params);
 	return (nni_posix_pollq_create(&nni_posix_global_pollq));
 }
 
diff --git a/src/platform/posix/posix_resolv_gai.c b/src/platform/posix/posix_resolv_gai.c
index f522499e..f85c27c1 100644
--- a/src/platform/posix/posix_resolv_gai.c
+++ b/src/platform/posix/posix_resolv_gai.c
@@ -8,6 +8,7 @@
 // found online at https://opensource.org/licenses/MIT.
 //
 
+#include "core/init.h"
 #include "core/nng_impl.h"
 
 #ifdef NNG_USE_POSIX_RESOLV_GAI
@@ -42,7 +43,7 @@ static nni_cv   resolv_cv   = NNI_CV_INITIALIZER(&resolv_mtx);
 static bool     resolv_fini = false;
 static nni_list resolv_aios;
 static nni_thr *resolv_thrs;
-static int      resolv_num_thr;
+static int16_t  resolv_num_thr;
 
 typedef struct resolv_item resolv_item;
 struct resolv_item {
@@ -477,22 +478,17 @@ nni_parse_ip_port(const char *addr, nni_sockaddr *sa)
 }
 
 int
-nni_posix_resolv_sysinit(void)
+nni_posix_resolv_sysinit(nng_init_params *params)
 {
 	resolv_fini = false;
 	nni_aio_list_init(&resolv_aios);
 
-#ifndef NNG_RESOLV_CONCURRENCY
-#define NNG_RESOLV_CONCURRENCY 4
-#endif
-
-	resolv_num_thr = (int) nni_init_get_param(
-	    NNG_INIT_NUM_RESOLVER_THREADS, NNG_RESOLV_CONCURRENCY);
+	resolv_num_thr = params->num_resolver_threads;
 	if (resolv_num_thr < 1) {
 		resolv_num_thr = 1;
 	}
+	params->num_resolver_threads = resolv_num_thr;
 	// no limit on the maximum for now
-	nni_init_set_effective(NNG_INIT_NUM_RESOLVER_THREADS, resolv_num_thr);
 	resolv_thrs = NNI_ALLOC_STRUCTS(resolv_thrs, resolv_num_thr);
 	if (resolv_thrs == NULL) {
 		return (NNG_ENOMEM);
diff --git a/src/platform/posix/posix_thread.c b/src/platform/posix/posix_thread.c
index b7cd5e1e..343a6ba5 100644
--- a/src/platform/posix/posix_thread.c
+++ b/src/platform/posix/posix_thread.c
@@ -1,5 +1,5 @@
 //
-// Copyright 2021 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2024 Staysail Systems, Inc. <info@staysail.tech>
 // Copyright 2018 Capitar IT Group BV <info@capitar.com>
 //
 // This software is supplied under the terms of the MIT License, a
@@ -11,6 +11,7 @@
 // POSIX threads.
 
 #include "core/nng_impl.h"
+#include "nng/nng.h"
 
 #ifdef NNG_PLATFORM_POSIX
 
@@ -33,9 +34,8 @@
 #include <sys/resource.h>
 #endif
 
-static pthread_mutex_t nni_plat_init_lock = PTHREAD_MUTEX_INITIALIZER;
-static volatile int    nni_plat_inited    = 0;
-static int             nni_plat_forked    = 0;
+static bool nni_plat_inited = 0;
+static bool nni_plat_forked = 0;
 
 pthread_condattr_t  nni_cvattr;
 pthread_mutexattr_t nni_mxattr;
@@ -143,6 +143,9 @@ nni_pthread_cond_timedwait(
 void
 nni_plat_mtx_lock(nni_plat_mtx *mtx)
 {
+	if (nni_plat_forked) {
+		nni_panic("nng is not fork-reentrant safe");
+	}
 	nni_pthread_mutex_lock(&mtx->mtx);
 }
 
@@ -339,35 +342,20 @@ nni_atfork_child(void)
 }
 
 int
-nni_plat_init(int (*helper)(void))
+nni_plat_init(nng_init_params *params)
 {
 	int rv;
 
-	if (nni_plat_forked) {
-		nni_panic("nng is not fork-reentrant safe");
-	}
-	if (nni_plat_inited) {
-		return (0); // fast path
-	}
-
-	pthread_mutex_lock(&nni_plat_init_lock);
-	if (nni_plat_inited) { // check again under the lock to be sure
-		pthread_mutex_unlock(&nni_plat_init_lock);
-		return (0);
-	}
-
 	if ((pthread_mutexattr_init(&nni_mxattr) != 0) ||
 	    (pthread_condattr_init(&nni_cvattr) != 0) ||
 	    (pthread_attr_init(&nni_thrattr) != 0)) {
 		// Technically this is leaking, but it should never
 		// occur, so really not worried about it.
-		pthread_mutex_unlock(&nni_plat_init_lock);
 		return (NNG_ENOMEM);
 	}
 
 #if !defined(NNG_USE_GETTIMEOFDAY) && NNG_USE_CLOCKID != CLOCK_REALTIME
 	if (pthread_condattr_setclock(&nni_cvattr, NNG_USE_CLOCKID) != 0) {
-		pthread_mutex_unlock(&nni_plat_init_lock);
 		pthread_mutexattr_destroy(&nni_mxattr);
 		pthread_condattr_destroy(&nni_cvattr);
 		pthread_attr_destroy(&nni_thrattr);
@@ -381,7 +369,6 @@ nni_plat_init(int (*helper)(void))
 	    (rl.rlim_cur != RLIM_INFINITY) &&
 	    (rl.rlim_cur >= PTHREAD_STACK_MIN) &&
 	    (pthread_attr_setstacksize(&nni_thrattr, rl.rlim_cur) != 0)) {
-		pthread_mutex_unlock(&nni_plat_init_lock);
 		pthread_mutexattr_destroy(&nni_mxattr);
 		pthread_condattr_destroy(&nni_cvattr);
 		pthread_attr_destroy(&nni_thrattr);
@@ -393,16 +380,14 @@ nni_plat_init(int (*helper)(void))
 	(void) pthread_mutexattr_settype(
 	    &nni_mxattr, PTHREAD_MUTEX_ERRORCHECK);
 
-	if ((rv = nni_posix_pollq_sysinit()) != 0) {
-		pthread_mutex_unlock(&nni_plat_init_lock);
+	if ((rv = nni_posix_pollq_sysinit(params)) != 0) {
 		pthread_mutexattr_destroy(&nni_mxattr);
 		pthread_condattr_destroy(&nni_cvattr);
 		pthread_attr_destroy(&nni_thrattr);
 		return (rv);
 	}
 
-	if ((rv = nni_posix_resolv_sysinit()) != 0) {
-		pthread_mutex_unlock(&nni_plat_init_lock);
+	if ((rv = nni_posix_resolv_sysinit(params)) != 0) {
 		nni_posix_pollq_sysfini();
 		pthread_mutexattr_destroy(&nni_mxattr);
 		pthread_condattr_destroy(&nni_cvattr);
@@ -411,7 +396,6 @@ nni_plat_init(int (*helper)(void))
 	}
 
 	if (pthread_atfork(NULL, NULL, nni_atfork_child) != 0) {
-		pthread_mutex_unlock(&nni_plat_init_lock);
 		nni_posix_resolv_sysfini();
 		nni_posix_pollq_sysfini();
 		pthread_mutexattr_destroy(&nni_mxattr);
@@ -419,10 +403,7 @@ nni_plat_init(int (*helper)(void))
 		pthread_attr_destroy(&nni_thrattr);
 		return (NNG_ENOMEM);
 	}
-	if ((rv = helper()) == 0) {
-		nni_plat_inited = 1;
-	}
-	pthread_mutex_unlock(&nni_plat_init_lock);
+	nni_plat_inited = 1;
 
 	return (rv);
 }
@@ -430,7 +411,6 @@ nni_plat_init(int (*helper)(void))
 void
 nni_plat_fini(void)
 {
-	pthread_mutex_lock(&nni_plat_init_lock);
 	if (nni_plat_inited) {
 		nni_posix_resolv_sysfini();
 		nni_posix_pollq_sysfini();
@@ -438,7 +418,6 @@ nni_plat_fini(void)
 		pthread_condattr_destroy(&nni_cvattr);
 		nni_plat_inited = 0;
 	}
-	pthread_mutex_unlock(&nni_plat_init_lock);
 }
 
 int