fixes #1572 nng creates too many threads

This further limits some of the thread counts, but principally it
offers a new runtime facility, nng_init_set_parameter(), which can
be used to set certain runtime parameters on the number of threads,
provided it is called before the rest of application start up.

This facility is quite intentionally "undocumented", at least for now,
as we want to limit our commitment to it.  Still this should be helpful
for applications that need to reduce the number of threads that are
created.

3 files changed, 83 insertions, 45 deletions

diff --git a/src/platform/posix/posix_resolv_gai.c b/src/platform/posix/posix_resolv_gai.c
index c6abee5f..8eaa29f2 100644
--- a/src/platform/posix/posix_resolv_gai.c
+++ b/src/platform/posix/posix_resolv_gai.c
@@ -1,5 +1,5 @@
 //
-// Copyright 2021 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2024 Staysail Systems, Inc. <info@staysail.tech>
 // Copyright 2018 Capitar IT Group BV <info@capitar.com>
 //
 // This software is supplied under the terms of the MIT License, a
@@ -25,14 +25,10 @@
 // for it to ensure that names can be looked up concurrently.  This isn't
 // as elegant or scalable as a true asynchronous resolver would be, but
 // it has the advantage of being fairly portable, and concurrent enough for
-// the vast, vast majority of use cases.  The total thread count can be
+// the vast majority of use cases.  The total thread count can be
 // changed with this define.  Note that some platforms may not have a
 // thread-safe getaddrinfo().  In that case they should set this to 1.
 
-#ifndef NNG_RESOLV_CONCURRENCY
-#define NNG_RESOLV_CONCURRENCY 4
-#endif
-
 #ifndef AI_NUMERICSERV
 #define AI_NUMERICSERV 0
 #endif
@@ -41,7 +37,8 @@ static nni_mtx  resolv_mtx  = NNI_MTX_INITIALIZER;
 static nni_cv   resolv_cv   = NNI_CV_INITIALIZER(&resolv_mtx);
 static bool     resolv_fini = false;
 static nni_list resolv_aios;
-static nni_thr  resolv_thrs[NNG_RESOLV_CONCURRENCY];
+static nni_thr *resolv_thrs;
+static int      resolv_num_thr;
 
 typedef struct resolv_item resolv_item;
 struct resolv_item {
@@ -450,14 +447,30 @@ nni_posix_resolv_sysinit(void)
 	resolv_fini = false;
 	nni_aio_list_init(&resolv_aios);
 
-	for (int i = 0; i < NNG_RESOLV_CONCURRENCY; i++) {
+#ifndef NNG_RESOLV_CONCURRENCY
+#define NNG_RESOLV_CONCURRENCY 4
+#endif
+
+	resolv_num_thr = (int) nni_init_get_param(
+	    NNG_INIT_NUM_RESOLVER_THREADS, NNG_RESOLV_CONCURRENCY);
+	if (resolv_num_thr < 1) {
+		resolv_num_thr = 1;
+	}
+	// no limit on the maximum for now
+	nni_init_set_effective(NNG_INIT_NUM_RESOLVER_THREADS, resolv_num_thr);
+	resolv_thrs = NNI_ALLOC_STRUCTS(resolv_thrs, resolv_num_thr);
+	if (resolv_thrs == NULL) {
+		return (NNG_ENOMEM);
+	}
+
+	for (int i = 0; i < resolv_num_thr; i++) {
 		int rv = nni_thr_init(&resolv_thrs[i], resolv_worker, NULL);
 		if (rv != 0) {
 			nni_posix_resolv_sysfini();
 			return (rv);
 		}
 	}
-	for (int i = 0; i < NNG_RESOLV_CONCURRENCY; i++) {
+	for (int i = 0; i < resolv_num_thr; i++) {
 		nni_thr_run(&resolv_thrs[i]);
 	}
 
@@ -472,8 +485,11 @@ nni_posix_resolv_sysfini(void)
 	nni_cv_wake(&resolv_cv);
 	nni_mtx_unlock(&resolv_mtx);
 
-	for (int i = 0; i < NNG_RESOLV_CONCURRENCY; i++) {
-		nni_thr_fini(&resolv_thrs[i]);
+	if (resolv_thrs != NULL) {
+		for (int i = 0; i < resolv_num_thr; i++) {
+			nni_thr_fini(&resolv_thrs[i]);
+		}
+		NNI_FREE_STRUCTS(resolv_thrs, resolv_num_thr);
 	}
 }
 
diff --git a/src/platform/windows/win_io.c b/src/platform/windows/win_io.c
index b08f2e4d..1e985130 100644
--- a/src/platform/windows/win_io.c
+++ b/src/platform/windows/win_io.c
@@ -1,5 +1,5 @@
 //
-// Copyright 2023 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2024 Staysail Systems, Inc. <info@staysail.tech>
 // Copyright 2018 Capitar IT Group BV <info@capitar.com>
 //
 // This software is supplied under the terms of the MIT License, a
@@ -89,26 +89,34 @@ nni_win_io_sysinit(void)
 	HANDLE h;
 	int    i;
 	int    rv;
-	int    nthr = nni_plat_ncpu() * 2;
+	int    num_thr;
+	int    max_thr;
 
-	// Limits on the thread count.  This is fairly arbitrary.
-	if (nthr < 2) {
-		nthr = 2;
-	}
 #ifndef NNG_MAX_POLLER_THREADS
 #define NNG_MAX_POLLER_THREADS 8
 #endif
-#if NNG_MAX_POLLER_THREADS > 0
-	if (nthr > NNG_MAX_POLLER_THREADS) {
-		nthr = NNG_MAX_POLLER_THREADS;
-	}
+#ifndef NNG_NUM_POLLER_THREADS
+#define NNG_NUM_POLLER_THREADS (nni_plat_ncpu())
 #endif
-	if ((win_io_thrs = NNI_ALLOC_STRUCTS(win_io_thrs, nthr)) == NULL) {
+	max_thr = (int) nni_init_get_param(
+	    NNG_INIT_MAX_POLLER_THREADS, NNG_MAX_POLLER_THREADS);
+
+	num_thr = (int) nni_init_get_param(
+	    NNG_INIT_NUM_POLLER_THREADS, NNG_NUM_POLLER_THREADS);
+
+	if ((max_thr > 0) && (num_thr > max_thr)) {
+		num_thr = max_thr;
+	}
+	if (num_thr < 1) {
+		num_thr = 1;
+	}
+	nni_init_set_effective(NNG_INIT_NUM_POLLER_THREADS, num_thr);
+	if ((win_io_thrs = NNI_ALLOC_STRUCTS(win_io_thrs, num_thr)) == NULL) {
 		return (NNG_ENOMEM);
 	}
-	win_io_nthr = nthr;
+	win_io_nthr = num_thr;
 
-	h = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, nthr);
+	h = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, num_thr);
 	if (h == NULL) {
 		return (nni_win_error(GetLastError()));
 	}
@@ -145,7 +153,7 @@ nni_win_io_sysfini(void)
 		nni_thr_fini(&win_io_thrs[i]);
 	}
 
-        NNI_FREE_STRUCTS(win_io_thrs, win_io_nthr);
+	NNI_FREE_STRUCTS(win_io_thrs, win_io_nthr);
 }
 
 #endif // NNG_PLATFORM_WINDOWS
diff --git a/src/platform/windows/win_resolv.c b/src/platform/windows/win_resolv.c
index 528da451..92c7461f 100644
--- a/src/platform/windows/win_resolv.c
+++ b/src/platform/windows/win_resolv.c
@@ -1,5 +1,5 @@
 //
-// Copyright 2021 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2024 Staysail Systems, Inc. <info@staysail.tech>
 // Copyright 2018 Capitar IT Group BV <info@capitar.com>
 //
 // This software is supplied under the terms of the MIT License, a
@@ -22,23 +22,20 @@
 // host file, WINS, or other naming services.  As a result, we just build
 // our own limited asynchronous resolver with threads.
 
-#ifndef NNG_RESOLV_CONCURRENCY
-#define NNG_RESOLV_CONCURRENCY 4
-#endif
-
-static nni_mtx  resolv_mtx = NNI_MTX_INITIALIZER;
-static nni_cv   resolv_cv = NNI_CV_INITIALIZER(&resolv_mtx);
+static nni_mtx  resolv_mtx  = NNI_MTX_INITIALIZER;
+static nni_cv   resolv_cv   = NNI_CV_INITIALIZER(&resolv_mtx);
 static bool     resolv_fini = false;
 static nni_list resolv_aios;
-static nni_thr  resolv_thrs[NNG_RESOLV_CONCURRENCY];
+static nni_thr *resolv_thrs;
+static int      resolv_num_thr;
 
 typedef struct resolv_item resolv_item;
 struct resolv_item {
 	int           family;
 	bool          passive;
-	char *        host;
-	char *        serv;
-	nni_aio *     aio;
+	char         *host;
+	char         *serv;
+	nni_aio      *aio;
 	nng_sockaddr *sa;
 };
 
@@ -159,9 +156,9 @@ resolv_task(resolv_item *item)
 
 	nni_mtx_lock(&resolv_mtx);
 	if ((probe != NULL) && (item->aio != NULL)) {
-		struct sockaddr_in * sin;
+		struct sockaddr_in  *sin;
 		struct sockaddr_in6 *sin6;
-		nni_sockaddr *       sa;
+		nni_sockaddr        *sa;
 
 		sa = item->sa;
 
@@ -270,7 +267,7 @@ resolv_worker(void *notused)
 
 	nni_mtx_lock(&resolv_mtx);
 	for (;;) {
-		nni_aio *    aio;
+		nni_aio     *aio;
 		resolv_item *item;
 		int          rv;
 
@@ -311,9 +308,9 @@ parse_ip(const char *addr, nng_sockaddr *sa, bool want_port)
 	int              rv;
 	bool             v6      = false;
 	bool             wrapped = false;
-	char *           port;
-	char *           host;
-	char *           buf;
+	char            *port;
+	char            *host;
+	char            *buf;
 	size_t           buf_len;
 
 	if (addr == NULL) {
@@ -411,7 +408,23 @@ nni_win_resolv_sysinit(void)
 	nni_aio_list_init(&resolv_aios);
 	resolv_fini = false;
 
-	for (int i = 0; i < NNG_RESOLV_CONCURRENCY; i++) {
+#ifndef NNG_RESOLV_CONCURRENCY
+#define NNG_RESOLV_CONCURRENCY 4
+#endif
+
+	resolv_num_thr = (int) nni_init_get_param(
+	    NNG_INIT_NUM_RESOLVER_THREADS, NNG_RESOLV_CONCURRENCY);
+	if (resolv_num_thr < 1) {
+		resolv_num_thr = 1;
+	}
+	// no limit on the maximum for now
+	nni_init_set_effective(NNG_INIT_NUM_RESOLVER_THREADS, resolv_num_thr);
+	resolv_thrs = NNI_ALLOC_STRUCTS(resolv_thrs, resolv_num_thr);
+	if (resolv_thrs == NULL) {
+		return (NNG_ENOMEM);
+	}
+
+	for (int i = 0; i < resolv_num_thr; i++) {
 		int rv = nni_thr_init(&resolv_thrs[i], resolv_worker, NULL);
 		if (rv != 0) {
 			nni_win_resolv_sysfini();
@@ -419,7 +432,7 @@ nni_win_resolv_sysinit(void)
 		}
 		nni_thr_set_name(&resolv_thrs[i], "nng:resolver");
 	}
-	for (int i = 0; i < NNG_RESOLV_CONCURRENCY; i++) {
+	for (int i = 0; i < resolv_num_thr; i++) {
 		nni_thr_run(&resolv_thrs[i]);
 	}
 	return (0);
@@ -432,9 +445,10 @@ nni_win_resolv_sysfini(void)
 	resolv_fini = true;
 	nni_cv_wake(&resolv_cv);
 	nni_mtx_unlock(&resolv_mtx);
-	for (int i = 0; i < NNG_RESOLV_CONCURRENCY; i++) {
+	for (int i = 0; i < resolv_num_thr; i++) {
 		nni_thr_fini(&resolv_thrs[i]);
 	}
+	NNI_FREE_STRUCTS(resolv_thrs, resolv_num_thr);
 }
 
 #endif // NNG_PLATFORM_WINDOWS