fixes #1572 nng creates too many threads

This further limits some of the thread counts, but principally it
offers a new runtime facility, nng_init_set_parameter(), which can
be used to set certain runtime parameters on the number of threads,
provided it is called before the rest of application start up.

This facility is quite intentionally "undocumented", at least for now,
as we want to limit our commitment to it.  Still this should be helpful
for applications that need to reduce the number of threads that are
created.

6 files changed, 295 insertions, 23 deletions

diff --git a/src/core/CMakeLists.txt b/src/core/CMakeLists.txt
index 9e5a6bec..009d6bb0 100644
--- a/src/core/CMakeLists.txt
+++ b/src/core/CMakeLists.txt
@@ -1,5 +1,5 @@
 #
-# Copyright 2021 Staysail Systems, Inc. <info@staystail.tech>
+# Copyright 2024 Staysail Systems, Inc. <info@staystail.tech>
 #
 # This software is supplied under the terms of the MIT License, a
 # copy of which should be located in the distribution where this
@@ -78,6 +78,7 @@ nng_test(aio_test)
 nng_test(buf_size_test)
 nng_test(errors_test)
 nng_test(id_test)
+nng_test(init_test)
 nng_test(list_test)
 nng_test(message_test)
 nng_test(reconnect_test)
diff --git a/src/core/aio.c b/src/core/aio.c
index 3d4a56c1..084795bd 100644
--- a/src/core/aio.c
+++ b/src/core/aio.c
@@ -1,5 +1,5 @@
 //
-// Copyright 2023 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2024 Staysail Systems, Inc. <info@staysail.tech>
 // Copyright 2018 Capitar IT Group BV <info@capitar.com>
 //
 // This software is supplied under the terms of the MIT License, a
@@ -843,18 +843,29 @@ int
 nni_aio_sys_init(void)
 {
 	int num_thr;
+	int max_thr;
 
-#ifndef NNG_NUM_EXPIRE_THREADS
-	num_thr = nni_plat_ncpu();
-#else
-	num_thr = NNG_NUM_EXPIRE_THREADS;
+#ifndef NNG_MAX_EXPIRE_THREADS
+#define NNG_MAX_EXPIRE_THREADS 8
 #endif
-#if NNG_MAX_EXPIRE_THREADS > 0
-	if (num_thr > NNG_MAX_EXPIRE_THREADS) {
-		num_thr = NNG_MAX_EXPIRE_THREADS;
-	}
+
+#ifndef NNG_NUM_EXPIRE_THREADS
+#define NNG_NUM_EXPIRE_THREADS (nni_plat_ncpu())
 #endif
 
+	max_thr = (int) nni_init_get_param(
+	    NNG_INIT_MAX_EXPIRE_THREADS, NNG_MAX_EXPIRE_THREADS);
+
+	num_thr = (int) nni_init_get_param(
+	    NNG_INIT_NUM_EXPIRE_THREADS, NNG_NUM_EXPIRE_THREADS);
+
+	if ((max_thr > 0) && (num_thr > max_thr)) {
+		num_thr = max_thr;
+	}
+	if (num_thr < 1) {
+		num_thr = 1;
+	}
+	nni_init_set_effective(NNG_INIT_NUM_EXPIRE_THREADS, num_thr);
 	nni_aio_expire_q_list =
 	    nni_zalloc(sizeof(nni_aio_expire_q *) * num_thr);
 	nni_aio_expire_q_cnt = num_thr;
diff --git a/src/core/init.c b/src/core/init.c
index f2195bcb..8f2e1056 100644
--- a/src/core/init.c
+++ b/src/core/init.c
@@ -1,5 +1,5 @@
 //
-// Copyright 2023 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2024 Staysail Systems, Inc. <info@staysail.tech>
 // Copyright 2018 Capitar IT Group BV <info@capitar.com>
 //
 // This software is supplied under the terms of the MIT License, a
@@ -54,10 +54,107 @@ nni_init(void)
 	return (nni_plat_init(nni_init_helper));
 }
 
+// accessing the list of parameters
+typedef struct nni_init_param {
+	nni_list_node      node;
+	nng_init_parameter param;
+	uint64_t           value;
+#ifdef NNG_TEST_LIB
+	uint64_t           effective;
+#endif
+} nni_init_param;
+
+static nni_list nni_init_params =
+    NNI_LIST_INITIALIZER(nni_init_params, nni_init_param, node);
+
+void
+nni_init_set_param(nng_init_parameter p, uint64_t value)
+{
+	if (nni_inited) {
+		// this is paranoia -- if some library code started already
+		// then we cannot safely change parameters, and modifying the
+		// list is not thread safe.
+		return;
+	}
+	nni_init_param *item;
+	NNI_LIST_FOREACH (&nni_init_params, item) {
+		if (item->param == p) {
+			item->value = value;
+			return;
+		}
+	}
+	if ((item = NNI_ALLOC_STRUCT(item)) != NULL) {
+		item->param = p;
+		item->value = value;
+		nni_list_append(&nni_init_params, item);
+	}
+}
+
+uint64_t
+nni_init_get_param(nng_init_parameter p, uint64_t default_value)
+{
+	nni_init_param *item;
+	NNI_LIST_FOREACH (&nni_init_params, item) {
+		if (item->param == p) {
+			return (item->value);
+		}
+	}
+	return (default_value);
+}
+
+void
+nni_init_set_effective(nng_init_parameter p, uint64_t value)
+{
+#ifdef NNG_TEST_LIB
+	nni_init_param *item;
+	NNI_LIST_FOREACH (&nni_init_params, item) {
+		if (item->param == p) {
+			item->effective = value;
+			return;
+		}
+	}
+	if ((item = NNI_ALLOC_STRUCT(item)) != NULL) {
+		item->param = p;
+		item->effective = value;
+		nni_list_append(&nni_init_params, item);
+	}
+#else
+	NNI_ARG_UNUSED(p);
+	NNI_ARG_UNUSED(value);
+#endif
+}
+
+#ifdef NNG_TEST_LIB
+uint64_t
+nni_init_get_effective(nng_init_parameter p)
+{
+	nni_init_param *item;
+	NNI_LIST_FOREACH (&nni_init_params, item) {
+		if (item->param == p) {
+			return (item->effective);
+		}
+	}
+	return ((uint64_t)-1);
+}
+#endif
+
+
+static void
+nni_init_params_fini(void)
+{
+	nni_init_param *item;
+	while ((item = nni_list_first(&nni_init_params)) != NULL) {
+		nni_list_remove(&nni_init_params, item);
+		NNI_FREE_STRUCT(item);
+	}
+}
+
 void
 nni_fini(void)
 {
 	if (!nni_inited) {
+		// make sure we discard parameters even if we didn't startup
+		nni_init_params_fini();
 		return;
 	}
 	nni_sp_tran_sys_fini();
@@ -67,6 +164,7 @@ nni_fini(void)
 	nni_taskq_sys_fini();
 	nni_reap_sys_fini(); // must be before timer and aio (expire)
 	nni_id_map_sys_fini();
+	nni_init_params_fini();
 
 	nni_plat_fini();
 	nni_inited = false;
diff --git a/src/core/init.h b/src/core/init.h
index 4340b15b..d20cf046 100644
--- a/src/core/init.h
+++ b/src/core/init.h
@@ -1,7 +1,6 @@
 //
-// Copyright 2017 Garrett D'Amore <garrett@damore.org>
+// Copyright 2024 Staysail Systems, Inc. <info@staysail.tech>
 // Copyright 2017 Capitar IT Group BV <info@capitar.com>
-// Copyright 2017 Staysail Systems, Inc. <info@staysail.tech>
 //
 // This software is supplied under the terms of the MIT License, a
 // copy of which should be located in the distribution where this
@@ -23,4 +22,15 @@ int nni_init(void);
 // that all resources used by the library are released back to the system.
 void nni_fini(void);
 
+// nni_init_param is used by applications (via nng_init_param) to configure
+// some tunable settings at runtime.  It must be called before any other NNG
+// functions are called, in order to have any effect at all.
+void nni_init_set_param(nng_init_parameter, uint64_t value);
+
+// subsystems can call this to obtain a parameter value.
+uint64_t nni_init_get_param(nng_init_parameter parameter, uint64_t default_value);
+
+// subsystems can set this to facilitate tests (only used in test code)
+void nni_init_set_effective(nng_init_parameter p, uint64_t value);
+
 #endif // CORE_INIT_H
diff --git a/src/core/init_test.c b/src/core/init_test.c
new file mode 100644
index 00000000..9b9e26b4
--- /dev/null
+++ b/src/core/init_test.c
@@ -0,0 +1,140 @@
+//
+// Copyright 2024 Staysail Systems, Inc. <info@staysail.tech>
+//
+// This software is supplied under the terms of the MIT License, a
+// copy of which should be located in the distribution where this
+// file was obtained (LICENSE.txt).  A copy of the license may also be
+// found online at https://opensource.org/licenses/MIT.
+//
+
+#include <nuts.h>
+
+uint64_t nni_init_get_param(
+    nng_init_parameter parameter, uint64_t default_value);
+uint64_t nni_init_get_effective(nng_init_parameter p);
+void     nni_init_set_effective(nng_init_parameter p, uint64_t);
+
+void
+test_init_param(void)
+{
+	NUTS_ASSERT(nni_init_get_param(NNG_INIT_PARAMETER_NONE, 456) == 456);
+	nng_init_set_parameter(NNG_INIT_PARAMETER_NONE, 123);
+	NUTS_ASSERT(nni_init_get_param(NNG_INIT_PARAMETER_NONE, 567) == 123);
+	nni_init_set_effective(NNG_INIT_PARAMETER_NONE, 124);
+	NUTS_ASSERT(nni_init_get_effective(NNG_INIT_PARAMETER_NONE) == 124);
+	NUTS_ASSERT(nni_init_get_param(NNG_INIT_PARAMETER_NONE, 567) == 123);
+	nng_fini();
+	NUTS_ASSERT(nni_init_get_param(NNG_INIT_PARAMETER_NONE, 567) == 567);
+}
+
+void
+test_set_effective(void)
+{
+	nni_init_set_effective(NNG_INIT_PARAMETER_NONE, 999);
+	NUTS_ASSERT(nni_init_get_param(NNG_INIT_PARAMETER_NONE, 0) == 0);
+	NUTS_ASSERT(nni_init_get_effective(NNG_INIT_PARAMETER_NONE) == 999);
+	nng_fini();
+}
+
+void
+test_init_zero_resolvers(void)
+{
+	nng_socket s;
+	nng_init_set_parameter(NNG_INIT_NUM_RESOLVER_THREADS, 0);
+	NUTS_OPEN(s);
+	NUTS_CLOSE(s);
+	NUTS_ASSERT(
+	    nni_init_get_effective(NNG_INIT_NUM_RESOLVER_THREADS) == 1);
+	nng_fini();
+}
+
+void
+test_init_one_task_thread(void)
+{
+	nng_socket s;
+	nng_init_set_parameter(NNG_INIT_NUM_TASK_THREADS, 0);
+	nng_init_set_parameter(NNG_INIT_MAX_TASK_THREADS, 1);
+	NUTS_OPEN(s);
+	NUTS_CLOSE(s);
+	NUTS_ASSERT(nni_init_get_effective(NNG_INIT_NUM_TASK_THREADS) == 2);
+	nng_fini();
+}
+
+void
+test_init_too_many_task_threads(void)
+{
+	nng_socket s;
+	nng_init_set_parameter(NNG_INIT_NUM_TASK_THREADS, 256);
+	nng_init_set_parameter(NNG_INIT_MAX_TASK_THREADS, 4);
+	NUTS_OPEN(s);
+	NUTS_CLOSE(s);
+	NUTS_ASSERT(nni_init_get_effective(NNG_INIT_NUM_TASK_THREADS) == 4);
+	nng_fini();
+}
+
+void
+test_init_no_expire_thread(void)
+{
+	nng_socket s;
+	nng_init_set_parameter(NNG_INIT_NUM_EXPIRE_THREADS, 0);
+	nng_init_set_parameter(NNG_INIT_MAX_EXPIRE_THREADS, 0);
+	NUTS_OPEN(s);
+	NUTS_CLOSE(s);
+	NUTS_ASSERT(nni_init_get_effective(NNG_INIT_NUM_EXPIRE_THREADS) == 1);
+	nng_fini();
+}
+
+void
+test_init_too_many_expire_threads(void)
+{
+	nng_socket s;
+	nng_init_set_parameter(NNG_INIT_NUM_EXPIRE_THREADS, 256);
+	nng_init_set_parameter(NNG_INIT_MAX_EXPIRE_THREADS, 2);
+	NUTS_OPEN(s);
+	NUTS_CLOSE(s);
+	NUTS_ASSERT(nni_init_get_effective(NNG_INIT_NUM_EXPIRE_THREADS) == 2);
+	nng_fini();
+}
+
+// poller tuning only supported on Windows right now
+#ifdef NNG_PLATFORM_WINDOWS
+void
+test_init_poller_no_threads(void)
+{
+	nng_socket s;
+	nng_init_set_parameter(NNG_INIT_NUM_POLLER_THREADS, 0);
+	nng_init_set_parameter(NNG_INIT_MAX_POLLER_THREADS, 0);
+	NUTS_OPEN(s);
+	NUTS_CLOSE(s);
+	NUTS_ASSERT(nni_init_get_effective(NNG_INIT_NUM_POLLER_THREADS) == 1);
+	nng_fini();
+}
+
+void
+test_init_too_many_poller_threads(void)
+{
+	nng_socket s;
+	nng_init_set_parameter(NNG_INIT_NUM_POLLER_THREADS, 256);
+	nng_init_set_parameter(NNG_INIT_MAX_POLLER_THREADS, 2);
+	NUTS_OPEN(s);
+	NUTS_CLOSE(s);
+	NUTS_ASSERT(nni_init_get_effective(NNG_INIT_NUM_POLLER_THREADS) == 2);
+	nng_fini();
+}
+#endif
+
+NUTS_TESTS = {
+	{ "init parameter", test_init_param },
+	{ "init set effective", test_set_effective },
+	{ "init zero resolvers", test_init_zero_resolvers },
+	{ "init one task thread", test_init_one_task_thread },
+	{ "init too many task threads", test_init_too_many_task_threads },
+	{ "init no expire thread", test_init_no_expire_thread },
+	{ "init too many expire threads", test_init_too_many_expire_threads },
+#ifdef NNG_PLATFORM_WINDOWS
+	{ "init no poller thread", test_init_poller_no_threads },
+	{ "init too many poller threads", test_init_too_many_poller_threads },
+#endif
+
+	{ NULL, NULL },
+};
\ No newline at end of file
diff --git a/src/core/taskq.c b/src/core/taskq.c
index d914093b..09886596 100644
--- a/src/core/taskq.c
+++ b/src/core/taskq.c
@@ -1,5 +1,5 @@
 //
-// Copyright 2022 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2024 Staysail Systems, Inc. <info@staysail.tech>
 // Copyright 2018 Capitar IT Group BV <info@capitar.com>
 //
 // This software is supplied under the terms of the MIT License, a
@@ -245,20 +245,32 @@ nni_task_fini(nni_task *task)
 int
 nni_taskq_sys_init(void)
 {
-	int nthrs;
+	int num_thr;
+	int max_thr;
 
 #ifndef NNG_NUM_TASKQ_THREADS
-	nthrs = nni_plat_ncpu() * 2;
-#else
-	nthrs = NNG_NUM_TASKQ_THREADS;
+#define NNG_NUM_TASKQ_THREADS (nni_plat_ncpu() * 2)
 #endif
-#if NNG_MAX_TASKQ_THREADS > 0
-	if (nthrs > NNG_MAX_TASKQ_THREADS) {
-		nthrs = NNG_MAX_TASKQ_THREADS;
-	}
+
+#ifndef NNG_MAX_TASKQ_THREADS
+#define NNG_MAX_TASKQ_THREADS 16
 #endif
 
-	return (nni_taskq_init(&nni_taskq_systq, nthrs));
+	max_thr = (int) nni_init_get_param(
+	    NNG_INIT_MAX_TASK_THREADS, NNG_MAX_TASKQ_THREADS);
+
+	num_thr = (int) nni_init_get_param(
+	    NNG_INIT_NUM_TASK_THREADS, NNG_NUM_TASKQ_THREADS);
+
+	if ((max_thr > 0) && (num_thr > max_thr)) {
+		num_thr = max_thr;
+	}
+	if (num_thr < 2) {
+		num_thr = 2;
+	}
+	nni_init_set_effective(NNG_INIT_NUM_TASK_THREADS, num_thr);
+
+	return (nni_taskq_init(&nni_taskq_systq, num_thr));
 }
 
 void