TLS: Remove support for dynamic engine initialization.

This saves some atomic lookups, and avoids possible races when the
engine is not yet initialized or being torn down.

6 files changed, 129 insertions, 154 deletions

diff --git a/src/supplemental/tls/mbedtls/CMakeLists.txt b/src/supplemental/tls/mbedtls/CMakeLists.txt
index 466f0a1b..573cf5ca 100644
--- a/src/supplemental/tls/mbedtls/CMakeLists.txt
+++ b/src/supplemental/tls/mbedtls/CMakeLists.txt
@@ -14,8 +14,6 @@ if (NNG_TLS_ENGINE STREQUAL "mbed")
         Consult a lawyer and the license files for details.
         ************************************************************")
     nng_sources(mbedtls.c)
-    nng_defines(NNG_TLS_ENGINE_INIT=nng_tls_engine_init_mbed)
-    nng_defines(NNG_TLS_ENGINE_FINI=nng_tls_engine_fini_mbed)
     nng_defines(NNG_SUPP_TLS)
     # For now Mbed TLS has PSK unconditionally
     nng_defines(NNG_SUPP_TLS_PSK)
diff --git a/src/supplemental/tls/mbedtls/mbedtls.c b/src/supplemental/tls/mbedtls/mbedtls.c
index 825fe21a..239f29fa 100644
--- a/src/supplemental/tls/mbedtls/mbedtls.c
+++ b/src/supplemental/tls/mbedtls/mbedtls.c
@@ -761,6 +761,48 @@ config_version(nng_tls_engine_config *cfg, nng_tls_version min_ver,
 	return (0);
 }
 
+static nng_err
+tls_engine_init(void)
+{
+	int rv;
+
+#ifdef MBEDTLS_PSA_CRYPTO_C
+	rv = psa_crypto_init();
+	if (rv != 0) {
+		tls_log_err(
+		    "NNG-TLS-INIT", "Failed initializing PSA crypto", rv);
+		return (tls_mk_err(rv));
+	}
+#endif
+	// Uncomment the following to have noisy debug from mbedTLS.
+	// This may be useful when trying to debug failures.
+	// mbedtls_debug_set_threshold(9);
+
+	mbedtls_ssl_cookie_init(&mbed_ssl_cookie_ctx);
+	rv = mbedtls_ssl_cookie_setup(&mbed_ssl_cookie_ctx, tls_random, NULL);
+	if (rv != 0) {
+		tls_log_err("NNG_TLS_INIT",
+		    "Failed initializing SSL cookie system", rv);
+		return (tls_mk_err(rv));
+	}
+	return (NNG_OK);
+}
+
+static void
+tls_engine_fini(void)
+{
+	mbedtls_ssl_cookie_free(&mbed_ssl_cookie_ctx);
+#ifdef MBEDTLS_PSA_CRYPTO_C
+	mbedtls_psa_crypto_free();
+#endif
+}
+
+static bool
+fips_mode(void)
+{
+	return (false);
+}
+
 static nng_tls_engine_config_ops config_ops = {
 	.init     = config_init,
 	.fini     = config_fini,
@@ -786,47 +828,13 @@ static nng_tls_engine_conn_ops conn_ops = {
 	.peer_alt_names = conn_peer_alt_names,
 };
 
-static nng_tls_engine tls_engine_mbed = {
+nng_tls_engine nng_tls_engine_ops = {
 	.version     = NNG_TLS_ENGINE_VERSION,
 	.config_ops  = &config_ops,
 	.conn_ops    = &conn_ops,
 	.name        = "mbed",
 	.description = MBEDTLS_VERSION_STRING_FULL,
-	.fips_mode   = false,
+	.init        = tls_engine_init,
+	.fini        = tls_engine_fini,
+	.fips_mode   = fips_mode,
 };
-
-int
-nng_tls_engine_init_mbed(void)
-{
-	int rv;
-
-#ifdef MBEDTLS_PSA_CRYPTO_C
-	rv = psa_crypto_init();
-	if (rv != 0) {
-		tls_log_err(
-		    "NNG-TLS-INIT", "Failed initializing PSA crypto", rv);
-		return (rv);
-	}
-#endif
-	// Uncomment the following to have noisy debug from mbedTLS.
-	// This may be useful when trying to debug failures.
-	// mbedtls_debug_set_threshold(9);
-
-	mbedtls_ssl_cookie_init(&mbed_ssl_cookie_ctx);
-	rv = mbedtls_ssl_cookie_setup(&mbed_ssl_cookie_ctx, tls_random, NULL);
-
-	if (rv == 0) {
-		rv = nng_tls_engine_register(&tls_engine_mbed);
-	}
-
-	return (rv);
-}
-
-void
-nng_tls_engine_fini_mbed(void)
-{
-	mbedtls_ssl_cookie_free(&mbed_ssl_cookie_ctx);
-#ifdef MBEDTLS_PSA_CRYPTO_C
-	mbedtls_psa_crypto_free();
-#endif
-}
diff --git a/src/supplemental/tls/tls_common.c b/src/supplemental/tls/tls_common.c
index dcaf6e49..13b40e6e 100644
--- a/src/supplemental/tls/tls_common.c
+++ b/src/supplemental/tls/tls_common.c
@@ -37,8 +37,6 @@
 // parts of TLS support that are invariant relative to different TLS
 // libraries, such as dialer and listener support.
 
-static nni_atomic_ptr tls_engine;
-
 static void tls_bio_send_cb(void *arg);
 static void tls_bio_recv_cb(void *arg);
 static void tls_do_send(nni_tls_conn *);
@@ -188,6 +186,8 @@ nni_tls_fini(nni_tls_conn *conn)
 	conn->ops.fini((void *) (conn + 1));
 	nni_aio_fini(&conn->bio_send);
 	nni_aio_fini(&conn->bio_recv);
+	nni_mtx_lock(&conn->lock);
+	nni_mtx_unlock(&conn->lock);
 	if (conn->cfg != NULL) {
 		nng_tls_config_free(conn->cfg); // this drops our hold on it
 	}
@@ -369,8 +369,11 @@ tls_do_send(nni_tls_conn *conn)
 nng_err
 nni_tls_run(nni_tls_conn *conn)
 {
-	nni_aio *aio;
-	nng_err  rv;
+	nni_aio            *aio;
+	nng_err             rv;
+	nni_aio_completions compq;
+
+	nni_aio_completions_init(&compq);
 	nni_mtx_lock(&conn->lock);
 	switch ((rv = tls_handshake(conn))) {
 	case NNG_OK:
@@ -383,11 +386,12 @@ nni_tls_run(nni_tls_conn *conn)
 		while (((aio = nni_list_first(&conn->send_queue)) != NULL) ||
 		    ((aio = nni_list_first(&conn->recv_queue)) != NULL)) {
 			nni_aio_list_remove(aio);
-			nni_aio_finish_error(aio, rv);
+			nni_aio_completions_add(&compq, aio, rv, 0);
 		}
 		break;
 	}
 	nni_mtx_unlock(&conn->lock);
+	nni_aio_completions_run(&compq);
 	return (rv);
 }
 
@@ -749,15 +753,9 @@ int
 nng_tls_config_alloc(nng_tls_config **cfg_p, nng_tls_mode mode)
 {
 	nng_tls_config       *cfg;
-	const nng_tls_engine *eng;
 	size_t                size;
 	int                   rv;
-
-	eng = nni_atomic_get_ptr(&tls_engine);
-
-	if (eng == NULL) {
-		return (NNG_ENOTSUP);
-	}
+	const nng_tls_engine *eng = &nng_tls_engine_ops;
 
 	size = NNI_ALIGN_UP(sizeof(*cfg)) + eng->config_ops->size;
 
@@ -806,90 +804,48 @@ nng_tls_config_hold(nng_tls_config *cfg)
 const char *
 nng_tls_engine_name(void)
 {
-	const nng_tls_engine *eng;
-
-	eng = nni_atomic_get_ptr(&tls_engine);
-
-	return (eng == NULL ? "none" : eng->name);
+	const nng_tls_engine *eng = &nng_tls_engine_ops;
+	return (eng->name);
 }
 
 const char *
 nng_tls_engine_description(void)
 {
-	const nng_tls_engine *eng;
-
-	eng = nni_atomic_get_ptr(&tls_engine);
-
-	return (eng == NULL ? "" : eng->description);
+	const nng_tls_engine *eng = &nng_tls_engine_ops;
+	return (eng->description);
 }
 
 bool
 nng_tls_engine_fips_mode(void)
 {
-	const nng_tls_engine *eng;
-
-	eng = nni_atomic_get_ptr(&tls_engine);
-
-	return (eng == NULL ? false : eng->fips_mode);
-}
-
-int
-nng_tls_engine_register(const nng_tls_engine *engine)
-{
-	if (engine->version != NNG_TLS_ENGINE_VERSION) {
-		nng_log_err("NNG-TLS-ENGINE-VER",
-		    "TLS Engine version mismatch: %d != %d", engine->version,
-		    NNG_TLS_ENGINE_VERSION);
-		return (NNG_ENOTSUP);
-	}
-	nng_log_info("NNG-TLS-INFO", "TLS Engine: %s", engine->description);
-	nni_atomic_set_ptr(&tls_engine, (void *) engine);
-	return (0);
+	const nng_tls_engine *eng = &nng_tls_engine_ops;
+	return (eng->fips_mode());
 }
 
 size_t
 nni_tls_engine_conn_size(void)
 {
-	const nng_tls_engine *eng;
-
-	eng = nni_atomic_get_ptr(&tls_engine);
-
-	return (eng == NULL ? 0 : eng->conn_ops->size);
-}
-
-#ifdef NNG_TLS_ENGINE_INIT
-extern int NNG_TLS_ENGINE_INIT(void);
-#else
-static int
-NNG_TLS_ENGINE_INIT(void)
-{
-	return (0);
-}
-#endif
-
-#ifdef NNG_TLS_ENGINE_FINI
-extern void NNG_TLS_ENGINE_FINI(void);
-#else
-static void
-NNG_TLS_ENGINE_FINI(void)
-{
+	const nng_tls_engine *eng = &nng_tls_engine_ops;
+	return (eng->conn_ops->size);
 }
-#endif
 
 int
 nni_tls_sys_init(void)
 {
-	int rv;
-
-	rv = NNG_TLS_ENGINE_INIT();
-	if (rv != 0) {
-		return (rv);
+	const nng_tls_engine *eng = &nng_tls_engine_ops;
+	if (eng->version != NNG_TLS_ENGINE_VERSION) {
+		nng_log_err("NNG-TLS-ENGINE-VER",
+		    "TLS Engine version mismatch: %d != %d", eng->version,
+		    NNG_TLS_ENGINE_VERSION);
+		return (NNG_ENOTSUP);
 	}
-	return (0);
+	nng_log_info("NNG-TLS-INFO", "TLS Engine: %s", eng->description);
+	return (eng->init());
 }
 
 void
 nni_tls_sys_fini(void)
 {
-	NNG_TLS_ENGINE_FINI();
+	const nng_tls_engine *eng = &nng_tls_engine_ops;
+	eng->fini();
 }
diff --git a/src/supplemental/tls/tls_engine.h b/src/supplemental/tls/tls_engine.h
index 66d40826..431048df 100644
--- a/src/supplemental/tls/tls_engine.h
+++ b/src/supplemental/tls/tls_engine.h
@@ -174,7 +174,8 @@ typedef enum nng_tls_engine_version_e {
 	NNG_TLS_ENGINE_V0      = 0,
 	NNG_TLS_ENGINE_V1      = 1, // adds FIPS, TLS 1.3 support
 	NNG_TLS_ENGINE_V2      = 2, // adds PSK support
-	NNG_TLS_ENGINE_VERSION = NNG_TLS_ENGINE_V2,
+	NNG_TLS_ENGINE_V3      = 3, // refactored API
+	NNG_TLS_ENGINE_VERSION = NNG_TLS_ENGINE_V3,
 } nng_tls_engine_version;
 
 typedef struct nng_tls_engine_s {
@@ -202,10 +203,16 @@ typedef struct nng_tls_engine_s {
 	// It is expected that this will be enabled either at compile
 	// time, or via environment variables at engine initialization.
 	// FIPS mode cannot be changed once the engine is registered.
-	bool fips_mode;
+	bool (*fips_mode)(void);
+
+	nng_err (*init)(void);
+
+	void (*fini)(void);
+
 } nng_tls_engine;
 
-extern int nng_tls_engine_register(const nng_tls_engine *);
+// Implementation supplies this ops vector.
+extern nng_tls_engine nng_tls_engine_ops;
 
 // nng_tls_engine_send is called by the engine to send data over the
 // underlying connection.  It returns zero on success, NNG_EAGAIN if
diff --git a/src/supplemental/tls/wolfssl/CMakeLists.txt b/src/supplemental/tls/wolfssl/CMakeLists.txt
index 7b129d9c..762757a3 100644
--- a/src/supplemental/tls/wolfssl/CMakeLists.txt
+++ b/src/supplemental/tls/wolfssl/CMakeLists.txt
@@ -69,8 +69,6 @@ if (NNG_TLS_ENGINE STREQUAL "wolf")
         message(STATUS "wolfSSL configured without pre-shared key (PSK) support.")
     endif()
 
-    nng_defines(NNG_TLS_ENGINE_INIT=nng_tls_engine_init_wolf)
-    nng_defines(NNG_TLS_ENGINE_FINI=nng_tls_engine_fini_wolf)
     nng_defines(NNG_SUPP_TLS)
     nng_defines(NNG_TLS_ENGINE_WOLFSSL)
 endif ()
diff --git a/src/supplemental/tls/wolfssl/wolfssl.c b/src/supplemental/tls/wolfssl/wolfssl.c
index 3eab0ada..f7f3732e 100644
--- a/src/supplemental/tls/wolfssl/wolfssl.c
+++ b/src/supplemental/tls/wolfssl/wolfssl.c
@@ -625,38 +625,6 @@ wolf_config_version(nng_tls_engine_config *cfg, nng_tls_version min_ver,
 	return (0);
 }
 
-static nng_tls_engine_config_ops wolf_config_ops = {
-	.init     = wolf_config_init,
-	.fini     = wolf_config_fini,
-	.size     = sizeof(nng_tls_engine_config),
-	.auth     = wolf_config_auth_mode,
-	.ca_chain = wolf_config_ca_chain,
-	.own_cert = wolf_config_own_cert,
-	.server   = wolf_config_server,
-	.psk      = wolf_config_psk,
-	.version  = wolf_config_version,
-};
-
-static nng_tls_engine_conn_ops wolf_conn_ops = {
-	.size      = sizeof(nng_tls_engine_conn),
-	.init      = wolf_conn_init,
-	.fini      = wolf_conn_fini,
-	.close     = wolf_conn_close,
-	.recv      = wolf_conn_recv,
-	.send      = wolf_conn_send,
-	.handshake = wolf_conn_handshake,
-	.verified  = wolf_conn_verified,
-};
-
-static nng_tls_engine wolf_engine = {
-	.version     = NNG_TLS_ENGINE_VERSION,
-	.config_ops  = &wolf_config_ops,
-	.conn_ops    = &wolf_conn_ops,
-	.name        = "wolf",
-	.description = "wolfSSL " LIBWOLFSSL_VERSION_STRING,
-	.fips_mode   = false, // commercial users only
-};
-
 static void
 wolf_logging_cb(const int level, const char *msg)
 {
@@ -679,8 +647,8 @@ wolf_logging_cb(const int level, const char *msg)
 	}
 }
 
-int
-nng_tls_engine_init_wolf(void)
+static nng_err
+tls_engine_init(void)
 {
 	switch (wolfSSL_Init()) {
 	case WOLFSSL_SUCCESS:
@@ -694,11 +662,51 @@ nng_tls_engine_init_wolf(void)
 	// Uncomment for full debug (also WolfSSL needs to be a debug build)
 	//
 	// wolfSSL_Debugging_ON();
-	return (nng_tls_engine_register(&wolf_engine));
+	return (NNG_OK);
 }
 
-void
-nng_tls_engine_fini_wolf(void)
+static void
+tls_engine_fini(void)
 {
 	(void) wolfSSL_Cleanup();
 }
+
+static bool
+fips_mode(void)
+{
+	return (false); // TODO: Support FIPS mode.
+}
+
+static nng_tls_engine_config_ops wolf_config_ops = {
+	.init     = wolf_config_init,
+	.fini     = wolf_config_fini,
+	.size     = sizeof(nng_tls_engine_config),
+	.auth     = wolf_config_auth_mode,
+	.ca_chain = wolf_config_ca_chain,
+	.own_cert = wolf_config_own_cert,
+	.server   = wolf_config_server,
+	.psk      = wolf_config_psk,
+	.version  = wolf_config_version,
+};
+
+static nng_tls_engine_conn_ops wolf_conn_ops = {
+	.size      = sizeof(nng_tls_engine_conn),
+	.init      = wolf_conn_init,
+	.fini      = wolf_conn_fini,
+	.close     = wolf_conn_close,
+	.recv      = wolf_conn_recv,
+	.send      = wolf_conn_send,
+	.handshake = wolf_conn_handshake,
+	.verified  = wolf_conn_verified,
+};
+
+nng_tls_engine nng_tls_engine_ops = {
+	.version     = NNG_TLS_ENGINE_VERSION,
+	.config_ops  = &wolf_config_ops,
+	.conn_ops    = &wolf_conn_ops,
+	.name        = "wolf",
+	.description = "wolfSSL " LIBWOLFSSL_VERSION_STRING,
+	.init        = tls_engine_init,
+	.fini        = tls_engine_fini,
+	.fips_mode   = fips_mode,
+};