fixes #523 dialers could support multiple outstanding dial requests

fixes #179 DNS resolution should be done at connect time
fixes #586 Windows IO completion port work could be better
fixes #339 Windows iocp could use synchronous completions
fixes #280 TCP abstraction improvements

This is a rather monstrous set of changes, which refactors TCP, and
the underlying Windows I/O completion path logic, in order to obtain
a cleaner, simpler API, with support for asynchronous DNS lookups performed
on connect rather than initialization time, the ability to have multiple
connects or accepts pending, as well as fewer extraneous function calls.

The Windows code also benefits from greatly reduced context switching,
fewer lock operations performed, and a reduced number of system calls
on the hot code path.  (We use automatic event resetting instead of manual.)

Some dead code was removed as well, and a few potential edge case leaks
on failure paths (in the websocket code) were plugged.

Note that all TCP based transports benefit from this work. The IPC code
on Windows still uses the legacy IOCP for now, as does the UDP code (used
for ZeroTier.)  We will be converting those soon too.

3 files changed, 22 insertions, 21 deletions

diff --git a/src/supplemental/tls/mbedtls/tls.c b/src/supplemental/tls/mbedtls/tls.c
index cdd226cd..0f4f67cc 100644
--- a/src/supplemental/tls/mbedtls/tls.c
+++ b/src/supplemental/tls/mbedtls/tls.c
@@ -59,7 +59,7 @@ typedef struct nni_tls_certkey {
 } nni_tls_certkey;
 
 struct nni_tls {
-	nni_plat_tcp_pipe * tcp;
+	nni_tcp_conn *      tcp;
 	mbedtls_ssl_context ctx;
 	nng_tls_config *    cfg; // kept so we can release it
 	nni_mtx             lk;
@@ -254,14 +254,14 @@ nni_tls_fini(nni_tls *tp)
 {
 	// Shut it all down first.
 	if (tp->tcp) {
-		nni_plat_tcp_pipe_close(tp->tcp);
+		nni_tcp_conn_close(tp->tcp);
 	}
 	nni_aio_stop(tp->tcp_send);
 	nni_aio_stop(tp->tcp_recv);
 
 	// And finalize / free everything.
 	if (tp->tcp) {
-		nni_plat_tcp_pipe_fini(tp->tcp);
+		nni_tcp_conn_fini(tp->tcp);
 	}
 	nni_aio_fini(tp->tcp_send);
 	nni_aio_fini(tp->tcp_recv);
@@ -306,7 +306,7 @@ nni_tls_mkerr(int err)
 }
 
 int
-nni_tls_init(nni_tls **tpp, nng_tls_config *cfg, nni_plat_tcp_pipe *tcp)
+nni_tls_init(nni_tls **tpp, nng_tls_config *cfg, nni_tcp_conn *tcp)
 {
 	nni_tls *tp;
 	int      rv;
@@ -314,7 +314,7 @@ nni_tls_init(nni_tls **tpp, nng_tls_config *cfg, nni_plat_tcp_pipe *tcp)
 	// During the handshake, disable Nagle to shorten the
 	// negotiation.  Once things are set up the caller can
 	// re-enable Nagle if so desired.
-	(void) nni_plat_tcp_pipe_set_nodelay(tcp, true);
+	(void) nni_tcp_conn_set_nodelay(tcp, true);
 
 	if ((tp = NNI_ALLOC_STRUCT(tp)) == NULL) {
 		return (NNG_ENOMEM);
@@ -387,7 +387,7 @@ nni_tls_fail(nni_tls *tp, int rv)
 {
 	nni_aio *aio;
 	tp->tls_closed = true;
-	nni_plat_tcp_pipe_close(tp->tcp);
+	nni_tcp_conn_close(tp->tcp);
 	tp->tcp_closed = true;
 	while ((aio = nni_list_first(&tp->recvs)) != NULL) {
 		nni_list_remove(&tp->recvs, aio);
@@ -408,7 +408,7 @@ nni_tls_send_cb(void *ctx)
 
 	nni_mtx_lock(&tp->lk);
 	if (nni_aio_result(aio) != 0) {
-		nni_plat_tcp_pipe_close(tp->tcp);
+		nni_tcp_conn_close(tp->tcp);
 		tp->tcp_closed = true;
 	} else {
 		size_t n = nni_aio_count(aio);
@@ -421,7 +421,7 @@ nni_tls_send_cb(void *ctx)
 			iov.iov_len = tp->sendlen;
 			nni_aio_set_iov(aio, 1, &iov);
 			nni_aio_set_timeout(aio, NNG_DURATION_INFINITE);
-			nni_plat_tcp_pipe_send(tp->tcp, aio);
+			nni_tcp_conn_send(tp->tcp, aio);
 			nni_mtx_unlock(&tp->lk);
 			return;
 		}
@@ -460,7 +460,7 @@ nni_tls_recv_start(nni_tls *tp)
 	iov.iov_len = NNG_TLS_MAX_RECV_SIZE;
 	nni_aio_set_iov(aio, 1, &iov);
 	nni_aio_set_timeout(tp->tcp_recv, NNG_DURATION_INFINITE);
-	nni_plat_tcp_pipe_recv(tp->tcp, aio);
+	nni_tcp_conn_recv(tp->tcp, aio);
 }
 
 static void
@@ -474,7 +474,7 @@ nni_tls_recv_cb(void *ctx)
 	if (nni_aio_result(aio) != 0) {
 		// Close the underlying TCP channel, but permit data we
 		// already received to continue to be received.
-		nni_plat_tcp_pipe_close(tp->tcp);
+		nni_tcp_conn_close(tp->tcp);
 		tp->tcp_closed = true;
 	} else {
 		NNI_ASSERT(tp->recvlen == 0);
@@ -531,7 +531,7 @@ nni_tls_net_send(void *ctx, const unsigned char *buf, size_t len)
 	iov.iov_len = len;
 	nni_aio_set_iov(tp->tcp_send, 1, &iov);
 	nni_aio_set_timeout(tp->tcp_send, NNG_DURATION_INFINITE);
-	nni_plat_tcp_pipe_send(tp->tcp, tp->tcp_send);
+	nni_tcp_conn_send(tp->tcp, tp->tcp_send);
 	return (len);
 }
 
@@ -615,25 +615,25 @@ nni_tls_recv(nni_tls *tp, nni_aio *aio)
 int
 nni_tls_peername(nni_tls *tp, nni_sockaddr *sa)
 {
-	return (nni_plat_tcp_pipe_peername(tp->tcp, sa));
+	return (nni_tcp_conn_peername(tp->tcp, sa));
 }
 
 int
 nni_tls_sockname(nni_tls *tp, nni_sockaddr *sa)
 {
-	return (nni_plat_tcp_pipe_sockname(tp->tcp, sa));
+	return (nni_tcp_conn_sockname(tp->tcp, sa));
 }
 
 int
 nni_tls_set_nodelay(nni_tls *tp, bool val)
 {
-	return (nni_plat_tcp_pipe_set_nodelay(tp->tcp, val));
+	return (nni_tcp_conn_set_nodelay(tp->tcp, val));
 }
 
 int
 nni_tls_set_keepalive(nni_tls *tp, bool val)
 {
-	return (nni_plat_tcp_pipe_set_keepalive(tp->tcp, val));
+	return (nni_tcp_conn_set_keepalive(tp->tcp, val));
 }
 
 static void
@@ -785,7 +785,7 @@ nni_tls_close(nni_tls *tp)
 		// connection at this point.
 		(void) mbedtls_ssl_close_notify(&tp->ctx);
 	} else {
-		nni_plat_tcp_pipe_close(tp->tcp);
+		nni_tcp_conn_close(tp->tcp);
 	}
 	nni_mtx_unlock(&tp->lk);
 }
diff --git a/src/supplemental/tls/none/tls.c b/src/supplemental/tls/none/tls.c
index 2fdc0c93..d7968758 100644
--- a/src/supplemental/tls/none/tls.c
+++ b/src/supplemental/tls/none/tls.c
@@ -47,7 +47,7 @@ nni_tls_fini(nni_tls *tp)
 }
 
 int
-nni_tls_init(nni_tls **tpp, nng_tls_config *cfg, nni_plat_tcp_pipe *tcp)
+nni_tls_init(nni_tls **tpp, nng_tls_config *cfg, nni_tcp_conn *tcp)
 {
 	NNI_ARG_UNUSED(tpp);
 	NNI_ARG_UNUSED(cfg);
@@ -163,7 +163,8 @@ nng_tls_config_cert_key_file(
 	return (NNG_ENOTSUP);
 }
 
-int nng_tls_config_key(nng_tls_config *cfg, const uint8_t * key, size_t size)
+int
+nng_tls_config_key(nng_tls_config *cfg, const uint8_t *key, size_t size)
 {
 	NNI_ARG_UNUSED(cfg);
 	NNI_ARG_UNUSED(key);
@@ -171,14 +172,14 @@ int nng_tls_config_key(nng_tls_config *cfg, const uint8_t * key, size_t size)
 	return (NNG_ENOTSUP);
 }
 
-int nng_tls_config_pass(nng_tls_config *cfg, const char *pass)
+int
+nng_tls_config_pass(nng_tls_config *cfg, const char *pass)
 {
 	NNI_ARG_UNUSED(cfg);
 	NNI_ARG_UNUSED(pass);
 	return (NNG_ENOTSUP);
 }
 
-
 int
 nng_tls_config_alloc(nng_tls_config **cfgp, nng_tls_mode mode)
 {
diff --git a/src/supplemental/tls/tls_api.h b/src/supplemental/tls/tls_api.h
index 8a40bcfb..53dba7fe 100644
--- a/src/supplemental/tls/tls_api.h
+++ b/src/supplemental/tls/tls_api.h
@@ -31,7 +31,7 @@ extern void nni_tls_config_fini(nng_tls_config *);
 // the configuration object is created with a hold on it.
 extern void nni_tls_config_hold(nng_tls_config *);
 
-extern int  nni_tls_init(nni_tls **, nng_tls_config *, nni_plat_tcp_pipe *);
+extern int  nni_tls_init(nni_tls **, nng_tls_config *, nni_tcp_conn *);
 extern void nni_tls_close(nni_tls *);
 extern void nni_tls_fini(nni_tls *);
 extern void nni_tls_send(nni_tls *, nng_aio *);