fixes #523 dialers could support multiple outstanding dial requests

fixes #179 DNS resolution should be done at connect time
fixes #586 Windows IO completion port work could be better
fixes #339 Windows iocp could use synchronous completions
fixes #280 TCP abstraction improvements

This is a rather monstrous set of changes, which refactors TCP, and
the underlying Windows I/O completion path logic, in order to obtain
a cleaner, simpler API, with support for asynchronous DNS lookups performed
on connect rather than initialization time, the ability to have multiple
connects or accepts pending, as well as fewer extraneous function calls.

The Windows code also benefits from greatly reduced context switching,
fewer lock operations performed, and a reduced number of system calls
on the hot code path.  (We use automatic event resetting instead of manual.)

Some dead code was removed as well, and a few potential edge case leaks
on failure paths (in the websocket code) were plugged.

Note that all TCP based transports benefit from this work. The IPC code
on Windows still uses the legacy IOCP for now, as does the UDP code (used
for ZeroTier.)  We will be converting those soon too.

3 files changed, 183 insertions, 178 deletions

diff --git a/tests/compat_tcp.c b/tests/compat_tcp.c
index fee271e9..95ceb546 100644
--- a/tests/compat_tcp.c
+++ b/tests/compat_tcp.c
@@ -119,9 +119,11 @@ int main (int argc, const char *argv[])
     rc = nn_connect (sc, "tcp://:5555");
     nn_assert (rc < 0);
     errno_assert (nn_errno () == EINVAL);
+#if 0 // NNG does not validate names apriori
     rc = nn_connect (sc, "tcp://-hostname:5555");
     nn_assert (rc < 0);
     errno_assert (nn_errno () == EINVAL);
+#endif
     rc = nn_connect (sc, "tcp://abc.123.---.#:5555");
     nn_assert (rc < 0);
     errno_assert (nn_errno () == EINVAL);
@@ -132,12 +134,14 @@ int main (int argc, const char *argv[])
     errno_assert (nn_errno () == EINVAL);
 #endif
 
+#if 0 // Again NNG is not validating names apriori.
     rc = nn_connect (sc, "tcp://abc...123:5555");
     nn_assert (rc < 0);
     errno_assert (nn_errno () == EINVAL);
     rc = nn_connect (sc, "tcp://.123:5555");
     nn_assert (rc < 0);
     errno_assert (nn_errno () == EINVAL);
+#endif
 
     /*  Connect correctly. Do so before binding the peer socket. */
     test_connect (sc, socket_address);
diff --git a/tests/compat_ws.c b/tests/compat_ws.c
index 2c302712..5dcfa277 100644
--- a/tests/compat_ws.c
+++ b/tests/compat_ws.c
@@ -5,8 +5,8 @@
     Copyright (c) 2014-2016 Jack R. Dunaway. All rights reserved.
     Copyright 2016 Franklin "Snaipe" Mathieu <franklinmathieu@gmail.com>
 
-    Permission is hereby granted, free of charge, to any person obtaining a copy
-    of this software and associated documentation files (the "Software"),
+    Permission is hereby granted, free of charge, to any person obtaining a
+   copy of this software and associated documentation files (the "Software"),
     to deal in the Software without restriction, including without limitation
     the rights to use, copy, modify, merge, publish, distribute, sublicense,
     and/or sell copies of the Software, and to permit persons to whom
@@ -20,8 +20,8 @@
     FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
     THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
     LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
-    IN THE SOFTWARE.
+    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+   DEALINGS IN THE SOFTWARE.
 */
 
 // This file began life in nanomsg, but we have made some significant changes
@@ -30,7 +30,7 @@
 // has support for IPv6, and does not support local interface binding.
 // We have improved the maximum receive size test, and verified that option
 // setting for the frame type conforms to NNG constraints.
- 
+
 #include <nanomsg/nn.h>
 #include <nanomsg/pair.h>
 #include <nanomsg/ws.h>
@@ -41,7 +41,7 @@ static char socket_address[128];
 
 /*  Basic tests for WebSocket transport. */
 
-#if 0  // NNG has no support for text frames.
+#if 0 // NNG has no support for text frames.
 /*  test_text() verifies that we drop messages properly when sending invalid
     UTF-8, but not when we send valid data. */
 void test_text ()
@@ -84,198 +84,200 @@ void test_text ()
 }
 #endif
 
-int main (int argc, const char *argv[])
+int
+main(int argc, const char *argv[])
 {
-    int rc;
-    int sb;
-    int sc;
-    int sb2;
-    int opt;
-    size_t sz;
-    int i;
-    char any_address[128];
-
-    test_addr_from (socket_address, "ws", "127.0.0.1",
-            get_test_port (argc, argv));
-
-    test_addr_from (any_address, "ws", "*",
-            get_test_port (argc, argv));
-
-    /*  Try closing bound but unconnected socket. */
-    sb = test_socket (AF_SP, NN_PAIR);
-    test_bind (sb, any_address);
-    test_close (sb);
-
-    /*  Try closing a TCP socket while it not connected. At the same time
-        test specifying the local address for the connection. */
-    sc = test_socket (AF_SP, NN_PAIR);
-    test_connect (sc, socket_address);
-    test_close (sc);
-
-    /*  Open the socket anew. */
-    sc = test_socket (AF_SP, NN_PAIR);
-
-    /*  Check socket options. */
-    sz = sizeof (opt);
-    rc = nn_getsockopt (sc, NN_WS, NN_WS_MSG_TYPE, &opt, &sz);
-    errno_assert (rc == 0);
-    nn_assert (sz == sizeof (opt));
-    nn_assert (opt == NN_WS_MSG_TYPE_BINARY);
-
-    /*  Default port 80 should be assumed if not explicitly declared. */
-    rc = nn_connect (sc, "ws://127.0.0.1");
-    errno_assert (rc >= 0);
-
-    /*  Try using invalid address strings. */
-    rc = nn_connect (sc, "ws://*:");
-    nn_assert (rc < 0);
-    errno_assert (nn_errno () == EINVAL);
-    rc = nn_connect (sc, "ws://*:1000000");
-    nn_assert (rc < 0);
-    errno_assert (nn_errno () == EINVAL);
-    rc = nn_connect (sc, "ws://*:some_port");
-    nn_assert (rc < 0);
+	int    rc;
+	int    sb;
+	int    sc;
+	int    sb2;
+	int    opt;
+	size_t sz;
+	int    i;
+	char   any_address[128];
+
+	test_addr_from(
+	    socket_address, "ws", "127.0.0.1", get_test_port(argc, argv));
+
+	test_addr_from(any_address, "ws", "*", get_test_port(argc, argv));
+
+	/*  Try closing bound but unconnected socket. */
+	sb = test_socket(AF_SP, NN_PAIR);
+	test_bind(sb, any_address);
+	test_close(sb);
+
+	/*  Try closing a TCP socket while it not connected. At the same time
+	    test specifying the local address for the connection. */
+	sc = test_socket(AF_SP, NN_PAIR);
+	test_connect(sc, socket_address);
+	test_close(sc);
+
+	/*  Open the socket anew. */
+	sc = test_socket(AF_SP, NN_PAIR);
+
+	/*  Check socket options. */
+	sz = sizeof(opt);
+	rc = nn_getsockopt(sc, NN_WS, NN_WS_MSG_TYPE, &opt, &sz);
+	errno_assert(rc == 0);
+	nn_assert(sz == sizeof(opt));
+	nn_assert(opt == NN_WS_MSG_TYPE_BINARY);
+
+	/*  Default port 80 should be assumed if not explicitly declared. */
+	rc = nn_connect(sc, "ws://127.0.0.1");
+	errno_assert(rc >= 0);
+
+	/*  Try using invalid address strings. */
+	rc = nn_connect(sc, "ws://*:");
+	nn_assert(rc < 0);
+	errno_assert(nn_errno() == EINVAL);
+	rc = nn_connect(sc, "ws://*:1000000");
+	nn_assert(rc < 0);
+	errno_assert(nn_errno() == EINVAL);
+	rc = nn_connect(sc, "ws://*:some_port");
+	nn_assert(rc < 0);
 #if 0 // NNG doesn't support device binding
     rc = nn_connect (sc, "ws://eth10000;127.0.0.1:5555");
     nn_assert (rc < 0);
     errno_assert (nn_errno () == ENODEV);
 #endif
 
-    rc = nn_bind (sc, "ws://127.0.0.1:");
-    nn_assert (rc < 0);
-    errno_assert (nn_errno () == EINVAL);
-    rc = nn_bind (sc, "ws://127.0.0.1:1000000");
-    nn_assert (rc < 0);
-    errno_assert (nn_errno () == EINVAL);
+	rc = nn_bind(sc, "ws://127.0.0.1:");
+	nn_assert(rc < 0);
+	errno_assert(nn_errno() == EINVAL);
+	rc = nn_bind(sc, "ws://127.0.0.1:1000000");
+	nn_assert(rc < 0);
+	errno_assert(nn_errno() == EINVAL);
 #if 0 // NNG doesn't support device binding
     rc = nn_bind (sc, "ws://eth10000:5555");
     nn_assert (rc < 0);
     errno_assert (nn_errno () == ENODEV);
 #endif
 
-    rc = nn_connect (sc, "ws://:5555");
-    nn_assert (rc < 0);
-    errno_assert (nn_errno () == EINVAL);
+	rc = nn_connect(sc, "ws://:5555");
+	nn_assert(rc < 0);
+	errno_assert(nn_errno() == EINVAL);
+#if 0 // NNG does not validate names apriori
     rc = nn_connect (sc, "ws://-hostname.:5555");
     nn_assert (rc < 0);
     errno_assert (nn_errno () == EINVAL);
     rc = nn_connect (sc, "ws://abc.123.---.#:5555");
     nn_assert (rc < 0);
     errno_assert (nn_errno () == EINVAL);
-#if 0 // Nothing wrong with this under NNG.  Valid IPv6 URL
     rc = nn_connect (sc, "ws://[::1]:5555");
     nn_assert (rc < 0);
+	errno_assert(nn_errno() == EINVAL);
+	rc = nn_connect(sc, "ws://abc...123:5555");
+	nn_assert(rc < 0);
+	errno_assert(nn_errno() == EINVAL);
+	rc = nn_connect(sc, "ws://.123:5555");
+	nn_assert(rc < 0);
+	errno_assert(nn_errno() == EINVAL);
 #endif
-    errno_assert (nn_errno () == EINVAL);
-    rc = nn_connect (sc, "ws://abc...123:5555");
-    nn_assert (rc < 0);
-    errno_assert (nn_errno () == EINVAL);
-    rc = nn_connect (sc, "ws://.123:5555");
-    nn_assert (rc < 0);
-    errno_assert (nn_errno () == EINVAL);
-
-    test_close (sc);
 
-    sb = test_socket (AF_SP, NN_PAIR);
-    test_bind (sb, socket_address);
-    sc = test_socket (AF_SP, NN_PAIR);
-    test_connect (sc, socket_address);
-
-    nn_sleep(100);
-    /*  Ping-pong test. */
-    for (i = 0; i != 100; ++i) {
-
-        test_send (sc, "ABC");
-        test_recv (sb, "ABC");
-
-        test_send (sb, "DEF");
-        test_recv (sc, "DEF");
-    }
-
-    /*  Batch transfer test. */
-    for (i = 0; i != 100; ++i) {
-        test_send (sc, "0123456789012345678901234567890123456789");
-    }
-    for (i = 0; i != 100; ++i) {
-        test_recv (sb, "0123456789012345678901234567890123456789");
-    }
-
-    test_close (sc);
-    test_close (sb);
-
-    /*  Test two sockets binding to the same address. */
-    sb = test_socket (AF_SP, NN_PAIR);
-    test_bind (sb, socket_address);
-    sb2 = test_socket (AF_SP, NN_PAIR);
-
-    rc = nn_bind (sb2, socket_address);
-    nn_assert (rc < 0);
-    errno_assert (nn_errno () == EADDRINUSE);
-    test_close(sb);
-    test_close(sb2);
-
-    /*  Test that NN_RCVMAXSIZE can be -1, but not lower */
-    sb = test_socket (AF_SP, NN_PAIR);
-    opt = -1;
-    rc = nn_setsockopt (sb, NN_SOL_SOCKET, NN_RCVMAXSIZE, &opt, sizeof (opt));
-    nn_assert (rc >= 0);
-    opt = -2;
-    rc = nn_setsockopt (sb, NN_SOL_SOCKET, NN_RCVMAXSIZE, &opt, sizeof (opt));
-    nn_assert (rc < 0);
-    errno_assert (nn_errno () == EINVAL);
-    test_close (sb);
-
-    /*  Test NN_RCVMAXSIZE limit */
-    sb = test_socket (AF_SP, NN_PAIR);
-    opt = 4;
-    test_setsockopt (sb, NN_SOL_SOCKET, NN_RCVMAXSIZE, &opt, sizeof (opt));
-    test_bind (sb, socket_address);
-    sc = test_socket (AF_SP, NN_PAIR);
-    test_connect (sc, socket_address);
-    opt = 1000;
-    test_setsockopt (sc, NN_SOL_SOCKET, NN_SNDTIMEO, &opt, sizeof (opt));
-    nn_assert (opt == 1000);
-    opt = 1000;
-    test_setsockopt (sb, NN_SOL_SOCKET, NN_RCVTIMEO, &opt, sizeof (opt));
-    nn_assert (opt == 1000);
-    test_send (sc, "ABC");
-    test_recv (sb, "ABC");
-    test_send (sc, "ABCD");
-    test_recv (sb, "ABCD");
-    test_send (sc, "ABCDE");
-    test_drop (sb, ETIMEDOUT);
-    test_close (sc);
-
-    /*  Increase the size limit, reconnect, then try sending again. */
-    opt = 5;
-    test_setsockopt (sb, NN_SOL_SOCKET, NN_RCVMAXSIZE, &opt, sizeof (opt));
-
-    sc = test_socket (AF_SP, NN_PAIR);
-    test_connect (sc, socket_address);
-    nn_sleep(200);
-
-    test_send (sc, "ABCDE");
-    test_recv (sb, "ABCDE");
-    test_close (sb);
-    test_close (sc);
+	test_close(sc);
+
+	sb = test_socket(AF_SP, NN_PAIR);
+	test_bind(sb, socket_address);
+	sc = test_socket(AF_SP, NN_PAIR);
+	test_connect(sc, socket_address);
+
+	nn_sleep(100);
+	/*  Ping-pong test. */
+	for (i = 0; i != 100; ++i) {
+
+		test_send(sc, "ABC");
+		test_recv(sb, "ABC");
+
+		test_send(sb, "DEF");
+		test_recv(sc, "DEF");
+	}
+
+	/*  Batch transfer test. */
+	for (i = 0; i != 100; ++i) {
+		test_send(sc, "0123456789012345678901234567890123456789");
+	}
+	for (i = 0; i != 100; ++i) {
+		test_recv(sb, "0123456789012345678901234567890123456789");
+	}
+
+	test_close(sc);
+	test_close(sb);
+
+	/*  Test two sockets binding to the same address. */
+	sb = test_socket(AF_SP, NN_PAIR);
+	test_bind(sb, socket_address);
+	sb2 = test_socket(AF_SP, NN_PAIR);
+
+	rc = nn_bind(sb2, socket_address);
+	nn_assert(rc < 0);
+	errno_assert(nn_errno() == EADDRINUSE);
+	test_close(sb);
+	test_close(sb2);
+
+	/*  Test that NN_RCVMAXSIZE can be -1, but not lower */
+	sb  = test_socket(AF_SP, NN_PAIR);
+	opt = -1;
+	rc =
+	    nn_setsockopt(sb, NN_SOL_SOCKET, NN_RCVMAXSIZE, &opt, sizeof(opt));
+	nn_assert(rc >= 0);
+	opt = -2;
+	rc =
+	    nn_setsockopt(sb, NN_SOL_SOCKET, NN_RCVMAXSIZE, &opt, sizeof(opt));
+	nn_assert(rc < 0);
+	errno_assert(nn_errno() == EINVAL);
+	test_close(sb);
+
+	/*  Test NN_RCVMAXSIZE limit */
+	sb  = test_socket(AF_SP, NN_PAIR);
+	opt = 4;
+	test_setsockopt(sb, NN_SOL_SOCKET, NN_RCVMAXSIZE, &opt, sizeof(opt));
+	test_bind(sb, socket_address);
+	sc = test_socket(AF_SP, NN_PAIR);
+	test_connect(sc, socket_address);
+	opt = 1000;
+	test_setsockopt(sc, NN_SOL_SOCKET, NN_SNDTIMEO, &opt, sizeof(opt));
+	nn_assert(opt == 1000);
+	opt = 1000;
+	test_setsockopt(sb, NN_SOL_SOCKET, NN_RCVTIMEO, &opt, sizeof(opt));
+	nn_assert(opt == 1000);
+	test_send(sc, "ABC");
+	test_recv(sb, "ABC");
+	test_send(sc, "ABCD");
+	test_recv(sb, "ABCD");
+	test_send(sc, "ABCDE");
+	test_drop(sb, ETIMEDOUT);
+	test_close(sc);
+
+	/*  Increase the size limit, reconnect, then try sending again. */
+	opt = 5;
+	test_setsockopt(sb, NN_SOL_SOCKET, NN_RCVMAXSIZE, &opt, sizeof(opt));
+
+	sc = test_socket(AF_SP, NN_PAIR);
+	test_connect(sc, socket_address);
+	nn_sleep(200);
+
+	test_send(sc, "ABCDE");
+	test_recv(sb, "ABCDE");
+	test_close(sb);
+	test_close(sc);
 
 #if 0 // NNG doesn't support text frames.
     test_text ();
 #else
-    opt = NN_WS_MSG_TYPE_TEXT;
-    rc = nn_setsockopt (sc, NN_WS, NN_WS_MSG_TYPE, &opt, sizeof (opt));
-    nn_assert (rc < 0);
-    errno_assert (nn_errno () == EINVAL);
+	opt = NN_WS_MSG_TYPE_TEXT;
+	rc  = nn_setsockopt(sc, NN_WS, NN_WS_MSG_TYPE, &opt, sizeof(opt));
+	nn_assert(rc < 0);
+	errno_assert(nn_errno() == EINVAL);
 
-    opt = NN_WS_MSG_TYPE_BINARY;
-    test_setsockopt (sb, NN_WS, NN_WS_MSG_TYPE, &opt, sizeof (opt));
+	opt = NN_WS_MSG_TYPE_BINARY;
+	test_setsockopt(sb, NN_WS, NN_WS_MSG_TYPE, &opt, sizeof(opt));
 #endif
 
-    /*  Test closing a socket that is waiting to connect. */
-    sc = test_socket (AF_SP, NN_PAIR);
-    test_connect (sc, socket_address);
-    nn_sleep (100);
-    test_close (sc);
+	/*  Test closing a socket that is waiting to connect. */
+	sc = test_socket(AF_SP, NN_PAIR);
+	test_connect(sc, socket_address);
+	nn_sleep(100);
+	test_close(sc);
 
-    return 0;
+	return 0;
 }
diff --git a/tests/resolv.c b/tests/resolv.c
index 1d502eec..90297a9f 100644
--- a/tests/resolv.c
+++ b/tests/resolv.c
@@ -57,8 +57,7 @@ ip6tostr(void *addr)
 		    nng_sockaddr sa;
 		    So(nng_aio_alloc(&aio, NULL, NULL) == 0);
 		    So(nng_aio_set_input(aio, 0, &sa) == 0);
-		    nni_plat_tcp_resolv("localhost", "80", NNG_AF_INET6, 1,
-		    aio);
+		    nni_tcp_resolv("localhost", "80", NNG_AF_INET6, 1, aio);
 		    nng_aio_wait(aio);
 		    So(nng_aio_result(aio) == 0);
 		    So(sa.s_in6.sa_family == NNG_AF_INET6);
@@ -79,7 +78,7 @@ TestMain("Resolver", {
 
 		So(nng_aio_alloc(&aio, NULL, NULL) == 0);
 		nng_aio_set_input(aio, 0, &sa);
-		nni_plat_tcp_resolv("google-public-dns-a.google.com", "80",
+		nni_tcp_resolv("google-public-dns-a.google.com", "80",
 		    NNG_AF_INET, 1, aio);
 		nng_aio_wait(aio);
 		So(nng_aio_result(aio) == 0);
@@ -96,7 +95,7 @@ TestMain("Resolver", {
 
 		So(nng_aio_alloc(&aio, NULL, NULL) == 0);
 		nng_aio_set_input(aio, 0, &sa);
-		nni_plat_udp_resolv("8.8.4.4", "69", NNG_AF_INET, 1, aio);
+		nni_udp_resolv("8.8.4.4", "69", NNG_AF_INET, 1, aio);
 		nng_aio_wait(aio);
 		So(nng_aio_result(aio) == 0);
 		So(sa.s_in.sa_family == NNG_AF_INET);
@@ -112,7 +111,7 @@ TestMain("Resolver", {
 
 		So(nng_aio_alloc(&aio, NULL, NULL) == 0);
 		nng_aio_set_input(aio, 0, &sa);
-		nni_plat_tcp_resolv("8.8.4.4", "80", NNG_AF_INET, 1, aio);
+		nni_tcp_resolv("8.8.4.4", "80", NNG_AF_INET, 1, aio);
 		nng_aio_wait(aio);
 		So(nng_aio_result(aio) == 0);
 		So(sa.s_in.sa_family == NNG_AF_INET);
@@ -137,7 +136,7 @@ TestMain("Resolver", {
 
 		So(nng_aio_alloc(&aio, NULL, NULL) == 0);
 		nng_aio_set_input(aio, 0, &sa);
-		nni_plat_tcp_resolv("::1", "80", NNG_AF_INET6, 1, aio);
+		nni_tcp_resolv("::1", "80", NNG_AF_INET6, 1, aio);
 		nng_aio_wait(aio);
 		So(nng_aio_result(aio) == 0);
 		So(sa.s_in6.sa_family == NNG_AF_INET6);
@@ -153,7 +152,7 @@ TestMain("Resolver", {
 
 		So(nng_aio_alloc(&aio, NULL, NULL) == 0);
 		nng_aio_set_input(aio, 0, &sa);
-		nni_plat_tcp_resolv("8.8.4.4", "http", NNG_AF_INET, 1, aio);
+		nni_tcp_resolv("8.8.4.4", "http", NNG_AF_INET, 1, aio);
 		nng_aio_wait(aio);
 		So(nng_aio_result(aio) == NNG_EADDRINVAL);
 		nng_aio_free(aio);
@@ -166,7 +165,7 @@ TestMain("Resolver", {
 
 		So(nng_aio_alloc(&aio, NULL, NULL) == 0);
 		nng_aio_set_input(aio, 0, &sa);
-		nni_plat_tcp_resolv("localhost", "80", NNG_AF_INET, 1, aio);
+		nni_tcp_resolv("localhost", "80", NNG_AF_INET, 1, aio);
 		nng_aio_wait(aio);
 		So(nng_aio_result(aio) == 0);
 		So(sa.s_in.sa_family == NNG_AF_INET);
@@ -184,7 +183,7 @@ TestMain("Resolver", {
 
 		So(nng_aio_alloc(&aio, NULL, NULL) == 0);
 		nng_aio_set_input(aio, 0, &sa);
-		nni_plat_tcp_resolv("localhost", "80", NNG_AF_UNSPEC, 1, aio);
+		nni_tcp_resolv("localhost", "80", NNG_AF_UNSPEC, 1, aio);
 		nng_aio_wait(aio);
 		So(nng_aio_result(aio) == 0);
 		So((sa.s_family == NNG_AF_INET) ||