Windows TCP now working.

There are lots of changes here, mostly stuff we did in support of
Windows TCP.  However, there are some bugs that were fixed, and we
added some new error codes, and generalized the handling of some failures
during accept.  Windows IPC (NamedPipes) is still missing.

10 files changed, 375 insertions, 180 deletions

diff --git a/src/platform/posix/posix_debug.c b/src/platform/posix/posix_debug.c
index 71006b25..6199df1d 100644
--- a/src/platform/posix/posix_debug.c
+++ b/src/platform/posix/posix_debug.c
@@ -40,8 +40,6 @@ nni_plat_strerror(int errnum)
 }
 
 
-#define NNI_ERR(x, y)    { x, y },
-
 // There are of course other errors than these, but these are the ones
 // that we might reasonably expect and want to handle "cleanly".  Most of
 // the others should be handled by the system error code.  Note that EFAULT
@@ -55,35 +53,36 @@ static struct {
 	int	nng_err;
 }
 nni_plat_errnos[] = {
-	NNI_ERR(EINTR, NNG_EINTR)
-	NNI_ERR(EINVAL, NNG_EINVAL)
-	NNI_ERR(ENOMEM, NNG_ENOMEM)
-	NNI_ERR(EACCES, NNG_EPERM)
-	NNI_ERR(EADDRINUSE, NNG_EADDRINUSE)
-	NNI_ERR(EADDRNOTAVAIL, NNG_EADDRINVAL)
-	NNI_ERR(EAFNOSUPPORT, NNG_ENOTSUP)
-	NNI_ERR(EAGAIN, NNG_EAGAIN)
-	NNI_ERR(EBADF, NNG_ECLOSED)
-	NNI_ERR(EBUSY, NNG_EBUSY)
-	NNI_ERR(ECONNABORTED, NNG_ECLOSED)
-	NNI_ERR(ECONNREFUSED, NNG_ECONNREFUSED)
-	NNI_ERR(ECONNRESET, NNG_ECLOSED)
-	NNI_ERR(EHOSTUNREACH, NNG_EUNREACHABLE)
-	NNI_ERR(ENETUNREACH, NNG_EUNREACHABLE)
-	NNI_ERR(ENAMETOOLONG, NNG_EINVAL)
-	NNI_ERR(ENOENT, NNG_ENOENT)
-	NNI_ERR(ENOBUFS, NNG_ENOMEM)
-	NNI_ERR(ENOPROTOOPT, NNG_ENOTSUP)
-	NNI_ERR(ENOSYS, NNG_ENOTSUP)
-	NNI_ERR(ENOTSUP, NNG_ENOTSUP)
-	NNI_ERR(EPERM, NNG_EPERM)
-	NNI_ERR(EPIPE, NNG_ECLOSED)
-	NNI_ERR(EPROTO, NNG_EPROTO)
-	NNI_ERR(EPROTONOSUPPORT, NNG_ENOTSUP)
-	NNI_ERR(ETIME, NNG_ETIMEDOUT)
-	NNI_ERR(ETIMEDOUT, NNG_ETIMEDOUT)
-	NNI_ERR(EWOULDBLOCK, NNG_EAGAIN)
-	NNI_ERR(0, 0)                   // must be last
+	{ EINTR,	   NNG_EINTR	    },
+	{ EINVAL,	   NNG_EINVAL	    },
+	{ ENOMEM,	   NNG_ENOMEM	    },
+	{ EACCES,	   NNG_EPERM	    },
+	{ EADDRINUSE,	   NNG_EADDRINUSE   },
+	{ EADDRNOTAVAIL,   NNG_EADDRINVAL   },
+	{ EAFNOSUPPORT,	   NNG_ENOTSUP	    },
+	{ EAGAIN,	   NNG_EAGAIN	    },
+	{ EBADF,	   NNG_ECLOSED	    },
+	{ EBUSY,	   NNG_EBUSY	    },
+	{ ECONNABORTED,	   NNG_ECONNABORTED },
+	{ ECONNREFUSED,	   NNG_ECONNREFUSED },
+	{ ECONNRESET,	   NNG_ECONNRESET   },
+	{ EHOSTUNREACH,	   NNG_EUNREACHABLE },
+	{ ENETUNREACH,	   NNG_EUNREACHABLE },
+	{ ENAMETOOLONG,	   NNG_EINVAL	    },
+	{ ENOENT,	   NNG_ENOENT	    },
+	{ ENOBUFS,	   NNG_ENOMEM	    },
+	{ ENOPROTOOPT,	   NNG_ENOTSUP	    },
+	{ ENOSYS,	   NNG_ENOTSUP	    },
+	{ ENOTSUP,	   NNG_ENOTSUP	    },
+	{ EPERM,	   NNG_EPERM	    },
+	{ EPIPE,	   NNG_ECLOSED	    },
+	{ EPROTO,	   NNG_EPROTO	    },
+	{ EPROTONOSUPPORT, NNG_ENOTSUP	    },
+	{ ETIME,	   NNG_ETIMEDOUT    },
+	{ ETIMEDOUT,	   NNG_ETIMEDOUT    },
+	{ EWOULDBLOCK,	   NNG_EAGAIN	    },
+	// must be last
+	{		0,		  0 },
 };
 
 int
diff --git a/src/platform/posix/posix_ipc.c b/src/platform/posix/posix_ipc.c
index 8c40397a..6044280e 100644
--- a/src/platform/posix/posix_ipc.c
+++ b/src/platform/posix/posix_ipc.c
@@ -73,7 +73,7 @@ nni_plat_ipc_send(nni_plat_ipcsock *s, nni_iov *iovs, int cnt)
 
 	i = 0;
 	while (resid) {
-		rv = writev(s->fd, iov, cnt);
+		rv = writev(s->fd, &iov[i], cnt);
 		if (rv < 0) {
 			if (rv == EINTR) {
 				continue;
@@ -121,7 +121,7 @@ nni_plat_ipc_recv(nni_plat_ipcsock *s, nni_iov *iovs, int cnt)
 	}
 	i = 0;
 	while (resid) {
-		rv = readv(s->fd, iov, cnt);
+		rv = readv(s->fd, &iov[i], cnt);
 		if (rv < 0) {
 			if (errno == EINTR) {
 				continue;
@@ -171,10 +171,11 @@ nni_plat_ipc_setopts(int fd)
 }
 
 
-void
+int
 nni_plat_ipc_init(nni_plat_ipcsock *s)
 {
 	s->fd = -1;
+	return (0);
 }
 
 
diff --git a/src/platform/posix/posix_net.c b/src/platform/posix/posix_net.c
index 76a40259..f9720b66 100644
--- a/src/platform/posix/posix_net.c
+++ b/src/platform/posix/posix_net.c
@@ -23,6 +23,12 @@
 #include <unistd.h>
 #include <netdb.h>
 
+#ifdef  SOCK_CLOEXEC
+#define NNI_TCP_SOCKTYPE	(SOCK_STREAM | SOCK_CLOEXEC)
+#else
+#define NNI_TCP_SOCKTYPE	SOCK_STREAM
+#endif
+
 static int
 nni_plat_to_sockaddr(struct sockaddr_storage *ss, const nni_sockaddr *sa)
 {
@@ -128,7 +134,7 @@ nni_plat_tcp_send(nni_plat_tcpsock *s, nni_iov *iovs, int cnt)
 
 	i = 0;
 	while (resid) {
-		rv = writev(s->fd, iov, cnt);
+		rv = writev(s->fd, &iov[i], cnt);
 		if (rv < 0) {
 			if (rv == EINTR) {
 				continue;
@@ -177,7 +183,7 @@ nni_plat_tcp_recv(nni_plat_tcpsock *s, nni_iov *iovs, int cnt)
 
 	i = 0;
 	while (resid) {
-		rv = readv(s->fd, iov, cnt);
+		rv = readv(s->fd, &iov[i], cnt);
 		if (rv < 0) {
 			if (errno == EINTR) {
 				continue;
@@ -233,10 +239,11 @@ nni_plat_tcp_setopts(int fd)
 }
 
 
-void
+int
 nni_plat_tcp_init(nni_plat_tcpsock *s)
 {
 	s->fd = -1;
+	return (0);
 }
 
 
@@ -282,12 +289,7 @@ nni_plat_tcp_listen(nni_plat_tcpsock *s, const nni_sockaddr *addr)
 		return (NNG_EADDRINVAL);
 	}
 
-#ifdef SOCK_CLOEXEC
-	fd = socket(ss.ss_family, SOCK_STREAM | SOCK_CLOEXEC, 0);
-#else
-	fd = socket(ss.ss_family, SOCK_STREAM, 0);
-#endif
-	if (fd < 0) {
+	if ((fd = socket(ss.ss_family, NNI_TCP_SOCKTYPE, 0)) < 0) {
 		return (nni_plat_errno(errno));
 	}
 
@@ -330,12 +332,7 @@ nni_plat_tcp_connect(nni_plat_tcpsock *s, const nni_sockaddr *addr,
 		return (NNG_EADDRINVAL);
 	}
 
-#ifdef  SOCK_CLOEXEC
-	fd = socket(ss.ss_family, SOCK_STREAM | SOCK_CLOEXEC, 0);
-#else
-	fd = socket(ss.ss_family, SOCK_STREAM, 0);
-#endif
-	if (fd < 0) {
+	if ((fd = socket(ss.ss_family, NNI_TCP_SOCKTYPE, 0)) < 0) {
 		return (nni_plat_errno(errno));
 	}
 
@@ -381,13 +378,6 @@ nni_plat_tcp_accept(nni_plat_tcpsock *s, nni_plat_tcpsock *server)
 #endif
 
 		if (fd < 0) {
-			if ((errno == EINTR) || (errno == ECONNABORTED)) {
-				// These are not fatal errors, keep trying
-				continue;
-			}
-			if ((errno == EAGAIN) || (errno == EWOULDBLOCK)) {
-				continue;
-			}
 			return (nni_plat_errno(errno));
 		} else {
 			break;
diff --git a/src/platform/posix/posix_thread.c b/src/platform/posix/posix_thread.c
index a429e071..7b4dbbdc 100644
--- a/src/platform/posix/posix_thread.c
+++ b/src/platform/posix/posix_thread.c
@@ -28,6 +28,7 @@ static int nni_plat_forked = 0;
 
 pthread_condattr_t nni_cvattr;
 pthread_mutexattr_t nni_mxattr;
+static pthread_attr_t nni_pthread_attr;
 
 // We open a /dev/null file descriptor so that we can dup2() it to
 // cause MacOS X to wakeup.  This gives us a "safe" close semantic.
@@ -189,7 +190,8 @@ nni_plat_thr_init(nni_plat_thr *thr, void (*fn)(void *), void *arg)
 	thr->arg = arg;
 
 	// POSIX wants functions to return a void *, but we don't care.
-	rv = pthread_create(&thr->tid, NULL, nni_plat_thr_main, thr);
+	rv = pthread_create(&thr->tid, &nni_pthread_attr,
+		nni_plat_thr_main, thr);
 	if (rv != 0) {
 		//nni_printf("pthread_create: %s", strerror(rv));
 		return (NNG_ENOMEM);
@@ -263,9 +265,27 @@ nni_plat_init(int (*helper)(void))
 		return (NNG_ENOMEM);
 	}
 
+	rv = pthread_attr_init(&nni_pthread_attr);
+	if (rv != 0) {
+		pthread_mutex_unlock(&nni_plat_lock);
+		(void) close(nni_plat_devnull);
+		pthread_mutexattr_destroy(&nni_mxattr);
+		pthread_condattr_destroy(&nni_cvattr);
+		return (NNG_ENOMEM);
+	}
+
+	// We don't force this, but we want to have it small... we could
+	// probably get by with even just 8k, but Linux usually wants 16k
+	// as a minimum.  If this fails, its not fatal, just we won't be
+	// as scalable / thrifty with our use of VM.
+	(void) pthread_attr_setstacksize(&nni_pthread_attr, 16384);
+
 	if (pthread_atfork(NULL, NULL, nni_atfork_child) != 0) {
 		pthread_mutex_unlock(&nni_plat_lock);
 		(void) close(nni_plat_devnull);
+		pthread_mutexattr_destroy(&nni_mxattr);
+		pthread_condattr_destroy(&nni_cvattr);
+		pthread_attr_destroy(&nni_pthread_attr);
 		return (NNG_ENOMEM);
 	}
 	if ((rv = helper()) == 0) {
@@ -284,6 +304,7 @@ nni_plat_fini(void)
 	if (nni_plat_inited) {
 		pthread_mutexattr_destroy(&nni_mxattr);
 		pthread_condattr_destroy(&nni_cvattr);
+		pthread_attr_destroy(&nni_pthread_attr);
 		(void) close(nni_plat_devnull);
 		nni_plat_devnull = -1;
 		nni_plat_inited = 0;
diff --git a/src/platform/windows/win_debug.c b/src/platform/windows/win_debug.c
index 189f5d4c..00fd6493 100644
--- a/src/platform/windows/win_debug.c
+++ b/src/platform/windows/win_debug.c
@@ -40,26 +40,24 @@ nni_plat_strerror(int errnum)
 }
 
 
-#define NNI_ERR(x, y)    { x, y },
-
 // Win32 has its own error codes, but these ones it shares with POSIX.
 static struct {
 	int	sys_err;
 	int	nng_err;
 }
 nni_plat_errnos[] = {
-	NNI_ERR(ENOENT, NNG_ENOENT)
-	NNI_ERR(EINTR, NNG_EINTR)
-	NNI_ERR(EINVAL, NNG_EINVAL)
-	NNI_ERR(ENOMEM, NNG_ENOMEM)
-	NNI_ERR(EACCES, NNG_EPERM)
-	NNI_ERR(EAGAIN, NNG_EAGAIN)
-	NNI_ERR(EBADF, NNG_ECLOSED)
-	NNI_ERR(EBUSY, NNG_EBUSY)
-	NNI_ERR(ENAMETOOLONG, NNG_EINVAL)
-	NNI_ERR(EPERM, NNG_EPERM)
-	NNI_ERR(EPIPE, NNG_ECLOSED)
-	NNI_ERR(0, 0)                   // must be last
+	{ ENOENT,	NNG_ENOENT  },
+	{ EINTR,	NNG_EINTR   },
+	{ EINVAL,	NNG_EINVAL  },
+	{ ENOMEM,	NNG_ENOMEM  },
+	{ EACCES,	NNG_EPERM   },
+	{ EAGAIN,	NNG_EAGAIN  },
+	{ EBADF,	NNG_ECLOSED },
+	{ EBUSY,	NNG_EBUSY   },
+	{ ENAMETOOLONG, NNG_EINVAL  },
+	{ EPERM,	NNG_EPERM   },
+	{ EPIPE,	NNG_ECLOSED },
+	{	     0,		  0 } // must be last
 };
 
 int
diff --git a/src/platform/windows/win_impl.h b/src/platform/windows/win_impl.h
index b2714b23..f675f510 100644
--- a/src/platform/windows/win_impl.h
+++ b/src/platform/windows/win_impl.h
@@ -27,7 +27,16 @@
 // elsewhere.
 
 struct nni_plat_tcpsock {
-	SOCKET s;
+	SOCKET		s;
+
+	WSAOVERLAPPED	recv_olpd;
+	WSAOVERLAPPED	send_olpd;
+	WSAOVERLAPPED	conn_olpd; // Use for both connect and accept
+
+	// We have to lookup some function pointers using ioctls.  Winsock,
+	// gotta love it.
+	LPFN_CONNECTEX	connectex;
+	LPFN_ACCEPTEX	acceptex;
 };
 
 struct nni_plat_ipcsock {
@@ -47,7 +56,7 @@ struct nni_plat_mtx {
 
 struct nni_plat_cv {
 	CONDITION_VARIABLE	cv;
-	CRITICAL_SECTION	*cs;
+	CRITICAL_SECTION *	cs;
 };
 
 #endif  // PLATFORM_WINDOWS
diff --git a/src/platform/windows/win_ipc.c b/src/platform/windows/win_ipc.c
index 682eb0a8..f4d4d004 100644
--- a/src/platform/windows/win_ipc.c
+++ b/src/platform/windows/win_ipc.c
@@ -115,10 +115,11 @@ nni_plat_ipc_recv(nni_plat_ipcsock *s, nni_iov *iovs, int cnt)
 }
 
 
-void
+int
 nni_plat_ipc_init(nni_plat_ipcsock *s)
 {
 	s->p = INVALID_HANDLE_VALUE;
+	return (0);
 }
 
 
@@ -136,7 +137,7 @@ void
 nni_plat_ipc_shutdown(nni_plat_ipcsock *s)
 {
 	if (s->p != INVALID_HANDLE_VALUE) {
-#if 0		
+#if 0
 		(void) shutdown(s->fd, SHUT_RDWR);
 		// This causes the equivalent of a close.  Hopefully waking
 		// up anything that didn't get the hint with the shutdown.
@@ -209,7 +210,7 @@ nni_plat_ipc_listen(nni_plat_ipcsock *s, const char *path)
 		return (rv);
 	}
 	s->fd = fd;
-#endif	
+#endif
 	return (NNG_ENOTSUP);
 }
 
@@ -244,7 +245,7 @@ nni_plat_ipc_connect(nni_plat_ipcsock *s, const char *path)
 		return (rv);
 	}
 	s->fd = fd;
-#endif	
+#endif
 	return (NNG_ENOTSUP);
 }
 
@@ -275,7 +276,7 @@ nni_plat_ipc_accept(nni_plat_ipcsock *s, nni_plat_ipcsock *server)
 	nni_plat_ipc_setopts(fd);
 
 	s->fd = fd;
-#endif	
+#endif
 	return (NNG_ENOTSUP);
 }
 
diff --git a/src/platform/windows/win_net.c b/src/platform/windows/win_net.c
index 3bd34b78..c7a7849d 100644
--- a/src/platform/windows/win_net.c
+++ b/src/platform/windows/win_net.c
@@ -11,46 +11,104 @@
 
 #ifdef PLATFORM_WINDOWS
 
+#include <stdio.h>
+
+// Windows has infinite numbers of error codes it seems.
 static struct {
 	int	wsa_err;
 	int	nng_err;
 }
 nni_plat_wsa_errnos[] = {
-	{ WSAECONNABORTED,	 NNG_ECLOSED	  },
-	{ WSAEINTR,		 NNG_EINTR	  },
+	{ WSA_INVALID_HANDLE,	    NNG_ECLOSED				 },
+	{ WSA_NOT_ENOUGH_MEMORY,    NNG_ENOMEM				 },
+	{ WSA_INVALID_PARAMETER,    NNG_EINVAL				 },
+	{ WSA_OPERATION_ABORTED,    NNG_ECLOSED				 },
+	{ WSA_IO_INCOMPLETE,	    NNG_EAGAIN				 },
+
+	{ WSAEINTR,		    NNG_EINTR				 },
+	{ WSAEBADF,		    NNG_ECLOSED				 },
+	{ WSAEACCES,		    NNG_EPERM				 },
+	{ WSAEFAULT,		    NNG_ESYSERR + WSAEFAULT		 },
+	{ WSAEWOULDBLOCK,	    NNG_EAGAIN				 },
+	{ WSAEINPROGRESS,	    NNG_EAGAIN				 },
+	{ WSAEALREADY,		    NNG_ESYSERR + WSAEALREADY		 },
+	{ WSAENOTSOCK,		    NNG_ECLOSED				 },
+	{ WSAEMSGSIZE,		    NNG_EMSGSIZE			 },
+	{ WSAEPROTOTYPE,	    NNG_ESYSERR + WSAEPROTOTYPE		 },
+	{ WSAENOPROTOOPT,	    NNG_ENOTSUP				 },
+	{ WSAEPROTONOSUPPORT,	    NNG_ENOTSUP				 },
+	{ WSAEPROTONOSUPPORT,	    NNG_ENOTSUP				 },
+	{ WSAEADDRINUSE,	    NNG_EADDRINUSE			 },
+	{ WSAEADDRNOTAVAIL,	    NNG_EADDRINVAL			 },
+	{ WSAENETDOWN,		    NNG_EUNREACHABLE			 },
+	{ WSAENETUNREACH,	    NNG_EUNREACHABLE			 },
+	{ WSAECONNABORTED,	    NNG_ETIMEDOUT			 },
+	{ WSAECONNRESET,	    NNG_ECLOSED				 },
+	{ WSAENOBUFS,		    NNG_ENOMEM				 },
+	{ WSAEISCONN,		    NNG_ESYSERR + WSAEISCONN		 },
+	{ WSAENOTCONN,		    NNG_ECLOSED				 },
+	{ WSAESHUTDOWN,		    NNG_ECLOSED				 },
+	{ WSAETOOMANYREFS,	    NNG_ESYSERR + WSAETOOMANYREFS	 },
+	{ WSAETIMEDOUT,		    NNG_ETIMEDOUT			 },
+	{ WSAECONNREFUSED,	    NNG_ECONNREFUSED			 },
+	{ WSAELOOP,		    NNG_ESYSERR + WSAELOOP		 },
+	{ WSAENAMETOOLONG,	    NNG_ESYSERR + WSAENAMETOOLONG	 },
+	{ WSAEHOSTDOWN,		    NNG_EUNREACHABLE			 },
+	{ WSAEHOSTUNREACH,	    NNG_EUNREACHABLE			 },
+	{ WSAENOTEMPTY,		    NNG_ESYSERR + WSAENOTEMPTY		 },
+	{ WSAEPROCLIM,		    NNG_ESYSERR + WSAEPROCLIM		 },
+	{ WSAEUSERS,		    NNG_ESYSERR + WSAEUSERS		 },
+	{ WSAEDQUOT,		    NNG_ESYSERR + WSAEDQUOT		 },
+	{ WSAESTALE,		    NNG_ESYSERR + WSAESTALE		 },
+	{ WSAEREMOTE,		    NNG_ESYSERR + WSAEREMOTE		 },
+	{ WSASYSNOTREADY,	    NNG_ESYSERR + WSASYSNOTREADY	 },
+	{ WSAVERNOTSUPPORTED,	    NNG_ENOTSUP				 },
+	{ WSANOTINITIALISED,	    NNG_ESYSERR + WSANOTINITIALISED	 },
+	{ WSAEDISCON,		    NNG_ECLOSED				 },
+	{ WSAENOMORE,		    NNG_ESYSERR + WSAENOMORE		 },
+	{ WSAECANCELLED,	    NNG_ESYSERR + WSAECANCELLED		 },
+	{ WSAEINVALIDPROVIDER,	    NNG_ESYSERR + WSAEINVALIDPROVIDER	 },
+	{ WSAEPROVIDERFAILEDINIT,   NNG_ESYSERR + WSAEPROVIDERFAILEDINIT },
+	{ WSASYSCALLFAILURE,	    NNG_ESYSERR + WSASYSCALLFAILURE	 },
+	{ WSASERVICE_NOT_FOUND,	    NNG_ESYSERR + WSASERVICE_NOT_FOUND	 },
+	{ WSATYPE_NOT_FOUND,	    NNG_ESYSERR + WSATYPE_NOT_FOUND	 },
+	{ WSA_E_CANCELLED,	    NNG_ESYSERR + WSA_E_CANCELLED	 },
+	{ WSAEREFUSED,		    NNG_ESYSERR + WSAEREFUSED		 },
+	{ WSAHOST_NOT_FOUND,	    NNG_EADDRINVAL			 },
+	{ WSATRY_AGAIN,		    NNG_EAGAIN				 },
+	{ WSANO_RECOVERY,	    NNG_ESYSERR + WSANO_RECOVERY	 },
+	{ WSANO_DATA,		    NNG_EADDRINVAL			 },
+	// Eliding all the QoS related errors.
+
+#if 0
 	// REVIEW THESE!!!
-	{ WSAECONNRESET,	 NNG_ECONNREFUSED },
-	{ WSAEMSGSIZE,		 NNG_EINVAL	  },
-	{ WSAENETDOWN,		 NNG_EUNREACHABLE },
-	{ WSAENETRESET,		 NNG_ECLOSED	  },
-	{ WSAENOBUFS,		 NNG_ENOMEM	  },
-	{ WSAESHUTDOWN,		 NNG_ECLOSED	  },
-	{ WSAEWOULDBLOCK,	 NNG_EAGAIN	  },
-	{ WSAEBADF,		 NNG_ECLOSED	  },
-	{ WSA_INVALID_HANDLE,	 NNG_ECLOSED	  },
-	{ WSA_NOT_ENOUGH_MEMORY, NNG_ENOMEM	  },
-	{ WSA_INVALID_PARAMETER, NNG_EINVAL	  },
-	{ WSAEACCES,		 NNG_EPERM	  },
-	{		      0,		0 }, // MUST BE LAST
+	{ ERROR_BROKEN_PIPE,	    NNG_ECLOSED				 },
+	{ ERROR_CONNECTION_REFUSED, NNG_ECONNREFUSED			 },
+	{ ERROR_NOT_CONNECTED,	    NNG_ECLOSED				 },
+	{ ERROR_PIPE_NOT_CONNECTED, NNG_ECLOSED				 },
+	{ ERROR_NO_DATA,	    NNG_ECLOSED				 },
+#endif
+	// Must be Last!!
+	{			 0,				       0 },
 };
 
 
 static int
-nni_plat_wsa_last_error(void)
+nni_winsock_error(int werr)
 {
-	int errnum = WSAGetLastError();
 	int i;
 
-	if (errnum == 0) {
+	if (werr == 0) {
 		return (0);
 	}
+
 	for (i = 0; nni_plat_wsa_errnos[i].nng_err != 0; i++) {
-		if (errnum == nni_plat_wsa_errnos[i].wsa_err) {
+		if (werr == nni_plat_wsa_errnos[i].wsa_err) {
 			return (nni_plat_wsa_errnos[i].nng_err);
 		}
 	}
 	// Other system errno.
-	return (NNG_ESYSERR + errnum);
+	return (NNG_ESYSERR + werr);
 }
 
 
@@ -140,22 +198,53 @@ nni_plat_tcp_send(nni_plat_tcpsock *s, nni_iov *iovs, int cnt)
 {
 	WSABUF iov[4];    // We never have more than 3 at present
 	int i;
-	DWORD sent = 0;
 	int rv;
+	DWORD offset;
+	DWORD nsent;
+	DWORD resid;
+	DWORD flags;
+	WSAOVERLAPPED *olp = &s->send_olpd;
 
 	if (cnt > 4) {
 		return (NNG_EINVAL);
 	}
 
-	for (i = 0; i < cnt; i++) {
+	for (i = 0, resid = 0; i < cnt; resid += iov[i].len, i++) {
 		iov[i].buf = iovs[i].iov_buf;
 		iov[i].len = iovs[i].iov_len;
 	}
 
-	rv = WSASend(s->s, iov, cnt, &sent, 0, NULL, NULL);
-	if (rv != 0) {
-		// XXX: CONVERT WSAGetLastError code.
-		return (nni_plat_wsa_last_error());
+	i = 0;
+	while (resid) {
+		flags = 0;
+		rv = WSASend(s->s, &iov[i], cnt, &nsent, flags, olp, NULL);
+		if (rv == SOCKET_ERROR) {
+			if ((rv = WSAGetLastError()) != WSA_IO_PENDING) {
+				return (nni_winsock_error(rv));
+			}
+			flags = 0;
+			if (!WSAGetOverlappedResult(s->s, olp, &nsent,
+			    TRUE, &flags)) {
+				return (nni_winsock_error(WSAGetLastError()));
+			}
+		}
+
+		if (nsent > resid) {
+			nni_panic("WSASend says it sent too much");
+		}
+
+		resid -= nsent;
+		while (nsent) {
+			if (iov[i].len <= nsent) {
+				nsent -= iov[i].len;
+				i++;
+				cnt--;
+			} else {
+				iov[i].len -= nsent;
+				iov[i].buf += nsent;
+				nsent = 0;
+			}
+		}
 	}
 
 	return (0);
@@ -167,29 +256,39 @@ nni_plat_tcp_recv(nni_plat_tcpsock *s, nni_iov *iovs, int cnt)
 {
 	WSABUF iov[4];    // We never have more than 3 at present
 	int i;
-	int offset;
-	int resid = 0;
 	int rv;
+	DWORD offset;
+	DWORD resid;
 	DWORD nrecv;
+	DWORD flags;
+	WSAOVERLAPPED *olp = &s->recv_olpd;
 
 	if (cnt > 4) {
 		return (NNG_EINVAL);
 	}
 
-	for (i = 0; i < cnt; i++) {
+	for (i = 0, resid = 0; i < cnt; resid += iov[i].len, i++) {
 		iov[i].buf = iovs[i].iov_buf;
 		iov[i].len = iovs[i].iov_len;
-		resid += iov[i].len;
 	}
 
 	i = 0;
 	while (resid) {
-		rv = WSARecv(s->s, iov, cnt, &nrecv, 0, NULL, NULL);
-		if (rv != 0) {
-			return (nni_plat_wsa_last_error());
+		flags = 0;
+		rv = WSARecv(s->s, &iov[i], cnt, &nrecv, &flags, olp, NULL);
+		if (rv == SOCKET_ERROR) {
+			if ((rv = WSAGetLastError()) != WSA_IO_PENDING) {
+				return (nni_winsock_error(rv));
+			}
+			flags = 0;
+			if (!WSAGetOverlappedResult(s->s, olp, &nrecv,
+			    TRUE, &flags)) {
+				return (nni_winsock_error(WSAGetLastError()));
+			}
 		}
+
 		if (nrecv > resid) {
-			nni_panic("readv says it read too much!");
+			nni_panic("WSARecv says it read too much!");
 		}
 
 		resid -= nrecv;
@@ -227,19 +326,74 @@ nni_plat_tcp_setopts(SOCKET fd)
 }
 
 
-void
+int
 nni_plat_tcp_init(nni_plat_tcpsock *s)
 {
 	s->s = INVALID_SOCKET;
+	return (0);
+}
+
+
+static int
+nni_plat_tcp_open(nni_plat_tcpsock *s)
+{
+	int rv;
+	DWORD nbytes;
+	GUID guid1 = WSAID_CONNECTEX;
+	GUID guid2 = WSAID_ACCEPTEX;
+
+	ZeroMemory(s, sizeof (*s));
+	s->s = WSASocket(AF_INET, SOCK_STREAM, IPPROTO_TCP, NULL, 0,
+		WSA_FLAG_NO_HANDLE_INHERIT|WSA_FLAG_OVERLAPPED);
+	if (s->s == INVALID_SOCKET) {
+		rv = WSAGetLastError();
+		return (nni_winsock_error(rv));
+	}
+
+	if (WSAIoctl(s->s, SIO_GET_EXTENSION_FUNCTION_POINTER,
+	    &guid1, sizeof (guid1), &s->connectex, sizeof (s->connectex),
+	    &nbytes, NULL, NULL) == SOCKET_ERROR) {
+		nni_panic("failed lookup for ConnectEx function");
+	}
+	if (WSAIoctl(s->s, SIO_GET_EXTENSION_FUNCTION_POINTER,
+	    &guid2, sizeof (guid2), &s->acceptex, sizeof (s->acceptex),
+	    &nbytes, NULL, NULL) == SOCKET_ERROR) {
+		nni_panic("failed lookup for AcceptEx function");
+	}
+
+	nni_plat_tcp_setopts(s->s);
+
+	return (0);
+}
+
+
+static void
+nni_plat_tcp_close(nni_plat_tcpsock *s)
+{
+	SOCKET fd;
+
+	if ((fd = s->s) != INVALID_SOCKET) {
+		s->s = INVALID_SOCKET;
+		(void) shutdown(fd, SD_BOTH);
+		(void) CancelIoEx((HANDLE) fd, &s->conn_olpd);
+		(void) CancelIoEx((HANDLE) fd, &s->recv_olpd);
+		(void) CancelIoEx((HANDLE) fd, &s->send_olpd);
+		(void) closesocket(fd);
+	}
 }
 
 
 void
 nni_plat_tcp_fini(nni_plat_tcpsock *s)
 {
-	if (s->s != INVALID_SOCKET) {
-		(void) closesocket(s->s);
+	SOCKET fd;
+
+	if ((fd = s->s) != INVALID_SOCKET) {
 		s->s = INVALID_SOCKET;
+		(void) CancelIoEx((HANDLE) fd, &s->conn_olpd);
+		(void) CancelIoEx((HANDLE) fd, &s->recv_olpd);
+		(void) CancelIoEx((HANDLE) fd, &s->send_olpd);
+		(void) closesocket(fd);
 	}
 }
 
@@ -247,9 +401,7 @@ nni_plat_tcp_fini(nni_plat_tcpsock *s)
 void
 nni_plat_tcp_shutdown(nni_plat_tcpsock *s)
 {
-	if (s->s != INVALID_SOCKET) {
-		(void) shutdown(s->s, SD_BOTH);
-	}
+	nni_plat_tcp_close(s);
 }
 
 
@@ -264,46 +416,39 @@ nni_plat_tcp_listen(nni_plat_tcpsock *s, const nni_sockaddr *addr)
 {
 	int len;
 	SOCKADDR_STORAGE ss;
+	ULONG yes;
 	int rv;
-	BOOL yes;
 
 	len = nni_plat_to_sockaddr(&ss, addr);
 	if (len < 0) {
 		return (NNG_EADDRINVAL);
 	}
 
-	s->s = WSASocket(ss.ss_family, SOCK_STREAM, 0, NULL, 0,
-		WSA_FLAG_NO_HANDLE_INHERIT);
-	if (s->s == INVALID_SOCKET) {
-		return (nni_plat_wsa_last_error());
+	if ((rv = nni_plat_tcp_open(s)) != 0) {
+		return (rv);
 	}
 
-	nni_plat_tcp_setopts(s->s);
-
 	// Make sure that we use the address exclusively.  Windows lets
 	// others hijack us by default.
 	yes = 1;
 	if (setsockopt(s->s, SOL_SOCKET, SO_EXCLUSIVEADDRUSE, (char *) &yes,
 	    sizeof (yes)) == SOCKET_ERROR) {
-		rv = nni_plat_wsa_last_error();
-		(void) closesocket(s->s);
-		s->s = INVALID_SOCKET;
-		return (rv);
+		rv = WSAGetLastError();
+		nni_plat_tcp_close(s);
+		return (nni_winsock_error(rv));
 	}
 	if (bind(s->s, (struct sockaddr *) &ss, len) != 0) {
-		rv = nni_plat_wsa_last_error();
-		(void) closesocket(s->s);
-		s->s = INVALID_SOCKET;
-		return (rv);
+		rv = WSAGetLastError();
+		nni_plat_tcp_close(s);
+		return (nni_winsock_error(rv));
 	}
 
 	// Listen -- 128 depth is probably sufficient.  If it isn't, other
 	// bad things are going to happen.
 	if (listen(s->s, 128) != 0) {
-		rv = nni_plat_wsa_last_error();
-		(void) closesocket(s->s);
-		s->s = INVALID_SOCKET;
-		return (rv);
+		rv = WSAGetLastError();
+		nni_plat_tcp_close(s);
+		return (nni_winsock_error(rv));
 	}
 
 	return (0);
@@ -320,6 +465,10 @@ nni_plat_tcp_connect(nni_plat_tcpsock *s, const nni_sockaddr *addr,
 	int len;
 	SOCKADDR_STORAGE ss;
 	SOCKADDR_STORAGE bss;
+	WSAOVERLAPPED *olp = &s->conn_olpd;
+	BOOL ok;
+	DWORD nbytes;
+	DWORD flags;
 	int rv;
 
 	len = nni_plat_to_sockaddr(&ss, addr);
@@ -327,39 +476,44 @@ nni_plat_tcp_connect(nni_plat_tcpsock *s, const nni_sockaddr *addr,
 		return (NNG_EADDRINVAL);
 	}
 
-	s->s = WSASocket(ss.ss_family, SOCK_STREAM, 0, NULL, 0,
-		WSA_FLAG_NO_HANDLE_INHERIT);
-	if (s->s == INVALID_SOCKET) {
-		return (nni_plat_wsa_last_error());
-	}
-
 	if (bindaddr != NULL) {
 		if (bindaddr->s_un.s_family != addr->s_un.s_family) {
-			(void) closesocket(s->s);
-			s->s = INVALID_SOCKET;
-			return (NNG_EINVAL);
+			return (NNG_EADDRINVAL);
 		}
 		if (nni_plat_to_sockaddr(&bss, bindaddr) < 0) {
-			(void) closesocket(s->s);
-			s->s = INVALID_SOCKET;
 			return (NNG_EADDRINVAL);
 		}
-		if (bind(s->s, (struct sockaddr *) &bss, len) < 0) {
-			rv = nni_plat_wsa_last_error();
-			(void) closesocket(s->s);
-			s->s = INVALID_SOCKET;
-			return (rv);
-		}
+	} else {
+		ZeroMemory(&bss, sizeof (bss));
+		bss.ss_family = ss.ss_family;
 	}
 
-	nni_plat_tcp_setopts(s->s);
-
-	if (connect(s->s, (struct sockaddr *) &ss, len) != 0) {
-		rv = nni_plat_wsa_last_error();
-		(void) closesocket(s->s);
-		s->s = INVALID_SOCKET;
+	if ((rv = nni_plat_tcp_open(s)) != 0) {
 		return (rv);
 	}
+
+	// ConnectEx must always be bound first.
+	if (bind(s->s, (struct sockaddr *) &bss, len) < 0) {
+		rv = WSAGetLastError();
+		nni_plat_tcp_close(s);
+		return (nni_winsock_error(rv));
+	}
+
+	if (s->connectex(s->s, (struct sockaddr *) &ss, len, NULL, 0, NULL,
+	    olp)) {
+		// Immediate completion?
+		return (0);
+	}
+	if ((rv = WSAGetLastError()) != ERROR_IO_PENDING) {
+		nni_plat_tcp_close(s);
+		return (nni_winsock_error(rv));
+	}
+	nbytes = flags = 0;
+	if (!WSAGetOverlappedResult(s->s, olp, &nbytes, TRUE, &flags)) {
+		rv = WSAGetLastError();
+		nni_plat_tcp_close(s);
+		return (nni_winsock_error(rv));
+	}
 	return (0);
 }
 
@@ -367,26 +521,32 @@ nni_plat_tcp_connect(nni_plat_tcpsock *s, const nni_sockaddr *addr,
 int
 nni_plat_tcp_accept(nni_plat_tcpsock *s, nni_plat_tcpsock *server)
 {
-	SOCKET fd;
-	int err;
-
-	for (;;) {
-		fd = accept(server->s, NULL, NULL);
+	DWORD nbytes;
+	DWORD flags;
+	WSAOVERLAPPED *olp = &s->conn_olpd;
+	char ainfo[512];
+	int rv;
 
-		if (fd == INVALID_SOCKET) {
-			err = WSAGetLastError();
-			if ((err == WSAECONNRESET) || (err == WSAEWOULDBLOCK)) {
-				continue;
-			}
-			return (nni_plat_wsa_last_error());
-		} else {
-			break;
-		}
+	if ((rv = nni_plat_tcp_open(s)) != 0) {
+		return (rv);
 	}
 
-	nni_plat_tcp_setopts(fd);
-
-	s->s = fd;
+	// 256 > (sizeof (SOCKADDR_STORAGE) + 16)
+	nbytes = 0;
+	if (s->acceptex(server->s, s->s, ainfo, 0, 256, 256, &nbytes, olp)) {
+		// Immediate completion?
+		return (0);
+	}
+	if ((rv = WSAGetLastError()) != ERROR_IO_PENDING) {
+		nni_plat_tcp_close(s);
+		return (nni_winsock_error(rv));
+	}
+	nbytes = flags = 0;
+	if (!WSAGetOverlappedResult(server->s, olp, &nbytes, TRUE, &flags)) {
+		rv = WSAGetLastError();
+		nni_plat_tcp_close(s);
+		return (nni_winsock_error(rv));
+	}
 	return (0);
 }
 
diff --git a/src/platform/windows/win_rand.c b/src/platform/windows/win_rand.c
index 1cabb6a0..8bca5927 100644
--- a/src/platform/windows/win_rand.c
+++ b/src/platform/windows/win_rand.c
@@ -17,13 +17,14 @@ void
 nni_plat_seed_prng(void *buf, size_t bufsz)
 {
 	unsigned val;
+
 	// The rand_s routine uses RtlGenRandom to get high quality
 	// pseudo random numbers (i.e. numbers that should be good enough
 	// for use with crypto keying.)
 	while (bufsz > sizeof (val)) {
 		rand_s(&val);
 		memcpy(buf, &val, sizeof (val));
-		buf = (((char *)buf) + sizeof (val));
+		buf = (((char *) buf) + sizeof (val));
 		bufsz -= sizeof (val);
 	}
 }
diff --git a/src/platform/windows/win_thread.c b/src/platform/windows/win_thread.c
index 4f323616..631883fc 100644
--- a/src/platform/windows/win_thread.c
+++ b/src/platform/windows/win_thread.c
@@ -125,8 +125,11 @@ nni_plat_thr_init(nni_plat_thr *thr, void (*fn)(void *), void *arg)
 	thr->func = fn;
 	thr->arg = arg;
 
-	thr->handle = (HANDLE) _beginthreadex(NULL, 0,
-		nni_plat_thr_main, thr, 0, NULL);
+	// We could probably even go down to 8k... but crypto for some
+	// protocols might get bigger than this.  1MB is waaay too big.
+	thr->handle = (HANDLE) _beginthreadex(NULL, 16384,
+		nni_plat_thr_main, thr, STACK_SIZE_PARAM_IS_A_RESERVATION,
+		NULL);
 	if (thr->handle == NULL) {
 		return (NNG_ENOMEM);    // Best guess...
 	}
@@ -166,6 +169,17 @@ nni_plat_init(int (*helper)(void))
 		Sleep(1);
 	}
 	if (!inited) {
+		WSADATA data;
+		WORD ver;
+		ver = MAKEWORD(2, 2);
+		if (WSAStartup(MAKEWORD(2, 2), &data) != 0) {
+			InterlockedExchange(&initing, 0);
+			if ((LOBYTE(data.wVersion) != 2) ||
+			    (HIBYTE(data.wVersion) != 2)) {
+				nni_panic("got back wrong winsock ver");
+			}
+			return (NNG_ENOMEM);
+		}
 		helper();
 		inited = 1;
 	}
@@ -178,6 +192,7 @@ nni_plat_init(int (*helper)(void))
 void
 nni_plat_fini(void)
 {
+	WSACleanup();
 }