Fix close related races (POSIX close is a PITA).

3 files changed, 54 insertions, 7 deletions

diff --git a/src/platform/posix/posix_impl.h b/src/platform/posix/posix_impl.h
index 2b72f574..ed52b6fb 100644
--- a/src/platform/posix/posix_impl.h
+++ b/src/platform/posix/posix_impl.h
@@ -35,13 +35,16 @@ extern int nni_plat_errno(int);
 
 #ifdef PLATFORM_POSIX_NET
 struct nni_plat_tcpsock {
-	int fd;
+	int	fd;
+	int	devnull; // used for shutting down blocking accept()
 };
 #endif
 
 // Define types that this platform uses.
 #ifdef PLATFORM_POSIX_THREAD
 
+extern int nni_plat_devnull;    // open descriptor on /dev/null
+
 #include <pthread.h>
 
 // These types are provided for here, to permit them to be directly inlined
diff --git a/src/platform/posix/posix_net.c b/src/platform/posix/posix_net.c
index d0cfb49c..151e14f1 100644
--- a/src/platform/posix/posix_net.c
+++ b/src/platform/posix/posix_net.c
@@ -231,12 +231,35 @@ nni_plat_tcp_setopts(int fd)
 
 
 void
-nni_plat_tcp_close(nni_plat_tcpsock *s)
+nni_plat_tcp_init(nni_plat_tcpsock *s)
 {
-	(void) close(s->fd);
 	s->fd = -1;
 }
 
+
+void
+nni_plat_tcp_fini(nni_plat_tcpsock *s)
+{
+	if (s->fd != -1) {
+		(void) close(s->fd);
+		s->fd = -1;
+	}
+}
+
+
+void
+nni_plat_tcp_shutdown(nni_plat_tcpsock *s)
+{
+	if (s->fd != -1) {
+		(void) shutdown(s->fd, SHUT_RDWR);
+		// This causes the equivalent of a close.  Hopefully waking
+		// up anything that didn't get the hint with the shutdown.
+		// (macOS does not see the shtudown).
+		(void) dup2(nni_plat_devnull, s->fd);
+	}
+}
+
+
 // nni_plat_tcp_bind creates a file descriptor bound to the given address.
 // This basically does the equivalent of socket, bind, and listen.  We have
 // chosen a default value for the listen backlog of 128, which should be
@@ -257,7 +280,7 @@ nni_plat_tcp_listen(nni_plat_tcpsock *s, const nni_sockaddr *addr)
 	}
 
 #ifdef SOCK_CLOEXEC
-	fd = socket(ss.ss_family, SOCK_STREAM, SOCK_CLOEXEC);
+	fd = socket(ss.ss_family, SOCK_STREAM | SOCK_CLOEXEC, 0);
 #else
 	fd = socket(ss.ss_family, SOCK_STREAM, 0);
 #endif
@@ -305,7 +328,7 @@ nni_plat_tcp_connect(nni_plat_tcpsock *s, const nni_sockaddr *addr,
 	}
 
 #ifdef  SOCK_CLOEXEC
-	fd = socket(ss.ss_family, SOCK_STREAM, SOCK_CLOEXEC);
+	fd = socket(ss.ss_family, SOCK_STREAM | SOCK_CLOEXEC, 0);
 #else
 	fd = socket(ss.ss_family, SOCK_STREAM, 0);
 #endif
@@ -346,9 +369,9 @@ nni_plat_tcp_accept(nni_plat_tcpsock *s, nni_plat_tcpsock *server)
 
 	for (;;) {
 #ifdef NNG_USE_ACCEPT4
-		fd = accept4(server, NULL, NULL, SOCK_CLOEXEC);
+		fd = accept4(server->fd, NULL, NULL, SOCK_CLOEXEC);
 		if ((fd < 0) && ((errrno == ENOSYS) || (errno == ENOTSUP))) {
-			fd = accept(server, NULL, NULL);
+			fd = accept(server->fd, NULL, NULL);
 		}
 #else
 		fd = accept(server->fd, NULL, NULL);
@@ -359,6 +382,9 @@ nni_plat_tcp_accept(nni_plat_tcpsock *s, nni_plat_tcpsock *server)
 				// These are not fatal errors, keep trying
 				continue;
 			}
+			if ((errno == EAGAIN) || (errno == EWOULDBLOCK)) {
+				continue;
+			}
 			return (nni_plat_errno(errno));
 		} else {
 			break;
diff --git a/src/platform/posix/posix_thread.c b/src/platform/posix/posix_thread.c
index 02f0b058..00419aac 100644
--- a/src/platform/posix/posix_thread.c
+++ b/src/platform/posix/posix_thread.c
@@ -19,6 +19,7 @@
 #include <stdlib.h>
 #include <sys/types.h>
 #include <unistd.h>
+#include <fcntl.h>
 
 static pthread_mutex_t nni_plat_lock = PTHREAD_MUTEX_INITIALIZER;
 static int nni_plat_inited = 0;
@@ -28,6 +29,11 @@ static int nni_plat_next = 0;
 pthread_condattr_t nni_cvattr;
 pthread_mutexattr_t nni_mxattr;
 
+// We open a /dev/null file descriptor so that we can dup2() it to
+// cause MacOS X to wakeup.  This gives us a "safe" close semantic.
+
+int nni_plat_devnull = -1;
+
 uint32_t
 nni_plat_nextid(void)
 {
@@ -228,30 +234,39 @@ nni_plat_init(int (*helper)(void))
 	if (nni_plat_inited) {
 		return (0);     // fast path
 	}
+
+	if ((nni_plat_devnull = open("/dev/null", O_RDONLY)) < 0) {
+		return (nni_plat_errno(errno));
+	}
 	pthread_mutex_lock(&nni_plat_lock);
 	if (nni_plat_inited) {        // check again under the lock to be sure
 		pthread_mutex_unlock(&nni_plat_lock);
+		(void) close(nni_plat_devnull);
 		return (0);
 	}
 	if (pthread_condattr_init(&nni_cvattr) != 0) {
 		pthread_mutex_unlock(&nni_plat_lock);
+		(void) close(nni_plat_devnull);
 		return (NNG_ENOMEM);
 	}
 #if !defined(NNG_USE_GETTIMEOFDAY) && NNG_USE_CLOCKID != CLOCK_REALTIME
 	if (pthread_condattr_setclock(&nni_cvattr, NNG_USE_CLOCKID) != 0) {
 		pthread_mutex_unlock(&nni_plat_lock);
+		(void) close(nni_plat_devnull);
 		return (NNG_ENOMEM);
 	}
 #endif
 
 	if (pthread_mutexattr_init(&nni_mxattr) != 0) {
 		pthread_mutex_unlock(&nni_plat_lock);
+		(void) close(nni_plat_devnull);
 		return (NNG_ENOMEM);
 	}
 
 	rv = pthread_mutexattr_settype(&nni_mxattr, PTHREAD_MUTEX_ERRORCHECK);
 	if (rv != 0) {
 		pthread_mutex_unlock(&nni_plat_lock);
+		(void) close(nni_plat_devnull);
 		return (NNG_ENOMEM);
 	}
 
@@ -276,6 +291,7 @@ nni_plat_init(int (*helper)(void))
 
 	if (pthread_atfork(NULL, NULL, nni_atfork_child) != 0) {
 		pthread_mutex_unlock(&nni_plat_lock);
+		(void) close(nni_plat_devnull);
 		return (NNG_ENOMEM);
 	}
 	if ((rv = helper()) == 0) {
@@ -294,6 +310,8 @@ nni_plat_fini(void)
 	if (nni_plat_inited) {
 		pthread_mutexattr_destroy(&nni_mxattr);
 		pthread_condattr_destroy(&nni_cvattr);
+		(void) close(nni_plat_devnull);
+		nni_plat_devnull = -1;
 		nni_plat_inited = 0;
 	}
 	pthread_mutex_unlock(&nni_plat_lock);