Yet more race condition fixes.

We need to remember that protocol stops can run synchronously, and
therefore we need to wait for the aio to complete.  Further, we need
to break apart shutting down aio activity from deallocation, as we need
to shut down *all* async activity before deallocating *anything*.

Noticed that we had a pipe race in the surveyor pattern too.

3 files changed, 19 insertions, 10 deletions

diff --git a/src/platform/posix/posix_epdesc.c b/src/platform/posix/posix_epdesc.c
index b89af982..7a91b4ec 100644
--- a/src/platform/posix/posix_epdesc.c
+++ b/src/platform/posix/posix_epdesc.c
@@ -195,6 +195,7 @@ nni_posix_epdesc_doclose(nni_posix_epdesc *ed)
 {
 	nni_aio *           aio;
 	struct sockaddr_un *sun;
+	int                 fd;
 
 	ed->closed = 1;
 	while ((aio = nni_list_first(&ed->acceptq)) != NULL) {
@@ -204,14 +205,14 @@ nni_posix_epdesc_doclose(nni_posix_epdesc *ed)
 		nni_posix_epdesc_finish(aio, NNG_ECLOSED, 0);
 	}
 
-	if (ed->node.fd != -1) {
-		(void) shutdown(ed->node.fd, SHUT_RDWR);
+	if ((fd = ed->node.fd) != -1) {
+		ed->node.fd = -1;
+		(void) shutdown(fd, SHUT_RDWR);
+		(void) close(fd);
 		sun = (void *) &ed->locaddr;
 		if ((sun->sun_family == AF_UNIX) && (ed->loclen != 0)) {
 			(void) unlink(sun->sun_path);
 		}
-		(void) close(ed->node.fd);
-		ed->node.fd = -1;
 	}
 }
 
@@ -511,9 +512,13 @@ nni_posix_epdesc_set_remote(nni_posix_epdesc *ed, void *sa, int len)
 void
 nni_posix_epdesc_fini(nni_posix_epdesc *ed)
 {
-	if (ed->node.fd >= 0) {
+	int fd;
+	nni_mtx_lock(&ed->mtx);
+	if ((fd = ed->node.fd) != -1) {
 		(void) close(ed->node.fd);
+		nni_posix_epdesc_doclose(ed);
 	}
+	nni_mtx_unlock(&ed->mtx);
 	nni_posix_pollq_remove(&ed->node);
 	nni_mtx_fini(&ed->mtx);
 	NNI_FREE_STRUCT(ed);
diff --git a/src/platform/posix/posix_pipedesc.c b/src/platform/posix/posix_pipedesc.c
index 4e61c2c4..b85e79a9 100644
--- a/src/platform/posix/posix_pipedesc.c
+++ b/src/platform/posix/posix_pipedesc.c
@@ -45,18 +45,21 @@ static void
 nni_posix_pipedesc_doclose(nni_posix_pipedesc *pd)
 {
 	nni_aio *aio;
+	int      fd;
 
 	pd->closed = 1;
-	if (pd->node.fd != -1) {
-		// Let any peer know we are closing.
-		(void) shutdown(pd->node.fd, SHUT_RDWR);
-	}
 	while ((aio = nni_list_first(&pd->readq)) != NULL) {
 		nni_posix_pipedesc_finish(aio, NNG_ECLOSED);
 	}
 	while ((aio = nni_list_first(&pd->writeq)) != NULL) {
 		nni_posix_pipedesc_finish(aio, NNG_ECLOSED);
 	}
+	if ((fd = pd->node.fd) != -1) {
+		// Let any peer know we are closing.
+		pd->node.fd = -1;
+		(void) shutdown(fd, SHUT_RDWR);
+		(void) close(fd);
+	}
 }
 
 static void
@@ -269,7 +272,7 @@ nni_posix_pipedesc_send(nni_posix_pipedesc *pd, nni_aio *aio)
 		nni_mtx_unlock(&pd->mtx);
 		return;
 	}
-	if (pd->closed < 0) {
+	if (pd->closed) {
 		nni_posix_pipedesc_finish(aio, NNG_ECLOSED);
 		nni_mtx_unlock(&pd->mtx);
 		return;
diff --git a/src/platform/posix/posix_thread.c b/src/platform/posix/posix_thread.c
index 45bbfcd3..071e6007 100644
--- a/src/platform/posix/posix_thread.c
+++ b/src/platform/posix/posix_thread.c
@@ -329,6 +329,7 @@ nni_plat_fini(void)
 {
 	pthread_mutex_lock(&nni_plat_lock);
 	if (nni_plat_inited) {
+		nni_posix_resolv_sysfini();
 		nni_posix_pollq_sysfini();
 		pthread_mutexattr_destroy(&nni_mxattr);
 		pthread_condattr_destroy(&nni_cvattr);