Fixes most of the raaces in posix; but at least one remains outstanding.

Apparently there are circumstances when a pipedesc may get orphaned form the
pollq.  This triggers an assertion failure when it occurs.  I am still
trying to understand how this can occur.  Stay tuned.

3 files changed, 36 insertions, 3 deletions

diff --git a/src/core/aio.c b/src/core/aio.c
index f9927e9d..238522b0 100644
--- a/src/core/aio.c
+++ b/src/core/aio.c
@@ -228,6 +228,32 @@ nni_aio_finish(nni_aio *aio, int result, size_t count)
 	return (0);
 }
 
+int
+nni_aio_finish_pipe(nni_aio *aio, int result, void *pipe)
+{
+	nni_mtx_lock(&aio->a_lk);
+	if (aio->a_flags & (NNI_AIO_DONE | NNI_AIO_FINI)) {
+		// Operation already done (canceled or timed out?)
+		nni_mtx_unlock(&aio->a_lk);
+		return (NNG_ESTATE);
+	}
+	aio->a_flags |= NNI_AIO_DONE;
+	aio->a_result      = result;
+	aio->a_count       = 0;
+	aio->a_prov_cancel = NULL;
+	aio->a_prov_data   = NULL;
+	aio->a_pipe        = pipe;
+
+	// This is guaranteed to just be a list operation at this point,
+	// because done wasn't set.
+	nni_aio_expire_remove(aio);
+	aio->a_expire = NNI_TIME_NEVER;
+
+	nni_taskq_dispatch(NULL, &aio->a_tqe);
+	nni_mtx_unlock(&aio->a_lk);
+	return (0);
+}
+
 void
 nni_aio_list_init(nni_list *list)
 {
diff --git a/src/core/aio.h b/src/core/aio.h
index 4f190aa1..ad5b560e 100644
--- a/src/core/aio.h
+++ b/src/core/aio.h
@@ -110,6 +110,7 @@ extern int  nni_aio_list_active(nni_aio *);
 // prevent resources (new pipes for example) from accidentally leaking
 // during close operations.
 extern int nni_aio_finish(nni_aio *, int, size_t);
+extern int nni_aio_finish_pipe(nni_aio *, int, void *);
 
 // nni_aio_cancel is used to cancel an operation.  Any pending I/O or
 // timeouts are canceled if possible, and the callback will be returned
diff --git a/src/core/endpt.c b/src/core/endpt.c
index 8048de2b..596329ab 100644
--- a/src/core/endpt.c
+++ b/src/core/endpt.c
@@ -59,16 +59,22 @@ nni_ep_destroy(nni_ep *ep)
 	if (ep == NULL) {
 		return;
 	}
+
+	// Remove us form the table so we cannot be found.
+	if (ep->ep_id != 0) {
+		nni_idhash_remove(nni_eps, ep->ep_id);
+	}
+
 	nni_aio_fini(&ep->ep_acc_aio);
 	nni_aio_fini(&ep->ep_con_aio);
 	nni_aio_fini(&ep->ep_con_syn);
 	nni_aio_fini(&ep->ep_backoff);
+
+	nni_mtx_lock(&ep->ep_mtx);
 	if (ep->ep_data != NULL) {
 		ep->ep_ops.ep_fini(ep->ep_data);
 	}
-	if (ep->ep_id != 0) {
-		nni_idhash_remove(nni_eps, ep->ep_id);
-	}
+	nni_mtx_unlock(&ep->ep_mtx);
 	nni_cv_fini(&ep->ep_cv);
 	nni_mtx_fini(&ep->ep_mtx);
 	NNI_FREE_STRUCT(ep);