Refactor AIO logic to close numerous races and reduce complexity.

This passes valgrind 100% clean for both helgrind and deep leak
checks.  This represents a complete rethink of how the AIOs work,
and much simpler synchronization; the provider API is a bit simpler
to boot, as a number of failure modes have been simply eliminated.

While here a few other minor bugs were squashed.

1 files changed, 11 insertions, 5 deletions

diff --git a/src/platform/posix/posix_epdesc.c b/src/platform/posix/posix_epdesc.c
index 7a91b4ec..a8c8395e 100644
--- a/src/platform/posix/posix_epdesc.c
+++ b/src/platform/posix/posix_epdesc.c
@@ -46,13 +46,17 @@ struct nni_posix_epdesc {
 };
 
 static void
-nni_posix_epdesc_cancel(nni_aio *aio)
+nni_posix_epdesc_cancel(nni_aio *aio, int rv)
 {
 	nni_posix_epdesc *ed = aio->a_prov_data;
 
+	NNI_ASSERT(rv != 0);
 	nni_mtx_lock(&ed->mtx);
-	nni_aio_list_remove(aio);
-	NNI_ASSERT(aio->a_pipe == NULL);
+	if (nni_aio_list_active(aio)) {
+		nni_aio_list_remove(aio);
+		NNI_ASSERT(aio->a_pipe == NULL);
+		nni_aio_finish_error(aio, rv);
+	}
 	nni_mtx_unlock(&ed->mtx);
 }
 
@@ -70,8 +74,10 @@ nni_posix_epdesc_finish(nni_aio *aio, int rv, int newfd)
 			(void) close(newfd);
 		}
 	}
-	if ((nni_aio_finish_pipe(aio, rv, pd) != 0) && (pd != NULL)) {
-		nni_posix_pipedesc_fini(pd);
+	if (rv != 0) {
+		nni_aio_finish_error(aio, rv);
+	} else {
+		nni_aio_finish_pipe(aio, pd);
 	}
 }