fixes #419 want to nni_aio_stop without blocking (#428)

* fixes #419 want to nni_aio_stop without blocking

This actually introduces an nni_aio_close() API that causes
nni_aio_begin to return NNG_ECLOSED, while scheduling a callback
on the AIO to do an NNG_ECLOSED as well.  This should be called
in non-blocking close() contexts instead of nni_aio_stop(), and
the cases where we call nni_aio_fini() multiple times are updated
updated to add nni_aio_stop() calls on all "interlinked" aios before
finalizing them.

Furthermore, we call nni_aio_close() as soon as practical in the
close path.  This closes an annoying race condition where the
callback from a lower subsystem could wind up rescheduling an
operation that we wanted to abort.

1 files changed, 21 insertions, 6 deletions

diff --git a/src/protocol/pair1/pair.c b/src/protocol/pair1/pair.c
index a3c01d46..ab01e451 100644
--- a/src/protocol/pair1/pair.c
+++ b/src/protocol/pair1/pair.c
@@ -117,9 +117,21 @@ pair1_sock_init_raw(void **sp, nni_sock *nsock)
 }
 
 static void
+pair1_pipe_stop(void *arg)
+{
+	pair1_pipe *p = arg;
+
+	nni_aio_stop(p->aio_send);
+	nni_aio_stop(p->aio_recv);
+	nni_aio_stop(p->aio_putq);
+	nni_aio_stop(p->aio_getq);
+}
+
+static void
 pair1_pipe_fini(void *arg)
 {
 	pair1_pipe *p = arg;
+
 	nni_aio_fini(p->aio_send);
 	nni_aio_fini(p->aio_recv);
 	nni_aio_fini(p->aio_putq);
@@ -198,21 +210,22 @@ pair1_pipe_start(void *arg)
 }
 
 static void
-pair1_pipe_stop(void *arg)
+pair1_pipe_close(void *arg)
 {
 	pair1_pipe *p = arg;
 	pair1_sock *s = p->psock;
 
+	nni_aio_close(p->aio_send);
+	nni_aio_close(p->aio_recv);
+	nni_aio_close(p->aio_putq);
+	nni_aio_close(p->aio_getq);
+
 	nni_mtx_lock(&s->mtx);
 	nni_idhash_remove(s->pipes, nni_pipe_id(p->npipe));
 	nni_list_node_remove(&p->node);
 	nni_mtx_unlock(&s->mtx);
 
 	nni_msgq_close(p->sendq);
-	nni_aio_stop(p->aio_send);
-	nni_aio_stop(p->aio_recv);
-	nni_aio_stop(p->aio_putq);
-	nni_aio_stop(p->aio_getq);
 }
 
 static void
@@ -405,7 +418,8 @@ pair1_sock_open(void *arg)
 static void
 pair1_sock_close(void *arg)
 {
-	NNI_ARG_UNUSED(arg);
+	pair1_sock *s = arg;
+	nni_aio_close(s->aio_getq);
 }
 
 static int
@@ -464,6 +478,7 @@ static nni_proto_pipe_ops pair1_pipe_ops = {
 	.pipe_init  = pair1_pipe_init,
 	.pipe_fini  = pair1_pipe_fini,
 	.pipe_start = pair1_pipe_start,
+	.pipe_close = pair1_pipe_close,
 	.pipe_stop  = pair1_pipe_stop,
 };