fixes #419 want to nni_aio_stop without blocking (#428)

* fixes #419 want to nni_aio_stop without blocking

This actually introduces an nni_aio_close() API that causes
nni_aio_begin to return NNG_ECLOSED, while scheduling a callback
on the AIO to do an NNG_ECLOSED as well.  This should be called
in non-blocking close() contexts instead of nni_aio_stop(), and
the cases where we call nni_aio_fini() multiple times are updated
updated to add nni_aio_stop() calls on all "interlinked" aios before
finalizing them.

Furthermore, we call nni_aio_close() as soon as practical in the
close path.  This closes an annoying race condition where the
callback from a lower subsystem could wind up rescheduling an
operation that we wanted to abort.

1 files changed, 32 insertions, 32 deletions

diff --git a/src/core/endpt.c b/src/core/endpt.c
index 7593fb42..87566d42 100644
--- a/src/core/endpt.c
+++ b/src/core/endpt.c
@@ -15,30 +15,30 @@
 #include <string.h>
 
 struct nni_ep {
-	nni_tran_ep   ep_ops;  // transport ops
-	nni_tran *    ep_tran; // transport pointer
-	void *        ep_data; // transport private
-	uint64_t      ep_id;   // endpoint id
-	nni_list_node ep_node; // per socket list
-	nni_sock *    ep_sock;
-	nni_url *     ep_url;
-	int           ep_mode;
-	int           ep_started;
-	int           ep_closed;  // full shutdown
-	int           ep_closing; // close pending (waiting on refcnt)
-	int           ep_refcnt;
-	bool          ep_tmo_run;
-	nni_mtx       ep_mtx;
-	nni_cv        ep_cv;
-	nni_list      ep_pipes;
-	nni_aio *     ep_acc_aio;
-	nni_aio *     ep_con_aio;
-	nni_aio *     ep_con_syn;  // used for sync connect
-	nni_aio *     ep_tmo_aio;  // backoff timer
-	nni_duration  ep_maxrtime; // maximum time for reconnect
-	nni_duration  ep_currtime; // current time for reconnect
-	nni_duration  ep_inirtime; // initial time for reconnect
-	nni_time      ep_conntime; // time of last good connect
+	nni_tran_ep_ops ep_ops;  // transport ops
+	nni_tran *      ep_tran; // transport pointer
+	void *          ep_data; // transport private
+	uint64_t        ep_id;   // endpoint id
+	nni_list_node   ep_node; // per socket list
+	nni_sock *      ep_sock;
+	nni_url *       ep_url;
+	int             ep_mode;
+	int             ep_started;
+	int             ep_closed;  // full shutdown
+	int             ep_closing; // close pending (waiting on refcnt)
+	int             ep_refcnt;
+	bool            ep_tmo_run;
+	nni_mtx         ep_mtx;
+	nni_cv          ep_cv;
+	nni_list        ep_pipes;
+	nni_aio *       ep_acc_aio;
+	nni_aio *       ep_con_aio;
+	nni_aio *       ep_con_syn;  // used for sync connect
+	nni_aio *       ep_tmo_aio;  // backoff timer
+	nni_duration    ep_maxrtime; // maximum time for reconnect
+	nni_duration    ep_currtime; // current time for reconnect
+	nni_duration    ep_inirtime; // initial time for reconnect
+	nni_time        ep_conntime; // time of last good connect
 };
 
 // Functionality related to end points.
@@ -249,10 +249,10 @@ nni_ep_shutdown(nni_ep *ep)
 	nni_mtx_unlock(&ep->ep_mtx);
 
 	// Abort any remaining in-flight operations.
-	nni_aio_abort(ep->ep_acc_aio, NNG_ECLOSED);
-	nni_aio_abort(ep->ep_con_aio, NNG_ECLOSED);
-	nni_aio_abort(ep->ep_con_syn, NNG_ECLOSED);
-	nni_aio_abort(ep->ep_tmo_aio, NNG_ECLOSED);
+	nni_aio_close(ep->ep_acc_aio);
+	nni_aio_close(ep->ep_con_aio);
+	nni_aio_close(ep->ep_con_syn);
+	nni_aio_close(ep->ep_tmo_aio);
 
 	// Stop the underlying transport.
 	ep->ep_ops.ep_close(ep->ep_data);
@@ -276,10 +276,10 @@ nni_ep_close(nni_ep *ep)
 
 	nni_ep_shutdown(ep);
 
-	nni_aio_stop(ep->ep_acc_aio);
-	nni_aio_stop(ep->ep_con_aio);
-	nni_aio_stop(ep->ep_con_syn);
-	nni_aio_stop(ep->ep_tmo_aio);
+	nni_aio_close(ep->ep_acc_aio);
+	nni_aio_close(ep->ep_con_aio);
+	nni_aio_close(ep->ep_con_syn);
+	nni_aio_close(ep->ep_tmo_aio);
 
 	nni_mtx_lock(&ep->ep_mtx);
 	NNI_LIST_FOREACH (&ep->ep_pipes, p) {