Rename ioev to aio. Eliminate generic cancel handling (not needed).

We will still need some kind of specific handling of cancellation for
msg queues, but it will be simpler to just implement that for the queues,
and not worry about cancellation in the general case around poll etc.
(The low level poll and I/O routines will get notified by their underlying
transport pipes/descriptors closing.)

6 files changed, 183 insertions, 281 deletions

diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index b23523e9..8cd94cdf 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -32,6 +32,8 @@ set (NNG_SOURCES
 
     core/defs.h
 
+    core/aio.c
+    core/aio.h
     core/clock.c
     core/clock.h
     core/device.c
@@ -44,8 +46,6 @@ set (NNG_SOURCES
     core/idhash.h
     core/init.c
     core/init.h
-    core/ioev.c
-    core/ioev.h
     core/list.c
     core/list.h
     core/message.c
diff --git a/src/core/aio.c b/src/core/aio.c
new file mode 100644
index 00000000..b8304e8a
--- /dev/null
+++ b/src/core/aio.c
@@ -0,0 +1,90 @@
+//
+// Copyright 2017 Garrett D'Amore <garrett@damore.org>
+//
+// This software is supplied under the terms of the MIT License, a
+// copy of which should be located in the distribution where this
+// file was obtained (LICENSE.txt).  A copy of the license may also be
+// found online at https://opensource.org/licenses/MIT.
+//
+
+#include <string.h>
+#include "core/nng_impl.h"
+
+#define NNI_AIO_WAKE    (1<<0)
+
+void
+nni_aio_init(nni_aio *aio, nni_cb cb, void *arg)
+{
+	if (cb == NULL) {
+		cb = (nni_cb) nni_aio_wake;
+		arg = aio;
+	}
+	memset(aio, 0, sizeof (*aio));
+	nni_mtx_init(&aio->a_lk);
+	nni_cv_init(&aio->a_cv, &aio->a_lk);
+	aio->a_cb = cb;
+	aio->a_cbarg = arg;
+}
+
+
+void
+nni_aio_fini(nni_aio *aio)
+{
+	nni_cv_fini(&aio->a_cv);
+	nni_mtx_fini(&aio->a_lk);
+}
+
+
+int
+nni_aio_result(nni_aio *aio)
+{
+	return (aio->a_result);
+}
+
+
+size_t
+nni_aio_count(nni_aio *aio)
+{
+	return (aio->a_count);
+}
+
+
+void
+nni_aio_wake(nni_aio *aio)
+{
+	nni_mtx_lock(&aio->a_lk);
+	aio->a_flags |= NNI_AIO_WAKE;
+	nni_cv_wake(&aio->a_cv);
+	nni_mtx_unlock(&aio->a_lk);
+}
+
+
+void
+nni_aio_wait(nni_aio *aio)
+{
+	nni_mtx_lock(&aio->a_lk);
+	while ((aio->a_flags & NNI_AIO_WAKE) == 0) {
+		nni_cv_wait(&aio->a_cv);
+	}
+	nni_mtx_unlock(&aio->a_lk);
+}
+
+
+// I/O provider related functions.
+
+void
+nni_aio_finish(nni_aio *aio, int result, size_t count)
+{
+	nni_cb cb;
+	void *arg;
+
+	nni_mtx_lock(&aio->a_lk);
+	aio->a_result = result;
+	aio->a_count = count;
+	cb = aio->a_cb;
+	arg = aio->a_cbarg;
+	nni_cv_wake(&aio->a_cv);
+	nni_mtx_unlock(&aio->a_lk);
+
+	cb(arg);
+}
diff --git a/src/core/aio.h b/src/core/aio.h
new file mode 100644
index 00000000..15746fc3
--- /dev/null
+++ b/src/core/aio.h
@@ -0,0 +1,90 @@
+//
+// Copyright 2017 Garrett D'Amore <garrett@damore.org>
+//
+// This software is supplied under the terms of the MIT License, a
+// copy of which should be located in the distribution where this
+// file was obtained (LICENSE.txt).  A copy of the license may also be
+// found online at https://opensource.org/licenses/MIT.
+//
+
+#ifndef CORE_AIO_H
+#define CORE_AIO_H
+
+#include "core/defs.h"
+#include "core/list.h"
+#include "core/thread.h"
+
+typedef struct nni_aio_ops	nni_aio_ops;
+typedef struct nni_aio		nni_aio;
+
+
+// An nni_aio is an async I/O handle.
+struct nni_aio {
+	int		a_result;       // Result code (nng_errno)
+	size_t		a_count;        // Bytes transferred (I/O only)
+	nni_cb		a_cb;           // User specified callback.
+	void *		a_cbarg;        // Callback argument.
+
+	// These fields are private to the io events framework.
+	nni_mtx		a_lk;
+	nni_cv		a_cv;
+	unsigned	a_flags;
+
+	// Read/write operations.
+	nni_iov		a_iov[4];
+	int		a_niov;
+
+	// Sendmsg operation.
+	nni_msg *	a_msg;
+
+	// Recvmsg operation.
+	nni_msg **	a_msgp;
+
+	// TBD: Resolver operations.
+
+	// Provider-use fields.
+	void *		a_prov_data;
+	nni_list_node	a_prov_node;
+};
+
+// nni_aio_init initializes an aio object.  The callback is called with
+// the supplied argument when the operation is complete.  If NULL is
+// supplied for the callback, then nni_aio_wake is used in its place,
+// and the aio is used for the argument.
+extern void nni_aio_init(nni_aio *, nni_cb, void *);
+
+// nni_aio_fini finalizes the aio, releasing resources (locks)
+// associated with it.  The caller is responsible for ensuring that any
+// associated I/O is unscheduled or complete.
+extern void nni_aio_fini(nni_aio *);
+
+// nni_aio_result returns the result code (0 on success, or an NNG errno)
+// for the operation.  It is only valid to call this when the operation is
+// complete (such as when the callback is executed or after nni_aio_wait
+// is performed).
+extern int nni_aio_result(nni_aio *);
+
+// nni_aio_count returns the number of bytes of data transferred, if any.
+// As with nni_aio_result, it is only defined if the I/O operation has
+// completed.
+extern size_t nni_aio_count(nni_aio *);
+
+// nni_aio_wake wakes any threads blocked in nni_aio_wait.  This is the
+// default callback if no other is supplied.  If a user callback is supplied
+// then that code must call this routine to wake any waiters (unless the
+// user code is certain that there are no such waiters).
+extern void nni_aio_wake(nni_aio *);
+
+// nni_aio_wait blocks the caller until the operation is complete, as indicated
+// by nni_aio_wake being called.  (Recall nni_aio_wake is the default
+// callback if none is supplied.)  If a user supplied callback is provided,
+// and that callback does not call nni_aio_wake, then this routine may
+// block the caller indefinitely.
+extern void nni_aio_wait(nni_aio *);
+
+// nni_aio_finish is called by the provider when an operation is complete.
+// The provider gives the result code (0 for success, an NNG errno otherwise),
+// and the amount of data transferred (if any).
+extern void nni_aio_finish(nni_aio *, int, size_t);
+
+#endif // CORE_AIO_H
diff --git a/src/core/ioev.c b/src/core/ioev.c
deleted file mode 100644
index 716d1313..00000000
--- a/src/core/ioev.c
+++ /dev/null
@@ -1,169 +0,0 @@
-//
-// Copyright 2017 Garrett D'Amore <garrett@damore.org>
-//
-// This software is supplied under the terms of the MIT License, a
-// copy of which should be located in the distribution where this
-// file was obtained (LICENSE.txt).  A copy of the license may also be
-// found online at https://opensource.org/licenses/MIT.
-//
-
-#include <string.h>
-#include "core/nng_impl.h"
-
-#define NNI_IOEV_DONE		(1<<0)
-#define NNI_IOEV_BUSY		(1<<1)
-#define NNI_IOEV_CANCEL		(1<<2)
-#define NNI_IOEV_WAKE		(1<<3)
-
-void
-nni_ioev_init(nni_ioev *ioev, nni_cb cb, void *arg)
-{
-	if (cb == NULL) {
-		cb = (nni_cb) nni_ioev_wake;
-		arg = ioev;
-	}
-	memset(ioev, 0, sizeof (*ioev));
-	nni_mtx_init(&ioev->ie_lk);
-	nni_cv_init(&ioev->ie_cv, &ioev->ie_lk);
-	ioev->ie_cb = cb;
-	ioev->ie_cbarg = arg;
-}
-
-
-void
-nni_ioev_fini(nni_ioev *ioev)
-{
-	nni_cv_fini(&ioev->ie_cv);
-	nni_mtx_fini(&ioev->ie_lk);
-}
-
-
-void
-nni_ioev_cancel(nni_ioev *ioev)
-{
-	nni_cb cb;
-	void *arg;
-
-	nni_mtx_lock(&ioev->ie_lk);
-	while ((ioev->ie_flags & NNI_IOEV_BUSY) != 0) {
-		nni_cv_wait(&ioev->ie_cv);
-	}
-	if ((ioev->ie_flags & NNI_IOEV_DONE) != 0) {
-		// Already finished the IO.
-		nni_mtx_unlock(&ioev->ie_lk);
-		return;
-	}
-	ioev->ie_flags |= NNI_IOEV_CANCEL;
-	nni_mtx_unlock(&ioev->ie_lk);
-
-	// Do not hold the lock across the provider!  The provider
-	// must not run on this because we have set the cancel flag,
-	// therefore "nni_ioev_start" will return failure.  The provider
-	// is responsible for dealing with any linked list issues or such,
-	// and freeing any provider data at this point.
-	ioev->ie_prov_ops.ip_cancel(ioev->ie_prov_data);
-
-	nni_mtx_lock(&ioev->ie_lk);
-	ioev->ie_result = NNG_ECANCELED;
-	cb = ioev->ie_cb;
-	arg = ioev->ie_cbarg;
-	nni_mtx_unlock(&ioev->ie_lk);
-
-	// Call the callback.  If none was registered, this will instead
-	// raise the done signal and wake anything blocked in nni_ioev_wait.
-	// (Because cb will be nni_ioev_wake, and arg will be the ioev itself.)
-	cb(arg);
-}
-
-
-int
-nni_ioev_result(nni_ioev *ioev)
-{
-	return (ioev->ie_result);
-}
-
-
-size_t
-nni_ioev_count(nni_ioev *ioev)
-{
-	return (ioev->ie_count);
-}
-
-
-void
-nni_ioev_wake(nni_ioev *ioev)
-{
-	nni_mtx_lock(&ioev->ie_lk);
-	ioev->ie_flags |= NNI_IOEV_WAKE;
-	nni_cv_wake(&ioev->ie_cv);
-	nni_mtx_unlock(&ioev->ie_lk);
-}
-
-
-void
-nni_ioev_wait(nni_ioev *ioev)
-{
-	nni_mtx_lock(&ioev->ie_lk);
-	while ((ioev->ie_flags & NNI_IOEV_WAKE) == 0) {
-		nni_cv_wait(&ioev->ie_cv);
-	}
-	nni_mtx_unlock(&ioev->ie_lk);
-}
-
-
-// I/O provider related functions.
-void
-nni_ioev_set_ops(nni_ioev *ioev, nni_ioev_ops *ops, void *data)
-{
-	memcpy(&ioev->ie_prov_ops, ops, sizeof (*ops));
-	ioev->ie_prov_data = data;
-}
-
-
-// nni_ioev_busy effectively "locks" the IOEV.  We'd like to use the
-// underlying mutex, and that would be faster, but the completion might
-// be executed by a thread other than the one that marked it busy, so we
-// use our own flags.
-int
-nni_ioev_busy(nni_ioev *ioev)
-{
-	nni_mtx_lock(&ioev->ie_lk);
-	if ((ioev->ie_flags & NNI_IOEV_CANCEL) != 0) {
-		nni_mtx_unlock(&ioev->ie_lk);
-		return (NNG_ECANCELED);
-	}
-
-	ioev->ie_flags |= NNI_IOEV_BUSY;
-	nni_mtx_unlock(&ioev->ie_lk);
-	return (0);
-}
-
-
-void
-nni_ioev_unbusy(nni_ioev *ioev)
-{
-	nni_mtx_lock(&ioev->ie_lk);
-	ioev->ie_flags &= ~(NNI_IOEV_BUSY);
-	nni_cv_wake(&ioev->ie_cv);
-	nni_mtx_unlock(&ioev->ie_lk);
-}
-
-
-void
-nni_ioev_finish(nni_ioev *ioev, int result, size_t count)
-{
-	nni_cb cb;
-	void *arg;
-
-	nni_mtx_lock(&ioev->ie_lk);
-	ioev->ie_result = result;
-	ioev->ie_count = count;
-	ioev->ie_flags &= ~(NNI_IOEV_BUSY);
-	ioev->ie_flags |= NNI_IOEV_DONE;
-	cb = ioev->ie_cb;
-	arg = ioev->ie_cbarg;
-	nni_cv_wake(&ioev->ie_cv);
-	nni_mtx_unlock(&ioev->ie_lk);
-
-	cb(arg);
-}
diff --git a/src/core/ioev.h b/src/core/ioev.h
deleted file mode 100644
index e635e0c0..00000000
--- a/src/core/ioev.h
+++ /dev/null
@@ -1,109 +0,0 @@
-//
-// Copyright 2017 Garrett D'Amore <garrett@damore.org>
-//
-// This software is supplied under the terms of the MIT License, a
-// copy of which should be located in the distribution where this
-// file was obtained (LICENSE.txt).  A copy of the license may also be
-// found online at https://opensource.org/licenses/MIT.
-//
-
-#ifndef CORE_IOEV_H
-#define CORE_IOEV_H
-
-#include "core/defs.h"
-#include "core/thread.h"
-
-typedef struct nni_ioev_ops	nni_ioev_ops;
-typedef struct nni_ioev		nni_ioev;
-
-// Provider specific operations on I/O events.  We only have cancellation
-// at present.
-struct nni_ioev_ops {
-	// Cancel the I/O.  This should not callback up, but instead should
-	// just do whatever is necessary to free provider specific state,
-	// and unlink the I/O from any schedule.  The I/O itself will not
-	// have been started if this is called.
-	void (*ip_cancel)(void *);
-};
-
-// An nni_iov is an I/O event handle, used to represent asynchronous I/O.
-// These take several different forms.
-struct nni_ioev {
-	int		ie_result;      // Result code (nng_errno)
-	size_t		ie_count;       // Bytes transferred (I/O only)
-	nni_cb		ie_cb;          // User specified callback.
-	void *		ie_cbarg;       // Callback argument.
-
-	// These fields are private to the io events framework.
-	nni_mtx		ie_lk;
-	nni_cv		ie_cv;
-	unsigned	ie_flags;
-
-	// Provider data.
-	nni_ioev_ops	ie_prov_ops;
-	void *		ie_prov_data;
-};
-
-// nni_ioev_init initializes an IO event.  The callback is called with
-// the supplied argument when the operation is complete.  If NULL is
-// supplied for the callback, then nni_ioev_wake is used in its place,
-// and the ioev is used for the argument.
-extern void nni_ioev_init(nni_ioev *, nni_cb, void *);
-
-// nni_ioev_fini finalizes the IO event, releasing resources (locks)
-// associated with it.  The caller is responsible for ensuring that any
-// associated I/O is unscheduled or complete.
-extern void nni_ioev_fini(nni_ioev *);
-
-// nni_ioev_cancel cancels the IO event.  The result will be NNG_ECANCELED,
-// unless the underlying IO has already completed.
-extern void nni_ioev_cancel(nni_ioev *);
-
-// nni_ioev_result returns the result code (0 on success, or an NNG errno)
-// for the operation.  It is only valid to call this when the operation is
-// complete (such as when the callback is executed or after nni_ioev_wait
-// is performed).
-extern int nni_ioev_result(nni_ioev *);
-
-// nni_ioev_count returns the number of bytes of data transferred, if any.
-// As with nni_ioev_result, it is only defined if the I/O operation has
-// completed.
-extern size_t nni_ioev_count(nni_ioev *);
-
-// nni_ioev_wake wakes any threads blocked in nni_ioev_wait.  This is the
-// default callback if no other is supplied.  If a user callback is supplied
-// then that code must call this routine to wake any waiters (unless the
-// user code is certain that there are no such waiters).
-extern void nni_ioev_wake(nni_ioev *);
-
-// nni_ioev_wait blocks the caller until the IO event is complete, as indicated
-// by nni_ioev_wake being called.  (Recall nni_ioev_wake is the default
-// callback if none is supplied.)  If a user supplied callback is provided,
-// and that callback does not call nni_ioev_wake, then this routine may
-// block the caller indefinitely.
-extern void nni_ioev_wait(nni_ioev *);
-
-// I/O provider related functions.
-extern void nni_ioev_set_ops(nni_ioev *, nni_ioev_ops *, void *);
-
-// nni_ioev_busy is called by the provider to begin an operation, marking
-// the IO busy.  The framework will avoid calling back into the provider
-// (for cancellation for example) while the ioev is busy.  It is important
-// that the busy state be held for only brief periods of time, such as while
-// a non-blocking I/O operation is in progress.  If the IO is canceled (or
-// a cancellation is in progress), the function will return NNG_ECANCELED.
-// In this case, the provider must not perform any further I/O operations,
-// and must not call the completion routine.  Otherwise zero is returned.
-extern int nni_ioev_busy(nni_ioev *);
-
-// nni_ioev_unbusy clears the "busy" state set by nni_ioev_busy.
-extern void nni_ioev_unbusy(nni_ioev *);
-
-// nni_ioev_finish is called by the provider when an operation is complete.
-// (This can be for any reason other than cancellation.)  The provider gives
-// the result code (0 for success, an NNG errno otherwise), and the amount of
-// data transferred (if any).  The ioev must have been marked busy when this
-// is called.  The ioev busy state is automatically cleared by this routine.
-extern void nni_ioev_finish(nni_ioev *, int, size_t);
-
-#endif // CORE_IOEV_H
diff --git a/src/core/nng_impl.h b/src/core/nng_impl.h
index da3424c5..d3e3e5c6 100644
--- a/src/core/nng_impl.h
+++ b/src/core/nng_impl.h
@@ -25,11 +25,11 @@
 
 #include "core/platform.h"
 
+#include "core/aio.h"
 #include "core/clock.h"
 #include "core/device.h"
 #include "core/idhash.h"
 #include "core/init.h"
-#include "core/ioev.h"
 #include "core/list.h"
 #include "core/message.h"
 #include "core/msgqueue.h"