fixes #1112 POSIX pollq finalizers could be simpler

We reap the connections when closing, to ensure that the clean up is
done outside the pollq thread.  This also reduces pressure on the
pollq, we think.  But more importantly it eliminates some complex
code that was meant to avoid deadlocks, but ultimately created other
use-after-free challenges.  This work is an enabler for further
simplifications in the aio/task logic.  While here we converted some
potentially racy locking of the dialers and reference counts to simpler
lock-free reference counting.

14 files changed, 211 insertions, 177 deletions

diff --git a/src/platform/posix/posix_ipc.h b/src/platform/posix/posix_ipc.h
index f0b9b5ef..894a5e96 100644
--- a/src/platform/posix/posix_ipc.h
+++ b/src/platform/posix/posix_ipc.h
@@ -1,5 +1,5 @@
 //
-// Copyright 2019 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2020 Staysail Systems, Inc. <info@staysail.tech>
 // Copyright 2018 Capitar IT Group BV <info@capitar.com>
 // Copyright 2019 Devolutions <info@devolutions.net>
 //
@@ -29,6 +29,7 @@ struct nni_ipc_conn {
 	nni_mtx         mtx;
 	nni_aio *       dial_aio;
 	nni_ipc_dialer *dialer;
+	nni_reap_item   reap;
 };
 
 struct nni_ipc_dialer {
@@ -37,11 +38,12 @@ struct nni_ipc_dialer {
 	bool              closed;
 	nni_mtx           mtx;
 	nng_sockaddr      sa;
-	int               refcnt;
-	bool              fini;
+	nni_atomic_u64    ref;
+	nni_atomic_bool   fini;
 };
 
-extern int  nni_posix_ipc_init(nni_ipc_conn **, nni_posix_pfd *);
+extern int  nni_posix_ipc_alloc(nni_ipc_conn **, nni_ipc_dialer *);
+extern void nni_posix_ipc_init(nni_ipc_conn *, nni_posix_pfd *);
 extern void nni_posix_ipc_start(nni_ipc_conn *);
 extern void nni_posix_ipc_dialer_rele(nni_ipc_dialer *);
 
diff --git a/src/platform/posix/posix_ipcconn.c b/src/platform/posix/posix_ipcconn.c
index 2b078aa5..e4d783f3 100644
--- a/src/platform/posix/posix_ipcconn.c
+++ b/src/platform/posix/posix_ipcconn.c
@@ -1,5 +1,5 @@
 //
-// Copyright 2019 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2020 Staysail Systems, Inc. <info@staysail.tech>
 // Copyright 2018 Capitar IT Group BV <info@capitar.com>
 // Copyright 2019 Devolutions <info@devolutions.net>
 //
@@ -200,7 +200,9 @@ ipc_close(void *arg)
 			nni_aio_list_remove(aio);
 			nni_aio_finish_error(aio, NNG_ECLOSED);
 		}
-		nni_posix_pfd_close(c->pfd);
+		if (c->pfd != NULL) {
+			nni_posix_pfd_close(c->pfd);
+		}
 	}
 	nni_mtx_unlock(&c->mtx);
 }
@@ -483,14 +485,13 @@ nni_posix_ipc_start(nni_ipc_conn *c)
 }
 
 static void
-ipc_free(void *arg)
+ipc_reap(void *arg)
 {
 	ipc_conn *c = arg;
 	ipc_close(c);
-	nni_posix_pfd_fini(c->pfd);
-	nni_mtx_lock(&c->mtx); // not strictly needed, but shut up TSAN
-	c->pfd = NULL;
-	nni_mtx_unlock(&c->mtx);
+	if (c->pfd != NULL) {
+		nni_posix_pfd_fini(c->pfd);
+	}
 	nni_mtx_fini(&c->mtx);
 
 	if (c->dialer != NULL) {
@@ -500,6 +501,13 @@ ipc_free(void *arg)
 	NNI_FREE_STRUCT(c);
 }
 
+static void
+ipc_free(void *arg)
+{
+	ipc_conn *c = arg;
+	nni_reap(&c->reap, ipc_reap, c);
+}
+
 static const nni_option ipc_options[] = {
 	{
 	    .o_name = NNG_OPT_LOCADDR,
@@ -545,7 +553,7 @@ ipc_setx(void *arg, const char *name, const void *val, size_t sz, nni_type t)
 }
 
 int
-nni_posix_ipc_init(nni_ipc_conn **cp, nni_posix_pfd *pfd)
+nni_posix_ipc_alloc(nni_ipc_conn **cp, nni_ipc_dialer *d)
 {
 	ipc_conn *c;
 
@@ -554,7 +562,7 @@ nni_posix_ipc_init(nni_ipc_conn **cp, nni_posix_pfd *pfd)
 	}
 
 	c->closed         = false;
-	c->pfd            = pfd;
+	c->dialer         = d;
 	c->stream.s_free  = ipc_free;
 	c->stream.s_close = ipc_close;
 	c->stream.s_send  = ipc_send;
@@ -569,3 +577,9 @@ nni_posix_ipc_init(nni_ipc_conn **cp, nni_posix_pfd *pfd)
 	*cp = c;
 	return (0);
 }
+
+void
+nni_posix_ipc_init(nni_ipc_conn *c, nni_posix_pfd *pfd)
+{
+	c->pfd = pfd;
+}
diff --git a/src/platform/posix/posix_ipcdial.c b/src/platform/posix/posix_ipcdial.c
index 50c7a897..65061767 100644
--- a/src/platform/posix/posix_ipcdial.c
+++ b/src/platform/posix/posix_ipcdial.c
@@ -1,5 +1,5 @@
 //
-// Copyright 2019 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2020 Staysail Systems, Inc. <info@staysail.tech>
 // Copyright 2018 Capitar IT Group BV <info@capitar.com>
 // Copyright 2019 Devolutions <info@devolutions.net>
 //
@@ -62,26 +62,17 @@ ipc_dialer_free(void *arg)
 	ipc_dialer *d = arg;
 
 	ipc_dialer_close(d);
-	nni_mtx_lock(&d->mtx);
-	d->fini = true;
-	if (d->refcnt) {
-		nni_mtx_unlock(&d->mtx);
-		return;
-	}
-	nni_mtx_unlock(&d->mtx);
-	ipc_dialer_fini(d);
+	nni_atomic_set_bool(&d->fini, true);
+	nni_posix_ipc_dialer_rele(d);
 }
 
 void
 nni_posix_ipc_dialer_rele(ipc_dialer *d)
 {
-	nni_mtx_lock(&d->mtx);
-	d->refcnt--;
-	if ((d->refcnt > 0) || (!d->fini)) {
-		nni_mtx_unlock(&d->mtx);
+	if (((nni_atomic_dec64_nv(&d->ref)) != 0) ||
+	    (!nni_atomic_get_bool(&d->fini))) {
 		return;
 	}
-	nni_mtx_unlock(&d->mtx);
 	ipc_dialer_fini(d);
 }
 
@@ -166,7 +157,7 @@ ipc_dialer_dial(void *arg, nni_aio *aio)
 	nni_ipc_conn *          c;
 	nni_posix_pfd *         pfd = NULL;
 	struct sockaddr_storage ss;
-	size_t                  sslen;
+	size_t                  len;
 	int                     fd;
 	int                     rv;
 
@@ -174,7 +165,7 @@ ipc_dialer_dial(void *arg, nni_aio *aio)
 		return;
 	}
 
-	if (((sslen = nni_posix_nn2sockaddr(&ss, &d->sa)) == 0) ||
+	if (((len = nni_posix_nn2sockaddr(&ss, &d->sa)) == 0) ||
 	    (ss.ss_family != AF_UNIX)) {
 		nni_aio_finish_error(aio, NNG_EADDRINVAL);
 		return;
@@ -185,23 +176,25 @@ ipc_dialer_dial(void *arg, nni_aio *aio)
 		return;
 	}
 
-	// This arranges for the fd to be in nonblocking mode, and adds the
-	// pollfd to the list.
-	if ((rv = nni_posix_pfd_init(&pfd, fd)) != 0) {
+	nni_atomic_inc64(&d->ref);
+
+	if ((rv = nni_posix_ipc_alloc(&c, d)) != 0) {
 		(void) close(fd);
+		nni_posix_ipc_dialer_rele(d);
 		nni_aio_finish_error(aio, rv);
 		return;
 	}
-	if ((rv = nni_posix_ipc_init(&c, pfd)) != 0) {
-		nni_posix_pfd_fini(pfd);
-		nni_aio_finish_error(aio, rv);
-		return;
+
+	// This arranges for the fd to be in non-blocking mode, and adds the
+	// poll fd to the list.
+	if ((rv = nni_posix_pfd_init(&pfd, fd)) != 0) {
+		goto error;
 	}
-	c->dialer = d;
+
+	nni_posix_ipc_init(c, pfd);
 	nni_posix_pfd_set_cb(pfd, ipc_dialer_cb, c);
 
 	nni_mtx_lock(&d->mtx);
-	d->refcnt++;
 	if (d->closed) {
 		rv = NNG_ECLOSED;
 		goto error;
@@ -209,7 +202,7 @@ ipc_dialer_dial(void *arg, nni_aio *aio)
 	if ((rv = nni_aio_schedule(aio, ipc_dialer_cancel, d)) != 0) {
 		goto error;
 	}
-	if (connect(fd, (void *) &ss, sslen) != 0) {
+	if (connect(fd, (void *) &ss, len) != 0) {
 		if (errno != EINPROGRESS) {
 			if (errno == ENOENT) {
 				// No socket present means nobody listening.
@@ -289,6 +282,9 @@ nni_ipc_dialer_alloc(nng_stream_dialer **dp, const nng_url *url)
 	d->sd.sd_dial  = ipc_dialer_dial;
 	d->sd.sd_getx  = ipc_dialer_getx;
 	d->sd.sd_setx  = ipc_dialer_setx;
+	nni_atomic_init_bool(&d->fini);
+	nni_atomic_init64(&d->ref);
+	nni_atomic_inc64(&d->ref);
 
 	*dp = (void *) d;
 	return (0);
diff --git a/src/platform/posix/posix_ipclisten.c b/src/platform/posix/posix_ipclisten.c
index da3eab7b..4104be99 100644
--- a/src/platform/posix/posix_ipclisten.c
+++ b/src/platform/posix/posix_ipclisten.c
@@ -1,5 +1,5 @@
 //
-// Copyright 2019 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2020 Staysail Systems, Inc. <info@staysail.tech>
 // Copyright 2018 Capitar IT Group BV <info@capitar.com>
 // Copyright 2019 Devolutions <info@devolutions.net>
 //
@@ -124,20 +124,22 @@ ipc_listener_doaccept(ipc_listener *l)
 			}
 		}
 
-		if ((rv = nni_posix_pfd_init(&pfd, newfd)) != 0) {
-			close(newfd);
+		if ((rv = nni_posix_ipc_alloc(&c, NULL)) != 0) {
+			(void) close(newfd);
 			nni_aio_list_remove(aio);
 			nni_aio_finish_error(aio, rv);
 			continue;
 		}
 
-		if ((rv = nni_posix_ipc_init(&c, pfd)) != 0) {
-			nni_posix_pfd_fini(pfd);
+		if ((rv = nni_posix_pfd_init(&pfd, newfd)) != 0) {
+			nng_stream_free(&c->stream);
 			nni_aio_list_remove(aio);
 			nni_aio_finish_error(aio, rv);
 			continue;
 		}
 
+		nni_posix_ipc_init(c, pfd);
+
 		nni_aio_list_remove(aio);
 		nni_posix_ipc_start(c);
 		nni_aio_set_output(aio, 0, c);
diff --git a/src/platform/posix/posix_pollq_epoll.c b/src/platform/posix/posix_pollq_epoll.c
index 8ec3bc3f..63cdaeb5 100644
--- a/src/platform/posix/posix_pollq_epoll.c
+++ b/src/platform/posix/posix_pollq_epoll.c
@@ -1,5 +1,5 @@
 //
-// Copyright 2019 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2020 Staysail Systems, Inc. <info@staysail.tech>
 // Copyright 2018 Capitar IT Group BV <info@capitar.com>
 // Copyright 2018 Liam Staskawicz <liam@stask.net>
 //
@@ -191,28 +191,27 @@ nni_posix_pfd_fini(nni_posix_pfd *pfd)
 
 	// We have to synchronize with the pollq thread (unless we are
 	// on that thread!)
-	if (!nni_thr_is_self(&pq->thr)) {
+	NNI_ASSERT(!nni_thr_is_self(&pq->thr));
 
-		uint64_t one = 1;
-
-		nni_mtx_lock(&pq->mtx);
-		nni_list_append(&pq->reapq, pfd);
+	uint64_t one = 1;
 
-		// Wake the remote side.  For now we assume this always
-		// succeeds.  The only failure modes here occur when we
-		// have already excessively signaled this (2^64 times
-		// with no read!!), or when the evfd is closed, or some
-		// kernel bug occurs.  Those errors would manifest as
-		// a hang waiting for the poller to reap the pfd in fini,
-		// if it were possible for them to occur.  (Barring other
-		// bugs, it isn't.)
-		(void) write(pq->evfd, &one, sizeof(one));
+	nni_mtx_lock(&pq->mtx);
+	nni_list_append(&pq->reapq, pfd);
+
+	// Wake the remote side.  For now we assume this always
+	// succeeds.  The only failure modes here occur when we
+	// have already excessively signaled this (2^64 times
+	// with no read!!), or when the evfd is closed, or some
+	// kernel bug occurs.  Those errors would manifest as
+	// a hang waiting for the poller to reap the pfd in fini,
+	// if it were possible for them to occur.  (Barring other
+	// bugs, it isn't.)
+	(void) write(pq->evfd, &one, sizeof(one));
 
-		while (!pfd->closed) {
-			nni_cv_wait(&pfd->cv);
-		}
-		nni_mtx_unlock(&pq->mtx);
+	while (!pfd->closed) {
+		nni_cv_wait(&pfd->cv);
 	}
+	nni_mtx_unlock(&pq->mtx);
 
 	// We're exclusive now.
 
diff --git a/src/platform/posix/posix_pollq_kqueue.c b/src/platform/posix/posix_pollq_kqueue.c
index 72d306c7..299479ab 100644
--- a/src/platform/posix/posix_pollq_kqueue.c
+++ b/src/platform/posix/posix_pollq_kqueue.c
@@ -1,5 +1,5 @@
 //
-// Copyright 2019 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2020 Staysail Systems, Inc. <info@staysail.tech>
 // Copyright 2018 Capitar IT Group BV <info@capitar.com>
 // Copyright 2018 Liam Staskawicz <liam@stask.net>
 //
@@ -124,22 +124,24 @@ nni_posix_pfd_fini(nni_posix_pfd *pf)
 
 	nni_posix_pfd_close(pf);
 
-	if (!nni_thr_is_self(&pq->thr)) {
-		struct kevent ev;
-		nni_mtx_lock(&pq->mtx);
-		nni_list_append(&pq->reapq, pf);
-		EV_SET(&ev, 0, EVFILT_USER, EV_ENABLE | EV_CLEAR, NOTE_TRIGGER,
-		    0, NULL);
-
-		// If this fails, the cleanup will stall.  That should
-		// only occur in a memory pressure situation, and it
-		// will self-heal when the next event comes in.
-		(void) kevent(pq->kq, &ev, 1, NULL, 0, NULL);
-		while (!pf->closed) {
-			nni_cv_wait(&pf->cv);
-		}
-		nni_mtx_unlock(&pq->mtx);
+	// All consumers take care to move finalization to the reap thread,
+	// unless they are synchronous on user threads.
+	NNI_ASSERT(!nni_thr_is_self(&pq->thr));
+
+	struct kevent ev;
+	nni_mtx_lock(&pq->mtx);
+	nni_list_append(&pq->reapq, pf);
+	EV_SET(
+	    &ev, 0, EVFILT_USER, EV_ENABLE | EV_CLEAR, NOTE_TRIGGER, 0, NULL);
+
+	// If this fails, the cleanup will stall.  That should
+	// only occur in a memory pressure situation, and it
+	// will self-heal when the next event comes in.
+	(void) kevent(pq->kq, &ev, 1, NULL, 0, NULL);
+	while (!pf->closed) {
+		nni_cv_wait(&pf->cv);
 	}
+	nni_mtx_unlock(&pq->mtx);
 
 	(void) close(pf->fd);
 	nni_cv_fini(&pf->cv);
diff --git a/src/platform/posix/posix_pollq_port.c b/src/platform/posix/posix_pollq_port.c
index 0a1110bf..bf628e47 100644
--- a/src/platform/posix/posix_pollq_port.c
+++ b/src/platform/posix/posix_pollq_port.c
@@ -105,22 +105,21 @@ nni_posix_pfd_fini(nni_posix_pfd *pfd)
 
 	nni_posix_pfd_close(pfd);
 
-	if (!nni_thr_is_self(&pq->thr)) {
+	NNI_ASSERT(!nni_thr_is_self(&pq->thr));
 
-		while (port_send(pq->port, 1, pfd) != 0) {
-			if ((errno == EBADF) || (errno == EBADFD)) {
-				pfd->closed = true;
-				break;
-			}
-			sched_yield(); // try again later...
+	while (port_send(pq->port, 1, pfd) != 0) {
+		if ((errno == EBADF) || (errno == EBADFD)) {
+			pfd->closed = true;
+			break;
 		}
+		sched_yield(); // try again later...
+	}
 
-		nni_mtx_lock(&pfd->mtx);
-		while (!pfd->closed) {
-			nni_cv_wait(&pfd->cv);
-		}
-		nni_mtx_unlock(&pfd->mtx);
+	nni_mtx_lock(&pfd->mtx);
+	while (!pfd->closed) {
+		nni_cv_wait(&pfd->cv);
 	}
+	nni_mtx_unlock(&pfd->mtx);
 
 	// We're exclusive now.
 	(void) close(pfd->fd);
diff --git a/src/platform/posix/posix_resolv_gai.c b/src/platform/posix/posix_resolv_gai.c
index 4b03a95e..a89c4623 100644
--- a/src/platform/posix/posix_resolv_gai.c
+++ b/src/platform/posix/posix_resolv_gai.c
@@ -42,6 +42,7 @@ typedef struct resolv_item resolv_item;
 struct resolv_item {
 	int          family;
 	int          passive;
+	char         name_buf[256];
 	char *       name;
 	int          proto;
 	int          socktype;
@@ -67,7 +68,6 @@ resolv_cancel(nni_aio *aio, void *arg, int rv)
 		// so we can just discard everything.
 		nni_aio_list_remove(aio);
 		nni_mtx_unlock(&resolv_mtx);
-		nni_strfree(item->name);
 		NNI_FREE_STRUCT(item);
 	} else {
 		// This case indicates the resolver is still processing our
@@ -255,14 +255,18 @@ resolv_ip(const char *host, const char *serv, int passive, int family,
 	}
 
 	// NB: must remain valid until this is completed.  So we have to
-	// make our own copy.
+	// keep our own copy.
 
 	if (host == NULL) {
 		item->name = NULL;
-	} else if ((item->name = nni_strdup(host)) == NULL) {
+
+	} else if (nni_strnlen(host, sizeof(item->name_buf)) >=
+	    sizeof(item->name_buf)) {
 		NNI_FREE_STRUCT(item);
-		nni_aio_finish_error(aio, NNG_ENOMEM);
-		return;
+		nni_aio_finish_error(aio, NNG_EADDRINVAL);
+	} else {
+		nni_strlcpy(item->name_buf, host, sizeof(item->name_buf));
+		item->name = item->name_buf;
 	}
 
 	memset(&item->sa, 0, sizeof(item->sa));
@@ -282,7 +286,6 @@ resolv_ip(const char *host, const char *serv, int passive, int family,
 	}
 	if (rv != 0) {
 		nni_mtx_unlock(&resolv_mtx);
-		nni_strfree(item->name);
 		NNI_FREE_STRUCT(item);
 		nni_aio_finish_error(aio, rv);
 		return;
@@ -343,7 +346,6 @@ resolv_worker(void *unused)
 			nni_aio_set_sockaddr(aio, &item->sa);
 			nni_aio_finish(aio, rv, 0);
 		}
-		nni_strfree(item->name);
 		NNI_FREE_STRUCT(item);
 	}
 	nni_mtx_unlock(&resolv_mtx);
diff --git a/src/platform/posix/posix_tcp.h b/src/platform/posix/posix_tcp.h
index e1a70bc3..87312dff 100644
--- a/src/platform/posix/posix_tcp.h
+++ b/src/platform/posix/posix_tcp.h
@@ -1,5 +1,5 @@
 //
-// Copyright 2019 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2020 Staysail Systems, Inc. <info@staysail.tech>
 // Copyright 2018 Capitar IT Group BV <info@capitar.com>
 // Copyright 2018 Devolutions <info@devolutions.net>
 //
@@ -27,7 +27,21 @@ struct nni_tcp_conn {
 	nni_tcp_dialer *dialer;
 	nni_reap_item   reap;
 };
-extern int  nni_posix_tcp_init(nni_tcp_conn **, nni_posix_pfd *);
+
+struct nni_tcp_dialer {
+	nni_list                connq; // pending connections
+	bool                    closed;
+	bool                    nodelay;
+	bool                    keepalive;
+	struct sockaddr_storage src;
+	size_t                  srclen;
+	nni_mtx                 mtx;
+	nni_atomic_u64          ref;
+	nni_atomic_bool         fini;
+};
+
+extern int  nni_posix_tcp_alloc(nni_tcp_conn **, nni_tcp_dialer *);
+extern void nni_posix_tcp_init(nni_tcp_conn *, nni_posix_pfd *);
 extern void nni_posix_tcp_start(nni_tcp_conn *, int, int);
 extern void nni_posix_tcp_dialer_rele(nni_tcp_dialer *);
 
diff --git a/src/platform/posix/posix_tcpconn.c b/src/platform/posix/posix_tcpconn.c
index 625fd7fd..2a209984 100644
--- a/src/platform/posix/posix_tcpconn.c
+++ b/src/platform/posix/posix_tcpconn.c
@@ -1,5 +1,5 @@
 //
-// Copyright 2019 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2020 Staysail Systems, Inc. <info@staysail.tech>
 // Copyright 2018 Capitar IT Group BV <info@capitar.com>
 // Copyright 2019 Devolutions <info@devolutions.net>
 //
@@ -173,7 +173,9 @@ tcp_error(void *arg, int err)
 		nni_aio_list_remove(aio);
 		nni_aio_finish_error(aio, err);
 	}
-	nni_posix_pfd_close(c->pfd);
+	if (c->pfd != NULL) {
+		nni_posix_pfd_close(c->pfd);
+	}
 	nni_mtx_unlock(&c->mtx);
 }
 
@@ -190,7 +192,9 @@ tcp_close(void *arg)
 			nni_aio_list_remove(aio);
 			nni_aio_finish_error(aio, NNG_ECLOSED);
 		}
-		nni_posix_pfd_close(c->pfd);
+		if (c->pfd != NULL) {
+			nni_posix_pfd_close(c->pfd);
+		}
 	}
 	nni_mtx_unlock(&c->mtx);
 }
@@ -202,10 +206,9 @@ tcp_fini(void *arg)
 {
 	nni_tcp_conn *c = arg;
 	tcp_close(c);
-	nni_posix_pfd_fini(c->pfd);
-	nni_mtx_lock(&c->mtx); // not strictly needed, but shut up TSAN
-	c->pfd = NULL;
-	nni_mtx_unlock(&c->mtx);
+	if (c->pfd != NULL) {
+		nni_posix_pfd_fini(c->pfd);
+	}
 	nni_mtx_fini(&c->mtx);
 
 	if (c->dialer != NULL) {
@@ -474,16 +477,15 @@ tcp_setx(void *arg, const char *name, const void *buf, size_t sz, nni_type t)
 }
 
 int
-nni_posix_tcp_init(nni_tcp_conn **cp, nni_posix_pfd *pfd)
+nni_posix_tcp_alloc(nni_tcp_conn **cp, nni_tcp_dialer *d)
 {
 	nni_tcp_conn *c;
-
 	if ((c = NNI_ALLOC_STRUCT(c)) == NULL) {
 		return (NNG_ENOMEM);
 	}
 
 	c->closed = false;
-	c->pfd    = pfd;
+	c->dialer = d;
 
 	nni_mtx_init(&c->mtx);
 	nni_aio_list_init(&c->readq);
@@ -501,6 +503,12 @@ nni_posix_tcp_init(nni_tcp_conn **cp, nni_posix_pfd *pfd)
 }
 
 void
+nni_posix_tcp_init(nni_tcp_conn *c, nni_posix_pfd *pfd)
+{
+	c->pfd = pfd;
+}
+
+void
 nni_posix_tcp_start(nni_tcp_conn *c, int nodelay, int keepalive)
 {
 	// Configure the initial socket options.
diff --git a/src/platform/posix/posix_tcpdial.c b/src/platform/posix/posix_tcpdial.c
index 418eb17e..3fabc28a 100644
--- a/src/platform/posix/posix_tcpdial.c
+++ b/src/platform/posix/posix_tcpdial.c
@@ -1,5 +1,5 @@
 //
-// Copyright 2019 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2020 Staysail Systems, Inc. <info@staysail.tech>
 // Copyright 2018 Capitar IT Group BV <info@capitar.com>
 // Copyright 2018 Devolutions <info@devolutions.net>
 //
@@ -23,18 +23,6 @@
 
 #include "posix_tcp.h"
 
-struct nni_tcp_dialer {
-	nni_list                connq; // pending connections
-	bool                    closed;
-	bool                    nodelay;
-	bool                    keepalive;
-	struct sockaddr_storage src;
-	size_t                  srclen;
-	nni_mtx                 mtx;
-	int                     refcnt;
-	bool                    fini;
-};
-
 // Dialer stuff.
 int
 nni_tcp_dialer_init(nni_tcp_dialer **dp)
@@ -47,6 +35,9 @@ nni_tcp_dialer_init(nni_tcp_dialer **dp)
 	nni_mtx_init(&d->mtx);
 	d->closed = false;
 	nni_aio_list_init(&d->connq);
+	nni_atomic_init_bool(&d->fini);
+	nni_atomic_init64(&d->ref);
+	nni_atomic_inc64(&d->ref);
 	*dp = d;
 	return (0);
 }
@@ -84,26 +75,17 @@ void
 nni_tcp_dialer_fini(nni_tcp_dialer *d)
 {
 	nni_tcp_dialer_close(d);
-	nni_mtx_lock(&d->mtx);
-	d->fini = true;
-	if (d->refcnt) {
-		nni_mtx_unlock(&d->mtx);
-		return;
-	}
-	nni_mtx_unlock(&d->mtx);
-	tcp_dialer_fini(d);
+	nni_atomic_set_bool(&d->fini, true);
+	nni_posix_tcp_dialer_rele(d);
 }
 
 void
 nni_posix_tcp_dialer_rele(nni_tcp_dialer *d)
 {
-	nni_mtx_lock(&d->mtx);
-	d->refcnt--;
-	if ((d->refcnt > 0) || (!d->fini)) {
-		nni_mtx_unlock(&d->mtx);
+	if (((nni_atomic_dec64_nv(&d->ref) != 0)) ||
+	    (!nni_atomic_get_bool(&d->fini))) {
 		return;
 	}
-	nni_mtx_unlock(&d->mtx);
 	tcp_dialer_fini(d);
 }
 
@@ -215,23 +197,25 @@ nni_tcp_dial(nni_tcp_dialer *d, nni_aio *aio)
 		return;
 	}
 
+	nni_atomic_inc64(&d->ref);
+
+	if ((rv = nni_posix_tcp_alloc(&c, d)) != 0) {
+		nni_aio_finish_error(aio, rv);
+		nni_posix_tcp_dialer_rele(d);
+		return;
+	}
+
 	// This arranges for the fd to be in non-blocking mode, and adds the
 	// poll fd to the list.
 	if ((rv = nni_posix_pfd_init(&pfd, fd)) != 0) {
 		(void) close(fd);
-		nni_aio_finish_error(aio, rv);
-		return;
-	}
-	if ((rv = nni_posix_tcp_init(&c, pfd)) != 0) {
-		nni_posix_pfd_fini(pfd);
-		nni_aio_finish_error(aio, rv);
-		return;
+		goto error;
 	}
-	c->dialer = d;
+
+	nni_posix_tcp_init(c, pfd);
 	nni_posix_pfd_set_cb(pfd, tcp_dialer_cb, c);
 
 	nni_mtx_lock(&d->mtx);
-	d->refcnt++;
 	if (d->closed) {
 		rv = NNG_ECLOSED;
 		goto error;
diff --git a/src/platform/posix/posix_tcplisten.c b/src/platform/posix/posix_tcplisten.c
index 10867453..a463a198 100644
--- a/src/platform/posix/posix_tcplisten.c
+++ b/src/platform/posix/posix_tcplisten.c
@@ -1,5 +1,5 @@
 //
-// Copyright 2018 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2020 Staysail Systems, Inc. <info@staysail.tech>
 // Copyright 2018 Capitar IT Group BV <info@capitar.com>
 // Copyright 2018 Devolutions <info@devolutions.net>
 //
@@ -136,20 +136,23 @@ tcp_listener_doaccept(nni_tcp_listener *l)
 			}
 		}
 
-		if ((rv = nni_posix_pfd_init(&pfd, newfd)) != 0) {
+		if ((rv = nni_posix_tcp_alloc(&c, NULL)) != 0) {
 			close(newfd);
 			nni_aio_list_remove(aio);
 			nni_aio_finish_error(aio, rv);
 			continue;
 		}
 
-		if ((rv = nni_posix_tcp_init(&c, pfd)) != 0) {
-			nni_posix_pfd_fini(pfd);
+		if ((rv = nni_posix_pfd_init(&pfd, newfd)) != 0) {
+			close(newfd);
+			nng_stream_free(&c->stream);
 			nni_aio_list_remove(aio);
 			nni_aio_finish_error(aio, rv);
 			continue;
 		}
 
+		nni_posix_tcp_init(c, pfd);
+
 		ka = l->keepalive ? 1 : 0;
 		nd = l->nodelay ? 1 : 0;
 		nni_aio_list_remove(aio);
diff --git a/src/supplemental/tcp/tcp.c b/src/supplemental/tcp/tcp.c
index 02a3351f..02d5d6ce 100644
--- a/src/supplemental/tcp/tcp.c
+++ b/src/supplemental/tcp/tcp.c
@@ -155,12 +155,15 @@ tcp_dialer_free(void *arg)
 		return;
 	}
 
+	nni_aio_stop(d->resaio);
+	nni_aio_stop(d->conaio);
+	nni_aio_fini(d->resaio);
+	nni_aio_fini(d->conaio);
+
 	if (d->d != NULL) {
 		nni_tcp_dialer_close(d->d);
 		nni_tcp_dialer_fini(d->d);
 	}
-	nni_aio_fini(d->resaio);
-	nni_aio_fini(d->conaio);
 	nni_mtx_fini(&d->mtx);
 	nni_strfree(d->host);
 	nni_strfree(d->port);
diff --git a/src/supplemental/tls/mbedtls/tls.c b/src/supplemental/tls/mbedtls/tls.c
index b7ed0575..d49c9de5 100644
--- a/src/supplemental/tls/mbedtls/tls.c
+++ b/src/supplemental/tls/mbedtls/tls.c
@@ -1,5 +1,5 @@
 //
-// Copyright 2019 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2020 Staysail Systems, Inc. <info@staysail.tech>
 // Copyright 2018 Capitar IT Group BV <info@capitar.com>
 // Copyright 2019 Devolutions <info@devolutions.net>
 //
@@ -81,6 +81,7 @@ typedef struct {
 	nni_list            sends;      // upper side sends
 	nni_list            recvs;      // upper recv aios
 	nni_aio *           handshake;  // handshake aio (upper)
+	nni_reap_item       reap;
 } tls;
 
 struct nng_tls_config {
@@ -279,36 +280,41 @@ tls_mkerr(int err)
 // The common code should call this only after it has released
 // it's upper layer stuff.
 static void
-tls_free(void *arg)
+tls_reap(void *arg)
 {
 	tls *tls = arg;
 
 	// Shut it all down first.
-	if (tls != NULL) {
-		if (tls->tcp != NULL) {
-			nng_stream_close(tls->tcp);
-		}
-		nni_aio_stop(tls->tcp_send);
-		nni_aio_stop(tls->tcp_recv);
-		nni_aio_fini(tls->com.aio);
-
-		// And finalize / free everything.
-		nng_stream_free(tls->tcp);
-		nni_aio_fini(tls->tcp_send);
-		nni_aio_fini(tls->tcp_recv);
-		mbedtls_ssl_free(&tls->ctx);
-		nng_tls_config_free(tls->com.cfg);
-
-		if (tls->recvbuf != NULL) {
-			nni_free(tls->recvbuf, NNG_TLS_MAX_RECV_SIZE);
-		}
-		if (tls->sendbuf != NULL) {
-			nni_free(tls->sendbuf, NNG_TLS_MAX_RECV_SIZE);
-		}
-		nni_mtx_fini(&tls->lk);
-		memset(tls, 0xff, sizeof(*tls));
-		NNI_FREE_STRUCT(tls);
+	if (tls->tcp != NULL) {
+		nng_stream_close(tls->tcp);
+	}
+	nni_aio_stop(tls->tcp_send);
+	nni_aio_stop(tls->tcp_recv);
+	nni_aio_fini(tls->com.aio);
+
+	// And finalize / free everything.
+	nng_stream_free(tls->tcp);
+	nni_aio_fini(tls->tcp_send);
+	nni_aio_fini(tls->tcp_recv);
+	mbedtls_ssl_free(&tls->ctx);
+	nng_tls_config_free(tls->com.cfg);
+
+	if (tls->recvbuf != NULL) {
+		nni_free(tls->recvbuf, NNG_TLS_MAX_RECV_SIZE);
+	}
+	if (tls->sendbuf != NULL) {
+		nni_free(tls->sendbuf, NNG_TLS_MAX_RECV_SIZE);
 	}
+	nni_mtx_fini(&tls->lk);
+	memset(tls, 0xff, sizeof(*tls));
+	NNI_FREE_STRUCT(tls);
+}
+
+static void
+tls_free(void *arg)
+{
+	tls *tls = arg;
+	nni_reap(&tls->reap, tls_reap, tls);
 }
 
 int