fixes #484 crashes in websocket transport

fixes #490 posix_epdesc use-after-free bug
fixes #489 Sanitizer based testing would help
fixes #492 Numerous memory leaks found with sanitizer

This introduces support for compiler-based sanitizers when using
clang or gcc (and not on Windows).  See NNG_SANITIZER for possible
settings such as "thread" or "address".

Furthermore, we have fixed the issues we found with both the
thread and address sanitizers.  We believe that the thread issues
pointed to a low frequency use-after-free responsible for rare
crashes in some of the tests.

The tests generally have their timeouts doubled when running under
a sanitizer, to account for the extra long times that the sanitizer
can cause these to take.

While here, we also changed the compat_ws test to avoid a particularly
painful and time consuming DNS lookup, and we made the nngcat_unlimited
test a bit more robust by waiting before sending traffic.

20 files changed, 204 insertions, 132 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 60b1726c..36ba269f 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -239,11 +239,17 @@ endif()
 #  Platform checks.
 
 if (CMAKE_C_COMPILER_ID STREQUAL "GNU")
-	set(NNG_WARN_FLAGS "-Wall -Wextra")
+	set(NNG_WARN_FLAGS "-Wall -Wextra -fno-omit-frame-pointer")
 elseif (CMAKE_C_COMPILER_ID STREQUAL "Clang")
-    set(NNG_WARN_FLAGS "-Wall -Wextra")
+    set(NNG_WARN_FLAGS "-Wall -Wextra -fno-omit-frame-pointer")
 elseif (CMAKE_C_COMPILER_ID STREQUAL "AppleClang")
-    set(NNG_WARN_FLAGS "-Wall -Wextra")
+    set(NNG_WARN_FLAGS "-Wall -Wextra -fno-omit-frame-pointer")
+endif()
+
+include (CheckSanitizer)
+CheckSanitizer()
+if (NOT NNG_SANITIZER STREQUAL "none")
+    set (NNG_SANITIZER_FLAGS "-fsanitize=${NNG_SANITIZER}")
 endif()
 
 if (NNG_ENABLE_COVERAGE)
@@ -261,8 +267,8 @@ if (NNG_ENABLE_COVERAGE)
     endif()
 endif()
 
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${NNG_WARN_FLAGS} ${NNG_COVERAGE_FLAGS}")
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${NNG_WARN_FLAGS} ${NNG_COVERAGE_FLAGS}")
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${NNG_WARN_FLAGS} ${NNG_COVERAGE_FLAGS} ${NNG_SANITIZER_FLAGS}")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${NNG_WARN_FLAGS} ${NNG_COVERAGE_FLAGS} ${NNG_SANITIZER_FLAGS}")
 
 TEST_BIG_ENDIAN(NNG_BIG_ENDIAN)
 if (NNG_BIG_ENDIAN)
@@ -280,6 +286,8 @@ if (NOT WIN32 AND NOT CYGWIN)
     endif()
 endif()
 
+
+
 find_package (Threads REQUIRED)
 
 if (CMAKE_SYSTEM_NAME MATCHES "Linux")
diff --git a/cmake/CheckSanitizer.cmake b/cmake/CheckSanitizer.cmake
new file mode 100644
index 00000000..a6b3501f
--- /dev/null
+++ b/cmake/CheckSanitizer.cmake
@@ -0,0 +1,30 @@
+#
+# Copyright 2018 Staysail Systems, Inc. <info@staysail.tech>
+# Copyright 2017 Capitar IT Group BV <info@capitar.com>
+#
+# This software is supplied under the terms of the MIT License, a
+# copy of which should be located in the distribution where this
+# file was obtained (LICENSE.txt).  A copy of the license may also be
+# found online at https://opensource.org/licenses/MIT.
+#
+
+macro (CheckSanitizer)
+
+    if (CMAKE_C_COMPILER_ID STREQUAL "GNU")
+        set(NNG_SAN_LIST none address leak memory thread undefined)
+    elseif (CMAKE_C_COMPILER_ID STREQUAL "Clang")
+        set(NNG_SAN_LIST none address leak memory thread undefined)
+    elseif (CMAKE_C_COMPILER_ID STREQUAL "AppleClang")
+        set(NNG_SAN_LIST none address thread undefined) 
+    else ()
+        set(NNG_SAN_LIST none)
+    endif ()
+    set (NNG_SANITIZER none CACHE STRING "Sanitizer to use (clang or gcc).")
+    set_property(CACHE NNG_SANITIZER PROPERTY STRINGS ${NNG_SAN_LIST})
+    mark_as_advanced (NNG_SANITIZER)
+
+    if (NOT NNG_SANITIZER STREQUAL "none")
+        set (NNG_C_FLAG_SANITIZER "-fsanitizer=${NNG_SANITIZER}")
+        message(STATUS "Enabling sanitizer: ${NNG_C_FLAG_SANITIZER}")
+    endif()
+endmacro ()
diff --git a/src/platform/posix/posix_epdesc.c b/src/platform/posix/posix_epdesc.c
index dfb750d4..69026051 100644
--- a/src/platform/posix/posix_epdesc.c
+++ b/src/platform/posix/posix_epdesc.c
@@ -469,11 +469,7 @@ nni_epdesc_connect_cb(nni_posix_pfd *pfd, int events, void *arg)
 	nni_mtx_lock(&ed->mtx);
 	if ((ed->closed) || ((aio = nni_list_first(&ed->connectq)) == NULL) ||
 	    (pfd != ed->pfd)) {
-		// Spurious completion.  Ignore it, but discard the PFD.
-		if (ed->pfd == pfd) {
-			ed->pfd = NULL;
-		}
-		nni_posix_pfd_fini(pfd);
+		// Spurious completion.  Just ignore it.
 		nni_mtx_unlock(&ed->mtx);
 		return;
 	}
diff --git a/src/platform/posix/posix_pollq_epoll.c b/src/platform/posix/posix_pollq_epoll.c
index a8d8693a..9c1ae682 100644
--- a/src/platform/posix/posix_pollq_epoll.c
+++ b/src/platform/posix/posix_pollq_epoll.c
@@ -93,6 +93,10 @@ nni_posix_pfd_init(nni_posix_pfd **pfdp, int fd)
 	if ((pfd = NNI_ALLOC_STRUCT(pfd)) == NULL) {
 		return (NNG_ENOMEM);
 	}
+	nni_mtx_init(&pfd->mtx);
+	nni_cv_init(&pfd->cv, &pq->mtx);
+
+	nni_mtx_lock(&pfd->mtx);
 	pfd->pq      = pq;
 	pfd->fd      = fd;
 	pfd->cb      = NULL;
@@ -101,9 +105,8 @@ nni_posix_pfd_init(nni_posix_pfd **pfdp, int fd)
 	pfd->closing = false;
 	pfd->closed  = false;
 
-	nni_mtx_init(&pfd->mtx);
-	nni_cv_init(&pfd->cv, &pq->mtx);
 	NNI_LIST_NODE_INIT(&pfd->node);
+	nni_mtx_unlock(&pfd->mtx);
 
 	// notifications disabled to begin with
 	ev.events   = 0;
diff --git a/src/platform/posix/posix_pollq_kqueue.c b/src/platform/posix/posix_pollq_kqueue.c
index 36ced3ff..c118d528 100644
--- a/src/platform/posix/posix_pollq_kqueue.c
+++ b/src/platform/posix/posix_pollq_kqueue.c
@@ -276,9 +276,9 @@ nni_posix_pollq_destroy(nni_posix_pollq *pq)
 {
 	if (pq->kq >= 0) {
 		close(pq->kq);
-		pq->kq = -1;
 	}
 	nni_thr_fini(&pq->thr);
+	pq->kq = -1;
 
 	nni_posix_pollq_reap(pq);
 
diff --git a/src/protocol/reqrep0/rep.c b/src/protocol/reqrep0/rep.c
index 24fc7335..33c9efcd 100644
--- a/src/protocol/reqrep0/rep.c
+++ b/src/protocol/reqrep0/rep.c
@@ -309,6 +309,12 @@ static void
 rep0_pipe_fini(void *arg)
 {
 	rep0_pipe *p = arg;
+	nng_msg *  msg;
+
+	if ((msg = nni_aio_get_msg(p->aio_recv)) != NULL) {
+		nni_aio_set_msg(p->aio_recv, NULL);
+		nni_msg_free(msg);
+	}
 
 	nni_aio_fini(p->aio_send);
 	nni_aio_fini(p->aio_recv);
@@ -538,6 +544,7 @@ rep0_pipe_recv_cb(void *arg)
 		if (nni_msg_len(msg) < 4) {
 			// Peer is speaking garbage. Kick it.
 			nni_msg_free(msg);
+			nni_aio_set_msg(p->aio_recv, NULL);
 			nni_pipe_stop(p->pipe);
 			return;
 		}
@@ -593,6 +600,7 @@ rep0_pipe_recv_cb(void *arg)
 
 drop:
 	nni_msg_free(msg);
+	nni_aio_set_msg(p->aio_recv, NULL);
 	nni_pipe_recv(p->pipe, p->aio_recv);
 }
 
diff --git a/src/protocol/survey0/respond.c b/src/protocol/survey0/respond.c
index 5f833ca6..569fd4ea 100644
--- a/src/protocol/survey0/respond.c
+++ b/src/protocol/survey0/respond.c
@@ -302,7 +302,12 @@ static void
 resp0_pipe_fini(void *arg)
 {
 	resp0_pipe *p = arg;
+	nng_msg *   msg;
 
+	if ((msg = nni_aio_get_msg(p->aio_recv)) != NULL) {
+		nni_aio_set_msg(p->aio_recv, NULL);
+		nni_msg_free(msg);
+	}
 	nni_aio_fini(p->aio_send);
 	nni_aio_fini(p->aio_recv);
 	NNI_FREE_STRUCT(p);
@@ -527,6 +532,7 @@ resp0_pipe_recv_cb(void *arg)
 		if (nni_msg_len(msg) < 4) {
 			// Peer is speaking garbage, kick it.
 			nni_msg_free(msg);
+			nni_aio_set_msg(p->aio_recv, NULL);
 			nni_pipe_stop(p->npipe);
 			return;
 		}
@@ -576,6 +582,7 @@ resp0_pipe_recv_cb(void *arg)
 
 drop:
 	nni_msg_free(msg);
+	nni_aio_set_msg(p->aio_recv, NULL);
 	nni_pipe_recv(p->npipe, p->aio_recv);
 }
 
diff --git a/src/supplemental/websocket/websocket.c b/src/supplemental/websocket/websocket.c
index 18491190..55befdb2 100644
--- a/src/supplemental/websocket/websocket.c
+++ b/src/supplemental/websocket/websocket.c
@@ -1968,7 +1968,9 @@ nni_ws_dialer_close(nni_ws_dialer *d)
 	d->closed = true;
 	while ((ws = nni_list_first(&d->wspend)) != 0) {
 		nni_list_remove(&d->wspend, ws);
-		nni_ws_close(ws);
+		nni_mtx_unlock(&d->mtx);
+		nni_ws_fini(ws);
+		nni_mtx_lock(&d->mtx);
 	}
 	nni_mtx_unlock(&d->mtx);
 }
diff --git a/src/transport/ipc/ipc.c b/src/transport/ipc/ipc.c
index 016e47ec..88e11fb1 100644
--- a/src/transport/ipc/ipc.c
+++ b/src/transport/ipc/ipc.c
@@ -803,7 +803,9 @@ nni_ipc_ep_setopt_recvmaxsz(void *arg, const void *data, size_t sz, int typ)
 
 	rv = nni_copyin_size(&val, data, sz, 0, NNI_MAXSZ, typ);
 	if ((rv == 0) && (ep != NULL)) {
+		nni_mtx_lock(&ep->mtx);
 		ep->rcvmax = val;
+		nni_mtx_unlock(&ep->mtx);
 	}
 	return (rv);
 }
@@ -833,7 +835,9 @@ nni_ipc_ep_setopt_permissions(void *arg, const void *data, size_t sz, int typ)
 	// meaningful chmod beyond the lower 9 bits.
 	rv = nni_copyin_int(&val, data, sz, 0, 0x7FFFFFFF, typ);
 	if ((rv == 0) && (ep != NULL)) {
+		nni_mtx_lock(&ep->mtx);
 		rv = nni_plat_ipc_ep_set_permissions(ep->iep, val);
+		nni_mtx_unlock(&ep->mtx);
 	}
 	return (rv);
 }
diff --git a/src/transport/tcp/tcp.c b/src/transport/tcp/tcp.c
index 419774e1..be0dd2b5 100644
--- a/src/transport/tcp/tcp.c
+++ b/src/transport/tcp/tcp.c
@@ -836,7 +836,9 @@ nni_tcp_ep_setopt_recvmaxsz(void *arg, const void *v, size_t sz, int typ)
 	int         rv;
 	rv = nni_copyin_size(&val, v, sz, 0, NNI_MAXSZ, typ);
 	if ((rv == 0) && (ep != NULL)) {
+		nni_mtx_lock(&ep->mtx);
 		ep->rcvmax = val;
+		nni_mtx_unlock(&ep->mtx);
 	}
 	return (rv);
 }
@@ -849,7 +851,9 @@ nni_tcp_ep_setopt_nodelay(void *arg, const void *v, size_t sz, int typ)
 	int         rv;
 	rv = nni_copyin_bool(&val, v, sz, typ);
 	if ((rv == 0) && (ep != NULL)) {
+		nni_mtx_lock(&ep->mtx);
 		ep->nodelay = val;
+		nni_mtx_unlock(&ep->mtx);
 	}
 	return (rv);
 }
@@ -869,7 +873,9 @@ nni_tcp_ep_setopt_keepalive(void *arg, const void *v, size_t sz, int typ)
 	int         rv;
 	rv = nni_copyin_bool(&val, v, sz, typ);
 	if ((rv == 0) && (ep != NULL)) {
+		nni_mtx_lock(&ep->mtx);
 		ep->keepalive = val;
+		nni_mtx_unlock(&ep->mtx);
 	}
 	return (rv);
 }
diff --git a/src/transport/tls/tls.c b/src/transport/tls/tls.c
index 385dd206..d1d21f6d 100644
--- a/src/transport/tls/tls.c
+++ b/src/transport/tls/tls.c
@@ -850,7 +850,9 @@ nni_tls_ep_setopt_nodelay(void *arg, const void *v, size_t sz, int typ)
 	int         rv;
 	rv = nni_copyin_bool(&val, v, sz, typ);
 	if ((rv == 0) && (ep != NULL)) {
+		nni_mtx_lock(&ep->mtx);
 		ep->nodelay = val;
+		nni_mtx_unlock(&ep->mtx);
 	}
 	return (rv);
 }
@@ -870,7 +872,9 @@ nni_tls_ep_setopt_keepalive(void *arg, const void *v, size_t sz, int typ)
 	int         rv;
 	rv = nni_copyin_bool(&val, v, sz, typ);
 	if ((rv == 0) && (ep != NULL)) {
+		nni_mtx_lock(&ep->mtx);
 		ep->keepalive = val;
+		nni_mtx_unlock(&ep->mtx);
 	}
 	return (rv);
 }
@@ -907,7 +911,9 @@ nni_tls_ep_setopt_recvmaxsz(void *arg, const void *v, size_t sz, int typ)
 
 	rv = nni_copyin_size(&val, v, sz, 0, NNI_MAXSZ, typ);
 	if ((rv == 0) && (ep != NULL)) {
+		nni_mtx_lock(&ep->mtx);
 		ep->rcvmax = val;
+		nni_mtx_unlock(&ep->mtx);
 	}
 	return (rv);
 }
diff --git a/src/transport/ws/websocket.c b/src/transport/ws/websocket.c
index 97175d49..62f43fd0 100644
--- a/src/transport/ws/websocket.c
+++ b/src/transport/ws/websocket.c
@@ -375,11 +375,13 @@ ws_ep_setopt_recvmaxsz(void *arg, const void *v, size_t sz, int typ)
 
 	rv = nni_copyin_size(&val, v, sz, 0, NNI_MAXSZ, typ);
 	if ((rv == 0) && (ep != NULL)) {
+		nni_mtx_lock(&ep->mtx);
 		ep->rcvmax = val;
+		nni_mtx_unlock(&ep->mtx);
 		if (ep->mode == NNI_EP_MODE_DIAL) {
-			nni_ws_dialer_set_maxframe(ep->dialer, ep->rcvmax);
+			nni_ws_dialer_set_maxframe(ep->dialer, val);
 		} else {
-			nni_ws_listener_set_maxframe(ep->listener, ep->rcvmax);
+			nni_ws_listener_set_maxframe(ep->listener, val);
 		}
 	}
 	return (rv);
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt
index 9ba25882..4139c046 100644
--- a/tests/CMakeLists.txt
+++ b/tests/CMakeLists.txt
@@ -38,11 +38,17 @@ include_directories(AFTER SYSTEM ${PROJECT_SOURCE_DIR}/src)
 
 if (NNG_TESTS)
 
+    if (NOT NNG_SANITIZER STREQUAL "none")
+        set(TIMEOUT_FACTOR 2)
+    else()
+        set (TIMEOUT_FACTOR 1)
+    endif()
     set (NNG_TEST_PORT 13000)
     macro (add_nng_test NAME TIMEOUT)
         add_executable (${NAME} ${NAME}.c convey.c)
         target_link_libraries (${NAME} ${PROJECT_NAME} Threads::Threads)
         add_test (NAME ${NAME} COMMAND ${NAME} -v -p TEST_PORT=${NNG_TEST_PORT})
+        math (EXPR TIMEOUT ${TIMEOUT}*${TIMEOUT_FACTOR})
         set_tests_properties (${NAME} PROPERTIES TIMEOUT ${TIMEOUT})
         math (EXPR NNG_TEST_PORT "${NNG_TEST_PORT}+20")
     endmacro (add_nng_test)
@@ -181,21 +187,21 @@ add_nng_test2(surveypoll 5 NNG_PROTO_SURVEYOR0 NNG_PROTO_RESPONDENT0)
 # is because we don't want to make modifications to partially enable some
 # of these tests.  Folks minimizing the library probably don't care too
 # much about these anyway.
-add_nng_compat_test(compat_block 5)
-add_nng_compat_test(compat_bug777 5)
-add_nng_compat_test(compat_bus 5)
-add_nng_compat_test(compat_cmsg 5)
-add_nng_compat_test(compat_msg 5)
-add_nng_compat_test(compat_iovec 5)
-add_nng_compat_test(compat_device 5)
-add_nng_compat_test(compat_pair 5)
-add_nng_compat_test(compat_pipeline 5)
-add_nng_compat_test(compat_poll 5)
-add_nng_compat_test(compat_reqrep 5)
-add_nng_compat_test(compat_survey 5)
-add_nng_compat_test(compat_reqttl 5)
-add_nng_compat_test(compat_shutdown 5)
-add_nng_compat_test(compat_surveyttl 5)
+add_nng_compat_test(compat_block 10)
+add_nng_compat_test(compat_bug777 10)
+add_nng_compat_test(compat_bus 10)
+add_nng_compat_test(compat_cmsg 10)
+add_nng_compat_test(compat_msg 10)
+add_nng_compat_test(compat_iovec 10)
+add_nng_compat_test(compat_device 10)
+add_nng_compat_test(compat_pair 10)
+add_nng_compat_test(compat_pipeline 10)
+add_nng_compat_test(compat_poll 10)
+add_nng_compat_test(compat_reqrep 10)
+add_nng_compat_test(compat_survey 10)
+add_nng_compat_test(compat_reqttl 10)
+add_nng_compat_test(compat_shutdown 10)
+add_nng_compat_test(compat_surveyttl 10)
 add_nng_compat_test(compat_tcp 60)
 add_nng_compat_test(compat_ws 60)
 
diff --git a/tests/aio.c b/tests/aio.c
index 2aa96de5..bd5c5097 100644
--- a/tests/aio.c
+++ b/tests/aio.c
@@ -36,7 +36,6 @@ sleepdone(void *arg)
 }
 
 Main({
-
 	Test("AIO operations", {
 		const char *addr = "inproc://aio";
 
@@ -170,7 +169,6 @@ Main({
 				So(done == 1);
 				So(nng_aio_result(a) == NNG_ECANCELED);
 			})
-
 		});
 
 		Convey("We cannot set insane IOVs", {
@@ -179,6 +177,7 @@ Main({
 
 			So(nng_aio_alloc(&aio, NULL, NULL) == 0);
 			So(nng_aio_set_iov(aio, 1024, &iov) == NNG_EINVAL);
+			nng_aio_free(aio);
 		});
 	});
 
diff --git a/tests/compat_ws.c b/tests/compat_ws.c
index 6d019364..2c302712 100644
--- a/tests/compat_ws.c
+++ b/tests/compat_ws.c
@@ -156,7 +156,7 @@ int main (int argc, const char *argv[])
     rc = nn_connect (sc, "ws://:5555");
     nn_assert (rc < 0);
     errno_assert (nn_errno () == EINVAL);
-    rc = nn_connect (sc, "ws://-hostname:5555");
+    rc = nn_connect (sc, "ws://-hostname.:5555");
     nn_assert (rc < 0);
     errno_assert (nn_errno () == EINVAL);
     rc = nn_connect (sc, "ws://abc.123.---.#:5555");
diff --git a/tests/platform.c b/tests/platform.c
index 168f7879..51dc41fc 100644
--- a/tests/platform.c
+++ b/tests/platform.c
@@ -15,35 +15,24 @@
 
 #include "stubs.h"
 
-// Add is for testing threads.
-void
-add(void *arg)
-{
-	*(int *) arg += 1;
-}
-
-// Notify tests for verifying condvars.
-struct notifyarg {
-	int          did;
-	nng_duration when;
-	nng_mtx *    mx;
-	nng_cv *     cv;
+struct addarg {
+	int      cnt;
+	nng_mtx *mx;
+	nng_cv * cv;
 };
 
 void
-notifyafter(void *arg)
+add(void *arg)
 {
-	struct notifyarg *na = arg;
+	struct addarg *aa = arg;
 
-	nng_msleep(na->when);
-	nng_mtx_lock(na->mx);
-	na->did = 1;
-	nng_cv_wake(na->cv);
-	nng_mtx_unlock(na->mx);
+	nng_mtx_lock(aa->mx);
+	aa->cnt++;
+	nng_cv_wake(aa->cv);
+	nng_mtx_unlock(aa->mx);
 }
 
 TestMain("Platform Operations", {
-
 	// This is required for anything else to work
 	Convey("The clock works", {
 		uint64_t now = getms();
@@ -79,35 +68,46 @@ TestMain("Platform Operations", {
 		So(nng_mtx_alloc(&mx) == 0);
 		Reset({ nng_mtx_free(mx); });
 
-		Convey("We can lock a mutex", {
+		Convey("We can lock and unlock mutex", {
 			nng_mtx_lock(mx);
 			So(1);
-			Convey("And we can unlock it", {
+			nng_mtx_unlock(mx);
+			So(1);
+			Convey("And then lock it again", {
+				nng_mtx_lock(mx);
+				So(1);
 				nng_mtx_unlock(mx);
 				So(1);
-				Convey("And then lock it again", {
-					nng_mtx_lock(mx);
-					So(1);
-					nng_mtx_unlock(mx);
-					So(1);
-				});
 			});
 		});
 	});
 
 	Convey("Threads work", {
 		static nng_thread *thr;
-		int                val = 0;
 		int                rv;
+		struct addarg      aa;
+
+		So(nng_mtx_alloc(&aa.mx) == 0);
+		So(nng_cv_alloc(&aa.cv, aa.mx) == 0);
+		Reset({
+			nng_mtx_free(aa.mx);
+			nng_cv_free(aa.cv);
+		});
+		aa.cnt = 0;
 
 		Convey("We can create threads", {
-			rv = nng_thread_create(&thr, add, &val);
+			rv = nng_thread_create(&thr, add, &aa);
 			So(rv == 0);
 
 			Reset({ nng_thread_destroy(thr); });
 
 			Convey("It ran", {
-				nng_msleep(50); // for context switch
+				int val;
+				nng_mtx_lock(aa.mx);
+				while ((val = aa.cnt) == 0) {
+					nng_cv_wait(aa.cv);
+				}
+				nng_mtx_unlock(aa.mx);
 				So(val == 1);
 			});
 		});
diff --git a/tests/survey.c b/tests/survey.c
index d6d35c1b..5f3001e7 100644
--- a/tests/survey.c
+++ b/tests/survey.c
@@ -131,6 +131,7 @@ TestMain("SURVEY pattern", {
 			So(nng_sendmsg(surv, msg, 0) == 0);
 			nng_aio_wait(aio);
 			So(nng_aio_result(aio) == NNG_ECANCELED);
+			nng_aio_free(aio);
 		});
 
 		Convey("Sending a NULL message does not panic", {
@@ -331,5 +332,4 @@ TestMain("SURVEY pattern", {
 		So(nng_sendmsg(resp, msg, 0) == 0);
 		So(nng_recvmsg(surv, &msg, 0) == NNG_ETIMEDOUT);
 	});
-
 })
diff --git a/tests/synch.c b/tests/synch.c
index e767cad8..cf75e20c 100644
--- a/tests/synch.c
+++ b/tests/synch.c
@@ -45,42 +45,39 @@ test_sync(void)
 		So(nng_mtx_alloc(&mx) == 0);
 		Reset({ nng_mtx_free(mx); });
 
-		Convey("We can lock a mutex", {
+		Convey("We can lock and unlock a mutex", {
 			nng_mtx_lock(mx);
 			So(1);
-			Convey("And we can unlock it", {
+			nng_mtx_unlock(mx);
+			So(1);
+			Convey("And then lock it again", {
+				nng_mtx_lock(mx);
+				So(1);
 				nng_mtx_unlock(mx);
 				So(1);
-				Convey("And then lock it again", {
-					nng_mtx_lock(mx);
-					So(1);
-					nng_mtx_unlock(mx);
-					So(1);
-				});
 			});
-			Convey("Things block properly", {
-				So(nng_mtx_alloc(&arg.mx) == 0);
-				So(nng_cv_alloc(&arg.cv, arg.mx) == 0);
-				arg.did  = 0;
-				arg.when = 0;
-				nng_mtx_lock(arg.mx);
-				So(nng_thread_create(
-				       &thr, notifyafter, &arg) == 0);
-				nng_msleep(10);
-				So(arg.did == 0);
-				nng_mtx_unlock(arg.mx);
-				nng_msleep(10);
-				nng_mtx_lock(arg.mx);
-				while (!arg.did) {
-					nng_cv_wait(arg.cv);
-				}
-				So(arg.did != 0);
-				nng_mtx_unlock(arg.mx);
-				nng_thread_destroy(thr);
-				nng_cv_free(arg.cv);
-				nng_mtx_free(arg.mx);
-			})
 		});
+		Convey("Things block properly", {
+			So(nng_mtx_alloc(&arg.mx) == 0);
+			So(nng_cv_alloc(&arg.cv, arg.mx) == 0);
+			arg.did  = 0;
+			arg.when = 0;
+			nng_mtx_lock(arg.mx);
+			So(nng_thread_create(&thr, notifyafter, &arg) == 0);
+			nng_msleep(10);
+			So(arg.did == 0);
+			nng_mtx_unlock(arg.mx);
+			nng_msleep(10);
+			nng_mtx_lock(arg.mx);
+			while (!arg.did) {
+				nng_cv_wait(arg.cv);
+			}
+			So(arg.did != 0);
+			nng_mtx_unlock(arg.mx);
+			nng_thread_destroy(thr);
+			nng_cv_free(arg.cv);
+			nng_mtx_free(arg.mx);
+		})
 	});
 
 	Convey("Condition variables work", {
diff --git a/tests/tcp6.c b/tests/tcp6.c
index c023997f..86898b46 100644
--- a/tests/tcp6.c
+++ b/tests/tcp6.c
@@ -45,46 +45,34 @@ check_props_v6(nng_msg *msg)
 
 	memset(loopback, 0, sizeof(loopback));
 	loopback[15] = 1;
-	Convey("IPv6 Local address property works", {
-		nng_sockaddr la;
-		z = sizeof(nng_sockaddr);
-		p = nng_msg_get_pipe(msg);
-		So(nng_pipe_id(p) > 0);
-		So(nng_pipe_getopt(p, NNG_OPT_LOCADDR, &la, &z) == 0);
-		So(z == sizeof(la));
-		So(la.s_family == NNG_AF_INET6);
-		// So(la.s_in.sa_port == (trantest_port - 1));
-		So(la.s_in6.sa_port != 0);
-		So(memcmp(la.s_in6.sa_addr, loopback, 16) == 0);
-	});
-
-	Convey("IPv6 Remote address property works", {
-		nng_sockaddr ra;
-		z = sizeof(nng_sockaddr);
-		p = nng_msg_get_pipe(msg);
-		So(nng_pipe_id(p) > 0);
-		So(nng_pipe_getopt(p, NNG_OPT_REMADDR, &ra, &z) == 0);
-		So(z == sizeof(ra));
-		So(ra.s_family == NNG_AF_INET6);
-		So(ra.s_in6.sa_port != 0);
-		So(memcmp(ra.s_in6.sa_addr, loopback, 16) == 0);
-	});
-
-	Convey("Malformed TCPv6 addresses do not panic", {
-		nng_socket s1;
 
-		So(nng_pair_open(&s1) == 0);
-		Reset({ nng_close(s1); });
-		So(nng_dial(s1, "tcp://::1", NULL, 0) == NNG_EADDRINVAL);
-		So(nng_dial(s1, "tcp://::1:5055", NULL, 0) == NNG_EADDRINVAL);
-		So(nng_dial(s1, "tcp://[::1]", NULL, 0) == NNG_EADDRINVAL);
-	});
+	// IPv6 Local address property works
+	nng_sockaddr la;
+	z = sizeof(nng_sockaddr);
+	p = nng_msg_get_pipe(msg);
+	So(nng_pipe_id(p) > 0);
+	So(nng_pipe_getopt(p, NNG_OPT_LOCADDR, &la, &z) == 0);
+	So(z == sizeof(la));
+	So(la.s_family == NNG_AF_INET6);
+	// So(la.s_in.sa_port == (trantest_port - 1));
+	So(la.s_in6.sa_port != 0);
+	So(memcmp(la.s_in6.sa_addr, loopback, 16) == 0);
+
+	// IPv6 Remote address property works
+	nng_sockaddr ra;
+	z = sizeof(nng_sockaddr);
+	p = nng_msg_get_pipe(msg);
+	So(nng_pipe_id(p) > 0);
+	So(nng_pipe_getopt(p, NNG_OPT_REMADDR, &ra, &z) == 0);
+	So(z == sizeof(ra));
+	So(ra.s_family == NNG_AF_INET6);
+	So(ra.s_in6.sa_port != 0);
+	So(memcmp(ra.s_in6.sa_addr, loopback, 16) == 0);
 
 	return (0);
 }
 
 TestMain("TCP (IPv6) Transport", {
-
 	nni_init();
 
 	if (has_v6()) {
@@ -93,5 +81,15 @@ TestMain("TCP (IPv6) Transport", {
 		SkipSo("IPv6 not available");
 	}
 
+	Convey("Malformed TCPv6 addresses do not panic", {
+		nng_socket s1;
+
+		So(nng_pair_open(&s1) == 0);
+		Reset({ nng_close(s1); });
+		So(nng_dial(s1, "tcp://::1", NULL, 0) == NNG_EADDRINVAL);
+		So(nng_dial(s1, "tcp://::1:5055", NULL, 0) == NNG_EADDRINVAL);
+		So(nng_dial(s1, "tcp://[::1]", NULL, 0) == NNG_EADDRINVAL);
+	});
+
 	nng_fini();
 })
diff --git a/tools/nngcat/nngcat_unlimited_test.sh b/tools/nngcat/nngcat_unlimited_test.sh
index 7b5b4f2c..5cc46f08 100644
--- a/tools/nngcat/nngcat_unlimited_test.sh
+++ b/tools/nngcat/nngcat_unlimited_test.sh
@@ -26,7 +26,7 @@ goodsum=${goodsum%% *}
 ${NNGCAT} --listen ${ADDR} --count=1 --recv-maxsz=0 --pull0 --raw > $OUTPUT 2>/dev/null &
 sleep 1
 # for speed of execution, run these in the background, they should be ignored
-${NNGCAT} --connect ${ADDR} --push0 --file ${INPUT}
+${NNGCAT} --connect ${ADDR} --delay=1 --push0 --file ${INPUT}
 wait $bgid 2>/dev/null
 
 sum=$(cksum ${OUTPUT})