streams: add explicit stop functions

This allows us to explicitly stop streams, dialers, and listeners,
before we start tearing down things. This hopefully will be useful
in resolving use-after-free bugs in http, tls, and websockets.

The new functions are not yet documented, but they are
nng_stream_stop, nng_stream_dialer_stop, and nng_stream_listener_stop.
They should be called after close, and before free.  The close
functions now close without blocking, but the stop function is
allowed to block.

2 files changed, 62 insertions, 16 deletions

diff --git a/src/supplemental/tls/tls_common.c b/src/supplemental/tls/tls_common.c
index c04d03a5..a871e74e 100644
--- a/src/supplemental/tls/tls_common.c
+++ b/src/supplemental/tls/tls_common.c
@@ -126,6 +126,14 @@ tls_dialer_free(void *arg)
 	}
 }
 
+static void
+tls_dialer_stop(void *arg)
+{
+	tls_dialer *d = arg;
+
+	nng_stream_dialer_stop(d->d);
+}
+
 // For dialing, we need to have our own completion callback, instead of
 // the user's completion callback.
 
@@ -281,6 +289,7 @@ nni_tls_dialer_alloc(nng_stream_dialer **dp, const nng_url *url)
 
 	d->ops.sd_close   = tls_dialer_close;
 	d->ops.sd_free    = tls_dialer_free;
+	d->ops.sd_stop    = tls_dialer_stop;
 	d->ops.sd_dial    = tls_dialer_dial;
 	d->ops.sd_get     = tls_dialer_get;
 	d->ops.sd_set     = tls_dialer_set;
@@ -307,6 +316,13 @@ tls_listener_close(void *arg)
 }
 
 static void
+tls_listener_stop(void *arg)
+{
+	tls_listener *l = arg;
+	nng_stream_listener_close(l->l);
+}
+
+static void
 tls_listener_free(void *arg)
 {
 	tls_listener *l;
@@ -436,6 +452,7 @@ nni_tls_listener_alloc(nng_stream_listener **lp, const nng_url *url)
 	}
 	l->ops.sl_free    = tls_listener_free;
 	l->ops.sl_close   = tls_listener_close;
+	l->ops.sl_stop    = tls_listener_stop;
 	l->ops.sl_accept  = tls_listener_accept;
 	l->ops.sl_listen  = tls_listener_listen;
 	l->ops.sl_get     = tls_listener_get;
@@ -526,6 +543,18 @@ tls_close(void *arg)
 	nng_stream_close(conn->tcp);
 }
 
+static void
+tls_stop(void *arg)
+{
+	tls_conn *conn = arg;
+
+	tls_close(conn);
+	nng_stream_stop(conn->tcp);
+	nni_aio_stop(&conn->conn_aio);
+	nni_aio_stop(&conn->tcp_send);
+	nni_aio_stop(&conn->tcp_recv);
+}
+
 static int
 tls_get_verified(void *arg, void *buf, size_t *szp, nni_type t)
 {
@@ -624,6 +653,7 @@ tls_alloc(tls_conn **conn_p, nng_tls_config *cfg, nng_aio *user_aio)
 
 	conn->stream.s_close = tls_close;
 	conn->stream.s_free  = tls_free;
+	conn->stream.s_stop  = tls_stop;
 	conn->stream.s_send  = tls_send;
 	conn->stream.s_recv  = tls_recv;
 	conn->stream.s_get   = tls_get;
@@ -638,14 +668,7 @@ tls_reap(void *arg)
 {
 	tls_conn *conn = arg;
 
-	// Shut it all down first.  We should be freed.
-	if (conn->tcp != NULL) {
-		nng_stream_close(conn->tcp);
-	}
-	nni_aio_stop(&conn->conn_aio);
-	nni_aio_stop(&conn->tcp_send);
-	nni_aio_stop(&conn->tcp_recv);
-
+	tls_stop(conn);
 	conn->ops.fini((void *) (conn + 1));
 	nni_aio_fini(&conn->conn_aio);
 	nni_aio_fini(&conn->tcp_send);
diff --git a/src/supplemental/tls/tls_test.c b/src/supplemental/tls/tls_test.c
index 43ce0c85..517be143 100644
--- a/src/supplemental/tls/tls_test.c
+++ b/src/supplemental/tls/tls_test.c
@@ -57,6 +57,7 @@ test_tls_conn_refused(void)
 	NUTS_FAIL(nng_aio_result(aio), NNG_ECONNREFUSED);
 
 	nng_aio_free(aio);
+	nng_stream_dialer_stop(dialer);
 	nng_stream_dialer_free(dialer);
 }
 
@@ -133,6 +134,10 @@ test_tls_large_message(void)
 
 	nng_free(buf1, size);
 	nng_free(buf2, size);
+	nng_stream_stop(s1);
+	nng_stream_stop(s2);
+	nng_stream_dialer_stop(d);
+	nng_stream_listener_stop(l);
 	nng_stream_free(s1);
 	nng_stream_free(s2);
 	nng_stream_dialer_free(d);
@@ -214,8 +219,10 @@ test_tls_ecdsa(void)
 	NUTS_PASS(nuts_stream_wait(t2));
 	NUTS_TRUE(memcmp(buf1, buf2, size) == 0);
 
-	nng_free(buf1, size);
-	nng_free(buf2, size);
+	nng_stream_stop(s1);
+	nng_stream_stop(s2);
+	nng_stream_dialer_stop(d);
+	nng_stream_listener_stop(l);
 	nng_stream_free(s1);
 	nng_stream_free(s2);
 	nng_stream_dialer_free(d);
@@ -224,6 +231,8 @@ test_tls_ecdsa(void)
 	nng_tls_config_free(c2);
 	nng_aio_free(aio1);
 	nng_aio_free(aio2);
+	nng_free(buf1, size);
+	nng_free(buf2, size);
 }
 
 void
@@ -241,6 +250,7 @@ test_tls_garbled_cert(void)
 	              c1, nuts_garbled_crt, nuts_server_key, NULL),
 	    NNG_ECRYPTO);
 
+	nng_stream_listener_stop(l);
 	nng_stream_listener_free(l);
 	nng_tls_config_free(c1);
 }
@@ -318,8 +328,10 @@ test_tls_psk(void)
 	NUTS_PASS(nuts_stream_wait(t2));
 	NUTS_TRUE(memcmp(buf1, buf2, size) == 0);
 
-	nng_free(buf1, size);
-	nng_free(buf2, size);
+	nng_stream_stop(s1);
+	nng_stream_stop(s2);
+	nng_stream_dialer_stop(d);
+	nng_stream_listener_stop(l);
 	nng_stream_free(s1);
 	nng_stream_free(s2);
 	nng_stream_dialer_free(d);
@@ -328,6 +340,8 @@ test_tls_psk(void)
 	nng_tls_config_free(c2);
 	nng_aio_free(aio1);
 	nng_aio_free(aio2);
+	nng_free(buf1, size);
+	nng_free(buf2, size);
 }
 
 void
@@ -408,8 +422,10 @@ test_tls_psk_server_identities(void)
 	NUTS_PASS(nuts_stream_wait(t2));
 	NUTS_TRUE(memcmp(buf1, buf2, size) == 0);
 
-	nng_free(buf1, size);
-	nng_free(buf2, size);
+	nng_stream_stop(s1);
+	nng_stream_stop(s2);
+	nng_stream_dialer_stop(d);
+	nng_stream_listener_stop(l);
 	nng_stream_free(s1);
 	nng_stream_free(s2);
 	nng_stream_dialer_free(d);
@@ -418,6 +434,8 @@ test_tls_psk_server_identities(void)
 	nng_tls_config_free(c2);
 	nng_aio_free(aio1);
 	nng_aio_free(aio2);
+	nng_free(buf1, size);
+	nng_free(buf2, size);
 }
 
 void
@@ -495,8 +513,10 @@ test_tls_psk_bad_identity(void)
 	NUTS_ASSERT(nuts_stream_wait(t1) != 0);
 	NUTS_ASSERT(nuts_stream_wait(t2) != 0);
 
-	nng_free(buf1, size);
-	nng_free(buf2, size);
+	nng_stream_stop(s1);
+	nng_stream_stop(s2);
+	nng_stream_dialer_stop(d);
+	nng_stream_listener_stop(l);
 	nng_stream_free(s1);
 	nng_stream_free(s2);
 	nng_stream_dialer_free(d);
@@ -505,6 +525,8 @@ test_tls_psk_bad_identity(void)
 	nng_tls_config_free(c2);
 	nng_aio_free(aio1);
 	nng_aio_free(aio2);
+	nng_free(buf1, size);
+	nng_free(buf2, size);
 }
 
 void
@@ -543,6 +565,7 @@ test_tls_psk_config_busy(void)
 	NUTS_FAIL(
 	    nng_tls_config_psk(c1, "identity2", key, sizeof(key)), NNG_EBUSY);
 
+	nng_stream_listener_stop(l);
 	nng_stream_listener_free(l);
 	nng_aio_free(aio);
 	nng_tls_config_free(c1);