Introduce DTLS transport for NNG.

This introduces a new experimental transport for DTLS, that
provides encryption over UDP.  It has a simpler protocol than
the current UDP SP protocol (but we intend to fix that by making
the UDP transport simpler in a follow up!)

There are a few other fixes in the TLS layer itself, and in
the build, that were needed to accomplish this work.

Also there was an endianness bug in the UDP protocol handling, which
is fixed here.

9 files changed, 237 insertions, 116 deletions

diff --git a/src/supplemental/tls/CMakeLists.txt b/src/supplemental/tls/CMakeLists.txt
index 400b1354..41587915 100644
--- a/src/supplemental/tls/CMakeLists.txt
+++ b/src/supplemental/tls/CMakeLists.txt
@@ -12,8 +12,10 @@
 #
 
 if (NNG_ENABLE_TLS)
+    # List of TLS engines we support.  TLS engines must support TLS 1.2 or better,
+    # and must also support DTLS.  Support for PSK is preferred.
     set(NNG_TLS_ENGINES mbed wolf none)
-    # We assume Mbed for now.  (Someday replaced perhaps with Bear.)
+    # We assume Mbed for now.
     set(NNG_TLS_ENGINE mbed CACHE STRING "TLS engine to use.")
     set_property(CACHE NNG_TLS_ENGINE PROPERTY STRINGS ${NNG_TLS_ENGINES})
 else ()
@@ -29,7 +31,7 @@ add_subdirectory(wolfssl)
 
 if (NNG_ENABLE_TLS)
     nng_sources(tls_common.c tls_dialer.c tls_listener.c tls_stream.c)
-    nng_sources(tls_api.h tls_engine.h)
+    nng_sources(tls_api.h tls_common.h tls_engine.h tls_stream.h)
 else()
     nng_sources(tls_stubs.c)
 endif()
diff --git a/src/supplemental/tls/mbedtls/CMakeLists.txt b/src/supplemental/tls/mbedtls/CMakeLists.txt
index acf852bd..466f0a1b 100644
--- a/src/supplemental/tls/mbedtls/CMakeLists.txt
+++ b/src/supplemental/tls/mbedtls/CMakeLists.txt
@@ -13,7 +13,7 @@ if (NNG_TLS_ENGINE STREQUAL "mbed")
         Linking against Mbed TLS may change license terms.
         Consult a lawyer and the license files for details.
         ************************************************************")
-    nng_sources(tls.c)
+    nng_sources(mbedtls.c)
     nng_defines(NNG_TLS_ENGINE_INIT=nng_tls_engine_init_mbed)
     nng_defines(NNG_TLS_ENGINE_FINI=nng_tls_engine_fini_mbed)
     nng_defines(NNG_SUPP_TLS)
diff --git a/src/supplemental/tls/mbedtls/tls.c b/src/supplemental/tls/mbedtls/mbedtls.c
index 7764bbbf..8250740f 100644
--- a/src/supplemental/tls/mbedtls/tls.c
+++ b/src/supplemental/tls/mbedtls/mbedtls.c
@@ -23,6 +23,9 @@
 
 #include "nng/nng.h"
 
+// We use a common cookie for our application.
+#include "mbedtls/ssl_cookie.h"
+
 #include "../tls_engine.h"
 
 // mbedTLS renamed this header for 2.4.0.
@@ -82,6 +85,8 @@ static nni_mtx                  rng_lock;
 struct nng_tls_engine_conn {
 	void               *tls; // parent conn
 	mbedtls_ssl_context ctx;
+	nng_time            exp1;
+	nng_time            exp2;
 };
 
 struct nng_tls_engine_config {
@@ -96,6 +101,8 @@ struct nng_tls_engine_config {
 	nni_list           psks;
 };
 
+static mbedtls_ssl_cookie_ctx mbed_ssl_cookie_ctx;
+
 static void
 tls_dbg(void *ctx, int level, const char *file, int line, const char *s)
 {
@@ -238,15 +245,44 @@ conn_fini(nng_tls_engine_conn *ec)
 	mbedtls_ssl_free(&ec->ctx);
 }
 
+static void
+conn_set_timer(void *arg, unsigned int t1, unsigned int t2)
+{
+	nng_time             now = nng_clock();
+	nng_tls_engine_conn *ec  = arg;
+	ec->exp1                 = t1 ? now + t1 : 0;
+	ec->exp2                 = t2 ? now + t2 : 0;
+}
+
 static int
-conn_init(nng_tls_engine_conn *ec, void *tls, nng_tls_engine_config *cfg)
+conn_get_timer(void *arg)
 {
-	int rv;
+	nng_tls_engine_conn *ec  = arg;
+	nng_time             now = nng_clock();
+	if (ec->exp2 == 0) {
+		return -1;
+	}
+	if (now > ec->exp2) {
+		return 2;
+	}
+	if (now > ec->exp1) {
+		return 1;
+	}
+	return (0);
+}
+
+static int
+conn_init(nng_tls_engine_conn *ec, void *tls, nng_tls_engine_config *cfg,
+    const nng_sockaddr *sa)
+{
+	int  rv;
+	char buf[NNG_MAXADDRSTRLEN];
 
 	ec->tls = tls;
 
 	mbedtls_ssl_init(&ec->ctx);
 	mbedtls_ssl_set_bio(&ec->ctx, tls, net_send, net_recv, NULL);
+	mbedtls_ssl_set_timer_cb(&ec->ctx, ec, conn_set_timer, conn_get_timer);
 
 	if ((rv = mbedtls_ssl_setup(&ec->ctx, &cfg->cfg_ctx)) != 0) {
 		tls_log_warn(
@@ -258,6 +294,12 @@ conn_init(nng_tls_engine_conn *ec, void *tls, nng_tls_engine_config *cfg)
 		mbedtls_ssl_set_hostname(&ec->ctx, cfg->server_name);
 	}
 
+	if (cfg->mode == NNG_TLS_MODE_SERVER) {
+		nng_str_sockaddr(sa, buf, sizeof(buf));
+		mbedtls_ssl_set_client_transport_id(
+		    &ec->ctx, (const void *) buf, strlen(buf));
+	}
+
 	return (0);
 }
 
@@ -484,6 +526,12 @@ config_init(nng_tls_engine_config *cfg, enum nng_tls_mode mode)
 	mbedtls_ssl_conf_rng(&cfg->cfg_ctx, tls_random, cfg);
 	mbedtls_ssl_conf_dbg(&cfg->cfg_ctx, tls_dbg, cfg);
 
+	if (cfg->mode == NNG_TLS_MODE_SERVER) {
+		mbedtls_ssl_conf_dtls_cookies(&cfg->cfg_ctx,
+		    mbedtls_ssl_cookie_write, mbedtls_ssl_cookie_check,
+		    &mbed_ssl_cookie_ctx);
+	}
+
 	return (0);
 }
 
@@ -793,9 +841,14 @@ nng_tls_engine_init_mbed(void)
 #endif
 	// Uncomment the following to have noisy debug from mbedTLS.
 	// This may be useful when trying to debug failures.
-	//	mbedtls_debug_set_threshold(9);
+	// mbedtls_debug_set_threshold(9);
+
+	mbedtls_ssl_cookie_init(&mbed_ssl_cookie_ctx);
+	rv = mbedtls_ssl_cookie_setup(&mbed_ssl_cookie_ctx, tls_random, NULL);
 
-	rv = nng_tls_engine_register(&tls_engine_mbed);
+	if (rv == 0) {
+		rv = nng_tls_engine_register(&tls_engine_mbed);
+	}
 
 #ifdef NNG_TLS_USE_CTR_DRBG
 	if (rv != 0) {
@@ -809,6 +862,7 @@ nng_tls_engine_init_mbed(void)
 void
 nng_tls_engine_fini_mbed(void)
 {
+	mbedtls_ssl_cookie_free(&mbed_ssl_cookie_ctx);
 #ifdef NNG_TLS_USE_CTR_DRBG
 	mbedtls_ctr_drbg_free(&rng_ctx);
 	nni_mtx_fini(&rng_lock);
diff --git a/src/supplemental/tls/tls_common.c b/src/supplemental/tls/tls_common.c
index c31e5fe9..40689f69 100644
--- a/src/supplemental/tls/tls_common.c
+++ b/src/supplemental/tls/tls_common.c
@@ -41,15 +41,15 @@ static nni_atomic_ptr tls_engine;
 
 static void tls_bio_send_cb(void *arg);
 static void tls_bio_recv_cb(void *arg);
-static void tls_do_send(tls_conn *);
-static void tls_do_recv(tls_conn *);
-static void tls_bio_send_start(tls_conn *);
-static void tls_bio_error(tls_conn *, int);
+static void tls_do_send(nni_tls_conn *);
+static void tls_do_recv(nni_tls_conn *);
+static void tls_bio_send_start(nni_tls_conn *);
+static void tls_bio_error(nni_tls_conn *, nng_err);
 
 static void
 tls_cancel(nni_aio *aio, void *arg, nng_err rv)
 {
-	tls_conn *conn = arg;
+	nni_tls_conn *conn = arg;
 	nni_mtx_lock(&conn->lock);
 	if (aio == nni_list_first(&conn->recv_queue)) {
 		nni_aio_abort(&conn->bio_recv, rv);
@@ -64,7 +64,7 @@ tls_cancel(nni_aio *aio, void *arg, nng_err rv)
 
 // tls_send implements the upper layer send operation.
 void
-nni_tls_send(tls_conn *conn, nni_aio *aio)
+nni_tls_send(nni_tls_conn *conn, nni_aio *aio)
 {
 	nni_aio_reset(aio);
 	nni_mtx_lock(&conn->lock);
@@ -83,7 +83,7 @@ nni_tls_send(tls_conn *conn, nni_aio *aio)
 }
 
 void
-nni_tls_recv(tls_conn *conn, nni_aio *aio)
+nni_tls_recv(nni_tls_conn *conn, nni_aio *aio)
 {
 	nni_aio_reset(aio);
 	nni_mtx_lock(&conn->lock);
@@ -103,21 +103,20 @@ nni_tls_recv(tls_conn *conn, nni_aio *aio)
 }
 
 void
-nni_tls_close(tls_conn *conn)
+nni_tls_close(nni_tls_conn *conn)
 {
 	if (!nni_atomic_flag_test_and_set(&conn->did_close)) {
 		nni_mtx_lock(&conn->lock);
 		conn->ops.close((void *) (conn + 1));
-		tls_bio_error(conn, NNG_ECLOSED);
 		nni_mtx_unlock(&conn->lock);
-		if (conn->bio != NULL) {
-			conn->bio_ops.bio_close(conn->bio);
-		}
+		nni_mtx_lock(&conn->bio_lock);
+		tls_bio_error(conn, NNG_ECLOSED);
+		nni_mtx_unlock(&conn->bio_lock);
 	}
 }
 
 void
-nni_tls_stop(tls_conn *conn)
+nni_tls_stop(nni_tls_conn *conn)
 {
 	nni_tls_close(conn);
 	if (conn->bio != NULL) {
@@ -128,7 +127,7 @@ nni_tls_stop(tls_conn *conn)
 }
 
 bool
-nni_tls_verified(tls_conn *conn)
+nni_tls_verified(nni_tls_conn *conn)
 {
 	bool result;
 	nni_mtx_lock(&conn->lock);
@@ -138,7 +137,7 @@ nni_tls_verified(tls_conn *conn)
 }
 
 const char *
-nni_tls_peer_cn(tls_conn *conn)
+nni_tls_peer_cn(nni_tls_conn *conn)
 {
 	const char *result;
 	nni_mtx_lock(&conn->lock);
@@ -148,7 +147,7 @@ nni_tls_peer_cn(tls_conn *conn)
 }
 
 int
-nni_tls_init(tls_conn *conn, nng_tls_config *cfg)
+nni_tls_init(nni_tls_conn *conn, nng_tls_config *cfg)
 {
 	const nng_tls_engine *eng;
 
@@ -158,9 +157,9 @@ nni_tls_init(tls_conn *conn, nng_tls_config *cfg)
 	cfg->busy = true;
 	nni_mtx_unlock(&cfg->lock);
 
-	if (((conn->bio_send_buf = nni_alloc(NNG_TLS_MAX_SEND_SIZE)) ==
+	if (((conn->bio_send_buf = nni_zalloc(NNG_TLS_MAX_SEND_SIZE)) ==
 	        NULL) ||
-	    ((conn->bio_recv_buf = nni_alloc(NNG_TLS_MAX_RECV_SIZE)) ==
+	    ((conn->bio_recv_buf = nni_zalloc(NNG_TLS_MAX_RECV_SIZE)) ==
 	        NULL)) {
 		return (NNG_ENOMEM);
 	}
@@ -173,6 +172,7 @@ nni_tls_init(tls_conn *conn, nng_tls_config *cfg)
 	nni_aio_list_init(&conn->send_queue);
 	nni_aio_list_init(&conn->recv_queue);
 	nni_mtx_init(&conn->lock);
+	nni_mtx_init(&conn->bio_lock);
 	nni_aio_set_timeout(&conn->bio_send, NNG_DURATION_INFINITE);
 	nni_aio_set_timeout(&conn->bio_recv, NNG_DURATION_INFINITE);
 	nni_atomic_flag_reset(&conn->did_close);
@@ -182,7 +182,7 @@ nni_tls_init(tls_conn *conn, nng_tls_config *cfg)
 }
 
 void
-nni_tls_fini(tls_conn *conn)
+nni_tls_fini(nni_tls_conn *conn)
 {
 	nni_tls_stop(conn);
 	conn->ops.fini((void *) (conn + 1));
@@ -200,11 +200,13 @@ nni_tls_fini(tls_conn *conn)
 	if (conn->bio != NULL) {
 		conn->bio_ops.bio_free(conn->bio);
 	}
+	nni_mtx_fini(&conn->bio_lock);
 	nni_mtx_fini(&conn->lock);
 }
 
 int
-nni_tls_start(tls_conn *conn, const nni_tls_bio_ops *biops, void *bio)
+nni_tls_start(nni_tls_conn *conn, const nni_tls_bio_ops *biops, void *bio,
+    const nng_sockaddr *sa)
 {
 	nng_tls_engine_config *cfg;
 	nng_tls_engine_conn   *econ;
@@ -215,48 +217,62 @@ nni_tls_start(tls_conn *conn, const nni_tls_bio_ops *biops, void *bio)
 	conn->bio_ops = *biops;
 	conn->bio     = bio;
 
-	return (conn->ops.init(econ, conn, cfg));
+	return (conn->ops.init(econ, conn, cfg, sa));
 }
 
 static void
-tls_bio_error(tls_conn *conn, int rv)
+tls_conn_err(nni_tls_conn *conn, nng_err rv)
 {
-	// An error here is fatal.  Shut it all down.
 	nni_aio *aio;
-	if (conn->bio != NULL) {
-		conn->bio_ops.bio_close(conn->bio);
-	}
-	nni_aio_close(&conn->bio_send);
-	nni_aio_close(&conn->bio_recv);
+	nni_mtx_lock(&conn->lock);
 	while (((aio = nni_list_first(&conn->send_queue)) != NULL) ||
 	    ((aio = nni_list_first(&conn->recv_queue)) != NULL)) {
 		nni_aio_list_remove(aio);
 		nni_aio_finish_error(aio, rv);
 	}
+	nni_mtx_unlock(&conn->lock);
+}
+static void
+tls_bio_error(nni_tls_conn *conn, nng_err rv)
+{
+	// An error here is fatal.  Shut it all down.
+	if (!conn->bio_closed) {
+		conn->bio_closed = true;
+		conn->bio_err    = rv;
+		if (conn->bio_send_active)
+			nni_aio_abort(&conn->bio_send, conn->bio_err);
+		if (conn->bio_recv_pend)
+			nni_aio_abort(&conn->bio_recv, conn->bio_err);
+		if (conn->bio != NULL) {
+			conn->bio_ops.bio_close(conn->bio);
+		}
+
+		nni_aio_close(&conn->bio_send);
+		nni_aio_close(&conn->bio_recv);
+	}
 }
 
-static bool
-tls_do_handshake(tls_conn *conn)
+static nng_err
+tls_handshake(nni_tls_conn *conn)
 {
 	int rv;
 	if (conn->hs_done) {
-		return (true);
+		return (NNG_OK);
 	}
 	rv = conn->ops.handshake((void *) (conn + 1));
 	if (rv == NNG_EAGAIN) {
 		// We need more data.
-		return (false);
+		return (rv);
 	}
-	if (rv == 0) {
+	if (rv == NNG_OK) {
 		conn->hs_done = true;
-		return (true);
+		return (rv);
 	}
-	tls_bio_error(conn, rv);
-	return (true);
+	return (rv);
 }
 
 static void
-tls_do_recv(tls_conn *conn)
+tls_do_recv(nni_tls_conn *conn)
 {
 	nni_aio *aio;
 
@@ -294,7 +310,7 @@ tls_do_recv(tls_conn *conn)
 		// caller as *soon* as we have some data.
 		nni_aio_list_remove(aio);
 
-		if (rv != 0) {
+		if (rv != NNG_OK) {
 			nni_aio_finish_error(aio, rv);
 		} else {
 			nni_aio_finish(aio, 0, len);
@@ -304,7 +320,7 @@ tls_do_recv(tls_conn *conn)
 
 // tls_do_send attempts to send user data.
 static void
-tls_do_send(tls_conn *conn)
+tls_do_send(nni_tls_conn *conn)
 {
 	nni_aio *aio;
 
@@ -350,20 +366,47 @@ tls_do_send(tls_conn *conn)
 	}
 }
 
+nng_err
+nni_tls_run(nni_tls_conn *conn)
+{
+	nni_aio *aio;
+	nng_err  rv;
+	nni_mtx_lock(&conn->lock);
+	switch ((rv = tls_handshake(conn))) {
+	case NNG_OK:
+		tls_do_recv(conn);
+		tls_do_send(conn);
+		break;
+	case NNG_EAGAIN:
+		break;
+	default:
+		while (((aio = nni_list_first(&conn->send_queue)) != NULL) ||
+		    ((aio = nni_list_first(&conn->recv_queue)) != NULL)) {
+			nni_aio_list_remove(aio);
+			nni_aio_finish_error(aio, rv);
+		}
+		break;
+	}
+	nni_mtx_unlock(&conn->lock);
+	return (rv);
+}
+
 static void
 tls_bio_send_cb(void *arg)
 {
-	tls_conn *conn = arg;
-	nng_aio  *aio  = &conn->bio_send;
-	int       rv;
-	size_t    count;
+	nni_tls_conn *conn = arg;
+	nng_aio      *aio  = &conn->bio_send;
+	int           rv;
+	size_t        count;
 
-	nni_mtx_lock(&conn->lock);
+	nni_mtx_lock(&conn->bio_lock);
 	conn->bio_send_active = false;
 
 	if ((rv = nni_aio_result(aio)) != 0) {
 		tls_bio_error(conn, rv);
-		nni_mtx_unlock(&conn->lock);
+		nni_mtx_unlock(&conn->bio_lock);
+
+		tls_conn_err(conn, rv);
 		return;
 	}
 
@@ -373,45 +416,37 @@ tls_bio_send_cb(void *arg)
 	conn->bio_send_tail += count;
 	conn->bio_send_tail %= NNG_TLS_MAX_SEND_SIZE;
 	tls_bio_send_start(conn);
+	nni_mtx_unlock(&conn->bio_lock);
 
-	if (tls_do_handshake(conn)) {
-		tls_do_send(conn);
-		tls_do_recv(conn);
-	}
-
-	nni_mtx_unlock(&conn->lock);
+	nni_tls_run(conn);
 }
 
 static void
 tls_bio_recv_cb(void *arg)
 {
-	tls_conn *conn = arg;
-	nni_aio  *aio  = &conn->bio_recv;
-	int       rv;
-
-	nni_mtx_lock(&conn->lock);
+	nni_tls_conn *conn = arg;
+	nni_aio      *aio  = &conn->bio_recv;
+	int           rv;
 
+	nni_mtx_lock(&conn->bio_lock);
 	conn->bio_recv_pend = false;
 	if ((rv = nni_aio_result(aio)) != 0) {
 		tls_bio_error(conn, rv);
-		nni_mtx_unlock(&conn->lock);
+		nni_mtx_unlock(&conn->bio_lock);
+		tls_conn_err(conn, rv);
 		return;
 	}
 
 	NNI_ASSERT(conn->bio_recv_len == 0);
 	NNI_ASSERT(conn->bio_recv_off == 0);
 	conn->bio_recv_len = nni_aio_count(aio);
+	nni_mtx_unlock(&conn->bio_lock);
 
-	if (tls_do_handshake(conn)) {
-		tls_do_recv(conn);
-		tls_do_send(conn);
-	}
-
-	nni_mtx_unlock(&conn->lock);
+	nni_tls_run(conn);
 }
 
 static void
-tls_bio_recv_start(tls_conn *conn)
+tls_bio_recv_start(nni_tls_conn *conn)
 {
 	nng_iov iov;
 
@@ -423,6 +458,9 @@ tls_bio_recv_start(tls_conn *conn)
 		// Already have a receive in flight.
 		return;
 	}
+	if (conn->bio_closed) {
+		return;
+	}
 	conn->bio_recv_off = 0;
 	iov.iov_len        = NNG_TLS_MAX_RECV_SIZE;
 	iov.iov_buf        = conn->bio_recv_buf;
@@ -434,7 +472,7 @@ tls_bio_recv_start(tls_conn *conn)
 }
 
 static void
-tls_bio_send_start(tls_conn *conn)
+tls_bio_send_start(nni_tls_conn *conn)
 {
 	nni_iov  iov[2];
 	unsigned nio = 0;
@@ -448,6 +486,9 @@ tls_bio_send_start(tls_conn *conn)
 	if (conn->bio_send_len == 0) {
 		return;
 	}
+	if (conn->bio_closed) {
+		return;
+	}
 	len  = conn->bio_send_len;
 	head = conn->bio_send_head;
 	tail = conn->bio_send_tail;
@@ -478,23 +519,23 @@ tls_bio_send_start(tls_conn *conn)
 int
 nng_tls_engine_send(void *arg, const uint8_t *buf, size_t *szp)
 {
-	tls_conn *conn = arg;
-	size_t    len  = *szp;
-	size_t    head = conn->bio_send_head;
-	size_t    tail = conn->bio_send_tail;
-	size_t    space;
-	size_t    cnt;
+	nni_tls_conn *conn = arg;
+	size_t        len  = *szp;
+	size_t        head;
+	size_t        tail;
+	size_t        space;
+	size_t        cnt;
 
+	nni_mtx_lock(&conn->bio_lock);
+	head  = conn->bio_send_head;
+	tail  = conn->bio_send_tail;
 	space = NNG_TLS_MAX_SEND_SIZE - conn->bio_send_len;
 
 	if (space == 0) {
+		nni_mtx_unlock(&conn->bio_lock);
 		return (NNG_EAGAIN);
 	}
 
-	if (conn->closed) {
-		return (NNG_ECLOSED);
-	}
-
 	if (len > space) {
 		len = space;
 	}
@@ -525,20 +566,20 @@ nng_tls_engine_send(void *arg, const uint8_t *buf, size_t *szp)
 	conn->bio_send_head = head;
 
 	tls_bio_send_start(conn);
+	nni_mtx_unlock(&conn->bio_lock);
 	return (0);
 }
 
 int
 nng_tls_engine_recv(void *arg, uint8_t *buf, size_t *szp)
 {
-	tls_conn *conn = arg;
-	size_t    len  = *szp;
+	nni_tls_conn *conn = arg;
+	size_t        len  = *szp;
 
-	if (conn->closed) {
-		return (NNG_ECLOSED);
-	}
+	nni_mtx_lock(&conn->bio_lock);
 	if (conn->bio_recv_len == 0) {
 		tls_bio_recv_start(conn);
+		nni_mtx_unlock(&conn->bio_lock);
 		return (NNG_EAGAIN);
 	}
 	if (len > conn->bio_recv_len) {
@@ -551,6 +592,7 @@ nng_tls_engine_recv(void *arg, uint8_t *buf, size_t *szp)
 	// If we still have data left in the buffer, then the following
 	// call is a no-op.
 	tls_bio_recv_start(conn);
+	nni_mtx_unlock(&conn->bio_lock);
 
 	*szp = len;
 	return (0);
@@ -805,6 +847,16 @@ nng_tls_engine_register(const nng_tls_engine *engine)
 	return (0);
 }
 
+size_t
+nni_tls_engine_conn_size(void)
+{
+	const nng_tls_engine *eng;
+
+	eng = nni_atomic_get_ptr(&tls_engine);
+
+	return (eng == NULL ? false : eng->conn_ops->size);
+}
+
 #ifdef NNG_TLS_ENGINE_INIT
 extern int NNG_TLS_ENGINE_INIT(void);
 #else
diff --git a/src/supplemental/tls/tls_common.h b/src/supplemental/tls/tls_common.h
index 3e703785..14bb0cf7 100644
--- a/src/supplemental/tls/tls_common.h
+++ b/src/supplemental/tls/tls_common.h
@@ -39,8 +39,6 @@
 // parts of TLS support that are invariant relative to different TLS
 // libraries, such as dialer and listener support.
 
-static nni_atomic_ptr tls_engine;
-
 struct nng_tls_config {
 	nng_tls_engine_config_ops ops;
 	const nng_tls_engine     *engine; // store this so we can verify
@@ -78,29 +76,34 @@ typedef struct {
 	nni_tls_bio_ops bio_ops;  // lower level ops vector
 	nni_aio         bio_send; // lower level send pending
 	nni_aio         bio_recv; // lower level recv pending
+	nni_mtx         bio_lock; // lock protecting lower layer operations
 	uint8_t        *bio_send_buf;
 	uint8_t        *bio_recv_buf;
 	size_t          bio_recv_len;
 	size_t          bio_recv_off;
 	bool            bio_recv_pend;
 	bool            bio_send_active;
+	bool            bio_closed;
+	nng_err         bio_err;
 	size_t          bio_send_len;
 	size_t          bio_send_head;
 	size_t          bio_send_tail;
 	nni_reap_node   reap;
 
 	// ... engine connection data follows
-} tls_conn;
-
-extern void nni_tls_fini(tls_conn *conn);
-extern int  nni_tls_init(tls_conn *conn, nng_tls_config *cfg);
-extern int  nni_tls_start(
-     tls_conn *conn, const nni_tls_bio_ops *biops, void *bio);
-extern void        nni_tls_stop(tls_conn *conn);
-extern void        nni_tls_close(tls_conn *conn);
-extern void        nni_tls_recv(tls_conn *conn, nni_aio *aio);
-extern void        nni_tls_send(tls_conn *conn, nni_aio *aio);
-extern bool        nni_tls_verified(tls_conn *conn);
-extern const char *nni_tls_peer_cn(tls_conn *conn);
+} nni_tls_conn;
+
+extern void nni_tls_fini(nni_tls_conn *conn);
+extern int  nni_tls_init(nni_tls_conn *conn, nng_tls_config *cfg);
+extern int  nni_tls_start(nni_tls_conn *conn, const nni_tls_bio_ops *biops,
+     void *bio, const nng_sockaddr *sa);
+extern void nni_tls_stop(nni_tls_conn *conn);
+extern void nni_tls_close(nni_tls_conn *conn);
+extern void nni_tls_recv(nni_tls_conn *conn, nni_aio *aio);
+extern void nni_tls_send(nni_tls_conn *conn, nni_aio *aio);
+extern bool nni_tls_verified(nni_tls_conn *conn);
+extern const char *nni_tls_peer_cn(nni_tls_conn *conn);
+extern nng_err     nni_tls_run(nni_tls_conn *conn);
+extern size_t      nni_tls_engine_conn_size(void);
 
 #endif // NNG_TLS_TLS_COMMON_H
diff --git a/src/supplemental/tls/tls_engine.h b/src/supplemental/tls/tls_engine.h
index bbc5a944..66d40826 100644
--- a/src/supplemental/tls/tls_engine.h
+++ b/src/supplemental/tls/tls_engine.h
@@ -44,7 +44,10 @@ typedef struct nng_tls_engine_conn_ops_s {
 	// init is used to initialize a connection object.
 	// The passed in connection state will be aligned naturally,
 	// and zeroed.  On success this returns 0, else an NNG error code.
-	int (*init)(nng_tls_engine_conn *, void *, nng_tls_engine_config *);
+	// The sockaddr is the peer's socket adress (needed for DTLS or
+	// possibly session resumption.)
+	int (*init)(nng_tls_engine_conn *, void *, nng_tls_engine_config *,
+	    const nng_sockaddr *);
 
 	// fini destroys a connection object.  This will
 	// be called only when no other external use of the connection
@@ -175,7 +178,7 @@ typedef enum nng_tls_engine_version_e {
 } nng_tls_engine_version;
 
 typedef struct nng_tls_engine_s {
-	// _version is the engine version.  This for now must
+	// version is the engine version.  This for now must
 	// be NNG_TLS_ENGINE_VERSION.  If the version does not match
 	// then registration of the engine will fail.
 	nng_tls_engine_version version;
@@ -212,7 +215,7 @@ extern int nng_tls_engine_register(const nng_tls_engine *);
 // is the context structure passed in when starting the engine.
 extern int nng_tls_engine_send(void *, const uint8_t *, size_t *);
 
-// nng_tls_engine_recv is called byu the engine to receive data over
+// nng_tls_engine_recv is called by the engine to receive data over
 // the underlying connection.  It returns zero on success, NNG_EAGAIN
 // if the operation can't be completed yet (there is no data available
 // for reading), or some other error.  On success the count is updated
diff --git a/src/supplemental/tls/tls_stream.c b/src/supplemental/tls/tls_stream.c
index 8a7f26d8..cd248686 100644
--- a/src/supplemental/tls/tls_stream.c
+++ b/src/supplemental/tls/tls_stream.c
@@ -110,9 +110,10 @@ tls_stream_recv(void *arg, nng_aio *aio)
 static void
 tls_stream_conn_cb(void *arg)
 {
-	tls_stream *ts = arg;
-	nng_stream *bio;
-	int         rv;
+	tls_stream  *ts = arg;
+	nng_stream  *bio;
+	int          rv;
+	nng_sockaddr sa;
 
 	if ((rv = nni_aio_result(&ts->conn_aio)) != 0) {
 		nni_aio_finish_error(ts->user_aio, rv);
@@ -121,8 +122,13 @@ tls_stream_conn_cb(void *arg)
 	}
 
 	bio = nni_aio_get_output(&ts->conn_aio, 0);
+	if ((rv = nng_stream_get_addr(bio, NNG_OPT_REMADDR, &sa)) != 0) {
+		nni_aio_finish_error(ts->user_aio, rv);
+		nni_tls_stream_free(ts);
+		return;
+	};
 
-	if ((rv = nni_tls_start(&ts->conn, &tls_stream_bio, bio)) != 0) {
+	if ((rv = nni_tls_start(&ts->conn, &tls_stream_bio, bio, &sa)) != 0) {
 		// NB: if this fails, it *will* have set the bio either way.
 		// So nni_tls_stream_free will also free the bio.
 		nni_aio_finish_error(ts->user_aio, rv);
@@ -140,13 +146,12 @@ static nng_err tls_stream_get(
 int
 nni_tls_stream_alloc(tls_stream **tsp, nng_tls_config *cfg, nng_aio *user_aio)
 {
-	tls_stream           *ts;
-	const nng_tls_engine *eng;
-	size_t                size;
-	int                   rv;
+	tls_stream *ts;
+	size_t      size;
+	int         rv;
 
-	eng  = cfg->engine;
-	size = NNI_ALIGN_UP(sizeof(*ts)) + eng->conn_ops->size;
+	size = NNI_ALIGN_UP(sizeof(*ts)) +
+	    NNI_ALIGN_UP(nni_tls_engine_conn_size());
 
 	if ((ts = nni_zalloc(size)) == NULL) {
 		return (NNG_ENOMEM);
diff --git a/src/supplemental/tls/tls_stream.h b/src/supplemental/tls/tls_stream.h
index 78760f82..dca439af 100644
--- a/src/supplemental/tls/tls_stream.h
+++ b/src/supplemental/tls/tls_stream.h
@@ -21,7 +21,7 @@ typedef struct tls_stream_s {
 	nni_reap_node reap;
 	nni_aio       conn_aio;
 	nni_aio      *user_aio;
-	tls_conn      conn; // NB: must be last!
+	nni_tls_conn  conn; // NB: must be last!
 } tls_stream;
 
 extern void nni_tls_stream_free(void *arg);
diff --git a/src/supplemental/tls/wolfssl/wolfssl.c b/src/supplemental/tls/wolfssl/wolfssl.c
index 1510a02a..3eab0ada 100644
--- a/src/supplemental/tls/wolfssl/wolfssl.c
+++ b/src/supplemental/tls/wolfssl/wolfssl.c
@@ -155,8 +155,10 @@ wolf_conn_fini(nng_tls_engine_conn *ec)
 }
 
 static int
-wolf_conn_init(nng_tls_engine_conn *ec, void *tls, nng_tls_engine_config *cfg)
+wolf_conn_init(nng_tls_engine_conn *ec, void *tls, nng_tls_engine_config *cfg,
+    const nng_sockaddr *sa)
 {
+	NNI_ARG_UNUSED(sa); // for now... revisit if we support DTLS ?
 	ec->tls       = tls;
 	ec->auth_mode = cfg->auth_mode;