fixes #572 Several locking errors found

fixes #573 atomic flags could help

This introduces a new atomic flag, and reduces some of the global
locking.  The lock refactoring work is not yet complete, but this is
a positive step forward, and should help with certain things.

While here we also fixed a compile warning due to incorrect types.

5 files changed, 126 insertions, 21 deletions

diff --git a/src/transport/ipc/ipc.c b/src/transport/ipc/ipc.c
index b48b82d9..7d99e507 100644
--- a/src/transport/ipc/ipc.c
+++ b/src/transport/ipc/ipc.c
@@ -827,14 +827,22 @@ static int
 ipc_ep_get_recvmaxsz(void *arg, void *data, size_t *szp, nni_opt_type t)
 {
 	ipc_ep *ep = arg;
-	return (nni_copyout_size(ep->rcvmax, data, szp, t));
+	int     rv;
+	nni_mtx_lock(&ep->mtx);
+	rv = nni_copyout_size(ep->rcvmax, data, szp, t);
+	nni_mtx_unlock(&ep->mtx);
+	return (rv);
 }
 
 static int
 ipc_ep_get_addr(void *arg, void *data, size_t *szp, nni_opt_type t)
 {
 	ipc_ep *ep = arg;
-	return (nni_copyout_sockaddr(&ep->sa, data, szp, t));
+	int     rv;
+	nni_mtx_lock(&ep->mtx);
+	rv = nni_copyout_sockaddr(&ep->sa, data, szp, t);
+	nni_mtx_unlock(&ep->mtx);
+	return (rv);
 }
 
 static int
@@ -868,7 +876,9 @@ ipc_ep_set_sec_desc(void *arg, const void *data, size_t sz, nni_opt_type t)
 	int     rv;
 
 	if ((rv = nni_copyin_ptr(&ptr, data, sz, t)) == 0) {
+		nni_mtx_lock(&ep->mtx);
 		rv = nni_plat_ipc_ep_set_security_descriptor(ep->iep, ptr);
+		nni_mtx_unlock(&ep->mtx);
 	}
 	return (rv);
 }
diff --git a/src/transport/tcp/tcp.c b/src/transport/tcp/tcp.c
index 1a183ecd..e8aa04d0 100644
--- a/src/transport/tcp/tcp.c
+++ b/src/transport/tcp/tcp.c
@@ -882,7 +882,11 @@ static int
 tcp_ep_get_nodelay(void *arg, void *v, size_t *szp, nni_opt_type t)
 {
 	tcp_ep *ep = arg;
-	return (nni_copyout_bool(ep->nodelay, v, szp, t));
+	int     rv;
+	nni_mtx_lock(&ep->mtx);
+	rv = nni_copyout_bool(ep->nodelay, v, szp, t);
+	nni_mtx_unlock(&ep->mtx);
+	return (rv);
 }
 
 static int
@@ -903,7 +907,11 @@ static int
 tcp_ep_get_keepalive(void *arg, void *v, size_t *szp, nni_opt_type t)
 {
 	tcp_ep *ep = arg;
-	return (nni_copyout_bool(ep->keepalive, v, szp, t));
+	int     rv;
+	nni_mtx_lock(&ep->mtx);
+	rv = nni_copyout_bool(ep->keepalive, v, szp, t);
+	nni_mtx_unlock(&ep->mtx);
+	return (rv);
 }
 
 static int
@@ -931,7 +939,11 @@ static int
 tcp_ep_get_recvmaxsz(void *arg, void *v, size_t *szp, nni_opt_type t)
 {
 	tcp_ep *ep = arg;
-	return (nni_copyout_size(ep->rcvmax, v, szp, t));
+	int     rv;
+	nni_mtx_lock(&ep->mtx);
+	rv = nni_copyout_size(ep->rcvmax, v, szp, t);
+	nni_mtx_unlock(&ep->mtx);
+	return (rv);
 }
 
 static nni_tran_option tcp_pipe_options[] = {
diff --git a/src/transport/tls/tls.c b/src/transport/tls/tls.c
index b4f555da..5e1a1e8d 100644
--- a/src/transport/tls/tls.c
+++ b/src/transport/tls/tls.c
@@ -877,7 +877,11 @@ static int
 tls_ep_get_nodelay(void *arg, void *v, size_t *szp, nni_opt_type t)
 {
 	tls_ep *ep = arg;
-	return (nni_copyout_bool(ep->nodelay, v, szp, t));
+	int     rv;
+	nni_mtx_lock(&ep->mtx);
+	rv = nni_copyout_bool(ep->nodelay, v, szp, t);
+	nni_mtx_unlock(&ep->mtx);
+	return (rv);
 }
 
 static int
@@ -898,7 +902,11 @@ static int
 tls_ep_get_keepalive(void *arg, void *v, size_t *szp, nni_opt_type t)
 {
 	tls_ep *ep = arg;
-	return (nni_copyout_bool(ep->keepalive, v, szp, t));
+	int     rv;
+	nni_mtx_lock(&ep->mtx);
+	rv = nni_copyout_bool(ep->keepalive, v, szp, t);
+	nni_mtx_unlock(&ep->mtx);
+	return (rv);
 }
 
 static int
@@ -947,7 +955,11 @@ static int
 tls_ep_get_recvmaxsz(void *arg, void *v, size_t *szp, nni_opt_type t)
 {
 	tls_ep *ep = arg;
-	return (nni_copyout_size(ep->rcvmax, v, szp, t));
+	int     rv;
+	nni_mtx_lock(&ep->mtx);
+	rv = nni_copyout_size(ep->rcvmax, v, szp, t);
+	nni_mtx_unlock(&ep->mtx);
+	return (rv);
 }
 
 static int
@@ -974,9 +986,11 @@ tls_ep_set_config(void *arg, const void *data, size_t sz, nni_opt_type t)
 	if (cfg == NULL) {
 		return (NNG_EINVAL);
 	}
+	nni_mtx_lock(&ep->mtx);
 	old = ep->cfg;
 	nni_tls_config_hold(cfg);
 	ep->cfg = cfg;
+	nni_mtx_unlock(&ep->mtx);
 	if (old != NULL) {
 		nni_tls_config_fini(old);
 	}
@@ -987,7 +1001,11 @@ static int
 tls_ep_get_config(void *arg, void *v, size_t *szp, nni_opt_type t)
 {
 	tls_ep *ep = arg;
-	return (nni_copyout_ptr(ep->cfg, v, szp, t));
+	int     rv;
+	nni_mtx_lock(&ep->mtx);
+	rv = nni_copyout_ptr(ep->cfg, v, szp, t);
+	nni_mtx_unlock(&ep->mtx);
+	return (rv);
 }
 
 static int
@@ -1055,7 +1073,9 @@ tls_ep_set_cert_key_file(void *arg, const void *v, size_t sz, nni_opt_type t)
 	int     rv;
 
 	if ((rv = tls_ep_chk_string(v, sz, t)) == 0) {
+		nni_mtx_lock(&ep->mtx);
 		rv = nng_tls_config_cert_key_file(ep->cfg, v, NULL);
+		nni_mtx_unlock(&ep->mtx);
 	}
 	return (rv);
 }
diff --git a/src/transport/ws/websocket.c b/src/transport/ws/websocket.c
index 2fa0fd67..b3aef756 100644
--- a/src/transport/ws/websocket.c
+++ b/src/transport/ws/websocket.c
@@ -416,7 +416,11 @@ static int
 ws_dialer_get_recvmaxsz(void *arg, void *v, size_t *szp, nni_opt_type t)
 {
 	ws_dialer *d = arg;
-	return (nni_copyout_size(d->rcvmax, v, szp, t));
+	int        rv;
+	nni_mtx_lock(&d->mtx);
+	rv = nni_copyout_size(d->rcvmax, v, szp, t);
+	nni_mtx_unlock(&d->mtx);
+	return (rv);
 }
 
 static int
@@ -439,7 +443,11 @@ static int
 ws_listener_get_recvmaxsz(void *arg, void *v, size_t *szp, nni_opt_type t)
 {
 	ws_listener *l = arg;
-	return (nni_copyout_size(l->rcvmax, v, szp, t));
+	int          rv;
+	nni_mtx_lock(&l->mtx);
+	rv = nni_copyout_size(l->rcvmax, v, szp, t);
+	nni_mtx_unlock(&l->mtx);
+	return (rv);
 }
 
 static int
@@ -538,7 +546,9 @@ ws_dialer_set_reqhdrs(void *arg, const void *v, size_t sz, nni_opt_type t)
 	}
 
 	if ((rv = ws_check_string(v, sz, t)) == 0) {
+		nni_mtx_lock(&d->mtx);
 		rv = ws_set_headers(&d->headers, v);
+		nni_mtx_unlock(&d->mtx);
 	}
 	return (rv);
 }
@@ -553,7 +563,9 @@ ws_listener_set_reshdrs(void *arg, const void *v, size_t sz, nni_opt_type t)
 		return (NNG_EBUSY);
 	}
 	if ((rv = ws_check_string(v, sz, t)) == 0) {
+		nni_mtx_lock(&l->mtx);
 		rv = ws_set_headers(&l->headers, v);
+		nni_mtx_unlock(&l->mtx);
 	}
 	return (rv);
 }
diff --git a/src/transport/zerotier/zerotier.c b/src/transport/zerotier/zerotier.c
index 3535a248..a2163e3f 100644
--- a/src/transport/zerotier/zerotier.c
+++ b/src/transport/zerotier/zerotier.c
@@ -2600,7 +2600,9 @@ zt_ep_set_recvmaxsz(void *arg, const void *data, size_t sz, nni_opt_type t)
 	int    rv;
 
 	if ((rv = nni_copyin_size(&val, data, sz, 0, NNI_MAXSZ, t)) == 0) {
+		nni_mtx_lock(&zt_lk);
 		ep->ze_rcvmax = val;
+		nni_mtx_unlock(&zt_lk);
 	}
 	return (rv);
 }
@@ -2609,7 +2611,11 @@ static int
 zt_ep_get_recvmaxsz(void *arg, void *data, size_t *szp, nni_opt_type t)
 {
 	zt_ep *ep = arg;
-	return (nni_copyout_size(ep->ze_rcvmax, data, szp, t));
+	int    rv;
+	nni_mtx_lock(&zt_lk);
+	rv = nni_copyout_size(ep->ze_rcvmax, data, szp, t);
+	nni_mtx_unlock(&zt_lk);
+	return (rv);
 }
 
 static int
@@ -2634,16 +2640,16 @@ zt_ep_set_home(void *arg, const void *data, size_t sz, nni_opt_type t)
 	zt_ep *ep = arg;
 
 	if ((rv = zt_ep_chk_string(data, sz, t)) == 0) {
+		nni_mtx_lock(&zt_lk);
 		if (ep->ze_running) {
 			rv = NNG_ESTATE;
 		} else {
-			nni_mtx_lock(&zt_lk);
 			nni_strlcpy(ep->ze_home, data, sizeof(ep->ze_home));
 			if ((rv = zt_node_find(ep)) != 0) {
 				ep->ze_ztn = NULL;
 			}
-			nni_mtx_unlock(&zt_lk);
 		}
+		nni_mtx_unlock(&zt_lk);
 	}
 
 	return (rv);
@@ -2653,7 +2659,12 @@ static int
 zt_ep_get_home(void *arg, void *data, size_t *szp, nni_opt_type t)
 {
 	zt_ep *ep = arg;
-	return (nni_copyout_str(ep->ze_home, data, szp, t));
+	int    rv;
+
+	nni_mtx_lock(&zt_lk);
+	rv = nni_copyout_str(ep->ze_home, data, szp, t);
+	nni_mtx_unlock(&zt_lk);
+	return (rv);
 }
 
 static int
@@ -2663,11 +2674,13 @@ zt_ep_get_url(void *arg, void *data, size_t *szp, nni_opt_type t)
 	zt_ep *  ep = arg;
 	uint64_t addr;
 
+	nni_mtx_lock(&zt_lk);
 	addr = ep->ze_mode == NNI_EP_MODE_DIAL ? ep->ze_raddr : ep->ze_laddr;
 	snprintf(ustr, sizeof(ustr), "zt://%llx.%llx:%u",
 	    (unsigned long long) addr >> zt_port_shift,
 	    (unsigned long long) ep->ze_nwid,
 	    (unsigned) (addr & zt_port_mask));
+	nni_mtx_unlock(&zt_lk);
 	return (nni_copyout_str(ustr, data, szp, t));
 }
 
@@ -2750,14 +2763,24 @@ static int
 zt_ep_get_node(void *arg, void *data, size_t *szp, nni_opt_type t)
 {
 	zt_ep *ep = arg;
-	return (nni_copyout_u64(ep->ze_ztn->zn_self, data, szp, t));
+	int    rv;
+
+	nni_mtx_lock(&zt_lk);
+	rv = nni_copyout_u64(ep->ze_ztn->zn_self, data, szp, t);
+	nni_mtx_unlock(&zt_lk);
+	return (rv);
 }
 
 static int
 zt_ep_get_nwid(void *arg, void *data, size_t *szp, nni_opt_type t)
 {
 	zt_ep *ep = arg;
-	return (nni_copyout_u64(ep->ze_nwid, data, szp, t));
+	int    rv;
+
+	nni_mtx_lock(&zt_lk);
+	rv = nni_copyout_u64(ep->ze_nwid, data, szp, t);
+	nni_mtx_unlock(&zt_lk);
+	return (rv);
 }
 
 static int
@@ -2788,7 +2811,9 @@ zt_ep_set_ping_time(void *arg, const void *data, size_t sz, nni_opt_type t)
 	int          rv;
 
 	if ((rv = nni_copyin_ms(&val, data, sz, t)) == 0) {
+		nni_mtx_lock(&zt_lk);
 		ep->ze_ping_time = val;
+		nni_mtx_unlock(&zt_lk);
 	}
 	return (rv);
 }
@@ -2797,7 +2822,12 @@ static int
 zt_ep_get_ping_time(void *arg, void *data, size_t *szp, nni_opt_type t)
 {
 	zt_ep *ep = arg;
-	return (nni_copyout_ms(ep->ze_ping_time, data, szp, t));
+	int    rv;
+
+	nni_mtx_lock(&zt_lk);
+	rv = nni_copyout_ms(ep->ze_ping_time, data, szp, t);
+	nni_mtx_unlock(&zt_lk);
+	return (rv);
 }
 
 static int
@@ -2814,7 +2844,9 @@ zt_ep_set_ping_tries(void *arg, const void *data, size_t sz, nni_opt_type t)
 	int    rv;
 
 	if ((rv = nni_copyin_int(&val, data, sz, 0, 1000000, t)) == 0) {
+		nni_mtx_lock(&zt_lk);
 		ep->ze_ping_tries = val;
+		nni_mtx_unlock(&zt_lk);
 	}
 	return (rv);
 }
@@ -2823,7 +2855,12 @@ static int
 zt_ep_get_ping_tries(void *arg, void *data, size_t *szp, nni_opt_type t)
 {
 	zt_ep *ep = arg;
-	return (nni_copyout_int(ep->ze_ping_tries, data, szp, t));
+	int    rv;
+
+	nni_mtx_lock(&zt_lk);
+	rv = nni_copyout_int(ep->ze_ping_tries, data, szp, t);
+	nni_mtx_unlock(&zt_lk);
+	return (rv);
 }
 
 static int
@@ -2834,7 +2871,9 @@ zt_ep_set_conn_time(void *arg, const void *data, size_t sz, nni_opt_type t)
 	int          rv;
 
 	if ((rv = nni_copyin_ms(&val, data, sz, t)) == 0) {
+		nni_mtx_lock(&zt_lk);
 		ep->ze_conn_time = val;
+		nni_mtx_unlock(&zt_lk);
 	}
 	return (rv);
 }
@@ -2843,7 +2882,12 @@ static int
 zt_ep_get_conn_time(void *arg, void *data, size_t *szp, nni_opt_type t)
 {
 	zt_ep *ep = arg;
-	return (nni_copyout_ms(ep->ze_conn_time, data, szp, t));
+	int    rv;
+
+	nni_mtx_lock(&zt_lk);
+	rv = nni_copyout_ms(ep->ze_conn_time, data, szp, t);
+	nni_mtx_unlock(&zt_lk);
+	return (rv);
 }
 
 static int
@@ -2854,7 +2898,9 @@ zt_ep_set_conn_tries(void *arg, const void *data, size_t sz, nni_opt_type t)
 	int    rv;
 
 	if ((rv = nni_copyin_int(&val, data, sz, 0, 1000000, t)) == 0) {
+		nni_mtx_lock(&zt_lk);
 		ep->ze_conn_tries = val;
+		nni_mtx_unlock(&zt_lk);
 	}
 	return (rv);
 }
@@ -2863,7 +2909,12 @@ static int
 zt_ep_get_conn_tries(void *arg, void *data, size_t *szp, nni_opt_type t)
 {
 	zt_ep *ep = arg;
-	return (nni_copyout_int(ep->ze_conn_tries, data, szp, t));
+	int    rv;
+
+	nni_mtx_lock(&zt_lk);
+	rv = nni_copyout_int(ep->ze_conn_tries, data, szp, t);
+	nni_mtx_unlock(&zt_lk);
+	return (rv);
 }
 
 static int