fixes #1289 zerotier should have it's own copy of the id hashing code

fixes #1288 id allocation can overallocate
fixes #1126 consider removing lock from idhash

This substantially refactors the id hash code, giving a cleaner API,
and eliminating a extra locking as well as some wasteful allocations.

The ZeroTier code has it's own copy, that is 64-bit friendly, as the
rest of the consumers need only a simpler 32-bit API.

8 files changed, 285 insertions, 422 deletions

diff --git a/src/core/dialer.c b/src/core/dialer.c
index fe9cb92f..80b93dc0 100644
--- a/src/core/dialer.c
+++ b/src/core/dialer.c
@@ -20,21 +20,16 @@ static void dialer_connect_start(nni_dialer *);
 static void dialer_connect_cb(void *);
 static void dialer_timer_cb(void *);
 
-static nni_idhash *dialers;
-static nni_mtx     dialers_lk;
+static nni_id_map dialers;
+static nni_mtx    dialers_lk;
 
 #define BUMP_STAT(x) nni_stat_inc_atomic(x, 1)
 
 int
 nni_dialer_sys_init(void)
 {
-	int rv;
-
-	if ((rv = nni_idhash_init(&dialers)) != 0) {
-		return (rv);
-	}
+	nni_id_map_init(&dialers, 1, 0x7fffffff, false);
 	nni_mtx_init(&dialers_lk);
-	nni_idhash_set_limits(dialers, 1, 0x7fffffff, 1);
 
 	return (0);
 }
@@ -44,8 +39,7 @@ nni_dialer_sys_fini(void)
 {
 	nni_reap_drain();
 	nni_mtx_fini(&dialers_lk);
-	nni_idhash_fini(dialers);
-	dialers = NULL;
+	nni_id_map_fini(&dialers);
 }
 
 uint32_t
@@ -57,11 +51,11 @@ nni_dialer_id(nni_dialer *d)
 void
 nni_dialer_destroy(nni_dialer *d)
 {
-	nni_aio_stop(d->d_con_aio);
-	nni_aio_stop(d->d_tmo_aio);
+	nni_aio_stop(&d->d_con_aio);
+	nni_aio_stop(&d->d_tmo_aio);
 
-	nni_aio_free(d->d_con_aio);
-	nni_aio_free(d->d_tmo_aio);
+	nni_aio_fini(&d->d_con_aio);
+	nni_aio_fini(&d->d_tmo_aio);
 
 	if (d->d_data != NULL) {
 		d->d_ops.d_fini(d->d_data);
@@ -112,7 +106,7 @@ dialer_stats_init(nni_dialer *d)
 	nni_stat_init_atomic(&st->s_etimedout, "timedout", "timed out");
 	nni_stat_add(root, &st->s_etimedout);
 
-	nni_stat_init_atomic(&st->s_eproto, "protoerr", "protcol errors");
+	nni_stat_init_atomic(&st->s_eproto, "protoerr", "protocol errors");
 	nni_stat_add(root, &st->s_eproto);
 
 	nni_stat_init_atomic(&st->s_eauth, "autherr", "auth errors");
@@ -204,11 +198,18 @@ nni_dialer_create(nni_dialer **dp, nni_sock *s, const char *urlstr)
 	nni_mtx_init(&d->d_mtx);
 
 	dialer_stats_init(d);
-	if (((rv = nni_aio_alloc(&d->d_con_aio, dialer_connect_cb, d)) != 0) ||
-	    ((rv = nni_aio_alloc(&d->d_tmo_aio, dialer_timer_cb, d)) != 0) ||
-	    ((rv = d->d_ops.d_init(&d->d_data, url, d)) != 0) ||
-	    ((rv = nni_idhash_alloc32(dialers, &d->d_id, d)) != 0) ||
+	nni_aio_init(&d->d_con_aio, dialer_connect_cb, d);
+	nni_aio_init(&d->d_tmo_aio, dialer_timer_cb, d);
+
+	nni_mtx_lock(&dialers_lk);
+	rv = nni_id_alloc(&dialers, &d->d_id, d);
+	nni_mtx_unlock(&dialers_lk);
+
+	if ((rv != 0) || ((rv = d->d_ops.d_init(&d->d_data, url, d)) != 0) ||
 	    ((rv = nni_sock_add_dialer(s, d)) != 0)) {
+		nni_mtx_lock(&dialers_lk);
+		nni_id_remove(&dialers, d->d_id);
+		nni_mtx_unlock(&dialers_lk);
 		nni_dialer_destroy(d);
 		return (rv);
 	}
@@ -232,16 +233,12 @@ nni_dialer_find(nni_dialer **dp, uint32_t id)
 	}
 
 	nni_mtx_lock(&dialers_lk);
-	if ((rv = nni_idhash_find(dialers, id, (void **) &d)) == 0) {
-		if (d->d_closed) {
-			rv = NNG_ECLOSED;
-		} else {
-			d->d_refcnt++;
-			*dp = d;
-		}
+	if ((d = nni_id_get(&dialers, id)) != NULL) {
+		d->d_refcnt++;
+		*dp = d;
 	}
 	nni_mtx_unlock(&dialers_lk);
-	return (rv);
+	return (d == NULL ? NNG_ENOENT : 0);
 }
 
 int
@@ -280,14 +277,9 @@ nni_dialer_close_rele(nni_dialer *d)
 		return;
 	}
 	d->d_closed = true;
+	nni_id_remove(&dialers, d->d_id);
 	nni_mtx_unlock(&dialers_lk);
 
-	// Remove us from the table so we cannot be found.
-	// This is done fairly early in the teardown process.
-	// If we're here, either the socket or the listener has been
-	// closed at the user request, so there would be a race anyway.
-	nni_idhash_remove(dialers, d->d_id);
-
 	nni_dialer_rele(d);
 }
 
@@ -301,14 +293,9 @@ nni_dialer_close(nni_dialer *d)
 		return;
 	}
 	d->d_closed = true;
+	nni_id_remove(&dialers, d->d_id);
 	nni_mtx_unlock(&dialers_lk);
 
-	// Remove us from the table so we cannot be found.
-	// This is done fairly early in the teardown process.
-	// If we're here, either the socket or the listener has been
-	// closed at the user request, so there would be a race anyway.
-	nni_idhash_remove(dialers, d->d_id);
-
 	nni_dialer_shutdown(d);
 
 	nni_dialer_rele(d);
@@ -317,10 +304,9 @@ nni_dialer_close(nni_dialer *d)
 static void
 dialer_timer_cb(void *arg)
 {
-	nni_dialer *d   = arg;
-	nni_aio *   aio = d->d_tmo_aio;
+	nni_dialer *d = arg;
 
-	if (nni_aio_result(aio) == 0) {
+	if (nni_aio_result(&d->d_tmo_aio) == 0) {
 		dialer_connect_start(d);
 	}
 }
@@ -329,7 +315,7 @@ static void
 dialer_connect_cb(void *arg)
 {
 	nni_dialer *d   = arg;
-	nni_aio *   aio = d->d_con_aio;
+	nni_aio *   aio = &d->d_con_aio;
 	nni_aio *   user_aio;
 	int         rv;
 
@@ -366,13 +352,11 @@ dialer_connect_cb(void *arg)
 static void
 dialer_connect_start(nni_dialer *d)
 {
-	nni_aio *aio = d->d_con_aio;
-
-	d->d_ops.d_connect(d->d_data, aio);
+	d->d_ops.d_connect(d->d_data, &d->d_con_aio);
 }
 
 int
-nni_dialer_start(nni_dialer *d, int flags)
+nni_dialer_start(nni_dialer *d, unsigned flags)
 {
 	int      rv = 0;
 	nni_aio *aio;
diff --git a/src/core/dialer.h b/src/core/dialer.h
index 200ad01d..df923209 100644
--- a/src/core/dialer.h
+++ b/src/core/dialer.h
@@ -1,5 +1,5 @@
 //
-// Copyright 2019 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2020 Staysail Systems, Inc. <info@staysail.tech>
 // Copyright 2018 Capitar IT Group BV <info@capitar.com>
 // Copyright 2018 Devolutions <info@devolutions.net>
 //
@@ -20,7 +20,7 @@ extern void      nni_dialer_rele(nni_dialer *);
 extern uint32_t  nni_dialer_id(nni_dialer *);
 extern int       nni_dialer_create(nni_dialer **, nni_sock *, const char *);
 extern void      nni_dialer_close(nni_dialer *);
-extern int       nni_dialer_start(nni_dialer *, int);
+extern int       nni_dialer_start(nni_dialer *, unsigned);
 extern nni_sock *nni_dialer_sock(nni_dialer *);
 
 extern int nni_dialer_setopt(
diff --git a/src/core/idhash.c b/src/core/idhash.c
index 80613ce0..0edf16dc 100644
--- a/src/core/idhash.c
+++ b/src/core/idhash.c
@@ -1,5 +1,5 @@
 //
-// Copyright 2018 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2020 Staysail Systems, Inc. <info@staysail.tech>
 // Copyright 2018 Capitar IT Group BV <info@capitar.com>
 //
 // This software is supplied under the terms of the MIT License, a
@@ -12,109 +12,77 @@
 
 #include <string.h>
 
-struct nni_idhash_entry {
-	uint64_t ihe_key;
-	void *   ihe_val;
-	uint32_t ihe_skips;
+struct nni_id_entry {
+	uint32_t key;
+	uint32_t skips;
+	void *   val;
 };
 
-struct nni_idhash {
-	size_t            ih_cap;
-	size_t            ih_count;
-	size_t            ih_load;
-	size_t            ih_minload; // considers placeholders
-	size_t            ih_maxload;
-	uint64_t          ih_minval;
-	uint64_t          ih_maxval;
-	uint64_t          ih_dynval;
-	nni_idhash_entry *ih_entries;
-	nni_mtx           ih_mtx;
-};
-
-int
-nni_idhash_init(nni_idhash **hp)
-{
-	nni_idhash *h;
-
-	if ((h = NNI_ALLOC_STRUCT(h)) == NULL) {
-		return (NNG_ENOMEM);
-	}
-	nni_mtx_init(&h->ih_mtx);
-	h->ih_entries = NULL;
-	h->ih_count   = 0;
-	h->ih_load    = 0;
-	h->ih_cap     = 0;
-	h->ih_maxload = 0;
-	h->ih_minload = 0; // never shrink below this
-	h->ih_minval  = 0;
-	h->ih_maxval  = 0xffffffff;
-	h->ih_dynval  = 0;
-	*hp           = h;
-	return (0);
-}
-
 void
-nni_idhash_fini(nni_idhash *h)
+nni_id_map_init(nni_id_map *m, uint32_t lo, uint32_t hi, bool randomize)
 {
-	if (h != NULL) {
-		if (h->ih_entries != NULL) {
-			NNI_FREE_STRUCTS(h->ih_entries, h->ih_cap);
-			h->ih_entries = NULL;
-			h->ih_cap = h->ih_count = 0;
-			h->ih_load = h->ih_minload = h->ih_maxload = 0;
-		}
-		nni_mtx_fini(&h->ih_mtx);
-		NNI_FREE_STRUCT(h);
+	if (lo == 0) {
+		lo = 1;
+	}
+	if (hi == 0) {
+		hi = 0xffffffffu;
+	}
+	NNI_ASSERT(lo != 0);
+	NNI_ASSERT(hi > lo);
+	m->id_entries  = NULL;
+	m->id_count    = 0;
+	m->id_load     = 0;
+	m->id_cap      = 0;
+	m->id_max_load = 0;
+	m->id_min_load = 0; // never shrink below this
+	m->id_min_val  = lo;
+	m->id_max_val  = hi;
+	if (randomize) {
+		// NB: The range is inclusive.
+		m->id_dyn_val = nni_random() % ((hi - lo) + 1) + lo;
+	} else {
+		m->id_dyn_val = lo;
 	}
 }
 
 void
-nni_idhash_set_limits(
-    nni_idhash *h, uint64_t minval, uint64_t maxval, uint64_t start)
+nni_id_map_fini(nni_id_map *m)
 {
-	if (start < minval) {
-		start = minval;
-	}
-	if (start > maxval) {
-		start = maxval;
+	if (m->id_entries != NULL) {
+		NNI_FREE_STRUCTS(m->id_entries, m->id_cap);
+		m->id_entries = NULL;
+		m->id_cap = m->id_count = 0;
+		m->id_load = m->id_min_load = m->id_max_load = 0;
 	}
-
-	nni_mtx_lock(&h->ih_mtx);
-	h->ih_minval = minval;
-	h->ih_maxval = maxval;
-	h->ih_dynval = start;
-	NNI_ASSERT(minval < maxval);
-	NNI_ASSERT(start >= minval);
-	NNI_ASSERT(start <= maxval);
-	nni_mtx_unlock(&h->ih_mtx);
 }
 
 // Inspired by Python dict implementation.  This probe will visit every
-// cell.  We always hash consecutively assigned IDs.
-#define NNI_IDHASH_NEXTPROBE(h, j) ((((j) *5) + 1) & (h->ih_cap - 1))
-#define NNI_IDHASH_INDEX(h, j) ((j) & (h->ih_cap - 1))
+// cell.  We always hash consecutively assigned IDs.  This requires that
+// the capacity is always a power of two.
+#define ID_NEXT(m, j) ((((j) *5) + 1) & (m->id_cap - 1))
+#define ID_INDEX(m, j) ((j) & (m->id_cap - 1))
 
 static size_t
-nni_hash_find_index(nni_idhash *h, uint64_t id)
+id_find(nni_id_map *m, uint32_t id)
 {
 	size_t index;
 	size_t start;
-	if (h->ih_count == 0) {
+	if (m->id_count == 0) {
 		return ((size_t) -1);
 	}
 
-	index = NNI_IDHASH_INDEX(h, id);
+	index = ID_INDEX(m, id);
 	start = index;
 	for (;;) {
 		// The value of ihe_key is only valid if ihe_val is not NULL.
-		if ((h->ih_entries[index].ihe_key == id) &&
-		    (h->ih_entries[index].ihe_val != NULL)) {
+		if ((m->id_entries[index].key == id) &&
+		    (m->id_entries[index].val != NULL)) {
 			return (index);
 		}
-		if (h->ih_entries[index].ihe_skips == 0) {
+		if (m->id_entries[index].skips == 0) {
 			return ((size_t) -1);
 		}
-		index = NNI_IDHASH_NEXTPROBE(h, index);
+		index = ID_NEXT(m, index);
 
 		if (index == start) {
 			break;
@@ -124,249 +92,196 @@ nni_hash_find_index(nni_idhash *h, uint64_t id)
 	return ((size_t) -1);
 }
 
-static int
-nni_hash_find(nni_idhash *h, uint64_t id, void **valp)
+void *
+nni_id_get(nni_id_map *m, uint32_t id)
 {
 	size_t index;
-	if ((index = nni_hash_find_index(h, id)) == (size_t) -1) {
-		return (NNG_ENOENT);
+	if ((index = id_find(m, id)) == (size_t) -1) {
+		return (NULL);
 	}
-	*valp = h->ih_entries[index].ihe_val;
-	return (0);
-}
-
-int
-nni_idhash_find(nni_idhash *h, uint64_t id, void **valp)
-{
-	int rv;
-
-	nni_mtx_lock(&h->ih_mtx);
-	rv = nni_hash_find(h, id, valp);
-	nni_mtx_unlock(&h->ih_mtx);
-	return (rv);
+	return (m->id_entries[index].val);
 }
 
 static int
-nni_hash_resize(nni_idhash *h)
+id_resize(nni_id_map *m)
 {
-	size_t            newsize;
-	size_t            oldsize;
-	nni_idhash_entry *newents;
-	nni_idhash_entry *oldents;
-	uint32_t          i;
+	size_t        new_cap;
+	size_t        old_cap;
+	nni_id_entry *new_entries;
+	nni_id_entry *old_entries;
+	uint32_t      i;
 
-	if ((h->ih_load < h->ih_maxload) && (h->ih_load >= h->ih_minload)) {
+	if ((m->id_load < m->id_max_load) && (m->id_load >= m->id_min_load)) {
 		// No resize needed.
 		return (0);
 	}
 
-	oldsize = h->ih_cap;
-
-	newsize = 8;
-	while (newsize < (h->ih_count * 2)) {
-		newsize *= 2;
+	old_cap = m->id_cap;
+	new_cap = 8;
+	while (new_cap < (m->id_count * 2)) {
+		new_cap *= 2;
 	}
-	if (newsize == oldsize) {
+	if (new_cap == old_cap) {
 		// Same size.
 		return (0);
 	}
 
-	oldents = h->ih_entries;
-	newents = NNI_ALLOC_STRUCTS(newents, newsize);
-	if (newents == NULL) {
+	old_entries = m->id_entries;
+	new_entries = NNI_ALLOC_STRUCTS(new_entries, new_cap);
+	if (new_entries == NULL) {
 		return (NNG_ENOMEM);
 	}
 
-	h->ih_entries = newents;
-	h->ih_cap     = newsize;
-	if (newsize > 8) {
-		h->ih_minload = newsize / 8;
-		h->ih_maxload = newsize * 2 / 3;
+	m->id_entries = new_entries;
+	m->id_cap     = new_cap;
+	m->id_load    = 0;
+	if (new_cap > 8) {
+		m->id_min_load = new_cap / 8;
+		m->id_max_load = new_cap * 2 / 3;
 	} else {
-		h->ih_minload = 0;
-		h->ih_maxload = 5;
+		m->id_min_load = 0;
+		m->id_max_load = 5;
 	}
-	for (i = 0; i < oldsize; i++) {
+	for (i = 0; i < old_cap; i++) {
 		size_t index;
-		if (oldents[i].ihe_val == NULL) {
+		if (old_entries[i].val == NULL) {
 			continue;
 		}
-		index = oldents[i].ihe_key & (newsize - 1);
+		index = old_entries[i].key & (new_cap - 1);
 		for (;;) {
 			// Increment the load unconditionally.  It counts
 			// once for every item stored, plus once for each
 			// hashing operation we use to store the item (i.e.
 			// one for the item, plus once for each rehash.)
-			h->ih_load++;
-			if (newents[index].ihe_val == NULL) {
+			m->id_load++;
+			if (new_entries[index].val == NULL) {
 				// As we are hitting this entry for the first
 				// time, it won't have any skips.
-				NNI_ASSERT(newents[index].ihe_skips == 0);
-				newents[index].ihe_val = oldents[i].ihe_val;
-				newents[index].ihe_key = oldents[i].ihe_key;
+				NNI_ASSERT(new_entries[index].skips == 0);
+				new_entries[index].val = old_entries[i].val;
+				new_entries[index].key = old_entries[i].key;
 				break;
 			}
-			newents[index].ihe_skips++;
-			index = NNI_IDHASH_NEXTPROBE(h, index);
+			new_entries[index].skips++;
+			index = ID_NEXT(m, index);
 		}
 	}
-	if (oldsize != 0) {
-		NNI_FREE_STRUCTS(oldents, oldsize);
+	if (old_cap != 0) {
+		NNI_FREE_STRUCTS(old_entries, old_cap);
 	}
 	return (0);
 }
 
 int
-nni_idhash_remove(nni_idhash *h, uint64_t id)
+nni_id_remove(nni_id_map *m, uint32_t id)
 {
-	size_t            index;
-	size_t            srch;
-	nni_idhash_entry *ent;
+	size_t index;
+	size_t probe;
 
-	nni_mtx_lock(&h->ih_mtx);
-	if ((index = nni_hash_find_index(h, id)) == (size_t) -1) {
-		nni_mtx_unlock(&h->ih_mtx);
+	if ((index = id_find(m, id)) == (size_t) -1) {
 		return (NNG_ENOENT);
 	}
 
 	// Now we have found the index where the object exists.  We are going
 	// to restart the search, until the index matches, to decrement the
 	// skips counter.
-	srch = (int) NNI_IDHASH_INDEX(h, id);
+	probe = ID_INDEX(m, id);
 
 	for (;;) {
+		nni_id_entry *entry;
+
 		// The load was increased once each hashing operation we used
 		// to place the the item.  Decrement it accordingly.
-		h->ih_load--;
-		ent = &h->ih_entries[srch];
-		if (srch == index) {
-			ent->ihe_val = NULL;
-			ent->ihe_key = (uint64_t) -1; // garbage key
+		m->id_load--;
+		entry = &m->id_entries[probe];
+		if (probe == index) {
+			entry->val = NULL;
+			entry->key = 0; // invalid key
 			break;
 		}
-		NNI_ASSERT(ent->ihe_skips > 0);
-		ent->ihe_skips--;
-		srch = NNI_IDHASH_NEXTPROBE(h, srch);
+		NNI_ASSERT(entry->skips > 0);
+		entry->skips--;
+		probe = ID_NEXT(m, probe);
 	}
 
-	h->ih_count--;
+	m->id_count--;
 
 	// Shrink -- but it's ok if we can't.
-	(void) nni_hash_resize(h);
-
-	nni_mtx_unlock(&h->ih_mtx);
+	(void) id_resize(m);
 
 	return (0);
 }
 
-static int
-nni_hash_insert(nni_idhash *h, uint64_t id, void *val)
+int
+nni_id_set(nni_id_map *m, uint32_t id, void *val)
 {
-	size_t            index;
-	nni_idhash_entry *ent;
+	size_t        index;
+	nni_id_entry *ent;
 
 	// Try to resize -- if we don't need to, this will be a no-op.
-	if (nni_hash_resize(h) != 0) {
+	if (id_resize(m) != 0) {
 		return (NNG_ENOMEM);
 	}
 
 	// If it already exists, just overwrite the old value.
-	if ((index = nni_hash_find_index(h, id)) != (size_t) -1) {
-		ent          = &h->ih_entries[index];
-		ent->ihe_val = val;
+	if ((index = id_find(m, id)) != (size_t) -1) {
+		ent      = &m->id_entries[index];
+		ent->val = val;
 		return (0);
 	}
 
-	index = NNI_IDHASH_INDEX(h, id);
+	index = ID_INDEX(m, id);
 	for (;;) {
-		ent = &h->ih_entries[index];
+		ent = &m->id_entries[index];
 
 		// Increment the load count.  We do this each time time we
-		// rehash.  This may overcount items that collide on the
+		// rehash.  This may over-count items that collide on the
 		// same rehashing, but this should just cause a table to
 		// grow sooner, which is probably a good thing.
-		h->ih_load++;
-		if (ent->ihe_val == NULL) {
-			h->ih_count++;
-			ent->ihe_key = id;
-			ent->ihe_val = val;
+		m->id_load++;
+		if (ent->val == NULL) {
+			m->id_count++;
+			ent->key = id;
+			ent->val = val;
 			return (0);
 		}
 		// Record the skip count.  This being non-zero informs
 		// that a rehash will be necessary.  Without this we
 		// would need to scan the entire hash for the match.
-		ent->ihe_skips++;
-		index = NNI_IDHASH_NEXTPROBE(h, index);
+		ent->skips++;
+		index = ID_NEXT(m, index);
 	}
 }
 
 int
-nni_idhash_insert(nni_idhash *h, uint64_t id, void *val)
-{
-	int rv;
-
-	nni_mtx_lock(&h->ih_mtx);
-	rv = nni_hash_insert(h, id, val);
-	nni_mtx_unlock(&h->ih_mtx);
-	return (rv);
-}
-
-int
-nni_idhash_alloc(nni_idhash *h, uint64_t *idp, void *val)
+nni_id_alloc(nni_id_map *m, uint32_t *idp, void *val)
 {
 	uint64_t id;
 	int      rv;
-	nni_mtx_lock(&h->ih_mtx);
 
 	NNI_ASSERT(val != NULL);
 
-	if (h->ih_count > (h->ih_maxval - h->ih_minval)) {
+	// range is inclusive, so > to get +1 effect.
+	if (m->id_count > (m->id_max_val - m->id_min_val)) {
 		// Really more like ENOSPC.. the table is filled to max.
-		nni_mtx_unlock(&h->ih_mtx);
-
 		return (NNG_ENOMEM);
 	}
 
 	for (;;) {
-		id = h->ih_dynval;
-		h->ih_dynval++;
-		if (h->ih_dynval > h->ih_maxval) {
-			h->ih_dynval = h->ih_minval;
+		id = m->id_dyn_val;
+		m->id_dyn_val++;
+		if (m->id_dyn_val > m->id_max_val) {
+			m->id_dyn_val = m->id_min_val;
 		}
 
-		if (nni_hash_find_index(h, id) == (size_t) -1) {
+		if (id_find(m, id) == (size_t) -1) {
 			break;
 		}
 	}
 
-	rv = nni_hash_insert(h, id, val);
+	rv = nni_id_set(m, id, val);
 	if (rv == 0) {
 		*idp = id;
 	}
-	nni_mtx_unlock(&h->ih_mtx);
-
 	return (rv);
 }
-
-int
-nni_idhash_alloc32(nni_idhash *h, uint32_t *idp, void *val)
-{
-	uint64_t id64;
-	int      rv;
-
-	if ((rv = nni_idhash_alloc(h, &id64, val)) == 0) {
-		if (id64 > 0xffffffffull) {
-			nni_idhash_remove(h, id64);
-			rv = NNG_EINVAL;
-		} else {
-			*idp = (uint32_t) id64;
-		}
-	}
-	return (rv);
-}
-
-size_t
-nni_idhash_count(nni_idhash *h)
-{
-	return (h->ih_count);
-}
diff --git a/src/core/idhash.h b/src/core/idhash.h
index 5b9ed593..0c9043ce 100644
--- a/src/core/idhash.h
+++ b/src/core/idhash.h
@@ -1,5 +1,5 @@
 //
-// Copyright 2018 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2020 Staysail Systems, Inc. <info@staysail.tech>
 // Copyright 2018 Capitar IT Group BV <info@capitar.com>
 //
 // This software is supplied under the terms of the MIT License, a
@@ -23,20 +23,28 @@
 // use table sizes that are powers of two.  Note that hash items
 // must be non-NULL.  The table is protected by an internal lock.
 
-typedef struct nni_idhash       nni_idhash;
-typedef struct nni_idhash_entry nni_idhash_entry;
-
-extern int  nni_idhash_init(nni_idhash **);
-extern void nni_idhash_fini(nni_idhash *);
-extern void nni_idhash_set_limits(nni_idhash *, uint64_t, uint64_t, uint64_t);
-extern int  nni_idhash_find(nni_idhash *, uint64_t, void **);
-extern int  nni_idhash_remove(nni_idhash *, uint64_t);
-extern int  nni_idhash_insert(nni_idhash *, uint64_t, void *);
-extern int  nni_idhash_alloc(nni_idhash *, uint64_t *, void *);
-
-// 32-bit version of idhash -- limits must have been set accordingly.
-extern int nni_idhash_alloc32(nni_idhash *, uint32_t *, void *);
-
-extern size_t nni_idhash_count(nni_idhash *);
+typedef struct nni_id_map       nni_id_map;
+typedef struct nni_id_entry     nni_id_entry;
+
+// NB: These details are entirely private to the hash implementation.
+// They are provided here to facilitate inlining in structures.
+struct nni_id_map {
+	size_t        id_cap;
+	size_t        id_count;
+	size_t        id_load;
+	size_t        id_min_load; // considers placeholders
+	size_t        id_max_load;
+	uint32_t      id_min_val;
+	uint32_t      id_max_val;
+	uint32_t      id_dyn_val;
+	nni_id_entry *id_entries;
+};
+
+extern void nni_id_map_init(nni_id_map *, uint32_t, uint32_t, bool);
+extern void nni_id_map_fini(nni_id_map *);
+extern void *nni_id_get(nni_id_map *, uint32_t);
+extern int nni_id_set(nni_id_map *, uint32_t, void *);
+extern int nni_id_alloc(nni_id_map *, uint32_t *, void *);
+extern int nni_id_remove(nni_id_map *, uint32_t);
 
 #endif // CORE_IDHASH_H
diff --git a/src/core/listener.c b/src/core/listener.c
index 1fde6e36..56850649 100644
--- a/src/core/listener.c
+++ b/src/core/listener.c
@@ -1,5 +1,5 @@
 //
-// Copyright 2019 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2020 Staysail Systems, Inc. <info@staysail.tech>
 // Copyright 2018 Capitar IT Group BV <info@capitar.com>
 // Copyright 2018 Devolutions <info@devolutions.net>
 //
@@ -21,21 +21,16 @@ static void listener_accept_start(nni_listener *);
 static void listener_accept_cb(void *);
 static void listener_timer_cb(void *);
 
-static nni_idhash *listeners;
-static nni_mtx     listeners_lk;
+static nni_id_map listeners;
+static nni_mtx    listeners_lk;
 
 #define BUMP_STAT(x) nni_stat_inc_atomic(x, 1)
 
 int
 nni_listener_sys_init(void)
 {
-	int rv;
-
-	if ((rv = nni_idhash_init(&listeners)) != 0) {
-		return (rv);
-	}
+	nni_id_map_init(&listeners, 1, 0x7fffffff, false);
 	nni_mtx_init(&listeners_lk);
-	nni_idhash_set_limits(listeners, 1, 0x7fffffff, 1);
 
 	return (0);
 }
@@ -45,8 +40,7 @@ nni_listener_sys_fini(void)
 {
 	nni_reap_drain();
 	nni_mtx_fini(&listeners_lk);
-	nni_idhash_fini(listeners);
-	listeners = NULL;
+	nni_id_map_fini(&listeners);
 }
 
 uint32_t
@@ -58,11 +52,11 @@ nni_listener_id(nni_listener *l)
 void
 nni_listener_destroy(nni_listener *l)
 {
-	nni_aio_stop(l->l_acc_aio);
-	nni_aio_stop(l->l_tmo_aio);
+	nni_aio_stop(&l->l_acc_aio);
+	nni_aio_stop(&l->l_tmo_aio);
 
-	nni_aio_free(l->l_acc_aio);
-	nni_aio_free(l->l_tmo_aio);
+	nni_aio_fini(&l->l_acc_aio);
+	nni_aio_fini(&l->l_tmo_aio);
 
 	if (l->l_data != NULL) {
 		l->l_ops.l_fini(l->l_data);
@@ -109,7 +103,7 @@ listener_stats_init(nni_listener *l)
 	nni_stat_init_atomic(&st->s_etimedout, "timedout", "timed out");
 	nni_stat_add(root, &st->s_etimedout);
 
-	nni_stat_init_atomic(&st->s_eproto, "protoerr", "protcol errors");
+	nni_stat_init_atomic(&st->s_eproto, "protoerr", "protocol errors");
 	nni_stat_add(root, &st->s_eproto);
 
 	nni_stat_init_atomic(&st->s_eauth, "autherr", "auth errors");
@@ -196,11 +190,18 @@ nni_listener_create(nni_listener **lp, nni_sock *s, const char *url_str)
 	NNI_LIST_INIT(&l->l_pipes, nni_pipe, p_ep_node);
 	listener_stats_init(l);
 
-	if (((rv = nni_aio_alloc(&l->l_acc_aio, listener_accept_cb, l)) != 0) ||
-	    ((rv = nni_aio_alloc(&l->l_tmo_aio, listener_timer_cb, l)) != 0) ||
-	    ((rv = l->l_ops.l_init(&l->l_data, url, l)) != 0) ||
-	    ((rv = nni_idhash_alloc32(listeners, &l->l_id, l)) != 0) ||
+	nni_aio_init(&l->l_acc_aio, listener_accept_cb, l);
+	nni_aio_init(&l->l_tmo_aio, listener_timer_cb, l);
+
+	nni_mtx_lock(&listeners_lk);
+	rv = nni_id_alloc(&listeners, &l->l_id, l);
+	nni_mtx_unlock(&listeners_lk);
+
+	if ((rv != 0) || ((rv = l->l_ops.l_init(&l->l_data, url, l)) != 0) ||
 	    ((rv = nni_sock_add_listener(s, l)) != 0)) {
+		nni_mtx_lock(&listeners_lk);
+		nni_id_remove(&listeners, l->l_id);
+		nni_mtx_unlock(&listeners_lk);
 		nni_listener_destroy(l);
 		return (rv);
 	}
@@ -226,16 +227,12 @@ nni_listener_find(nni_listener **lp, uint32_t id)
 	}
 
 	nni_mtx_lock(&listeners_lk);
-	if ((rv = nni_idhash_find(listeners, id, (void **) &l)) == 0) {
-		if (l->l_closed) {
-			rv = NNG_ECLOSED;
-		} else {
-			l->l_refcnt++;
-			*lp = l;
-		}
+	if ((l = nni_id_get(&listeners, id)) != NULL) {
+		l->l_refcnt++;
+		*lp = l;
 	}
 	nni_mtx_unlock(&listeners_lk);
-	return (rv);
+	return (l == NULL ? NNG_ENOENT : 0);
 }
 
 int
@@ -274,14 +271,9 @@ nni_listener_close(nni_listener *l)
 		return;
 	}
 	l->l_closed = true;
+	nni_id_remove(&listeners, l->l_id);
 	nni_mtx_unlock(&listeners_lk);
 
-	// Remove us from the table so we cannot be found.
-	// This is done fairly early in the teardown process.
-	// If we're here, either the socket or the listener has been
-	// closed at the user request, so there would be a race anyway.
-	nni_idhash_remove(listeners, l->l_id);
-
 	nni_listener_shutdown(l);
 
 	nni_listener_rele(l); // This will trigger a reap if id count is zero.
@@ -298,23 +290,18 @@ nni_listener_close_rele(nni_listener *l)
 		return;
 	}
 	l->l_closed = true;
+	nni_id_remove(&listeners, l->l_id);
 	nni_mtx_unlock(&listeners_lk);
 
-	// Remove us from the table so we cannot be found.
-	// This is done fairly early in the teardown process.
-	// If we're here, either the socket or the listener has been
-	// closed at the user request, so there would be a race anyway.
-	nni_idhash_remove(listeners, l->l_id);
 	nni_listener_rele(l); // This will trigger a reap if id count is zero.
 }
 
 static void
 listener_timer_cb(void *arg)
 {
-	nni_listener *l   = arg;
-	nni_aio *     aio = l->l_tmo_aio;
+	nni_listener *l = arg;
 
-	if (nni_aio_result(aio) == 0) {
+	if (nni_aio_result(&l->l_tmo_aio) == 0) {
 		listener_accept_start(l);
 	}
 }
@@ -323,8 +310,8 @@ static void
 listener_accept_cb(void *arg)
 {
 	nni_listener *l   = arg;
-	nni_aio *     aio = l->l_acc_aio;
-	int rv;
+	nni_aio *     aio = &l->l_acc_aio;
+	int           rv;
 
 	switch ((rv = nni_aio_result(aio))) {
 	case 0:
@@ -350,7 +337,7 @@ listener_accept_cb(void *arg)
 		// by not thrashing we give the system a chance to
 		// recover.  100 ms is enough to cool down.
 		nni_listener_bump_error(l, rv);
-		nni_sleep_aio(100, l->l_tmo_aio);
+		nni_sleep_aio(100, &l->l_tmo_aio);
 		break;
 	}
 }
@@ -358,16 +345,14 @@ listener_accept_cb(void *arg)
 static void
 listener_accept_start(nni_listener *l)
 {
-	nni_aio *aio = l->l_acc_aio;
-
 	// Call with the listener lock held.
-	l->l_ops.l_accept(l->l_data, aio);
+	l->l_ops.l_accept(l->l_data, &l->l_acc_aio);
 }
 
 int
 nni_listener_start(nni_listener *l, int flags)
 {
-	int rv = 0;
+	int rv;
 	NNI_ARG_UNUSED(flags);
 
 	if (nni_atomic_flag_test_and_set(&l->l_started)) {
diff --git a/src/core/pipe.c b/src/core/pipe.c
index 4076b62c..b93d9f64 100644
--- a/src/core/pipe.c
+++ b/src/core/pipe.c
@@ -13,34 +13,23 @@
 #include "sockimpl.h"
 
 #include <stdio.h>
-#include <string.h>
 
 // This file contains functions relating to pipes.
 //
 // Operations on pipes (to the transport) are generally blocking operations,
 // performed in the context of the protocol.
 
-static nni_idhash *nni_pipes;
-static nni_mtx     nni_pipe_lk;
+static nni_id_map pipes;
+static nni_mtx    pipes_lk;
 
 int
 nni_pipe_sys_init(void)
 {
-	int rv;
-
-	nni_mtx_init(&nni_pipe_lk);
-
-	if ((rv = nni_idhash_init(&nni_pipes)) != 0) {
-		return (rv);
-	}
+	nni_mtx_init(&pipes_lk);
 
-	// Note that pipes have their own namespace.  ID hash will
-	// guarantee the that the first value is reasonable (non-zero),
-	// if we supply an out of range value (0).  (Consequently the
-	// value "1" has a bias -- its roughly twice as likely to be
-	// chosen as any other value.  This does not mater.)
-	nni_idhash_set_limits(
-	    nni_pipes, 1, 0x7fffffff, nni_random() & 0x7fffffffu);
+	// Pipe IDs needs to have high order bit clear, and we want
+	// them to start at a random value.
+	nni_id_map_init(&pipes, 1, 0x7fffffff, true);
 
 	return (0);
 }
@@ -49,11 +38,8 @@ void
 nni_pipe_sys_fini(void)
 {
 	nni_reap_drain();
-	nni_mtx_fini(&nni_pipe_lk);
-	if (nni_pipes != NULL) {
-		nni_idhash_fini(nni_pipes);
-		nni_pipes = NULL;
-	}
+	nni_mtx_fini(&pipes_lk);
+	nni_id_map_fini(&pipes);
 }
 
 static void
@@ -67,15 +53,15 @@ pipe_destroy(nni_pipe *p)
 
 	// Make sure any unlocked holders are done with this.
 	// This happens during initialization for example.
-	nni_mtx_lock(&nni_pipe_lk);
+	nni_mtx_lock(&pipes_lk);
 	if (p->p_id != 0) {
-		nni_idhash_remove(nni_pipes, p->p_id);
+		nni_id_remove(&pipes, p->p_id);
 	}
 	// This wait guarantees that all callers are done with us.
 	while (p->p_refcnt != 0) {
 		nni_cv_wait(&p->p_cv);
 	}
-	nni_mtx_unlock(&nni_pipe_lk);
+	nni_mtx_unlock(&pipes_lk);
 
 	if (p->p_proto_data != NULL) {
 		p->p_proto_ops.pipe_stop(p->p_proto_data);
@@ -101,31 +87,30 @@ pipe_destroy(nni_pipe *p)
 int
 nni_pipe_find(nni_pipe **pp, uint32_t id)
 {
-	int       rv;
 	nni_pipe *p;
-	nni_mtx_lock(&nni_pipe_lk);
 
 	// We don't care if the pipe is "closed".  End users only have
 	// access to the pipe in order to obtain properties (which may
 	// be retried during the post-close notification callback) or to
 	// close the pipe.
-	if ((rv = nni_idhash_find(nni_pipes, id, (void **) &p)) == 0) {
+        nni_mtx_lock(&pipes_lk);
+	if ((p = nni_id_get(&pipes, id)) != NULL) {
 		p->p_refcnt++;
 		*pp = p;
 	}
-	nni_mtx_unlock(&nni_pipe_lk);
-	return (rv);
+	nni_mtx_unlock(&pipes_lk);
+	return (p == NULL ? NNG_ENOENT : 0);
 }
 
 void
 nni_pipe_rele(nni_pipe *p)
 {
-	nni_mtx_lock(&nni_pipe_lk);
+	nni_mtx_lock(&pipes_lk);
 	p->p_refcnt--;
 	if (p->p_refcnt == 0) {
 		nni_cv_wake(&p->p_cv);
 	}
-	nni_mtx_unlock(&nni_pipe_lk);
+	nni_mtx_unlock(&pipes_lk);
 }
 
 // nni_pipe_id returns the 32-bit pipe id, which can be used in backtraces.
@@ -188,9 +173,9 @@ pipe_create(nni_pipe **pp, nni_sock *sock, nni_tran *tran, void *tdata)
 	void *              sdata = nni_sock_proto_data(sock);
 	nni_proto_pipe_ops *pops  = nni_sock_proto_pipe_ops(sock);
 	nni_pipe_stats *    st;
-	size_t sz;
+	size_t              sz;
 
-	sz = NNI_ALIGN_UP(sizeof (*p)) + pops->pipe_size;
+	sz = NNI_ALIGN_UP(sizeof(*p)) + pops->pipe_size;
 
 	if ((p = nni_zalloc(sz)) == NULL) {
 		// In this case we just toss the pipe...
@@ -198,7 +183,7 @@ pipe_create(nni_pipe **pp, nni_sock *sock, nni_tran *tran, void *tdata)
 		return (NNG_ENOMEM);
 	}
 
-	p->p_size = sz;
+	p->p_size       = sz;
 	p->p_proto_data = p + 1;
 	p->p_tran_ops   = *tran->tran_pipe;
 	p->p_tran_data  = tdata;
@@ -214,13 +199,13 @@ pipe_create(nni_pipe **pp, nni_sock *sock, nni_tran *tran, void *tdata)
 	NNI_LIST_NODE_INIT(&p->p_ep_node);
 
 	nni_mtx_init(&p->p_mtx);
-	nni_cv_init(&p->p_cv, &nni_pipe_lk);
+	nni_cv_init(&p->p_cv, &pipes_lk);
 
-	nni_mtx_lock(&nni_pipe_lk);
-	if ((rv = nni_idhash_alloc32(nni_pipes, &p->p_id, p)) == 0) {
+	nni_mtx_lock(&pipes_lk);
+	if ((rv = nni_id_alloc(&pipes, &p->p_id, p)) == 0) {
 		p->p_refcnt = 1;
 	}
-	nni_mtx_unlock(&nni_pipe_lk);
+	nni_mtx_unlock(&pipes_lk);
 
 	snprintf(st->s_scope, sizeof(st->s_scope), "pipe%u", p->p_id);
 
diff --git a/src/core/socket.c b/src/core/socket.c
index 6e3f14d0..66130d4a 100644
--- a/src/core/socket.c
+++ b/src/core/socket.c
@@ -16,10 +16,11 @@
 
 // Socket implementation.
 
-static nni_list    sock_list;
-static nni_idhash *sock_hash;
-static nni_mtx     sock_lk;
-static nni_idhash *ctx_hash;
+static nni_list   sock_list;
+static nni_id_map sock_ids;
+static nni_mtx    sock_lk;
+static nni_id_map ctx_ids;
+static bool       inited;
 
 struct nni_ctx {
 	nni_list_node     c_node;
@@ -117,7 +118,7 @@ static void listener_shutdown_locked(nni_listener *);
 #define SOCK(s) ((nni_sock *) (s))
 
 static int
-sock_get_fd(void *s, int flag, int *fdp)
+sock_get_fd(void *s, unsigned flag, int *fdp)
 {
 	int           rv;
 	nni_pollable *p;
@@ -374,19 +375,17 @@ nni_sock_find(nni_sock **sockp, uint32_t id)
 		return (rv);
 	}
 	nni_mtx_lock(&sock_lk);
-	if ((rv = nni_idhash_find(sock_hash, id, (void **) &s)) == 0) {
+	if ((s = nni_id_get(&sock_ids, id)) != NULL) {
 		if (s->s_closed) {
 			rv = NNG_ECLOSED;
 		} else {
 			s->s_refcnt++;
 			*sockp = s;
 		}
-	}
-	nni_mtx_unlock(&sock_lk);
-
-	if (rv == NNG_ENOENT) {
+	} else {
 		rv = NNG_ECLOSED;
 	}
+	nni_mtx_unlock(&sock_lk);
 
 	return (rv);
 }
@@ -582,33 +581,22 @@ nni_sock_create(nni_sock **sp, const nni_proto *proto)
 int
 nni_sock_sys_init(void)
 {
-	int rv;
-
 	NNI_LIST_INIT(&sock_list, nni_sock, s_node);
 	nni_mtx_init(&sock_lk);
 
-	if (((rv = nni_idhash_init(&sock_hash)) != 0) ||
-	    ((rv = nni_idhash_init(&ctx_hash)) != 0)) {
-		nni_sock_sys_fini();
-		return (rv);
-	}
-	nni_idhash_set_limits(sock_hash, 1, 0x7fffffff, 1);
-	nni_idhash_set_limits(ctx_hash, 1, 0x7fffffff, 1);
+	nni_id_map_init(&sock_ids, 1, 0x7fffffff, false);
+	nni_id_map_init(&ctx_ids, 1, 0x7fffffff, false);
+	inited = true;
 	return (0);
 }
 
 void
 nni_sock_sys_fini(void)
 {
-	if (sock_hash != NULL) {
-		nni_idhash_fini(sock_hash);
-		sock_hash = NULL;
-	}
-	if (ctx_hash != NULL) {
-		nni_idhash_fini(ctx_hash);
-		ctx_hash = NULL;
-	}
+	nni_id_map_fini(&sock_ids);
+	nni_id_map_fini(&ctx_ids);
 	nni_mtx_fini(&sock_lk);
+	inited = false;
 }
 
 int
@@ -628,7 +616,7 @@ nni_sock_open(nni_sock **sockp, const nni_proto *proto)
 	}
 
 	nni_mtx_lock(&sock_lk);
-	if (nni_idhash_alloc32(sock_hash, &s->s_id, s) != 0) {
+	if (nni_id_alloc(&sock_ids, &s->s_id, s) != 0) {
 		sock_destroy(s);
 	} else {
 		nni_list_append(&sock_list, s);
@@ -691,7 +679,7 @@ nni_sock_shutdown(nni_sock *sock)
 		ctx->c_closed = true;
 		if (ctx->c_refcnt == 0) {
 			// No open operations.  So close it.
-			nni_idhash_remove(ctx_hash, ctx->c_id);
+			nni_id_remove(&ctx_ids, ctx->c_id);
 			nni_list_remove(&sock->s_ctxs, ctx);
 			nni_ctx_destroy(ctx);
 		}
@@ -790,7 +778,7 @@ nni_sock_close(nni_sock *s)
 		return;
 	}
 	s->s_closed = true;
-	nni_idhash_remove(sock_hash, s->s_id);
+	nni_id_remove(&sock_ids, s->s_id);
 
 	// We might have been removed from the list already, e.g. by
 	// nni_sock_closeall.  This is idempotent.
@@ -820,7 +808,7 @@ nni_sock_closeall(void)
 {
 	nni_sock *s;
 
-	if (sock_hash == NULL) {
+	if (!inited) {
 		return;
 	}
 	for (;;) {
@@ -1152,7 +1140,7 @@ nni_sock_set_pipe_cb(nni_sock *s, int ev, nng_pipe_cb cb, void *arg)
 }
 
 int
-nni_ctx_find(nni_ctx **ctxp, uint32_t id, bool closing)
+nni_ctx_find(nni_ctx **cp, uint32_t id, bool closing)
 {
 	int      rv;
 	nni_ctx *ctx;
@@ -1161,7 +1149,7 @@ nni_ctx_find(nni_ctx **ctxp, uint32_t id, bool closing)
 		return (rv);
 	}
 	nni_mtx_lock(&sock_lk);
-	if ((rv = nni_idhash_find(ctx_hash, id, (void **) &ctx)) == 0) {
+	if ((ctx = nni_id_get(&ctx_ids, id)) != NULL) {
 		// We refuse a reference if either the socket is
 		// closed, or the context is closed.  (If the socket
 		// is closed, and we are only getting the reference so
@@ -1172,14 +1160,12 @@ nni_ctx_find(nni_ctx **ctxp, uint32_t id, bool closing)
 			rv = NNG_ECLOSED;
 		} else {
 			ctx->c_refcnt++;
-			*ctxp = ctx;
+			*cp = ctx;
 		}
-	}
-	nni_mtx_unlock(&sock_lk);
-
-	if (rv == NNG_ENOENT) {
+	} else {
 		rv = NNG_ECLOSED;
 	}
+	nni_mtx_unlock(&sock_lk);
 
 	return (rv);
 }
@@ -1211,7 +1197,7 @@ nni_ctx_rele(nni_ctx *ctx)
 	// Remove us from the hash, so we can't be found any more.
 	// This allows our ID to be reused later, although the system
 	// tries to avoid ID reuse.
-	nni_idhash_remove(ctx_hash, ctx->c_id);
+	nni_id_remove(&ctx_ids, ctx->c_id);
 	nni_list_remove(&sock->s_ctxs, ctx);
 	if (sock->s_closed || sock->s_ctxwait) {
 		nni_cv_wake(&sock->s_close_cv);
@@ -1236,8 +1222,8 @@ nni_ctx_open(nni_ctx **ctxp, nni_sock *sock)
 	if ((ctx = nni_zalloc(sz)) == NULL) {
 		return (NNG_ENOMEM);
 	}
-	ctx->c_size = sz;
-	ctx->c_data = ctx + 1;
+	ctx->c_size     = sz;
+	ctx->c_data     = ctx + 1;
 	ctx->c_closed   = false;
 	ctx->c_refcnt   = 1; // Caller implicitly gets a reference.
 	ctx->c_sock     = sock;
@@ -1251,14 +1237,14 @@ nni_ctx_open(nni_ctx **ctxp, nni_sock *sock)
 		nni_free(ctx, ctx->c_size);
 		return (NNG_ECLOSED);
 	}
-	if ((rv = nni_idhash_alloc32(ctx_hash, &ctx->c_id, ctx)) != 0) {
+	if ((rv = nni_id_alloc(&ctx_ids, &ctx->c_id, ctx)) != 0) {
 		nni_mtx_unlock(&sock_lk);
 		nni_free(ctx, ctx->c_size);
 		return (rv);
 	}
 
 	if ((rv = sock->s_ctx_ops.ctx_init(ctx->c_data, sock->s_data)) != 0) {
-		nni_idhash_remove(ctx_hash, ctx->c_id);
+		nni_id_remove(&ctx_ids, ctx->c_id);
 		nni_mtx_unlock(&sock_lk);
 		nni_free(ctx, ctx->c_size);
 		return (rv);
@@ -1396,7 +1382,7 @@ dialer_timer_start_locked(nni_dialer *d)
 	// This algorithm may lead to slight biases because we don't
 	// have a statistically perfect distribution with the modulo of
 	// the random number, but this really doesn't matter.
-	nni_sleep_aio(back_off ? nni_random() % back_off : 0, d->d_tmo_aio);
+	nni_sleep_aio(back_off ? nni_random() % back_off : 0, &d->d_tmo_aio);
 }
 
 void
@@ -1460,8 +1446,8 @@ dialer_shutdown_impl(nni_dialer *d)
 	nni_pipe *p;
 
 	// Abort any remaining in-flight operations.
-	nni_aio_close(d->d_con_aio);
-	nni_aio_close(d->d_tmo_aio);
+	nni_aio_close(&d->d_con_aio);
+	nni_aio_close(&d->d_tmo_aio);
 
 	// Stop the underlying transport.
 	d->d_ops.d_close(d->d_data);
@@ -1494,8 +1480,8 @@ nni_dialer_reap(nni_dialer *d)
 {
 	nni_sock *s = d->d_sock;
 
-	nni_aio_stop(d->d_tmo_aio);
-	nni_aio_stop(d->d_con_aio);
+	nni_aio_stop(&d->d_tmo_aio);
+	nni_aio_stop(&d->d_con_aio);
 
 	nni_stat_unregister(&d->d_stats.s_root);
 
@@ -1571,8 +1557,8 @@ listener_shutdown_impl(nni_listener *l)
 	nni_pipe *p;
 
 	// Abort any remaining in-flight accepts.
-	nni_aio_close(l->l_acc_aio);
-	nni_aio_close(l->l_tmo_aio);
+	nni_aio_close(&l->l_acc_aio);
+	nni_aio_close(&l->l_tmo_aio);
 
 	// Stop the underlying transport.
 	l->l_ops.l_close(l->l_data);
@@ -1606,8 +1592,8 @@ nni_listener_reap(nni_listener *l)
 {
 	nni_sock *s = l->l_sock;
 
-	nni_aio_stop(l->l_tmo_aio);
-	nni_aio_stop(l->l_acc_aio);
+	nni_aio_stop(&l->l_tmo_aio);
+	nni_aio_stop(&l->l_acc_aio);
 
 	nni_stat_unregister(&l->l_stats.s_root);
 
diff --git a/src/core/sockimpl.h b/src/core/sockimpl.h
index 16596c63..732e2285 100644
--- a/src/core/sockimpl.h
+++ b/src/core/sockimpl.h
@@ -50,8 +50,8 @@ struct nni_dialer {
 	nni_mtx             d_mtx;
 	nni_list            d_pipes;
 	nni_aio *           d_user_aio;
-	nni_aio *           d_con_aio;
-	nni_aio *           d_tmo_aio;  // backoff timer
+	nni_aio             d_con_aio;
+	nni_aio             d_tmo_aio;  // backoff timer
 	nni_duration        d_maxrtime; // maximum time for reconnect
 	nni_duration        d_currtime; // current time for reconnect
 	nni_duration        d_inirtime; // initial time for reconnect
@@ -91,8 +91,8 @@ struct nni_listener {
 	bool                  l_closing; // close started (shutdown)
 	nni_atomic_flag       l_started;
 	nni_list              l_pipes;
-	nni_aio *             l_acc_aio;
-	nni_aio *             l_tmo_aio;
+	nni_aio               l_acc_aio;
+	nni_aio               l_tmo_aio;
 	nni_reap_item         l_reap;
 	nni_listener_stats    l_stats;
 };