Introduce 'porting layer' Public API.

This introduces portable primitives for time, random numbers,
synchronization primitives, and threading.  These are somewhat
primitive (least common denominiators), but they can help with writing
portable applications, especially our own demo apps.

19 files changed, 469 insertions, 245 deletions

diff --git a/perf/perf.c b/perf/perf.c
index 84a1c66f..593f635e 100644
--- a/perf/perf.c
+++ b/perf/perf.c
@@ -1,6 +1,6 @@
 //
-// Copyright 2017 Garrett D'Amore <garrett@damore.org>
-// Copyright 2017 Capitar IT Group BV <info@capitar.com>
+// Copyright 2018 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2018 Capitar IT Group BV <info@capitar.com>
 //
 // This software is supplied under the terms of the MIT License, a
 // copy of which should be located in the distribution where this
@@ -16,11 +16,7 @@
 #include <stdlib.h>
 #include <string.h>
 
-// We steal access to the clock and thread functions so that we can
-// work on Windows too.  These functions are *not* part of nng's public
-// API, so don't be lazy like this!  All nni_ symbols are subject to
-// change without notice, and not part of the stable API or ABI.
-#include "core/nng_impl.h"
+#include "supplemental/util/platform.h"
 
 #if defined(NNG_ENABLE_PAIR1)
 #include "protocol/pair1/pair.h"
@@ -224,11 +220,10 @@ do_inproc(void *args)
 void
 do_inproc_lat(int argc, char **argv)
 {
-	nni_thr            thr;
+	nng_thread         *thr;
 	struct inproc_args ia;
 	int                rv;
 
-	nni_init();
 	if (argc != 2) {
 		die("Usage: inproc_lat <msg-size> <count>");
 	}
@@ -238,26 +233,24 @@ do_inproc_lat(int argc, char **argv)
 	ia.count   = parse_int(argv[1], "count");
 	ia.func    = latency_server;
 
-	if ((rv = nni_thr_init(&thr, do_inproc, &ia)) != 0) {
+	if ((rv = nng_thread_create(&thr, do_inproc, &ia)) != 0) {
 		die("Cannot create thread: %s", nng_strerror(rv));
 	}
-	nni_thr_run(&thr);
 
 	// Sleep a bit.
 	nng_msleep(100);
 
 	latency_client("inproc://latency_test", ia.msgsize, ia.count);
-	nni_thr_fini(&thr);
+	nng_thread_destroy(thr);
 }
 
 void
 do_inproc_thr(int argc, char **argv)
 {
-	nni_thr            thr;
+	nng_thread         *thr;
 	struct inproc_args ia;
 	int                rv;
 
-	nni_init();
 	if (argc != 2) {
 		die("Usage: inproc_thr <msg-size> <count>");
 	}
@@ -267,12 +260,11 @@ do_inproc_thr(int argc, char **argv)
 	ia.count   = parse_int(argv[1], "count");
 	ia.func    = throughput_client;
 
-	if ((rv = nni_thr_init(&thr, do_inproc, &ia)) != 0) {
+	if ((rv = nng_thread_create(&thr, do_inproc, &ia)) != 0) {
 		die("Cannot create thread: %s", nng_strerror(rv));
 	}
-	nni_thr_run(&thr);
 	throughput_server("inproc://tput_test", ia.msgsize, ia.count);
-	nni_thr_fini(&thr);
+	nng_thread_destroy(thr);
 }
 
 void
@@ -280,7 +272,7 @@ latency_client(const char *addr, size_t msgsize, int trips)
 {
 	nng_socket s;
 	nng_msg *  msg;
-	nni_time   start, end;
+	nng_time   start, end;
 	int        rv;
 	int        i;
 	float      total;
@@ -301,7 +293,7 @@ latency_client(const char *addr, size_t msgsize, int trips)
 		die("nng_msg_alloc: %s", nng_strerror(rv));
 	}
 
-	start = nni_clock();
+	start = nng_clock();
 	for (i = 0; i < trips; i++) {
 		if ((rv = nng_sendmsg(s, msg, 0)) != 0) {
 			die("nng_sendmsg: %s", nng_strerror(rv));
@@ -311,9 +303,9 @@ latency_client(const char *addr, size_t msgsize, int trips)
 			die("nng_recvmsg: %s", nng_strerror(rv));
 		}
 	}
-	end = nni_clock();
+	end = nng_clock();
 
-	nni_msg_free(msg);
+	nng_msg_free(msg);
 	nng_close(s);
 
 	total   = (float) ((end - start)) / 1000;
@@ -395,8 +387,8 @@ throughput_server(const char *addr, size_t msgsize, int count)
 	if ((rv = nng_recvmsg(s, &msg, 0)) != 0) {
 		die("nng_recvmsg: %s", nng_strerror(rv));
 	}
-	nni_msg_free(msg);
-	start = nni_clock();
+	nng_msg_free(msg);
+	start = nng_clock();
 
 	for (i = 0; i < count; i++) {
 		if ((rv = nng_recvmsg(s, &msg, 0)) != 0) {
@@ -406,9 +398,9 @@ throughput_server(const char *addr, size_t msgsize, int count)
 			die("wrong message size: %d != %d", nng_msg_len(msg),
 			    msgsize);
 		}
-		nni_msg_free(msg);
+		nng_msg_free(msg);
 	}
-	end = nni_clock();
+	end = nng_clock();
 	nng_close(s);
 	total     = (float) ((end - start)) / 1000;
 	msgpersec = (float) (count) / total;
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 48262751..b245d6ae 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -1,8 +1,9 @@
 #
-#   Copyright (c) 2012-2013 Martin Sustrik  All rights reserved.
-#   Copyright (c) 2013 GoPivotal, Inc.  All rights reserved.
-#   Copyright (c) 2015-2016 Jack R. Dunaway. All rights reserved.
-#   Copyright 2017 Garrett D'Amore <garrett@damore.org>
+# Copyright 2018 Capitar IT Group BV <info@capitar.com>
+# Copyright 2018 Staysail Systems, Inc. <info@staysail.tech>
+# Copyright (c) 2012-2013 Martin Sustrik  All rights reserved.
+# Copyright (c) 2013 GoPivotal, Inc.  All rights reserved.
+# Copyright (c) 2015-2016 Jack R. Dunaway. All rights reserved.
 #
 #   Permission is hereby granted, free of charge, to any person obtaining a copy
 #   of this software and associated documentation files (the "Software"),
@@ -129,6 +130,7 @@ add_subdirectory(supplemental/base64)
 add_subdirectory(supplemental/http)
 add_subdirectory(supplemental/sha1)
 add_subdirectory(supplemental/tls)
+add_subdirectory(supplemental/util)
 add_subdirectory(supplemental/websocket)
 
 add_subdirectory(protocol/bus0)
@@ -152,6 +154,7 @@ include_directories(AFTER SYSTEM ${PROJECT_SOURCE_DIR}/src
 add_definitions(${NNG_DEFINES})
 
 # Provide same folder structure in IDE as on disk
+# XXX: Consider replacing this with source_group(TREE...)
 foreach (f ${NNG_SOURCES})
     # Get the path of the file relative to source directory
     if (IS_ABSOLUTE "${f}")
diff --git a/src/core/platform.h b/src/core/platform.h
index 73b6785b..cf635f4a 100644
--- a/src/core/platform.h
+++ b/src/core/platform.h
@@ -145,12 +145,12 @@ extern void nni_plat_thr_fini(nni_plat_thr *);
 // Clock Support
 //
 
-// nn_plat_clock returns a number of microseconds since some arbitrary time
+// nn_plat_clock returns a number of milliseconds since some arbitrary time
 // in the past.  The values returned by nni_clock must use the same base
 // as the times used in nni_plat_cond_waituntil.  The nni_plat_clock() must
-// returnvalues > 0, and must return values smaller than 2^63.  (We could relax
-// this last constraint, but there is no reason to, and leaves us the option
-// of using negative values for other purposes in the future.)
+// return values > 0, and must return values smaller than 2^63.  (We could
+// relax this last constraint, but there is no reason to, and leaves us the
+// option of using negative values for other purposes in the future.)
 extern nni_time nni_plat_clock(void);
 
 // nni_plat_sleep sleeps for the specified number of milliseconds (at least).
diff --git a/src/nng.c b/src/nng.c
index f552e26b..ce296782 100644
--- a/src/nng.c
+++ b/src/nng.c
@@ -1194,6 +1194,7 @@ nng_stat_value(nng_stat *stat)
 }
 #endif
 
+#if 0
 // These routines exist as utility functions, exposing some of our
 // "guts" to the external world for the purposes of test code and
 // bundled utilities. They should not be considered part of our public
@@ -1237,6 +1238,7 @@ nng_thread_destroy(void *arg)
 
 	NNI_FREE_STRUCT(thr);
 }
+#endif
 
 int
 nng_url_parse(nng_url **result, const char *ustr)
diff --git a/src/nng.h b/src/nng.h
index f4922c54..1d590b70 100644
--- a/src/nng.h
+++ b/src/nng.h
@@ -536,22 +536,6 @@ NNG_DECL int64_t nng_stat_value(nng_stat *);
 // which means that messages from one side are forwarded to the other.
 NNG_DECL int nng_device(nng_socket, nng_socket);
 
-// The following functions are not intrinsic to nanomsg, and so do not
-// represent our public API.  Avoid their use in other applications.
-
-#ifdef NNG_PRIVATE
-
-// Sleep for specified msecs.
-NNG_DECL void nng_msleep(nng_duration);
-
-// Create and start a thread.
-NNG_DECL int nng_thread_create(void **, void (*)(void *), void *);
-
-// Destroy a thread (waiting for it to complete.)
-NNG_DECL void nng_thread_destroy(void *);
-
-#endif // NNG_PRIVATE
-
 // Symbol name and visibility.  TBD.  The only symbols that really should
 // be directly exported to runtimes IMO are the option symbols.  And frankly
 // they have enough special logic around them that it might be best not to
@@ -985,7 +969,7 @@ NNG_DECL int nng_http_res_set_data(nng_http_res *, const void *, size_t);
 // probably set the content-type header.
 NNG_DECL int nng_http_res_copy_data(nng_http_res *, const void *, size_t);
 
-// An nng_http_conn represents an underlyinjg "connection".  It may be
+// An nng_http_conn represents an underlying "connection".  It may be
 // a TCP channel, or a TLS channel, but the main thing is that this is
 // normally only used for exchanging HTTP requests and responses.
 typedef struct nng_http_conn nng_http_conn;
diff --git a/src/supplemental/util/CMakeLists.txt b/src/supplemental/util/CMakeLists.txt
new file mode 100644
index 00000000..07dba939
--- /dev/null
+++ b/src/supplemental/util/CMakeLists.txt
@@ -0,0 +1,15 @@
+#
+# Copyright 2018 Capitar IT Group BV <info@capitar.com>
+# Copyright 2018 Staysail Systems, Inc. <info@staysail.tech>
+#
+# This software is supplied under the terms of the MIT License, a
+# copy of which should be located in the distribution where this
+# file was obtained (LICENSE.txt).  A copy of the license may also be
+# found online at https://opensource.org/licenses/MIT.
+#
+
+set(SUPP_PLATFORM_SOURCES
+	supplemental/util/platform.c)
+
+install(FILES platform.h DESTINATION include/nng/supplemental/util)
+set(NNG_SOURCES ${NNG_SOURCES} ${SUPP_PLATFORM_SOURCES} PARENT_SCOPE)
diff --git a/src/supplemental/util/platform.c b/src/supplemental/util/platform.c
new file mode 100644
index 00000000..ce52491a
--- /dev/null
+++ b/src/supplemental/util/platform.c
@@ -0,0 +1,166 @@
+//
+// Copyright 2018 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2018 Capitar IT Group BV <info@capitar.com>
+//
+// This software is supplied under the terms of the MIT License, a
+// copy of which should be located in the distribution where this
+// file was obtained (LICENSE.txt).  A copy of the license may also be
+// found online at https://opensource.org/licenses/MIT.
+//
+
+#include <stdlib.h>
+#include <string.h>
+
+#include "core/nng_impl.h"
+#include "supplemental/util/platform.h"
+
+nng_time
+nng_clock(void)
+{
+	(void) nni_init();
+	return (nni_plat_clock());
+}
+
+// Sleep for specified msecs.
+void
+nng_msleep(nng_duration dur)
+{
+	(void) nni_init();
+	nni_msleep(dur);
+}
+
+// nng_thread is a handle to a "thread", which may be a real system
+// thread, or a coroutine on some platforms.
+typedef struct nng_thread nng_thread;
+
+// Create and start a thread.  Note that on some platforms, this might
+// actually be a coroutine, with limitations about what system APIs
+// you can call.  Therefore, these threads should only be used with the
+// I/O APIs provided by nng.  The thread runs until completion.
+int
+nng_thread_create(nng_thread **thrp, void (*func)(void *), void *arg)
+{
+	nni_thr *thr;
+	int      rv;
+
+	(void) nni_init();
+
+	if ((thr = NNI_ALLOC_STRUCT(thr)) == NULL) {
+		return (NNG_ENOMEM);
+	}
+	memset(thr, 0, sizeof(*thr));
+	*thrp = (void *) thr;
+	if ((rv = nni_thr_init(thr, func, arg)) != 0) {
+		return (rv);
+	}
+	nni_thr_run(thr);
+	return (0);
+}
+
+// Destroy a thread (waiting for it to complete.)  When this function
+// returns all resources for the thread are cleaned up.
+void
+nng_thread_destroy(nng_thread *thrp)
+{
+	nni_thr *t = (void *) thrp;
+	nni_thr_fini(t);
+	NNI_FREE_STRUCT(t);
+}
+
+struct nng_mtx {
+	nni_mtx m;
+};
+
+int
+nng_mtx_alloc(nng_mtx **mpp)
+{
+	nng_mtx *mp;
+
+	(void) nni_init();
+
+	if ((mp = NNI_ALLOC_STRUCT(mp)) == NULL) {
+		return (NNG_ENOMEM);
+	}
+	nni_mtx_init(&mp->m);
+	*mpp = mp;
+	return (0);
+}
+
+void
+nng_mtx_free(nng_mtx *mp)
+{
+	if (mp != NULL) {
+		nni_mtx_fini(&mp->m);
+		NNI_FREE_STRUCT(mp);
+	}
+}
+
+void
+nng_mtx_lock(nng_mtx *mp)
+{
+	nni_mtx_lock(&mp->m);
+}
+
+void
+nng_mtx_unlock(nng_mtx *mp)
+{
+	nni_mtx_unlock(&mp->m);
+}
+
+struct nng_cv {
+	nni_cv c;
+};
+
+typedef struct nng_cv nng_cv;
+
+int
+nng_cv_alloc(nng_cv **cvp, nng_mtx *mx)
+{
+	nng_cv *cv;
+
+	if ((cv = NNI_ALLOC_STRUCT(cv)) == NULL) {
+		return (NNG_ENOMEM);
+	}
+	nni_cv_init(&cv->c, &mx->m);
+	*cvp = cv;
+	return (0);
+}
+
+void
+nng_cv_free(nng_cv *cv)
+{
+	if (cv != NULL) {
+		nni_cv_fini(&cv->c);
+		NNI_FREE_STRUCT(cv);
+	}
+}
+
+void
+nng_cv_wait(nng_cv *cv)
+{
+	nni_cv_wait(&cv->c);
+}
+
+int
+nng_cv_until(nng_cv *cv, nng_time when)
+{
+	return (nni_cv_until(&cv->c, (nni_time) when));
+}
+
+void
+nng_cv_wake(nng_cv *cv)
+{
+	nni_cv_wake(&cv->c);
+}
+
+void
+nng_cv_wake1(nng_cv *cv)
+{
+	nni_cv_wake1(&cv->c);
+}
+
+uint32_t
+nng_random(void)
+{
+	return (nni_random());
+}
diff --git a/src/supplemental/util/platform.h b/src/supplemental/util/platform.h
new file mode 100644
index 00000000..1383fe15
--- /dev/null
+++ b/src/supplemental/util/platform.h
@@ -0,0 +1,106 @@
+//
+// Copyright 2018 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2018 Capitar IT Group BV <info@capitar.com>
+//
+// This software is supplied under the terms of the MIT License, a
+// copy of which should be located in the distribution where this
+// file was obtained (LICENSE.txt).  A copy of the license may also be
+// found online at https://opensource.org/licenses/MIT.
+//
+
+#ifndef NNG_SUPPLEMENTAL_UTIL_PLATFORM_H
+#define NNG_SUPPLEMENTAL_UTIL_PLATFORM_H
+
+// The declarations in this file are provided to assist with application
+// portability.  Conceptually these APIs are based on work we have already
+// done for NNG internals, and we find that they are useful in building
+// portable applications.
+
+// If it is more natural to use native system APIs like pthreads or C11
+// APIs or Windows APIs, then by all means please feel free to simply
+// ignore this.
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stddef.h>
+#include <stdint.h>
+
+// nng_time represents an absolute time since some arbitrary point in the
+// past, measured in milliseconds.  The values are always positive.
+typedef uint64_t nng_time;
+
+// Return an absolute time from some arbitrary point.  The value is
+// provided in milliseconds, and is of limited resolution based on the
+// system clock.  (Do not use it for fine grained performance measurements.)
+NNG_DECL uint64_t nng_clock(void);
+
+// Sleep for specified msecs.
+NNG_DECL void nng_msleep(nng_duration);
+
+// nng_thread is a handle to a "thread", which may be a real system
+// thread, or a coroutine on some platforms.
+typedef struct nng_thread nng_thread;
+
+// Create and start a thread.  Note that on some platforms, this might
+// actually be a coroutine, with limitations about what system APIs
+// you can call.  Therefore, these threads should only be used with the
+// I/O APIs provided by nng.  The thread runs until completion.
+NNG_DECL int nng_thread_create(nng_thread **, void (*)(void *), void *);
+
+// Destroy a thread (waiting for it to complete.)  When this function
+// returns all resources for the thread are cleaned up.
+NNG_DECL void nng_thread_destroy(nng_thread *);
+
+// nng_mtx represents a mutex, which is a simple, non-retrant, boolean lock.
+typedef struct nng_mtx nng_mtx;
+
+// nng_mtx_alloc allocates a mutex structure.
+NNG_DECL int nng_mtx_alloc(nng_mtx **);
+
+// nng_mtx_free frees the mutex.  It most not be locked.
+NNG_DECL void nng_mtx_free(nng_mtx *);
+
+// nng_mtx_lock locks the mutex; if it is already locked it will block
+// until it can be locked.  If the caller already holds the lock, the
+// results are undefined (a panic may occur).
+NNG_DECL void nng_mtx_lock(nng_mtx *);
+
+// nng_mtx_unlock unlocks a previously locked mutex.  It is an error to
+// call this on a mutex which is not owned by caller.
+NNG_DECL void nng_mtx_unlock(nng_mtx *);
+
+// nng_cv is a condition variable.  It is always allocated with an
+// associated mutex, which must be held when waiting for it, or
+// when signaling it.
+typedef struct nng_cv nng_cv;
+
+NNG_DECL int nng_cv_alloc(nng_cv **, nng_mtx *);
+
+// nng_cv_free frees the condition variable.
+NNG_DECL void nng_cv_free(nng_cv *);
+
+// nng_cv_wait waits until the condition variable is "signaled".
+NNG_DECL void nng_cv_wait(nng_cv *);
+
+// nng_cv_until waits until either the condition is signaled, or
+// the timeout expires.  It returns NNG_ETIMEDOUT in that case.
+NNG_DECL int nng_cv_until(nng_cv *, nng_time);
+
+// nng_cv_wake wakes all threads waiting on the condition.
+NNG_DECL void nng_cv_wake(nng_cv *);
+
+// nng_cv_wake1 wakes only one thread waiting on the condition.  This may
+// reduce the thundering herd problem, but care must be taken to ensure
+// that no waiter starves forvever.
+NNG_DECL void nng_cv_wake1(nng_cv *);
+
+// nng_random returns a "strong" (cryptographic sense) random number.
+NNG_DECL uint32_t nng_random(void);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // NNG_SUPPLEMENTAL_UTIL_PLATFORM_H
diff --git a/tests/device.c b/tests/device.c
index 56ad2cb5..535371e6 100644
--- a/tests/device.c
+++ b/tests/device.c
@@ -1,6 +1,6 @@
 //
-// Copyright 2017 Garrett D'Amore <garrett@damore.org>
-// Copyright 2017 Capitar IT Group BV <info@capitar.com>
+// Copyright 2018 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2018 Capitar IT Group BV <info@capitar.com>
 //
 // This software is supplied under the terms of the MIT License, a
 // copy of which should be located in the distribution where this
@@ -11,6 +11,7 @@
 #include "convey.h"
 #include "nng.h"
 #include "protocol/pair1/pair.h"
+#include "supplemental/util/platform.h"
 #include "stubs.h"
 
 #include <string.h>
@@ -48,7 +49,7 @@ Main({
 			nng_socket   end2;
 			nng_duration tmo;
 			nng_msg *    msg;
-			void *       thr;
+			nng_thread * thr;
 
 			So(nng_pair1_open(&dev1) == 0);
 			So(nng_pair1_open(&dev2) == 0);
diff --git a/tests/multistress.c b/tests/multistress.c
index 6a41a551..556718fa 100644
--- a/tests/multistress.c
+++ b/tests/multistress.c
@@ -1,6 +1,6 @@
 //
-// Copyright 2017 Garrett D'Amore <garrett@damore.org>
-// Copyright 2017 Capitar IT Group BV <info@capitar.com>
+// Copyright 2018 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2018 Capitar IT Group BV <info@capitar.com>
 //
 // This software is supplied under the terms of the MIT License, a
 // copy of which should be located in the distribution where this
@@ -22,6 +22,7 @@
 #include "protocol/reqrep0/req.h"
 #include "protocol/survey0/respond.h"
 #include "protocol/survey0/survey.h"
+#include "supplemental/util/platform.h"
 #include "transport/inproc/inproc.h"
 #include "transport/ipc/ipc.h"
 #include "transport/tcp/tcp.h"
@@ -69,7 +70,7 @@ int    allocaddrs;
 typedef struct test_case {
 	nng_socket  socket;
 	const char *name;
-	void *      thr;
+	nng_thread *thr;
 	int         nrecv;
 	int         nsend;
 	int         nfail;
diff --git a/tests/pipeline.c b/tests/pipeline.c
index 96ae3d17..2b61d99a 100644
--- a/tests/pipeline.c
+++ b/tests/pipeline.c
@@ -1,6 +1,6 @@
 //
-// Copyright 2017 Garrett D'Amore <garrett@damore.org>
-// Copyright 2017 Capitar IT Group BV <info@capitar.com>
+// Copyright 2018 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2018 Capitar IT Group BV <info@capitar.com>
 //
 // This software is supplied under the terms of the MIT License, a
 // copy of which should be located in the distribution where this
@@ -13,6 +13,7 @@
 #include "protocol/pipeline0/pull.h"
 #include "protocol/pipeline0/push.h"
 #include "stubs.h"
+#include "supplemental/util/platform.h"
 
 #include <string.h>
 
diff --git a/tests/platform.c b/tests/platform.c
index 10278e7d..168f7879 100644
--- a/tests/platform.c
+++ b/tests/platform.c
@@ -1,6 +1,6 @@
 //
-// Copyright 2017 Garrett D'Amore <garrett@damore.org>
-// Copyright 2017 Capitar IT Group BV <info@capitar.com>
+// Copyright 2018 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2018 Capitar IT Group BV <info@capitar.com>
 //
 // This software is supplied under the terms of the MIT License, a
 // copy of which should be located in the distribution where this
@@ -9,8 +9,10 @@
 //
 
 #include "convey.h"
-#include "core/nng_impl.h"
+
 #include "nng.h"
+#include "supplemental/util/platform.h"
+
 #include "stubs.h"
 
 // Add is for testing threads.
@@ -24,8 +26,8 @@ add(void *arg)
 struct notifyarg {
 	int          did;
 	nng_duration when;
-	nni_mtx      mx;
-	nni_cv       cv;
+	nng_mtx *    mx;
+	nng_cv *     cv;
 };
 
 void
@@ -34,16 +36,14 @@ notifyafter(void *arg)
 	struct notifyarg *na = arg;
 
 	nng_msleep(na->when);
-	nni_mtx_lock(&na->mx);
+	nng_mtx_lock(na->mx);
 	na->did = 1;
-	nni_cv_wake(&na->cv);
-	nni_mtx_unlock(&na->mx);
+	nng_cv_wake(na->cv);
+	nng_mtx_unlock(na->mx);
 }
 
 TestMain("Platform Operations", {
 
-	nni_init();
-
 	// This is required for anything else to work
 	Convey("The clock works", {
 		uint64_t now = getms();
@@ -58,10 +58,10 @@ TestMain("Platform Operations", {
 			uint64_t msend;
 			int      usdelta;
 			int      msdelta;
-			nni_time usend;
-			nni_time usnow = nni_clock();
+			nng_time usend;
+			nng_time usnow = nng_clock();
 			nng_msleep(200);
-			usend = nni_clock();
+			usend = nng_clock();
 			msend = getms();
 
 			So(usend > usnow);
@@ -74,38 +74,37 @@ TestMain("Platform Operations", {
 		});
 	});
 	Convey("Mutexes work", {
-		static nni_mtx mx;
+		static nng_mtx *mx;
 
-		nni_mtx_init(&mx);
+		So(nng_mtx_alloc(&mx) == 0);
+		Reset({ nng_mtx_free(mx); });
 
 		Convey("We can lock a mutex", {
-			nni_mtx_lock(&mx);
+			nng_mtx_lock(mx);
 			So(1);
 			Convey("And we can unlock it", {
-				nni_mtx_unlock(&mx);
+				nng_mtx_unlock(mx);
 				So(1);
 				Convey("And then lock it again", {
-					nni_mtx_lock(&mx);
+					nng_mtx_lock(mx);
 					So(1);
-					nni_mtx_unlock(&mx);
+					nng_mtx_unlock(mx);
 					So(1);
 				});
 			});
 		});
-		Convey("We can finalize it", { nni_mtx_fini(&mx); });
 	});
 
 	Convey("Threads work", {
-		static nni_thr thr;
-		int            val = 0;
-		int            rv;
+		static nng_thread *thr;
+		int                val = 0;
+		int                rv;
 
 		Convey("We can create threads", {
-			rv = nni_thr_init(&thr, add, &val);
+			rv = nng_thread_create(&thr, add, &val);
 			So(rv == 0);
-			nni_thr_run(&thr);
 
-			Reset({ nni_thr_fini(&thr); });
+			Reset({ nng_thread_destroy(thr); });
 
 			Convey("It ran", {
 				nng_msleep(50); // for context switch
@@ -113,55 +112,4 @@ TestMain("Platform Operations", {
 			});
 		});
 	});
-	Convey("Condition variables work", {
-		static struct notifyarg arg;
-		static nni_thr          thr;
-
-		nni_mtx_init(&arg.mx);
-		nni_cv_init(&arg.cv, &arg.mx);
-		So(nni_thr_init(&thr, notifyafter, &arg) == 0);
-
-		Reset({
-			nni_cv_fini(&arg.cv);
-			nni_mtx_fini(&arg.mx);
-			nni_thr_fini(&thr);
-		});
-
-		Convey("Notification works", {
-			arg.did  = 0;
-			arg.when = 10;
-			nni_thr_run(&thr);
-
-			nni_mtx_lock(&arg.mx);
-			if (!arg.did) {
-				nni_cv_wait(&arg.cv);
-			}
-			nni_mtx_unlock(&arg.mx);
-			nni_thr_wait(&thr);
-			So(arg.did == 1);
-		});
-
-		Convey("Timeout works", {
-			arg.did  = 0;
-			arg.when = 200;
-			nni_thr_run(&thr);
-			nni_mtx_lock(&arg.mx);
-			if (!arg.did) {
-				nni_cv_until(&arg.cv, nni_clock() + 10);
-			}
-			So(arg.did == 0);
-			nni_mtx_unlock(&arg.mx);
-			nni_thr_wait(&thr);
-		});
-		Convey("Not running works", {
-			arg.did  = 0;
-			arg.when = 1;
-			nni_mtx_lock(&arg.mx);
-			if (!arg.did) {
-				nni_cv_until(&arg.cv, nni_clock() + 10);
-			}
-			So(arg.did == 0);
-			nni_mtx_unlock(&arg.mx);
-		});
-	});
 })
diff --git a/tests/pollfd.c b/tests/pollfd.c
index 8526c705..7a87fe9b 100644
--- a/tests/pollfd.c
+++ b/tests/pollfd.c
@@ -1,6 +1,6 @@
 //
-// Copyright 2017 Garrett D'Amore <garrett@damore.org>
-// Copyright 2017 Capitar IT Group BV <info@capitar.com>
+// Copyright 2018 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2018 Capitar IT Group BV <info@capitar.com>
 //
 // This software is supplied under the terms of the MIT License, a
 // copy of which should be located in the distribution where this
@@ -35,6 +35,7 @@
 #include "protocol/pipeline0/pull.h"
 #include "protocol/pipeline0/push.h"
 #include "stubs.h"
+#include "supplemental/util/platform.h"
 
 TestMain("Poll FDs", {
 
diff --git a/tests/pubsub.c b/tests/pubsub.c
index 796d951e..3fd95b42 100644
--- a/tests/pubsub.c
+++ b/tests/pubsub.c
@@ -1,6 +1,6 @@
 //
-// Copyright 2017 Garrett D'Amore <garrett@damore.org>
-// Copyright 2017 Capitar IT Group BV <info@capitar.com>
+// Copyright 2018 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2018 Capitar IT Group BV <info@capitar.com>
 //
 // This software is supplied under the terms of the MIT License, a
 // copy of which should be located in the distribution where this
@@ -13,6 +13,7 @@
 #include "protocol/pubsub0/pub.h"
 #include "protocol/pubsub0/sub.h"
 #include "stubs.h"
+#include "supplemental/util/platform.h"
 
 #include <string.h>
 
diff --git a/tests/reconnect.c b/tests/reconnect.c
index e191f70a..6e7ab79f 100644
--- a/tests/reconnect.c
+++ b/tests/reconnect.c
@@ -1,6 +1,6 @@
 //
-// Copyright 2017 Garrett D'Amore <garrett@damore.org>
-// Copyright 2017 Capitar IT Group BV <info@capitar.com>
+// Copyright 2018 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2018 Capitar IT Group BV <info@capitar.com>
 //
 // This software is supplied under the terms of the MIT License, a
 // copy of which should be located in the distribution where this
@@ -12,8 +12,9 @@
 #include "nng.h"
 #include "protocol/pipeline0/pull.h"
 #include "protocol/pipeline0/push.h"
-#include "stubs.h"
+#include "supplemental/util/platform.h"
 
+#include "stubs.h"
 #include <string.h>
 
 #define APPENDSTR(m, s) nng_msg_append(m, s, strlen(s))
diff --git a/tests/scalability.c b/tests/scalability.c
index 9695c53c..84395f9d 100644
--- a/tests/scalability.c
+++ b/tests/scalability.c
@@ -13,6 +13,7 @@
 
 #include "protocol/reqrep0/rep.h"
 #include "protocol/reqrep0/req.h"
+#include "supplemental/util/platform.h"
 #include "stubs.h"
 
 #include <string.h>
@@ -21,7 +22,7 @@ static int nclients = 200;
 
 static char *addr = "inproc:///atscale";
 nng_socket   rep;
-void *       server;
+nng_thread *server;
 
 void
 serve(void *arg)
diff --git a/tests/sock.c b/tests/sock.c
index 404e160b..1f7567f8 100644
--- a/tests/sock.c
+++ b/tests/sock.c
@@ -1,6 +1,6 @@
 //
-// Copyright 2017 Garrett D'Amore <garrett@damore.org>
-// Copyright 2017 Capitar IT Group BV <info@capitar.com>
+// Copyright 2018 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2018 Capitar IT Group BV <info@capitar.com>
 //
 // This software is supplied under the terms of the MIT License, a
 // copy of which should be located in the distribution where this
@@ -15,6 +15,9 @@
 #include "protocol/pubsub0/sub.h"
 
 #include "protocol/pair1/pair.h"
+
+#include "supplemental/util/platform.h"
+
 #include "stubs.h"
 
 #include <string.h>
diff --git a/tests/synch.c b/tests/synch.c
index c9de3deb..de0d7e16 100644
--- a/tests/synch.c
+++ b/tests/synch.c
@@ -9,15 +9,16 @@
 //
 
 #include "convey.h"
-#include "core/nng_impl.h"
+//#include "core/nng_impl.h"
 #include "nng.h"
+#include "supplemental/util/platform.h"
 
 // Notify tests for verifying condvars.
 struct notifyarg {
 	int          did;
 	nng_duration when;
-	nni_mtx      mx;
-	nni_cv       cv;
+	nng_mtx *    mx;
+	nng_cv *     cv;
 };
 
 #ifdef NNG_PLATFORM_POSIX
@@ -31,118 +32,116 @@ notifyafter(void *arg)
 {
 	struct notifyarg *na = arg;
 
-	nni_msleep(na->when);
-	nni_mtx_lock(&na->mx);
+	nng_msleep(na->when);
+	nng_mtx_lock(na->mx);
 	na->did = 1;
-	nni_cv_wake(&na->cv);
-	nni_mtx_unlock(&na->mx);
+	nng_cv_wake(na->cv);
+	nng_mtx_unlock(na->mx);
 }
 
 struct notifyarg arg;
-nni_thr          thr;
+nng_thread *     thr;
 
 static void
 test_sync(void)
 {
 	Convey("Mutexes work", {
-		nni_mtx mx;
+		nng_mtx *mx;
 
-		nni_mtx_init(&mx);
+		So(nng_mtx_alloc(&mx) == 0);
+		Reset({ nng_mtx_free(mx); });
 
 		Convey("We can lock a mutex", {
-			nni_mtx_lock(&mx);
+			nng_mtx_lock(mx);
 			So(1);
 			Convey("And we can unlock it", {
-				nni_mtx_unlock(&mx);
+				nng_mtx_unlock(mx);
 				So(1);
 				Convey("And then lock it again", {
-					nni_mtx_lock(&mx);
+					nng_mtx_lock(mx);
 					So(1);
-					nni_mtx_unlock(&mx);
+					nng_mtx_unlock(mx);
 					So(1);
 				});
 			});
 			Convey("Things block properly", {
 
-				nni_mtx_init(&arg.mx);
-				nni_cv_init(&arg.cv, &arg.mx);
-				So(nni_thr_init(&thr, notifyafter, &arg) == 0);
+				So(nng_mtx_alloc(&arg.mx) == 0);
+				So(nng_cv_alloc(&arg.cv, arg.mx) == 0);
 				arg.did  = 0;
 				arg.when = 0;
-				nni_mtx_lock(&arg.mx);
-				nni_thr_run(&thr);
+				nng_mtx_lock(arg.mx);
+				So(nng_thread_create(
+				       &thr, notifyafter, &arg) == 0);
 				nng_msleep(10);
 				So(arg.did == 0);
-				nni_mtx_unlock(&arg.mx);
+				nng_mtx_unlock(arg.mx);
 				nng_msleep(10);
-				nni_mtx_lock(&arg.mx);
+				nng_mtx_lock(arg.mx);
 				while (!arg.did) {
-					nni_cv_wait(&arg.cv);
+					nng_cv_wait(arg.cv);
 				}
 				So(arg.did != 0);
-				nni_mtx_unlock(&arg.mx);
-				nni_thr_fini(&thr);
-				nni_cv_fini(&arg.cv);
-				nni_mtx_fini(&arg.mx);
+				nng_mtx_unlock(arg.mx);
+				nng_thread_destroy(thr);
+				nng_cv_free(arg.cv);
+				nng_mtx_free(arg.mx);
 			})
 		});
-		Convey("We can finalize it", { nni_mtx_fini(&mx); });
 	});
 
 	Convey("Condition variables work", {
 
-		nni_mtx_init(&arg.mx);
-		nni_cv_init(&arg.cv, &arg.mx);
-		So(nni_thr_init(&thr, notifyafter, &arg) == 0);
+		So(nng_mtx_alloc(&arg.mx) == 0);
+		So(nng_cv_alloc(&arg.cv, arg.mx) == 0);
 
 		Reset({
-			nni_cv_fini(&arg.cv);
-			nni_mtx_fini(&arg.mx);
-			nni_thr_fini(&thr);
+			nng_cv_free(arg.cv);
+			nng_mtx_free(arg.mx);
 		});
 
 		Convey("Notification works", {
 			arg.did  = 0;
 			arg.when = 10;
-			nni_thr_run(&thr);
+			So(nng_thread_create(&thr, notifyafter, &arg) == 0);
 
-			nni_mtx_lock(&arg.mx);
+			nng_mtx_lock(arg.mx);
 			if (!arg.did) {
-				nni_cv_wait(&arg.cv);
+				nng_cv_wait(arg.cv);
 			}
-			nni_mtx_unlock(&arg.mx);
-			nni_thr_wait(&thr);
+			nng_mtx_unlock(arg.mx);
+			nng_thread_destroy(thr);
 			So(arg.did == 1);
 		});
 
 		Convey("Timeout works", {
 			arg.did  = 0;
 			arg.when = 200;
-			nni_thr_run(&thr);
-			nni_mtx_lock(&arg.mx);
+			So(nng_thread_create(&thr, notifyafter, &arg) == 0);
+			nng_mtx_lock(arg.mx);
 			if (!arg.did) {
-				nni_cv_until(&arg.cv, nni_clock() + 10);
+				nng_cv_until(arg.cv, nng_clock() + 10);
 			}
 			So(arg.did == 0);
-			nni_mtx_unlock(&arg.mx);
-			nni_thr_wait(&thr);
+			nng_mtx_unlock(arg.mx);
+			nng_thread_destroy(thr);
 		});
 
 		Convey("Empty timeout is EAGAIN", {
-			nni_mtx_lock(&arg.mx);
-			So(nni_cv_until(&arg.cv, 0) == NNG_EAGAIN);
-			nni_mtx_unlock(&arg.mx);
+			nng_mtx_lock(arg.mx);
+			So(nng_cv_until(arg.cv, 0) == NNG_EAGAIN);
+			nng_mtx_unlock(arg.mx);
 		});
 
 		Convey("Not running works", {
 			arg.did  = 0;
 			arg.when = 1;
-			nni_mtx_lock(&arg.mx);
+			nng_mtx_lock(arg.mx);
 			if (!arg.did) {
-				nni_cv_until(&arg.cv, nni_clock() + 10);
+				nng_cv_until(arg.cv, nng_clock() + 10);
 			}
 			So(arg.did == 0);
-			nni_mtx_unlock(&arg.mx);
+			nng_mtx_unlock(arg.mx);
 		});
 	});
 }
@@ -157,104 +156,102 @@ test_sync_fallback(void)
 {
 	nni_plat_sync_fallback = 1;
 	Convey("Mutexes work", {
-		nni_mtx mx;
+		nng_mtx *mx;
 
-		nni_mtx_init(&mx);
+		So(nng_mtx_alloc(&mx) == 0);
+		Reset({ nng_mtx_free(mx); });
 
 		Convey("We can lock a mutex", {
-			nni_mtx_lock(&mx);
+			nng_mtx_lock(mx);
 			So(1);
 			Convey("And we can unlock it", {
-				nni_mtx_unlock(&mx);
+				nng_mtx_unlock(mx);
 				So(1);
 				Convey("And then lock it again", {
-					nni_mtx_lock(&mx);
+					nng_mtx_lock(mx);
 					So(1);
-					nni_mtx_unlock(&mx);
+					nng_mtx_unlock(mx);
 					So(1);
 				});
 			});
 			Convey("Things block properly", {
 
-				nni_mtx_init(&arg.mx);
-				nni_cv_init(&arg.cv, &arg.mx);
-				So(nni_thr_init(&thr, notifyafter, &arg) == 0);
+				So(nng_mtx_alloc(&arg.mx) == 0);
+				So(nng_cv_alloc(&arg.cv, arg.mx) == 0);
 				arg.did  = 0;
 				arg.when = 0;
-				nni_mtx_lock(&arg.mx);
-				nni_thr_run(&thr);
+				nng_mtx_lock(arg.mx);
+				So(nng_thread_create(
+				       &thr, notifyafter, &arg) == 0);
 				nng_msleep(10);
 				So(arg.did == 0);
-				nni_mtx_unlock(&arg.mx);
+				nng_mtx_unlock(arg.mx);
 				nng_msleep(10);
-				nni_mtx_lock(&arg.mx);
+				nng_mtx_lock(arg.mx);
 				while (!arg.did) {
-					nni_cv_wait(&arg.cv);
+					nng_cv_wait(arg.cv);
 				}
 				So(arg.did != 0);
-				nni_mtx_unlock(&arg.mx);
-				nni_thr_fini(&thr);
-				nni_cv_fini(&arg.cv);
-				nni_mtx_fini(&arg.mx);
+				nng_mtx_unlock(arg.mx);
+				nng_thread_destroy(thr);
+				nng_cv_free(arg.cv);
+				nng_mtx_free(arg.mx);
 			})
 		});
-		Convey("We can finalize it", { nni_mtx_fini(&mx); });
 	});
 
 	Convey("Condition variables work", {
 
-		nni_mtx_init(&arg.mx);
-		nni_cv_init(&arg.cv, &arg.mx);
-		So(nni_thr_init(&thr, notifyafter, &arg) == 0);
+		So(nng_mtx_alloc(&arg.mx) == 0);
+		So(nng_cv_alloc(&arg.cv, arg.mx) == 0);
 
 		Reset({
-			nni_cv_fini(&arg.cv);
-			nni_mtx_fini(&arg.mx);
-			nni_thr_fini(&thr);
+			nng_cv_free(arg.cv);
+			nng_mtx_free(arg.mx);
 		});
 
 		Convey("Notification works", {
 			arg.did  = 0;
 			arg.when = 10;
-			nni_thr_run(&thr);
+			So(nng_thread_create(&thr, notifyafter, &arg) == 0);
 
-			nni_mtx_lock(&arg.mx);
+			nng_mtx_lock(arg.mx);
 			if (!arg.did) {
-				nni_cv_wait(&arg.cv);
+				nng_cv_wait(arg.cv);
 			}
-			nni_mtx_unlock(&arg.mx);
-			nni_thr_wait(&thr);
+			nng_mtx_unlock(arg.mx);
+			nng_thread_destroy(thr);
 			So(arg.did == 1);
 		});
 
 		Convey("Timeout works", {
 			arg.did  = 0;
 			arg.when = 200;
-			nni_thr_run(&thr);
-			nni_mtx_lock(&arg.mx);
+			So(nng_thread_create(&thr, notifyafter, &arg) == 0);
+			nng_mtx_lock(arg.mx);
 			if (!arg.did) {
-				nni_cv_until(&arg.cv, nni_clock() + 10);
+				nng_cv_until(arg.cv, nng_clock() + 10);
 			}
 			So(arg.did == 0);
-			nni_mtx_unlock(&arg.mx);
-			nni_thr_wait(&thr);
+			nng_mtx_unlock(arg.mx);
+			nng_thread_destroy(thr);
 		});
 
 		Convey("Empty timeout is EAGAIN", {
-			nni_mtx_lock(&arg.mx);
-			So(nni_cv_until(&arg.cv, 0) == NNG_EAGAIN);
-			nni_mtx_unlock(&arg.mx);
+			nng_mtx_lock(arg.mx);
+			So(nng_cv_until(arg.cv, 0) == NNG_EAGAIN);
+			nng_mtx_unlock(arg.mx);
 		});
 
 		Convey("Not running works", {
 			arg.did  = 0;
 			arg.when = 1;
-			nni_mtx_lock(&arg.mx);
+			nng_mtx_lock(arg.mx);
 			if (!arg.did) {
-				nni_cv_until(&arg.cv, nni_clock() + 10);
+				nng_cv_until(arg.cv, nng_clock() + 10);
 			}
 			So(arg.did == 0);
-			nni_mtx_unlock(&arg.mx);
+			nng_mtx_unlock(arg.mx);
 		});
 	});
 }
@@ -263,7 +260,6 @@ test_sync_fallback(void)
 #endif
 
 TestMain("Synchronization", {
-	nni_init();
 
 	Convey("Synchronization works", { test_sync(); });
 
@@ -271,21 +267,22 @@ TestMain("Synchronization", {
 
 	ConveyFB("Transform works", {
 		nni_plat_sync_fallback = 0;
-		nni_mtx_init(&arg.mx);
+		So(nng_mtx_alloc(&arg.mx) == 0);
 		nni_plat_sync_fallback = 1;
-		nni_cv_init(&arg.cv, &arg.mx);
-		So(nni_thr_init(&thr, notifyafter, &arg) == 0);
+		So(nng_cv_alloc(&arg.cv, arg.mx) == 0);
 
 		arg.did  = 0;
 		arg.when = 10;
-		nni_thr_run(&thr);
+		So(nng_thread_create(&thr, notifyafter, &arg) == 0);
 
-		nni_mtx_lock(&arg.mx);
+		nng_mtx_lock(arg.mx);
 		if (!arg.did) {
-			nni_cv_wait(&arg.cv);
+			nng_cv_wait(arg.cv);
 		}
-		nni_mtx_unlock(&arg.mx);
-		nni_thr_wait(&thr);
+		nng_mtx_unlock(arg.mx);
+		nng_thread_destroy(thr);
 		So(arg.did == 1);
+		nng_cv_free(arg.cv);
+		nng_mtx_free(arg.mx);
 	});
 })
diff --git a/tests/trantest.h b/tests/trantest.h
index 13445eaf..feb34be8 100644
--- a/tests/trantest.h
+++ b/tests/trantest.h
@@ -1,6 +1,6 @@
 //
-// Copyright 2018 Capitar IT Group BV <info@capitar.com>
 // Copyright 2018 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2018 Capitar IT Group BV <info@capitar.com>
 //
 // This software is supplied under the terms of the MIT License, a
 // copy of which should be located in the distribution where this
@@ -13,6 +13,7 @@
 #include "nng.h"
 #include "protocol/reqrep0/rep.h"
 #include "protocol/reqrep0/req.h"
+#include "supplemental/util/platform.h"
 #include <stdlib.h>
 #include <string.h>