fixes #1032 Figure out Darwin bustedness

fixes #1035 Convey is awkward -- consider acutest.h

This represents a rather large effort towards cleaning up our
testing and optional configuration infrastructure.

A separate test library is built by default, which is static, and
includes some useful utilities design to make it easier to write
shorter and more robust (not timing dependent) tests.  This also means
that we can cover pretty nearly all the tests (protocols etc.) in
every case, even if the shipped image will be minimized.

Subsystems which are optional can now use a few new macros to configure
what they need see nng_sources_if, nng_headers_if, and nng_defines_if.
This goes a long way to making the distributed CMakefiles a lot simpler.

Additionally, tests for different parts of the tree can now be located
outside of the tests/ tree, so that they can be placed next to the code
that they are testing.

Beyond the enabling work, the work has only begun, but these changes
have resolved the most often failing tests for Darwin in the cloud.

2 files changed, 310 insertions, 22 deletions

diff --git a/src/protocol/reqrep0/CMakeLists.txt b/src/protocol/reqrep0/CMakeLists.txt
index bae31433..4778f4ea 100644
--- a/src/protocol/reqrep0/CMakeLists.txt
+++ b/src/protocol/reqrep0/CMakeLists.txt
@@ -1,5 +1,5 @@
 #
-# Copyright 2018 Staysail Systems, Inc. <info@staysail.tech>
+# Copyright 2019 Staysail Systems, Inc. <info@staysail.tech>
 # Copyright 2018 Capitar IT Group BV <info@capitar.com>
 #
 # This software is supplied under the terms of the MIT License, a
@@ -9,30 +9,18 @@
 #
 
 #  Req/Rep protocol
-option (NNG_PROTO_REQ0 "Enable REQv0 protocol." ON)
+option(NNG_PROTO_REQ0 "Enable REQv0 protocol." ON)
 mark_as_advanced(NNG_PROTO_REQ0)
 
-option (NNG_PROTO_REP0 "Enable REPv0 protocol." ON)
+option(NNG_PROTO_REP0 "Enable REPv0 protocol." ON)
 mark_as_advanced(NNG_PROTO_REP0)
 
-set(_DEFS)
-set(_SRCS)
+nng_sources_if(NNG_PROTO_REQ0 req.c xreq.c)
+nng_headers_if(NNG_PROTO_REQ0 nng/protocol/reqrep0/req.h)
+nng_defines_if(NNG_PROTO_REQ0 NNG_HAVE_REQ0)
 
-if (NNG_PROTO_REQ0)
-    list(APPEND _DEFS -DNNG_HAVE_REQ0)
-    list(APPEND _SRCS
-        protocol/reqrep0/req.c protocol/reqrep0/xreq.c
-        ${PROJECT_SOURCE_DIR}/include/nng/protocol/reqrep0/req.h)
-   
-endif()
+nng_sources_if(NNG_PROTO_REP0 rep.c xrep.c)
+nng_headers_if(NNG_PROTO_REP0 nng/protocol/reqrep0/rep.h)
+nng_defines_if(NNG_PROTO_REP0 NNG_HAVE_REP0)
 
-if (NNG_PROTO_REP0)
-    list(APPEND _DEFS -DNNG_HAVE_REP0)
-    list(APPEND _SRCS
-        protocol/reqrep0/rep.c protocol/reqrep0/xrep.c
-        ${PROJECT_SOURCE_DIR}/include/nng/protocol/reqrep0/rep.h)
-   
-endif()
-
-set(NNG_DEFS ${NNG_DEFS} ${_DEFS} PARENT_SCOPE)
-set(NNG_SRCS ${NNG_SRCS} ${_SRCS} PARENT_SCOPE)
+nng_test(reqrep_test)
diff --git a/src/protocol/reqrep0/reqrep_test.c b/src/protocol/reqrep0/reqrep_test.c
new file mode 100644
index 00000000..564cf158
--- /dev/null
+++ b/src/protocol/reqrep0/reqrep_test.c
@@ -0,0 +1,300 @@
+//
+// Copyright 2019 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2018 Capitar IT Group BV <info@capitar.com>
+//
+// This software is supplied under the terms of the MIT License, a
+// copy of which should be located in the distribution where this
+// file was obtained (LICENSE.txt).  A copy of the license may also be
+// found online at https://opensource.org/licenses/MIT.
+//
+
+#include <string.h>
+
+#include <nng/nng.h>
+#include <nng/protocol/reqrep0/rep.h>
+#include <nng/protocol/reqrep0/req.h>
+
+#include <acutest.h>
+#include <testutil.h>
+
+#ifndef NNI_PROTO
+#define NNI_PROTO(x, y) (((x) << 4u) | (y))
+#endif
+
+void
+test_req_rep_identity(void)
+{
+	nng_socket s;
+	int        p;
+	char *     n;
+
+	TEST_CHECK(nng_req0_open(&s) == 0);
+	TEST_CHECK(nng_getopt_int(s, NNG_OPT_PROTO, &p) == 0);
+	TEST_CHECK(p == NNI_PROTO(3u, 0u)); // 48
+	TEST_CHECK(nng_getopt_int(s, NNG_OPT_PEER, &p) == 0);
+	TEST_CHECK(p == NNI_PROTO(3u, 1u)); // 49
+	TEST_CHECK(nng_getopt_string(s, NNG_OPT_PROTONAME, &n) == 0);
+	TEST_CHECK(strcmp(n, "req") == 0);
+	TEST_CHECK(nng_getopt_string(s, NNG_OPT_PEERNAME, &n) == 0);
+	TEST_CHECK(strcmp(n, "rep") == 0);
+	TEST_CHECK(nng_close(s) == 0);
+
+	TEST_CHECK(nng_rep0_open(&s) == 0);
+	TEST_CHECK(nng_getopt_int(s, NNG_OPT_PROTO, &p) == 0);
+	TEST_CHECK(p == NNI_PROTO(3u, 1u)); // 49
+	TEST_CHECK(nng_getopt_int(s, NNG_OPT_PEER, &p) == 0);
+	TEST_CHECK(p == NNI_PROTO(3u, 0u)); // 48
+	TEST_CHECK(nng_getopt_string(s, NNG_OPT_PROTONAME, &n) == 0);
+	TEST_CHECK(strcmp(n, "rep") == 0);
+	TEST_CHECK(nng_getopt_string(s, NNG_OPT_PEERNAME, &n) == 0);
+	TEST_CHECK(strcmp(n, "req") == 0);
+	TEST_CHECK(nng_close(s) == 0);
+}
+
+void
+test_resend_option(void)
+{
+	nng_socket  req;
+	bool        b;
+	size_t      sz  = sizeof(b);
+	const char *opt = NNG_OPT_REQ_RESENDTIME;
+
+	TEST_CHECK(nng_req0_open(&req) == 0);
+
+	TEST_CHECK(nng_setopt_ms(req, opt, 10) == 0);
+	TEST_CHECK(nng_setopt(req, opt, "", 1) == NNG_EINVAL);
+	TEST_CHECK(nng_getopt(req, opt, &b, &sz) == NNG_EINVAL);
+	TEST_CHECK(nng_setopt_bool(req, opt, true) == NNG_EBADTYPE);
+	TEST_CHECK(nng_getopt_bool(req, opt, &b) == NNG_EBADTYPE);
+
+	TEST_CHECK(nng_close(req) == 0);
+}
+
+void
+test_req_recv_bad_state(void)
+{
+	nng_socket req;
+	nng_msg *  msg = NULL;
+
+	TEST_CHECK(nng_req0_open(&req) == 0);
+	TEST_CHECK(nng_recvmsg(req, &msg, 0) == NNG_ESTATE);
+	TEST_CHECK(msg == NULL);
+	TEST_CHECK(nng_close(req) == 0);
+}
+
+void
+test_rep_send_bad_state(void)
+{
+	nng_socket rep;
+	nng_msg *  msg = NULL;
+
+	TEST_CHECK(nng_rep0_open(&rep) == 0);
+	TEST_CHECK(nng_msg_alloc(&msg, 0) == 0);
+	TEST_CHECK(nng_sendmsg(rep, msg, 0) == NNG_ESTATE);
+	nng_msg_free(msg);
+	TEST_CHECK(nng_close(rep) == 0);
+}
+
+#define SECOND 1000
+
+void
+test_req_rep_exchange(void)
+{
+	nng_socket  req;
+	nng_socket  rep;
+	nng_msg *   msg  = NULL;
+
+	TEST_CHECK(nng_req0_open(&req) == 0);
+	TEST_CHECK(nng_rep0_open(&rep) == 0);
+
+	TEST_CHECK(nng_setopt_ms(req, NNG_OPT_RECVTIMEO, SECOND) == 0);
+	TEST_CHECK(nng_setopt_ms(rep, NNG_OPT_RECVTIMEO, SECOND) == 0);
+	TEST_CHECK(nng_setopt_ms(req, NNG_OPT_SENDTIMEO, SECOND) == 0);
+	TEST_CHECK(nng_setopt_ms(rep, NNG_OPT_SENDTIMEO, SECOND) == 0);
+
+	TEST_CHECK(testutil_marry(rep, req) == 0);
+
+	TEST_CHECK(nng_msg_alloc(&msg, 0) == 0);
+	TEST_CHECK(nng_msg_append(msg, "ping", 5) == 0);
+	TEST_CHECK(nng_msg_len(msg) == 5);
+	TEST_CHECK(strcmp(nng_msg_body(msg), "ping") == 0);
+	TEST_CHECK(nng_sendmsg(req, msg, 0) == 0);
+	msg = NULL;
+	TEST_CHECK(nng_recvmsg(rep, &msg, 0) == 0);
+	TEST_CHECK(msg != NULL);
+	TEST_CHECK(nng_msg_len(msg) == 5);
+	TEST_CHECK(strcmp(nng_msg_body(msg), "ping") == 0);
+	nng_msg_trim(msg, 5);
+	TEST_CHECK(nng_msg_append(msg, "pong", 5) == 0);
+	TEST_CHECK(nng_sendmsg(rep, msg, 0) == 0);
+	msg = NULL;
+	TEST_CHECK(nng_recvmsg(req, &msg, 0) == 0);
+	TEST_CHECK(msg != NULL);
+	TEST_CHECK(nng_msg_len(msg) == 5);
+	TEST_CHECK(strcmp(nng_msg_body(msg), "pong") == 0);
+	nng_msg_free(msg);
+
+	TEST_CHECK(nng_close(req) == 0);
+	TEST_CHECK(nng_close(rep) == 0);
+}
+
+void
+test_req_cancel(void)
+{
+	nng_msg *    abc;
+	nng_msg *    def;
+	nng_msg *    cmd;
+	nng_duration retry = SECOND;
+	nng_socket   req;
+	nng_socket   rep;
+
+	TEST_CHECK(nng_rep_open(&rep) == 0);
+	TEST_CHECK(nng_req_open(&req) == 0);
+
+	TEST_CHECK(nng_setopt_ms(req, NNG_OPT_RECVTIMEO, SECOND) == 0);
+	TEST_CHECK(nng_setopt_ms(rep, NNG_OPT_RECVTIMEO, SECOND) == 0);
+	TEST_CHECK(nng_setopt_ms(req, NNG_OPT_SENDTIMEO, SECOND) == 0);
+	TEST_CHECK(nng_setopt_ms(rep, NNG_OPT_SENDTIMEO, SECOND) == 0);
+	TEST_CHECK(nng_setopt_ms(req, NNG_OPT_REQ_RESENDTIME, retry) == 0);
+	TEST_CHECK(nng_setopt_int(req, NNG_OPT_SENDBUF, 16) == 0);
+
+	TEST_CHECK(nng_msg_alloc(&abc, 0) == 0);
+	TEST_CHECK(nng_msg_append(abc, "abc", 4) == 0);
+	TEST_CHECK(nng_msg_alloc(&def, 0) == 0);
+	TEST_CHECK(nng_msg_append(def, "def", 4) == 0);
+
+	TEST_CHECK(testutil_marry(rep, req) == 0);
+
+	// Send req #1 (abc).
+	TEST_CHECK(nng_sendmsg(req, abc, 0) == 0);
+
+	// Sleep a bit.  This is so that we ensure that our request gets
+	// to the far side.  (If we cancel too fast, then our outgoing send
+	// will be canceled before it gets to the peer.)
+	testutil_sleep(100);
+
+	// Send the next next request ("def").  Note that
+	// the REP side server will have already buffered the receive
+	// request, and should simply be waiting for us to reply to abc.
+	TEST_CHECK(nng_sendmsg(req, def, 0) == 0);
+
+	// Receive the first request (should be abc) on the REP server.
+	TEST_CHECK(nng_recvmsg(rep, &cmd, 0) == 0);
+	TEST_ASSERT(cmd != NULL);
+	TEST_CHECK(nng_msg_len(cmd) == 4);
+	TEST_CHECK(strcmp(nng_msg_body(cmd), "abc") == 0);
+
+	// REP sends the reply to first command.  This will be discarded
+	// by the REQ socket.
+	TEST_CHECK(nng_sendmsg(rep, cmd, 0) == 0);
+
+	// Now get the next command from the REP; should be "def".
+	TEST_CHECK(nng_recvmsg(rep, &cmd, 0) == 0);
+	TEST_ASSERT(cmd != NULL);
+	TEST_CHECK(nng_msg_len(cmd) == 4);
+	TEST_CHECK(strcmp(nng_msg_body(cmd), "def") == 0);
+	TEST_MSG("Received body was %s", nng_msg_body(cmd));
+
+	// And send it back to REQ.
+	TEST_CHECK(nng_sendmsg(rep, cmd, 0) == 0);
+
+	// Try a req command.  This should give back "def"
+	TEST_CHECK(nng_recvmsg(req, &cmd, 0) == 0);
+	TEST_CHECK(nng_msg_len(cmd) == 4);
+	TEST_CHECK(strcmp(nng_msg_body(cmd), "def") == 0);
+	nng_msg_free(cmd);
+
+	TEST_CHECK(nng_close(req) == 0);
+	TEST_CHECK(nng_close(rep) == 0);
+}
+
+void
+test_req_cancel_abort_recv(void)
+{
+
+	nng_msg *    abc;
+	nng_msg *    def;
+	nng_msg *    cmd;
+	nng_aio *    aio;
+	nng_duration retry = SECOND * 10; // 10s (kind of never)
+	nng_socket   req;
+	nng_socket   rep;
+
+	TEST_CHECK(nng_rep_open(&rep) == 0);
+	TEST_CHECK(nng_req_open(&req) == 0);
+	TEST_CHECK(nng_aio_alloc(&aio, NULL, NULL) == 0);
+
+	TEST_CHECK(nng_setopt_ms(req, NNG_OPT_REQ_RESENDTIME, retry) == 0);
+	TEST_CHECK(nng_setopt_int(req, NNG_OPT_SENDBUF, 16) == 0);
+	TEST_CHECK(nng_setopt_ms(req, NNG_OPT_RECVTIMEO, 5 * SECOND) == 0);
+	TEST_CHECK(nng_setopt_ms(rep, NNG_OPT_RECVTIMEO, 5 * SECOND) == 0);
+	TEST_CHECK(nng_setopt_ms(req, NNG_OPT_SENDTIMEO, 5 * SECOND) == 0);
+	TEST_CHECK(nng_setopt_ms(rep, NNG_OPT_SENDTIMEO, 5 * SECOND) == 0);
+
+	TEST_CHECK(nng_msg_alloc(&abc, 0) == 0);
+	TEST_CHECK(nng_msg_append(abc, "abc", 4) == 0);
+	TEST_CHECK(nng_msg_alloc(&def, 0) == 0);
+	TEST_CHECK(nng_msg_append(def, "def", 4) == 0);
+
+	TEST_CHECK(testutil_marry(rep, req) == 0);
+
+	// Send req #1 (abc).
+	TEST_CHECK(nng_sendmsg(req, abc, 0) == 0);
+
+	// Wait for it to get ot the other side.
+	testutil_sleep(100);
+
+	nng_aio_set_timeout(aio, 5 * SECOND);
+	nng_recv_aio(req, aio);
+
+	// Give time for this recv to post properly.
+	testutil_sleep(100);
+
+	// Send the next next request ("def").  Note that
+	// the REP side server will have already buffered the receive
+	// request, and should simply be waiting for us to reply to
+	// abc.
+	TEST_CHECK(nng_sendmsg(req, def, 0) == 0);
+
+	// Our pending I/O should have been canceled.
+	nng_aio_wait(aio);
+	TEST_CHECK(nng_aio_result(aio) == NNG_ECANCELED);
+
+	// Receive the first request (should be abc) on the REP server.
+	TEST_CHECK(nng_recvmsg(rep, &cmd, 0) == 0);
+	TEST_CHECK(nng_msg_len(cmd) == 4);
+	TEST_CHECK(strcmp(nng_msg_body(cmd), "abc") == 0);
+
+	// REP sends the reply to first command.  This will be
+	// discarded by the REQ socket.
+	TEST_CHECK(nng_sendmsg(rep, cmd, 0) == 0);
+
+	// Now get the next command from the REP; should be "def".
+	TEST_CHECK(nng_recvmsg(rep, &cmd, 0) == 0);
+	TEST_CHECK(nng_msg_len(cmd) == 4);
+	TEST_CHECK(strcmp(nng_msg_body(cmd), "def") == 0);
+
+	// And send it back to REQ.
+	TEST_CHECK(nng_sendmsg(rep, cmd, 0) == 0);
+
+	// Try a req command.  This should give back "def"
+	TEST_CHECK(nng_recvmsg(req, &cmd, 0) == 0);
+	TEST_CHECK(nng_msg_len(cmd) == 4);
+	TEST_CHECK(strcmp(nng_msg_body(cmd), "def") == 0);
+	nng_msg_free(cmd);
+
+	nng_aio_free(aio);
+	TEST_CHECK(nng_close(req) == 0);
+	TEST_CHECK(nng_close(rep) == 0);
+}
+
+TEST_LIST = {
+	{ "req rep identity", test_req_rep_identity },
+	{ "resend option", test_resend_option },
+	{ "req recv bad state", test_req_recv_bad_state },
+	{ "rep send bad state", test_rep_send_bad_state },
+	{ "req rep exchange", test_req_rep_exchange },
+	{ "req cancel", test_req_cancel },
+	{ "req cancel abort recv", test_req_cancel_abort_recv },
+	{ NULL, NULL },
+};