fixes #1124 data race on ttl in xrep

3 files changed, 521 insertions, 71 deletions

diff --git a/src/protocol/reqrep0/CMakeLists.txt b/src/protocol/reqrep0/CMakeLists.txt
index 4d2903c6..3cd8d366 100644
--- a/src/protocol/reqrep0/CMakeLists.txt
+++ b/src/protocol/reqrep0/CMakeLists.txt
@@ -1,5 +1,5 @@
 #
-# Copyright 2019 Staysail Systems, Inc. <info@staysail.tech>
+# Copyright 2020 Staysail Systems, Inc. <info@staysail.tech>
 # Copyright 2018 Capitar IT Group BV <info@capitar.com>
 #
 # This software is supplied under the terms of the MIT License, a
@@ -24,4 +24,5 @@ nng_headers_if(NNG_PROTO_REP0 nng/protocol/reqrep0/rep.h)
 nng_defines_if(NNG_PROTO_REP0 NNG_HAVE_REP0)
 
 nng_test(reqrep_test)
-nng_test(rep_test)
\ No newline at end of file
+nng_test(rep_test)
+nng_test(xrep_test)
diff --git a/src/protocol/reqrep0/xrep.c b/src/protocol/reqrep0/xrep.c
index 308c0f0e..aac6d6b9 100644
--- a/src/protocol/reqrep0/xrep.c
+++ b/src/protocol/reqrep0/xrep.c
@@ -8,7 +8,6 @@
 // found online at https://opensource.org/licenses/MIT.
 //
 
-#include <stdlib.h>
 #include <string.h>
 
 #include "core/nng_impl.h"
@@ -38,12 +37,12 @@ static void xrep0_pipe_fini(void *);
 
 // xrep0_sock is our per-socket protocol private structure.
 struct xrep0_sock {
-	nni_msgq *  uwq;
-	nni_msgq *  urq;
-	nni_mtx     lk;
-	int         ttl;
-	nni_idhash *pipes;
-	nni_aio *   aio_getq;
+	nni_msgq *     uwq;
+	nni_msgq *     urq;
+	nni_mtx        lk;
+	nni_atomic_u64 ttl;
+	nni_idhash *   pipes;
+	nni_aio        aio_getq;
 };
 
 // xrep0_pipe is our per-pipe protocol private structure.
@@ -51,10 +50,10 @@ struct xrep0_pipe {
 	nni_pipe *  pipe;
 	xrep0_sock *rep;
 	nni_msgq *  sendq;
-	nni_aio *   aio_getq;
-	nni_aio *   aio_send;
-	nni_aio *   aio_recv;
-	nni_aio *   aio_putq;
+	nni_aio     aio_getq;
+	nni_aio     aio_send;
+	nni_aio     aio_recv;
+	nni_aio     aio_putq;
 };
 
 static void
@@ -62,7 +61,7 @@ xrep0_sock_fini(void *arg)
 {
 	xrep0_sock *s = arg;
 
-	nni_aio_free(s->aio_getq);
+	nni_aio_fini(&s->aio_getq);
 	nni_idhash_fini(s->pipes);
 	nni_mtx_fini(&s->lk);
 }
@@ -74,16 +73,17 @@ xrep0_sock_init(void *arg, nni_sock *sock)
 	int         rv;
 
 	nni_mtx_init(&s->lk);
-	if (((rv = nni_idhash_init(&s->pipes)) != 0) ||
-	    ((rv = nni_aio_alloc(&s->aio_getq, xrep0_sock_getq_cb, s)) != 0)) {
+	nni_aio_init(&s->aio_getq, xrep0_sock_getq_cb, s);
+	nni_atomic_init64(&s->ttl);
+	nni_atomic_set64(&s->ttl, 8); // Per RFC
+	s->uwq = nni_sock_sendq(sock);
+	s->urq = nni_sock_recvq(sock);
+
+	if ((rv = nni_idhash_init(&s->pipes)) != 0) {
 		xrep0_sock_fini(s);
 		return (rv);
 	}
 
-	s->ttl = 8; // Per RFC
-	s->uwq = nni_sock_sendq(sock);
-	s->urq = nni_sock_recvq(sock);
-
 	return (0);
 }
 
@@ -93,7 +93,7 @@ xrep0_sock_open(void *arg)
 	xrep0_sock *s = arg;
 
 	// This starts us retrieving message from the upper write q.
-	nni_msgq_aio_get(s->uwq, s->aio_getq);
+	nni_msgq_aio_get(s->uwq, &s->aio_getq);
 }
 
 static void
@@ -101,7 +101,7 @@ xrep0_sock_close(void *arg)
 {
 	xrep0_sock *s = arg;
 
-	nni_aio_close(s->aio_getq);
+	nni_aio_close(&s->aio_getq);
 }
 
 static void
@@ -109,10 +109,10 @@ xrep0_pipe_stop(void *arg)
 {
 	xrep0_pipe *p = arg;
 
-	nni_aio_stop(p->aio_getq);
-	nni_aio_stop(p->aio_send);
-	nni_aio_stop(p->aio_recv);
-	nni_aio_stop(p->aio_putq);
+	nni_aio_stop(&p->aio_getq);
+	nni_aio_stop(&p->aio_send);
+	nni_aio_stop(&p->aio_recv);
+	nni_aio_stop(&p->aio_putq);
 }
 
 static void
@@ -120,10 +120,10 @@ xrep0_pipe_fini(void *arg)
 {
 	xrep0_pipe *p = arg;
 
-	nni_aio_free(p->aio_getq);
-	nni_aio_free(p->aio_send);
-	nni_aio_free(p->aio_recv);
-	nni_aio_free(p->aio_putq);
+	nni_aio_fini(&p->aio_getq);
+	nni_aio_fini(&p->aio_send);
+	nni_aio_fini(&p->aio_recv);
+	nni_aio_fini(&p->aio_putq);
 	nni_msgq_fini(p->sendq);
 }
 
@@ -133,6 +133,14 @@ xrep0_pipe_init(void *arg, nni_pipe *pipe, void *s)
 	xrep0_pipe *p = arg;
 	int         rv;
 
+	nni_aio_init(&p->aio_getq, xrep0_pipe_getq_cb, p);
+	nni_aio_init(&p->aio_send, xrep0_pipe_send_cb, p);
+	nni_aio_init(&p->aio_recv, xrep0_pipe_recv_cb, p);
+	nni_aio_init(&p->aio_putq, xrep0_pipe_putq_cb, p);
+
+	p->pipe = pipe;
+	p->rep  = s;
+
 	// We want a pretty deep send queue on pipes. The rationale here is
 	// that the send rate will be mitigated by the receive rate.
 	// If a slow pipe (req pipe not reading its own responses!?)
@@ -145,17 +153,10 @@ xrep0_pipe_init(void *arg, nni_pipe *pipe, void *s)
 	// essentially don't let peers send requests faster than they are
 	// willing to receive replies.  Something to think about for the
 	// future.)
-	if (((rv = nni_msgq_init(&p->sendq, 64)) != 0) ||
-	    ((rv = nni_aio_alloc(&p->aio_getq, xrep0_pipe_getq_cb, p)) != 0) ||
-	    ((rv = nni_aio_alloc(&p->aio_send, xrep0_pipe_send_cb, p)) != 0) ||
-	    ((rv = nni_aio_alloc(&p->aio_recv, xrep0_pipe_recv_cb, p)) != 0) ||
-	    ((rv = nni_aio_alloc(&p->aio_putq, xrep0_pipe_putq_cb, p)) != 0)) {
+	if ((rv = nni_msgq_init(&p->sendq, 64)) != 0) {
 		xrep0_pipe_fini(p);
 		return (rv);
 	}
-
-	p->pipe = pipe;
-	p->rep  = s;
 	return (0);
 }
 
@@ -175,8 +176,8 @@ xrep0_pipe_start(void *arg)
 		return (rv);
 	}
 
-	nni_msgq_aio_get(p->sendq, p->aio_getq);
-	nni_pipe_recv(p->pipe, p->aio_recv);
+	nni_msgq_aio_get(p->sendq, &p->aio_getq);
+	nni_pipe_recv(p->pipe, &p->aio_recv);
 	return (0);
 }
 
@@ -186,10 +187,10 @@ xrep0_pipe_close(void *arg)
 	xrep0_pipe *p = arg;
 	xrep0_sock *s = p->rep;
 
-	nni_aio_close(p->aio_getq);
-	nni_aio_close(p->aio_send);
-	nni_aio_close(p->aio_recv);
-	nni_aio_close(p->aio_putq);
+	nni_aio_close(&p->aio_getq);
+	nni_aio_close(&p->aio_send);
+	nni_aio_close(&p->aio_recv);
+	nni_aio_close(&p->aio_putq);
 	nni_msgq_close(p->sendq);
 
 	nni_mtx_lock(&s->lk);
@@ -211,27 +212,27 @@ xrep0_sock_getq_cb(void *arg)
 	// destination pipe via a separate queue.  This prevents a single bad
 	// or slow pipe from gumming up the works for the entire socket.
 
-	if (nni_aio_result(s->aio_getq) != 0) {
+	if (nni_aio_result(&s->aio_getq) != 0) {
 		// Closed socket?
 		return;
 	}
 
-	msg = nni_aio_get_msg(s->aio_getq);
-	nni_aio_set_msg(s->aio_getq, NULL);
+	msg = nni_aio_get_msg(&s->aio_getq);
+	nni_aio_set_msg(&s->aio_getq, NULL);
 
 	// We yank the outgoing pipe id from the header
 	if (nni_msg_header_len(msg) < 4) {
 		nni_msg_free(msg);
 
 		// Look for another message on the upper write queue.
-		nni_msgq_aio_get(uwq, s->aio_getq);
+		nni_msgq_aio_get(uwq, &s->aio_getq);
 		return;
 	}
 
 	id = nni_msg_header_trim_u32(msg);
 
 	// Look for the pipe, and attempt to put the message there
-	// (nonblocking) if we can.  If we can't for any reason, then we
+	// (non-blocking) if we can.  If we can't for any reason, then we
 	// free the message.
 	nni_mtx_lock(&s->lk);
 	if (((nni_idhash_find(s->pipes, id, (void **) &p)) != 0) ||
@@ -241,7 +242,7 @@ xrep0_sock_getq_cb(void *arg)
 	nni_mtx_unlock(&s->lk);
 
 	// Now look for another message on the upper write queue.
-	nni_msgq_aio_get(uwq, s->aio_getq);
+	nni_msgq_aio_get(uwq, &s->aio_getq);
 }
 
 static void
@@ -249,15 +250,15 @@ xrep0_pipe_getq_cb(void *arg)
 {
 	xrep0_pipe *p = arg;
 
-	if (nni_aio_result(p->aio_getq) != 0) {
+	if (nni_aio_result(&p->aio_getq) != 0) {
 		nni_pipe_close(p->pipe);
 		return;
 	}
 
-	nni_aio_set_msg(p->aio_send, nni_aio_get_msg(p->aio_getq));
-	nni_aio_set_msg(p->aio_getq, NULL);
+	nni_aio_set_msg(&p->aio_send, nni_aio_get_msg(&p->aio_getq));
+	nni_aio_set_msg(&p->aio_getq, NULL);
 
-	nni_pipe_send(p->pipe, p->aio_send);
+	nni_pipe_send(p->pipe, &p->aio_send);
 }
 
 static void
@@ -265,14 +266,14 @@ xrep0_pipe_send_cb(void *arg)
 {
 	xrep0_pipe *p = arg;
 
-	if (nni_aio_result(p->aio_send) != 0) {
-		nni_msg_free(nni_aio_get_msg(p->aio_send));
-		nni_aio_set_msg(p->aio_send, NULL);
+	if (nni_aio_result(&p->aio_send) != 0) {
+		nni_msg_free(nni_aio_get_msg(&p->aio_send));
+		nni_aio_set_msg(&p->aio_send, NULL);
 		nni_pipe_close(p->pipe);
 		return;
 	}
 
-	nni_msgq_aio_get(p->sendq, p->aio_getq);
+	nni_msgq_aio_get(p->sendq, &p->aio_getq);
 }
 
 static void
@@ -283,13 +284,13 @@ xrep0_pipe_recv_cb(void *arg)
 	nni_msg *   msg;
 	int         hops;
 
-	if (nni_aio_result(p->aio_recv) != 0) {
+	if (nni_aio_result(&p->aio_recv) != 0) {
 		nni_pipe_close(p->pipe);
 		return;
 	}
 
-	msg = nni_aio_get_msg(p->aio_recv);
-	nni_aio_set_msg(p->aio_recv, NULL);
+	msg = nni_aio_get_msg(&p->aio_recv);
+	nni_aio_set_msg(&p->aio_recv, NULL);
 
 	nni_msg_set_pipe(msg, nni_pipe_id(p->pipe));
 
@@ -304,7 +305,7 @@ xrep0_pipe_recv_cb(void *arg)
 	for (;;) {
 		bool     end = 0;
 		uint8_t *body;
-		if (hops > s->ttl) {
+		if (hops > (int)nni_atomic_get64(&s->ttl)) {
 			// This isn't malformed, but it has gone through
 			// too many hops.  Do not disconnect, because we
 			// can legitimately receive messages with too many
@@ -332,13 +333,13 @@ xrep0_pipe_recv_cb(void *arg)
 	}
 
 	// Go ahead and send it up.
-	nni_aio_set_msg(p->aio_putq, msg);
-	nni_msgq_aio_put(s->urq, p->aio_putq);
+	nni_aio_set_msg(&p->aio_putq, msg);
+	nni_msgq_aio_put(s->urq, &p->aio_putq);
 	return;
 
 drop:
 	nni_msg_free(msg);
-	nni_pipe_recv(p->pipe, p->aio_recv);
+	nni_pipe_recv(p->pipe, &p->aio_recv);
 }
 
 static void
@@ -346,28 +347,33 @@ xrep0_pipe_putq_cb(void *arg)
 {
 	xrep0_pipe *p = arg;
 
-	if (nni_aio_result(p->aio_putq) != 0) {
-		nni_msg_free(nni_aio_get_msg(p->aio_putq));
-		nni_aio_set_msg(p->aio_putq, NULL);
+	if (nni_aio_result(&p->aio_putq) != 0) {
+		nni_msg_free(nni_aio_get_msg(&p->aio_putq));
+		nni_aio_set_msg(&p->aio_putq, NULL);
 		nni_pipe_close(p->pipe);
 		return;
 	}
 
-	nni_pipe_recv(p->pipe, p->aio_recv);
+	nni_pipe_recv(p->pipe, &p->aio_recv);
 }
 
 static int
 xrep0_sock_set_maxttl(void *arg, const void *buf, size_t sz, nni_opt_type t)
 {
 	xrep0_sock *s = arg;
-	return (nni_copyin_int(&s->ttl, buf, sz, 1, 255, t));
+	int         ttl;
+	int         rv;
+	if ((rv = nni_copyin_int(&ttl, buf, sz, 1, 255, t)) == 0) {
+		nni_atomic_set64(&s->ttl, (uint64_t) ttl);
+	}
+	return (rv);
 }
 
 static int
 xrep0_sock_get_maxttl(void *arg, void *buf, size_t *szp, nni_opt_type t)
 {
 	xrep0_sock *s = arg;
-	return (nni_copyout_int(s->ttl, buf, szp, t));
+	return (nni_copyout_int((int) nni_atomic_get64(&s->ttl), buf, szp, t));
 }
 
 static void
diff --git a/src/protocol/reqrep0/xrep_test.c b/src/protocol/reqrep0/xrep_test.c
new file mode 100644
index 00000000..93573be1
--- /dev/null
+++ b/src/protocol/reqrep0/xrep_test.c
@@ -0,0 +1,443 @@
+//
+// Copyright 2020 Staysail Systems, Inc. <info@staysail.tech>
+//
+// This software is supplied under the terms of the MIT License, a
+// copy of which should be located in the distribution where this
+// file was obtained (LICENSE.txt).  A copy of the license may also be
+// found online at https://opensource.org/licenses/MIT.
+//
+
+#include <string.h>
+
+#include <nng/nng.h>
+#include <nng/protocol/reqrep0/rep.h>
+#include <nng/protocol/reqrep0/req.h>
+
+#include <acutest.h>
+#include <testutil.h>
+
+#ifndef NNI_PROTO
+#define NNI_PROTO(x, y) (((x) << 4u) | (y))
+#endif
+
+void
+test_xrep_identity(void)
+{
+	nng_socket s;
+	int        p;
+	char *     n;
+
+	TEST_CHECK(nng_rep0_open_raw(&s) == 0);
+	TEST_CHECK(nng_getopt_int(s, NNG_OPT_PROTO, &p) == 0);
+	TEST_CHECK(p == NNI_PROTO(3u, 1u)); // 49
+	TEST_CHECK(nng_getopt_int(s, NNG_OPT_PEER, &p) == 0);
+	TEST_CHECK(p == NNI_PROTO(3u, 0u)); // 48
+	TEST_CHECK(nng_getopt_string(s, NNG_OPT_PROTONAME, &n) == 0);
+	TEST_CHECK(strcmp(n, "rep") == 0);
+	nng_strfree(n);
+	TEST_CHECK(nng_getopt_string(s, NNG_OPT_PEERNAME, &n) == 0);
+	TEST_CHECK(strcmp(n, "req") == 0);
+	nng_strfree(n);
+	TEST_CHECK(nng_close(s) == 0);
+}
+
+void
+test_xrep_raw(void)
+{
+	nng_socket s;
+	bool       b;
+
+	TEST_NNG_PASS(nng_rep0_open_raw(&s));
+	TEST_NNG_PASS(nng_getopt_bool(s, NNG_OPT_RAW, &b));
+	TEST_CHECK(b);
+	TEST_NNG_PASS(nng_close(s));
+}
+
+void
+test_xrep_no_context(void)
+{
+	nng_socket s;
+	nng_ctx    ctx;
+
+	TEST_NNG_PASS(nng_rep0_open_raw(&s));
+	TEST_NNG_FAIL(nng_ctx_open(&ctx, s), NNG_ENOTSUP);
+	TEST_NNG_PASS(nng_close(s));
+}
+
+void
+test_xrep_poll_writeable(void)
+{
+	int        fd;
+	nng_socket req;
+	nng_socket rep;
+
+	TEST_NNG_PASS(nng_rep0_open_raw(&rep));
+	TEST_NNG_PASS(nng_req0_open(&req));
+	TEST_NNG_PASS(nng_getopt_int(rep, NNG_OPT_SENDFD, &fd));
+	TEST_CHECK(fd >= 0);
+
+	// We are always writeable, even before connect.  This is so that
+	// back-pressure from a bad peer can't trash others.  We assume
+	// that peers won't send us requests faster than they can consume
+	// the answers.  If they do, they will lose their answers.
+	TEST_CHECK(testutil_pollfd(fd) == true);
+
+	TEST_NNG_PASS(testutil_marry(req, rep));
+
+	// Now it's writable.
+	TEST_CHECK(testutil_pollfd(fd) == true);
+
+	TEST_NNG_PASS(nng_close(req));
+	TEST_NNG_PASS(nng_close(rep));
+}
+
+void
+test_xrep_poll_readable(void)
+{
+	int        fd;
+	nng_socket req;
+	nng_socket rep;
+	nng_msg *  msg;
+
+	TEST_NNG_PASS(nng_req0_open(&req));
+	TEST_NNG_PASS(nng_rep0_open_raw(&rep));
+	TEST_NNG_PASS(nng_getopt_int(rep, NNG_OPT_RECVFD, &fd));
+	TEST_CHECK(fd >= 0);
+
+	// Not readable if not connected!
+	TEST_CHECK(testutil_pollfd(fd) == false);
+
+	// Even after connect (no message yet)
+	TEST_NNG_PASS(testutil_marry(req, rep));
+	TEST_CHECK(testutil_pollfd(fd) == false);
+
+	// But once we send messages, it is.
+	// We have to send a request, in order to send a reply.
+	TEST_NNG_SEND_STR(req, "abc");
+	testutil_sleep(100);
+
+	TEST_CHECK(testutil_pollfd(fd) == true);
+
+	// and receiving makes it no longer pollable
+	TEST_NNG_PASS(nng_recvmsg(rep, &msg, 0));
+	nng_msg_free(msg);
+	TEST_CHECK(testutil_pollfd(fd) == false);
+
+	TEST_NNG_PASS(nng_close(req));
+	TEST_NNG_PASS(nng_close(rep));
+}
+
+void
+test_xrep_validate_peer(void)
+{
+	nng_socket s1, s2;
+	nng_stat * stats;
+	nng_stat * reject;
+	char       addr[64];
+
+	testutil_scratch_addr("inproc", sizeof(addr), addr);
+
+	TEST_NNG_PASS(nng_rep0_open_raw(&s1));
+	TEST_NNG_PASS(nng_rep0_open(&s2));
+
+	TEST_NNG_PASS(nng_listen(s1, addr, NULL, 0));
+	TEST_NNG_PASS(nng_dial(s2, addr, NULL, NNG_FLAG_NONBLOCK));
+
+	testutil_sleep(100);
+	TEST_NNG_PASS(nng_stats_get(&stats));
+
+	TEST_CHECK(stats != NULL);
+	TEST_CHECK((reject = nng_stat_find_socket(stats, s1)) != NULL);
+	TEST_CHECK((reject = nng_stat_find(reject, "reject")) != NULL);
+
+	TEST_CHECK(nng_stat_type(reject) == NNG_STAT_COUNTER);
+	TEST_CHECK(nng_stat_value(reject) > 0);
+
+	TEST_NNG_PASS(nng_close(s1));
+	TEST_NNG_PASS(nng_close(s2));
+	nng_stats_free(stats);
+}
+
+void
+test_xrep_close_pipe_before_send(void)
+{
+	nng_socket rep;
+	nng_socket req;
+	nng_pipe   p;
+	nng_aio *  aio1;
+	nng_msg *  m;
+
+	TEST_NNG_PASS(nng_rep0_open_raw(&rep));
+	TEST_NNG_PASS(nng_req0_open(&req));
+	TEST_NNG_PASS(nng_setopt_ms(rep, NNG_OPT_RECVTIMEO, 1000));
+	TEST_NNG_PASS(nng_setopt_ms(rep, NNG_OPT_SENDTIMEO, 1000));
+	TEST_NNG_PASS(nng_setopt_ms(req, NNG_OPT_SENDTIMEO, 1000));
+	TEST_NNG_PASS(nng_aio_alloc(&aio1, NULL, NULL));
+
+	TEST_NNG_PASS(testutil_marry(req, rep));
+	TEST_NNG_SEND_STR(req, "test");
+
+	nng_recv_aio(rep, aio1);
+	nng_aio_wait(aio1);
+	TEST_NNG_PASS(nng_aio_result(aio1));
+	TEST_CHECK((m = nng_aio_get_msg(aio1)) != NULL);
+	p = nng_msg_get_pipe(m);
+	TEST_NNG_PASS(nng_pipe_close(p));
+	TEST_NNG_PASS(nng_sendmsg(rep, m, 0));
+
+	TEST_NNG_PASS(nng_close(req));
+	TEST_NNG_PASS(nng_close(rep));
+	nng_aio_free(aio1);
+}
+
+void
+test_xrep_close_pipe_during_send(void)
+{
+	nng_socket rep;
+	nng_socket req;
+	nng_pipe   p;
+	nng_msg *  m;
+
+	TEST_NNG_PASS(nng_rep0_open_raw(&rep));
+	TEST_NNG_PASS(nng_req0_open_raw(&req));
+	TEST_NNG_PASS(nng_setopt_ms(rep, NNG_OPT_RECVTIMEO, 1000));
+	TEST_NNG_PASS(nng_setopt_ms(rep, NNG_OPT_SENDTIMEO, 200));
+	TEST_NNG_PASS(nng_setopt_ms(req, NNG_OPT_SENDTIMEO, 1000));
+	TEST_NNG_PASS(nng_setopt_int(rep, NNG_OPT_SENDBUF, 20));
+	TEST_NNG_PASS(nng_setopt_int(rep, NNG_OPT_RECVBUF, 20));
+	TEST_NNG_PASS(nng_setopt_int(req, NNG_OPT_SENDBUF, 20));
+	TEST_NNG_PASS(nng_setopt_int(req, NNG_OPT_RECVBUF, 1));
+
+	TEST_NNG_PASS(testutil_marry(req, rep));
+
+	TEST_NNG_PASS(nng_msg_alloc(&m, 4));
+	TEST_NNG_PASS(nng_msg_append_u32(m, (unsigned) 0x81000000u));
+	TEST_NNG_PASS(nng_sendmsg(req, m, 0));
+	TEST_NNG_PASS(nng_recvmsg(rep, &m, 0));
+	p = nng_msg_get_pipe(m);
+	nng_msg_free(m);
+
+	for (int i = 0; i < 100; i++) {
+		TEST_NNG_PASS(nng_msg_alloc(&m, 4));
+		TEST_NNG_PASS(nng_msg_header_append_u32(m, nng_pipe_id(p)));
+		TEST_NNG_PASS(
+		    nng_msg_header_append_u32(m, (unsigned) i | 0x80000000u));
+		// xrep does not exert back-pressure
+		TEST_NNG_PASS(nng_sendmsg(rep, m, 0));
+	}
+	TEST_NNG_PASS(nng_pipe_close(p));
+
+	TEST_NNG_PASS(nng_close(req));
+	TEST_NNG_PASS(nng_close(rep));
+}
+
+void
+test_xrep_close_during_recv(void)
+{
+	nng_socket rep;
+	nng_socket req;
+	nng_msg *  m;
+
+	TEST_NNG_PASS(nng_rep0_open_raw(&rep));
+	TEST_NNG_PASS(nng_req0_open_raw(&req));
+	TEST_NNG_PASS(nng_setopt_ms(rep, NNG_OPT_RECVTIMEO, 1000));
+	TEST_NNG_PASS(nng_setopt_ms(req, NNG_OPT_SENDTIMEO, 100));
+	TEST_NNG_PASS(nng_setopt_int(rep, NNG_OPT_RECVBUF, 5));
+	TEST_NNG_PASS(nng_setopt_int(req, NNG_OPT_SENDBUF, 20));
+
+	TEST_NNG_PASS(testutil_marry(req, rep));
+
+	for (unsigned i = 0; i < 100; i++) {
+		int rv;
+		TEST_NNG_PASS(nng_msg_alloc(&m, 4));
+		TEST_NNG_PASS(nng_msg_header_append_u32(m, i | 0x80000000u));
+		rv = nng_sendmsg(req, m, 0);
+		if (rv == NNG_ETIMEDOUT) {
+			nng_msg_free(m);
+			break;
+		}
+	}
+	TEST_NNG_PASS(nng_close(req));
+	TEST_NNG_PASS(nng_close(rep));
+}
+
+void
+test_xrep_recv_aio_stopped(void)
+{
+	nng_socket rep;
+	nng_aio *  aio;
+
+	TEST_NNG_PASS(nng_rep0_open_raw(&rep));
+	TEST_NNG_PASS(nng_aio_alloc(&aio, NULL, NULL));
+
+	nng_aio_stop(aio);
+	nng_recv_aio(rep, aio);
+	nng_aio_wait(aio);
+	TEST_NNG_FAIL(nng_aio_result(aio), NNG_ECANCELED);
+	TEST_NNG_PASS(nng_close(rep));
+	nng_aio_free(aio);
+}
+
+void
+test_xrep_send_no_header(void)
+{
+	nng_socket rep;
+	nng_socket req;
+	nng_msg *  m;
+
+	TEST_NNG_PASS(nng_req0_open_raw(&req));
+	TEST_NNG_PASS(nng_rep0_open_raw(&rep));
+	TEST_NNG_PASS(nng_setopt_ms(rep, NNG_OPT_RECVTIMEO, 100));
+	TEST_NNG_PASS(nng_setopt_ms(req, NNG_OPT_RECVTIMEO, 100));
+	TEST_NNG_PASS(nng_setopt_ms(rep, NNG_OPT_SENDTIMEO, 1000));
+	TEST_NNG_PASS(nng_setopt_ms(req, NNG_OPT_SENDTIMEO, 1000));
+
+	TEST_NNG_PASS(testutil_marry(req, rep));
+
+	TEST_NNG_PASS(nng_msg_alloc(&m, 4));
+	TEST_NNG_PASS(nng_sendmsg(rep, m, 0));
+	TEST_NNG_FAIL(nng_recvmsg(rep, &m, 0), NNG_ETIMEDOUT);
+
+	TEST_NNG_PASS(nng_close(req));
+	TEST_NNG_PASS(nng_close(rep));
+}
+
+void
+test_xrep_recv_garbage(void)
+{
+	nng_socket rep;
+	nng_socket req;
+	nng_msg *  m;
+
+	TEST_NNG_PASS(nng_rep0_open_raw(&rep));
+	TEST_NNG_PASS(nng_req0_open_raw(&req));
+	TEST_NNG_PASS(nng_setopt_ms(rep, NNG_OPT_RECVTIMEO, 100));
+	TEST_NNG_PASS(nng_setopt_ms(rep, NNG_OPT_SENDTIMEO, 100));
+	TEST_NNG_PASS(nng_setopt_ms(req, NNG_OPT_SENDTIMEO, 1000));
+
+	TEST_NNG_PASS(testutil_marry(req, rep));
+
+	TEST_NNG_PASS(nng_msg_alloc(&m, 4));
+	TEST_NNG_PASS(nng_msg_append_u32(m, 1u));
+	TEST_NNG_PASS(nng_sendmsg(req, m, 0));
+	TEST_NNG_FAIL(nng_recvmsg(rep, &m, 0), NNG_ETIMEDOUT);
+
+	TEST_NNG_PASS(nng_close(req));
+	TEST_NNG_PASS(nng_close(rep));
+}
+
+void
+test_xrep_ttl_option(void)
+{
+	nng_socket  rep;
+	int         v;
+	bool        b;
+	size_t      sz  = sizeof(v);
+	const char *opt = NNG_OPT_MAXTTL;
+
+	TEST_NNG_PASS(nng_rep0_open_raw(&rep));
+
+	TEST_NNG_PASS(nng_setopt_int(rep, opt, 1));
+	TEST_NNG_FAIL(nng_setopt_int(rep, opt, 0), NNG_EINVAL);
+	TEST_NNG_FAIL(nng_setopt_int(rep, opt, -1), NNG_EINVAL);
+	TEST_NNG_FAIL(nng_setopt_int(rep, opt, 256), NNG_EINVAL);
+	TEST_NNG_PASS(nng_setopt_int(rep, opt, 3));
+	TEST_NNG_PASS(nng_getopt_int(rep, opt, &v));
+	TEST_CHECK(v == 3);
+	v  = 0;
+	sz = sizeof(v);
+	TEST_NNG_PASS(nng_getopt(rep, opt, &v, &sz));
+	TEST_CHECK(v == 3);
+	TEST_CHECK(sz == sizeof(v));
+
+	TEST_CHECK(nng_setopt(rep, opt, "", 1) == NNG_EINVAL);
+	sz = 1;
+	TEST_CHECK(nng_getopt(rep, opt, &v, &sz) == NNG_EINVAL);
+	TEST_CHECK(nng_setopt_bool(rep, opt, true) == NNG_EBADTYPE);
+	TEST_CHECK(nng_getopt_bool(rep, opt, &b) == NNG_EBADTYPE);
+
+	TEST_CHECK(nng_close(rep) == 0);
+}
+
+void
+test_xrep_ttl_drop(void)
+{
+	nng_socket rep;
+	nng_socket req;
+	nng_msg *  m;
+
+	TEST_NNG_PASS(nng_rep0_open_raw(&rep));
+	TEST_NNG_PASS(nng_req0_open_raw(&req));
+	TEST_NNG_PASS(nng_setopt_int(rep, NNG_OPT_MAXTTL, 3));
+	TEST_NNG_PASS(nng_setopt_ms(rep, NNG_OPT_RECVTIMEO, 200));
+	TEST_NNG_PASS(nng_setopt_ms(req, NNG_OPT_SENDTIMEO, 1000));
+
+	TEST_NNG_PASS(testutil_marry(req, rep));
+
+	// Send messages.  Note that xrep implicitly adds a hop on receive.
+
+	TEST_NNG_PASS(nng_msg_alloc(&m, 0));
+	TEST_NNG_PASS(nng_msg_append_u32(m, 1u)); // 2 hops
+	TEST_NNG_PASS(nng_msg_append_u32(m, 0x80000001u));
+	TEST_NNG_PASS(nng_msg_append(m, "PASS1", 6));
+	TEST_NNG_PASS(nng_sendmsg(req, m, 0));
+
+	TEST_NNG_PASS(nng_msg_alloc(&m, 0));
+	TEST_NNG_PASS(nng_msg_append_u32(m, 1u)); // 4 hops -- discard!
+	TEST_NNG_PASS(nng_msg_append_u32(m, 2u));
+	TEST_NNG_PASS(nng_msg_append_u32(m, 3u));
+	TEST_NNG_PASS(nng_msg_append_u32(m, 0x80000002u));
+	TEST_NNG_PASS(nng_msg_append(m, "FAIL2", 6));
+	TEST_NNG_PASS(nng_sendmsg(req, m, 0));
+
+	TEST_NNG_PASS(nng_msg_alloc(&m, 0));
+	TEST_NNG_PASS(nng_msg_append_u32(m, 1u)); // 3 hops - passes
+	TEST_NNG_PASS(nng_msg_append_u32(m, 2u));
+	TEST_NNG_PASS(nng_msg_append_u32(m, 0x80000003u));
+	TEST_NNG_PASS(nng_msg_append(m, "PASS3", 6));
+	TEST_NNG_PASS(nng_sendmsg(req, m, 0));
+
+	TEST_NNG_PASS(nng_msg_alloc(&m, 0));
+	TEST_NNG_PASS(nng_msg_append_u32(m, 1u)); // 4 hops -- discard!
+	TEST_NNG_PASS(nng_msg_append_u32(m, 2u));
+	TEST_NNG_PASS(nng_msg_append_u32(m, 3u));
+	TEST_NNG_PASS(nng_msg_append_u32(m, 0x80000003u));
+	TEST_NNG_PASS(nng_msg_append(m, "FAIL4", 6));
+	TEST_NNG_PASS(nng_sendmsg(req, m, 0));
+
+	// So on receive we should see 80000001 and 80000003.
+	TEST_NNG_PASS(nng_recvmsg(rep, &m, 0));
+	TEST_CHECK(nng_msg_header_len(m) == 12);
+	TEST_CHECK(nng_msg_len(m) == 6);
+	TEST_CHECK(strcmp(nng_msg_body(m), "PASS1") == 0);
+	nng_msg_free(m);
+
+	TEST_NNG_PASS(nng_recvmsg(rep, &m, 0));
+	TEST_CHECK(nng_msg_header_len(m) == 16); // 3 hops + ID
+	TEST_CHECK(nng_msg_len(m) == 6);
+	TEST_CHECK(strcmp(nng_msg_body(m), "PASS3") == 0);
+	nng_msg_free(m);
+
+	TEST_NNG_FAIL(nng_recvmsg(rep, &m, 0), NNG_ETIMEDOUT);
+
+	TEST_NNG_PASS(nng_close(req));
+	TEST_NNG_PASS(nng_close(rep));
+}
+
+TEST_LIST = {
+	{ "xrep identity", test_xrep_identity },
+	{ "xrep raw", test_xrep_raw },
+	{ "xrep no context", test_xrep_no_context },
+	{ "xrep poll readable", test_xrep_poll_readable },
+	{ "xrep poll writable", test_xrep_poll_writeable },
+	{ "xrep validate peer", test_xrep_validate_peer },
+	{ "xrep close pipe before send", test_xrep_close_pipe_before_send },
+	{ "xrep close pipe during send", test_xrep_close_pipe_during_send },
+	{ "xrep close during recv", test_xrep_close_during_recv },
+	{ "xrep recv aio stopped", test_xrep_recv_aio_stopped },
+	{ "xrep send no header", test_xrep_send_no_header },
+	{ "xrep recv garbage", test_xrep_recv_garbage },
+	{ "xrep ttl option", test_xrep_ttl_option },
+	{ "xrep ttl drop", test_xrep_ttl_drop },
+	{ NULL, NULL },
+};