From ed542ac45e00c9b2faa0b41f3c00de6e291e5678 Mon Sep 17 00:00:00 2001 From: Garrett D'Amore Date: Fri, 1 Jan 2021 11:30:03 -0800 Subject: fixes #1345 Restructure the source tree This is not quite complete, but it sets the stage for other protocols (such as zmq or mqtt) to be added to the project. --- src/CMakeLists.txt | 19 +- src/compat/CMakeLists.txt | 12 + src/protocol/CMakeLists.txt | 20 - src/protocol/bus0/CMakeLists.txt | 18 - src/protocol/bus0/bug1247_test.c | 35 - src/protocol/bus0/bus.c | 466 ---- src/protocol/pair0/CMakeLists.txt | 16 - src/protocol/pair0/pair.c | 305 --- src/protocol/pair1/CMakeLists.txt | 20 - src/protocol/pair1/pair.c | 540 ----- src/protocol/pair1/pair1_poly.c | 535 ----- src/protocol/pair1/pair1_poly_test.c | 370 ---- src/protocol/pair1/pair1_test.c | 433 ---- src/protocol/pipeline0/CMakeLists.txt | 23 - src/protocol/pipeline0/pull.c | 325 --- src/protocol/pipeline0/pull_test.c | 264 --- src/protocol/pipeline0/push.c | 442 ---- src/protocol/pipeline0/push_test.c | 525 ----- src/protocol/pubsub0/CMakeLists.txt | 24 - src/protocol/pubsub0/pub.c | 383 ---- src/protocol/pubsub0/pub_test.c | 331 --- src/protocol/pubsub0/sub.c | 755 ------- src/protocol/pubsub0/sub_test.c | 624 ------ src/protocol/pubsub0/xsub.c | 211 -- src/protocol/pubsub0/xsub_test.c | 376 ---- src/protocol/reqrep0/CMakeLists.txt | 25 - src/protocol/reqrep0/rep.c | 705 ------ src/protocol/reqrep0/rep_test.c | 669 ------ src/protocol/reqrep0/req.c | 869 -------- src/protocol/reqrep0/req_test.c | 968 --------- src/protocol/reqrep0/xrep.c | 432 ---- src/protocol/reqrep0/xrep_test.c | 434 ---- src/protocol/reqrep0/xreq.c | 319 --- src/protocol/reqrep0/xreq_test.c | 367 ---- src/protocol/survey0/CMakeLists.txt | 25 - src/protocol/survey0/respond.c | 693 ------ src/protocol/survey0/respond_test.c | 586 ----- src/protocol/survey0/survey.c | 663 ------ src/protocol/survey0/survey_test.c | 626 ------ src/protocol/survey0/xrespond.c | 417 ---- src/protocol/survey0/xrespond_test.c | 436 ---- src/protocol/survey0/xsurvey.c | 379 ---- src/protocol/survey0/xsurvey_test.c | 399 ---- src/sp/CMakeLists.txt | 13 + src/sp/protocol/CMakeLists.txt | 20 + src/sp/protocol/bus0/CMakeLists.txt | 18 + src/sp/protocol/bus0/bug1247_test.c | 35 + src/sp/protocol/bus0/bus.c | 466 ++++ src/sp/protocol/pair0/CMakeLists.txt | 16 + src/sp/protocol/pair0/pair.c | 305 +++ src/sp/protocol/pair1/CMakeLists.txt | 20 + src/sp/protocol/pair1/pair.c | 540 +++++ src/sp/protocol/pair1/pair1_poly.c | 535 +++++ src/sp/protocol/pair1/pair1_poly_test.c | 370 ++++ src/sp/protocol/pair1/pair1_test.c | 433 ++++ src/sp/protocol/pipeline0/CMakeLists.txt | 23 + src/sp/protocol/pipeline0/pull.c | 325 +++ src/sp/protocol/pipeline0/pull_test.c | 264 +++ src/sp/protocol/pipeline0/push.c | 442 ++++ src/sp/protocol/pipeline0/push_test.c | 525 +++++ src/sp/protocol/pubsub0/CMakeLists.txt | 24 + src/sp/protocol/pubsub0/pub.c | 383 ++++ src/sp/protocol/pubsub0/pub_test.c | 331 +++ src/sp/protocol/pubsub0/sub.c | 755 +++++++ src/sp/protocol/pubsub0/sub_test.c | 624 ++++++ src/sp/protocol/pubsub0/xsub.c | 211 ++ src/sp/protocol/pubsub0/xsub_test.c | 376 ++++ src/sp/protocol/reqrep0/CMakeLists.txt | 25 + src/sp/protocol/reqrep0/rep.c | 705 ++++++ src/sp/protocol/reqrep0/rep_test.c | 669 ++++++ src/sp/protocol/reqrep0/req.c | 869 ++++++++ src/sp/protocol/reqrep0/req_test.c | 968 +++++++++ src/sp/protocol/reqrep0/xrep.c | 432 ++++ src/sp/protocol/reqrep0/xrep_test.c | 434 ++++ src/sp/protocol/reqrep0/xreq.c | 319 +++ src/sp/protocol/reqrep0/xreq_test.c | 367 ++++ src/sp/protocol/survey0/CMakeLists.txt | 25 + src/sp/protocol/survey0/respond.c | 693 ++++++ src/sp/protocol/survey0/respond_test.c | 586 +++++ src/sp/protocol/survey0/survey.c | 663 ++++++ src/sp/protocol/survey0/survey_test.c | 626 ++++++ src/sp/protocol/survey0/xrespond.c | 417 ++++ src/sp/protocol/survey0/xrespond_test.c | 436 ++++ src/sp/protocol/survey0/xsurvey.c | 379 ++++ src/sp/protocol/survey0/xsurvey_test.c | 399 ++++ src/sp/transport/CMakeLists.txt | 19 + src/sp/transport/inproc/CMakeLists.txt | 16 + src/sp/transport/inproc/inproc.c | 692 ++++++ src/sp/transport/ipc/CMakeLists.txt | 17 + src/sp/transport/ipc/ipc.c | 1171 ++++++++++ src/sp/transport/ipc/ipc_test.c | 395 ++++ src/sp/transport/tcp/CMakeLists.txt | 17 + src/sp/transport/tcp/tcp.c | 1263 +++++++++++ src/sp/transport/tcp/tcp_test.c | 297 +++ src/sp/transport/tls/CMakeLists.txt | 16 + src/sp/transport/tls/tls.c | 1292 +++++++++++ src/sp/transport/ws/CMakeLists.txt | 24 + src/sp/transport/ws/README.adoc | 38 + src/sp/transport/ws/websocket.c | 740 +++++++ src/sp/transport/ws/ws_test.c | 181 ++ src/sp/transport/zerotier/CMakeLists.txt | 37 + src/sp/transport/zerotier/zerotier.c | 3241 ++++++++++++++++++++++++++++ src/sp/transport/zerotier/zthash.c | 302 +++ src/sp/transport/zerotier/zthash.h | 43 + src/supplemental/CMakeLists.txt | 17 + src/tools/CMakeLists.txt | 13 + src/tools/nngcat/CMakeLists.txt | 41 + src/tools/nngcat/nngcat.c | 1217 +++++++++++ src/tools/nngcat/nngcat_ambiguous_test.sh | 31 + src/tools/nngcat/nngcat_async_test.sh | 32 + src/tools/nngcat/nngcat_dup_proto_test.sh | 23 + src/tools/nngcat/nngcat_help_test.sh | 32 + src/tools/nngcat/nngcat_incompat_test.sh | 73 + src/tools/nngcat/nngcat_need_proto_test.sh | 23 + src/tools/nngcat/nngcat_pubsub_test.sh | 45 + src/tools/nngcat/nngcat_recvmaxsz_test.sh | 46 + src/tools/nngcat/nngcat_stdin_pipe_test.sh | 44 + src/tools/nngcat/nngcat_unlimited_test.sh | 46 + src/tools/perf/CMakeLists.txt | 35 + src/tools/perf/perf.c | 662 ++++++ src/tools/perf/pubdrop.c | 325 +++ src/transport/CMakeLists.txt | 19 - src/transport/inproc/CMakeLists.txt | 16 - src/transport/inproc/inproc.c | 692 ------ src/transport/ipc/CMakeLists.txt | 17 - src/transport/ipc/ipc.c | 1171 ---------- src/transport/ipc/ipc_test.c | 395 ---- src/transport/tcp/CMakeLists.txt | 17 - src/transport/tcp/tcp.c | 1263 ----------- src/transport/tcp/tcp_test.c | 297 --- src/transport/tls/CMakeLists.txt | 16 - src/transport/tls/tls.c | 1292 ----------- src/transport/ws/CMakeLists.txt | 24 - src/transport/ws/README.adoc | 38 - src/transport/ws/websocket.c | 740 ------- src/transport/ws/ws_test.c | 181 -- src/transport/zerotier/CMakeLists.txt | 37 - src/transport/zerotier/zerotier.c | 3241 ---------------------------- src/transport/zerotier/zthash.c | 302 --- src/transport/zerotier/zthash.h | 43 - 140 files changed, 28589 insertions(+), 25868 deletions(-) create mode 100644 src/compat/CMakeLists.txt delete mode 100644 src/protocol/CMakeLists.txt delete mode 100644 src/protocol/bus0/CMakeLists.txt delete mode 100644 src/protocol/bus0/bug1247_test.c delete mode 100644 src/protocol/bus0/bus.c delete mode 100644 src/protocol/pair0/CMakeLists.txt delete mode 100644 src/protocol/pair0/pair.c delete mode 100644 src/protocol/pair1/CMakeLists.txt delete mode 100644 src/protocol/pair1/pair.c delete mode 100644 src/protocol/pair1/pair1_poly.c delete mode 100644 src/protocol/pair1/pair1_poly_test.c delete mode 100644 src/protocol/pair1/pair1_test.c delete mode 100644 src/protocol/pipeline0/CMakeLists.txt delete mode 100644 src/protocol/pipeline0/pull.c delete mode 100644 src/protocol/pipeline0/pull_test.c delete mode 100644 src/protocol/pipeline0/push.c delete mode 100644 src/protocol/pipeline0/push_test.c delete mode 100644 src/protocol/pubsub0/CMakeLists.txt delete mode 100644 src/protocol/pubsub0/pub.c delete mode 100644 src/protocol/pubsub0/pub_test.c delete mode 100644 src/protocol/pubsub0/sub.c delete mode 100644 src/protocol/pubsub0/sub_test.c delete mode 100644 src/protocol/pubsub0/xsub.c delete mode 100644 src/protocol/pubsub0/xsub_test.c delete mode 100644 src/protocol/reqrep0/CMakeLists.txt delete mode 100644 src/protocol/reqrep0/rep.c delete mode 100644 src/protocol/reqrep0/rep_test.c delete mode 100644 src/protocol/reqrep0/req.c delete mode 100644 src/protocol/reqrep0/req_test.c delete mode 100644 src/protocol/reqrep0/xrep.c delete mode 100644 src/protocol/reqrep0/xrep_test.c delete mode 100644 src/protocol/reqrep0/xreq.c delete mode 100644 src/protocol/reqrep0/xreq_test.c delete mode 100644 src/protocol/survey0/CMakeLists.txt delete mode 100644 src/protocol/survey0/respond.c delete mode 100644 src/protocol/survey0/respond_test.c delete mode 100644 src/protocol/survey0/survey.c delete mode 100644 src/protocol/survey0/survey_test.c delete mode 100644 src/protocol/survey0/xrespond.c delete mode 100644 src/protocol/survey0/xrespond_test.c delete mode 100644 src/protocol/survey0/xsurvey.c delete mode 100644 src/protocol/survey0/xsurvey_test.c create mode 100644 src/sp/CMakeLists.txt create mode 100644 src/sp/protocol/CMakeLists.txt create mode 100644 src/sp/protocol/bus0/CMakeLists.txt create mode 100644 src/sp/protocol/bus0/bug1247_test.c create mode 100644 src/sp/protocol/bus0/bus.c create mode 100644 src/sp/protocol/pair0/CMakeLists.txt create mode 100644 src/sp/protocol/pair0/pair.c create mode 100644 src/sp/protocol/pair1/CMakeLists.txt create mode 100644 src/sp/protocol/pair1/pair.c create mode 100644 src/sp/protocol/pair1/pair1_poly.c create mode 100644 src/sp/protocol/pair1/pair1_poly_test.c create mode 100644 src/sp/protocol/pair1/pair1_test.c create mode 100644 src/sp/protocol/pipeline0/CMakeLists.txt create mode 100644 src/sp/protocol/pipeline0/pull.c create mode 100644 src/sp/protocol/pipeline0/pull_test.c create mode 100644 src/sp/protocol/pipeline0/push.c create mode 100644 src/sp/protocol/pipeline0/push_test.c create mode 100644 src/sp/protocol/pubsub0/CMakeLists.txt create mode 100644 src/sp/protocol/pubsub0/pub.c create mode 100644 src/sp/protocol/pubsub0/pub_test.c create mode 100644 src/sp/protocol/pubsub0/sub.c create mode 100644 src/sp/protocol/pubsub0/sub_test.c create mode 100644 src/sp/protocol/pubsub0/xsub.c create mode 100644 src/sp/protocol/pubsub0/xsub_test.c create mode 100644 src/sp/protocol/reqrep0/CMakeLists.txt create mode 100644 src/sp/protocol/reqrep0/rep.c create mode 100644 src/sp/protocol/reqrep0/rep_test.c create mode 100644 src/sp/protocol/reqrep0/req.c create mode 100644 src/sp/protocol/reqrep0/req_test.c create mode 100644 src/sp/protocol/reqrep0/xrep.c create mode 100644 src/sp/protocol/reqrep0/xrep_test.c create mode 100644 src/sp/protocol/reqrep0/xreq.c create mode 100644 src/sp/protocol/reqrep0/xreq_test.c create mode 100644 src/sp/protocol/survey0/CMakeLists.txt create mode 100644 src/sp/protocol/survey0/respond.c create mode 100644 src/sp/protocol/survey0/respond_test.c create mode 100644 src/sp/protocol/survey0/survey.c create mode 100644 src/sp/protocol/survey0/survey_test.c create mode 100644 src/sp/protocol/survey0/xrespond.c create mode 100644 src/sp/protocol/survey0/xrespond_test.c create mode 100644 src/sp/protocol/survey0/xsurvey.c create mode 100644 src/sp/protocol/survey0/xsurvey_test.c create mode 100644 src/sp/transport/CMakeLists.txt create mode 100644 src/sp/transport/inproc/CMakeLists.txt create mode 100644 src/sp/transport/inproc/inproc.c create mode 100644 src/sp/transport/ipc/CMakeLists.txt create mode 100644 src/sp/transport/ipc/ipc.c create mode 100644 src/sp/transport/ipc/ipc_test.c create mode 100644 src/sp/transport/tcp/CMakeLists.txt create mode 100644 src/sp/transport/tcp/tcp.c create mode 100644 src/sp/transport/tcp/tcp_test.c create mode 100644 src/sp/transport/tls/CMakeLists.txt create mode 100644 src/sp/transport/tls/tls.c create mode 100644 src/sp/transport/ws/CMakeLists.txt create mode 100644 src/sp/transport/ws/README.adoc create mode 100644 src/sp/transport/ws/websocket.c create mode 100644 src/sp/transport/ws/ws_test.c create mode 100644 src/sp/transport/zerotier/CMakeLists.txt create mode 100644 src/sp/transport/zerotier/zerotier.c create mode 100644 src/sp/transport/zerotier/zthash.c create mode 100644 src/sp/transport/zerotier/zthash.h create mode 100644 src/supplemental/CMakeLists.txt create mode 100644 src/tools/CMakeLists.txt create mode 100644 src/tools/nngcat/CMakeLists.txt create mode 100644 src/tools/nngcat/nngcat.c create mode 100755 src/tools/nngcat/nngcat_ambiguous_test.sh create mode 100755 src/tools/nngcat/nngcat_async_test.sh create mode 100755 src/tools/nngcat/nngcat_dup_proto_test.sh create mode 100755 src/tools/nngcat/nngcat_help_test.sh create mode 100755 src/tools/nngcat/nngcat_incompat_test.sh create mode 100755 src/tools/nngcat/nngcat_need_proto_test.sh create mode 100755 src/tools/nngcat/nngcat_pubsub_test.sh create mode 100755 src/tools/nngcat/nngcat_recvmaxsz_test.sh create mode 100755 src/tools/nngcat/nngcat_stdin_pipe_test.sh create mode 100755 src/tools/nngcat/nngcat_unlimited_test.sh create mode 100644 src/tools/perf/CMakeLists.txt create mode 100644 src/tools/perf/perf.c create mode 100644 src/tools/perf/pubdrop.c delete mode 100644 src/transport/CMakeLists.txt delete mode 100644 src/transport/inproc/CMakeLists.txt delete mode 100644 src/transport/inproc/inproc.c delete mode 100644 src/transport/ipc/CMakeLists.txt delete mode 100644 src/transport/ipc/ipc.c delete mode 100644 src/transport/ipc/ipc_test.c delete mode 100644 src/transport/tcp/CMakeLists.txt delete mode 100644 src/transport/tcp/tcp.c delete mode 100644 src/transport/tcp/tcp_test.c delete mode 100644 src/transport/tls/CMakeLists.txt delete mode 100644 src/transport/tls/tls.c delete mode 100644 src/transport/ws/CMakeLists.txt delete mode 100644 src/transport/ws/README.adoc delete mode 100644 src/transport/ws/websocket.c delete mode 100644 src/transport/ws/ws_test.c delete mode 100644 src/transport/zerotier/CMakeLists.txt delete mode 100644 src/transport/zerotier/zerotier.c delete mode 100644 src/transport/zerotier/zthash.c delete mode 100644 src/transport/zerotier/zthash.h (limited to 'src') diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 57679309..20a7bef0 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,5 +1,5 @@ # -# Copyright 2020 Staysail Systems, Inc. +# Copyright 2021 Staysail Systems, Inc. # Copyright 2018 Capitar IT Group BV # # This software is supplied under the terms of the MIT License, a @@ -15,20 +15,11 @@ target_include_directories(nng PRIVATE ${PROJECT_SOURCE_DIR}/src) target_include_directories(nng_testing PRIVATE ${PROJECT_SOURCE_DIR}/src) add_subdirectory(core) - add_subdirectory(platform) -add_subdirectory(compat/nanomsg) - -add_subdirectory(protocol) -add_subdirectory(transport) - -add_subdirectory(supplemental/base64) -add_subdirectory(supplemental/http) -add_subdirectory(supplemental/sha1) -add_subdirectory(supplemental/tls) -add_subdirectory(supplemental/util) -add_subdirectory(supplemental/websocket) - +add_subdirectory(compat) +add_subdirectory(sp) +add_subdirectory(supplemental) +add_subdirectory(tools) add_subdirectory(testing) # When building shared libraries we prefer to suppress default symbol diff --git a/src/compat/CMakeLists.txt b/src/compat/CMakeLists.txt new file mode 100644 index 00000000..bf685e90 --- /dev/null +++ b/src/compat/CMakeLists.txt @@ -0,0 +1,12 @@ +# +# Copyright 2021 Staysail Systems, Inc. +# +# This software is supplied under the terms of the MIT License, a +# copy of which should be located in the distribution where this +# file was obtained (LICENSE.txt). A copy of the license may also be +# found online at https://opensource.org/licenses/MIT. +# + +nng_directory(compat) + +add_subdirectory(nanomsg) diff --git a/src/protocol/CMakeLists.txt b/src/protocol/CMakeLists.txt deleted file mode 100644 index fd480523..00000000 --- a/src/protocol/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ -# -# Copyright 2020 Staysail Systems, Inc. -# -# This software is supplied under the terms of the MIT License, a -# copy of which should be located in the distribution where this -# file was obtained (LICENSE.txt). A copy of the license may also be -# found online at https://opensource.org/licenses/MIT. -# - -# Protocols. -nng_directory(protocol) - -add_subdirectory(bus0) -add_subdirectory(pair0) -add_subdirectory(pair1) -add_subdirectory(pipeline0) -add_subdirectory(pubsub0) -add_subdirectory(reqrep0) -add_subdirectory(survey0) - diff --git a/src/protocol/bus0/CMakeLists.txt b/src/protocol/bus0/CMakeLists.txt deleted file mode 100644 index 01c0b05b..00000000 --- a/src/protocol/bus0/CMakeLists.txt +++ /dev/null @@ -1,18 +0,0 @@ -# -# Copyright 2020 Staysail Systems, Inc. -# Copyright 2018 Capitar IT Group BV -# -# This software is supplied under the terms of the MIT License, a -# copy of which should be located in the distribution where this -# file was obtained (LICENSE.txt). A copy of the license may also be -# found online at https://opensource.org/licenses/MIT. -# - -# Bus protocol -nng_directory(bus0) - -nng_sources_if(NNG_PROTO_BUS0 bus.c) -nng_headers_if(NNG_PROTO_BUS0 nng/protocol/bus0/bus.h) -nng_defines_if(NNG_PROTO_BUS0 NNG_HAVE_BUS0) - -nng_test(bug1247_test) \ No newline at end of file diff --git a/src/protocol/bus0/bug1247_test.c b/src/protocol/bus0/bug1247_test.c deleted file mode 100644 index bbc6958b..00000000 --- a/src/protocol/bus0/bug1247_test.c +++ /dev/null @@ -1,35 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - -#include - -#include - -void -test_bug1247(void) -{ - nng_socket bus1, bus2; - char * addr; - - NUTS_ADDR(addr, "tcp"); - - NUTS_PASS(nng_bus0_open(&bus1)); - NUTS_PASS(nng_bus0_open(&bus2)); - - NUTS_PASS(nng_listen(bus1, addr, NULL, 0)); - NUTS_FAIL(nng_listen(bus2, addr, NULL, 0), NNG_EADDRINUSE); - - NUTS_PASS(nng_close(bus2)); - NUTS_PASS(nng_close(bus1)); -} - -TEST_LIST = { - { "bug1247", test_bug1247 }, - { NULL, NULL }, -}; diff --git a/src/protocol/bus0/bus.c b/src/protocol/bus0/bus.c deleted file mode 100644 index 9a610ac6..00000000 --- a/src/protocol/bus0/bus.c +++ /dev/null @@ -1,466 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// Copyright 2018 Capitar IT Group BV -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - -#include -#include - -#include "core/nng_impl.h" -#include "nng/protocol/bus0/bus.h" - -// Bus protocol. The BUS protocol, each peer sends a message to its peers. -// However, bus protocols do not "forward" (absent a device). So in order -// for each participant to receive the message, each sender must be connected -// to every other node in the network (full mesh). - -#ifndef NNI_PROTO_BUS_V0 -#define NNI_PROTO_BUS_V0 NNI_PROTO(7, 0) -#endif - -typedef struct bus0_pipe bus0_pipe; -typedef struct bus0_sock bus0_sock; - -static void bus0_sock_getq(bus0_sock *); -static void bus0_sock_send(void *, nni_aio *); -static void bus0_sock_recv(void *, nni_aio *); - -static void bus0_pipe_getq(bus0_pipe *); -static void bus0_pipe_recv(bus0_pipe *); - -static void bus0_sock_getq_cb(void *); -static void bus0_sock_getq_cb_raw(void *); -static void bus0_pipe_getq_cb(void *); -static void bus0_pipe_send_cb(void *); -static void bus0_pipe_recv_cb(void *); -static void bus0_pipe_putq_cb(void *); - -// bus0_sock is our per-socket protocol private structure. -struct bus0_sock { - nni_aio * aio_getq; - nni_list pipes; - nni_mtx mtx; - nni_msgq *uwq; - nni_msgq *urq; - bool raw; -}; - -// bus0_pipe is our per-pipe protocol private structure. -struct bus0_pipe { - nni_pipe * npipe; - bus0_sock * psock; - nni_msgq * sendq; - nni_list_node node; - nni_aio * aio_getq; - nni_aio * aio_recv; - nni_aio * aio_send; - nni_aio * aio_putq; - nni_mtx mtx; -}; - -static void -bus0_sock_fini(void *arg) -{ - bus0_sock *s = arg; - - nni_aio_free(s->aio_getq); - nni_mtx_fini(&s->mtx); -} - -static int -bus0_sock_init(void *arg, nni_sock *nsock) -{ - bus0_sock *s = arg; - int rv; - - NNI_LIST_INIT(&s->pipes, bus0_pipe, node); - nni_mtx_init(&s->mtx); - if ((rv = nni_aio_alloc(&s->aio_getq, bus0_sock_getq_cb, s)) != 0) { - bus0_sock_fini(s); - return (rv); - } - s->uwq = nni_sock_sendq(nsock); - s->urq = nni_sock_recvq(nsock); - s->raw = false; - - return (0); -} - -static int -bus0_sock_init_raw(void *arg, nni_sock *nsock) -{ - bus0_sock *s = arg; - int rv; - - NNI_LIST_INIT(&s->pipes, bus0_pipe, node); - nni_mtx_init(&s->mtx); - if ((rv = nni_aio_alloc(&s->aio_getq, bus0_sock_getq_cb_raw, s)) != - 0) { - bus0_sock_fini(s); - return (rv); - } - s->uwq = nni_sock_sendq(nsock); - s->urq = nni_sock_recvq(nsock); - s->raw = true; - - return (0); -} - -static void -bus0_sock_open(void *arg) -{ - bus0_sock *s = arg; - - bus0_sock_getq(s); -} - -static void -bus0_sock_close(void *arg) -{ - bus0_sock *s = arg; - - nni_aio_close(s->aio_getq); -} - -static void -bus0_pipe_stop(void *arg) -{ - bus0_pipe *p = arg; - - nni_aio_stop(p->aio_getq); - nni_aio_stop(p->aio_send); - nni_aio_stop(p->aio_recv); - nni_aio_stop(p->aio_putq); -} - -static void -bus0_pipe_fini(void *arg) -{ - bus0_pipe *p = arg; - - nni_aio_free(p->aio_getq); - nni_aio_free(p->aio_send); - nni_aio_free(p->aio_recv); - nni_aio_free(p->aio_putq); - nni_msgq_fini(p->sendq); - nni_mtx_fini(&p->mtx); -} - -static int -bus0_pipe_init(void *arg, nni_pipe *npipe, void *s) -{ - bus0_pipe *p = arg; - int rv; - - NNI_LIST_NODE_INIT(&p->node); - nni_mtx_init(&p->mtx); - if (((rv = nni_msgq_init(&p->sendq, 16)) != 0) || - ((rv = nni_aio_alloc(&p->aio_getq, bus0_pipe_getq_cb, p)) != 0) || - ((rv = nni_aio_alloc(&p->aio_send, bus0_pipe_send_cb, p)) != 0) || - ((rv = nni_aio_alloc(&p->aio_recv, bus0_pipe_recv_cb, p)) != 0) || - ((rv = nni_aio_alloc(&p->aio_putq, bus0_pipe_putq_cb, p)) != 0)) { - bus0_pipe_fini(p); - return (rv); - } - - p->npipe = npipe; - p->psock = s; - return (0); -} - -static int -bus0_pipe_start(void *arg) -{ - bus0_pipe *p = arg; - bus0_sock *s = p->psock; - - if (nni_pipe_peer(p->npipe) != NNI_PROTO_BUS_V0) { - // Peer protocol mismatch. - return (NNG_EPROTO); - } - - nni_mtx_lock(&s->mtx); - nni_list_append(&s->pipes, p); - nni_mtx_unlock(&s->mtx); - - bus0_pipe_recv(p); - bus0_pipe_getq(p); - - return (0); -} - -static void -bus0_pipe_close(void *arg) -{ - bus0_pipe *p = arg; - bus0_sock *s = p->psock; - - nni_aio_close(p->aio_getq); - nni_aio_close(p->aio_send); - nni_aio_close(p->aio_recv); - nni_aio_close(p->aio_putq); - nni_msgq_close(p->sendq); - - nni_mtx_lock(&s->mtx); - if (nni_list_active(&s->pipes, p)) { - nni_list_remove(&s->pipes, p); - } - nni_mtx_unlock(&s->mtx); -} - -static void -bus0_pipe_getq_cb(void *arg) -{ - bus0_pipe *p = arg; - - if (nni_aio_result(p->aio_getq) != 0) { - // closed? - nni_pipe_close(p->npipe); - return; - } - nni_aio_set_msg(p->aio_send, nni_aio_get_msg(p->aio_getq)); - nni_aio_set_msg(p->aio_getq, NULL); - - nni_pipe_send(p->npipe, p->aio_send); -} - -static void -bus0_pipe_send_cb(void *arg) -{ - bus0_pipe *p = arg; - - if (nni_aio_result(p->aio_send) != 0) { - // closed? - nni_msg_free(nni_aio_get_msg(p->aio_send)); - nni_aio_set_msg(p->aio_send, NULL); - nni_pipe_close(p->npipe); - return; - } - - bus0_pipe_getq(p); -} - -static void -bus0_pipe_recv_cb(void *arg) -{ - bus0_pipe *p = arg; - bus0_sock *s = p->psock; - nni_msg * msg; - - if (nni_aio_result(p->aio_recv) != 0) { - nni_pipe_close(p->npipe); - return; - } - msg = nni_aio_get_msg(p->aio_recv); - - if (s->raw) { - nni_msg_header_append_u32(msg, nni_pipe_id(p->npipe)); - } - - nni_msg_set_pipe(msg, nni_pipe_id(p->npipe)); - nni_aio_set_msg(p->aio_putq, msg); - nni_aio_set_msg(p->aio_recv, NULL); - nni_msgq_aio_put(s->urq, p->aio_putq); -} - -static void -bus0_pipe_putq_cb(void *arg) -{ - bus0_pipe *p = arg; - - if (nni_aio_result(p->aio_putq) != 0) { - nni_msg_free(nni_aio_get_msg(p->aio_putq)); - nni_aio_set_msg(p->aio_putq, NULL); - nni_pipe_close(p->npipe); - return; - } - - // Wait for another recv. - bus0_pipe_recv(p); -} - -static void -bus0_sock_getq_cb(void *arg) -{ - bus0_sock *s = arg; - bus0_pipe *p; - bus0_pipe *lastp; - nni_msg * msg; - nni_msg * dup; - - if (nni_aio_result(s->aio_getq) != 0) { - return; - } - - msg = nni_aio_get_msg(s->aio_getq); - - // We ignore any headers present for cooked mode. - nni_msg_header_clear(msg); - - nni_mtx_lock(&s->mtx); - lastp = nni_list_last(&s->pipes); - NNI_LIST_FOREACH (&s->pipes, p) { - if (p != lastp) { - if (nni_msg_dup(&dup, msg) != 0) { - continue; - } - } else { - dup = msg; - msg = NULL; - } - if (nni_msgq_tryput(p->sendq, dup) != 0) { - nni_msg_free(dup); - } - } - nni_mtx_unlock(&s->mtx); - nni_msg_free(msg); - - bus0_sock_getq(s); -} - -static void -bus0_sock_getq_cb_raw(void *arg) -{ - bus0_sock *s = arg; - bus0_pipe *p; - nni_msg * msg; - uint32_t sender; - - if (nni_aio_result(s->aio_getq) != 0) { - return; - } - - msg = nni_aio_get_msg(s->aio_getq); - - // The header being present indicates that the message - // was received locally and we are rebroadcasting. (Device - // is doing this probably.) In this case grab the pipe - // ID from the header, so we can exclude it. - if (nni_msg_header_len(msg) >= 4) { - sender = nni_msg_header_trim_u32(msg); - } else { - sender = 0; - } - - nni_mtx_lock(&s->mtx); - NNI_LIST_FOREACH (&s->pipes, p) { - if (nni_pipe_id(p->npipe) == sender) { - continue; - } - nni_msg_clone(msg); - if (nni_msgq_tryput(p->sendq, msg) != 0) { - nni_msg_free(msg); - } - } - nni_mtx_unlock(&s->mtx); - nni_msg_free(msg); - - bus0_sock_getq(s); -} - -static void -bus0_sock_getq(bus0_sock *s) -{ - nni_msgq_aio_get(s->uwq, s->aio_getq); -} - -static void -bus0_pipe_getq(bus0_pipe *p) -{ - nni_msgq_aio_get(p->sendq, p->aio_getq); -} - -static void -bus0_pipe_recv(bus0_pipe *p) -{ - nni_pipe_recv(p->npipe, p->aio_recv); -} - -static void -bus0_sock_send(void *arg, nni_aio *aio) -{ - bus0_sock *s = arg; - - nni_msgq_aio_put(s->uwq, aio); -} - -static void -bus0_sock_recv(void *arg, nni_aio *aio) -{ - bus0_sock *s = arg; - - nni_msgq_aio_get(s->urq, aio); -} - -static nni_proto_pipe_ops bus0_pipe_ops = { - .pipe_size = sizeof(bus0_pipe), - .pipe_init = bus0_pipe_init, - .pipe_fini = bus0_pipe_fini, - .pipe_start = bus0_pipe_start, - .pipe_close = bus0_pipe_close, - .pipe_stop = bus0_pipe_stop, -}; - -static nni_option bus0_sock_options[] = { - // terminate list - { - .o_name = NULL, - }, -}; - -static nni_proto_sock_ops bus0_sock_ops = { - .sock_size = sizeof(bus0_sock), - .sock_init = bus0_sock_init, - .sock_fini = bus0_sock_fini, - .sock_open = bus0_sock_open, - .sock_close = bus0_sock_close, - .sock_send = bus0_sock_send, - .sock_recv = bus0_sock_recv, - .sock_options = bus0_sock_options, -}; - -static nni_proto_sock_ops bus0_sock_ops_raw = { - .sock_size = sizeof(bus0_sock), - .sock_init = bus0_sock_init_raw, - .sock_fini = bus0_sock_fini, - .sock_open = bus0_sock_open, - .sock_close = bus0_sock_close, - .sock_send = bus0_sock_send, - .sock_recv = bus0_sock_recv, - .sock_options = bus0_sock_options, -}; - -static nni_proto bus0_proto = { - .proto_version = NNI_PROTOCOL_VERSION, - .proto_self = { NNI_PROTO_BUS_V0, "bus" }, - .proto_peer = { NNI_PROTO_BUS_V0, "bus" }, - .proto_flags = NNI_PROTO_FLAG_SNDRCV, - .proto_sock_ops = &bus0_sock_ops, - .proto_pipe_ops = &bus0_pipe_ops, -}; - -static nni_proto bus0_proto_raw = { - .proto_version = NNI_PROTOCOL_VERSION, - .proto_self = { NNI_PROTO_BUS_V0, "bus" }, - .proto_peer = { NNI_PROTO_BUS_V0, "bus" }, - .proto_flags = NNI_PROTO_FLAG_SNDRCV | NNI_PROTO_FLAG_RAW, - .proto_sock_ops = &bus0_sock_ops_raw, - .proto_pipe_ops = &bus0_pipe_ops, -}; - -int -nng_bus0_open(nng_socket *sidp) -{ - return (nni_proto_open(sidp, &bus0_proto)); -} - -int -nng_bus0_open_raw(nng_socket *sidp) -{ - return (nni_proto_open(sidp, &bus0_proto_raw)); -} diff --git a/src/protocol/pair0/CMakeLists.txt b/src/protocol/pair0/CMakeLists.txt deleted file mode 100644 index b12583ab..00000000 --- a/src/protocol/pair0/CMakeLists.txt +++ /dev/null @@ -1,16 +0,0 @@ -# -# Copyright 2020 Staysail Systems, Inc. -# Copyright 2018 Capitar IT Group BV -# -# This software is supplied under the terms of the MIT License, a -# copy of which should be located in the distribution where this -# file was obtained (LICENSE.txt). A copy of the license may also be -# found online at https://opensource.org/licenses/MIT. -# - -# PAIRv0 protocol -nng_directory(pair0) - -nng_sources_if(NNG_PROTO_PAIR0 pair.c) -nng_headers_if(NNG_PROTO_PAIR0 nng/protocol/pair0/pair.h) -nng_defines_if(NNG_PROTO_PAIR0 NNG_HAVE_PAIR0) \ No newline at end of file diff --git a/src/protocol/pair0/pair.c b/src/protocol/pair0/pair.c deleted file mode 100644 index 41f88c7c..00000000 --- a/src/protocol/pair0/pair.c +++ /dev/null @@ -1,305 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// Copyright 2018 Capitar IT Group BV -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - -#include - -#include "core/nng_impl.h" -#include "nng/protocol/pair0/pair.h" - -// Pair protocol. The PAIR protocol is a simple 1:1 messaging pattern. -// While a peer is connected to the server, all other peer connection -// attempts are discarded. - -#ifndef NNI_PROTO_PAIR_V0 -#define NNI_PROTO_PAIR_V0 NNI_PROTO(1, 0) -#endif - -typedef struct pair0_pipe pair0_pipe; -typedef struct pair0_sock pair0_sock; - -static void pair0_send_cb(void *); -static void pair0_recv_cb(void *); -static void pair0_getq_cb(void *); -static void pair0_putq_cb(void *); -static void pair0_pipe_fini(void *); - -// pair0_sock is our per-socket protocol private structure. -struct pair0_sock { - pair0_pipe *ppipe; - nni_msgq * uwq; - nni_msgq * urq; - nni_mtx mtx; -}; - -// An pair0_pipe is our per-pipe protocol private structure. We keep -// one of these even though in theory we'd only have a single underlying -// pipe. The separate data structure is more like other protocols that do -// manage multiple pipes. -struct pair0_pipe { - nni_pipe * npipe; - pair0_sock *psock; - nni_aio aio_send; - nni_aio aio_recv; - nni_aio aio_getq; - nni_aio aio_putq; -}; - -static int -pair0_sock_init(void *arg, nni_sock *nsock) -{ - pair0_sock *s = arg; - - nni_mtx_init(&s->mtx); - s->ppipe = NULL; - s->uwq = nni_sock_sendq(nsock); - s->urq = nni_sock_recvq(nsock); - return (0); -} - -static void -pair0_sock_fini(void *arg) -{ - pair0_sock *s = arg; - - nni_mtx_fini(&s->mtx); -} - -static void -pair0_pipe_stop(void *arg) -{ - pair0_pipe *p = arg; - - nni_aio_stop(&p->aio_send); - nni_aio_stop(&p->aio_recv); - nni_aio_stop(&p->aio_putq); - nni_aio_stop(&p->aio_getq); -} - -static void -pair0_pipe_fini(void *arg) -{ - pair0_pipe *p = arg; - - nni_aio_fini(&p->aio_send); - nni_aio_fini(&p->aio_recv); - nni_aio_fini(&p->aio_putq); - nni_aio_fini(&p->aio_getq); -} - -static int -pair0_pipe_init(void *arg, nni_pipe *npipe, void *psock) -{ - pair0_pipe *p = arg; - - nni_aio_init(&p->aio_send, pair0_send_cb, p); - nni_aio_init(&p->aio_recv, pair0_recv_cb, p); - nni_aio_init(&p->aio_getq, pair0_getq_cb, p); - nni_aio_init(&p->aio_putq, pair0_putq_cb, p); - - p->npipe = npipe; - p->psock = psock; - return (0); -} - -static int -pair0_pipe_start(void *arg) -{ - pair0_pipe *p = arg; - pair0_sock *s = p->psock; - - if (nni_pipe_peer(p->npipe) != NNI_PROTO_PAIR_V0) { - // Peer protocol mismatch. - return (NNG_EPROTO); - } - - nni_mtx_lock(&s->mtx); - if (s->ppipe != NULL) { - nni_mtx_unlock(&s->mtx); - return (NNG_EBUSY); // Already have a peer, denied. - } - s->ppipe = p; - nni_mtx_unlock(&s->mtx); - - // Schedule a getq on the upper, and a read from the pipe. - // Each of these also sets up another hold on the pipe itself. - nni_msgq_aio_get(s->uwq, &p->aio_getq); - nni_pipe_recv(p->npipe, &p->aio_recv); - - return (0); -} - -static void -pair0_pipe_close(void *arg) -{ - pair0_pipe *p = arg; - pair0_sock *s = p->psock; - - nni_aio_close(&p->aio_send); - nni_aio_close(&p->aio_recv); - nni_aio_close(&p->aio_putq); - nni_aio_close(&p->aio_getq); - - nni_mtx_lock(&s->mtx); - if (s->ppipe == p) { - s->ppipe = NULL; - } - nni_mtx_unlock(&s->mtx); -} - -static void -pair0_recv_cb(void *arg) -{ - pair0_pipe *p = arg; - pair0_sock *s = p->psock; - nni_msg * msg; - - if (nni_aio_result(&p->aio_recv) != 0) { - nni_pipe_close(p->npipe); - return; - } - - msg = nni_aio_get_msg(&p->aio_recv); - nni_aio_set_msg(&p->aio_putq, msg); - nni_aio_set_msg(&p->aio_recv, NULL); - - nni_msg_set_pipe(msg, nni_pipe_id(p->npipe)); - nni_msgq_aio_put(s->urq, &p->aio_putq); -} - -static void -pair0_putq_cb(void *arg) -{ - pair0_pipe *p = arg; - - if (nni_aio_result(&p->aio_putq) != 0) { - nni_msg_free(nni_aio_get_msg(&p->aio_putq)); - nni_aio_set_msg(&p->aio_putq, NULL); - nni_pipe_close(p->npipe); - return; - } - nni_pipe_recv(p->npipe, &p->aio_recv); -} - -static void -pair0_getq_cb(void *arg) -{ - pair0_pipe *p = arg; - - if (nni_aio_result(&p->aio_getq) != 0) { - nni_pipe_close(p->npipe); - return; - } - - nni_aio_set_msg(&p->aio_send, nni_aio_get_msg(&p->aio_getq)); - nni_aio_set_msg(&p->aio_getq, NULL); - nni_pipe_send(p->npipe, &p->aio_send); -} - -static void -pair0_send_cb(void *arg) -{ - pair0_pipe *p = arg; - pair0_sock *s = p->psock; - - if (nni_aio_result(&p->aio_send) != 0) { - nni_msg_free(nni_aio_get_msg(&p->aio_send)); - nni_aio_set_msg(&p->aio_send, NULL); - nni_pipe_close(p->npipe); - return; - } - - nni_msgq_aio_get(s->uwq, &p->aio_getq); -} - -static void -pair0_sock_open(void *arg) -{ - NNI_ARG_UNUSED(arg); -} - -static void -pair0_sock_close(void *arg) -{ - NNI_ARG_UNUSED(arg); -} - -static void -pair0_sock_send(void *arg, nni_aio *aio) -{ - pair0_sock *s = arg; - - nni_msgq_aio_put(s->uwq, aio); -} - -static void -pair0_sock_recv(void *arg, nni_aio *aio) -{ - pair0_sock *s = arg; - - nni_msgq_aio_get(s->urq, aio); -} - -static nni_proto_pipe_ops pair0_pipe_ops = { - .pipe_size = sizeof(pair0_pipe), - .pipe_init = pair0_pipe_init, - .pipe_fini = pair0_pipe_fini, - .pipe_start = pair0_pipe_start, - .pipe_close = pair0_pipe_close, - .pipe_stop = pair0_pipe_stop, -}; - -static nni_option pair0_sock_options[] = { - // terminate list - { - .o_name = NULL, - } -}; - -static nni_proto_sock_ops pair0_sock_ops = { - .sock_size = sizeof(pair0_sock), - .sock_init = pair0_sock_init, - .sock_fini = pair0_sock_fini, - .sock_open = pair0_sock_open, - .sock_close = pair0_sock_close, - .sock_send = pair0_sock_send, - .sock_recv = pair0_sock_recv, - .sock_options = pair0_sock_options, -}; - -// Legacy protocol (v0) -static nni_proto pair0_proto = { - .proto_version = NNI_PROTOCOL_VERSION, - .proto_self = { NNI_PROTO_PAIR_V0, "pair" }, - .proto_peer = { NNI_PROTO_PAIR_V0, "pair" }, - .proto_flags = NNI_PROTO_FLAG_SNDRCV, - .proto_sock_ops = &pair0_sock_ops, - .proto_pipe_ops = &pair0_pipe_ops, -}; - -static nni_proto pair0_proto_raw = { - .proto_version = NNI_PROTOCOL_VERSION, - .proto_self = { NNI_PROTO_PAIR_V0, "pair" }, - .proto_peer = { NNI_PROTO_PAIR_V0, "pair" }, - .proto_flags = NNI_PROTO_FLAG_SNDRCV | NNI_PROTO_FLAG_RAW, - .proto_sock_ops = &pair0_sock_ops, - .proto_pipe_ops = &pair0_pipe_ops, -}; - -int -nng_pair0_open(nng_socket *sidp) -{ - return (nni_proto_open(sidp, &pair0_proto)); -} - -int -nng_pair0_open_raw(nng_socket *sidp) -{ - return (nni_proto_open(sidp, &pair0_proto_raw)); -} diff --git a/src/protocol/pair1/CMakeLists.txt b/src/protocol/pair1/CMakeLists.txt deleted file mode 100644 index 12e12607..00000000 --- a/src/protocol/pair1/CMakeLists.txt +++ /dev/null @@ -1,20 +0,0 @@ -# -# Copyright 2019 Staysail Systems, Inc. -# Copyright 2018 Capitar IT Group BV -# -# This software is supplied under the terms of the MIT License, a -# copy of which should be located in the distribution where this -# file was obtained (LICENSE.txt). A copy of the license may also be -# found online at https://opensource.org/licenses/MIT. -# - -# PAIRv1 protocol -nng_directory(pair1) - -# XXX: break pair1_poly into an ifdef. -nng_sources_if(NNG_PROTO_PAIR1 pair.c pair1_poly.c) -nng_headers_if(NNG_PROTO_PAIR1 nng/protocol/pair1/pair.h) -nng_defines_if(NNG_PROTO_PAIR1 NNG_HAVE_PAIR1) - -nng_test(pair1_test) -nng_test(pair1_poly_test) \ No newline at end of file diff --git a/src/protocol/pair1/pair.c b/src/protocol/pair1/pair.c deleted file mode 100644 index ba497c42..00000000 --- a/src/protocol/pair1/pair.c +++ /dev/null @@ -1,540 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// Copyright 2018 Capitar IT Group BV -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - -#include - -#include "core/nng_impl.h" -#include "nng/protocol/pair1/pair.h" - -// Pair protocol. The PAIRv1 protocol is a simple 1:1 messaging pattern. - -#ifdef NNG_ENABLE_STATS -#define BUMP_STAT(x) nni_stat_inc(x, 1) -#else -#define BUMP_STAT(x) -#endif - -typedef struct pair1_pipe pair1_pipe; -typedef struct pair1_sock pair1_sock; - -static void pair1_pipe_send_cb(void *); -static void pair1_pipe_recv_cb(void *); -static void pair1_pipe_get_cb(void *); -static void pair1_pipe_put_cb(void *); -static void pair1_pipe_fini(void *); - -// pair1_sock is our per-socket protocol private structure. -struct pair1_sock { - nni_msgq * uwq; - nni_msgq * urq; - nni_sock * sock; - bool raw; - pair1_pipe * p; - nni_atomic_int ttl; - nni_mtx mtx; -#ifdef NNG_ENABLE_STATS - nni_stat_item stat_poly; - nni_stat_item stat_raw; - nni_stat_item stat_reject_mismatch; - nni_stat_item stat_reject_already; - nni_stat_item stat_ttl_drop; - nni_stat_item stat_rx_malformed; - nni_stat_item stat_tx_malformed; - nni_stat_item stat_tx_drop; -#endif -#ifdef NNG_TEST_LIB - bool inject_header; -#endif -}; - -// pair1_pipe is our per-pipe protocol private structure. -struct pair1_pipe { - nni_pipe * pipe; - pair1_sock *pair; - nni_aio aio_send; - nni_aio aio_recv; - nni_aio aio_get; - nni_aio aio_put; -}; - -static void -pair1_sock_fini(void *arg) -{ - pair1_sock *s = arg; - - nni_mtx_fini(&s->mtx); -} - -#ifdef NNG_ENABLE_STATS -static void -pair1_add_sock_stat( - pair1_sock *s, nni_stat_item *item, const nni_stat_info *info) -{ - nni_stat_init(item, info); - nni_sock_add_stat(s->sock, item); -} -#endif - -static int -pair1_sock_init_impl(void *arg, nni_sock *sock, bool raw) -{ - pair1_sock *s = arg; - - // Raw mode uses this. - nni_mtx_init(&s->mtx); - s->sock = sock; - -#ifdef NNG_ENABLE_STATS - static const nni_stat_info poly_info = { - .si_name = "poly", - .si_desc = "polyamorous mode?", - .si_type = NNG_STAT_BOOLEAN, - }; - static const nni_stat_info raw_info = { - .si_name = "raw", - .si_desc = "raw mode?", - .si_type = NNG_STAT_BOOLEAN, - }; - static const nni_stat_info mismatch_info = { - .si_name = "mismatch", - .si_desc = "pipes rejected (protocol mismatch)", - .si_type = NNG_STAT_COUNTER, - .si_atomic = true, - }; - static const nni_stat_info already_info = { - .si_name = "already", - .si_desc = "pipes rejected (already connected)", - .si_type = NNG_STAT_COUNTER, - .si_atomic = true, - }; - static const nni_stat_info ttl_drop_info = { - .si_name = "ttl_drop", - .si_desc = "messages dropped due to too many hops", - .si_type = NNG_STAT_COUNTER, - .si_unit = NNG_UNIT_MESSAGES, - .si_atomic = true, - }; - static const nni_stat_info tx_drop_info = { - .si_name = "tx_drop", - .si_desc = "messages dropped undeliverable", - .si_type = NNG_STAT_COUNTER, - .si_unit = NNG_UNIT_MESSAGES, - .si_atomic = true, - }; - static const nni_stat_info rx_malformed_info = { - .si_name = "rx_malformed", - .si_desc = "malformed messages received", - .si_type = NNG_STAT_COUNTER, - .si_unit = NNG_UNIT_MESSAGES, - .si_atomic = true, - }; - static const nni_stat_info tx_malformed_info = { - .si_name = "tx_malformed", - .si_desc = "malformed messages not sent", - .si_type = NNG_STAT_COUNTER, - .si_unit = NNG_UNIT_MESSAGES, - .si_atomic = true, - }; - - pair1_add_sock_stat(s, &s->stat_poly, &poly_info); - pair1_add_sock_stat(s, &s->stat_raw, &raw_info); - pair1_add_sock_stat(s, &s->stat_reject_mismatch, &mismatch_info); - pair1_add_sock_stat(s, &s->stat_reject_already, &already_info); - pair1_add_sock_stat(s, &s->stat_ttl_drop, &ttl_drop_info); - pair1_add_sock_stat(s, &s->stat_tx_drop, &tx_drop_info); - pair1_add_sock_stat(s, &s->stat_rx_malformed, &rx_malformed_info); - - if (raw) { - // This stat only makes sense in raw mode. - pair1_add_sock_stat( - s, &s->stat_tx_malformed, &tx_malformed_info); - } - - nni_stat_set_bool(&s->stat_raw, raw); - nni_stat_set_bool(&s->stat_poly, false); -#endif - - s->raw = raw; - s->uwq = nni_sock_sendq(sock); - s->urq = nni_sock_recvq(sock); - nni_atomic_init(&s->ttl); - nni_atomic_set(&s->ttl, 8); - - return (0); -} - -static int -pair1_sock_init(void *arg, nni_sock *sock) -{ - return (pair1_sock_init_impl(arg, sock, false)); -} - -static int -pair1_sock_init_raw(void *arg, nni_sock *sock) -{ - return (pair1_sock_init_impl(arg, sock, true)); -} - -static void -pair1_pipe_stop(void *arg) -{ - pair1_pipe *p = arg; - pair1_sock *s = p->pair; - - nni_mtx_lock(&s->mtx); - if (s->p == p) { - s->p = NULL; - } - nni_mtx_unlock(&s->mtx); - nni_aio_stop(&p->aio_send); - nni_aio_stop(&p->aio_recv); - nni_aio_stop(&p->aio_put); - nni_aio_stop(&p->aio_get); -} - -static void -pair1_pipe_fini(void *arg) -{ - pair1_pipe *p = arg; - - nni_aio_fini(&p->aio_send); - nni_aio_fini(&p->aio_recv); - nni_aio_fini(&p->aio_put); - nni_aio_fini(&p->aio_get); -} - -static int -pair1_pipe_init(void *arg, nni_pipe *pipe, void *pair) -{ - pair1_pipe *p = arg; - - nni_aio_init(&p->aio_send, pair1_pipe_send_cb, p); - nni_aio_init(&p->aio_recv, pair1_pipe_recv_cb, p); - nni_aio_init(&p->aio_get, pair1_pipe_get_cb, p); - nni_aio_init(&p->aio_put, pair1_pipe_put_cb, p); - - p->pipe = pipe; - p->pair = pair; - - return (0); -} - -static int -pair1_pipe_start(void *arg) -{ - pair1_pipe *p = arg; - pair1_sock *s = p->pair; - - if (nni_pipe_peer(p->pipe) != NNG_PAIR1_PEER) { - BUMP_STAT(&s->stat_reject_mismatch); - // Peer protocol mismatch. - return (NNG_EPROTO); - } - - nni_mtx_lock(&s->mtx); - if (s->p != NULL) { - nni_mtx_unlock(&s->mtx); - BUMP_STAT(&s->stat_reject_already); - return (NNG_EBUSY); - } - s->p = p; - nni_mtx_unlock(&s->mtx); - - // Schedule a get. - nni_msgq_aio_get(s->uwq, &p->aio_get); - - // And the pipe read of course. - nni_pipe_recv(p->pipe, &p->aio_recv); - - return (0); -} - -static void -pair1_pipe_close(void *arg) -{ - pair1_pipe *p = arg; - - nni_aio_close(&p->aio_send); - nni_aio_close(&p->aio_recv); - nni_aio_close(&p->aio_put); - nni_aio_close(&p->aio_get); -} - -static void -pair1_pipe_recv_cb(void *arg) -{ - pair1_pipe *p = arg; - pair1_sock *s = p->pair; - nni_msg * msg; - uint32_t hdr; - nni_pipe * pipe = p->pipe; - size_t len; - - if (nni_aio_result(&p->aio_recv) != 0) { - nni_pipe_close(p->pipe); - return; - } - - msg = nni_aio_get_msg(&p->aio_recv); - nni_aio_set_msg(&p->aio_recv, NULL); - - // Store the pipe ID. - nni_msg_set_pipe(msg, nni_pipe_id(p->pipe)); - - // If the message is missing the hop count header, scrap it. - if ((nni_msg_len(msg) < sizeof(uint32_t)) || - ((hdr = nni_msg_trim_u32(msg)) > 0xff)) { - BUMP_STAT(&s->stat_rx_malformed); - nni_msg_free(msg); - nni_pipe_close(pipe); - return; - } - - len = nni_msg_len(msg); - - // If we bounced too many times, discard the message, but - // keep getting more. - if ((int) hdr > nni_atomic_get(&s->ttl)) { - BUMP_STAT(&s->stat_ttl_drop); - nni_msg_free(msg); - nni_pipe_recv(pipe, &p->aio_recv); - return; - } - - // Store the hop count in the header. - nni_msg_header_append_u32(msg, hdr); - - // Send the message up. - nni_aio_set_msg(&p->aio_put, msg); - nni_sock_bump_rx(s->sock, len); - nni_msgq_aio_put(s->urq, &p->aio_put); -} - -static void -pair1_pipe_put_cb(void *arg) -{ - pair1_pipe *p = arg; - - if (nni_aio_result(&p->aio_put) != 0) { - nni_msg_free(nni_aio_get_msg(&p->aio_put)); - nni_aio_set_msg(&p->aio_put, NULL); - nni_pipe_close(p->pipe); - return; - } - nni_pipe_recv(p->pipe, &p->aio_recv); -} - -static void -pair1_pipe_get_cb(void *arg) -{ - pair1_pipe *p = arg; - pair1_sock *s = p->pair; - nni_msg * msg; - uint32_t hops; - - if (nni_aio_result(&p->aio_get) != 0) { - nni_pipe_close(p->pipe); - return; - } - - msg = nni_aio_get_msg(&p->aio_get); - nni_aio_set_msg(&p->aio_get, NULL); - - // Raw mode messages have the header already formed, with a hop count. - // Cooked mode messages have no header so we have to add one. - if (s->raw) { - if ((nni_msg_header_len(msg) != sizeof(uint32_t)) || - ((hops = nni_msg_header_trim_u32(msg)) > 254)) { - BUMP_STAT(&s->stat_tx_malformed); - nni_msg_free(msg); - nni_msgq_aio_get(s->uwq, &p->aio_get); - return; - } -#if NNG_TEST_LIB - } else if (s->inject_header) { - nni_aio_set_msg(&p->aio_send, msg); - nni_pipe_send(p->pipe, &p->aio_send); - return; -#endif - } else { - // Strip off any previously existing header, such as when - // replying to messages. - nni_msg_header_clear(msg); - hops = 0; - } - - hops++; - - // Insert the hops header. - nni_msg_header_append_u32(msg, hops); - - nni_aio_set_msg(&p->aio_send, msg); - nni_pipe_send(p->pipe, &p->aio_send); -} - -static void -pair1_pipe_send_cb(void *arg) -{ - pair1_pipe *p = arg; - pair1_sock *s = p->pair; - - if (nni_aio_result(&p->aio_send) != 0) { - nni_msg_free(nni_aio_get_msg(&p->aio_send)); - nni_aio_set_msg(&p->aio_send, NULL); - nni_pipe_close(p->pipe); - return; - } - - nni_msgq_aio_get(s->uwq, &p->aio_get); -} - -static void -pair1_sock_open(void *arg) -{ - NNI_ARG_UNUSED(arg); -} - -static void -pair1_sock_close(void *arg) -{ - NNI_ARG_UNUSED(arg); -} - -static int -pair1_sock_set_max_ttl(void *arg, const void *buf, size_t sz, nni_opt_type t) -{ - pair1_sock *s = arg; - int rv; - int ttl; - - if ((rv = nni_copyin_int(&ttl, buf, sz, 1, NNI_MAX_MAX_TTL, t)) == 0) { - nni_atomic_set(&s->ttl, ttl); - } - - return (rv); -} - -static int -pair1_sock_get_max_ttl(void *arg, void *buf, size_t *szp, nni_opt_type t) -{ - pair1_sock *s = arg; - return (nni_copyout_int(nni_atomic_get(&s->ttl), buf, szp, t)); -} - -#ifdef NNG_TEST_LIB -static int -pair1_set_test_inject_header(void *arg, const void *buf, size_t sz, nni_type t) -{ - pair1_sock *s = arg; - int rv; - nni_mtx_lock(&s->mtx); - rv = nni_copyin_bool(&s->inject_header, buf, sz, t); - nni_mtx_unlock(&s->mtx); - return (rv); -} -#endif - -static void -pair1_sock_send(void *arg, nni_aio *aio) -{ - pair1_sock *s = arg; - - nni_sock_bump_tx(s->sock, nni_msg_len(nni_aio_get_msg(aio))); - nni_msgq_aio_put(s->uwq, aio); -} - -static void -pair1_sock_recv(void *arg, nni_aio *aio) -{ - pair1_sock *s = arg; - - nni_msgq_aio_get(s->urq, aio); -} - -static nni_proto_pipe_ops pair1_pipe_ops = { - .pipe_size = sizeof(pair1_pipe), - .pipe_init = pair1_pipe_init, - .pipe_fini = pair1_pipe_fini, - .pipe_start = pair1_pipe_start, - .pipe_close = pair1_pipe_close, - .pipe_stop = pair1_pipe_stop, -}; - -static nni_option pair1_sock_options[] = { - { - .o_name = NNG_OPT_MAXTTL, - .o_get = pair1_sock_get_max_ttl, - .o_set = pair1_sock_set_max_ttl, - }, -#ifdef NNG_TEST_LIB - { - // Test only option to pass header unmolested. This allows - // us to inject bad header contents. - .o_name = "pair1_test_inject_header", - .o_set = pair1_set_test_inject_header, - }, -#endif - // terminate list - { - .o_name = NULL, - }, -}; - -static nni_proto_sock_ops pair1_sock_ops = { - .sock_size = sizeof(pair1_sock), - .sock_init = pair1_sock_init, - .sock_fini = pair1_sock_fini, - .sock_open = pair1_sock_open, - .sock_close = pair1_sock_close, - .sock_recv = pair1_sock_recv, - .sock_send = pair1_sock_send, - .sock_options = pair1_sock_options, -}; - -static nni_proto pair1_proto = { - .proto_version = NNI_PROTOCOL_VERSION, - .proto_self = { NNG_PAIR1_SELF, NNG_PAIR1_SELF_NAME }, - .proto_peer = { NNG_PAIR1_PEER, NNG_PAIR1_PEER_NAME }, - .proto_flags = NNI_PROTO_FLAG_SNDRCV, - .proto_sock_ops = &pair1_sock_ops, - .proto_pipe_ops = &pair1_pipe_ops, -}; - -int -nng_pair1_open(nng_socket *sock) -{ - return (nni_proto_open(sock, &pair1_proto)); -} - -static nni_proto_sock_ops pair1_sock_ops_raw = { - .sock_size = sizeof(pair1_sock), - .sock_init = pair1_sock_init_raw, - .sock_fini = pair1_sock_fini, - .sock_open = pair1_sock_open, - .sock_close = pair1_sock_close, - .sock_recv = pair1_sock_recv, - .sock_send = pair1_sock_send, - .sock_options = pair1_sock_options, -}; - -static nni_proto pair1_proto_raw = { - .proto_version = NNI_PROTOCOL_VERSION, - .proto_self = { NNG_PAIR1_SELF, NNG_PAIR1_SELF_NAME }, - .proto_peer = { NNG_PAIR1_PEER, NNG_PAIR1_PEER_NAME }, - .proto_flags = NNI_PROTO_FLAG_SNDRCV | NNI_PROTO_FLAG_RAW, - .proto_sock_ops = &pair1_sock_ops_raw, - .proto_pipe_ops = &pair1_pipe_ops, -}; - -int -nng_pair1_open_raw(nng_socket *sock) -{ - return (nni_proto_open(sock, &pair1_proto_raw)); -} diff --git a/src/protocol/pair1/pair1_poly.c b/src/protocol/pair1/pair1_poly.c deleted file mode 100644 index 6c16745c..00000000 --- a/src/protocol/pair1/pair1_poly.c +++ /dev/null @@ -1,535 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// Copyright 2018 Capitar IT Group BV -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - -#include - -#include "core/nng_impl.h" -#include "nng/protocol/pair1/pair.h" - -// Pair1 polyamorous mode. The PAIRv1 protocol is normally a simple 1:1 -// messaging pattern, but this mode offers the ability to use a best-effort -// multicast type of communication. There are limitations however. -// Most notably this does not interact well with nng_device type -// proxies, and there is no support for raw mode. - -// THIS FEATURE IS DEPRECATED. We discourage use in new applications. - -#ifdef NNG_ENABLE_STATS -#define BUMP_STAT(x) nni_stat_inc(x, 1) -#else -#define BUMP_STAT(x) -#endif - -typedef struct pair1poly_pipe pair1poly_pipe; -typedef struct pair1poly_sock pair1poly_sock; - -static void pair1poly_sock_get_cb(void *); -static void pair1poly_pipe_send_cb(void *); -static void pair1poly_pipe_recv_cb(void *); -static void pair1poly_pipe_get_cb(void *); -static void pair1poly_pipe_put_cb(void *); -static void pair1poly_pipe_fini(void *); - -// pair1poly_sock is our per-socket protocol private structure. -struct pair1poly_sock { - nni_msgq * uwq; - nni_msgq * urq; - nni_sock * sock; - nni_atomic_int ttl; - nni_mtx mtx; - nni_id_map pipes; - nni_list plist; - bool started; - nni_aio aio_get; - nni_stat_item stat_poly; - nni_stat_item stat_raw; - nni_stat_item stat_reject_mismatch; - nni_stat_item stat_reject_already; - nni_stat_item stat_ttl_drop; - nni_stat_item stat_rx_malformed; - nni_stat_item stat_tx_malformed; - nni_stat_item stat_tx_drop; -}; - -// pair1poly_pipe is our per-pipe protocol private structure. -struct pair1poly_pipe { - nni_pipe * pipe; - pair1poly_sock *pair; - nni_msgq * send_queue; - nni_aio aio_send; - nni_aio aio_recv; - nni_aio aio_get; - nni_aio aio_put; - nni_list_node node; -}; - -static void -pair1poly_sock_fini(void *arg) -{ - pair1poly_sock *s = arg; - - nni_aio_fini(&s->aio_get); - nni_id_map_fini(&s->pipes); - nni_mtx_fini(&s->mtx); -} - -#ifdef NNG_ENABLE_STATS -static void -pair1_add_sock_stat( - pair1poly_sock *s, nni_stat_item *item, const nni_stat_info *info) -{ - nni_stat_init(item, info); - nni_sock_add_stat(s->sock, item); -} -#endif - -static int -pair1poly_sock_init(void *arg, nni_sock *sock) -{ - pair1poly_sock *s = arg; - - nni_id_map_init(&s->pipes, 0, 0, false); - NNI_LIST_INIT(&s->plist, pair1poly_pipe, node); - s->sock = sock; - - // Raw mode uses this. - nni_mtx_init(&s->mtx); - - nni_aio_init(&s->aio_get, pair1poly_sock_get_cb, s); - -#ifdef NNG_ENABLE_STATS - static const nni_stat_info poly_info = { - .si_name = "poly", - .si_desc = "polyamorous mode?", - .si_type = NNG_STAT_BOOLEAN, - }; - static const nni_stat_info raw_info = { - .si_name = "raw", - .si_desc = "raw mode?", - .si_type = NNG_STAT_BOOLEAN, - }; - static const nni_stat_info mismatch_info = { - .si_name = "mismatch", - .si_desc = "pipes rejected (protocol mismatch)", - .si_type = NNG_STAT_COUNTER, - .si_atomic = true, - }; - static const nni_stat_info already_info = { - .si_name = "already", - .si_desc = "pipes rejected (already connected)", - .si_type = NNG_STAT_COUNTER, - .si_atomic = true, - }; - static const nni_stat_info ttl_drop_info = { - .si_name = "ttl_drop", - .si_desc = "messages dropped due to too many hops", - .si_type = NNG_STAT_COUNTER, - .si_unit = NNG_UNIT_MESSAGES, - .si_atomic = true, - }; - static const nni_stat_info tx_drop_info = { - .si_name = "tx_drop", - .si_desc = "messages dropped undeliverable", - .si_type = NNG_STAT_COUNTER, - .si_unit = NNG_UNIT_MESSAGES, - .si_atomic = true, - }; - static const nni_stat_info rx_malformed_info = { - .si_name = "rx_malformed", - .si_desc = "malformed messages received", - .si_type = NNG_STAT_COUNTER, - .si_unit = NNG_UNIT_MESSAGES, - .si_atomic = true, - }; - static const nni_stat_info tx_malformed_info = { - .si_name = "tx_malformed", - .si_desc = "malformed messages not sent", - .si_type = NNG_STAT_COUNTER, - .si_unit = NNG_UNIT_MESSAGES, - .si_atomic = true, - }; - - pair1_add_sock_stat(s, &s->stat_poly, &poly_info); - pair1_add_sock_stat(s, &s->stat_raw, &raw_info); - pair1_add_sock_stat(s, &s->stat_reject_mismatch, &mismatch_info); - pair1_add_sock_stat(s, &s->stat_reject_already, &already_info); - pair1_add_sock_stat(s, &s->stat_ttl_drop, &ttl_drop_info); - pair1_add_sock_stat(s, &s->stat_tx_drop, &tx_drop_info); - pair1_add_sock_stat(s, &s->stat_rx_malformed, &rx_malformed_info); - pair1_add_sock_stat(s, &s->stat_tx_malformed, &tx_malformed_info); - - nni_stat_set_bool(&s->stat_raw, false); - nni_stat_set_bool(&s->stat_poly, true); -#endif - - s->uwq = nni_sock_sendq(sock); - s->urq = nni_sock_recvq(sock); - nni_atomic_init(&s->ttl); - nni_atomic_set(&s->ttl, 8); - - return (0); -} - -static void -pair1poly_pipe_stop(void *arg) -{ - pair1poly_pipe *p = arg; - - nni_aio_stop(&p->aio_send); - nni_aio_stop(&p->aio_recv); - nni_aio_stop(&p->aio_put); - nni_aio_stop(&p->aio_get); -} - -static void -pair1poly_pipe_fini(void *arg) -{ - pair1poly_pipe *p = arg; - - nni_aio_fini(&p->aio_send); - nni_aio_fini(&p->aio_recv); - nni_aio_fini(&p->aio_put); - nni_aio_fini(&p->aio_get); - nni_msgq_fini(p->send_queue); -} - -static int -pair1poly_pipe_init(void *arg, nni_pipe *pipe, void *pair) -{ - pair1poly_pipe *p = arg; - int rv; - - nni_aio_init(&p->aio_send, pair1poly_pipe_send_cb, p); - nni_aio_init(&p->aio_recv, pair1poly_pipe_recv_cb, p); - nni_aio_init(&p->aio_get, pair1poly_pipe_get_cb, p); - nni_aio_init(&p->aio_put, pair1poly_pipe_put_cb, p); - - if ((rv = nni_msgq_init(&p->send_queue, 2)) != 0) { - pair1poly_pipe_fini(p); - return (rv); - } - - p->pipe = pipe; - p->pair = pair; - - return (0); -} - -static int -pair1poly_pipe_start(void *arg) -{ - pair1poly_pipe *p = arg; - pair1poly_sock *s = p->pair; - uint32_t id; - int rv; - - nni_mtx_lock(&s->mtx); - if (nni_pipe_peer(p->pipe) != NNG_PAIR1_PEER) { - nni_mtx_unlock(&s->mtx); - BUMP_STAT(&s->stat_reject_mismatch); - // Peer protocol mismatch. - return (NNG_EPROTO); - } - - id = nni_pipe_id(p->pipe); - if ((rv = nni_id_set(&s->pipes, id, p)) != 0) { - nni_mtx_unlock(&s->mtx); - return (rv); - } - if (!s->started) { - nni_msgq_aio_get(s->uwq, &s->aio_get); - } - nni_list_append(&s->plist, p); - s->started = true; - nni_mtx_unlock(&s->mtx); - - // Schedule a get. In polyamorous mode we get on the per pipe - // send_queue, as the socket distributes to us. In monogamous mode - // we bypass and get from the upper write queue directly (saving a - // set of context switches). - nni_msgq_aio_get(p->send_queue, &p->aio_get); - - // And the pipe read of course. - nni_pipe_recv(p->pipe, &p->aio_recv); - - return (0); -} - -static void -pair1poly_pipe_close(void *arg) -{ - pair1poly_pipe *p = arg; - pair1poly_sock *s = p->pair; - - nni_aio_close(&p->aio_send); - nni_aio_close(&p->aio_recv); - nni_aio_close(&p->aio_put); - nni_aio_close(&p->aio_get); - - nni_mtx_lock(&s->mtx); - nni_id_remove(&s->pipes, nni_pipe_id(p->pipe)); - nni_list_node_remove(&p->node); - nni_mtx_unlock(&s->mtx); - - nni_msgq_close(p->send_queue); -} - -static void -pair1poly_pipe_recv_cb(void *arg) -{ - pair1poly_pipe *p = arg; - pair1poly_sock *s = p->pair; - nni_msg * msg; - uint32_t hdr; - nni_pipe * pipe = p->pipe; - size_t len; - - if (nni_aio_result(&p->aio_recv) != 0) { - nni_pipe_close(p->pipe); - return; - } - - msg = nni_aio_get_msg(&p->aio_recv); - nni_aio_set_msg(&p->aio_recv, NULL); - - // Store the pipe ID. - nni_msg_set_pipe(msg, nni_pipe_id(p->pipe)); - - // If the message is missing the hop count header, scrap it. - if ((nni_msg_len(msg) < sizeof(uint32_t)) || - ((hdr = nni_msg_trim_u32(msg)) > 0xff)) { - BUMP_STAT(&s->stat_rx_malformed); - nni_msg_free(msg); - nni_pipe_close(pipe); - return; - } - - len = nni_msg_len(msg); - - // If we bounced too many times, discard the message, but - // keep getting more. - if ((int) hdr > nni_atomic_get(&s->ttl)) { - BUMP_STAT(&s->stat_ttl_drop); - nni_msg_free(msg); - nni_pipe_recv(pipe, &p->aio_recv); - return; - } - - // Store the hop count in the header. - nni_msg_header_append_u32(msg, hdr); - - // Send the message up. - nni_aio_set_msg(&p->aio_put, msg); - nni_sock_bump_rx(s->sock, len); - nni_msgq_aio_put(s->urq, &p->aio_put); -} - -static void -pair1poly_sock_get_cb(void *arg) -{ - pair1poly_pipe *p; - pair1poly_sock *s = arg; - nni_msg * msg; - uint32_t id; - - if (nni_aio_result(&s->aio_get) != 0) { - // Socket closing... - return; - } - - msg = nni_aio_get_msg(&s->aio_get); - nni_aio_set_msg(&s->aio_get, NULL); - - p = NULL; - nni_mtx_lock(&s->mtx); - // If no pipe was requested, we look for any connected peer. - if (((id = nni_msg_get_pipe(msg)) == 0) && - (!nni_list_empty(&s->plist))) { - p = nni_list_first(&s->plist); - } else { - p = nni_id_get(&s->pipes, id); - } - - // Try a non-blocking send. If this fails we just discard the - // message. We have to do this to avoid head-of-line blocking - // for messages sent to other pipes. Note that there is some - // buffering in the send_queue. - if ((p == NULL) || nni_msgq_tryput(p->send_queue, msg) != 0) { - BUMP_STAT(&s->stat_tx_drop); - nni_msg_free(msg); - } - - nni_mtx_unlock(&s->mtx); - nni_msgq_aio_get(s->uwq, &s->aio_get); -} - -static void -pair1poly_pipe_put_cb(void *arg) -{ - pair1poly_pipe *p = arg; - - if (nni_aio_result(&p->aio_put) != 0) { - nni_msg_free(nni_aio_get_msg(&p->aio_put)); - nni_aio_set_msg(&p->aio_put, NULL); - nni_pipe_close(p->pipe); - return; - } - nni_pipe_recv(p->pipe, &p->aio_recv); -} - -static void -pair1poly_pipe_get_cb(void *arg) -{ - pair1poly_pipe *p = arg; - nni_msg * msg; - - if (nni_aio_result(&p->aio_get) != 0) { - nni_pipe_close(p->pipe); - return; - } - - msg = nni_aio_get_msg(&p->aio_get); - nni_aio_set_msg(&p->aio_get, NULL); - - // Cooked mode messages have no header so we have to add one. - // Strip off any previously existing header, such as when - // replying to messages. - nni_msg_header_clear(msg); - - // Insert the hops header. - nni_msg_header_append_u32(msg, 1); - - nni_aio_set_msg(&p->aio_send, msg); - nni_pipe_send(p->pipe, &p->aio_send); -} - -static void -pair1poly_pipe_send_cb(void *arg) -{ - pair1poly_pipe *p = arg; - - if (nni_aio_result(&p->aio_send) != 0) { - nni_msg_free(nni_aio_get_msg(&p->aio_send)); - nni_aio_set_msg(&p->aio_send, NULL); - nni_pipe_close(p->pipe); - return; - } - - nni_msgq_aio_get(p->send_queue, &p->aio_get); -} - -static void -pair1poly_sock_open(void *arg) -{ - NNI_ARG_UNUSED(arg); -} - -static void -pair1poly_sock_close(void *arg) -{ - pair1poly_sock *s = arg; - nni_aio_close(&s->aio_get); -} - -static int -pair1poly_set_max_ttl(void *arg, const void *buf, size_t sz, nni_opt_type t) -{ - pair1poly_sock *s = arg; - int rv; - int ttl; - - if ((rv = nni_copyin_int(&ttl, buf, sz, 1, NNI_MAX_MAX_TTL, t)) == 0) { - nni_atomic_set(&s->ttl, ttl); - } - - return (rv); -} - -static int -pair1poly_get_max_ttl(void *arg, void *buf, size_t *szp, nni_opt_type t) -{ - pair1poly_sock *s = arg; - return (nni_copyout_int(nni_atomic_get(&s->ttl), buf, szp, t)); -} - -static int -pair1poly_get_poly(void *arg, void *buf, size_t *szp, nni_opt_type t) -{ - NNI_ARG_UNUSED(arg); - return (nni_copyout_bool(true, buf, szp, t)); -} - -static void -pair1poly_sock_send(void *arg, nni_aio *aio) -{ - pair1poly_sock *s = arg; - - nni_sock_bump_tx(s->sock, nni_msg_len(nni_aio_get_msg(aio))); - nni_msgq_aio_put(s->uwq, aio); -} - -static void -pair1poly_sock_recv(void *arg, nni_aio *aio) -{ - pair1poly_sock *s = arg; - - nni_msgq_aio_get(s->urq, aio); -} - -static nni_proto_pipe_ops pair1poly_pipe_ops = { - .pipe_size = sizeof(pair1poly_pipe), - .pipe_init = pair1poly_pipe_init, - .pipe_fini = pair1poly_pipe_fini, - .pipe_start = pair1poly_pipe_start, - .pipe_close = pair1poly_pipe_close, - .pipe_stop = pair1poly_pipe_stop, -}; - -static nni_option pair1poly_sock_options[] = { - { - .o_name = NNG_OPT_MAXTTL, - .o_get = pair1poly_get_max_ttl, - .o_set = pair1poly_set_max_ttl, - }, - { - .o_name = NNG_OPT_PAIR1_POLY, - .o_get = pair1poly_get_poly, - }, - // terminate list - { - .o_name = NULL, - }, -}; - -static nni_proto_sock_ops pair1poly_sock_ops = { - .sock_size = sizeof(pair1poly_sock), - .sock_init = pair1poly_sock_init, - .sock_fini = pair1poly_sock_fini, - .sock_open = pair1poly_sock_open, - .sock_close = pair1poly_sock_close, - .sock_recv = pair1poly_sock_recv, - .sock_send = pair1poly_sock_send, - .sock_options = pair1poly_sock_options, -}; - -static nni_proto pair1poly_proto = { - .proto_version = NNI_PROTOCOL_VERSION, - .proto_self = { NNG_PAIR1_SELF, NNG_PAIR1_SELF_NAME }, - .proto_peer = { NNG_PAIR1_PEER, NNG_PAIR1_PEER_NAME }, - .proto_flags = NNI_PROTO_FLAG_SNDRCV, - .proto_sock_ops = &pair1poly_sock_ops, - .proto_pipe_ops = &pair1poly_pipe_ops, -}; - -int -nng_pair1_open_poly(nng_socket *sock) -{ - return (nni_proto_open(sock, &pair1poly_proto)); -} diff --git a/src/protocol/pair1/pair1_poly_test.c b/src/protocol/pair1/pair1_poly_test.c deleted file mode 100644 index f26f7809..00000000 --- a/src/protocol/pair1/pair1_poly_test.c +++ /dev/null @@ -1,370 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// Copyright 2017 Capitar IT Group BV -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - -#include - -#define SECOND 1000 - -#define APPEND_STR(m, s) NUTS_PASS(nng_msg_append(m, s, strlen(s))) -#define CHECK_STR(m, s) \ - NUTS_TRUE(nng_msg_len(m) == strlen(s)); \ - NUTS_TRUE(memcmp(nng_msg_body(m), s, strlen(s)) == 0) - -static void -test_poly_identity(void) -{ - nng_socket s; - int p; - char * n; - - NUTS_PASS(nng_pair1_open_poly(&s)); - NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PROTO, &p)); - NUTS_TRUE(p == NUTS_PROTO(1u, 1u)); // 32 - NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PEER, &p)); - NUTS_TRUE(p == NUTS_PROTO(1u, 1u)); // 33 - NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PROTONAME, &n)); - NUTS_MATCH(n, "pair1"); - nng_strfree(n); - NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PEERNAME, &n)); - NUTS_MATCH(n, "pair1"); - nng_strfree(n); - NUTS_CLOSE(s); -} - -void -test_poly_best_effort(void) -{ - nng_socket s1; - nng_socket c1; - nng_msg * msg; - - NUTS_PASS(nng_pair1_open_poly(&s1)); - NUTS_PASS(nng_pair1_open(&c1)); - - NUTS_PASS(nng_setopt_int(s1, NNG_OPT_RECVBUF, 1)); - NUTS_PASS(nng_setopt_int(s1, NNG_OPT_SENDBUF, 1)); - NUTS_PASS(nng_setopt_int(c1, NNG_OPT_RECVBUF, 1)); - NUTS_PASS(nng_setopt_ms(s1, NNG_OPT_SENDTIMEO, SECOND)); - - NUTS_MARRY(s1, c1); - - for (int i = 0; i < 10; i++) { - NUTS_PASS(nng_msg_alloc(&msg, 0)); - NUTS_PASS(nng_sendmsg(s1, msg, 0)); - } - - NUTS_CLOSE(s1); - NUTS_CLOSE(c1); -} - -void -test_poly_cooked(void) -{ - nng_socket s1; - nng_socket c1; - nng_socket c2; - nng_msg * msg; - bool v; - nng_pipe p1; - nng_pipe p2; - - NUTS_PASS(nng_pair1_open_poly(&s1)); - NUTS_PASS(nng_pair1_open(&c1)); - NUTS_PASS(nng_pair1_open(&c2)); - NUTS_PASS(nng_setopt_ms(s1, NNG_OPT_SENDTIMEO, SECOND)); - NUTS_PASS(nng_setopt_ms(c1, NNG_OPT_SENDTIMEO, SECOND)); - NUTS_PASS(nng_setopt_ms(c2, NNG_OPT_SENDTIMEO, SECOND)); - NUTS_PASS(nng_setopt_ms(s1, NNG_OPT_RECVTIMEO, SECOND / 10)); - NUTS_PASS(nng_setopt_ms(c1, NNG_OPT_RECVTIMEO, SECOND / 10)); - NUTS_PASS(nng_setopt_ms(c2, NNG_OPT_RECVTIMEO, SECOND / 10)); - - NUTS_PASS(nng_getopt_bool(s1, NNG_OPT_PAIR1_POLY, &v)); - NUTS_TRUE(v); - - NUTS_MARRY(s1, c1); - NUTS_MARRY(s1, c2); - - NUTS_PASS(nng_msg_alloc(&msg, 0)); - APPEND_STR(msg, "ONE"); - NUTS_PASS(nng_sendmsg(c1, msg, 0)); - NUTS_PASS(nng_recvmsg(s1, &msg, 0)); - CHECK_STR(msg, "ONE"); - p1 = nng_msg_get_pipe(msg); - NUTS_TRUE(nng_pipe_id(p1) > 0); - nng_msg_free(msg); - - NUTS_PASS(nng_msg_alloc(&msg, 0)); - APPEND_STR(msg, "TWO"); - NUTS_PASS(nng_sendmsg(c2, msg, 0)); - NUTS_PASS(nng_recvmsg(s1, &msg, 0)); - CHECK_STR(msg, "TWO"); - p2 = nng_msg_get_pipe(msg); - NUTS_TRUE(nng_pipe_id(p2) > 0); - nng_msg_free(msg); - - NUTS_TRUE(nng_pipe_id(p1) != nng_pipe_id(p2)); - - NUTS_PASS(nng_msg_alloc(&msg, 0)); - - nng_msg_set_pipe(msg, p1); - APPEND_STR(msg, "UNO"); - NUTS_PASS(nng_sendmsg(s1, msg, 0)); - NUTS_PASS(nng_recvmsg(c1, &msg, 0)); - CHECK_STR(msg, "UNO"); - nng_msg_free(msg); - - NUTS_PASS(nng_msg_alloc(&msg, 0)); - nng_msg_set_pipe(msg, p2); - APPEND_STR(msg, "DOS"); - NUTS_PASS(nng_sendmsg(s1, msg, 0)); - NUTS_PASS(nng_recvmsg(c2, &msg, 0)); - CHECK_STR(msg, "DOS"); - nng_msg_free(msg); - - NUTS_PASS(nng_close(c1)); - - NUTS_PASS(nng_msg_alloc(&msg, 0)); - nng_msg_set_pipe(msg, p1); - APPEND_STR(msg, "EIN"); - NUTS_PASS(nng_sendmsg(s1, msg, 0)); - NUTS_FAIL(nng_recvmsg(c2, &msg, 0), NNG_ETIMEDOUT); - - NUTS_CLOSE(s1); - NUTS_CLOSE(c2); -} - -void -test_poly_default(void) -{ - nng_socket s1; - nng_socket c1; - nng_socket c2; - nng_msg * msg; - - NUTS_PASS(nng_pair1_open_poly(&s1)); - NUTS_PASS(nng_pair1_open(&c1)); - NUTS_PASS(nng_pair1_open(&c2)); - NUTS_PASS(nng_setopt_ms(s1, NNG_OPT_SENDTIMEO, SECOND)); - NUTS_PASS(nng_setopt_ms(c1, NNG_OPT_SENDTIMEO, SECOND)); - NUTS_PASS(nng_setopt_ms(c2, NNG_OPT_SENDTIMEO, SECOND)); - - NUTS_MARRY(s1, c1); - NUTS_MARRY(s1, c2); - - // This assumes poly picks the first suitor. Applications - // should not make the same assumption. - NUTS_PASS(nng_msg_alloc(&msg, 0)); - APPEND_STR(msg, "YES"); - NUTS_PASS(nng_sendmsg(s1, msg, 0)); - NUTS_PASS(nng_recvmsg(c1, &msg, 0)); - CHECK_STR(msg, "YES"); - nng_msg_free(msg); - - NUTS_CLOSE(c1); - NUTS_SLEEP(10); - - // Verify that the other pipe is chosen as the next suitor. - NUTS_PASS(nng_msg_alloc(&msg, 0)); - APPEND_STR(msg, "AGAIN"); - NUTS_PASS(nng_sendmsg(s1, msg, 0)); - NUTS_PASS(nng_recvmsg(c2, &msg, 0)); - CHECK_STR(msg, "AGAIN"); - nng_msg_free(msg); - - NUTS_CLOSE(s1); - NUTS_CLOSE(c2); -} - -void -test_poly_close_abort(void) -{ - nng_socket s; - nng_socket c; - - NUTS_PASS(nng_pair1_open_poly(&s)); - NUTS_PASS(nng_pair1_open(&c)); - NUTS_PASS(nng_setopt_ms(s, NNG_OPT_RECVTIMEO, 100)); - NUTS_PASS(nng_setopt_ms(s, NNG_OPT_SENDTIMEO, 200)); - NUTS_PASS(nng_setopt_int(s, NNG_OPT_RECVBUF, 1)); - NUTS_PASS(nng_setopt_int(c, NNG_OPT_SENDBUF, 20)); - - NUTS_MARRY(c, s); - - for (int i = 0; i < 20; i++) { - NUTS_SEND(c, "TEST"); - } - NUTS_SLEEP(50); - - NUTS_CLOSE(s); - NUTS_CLOSE(c); -} - -void -test_poly_recv_no_header(void) -{ - nng_socket s; - nng_socket c; - nng_msg * m; - - NUTS_PASS(nng_pair1_open_poly(&s)); - NUTS_PASS(nng_pair1_open(&c)); - NUTS_PASS(nng_setopt_bool(c, "pair1_test_inject_header", true)); - NUTS_PASS(nng_setopt_ms(s, NNG_OPT_RECVTIMEO, 100)); - NUTS_PASS(nng_setopt_ms(s, NNG_OPT_SENDTIMEO, 200)); - - NUTS_MARRY(c, s); - - NUTS_PASS(nng_msg_alloc(&m, 0)); - NUTS_PASS(nng_sendmsg(c, m, 0)); - NUTS_FAIL(nng_recvmsg(s, &m, 0), NNG_ETIMEDOUT); - - NUTS_CLOSE(c); - NUTS_CLOSE(s); -} - -void -test_poly_recv_garbage(void) -{ - nng_socket s; - nng_socket c; - nng_msg * m; - - NUTS_PASS(nng_pair1_open_poly(&s)); - NUTS_PASS(nng_pair1_open(&c)); - NUTS_PASS(nng_setopt_bool(c, "pair1_test_inject_header", true)); - NUTS_PASS(nng_setopt_ms(s, NNG_OPT_RECVTIMEO, 100)); - NUTS_PASS(nng_setopt_ms(s, NNG_OPT_SENDTIMEO, 200)); - - NUTS_MARRY(c, s); - - // ridiculous hop count - NUTS_PASS(nng_msg_alloc(&m, 0)); - NUTS_PASS(nng_msg_append_u32(m, 0x1000)); - NUTS_PASS(nng_sendmsg(c, m, 0)); - NUTS_FAIL(nng_recvmsg(s, &m, 0), NNG_ETIMEDOUT); - - NUTS_CLOSE(c); - NUTS_CLOSE(s); -} - -void -test_poly_ttl(void) -{ - nng_socket s1; - nng_socket c1; - nng_msg * msg; - uint32_t val; - int ttl; - - NUTS_PASS(nng_pair1_open_poly(&s1)); - NUTS_PASS(nng_pair1_open_raw(&c1)); - NUTS_PASS(nng_setopt_ms(s1, NNG_OPT_RECVTIMEO, SECOND / 5)); - NUTS_PASS(nng_setopt_ms(c1, NNG_OPT_RECVTIMEO, SECOND / 5)); - - // cannot set insane TTLs - NUTS_FAIL(nng_setopt_int(s1, NNG_OPT_MAXTTL, 0), NNG_EINVAL); - NUTS_FAIL(nng_setopt_int(s1, NNG_OPT_MAXTTL, 1000), NNG_EINVAL); - ttl = 8; - NUTS_FAIL(nng_setopt(s1, NNG_OPT_MAXTTL, &ttl, 1), NNG_EINVAL); - NUTS_FAIL(nng_setopt_bool(s1, NNG_OPT_MAXTTL, true), NNG_EBADTYPE); - - NUTS_MARRY(s1, c1); - - // Let's check enforcement of TTL - NUTS_PASS(nng_setopt_int(s1, NNG_OPT_MAXTTL, 4)); - NUTS_PASS(nng_getopt_int(s1, NNG_OPT_MAXTTL, &ttl)); - NUTS_TRUE(ttl == 4); - - // Bad TTL bounces - NUTS_PASS(nng_msg_alloc(&msg, 0)); - NUTS_PASS(nng_msg_header_append_u32(msg, 4)); - NUTS_PASS(nng_sendmsg(c1, msg, 0)); - NUTS_FAIL(nng_recvmsg(s1, &msg, 0), NNG_ETIMEDOUT); - - // Good TTL passes - NUTS_PASS(nng_msg_alloc(&msg, 0)); - NUTS_PASS(nng_msg_append_u32(msg, 0xFEEDFACE)); - NUTS_PASS(nng_msg_header_append_u32(msg, 3)); - NUTS_PASS(nng_sendmsg(c1, msg, 0)); - NUTS_PASS(nng_recvmsg(s1, &msg, 0)); - NUTS_PASS(nng_msg_trim_u32(msg, &val)); - NUTS_TRUE(val == 0xFEEDFACE); - NUTS_PASS(nng_msg_header_trim_u32(msg, &val)); - NUTS_TRUE(val == 4); - nng_msg_free(msg); - - // Large TTL passes - NUTS_PASS(nng_setopt_int(s1, NNG_OPT_MAXTTL, 15)); - NUTS_PASS(nng_msg_alloc(&msg, 0)); - NUTS_PASS(nng_msg_append_u32(msg, 1234)); - NUTS_PASS(nng_msg_header_append_u32(msg, 14)); - NUTS_PASS(nng_sendmsg(c1, msg, 0)); - NUTS_PASS(nng_recvmsg(s1, &msg, 0)); - NUTS_PASS(nng_msg_trim_u32(msg, &val)); - NUTS_TRUE(val == 1234); - NUTS_PASS(nng_msg_header_trim_u32(msg, &val)); - NUTS_TRUE(val == 15); - nng_msg_free(msg); - - // Max TTL fails - NUTS_PASS(nng_setopt_int(s1, NNG_OPT_MAXTTL, 15)); - NUTS_PASS(nng_msg_alloc(&msg, 0)); - NUTS_PASS(nng_msg_header_append_u32(msg, 15)); - NUTS_PASS(nng_sendmsg(c1, msg, 0)); - NUTS_FAIL(nng_recvmsg(s1, &msg, 0), NNG_ETIMEDOUT); - - NUTS_CLOSE(s1); - NUTS_CLOSE(c1); -} - -void -test_poly_validate_peer(void) -{ - nng_socket s1, s2; - nng_stat * stats; - nng_stat * reject; - char * addr; - - NUTS_ADDR(addr, "inproc"); - - NUTS_PASS(nng_pair1_open_poly(&s1)); - NUTS_PASS(nng_pair0_open(&s2)); - - NUTS_PASS(nng_listen(s1, addr, NULL, 0)); - NUTS_PASS(nng_dial(s2, addr, NULL, NNG_FLAG_NONBLOCK)); - - NUTS_SLEEP(100); - NUTS_PASS(nng_stats_get(&stats)); - - NUTS_TRUE(stats != NULL); - NUTS_TRUE((reject = nng_stat_find_socket(stats, s1)) != NULL); - NUTS_TRUE((reject = nng_stat_find(reject, "reject")) != NULL); - - NUTS_TRUE(nng_stat_type(reject) == NNG_STAT_COUNTER); - NUTS_TRUE(nng_stat_value(reject) > 0); - - NUTS_CLOSE(s1); - NUTS_CLOSE(s2); - nng_stats_free(stats); -} - -TEST_LIST = { - { "pair1 poly identity", test_poly_identity }, - { "pair1 poly best effort", test_poly_best_effort }, - { "pair1 poly cooked", test_poly_cooked }, - { "pair1 poly default", test_poly_default }, - { "pair1 poly recv no header", test_poly_recv_no_header }, - { "pair1 poly recv garbage", test_poly_recv_garbage }, - { "pair1 poly ttl", test_poly_ttl }, - { "pair1 poly close abort", test_poly_close_abort }, - { "pair1 poly validate peer", test_poly_validate_peer }, - - { NULL, NULL }, -}; diff --git a/src/protocol/pair1/pair1_test.c b/src/protocol/pair1/pair1_test.c deleted file mode 100644 index 881c4ac8..00000000 --- a/src/protocol/pair1/pair1_test.c +++ /dev/null @@ -1,433 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// Copyright 2017 Capitar IT Group BV -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - -#include - -#define SECOND 1000 - -#define APPEND_STR(m, s) NUTS_TRUE(nng_msg_append(m, s, strlen(s)) == 0) -#define CHECK_STR(m, s) \ - NUTS_TRUE(nng_msg_len(m) == strlen(s)); \ - NUTS_TRUE(memcmp(nng_msg_body(m), s, strlen(s)) == 0) - -static void -test_mono_identity(void) -{ - nng_socket s; - int p; - char * n; - - NUTS_PASS(nng_pair1_open(&s)); - NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PROTO, &p)); - NUTS_TRUE(p == NUTS_PROTO(1u, 1u)); // 32 - NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PEER, &p)); - NUTS_TRUE(p == NUTS_PROTO(1u, 1u)); // 33 - NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PROTONAME, &n)); - NUTS_MATCH(n, "pair1"); - nng_strfree(n); - NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PEERNAME, &n)); - NUTS_MATCH(n, "pair1"); - nng_strfree(n); - NUTS_CLOSE(s); -} - -void -test_mono_cooked(void) -{ - nng_socket s1; - nng_socket c1; - nng_msg * msg; - - NUTS_PASS(nng_pair1_open(&s1)); - NUTS_PASS(nng_pair1_open(&c1)); - NUTS_PASS(nuts_marry(s1, c1)); - - NUTS_PASS(nng_msg_alloc(&msg, 0)); - NUTS_PASS(nng_msg_append(msg, "ALPHA", strlen("ALPHA") + 1)); - NUTS_PASS(nng_sendmsg(c1, msg, 0)); - NUTS_PASS(nng_recvmsg(s1, &msg, 0)); - NUTS_TRUE(nng_msg_len(msg) == strlen("ALPHA") + 1); - NUTS_MATCH(nng_msg_body(msg), "ALPHA"); - nng_msg_free(msg); - - NUTS_PASS(nng_msg_alloc(&msg, 0)); - NUTS_PASS(nng_msg_append(msg, "BETA", strlen("BETA") + 1)); - NUTS_PASS(nng_sendmsg(s1, msg, 0)); - NUTS_PASS(nng_recvmsg(c1, &msg, 0)); - NUTS_TRUE(nng_msg_len(msg) == strlen("BETA") + 1); - NUTS_MATCH(nng_msg_body(msg), "BETA"); - - nng_msg_free(msg); - NUTS_CLOSE(c1); - NUTS_CLOSE(s1); -} - -void -test_mono_faithful(void) -{ - nng_socket s1; - nng_socket c1; - nng_socket c2; - nng_msg * msg; - const char *addr = "inproc://pair1_mono_faithful"; - - NUTS_PASS(nng_pair1_open(&s1)); - NUTS_PASS(nng_pair1_open(&c1)); - NUTS_PASS(nng_pair1_open(&c2)); - NUTS_PASS(nng_setopt_ms(s1, NNG_OPT_RECVTIMEO, SECOND / 4)); - NUTS_PASS(nng_setopt_ms(c1, NNG_OPT_SENDTIMEO, SECOND)); - NUTS_PASS(nng_setopt_ms(c2, NNG_OPT_SENDTIMEO, SECOND)); - NUTS_PASS(nng_setopt_int(c2, NNG_OPT_SENDBUF, 2)); - - NUTS_PASS(nng_listen(s1, addr, NULL, 0)); - NUTS_MARRY(s1, c1); - NUTS_PASS(nng_dial(c2, addr, NULL, 0)); - - NUTS_SLEEP(100); - - NUTS_PASS(nng_msg_alloc(&msg, 0)); - APPEND_STR(msg, "ONE"); - NUTS_PASS(nng_sendmsg(c1, msg, 0)); - NUTS_PASS(nng_recvmsg(s1, &msg, 0)); - CHECK_STR(msg, "ONE"); - nng_msg_free(msg); - - NUTS_PASS(nng_msg_alloc(&msg, 0)); - APPEND_STR(msg, "TWO"); - NUTS_PASS(nng_sendmsg(c2, msg, 0)); - NUTS_FAIL(nng_recvmsg(s1, &msg, 0), NNG_ETIMEDOUT); - - NUTS_CLOSE(s1); - NUTS_CLOSE(c1); - NUTS_CLOSE(c2); -} - -void -test_mono_back_pressure(void) -{ - nng_socket s1; - nng_socket c1; - int i; - int rv; - nng_msg * msg; - nng_duration to = 100; - - NUTS_PASS(nng_pair1_open(&s1)); - NUTS_PASS(nng_pair1_open(&c1)); - NUTS_PASS(nng_setopt_int(s1, NNG_OPT_RECVBUF, 1)); - NUTS_PASS(nng_setopt_int(s1, NNG_OPT_SENDBUF, 1)); - NUTS_PASS(nng_setopt_int(c1, NNG_OPT_RECVBUF, 1)); - NUTS_PASS(nng_setopt_ms(s1, NNG_OPT_SENDTIMEO, to)); - - NUTS_MARRY(s1, c1); - - // We choose to allow some buffering. In reality the - // buffer size is just 1, and we will fail after 2. - for (i = 0, rv = 0; i < 10; i++) { - NUTS_PASS(nng_msg_alloc(&msg, 0)); - if ((rv = nng_sendmsg(s1, msg, 0)) != 0) { - nng_msg_free(msg); - break; - } - } - NUTS_FAIL(rv, NNG_ETIMEDOUT); - NUTS_TRUE(i < 10); - NUTS_CLOSE(s1); - NUTS_CLOSE(c1); -} - -void -test_mono_raw_exchange(void) -{ - nng_socket s1; - nng_socket c1; - - nng_msg *msg; - uint32_t hops; - - NUTS_PASS(nng_pair1_open_raw(&s1)); - NUTS_PASS(nng_pair1_open_raw(&c1)); - - NUTS_PASS(nng_setopt_ms(s1, NNG_OPT_RECVTIMEO, SECOND)); - NUTS_PASS(nng_setopt_ms(c1, NNG_OPT_RECVTIMEO, SECOND)); - NUTS_MARRY(s1, c1); - - nng_pipe p = NNG_PIPE_INITIALIZER; - NUTS_PASS(nng_msg_alloc(&msg, 0)); - APPEND_STR(msg, "GAMMA"); - NUTS_PASS(nng_msg_header_append_u32(msg, 1)); - NUTS_TRUE(nng_msg_header_len(msg) == sizeof(uint32_t)); - NUTS_PASS(nng_sendmsg(c1, msg, 0)); - NUTS_PASS(nng_recvmsg(s1, &msg, 0)); - p = nng_msg_get_pipe(msg); - NUTS_TRUE(nng_pipe_id(p) > 0); - - CHECK_STR(msg, "GAMMA"); - NUTS_TRUE(nng_msg_header_len(msg) == sizeof(uint32_t)); - NUTS_PASS(nng_msg_header_trim_u32(msg, &hops)); - NUTS_TRUE(hops == 2); - nng_msg_free(msg); - - NUTS_PASS(nng_msg_alloc(&msg, 0)); - APPEND_STR(msg, "EPSILON"); - NUTS_PASS(nng_msg_header_append_u32(msg, 1)); - NUTS_PASS(nng_sendmsg(s1, msg, 0)); - NUTS_PASS(nng_recvmsg(c1, &msg, 0)); - CHECK_STR(msg, "EPSILON"); - NUTS_TRUE(nng_msg_header_len(msg) == sizeof(uint32_t)); - NUTS_PASS(nng_msg_header_trim_u32(msg, &hops)); - p = nng_msg_get_pipe(msg); - NUTS_TRUE(nng_pipe_id(p) > 0); - - NUTS_TRUE(hops == 2); - nng_msg_free(msg); - - NUTS_CLOSE(s1); - NUTS_CLOSE(c1); -} - -void -test_mono_raw_header(void) -{ - nng_socket s1; - nng_socket c1; - nng_msg * msg; - uint32_t v; - - NUTS_PASS(nng_pair1_open_raw(&s1)); - NUTS_PASS(nng_pair1_open_raw(&c1)); - - NUTS_PASS(nng_setopt_ms(s1, NNG_OPT_RECVTIMEO, SECOND / 5)); - NUTS_PASS(nng_setopt_ms(c1, NNG_OPT_RECVTIMEO, SECOND / 5)); - NUTS_MARRY(s1, c1); - - // Missing bits in the header - NUTS_PASS(nng_msg_alloc(&msg, 0)); - NUTS_PASS(nng_sendmsg(c1, msg, 0)); - NUTS_FAIL(nng_recvmsg(s1, &msg, 0), NNG_ETIMEDOUT); - - // Valid header works - NUTS_PASS(nng_msg_alloc(&msg, 0)); - NUTS_PASS(nng_msg_append_u32(msg, 0xFEEDFACE)); - NUTS_PASS(nng_msg_header_append_u32(msg, 1)); - NUTS_PASS(nng_sendmsg(c1, msg, 0)); - NUTS_PASS(nng_recvmsg(s1, &msg, 0)); - NUTS_PASS(nng_msg_trim_u32(msg, &v)); - NUTS_TRUE(v == 0xFEEDFACE); - nng_msg_free(msg); - - // Header with reserved bits set dropped - NUTS_PASS(nng_msg_alloc(&msg, 0)); - NUTS_PASS(nng_msg_header_append_u32(msg, 0xDEAD0000)); - NUTS_PASS(nng_sendmsg(c1, msg, 0)); - NUTS_FAIL(nng_recvmsg(s1, &msg, 0), NNG_ETIMEDOUT); - - // Header with no chance to add another hop gets dropped - NUTS_PASS(nng_msg_alloc(&msg, 0)); - NUTS_PASS(nng_msg_header_append_u32(msg, 0xff)); - NUTS_PASS(nng_sendmsg(c1, msg, 0)); - NUTS_FAIL(nng_recvmsg(s1, &msg, 0), NNG_ETIMEDOUT); - - // With the same bits clear it works - NUTS_PASS(nng_msg_alloc(&msg, 0)); - NUTS_PASS(nng_msg_append_u32(msg, 0xFEEDFACE)); - NUTS_PASS(nng_msg_header_append_u32(msg, 1)); - NUTS_PASS(nng_sendmsg(c1, msg, 0)); - NUTS_PASS(nng_recvmsg(s1, &msg, 0)); - NUTS_PASS(nng_msg_trim_u32(msg, &v)); - NUTS_TRUE(v == 0xFEEDFACE); - nng_msg_free(msg); - - NUTS_CLOSE(s1); - NUTS_CLOSE(c1); -} - -void -test_pair1_raw(void) -{ - nng_socket s1; - bool raw; - - NUTS_PASS(nng_pair1_open(&s1)); - NUTS_PASS(nng_getopt_bool(s1, NNG_OPT_RAW, &raw)); - NUTS_TRUE(raw == false); - NUTS_FAIL(nng_setopt_bool(s1, NNG_OPT_RAW, true), NNG_EREADONLY); - NUTS_PASS(nng_close(s1)); - - NUTS_PASS(nng_pair1_open_raw(&s1)); - NUTS_PASS(nng_getopt_bool(s1, NNG_OPT_RAW, &raw)); - NUTS_TRUE(raw == true); - NUTS_FAIL(nng_setopt_bool(s1, NNG_OPT_RAW, false), NNG_EREADONLY); - NUTS_PASS(nng_close(s1)); -} - -void -test_pair1_ttl(void) -{ - nng_socket s1; - nng_socket c1; - nng_msg * msg; - uint32_t val; - int ttl; - - NUTS_PASS(nng_pair1_open_raw(&s1)); - NUTS_PASS(nng_pair1_open_raw(&c1)); - NUTS_PASS(nng_setopt_ms(s1, NNG_OPT_RECVTIMEO, SECOND / 5)); - NUTS_PASS(nng_setopt_ms(c1, NNG_OPT_RECVTIMEO, SECOND / 5)); - - // cannot set insane TTLs - NUTS_FAIL(nng_setopt_int(s1, NNG_OPT_MAXTTL, 0), NNG_EINVAL); - NUTS_FAIL(nng_setopt_int(s1, NNG_OPT_MAXTTL, 1000), NNG_EINVAL); - ttl = 8; - NUTS_FAIL(nng_setopt(s1, NNG_OPT_MAXTTL, &ttl, 1), NNG_EINVAL); - NUTS_FAIL(nng_setopt_bool(s1, NNG_OPT_MAXTTL, true), NNG_EBADTYPE); - - NUTS_MARRY(s1, c1); - - // Let's check enforcement of TTL - NUTS_PASS(nng_setopt_int(s1, NNG_OPT_MAXTTL, 4)); - NUTS_PASS(nng_getopt_int(s1, NNG_OPT_MAXTTL, &ttl)); - NUTS_TRUE(ttl == 4); - - // Bad TTL bounces - NUTS_PASS(nng_msg_alloc(&msg, 0)); - NUTS_PASS(nng_msg_header_append_u32(msg, 4)); - NUTS_PASS(nng_sendmsg(c1, msg, 0)); - NUTS_FAIL(nng_recvmsg(s1, &msg, 0), NNG_ETIMEDOUT); - - // Good TTL passes - NUTS_PASS(nng_msg_alloc(&msg, 0)); - NUTS_PASS(nng_msg_append_u32(msg, 0xFEEDFACE)); - NUTS_PASS(nng_msg_header_append_u32(msg, 3)); - NUTS_PASS(nng_sendmsg(c1, msg, 0)); - NUTS_PASS(nng_recvmsg(s1, &msg, 0)); - NUTS_PASS(nng_msg_trim_u32(msg, &val)); - NUTS_TRUE(val == 0xFEEDFACE); - NUTS_PASS(nng_msg_header_trim_u32(msg, &val)); - NUTS_TRUE(val == 4); - nng_msg_free(msg); - - // Large TTL passes - NUTS_PASS(nng_setopt_int(s1, NNG_OPT_MAXTTL, 15)); - NUTS_PASS(nng_msg_alloc(&msg, 0)); - NUTS_PASS(nng_msg_append_u32(msg, 1234)); - NUTS_PASS(nng_msg_header_append_u32(msg, 14)); - NUTS_PASS(nng_sendmsg(c1, msg, 0)); - NUTS_PASS(nng_recvmsg(s1, &msg, 0)); - NUTS_PASS(nng_msg_trim_u32(msg, &val)); - NUTS_TRUE(val == 1234); - NUTS_PASS(nng_msg_header_trim_u32(msg, &val)); - NUTS_TRUE(val == 15); - nng_msg_free(msg); - - // Max TTL fails - NUTS_PASS(nng_setopt_int(s1, NNG_OPT_MAXTTL, 15)); - NUTS_PASS(nng_msg_alloc(&msg, 0)); - NUTS_PASS(nng_msg_header_append_u32(msg, 15)); - NUTS_PASS(nng_sendmsg(c1, msg, 0)); - NUTS_FAIL(nng_recvmsg(s1, &msg, 0), NNG_ETIMEDOUT); - - NUTS_CLOSE(s1); - NUTS_CLOSE(c1); -} - -void -test_pair1_validate_peer(void) -{ - nng_socket s1, s2; - nng_stat * stats; - nng_stat * reject; - char * addr; - - NUTS_ADDR(addr, "inproc"); - NUTS_PASS(nng_pair1_open(&s1)); - NUTS_PASS(nng_pair0_open(&s2)); - - NUTS_PASS(nng_listen(s1, addr, NULL, 0)); - NUTS_PASS(nng_dial(s2, addr, NULL, NNG_FLAG_NONBLOCK)); - - NUTS_SLEEP(100); - NUTS_PASS(nng_stats_get(&stats)); - - NUTS_TRUE(stats != NULL); - NUTS_TRUE((reject = nng_stat_find_socket(stats, s1)) != NULL); - NUTS_TRUE((reject = nng_stat_find(reject, "reject")) != NULL); - - NUTS_TRUE(nng_stat_type(reject) == NNG_STAT_COUNTER); - NUTS_TRUE(nng_stat_value(reject) > 0); - - NUTS_CLOSE(s1); - NUTS_CLOSE(s2); - nng_stats_free(stats); -} - -void -test_pair1_recv_no_header(void) -{ - nng_socket s; - nng_socket c; - nng_msg * m; - - NUTS_PASS(nng_pair1_open(&s)); - NUTS_PASS(nng_pair1_open(&c)); - NUTS_PASS(nng_setopt_bool(c, "pair1_test_inject_header", true)); - NUTS_PASS(nng_setopt_ms(s, NNG_OPT_RECVTIMEO, 100)); - NUTS_PASS(nng_setopt_ms(s, NNG_OPT_SENDTIMEO, 200)); - - NUTS_MARRY(c, s); - - NUTS_PASS(nng_msg_alloc(&m, 0)); - NUTS_PASS(nng_sendmsg(c, m, 0)); - NUTS_FAIL(nng_recvmsg(s, &m, 0), NNG_ETIMEDOUT); - - NUTS_CLOSE(c); - NUTS_CLOSE(s); -} - -void -test_pair1_recv_garbage(void) -{ - nng_socket s; - nng_socket c; - nng_msg * m; - - NUTS_PASS(nng_pair1_open(&s)); - NUTS_PASS(nng_pair1_open(&c)); - NUTS_PASS(nng_setopt_bool(c, "pair1_test_inject_header", true)); - NUTS_PASS(nng_setopt_ms(s, NNG_OPT_RECVTIMEO, 100)); - NUTS_PASS(nng_setopt_ms(s, NNG_OPT_SENDTIMEO, 200)); - - NUTS_MARRY(c, s); - - // ridiculous hop count - NUTS_PASS(nng_msg_alloc(&m, 0)); - NUTS_PASS(nng_msg_append_u32(m, 0x1000)); - NUTS_PASS(nng_sendmsg(c, m, 0)); - NUTS_FAIL(nng_recvmsg(s, &m, 0), NNG_ETIMEDOUT); - - NUTS_CLOSE(c); - NUTS_CLOSE(s); -} - -NUTS_TESTS = { - { "pair1 mono identity", test_mono_identity }, - { "pair1 mono cooked", test_mono_cooked }, - { "pair1 mono faithful", test_mono_faithful }, - { "pair1 mono back pressure", test_mono_back_pressure }, - { "pair1 mono raw exchange", test_mono_raw_exchange }, - { "pair1 mono raw header", test_mono_raw_header }, - { "pair1 raw", test_pair1_raw }, - { "pair1 ttl", test_pair1_ttl }, - { "pair1 validate peer", test_pair1_validate_peer }, - { "pair1 recv no header", test_pair1_recv_no_header }, - { "pair1 recv garbage", test_pair1_recv_garbage }, - - { NULL, NULL }, -}; diff --git a/src/protocol/pipeline0/CMakeLists.txt b/src/protocol/pipeline0/CMakeLists.txt deleted file mode 100644 index 8a10eab7..00000000 --- a/src/protocol/pipeline0/CMakeLists.txt +++ /dev/null @@ -1,23 +0,0 @@ -# -# Copyright 2019 Staysail Systems, Inc. -# Copyright 2018 Capitar IT Group BV -# -# This software is supplied under the terms of the MIT License, a -# copy of which should be located in the distribution where this -# file was obtained (LICENSE.txt). A copy of the license may also be -# found online at https://opensource.org/licenses/MIT. -# - -# Pipeline protocol -nng_directory(pipeline0) - -nng_sources_if(NNG_PROTO_PUSH0 push.c) -nng_headers_if(NNG_PROTO_PUSH0 nng/protocol/pipeline0/push.h) -nng_defines_if(NNG_PROTO_PUSH0 NNG_HAVE_PUSH0) - -nng_sources_if(NNG_PROTO_PULL0 pull.c) -nng_headers_if(NNG_PROTO_PULL0 nng/protocol/pipeline0/pull.h) -nng_defines_if(NNG_PROTO_PULL0 NNG_HAVE_PULL0) - -nng_test(pull_test) -nng_test(push_test) diff --git a/src/protocol/pipeline0/pull.c b/src/protocol/pipeline0/pull.c deleted file mode 100644 index 616b0817..00000000 --- a/src/protocol/pipeline0/pull.c +++ /dev/null @@ -1,325 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// Copyright 2018 Capitar IT Group BV -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - -#include - -#include "core/nng_impl.h" -#include "nng/protocol/pipeline0/pull.h" - -// Pull protocol. The PULL protocol is the "read" side of a pipeline. - -#ifndef NNI_PROTO_PULL_V0 -#define NNI_PROTO_PULL_V0 NNI_PROTO(5, 1) -#endif - -#ifndef NNI_PROTO_PUSH_V0 -#define NNI_PROTO_PUSH_V0 NNI_PROTO(5, 0) -#endif - -typedef struct pull0_pipe pull0_pipe; -typedef struct pull0_sock pull0_sock; - -static void pull0_recv_cb(void *); - -// pull0_sock is our per-socket protocol private structure. -struct pull0_sock { - bool raw; - nni_list pl; // pipe list (pipes with data ready) - nni_list rq; // recv queue (aio list) - nni_mtx m; - nni_pollable readable; -}; - -// pull0_pipe is our per-pipe protocol private structure. -struct pull0_pipe { - nni_pipe * p; - pull0_sock * s; - nni_msg * m; - nni_aio aio; - bool closed; - nni_list_node node; -}; - -static int -pull0_sock_init(void *arg, nni_sock *sock) -{ - pull0_sock *s = arg; - NNI_ARG_UNUSED(sock); - - nni_aio_list_init(&s->rq); - NNI_LIST_INIT(&s->pl, pull0_pipe, node); - nni_mtx_init(&s->m); - nni_pollable_init(&s->readable); - return (0); -} - -static void -pull0_sock_fini(void *arg) -{ - pull0_sock *s = arg; - nni_mtx_fini(&s->m); - nni_pollable_fini(&s->readable); -} - -static void -pull0_pipe_stop(void *arg) -{ - pull0_pipe *p = arg; - - nni_aio_stop(&p->aio); -} - -static void -pull0_pipe_fini(void *arg) -{ - pull0_pipe *p = arg; - - nni_aio_fini(&p->aio); - if (p->m) { - nni_msg_free(p->m); - } -} - -static int -pull0_pipe_init(void *arg, nni_pipe *pipe, void *s) -{ - pull0_pipe *p = arg; - - nni_aio_init(&p->aio, pull0_recv_cb, p); - p->p = pipe; - p->s = s; - return (0); -} - -static int -pull0_pipe_start(void *arg) -{ - pull0_pipe *p = arg; - - if (nni_pipe_peer(p->p) != NNI_PROTO_PUSH_V0) { - // Peer protocol mismatch. - return (NNG_EPROTO); - } - - // Start the pending receive... - nni_pipe_recv(p->p, &p->aio); - - return (0); -} - -static void -pull0_pipe_close(void *arg) -{ - pull0_pipe *p = arg; - pull0_sock *s = p->s; - - nni_mtx_lock(&s->m); - p->closed = true; - if (nni_list_node_active(&p->node)) { - nni_list_node_remove(&p->node); - if (nni_list_empty(&s->pl)) { - nni_pollable_clear(&s->readable); - } - } - nni_mtx_unlock(&s->m); - - nni_aio_close(&p->aio); -} - -static void -pull0_recv_cb(void *arg) -{ - pull0_pipe *p = arg; - pull0_sock *s = p->s; - nni_aio * ap = &p->aio; - nni_aio * as; - nni_msg * m; - - if (nni_aio_result(ap) != 0) { - // Failed to get a message, probably the pipe is closed. - nni_pipe_close(p->p); - return; - } - - // Got a message... start the put to send it up to the application. - m = nni_aio_get_msg(ap); - nni_aio_set_msg(ap, NULL); - nni_msg_set_pipe(m, nni_pipe_id(p->p)); - - nni_mtx_lock(&s->m); - if (p->closed) { - nni_mtx_unlock(&s->m); - nni_msg_free(m); - return; - } - if (nni_list_empty(&s->rq)) { - nni_list_append(&s->pl, p); - if (nni_list_first(&s->pl) == p) { - nni_pollable_raise(&s->readable); - } - p->m = m; - nni_mtx_unlock(&s->m); - return; - } - nni_pipe_recv(p->p, ap); - as = nni_list_first(&s->rq); - nni_aio_list_remove(as); - nni_mtx_unlock(&s->m); - nni_aio_set_msg(as, m); - nni_aio_finish_sync(as, 0, nni_msg_len(m)); -} - -static void -pull0_sock_open(void *arg) -{ - NNI_ARG_UNUSED(arg); -} - -static void -pull0_sock_close(void *arg) -{ - pull0_sock *s = arg; - nni_aio * a; - nni_mtx_lock(&s->m); - while ((a = nni_list_first(&s->rq)) != NULL) { - nni_aio_list_remove(a); - nni_aio_finish_error(a, NNG_ECLOSED); - } - // NB: The common socket framework closes pipes before this. - nni_mtx_unlock(&s->m); -} - -static void -pull0_sock_send(void *arg, nni_aio *aio) -{ - NNI_ARG_UNUSED(arg); - nni_aio_finish_error(aio, NNG_ENOTSUP); -} - -static void -pull0_cancel(nni_aio *aio, void *arg, int rv) -{ - pull0_sock *s = arg; - nni_mtx_lock(&s->m); - if (nni_aio_list_active(aio)) { - nni_aio_list_remove(aio); - nni_aio_finish_error(aio, rv); - } - nni_mtx_unlock(&s->m); -} - -static void -pull0_sock_recv(void *arg, nni_aio *aio) -{ - pull0_sock *s = arg; - pull0_pipe *p; - - if (nni_aio_begin(aio) != 0) { - return; - } - - nni_mtx_lock(&s->m); - if ((p = nni_list_first(&s->pl)) == NULL) { - - int rv; - if ((rv = nni_aio_schedule(aio, pull0_cancel, s)) != 0) { - nni_mtx_unlock(&s->m); - nni_aio_finish_error(aio, rv); - return; - } - - nni_aio_list_append(&s->rq, aio); - nni_mtx_unlock(&s->m); - return; - } - - nni_list_remove(&s->pl, p); - if (nni_list_empty(&s->pl)) { - nni_pollable_clear(&s->readable); - } - nni_aio_finish_msg(aio, p->m); - p->m = NULL; - nni_pipe_recv(p->p, &p->aio); - nni_mtx_unlock(&s->m); -} - -static int -pull0_sock_get_recv_fd(void *arg, void *buf, size_t *szp, nni_opt_type t) -{ - pull0_sock *s = arg; - int rv; - int fd; - - if ((rv = nni_pollable_getfd(&s->readable, &fd)) != 0) { - return (rv); - } - return (nni_copyout_int(fd, buf, szp, t)); -} - -static nni_option pull0_sock_options[] = { - { - .o_name = NNG_OPT_RECVFD, - .o_get = pull0_sock_get_recv_fd, - }, - // terminate list - { - .o_name = NULL, - }, -}; - -static nni_proto_pipe_ops pull0_pipe_ops = { - .pipe_size = sizeof(pull0_pipe), - .pipe_init = pull0_pipe_init, - .pipe_fini = pull0_pipe_fini, - .pipe_start = pull0_pipe_start, - .pipe_close = pull0_pipe_close, - .pipe_stop = pull0_pipe_stop, -}; - -static nni_proto_sock_ops pull0_sock_ops = { - .sock_size = sizeof(pull0_sock), - .sock_init = pull0_sock_init, - .sock_fini = pull0_sock_fini, - .sock_open = pull0_sock_open, - .sock_close = pull0_sock_close, - .sock_send = pull0_sock_send, - .sock_recv = pull0_sock_recv, - .sock_options = pull0_sock_options, -}; - -static nni_proto pull0_proto = { - .proto_version = NNI_PROTOCOL_VERSION, - .proto_self = { NNI_PROTO_PULL_V0, "pull" }, - .proto_peer = { NNI_PROTO_PUSH_V0, "push" }, - .proto_flags = NNI_PROTO_FLAG_RCV, - .proto_pipe_ops = &pull0_pipe_ops, - .proto_sock_ops = &pull0_sock_ops, -}; - -static nni_proto pull0_proto_raw = { - .proto_version = NNI_PROTOCOL_VERSION, - .proto_self = { NNI_PROTO_PULL_V0, "pull" }, - .proto_peer = { NNI_PROTO_PUSH_V0, "push" }, - .proto_flags = NNI_PROTO_FLAG_RCV | NNI_PROTO_FLAG_RAW, - .proto_pipe_ops = &pull0_pipe_ops, - .proto_sock_ops = &pull0_sock_ops, -}; - -int -nng_pull0_open(nng_socket *s) -{ - return (nni_proto_open(s, &pull0_proto)); -} - -int -nng_pull0_open_raw(nng_socket *s) -{ - return (nni_proto_open(s, &pull0_proto_raw)); -} diff --git a/src/protocol/pipeline0/pull_test.c b/src/protocol/pipeline0/pull_test.c deleted file mode 100644 index 25066093..00000000 --- a/src/protocol/pipeline0/pull_test.c +++ /dev/null @@ -1,264 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - -#include - -static void -test_pull_identity(void) -{ - nng_socket s; - int p; - char * n; - - NUTS_PASS(nng_pull0_open(&s)); - NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PROTO, &p)); - NUTS_TRUE(p == NUTS_PROTO(5u, 1u)); // 81 - NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PEER, &p)); - NUTS_TRUE(p == NUTS_PROTO(5u, 0u)); // 80 - NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PROTONAME, &n)); - NUTS_MATCH(n, "pull"); - nng_strfree(n); - NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PEERNAME, &n)); - NUTS_MATCH(n, "push"); - nng_strfree(n); - NUTS_CLOSE(s); -} - -static void -test_pull_cannot_send(void) -{ - nng_socket s; - - NUTS_PASS(nng_pull0_open(&s)); - NUTS_FAIL(nng_send(s, "", 0, 0), NNG_ENOTSUP); - NUTS_CLOSE(s); -} - -static void -test_pull_no_context(void) -{ - nng_socket s; - nng_ctx ctx; - - NUTS_PASS(nng_pull0_open(&s)); - NUTS_FAIL(nng_ctx_open(&ctx, s), NNG_ENOTSUP); - NUTS_CLOSE(s); -} - -static void -test_pull_not_writeable(void) -{ - int fd; - nng_socket s; - - NUTS_PASS(nng_pull0_open(&s)); - NUTS_FAIL(nng_socket_get_int(s, NNG_OPT_SENDFD, &fd), NNG_ENOTSUP); - NUTS_CLOSE(s); -} - -static void -test_pull_poll_readable(void) -{ - int fd; - nng_socket pull; - nng_socket push; - - NUTS_PASS(nng_pull0_open(&pull)); - NUTS_PASS(nng_push0_open(&push)); - NUTS_PASS(nng_socket_set_ms(pull, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(push, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_get_int(pull, NNG_OPT_RECVFD, &fd)); - NUTS_TRUE(fd >= 0); - - // Not readable if not connected! - NUTS_TRUE(nuts_poll_fd(fd) == false); - - // Even after connect (no message yet) - NUTS_MARRY(pull, push); - NUTS_TRUE(nuts_poll_fd(fd) == false); - - // But once we send messages, it is. - // We have to send a request, in order to send a reply. - NUTS_SEND(push, "abc"); - NUTS_SLEEP(100); - NUTS_TRUE(nuts_poll_fd(fd)); - - // and receiving makes it no longer ready - NUTS_RECV(pull, "abc"); - NUTS_TRUE(nuts_poll_fd(fd) == false); - - NUTS_CLOSE(pull); - NUTS_CLOSE(push); -} - -static void -test_pull_close_pending(void) -{ - int fd; - nng_socket pull; - nng_socket push; - nng_pipe p1, p2; - char * addr; - - NUTS_ADDR(addr, "inproc"); - - NUTS_PASS(nng_pull0_open(&pull)); - NUTS_PASS(nng_push0_open(&push)); - NUTS_PASS(nng_socket_get_int(pull, NNG_OPT_RECVFD, &fd)); - NUTS_TRUE(fd >= 0); - NUTS_MARRY_EX(pull, push, addr, &p1, &p2); - - // Send a message -- it's ready for reading. - NUTS_SEND(push, "abc"); - NUTS_SLEEP(100); - NUTS_TRUE(nuts_poll_fd(fd)); - - // NB: We have to close the pipe instead of the socket. - // This is because the socket won't notice the remote pipe - // disconnect until we collect the message and start another - // receive operation. - nng_pipe_close(p1); - nng_pipe_close(p2); - - NUTS_SLEEP(100); - NUTS_TRUE(nuts_poll_fd(fd) == false); - - NUTS_CLOSE(pull); - NUTS_CLOSE(push); -} - -void -test_pull_validate_peer(void) -{ - nng_socket s1, s2; - nng_stat * stats; - nng_stat * reject; - char * addr; - - NUTS_ADDR(addr, "inproc"); - - NUTS_PASS(nng_pull0_open(&s1)); - NUTS_PASS(nng_pull0_open(&s2)); - - NUTS_PASS(nng_listen(s1, addr, NULL, 0)); - NUTS_PASS(nng_dial(s2, addr, NULL, NNG_FLAG_NONBLOCK)); - - NUTS_SLEEP(100); - NUTS_PASS(nng_stats_get(&stats)); - - NUTS_TRUE(stats != NULL); - NUTS_TRUE((reject = nng_stat_find_socket(stats, s1)) != NULL); - NUTS_TRUE((reject = nng_stat_find(reject, "reject")) != NULL); - - NUTS_TRUE(nng_stat_type(reject) == NNG_STAT_COUNTER); - NUTS_TRUE(nng_stat_value(reject) > 0); - - NUTS_CLOSE(s1); - NUTS_CLOSE(s2); - nng_stats_free(stats); -} - -static void -test_pull_recv_aio_stopped(void) -{ - nng_socket s; - nng_aio * aio; - - NUTS_PASS(nng_pull0_open(&s)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - - nng_aio_stop(aio); - nng_recv_aio(s, aio); - nng_aio_wait(aio); - NUTS_FAIL(nng_aio_result(aio), NNG_ECANCELED); - NUTS_CLOSE(s); - nng_aio_free(aio); -} - -static void -test_pull_close_recv(void) -{ - nng_socket s; - nng_aio * aio; - - NUTS_PASS(nng_pull0_open(&s)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - nng_aio_set_timeout(aio, 1000); - nng_recv_aio(s, aio); - NUTS_PASS(nng_close(s)); - nng_aio_wait(aio); - NUTS_FAIL(nng_aio_result(aio), NNG_ECLOSED); - - nng_aio_free(aio); -} - -static void -test_pull_recv_nonblock(void) -{ - nng_socket s; - nng_aio * aio; - - NUTS_PASS(nng_pull0_open(&s)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - - nng_aio_set_timeout(aio, 0); // Instant timeout - nng_recv_aio(s, aio); - - nng_aio_wait(aio); - NUTS_FAIL(nng_aio_result(aio), NNG_ETIMEDOUT); - NUTS_CLOSE(s); - nng_aio_free(aio); -} - -static void -test_pull_recv_cancel(void) -{ - nng_socket s; - nng_aio * aio; - - NUTS_PASS(nng_pull0_open(&s)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - - nng_aio_set_timeout(aio, 1000); - nng_recv_aio(s, aio); - nng_aio_abort(aio, NNG_ECANCELED); - - nng_aio_wait(aio); - NUTS_FAIL(nng_aio_result(aio), NNG_ECANCELED); - NUTS_CLOSE(s); - nng_aio_free(aio); -} - -static void -test_pull_cooked(void) -{ - nng_socket s; - bool b; - - NUTS_PASS(nng_pull0_open(&s)); - NUTS_PASS(nng_socket_get_bool(s, NNG_OPT_RAW, &b)); - NUTS_TRUE(!b); - NUTS_CLOSE(s); -} - -TEST_LIST = { - { "pull identity", test_pull_identity }, - { "pull cannot send", test_pull_cannot_send }, - { "pull no context", test_pull_no_context }, - { "pull not writeable", test_pull_not_writeable }, - { "pull poll readable", test_pull_poll_readable }, - { "pull close pending", test_pull_close_pending }, - { "pull validate peer", test_pull_validate_peer }, - { "pull recv aio stopped", test_pull_recv_aio_stopped }, - { "pull close recv", test_pull_close_recv }, - { "pull recv nonblock", test_pull_recv_nonblock }, - { "pull recv cancel", test_pull_recv_cancel }, - { "pull cooked", test_pull_cooked }, - { NULL, NULL }, -}; diff --git a/src/protocol/pipeline0/push.c b/src/protocol/pipeline0/push.c deleted file mode 100644 index ad43d967..00000000 --- a/src/protocol/pipeline0/push.c +++ /dev/null @@ -1,442 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// Copyright 2018 Capitar IT Group BV -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - -#include - -#include "core/nng_impl.h" -#include "nng/protocol/pipeline0/push.h" - -// Push protocol. The PUSH protocol is the "write" side of a pipeline. -// Push distributes fairly, or tries to, by giving messages in round-robin -// order. - -#ifndef NNI_PROTO_PULL_V0 -#define NNI_PROTO_PULL_V0 NNI_PROTO(5, 1) -#endif - -#ifndef NNI_PROTO_PUSH_V0 -#define NNI_PROTO_PUSH_V0 NNI_PROTO(5, 0) -#endif - -typedef struct push0_pipe push0_pipe; -typedef struct push0_sock push0_sock; - -static void push0_send_cb(void *); -static void push0_recv_cb(void *); -static void push0_pipe_ready(push0_pipe *); - -// push0_sock is our per-socket protocol private structure. -struct push0_sock { - nni_lmq wq; // list of messages queued - nni_list aq; // list of aio senders waiting - nni_list pl; // list of pipes ready to send - nni_pollable writable; - nni_mtx m; -}; - -// push0_pipe is our per-pipe protocol private structure. -struct push0_pipe { - nni_pipe * pipe; - push0_sock * push; - nni_list_node node; - - nni_aio aio_recv; - nni_aio aio_send; -}; - -static int -push0_sock_init(void *arg, nni_sock *sock) -{ - push0_sock *s = arg; - NNI_ARG_UNUSED(sock); - - nni_mtx_init(&s->m); - nni_aio_list_init(&s->aq); - NNI_LIST_INIT(&s->pl, push0_pipe, node); - nni_lmq_init(&s->wq, 0); // initially we start unbuffered. - nni_pollable_init(&s->writable); - - return (0); -} - -static void -push0_sock_fini(void *arg) -{ - push0_sock *s = arg; - nni_pollable_fini(&s->writable); - nni_lmq_fini(&s->wq); - nni_mtx_fini(&s->m); -} - -static void -push0_sock_open(void *arg) -{ - NNI_ARG_UNUSED(arg); -} - -static void -push0_sock_close(void *arg) -{ - push0_sock *s = arg; - nni_aio * a; - nni_mtx_lock(&s->m); - while ((a = nni_list_first(&s->aq)) != NULL) { - nni_aio_list_remove(a); - nni_aio_finish_error(a, NNG_ECLOSED); - } - nni_mtx_unlock(&s->m); -} - -static void -push0_pipe_stop(void *arg) -{ - push0_pipe *p = arg; - - nni_aio_stop(&p->aio_recv); - nni_aio_stop(&p->aio_send); -} - -static void -push0_pipe_fini(void *arg) -{ - push0_pipe *p = arg; - - nni_aio_fini(&p->aio_recv); - nni_aio_fini(&p->aio_send); -} - -static int -push0_pipe_init(void *arg, nni_pipe *pipe, void *s) -{ - push0_pipe *p = arg; - - nni_aio_init(&p->aio_recv, push0_recv_cb, p); - nni_aio_init(&p->aio_send, push0_send_cb, p); - NNI_LIST_NODE_INIT(&p->node); - p->pipe = pipe; - p->push = s; - return (0); -} - -static int -push0_pipe_start(void *arg) -{ - push0_pipe *p = arg; - - if (nni_pipe_peer(p->pipe) != NNI_PROTO_PULL_V0) { - return (NNG_EPROTO); - } - - // Schedule a receiver. This is mostly so that we can detect - // a closed transport pipe. - nni_pipe_recv(p->pipe, &p->aio_recv); - push0_pipe_ready(p); - - return (0); -} - -static void -push0_pipe_close(void *arg) -{ - push0_pipe *p = arg; - push0_sock *s = p->push; - - nni_aio_close(&p->aio_recv); - nni_aio_close(&p->aio_send); - - nni_mtx_lock(&s->m); - if (nni_list_node_active(&p->node)) { - nni_list_node_remove(&p->node); - - if (nni_list_empty(&s->pl) && nni_lmq_full(&s->wq)) { - nni_pollable_clear(&s->writable); - } - } - nni_mtx_unlock(&s->m); -} - -static void -push0_recv_cb(void *arg) -{ - push0_pipe *p = arg; - - // We normally expect to receive an error. If a pipe actually - // sends us data, we just discard it. - if (nni_aio_result(&p->aio_recv) != 0) { - nni_pipe_close(p->pipe); - return; - } - nni_msg_free(nni_aio_get_msg(&p->aio_recv)); - nni_aio_set_msg(&p->aio_recv, NULL); - nni_pipe_recv(p->pipe, &p->aio_recv); -} - -static void -push0_pipe_ready(push0_pipe *p) -{ - push0_sock *s = p->push; - nni_msg * m; - nni_aio * a = NULL; - size_t l; - bool blocked; - - nni_mtx_lock(&s->m); - - blocked = nni_lmq_full(&s->wq) && nni_list_empty(&s->pl); - - // if message is waiting in the buffered queue - // then we prefer that. - if (nni_lmq_getq(&s->wq, &m) == 0) { - nni_aio_set_msg(&p->aio_send, m); - nni_pipe_send(p->pipe, &p->aio_send); - - if ((a = nni_list_first(&s->aq)) != NULL) { - nni_aio_list_remove(a); - m = nni_aio_get_msg(a); - l = nni_msg_len(m); - nni_lmq_putq(&s->wq, m); - } - - } else if ((a = nni_list_first(&s->aq)) != NULL) { - // Looks like we had the unbuffered case, but - // someone was waiting. - nni_aio_list_remove(a); - m = nni_aio_get_msg(a); - l = nni_msg_len(m); - - nni_aio_set_msg(&p->aio_send, m); - nni_pipe_send(p->pipe, &p->aio_send); - } else { - // We had nothing to send. Just put us in the ready list. - nni_list_append(&s->pl, p); - } - - if (blocked) { - // if we blocked, then toggle the status. - if ((!nni_lmq_full(&s->wq)) || (!nni_list_empty(&s->pl))) { - nni_pollable_raise(&s->writable); - } - } - - nni_mtx_unlock(&s->m); - - if (a != NULL) { - nni_aio_set_msg(a, NULL); - nni_aio_finish_sync(a, 0, l); - } -} - -static void -push0_send_cb(void *arg) -{ - push0_pipe *p = arg; - - if (nni_aio_result(&p->aio_send) != 0) { - nni_msg_free(nni_aio_get_msg(&p->aio_send)); - nni_aio_set_msg(&p->aio_send, NULL); - nni_pipe_close(p->pipe); - return; - } - - push0_pipe_ready(p); -} - -static void -push0_cancel(nni_aio *aio, void *arg, int rv) -{ - push0_sock *s = arg; - - nni_mtx_lock(&s->m); - if (nni_aio_list_active(aio)) { - nni_aio_list_remove(aio); - nni_aio_finish_error(aio, rv); - } - nni_mtx_unlock(&s->m); -} - -static void -push0_sock_send(void *arg, nni_aio *aio) -{ - push0_sock *s = arg; - push0_pipe *p; - nni_msg * m; - size_t l; - int rv; - - if (nni_aio_begin(aio) != 0) { - return; - } - - m = nni_aio_get_msg(aio); - l = nni_msg_len(m); - - nni_mtx_lock(&s->m); - - // First we want to see if we can send it right now. - // Note that we don't block the sender until the read is complete, - // only until we have committed to send it. - if ((p = nni_list_first(&s->pl)) != NULL) { - nni_list_remove(&s->pl, p); - // NB: We won't have had any waiters in the message queue - // or the aio queue, because we would not put the pipe - // in the ready list in that case. Note though that the - // wq may be "full" if we are unbuffered. - if (nni_list_empty(&s->pl) && (nni_lmq_full(&s->wq))) { - nni_pollable_clear(&s->writable); - } - nni_aio_set_msg(aio, NULL); - nni_aio_finish(aio, 0, l); - nni_aio_set_msg(&p->aio_send, m); - nni_pipe_send(p->pipe, &p->aio_send); - nni_mtx_unlock(&s->m); - return; - } - - // Can we maybe queue it. - if (nni_lmq_putq(&s->wq, m) == 0) { - // Yay, we can. So we're done. - nni_aio_set_msg(aio, NULL); - nni_aio_finish(aio, 0, l); - if (nni_lmq_full(&s->wq)) { - nni_pollable_clear(&s->writable); - } - nni_mtx_unlock(&s->m); - return; - } - - if ((rv = nni_aio_schedule(aio, push0_cancel, s)) != 0) { - nni_aio_finish_error(aio, rv); - nni_mtx_unlock(&s->m); - return; - } - nni_aio_list_append(&s->aq, aio); - nni_mtx_unlock(&s->m); -} - -static void -push0_sock_recv(void *arg, nni_aio *aio) -{ - NNI_ARG_UNUSED(arg); - nni_aio_finish_error(aio, NNG_ENOTSUP); -} - -static int -push0_set_send_buf_len(void *arg, const void *buf, size_t sz, nni_type t) -{ - push0_sock *s = arg; - int val; - int rv; - - if ((rv = nni_copyin_int(&val, buf, sz, 0, 8192, t)) != 0) { - return (rv); - } - nni_mtx_lock(&s->m); - rv = nni_lmq_resize(&s->wq, (size_t) val); - // Changing the size of the queue can affect our readiness. - if (!nni_lmq_full(&s->wq)) { - nni_pollable_raise(&s->writable); - } else if (nni_list_empty(&s->pl)) { - nni_pollable_clear(&s->writable); - } - nni_mtx_unlock(&s->m); - return (rv); -} - -static int -push0_get_send_buf_len(void *arg, void *buf, size_t *szp, nni_opt_type t) -{ - push0_sock *s = arg; - int val; - - nni_mtx_lock(&s->m); - val = nni_lmq_cap(&s->wq); - nni_mtx_unlock(&s->m); - - return (nni_copyout_int(val, buf, szp, t)); -} - -static int -push0_sock_get_send_fd(void *arg, void *buf, size_t *szp, nni_opt_type t) -{ - push0_sock *s = arg; - int rv; - int fd; - - if ((rv = nni_pollable_getfd(&s->writable, &fd)) != 0) { - return (rv); - } - return (nni_copyout_int(fd, buf, szp, t)); -} - -static nni_proto_pipe_ops push0_pipe_ops = { - .pipe_size = sizeof(push0_pipe), - .pipe_init = push0_pipe_init, - .pipe_fini = push0_pipe_fini, - .pipe_start = push0_pipe_start, - .pipe_close = push0_pipe_close, - .pipe_stop = push0_pipe_stop, -}; - -static nni_option push0_sock_options[] = { - { - .o_name = NNG_OPT_SENDFD, - .o_get = push0_sock_get_send_fd, - }, - { - .o_name = NNG_OPT_SENDBUF, - .o_get = push0_get_send_buf_len, - .o_set = push0_set_send_buf_len, - }, - // terminate list - { - .o_name = NULL, - }, -}; - -static nni_proto_sock_ops push0_sock_ops = { - .sock_size = sizeof(push0_sock), - .sock_init = push0_sock_init, - .sock_fini = push0_sock_fini, - .sock_open = push0_sock_open, - .sock_close = push0_sock_close, - .sock_options = push0_sock_options, - .sock_send = push0_sock_send, - .sock_recv = push0_sock_recv, -}; - -static nni_proto push0_proto = { - .proto_version = NNI_PROTOCOL_VERSION, - .proto_self = { NNI_PROTO_PUSH_V0, "push" }, - .proto_peer = { NNI_PROTO_PULL_V0, "pull" }, - .proto_flags = NNI_PROTO_FLAG_SND, - .proto_pipe_ops = &push0_pipe_ops, - .proto_sock_ops = &push0_sock_ops, -}; - -static nni_proto push0_proto_raw = { - .proto_version = NNI_PROTOCOL_VERSION, - .proto_self = { NNI_PROTO_PUSH_V0, "push" }, - .proto_peer = { NNI_PROTO_PULL_V0, "pull" }, - .proto_flags = NNI_PROTO_FLAG_SND | NNI_PROTO_FLAG_RAW, - .proto_pipe_ops = &push0_pipe_ops, - .proto_sock_ops = &push0_sock_ops, -}; - -int -nng_push0_open(nng_socket *s) -{ - return (nni_proto_open(s, &push0_proto)); -} - -int -nng_push0_open_raw(nng_socket *s) -{ - return (nni_proto_open(s, &push0_proto_raw)); -} diff --git a/src/protocol/pipeline0/push_test.c b/src/protocol/pipeline0/push_test.c deleted file mode 100644 index d22ccaa4..00000000 --- a/src/protocol/pipeline0/push_test.c +++ /dev/null @@ -1,525 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - -#include - -static void -test_push_identity(void) -{ - nng_socket s; - int p; - char * n; - - NUTS_PASS(nng_push0_open(&s)); - NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PROTO, &p)); - NUTS_TRUE(p == NUTS_PROTO(5u, 0u)); // 80 - NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PEER, &p)); - NUTS_TRUE(p == NUTS_PROTO(5u, 1u)); // 81 - NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PROTONAME, &n)); - NUTS_MATCH(n, "push"); - nng_strfree(n); - NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PEERNAME, &n)); - NUTS_MATCH(n, "pull"); - nng_strfree(n); - NUTS_CLOSE(s); -} - -static void -test_push_cannot_recv(void) -{ - nng_socket s; - nng_msg * m = NULL; - - NUTS_PASS(nng_push0_open(&s)); - NUTS_FAIL(nng_recvmsg(s, &m, 0), NNG_ENOTSUP); - NUTS_CLOSE(s); -} - -static void -test_push_no_context(void) -{ - nng_socket s; - nng_ctx ctx; - - NUTS_PASS(nng_push0_open(&s)); - NUTS_FAIL(nng_ctx_open(&ctx, s), NNG_ENOTSUP); - NUTS_CLOSE(s); -} - -static void -test_push_not_readable(void) -{ - int fd; - nng_socket s; - - NUTS_PASS(nng_push0_open(&s)); - NUTS_FAIL(nng_socket_get_int(s, NNG_OPT_RECVFD, &fd), NNG_ENOTSUP); - NUTS_CLOSE(s); -} - -static void -test_push_poll_writable(void) -{ - int fd; - nng_socket pull; - nng_socket push; - - NUTS_PASS(nng_pull0_open(&pull)); - NUTS_PASS(nng_push0_open(&push)); - NUTS_PASS(nng_socket_set_ms(pull, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(push, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_get_int(push, NNG_OPT_SENDFD, &fd)); - NUTS_TRUE(fd >= 0); - - // This tests unbuffered sockets for now. - // Note that for this we are using unbuffered inproc. - // If using TCP or similar, then transport buffering will - // break assumptions in this test. - - // Not writable if not connected! - NUTS_TRUE(nuts_poll_fd(fd) == false); - - // After connect we can write. - NUTS_MARRY(pull, push); - NUTS_TRUE(nuts_poll_fd(fd) == true); - - // But once we send a message, it is not anymore. - NUTS_SEND(push, "abc"); - // Have to send a second message, because the remote socket - // will have consumed the first one. - NUTS_SEND(push, "def"); - NUTS_SLEEP(100); - NUTS_TRUE(nuts_poll_fd(fd) == false); - - // and receiving receiving the message makes it possible again. - NUTS_RECV(pull, "abc"); - NUTS_SLEEP(100); - NUTS_TRUE(nuts_poll_fd(fd)); - - NUTS_CLOSE(pull); - NUTS_CLOSE(push); -} - -static void -test_push_poll_buffered(void) -{ - int fd; - nng_socket pull; - nng_socket push; - - NUTS_PASS(nng_pull0_open(&pull)); - NUTS_PASS(nng_push0_open(&push)); - NUTS_PASS(nng_socket_set_ms(pull, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(push, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_int(push, NNG_OPT_SENDBUF, 2)); - NUTS_PASS(nng_socket_get_int(push, NNG_OPT_SENDFD, &fd)); - NUTS_TRUE(fd >= 0); - - // We can write two message while unbuffered. - NUTS_TRUE(nuts_poll_fd(fd)); - NUTS_SEND(push, "abc"); - NUTS_TRUE(nuts_poll_fd(fd)); - NUTS_SEND(push, "def"); - NUTS_TRUE(nuts_poll_fd(fd) == false); - - // After connect we remote end will pick up one of them. - // Also, the local pipe itself will pick up one. So we - // have two. - NUTS_MARRY(pull, push); - NUTS_SLEEP(100); - NUTS_TRUE(nuts_poll_fd(fd)); - NUTS_SEND(push, "ghi"); - NUTS_SLEEP(100); - NUTS_TRUE(nuts_poll_fd(fd)); - NUTS_SEND(push, "jkl"); - // Now it should be full. - NUTS_TRUE(nuts_poll_fd(fd) == false); - - // and receiving receiving the message makes it possible again. - NUTS_RECV(pull, "abc"); - NUTS_SLEEP(100); - NUTS_TRUE(nuts_poll_fd(fd)); - NUTS_RECV(pull, "def"); - NUTS_RECV(pull, "ghi"); - NUTS_RECV(pull, "jkl"); - - NUTS_CLOSE(pull); - NUTS_CLOSE(push); -} - -static void -test_push_poll_truncate(void) -{ - int fd; - nng_socket pull; - nng_socket push; - - // This test starts with a buffer and then truncates it to see - // that shortening the buffer has an impact. - - NUTS_PASS(nng_pull0_open(&pull)); - NUTS_PASS(nng_push0_open(&push)); - NUTS_PASS(nng_socket_set_ms(pull, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(push, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_int(push, NNG_OPT_SENDBUF, 3)); - NUTS_PASS(nng_socket_get_int(push, NNG_OPT_SENDFD, &fd)); - NUTS_TRUE(fd >= 0); - - // We can write two message while unbuffered. - NUTS_TRUE(nuts_poll_fd(fd)); - NUTS_SEND(push, "abc"); - NUTS_TRUE(nuts_poll_fd(fd)); - NUTS_SEND(push, "def"); - NUTS_TRUE(nuts_poll_fd(fd)); - - NUTS_PASS(nng_socket_set_int(push, NNG_OPT_SENDBUF, 1)); - NUTS_TRUE(nuts_poll_fd(fd) == false); - - NUTS_MARRY(pull, push); - NUTS_RECV(pull, "abc"); - // def got dropped - NUTS_SEND(push, "ghi"); - NUTS_RECV(pull, "ghi"); - - NUTS_CLOSE(pull); - NUTS_SLEEP(100); - - // We have a buffer of one. - NUTS_SEND(push, "jkl"); - // Resize to 0 (unbuffered) - NUTS_PASS(nng_socket_set_int(push, NNG_OPT_SENDBUF, 0)); - - // reopen the pull socket and connect it - NUTS_PASS(nng_pull0_open(&pull)); - NUTS_MARRY(push, pull); - - // jkl got dropped. - NUTS_SEND(push, "mno"); - NUTS_RECV(pull, "mno"); - - NUTS_CLOSE(pull); - NUTS_CLOSE(push); -} - -void -test_push_validate_peer(void) -{ - nng_socket s1, s2; - nng_stat * stats; - nng_stat * reject; - char * addr; - - NUTS_ADDR(addr, "inproc"); - - NUTS_PASS(nng_push0_open(&s1)); - NUTS_PASS(nng_push0_open(&s2)); - - NUTS_PASS(nng_listen(s1, addr, NULL, 0)); - NUTS_PASS(nng_dial(s2, addr, NULL, NNG_FLAG_NONBLOCK)); - - NUTS_SLEEP(100); - NUTS_PASS(nng_stats_get(&stats)); - - NUTS_TRUE(stats != NULL); - NUTS_TRUE((reject = nng_stat_find_socket(stats, s1)) != NULL); - NUTS_TRUE((reject = nng_stat_find(reject, "reject")) != NULL); - - NUTS_TRUE(nng_stat_type(reject) == NNG_STAT_COUNTER); - NUTS_TRUE(nng_stat_value(reject) > 0); - - NUTS_CLOSE(s1); - NUTS_CLOSE(s2); - nng_stats_free(stats); -} - -static void -test_push_send_aio_stopped(void) -{ - nng_socket s; - nng_aio * aio; - nng_msg * m; - - NUTS_PASS(nng_push0_open(&s)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - NUTS_PASS(nng_msg_alloc(&m, 0)); - - nng_aio_set_msg(aio, m); - nng_aio_stop(aio); - nng_send_aio(s, aio); - nng_aio_wait(aio); - NUTS_FAIL(nng_aio_result(aio), NNG_ECANCELED); - NUTS_CLOSE(s); - nng_aio_free(aio); - nng_msg_free(m); -} - -static void -test_push_close_send(void) -{ - nng_socket s; - nng_aio * aio; - nng_msg * m; - - NUTS_PASS(nng_push0_open(&s)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - NUTS_PASS(nng_msg_alloc(&m, 0)); - nng_aio_set_timeout(aio, 1000); - nng_aio_set_msg(aio, m); - nng_send_aio(s, aio); - NUTS_PASS(nng_close(s)); - nng_aio_wait(aio); - NUTS_FAIL(nng_aio_result(aio), NNG_ECLOSED); - - nng_aio_free(aio); - nng_msg_free(m); -} - -static void -test_push_send_nonblock(void) -{ - nng_socket s; - nng_aio * aio; - nng_msg * m; - - NUTS_PASS(nng_push0_open(&s)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - NUTS_PASS(nng_msg_alloc(&m, 0)); - - nng_aio_set_timeout(aio, 0); // Instant timeout - nng_aio_set_msg(aio, m); - nng_send_aio(s, aio); - - nng_aio_wait(aio); - NUTS_FAIL(nng_aio_result(aio), NNG_ETIMEDOUT); - NUTS_CLOSE(s); - nng_aio_free(aio); - nng_msg_free(m); -} - -static void -test_push_send_timeout(void) -{ - nng_socket s; - nng_aio * aio; - nng_msg * m; - - NUTS_PASS(nng_push0_open(&s)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - NUTS_PASS(nng_msg_alloc(&m, 0)); - - nng_aio_set_timeout(aio, 10); - nng_aio_set_msg(aio, m); - nng_send_aio(s, aio); - - nng_aio_wait(aio); - NUTS_FAIL(nng_aio_result(aio), NNG_ETIMEDOUT); - NUTS_CLOSE(s); - nng_aio_free(aio); - nng_msg_free(m); -} - -static void -test_push_send_cancel(void) -{ - nng_socket s; - nng_aio * aio; - nng_msg * m; - - NUTS_PASS(nng_push0_open(&s)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - NUTS_PASS(nng_msg_alloc(&m, 0)); - - nng_aio_set_timeout(aio, 1000); - nng_aio_set_msg(aio, m); - nng_send_aio(s, aio); - nng_aio_abort(aio, NNG_ECANCELED); - - nng_aio_wait(aio); - NUTS_FAIL(nng_aio_result(aio), NNG_ECANCELED); - NUTS_CLOSE(s); - nng_aio_free(aio); - nng_msg_free(m); -} - -static void -test_push_send_late_unbuffered(void) -{ - nng_socket s; - nng_socket pull; - nng_aio * aio; - nng_msg * m; - - NUTS_PASS(nng_push0_open(&s)); - NUTS_PASS(nng_pull0_open(&pull)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - NUTS_PASS(nng_msg_alloc(&m, 0)); - NUTS_PASS(nng_msg_append(m, "123\0", 4)); - - nng_aio_set_timeout(aio, 1000); - nng_aio_set_msg(aio, m); - nng_send_aio(s, aio); - - NUTS_MARRY(s, pull); - - NUTS_RECV(pull, "123"); - - nng_aio_wait(aio); - NUTS_PASS(nng_aio_result(aio)); - NUTS_CLOSE(s); - nng_aio_free(aio); -} - - -static void -test_push_send_late_buffered(void) -{ - nng_socket s; - nng_socket pull; - nng_aio * aio; - nng_msg * m; - - NUTS_PASS(nng_push0_open(&s)); - NUTS_PASS(nng_pull0_open(&pull)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - NUTS_PASS(nng_setopt_int(s, NNG_OPT_SENDBUF, 2)); - NUTS_PASS(nng_msg_alloc(&m, 0)); - NUTS_PASS(nng_msg_append(m, "123\0", 4)); - - nng_aio_set_timeout(aio, 1000); - nng_aio_set_msg(aio, m); - nng_send_aio(s, aio); - - NUTS_MARRY(s, pull); - - NUTS_RECV(pull, "123"); - - nng_aio_wait(aio); - NUTS_PASS(nng_aio_result(aio)); - NUTS_CLOSE(s); - nng_aio_free(aio); -} - -static void -test_push_cooked(void) -{ - nng_socket s; - bool b; - - NUTS_PASS(nng_push0_open(&s)); - NUTS_PASS(nng_socket_get_bool(s, NNG_OPT_RAW, &b)); - NUTS_TRUE(!b); - NUTS_CLOSE(s); -} - -static void -test_push_load_balance_buffered(void) -{ - nng_socket s; - nng_socket pull1; - nng_socket pull2; - nng_socket pull3; - - NUTS_PASS(nng_push0_open(&s)); - NUTS_PASS(nng_pull0_open(&pull1)); - NUTS_PASS(nng_pull0_open(&pull2)); - NUTS_PASS(nng_pull0_open(&pull3)); - NUTS_PASS(nng_setopt_int(s, NNG_OPT_SENDBUF, 4)); - NUTS_MARRY(s, pull1); - NUTS_MARRY(s, pull2); - NUTS_MARRY(s, pull3); - NUTS_SLEEP(100); - NUTS_SEND(s, "one"); - NUTS_SEND(s, "two"); - NUTS_SEND(s, "three"); - NUTS_RECV(pull1, "one"); - NUTS_RECV(pull2, "two"); - NUTS_RECV(pull3, "three"); - NUTS_CLOSE(s); - NUTS_CLOSE(pull1); - NUTS_CLOSE(pull2); - NUTS_CLOSE(pull3); -} - -static void -test_push_load_balance_unbuffered(void) -{ - nng_socket s; - nng_socket pull1; - nng_socket pull2; - nng_socket pull3; - - NUTS_PASS(nng_push0_open(&s)); - NUTS_PASS(nng_pull0_open(&pull1)); - NUTS_PASS(nng_pull0_open(&pull2)); - NUTS_PASS(nng_pull0_open(&pull3)); - NUTS_MARRY(s, pull1); - NUTS_MARRY(s, pull2); - NUTS_MARRY(s, pull3); - NUTS_SLEEP(100); - NUTS_SEND(s, "one"); - NUTS_SEND(s, "two"); - NUTS_SEND(s, "three"); - NUTS_RECV(pull1, "one"); - NUTS_RECV(pull2, "two"); - NUTS_RECV(pull3, "three"); - // Loop around is unpredictable somewhat, because the the - // pull sockets can take different periods of time to return - // back to readiness. - NUTS_CLOSE(s); - NUTS_CLOSE(pull1); - NUTS_CLOSE(pull2); - NUTS_CLOSE(pull3); -} - -static void -test_push_send_buffer(void) -{ - nng_socket s; - int v; - bool b; - size_t sz; - - NUTS_PASS(nng_push0_open(&s)); - NUTS_PASS(nng_getopt_int(s, NNG_OPT_SENDBUF, &v)); - NUTS_TRUE(v == 0); - NUTS_FAIL(nng_getopt_bool(s, NNG_OPT_SENDBUF, &b), NNG_EBADTYPE); - sz = 1; - NUTS_FAIL(nng_getopt(s, NNG_OPT_SENDBUF, &b, &sz), NNG_EINVAL); - NUTS_FAIL(nng_setopt_int(s, NNG_OPT_SENDBUF, -1), NNG_EINVAL); - NUTS_FAIL(nng_setopt_int(s, NNG_OPT_SENDBUF, 100000), NNG_EINVAL); - NUTS_FAIL(nng_setopt_bool(s, NNG_OPT_SENDBUF, false), NNG_EBADTYPE); - NUTS_FAIL(nng_setopt(s, NNG_OPT_SENDBUF, &b, 1), NNG_EINVAL); - NUTS_PASS(nng_setopt_int(s, NNG_OPT_SENDBUF, 100)); - NUTS_PASS(nng_getopt_int(s, NNG_OPT_SENDBUF, &v)); - NUTS_TRUE(v == 100); - NUTS_CLOSE(s); -} - -TEST_LIST = { - { "push identity", test_push_identity }, - { "push cannot recv", test_push_cannot_recv }, - { "push no context", test_push_no_context }, - { "push not readable", test_push_not_readable }, - { "push poll writable", test_push_poll_writable }, - { "push poll buffered", test_push_poll_buffered }, - { "push poll truncate", test_push_poll_truncate }, - { "push validate peer", test_push_validate_peer }, - { "push send aio stopped", test_push_send_aio_stopped }, - { "push close send", test_push_close_send }, - { "push send nonblock", test_push_send_nonblock }, - { "push send timeout", test_push_send_timeout }, - { "push send cancel", test_push_send_cancel }, - { "push send late buffered", test_push_send_late_buffered }, - { "push send late unbuffered", test_push_send_late_unbuffered }, - { "push cooked", test_push_cooked }, - { "push load balance buffered", test_push_load_balance_buffered }, - { "push load balance unbuffered", test_push_load_balance_unbuffered }, - { "push send buffer", test_push_send_buffer }, - { NULL, NULL }, -}; diff --git a/src/protocol/pubsub0/CMakeLists.txt b/src/protocol/pubsub0/CMakeLists.txt deleted file mode 100644 index 160b7462..00000000 --- a/src/protocol/pubsub0/CMakeLists.txt +++ /dev/null @@ -1,24 +0,0 @@ -# -# Copyright 2019 Staysail Systems, Inc. -# Copyright 2018 Capitar IT Group BV -# -# This software is supplied under the terms of the MIT License, a -# copy of which should be located in the distribution where this -# file was obtained (LICENSE.txt). A copy of the license may also be -# found online at https://opensource.org/licenses/MIT. -# - -# Pub/Sub protocol -nng_directory(pubsub0) - -nng_sources_if(NNG_PROTO_PUB0 pub.c) -nng_headers_if(NNG_PROTO_PUB0 nng/protocol/pubsub0/pub.h) -nng_defines_if(NNG_PROTO_PUB0 NNG_HAVE_PUB0) - -nng_sources_if(NNG_PROTO_SUB0 sub.c xsub.c) -nng_headers_if(NNG_PROTO_SUB0 nng/protocol/pubsub0/sub.h) -nng_defines_if(NNG_PROTO_SUB0 NNG_HAVE_SUB0) - -nng_test(pub_test) -nng_test(sub_test) -nng_test(xsub_test) diff --git a/src/protocol/pubsub0/pub.c b/src/protocol/pubsub0/pub.c deleted file mode 100644 index e3d4f16a..00000000 --- a/src/protocol/pubsub0/pub.c +++ /dev/null @@ -1,383 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// Copyright 2018 Capitar IT Group BV -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - -#include - -#include "core/nng_impl.h" -#include "nng/protocol/pubsub0/pub.h" - -// Publish protocol. The PUB protocol simply sends messages out, as -// a broadcast. It has nothing more sophisticated because it does not -// perform sender-side filtering. Its best effort delivery, so anything -// that can't receive the message won't get one. - -#ifndef NNI_PROTO_SUB_V0 -#define NNI_PROTO_SUB_V0 NNI_PROTO(2, 1) -#endif - -#ifndef NNI_PROTO_PUB_V0 -#define NNI_PROTO_PUB_V0 NNI_PROTO(2, 0) -#endif - -typedef struct pub0_pipe pub0_pipe; -typedef struct pub0_sock pub0_sock; - -static void pub0_pipe_recv_cb(void *); -static void pub0_pipe_send_cb(void *); -static void pub0_sock_fini(void *); -static void pub0_pipe_fini(void *); - -// pub0_sock is our per-socket protocol private structure. -struct pub0_sock { - nni_list pipes; - nni_mtx mtx; - bool closed; - size_t sendbuf; - nni_pollable *sendable; -}; - -// pub0_pipe is our per-pipe protocol private structure. -struct pub0_pipe { - nni_pipe * pipe; - pub0_sock * pub; - nni_lmq sendq; - bool closed; - bool busy; - nni_aio * aio_send; - nni_aio * aio_recv; - nni_list_node node; -}; - -static void -pub0_sock_fini(void *arg) -{ - pub0_sock *s = arg; - - nni_pollable_free(s->sendable); - nni_mtx_fini(&s->mtx); -} - -static int -pub0_sock_init(void *arg, nni_sock *nsock) -{ - pub0_sock *sock = arg; - int rv; - NNI_ARG_UNUSED(nsock); - - if ((rv = nni_pollable_alloc(&sock->sendable)) != 0) { - return (rv); - } - nni_mtx_init(&sock->mtx); - NNI_LIST_INIT(&sock->pipes, pub0_pipe, node); - sock->sendbuf = 16; // fairly arbitrary - return (0); -} - -static void -pub0_sock_open(void *arg) -{ - NNI_ARG_UNUSED(arg); -} - -static void -pub0_sock_close(void *arg) -{ - NNI_ARG_UNUSED(arg); -} - -static void -pub0_pipe_stop(void *arg) -{ - pub0_pipe *p = arg; - - nni_aio_stop(p->aio_send); - nni_aio_stop(p->aio_recv); -} - -static void -pub0_pipe_fini(void *arg) -{ - pub0_pipe *p = arg; - - nni_aio_free(p->aio_send); - nni_aio_free(p->aio_recv); - nni_lmq_fini(&p->sendq); -} - -static int -pub0_pipe_init(void *arg, nni_pipe *pipe, void *s) -{ - pub0_pipe *p = arg; - pub0_sock *sock = s; - int rv; - size_t len; - - nni_mtx_lock(&sock->mtx); - len = sock->sendbuf; - nni_mtx_unlock(&sock->mtx); - - // XXX: consider making this depth tunable - if (((rv = nni_lmq_init(&p->sendq, len)) != 0) || - ((rv = nni_aio_alloc(&p->aio_send, pub0_pipe_send_cb, p)) != 0) || - ((rv = nni_aio_alloc(&p->aio_recv, pub0_pipe_recv_cb, p)) != 0)) { - - pub0_pipe_fini(p); - return (rv); - } - - p->busy = false; - p->pipe = pipe; - p->pub = s; - return (0); -} - -static int -pub0_pipe_start(void *arg) -{ - pub0_pipe *p = arg; - pub0_sock *sock = p->pub; - - if (nni_pipe_peer(p->pipe) != NNI_PROTO_SUB_V0) { - return (NNG_EPROTO); - } - nni_mtx_lock(&sock->mtx); - nni_list_append(&sock->pipes, p); - nni_mtx_unlock(&sock->mtx); - - // Start the receiver. - nni_pipe_recv(p->pipe, p->aio_recv); - - return (0); -} - -static void -pub0_pipe_close(void *arg) -{ - pub0_pipe *p = arg; - pub0_sock *sock = p->pub; - - nni_aio_close(p->aio_send); - nni_aio_close(p->aio_recv); - - nni_mtx_lock(&sock->mtx); - p->closed = true; - nni_lmq_flush(&p->sendq); - - if (nni_list_active(&sock->pipes, p)) { - nni_list_remove(&sock->pipes, p); - } - nni_mtx_unlock(&sock->mtx); -} - -static void -pub0_pipe_recv_cb(void *arg) -{ - pub0_pipe *p = arg; - - // We should never receive a message -- the only valid reason for us to - // be here is on pipe close. - if (nni_aio_result(p->aio_recv) == 0) { - nni_msg_free(nni_aio_get_msg(p->aio_recv)); - } - nni_pipe_close(p->pipe); -} - -static void -pub0_pipe_send_cb(void *arg) -{ - pub0_pipe *p = arg; - pub0_sock *sock = p->pub; - nni_msg * msg; - - if (nni_aio_result(p->aio_send) != 0) { - nni_msg_free(nni_aio_get_msg(p->aio_send)); - nni_aio_set_msg(p->aio_send, NULL); - nni_pipe_close(p->pipe); - return; - } - - nni_mtx_lock(&sock->mtx); - if (p->closed) { - nni_mtx_unlock(&sock->mtx); - return; - } - if (nni_lmq_getq(&p->sendq, &msg) == 0) { - nni_aio_set_msg(p->aio_send, msg); - nni_pipe_send(p->pipe, p->aio_send); - } else { - p->busy = false; - } - nni_mtx_unlock(&sock->mtx); -} - -static void -pub0_sock_recv(void *arg, nni_aio *aio) -{ - NNI_ARG_UNUSED(arg); - if (nni_aio_begin(aio) == 0) { - nni_aio_finish_error(aio, NNG_ENOTSUP); - } -} - -static void -pub0_sock_send(void *arg, nni_aio *aio) -{ - pub0_sock *sock = arg; - pub0_pipe *p; - nng_msg * msg; - size_t len; - - msg = nni_aio_get_msg(aio); - len = nni_msg_len(msg); - nni_mtx_lock(&sock->mtx); - NNI_LIST_FOREACH (&sock->pipes, p) { - - nni_msg_clone(msg); - if (p->busy) { - if (nni_lmq_full(&p->sendq)) { - // Make space for the new message. - nni_msg *old; - (void) nni_lmq_getq(&p->sendq, &old); - nni_msg_free(old); - } - nni_lmq_putq(&p->sendq, msg); - } else { - p->busy = true; - nni_aio_set_msg(p->aio_send, msg); - nni_pipe_send(p->pipe, p->aio_send); - } - } - nni_mtx_unlock(&sock->mtx); - nng_msg_free(msg); - nni_aio_finish(aio, 0, len); -} - -static int -pub0_sock_get_sendfd(void *arg, void *buf, size_t *szp, nni_type t) -{ - pub0_sock *sock = arg; - int fd; - int rv; - nni_mtx_lock(&sock->mtx); - // PUB sockets are *always* writable. - nni_pollable_raise(sock->sendable); - rv = nni_pollable_getfd(sock->sendable, &fd); - nni_mtx_unlock(&sock->mtx); - - if (rv == 0) { - rv = nni_copyout_int(fd, buf, szp, t); - } - return (rv); -} - -static int -pub0_sock_set_sendbuf(void *arg, const void *buf, size_t sz, nni_type t) -{ - pub0_sock *sock = arg; - pub0_pipe *p; - int val; - int rv; - - if ((rv = nni_copyin_int(&val, buf, sz, 1, 8192, t)) != 0) { - return (rv); - } - - nni_mtx_lock(&sock->mtx); - sock->sendbuf = (size_t) val; - NNI_LIST_FOREACH (&sock->pipes, p) { - // If we fail part way thru (should only be ENOMEM), we - // stop short. The others would likely fail for ENOMEM as - // well anyway. There is a weird effect here where the - // buffers may have been set for *some* of the pipes, but - // we have no way to correct partial failure. - if ((rv = nni_lmq_resize(&p->sendq, (size_t) val)) != 0) { - break; - } - } - nni_mtx_unlock(&sock->mtx); - return (rv); -} - -static int -pub0_sock_get_sendbuf(void *arg, void *buf, size_t *szp, nni_type t) -{ - pub0_sock *sock = arg; - int val; - nni_mtx_lock(&sock->mtx); - val = (int) sock->sendbuf; - nni_mtx_unlock(&sock->mtx); - return (nni_copyout_int(val, buf, szp, t)); -} - -static nni_proto_pipe_ops pub0_pipe_ops = { - .pipe_size = sizeof(pub0_pipe), - .pipe_init = pub0_pipe_init, - .pipe_fini = pub0_pipe_fini, - .pipe_start = pub0_pipe_start, - .pipe_close = pub0_pipe_close, - .pipe_stop = pub0_pipe_stop, -}; - -static nni_option pub0_sock_options[] = { - // terminate list - { - .o_name = NNG_OPT_SENDFD, - .o_get = pub0_sock_get_sendfd, - }, - { - .o_name = NNG_OPT_SENDBUF, - .o_get = pub0_sock_get_sendbuf, - .o_set = pub0_sock_set_sendbuf, - }, - { - .o_name = NULL, - }, -}; - -static nni_proto_sock_ops pub0_sock_ops = { - .sock_size = sizeof(pub0_sock), - .sock_init = pub0_sock_init, - .sock_fini = pub0_sock_fini, - .sock_open = pub0_sock_open, - .sock_close = pub0_sock_close, - .sock_send = pub0_sock_send, - .sock_recv = pub0_sock_recv, - .sock_options = pub0_sock_options, -}; - -static nni_proto pub0_proto = { - .proto_version = NNI_PROTOCOL_VERSION, - .proto_self = { NNI_PROTO_PUB_V0, "pub" }, - .proto_peer = { NNI_PROTO_SUB_V0, "sub" }, - .proto_flags = NNI_PROTO_FLAG_SND, - .proto_sock_ops = &pub0_sock_ops, - .proto_pipe_ops = &pub0_pipe_ops, -}; - -static nni_proto pub0_proto_raw = { - .proto_version = NNI_PROTOCOL_VERSION, - .proto_self = { NNI_PROTO_PUB_V0, "pub" }, - .proto_peer = { NNI_PROTO_SUB_V0, "sub" }, - .proto_flags = NNI_PROTO_FLAG_SND | NNI_PROTO_FLAG_RAW, - .proto_sock_ops = &pub0_sock_ops, - .proto_pipe_ops = &pub0_pipe_ops, -}; - -int -nng_pub0_open(nng_socket *sidp) -{ - return (nni_proto_open(sidp, &pub0_proto)); -} - -int -nng_pub0_open_raw(nng_socket *sidp) -{ - return (nni_proto_open(sidp, &pub0_proto_raw)); -} diff --git a/src/protocol/pubsub0/pub_test.c b/src/protocol/pubsub0/pub_test.c deleted file mode 100644 index a430b610..00000000 --- a/src/protocol/pubsub0/pub_test.c +++ /dev/null @@ -1,331 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - -#include - -static void -test_pub_identity(void) -{ - nng_socket s; - int p; - char * n; - - NUTS_PASS(nng_pub0_open(&s)); - NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PROTO, &p)); - NUTS_TRUE(p == NUTS_PROTO(2u, 0u)); // 32 - NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PEER, &p)); - NUTS_TRUE(p == NUTS_PROTO(2u, 1u)); // 33 - NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PROTONAME, &n)); - NUTS_MATCH(n, "pub"); - nng_strfree(n); - NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PEERNAME, &n)); - NUTS_MATCH(n, "sub"); - nng_strfree(n); - NUTS_CLOSE(s); -} - -static void -test_pub_cannot_recv(void) -{ - nng_socket pub; - - NUTS_PASS(nng_pub0_open(&pub)); - NUTS_FAIL(nng_recv(pub, "", 0, 0), NNG_ENOTSUP); - NUTS_CLOSE(pub); -} - -static void -test_pub_no_context(void) -{ - nng_socket pub; - nng_ctx ctx; - - NUTS_PASS(nng_pub0_open(&pub)); - NUTS_FAIL(nng_ctx_open(&ctx, pub), NNG_ENOTSUP); - NUTS_CLOSE(pub); -} - -static void -test_pub_not_readable(void) -{ - int fd; - nng_socket pub; - - NUTS_PASS(nng_pub0_open(&pub)); - NUTS_FAIL(nng_socket_get_int(pub, NNG_OPT_RECVFD, &fd), NNG_ENOTSUP); - NUTS_CLOSE(pub); -} - -static void -test_pub_poll_writeable(void) -{ - int fd; - nng_socket pub; - nng_socket sub; - - NUTS_PASS(nng_sub0_open(&sub)); - NUTS_PASS(nng_pub0_open(&pub)); - NUTS_PASS(nng_socket_get_int(pub, NNG_OPT_SENDFD, &fd)); - NUTS_TRUE(fd >= 0); - - // Pub is *always* writeable - NUTS_TRUE(nuts_poll_fd(fd)); - - // Even after connect (no message yet) - NUTS_MARRY(pub, sub); - NUTS_TRUE(nuts_poll_fd(fd)); - - // Even if we send messages. - NUTS_SEND(pub, "abc"); - NUTS_TRUE(nuts_poll_fd(fd)); - - NUTS_CLOSE(pub); - NUTS_CLOSE(sub); -} - -static void -test_pub_send_no_pipes(void) -{ - nng_socket pub; - - NUTS_PASS(nng_pub0_open(&pub)); - NUTS_SEND(pub, "DROP1"); - NUTS_SEND(pub, "DROP2"); - NUTS_CLOSE(pub); -} - -void -test_pub_validate_peer(void) -{ - nng_socket s1, s2; - nng_stat * stats; - nng_stat * reject; - char *addr; - - NUTS_ADDR(addr, "inproc"); - - NUTS_PASS(nng_pub0_open(&s1)); - NUTS_PASS(nng_pub0_open(&s2)); - - NUTS_PASS(nng_listen(s1, addr, NULL, 0)); - NUTS_PASS(nng_dial(s2, addr, NULL, NNG_FLAG_NONBLOCK)); - - NUTS_SLEEP(100); - NUTS_PASS(nng_stats_get(&stats)); - - NUTS_TRUE(stats != NULL); - NUTS_TRUE((reject = nng_stat_find_socket(stats, s1)) != NULL); - NUTS_TRUE((reject = nng_stat_find(reject, "reject")) != NULL); - - NUTS_TRUE(nng_stat_type(reject) == NNG_STAT_COUNTER); - NUTS_TRUE(nng_stat_value(reject) > 0); - - NUTS_CLOSE(s1); - NUTS_CLOSE(s2); - nng_stats_free(stats); -} - -static void -test_pub_send_queued(void) -{ - nng_socket pub; - nng_socket sub; - - // MB: What we really need is a mock so that we can send harder - // than we receive -- we need a way to apply back-pressure for this - // test to be really meaningful. - NUTS_PASS(nng_pub0_open(&pub)); - NUTS_PASS(nng_sub0_open(&sub)); - NUTS_PASS(nng_socket_set(sub, NNG_OPT_SUB_SUBSCRIBE, "", 0)); - NUTS_PASS(nng_socket_set_int(pub, NNG_OPT_SENDBUF, 10)); - NUTS_PASS(nng_socket_set_int(sub, NNG_OPT_RECVBUF, 10)); - NUTS_PASS(nng_socket_set_ms(pub, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(sub, NNG_OPT_RECVTIMEO, 1000)); - NUTS_MARRY(pub, sub); - NUTS_SEND(pub, "first"); - NUTS_SEND(pub, "second"); - NUTS_SEND(pub, "three musketeers"); - NUTS_SEND(pub, "four"); - NUTS_SLEEP(50); - NUTS_RECV(sub, "first"); - NUTS_RECV(sub, "second"); - NUTS_RECV(sub, "three musketeers"); - NUTS_RECV(sub, "four"); - - NUTS_CLOSE(pub); - NUTS_CLOSE(sub); -} -static void -test_sub_recv_ctx_closed(void) -{ - nng_socket sub; - nng_ctx ctx; - nng_aio * aio; - NUTS_PASS(nng_sub0_open(&sub)); - NUTS_PASS(nng_ctx_open(&ctx, sub)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - nng_ctx_close(ctx); - nng_ctx_recv(ctx, aio); - nng_aio_wait(aio); - NUTS_FAIL(nng_aio_result(aio), NNG_ECLOSED); - nng_aio_free(aio); - NUTS_CLOSE(sub); -} - -static void -test_sub_ctx_recv_aio_stopped(void) -{ - nng_socket sub; - nng_ctx ctx; - nng_aio * aio; - - NUTS_PASS(nng_sub0_open(&sub)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - NUTS_PASS(nng_ctx_open(&ctx, sub)); - - nng_aio_stop(aio); - nng_ctx_recv(ctx, aio); - nng_aio_wait(aio); - NUTS_FAIL(nng_aio_result(aio), NNG_ECANCELED); - NUTS_PASS(nng_ctx_close(ctx)); - NUTS_CLOSE(sub); - nng_aio_free(aio); -} - -static void -test_sub_close_context_recv(void) -{ - nng_socket sub; - nng_ctx ctx; - nng_aio * aio; - - NUTS_PASS(nng_sub0_open(&sub)); - NUTS_PASS(nng_ctx_open(&ctx, sub)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - nng_aio_set_timeout(aio, 1000); - nng_ctx_recv(ctx, aio); - NUTS_PASS(nng_ctx_close(ctx)); - nng_aio_wait(aio); - NUTS_FAIL(nng_aio_result(aio), NNG_ECLOSED); - - NUTS_CLOSE(sub); - nng_aio_free(aio); -} - -static void -test_sub_ctx_recv_nonblock(void) -{ - nng_socket sub; - nng_ctx ctx; - nng_aio * aio; - - NUTS_PASS(nng_sub0_open(&sub)); - NUTS_PASS(nng_ctx_open(&ctx, sub)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - - nng_aio_set_timeout(aio, 0); // Instant timeout - nng_ctx_recv(ctx, aio); - - nng_aio_wait(aio); - NUTS_FAIL(nng_aio_result(aio), NNG_ETIMEDOUT); - NUTS_CLOSE(sub); - nng_aio_free(aio); -} - -static void -test_sub_ctx_recv_cancel(void) -{ - nng_socket sub; - nng_ctx ctx; - nng_aio * aio; - - NUTS_PASS(nng_sub0_open(&sub)); - NUTS_PASS(nng_ctx_open(&ctx, sub)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - - nng_aio_set_timeout(aio, 1000); - nng_ctx_recv(ctx, aio); - nng_aio_abort(aio, NNG_ECANCELED); - - nng_aio_wait(aio); - NUTS_FAIL(nng_aio_result(aio), NNG_ECANCELED); - NUTS_CLOSE(sub); - nng_aio_free(aio); -} - -static void -test_pub_send_buf_option(void) -{ - nng_socket pub; - int v; - bool b; - size_t sz; - const char *opt = NNG_OPT_SENDBUF; - - NUTS_PASS(nng_pub0_open(&pub)); - - NUTS_PASS(nng_socket_set_int(pub, opt, 1)); - NUTS_FAIL(nng_socket_set_int(pub, opt, 0), NNG_EINVAL); - NUTS_FAIL(nng_socket_set_int(pub, opt, -1), NNG_EINVAL); - NUTS_FAIL(nng_socket_set_int(pub, opt, 1000000), NNG_EINVAL); - NUTS_PASS(nng_socket_set_int(pub, opt, 3)); - NUTS_PASS(nng_socket_get_int(pub, opt, &v)); - NUTS_TRUE(v == 3); - v = 0; - sz = sizeof(v); - NUTS_PASS(nng_socket_get(pub, opt, &v, &sz)); - NUTS_TRUE(v == 3); - NUTS_TRUE(sz == sizeof(v)); - - NUTS_FAIL(nng_socket_set(pub, opt, "", 1), NNG_EINVAL); - sz = 1; - NUTS_FAIL(nng_socket_get(pub, opt, &v, &sz), NNG_EINVAL); - NUTS_FAIL(nng_socket_set_bool(pub, opt, true), NNG_EBADTYPE); - NUTS_FAIL(nng_socket_get_bool(pub, opt, &b), NNG_EBADTYPE); - - NUTS_CLOSE(pub); -} - -static void -test_pub_cooked(void) -{ - nng_socket s; - bool b; - - NUTS_PASS(nng_pub0_open(&s)); - NUTS_PASS(nng_socket_get_bool(s, NNG_OPT_RAW, &b)); - NUTS_TRUE(!b); - NUTS_FAIL(nng_socket_set_bool(s, NNG_OPT_RAW, true), NNG_EREADONLY); - NUTS_PASS(nng_close(s)); - - // raw pub only differs in the option setting - NUTS_PASS(nng_pub0_open_raw(&s)); - NUTS_PASS(nng_socket_get_bool(s, NNG_OPT_RAW, &b)); - NUTS_TRUE(b); - NUTS_CLOSE(s); -} - -NUTS_TESTS = { - { "pub identity", test_pub_identity }, - { "pub cannot recv", test_pub_cannot_recv }, - { "put no context", test_pub_no_context }, - { "pub not readable", test_pub_not_readable }, - { "pub poll writeable", test_pub_poll_writeable }, - { "pub validate peer", test_pub_validate_peer }, - { "pub send queued", test_pub_send_queued }, - { "pub send no pipes", test_pub_send_no_pipes }, - { "sub recv ctx closed", test_sub_recv_ctx_closed }, - { "sub recv aio ctx stopped", test_sub_ctx_recv_aio_stopped }, - { "sub close context recv", test_sub_close_context_recv }, - { "sub context recv nonblock", test_sub_ctx_recv_nonblock }, - { "sub context recv cancel", test_sub_ctx_recv_cancel }, - { "pub send buf option", test_pub_send_buf_option }, - { "pub cooked", test_pub_cooked }, - { NULL, NULL }, -}; diff --git a/src/protocol/pubsub0/sub.c b/src/protocol/pubsub0/sub.c deleted file mode 100644 index 9f3f2283..00000000 --- a/src/protocol/pubsub0/sub.c +++ /dev/null @@ -1,755 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// Copyright 2018 Capitar IT Group BV -// Copyright 2019 Nathan Kent -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - -#include -#include - -#include "core/nng_impl.h" -#include "nng/protocol/pubsub0/sub.h" - -// Subscriber protocol. The SUB protocol receives messages sent to -// it from publishers, and filters out those it is not interested in, -// only passing up ones that match known subscriptions. - -#ifndef NNI_PROTO_SUB_V0 -#define NNI_PROTO_SUB_V0 NNI_PROTO(2, 1) -#endif - -#ifndef NNI_PROTO_PUB_V0 -#define NNI_PROTO_PUB_V0 NNI_PROTO(2, 0) -#endif - -// By default we accept 128 messages. -#define SUB0_DEFAULT_RECV_BUF_LEN 128 - -// By default, prefer new messages when the queue is full. -#define SUB0_DEFAULT_PREFER_NEW true - -typedef struct sub0_pipe sub0_pipe; -typedef struct sub0_sock sub0_sock; -typedef struct sub0_ctx sub0_ctx; -typedef struct sub0_topic sub0_topic; - -static void sub0_recv_cb(void *); -static void sub0_pipe_fini(void *); - -struct sub0_topic { - nni_list_node node; - size_t len; - void * buf; -}; - -// sub0_ctx is a context for a SUB socket. The advantage of contexts is -// that different contexts can maintain different subscriptions. -struct sub0_ctx { - nni_list_node node; - sub0_sock * sock; - nni_list topics; // TODO: Consider patricia trie - nni_list recv_queue; // can have multiple pending receives - nni_lmq lmq; - bool prefer_new; -}; - -// sub0_sock is our per-socket protocol private structure. -struct sub0_sock { - nni_pollable readable; - sub0_ctx master; // default context - nni_list contexts; // all contexts - int num_contexts; - size_t recv_buf_len; - bool prefer_new; - nni_mtx lk; -}; - -// sub0_pipe is our per-pipe protocol private structure. -struct sub0_pipe { - nni_pipe * pipe; - sub0_sock *sub; - nni_aio aio_recv; -}; - -static void -sub0_ctx_cancel(nng_aio *aio, void *arg, int rv) -{ - sub0_ctx * ctx = arg; - sub0_sock *sock = ctx->sock; - nni_mtx_lock(&sock->lk); - if (nni_list_active(&ctx->recv_queue, aio)) { - nni_list_remove(&ctx->recv_queue, aio); - nni_aio_finish_error(aio, rv); - } - nni_mtx_unlock(&sock->lk); -} - -static void -sub0_ctx_recv(void *arg, nni_aio *aio) -{ - sub0_ctx * ctx = arg; - sub0_sock *sock = ctx->sock; - nni_msg * msg; - - if (nni_aio_begin(aio) != 0) { - return; - } - - nni_mtx_lock(&sock->lk); - -again: - if (nni_lmq_empty(&ctx->lmq)) { - int rv; - if ((rv = nni_aio_schedule(aio, sub0_ctx_cancel, ctx)) != 0) { - nni_mtx_unlock(&sock->lk); - nni_aio_finish_error(aio, rv); - return; - } - nni_list_append(&ctx->recv_queue, aio); - nni_mtx_unlock(&sock->lk); - return; - } - - (void) nni_lmq_getq(&ctx->lmq, &msg); - - if (nni_lmq_empty(&ctx->lmq) && (ctx == &sock->master)) { - nni_pollable_clear(&sock->readable); - } - if ((msg = nni_msg_unique(msg)) == NULL) { - goto again; - } - nni_aio_set_msg(aio, msg); - nni_mtx_unlock(&sock->lk); - nni_aio_finish(aio, 0, nni_msg_len(msg)); -} - -static void -sub0_ctx_send(void *arg, nni_aio *aio) -{ - NNI_ARG_UNUSED(arg); - if (nni_aio_begin(aio) == 0) { - nni_aio_finish_error(aio, NNG_ENOTSUP); - } -} - -static void -sub0_ctx_close(void *arg) -{ - sub0_ctx * ctx = arg; - sub0_sock *sock = ctx->sock; - nni_aio * aio; - - nni_mtx_lock(&sock->lk); - while ((aio = nni_list_first(&ctx->recv_queue)) != NULL) { - nni_list_remove(&ctx->recv_queue, aio); - nni_aio_finish_error(aio, NNG_ECLOSED); - } - nni_mtx_unlock(&sock->lk); -} - -static void -sub0_ctx_fini(void *arg) -{ - sub0_ctx * ctx = arg; - sub0_sock * sock = ctx->sock; - sub0_topic *topic; - - sub0_ctx_close(ctx); - - nni_mtx_lock(&sock->lk); - nni_list_remove(&sock->contexts, ctx); - sock->num_contexts--; - nni_mtx_unlock(&sock->lk); - - while ((topic = nni_list_first(&ctx->topics)) != 0) { - nni_list_remove(&ctx->topics, topic); - nni_free(topic->buf, topic->len); - NNI_FREE_STRUCT(topic); - } - - nni_lmq_fini(&ctx->lmq); -} - -static int -sub0_ctx_init(void *ctx_arg, void *sock_arg) -{ - sub0_sock *sock = sock_arg; - sub0_ctx * ctx = ctx_arg; - size_t len; - bool prefer_new; - int rv; - - nni_mtx_lock(&sock->lk); - len = sock->recv_buf_len; - prefer_new = sock->prefer_new; - - if ((rv = nni_lmq_init(&ctx->lmq, len)) != 0) { - return (rv); - } - ctx->prefer_new = prefer_new; - - nni_aio_list_init(&ctx->recv_queue); - NNI_LIST_INIT(&ctx->topics, sub0_topic, node); - - ctx->sock = sock; - - nni_list_append(&sock->contexts, ctx); - sock->num_contexts++; - nni_mtx_unlock(&sock->lk); - - return (0); -} - -static void -sub0_sock_fini(void *arg) -{ - sub0_sock *sock = arg; - - sub0_ctx_fini(&sock->master); - nni_pollable_fini(&sock->readable); - nni_mtx_fini(&sock->lk); -} - -static int -sub0_sock_init(void *arg, nni_sock *unused) -{ - sub0_sock *sock = arg; - int rv; - - NNI_ARG_UNUSED(unused); - - NNI_LIST_INIT(&sock->contexts, sub0_ctx, node); - nni_mtx_init(&sock->lk); - sock->recv_buf_len = SUB0_DEFAULT_RECV_BUF_LEN; - sock->prefer_new = SUB0_DEFAULT_PREFER_NEW; - nni_pollable_init(&sock->readable); - - if ((rv = sub0_ctx_init(&sock->master, sock)) != 0) { - sub0_sock_fini(sock); - return (rv); - } - - return (0); -} - -static void -sub0_sock_open(void *arg) -{ - NNI_ARG_UNUSED(arg); -} - -static void -sub0_sock_close(void *arg) -{ - sub0_sock *sock = arg; - sub0_ctx_close(&sock->master); -} - -static void -sub0_pipe_stop(void *arg) -{ - sub0_pipe *p = arg; - - nni_aio_stop(&p->aio_recv); -} - -static void -sub0_pipe_fini(void *arg) -{ - sub0_pipe *p = arg; - - nni_aio_fini(&p->aio_recv); -} - -static int -sub0_pipe_init(void *arg, nni_pipe *pipe, void *s) -{ - sub0_pipe *p = arg; - - nni_aio_init(&p->aio_recv, sub0_recv_cb, p); - - p->pipe = pipe; - p->sub = s; - return (0); -} - -static int -sub0_pipe_start(void *arg) -{ - sub0_pipe *p = arg; - - if (nni_pipe_peer(p->pipe) != NNI_PROTO_PUB_V0) { - // Peer protocol mismatch. - return (NNG_EPROTO); - } - - nni_pipe_recv(p->pipe, &p->aio_recv); - return (0); -} - -static void -sub0_pipe_close(void *arg) -{ - sub0_pipe *p = arg; - - nni_aio_close(&p->aio_recv); -} - -static bool -sub0_matches(sub0_ctx *ctx, uint8_t *body, size_t len) -{ - sub0_topic *topic; - - // This is a naive and trivial matcher. Replace with a real - // patricia trie later. - NNI_LIST_FOREACH (&ctx->topics, topic) { - if (len < topic->len) { - continue; - } - if ((topic->len == 0) || - (memcmp(topic->buf, body, topic->len) == 0)) { - return (true); - } - } - return (false); -} - -static void -sub0_recv_cb(void *arg) -{ - sub0_pipe *p = arg; - sub0_sock *sock = p->sub; - sub0_ctx * ctx; - nni_msg * msg; - size_t len; - uint8_t * body; - nni_list finish; - nng_aio * aio; - nni_msg * dup_msg; - - if (nni_aio_result(&p->aio_recv) != 0) { - nni_pipe_close(p->pipe); - return; - } - - nni_aio_list_init(&finish); - - msg = nni_aio_get_msg(&p->aio_recv); - nni_aio_set_msg(&p->aio_recv, NULL); - nni_msg_set_pipe(msg, nni_pipe_id(p->pipe)); - - body = nni_msg_body(msg); - len = nni_msg_len(msg); - dup_msg = NULL; - - nni_mtx_lock(&sock->lk); - // Go through all contexts. We will try to send up. - NNI_LIST_FOREACH (&sock->contexts, ctx) { - bool queued = false; - - if (nni_lmq_full(&ctx->lmq) && !ctx->prefer_new) { - // Cannot deliver here, as receive buffer is full. - continue; - } - - if (!sub0_matches(ctx, body, len)) { - continue; - } - - // This is a performance optimization, that ensures we - // do not duplicate a message in the common case, where there - // is only a single context. - if (sock->num_contexts > 1) { - if (nni_msg_dup(&dup_msg, msg) != 0) { - // if we cannot dup it, continue on - continue; - } - } else { - // We only have one context, so it's the only - // possible message. - dup_msg = msg; - } - - if (!nni_list_empty(&ctx->recv_queue)) { - aio = nni_list_first(&ctx->recv_queue); - nni_list_remove(&ctx->recv_queue, aio); - nni_aio_set_msg(aio, dup_msg); - - // Save for synchronous completion - nni_list_append(&finish, aio); - } else if (nni_lmq_full(&ctx->lmq)) { - // Make space for the new message. - nni_msg *old; - (void) nni_lmq_getq(&ctx->lmq, &old); - nni_msg_free(old); - - (void) nni_lmq_putq(&ctx->lmq, dup_msg); - queued = true; - - } else { - (void) nni_lmq_putq(&ctx->lmq, dup_msg); - queued = true; - } - if (queued && ctx == &sock->master) { - nni_pollable_raise(&sock->readable); - } - } - nni_mtx_unlock(&sock->lk); - - // NB: This is slightly less efficient in that we may have - // created an extra copy in the face of e.g. two subscriptions, - // but optimizing this further would require checking the subscription - // list twice, adding complexity. If this turns out to be a problem - // we could probably add some other sophistication with a counter - // and flags on the contexts themselves. - if (msg != dup_msg) { - // If we didn't just use the message, then free our copy. - nni_msg_free(msg); - } - - while ((aio = nni_list_first(&finish)) != NULL) { - nni_list_remove(&finish, aio); - nni_aio_finish_sync(aio, 0, len); - } - - nni_pipe_recv(p->pipe, &p->aio_recv); -} - -static int -sub0_ctx_get_recv_buf_len(void *arg, void *buf, size_t *szp, nni_type t) -{ - sub0_ctx * ctx = arg; - sub0_sock *sock = ctx->sock; - int val; - nni_mtx_lock(&sock->lk); - val = (int) nni_lmq_cap(&ctx->lmq); - nni_mtx_unlock(&sock->lk); - - return (nni_copyout_int(val, buf, szp, t)); -} - -static int -sub0_ctx_set_recv_buf_len(void *arg, const void *buf, size_t sz, nni_type t) -{ - sub0_ctx * ctx = arg; - sub0_sock *sock = ctx->sock; - int val; - int rv; - - if ((rv = nni_copyin_int(&val, buf, sz, 1, 8192, t)) != 0) { - return (rv); - } - nni_mtx_lock(&sock->lk); - if ((rv = nni_lmq_resize(&ctx->lmq, (size_t) val)) != 0) { - nni_mtx_unlock(&sock->lk); - return (rv); - } - - // If we change the socket, then this will change the queue for - // any new contexts. (Previously constructed contexts are unaffected.) - if (&sock->master == ctx) { - sock->recv_buf_len = (size_t) val; - } - nni_mtx_unlock(&sock->lk); - return (0); -} - -// For now we maintain subscriptions on a sorted linked list. As we do not -// expect to have huge numbers of subscriptions, and as the operation is -// really O(n), we think this is acceptable. In the future we might decide -// to replace this with a patricia trie, like old nanomsg had. - -static int -sub0_ctx_subscribe(void *arg, const void *buf, size_t sz, nni_type t) -{ - sub0_ctx * ctx = arg; - sub0_sock * sock = ctx->sock; - sub0_topic *topic; - sub0_topic *new_topic; - NNI_ARG_UNUSED(t); - - nni_mtx_lock(&sock->lk); - NNI_LIST_FOREACH (&ctx->topics, topic) { - if (topic->len != sz) { - continue; - } - if (memcmp(topic->buf, buf, sz) == 0) { - // Already have it. - nni_mtx_unlock(&sock->lk); - return (0); - } - } - if ((new_topic = NNI_ALLOC_STRUCT(new_topic)) == NULL) { - nni_mtx_unlock(&sock->lk); - return (NNG_ENOMEM); - } - if ((sz > 0) && ((new_topic->buf = nni_alloc(sz)) == NULL)) { - nni_mtx_unlock(&sock->lk); - NNI_FREE_STRUCT(new_topic); - return (NNG_ENOMEM); - } - memcpy(new_topic->buf, buf, sz); - new_topic->len = sz; - nni_list_append(&ctx->topics, new_topic); - nni_mtx_unlock(&sock->lk); - return (0); -} - -static int -sub0_ctx_unsubscribe(void *arg, const void *buf, size_t sz, nni_type t) -{ - sub0_ctx * ctx = arg; - sub0_sock * sock = ctx->sock; - sub0_topic *topic; - size_t len; - NNI_ARG_UNUSED(t); - - nni_mtx_lock(&sock->lk); - NNI_LIST_FOREACH (&ctx->topics, topic) { - if (topic->len != sz) { - continue; - } - if (memcmp(topic->buf, buf, sz) == 0) { - // Matched! - break; - } - } - if (topic == NULL) { - nni_mtx_unlock(&sock->lk); - return (NNG_ENOENT); - } - nni_list_remove(&ctx->topics, topic); - - // Now we need to make sure that any messages that are waiting still - // match the subscription. We basically just run through the queue - // and requeue those messages we need. - len = nni_lmq_len(&ctx->lmq); - for (size_t i = 0; i < len; i++) { - nni_msg *msg; - - (void) nni_lmq_getq(&ctx->lmq, &msg); - if (sub0_matches(ctx, nni_msg_body(msg), nni_msg_len(msg))) { - (void) nni_lmq_putq(&ctx->lmq, msg); - } else { - nni_msg_free(msg); - } - } - nni_mtx_unlock(&sock->lk); - - nni_free(topic->buf, topic->len); - NNI_FREE_STRUCT(topic); - return (0); -} - -static int -sub0_ctx_get_prefer_new(void *arg, void *buf, size_t *szp, nni_type t) -{ - sub0_ctx * ctx = arg; - sub0_sock *sock = ctx->sock; - bool val; - - nni_mtx_lock(&sock->lk); - val = ctx->prefer_new; - nni_mtx_unlock(&sock->lk); - - return (nni_copyout_bool(val, buf, szp, t)); -} - -static int -sub0_ctx_set_prefer_new(void *arg, const void *buf, size_t sz, nni_type t) -{ - sub0_ctx * ctx = arg; - sub0_sock *sock = ctx->sock; - bool val; - int rv; - - if ((rv = nni_copyin_bool(&val, buf, sz, t)) != 0) { - return (rv); - } - - nni_mtx_lock(&sock->lk); - ctx->prefer_new = val; - if (&sock->master == ctx) { - sock->prefer_new = val; - } - nni_mtx_unlock(&sock->lk); - - return (0); -} - -static nni_option sub0_ctx_options[] = { - { - .o_name = NNG_OPT_RECVBUF, - .o_get = sub0_ctx_get_recv_buf_len, - .o_set = sub0_ctx_set_recv_buf_len, - }, - { - .o_name = NNG_OPT_SUB_SUBSCRIBE, - .o_set = sub0_ctx_subscribe, - }, - { - .o_name = NNG_OPT_SUB_UNSUBSCRIBE, - .o_set = sub0_ctx_unsubscribe, - }, - { - .o_name = NNG_OPT_SUB_PREFNEW, - .o_get = sub0_ctx_get_prefer_new, - .o_set = sub0_ctx_set_prefer_new, - }, - { - .o_name = NULL, - }, -}; - -static void -sub0_sock_send(void *arg, nni_aio *aio) -{ - NNI_ARG_UNUSED(arg); - if (nni_aio_begin(aio) == 0) { - nni_aio_finish_error(aio, NNG_ENOTSUP); - } -} - -static void -sub0_sock_recv(void *arg, nni_aio *aio) -{ - sub0_sock *sock = arg; - - sub0_ctx_recv(&sock->master, aio); -} - -static int -sub0_sock_get_recv_fd(void *arg, void *buf, size_t *szp, nni_opt_type t) -{ - sub0_sock *sock = arg; - int rv; - int fd; - - if ((rv = nni_pollable_getfd(&sock->readable, &fd)) != 0) { - return (rv); - } - return (nni_copyout_int(fd, buf, szp, t)); -} - -static int -sub0_sock_get_recv_buf_len(void *arg, void *buf, size_t *szp, nni_type t) -{ - sub0_sock *sock = arg; - return (sub0_ctx_get_recv_buf_len(&sock->master, buf, szp, t)); -} - -static int -sub0_sock_set_recv_buf_len(void *arg, const void *buf, size_t sz, nni_type t) -{ - sub0_sock *sock = arg; - return (sub0_ctx_set_recv_buf_len(&sock->master, buf, sz, t)); -} - -static int -sub0_sock_subscribe(void *arg, const void *buf, size_t sz, nni_type t) -{ - sub0_sock *sock = arg; - return (sub0_ctx_subscribe(&sock->master, buf, sz, t)); -} - -static int -sub0_sock_unsubscribe(void *arg, const void *buf, size_t sz, nni_type t) -{ - sub0_sock *sock = arg; - return (sub0_ctx_unsubscribe(&sock->master, buf, sz, t)); -} - -static int -sub0_sock_get_prefer_new(void *arg, void *buf, size_t *szp, nni_type t) -{ - sub0_sock *sock = arg; - return (sub0_ctx_get_prefer_new(&sock->master, buf, szp, t)); -} - -static int -sub0_sock_set_prefer_new(void *arg, const void *buf, size_t sz, nni_type t) -{ - sub0_sock *sock = arg; - return (sub0_ctx_set_prefer_new(&sock->master, buf, sz, t)); -} - -// This is the global protocol structure -- our linkage to the core. -// This should be the only global non-static symbol in this file. -static nni_proto_pipe_ops sub0_pipe_ops = { - .pipe_size = sizeof(sub0_pipe), - .pipe_init = sub0_pipe_init, - .pipe_fini = sub0_pipe_fini, - .pipe_start = sub0_pipe_start, - .pipe_close = sub0_pipe_close, - .pipe_stop = sub0_pipe_stop, -}; - -static nni_proto_ctx_ops sub0_ctx_ops = { - .ctx_size = sizeof(sub0_ctx), - .ctx_init = sub0_ctx_init, - .ctx_fini = sub0_ctx_fini, - .ctx_send = sub0_ctx_send, - .ctx_recv = sub0_ctx_recv, - .ctx_options = sub0_ctx_options, -}; - -static nni_option sub0_sock_options[] = { - { - .o_name = NNG_OPT_SUB_SUBSCRIBE, - .o_set = sub0_sock_subscribe, - }, - { - .o_name = NNG_OPT_SUB_UNSUBSCRIBE, - .o_set = sub0_sock_unsubscribe, - }, - { - .o_name = NNG_OPT_RECVFD, - .o_get = sub0_sock_get_recv_fd, - }, - { - .o_name = NNG_OPT_RECVBUF, - .o_get = sub0_sock_get_recv_buf_len, - .o_set = sub0_sock_set_recv_buf_len, - }, - { - .o_name = NNG_OPT_SUB_PREFNEW, - .o_get = sub0_sock_get_prefer_new, - .o_set = sub0_sock_set_prefer_new, - }, - // terminate list - { - .o_name = NULL, - }, -}; - -static nni_proto_sock_ops sub0_sock_ops = { - .sock_size = sizeof(sub0_sock), - .sock_init = sub0_sock_init, - .sock_fini = sub0_sock_fini, - .sock_open = sub0_sock_open, - .sock_close = sub0_sock_close, - .sock_send = sub0_sock_send, - .sock_recv = sub0_sock_recv, - .sock_options = sub0_sock_options, -}; - -static nni_proto sub0_proto = { - .proto_version = NNI_PROTOCOL_VERSION, - .proto_self = { NNI_PROTO_SUB_V0, "sub" }, - .proto_peer = { NNI_PROTO_PUB_V0, "pub" }, - .proto_flags = NNI_PROTO_FLAG_RCV, - .proto_sock_ops = &sub0_sock_ops, - .proto_pipe_ops = &sub0_pipe_ops, - .proto_ctx_ops = &sub0_ctx_ops, -}; - -int -nng_sub0_open(nng_socket *sock) -{ - return (nni_proto_open(sock, &sub0_proto)); -} diff --git a/src/protocol/pubsub0/sub_test.c b/src/protocol/pubsub0/sub_test.c deleted file mode 100644 index b830ae80..00000000 --- a/src/protocol/pubsub0/sub_test.c +++ /dev/null @@ -1,624 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - -#include - -static void -test_sub_identity(void) -{ - nng_socket s; - int p; - char * n; - - NUTS_PASS(nng_sub0_open(&s)); - NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PROTO, &p)); - NUTS_TRUE(p == NUTS_PROTO(2u, 1u)); // 33 - NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PEER, &p)); - NUTS_TRUE(p == NUTS_PROTO(2u, 0u)); // 32 - NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PROTONAME, &n)); - NUTS_MATCH(n, "sub"); - nng_strfree(n); - NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PEERNAME, &n)); - NUTS_MATCH(n, "pub"); - nng_strfree(n); - NUTS_CLOSE(s); -} - -static void -test_sub_cannot_send(void) -{ - nng_socket sub; - - NUTS_PASS(nng_sub0_open(&sub)); - NUTS_FAIL(nng_send(sub, "", 0, 0), NNG_ENOTSUP); - NUTS_CLOSE(sub); -} - -static void -test_sub_context_cannot_send(void) -{ - nng_socket sub; - nng_ctx ctx; - nng_msg * m; - nng_aio * aio; - - NUTS_PASS(nng_sub0_open(&sub)); - NUTS_PASS(nng_ctx_open(&ctx, sub)); - NUTS_PASS(nng_msg_alloc(&m, 0)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - nng_aio_set_msg(aio, m); - nng_aio_set_timeout(aio, 1000); - nng_ctx_send(ctx, aio); - nng_aio_wait(aio); - NUTS_FAIL(nng_aio_result(aio), NNG_ENOTSUP); - NUTS_PASS(nng_ctx_close(ctx)); - NUTS_CLOSE(sub); - nng_aio_free(aio); - nng_msg_free(m); -} - -static void -test_sub_not_writeable(void) -{ - int fd; - nng_socket sub; - - NUTS_PASS(nng_sub0_open(&sub)); - NUTS_FAIL(nng_socket_get_int(sub, NNG_OPT_SENDFD, &fd), NNG_ENOTSUP); - NUTS_CLOSE(sub); -} - -static void -test_sub_poll_readable(void) -{ - int fd; - nng_socket pub; - nng_socket sub; - - NUTS_PASS(nng_sub0_open(&sub)); - NUTS_PASS(nng_pub0_open(&pub)); - NUTS_PASS(nng_socket_set(sub, NNG_OPT_SUB_SUBSCRIBE, "a", 1)); - NUTS_PASS(nng_socket_set_ms(sub, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(pub, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_get_int(sub, NNG_OPT_RECVFD, &fd)); - NUTS_TRUE(fd >= 0); - - // Not readable if not connected! - NUTS_TRUE(nuts_poll_fd(fd) == false); - - // Even after connect (no message yet) - NUTS_MARRY(pub, sub); - NUTS_TRUE(nuts_poll_fd(fd) == false); - - // If we send a message we didn't subscribe to, that doesn't matter. - NUTS_SEND(pub, "def"); - NUTS_SLEEP(100); - NUTS_TRUE(nuts_poll_fd(fd) == false); - - // But once we send messages, it is. - // We have to send a request, in order to send a reply. - NUTS_SEND(pub, "abc"); - NUTS_SLEEP(100); - NUTS_TRUE(nuts_poll_fd(fd)); - - // and receiving makes it no longer ready - NUTS_RECV(sub, "abc"); - NUTS_TRUE(nuts_poll_fd(fd) == false); - - NUTS_CLOSE(pub); - NUTS_CLOSE(sub); -} - -static void -test_sub_recv_late(void) -{ - int fd; - nng_socket pub; - nng_socket sub; - nng_aio * aio; - nng_msg * msg; - - NUTS_PASS(nng_sub0_open(&sub)); - NUTS_PASS(nng_pub0_open(&pub)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - NUTS_PASS(nng_socket_set(sub, NNG_OPT_SUB_SUBSCRIBE, "", 0)); - NUTS_PASS(nng_socket_set_ms(sub, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(pub, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_get_int(sub, NNG_OPT_RECVFD, &fd)); - NUTS_TRUE(fd >= 0); - - // Not readable if not connected! - NUTS_TRUE(nuts_poll_fd(fd) == false); - - // Even after connect (no message yet) - NUTS_MARRY(pub, sub); - NUTS_TRUE(nuts_poll_fd(fd) == false); - - nng_recv_aio(sub, aio); - - // But once we send messages, it is. - // We have to send a request, in order to send a reply. - NUTS_SEND(pub, "abc"); - NUTS_SLEEP(200); - - nng_aio_wait(aio); - NUTS_PASS(nng_aio_result(aio)); - msg = nng_aio_get_msg(aio); - nng_aio_set_msg(aio, NULL); - NUTS_TRUE(nng_msg_len(msg) == 4); - NUTS_MATCH(nng_msg_body(msg), "abc"); - - nng_msg_free(msg); - nng_aio_free(aio); - - NUTS_CLOSE(pub); - NUTS_CLOSE(sub); -} - -void -test_sub_context_no_poll(void) -{ - int fd; - nng_socket sub; - nng_ctx ctx; - - NUTS_PASS(nng_sub0_open(&sub)); - NUTS_PASS(nng_ctx_open(&ctx, sub)); - NUTS_FAIL(nng_ctx_get_int(ctx, NNG_OPT_SENDFD, &fd), NNG_ENOTSUP); - NUTS_FAIL(nng_ctx_get_int(ctx, NNG_OPT_RECVFD, &fd), NNG_ENOTSUP); - NUTS_PASS(nng_ctx_close(ctx)); - NUTS_CLOSE(sub); -} - -void -test_sub_validate_peer(void) -{ - nng_socket s1, s2; - nng_stat * stats; - nng_stat * reject; - char * addr; - - NUTS_ADDR(addr, "inproc"); - - NUTS_PASS(nng_sub0_open(&s1)); - NUTS_PASS(nng_sub0_open(&s2)); - - NUTS_PASS(nng_listen(s1, addr, NULL, 0)); - NUTS_PASS(nng_dial(s2, addr, NULL, NNG_FLAG_NONBLOCK)); - - NUTS_SLEEP(100); - NUTS_PASS(nng_stats_get(&stats)); - - NUTS_TRUE(stats != NULL); - NUTS_TRUE((reject = nng_stat_find_socket(stats, s1)) != NULL); - NUTS_TRUE((reject = nng_stat_find(reject, "reject")) != NULL); - - NUTS_TRUE(nng_stat_type(reject) == NNG_STAT_COUNTER); - NUTS_TRUE(nng_stat_value(reject) > 0); - - NUTS_CLOSE(s1); - NUTS_CLOSE(s2); - nng_stats_free(stats); -} - -static void -test_sub_recv_ctx_closed(void) -{ - nng_socket sub; - nng_ctx ctx; - nng_aio * aio; - NUTS_PASS(nng_sub0_open(&sub)); - NUTS_PASS(nng_ctx_open(&ctx, sub)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - nng_ctx_close(ctx); - nng_ctx_recv(ctx, aio); - nng_aio_wait(aio); - NUTS_FAIL(nng_aio_result(aio), NNG_ECLOSED); - nng_aio_free(aio); - NUTS_CLOSE(sub); -} - -static void -test_sub_ctx_recv_aio_stopped(void) -{ - nng_socket sub; - nng_ctx ctx; - nng_aio * aio; - - NUTS_PASS(nng_sub0_open(&sub)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - NUTS_PASS(nng_ctx_open(&ctx, sub)); - - nng_aio_stop(aio); - nng_ctx_recv(ctx, aio); - nng_aio_wait(aio); - NUTS_FAIL(nng_aio_result(aio), NNG_ECANCELED); - NUTS_PASS(nng_ctx_close(ctx)); - NUTS_CLOSE(sub); - nng_aio_free(aio); -} - -static void -test_sub_close_context_recv(void) -{ - nng_socket sub; - nng_ctx ctx; - nng_aio * aio; - - NUTS_PASS(nng_sub0_open(&sub)); - NUTS_PASS(nng_ctx_open(&ctx, sub)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - nng_aio_set_timeout(aio, 1000); - nng_ctx_recv(ctx, aio); - NUTS_PASS(nng_ctx_close(ctx)); - nng_aio_wait(aio); - NUTS_FAIL(nng_aio_result(aio), NNG_ECLOSED); - - NUTS_CLOSE(sub); - nng_aio_free(aio); -} - -static void -test_sub_ctx_recv_nonblock(void) -{ - nng_socket sub; - nng_ctx ctx; - nng_aio * aio; - - NUTS_PASS(nng_sub0_open(&sub)); - NUTS_PASS(nng_ctx_open(&ctx, sub)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - - nng_aio_set_timeout(aio, 0); // Instant timeout - nng_ctx_recv(ctx, aio); - - nng_aio_wait(aio); - NUTS_FAIL(nng_aio_result(aio), NNG_ETIMEDOUT); - NUTS_CLOSE(sub); - nng_aio_free(aio); -} - -static void -test_sub_ctx_recv_cancel(void) -{ - nng_socket sub; - nng_ctx ctx; - nng_aio * aio; - - NUTS_PASS(nng_sub0_open(&sub)); - NUTS_PASS(nng_ctx_open(&ctx, sub)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - - nng_aio_set_timeout(aio, 1000); - nng_ctx_recv(ctx, aio); - nng_aio_abort(aio, NNG_ECANCELED); - - nng_aio_wait(aio); - NUTS_FAIL(nng_aio_result(aio), NNG_ECANCELED); - NUTS_CLOSE(sub); - nng_aio_free(aio); -} - -static void -test_sub_recv_buf_option(void) -{ - nng_socket sub; - int v; - bool b; - size_t sz; - const char *opt = NNG_OPT_RECVBUF; - - NUTS_PASS(nng_sub0_open(&sub)); - - NUTS_PASS(nng_socket_set_int(sub, opt, 1)); - NUTS_FAIL(nng_socket_set_int(sub, opt, 0), NNG_EINVAL); - NUTS_FAIL(nng_socket_set_int(sub, opt, -1), NNG_EINVAL); - NUTS_FAIL(nng_socket_set_int(sub, opt, 1000000), NNG_EINVAL); - NUTS_PASS(nng_socket_set_int(sub, opt, 3)); - NUTS_PASS(nng_socket_get_int(sub, opt, &v)); - NUTS_TRUE(v == 3); - v = 0; - sz = sizeof(v); - NUTS_PASS(nng_socket_get(sub, opt, &v, &sz)); - NUTS_TRUE(v == 3); - NUTS_TRUE(sz == sizeof(v)); - - NUTS_FAIL(nng_socket_set(sub, opt, "", 1), NNG_EINVAL); - sz = 1; - NUTS_FAIL(nng_socket_get(sub, opt, &v, &sz), NNG_EINVAL); - NUTS_FAIL(nng_socket_set_bool(sub, opt, true), NNG_EBADTYPE); - NUTS_FAIL(nng_socket_get_bool(sub, opt, &b), NNG_EBADTYPE); - - NUTS_CLOSE(sub); -} - -static void -test_sub_subscribe_option(void) -{ - nng_socket sub; - size_t sz; - int v; - const char *opt = NNG_OPT_SUB_SUBSCRIBE; - - NUTS_PASS(nng_sub0_open(&sub)); - - NUTS_PASS(nng_socket_set(sub, opt, "abc", 3)); - NUTS_PASS(nng_socket_set(sub, opt, "abc", 3)); // duplicate - NUTS_PASS(nng_socket_set_bool(sub, opt, false)); - NUTS_PASS(nng_socket_set_int(sub, opt, 32)); - sz = sizeof(v); - NUTS_FAIL(nng_socket_get(sub, opt, &v, &sz), NNG_EWRITEONLY); - - NUTS_CLOSE(sub); -} - -static void -test_sub_unsubscribe_option(void) -{ - nng_socket sub; - size_t sz; - int v; - const char *opt1 = NNG_OPT_SUB_SUBSCRIBE; - const char *opt2 = NNG_OPT_SUB_UNSUBSCRIBE; - - NUTS_PASS(nng_sub0_open(&sub)); - - NUTS_PASS(nng_socket_set(sub, opt1, "abc", 3)); - NUTS_FAIL(nng_socket_set(sub, opt2, "abc123", 6), NNG_ENOENT); - NUTS_PASS(nng_socket_set(sub, opt2, "abc", 3)); - NUTS_FAIL(nng_socket_set(sub, opt2, "abc", 3), NNG_ENOENT); - NUTS_PASS(nng_socket_set_int(sub, opt1, 32)); - NUTS_FAIL(nng_socket_set_int(sub, opt2, 23), NNG_ENOENT); - NUTS_PASS(nng_socket_set_int(sub, opt2, 32)); - sz = sizeof(v); - NUTS_FAIL(nng_socket_get(sub, opt2, &v, &sz), NNG_EWRITEONLY); - - NUTS_CLOSE(sub); -} - -static void -test_sub_prefer_new_option(void) -{ - nng_socket sub; - bool b; - size_t sz; - const char *opt = NNG_OPT_SUB_PREFNEW; - - NUTS_PASS(nng_sub0_open(&sub)); - - NUTS_PASS(nng_socket_set_bool(sub, opt, true)); - NUTS_PASS(nng_socket_set_bool(sub, opt, false)); - NUTS_PASS(nng_socket_get_bool(sub, opt, &b)); - NUTS_TRUE(b == false); - sz = sizeof(b); - b = true; - NUTS_PASS(nng_socket_get(sub, opt, &b, &sz)); - NUTS_TRUE(b == false); - NUTS_TRUE(sz == sizeof(bool)); - - NUTS_FAIL(nng_socket_set(sub, opt, "abc", 3), NNG_EINVAL); - NUTS_FAIL(nng_socket_set_int(sub, opt, 1), NNG_EBADTYPE); - - NUTS_CLOSE(sub); -} - -void -test_sub_drop_new(void) -{ - nng_socket sub; - nng_socket pub; - nng_msg * msg; - - NUTS_PASS(nng_sub0_open(&sub)); - NUTS_PASS(nng_pub0_open(&pub)); - NUTS_PASS(nng_socket_set_int(sub, NNG_OPT_RECVBUF, 2)); - NUTS_PASS(nng_socket_set_bool(sub, NNG_OPT_SUB_PREFNEW, false)); - NUTS_PASS(nng_socket_set(sub, NNG_OPT_SUB_SUBSCRIBE, NULL, 0)); - NUTS_PASS(nng_socket_set_ms(sub, NNG_OPT_RECVTIMEO, 200)); - NUTS_PASS(nng_socket_set_ms(pub, NNG_OPT_SENDTIMEO, 1000)); - NUTS_MARRY(pub, sub); - NUTS_SEND(pub, "one"); - NUTS_SEND(pub, "two"); - NUTS_SEND(pub, "three"); - NUTS_SLEEP(100); - NUTS_RECV(sub, "one"); - NUTS_RECV(sub, "two"); - NUTS_FAIL(nng_recvmsg(sub, &msg, 0), NNG_ETIMEDOUT); - NUTS_CLOSE(pub); - NUTS_CLOSE(sub); -} - -void -test_sub_drop_old(void) -{ - nng_socket sub; - nng_socket pub; - nng_msg * msg; - - NUTS_PASS(nng_sub0_open(&sub)); - NUTS_PASS(nng_pub0_open(&pub)); - NUTS_PASS(nng_socket_set_int(sub, NNG_OPT_RECVBUF, 2)); - NUTS_PASS(nng_socket_set_bool(sub, NNG_OPT_SUB_PREFNEW, true)); - NUTS_PASS(nng_socket_set(sub, NNG_OPT_SUB_SUBSCRIBE, NULL, 0)); - NUTS_PASS(nng_socket_set_ms(sub, NNG_OPT_RECVTIMEO, 200)); - NUTS_PASS(nng_socket_set_ms(pub, NNG_OPT_SENDTIMEO, 1000)); - NUTS_MARRY(pub, sub); - NUTS_SEND(pub, "one"); - NUTS_SEND(pub, "two"); - NUTS_SEND(pub, "three"); - NUTS_SLEEP(100); - NUTS_RECV(sub, "two"); - NUTS_RECV(sub, "three"); - NUTS_FAIL(nng_recvmsg(sub, &msg, 0), NNG_ETIMEDOUT); - NUTS_CLOSE(pub); - NUTS_CLOSE(sub); -} - -static void -test_sub_filter(void) -{ - nng_socket sub; - nng_socket pub; - char buf[32]; - size_t sz; - - NUTS_PASS(nng_sub0_open(&sub)); - NUTS_PASS(nng_pub0_open(&pub)); - NUTS_PASS(nng_socket_set_ms(pub, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(sub, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_socket_set_int(sub, NNG_OPT_RECVBUF, 10)); - - // Set up some default filters - NUTS_PASS(nng_socket_set(sub, NNG_OPT_SUB_SUBSCRIBE, "abc", 3)); - NUTS_PASS(nng_socket_set(sub, NNG_OPT_SUB_SUBSCRIBE, "def", 3)); - NUTS_PASS(nng_socket_set(sub, NNG_OPT_SUB_SUBSCRIBE, "ghi", 3)); - NUTS_PASS(nng_socket_set(sub, NNG_OPT_SUB_SUBSCRIBE, "jkl", 3)); - - NUTS_MARRY(pub, sub); - - NUTS_PASS(nng_send(pub, "def", 3, 0)); - NUTS_PASS(nng_send(pub, "de", 2, 0)); // will not go through - NUTS_PASS(nng_send(pub, "abc123", 6, 0)); - NUTS_PASS(nng_send(pub, "xzy", 3, 0)); // does not match - NUTS_PASS(nng_send(pub, "ghi-drop", 7, 0)); // dropped by unsub - NUTS_PASS(nng_send(pub, "jkl-mno", 6, 0)); - - NUTS_SLEEP(100); - NUTS_PASS(nng_socket_set(sub, NNG_OPT_SUB_UNSUBSCRIBE, "ghi", 3)); - sz = sizeof(buf); - NUTS_PASS(nng_recv(sub, buf, &sz, 0)); - NUTS_TRUE(sz == 3); - NUTS_TRUE(memcmp(buf, "def", 3) == 0); - - sz = sizeof(buf); - NUTS_PASS(nng_recv(sub, buf, &sz, 0)); - NUTS_TRUE(sz == 6); - NUTS_TRUE(memcmp(buf, "abc123", 6) == 0); - - sz = sizeof(buf); - NUTS_PASS(nng_recv(sub, buf, &sz, 0)); - NUTS_TRUE(sz == 6); - NUTS_TRUE(memcmp(buf, "jkl-mno", 6) == 0); - - NUTS_CLOSE(sub); - NUTS_CLOSE(pub); -} - -static void -test_sub_multi_context(void) -{ - nng_socket sub; - nng_socket pub; - nng_ctx c1; - nng_ctx c2; - nng_aio * aio1; - nng_aio * aio2; - nng_msg * m; - - NUTS_PASS(nng_sub0_open(&sub)); - NUTS_PASS(nng_pub0_open(&pub)); - NUTS_PASS(nng_aio_alloc(&aio1, NULL, NULL)); - NUTS_PASS(nng_aio_alloc(&aio2, NULL, NULL)); - NUTS_PASS(nng_ctx_open(&c1, sub)); - NUTS_PASS(nng_ctx_open(&c2, sub)); - - NUTS_PASS(nng_ctx_set(c1, NNG_OPT_SUB_SUBSCRIBE, "one", 3)); - NUTS_PASS(nng_ctx_set(c1, NNG_OPT_SUB_SUBSCRIBE, "all", 3)); - - NUTS_PASS(nng_ctx_set(c2, NNG_OPT_SUB_SUBSCRIBE, "two", 3)); - NUTS_PASS(nng_ctx_set(c2, NNG_OPT_SUB_SUBSCRIBE, "all", 3)); - - nng_aio_set_timeout(aio1, 100); - nng_aio_set_timeout(aio2, 100); - - NUTS_MARRY(pub, sub); - - NUTS_SEND(pub, "one for the money"); - NUTS_SEND(pub, "all dogs go to heaven"); - NUTS_SEND(pub, "nobody likes a snitch"); - NUTS_SEND(pub, "two for the show"); - - nng_ctx_recv(c1, aio1); - nng_aio_wait(aio1); - NUTS_PASS(nng_aio_result(aio1)); - m = nng_aio_get_msg(aio1); - NUTS_MATCH(nng_msg_body(m), "one for the money"); - nng_msg_free(m); - - nng_ctx_recv(c1, aio1); - nng_aio_wait(aio1); - NUTS_PASS(nng_aio_result(aio1)); - m = nng_aio_get_msg(aio1); - NUTS_MATCH(nng_msg_body(m), "all dogs go to heaven"); - nng_msg_free(m); - - nng_ctx_recv(c2, aio1); - nng_aio_wait(aio1); - NUTS_PASS(nng_aio_result(aio1)); - m = nng_aio_get_msg(aio1); - NUTS_MATCH(nng_msg_body(m), "all dogs go to heaven"); - nng_msg_free(m); - - nng_ctx_recv(c2, aio1); - nng_aio_wait(aio1); - NUTS_PASS(nng_aio_result(aio1)); - m = nng_aio_get_msg(aio1); - NUTS_MATCH(nng_msg_body(m), "two for the show"); - nng_msg_free(m); - - nng_ctx_recv(c1, aio1); - nng_ctx_recv(c2, aio2); - - nng_aio_wait(aio1); - nng_aio_wait(aio2); - NUTS_FAIL(nng_aio_result(aio1), NNG_ETIMEDOUT); - NUTS_FAIL(nng_aio_result(aio2), NNG_ETIMEDOUT); - NUTS_CLOSE(sub); - NUTS_CLOSE(pub); - nng_aio_free(aio1); - nng_aio_free(aio2); -} - -static void -test_sub_cooked(void) -{ - nng_socket s; - bool b; - - NUTS_PASS(nng_sub0_open(&s)); - NUTS_PASS(nng_socket_get_bool(s, NNG_OPT_RAW, &b)); - NUTS_TRUE(!b); - NUTS_CLOSE(s); -} - -TEST_LIST = { - { "sub identity", test_sub_identity }, - { "sub cannot send", test_sub_cannot_send }, - { "sub context cannot send", test_sub_context_cannot_send }, - { "sub not writeable", test_sub_not_writeable }, - { "sub poll readable", test_sub_poll_readable }, - { "sub context does not poll", test_sub_context_no_poll }, - { "sub validate peer", test_sub_validate_peer }, - { "sub recv late", test_sub_recv_late }, - { "sub recv ctx closed", test_sub_recv_ctx_closed }, - { "sub recv aio ctx stopped", test_sub_ctx_recv_aio_stopped }, - { "sub close context recv", test_sub_close_context_recv }, - { "sub context recv nonblock", test_sub_ctx_recv_nonblock }, - { "sub context recv cancel", test_sub_ctx_recv_cancel }, - { "sub recv buf option", test_sub_recv_buf_option }, - { "sub subscribe option", test_sub_subscribe_option }, - { "sub unsubscribe option", test_sub_unsubscribe_option }, - { "sub prefer new option", test_sub_prefer_new_option }, - { "sub drop new", test_sub_drop_new }, - { "sub drop old", test_sub_drop_old }, - { "sub filter", test_sub_filter }, - { "sub multi context", test_sub_multi_context }, - { "sub cooked", test_sub_cooked }, - { NULL, NULL }, -}; diff --git a/src/protocol/pubsub0/xsub.c b/src/protocol/pubsub0/xsub.c deleted file mode 100644 index 0013b8b3..00000000 --- a/src/protocol/pubsub0/xsub.c +++ /dev/null @@ -1,211 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// Copyright 2018 Capitar IT Group BV -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - -#include - -#include "core/nng_impl.h" -#include "nng/protocol/pubsub0/sub.h" - -// Subscriber protocol. The SUB protocol receives messages sent to -// it from publishers, and filters out those it is not interested in, -// only passing up ones that match known subscriptions. - -#ifndef NNI_PROTO_SUB_V0 -#define NNI_PROTO_SUB_V0 NNI_PROTO(2, 1) -#endif - -#ifndef NNI_PROTO_PUB_V0 -#define NNI_PROTO_PUB_V0 NNI_PROTO(2, 0) -#endif - -typedef struct xsub0_pipe xsub0_pipe; -typedef struct xsub0_sock xsub0_sock; - -static void xsub0_recv_cb(void *); -static void xsub0_pipe_fini(void *); - -// xsub0_sock is our per-socket protocol private structure. -struct xsub0_sock { - nni_msgq *urq; - nni_mtx lk; -}; - -// sub0_pipe is our per-pipe protocol private structure. -struct xsub0_pipe { - nni_pipe * pipe; - xsub0_sock *sub; - nni_aio aio_recv; -}; - -static int -xsub0_sock_init(void *arg, nni_sock *sock) -{ - xsub0_sock *s = arg; - - s->urq = nni_sock_recvq(sock); - return (0); -} - -static void -xsub0_sock_fini(void *arg) -{ - xsub0_sock *s = arg; - nni_mtx_fini(&s->lk); -} - -static void -xsub0_sock_open(void *arg) -{ - NNI_ARG_UNUSED(arg); -} - -static void -xsub0_sock_close(void *arg) -{ - NNI_ARG_UNUSED(arg); -} - -static void -xsub0_pipe_stop(void *arg) -{ - xsub0_pipe *p = arg; - - nni_aio_stop(&p->aio_recv); -} - -static void -xsub0_pipe_fini(void *arg) -{ - xsub0_pipe *p = arg; - - nni_aio_fini(&p->aio_recv); -} - -static int -xsub0_pipe_init(void *arg, nni_pipe *pipe, void *s) -{ - xsub0_pipe *p = arg; - - nni_aio_init(&p->aio_recv, xsub0_recv_cb, p); - - p->pipe = pipe; - p->sub = s; - return (0); -} - -static int -xsub0_pipe_start(void *arg) -{ - xsub0_pipe *p = arg; - - if (nni_pipe_peer(p->pipe) != NNI_PROTO_PUB_V0) { - // Peer protocol mismatch. - return (NNG_EPROTO); - } - - nni_pipe_recv(p->pipe, &p->aio_recv); - return (0); -} - -static void -xsub0_pipe_close(void *arg) -{ - xsub0_pipe *p = arg; - - nni_aio_close(&p->aio_recv); -} - -static void -xsub0_recv_cb(void *arg) -{ - xsub0_pipe *p = arg; - xsub0_sock *s = p->sub; - nni_msgq * urq = s->urq; - nni_msg * msg; - - if (nni_aio_result(&p->aio_recv) != 0) { - nni_pipe_close(p->pipe); - return; - } - - msg = nni_aio_get_msg(&p->aio_recv); - nni_aio_set_msg(&p->aio_recv, NULL); - nni_msg_set_pipe(msg, nni_pipe_id(p->pipe)); - - if (nni_msgq_tryput(urq, msg) != 0) { - // This only happens for two reasons. For flow control, - // in which case we just want to discard the message and - // carry on, and for a close of the socket (which is very - // hard to achieve, since we close the pipes.) In either - // case the easiest thing to do is just free the message - // and try again. - nni_msg_free(msg); - } - nni_pipe_recv(p->pipe, &p->aio_recv); -} - -static void -xsub0_sock_send(void *arg, nni_aio *aio) -{ - NNI_ARG_UNUSED(arg); - nni_aio_finish_error(aio, NNG_ENOTSUP); -} - -static void -xsub0_sock_recv(void *arg, nni_aio *aio) -{ - xsub0_sock *s = arg; - - nni_msgq_aio_get(s->urq, aio); -} - -// This is the global protocol structure -- our linkage to the core. -// This should be the only global non-static symbol in this file. -static nni_proto_pipe_ops xsub0_pipe_ops = { - .pipe_size = sizeof(xsub0_pipe), - .pipe_init = xsub0_pipe_init, - .pipe_fini = xsub0_pipe_fini, - .pipe_start = xsub0_pipe_start, - .pipe_close = xsub0_pipe_close, - .pipe_stop = xsub0_pipe_stop, -}; - -static nni_option xsub0_sock_options[] = { - // terminate list - { - .o_name = NULL, - }, -}; - -static nni_proto_sock_ops xsub0_sock_ops = { - .sock_size = sizeof(xsub0_sock), - .sock_init = xsub0_sock_init, - .sock_fini = xsub0_sock_fini, - .sock_open = xsub0_sock_open, - .sock_close = xsub0_sock_close, - .sock_send = xsub0_sock_send, - .sock_recv = xsub0_sock_recv, - .sock_options = xsub0_sock_options, -}; - -static nni_proto xsub0_proto = { - .proto_version = NNI_PROTOCOL_VERSION, - .proto_self = { NNI_PROTO_SUB_V0, "sub" }, - .proto_peer = { NNI_PROTO_PUB_V0, "pub" }, - .proto_flags = NNI_PROTO_FLAG_RCV | NNI_PROTO_FLAG_RAW, - .proto_sock_ops = &xsub0_sock_ops, - .proto_pipe_ops = &xsub0_pipe_ops, -}; - -int -nng_sub0_open_raw(nng_socket *sidp) -{ - return (nni_proto_open(sidp, &xsub0_proto)); -} diff --git a/src/protocol/pubsub0/xsub_test.c b/src/protocol/pubsub0/xsub_test.c deleted file mode 100644 index 19815661..00000000 --- a/src/protocol/pubsub0/xsub_test.c +++ /dev/null @@ -1,376 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - -#include - -static void -test_xsub_identity(void) -{ - nng_socket s; - int p; - char * n; - - NUTS_PASS(nng_sub0_open_raw(&s)); - NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PROTO, &p)); - NUTS_TRUE(p == NUTS_PROTO(2u, 1u)); // 33 - NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PEER, &p)); - NUTS_TRUE(p == NUTS_PROTO(2u, 0u)); // 32 - NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PROTONAME, &n)); - NUTS_MATCH(n, "sub"); - nng_strfree(n); - NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PEERNAME, &n)); - NUTS_MATCH(n, "pub"); - nng_strfree(n); - NUTS_CLOSE(s); -} - -static void -test_xsub_cannot_send(void) -{ - nng_socket sub; - - NUTS_PASS(nng_sub0_open_raw(&sub)); - NUTS_FAIL(nng_send(sub, "", 0, 0), NNG_ENOTSUP); - NUTS_CLOSE(sub); -} - -static void -test_xsub_not_writeable(void) -{ - int fd; - nng_socket sub; - - NUTS_PASS(nng_sub0_open_raw(&sub)); - NUTS_FAIL(nng_socket_get_int(sub, NNG_OPT_SENDFD, &fd), NNG_ENOTSUP); - NUTS_CLOSE(sub); -} - -static void -test_xsub_poll_readable(void) -{ - int fd; - nng_socket pub; - nng_socket sub; - - NUTS_PASS(nng_sub0_open_raw(&sub)); - NUTS_PASS(nng_pub0_open(&pub)); - NUTS_PASS(nng_socket_set_ms(sub, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(pub, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_get_int(sub, NNG_OPT_RECVFD, &fd)); - NUTS_TRUE(fd >= 0); - - // Not readable if not connected! - NUTS_TRUE(nuts_poll_fd(fd) == false); - - // Even after connect (no message yet) - NUTS_MARRY(pub, sub); - NUTS_TRUE(nuts_poll_fd(fd) == false); - - // But once we send messages, it is. - // We have to send a request, in order to send a reply. - NUTS_SEND(pub, "abc"); - NUTS_SLEEP(200); - - NUTS_TRUE(nuts_poll_fd(fd)); - - // and receiving makes it no longer ready - NUTS_RECV(sub, "abc"); - NUTS_TRUE(nuts_poll_fd(fd) == false); - - NUTS_CLOSE(pub); - NUTS_CLOSE(sub); -} - -static void -test_xsub_recv_late(void) -{ - int fd; - nng_socket pub; - nng_socket sub; - nng_aio * aio; - nng_msg * msg; - - NUTS_PASS(nng_sub0_open_raw(&sub)); - NUTS_PASS(nng_pub0_open(&pub)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - NUTS_PASS(nng_socket_set_ms(sub, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(pub, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_get_int(sub, NNG_OPT_RECVFD, &fd)); - NUTS_TRUE(fd >= 0); - - // Not readable if not connected! - NUTS_TRUE(nuts_poll_fd(fd) == false); - - // Even after connect (no message yet) - NUTS_MARRY(pub, sub); - NUTS_TRUE(nuts_poll_fd(fd) == false); - - nng_recv_aio(sub, aio); - - // But once we send messages, it is. - // We have to send a request, in order to send a reply. - NUTS_SEND(pub, "abc"); - NUTS_SLEEP(200); - - nng_aio_wait(aio); - NUTS_PASS(nng_aio_result(aio)); - msg = nng_aio_get_msg(aio); - nng_aio_set_msg(aio, NULL); - NUTS_TRUE(nng_msg_len(msg) == 4); - NUTS_TRUE(strcmp(nng_msg_body(msg), "abc") == 0); - - nng_msg_free(msg); - nng_aio_free(aio); - - NUTS_CLOSE(pub); - NUTS_CLOSE(sub); -} - -void -test_xsub_no_context(void) -{ - nng_socket sub; - nng_ctx ctx; - - NUTS_PASS(nng_sub0_open_raw(&sub)); - NUTS_FAIL(nng_ctx_open(&ctx, sub), NNG_ENOTSUP); - NUTS_CLOSE(sub); -} - -void -test_xsub_validate_peer(void) -{ - nng_socket s1, s2; - nng_stat * stats; - nng_stat * reject; - char * addr; - - NUTS_ADDR(addr, "inproc"); - - NUTS_PASS(nng_sub0_open_raw(&s1)); - NUTS_PASS(nng_sub0_open_raw(&s2)); - - NUTS_PASS(nng_listen(s1, addr, NULL, 0)); - NUTS_PASS(nng_dial(s2, addr, NULL, NNG_FLAG_NONBLOCK)); - - NUTS_SLEEP(100); - NUTS_PASS(nng_stats_get(&stats)); - - NUTS_TRUE(stats != NULL); - NUTS_TRUE((reject = nng_stat_find_socket(stats, s1)) != NULL); - NUTS_TRUE((reject = nng_stat_find(reject, "reject")) != NULL); - - NUTS_TRUE(nng_stat_type(reject) == NNG_STAT_COUNTER); - NUTS_TRUE(nng_stat_value(reject) > 0); - - NUTS_CLOSE(s1); - NUTS_CLOSE(s2); - nng_stats_free(stats); -} - -static void -test_xsub_recv_closed(void) -{ - nng_socket sub; - nng_aio * aio; - NUTS_PASS(nng_sub0_open_raw(&sub)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - NUTS_CLOSE(sub); - nng_recv_aio(sub, aio); - nng_aio_wait(aio); - NUTS_FAIL(nng_aio_result(aio), NNG_ECLOSED); - nng_aio_free(aio); -} - -static void -test_xsub_close_recv(void) -{ - nng_socket sub; - nng_aio * aio; - - NUTS_PASS(nng_sub0_open_raw(&sub)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - nng_aio_set_timeout(aio, 1000); - nng_recv_aio(sub, aio); - NUTS_CLOSE(sub); - nng_aio_wait(aio); - NUTS_FAIL(nng_aio_result(aio), NNG_ECLOSED); - - nng_aio_free(aio); -} - -static void -test_xsub_recv_nonblock(void) -{ - nng_socket sub; - nng_aio * aio; - - NUTS_PASS(nng_sub0_open_raw(&sub)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - - nng_aio_set_timeout(aio, 0); // Instant timeout - nng_recv_aio(sub, aio); - - nng_aio_wait(aio); - NUTS_FAIL(nng_aio_result(aio), NNG_ETIMEDOUT); - NUTS_CLOSE(sub); - nng_aio_free(aio); -} - -static void -test_xsub_recv_buf_option(void) -{ - nng_socket sub; - int v; - bool b; - size_t sz; - const char *opt = NNG_OPT_RECVBUF; - - NUTS_PASS(nng_sub0_open_raw(&sub)); - - NUTS_PASS(nng_socket_set_int(sub, opt, 1)); - NUTS_FAIL(nng_socket_set_int(sub, opt, -1), NNG_EINVAL); - NUTS_FAIL(nng_socket_set_int(sub, opt, 1000000), NNG_EINVAL); - NUTS_PASS(nng_socket_set_int(sub, opt, 3)); - NUTS_PASS(nng_socket_get_int(sub, opt, &v)); - NUTS_TRUE(v == 3); - v = 0; - sz = sizeof(v); - NUTS_PASS(nng_socket_get(sub, opt, &v, &sz)); - NUTS_TRUE(v == 3); - NUTS_TRUE(sz == sizeof(v)); - - NUTS_FAIL(nng_socket_set(sub, opt, "", 1), NNG_EINVAL); - sz = 1; - NUTS_FAIL(nng_socket_get(sub, opt, &v, &sz), NNG_EINVAL); - NUTS_FAIL(nng_socket_set_bool(sub, opt, true), NNG_EBADTYPE); - NUTS_FAIL(nng_socket_get_bool(sub, opt, &b), NNG_EBADTYPE); - - NUTS_CLOSE(sub); -} - -static void -test_xsub_subscribe_option(void) -{ - nng_socket sub; - const char *opt = NNG_OPT_SUB_SUBSCRIBE; - - NUTS_PASS(nng_sub0_open_raw(&sub)); - NUTS_FAIL(nng_socket_set(sub, opt, "abc", 3), NNG_ENOTSUP); - NUTS_CLOSE(sub); -} - -static void -test_xsub_unsubscribe_option(void) -{ - nng_socket sub; - const char *opt = NNG_OPT_SUB_UNSUBSCRIBE; - - NUTS_PASS(nng_sub0_open_raw(&sub)); - NUTS_FAIL(nng_socket_set(sub, opt, "abc", 3), NNG_ENOTSUP); - NUTS_CLOSE(sub); -} - -static void -test_xsub_raw(void) -{ - nng_socket s; - bool b; - - NUTS_PASS(nng_sub0_open_raw(&s)); - NUTS_PASS(nng_socket_get_bool(s, NNG_OPT_RAW, &b)); - NUTS_TRUE(b); - NUTS_CLOSE(s); -} - -static void -test_xsub_close_during_recv(void) -{ - nng_socket sub; - nng_socket pub; - - NUTS_PASS(nng_sub0_open_raw(&sub)); - NUTS_PASS(nng_pub0_open(&pub)); - NUTS_PASS(nng_socket_set_ms(sub, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(pub, NNG_OPT_SENDTIMEO, 100)); - NUTS_PASS(nng_socket_set_int(sub, NNG_OPT_RECVBUF, 5)); - NUTS_PASS(nng_socket_set_int(pub, NNG_OPT_SENDBUF, 20)); - - NUTS_MARRY(pub, sub); - - for (unsigned i = 0; i < 100; i++) { - NUTS_PASS(nng_send(pub, "abc", 3, 0)); - } - NUTS_CLOSE(pub); - NUTS_CLOSE(sub); -} - -static void -test_xsub_close_during_pipe_recv(void) -{ - nng_socket sub; - nng_socket pub; - - NUTS_PASS(nng_sub0_open_raw(&sub)); - NUTS_PASS(nng_pub0_open(&pub)); - NUTS_PASS(nng_socket_set_ms(sub, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(pub, NNG_OPT_SENDTIMEO, 100)); - NUTS_PASS(nng_socket_set_int(sub, NNG_OPT_RECVBUF, 5)); - NUTS_PASS(nng_socket_set_int(pub, NNG_OPT_SENDBUF, 20)); - - NUTS_MARRY(pub, sub); - - for (unsigned i = 0; i < 100; i++) { - int rv; - rv = nng_send(pub, "abc", 3, 0); - if (rv == NNG_ETIMEDOUT) { - break; - } - NUTS_SLEEP(1); - } - NUTS_CLOSE(sub); -} - -static void -test_xsub_recv_aio_stopped(void) -{ - nng_socket sub; - nng_aio * aio; - - NUTS_PASS(nng_sub0_open_raw(&sub)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - - nng_aio_stop(aio); - nng_recv_aio(sub, aio); - nng_aio_wait(aio); - NUTS_FAIL(nng_aio_result(aio), NNG_ECANCELED); - NUTS_CLOSE(sub); - nng_aio_free(aio); -} - -TEST_LIST = { - { "xsub identity", test_xsub_identity }, - { "xsub cannot send", test_xsub_cannot_send }, - { "xsub not writeable", test_xsub_not_writeable }, - { "xsub poll readable", test_xsub_poll_readable }, - { "xsub validate peer", test_xsub_validate_peer }, - { "xsub recv late", test_xsub_recv_late }, - { "xsub recv closed", test_xsub_recv_closed }, - { "xsub close recv", test_xsub_close_recv }, - { "xsub recv nonblock", test_xsub_recv_nonblock }, - { "xsub recv buf option", test_xsub_recv_buf_option }, - { "xsub subscribe option", test_xsub_subscribe_option }, - { "xsub unsubscribe option", test_xsub_unsubscribe_option }, - { "xsub no context", test_xsub_no_context }, - { "xsub raw", test_xsub_raw }, - { "xsub recv aio stopped", test_xsub_recv_aio_stopped }, - { "xsub close during recv ", test_xsub_close_during_recv }, - { "xsub close during pipe recv", test_xsub_close_during_pipe_recv }, - { NULL, NULL }, -}; diff --git a/src/protocol/reqrep0/CMakeLists.txt b/src/protocol/reqrep0/CMakeLists.txt deleted file mode 100644 index a3cecfd0..00000000 --- a/src/protocol/reqrep0/CMakeLists.txt +++ /dev/null @@ -1,25 +0,0 @@ -# -# Copyright 2020 Staysail Systems, Inc. -# Copyright 2018 Capitar IT Group BV -# -# This software is supplied under the terms of the MIT License, a -# copy of which should be located in the distribution where this -# file was obtained (LICENSE.txt). A copy of the license may also be -# found online at https://opensource.org/licenses/MIT. -# - -# Req/Rep protocol -nng_directory(reqrep0) - -nng_sources_if(NNG_PROTO_REQ0 req.c xreq.c) -nng_headers_if(NNG_PROTO_REQ0 nng/protocol/reqrep0/req.h) -nng_defines_if(NNG_PROTO_REQ0 NNG_HAVE_REQ0) - -nng_sources_if(NNG_PROTO_REP0 rep.c xrep.c) -nng_headers_if(NNG_PROTO_REP0 nng/protocol/reqrep0/rep.h) -nng_defines_if(NNG_PROTO_REP0 NNG_HAVE_REP0) - -nng_test(req_test) -nng_test(rep_test) -nng_test(xrep_test) -nng_test(xreq_test) diff --git a/src/protocol/reqrep0/rep.c b/src/protocol/reqrep0/rep.c deleted file mode 100644 index aa32d249..00000000 --- a/src/protocol/reqrep0/rep.c +++ /dev/null @@ -1,705 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// Copyright 2018 Capitar IT Group BV -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - -#include - -#include "core/nng_impl.h" -#include "nng/protocol/reqrep0/rep.h" - -// Response protocol. The REP protocol is the "reply" side of a -// request-reply pair. This is useful for building RPC servers, for -// example. - -typedef struct rep0_pipe rep0_pipe; -typedef struct rep0_sock rep0_sock; -typedef struct rep0_ctx rep0_ctx; - -static void rep0_pipe_send_cb(void *); -static void rep0_pipe_recv_cb(void *); -static void rep0_pipe_fini(void *); - -struct rep0_ctx { - rep0_sock * sock; - uint32_t pipe_id; - rep0_pipe * spipe; // send pipe - nni_aio * saio; // send aio - nni_aio * raio; // recv aio - nni_list_node sqnode; - nni_list_node rqnode; - size_t btrace_len; - uint32_t btrace[NNI_MAX_MAX_TTL + 1]; -}; - -// rep0_sock is our per-socket protocol private structure. -struct rep0_sock { - nni_mtx lk; - nni_atomic_int ttl; - nni_id_map pipes; - nni_list recvpipes; // list of pipes with data to receive - nni_list recvq; - rep0_ctx ctx; - nni_pollable readable; - nni_pollable writable; -}; - -// rep0_pipe is our per-pipe protocol private structure. -struct rep0_pipe { - nni_pipe * pipe; - rep0_sock * rep; - uint32_t id; - nni_aio aio_send; - nni_aio aio_recv; - nni_list_node rnode; // receivable list linkage - nni_list sendq; // contexts waiting to send - bool busy; - bool closed; -}; - -static void -rep0_ctx_close(void *arg) -{ - rep0_ctx * ctx = arg; - rep0_sock *s = ctx->sock; - nni_aio * aio; - - nni_mtx_lock(&s->lk); - if ((aio = ctx->saio) != NULL) { - rep0_pipe *pipe = ctx->spipe; - ctx->saio = NULL; - ctx->spipe = NULL; - nni_list_remove(&pipe->sendq, ctx); - nni_aio_finish_error(aio, NNG_ECLOSED); - } - if ((aio = ctx->raio) != NULL) { - nni_list_remove(&s->recvq, ctx); - ctx->raio = NULL; - nni_aio_finish_error(aio, NNG_ECLOSED); - } - nni_mtx_unlock(&s->lk); -} - -static void -rep0_ctx_fini(void *arg) -{ - rep0_ctx *ctx = arg; - - rep0_ctx_close(ctx); -} - -static int -rep0_ctx_init(void *carg, void *sarg) -{ - rep0_sock *s = sarg; - rep0_ctx * ctx = carg; - - NNI_LIST_NODE_INIT(&ctx->sqnode); - NNI_LIST_NODE_INIT(&ctx->rqnode); - ctx->btrace_len = 0; - ctx->sock = s; - ctx->pipe_id = 0; - - return (0); -} - -static void -rep0_ctx_cancel_send(nni_aio *aio, void *arg, int rv) -{ - rep0_ctx * ctx = arg; - rep0_sock *s = ctx->sock; - - nni_mtx_lock(&s->lk); - if (ctx->saio != aio) { - nni_mtx_unlock(&s->lk); - return; - } - nni_list_node_remove(&ctx->sqnode); - ctx->saio = NULL; - nni_mtx_unlock(&s->lk); - - nni_msg_header_clear(nni_aio_get_msg(aio)); // reset the headers - nni_aio_finish_error(aio, rv); -} - -static void -rep0_ctx_send(void *arg, nni_aio *aio) -{ - rep0_ctx * ctx = arg; - rep0_sock *s = ctx->sock; - rep0_pipe *p; - nni_msg * msg; - int rv; - size_t len; - uint32_t p_id; // pipe id - - msg = nni_aio_get_msg(aio); - nni_msg_header_clear(msg); - - if (nni_aio_begin(aio) != 0) { - return; - } - - nni_mtx_lock(&s->lk); - len = ctx->btrace_len; - p_id = ctx->pipe_id; - - // Assert "completion" of the previous req request. This ensures - // exactly one send for one receive ordering. - ctx->btrace_len = 0; - ctx->pipe_id = 0; - - if (ctx == &s->ctx) { - // No matter how this goes, we will no longer be able - // to send on the socket (root context). That's because - // we will have finished (successfully or otherwise) the - // reply for the single request we got. - nni_pollable_clear(&s->writable); - } - if (len == 0) { - nni_mtx_unlock(&s->lk); - nni_aio_finish_error(aio, NNG_ESTATE); - return; - } - if ((rv = nni_msg_header_append(msg, ctx->btrace, len)) != 0) { - nni_mtx_unlock(&s->lk); - nni_aio_finish_error(aio, rv); - return; - } - if ((p = nni_id_get(&s->pipes, p_id)) == NULL) { - // Pipe is gone. Make this look like a good send to avoid - // disrupting the state machine. We don't care if the peer - // lost interest in our reply. - nni_mtx_unlock(&s->lk); - nni_aio_set_msg(aio, NULL); - nni_aio_finish(aio, 0, nni_msg_len(msg)); - nni_msg_free(msg); - return; - } - if (!p->busy) { - p->busy = true; - len = nni_msg_len(msg); - nni_aio_set_msg(&p->aio_send, msg); - nni_pipe_send(p->pipe, &p->aio_send); - nni_mtx_unlock(&s->lk); - - nni_aio_set_msg(aio, NULL); - nni_aio_finish(aio, 0, len); - return; - } - - if ((rv = nni_aio_schedule(aio, rep0_ctx_cancel_send, ctx)) != 0) { - nni_mtx_unlock(&s->lk); - nni_aio_finish_error(aio, rv); - return; - } - - ctx->saio = aio; - ctx->spipe = p; - nni_list_append(&p->sendq, ctx); - nni_mtx_unlock(&s->lk); -} - -static void -rep0_sock_fini(void *arg) -{ - rep0_sock *s = arg; - - nni_id_map_fini(&s->pipes); - rep0_ctx_fini(&s->ctx); - nni_pollable_fini(&s->writable); - nni_pollable_fini(&s->readable); - nni_mtx_fini(&s->lk); -} - -static int -rep0_sock_init(void *arg, nni_sock *sock) -{ - rep0_sock *s = arg; - - NNI_ARG_UNUSED(sock); - - nni_mtx_init(&s->lk); - nni_id_map_init(&s->pipes, 0, 0, false); - NNI_LIST_INIT(&s->recvq, rep0_ctx, rqnode); - NNI_LIST_INIT(&s->recvpipes, rep0_pipe, rnode); - nni_atomic_init(&s->ttl); - nni_atomic_set(&s->ttl, 8); - - (void) rep0_ctx_init(&s->ctx, s); - - // We start off without being either readable or writable. - // Readability comes when there is something on the socket. - nni_pollable_init(&s->writable); - nni_pollable_init(&s->readable); - - return (0); -} - -static void -rep0_sock_open(void *arg) -{ - NNI_ARG_UNUSED(arg); -} - -static void -rep0_sock_close(void *arg) -{ - rep0_sock *s = arg; - - rep0_ctx_close(&s->ctx); -} - -static void -rep0_pipe_stop(void *arg) -{ - rep0_pipe *p = arg; - - nni_aio_stop(&p->aio_send); - nni_aio_stop(&p->aio_recv); -} - -static void -rep0_pipe_fini(void *arg) -{ - rep0_pipe *p = arg; - nng_msg * msg; - - if ((msg = nni_aio_get_msg(&p->aio_recv)) != NULL) { - nni_aio_set_msg(&p->aio_recv, NULL); - nni_msg_free(msg); - } - - nni_aio_fini(&p->aio_send); - nni_aio_fini(&p->aio_recv); -} - -static int -rep0_pipe_init(void *arg, nni_pipe *pipe, void *s) -{ - rep0_pipe *p = arg; - - nni_aio_init(&p->aio_send, rep0_pipe_send_cb, p); - nni_aio_init(&p->aio_recv, rep0_pipe_recv_cb, p); - - NNI_LIST_INIT(&p->sendq, rep0_ctx, sqnode); - - p->id = nni_pipe_id(pipe); - p->pipe = pipe; - p->rep = s; - return (0); -} - -static int -rep0_pipe_start(void *arg) -{ - rep0_pipe *p = arg; - rep0_sock *s = p->rep; - int rv; - - if (nni_pipe_peer(p->pipe) != NNG_REP0_PEER) { - // Peer protocol mismatch. - return (NNG_EPROTO); - } - - nni_mtx_lock(&s->lk); - rv = nni_id_set(&s->pipes, nni_pipe_id(p->pipe), p); - nni_mtx_unlock(&s->lk); - if (rv != 0) { - return (rv); - } - // By definition, we have not received a request yet on this pipe, - // so it cannot cause us to become writable. - nni_pipe_recv(p->pipe, &p->aio_recv); - return (0); -} - -static void -rep0_pipe_close(void *arg) -{ - rep0_pipe *p = arg; - rep0_sock *s = p->rep; - rep0_ctx * ctx; - - nni_aio_close(&p->aio_send); - nni_aio_close(&p->aio_recv); - - nni_mtx_lock(&s->lk); - p->closed = true; - if (nni_list_active(&s->recvpipes, p)) { - // We are no longer "receivable". - nni_list_remove(&s->recvpipes, p); - } - while ((ctx = nni_list_first(&p->sendq)) != NULL) { - nni_aio *aio; - nni_msg *msg; - // Pipe was closed. To avoid pushing an error back to the - // entire socket, we pretend we completed this successfully. - nni_list_remove(&p->sendq, ctx); - aio = ctx->saio; - ctx->saio = NULL; - msg = nni_aio_get_msg(aio); - nni_aio_set_msg(aio, NULL); - nni_aio_finish(aio, 0, nni_msg_len(msg)); - nni_msg_free(msg); - } - if (p->id == s->ctx.pipe_id) { - // We "can" send. (Well, not really, but we will happily - // accept a message and discard it.) - nni_pollable_raise(&s->writable); - } - nni_id_remove(&s->pipes, nni_pipe_id(p->pipe)); - nni_mtx_unlock(&s->lk); -} - -static void -rep0_pipe_send_cb(void *arg) -{ - rep0_pipe *p = arg; - rep0_sock *s = p->rep; - rep0_ctx * ctx; - nni_aio * aio; - nni_msg * msg; - size_t len; - - if (nni_aio_result(&p->aio_send) != 0) { - nni_msg_free(nni_aio_get_msg(&p->aio_send)); - nni_aio_set_msg(&p->aio_send, NULL); - nni_pipe_close(p->pipe); - return; - } - nni_mtx_lock(&s->lk); - p->busy = false; - if ((ctx = nni_list_first(&p->sendq)) == NULL) { - // Nothing else to send. - if (p->id == s->ctx.pipe_id) { - // Mark us ready for the other side to send! - nni_pollable_raise(&s->writable); - } - nni_mtx_unlock(&s->lk); - return; - } - - nni_list_remove(&p->sendq, ctx); - aio = ctx->saio; - ctx->saio = NULL; - ctx->spipe = NULL; - p->busy = true; - msg = nni_aio_get_msg(aio); - len = nni_msg_len(msg); - nni_aio_set_msg(aio, NULL); - nni_aio_set_msg(&p->aio_send, msg); - nni_pipe_send(p->pipe, &p->aio_send); - - nni_mtx_unlock(&s->lk); - - nni_aio_finish_sync(aio, 0, len); -} - -static void -rep0_cancel_recv(nni_aio *aio, void *arg, int rv) -{ - rep0_ctx * ctx = arg; - rep0_sock *s = ctx->sock; - - nni_mtx_lock(&s->lk); - if (ctx->raio == aio) { - nni_list_remove(&s->recvq, ctx); - ctx->raio = NULL; - nni_aio_finish_error(aio, rv); - } - nni_mtx_unlock(&s->lk); -} - -static void -rep0_ctx_recv(void *arg, nni_aio *aio) -{ - rep0_ctx * ctx = arg; - rep0_sock *s = ctx->sock; - rep0_pipe *p; - size_t len; - nni_msg * msg; - - if (nni_aio_begin(aio) != 0) { - return; - } - nni_mtx_lock(&s->lk); - if ((p = nni_list_first(&s->recvpipes)) == NULL) { - int rv; - if ((rv = nni_aio_schedule(aio, rep0_cancel_recv, ctx)) != 0) { - nni_mtx_unlock(&s->lk); - nni_aio_finish_error(aio, rv); - return; - } - if (ctx->raio != NULL) { - // Cannot have a second receive operation pending. - // This could be ESTATE, or we could cancel the first - // with ECANCELED. We elect the former. - nni_mtx_unlock(&s->lk); - nni_aio_finish_error(aio, NNG_ESTATE); - return; - } - ctx->raio = aio; - nni_list_append(&s->recvq, ctx); - nni_mtx_unlock(&s->lk); - return; - } - msg = nni_aio_get_msg(&p->aio_recv); - nni_aio_set_msg(&p->aio_recv, NULL); - nni_list_remove(&s->recvpipes, p); - if (nni_list_empty(&s->recvpipes)) { - nni_pollable_clear(&s->readable); - } - nni_pipe_recv(p->pipe, &p->aio_recv); - if ((ctx == &s->ctx) && !p->busy) { - nni_pollable_raise(&s->writable); - } - - len = nni_msg_header_len(msg); - memcpy(ctx->btrace, nni_msg_header(msg), len); - ctx->btrace_len = len; - ctx->pipe_id = nni_pipe_id(p->pipe); - nni_mtx_unlock(&s->lk); - - nni_msg_header_clear(msg); - nni_aio_set_msg(aio, msg); - nni_aio_finish(aio, 0, nni_msg_len(msg)); -} - -static void -rep0_pipe_recv_cb(void *arg) -{ - rep0_pipe *p = arg; - rep0_sock *s = p->rep; - rep0_ctx * ctx; - nni_msg * msg; - uint8_t * body; - nni_aio * aio; - size_t len; - int hops; - int ttl; - - if (nni_aio_result(&p->aio_recv) != 0) { - nni_pipe_close(p->pipe); - return; - } - - msg = nni_aio_get_msg(&p->aio_recv); - ttl = nni_atomic_get(&s->ttl); - - nni_msg_set_pipe(msg, p->id); - - // Move backtrace from body to header - hops = 1; - for (;;) { - bool end; - - if (hops > ttl) { - // This isn't malformed, but it has gone - // through too many hops. Do not disconnect, - // because we can legitimately receive messages - // with too many hops from devices, etc. - goto drop; - } - hops++; - if (nni_msg_len(msg) < 4) { - // Peer is speaking garbage. Kick it. - nni_msg_free(msg); - nni_aio_set_msg(&p->aio_recv, NULL); - nni_pipe_close(p->pipe); - return; - } - body = nni_msg_body(msg); - end = ((body[0] & 0x80u) != 0); - if (nni_msg_header_append(msg, body, 4) != 0) { - // Out of memory, so drop it. - goto drop; - } - nni_msg_trim(msg, 4); - if (end) { - break; - } - } - - len = nni_msg_header_len(msg); - - nni_mtx_lock(&s->lk); - - if (p->closed) { - // If we are closed, then we can't return data. - nni_aio_set_msg(&p->aio_recv, NULL); - nni_mtx_unlock(&s->lk); - nni_msg_free(msg); - return; - } - - if ((ctx = nni_list_first(&s->recvq)) == NULL) { - // No one waiting to receive yet, holding pattern. - nni_list_append(&s->recvpipes, p); - nni_pollable_raise(&s->readable); - nni_mtx_unlock(&s->lk); - return; - } - - nni_list_remove(&s->recvq, ctx); - aio = ctx->raio; - ctx->raio = NULL; - nni_aio_set_msg(&p->aio_recv, NULL); - if ((ctx == &s->ctx) && !p->busy) { - nni_pollable_raise(&s->writable); - } - - // schedule another receive - nni_pipe_recv(p->pipe, &p->aio_recv); - - ctx->btrace_len = len; - memcpy(ctx->btrace, nni_msg_header(msg), len); - nni_msg_header_clear(msg); - ctx->pipe_id = p->id; - - nni_mtx_unlock(&s->lk); - - nni_aio_set_msg(aio, msg); - nni_aio_finish_sync(aio, 0, nni_msg_len(msg)); - return; - -drop: - nni_msg_free(msg); - nni_aio_set_msg(&p->aio_recv, NULL); - nni_pipe_recv(p->pipe, &p->aio_recv); -} - -static int -rep0_sock_set_max_ttl(void *arg, const void *buf, size_t sz, nni_opt_type t) -{ - rep0_sock *s = arg; - int ttl; - int rv; - - if ((rv = nni_copyin_int(&ttl, buf, sz, 1, NNI_MAX_MAX_TTL, t)) == 0) { - nni_atomic_set(&s->ttl, ttl); - } - return (rv); -} - -static int -rep0_sock_get_max_ttl(void *arg, void *buf, size_t *szp, nni_opt_type t) -{ - rep0_sock *s = arg; - - return (nni_copyout_int(nni_atomic_get(&s->ttl), buf, szp, t)); -} - -static int -rep0_sock_get_sendfd(void *arg, void *buf, size_t *szp, nni_opt_type t) -{ - rep0_sock *s = arg; - int rv; - int fd; - - if ((rv = nni_pollable_getfd(&s->writable, &fd)) != 0) { - return (rv); - } - return (nni_copyout_int(fd, buf, szp, t)); -} - -static int -rep0_sock_get_recvfd(void *arg, void *buf, size_t *szp, nni_opt_type t) -{ - rep0_sock *s = arg; - int rv; - int fd; - - if ((rv = nni_pollable_getfd(&s->readable, &fd)) != 0) { - return (rv); - } - - return (nni_copyout_int(fd, buf, szp, t)); -} - -static void -rep0_sock_send(void *arg, nni_aio *aio) -{ - rep0_sock *s = arg; - - rep0_ctx_send(&s->ctx, aio); -} - -static void -rep0_sock_recv(void *arg, nni_aio *aio) -{ - rep0_sock *s = arg; - - rep0_ctx_recv(&s->ctx, aio); -} - -// This is the global protocol structure -- our linkage to the core. -// This should be the only global non-static symbol in this file. -static nni_proto_pipe_ops rep0_pipe_ops = { - .pipe_size = sizeof(rep0_pipe), - .pipe_init = rep0_pipe_init, - .pipe_fini = rep0_pipe_fini, - .pipe_start = rep0_pipe_start, - .pipe_close = rep0_pipe_close, - .pipe_stop = rep0_pipe_stop, -}; - -static nni_proto_ctx_ops rep0_ctx_ops = { - .ctx_size = sizeof(rep0_ctx), - .ctx_init = rep0_ctx_init, - .ctx_fini = rep0_ctx_fini, - .ctx_send = rep0_ctx_send, - .ctx_recv = rep0_ctx_recv, -}; - -static nni_option rep0_sock_options[] = { - { - .o_name = NNG_OPT_MAXTTL, - .o_get = rep0_sock_get_max_ttl, - .o_set = rep0_sock_set_max_ttl, - }, - { - .o_name = NNG_OPT_RECVFD, - .o_get = rep0_sock_get_recvfd, - }, - { - .o_name = NNG_OPT_SENDFD, - .o_get = rep0_sock_get_sendfd, - }, - // terminate list - { - .o_name = NULL, - }, -}; - -static nni_proto_sock_ops rep0_sock_ops = { - .sock_size = sizeof(rep0_sock), - .sock_init = rep0_sock_init, - .sock_fini = rep0_sock_fini, - .sock_open = rep0_sock_open, - .sock_close = rep0_sock_close, - .sock_options = rep0_sock_options, - .sock_send = rep0_sock_send, - .sock_recv = rep0_sock_recv, -}; - -static nni_proto rep0_proto = { - .proto_version = NNI_PROTOCOL_VERSION, - .proto_self = { NNG_REP0_SELF, NNG_REP0_SELF_NAME }, - .proto_peer = { NNG_REP0_PEER, NNG_REP0_PEER_NAME }, - .proto_flags = NNI_PROTO_FLAG_SNDRCV, - .proto_sock_ops = &rep0_sock_ops, - .proto_pipe_ops = &rep0_pipe_ops, - .proto_ctx_ops = &rep0_ctx_ops, -}; - -int -nng_rep0_open(nng_socket *sidp) -{ - return (nni_proto_open(sidp, &rep0_proto)); -} diff --git a/src/protocol/reqrep0/rep_test.c b/src/protocol/reqrep0/rep_test.c deleted file mode 100644 index 5a47e67a..00000000 --- a/src/protocol/reqrep0/rep_test.c +++ /dev/null @@ -1,669 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - -#include - -static void -test_rep_identity(void) -{ - nng_socket s; - int p1, p2; - char * n1; - char * n2; - - NUTS_PASS(nng_rep0_open(&s)); - NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PROTO, &p1)); - NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PEER, &p2)); - NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PROTONAME, &n1)); - NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PEERNAME, &n2)); - NUTS_CLOSE(s); - NUTS_TRUE(p1 == NNG_REP0_SELF); - NUTS_TRUE(p2 == NNG_REP0_PEER); - NUTS_MATCH(n1, NNG_REP0_SELF_NAME); - NUTS_MATCH(n2, NNG_REP0_PEER_NAME); - nng_strfree(n1); - nng_strfree(n2); -} - -void -test_rep_send_bad_state(void) -{ - nng_socket rep; - nng_msg * msg = NULL; - - NUTS_TRUE(nng_rep0_open(&rep) == 0); - NUTS_TRUE(nng_msg_alloc(&msg, 0) == 0); - NUTS_TRUE(nng_sendmsg(rep, msg, 0) == NNG_ESTATE); - nng_msg_free(msg); - NUTS_CLOSE(rep); -} - -void -test_rep_poll_writeable(void) -{ - int fd; - nng_socket req; - nng_socket rep; - - NUTS_PASS(nng_req0_open(&req)); - NUTS_PASS(nng_rep0_open(&rep)); - NUTS_PASS(nng_socket_get_int(rep, NNG_OPT_SENDFD, &fd)); - NUTS_TRUE(fd >= 0); - - // Not writable before connect. - NUTS_TRUE(nuts_poll_fd(fd) == false); - - NUTS_MARRY(req, rep); - - // Still not writable. - NUTS_TRUE(nuts_poll_fd(fd) == false); - - // If we get a job, *then* we become writable - NUTS_SEND(req, "abc"); - NUTS_RECV(rep, "abc"); - NUTS_TRUE(nuts_poll_fd(fd) == true); - - // And is no longer writable once we send a message - NUTS_SEND(rep, "def"); - NUTS_TRUE(nuts_poll_fd(fd) == false); - // Even after receiving it - NUTS_RECV(req, "def"); - NUTS_TRUE(nuts_poll_fd(fd) == false); - - NUTS_CLOSE(req); - NUTS_CLOSE(rep); -} - -void -test_rep_poll_readable(void) -{ - int fd; - nng_socket req; - nng_socket rep; - nng_msg * msg; - - NUTS_PASS(nng_req0_open(&req)); - NUTS_PASS(nng_rep0_open(&rep)); - NUTS_PASS(nng_socket_get_int(rep, NNG_OPT_RECVFD, &fd)); - NUTS_TRUE(fd >= 0); - - // Not readable if not connected! - NUTS_TRUE(nuts_poll_fd(fd) == false); - - // Even after connect (no message yet) - NUTS_MARRY(req, rep); - NUTS_TRUE(nuts_poll_fd(fd) == false); - - // But once we send messages, it is. - // We have to send a request, in order to send a reply. - NUTS_SEND(req, "abc"); - NUTS_SLEEP(100); - - NUTS_TRUE(nuts_poll_fd(fd) == true); - - // and receiving makes it no longer ready - NUTS_PASS(nng_recvmsg(rep, &msg, 0)); - nng_msg_free(msg); - NUTS_TRUE(nuts_poll_fd(fd) == false); - - // TODO verify unsolicited response - - NUTS_CLOSE(req); - NUTS_CLOSE(rep); -} - -void -test_rep_context_no_poll(void) -{ - int fd; - nng_socket req; - nng_ctx ctx; - - NUTS_PASS(nng_rep0_open(&req)); - NUTS_PASS(nng_ctx_open(&ctx, req)); - NUTS_FAIL(nng_ctx_get_int(ctx, NNG_OPT_SENDFD, &fd), NNG_ENOTSUP); - NUTS_FAIL(nng_ctx_get_int(ctx, NNG_OPT_RECVFD, &fd), NNG_ENOTSUP); - NUTS_PASS(nng_ctx_close(ctx)); - NUTS_CLOSE(req); -} - -void -test_rep_validate_peer(void) -{ - nng_socket s1, s2; - nng_stat * stats; - nng_stat * reject; - char * addr; - - NUTS_ADDR(addr, "inproc"); - NUTS_PASS(nng_rep0_open(&s1)); - NUTS_PASS(nng_rep0_open(&s2)); - - NUTS_PASS(nng_listen(s1, addr, NULL, 0)); - NUTS_PASS(nng_dial(s2, addr, NULL, NNG_FLAG_NONBLOCK)); - - NUTS_SLEEP(100); - NUTS_PASS(nng_stats_get(&stats)); - - NUTS_TRUE(stats != NULL); - NUTS_TRUE((reject = nng_stat_find_socket(stats, s1)) != NULL); - NUTS_TRUE((reject = nng_stat_find(reject, "reject")) != NULL); - - NUTS_TRUE(nng_stat_type(reject) == NNG_STAT_COUNTER); - NUTS_TRUE(nng_stat_value(reject) > 0); - - NUTS_CLOSE(s1); - NUTS_CLOSE(s2); - nng_stats_free(stats); -} - -void -test_rep_double_recv(void) -{ - nng_socket s1; - nng_aio * aio1; - nng_aio * aio2; - - NUTS_PASS(nng_rep0_open(&s1)); - NUTS_PASS(nng_aio_alloc(&aio1, NULL, NULL)); - NUTS_PASS(nng_aio_alloc(&aio2, NULL, NULL)); - - nng_recv_aio(s1, aio1); - nng_recv_aio(s1, aio2); - - nng_aio_wait(aio2); - NUTS_FAIL(nng_aio_result(aio2), NNG_ESTATE); - NUTS_CLOSE(s1); - NUTS_FAIL(nng_aio_result(aio1), NNG_ECLOSED); - nng_aio_free(aio1); - nng_aio_free(aio2); -} - -void -test_rep_close_pipe_before_send(void) -{ - nng_socket rep; - nng_socket req; - nng_pipe p; - nng_aio * aio1; - nng_msg * m; - - NUTS_PASS(nng_rep0_open(&rep)); - NUTS_PASS(nng_req0_open(&req)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_aio_alloc(&aio1, NULL, NULL)); - - NUTS_MARRY(req, rep); - NUTS_SEND(req, "test"); - - nng_recv_aio(rep, aio1); - nng_aio_wait(aio1); - NUTS_PASS(nng_aio_result(aio1)); - NUTS_TRUE((m = nng_aio_get_msg(aio1)) != NULL); - p = nng_msg_get_pipe(m); - NUTS_PASS(nng_pipe_close(p)); - NUTS_PASS(nng_sendmsg(rep, m, 0)); - - NUTS_CLOSE(req); - NUTS_CLOSE(rep); - nng_aio_free(aio1); -} - -void -test_rep_close_pipe_during_send(void) -{ - nng_socket rep; - nng_socket req; - nng_pipe p = NNG_PIPE_INITIALIZER; - nng_msg * m; - - NUTS_PASS(nng_rep0_open(&rep)); - NUTS_PASS(nng_req0_open_raw(&req)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 200)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_int(rep, NNG_OPT_SENDBUF, 20)); - NUTS_PASS(nng_socket_set_int(rep, NNG_OPT_RECVBUF, 20)); - NUTS_PASS(nng_socket_set_int(req, NNG_OPT_SENDBUF, 20)); - NUTS_PASS(nng_socket_set_int(req, NNG_OPT_RECVBUF, 1)); - - NUTS_MARRY(req, rep); - - for (int i = 0; i < 100; i++) { - int rv; - NUTS_PASS(nng_msg_alloc(&m, 4)); - NUTS_PASS(nng_msg_append_u32(m, (unsigned) i | 0x80000000u)); - NUTS_PASS(nng_sendmsg(req, m, 0)); - NUTS_PASS(nng_recvmsg(rep, &m, 0)); - p = nng_msg_get_pipe(m); - rv = nng_sendmsg(rep, m, 0); - if (rv == NNG_ETIMEDOUT) { - // Queue is backed up, senders are busy. - nng_msg_free(m); - break; - } - NUTS_PASS(rv); - } - NUTS_PASS(nng_pipe_close(p)); - - NUTS_CLOSE(req); - NUTS_CLOSE(rep); -} - -void -test_rep_ctx_recv_aio_stopped(void) -{ - nng_socket rep; - nng_ctx ctx; - nng_aio * aio; - - NUTS_PASS(nng_rep0_open(&rep)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - NUTS_PASS(nng_ctx_open(&ctx, rep)); - - nng_aio_stop(aio); - nng_ctx_recv(ctx, aio); - nng_aio_wait(aio); - NUTS_FAIL(nng_aio_result(aio), NNG_ECANCELED); - NUTS_PASS(nng_ctx_close(ctx)); - NUTS_CLOSE(rep); - nng_aio_free(aio); -} - -void -test_rep_close_pipe_context_send(void) -{ - nng_socket rep; - nng_socket req; - nng_pipe p = NNG_PIPE_INITIALIZER; - nng_msg * m; - nng_ctx ctx[100]; - nng_aio * aio[100]; - int i; - - NUTS_PASS(nng_rep0_open(&rep)); - NUTS_PASS(nng_req0_open_raw(&req)); - for (i = 0; i < 100; i++) { - NUTS_PASS(nng_ctx_open(&ctx[i], rep)); - NUTS_PASS(nng_aio_alloc(&aio[i], NULL, NULL)); - } - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_int(rep, NNG_OPT_SENDBUF, 1)); - NUTS_PASS(nng_socket_set_int(rep, NNG_OPT_RECVBUF, 1)); - NUTS_PASS(nng_socket_set_int(req, NNG_OPT_SENDBUF, 1)); - NUTS_PASS(nng_socket_set_int(req, NNG_OPT_RECVBUF, 1)); - - NUTS_MARRY(req, rep); - - for (i = 0; i < 100; i++) { - NUTS_PASS(nng_msg_alloc(&m, 4)); - NUTS_PASS(nng_msg_append_u32(m, (unsigned) i | 0x80000000u)); - NUTS_PASS(nng_sendmsg(req, m, 0)); - nng_ctx_recv(ctx[i], aio[i]); - } - for (i = 0; i < 100; i++) { - nng_aio_wait(aio[i]); - NUTS_PASS(nng_aio_result(aio[i])); - NUTS_TRUE((m = nng_aio_get_msg(aio[i])) != NULL); - p = nng_msg_get_pipe(m); - nng_aio_set_msg(aio[i], m); - nng_ctx_send(ctx[i], aio[i]); - } - - // Note that REQ socket is not reading the results. - NUTS_PASS(nng_pipe_close(p)); - - for (i = 0; i < 100; i++) { - int rv; - nng_aio_wait(aio[i]); - rv = nng_aio_result(aio[i]); - if (rv != 0) { - NUTS_FAIL(rv, NNG_ECLOSED); - nng_msg_free(nng_aio_get_msg(aio[i])); - } - nng_aio_free(aio[i]); - NUTS_PASS(nng_ctx_close(ctx[i])); - } - NUTS_CLOSE(req); - NUTS_CLOSE(rep); -} - -void -test_rep_close_context_send(void) -{ - nng_socket rep; - nng_socket req; - nng_msg * m; - nng_ctx ctx[100]; - nng_aio * aio[100]; - int i; - - NUTS_PASS(nng_rep0_open(&rep)); - NUTS_PASS(nng_req0_open_raw(&req)); - for (i = 0; i < 100; i++) { - NUTS_PASS(nng_ctx_open(&ctx[i], rep)); - NUTS_PASS(nng_aio_alloc(&aio[i], NULL, NULL)); - } - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_int(rep, NNG_OPT_SENDBUF, 1)); - NUTS_PASS(nng_socket_set_int(rep, NNG_OPT_RECVBUF, 1)); - NUTS_PASS(nng_socket_set_int(req, NNG_OPT_SENDBUF, 1)); - NUTS_PASS(nng_socket_set_int(req, NNG_OPT_RECVBUF, 1)); - - NUTS_MARRY(req, rep); - - for (i = 0; i < 100; i++) { - NUTS_PASS(nng_msg_alloc(&m, 4)); - NUTS_PASS(nng_msg_append_u32(m, (unsigned) i | 0x80000000u)); - NUTS_PASS(nng_sendmsg(req, m, 0)); - nng_ctx_recv(ctx[i], aio[i]); - } - for (i = 0; i < 100; i++) { - nng_aio_wait(aio[i]); - NUTS_PASS(nng_aio_result(aio[i])); - NUTS_TRUE((m = nng_aio_get_msg(aio[i])) != NULL); - nng_aio_set_msg(aio[i], m); - nng_ctx_send(ctx[i], aio[i]); - } - - // Note that REQ socket is not reading the results. - for (i = 0; i < 100; i++) { - int rv; - NUTS_PASS(nng_ctx_close(ctx[i])); - nng_aio_wait(aio[i]); - rv = nng_aio_result(aio[i]); - if (rv != 0) { - NUTS_FAIL(rv, NNG_ECLOSED); - nng_msg_free(nng_aio_get_msg(aio[i])); - } - nng_aio_free(aio[i]); - } - NUTS_CLOSE(req); - NUTS_CLOSE(rep); -} - -void -test_rep_close_recv(void) -{ - nng_socket rep; - nng_socket req; - nng_aio * aio; - - NUTS_PASS(nng_rep0_open(&rep)); - NUTS_PASS(nng_req0_open_raw(&req)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); - - NUTS_MARRY(req, rep); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - nng_recv_aio(rep, aio); - NUTS_CLOSE(rep); - NUTS_CLOSE(req); - nng_aio_wait(aio); - NUTS_FAIL(nng_aio_result(aio), NNG_ECLOSED); - nng_aio_free(aio); -} - -struct rep_close_recv_cb_state { - nng_aio *aio; - nng_mtx *mtx; - nng_cv * cv; - int done; - int result; - nng_msg *msg; -}; - -static void -rep_close_recv_cb(void *arg) -{ - struct rep_close_recv_cb_state *state = arg; - - nng_mtx_lock(state->mtx); - state->result = nng_aio_result(state->aio); - state->msg = nng_aio_get_msg(state->aio); - state->done = true; - nng_cv_wake(state->cv); - nng_mtx_unlock(state->mtx); -} - -void -test_rep_close_recv_cb(void) -{ - nng_socket rep; - nng_socket req; - struct rep_close_recv_cb_state state; - - memset(&state, 0, sizeof(state)); - NUTS_PASS(nng_mtx_alloc(&state.mtx)); - NUTS_PASS(nng_cv_alloc(&state.cv, state.mtx)); - - NUTS_PASS(nng_rep0_open(&rep)); - NUTS_PASS(nng_req0_open_raw(&req)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); - - NUTS_MARRY(req, rep); - NUTS_PASS(nng_aio_alloc(&state.aio, rep_close_recv_cb, &state)); - nng_recv_aio(rep, state.aio); - NUTS_CLOSE(rep); - NUTS_CLOSE(req); - nng_mtx_lock(state.mtx); - while (!state.done) { - NUTS_PASS(nng_cv_until(state.cv, nng_clock() + 1000)); - } - nng_mtx_unlock(state.mtx); - NUTS_TRUE(state.done != 0); - NUTS_FAIL(nng_aio_result(state.aio), NNG_ECLOSED); - NUTS_TRUE(nng_aio_get_msg(state.aio) == NULL); - nng_aio_free(state.aio); - nng_cv_free(state.cv); - nng_mtx_free(state.mtx); -} - -static void -test_rep_ctx_recv_nonblock(void) -{ - nng_socket rep; - nng_ctx ctx; - nng_aio * aio; - - NUTS_PASS(nng_rep0_open(&rep)); - NUTS_PASS(nng_ctx_open(&ctx, rep)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - - nng_aio_set_timeout(aio, 0); // Instant timeout - nng_ctx_recv(ctx, aio); - - nng_aio_wait(aio); - NUTS_FAIL(nng_aio_result(aio), NNG_ETIMEDOUT); - NUTS_CLOSE(rep); - nng_aio_free(aio); -} - -static void -test_rep_ctx_send_nonblock(void) -{ - nng_socket rep; - nng_socket req; - nng_ctx ctx; - nng_aio * aio; - nng_msg * msg; - - NUTS_PASS(nng_req0_open(&req)); - NUTS_PASS(nng_rep0_open(&rep)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 2000)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_ctx_open(&ctx, rep)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - NUTS_MARRY(req, rep); - - NUTS_SEND(req, "SEND"); - nng_ctx_recv(ctx, aio); - nng_aio_wait(aio); - NUTS_PASS(nng_aio_result(aio)); - // message carries over - msg = nng_aio_get_msg(aio); - nng_aio_set_msg(aio, msg); - nng_aio_set_timeout(aio, 0); // Instant timeout - nng_ctx_send(ctx, aio); - - nng_aio_wait(aio); - NUTS_PASS(nng_aio_result(aio)); - NUTS_CLOSE(rep); - NUTS_CLOSE(req); - nng_aio_free(aio); -} - -static void -test_rep_ctx_send_nonblock2(void) -{ - nng_socket rep; - nng_socket req; - nng_ctx rep_ctx[10]; - nng_aio * rep_aio[10]; - int num_good = 0; - int num_fail = 0; - - // We are going to send a bunch of requests, receive them, - // but then see that non-block pressure exerts for some, but - // that at least one non-blocking send works. - NUTS_PASS(nng_req0_open_raw(&req)); - NUTS_PASS(nng_rep0_open(&rep)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 1000)); - for (int i = 0; i < 10; i++) { - NUTS_PASS(nng_ctx_open(&rep_ctx[i], rep)); - NUTS_PASS(nng_aio_alloc(&rep_aio[i], NULL, NULL)); - } - NUTS_MARRY(req, rep); - - for (int i = 0; i < 10; i++) { - nng_msg *msg; - NUTS_PASS(nng_msg_alloc(&msg, 4)); - NUTS_PASS(nng_msg_append_u32(msg, (unsigned) i | 0x80000000u)); - nng_ctx_recv(rep_ctx[i], rep_aio[i]); - NUTS_PASS(nng_sendmsg(req, msg, 0)); - } - for (int i = 0; i < 10; i++) { - nng_msg *msg; - nng_aio_wait(rep_aio[i]); - NUTS_PASS(nng_aio_result(rep_aio[i])); - msg = nng_aio_get_msg(rep_aio[i]); - nng_aio_set_timeout(rep_aio[i], 0); - nng_aio_set_msg(rep_aio[i], msg); - nng_ctx_send(rep_ctx[i], rep_aio[i]); - } - - for (int i = 0; i < 10; i++) { - int rv; - nng_aio_wait(rep_aio[i]); - rv = nng_aio_result(rep_aio[i]); - if (rv == 0) { - num_good++; - } else { - NUTS_FAIL(rv, NNG_ETIMEDOUT); - nng_msg_free(nng_aio_get_msg(rep_aio[i])); - num_fail++; - } - } - - TEST_ASSERT(num_good > 0); - TEST_ASSERT(num_fail > 0); - - for (int i = 0; i < 10; i++) { - nng_aio_free(rep_aio[i]); - nng_ctx_close(rep_ctx[i]); - } - NUTS_CLOSE(rep); - NUTS_CLOSE(req); -} - -static void -test_rep_send_nonblock(void) -{ - nng_socket rep; - nng_socket req; - int rv; - - NUTS_PASS(nng_req0_open(&req)); - NUTS_PASS(nng_rep0_open(&rep)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 1000)); - NUTS_MARRY(req, rep); - - NUTS_SEND(req, "SEND"); - NUTS_RECV(rep, "SEND"); - - // Use the nonblock flag - rv = nng_send(rep, "RECV", 5, NNG_FLAG_NONBLOCK); - - NUTS_PASS(rv); - NUTS_RECV(req, "RECV"); - NUTS_CLOSE(rep); - NUTS_CLOSE(req); -} - -void -test_rep_recv_garbage(void) -{ - nng_socket rep; - nng_socket req; - nng_msg * m; - - NUTS_PASS(nng_rep0_open(&rep)); - NUTS_PASS(nng_req0_open_raw(&req)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 200)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 200)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); - - NUTS_MARRY(req, rep); - - NUTS_PASS(nng_msg_alloc(&m, 4)); - NUTS_PASS(nng_msg_append_u32(m, 1u)); - NUTS_PASS(nng_sendmsg(req, m, 0)); - NUTS_FAIL(nng_recvmsg(rep, &m, 0), NNG_ETIMEDOUT); - - NUTS_CLOSE(req); - NUTS_CLOSE(rep); -} - -NUTS_TESTS = { - { "rep identity", test_rep_identity }, - { "rep send bad state", test_rep_send_bad_state }, - { "rep poll readable", test_rep_poll_readable }, - { "rep poll writable", test_rep_poll_writeable }, - { "rep context does not poll", test_rep_context_no_poll }, - { "rep validate peer", test_rep_validate_peer }, - { "rep double recv", test_rep_double_recv }, - { "rep send nonblock", test_rep_send_nonblock }, - { "rep close pipe before send", test_rep_close_pipe_before_send }, - { "rep close pipe during send", test_rep_close_pipe_during_send }, - { "rep recv aio ctx stopped", test_rep_ctx_recv_aio_stopped }, - { "rep close pipe context send", test_rep_close_pipe_context_send }, - { "rep close context send", test_rep_close_context_send }, - { "rep close recv", test_rep_close_recv }, - { "rep close recv cb", test_rep_close_recv_cb }, - { "rep context send nonblock", test_rep_ctx_send_nonblock }, - { "rep context send nonblock 2", test_rep_ctx_send_nonblock2 }, - { "rep context recv nonblock", test_rep_ctx_recv_nonblock }, - { "rep recv garbage", test_rep_recv_garbage }, - { NULL, NULL }, -}; diff --git a/src/protocol/reqrep0/req.c b/src/protocol/reqrep0/req.c deleted file mode 100644 index cb3c9395..00000000 --- a/src/protocol/reqrep0/req.c +++ /dev/null @@ -1,869 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// Copyright 2018 Capitar IT Group BV -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// -#include - -#include "core/nng_impl.h" -#include "nng/protocol/reqrep0/req.h" - -// Request protocol. The REQ protocol is the "request" side of a -// request-reply pair. This is useful for building RPC clients, for example. - -typedef struct req0_pipe req0_pipe; -typedef struct req0_sock req0_sock; -typedef struct req0_ctx req0_ctx; - -static void req0_run_send_queue(req0_sock *, nni_list *); -static void req0_ctx_reset(req0_ctx *); -static void req0_ctx_timeout(void *); -static void req0_pipe_fini(void *); -static void req0_ctx_fini(void *); -static int req0_ctx_init(void *, void *); - -// A req0_ctx is a "context" for the request. It uses most of the -// socket, but keeps track of its own outstanding replays, the request ID, -// and so forth. -struct req0_ctx { - req0_sock * sock; - nni_list_node sock_node; // node on the socket context list - nni_list_node send_node; // node on the send_queue - nni_list_node pipe_node; // node on the pipe list - uint32_t request_id; // request ID, without high bit set - nni_aio * recv_aio; // user aio waiting to recv - only one! - nni_aio * send_aio; // user aio waiting to send - nng_msg * req_msg; // request message (owned by protocol) - size_t req_len; // length of request message (for stats) - nng_msg * rep_msg; // reply message - nni_timer_node timer; - nni_duration retry; - bool conn_reset; // sent message w/o retry, peer disconnect -}; - -// A req0_sock is our per-socket protocol private structure. -struct req0_sock { - nni_duration retry; - bool closed; - nni_atomic_int ttl; - req0_ctx master; // base socket master - nni_list ready_pipes; - nni_list busy_pipes; - nni_list stop_pipes; - nni_list contexts; - nni_list send_queue; // contexts waiting to send. - nni_id_map requests; // contexts by request ID - nni_pollable readable; - nni_pollable writable; - nni_mtx mtx; -}; - -// A req0_pipe is our per-pipe protocol private structure. -struct req0_pipe { - nni_pipe * pipe; - req0_sock * req; - nni_list_node node; - nni_list contexts; // contexts with pending traffic - bool closed; - nni_aio aio_send; - nni_aio aio_recv; -}; - -static void req0_sock_fini(void *); -static void req0_send_cb(void *); -static void req0_recv_cb(void *); - -static int -req0_sock_init(void *arg, nni_sock *sock) -{ - req0_sock *s = arg; - - NNI_ARG_UNUSED(sock); - - // Request IDs are 32 bits, with the high order bit set. - // We start at a random point, to minimize likelihood of - // accidental collision across restarts. - nni_id_map_init(&s->requests, 0x80000000u, 0xffffffffu, true); - - nni_mtx_init(&s->mtx); - - NNI_LIST_INIT(&s->ready_pipes, req0_pipe, node); - NNI_LIST_INIT(&s->busy_pipes, req0_pipe, node); - NNI_LIST_INIT(&s->stop_pipes, req0_pipe, node); - NNI_LIST_INIT(&s->send_queue, req0_ctx, send_node); - NNI_LIST_INIT(&s->contexts, req0_ctx, sock_node); - - // this is "semi random" start for request IDs. - s->retry = NNI_SECOND * 60; - - (void) req0_ctx_init(&s->master, s); - - nni_pollable_init(&s->writable); - nni_pollable_init(&s->readable); - - nni_atomic_init(&s->ttl); - nni_atomic_set(&s->ttl, 8); - return (0); -} - -static void -req0_sock_open(void *arg) -{ - NNI_ARG_UNUSED(arg); -} - -static void -req0_sock_close(void *arg) -{ - req0_sock *s = arg; - - nni_mtx_lock(&s->mtx); - s->closed = true; - nni_mtx_unlock(&s->mtx); -} - -static void -req0_sock_fini(void *arg) -{ - req0_sock *s = arg; - - nni_mtx_lock(&s->mtx); - NNI_ASSERT(nni_list_empty(&s->busy_pipes)); - NNI_ASSERT(nni_list_empty(&s->stop_pipes)); - NNI_ASSERT(nni_list_empty(&s->ready_pipes)); - nni_mtx_unlock(&s->mtx); - - req0_ctx_fini(&s->master); - nni_pollable_fini(&s->readable); - nni_pollable_fini(&s->writable); - nni_id_map_fini(&s->requests); - nni_mtx_fini(&s->mtx); -} - -static void -req0_pipe_stop(void *arg) -{ - req0_pipe *p = arg; - req0_sock *s = p->req; - - nni_aio_stop(&p->aio_recv); - nni_aio_stop(&p->aio_send); - nni_mtx_lock(&s->mtx); - nni_list_node_remove(&p->node); - nni_mtx_unlock(&s->mtx); -} - -static void -req0_pipe_fini(void *arg) -{ - req0_pipe *p = arg; - - nni_aio_fini(&p->aio_recv); - nni_aio_fini(&p->aio_send); -} - -static int -req0_pipe_init(void *arg, nni_pipe *pipe, void *s) -{ - req0_pipe *p = arg; - - nni_aio_init(&p->aio_recv, req0_recv_cb, p); - nni_aio_init(&p->aio_send, req0_send_cb, p); - NNI_LIST_NODE_INIT(&p->node); - NNI_LIST_INIT(&p->contexts, req0_ctx, pipe_node); - p->pipe = pipe; - p->req = s; - return (0); -} - -static int -req0_pipe_start(void *arg) -{ - req0_pipe *p = arg; - req0_sock *s = p->req; - - if (nni_pipe_peer(p->pipe) != NNG_REQ0_PEER) { - return (NNG_EPROTO); - } - - nni_mtx_lock(&s->mtx); - nni_list_append(&s->ready_pipes, p); - nni_pollable_raise(&s->writable); - req0_run_send_queue(s, NULL); - nni_mtx_unlock(&s->mtx); - - nni_pipe_recv(p->pipe, &p->aio_recv); - return (0); -} - -static void -req0_pipe_close(void *arg) -{ - req0_pipe *p = arg; - req0_sock *s = p->req; - req0_ctx * ctx; - - nni_aio_close(&p->aio_recv); - nni_aio_close(&p->aio_send); - - nni_mtx_lock(&s->mtx); - // This removes the node from either busy_pipes or ready_pipes. - // It doesn't much matter which. We stick the pipe on the stop - // list, so that we can wait for that to close down safely. - p->closed = true; - nni_list_node_remove(&p->node); - nni_list_append(&s->stop_pipes, p); - if (nni_list_empty(&s->ready_pipes)) { - nni_pollable_clear(&s->writable); - } - - while ((ctx = nni_list_first(&p->contexts)) != NULL) { - nni_list_remove(&p->contexts, ctx); - nng_aio *aio; - if (ctx->retry <= 0) { - // If we can't retry, then just cancel the operation - // altogether. We should only be waiting for recv, - // because we will already have sent if we are here. - if ((aio = ctx->recv_aio) != NULL) { - ctx->recv_aio = NULL; - nni_aio_finish_error(aio, NNG_ECONNRESET); - req0_ctx_reset(ctx); - } else { - req0_ctx_reset(ctx); - ctx->conn_reset = true; - } - } else { - // Reset the timer on this so it expires immediately. - // This is actually easier than canceling the timer and - // running the send_queue separately. (In particular, - // it avoids a potential deadlock on cancelling the - // timer.) - nni_timer_schedule(&ctx->timer, NNI_TIME_ZERO); - } - } - nni_mtx_unlock(&s->mtx); -} - -// For cooked mode, we use a context, and send out that way. This -// completely bypasses the upper write queue. Each context keeps one -// message pending; these are "scheduled" via the send_queue. The send_queue -// is ordered, so FIFO ordering between contexts is provided for. - -static void -req0_send_cb(void *arg) -{ - req0_pipe *p = arg; - req0_sock *s = p->req; - nni_aio * aio; - nni_list sent_list; - - nni_aio_list_init(&sent_list); - if (nni_aio_result(&p->aio_send) != 0) { - // We failed to send... clean up and deal with it. - nni_msg_free(nni_aio_get_msg(&p->aio_send)); - nni_aio_set_msg(&p->aio_send, NULL); - nni_pipe_close(p->pipe); - return; - } - - // We completed a cooked send, so we need to reinsert ourselves - // in the ready list, and re-run the send_queue. - - nni_mtx_lock(&s->mtx); - if (p->closed || s->closed) { - // This occurs if the req0_pipe_close has been called. - // In that case we don't want any more processing. - nni_mtx_unlock(&s->mtx); - return; - } - nni_list_remove(&s->busy_pipes, p); - nni_list_append(&s->ready_pipes, p); - if (nni_list_empty(&s->send_queue)) { - nni_pollable_raise(&s->writable); - } - req0_run_send_queue(s, &sent_list); - nni_mtx_unlock(&s->mtx); - - while ((aio = nni_list_first(&sent_list)) != NULL) { - nni_list_remove(&sent_list, aio); - nni_aio_finish_sync(aio, 0, 0); - } -} - -static void -req0_recv_cb(void *arg) -{ - req0_pipe *p = arg; - req0_sock *s = p->req; - req0_ctx * ctx; - nni_msg * msg; - nni_aio * aio; - uint32_t id; - - if (nni_aio_result(&p->aio_recv) != 0) { - nni_pipe_close(p->pipe); - return; - } - - msg = nni_aio_get_msg(&p->aio_recv); - nni_aio_set_msg(&p->aio_recv, NULL); - nni_msg_set_pipe(msg, nni_pipe_id(p->pipe)); - - // We yank 4 bytes from front of body, and move them to the header. - if (nni_msg_len(msg) < 4) { - // Malformed message. - goto malformed; - } - id = nni_msg_trim_u32(msg); - - // Schedule another receive while we are processing this. - nni_mtx_lock(&s->mtx); - - // NB: If close was called, then this will just abort. - nni_pipe_recv(p->pipe, &p->aio_recv); - - // Look for a context to receive it. - if (((ctx = nni_id_get(&s->requests, id)) == NULL) || - (ctx->send_aio != NULL) || (ctx->rep_msg != NULL)) { - nni_mtx_unlock(&s->mtx); - // No waiting context, we have not sent the request out to - // the wire yet, or context already has a reply ready. - // Discard the message. - nni_msg_free(msg); - return; - } - - // We have our match, so we can remove this. - nni_list_node_remove(&ctx->send_node); - nni_id_remove(&s->requests, id); - ctx->request_id = 0; - if (ctx->req_msg != NULL) { - nni_msg_free(ctx->req_msg); - ctx->req_msg = NULL; - } - - // Is there an aio waiting for us? - if ((aio = ctx->recv_aio) != NULL) { - ctx->recv_aio = NULL; - nni_mtx_unlock(&s->mtx); - nni_aio_set_msg(aio, msg); - nni_aio_finish_sync(aio, 0, nni_msg_len(msg)); - } else { - // No AIO, so stash msg. Receive will pick it up later. - ctx->rep_msg = msg; - if (ctx == &s->master) { - nni_pollable_raise(&s->readable); - } - nni_mtx_unlock(&s->mtx); - } - return; - -malformed: - nni_msg_free(msg); - nni_pipe_close(p->pipe); -} - -static void -req0_ctx_timeout(void *arg) -{ - req0_ctx * ctx = arg; - req0_sock *s = ctx->sock; - - nni_mtx_lock(&s->mtx); - if ((ctx->req_msg != NULL) && (!s->closed)) { - if (!nni_list_node_active(&ctx->send_node)) { - nni_list_append(&s->send_queue, ctx); - } - req0_run_send_queue(s, NULL); - } - nni_mtx_unlock(&s->mtx); -} - -static int -req0_ctx_init(void *arg, void *sock) -{ - req0_sock *s = sock; - req0_ctx * ctx = arg; - - nni_timer_init(&ctx->timer, req0_ctx_timeout, ctx); - - nni_mtx_lock(&s->mtx); - ctx->sock = s; - ctx->recv_aio = NULL; - ctx->retry = s->retry; - nni_list_append(&s->contexts, ctx); - nni_mtx_unlock(&s->mtx); - - return (0); -} - -static void -req0_ctx_fini(void *arg) -{ - req0_ctx * ctx = arg; - req0_sock *s = ctx->sock; - nni_aio * aio; - - nni_mtx_lock(&s->mtx); - if ((aio = ctx->recv_aio) != NULL) { - ctx->recv_aio = NULL; - nni_aio_finish_error(aio, NNG_ECLOSED); - } - if ((aio = ctx->send_aio) != NULL) { - ctx->send_aio = NULL; - nni_aio_set_msg(aio, ctx->req_msg); - ctx->req_msg = NULL; - nni_aio_finish_error(aio, NNG_ECLOSED); - } - req0_ctx_reset(ctx); - nni_list_remove(&s->contexts, ctx); - nni_mtx_unlock(&s->mtx); - - nni_timer_cancel(&ctx->timer); - nni_timer_fini(&ctx->timer); -} - -static int -req0_ctx_set_resend_time(void *arg, const void *buf, size_t sz, nni_opt_type t) -{ - req0_ctx *ctx = arg; - return (nni_copyin_ms(&ctx->retry, buf, sz, t)); -} - -static int -req0_ctx_get_resend_time(void *arg, void *buf, size_t *szp, nni_opt_type t) -{ - req0_ctx *ctx = arg; - return (nni_copyout_ms(ctx->retry, buf, szp, t)); -} - -static void -req0_run_send_queue(req0_sock *s, nni_list *sent_list) -{ - req0_ctx *ctx; - nni_aio * aio; - - // Note: This routine should be called with the socket lock held. - while ((ctx = nni_list_first(&s->send_queue)) != NULL) { - req0_pipe *p; - - if ((p = nni_list_first(&s->ready_pipes)) == NULL) { - return; - } - - // We have a place to send it, so do the send. - // If a sending error occurs that causes the message to - // be dropped, we rely on the resend timer to pick it up. - // We also notify the completion callback if this is the - // first send attempt. - nni_list_remove(&s->send_queue, ctx); - - // Schedule a resubmit timer. We only do this if we got - // a pipe to send to. Otherwise, we should get handled - // the next time that the send_queue is run. We don't do this - // if the retry is "disabled" with NNG_DURATION_INFINITE. - if (ctx->retry > 0) { - nni_timer_schedule( - &ctx->timer, nni_clock() + ctx->retry); - } - - // Put us on the pipe list of active contexts. - // This gives the pipe a chance to kick a resubmit - // if the pipe is removed. - nni_list_node_remove(&ctx->pipe_node); - nni_list_append(&p->contexts, ctx); - - nni_list_remove(&s->ready_pipes, p); - nni_list_append(&s->busy_pipes, p); - if (nni_list_empty(&s->ready_pipes)) { - nni_pollable_clear(&s->writable); - } - - if ((aio = ctx->send_aio) != NULL) { - ctx->send_aio = NULL; - nni_aio_bump_count(aio, ctx->req_len); - // If the list was passed in, we want to do a - // synchronous completion later. - if (sent_list != NULL) { - nni_list_append(sent_list, aio); - } else { - nni_aio_finish(aio, 0, 0); - } - } - - // At this point, we will never give this message back to - // to the user, so we don't have to worry about making it - // unique. We can freely clone it. - nni_msg_clone(ctx->req_msg); - nni_aio_set_msg(&p->aio_send, ctx->req_msg); - nni_pipe_send(p->pipe, &p->aio_send); - } -} - -void -req0_ctx_reset(req0_ctx *ctx) -{ - req0_sock *s = ctx->sock; - // Call with sock lock held! - - // We cannot safely "wait" using nni_timer_cancel, but this removes - // any scheduled timer activation. If the timeout is already running - // concurrently, it will still run. It should do nothing, because - // we toss the request. There is still a very narrow race if the - // timeout fires, but doesn't actually start running before we - // both finish this function, *and* manage to reschedule another - // request. The consequence of that occurring is that the request - // will be emitted on the wire twice. This is not actually tragic. - nni_timer_schedule(&ctx->timer, NNI_TIME_NEVER); - - nni_list_node_remove(&ctx->pipe_node); - nni_list_node_remove(&ctx->send_node); - if (ctx->request_id != 0) { - nni_id_remove(&s->requests, ctx->request_id); - ctx->request_id = 0; - } - if (ctx->req_msg != NULL) { - nni_msg_free(ctx->req_msg); - ctx->req_msg = NULL; - } - if (ctx->rep_msg != NULL) { - nni_msg_free(ctx->rep_msg); - ctx->rep_msg = NULL; - } - ctx->conn_reset = false; -} - -static void -req0_ctx_cancel_recv(nni_aio *aio, void *arg, int rv) -{ - req0_ctx * ctx = arg; - req0_sock *s = ctx->sock; - - nni_mtx_lock(&s->mtx); - if (ctx->recv_aio == aio) { - ctx->recv_aio = NULL; - - // Cancellation of a pending receive is treated as aborting the - // entire state machine. This allows us to preserve the - // semantic of exactly one receive operation per send - // operation, and should be the least surprising for users. The - // main consequence is that if a receive operation is completed - // (in error or otherwise), the user must submit a new send - // operation to restart the state machine. - req0_ctx_reset(ctx); - - nni_aio_finish_error(aio, rv); - } - nni_mtx_unlock(&s->mtx); -} - -static void -req0_ctx_recv(void *arg, nni_aio *aio) -{ - req0_ctx * ctx = arg; - req0_sock *s = ctx->sock; - nni_msg * msg; - - if (nni_aio_begin(aio) != 0) { - return; - } - nni_mtx_lock(&s->mtx); - if ((ctx->recv_aio != NULL) || - ((ctx->req_msg == NULL) && (ctx->rep_msg == NULL))) { - // We have already got a pending receive or have not - // tried to send a request yet. - // Either of these violate our basic state assumptions. - int rv; - if (ctx->conn_reset) { - ctx->conn_reset = false; - rv = NNG_ECONNRESET; - } else { - rv = NNG_ESTATE; - } - nni_mtx_unlock(&s->mtx); - nni_aio_finish_error(aio, rv); - return; - } - - if ((msg = ctx->rep_msg) == NULL) { - int rv; - rv = nni_aio_schedule(aio, req0_ctx_cancel_recv, ctx); - if (rv != 0) { - nni_mtx_unlock(&s->mtx); - nni_aio_finish_error(aio, rv); - return; - } - ctx->recv_aio = aio; - nni_mtx_unlock(&s->mtx); - return; - } - - ctx->rep_msg = NULL; - - // We have got a message to pass up, yay! - nni_aio_set_msg(aio, msg); - if (ctx == &s->master) { - nni_pollable_clear(&s->readable); - } - nni_mtx_unlock(&s->mtx); - nni_aio_finish(aio, 0, nni_msg_len(msg)); -} - -static void -req0_ctx_cancel_send(nni_aio *aio, void *arg, int rv) -{ - req0_ctx * ctx = arg; - req0_sock *s = ctx->sock; - - nni_mtx_lock(&s->mtx); - if (ctx->send_aio == aio) { - // There should not be a pending reply, because we canceled - // it while we were waiting. - NNI_ASSERT(ctx->recv_aio == NULL); - ctx->send_aio = NULL; - // Restore the message back to the aio. - nni_aio_set_msg(aio, ctx->req_msg); - nni_msg_header_clear(ctx->req_msg); - ctx->req_msg = NULL; - - // Cancellation of a pending receive is treated as aborting the - // entire state machine. This allows us to preserve the - // semantic of exactly one receive operation per send - // operation, and should be the least surprising for users. The - // main consequence is that if a receive operation is completed - // (in error or otherwise), the user must submit a new send - // operation to restart the state machine. - req0_ctx_reset(ctx); - - nni_aio_finish_error(aio, rv); - } - nni_mtx_unlock(&s->mtx); -} - -static void -req0_ctx_send(void *arg, nni_aio *aio) -{ - req0_ctx * ctx = arg; - req0_sock *s = ctx->sock; - nng_msg * msg = nni_aio_get_msg(aio); - int rv; - - if (nni_aio_begin(aio) != 0) { - return; - } - nni_mtx_lock(&s->mtx); - if (s->closed) { - nni_mtx_unlock(&s->mtx); - nni_aio_finish_error(aio, NNG_ECLOSED); - return; - } - // Sending a new request cancels the old one, including any - // outstanding reply. - if (ctx->recv_aio != NULL) { - nni_aio_finish_error(ctx->recv_aio, NNG_ECANCELED); - ctx->recv_aio = NULL; - } - if (ctx->send_aio != NULL) { - nni_aio_set_msg(ctx->send_aio, ctx->req_msg); - nni_msg_header_clear(ctx->req_msg); - ctx->req_msg = NULL; - nni_aio_finish_error(ctx->send_aio, NNG_ECANCELED); - ctx->send_aio = NULL; - nni_list_remove(&s->send_queue, ctx); - } - - // This resets the entire state machine. - req0_ctx_reset(ctx); - - // Insert us on the per ID hash list, so that receives can find us. - if ((rv = nni_id_alloc(&s->requests, &ctx->request_id, ctx)) != 0) { - nni_mtx_unlock(&s->mtx); - nni_aio_finish_error(aio, rv); - return; - } - nni_msg_header_clear(msg); - nni_msg_header_append_u32(msg, ctx->request_id); - - // If no pipes are ready, and the request was a poll (no background - // schedule), then fail it. Should be NNG_ETIMEDOUT. - rv = nni_aio_schedule(aio, req0_ctx_cancel_send, ctx); - if ((rv != 0) && (nni_list_empty(&s->ready_pipes))) { - nni_id_remove(&s->requests, ctx->request_id); - nni_mtx_unlock(&s->mtx); - nni_aio_finish_error(aio, rv); - return; - } - ctx->req_len = nni_msg_len(msg); - ctx->req_msg = msg; - ctx->send_aio = aio; - nni_aio_set_msg(aio, NULL); - - // Stick us on the send_queue list. - nni_list_append(&s->send_queue, ctx); - - req0_run_send_queue(s, NULL); - nni_mtx_unlock(&s->mtx); -} - -static void -req0_sock_send(void *arg, nni_aio *aio) -{ - req0_sock *s = arg; - req0_ctx_send(&s->master, aio); -} - -static void -req0_sock_recv(void *arg, nni_aio *aio) -{ - req0_sock *s = arg; - req0_ctx_recv(&s->master, aio); -} - -static int -req0_sock_set_max_ttl(void *arg, const void *buf, size_t sz, nni_opt_type t) -{ - req0_sock *s = arg; - int ttl; - int rv; - if ((rv = nni_copyin_int(&ttl, buf, sz, 1, NNI_MAX_MAX_TTL, t)) == 0) { - nni_atomic_set(&s->ttl, ttl); - } - return (rv); -} - -static int -req0_sock_get_max_ttl(void *arg, void *buf, size_t *szp, nni_opt_type t) -{ - req0_sock *s = arg; - return (nni_copyout_int(nni_atomic_get(&s->ttl), buf, szp, t)); -} - -static int -req0_sock_set_resend_time( - void *arg, const void *buf, size_t sz, nni_opt_type t) -{ - req0_sock *s = arg; - int rv; - rv = req0_ctx_set_resend_time(&s->master, buf, sz, t); - s->retry = s->master.retry; - return (rv); -} - -static int -req0_sock_get_resend_time(void *arg, void *buf, size_t *szp, nni_opt_type t) -{ - req0_sock *s = arg; - return (req0_ctx_get_resend_time(&s->master, buf, szp, t)); -} - -static int -req0_sock_get_send_fd(void *arg, void *buf, size_t *szp, nni_opt_type t) -{ - req0_sock *s = arg; - int rv; - int fd; - - if ((rv = nni_pollable_getfd(&s->writable, &fd)) != 0) { - return (rv); - } - return (nni_copyout_int(fd, buf, szp, t)); -} - -static int -req0_sock_get_recv_fd(void *arg, void *buf, size_t *szp, nni_opt_type t) -{ - req0_sock *s = arg; - int rv; - int fd; - - if ((rv = nni_pollable_getfd(&s->readable, &fd)) != 0) { - return (rv); - } - - return (nni_copyout_int(fd, buf, szp, t)); -} - -static nni_proto_pipe_ops req0_pipe_ops = { - .pipe_size = sizeof(req0_pipe), - .pipe_init = req0_pipe_init, - .pipe_fini = req0_pipe_fini, - .pipe_start = req0_pipe_start, - .pipe_close = req0_pipe_close, - .pipe_stop = req0_pipe_stop, -}; - -static nni_option req0_ctx_options[] = { - { - .o_name = NNG_OPT_REQ_RESENDTIME, - .o_get = req0_ctx_get_resend_time, - .o_set = req0_ctx_set_resend_time, - }, - { - .o_name = NULL, - }, -}; - -static nni_proto_ctx_ops req0_ctx_ops = { - .ctx_size = sizeof(req0_ctx), - .ctx_init = req0_ctx_init, - .ctx_fini = req0_ctx_fini, - .ctx_recv = req0_ctx_recv, - .ctx_send = req0_ctx_send, - .ctx_options = req0_ctx_options, -}; - -static nni_option req0_sock_options[] = { - { - .o_name = NNG_OPT_MAXTTL, - .o_get = req0_sock_get_max_ttl, - .o_set = req0_sock_set_max_ttl, - }, - { - .o_name = NNG_OPT_REQ_RESENDTIME, - .o_get = req0_sock_get_resend_time, - .o_set = req0_sock_set_resend_time, - }, - { - .o_name = NNG_OPT_RECVFD, - .o_get = req0_sock_get_recv_fd, - }, - { - .o_name = NNG_OPT_SENDFD, - .o_get = req0_sock_get_send_fd, - }, - // terminate list - { - .o_name = NULL, - }, -}; - -static nni_proto_sock_ops req0_sock_ops = { - .sock_size = sizeof(req0_sock), - .sock_init = req0_sock_init, - .sock_fini = req0_sock_fini, - .sock_open = req0_sock_open, - .sock_close = req0_sock_close, - .sock_options = req0_sock_options, - .sock_send = req0_sock_send, - .sock_recv = req0_sock_recv, -}; - -static nni_proto req0_proto = { - .proto_version = NNI_PROTOCOL_VERSION, - .proto_self = { NNG_REQ0_SELF, NNG_REQ0_SELF_NAME }, - .proto_peer = { NNG_REQ0_PEER, NNG_REQ0_PEER_NAME }, - .proto_flags = NNI_PROTO_FLAG_SNDRCV, - .proto_sock_ops = &req0_sock_ops, - .proto_pipe_ops = &req0_pipe_ops, - .proto_ctx_ops = &req0_ctx_ops, -}; - -int -nng_req0_open(nng_socket *sock) -{ - return (nni_proto_open(sock, &req0_proto)); -} diff --git a/src/protocol/reqrep0/req_test.c b/src/protocol/reqrep0/req_test.c deleted file mode 100644 index fb78efa0..00000000 --- a/src/protocol/reqrep0/req_test.c +++ /dev/null @@ -1,968 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// Copyright 2018 Capitar IT Group BV -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - -#include - -static void -test_req_identity(void) -{ - nng_socket s; - int p; - char * n; - - NUTS_PASS(nng_req0_open(&s)); - NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PROTO, &p)); - NUTS_TRUE(p == NNG_REQ0_SELF); - NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PEER, &p)); - NUTS_TRUE(p == NNG_REQ0_PEER); // 49 - NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PROTONAME, &n)); - NUTS_MATCH(n, NNG_REQ0_SELF_NAME); - nng_strfree(n); - NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PEERNAME, &n)); - NUTS_MATCH(n, NNG_REQ0_PEER_NAME); - nng_strfree(n); - NUTS_CLOSE(s); -} - -static void -test_req_ttl_option(void) -{ - nng_socket req; - int v; - bool b; - size_t sz; - const char *opt = NNG_OPT_MAXTTL; - - NUTS_PASS(nng_req0_open(&req)); - - NUTS_PASS(nng_socket_set_int(req, opt, 1)); - NUTS_FAIL(nng_socket_set_int(req, opt, 0), NNG_EINVAL); - NUTS_FAIL(nng_socket_set_int(req, opt, -1), NNG_EINVAL); - // This test will fail if the NNI_MAX_MAX_TTL is changed from the - // builtin default of 15. - NUTS_FAIL(nng_socket_set_int(req, opt, 16), NNG_EINVAL); - NUTS_FAIL(nng_socket_set_int(req, opt, 256), NNG_EINVAL); - NUTS_PASS(nng_socket_set_int(req, opt, 3)); - NUTS_PASS(nng_socket_get_int(req, opt, &v)); - NUTS_TRUE(v == 3); - v = 0; - sz = sizeof(v); - NUTS_PASS(nng_socket_get(req, opt, &v, &sz)); - NUTS_TRUE(v == 3); - NUTS_TRUE(sz == sizeof(v)); - - NUTS_FAIL(nng_socket_set(req, opt, "", 1), NNG_EINVAL); - sz = 1; - NUTS_FAIL(nng_socket_get(req, opt, &v, &sz), NNG_EINVAL); - NUTS_FAIL(nng_socket_set_bool(req, opt, true), NNG_EBADTYPE); - NUTS_FAIL(nng_socket_get_bool(req, opt, &b), NNG_EBADTYPE); - - NUTS_CLOSE(req); -} - -static void -test_req_resend_option(void) -{ - nng_socket req; - nng_duration d; - bool b; - size_t sz = sizeof(b); - const char * opt = NNG_OPT_REQ_RESENDTIME; - - NUTS_PASS(nng_req0_open(&req)); - - NUTS_TRUE(nng_socket_set_ms(req, opt, 10) == 0); - NUTS_FAIL(nng_socket_set(req, opt, "", 1), NNG_EINVAL); - NUTS_FAIL(nng_socket_get(req, opt, &b, &sz), NNG_EINVAL); - NUTS_FAIL(nng_socket_set_bool(req, opt, true), NNG_EBADTYPE); - NUTS_FAIL(nng_socket_get_bool(req, opt, &b), NNG_EBADTYPE); - - NUTS_PASS(nng_socket_get_ms(req, opt, &d)); - NUTS_TRUE(d == 10); - NUTS_CLOSE(req); -} - -void -test_req_recv_bad_state(void) -{ - nng_socket req; - nng_msg * msg = NULL; - - NUTS_TRUE(nng_req0_open(&req) == 0); - NUTS_TRUE(nng_recvmsg(req, &msg, 0) == NNG_ESTATE); - NUTS_NULL(msg); - NUTS_CLOSE(req); -} - -static void -test_req_recv_garbage(void) -{ - nng_socket rep; - nng_socket req; - nng_msg * m; - uint32_t req_id; - - NUTS_PASS(nng_rep0_open_raw(&rep)); - NUTS_PASS(nng_req0_open(&req)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_RECVTIMEO, 100)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 1000)); - - NUTS_MARRY(req, rep); - - NUTS_PASS(nng_msg_alloc(&m, 0)); - NUTS_PASS(nng_sendmsg(req, m, 0)); - - NUTS_PASS(nng_recvmsg(rep, &m, 0)); - - // The message will have a header that contains the 32-bit pipe ID, - // followed by the 32-bit request ID. We will discard the request - // ID before sending it out. - NUTS_TRUE(nng_msg_header_len(m) == 8); - NUTS_PASS(nng_msg_header_chop_u32(m, &req_id)); - - NUTS_PASS(nng_sendmsg(rep, m, 0)); - NUTS_FAIL(nng_recvmsg(req, &m, 0), NNG_ETIMEDOUT); - - NUTS_CLOSE(req); - NUTS_CLOSE(rep); -} - -#define SECOND 1000 - -void -test_req_rep_exchange(void) -{ - nng_socket req; - nng_socket rep; - - NUTS_TRUE(nng_req0_open(&req) == 0); - NUTS_TRUE(nng_rep0_open(&rep) == 0); - - NUTS_TRUE(nng_socket_set_ms(req, NNG_OPT_RECVTIMEO, SECOND) == 0); - NUTS_TRUE(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, SECOND) == 0); - NUTS_TRUE(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, SECOND) == 0); - NUTS_TRUE(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, SECOND) == 0); - - NUTS_MARRY(rep, req); - - NUTS_SEND(req, "ping"); - NUTS_RECV(rep, "ping"); - NUTS_SEND(rep, "pong"); - NUTS_RECV(req, "pong"); - - NUTS_CLOSE(req); - NUTS_CLOSE(rep); -} - -void -test_req_resend(void) -{ - nng_socket req; - nng_socket rep; - - NUTS_PASS(nng_req0_open(&req)); - NUTS_PASS(nng_rep0_open(&rep)); - - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_RECVTIMEO, SECOND)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, SECOND)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, SECOND)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, SECOND)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_REQ_RESENDTIME, 10)); - - NUTS_MARRY(rep, req); - - NUTS_SEND(req, "ping"); - NUTS_RECV(rep, "ping"); - NUTS_RECV(rep, "ping"); - NUTS_RECV(rep, "ping"); - - NUTS_CLOSE(req); - NUTS_CLOSE(rep); -} - -void -test_req_resend_reconnect(void) -{ - nng_socket req; - nng_socket rep1; - nng_socket rep2; - - NUTS_PASS(nng_req0_open(&req)); - NUTS_PASS(nng_rep0_open(&rep1)); - NUTS_PASS(nng_rep0_open(&rep2)); - - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_RECVTIMEO, SECOND)); - NUTS_PASS(nng_socket_set_ms(rep1, NNG_OPT_RECVTIMEO, SECOND)); - NUTS_PASS(nng_socket_set_ms(rep2, NNG_OPT_RECVTIMEO, SECOND)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, SECOND)); - NUTS_PASS(nng_socket_set_ms(rep1, NNG_OPT_SENDTIMEO, SECOND)); - NUTS_PASS(nng_socket_set_ms(rep2, NNG_OPT_SENDTIMEO, SECOND)); - // We intentionally set the retry time long; that way we only see - // the retry from loss of our original peer. - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_REQ_RESENDTIME, 60 * SECOND)); - - NUTS_MARRY(rep1, req); - - NUTS_SEND(req, "ping"); - NUTS_RECV(rep1, "ping"); - - NUTS_CLOSE(rep1); - NUTS_MARRY(rep2, req); - - NUTS_RECV(rep2, "ping"); - NUTS_SEND(rep2, "rep2"); - NUTS_RECV(req, "rep2"); - - NUTS_CLOSE(req); - NUTS_CLOSE(rep2); -} - -void -test_req_resend_disconnect(void) -{ - nng_socket req; - nng_socket rep1; - nng_socket rep2; - - NUTS_PASS(nng_req0_open(&req)); - NUTS_PASS(nng_rep0_open(&rep1)); - NUTS_PASS(nng_rep0_open(&rep2)); - - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_RECVTIMEO, SECOND)); - NUTS_PASS(nng_socket_set_ms(rep1, NNG_OPT_RECVTIMEO, SECOND)); - NUTS_PASS(nng_socket_set_ms(rep2, NNG_OPT_RECVTIMEO, SECOND)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, SECOND)); - NUTS_PASS(nng_socket_set_ms(rep1, NNG_OPT_SENDTIMEO, SECOND)); - NUTS_PASS(nng_socket_set_ms(rep2, NNG_OPT_SENDTIMEO, SECOND)); - // We intentionally set the retry time long; that way we only see - // the retry from loss of our original peer. - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_REQ_RESENDTIME, 60 * SECOND)); - - NUTS_MARRY(rep1, req); - NUTS_SEND(req, "ping"); - NUTS_RECV(rep1, "ping"); - - NUTS_MARRY(rep2, req); - NUTS_CLOSE(rep1); - - NUTS_RECV(rep2, "ping"); - NUTS_SEND(rep2, "rep2"); - NUTS_RECV(req, "rep2"); - - NUTS_CLOSE(req); - NUTS_CLOSE(rep2); -} - -void -test_req_disconnect_no_retry(void) -{ - nng_socket req; - nng_socket rep1; - nng_socket rep2; - - NUTS_PASS(nng_req0_open(&req)); - NUTS_PASS(nng_rep0_open(&rep1)); - NUTS_PASS(nng_rep0_open(&rep2)); - - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_RECVTIMEO, SECOND)); - NUTS_PASS(nng_socket_set_ms(rep1, NNG_OPT_RECVTIMEO, SECOND)); - NUTS_PASS(nng_socket_set_ms(rep2, NNG_OPT_RECVTIMEO, SECOND)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, SECOND)); - NUTS_PASS(nng_socket_set_ms(rep1, NNG_OPT_SENDTIMEO, SECOND / 10)); - // Setting the resend time to zero so we will force an error - // if the peer disconnects without sending us an answer. - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_REQ_RESENDTIME, 0)); - - NUTS_MARRY(rep1, req); - NUTS_SEND(req, "ping"); - NUTS_RECV(rep1, "ping"); - - NUTS_MARRY(rep2, req); - NUTS_CLOSE(rep1); - - nng_msg *msg = NULL; - NUTS_FAIL(nng_recvmsg(req, &msg, 0), NNG_ECONNRESET); - NUTS_FAIL(nng_recvmsg(rep2, &msg, 0), NNG_ETIMEDOUT); - - NUTS_CLOSE(req); - NUTS_CLOSE(rep2); -} - -void -test_req_disconnect_abort(void) -{ - nng_socket req; - nng_socket rep1; - nng_socket rep2; - nng_aio * aio; - - NUTS_PASS(nng_req0_open(&req)); - NUTS_PASS(nng_rep0_open(&rep1)); - NUTS_PASS(nng_rep0_open(&rep2)); - NUTS_PASS(nng_aio_alloc(&aio, 0, 0)); - - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_RECVTIMEO, SECOND)); - NUTS_PASS(nng_socket_set_ms(rep1, NNG_OPT_RECVTIMEO, SECOND)); - NUTS_PASS(nng_socket_set_ms(rep2, NNG_OPT_RECVTIMEO, SECOND)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, SECOND)); - NUTS_PASS(nng_socket_set_ms(rep1, NNG_OPT_SENDTIMEO, SECOND / 10)); - // Setting the resend time to zero so we will force an error - // if the peer disconnects without sending us an answer. - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_REQ_RESENDTIME, 0)); - - NUTS_MARRY(rep1, req); - NUTS_SEND(req, "ping"); - NUTS_RECV(rep1, "ping"); - nng_recv_aio(req, aio); - - NUTS_MARRY(rep2, req); - NUTS_CLOSE(rep1); - - nng_msg *msg = NULL; - nng_aio_wait(aio); - NUTS_FAIL(nng_aio_result(aio), NNG_ECONNRESET); - NUTS_FAIL(nng_recvmsg(rep2, &msg, 0), NNG_ETIMEDOUT); - nng_aio_free(aio); - - NUTS_CLOSE(req); - NUTS_CLOSE(rep2); -} - -void -test_req_cancel(void) -{ - nng_duration retry = SECOND; - nng_socket req; - nng_socket rep; - - NUTS_PASS(nng_rep_open(&rep)); - NUTS_PASS(nng_req_open(&req)); - - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_RECVTIMEO, SECOND)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, SECOND)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 5 * SECOND)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 5 * SECOND)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_REQ_RESENDTIME, retry)); - NUTS_PASS(nng_socket_set_int(req, NNG_OPT_SENDBUF, 16)); - - NUTS_MARRY(rep, req); - - // Send req #1 (abc). - NUTS_SEND(req, "abc"); - - // Sleep a bit. This is so that we ensure that our request gets - // to the far side. (If we cancel too fast, then our outgoing send - // will be canceled before it gets to the peer.) - NUTS_SLEEP(100); - - // Send the next next request ("def"). Note that - // the REP side server will have already buffered the receive - // request, and should simply be waiting for us to reply to abc. - NUTS_SEND(req, "def"); - - // Receive the first request (should be abc) on the REP server. - NUTS_RECV(rep, "abc"); - - // REP sends the reply to first command. This will be discarded - // by the REQ socket. - NUTS_SEND(rep, "abc"); - - // Now get the next command from the REP; should be "def". - NUTS_RECV(rep, "def"); - - // And send it back to REQ. - NUTS_SEND(rep, "def"); - - // And we got back only the second result. - NUTS_RECV(req, "def"); - - NUTS_CLOSE(req); - NUTS_CLOSE(rep); -} - -void -test_req_cancel_abort_recv(void) -{ - nng_aio * aio; - nng_duration retry = SECOND * 10; // 10s (kind of never) - nng_socket req; - nng_socket rep; - - NUTS_PASS(nng_rep_open(&rep)); - NUTS_PASS(nng_req_open(&req)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_REQ_RESENDTIME, retry)); - NUTS_PASS(nng_socket_set_int(req, NNG_OPT_SENDBUF, 16)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_RECVTIMEO, 5 * SECOND)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 5 * SECOND)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 5 * SECOND)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 5 * SECOND)); - - NUTS_MARRY(rep, req); - - // Send req #1 (abc). - NUTS_SEND(req, "abc"); - - // Wait for it to get ot the other side. - NUTS_SLEEP(100); - - nng_aio_set_timeout(aio, 5 * SECOND); - nng_recv_aio(req, aio); - - // Give time for this recv to post properly. - NUTS_SLEEP(100); - - // Send the next next request ("def"). Note that - // the REP side server will have already buffered the receive - // request, and should simply be waiting for us to reply to - // abc. - NUTS_SEND(req, "def"); - - // Our pending I/O should have been canceled. - nng_aio_wait(aio); - NUTS_FAIL(nng_aio_result(aio), NNG_ECANCELED); - - // Receive the first request (should be abc) on the REP server. - NUTS_RECV(rep, "abc"); - - // REP sends the reply to first command. This will be - // discarded by the REQ socket. - NUTS_SEND(rep, "abc"); - - // Now get the next command from the REP; should be "def". - NUTS_RECV(rep, "def"); - - // And send it back to REQ. - NUTS_SEND(rep, "def"); - - // Try a req command. This should give back "def" - NUTS_RECV(req, "def"); - - nng_aio_free(aio); - NUTS_CLOSE(req); - NUTS_CLOSE(rep); -} - -static void -test_req_cancel_post_recv(void) -{ - nng_socket req; - nng_socket rep; - - NUTS_PASS(nng_req0_open(&req)); - NUTS_PASS(nng_rep0_open(&rep)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 1000)); - NUTS_MARRY(req, rep); - - NUTS_SEND(req, "ONE"); - NUTS_RECV(rep, "ONE"); - NUTS_SEND(rep, "one"); - NUTS_SLEEP(100); // Make sure reply arrives! - NUTS_SEND(req, "TWO"); - NUTS_RECV(rep, "TWO"); - NUTS_SEND(rep, "two"); - NUTS_RECV(req, "two"); - - NUTS_CLOSE(req); - NUTS_CLOSE(rep); -} - -void -test_req_poll_writeable(void) -{ - int fd; - nng_socket req; - nng_socket rep; - - NUTS_PASS(nng_req0_open(&req)); - NUTS_PASS(nng_rep0_open(&rep)); - NUTS_PASS(nng_socket_get_int(req, NNG_OPT_SENDFD, &fd)); - NUTS_TRUE(fd >= 0); - - // Not writable before connect. - NUTS_TRUE(nuts_poll_fd(fd) == false); - - NUTS_MARRY(req, rep); - - // It should be writable now. - NUTS_TRUE(nuts_poll_fd(fd)); - - // Submit a bunch of jobs. Note that we have to stall a bit - // between each message to let it queue up. - for (int i = 0; i < 10; i++) { - int rv = nng_send(req, "", 0, NNG_FLAG_NONBLOCK); - if (rv == NNG_EAGAIN) { - break; - } - NUTS_PASS(rv); - NUTS_SLEEP(50); - } - NUTS_TRUE(nuts_poll_fd(fd) == false); - NUTS_CLOSE(req); - NUTS_CLOSE(rep); -} - -void -test_req_poll_contention(void) -{ - int fd; - nng_socket req; - nng_socket rep; - nng_aio * aio; - nng_ctx ctx[5]; - nng_aio * ctx_aio[5]; - nng_msg * ctx_msg[5]; - nng_msg * msg; - - NUTS_PASS(nng_req0_open(&req)); - NUTS_PASS(nng_rep0_open(&rep)); - NUTS_PASS(nng_socket_set_int(req, NNG_OPT_SENDBUF, 1)); - NUTS_PASS(nng_socket_set_int(rep, NNG_OPT_RECVBUF, 1)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 1000)); - - for (int i = 0; i < 5; i++) { - NUTS_PASS(nng_ctx_open(&ctx[i], req)); - NUTS_PASS(nng_aio_alloc(&ctx_aio[i], NULL, NULL)); - NUTS_PASS(nng_msg_alloc(&ctx_msg[i], 0)); - } - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - NUTS_PASS(nng_msg_alloc(&msg, 0)); - - NUTS_PASS(nng_socket_get_int(req, NNG_OPT_SENDFD, &fd)); - NUTS_TRUE(fd >= 0); - - // Not writable before connect. - NUTS_TRUE(nuts_poll_fd(fd) == false); - - nng_aio_set_msg(aio, msg); - nng_send_aio(req, aio); - for (int i = 0; i < 5; i++) { - nng_aio_set_msg(ctx_aio[i], ctx_msg[i]); - nng_ctx_send(ctx[i], ctx_aio[i]); - } - NUTS_SLEEP(50); // so everything is queued steady state - - NUTS_MARRY(req, rep); - - // It should not be writable now. - NUTS_TRUE(nuts_poll_fd(fd) == false); - - NUTS_PASS(nng_recvmsg(rep, &msg, 0)); - nng_msg_free(msg); - - // Still not writeable... - NUTS_TRUE(nuts_poll_fd(fd) == false); - for (int i = 0; i < 5; i++) { - NUTS_PASS(nng_recvmsg(rep, &msg, 0)); - nng_msg_free(msg); - } - // It can take a little bit of time for the eased back-pressure - // to reflect across the network. - NUTS_SLEEP(100); - - // Should be come writeable now... - NUTS_TRUE(nuts_poll_fd(fd) == true); - - for (int i = 0; i < 5; i++) { - nng_aio_free(ctx_aio[i]); - } - nng_aio_free(aio); - NUTS_CLOSE(req); - NUTS_CLOSE(rep); -} - -void -test_req_poll_multi_pipe(void) -{ - int fd; - nng_socket req; - nng_socket rep1; - nng_socket rep2; - - NUTS_PASS(nng_req0_open(&req)); - NUTS_PASS(nng_rep0_open(&rep1)); - NUTS_PASS(nng_rep0_open(&rep2)); - NUTS_PASS(nng_socket_set_int(req, NNG_OPT_SENDBUF, 1)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); - - NUTS_PASS(nng_socket_get_int(req, NNG_OPT_SENDFD, &fd)); - NUTS_TRUE(fd >= 0); - - // Not writable before connect. - NUTS_TRUE(nuts_poll_fd(fd) == false); - - NUTS_MARRY(req, rep1); - NUTS_MARRY(req, rep2); - - NUTS_TRUE(nuts_poll_fd(fd) == true); - NUTS_SEND(req, "ONE"); - NUTS_TRUE(nuts_poll_fd(fd) == true); - - NUTS_CLOSE(req); - NUTS_CLOSE(rep1); - NUTS_CLOSE(rep2); -} - -void -test_req_poll_readable(void) -{ - int fd; - nng_socket req; - nng_socket rep; - nng_msg * msg; - - NUTS_PASS(nng_req0_open(&req)); - NUTS_PASS(nng_rep0_open(&rep)); - NUTS_PASS(nng_socket_get_int(req, NNG_OPT_RECVFD, &fd)); - NUTS_TRUE(fd >= 0); - - // Not readable if not connected! - NUTS_TRUE(nuts_poll_fd(fd) == false); - - // Even after connect (no message yet) - NUTS_MARRY(req, rep); - NUTS_TRUE(nuts_poll_fd(fd) == false); - - // But once we send messages, it is. - // We have to send a request, in order to send a reply. - - NUTS_PASS(nng_msg_alloc(&msg, 0)); - NUTS_PASS(nng_msg_append(msg, "xyz", 3)); - NUTS_PASS(nng_sendmsg(req, msg, 0)); - NUTS_PASS(nng_recvmsg(rep, &msg, 0)); // recv on rep - NUTS_PASS(nng_sendmsg(rep, msg, 0)); // echo it back - NUTS_SLEEP(200); // give time for message to arrive - - NUTS_TRUE(nuts_poll_fd(fd) == true); - - // and receiving makes it no longer ready - NUTS_PASS(nng_recvmsg(req, &msg, 0)); - nng_msg_free(msg); - NUTS_TRUE(nuts_poll_fd(fd) == false); - - // TODO verify unsolicited response - - NUTS_CLOSE(req); - NUTS_CLOSE(rep); -} - -static void -test_req_ctx_no_poll(void) -{ - int fd; - nng_socket req; - nng_ctx ctx; - - NUTS_PASS(nng_req0_open(&req)); - NUTS_PASS(nng_ctx_open(&ctx, req)); - NUTS_FAIL(nng_ctx_getopt_int(ctx, NNG_OPT_SENDFD, &fd), NNG_ENOTSUP); - NUTS_FAIL(nng_ctx_getopt_int(ctx, NNG_OPT_RECVFD, &fd), NNG_ENOTSUP); - NUTS_PASS(nng_ctx_close(ctx)); - NUTS_CLOSE(req); -} - -static void -test_req_ctx_send_queued(void) -{ - nng_socket req; - nng_socket rep; - nng_ctx ctx[3]; - nng_aio * aio[3]; - nng_msg * msg[3]; - - NUTS_PASS(nng_req0_open(&req)); - NUTS_PASS(nng_rep0_open(&rep)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 100)); - - for (int i = 0; i < 3; i++) { - NUTS_PASS(nng_ctx_open(&ctx[i], req)); - NUTS_PASS(nng_aio_alloc(&aio[i], NULL, NULL)); - NUTS_PASS(nng_msg_alloc(&msg[i], 0)); - } - - for (int i = 0; i < 3; i++) { - nng_aio_set_msg(aio[i], msg[i]); - nng_ctx_send(ctx[i], aio[i]); - } - - NUTS_MARRY(req, rep); - - NUTS_SLEEP(50); // Only to ensure stuff queues up - for (int i = 0; i < 3; i++) { - nng_msg *m; - NUTS_PASS(nng_recvmsg(rep, &m, 0)); - nng_msg_free(m); - } - - NUTS_CLOSE(req); - NUTS_CLOSE(rep); - for (int i = 0; i < 3; i++) { - nng_aio_wait(aio[i]); - NUTS_PASS(nng_aio_result(aio[i])); - nng_aio_free(aio[i]); - } -} - -static void -test_req_ctx_send_close(void) -{ - nng_socket req; - nng_ctx ctx[3]; - nng_aio * aio[3]; - nng_msg * msg[3]; - - NUTS_PASS(nng_req0_open(&req)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); - - for (int i = 0; i < 3; i++) { - NUTS_PASS(nng_ctx_open(&ctx[i], req)); - NUTS_PASS(nng_aio_alloc(&aio[i], NULL, NULL)); - NUTS_PASS(nng_msg_alloc(&msg[i], 0)); - } - - for (int i = 0; i < 3; i++) { - nng_aio_set_msg(aio[i], msg[i]); - nng_ctx_send(ctx[i], aio[i]); - } - - for (int i = 0; i < 3; i++) { - nng_ctx_close(ctx[i]); - } - - for (int i = 0; i < 3; i++) { - nng_aio_wait(aio[i]); - NUTS_FAIL(nng_aio_result(aio[i]), NNG_ECLOSED); - nng_aio_free(aio[i]); - nng_msg_free(msg[i]); - } - NUTS_CLOSE(req); -} - -static void -test_req_ctx_send_abort(void) -{ - nng_socket req; - nng_ctx ctx[3]; - nng_aio * aio[3]; - nng_msg * msg[3]; - - NUTS_PASS(nng_req0_open(&req)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); - - for (int i = 0; i < 3; i++) { - NUTS_PASS(nng_ctx_open(&ctx[i], req)); - NUTS_PASS(nng_aio_alloc(&aio[i], NULL, NULL)); - NUTS_PASS(nng_msg_alloc(&msg[i], 0)); - } - - for (int i = 0; i < 3; i++) { - nng_aio_set_msg(aio[i], msg[i]); - nng_ctx_send(ctx[i], aio[i]); - } - - for (int i = 0; i < 3; i++) { - nng_aio_abort(aio[i], NNG_ECANCELED); - } - - for (int i = 0; i < 3; i++) { - nng_aio_wait(aio[i]); - NUTS_FAIL(nng_aio_result(aio[i]), NNG_ECANCELED); - nng_aio_free(aio[i]); - nng_msg_free(msg[i]); - } - NUTS_CLOSE(req); -} - -static void -test_req_ctx_send_twice(void) -{ - nng_socket req; - nng_ctx ctx; - nng_aio * aio[2]; - nng_msg * msg[2]; - - NUTS_PASS(nng_req0_open(&req)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_ctx_open(&ctx, req)); - - for (int i = 0; i < 2; i++) { - NUTS_PASS(nng_aio_alloc(&aio[i], NULL, NULL)); - NUTS_PASS(nng_msg_alloc(&msg[i], 0)); - } - - for (int i = 0; i < 2; i++) { - nng_aio_set_msg(aio[i], msg[i]); - nng_ctx_send(ctx, aio[i]); - NUTS_SLEEP(50); - } - - NUTS_CLOSE(req); - nng_aio_wait(aio[0]); - nng_aio_wait(aio[1]); - NUTS_FAIL(nng_aio_result(aio[0]), NNG_ECANCELED); - NUTS_FAIL(nng_aio_result(aio[1]), NNG_ECLOSED); - - for (int i = 0; i < 2; i++) { - nng_aio_free(aio[i]); - nng_msg_free(msg[i]); - } -} - -static void -test_req_ctx_recv_nonblock(void) -{ - nng_socket req; - nng_socket rep; - nng_ctx ctx; - nng_aio * aio; - nng_msg * msg; - - NUTS_PASS(nng_req0_open(&req)); - NUTS_PASS(nng_rep0_open(&rep)); - NUTS_PASS(nng_ctx_open(&ctx, req)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - NUTS_PASS(nng_msg_alloc(&msg, 0)); - - NUTS_MARRY(req, rep); - - nng_aio_set_msg(aio, msg); - nng_ctx_send(ctx, aio); - nng_aio_wait(aio); - NUTS_PASS(nng_aio_result(aio)); - nng_aio_set_timeout(aio, 0); // Instant timeout - nng_ctx_recv(ctx, aio); - - nng_aio_wait(aio); - NUTS_FAIL(nng_aio_result(aio), NNG_ETIMEDOUT); - NUTS_CLOSE(req); - NUTS_CLOSE(rep); - nng_aio_free(aio); -} - -static void -test_req_ctx_send_nonblock(void) -{ - nng_socket req; - nng_ctx ctx; - nng_aio * aio; - nng_msg * msg; - - NUTS_PASS(nng_req0_open(&req)); - NUTS_PASS(nng_ctx_open(&ctx, req)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - NUTS_PASS(nng_msg_alloc(&msg, 0)); - - nng_aio_set_msg(aio, msg); - nng_aio_set_timeout(aio, 0); // Instant timeout - nng_ctx_send(ctx, aio); - nng_aio_wait(aio); - NUTS_FAIL(nng_aio_result(aio), NNG_ETIMEDOUT); - NUTS_CLOSE(req); - nng_aio_free(aio); - nng_msg_free(msg); -} - -static void -test_req_ctx_recv_close_socket(void) -{ - nng_socket req; - nng_socket rep; - nng_ctx ctx; - nng_aio * aio; - nng_msg * m; - - NUTS_PASS(nng_req0_open(&req)); - NUTS_PASS(nng_rep0_open(&rep)); - NUTS_PASS(nng_ctx_open(&ctx, req)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - NUTS_MARRY(req, rep); - NUTS_PASS(nng_msg_alloc(&m, 0)); - nng_aio_set_msg(aio, m); - nng_ctx_send(ctx, aio); - nng_aio_wait(aio); - NUTS_PASS(nng_aio_result(aio)); - - nng_ctx_recv(ctx, aio); - nng_close(req); - - NUTS_FAIL(nng_aio_result(aio), NNG_ECLOSED); - nng_aio_free(aio); - NUTS_CLOSE(rep); -} - -static void -test_req_validate_peer(void) -{ - nng_socket s1, s2; - nng_stat * stats; - nng_stat * reject; - char * addr; - - NUTS_ADDR(addr, "inproc"); - - NUTS_PASS(nng_req0_open(&s1)); - NUTS_PASS(nng_req0_open(&s2)); - - NUTS_PASS(nng_listen(s1, addr, NULL, 0)); - NUTS_PASS(nng_dial(s2, addr, NULL, NNG_FLAG_NONBLOCK)); - - NUTS_SLEEP(100); - NUTS_PASS(nng_stats_get(&stats)); - - NUTS_TRUE(stats != NULL); - NUTS_TRUE((reject = nng_stat_find_socket(stats, s1)) != NULL); - NUTS_TRUE((reject = nng_stat_find(reject, "reject")) != NULL); - - NUTS_TRUE(nng_stat_type(reject) == NNG_STAT_COUNTER); - NUTS_TRUE(nng_stat_value(reject) > 0); - - NUTS_CLOSE(s1); - NUTS_CLOSE(s2); - nng_stats_free(stats); -} - -NUTS_TESTS = { - { "req identity", test_req_identity }, - { "req ttl option", test_req_ttl_option }, - { "req resend option", test_req_resend_option }, - { "req recv bad state", test_req_recv_bad_state }, - { "req recv garbage", test_req_recv_garbage }, - { "req rep exchange", test_req_rep_exchange }, - { "req resend", test_req_resend }, - { "req resend disconnect", test_req_resend_disconnect }, - { "req disconnect no retry", test_req_disconnect_no_retry }, - { "req disconnect abort", test_req_disconnect_abort }, - { "req resend reconnect", test_req_resend_reconnect }, - { "req cancel", test_req_cancel }, - { "req cancel abort recv", test_req_cancel_abort_recv }, - { "req cancel post recv", test_req_cancel_post_recv }, - { "req poll writable", test_req_poll_writeable }, - { "req poll contention", test_req_poll_contention }, - { "req poll multi pipe", test_req_poll_multi_pipe }, - { "req poll readable", test_req_poll_readable }, - { "req context send queued", test_req_ctx_send_queued }, - { "req context send close", test_req_ctx_send_close }, - { "req context send abort", test_req_ctx_send_abort }, - { "req context send twice", test_req_ctx_send_twice }, - { "req context does not poll", test_req_ctx_no_poll }, - { "req context recv close socket", test_req_ctx_recv_close_socket }, - { "req context recv nonblock", test_req_ctx_recv_nonblock }, - { "req context send nonblock", test_req_ctx_send_nonblock }, - { "req validate peer", test_req_validate_peer }, - { NULL, NULL }, -}; diff --git a/src/protocol/reqrep0/xrep.c b/src/protocol/reqrep0/xrep.c deleted file mode 100644 index 9737c600..00000000 --- a/src/protocol/reqrep0/xrep.c +++ /dev/null @@ -1,432 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// Copyright 2018 Capitar IT Group BV -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - -#include - -#include "core/nng_impl.h" -#include "nng/protocol/reqrep0/rep.h" - -// Response protocol in raw mode. The REP protocol is the "reply" side of a -// request-reply pair. This is useful for building RPC servers, for -// example. - -typedef struct xrep0_pipe xrep0_pipe; -typedef struct xrep0_sock xrep0_sock; - -static void xrep0_sock_getq_cb(void *); -static void xrep0_pipe_getq_cb(void *); -static void xrep0_pipe_putq_cb(void *); -static void xrep0_pipe_send_cb(void *); -static void xrep0_pipe_recv_cb(void *); -static void xrep0_pipe_fini(void *); - -// xrep0_sock is our per-socket protocol private structure. -struct xrep0_sock { - nni_msgq * uwq; - nni_msgq * urq; - nni_mtx lk; - nni_atomic_int ttl; - nni_id_map pipes; - nni_aio aio_getq; -}; - -// xrep0_pipe is our per-pipe protocol private structure. -struct xrep0_pipe { - nni_pipe * pipe; - xrep0_sock *rep; - nni_msgq * sendq; - nni_aio aio_getq; - nni_aio aio_send; - nni_aio aio_recv; - nni_aio aio_putq; -}; - -static void -xrep0_sock_fini(void *arg) -{ - xrep0_sock *s = arg; - - nni_aio_fini(&s->aio_getq); - nni_id_map_fini(&s->pipes); - nni_mtx_fini(&s->lk); -} - -static int -xrep0_sock_init(void *arg, nni_sock *sock) -{ - xrep0_sock *s = arg; - - nni_mtx_init(&s->lk); - nni_aio_init(&s->aio_getq, xrep0_sock_getq_cb, s); - nni_atomic_init(&s->ttl); - nni_atomic_set(&s->ttl, 8); // Per RFC - s->uwq = nni_sock_sendq(sock); - s->urq = nni_sock_recvq(sock); - - nni_id_map_init(&s->pipes, 0, 0, false); - return (0); -} - -static void -xrep0_sock_open(void *arg) -{ - xrep0_sock *s = arg; - - // This starts us retrieving message from the upper write q. - nni_msgq_aio_get(s->uwq, &s->aio_getq); -} - -static void -xrep0_sock_close(void *arg) -{ - xrep0_sock *s = arg; - - nni_aio_close(&s->aio_getq); -} - -static void -xrep0_pipe_stop(void *arg) -{ - xrep0_pipe *p = arg; - - nni_aio_stop(&p->aio_getq); - nni_aio_stop(&p->aio_send); - nni_aio_stop(&p->aio_recv); - nni_aio_stop(&p->aio_putq); -} - -static void -xrep0_pipe_fini(void *arg) -{ - xrep0_pipe *p = arg; - - nni_aio_fini(&p->aio_getq); - nni_aio_fini(&p->aio_send); - nni_aio_fini(&p->aio_recv); - nni_aio_fini(&p->aio_putq); - nni_msgq_fini(p->sendq); -} - -static int -xrep0_pipe_init(void *arg, nni_pipe *pipe, void *s) -{ - xrep0_pipe *p = arg; - int rv; - - nni_aio_init(&p->aio_getq, xrep0_pipe_getq_cb, p); - nni_aio_init(&p->aio_send, xrep0_pipe_send_cb, p); - nni_aio_init(&p->aio_recv, xrep0_pipe_recv_cb, p); - nni_aio_init(&p->aio_putq, xrep0_pipe_putq_cb, p); - - p->pipe = pipe; - p->rep = s; - - // We want a pretty deep send queue on pipes. The rationale here is - // that the send rate will be mitigated by the receive rate. - // If a slow pipe (req pipe not reading its own responses!?) - // comes up, then we will start discarding its replies eventually, - // but it takes some time. It would be poor form for a peer to - // smash us with requests, but be unable to handle replies faster - // than we can forward them. If they do that, their replies get - // dropped. (From a DDoS perspective, it might be nice in the - // future if we had a way to exert back pressure to the send side -- - // essentially don't let peers send requests faster than they are - // willing to receive replies. Something to think about for the - // future.) - if ((rv = nni_msgq_init(&p->sendq, 64)) != 0) { - xrep0_pipe_fini(p); - return (rv); - } - return (0); -} - -static int -xrep0_pipe_start(void *arg) -{ - xrep0_pipe *p = arg; - xrep0_sock *s = p->rep; - int rv; - - if (nni_pipe_peer(p->pipe) != NNG_REP0_PEER) { - // Peer protocol mismatch. - return (NNG_EPROTO); - } - - nni_mtx_lock(&s->lk); - rv = nni_id_set(&s->pipes, nni_pipe_id(p->pipe), p); - nni_mtx_unlock(&s->lk); - if (rv != 0) { - return (rv); - } - - nni_msgq_aio_get(p->sendq, &p->aio_getq); - nni_pipe_recv(p->pipe, &p->aio_recv); - return (0); -} - -static void -xrep0_pipe_close(void *arg) -{ - xrep0_pipe *p = arg; - xrep0_sock *s = p->rep; - - nni_aio_close(&p->aio_getq); - nni_aio_close(&p->aio_send); - nni_aio_close(&p->aio_recv); - nni_aio_close(&p->aio_putq); - nni_msgq_close(p->sendq); - - nni_mtx_lock(&s->lk); - nni_id_remove(&s->pipes, nni_pipe_id(p->pipe)); - nni_mtx_unlock(&s->lk); -} - -static void -xrep0_sock_getq_cb(void *arg) -{ - xrep0_sock *s = arg; - nni_msgq * uwq = s->uwq; - nni_msg * msg; - uint32_t id; - xrep0_pipe *p; - - // This watches for messages from the upper write queue, - // extracts the destination pipe, and forwards it to the appropriate - // destination pipe via a separate queue. This prevents a single bad - // or slow pipe from gumming up the works for the entire socket. - - if (nni_aio_result(&s->aio_getq) != 0) { - // Closed socket? - return; - } - - msg = nni_aio_get_msg(&s->aio_getq); - nni_aio_set_msg(&s->aio_getq, NULL); - - // We yank the outgoing pipe id from the header - if (nni_msg_header_len(msg) < 4) { - nni_msg_free(msg); - - // Look for another message on the upper write queue. - nni_msgq_aio_get(uwq, &s->aio_getq); - return; - } - - id = nni_msg_header_trim_u32(msg); - - // Look for the pipe, and attempt to put the message there - // (non-blocking) if we can. If we can't for any reason, then we - // free the message. - nni_mtx_lock(&s->lk); - if (((p = nni_id_get(&s->pipes, id)) == NULL) || - (nni_msgq_tryput(p->sendq, msg) != 0)) { - nni_msg_free(msg); - } - nni_mtx_unlock(&s->lk); - - // Now look for another message on the upper write queue. - nni_msgq_aio_get(uwq, &s->aio_getq); -} - -static void -xrep0_pipe_getq_cb(void *arg) -{ - xrep0_pipe *p = arg; - - if (nni_aio_result(&p->aio_getq) != 0) { - nni_pipe_close(p->pipe); - return; - } - - nni_aio_set_msg(&p->aio_send, nni_aio_get_msg(&p->aio_getq)); - nni_aio_set_msg(&p->aio_getq, NULL); - - nni_pipe_send(p->pipe, &p->aio_send); -} - -static void -xrep0_pipe_send_cb(void *arg) -{ - xrep0_pipe *p = arg; - - if (nni_aio_result(&p->aio_send) != 0) { - nni_msg_free(nni_aio_get_msg(&p->aio_send)); - nni_aio_set_msg(&p->aio_send, NULL); - nni_pipe_close(p->pipe); - return; - } - - nni_msgq_aio_get(p->sendq, &p->aio_getq); -} - -static void -xrep0_pipe_recv_cb(void *arg) -{ - xrep0_pipe *p = arg; - xrep0_sock *s = p->rep; - nni_msg * msg; - int hops; - int ttl; - - if (nni_aio_result(&p->aio_recv) != 0) { - nni_pipe_close(p->pipe); - return; - } - - ttl = nni_atomic_get(&s->ttl); - - msg = nni_aio_get_msg(&p->aio_recv); - nni_aio_set_msg(&p->aio_recv, NULL); - - nni_msg_set_pipe(msg, nni_pipe_id(p->pipe)); - - // Store the pipe id in the header, first thing. - nni_msg_header_append_u32(msg, nni_pipe_id(p->pipe)); - - // Move backtrace from body to header - hops = 1; - for (;;) { - bool end; - uint8_t *body; - if (hops > ttl) { - // This isn't malformed, but it has gone through - // too many hops. Do not disconnect, because we - // can legitimately receive messages with too many - // hops from devices, etc. - goto drop; - } - hops++; - if (nni_msg_len(msg) < 4) { - // Peer is speaking garbage. Kick it. - nni_msg_free(msg); - nni_pipe_close(p->pipe); - return; - } - body = nni_msg_body(msg); - end = ((body[0] & 0x80u) != 0); - if (nni_msg_header_append(msg, body, 4) != 0) { - // Out of memory most likely, but keep going to - // avoid breaking things. - goto drop; - } - nni_msg_trim(msg, 4); - if (end) { - break; - } - } - - // Go ahead and send it up. - nni_aio_set_msg(&p->aio_putq, msg); - nni_msgq_aio_put(s->urq, &p->aio_putq); - return; - -drop: - nni_msg_free(msg); - nni_pipe_recv(p->pipe, &p->aio_recv); -} - -static void -xrep0_pipe_putq_cb(void *arg) -{ - xrep0_pipe *p = arg; - - if (nni_aio_result(&p->aio_putq) != 0) { - nni_msg_free(nni_aio_get_msg(&p->aio_putq)); - nni_aio_set_msg(&p->aio_putq, NULL); - nni_pipe_close(p->pipe); - return; - } - - nni_pipe_recv(p->pipe, &p->aio_recv); -} - -static int -xrep0_sock_set_maxttl(void *arg, const void *buf, size_t sz, nni_opt_type t) -{ - xrep0_sock *s = arg; - int ttl; - int rv; - if ((rv = nni_copyin_int(&ttl, buf, sz, 1, NNI_MAX_MAX_TTL, t)) == 0) { - nni_atomic_set(&s->ttl, ttl); - } - return (rv); -} - -static int -xrep0_sock_get_maxttl(void *arg, void *buf, size_t *szp, nni_opt_type t) -{ - xrep0_sock *s = arg; - return (nni_copyout_int(nni_atomic_get(&s->ttl), buf, szp, t)); -} - -static void -xrep0_sock_send(void *arg, nni_aio *aio) -{ - xrep0_sock *s = arg; - - nni_msgq_aio_put(s->uwq, aio); -} - -static void -xrep0_sock_recv(void *arg, nni_aio *aio) -{ - xrep0_sock *s = arg; - - nni_msgq_aio_get(s->urq, aio); -} - -// This is the global protocol structure -- our linkage to the core. -// This should be the only global non-static symbol in this file. -static nni_proto_pipe_ops xrep0_pipe_ops = { - .pipe_size = sizeof(xrep0_pipe), - .pipe_init = xrep0_pipe_init, - .pipe_fini = xrep0_pipe_fini, - .pipe_start = xrep0_pipe_start, - .pipe_close = xrep0_pipe_close, - .pipe_stop = xrep0_pipe_stop, -}; - -static nni_option xrep0_sock_options[] = { - { - .o_name = NNG_OPT_MAXTTL, - .o_get = xrep0_sock_get_maxttl, - .o_set = xrep0_sock_set_maxttl, - }, - // terminate list - { - .o_name = NULL, - }, -}; - -static nni_proto_sock_ops xrep0_sock_ops = { - .sock_size = sizeof(xrep0_sock), - .sock_init = xrep0_sock_init, - .sock_fini = xrep0_sock_fini, - .sock_open = xrep0_sock_open, - .sock_close = xrep0_sock_close, - .sock_options = xrep0_sock_options, - .sock_send = xrep0_sock_send, - .sock_recv = xrep0_sock_recv, -}; - -static nni_proto xrep0_proto = { - .proto_version = NNI_PROTOCOL_VERSION, - .proto_self = { NNG_REP0_SELF, NNG_REP0_SELF_NAME }, - .proto_peer = { NNG_REP0_PEER, NNG_REP0_PEER_NAME }, - .proto_flags = NNI_PROTO_FLAG_SNDRCV | NNI_PROTO_FLAG_RAW, - .proto_sock_ops = &xrep0_sock_ops, - .proto_pipe_ops = &xrep0_pipe_ops, -}; - -int -nng_rep0_open_raw(nng_socket *sidp) -{ - return (nni_proto_open(sidp, &xrep0_proto)); -} diff --git a/src/protocol/reqrep0/xrep_test.c b/src/protocol/reqrep0/xrep_test.c deleted file mode 100644 index 6f1564eb..00000000 --- a/src/protocol/reqrep0/xrep_test.c +++ /dev/null @@ -1,434 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - -#include - -static void -test_xrep_identity(void) -{ - nng_socket s; - int p1, p2; - char * n1; - char * n2; - - NUTS_PASS(nng_rep0_open_raw(&s)); - NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PROTO, &p1)); - NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PEER, &p2)); - NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PROTONAME, &n1)); - NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PEERNAME, &n2)); - NUTS_CLOSE(s); - NUTS_TRUE(p1 == NNG_REP0_SELF); - NUTS_TRUE(p2 == NNG_REP0_PEER); - NUTS_MATCH(n1, NNG_REP0_SELF_NAME); - NUTS_MATCH(n2, NNG_REP0_PEER_NAME); - nng_strfree(n1); - nng_strfree(n2); -} - -static void -test_xrep_raw(void) -{ - nng_socket s; - bool b; - - NUTS_PASS(nng_rep0_open_raw(&s)); - NUTS_PASS(nng_socket_get_bool(s, NNG_OPT_RAW, &b)); - NUTS_TRUE(b); - NUTS_CLOSE(s); -} - -static void -test_xrep_no_context(void) -{ - nng_socket s; - nng_ctx ctx; - - NUTS_PASS(nng_rep0_open_raw(&s)); - NUTS_FAIL(nng_ctx_open(&ctx, s), NNG_ENOTSUP); - NUTS_CLOSE(s); -} - -static void -test_xrep_poll_writeable(void) -{ - int fd; - nng_socket req; - nng_socket rep; - - NUTS_PASS(nng_rep0_open_raw(&rep)); - NUTS_PASS(nng_req0_open(&req)); - NUTS_PASS(nng_socket_get_int(rep, NNG_OPT_SENDFD, &fd)); - NUTS_TRUE(fd >= 0); - - // We are always writeable, even before connect. This is so that - // back-pressure from a bad peer can't trash others. We assume - // that peers won't send us requests faster than they can consume - // the answers. If they do, they will lose their answers. - NUTS_TRUE(nuts_poll_fd(fd) == true); - - NUTS_MARRY(req, rep); - - // Now it's writable. - NUTS_TRUE(nuts_poll_fd(fd) == true); - - NUTS_CLOSE(req); - NUTS_CLOSE(rep); -} - -static void -test_xrep_poll_readable(void) -{ - int fd; - nng_socket req; - nng_socket rep; - nng_msg * msg; - - NUTS_PASS(nng_req0_open(&req)); - NUTS_PASS(nng_rep0_open_raw(&rep)); - NUTS_PASS(nng_socket_get_int(rep, NNG_OPT_RECVFD, &fd)); - NUTS_TRUE(fd >= 0); - - // Not readable if not connected! - NUTS_TRUE(nuts_poll_fd(fd) == false); - - // Even after connect (no message yet) - NUTS_MARRY(req, rep); - NUTS_TRUE(nuts_poll_fd(fd) == false); - - // But once we send messages, it is. - // We have to send a request, in order to send a reply. - NUTS_SEND(req, "abc"); - NUTS_SLEEP(100); - - NUTS_TRUE(nuts_poll_fd(fd) == true); - - // and receiving makes it no longer ready - NUTS_PASS(nng_recvmsg(rep, &msg, 0)); - nng_msg_free(msg); - NUTS_TRUE(nuts_poll_fd(fd) == false); - - NUTS_CLOSE(req); - NUTS_CLOSE(rep); -} - -static void -test_xrep_validate_peer(void) -{ - nng_socket s1, s2; - nng_stat * stats; - nng_stat * reject; - char *addr; - - NUTS_ADDR(addr, "inproc"); - - NUTS_PASS(nng_rep0_open_raw(&s1)); - NUTS_PASS(nng_rep0_open(&s2)); - - NUTS_PASS(nng_listen(s1, addr, NULL, 0)); - NUTS_PASS(nng_dial(s2, addr, NULL, NNG_FLAG_NONBLOCK)); - - NUTS_SLEEP(100); - NUTS_PASS(nng_stats_get(&stats)); - - NUTS_TRUE(stats != NULL); - NUTS_TRUE((reject = nng_stat_find_socket(stats, s1)) != NULL); - NUTS_TRUE((reject = nng_stat_find(reject, "reject")) != NULL); - - NUTS_TRUE(nng_stat_type(reject) == NNG_STAT_COUNTER); - NUTS_TRUE(nng_stat_value(reject) > 0); - - NUTS_PASS(nng_close(s1)); - NUTS_PASS(nng_close(s2)); - nng_stats_free(stats); -} - -static void -test_xrep_close_pipe_before_send(void) -{ - nng_socket rep; - nng_socket req; - nng_pipe p; - nng_aio * aio1; - nng_msg * m; - - NUTS_PASS(nng_rep0_open_raw(&rep)); - NUTS_PASS(nng_req0_open(&req)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_aio_alloc(&aio1, NULL, NULL)); - - NUTS_MARRY(req, rep); - NUTS_SEND(req, "test"); - - nng_recv_aio(rep, aio1); - nng_aio_wait(aio1); - NUTS_PASS(nng_aio_result(aio1)); - NUTS_TRUE((m = nng_aio_get_msg(aio1)) != NULL); - p = nng_msg_get_pipe(m); - NUTS_PASS(nng_pipe_close(p)); - NUTS_PASS(nng_sendmsg(rep, m, 0)); - - NUTS_CLOSE(req); - NUTS_CLOSE(rep); - nng_aio_free(aio1); -} - -static void -test_xrep_close_pipe_during_send(void) -{ - nng_socket rep; - nng_socket req; - nng_pipe p; - nng_msg * m; - - NUTS_PASS(nng_rep0_open_raw(&rep)); - NUTS_PASS(nng_req0_open_raw(&req)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 200)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_int(rep, NNG_OPT_SENDBUF, 20)); - NUTS_PASS(nng_socket_set_int(rep, NNG_OPT_RECVBUF, 20)); - NUTS_PASS(nng_socket_set_int(req, NNG_OPT_SENDBUF, 20)); - NUTS_PASS(nng_socket_set_int(req, NNG_OPT_RECVBUF, 1)); - - NUTS_MARRY(req, rep); - - NUTS_PASS(nng_msg_alloc(&m, 4)); - NUTS_PASS(nng_msg_append_u32(m, (unsigned) 0x81000000u)); - NUTS_PASS(nng_sendmsg(req, m, 0)); - NUTS_PASS(nng_recvmsg(rep, &m, 0)); - p = nng_msg_get_pipe(m); - nng_msg_free(m); - - for (int i = 0; i < 100; i++) { - NUTS_PASS(nng_msg_alloc(&m, 4)); - NUTS_PASS(nng_msg_header_append_u32(m, nng_pipe_id(p))); - NUTS_PASS( - nng_msg_header_append_u32(m, (unsigned) i | 0x80000000u)); - // xrep does not exert back-pressure - NUTS_PASS(nng_sendmsg(rep, m, 0)); - } - NUTS_PASS(nng_pipe_close(p)); - - NUTS_CLOSE(req); - NUTS_CLOSE(rep); -} - -static void -test_xrep_close_during_recv(void) -{ - nng_socket rep; - nng_socket req; - nng_msg * m; - - NUTS_PASS(nng_rep0_open_raw(&rep)); - NUTS_PASS(nng_req0_open_raw(&req)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 100)); - NUTS_PASS(nng_socket_set_int(rep, NNG_OPT_RECVBUF, 5)); - NUTS_PASS(nng_socket_set_int(req, NNG_OPT_SENDBUF, 20)); - - NUTS_MARRY(req, rep); - - for (unsigned i = 0; i < 100; i++) { - int rv; - NUTS_PASS(nng_msg_alloc(&m, 4)); - NUTS_PASS(nng_msg_header_append_u32(m, i | 0x80000000u)); - rv = nng_sendmsg(req, m, 0); - if (rv == NNG_ETIMEDOUT) { - nng_msg_free(m); - break; - } - } - NUTS_CLOSE(req); - NUTS_CLOSE(rep); -} - -static void -test_xrep_recv_aio_stopped(void) -{ - nng_socket rep; - nng_aio * aio; - - NUTS_PASS(nng_rep0_open_raw(&rep)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - - nng_aio_stop(aio); - nng_recv_aio(rep, aio); - nng_aio_wait(aio); - NUTS_FAIL(nng_aio_result(aio), NNG_ECANCELED); - NUTS_CLOSE(rep); - nng_aio_free(aio); -} - -static void -test_xrep_send_no_header(void) -{ - nng_socket rep; - nng_socket req; - nng_msg * m; - - NUTS_PASS(nng_req0_open_raw(&req)); - NUTS_PASS(nng_rep0_open_raw(&rep)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 100)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_RECVTIMEO, 100)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); - - NUTS_MARRY(req, rep); - - NUTS_PASS(nng_msg_alloc(&m, 4)); - NUTS_PASS(nng_sendmsg(rep, m, 0)); - NUTS_FAIL(nng_recvmsg(rep, &m, 0), NNG_ETIMEDOUT); - - NUTS_CLOSE(req); - NUTS_CLOSE(rep); -} - -static void -test_xrep_recv_garbage(void) -{ - nng_socket rep; - nng_socket req; - nng_msg * m; - - NUTS_PASS(nng_rep0_open_raw(&rep)); - NUTS_PASS(nng_req0_open_raw(&req)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 100)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 100)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); - - NUTS_MARRY(req, rep); - - NUTS_PASS(nng_msg_alloc(&m, 4)); - NUTS_PASS(nng_msg_append_u32(m, 1u)); - NUTS_PASS(nng_sendmsg(req, m, 0)); - NUTS_FAIL(nng_recvmsg(rep, &m, 0), NNG_ETIMEDOUT); - - NUTS_CLOSE(req); - NUTS_CLOSE(rep); -} - -static void -test_xrep_ttl_option(void) -{ - nng_socket rep; - int v; - bool b; - size_t sz; - const char *opt = NNG_OPT_MAXTTL; - - NUTS_PASS(nng_rep0_open_raw(&rep)); - - NUTS_PASS(nng_socket_set_int(rep, opt, 1)); - NUTS_FAIL(nng_socket_set_int(rep, opt, 0), NNG_EINVAL); - NUTS_FAIL(nng_socket_set_int(rep, opt, -1), NNG_EINVAL); - NUTS_FAIL(nng_socket_set_int(rep, opt, 16), NNG_EINVAL); - NUTS_FAIL(nng_socket_set_int(rep, opt, 256), NNG_EINVAL); - NUTS_PASS(nng_socket_set_int(rep, opt, 3)); - NUTS_PASS(nng_socket_get_int(rep, opt, &v)); - NUTS_TRUE(v == 3); - v = 0; - sz = sizeof(v); - NUTS_PASS(nng_socket_get(rep, opt, &v, &sz)); - NUTS_TRUE(v == 3); - NUTS_TRUE(sz == sizeof(v)); - - NUTS_TRUE(nng_socket_set(rep, opt, "", 1) == NNG_EINVAL); - sz = 1; - NUTS_TRUE(nng_socket_get(rep, opt, &v, &sz) == NNG_EINVAL); - NUTS_TRUE(nng_socket_set_bool(rep, opt, true) == NNG_EBADTYPE); - NUTS_TRUE(nng_socket_get_bool(rep, opt, &b) == NNG_EBADTYPE); - - NUTS_TRUE(nng_close(rep) == 0); -} - -static void -test_xrep_ttl_drop(void) -{ - nng_socket rep; - nng_socket req; - nng_msg * m; - - NUTS_PASS(nng_rep0_open_raw(&rep)); - NUTS_PASS(nng_req0_open_raw(&req)); - NUTS_PASS(nng_socket_set_int(rep, NNG_OPT_MAXTTL, 3)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 200)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); - - NUTS_MARRY(req, rep); - - // Send messages. Note that xrep implicitly adds a hop on receive. - - NUTS_PASS(nng_msg_alloc(&m, 0)); - NUTS_PASS(nng_msg_append_u32(m, 1u)); // 2 hops - NUTS_PASS(nng_msg_append_u32(m, 0x80000001u)); - NUTS_PASS(nng_msg_append(m, "PASS1", 6)); - NUTS_PASS(nng_sendmsg(req, m, 0)); - - NUTS_PASS(nng_msg_alloc(&m, 0)); - NUTS_PASS(nng_msg_append_u32(m, 1u)); // 4 hops -- discard! - NUTS_PASS(nng_msg_append_u32(m, 2u)); - NUTS_PASS(nng_msg_append_u32(m, 3u)); - NUTS_PASS(nng_msg_append_u32(m, 0x80000002u)); - NUTS_PASS(nng_msg_append(m, "FAIL2", 6)); - NUTS_PASS(nng_sendmsg(req, m, 0)); - - NUTS_PASS(nng_msg_alloc(&m, 0)); - NUTS_PASS(nng_msg_append_u32(m, 1u)); // 3 hops - passes - NUTS_PASS(nng_msg_append_u32(m, 2u)); - NUTS_PASS(nng_msg_append_u32(m, 0x80000003u)); - NUTS_PASS(nng_msg_append(m, "PASS3", 6)); - NUTS_PASS(nng_sendmsg(req, m, 0)); - - NUTS_PASS(nng_msg_alloc(&m, 0)); - NUTS_PASS(nng_msg_append_u32(m, 1u)); // 4 hops -- discard! - NUTS_PASS(nng_msg_append_u32(m, 2u)); - NUTS_PASS(nng_msg_append_u32(m, 3u)); - NUTS_PASS(nng_msg_append_u32(m, 0x80000003u)); - NUTS_PASS(nng_msg_append(m, "FAIL4", 6)); - NUTS_PASS(nng_sendmsg(req, m, 0)); - - // So on receive we should see 80000001 and 80000003. - NUTS_PASS(nng_recvmsg(rep, &m, 0)); - NUTS_TRUE(nng_msg_header_len(m) == 12); - NUTS_TRUE(nng_msg_len(m) == 6); - NUTS_TRUE(strcmp(nng_msg_body(m), "PASS1") == 0); - nng_msg_free(m); - - NUTS_PASS(nng_recvmsg(rep, &m, 0)); - NUTS_TRUE(nng_msg_header_len(m) == 16); // 3 hops + ID - NUTS_TRUE(nng_msg_len(m) == 6); - NUTS_TRUE(strcmp(nng_msg_body(m), "PASS3") == 0); - nng_msg_free(m); - - NUTS_FAIL(nng_recvmsg(rep, &m, 0), NNG_ETIMEDOUT); - - NUTS_CLOSE(req); - NUTS_CLOSE(rep); -} - -NUTS_TESTS = { - { "xrep identity", test_xrep_identity }, - { "xrep raw", test_xrep_raw }, - { "xrep no context", test_xrep_no_context }, - { "xrep poll readable", test_xrep_poll_readable }, - { "xrep poll writable", test_xrep_poll_writeable }, - { "xrep validate peer", test_xrep_validate_peer }, - { "xrep close pipe before send", test_xrep_close_pipe_before_send }, - { "xrep close pipe during send", test_xrep_close_pipe_during_send }, - { "xrep close during recv", test_xrep_close_during_recv }, - { "xrep recv aio stopped", test_xrep_recv_aio_stopped }, - { "xrep send no header", test_xrep_send_no_header }, - { "xrep recv garbage", test_xrep_recv_garbage }, - { "xrep ttl option", test_xrep_ttl_option }, - { "xrep ttl drop", test_xrep_ttl_drop }, - { NULL, NULL }, -}; diff --git a/src/protocol/reqrep0/xreq.c b/src/protocol/reqrep0/xreq.c deleted file mode 100644 index bcb218bf..00000000 --- a/src/protocol/reqrep0/xreq.c +++ /dev/null @@ -1,319 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// Copyright 2018 Capitar IT Group BV -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - -#include - -#include "core/nng_impl.h" -#include "nng/protocol/reqrep0/req.h" - -// Request protocol. The REQ protocol is the "request" side of a -// request-reply pair. This is useful for building RPC clients, for example. - -typedef struct xreq0_pipe xreq0_pipe; -typedef struct xreq0_sock xreq0_sock; - -// An xreq0_sock is our per-socket protocol private structure. -struct xreq0_sock { - nni_msgq * uwq; - nni_msgq * urq; - nni_atomic_int ttl; -}; - -// A req0_pipe is our per-pipe protocol private structure. -struct xreq0_pipe { - nni_pipe * pipe; - xreq0_sock *req; - nni_aio aio_getq; - nni_aio aio_send; - nni_aio aio_recv; - nni_aio aio_putq; -}; - -static void xreq0_sock_fini(void *); -static void xreq0_getq_cb(void *); -static void xreq0_send_cb(void *); -static void xreq0_recv_cb(void *); -static void xreq0_putq_cb(void *); - -static int -xreq0_sock_init(void *arg, nni_sock *sock) -{ - xreq0_sock *s = arg; - - nni_atomic_init(&s->ttl); - nni_atomic_set(&s->ttl, 8); - s->uwq = nni_sock_sendq(sock); - s->urq = nni_sock_recvq(sock); - - return (0); -} - -static void -xreq0_sock_open(void *arg) -{ - NNI_ARG_UNUSED(arg); -} - -static void -xreq0_sock_close(void *arg) -{ - NNI_ARG_UNUSED(arg); -} - -static void -xreq0_sock_fini(void *arg) -{ - NNI_ARG_UNUSED(arg); -} - -static void -xreq0_pipe_stop(void *arg) -{ - xreq0_pipe *p = arg; - - nni_aio_stop(&p->aio_getq); - nni_aio_stop(&p->aio_putq); - nni_aio_stop(&p->aio_recv); - nni_aio_stop(&p->aio_send); -} - -static void -xreq0_pipe_fini(void *arg) -{ - xreq0_pipe *p = arg; - - nni_aio_fini(&p->aio_getq); - nni_aio_fini(&p->aio_putq); - nni_aio_fini(&p->aio_recv); - nni_aio_fini(&p->aio_send); -} - -static int -xreq0_pipe_init(void *arg, nni_pipe *pipe, void *s) -{ - xreq0_pipe *p = arg; - - nni_aio_init(&p->aio_getq, xreq0_getq_cb, p); - nni_aio_init(&p->aio_putq, xreq0_putq_cb, p); - nni_aio_init(&p->aio_recv, xreq0_recv_cb, p); - nni_aio_init(&p->aio_send, xreq0_send_cb, p); - - p->pipe = pipe; - p->req = s; - return (0); -} - -static int -xreq0_pipe_start(void *arg) -{ - xreq0_pipe *p = arg; - xreq0_sock *s = p->req; - - if (nni_pipe_peer(p->pipe) != NNG_REQ0_PEER) { - return (NNG_EPROTO); - } - - nni_msgq_aio_get(s->uwq, &p->aio_getq); - nni_pipe_recv(p->pipe, &p->aio_recv); - return (0); -} - -static void -xreq0_pipe_close(void *arg) -{ - xreq0_pipe *p = arg; - - nni_aio_close(&p->aio_getq); - nni_aio_close(&p->aio_putq); - nni_aio_close(&p->aio_recv); - nni_aio_close(&p->aio_send); -} - -// For raw mode we can just let the pipes "contend" via get queue to get a -// message from the upper write queue. The msg queue implementation -// actually provides ordering, so load will be spread automatically. -// (NB: We may have to revise this in the future if we want to provide some -// kind of priority.) - -static void -xreq0_getq_cb(void *arg) -{ - xreq0_pipe *p = arg; - - if (nni_aio_result(&p->aio_getq) != 0) { - nni_pipe_close(p->pipe); - return; - } - - nni_aio_set_msg(&p->aio_send, nni_aio_get_msg(&p->aio_getq)); - nni_aio_set_msg(&p->aio_getq, NULL); - - nni_pipe_send(p->pipe, &p->aio_send); -} - -static void -xreq0_send_cb(void *arg) -{ - xreq0_pipe *p = arg; - - if (nni_aio_result(&p->aio_send) != 0) { - nni_msg_free(nni_aio_get_msg(&p->aio_send)); - nni_aio_set_msg(&p->aio_send, NULL); - nni_pipe_close(p->pipe); - return; - } - - // Sent a message so we just need to look for another one. - nni_msgq_aio_get(p->req->uwq, &p->aio_getq); -} - -static void -xreq0_putq_cb(void *arg) -{ - xreq0_pipe *p = arg; - - if (nni_aio_result(&p->aio_putq) != 0) { - nni_msg_free(nni_aio_get_msg(&p->aio_putq)); - nni_aio_set_msg(&p->aio_putq, NULL); - nni_pipe_close(p->pipe); - return; - } - nni_aio_set_msg(&p->aio_putq, NULL); - - nni_pipe_recv(p->pipe, &p->aio_recv); -} - -static void -xreq0_recv_cb(void *arg) -{ - xreq0_pipe *p = arg; - xreq0_sock *sock = p->req; - nni_msg * msg; - bool end; - - if (nni_aio_result(&p->aio_recv) != 0) { - nni_pipe_close(p->pipe); - return; - } - - msg = nni_aio_get_msg(&p->aio_recv); - nni_aio_set_msg(&p->aio_recv, NULL); - nni_msg_set_pipe(msg, nni_pipe_id(p->pipe)); - end = false; - - while (!end) { - uint8_t *body; - - if (nni_msg_len(msg) < 4) { - // Peer gave us garbage, so kick it. - nni_msg_free(msg); - nni_pipe_close(p->pipe); - return; - } - body = nni_msg_body(msg); - end = ((body[0] & 0x80u) != 0); - - if (nng_msg_header_append(msg, body, sizeof (uint32_t)) != 0) { - // TODO: bump a no-memory stat - nni_msg_free(msg); - // Closing the pipe may release some memory. - // It at least gives an indication to the peer - // that we've lost the message. - nni_pipe_close(p->pipe); - return; - } - nni_msg_trim(msg, sizeof (uint32_t)); - } - nni_aio_set_msg(&p->aio_putq, msg); - nni_msgq_aio_put(sock->urq, &p->aio_putq); -} - -static void -xreq0_sock_send(void *arg, nni_aio *aio) -{ - xreq0_sock *s = arg; - - nni_msgq_aio_put(s->uwq, aio); -} - -static void -xreq0_sock_recv(void *arg, nni_aio *aio) -{ - xreq0_sock *s = arg; - - nni_msgq_aio_get(s->urq, aio); -} - -static int -xreq0_sock_set_max_ttl(void *arg, const void *buf, size_t sz, nni_opt_type t) -{ - xreq0_sock *s = arg; - int ttl; - int rv; - if ((rv = nni_copyin_int(&ttl, buf, sz, 1, NNI_MAX_MAX_TTL, t)) == 0) { - nni_atomic_set(&s->ttl, ttl); - } - return (rv); -} - -static int -xreq0_sock_get_max_ttl(void *arg, void *buf, size_t *szp, nni_opt_type t) -{ - xreq0_sock *s = arg; - return (nni_copyout_int(nni_atomic_get(&s->ttl), buf, szp, t)); -} - -static nni_proto_pipe_ops xreq0_pipe_ops = { - .pipe_size = sizeof(xreq0_pipe), - .pipe_init = xreq0_pipe_init, - .pipe_fini = xreq0_pipe_fini, - .pipe_start = xreq0_pipe_start, - .pipe_close = xreq0_pipe_close, - .pipe_stop = xreq0_pipe_stop, -}; - -static nni_option xreq0_sock_options[] = { - { - .o_name = NNG_OPT_MAXTTL, - .o_get = xreq0_sock_get_max_ttl, - .o_set = xreq0_sock_set_max_ttl, - }, - // terminate list - { - .o_name = NULL, - }, -}; - -static nni_proto_sock_ops xreq0_sock_ops = { - .sock_size = sizeof(xreq0_sock), - .sock_init = xreq0_sock_init, - .sock_fini = xreq0_sock_fini, - .sock_open = xreq0_sock_open, - .sock_close = xreq0_sock_close, - .sock_options = xreq0_sock_options, - .sock_send = xreq0_sock_send, - .sock_recv = xreq0_sock_recv, -}; - -static nni_proto xreq0_proto = { - .proto_version = NNI_PROTOCOL_VERSION, - .proto_self = { NNG_REQ0_SELF, NNG_REQ0_SELF_NAME }, - .proto_peer = { NNG_REQ0_PEER, NNG_REQ0_PEER_NAME }, - .proto_flags = NNI_PROTO_FLAG_SNDRCV | NNI_PROTO_FLAG_RAW, - .proto_sock_ops = &xreq0_sock_ops, - .proto_pipe_ops = &xreq0_pipe_ops, - .proto_ctx_ops = NULL, // raw mode does not support contexts -}; - -int -nng_req0_open_raw(nng_socket *sock) -{ - return (nni_proto_open(sock, &xreq0_proto)); -} diff --git a/src/protocol/reqrep0/xreq_test.c b/src/protocol/reqrep0/xreq_test.c deleted file mode 100644 index 8c850cba..00000000 --- a/src/protocol/reqrep0/xreq_test.c +++ /dev/null @@ -1,367 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - -#include - -static void -test_xreq_identity(void) -{ - nng_socket s; - int p1, p2; - char * n1; - char * n2; - - NUTS_PASS(nng_req0_open_raw(&s)); - NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PROTO, &p1)); - NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PEER, &p2)); - NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PROTONAME, &n1)); - NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PEERNAME, &n2)); - NUTS_CLOSE(s); - NUTS_TRUE(p1 == NNG_REQ0_SELF); - NUTS_TRUE(p2 == NNG_REQ0_PEER); - NUTS_MATCH(n1, NNG_REQ0_SELF_NAME); - NUTS_MATCH(n2, NNG_REQ0_PEER_NAME); - nng_strfree(n1); - nng_strfree(n2); -} - -static void -test_xreq_raw(void) -{ - nng_socket s; - bool b; - - NUTS_PASS(nng_req0_open_raw(&s)); - NUTS_PASS(nng_socket_get_bool(s, NNG_OPT_RAW, &b)); - NUTS_TRUE(b); - NUTS_CLOSE(s); -} - -static void -test_xreq_no_context(void) -{ - nng_socket s; - nng_ctx ctx; - - NUTS_PASS(nng_req0_open_raw(&s)); - NUTS_FAIL(nng_ctx_open(&ctx, s), NNG_ENOTSUP); - NUTS_CLOSE(s); -} - -static void -test_xreq_poll_writeable(void) -{ - int fd; - nng_socket req; - nng_socket rep; - - NUTS_PASS(nng_req0_open_raw(&req)); - NUTS_PASS(nng_rep0_open(&rep)); - NUTS_PASS(nng_socket_get_int(req, NNG_OPT_SENDFD, &fd)); - NUTS_TRUE(fd >= 0); - - // We can't write until we have a connection. - NUTS_TRUE(nuts_poll_fd(fd) == false); - - NUTS_MARRY(req, rep); - - // Now it's writable. - NUTS_TRUE(nuts_poll_fd(fd) == true); - - NUTS_CLOSE(req); - NUTS_CLOSE(rep); -} - -static void -test_xreq_poll_readable(void) -{ - int fd; - nng_socket req; - nng_socket rep; - nng_msg * msg; - - NUTS_PASS(nng_req0_open_raw(&req)); - NUTS_PASS(nng_rep0_open(&rep)); - NUTS_PASS(nng_socket_get_int(req, NNG_OPT_RECVFD, &fd)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); - - NUTS_TRUE(fd >= 0); - - // Not readable if not connected! - NUTS_TRUE(nuts_poll_fd(fd) == false); - - // Even after connect (no message yet) - NUTS_MARRY(req, rep); - NUTS_TRUE(nuts_poll_fd(fd) == false); - - // But once we send messages, it is. - // We have to send a request, in order to send a reply. - NUTS_PASS(nng_msg_alloc(&msg, 0)); - // Request ID - NUTS_PASS(nng_msg_append_u32(msg, 0x80000000)); - NUTS_PASS(nng_sendmsg(req, msg, 0)); - - NUTS_PASS(nng_recvmsg(rep, &msg, 0)); - NUTS_PASS(nng_sendmsg(rep, msg, 0)); - - NUTS_SLEEP(100); - - NUTS_TRUE(nuts_poll_fd(fd) == true); - - // and receiving makes it no longer ready - NUTS_PASS(nng_recvmsg(req, &msg, 0)); - nng_msg_free(msg); - NUTS_TRUE(nuts_poll_fd(fd) == false); - - NUTS_CLOSE(req); - NUTS_CLOSE(rep); -} - -static void -test_xreq_validate_peer(void) -{ - nng_socket s1, s2; - nng_stat * stats; - nng_stat * reject; - char * addr; - - NUTS_ADDR(addr, "inproc"); - - NUTS_PASS(nng_req0_open_raw(&s1)); - NUTS_PASS(nng_req0_open(&s2)); - - NUTS_PASS(nng_listen(s1, addr, NULL, 0)); - NUTS_PASS(nng_dial(s2, addr, NULL, NNG_FLAG_NONBLOCK)); - - NUTS_SLEEP(100); - NUTS_PASS(nng_stats_get(&stats)); - - NUTS_TRUE(stats != NULL); - NUTS_TRUE((reject = nng_stat_find_socket(stats, s1)) != NULL); - NUTS_TRUE((reject = nng_stat_find(reject, "reject")) != NULL); - - NUTS_TRUE(nng_stat_type(reject) == NNG_STAT_COUNTER); - NUTS_TRUE(nng_stat_value(reject) > 0); - - NUTS_CLOSE(s1); - NUTS_CLOSE(s2); - nng_stats_free(stats); -} - -static void -test_xreq_recv_aio_stopped(void) -{ - nng_socket req; - nng_aio * aio; - - NUTS_PASS(nng_req0_open_raw(&req)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - - nng_aio_stop(aio); - nng_recv_aio(req, aio); - nng_aio_wait(aio); - NUTS_FAIL(nng_aio_result(aio), NNG_ECANCELED); - NUTS_CLOSE(req); - nng_aio_free(aio); -} - -static void -test_xreq_recv_garbage(void) -{ - nng_socket rep; - nng_socket req; - nng_msg * m; - uint32_t req_id; - - NUTS_PASS(nng_rep0_open_raw(&rep)); - NUTS_PASS(nng_req0_open_raw(&req)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_RECVTIMEO, 100)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 1000)); - - NUTS_MARRY(req, rep); - - NUTS_PASS(nng_msg_alloc(&m, 0)); - NUTS_PASS(nng_msg_append_u32(m, 0x80000000)); - NUTS_PASS(nng_sendmsg(req, m, 0)); - - NUTS_PASS(nng_recvmsg(rep, &m, 0)); - - // The message will have a header that contains the 32-bit pipe ID, - // followed by the 32-bit request ID. We will discard the request - // ID before sending it out. - NUTS_TRUE(nng_msg_header_len(m) == 8); - NUTS_PASS(nng_msg_header_chop_u32(m, &req_id)); - NUTS_TRUE(req_id == 0x80000000); - - NUTS_PASS(nng_sendmsg(rep, m, 0)); - NUTS_FAIL(nng_recvmsg(req, &m, 0), NNG_ETIMEDOUT); - - NUTS_CLOSE(req); - NUTS_CLOSE(rep); -} - -static void -test_xreq_recv_header(void) -{ - nng_socket rep; - nng_socket req; - nng_msg * m; - nng_pipe p1, p2; - uint32_t id; - - NUTS_PASS(nng_rep0_open_raw(&rep)); - NUTS_PASS(nng_req0_open_raw(&req)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 1000)); - - NUTS_MARRY_EX(req, rep, NULL, &p1, &p2); - - // Simulate a few hops. - NUTS_PASS(nng_msg_alloc(&m, 0)); - NUTS_PASS(nng_msg_header_append_u32(m, nng_pipe_id(p2))); - NUTS_PASS(nng_msg_header_append_u32(m, 0x2)); - NUTS_PASS(nng_msg_header_append_u32(m, 0x1)); - NUTS_PASS(nng_msg_header_append_u32(m, 0x80000123u)); - - NUTS_PASS(nng_sendmsg(rep, m, 0)); - - NUTS_PASS(nng_recvmsg(req, &m, 0)); - NUTS_TRUE(nng_msg_header_len(m) == 12); - NUTS_PASS(nng_msg_header_trim_u32(m, &id)); - NUTS_TRUE(id == 0x2); - NUTS_PASS(nng_msg_header_trim_u32(m, &id)); - NUTS_TRUE(id == 0x1); - NUTS_PASS(nng_msg_header_trim_u32(m, &id)); - NUTS_TRUE(id == 0x80000123u); - - nng_msg_free(m); - - NUTS_CLOSE(req); - NUTS_CLOSE(rep); -} - -static void -test_xreq_close_during_recv(void) -{ - nng_socket rep; - nng_socket req; - nng_msg * m; - nng_pipe p1; - nng_pipe p2; - - NUTS_PASS(nng_rep0_open_raw(&rep)); - NUTS_PASS(nng_req0_open_raw(&req)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 100)); - NUTS_PASS(nng_socket_set_int(req, NNG_OPT_RECVBUF, 5)); - NUTS_PASS(nng_socket_set_int(rep, NNG_OPT_SENDBUF, 20)); - - NUTS_MARRY_EX(req, rep, NULL, &p1, &p2); - NUTS_TRUE(nng_pipe_id(p1) > 0); - NUTS_TRUE(nng_pipe_id(p2) > 0); - - for (unsigned i = 0; i < 20; i++) { - NUTS_PASS(nng_msg_alloc(&m, 4)); - NUTS_PASS(nng_msg_header_append_u32(m, nng_pipe_id(p2))); - NUTS_PASS(nng_msg_header_append_u32(m, i | 0x80000000u)); - NUTS_SLEEP(10); - NUTS_PASS(nng_sendmsg(rep, m, 0)); - } - NUTS_CLOSE(req); - NUTS_CLOSE(rep); -} - -static void -test_xreq_close_pipe_during_send(void) -{ - nng_socket rep; - nng_socket req; - nng_msg * m; - nng_pipe p1; - nng_pipe p2; - - NUTS_PASS(nng_rep0_open_raw(&rep)); - NUTS_PASS(nng_req0_open_raw(&req)); - NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 100)); - NUTS_PASS(nng_socket_set_int(rep, NNG_OPT_RECVBUF, 5)); - NUTS_PASS(nng_socket_set_int(req, NNG_OPT_SENDBUF, 20)); - - NUTS_MARRY_EX(req, rep, NULL, &p1, &p2); - NUTS_TRUE(nng_pipe_id(p1) > 0); - NUTS_TRUE(nng_pipe_id(p2) > 0); - - for (unsigned i = 0; i < 20; i++) { - NUTS_PASS(nng_msg_alloc(&m, 4)); - NUTS_PASS(nng_msg_header_append_u32(m, i | 0x80000000u)); - NUTS_SLEEP(10); - NUTS_PASS(nng_sendmsg(req, m, 0)); - } - - NUTS_PASS(nng_pipe_close(p1)); - NUTS_CLOSE(req); - NUTS_CLOSE(rep); -} - -static void -test_xreq_ttl_option(void) -{ - nng_socket rep; - int v; - bool b; - size_t sz; - const char *opt = NNG_OPT_MAXTTL; - - NUTS_PASS(nng_req0_open_raw(&rep)); - - NUTS_PASS(nng_socket_set_int(rep, opt, 1)); - NUTS_FAIL(nng_socket_set_int(rep, opt, 0), NNG_EINVAL); - NUTS_FAIL(nng_socket_set_int(rep, opt, -1), NNG_EINVAL); - NUTS_FAIL(nng_socket_set_int(rep, opt, 16), NNG_EINVAL); - NUTS_FAIL(nng_socket_set_int(rep, opt, 256), NNG_EINVAL); - NUTS_PASS(nng_socket_set_int(rep, opt, 3)); - NUTS_PASS(nng_socket_get_int(rep, opt, &v)); - NUTS_TRUE(v == 3); - v = 0; - sz = sizeof(v); - NUTS_PASS(nng_socket_get(rep, opt, &v, &sz)); - NUTS_TRUE(v == 3); - NUTS_TRUE(sz == sizeof(v)); - - NUTS_TRUE(nng_socket_set(rep, opt, "", 1) == NNG_EINVAL); - sz = 1; - NUTS_TRUE(nng_socket_get(rep, opt, &v, &sz) == NNG_EINVAL); - NUTS_TRUE(nng_socket_set_bool(rep, opt, true) == NNG_EBADTYPE); - NUTS_TRUE(nng_socket_get_bool(rep, opt, &b) == NNG_EBADTYPE); - - NUTS_TRUE(nng_close(rep) == 0); -} - -NUTS_TESTS = { - { "xreq identity", test_xreq_identity }, - { "xreq raw", test_xreq_raw }, - { "xreq no context", test_xreq_no_context }, - { "xreq poll readable", test_xreq_poll_readable }, - { "xreq poll writable", test_xreq_poll_writeable }, - { "xreq validate peer", test_xreq_validate_peer }, - { "xreq recv aio stopped", test_xreq_recv_aio_stopped }, - { "xreq recv garbage", test_xreq_recv_garbage }, - { "xreq recv header", test_xreq_recv_header }, - { "xreq close during recv", test_xreq_close_during_recv }, - { "xreq close pipe during send", test_xreq_close_pipe_during_send }, - { "xreq ttl option", test_xreq_ttl_option }, - { NULL, NULL }, -}; diff --git a/src/protocol/survey0/CMakeLists.txt b/src/protocol/survey0/CMakeLists.txt deleted file mode 100644 index b5daca41..00000000 --- a/src/protocol/survey0/CMakeLists.txt +++ /dev/null @@ -1,25 +0,0 @@ -# -# Copyright 2020 Staysail Systems, Inc. -# Copyright 2018 Capitar IT Group BV -# -# This software is supplied under the terms of the MIT License, a -# copy of which should be located in the distribution where this -# file was obtained (LICENSE.txt). A copy of the license may also be -# found online at https://opensource.org/licenses/MIT. -# - -# Surveyor/Respondent protocol -nng_directory(survey0) - -nng_sources_if(NNG_PROTO_SURVEYOR0 survey.c xsurvey.c) -nng_headers_if(NNG_PROTO_SURVEYOR0 nng/protocol/survey0/survey.h) -nng_defines_if(NNG_PROTO_SURVEYOR0 NNG_HAVE_SURVEYOR0) - -nng_sources_if(NNG_PROTO_RESPONDENT0 respond.c xrespond.c) -nng_headers_if(NNG_PROTO_RESPONDENT0 nng/protocol/survey0/respond.h) -nng_defines_if(NNG_PROTO_RESPONDENT0 NNG_HAVE_RESPONDENT0) - -nng_test(respond_test) -nng_test(survey_test) -nng_test(xrespond_test) -nng_test(xsurvey_test) \ No newline at end of file diff --git a/src/protocol/survey0/respond.c b/src/protocol/survey0/respond.c deleted file mode 100644 index ad551c8f..00000000 --- a/src/protocol/survey0/respond.c +++ /dev/null @@ -1,693 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// Copyright 2018 Capitar IT Group BV -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - -#include -#include - -#include "core/nng_impl.h" -#include "nng/protocol/survey0/respond.h" - -// Respondent protocol. The RESPONDENT protocol is the "replier" side of -// the surveyor pattern. This is useful for building service discovery, or -// voting algorithms, for example. - -#ifndef NNI_PROTO_SURVEYOR_V0 -#define NNI_PROTO_SURVEYOR_V0 NNI_PROTO(6, 2) -#endif - -#ifndef NNI_PROTO_RESPONDENT_V0 -#define NNI_PROTO_RESPONDENT_V0 NNI_PROTO(6, 3) -#endif - -typedef struct resp0_pipe resp0_pipe; -typedef struct resp0_sock resp0_sock; -typedef struct resp0_ctx resp0_ctx; - -static void resp0_pipe_send_cb(void *); -static void resp0_pipe_recv_cb(void *); -static void resp0_pipe_fini(void *); - -struct resp0_ctx { - resp0_sock * sock; - uint32_t pipe_id; - resp0_pipe * spipe; // send pipe - nni_aio * saio; // send aio - nni_aio * raio; // recv aio - nni_list_node sqnode; - nni_list_node rqnode; - size_t btrace_len; - uint32_t btrace[NNI_MAX_MAX_TTL + 1]; -}; - -// resp0_sock is our per-socket protocol private structure. -struct resp0_sock { - nni_mtx mtx; - nni_atomic_int ttl; - nni_id_map pipes; - resp0_ctx ctx; - nni_list recvpipes; - nni_list recvq; - nni_pollable readable; - nni_pollable writable; -}; - -// resp0_pipe is our per-pipe protocol private structure. -struct resp0_pipe { - nni_pipe * npipe; - resp0_sock * psock; - bool busy; - bool closed; - uint32_t id; - nni_list sendq; // contexts waiting to send - nni_aio aio_send; - nni_aio aio_recv; - nni_list_node rnode; // receivable linkage -}; - -static void -resp0_ctx_close(void *arg) -{ - resp0_ctx * ctx = arg; - resp0_sock *s = ctx->sock; - nni_aio * aio; - - // complete any outstanding operations here, cancellation, etc. - - nni_mtx_lock(&s->mtx); - if ((aio = ctx->saio) != NULL) { - resp0_pipe *p = ctx->spipe; - ctx->saio = NULL; - ctx->spipe = NULL; - nni_list_remove(&p->sendq, ctx); - nni_aio_finish_error(aio, NNG_ECLOSED); - } - if ((aio = ctx->raio) != NULL) { - ctx->raio = NULL; - nni_list_remove(&s->recvq, ctx); - nni_aio_finish_error(aio, NNG_ECLOSED); - } - nni_mtx_unlock(&s->mtx); -} - -static void -resp0_ctx_fini(void *arg) -{ - resp0_ctx *ctx = arg; - - resp0_ctx_close(ctx); -} - -static int -resp0_ctx_init(void *carg, void *sarg) -{ - resp0_sock *s = sarg; - resp0_ctx * ctx = carg; - - NNI_LIST_NODE_INIT(&ctx->sqnode); - NNI_LIST_NODE_INIT(&ctx->rqnode); - ctx->btrace_len = 0; - ctx->sock = s; - ctx->pipe_id = 0; - - return (0); -} - -static void -resp0_ctx_cancel_send(nni_aio *aio, void *arg, int rv) -{ - resp0_ctx * ctx = arg; - resp0_sock *s = ctx->sock; - - nni_mtx_lock(&s->mtx); - if (ctx->saio != aio) { - nni_mtx_unlock(&s->mtx); - return; - } - nni_list_node_remove(&ctx->sqnode); - ctx->saio = NULL; - nni_mtx_unlock(&s->mtx); - nni_msg_header_clear(nni_aio_get_msg(aio)); // reset the headers - nni_aio_finish_error(aio, rv); -} - -static void -resp0_ctx_send(void *arg, nni_aio *aio) -{ - resp0_ctx * ctx = arg; - resp0_sock *s = ctx->sock; - resp0_pipe *p; - nni_msg * msg; - size_t len; - uint32_t pid; - int rv; - - if (nni_aio_begin(aio) != 0) { - return; - } - msg = nni_aio_get_msg(aio); - nni_msg_header_clear(msg); - - if (ctx == &s->ctx) { - // We can't send anymore, because only one send per request. - nni_pollable_clear(&s->writable); - } - - nni_mtx_lock(&s->mtx); - if ((rv = nni_aio_schedule(aio, resp0_ctx_cancel_send, ctx)) != 0) { - nni_mtx_unlock(&s->mtx); - nni_aio_finish_error(aio, rv); - return; - } - - if ((len = ctx->btrace_len) == 0) { - nni_mtx_unlock(&s->mtx); - nni_aio_finish_error(aio, NNG_ESTATE); - return; - } - pid = ctx->pipe_id; - ctx->pipe_id = 0; - ctx->btrace_len = 0; - - if ((rv = nni_msg_header_append(msg, ctx->btrace, len)) != 0) { - nni_mtx_unlock(&s->mtx); - nni_aio_finish_error(aio, rv); - return; - } - - if ((p = nni_id_get(&s->pipes, pid)) == NULL) { - // Surveyor has left the building. Just discard the reply. - nni_mtx_unlock(&s->mtx); - nni_aio_set_msg(aio, NULL); - nni_aio_finish(aio, 0, nni_msg_len(msg)); - nni_msg_free(msg); - return; - } - - if (!p->busy) { - p->busy = true; - len = nni_msg_len(msg); - nni_aio_set_msg(&p->aio_send, msg); - nni_pipe_send(p->npipe, &p->aio_send); - nni_mtx_unlock(&s->mtx); - - nni_aio_set_msg(aio, NULL); - nni_aio_finish(aio, 0, len); - return; - } - - ctx->saio = aio; - ctx->spipe = p; - nni_list_append(&p->sendq, ctx); - nni_mtx_unlock(&s->mtx); -} - -static void -resp0_sock_fini(void *arg) -{ - resp0_sock *s = arg; - - nni_id_map_fini(&s->pipes); - resp0_ctx_fini(&s->ctx); - nni_pollable_fini(&s->writable); - nni_pollable_fini(&s->readable); - nni_mtx_fini(&s->mtx); -} - -static int -resp0_sock_init(void *arg, nni_sock *nsock) -{ - resp0_sock *s = arg; - - NNI_ARG_UNUSED(nsock); - - nni_mtx_init(&s->mtx); - nni_id_map_init(&s->pipes, 0, 0, false); - - NNI_LIST_INIT(&s->recvq, resp0_ctx, rqnode); - NNI_LIST_INIT(&s->recvpipes, resp0_pipe, rnode); - - nni_atomic_init(&s->ttl); - nni_atomic_set(&s->ttl, 8); // Per RFC - - (void) resp0_ctx_init(&s->ctx, s); - - // We start off without being either readable or writable. - // Readability comes when there is something on the socket. - nni_pollable_init(&s->writable); - nni_pollable_init(&s->readable); - return (0); -} - -static void -resp0_sock_open(void *arg) -{ - NNI_ARG_UNUSED(arg); -} - -static void -resp0_sock_close(void *arg) -{ - resp0_sock *s = arg; - - resp0_ctx_close(&s->ctx); -} - -static void -resp0_pipe_stop(void *arg) -{ - resp0_pipe *p = arg; - - nni_aio_stop(&p->aio_send); - nni_aio_stop(&p->aio_recv); -} - -static void -resp0_pipe_fini(void *arg) -{ - resp0_pipe *p = arg; - nng_msg * msg; - - if ((msg = nni_aio_get_msg(&p->aio_recv)) != NULL) { - nni_aio_set_msg(&p->aio_recv, NULL); - nni_msg_free(msg); - } - nni_aio_fini(&p->aio_send); - nni_aio_fini(&p->aio_recv); -} - -static int -resp0_pipe_init(void *arg, nni_pipe *npipe, void *s) -{ - resp0_pipe *p = arg; - - nni_aio_init(&p->aio_recv, resp0_pipe_recv_cb, p); - nni_aio_init(&p->aio_send, resp0_pipe_send_cb, p); - - NNI_LIST_INIT(&p->sendq, resp0_ctx, sqnode); - - p->npipe = npipe; - p->psock = s; - p->busy = false; - p->id = nni_pipe_id(npipe); - - return (0); -} - -static int -resp0_pipe_start(void *arg) -{ - resp0_pipe *p = arg; - resp0_sock *s = p->psock; - int rv; - - if (nni_pipe_peer(p->npipe) != NNI_PROTO_SURVEYOR_V0) { - return (NNG_EPROTO); - } - - nni_mtx_lock(&s->mtx); - rv = nni_id_set(&s->pipes, p->id, p); - nni_mtx_unlock(&s->mtx); - if (rv != 0) { - return (rv); - } - - nni_pipe_recv(p->npipe, &p->aio_recv); - return (rv); -} - -static void -resp0_pipe_close(void *arg) -{ - resp0_pipe *p = arg; - resp0_sock *s = p->psock; - resp0_ctx * ctx; - - nni_aio_close(&p->aio_send); - nni_aio_close(&p->aio_recv); - - nni_mtx_lock(&s->mtx); - p->closed = true; - while ((ctx = nni_list_first(&p->sendq)) != NULL) { - nni_aio *aio; - nni_msg *msg; - nni_list_remove(&p->sendq, ctx); - aio = ctx->saio; - ctx->saio = NULL; - msg = nni_aio_get_msg(aio); - nni_aio_set_msg(aio, NULL); - nni_aio_finish(aio, 0, nni_msg_len(msg)); - nni_msg_free(msg); - } - if (p->id == s->ctx.pipe_id) { - // Make sure user space knows they can send a message to us, - // which we will happily discard. - nni_pollable_raise(&s->writable); - } - nni_id_remove(&s->pipes, p->id); - nni_mtx_unlock(&s->mtx); -} - -static void -resp0_pipe_send_cb(void *arg) -{ - resp0_pipe *p = arg; - resp0_sock *s = p->psock; - resp0_ctx * ctx; - nni_aio * aio; - nni_msg * msg; - size_t len; - - if (nni_aio_result(&p->aio_send) != 0) { - nni_msg_free(nni_aio_get_msg(&p->aio_send)); - nni_aio_set_msg(&p->aio_send, NULL); - nni_pipe_close(p->npipe); - return; - } - nni_mtx_lock(&s->mtx); - p->busy = false; - if ((ctx = nni_list_first(&p->sendq)) == NULL) { - // Nothing else to send. - if (p->id == s->ctx.pipe_id) { - // Mark us ready for the other side to send! - nni_pollable_raise(&s->writable); - } - nni_mtx_unlock(&s->mtx); - return; - } - - nni_list_remove(&p->sendq, ctx); - aio = ctx->saio; - ctx->saio = NULL; - ctx->spipe = NULL; - p->busy = true; - msg = nni_aio_get_msg(aio); - len = nni_msg_len(msg); - nni_aio_set_msg(aio, NULL); - nni_aio_set_msg(&p->aio_send, msg); - nni_pipe_send(p->npipe, &p->aio_send); - - nni_mtx_unlock(&s->mtx); - - nni_aio_finish_sync(aio, 0, len); -} - -static void -resp0_cancel_recv(nni_aio *aio, void *arg, int rv) -{ - resp0_ctx * ctx = arg; - resp0_sock *s = ctx->sock; - - nni_mtx_lock(&s->mtx); - if (ctx->raio == aio) { - nni_list_remove(&s->recvq, ctx); - ctx->raio = NULL; - nni_aio_finish_error(aio, rv); - } - nni_mtx_unlock(&s->mtx); -} - -static void -resp0_ctx_recv(void *arg, nni_aio *aio) -{ - resp0_ctx * ctx = arg; - resp0_sock *s = ctx->sock; - resp0_pipe *p; - size_t len; - nni_msg * msg; - - if (nni_aio_begin(aio) != 0) { - return; - } - nni_mtx_lock(&s->mtx); - if ((p = nni_list_first(&s->recvpipes)) == NULL) { - int rv; - rv = nni_aio_schedule(aio, resp0_cancel_recv, ctx); - if (rv != 0) { - nni_mtx_unlock(&s->mtx); - nni_aio_finish_error(aio, rv); - return; - } - // We cannot have two concurrent receive requests on the same - // context... - if (ctx->raio != NULL) { - nni_mtx_unlock(&s->mtx); - nni_aio_finish_error(aio, NNG_ESTATE); - return; - } - ctx->raio = aio; - nni_list_append(&s->recvq, ctx); - nni_mtx_unlock(&s->mtx); - return; - } - msg = nni_aio_get_msg(&p->aio_recv); - nni_aio_set_msg(&p->aio_recv, NULL); - nni_list_remove(&s->recvpipes, p); - if (nni_list_empty(&s->recvpipes)) { - nni_pollable_clear(&s->readable); - } - nni_pipe_recv(p->npipe, &p->aio_recv); - - len = nni_msg_header_len(msg); - memcpy(ctx->btrace, nni_msg_header(msg), len); - ctx->btrace_len = len; - ctx->pipe_id = p->id; - if (ctx == &s->ctx) { - nni_pollable_raise(&s->writable); - } - nni_mtx_unlock(&s->mtx); - - nni_msg_header_clear(msg); - nni_aio_set_msg(aio, msg); - nni_aio_finish(aio, 0, nni_msg_len(msg)); -} - -static void -resp0_pipe_recv_cb(void *arg) -{ - resp0_pipe *p = arg; - resp0_sock *s = p->psock; - resp0_ctx * ctx; - nni_msg * msg; - nni_aio * aio; - int hops; - size_t len; - int ttl; - - if (nni_aio_result(&p->aio_recv) != 0) { - nni_pipe_close(p->npipe); - return; - } - - ttl = nni_atomic_get(&s->ttl); - msg = nni_aio_get_msg(&p->aio_recv); - nni_msg_set_pipe(msg, p->id); - - // Move backtrace from body to header - hops = 1; - for (;;) { - bool end = 0; - uint8_t *body; - - if (hops > ttl) { - goto drop; - } - hops++; - if (nni_msg_len(msg) < 4) { - // Peer is speaking garbage, kick it. - nni_msg_free(msg); - nni_aio_set_msg(&p->aio_recv, NULL); - nni_pipe_close(p->npipe); - return; - } - body = nni_msg_body(msg); - end = ((body[0] & 0x80u) != 0); - if (nni_msg_header_append(msg, body, 4) != 0) { - goto drop; - } - nni_msg_trim(msg, 4); - if (end) { - break; - } - } - - len = nni_msg_header_len(msg); - - nni_mtx_lock(&s->mtx); - - if (p->closed) { - // If pipe was closed, we just abandon the data from it. - nni_aio_set_msg(&p->aio_recv, NULL); - nni_mtx_unlock(&s->mtx); - nni_msg_free(msg); - return; - } - if ((ctx = nni_list_first(&s->recvq)) == NULL) { - // No one blocked in recv, stall. - nni_list_append(&s->recvpipes, p); - nni_pollable_raise(&s->readable); - nni_mtx_unlock(&s->mtx); - return; - } - - nni_list_remove(&s->recvq, ctx); - aio = ctx->raio; - ctx->raio = NULL; - nni_aio_set_msg(&p->aio_recv, NULL); - - // Start the next receive. - nni_pipe_recv(p->npipe, &p->aio_recv); - - ctx->btrace_len = len; - memcpy(ctx->btrace, nni_msg_header(msg), len); - nni_msg_header_clear(msg); - ctx->pipe_id = p->id; - - if ((ctx == &s->ctx) && (!p->busy)) { - nni_pollable_raise(&s->writable); - } - nni_mtx_unlock(&s->mtx); - - nni_aio_set_msg(aio, msg); - nni_aio_finish_sync(aio, 0, nni_msg_len(msg)); - return; - -drop: - nni_msg_free(msg); - nni_aio_set_msg(&p->aio_recv, NULL); - nni_pipe_recv(p->npipe, &p->aio_recv); -} - -static int -resp0_sock_set_max_ttl(void *arg, const void *buf, size_t sz, nni_opt_type t) -{ - resp0_sock *s = arg; - int ttl; - int rv; - - if ((rv = nni_copyin_int(&ttl, buf, sz, 1, NNI_MAX_MAX_TTL, t)) == 0) { - nni_atomic_set(&s->ttl, ttl); - } - return (rv); -} - -static int -resp0_sock_get_max_ttl(void *arg, void *buf, size_t *szp, nni_opt_type t) -{ - resp0_sock *s = arg; - return (nni_copyout_int(nni_atomic_get(&s->ttl), buf, szp, t)); -} - -static int -resp0_sock_get_sendfd(void *arg, void *buf, size_t *szp, nni_opt_type t) -{ - resp0_sock *s = arg; - int rv; - int fd; - - if ((rv = nni_pollable_getfd(&s->writable, &fd)) != 0) { - return (rv); - } - return (nni_copyout_int(fd, buf, szp, t)); -} - -static int -resp0_sock_get_recvfd(void *arg, void *buf, size_t *szp, nni_opt_type t) -{ - resp0_sock *s = arg; - int rv; - int fd; - - if ((rv = nni_pollable_getfd(&s->readable, &fd)) != 0) { - return (rv); - } - return (nni_copyout_int(fd, buf, szp, t)); -} - -static void -resp0_sock_send(void *arg, nni_aio *aio) -{ - resp0_sock *s = arg; - - resp0_ctx_send(&s->ctx, aio); -} - -static void -resp0_sock_recv(void *arg, nni_aio *aio) -{ - resp0_sock *s = arg; - - resp0_ctx_recv(&s->ctx, aio); -} - -static nni_proto_pipe_ops resp0_pipe_ops = { - .pipe_size = sizeof(resp0_pipe), - .pipe_init = resp0_pipe_init, - .pipe_fini = resp0_pipe_fini, - .pipe_start = resp0_pipe_start, - .pipe_close = resp0_pipe_close, - .pipe_stop = resp0_pipe_stop, -}; - -static nni_proto_ctx_ops resp0_ctx_ops = { - .ctx_size = sizeof(resp0_ctx), - .ctx_init = resp0_ctx_init, - .ctx_fini = resp0_ctx_fini, - .ctx_send = resp0_ctx_send, - .ctx_recv = resp0_ctx_recv, -}; - -static nni_option resp0_sock_options[] = { - { - .o_name = NNG_OPT_MAXTTL, - .o_get = resp0_sock_get_max_ttl, - .o_set = resp0_sock_set_max_ttl, - }, - { - .o_name = NNG_OPT_RECVFD, - .o_get = resp0_sock_get_recvfd, - .o_set = NULL, - }, - { - .o_name = NNG_OPT_SENDFD, - .o_get = resp0_sock_get_sendfd, - .o_set = NULL, - }, - // terminate list - { - .o_name = NULL, - }, -}; - -static nni_proto_sock_ops resp0_sock_ops = { - .sock_size = sizeof(resp0_sock), - .sock_init = resp0_sock_init, - .sock_fini = resp0_sock_fini, - .sock_open = resp0_sock_open, - .sock_close = resp0_sock_close, - .sock_send = resp0_sock_send, - .sock_recv = resp0_sock_recv, - .sock_options = resp0_sock_options, -}; - -static nni_proto resp0_proto = { - .proto_version = NNI_PROTOCOL_VERSION, - .proto_self = { NNI_PROTO_RESPONDENT_V0, "respondent" }, - .proto_peer = { NNI_PROTO_SURVEYOR_V0, "surveyor" }, - .proto_flags = NNI_PROTO_FLAG_SNDRCV, - .proto_sock_ops = &resp0_sock_ops, - .proto_pipe_ops = &resp0_pipe_ops, - .proto_ctx_ops = &resp0_ctx_ops, -}; - -int -nng_respondent0_open(nng_socket *sidp) -{ - return (nni_proto_open(sidp, &resp0_proto)); -} diff --git a/src/protocol/survey0/respond_test.c b/src/protocol/survey0/respond_test.c deleted file mode 100644 index 51844c76..00000000 --- a/src/protocol/survey0/respond_test.c +++ /dev/null @@ -1,586 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - -#include - -void -test_resp_identity(void) -{ - nng_socket s; - int p; - char * n; - - NUTS_PASS(nng_respondent0_open(&s)); - NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PROTO, &p)); - NUTS_TRUE(p == NNG_RESPONDENT0_SELF); - NUTS_TRUE(nng_socket_get_int(s, NNG_OPT_PEER, &p) == 0); - NUTS_TRUE(p == NNG_RESPONDENT0_PEER); - NUTS_TRUE(nng_socket_get_string(s, NNG_OPT_PROTONAME, &n) == 0); - NUTS_MATCH(n, NNG_RESPONDENT0_SELF_NAME); - nng_strfree(n); - NUTS_TRUE(nng_socket_get_string(s, NNG_OPT_PEERNAME, &n) == 0); - NUTS_MATCH(n, NNG_RESPONDENT0_PEER_NAME); - nng_strfree(n); - NUTS_CLOSE(s); -} - -void -test_resp_send_bad_state(void) -{ - nng_socket resp; - nng_msg * msg = NULL; - - NUTS_PASS(nng_respondent0_open(&resp)); - NUTS_PASS(nng_msg_alloc(&msg, 0)); - NUTS_FAIL(nng_sendmsg(resp, msg, 0), NNG_ESTATE); - nng_msg_free(msg); - NUTS_CLOSE(resp); -} - -void -test_resp_poll_writeable(void) -{ - int fd; - nng_socket surv; - nng_socket resp; - - NUTS_PASS(nng_surveyor0_open(&surv)); - NUTS_PASS(nng_respondent0_open(&resp)); - NUTS_PASS(nng_socket_get_int(resp, NNG_OPT_SENDFD, &fd)); - NUTS_TRUE(fd >= 0); - - // Not writable before connect. - NUTS_TRUE(nuts_poll_fd(fd) == false); - - NUTS_MARRY(surv, resp); - - // Still not writable. - NUTS_TRUE(nuts_poll_fd(fd) == false); - - // If we get a job, *then* we become writable - NUTS_SEND(surv, "abc"); - NUTS_RECV(resp, "abc"); - NUTS_TRUE(nuts_poll_fd(fd) == true); - - // And is no longer writable once we send a message - NUTS_SEND(resp, "def"); - NUTS_TRUE(nuts_poll_fd(fd) == false); - // Even after receiving it - NUTS_RECV(surv, "def"); - NUTS_TRUE(nuts_poll_fd(fd) == false); - - NUTS_CLOSE(surv); - NUTS_CLOSE(resp); -} - -void -test_resp_poll_readable(void) -{ - int fd; - nng_socket surv; - nng_socket resp; - nng_msg * msg; - - NUTS_PASS(nng_surveyor0_open(&surv)); - NUTS_PASS(nng_respondent0_open(&resp)); - NUTS_PASS(nng_socket_get_int(resp, NNG_OPT_RECVFD, &fd)); - NUTS_TRUE(fd >= 0); - - // Not readable if not connected! - NUTS_TRUE(nuts_poll_fd(fd) == false); - - // Even after connect (no message yet) - NUTS_MARRY(surv, resp); - NUTS_TRUE(nuts_poll_fd(fd) == false); - - // But once we send messages, it is. - // We have to send a request, in order to send a reply. - NUTS_SEND(surv, "abc"); - NUTS_SLEEP(100); - - NUTS_TRUE(nuts_poll_fd(fd) == true); - - // and receiving makes it no longer ready - NUTS_PASS(nng_recvmsg(resp, &msg, 0)); - nng_msg_free(msg); - NUTS_TRUE(nuts_poll_fd(fd) == false); - - // TODO verify unsolicited response - - NUTS_CLOSE(surv); - NUTS_CLOSE(resp); -} - -void -test_resp_context_no_poll(void) -{ - int fd; - nng_socket resp; - nng_ctx ctx; - - NUTS_PASS(nng_respondent0_open(&resp)); - NUTS_PASS(nng_ctx_open(&ctx, resp)); - NUTS_FAIL(nng_ctx_get_int(ctx, NNG_OPT_SENDFD, &fd), NNG_ENOTSUP); - NUTS_FAIL(nng_ctx_get_int(ctx, NNG_OPT_RECVFD, &fd), NNG_ENOTSUP); - NUTS_PASS(nng_ctx_close(ctx)); - NUTS_CLOSE(resp); -} - -void -test_resp_validate_peer(void) -{ - nng_socket s1, s2; - nng_stat * stats; - nng_stat * reject; - char * addr; - - NUTS_ADDR(addr, "inproc"); - - NUTS_PASS(nng_respondent0_open(&s1)); - NUTS_PASS(nng_respondent0_open(&s2)); - - NUTS_PASS(nng_listen(s1, addr, NULL, 0)); - NUTS_PASS(nng_dial(s2, addr, NULL, NNG_FLAG_NONBLOCK)); - - NUTS_SLEEP(100); - NUTS_PASS(nng_stats_get(&stats)); - - NUTS_TRUE(stats != NULL); - NUTS_TRUE((reject = nng_stat_find_socket(stats, s1)) != NULL); - NUTS_TRUE((reject = nng_stat_find(reject, "reject")) != NULL); - - NUTS_TRUE(nng_stat_type(reject) == NNG_STAT_COUNTER); - NUTS_TRUE(nng_stat_value(reject) > 0); - - NUTS_CLOSE(s1); - NUTS_CLOSE(s2); - nng_stats_free(stats); -} - -void -test_resp_double_recv(void) -{ - nng_socket s1; - nng_aio * aio1; - nng_aio * aio2; - - NUTS_PASS(nng_respondent0_open(&s1)); - NUTS_PASS(nng_aio_alloc(&aio1, NULL, NULL)); - NUTS_PASS(nng_aio_alloc(&aio2, NULL, NULL)); - - nng_recv_aio(s1, aio1); - nng_recv_aio(s1, aio2); - - nng_aio_wait(aio2); - NUTS_FAIL(nng_aio_result(aio2), NNG_ESTATE); - NUTS_CLOSE(s1); - NUTS_FAIL(nng_aio_result(aio1), NNG_ECLOSED); - nng_aio_free(aio1); - nng_aio_free(aio2); -} - -void -test_resp_close_pipe_before_send(void) -{ - nng_socket resp; - nng_socket surv; - nng_pipe p; - nng_aio * aio1; - nng_msg * m; - - NUTS_PASS(nng_respondent0_open(&resp)); - NUTS_PASS(nng_surveyor0_open(&surv)); - NUTS_PASS(nng_setopt_ms(resp, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_setopt_ms(resp, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_setopt_ms(surv, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_aio_alloc(&aio1, NULL, NULL)); - - NUTS_MARRY(surv, resp); - NUTS_SEND(surv, "test"); - - nng_recv_aio(resp, aio1); - nng_aio_wait(aio1); - NUTS_PASS(nng_aio_result(aio1)); - NUTS_TRUE((m = nng_aio_get_msg(aio1)) != NULL); - p = nng_msg_get_pipe(m); - NUTS_PASS(nng_pipe_close(p)); - NUTS_PASS(nng_sendmsg(resp, m, 0)); - - NUTS_CLOSE(surv); - NUTS_CLOSE(resp); - nng_aio_free(aio1); -} - -void -test_resp_close_pipe_during_send(void) -{ - nng_socket resp; - nng_socket surv; - nng_pipe p = NNG_PIPE_INITIALIZER; - nng_msg * m; - - NUTS_PASS(nng_respondent0_open(&resp)); - NUTS_PASS(nng_surveyor0_open_raw(&surv)); - NUTS_PASS(nng_setopt_ms(resp, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_setopt_ms(resp, NNG_OPT_SENDTIMEO, 200)); - NUTS_PASS(nng_setopt_ms(surv, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_setopt_int(resp, NNG_OPT_SENDBUF, 20)); - NUTS_PASS(nng_setopt_int(resp, NNG_OPT_RECVBUF, 20)); - NUTS_PASS(nng_setopt_int(surv, NNG_OPT_SENDBUF, 20)); - NUTS_PASS(nng_setopt_int(surv, NNG_OPT_RECVBUF, 1)); - - NUTS_MARRY(surv, resp); - - for (int i = 0; i < 100; i++) { - int rv; - NUTS_PASS(nng_msg_alloc(&m, 4)); - NUTS_PASS(nng_msg_append_u32(m, (unsigned) i | 0x80000000u)); - NUTS_PASS(nng_sendmsg(surv, m, 0)); - NUTS_PASS(nng_recvmsg(resp, &m, 0)); - p = nng_msg_get_pipe(m); - rv = nng_sendmsg(resp, m, 0); - if (rv == NNG_ETIMEDOUT) { - // Queue is backed up, senders are busy. - nng_msg_free(m); - break; - } - NUTS_PASS(rv); - } - NUTS_PASS(nng_pipe_close(p)); - - NUTS_CLOSE(surv); - NUTS_CLOSE(resp); -} - -void -test_resp_ctx_recv_aio_stopped(void) -{ - nng_socket resp; - nng_ctx ctx; - nng_aio * aio; - - NUTS_PASS(nng_respondent0_open(&resp)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - NUTS_PASS(nng_ctx_open(&ctx, resp)); - - nng_aio_stop(aio); - nng_ctx_recv(ctx, aio); - nng_aio_wait(aio); - NUTS_FAIL(nng_aio_result(aio), NNG_ECANCELED); - NUTS_PASS(nng_ctx_close(ctx)); - NUTS_CLOSE(resp); - nng_aio_free(aio); -} - -void -test_resp_close_pipe_context_send(void) -{ - nng_socket resp; - nng_socket surv; - nng_pipe p = NNG_PIPE_INITIALIZER; - nng_msg * m; - nng_ctx ctx[10]; - nng_aio * aio[10]; - int i; - - NUTS_PASS(nng_respondent0_open(&resp)); - NUTS_PASS(nng_surveyor0_open_raw(&surv)); - NUTS_PASS(nng_setopt_ms(resp, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_setopt_ms(resp, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_setopt_ms(surv, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_setopt_int(resp, NNG_OPT_SENDBUF, 1)); - NUTS_PASS(nng_setopt_int(resp, NNG_OPT_RECVBUF, 1)); - NUTS_PASS(nng_setopt_int(surv, NNG_OPT_SENDBUF, 1)); - NUTS_PASS(nng_setopt_int(surv, NNG_OPT_RECVBUF, 1)); - for (i = 0; i < 10; i++) { - NUTS_PASS(nng_ctx_open(&ctx[i], resp)); - NUTS_PASS(nng_aio_alloc(&aio[i], NULL, NULL)); - } - - NUTS_MARRY(surv, resp); - - for (i = 0; i < 10; i++) { - NUTS_PASS(nng_msg_alloc(&m, 4)); - NUTS_PASS(nng_msg_append_u32(m, (unsigned) i | 0x80000000u)); - NUTS_PASS(nng_sendmsg(surv, m, 0)); - nng_ctx_recv(ctx[i], aio[i]); - } - for (i = 0; i < 10; i++) { - nng_aio_wait(aio[i]); - NUTS_PASS(nng_aio_result(aio[i])); - NUTS_TRUE((m = nng_aio_get_msg(aio[i])) != NULL); - p = nng_msg_get_pipe(m); - nng_aio_set_msg(aio[i], m); - nng_ctx_send(ctx[i], aio[i]); - } - - // Note that SURVEYOR socket is not reading the results. - NUTS_PASS(nng_pipe_close(p)); - - for (i = 0; i < 10; i++) { - int rv; - nng_aio_wait(aio[i]); - rv = nng_aio_result(aio[i]); - if (rv != 0) { - NUTS_FAIL(rv, NNG_ECLOSED); - nng_msg_free(nng_aio_get_msg(aio[i])); - } - nng_aio_free(aio[i]); - NUTS_PASS(nng_ctx_close(ctx[i])); - } - NUTS_CLOSE(surv); - NUTS_CLOSE(resp); -} - -void -test_resp_close_context_send(void) -{ - nng_socket resp; - nng_socket surv; - nng_msg * m; - nng_ctx ctx[10]; - nng_aio * aio[10]; - int i; - - NUTS_PASS(nng_respondent0_open(&resp)); - NUTS_PASS(nng_surveyor0_open_raw(&surv)); - NUTS_PASS(nng_setopt_ms(resp, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_setopt_ms(resp, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_setopt_ms(surv, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_setopt_int(resp, NNG_OPT_SENDBUF, 1)); - NUTS_PASS(nng_setopt_int(resp, NNG_OPT_RECVBUF, 1)); - NUTS_PASS(nng_setopt_int(surv, NNG_OPT_SENDBUF, 1)); - NUTS_PASS(nng_setopt_int(surv, NNG_OPT_RECVBUF, 1)); - for (i = 0; i < 10; i++) { - NUTS_PASS(nng_ctx_open(&ctx[i], resp)); - NUTS_PASS(nng_aio_alloc(&aio[i], NULL, NULL)); - } - - NUTS_MARRY(surv, resp); - - for (i = 0; i < 10; i++) { - NUTS_PASS(nng_msg_alloc(&m, 4)); - NUTS_PASS(nng_msg_append_u32(m, (unsigned) i | 0x80000000u)); - NUTS_PASS(nng_sendmsg(surv, m, 0)); - nng_ctx_recv(ctx[i], aio[i]); - } - for (i = 0; i < 10; i++) { - nng_aio_wait(aio[i]); - NUTS_PASS(nng_aio_result(aio[i])); - NUTS_TRUE((m = nng_aio_get_msg(aio[i])) != NULL); - nng_aio_set_msg(aio[i], m); - nng_ctx_send(ctx[i], aio[i]); - } - - // Note that REQ socket is not reading the results. - for (i = 0; i < 10; i++) { - int rv; - NUTS_PASS(nng_ctx_close(ctx[i])); - nng_aio_wait(aio[i]); - rv = nng_aio_result(aio[i]); - if (rv != 0) { - NUTS_FAIL(rv, NNG_ECLOSED); - nng_msg_free(nng_aio_get_msg(aio[i])); - } - nng_aio_free(aio[i]); - } - NUTS_CLOSE(surv); - NUTS_CLOSE(resp); -} - -static void -test_resp_ctx_recv_nonblock(void) -{ - nng_socket resp; - nng_ctx ctx; - nng_aio * aio; - - NUTS_PASS(nng_respondent0_open(&resp)); - NUTS_PASS(nng_ctx_open(&ctx, resp)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - - nng_aio_set_timeout(aio, 0); // Instant timeout - nng_ctx_recv(ctx, aio); - - nng_aio_wait(aio); - NUTS_FAIL(nng_aio_result(aio), NNG_ETIMEDOUT); - NUTS_CLOSE(resp); - nng_aio_free(aio); -} - -static void -test_resp_ctx_send_nonblock(void) -{ - nng_socket resp; - nng_socket surv; - nng_ctx ctx; - nng_aio * aio; - nng_msg * msg; - - NUTS_PASS(nng_surveyor0_open(&surv)); - NUTS_PASS(nng_respondent0_open(&resp)); - NUTS_PASS(nng_setopt_ms(surv, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_setopt_ms(resp, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_setopt_ms(resp, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_ctx_open(&ctx, resp)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - NUTS_MARRY(surv, resp); - - NUTS_SEND(surv, "SEND"); - nng_ctx_recv(ctx, aio); - nng_aio_wait(aio); - NUTS_PASS(nng_aio_result(aio)); - // message carries over - msg = nng_aio_get_msg(aio); - nng_aio_set_msg(aio, msg); - nng_aio_set_timeout(aio, 0); // Instant timeout - nng_ctx_send(ctx, aio); - - nng_aio_wait(aio); - NUTS_FAIL(nng_aio_result(aio), NNG_ETIMEDOUT); - NUTS_CLOSE(surv); - NUTS_CLOSE(resp); - nng_aio_free(aio); - nng_msg_free(msg); -} - -void -test_resp_recv_garbage(void) -{ - nng_socket resp; - nng_socket surv; - nng_msg * m; - - NUTS_PASS(nng_respondent0_open(&resp)); - NUTS_PASS(nng_surveyor0_open_raw(&surv)); - NUTS_PASS(nng_setopt_ms(resp, NNG_OPT_RECVTIMEO, 200)); - NUTS_PASS(nng_setopt_ms(resp, NNG_OPT_SENDTIMEO, 200)); - NUTS_PASS(nng_setopt_ms(surv, NNG_OPT_SENDTIMEO, 1000)); - - NUTS_MARRY(surv, resp); - - NUTS_PASS(nng_msg_alloc(&m, 4)); - NUTS_PASS(nng_msg_append_u32(m, 1u)); - NUTS_PASS(nng_sendmsg(surv, m, 0)); - NUTS_FAIL(nng_recvmsg(resp, &m, 0), NNG_ETIMEDOUT); - - NUTS_CLOSE(surv); - NUTS_CLOSE(resp); -} - -static void -test_resp_ttl_option(void) -{ - nng_socket resp; - int v; - bool b; - size_t sz; - const char *opt = NNG_OPT_MAXTTL; - - NUTS_PASS(nng_respondent0_open(&resp)); - - NUTS_PASS(nng_setopt_int(resp, opt, 1)); - NUTS_FAIL(nng_setopt_int(resp, opt, 0), NNG_EINVAL); - NUTS_FAIL(nng_setopt_int(resp, opt, -1), NNG_EINVAL); - NUTS_FAIL(nng_setopt_int(resp, opt, 16), NNG_EINVAL); - NUTS_FAIL(nng_setopt_int(resp, opt, 256), NNG_EINVAL); - NUTS_PASS(nng_setopt_int(resp, opt, 3)); - NUTS_PASS(nng_socket_get_int(resp, opt, &v)); - NUTS_TRUE(v == 3); - v = 0; - sz = sizeof(v); - NUTS_PASS(nng_socket_get(resp, opt, &v, &sz)); - NUTS_TRUE(v == 3); - NUTS_TRUE(sz == sizeof(v)); - - NUTS_FAIL(nng_setopt(resp, opt, "", 1), NNG_EINVAL); - sz = 1; - NUTS_FAIL(nng_socket_get(resp, opt, &v, &sz), NNG_EINVAL); - NUTS_FAIL(nng_setopt_bool(resp, opt, true), NNG_EBADTYPE); - NUTS_FAIL(nng_socket_get_bool(resp, opt, &b), NNG_EBADTYPE); - - NUTS_CLOSE(resp); -} - -static void -test_resp_ttl_drop(void) -{ - nng_socket resp; - nng_socket surv; - nng_msg * m; - - NUTS_PASS(nng_respondent0_open(&resp)); - NUTS_PASS(nng_surveyor0_open_raw(&surv)); - NUTS_PASS(nng_setopt_int(resp, NNG_OPT_MAXTTL, 3)); - NUTS_PASS(nng_setopt_ms(resp, NNG_OPT_RECVTIMEO, 200)); - NUTS_PASS(nng_setopt_ms(surv, NNG_OPT_SENDTIMEO, 1000)); - - NUTS_MARRY(surv, resp); - - // Send messages. Note that xrep implicitly adds a hop on receive. - - NUTS_PASS(nng_msg_alloc(&m, 0)); - NUTS_PASS(nng_msg_append_u32(m, 1u)); // 2 hops - NUTS_PASS(nng_msg_append_u32(m, 0x80000001u)); - NUTS_PASS(nng_msg_append(m, "PASS1", 6)); - NUTS_PASS(nng_sendmsg(surv, m, 0)); - - NUTS_PASS(nng_msg_alloc(&m, 0)); - NUTS_PASS(nng_msg_append_u32(m, 1u)); // 4 hops -- discard! - NUTS_PASS(nng_msg_append_u32(m, 2u)); - NUTS_PASS(nng_msg_append_u32(m, 3u)); - NUTS_PASS(nng_msg_append_u32(m, 0x80000002u)); - NUTS_PASS(nng_msg_append(m, "FAIL2", 6)); - NUTS_PASS(nng_sendmsg(surv, m, 0)); - - NUTS_PASS(nng_msg_alloc(&m, 0)); - NUTS_PASS(nng_msg_append_u32(m, 1u)); // 3 hops - passes - NUTS_PASS(nng_msg_append_u32(m, 2u)); - NUTS_PASS(nng_msg_append_u32(m, 0x80000003u)); - NUTS_PASS(nng_msg_append(m, "PASS3", 6)); - NUTS_PASS(nng_sendmsg(surv, m, 0)); - - NUTS_PASS(nng_msg_alloc(&m, 0)); - NUTS_PASS(nng_msg_append_u32(m, 1u)); // 4 hops -- discard! - NUTS_PASS(nng_msg_append_u32(m, 2u)); - NUTS_PASS(nng_msg_append_u32(m, 3u)); - NUTS_PASS(nng_msg_append_u32(m, 0x80000003u)); - NUTS_PASS(nng_msg_append(m, "FAIL4", 6)); - NUTS_PASS(nng_sendmsg(surv, m, 0)); - - NUTS_RECV(resp, "PASS1"); - NUTS_RECV(resp, "PASS3"); - - NUTS_FAIL(nng_recvmsg(resp, &m, 0), NNG_ETIMEDOUT); - - NUTS_CLOSE(resp); - NUTS_CLOSE(surv); -} - -TEST_LIST = { - { "respond identity", test_resp_identity }, - { "respond send bad state", test_resp_send_bad_state }, - { "respond poll readable", test_resp_poll_readable }, - { "respond poll writable", test_resp_poll_writeable }, - { "respond context does not poll", test_resp_context_no_poll }, - { "respond validate peer", test_resp_validate_peer }, - { "respond double recv", test_resp_double_recv }, - { "respond close pipe before send", test_resp_close_pipe_before_send }, - { "respond close pipe during send", test_resp_close_pipe_during_send }, - { "respond recv aio ctx stopped", test_resp_ctx_recv_aio_stopped }, - { "respond close pipe context send", - test_resp_close_pipe_context_send }, - { "respond close context send", test_resp_close_context_send }, - { "respond context send nonblock", test_resp_ctx_send_nonblock }, - { "respond context recv nonblock", test_resp_ctx_recv_nonblock }, - { "respond recv garbage", test_resp_recv_garbage }, - { "respond ttl option", test_resp_ttl_option }, - { "respond ttl drop", test_resp_ttl_drop }, - { NULL, NULL }, -}; diff --git a/src/protocol/survey0/survey.c b/src/protocol/survey0/survey.c deleted file mode 100644 index ce1ed601..00000000 --- a/src/protocol/survey0/survey.c +++ /dev/null @@ -1,663 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// Copyright 2018 Capitar IT Group BV -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - -#include - -#include "core/nng_impl.h" -#include "nng/protocol/survey0/survey.h" - -// Surveyor protocol. The SURVEYOR protocol is the "survey" side of the -// survey pattern. This is useful for building service discovery, voting, etc. -// Note that this pattern is not optimized for extreme low latency, as it makes -// multiple use of queues for simplicity. Typically this is used in cases -// where a few dozen extra microseconds does not matter. - -typedef struct surv0_pipe surv0_pipe; -typedef struct surv0_sock surv0_sock; -typedef struct surv0_ctx surv0_ctx; - -static void surv0_pipe_send_cb(void *); -static void surv0_pipe_recv_cb(void *); -static void surv0_ctx_timeout(void *); - -struct surv0_ctx { - surv0_sock * sock; - uint32_t survey_id; // survey id - nni_timer_node timer; - nni_time expire; - nni_lmq recv_lmq; - nni_list recv_queue; - nni_atomic_int recv_buf; - nni_atomic_int survey_time; - int err; -}; - -// surv0_sock is our per-socket protocol private structure. -struct surv0_sock { - int ttl; - nni_list pipes; - nni_mtx mtx; - surv0_ctx ctx; - nni_id_map surveys; - nni_pollable writable; - nni_pollable readable; - nni_atomic_int send_buf; -}; - -// surv0_pipe is our per-pipe protocol private structure. -struct surv0_pipe { - nni_pipe * pipe; - surv0_sock * sock; - nni_lmq send_queue; - nni_list_node node; - nni_aio aio_send; - nni_aio aio_recv; - bool busy; - bool closed; -}; - -static void -surv0_ctx_abort(surv0_ctx *ctx, int err) -{ - nni_aio * aio; - surv0_sock *sock = ctx->sock; - - while ((aio = nni_list_first(&ctx->recv_queue)) != NULL) { - nni_list_remove(&ctx->recv_queue, aio); - nni_aio_finish_error(aio, err); - } - nni_lmq_flush(&ctx->recv_lmq); - if (ctx->survey_id != 0) { - nni_id_remove(&sock->surveys, ctx->survey_id); - ctx->survey_id = 0; - } - if (ctx == &sock->ctx) { - nni_pollable_clear(&sock->readable); - } -} - -static void -surv0_ctx_close(surv0_ctx *ctx) -{ - surv0_sock *sock = ctx->sock; - - nni_mtx_lock(&sock->mtx); - surv0_ctx_abort(ctx, NNG_ECLOSED); - nni_mtx_unlock(&sock->mtx); -} - -static void -surv0_ctx_fini(void *arg) -{ - surv0_ctx *ctx = arg; - - surv0_ctx_close(ctx); - nni_timer_cancel(&ctx->timer); - nni_lmq_fini(&ctx->recv_lmq); -} - -static int -surv0_ctx_init(void *c, void *s) -{ - surv0_ctx * ctx = c; - surv0_sock * sock = s; - int rv; - int len; - nng_duration tmo; - - nni_aio_list_init(&ctx->recv_queue); - nni_atomic_init(&ctx->recv_buf); - nni_atomic_init(&ctx->survey_time); - - if (ctx == &sock->ctx) { - len = 128; - tmo = NNI_SECOND; // survey timeout - } else { - len = nni_atomic_get(&sock->ctx.recv_buf); - tmo = nni_atomic_get(&sock->ctx.survey_time); - } - - nni_atomic_set(&ctx->recv_buf, len); - nni_atomic_set(&ctx->survey_time, tmo); - - ctx->sock = sock; - - if ((rv = nni_lmq_init(&ctx->recv_lmq, len)) != 0) { - surv0_ctx_fini(ctx); - return (rv); - } - nni_timer_init(&ctx->timer, surv0_ctx_timeout, ctx); - return (0); -} - -static void -surv0_ctx_cancel(nni_aio *aio, void *arg, int rv) -{ - surv0_ctx * ctx = arg; - surv0_sock *sock = ctx->sock; - nni_mtx_lock(&sock->mtx); - if (nni_list_active(&ctx->recv_queue, aio)) { - nni_list_remove(&ctx->recv_queue, aio); - nni_aio_finish_error(aio, rv); - } - if (ctx->survey_id != 0) { - nni_id_remove(&sock->surveys, ctx->survey_id); - ctx->survey_id = 0; - } - nni_mtx_unlock(&sock->mtx); -} - -static void -surv0_ctx_recv(void *arg, nni_aio *aio) -{ - surv0_ctx * ctx = arg; - surv0_sock *sock = ctx->sock; - nni_msg * msg; - - if (nni_aio_begin(aio) != 0) { - return; - } - - nni_mtx_lock(&sock->mtx); - if (ctx->survey_id == 0) { - nni_mtx_unlock(&sock->mtx); - nni_aio_finish_error(aio, NNG_ESTATE); - return; - } -again: - if (nni_lmq_getq(&ctx->recv_lmq, &msg) != 0) { - int rv; - if ((rv = nni_aio_schedule(aio, &surv0_ctx_cancel, ctx)) != - 0) { - nni_mtx_unlock(&sock->mtx); - nni_aio_finish_error(aio, rv); - return; - } - nni_list_append(&ctx->recv_queue, aio); - nni_mtx_unlock(&sock->mtx); - return; - } - if (nni_lmq_empty(&ctx->recv_lmq) && (ctx == &sock->ctx)) { - nni_pollable_clear(&sock->readable); - } - if ((msg = nni_msg_unique(msg)) == NULL) { - goto again; - } - - nni_mtx_unlock(&sock->mtx); - nni_aio_finish_msg(aio, msg); -} - -void -surv0_ctx_timeout(void *arg) -{ - surv0_ctx * ctx = arg; - surv0_sock *sock = ctx->sock; - - nni_mtx_lock(&sock->mtx); - if (nni_clock() < ctx->expire) { - nni_mtx_unlock(&sock->mtx); - return; - } - - // Abort any pending receives. - surv0_ctx_abort(ctx, NNG_ETIMEDOUT); - nni_mtx_unlock(&sock->mtx); -} - -static void -surv0_ctx_send(void *arg, nni_aio *aio) -{ - surv0_ctx * ctx = arg; - surv0_sock * sock = ctx->sock; - surv0_pipe * pipe; - nni_msg * msg = nni_aio_get_msg(aio); - size_t len = nni_msg_len(msg); - nni_time now = nni_clock(); - nng_duration survey_time; - int rv; - - if (nni_aio_begin(aio) != 0) { - return; - } - - survey_time = nni_atomic_get(&ctx->survey_time); - - nni_mtx_lock(&sock->mtx); - - // Abort everything outstanding. - surv0_ctx_abort(ctx, NNG_ECANCELED); - nni_timer_cancel(&ctx->timer); - - // Allocate the new ID. - if ((rv = nni_id_alloc(&sock->surveys, &ctx->survey_id, ctx)) != 0) { - nni_mtx_unlock(&sock->mtx); - nni_aio_finish_error(aio, rv); - return; - } - nni_msg_header_clear(msg); - nni_msg_header_append_u32(msg, (uint32_t) ctx->survey_id); - - // From this point, we're committed to success. Note that we send - // regardless of whether there are any pipes or not. If no pipes, - // then it just gets discarded. - nni_aio_set_msg(aio, NULL); - NNI_LIST_FOREACH (&sock->pipes, pipe) { - - // if the pipe isn't busy, then send this message direct. - if (!pipe->busy) { - pipe->busy = true; - nni_msg_clone(msg); - nni_aio_set_msg(&pipe->aio_send, msg); - nni_pipe_send(pipe->pipe, &pipe->aio_send); - } else if (!nni_lmq_full(&pipe->send_queue)) { - nni_msg_clone(msg); - nni_lmq_putq(&pipe->send_queue, msg); - } - } - - ctx->expire = now + survey_time; - nni_timer_schedule(&ctx->timer, ctx->expire); - - nni_mtx_unlock(&sock->mtx); - nni_msg_free(msg); - - nni_aio_finish(aio, 0, len); -} - -static void -surv0_sock_fini(void *arg) -{ - surv0_sock *sock = arg; - - surv0_ctx_fini(&sock->ctx); - nni_id_map_fini(&sock->surveys); - nni_pollable_fini(&sock->writable); - nni_pollable_fini(&sock->readable); - nni_mtx_fini(&sock->mtx); -} - -static int -surv0_sock_init(void *arg, nni_sock *s) -{ - surv0_sock *sock = arg; - int rv; - - NNI_ARG_UNUSED(s); - - NNI_LIST_INIT(&sock->pipes, surv0_pipe, node); - nni_mtx_init(&sock->mtx); - nni_pollable_init(&sock->readable); - nni_pollable_init(&sock->writable); - // We are always writable. - nni_pollable_raise(&sock->writable); - - // We allow for some buffering on a per pipe basis, to allow for - // multiple contexts to have surveys outstanding. It is recommended - // to increase this if many contexts will want to publish - // at nearly the same time. - nni_atomic_init(&sock->send_buf); - nni_atomic_set(&sock->send_buf, 8); - - // Survey IDs are 32 bits, with the high order bit set. - // We start at a random point, to minimize likelihood of - // accidental collision across restarts. - nni_id_map_init(&sock->surveys, 0x80000000u, 0xffffffffu, true); - - if ((rv = surv0_ctx_init(&sock->ctx, sock)) != 0) { - surv0_sock_fini(sock); - return (rv); - } - - sock->ttl = 8; - - return (0); -} - -static void -surv0_sock_open(void *arg) -{ - NNI_ARG_UNUSED(arg); -} - -static void -surv0_sock_close(void *arg) -{ - surv0_sock *s = arg; - - surv0_ctx_close(&s->ctx); -} - -static void -surv0_pipe_stop(void *arg) -{ - surv0_pipe *p = arg; - - nni_aio_stop(&p->aio_send); - nni_aio_stop(&p->aio_recv); -} - -static void -surv0_pipe_fini(void *arg) -{ - surv0_pipe *p = arg; - - nni_aio_fini(&p->aio_send); - nni_aio_fini(&p->aio_recv); - nni_lmq_fini(&p->send_queue); -} - -static int -surv0_pipe_init(void *arg, nni_pipe *pipe, void *s) -{ - surv0_pipe *p = arg; - surv0_sock *sock = s; - int rv; - int len; - - len = nni_atomic_get(&sock->send_buf); - nni_aio_init(&p->aio_send, surv0_pipe_send_cb, p); - nni_aio_init(&p->aio_recv, surv0_pipe_recv_cb, p); - - // This depth could be tunable. The deeper the queue, the more - // concurrent surveys that can be delivered (multiple contexts). - // Note that surveys can be *outstanding*, but not yet put on the wire. - if ((rv = nni_lmq_init(&p->send_queue, len)) != 0) { - surv0_pipe_fini(p); - return (rv); - } - - p->pipe = pipe; - p->sock = sock; - return (0); -} - -static int -surv0_pipe_start(void *arg) -{ - surv0_pipe *p = arg; - surv0_sock *s = p->sock; - - if (nni_pipe_peer(p->pipe) != NNG_SURVEYOR0_PEER) { - return (NNG_EPROTO); - } - - nni_mtx_lock(&s->mtx); - nni_list_append(&s->pipes, p); - nni_mtx_unlock(&s->mtx); - - nni_pipe_recv(p->pipe, &p->aio_recv); - return (0); -} - -static void -surv0_pipe_close(void *arg) -{ - surv0_pipe *p = arg; - surv0_sock *s = p->sock; - - nni_aio_close(&p->aio_send); - nni_aio_close(&p->aio_recv); - - nni_mtx_lock(&s->mtx); - p->closed = true; - nni_lmq_flush(&p->send_queue); - if (nni_list_active(&s->pipes, p)) { - nni_list_remove(&s->pipes, p); - } - nni_mtx_unlock(&s->mtx); -} - -static void -surv0_pipe_send_cb(void *arg) -{ - surv0_pipe *p = arg; - surv0_sock *sock = p->sock; - nni_msg * msg; - - if (nni_aio_result(&p->aio_send) != 0) { - nni_msg_free(nni_aio_get_msg(&p->aio_send)); - nni_aio_set_msg(&p->aio_send, NULL); - nni_pipe_close(p->pipe); - return; - } - - nni_mtx_lock(&sock->mtx); - if (p->closed) { - nni_mtx_unlock(&sock->mtx); - return; - } - if (nni_lmq_getq(&p->send_queue, &msg) == 0) { - nni_aio_set_msg(&p->aio_send, msg); - nni_pipe_send(p->pipe, &p->aio_send); - } else { - p->busy = false; - } - nni_mtx_unlock(&sock->mtx); -} - -static void -surv0_pipe_recv_cb(void *arg) -{ - surv0_pipe *p = arg; - surv0_sock *sock = p->sock; - surv0_ctx * ctx; - nni_msg * msg; - uint32_t id; - nni_aio * aio; - - if (nni_aio_result(&p->aio_recv) != 0) { - nni_pipe_close(p->pipe); - return; - } - - msg = nni_aio_get_msg(&p->aio_recv); - nni_aio_set_msg(&p->aio_recv, NULL); - nni_msg_set_pipe(msg, nni_pipe_id(p->pipe)); - - // We yank 4 bytes of body, and move them to the header. - if (nni_msg_len(msg) < 4) { - // Peer sent us garbage. Kick it. - nni_msg_free(msg); - nni_pipe_close(p->pipe); - return; - } - id = nni_msg_trim_u32(msg); - nni_msg_header_append_u32(msg, id); - - nni_mtx_lock(&sock->mtx); - // Best effort at delivery. Discard if no context or context is - // unable to receive it. - if (((ctx = nni_id_get(&sock->surveys, id)) == NULL) || - (nni_lmq_full(&ctx->recv_lmq))) { - nni_msg_free(msg); - } else if ((aio = nni_list_first(&ctx->recv_queue)) != NULL) { - nni_list_remove(&ctx->recv_queue, aio); - nni_aio_finish_msg(aio, msg); - } else { - nni_lmq_putq(&ctx->recv_lmq, msg); - if (ctx == &sock->ctx) { - nni_pollable_raise(&sock->readable); - } - } - nni_mtx_unlock(&sock->mtx); - - nni_pipe_recv(p->pipe, &p->aio_recv); -} - -static int -surv0_ctx_set_survey_time( - void *arg, const void *buf, size_t sz, nni_opt_type t) -{ - surv0_ctx * ctx = arg; - nng_duration expire; - int rv; - if ((rv = nni_copyin_ms(&expire, buf, sz, t)) == 0) { - nni_atomic_set(&ctx->survey_time, expire); - } - return (rv); -} - -static int -surv0_ctx_get_survey_time(void *arg, void *buf, size_t *szp, nni_opt_type t) -{ - surv0_ctx *ctx = arg; - return ( - nni_copyout_ms(nni_atomic_get(&ctx->survey_time), buf, szp, t)); -} - -static int -surv0_sock_set_max_ttl(void *arg, const void *buf, size_t sz, nni_opt_type t) -{ - surv0_sock *s = arg; - return (nni_copyin_int(&s->ttl, buf, sz, 1, NNI_MAX_MAX_TTL, t)); -} - -static int -surv0_sock_get_max_ttl(void *arg, void *buf, size_t *szp, nni_opt_type t) -{ - surv0_sock *s = arg; - return (nni_copyout_int(s->ttl, buf, szp, t)); -} - -static int -surv0_sock_set_survey_time( - void *arg, const void *buf, size_t sz, nni_opt_type t) -{ - surv0_sock *s = arg; - return (surv0_ctx_set_survey_time(&s->ctx, buf, sz, t)); -} - -static int -surv0_sock_get_survey_time(void *arg, void *buf, size_t *szp, nni_opt_type t) -{ - surv0_sock *s = arg; - return (surv0_ctx_get_survey_time(&s->ctx, buf, szp, t)); -} - -static int -surv0_sock_get_send_fd(void *arg, void *buf, size_t *szp, nni_opt_type t) -{ - surv0_sock *sock = arg; - int rv; - int fd; - - if ((rv = nni_pollable_getfd(&sock->writable, &fd)) != 0) { - return (rv); - } - return (nni_copyout_int(fd, buf, szp, t)); -} - -static int -surv0_sock_get_recv_fd(void *arg, void *buf, size_t *szp, nni_opt_type t) -{ - surv0_sock *sock = arg; - int rv; - int fd; - - if ((rv = nni_pollable_getfd(&sock->readable, &fd)) != 0) { - return (rv); - } - return (nni_copyout_int(fd, buf, szp, t)); -} - -static void -surv0_sock_recv(void *arg, nni_aio *aio) -{ - surv0_sock *s = arg; - surv0_ctx_recv(&s->ctx, aio); -} - -static void -surv0_sock_send(void *arg, nni_aio *aio) -{ - surv0_sock *s = arg; - surv0_ctx_send(&s->ctx, aio); -} - -static nni_proto_pipe_ops surv0_pipe_ops = { - .pipe_size = sizeof(surv0_pipe), - .pipe_init = surv0_pipe_init, - .pipe_fini = surv0_pipe_fini, - .pipe_start = surv0_pipe_start, - .pipe_close = surv0_pipe_close, - .pipe_stop = surv0_pipe_stop, -}; - -static nni_option surv0_ctx_options[] = { - { - .o_name = NNG_OPT_SURVEYOR_SURVEYTIME, - .o_get = surv0_ctx_get_survey_time, - .o_set = surv0_ctx_set_survey_time, - }, - { - .o_name = NULL, - } -}; -static nni_proto_ctx_ops surv0_ctx_ops = { - .ctx_size = sizeof(surv0_ctx), - .ctx_init = surv0_ctx_init, - .ctx_fini = surv0_ctx_fini, - .ctx_send = surv0_ctx_send, - .ctx_recv = surv0_ctx_recv, - .ctx_options = surv0_ctx_options, -}; - -static nni_option surv0_sock_options[] = { - { - .o_name = NNG_OPT_SURVEYOR_SURVEYTIME, - .o_get = surv0_sock_get_survey_time, - .o_set = surv0_sock_set_survey_time, - }, - { - .o_name = NNG_OPT_MAXTTL, - .o_get = surv0_sock_get_max_ttl, - .o_set = surv0_sock_set_max_ttl, - }, - { - .o_name = NNG_OPT_RECVFD, - .o_get = surv0_sock_get_recv_fd, - }, - { - .o_name = NNG_OPT_SENDFD, - .o_get = surv0_sock_get_send_fd, - }, - // terminate list - { - .o_name = NULL, - }, -}; - -static nni_proto_sock_ops surv0_sock_ops = { - .sock_size = sizeof(surv0_sock), - .sock_init = surv0_sock_init, - .sock_fini = surv0_sock_fini, - .sock_open = surv0_sock_open, - .sock_close = surv0_sock_close, - .sock_send = surv0_sock_send, - .sock_recv = surv0_sock_recv, - .sock_options = surv0_sock_options, -}; - -static nni_proto surv0_proto = { - .proto_version = NNI_PROTOCOL_VERSION, - .proto_self = { NNG_SURVEYOR0_SELF, NNG_SURVEYOR0_SELF_NAME }, - .proto_peer = { NNG_SURVEYOR0_PEER, NNG_SURVEYOR0_PEER_NAME }, - .proto_flags = NNI_PROTO_FLAG_SNDRCV, - .proto_sock_ops = &surv0_sock_ops, - .proto_pipe_ops = &surv0_pipe_ops, - .proto_ctx_ops = &surv0_ctx_ops, -}; - -int -nng_surveyor0_open(nng_socket *sock) -{ - return (nni_proto_open(sock, &surv0_proto)); -} diff --git a/src/protocol/survey0/survey_test.c b/src/protocol/survey0/survey_test.c deleted file mode 100644 index 95d27adf..00000000 --- a/src/protocol/survey0/survey_test.c +++ /dev/null @@ -1,626 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// Copyright 2018 Capitar IT Group BV -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - -#include - -static void -test_surv_identity(void) -{ - nng_socket s; - int p; - char * n; - - NUTS_PASS(nng_surveyor0_open(&s)); - NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PROTO, &p)); - NUTS_TRUE(p == NNG_SURVEYOR0_SELF); - NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PEER, &p)); - NUTS_TRUE(p == NNG_SURVEYOR0_PEER); // 49 - NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PROTONAME, &n)); - NUTS_MATCH(n, NNG_SURVEYOR0_SELF_NAME); - nng_strfree(n); - NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PEERNAME, &n)); - NUTS_MATCH(n, NNG_SURVEYOR0_PEER_NAME); - nng_strfree(n); - NUTS_CLOSE(s); -} - -static void -test_surv_ttl_option(void) -{ - nng_socket surv; - int v; - bool b; - size_t sz; - const char *opt = NNG_OPT_MAXTTL; - - NUTS_PASS(nng_surveyor0_open(&surv)); - - NUTS_PASS(nng_socket_set_int(surv, opt, 1)); - NUTS_FAIL(nng_socket_set_int(surv, opt, 0), NNG_EINVAL); - NUTS_FAIL(nng_socket_set_int(surv, opt, -1), NNG_EINVAL); - // This test will fail if the NNI_MAX_MAX_TTL is changed from the - // builtin default of 15. - NUTS_FAIL(nng_socket_set_int(surv, opt, 16), NNG_EINVAL); - NUTS_FAIL(nng_socket_set_int(surv, opt, 256), NNG_EINVAL); - NUTS_PASS(nng_socket_set_int(surv, opt, 3)); - NUTS_PASS(nng_socket_get_int(surv, opt, &v)); - NUTS_TRUE(v == 3); - v = 0; - sz = sizeof(v); - NUTS_PASS(nng_socket_get(surv, opt, &v, &sz)); - NUTS_TRUE(v == 3); - NUTS_TRUE(sz == sizeof(v)); - - NUTS_FAIL(nng_socket_set(surv, opt, "", 1), NNG_EINVAL); - sz = 1; - NUTS_FAIL(nng_socket_get(surv, opt, &v, &sz), NNG_EINVAL); - NUTS_FAIL(nng_socket_set_bool(surv, opt, true), NNG_EBADTYPE); - NUTS_FAIL(nng_socket_get_bool(surv, opt, &b), NNG_EBADTYPE); - - NUTS_CLOSE(surv); -} - -static void -test_surv_survey_time_option(void) -{ - nng_socket surv; - nng_duration d; - bool b; - size_t sz = sizeof(b); - const char * opt = NNG_OPT_SURVEYOR_SURVEYTIME; - - NUTS_PASS(nng_surveyor0_open(&surv)); - - NUTS_PASS(nng_socket_set_ms(surv, opt, 10)); - NUTS_FAIL(nng_socket_set(surv, opt, "", 1), NNG_EINVAL); - NUTS_FAIL(nng_socket_get(surv, opt, &b, &sz), NNG_EINVAL); - NUTS_FAIL(nng_socket_set_bool(surv, opt, true), NNG_EBADTYPE); - NUTS_FAIL(nng_socket_get_bool(surv, opt, &b), NNG_EBADTYPE); - - NUTS_PASS(nng_socket_get_ms(surv, opt, &d)); - NUTS_TRUE(d == 10); - NUTS_CLOSE(surv); -} - -void -test_surv_recv_bad_state(void) -{ - nng_socket surv; - nng_msg * msg = NULL; - - NUTS_PASS(nng_surveyor0_open(&surv)); - NUTS_FAIL(nng_recvmsg(surv, &msg, 0), NNG_ESTATE); - NUTS_TRUE(msg == NULL); - NUTS_CLOSE(surv); -} - -static void -test_surv_recv_garbage(void) -{ - nng_socket resp; - nng_socket surv; - nng_msg * m; - uint32_t surv_id; - - NUTS_PASS(nng_respondent0_open_raw(&resp)); - NUTS_PASS(nng_surveyor0_open(&surv)); - NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_RECVTIMEO, 100)); - NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_SENDTIMEO, 1000)); - - NUTS_MARRY(surv, resp); - - NUTS_PASS(nng_msg_alloc(&m, 0)); - NUTS_PASS(nng_sendmsg(surv, m, 0)); - - NUTS_PASS(nng_recvmsg(resp, &m, 0)); - - // The message will have a header that contains the 32-bit pipe ID, - // followed by the 32-bit request ID. We will discard the request - // ID before sending it out. - NUTS_TRUE(nng_msg_header_len(m) == 8); - NUTS_PASS(nng_msg_header_chop_u32(m, &surv_id)); - - NUTS_PASS(nng_sendmsg(resp, m, 0)); - NUTS_FAIL(nng_recvmsg(surv, &m, 0), NNG_ETIMEDOUT); - - NUTS_CLOSE(surv); - NUTS_CLOSE(resp); -} - -#define SECOND 1000 - -void -test_surv_resp_exchange(void) -{ - nng_socket surv; - nng_socket resp; - - NUTS_PASS(nng_surveyor0_open(&surv)); - NUTS_PASS(nng_respondent0_open(&resp)); - - NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_RECVTIMEO, SECOND)); - NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_RECVTIMEO, SECOND)); - NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_SENDTIMEO, SECOND)); - NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_SENDTIMEO, SECOND)); - - NUTS_MARRY(resp, surv); - - NUTS_SEND(surv, "ping"); - NUTS_RECV(resp, "ping"); - NUTS_SEND(resp, "pong"); - NUTS_RECV(surv, "pong"); - - NUTS_CLOSE(surv); - NUTS_CLOSE(resp); -} - -void -test_surv_cancel(void) -{ - nng_socket surv; - nng_socket resp; - - NUTS_PASS(nng_respondent0_open(&resp)); - NUTS_PASS(nng_surveyor0_open(&surv)); - - NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_RECVTIMEO, SECOND)); - NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_RECVTIMEO, SECOND)); - NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_SENDTIMEO, 5 * SECOND)); - NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_SENDTIMEO, 5 * SECOND)); - NUTS_PASS(nng_socket_set_int(surv, NNG_OPT_SENDBUF, 16)); - - NUTS_MARRY(resp, surv); - - // Send req #1 (abc). - NUTS_SEND(surv, "abc"); - - // Sleep a bit. This is so that we ensure that our request gets - // to the far side. (If we cancel too fast, then our outgoing send - // will be canceled before it gets to the peer.) - NUTS_SLEEP(100); - - // Send the next next request ("def"). Note that - // the RESP side server will have already buffered the receive - // request, and should simply be waiting for us to reply to abc. - NUTS_SEND(surv, "def"); - - // Receive the first request (should be abc) on the REP server. - NUTS_RECV(resp, "abc"); - - // RESP sends the reply to first command. This will be discarded - // by the SURV socket. - NUTS_SEND(resp, "abc"); - - // Now get the next command from the REP; should be "def". - NUTS_RECV(resp, "def"); - - // And send it back to REQ. - NUTS_SEND(resp, "def"); - - // Try a req command. This should give back "def" - NUTS_RECV(surv, "def"); - - NUTS_CLOSE(surv); - NUTS_CLOSE(resp); -} - -void -test_surv_cancel_abort_recv(void) -{ - nng_aio * aio; - nng_duration time = SECOND * 10; // 10s (kind of never) - nng_socket surv; - nng_socket resp; - - NUTS_PASS(nng_respondent0_open(&resp)); - NUTS_PASS(nng_surveyor0_open(&surv)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - - NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_SURVEYOR_SURVEYTIME, time)); - NUTS_PASS(nng_socket_set_int(surv, NNG_OPT_SENDBUF, 16)); - NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_RECVTIMEO, 5 * SECOND)); - NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_RECVTIMEO, 5 * SECOND)); - NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_SENDTIMEO, 5 * SECOND)); - NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_SENDTIMEO, 5 * SECOND)); - - NUTS_MARRY(resp, surv); - - // Send survey #1 (abc). - NUTS_SEND(surv, "abc"); - - // Wait for it to get ot the other side. - NUTS_SLEEP(100); - - nng_aio_set_timeout(aio, 5 * SECOND); - nng_recv_aio(surv, aio); - - // Give time for this recv to post properly. - NUTS_SLEEP(100); - - // Send the next next request ("def"). Note that - // the respondent side server will have already buffered the receive - // request, and should simply be waiting for us to reply to - // abc. - NUTS_SEND(surv, "def"); - - // Our pending I/O should have been canceled. - nng_aio_wait(aio); - NUTS_FAIL(nng_aio_result(aio), NNG_ECANCELED); - - // Receive the first request (should be abc) on the respondent. - NUTS_RECV(resp, "abc"); - - // Respondent sends the reply to first survey. This will be - // discarded by the SURV socket. - NUTS_SEND(resp, "abc"); - - // Now get the next survey from the RESP; should be "def". - NUTS_RECV(resp, "def"); - - // And send it back to REQ. - NUTS_SEND(resp, "def"); - - // Try a req command. This should give back "def" - NUTS_RECV(surv, "def"); - - nng_aio_free(aio); - NUTS_CLOSE(surv); - NUTS_CLOSE(resp); -} - -static void -test_surv_cancel_post_recv(void) -{ - nng_socket surv; - nng_socket resp; - - NUTS_PASS(nng_surveyor0_open(&surv)); - NUTS_PASS(nng_respondent0_open(&resp)); - NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_RECVTIMEO, 1000)); - NUTS_MARRY(surv, resp); - - NUTS_SEND(surv, "ONE"); - NUTS_RECV(resp, "ONE"); - NUTS_SEND(resp, "one"); - NUTS_SLEEP(100); // Make sure reply arrives! - NUTS_SEND(surv, "TWO"); - NUTS_RECV(resp, "TWO"); - NUTS_SEND(resp, "two"); - NUTS_RECV(surv, "two"); - - NUTS_CLOSE(surv); - NUTS_CLOSE(resp); -} - -static void -test_surv_poll_writeable(void) -{ - int fd; - nng_socket surv; - nng_socket resp; - - NUTS_PASS(nng_surveyor0_open(&surv)); - NUTS_PASS(nng_respondent0_open(&resp)); - NUTS_PASS(nng_socket_get_int(surv, NNG_OPT_SENDFD, &fd)); - NUTS_TRUE(fd >= 0); - - // Survey is broadcast, so we can always write. - NUTS_TRUE(nuts_poll_fd(fd)); - - NUTS_MARRY(surv, resp); - - // Now it's writable. - NUTS_TRUE(nuts_poll_fd(fd)); - - NUTS_CLOSE(surv); - NUTS_CLOSE(resp); -} - -void -test_surv_poll_readable(void) -{ - int fd; - nng_socket surv; - nng_socket resp; - nng_msg * msg; - - NUTS_PASS(nng_surveyor0_open(&surv)); - NUTS_PASS(nng_respondent0_open(&resp)); - NUTS_PASS(nng_socket_get_int(surv, NNG_OPT_RECVFD, &fd)); - NUTS_TRUE(fd >= 0); - - // Not readable if not connected! - NUTS_TRUE(nuts_poll_fd(fd) == false); - - // Even after connect (no message yet) - NUTS_MARRY(surv, resp); - NUTS_TRUE(nuts_poll_fd(fd) == false); - - // But once we send messages, it is. - // We have to send a request, in order to send a reply. - - NUTS_PASS(nng_msg_alloc(&msg, 0)); - NUTS_PASS(nng_msg_append(msg, "xyz", 3)); - NUTS_PASS(nng_sendmsg(surv, msg, 0)); - NUTS_PASS(nng_recvmsg(resp, &msg, 0)); // recv on rep - NUTS_PASS(nng_sendmsg(resp, msg, 0)); // echo it back - NUTS_SLEEP(200); // give time for message to arrive - - NUTS_TRUE(nuts_poll_fd(fd) == true); - - // and receiving makes it no longer ready - NUTS_PASS(nng_recvmsg(surv, &msg, 0)); - nng_msg_free(msg); - NUTS_TRUE(nuts_poll_fd(fd) == false); - - // TODO verify unsolicited response - - NUTS_CLOSE(surv); - NUTS_CLOSE(resp); -} - -static void -test_surv_ctx_no_poll(void) -{ - int fd; - nng_socket surv; - nng_ctx ctx; - - NUTS_PASS(nng_surveyor0_open(&surv)); - NUTS_PASS(nng_ctx_open(&ctx, surv)); - NUTS_FAIL(nng_ctx_get_int(ctx, NNG_OPT_SENDFD, &fd), NNG_ENOTSUP); - NUTS_FAIL(nng_ctx_get_int(ctx, NNG_OPT_RECVFD, &fd), NNG_ENOTSUP); - NUTS_PASS(nng_ctx_close(ctx)); - NUTS_CLOSE(surv); -} - -static void -test_surv_ctx_recv_nonblock(void) -{ - nng_socket surv; - nng_socket resp; - nng_ctx ctx; - nng_aio * aio; - nng_msg * msg; - - NUTS_PASS(nng_surveyor0_open(&surv)); - NUTS_PASS(nng_respondent0_open(&resp)); - NUTS_PASS(nng_ctx_open(&ctx, surv)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - NUTS_PASS(nng_msg_alloc(&msg, 0)); - - NUTS_MARRY(surv, resp); - - nng_aio_set_msg(aio, msg); - nng_ctx_send(ctx, aio); - nng_aio_wait(aio); - NUTS_PASS(nng_aio_result(aio)); - nng_aio_set_timeout(aio, 0); // Instant timeout - nng_ctx_recv(ctx, aio); - - nng_aio_wait(aio); - NUTS_FAIL(nng_aio_result(aio), NNG_ETIMEDOUT); - NUTS_CLOSE(surv); - NUTS_CLOSE(resp); - nng_aio_free(aio); -} - -static void -test_surv_ctx_send_nonblock(void) -{ - nng_socket surv; - nng_ctx ctx; - nng_aio * aio; - nng_msg * msg; - - NUTS_PASS(nng_surveyor0_open(&surv)); - NUTS_PASS(nng_ctx_open(&ctx, surv)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - NUTS_PASS(nng_msg_alloc(&msg, 0)); - - nng_aio_set_msg(aio, msg); - nng_aio_set_timeout(aio, 0); // Instant timeout - nng_ctx_send(ctx, aio); - nng_aio_wait(aio); - NUTS_PASS(nng_aio_result(aio)); // We never block - NUTS_CLOSE(surv); - nng_aio_free(aio); -} - -static void -test_surv_send_best_effort(void) -{ - nng_socket surv; - nng_socket resp; - - NUTS_PASS(nng_surveyor0_open(&surv)); - NUTS_PASS(nng_respondent0_open(&resp)); - NUTS_MARRY(surv, resp); - - for (int i = 0; i < 200; i++) { - NUTS_SEND(surv, "junk"); - } - - NUTS_CLOSE(surv); - NUTS_CLOSE(resp); -} - -static void -test_surv_survey_timeout(void) -{ - nng_socket surv; - nng_socket resp; - char buf[16]; - size_t sz; - - NUTS_PASS(nng_surveyor0_open(&surv)); - NUTS_PASS(nng_respondent0_open(&resp)); - NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_SURVEYOR_SURVEYTIME, 50)); - NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_RECVTIMEO, 100)); - - NUTS_MARRY(surv, resp); - - NUTS_SEND(surv, "hello"); - NUTS_RECV(resp, "hello"); - - sz = sizeof(buf); - NUTS_FAIL(nng_recv(surv, buf, &sz, 0), NNG_ETIMEDOUT); - NUTS_SEND(resp, "world"); - NUTS_FAIL(nng_recv(surv, buf, &sz, 0), NNG_ESTATE); - - NUTS_CLOSE(surv); - NUTS_CLOSE(resp); -} - -static void -test_surv_ctx_recv_close_socket(void) -{ - nng_socket surv; - nng_socket resp; - nng_ctx ctx; - nng_aio * aio; - nng_msg * m; - - NUTS_PASS(nng_surveyor0_open(&surv)); - NUTS_PASS(nng_respondent0_open(&resp)); - NUTS_PASS(nng_ctx_open(&ctx, surv)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - NUTS_MARRY(surv, resp); - NUTS_PASS(nng_msg_alloc(&m, 0)); - nng_aio_set_msg(aio, m); - nng_ctx_send(ctx, aio); - nng_aio_wait(aio); - NUTS_PASS(nng_aio_result(aio)); - - nng_ctx_recv(ctx, aio); - nng_close(surv); - - NUTS_FAIL(nng_aio_result(aio), NNG_ECLOSED); - nng_aio_free(aio); - NUTS_CLOSE(resp); -} - -static void -test_surv_context_multi(void) -{ - nng_socket surv; - nng_socket resp; - nng_ctx c[5]; - nng_aio * aio; - nng_msg * m; - int cnt = sizeof(c) / sizeof(c[0]); - - NUTS_PASS(nng_surveyor0_open(&surv)); - NUTS_PASS(nng_respondent0_open(&resp)); - NUTS_MARRY(surv, resp); - NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_SURVEYOR_SURVEYTIME, 200)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - - for (int i = 0; i < cnt; i++) { - NUTS_PASS(nng_ctx_open(&c[i], surv)); - } - - for (int i = 0; i < cnt; i++) { - NUTS_PASS(nng_msg_alloc(&m, 0)); - NUTS_PASS(nng_msg_append_u32(m, i)); - nng_aio_set_msg(aio, m); - nng_ctx_send(c[i], aio); - nng_aio_wait(aio); - NUTS_PASS(nng_aio_result(aio)); - } - - for (int i = 0; i < cnt; i++) { - NUTS_PASS(nng_recvmsg(resp, &m, 0)); - NUTS_PASS(nng_sendmsg(resp, m, 0)); - } - - for (int i = cnt - 1; i >= 0; i--) { - uint32_t x; - nng_ctx_recv(c[i], aio); - nng_aio_wait(aio); - NUTS_PASS(nng_aio_result(aio)); - m = nng_aio_get_msg(aio); - TEST_ASSERT(m != NULL); - NUTS_PASS(nng_msg_trim_u32(m, &x)); - NUTS_TRUE(x == (uint32_t) i); - nng_msg_free(m); - } - - for (int i = 0; i < cnt; i++) { - nng_ctx_recv(c[i], aio); - nng_aio_wait(aio); - NUTS_TRUE(nng_aio_result(aio) != 0); - } - for (int i = 0; i < cnt; i++) { - nng_ctx_close(c[i]); - } - NUTS_CLOSE(surv); - NUTS_CLOSE(resp); - nng_aio_free(aio); -} - -static void -test_surv_validate_peer(void) -{ - nng_socket s1, s2; - nng_stat * stats; - nng_stat * reject; - char * addr; - - NUTS_ADDR(addr, "inproc"); - NUTS_PASS(nng_surveyor0_open(&s1)); - NUTS_PASS(nng_surveyor0_open(&s2)); - - NUTS_PASS(nng_listen(s1, addr, NULL, 0)); - NUTS_PASS(nng_dial(s2, addr, NULL, NNG_FLAG_NONBLOCK)); - - NUTS_SLEEP(100); - NUTS_PASS(nng_stats_get(&stats)); - - NUTS_TRUE(stats != NULL); - NUTS_TRUE((reject = nng_stat_find_socket(stats, s1)) != NULL); - NUTS_TRUE((reject = nng_stat_find(reject, "reject")) != NULL); - - NUTS_TRUE(nng_stat_type(reject) == NNG_STAT_COUNTER); - NUTS_TRUE(nng_stat_value(reject) > 0); - - NUTS_PASS(nng_close(s1)); - NUTS_PASS(nng_close(s2)); - nng_stats_free(stats); -} - -TEST_LIST = { - { "survey identity", test_surv_identity }, - { "survey ttl option", test_surv_ttl_option }, - { "survey survey time option", test_surv_survey_time_option }, - { "survey recv bad state", test_surv_recv_bad_state }, - { "survey recv garbage", test_surv_recv_garbage }, - { "survey respondent exchange", test_surv_resp_exchange }, - { "survey cancel", test_surv_cancel }, - { "survey cancel abort recv", test_surv_cancel_abort_recv }, - { "survey cancel post recv", test_surv_cancel_post_recv }, - { "survey poll writable", test_surv_poll_writeable }, - { "survey poll readable", test_surv_poll_readable }, - { "survey context does not poll", test_surv_ctx_no_poll }, - { "survey context recv close socket", - test_surv_ctx_recv_close_socket }, - { "survey context recv nonblock", test_surv_ctx_recv_nonblock }, - { "survey context send nonblock", test_surv_ctx_send_nonblock }, - { "survey timeout", test_surv_survey_timeout }, - { "survey send best effort", test_surv_send_best_effort }, - { "survey context multi", test_surv_context_multi }, - { "survey validate peer", test_surv_validate_peer }, - { NULL, NULL }, -}; diff --git a/src/protocol/survey0/xrespond.c b/src/protocol/survey0/xrespond.c deleted file mode 100644 index b2f203c3..00000000 --- a/src/protocol/survey0/xrespond.c +++ /dev/null @@ -1,417 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// Copyright 2018 Capitar IT Group BV -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - -#include - -#include "core/nng_impl.h" -#include "nng/protocol/survey0/respond.h" - -// Respondent protocol. The RESPONDENT protocol is the "replier" side of -// the surveyor pattern. This is useful for building service discovery, or -// voting algorithms, for example. - -#ifndef NNI_PROTO_SURVEYOR_V0 -#define NNI_PROTO_SURVEYOR_V0 NNI_PROTO(6, 2) -#endif - -#ifndef NNI_PROTO_RESPONDENT_V0 -#define NNI_PROTO_RESPONDENT_V0 NNI_PROTO(6, 3) -#endif - -typedef struct xresp0_pipe xresp0_pipe; -typedef struct xresp0_sock xresp0_sock; - -static void xresp0_recv_cb(void *); -static void xresp0_putq_cb(void *); -static void xresp0_getq_cb(void *); -static void xresp0_send_cb(void *); -static void xresp0_sock_getq_cb(void *); -static void xresp0_pipe_fini(void *); - -// resp0_sock is our per-socket protocol private structure. -struct xresp0_sock { - nni_msgq * urq; - nni_msgq * uwq; - nni_atomic_int ttl; - nni_id_map pipes; - nni_aio aio_getq; - nni_mtx mtx; -}; - -// resp0_pipe is our per-pipe protocol private structure. -struct xresp0_pipe { - nni_pipe * npipe; - xresp0_sock *psock; - uint32_t id; - nni_msgq * sendq; - nni_aio aio_getq; - nni_aio aio_putq; - nni_aio aio_send; - nni_aio aio_recv; -}; - -static void -xresp0_sock_fini(void *arg) -{ - xresp0_sock *s = arg; - - nni_aio_fini(&s->aio_getq); - nni_id_map_fini(&s->pipes); - nni_mtx_fini(&s->mtx); -} - -static int -xresp0_sock_init(void *arg, nni_sock *nsock) -{ - xresp0_sock *s = arg; - - nni_mtx_init(&s->mtx); - nni_atomic_init(&s->ttl); - nni_atomic_set(&s->ttl, 8); // Per RFC - nni_id_map_init(&s->pipes, 0, 0, false); - nni_aio_init(&s->aio_getq, xresp0_sock_getq_cb, s); - - s->urq = nni_sock_recvq(nsock); - s->uwq = nni_sock_sendq(nsock); - - return (0); -} - -static void -xresp0_sock_open(void *arg) -{ - xresp0_sock *s = arg; - - nni_msgq_aio_get(s->uwq, &s->aio_getq); -} - -static void -xresp0_sock_close(void *arg) -{ - xresp0_sock *s = arg; - - nni_aio_close(&s->aio_getq); -} - -static void -xresp0_pipe_stop(void *arg) -{ - xresp0_pipe *p = arg; - - nni_aio_stop(&p->aio_putq); - nni_aio_stop(&p->aio_getq); - nni_aio_stop(&p->aio_send); - nni_aio_stop(&p->aio_recv); -} - -static void -xresp0_pipe_fini(void *arg) -{ - xresp0_pipe *p = arg; - - nni_aio_fini(&p->aio_putq); - nni_aio_fini(&p->aio_getq); - nni_aio_fini(&p->aio_send); - nni_aio_fini(&p->aio_recv); - nni_msgq_fini(p->sendq); -} - -static int -xresp0_pipe_init(void *arg, nni_pipe *npipe, void *s) -{ - xresp0_pipe *p = arg; - int rv; - - nni_aio_init(&p->aio_putq, xresp0_putq_cb, p); - nni_aio_init(&p->aio_recv, xresp0_recv_cb, p); - nni_aio_init(&p->aio_getq, xresp0_getq_cb, p); - nni_aio_init(&p->aio_send, xresp0_send_cb, p); - - if ((rv = nni_msgq_init(&p->sendq, 2)) != 0) { - xresp0_pipe_fini(p); - return (rv); - } - - p->npipe = npipe; - p->psock = s; - return (0); -} - -static int -xresp0_pipe_start(void *arg) -{ - xresp0_pipe *p = arg; - xresp0_sock *s = p->psock; - int rv; - - if (nni_pipe_peer(p->npipe) != NNI_PROTO_SURVEYOR_V0) { - return (NNG_EPROTO); - } - - p->id = nni_pipe_id(p->npipe); - - nni_mtx_lock(&s->mtx); - rv = nni_id_set(&s->pipes, p->id, p); - nni_mtx_unlock(&s->mtx); - if (rv != 0) { - return (rv); - } - - nni_pipe_recv(p->npipe, &p->aio_recv); - nni_msgq_aio_get(p->sendq, &p->aio_getq); - - return (rv); -} - -static void -xresp0_pipe_close(void *arg) -{ - xresp0_pipe *p = arg; - xresp0_sock *s = p->psock; - - nni_aio_close(&p->aio_putq); - nni_aio_close(&p->aio_getq); - nni_aio_close(&p->aio_send); - nni_aio_close(&p->aio_recv); - - nni_msgq_close(p->sendq); - - nni_mtx_lock(&s->mtx); - nni_id_remove(&s->pipes, p->id); - nni_mtx_unlock(&s->mtx); -} - -// resp0_sock_send watches for messages from the upper write queue, -// extracts the destination pipe, and forwards it to the appropriate -// destination pipe via a separate queue. This prevents a single bad -// or slow pipe from gumming up the works for the entire socket.s - -void -xresp0_sock_getq_cb(void *arg) -{ - xresp0_sock *s = arg; - nni_msg * msg; - uint32_t id; - xresp0_pipe *p; - - if (nni_aio_result(&s->aio_getq) != 0) { - return; - } - msg = nni_aio_get_msg(&s->aio_getq); - nni_aio_set_msg(&s->aio_getq, NULL); - - // We yank the outgoing pipe id from the header - if (nni_msg_header_len(msg) < 4) { - nni_msg_free(msg); - // We can't really close down the socket, so just keep going. - nni_msgq_aio_get(s->uwq, &s->aio_getq); - return; - } - id = nni_msg_header_trim_u32(msg); - - nni_mtx_lock(&s->mtx); - // Look for the pipe, and attempt to put the message there - // (nonblocking) if we can. If we can't for any reason, then we - // free the message. - if (((p = nni_id_get(&s->pipes, id)) == NULL) || - (nni_msgq_tryput(p->sendq, msg) != 0)) { - nni_msg_free(msg); - } - nni_mtx_unlock(&s->mtx); - nni_msgq_aio_get(s->uwq, &s->aio_getq); -} - -void -xresp0_getq_cb(void *arg) -{ - xresp0_pipe *p = arg; - - if (nni_aio_result(&p->aio_getq) != 0) { - nni_pipe_close(p->npipe); - return; - } - - nni_aio_set_msg(&p->aio_send, nni_aio_get_msg(&p->aio_getq)); - nni_aio_set_msg(&p->aio_getq, NULL); - - nni_pipe_send(p->npipe, &p->aio_send); -} - -void -xresp0_send_cb(void *arg) -{ - xresp0_pipe *p = arg; - - if (nni_aio_result(&p->aio_send) != 0) { - nni_msg_free(nni_aio_get_msg(&p->aio_send)); - nni_aio_set_msg(&p->aio_send, NULL); - nni_pipe_close(p->npipe); - return; - } - - nni_msgq_aio_get(p->sendq, &p->aio_getq); -} - -static void -xresp0_recv_cb(void *arg) -{ - xresp0_pipe *p = arg; - xresp0_sock *s = p->psock; - nni_msgq * urq = s->urq; - nni_msg * msg; - int hops; - int ttl; - - if (nni_aio_result(&p->aio_recv) != 0) { - nni_pipe_close(p->npipe); - return; - } - - ttl = nni_atomic_get(&s->ttl); - msg = nni_aio_get_msg(&p->aio_recv); - nni_aio_set_msg(&p->aio_recv, NULL); - nni_msg_set_pipe(msg, p->id); - - // Store the pipe id in the header, first thing. - nni_msg_header_append_u32(msg, p->id); - - // Move backtrace from body to header - hops = 1; - for (;;) { - bool end; - uint8_t *body; - - if (hops > ttl) { - goto drop; - } - hops++; - if (nni_msg_len(msg) < 4) { - // Peer sent us garbage, so kick it. - nni_msg_free(msg); - nni_pipe_close(p->npipe); - return; - } - body = nni_msg_body(msg); - end = ((body[0] & 0x80u) != 0); - if (nni_msg_header_append(msg, body, 4) != 0) { - goto drop; - } - nni_msg_trim(msg, 4); - if (end) { - break; - } - } - - // Now send it up. - nni_aio_set_msg(&p->aio_putq, msg); - nni_msgq_aio_put(urq, &p->aio_putq); - return; - -drop: - nni_msg_free(msg); - nni_pipe_recv(p->npipe, &p->aio_recv); -} - -static void -xresp0_putq_cb(void *arg) -{ - xresp0_pipe *p = arg; - - if (nni_aio_result(&p->aio_putq) != 0) { - nni_msg_free(nni_aio_get_msg(&p->aio_putq)); - nni_aio_set_msg(&p->aio_putq, NULL); - nni_pipe_close(p->npipe); - return; - } - - nni_pipe_recv(p->npipe, &p->aio_recv); -} - -static int -xresp0_sock_set_maxttl(void *arg, const void *buf, size_t sz, nni_opt_type t) -{ - xresp0_sock *s = arg; - int ttl; - int rv; - if ((rv = nni_copyin_int(&ttl, buf, sz, 1, NNI_MAX_MAX_TTL, t)) == 0) { - nni_atomic_set(&s->ttl, ttl); - } - return (rv); -} - -static int -xresp0_sock_get_maxttl(void *arg, void *buf, size_t *szp, nni_opt_type t) -{ - xresp0_sock *s = arg; - return (nni_copyout_int(nni_atomic_get(&s->ttl), buf, szp, t)); -} - -static void -xresp0_sock_send(void *arg, nni_aio *aio) -{ - xresp0_sock *s = arg; - - nni_msgq_aio_put(s->uwq, aio); -} - -static void -xresp0_sock_recv(void *arg, nni_aio *aio) -{ - xresp0_sock *s = arg; - - nni_msgq_aio_get(s->urq, aio); -} - -static nni_proto_pipe_ops xresp0_pipe_ops = { - .pipe_size = sizeof(xresp0_pipe), - .pipe_init = xresp0_pipe_init, - .pipe_fini = xresp0_pipe_fini, - .pipe_start = xresp0_pipe_start, - .pipe_close = xresp0_pipe_close, - .pipe_stop = xresp0_pipe_stop, -}; - -static nni_option xresp0_sock_options[] = { - { - .o_name = NNG_OPT_MAXTTL, - .o_get = xresp0_sock_get_maxttl, - .o_set = xresp0_sock_set_maxttl, - }, - // terminate list - { - .o_name = NULL, - }, -}; - -static nni_proto_sock_ops xresp0_sock_ops = { - .sock_size = sizeof(xresp0_sock), - .sock_init = xresp0_sock_init, - .sock_fini = xresp0_sock_fini, - .sock_open = xresp0_sock_open, - .sock_close = xresp0_sock_close, - .sock_send = xresp0_sock_send, - .sock_recv = xresp0_sock_recv, - .sock_options = xresp0_sock_options, -}; - -static nni_proto xresp0_proto = { - .proto_version = NNI_PROTOCOL_VERSION, - .proto_self = { NNI_PROTO_RESPONDENT_V0, "respondent" }, - .proto_peer = { NNI_PROTO_SURVEYOR_V0, "surveyor" }, - .proto_flags = NNI_PROTO_FLAG_SNDRCV | NNI_PROTO_FLAG_RAW, - .proto_sock_ops = &xresp0_sock_ops, - .proto_pipe_ops = &xresp0_pipe_ops, -}; - -int -nng_respondent0_open_raw(nng_socket *sidp) -{ - return (nni_proto_open(sidp, &xresp0_proto)); -} diff --git a/src/protocol/survey0/xrespond_test.c b/src/protocol/survey0/xrespond_test.c deleted file mode 100644 index ec5e99a3..00000000 --- a/src/protocol/survey0/xrespond_test.c +++ /dev/null @@ -1,436 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - -#include - -static void -test_xresp_identity(void) -{ - nng_socket s; - int p1, p2; - char * n1; - char * n2; - - NUTS_PASS(nng_respondent0_open_raw(&s)); - NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PROTO, &p1)); - NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PEER, &p2)); - NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PROTONAME, &n1)); - NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PEERNAME, &n2)); - NUTS_CLOSE(s); - NUTS_TRUE(p1 == NNG_RESPONDENT0_SELF); - NUTS_TRUE(p2 == NNG_RESPONDENT0_PEER); - NUTS_MATCH(n1, NNG_RESPONDENT0_SELF_NAME); - NUTS_MATCH(n2, NNG_RESPONDENT0_PEER_NAME); - nng_strfree(n1); - nng_strfree(n2); -} - -static void -test_xresp_raw(void) -{ - nng_socket s; - bool b; - - NUTS_PASS(nng_respondent0_open_raw(&s)); - NUTS_PASS(nng_socket_get_bool(s, NNG_OPT_RAW, &b)); - NUTS_TRUE(b); - NUTS_CLOSE(s); -} - -static void -test_xresp_no_context(void) -{ - nng_socket s; - nng_ctx ctx; - - NUTS_PASS(nng_respondent0_open_raw(&s)); - NUTS_FAIL(nng_ctx_open(&ctx, s), NNG_ENOTSUP); - NUTS_CLOSE(s); -} - -static void -test_xresp_poll_writeable(void) -{ - int fd; - nng_socket surv; - nng_socket resp; - - NUTS_PASS(nng_respondent0_open_raw(&resp)); - NUTS_PASS(nng_surveyor0_open(&surv)); - NUTS_PASS(nng_socket_get_int(resp, NNG_OPT_SENDFD, &fd)); - NUTS_TRUE(fd >= 0); - - // We are always writeable, even before connect. This is so that - // back-pressure from a bad peer can't trash others. We assume - // that peers won't send us requests faster than they can consume - // the answers. If they do, they will lose their answers. - NUTS_TRUE(nuts_poll_fd(fd) == true); - - NUTS_MARRY(surv, resp); - - // Now it's writable. - NUTS_TRUE(nuts_poll_fd(fd) == true); - - NUTS_CLOSE(surv); - NUTS_CLOSE(resp); -} - -static void -test_xresp_poll_readable(void) -{ - int fd; - nng_socket surv; - nng_socket resp; - nng_msg * msg; - - NUTS_PASS(nng_surveyor0_open(&surv)); - NUTS_PASS(nng_respondent0_open_raw(&resp)); - NUTS_PASS(nng_socket_get_int(resp, NNG_OPT_RECVFD, &fd)); - NUTS_TRUE(fd >= 0); - - // Not readable if not connected! - NUTS_TRUE(nuts_poll_fd(fd) == false); - - // Even after connect (no message yet) - NUTS_MARRY(surv, resp); - NUTS_TRUE(nuts_poll_fd(fd) == false); - - // But once we send messages, it is. - // We have to send a request, in order to send a reply. - NUTS_SEND(surv, "abc"); - NUTS_SLEEP(100); - - NUTS_TRUE(nuts_poll_fd(fd) == true); - - // and receiving makes it no longer ready - NUTS_PASS(nng_recvmsg(resp, &msg, 0)); - nng_msg_free(msg); - NUTS_TRUE(nuts_poll_fd(fd) == false); - - NUTS_CLOSE(surv); - NUTS_CLOSE(resp); -} - -static void -test_xresp_validate_peer(void) -{ - nng_socket s1, s2; - nng_stat * stats; - nng_stat * reject; - char * addr; - - NUTS_ADDR(addr, "inproc"); - - NUTS_PASS(nng_respondent0_open_raw(&s1)); - NUTS_PASS(nng_respondent0_open(&s2)); - - NUTS_PASS(nng_listen(s1, addr, NULL, 0)); - NUTS_PASS(nng_dial(s2, addr, NULL, NNG_FLAG_NONBLOCK)); - - NUTS_SLEEP(100); - NUTS_PASS(nng_stats_get(&stats)); - - NUTS_TRUE(stats != NULL); - NUTS_TRUE((reject = nng_stat_find_socket(stats, s1)) != NULL); - NUTS_TRUE((reject = nng_stat_find(reject, "reject")) != NULL); - - NUTS_TRUE(nng_stat_type(reject) == NNG_STAT_COUNTER); - NUTS_TRUE(nng_stat_value(reject) > 0); - - NUTS_CLOSE(s1); - NUTS_CLOSE(s2); - nng_stats_free(stats); -} - -static void -test_xresp_close_pipe_before_send(void) -{ - nng_socket resp; - nng_socket surv; - nng_pipe p; - nng_aio * aio1; - nng_msg * m; - - NUTS_PASS(nng_respondent0_open_raw(&resp)); - NUTS_PASS(nng_surveyor0_open(&surv)); - NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_aio_alloc(&aio1, NULL, NULL)); - - NUTS_MARRY(surv, resp); - NUTS_SEND(surv, "test"); - - nng_recv_aio(resp, aio1); - nng_aio_wait(aio1); - NUTS_PASS(nng_aio_result(aio1)); - NUTS_TRUE((m = nng_aio_get_msg(aio1)) != NULL); - p = nng_msg_get_pipe(m); - NUTS_PASS(nng_pipe_close(p)); - NUTS_PASS(nng_sendmsg(resp, m, 0)); - - NUTS_CLOSE(surv); - NUTS_CLOSE(resp); - nng_aio_free(aio1); -} - -static void -test_xresp_close_pipe_during_send(void) -{ - nng_socket resp; - nng_socket surv; - nng_pipe p; - nng_msg * m; - - NUTS_PASS(nng_respondent_open_raw(&resp)); - NUTS_PASS(nng_surveyor0_open_raw(&surv)); - NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_SENDTIMEO, 200)); - NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_int(resp, NNG_OPT_SENDBUF, 20)); - NUTS_PASS(nng_socket_set_int(resp, NNG_OPT_RECVBUF, 20)); - NUTS_PASS(nng_socket_set_int(surv, NNG_OPT_SENDBUF, 20)); - NUTS_PASS(nng_socket_set_int(surv, NNG_OPT_RECVBUF, 1)); - - NUTS_MARRY(surv, resp); - - NUTS_PASS(nng_msg_alloc(&m, 4)); - NUTS_PASS(nng_msg_append_u32(m, (unsigned) 0x81000000u)); - NUTS_PASS(nng_sendmsg(surv, m, 0)); - NUTS_PASS(nng_recvmsg(resp, &m, 0)); - p = nng_msg_get_pipe(m); - nng_msg_free(m); - - for (int i = 0; i < 100; i++) { - NUTS_PASS(nng_msg_alloc(&m, 4)); - NUTS_PASS(nng_msg_header_append_u32(m, nng_pipe_id(p))); - NUTS_PASS( - nng_msg_header_append_u32(m, (unsigned) i | 0x80000000u)); - // protocol does not exert back-pressure - NUTS_PASS(nng_sendmsg(resp, m, 0)); - } - NUTS_PASS(nng_pipe_close(p)); - - NUTS_CLOSE(surv); - NUTS_CLOSE(resp); -} - -static void -test_xresp_close_during_recv(void) -{ - nng_socket resp; - nng_socket surv; - nng_msg * m; - - NUTS_PASS(nng_respondent0_open_raw(&resp)); - NUTS_PASS(nng_surveyor0_open_raw(&surv)); - NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_SENDTIMEO, 100)); - NUTS_PASS(nng_socket_set_int(resp, NNG_OPT_RECVBUF, 5)); - NUTS_PASS(nng_socket_set_int(surv, NNG_OPT_SENDBUF, 20)); - - NUTS_MARRY(surv, resp); - - for (unsigned i = 0; i < 100; i++) { - int rv; - NUTS_PASS(nng_msg_alloc(&m, 4)); - NUTS_PASS(nng_msg_header_append_u32(m, i | 0x80000000u)); - rv = nng_sendmsg(surv, m, 0); - if (rv == NNG_ETIMEDOUT) { - nng_msg_free(m); - break; - } - } - NUTS_CLOSE(surv); - NUTS_CLOSE(resp); -} - -static void -test_xresp_recv_aio_stopped(void) -{ - nng_socket resp; - nng_aio * aio; - - NUTS_PASS(nng_respondent0_open_raw(&resp)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - - nng_aio_stop(aio); - nng_recv_aio(resp, aio); - nng_aio_wait(aio); - NUTS_FAIL(nng_aio_result(aio), NNG_ECANCELED); - NUTS_CLOSE(resp); - nng_aio_free(aio); -} - -static void -test_xresp_send_no_header(void) -{ - nng_socket resp; - nng_socket surv; - nng_msg * m; - - NUTS_PASS(nng_surveyor0_open_raw(&surv)); - NUTS_PASS(nng_respondent0_open_raw(&resp)); - NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_RECVTIMEO, 100)); - NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_RECVTIMEO, 100)); - NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_SENDTIMEO, 1000)); - - NUTS_MARRY(surv, resp); - - NUTS_PASS(nng_msg_alloc(&m, 4)); - NUTS_PASS(nng_sendmsg(resp, m, 0)); - NUTS_FAIL(nng_recvmsg(resp, &m, 0), NNG_ETIMEDOUT); - - NUTS_CLOSE(surv); - NUTS_CLOSE(resp); -} - -static void -test_xresp_recv_garbage(void) -{ - nng_socket resp; - nng_socket surv; - nng_msg * m; - - NUTS_PASS(nng_respondent0_open_raw(&resp)); - NUTS_PASS(nng_surveyor0_open_raw(&surv)); - NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_RECVTIMEO, 100)); - NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_SENDTIMEO, 100)); - NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_SENDTIMEO, 1000)); - - NUTS_MARRY(surv, resp); - - NUTS_PASS(nng_msg_alloc(&m, 4)); - NUTS_PASS(nng_msg_append_u32(m, 1u)); - NUTS_PASS(nng_sendmsg(surv, m, 0)); - NUTS_FAIL(nng_recvmsg(resp, &m, 0), NNG_ETIMEDOUT); - - NUTS_CLOSE(surv); - NUTS_CLOSE(resp); -} - -static void -test_xresp_ttl_option(void) -{ - nng_socket resp; - int v; - bool b; - size_t sz; - const char *opt = NNG_OPT_MAXTTL; - - NUTS_PASS(nng_respondent0_open_raw(&resp)); - - NUTS_PASS(nng_socket_set_int(resp, opt, 1)); - NUTS_FAIL(nng_socket_set_int(resp, opt, 0), NNG_EINVAL); - NUTS_FAIL(nng_socket_set_int(resp, opt, -1), NNG_EINVAL); - NUTS_FAIL(nng_socket_set_int(resp, opt, 16), NNG_EINVAL); - NUTS_FAIL(nng_socket_set_int(resp, opt, 256), NNG_EINVAL); - NUTS_PASS(nng_socket_set_int(resp, opt, 3)); - NUTS_PASS(nng_socket_get_int(resp, opt, &v)); - NUTS_TRUE(v == 3); - v = 0; - sz = sizeof(v); - NUTS_PASS(nng_socket_get(resp, opt, &v, &sz)); - NUTS_TRUE(v == 3); - NUTS_TRUE(sz == sizeof(v)); - - NUTS_FAIL(nng_socket_set(resp, opt, "", 1), NNG_EINVAL); - sz = 1; - NUTS_FAIL(nng_socket_get(resp, opt, &v, &sz), NNG_EINVAL); - NUTS_FAIL(nng_socket_set_bool(resp, opt, true), NNG_EBADTYPE); - NUTS_FAIL(nng_socket_get_bool(resp, opt, &b), NNG_EBADTYPE); - - NUTS_CLOSE(resp); -} - -static void -test_xresp_ttl_drop(void) -{ - nng_socket resp; - nng_socket surv; - nng_msg * m; - - NUTS_PASS(nng_respondent0_open_raw(&resp)); - NUTS_PASS(nng_surveyor0_open_raw(&surv)); - NUTS_PASS(nng_socket_set_int(resp, NNG_OPT_MAXTTL, 3)); - NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_RECVTIMEO, 200)); - NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_SENDTIMEO, 1000)); - - NUTS_MARRY(surv, resp); - - // Send messages. Note that xresp implicitly adds a hop on receive. - - NUTS_PASS(nng_msg_alloc(&m, 0)); - NUTS_PASS(nng_msg_append_u32(m, 1u)); // 2 hops - NUTS_PASS(nng_msg_append_u32(m, 0x80000001u)); - NUTS_PASS(nng_msg_append(m, "PASS1", 6)); - NUTS_PASS(nng_sendmsg(surv, m, 0)); - - NUTS_PASS(nng_msg_alloc(&m, 0)); - NUTS_PASS(nng_msg_append_u32(m, 1u)); // 4 hops -- discard! - NUTS_PASS(nng_msg_append_u32(m, 2u)); - NUTS_PASS(nng_msg_append_u32(m, 3u)); - NUTS_PASS(nng_msg_append_u32(m, 0x80000002u)); - NUTS_PASS(nng_msg_append(m, "FAIL2", 6)); - NUTS_PASS(nng_sendmsg(surv, m, 0)); - - NUTS_PASS(nng_msg_alloc(&m, 0)); - NUTS_PASS(nng_msg_append_u32(m, 1u)); // 3 hops - passes - NUTS_PASS(nng_msg_append_u32(m, 2u)); - NUTS_PASS(nng_msg_append_u32(m, 0x80000003u)); - NUTS_PASS(nng_msg_append(m, "PASS3", 6)); - NUTS_PASS(nng_sendmsg(surv, m, 0)); - - NUTS_PASS(nng_msg_alloc(&m, 0)); - NUTS_PASS(nng_msg_append_u32(m, 1u)); // 4 hops -- discard! - NUTS_PASS(nng_msg_append_u32(m, 2u)); - NUTS_PASS(nng_msg_append_u32(m, 3u)); - NUTS_PASS(nng_msg_append_u32(m, 0x80000003u)); - NUTS_PASS(nng_msg_append(m, "FAIL4", 6)); - NUTS_PASS(nng_sendmsg(surv, m, 0)); - - // So on receive we should see 80000001 and 80000003. - NUTS_PASS(nng_recvmsg(resp, &m, 0)); - NUTS_TRUE(nng_msg_header_len(m) == 12); - NUTS_TRUE(nng_msg_len(m) == 6); - NUTS_MATCH(nng_msg_body(m), "PASS1"); - nng_msg_free(m); - - NUTS_PASS(nng_recvmsg(resp, &m, 0)); - NUTS_TRUE(nng_msg_header_len(m) == 16); // 3 hops + ID - NUTS_TRUE(nng_msg_len(m) == 6); - NUTS_MATCH(nng_msg_body(m), "PASS3"); - nng_msg_free(m); - - NUTS_FAIL(nng_recvmsg(resp, &m, 0), NNG_ETIMEDOUT); - - NUTS_CLOSE(surv); - NUTS_CLOSE(resp); -} - -NUTS_TESTS = { - { "xrespond identity", test_xresp_identity }, - { "xrespond raw", test_xresp_raw }, - { "xrespond no context", test_xresp_no_context }, - { "xrespond poll readable", test_xresp_poll_readable }, - { "xrespond poll writable", test_xresp_poll_writeable }, - { "xrespond validate peer", test_xresp_validate_peer }, - { "xrespond close pipe before send", - test_xresp_close_pipe_before_send }, - { "xrespond close pipe during send", - test_xresp_close_pipe_during_send }, - { "xrespond close during recv", test_xresp_close_during_recv }, - { "xrespond recv aio stopped", test_xresp_recv_aio_stopped }, - { "xrespond send no header", test_xresp_send_no_header }, - { "xrespond recv garbage", test_xresp_recv_garbage }, - { "xrespond ttl option", test_xresp_ttl_option }, - { "xrespond ttl drop", test_xresp_ttl_drop }, - { NULL, NULL }, -}; diff --git a/src/protocol/survey0/xsurvey.c b/src/protocol/survey0/xsurvey.c deleted file mode 100644 index 2a198662..00000000 --- a/src/protocol/survey0/xsurvey.c +++ /dev/null @@ -1,379 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// Copyright 2018 Capitar IT Group BV -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - -#include "core/nng_impl.h" -#include "nng/protocol/survey0/survey.h" - -// Surveyor protocol. The SURVEYOR protocol is the "survey" side of the -// survey pattern. This is useful for building service discovery, voting, etc. - -typedef struct xsurv0_pipe xsurv0_pipe; -typedef struct xsurv0_sock xsurv0_sock; - -static void xsurv0_sock_getq_cb(void *); -static void xsurv0_getq_cb(void *); -static void xsurv0_putq_cb(void *); -static void xsurv0_send_cb(void *); -static void xsurv0_recv_cb(void *); - -// surv0_sock is our per-socket protocol private structure. -struct xsurv0_sock { - nni_list pipes; - nni_aio aio_getq; - nni_msgq * uwq; - nni_msgq * urq; - nni_mtx mtx; - nni_atomic_int ttl; -}; - -// surv0_pipe is our per-pipe protocol private structure. -struct xsurv0_pipe { - nni_pipe * npipe; - xsurv0_sock * psock; - nni_msgq * sendq; - nni_list_node node; - nni_aio aio_getq; - nni_aio aio_putq; - nni_aio aio_send; - nni_aio aio_recv; -}; - -static void -xsurv0_sock_fini(void *arg) -{ - xsurv0_sock *s = arg; - - nni_aio_fini(&s->aio_getq); - nni_mtx_fini(&s->mtx); -} - -static int -xsurv0_sock_init(void *arg, nni_sock *nsock) -{ - xsurv0_sock *s = arg; - - nni_aio_init(&s->aio_getq, xsurv0_sock_getq_cb, s); - NNI_LIST_INIT(&s->pipes, xsurv0_pipe, node); - nni_mtx_init(&s->mtx); - - s->uwq = nni_sock_sendq(nsock); - s->urq = nni_sock_recvq(nsock); - nni_atomic_init(&s->ttl); - nni_atomic_set(&s->ttl, 8); - - return (0); -} - -static void -xsurv0_sock_open(void *arg) -{ - xsurv0_sock *s = arg; - - nni_msgq_aio_get(s->uwq, &s->aio_getq); -} - -static void -xsurv0_sock_close(void *arg) -{ - xsurv0_sock *s = arg; - - nni_aio_close(&s->aio_getq); -} - -static void -xsurv0_pipe_stop(void *arg) -{ - xsurv0_pipe *p = arg; - - nni_aio_stop(&p->aio_getq); - nni_aio_stop(&p->aio_send); - nni_aio_stop(&p->aio_recv); - nni_aio_stop(&p->aio_putq); -} - -static void -xsurv0_pipe_fini(void *arg) -{ - xsurv0_pipe *p = arg; - - nni_aio_fini(&p->aio_getq); - nni_aio_fini(&p->aio_send); - nni_aio_fini(&p->aio_recv); - nni_aio_fini(&p->aio_putq); - nni_msgq_fini(p->sendq); -} - -static int -xsurv0_pipe_init(void *arg, nni_pipe *npipe, void *s) -{ - xsurv0_pipe *p = arg; - int rv; - - nni_aio_init(&p->aio_getq, xsurv0_getq_cb, p); - nni_aio_init(&p->aio_putq, xsurv0_putq_cb, p); - nni_aio_init(&p->aio_send, xsurv0_send_cb, p); - nni_aio_init(&p->aio_recv, xsurv0_recv_cb, p); - - // This depth could be tunable. The queue exists so that if we - // have multiple requests coming in faster than we can deliver them, - // we try to avoid dropping them. We don't really have a solution - // for applying back pressure. It would be nice if surveys carried - // an expiration with them, so that we could discard any that are - // not delivered before their expiration date. - if ((rv = nni_msgq_init(&p->sendq, 16)) != 0) { - xsurv0_pipe_fini(p); - return (rv); - } - - p->npipe = npipe; - p->psock = s; - return (0); -} - -static int -xsurv0_pipe_start(void *arg) -{ - xsurv0_pipe *p = arg; - xsurv0_sock *s = p->psock; - - if (nni_pipe_peer(p->npipe) != NNG_SURVEYOR0_PEER) { - return (NNG_EPROTO); - } - - nni_mtx_lock(&s->mtx); - nni_list_append(&s->pipes, p); - nni_mtx_unlock(&s->mtx); - - nni_msgq_aio_get(p->sendq, &p->aio_getq); - nni_pipe_recv(p->npipe, &p->aio_recv); - return (0); -} - -static void -xsurv0_pipe_close(void *arg) -{ - xsurv0_pipe *p = arg; - xsurv0_sock *s = p->psock; - - nni_aio_close(&p->aio_getq); - nni_aio_close(&p->aio_send); - nni_aio_close(&p->aio_recv); - nni_aio_close(&p->aio_putq); - - nni_msgq_close(p->sendq); - - nni_mtx_lock(&s->mtx); - if (nni_list_active(&s->pipes, p)) { - nni_list_remove(&s->pipes, p); - } - nni_mtx_unlock(&s->mtx); -} - -static void -xsurv0_getq_cb(void *arg) -{ - xsurv0_pipe *p = arg; - - if (nni_aio_result(&p->aio_getq) != 0) { - nni_pipe_close(p->npipe); - return; - } - - nni_aio_set_msg(&p->aio_send, nni_aio_get_msg(&p->aio_getq)); - nni_aio_set_msg(&p->aio_getq, NULL); - - nni_pipe_send(p->npipe, &p->aio_send); -} - -static void -xsurv0_send_cb(void *arg) -{ - xsurv0_pipe *p = arg; - - if (nni_aio_result(&p->aio_send) != 0) { - nni_msg_free(nni_aio_get_msg(&p->aio_send)); - nni_aio_set_msg(&p->aio_send, NULL); - nni_pipe_close(p->npipe); - return; - } - - nni_msgq_aio_get(p->sendq, &p->aio_getq); -} - -static void -xsurv0_putq_cb(void *arg) -{ - xsurv0_pipe *p = arg; - - if (nni_aio_result(&p->aio_putq) != 0) { - nni_msg_free(nni_aio_get_msg(&p->aio_putq)); - nni_aio_set_msg(&p->aio_putq, NULL); - nni_pipe_close(p->npipe); - return; - } - - nni_pipe_recv(p->npipe, &p->aio_recv); -} - -static void -xsurv0_recv_cb(void *arg) -{ - xsurv0_pipe *p = arg; - nni_msg * msg; - bool end; - - if (nni_aio_result(&p->aio_recv) != 0) { - nni_pipe_close(p->npipe); - return; - } - - msg = nni_aio_get_msg(&p->aio_recv); - nni_aio_set_msg(&p->aio_recv, NULL); - nni_msg_set_pipe(msg, nni_pipe_id(p->npipe)); - end = false; - - while (!end) { - uint8_t *body; - - if (nni_msg_len(msg) < 4) { - // Peer gave us garbage, so kick it. - nni_msg_free(msg); - nni_pipe_close(p->npipe); - return; - } - body = nni_msg_body(msg); - end = ((body[0] & 0x80u) != 0); - - if (nni_msg_header_append(msg, body, sizeof(uint32_t)) != 0) { - // TODO: bump a no-memory stat - nni_msg_free(msg); - // Closing the pipe may release some memory. - // It at least gives an indication to the peer - // that we've lost the message. - nni_pipe_close(p->npipe); - return; - } - nni_msg_trim(msg, sizeof(uint32_t)); - } - - nni_aio_set_msg(&p->aio_putq, msg); - nni_msgq_aio_put(p->psock->urq, &p->aio_putq); -} - -static int -xsurv0_sock_set_max_ttl(void *arg, const void *buf, size_t sz, nni_opt_type t) -{ - xsurv0_sock *s = arg; - int ttl; - int rv; - if ((rv = nni_copyin_int(&ttl, buf, sz, 1, NNI_MAX_MAX_TTL, t)) == 0) { - nni_atomic_set(&s->ttl, ttl); - } - return (rv); -} - -static int -xsurv0_sock_get_max_ttl(void *arg, void *buf, size_t *szp, nni_opt_type t) -{ - xsurv0_sock *s = arg; - return (nni_copyout_int(nni_atomic_get(&s->ttl), buf, szp, t)); -} - -static void -xsurv0_sock_getq_cb(void *arg) -{ - xsurv0_sock *s = arg; - xsurv0_pipe *p; - nni_msg * msg; - - if (nni_aio_result(&s->aio_getq) != 0) { - // Should be NNG_ECLOSED. - return; - } - msg = nni_aio_get_msg(&s->aio_getq); - nni_aio_set_msg(&s->aio_getq, NULL); - - nni_mtx_lock(&s->mtx); - NNI_LIST_FOREACH (&s->pipes, p) { - nni_msg_clone(msg); - if (nni_msgq_tryput(p->sendq, msg) != 0) { - nni_msg_free(msg); - } - } - - nni_msgq_aio_get(s->uwq, &s->aio_getq); - nni_mtx_unlock(&s->mtx); - - // If there were no pipes to send on, just toss the message. - nni_msg_free(msg); -} - -static void -xsurv0_sock_recv(void *arg, nni_aio *aio) -{ - xsurv0_sock *s = arg; - - nni_msgq_aio_get(s->urq, aio); -} - -static void -xsurv0_sock_send(void *arg, nni_aio *aio) -{ - xsurv0_sock *s = arg; - - nni_msgq_aio_put(s->uwq, aio); -} - -static nni_proto_pipe_ops xsurv0_pipe_ops = { - .pipe_size = sizeof(xsurv0_pipe), - .pipe_init = xsurv0_pipe_init, - .pipe_fini = xsurv0_pipe_fini, - .pipe_start = xsurv0_pipe_start, - .pipe_close = xsurv0_pipe_close, - .pipe_stop = xsurv0_pipe_stop, -}; - -static nni_option xsurv0_sock_options[] = { - { - .o_name = NNG_OPT_MAXTTL, - .o_get = xsurv0_sock_get_max_ttl, - .o_set = xsurv0_sock_set_max_ttl, - }, - // terminate list - { - .o_name = NULL, - }, -}; - -static nni_proto_sock_ops xsurv0_sock_ops = { - .sock_size = sizeof(xsurv0_sock), - .sock_init = xsurv0_sock_init, - .sock_fini = xsurv0_sock_fini, - .sock_open = xsurv0_sock_open, - .sock_close = xsurv0_sock_close, - .sock_send = xsurv0_sock_send, - .sock_recv = xsurv0_sock_recv, - .sock_options = xsurv0_sock_options, -}; - -static nni_proto xsurv0_proto = { - .proto_version = NNI_PROTOCOL_VERSION, - .proto_self = { NNG_SURVEYOR0_SELF, NNG_SURVEYOR0_SELF_NAME }, - .proto_peer = { NNG_SURVEYOR0_PEER, NNG_SURVEYOR0_PEER_NAME }, - .proto_flags = NNI_PROTO_FLAG_SNDRCV | NNI_PROTO_FLAG_RAW, - .proto_sock_ops = &xsurv0_sock_ops, - .proto_pipe_ops = &xsurv0_pipe_ops, -}; - -int -nng_surveyor0_open_raw(nng_socket *sidp) -{ - return (nni_proto_open(sidp, &xsurv0_proto)); -} diff --git a/src/protocol/survey0/xsurvey_test.c b/src/protocol/survey0/xsurvey_test.c deleted file mode 100644 index f8e9d401..00000000 --- a/src/protocol/survey0/xsurvey_test.c +++ /dev/null @@ -1,399 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - -#include - -static void -test_xsurveyor_identity(void) -{ - nng_socket s; - int p; - char * n; - - NUTS_PASS(nng_surveyor0_open_raw(&s)); - NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PROTO, &p)); - NUTS_TRUE(p == NNG_SURVEYOR0_SELF); // 0x62 - NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PEER, &p)); - NUTS_TRUE(p == NNG_SURVEYOR0_PEER); // 0x62 - NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PROTONAME, &n)); - NUTS_MATCH(n, NNG_SURVEYOR0_SELF_NAME); - nng_strfree(n); - NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PEERNAME, &n)); - NUTS_MATCH(n, NNG_SURVEYOR0_PEER_NAME); - nng_strfree(n); - NUTS_CLOSE(s); -} - -static void -test_xsurveyor_raw(void) -{ - nng_socket s; - bool b; - - NUTS_PASS(nng_surveyor0_open_raw(&s)); - NUTS_PASS(nng_socket_get_bool(s, NNG_OPT_RAW, &b)); - NUTS_TRUE(b); - NUTS_CLOSE(s); -} - -static void -test_xsurvey_no_context(void) -{ - nng_socket s; - nng_ctx ctx; - - NUTS_PASS(nng_surveyor0_open_raw(&s)); - NUTS_FAIL(nng_ctx_open(&ctx, s), NNG_ENOTSUP); - NUTS_CLOSE(s); -} - -static void -test_xsurvey_poll_writeable(void) -{ - int fd; - nng_socket surv; - nng_socket resp; - - NUTS_PASS(nng_surveyor0_open_raw(&surv)); - NUTS_PASS(nng_respondent0_open(&resp)); - NUTS_PASS(nng_socket_get_int(surv, NNG_OPT_SENDFD, &fd)); - NUTS_TRUE(fd >= 0); - - // Survey is broadcast, so we can always write. - NUTS_TRUE(nuts_poll_fd(fd)); - - NUTS_MARRY(surv, resp); - - // Now it's writable. - NUTS_TRUE(nuts_poll_fd(fd)); - - NUTS_CLOSE(surv); - NUTS_CLOSE(resp); -} - -static void -test_xsurvey_poll_readable(void) -{ - int fd; - nng_socket surv; - nng_socket resp; - nng_msg * msg; - - NUTS_PASS(nng_surveyor0_open_raw(&surv)); - NUTS_PASS(nng_respondent0_open(&resp)); - NUTS_PASS(nng_socket_get_int(surv, NNG_OPT_RECVFD, &fd)); - NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_SENDTIMEO, 1000)); - - NUTS_TRUE(fd >= 0); - - // Not readable if not connected! - NUTS_TRUE(nuts_poll_fd(fd) == false); - - // Even after connect (no message yet) - NUTS_MARRY(surv, resp); - NUTS_TRUE(nuts_poll_fd(fd) == false); - - // But once we send messages, it is. - // We have to send a request, in order to send a reply. - NUTS_PASS(nng_msg_alloc(&msg, 0)); - // Request ID - NUTS_PASS(nng_msg_append_u32(msg, 0x80000000)); - NUTS_PASS(nng_sendmsg(surv, msg, 0)); - - NUTS_PASS(nng_recvmsg(resp, &msg, 0)); - NUTS_PASS(nng_sendmsg(resp, msg, 0)); - - NUTS_SLEEP(100); - - NUTS_TRUE(nuts_poll_fd(fd) ); - - // and receiving makes it no longer ready - NUTS_PASS(nng_recvmsg(surv, &msg, 0)); - nng_msg_free(msg); - NUTS_TRUE(nuts_poll_fd(fd) == false); - - NUTS_CLOSE(surv); - NUTS_CLOSE(resp); -} - -static void -test_xsurvey_validate_peer(void) -{ - nng_socket s1, s2; - nng_stat * stats; - nng_stat * reject; - char *addr; - - NUTS_ADDR(addr, "inproc"); - - NUTS_PASS(nng_surveyor0_open_raw(&s1)); - NUTS_PASS(nng_surveyor0_open(&s2)); - - NUTS_PASS(nng_listen(s1, addr, NULL, 0)); - NUTS_PASS(nng_dial(s2, addr, NULL, NNG_FLAG_NONBLOCK)); - - NUTS_SLEEP(100); - NUTS_PASS(nng_stats_get(&stats)); - - NUTS_TRUE(stats != NULL); - NUTS_TRUE((reject = nng_stat_find_socket(stats, s1)) != NULL); - NUTS_TRUE((reject = nng_stat_find(reject, "reject")) != NULL); - - NUTS_TRUE(nng_stat_type(reject) == NNG_STAT_COUNTER); - NUTS_TRUE(nng_stat_value(reject) > 0); - - NUTS_CLOSE(s1); - NUTS_CLOSE(s2); - nng_stats_free(stats); -} - -static void -test_xsurvey_recv_aio_stopped(void) -{ - nng_socket surv; - nng_aio * aio; - - NUTS_PASS(nng_surveyor0_open_raw(&surv)); - NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); - - nng_aio_stop(aio); - nng_recv_aio(surv, aio); - nng_aio_wait(aio); - NUTS_FAIL(nng_aio_result(aio), NNG_ECANCELED); - NUTS_CLOSE(surv); - nng_aio_free(aio); -} - -static void -test_xsurvey_recv_garbage(void) -{ - nng_socket resp; - nng_socket surv; - nng_msg * m; - uint32_t req_id; - - NUTS_PASS(nng_respondent0_open_raw(&resp)); - NUTS_PASS(nng_surveyor0_open_raw(&surv)); - NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_RECVTIMEO, 100)); - NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_SENDTIMEO, 1000)); - - NUTS_MARRY(surv, resp); - - NUTS_PASS(nng_msg_alloc(&m, 0)); - NUTS_PASS(nng_msg_append_u32(m, 0x80000000)); - NUTS_PASS(nng_sendmsg(surv, m, 0)); - - NUTS_PASS(nng_recvmsg(resp, &m, 0)); - - // The message will have a header that contains the 32-bit pipe ID, - // followed by the 32-bit request ID. We will discard the request - // ID before sending it out. - NUTS_TRUE(nng_msg_header_len(m) == 8); - NUTS_PASS(nng_msg_header_chop_u32(m, &req_id)); - NUTS_TRUE(req_id == 0x80000000); - - NUTS_PASS(nng_sendmsg(resp, m, 0)); - NUTS_FAIL(nng_recvmsg(surv, &m, 0), NNG_ETIMEDOUT); - - NUTS_CLOSE(surv); - NUTS_CLOSE(resp); -} - -static void -test_xsurvey_recv_header(void) -{ - nng_socket resp; - nng_socket surv; - nng_msg * m; - nng_pipe p; - uint32_t id; - - NUTS_PASS(nng_respondent0_open_raw(&resp)); - NUTS_PASS(nng_surveyor0_open_raw(&surv)); - NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_SENDTIMEO, 1000)); - - NUTS_MARRY_EX(surv, resp, NULL, NULL, &p); - - // Simulate a few hops. - NUTS_PASS(nng_msg_alloc(&m, 0)); - NUTS_PASS(nng_msg_header_append_u32(m, nng_pipe_id(p))); - NUTS_PASS(nng_msg_header_append_u32(m, 0x2)); - NUTS_PASS(nng_msg_header_append_u32(m, 0x1)); - NUTS_PASS(nng_msg_header_append_u32(m, 0x80000123u)); - - NUTS_PASS(nng_sendmsg(resp, m, 0)); - - NUTS_PASS(nng_recvmsg(surv, &m, 0)); - NUTS_TRUE(nng_msg_header_len(m) == 12); - NUTS_PASS(nng_msg_header_trim_u32(m, &id)); - NUTS_TRUE(id == 0x2); - NUTS_PASS(nng_msg_header_trim_u32(m, &id)); - NUTS_TRUE(id == 0x1); - NUTS_PASS(nng_msg_header_trim_u32(m, &id)); - NUTS_TRUE(id == 0x80000123u); - - nng_msg_free(m); - - NUTS_CLOSE(surv); - NUTS_CLOSE(resp); -} - -static void -test_xsurvey_close_during_recv(void) -{ - nng_socket resp; - nng_socket surv; - nng_msg * m; - nng_pipe p1; - nng_pipe p2; - - NUTS_PASS(nng_respondent0_open_raw(&resp)); - NUTS_PASS(nng_surveyor0_open_raw(&surv)); - NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_SENDTIMEO, 100)); - NUTS_PASS(nng_socket_set_int(surv, NNG_OPT_RECVBUF, 1)); - NUTS_PASS(nng_socket_set_int(resp, NNG_OPT_SENDBUF, 20)); - - NUTS_MARRY_EX(surv, resp, NULL, &p1, &p2); - NUTS_TRUE(nng_pipe_id(p1) > 0); - NUTS_TRUE(nng_pipe_id(p2) > 0); - - for (unsigned i = 0; i < 20; i++) { - NUTS_PASS(nng_msg_alloc(&m, 4)); - NUTS_PASS(nng_msg_header_append_u32(m, nng_pipe_id(p2))); - NUTS_PASS(nng_msg_header_append_u32(m, i | 0x80000000u)); - NUTS_SLEEP(10); - NUTS_PASS(nng_sendmsg(resp, m, 0)); - } - NUTS_CLOSE(surv); - NUTS_CLOSE(resp); -} - -static void -test_xsurvey_close_pipe_during_send(void) -{ - nng_socket resp; - nng_socket surv; - nng_msg * m; - nng_pipe p1; - nng_pipe p2; - - NUTS_PASS(nng_respondent0_open_raw(&resp)); - NUTS_PASS(nng_surveyor0_open_raw(&surv)); - NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_SENDTIMEO, 100)); - NUTS_PASS(nng_socket_set_int(resp, NNG_OPT_RECVBUF, 5)); - NUTS_PASS(nng_socket_set_int(surv, NNG_OPT_SENDBUF, 20)); - - NUTS_MARRY_EX(surv, resp, NULL, &p1, &p2); - NUTS_TRUE(nng_pipe_id(p1) > 0); - NUTS_TRUE(nng_pipe_id(p2) > 0); - - for (unsigned i = 0; i < 20; i++) { - NUTS_PASS(nng_msg_alloc(&m, 4)); - NUTS_PASS(nng_msg_header_append_u32(m, i | 0x80000000u)); - NUTS_SLEEP(10); - NUTS_PASS(nng_sendmsg(surv, m, 0)); - } - - NUTS_PASS(nng_pipe_close(p1)); - NUTS_CLOSE(surv); - NUTS_CLOSE(resp); -} - -static void -test_xsurvey_ttl_option(void) -{ - nng_socket s; - int v; - bool b; - size_t sz; - const char *opt = NNG_OPT_MAXTTL; - - NUTS_PASS(nng_surveyor0_open_raw(&s)); - - NUTS_PASS(nng_socket_set_int(s, opt, 1)); - NUTS_FAIL(nng_socket_set_int(s, opt, 0), NNG_EINVAL); - NUTS_FAIL(nng_socket_set_int(s, opt, -1), NNG_EINVAL); - NUTS_FAIL(nng_socket_set_int(s, opt, 16), NNG_EINVAL); - NUTS_FAIL(nng_socket_set_int(s, opt, 256), NNG_EINVAL); - NUTS_PASS(nng_socket_set_int(s, opt, 3)); - NUTS_PASS(nng_socket_get_int(s, opt, &v)); - NUTS_TRUE(v == 3); - v = 0; - sz = sizeof(v); - NUTS_PASS(nng_socket_get(s, opt, &v, &sz)); - NUTS_TRUE(v == 3); - NUTS_TRUE(sz == sizeof(v)); - - NUTS_FAIL(nng_socket_set(s, opt, "", 1) , NNG_EINVAL); - sz = 1; - NUTS_FAIL(nng_socket_get(s, opt, &v, &sz) , NNG_EINVAL); - NUTS_FAIL(nng_socket_set_bool(s, opt, true) , NNG_EBADTYPE); - NUTS_FAIL(nng_socket_get_bool(s, opt, &b) , NNG_EBADTYPE); - - NUTS_CLOSE(s); -} - -static void -test_xsurvey_broadcast(void) -{ - nng_socket resp1; - nng_socket resp2; - nng_socket surv; - nng_msg * m; - - NUTS_PASS(nng_respondent0_open(&resp1)); - NUTS_PASS(nng_respondent0_open(&resp2)); - NUTS_PASS(nng_surveyor0_open_raw(&surv)); - NUTS_PASS(nng_socket_set_ms(resp1, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(resp2, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_SENDTIMEO, 100)); - - NUTS_MARRY(surv, resp1); - NUTS_MARRY(surv, resp2); - - NUTS_PASS(nng_msg_alloc(&m, 0)); - NUTS_PASS(nng_msg_header_append_u32(m, 0x80000002u)); - NUTS_PASS(nng_msg_append(m, "hello", 6)); - - NUTS_PASS(nng_sendmsg(surv, m, 0)); - NUTS_RECV(resp1, "hello"); - NUTS_RECV(resp2, "hello"); - - NUTS_CLOSE(surv); - NUTS_CLOSE(resp1); - NUTS_CLOSE(resp2); -} - -TEST_LIST = { - { "xsurvey identity", test_xsurveyor_identity }, - { "xsurvey raw", test_xsurveyor_raw }, - { "xsurvey no context", test_xsurvey_no_context }, - { "xsurvey poll readable", test_xsurvey_poll_readable }, - { "xsurvey poll writable", test_xsurvey_poll_writeable }, - { "xsurvey validate peer", test_xsurvey_validate_peer }, - { "xsurvey recv aio stopped", test_xsurvey_recv_aio_stopped }, - { "xsurvey recv garbage", test_xsurvey_recv_garbage }, - { "xsurvey recv header", test_xsurvey_recv_header }, - { "xsurvey close during recv", test_xsurvey_close_during_recv }, - { "xsurvey close pipe during send", - test_xsurvey_close_pipe_during_send }, - { "xsurvey ttl option", test_xsurvey_ttl_option }, - { "xsurvey broadcast", test_xsurvey_broadcast }, - { NULL, NULL }, -}; diff --git a/src/sp/CMakeLists.txt b/src/sp/CMakeLists.txt new file mode 100644 index 00000000..b84952ee --- /dev/null +++ b/src/sp/CMakeLists.txt @@ -0,0 +1,13 @@ +# +# Copyright 2021 Staysail Systems, Inc. +# +# This software is supplied under the terms of the MIT License, a +# copy of which should be located in the distribution where this +# file was obtained (LICENSE.txt). A copy of the license may also be +# found online at https://opensource.org/licenses/MIT. +# + +nng_directory(sp) + +add_subdirectory(protocol) +add_subdirectory(transport) diff --git a/src/sp/protocol/CMakeLists.txt b/src/sp/protocol/CMakeLists.txt new file mode 100644 index 00000000..fd480523 --- /dev/null +++ b/src/sp/protocol/CMakeLists.txt @@ -0,0 +1,20 @@ +# +# Copyright 2020 Staysail Systems, Inc. +# +# This software is supplied under the terms of the MIT License, a +# copy of which should be located in the distribution where this +# file was obtained (LICENSE.txt). A copy of the license may also be +# found online at https://opensource.org/licenses/MIT. +# + +# Protocols. +nng_directory(protocol) + +add_subdirectory(bus0) +add_subdirectory(pair0) +add_subdirectory(pair1) +add_subdirectory(pipeline0) +add_subdirectory(pubsub0) +add_subdirectory(reqrep0) +add_subdirectory(survey0) + diff --git a/src/sp/protocol/bus0/CMakeLists.txt b/src/sp/protocol/bus0/CMakeLists.txt new file mode 100644 index 00000000..01c0b05b --- /dev/null +++ b/src/sp/protocol/bus0/CMakeLists.txt @@ -0,0 +1,18 @@ +# +# Copyright 2020 Staysail Systems, Inc. +# Copyright 2018 Capitar IT Group BV +# +# This software is supplied under the terms of the MIT License, a +# copy of which should be located in the distribution where this +# file was obtained (LICENSE.txt). A copy of the license may also be +# found online at https://opensource.org/licenses/MIT. +# + +# Bus protocol +nng_directory(bus0) + +nng_sources_if(NNG_PROTO_BUS0 bus.c) +nng_headers_if(NNG_PROTO_BUS0 nng/protocol/bus0/bus.h) +nng_defines_if(NNG_PROTO_BUS0 NNG_HAVE_BUS0) + +nng_test(bug1247_test) \ No newline at end of file diff --git a/src/sp/protocol/bus0/bug1247_test.c b/src/sp/protocol/bus0/bug1247_test.c new file mode 100644 index 00000000..bbc6958b --- /dev/null +++ b/src/sp/protocol/bus0/bug1247_test.c @@ -0,0 +1,35 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include + +#include + +void +test_bug1247(void) +{ + nng_socket bus1, bus2; + char * addr; + + NUTS_ADDR(addr, "tcp"); + + NUTS_PASS(nng_bus0_open(&bus1)); + NUTS_PASS(nng_bus0_open(&bus2)); + + NUTS_PASS(nng_listen(bus1, addr, NULL, 0)); + NUTS_FAIL(nng_listen(bus2, addr, NULL, 0), NNG_EADDRINUSE); + + NUTS_PASS(nng_close(bus2)); + NUTS_PASS(nng_close(bus1)); +} + +TEST_LIST = { + { "bug1247", test_bug1247 }, + { NULL, NULL }, +}; diff --git a/src/sp/protocol/bus0/bus.c b/src/sp/protocol/bus0/bus.c new file mode 100644 index 00000000..9a610ac6 --- /dev/null +++ b/src/sp/protocol/bus0/bus.c @@ -0,0 +1,466 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// Copyright 2018 Capitar IT Group BV +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include +#include + +#include "core/nng_impl.h" +#include "nng/protocol/bus0/bus.h" + +// Bus protocol. The BUS protocol, each peer sends a message to its peers. +// However, bus protocols do not "forward" (absent a device). So in order +// for each participant to receive the message, each sender must be connected +// to every other node in the network (full mesh). + +#ifndef NNI_PROTO_BUS_V0 +#define NNI_PROTO_BUS_V0 NNI_PROTO(7, 0) +#endif + +typedef struct bus0_pipe bus0_pipe; +typedef struct bus0_sock bus0_sock; + +static void bus0_sock_getq(bus0_sock *); +static void bus0_sock_send(void *, nni_aio *); +static void bus0_sock_recv(void *, nni_aio *); + +static void bus0_pipe_getq(bus0_pipe *); +static void bus0_pipe_recv(bus0_pipe *); + +static void bus0_sock_getq_cb(void *); +static void bus0_sock_getq_cb_raw(void *); +static void bus0_pipe_getq_cb(void *); +static void bus0_pipe_send_cb(void *); +static void bus0_pipe_recv_cb(void *); +static void bus0_pipe_putq_cb(void *); + +// bus0_sock is our per-socket protocol private structure. +struct bus0_sock { + nni_aio * aio_getq; + nni_list pipes; + nni_mtx mtx; + nni_msgq *uwq; + nni_msgq *urq; + bool raw; +}; + +// bus0_pipe is our per-pipe protocol private structure. +struct bus0_pipe { + nni_pipe * npipe; + bus0_sock * psock; + nni_msgq * sendq; + nni_list_node node; + nni_aio * aio_getq; + nni_aio * aio_recv; + nni_aio * aio_send; + nni_aio * aio_putq; + nni_mtx mtx; +}; + +static void +bus0_sock_fini(void *arg) +{ + bus0_sock *s = arg; + + nni_aio_free(s->aio_getq); + nni_mtx_fini(&s->mtx); +} + +static int +bus0_sock_init(void *arg, nni_sock *nsock) +{ + bus0_sock *s = arg; + int rv; + + NNI_LIST_INIT(&s->pipes, bus0_pipe, node); + nni_mtx_init(&s->mtx); + if ((rv = nni_aio_alloc(&s->aio_getq, bus0_sock_getq_cb, s)) != 0) { + bus0_sock_fini(s); + return (rv); + } + s->uwq = nni_sock_sendq(nsock); + s->urq = nni_sock_recvq(nsock); + s->raw = false; + + return (0); +} + +static int +bus0_sock_init_raw(void *arg, nni_sock *nsock) +{ + bus0_sock *s = arg; + int rv; + + NNI_LIST_INIT(&s->pipes, bus0_pipe, node); + nni_mtx_init(&s->mtx); + if ((rv = nni_aio_alloc(&s->aio_getq, bus0_sock_getq_cb_raw, s)) != + 0) { + bus0_sock_fini(s); + return (rv); + } + s->uwq = nni_sock_sendq(nsock); + s->urq = nni_sock_recvq(nsock); + s->raw = true; + + return (0); +} + +static void +bus0_sock_open(void *arg) +{ + bus0_sock *s = arg; + + bus0_sock_getq(s); +} + +static void +bus0_sock_close(void *arg) +{ + bus0_sock *s = arg; + + nni_aio_close(s->aio_getq); +} + +static void +bus0_pipe_stop(void *arg) +{ + bus0_pipe *p = arg; + + nni_aio_stop(p->aio_getq); + nni_aio_stop(p->aio_send); + nni_aio_stop(p->aio_recv); + nni_aio_stop(p->aio_putq); +} + +static void +bus0_pipe_fini(void *arg) +{ + bus0_pipe *p = arg; + + nni_aio_free(p->aio_getq); + nni_aio_free(p->aio_send); + nni_aio_free(p->aio_recv); + nni_aio_free(p->aio_putq); + nni_msgq_fini(p->sendq); + nni_mtx_fini(&p->mtx); +} + +static int +bus0_pipe_init(void *arg, nni_pipe *npipe, void *s) +{ + bus0_pipe *p = arg; + int rv; + + NNI_LIST_NODE_INIT(&p->node); + nni_mtx_init(&p->mtx); + if (((rv = nni_msgq_init(&p->sendq, 16)) != 0) || + ((rv = nni_aio_alloc(&p->aio_getq, bus0_pipe_getq_cb, p)) != 0) || + ((rv = nni_aio_alloc(&p->aio_send, bus0_pipe_send_cb, p)) != 0) || + ((rv = nni_aio_alloc(&p->aio_recv, bus0_pipe_recv_cb, p)) != 0) || + ((rv = nni_aio_alloc(&p->aio_putq, bus0_pipe_putq_cb, p)) != 0)) { + bus0_pipe_fini(p); + return (rv); + } + + p->npipe = npipe; + p->psock = s; + return (0); +} + +static int +bus0_pipe_start(void *arg) +{ + bus0_pipe *p = arg; + bus0_sock *s = p->psock; + + if (nni_pipe_peer(p->npipe) != NNI_PROTO_BUS_V0) { + // Peer protocol mismatch. + return (NNG_EPROTO); + } + + nni_mtx_lock(&s->mtx); + nni_list_append(&s->pipes, p); + nni_mtx_unlock(&s->mtx); + + bus0_pipe_recv(p); + bus0_pipe_getq(p); + + return (0); +} + +static void +bus0_pipe_close(void *arg) +{ + bus0_pipe *p = arg; + bus0_sock *s = p->psock; + + nni_aio_close(p->aio_getq); + nni_aio_close(p->aio_send); + nni_aio_close(p->aio_recv); + nni_aio_close(p->aio_putq); + nni_msgq_close(p->sendq); + + nni_mtx_lock(&s->mtx); + if (nni_list_active(&s->pipes, p)) { + nni_list_remove(&s->pipes, p); + } + nni_mtx_unlock(&s->mtx); +} + +static void +bus0_pipe_getq_cb(void *arg) +{ + bus0_pipe *p = arg; + + if (nni_aio_result(p->aio_getq) != 0) { + // closed? + nni_pipe_close(p->npipe); + return; + } + nni_aio_set_msg(p->aio_send, nni_aio_get_msg(p->aio_getq)); + nni_aio_set_msg(p->aio_getq, NULL); + + nni_pipe_send(p->npipe, p->aio_send); +} + +static void +bus0_pipe_send_cb(void *arg) +{ + bus0_pipe *p = arg; + + if (nni_aio_result(p->aio_send) != 0) { + // closed? + nni_msg_free(nni_aio_get_msg(p->aio_send)); + nni_aio_set_msg(p->aio_send, NULL); + nni_pipe_close(p->npipe); + return; + } + + bus0_pipe_getq(p); +} + +static void +bus0_pipe_recv_cb(void *arg) +{ + bus0_pipe *p = arg; + bus0_sock *s = p->psock; + nni_msg * msg; + + if (nni_aio_result(p->aio_recv) != 0) { + nni_pipe_close(p->npipe); + return; + } + msg = nni_aio_get_msg(p->aio_recv); + + if (s->raw) { + nni_msg_header_append_u32(msg, nni_pipe_id(p->npipe)); + } + + nni_msg_set_pipe(msg, nni_pipe_id(p->npipe)); + nni_aio_set_msg(p->aio_putq, msg); + nni_aio_set_msg(p->aio_recv, NULL); + nni_msgq_aio_put(s->urq, p->aio_putq); +} + +static void +bus0_pipe_putq_cb(void *arg) +{ + bus0_pipe *p = arg; + + if (nni_aio_result(p->aio_putq) != 0) { + nni_msg_free(nni_aio_get_msg(p->aio_putq)); + nni_aio_set_msg(p->aio_putq, NULL); + nni_pipe_close(p->npipe); + return; + } + + // Wait for another recv. + bus0_pipe_recv(p); +} + +static void +bus0_sock_getq_cb(void *arg) +{ + bus0_sock *s = arg; + bus0_pipe *p; + bus0_pipe *lastp; + nni_msg * msg; + nni_msg * dup; + + if (nni_aio_result(s->aio_getq) != 0) { + return; + } + + msg = nni_aio_get_msg(s->aio_getq); + + // We ignore any headers present for cooked mode. + nni_msg_header_clear(msg); + + nni_mtx_lock(&s->mtx); + lastp = nni_list_last(&s->pipes); + NNI_LIST_FOREACH (&s->pipes, p) { + if (p != lastp) { + if (nni_msg_dup(&dup, msg) != 0) { + continue; + } + } else { + dup = msg; + msg = NULL; + } + if (nni_msgq_tryput(p->sendq, dup) != 0) { + nni_msg_free(dup); + } + } + nni_mtx_unlock(&s->mtx); + nni_msg_free(msg); + + bus0_sock_getq(s); +} + +static void +bus0_sock_getq_cb_raw(void *arg) +{ + bus0_sock *s = arg; + bus0_pipe *p; + nni_msg * msg; + uint32_t sender; + + if (nni_aio_result(s->aio_getq) != 0) { + return; + } + + msg = nni_aio_get_msg(s->aio_getq); + + // The header being present indicates that the message + // was received locally and we are rebroadcasting. (Device + // is doing this probably.) In this case grab the pipe + // ID from the header, so we can exclude it. + if (nni_msg_header_len(msg) >= 4) { + sender = nni_msg_header_trim_u32(msg); + } else { + sender = 0; + } + + nni_mtx_lock(&s->mtx); + NNI_LIST_FOREACH (&s->pipes, p) { + if (nni_pipe_id(p->npipe) == sender) { + continue; + } + nni_msg_clone(msg); + if (nni_msgq_tryput(p->sendq, msg) != 0) { + nni_msg_free(msg); + } + } + nni_mtx_unlock(&s->mtx); + nni_msg_free(msg); + + bus0_sock_getq(s); +} + +static void +bus0_sock_getq(bus0_sock *s) +{ + nni_msgq_aio_get(s->uwq, s->aio_getq); +} + +static void +bus0_pipe_getq(bus0_pipe *p) +{ + nni_msgq_aio_get(p->sendq, p->aio_getq); +} + +static void +bus0_pipe_recv(bus0_pipe *p) +{ + nni_pipe_recv(p->npipe, p->aio_recv); +} + +static void +bus0_sock_send(void *arg, nni_aio *aio) +{ + bus0_sock *s = arg; + + nni_msgq_aio_put(s->uwq, aio); +} + +static void +bus0_sock_recv(void *arg, nni_aio *aio) +{ + bus0_sock *s = arg; + + nni_msgq_aio_get(s->urq, aio); +} + +static nni_proto_pipe_ops bus0_pipe_ops = { + .pipe_size = sizeof(bus0_pipe), + .pipe_init = bus0_pipe_init, + .pipe_fini = bus0_pipe_fini, + .pipe_start = bus0_pipe_start, + .pipe_close = bus0_pipe_close, + .pipe_stop = bus0_pipe_stop, +}; + +static nni_option bus0_sock_options[] = { + // terminate list + { + .o_name = NULL, + }, +}; + +static nni_proto_sock_ops bus0_sock_ops = { + .sock_size = sizeof(bus0_sock), + .sock_init = bus0_sock_init, + .sock_fini = bus0_sock_fini, + .sock_open = bus0_sock_open, + .sock_close = bus0_sock_close, + .sock_send = bus0_sock_send, + .sock_recv = bus0_sock_recv, + .sock_options = bus0_sock_options, +}; + +static nni_proto_sock_ops bus0_sock_ops_raw = { + .sock_size = sizeof(bus0_sock), + .sock_init = bus0_sock_init_raw, + .sock_fini = bus0_sock_fini, + .sock_open = bus0_sock_open, + .sock_close = bus0_sock_close, + .sock_send = bus0_sock_send, + .sock_recv = bus0_sock_recv, + .sock_options = bus0_sock_options, +}; + +static nni_proto bus0_proto = { + .proto_version = NNI_PROTOCOL_VERSION, + .proto_self = { NNI_PROTO_BUS_V0, "bus" }, + .proto_peer = { NNI_PROTO_BUS_V0, "bus" }, + .proto_flags = NNI_PROTO_FLAG_SNDRCV, + .proto_sock_ops = &bus0_sock_ops, + .proto_pipe_ops = &bus0_pipe_ops, +}; + +static nni_proto bus0_proto_raw = { + .proto_version = NNI_PROTOCOL_VERSION, + .proto_self = { NNI_PROTO_BUS_V0, "bus" }, + .proto_peer = { NNI_PROTO_BUS_V0, "bus" }, + .proto_flags = NNI_PROTO_FLAG_SNDRCV | NNI_PROTO_FLAG_RAW, + .proto_sock_ops = &bus0_sock_ops_raw, + .proto_pipe_ops = &bus0_pipe_ops, +}; + +int +nng_bus0_open(nng_socket *sidp) +{ + return (nni_proto_open(sidp, &bus0_proto)); +} + +int +nng_bus0_open_raw(nng_socket *sidp) +{ + return (nni_proto_open(sidp, &bus0_proto_raw)); +} diff --git a/src/sp/protocol/pair0/CMakeLists.txt b/src/sp/protocol/pair0/CMakeLists.txt new file mode 100644 index 00000000..b12583ab --- /dev/null +++ b/src/sp/protocol/pair0/CMakeLists.txt @@ -0,0 +1,16 @@ +# +# Copyright 2020 Staysail Systems, Inc. +# Copyright 2018 Capitar IT Group BV +# +# This software is supplied under the terms of the MIT License, a +# copy of which should be located in the distribution where this +# file was obtained (LICENSE.txt). A copy of the license may also be +# found online at https://opensource.org/licenses/MIT. +# + +# PAIRv0 protocol +nng_directory(pair0) + +nng_sources_if(NNG_PROTO_PAIR0 pair.c) +nng_headers_if(NNG_PROTO_PAIR0 nng/protocol/pair0/pair.h) +nng_defines_if(NNG_PROTO_PAIR0 NNG_HAVE_PAIR0) \ No newline at end of file diff --git a/src/sp/protocol/pair0/pair.c b/src/sp/protocol/pair0/pair.c new file mode 100644 index 00000000..41f88c7c --- /dev/null +++ b/src/sp/protocol/pair0/pair.c @@ -0,0 +1,305 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// Copyright 2018 Capitar IT Group BV +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include + +#include "core/nng_impl.h" +#include "nng/protocol/pair0/pair.h" + +// Pair protocol. The PAIR protocol is a simple 1:1 messaging pattern. +// While a peer is connected to the server, all other peer connection +// attempts are discarded. + +#ifndef NNI_PROTO_PAIR_V0 +#define NNI_PROTO_PAIR_V0 NNI_PROTO(1, 0) +#endif + +typedef struct pair0_pipe pair0_pipe; +typedef struct pair0_sock pair0_sock; + +static void pair0_send_cb(void *); +static void pair0_recv_cb(void *); +static void pair0_getq_cb(void *); +static void pair0_putq_cb(void *); +static void pair0_pipe_fini(void *); + +// pair0_sock is our per-socket protocol private structure. +struct pair0_sock { + pair0_pipe *ppipe; + nni_msgq * uwq; + nni_msgq * urq; + nni_mtx mtx; +}; + +// An pair0_pipe is our per-pipe protocol private structure. We keep +// one of these even though in theory we'd only have a single underlying +// pipe. The separate data structure is more like other protocols that do +// manage multiple pipes. +struct pair0_pipe { + nni_pipe * npipe; + pair0_sock *psock; + nni_aio aio_send; + nni_aio aio_recv; + nni_aio aio_getq; + nni_aio aio_putq; +}; + +static int +pair0_sock_init(void *arg, nni_sock *nsock) +{ + pair0_sock *s = arg; + + nni_mtx_init(&s->mtx); + s->ppipe = NULL; + s->uwq = nni_sock_sendq(nsock); + s->urq = nni_sock_recvq(nsock); + return (0); +} + +static void +pair0_sock_fini(void *arg) +{ + pair0_sock *s = arg; + + nni_mtx_fini(&s->mtx); +} + +static void +pair0_pipe_stop(void *arg) +{ + pair0_pipe *p = arg; + + nni_aio_stop(&p->aio_send); + nni_aio_stop(&p->aio_recv); + nni_aio_stop(&p->aio_putq); + nni_aio_stop(&p->aio_getq); +} + +static void +pair0_pipe_fini(void *arg) +{ + pair0_pipe *p = arg; + + nni_aio_fini(&p->aio_send); + nni_aio_fini(&p->aio_recv); + nni_aio_fini(&p->aio_putq); + nni_aio_fini(&p->aio_getq); +} + +static int +pair0_pipe_init(void *arg, nni_pipe *npipe, void *psock) +{ + pair0_pipe *p = arg; + + nni_aio_init(&p->aio_send, pair0_send_cb, p); + nni_aio_init(&p->aio_recv, pair0_recv_cb, p); + nni_aio_init(&p->aio_getq, pair0_getq_cb, p); + nni_aio_init(&p->aio_putq, pair0_putq_cb, p); + + p->npipe = npipe; + p->psock = psock; + return (0); +} + +static int +pair0_pipe_start(void *arg) +{ + pair0_pipe *p = arg; + pair0_sock *s = p->psock; + + if (nni_pipe_peer(p->npipe) != NNI_PROTO_PAIR_V0) { + // Peer protocol mismatch. + return (NNG_EPROTO); + } + + nni_mtx_lock(&s->mtx); + if (s->ppipe != NULL) { + nni_mtx_unlock(&s->mtx); + return (NNG_EBUSY); // Already have a peer, denied. + } + s->ppipe = p; + nni_mtx_unlock(&s->mtx); + + // Schedule a getq on the upper, and a read from the pipe. + // Each of these also sets up another hold on the pipe itself. + nni_msgq_aio_get(s->uwq, &p->aio_getq); + nni_pipe_recv(p->npipe, &p->aio_recv); + + return (0); +} + +static void +pair0_pipe_close(void *arg) +{ + pair0_pipe *p = arg; + pair0_sock *s = p->psock; + + nni_aio_close(&p->aio_send); + nni_aio_close(&p->aio_recv); + nni_aio_close(&p->aio_putq); + nni_aio_close(&p->aio_getq); + + nni_mtx_lock(&s->mtx); + if (s->ppipe == p) { + s->ppipe = NULL; + } + nni_mtx_unlock(&s->mtx); +} + +static void +pair0_recv_cb(void *arg) +{ + pair0_pipe *p = arg; + pair0_sock *s = p->psock; + nni_msg * msg; + + if (nni_aio_result(&p->aio_recv) != 0) { + nni_pipe_close(p->npipe); + return; + } + + msg = nni_aio_get_msg(&p->aio_recv); + nni_aio_set_msg(&p->aio_putq, msg); + nni_aio_set_msg(&p->aio_recv, NULL); + + nni_msg_set_pipe(msg, nni_pipe_id(p->npipe)); + nni_msgq_aio_put(s->urq, &p->aio_putq); +} + +static void +pair0_putq_cb(void *arg) +{ + pair0_pipe *p = arg; + + if (nni_aio_result(&p->aio_putq) != 0) { + nni_msg_free(nni_aio_get_msg(&p->aio_putq)); + nni_aio_set_msg(&p->aio_putq, NULL); + nni_pipe_close(p->npipe); + return; + } + nni_pipe_recv(p->npipe, &p->aio_recv); +} + +static void +pair0_getq_cb(void *arg) +{ + pair0_pipe *p = arg; + + if (nni_aio_result(&p->aio_getq) != 0) { + nni_pipe_close(p->npipe); + return; + } + + nni_aio_set_msg(&p->aio_send, nni_aio_get_msg(&p->aio_getq)); + nni_aio_set_msg(&p->aio_getq, NULL); + nni_pipe_send(p->npipe, &p->aio_send); +} + +static void +pair0_send_cb(void *arg) +{ + pair0_pipe *p = arg; + pair0_sock *s = p->psock; + + if (nni_aio_result(&p->aio_send) != 0) { + nni_msg_free(nni_aio_get_msg(&p->aio_send)); + nni_aio_set_msg(&p->aio_send, NULL); + nni_pipe_close(p->npipe); + return; + } + + nni_msgq_aio_get(s->uwq, &p->aio_getq); +} + +static void +pair0_sock_open(void *arg) +{ + NNI_ARG_UNUSED(arg); +} + +static void +pair0_sock_close(void *arg) +{ + NNI_ARG_UNUSED(arg); +} + +static void +pair0_sock_send(void *arg, nni_aio *aio) +{ + pair0_sock *s = arg; + + nni_msgq_aio_put(s->uwq, aio); +} + +static void +pair0_sock_recv(void *arg, nni_aio *aio) +{ + pair0_sock *s = arg; + + nni_msgq_aio_get(s->urq, aio); +} + +static nni_proto_pipe_ops pair0_pipe_ops = { + .pipe_size = sizeof(pair0_pipe), + .pipe_init = pair0_pipe_init, + .pipe_fini = pair0_pipe_fini, + .pipe_start = pair0_pipe_start, + .pipe_close = pair0_pipe_close, + .pipe_stop = pair0_pipe_stop, +}; + +static nni_option pair0_sock_options[] = { + // terminate list + { + .o_name = NULL, + } +}; + +static nni_proto_sock_ops pair0_sock_ops = { + .sock_size = sizeof(pair0_sock), + .sock_init = pair0_sock_init, + .sock_fini = pair0_sock_fini, + .sock_open = pair0_sock_open, + .sock_close = pair0_sock_close, + .sock_send = pair0_sock_send, + .sock_recv = pair0_sock_recv, + .sock_options = pair0_sock_options, +}; + +// Legacy protocol (v0) +static nni_proto pair0_proto = { + .proto_version = NNI_PROTOCOL_VERSION, + .proto_self = { NNI_PROTO_PAIR_V0, "pair" }, + .proto_peer = { NNI_PROTO_PAIR_V0, "pair" }, + .proto_flags = NNI_PROTO_FLAG_SNDRCV, + .proto_sock_ops = &pair0_sock_ops, + .proto_pipe_ops = &pair0_pipe_ops, +}; + +static nni_proto pair0_proto_raw = { + .proto_version = NNI_PROTOCOL_VERSION, + .proto_self = { NNI_PROTO_PAIR_V0, "pair" }, + .proto_peer = { NNI_PROTO_PAIR_V0, "pair" }, + .proto_flags = NNI_PROTO_FLAG_SNDRCV | NNI_PROTO_FLAG_RAW, + .proto_sock_ops = &pair0_sock_ops, + .proto_pipe_ops = &pair0_pipe_ops, +}; + +int +nng_pair0_open(nng_socket *sidp) +{ + return (nni_proto_open(sidp, &pair0_proto)); +} + +int +nng_pair0_open_raw(nng_socket *sidp) +{ + return (nni_proto_open(sidp, &pair0_proto_raw)); +} diff --git a/src/sp/protocol/pair1/CMakeLists.txt b/src/sp/protocol/pair1/CMakeLists.txt new file mode 100644 index 00000000..12e12607 --- /dev/null +++ b/src/sp/protocol/pair1/CMakeLists.txt @@ -0,0 +1,20 @@ +# +# Copyright 2019 Staysail Systems, Inc. +# Copyright 2018 Capitar IT Group BV +# +# This software is supplied under the terms of the MIT License, a +# copy of which should be located in the distribution where this +# file was obtained (LICENSE.txt). A copy of the license may also be +# found online at https://opensource.org/licenses/MIT. +# + +# PAIRv1 protocol +nng_directory(pair1) + +# XXX: break pair1_poly into an ifdef. +nng_sources_if(NNG_PROTO_PAIR1 pair.c pair1_poly.c) +nng_headers_if(NNG_PROTO_PAIR1 nng/protocol/pair1/pair.h) +nng_defines_if(NNG_PROTO_PAIR1 NNG_HAVE_PAIR1) + +nng_test(pair1_test) +nng_test(pair1_poly_test) \ No newline at end of file diff --git a/src/sp/protocol/pair1/pair.c b/src/sp/protocol/pair1/pair.c new file mode 100644 index 00000000..ba497c42 --- /dev/null +++ b/src/sp/protocol/pair1/pair.c @@ -0,0 +1,540 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// Copyright 2018 Capitar IT Group BV +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include + +#include "core/nng_impl.h" +#include "nng/protocol/pair1/pair.h" + +// Pair protocol. The PAIRv1 protocol is a simple 1:1 messaging pattern. + +#ifdef NNG_ENABLE_STATS +#define BUMP_STAT(x) nni_stat_inc(x, 1) +#else +#define BUMP_STAT(x) +#endif + +typedef struct pair1_pipe pair1_pipe; +typedef struct pair1_sock pair1_sock; + +static void pair1_pipe_send_cb(void *); +static void pair1_pipe_recv_cb(void *); +static void pair1_pipe_get_cb(void *); +static void pair1_pipe_put_cb(void *); +static void pair1_pipe_fini(void *); + +// pair1_sock is our per-socket protocol private structure. +struct pair1_sock { + nni_msgq * uwq; + nni_msgq * urq; + nni_sock * sock; + bool raw; + pair1_pipe * p; + nni_atomic_int ttl; + nni_mtx mtx; +#ifdef NNG_ENABLE_STATS + nni_stat_item stat_poly; + nni_stat_item stat_raw; + nni_stat_item stat_reject_mismatch; + nni_stat_item stat_reject_already; + nni_stat_item stat_ttl_drop; + nni_stat_item stat_rx_malformed; + nni_stat_item stat_tx_malformed; + nni_stat_item stat_tx_drop; +#endif +#ifdef NNG_TEST_LIB + bool inject_header; +#endif +}; + +// pair1_pipe is our per-pipe protocol private structure. +struct pair1_pipe { + nni_pipe * pipe; + pair1_sock *pair; + nni_aio aio_send; + nni_aio aio_recv; + nni_aio aio_get; + nni_aio aio_put; +}; + +static void +pair1_sock_fini(void *arg) +{ + pair1_sock *s = arg; + + nni_mtx_fini(&s->mtx); +} + +#ifdef NNG_ENABLE_STATS +static void +pair1_add_sock_stat( + pair1_sock *s, nni_stat_item *item, const nni_stat_info *info) +{ + nni_stat_init(item, info); + nni_sock_add_stat(s->sock, item); +} +#endif + +static int +pair1_sock_init_impl(void *arg, nni_sock *sock, bool raw) +{ + pair1_sock *s = arg; + + // Raw mode uses this. + nni_mtx_init(&s->mtx); + s->sock = sock; + +#ifdef NNG_ENABLE_STATS + static const nni_stat_info poly_info = { + .si_name = "poly", + .si_desc = "polyamorous mode?", + .si_type = NNG_STAT_BOOLEAN, + }; + static const nni_stat_info raw_info = { + .si_name = "raw", + .si_desc = "raw mode?", + .si_type = NNG_STAT_BOOLEAN, + }; + static const nni_stat_info mismatch_info = { + .si_name = "mismatch", + .si_desc = "pipes rejected (protocol mismatch)", + .si_type = NNG_STAT_COUNTER, + .si_atomic = true, + }; + static const nni_stat_info already_info = { + .si_name = "already", + .si_desc = "pipes rejected (already connected)", + .si_type = NNG_STAT_COUNTER, + .si_atomic = true, + }; + static const nni_stat_info ttl_drop_info = { + .si_name = "ttl_drop", + .si_desc = "messages dropped due to too many hops", + .si_type = NNG_STAT_COUNTER, + .si_unit = NNG_UNIT_MESSAGES, + .si_atomic = true, + }; + static const nni_stat_info tx_drop_info = { + .si_name = "tx_drop", + .si_desc = "messages dropped undeliverable", + .si_type = NNG_STAT_COUNTER, + .si_unit = NNG_UNIT_MESSAGES, + .si_atomic = true, + }; + static const nni_stat_info rx_malformed_info = { + .si_name = "rx_malformed", + .si_desc = "malformed messages received", + .si_type = NNG_STAT_COUNTER, + .si_unit = NNG_UNIT_MESSAGES, + .si_atomic = true, + }; + static const nni_stat_info tx_malformed_info = { + .si_name = "tx_malformed", + .si_desc = "malformed messages not sent", + .si_type = NNG_STAT_COUNTER, + .si_unit = NNG_UNIT_MESSAGES, + .si_atomic = true, + }; + + pair1_add_sock_stat(s, &s->stat_poly, &poly_info); + pair1_add_sock_stat(s, &s->stat_raw, &raw_info); + pair1_add_sock_stat(s, &s->stat_reject_mismatch, &mismatch_info); + pair1_add_sock_stat(s, &s->stat_reject_already, &already_info); + pair1_add_sock_stat(s, &s->stat_ttl_drop, &ttl_drop_info); + pair1_add_sock_stat(s, &s->stat_tx_drop, &tx_drop_info); + pair1_add_sock_stat(s, &s->stat_rx_malformed, &rx_malformed_info); + + if (raw) { + // This stat only makes sense in raw mode. + pair1_add_sock_stat( + s, &s->stat_tx_malformed, &tx_malformed_info); + } + + nni_stat_set_bool(&s->stat_raw, raw); + nni_stat_set_bool(&s->stat_poly, false); +#endif + + s->raw = raw; + s->uwq = nni_sock_sendq(sock); + s->urq = nni_sock_recvq(sock); + nni_atomic_init(&s->ttl); + nni_atomic_set(&s->ttl, 8); + + return (0); +} + +static int +pair1_sock_init(void *arg, nni_sock *sock) +{ + return (pair1_sock_init_impl(arg, sock, false)); +} + +static int +pair1_sock_init_raw(void *arg, nni_sock *sock) +{ + return (pair1_sock_init_impl(arg, sock, true)); +} + +static void +pair1_pipe_stop(void *arg) +{ + pair1_pipe *p = arg; + pair1_sock *s = p->pair; + + nni_mtx_lock(&s->mtx); + if (s->p == p) { + s->p = NULL; + } + nni_mtx_unlock(&s->mtx); + nni_aio_stop(&p->aio_send); + nni_aio_stop(&p->aio_recv); + nni_aio_stop(&p->aio_put); + nni_aio_stop(&p->aio_get); +} + +static void +pair1_pipe_fini(void *arg) +{ + pair1_pipe *p = arg; + + nni_aio_fini(&p->aio_send); + nni_aio_fini(&p->aio_recv); + nni_aio_fini(&p->aio_put); + nni_aio_fini(&p->aio_get); +} + +static int +pair1_pipe_init(void *arg, nni_pipe *pipe, void *pair) +{ + pair1_pipe *p = arg; + + nni_aio_init(&p->aio_send, pair1_pipe_send_cb, p); + nni_aio_init(&p->aio_recv, pair1_pipe_recv_cb, p); + nni_aio_init(&p->aio_get, pair1_pipe_get_cb, p); + nni_aio_init(&p->aio_put, pair1_pipe_put_cb, p); + + p->pipe = pipe; + p->pair = pair; + + return (0); +} + +static int +pair1_pipe_start(void *arg) +{ + pair1_pipe *p = arg; + pair1_sock *s = p->pair; + + if (nni_pipe_peer(p->pipe) != NNG_PAIR1_PEER) { + BUMP_STAT(&s->stat_reject_mismatch); + // Peer protocol mismatch. + return (NNG_EPROTO); + } + + nni_mtx_lock(&s->mtx); + if (s->p != NULL) { + nni_mtx_unlock(&s->mtx); + BUMP_STAT(&s->stat_reject_already); + return (NNG_EBUSY); + } + s->p = p; + nni_mtx_unlock(&s->mtx); + + // Schedule a get. + nni_msgq_aio_get(s->uwq, &p->aio_get); + + // And the pipe read of course. + nni_pipe_recv(p->pipe, &p->aio_recv); + + return (0); +} + +static void +pair1_pipe_close(void *arg) +{ + pair1_pipe *p = arg; + + nni_aio_close(&p->aio_send); + nni_aio_close(&p->aio_recv); + nni_aio_close(&p->aio_put); + nni_aio_close(&p->aio_get); +} + +static void +pair1_pipe_recv_cb(void *arg) +{ + pair1_pipe *p = arg; + pair1_sock *s = p->pair; + nni_msg * msg; + uint32_t hdr; + nni_pipe * pipe = p->pipe; + size_t len; + + if (nni_aio_result(&p->aio_recv) != 0) { + nni_pipe_close(p->pipe); + return; + } + + msg = nni_aio_get_msg(&p->aio_recv); + nni_aio_set_msg(&p->aio_recv, NULL); + + // Store the pipe ID. + nni_msg_set_pipe(msg, nni_pipe_id(p->pipe)); + + // If the message is missing the hop count header, scrap it. + if ((nni_msg_len(msg) < sizeof(uint32_t)) || + ((hdr = nni_msg_trim_u32(msg)) > 0xff)) { + BUMP_STAT(&s->stat_rx_malformed); + nni_msg_free(msg); + nni_pipe_close(pipe); + return; + } + + len = nni_msg_len(msg); + + // If we bounced too many times, discard the message, but + // keep getting more. + if ((int) hdr > nni_atomic_get(&s->ttl)) { + BUMP_STAT(&s->stat_ttl_drop); + nni_msg_free(msg); + nni_pipe_recv(pipe, &p->aio_recv); + return; + } + + // Store the hop count in the header. + nni_msg_header_append_u32(msg, hdr); + + // Send the message up. + nni_aio_set_msg(&p->aio_put, msg); + nni_sock_bump_rx(s->sock, len); + nni_msgq_aio_put(s->urq, &p->aio_put); +} + +static void +pair1_pipe_put_cb(void *arg) +{ + pair1_pipe *p = arg; + + if (nni_aio_result(&p->aio_put) != 0) { + nni_msg_free(nni_aio_get_msg(&p->aio_put)); + nni_aio_set_msg(&p->aio_put, NULL); + nni_pipe_close(p->pipe); + return; + } + nni_pipe_recv(p->pipe, &p->aio_recv); +} + +static void +pair1_pipe_get_cb(void *arg) +{ + pair1_pipe *p = arg; + pair1_sock *s = p->pair; + nni_msg * msg; + uint32_t hops; + + if (nni_aio_result(&p->aio_get) != 0) { + nni_pipe_close(p->pipe); + return; + } + + msg = nni_aio_get_msg(&p->aio_get); + nni_aio_set_msg(&p->aio_get, NULL); + + // Raw mode messages have the header already formed, with a hop count. + // Cooked mode messages have no header so we have to add one. + if (s->raw) { + if ((nni_msg_header_len(msg) != sizeof(uint32_t)) || + ((hops = nni_msg_header_trim_u32(msg)) > 254)) { + BUMP_STAT(&s->stat_tx_malformed); + nni_msg_free(msg); + nni_msgq_aio_get(s->uwq, &p->aio_get); + return; + } +#if NNG_TEST_LIB + } else if (s->inject_header) { + nni_aio_set_msg(&p->aio_send, msg); + nni_pipe_send(p->pipe, &p->aio_send); + return; +#endif + } else { + // Strip off any previously existing header, such as when + // replying to messages. + nni_msg_header_clear(msg); + hops = 0; + } + + hops++; + + // Insert the hops header. + nni_msg_header_append_u32(msg, hops); + + nni_aio_set_msg(&p->aio_send, msg); + nni_pipe_send(p->pipe, &p->aio_send); +} + +static void +pair1_pipe_send_cb(void *arg) +{ + pair1_pipe *p = arg; + pair1_sock *s = p->pair; + + if (nni_aio_result(&p->aio_send) != 0) { + nni_msg_free(nni_aio_get_msg(&p->aio_send)); + nni_aio_set_msg(&p->aio_send, NULL); + nni_pipe_close(p->pipe); + return; + } + + nni_msgq_aio_get(s->uwq, &p->aio_get); +} + +static void +pair1_sock_open(void *arg) +{ + NNI_ARG_UNUSED(arg); +} + +static void +pair1_sock_close(void *arg) +{ + NNI_ARG_UNUSED(arg); +} + +static int +pair1_sock_set_max_ttl(void *arg, const void *buf, size_t sz, nni_opt_type t) +{ + pair1_sock *s = arg; + int rv; + int ttl; + + if ((rv = nni_copyin_int(&ttl, buf, sz, 1, NNI_MAX_MAX_TTL, t)) == 0) { + nni_atomic_set(&s->ttl, ttl); + } + + return (rv); +} + +static int +pair1_sock_get_max_ttl(void *arg, void *buf, size_t *szp, nni_opt_type t) +{ + pair1_sock *s = arg; + return (nni_copyout_int(nni_atomic_get(&s->ttl), buf, szp, t)); +} + +#ifdef NNG_TEST_LIB +static int +pair1_set_test_inject_header(void *arg, const void *buf, size_t sz, nni_type t) +{ + pair1_sock *s = arg; + int rv; + nni_mtx_lock(&s->mtx); + rv = nni_copyin_bool(&s->inject_header, buf, sz, t); + nni_mtx_unlock(&s->mtx); + return (rv); +} +#endif + +static void +pair1_sock_send(void *arg, nni_aio *aio) +{ + pair1_sock *s = arg; + + nni_sock_bump_tx(s->sock, nni_msg_len(nni_aio_get_msg(aio))); + nni_msgq_aio_put(s->uwq, aio); +} + +static void +pair1_sock_recv(void *arg, nni_aio *aio) +{ + pair1_sock *s = arg; + + nni_msgq_aio_get(s->urq, aio); +} + +static nni_proto_pipe_ops pair1_pipe_ops = { + .pipe_size = sizeof(pair1_pipe), + .pipe_init = pair1_pipe_init, + .pipe_fini = pair1_pipe_fini, + .pipe_start = pair1_pipe_start, + .pipe_close = pair1_pipe_close, + .pipe_stop = pair1_pipe_stop, +}; + +static nni_option pair1_sock_options[] = { + { + .o_name = NNG_OPT_MAXTTL, + .o_get = pair1_sock_get_max_ttl, + .o_set = pair1_sock_set_max_ttl, + }, +#ifdef NNG_TEST_LIB + { + // Test only option to pass header unmolested. This allows + // us to inject bad header contents. + .o_name = "pair1_test_inject_header", + .o_set = pair1_set_test_inject_header, + }, +#endif + // terminate list + { + .o_name = NULL, + }, +}; + +static nni_proto_sock_ops pair1_sock_ops = { + .sock_size = sizeof(pair1_sock), + .sock_init = pair1_sock_init, + .sock_fini = pair1_sock_fini, + .sock_open = pair1_sock_open, + .sock_close = pair1_sock_close, + .sock_recv = pair1_sock_recv, + .sock_send = pair1_sock_send, + .sock_options = pair1_sock_options, +}; + +static nni_proto pair1_proto = { + .proto_version = NNI_PROTOCOL_VERSION, + .proto_self = { NNG_PAIR1_SELF, NNG_PAIR1_SELF_NAME }, + .proto_peer = { NNG_PAIR1_PEER, NNG_PAIR1_PEER_NAME }, + .proto_flags = NNI_PROTO_FLAG_SNDRCV, + .proto_sock_ops = &pair1_sock_ops, + .proto_pipe_ops = &pair1_pipe_ops, +}; + +int +nng_pair1_open(nng_socket *sock) +{ + return (nni_proto_open(sock, &pair1_proto)); +} + +static nni_proto_sock_ops pair1_sock_ops_raw = { + .sock_size = sizeof(pair1_sock), + .sock_init = pair1_sock_init_raw, + .sock_fini = pair1_sock_fini, + .sock_open = pair1_sock_open, + .sock_close = pair1_sock_close, + .sock_recv = pair1_sock_recv, + .sock_send = pair1_sock_send, + .sock_options = pair1_sock_options, +}; + +static nni_proto pair1_proto_raw = { + .proto_version = NNI_PROTOCOL_VERSION, + .proto_self = { NNG_PAIR1_SELF, NNG_PAIR1_SELF_NAME }, + .proto_peer = { NNG_PAIR1_PEER, NNG_PAIR1_PEER_NAME }, + .proto_flags = NNI_PROTO_FLAG_SNDRCV | NNI_PROTO_FLAG_RAW, + .proto_sock_ops = &pair1_sock_ops_raw, + .proto_pipe_ops = &pair1_pipe_ops, +}; + +int +nng_pair1_open_raw(nng_socket *sock) +{ + return (nni_proto_open(sock, &pair1_proto_raw)); +} diff --git a/src/sp/protocol/pair1/pair1_poly.c b/src/sp/protocol/pair1/pair1_poly.c new file mode 100644 index 00000000..6c16745c --- /dev/null +++ b/src/sp/protocol/pair1/pair1_poly.c @@ -0,0 +1,535 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// Copyright 2018 Capitar IT Group BV +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include + +#include "core/nng_impl.h" +#include "nng/protocol/pair1/pair.h" + +// Pair1 polyamorous mode. The PAIRv1 protocol is normally a simple 1:1 +// messaging pattern, but this mode offers the ability to use a best-effort +// multicast type of communication. There are limitations however. +// Most notably this does not interact well with nng_device type +// proxies, and there is no support for raw mode. + +// THIS FEATURE IS DEPRECATED. We discourage use in new applications. + +#ifdef NNG_ENABLE_STATS +#define BUMP_STAT(x) nni_stat_inc(x, 1) +#else +#define BUMP_STAT(x) +#endif + +typedef struct pair1poly_pipe pair1poly_pipe; +typedef struct pair1poly_sock pair1poly_sock; + +static void pair1poly_sock_get_cb(void *); +static void pair1poly_pipe_send_cb(void *); +static void pair1poly_pipe_recv_cb(void *); +static void pair1poly_pipe_get_cb(void *); +static void pair1poly_pipe_put_cb(void *); +static void pair1poly_pipe_fini(void *); + +// pair1poly_sock is our per-socket protocol private structure. +struct pair1poly_sock { + nni_msgq * uwq; + nni_msgq * urq; + nni_sock * sock; + nni_atomic_int ttl; + nni_mtx mtx; + nni_id_map pipes; + nni_list plist; + bool started; + nni_aio aio_get; + nni_stat_item stat_poly; + nni_stat_item stat_raw; + nni_stat_item stat_reject_mismatch; + nni_stat_item stat_reject_already; + nni_stat_item stat_ttl_drop; + nni_stat_item stat_rx_malformed; + nni_stat_item stat_tx_malformed; + nni_stat_item stat_tx_drop; +}; + +// pair1poly_pipe is our per-pipe protocol private structure. +struct pair1poly_pipe { + nni_pipe * pipe; + pair1poly_sock *pair; + nni_msgq * send_queue; + nni_aio aio_send; + nni_aio aio_recv; + nni_aio aio_get; + nni_aio aio_put; + nni_list_node node; +}; + +static void +pair1poly_sock_fini(void *arg) +{ + pair1poly_sock *s = arg; + + nni_aio_fini(&s->aio_get); + nni_id_map_fini(&s->pipes); + nni_mtx_fini(&s->mtx); +} + +#ifdef NNG_ENABLE_STATS +static void +pair1_add_sock_stat( + pair1poly_sock *s, nni_stat_item *item, const nni_stat_info *info) +{ + nni_stat_init(item, info); + nni_sock_add_stat(s->sock, item); +} +#endif + +static int +pair1poly_sock_init(void *arg, nni_sock *sock) +{ + pair1poly_sock *s = arg; + + nni_id_map_init(&s->pipes, 0, 0, false); + NNI_LIST_INIT(&s->plist, pair1poly_pipe, node); + s->sock = sock; + + // Raw mode uses this. + nni_mtx_init(&s->mtx); + + nni_aio_init(&s->aio_get, pair1poly_sock_get_cb, s); + +#ifdef NNG_ENABLE_STATS + static const nni_stat_info poly_info = { + .si_name = "poly", + .si_desc = "polyamorous mode?", + .si_type = NNG_STAT_BOOLEAN, + }; + static const nni_stat_info raw_info = { + .si_name = "raw", + .si_desc = "raw mode?", + .si_type = NNG_STAT_BOOLEAN, + }; + static const nni_stat_info mismatch_info = { + .si_name = "mismatch", + .si_desc = "pipes rejected (protocol mismatch)", + .si_type = NNG_STAT_COUNTER, + .si_atomic = true, + }; + static const nni_stat_info already_info = { + .si_name = "already", + .si_desc = "pipes rejected (already connected)", + .si_type = NNG_STAT_COUNTER, + .si_atomic = true, + }; + static const nni_stat_info ttl_drop_info = { + .si_name = "ttl_drop", + .si_desc = "messages dropped due to too many hops", + .si_type = NNG_STAT_COUNTER, + .si_unit = NNG_UNIT_MESSAGES, + .si_atomic = true, + }; + static const nni_stat_info tx_drop_info = { + .si_name = "tx_drop", + .si_desc = "messages dropped undeliverable", + .si_type = NNG_STAT_COUNTER, + .si_unit = NNG_UNIT_MESSAGES, + .si_atomic = true, + }; + static const nni_stat_info rx_malformed_info = { + .si_name = "rx_malformed", + .si_desc = "malformed messages received", + .si_type = NNG_STAT_COUNTER, + .si_unit = NNG_UNIT_MESSAGES, + .si_atomic = true, + }; + static const nni_stat_info tx_malformed_info = { + .si_name = "tx_malformed", + .si_desc = "malformed messages not sent", + .si_type = NNG_STAT_COUNTER, + .si_unit = NNG_UNIT_MESSAGES, + .si_atomic = true, + }; + + pair1_add_sock_stat(s, &s->stat_poly, &poly_info); + pair1_add_sock_stat(s, &s->stat_raw, &raw_info); + pair1_add_sock_stat(s, &s->stat_reject_mismatch, &mismatch_info); + pair1_add_sock_stat(s, &s->stat_reject_already, &already_info); + pair1_add_sock_stat(s, &s->stat_ttl_drop, &ttl_drop_info); + pair1_add_sock_stat(s, &s->stat_tx_drop, &tx_drop_info); + pair1_add_sock_stat(s, &s->stat_rx_malformed, &rx_malformed_info); + pair1_add_sock_stat(s, &s->stat_tx_malformed, &tx_malformed_info); + + nni_stat_set_bool(&s->stat_raw, false); + nni_stat_set_bool(&s->stat_poly, true); +#endif + + s->uwq = nni_sock_sendq(sock); + s->urq = nni_sock_recvq(sock); + nni_atomic_init(&s->ttl); + nni_atomic_set(&s->ttl, 8); + + return (0); +} + +static void +pair1poly_pipe_stop(void *arg) +{ + pair1poly_pipe *p = arg; + + nni_aio_stop(&p->aio_send); + nni_aio_stop(&p->aio_recv); + nni_aio_stop(&p->aio_put); + nni_aio_stop(&p->aio_get); +} + +static void +pair1poly_pipe_fini(void *arg) +{ + pair1poly_pipe *p = arg; + + nni_aio_fini(&p->aio_send); + nni_aio_fini(&p->aio_recv); + nni_aio_fini(&p->aio_put); + nni_aio_fini(&p->aio_get); + nni_msgq_fini(p->send_queue); +} + +static int +pair1poly_pipe_init(void *arg, nni_pipe *pipe, void *pair) +{ + pair1poly_pipe *p = arg; + int rv; + + nni_aio_init(&p->aio_send, pair1poly_pipe_send_cb, p); + nni_aio_init(&p->aio_recv, pair1poly_pipe_recv_cb, p); + nni_aio_init(&p->aio_get, pair1poly_pipe_get_cb, p); + nni_aio_init(&p->aio_put, pair1poly_pipe_put_cb, p); + + if ((rv = nni_msgq_init(&p->send_queue, 2)) != 0) { + pair1poly_pipe_fini(p); + return (rv); + } + + p->pipe = pipe; + p->pair = pair; + + return (0); +} + +static int +pair1poly_pipe_start(void *arg) +{ + pair1poly_pipe *p = arg; + pair1poly_sock *s = p->pair; + uint32_t id; + int rv; + + nni_mtx_lock(&s->mtx); + if (nni_pipe_peer(p->pipe) != NNG_PAIR1_PEER) { + nni_mtx_unlock(&s->mtx); + BUMP_STAT(&s->stat_reject_mismatch); + // Peer protocol mismatch. + return (NNG_EPROTO); + } + + id = nni_pipe_id(p->pipe); + if ((rv = nni_id_set(&s->pipes, id, p)) != 0) { + nni_mtx_unlock(&s->mtx); + return (rv); + } + if (!s->started) { + nni_msgq_aio_get(s->uwq, &s->aio_get); + } + nni_list_append(&s->plist, p); + s->started = true; + nni_mtx_unlock(&s->mtx); + + // Schedule a get. In polyamorous mode we get on the per pipe + // send_queue, as the socket distributes to us. In monogamous mode + // we bypass and get from the upper write queue directly (saving a + // set of context switches). + nni_msgq_aio_get(p->send_queue, &p->aio_get); + + // And the pipe read of course. + nni_pipe_recv(p->pipe, &p->aio_recv); + + return (0); +} + +static void +pair1poly_pipe_close(void *arg) +{ + pair1poly_pipe *p = arg; + pair1poly_sock *s = p->pair; + + nni_aio_close(&p->aio_send); + nni_aio_close(&p->aio_recv); + nni_aio_close(&p->aio_put); + nni_aio_close(&p->aio_get); + + nni_mtx_lock(&s->mtx); + nni_id_remove(&s->pipes, nni_pipe_id(p->pipe)); + nni_list_node_remove(&p->node); + nni_mtx_unlock(&s->mtx); + + nni_msgq_close(p->send_queue); +} + +static void +pair1poly_pipe_recv_cb(void *arg) +{ + pair1poly_pipe *p = arg; + pair1poly_sock *s = p->pair; + nni_msg * msg; + uint32_t hdr; + nni_pipe * pipe = p->pipe; + size_t len; + + if (nni_aio_result(&p->aio_recv) != 0) { + nni_pipe_close(p->pipe); + return; + } + + msg = nni_aio_get_msg(&p->aio_recv); + nni_aio_set_msg(&p->aio_recv, NULL); + + // Store the pipe ID. + nni_msg_set_pipe(msg, nni_pipe_id(p->pipe)); + + // If the message is missing the hop count header, scrap it. + if ((nni_msg_len(msg) < sizeof(uint32_t)) || + ((hdr = nni_msg_trim_u32(msg)) > 0xff)) { + BUMP_STAT(&s->stat_rx_malformed); + nni_msg_free(msg); + nni_pipe_close(pipe); + return; + } + + len = nni_msg_len(msg); + + // If we bounced too many times, discard the message, but + // keep getting more. + if ((int) hdr > nni_atomic_get(&s->ttl)) { + BUMP_STAT(&s->stat_ttl_drop); + nni_msg_free(msg); + nni_pipe_recv(pipe, &p->aio_recv); + return; + } + + // Store the hop count in the header. + nni_msg_header_append_u32(msg, hdr); + + // Send the message up. + nni_aio_set_msg(&p->aio_put, msg); + nni_sock_bump_rx(s->sock, len); + nni_msgq_aio_put(s->urq, &p->aio_put); +} + +static void +pair1poly_sock_get_cb(void *arg) +{ + pair1poly_pipe *p; + pair1poly_sock *s = arg; + nni_msg * msg; + uint32_t id; + + if (nni_aio_result(&s->aio_get) != 0) { + // Socket closing... + return; + } + + msg = nni_aio_get_msg(&s->aio_get); + nni_aio_set_msg(&s->aio_get, NULL); + + p = NULL; + nni_mtx_lock(&s->mtx); + // If no pipe was requested, we look for any connected peer. + if (((id = nni_msg_get_pipe(msg)) == 0) && + (!nni_list_empty(&s->plist))) { + p = nni_list_first(&s->plist); + } else { + p = nni_id_get(&s->pipes, id); + } + + // Try a non-blocking send. If this fails we just discard the + // message. We have to do this to avoid head-of-line blocking + // for messages sent to other pipes. Note that there is some + // buffering in the send_queue. + if ((p == NULL) || nni_msgq_tryput(p->send_queue, msg) != 0) { + BUMP_STAT(&s->stat_tx_drop); + nni_msg_free(msg); + } + + nni_mtx_unlock(&s->mtx); + nni_msgq_aio_get(s->uwq, &s->aio_get); +} + +static void +pair1poly_pipe_put_cb(void *arg) +{ + pair1poly_pipe *p = arg; + + if (nni_aio_result(&p->aio_put) != 0) { + nni_msg_free(nni_aio_get_msg(&p->aio_put)); + nni_aio_set_msg(&p->aio_put, NULL); + nni_pipe_close(p->pipe); + return; + } + nni_pipe_recv(p->pipe, &p->aio_recv); +} + +static void +pair1poly_pipe_get_cb(void *arg) +{ + pair1poly_pipe *p = arg; + nni_msg * msg; + + if (nni_aio_result(&p->aio_get) != 0) { + nni_pipe_close(p->pipe); + return; + } + + msg = nni_aio_get_msg(&p->aio_get); + nni_aio_set_msg(&p->aio_get, NULL); + + // Cooked mode messages have no header so we have to add one. + // Strip off any previously existing header, such as when + // replying to messages. + nni_msg_header_clear(msg); + + // Insert the hops header. + nni_msg_header_append_u32(msg, 1); + + nni_aio_set_msg(&p->aio_send, msg); + nni_pipe_send(p->pipe, &p->aio_send); +} + +static void +pair1poly_pipe_send_cb(void *arg) +{ + pair1poly_pipe *p = arg; + + if (nni_aio_result(&p->aio_send) != 0) { + nni_msg_free(nni_aio_get_msg(&p->aio_send)); + nni_aio_set_msg(&p->aio_send, NULL); + nni_pipe_close(p->pipe); + return; + } + + nni_msgq_aio_get(p->send_queue, &p->aio_get); +} + +static void +pair1poly_sock_open(void *arg) +{ + NNI_ARG_UNUSED(arg); +} + +static void +pair1poly_sock_close(void *arg) +{ + pair1poly_sock *s = arg; + nni_aio_close(&s->aio_get); +} + +static int +pair1poly_set_max_ttl(void *arg, const void *buf, size_t sz, nni_opt_type t) +{ + pair1poly_sock *s = arg; + int rv; + int ttl; + + if ((rv = nni_copyin_int(&ttl, buf, sz, 1, NNI_MAX_MAX_TTL, t)) == 0) { + nni_atomic_set(&s->ttl, ttl); + } + + return (rv); +} + +static int +pair1poly_get_max_ttl(void *arg, void *buf, size_t *szp, nni_opt_type t) +{ + pair1poly_sock *s = arg; + return (nni_copyout_int(nni_atomic_get(&s->ttl), buf, szp, t)); +} + +static int +pair1poly_get_poly(void *arg, void *buf, size_t *szp, nni_opt_type t) +{ + NNI_ARG_UNUSED(arg); + return (nni_copyout_bool(true, buf, szp, t)); +} + +static void +pair1poly_sock_send(void *arg, nni_aio *aio) +{ + pair1poly_sock *s = arg; + + nni_sock_bump_tx(s->sock, nni_msg_len(nni_aio_get_msg(aio))); + nni_msgq_aio_put(s->uwq, aio); +} + +static void +pair1poly_sock_recv(void *arg, nni_aio *aio) +{ + pair1poly_sock *s = arg; + + nni_msgq_aio_get(s->urq, aio); +} + +static nni_proto_pipe_ops pair1poly_pipe_ops = { + .pipe_size = sizeof(pair1poly_pipe), + .pipe_init = pair1poly_pipe_init, + .pipe_fini = pair1poly_pipe_fini, + .pipe_start = pair1poly_pipe_start, + .pipe_close = pair1poly_pipe_close, + .pipe_stop = pair1poly_pipe_stop, +}; + +static nni_option pair1poly_sock_options[] = { + { + .o_name = NNG_OPT_MAXTTL, + .o_get = pair1poly_get_max_ttl, + .o_set = pair1poly_set_max_ttl, + }, + { + .o_name = NNG_OPT_PAIR1_POLY, + .o_get = pair1poly_get_poly, + }, + // terminate list + { + .o_name = NULL, + }, +}; + +static nni_proto_sock_ops pair1poly_sock_ops = { + .sock_size = sizeof(pair1poly_sock), + .sock_init = pair1poly_sock_init, + .sock_fini = pair1poly_sock_fini, + .sock_open = pair1poly_sock_open, + .sock_close = pair1poly_sock_close, + .sock_recv = pair1poly_sock_recv, + .sock_send = pair1poly_sock_send, + .sock_options = pair1poly_sock_options, +}; + +static nni_proto pair1poly_proto = { + .proto_version = NNI_PROTOCOL_VERSION, + .proto_self = { NNG_PAIR1_SELF, NNG_PAIR1_SELF_NAME }, + .proto_peer = { NNG_PAIR1_PEER, NNG_PAIR1_PEER_NAME }, + .proto_flags = NNI_PROTO_FLAG_SNDRCV, + .proto_sock_ops = &pair1poly_sock_ops, + .proto_pipe_ops = &pair1poly_pipe_ops, +}; + +int +nng_pair1_open_poly(nng_socket *sock) +{ + return (nni_proto_open(sock, &pair1poly_proto)); +} diff --git a/src/sp/protocol/pair1/pair1_poly_test.c b/src/sp/protocol/pair1/pair1_poly_test.c new file mode 100644 index 00000000..f26f7809 --- /dev/null +++ b/src/sp/protocol/pair1/pair1_poly_test.c @@ -0,0 +1,370 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// Copyright 2017 Capitar IT Group BV +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include + +#define SECOND 1000 + +#define APPEND_STR(m, s) NUTS_PASS(nng_msg_append(m, s, strlen(s))) +#define CHECK_STR(m, s) \ + NUTS_TRUE(nng_msg_len(m) == strlen(s)); \ + NUTS_TRUE(memcmp(nng_msg_body(m), s, strlen(s)) == 0) + +static void +test_poly_identity(void) +{ + nng_socket s; + int p; + char * n; + + NUTS_PASS(nng_pair1_open_poly(&s)); + NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PROTO, &p)); + NUTS_TRUE(p == NUTS_PROTO(1u, 1u)); // 32 + NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PEER, &p)); + NUTS_TRUE(p == NUTS_PROTO(1u, 1u)); // 33 + NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PROTONAME, &n)); + NUTS_MATCH(n, "pair1"); + nng_strfree(n); + NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PEERNAME, &n)); + NUTS_MATCH(n, "pair1"); + nng_strfree(n); + NUTS_CLOSE(s); +} + +void +test_poly_best_effort(void) +{ + nng_socket s1; + nng_socket c1; + nng_msg * msg; + + NUTS_PASS(nng_pair1_open_poly(&s1)); + NUTS_PASS(nng_pair1_open(&c1)); + + NUTS_PASS(nng_setopt_int(s1, NNG_OPT_RECVBUF, 1)); + NUTS_PASS(nng_setopt_int(s1, NNG_OPT_SENDBUF, 1)); + NUTS_PASS(nng_setopt_int(c1, NNG_OPT_RECVBUF, 1)); + NUTS_PASS(nng_setopt_ms(s1, NNG_OPT_SENDTIMEO, SECOND)); + + NUTS_MARRY(s1, c1); + + for (int i = 0; i < 10; i++) { + NUTS_PASS(nng_msg_alloc(&msg, 0)); + NUTS_PASS(nng_sendmsg(s1, msg, 0)); + } + + NUTS_CLOSE(s1); + NUTS_CLOSE(c1); +} + +void +test_poly_cooked(void) +{ + nng_socket s1; + nng_socket c1; + nng_socket c2; + nng_msg * msg; + bool v; + nng_pipe p1; + nng_pipe p2; + + NUTS_PASS(nng_pair1_open_poly(&s1)); + NUTS_PASS(nng_pair1_open(&c1)); + NUTS_PASS(nng_pair1_open(&c2)); + NUTS_PASS(nng_setopt_ms(s1, NNG_OPT_SENDTIMEO, SECOND)); + NUTS_PASS(nng_setopt_ms(c1, NNG_OPT_SENDTIMEO, SECOND)); + NUTS_PASS(nng_setopt_ms(c2, NNG_OPT_SENDTIMEO, SECOND)); + NUTS_PASS(nng_setopt_ms(s1, NNG_OPT_RECVTIMEO, SECOND / 10)); + NUTS_PASS(nng_setopt_ms(c1, NNG_OPT_RECVTIMEO, SECOND / 10)); + NUTS_PASS(nng_setopt_ms(c2, NNG_OPT_RECVTIMEO, SECOND / 10)); + + NUTS_PASS(nng_getopt_bool(s1, NNG_OPT_PAIR1_POLY, &v)); + NUTS_TRUE(v); + + NUTS_MARRY(s1, c1); + NUTS_MARRY(s1, c2); + + NUTS_PASS(nng_msg_alloc(&msg, 0)); + APPEND_STR(msg, "ONE"); + NUTS_PASS(nng_sendmsg(c1, msg, 0)); + NUTS_PASS(nng_recvmsg(s1, &msg, 0)); + CHECK_STR(msg, "ONE"); + p1 = nng_msg_get_pipe(msg); + NUTS_TRUE(nng_pipe_id(p1) > 0); + nng_msg_free(msg); + + NUTS_PASS(nng_msg_alloc(&msg, 0)); + APPEND_STR(msg, "TWO"); + NUTS_PASS(nng_sendmsg(c2, msg, 0)); + NUTS_PASS(nng_recvmsg(s1, &msg, 0)); + CHECK_STR(msg, "TWO"); + p2 = nng_msg_get_pipe(msg); + NUTS_TRUE(nng_pipe_id(p2) > 0); + nng_msg_free(msg); + + NUTS_TRUE(nng_pipe_id(p1) != nng_pipe_id(p2)); + + NUTS_PASS(nng_msg_alloc(&msg, 0)); + + nng_msg_set_pipe(msg, p1); + APPEND_STR(msg, "UNO"); + NUTS_PASS(nng_sendmsg(s1, msg, 0)); + NUTS_PASS(nng_recvmsg(c1, &msg, 0)); + CHECK_STR(msg, "UNO"); + nng_msg_free(msg); + + NUTS_PASS(nng_msg_alloc(&msg, 0)); + nng_msg_set_pipe(msg, p2); + APPEND_STR(msg, "DOS"); + NUTS_PASS(nng_sendmsg(s1, msg, 0)); + NUTS_PASS(nng_recvmsg(c2, &msg, 0)); + CHECK_STR(msg, "DOS"); + nng_msg_free(msg); + + NUTS_PASS(nng_close(c1)); + + NUTS_PASS(nng_msg_alloc(&msg, 0)); + nng_msg_set_pipe(msg, p1); + APPEND_STR(msg, "EIN"); + NUTS_PASS(nng_sendmsg(s1, msg, 0)); + NUTS_FAIL(nng_recvmsg(c2, &msg, 0), NNG_ETIMEDOUT); + + NUTS_CLOSE(s1); + NUTS_CLOSE(c2); +} + +void +test_poly_default(void) +{ + nng_socket s1; + nng_socket c1; + nng_socket c2; + nng_msg * msg; + + NUTS_PASS(nng_pair1_open_poly(&s1)); + NUTS_PASS(nng_pair1_open(&c1)); + NUTS_PASS(nng_pair1_open(&c2)); + NUTS_PASS(nng_setopt_ms(s1, NNG_OPT_SENDTIMEO, SECOND)); + NUTS_PASS(nng_setopt_ms(c1, NNG_OPT_SENDTIMEO, SECOND)); + NUTS_PASS(nng_setopt_ms(c2, NNG_OPT_SENDTIMEO, SECOND)); + + NUTS_MARRY(s1, c1); + NUTS_MARRY(s1, c2); + + // This assumes poly picks the first suitor. Applications + // should not make the same assumption. + NUTS_PASS(nng_msg_alloc(&msg, 0)); + APPEND_STR(msg, "YES"); + NUTS_PASS(nng_sendmsg(s1, msg, 0)); + NUTS_PASS(nng_recvmsg(c1, &msg, 0)); + CHECK_STR(msg, "YES"); + nng_msg_free(msg); + + NUTS_CLOSE(c1); + NUTS_SLEEP(10); + + // Verify that the other pipe is chosen as the next suitor. + NUTS_PASS(nng_msg_alloc(&msg, 0)); + APPEND_STR(msg, "AGAIN"); + NUTS_PASS(nng_sendmsg(s1, msg, 0)); + NUTS_PASS(nng_recvmsg(c2, &msg, 0)); + CHECK_STR(msg, "AGAIN"); + nng_msg_free(msg); + + NUTS_CLOSE(s1); + NUTS_CLOSE(c2); +} + +void +test_poly_close_abort(void) +{ + nng_socket s; + nng_socket c; + + NUTS_PASS(nng_pair1_open_poly(&s)); + NUTS_PASS(nng_pair1_open(&c)); + NUTS_PASS(nng_setopt_ms(s, NNG_OPT_RECVTIMEO, 100)); + NUTS_PASS(nng_setopt_ms(s, NNG_OPT_SENDTIMEO, 200)); + NUTS_PASS(nng_setopt_int(s, NNG_OPT_RECVBUF, 1)); + NUTS_PASS(nng_setopt_int(c, NNG_OPT_SENDBUF, 20)); + + NUTS_MARRY(c, s); + + for (int i = 0; i < 20; i++) { + NUTS_SEND(c, "TEST"); + } + NUTS_SLEEP(50); + + NUTS_CLOSE(s); + NUTS_CLOSE(c); +} + +void +test_poly_recv_no_header(void) +{ + nng_socket s; + nng_socket c; + nng_msg * m; + + NUTS_PASS(nng_pair1_open_poly(&s)); + NUTS_PASS(nng_pair1_open(&c)); + NUTS_PASS(nng_setopt_bool(c, "pair1_test_inject_header", true)); + NUTS_PASS(nng_setopt_ms(s, NNG_OPT_RECVTIMEO, 100)); + NUTS_PASS(nng_setopt_ms(s, NNG_OPT_SENDTIMEO, 200)); + + NUTS_MARRY(c, s); + + NUTS_PASS(nng_msg_alloc(&m, 0)); + NUTS_PASS(nng_sendmsg(c, m, 0)); + NUTS_FAIL(nng_recvmsg(s, &m, 0), NNG_ETIMEDOUT); + + NUTS_CLOSE(c); + NUTS_CLOSE(s); +} + +void +test_poly_recv_garbage(void) +{ + nng_socket s; + nng_socket c; + nng_msg * m; + + NUTS_PASS(nng_pair1_open_poly(&s)); + NUTS_PASS(nng_pair1_open(&c)); + NUTS_PASS(nng_setopt_bool(c, "pair1_test_inject_header", true)); + NUTS_PASS(nng_setopt_ms(s, NNG_OPT_RECVTIMEO, 100)); + NUTS_PASS(nng_setopt_ms(s, NNG_OPT_SENDTIMEO, 200)); + + NUTS_MARRY(c, s); + + // ridiculous hop count + NUTS_PASS(nng_msg_alloc(&m, 0)); + NUTS_PASS(nng_msg_append_u32(m, 0x1000)); + NUTS_PASS(nng_sendmsg(c, m, 0)); + NUTS_FAIL(nng_recvmsg(s, &m, 0), NNG_ETIMEDOUT); + + NUTS_CLOSE(c); + NUTS_CLOSE(s); +} + +void +test_poly_ttl(void) +{ + nng_socket s1; + nng_socket c1; + nng_msg * msg; + uint32_t val; + int ttl; + + NUTS_PASS(nng_pair1_open_poly(&s1)); + NUTS_PASS(nng_pair1_open_raw(&c1)); + NUTS_PASS(nng_setopt_ms(s1, NNG_OPT_RECVTIMEO, SECOND / 5)); + NUTS_PASS(nng_setopt_ms(c1, NNG_OPT_RECVTIMEO, SECOND / 5)); + + // cannot set insane TTLs + NUTS_FAIL(nng_setopt_int(s1, NNG_OPT_MAXTTL, 0), NNG_EINVAL); + NUTS_FAIL(nng_setopt_int(s1, NNG_OPT_MAXTTL, 1000), NNG_EINVAL); + ttl = 8; + NUTS_FAIL(nng_setopt(s1, NNG_OPT_MAXTTL, &ttl, 1), NNG_EINVAL); + NUTS_FAIL(nng_setopt_bool(s1, NNG_OPT_MAXTTL, true), NNG_EBADTYPE); + + NUTS_MARRY(s1, c1); + + // Let's check enforcement of TTL + NUTS_PASS(nng_setopt_int(s1, NNG_OPT_MAXTTL, 4)); + NUTS_PASS(nng_getopt_int(s1, NNG_OPT_MAXTTL, &ttl)); + NUTS_TRUE(ttl == 4); + + // Bad TTL bounces + NUTS_PASS(nng_msg_alloc(&msg, 0)); + NUTS_PASS(nng_msg_header_append_u32(msg, 4)); + NUTS_PASS(nng_sendmsg(c1, msg, 0)); + NUTS_FAIL(nng_recvmsg(s1, &msg, 0), NNG_ETIMEDOUT); + + // Good TTL passes + NUTS_PASS(nng_msg_alloc(&msg, 0)); + NUTS_PASS(nng_msg_append_u32(msg, 0xFEEDFACE)); + NUTS_PASS(nng_msg_header_append_u32(msg, 3)); + NUTS_PASS(nng_sendmsg(c1, msg, 0)); + NUTS_PASS(nng_recvmsg(s1, &msg, 0)); + NUTS_PASS(nng_msg_trim_u32(msg, &val)); + NUTS_TRUE(val == 0xFEEDFACE); + NUTS_PASS(nng_msg_header_trim_u32(msg, &val)); + NUTS_TRUE(val == 4); + nng_msg_free(msg); + + // Large TTL passes + NUTS_PASS(nng_setopt_int(s1, NNG_OPT_MAXTTL, 15)); + NUTS_PASS(nng_msg_alloc(&msg, 0)); + NUTS_PASS(nng_msg_append_u32(msg, 1234)); + NUTS_PASS(nng_msg_header_append_u32(msg, 14)); + NUTS_PASS(nng_sendmsg(c1, msg, 0)); + NUTS_PASS(nng_recvmsg(s1, &msg, 0)); + NUTS_PASS(nng_msg_trim_u32(msg, &val)); + NUTS_TRUE(val == 1234); + NUTS_PASS(nng_msg_header_trim_u32(msg, &val)); + NUTS_TRUE(val == 15); + nng_msg_free(msg); + + // Max TTL fails + NUTS_PASS(nng_setopt_int(s1, NNG_OPT_MAXTTL, 15)); + NUTS_PASS(nng_msg_alloc(&msg, 0)); + NUTS_PASS(nng_msg_header_append_u32(msg, 15)); + NUTS_PASS(nng_sendmsg(c1, msg, 0)); + NUTS_FAIL(nng_recvmsg(s1, &msg, 0), NNG_ETIMEDOUT); + + NUTS_CLOSE(s1); + NUTS_CLOSE(c1); +} + +void +test_poly_validate_peer(void) +{ + nng_socket s1, s2; + nng_stat * stats; + nng_stat * reject; + char * addr; + + NUTS_ADDR(addr, "inproc"); + + NUTS_PASS(nng_pair1_open_poly(&s1)); + NUTS_PASS(nng_pair0_open(&s2)); + + NUTS_PASS(nng_listen(s1, addr, NULL, 0)); + NUTS_PASS(nng_dial(s2, addr, NULL, NNG_FLAG_NONBLOCK)); + + NUTS_SLEEP(100); + NUTS_PASS(nng_stats_get(&stats)); + + NUTS_TRUE(stats != NULL); + NUTS_TRUE((reject = nng_stat_find_socket(stats, s1)) != NULL); + NUTS_TRUE((reject = nng_stat_find(reject, "reject")) != NULL); + + NUTS_TRUE(nng_stat_type(reject) == NNG_STAT_COUNTER); + NUTS_TRUE(nng_stat_value(reject) > 0); + + NUTS_CLOSE(s1); + NUTS_CLOSE(s2); + nng_stats_free(stats); +} + +TEST_LIST = { + { "pair1 poly identity", test_poly_identity }, + { "pair1 poly best effort", test_poly_best_effort }, + { "pair1 poly cooked", test_poly_cooked }, + { "pair1 poly default", test_poly_default }, + { "pair1 poly recv no header", test_poly_recv_no_header }, + { "pair1 poly recv garbage", test_poly_recv_garbage }, + { "pair1 poly ttl", test_poly_ttl }, + { "pair1 poly close abort", test_poly_close_abort }, + { "pair1 poly validate peer", test_poly_validate_peer }, + + { NULL, NULL }, +}; diff --git a/src/sp/protocol/pair1/pair1_test.c b/src/sp/protocol/pair1/pair1_test.c new file mode 100644 index 00000000..881c4ac8 --- /dev/null +++ b/src/sp/protocol/pair1/pair1_test.c @@ -0,0 +1,433 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// Copyright 2017 Capitar IT Group BV +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include + +#define SECOND 1000 + +#define APPEND_STR(m, s) NUTS_TRUE(nng_msg_append(m, s, strlen(s)) == 0) +#define CHECK_STR(m, s) \ + NUTS_TRUE(nng_msg_len(m) == strlen(s)); \ + NUTS_TRUE(memcmp(nng_msg_body(m), s, strlen(s)) == 0) + +static void +test_mono_identity(void) +{ + nng_socket s; + int p; + char * n; + + NUTS_PASS(nng_pair1_open(&s)); + NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PROTO, &p)); + NUTS_TRUE(p == NUTS_PROTO(1u, 1u)); // 32 + NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PEER, &p)); + NUTS_TRUE(p == NUTS_PROTO(1u, 1u)); // 33 + NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PROTONAME, &n)); + NUTS_MATCH(n, "pair1"); + nng_strfree(n); + NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PEERNAME, &n)); + NUTS_MATCH(n, "pair1"); + nng_strfree(n); + NUTS_CLOSE(s); +} + +void +test_mono_cooked(void) +{ + nng_socket s1; + nng_socket c1; + nng_msg * msg; + + NUTS_PASS(nng_pair1_open(&s1)); + NUTS_PASS(nng_pair1_open(&c1)); + NUTS_PASS(nuts_marry(s1, c1)); + + NUTS_PASS(nng_msg_alloc(&msg, 0)); + NUTS_PASS(nng_msg_append(msg, "ALPHA", strlen("ALPHA") + 1)); + NUTS_PASS(nng_sendmsg(c1, msg, 0)); + NUTS_PASS(nng_recvmsg(s1, &msg, 0)); + NUTS_TRUE(nng_msg_len(msg) == strlen("ALPHA") + 1); + NUTS_MATCH(nng_msg_body(msg), "ALPHA"); + nng_msg_free(msg); + + NUTS_PASS(nng_msg_alloc(&msg, 0)); + NUTS_PASS(nng_msg_append(msg, "BETA", strlen("BETA") + 1)); + NUTS_PASS(nng_sendmsg(s1, msg, 0)); + NUTS_PASS(nng_recvmsg(c1, &msg, 0)); + NUTS_TRUE(nng_msg_len(msg) == strlen("BETA") + 1); + NUTS_MATCH(nng_msg_body(msg), "BETA"); + + nng_msg_free(msg); + NUTS_CLOSE(c1); + NUTS_CLOSE(s1); +} + +void +test_mono_faithful(void) +{ + nng_socket s1; + nng_socket c1; + nng_socket c2; + nng_msg * msg; + const char *addr = "inproc://pair1_mono_faithful"; + + NUTS_PASS(nng_pair1_open(&s1)); + NUTS_PASS(nng_pair1_open(&c1)); + NUTS_PASS(nng_pair1_open(&c2)); + NUTS_PASS(nng_setopt_ms(s1, NNG_OPT_RECVTIMEO, SECOND / 4)); + NUTS_PASS(nng_setopt_ms(c1, NNG_OPT_SENDTIMEO, SECOND)); + NUTS_PASS(nng_setopt_ms(c2, NNG_OPT_SENDTIMEO, SECOND)); + NUTS_PASS(nng_setopt_int(c2, NNG_OPT_SENDBUF, 2)); + + NUTS_PASS(nng_listen(s1, addr, NULL, 0)); + NUTS_MARRY(s1, c1); + NUTS_PASS(nng_dial(c2, addr, NULL, 0)); + + NUTS_SLEEP(100); + + NUTS_PASS(nng_msg_alloc(&msg, 0)); + APPEND_STR(msg, "ONE"); + NUTS_PASS(nng_sendmsg(c1, msg, 0)); + NUTS_PASS(nng_recvmsg(s1, &msg, 0)); + CHECK_STR(msg, "ONE"); + nng_msg_free(msg); + + NUTS_PASS(nng_msg_alloc(&msg, 0)); + APPEND_STR(msg, "TWO"); + NUTS_PASS(nng_sendmsg(c2, msg, 0)); + NUTS_FAIL(nng_recvmsg(s1, &msg, 0), NNG_ETIMEDOUT); + + NUTS_CLOSE(s1); + NUTS_CLOSE(c1); + NUTS_CLOSE(c2); +} + +void +test_mono_back_pressure(void) +{ + nng_socket s1; + nng_socket c1; + int i; + int rv; + nng_msg * msg; + nng_duration to = 100; + + NUTS_PASS(nng_pair1_open(&s1)); + NUTS_PASS(nng_pair1_open(&c1)); + NUTS_PASS(nng_setopt_int(s1, NNG_OPT_RECVBUF, 1)); + NUTS_PASS(nng_setopt_int(s1, NNG_OPT_SENDBUF, 1)); + NUTS_PASS(nng_setopt_int(c1, NNG_OPT_RECVBUF, 1)); + NUTS_PASS(nng_setopt_ms(s1, NNG_OPT_SENDTIMEO, to)); + + NUTS_MARRY(s1, c1); + + // We choose to allow some buffering. In reality the + // buffer size is just 1, and we will fail after 2. + for (i = 0, rv = 0; i < 10; i++) { + NUTS_PASS(nng_msg_alloc(&msg, 0)); + if ((rv = nng_sendmsg(s1, msg, 0)) != 0) { + nng_msg_free(msg); + break; + } + } + NUTS_FAIL(rv, NNG_ETIMEDOUT); + NUTS_TRUE(i < 10); + NUTS_CLOSE(s1); + NUTS_CLOSE(c1); +} + +void +test_mono_raw_exchange(void) +{ + nng_socket s1; + nng_socket c1; + + nng_msg *msg; + uint32_t hops; + + NUTS_PASS(nng_pair1_open_raw(&s1)); + NUTS_PASS(nng_pair1_open_raw(&c1)); + + NUTS_PASS(nng_setopt_ms(s1, NNG_OPT_RECVTIMEO, SECOND)); + NUTS_PASS(nng_setopt_ms(c1, NNG_OPT_RECVTIMEO, SECOND)); + NUTS_MARRY(s1, c1); + + nng_pipe p = NNG_PIPE_INITIALIZER; + NUTS_PASS(nng_msg_alloc(&msg, 0)); + APPEND_STR(msg, "GAMMA"); + NUTS_PASS(nng_msg_header_append_u32(msg, 1)); + NUTS_TRUE(nng_msg_header_len(msg) == sizeof(uint32_t)); + NUTS_PASS(nng_sendmsg(c1, msg, 0)); + NUTS_PASS(nng_recvmsg(s1, &msg, 0)); + p = nng_msg_get_pipe(msg); + NUTS_TRUE(nng_pipe_id(p) > 0); + + CHECK_STR(msg, "GAMMA"); + NUTS_TRUE(nng_msg_header_len(msg) == sizeof(uint32_t)); + NUTS_PASS(nng_msg_header_trim_u32(msg, &hops)); + NUTS_TRUE(hops == 2); + nng_msg_free(msg); + + NUTS_PASS(nng_msg_alloc(&msg, 0)); + APPEND_STR(msg, "EPSILON"); + NUTS_PASS(nng_msg_header_append_u32(msg, 1)); + NUTS_PASS(nng_sendmsg(s1, msg, 0)); + NUTS_PASS(nng_recvmsg(c1, &msg, 0)); + CHECK_STR(msg, "EPSILON"); + NUTS_TRUE(nng_msg_header_len(msg) == sizeof(uint32_t)); + NUTS_PASS(nng_msg_header_trim_u32(msg, &hops)); + p = nng_msg_get_pipe(msg); + NUTS_TRUE(nng_pipe_id(p) > 0); + + NUTS_TRUE(hops == 2); + nng_msg_free(msg); + + NUTS_CLOSE(s1); + NUTS_CLOSE(c1); +} + +void +test_mono_raw_header(void) +{ + nng_socket s1; + nng_socket c1; + nng_msg * msg; + uint32_t v; + + NUTS_PASS(nng_pair1_open_raw(&s1)); + NUTS_PASS(nng_pair1_open_raw(&c1)); + + NUTS_PASS(nng_setopt_ms(s1, NNG_OPT_RECVTIMEO, SECOND / 5)); + NUTS_PASS(nng_setopt_ms(c1, NNG_OPT_RECVTIMEO, SECOND / 5)); + NUTS_MARRY(s1, c1); + + // Missing bits in the header + NUTS_PASS(nng_msg_alloc(&msg, 0)); + NUTS_PASS(nng_sendmsg(c1, msg, 0)); + NUTS_FAIL(nng_recvmsg(s1, &msg, 0), NNG_ETIMEDOUT); + + // Valid header works + NUTS_PASS(nng_msg_alloc(&msg, 0)); + NUTS_PASS(nng_msg_append_u32(msg, 0xFEEDFACE)); + NUTS_PASS(nng_msg_header_append_u32(msg, 1)); + NUTS_PASS(nng_sendmsg(c1, msg, 0)); + NUTS_PASS(nng_recvmsg(s1, &msg, 0)); + NUTS_PASS(nng_msg_trim_u32(msg, &v)); + NUTS_TRUE(v == 0xFEEDFACE); + nng_msg_free(msg); + + // Header with reserved bits set dropped + NUTS_PASS(nng_msg_alloc(&msg, 0)); + NUTS_PASS(nng_msg_header_append_u32(msg, 0xDEAD0000)); + NUTS_PASS(nng_sendmsg(c1, msg, 0)); + NUTS_FAIL(nng_recvmsg(s1, &msg, 0), NNG_ETIMEDOUT); + + // Header with no chance to add another hop gets dropped + NUTS_PASS(nng_msg_alloc(&msg, 0)); + NUTS_PASS(nng_msg_header_append_u32(msg, 0xff)); + NUTS_PASS(nng_sendmsg(c1, msg, 0)); + NUTS_FAIL(nng_recvmsg(s1, &msg, 0), NNG_ETIMEDOUT); + + // With the same bits clear it works + NUTS_PASS(nng_msg_alloc(&msg, 0)); + NUTS_PASS(nng_msg_append_u32(msg, 0xFEEDFACE)); + NUTS_PASS(nng_msg_header_append_u32(msg, 1)); + NUTS_PASS(nng_sendmsg(c1, msg, 0)); + NUTS_PASS(nng_recvmsg(s1, &msg, 0)); + NUTS_PASS(nng_msg_trim_u32(msg, &v)); + NUTS_TRUE(v == 0xFEEDFACE); + nng_msg_free(msg); + + NUTS_CLOSE(s1); + NUTS_CLOSE(c1); +} + +void +test_pair1_raw(void) +{ + nng_socket s1; + bool raw; + + NUTS_PASS(nng_pair1_open(&s1)); + NUTS_PASS(nng_getopt_bool(s1, NNG_OPT_RAW, &raw)); + NUTS_TRUE(raw == false); + NUTS_FAIL(nng_setopt_bool(s1, NNG_OPT_RAW, true), NNG_EREADONLY); + NUTS_PASS(nng_close(s1)); + + NUTS_PASS(nng_pair1_open_raw(&s1)); + NUTS_PASS(nng_getopt_bool(s1, NNG_OPT_RAW, &raw)); + NUTS_TRUE(raw == true); + NUTS_FAIL(nng_setopt_bool(s1, NNG_OPT_RAW, false), NNG_EREADONLY); + NUTS_PASS(nng_close(s1)); +} + +void +test_pair1_ttl(void) +{ + nng_socket s1; + nng_socket c1; + nng_msg * msg; + uint32_t val; + int ttl; + + NUTS_PASS(nng_pair1_open_raw(&s1)); + NUTS_PASS(nng_pair1_open_raw(&c1)); + NUTS_PASS(nng_setopt_ms(s1, NNG_OPT_RECVTIMEO, SECOND / 5)); + NUTS_PASS(nng_setopt_ms(c1, NNG_OPT_RECVTIMEO, SECOND / 5)); + + // cannot set insane TTLs + NUTS_FAIL(nng_setopt_int(s1, NNG_OPT_MAXTTL, 0), NNG_EINVAL); + NUTS_FAIL(nng_setopt_int(s1, NNG_OPT_MAXTTL, 1000), NNG_EINVAL); + ttl = 8; + NUTS_FAIL(nng_setopt(s1, NNG_OPT_MAXTTL, &ttl, 1), NNG_EINVAL); + NUTS_FAIL(nng_setopt_bool(s1, NNG_OPT_MAXTTL, true), NNG_EBADTYPE); + + NUTS_MARRY(s1, c1); + + // Let's check enforcement of TTL + NUTS_PASS(nng_setopt_int(s1, NNG_OPT_MAXTTL, 4)); + NUTS_PASS(nng_getopt_int(s1, NNG_OPT_MAXTTL, &ttl)); + NUTS_TRUE(ttl == 4); + + // Bad TTL bounces + NUTS_PASS(nng_msg_alloc(&msg, 0)); + NUTS_PASS(nng_msg_header_append_u32(msg, 4)); + NUTS_PASS(nng_sendmsg(c1, msg, 0)); + NUTS_FAIL(nng_recvmsg(s1, &msg, 0), NNG_ETIMEDOUT); + + // Good TTL passes + NUTS_PASS(nng_msg_alloc(&msg, 0)); + NUTS_PASS(nng_msg_append_u32(msg, 0xFEEDFACE)); + NUTS_PASS(nng_msg_header_append_u32(msg, 3)); + NUTS_PASS(nng_sendmsg(c1, msg, 0)); + NUTS_PASS(nng_recvmsg(s1, &msg, 0)); + NUTS_PASS(nng_msg_trim_u32(msg, &val)); + NUTS_TRUE(val == 0xFEEDFACE); + NUTS_PASS(nng_msg_header_trim_u32(msg, &val)); + NUTS_TRUE(val == 4); + nng_msg_free(msg); + + // Large TTL passes + NUTS_PASS(nng_setopt_int(s1, NNG_OPT_MAXTTL, 15)); + NUTS_PASS(nng_msg_alloc(&msg, 0)); + NUTS_PASS(nng_msg_append_u32(msg, 1234)); + NUTS_PASS(nng_msg_header_append_u32(msg, 14)); + NUTS_PASS(nng_sendmsg(c1, msg, 0)); + NUTS_PASS(nng_recvmsg(s1, &msg, 0)); + NUTS_PASS(nng_msg_trim_u32(msg, &val)); + NUTS_TRUE(val == 1234); + NUTS_PASS(nng_msg_header_trim_u32(msg, &val)); + NUTS_TRUE(val == 15); + nng_msg_free(msg); + + // Max TTL fails + NUTS_PASS(nng_setopt_int(s1, NNG_OPT_MAXTTL, 15)); + NUTS_PASS(nng_msg_alloc(&msg, 0)); + NUTS_PASS(nng_msg_header_append_u32(msg, 15)); + NUTS_PASS(nng_sendmsg(c1, msg, 0)); + NUTS_FAIL(nng_recvmsg(s1, &msg, 0), NNG_ETIMEDOUT); + + NUTS_CLOSE(s1); + NUTS_CLOSE(c1); +} + +void +test_pair1_validate_peer(void) +{ + nng_socket s1, s2; + nng_stat * stats; + nng_stat * reject; + char * addr; + + NUTS_ADDR(addr, "inproc"); + NUTS_PASS(nng_pair1_open(&s1)); + NUTS_PASS(nng_pair0_open(&s2)); + + NUTS_PASS(nng_listen(s1, addr, NULL, 0)); + NUTS_PASS(nng_dial(s2, addr, NULL, NNG_FLAG_NONBLOCK)); + + NUTS_SLEEP(100); + NUTS_PASS(nng_stats_get(&stats)); + + NUTS_TRUE(stats != NULL); + NUTS_TRUE((reject = nng_stat_find_socket(stats, s1)) != NULL); + NUTS_TRUE((reject = nng_stat_find(reject, "reject")) != NULL); + + NUTS_TRUE(nng_stat_type(reject) == NNG_STAT_COUNTER); + NUTS_TRUE(nng_stat_value(reject) > 0); + + NUTS_CLOSE(s1); + NUTS_CLOSE(s2); + nng_stats_free(stats); +} + +void +test_pair1_recv_no_header(void) +{ + nng_socket s; + nng_socket c; + nng_msg * m; + + NUTS_PASS(nng_pair1_open(&s)); + NUTS_PASS(nng_pair1_open(&c)); + NUTS_PASS(nng_setopt_bool(c, "pair1_test_inject_header", true)); + NUTS_PASS(nng_setopt_ms(s, NNG_OPT_RECVTIMEO, 100)); + NUTS_PASS(nng_setopt_ms(s, NNG_OPT_SENDTIMEO, 200)); + + NUTS_MARRY(c, s); + + NUTS_PASS(nng_msg_alloc(&m, 0)); + NUTS_PASS(nng_sendmsg(c, m, 0)); + NUTS_FAIL(nng_recvmsg(s, &m, 0), NNG_ETIMEDOUT); + + NUTS_CLOSE(c); + NUTS_CLOSE(s); +} + +void +test_pair1_recv_garbage(void) +{ + nng_socket s; + nng_socket c; + nng_msg * m; + + NUTS_PASS(nng_pair1_open(&s)); + NUTS_PASS(nng_pair1_open(&c)); + NUTS_PASS(nng_setopt_bool(c, "pair1_test_inject_header", true)); + NUTS_PASS(nng_setopt_ms(s, NNG_OPT_RECVTIMEO, 100)); + NUTS_PASS(nng_setopt_ms(s, NNG_OPT_SENDTIMEO, 200)); + + NUTS_MARRY(c, s); + + // ridiculous hop count + NUTS_PASS(nng_msg_alloc(&m, 0)); + NUTS_PASS(nng_msg_append_u32(m, 0x1000)); + NUTS_PASS(nng_sendmsg(c, m, 0)); + NUTS_FAIL(nng_recvmsg(s, &m, 0), NNG_ETIMEDOUT); + + NUTS_CLOSE(c); + NUTS_CLOSE(s); +} + +NUTS_TESTS = { + { "pair1 mono identity", test_mono_identity }, + { "pair1 mono cooked", test_mono_cooked }, + { "pair1 mono faithful", test_mono_faithful }, + { "pair1 mono back pressure", test_mono_back_pressure }, + { "pair1 mono raw exchange", test_mono_raw_exchange }, + { "pair1 mono raw header", test_mono_raw_header }, + { "pair1 raw", test_pair1_raw }, + { "pair1 ttl", test_pair1_ttl }, + { "pair1 validate peer", test_pair1_validate_peer }, + { "pair1 recv no header", test_pair1_recv_no_header }, + { "pair1 recv garbage", test_pair1_recv_garbage }, + + { NULL, NULL }, +}; diff --git a/src/sp/protocol/pipeline0/CMakeLists.txt b/src/sp/protocol/pipeline0/CMakeLists.txt new file mode 100644 index 00000000..8a10eab7 --- /dev/null +++ b/src/sp/protocol/pipeline0/CMakeLists.txt @@ -0,0 +1,23 @@ +# +# Copyright 2019 Staysail Systems, Inc. +# Copyright 2018 Capitar IT Group BV +# +# This software is supplied under the terms of the MIT License, a +# copy of which should be located in the distribution where this +# file was obtained (LICENSE.txt). A copy of the license may also be +# found online at https://opensource.org/licenses/MIT. +# + +# Pipeline protocol +nng_directory(pipeline0) + +nng_sources_if(NNG_PROTO_PUSH0 push.c) +nng_headers_if(NNG_PROTO_PUSH0 nng/protocol/pipeline0/push.h) +nng_defines_if(NNG_PROTO_PUSH0 NNG_HAVE_PUSH0) + +nng_sources_if(NNG_PROTO_PULL0 pull.c) +nng_headers_if(NNG_PROTO_PULL0 nng/protocol/pipeline0/pull.h) +nng_defines_if(NNG_PROTO_PULL0 NNG_HAVE_PULL0) + +nng_test(pull_test) +nng_test(push_test) diff --git a/src/sp/protocol/pipeline0/pull.c b/src/sp/protocol/pipeline0/pull.c new file mode 100644 index 00000000..616b0817 --- /dev/null +++ b/src/sp/protocol/pipeline0/pull.c @@ -0,0 +1,325 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// Copyright 2018 Capitar IT Group BV +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include + +#include "core/nng_impl.h" +#include "nng/protocol/pipeline0/pull.h" + +// Pull protocol. The PULL protocol is the "read" side of a pipeline. + +#ifndef NNI_PROTO_PULL_V0 +#define NNI_PROTO_PULL_V0 NNI_PROTO(5, 1) +#endif + +#ifndef NNI_PROTO_PUSH_V0 +#define NNI_PROTO_PUSH_V0 NNI_PROTO(5, 0) +#endif + +typedef struct pull0_pipe pull0_pipe; +typedef struct pull0_sock pull0_sock; + +static void pull0_recv_cb(void *); + +// pull0_sock is our per-socket protocol private structure. +struct pull0_sock { + bool raw; + nni_list pl; // pipe list (pipes with data ready) + nni_list rq; // recv queue (aio list) + nni_mtx m; + nni_pollable readable; +}; + +// pull0_pipe is our per-pipe protocol private structure. +struct pull0_pipe { + nni_pipe * p; + pull0_sock * s; + nni_msg * m; + nni_aio aio; + bool closed; + nni_list_node node; +}; + +static int +pull0_sock_init(void *arg, nni_sock *sock) +{ + pull0_sock *s = arg; + NNI_ARG_UNUSED(sock); + + nni_aio_list_init(&s->rq); + NNI_LIST_INIT(&s->pl, pull0_pipe, node); + nni_mtx_init(&s->m); + nni_pollable_init(&s->readable); + return (0); +} + +static void +pull0_sock_fini(void *arg) +{ + pull0_sock *s = arg; + nni_mtx_fini(&s->m); + nni_pollable_fini(&s->readable); +} + +static void +pull0_pipe_stop(void *arg) +{ + pull0_pipe *p = arg; + + nni_aio_stop(&p->aio); +} + +static void +pull0_pipe_fini(void *arg) +{ + pull0_pipe *p = arg; + + nni_aio_fini(&p->aio); + if (p->m) { + nni_msg_free(p->m); + } +} + +static int +pull0_pipe_init(void *arg, nni_pipe *pipe, void *s) +{ + pull0_pipe *p = arg; + + nni_aio_init(&p->aio, pull0_recv_cb, p); + p->p = pipe; + p->s = s; + return (0); +} + +static int +pull0_pipe_start(void *arg) +{ + pull0_pipe *p = arg; + + if (nni_pipe_peer(p->p) != NNI_PROTO_PUSH_V0) { + // Peer protocol mismatch. + return (NNG_EPROTO); + } + + // Start the pending receive... + nni_pipe_recv(p->p, &p->aio); + + return (0); +} + +static void +pull0_pipe_close(void *arg) +{ + pull0_pipe *p = arg; + pull0_sock *s = p->s; + + nni_mtx_lock(&s->m); + p->closed = true; + if (nni_list_node_active(&p->node)) { + nni_list_node_remove(&p->node); + if (nni_list_empty(&s->pl)) { + nni_pollable_clear(&s->readable); + } + } + nni_mtx_unlock(&s->m); + + nni_aio_close(&p->aio); +} + +static void +pull0_recv_cb(void *arg) +{ + pull0_pipe *p = arg; + pull0_sock *s = p->s; + nni_aio * ap = &p->aio; + nni_aio * as; + nni_msg * m; + + if (nni_aio_result(ap) != 0) { + // Failed to get a message, probably the pipe is closed. + nni_pipe_close(p->p); + return; + } + + // Got a message... start the put to send it up to the application. + m = nni_aio_get_msg(ap); + nni_aio_set_msg(ap, NULL); + nni_msg_set_pipe(m, nni_pipe_id(p->p)); + + nni_mtx_lock(&s->m); + if (p->closed) { + nni_mtx_unlock(&s->m); + nni_msg_free(m); + return; + } + if (nni_list_empty(&s->rq)) { + nni_list_append(&s->pl, p); + if (nni_list_first(&s->pl) == p) { + nni_pollable_raise(&s->readable); + } + p->m = m; + nni_mtx_unlock(&s->m); + return; + } + nni_pipe_recv(p->p, ap); + as = nni_list_first(&s->rq); + nni_aio_list_remove(as); + nni_mtx_unlock(&s->m); + nni_aio_set_msg(as, m); + nni_aio_finish_sync(as, 0, nni_msg_len(m)); +} + +static void +pull0_sock_open(void *arg) +{ + NNI_ARG_UNUSED(arg); +} + +static void +pull0_sock_close(void *arg) +{ + pull0_sock *s = arg; + nni_aio * a; + nni_mtx_lock(&s->m); + while ((a = nni_list_first(&s->rq)) != NULL) { + nni_aio_list_remove(a); + nni_aio_finish_error(a, NNG_ECLOSED); + } + // NB: The common socket framework closes pipes before this. + nni_mtx_unlock(&s->m); +} + +static void +pull0_sock_send(void *arg, nni_aio *aio) +{ + NNI_ARG_UNUSED(arg); + nni_aio_finish_error(aio, NNG_ENOTSUP); +} + +static void +pull0_cancel(nni_aio *aio, void *arg, int rv) +{ + pull0_sock *s = arg; + nni_mtx_lock(&s->m); + if (nni_aio_list_active(aio)) { + nni_aio_list_remove(aio); + nni_aio_finish_error(aio, rv); + } + nni_mtx_unlock(&s->m); +} + +static void +pull0_sock_recv(void *arg, nni_aio *aio) +{ + pull0_sock *s = arg; + pull0_pipe *p; + + if (nni_aio_begin(aio) != 0) { + return; + } + + nni_mtx_lock(&s->m); + if ((p = nni_list_first(&s->pl)) == NULL) { + + int rv; + if ((rv = nni_aio_schedule(aio, pull0_cancel, s)) != 0) { + nni_mtx_unlock(&s->m); + nni_aio_finish_error(aio, rv); + return; + } + + nni_aio_list_append(&s->rq, aio); + nni_mtx_unlock(&s->m); + return; + } + + nni_list_remove(&s->pl, p); + if (nni_list_empty(&s->pl)) { + nni_pollable_clear(&s->readable); + } + nni_aio_finish_msg(aio, p->m); + p->m = NULL; + nni_pipe_recv(p->p, &p->aio); + nni_mtx_unlock(&s->m); +} + +static int +pull0_sock_get_recv_fd(void *arg, void *buf, size_t *szp, nni_opt_type t) +{ + pull0_sock *s = arg; + int rv; + int fd; + + if ((rv = nni_pollable_getfd(&s->readable, &fd)) != 0) { + return (rv); + } + return (nni_copyout_int(fd, buf, szp, t)); +} + +static nni_option pull0_sock_options[] = { + { + .o_name = NNG_OPT_RECVFD, + .o_get = pull0_sock_get_recv_fd, + }, + // terminate list + { + .o_name = NULL, + }, +}; + +static nni_proto_pipe_ops pull0_pipe_ops = { + .pipe_size = sizeof(pull0_pipe), + .pipe_init = pull0_pipe_init, + .pipe_fini = pull0_pipe_fini, + .pipe_start = pull0_pipe_start, + .pipe_close = pull0_pipe_close, + .pipe_stop = pull0_pipe_stop, +}; + +static nni_proto_sock_ops pull0_sock_ops = { + .sock_size = sizeof(pull0_sock), + .sock_init = pull0_sock_init, + .sock_fini = pull0_sock_fini, + .sock_open = pull0_sock_open, + .sock_close = pull0_sock_close, + .sock_send = pull0_sock_send, + .sock_recv = pull0_sock_recv, + .sock_options = pull0_sock_options, +}; + +static nni_proto pull0_proto = { + .proto_version = NNI_PROTOCOL_VERSION, + .proto_self = { NNI_PROTO_PULL_V0, "pull" }, + .proto_peer = { NNI_PROTO_PUSH_V0, "push" }, + .proto_flags = NNI_PROTO_FLAG_RCV, + .proto_pipe_ops = &pull0_pipe_ops, + .proto_sock_ops = &pull0_sock_ops, +}; + +static nni_proto pull0_proto_raw = { + .proto_version = NNI_PROTOCOL_VERSION, + .proto_self = { NNI_PROTO_PULL_V0, "pull" }, + .proto_peer = { NNI_PROTO_PUSH_V0, "push" }, + .proto_flags = NNI_PROTO_FLAG_RCV | NNI_PROTO_FLAG_RAW, + .proto_pipe_ops = &pull0_pipe_ops, + .proto_sock_ops = &pull0_sock_ops, +}; + +int +nng_pull0_open(nng_socket *s) +{ + return (nni_proto_open(s, &pull0_proto)); +} + +int +nng_pull0_open_raw(nng_socket *s) +{ + return (nni_proto_open(s, &pull0_proto_raw)); +} diff --git a/src/sp/protocol/pipeline0/pull_test.c b/src/sp/protocol/pipeline0/pull_test.c new file mode 100644 index 00000000..25066093 --- /dev/null +++ b/src/sp/protocol/pipeline0/pull_test.c @@ -0,0 +1,264 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include + +static void +test_pull_identity(void) +{ + nng_socket s; + int p; + char * n; + + NUTS_PASS(nng_pull0_open(&s)); + NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PROTO, &p)); + NUTS_TRUE(p == NUTS_PROTO(5u, 1u)); // 81 + NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PEER, &p)); + NUTS_TRUE(p == NUTS_PROTO(5u, 0u)); // 80 + NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PROTONAME, &n)); + NUTS_MATCH(n, "pull"); + nng_strfree(n); + NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PEERNAME, &n)); + NUTS_MATCH(n, "push"); + nng_strfree(n); + NUTS_CLOSE(s); +} + +static void +test_pull_cannot_send(void) +{ + nng_socket s; + + NUTS_PASS(nng_pull0_open(&s)); + NUTS_FAIL(nng_send(s, "", 0, 0), NNG_ENOTSUP); + NUTS_CLOSE(s); +} + +static void +test_pull_no_context(void) +{ + nng_socket s; + nng_ctx ctx; + + NUTS_PASS(nng_pull0_open(&s)); + NUTS_FAIL(nng_ctx_open(&ctx, s), NNG_ENOTSUP); + NUTS_CLOSE(s); +} + +static void +test_pull_not_writeable(void) +{ + int fd; + nng_socket s; + + NUTS_PASS(nng_pull0_open(&s)); + NUTS_FAIL(nng_socket_get_int(s, NNG_OPT_SENDFD, &fd), NNG_ENOTSUP); + NUTS_CLOSE(s); +} + +static void +test_pull_poll_readable(void) +{ + int fd; + nng_socket pull; + nng_socket push; + + NUTS_PASS(nng_pull0_open(&pull)); + NUTS_PASS(nng_push0_open(&push)); + NUTS_PASS(nng_socket_set_ms(pull, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(push, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_get_int(pull, NNG_OPT_RECVFD, &fd)); + NUTS_TRUE(fd >= 0); + + // Not readable if not connected! + NUTS_TRUE(nuts_poll_fd(fd) == false); + + // Even after connect (no message yet) + NUTS_MARRY(pull, push); + NUTS_TRUE(nuts_poll_fd(fd) == false); + + // But once we send messages, it is. + // We have to send a request, in order to send a reply. + NUTS_SEND(push, "abc"); + NUTS_SLEEP(100); + NUTS_TRUE(nuts_poll_fd(fd)); + + // and receiving makes it no longer ready + NUTS_RECV(pull, "abc"); + NUTS_TRUE(nuts_poll_fd(fd) == false); + + NUTS_CLOSE(pull); + NUTS_CLOSE(push); +} + +static void +test_pull_close_pending(void) +{ + int fd; + nng_socket pull; + nng_socket push; + nng_pipe p1, p2; + char * addr; + + NUTS_ADDR(addr, "inproc"); + + NUTS_PASS(nng_pull0_open(&pull)); + NUTS_PASS(nng_push0_open(&push)); + NUTS_PASS(nng_socket_get_int(pull, NNG_OPT_RECVFD, &fd)); + NUTS_TRUE(fd >= 0); + NUTS_MARRY_EX(pull, push, addr, &p1, &p2); + + // Send a message -- it's ready for reading. + NUTS_SEND(push, "abc"); + NUTS_SLEEP(100); + NUTS_TRUE(nuts_poll_fd(fd)); + + // NB: We have to close the pipe instead of the socket. + // This is because the socket won't notice the remote pipe + // disconnect until we collect the message and start another + // receive operation. + nng_pipe_close(p1); + nng_pipe_close(p2); + + NUTS_SLEEP(100); + NUTS_TRUE(nuts_poll_fd(fd) == false); + + NUTS_CLOSE(pull); + NUTS_CLOSE(push); +} + +void +test_pull_validate_peer(void) +{ + nng_socket s1, s2; + nng_stat * stats; + nng_stat * reject; + char * addr; + + NUTS_ADDR(addr, "inproc"); + + NUTS_PASS(nng_pull0_open(&s1)); + NUTS_PASS(nng_pull0_open(&s2)); + + NUTS_PASS(nng_listen(s1, addr, NULL, 0)); + NUTS_PASS(nng_dial(s2, addr, NULL, NNG_FLAG_NONBLOCK)); + + NUTS_SLEEP(100); + NUTS_PASS(nng_stats_get(&stats)); + + NUTS_TRUE(stats != NULL); + NUTS_TRUE((reject = nng_stat_find_socket(stats, s1)) != NULL); + NUTS_TRUE((reject = nng_stat_find(reject, "reject")) != NULL); + + NUTS_TRUE(nng_stat_type(reject) == NNG_STAT_COUNTER); + NUTS_TRUE(nng_stat_value(reject) > 0); + + NUTS_CLOSE(s1); + NUTS_CLOSE(s2); + nng_stats_free(stats); +} + +static void +test_pull_recv_aio_stopped(void) +{ + nng_socket s; + nng_aio * aio; + + NUTS_PASS(nng_pull0_open(&s)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + + nng_aio_stop(aio); + nng_recv_aio(s, aio); + nng_aio_wait(aio); + NUTS_FAIL(nng_aio_result(aio), NNG_ECANCELED); + NUTS_CLOSE(s); + nng_aio_free(aio); +} + +static void +test_pull_close_recv(void) +{ + nng_socket s; + nng_aio * aio; + + NUTS_PASS(nng_pull0_open(&s)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + nng_aio_set_timeout(aio, 1000); + nng_recv_aio(s, aio); + NUTS_PASS(nng_close(s)); + nng_aio_wait(aio); + NUTS_FAIL(nng_aio_result(aio), NNG_ECLOSED); + + nng_aio_free(aio); +} + +static void +test_pull_recv_nonblock(void) +{ + nng_socket s; + nng_aio * aio; + + NUTS_PASS(nng_pull0_open(&s)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + + nng_aio_set_timeout(aio, 0); // Instant timeout + nng_recv_aio(s, aio); + + nng_aio_wait(aio); + NUTS_FAIL(nng_aio_result(aio), NNG_ETIMEDOUT); + NUTS_CLOSE(s); + nng_aio_free(aio); +} + +static void +test_pull_recv_cancel(void) +{ + nng_socket s; + nng_aio * aio; + + NUTS_PASS(nng_pull0_open(&s)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + + nng_aio_set_timeout(aio, 1000); + nng_recv_aio(s, aio); + nng_aio_abort(aio, NNG_ECANCELED); + + nng_aio_wait(aio); + NUTS_FAIL(nng_aio_result(aio), NNG_ECANCELED); + NUTS_CLOSE(s); + nng_aio_free(aio); +} + +static void +test_pull_cooked(void) +{ + nng_socket s; + bool b; + + NUTS_PASS(nng_pull0_open(&s)); + NUTS_PASS(nng_socket_get_bool(s, NNG_OPT_RAW, &b)); + NUTS_TRUE(!b); + NUTS_CLOSE(s); +} + +TEST_LIST = { + { "pull identity", test_pull_identity }, + { "pull cannot send", test_pull_cannot_send }, + { "pull no context", test_pull_no_context }, + { "pull not writeable", test_pull_not_writeable }, + { "pull poll readable", test_pull_poll_readable }, + { "pull close pending", test_pull_close_pending }, + { "pull validate peer", test_pull_validate_peer }, + { "pull recv aio stopped", test_pull_recv_aio_stopped }, + { "pull close recv", test_pull_close_recv }, + { "pull recv nonblock", test_pull_recv_nonblock }, + { "pull recv cancel", test_pull_recv_cancel }, + { "pull cooked", test_pull_cooked }, + { NULL, NULL }, +}; diff --git a/src/sp/protocol/pipeline0/push.c b/src/sp/protocol/pipeline0/push.c new file mode 100644 index 00000000..ad43d967 --- /dev/null +++ b/src/sp/protocol/pipeline0/push.c @@ -0,0 +1,442 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// Copyright 2018 Capitar IT Group BV +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include + +#include "core/nng_impl.h" +#include "nng/protocol/pipeline0/push.h" + +// Push protocol. The PUSH protocol is the "write" side of a pipeline. +// Push distributes fairly, or tries to, by giving messages in round-robin +// order. + +#ifndef NNI_PROTO_PULL_V0 +#define NNI_PROTO_PULL_V0 NNI_PROTO(5, 1) +#endif + +#ifndef NNI_PROTO_PUSH_V0 +#define NNI_PROTO_PUSH_V0 NNI_PROTO(5, 0) +#endif + +typedef struct push0_pipe push0_pipe; +typedef struct push0_sock push0_sock; + +static void push0_send_cb(void *); +static void push0_recv_cb(void *); +static void push0_pipe_ready(push0_pipe *); + +// push0_sock is our per-socket protocol private structure. +struct push0_sock { + nni_lmq wq; // list of messages queued + nni_list aq; // list of aio senders waiting + nni_list pl; // list of pipes ready to send + nni_pollable writable; + nni_mtx m; +}; + +// push0_pipe is our per-pipe protocol private structure. +struct push0_pipe { + nni_pipe * pipe; + push0_sock * push; + nni_list_node node; + + nni_aio aio_recv; + nni_aio aio_send; +}; + +static int +push0_sock_init(void *arg, nni_sock *sock) +{ + push0_sock *s = arg; + NNI_ARG_UNUSED(sock); + + nni_mtx_init(&s->m); + nni_aio_list_init(&s->aq); + NNI_LIST_INIT(&s->pl, push0_pipe, node); + nni_lmq_init(&s->wq, 0); // initially we start unbuffered. + nni_pollable_init(&s->writable); + + return (0); +} + +static void +push0_sock_fini(void *arg) +{ + push0_sock *s = arg; + nni_pollable_fini(&s->writable); + nni_lmq_fini(&s->wq); + nni_mtx_fini(&s->m); +} + +static void +push0_sock_open(void *arg) +{ + NNI_ARG_UNUSED(arg); +} + +static void +push0_sock_close(void *arg) +{ + push0_sock *s = arg; + nni_aio * a; + nni_mtx_lock(&s->m); + while ((a = nni_list_first(&s->aq)) != NULL) { + nni_aio_list_remove(a); + nni_aio_finish_error(a, NNG_ECLOSED); + } + nni_mtx_unlock(&s->m); +} + +static void +push0_pipe_stop(void *arg) +{ + push0_pipe *p = arg; + + nni_aio_stop(&p->aio_recv); + nni_aio_stop(&p->aio_send); +} + +static void +push0_pipe_fini(void *arg) +{ + push0_pipe *p = arg; + + nni_aio_fini(&p->aio_recv); + nni_aio_fini(&p->aio_send); +} + +static int +push0_pipe_init(void *arg, nni_pipe *pipe, void *s) +{ + push0_pipe *p = arg; + + nni_aio_init(&p->aio_recv, push0_recv_cb, p); + nni_aio_init(&p->aio_send, push0_send_cb, p); + NNI_LIST_NODE_INIT(&p->node); + p->pipe = pipe; + p->push = s; + return (0); +} + +static int +push0_pipe_start(void *arg) +{ + push0_pipe *p = arg; + + if (nni_pipe_peer(p->pipe) != NNI_PROTO_PULL_V0) { + return (NNG_EPROTO); + } + + // Schedule a receiver. This is mostly so that we can detect + // a closed transport pipe. + nni_pipe_recv(p->pipe, &p->aio_recv); + push0_pipe_ready(p); + + return (0); +} + +static void +push0_pipe_close(void *arg) +{ + push0_pipe *p = arg; + push0_sock *s = p->push; + + nni_aio_close(&p->aio_recv); + nni_aio_close(&p->aio_send); + + nni_mtx_lock(&s->m); + if (nni_list_node_active(&p->node)) { + nni_list_node_remove(&p->node); + + if (nni_list_empty(&s->pl) && nni_lmq_full(&s->wq)) { + nni_pollable_clear(&s->writable); + } + } + nni_mtx_unlock(&s->m); +} + +static void +push0_recv_cb(void *arg) +{ + push0_pipe *p = arg; + + // We normally expect to receive an error. If a pipe actually + // sends us data, we just discard it. + if (nni_aio_result(&p->aio_recv) != 0) { + nni_pipe_close(p->pipe); + return; + } + nni_msg_free(nni_aio_get_msg(&p->aio_recv)); + nni_aio_set_msg(&p->aio_recv, NULL); + nni_pipe_recv(p->pipe, &p->aio_recv); +} + +static void +push0_pipe_ready(push0_pipe *p) +{ + push0_sock *s = p->push; + nni_msg * m; + nni_aio * a = NULL; + size_t l; + bool blocked; + + nni_mtx_lock(&s->m); + + blocked = nni_lmq_full(&s->wq) && nni_list_empty(&s->pl); + + // if message is waiting in the buffered queue + // then we prefer that. + if (nni_lmq_getq(&s->wq, &m) == 0) { + nni_aio_set_msg(&p->aio_send, m); + nni_pipe_send(p->pipe, &p->aio_send); + + if ((a = nni_list_first(&s->aq)) != NULL) { + nni_aio_list_remove(a); + m = nni_aio_get_msg(a); + l = nni_msg_len(m); + nni_lmq_putq(&s->wq, m); + } + + } else if ((a = nni_list_first(&s->aq)) != NULL) { + // Looks like we had the unbuffered case, but + // someone was waiting. + nni_aio_list_remove(a); + m = nni_aio_get_msg(a); + l = nni_msg_len(m); + + nni_aio_set_msg(&p->aio_send, m); + nni_pipe_send(p->pipe, &p->aio_send); + } else { + // We had nothing to send. Just put us in the ready list. + nni_list_append(&s->pl, p); + } + + if (blocked) { + // if we blocked, then toggle the status. + if ((!nni_lmq_full(&s->wq)) || (!nni_list_empty(&s->pl))) { + nni_pollable_raise(&s->writable); + } + } + + nni_mtx_unlock(&s->m); + + if (a != NULL) { + nni_aio_set_msg(a, NULL); + nni_aio_finish_sync(a, 0, l); + } +} + +static void +push0_send_cb(void *arg) +{ + push0_pipe *p = arg; + + if (nni_aio_result(&p->aio_send) != 0) { + nni_msg_free(nni_aio_get_msg(&p->aio_send)); + nni_aio_set_msg(&p->aio_send, NULL); + nni_pipe_close(p->pipe); + return; + } + + push0_pipe_ready(p); +} + +static void +push0_cancel(nni_aio *aio, void *arg, int rv) +{ + push0_sock *s = arg; + + nni_mtx_lock(&s->m); + if (nni_aio_list_active(aio)) { + nni_aio_list_remove(aio); + nni_aio_finish_error(aio, rv); + } + nni_mtx_unlock(&s->m); +} + +static void +push0_sock_send(void *arg, nni_aio *aio) +{ + push0_sock *s = arg; + push0_pipe *p; + nni_msg * m; + size_t l; + int rv; + + if (nni_aio_begin(aio) != 0) { + return; + } + + m = nni_aio_get_msg(aio); + l = nni_msg_len(m); + + nni_mtx_lock(&s->m); + + // First we want to see if we can send it right now. + // Note that we don't block the sender until the read is complete, + // only until we have committed to send it. + if ((p = nni_list_first(&s->pl)) != NULL) { + nni_list_remove(&s->pl, p); + // NB: We won't have had any waiters in the message queue + // or the aio queue, because we would not put the pipe + // in the ready list in that case. Note though that the + // wq may be "full" if we are unbuffered. + if (nni_list_empty(&s->pl) && (nni_lmq_full(&s->wq))) { + nni_pollable_clear(&s->writable); + } + nni_aio_set_msg(aio, NULL); + nni_aio_finish(aio, 0, l); + nni_aio_set_msg(&p->aio_send, m); + nni_pipe_send(p->pipe, &p->aio_send); + nni_mtx_unlock(&s->m); + return; + } + + // Can we maybe queue it. + if (nni_lmq_putq(&s->wq, m) == 0) { + // Yay, we can. So we're done. + nni_aio_set_msg(aio, NULL); + nni_aio_finish(aio, 0, l); + if (nni_lmq_full(&s->wq)) { + nni_pollable_clear(&s->writable); + } + nni_mtx_unlock(&s->m); + return; + } + + if ((rv = nni_aio_schedule(aio, push0_cancel, s)) != 0) { + nni_aio_finish_error(aio, rv); + nni_mtx_unlock(&s->m); + return; + } + nni_aio_list_append(&s->aq, aio); + nni_mtx_unlock(&s->m); +} + +static void +push0_sock_recv(void *arg, nni_aio *aio) +{ + NNI_ARG_UNUSED(arg); + nni_aio_finish_error(aio, NNG_ENOTSUP); +} + +static int +push0_set_send_buf_len(void *arg, const void *buf, size_t sz, nni_type t) +{ + push0_sock *s = arg; + int val; + int rv; + + if ((rv = nni_copyin_int(&val, buf, sz, 0, 8192, t)) != 0) { + return (rv); + } + nni_mtx_lock(&s->m); + rv = nni_lmq_resize(&s->wq, (size_t) val); + // Changing the size of the queue can affect our readiness. + if (!nni_lmq_full(&s->wq)) { + nni_pollable_raise(&s->writable); + } else if (nni_list_empty(&s->pl)) { + nni_pollable_clear(&s->writable); + } + nni_mtx_unlock(&s->m); + return (rv); +} + +static int +push0_get_send_buf_len(void *arg, void *buf, size_t *szp, nni_opt_type t) +{ + push0_sock *s = arg; + int val; + + nni_mtx_lock(&s->m); + val = nni_lmq_cap(&s->wq); + nni_mtx_unlock(&s->m); + + return (nni_copyout_int(val, buf, szp, t)); +} + +static int +push0_sock_get_send_fd(void *arg, void *buf, size_t *szp, nni_opt_type t) +{ + push0_sock *s = arg; + int rv; + int fd; + + if ((rv = nni_pollable_getfd(&s->writable, &fd)) != 0) { + return (rv); + } + return (nni_copyout_int(fd, buf, szp, t)); +} + +static nni_proto_pipe_ops push0_pipe_ops = { + .pipe_size = sizeof(push0_pipe), + .pipe_init = push0_pipe_init, + .pipe_fini = push0_pipe_fini, + .pipe_start = push0_pipe_start, + .pipe_close = push0_pipe_close, + .pipe_stop = push0_pipe_stop, +}; + +static nni_option push0_sock_options[] = { + { + .o_name = NNG_OPT_SENDFD, + .o_get = push0_sock_get_send_fd, + }, + { + .o_name = NNG_OPT_SENDBUF, + .o_get = push0_get_send_buf_len, + .o_set = push0_set_send_buf_len, + }, + // terminate list + { + .o_name = NULL, + }, +}; + +static nni_proto_sock_ops push0_sock_ops = { + .sock_size = sizeof(push0_sock), + .sock_init = push0_sock_init, + .sock_fini = push0_sock_fini, + .sock_open = push0_sock_open, + .sock_close = push0_sock_close, + .sock_options = push0_sock_options, + .sock_send = push0_sock_send, + .sock_recv = push0_sock_recv, +}; + +static nni_proto push0_proto = { + .proto_version = NNI_PROTOCOL_VERSION, + .proto_self = { NNI_PROTO_PUSH_V0, "push" }, + .proto_peer = { NNI_PROTO_PULL_V0, "pull" }, + .proto_flags = NNI_PROTO_FLAG_SND, + .proto_pipe_ops = &push0_pipe_ops, + .proto_sock_ops = &push0_sock_ops, +}; + +static nni_proto push0_proto_raw = { + .proto_version = NNI_PROTOCOL_VERSION, + .proto_self = { NNI_PROTO_PUSH_V0, "push" }, + .proto_peer = { NNI_PROTO_PULL_V0, "pull" }, + .proto_flags = NNI_PROTO_FLAG_SND | NNI_PROTO_FLAG_RAW, + .proto_pipe_ops = &push0_pipe_ops, + .proto_sock_ops = &push0_sock_ops, +}; + +int +nng_push0_open(nng_socket *s) +{ + return (nni_proto_open(s, &push0_proto)); +} + +int +nng_push0_open_raw(nng_socket *s) +{ + return (nni_proto_open(s, &push0_proto_raw)); +} diff --git a/src/sp/protocol/pipeline0/push_test.c b/src/sp/protocol/pipeline0/push_test.c new file mode 100644 index 00000000..d22ccaa4 --- /dev/null +++ b/src/sp/protocol/pipeline0/push_test.c @@ -0,0 +1,525 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include + +static void +test_push_identity(void) +{ + nng_socket s; + int p; + char * n; + + NUTS_PASS(nng_push0_open(&s)); + NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PROTO, &p)); + NUTS_TRUE(p == NUTS_PROTO(5u, 0u)); // 80 + NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PEER, &p)); + NUTS_TRUE(p == NUTS_PROTO(5u, 1u)); // 81 + NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PROTONAME, &n)); + NUTS_MATCH(n, "push"); + nng_strfree(n); + NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PEERNAME, &n)); + NUTS_MATCH(n, "pull"); + nng_strfree(n); + NUTS_CLOSE(s); +} + +static void +test_push_cannot_recv(void) +{ + nng_socket s; + nng_msg * m = NULL; + + NUTS_PASS(nng_push0_open(&s)); + NUTS_FAIL(nng_recvmsg(s, &m, 0), NNG_ENOTSUP); + NUTS_CLOSE(s); +} + +static void +test_push_no_context(void) +{ + nng_socket s; + nng_ctx ctx; + + NUTS_PASS(nng_push0_open(&s)); + NUTS_FAIL(nng_ctx_open(&ctx, s), NNG_ENOTSUP); + NUTS_CLOSE(s); +} + +static void +test_push_not_readable(void) +{ + int fd; + nng_socket s; + + NUTS_PASS(nng_push0_open(&s)); + NUTS_FAIL(nng_socket_get_int(s, NNG_OPT_RECVFD, &fd), NNG_ENOTSUP); + NUTS_CLOSE(s); +} + +static void +test_push_poll_writable(void) +{ + int fd; + nng_socket pull; + nng_socket push; + + NUTS_PASS(nng_pull0_open(&pull)); + NUTS_PASS(nng_push0_open(&push)); + NUTS_PASS(nng_socket_set_ms(pull, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(push, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_get_int(push, NNG_OPT_SENDFD, &fd)); + NUTS_TRUE(fd >= 0); + + // This tests unbuffered sockets for now. + // Note that for this we are using unbuffered inproc. + // If using TCP or similar, then transport buffering will + // break assumptions in this test. + + // Not writable if not connected! + NUTS_TRUE(nuts_poll_fd(fd) == false); + + // After connect we can write. + NUTS_MARRY(pull, push); + NUTS_TRUE(nuts_poll_fd(fd) == true); + + // But once we send a message, it is not anymore. + NUTS_SEND(push, "abc"); + // Have to send a second message, because the remote socket + // will have consumed the first one. + NUTS_SEND(push, "def"); + NUTS_SLEEP(100); + NUTS_TRUE(nuts_poll_fd(fd) == false); + + // and receiving receiving the message makes it possible again. + NUTS_RECV(pull, "abc"); + NUTS_SLEEP(100); + NUTS_TRUE(nuts_poll_fd(fd)); + + NUTS_CLOSE(pull); + NUTS_CLOSE(push); +} + +static void +test_push_poll_buffered(void) +{ + int fd; + nng_socket pull; + nng_socket push; + + NUTS_PASS(nng_pull0_open(&pull)); + NUTS_PASS(nng_push0_open(&push)); + NUTS_PASS(nng_socket_set_ms(pull, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(push, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_int(push, NNG_OPT_SENDBUF, 2)); + NUTS_PASS(nng_socket_get_int(push, NNG_OPT_SENDFD, &fd)); + NUTS_TRUE(fd >= 0); + + // We can write two message while unbuffered. + NUTS_TRUE(nuts_poll_fd(fd)); + NUTS_SEND(push, "abc"); + NUTS_TRUE(nuts_poll_fd(fd)); + NUTS_SEND(push, "def"); + NUTS_TRUE(nuts_poll_fd(fd) == false); + + // After connect we remote end will pick up one of them. + // Also, the local pipe itself will pick up one. So we + // have two. + NUTS_MARRY(pull, push); + NUTS_SLEEP(100); + NUTS_TRUE(nuts_poll_fd(fd)); + NUTS_SEND(push, "ghi"); + NUTS_SLEEP(100); + NUTS_TRUE(nuts_poll_fd(fd)); + NUTS_SEND(push, "jkl"); + // Now it should be full. + NUTS_TRUE(nuts_poll_fd(fd) == false); + + // and receiving receiving the message makes it possible again. + NUTS_RECV(pull, "abc"); + NUTS_SLEEP(100); + NUTS_TRUE(nuts_poll_fd(fd)); + NUTS_RECV(pull, "def"); + NUTS_RECV(pull, "ghi"); + NUTS_RECV(pull, "jkl"); + + NUTS_CLOSE(pull); + NUTS_CLOSE(push); +} + +static void +test_push_poll_truncate(void) +{ + int fd; + nng_socket pull; + nng_socket push; + + // This test starts with a buffer and then truncates it to see + // that shortening the buffer has an impact. + + NUTS_PASS(nng_pull0_open(&pull)); + NUTS_PASS(nng_push0_open(&push)); + NUTS_PASS(nng_socket_set_ms(pull, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(push, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_int(push, NNG_OPT_SENDBUF, 3)); + NUTS_PASS(nng_socket_get_int(push, NNG_OPT_SENDFD, &fd)); + NUTS_TRUE(fd >= 0); + + // We can write two message while unbuffered. + NUTS_TRUE(nuts_poll_fd(fd)); + NUTS_SEND(push, "abc"); + NUTS_TRUE(nuts_poll_fd(fd)); + NUTS_SEND(push, "def"); + NUTS_TRUE(nuts_poll_fd(fd)); + + NUTS_PASS(nng_socket_set_int(push, NNG_OPT_SENDBUF, 1)); + NUTS_TRUE(nuts_poll_fd(fd) == false); + + NUTS_MARRY(pull, push); + NUTS_RECV(pull, "abc"); + // def got dropped + NUTS_SEND(push, "ghi"); + NUTS_RECV(pull, "ghi"); + + NUTS_CLOSE(pull); + NUTS_SLEEP(100); + + // We have a buffer of one. + NUTS_SEND(push, "jkl"); + // Resize to 0 (unbuffered) + NUTS_PASS(nng_socket_set_int(push, NNG_OPT_SENDBUF, 0)); + + // reopen the pull socket and connect it + NUTS_PASS(nng_pull0_open(&pull)); + NUTS_MARRY(push, pull); + + // jkl got dropped. + NUTS_SEND(push, "mno"); + NUTS_RECV(pull, "mno"); + + NUTS_CLOSE(pull); + NUTS_CLOSE(push); +} + +void +test_push_validate_peer(void) +{ + nng_socket s1, s2; + nng_stat * stats; + nng_stat * reject; + char * addr; + + NUTS_ADDR(addr, "inproc"); + + NUTS_PASS(nng_push0_open(&s1)); + NUTS_PASS(nng_push0_open(&s2)); + + NUTS_PASS(nng_listen(s1, addr, NULL, 0)); + NUTS_PASS(nng_dial(s2, addr, NULL, NNG_FLAG_NONBLOCK)); + + NUTS_SLEEP(100); + NUTS_PASS(nng_stats_get(&stats)); + + NUTS_TRUE(stats != NULL); + NUTS_TRUE((reject = nng_stat_find_socket(stats, s1)) != NULL); + NUTS_TRUE((reject = nng_stat_find(reject, "reject")) != NULL); + + NUTS_TRUE(nng_stat_type(reject) == NNG_STAT_COUNTER); + NUTS_TRUE(nng_stat_value(reject) > 0); + + NUTS_CLOSE(s1); + NUTS_CLOSE(s2); + nng_stats_free(stats); +} + +static void +test_push_send_aio_stopped(void) +{ + nng_socket s; + nng_aio * aio; + nng_msg * m; + + NUTS_PASS(nng_push0_open(&s)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + NUTS_PASS(nng_msg_alloc(&m, 0)); + + nng_aio_set_msg(aio, m); + nng_aio_stop(aio); + nng_send_aio(s, aio); + nng_aio_wait(aio); + NUTS_FAIL(nng_aio_result(aio), NNG_ECANCELED); + NUTS_CLOSE(s); + nng_aio_free(aio); + nng_msg_free(m); +} + +static void +test_push_close_send(void) +{ + nng_socket s; + nng_aio * aio; + nng_msg * m; + + NUTS_PASS(nng_push0_open(&s)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + NUTS_PASS(nng_msg_alloc(&m, 0)); + nng_aio_set_timeout(aio, 1000); + nng_aio_set_msg(aio, m); + nng_send_aio(s, aio); + NUTS_PASS(nng_close(s)); + nng_aio_wait(aio); + NUTS_FAIL(nng_aio_result(aio), NNG_ECLOSED); + + nng_aio_free(aio); + nng_msg_free(m); +} + +static void +test_push_send_nonblock(void) +{ + nng_socket s; + nng_aio * aio; + nng_msg * m; + + NUTS_PASS(nng_push0_open(&s)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + NUTS_PASS(nng_msg_alloc(&m, 0)); + + nng_aio_set_timeout(aio, 0); // Instant timeout + nng_aio_set_msg(aio, m); + nng_send_aio(s, aio); + + nng_aio_wait(aio); + NUTS_FAIL(nng_aio_result(aio), NNG_ETIMEDOUT); + NUTS_CLOSE(s); + nng_aio_free(aio); + nng_msg_free(m); +} + +static void +test_push_send_timeout(void) +{ + nng_socket s; + nng_aio * aio; + nng_msg * m; + + NUTS_PASS(nng_push0_open(&s)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + NUTS_PASS(nng_msg_alloc(&m, 0)); + + nng_aio_set_timeout(aio, 10); + nng_aio_set_msg(aio, m); + nng_send_aio(s, aio); + + nng_aio_wait(aio); + NUTS_FAIL(nng_aio_result(aio), NNG_ETIMEDOUT); + NUTS_CLOSE(s); + nng_aio_free(aio); + nng_msg_free(m); +} + +static void +test_push_send_cancel(void) +{ + nng_socket s; + nng_aio * aio; + nng_msg * m; + + NUTS_PASS(nng_push0_open(&s)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + NUTS_PASS(nng_msg_alloc(&m, 0)); + + nng_aio_set_timeout(aio, 1000); + nng_aio_set_msg(aio, m); + nng_send_aio(s, aio); + nng_aio_abort(aio, NNG_ECANCELED); + + nng_aio_wait(aio); + NUTS_FAIL(nng_aio_result(aio), NNG_ECANCELED); + NUTS_CLOSE(s); + nng_aio_free(aio); + nng_msg_free(m); +} + +static void +test_push_send_late_unbuffered(void) +{ + nng_socket s; + nng_socket pull; + nng_aio * aio; + nng_msg * m; + + NUTS_PASS(nng_push0_open(&s)); + NUTS_PASS(nng_pull0_open(&pull)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + NUTS_PASS(nng_msg_alloc(&m, 0)); + NUTS_PASS(nng_msg_append(m, "123\0", 4)); + + nng_aio_set_timeout(aio, 1000); + nng_aio_set_msg(aio, m); + nng_send_aio(s, aio); + + NUTS_MARRY(s, pull); + + NUTS_RECV(pull, "123"); + + nng_aio_wait(aio); + NUTS_PASS(nng_aio_result(aio)); + NUTS_CLOSE(s); + nng_aio_free(aio); +} + + +static void +test_push_send_late_buffered(void) +{ + nng_socket s; + nng_socket pull; + nng_aio * aio; + nng_msg * m; + + NUTS_PASS(nng_push0_open(&s)); + NUTS_PASS(nng_pull0_open(&pull)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + NUTS_PASS(nng_setopt_int(s, NNG_OPT_SENDBUF, 2)); + NUTS_PASS(nng_msg_alloc(&m, 0)); + NUTS_PASS(nng_msg_append(m, "123\0", 4)); + + nng_aio_set_timeout(aio, 1000); + nng_aio_set_msg(aio, m); + nng_send_aio(s, aio); + + NUTS_MARRY(s, pull); + + NUTS_RECV(pull, "123"); + + nng_aio_wait(aio); + NUTS_PASS(nng_aio_result(aio)); + NUTS_CLOSE(s); + nng_aio_free(aio); +} + +static void +test_push_cooked(void) +{ + nng_socket s; + bool b; + + NUTS_PASS(nng_push0_open(&s)); + NUTS_PASS(nng_socket_get_bool(s, NNG_OPT_RAW, &b)); + NUTS_TRUE(!b); + NUTS_CLOSE(s); +} + +static void +test_push_load_balance_buffered(void) +{ + nng_socket s; + nng_socket pull1; + nng_socket pull2; + nng_socket pull3; + + NUTS_PASS(nng_push0_open(&s)); + NUTS_PASS(nng_pull0_open(&pull1)); + NUTS_PASS(nng_pull0_open(&pull2)); + NUTS_PASS(nng_pull0_open(&pull3)); + NUTS_PASS(nng_setopt_int(s, NNG_OPT_SENDBUF, 4)); + NUTS_MARRY(s, pull1); + NUTS_MARRY(s, pull2); + NUTS_MARRY(s, pull3); + NUTS_SLEEP(100); + NUTS_SEND(s, "one"); + NUTS_SEND(s, "two"); + NUTS_SEND(s, "three"); + NUTS_RECV(pull1, "one"); + NUTS_RECV(pull2, "two"); + NUTS_RECV(pull3, "three"); + NUTS_CLOSE(s); + NUTS_CLOSE(pull1); + NUTS_CLOSE(pull2); + NUTS_CLOSE(pull3); +} + +static void +test_push_load_balance_unbuffered(void) +{ + nng_socket s; + nng_socket pull1; + nng_socket pull2; + nng_socket pull3; + + NUTS_PASS(nng_push0_open(&s)); + NUTS_PASS(nng_pull0_open(&pull1)); + NUTS_PASS(nng_pull0_open(&pull2)); + NUTS_PASS(nng_pull0_open(&pull3)); + NUTS_MARRY(s, pull1); + NUTS_MARRY(s, pull2); + NUTS_MARRY(s, pull3); + NUTS_SLEEP(100); + NUTS_SEND(s, "one"); + NUTS_SEND(s, "two"); + NUTS_SEND(s, "three"); + NUTS_RECV(pull1, "one"); + NUTS_RECV(pull2, "two"); + NUTS_RECV(pull3, "three"); + // Loop around is unpredictable somewhat, because the the + // pull sockets can take different periods of time to return + // back to readiness. + NUTS_CLOSE(s); + NUTS_CLOSE(pull1); + NUTS_CLOSE(pull2); + NUTS_CLOSE(pull3); +} + +static void +test_push_send_buffer(void) +{ + nng_socket s; + int v; + bool b; + size_t sz; + + NUTS_PASS(nng_push0_open(&s)); + NUTS_PASS(nng_getopt_int(s, NNG_OPT_SENDBUF, &v)); + NUTS_TRUE(v == 0); + NUTS_FAIL(nng_getopt_bool(s, NNG_OPT_SENDBUF, &b), NNG_EBADTYPE); + sz = 1; + NUTS_FAIL(nng_getopt(s, NNG_OPT_SENDBUF, &b, &sz), NNG_EINVAL); + NUTS_FAIL(nng_setopt_int(s, NNG_OPT_SENDBUF, -1), NNG_EINVAL); + NUTS_FAIL(nng_setopt_int(s, NNG_OPT_SENDBUF, 100000), NNG_EINVAL); + NUTS_FAIL(nng_setopt_bool(s, NNG_OPT_SENDBUF, false), NNG_EBADTYPE); + NUTS_FAIL(nng_setopt(s, NNG_OPT_SENDBUF, &b, 1), NNG_EINVAL); + NUTS_PASS(nng_setopt_int(s, NNG_OPT_SENDBUF, 100)); + NUTS_PASS(nng_getopt_int(s, NNG_OPT_SENDBUF, &v)); + NUTS_TRUE(v == 100); + NUTS_CLOSE(s); +} + +TEST_LIST = { + { "push identity", test_push_identity }, + { "push cannot recv", test_push_cannot_recv }, + { "push no context", test_push_no_context }, + { "push not readable", test_push_not_readable }, + { "push poll writable", test_push_poll_writable }, + { "push poll buffered", test_push_poll_buffered }, + { "push poll truncate", test_push_poll_truncate }, + { "push validate peer", test_push_validate_peer }, + { "push send aio stopped", test_push_send_aio_stopped }, + { "push close send", test_push_close_send }, + { "push send nonblock", test_push_send_nonblock }, + { "push send timeout", test_push_send_timeout }, + { "push send cancel", test_push_send_cancel }, + { "push send late buffered", test_push_send_late_buffered }, + { "push send late unbuffered", test_push_send_late_unbuffered }, + { "push cooked", test_push_cooked }, + { "push load balance buffered", test_push_load_balance_buffered }, + { "push load balance unbuffered", test_push_load_balance_unbuffered }, + { "push send buffer", test_push_send_buffer }, + { NULL, NULL }, +}; diff --git a/src/sp/protocol/pubsub0/CMakeLists.txt b/src/sp/protocol/pubsub0/CMakeLists.txt new file mode 100644 index 00000000..160b7462 --- /dev/null +++ b/src/sp/protocol/pubsub0/CMakeLists.txt @@ -0,0 +1,24 @@ +# +# Copyright 2019 Staysail Systems, Inc. +# Copyright 2018 Capitar IT Group BV +# +# This software is supplied under the terms of the MIT License, a +# copy of which should be located in the distribution where this +# file was obtained (LICENSE.txt). A copy of the license may also be +# found online at https://opensource.org/licenses/MIT. +# + +# Pub/Sub protocol +nng_directory(pubsub0) + +nng_sources_if(NNG_PROTO_PUB0 pub.c) +nng_headers_if(NNG_PROTO_PUB0 nng/protocol/pubsub0/pub.h) +nng_defines_if(NNG_PROTO_PUB0 NNG_HAVE_PUB0) + +nng_sources_if(NNG_PROTO_SUB0 sub.c xsub.c) +nng_headers_if(NNG_PROTO_SUB0 nng/protocol/pubsub0/sub.h) +nng_defines_if(NNG_PROTO_SUB0 NNG_HAVE_SUB0) + +nng_test(pub_test) +nng_test(sub_test) +nng_test(xsub_test) diff --git a/src/sp/protocol/pubsub0/pub.c b/src/sp/protocol/pubsub0/pub.c new file mode 100644 index 00000000..e3d4f16a --- /dev/null +++ b/src/sp/protocol/pubsub0/pub.c @@ -0,0 +1,383 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// Copyright 2018 Capitar IT Group BV +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include + +#include "core/nng_impl.h" +#include "nng/protocol/pubsub0/pub.h" + +// Publish protocol. The PUB protocol simply sends messages out, as +// a broadcast. It has nothing more sophisticated because it does not +// perform sender-side filtering. Its best effort delivery, so anything +// that can't receive the message won't get one. + +#ifndef NNI_PROTO_SUB_V0 +#define NNI_PROTO_SUB_V0 NNI_PROTO(2, 1) +#endif + +#ifndef NNI_PROTO_PUB_V0 +#define NNI_PROTO_PUB_V0 NNI_PROTO(2, 0) +#endif + +typedef struct pub0_pipe pub0_pipe; +typedef struct pub0_sock pub0_sock; + +static void pub0_pipe_recv_cb(void *); +static void pub0_pipe_send_cb(void *); +static void pub0_sock_fini(void *); +static void pub0_pipe_fini(void *); + +// pub0_sock is our per-socket protocol private structure. +struct pub0_sock { + nni_list pipes; + nni_mtx mtx; + bool closed; + size_t sendbuf; + nni_pollable *sendable; +}; + +// pub0_pipe is our per-pipe protocol private structure. +struct pub0_pipe { + nni_pipe * pipe; + pub0_sock * pub; + nni_lmq sendq; + bool closed; + bool busy; + nni_aio * aio_send; + nni_aio * aio_recv; + nni_list_node node; +}; + +static void +pub0_sock_fini(void *arg) +{ + pub0_sock *s = arg; + + nni_pollable_free(s->sendable); + nni_mtx_fini(&s->mtx); +} + +static int +pub0_sock_init(void *arg, nni_sock *nsock) +{ + pub0_sock *sock = arg; + int rv; + NNI_ARG_UNUSED(nsock); + + if ((rv = nni_pollable_alloc(&sock->sendable)) != 0) { + return (rv); + } + nni_mtx_init(&sock->mtx); + NNI_LIST_INIT(&sock->pipes, pub0_pipe, node); + sock->sendbuf = 16; // fairly arbitrary + return (0); +} + +static void +pub0_sock_open(void *arg) +{ + NNI_ARG_UNUSED(arg); +} + +static void +pub0_sock_close(void *arg) +{ + NNI_ARG_UNUSED(arg); +} + +static void +pub0_pipe_stop(void *arg) +{ + pub0_pipe *p = arg; + + nni_aio_stop(p->aio_send); + nni_aio_stop(p->aio_recv); +} + +static void +pub0_pipe_fini(void *arg) +{ + pub0_pipe *p = arg; + + nni_aio_free(p->aio_send); + nni_aio_free(p->aio_recv); + nni_lmq_fini(&p->sendq); +} + +static int +pub0_pipe_init(void *arg, nni_pipe *pipe, void *s) +{ + pub0_pipe *p = arg; + pub0_sock *sock = s; + int rv; + size_t len; + + nni_mtx_lock(&sock->mtx); + len = sock->sendbuf; + nni_mtx_unlock(&sock->mtx); + + // XXX: consider making this depth tunable + if (((rv = nni_lmq_init(&p->sendq, len)) != 0) || + ((rv = nni_aio_alloc(&p->aio_send, pub0_pipe_send_cb, p)) != 0) || + ((rv = nni_aio_alloc(&p->aio_recv, pub0_pipe_recv_cb, p)) != 0)) { + + pub0_pipe_fini(p); + return (rv); + } + + p->busy = false; + p->pipe = pipe; + p->pub = s; + return (0); +} + +static int +pub0_pipe_start(void *arg) +{ + pub0_pipe *p = arg; + pub0_sock *sock = p->pub; + + if (nni_pipe_peer(p->pipe) != NNI_PROTO_SUB_V0) { + return (NNG_EPROTO); + } + nni_mtx_lock(&sock->mtx); + nni_list_append(&sock->pipes, p); + nni_mtx_unlock(&sock->mtx); + + // Start the receiver. + nni_pipe_recv(p->pipe, p->aio_recv); + + return (0); +} + +static void +pub0_pipe_close(void *arg) +{ + pub0_pipe *p = arg; + pub0_sock *sock = p->pub; + + nni_aio_close(p->aio_send); + nni_aio_close(p->aio_recv); + + nni_mtx_lock(&sock->mtx); + p->closed = true; + nni_lmq_flush(&p->sendq); + + if (nni_list_active(&sock->pipes, p)) { + nni_list_remove(&sock->pipes, p); + } + nni_mtx_unlock(&sock->mtx); +} + +static void +pub0_pipe_recv_cb(void *arg) +{ + pub0_pipe *p = arg; + + // We should never receive a message -- the only valid reason for us to + // be here is on pipe close. + if (nni_aio_result(p->aio_recv) == 0) { + nni_msg_free(nni_aio_get_msg(p->aio_recv)); + } + nni_pipe_close(p->pipe); +} + +static void +pub0_pipe_send_cb(void *arg) +{ + pub0_pipe *p = arg; + pub0_sock *sock = p->pub; + nni_msg * msg; + + if (nni_aio_result(p->aio_send) != 0) { + nni_msg_free(nni_aio_get_msg(p->aio_send)); + nni_aio_set_msg(p->aio_send, NULL); + nni_pipe_close(p->pipe); + return; + } + + nni_mtx_lock(&sock->mtx); + if (p->closed) { + nni_mtx_unlock(&sock->mtx); + return; + } + if (nni_lmq_getq(&p->sendq, &msg) == 0) { + nni_aio_set_msg(p->aio_send, msg); + nni_pipe_send(p->pipe, p->aio_send); + } else { + p->busy = false; + } + nni_mtx_unlock(&sock->mtx); +} + +static void +pub0_sock_recv(void *arg, nni_aio *aio) +{ + NNI_ARG_UNUSED(arg); + if (nni_aio_begin(aio) == 0) { + nni_aio_finish_error(aio, NNG_ENOTSUP); + } +} + +static void +pub0_sock_send(void *arg, nni_aio *aio) +{ + pub0_sock *sock = arg; + pub0_pipe *p; + nng_msg * msg; + size_t len; + + msg = nni_aio_get_msg(aio); + len = nni_msg_len(msg); + nni_mtx_lock(&sock->mtx); + NNI_LIST_FOREACH (&sock->pipes, p) { + + nni_msg_clone(msg); + if (p->busy) { + if (nni_lmq_full(&p->sendq)) { + // Make space for the new message. + nni_msg *old; + (void) nni_lmq_getq(&p->sendq, &old); + nni_msg_free(old); + } + nni_lmq_putq(&p->sendq, msg); + } else { + p->busy = true; + nni_aio_set_msg(p->aio_send, msg); + nni_pipe_send(p->pipe, p->aio_send); + } + } + nni_mtx_unlock(&sock->mtx); + nng_msg_free(msg); + nni_aio_finish(aio, 0, len); +} + +static int +pub0_sock_get_sendfd(void *arg, void *buf, size_t *szp, nni_type t) +{ + pub0_sock *sock = arg; + int fd; + int rv; + nni_mtx_lock(&sock->mtx); + // PUB sockets are *always* writable. + nni_pollable_raise(sock->sendable); + rv = nni_pollable_getfd(sock->sendable, &fd); + nni_mtx_unlock(&sock->mtx); + + if (rv == 0) { + rv = nni_copyout_int(fd, buf, szp, t); + } + return (rv); +} + +static int +pub0_sock_set_sendbuf(void *arg, const void *buf, size_t sz, nni_type t) +{ + pub0_sock *sock = arg; + pub0_pipe *p; + int val; + int rv; + + if ((rv = nni_copyin_int(&val, buf, sz, 1, 8192, t)) != 0) { + return (rv); + } + + nni_mtx_lock(&sock->mtx); + sock->sendbuf = (size_t) val; + NNI_LIST_FOREACH (&sock->pipes, p) { + // If we fail part way thru (should only be ENOMEM), we + // stop short. The others would likely fail for ENOMEM as + // well anyway. There is a weird effect here where the + // buffers may have been set for *some* of the pipes, but + // we have no way to correct partial failure. + if ((rv = nni_lmq_resize(&p->sendq, (size_t) val)) != 0) { + break; + } + } + nni_mtx_unlock(&sock->mtx); + return (rv); +} + +static int +pub0_sock_get_sendbuf(void *arg, void *buf, size_t *szp, nni_type t) +{ + pub0_sock *sock = arg; + int val; + nni_mtx_lock(&sock->mtx); + val = (int) sock->sendbuf; + nni_mtx_unlock(&sock->mtx); + return (nni_copyout_int(val, buf, szp, t)); +} + +static nni_proto_pipe_ops pub0_pipe_ops = { + .pipe_size = sizeof(pub0_pipe), + .pipe_init = pub0_pipe_init, + .pipe_fini = pub0_pipe_fini, + .pipe_start = pub0_pipe_start, + .pipe_close = pub0_pipe_close, + .pipe_stop = pub0_pipe_stop, +}; + +static nni_option pub0_sock_options[] = { + // terminate list + { + .o_name = NNG_OPT_SENDFD, + .o_get = pub0_sock_get_sendfd, + }, + { + .o_name = NNG_OPT_SENDBUF, + .o_get = pub0_sock_get_sendbuf, + .o_set = pub0_sock_set_sendbuf, + }, + { + .o_name = NULL, + }, +}; + +static nni_proto_sock_ops pub0_sock_ops = { + .sock_size = sizeof(pub0_sock), + .sock_init = pub0_sock_init, + .sock_fini = pub0_sock_fini, + .sock_open = pub0_sock_open, + .sock_close = pub0_sock_close, + .sock_send = pub0_sock_send, + .sock_recv = pub0_sock_recv, + .sock_options = pub0_sock_options, +}; + +static nni_proto pub0_proto = { + .proto_version = NNI_PROTOCOL_VERSION, + .proto_self = { NNI_PROTO_PUB_V0, "pub" }, + .proto_peer = { NNI_PROTO_SUB_V0, "sub" }, + .proto_flags = NNI_PROTO_FLAG_SND, + .proto_sock_ops = &pub0_sock_ops, + .proto_pipe_ops = &pub0_pipe_ops, +}; + +static nni_proto pub0_proto_raw = { + .proto_version = NNI_PROTOCOL_VERSION, + .proto_self = { NNI_PROTO_PUB_V0, "pub" }, + .proto_peer = { NNI_PROTO_SUB_V0, "sub" }, + .proto_flags = NNI_PROTO_FLAG_SND | NNI_PROTO_FLAG_RAW, + .proto_sock_ops = &pub0_sock_ops, + .proto_pipe_ops = &pub0_pipe_ops, +}; + +int +nng_pub0_open(nng_socket *sidp) +{ + return (nni_proto_open(sidp, &pub0_proto)); +} + +int +nng_pub0_open_raw(nng_socket *sidp) +{ + return (nni_proto_open(sidp, &pub0_proto_raw)); +} diff --git a/src/sp/protocol/pubsub0/pub_test.c b/src/sp/protocol/pubsub0/pub_test.c new file mode 100644 index 00000000..a430b610 --- /dev/null +++ b/src/sp/protocol/pubsub0/pub_test.c @@ -0,0 +1,331 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include + +static void +test_pub_identity(void) +{ + nng_socket s; + int p; + char * n; + + NUTS_PASS(nng_pub0_open(&s)); + NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PROTO, &p)); + NUTS_TRUE(p == NUTS_PROTO(2u, 0u)); // 32 + NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PEER, &p)); + NUTS_TRUE(p == NUTS_PROTO(2u, 1u)); // 33 + NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PROTONAME, &n)); + NUTS_MATCH(n, "pub"); + nng_strfree(n); + NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PEERNAME, &n)); + NUTS_MATCH(n, "sub"); + nng_strfree(n); + NUTS_CLOSE(s); +} + +static void +test_pub_cannot_recv(void) +{ + nng_socket pub; + + NUTS_PASS(nng_pub0_open(&pub)); + NUTS_FAIL(nng_recv(pub, "", 0, 0), NNG_ENOTSUP); + NUTS_CLOSE(pub); +} + +static void +test_pub_no_context(void) +{ + nng_socket pub; + nng_ctx ctx; + + NUTS_PASS(nng_pub0_open(&pub)); + NUTS_FAIL(nng_ctx_open(&ctx, pub), NNG_ENOTSUP); + NUTS_CLOSE(pub); +} + +static void +test_pub_not_readable(void) +{ + int fd; + nng_socket pub; + + NUTS_PASS(nng_pub0_open(&pub)); + NUTS_FAIL(nng_socket_get_int(pub, NNG_OPT_RECVFD, &fd), NNG_ENOTSUP); + NUTS_CLOSE(pub); +} + +static void +test_pub_poll_writeable(void) +{ + int fd; + nng_socket pub; + nng_socket sub; + + NUTS_PASS(nng_sub0_open(&sub)); + NUTS_PASS(nng_pub0_open(&pub)); + NUTS_PASS(nng_socket_get_int(pub, NNG_OPT_SENDFD, &fd)); + NUTS_TRUE(fd >= 0); + + // Pub is *always* writeable + NUTS_TRUE(nuts_poll_fd(fd)); + + // Even after connect (no message yet) + NUTS_MARRY(pub, sub); + NUTS_TRUE(nuts_poll_fd(fd)); + + // Even if we send messages. + NUTS_SEND(pub, "abc"); + NUTS_TRUE(nuts_poll_fd(fd)); + + NUTS_CLOSE(pub); + NUTS_CLOSE(sub); +} + +static void +test_pub_send_no_pipes(void) +{ + nng_socket pub; + + NUTS_PASS(nng_pub0_open(&pub)); + NUTS_SEND(pub, "DROP1"); + NUTS_SEND(pub, "DROP2"); + NUTS_CLOSE(pub); +} + +void +test_pub_validate_peer(void) +{ + nng_socket s1, s2; + nng_stat * stats; + nng_stat * reject; + char *addr; + + NUTS_ADDR(addr, "inproc"); + + NUTS_PASS(nng_pub0_open(&s1)); + NUTS_PASS(nng_pub0_open(&s2)); + + NUTS_PASS(nng_listen(s1, addr, NULL, 0)); + NUTS_PASS(nng_dial(s2, addr, NULL, NNG_FLAG_NONBLOCK)); + + NUTS_SLEEP(100); + NUTS_PASS(nng_stats_get(&stats)); + + NUTS_TRUE(stats != NULL); + NUTS_TRUE((reject = nng_stat_find_socket(stats, s1)) != NULL); + NUTS_TRUE((reject = nng_stat_find(reject, "reject")) != NULL); + + NUTS_TRUE(nng_stat_type(reject) == NNG_STAT_COUNTER); + NUTS_TRUE(nng_stat_value(reject) > 0); + + NUTS_CLOSE(s1); + NUTS_CLOSE(s2); + nng_stats_free(stats); +} + +static void +test_pub_send_queued(void) +{ + nng_socket pub; + nng_socket sub; + + // MB: What we really need is a mock so that we can send harder + // than we receive -- we need a way to apply back-pressure for this + // test to be really meaningful. + NUTS_PASS(nng_pub0_open(&pub)); + NUTS_PASS(nng_sub0_open(&sub)); + NUTS_PASS(nng_socket_set(sub, NNG_OPT_SUB_SUBSCRIBE, "", 0)); + NUTS_PASS(nng_socket_set_int(pub, NNG_OPT_SENDBUF, 10)); + NUTS_PASS(nng_socket_set_int(sub, NNG_OPT_RECVBUF, 10)); + NUTS_PASS(nng_socket_set_ms(pub, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(sub, NNG_OPT_RECVTIMEO, 1000)); + NUTS_MARRY(pub, sub); + NUTS_SEND(pub, "first"); + NUTS_SEND(pub, "second"); + NUTS_SEND(pub, "three musketeers"); + NUTS_SEND(pub, "four"); + NUTS_SLEEP(50); + NUTS_RECV(sub, "first"); + NUTS_RECV(sub, "second"); + NUTS_RECV(sub, "three musketeers"); + NUTS_RECV(sub, "four"); + + NUTS_CLOSE(pub); + NUTS_CLOSE(sub); +} +static void +test_sub_recv_ctx_closed(void) +{ + nng_socket sub; + nng_ctx ctx; + nng_aio * aio; + NUTS_PASS(nng_sub0_open(&sub)); + NUTS_PASS(nng_ctx_open(&ctx, sub)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + nng_ctx_close(ctx); + nng_ctx_recv(ctx, aio); + nng_aio_wait(aio); + NUTS_FAIL(nng_aio_result(aio), NNG_ECLOSED); + nng_aio_free(aio); + NUTS_CLOSE(sub); +} + +static void +test_sub_ctx_recv_aio_stopped(void) +{ + nng_socket sub; + nng_ctx ctx; + nng_aio * aio; + + NUTS_PASS(nng_sub0_open(&sub)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + NUTS_PASS(nng_ctx_open(&ctx, sub)); + + nng_aio_stop(aio); + nng_ctx_recv(ctx, aio); + nng_aio_wait(aio); + NUTS_FAIL(nng_aio_result(aio), NNG_ECANCELED); + NUTS_PASS(nng_ctx_close(ctx)); + NUTS_CLOSE(sub); + nng_aio_free(aio); +} + +static void +test_sub_close_context_recv(void) +{ + nng_socket sub; + nng_ctx ctx; + nng_aio * aio; + + NUTS_PASS(nng_sub0_open(&sub)); + NUTS_PASS(nng_ctx_open(&ctx, sub)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + nng_aio_set_timeout(aio, 1000); + nng_ctx_recv(ctx, aio); + NUTS_PASS(nng_ctx_close(ctx)); + nng_aio_wait(aio); + NUTS_FAIL(nng_aio_result(aio), NNG_ECLOSED); + + NUTS_CLOSE(sub); + nng_aio_free(aio); +} + +static void +test_sub_ctx_recv_nonblock(void) +{ + nng_socket sub; + nng_ctx ctx; + nng_aio * aio; + + NUTS_PASS(nng_sub0_open(&sub)); + NUTS_PASS(nng_ctx_open(&ctx, sub)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + + nng_aio_set_timeout(aio, 0); // Instant timeout + nng_ctx_recv(ctx, aio); + + nng_aio_wait(aio); + NUTS_FAIL(nng_aio_result(aio), NNG_ETIMEDOUT); + NUTS_CLOSE(sub); + nng_aio_free(aio); +} + +static void +test_sub_ctx_recv_cancel(void) +{ + nng_socket sub; + nng_ctx ctx; + nng_aio * aio; + + NUTS_PASS(nng_sub0_open(&sub)); + NUTS_PASS(nng_ctx_open(&ctx, sub)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + + nng_aio_set_timeout(aio, 1000); + nng_ctx_recv(ctx, aio); + nng_aio_abort(aio, NNG_ECANCELED); + + nng_aio_wait(aio); + NUTS_FAIL(nng_aio_result(aio), NNG_ECANCELED); + NUTS_CLOSE(sub); + nng_aio_free(aio); +} + +static void +test_pub_send_buf_option(void) +{ + nng_socket pub; + int v; + bool b; + size_t sz; + const char *opt = NNG_OPT_SENDBUF; + + NUTS_PASS(nng_pub0_open(&pub)); + + NUTS_PASS(nng_socket_set_int(pub, opt, 1)); + NUTS_FAIL(nng_socket_set_int(pub, opt, 0), NNG_EINVAL); + NUTS_FAIL(nng_socket_set_int(pub, opt, -1), NNG_EINVAL); + NUTS_FAIL(nng_socket_set_int(pub, opt, 1000000), NNG_EINVAL); + NUTS_PASS(nng_socket_set_int(pub, opt, 3)); + NUTS_PASS(nng_socket_get_int(pub, opt, &v)); + NUTS_TRUE(v == 3); + v = 0; + sz = sizeof(v); + NUTS_PASS(nng_socket_get(pub, opt, &v, &sz)); + NUTS_TRUE(v == 3); + NUTS_TRUE(sz == sizeof(v)); + + NUTS_FAIL(nng_socket_set(pub, opt, "", 1), NNG_EINVAL); + sz = 1; + NUTS_FAIL(nng_socket_get(pub, opt, &v, &sz), NNG_EINVAL); + NUTS_FAIL(nng_socket_set_bool(pub, opt, true), NNG_EBADTYPE); + NUTS_FAIL(nng_socket_get_bool(pub, opt, &b), NNG_EBADTYPE); + + NUTS_CLOSE(pub); +} + +static void +test_pub_cooked(void) +{ + nng_socket s; + bool b; + + NUTS_PASS(nng_pub0_open(&s)); + NUTS_PASS(nng_socket_get_bool(s, NNG_OPT_RAW, &b)); + NUTS_TRUE(!b); + NUTS_FAIL(nng_socket_set_bool(s, NNG_OPT_RAW, true), NNG_EREADONLY); + NUTS_PASS(nng_close(s)); + + // raw pub only differs in the option setting + NUTS_PASS(nng_pub0_open_raw(&s)); + NUTS_PASS(nng_socket_get_bool(s, NNG_OPT_RAW, &b)); + NUTS_TRUE(b); + NUTS_CLOSE(s); +} + +NUTS_TESTS = { + { "pub identity", test_pub_identity }, + { "pub cannot recv", test_pub_cannot_recv }, + { "put no context", test_pub_no_context }, + { "pub not readable", test_pub_not_readable }, + { "pub poll writeable", test_pub_poll_writeable }, + { "pub validate peer", test_pub_validate_peer }, + { "pub send queued", test_pub_send_queued }, + { "pub send no pipes", test_pub_send_no_pipes }, + { "sub recv ctx closed", test_sub_recv_ctx_closed }, + { "sub recv aio ctx stopped", test_sub_ctx_recv_aio_stopped }, + { "sub close context recv", test_sub_close_context_recv }, + { "sub context recv nonblock", test_sub_ctx_recv_nonblock }, + { "sub context recv cancel", test_sub_ctx_recv_cancel }, + { "pub send buf option", test_pub_send_buf_option }, + { "pub cooked", test_pub_cooked }, + { NULL, NULL }, +}; diff --git a/src/sp/protocol/pubsub0/sub.c b/src/sp/protocol/pubsub0/sub.c new file mode 100644 index 00000000..9f3f2283 --- /dev/null +++ b/src/sp/protocol/pubsub0/sub.c @@ -0,0 +1,755 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// Copyright 2018 Capitar IT Group BV +// Copyright 2019 Nathan Kent +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include +#include + +#include "core/nng_impl.h" +#include "nng/protocol/pubsub0/sub.h" + +// Subscriber protocol. The SUB protocol receives messages sent to +// it from publishers, and filters out those it is not interested in, +// only passing up ones that match known subscriptions. + +#ifndef NNI_PROTO_SUB_V0 +#define NNI_PROTO_SUB_V0 NNI_PROTO(2, 1) +#endif + +#ifndef NNI_PROTO_PUB_V0 +#define NNI_PROTO_PUB_V0 NNI_PROTO(2, 0) +#endif + +// By default we accept 128 messages. +#define SUB0_DEFAULT_RECV_BUF_LEN 128 + +// By default, prefer new messages when the queue is full. +#define SUB0_DEFAULT_PREFER_NEW true + +typedef struct sub0_pipe sub0_pipe; +typedef struct sub0_sock sub0_sock; +typedef struct sub0_ctx sub0_ctx; +typedef struct sub0_topic sub0_topic; + +static void sub0_recv_cb(void *); +static void sub0_pipe_fini(void *); + +struct sub0_topic { + nni_list_node node; + size_t len; + void * buf; +}; + +// sub0_ctx is a context for a SUB socket. The advantage of contexts is +// that different contexts can maintain different subscriptions. +struct sub0_ctx { + nni_list_node node; + sub0_sock * sock; + nni_list topics; // TODO: Consider patricia trie + nni_list recv_queue; // can have multiple pending receives + nni_lmq lmq; + bool prefer_new; +}; + +// sub0_sock is our per-socket protocol private structure. +struct sub0_sock { + nni_pollable readable; + sub0_ctx master; // default context + nni_list contexts; // all contexts + int num_contexts; + size_t recv_buf_len; + bool prefer_new; + nni_mtx lk; +}; + +// sub0_pipe is our per-pipe protocol private structure. +struct sub0_pipe { + nni_pipe * pipe; + sub0_sock *sub; + nni_aio aio_recv; +}; + +static void +sub0_ctx_cancel(nng_aio *aio, void *arg, int rv) +{ + sub0_ctx * ctx = arg; + sub0_sock *sock = ctx->sock; + nni_mtx_lock(&sock->lk); + if (nni_list_active(&ctx->recv_queue, aio)) { + nni_list_remove(&ctx->recv_queue, aio); + nni_aio_finish_error(aio, rv); + } + nni_mtx_unlock(&sock->lk); +} + +static void +sub0_ctx_recv(void *arg, nni_aio *aio) +{ + sub0_ctx * ctx = arg; + sub0_sock *sock = ctx->sock; + nni_msg * msg; + + if (nni_aio_begin(aio) != 0) { + return; + } + + nni_mtx_lock(&sock->lk); + +again: + if (nni_lmq_empty(&ctx->lmq)) { + int rv; + if ((rv = nni_aio_schedule(aio, sub0_ctx_cancel, ctx)) != 0) { + nni_mtx_unlock(&sock->lk); + nni_aio_finish_error(aio, rv); + return; + } + nni_list_append(&ctx->recv_queue, aio); + nni_mtx_unlock(&sock->lk); + return; + } + + (void) nni_lmq_getq(&ctx->lmq, &msg); + + if (nni_lmq_empty(&ctx->lmq) && (ctx == &sock->master)) { + nni_pollable_clear(&sock->readable); + } + if ((msg = nni_msg_unique(msg)) == NULL) { + goto again; + } + nni_aio_set_msg(aio, msg); + nni_mtx_unlock(&sock->lk); + nni_aio_finish(aio, 0, nni_msg_len(msg)); +} + +static void +sub0_ctx_send(void *arg, nni_aio *aio) +{ + NNI_ARG_UNUSED(arg); + if (nni_aio_begin(aio) == 0) { + nni_aio_finish_error(aio, NNG_ENOTSUP); + } +} + +static void +sub0_ctx_close(void *arg) +{ + sub0_ctx * ctx = arg; + sub0_sock *sock = ctx->sock; + nni_aio * aio; + + nni_mtx_lock(&sock->lk); + while ((aio = nni_list_first(&ctx->recv_queue)) != NULL) { + nni_list_remove(&ctx->recv_queue, aio); + nni_aio_finish_error(aio, NNG_ECLOSED); + } + nni_mtx_unlock(&sock->lk); +} + +static void +sub0_ctx_fini(void *arg) +{ + sub0_ctx * ctx = arg; + sub0_sock * sock = ctx->sock; + sub0_topic *topic; + + sub0_ctx_close(ctx); + + nni_mtx_lock(&sock->lk); + nni_list_remove(&sock->contexts, ctx); + sock->num_contexts--; + nni_mtx_unlock(&sock->lk); + + while ((topic = nni_list_first(&ctx->topics)) != 0) { + nni_list_remove(&ctx->topics, topic); + nni_free(topic->buf, topic->len); + NNI_FREE_STRUCT(topic); + } + + nni_lmq_fini(&ctx->lmq); +} + +static int +sub0_ctx_init(void *ctx_arg, void *sock_arg) +{ + sub0_sock *sock = sock_arg; + sub0_ctx * ctx = ctx_arg; + size_t len; + bool prefer_new; + int rv; + + nni_mtx_lock(&sock->lk); + len = sock->recv_buf_len; + prefer_new = sock->prefer_new; + + if ((rv = nni_lmq_init(&ctx->lmq, len)) != 0) { + return (rv); + } + ctx->prefer_new = prefer_new; + + nni_aio_list_init(&ctx->recv_queue); + NNI_LIST_INIT(&ctx->topics, sub0_topic, node); + + ctx->sock = sock; + + nni_list_append(&sock->contexts, ctx); + sock->num_contexts++; + nni_mtx_unlock(&sock->lk); + + return (0); +} + +static void +sub0_sock_fini(void *arg) +{ + sub0_sock *sock = arg; + + sub0_ctx_fini(&sock->master); + nni_pollable_fini(&sock->readable); + nni_mtx_fini(&sock->lk); +} + +static int +sub0_sock_init(void *arg, nni_sock *unused) +{ + sub0_sock *sock = arg; + int rv; + + NNI_ARG_UNUSED(unused); + + NNI_LIST_INIT(&sock->contexts, sub0_ctx, node); + nni_mtx_init(&sock->lk); + sock->recv_buf_len = SUB0_DEFAULT_RECV_BUF_LEN; + sock->prefer_new = SUB0_DEFAULT_PREFER_NEW; + nni_pollable_init(&sock->readable); + + if ((rv = sub0_ctx_init(&sock->master, sock)) != 0) { + sub0_sock_fini(sock); + return (rv); + } + + return (0); +} + +static void +sub0_sock_open(void *arg) +{ + NNI_ARG_UNUSED(arg); +} + +static void +sub0_sock_close(void *arg) +{ + sub0_sock *sock = arg; + sub0_ctx_close(&sock->master); +} + +static void +sub0_pipe_stop(void *arg) +{ + sub0_pipe *p = arg; + + nni_aio_stop(&p->aio_recv); +} + +static void +sub0_pipe_fini(void *arg) +{ + sub0_pipe *p = arg; + + nni_aio_fini(&p->aio_recv); +} + +static int +sub0_pipe_init(void *arg, nni_pipe *pipe, void *s) +{ + sub0_pipe *p = arg; + + nni_aio_init(&p->aio_recv, sub0_recv_cb, p); + + p->pipe = pipe; + p->sub = s; + return (0); +} + +static int +sub0_pipe_start(void *arg) +{ + sub0_pipe *p = arg; + + if (nni_pipe_peer(p->pipe) != NNI_PROTO_PUB_V0) { + // Peer protocol mismatch. + return (NNG_EPROTO); + } + + nni_pipe_recv(p->pipe, &p->aio_recv); + return (0); +} + +static void +sub0_pipe_close(void *arg) +{ + sub0_pipe *p = arg; + + nni_aio_close(&p->aio_recv); +} + +static bool +sub0_matches(sub0_ctx *ctx, uint8_t *body, size_t len) +{ + sub0_topic *topic; + + // This is a naive and trivial matcher. Replace with a real + // patricia trie later. + NNI_LIST_FOREACH (&ctx->topics, topic) { + if (len < topic->len) { + continue; + } + if ((topic->len == 0) || + (memcmp(topic->buf, body, topic->len) == 0)) { + return (true); + } + } + return (false); +} + +static void +sub0_recv_cb(void *arg) +{ + sub0_pipe *p = arg; + sub0_sock *sock = p->sub; + sub0_ctx * ctx; + nni_msg * msg; + size_t len; + uint8_t * body; + nni_list finish; + nng_aio * aio; + nni_msg * dup_msg; + + if (nni_aio_result(&p->aio_recv) != 0) { + nni_pipe_close(p->pipe); + return; + } + + nni_aio_list_init(&finish); + + msg = nni_aio_get_msg(&p->aio_recv); + nni_aio_set_msg(&p->aio_recv, NULL); + nni_msg_set_pipe(msg, nni_pipe_id(p->pipe)); + + body = nni_msg_body(msg); + len = nni_msg_len(msg); + dup_msg = NULL; + + nni_mtx_lock(&sock->lk); + // Go through all contexts. We will try to send up. + NNI_LIST_FOREACH (&sock->contexts, ctx) { + bool queued = false; + + if (nni_lmq_full(&ctx->lmq) && !ctx->prefer_new) { + // Cannot deliver here, as receive buffer is full. + continue; + } + + if (!sub0_matches(ctx, body, len)) { + continue; + } + + // This is a performance optimization, that ensures we + // do not duplicate a message in the common case, where there + // is only a single context. + if (sock->num_contexts > 1) { + if (nni_msg_dup(&dup_msg, msg) != 0) { + // if we cannot dup it, continue on + continue; + } + } else { + // We only have one context, so it's the only + // possible message. + dup_msg = msg; + } + + if (!nni_list_empty(&ctx->recv_queue)) { + aio = nni_list_first(&ctx->recv_queue); + nni_list_remove(&ctx->recv_queue, aio); + nni_aio_set_msg(aio, dup_msg); + + // Save for synchronous completion + nni_list_append(&finish, aio); + } else if (nni_lmq_full(&ctx->lmq)) { + // Make space for the new message. + nni_msg *old; + (void) nni_lmq_getq(&ctx->lmq, &old); + nni_msg_free(old); + + (void) nni_lmq_putq(&ctx->lmq, dup_msg); + queued = true; + + } else { + (void) nni_lmq_putq(&ctx->lmq, dup_msg); + queued = true; + } + if (queued && ctx == &sock->master) { + nni_pollable_raise(&sock->readable); + } + } + nni_mtx_unlock(&sock->lk); + + // NB: This is slightly less efficient in that we may have + // created an extra copy in the face of e.g. two subscriptions, + // but optimizing this further would require checking the subscription + // list twice, adding complexity. If this turns out to be a problem + // we could probably add some other sophistication with a counter + // and flags on the contexts themselves. + if (msg != dup_msg) { + // If we didn't just use the message, then free our copy. + nni_msg_free(msg); + } + + while ((aio = nni_list_first(&finish)) != NULL) { + nni_list_remove(&finish, aio); + nni_aio_finish_sync(aio, 0, len); + } + + nni_pipe_recv(p->pipe, &p->aio_recv); +} + +static int +sub0_ctx_get_recv_buf_len(void *arg, void *buf, size_t *szp, nni_type t) +{ + sub0_ctx * ctx = arg; + sub0_sock *sock = ctx->sock; + int val; + nni_mtx_lock(&sock->lk); + val = (int) nni_lmq_cap(&ctx->lmq); + nni_mtx_unlock(&sock->lk); + + return (nni_copyout_int(val, buf, szp, t)); +} + +static int +sub0_ctx_set_recv_buf_len(void *arg, const void *buf, size_t sz, nni_type t) +{ + sub0_ctx * ctx = arg; + sub0_sock *sock = ctx->sock; + int val; + int rv; + + if ((rv = nni_copyin_int(&val, buf, sz, 1, 8192, t)) != 0) { + return (rv); + } + nni_mtx_lock(&sock->lk); + if ((rv = nni_lmq_resize(&ctx->lmq, (size_t) val)) != 0) { + nni_mtx_unlock(&sock->lk); + return (rv); + } + + // If we change the socket, then this will change the queue for + // any new contexts. (Previously constructed contexts are unaffected.) + if (&sock->master == ctx) { + sock->recv_buf_len = (size_t) val; + } + nni_mtx_unlock(&sock->lk); + return (0); +} + +// For now we maintain subscriptions on a sorted linked list. As we do not +// expect to have huge numbers of subscriptions, and as the operation is +// really O(n), we think this is acceptable. In the future we might decide +// to replace this with a patricia trie, like old nanomsg had. + +static int +sub0_ctx_subscribe(void *arg, const void *buf, size_t sz, nni_type t) +{ + sub0_ctx * ctx = arg; + sub0_sock * sock = ctx->sock; + sub0_topic *topic; + sub0_topic *new_topic; + NNI_ARG_UNUSED(t); + + nni_mtx_lock(&sock->lk); + NNI_LIST_FOREACH (&ctx->topics, topic) { + if (topic->len != sz) { + continue; + } + if (memcmp(topic->buf, buf, sz) == 0) { + // Already have it. + nni_mtx_unlock(&sock->lk); + return (0); + } + } + if ((new_topic = NNI_ALLOC_STRUCT(new_topic)) == NULL) { + nni_mtx_unlock(&sock->lk); + return (NNG_ENOMEM); + } + if ((sz > 0) && ((new_topic->buf = nni_alloc(sz)) == NULL)) { + nni_mtx_unlock(&sock->lk); + NNI_FREE_STRUCT(new_topic); + return (NNG_ENOMEM); + } + memcpy(new_topic->buf, buf, sz); + new_topic->len = sz; + nni_list_append(&ctx->topics, new_topic); + nni_mtx_unlock(&sock->lk); + return (0); +} + +static int +sub0_ctx_unsubscribe(void *arg, const void *buf, size_t sz, nni_type t) +{ + sub0_ctx * ctx = arg; + sub0_sock * sock = ctx->sock; + sub0_topic *topic; + size_t len; + NNI_ARG_UNUSED(t); + + nni_mtx_lock(&sock->lk); + NNI_LIST_FOREACH (&ctx->topics, topic) { + if (topic->len != sz) { + continue; + } + if (memcmp(topic->buf, buf, sz) == 0) { + // Matched! + break; + } + } + if (topic == NULL) { + nni_mtx_unlock(&sock->lk); + return (NNG_ENOENT); + } + nni_list_remove(&ctx->topics, topic); + + // Now we need to make sure that any messages that are waiting still + // match the subscription. We basically just run through the queue + // and requeue those messages we need. + len = nni_lmq_len(&ctx->lmq); + for (size_t i = 0; i < len; i++) { + nni_msg *msg; + + (void) nni_lmq_getq(&ctx->lmq, &msg); + if (sub0_matches(ctx, nni_msg_body(msg), nni_msg_len(msg))) { + (void) nni_lmq_putq(&ctx->lmq, msg); + } else { + nni_msg_free(msg); + } + } + nni_mtx_unlock(&sock->lk); + + nni_free(topic->buf, topic->len); + NNI_FREE_STRUCT(topic); + return (0); +} + +static int +sub0_ctx_get_prefer_new(void *arg, void *buf, size_t *szp, nni_type t) +{ + sub0_ctx * ctx = arg; + sub0_sock *sock = ctx->sock; + bool val; + + nni_mtx_lock(&sock->lk); + val = ctx->prefer_new; + nni_mtx_unlock(&sock->lk); + + return (nni_copyout_bool(val, buf, szp, t)); +} + +static int +sub0_ctx_set_prefer_new(void *arg, const void *buf, size_t sz, nni_type t) +{ + sub0_ctx * ctx = arg; + sub0_sock *sock = ctx->sock; + bool val; + int rv; + + if ((rv = nni_copyin_bool(&val, buf, sz, t)) != 0) { + return (rv); + } + + nni_mtx_lock(&sock->lk); + ctx->prefer_new = val; + if (&sock->master == ctx) { + sock->prefer_new = val; + } + nni_mtx_unlock(&sock->lk); + + return (0); +} + +static nni_option sub0_ctx_options[] = { + { + .o_name = NNG_OPT_RECVBUF, + .o_get = sub0_ctx_get_recv_buf_len, + .o_set = sub0_ctx_set_recv_buf_len, + }, + { + .o_name = NNG_OPT_SUB_SUBSCRIBE, + .o_set = sub0_ctx_subscribe, + }, + { + .o_name = NNG_OPT_SUB_UNSUBSCRIBE, + .o_set = sub0_ctx_unsubscribe, + }, + { + .o_name = NNG_OPT_SUB_PREFNEW, + .o_get = sub0_ctx_get_prefer_new, + .o_set = sub0_ctx_set_prefer_new, + }, + { + .o_name = NULL, + }, +}; + +static void +sub0_sock_send(void *arg, nni_aio *aio) +{ + NNI_ARG_UNUSED(arg); + if (nni_aio_begin(aio) == 0) { + nni_aio_finish_error(aio, NNG_ENOTSUP); + } +} + +static void +sub0_sock_recv(void *arg, nni_aio *aio) +{ + sub0_sock *sock = arg; + + sub0_ctx_recv(&sock->master, aio); +} + +static int +sub0_sock_get_recv_fd(void *arg, void *buf, size_t *szp, nni_opt_type t) +{ + sub0_sock *sock = arg; + int rv; + int fd; + + if ((rv = nni_pollable_getfd(&sock->readable, &fd)) != 0) { + return (rv); + } + return (nni_copyout_int(fd, buf, szp, t)); +} + +static int +sub0_sock_get_recv_buf_len(void *arg, void *buf, size_t *szp, nni_type t) +{ + sub0_sock *sock = arg; + return (sub0_ctx_get_recv_buf_len(&sock->master, buf, szp, t)); +} + +static int +sub0_sock_set_recv_buf_len(void *arg, const void *buf, size_t sz, nni_type t) +{ + sub0_sock *sock = arg; + return (sub0_ctx_set_recv_buf_len(&sock->master, buf, sz, t)); +} + +static int +sub0_sock_subscribe(void *arg, const void *buf, size_t sz, nni_type t) +{ + sub0_sock *sock = arg; + return (sub0_ctx_subscribe(&sock->master, buf, sz, t)); +} + +static int +sub0_sock_unsubscribe(void *arg, const void *buf, size_t sz, nni_type t) +{ + sub0_sock *sock = arg; + return (sub0_ctx_unsubscribe(&sock->master, buf, sz, t)); +} + +static int +sub0_sock_get_prefer_new(void *arg, void *buf, size_t *szp, nni_type t) +{ + sub0_sock *sock = arg; + return (sub0_ctx_get_prefer_new(&sock->master, buf, szp, t)); +} + +static int +sub0_sock_set_prefer_new(void *arg, const void *buf, size_t sz, nni_type t) +{ + sub0_sock *sock = arg; + return (sub0_ctx_set_prefer_new(&sock->master, buf, sz, t)); +} + +// This is the global protocol structure -- our linkage to the core. +// This should be the only global non-static symbol in this file. +static nni_proto_pipe_ops sub0_pipe_ops = { + .pipe_size = sizeof(sub0_pipe), + .pipe_init = sub0_pipe_init, + .pipe_fini = sub0_pipe_fini, + .pipe_start = sub0_pipe_start, + .pipe_close = sub0_pipe_close, + .pipe_stop = sub0_pipe_stop, +}; + +static nni_proto_ctx_ops sub0_ctx_ops = { + .ctx_size = sizeof(sub0_ctx), + .ctx_init = sub0_ctx_init, + .ctx_fini = sub0_ctx_fini, + .ctx_send = sub0_ctx_send, + .ctx_recv = sub0_ctx_recv, + .ctx_options = sub0_ctx_options, +}; + +static nni_option sub0_sock_options[] = { + { + .o_name = NNG_OPT_SUB_SUBSCRIBE, + .o_set = sub0_sock_subscribe, + }, + { + .o_name = NNG_OPT_SUB_UNSUBSCRIBE, + .o_set = sub0_sock_unsubscribe, + }, + { + .o_name = NNG_OPT_RECVFD, + .o_get = sub0_sock_get_recv_fd, + }, + { + .o_name = NNG_OPT_RECVBUF, + .o_get = sub0_sock_get_recv_buf_len, + .o_set = sub0_sock_set_recv_buf_len, + }, + { + .o_name = NNG_OPT_SUB_PREFNEW, + .o_get = sub0_sock_get_prefer_new, + .o_set = sub0_sock_set_prefer_new, + }, + // terminate list + { + .o_name = NULL, + }, +}; + +static nni_proto_sock_ops sub0_sock_ops = { + .sock_size = sizeof(sub0_sock), + .sock_init = sub0_sock_init, + .sock_fini = sub0_sock_fini, + .sock_open = sub0_sock_open, + .sock_close = sub0_sock_close, + .sock_send = sub0_sock_send, + .sock_recv = sub0_sock_recv, + .sock_options = sub0_sock_options, +}; + +static nni_proto sub0_proto = { + .proto_version = NNI_PROTOCOL_VERSION, + .proto_self = { NNI_PROTO_SUB_V0, "sub" }, + .proto_peer = { NNI_PROTO_PUB_V0, "pub" }, + .proto_flags = NNI_PROTO_FLAG_RCV, + .proto_sock_ops = &sub0_sock_ops, + .proto_pipe_ops = &sub0_pipe_ops, + .proto_ctx_ops = &sub0_ctx_ops, +}; + +int +nng_sub0_open(nng_socket *sock) +{ + return (nni_proto_open(sock, &sub0_proto)); +} diff --git a/src/sp/protocol/pubsub0/sub_test.c b/src/sp/protocol/pubsub0/sub_test.c new file mode 100644 index 00000000..b830ae80 --- /dev/null +++ b/src/sp/protocol/pubsub0/sub_test.c @@ -0,0 +1,624 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include + +static void +test_sub_identity(void) +{ + nng_socket s; + int p; + char * n; + + NUTS_PASS(nng_sub0_open(&s)); + NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PROTO, &p)); + NUTS_TRUE(p == NUTS_PROTO(2u, 1u)); // 33 + NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PEER, &p)); + NUTS_TRUE(p == NUTS_PROTO(2u, 0u)); // 32 + NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PROTONAME, &n)); + NUTS_MATCH(n, "sub"); + nng_strfree(n); + NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PEERNAME, &n)); + NUTS_MATCH(n, "pub"); + nng_strfree(n); + NUTS_CLOSE(s); +} + +static void +test_sub_cannot_send(void) +{ + nng_socket sub; + + NUTS_PASS(nng_sub0_open(&sub)); + NUTS_FAIL(nng_send(sub, "", 0, 0), NNG_ENOTSUP); + NUTS_CLOSE(sub); +} + +static void +test_sub_context_cannot_send(void) +{ + nng_socket sub; + nng_ctx ctx; + nng_msg * m; + nng_aio * aio; + + NUTS_PASS(nng_sub0_open(&sub)); + NUTS_PASS(nng_ctx_open(&ctx, sub)); + NUTS_PASS(nng_msg_alloc(&m, 0)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + nng_aio_set_msg(aio, m); + nng_aio_set_timeout(aio, 1000); + nng_ctx_send(ctx, aio); + nng_aio_wait(aio); + NUTS_FAIL(nng_aio_result(aio), NNG_ENOTSUP); + NUTS_PASS(nng_ctx_close(ctx)); + NUTS_CLOSE(sub); + nng_aio_free(aio); + nng_msg_free(m); +} + +static void +test_sub_not_writeable(void) +{ + int fd; + nng_socket sub; + + NUTS_PASS(nng_sub0_open(&sub)); + NUTS_FAIL(nng_socket_get_int(sub, NNG_OPT_SENDFD, &fd), NNG_ENOTSUP); + NUTS_CLOSE(sub); +} + +static void +test_sub_poll_readable(void) +{ + int fd; + nng_socket pub; + nng_socket sub; + + NUTS_PASS(nng_sub0_open(&sub)); + NUTS_PASS(nng_pub0_open(&pub)); + NUTS_PASS(nng_socket_set(sub, NNG_OPT_SUB_SUBSCRIBE, "a", 1)); + NUTS_PASS(nng_socket_set_ms(sub, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(pub, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_get_int(sub, NNG_OPT_RECVFD, &fd)); + NUTS_TRUE(fd >= 0); + + // Not readable if not connected! + NUTS_TRUE(nuts_poll_fd(fd) == false); + + // Even after connect (no message yet) + NUTS_MARRY(pub, sub); + NUTS_TRUE(nuts_poll_fd(fd) == false); + + // If we send a message we didn't subscribe to, that doesn't matter. + NUTS_SEND(pub, "def"); + NUTS_SLEEP(100); + NUTS_TRUE(nuts_poll_fd(fd) == false); + + // But once we send messages, it is. + // We have to send a request, in order to send a reply. + NUTS_SEND(pub, "abc"); + NUTS_SLEEP(100); + NUTS_TRUE(nuts_poll_fd(fd)); + + // and receiving makes it no longer ready + NUTS_RECV(sub, "abc"); + NUTS_TRUE(nuts_poll_fd(fd) == false); + + NUTS_CLOSE(pub); + NUTS_CLOSE(sub); +} + +static void +test_sub_recv_late(void) +{ + int fd; + nng_socket pub; + nng_socket sub; + nng_aio * aio; + nng_msg * msg; + + NUTS_PASS(nng_sub0_open(&sub)); + NUTS_PASS(nng_pub0_open(&pub)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + NUTS_PASS(nng_socket_set(sub, NNG_OPT_SUB_SUBSCRIBE, "", 0)); + NUTS_PASS(nng_socket_set_ms(sub, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(pub, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_get_int(sub, NNG_OPT_RECVFD, &fd)); + NUTS_TRUE(fd >= 0); + + // Not readable if not connected! + NUTS_TRUE(nuts_poll_fd(fd) == false); + + // Even after connect (no message yet) + NUTS_MARRY(pub, sub); + NUTS_TRUE(nuts_poll_fd(fd) == false); + + nng_recv_aio(sub, aio); + + // But once we send messages, it is. + // We have to send a request, in order to send a reply. + NUTS_SEND(pub, "abc"); + NUTS_SLEEP(200); + + nng_aio_wait(aio); + NUTS_PASS(nng_aio_result(aio)); + msg = nng_aio_get_msg(aio); + nng_aio_set_msg(aio, NULL); + NUTS_TRUE(nng_msg_len(msg) == 4); + NUTS_MATCH(nng_msg_body(msg), "abc"); + + nng_msg_free(msg); + nng_aio_free(aio); + + NUTS_CLOSE(pub); + NUTS_CLOSE(sub); +} + +void +test_sub_context_no_poll(void) +{ + int fd; + nng_socket sub; + nng_ctx ctx; + + NUTS_PASS(nng_sub0_open(&sub)); + NUTS_PASS(nng_ctx_open(&ctx, sub)); + NUTS_FAIL(nng_ctx_get_int(ctx, NNG_OPT_SENDFD, &fd), NNG_ENOTSUP); + NUTS_FAIL(nng_ctx_get_int(ctx, NNG_OPT_RECVFD, &fd), NNG_ENOTSUP); + NUTS_PASS(nng_ctx_close(ctx)); + NUTS_CLOSE(sub); +} + +void +test_sub_validate_peer(void) +{ + nng_socket s1, s2; + nng_stat * stats; + nng_stat * reject; + char * addr; + + NUTS_ADDR(addr, "inproc"); + + NUTS_PASS(nng_sub0_open(&s1)); + NUTS_PASS(nng_sub0_open(&s2)); + + NUTS_PASS(nng_listen(s1, addr, NULL, 0)); + NUTS_PASS(nng_dial(s2, addr, NULL, NNG_FLAG_NONBLOCK)); + + NUTS_SLEEP(100); + NUTS_PASS(nng_stats_get(&stats)); + + NUTS_TRUE(stats != NULL); + NUTS_TRUE((reject = nng_stat_find_socket(stats, s1)) != NULL); + NUTS_TRUE((reject = nng_stat_find(reject, "reject")) != NULL); + + NUTS_TRUE(nng_stat_type(reject) == NNG_STAT_COUNTER); + NUTS_TRUE(nng_stat_value(reject) > 0); + + NUTS_CLOSE(s1); + NUTS_CLOSE(s2); + nng_stats_free(stats); +} + +static void +test_sub_recv_ctx_closed(void) +{ + nng_socket sub; + nng_ctx ctx; + nng_aio * aio; + NUTS_PASS(nng_sub0_open(&sub)); + NUTS_PASS(nng_ctx_open(&ctx, sub)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + nng_ctx_close(ctx); + nng_ctx_recv(ctx, aio); + nng_aio_wait(aio); + NUTS_FAIL(nng_aio_result(aio), NNG_ECLOSED); + nng_aio_free(aio); + NUTS_CLOSE(sub); +} + +static void +test_sub_ctx_recv_aio_stopped(void) +{ + nng_socket sub; + nng_ctx ctx; + nng_aio * aio; + + NUTS_PASS(nng_sub0_open(&sub)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + NUTS_PASS(nng_ctx_open(&ctx, sub)); + + nng_aio_stop(aio); + nng_ctx_recv(ctx, aio); + nng_aio_wait(aio); + NUTS_FAIL(nng_aio_result(aio), NNG_ECANCELED); + NUTS_PASS(nng_ctx_close(ctx)); + NUTS_CLOSE(sub); + nng_aio_free(aio); +} + +static void +test_sub_close_context_recv(void) +{ + nng_socket sub; + nng_ctx ctx; + nng_aio * aio; + + NUTS_PASS(nng_sub0_open(&sub)); + NUTS_PASS(nng_ctx_open(&ctx, sub)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + nng_aio_set_timeout(aio, 1000); + nng_ctx_recv(ctx, aio); + NUTS_PASS(nng_ctx_close(ctx)); + nng_aio_wait(aio); + NUTS_FAIL(nng_aio_result(aio), NNG_ECLOSED); + + NUTS_CLOSE(sub); + nng_aio_free(aio); +} + +static void +test_sub_ctx_recv_nonblock(void) +{ + nng_socket sub; + nng_ctx ctx; + nng_aio * aio; + + NUTS_PASS(nng_sub0_open(&sub)); + NUTS_PASS(nng_ctx_open(&ctx, sub)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + + nng_aio_set_timeout(aio, 0); // Instant timeout + nng_ctx_recv(ctx, aio); + + nng_aio_wait(aio); + NUTS_FAIL(nng_aio_result(aio), NNG_ETIMEDOUT); + NUTS_CLOSE(sub); + nng_aio_free(aio); +} + +static void +test_sub_ctx_recv_cancel(void) +{ + nng_socket sub; + nng_ctx ctx; + nng_aio * aio; + + NUTS_PASS(nng_sub0_open(&sub)); + NUTS_PASS(nng_ctx_open(&ctx, sub)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + + nng_aio_set_timeout(aio, 1000); + nng_ctx_recv(ctx, aio); + nng_aio_abort(aio, NNG_ECANCELED); + + nng_aio_wait(aio); + NUTS_FAIL(nng_aio_result(aio), NNG_ECANCELED); + NUTS_CLOSE(sub); + nng_aio_free(aio); +} + +static void +test_sub_recv_buf_option(void) +{ + nng_socket sub; + int v; + bool b; + size_t sz; + const char *opt = NNG_OPT_RECVBUF; + + NUTS_PASS(nng_sub0_open(&sub)); + + NUTS_PASS(nng_socket_set_int(sub, opt, 1)); + NUTS_FAIL(nng_socket_set_int(sub, opt, 0), NNG_EINVAL); + NUTS_FAIL(nng_socket_set_int(sub, opt, -1), NNG_EINVAL); + NUTS_FAIL(nng_socket_set_int(sub, opt, 1000000), NNG_EINVAL); + NUTS_PASS(nng_socket_set_int(sub, opt, 3)); + NUTS_PASS(nng_socket_get_int(sub, opt, &v)); + NUTS_TRUE(v == 3); + v = 0; + sz = sizeof(v); + NUTS_PASS(nng_socket_get(sub, opt, &v, &sz)); + NUTS_TRUE(v == 3); + NUTS_TRUE(sz == sizeof(v)); + + NUTS_FAIL(nng_socket_set(sub, opt, "", 1), NNG_EINVAL); + sz = 1; + NUTS_FAIL(nng_socket_get(sub, opt, &v, &sz), NNG_EINVAL); + NUTS_FAIL(nng_socket_set_bool(sub, opt, true), NNG_EBADTYPE); + NUTS_FAIL(nng_socket_get_bool(sub, opt, &b), NNG_EBADTYPE); + + NUTS_CLOSE(sub); +} + +static void +test_sub_subscribe_option(void) +{ + nng_socket sub; + size_t sz; + int v; + const char *opt = NNG_OPT_SUB_SUBSCRIBE; + + NUTS_PASS(nng_sub0_open(&sub)); + + NUTS_PASS(nng_socket_set(sub, opt, "abc", 3)); + NUTS_PASS(nng_socket_set(sub, opt, "abc", 3)); // duplicate + NUTS_PASS(nng_socket_set_bool(sub, opt, false)); + NUTS_PASS(nng_socket_set_int(sub, opt, 32)); + sz = sizeof(v); + NUTS_FAIL(nng_socket_get(sub, opt, &v, &sz), NNG_EWRITEONLY); + + NUTS_CLOSE(sub); +} + +static void +test_sub_unsubscribe_option(void) +{ + nng_socket sub; + size_t sz; + int v; + const char *opt1 = NNG_OPT_SUB_SUBSCRIBE; + const char *opt2 = NNG_OPT_SUB_UNSUBSCRIBE; + + NUTS_PASS(nng_sub0_open(&sub)); + + NUTS_PASS(nng_socket_set(sub, opt1, "abc", 3)); + NUTS_FAIL(nng_socket_set(sub, opt2, "abc123", 6), NNG_ENOENT); + NUTS_PASS(nng_socket_set(sub, opt2, "abc", 3)); + NUTS_FAIL(nng_socket_set(sub, opt2, "abc", 3), NNG_ENOENT); + NUTS_PASS(nng_socket_set_int(sub, opt1, 32)); + NUTS_FAIL(nng_socket_set_int(sub, opt2, 23), NNG_ENOENT); + NUTS_PASS(nng_socket_set_int(sub, opt2, 32)); + sz = sizeof(v); + NUTS_FAIL(nng_socket_get(sub, opt2, &v, &sz), NNG_EWRITEONLY); + + NUTS_CLOSE(sub); +} + +static void +test_sub_prefer_new_option(void) +{ + nng_socket sub; + bool b; + size_t sz; + const char *opt = NNG_OPT_SUB_PREFNEW; + + NUTS_PASS(nng_sub0_open(&sub)); + + NUTS_PASS(nng_socket_set_bool(sub, opt, true)); + NUTS_PASS(nng_socket_set_bool(sub, opt, false)); + NUTS_PASS(nng_socket_get_bool(sub, opt, &b)); + NUTS_TRUE(b == false); + sz = sizeof(b); + b = true; + NUTS_PASS(nng_socket_get(sub, opt, &b, &sz)); + NUTS_TRUE(b == false); + NUTS_TRUE(sz == sizeof(bool)); + + NUTS_FAIL(nng_socket_set(sub, opt, "abc", 3), NNG_EINVAL); + NUTS_FAIL(nng_socket_set_int(sub, opt, 1), NNG_EBADTYPE); + + NUTS_CLOSE(sub); +} + +void +test_sub_drop_new(void) +{ + nng_socket sub; + nng_socket pub; + nng_msg * msg; + + NUTS_PASS(nng_sub0_open(&sub)); + NUTS_PASS(nng_pub0_open(&pub)); + NUTS_PASS(nng_socket_set_int(sub, NNG_OPT_RECVBUF, 2)); + NUTS_PASS(nng_socket_set_bool(sub, NNG_OPT_SUB_PREFNEW, false)); + NUTS_PASS(nng_socket_set(sub, NNG_OPT_SUB_SUBSCRIBE, NULL, 0)); + NUTS_PASS(nng_socket_set_ms(sub, NNG_OPT_RECVTIMEO, 200)); + NUTS_PASS(nng_socket_set_ms(pub, NNG_OPT_SENDTIMEO, 1000)); + NUTS_MARRY(pub, sub); + NUTS_SEND(pub, "one"); + NUTS_SEND(pub, "two"); + NUTS_SEND(pub, "three"); + NUTS_SLEEP(100); + NUTS_RECV(sub, "one"); + NUTS_RECV(sub, "two"); + NUTS_FAIL(nng_recvmsg(sub, &msg, 0), NNG_ETIMEDOUT); + NUTS_CLOSE(pub); + NUTS_CLOSE(sub); +} + +void +test_sub_drop_old(void) +{ + nng_socket sub; + nng_socket pub; + nng_msg * msg; + + NUTS_PASS(nng_sub0_open(&sub)); + NUTS_PASS(nng_pub0_open(&pub)); + NUTS_PASS(nng_socket_set_int(sub, NNG_OPT_RECVBUF, 2)); + NUTS_PASS(nng_socket_set_bool(sub, NNG_OPT_SUB_PREFNEW, true)); + NUTS_PASS(nng_socket_set(sub, NNG_OPT_SUB_SUBSCRIBE, NULL, 0)); + NUTS_PASS(nng_socket_set_ms(sub, NNG_OPT_RECVTIMEO, 200)); + NUTS_PASS(nng_socket_set_ms(pub, NNG_OPT_SENDTIMEO, 1000)); + NUTS_MARRY(pub, sub); + NUTS_SEND(pub, "one"); + NUTS_SEND(pub, "two"); + NUTS_SEND(pub, "three"); + NUTS_SLEEP(100); + NUTS_RECV(sub, "two"); + NUTS_RECV(sub, "three"); + NUTS_FAIL(nng_recvmsg(sub, &msg, 0), NNG_ETIMEDOUT); + NUTS_CLOSE(pub); + NUTS_CLOSE(sub); +} + +static void +test_sub_filter(void) +{ + nng_socket sub; + nng_socket pub; + char buf[32]; + size_t sz; + + NUTS_PASS(nng_sub0_open(&sub)); + NUTS_PASS(nng_pub0_open(&pub)); + NUTS_PASS(nng_socket_set_ms(pub, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(sub, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_socket_set_int(sub, NNG_OPT_RECVBUF, 10)); + + // Set up some default filters + NUTS_PASS(nng_socket_set(sub, NNG_OPT_SUB_SUBSCRIBE, "abc", 3)); + NUTS_PASS(nng_socket_set(sub, NNG_OPT_SUB_SUBSCRIBE, "def", 3)); + NUTS_PASS(nng_socket_set(sub, NNG_OPT_SUB_SUBSCRIBE, "ghi", 3)); + NUTS_PASS(nng_socket_set(sub, NNG_OPT_SUB_SUBSCRIBE, "jkl", 3)); + + NUTS_MARRY(pub, sub); + + NUTS_PASS(nng_send(pub, "def", 3, 0)); + NUTS_PASS(nng_send(pub, "de", 2, 0)); // will not go through + NUTS_PASS(nng_send(pub, "abc123", 6, 0)); + NUTS_PASS(nng_send(pub, "xzy", 3, 0)); // does not match + NUTS_PASS(nng_send(pub, "ghi-drop", 7, 0)); // dropped by unsub + NUTS_PASS(nng_send(pub, "jkl-mno", 6, 0)); + + NUTS_SLEEP(100); + NUTS_PASS(nng_socket_set(sub, NNG_OPT_SUB_UNSUBSCRIBE, "ghi", 3)); + sz = sizeof(buf); + NUTS_PASS(nng_recv(sub, buf, &sz, 0)); + NUTS_TRUE(sz == 3); + NUTS_TRUE(memcmp(buf, "def", 3) == 0); + + sz = sizeof(buf); + NUTS_PASS(nng_recv(sub, buf, &sz, 0)); + NUTS_TRUE(sz == 6); + NUTS_TRUE(memcmp(buf, "abc123", 6) == 0); + + sz = sizeof(buf); + NUTS_PASS(nng_recv(sub, buf, &sz, 0)); + NUTS_TRUE(sz == 6); + NUTS_TRUE(memcmp(buf, "jkl-mno", 6) == 0); + + NUTS_CLOSE(sub); + NUTS_CLOSE(pub); +} + +static void +test_sub_multi_context(void) +{ + nng_socket sub; + nng_socket pub; + nng_ctx c1; + nng_ctx c2; + nng_aio * aio1; + nng_aio * aio2; + nng_msg * m; + + NUTS_PASS(nng_sub0_open(&sub)); + NUTS_PASS(nng_pub0_open(&pub)); + NUTS_PASS(nng_aio_alloc(&aio1, NULL, NULL)); + NUTS_PASS(nng_aio_alloc(&aio2, NULL, NULL)); + NUTS_PASS(nng_ctx_open(&c1, sub)); + NUTS_PASS(nng_ctx_open(&c2, sub)); + + NUTS_PASS(nng_ctx_set(c1, NNG_OPT_SUB_SUBSCRIBE, "one", 3)); + NUTS_PASS(nng_ctx_set(c1, NNG_OPT_SUB_SUBSCRIBE, "all", 3)); + + NUTS_PASS(nng_ctx_set(c2, NNG_OPT_SUB_SUBSCRIBE, "two", 3)); + NUTS_PASS(nng_ctx_set(c2, NNG_OPT_SUB_SUBSCRIBE, "all", 3)); + + nng_aio_set_timeout(aio1, 100); + nng_aio_set_timeout(aio2, 100); + + NUTS_MARRY(pub, sub); + + NUTS_SEND(pub, "one for the money"); + NUTS_SEND(pub, "all dogs go to heaven"); + NUTS_SEND(pub, "nobody likes a snitch"); + NUTS_SEND(pub, "two for the show"); + + nng_ctx_recv(c1, aio1); + nng_aio_wait(aio1); + NUTS_PASS(nng_aio_result(aio1)); + m = nng_aio_get_msg(aio1); + NUTS_MATCH(nng_msg_body(m), "one for the money"); + nng_msg_free(m); + + nng_ctx_recv(c1, aio1); + nng_aio_wait(aio1); + NUTS_PASS(nng_aio_result(aio1)); + m = nng_aio_get_msg(aio1); + NUTS_MATCH(nng_msg_body(m), "all dogs go to heaven"); + nng_msg_free(m); + + nng_ctx_recv(c2, aio1); + nng_aio_wait(aio1); + NUTS_PASS(nng_aio_result(aio1)); + m = nng_aio_get_msg(aio1); + NUTS_MATCH(nng_msg_body(m), "all dogs go to heaven"); + nng_msg_free(m); + + nng_ctx_recv(c2, aio1); + nng_aio_wait(aio1); + NUTS_PASS(nng_aio_result(aio1)); + m = nng_aio_get_msg(aio1); + NUTS_MATCH(nng_msg_body(m), "two for the show"); + nng_msg_free(m); + + nng_ctx_recv(c1, aio1); + nng_ctx_recv(c2, aio2); + + nng_aio_wait(aio1); + nng_aio_wait(aio2); + NUTS_FAIL(nng_aio_result(aio1), NNG_ETIMEDOUT); + NUTS_FAIL(nng_aio_result(aio2), NNG_ETIMEDOUT); + NUTS_CLOSE(sub); + NUTS_CLOSE(pub); + nng_aio_free(aio1); + nng_aio_free(aio2); +} + +static void +test_sub_cooked(void) +{ + nng_socket s; + bool b; + + NUTS_PASS(nng_sub0_open(&s)); + NUTS_PASS(nng_socket_get_bool(s, NNG_OPT_RAW, &b)); + NUTS_TRUE(!b); + NUTS_CLOSE(s); +} + +TEST_LIST = { + { "sub identity", test_sub_identity }, + { "sub cannot send", test_sub_cannot_send }, + { "sub context cannot send", test_sub_context_cannot_send }, + { "sub not writeable", test_sub_not_writeable }, + { "sub poll readable", test_sub_poll_readable }, + { "sub context does not poll", test_sub_context_no_poll }, + { "sub validate peer", test_sub_validate_peer }, + { "sub recv late", test_sub_recv_late }, + { "sub recv ctx closed", test_sub_recv_ctx_closed }, + { "sub recv aio ctx stopped", test_sub_ctx_recv_aio_stopped }, + { "sub close context recv", test_sub_close_context_recv }, + { "sub context recv nonblock", test_sub_ctx_recv_nonblock }, + { "sub context recv cancel", test_sub_ctx_recv_cancel }, + { "sub recv buf option", test_sub_recv_buf_option }, + { "sub subscribe option", test_sub_subscribe_option }, + { "sub unsubscribe option", test_sub_unsubscribe_option }, + { "sub prefer new option", test_sub_prefer_new_option }, + { "sub drop new", test_sub_drop_new }, + { "sub drop old", test_sub_drop_old }, + { "sub filter", test_sub_filter }, + { "sub multi context", test_sub_multi_context }, + { "sub cooked", test_sub_cooked }, + { NULL, NULL }, +}; diff --git a/src/sp/protocol/pubsub0/xsub.c b/src/sp/protocol/pubsub0/xsub.c new file mode 100644 index 00000000..0013b8b3 --- /dev/null +++ b/src/sp/protocol/pubsub0/xsub.c @@ -0,0 +1,211 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// Copyright 2018 Capitar IT Group BV +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include + +#include "core/nng_impl.h" +#include "nng/protocol/pubsub0/sub.h" + +// Subscriber protocol. The SUB protocol receives messages sent to +// it from publishers, and filters out those it is not interested in, +// only passing up ones that match known subscriptions. + +#ifndef NNI_PROTO_SUB_V0 +#define NNI_PROTO_SUB_V0 NNI_PROTO(2, 1) +#endif + +#ifndef NNI_PROTO_PUB_V0 +#define NNI_PROTO_PUB_V0 NNI_PROTO(2, 0) +#endif + +typedef struct xsub0_pipe xsub0_pipe; +typedef struct xsub0_sock xsub0_sock; + +static void xsub0_recv_cb(void *); +static void xsub0_pipe_fini(void *); + +// xsub0_sock is our per-socket protocol private structure. +struct xsub0_sock { + nni_msgq *urq; + nni_mtx lk; +}; + +// sub0_pipe is our per-pipe protocol private structure. +struct xsub0_pipe { + nni_pipe * pipe; + xsub0_sock *sub; + nni_aio aio_recv; +}; + +static int +xsub0_sock_init(void *arg, nni_sock *sock) +{ + xsub0_sock *s = arg; + + s->urq = nni_sock_recvq(sock); + return (0); +} + +static void +xsub0_sock_fini(void *arg) +{ + xsub0_sock *s = arg; + nni_mtx_fini(&s->lk); +} + +static void +xsub0_sock_open(void *arg) +{ + NNI_ARG_UNUSED(arg); +} + +static void +xsub0_sock_close(void *arg) +{ + NNI_ARG_UNUSED(arg); +} + +static void +xsub0_pipe_stop(void *arg) +{ + xsub0_pipe *p = arg; + + nni_aio_stop(&p->aio_recv); +} + +static void +xsub0_pipe_fini(void *arg) +{ + xsub0_pipe *p = arg; + + nni_aio_fini(&p->aio_recv); +} + +static int +xsub0_pipe_init(void *arg, nni_pipe *pipe, void *s) +{ + xsub0_pipe *p = arg; + + nni_aio_init(&p->aio_recv, xsub0_recv_cb, p); + + p->pipe = pipe; + p->sub = s; + return (0); +} + +static int +xsub0_pipe_start(void *arg) +{ + xsub0_pipe *p = arg; + + if (nni_pipe_peer(p->pipe) != NNI_PROTO_PUB_V0) { + // Peer protocol mismatch. + return (NNG_EPROTO); + } + + nni_pipe_recv(p->pipe, &p->aio_recv); + return (0); +} + +static void +xsub0_pipe_close(void *arg) +{ + xsub0_pipe *p = arg; + + nni_aio_close(&p->aio_recv); +} + +static void +xsub0_recv_cb(void *arg) +{ + xsub0_pipe *p = arg; + xsub0_sock *s = p->sub; + nni_msgq * urq = s->urq; + nni_msg * msg; + + if (nni_aio_result(&p->aio_recv) != 0) { + nni_pipe_close(p->pipe); + return; + } + + msg = nni_aio_get_msg(&p->aio_recv); + nni_aio_set_msg(&p->aio_recv, NULL); + nni_msg_set_pipe(msg, nni_pipe_id(p->pipe)); + + if (nni_msgq_tryput(urq, msg) != 0) { + // This only happens for two reasons. For flow control, + // in which case we just want to discard the message and + // carry on, and for a close of the socket (which is very + // hard to achieve, since we close the pipes.) In either + // case the easiest thing to do is just free the message + // and try again. + nni_msg_free(msg); + } + nni_pipe_recv(p->pipe, &p->aio_recv); +} + +static void +xsub0_sock_send(void *arg, nni_aio *aio) +{ + NNI_ARG_UNUSED(arg); + nni_aio_finish_error(aio, NNG_ENOTSUP); +} + +static void +xsub0_sock_recv(void *arg, nni_aio *aio) +{ + xsub0_sock *s = arg; + + nni_msgq_aio_get(s->urq, aio); +} + +// This is the global protocol structure -- our linkage to the core. +// This should be the only global non-static symbol in this file. +static nni_proto_pipe_ops xsub0_pipe_ops = { + .pipe_size = sizeof(xsub0_pipe), + .pipe_init = xsub0_pipe_init, + .pipe_fini = xsub0_pipe_fini, + .pipe_start = xsub0_pipe_start, + .pipe_close = xsub0_pipe_close, + .pipe_stop = xsub0_pipe_stop, +}; + +static nni_option xsub0_sock_options[] = { + // terminate list + { + .o_name = NULL, + }, +}; + +static nni_proto_sock_ops xsub0_sock_ops = { + .sock_size = sizeof(xsub0_sock), + .sock_init = xsub0_sock_init, + .sock_fini = xsub0_sock_fini, + .sock_open = xsub0_sock_open, + .sock_close = xsub0_sock_close, + .sock_send = xsub0_sock_send, + .sock_recv = xsub0_sock_recv, + .sock_options = xsub0_sock_options, +}; + +static nni_proto xsub0_proto = { + .proto_version = NNI_PROTOCOL_VERSION, + .proto_self = { NNI_PROTO_SUB_V0, "sub" }, + .proto_peer = { NNI_PROTO_PUB_V0, "pub" }, + .proto_flags = NNI_PROTO_FLAG_RCV | NNI_PROTO_FLAG_RAW, + .proto_sock_ops = &xsub0_sock_ops, + .proto_pipe_ops = &xsub0_pipe_ops, +}; + +int +nng_sub0_open_raw(nng_socket *sidp) +{ + return (nni_proto_open(sidp, &xsub0_proto)); +} diff --git a/src/sp/protocol/pubsub0/xsub_test.c b/src/sp/protocol/pubsub0/xsub_test.c new file mode 100644 index 00000000..19815661 --- /dev/null +++ b/src/sp/protocol/pubsub0/xsub_test.c @@ -0,0 +1,376 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include + +static void +test_xsub_identity(void) +{ + nng_socket s; + int p; + char * n; + + NUTS_PASS(nng_sub0_open_raw(&s)); + NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PROTO, &p)); + NUTS_TRUE(p == NUTS_PROTO(2u, 1u)); // 33 + NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PEER, &p)); + NUTS_TRUE(p == NUTS_PROTO(2u, 0u)); // 32 + NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PROTONAME, &n)); + NUTS_MATCH(n, "sub"); + nng_strfree(n); + NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PEERNAME, &n)); + NUTS_MATCH(n, "pub"); + nng_strfree(n); + NUTS_CLOSE(s); +} + +static void +test_xsub_cannot_send(void) +{ + nng_socket sub; + + NUTS_PASS(nng_sub0_open_raw(&sub)); + NUTS_FAIL(nng_send(sub, "", 0, 0), NNG_ENOTSUP); + NUTS_CLOSE(sub); +} + +static void +test_xsub_not_writeable(void) +{ + int fd; + nng_socket sub; + + NUTS_PASS(nng_sub0_open_raw(&sub)); + NUTS_FAIL(nng_socket_get_int(sub, NNG_OPT_SENDFD, &fd), NNG_ENOTSUP); + NUTS_CLOSE(sub); +} + +static void +test_xsub_poll_readable(void) +{ + int fd; + nng_socket pub; + nng_socket sub; + + NUTS_PASS(nng_sub0_open_raw(&sub)); + NUTS_PASS(nng_pub0_open(&pub)); + NUTS_PASS(nng_socket_set_ms(sub, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(pub, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_get_int(sub, NNG_OPT_RECVFD, &fd)); + NUTS_TRUE(fd >= 0); + + // Not readable if not connected! + NUTS_TRUE(nuts_poll_fd(fd) == false); + + // Even after connect (no message yet) + NUTS_MARRY(pub, sub); + NUTS_TRUE(nuts_poll_fd(fd) == false); + + // But once we send messages, it is. + // We have to send a request, in order to send a reply. + NUTS_SEND(pub, "abc"); + NUTS_SLEEP(200); + + NUTS_TRUE(nuts_poll_fd(fd)); + + // and receiving makes it no longer ready + NUTS_RECV(sub, "abc"); + NUTS_TRUE(nuts_poll_fd(fd) == false); + + NUTS_CLOSE(pub); + NUTS_CLOSE(sub); +} + +static void +test_xsub_recv_late(void) +{ + int fd; + nng_socket pub; + nng_socket sub; + nng_aio * aio; + nng_msg * msg; + + NUTS_PASS(nng_sub0_open_raw(&sub)); + NUTS_PASS(nng_pub0_open(&pub)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + NUTS_PASS(nng_socket_set_ms(sub, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(pub, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_get_int(sub, NNG_OPT_RECVFD, &fd)); + NUTS_TRUE(fd >= 0); + + // Not readable if not connected! + NUTS_TRUE(nuts_poll_fd(fd) == false); + + // Even after connect (no message yet) + NUTS_MARRY(pub, sub); + NUTS_TRUE(nuts_poll_fd(fd) == false); + + nng_recv_aio(sub, aio); + + // But once we send messages, it is. + // We have to send a request, in order to send a reply. + NUTS_SEND(pub, "abc"); + NUTS_SLEEP(200); + + nng_aio_wait(aio); + NUTS_PASS(nng_aio_result(aio)); + msg = nng_aio_get_msg(aio); + nng_aio_set_msg(aio, NULL); + NUTS_TRUE(nng_msg_len(msg) == 4); + NUTS_TRUE(strcmp(nng_msg_body(msg), "abc") == 0); + + nng_msg_free(msg); + nng_aio_free(aio); + + NUTS_CLOSE(pub); + NUTS_CLOSE(sub); +} + +void +test_xsub_no_context(void) +{ + nng_socket sub; + nng_ctx ctx; + + NUTS_PASS(nng_sub0_open_raw(&sub)); + NUTS_FAIL(nng_ctx_open(&ctx, sub), NNG_ENOTSUP); + NUTS_CLOSE(sub); +} + +void +test_xsub_validate_peer(void) +{ + nng_socket s1, s2; + nng_stat * stats; + nng_stat * reject; + char * addr; + + NUTS_ADDR(addr, "inproc"); + + NUTS_PASS(nng_sub0_open_raw(&s1)); + NUTS_PASS(nng_sub0_open_raw(&s2)); + + NUTS_PASS(nng_listen(s1, addr, NULL, 0)); + NUTS_PASS(nng_dial(s2, addr, NULL, NNG_FLAG_NONBLOCK)); + + NUTS_SLEEP(100); + NUTS_PASS(nng_stats_get(&stats)); + + NUTS_TRUE(stats != NULL); + NUTS_TRUE((reject = nng_stat_find_socket(stats, s1)) != NULL); + NUTS_TRUE((reject = nng_stat_find(reject, "reject")) != NULL); + + NUTS_TRUE(nng_stat_type(reject) == NNG_STAT_COUNTER); + NUTS_TRUE(nng_stat_value(reject) > 0); + + NUTS_CLOSE(s1); + NUTS_CLOSE(s2); + nng_stats_free(stats); +} + +static void +test_xsub_recv_closed(void) +{ + nng_socket sub; + nng_aio * aio; + NUTS_PASS(nng_sub0_open_raw(&sub)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + NUTS_CLOSE(sub); + nng_recv_aio(sub, aio); + nng_aio_wait(aio); + NUTS_FAIL(nng_aio_result(aio), NNG_ECLOSED); + nng_aio_free(aio); +} + +static void +test_xsub_close_recv(void) +{ + nng_socket sub; + nng_aio * aio; + + NUTS_PASS(nng_sub0_open_raw(&sub)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + nng_aio_set_timeout(aio, 1000); + nng_recv_aio(sub, aio); + NUTS_CLOSE(sub); + nng_aio_wait(aio); + NUTS_FAIL(nng_aio_result(aio), NNG_ECLOSED); + + nng_aio_free(aio); +} + +static void +test_xsub_recv_nonblock(void) +{ + nng_socket sub; + nng_aio * aio; + + NUTS_PASS(nng_sub0_open_raw(&sub)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + + nng_aio_set_timeout(aio, 0); // Instant timeout + nng_recv_aio(sub, aio); + + nng_aio_wait(aio); + NUTS_FAIL(nng_aio_result(aio), NNG_ETIMEDOUT); + NUTS_CLOSE(sub); + nng_aio_free(aio); +} + +static void +test_xsub_recv_buf_option(void) +{ + nng_socket sub; + int v; + bool b; + size_t sz; + const char *opt = NNG_OPT_RECVBUF; + + NUTS_PASS(nng_sub0_open_raw(&sub)); + + NUTS_PASS(nng_socket_set_int(sub, opt, 1)); + NUTS_FAIL(nng_socket_set_int(sub, opt, -1), NNG_EINVAL); + NUTS_FAIL(nng_socket_set_int(sub, opt, 1000000), NNG_EINVAL); + NUTS_PASS(nng_socket_set_int(sub, opt, 3)); + NUTS_PASS(nng_socket_get_int(sub, opt, &v)); + NUTS_TRUE(v == 3); + v = 0; + sz = sizeof(v); + NUTS_PASS(nng_socket_get(sub, opt, &v, &sz)); + NUTS_TRUE(v == 3); + NUTS_TRUE(sz == sizeof(v)); + + NUTS_FAIL(nng_socket_set(sub, opt, "", 1), NNG_EINVAL); + sz = 1; + NUTS_FAIL(nng_socket_get(sub, opt, &v, &sz), NNG_EINVAL); + NUTS_FAIL(nng_socket_set_bool(sub, opt, true), NNG_EBADTYPE); + NUTS_FAIL(nng_socket_get_bool(sub, opt, &b), NNG_EBADTYPE); + + NUTS_CLOSE(sub); +} + +static void +test_xsub_subscribe_option(void) +{ + nng_socket sub; + const char *opt = NNG_OPT_SUB_SUBSCRIBE; + + NUTS_PASS(nng_sub0_open_raw(&sub)); + NUTS_FAIL(nng_socket_set(sub, opt, "abc", 3), NNG_ENOTSUP); + NUTS_CLOSE(sub); +} + +static void +test_xsub_unsubscribe_option(void) +{ + nng_socket sub; + const char *opt = NNG_OPT_SUB_UNSUBSCRIBE; + + NUTS_PASS(nng_sub0_open_raw(&sub)); + NUTS_FAIL(nng_socket_set(sub, opt, "abc", 3), NNG_ENOTSUP); + NUTS_CLOSE(sub); +} + +static void +test_xsub_raw(void) +{ + nng_socket s; + bool b; + + NUTS_PASS(nng_sub0_open_raw(&s)); + NUTS_PASS(nng_socket_get_bool(s, NNG_OPT_RAW, &b)); + NUTS_TRUE(b); + NUTS_CLOSE(s); +} + +static void +test_xsub_close_during_recv(void) +{ + nng_socket sub; + nng_socket pub; + + NUTS_PASS(nng_sub0_open_raw(&sub)); + NUTS_PASS(nng_pub0_open(&pub)); + NUTS_PASS(nng_socket_set_ms(sub, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(pub, NNG_OPT_SENDTIMEO, 100)); + NUTS_PASS(nng_socket_set_int(sub, NNG_OPT_RECVBUF, 5)); + NUTS_PASS(nng_socket_set_int(pub, NNG_OPT_SENDBUF, 20)); + + NUTS_MARRY(pub, sub); + + for (unsigned i = 0; i < 100; i++) { + NUTS_PASS(nng_send(pub, "abc", 3, 0)); + } + NUTS_CLOSE(pub); + NUTS_CLOSE(sub); +} + +static void +test_xsub_close_during_pipe_recv(void) +{ + nng_socket sub; + nng_socket pub; + + NUTS_PASS(nng_sub0_open_raw(&sub)); + NUTS_PASS(nng_pub0_open(&pub)); + NUTS_PASS(nng_socket_set_ms(sub, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(pub, NNG_OPT_SENDTIMEO, 100)); + NUTS_PASS(nng_socket_set_int(sub, NNG_OPT_RECVBUF, 5)); + NUTS_PASS(nng_socket_set_int(pub, NNG_OPT_SENDBUF, 20)); + + NUTS_MARRY(pub, sub); + + for (unsigned i = 0; i < 100; i++) { + int rv; + rv = nng_send(pub, "abc", 3, 0); + if (rv == NNG_ETIMEDOUT) { + break; + } + NUTS_SLEEP(1); + } + NUTS_CLOSE(sub); +} + +static void +test_xsub_recv_aio_stopped(void) +{ + nng_socket sub; + nng_aio * aio; + + NUTS_PASS(nng_sub0_open_raw(&sub)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + + nng_aio_stop(aio); + nng_recv_aio(sub, aio); + nng_aio_wait(aio); + NUTS_FAIL(nng_aio_result(aio), NNG_ECANCELED); + NUTS_CLOSE(sub); + nng_aio_free(aio); +} + +TEST_LIST = { + { "xsub identity", test_xsub_identity }, + { "xsub cannot send", test_xsub_cannot_send }, + { "xsub not writeable", test_xsub_not_writeable }, + { "xsub poll readable", test_xsub_poll_readable }, + { "xsub validate peer", test_xsub_validate_peer }, + { "xsub recv late", test_xsub_recv_late }, + { "xsub recv closed", test_xsub_recv_closed }, + { "xsub close recv", test_xsub_close_recv }, + { "xsub recv nonblock", test_xsub_recv_nonblock }, + { "xsub recv buf option", test_xsub_recv_buf_option }, + { "xsub subscribe option", test_xsub_subscribe_option }, + { "xsub unsubscribe option", test_xsub_unsubscribe_option }, + { "xsub no context", test_xsub_no_context }, + { "xsub raw", test_xsub_raw }, + { "xsub recv aio stopped", test_xsub_recv_aio_stopped }, + { "xsub close during recv ", test_xsub_close_during_recv }, + { "xsub close during pipe recv", test_xsub_close_during_pipe_recv }, + { NULL, NULL }, +}; diff --git a/src/sp/protocol/reqrep0/CMakeLists.txt b/src/sp/protocol/reqrep0/CMakeLists.txt new file mode 100644 index 00000000..a3cecfd0 --- /dev/null +++ b/src/sp/protocol/reqrep0/CMakeLists.txt @@ -0,0 +1,25 @@ +# +# Copyright 2020 Staysail Systems, Inc. +# Copyright 2018 Capitar IT Group BV +# +# This software is supplied under the terms of the MIT License, a +# copy of which should be located in the distribution where this +# file was obtained (LICENSE.txt). A copy of the license may also be +# found online at https://opensource.org/licenses/MIT. +# + +# Req/Rep protocol +nng_directory(reqrep0) + +nng_sources_if(NNG_PROTO_REQ0 req.c xreq.c) +nng_headers_if(NNG_PROTO_REQ0 nng/protocol/reqrep0/req.h) +nng_defines_if(NNG_PROTO_REQ0 NNG_HAVE_REQ0) + +nng_sources_if(NNG_PROTO_REP0 rep.c xrep.c) +nng_headers_if(NNG_PROTO_REP0 nng/protocol/reqrep0/rep.h) +nng_defines_if(NNG_PROTO_REP0 NNG_HAVE_REP0) + +nng_test(req_test) +nng_test(rep_test) +nng_test(xrep_test) +nng_test(xreq_test) diff --git a/src/sp/protocol/reqrep0/rep.c b/src/sp/protocol/reqrep0/rep.c new file mode 100644 index 00000000..aa32d249 --- /dev/null +++ b/src/sp/protocol/reqrep0/rep.c @@ -0,0 +1,705 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// Copyright 2018 Capitar IT Group BV +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include + +#include "core/nng_impl.h" +#include "nng/protocol/reqrep0/rep.h" + +// Response protocol. The REP protocol is the "reply" side of a +// request-reply pair. This is useful for building RPC servers, for +// example. + +typedef struct rep0_pipe rep0_pipe; +typedef struct rep0_sock rep0_sock; +typedef struct rep0_ctx rep0_ctx; + +static void rep0_pipe_send_cb(void *); +static void rep0_pipe_recv_cb(void *); +static void rep0_pipe_fini(void *); + +struct rep0_ctx { + rep0_sock * sock; + uint32_t pipe_id; + rep0_pipe * spipe; // send pipe + nni_aio * saio; // send aio + nni_aio * raio; // recv aio + nni_list_node sqnode; + nni_list_node rqnode; + size_t btrace_len; + uint32_t btrace[NNI_MAX_MAX_TTL + 1]; +}; + +// rep0_sock is our per-socket protocol private structure. +struct rep0_sock { + nni_mtx lk; + nni_atomic_int ttl; + nni_id_map pipes; + nni_list recvpipes; // list of pipes with data to receive + nni_list recvq; + rep0_ctx ctx; + nni_pollable readable; + nni_pollable writable; +}; + +// rep0_pipe is our per-pipe protocol private structure. +struct rep0_pipe { + nni_pipe * pipe; + rep0_sock * rep; + uint32_t id; + nni_aio aio_send; + nni_aio aio_recv; + nni_list_node rnode; // receivable list linkage + nni_list sendq; // contexts waiting to send + bool busy; + bool closed; +}; + +static void +rep0_ctx_close(void *arg) +{ + rep0_ctx * ctx = arg; + rep0_sock *s = ctx->sock; + nni_aio * aio; + + nni_mtx_lock(&s->lk); + if ((aio = ctx->saio) != NULL) { + rep0_pipe *pipe = ctx->spipe; + ctx->saio = NULL; + ctx->spipe = NULL; + nni_list_remove(&pipe->sendq, ctx); + nni_aio_finish_error(aio, NNG_ECLOSED); + } + if ((aio = ctx->raio) != NULL) { + nni_list_remove(&s->recvq, ctx); + ctx->raio = NULL; + nni_aio_finish_error(aio, NNG_ECLOSED); + } + nni_mtx_unlock(&s->lk); +} + +static void +rep0_ctx_fini(void *arg) +{ + rep0_ctx *ctx = arg; + + rep0_ctx_close(ctx); +} + +static int +rep0_ctx_init(void *carg, void *sarg) +{ + rep0_sock *s = sarg; + rep0_ctx * ctx = carg; + + NNI_LIST_NODE_INIT(&ctx->sqnode); + NNI_LIST_NODE_INIT(&ctx->rqnode); + ctx->btrace_len = 0; + ctx->sock = s; + ctx->pipe_id = 0; + + return (0); +} + +static void +rep0_ctx_cancel_send(nni_aio *aio, void *arg, int rv) +{ + rep0_ctx * ctx = arg; + rep0_sock *s = ctx->sock; + + nni_mtx_lock(&s->lk); + if (ctx->saio != aio) { + nni_mtx_unlock(&s->lk); + return; + } + nni_list_node_remove(&ctx->sqnode); + ctx->saio = NULL; + nni_mtx_unlock(&s->lk); + + nni_msg_header_clear(nni_aio_get_msg(aio)); // reset the headers + nni_aio_finish_error(aio, rv); +} + +static void +rep0_ctx_send(void *arg, nni_aio *aio) +{ + rep0_ctx * ctx = arg; + rep0_sock *s = ctx->sock; + rep0_pipe *p; + nni_msg * msg; + int rv; + size_t len; + uint32_t p_id; // pipe id + + msg = nni_aio_get_msg(aio); + nni_msg_header_clear(msg); + + if (nni_aio_begin(aio) != 0) { + return; + } + + nni_mtx_lock(&s->lk); + len = ctx->btrace_len; + p_id = ctx->pipe_id; + + // Assert "completion" of the previous req request. This ensures + // exactly one send for one receive ordering. + ctx->btrace_len = 0; + ctx->pipe_id = 0; + + if (ctx == &s->ctx) { + // No matter how this goes, we will no longer be able + // to send on the socket (root context). That's because + // we will have finished (successfully or otherwise) the + // reply for the single request we got. + nni_pollable_clear(&s->writable); + } + if (len == 0) { + nni_mtx_unlock(&s->lk); + nni_aio_finish_error(aio, NNG_ESTATE); + return; + } + if ((rv = nni_msg_header_append(msg, ctx->btrace, len)) != 0) { + nni_mtx_unlock(&s->lk); + nni_aio_finish_error(aio, rv); + return; + } + if ((p = nni_id_get(&s->pipes, p_id)) == NULL) { + // Pipe is gone. Make this look like a good send to avoid + // disrupting the state machine. We don't care if the peer + // lost interest in our reply. + nni_mtx_unlock(&s->lk); + nni_aio_set_msg(aio, NULL); + nni_aio_finish(aio, 0, nni_msg_len(msg)); + nni_msg_free(msg); + return; + } + if (!p->busy) { + p->busy = true; + len = nni_msg_len(msg); + nni_aio_set_msg(&p->aio_send, msg); + nni_pipe_send(p->pipe, &p->aio_send); + nni_mtx_unlock(&s->lk); + + nni_aio_set_msg(aio, NULL); + nni_aio_finish(aio, 0, len); + return; + } + + if ((rv = nni_aio_schedule(aio, rep0_ctx_cancel_send, ctx)) != 0) { + nni_mtx_unlock(&s->lk); + nni_aio_finish_error(aio, rv); + return; + } + + ctx->saio = aio; + ctx->spipe = p; + nni_list_append(&p->sendq, ctx); + nni_mtx_unlock(&s->lk); +} + +static void +rep0_sock_fini(void *arg) +{ + rep0_sock *s = arg; + + nni_id_map_fini(&s->pipes); + rep0_ctx_fini(&s->ctx); + nni_pollable_fini(&s->writable); + nni_pollable_fini(&s->readable); + nni_mtx_fini(&s->lk); +} + +static int +rep0_sock_init(void *arg, nni_sock *sock) +{ + rep0_sock *s = arg; + + NNI_ARG_UNUSED(sock); + + nni_mtx_init(&s->lk); + nni_id_map_init(&s->pipes, 0, 0, false); + NNI_LIST_INIT(&s->recvq, rep0_ctx, rqnode); + NNI_LIST_INIT(&s->recvpipes, rep0_pipe, rnode); + nni_atomic_init(&s->ttl); + nni_atomic_set(&s->ttl, 8); + + (void) rep0_ctx_init(&s->ctx, s); + + // We start off without being either readable or writable. + // Readability comes when there is something on the socket. + nni_pollable_init(&s->writable); + nni_pollable_init(&s->readable); + + return (0); +} + +static void +rep0_sock_open(void *arg) +{ + NNI_ARG_UNUSED(arg); +} + +static void +rep0_sock_close(void *arg) +{ + rep0_sock *s = arg; + + rep0_ctx_close(&s->ctx); +} + +static void +rep0_pipe_stop(void *arg) +{ + rep0_pipe *p = arg; + + nni_aio_stop(&p->aio_send); + nni_aio_stop(&p->aio_recv); +} + +static void +rep0_pipe_fini(void *arg) +{ + rep0_pipe *p = arg; + nng_msg * msg; + + if ((msg = nni_aio_get_msg(&p->aio_recv)) != NULL) { + nni_aio_set_msg(&p->aio_recv, NULL); + nni_msg_free(msg); + } + + nni_aio_fini(&p->aio_send); + nni_aio_fini(&p->aio_recv); +} + +static int +rep0_pipe_init(void *arg, nni_pipe *pipe, void *s) +{ + rep0_pipe *p = arg; + + nni_aio_init(&p->aio_send, rep0_pipe_send_cb, p); + nni_aio_init(&p->aio_recv, rep0_pipe_recv_cb, p); + + NNI_LIST_INIT(&p->sendq, rep0_ctx, sqnode); + + p->id = nni_pipe_id(pipe); + p->pipe = pipe; + p->rep = s; + return (0); +} + +static int +rep0_pipe_start(void *arg) +{ + rep0_pipe *p = arg; + rep0_sock *s = p->rep; + int rv; + + if (nni_pipe_peer(p->pipe) != NNG_REP0_PEER) { + // Peer protocol mismatch. + return (NNG_EPROTO); + } + + nni_mtx_lock(&s->lk); + rv = nni_id_set(&s->pipes, nni_pipe_id(p->pipe), p); + nni_mtx_unlock(&s->lk); + if (rv != 0) { + return (rv); + } + // By definition, we have not received a request yet on this pipe, + // so it cannot cause us to become writable. + nni_pipe_recv(p->pipe, &p->aio_recv); + return (0); +} + +static void +rep0_pipe_close(void *arg) +{ + rep0_pipe *p = arg; + rep0_sock *s = p->rep; + rep0_ctx * ctx; + + nni_aio_close(&p->aio_send); + nni_aio_close(&p->aio_recv); + + nni_mtx_lock(&s->lk); + p->closed = true; + if (nni_list_active(&s->recvpipes, p)) { + // We are no longer "receivable". + nni_list_remove(&s->recvpipes, p); + } + while ((ctx = nni_list_first(&p->sendq)) != NULL) { + nni_aio *aio; + nni_msg *msg; + // Pipe was closed. To avoid pushing an error back to the + // entire socket, we pretend we completed this successfully. + nni_list_remove(&p->sendq, ctx); + aio = ctx->saio; + ctx->saio = NULL; + msg = nni_aio_get_msg(aio); + nni_aio_set_msg(aio, NULL); + nni_aio_finish(aio, 0, nni_msg_len(msg)); + nni_msg_free(msg); + } + if (p->id == s->ctx.pipe_id) { + // We "can" send. (Well, not really, but we will happily + // accept a message and discard it.) + nni_pollable_raise(&s->writable); + } + nni_id_remove(&s->pipes, nni_pipe_id(p->pipe)); + nni_mtx_unlock(&s->lk); +} + +static void +rep0_pipe_send_cb(void *arg) +{ + rep0_pipe *p = arg; + rep0_sock *s = p->rep; + rep0_ctx * ctx; + nni_aio * aio; + nni_msg * msg; + size_t len; + + if (nni_aio_result(&p->aio_send) != 0) { + nni_msg_free(nni_aio_get_msg(&p->aio_send)); + nni_aio_set_msg(&p->aio_send, NULL); + nni_pipe_close(p->pipe); + return; + } + nni_mtx_lock(&s->lk); + p->busy = false; + if ((ctx = nni_list_first(&p->sendq)) == NULL) { + // Nothing else to send. + if (p->id == s->ctx.pipe_id) { + // Mark us ready for the other side to send! + nni_pollable_raise(&s->writable); + } + nni_mtx_unlock(&s->lk); + return; + } + + nni_list_remove(&p->sendq, ctx); + aio = ctx->saio; + ctx->saio = NULL; + ctx->spipe = NULL; + p->busy = true; + msg = nni_aio_get_msg(aio); + len = nni_msg_len(msg); + nni_aio_set_msg(aio, NULL); + nni_aio_set_msg(&p->aio_send, msg); + nni_pipe_send(p->pipe, &p->aio_send); + + nni_mtx_unlock(&s->lk); + + nni_aio_finish_sync(aio, 0, len); +} + +static void +rep0_cancel_recv(nni_aio *aio, void *arg, int rv) +{ + rep0_ctx * ctx = arg; + rep0_sock *s = ctx->sock; + + nni_mtx_lock(&s->lk); + if (ctx->raio == aio) { + nni_list_remove(&s->recvq, ctx); + ctx->raio = NULL; + nni_aio_finish_error(aio, rv); + } + nni_mtx_unlock(&s->lk); +} + +static void +rep0_ctx_recv(void *arg, nni_aio *aio) +{ + rep0_ctx * ctx = arg; + rep0_sock *s = ctx->sock; + rep0_pipe *p; + size_t len; + nni_msg * msg; + + if (nni_aio_begin(aio) != 0) { + return; + } + nni_mtx_lock(&s->lk); + if ((p = nni_list_first(&s->recvpipes)) == NULL) { + int rv; + if ((rv = nni_aio_schedule(aio, rep0_cancel_recv, ctx)) != 0) { + nni_mtx_unlock(&s->lk); + nni_aio_finish_error(aio, rv); + return; + } + if (ctx->raio != NULL) { + // Cannot have a second receive operation pending. + // This could be ESTATE, or we could cancel the first + // with ECANCELED. We elect the former. + nni_mtx_unlock(&s->lk); + nni_aio_finish_error(aio, NNG_ESTATE); + return; + } + ctx->raio = aio; + nni_list_append(&s->recvq, ctx); + nni_mtx_unlock(&s->lk); + return; + } + msg = nni_aio_get_msg(&p->aio_recv); + nni_aio_set_msg(&p->aio_recv, NULL); + nni_list_remove(&s->recvpipes, p); + if (nni_list_empty(&s->recvpipes)) { + nni_pollable_clear(&s->readable); + } + nni_pipe_recv(p->pipe, &p->aio_recv); + if ((ctx == &s->ctx) && !p->busy) { + nni_pollable_raise(&s->writable); + } + + len = nni_msg_header_len(msg); + memcpy(ctx->btrace, nni_msg_header(msg), len); + ctx->btrace_len = len; + ctx->pipe_id = nni_pipe_id(p->pipe); + nni_mtx_unlock(&s->lk); + + nni_msg_header_clear(msg); + nni_aio_set_msg(aio, msg); + nni_aio_finish(aio, 0, nni_msg_len(msg)); +} + +static void +rep0_pipe_recv_cb(void *arg) +{ + rep0_pipe *p = arg; + rep0_sock *s = p->rep; + rep0_ctx * ctx; + nni_msg * msg; + uint8_t * body; + nni_aio * aio; + size_t len; + int hops; + int ttl; + + if (nni_aio_result(&p->aio_recv) != 0) { + nni_pipe_close(p->pipe); + return; + } + + msg = nni_aio_get_msg(&p->aio_recv); + ttl = nni_atomic_get(&s->ttl); + + nni_msg_set_pipe(msg, p->id); + + // Move backtrace from body to header + hops = 1; + for (;;) { + bool end; + + if (hops > ttl) { + // This isn't malformed, but it has gone + // through too many hops. Do not disconnect, + // because we can legitimately receive messages + // with too many hops from devices, etc. + goto drop; + } + hops++; + if (nni_msg_len(msg) < 4) { + // Peer is speaking garbage. Kick it. + nni_msg_free(msg); + nni_aio_set_msg(&p->aio_recv, NULL); + nni_pipe_close(p->pipe); + return; + } + body = nni_msg_body(msg); + end = ((body[0] & 0x80u) != 0); + if (nni_msg_header_append(msg, body, 4) != 0) { + // Out of memory, so drop it. + goto drop; + } + nni_msg_trim(msg, 4); + if (end) { + break; + } + } + + len = nni_msg_header_len(msg); + + nni_mtx_lock(&s->lk); + + if (p->closed) { + // If we are closed, then we can't return data. + nni_aio_set_msg(&p->aio_recv, NULL); + nni_mtx_unlock(&s->lk); + nni_msg_free(msg); + return; + } + + if ((ctx = nni_list_first(&s->recvq)) == NULL) { + // No one waiting to receive yet, holding pattern. + nni_list_append(&s->recvpipes, p); + nni_pollable_raise(&s->readable); + nni_mtx_unlock(&s->lk); + return; + } + + nni_list_remove(&s->recvq, ctx); + aio = ctx->raio; + ctx->raio = NULL; + nni_aio_set_msg(&p->aio_recv, NULL); + if ((ctx == &s->ctx) && !p->busy) { + nni_pollable_raise(&s->writable); + } + + // schedule another receive + nni_pipe_recv(p->pipe, &p->aio_recv); + + ctx->btrace_len = len; + memcpy(ctx->btrace, nni_msg_header(msg), len); + nni_msg_header_clear(msg); + ctx->pipe_id = p->id; + + nni_mtx_unlock(&s->lk); + + nni_aio_set_msg(aio, msg); + nni_aio_finish_sync(aio, 0, nni_msg_len(msg)); + return; + +drop: + nni_msg_free(msg); + nni_aio_set_msg(&p->aio_recv, NULL); + nni_pipe_recv(p->pipe, &p->aio_recv); +} + +static int +rep0_sock_set_max_ttl(void *arg, const void *buf, size_t sz, nni_opt_type t) +{ + rep0_sock *s = arg; + int ttl; + int rv; + + if ((rv = nni_copyin_int(&ttl, buf, sz, 1, NNI_MAX_MAX_TTL, t)) == 0) { + nni_atomic_set(&s->ttl, ttl); + } + return (rv); +} + +static int +rep0_sock_get_max_ttl(void *arg, void *buf, size_t *szp, nni_opt_type t) +{ + rep0_sock *s = arg; + + return (nni_copyout_int(nni_atomic_get(&s->ttl), buf, szp, t)); +} + +static int +rep0_sock_get_sendfd(void *arg, void *buf, size_t *szp, nni_opt_type t) +{ + rep0_sock *s = arg; + int rv; + int fd; + + if ((rv = nni_pollable_getfd(&s->writable, &fd)) != 0) { + return (rv); + } + return (nni_copyout_int(fd, buf, szp, t)); +} + +static int +rep0_sock_get_recvfd(void *arg, void *buf, size_t *szp, nni_opt_type t) +{ + rep0_sock *s = arg; + int rv; + int fd; + + if ((rv = nni_pollable_getfd(&s->readable, &fd)) != 0) { + return (rv); + } + + return (nni_copyout_int(fd, buf, szp, t)); +} + +static void +rep0_sock_send(void *arg, nni_aio *aio) +{ + rep0_sock *s = arg; + + rep0_ctx_send(&s->ctx, aio); +} + +static void +rep0_sock_recv(void *arg, nni_aio *aio) +{ + rep0_sock *s = arg; + + rep0_ctx_recv(&s->ctx, aio); +} + +// This is the global protocol structure -- our linkage to the core. +// This should be the only global non-static symbol in this file. +static nni_proto_pipe_ops rep0_pipe_ops = { + .pipe_size = sizeof(rep0_pipe), + .pipe_init = rep0_pipe_init, + .pipe_fini = rep0_pipe_fini, + .pipe_start = rep0_pipe_start, + .pipe_close = rep0_pipe_close, + .pipe_stop = rep0_pipe_stop, +}; + +static nni_proto_ctx_ops rep0_ctx_ops = { + .ctx_size = sizeof(rep0_ctx), + .ctx_init = rep0_ctx_init, + .ctx_fini = rep0_ctx_fini, + .ctx_send = rep0_ctx_send, + .ctx_recv = rep0_ctx_recv, +}; + +static nni_option rep0_sock_options[] = { + { + .o_name = NNG_OPT_MAXTTL, + .o_get = rep0_sock_get_max_ttl, + .o_set = rep0_sock_set_max_ttl, + }, + { + .o_name = NNG_OPT_RECVFD, + .o_get = rep0_sock_get_recvfd, + }, + { + .o_name = NNG_OPT_SENDFD, + .o_get = rep0_sock_get_sendfd, + }, + // terminate list + { + .o_name = NULL, + }, +}; + +static nni_proto_sock_ops rep0_sock_ops = { + .sock_size = sizeof(rep0_sock), + .sock_init = rep0_sock_init, + .sock_fini = rep0_sock_fini, + .sock_open = rep0_sock_open, + .sock_close = rep0_sock_close, + .sock_options = rep0_sock_options, + .sock_send = rep0_sock_send, + .sock_recv = rep0_sock_recv, +}; + +static nni_proto rep0_proto = { + .proto_version = NNI_PROTOCOL_VERSION, + .proto_self = { NNG_REP0_SELF, NNG_REP0_SELF_NAME }, + .proto_peer = { NNG_REP0_PEER, NNG_REP0_PEER_NAME }, + .proto_flags = NNI_PROTO_FLAG_SNDRCV, + .proto_sock_ops = &rep0_sock_ops, + .proto_pipe_ops = &rep0_pipe_ops, + .proto_ctx_ops = &rep0_ctx_ops, +}; + +int +nng_rep0_open(nng_socket *sidp) +{ + return (nni_proto_open(sidp, &rep0_proto)); +} diff --git a/src/sp/protocol/reqrep0/rep_test.c b/src/sp/protocol/reqrep0/rep_test.c new file mode 100644 index 00000000..5a47e67a --- /dev/null +++ b/src/sp/protocol/reqrep0/rep_test.c @@ -0,0 +1,669 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include + +static void +test_rep_identity(void) +{ + nng_socket s; + int p1, p2; + char * n1; + char * n2; + + NUTS_PASS(nng_rep0_open(&s)); + NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PROTO, &p1)); + NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PEER, &p2)); + NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PROTONAME, &n1)); + NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PEERNAME, &n2)); + NUTS_CLOSE(s); + NUTS_TRUE(p1 == NNG_REP0_SELF); + NUTS_TRUE(p2 == NNG_REP0_PEER); + NUTS_MATCH(n1, NNG_REP0_SELF_NAME); + NUTS_MATCH(n2, NNG_REP0_PEER_NAME); + nng_strfree(n1); + nng_strfree(n2); +} + +void +test_rep_send_bad_state(void) +{ + nng_socket rep; + nng_msg * msg = NULL; + + NUTS_TRUE(nng_rep0_open(&rep) == 0); + NUTS_TRUE(nng_msg_alloc(&msg, 0) == 0); + NUTS_TRUE(nng_sendmsg(rep, msg, 0) == NNG_ESTATE); + nng_msg_free(msg); + NUTS_CLOSE(rep); +} + +void +test_rep_poll_writeable(void) +{ + int fd; + nng_socket req; + nng_socket rep; + + NUTS_PASS(nng_req0_open(&req)); + NUTS_PASS(nng_rep0_open(&rep)); + NUTS_PASS(nng_socket_get_int(rep, NNG_OPT_SENDFD, &fd)); + NUTS_TRUE(fd >= 0); + + // Not writable before connect. + NUTS_TRUE(nuts_poll_fd(fd) == false); + + NUTS_MARRY(req, rep); + + // Still not writable. + NUTS_TRUE(nuts_poll_fd(fd) == false); + + // If we get a job, *then* we become writable + NUTS_SEND(req, "abc"); + NUTS_RECV(rep, "abc"); + NUTS_TRUE(nuts_poll_fd(fd) == true); + + // And is no longer writable once we send a message + NUTS_SEND(rep, "def"); + NUTS_TRUE(nuts_poll_fd(fd) == false); + // Even after receiving it + NUTS_RECV(req, "def"); + NUTS_TRUE(nuts_poll_fd(fd) == false); + + NUTS_CLOSE(req); + NUTS_CLOSE(rep); +} + +void +test_rep_poll_readable(void) +{ + int fd; + nng_socket req; + nng_socket rep; + nng_msg * msg; + + NUTS_PASS(nng_req0_open(&req)); + NUTS_PASS(nng_rep0_open(&rep)); + NUTS_PASS(nng_socket_get_int(rep, NNG_OPT_RECVFD, &fd)); + NUTS_TRUE(fd >= 0); + + // Not readable if not connected! + NUTS_TRUE(nuts_poll_fd(fd) == false); + + // Even after connect (no message yet) + NUTS_MARRY(req, rep); + NUTS_TRUE(nuts_poll_fd(fd) == false); + + // But once we send messages, it is. + // We have to send a request, in order to send a reply. + NUTS_SEND(req, "abc"); + NUTS_SLEEP(100); + + NUTS_TRUE(nuts_poll_fd(fd) == true); + + // and receiving makes it no longer ready + NUTS_PASS(nng_recvmsg(rep, &msg, 0)); + nng_msg_free(msg); + NUTS_TRUE(nuts_poll_fd(fd) == false); + + // TODO verify unsolicited response + + NUTS_CLOSE(req); + NUTS_CLOSE(rep); +} + +void +test_rep_context_no_poll(void) +{ + int fd; + nng_socket req; + nng_ctx ctx; + + NUTS_PASS(nng_rep0_open(&req)); + NUTS_PASS(nng_ctx_open(&ctx, req)); + NUTS_FAIL(nng_ctx_get_int(ctx, NNG_OPT_SENDFD, &fd), NNG_ENOTSUP); + NUTS_FAIL(nng_ctx_get_int(ctx, NNG_OPT_RECVFD, &fd), NNG_ENOTSUP); + NUTS_PASS(nng_ctx_close(ctx)); + NUTS_CLOSE(req); +} + +void +test_rep_validate_peer(void) +{ + nng_socket s1, s2; + nng_stat * stats; + nng_stat * reject; + char * addr; + + NUTS_ADDR(addr, "inproc"); + NUTS_PASS(nng_rep0_open(&s1)); + NUTS_PASS(nng_rep0_open(&s2)); + + NUTS_PASS(nng_listen(s1, addr, NULL, 0)); + NUTS_PASS(nng_dial(s2, addr, NULL, NNG_FLAG_NONBLOCK)); + + NUTS_SLEEP(100); + NUTS_PASS(nng_stats_get(&stats)); + + NUTS_TRUE(stats != NULL); + NUTS_TRUE((reject = nng_stat_find_socket(stats, s1)) != NULL); + NUTS_TRUE((reject = nng_stat_find(reject, "reject")) != NULL); + + NUTS_TRUE(nng_stat_type(reject) == NNG_STAT_COUNTER); + NUTS_TRUE(nng_stat_value(reject) > 0); + + NUTS_CLOSE(s1); + NUTS_CLOSE(s2); + nng_stats_free(stats); +} + +void +test_rep_double_recv(void) +{ + nng_socket s1; + nng_aio * aio1; + nng_aio * aio2; + + NUTS_PASS(nng_rep0_open(&s1)); + NUTS_PASS(nng_aio_alloc(&aio1, NULL, NULL)); + NUTS_PASS(nng_aio_alloc(&aio2, NULL, NULL)); + + nng_recv_aio(s1, aio1); + nng_recv_aio(s1, aio2); + + nng_aio_wait(aio2); + NUTS_FAIL(nng_aio_result(aio2), NNG_ESTATE); + NUTS_CLOSE(s1); + NUTS_FAIL(nng_aio_result(aio1), NNG_ECLOSED); + nng_aio_free(aio1); + nng_aio_free(aio2); +} + +void +test_rep_close_pipe_before_send(void) +{ + nng_socket rep; + nng_socket req; + nng_pipe p; + nng_aio * aio1; + nng_msg * m; + + NUTS_PASS(nng_rep0_open(&rep)); + NUTS_PASS(nng_req0_open(&req)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_aio_alloc(&aio1, NULL, NULL)); + + NUTS_MARRY(req, rep); + NUTS_SEND(req, "test"); + + nng_recv_aio(rep, aio1); + nng_aio_wait(aio1); + NUTS_PASS(nng_aio_result(aio1)); + NUTS_TRUE((m = nng_aio_get_msg(aio1)) != NULL); + p = nng_msg_get_pipe(m); + NUTS_PASS(nng_pipe_close(p)); + NUTS_PASS(nng_sendmsg(rep, m, 0)); + + NUTS_CLOSE(req); + NUTS_CLOSE(rep); + nng_aio_free(aio1); +} + +void +test_rep_close_pipe_during_send(void) +{ + nng_socket rep; + nng_socket req; + nng_pipe p = NNG_PIPE_INITIALIZER; + nng_msg * m; + + NUTS_PASS(nng_rep0_open(&rep)); + NUTS_PASS(nng_req0_open_raw(&req)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 200)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_int(rep, NNG_OPT_SENDBUF, 20)); + NUTS_PASS(nng_socket_set_int(rep, NNG_OPT_RECVBUF, 20)); + NUTS_PASS(nng_socket_set_int(req, NNG_OPT_SENDBUF, 20)); + NUTS_PASS(nng_socket_set_int(req, NNG_OPT_RECVBUF, 1)); + + NUTS_MARRY(req, rep); + + for (int i = 0; i < 100; i++) { + int rv; + NUTS_PASS(nng_msg_alloc(&m, 4)); + NUTS_PASS(nng_msg_append_u32(m, (unsigned) i | 0x80000000u)); + NUTS_PASS(nng_sendmsg(req, m, 0)); + NUTS_PASS(nng_recvmsg(rep, &m, 0)); + p = nng_msg_get_pipe(m); + rv = nng_sendmsg(rep, m, 0); + if (rv == NNG_ETIMEDOUT) { + // Queue is backed up, senders are busy. + nng_msg_free(m); + break; + } + NUTS_PASS(rv); + } + NUTS_PASS(nng_pipe_close(p)); + + NUTS_CLOSE(req); + NUTS_CLOSE(rep); +} + +void +test_rep_ctx_recv_aio_stopped(void) +{ + nng_socket rep; + nng_ctx ctx; + nng_aio * aio; + + NUTS_PASS(nng_rep0_open(&rep)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + NUTS_PASS(nng_ctx_open(&ctx, rep)); + + nng_aio_stop(aio); + nng_ctx_recv(ctx, aio); + nng_aio_wait(aio); + NUTS_FAIL(nng_aio_result(aio), NNG_ECANCELED); + NUTS_PASS(nng_ctx_close(ctx)); + NUTS_CLOSE(rep); + nng_aio_free(aio); +} + +void +test_rep_close_pipe_context_send(void) +{ + nng_socket rep; + nng_socket req; + nng_pipe p = NNG_PIPE_INITIALIZER; + nng_msg * m; + nng_ctx ctx[100]; + nng_aio * aio[100]; + int i; + + NUTS_PASS(nng_rep0_open(&rep)); + NUTS_PASS(nng_req0_open_raw(&req)); + for (i = 0; i < 100; i++) { + NUTS_PASS(nng_ctx_open(&ctx[i], rep)); + NUTS_PASS(nng_aio_alloc(&aio[i], NULL, NULL)); + } + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_int(rep, NNG_OPT_SENDBUF, 1)); + NUTS_PASS(nng_socket_set_int(rep, NNG_OPT_RECVBUF, 1)); + NUTS_PASS(nng_socket_set_int(req, NNG_OPT_SENDBUF, 1)); + NUTS_PASS(nng_socket_set_int(req, NNG_OPT_RECVBUF, 1)); + + NUTS_MARRY(req, rep); + + for (i = 0; i < 100; i++) { + NUTS_PASS(nng_msg_alloc(&m, 4)); + NUTS_PASS(nng_msg_append_u32(m, (unsigned) i | 0x80000000u)); + NUTS_PASS(nng_sendmsg(req, m, 0)); + nng_ctx_recv(ctx[i], aio[i]); + } + for (i = 0; i < 100; i++) { + nng_aio_wait(aio[i]); + NUTS_PASS(nng_aio_result(aio[i])); + NUTS_TRUE((m = nng_aio_get_msg(aio[i])) != NULL); + p = nng_msg_get_pipe(m); + nng_aio_set_msg(aio[i], m); + nng_ctx_send(ctx[i], aio[i]); + } + + // Note that REQ socket is not reading the results. + NUTS_PASS(nng_pipe_close(p)); + + for (i = 0; i < 100; i++) { + int rv; + nng_aio_wait(aio[i]); + rv = nng_aio_result(aio[i]); + if (rv != 0) { + NUTS_FAIL(rv, NNG_ECLOSED); + nng_msg_free(nng_aio_get_msg(aio[i])); + } + nng_aio_free(aio[i]); + NUTS_PASS(nng_ctx_close(ctx[i])); + } + NUTS_CLOSE(req); + NUTS_CLOSE(rep); +} + +void +test_rep_close_context_send(void) +{ + nng_socket rep; + nng_socket req; + nng_msg * m; + nng_ctx ctx[100]; + nng_aio * aio[100]; + int i; + + NUTS_PASS(nng_rep0_open(&rep)); + NUTS_PASS(nng_req0_open_raw(&req)); + for (i = 0; i < 100; i++) { + NUTS_PASS(nng_ctx_open(&ctx[i], rep)); + NUTS_PASS(nng_aio_alloc(&aio[i], NULL, NULL)); + } + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_int(rep, NNG_OPT_SENDBUF, 1)); + NUTS_PASS(nng_socket_set_int(rep, NNG_OPT_RECVBUF, 1)); + NUTS_PASS(nng_socket_set_int(req, NNG_OPT_SENDBUF, 1)); + NUTS_PASS(nng_socket_set_int(req, NNG_OPT_RECVBUF, 1)); + + NUTS_MARRY(req, rep); + + for (i = 0; i < 100; i++) { + NUTS_PASS(nng_msg_alloc(&m, 4)); + NUTS_PASS(nng_msg_append_u32(m, (unsigned) i | 0x80000000u)); + NUTS_PASS(nng_sendmsg(req, m, 0)); + nng_ctx_recv(ctx[i], aio[i]); + } + for (i = 0; i < 100; i++) { + nng_aio_wait(aio[i]); + NUTS_PASS(nng_aio_result(aio[i])); + NUTS_TRUE((m = nng_aio_get_msg(aio[i])) != NULL); + nng_aio_set_msg(aio[i], m); + nng_ctx_send(ctx[i], aio[i]); + } + + // Note that REQ socket is not reading the results. + for (i = 0; i < 100; i++) { + int rv; + NUTS_PASS(nng_ctx_close(ctx[i])); + nng_aio_wait(aio[i]); + rv = nng_aio_result(aio[i]); + if (rv != 0) { + NUTS_FAIL(rv, NNG_ECLOSED); + nng_msg_free(nng_aio_get_msg(aio[i])); + } + nng_aio_free(aio[i]); + } + NUTS_CLOSE(req); + NUTS_CLOSE(rep); +} + +void +test_rep_close_recv(void) +{ + nng_socket rep; + nng_socket req; + nng_aio * aio; + + NUTS_PASS(nng_rep0_open(&rep)); + NUTS_PASS(nng_req0_open_raw(&req)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); + + NUTS_MARRY(req, rep); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + nng_recv_aio(rep, aio); + NUTS_CLOSE(rep); + NUTS_CLOSE(req); + nng_aio_wait(aio); + NUTS_FAIL(nng_aio_result(aio), NNG_ECLOSED); + nng_aio_free(aio); +} + +struct rep_close_recv_cb_state { + nng_aio *aio; + nng_mtx *mtx; + nng_cv * cv; + int done; + int result; + nng_msg *msg; +}; + +static void +rep_close_recv_cb(void *arg) +{ + struct rep_close_recv_cb_state *state = arg; + + nng_mtx_lock(state->mtx); + state->result = nng_aio_result(state->aio); + state->msg = nng_aio_get_msg(state->aio); + state->done = true; + nng_cv_wake(state->cv); + nng_mtx_unlock(state->mtx); +} + +void +test_rep_close_recv_cb(void) +{ + nng_socket rep; + nng_socket req; + struct rep_close_recv_cb_state state; + + memset(&state, 0, sizeof(state)); + NUTS_PASS(nng_mtx_alloc(&state.mtx)); + NUTS_PASS(nng_cv_alloc(&state.cv, state.mtx)); + + NUTS_PASS(nng_rep0_open(&rep)); + NUTS_PASS(nng_req0_open_raw(&req)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); + + NUTS_MARRY(req, rep); + NUTS_PASS(nng_aio_alloc(&state.aio, rep_close_recv_cb, &state)); + nng_recv_aio(rep, state.aio); + NUTS_CLOSE(rep); + NUTS_CLOSE(req); + nng_mtx_lock(state.mtx); + while (!state.done) { + NUTS_PASS(nng_cv_until(state.cv, nng_clock() + 1000)); + } + nng_mtx_unlock(state.mtx); + NUTS_TRUE(state.done != 0); + NUTS_FAIL(nng_aio_result(state.aio), NNG_ECLOSED); + NUTS_TRUE(nng_aio_get_msg(state.aio) == NULL); + nng_aio_free(state.aio); + nng_cv_free(state.cv); + nng_mtx_free(state.mtx); +} + +static void +test_rep_ctx_recv_nonblock(void) +{ + nng_socket rep; + nng_ctx ctx; + nng_aio * aio; + + NUTS_PASS(nng_rep0_open(&rep)); + NUTS_PASS(nng_ctx_open(&ctx, rep)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + + nng_aio_set_timeout(aio, 0); // Instant timeout + nng_ctx_recv(ctx, aio); + + nng_aio_wait(aio); + NUTS_FAIL(nng_aio_result(aio), NNG_ETIMEDOUT); + NUTS_CLOSE(rep); + nng_aio_free(aio); +} + +static void +test_rep_ctx_send_nonblock(void) +{ + nng_socket rep; + nng_socket req; + nng_ctx ctx; + nng_aio * aio; + nng_msg * msg; + + NUTS_PASS(nng_req0_open(&req)); + NUTS_PASS(nng_rep0_open(&rep)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 2000)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_ctx_open(&ctx, rep)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + NUTS_MARRY(req, rep); + + NUTS_SEND(req, "SEND"); + nng_ctx_recv(ctx, aio); + nng_aio_wait(aio); + NUTS_PASS(nng_aio_result(aio)); + // message carries over + msg = nng_aio_get_msg(aio); + nng_aio_set_msg(aio, msg); + nng_aio_set_timeout(aio, 0); // Instant timeout + nng_ctx_send(ctx, aio); + + nng_aio_wait(aio); + NUTS_PASS(nng_aio_result(aio)); + NUTS_CLOSE(rep); + NUTS_CLOSE(req); + nng_aio_free(aio); +} + +static void +test_rep_ctx_send_nonblock2(void) +{ + nng_socket rep; + nng_socket req; + nng_ctx rep_ctx[10]; + nng_aio * rep_aio[10]; + int num_good = 0; + int num_fail = 0; + + // We are going to send a bunch of requests, receive them, + // but then see that non-block pressure exerts for some, but + // that at least one non-blocking send works. + NUTS_PASS(nng_req0_open_raw(&req)); + NUTS_PASS(nng_rep0_open(&rep)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 1000)); + for (int i = 0; i < 10; i++) { + NUTS_PASS(nng_ctx_open(&rep_ctx[i], rep)); + NUTS_PASS(nng_aio_alloc(&rep_aio[i], NULL, NULL)); + } + NUTS_MARRY(req, rep); + + for (int i = 0; i < 10; i++) { + nng_msg *msg; + NUTS_PASS(nng_msg_alloc(&msg, 4)); + NUTS_PASS(nng_msg_append_u32(msg, (unsigned) i | 0x80000000u)); + nng_ctx_recv(rep_ctx[i], rep_aio[i]); + NUTS_PASS(nng_sendmsg(req, msg, 0)); + } + for (int i = 0; i < 10; i++) { + nng_msg *msg; + nng_aio_wait(rep_aio[i]); + NUTS_PASS(nng_aio_result(rep_aio[i])); + msg = nng_aio_get_msg(rep_aio[i]); + nng_aio_set_timeout(rep_aio[i], 0); + nng_aio_set_msg(rep_aio[i], msg); + nng_ctx_send(rep_ctx[i], rep_aio[i]); + } + + for (int i = 0; i < 10; i++) { + int rv; + nng_aio_wait(rep_aio[i]); + rv = nng_aio_result(rep_aio[i]); + if (rv == 0) { + num_good++; + } else { + NUTS_FAIL(rv, NNG_ETIMEDOUT); + nng_msg_free(nng_aio_get_msg(rep_aio[i])); + num_fail++; + } + } + + TEST_ASSERT(num_good > 0); + TEST_ASSERT(num_fail > 0); + + for (int i = 0; i < 10; i++) { + nng_aio_free(rep_aio[i]); + nng_ctx_close(rep_ctx[i]); + } + NUTS_CLOSE(rep); + NUTS_CLOSE(req); +} + +static void +test_rep_send_nonblock(void) +{ + nng_socket rep; + nng_socket req; + int rv; + + NUTS_PASS(nng_req0_open(&req)); + NUTS_PASS(nng_rep0_open(&rep)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 1000)); + NUTS_MARRY(req, rep); + + NUTS_SEND(req, "SEND"); + NUTS_RECV(rep, "SEND"); + + // Use the nonblock flag + rv = nng_send(rep, "RECV", 5, NNG_FLAG_NONBLOCK); + + NUTS_PASS(rv); + NUTS_RECV(req, "RECV"); + NUTS_CLOSE(rep); + NUTS_CLOSE(req); +} + +void +test_rep_recv_garbage(void) +{ + nng_socket rep; + nng_socket req; + nng_msg * m; + + NUTS_PASS(nng_rep0_open(&rep)); + NUTS_PASS(nng_req0_open_raw(&req)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 200)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 200)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); + + NUTS_MARRY(req, rep); + + NUTS_PASS(nng_msg_alloc(&m, 4)); + NUTS_PASS(nng_msg_append_u32(m, 1u)); + NUTS_PASS(nng_sendmsg(req, m, 0)); + NUTS_FAIL(nng_recvmsg(rep, &m, 0), NNG_ETIMEDOUT); + + NUTS_CLOSE(req); + NUTS_CLOSE(rep); +} + +NUTS_TESTS = { + { "rep identity", test_rep_identity }, + { "rep send bad state", test_rep_send_bad_state }, + { "rep poll readable", test_rep_poll_readable }, + { "rep poll writable", test_rep_poll_writeable }, + { "rep context does not poll", test_rep_context_no_poll }, + { "rep validate peer", test_rep_validate_peer }, + { "rep double recv", test_rep_double_recv }, + { "rep send nonblock", test_rep_send_nonblock }, + { "rep close pipe before send", test_rep_close_pipe_before_send }, + { "rep close pipe during send", test_rep_close_pipe_during_send }, + { "rep recv aio ctx stopped", test_rep_ctx_recv_aio_stopped }, + { "rep close pipe context send", test_rep_close_pipe_context_send }, + { "rep close context send", test_rep_close_context_send }, + { "rep close recv", test_rep_close_recv }, + { "rep close recv cb", test_rep_close_recv_cb }, + { "rep context send nonblock", test_rep_ctx_send_nonblock }, + { "rep context send nonblock 2", test_rep_ctx_send_nonblock2 }, + { "rep context recv nonblock", test_rep_ctx_recv_nonblock }, + { "rep recv garbage", test_rep_recv_garbage }, + { NULL, NULL }, +}; diff --git a/src/sp/protocol/reqrep0/req.c b/src/sp/protocol/reqrep0/req.c new file mode 100644 index 00000000..cb3c9395 --- /dev/null +++ b/src/sp/protocol/reqrep0/req.c @@ -0,0 +1,869 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// Copyright 2018 Capitar IT Group BV +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// +#include + +#include "core/nng_impl.h" +#include "nng/protocol/reqrep0/req.h" + +// Request protocol. The REQ protocol is the "request" side of a +// request-reply pair. This is useful for building RPC clients, for example. + +typedef struct req0_pipe req0_pipe; +typedef struct req0_sock req0_sock; +typedef struct req0_ctx req0_ctx; + +static void req0_run_send_queue(req0_sock *, nni_list *); +static void req0_ctx_reset(req0_ctx *); +static void req0_ctx_timeout(void *); +static void req0_pipe_fini(void *); +static void req0_ctx_fini(void *); +static int req0_ctx_init(void *, void *); + +// A req0_ctx is a "context" for the request. It uses most of the +// socket, but keeps track of its own outstanding replays, the request ID, +// and so forth. +struct req0_ctx { + req0_sock * sock; + nni_list_node sock_node; // node on the socket context list + nni_list_node send_node; // node on the send_queue + nni_list_node pipe_node; // node on the pipe list + uint32_t request_id; // request ID, without high bit set + nni_aio * recv_aio; // user aio waiting to recv - only one! + nni_aio * send_aio; // user aio waiting to send + nng_msg * req_msg; // request message (owned by protocol) + size_t req_len; // length of request message (for stats) + nng_msg * rep_msg; // reply message + nni_timer_node timer; + nni_duration retry; + bool conn_reset; // sent message w/o retry, peer disconnect +}; + +// A req0_sock is our per-socket protocol private structure. +struct req0_sock { + nni_duration retry; + bool closed; + nni_atomic_int ttl; + req0_ctx master; // base socket master + nni_list ready_pipes; + nni_list busy_pipes; + nni_list stop_pipes; + nni_list contexts; + nni_list send_queue; // contexts waiting to send. + nni_id_map requests; // contexts by request ID + nni_pollable readable; + nni_pollable writable; + nni_mtx mtx; +}; + +// A req0_pipe is our per-pipe protocol private structure. +struct req0_pipe { + nni_pipe * pipe; + req0_sock * req; + nni_list_node node; + nni_list contexts; // contexts with pending traffic + bool closed; + nni_aio aio_send; + nni_aio aio_recv; +}; + +static void req0_sock_fini(void *); +static void req0_send_cb(void *); +static void req0_recv_cb(void *); + +static int +req0_sock_init(void *arg, nni_sock *sock) +{ + req0_sock *s = arg; + + NNI_ARG_UNUSED(sock); + + // Request IDs are 32 bits, with the high order bit set. + // We start at a random point, to minimize likelihood of + // accidental collision across restarts. + nni_id_map_init(&s->requests, 0x80000000u, 0xffffffffu, true); + + nni_mtx_init(&s->mtx); + + NNI_LIST_INIT(&s->ready_pipes, req0_pipe, node); + NNI_LIST_INIT(&s->busy_pipes, req0_pipe, node); + NNI_LIST_INIT(&s->stop_pipes, req0_pipe, node); + NNI_LIST_INIT(&s->send_queue, req0_ctx, send_node); + NNI_LIST_INIT(&s->contexts, req0_ctx, sock_node); + + // this is "semi random" start for request IDs. + s->retry = NNI_SECOND * 60; + + (void) req0_ctx_init(&s->master, s); + + nni_pollable_init(&s->writable); + nni_pollable_init(&s->readable); + + nni_atomic_init(&s->ttl); + nni_atomic_set(&s->ttl, 8); + return (0); +} + +static void +req0_sock_open(void *arg) +{ + NNI_ARG_UNUSED(arg); +} + +static void +req0_sock_close(void *arg) +{ + req0_sock *s = arg; + + nni_mtx_lock(&s->mtx); + s->closed = true; + nni_mtx_unlock(&s->mtx); +} + +static void +req0_sock_fini(void *arg) +{ + req0_sock *s = arg; + + nni_mtx_lock(&s->mtx); + NNI_ASSERT(nni_list_empty(&s->busy_pipes)); + NNI_ASSERT(nni_list_empty(&s->stop_pipes)); + NNI_ASSERT(nni_list_empty(&s->ready_pipes)); + nni_mtx_unlock(&s->mtx); + + req0_ctx_fini(&s->master); + nni_pollable_fini(&s->readable); + nni_pollable_fini(&s->writable); + nni_id_map_fini(&s->requests); + nni_mtx_fini(&s->mtx); +} + +static void +req0_pipe_stop(void *arg) +{ + req0_pipe *p = arg; + req0_sock *s = p->req; + + nni_aio_stop(&p->aio_recv); + nni_aio_stop(&p->aio_send); + nni_mtx_lock(&s->mtx); + nni_list_node_remove(&p->node); + nni_mtx_unlock(&s->mtx); +} + +static void +req0_pipe_fini(void *arg) +{ + req0_pipe *p = arg; + + nni_aio_fini(&p->aio_recv); + nni_aio_fini(&p->aio_send); +} + +static int +req0_pipe_init(void *arg, nni_pipe *pipe, void *s) +{ + req0_pipe *p = arg; + + nni_aio_init(&p->aio_recv, req0_recv_cb, p); + nni_aio_init(&p->aio_send, req0_send_cb, p); + NNI_LIST_NODE_INIT(&p->node); + NNI_LIST_INIT(&p->contexts, req0_ctx, pipe_node); + p->pipe = pipe; + p->req = s; + return (0); +} + +static int +req0_pipe_start(void *arg) +{ + req0_pipe *p = arg; + req0_sock *s = p->req; + + if (nni_pipe_peer(p->pipe) != NNG_REQ0_PEER) { + return (NNG_EPROTO); + } + + nni_mtx_lock(&s->mtx); + nni_list_append(&s->ready_pipes, p); + nni_pollable_raise(&s->writable); + req0_run_send_queue(s, NULL); + nni_mtx_unlock(&s->mtx); + + nni_pipe_recv(p->pipe, &p->aio_recv); + return (0); +} + +static void +req0_pipe_close(void *arg) +{ + req0_pipe *p = arg; + req0_sock *s = p->req; + req0_ctx * ctx; + + nni_aio_close(&p->aio_recv); + nni_aio_close(&p->aio_send); + + nni_mtx_lock(&s->mtx); + // This removes the node from either busy_pipes or ready_pipes. + // It doesn't much matter which. We stick the pipe on the stop + // list, so that we can wait for that to close down safely. + p->closed = true; + nni_list_node_remove(&p->node); + nni_list_append(&s->stop_pipes, p); + if (nni_list_empty(&s->ready_pipes)) { + nni_pollable_clear(&s->writable); + } + + while ((ctx = nni_list_first(&p->contexts)) != NULL) { + nni_list_remove(&p->contexts, ctx); + nng_aio *aio; + if (ctx->retry <= 0) { + // If we can't retry, then just cancel the operation + // altogether. We should only be waiting for recv, + // because we will already have sent if we are here. + if ((aio = ctx->recv_aio) != NULL) { + ctx->recv_aio = NULL; + nni_aio_finish_error(aio, NNG_ECONNRESET); + req0_ctx_reset(ctx); + } else { + req0_ctx_reset(ctx); + ctx->conn_reset = true; + } + } else { + // Reset the timer on this so it expires immediately. + // This is actually easier than canceling the timer and + // running the send_queue separately. (In particular, + // it avoids a potential deadlock on cancelling the + // timer.) + nni_timer_schedule(&ctx->timer, NNI_TIME_ZERO); + } + } + nni_mtx_unlock(&s->mtx); +} + +// For cooked mode, we use a context, and send out that way. This +// completely bypasses the upper write queue. Each context keeps one +// message pending; these are "scheduled" via the send_queue. The send_queue +// is ordered, so FIFO ordering between contexts is provided for. + +static void +req0_send_cb(void *arg) +{ + req0_pipe *p = arg; + req0_sock *s = p->req; + nni_aio * aio; + nni_list sent_list; + + nni_aio_list_init(&sent_list); + if (nni_aio_result(&p->aio_send) != 0) { + // We failed to send... clean up and deal with it. + nni_msg_free(nni_aio_get_msg(&p->aio_send)); + nni_aio_set_msg(&p->aio_send, NULL); + nni_pipe_close(p->pipe); + return; + } + + // We completed a cooked send, so we need to reinsert ourselves + // in the ready list, and re-run the send_queue. + + nni_mtx_lock(&s->mtx); + if (p->closed || s->closed) { + // This occurs if the req0_pipe_close has been called. + // In that case we don't want any more processing. + nni_mtx_unlock(&s->mtx); + return; + } + nni_list_remove(&s->busy_pipes, p); + nni_list_append(&s->ready_pipes, p); + if (nni_list_empty(&s->send_queue)) { + nni_pollable_raise(&s->writable); + } + req0_run_send_queue(s, &sent_list); + nni_mtx_unlock(&s->mtx); + + while ((aio = nni_list_first(&sent_list)) != NULL) { + nni_list_remove(&sent_list, aio); + nni_aio_finish_sync(aio, 0, 0); + } +} + +static void +req0_recv_cb(void *arg) +{ + req0_pipe *p = arg; + req0_sock *s = p->req; + req0_ctx * ctx; + nni_msg * msg; + nni_aio * aio; + uint32_t id; + + if (nni_aio_result(&p->aio_recv) != 0) { + nni_pipe_close(p->pipe); + return; + } + + msg = nni_aio_get_msg(&p->aio_recv); + nni_aio_set_msg(&p->aio_recv, NULL); + nni_msg_set_pipe(msg, nni_pipe_id(p->pipe)); + + // We yank 4 bytes from front of body, and move them to the header. + if (nni_msg_len(msg) < 4) { + // Malformed message. + goto malformed; + } + id = nni_msg_trim_u32(msg); + + // Schedule another receive while we are processing this. + nni_mtx_lock(&s->mtx); + + // NB: If close was called, then this will just abort. + nni_pipe_recv(p->pipe, &p->aio_recv); + + // Look for a context to receive it. + if (((ctx = nni_id_get(&s->requests, id)) == NULL) || + (ctx->send_aio != NULL) || (ctx->rep_msg != NULL)) { + nni_mtx_unlock(&s->mtx); + // No waiting context, we have not sent the request out to + // the wire yet, or context already has a reply ready. + // Discard the message. + nni_msg_free(msg); + return; + } + + // We have our match, so we can remove this. + nni_list_node_remove(&ctx->send_node); + nni_id_remove(&s->requests, id); + ctx->request_id = 0; + if (ctx->req_msg != NULL) { + nni_msg_free(ctx->req_msg); + ctx->req_msg = NULL; + } + + // Is there an aio waiting for us? + if ((aio = ctx->recv_aio) != NULL) { + ctx->recv_aio = NULL; + nni_mtx_unlock(&s->mtx); + nni_aio_set_msg(aio, msg); + nni_aio_finish_sync(aio, 0, nni_msg_len(msg)); + } else { + // No AIO, so stash msg. Receive will pick it up later. + ctx->rep_msg = msg; + if (ctx == &s->master) { + nni_pollable_raise(&s->readable); + } + nni_mtx_unlock(&s->mtx); + } + return; + +malformed: + nni_msg_free(msg); + nni_pipe_close(p->pipe); +} + +static void +req0_ctx_timeout(void *arg) +{ + req0_ctx * ctx = arg; + req0_sock *s = ctx->sock; + + nni_mtx_lock(&s->mtx); + if ((ctx->req_msg != NULL) && (!s->closed)) { + if (!nni_list_node_active(&ctx->send_node)) { + nni_list_append(&s->send_queue, ctx); + } + req0_run_send_queue(s, NULL); + } + nni_mtx_unlock(&s->mtx); +} + +static int +req0_ctx_init(void *arg, void *sock) +{ + req0_sock *s = sock; + req0_ctx * ctx = arg; + + nni_timer_init(&ctx->timer, req0_ctx_timeout, ctx); + + nni_mtx_lock(&s->mtx); + ctx->sock = s; + ctx->recv_aio = NULL; + ctx->retry = s->retry; + nni_list_append(&s->contexts, ctx); + nni_mtx_unlock(&s->mtx); + + return (0); +} + +static void +req0_ctx_fini(void *arg) +{ + req0_ctx * ctx = arg; + req0_sock *s = ctx->sock; + nni_aio * aio; + + nni_mtx_lock(&s->mtx); + if ((aio = ctx->recv_aio) != NULL) { + ctx->recv_aio = NULL; + nni_aio_finish_error(aio, NNG_ECLOSED); + } + if ((aio = ctx->send_aio) != NULL) { + ctx->send_aio = NULL; + nni_aio_set_msg(aio, ctx->req_msg); + ctx->req_msg = NULL; + nni_aio_finish_error(aio, NNG_ECLOSED); + } + req0_ctx_reset(ctx); + nni_list_remove(&s->contexts, ctx); + nni_mtx_unlock(&s->mtx); + + nni_timer_cancel(&ctx->timer); + nni_timer_fini(&ctx->timer); +} + +static int +req0_ctx_set_resend_time(void *arg, const void *buf, size_t sz, nni_opt_type t) +{ + req0_ctx *ctx = arg; + return (nni_copyin_ms(&ctx->retry, buf, sz, t)); +} + +static int +req0_ctx_get_resend_time(void *arg, void *buf, size_t *szp, nni_opt_type t) +{ + req0_ctx *ctx = arg; + return (nni_copyout_ms(ctx->retry, buf, szp, t)); +} + +static void +req0_run_send_queue(req0_sock *s, nni_list *sent_list) +{ + req0_ctx *ctx; + nni_aio * aio; + + // Note: This routine should be called with the socket lock held. + while ((ctx = nni_list_first(&s->send_queue)) != NULL) { + req0_pipe *p; + + if ((p = nni_list_first(&s->ready_pipes)) == NULL) { + return; + } + + // We have a place to send it, so do the send. + // If a sending error occurs that causes the message to + // be dropped, we rely on the resend timer to pick it up. + // We also notify the completion callback if this is the + // first send attempt. + nni_list_remove(&s->send_queue, ctx); + + // Schedule a resubmit timer. We only do this if we got + // a pipe to send to. Otherwise, we should get handled + // the next time that the send_queue is run. We don't do this + // if the retry is "disabled" with NNG_DURATION_INFINITE. + if (ctx->retry > 0) { + nni_timer_schedule( + &ctx->timer, nni_clock() + ctx->retry); + } + + // Put us on the pipe list of active contexts. + // This gives the pipe a chance to kick a resubmit + // if the pipe is removed. + nni_list_node_remove(&ctx->pipe_node); + nni_list_append(&p->contexts, ctx); + + nni_list_remove(&s->ready_pipes, p); + nni_list_append(&s->busy_pipes, p); + if (nni_list_empty(&s->ready_pipes)) { + nni_pollable_clear(&s->writable); + } + + if ((aio = ctx->send_aio) != NULL) { + ctx->send_aio = NULL; + nni_aio_bump_count(aio, ctx->req_len); + // If the list was passed in, we want to do a + // synchronous completion later. + if (sent_list != NULL) { + nni_list_append(sent_list, aio); + } else { + nni_aio_finish(aio, 0, 0); + } + } + + // At this point, we will never give this message back to + // to the user, so we don't have to worry about making it + // unique. We can freely clone it. + nni_msg_clone(ctx->req_msg); + nni_aio_set_msg(&p->aio_send, ctx->req_msg); + nni_pipe_send(p->pipe, &p->aio_send); + } +} + +void +req0_ctx_reset(req0_ctx *ctx) +{ + req0_sock *s = ctx->sock; + // Call with sock lock held! + + // We cannot safely "wait" using nni_timer_cancel, but this removes + // any scheduled timer activation. If the timeout is already running + // concurrently, it will still run. It should do nothing, because + // we toss the request. There is still a very narrow race if the + // timeout fires, but doesn't actually start running before we + // both finish this function, *and* manage to reschedule another + // request. The consequence of that occurring is that the request + // will be emitted on the wire twice. This is not actually tragic. + nni_timer_schedule(&ctx->timer, NNI_TIME_NEVER); + + nni_list_node_remove(&ctx->pipe_node); + nni_list_node_remove(&ctx->send_node); + if (ctx->request_id != 0) { + nni_id_remove(&s->requests, ctx->request_id); + ctx->request_id = 0; + } + if (ctx->req_msg != NULL) { + nni_msg_free(ctx->req_msg); + ctx->req_msg = NULL; + } + if (ctx->rep_msg != NULL) { + nni_msg_free(ctx->rep_msg); + ctx->rep_msg = NULL; + } + ctx->conn_reset = false; +} + +static void +req0_ctx_cancel_recv(nni_aio *aio, void *arg, int rv) +{ + req0_ctx * ctx = arg; + req0_sock *s = ctx->sock; + + nni_mtx_lock(&s->mtx); + if (ctx->recv_aio == aio) { + ctx->recv_aio = NULL; + + // Cancellation of a pending receive is treated as aborting the + // entire state machine. This allows us to preserve the + // semantic of exactly one receive operation per send + // operation, and should be the least surprising for users. The + // main consequence is that if a receive operation is completed + // (in error or otherwise), the user must submit a new send + // operation to restart the state machine. + req0_ctx_reset(ctx); + + nni_aio_finish_error(aio, rv); + } + nni_mtx_unlock(&s->mtx); +} + +static void +req0_ctx_recv(void *arg, nni_aio *aio) +{ + req0_ctx * ctx = arg; + req0_sock *s = ctx->sock; + nni_msg * msg; + + if (nni_aio_begin(aio) != 0) { + return; + } + nni_mtx_lock(&s->mtx); + if ((ctx->recv_aio != NULL) || + ((ctx->req_msg == NULL) && (ctx->rep_msg == NULL))) { + // We have already got a pending receive or have not + // tried to send a request yet. + // Either of these violate our basic state assumptions. + int rv; + if (ctx->conn_reset) { + ctx->conn_reset = false; + rv = NNG_ECONNRESET; + } else { + rv = NNG_ESTATE; + } + nni_mtx_unlock(&s->mtx); + nni_aio_finish_error(aio, rv); + return; + } + + if ((msg = ctx->rep_msg) == NULL) { + int rv; + rv = nni_aio_schedule(aio, req0_ctx_cancel_recv, ctx); + if (rv != 0) { + nni_mtx_unlock(&s->mtx); + nni_aio_finish_error(aio, rv); + return; + } + ctx->recv_aio = aio; + nni_mtx_unlock(&s->mtx); + return; + } + + ctx->rep_msg = NULL; + + // We have got a message to pass up, yay! + nni_aio_set_msg(aio, msg); + if (ctx == &s->master) { + nni_pollable_clear(&s->readable); + } + nni_mtx_unlock(&s->mtx); + nni_aio_finish(aio, 0, nni_msg_len(msg)); +} + +static void +req0_ctx_cancel_send(nni_aio *aio, void *arg, int rv) +{ + req0_ctx * ctx = arg; + req0_sock *s = ctx->sock; + + nni_mtx_lock(&s->mtx); + if (ctx->send_aio == aio) { + // There should not be a pending reply, because we canceled + // it while we were waiting. + NNI_ASSERT(ctx->recv_aio == NULL); + ctx->send_aio = NULL; + // Restore the message back to the aio. + nni_aio_set_msg(aio, ctx->req_msg); + nni_msg_header_clear(ctx->req_msg); + ctx->req_msg = NULL; + + // Cancellation of a pending receive is treated as aborting the + // entire state machine. This allows us to preserve the + // semantic of exactly one receive operation per send + // operation, and should be the least surprising for users. The + // main consequence is that if a receive operation is completed + // (in error or otherwise), the user must submit a new send + // operation to restart the state machine. + req0_ctx_reset(ctx); + + nni_aio_finish_error(aio, rv); + } + nni_mtx_unlock(&s->mtx); +} + +static void +req0_ctx_send(void *arg, nni_aio *aio) +{ + req0_ctx * ctx = arg; + req0_sock *s = ctx->sock; + nng_msg * msg = nni_aio_get_msg(aio); + int rv; + + if (nni_aio_begin(aio) != 0) { + return; + } + nni_mtx_lock(&s->mtx); + if (s->closed) { + nni_mtx_unlock(&s->mtx); + nni_aio_finish_error(aio, NNG_ECLOSED); + return; + } + // Sending a new request cancels the old one, including any + // outstanding reply. + if (ctx->recv_aio != NULL) { + nni_aio_finish_error(ctx->recv_aio, NNG_ECANCELED); + ctx->recv_aio = NULL; + } + if (ctx->send_aio != NULL) { + nni_aio_set_msg(ctx->send_aio, ctx->req_msg); + nni_msg_header_clear(ctx->req_msg); + ctx->req_msg = NULL; + nni_aio_finish_error(ctx->send_aio, NNG_ECANCELED); + ctx->send_aio = NULL; + nni_list_remove(&s->send_queue, ctx); + } + + // This resets the entire state machine. + req0_ctx_reset(ctx); + + // Insert us on the per ID hash list, so that receives can find us. + if ((rv = nni_id_alloc(&s->requests, &ctx->request_id, ctx)) != 0) { + nni_mtx_unlock(&s->mtx); + nni_aio_finish_error(aio, rv); + return; + } + nni_msg_header_clear(msg); + nni_msg_header_append_u32(msg, ctx->request_id); + + // If no pipes are ready, and the request was a poll (no background + // schedule), then fail it. Should be NNG_ETIMEDOUT. + rv = nni_aio_schedule(aio, req0_ctx_cancel_send, ctx); + if ((rv != 0) && (nni_list_empty(&s->ready_pipes))) { + nni_id_remove(&s->requests, ctx->request_id); + nni_mtx_unlock(&s->mtx); + nni_aio_finish_error(aio, rv); + return; + } + ctx->req_len = nni_msg_len(msg); + ctx->req_msg = msg; + ctx->send_aio = aio; + nni_aio_set_msg(aio, NULL); + + // Stick us on the send_queue list. + nni_list_append(&s->send_queue, ctx); + + req0_run_send_queue(s, NULL); + nni_mtx_unlock(&s->mtx); +} + +static void +req0_sock_send(void *arg, nni_aio *aio) +{ + req0_sock *s = arg; + req0_ctx_send(&s->master, aio); +} + +static void +req0_sock_recv(void *arg, nni_aio *aio) +{ + req0_sock *s = arg; + req0_ctx_recv(&s->master, aio); +} + +static int +req0_sock_set_max_ttl(void *arg, const void *buf, size_t sz, nni_opt_type t) +{ + req0_sock *s = arg; + int ttl; + int rv; + if ((rv = nni_copyin_int(&ttl, buf, sz, 1, NNI_MAX_MAX_TTL, t)) == 0) { + nni_atomic_set(&s->ttl, ttl); + } + return (rv); +} + +static int +req0_sock_get_max_ttl(void *arg, void *buf, size_t *szp, nni_opt_type t) +{ + req0_sock *s = arg; + return (nni_copyout_int(nni_atomic_get(&s->ttl), buf, szp, t)); +} + +static int +req0_sock_set_resend_time( + void *arg, const void *buf, size_t sz, nni_opt_type t) +{ + req0_sock *s = arg; + int rv; + rv = req0_ctx_set_resend_time(&s->master, buf, sz, t); + s->retry = s->master.retry; + return (rv); +} + +static int +req0_sock_get_resend_time(void *arg, void *buf, size_t *szp, nni_opt_type t) +{ + req0_sock *s = arg; + return (req0_ctx_get_resend_time(&s->master, buf, szp, t)); +} + +static int +req0_sock_get_send_fd(void *arg, void *buf, size_t *szp, nni_opt_type t) +{ + req0_sock *s = arg; + int rv; + int fd; + + if ((rv = nni_pollable_getfd(&s->writable, &fd)) != 0) { + return (rv); + } + return (nni_copyout_int(fd, buf, szp, t)); +} + +static int +req0_sock_get_recv_fd(void *arg, void *buf, size_t *szp, nni_opt_type t) +{ + req0_sock *s = arg; + int rv; + int fd; + + if ((rv = nni_pollable_getfd(&s->readable, &fd)) != 0) { + return (rv); + } + + return (nni_copyout_int(fd, buf, szp, t)); +} + +static nni_proto_pipe_ops req0_pipe_ops = { + .pipe_size = sizeof(req0_pipe), + .pipe_init = req0_pipe_init, + .pipe_fini = req0_pipe_fini, + .pipe_start = req0_pipe_start, + .pipe_close = req0_pipe_close, + .pipe_stop = req0_pipe_stop, +}; + +static nni_option req0_ctx_options[] = { + { + .o_name = NNG_OPT_REQ_RESENDTIME, + .o_get = req0_ctx_get_resend_time, + .o_set = req0_ctx_set_resend_time, + }, + { + .o_name = NULL, + }, +}; + +static nni_proto_ctx_ops req0_ctx_ops = { + .ctx_size = sizeof(req0_ctx), + .ctx_init = req0_ctx_init, + .ctx_fini = req0_ctx_fini, + .ctx_recv = req0_ctx_recv, + .ctx_send = req0_ctx_send, + .ctx_options = req0_ctx_options, +}; + +static nni_option req0_sock_options[] = { + { + .o_name = NNG_OPT_MAXTTL, + .o_get = req0_sock_get_max_ttl, + .o_set = req0_sock_set_max_ttl, + }, + { + .o_name = NNG_OPT_REQ_RESENDTIME, + .o_get = req0_sock_get_resend_time, + .o_set = req0_sock_set_resend_time, + }, + { + .o_name = NNG_OPT_RECVFD, + .o_get = req0_sock_get_recv_fd, + }, + { + .o_name = NNG_OPT_SENDFD, + .o_get = req0_sock_get_send_fd, + }, + // terminate list + { + .o_name = NULL, + }, +}; + +static nni_proto_sock_ops req0_sock_ops = { + .sock_size = sizeof(req0_sock), + .sock_init = req0_sock_init, + .sock_fini = req0_sock_fini, + .sock_open = req0_sock_open, + .sock_close = req0_sock_close, + .sock_options = req0_sock_options, + .sock_send = req0_sock_send, + .sock_recv = req0_sock_recv, +}; + +static nni_proto req0_proto = { + .proto_version = NNI_PROTOCOL_VERSION, + .proto_self = { NNG_REQ0_SELF, NNG_REQ0_SELF_NAME }, + .proto_peer = { NNG_REQ0_PEER, NNG_REQ0_PEER_NAME }, + .proto_flags = NNI_PROTO_FLAG_SNDRCV, + .proto_sock_ops = &req0_sock_ops, + .proto_pipe_ops = &req0_pipe_ops, + .proto_ctx_ops = &req0_ctx_ops, +}; + +int +nng_req0_open(nng_socket *sock) +{ + return (nni_proto_open(sock, &req0_proto)); +} diff --git a/src/sp/protocol/reqrep0/req_test.c b/src/sp/protocol/reqrep0/req_test.c new file mode 100644 index 00000000..fb78efa0 --- /dev/null +++ b/src/sp/protocol/reqrep0/req_test.c @@ -0,0 +1,968 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// Copyright 2018 Capitar IT Group BV +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include + +static void +test_req_identity(void) +{ + nng_socket s; + int p; + char * n; + + NUTS_PASS(nng_req0_open(&s)); + NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PROTO, &p)); + NUTS_TRUE(p == NNG_REQ0_SELF); + NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PEER, &p)); + NUTS_TRUE(p == NNG_REQ0_PEER); // 49 + NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PROTONAME, &n)); + NUTS_MATCH(n, NNG_REQ0_SELF_NAME); + nng_strfree(n); + NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PEERNAME, &n)); + NUTS_MATCH(n, NNG_REQ0_PEER_NAME); + nng_strfree(n); + NUTS_CLOSE(s); +} + +static void +test_req_ttl_option(void) +{ + nng_socket req; + int v; + bool b; + size_t sz; + const char *opt = NNG_OPT_MAXTTL; + + NUTS_PASS(nng_req0_open(&req)); + + NUTS_PASS(nng_socket_set_int(req, opt, 1)); + NUTS_FAIL(nng_socket_set_int(req, opt, 0), NNG_EINVAL); + NUTS_FAIL(nng_socket_set_int(req, opt, -1), NNG_EINVAL); + // This test will fail if the NNI_MAX_MAX_TTL is changed from the + // builtin default of 15. + NUTS_FAIL(nng_socket_set_int(req, opt, 16), NNG_EINVAL); + NUTS_FAIL(nng_socket_set_int(req, opt, 256), NNG_EINVAL); + NUTS_PASS(nng_socket_set_int(req, opt, 3)); + NUTS_PASS(nng_socket_get_int(req, opt, &v)); + NUTS_TRUE(v == 3); + v = 0; + sz = sizeof(v); + NUTS_PASS(nng_socket_get(req, opt, &v, &sz)); + NUTS_TRUE(v == 3); + NUTS_TRUE(sz == sizeof(v)); + + NUTS_FAIL(nng_socket_set(req, opt, "", 1), NNG_EINVAL); + sz = 1; + NUTS_FAIL(nng_socket_get(req, opt, &v, &sz), NNG_EINVAL); + NUTS_FAIL(nng_socket_set_bool(req, opt, true), NNG_EBADTYPE); + NUTS_FAIL(nng_socket_get_bool(req, opt, &b), NNG_EBADTYPE); + + NUTS_CLOSE(req); +} + +static void +test_req_resend_option(void) +{ + nng_socket req; + nng_duration d; + bool b; + size_t sz = sizeof(b); + const char * opt = NNG_OPT_REQ_RESENDTIME; + + NUTS_PASS(nng_req0_open(&req)); + + NUTS_TRUE(nng_socket_set_ms(req, opt, 10) == 0); + NUTS_FAIL(nng_socket_set(req, opt, "", 1), NNG_EINVAL); + NUTS_FAIL(nng_socket_get(req, opt, &b, &sz), NNG_EINVAL); + NUTS_FAIL(nng_socket_set_bool(req, opt, true), NNG_EBADTYPE); + NUTS_FAIL(nng_socket_get_bool(req, opt, &b), NNG_EBADTYPE); + + NUTS_PASS(nng_socket_get_ms(req, opt, &d)); + NUTS_TRUE(d == 10); + NUTS_CLOSE(req); +} + +void +test_req_recv_bad_state(void) +{ + nng_socket req; + nng_msg * msg = NULL; + + NUTS_TRUE(nng_req0_open(&req) == 0); + NUTS_TRUE(nng_recvmsg(req, &msg, 0) == NNG_ESTATE); + NUTS_NULL(msg); + NUTS_CLOSE(req); +} + +static void +test_req_recv_garbage(void) +{ + nng_socket rep; + nng_socket req; + nng_msg * m; + uint32_t req_id; + + NUTS_PASS(nng_rep0_open_raw(&rep)); + NUTS_PASS(nng_req0_open(&req)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_RECVTIMEO, 100)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 1000)); + + NUTS_MARRY(req, rep); + + NUTS_PASS(nng_msg_alloc(&m, 0)); + NUTS_PASS(nng_sendmsg(req, m, 0)); + + NUTS_PASS(nng_recvmsg(rep, &m, 0)); + + // The message will have a header that contains the 32-bit pipe ID, + // followed by the 32-bit request ID. We will discard the request + // ID before sending it out. + NUTS_TRUE(nng_msg_header_len(m) == 8); + NUTS_PASS(nng_msg_header_chop_u32(m, &req_id)); + + NUTS_PASS(nng_sendmsg(rep, m, 0)); + NUTS_FAIL(nng_recvmsg(req, &m, 0), NNG_ETIMEDOUT); + + NUTS_CLOSE(req); + NUTS_CLOSE(rep); +} + +#define SECOND 1000 + +void +test_req_rep_exchange(void) +{ + nng_socket req; + nng_socket rep; + + NUTS_TRUE(nng_req0_open(&req) == 0); + NUTS_TRUE(nng_rep0_open(&rep) == 0); + + NUTS_TRUE(nng_socket_set_ms(req, NNG_OPT_RECVTIMEO, SECOND) == 0); + NUTS_TRUE(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, SECOND) == 0); + NUTS_TRUE(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, SECOND) == 0); + NUTS_TRUE(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, SECOND) == 0); + + NUTS_MARRY(rep, req); + + NUTS_SEND(req, "ping"); + NUTS_RECV(rep, "ping"); + NUTS_SEND(rep, "pong"); + NUTS_RECV(req, "pong"); + + NUTS_CLOSE(req); + NUTS_CLOSE(rep); +} + +void +test_req_resend(void) +{ + nng_socket req; + nng_socket rep; + + NUTS_PASS(nng_req0_open(&req)); + NUTS_PASS(nng_rep0_open(&rep)); + + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_RECVTIMEO, SECOND)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, SECOND)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, SECOND)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, SECOND)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_REQ_RESENDTIME, 10)); + + NUTS_MARRY(rep, req); + + NUTS_SEND(req, "ping"); + NUTS_RECV(rep, "ping"); + NUTS_RECV(rep, "ping"); + NUTS_RECV(rep, "ping"); + + NUTS_CLOSE(req); + NUTS_CLOSE(rep); +} + +void +test_req_resend_reconnect(void) +{ + nng_socket req; + nng_socket rep1; + nng_socket rep2; + + NUTS_PASS(nng_req0_open(&req)); + NUTS_PASS(nng_rep0_open(&rep1)); + NUTS_PASS(nng_rep0_open(&rep2)); + + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_RECVTIMEO, SECOND)); + NUTS_PASS(nng_socket_set_ms(rep1, NNG_OPT_RECVTIMEO, SECOND)); + NUTS_PASS(nng_socket_set_ms(rep2, NNG_OPT_RECVTIMEO, SECOND)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, SECOND)); + NUTS_PASS(nng_socket_set_ms(rep1, NNG_OPT_SENDTIMEO, SECOND)); + NUTS_PASS(nng_socket_set_ms(rep2, NNG_OPT_SENDTIMEO, SECOND)); + // We intentionally set the retry time long; that way we only see + // the retry from loss of our original peer. + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_REQ_RESENDTIME, 60 * SECOND)); + + NUTS_MARRY(rep1, req); + + NUTS_SEND(req, "ping"); + NUTS_RECV(rep1, "ping"); + + NUTS_CLOSE(rep1); + NUTS_MARRY(rep2, req); + + NUTS_RECV(rep2, "ping"); + NUTS_SEND(rep2, "rep2"); + NUTS_RECV(req, "rep2"); + + NUTS_CLOSE(req); + NUTS_CLOSE(rep2); +} + +void +test_req_resend_disconnect(void) +{ + nng_socket req; + nng_socket rep1; + nng_socket rep2; + + NUTS_PASS(nng_req0_open(&req)); + NUTS_PASS(nng_rep0_open(&rep1)); + NUTS_PASS(nng_rep0_open(&rep2)); + + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_RECVTIMEO, SECOND)); + NUTS_PASS(nng_socket_set_ms(rep1, NNG_OPT_RECVTIMEO, SECOND)); + NUTS_PASS(nng_socket_set_ms(rep2, NNG_OPT_RECVTIMEO, SECOND)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, SECOND)); + NUTS_PASS(nng_socket_set_ms(rep1, NNG_OPT_SENDTIMEO, SECOND)); + NUTS_PASS(nng_socket_set_ms(rep2, NNG_OPT_SENDTIMEO, SECOND)); + // We intentionally set the retry time long; that way we only see + // the retry from loss of our original peer. + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_REQ_RESENDTIME, 60 * SECOND)); + + NUTS_MARRY(rep1, req); + NUTS_SEND(req, "ping"); + NUTS_RECV(rep1, "ping"); + + NUTS_MARRY(rep2, req); + NUTS_CLOSE(rep1); + + NUTS_RECV(rep2, "ping"); + NUTS_SEND(rep2, "rep2"); + NUTS_RECV(req, "rep2"); + + NUTS_CLOSE(req); + NUTS_CLOSE(rep2); +} + +void +test_req_disconnect_no_retry(void) +{ + nng_socket req; + nng_socket rep1; + nng_socket rep2; + + NUTS_PASS(nng_req0_open(&req)); + NUTS_PASS(nng_rep0_open(&rep1)); + NUTS_PASS(nng_rep0_open(&rep2)); + + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_RECVTIMEO, SECOND)); + NUTS_PASS(nng_socket_set_ms(rep1, NNG_OPT_RECVTIMEO, SECOND)); + NUTS_PASS(nng_socket_set_ms(rep2, NNG_OPT_RECVTIMEO, SECOND)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, SECOND)); + NUTS_PASS(nng_socket_set_ms(rep1, NNG_OPT_SENDTIMEO, SECOND / 10)); + // Setting the resend time to zero so we will force an error + // if the peer disconnects without sending us an answer. + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_REQ_RESENDTIME, 0)); + + NUTS_MARRY(rep1, req); + NUTS_SEND(req, "ping"); + NUTS_RECV(rep1, "ping"); + + NUTS_MARRY(rep2, req); + NUTS_CLOSE(rep1); + + nng_msg *msg = NULL; + NUTS_FAIL(nng_recvmsg(req, &msg, 0), NNG_ECONNRESET); + NUTS_FAIL(nng_recvmsg(rep2, &msg, 0), NNG_ETIMEDOUT); + + NUTS_CLOSE(req); + NUTS_CLOSE(rep2); +} + +void +test_req_disconnect_abort(void) +{ + nng_socket req; + nng_socket rep1; + nng_socket rep2; + nng_aio * aio; + + NUTS_PASS(nng_req0_open(&req)); + NUTS_PASS(nng_rep0_open(&rep1)); + NUTS_PASS(nng_rep0_open(&rep2)); + NUTS_PASS(nng_aio_alloc(&aio, 0, 0)); + + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_RECVTIMEO, SECOND)); + NUTS_PASS(nng_socket_set_ms(rep1, NNG_OPT_RECVTIMEO, SECOND)); + NUTS_PASS(nng_socket_set_ms(rep2, NNG_OPT_RECVTIMEO, SECOND)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, SECOND)); + NUTS_PASS(nng_socket_set_ms(rep1, NNG_OPT_SENDTIMEO, SECOND / 10)); + // Setting the resend time to zero so we will force an error + // if the peer disconnects without sending us an answer. + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_REQ_RESENDTIME, 0)); + + NUTS_MARRY(rep1, req); + NUTS_SEND(req, "ping"); + NUTS_RECV(rep1, "ping"); + nng_recv_aio(req, aio); + + NUTS_MARRY(rep2, req); + NUTS_CLOSE(rep1); + + nng_msg *msg = NULL; + nng_aio_wait(aio); + NUTS_FAIL(nng_aio_result(aio), NNG_ECONNRESET); + NUTS_FAIL(nng_recvmsg(rep2, &msg, 0), NNG_ETIMEDOUT); + nng_aio_free(aio); + + NUTS_CLOSE(req); + NUTS_CLOSE(rep2); +} + +void +test_req_cancel(void) +{ + nng_duration retry = SECOND; + nng_socket req; + nng_socket rep; + + NUTS_PASS(nng_rep_open(&rep)); + NUTS_PASS(nng_req_open(&req)); + + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_RECVTIMEO, SECOND)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, SECOND)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 5 * SECOND)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 5 * SECOND)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_REQ_RESENDTIME, retry)); + NUTS_PASS(nng_socket_set_int(req, NNG_OPT_SENDBUF, 16)); + + NUTS_MARRY(rep, req); + + // Send req #1 (abc). + NUTS_SEND(req, "abc"); + + // Sleep a bit. This is so that we ensure that our request gets + // to the far side. (If we cancel too fast, then our outgoing send + // will be canceled before it gets to the peer.) + NUTS_SLEEP(100); + + // Send the next next request ("def"). Note that + // the REP side server will have already buffered the receive + // request, and should simply be waiting for us to reply to abc. + NUTS_SEND(req, "def"); + + // Receive the first request (should be abc) on the REP server. + NUTS_RECV(rep, "abc"); + + // REP sends the reply to first command. This will be discarded + // by the REQ socket. + NUTS_SEND(rep, "abc"); + + // Now get the next command from the REP; should be "def". + NUTS_RECV(rep, "def"); + + // And send it back to REQ. + NUTS_SEND(rep, "def"); + + // And we got back only the second result. + NUTS_RECV(req, "def"); + + NUTS_CLOSE(req); + NUTS_CLOSE(rep); +} + +void +test_req_cancel_abort_recv(void) +{ + nng_aio * aio; + nng_duration retry = SECOND * 10; // 10s (kind of never) + nng_socket req; + nng_socket rep; + + NUTS_PASS(nng_rep_open(&rep)); + NUTS_PASS(nng_req_open(&req)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_REQ_RESENDTIME, retry)); + NUTS_PASS(nng_socket_set_int(req, NNG_OPT_SENDBUF, 16)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_RECVTIMEO, 5 * SECOND)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 5 * SECOND)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 5 * SECOND)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 5 * SECOND)); + + NUTS_MARRY(rep, req); + + // Send req #1 (abc). + NUTS_SEND(req, "abc"); + + // Wait for it to get ot the other side. + NUTS_SLEEP(100); + + nng_aio_set_timeout(aio, 5 * SECOND); + nng_recv_aio(req, aio); + + // Give time for this recv to post properly. + NUTS_SLEEP(100); + + // Send the next next request ("def"). Note that + // the REP side server will have already buffered the receive + // request, and should simply be waiting for us to reply to + // abc. + NUTS_SEND(req, "def"); + + // Our pending I/O should have been canceled. + nng_aio_wait(aio); + NUTS_FAIL(nng_aio_result(aio), NNG_ECANCELED); + + // Receive the first request (should be abc) on the REP server. + NUTS_RECV(rep, "abc"); + + // REP sends the reply to first command. This will be + // discarded by the REQ socket. + NUTS_SEND(rep, "abc"); + + // Now get the next command from the REP; should be "def". + NUTS_RECV(rep, "def"); + + // And send it back to REQ. + NUTS_SEND(rep, "def"); + + // Try a req command. This should give back "def" + NUTS_RECV(req, "def"); + + nng_aio_free(aio); + NUTS_CLOSE(req); + NUTS_CLOSE(rep); +} + +static void +test_req_cancel_post_recv(void) +{ + nng_socket req; + nng_socket rep; + + NUTS_PASS(nng_req0_open(&req)); + NUTS_PASS(nng_rep0_open(&rep)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 1000)); + NUTS_MARRY(req, rep); + + NUTS_SEND(req, "ONE"); + NUTS_RECV(rep, "ONE"); + NUTS_SEND(rep, "one"); + NUTS_SLEEP(100); // Make sure reply arrives! + NUTS_SEND(req, "TWO"); + NUTS_RECV(rep, "TWO"); + NUTS_SEND(rep, "two"); + NUTS_RECV(req, "two"); + + NUTS_CLOSE(req); + NUTS_CLOSE(rep); +} + +void +test_req_poll_writeable(void) +{ + int fd; + nng_socket req; + nng_socket rep; + + NUTS_PASS(nng_req0_open(&req)); + NUTS_PASS(nng_rep0_open(&rep)); + NUTS_PASS(nng_socket_get_int(req, NNG_OPT_SENDFD, &fd)); + NUTS_TRUE(fd >= 0); + + // Not writable before connect. + NUTS_TRUE(nuts_poll_fd(fd) == false); + + NUTS_MARRY(req, rep); + + // It should be writable now. + NUTS_TRUE(nuts_poll_fd(fd)); + + // Submit a bunch of jobs. Note that we have to stall a bit + // between each message to let it queue up. + for (int i = 0; i < 10; i++) { + int rv = nng_send(req, "", 0, NNG_FLAG_NONBLOCK); + if (rv == NNG_EAGAIN) { + break; + } + NUTS_PASS(rv); + NUTS_SLEEP(50); + } + NUTS_TRUE(nuts_poll_fd(fd) == false); + NUTS_CLOSE(req); + NUTS_CLOSE(rep); +} + +void +test_req_poll_contention(void) +{ + int fd; + nng_socket req; + nng_socket rep; + nng_aio * aio; + nng_ctx ctx[5]; + nng_aio * ctx_aio[5]; + nng_msg * ctx_msg[5]; + nng_msg * msg; + + NUTS_PASS(nng_req0_open(&req)); + NUTS_PASS(nng_rep0_open(&rep)); + NUTS_PASS(nng_socket_set_int(req, NNG_OPT_SENDBUF, 1)); + NUTS_PASS(nng_socket_set_int(rep, NNG_OPT_RECVBUF, 1)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 1000)); + + for (int i = 0; i < 5; i++) { + NUTS_PASS(nng_ctx_open(&ctx[i], req)); + NUTS_PASS(nng_aio_alloc(&ctx_aio[i], NULL, NULL)); + NUTS_PASS(nng_msg_alloc(&ctx_msg[i], 0)); + } + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + NUTS_PASS(nng_msg_alloc(&msg, 0)); + + NUTS_PASS(nng_socket_get_int(req, NNG_OPT_SENDFD, &fd)); + NUTS_TRUE(fd >= 0); + + // Not writable before connect. + NUTS_TRUE(nuts_poll_fd(fd) == false); + + nng_aio_set_msg(aio, msg); + nng_send_aio(req, aio); + for (int i = 0; i < 5; i++) { + nng_aio_set_msg(ctx_aio[i], ctx_msg[i]); + nng_ctx_send(ctx[i], ctx_aio[i]); + } + NUTS_SLEEP(50); // so everything is queued steady state + + NUTS_MARRY(req, rep); + + // It should not be writable now. + NUTS_TRUE(nuts_poll_fd(fd) == false); + + NUTS_PASS(nng_recvmsg(rep, &msg, 0)); + nng_msg_free(msg); + + // Still not writeable... + NUTS_TRUE(nuts_poll_fd(fd) == false); + for (int i = 0; i < 5; i++) { + NUTS_PASS(nng_recvmsg(rep, &msg, 0)); + nng_msg_free(msg); + } + // It can take a little bit of time for the eased back-pressure + // to reflect across the network. + NUTS_SLEEP(100); + + // Should be come writeable now... + NUTS_TRUE(nuts_poll_fd(fd) == true); + + for (int i = 0; i < 5; i++) { + nng_aio_free(ctx_aio[i]); + } + nng_aio_free(aio); + NUTS_CLOSE(req); + NUTS_CLOSE(rep); +} + +void +test_req_poll_multi_pipe(void) +{ + int fd; + nng_socket req; + nng_socket rep1; + nng_socket rep2; + + NUTS_PASS(nng_req0_open(&req)); + NUTS_PASS(nng_rep0_open(&rep1)); + NUTS_PASS(nng_rep0_open(&rep2)); + NUTS_PASS(nng_socket_set_int(req, NNG_OPT_SENDBUF, 1)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); + + NUTS_PASS(nng_socket_get_int(req, NNG_OPT_SENDFD, &fd)); + NUTS_TRUE(fd >= 0); + + // Not writable before connect. + NUTS_TRUE(nuts_poll_fd(fd) == false); + + NUTS_MARRY(req, rep1); + NUTS_MARRY(req, rep2); + + NUTS_TRUE(nuts_poll_fd(fd) == true); + NUTS_SEND(req, "ONE"); + NUTS_TRUE(nuts_poll_fd(fd) == true); + + NUTS_CLOSE(req); + NUTS_CLOSE(rep1); + NUTS_CLOSE(rep2); +} + +void +test_req_poll_readable(void) +{ + int fd; + nng_socket req; + nng_socket rep; + nng_msg * msg; + + NUTS_PASS(nng_req0_open(&req)); + NUTS_PASS(nng_rep0_open(&rep)); + NUTS_PASS(nng_socket_get_int(req, NNG_OPT_RECVFD, &fd)); + NUTS_TRUE(fd >= 0); + + // Not readable if not connected! + NUTS_TRUE(nuts_poll_fd(fd) == false); + + // Even after connect (no message yet) + NUTS_MARRY(req, rep); + NUTS_TRUE(nuts_poll_fd(fd) == false); + + // But once we send messages, it is. + // We have to send a request, in order to send a reply. + + NUTS_PASS(nng_msg_alloc(&msg, 0)); + NUTS_PASS(nng_msg_append(msg, "xyz", 3)); + NUTS_PASS(nng_sendmsg(req, msg, 0)); + NUTS_PASS(nng_recvmsg(rep, &msg, 0)); // recv on rep + NUTS_PASS(nng_sendmsg(rep, msg, 0)); // echo it back + NUTS_SLEEP(200); // give time for message to arrive + + NUTS_TRUE(nuts_poll_fd(fd) == true); + + // and receiving makes it no longer ready + NUTS_PASS(nng_recvmsg(req, &msg, 0)); + nng_msg_free(msg); + NUTS_TRUE(nuts_poll_fd(fd) == false); + + // TODO verify unsolicited response + + NUTS_CLOSE(req); + NUTS_CLOSE(rep); +} + +static void +test_req_ctx_no_poll(void) +{ + int fd; + nng_socket req; + nng_ctx ctx; + + NUTS_PASS(nng_req0_open(&req)); + NUTS_PASS(nng_ctx_open(&ctx, req)); + NUTS_FAIL(nng_ctx_getopt_int(ctx, NNG_OPT_SENDFD, &fd), NNG_ENOTSUP); + NUTS_FAIL(nng_ctx_getopt_int(ctx, NNG_OPT_RECVFD, &fd), NNG_ENOTSUP); + NUTS_PASS(nng_ctx_close(ctx)); + NUTS_CLOSE(req); +} + +static void +test_req_ctx_send_queued(void) +{ + nng_socket req; + nng_socket rep; + nng_ctx ctx[3]; + nng_aio * aio[3]; + nng_msg * msg[3]; + + NUTS_PASS(nng_req0_open(&req)); + NUTS_PASS(nng_rep0_open(&rep)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 100)); + + for (int i = 0; i < 3; i++) { + NUTS_PASS(nng_ctx_open(&ctx[i], req)); + NUTS_PASS(nng_aio_alloc(&aio[i], NULL, NULL)); + NUTS_PASS(nng_msg_alloc(&msg[i], 0)); + } + + for (int i = 0; i < 3; i++) { + nng_aio_set_msg(aio[i], msg[i]); + nng_ctx_send(ctx[i], aio[i]); + } + + NUTS_MARRY(req, rep); + + NUTS_SLEEP(50); // Only to ensure stuff queues up + for (int i = 0; i < 3; i++) { + nng_msg *m; + NUTS_PASS(nng_recvmsg(rep, &m, 0)); + nng_msg_free(m); + } + + NUTS_CLOSE(req); + NUTS_CLOSE(rep); + for (int i = 0; i < 3; i++) { + nng_aio_wait(aio[i]); + NUTS_PASS(nng_aio_result(aio[i])); + nng_aio_free(aio[i]); + } +} + +static void +test_req_ctx_send_close(void) +{ + nng_socket req; + nng_ctx ctx[3]; + nng_aio * aio[3]; + nng_msg * msg[3]; + + NUTS_PASS(nng_req0_open(&req)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); + + for (int i = 0; i < 3; i++) { + NUTS_PASS(nng_ctx_open(&ctx[i], req)); + NUTS_PASS(nng_aio_alloc(&aio[i], NULL, NULL)); + NUTS_PASS(nng_msg_alloc(&msg[i], 0)); + } + + for (int i = 0; i < 3; i++) { + nng_aio_set_msg(aio[i], msg[i]); + nng_ctx_send(ctx[i], aio[i]); + } + + for (int i = 0; i < 3; i++) { + nng_ctx_close(ctx[i]); + } + + for (int i = 0; i < 3; i++) { + nng_aio_wait(aio[i]); + NUTS_FAIL(nng_aio_result(aio[i]), NNG_ECLOSED); + nng_aio_free(aio[i]); + nng_msg_free(msg[i]); + } + NUTS_CLOSE(req); +} + +static void +test_req_ctx_send_abort(void) +{ + nng_socket req; + nng_ctx ctx[3]; + nng_aio * aio[3]; + nng_msg * msg[3]; + + NUTS_PASS(nng_req0_open(&req)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); + + for (int i = 0; i < 3; i++) { + NUTS_PASS(nng_ctx_open(&ctx[i], req)); + NUTS_PASS(nng_aio_alloc(&aio[i], NULL, NULL)); + NUTS_PASS(nng_msg_alloc(&msg[i], 0)); + } + + for (int i = 0; i < 3; i++) { + nng_aio_set_msg(aio[i], msg[i]); + nng_ctx_send(ctx[i], aio[i]); + } + + for (int i = 0; i < 3; i++) { + nng_aio_abort(aio[i], NNG_ECANCELED); + } + + for (int i = 0; i < 3; i++) { + nng_aio_wait(aio[i]); + NUTS_FAIL(nng_aio_result(aio[i]), NNG_ECANCELED); + nng_aio_free(aio[i]); + nng_msg_free(msg[i]); + } + NUTS_CLOSE(req); +} + +static void +test_req_ctx_send_twice(void) +{ + nng_socket req; + nng_ctx ctx; + nng_aio * aio[2]; + nng_msg * msg[2]; + + NUTS_PASS(nng_req0_open(&req)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_ctx_open(&ctx, req)); + + for (int i = 0; i < 2; i++) { + NUTS_PASS(nng_aio_alloc(&aio[i], NULL, NULL)); + NUTS_PASS(nng_msg_alloc(&msg[i], 0)); + } + + for (int i = 0; i < 2; i++) { + nng_aio_set_msg(aio[i], msg[i]); + nng_ctx_send(ctx, aio[i]); + NUTS_SLEEP(50); + } + + NUTS_CLOSE(req); + nng_aio_wait(aio[0]); + nng_aio_wait(aio[1]); + NUTS_FAIL(nng_aio_result(aio[0]), NNG_ECANCELED); + NUTS_FAIL(nng_aio_result(aio[1]), NNG_ECLOSED); + + for (int i = 0; i < 2; i++) { + nng_aio_free(aio[i]); + nng_msg_free(msg[i]); + } +} + +static void +test_req_ctx_recv_nonblock(void) +{ + nng_socket req; + nng_socket rep; + nng_ctx ctx; + nng_aio * aio; + nng_msg * msg; + + NUTS_PASS(nng_req0_open(&req)); + NUTS_PASS(nng_rep0_open(&rep)); + NUTS_PASS(nng_ctx_open(&ctx, req)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + NUTS_PASS(nng_msg_alloc(&msg, 0)); + + NUTS_MARRY(req, rep); + + nng_aio_set_msg(aio, msg); + nng_ctx_send(ctx, aio); + nng_aio_wait(aio); + NUTS_PASS(nng_aio_result(aio)); + nng_aio_set_timeout(aio, 0); // Instant timeout + nng_ctx_recv(ctx, aio); + + nng_aio_wait(aio); + NUTS_FAIL(nng_aio_result(aio), NNG_ETIMEDOUT); + NUTS_CLOSE(req); + NUTS_CLOSE(rep); + nng_aio_free(aio); +} + +static void +test_req_ctx_send_nonblock(void) +{ + nng_socket req; + nng_ctx ctx; + nng_aio * aio; + nng_msg * msg; + + NUTS_PASS(nng_req0_open(&req)); + NUTS_PASS(nng_ctx_open(&ctx, req)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + NUTS_PASS(nng_msg_alloc(&msg, 0)); + + nng_aio_set_msg(aio, msg); + nng_aio_set_timeout(aio, 0); // Instant timeout + nng_ctx_send(ctx, aio); + nng_aio_wait(aio); + NUTS_FAIL(nng_aio_result(aio), NNG_ETIMEDOUT); + NUTS_CLOSE(req); + nng_aio_free(aio); + nng_msg_free(msg); +} + +static void +test_req_ctx_recv_close_socket(void) +{ + nng_socket req; + nng_socket rep; + nng_ctx ctx; + nng_aio * aio; + nng_msg * m; + + NUTS_PASS(nng_req0_open(&req)); + NUTS_PASS(nng_rep0_open(&rep)); + NUTS_PASS(nng_ctx_open(&ctx, req)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + NUTS_MARRY(req, rep); + NUTS_PASS(nng_msg_alloc(&m, 0)); + nng_aio_set_msg(aio, m); + nng_ctx_send(ctx, aio); + nng_aio_wait(aio); + NUTS_PASS(nng_aio_result(aio)); + + nng_ctx_recv(ctx, aio); + nng_close(req); + + NUTS_FAIL(nng_aio_result(aio), NNG_ECLOSED); + nng_aio_free(aio); + NUTS_CLOSE(rep); +} + +static void +test_req_validate_peer(void) +{ + nng_socket s1, s2; + nng_stat * stats; + nng_stat * reject; + char * addr; + + NUTS_ADDR(addr, "inproc"); + + NUTS_PASS(nng_req0_open(&s1)); + NUTS_PASS(nng_req0_open(&s2)); + + NUTS_PASS(nng_listen(s1, addr, NULL, 0)); + NUTS_PASS(nng_dial(s2, addr, NULL, NNG_FLAG_NONBLOCK)); + + NUTS_SLEEP(100); + NUTS_PASS(nng_stats_get(&stats)); + + NUTS_TRUE(stats != NULL); + NUTS_TRUE((reject = nng_stat_find_socket(stats, s1)) != NULL); + NUTS_TRUE((reject = nng_stat_find(reject, "reject")) != NULL); + + NUTS_TRUE(nng_stat_type(reject) == NNG_STAT_COUNTER); + NUTS_TRUE(nng_stat_value(reject) > 0); + + NUTS_CLOSE(s1); + NUTS_CLOSE(s2); + nng_stats_free(stats); +} + +NUTS_TESTS = { + { "req identity", test_req_identity }, + { "req ttl option", test_req_ttl_option }, + { "req resend option", test_req_resend_option }, + { "req recv bad state", test_req_recv_bad_state }, + { "req recv garbage", test_req_recv_garbage }, + { "req rep exchange", test_req_rep_exchange }, + { "req resend", test_req_resend }, + { "req resend disconnect", test_req_resend_disconnect }, + { "req disconnect no retry", test_req_disconnect_no_retry }, + { "req disconnect abort", test_req_disconnect_abort }, + { "req resend reconnect", test_req_resend_reconnect }, + { "req cancel", test_req_cancel }, + { "req cancel abort recv", test_req_cancel_abort_recv }, + { "req cancel post recv", test_req_cancel_post_recv }, + { "req poll writable", test_req_poll_writeable }, + { "req poll contention", test_req_poll_contention }, + { "req poll multi pipe", test_req_poll_multi_pipe }, + { "req poll readable", test_req_poll_readable }, + { "req context send queued", test_req_ctx_send_queued }, + { "req context send close", test_req_ctx_send_close }, + { "req context send abort", test_req_ctx_send_abort }, + { "req context send twice", test_req_ctx_send_twice }, + { "req context does not poll", test_req_ctx_no_poll }, + { "req context recv close socket", test_req_ctx_recv_close_socket }, + { "req context recv nonblock", test_req_ctx_recv_nonblock }, + { "req context send nonblock", test_req_ctx_send_nonblock }, + { "req validate peer", test_req_validate_peer }, + { NULL, NULL }, +}; diff --git a/src/sp/protocol/reqrep0/xrep.c b/src/sp/protocol/reqrep0/xrep.c new file mode 100644 index 00000000..9737c600 --- /dev/null +++ b/src/sp/protocol/reqrep0/xrep.c @@ -0,0 +1,432 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// Copyright 2018 Capitar IT Group BV +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include + +#include "core/nng_impl.h" +#include "nng/protocol/reqrep0/rep.h" + +// Response protocol in raw mode. The REP protocol is the "reply" side of a +// request-reply pair. This is useful for building RPC servers, for +// example. + +typedef struct xrep0_pipe xrep0_pipe; +typedef struct xrep0_sock xrep0_sock; + +static void xrep0_sock_getq_cb(void *); +static void xrep0_pipe_getq_cb(void *); +static void xrep0_pipe_putq_cb(void *); +static void xrep0_pipe_send_cb(void *); +static void xrep0_pipe_recv_cb(void *); +static void xrep0_pipe_fini(void *); + +// xrep0_sock is our per-socket protocol private structure. +struct xrep0_sock { + nni_msgq * uwq; + nni_msgq * urq; + nni_mtx lk; + nni_atomic_int ttl; + nni_id_map pipes; + nni_aio aio_getq; +}; + +// xrep0_pipe is our per-pipe protocol private structure. +struct xrep0_pipe { + nni_pipe * pipe; + xrep0_sock *rep; + nni_msgq * sendq; + nni_aio aio_getq; + nni_aio aio_send; + nni_aio aio_recv; + nni_aio aio_putq; +}; + +static void +xrep0_sock_fini(void *arg) +{ + xrep0_sock *s = arg; + + nni_aio_fini(&s->aio_getq); + nni_id_map_fini(&s->pipes); + nni_mtx_fini(&s->lk); +} + +static int +xrep0_sock_init(void *arg, nni_sock *sock) +{ + xrep0_sock *s = arg; + + nni_mtx_init(&s->lk); + nni_aio_init(&s->aio_getq, xrep0_sock_getq_cb, s); + nni_atomic_init(&s->ttl); + nni_atomic_set(&s->ttl, 8); // Per RFC + s->uwq = nni_sock_sendq(sock); + s->urq = nni_sock_recvq(sock); + + nni_id_map_init(&s->pipes, 0, 0, false); + return (0); +} + +static void +xrep0_sock_open(void *arg) +{ + xrep0_sock *s = arg; + + // This starts us retrieving message from the upper write q. + nni_msgq_aio_get(s->uwq, &s->aio_getq); +} + +static void +xrep0_sock_close(void *arg) +{ + xrep0_sock *s = arg; + + nni_aio_close(&s->aio_getq); +} + +static void +xrep0_pipe_stop(void *arg) +{ + xrep0_pipe *p = arg; + + nni_aio_stop(&p->aio_getq); + nni_aio_stop(&p->aio_send); + nni_aio_stop(&p->aio_recv); + nni_aio_stop(&p->aio_putq); +} + +static void +xrep0_pipe_fini(void *arg) +{ + xrep0_pipe *p = arg; + + nni_aio_fini(&p->aio_getq); + nni_aio_fini(&p->aio_send); + nni_aio_fini(&p->aio_recv); + nni_aio_fini(&p->aio_putq); + nni_msgq_fini(p->sendq); +} + +static int +xrep0_pipe_init(void *arg, nni_pipe *pipe, void *s) +{ + xrep0_pipe *p = arg; + int rv; + + nni_aio_init(&p->aio_getq, xrep0_pipe_getq_cb, p); + nni_aio_init(&p->aio_send, xrep0_pipe_send_cb, p); + nni_aio_init(&p->aio_recv, xrep0_pipe_recv_cb, p); + nni_aio_init(&p->aio_putq, xrep0_pipe_putq_cb, p); + + p->pipe = pipe; + p->rep = s; + + // We want a pretty deep send queue on pipes. The rationale here is + // that the send rate will be mitigated by the receive rate. + // If a slow pipe (req pipe not reading its own responses!?) + // comes up, then we will start discarding its replies eventually, + // but it takes some time. It would be poor form for a peer to + // smash us with requests, but be unable to handle replies faster + // than we can forward them. If they do that, their replies get + // dropped. (From a DDoS perspective, it might be nice in the + // future if we had a way to exert back pressure to the send side -- + // essentially don't let peers send requests faster than they are + // willing to receive replies. Something to think about for the + // future.) + if ((rv = nni_msgq_init(&p->sendq, 64)) != 0) { + xrep0_pipe_fini(p); + return (rv); + } + return (0); +} + +static int +xrep0_pipe_start(void *arg) +{ + xrep0_pipe *p = arg; + xrep0_sock *s = p->rep; + int rv; + + if (nni_pipe_peer(p->pipe) != NNG_REP0_PEER) { + // Peer protocol mismatch. + return (NNG_EPROTO); + } + + nni_mtx_lock(&s->lk); + rv = nni_id_set(&s->pipes, nni_pipe_id(p->pipe), p); + nni_mtx_unlock(&s->lk); + if (rv != 0) { + return (rv); + } + + nni_msgq_aio_get(p->sendq, &p->aio_getq); + nni_pipe_recv(p->pipe, &p->aio_recv); + return (0); +} + +static void +xrep0_pipe_close(void *arg) +{ + xrep0_pipe *p = arg; + xrep0_sock *s = p->rep; + + nni_aio_close(&p->aio_getq); + nni_aio_close(&p->aio_send); + nni_aio_close(&p->aio_recv); + nni_aio_close(&p->aio_putq); + nni_msgq_close(p->sendq); + + nni_mtx_lock(&s->lk); + nni_id_remove(&s->pipes, nni_pipe_id(p->pipe)); + nni_mtx_unlock(&s->lk); +} + +static void +xrep0_sock_getq_cb(void *arg) +{ + xrep0_sock *s = arg; + nni_msgq * uwq = s->uwq; + nni_msg * msg; + uint32_t id; + xrep0_pipe *p; + + // This watches for messages from the upper write queue, + // extracts the destination pipe, and forwards it to the appropriate + // destination pipe via a separate queue. This prevents a single bad + // or slow pipe from gumming up the works for the entire socket. + + if (nni_aio_result(&s->aio_getq) != 0) { + // Closed socket? + return; + } + + msg = nni_aio_get_msg(&s->aio_getq); + nni_aio_set_msg(&s->aio_getq, NULL); + + // We yank the outgoing pipe id from the header + if (nni_msg_header_len(msg) < 4) { + nni_msg_free(msg); + + // Look for another message on the upper write queue. + nni_msgq_aio_get(uwq, &s->aio_getq); + return; + } + + id = nni_msg_header_trim_u32(msg); + + // Look for the pipe, and attempt to put the message there + // (non-blocking) if we can. If we can't for any reason, then we + // free the message. + nni_mtx_lock(&s->lk); + if (((p = nni_id_get(&s->pipes, id)) == NULL) || + (nni_msgq_tryput(p->sendq, msg) != 0)) { + nni_msg_free(msg); + } + nni_mtx_unlock(&s->lk); + + // Now look for another message on the upper write queue. + nni_msgq_aio_get(uwq, &s->aio_getq); +} + +static void +xrep0_pipe_getq_cb(void *arg) +{ + xrep0_pipe *p = arg; + + if (nni_aio_result(&p->aio_getq) != 0) { + nni_pipe_close(p->pipe); + return; + } + + nni_aio_set_msg(&p->aio_send, nni_aio_get_msg(&p->aio_getq)); + nni_aio_set_msg(&p->aio_getq, NULL); + + nni_pipe_send(p->pipe, &p->aio_send); +} + +static void +xrep0_pipe_send_cb(void *arg) +{ + xrep0_pipe *p = arg; + + if (nni_aio_result(&p->aio_send) != 0) { + nni_msg_free(nni_aio_get_msg(&p->aio_send)); + nni_aio_set_msg(&p->aio_send, NULL); + nni_pipe_close(p->pipe); + return; + } + + nni_msgq_aio_get(p->sendq, &p->aio_getq); +} + +static void +xrep0_pipe_recv_cb(void *arg) +{ + xrep0_pipe *p = arg; + xrep0_sock *s = p->rep; + nni_msg * msg; + int hops; + int ttl; + + if (nni_aio_result(&p->aio_recv) != 0) { + nni_pipe_close(p->pipe); + return; + } + + ttl = nni_atomic_get(&s->ttl); + + msg = nni_aio_get_msg(&p->aio_recv); + nni_aio_set_msg(&p->aio_recv, NULL); + + nni_msg_set_pipe(msg, nni_pipe_id(p->pipe)); + + // Store the pipe id in the header, first thing. + nni_msg_header_append_u32(msg, nni_pipe_id(p->pipe)); + + // Move backtrace from body to header + hops = 1; + for (;;) { + bool end; + uint8_t *body; + if (hops > ttl) { + // This isn't malformed, but it has gone through + // too many hops. Do not disconnect, because we + // can legitimately receive messages with too many + // hops from devices, etc. + goto drop; + } + hops++; + if (nni_msg_len(msg) < 4) { + // Peer is speaking garbage. Kick it. + nni_msg_free(msg); + nni_pipe_close(p->pipe); + return; + } + body = nni_msg_body(msg); + end = ((body[0] & 0x80u) != 0); + if (nni_msg_header_append(msg, body, 4) != 0) { + // Out of memory most likely, but keep going to + // avoid breaking things. + goto drop; + } + nni_msg_trim(msg, 4); + if (end) { + break; + } + } + + // Go ahead and send it up. + nni_aio_set_msg(&p->aio_putq, msg); + nni_msgq_aio_put(s->urq, &p->aio_putq); + return; + +drop: + nni_msg_free(msg); + nni_pipe_recv(p->pipe, &p->aio_recv); +} + +static void +xrep0_pipe_putq_cb(void *arg) +{ + xrep0_pipe *p = arg; + + if (nni_aio_result(&p->aio_putq) != 0) { + nni_msg_free(nni_aio_get_msg(&p->aio_putq)); + nni_aio_set_msg(&p->aio_putq, NULL); + nni_pipe_close(p->pipe); + return; + } + + nni_pipe_recv(p->pipe, &p->aio_recv); +} + +static int +xrep0_sock_set_maxttl(void *arg, const void *buf, size_t sz, nni_opt_type t) +{ + xrep0_sock *s = arg; + int ttl; + int rv; + if ((rv = nni_copyin_int(&ttl, buf, sz, 1, NNI_MAX_MAX_TTL, t)) == 0) { + nni_atomic_set(&s->ttl, ttl); + } + return (rv); +} + +static int +xrep0_sock_get_maxttl(void *arg, void *buf, size_t *szp, nni_opt_type t) +{ + xrep0_sock *s = arg; + return (nni_copyout_int(nni_atomic_get(&s->ttl), buf, szp, t)); +} + +static void +xrep0_sock_send(void *arg, nni_aio *aio) +{ + xrep0_sock *s = arg; + + nni_msgq_aio_put(s->uwq, aio); +} + +static void +xrep0_sock_recv(void *arg, nni_aio *aio) +{ + xrep0_sock *s = arg; + + nni_msgq_aio_get(s->urq, aio); +} + +// This is the global protocol structure -- our linkage to the core. +// This should be the only global non-static symbol in this file. +static nni_proto_pipe_ops xrep0_pipe_ops = { + .pipe_size = sizeof(xrep0_pipe), + .pipe_init = xrep0_pipe_init, + .pipe_fini = xrep0_pipe_fini, + .pipe_start = xrep0_pipe_start, + .pipe_close = xrep0_pipe_close, + .pipe_stop = xrep0_pipe_stop, +}; + +static nni_option xrep0_sock_options[] = { + { + .o_name = NNG_OPT_MAXTTL, + .o_get = xrep0_sock_get_maxttl, + .o_set = xrep0_sock_set_maxttl, + }, + // terminate list + { + .o_name = NULL, + }, +}; + +static nni_proto_sock_ops xrep0_sock_ops = { + .sock_size = sizeof(xrep0_sock), + .sock_init = xrep0_sock_init, + .sock_fini = xrep0_sock_fini, + .sock_open = xrep0_sock_open, + .sock_close = xrep0_sock_close, + .sock_options = xrep0_sock_options, + .sock_send = xrep0_sock_send, + .sock_recv = xrep0_sock_recv, +}; + +static nni_proto xrep0_proto = { + .proto_version = NNI_PROTOCOL_VERSION, + .proto_self = { NNG_REP0_SELF, NNG_REP0_SELF_NAME }, + .proto_peer = { NNG_REP0_PEER, NNG_REP0_PEER_NAME }, + .proto_flags = NNI_PROTO_FLAG_SNDRCV | NNI_PROTO_FLAG_RAW, + .proto_sock_ops = &xrep0_sock_ops, + .proto_pipe_ops = &xrep0_pipe_ops, +}; + +int +nng_rep0_open_raw(nng_socket *sidp) +{ + return (nni_proto_open(sidp, &xrep0_proto)); +} diff --git a/src/sp/protocol/reqrep0/xrep_test.c b/src/sp/protocol/reqrep0/xrep_test.c new file mode 100644 index 00000000..6f1564eb --- /dev/null +++ b/src/sp/protocol/reqrep0/xrep_test.c @@ -0,0 +1,434 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include + +static void +test_xrep_identity(void) +{ + nng_socket s; + int p1, p2; + char * n1; + char * n2; + + NUTS_PASS(nng_rep0_open_raw(&s)); + NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PROTO, &p1)); + NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PEER, &p2)); + NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PROTONAME, &n1)); + NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PEERNAME, &n2)); + NUTS_CLOSE(s); + NUTS_TRUE(p1 == NNG_REP0_SELF); + NUTS_TRUE(p2 == NNG_REP0_PEER); + NUTS_MATCH(n1, NNG_REP0_SELF_NAME); + NUTS_MATCH(n2, NNG_REP0_PEER_NAME); + nng_strfree(n1); + nng_strfree(n2); +} + +static void +test_xrep_raw(void) +{ + nng_socket s; + bool b; + + NUTS_PASS(nng_rep0_open_raw(&s)); + NUTS_PASS(nng_socket_get_bool(s, NNG_OPT_RAW, &b)); + NUTS_TRUE(b); + NUTS_CLOSE(s); +} + +static void +test_xrep_no_context(void) +{ + nng_socket s; + nng_ctx ctx; + + NUTS_PASS(nng_rep0_open_raw(&s)); + NUTS_FAIL(nng_ctx_open(&ctx, s), NNG_ENOTSUP); + NUTS_CLOSE(s); +} + +static void +test_xrep_poll_writeable(void) +{ + int fd; + nng_socket req; + nng_socket rep; + + NUTS_PASS(nng_rep0_open_raw(&rep)); + NUTS_PASS(nng_req0_open(&req)); + NUTS_PASS(nng_socket_get_int(rep, NNG_OPT_SENDFD, &fd)); + NUTS_TRUE(fd >= 0); + + // We are always writeable, even before connect. This is so that + // back-pressure from a bad peer can't trash others. We assume + // that peers won't send us requests faster than they can consume + // the answers. If they do, they will lose their answers. + NUTS_TRUE(nuts_poll_fd(fd) == true); + + NUTS_MARRY(req, rep); + + // Now it's writable. + NUTS_TRUE(nuts_poll_fd(fd) == true); + + NUTS_CLOSE(req); + NUTS_CLOSE(rep); +} + +static void +test_xrep_poll_readable(void) +{ + int fd; + nng_socket req; + nng_socket rep; + nng_msg * msg; + + NUTS_PASS(nng_req0_open(&req)); + NUTS_PASS(nng_rep0_open_raw(&rep)); + NUTS_PASS(nng_socket_get_int(rep, NNG_OPT_RECVFD, &fd)); + NUTS_TRUE(fd >= 0); + + // Not readable if not connected! + NUTS_TRUE(nuts_poll_fd(fd) == false); + + // Even after connect (no message yet) + NUTS_MARRY(req, rep); + NUTS_TRUE(nuts_poll_fd(fd) == false); + + // But once we send messages, it is. + // We have to send a request, in order to send a reply. + NUTS_SEND(req, "abc"); + NUTS_SLEEP(100); + + NUTS_TRUE(nuts_poll_fd(fd) == true); + + // and receiving makes it no longer ready + NUTS_PASS(nng_recvmsg(rep, &msg, 0)); + nng_msg_free(msg); + NUTS_TRUE(nuts_poll_fd(fd) == false); + + NUTS_CLOSE(req); + NUTS_CLOSE(rep); +} + +static void +test_xrep_validate_peer(void) +{ + nng_socket s1, s2; + nng_stat * stats; + nng_stat * reject; + char *addr; + + NUTS_ADDR(addr, "inproc"); + + NUTS_PASS(nng_rep0_open_raw(&s1)); + NUTS_PASS(nng_rep0_open(&s2)); + + NUTS_PASS(nng_listen(s1, addr, NULL, 0)); + NUTS_PASS(nng_dial(s2, addr, NULL, NNG_FLAG_NONBLOCK)); + + NUTS_SLEEP(100); + NUTS_PASS(nng_stats_get(&stats)); + + NUTS_TRUE(stats != NULL); + NUTS_TRUE((reject = nng_stat_find_socket(stats, s1)) != NULL); + NUTS_TRUE((reject = nng_stat_find(reject, "reject")) != NULL); + + NUTS_TRUE(nng_stat_type(reject) == NNG_STAT_COUNTER); + NUTS_TRUE(nng_stat_value(reject) > 0); + + NUTS_PASS(nng_close(s1)); + NUTS_PASS(nng_close(s2)); + nng_stats_free(stats); +} + +static void +test_xrep_close_pipe_before_send(void) +{ + nng_socket rep; + nng_socket req; + nng_pipe p; + nng_aio * aio1; + nng_msg * m; + + NUTS_PASS(nng_rep0_open_raw(&rep)); + NUTS_PASS(nng_req0_open(&req)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_aio_alloc(&aio1, NULL, NULL)); + + NUTS_MARRY(req, rep); + NUTS_SEND(req, "test"); + + nng_recv_aio(rep, aio1); + nng_aio_wait(aio1); + NUTS_PASS(nng_aio_result(aio1)); + NUTS_TRUE((m = nng_aio_get_msg(aio1)) != NULL); + p = nng_msg_get_pipe(m); + NUTS_PASS(nng_pipe_close(p)); + NUTS_PASS(nng_sendmsg(rep, m, 0)); + + NUTS_CLOSE(req); + NUTS_CLOSE(rep); + nng_aio_free(aio1); +} + +static void +test_xrep_close_pipe_during_send(void) +{ + nng_socket rep; + nng_socket req; + nng_pipe p; + nng_msg * m; + + NUTS_PASS(nng_rep0_open_raw(&rep)); + NUTS_PASS(nng_req0_open_raw(&req)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 200)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_int(rep, NNG_OPT_SENDBUF, 20)); + NUTS_PASS(nng_socket_set_int(rep, NNG_OPT_RECVBUF, 20)); + NUTS_PASS(nng_socket_set_int(req, NNG_OPT_SENDBUF, 20)); + NUTS_PASS(nng_socket_set_int(req, NNG_OPT_RECVBUF, 1)); + + NUTS_MARRY(req, rep); + + NUTS_PASS(nng_msg_alloc(&m, 4)); + NUTS_PASS(nng_msg_append_u32(m, (unsigned) 0x81000000u)); + NUTS_PASS(nng_sendmsg(req, m, 0)); + NUTS_PASS(nng_recvmsg(rep, &m, 0)); + p = nng_msg_get_pipe(m); + nng_msg_free(m); + + for (int i = 0; i < 100; i++) { + NUTS_PASS(nng_msg_alloc(&m, 4)); + NUTS_PASS(nng_msg_header_append_u32(m, nng_pipe_id(p))); + NUTS_PASS( + nng_msg_header_append_u32(m, (unsigned) i | 0x80000000u)); + // xrep does not exert back-pressure + NUTS_PASS(nng_sendmsg(rep, m, 0)); + } + NUTS_PASS(nng_pipe_close(p)); + + NUTS_CLOSE(req); + NUTS_CLOSE(rep); +} + +static void +test_xrep_close_during_recv(void) +{ + nng_socket rep; + nng_socket req; + nng_msg * m; + + NUTS_PASS(nng_rep0_open_raw(&rep)); + NUTS_PASS(nng_req0_open_raw(&req)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 100)); + NUTS_PASS(nng_socket_set_int(rep, NNG_OPT_RECVBUF, 5)); + NUTS_PASS(nng_socket_set_int(req, NNG_OPT_SENDBUF, 20)); + + NUTS_MARRY(req, rep); + + for (unsigned i = 0; i < 100; i++) { + int rv; + NUTS_PASS(nng_msg_alloc(&m, 4)); + NUTS_PASS(nng_msg_header_append_u32(m, i | 0x80000000u)); + rv = nng_sendmsg(req, m, 0); + if (rv == NNG_ETIMEDOUT) { + nng_msg_free(m); + break; + } + } + NUTS_CLOSE(req); + NUTS_CLOSE(rep); +} + +static void +test_xrep_recv_aio_stopped(void) +{ + nng_socket rep; + nng_aio * aio; + + NUTS_PASS(nng_rep0_open_raw(&rep)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + + nng_aio_stop(aio); + nng_recv_aio(rep, aio); + nng_aio_wait(aio); + NUTS_FAIL(nng_aio_result(aio), NNG_ECANCELED); + NUTS_CLOSE(rep); + nng_aio_free(aio); +} + +static void +test_xrep_send_no_header(void) +{ + nng_socket rep; + nng_socket req; + nng_msg * m; + + NUTS_PASS(nng_req0_open_raw(&req)); + NUTS_PASS(nng_rep0_open_raw(&rep)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 100)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_RECVTIMEO, 100)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); + + NUTS_MARRY(req, rep); + + NUTS_PASS(nng_msg_alloc(&m, 4)); + NUTS_PASS(nng_sendmsg(rep, m, 0)); + NUTS_FAIL(nng_recvmsg(rep, &m, 0), NNG_ETIMEDOUT); + + NUTS_CLOSE(req); + NUTS_CLOSE(rep); +} + +static void +test_xrep_recv_garbage(void) +{ + nng_socket rep; + nng_socket req; + nng_msg * m; + + NUTS_PASS(nng_rep0_open_raw(&rep)); + NUTS_PASS(nng_req0_open_raw(&req)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 100)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 100)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); + + NUTS_MARRY(req, rep); + + NUTS_PASS(nng_msg_alloc(&m, 4)); + NUTS_PASS(nng_msg_append_u32(m, 1u)); + NUTS_PASS(nng_sendmsg(req, m, 0)); + NUTS_FAIL(nng_recvmsg(rep, &m, 0), NNG_ETIMEDOUT); + + NUTS_CLOSE(req); + NUTS_CLOSE(rep); +} + +static void +test_xrep_ttl_option(void) +{ + nng_socket rep; + int v; + bool b; + size_t sz; + const char *opt = NNG_OPT_MAXTTL; + + NUTS_PASS(nng_rep0_open_raw(&rep)); + + NUTS_PASS(nng_socket_set_int(rep, opt, 1)); + NUTS_FAIL(nng_socket_set_int(rep, opt, 0), NNG_EINVAL); + NUTS_FAIL(nng_socket_set_int(rep, opt, -1), NNG_EINVAL); + NUTS_FAIL(nng_socket_set_int(rep, opt, 16), NNG_EINVAL); + NUTS_FAIL(nng_socket_set_int(rep, opt, 256), NNG_EINVAL); + NUTS_PASS(nng_socket_set_int(rep, opt, 3)); + NUTS_PASS(nng_socket_get_int(rep, opt, &v)); + NUTS_TRUE(v == 3); + v = 0; + sz = sizeof(v); + NUTS_PASS(nng_socket_get(rep, opt, &v, &sz)); + NUTS_TRUE(v == 3); + NUTS_TRUE(sz == sizeof(v)); + + NUTS_TRUE(nng_socket_set(rep, opt, "", 1) == NNG_EINVAL); + sz = 1; + NUTS_TRUE(nng_socket_get(rep, opt, &v, &sz) == NNG_EINVAL); + NUTS_TRUE(nng_socket_set_bool(rep, opt, true) == NNG_EBADTYPE); + NUTS_TRUE(nng_socket_get_bool(rep, opt, &b) == NNG_EBADTYPE); + + NUTS_TRUE(nng_close(rep) == 0); +} + +static void +test_xrep_ttl_drop(void) +{ + nng_socket rep; + nng_socket req; + nng_msg * m; + + NUTS_PASS(nng_rep0_open_raw(&rep)); + NUTS_PASS(nng_req0_open_raw(&req)); + NUTS_PASS(nng_socket_set_int(rep, NNG_OPT_MAXTTL, 3)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 200)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); + + NUTS_MARRY(req, rep); + + // Send messages. Note that xrep implicitly adds a hop on receive. + + NUTS_PASS(nng_msg_alloc(&m, 0)); + NUTS_PASS(nng_msg_append_u32(m, 1u)); // 2 hops + NUTS_PASS(nng_msg_append_u32(m, 0x80000001u)); + NUTS_PASS(nng_msg_append(m, "PASS1", 6)); + NUTS_PASS(nng_sendmsg(req, m, 0)); + + NUTS_PASS(nng_msg_alloc(&m, 0)); + NUTS_PASS(nng_msg_append_u32(m, 1u)); // 4 hops -- discard! + NUTS_PASS(nng_msg_append_u32(m, 2u)); + NUTS_PASS(nng_msg_append_u32(m, 3u)); + NUTS_PASS(nng_msg_append_u32(m, 0x80000002u)); + NUTS_PASS(nng_msg_append(m, "FAIL2", 6)); + NUTS_PASS(nng_sendmsg(req, m, 0)); + + NUTS_PASS(nng_msg_alloc(&m, 0)); + NUTS_PASS(nng_msg_append_u32(m, 1u)); // 3 hops - passes + NUTS_PASS(nng_msg_append_u32(m, 2u)); + NUTS_PASS(nng_msg_append_u32(m, 0x80000003u)); + NUTS_PASS(nng_msg_append(m, "PASS3", 6)); + NUTS_PASS(nng_sendmsg(req, m, 0)); + + NUTS_PASS(nng_msg_alloc(&m, 0)); + NUTS_PASS(nng_msg_append_u32(m, 1u)); // 4 hops -- discard! + NUTS_PASS(nng_msg_append_u32(m, 2u)); + NUTS_PASS(nng_msg_append_u32(m, 3u)); + NUTS_PASS(nng_msg_append_u32(m, 0x80000003u)); + NUTS_PASS(nng_msg_append(m, "FAIL4", 6)); + NUTS_PASS(nng_sendmsg(req, m, 0)); + + // So on receive we should see 80000001 and 80000003. + NUTS_PASS(nng_recvmsg(rep, &m, 0)); + NUTS_TRUE(nng_msg_header_len(m) == 12); + NUTS_TRUE(nng_msg_len(m) == 6); + NUTS_TRUE(strcmp(nng_msg_body(m), "PASS1") == 0); + nng_msg_free(m); + + NUTS_PASS(nng_recvmsg(rep, &m, 0)); + NUTS_TRUE(nng_msg_header_len(m) == 16); // 3 hops + ID + NUTS_TRUE(nng_msg_len(m) == 6); + NUTS_TRUE(strcmp(nng_msg_body(m), "PASS3") == 0); + nng_msg_free(m); + + NUTS_FAIL(nng_recvmsg(rep, &m, 0), NNG_ETIMEDOUT); + + NUTS_CLOSE(req); + NUTS_CLOSE(rep); +} + +NUTS_TESTS = { + { "xrep identity", test_xrep_identity }, + { "xrep raw", test_xrep_raw }, + { "xrep no context", test_xrep_no_context }, + { "xrep poll readable", test_xrep_poll_readable }, + { "xrep poll writable", test_xrep_poll_writeable }, + { "xrep validate peer", test_xrep_validate_peer }, + { "xrep close pipe before send", test_xrep_close_pipe_before_send }, + { "xrep close pipe during send", test_xrep_close_pipe_during_send }, + { "xrep close during recv", test_xrep_close_during_recv }, + { "xrep recv aio stopped", test_xrep_recv_aio_stopped }, + { "xrep send no header", test_xrep_send_no_header }, + { "xrep recv garbage", test_xrep_recv_garbage }, + { "xrep ttl option", test_xrep_ttl_option }, + { "xrep ttl drop", test_xrep_ttl_drop }, + { NULL, NULL }, +}; diff --git a/src/sp/protocol/reqrep0/xreq.c b/src/sp/protocol/reqrep0/xreq.c new file mode 100644 index 00000000..bcb218bf --- /dev/null +++ b/src/sp/protocol/reqrep0/xreq.c @@ -0,0 +1,319 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// Copyright 2018 Capitar IT Group BV +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include + +#include "core/nng_impl.h" +#include "nng/protocol/reqrep0/req.h" + +// Request protocol. The REQ protocol is the "request" side of a +// request-reply pair. This is useful for building RPC clients, for example. + +typedef struct xreq0_pipe xreq0_pipe; +typedef struct xreq0_sock xreq0_sock; + +// An xreq0_sock is our per-socket protocol private structure. +struct xreq0_sock { + nni_msgq * uwq; + nni_msgq * urq; + nni_atomic_int ttl; +}; + +// A req0_pipe is our per-pipe protocol private structure. +struct xreq0_pipe { + nni_pipe * pipe; + xreq0_sock *req; + nni_aio aio_getq; + nni_aio aio_send; + nni_aio aio_recv; + nni_aio aio_putq; +}; + +static void xreq0_sock_fini(void *); +static void xreq0_getq_cb(void *); +static void xreq0_send_cb(void *); +static void xreq0_recv_cb(void *); +static void xreq0_putq_cb(void *); + +static int +xreq0_sock_init(void *arg, nni_sock *sock) +{ + xreq0_sock *s = arg; + + nni_atomic_init(&s->ttl); + nni_atomic_set(&s->ttl, 8); + s->uwq = nni_sock_sendq(sock); + s->urq = nni_sock_recvq(sock); + + return (0); +} + +static void +xreq0_sock_open(void *arg) +{ + NNI_ARG_UNUSED(arg); +} + +static void +xreq0_sock_close(void *arg) +{ + NNI_ARG_UNUSED(arg); +} + +static void +xreq0_sock_fini(void *arg) +{ + NNI_ARG_UNUSED(arg); +} + +static void +xreq0_pipe_stop(void *arg) +{ + xreq0_pipe *p = arg; + + nni_aio_stop(&p->aio_getq); + nni_aio_stop(&p->aio_putq); + nni_aio_stop(&p->aio_recv); + nni_aio_stop(&p->aio_send); +} + +static void +xreq0_pipe_fini(void *arg) +{ + xreq0_pipe *p = arg; + + nni_aio_fini(&p->aio_getq); + nni_aio_fini(&p->aio_putq); + nni_aio_fini(&p->aio_recv); + nni_aio_fini(&p->aio_send); +} + +static int +xreq0_pipe_init(void *arg, nni_pipe *pipe, void *s) +{ + xreq0_pipe *p = arg; + + nni_aio_init(&p->aio_getq, xreq0_getq_cb, p); + nni_aio_init(&p->aio_putq, xreq0_putq_cb, p); + nni_aio_init(&p->aio_recv, xreq0_recv_cb, p); + nni_aio_init(&p->aio_send, xreq0_send_cb, p); + + p->pipe = pipe; + p->req = s; + return (0); +} + +static int +xreq0_pipe_start(void *arg) +{ + xreq0_pipe *p = arg; + xreq0_sock *s = p->req; + + if (nni_pipe_peer(p->pipe) != NNG_REQ0_PEER) { + return (NNG_EPROTO); + } + + nni_msgq_aio_get(s->uwq, &p->aio_getq); + nni_pipe_recv(p->pipe, &p->aio_recv); + return (0); +} + +static void +xreq0_pipe_close(void *arg) +{ + xreq0_pipe *p = arg; + + nni_aio_close(&p->aio_getq); + nni_aio_close(&p->aio_putq); + nni_aio_close(&p->aio_recv); + nni_aio_close(&p->aio_send); +} + +// For raw mode we can just let the pipes "contend" via get queue to get a +// message from the upper write queue. The msg queue implementation +// actually provides ordering, so load will be spread automatically. +// (NB: We may have to revise this in the future if we want to provide some +// kind of priority.) + +static void +xreq0_getq_cb(void *arg) +{ + xreq0_pipe *p = arg; + + if (nni_aio_result(&p->aio_getq) != 0) { + nni_pipe_close(p->pipe); + return; + } + + nni_aio_set_msg(&p->aio_send, nni_aio_get_msg(&p->aio_getq)); + nni_aio_set_msg(&p->aio_getq, NULL); + + nni_pipe_send(p->pipe, &p->aio_send); +} + +static void +xreq0_send_cb(void *arg) +{ + xreq0_pipe *p = arg; + + if (nni_aio_result(&p->aio_send) != 0) { + nni_msg_free(nni_aio_get_msg(&p->aio_send)); + nni_aio_set_msg(&p->aio_send, NULL); + nni_pipe_close(p->pipe); + return; + } + + // Sent a message so we just need to look for another one. + nni_msgq_aio_get(p->req->uwq, &p->aio_getq); +} + +static void +xreq0_putq_cb(void *arg) +{ + xreq0_pipe *p = arg; + + if (nni_aio_result(&p->aio_putq) != 0) { + nni_msg_free(nni_aio_get_msg(&p->aio_putq)); + nni_aio_set_msg(&p->aio_putq, NULL); + nni_pipe_close(p->pipe); + return; + } + nni_aio_set_msg(&p->aio_putq, NULL); + + nni_pipe_recv(p->pipe, &p->aio_recv); +} + +static void +xreq0_recv_cb(void *arg) +{ + xreq0_pipe *p = arg; + xreq0_sock *sock = p->req; + nni_msg * msg; + bool end; + + if (nni_aio_result(&p->aio_recv) != 0) { + nni_pipe_close(p->pipe); + return; + } + + msg = nni_aio_get_msg(&p->aio_recv); + nni_aio_set_msg(&p->aio_recv, NULL); + nni_msg_set_pipe(msg, nni_pipe_id(p->pipe)); + end = false; + + while (!end) { + uint8_t *body; + + if (nni_msg_len(msg) < 4) { + // Peer gave us garbage, so kick it. + nni_msg_free(msg); + nni_pipe_close(p->pipe); + return; + } + body = nni_msg_body(msg); + end = ((body[0] & 0x80u) != 0); + + if (nng_msg_header_append(msg, body, sizeof (uint32_t)) != 0) { + // TODO: bump a no-memory stat + nni_msg_free(msg); + // Closing the pipe may release some memory. + // It at least gives an indication to the peer + // that we've lost the message. + nni_pipe_close(p->pipe); + return; + } + nni_msg_trim(msg, sizeof (uint32_t)); + } + nni_aio_set_msg(&p->aio_putq, msg); + nni_msgq_aio_put(sock->urq, &p->aio_putq); +} + +static void +xreq0_sock_send(void *arg, nni_aio *aio) +{ + xreq0_sock *s = arg; + + nni_msgq_aio_put(s->uwq, aio); +} + +static void +xreq0_sock_recv(void *arg, nni_aio *aio) +{ + xreq0_sock *s = arg; + + nni_msgq_aio_get(s->urq, aio); +} + +static int +xreq0_sock_set_max_ttl(void *arg, const void *buf, size_t sz, nni_opt_type t) +{ + xreq0_sock *s = arg; + int ttl; + int rv; + if ((rv = nni_copyin_int(&ttl, buf, sz, 1, NNI_MAX_MAX_TTL, t)) == 0) { + nni_atomic_set(&s->ttl, ttl); + } + return (rv); +} + +static int +xreq0_sock_get_max_ttl(void *arg, void *buf, size_t *szp, nni_opt_type t) +{ + xreq0_sock *s = arg; + return (nni_copyout_int(nni_atomic_get(&s->ttl), buf, szp, t)); +} + +static nni_proto_pipe_ops xreq0_pipe_ops = { + .pipe_size = sizeof(xreq0_pipe), + .pipe_init = xreq0_pipe_init, + .pipe_fini = xreq0_pipe_fini, + .pipe_start = xreq0_pipe_start, + .pipe_close = xreq0_pipe_close, + .pipe_stop = xreq0_pipe_stop, +}; + +static nni_option xreq0_sock_options[] = { + { + .o_name = NNG_OPT_MAXTTL, + .o_get = xreq0_sock_get_max_ttl, + .o_set = xreq0_sock_set_max_ttl, + }, + // terminate list + { + .o_name = NULL, + }, +}; + +static nni_proto_sock_ops xreq0_sock_ops = { + .sock_size = sizeof(xreq0_sock), + .sock_init = xreq0_sock_init, + .sock_fini = xreq0_sock_fini, + .sock_open = xreq0_sock_open, + .sock_close = xreq0_sock_close, + .sock_options = xreq0_sock_options, + .sock_send = xreq0_sock_send, + .sock_recv = xreq0_sock_recv, +}; + +static nni_proto xreq0_proto = { + .proto_version = NNI_PROTOCOL_VERSION, + .proto_self = { NNG_REQ0_SELF, NNG_REQ0_SELF_NAME }, + .proto_peer = { NNG_REQ0_PEER, NNG_REQ0_PEER_NAME }, + .proto_flags = NNI_PROTO_FLAG_SNDRCV | NNI_PROTO_FLAG_RAW, + .proto_sock_ops = &xreq0_sock_ops, + .proto_pipe_ops = &xreq0_pipe_ops, + .proto_ctx_ops = NULL, // raw mode does not support contexts +}; + +int +nng_req0_open_raw(nng_socket *sock) +{ + return (nni_proto_open(sock, &xreq0_proto)); +} diff --git a/src/sp/protocol/reqrep0/xreq_test.c b/src/sp/protocol/reqrep0/xreq_test.c new file mode 100644 index 00000000..8c850cba --- /dev/null +++ b/src/sp/protocol/reqrep0/xreq_test.c @@ -0,0 +1,367 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include + +static void +test_xreq_identity(void) +{ + nng_socket s; + int p1, p2; + char * n1; + char * n2; + + NUTS_PASS(nng_req0_open_raw(&s)); + NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PROTO, &p1)); + NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PEER, &p2)); + NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PROTONAME, &n1)); + NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PEERNAME, &n2)); + NUTS_CLOSE(s); + NUTS_TRUE(p1 == NNG_REQ0_SELF); + NUTS_TRUE(p2 == NNG_REQ0_PEER); + NUTS_MATCH(n1, NNG_REQ0_SELF_NAME); + NUTS_MATCH(n2, NNG_REQ0_PEER_NAME); + nng_strfree(n1); + nng_strfree(n2); +} + +static void +test_xreq_raw(void) +{ + nng_socket s; + bool b; + + NUTS_PASS(nng_req0_open_raw(&s)); + NUTS_PASS(nng_socket_get_bool(s, NNG_OPT_RAW, &b)); + NUTS_TRUE(b); + NUTS_CLOSE(s); +} + +static void +test_xreq_no_context(void) +{ + nng_socket s; + nng_ctx ctx; + + NUTS_PASS(nng_req0_open_raw(&s)); + NUTS_FAIL(nng_ctx_open(&ctx, s), NNG_ENOTSUP); + NUTS_CLOSE(s); +} + +static void +test_xreq_poll_writeable(void) +{ + int fd; + nng_socket req; + nng_socket rep; + + NUTS_PASS(nng_req0_open_raw(&req)); + NUTS_PASS(nng_rep0_open(&rep)); + NUTS_PASS(nng_socket_get_int(req, NNG_OPT_SENDFD, &fd)); + NUTS_TRUE(fd >= 0); + + // We can't write until we have a connection. + NUTS_TRUE(nuts_poll_fd(fd) == false); + + NUTS_MARRY(req, rep); + + // Now it's writable. + NUTS_TRUE(nuts_poll_fd(fd) == true); + + NUTS_CLOSE(req); + NUTS_CLOSE(rep); +} + +static void +test_xreq_poll_readable(void) +{ + int fd; + nng_socket req; + nng_socket rep; + nng_msg * msg; + + NUTS_PASS(nng_req0_open_raw(&req)); + NUTS_PASS(nng_rep0_open(&rep)); + NUTS_PASS(nng_socket_get_int(req, NNG_OPT_RECVFD, &fd)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); + + NUTS_TRUE(fd >= 0); + + // Not readable if not connected! + NUTS_TRUE(nuts_poll_fd(fd) == false); + + // Even after connect (no message yet) + NUTS_MARRY(req, rep); + NUTS_TRUE(nuts_poll_fd(fd) == false); + + // But once we send messages, it is. + // We have to send a request, in order to send a reply. + NUTS_PASS(nng_msg_alloc(&msg, 0)); + // Request ID + NUTS_PASS(nng_msg_append_u32(msg, 0x80000000)); + NUTS_PASS(nng_sendmsg(req, msg, 0)); + + NUTS_PASS(nng_recvmsg(rep, &msg, 0)); + NUTS_PASS(nng_sendmsg(rep, msg, 0)); + + NUTS_SLEEP(100); + + NUTS_TRUE(nuts_poll_fd(fd) == true); + + // and receiving makes it no longer ready + NUTS_PASS(nng_recvmsg(req, &msg, 0)); + nng_msg_free(msg); + NUTS_TRUE(nuts_poll_fd(fd) == false); + + NUTS_CLOSE(req); + NUTS_CLOSE(rep); +} + +static void +test_xreq_validate_peer(void) +{ + nng_socket s1, s2; + nng_stat * stats; + nng_stat * reject; + char * addr; + + NUTS_ADDR(addr, "inproc"); + + NUTS_PASS(nng_req0_open_raw(&s1)); + NUTS_PASS(nng_req0_open(&s2)); + + NUTS_PASS(nng_listen(s1, addr, NULL, 0)); + NUTS_PASS(nng_dial(s2, addr, NULL, NNG_FLAG_NONBLOCK)); + + NUTS_SLEEP(100); + NUTS_PASS(nng_stats_get(&stats)); + + NUTS_TRUE(stats != NULL); + NUTS_TRUE((reject = nng_stat_find_socket(stats, s1)) != NULL); + NUTS_TRUE((reject = nng_stat_find(reject, "reject")) != NULL); + + NUTS_TRUE(nng_stat_type(reject) == NNG_STAT_COUNTER); + NUTS_TRUE(nng_stat_value(reject) > 0); + + NUTS_CLOSE(s1); + NUTS_CLOSE(s2); + nng_stats_free(stats); +} + +static void +test_xreq_recv_aio_stopped(void) +{ + nng_socket req; + nng_aio * aio; + + NUTS_PASS(nng_req0_open_raw(&req)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + + nng_aio_stop(aio); + nng_recv_aio(req, aio); + nng_aio_wait(aio); + NUTS_FAIL(nng_aio_result(aio), NNG_ECANCELED); + NUTS_CLOSE(req); + nng_aio_free(aio); +} + +static void +test_xreq_recv_garbage(void) +{ + nng_socket rep; + nng_socket req; + nng_msg * m; + uint32_t req_id; + + NUTS_PASS(nng_rep0_open_raw(&rep)); + NUTS_PASS(nng_req0_open_raw(&req)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_RECVTIMEO, 100)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 1000)); + + NUTS_MARRY(req, rep); + + NUTS_PASS(nng_msg_alloc(&m, 0)); + NUTS_PASS(nng_msg_append_u32(m, 0x80000000)); + NUTS_PASS(nng_sendmsg(req, m, 0)); + + NUTS_PASS(nng_recvmsg(rep, &m, 0)); + + // The message will have a header that contains the 32-bit pipe ID, + // followed by the 32-bit request ID. We will discard the request + // ID before sending it out. + NUTS_TRUE(nng_msg_header_len(m) == 8); + NUTS_PASS(nng_msg_header_chop_u32(m, &req_id)); + NUTS_TRUE(req_id == 0x80000000); + + NUTS_PASS(nng_sendmsg(rep, m, 0)); + NUTS_FAIL(nng_recvmsg(req, &m, 0), NNG_ETIMEDOUT); + + NUTS_CLOSE(req); + NUTS_CLOSE(rep); +} + +static void +test_xreq_recv_header(void) +{ + nng_socket rep; + nng_socket req; + nng_msg * m; + nng_pipe p1, p2; + uint32_t id; + + NUTS_PASS(nng_rep0_open_raw(&rep)); + NUTS_PASS(nng_req0_open_raw(&req)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_SENDTIMEO, 1000)); + + NUTS_MARRY_EX(req, rep, NULL, &p1, &p2); + + // Simulate a few hops. + NUTS_PASS(nng_msg_alloc(&m, 0)); + NUTS_PASS(nng_msg_header_append_u32(m, nng_pipe_id(p2))); + NUTS_PASS(nng_msg_header_append_u32(m, 0x2)); + NUTS_PASS(nng_msg_header_append_u32(m, 0x1)); + NUTS_PASS(nng_msg_header_append_u32(m, 0x80000123u)); + + NUTS_PASS(nng_sendmsg(rep, m, 0)); + + NUTS_PASS(nng_recvmsg(req, &m, 0)); + NUTS_TRUE(nng_msg_header_len(m) == 12); + NUTS_PASS(nng_msg_header_trim_u32(m, &id)); + NUTS_TRUE(id == 0x2); + NUTS_PASS(nng_msg_header_trim_u32(m, &id)); + NUTS_TRUE(id == 0x1); + NUTS_PASS(nng_msg_header_trim_u32(m, &id)); + NUTS_TRUE(id == 0x80000123u); + + nng_msg_free(m); + + NUTS_CLOSE(req); + NUTS_CLOSE(rep); +} + +static void +test_xreq_close_during_recv(void) +{ + nng_socket rep; + nng_socket req; + nng_msg * m; + nng_pipe p1; + nng_pipe p2; + + NUTS_PASS(nng_rep0_open_raw(&rep)); + NUTS_PASS(nng_req0_open_raw(&req)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 100)); + NUTS_PASS(nng_socket_set_int(req, NNG_OPT_RECVBUF, 5)); + NUTS_PASS(nng_socket_set_int(rep, NNG_OPT_SENDBUF, 20)); + + NUTS_MARRY_EX(req, rep, NULL, &p1, &p2); + NUTS_TRUE(nng_pipe_id(p1) > 0); + NUTS_TRUE(nng_pipe_id(p2) > 0); + + for (unsigned i = 0; i < 20; i++) { + NUTS_PASS(nng_msg_alloc(&m, 4)); + NUTS_PASS(nng_msg_header_append_u32(m, nng_pipe_id(p2))); + NUTS_PASS(nng_msg_header_append_u32(m, i | 0x80000000u)); + NUTS_SLEEP(10); + NUTS_PASS(nng_sendmsg(rep, m, 0)); + } + NUTS_CLOSE(req); + NUTS_CLOSE(rep); +} + +static void +test_xreq_close_pipe_during_send(void) +{ + nng_socket rep; + nng_socket req; + nng_msg * m; + nng_pipe p1; + nng_pipe p2; + + NUTS_PASS(nng_rep0_open_raw(&rep)); + NUTS_PASS(nng_req0_open_raw(&req)); + NUTS_PASS(nng_socket_set_ms(rep, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(req, NNG_OPT_SENDTIMEO, 100)); + NUTS_PASS(nng_socket_set_int(rep, NNG_OPT_RECVBUF, 5)); + NUTS_PASS(nng_socket_set_int(req, NNG_OPT_SENDBUF, 20)); + + NUTS_MARRY_EX(req, rep, NULL, &p1, &p2); + NUTS_TRUE(nng_pipe_id(p1) > 0); + NUTS_TRUE(nng_pipe_id(p2) > 0); + + for (unsigned i = 0; i < 20; i++) { + NUTS_PASS(nng_msg_alloc(&m, 4)); + NUTS_PASS(nng_msg_header_append_u32(m, i | 0x80000000u)); + NUTS_SLEEP(10); + NUTS_PASS(nng_sendmsg(req, m, 0)); + } + + NUTS_PASS(nng_pipe_close(p1)); + NUTS_CLOSE(req); + NUTS_CLOSE(rep); +} + +static void +test_xreq_ttl_option(void) +{ + nng_socket rep; + int v; + bool b; + size_t sz; + const char *opt = NNG_OPT_MAXTTL; + + NUTS_PASS(nng_req0_open_raw(&rep)); + + NUTS_PASS(nng_socket_set_int(rep, opt, 1)); + NUTS_FAIL(nng_socket_set_int(rep, opt, 0), NNG_EINVAL); + NUTS_FAIL(nng_socket_set_int(rep, opt, -1), NNG_EINVAL); + NUTS_FAIL(nng_socket_set_int(rep, opt, 16), NNG_EINVAL); + NUTS_FAIL(nng_socket_set_int(rep, opt, 256), NNG_EINVAL); + NUTS_PASS(nng_socket_set_int(rep, opt, 3)); + NUTS_PASS(nng_socket_get_int(rep, opt, &v)); + NUTS_TRUE(v == 3); + v = 0; + sz = sizeof(v); + NUTS_PASS(nng_socket_get(rep, opt, &v, &sz)); + NUTS_TRUE(v == 3); + NUTS_TRUE(sz == sizeof(v)); + + NUTS_TRUE(nng_socket_set(rep, opt, "", 1) == NNG_EINVAL); + sz = 1; + NUTS_TRUE(nng_socket_get(rep, opt, &v, &sz) == NNG_EINVAL); + NUTS_TRUE(nng_socket_set_bool(rep, opt, true) == NNG_EBADTYPE); + NUTS_TRUE(nng_socket_get_bool(rep, opt, &b) == NNG_EBADTYPE); + + NUTS_TRUE(nng_close(rep) == 0); +} + +NUTS_TESTS = { + { "xreq identity", test_xreq_identity }, + { "xreq raw", test_xreq_raw }, + { "xreq no context", test_xreq_no_context }, + { "xreq poll readable", test_xreq_poll_readable }, + { "xreq poll writable", test_xreq_poll_writeable }, + { "xreq validate peer", test_xreq_validate_peer }, + { "xreq recv aio stopped", test_xreq_recv_aio_stopped }, + { "xreq recv garbage", test_xreq_recv_garbage }, + { "xreq recv header", test_xreq_recv_header }, + { "xreq close during recv", test_xreq_close_during_recv }, + { "xreq close pipe during send", test_xreq_close_pipe_during_send }, + { "xreq ttl option", test_xreq_ttl_option }, + { NULL, NULL }, +}; diff --git a/src/sp/protocol/survey0/CMakeLists.txt b/src/sp/protocol/survey0/CMakeLists.txt new file mode 100644 index 00000000..b5daca41 --- /dev/null +++ b/src/sp/protocol/survey0/CMakeLists.txt @@ -0,0 +1,25 @@ +# +# Copyright 2020 Staysail Systems, Inc. +# Copyright 2018 Capitar IT Group BV +# +# This software is supplied under the terms of the MIT License, a +# copy of which should be located in the distribution where this +# file was obtained (LICENSE.txt). A copy of the license may also be +# found online at https://opensource.org/licenses/MIT. +# + +# Surveyor/Respondent protocol +nng_directory(survey0) + +nng_sources_if(NNG_PROTO_SURVEYOR0 survey.c xsurvey.c) +nng_headers_if(NNG_PROTO_SURVEYOR0 nng/protocol/survey0/survey.h) +nng_defines_if(NNG_PROTO_SURVEYOR0 NNG_HAVE_SURVEYOR0) + +nng_sources_if(NNG_PROTO_RESPONDENT0 respond.c xrespond.c) +nng_headers_if(NNG_PROTO_RESPONDENT0 nng/protocol/survey0/respond.h) +nng_defines_if(NNG_PROTO_RESPONDENT0 NNG_HAVE_RESPONDENT0) + +nng_test(respond_test) +nng_test(survey_test) +nng_test(xrespond_test) +nng_test(xsurvey_test) \ No newline at end of file diff --git a/src/sp/protocol/survey0/respond.c b/src/sp/protocol/survey0/respond.c new file mode 100644 index 00000000..ad551c8f --- /dev/null +++ b/src/sp/protocol/survey0/respond.c @@ -0,0 +1,693 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// Copyright 2018 Capitar IT Group BV +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include +#include + +#include "core/nng_impl.h" +#include "nng/protocol/survey0/respond.h" + +// Respondent protocol. The RESPONDENT protocol is the "replier" side of +// the surveyor pattern. This is useful for building service discovery, or +// voting algorithms, for example. + +#ifndef NNI_PROTO_SURVEYOR_V0 +#define NNI_PROTO_SURVEYOR_V0 NNI_PROTO(6, 2) +#endif + +#ifndef NNI_PROTO_RESPONDENT_V0 +#define NNI_PROTO_RESPONDENT_V0 NNI_PROTO(6, 3) +#endif + +typedef struct resp0_pipe resp0_pipe; +typedef struct resp0_sock resp0_sock; +typedef struct resp0_ctx resp0_ctx; + +static void resp0_pipe_send_cb(void *); +static void resp0_pipe_recv_cb(void *); +static void resp0_pipe_fini(void *); + +struct resp0_ctx { + resp0_sock * sock; + uint32_t pipe_id; + resp0_pipe * spipe; // send pipe + nni_aio * saio; // send aio + nni_aio * raio; // recv aio + nni_list_node sqnode; + nni_list_node rqnode; + size_t btrace_len; + uint32_t btrace[NNI_MAX_MAX_TTL + 1]; +}; + +// resp0_sock is our per-socket protocol private structure. +struct resp0_sock { + nni_mtx mtx; + nni_atomic_int ttl; + nni_id_map pipes; + resp0_ctx ctx; + nni_list recvpipes; + nni_list recvq; + nni_pollable readable; + nni_pollable writable; +}; + +// resp0_pipe is our per-pipe protocol private structure. +struct resp0_pipe { + nni_pipe * npipe; + resp0_sock * psock; + bool busy; + bool closed; + uint32_t id; + nni_list sendq; // contexts waiting to send + nni_aio aio_send; + nni_aio aio_recv; + nni_list_node rnode; // receivable linkage +}; + +static void +resp0_ctx_close(void *arg) +{ + resp0_ctx * ctx = arg; + resp0_sock *s = ctx->sock; + nni_aio * aio; + + // complete any outstanding operations here, cancellation, etc. + + nni_mtx_lock(&s->mtx); + if ((aio = ctx->saio) != NULL) { + resp0_pipe *p = ctx->spipe; + ctx->saio = NULL; + ctx->spipe = NULL; + nni_list_remove(&p->sendq, ctx); + nni_aio_finish_error(aio, NNG_ECLOSED); + } + if ((aio = ctx->raio) != NULL) { + ctx->raio = NULL; + nni_list_remove(&s->recvq, ctx); + nni_aio_finish_error(aio, NNG_ECLOSED); + } + nni_mtx_unlock(&s->mtx); +} + +static void +resp0_ctx_fini(void *arg) +{ + resp0_ctx *ctx = arg; + + resp0_ctx_close(ctx); +} + +static int +resp0_ctx_init(void *carg, void *sarg) +{ + resp0_sock *s = sarg; + resp0_ctx * ctx = carg; + + NNI_LIST_NODE_INIT(&ctx->sqnode); + NNI_LIST_NODE_INIT(&ctx->rqnode); + ctx->btrace_len = 0; + ctx->sock = s; + ctx->pipe_id = 0; + + return (0); +} + +static void +resp0_ctx_cancel_send(nni_aio *aio, void *arg, int rv) +{ + resp0_ctx * ctx = arg; + resp0_sock *s = ctx->sock; + + nni_mtx_lock(&s->mtx); + if (ctx->saio != aio) { + nni_mtx_unlock(&s->mtx); + return; + } + nni_list_node_remove(&ctx->sqnode); + ctx->saio = NULL; + nni_mtx_unlock(&s->mtx); + nni_msg_header_clear(nni_aio_get_msg(aio)); // reset the headers + nni_aio_finish_error(aio, rv); +} + +static void +resp0_ctx_send(void *arg, nni_aio *aio) +{ + resp0_ctx * ctx = arg; + resp0_sock *s = ctx->sock; + resp0_pipe *p; + nni_msg * msg; + size_t len; + uint32_t pid; + int rv; + + if (nni_aio_begin(aio) != 0) { + return; + } + msg = nni_aio_get_msg(aio); + nni_msg_header_clear(msg); + + if (ctx == &s->ctx) { + // We can't send anymore, because only one send per request. + nni_pollable_clear(&s->writable); + } + + nni_mtx_lock(&s->mtx); + if ((rv = nni_aio_schedule(aio, resp0_ctx_cancel_send, ctx)) != 0) { + nni_mtx_unlock(&s->mtx); + nni_aio_finish_error(aio, rv); + return; + } + + if ((len = ctx->btrace_len) == 0) { + nni_mtx_unlock(&s->mtx); + nni_aio_finish_error(aio, NNG_ESTATE); + return; + } + pid = ctx->pipe_id; + ctx->pipe_id = 0; + ctx->btrace_len = 0; + + if ((rv = nni_msg_header_append(msg, ctx->btrace, len)) != 0) { + nni_mtx_unlock(&s->mtx); + nni_aio_finish_error(aio, rv); + return; + } + + if ((p = nni_id_get(&s->pipes, pid)) == NULL) { + // Surveyor has left the building. Just discard the reply. + nni_mtx_unlock(&s->mtx); + nni_aio_set_msg(aio, NULL); + nni_aio_finish(aio, 0, nni_msg_len(msg)); + nni_msg_free(msg); + return; + } + + if (!p->busy) { + p->busy = true; + len = nni_msg_len(msg); + nni_aio_set_msg(&p->aio_send, msg); + nni_pipe_send(p->npipe, &p->aio_send); + nni_mtx_unlock(&s->mtx); + + nni_aio_set_msg(aio, NULL); + nni_aio_finish(aio, 0, len); + return; + } + + ctx->saio = aio; + ctx->spipe = p; + nni_list_append(&p->sendq, ctx); + nni_mtx_unlock(&s->mtx); +} + +static void +resp0_sock_fini(void *arg) +{ + resp0_sock *s = arg; + + nni_id_map_fini(&s->pipes); + resp0_ctx_fini(&s->ctx); + nni_pollable_fini(&s->writable); + nni_pollable_fini(&s->readable); + nni_mtx_fini(&s->mtx); +} + +static int +resp0_sock_init(void *arg, nni_sock *nsock) +{ + resp0_sock *s = arg; + + NNI_ARG_UNUSED(nsock); + + nni_mtx_init(&s->mtx); + nni_id_map_init(&s->pipes, 0, 0, false); + + NNI_LIST_INIT(&s->recvq, resp0_ctx, rqnode); + NNI_LIST_INIT(&s->recvpipes, resp0_pipe, rnode); + + nni_atomic_init(&s->ttl); + nni_atomic_set(&s->ttl, 8); // Per RFC + + (void) resp0_ctx_init(&s->ctx, s); + + // We start off without being either readable or writable. + // Readability comes when there is something on the socket. + nni_pollable_init(&s->writable); + nni_pollable_init(&s->readable); + return (0); +} + +static void +resp0_sock_open(void *arg) +{ + NNI_ARG_UNUSED(arg); +} + +static void +resp0_sock_close(void *arg) +{ + resp0_sock *s = arg; + + resp0_ctx_close(&s->ctx); +} + +static void +resp0_pipe_stop(void *arg) +{ + resp0_pipe *p = arg; + + nni_aio_stop(&p->aio_send); + nni_aio_stop(&p->aio_recv); +} + +static void +resp0_pipe_fini(void *arg) +{ + resp0_pipe *p = arg; + nng_msg * msg; + + if ((msg = nni_aio_get_msg(&p->aio_recv)) != NULL) { + nni_aio_set_msg(&p->aio_recv, NULL); + nni_msg_free(msg); + } + nni_aio_fini(&p->aio_send); + nni_aio_fini(&p->aio_recv); +} + +static int +resp0_pipe_init(void *arg, nni_pipe *npipe, void *s) +{ + resp0_pipe *p = arg; + + nni_aio_init(&p->aio_recv, resp0_pipe_recv_cb, p); + nni_aio_init(&p->aio_send, resp0_pipe_send_cb, p); + + NNI_LIST_INIT(&p->sendq, resp0_ctx, sqnode); + + p->npipe = npipe; + p->psock = s; + p->busy = false; + p->id = nni_pipe_id(npipe); + + return (0); +} + +static int +resp0_pipe_start(void *arg) +{ + resp0_pipe *p = arg; + resp0_sock *s = p->psock; + int rv; + + if (nni_pipe_peer(p->npipe) != NNI_PROTO_SURVEYOR_V0) { + return (NNG_EPROTO); + } + + nni_mtx_lock(&s->mtx); + rv = nni_id_set(&s->pipes, p->id, p); + nni_mtx_unlock(&s->mtx); + if (rv != 0) { + return (rv); + } + + nni_pipe_recv(p->npipe, &p->aio_recv); + return (rv); +} + +static void +resp0_pipe_close(void *arg) +{ + resp0_pipe *p = arg; + resp0_sock *s = p->psock; + resp0_ctx * ctx; + + nni_aio_close(&p->aio_send); + nni_aio_close(&p->aio_recv); + + nni_mtx_lock(&s->mtx); + p->closed = true; + while ((ctx = nni_list_first(&p->sendq)) != NULL) { + nni_aio *aio; + nni_msg *msg; + nni_list_remove(&p->sendq, ctx); + aio = ctx->saio; + ctx->saio = NULL; + msg = nni_aio_get_msg(aio); + nni_aio_set_msg(aio, NULL); + nni_aio_finish(aio, 0, nni_msg_len(msg)); + nni_msg_free(msg); + } + if (p->id == s->ctx.pipe_id) { + // Make sure user space knows they can send a message to us, + // which we will happily discard. + nni_pollable_raise(&s->writable); + } + nni_id_remove(&s->pipes, p->id); + nni_mtx_unlock(&s->mtx); +} + +static void +resp0_pipe_send_cb(void *arg) +{ + resp0_pipe *p = arg; + resp0_sock *s = p->psock; + resp0_ctx * ctx; + nni_aio * aio; + nni_msg * msg; + size_t len; + + if (nni_aio_result(&p->aio_send) != 0) { + nni_msg_free(nni_aio_get_msg(&p->aio_send)); + nni_aio_set_msg(&p->aio_send, NULL); + nni_pipe_close(p->npipe); + return; + } + nni_mtx_lock(&s->mtx); + p->busy = false; + if ((ctx = nni_list_first(&p->sendq)) == NULL) { + // Nothing else to send. + if (p->id == s->ctx.pipe_id) { + // Mark us ready for the other side to send! + nni_pollable_raise(&s->writable); + } + nni_mtx_unlock(&s->mtx); + return; + } + + nni_list_remove(&p->sendq, ctx); + aio = ctx->saio; + ctx->saio = NULL; + ctx->spipe = NULL; + p->busy = true; + msg = nni_aio_get_msg(aio); + len = nni_msg_len(msg); + nni_aio_set_msg(aio, NULL); + nni_aio_set_msg(&p->aio_send, msg); + nni_pipe_send(p->npipe, &p->aio_send); + + nni_mtx_unlock(&s->mtx); + + nni_aio_finish_sync(aio, 0, len); +} + +static void +resp0_cancel_recv(nni_aio *aio, void *arg, int rv) +{ + resp0_ctx * ctx = arg; + resp0_sock *s = ctx->sock; + + nni_mtx_lock(&s->mtx); + if (ctx->raio == aio) { + nni_list_remove(&s->recvq, ctx); + ctx->raio = NULL; + nni_aio_finish_error(aio, rv); + } + nni_mtx_unlock(&s->mtx); +} + +static void +resp0_ctx_recv(void *arg, nni_aio *aio) +{ + resp0_ctx * ctx = arg; + resp0_sock *s = ctx->sock; + resp0_pipe *p; + size_t len; + nni_msg * msg; + + if (nni_aio_begin(aio) != 0) { + return; + } + nni_mtx_lock(&s->mtx); + if ((p = nni_list_first(&s->recvpipes)) == NULL) { + int rv; + rv = nni_aio_schedule(aio, resp0_cancel_recv, ctx); + if (rv != 0) { + nni_mtx_unlock(&s->mtx); + nni_aio_finish_error(aio, rv); + return; + } + // We cannot have two concurrent receive requests on the same + // context... + if (ctx->raio != NULL) { + nni_mtx_unlock(&s->mtx); + nni_aio_finish_error(aio, NNG_ESTATE); + return; + } + ctx->raio = aio; + nni_list_append(&s->recvq, ctx); + nni_mtx_unlock(&s->mtx); + return; + } + msg = nni_aio_get_msg(&p->aio_recv); + nni_aio_set_msg(&p->aio_recv, NULL); + nni_list_remove(&s->recvpipes, p); + if (nni_list_empty(&s->recvpipes)) { + nni_pollable_clear(&s->readable); + } + nni_pipe_recv(p->npipe, &p->aio_recv); + + len = nni_msg_header_len(msg); + memcpy(ctx->btrace, nni_msg_header(msg), len); + ctx->btrace_len = len; + ctx->pipe_id = p->id; + if (ctx == &s->ctx) { + nni_pollable_raise(&s->writable); + } + nni_mtx_unlock(&s->mtx); + + nni_msg_header_clear(msg); + nni_aio_set_msg(aio, msg); + nni_aio_finish(aio, 0, nni_msg_len(msg)); +} + +static void +resp0_pipe_recv_cb(void *arg) +{ + resp0_pipe *p = arg; + resp0_sock *s = p->psock; + resp0_ctx * ctx; + nni_msg * msg; + nni_aio * aio; + int hops; + size_t len; + int ttl; + + if (nni_aio_result(&p->aio_recv) != 0) { + nni_pipe_close(p->npipe); + return; + } + + ttl = nni_atomic_get(&s->ttl); + msg = nni_aio_get_msg(&p->aio_recv); + nni_msg_set_pipe(msg, p->id); + + // Move backtrace from body to header + hops = 1; + for (;;) { + bool end = 0; + uint8_t *body; + + if (hops > ttl) { + goto drop; + } + hops++; + if (nni_msg_len(msg) < 4) { + // Peer is speaking garbage, kick it. + nni_msg_free(msg); + nni_aio_set_msg(&p->aio_recv, NULL); + nni_pipe_close(p->npipe); + return; + } + body = nni_msg_body(msg); + end = ((body[0] & 0x80u) != 0); + if (nni_msg_header_append(msg, body, 4) != 0) { + goto drop; + } + nni_msg_trim(msg, 4); + if (end) { + break; + } + } + + len = nni_msg_header_len(msg); + + nni_mtx_lock(&s->mtx); + + if (p->closed) { + // If pipe was closed, we just abandon the data from it. + nni_aio_set_msg(&p->aio_recv, NULL); + nni_mtx_unlock(&s->mtx); + nni_msg_free(msg); + return; + } + if ((ctx = nni_list_first(&s->recvq)) == NULL) { + // No one blocked in recv, stall. + nni_list_append(&s->recvpipes, p); + nni_pollable_raise(&s->readable); + nni_mtx_unlock(&s->mtx); + return; + } + + nni_list_remove(&s->recvq, ctx); + aio = ctx->raio; + ctx->raio = NULL; + nni_aio_set_msg(&p->aio_recv, NULL); + + // Start the next receive. + nni_pipe_recv(p->npipe, &p->aio_recv); + + ctx->btrace_len = len; + memcpy(ctx->btrace, nni_msg_header(msg), len); + nni_msg_header_clear(msg); + ctx->pipe_id = p->id; + + if ((ctx == &s->ctx) && (!p->busy)) { + nni_pollable_raise(&s->writable); + } + nni_mtx_unlock(&s->mtx); + + nni_aio_set_msg(aio, msg); + nni_aio_finish_sync(aio, 0, nni_msg_len(msg)); + return; + +drop: + nni_msg_free(msg); + nni_aio_set_msg(&p->aio_recv, NULL); + nni_pipe_recv(p->npipe, &p->aio_recv); +} + +static int +resp0_sock_set_max_ttl(void *arg, const void *buf, size_t sz, nni_opt_type t) +{ + resp0_sock *s = arg; + int ttl; + int rv; + + if ((rv = nni_copyin_int(&ttl, buf, sz, 1, NNI_MAX_MAX_TTL, t)) == 0) { + nni_atomic_set(&s->ttl, ttl); + } + return (rv); +} + +static int +resp0_sock_get_max_ttl(void *arg, void *buf, size_t *szp, nni_opt_type t) +{ + resp0_sock *s = arg; + return (nni_copyout_int(nni_atomic_get(&s->ttl), buf, szp, t)); +} + +static int +resp0_sock_get_sendfd(void *arg, void *buf, size_t *szp, nni_opt_type t) +{ + resp0_sock *s = arg; + int rv; + int fd; + + if ((rv = nni_pollable_getfd(&s->writable, &fd)) != 0) { + return (rv); + } + return (nni_copyout_int(fd, buf, szp, t)); +} + +static int +resp0_sock_get_recvfd(void *arg, void *buf, size_t *szp, nni_opt_type t) +{ + resp0_sock *s = arg; + int rv; + int fd; + + if ((rv = nni_pollable_getfd(&s->readable, &fd)) != 0) { + return (rv); + } + return (nni_copyout_int(fd, buf, szp, t)); +} + +static void +resp0_sock_send(void *arg, nni_aio *aio) +{ + resp0_sock *s = arg; + + resp0_ctx_send(&s->ctx, aio); +} + +static void +resp0_sock_recv(void *arg, nni_aio *aio) +{ + resp0_sock *s = arg; + + resp0_ctx_recv(&s->ctx, aio); +} + +static nni_proto_pipe_ops resp0_pipe_ops = { + .pipe_size = sizeof(resp0_pipe), + .pipe_init = resp0_pipe_init, + .pipe_fini = resp0_pipe_fini, + .pipe_start = resp0_pipe_start, + .pipe_close = resp0_pipe_close, + .pipe_stop = resp0_pipe_stop, +}; + +static nni_proto_ctx_ops resp0_ctx_ops = { + .ctx_size = sizeof(resp0_ctx), + .ctx_init = resp0_ctx_init, + .ctx_fini = resp0_ctx_fini, + .ctx_send = resp0_ctx_send, + .ctx_recv = resp0_ctx_recv, +}; + +static nni_option resp0_sock_options[] = { + { + .o_name = NNG_OPT_MAXTTL, + .o_get = resp0_sock_get_max_ttl, + .o_set = resp0_sock_set_max_ttl, + }, + { + .o_name = NNG_OPT_RECVFD, + .o_get = resp0_sock_get_recvfd, + .o_set = NULL, + }, + { + .o_name = NNG_OPT_SENDFD, + .o_get = resp0_sock_get_sendfd, + .o_set = NULL, + }, + // terminate list + { + .o_name = NULL, + }, +}; + +static nni_proto_sock_ops resp0_sock_ops = { + .sock_size = sizeof(resp0_sock), + .sock_init = resp0_sock_init, + .sock_fini = resp0_sock_fini, + .sock_open = resp0_sock_open, + .sock_close = resp0_sock_close, + .sock_send = resp0_sock_send, + .sock_recv = resp0_sock_recv, + .sock_options = resp0_sock_options, +}; + +static nni_proto resp0_proto = { + .proto_version = NNI_PROTOCOL_VERSION, + .proto_self = { NNI_PROTO_RESPONDENT_V0, "respondent" }, + .proto_peer = { NNI_PROTO_SURVEYOR_V0, "surveyor" }, + .proto_flags = NNI_PROTO_FLAG_SNDRCV, + .proto_sock_ops = &resp0_sock_ops, + .proto_pipe_ops = &resp0_pipe_ops, + .proto_ctx_ops = &resp0_ctx_ops, +}; + +int +nng_respondent0_open(nng_socket *sidp) +{ + return (nni_proto_open(sidp, &resp0_proto)); +} diff --git a/src/sp/protocol/survey0/respond_test.c b/src/sp/protocol/survey0/respond_test.c new file mode 100644 index 00000000..51844c76 --- /dev/null +++ b/src/sp/protocol/survey0/respond_test.c @@ -0,0 +1,586 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include + +void +test_resp_identity(void) +{ + nng_socket s; + int p; + char * n; + + NUTS_PASS(nng_respondent0_open(&s)); + NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PROTO, &p)); + NUTS_TRUE(p == NNG_RESPONDENT0_SELF); + NUTS_TRUE(nng_socket_get_int(s, NNG_OPT_PEER, &p) == 0); + NUTS_TRUE(p == NNG_RESPONDENT0_PEER); + NUTS_TRUE(nng_socket_get_string(s, NNG_OPT_PROTONAME, &n) == 0); + NUTS_MATCH(n, NNG_RESPONDENT0_SELF_NAME); + nng_strfree(n); + NUTS_TRUE(nng_socket_get_string(s, NNG_OPT_PEERNAME, &n) == 0); + NUTS_MATCH(n, NNG_RESPONDENT0_PEER_NAME); + nng_strfree(n); + NUTS_CLOSE(s); +} + +void +test_resp_send_bad_state(void) +{ + nng_socket resp; + nng_msg * msg = NULL; + + NUTS_PASS(nng_respondent0_open(&resp)); + NUTS_PASS(nng_msg_alloc(&msg, 0)); + NUTS_FAIL(nng_sendmsg(resp, msg, 0), NNG_ESTATE); + nng_msg_free(msg); + NUTS_CLOSE(resp); +} + +void +test_resp_poll_writeable(void) +{ + int fd; + nng_socket surv; + nng_socket resp; + + NUTS_PASS(nng_surveyor0_open(&surv)); + NUTS_PASS(nng_respondent0_open(&resp)); + NUTS_PASS(nng_socket_get_int(resp, NNG_OPT_SENDFD, &fd)); + NUTS_TRUE(fd >= 0); + + // Not writable before connect. + NUTS_TRUE(nuts_poll_fd(fd) == false); + + NUTS_MARRY(surv, resp); + + // Still not writable. + NUTS_TRUE(nuts_poll_fd(fd) == false); + + // If we get a job, *then* we become writable + NUTS_SEND(surv, "abc"); + NUTS_RECV(resp, "abc"); + NUTS_TRUE(nuts_poll_fd(fd) == true); + + // And is no longer writable once we send a message + NUTS_SEND(resp, "def"); + NUTS_TRUE(nuts_poll_fd(fd) == false); + // Even after receiving it + NUTS_RECV(surv, "def"); + NUTS_TRUE(nuts_poll_fd(fd) == false); + + NUTS_CLOSE(surv); + NUTS_CLOSE(resp); +} + +void +test_resp_poll_readable(void) +{ + int fd; + nng_socket surv; + nng_socket resp; + nng_msg * msg; + + NUTS_PASS(nng_surveyor0_open(&surv)); + NUTS_PASS(nng_respondent0_open(&resp)); + NUTS_PASS(nng_socket_get_int(resp, NNG_OPT_RECVFD, &fd)); + NUTS_TRUE(fd >= 0); + + // Not readable if not connected! + NUTS_TRUE(nuts_poll_fd(fd) == false); + + // Even after connect (no message yet) + NUTS_MARRY(surv, resp); + NUTS_TRUE(nuts_poll_fd(fd) == false); + + // But once we send messages, it is. + // We have to send a request, in order to send a reply. + NUTS_SEND(surv, "abc"); + NUTS_SLEEP(100); + + NUTS_TRUE(nuts_poll_fd(fd) == true); + + // and receiving makes it no longer ready + NUTS_PASS(nng_recvmsg(resp, &msg, 0)); + nng_msg_free(msg); + NUTS_TRUE(nuts_poll_fd(fd) == false); + + // TODO verify unsolicited response + + NUTS_CLOSE(surv); + NUTS_CLOSE(resp); +} + +void +test_resp_context_no_poll(void) +{ + int fd; + nng_socket resp; + nng_ctx ctx; + + NUTS_PASS(nng_respondent0_open(&resp)); + NUTS_PASS(nng_ctx_open(&ctx, resp)); + NUTS_FAIL(nng_ctx_get_int(ctx, NNG_OPT_SENDFD, &fd), NNG_ENOTSUP); + NUTS_FAIL(nng_ctx_get_int(ctx, NNG_OPT_RECVFD, &fd), NNG_ENOTSUP); + NUTS_PASS(nng_ctx_close(ctx)); + NUTS_CLOSE(resp); +} + +void +test_resp_validate_peer(void) +{ + nng_socket s1, s2; + nng_stat * stats; + nng_stat * reject; + char * addr; + + NUTS_ADDR(addr, "inproc"); + + NUTS_PASS(nng_respondent0_open(&s1)); + NUTS_PASS(nng_respondent0_open(&s2)); + + NUTS_PASS(nng_listen(s1, addr, NULL, 0)); + NUTS_PASS(nng_dial(s2, addr, NULL, NNG_FLAG_NONBLOCK)); + + NUTS_SLEEP(100); + NUTS_PASS(nng_stats_get(&stats)); + + NUTS_TRUE(stats != NULL); + NUTS_TRUE((reject = nng_stat_find_socket(stats, s1)) != NULL); + NUTS_TRUE((reject = nng_stat_find(reject, "reject")) != NULL); + + NUTS_TRUE(nng_stat_type(reject) == NNG_STAT_COUNTER); + NUTS_TRUE(nng_stat_value(reject) > 0); + + NUTS_CLOSE(s1); + NUTS_CLOSE(s2); + nng_stats_free(stats); +} + +void +test_resp_double_recv(void) +{ + nng_socket s1; + nng_aio * aio1; + nng_aio * aio2; + + NUTS_PASS(nng_respondent0_open(&s1)); + NUTS_PASS(nng_aio_alloc(&aio1, NULL, NULL)); + NUTS_PASS(nng_aio_alloc(&aio2, NULL, NULL)); + + nng_recv_aio(s1, aio1); + nng_recv_aio(s1, aio2); + + nng_aio_wait(aio2); + NUTS_FAIL(nng_aio_result(aio2), NNG_ESTATE); + NUTS_CLOSE(s1); + NUTS_FAIL(nng_aio_result(aio1), NNG_ECLOSED); + nng_aio_free(aio1); + nng_aio_free(aio2); +} + +void +test_resp_close_pipe_before_send(void) +{ + nng_socket resp; + nng_socket surv; + nng_pipe p; + nng_aio * aio1; + nng_msg * m; + + NUTS_PASS(nng_respondent0_open(&resp)); + NUTS_PASS(nng_surveyor0_open(&surv)); + NUTS_PASS(nng_setopt_ms(resp, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_setopt_ms(resp, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_setopt_ms(surv, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_aio_alloc(&aio1, NULL, NULL)); + + NUTS_MARRY(surv, resp); + NUTS_SEND(surv, "test"); + + nng_recv_aio(resp, aio1); + nng_aio_wait(aio1); + NUTS_PASS(nng_aio_result(aio1)); + NUTS_TRUE((m = nng_aio_get_msg(aio1)) != NULL); + p = nng_msg_get_pipe(m); + NUTS_PASS(nng_pipe_close(p)); + NUTS_PASS(nng_sendmsg(resp, m, 0)); + + NUTS_CLOSE(surv); + NUTS_CLOSE(resp); + nng_aio_free(aio1); +} + +void +test_resp_close_pipe_during_send(void) +{ + nng_socket resp; + nng_socket surv; + nng_pipe p = NNG_PIPE_INITIALIZER; + nng_msg * m; + + NUTS_PASS(nng_respondent0_open(&resp)); + NUTS_PASS(nng_surveyor0_open_raw(&surv)); + NUTS_PASS(nng_setopt_ms(resp, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_setopt_ms(resp, NNG_OPT_SENDTIMEO, 200)); + NUTS_PASS(nng_setopt_ms(surv, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_setopt_int(resp, NNG_OPT_SENDBUF, 20)); + NUTS_PASS(nng_setopt_int(resp, NNG_OPT_RECVBUF, 20)); + NUTS_PASS(nng_setopt_int(surv, NNG_OPT_SENDBUF, 20)); + NUTS_PASS(nng_setopt_int(surv, NNG_OPT_RECVBUF, 1)); + + NUTS_MARRY(surv, resp); + + for (int i = 0; i < 100; i++) { + int rv; + NUTS_PASS(nng_msg_alloc(&m, 4)); + NUTS_PASS(nng_msg_append_u32(m, (unsigned) i | 0x80000000u)); + NUTS_PASS(nng_sendmsg(surv, m, 0)); + NUTS_PASS(nng_recvmsg(resp, &m, 0)); + p = nng_msg_get_pipe(m); + rv = nng_sendmsg(resp, m, 0); + if (rv == NNG_ETIMEDOUT) { + // Queue is backed up, senders are busy. + nng_msg_free(m); + break; + } + NUTS_PASS(rv); + } + NUTS_PASS(nng_pipe_close(p)); + + NUTS_CLOSE(surv); + NUTS_CLOSE(resp); +} + +void +test_resp_ctx_recv_aio_stopped(void) +{ + nng_socket resp; + nng_ctx ctx; + nng_aio * aio; + + NUTS_PASS(nng_respondent0_open(&resp)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + NUTS_PASS(nng_ctx_open(&ctx, resp)); + + nng_aio_stop(aio); + nng_ctx_recv(ctx, aio); + nng_aio_wait(aio); + NUTS_FAIL(nng_aio_result(aio), NNG_ECANCELED); + NUTS_PASS(nng_ctx_close(ctx)); + NUTS_CLOSE(resp); + nng_aio_free(aio); +} + +void +test_resp_close_pipe_context_send(void) +{ + nng_socket resp; + nng_socket surv; + nng_pipe p = NNG_PIPE_INITIALIZER; + nng_msg * m; + nng_ctx ctx[10]; + nng_aio * aio[10]; + int i; + + NUTS_PASS(nng_respondent0_open(&resp)); + NUTS_PASS(nng_surveyor0_open_raw(&surv)); + NUTS_PASS(nng_setopt_ms(resp, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_setopt_ms(resp, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_setopt_ms(surv, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_setopt_int(resp, NNG_OPT_SENDBUF, 1)); + NUTS_PASS(nng_setopt_int(resp, NNG_OPT_RECVBUF, 1)); + NUTS_PASS(nng_setopt_int(surv, NNG_OPT_SENDBUF, 1)); + NUTS_PASS(nng_setopt_int(surv, NNG_OPT_RECVBUF, 1)); + for (i = 0; i < 10; i++) { + NUTS_PASS(nng_ctx_open(&ctx[i], resp)); + NUTS_PASS(nng_aio_alloc(&aio[i], NULL, NULL)); + } + + NUTS_MARRY(surv, resp); + + for (i = 0; i < 10; i++) { + NUTS_PASS(nng_msg_alloc(&m, 4)); + NUTS_PASS(nng_msg_append_u32(m, (unsigned) i | 0x80000000u)); + NUTS_PASS(nng_sendmsg(surv, m, 0)); + nng_ctx_recv(ctx[i], aio[i]); + } + for (i = 0; i < 10; i++) { + nng_aio_wait(aio[i]); + NUTS_PASS(nng_aio_result(aio[i])); + NUTS_TRUE((m = nng_aio_get_msg(aio[i])) != NULL); + p = nng_msg_get_pipe(m); + nng_aio_set_msg(aio[i], m); + nng_ctx_send(ctx[i], aio[i]); + } + + // Note that SURVEYOR socket is not reading the results. + NUTS_PASS(nng_pipe_close(p)); + + for (i = 0; i < 10; i++) { + int rv; + nng_aio_wait(aio[i]); + rv = nng_aio_result(aio[i]); + if (rv != 0) { + NUTS_FAIL(rv, NNG_ECLOSED); + nng_msg_free(nng_aio_get_msg(aio[i])); + } + nng_aio_free(aio[i]); + NUTS_PASS(nng_ctx_close(ctx[i])); + } + NUTS_CLOSE(surv); + NUTS_CLOSE(resp); +} + +void +test_resp_close_context_send(void) +{ + nng_socket resp; + nng_socket surv; + nng_msg * m; + nng_ctx ctx[10]; + nng_aio * aio[10]; + int i; + + NUTS_PASS(nng_respondent0_open(&resp)); + NUTS_PASS(nng_surveyor0_open_raw(&surv)); + NUTS_PASS(nng_setopt_ms(resp, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_setopt_ms(resp, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_setopt_ms(surv, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_setopt_int(resp, NNG_OPT_SENDBUF, 1)); + NUTS_PASS(nng_setopt_int(resp, NNG_OPT_RECVBUF, 1)); + NUTS_PASS(nng_setopt_int(surv, NNG_OPT_SENDBUF, 1)); + NUTS_PASS(nng_setopt_int(surv, NNG_OPT_RECVBUF, 1)); + for (i = 0; i < 10; i++) { + NUTS_PASS(nng_ctx_open(&ctx[i], resp)); + NUTS_PASS(nng_aio_alloc(&aio[i], NULL, NULL)); + } + + NUTS_MARRY(surv, resp); + + for (i = 0; i < 10; i++) { + NUTS_PASS(nng_msg_alloc(&m, 4)); + NUTS_PASS(nng_msg_append_u32(m, (unsigned) i | 0x80000000u)); + NUTS_PASS(nng_sendmsg(surv, m, 0)); + nng_ctx_recv(ctx[i], aio[i]); + } + for (i = 0; i < 10; i++) { + nng_aio_wait(aio[i]); + NUTS_PASS(nng_aio_result(aio[i])); + NUTS_TRUE((m = nng_aio_get_msg(aio[i])) != NULL); + nng_aio_set_msg(aio[i], m); + nng_ctx_send(ctx[i], aio[i]); + } + + // Note that REQ socket is not reading the results. + for (i = 0; i < 10; i++) { + int rv; + NUTS_PASS(nng_ctx_close(ctx[i])); + nng_aio_wait(aio[i]); + rv = nng_aio_result(aio[i]); + if (rv != 0) { + NUTS_FAIL(rv, NNG_ECLOSED); + nng_msg_free(nng_aio_get_msg(aio[i])); + } + nng_aio_free(aio[i]); + } + NUTS_CLOSE(surv); + NUTS_CLOSE(resp); +} + +static void +test_resp_ctx_recv_nonblock(void) +{ + nng_socket resp; + nng_ctx ctx; + nng_aio * aio; + + NUTS_PASS(nng_respondent0_open(&resp)); + NUTS_PASS(nng_ctx_open(&ctx, resp)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + + nng_aio_set_timeout(aio, 0); // Instant timeout + nng_ctx_recv(ctx, aio); + + nng_aio_wait(aio); + NUTS_FAIL(nng_aio_result(aio), NNG_ETIMEDOUT); + NUTS_CLOSE(resp); + nng_aio_free(aio); +} + +static void +test_resp_ctx_send_nonblock(void) +{ + nng_socket resp; + nng_socket surv; + nng_ctx ctx; + nng_aio * aio; + nng_msg * msg; + + NUTS_PASS(nng_surveyor0_open(&surv)); + NUTS_PASS(nng_respondent0_open(&resp)); + NUTS_PASS(nng_setopt_ms(surv, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_setopt_ms(resp, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_setopt_ms(resp, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_ctx_open(&ctx, resp)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + NUTS_MARRY(surv, resp); + + NUTS_SEND(surv, "SEND"); + nng_ctx_recv(ctx, aio); + nng_aio_wait(aio); + NUTS_PASS(nng_aio_result(aio)); + // message carries over + msg = nng_aio_get_msg(aio); + nng_aio_set_msg(aio, msg); + nng_aio_set_timeout(aio, 0); // Instant timeout + nng_ctx_send(ctx, aio); + + nng_aio_wait(aio); + NUTS_FAIL(nng_aio_result(aio), NNG_ETIMEDOUT); + NUTS_CLOSE(surv); + NUTS_CLOSE(resp); + nng_aio_free(aio); + nng_msg_free(msg); +} + +void +test_resp_recv_garbage(void) +{ + nng_socket resp; + nng_socket surv; + nng_msg * m; + + NUTS_PASS(nng_respondent0_open(&resp)); + NUTS_PASS(nng_surveyor0_open_raw(&surv)); + NUTS_PASS(nng_setopt_ms(resp, NNG_OPT_RECVTIMEO, 200)); + NUTS_PASS(nng_setopt_ms(resp, NNG_OPT_SENDTIMEO, 200)); + NUTS_PASS(nng_setopt_ms(surv, NNG_OPT_SENDTIMEO, 1000)); + + NUTS_MARRY(surv, resp); + + NUTS_PASS(nng_msg_alloc(&m, 4)); + NUTS_PASS(nng_msg_append_u32(m, 1u)); + NUTS_PASS(nng_sendmsg(surv, m, 0)); + NUTS_FAIL(nng_recvmsg(resp, &m, 0), NNG_ETIMEDOUT); + + NUTS_CLOSE(surv); + NUTS_CLOSE(resp); +} + +static void +test_resp_ttl_option(void) +{ + nng_socket resp; + int v; + bool b; + size_t sz; + const char *opt = NNG_OPT_MAXTTL; + + NUTS_PASS(nng_respondent0_open(&resp)); + + NUTS_PASS(nng_setopt_int(resp, opt, 1)); + NUTS_FAIL(nng_setopt_int(resp, opt, 0), NNG_EINVAL); + NUTS_FAIL(nng_setopt_int(resp, opt, -1), NNG_EINVAL); + NUTS_FAIL(nng_setopt_int(resp, opt, 16), NNG_EINVAL); + NUTS_FAIL(nng_setopt_int(resp, opt, 256), NNG_EINVAL); + NUTS_PASS(nng_setopt_int(resp, opt, 3)); + NUTS_PASS(nng_socket_get_int(resp, opt, &v)); + NUTS_TRUE(v == 3); + v = 0; + sz = sizeof(v); + NUTS_PASS(nng_socket_get(resp, opt, &v, &sz)); + NUTS_TRUE(v == 3); + NUTS_TRUE(sz == sizeof(v)); + + NUTS_FAIL(nng_setopt(resp, opt, "", 1), NNG_EINVAL); + sz = 1; + NUTS_FAIL(nng_socket_get(resp, opt, &v, &sz), NNG_EINVAL); + NUTS_FAIL(nng_setopt_bool(resp, opt, true), NNG_EBADTYPE); + NUTS_FAIL(nng_socket_get_bool(resp, opt, &b), NNG_EBADTYPE); + + NUTS_CLOSE(resp); +} + +static void +test_resp_ttl_drop(void) +{ + nng_socket resp; + nng_socket surv; + nng_msg * m; + + NUTS_PASS(nng_respondent0_open(&resp)); + NUTS_PASS(nng_surveyor0_open_raw(&surv)); + NUTS_PASS(nng_setopt_int(resp, NNG_OPT_MAXTTL, 3)); + NUTS_PASS(nng_setopt_ms(resp, NNG_OPT_RECVTIMEO, 200)); + NUTS_PASS(nng_setopt_ms(surv, NNG_OPT_SENDTIMEO, 1000)); + + NUTS_MARRY(surv, resp); + + // Send messages. Note that xrep implicitly adds a hop on receive. + + NUTS_PASS(nng_msg_alloc(&m, 0)); + NUTS_PASS(nng_msg_append_u32(m, 1u)); // 2 hops + NUTS_PASS(nng_msg_append_u32(m, 0x80000001u)); + NUTS_PASS(nng_msg_append(m, "PASS1", 6)); + NUTS_PASS(nng_sendmsg(surv, m, 0)); + + NUTS_PASS(nng_msg_alloc(&m, 0)); + NUTS_PASS(nng_msg_append_u32(m, 1u)); // 4 hops -- discard! + NUTS_PASS(nng_msg_append_u32(m, 2u)); + NUTS_PASS(nng_msg_append_u32(m, 3u)); + NUTS_PASS(nng_msg_append_u32(m, 0x80000002u)); + NUTS_PASS(nng_msg_append(m, "FAIL2", 6)); + NUTS_PASS(nng_sendmsg(surv, m, 0)); + + NUTS_PASS(nng_msg_alloc(&m, 0)); + NUTS_PASS(nng_msg_append_u32(m, 1u)); // 3 hops - passes + NUTS_PASS(nng_msg_append_u32(m, 2u)); + NUTS_PASS(nng_msg_append_u32(m, 0x80000003u)); + NUTS_PASS(nng_msg_append(m, "PASS3", 6)); + NUTS_PASS(nng_sendmsg(surv, m, 0)); + + NUTS_PASS(nng_msg_alloc(&m, 0)); + NUTS_PASS(nng_msg_append_u32(m, 1u)); // 4 hops -- discard! + NUTS_PASS(nng_msg_append_u32(m, 2u)); + NUTS_PASS(nng_msg_append_u32(m, 3u)); + NUTS_PASS(nng_msg_append_u32(m, 0x80000003u)); + NUTS_PASS(nng_msg_append(m, "FAIL4", 6)); + NUTS_PASS(nng_sendmsg(surv, m, 0)); + + NUTS_RECV(resp, "PASS1"); + NUTS_RECV(resp, "PASS3"); + + NUTS_FAIL(nng_recvmsg(resp, &m, 0), NNG_ETIMEDOUT); + + NUTS_CLOSE(resp); + NUTS_CLOSE(surv); +} + +TEST_LIST = { + { "respond identity", test_resp_identity }, + { "respond send bad state", test_resp_send_bad_state }, + { "respond poll readable", test_resp_poll_readable }, + { "respond poll writable", test_resp_poll_writeable }, + { "respond context does not poll", test_resp_context_no_poll }, + { "respond validate peer", test_resp_validate_peer }, + { "respond double recv", test_resp_double_recv }, + { "respond close pipe before send", test_resp_close_pipe_before_send }, + { "respond close pipe during send", test_resp_close_pipe_during_send }, + { "respond recv aio ctx stopped", test_resp_ctx_recv_aio_stopped }, + { "respond close pipe context send", + test_resp_close_pipe_context_send }, + { "respond close context send", test_resp_close_context_send }, + { "respond context send nonblock", test_resp_ctx_send_nonblock }, + { "respond context recv nonblock", test_resp_ctx_recv_nonblock }, + { "respond recv garbage", test_resp_recv_garbage }, + { "respond ttl option", test_resp_ttl_option }, + { "respond ttl drop", test_resp_ttl_drop }, + { NULL, NULL }, +}; diff --git a/src/sp/protocol/survey0/survey.c b/src/sp/protocol/survey0/survey.c new file mode 100644 index 00000000..ce1ed601 --- /dev/null +++ b/src/sp/protocol/survey0/survey.c @@ -0,0 +1,663 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// Copyright 2018 Capitar IT Group BV +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include + +#include "core/nng_impl.h" +#include "nng/protocol/survey0/survey.h" + +// Surveyor protocol. The SURVEYOR protocol is the "survey" side of the +// survey pattern. This is useful for building service discovery, voting, etc. +// Note that this pattern is not optimized for extreme low latency, as it makes +// multiple use of queues for simplicity. Typically this is used in cases +// where a few dozen extra microseconds does not matter. + +typedef struct surv0_pipe surv0_pipe; +typedef struct surv0_sock surv0_sock; +typedef struct surv0_ctx surv0_ctx; + +static void surv0_pipe_send_cb(void *); +static void surv0_pipe_recv_cb(void *); +static void surv0_ctx_timeout(void *); + +struct surv0_ctx { + surv0_sock * sock; + uint32_t survey_id; // survey id + nni_timer_node timer; + nni_time expire; + nni_lmq recv_lmq; + nni_list recv_queue; + nni_atomic_int recv_buf; + nni_atomic_int survey_time; + int err; +}; + +// surv0_sock is our per-socket protocol private structure. +struct surv0_sock { + int ttl; + nni_list pipes; + nni_mtx mtx; + surv0_ctx ctx; + nni_id_map surveys; + nni_pollable writable; + nni_pollable readable; + nni_atomic_int send_buf; +}; + +// surv0_pipe is our per-pipe protocol private structure. +struct surv0_pipe { + nni_pipe * pipe; + surv0_sock * sock; + nni_lmq send_queue; + nni_list_node node; + nni_aio aio_send; + nni_aio aio_recv; + bool busy; + bool closed; +}; + +static void +surv0_ctx_abort(surv0_ctx *ctx, int err) +{ + nni_aio * aio; + surv0_sock *sock = ctx->sock; + + while ((aio = nni_list_first(&ctx->recv_queue)) != NULL) { + nni_list_remove(&ctx->recv_queue, aio); + nni_aio_finish_error(aio, err); + } + nni_lmq_flush(&ctx->recv_lmq); + if (ctx->survey_id != 0) { + nni_id_remove(&sock->surveys, ctx->survey_id); + ctx->survey_id = 0; + } + if (ctx == &sock->ctx) { + nni_pollable_clear(&sock->readable); + } +} + +static void +surv0_ctx_close(surv0_ctx *ctx) +{ + surv0_sock *sock = ctx->sock; + + nni_mtx_lock(&sock->mtx); + surv0_ctx_abort(ctx, NNG_ECLOSED); + nni_mtx_unlock(&sock->mtx); +} + +static void +surv0_ctx_fini(void *arg) +{ + surv0_ctx *ctx = arg; + + surv0_ctx_close(ctx); + nni_timer_cancel(&ctx->timer); + nni_lmq_fini(&ctx->recv_lmq); +} + +static int +surv0_ctx_init(void *c, void *s) +{ + surv0_ctx * ctx = c; + surv0_sock * sock = s; + int rv; + int len; + nng_duration tmo; + + nni_aio_list_init(&ctx->recv_queue); + nni_atomic_init(&ctx->recv_buf); + nni_atomic_init(&ctx->survey_time); + + if (ctx == &sock->ctx) { + len = 128; + tmo = NNI_SECOND; // survey timeout + } else { + len = nni_atomic_get(&sock->ctx.recv_buf); + tmo = nni_atomic_get(&sock->ctx.survey_time); + } + + nni_atomic_set(&ctx->recv_buf, len); + nni_atomic_set(&ctx->survey_time, tmo); + + ctx->sock = sock; + + if ((rv = nni_lmq_init(&ctx->recv_lmq, len)) != 0) { + surv0_ctx_fini(ctx); + return (rv); + } + nni_timer_init(&ctx->timer, surv0_ctx_timeout, ctx); + return (0); +} + +static void +surv0_ctx_cancel(nni_aio *aio, void *arg, int rv) +{ + surv0_ctx * ctx = arg; + surv0_sock *sock = ctx->sock; + nni_mtx_lock(&sock->mtx); + if (nni_list_active(&ctx->recv_queue, aio)) { + nni_list_remove(&ctx->recv_queue, aio); + nni_aio_finish_error(aio, rv); + } + if (ctx->survey_id != 0) { + nni_id_remove(&sock->surveys, ctx->survey_id); + ctx->survey_id = 0; + } + nni_mtx_unlock(&sock->mtx); +} + +static void +surv0_ctx_recv(void *arg, nni_aio *aio) +{ + surv0_ctx * ctx = arg; + surv0_sock *sock = ctx->sock; + nni_msg * msg; + + if (nni_aio_begin(aio) != 0) { + return; + } + + nni_mtx_lock(&sock->mtx); + if (ctx->survey_id == 0) { + nni_mtx_unlock(&sock->mtx); + nni_aio_finish_error(aio, NNG_ESTATE); + return; + } +again: + if (nni_lmq_getq(&ctx->recv_lmq, &msg) != 0) { + int rv; + if ((rv = nni_aio_schedule(aio, &surv0_ctx_cancel, ctx)) != + 0) { + nni_mtx_unlock(&sock->mtx); + nni_aio_finish_error(aio, rv); + return; + } + nni_list_append(&ctx->recv_queue, aio); + nni_mtx_unlock(&sock->mtx); + return; + } + if (nni_lmq_empty(&ctx->recv_lmq) && (ctx == &sock->ctx)) { + nni_pollable_clear(&sock->readable); + } + if ((msg = nni_msg_unique(msg)) == NULL) { + goto again; + } + + nni_mtx_unlock(&sock->mtx); + nni_aio_finish_msg(aio, msg); +} + +void +surv0_ctx_timeout(void *arg) +{ + surv0_ctx * ctx = arg; + surv0_sock *sock = ctx->sock; + + nni_mtx_lock(&sock->mtx); + if (nni_clock() < ctx->expire) { + nni_mtx_unlock(&sock->mtx); + return; + } + + // Abort any pending receives. + surv0_ctx_abort(ctx, NNG_ETIMEDOUT); + nni_mtx_unlock(&sock->mtx); +} + +static void +surv0_ctx_send(void *arg, nni_aio *aio) +{ + surv0_ctx * ctx = arg; + surv0_sock * sock = ctx->sock; + surv0_pipe * pipe; + nni_msg * msg = nni_aio_get_msg(aio); + size_t len = nni_msg_len(msg); + nni_time now = nni_clock(); + nng_duration survey_time; + int rv; + + if (nni_aio_begin(aio) != 0) { + return; + } + + survey_time = nni_atomic_get(&ctx->survey_time); + + nni_mtx_lock(&sock->mtx); + + // Abort everything outstanding. + surv0_ctx_abort(ctx, NNG_ECANCELED); + nni_timer_cancel(&ctx->timer); + + // Allocate the new ID. + if ((rv = nni_id_alloc(&sock->surveys, &ctx->survey_id, ctx)) != 0) { + nni_mtx_unlock(&sock->mtx); + nni_aio_finish_error(aio, rv); + return; + } + nni_msg_header_clear(msg); + nni_msg_header_append_u32(msg, (uint32_t) ctx->survey_id); + + // From this point, we're committed to success. Note that we send + // regardless of whether there are any pipes or not. If no pipes, + // then it just gets discarded. + nni_aio_set_msg(aio, NULL); + NNI_LIST_FOREACH (&sock->pipes, pipe) { + + // if the pipe isn't busy, then send this message direct. + if (!pipe->busy) { + pipe->busy = true; + nni_msg_clone(msg); + nni_aio_set_msg(&pipe->aio_send, msg); + nni_pipe_send(pipe->pipe, &pipe->aio_send); + } else if (!nni_lmq_full(&pipe->send_queue)) { + nni_msg_clone(msg); + nni_lmq_putq(&pipe->send_queue, msg); + } + } + + ctx->expire = now + survey_time; + nni_timer_schedule(&ctx->timer, ctx->expire); + + nni_mtx_unlock(&sock->mtx); + nni_msg_free(msg); + + nni_aio_finish(aio, 0, len); +} + +static void +surv0_sock_fini(void *arg) +{ + surv0_sock *sock = arg; + + surv0_ctx_fini(&sock->ctx); + nni_id_map_fini(&sock->surveys); + nni_pollable_fini(&sock->writable); + nni_pollable_fini(&sock->readable); + nni_mtx_fini(&sock->mtx); +} + +static int +surv0_sock_init(void *arg, nni_sock *s) +{ + surv0_sock *sock = arg; + int rv; + + NNI_ARG_UNUSED(s); + + NNI_LIST_INIT(&sock->pipes, surv0_pipe, node); + nni_mtx_init(&sock->mtx); + nni_pollable_init(&sock->readable); + nni_pollable_init(&sock->writable); + // We are always writable. + nni_pollable_raise(&sock->writable); + + // We allow for some buffering on a per pipe basis, to allow for + // multiple contexts to have surveys outstanding. It is recommended + // to increase this if many contexts will want to publish + // at nearly the same time. + nni_atomic_init(&sock->send_buf); + nni_atomic_set(&sock->send_buf, 8); + + // Survey IDs are 32 bits, with the high order bit set. + // We start at a random point, to minimize likelihood of + // accidental collision across restarts. + nni_id_map_init(&sock->surveys, 0x80000000u, 0xffffffffu, true); + + if ((rv = surv0_ctx_init(&sock->ctx, sock)) != 0) { + surv0_sock_fini(sock); + return (rv); + } + + sock->ttl = 8; + + return (0); +} + +static void +surv0_sock_open(void *arg) +{ + NNI_ARG_UNUSED(arg); +} + +static void +surv0_sock_close(void *arg) +{ + surv0_sock *s = arg; + + surv0_ctx_close(&s->ctx); +} + +static void +surv0_pipe_stop(void *arg) +{ + surv0_pipe *p = arg; + + nni_aio_stop(&p->aio_send); + nni_aio_stop(&p->aio_recv); +} + +static void +surv0_pipe_fini(void *arg) +{ + surv0_pipe *p = arg; + + nni_aio_fini(&p->aio_send); + nni_aio_fini(&p->aio_recv); + nni_lmq_fini(&p->send_queue); +} + +static int +surv0_pipe_init(void *arg, nni_pipe *pipe, void *s) +{ + surv0_pipe *p = arg; + surv0_sock *sock = s; + int rv; + int len; + + len = nni_atomic_get(&sock->send_buf); + nni_aio_init(&p->aio_send, surv0_pipe_send_cb, p); + nni_aio_init(&p->aio_recv, surv0_pipe_recv_cb, p); + + // This depth could be tunable. The deeper the queue, the more + // concurrent surveys that can be delivered (multiple contexts). + // Note that surveys can be *outstanding*, but not yet put on the wire. + if ((rv = nni_lmq_init(&p->send_queue, len)) != 0) { + surv0_pipe_fini(p); + return (rv); + } + + p->pipe = pipe; + p->sock = sock; + return (0); +} + +static int +surv0_pipe_start(void *arg) +{ + surv0_pipe *p = arg; + surv0_sock *s = p->sock; + + if (nni_pipe_peer(p->pipe) != NNG_SURVEYOR0_PEER) { + return (NNG_EPROTO); + } + + nni_mtx_lock(&s->mtx); + nni_list_append(&s->pipes, p); + nni_mtx_unlock(&s->mtx); + + nni_pipe_recv(p->pipe, &p->aio_recv); + return (0); +} + +static void +surv0_pipe_close(void *arg) +{ + surv0_pipe *p = arg; + surv0_sock *s = p->sock; + + nni_aio_close(&p->aio_send); + nni_aio_close(&p->aio_recv); + + nni_mtx_lock(&s->mtx); + p->closed = true; + nni_lmq_flush(&p->send_queue); + if (nni_list_active(&s->pipes, p)) { + nni_list_remove(&s->pipes, p); + } + nni_mtx_unlock(&s->mtx); +} + +static void +surv0_pipe_send_cb(void *arg) +{ + surv0_pipe *p = arg; + surv0_sock *sock = p->sock; + nni_msg * msg; + + if (nni_aio_result(&p->aio_send) != 0) { + nni_msg_free(nni_aio_get_msg(&p->aio_send)); + nni_aio_set_msg(&p->aio_send, NULL); + nni_pipe_close(p->pipe); + return; + } + + nni_mtx_lock(&sock->mtx); + if (p->closed) { + nni_mtx_unlock(&sock->mtx); + return; + } + if (nni_lmq_getq(&p->send_queue, &msg) == 0) { + nni_aio_set_msg(&p->aio_send, msg); + nni_pipe_send(p->pipe, &p->aio_send); + } else { + p->busy = false; + } + nni_mtx_unlock(&sock->mtx); +} + +static void +surv0_pipe_recv_cb(void *arg) +{ + surv0_pipe *p = arg; + surv0_sock *sock = p->sock; + surv0_ctx * ctx; + nni_msg * msg; + uint32_t id; + nni_aio * aio; + + if (nni_aio_result(&p->aio_recv) != 0) { + nni_pipe_close(p->pipe); + return; + } + + msg = nni_aio_get_msg(&p->aio_recv); + nni_aio_set_msg(&p->aio_recv, NULL); + nni_msg_set_pipe(msg, nni_pipe_id(p->pipe)); + + // We yank 4 bytes of body, and move them to the header. + if (nni_msg_len(msg) < 4) { + // Peer sent us garbage. Kick it. + nni_msg_free(msg); + nni_pipe_close(p->pipe); + return; + } + id = nni_msg_trim_u32(msg); + nni_msg_header_append_u32(msg, id); + + nni_mtx_lock(&sock->mtx); + // Best effort at delivery. Discard if no context or context is + // unable to receive it. + if (((ctx = nni_id_get(&sock->surveys, id)) == NULL) || + (nni_lmq_full(&ctx->recv_lmq))) { + nni_msg_free(msg); + } else if ((aio = nni_list_first(&ctx->recv_queue)) != NULL) { + nni_list_remove(&ctx->recv_queue, aio); + nni_aio_finish_msg(aio, msg); + } else { + nni_lmq_putq(&ctx->recv_lmq, msg); + if (ctx == &sock->ctx) { + nni_pollable_raise(&sock->readable); + } + } + nni_mtx_unlock(&sock->mtx); + + nni_pipe_recv(p->pipe, &p->aio_recv); +} + +static int +surv0_ctx_set_survey_time( + void *arg, const void *buf, size_t sz, nni_opt_type t) +{ + surv0_ctx * ctx = arg; + nng_duration expire; + int rv; + if ((rv = nni_copyin_ms(&expire, buf, sz, t)) == 0) { + nni_atomic_set(&ctx->survey_time, expire); + } + return (rv); +} + +static int +surv0_ctx_get_survey_time(void *arg, void *buf, size_t *szp, nni_opt_type t) +{ + surv0_ctx *ctx = arg; + return ( + nni_copyout_ms(nni_atomic_get(&ctx->survey_time), buf, szp, t)); +} + +static int +surv0_sock_set_max_ttl(void *arg, const void *buf, size_t sz, nni_opt_type t) +{ + surv0_sock *s = arg; + return (nni_copyin_int(&s->ttl, buf, sz, 1, NNI_MAX_MAX_TTL, t)); +} + +static int +surv0_sock_get_max_ttl(void *arg, void *buf, size_t *szp, nni_opt_type t) +{ + surv0_sock *s = arg; + return (nni_copyout_int(s->ttl, buf, szp, t)); +} + +static int +surv0_sock_set_survey_time( + void *arg, const void *buf, size_t sz, nni_opt_type t) +{ + surv0_sock *s = arg; + return (surv0_ctx_set_survey_time(&s->ctx, buf, sz, t)); +} + +static int +surv0_sock_get_survey_time(void *arg, void *buf, size_t *szp, nni_opt_type t) +{ + surv0_sock *s = arg; + return (surv0_ctx_get_survey_time(&s->ctx, buf, szp, t)); +} + +static int +surv0_sock_get_send_fd(void *arg, void *buf, size_t *szp, nni_opt_type t) +{ + surv0_sock *sock = arg; + int rv; + int fd; + + if ((rv = nni_pollable_getfd(&sock->writable, &fd)) != 0) { + return (rv); + } + return (nni_copyout_int(fd, buf, szp, t)); +} + +static int +surv0_sock_get_recv_fd(void *arg, void *buf, size_t *szp, nni_opt_type t) +{ + surv0_sock *sock = arg; + int rv; + int fd; + + if ((rv = nni_pollable_getfd(&sock->readable, &fd)) != 0) { + return (rv); + } + return (nni_copyout_int(fd, buf, szp, t)); +} + +static void +surv0_sock_recv(void *arg, nni_aio *aio) +{ + surv0_sock *s = arg; + surv0_ctx_recv(&s->ctx, aio); +} + +static void +surv0_sock_send(void *arg, nni_aio *aio) +{ + surv0_sock *s = arg; + surv0_ctx_send(&s->ctx, aio); +} + +static nni_proto_pipe_ops surv0_pipe_ops = { + .pipe_size = sizeof(surv0_pipe), + .pipe_init = surv0_pipe_init, + .pipe_fini = surv0_pipe_fini, + .pipe_start = surv0_pipe_start, + .pipe_close = surv0_pipe_close, + .pipe_stop = surv0_pipe_stop, +}; + +static nni_option surv0_ctx_options[] = { + { + .o_name = NNG_OPT_SURVEYOR_SURVEYTIME, + .o_get = surv0_ctx_get_survey_time, + .o_set = surv0_ctx_set_survey_time, + }, + { + .o_name = NULL, + } +}; +static nni_proto_ctx_ops surv0_ctx_ops = { + .ctx_size = sizeof(surv0_ctx), + .ctx_init = surv0_ctx_init, + .ctx_fini = surv0_ctx_fini, + .ctx_send = surv0_ctx_send, + .ctx_recv = surv0_ctx_recv, + .ctx_options = surv0_ctx_options, +}; + +static nni_option surv0_sock_options[] = { + { + .o_name = NNG_OPT_SURVEYOR_SURVEYTIME, + .o_get = surv0_sock_get_survey_time, + .o_set = surv0_sock_set_survey_time, + }, + { + .o_name = NNG_OPT_MAXTTL, + .o_get = surv0_sock_get_max_ttl, + .o_set = surv0_sock_set_max_ttl, + }, + { + .o_name = NNG_OPT_RECVFD, + .o_get = surv0_sock_get_recv_fd, + }, + { + .o_name = NNG_OPT_SENDFD, + .o_get = surv0_sock_get_send_fd, + }, + // terminate list + { + .o_name = NULL, + }, +}; + +static nni_proto_sock_ops surv0_sock_ops = { + .sock_size = sizeof(surv0_sock), + .sock_init = surv0_sock_init, + .sock_fini = surv0_sock_fini, + .sock_open = surv0_sock_open, + .sock_close = surv0_sock_close, + .sock_send = surv0_sock_send, + .sock_recv = surv0_sock_recv, + .sock_options = surv0_sock_options, +}; + +static nni_proto surv0_proto = { + .proto_version = NNI_PROTOCOL_VERSION, + .proto_self = { NNG_SURVEYOR0_SELF, NNG_SURVEYOR0_SELF_NAME }, + .proto_peer = { NNG_SURVEYOR0_PEER, NNG_SURVEYOR0_PEER_NAME }, + .proto_flags = NNI_PROTO_FLAG_SNDRCV, + .proto_sock_ops = &surv0_sock_ops, + .proto_pipe_ops = &surv0_pipe_ops, + .proto_ctx_ops = &surv0_ctx_ops, +}; + +int +nng_surveyor0_open(nng_socket *sock) +{ + return (nni_proto_open(sock, &surv0_proto)); +} diff --git a/src/sp/protocol/survey0/survey_test.c b/src/sp/protocol/survey0/survey_test.c new file mode 100644 index 00000000..95d27adf --- /dev/null +++ b/src/sp/protocol/survey0/survey_test.c @@ -0,0 +1,626 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// Copyright 2018 Capitar IT Group BV +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include + +static void +test_surv_identity(void) +{ + nng_socket s; + int p; + char * n; + + NUTS_PASS(nng_surveyor0_open(&s)); + NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PROTO, &p)); + NUTS_TRUE(p == NNG_SURVEYOR0_SELF); + NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PEER, &p)); + NUTS_TRUE(p == NNG_SURVEYOR0_PEER); // 49 + NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PROTONAME, &n)); + NUTS_MATCH(n, NNG_SURVEYOR0_SELF_NAME); + nng_strfree(n); + NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PEERNAME, &n)); + NUTS_MATCH(n, NNG_SURVEYOR0_PEER_NAME); + nng_strfree(n); + NUTS_CLOSE(s); +} + +static void +test_surv_ttl_option(void) +{ + nng_socket surv; + int v; + bool b; + size_t sz; + const char *opt = NNG_OPT_MAXTTL; + + NUTS_PASS(nng_surveyor0_open(&surv)); + + NUTS_PASS(nng_socket_set_int(surv, opt, 1)); + NUTS_FAIL(nng_socket_set_int(surv, opt, 0), NNG_EINVAL); + NUTS_FAIL(nng_socket_set_int(surv, opt, -1), NNG_EINVAL); + // This test will fail if the NNI_MAX_MAX_TTL is changed from the + // builtin default of 15. + NUTS_FAIL(nng_socket_set_int(surv, opt, 16), NNG_EINVAL); + NUTS_FAIL(nng_socket_set_int(surv, opt, 256), NNG_EINVAL); + NUTS_PASS(nng_socket_set_int(surv, opt, 3)); + NUTS_PASS(nng_socket_get_int(surv, opt, &v)); + NUTS_TRUE(v == 3); + v = 0; + sz = sizeof(v); + NUTS_PASS(nng_socket_get(surv, opt, &v, &sz)); + NUTS_TRUE(v == 3); + NUTS_TRUE(sz == sizeof(v)); + + NUTS_FAIL(nng_socket_set(surv, opt, "", 1), NNG_EINVAL); + sz = 1; + NUTS_FAIL(nng_socket_get(surv, opt, &v, &sz), NNG_EINVAL); + NUTS_FAIL(nng_socket_set_bool(surv, opt, true), NNG_EBADTYPE); + NUTS_FAIL(nng_socket_get_bool(surv, opt, &b), NNG_EBADTYPE); + + NUTS_CLOSE(surv); +} + +static void +test_surv_survey_time_option(void) +{ + nng_socket surv; + nng_duration d; + bool b; + size_t sz = sizeof(b); + const char * opt = NNG_OPT_SURVEYOR_SURVEYTIME; + + NUTS_PASS(nng_surveyor0_open(&surv)); + + NUTS_PASS(nng_socket_set_ms(surv, opt, 10)); + NUTS_FAIL(nng_socket_set(surv, opt, "", 1), NNG_EINVAL); + NUTS_FAIL(nng_socket_get(surv, opt, &b, &sz), NNG_EINVAL); + NUTS_FAIL(nng_socket_set_bool(surv, opt, true), NNG_EBADTYPE); + NUTS_FAIL(nng_socket_get_bool(surv, opt, &b), NNG_EBADTYPE); + + NUTS_PASS(nng_socket_get_ms(surv, opt, &d)); + NUTS_TRUE(d == 10); + NUTS_CLOSE(surv); +} + +void +test_surv_recv_bad_state(void) +{ + nng_socket surv; + nng_msg * msg = NULL; + + NUTS_PASS(nng_surveyor0_open(&surv)); + NUTS_FAIL(nng_recvmsg(surv, &msg, 0), NNG_ESTATE); + NUTS_TRUE(msg == NULL); + NUTS_CLOSE(surv); +} + +static void +test_surv_recv_garbage(void) +{ + nng_socket resp; + nng_socket surv; + nng_msg * m; + uint32_t surv_id; + + NUTS_PASS(nng_respondent0_open_raw(&resp)); + NUTS_PASS(nng_surveyor0_open(&surv)); + NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_RECVTIMEO, 100)); + NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_SENDTIMEO, 1000)); + + NUTS_MARRY(surv, resp); + + NUTS_PASS(nng_msg_alloc(&m, 0)); + NUTS_PASS(nng_sendmsg(surv, m, 0)); + + NUTS_PASS(nng_recvmsg(resp, &m, 0)); + + // The message will have a header that contains the 32-bit pipe ID, + // followed by the 32-bit request ID. We will discard the request + // ID before sending it out. + NUTS_TRUE(nng_msg_header_len(m) == 8); + NUTS_PASS(nng_msg_header_chop_u32(m, &surv_id)); + + NUTS_PASS(nng_sendmsg(resp, m, 0)); + NUTS_FAIL(nng_recvmsg(surv, &m, 0), NNG_ETIMEDOUT); + + NUTS_CLOSE(surv); + NUTS_CLOSE(resp); +} + +#define SECOND 1000 + +void +test_surv_resp_exchange(void) +{ + nng_socket surv; + nng_socket resp; + + NUTS_PASS(nng_surveyor0_open(&surv)); + NUTS_PASS(nng_respondent0_open(&resp)); + + NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_RECVTIMEO, SECOND)); + NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_RECVTIMEO, SECOND)); + NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_SENDTIMEO, SECOND)); + NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_SENDTIMEO, SECOND)); + + NUTS_MARRY(resp, surv); + + NUTS_SEND(surv, "ping"); + NUTS_RECV(resp, "ping"); + NUTS_SEND(resp, "pong"); + NUTS_RECV(surv, "pong"); + + NUTS_CLOSE(surv); + NUTS_CLOSE(resp); +} + +void +test_surv_cancel(void) +{ + nng_socket surv; + nng_socket resp; + + NUTS_PASS(nng_respondent0_open(&resp)); + NUTS_PASS(nng_surveyor0_open(&surv)); + + NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_RECVTIMEO, SECOND)); + NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_RECVTIMEO, SECOND)); + NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_SENDTIMEO, 5 * SECOND)); + NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_SENDTIMEO, 5 * SECOND)); + NUTS_PASS(nng_socket_set_int(surv, NNG_OPT_SENDBUF, 16)); + + NUTS_MARRY(resp, surv); + + // Send req #1 (abc). + NUTS_SEND(surv, "abc"); + + // Sleep a bit. This is so that we ensure that our request gets + // to the far side. (If we cancel too fast, then our outgoing send + // will be canceled before it gets to the peer.) + NUTS_SLEEP(100); + + // Send the next next request ("def"). Note that + // the RESP side server will have already buffered the receive + // request, and should simply be waiting for us to reply to abc. + NUTS_SEND(surv, "def"); + + // Receive the first request (should be abc) on the REP server. + NUTS_RECV(resp, "abc"); + + // RESP sends the reply to first command. This will be discarded + // by the SURV socket. + NUTS_SEND(resp, "abc"); + + // Now get the next command from the REP; should be "def". + NUTS_RECV(resp, "def"); + + // And send it back to REQ. + NUTS_SEND(resp, "def"); + + // Try a req command. This should give back "def" + NUTS_RECV(surv, "def"); + + NUTS_CLOSE(surv); + NUTS_CLOSE(resp); +} + +void +test_surv_cancel_abort_recv(void) +{ + nng_aio * aio; + nng_duration time = SECOND * 10; // 10s (kind of never) + nng_socket surv; + nng_socket resp; + + NUTS_PASS(nng_respondent0_open(&resp)); + NUTS_PASS(nng_surveyor0_open(&surv)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + + NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_SURVEYOR_SURVEYTIME, time)); + NUTS_PASS(nng_socket_set_int(surv, NNG_OPT_SENDBUF, 16)); + NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_RECVTIMEO, 5 * SECOND)); + NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_RECVTIMEO, 5 * SECOND)); + NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_SENDTIMEO, 5 * SECOND)); + NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_SENDTIMEO, 5 * SECOND)); + + NUTS_MARRY(resp, surv); + + // Send survey #1 (abc). + NUTS_SEND(surv, "abc"); + + // Wait for it to get ot the other side. + NUTS_SLEEP(100); + + nng_aio_set_timeout(aio, 5 * SECOND); + nng_recv_aio(surv, aio); + + // Give time for this recv to post properly. + NUTS_SLEEP(100); + + // Send the next next request ("def"). Note that + // the respondent side server will have already buffered the receive + // request, and should simply be waiting for us to reply to + // abc. + NUTS_SEND(surv, "def"); + + // Our pending I/O should have been canceled. + nng_aio_wait(aio); + NUTS_FAIL(nng_aio_result(aio), NNG_ECANCELED); + + // Receive the first request (should be abc) on the respondent. + NUTS_RECV(resp, "abc"); + + // Respondent sends the reply to first survey. This will be + // discarded by the SURV socket. + NUTS_SEND(resp, "abc"); + + // Now get the next survey from the RESP; should be "def". + NUTS_RECV(resp, "def"); + + // And send it back to REQ. + NUTS_SEND(resp, "def"); + + // Try a req command. This should give back "def" + NUTS_RECV(surv, "def"); + + nng_aio_free(aio); + NUTS_CLOSE(surv); + NUTS_CLOSE(resp); +} + +static void +test_surv_cancel_post_recv(void) +{ + nng_socket surv; + nng_socket resp; + + NUTS_PASS(nng_surveyor0_open(&surv)); + NUTS_PASS(nng_respondent0_open(&resp)); + NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_RECVTIMEO, 1000)); + NUTS_MARRY(surv, resp); + + NUTS_SEND(surv, "ONE"); + NUTS_RECV(resp, "ONE"); + NUTS_SEND(resp, "one"); + NUTS_SLEEP(100); // Make sure reply arrives! + NUTS_SEND(surv, "TWO"); + NUTS_RECV(resp, "TWO"); + NUTS_SEND(resp, "two"); + NUTS_RECV(surv, "two"); + + NUTS_CLOSE(surv); + NUTS_CLOSE(resp); +} + +static void +test_surv_poll_writeable(void) +{ + int fd; + nng_socket surv; + nng_socket resp; + + NUTS_PASS(nng_surveyor0_open(&surv)); + NUTS_PASS(nng_respondent0_open(&resp)); + NUTS_PASS(nng_socket_get_int(surv, NNG_OPT_SENDFD, &fd)); + NUTS_TRUE(fd >= 0); + + // Survey is broadcast, so we can always write. + NUTS_TRUE(nuts_poll_fd(fd)); + + NUTS_MARRY(surv, resp); + + // Now it's writable. + NUTS_TRUE(nuts_poll_fd(fd)); + + NUTS_CLOSE(surv); + NUTS_CLOSE(resp); +} + +void +test_surv_poll_readable(void) +{ + int fd; + nng_socket surv; + nng_socket resp; + nng_msg * msg; + + NUTS_PASS(nng_surveyor0_open(&surv)); + NUTS_PASS(nng_respondent0_open(&resp)); + NUTS_PASS(nng_socket_get_int(surv, NNG_OPT_RECVFD, &fd)); + NUTS_TRUE(fd >= 0); + + // Not readable if not connected! + NUTS_TRUE(nuts_poll_fd(fd) == false); + + // Even after connect (no message yet) + NUTS_MARRY(surv, resp); + NUTS_TRUE(nuts_poll_fd(fd) == false); + + // But once we send messages, it is. + // We have to send a request, in order to send a reply. + + NUTS_PASS(nng_msg_alloc(&msg, 0)); + NUTS_PASS(nng_msg_append(msg, "xyz", 3)); + NUTS_PASS(nng_sendmsg(surv, msg, 0)); + NUTS_PASS(nng_recvmsg(resp, &msg, 0)); // recv on rep + NUTS_PASS(nng_sendmsg(resp, msg, 0)); // echo it back + NUTS_SLEEP(200); // give time for message to arrive + + NUTS_TRUE(nuts_poll_fd(fd) == true); + + // and receiving makes it no longer ready + NUTS_PASS(nng_recvmsg(surv, &msg, 0)); + nng_msg_free(msg); + NUTS_TRUE(nuts_poll_fd(fd) == false); + + // TODO verify unsolicited response + + NUTS_CLOSE(surv); + NUTS_CLOSE(resp); +} + +static void +test_surv_ctx_no_poll(void) +{ + int fd; + nng_socket surv; + nng_ctx ctx; + + NUTS_PASS(nng_surveyor0_open(&surv)); + NUTS_PASS(nng_ctx_open(&ctx, surv)); + NUTS_FAIL(nng_ctx_get_int(ctx, NNG_OPT_SENDFD, &fd), NNG_ENOTSUP); + NUTS_FAIL(nng_ctx_get_int(ctx, NNG_OPT_RECVFD, &fd), NNG_ENOTSUP); + NUTS_PASS(nng_ctx_close(ctx)); + NUTS_CLOSE(surv); +} + +static void +test_surv_ctx_recv_nonblock(void) +{ + nng_socket surv; + nng_socket resp; + nng_ctx ctx; + nng_aio * aio; + nng_msg * msg; + + NUTS_PASS(nng_surveyor0_open(&surv)); + NUTS_PASS(nng_respondent0_open(&resp)); + NUTS_PASS(nng_ctx_open(&ctx, surv)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + NUTS_PASS(nng_msg_alloc(&msg, 0)); + + NUTS_MARRY(surv, resp); + + nng_aio_set_msg(aio, msg); + nng_ctx_send(ctx, aio); + nng_aio_wait(aio); + NUTS_PASS(nng_aio_result(aio)); + nng_aio_set_timeout(aio, 0); // Instant timeout + nng_ctx_recv(ctx, aio); + + nng_aio_wait(aio); + NUTS_FAIL(nng_aio_result(aio), NNG_ETIMEDOUT); + NUTS_CLOSE(surv); + NUTS_CLOSE(resp); + nng_aio_free(aio); +} + +static void +test_surv_ctx_send_nonblock(void) +{ + nng_socket surv; + nng_ctx ctx; + nng_aio * aio; + nng_msg * msg; + + NUTS_PASS(nng_surveyor0_open(&surv)); + NUTS_PASS(nng_ctx_open(&ctx, surv)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + NUTS_PASS(nng_msg_alloc(&msg, 0)); + + nng_aio_set_msg(aio, msg); + nng_aio_set_timeout(aio, 0); // Instant timeout + nng_ctx_send(ctx, aio); + nng_aio_wait(aio); + NUTS_PASS(nng_aio_result(aio)); // We never block + NUTS_CLOSE(surv); + nng_aio_free(aio); +} + +static void +test_surv_send_best_effort(void) +{ + nng_socket surv; + nng_socket resp; + + NUTS_PASS(nng_surveyor0_open(&surv)); + NUTS_PASS(nng_respondent0_open(&resp)); + NUTS_MARRY(surv, resp); + + for (int i = 0; i < 200; i++) { + NUTS_SEND(surv, "junk"); + } + + NUTS_CLOSE(surv); + NUTS_CLOSE(resp); +} + +static void +test_surv_survey_timeout(void) +{ + nng_socket surv; + nng_socket resp; + char buf[16]; + size_t sz; + + NUTS_PASS(nng_surveyor0_open(&surv)); + NUTS_PASS(nng_respondent0_open(&resp)); + NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_SURVEYOR_SURVEYTIME, 50)); + NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_RECVTIMEO, 100)); + + NUTS_MARRY(surv, resp); + + NUTS_SEND(surv, "hello"); + NUTS_RECV(resp, "hello"); + + sz = sizeof(buf); + NUTS_FAIL(nng_recv(surv, buf, &sz, 0), NNG_ETIMEDOUT); + NUTS_SEND(resp, "world"); + NUTS_FAIL(nng_recv(surv, buf, &sz, 0), NNG_ESTATE); + + NUTS_CLOSE(surv); + NUTS_CLOSE(resp); +} + +static void +test_surv_ctx_recv_close_socket(void) +{ + nng_socket surv; + nng_socket resp; + nng_ctx ctx; + nng_aio * aio; + nng_msg * m; + + NUTS_PASS(nng_surveyor0_open(&surv)); + NUTS_PASS(nng_respondent0_open(&resp)); + NUTS_PASS(nng_ctx_open(&ctx, surv)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + NUTS_MARRY(surv, resp); + NUTS_PASS(nng_msg_alloc(&m, 0)); + nng_aio_set_msg(aio, m); + nng_ctx_send(ctx, aio); + nng_aio_wait(aio); + NUTS_PASS(nng_aio_result(aio)); + + nng_ctx_recv(ctx, aio); + nng_close(surv); + + NUTS_FAIL(nng_aio_result(aio), NNG_ECLOSED); + nng_aio_free(aio); + NUTS_CLOSE(resp); +} + +static void +test_surv_context_multi(void) +{ + nng_socket surv; + nng_socket resp; + nng_ctx c[5]; + nng_aio * aio; + nng_msg * m; + int cnt = sizeof(c) / sizeof(c[0]); + + NUTS_PASS(nng_surveyor0_open(&surv)); + NUTS_PASS(nng_respondent0_open(&resp)); + NUTS_MARRY(surv, resp); + NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_SURVEYOR_SURVEYTIME, 200)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + + for (int i = 0; i < cnt; i++) { + NUTS_PASS(nng_ctx_open(&c[i], surv)); + } + + for (int i = 0; i < cnt; i++) { + NUTS_PASS(nng_msg_alloc(&m, 0)); + NUTS_PASS(nng_msg_append_u32(m, i)); + nng_aio_set_msg(aio, m); + nng_ctx_send(c[i], aio); + nng_aio_wait(aio); + NUTS_PASS(nng_aio_result(aio)); + } + + for (int i = 0; i < cnt; i++) { + NUTS_PASS(nng_recvmsg(resp, &m, 0)); + NUTS_PASS(nng_sendmsg(resp, m, 0)); + } + + for (int i = cnt - 1; i >= 0; i--) { + uint32_t x; + nng_ctx_recv(c[i], aio); + nng_aio_wait(aio); + NUTS_PASS(nng_aio_result(aio)); + m = nng_aio_get_msg(aio); + TEST_ASSERT(m != NULL); + NUTS_PASS(nng_msg_trim_u32(m, &x)); + NUTS_TRUE(x == (uint32_t) i); + nng_msg_free(m); + } + + for (int i = 0; i < cnt; i++) { + nng_ctx_recv(c[i], aio); + nng_aio_wait(aio); + NUTS_TRUE(nng_aio_result(aio) != 0); + } + for (int i = 0; i < cnt; i++) { + nng_ctx_close(c[i]); + } + NUTS_CLOSE(surv); + NUTS_CLOSE(resp); + nng_aio_free(aio); +} + +static void +test_surv_validate_peer(void) +{ + nng_socket s1, s2; + nng_stat * stats; + nng_stat * reject; + char * addr; + + NUTS_ADDR(addr, "inproc"); + NUTS_PASS(nng_surveyor0_open(&s1)); + NUTS_PASS(nng_surveyor0_open(&s2)); + + NUTS_PASS(nng_listen(s1, addr, NULL, 0)); + NUTS_PASS(nng_dial(s2, addr, NULL, NNG_FLAG_NONBLOCK)); + + NUTS_SLEEP(100); + NUTS_PASS(nng_stats_get(&stats)); + + NUTS_TRUE(stats != NULL); + NUTS_TRUE((reject = nng_stat_find_socket(stats, s1)) != NULL); + NUTS_TRUE((reject = nng_stat_find(reject, "reject")) != NULL); + + NUTS_TRUE(nng_stat_type(reject) == NNG_STAT_COUNTER); + NUTS_TRUE(nng_stat_value(reject) > 0); + + NUTS_PASS(nng_close(s1)); + NUTS_PASS(nng_close(s2)); + nng_stats_free(stats); +} + +TEST_LIST = { + { "survey identity", test_surv_identity }, + { "survey ttl option", test_surv_ttl_option }, + { "survey survey time option", test_surv_survey_time_option }, + { "survey recv bad state", test_surv_recv_bad_state }, + { "survey recv garbage", test_surv_recv_garbage }, + { "survey respondent exchange", test_surv_resp_exchange }, + { "survey cancel", test_surv_cancel }, + { "survey cancel abort recv", test_surv_cancel_abort_recv }, + { "survey cancel post recv", test_surv_cancel_post_recv }, + { "survey poll writable", test_surv_poll_writeable }, + { "survey poll readable", test_surv_poll_readable }, + { "survey context does not poll", test_surv_ctx_no_poll }, + { "survey context recv close socket", + test_surv_ctx_recv_close_socket }, + { "survey context recv nonblock", test_surv_ctx_recv_nonblock }, + { "survey context send nonblock", test_surv_ctx_send_nonblock }, + { "survey timeout", test_surv_survey_timeout }, + { "survey send best effort", test_surv_send_best_effort }, + { "survey context multi", test_surv_context_multi }, + { "survey validate peer", test_surv_validate_peer }, + { NULL, NULL }, +}; diff --git a/src/sp/protocol/survey0/xrespond.c b/src/sp/protocol/survey0/xrespond.c new file mode 100644 index 00000000..b2f203c3 --- /dev/null +++ b/src/sp/protocol/survey0/xrespond.c @@ -0,0 +1,417 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// Copyright 2018 Capitar IT Group BV +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include + +#include "core/nng_impl.h" +#include "nng/protocol/survey0/respond.h" + +// Respondent protocol. The RESPONDENT protocol is the "replier" side of +// the surveyor pattern. This is useful for building service discovery, or +// voting algorithms, for example. + +#ifndef NNI_PROTO_SURVEYOR_V0 +#define NNI_PROTO_SURVEYOR_V0 NNI_PROTO(6, 2) +#endif + +#ifndef NNI_PROTO_RESPONDENT_V0 +#define NNI_PROTO_RESPONDENT_V0 NNI_PROTO(6, 3) +#endif + +typedef struct xresp0_pipe xresp0_pipe; +typedef struct xresp0_sock xresp0_sock; + +static void xresp0_recv_cb(void *); +static void xresp0_putq_cb(void *); +static void xresp0_getq_cb(void *); +static void xresp0_send_cb(void *); +static void xresp0_sock_getq_cb(void *); +static void xresp0_pipe_fini(void *); + +// resp0_sock is our per-socket protocol private structure. +struct xresp0_sock { + nni_msgq * urq; + nni_msgq * uwq; + nni_atomic_int ttl; + nni_id_map pipes; + nni_aio aio_getq; + nni_mtx mtx; +}; + +// resp0_pipe is our per-pipe protocol private structure. +struct xresp0_pipe { + nni_pipe * npipe; + xresp0_sock *psock; + uint32_t id; + nni_msgq * sendq; + nni_aio aio_getq; + nni_aio aio_putq; + nni_aio aio_send; + nni_aio aio_recv; +}; + +static void +xresp0_sock_fini(void *arg) +{ + xresp0_sock *s = arg; + + nni_aio_fini(&s->aio_getq); + nni_id_map_fini(&s->pipes); + nni_mtx_fini(&s->mtx); +} + +static int +xresp0_sock_init(void *arg, nni_sock *nsock) +{ + xresp0_sock *s = arg; + + nni_mtx_init(&s->mtx); + nni_atomic_init(&s->ttl); + nni_atomic_set(&s->ttl, 8); // Per RFC + nni_id_map_init(&s->pipes, 0, 0, false); + nni_aio_init(&s->aio_getq, xresp0_sock_getq_cb, s); + + s->urq = nni_sock_recvq(nsock); + s->uwq = nni_sock_sendq(nsock); + + return (0); +} + +static void +xresp0_sock_open(void *arg) +{ + xresp0_sock *s = arg; + + nni_msgq_aio_get(s->uwq, &s->aio_getq); +} + +static void +xresp0_sock_close(void *arg) +{ + xresp0_sock *s = arg; + + nni_aio_close(&s->aio_getq); +} + +static void +xresp0_pipe_stop(void *arg) +{ + xresp0_pipe *p = arg; + + nni_aio_stop(&p->aio_putq); + nni_aio_stop(&p->aio_getq); + nni_aio_stop(&p->aio_send); + nni_aio_stop(&p->aio_recv); +} + +static void +xresp0_pipe_fini(void *arg) +{ + xresp0_pipe *p = arg; + + nni_aio_fini(&p->aio_putq); + nni_aio_fini(&p->aio_getq); + nni_aio_fini(&p->aio_send); + nni_aio_fini(&p->aio_recv); + nni_msgq_fini(p->sendq); +} + +static int +xresp0_pipe_init(void *arg, nni_pipe *npipe, void *s) +{ + xresp0_pipe *p = arg; + int rv; + + nni_aio_init(&p->aio_putq, xresp0_putq_cb, p); + nni_aio_init(&p->aio_recv, xresp0_recv_cb, p); + nni_aio_init(&p->aio_getq, xresp0_getq_cb, p); + nni_aio_init(&p->aio_send, xresp0_send_cb, p); + + if ((rv = nni_msgq_init(&p->sendq, 2)) != 0) { + xresp0_pipe_fini(p); + return (rv); + } + + p->npipe = npipe; + p->psock = s; + return (0); +} + +static int +xresp0_pipe_start(void *arg) +{ + xresp0_pipe *p = arg; + xresp0_sock *s = p->psock; + int rv; + + if (nni_pipe_peer(p->npipe) != NNI_PROTO_SURVEYOR_V0) { + return (NNG_EPROTO); + } + + p->id = nni_pipe_id(p->npipe); + + nni_mtx_lock(&s->mtx); + rv = nni_id_set(&s->pipes, p->id, p); + nni_mtx_unlock(&s->mtx); + if (rv != 0) { + return (rv); + } + + nni_pipe_recv(p->npipe, &p->aio_recv); + nni_msgq_aio_get(p->sendq, &p->aio_getq); + + return (rv); +} + +static void +xresp0_pipe_close(void *arg) +{ + xresp0_pipe *p = arg; + xresp0_sock *s = p->psock; + + nni_aio_close(&p->aio_putq); + nni_aio_close(&p->aio_getq); + nni_aio_close(&p->aio_send); + nni_aio_close(&p->aio_recv); + + nni_msgq_close(p->sendq); + + nni_mtx_lock(&s->mtx); + nni_id_remove(&s->pipes, p->id); + nni_mtx_unlock(&s->mtx); +} + +// resp0_sock_send watches for messages from the upper write queue, +// extracts the destination pipe, and forwards it to the appropriate +// destination pipe via a separate queue. This prevents a single bad +// or slow pipe from gumming up the works for the entire socket.s + +void +xresp0_sock_getq_cb(void *arg) +{ + xresp0_sock *s = arg; + nni_msg * msg; + uint32_t id; + xresp0_pipe *p; + + if (nni_aio_result(&s->aio_getq) != 0) { + return; + } + msg = nni_aio_get_msg(&s->aio_getq); + nni_aio_set_msg(&s->aio_getq, NULL); + + // We yank the outgoing pipe id from the header + if (nni_msg_header_len(msg) < 4) { + nni_msg_free(msg); + // We can't really close down the socket, so just keep going. + nni_msgq_aio_get(s->uwq, &s->aio_getq); + return; + } + id = nni_msg_header_trim_u32(msg); + + nni_mtx_lock(&s->mtx); + // Look for the pipe, and attempt to put the message there + // (nonblocking) if we can. If we can't for any reason, then we + // free the message. + if (((p = nni_id_get(&s->pipes, id)) == NULL) || + (nni_msgq_tryput(p->sendq, msg) != 0)) { + nni_msg_free(msg); + } + nni_mtx_unlock(&s->mtx); + nni_msgq_aio_get(s->uwq, &s->aio_getq); +} + +void +xresp0_getq_cb(void *arg) +{ + xresp0_pipe *p = arg; + + if (nni_aio_result(&p->aio_getq) != 0) { + nni_pipe_close(p->npipe); + return; + } + + nni_aio_set_msg(&p->aio_send, nni_aio_get_msg(&p->aio_getq)); + nni_aio_set_msg(&p->aio_getq, NULL); + + nni_pipe_send(p->npipe, &p->aio_send); +} + +void +xresp0_send_cb(void *arg) +{ + xresp0_pipe *p = arg; + + if (nni_aio_result(&p->aio_send) != 0) { + nni_msg_free(nni_aio_get_msg(&p->aio_send)); + nni_aio_set_msg(&p->aio_send, NULL); + nni_pipe_close(p->npipe); + return; + } + + nni_msgq_aio_get(p->sendq, &p->aio_getq); +} + +static void +xresp0_recv_cb(void *arg) +{ + xresp0_pipe *p = arg; + xresp0_sock *s = p->psock; + nni_msgq * urq = s->urq; + nni_msg * msg; + int hops; + int ttl; + + if (nni_aio_result(&p->aio_recv) != 0) { + nni_pipe_close(p->npipe); + return; + } + + ttl = nni_atomic_get(&s->ttl); + msg = nni_aio_get_msg(&p->aio_recv); + nni_aio_set_msg(&p->aio_recv, NULL); + nni_msg_set_pipe(msg, p->id); + + // Store the pipe id in the header, first thing. + nni_msg_header_append_u32(msg, p->id); + + // Move backtrace from body to header + hops = 1; + for (;;) { + bool end; + uint8_t *body; + + if (hops > ttl) { + goto drop; + } + hops++; + if (nni_msg_len(msg) < 4) { + // Peer sent us garbage, so kick it. + nni_msg_free(msg); + nni_pipe_close(p->npipe); + return; + } + body = nni_msg_body(msg); + end = ((body[0] & 0x80u) != 0); + if (nni_msg_header_append(msg, body, 4) != 0) { + goto drop; + } + nni_msg_trim(msg, 4); + if (end) { + break; + } + } + + // Now send it up. + nni_aio_set_msg(&p->aio_putq, msg); + nni_msgq_aio_put(urq, &p->aio_putq); + return; + +drop: + nni_msg_free(msg); + nni_pipe_recv(p->npipe, &p->aio_recv); +} + +static void +xresp0_putq_cb(void *arg) +{ + xresp0_pipe *p = arg; + + if (nni_aio_result(&p->aio_putq) != 0) { + nni_msg_free(nni_aio_get_msg(&p->aio_putq)); + nni_aio_set_msg(&p->aio_putq, NULL); + nni_pipe_close(p->npipe); + return; + } + + nni_pipe_recv(p->npipe, &p->aio_recv); +} + +static int +xresp0_sock_set_maxttl(void *arg, const void *buf, size_t sz, nni_opt_type t) +{ + xresp0_sock *s = arg; + int ttl; + int rv; + if ((rv = nni_copyin_int(&ttl, buf, sz, 1, NNI_MAX_MAX_TTL, t)) == 0) { + nni_atomic_set(&s->ttl, ttl); + } + return (rv); +} + +static int +xresp0_sock_get_maxttl(void *arg, void *buf, size_t *szp, nni_opt_type t) +{ + xresp0_sock *s = arg; + return (nni_copyout_int(nni_atomic_get(&s->ttl), buf, szp, t)); +} + +static void +xresp0_sock_send(void *arg, nni_aio *aio) +{ + xresp0_sock *s = arg; + + nni_msgq_aio_put(s->uwq, aio); +} + +static void +xresp0_sock_recv(void *arg, nni_aio *aio) +{ + xresp0_sock *s = arg; + + nni_msgq_aio_get(s->urq, aio); +} + +static nni_proto_pipe_ops xresp0_pipe_ops = { + .pipe_size = sizeof(xresp0_pipe), + .pipe_init = xresp0_pipe_init, + .pipe_fini = xresp0_pipe_fini, + .pipe_start = xresp0_pipe_start, + .pipe_close = xresp0_pipe_close, + .pipe_stop = xresp0_pipe_stop, +}; + +static nni_option xresp0_sock_options[] = { + { + .o_name = NNG_OPT_MAXTTL, + .o_get = xresp0_sock_get_maxttl, + .o_set = xresp0_sock_set_maxttl, + }, + // terminate list + { + .o_name = NULL, + }, +}; + +static nni_proto_sock_ops xresp0_sock_ops = { + .sock_size = sizeof(xresp0_sock), + .sock_init = xresp0_sock_init, + .sock_fini = xresp0_sock_fini, + .sock_open = xresp0_sock_open, + .sock_close = xresp0_sock_close, + .sock_send = xresp0_sock_send, + .sock_recv = xresp0_sock_recv, + .sock_options = xresp0_sock_options, +}; + +static nni_proto xresp0_proto = { + .proto_version = NNI_PROTOCOL_VERSION, + .proto_self = { NNI_PROTO_RESPONDENT_V0, "respondent" }, + .proto_peer = { NNI_PROTO_SURVEYOR_V0, "surveyor" }, + .proto_flags = NNI_PROTO_FLAG_SNDRCV | NNI_PROTO_FLAG_RAW, + .proto_sock_ops = &xresp0_sock_ops, + .proto_pipe_ops = &xresp0_pipe_ops, +}; + +int +nng_respondent0_open_raw(nng_socket *sidp) +{ + return (nni_proto_open(sidp, &xresp0_proto)); +} diff --git a/src/sp/protocol/survey0/xrespond_test.c b/src/sp/protocol/survey0/xrespond_test.c new file mode 100644 index 00000000..ec5e99a3 --- /dev/null +++ b/src/sp/protocol/survey0/xrespond_test.c @@ -0,0 +1,436 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include + +static void +test_xresp_identity(void) +{ + nng_socket s; + int p1, p2; + char * n1; + char * n2; + + NUTS_PASS(nng_respondent0_open_raw(&s)); + NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PROTO, &p1)); + NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PEER, &p2)); + NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PROTONAME, &n1)); + NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PEERNAME, &n2)); + NUTS_CLOSE(s); + NUTS_TRUE(p1 == NNG_RESPONDENT0_SELF); + NUTS_TRUE(p2 == NNG_RESPONDENT0_PEER); + NUTS_MATCH(n1, NNG_RESPONDENT0_SELF_NAME); + NUTS_MATCH(n2, NNG_RESPONDENT0_PEER_NAME); + nng_strfree(n1); + nng_strfree(n2); +} + +static void +test_xresp_raw(void) +{ + nng_socket s; + bool b; + + NUTS_PASS(nng_respondent0_open_raw(&s)); + NUTS_PASS(nng_socket_get_bool(s, NNG_OPT_RAW, &b)); + NUTS_TRUE(b); + NUTS_CLOSE(s); +} + +static void +test_xresp_no_context(void) +{ + nng_socket s; + nng_ctx ctx; + + NUTS_PASS(nng_respondent0_open_raw(&s)); + NUTS_FAIL(nng_ctx_open(&ctx, s), NNG_ENOTSUP); + NUTS_CLOSE(s); +} + +static void +test_xresp_poll_writeable(void) +{ + int fd; + nng_socket surv; + nng_socket resp; + + NUTS_PASS(nng_respondent0_open_raw(&resp)); + NUTS_PASS(nng_surveyor0_open(&surv)); + NUTS_PASS(nng_socket_get_int(resp, NNG_OPT_SENDFD, &fd)); + NUTS_TRUE(fd >= 0); + + // We are always writeable, even before connect. This is so that + // back-pressure from a bad peer can't trash others. We assume + // that peers won't send us requests faster than they can consume + // the answers. If they do, they will lose their answers. + NUTS_TRUE(nuts_poll_fd(fd) == true); + + NUTS_MARRY(surv, resp); + + // Now it's writable. + NUTS_TRUE(nuts_poll_fd(fd) == true); + + NUTS_CLOSE(surv); + NUTS_CLOSE(resp); +} + +static void +test_xresp_poll_readable(void) +{ + int fd; + nng_socket surv; + nng_socket resp; + nng_msg * msg; + + NUTS_PASS(nng_surveyor0_open(&surv)); + NUTS_PASS(nng_respondent0_open_raw(&resp)); + NUTS_PASS(nng_socket_get_int(resp, NNG_OPT_RECVFD, &fd)); + NUTS_TRUE(fd >= 0); + + // Not readable if not connected! + NUTS_TRUE(nuts_poll_fd(fd) == false); + + // Even after connect (no message yet) + NUTS_MARRY(surv, resp); + NUTS_TRUE(nuts_poll_fd(fd) == false); + + // But once we send messages, it is. + // We have to send a request, in order to send a reply. + NUTS_SEND(surv, "abc"); + NUTS_SLEEP(100); + + NUTS_TRUE(nuts_poll_fd(fd) == true); + + // and receiving makes it no longer ready + NUTS_PASS(nng_recvmsg(resp, &msg, 0)); + nng_msg_free(msg); + NUTS_TRUE(nuts_poll_fd(fd) == false); + + NUTS_CLOSE(surv); + NUTS_CLOSE(resp); +} + +static void +test_xresp_validate_peer(void) +{ + nng_socket s1, s2; + nng_stat * stats; + nng_stat * reject; + char * addr; + + NUTS_ADDR(addr, "inproc"); + + NUTS_PASS(nng_respondent0_open_raw(&s1)); + NUTS_PASS(nng_respondent0_open(&s2)); + + NUTS_PASS(nng_listen(s1, addr, NULL, 0)); + NUTS_PASS(nng_dial(s2, addr, NULL, NNG_FLAG_NONBLOCK)); + + NUTS_SLEEP(100); + NUTS_PASS(nng_stats_get(&stats)); + + NUTS_TRUE(stats != NULL); + NUTS_TRUE((reject = nng_stat_find_socket(stats, s1)) != NULL); + NUTS_TRUE((reject = nng_stat_find(reject, "reject")) != NULL); + + NUTS_TRUE(nng_stat_type(reject) == NNG_STAT_COUNTER); + NUTS_TRUE(nng_stat_value(reject) > 0); + + NUTS_CLOSE(s1); + NUTS_CLOSE(s2); + nng_stats_free(stats); +} + +static void +test_xresp_close_pipe_before_send(void) +{ + nng_socket resp; + nng_socket surv; + nng_pipe p; + nng_aio * aio1; + nng_msg * m; + + NUTS_PASS(nng_respondent0_open_raw(&resp)); + NUTS_PASS(nng_surveyor0_open(&surv)); + NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_aio_alloc(&aio1, NULL, NULL)); + + NUTS_MARRY(surv, resp); + NUTS_SEND(surv, "test"); + + nng_recv_aio(resp, aio1); + nng_aio_wait(aio1); + NUTS_PASS(nng_aio_result(aio1)); + NUTS_TRUE((m = nng_aio_get_msg(aio1)) != NULL); + p = nng_msg_get_pipe(m); + NUTS_PASS(nng_pipe_close(p)); + NUTS_PASS(nng_sendmsg(resp, m, 0)); + + NUTS_CLOSE(surv); + NUTS_CLOSE(resp); + nng_aio_free(aio1); +} + +static void +test_xresp_close_pipe_during_send(void) +{ + nng_socket resp; + nng_socket surv; + nng_pipe p; + nng_msg * m; + + NUTS_PASS(nng_respondent_open_raw(&resp)); + NUTS_PASS(nng_surveyor0_open_raw(&surv)); + NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_SENDTIMEO, 200)); + NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_int(resp, NNG_OPT_SENDBUF, 20)); + NUTS_PASS(nng_socket_set_int(resp, NNG_OPT_RECVBUF, 20)); + NUTS_PASS(nng_socket_set_int(surv, NNG_OPT_SENDBUF, 20)); + NUTS_PASS(nng_socket_set_int(surv, NNG_OPT_RECVBUF, 1)); + + NUTS_MARRY(surv, resp); + + NUTS_PASS(nng_msg_alloc(&m, 4)); + NUTS_PASS(nng_msg_append_u32(m, (unsigned) 0x81000000u)); + NUTS_PASS(nng_sendmsg(surv, m, 0)); + NUTS_PASS(nng_recvmsg(resp, &m, 0)); + p = nng_msg_get_pipe(m); + nng_msg_free(m); + + for (int i = 0; i < 100; i++) { + NUTS_PASS(nng_msg_alloc(&m, 4)); + NUTS_PASS(nng_msg_header_append_u32(m, nng_pipe_id(p))); + NUTS_PASS( + nng_msg_header_append_u32(m, (unsigned) i | 0x80000000u)); + // protocol does not exert back-pressure + NUTS_PASS(nng_sendmsg(resp, m, 0)); + } + NUTS_PASS(nng_pipe_close(p)); + + NUTS_CLOSE(surv); + NUTS_CLOSE(resp); +} + +static void +test_xresp_close_during_recv(void) +{ + nng_socket resp; + nng_socket surv; + nng_msg * m; + + NUTS_PASS(nng_respondent0_open_raw(&resp)); + NUTS_PASS(nng_surveyor0_open_raw(&surv)); + NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_SENDTIMEO, 100)); + NUTS_PASS(nng_socket_set_int(resp, NNG_OPT_RECVBUF, 5)); + NUTS_PASS(nng_socket_set_int(surv, NNG_OPT_SENDBUF, 20)); + + NUTS_MARRY(surv, resp); + + for (unsigned i = 0; i < 100; i++) { + int rv; + NUTS_PASS(nng_msg_alloc(&m, 4)); + NUTS_PASS(nng_msg_header_append_u32(m, i | 0x80000000u)); + rv = nng_sendmsg(surv, m, 0); + if (rv == NNG_ETIMEDOUT) { + nng_msg_free(m); + break; + } + } + NUTS_CLOSE(surv); + NUTS_CLOSE(resp); +} + +static void +test_xresp_recv_aio_stopped(void) +{ + nng_socket resp; + nng_aio * aio; + + NUTS_PASS(nng_respondent0_open_raw(&resp)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + + nng_aio_stop(aio); + nng_recv_aio(resp, aio); + nng_aio_wait(aio); + NUTS_FAIL(nng_aio_result(aio), NNG_ECANCELED); + NUTS_CLOSE(resp); + nng_aio_free(aio); +} + +static void +test_xresp_send_no_header(void) +{ + nng_socket resp; + nng_socket surv; + nng_msg * m; + + NUTS_PASS(nng_surveyor0_open_raw(&surv)); + NUTS_PASS(nng_respondent0_open_raw(&resp)); + NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_RECVTIMEO, 100)); + NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_RECVTIMEO, 100)); + NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_SENDTIMEO, 1000)); + + NUTS_MARRY(surv, resp); + + NUTS_PASS(nng_msg_alloc(&m, 4)); + NUTS_PASS(nng_sendmsg(resp, m, 0)); + NUTS_FAIL(nng_recvmsg(resp, &m, 0), NNG_ETIMEDOUT); + + NUTS_CLOSE(surv); + NUTS_CLOSE(resp); +} + +static void +test_xresp_recv_garbage(void) +{ + nng_socket resp; + nng_socket surv; + nng_msg * m; + + NUTS_PASS(nng_respondent0_open_raw(&resp)); + NUTS_PASS(nng_surveyor0_open_raw(&surv)); + NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_RECVTIMEO, 100)); + NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_SENDTIMEO, 100)); + NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_SENDTIMEO, 1000)); + + NUTS_MARRY(surv, resp); + + NUTS_PASS(nng_msg_alloc(&m, 4)); + NUTS_PASS(nng_msg_append_u32(m, 1u)); + NUTS_PASS(nng_sendmsg(surv, m, 0)); + NUTS_FAIL(nng_recvmsg(resp, &m, 0), NNG_ETIMEDOUT); + + NUTS_CLOSE(surv); + NUTS_CLOSE(resp); +} + +static void +test_xresp_ttl_option(void) +{ + nng_socket resp; + int v; + bool b; + size_t sz; + const char *opt = NNG_OPT_MAXTTL; + + NUTS_PASS(nng_respondent0_open_raw(&resp)); + + NUTS_PASS(nng_socket_set_int(resp, opt, 1)); + NUTS_FAIL(nng_socket_set_int(resp, opt, 0), NNG_EINVAL); + NUTS_FAIL(nng_socket_set_int(resp, opt, -1), NNG_EINVAL); + NUTS_FAIL(nng_socket_set_int(resp, opt, 16), NNG_EINVAL); + NUTS_FAIL(nng_socket_set_int(resp, opt, 256), NNG_EINVAL); + NUTS_PASS(nng_socket_set_int(resp, opt, 3)); + NUTS_PASS(nng_socket_get_int(resp, opt, &v)); + NUTS_TRUE(v == 3); + v = 0; + sz = sizeof(v); + NUTS_PASS(nng_socket_get(resp, opt, &v, &sz)); + NUTS_TRUE(v == 3); + NUTS_TRUE(sz == sizeof(v)); + + NUTS_FAIL(nng_socket_set(resp, opt, "", 1), NNG_EINVAL); + sz = 1; + NUTS_FAIL(nng_socket_get(resp, opt, &v, &sz), NNG_EINVAL); + NUTS_FAIL(nng_socket_set_bool(resp, opt, true), NNG_EBADTYPE); + NUTS_FAIL(nng_socket_get_bool(resp, opt, &b), NNG_EBADTYPE); + + NUTS_CLOSE(resp); +} + +static void +test_xresp_ttl_drop(void) +{ + nng_socket resp; + nng_socket surv; + nng_msg * m; + + NUTS_PASS(nng_respondent0_open_raw(&resp)); + NUTS_PASS(nng_surveyor0_open_raw(&surv)); + NUTS_PASS(nng_socket_set_int(resp, NNG_OPT_MAXTTL, 3)); + NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_RECVTIMEO, 200)); + NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_SENDTIMEO, 1000)); + + NUTS_MARRY(surv, resp); + + // Send messages. Note that xresp implicitly adds a hop on receive. + + NUTS_PASS(nng_msg_alloc(&m, 0)); + NUTS_PASS(nng_msg_append_u32(m, 1u)); // 2 hops + NUTS_PASS(nng_msg_append_u32(m, 0x80000001u)); + NUTS_PASS(nng_msg_append(m, "PASS1", 6)); + NUTS_PASS(nng_sendmsg(surv, m, 0)); + + NUTS_PASS(nng_msg_alloc(&m, 0)); + NUTS_PASS(nng_msg_append_u32(m, 1u)); // 4 hops -- discard! + NUTS_PASS(nng_msg_append_u32(m, 2u)); + NUTS_PASS(nng_msg_append_u32(m, 3u)); + NUTS_PASS(nng_msg_append_u32(m, 0x80000002u)); + NUTS_PASS(nng_msg_append(m, "FAIL2", 6)); + NUTS_PASS(nng_sendmsg(surv, m, 0)); + + NUTS_PASS(nng_msg_alloc(&m, 0)); + NUTS_PASS(nng_msg_append_u32(m, 1u)); // 3 hops - passes + NUTS_PASS(nng_msg_append_u32(m, 2u)); + NUTS_PASS(nng_msg_append_u32(m, 0x80000003u)); + NUTS_PASS(nng_msg_append(m, "PASS3", 6)); + NUTS_PASS(nng_sendmsg(surv, m, 0)); + + NUTS_PASS(nng_msg_alloc(&m, 0)); + NUTS_PASS(nng_msg_append_u32(m, 1u)); // 4 hops -- discard! + NUTS_PASS(nng_msg_append_u32(m, 2u)); + NUTS_PASS(nng_msg_append_u32(m, 3u)); + NUTS_PASS(nng_msg_append_u32(m, 0x80000003u)); + NUTS_PASS(nng_msg_append(m, "FAIL4", 6)); + NUTS_PASS(nng_sendmsg(surv, m, 0)); + + // So on receive we should see 80000001 and 80000003. + NUTS_PASS(nng_recvmsg(resp, &m, 0)); + NUTS_TRUE(nng_msg_header_len(m) == 12); + NUTS_TRUE(nng_msg_len(m) == 6); + NUTS_MATCH(nng_msg_body(m), "PASS1"); + nng_msg_free(m); + + NUTS_PASS(nng_recvmsg(resp, &m, 0)); + NUTS_TRUE(nng_msg_header_len(m) == 16); // 3 hops + ID + NUTS_TRUE(nng_msg_len(m) == 6); + NUTS_MATCH(nng_msg_body(m), "PASS3"); + nng_msg_free(m); + + NUTS_FAIL(nng_recvmsg(resp, &m, 0), NNG_ETIMEDOUT); + + NUTS_CLOSE(surv); + NUTS_CLOSE(resp); +} + +NUTS_TESTS = { + { "xrespond identity", test_xresp_identity }, + { "xrespond raw", test_xresp_raw }, + { "xrespond no context", test_xresp_no_context }, + { "xrespond poll readable", test_xresp_poll_readable }, + { "xrespond poll writable", test_xresp_poll_writeable }, + { "xrespond validate peer", test_xresp_validate_peer }, + { "xrespond close pipe before send", + test_xresp_close_pipe_before_send }, + { "xrespond close pipe during send", + test_xresp_close_pipe_during_send }, + { "xrespond close during recv", test_xresp_close_during_recv }, + { "xrespond recv aio stopped", test_xresp_recv_aio_stopped }, + { "xrespond send no header", test_xresp_send_no_header }, + { "xrespond recv garbage", test_xresp_recv_garbage }, + { "xrespond ttl option", test_xresp_ttl_option }, + { "xrespond ttl drop", test_xresp_ttl_drop }, + { NULL, NULL }, +}; diff --git a/src/sp/protocol/survey0/xsurvey.c b/src/sp/protocol/survey0/xsurvey.c new file mode 100644 index 00000000..2a198662 --- /dev/null +++ b/src/sp/protocol/survey0/xsurvey.c @@ -0,0 +1,379 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// Copyright 2018 Capitar IT Group BV +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include "core/nng_impl.h" +#include "nng/protocol/survey0/survey.h" + +// Surveyor protocol. The SURVEYOR protocol is the "survey" side of the +// survey pattern. This is useful for building service discovery, voting, etc. + +typedef struct xsurv0_pipe xsurv0_pipe; +typedef struct xsurv0_sock xsurv0_sock; + +static void xsurv0_sock_getq_cb(void *); +static void xsurv0_getq_cb(void *); +static void xsurv0_putq_cb(void *); +static void xsurv0_send_cb(void *); +static void xsurv0_recv_cb(void *); + +// surv0_sock is our per-socket protocol private structure. +struct xsurv0_sock { + nni_list pipes; + nni_aio aio_getq; + nni_msgq * uwq; + nni_msgq * urq; + nni_mtx mtx; + nni_atomic_int ttl; +}; + +// surv0_pipe is our per-pipe protocol private structure. +struct xsurv0_pipe { + nni_pipe * npipe; + xsurv0_sock * psock; + nni_msgq * sendq; + nni_list_node node; + nni_aio aio_getq; + nni_aio aio_putq; + nni_aio aio_send; + nni_aio aio_recv; +}; + +static void +xsurv0_sock_fini(void *arg) +{ + xsurv0_sock *s = arg; + + nni_aio_fini(&s->aio_getq); + nni_mtx_fini(&s->mtx); +} + +static int +xsurv0_sock_init(void *arg, nni_sock *nsock) +{ + xsurv0_sock *s = arg; + + nni_aio_init(&s->aio_getq, xsurv0_sock_getq_cb, s); + NNI_LIST_INIT(&s->pipes, xsurv0_pipe, node); + nni_mtx_init(&s->mtx); + + s->uwq = nni_sock_sendq(nsock); + s->urq = nni_sock_recvq(nsock); + nni_atomic_init(&s->ttl); + nni_atomic_set(&s->ttl, 8); + + return (0); +} + +static void +xsurv0_sock_open(void *arg) +{ + xsurv0_sock *s = arg; + + nni_msgq_aio_get(s->uwq, &s->aio_getq); +} + +static void +xsurv0_sock_close(void *arg) +{ + xsurv0_sock *s = arg; + + nni_aio_close(&s->aio_getq); +} + +static void +xsurv0_pipe_stop(void *arg) +{ + xsurv0_pipe *p = arg; + + nni_aio_stop(&p->aio_getq); + nni_aio_stop(&p->aio_send); + nni_aio_stop(&p->aio_recv); + nni_aio_stop(&p->aio_putq); +} + +static void +xsurv0_pipe_fini(void *arg) +{ + xsurv0_pipe *p = arg; + + nni_aio_fini(&p->aio_getq); + nni_aio_fini(&p->aio_send); + nni_aio_fini(&p->aio_recv); + nni_aio_fini(&p->aio_putq); + nni_msgq_fini(p->sendq); +} + +static int +xsurv0_pipe_init(void *arg, nni_pipe *npipe, void *s) +{ + xsurv0_pipe *p = arg; + int rv; + + nni_aio_init(&p->aio_getq, xsurv0_getq_cb, p); + nni_aio_init(&p->aio_putq, xsurv0_putq_cb, p); + nni_aio_init(&p->aio_send, xsurv0_send_cb, p); + nni_aio_init(&p->aio_recv, xsurv0_recv_cb, p); + + // This depth could be tunable. The queue exists so that if we + // have multiple requests coming in faster than we can deliver them, + // we try to avoid dropping them. We don't really have a solution + // for applying back pressure. It would be nice if surveys carried + // an expiration with them, so that we could discard any that are + // not delivered before their expiration date. + if ((rv = nni_msgq_init(&p->sendq, 16)) != 0) { + xsurv0_pipe_fini(p); + return (rv); + } + + p->npipe = npipe; + p->psock = s; + return (0); +} + +static int +xsurv0_pipe_start(void *arg) +{ + xsurv0_pipe *p = arg; + xsurv0_sock *s = p->psock; + + if (nni_pipe_peer(p->npipe) != NNG_SURVEYOR0_PEER) { + return (NNG_EPROTO); + } + + nni_mtx_lock(&s->mtx); + nni_list_append(&s->pipes, p); + nni_mtx_unlock(&s->mtx); + + nni_msgq_aio_get(p->sendq, &p->aio_getq); + nni_pipe_recv(p->npipe, &p->aio_recv); + return (0); +} + +static void +xsurv0_pipe_close(void *arg) +{ + xsurv0_pipe *p = arg; + xsurv0_sock *s = p->psock; + + nni_aio_close(&p->aio_getq); + nni_aio_close(&p->aio_send); + nni_aio_close(&p->aio_recv); + nni_aio_close(&p->aio_putq); + + nni_msgq_close(p->sendq); + + nni_mtx_lock(&s->mtx); + if (nni_list_active(&s->pipes, p)) { + nni_list_remove(&s->pipes, p); + } + nni_mtx_unlock(&s->mtx); +} + +static void +xsurv0_getq_cb(void *arg) +{ + xsurv0_pipe *p = arg; + + if (nni_aio_result(&p->aio_getq) != 0) { + nni_pipe_close(p->npipe); + return; + } + + nni_aio_set_msg(&p->aio_send, nni_aio_get_msg(&p->aio_getq)); + nni_aio_set_msg(&p->aio_getq, NULL); + + nni_pipe_send(p->npipe, &p->aio_send); +} + +static void +xsurv0_send_cb(void *arg) +{ + xsurv0_pipe *p = arg; + + if (nni_aio_result(&p->aio_send) != 0) { + nni_msg_free(nni_aio_get_msg(&p->aio_send)); + nni_aio_set_msg(&p->aio_send, NULL); + nni_pipe_close(p->npipe); + return; + } + + nni_msgq_aio_get(p->sendq, &p->aio_getq); +} + +static void +xsurv0_putq_cb(void *arg) +{ + xsurv0_pipe *p = arg; + + if (nni_aio_result(&p->aio_putq) != 0) { + nni_msg_free(nni_aio_get_msg(&p->aio_putq)); + nni_aio_set_msg(&p->aio_putq, NULL); + nni_pipe_close(p->npipe); + return; + } + + nni_pipe_recv(p->npipe, &p->aio_recv); +} + +static void +xsurv0_recv_cb(void *arg) +{ + xsurv0_pipe *p = arg; + nni_msg * msg; + bool end; + + if (nni_aio_result(&p->aio_recv) != 0) { + nni_pipe_close(p->npipe); + return; + } + + msg = nni_aio_get_msg(&p->aio_recv); + nni_aio_set_msg(&p->aio_recv, NULL); + nni_msg_set_pipe(msg, nni_pipe_id(p->npipe)); + end = false; + + while (!end) { + uint8_t *body; + + if (nni_msg_len(msg) < 4) { + // Peer gave us garbage, so kick it. + nni_msg_free(msg); + nni_pipe_close(p->npipe); + return; + } + body = nni_msg_body(msg); + end = ((body[0] & 0x80u) != 0); + + if (nni_msg_header_append(msg, body, sizeof(uint32_t)) != 0) { + // TODO: bump a no-memory stat + nni_msg_free(msg); + // Closing the pipe may release some memory. + // It at least gives an indication to the peer + // that we've lost the message. + nni_pipe_close(p->npipe); + return; + } + nni_msg_trim(msg, sizeof(uint32_t)); + } + + nni_aio_set_msg(&p->aio_putq, msg); + nni_msgq_aio_put(p->psock->urq, &p->aio_putq); +} + +static int +xsurv0_sock_set_max_ttl(void *arg, const void *buf, size_t sz, nni_opt_type t) +{ + xsurv0_sock *s = arg; + int ttl; + int rv; + if ((rv = nni_copyin_int(&ttl, buf, sz, 1, NNI_MAX_MAX_TTL, t)) == 0) { + nni_atomic_set(&s->ttl, ttl); + } + return (rv); +} + +static int +xsurv0_sock_get_max_ttl(void *arg, void *buf, size_t *szp, nni_opt_type t) +{ + xsurv0_sock *s = arg; + return (nni_copyout_int(nni_atomic_get(&s->ttl), buf, szp, t)); +} + +static void +xsurv0_sock_getq_cb(void *arg) +{ + xsurv0_sock *s = arg; + xsurv0_pipe *p; + nni_msg * msg; + + if (nni_aio_result(&s->aio_getq) != 0) { + // Should be NNG_ECLOSED. + return; + } + msg = nni_aio_get_msg(&s->aio_getq); + nni_aio_set_msg(&s->aio_getq, NULL); + + nni_mtx_lock(&s->mtx); + NNI_LIST_FOREACH (&s->pipes, p) { + nni_msg_clone(msg); + if (nni_msgq_tryput(p->sendq, msg) != 0) { + nni_msg_free(msg); + } + } + + nni_msgq_aio_get(s->uwq, &s->aio_getq); + nni_mtx_unlock(&s->mtx); + + // If there were no pipes to send on, just toss the message. + nni_msg_free(msg); +} + +static void +xsurv0_sock_recv(void *arg, nni_aio *aio) +{ + xsurv0_sock *s = arg; + + nni_msgq_aio_get(s->urq, aio); +} + +static void +xsurv0_sock_send(void *arg, nni_aio *aio) +{ + xsurv0_sock *s = arg; + + nni_msgq_aio_put(s->uwq, aio); +} + +static nni_proto_pipe_ops xsurv0_pipe_ops = { + .pipe_size = sizeof(xsurv0_pipe), + .pipe_init = xsurv0_pipe_init, + .pipe_fini = xsurv0_pipe_fini, + .pipe_start = xsurv0_pipe_start, + .pipe_close = xsurv0_pipe_close, + .pipe_stop = xsurv0_pipe_stop, +}; + +static nni_option xsurv0_sock_options[] = { + { + .o_name = NNG_OPT_MAXTTL, + .o_get = xsurv0_sock_get_max_ttl, + .o_set = xsurv0_sock_set_max_ttl, + }, + // terminate list + { + .o_name = NULL, + }, +}; + +static nni_proto_sock_ops xsurv0_sock_ops = { + .sock_size = sizeof(xsurv0_sock), + .sock_init = xsurv0_sock_init, + .sock_fini = xsurv0_sock_fini, + .sock_open = xsurv0_sock_open, + .sock_close = xsurv0_sock_close, + .sock_send = xsurv0_sock_send, + .sock_recv = xsurv0_sock_recv, + .sock_options = xsurv0_sock_options, +}; + +static nni_proto xsurv0_proto = { + .proto_version = NNI_PROTOCOL_VERSION, + .proto_self = { NNG_SURVEYOR0_SELF, NNG_SURVEYOR0_SELF_NAME }, + .proto_peer = { NNG_SURVEYOR0_PEER, NNG_SURVEYOR0_PEER_NAME }, + .proto_flags = NNI_PROTO_FLAG_SNDRCV | NNI_PROTO_FLAG_RAW, + .proto_sock_ops = &xsurv0_sock_ops, + .proto_pipe_ops = &xsurv0_pipe_ops, +}; + +int +nng_surveyor0_open_raw(nng_socket *sidp) +{ + return (nni_proto_open(sidp, &xsurv0_proto)); +} diff --git a/src/sp/protocol/survey0/xsurvey_test.c b/src/sp/protocol/survey0/xsurvey_test.c new file mode 100644 index 00000000..f8e9d401 --- /dev/null +++ b/src/sp/protocol/survey0/xsurvey_test.c @@ -0,0 +1,399 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include + +static void +test_xsurveyor_identity(void) +{ + nng_socket s; + int p; + char * n; + + NUTS_PASS(nng_surveyor0_open_raw(&s)); + NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PROTO, &p)); + NUTS_TRUE(p == NNG_SURVEYOR0_SELF); // 0x62 + NUTS_PASS(nng_socket_get_int(s, NNG_OPT_PEER, &p)); + NUTS_TRUE(p == NNG_SURVEYOR0_PEER); // 0x62 + NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PROTONAME, &n)); + NUTS_MATCH(n, NNG_SURVEYOR0_SELF_NAME); + nng_strfree(n); + NUTS_PASS(nng_socket_get_string(s, NNG_OPT_PEERNAME, &n)); + NUTS_MATCH(n, NNG_SURVEYOR0_PEER_NAME); + nng_strfree(n); + NUTS_CLOSE(s); +} + +static void +test_xsurveyor_raw(void) +{ + nng_socket s; + bool b; + + NUTS_PASS(nng_surveyor0_open_raw(&s)); + NUTS_PASS(nng_socket_get_bool(s, NNG_OPT_RAW, &b)); + NUTS_TRUE(b); + NUTS_CLOSE(s); +} + +static void +test_xsurvey_no_context(void) +{ + nng_socket s; + nng_ctx ctx; + + NUTS_PASS(nng_surveyor0_open_raw(&s)); + NUTS_FAIL(nng_ctx_open(&ctx, s), NNG_ENOTSUP); + NUTS_CLOSE(s); +} + +static void +test_xsurvey_poll_writeable(void) +{ + int fd; + nng_socket surv; + nng_socket resp; + + NUTS_PASS(nng_surveyor0_open_raw(&surv)); + NUTS_PASS(nng_respondent0_open(&resp)); + NUTS_PASS(nng_socket_get_int(surv, NNG_OPT_SENDFD, &fd)); + NUTS_TRUE(fd >= 0); + + // Survey is broadcast, so we can always write. + NUTS_TRUE(nuts_poll_fd(fd)); + + NUTS_MARRY(surv, resp); + + // Now it's writable. + NUTS_TRUE(nuts_poll_fd(fd)); + + NUTS_CLOSE(surv); + NUTS_CLOSE(resp); +} + +static void +test_xsurvey_poll_readable(void) +{ + int fd; + nng_socket surv; + nng_socket resp; + nng_msg * msg; + + NUTS_PASS(nng_surveyor0_open_raw(&surv)); + NUTS_PASS(nng_respondent0_open(&resp)); + NUTS_PASS(nng_socket_get_int(surv, NNG_OPT_RECVFD, &fd)); + NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_SENDTIMEO, 1000)); + + NUTS_TRUE(fd >= 0); + + // Not readable if not connected! + NUTS_TRUE(nuts_poll_fd(fd) == false); + + // Even after connect (no message yet) + NUTS_MARRY(surv, resp); + NUTS_TRUE(nuts_poll_fd(fd) == false); + + // But once we send messages, it is. + // We have to send a request, in order to send a reply. + NUTS_PASS(nng_msg_alloc(&msg, 0)); + // Request ID + NUTS_PASS(nng_msg_append_u32(msg, 0x80000000)); + NUTS_PASS(nng_sendmsg(surv, msg, 0)); + + NUTS_PASS(nng_recvmsg(resp, &msg, 0)); + NUTS_PASS(nng_sendmsg(resp, msg, 0)); + + NUTS_SLEEP(100); + + NUTS_TRUE(nuts_poll_fd(fd) ); + + // and receiving makes it no longer ready + NUTS_PASS(nng_recvmsg(surv, &msg, 0)); + nng_msg_free(msg); + NUTS_TRUE(nuts_poll_fd(fd) == false); + + NUTS_CLOSE(surv); + NUTS_CLOSE(resp); +} + +static void +test_xsurvey_validate_peer(void) +{ + nng_socket s1, s2; + nng_stat * stats; + nng_stat * reject; + char *addr; + + NUTS_ADDR(addr, "inproc"); + + NUTS_PASS(nng_surveyor0_open_raw(&s1)); + NUTS_PASS(nng_surveyor0_open(&s2)); + + NUTS_PASS(nng_listen(s1, addr, NULL, 0)); + NUTS_PASS(nng_dial(s2, addr, NULL, NNG_FLAG_NONBLOCK)); + + NUTS_SLEEP(100); + NUTS_PASS(nng_stats_get(&stats)); + + NUTS_TRUE(stats != NULL); + NUTS_TRUE((reject = nng_stat_find_socket(stats, s1)) != NULL); + NUTS_TRUE((reject = nng_stat_find(reject, "reject")) != NULL); + + NUTS_TRUE(nng_stat_type(reject) == NNG_STAT_COUNTER); + NUTS_TRUE(nng_stat_value(reject) > 0); + + NUTS_CLOSE(s1); + NUTS_CLOSE(s2); + nng_stats_free(stats); +} + +static void +test_xsurvey_recv_aio_stopped(void) +{ + nng_socket surv; + nng_aio * aio; + + NUTS_PASS(nng_surveyor0_open_raw(&surv)); + NUTS_PASS(nng_aio_alloc(&aio, NULL, NULL)); + + nng_aio_stop(aio); + nng_recv_aio(surv, aio); + nng_aio_wait(aio); + NUTS_FAIL(nng_aio_result(aio), NNG_ECANCELED); + NUTS_CLOSE(surv); + nng_aio_free(aio); +} + +static void +test_xsurvey_recv_garbage(void) +{ + nng_socket resp; + nng_socket surv; + nng_msg * m; + uint32_t req_id; + + NUTS_PASS(nng_respondent0_open_raw(&resp)); + NUTS_PASS(nng_surveyor0_open_raw(&surv)); + NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_RECVTIMEO, 100)); + NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_SENDTIMEO, 1000)); + + NUTS_MARRY(surv, resp); + + NUTS_PASS(nng_msg_alloc(&m, 0)); + NUTS_PASS(nng_msg_append_u32(m, 0x80000000)); + NUTS_PASS(nng_sendmsg(surv, m, 0)); + + NUTS_PASS(nng_recvmsg(resp, &m, 0)); + + // The message will have a header that contains the 32-bit pipe ID, + // followed by the 32-bit request ID. We will discard the request + // ID before sending it out. + NUTS_TRUE(nng_msg_header_len(m) == 8); + NUTS_PASS(nng_msg_header_chop_u32(m, &req_id)); + NUTS_TRUE(req_id == 0x80000000); + + NUTS_PASS(nng_sendmsg(resp, m, 0)); + NUTS_FAIL(nng_recvmsg(surv, &m, 0), NNG_ETIMEDOUT); + + NUTS_CLOSE(surv); + NUTS_CLOSE(resp); +} + +static void +test_xsurvey_recv_header(void) +{ + nng_socket resp; + nng_socket surv; + nng_msg * m; + nng_pipe p; + uint32_t id; + + NUTS_PASS(nng_respondent0_open_raw(&resp)); + NUTS_PASS(nng_surveyor0_open_raw(&surv)); + NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_SENDTIMEO, 1000)); + + NUTS_MARRY_EX(surv, resp, NULL, NULL, &p); + + // Simulate a few hops. + NUTS_PASS(nng_msg_alloc(&m, 0)); + NUTS_PASS(nng_msg_header_append_u32(m, nng_pipe_id(p))); + NUTS_PASS(nng_msg_header_append_u32(m, 0x2)); + NUTS_PASS(nng_msg_header_append_u32(m, 0x1)); + NUTS_PASS(nng_msg_header_append_u32(m, 0x80000123u)); + + NUTS_PASS(nng_sendmsg(resp, m, 0)); + + NUTS_PASS(nng_recvmsg(surv, &m, 0)); + NUTS_TRUE(nng_msg_header_len(m) == 12); + NUTS_PASS(nng_msg_header_trim_u32(m, &id)); + NUTS_TRUE(id == 0x2); + NUTS_PASS(nng_msg_header_trim_u32(m, &id)); + NUTS_TRUE(id == 0x1); + NUTS_PASS(nng_msg_header_trim_u32(m, &id)); + NUTS_TRUE(id == 0x80000123u); + + nng_msg_free(m); + + NUTS_CLOSE(surv); + NUTS_CLOSE(resp); +} + +static void +test_xsurvey_close_during_recv(void) +{ + nng_socket resp; + nng_socket surv; + nng_msg * m; + nng_pipe p1; + nng_pipe p2; + + NUTS_PASS(nng_respondent0_open_raw(&resp)); + NUTS_PASS(nng_surveyor0_open_raw(&surv)); + NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_SENDTIMEO, 100)); + NUTS_PASS(nng_socket_set_int(surv, NNG_OPT_RECVBUF, 1)); + NUTS_PASS(nng_socket_set_int(resp, NNG_OPT_SENDBUF, 20)); + + NUTS_MARRY_EX(surv, resp, NULL, &p1, &p2); + NUTS_TRUE(nng_pipe_id(p1) > 0); + NUTS_TRUE(nng_pipe_id(p2) > 0); + + for (unsigned i = 0; i < 20; i++) { + NUTS_PASS(nng_msg_alloc(&m, 4)); + NUTS_PASS(nng_msg_header_append_u32(m, nng_pipe_id(p2))); + NUTS_PASS(nng_msg_header_append_u32(m, i | 0x80000000u)); + NUTS_SLEEP(10); + NUTS_PASS(nng_sendmsg(resp, m, 0)); + } + NUTS_CLOSE(surv); + NUTS_CLOSE(resp); +} + +static void +test_xsurvey_close_pipe_during_send(void) +{ + nng_socket resp; + nng_socket surv; + nng_msg * m; + nng_pipe p1; + nng_pipe p2; + + NUTS_PASS(nng_respondent0_open_raw(&resp)); + NUTS_PASS(nng_surveyor0_open_raw(&surv)); + NUTS_PASS(nng_socket_set_ms(resp, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_SENDTIMEO, 100)); + NUTS_PASS(nng_socket_set_int(resp, NNG_OPT_RECVBUF, 5)); + NUTS_PASS(nng_socket_set_int(surv, NNG_OPT_SENDBUF, 20)); + + NUTS_MARRY_EX(surv, resp, NULL, &p1, &p2); + NUTS_TRUE(nng_pipe_id(p1) > 0); + NUTS_TRUE(nng_pipe_id(p2) > 0); + + for (unsigned i = 0; i < 20; i++) { + NUTS_PASS(nng_msg_alloc(&m, 4)); + NUTS_PASS(nng_msg_header_append_u32(m, i | 0x80000000u)); + NUTS_SLEEP(10); + NUTS_PASS(nng_sendmsg(surv, m, 0)); + } + + NUTS_PASS(nng_pipe_close(p1)); + NUTS_CLOSE(surv); + NUTS_CLOSE(resp); +} + +static void +test_xsurvey_ttl_option(void) +{ + nng_socket s; + int v; + bool b; + size_t sz; + const char *opt = NNG_OPT_MAXTTL; + + NUTS_PASS(nng_surveyor0_open_raw(&s)); + + NUTS_PASS(nng_socket_set_int(s, opt, 1)); + NUTS_FAIL(nng_socket_set_int(s, opt, 0), NNG_EINVAL); + NUTS_FAIL(nng_socket_set_int(s, opt, -1), NNG_EINVAL); + NUTS_FAIL(nng_socket_set_int(s, opt, 16), NNG_EINVAL); + NUTS_FAIL(nng_socket_set_int(s, opt, 256), NNG_EINVAL); + NUTS_PASS(nng_socket_set_int(s, opt, 3)); + NUTS_PASS(nng_socket_get_int(s, opt, &v)); + NUTS_TRUE(v == 3); + v = 0; + sz = sizeof(v); + NUTS_PASS(nng_socket_get(s, opt, &v, &sz)); + NUTS_TRUE(v == 3); + NUTS_TRUE(sz == sizeof(v)); + + NUTS_FAIL(nng_socket_set(s, opt, "", 1) , NNG_EINVAL); + sz = 1; + NUTS_FAIL(nng_socket_get(s, opt, &v, &sz) , NNG_EINVAL); + NUTS_FAIL(nng_socket_set_bool(s, opt, true) , NNG_EBADTYPE); + NUTS_FAIL(nng_socket_get_bool(s, opt, &b) , NNG_EBADTYPE); + + NUTS_CLOSE(s); +} + +static void +test_xsurvey_broadcast(void) +{ + nng_socket resp1; + nng_socket resp2; + nng_socket surv; + nng_msg * m; + + NUTS_PASS(nng_respondent0_open(&resp1)); + NUTS_PASS(nng_respondent0_open(&resp2)); + NUTS_PASS(nng_surveyor0_open_raw(&surv)); + NUTS_PASS(nng_socket_set_ms(resp1, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(resp2, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(surv, NNG_OPT_SENDTIMEO, 100)); + + NUTS_MARRY(surv, resp1); + NUTS_MARRY(surv, resp2); + + NUTS_PASS(nng_msg_alloc(&m, 0)); + NUTS_PASS(nng_msg_header_append_u32(m, 0x80000002u)); + NUTS_PASS(nng_msg_append(m, "hello", 6)); + + NUTS_PASS(nng_sendmsg(surv, m, 0)); + NUTS_RECV(resp1, "hello"); + NUTS_RECV(resp2, "hello"); + + NUTS_CLOSE(surv); + NUTS_CLOSE(resp1); + NUTS_CLOSE(resp2); +} + +TEST_LIST = { + { "xsurvey identity", test_xsurveyor_identity }, + { "xsurvey raw", test_xsurveyor_raw }, + { "xsurvey no context", test_xsurvey_no_context }, + { "xsurvey poll readable", test_xsurvey_poll_readable }, + { "xsurvey poll writable", test_xsurvey_poll_writeable }, + { "xsurvey validate peer", test_xsurvey_validate_peer }, + { "xsurvey recv aio stopped", test_xsurvey_recv_aio_stopped }, + { "xsurvey recv garbage", test_xsurvey_recv_garbage }, + { "xsurvey recv header", test_xsurvey_recv_header }, + { "xsurvey close during recv", test_xsurvey_close_during_recv }, + { "xsurvey close pipe during send", + test_xsurvey_close_pipe_during_send }, + { "xsurvey ttl option", test_xsurvey_ttl_option }, + { "xsurvey broadcast", test_xsurvey_broadcast }, + { NULL, NULL }, +}; diff --git a/src/sp/transport/CMakeLists.txt b/src/sp/transport/CMakeLists.txt new file mode 100644 index 00000000..add8a9c9 --- /dev/null +++ b/src/sp/transport/CMakeLists.txt @@ -0,0 +1,19 @@ +# +# Copyright 2020 Staysail Systems, Inc. +# +# This software is supplied under the terms of the MIT License, a +# copy of which should be located in the distribution where this +# file was obtained (LICENSE.txt). A copy of the license may also be +# found online at https://opensource.org/licenses/MIT. +# + +# Transports. +nng_directory(transport) + +add_subdirectory(inproc) +add_subdirectory(ipc) +add_subdirectory(tcp) +add_subdirectory(tls) +add_subdirectory(ws) +add_subdirectory(zerotier) + diff --git a/src/sp/transport/inproc/CMakeLists.txt b/src/sp/transport/inproc/CMakeLists.txt new file mode 100644 index 00000000..317686bb --- /dev/null +++ b/src/sp/transport/inproc/CMakeLists.txt @@ -0,0 +1,16 @@ +# +# Copyright 2020 Staysail Systems, Inc. +# Copyright 2018 Capitar IT Group BV +# +# This software is supplied under the terms of the MIT License, a +# copy of which should be located in the distribution where this +# file was obtained (LICENSE.txt). A copy of the license may also be +# found online at https://opensource.org/licenses/MIT. +# + +# inproc protocol +nng_directory(inproc) + +nng_sources_if(NNG_TRANSPORT_INPROC inproc.c) +nng_headers_if(NNG_TRANSPORT_INPROC nng/transport/inproc/inproc.h) +nng_defines_if(NNG_TRANSPORT_INPROC NNG_TRANSPORT_INPROC) \ No newline at end of file diff --git a/src/sp/transport/inproc/inproc.c b/src/sp/transport/inproc/inproc.c new file mode 100644 index 00000000..84e2c625 --- /dev/null +++ b/src/sp/transport/inproc/inproc.c @@ -0,0 +1,692 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// Copyright 2018 Capitar IT Group BV +// Copyright 2018 Devolutions +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include + +#include "core/nng_impl.h" + +// Inproc transport. This just transports messages from one +// peer to another. The inproc transport is only valid within the same +// process. + +typedef struct inproc_pair inproc_pair; +typedef struct inproc_pipe inproc_pipe; +typedef struct inproc_ep inproc_ep; +typedef struct inproc_queue inproc_queue; + +typedef struct { + nni_mtx mx; + nni_list servers; +} inproc_global; + +// inproc_pipe represents one half of a connection. +struct inproc_pipe { + const char * addr; + inproc_pair * pair; + inproc_queue *recv_queue; + inproc_queue *send_queue; + uint16_t peer; + uint16_t proto; +}; + +struct inproc_queue { + nni_list readers; + nni_list writers; + nni_mtx lock; + bool closed; +}; + +// inproc_pair represents a pair of pipes. Because we control both +// sides of the pipes, we can allocate and free this in one structure. +struct inproc_pair { + nni_atomic_int ref; + inproc_queue queues[2]; +}; + +struct inproc_ep { + const char * addr; + bool listener; + nni_list_node node; + uint16_t proto; + nni_cv cv; + nni_list clients; + nni_list aios; + size_t rcvmax; + nni_mtx mtx; +}; + +// nni_inproc is our global state - this contains the list of active endpoints +// which we use for coordinating rendezvous. +static inproc_global nni_inproc; + +static int +inproc_init(void) +{ + NNI_LIST_INIT(&nni_inproc.servers, inproc_ep, node); + + nni_mtx_init(&nni_inproc.mx); + return (0); +} + +static void +inproc_fini(void) +{ + nni_mtx_fini(&nni_inproc.mx); +} + +// inproc_pair destroy is called when both pipe-ends of the pipe +// have been destroyed. +static void +inproc_pair_destroy(inproc_pair *pair) +{ + for (int i = 0; i < 2; i++) { + nni_mtx_fini(&pair->queues[i].lock); + } + NNI_FREE_STRUCT(pair); +} + +static int +inproc_pipe_alloc(inproc_pipe **pipep, inproc_ep *ep) +{ + inproc_pipe *pipe; + + if ((pipe = NNI_ALLOC_STRUCT(pipe)) == NULL) { + return (NNG_ENOMEM); + } + + pipe->proto = ep->proto; + pipe->addr = ep->addr; + *pipep = pipe; + return (0); +} + +static int +inproc_pipe_init(void *arg, nni_pipe *p) +{ + NNI_ARG_UNUSED(arg); + NNI_ARG_UNUSED(p); + return (0); +} + +static void +inproc_pipe_fini(void *arg) +{ + inproc_pipe *pipe = arg; + inproc_pair *pair; + + if ((pair = pipe->pair) != NULL) { + // If we are the last peer, then toss the pair structure. + if (nni_atomic_dec_nv(&pair->ref) == 0) { + inproc_pair_destroy(pair); + } + } + + NNI_FREE_STRUCT(pipe); +} + +static void +inproc_queue_run_closed(inproc_queue *queue) +{ + nni_aio *aio; + while (((aio = nni_list_first(&queue->readers)) != NULL) || + ((aio = nni_list_first(&queue->writers)) != NULL)) { + nni_aio_list_remove(aio); + nni_aio_finish_error(aio, NNG_ECLOSED); + } +} + +static void +inproc_queue_run(inproc_queue *queue) +{ + if (queue->closed) { + inproc_queue_run_closed(queue); + } + for (;;) { + nni_aio *rd; + nni_aio *wr; + nni_msg *msg; + nni_msg *pu; + + if (((rd = nni_list_first(&queue->readers)) == NULL) || + ((wr = nni_list_first(&queue->writers)) == NULL)) { + return; + } + + msg = nni_aio_get_msg(wr); + NNI_ASSERT(msg != NULL); + + // At this point, we pass success back to the caller. If + // we drop the message for any reason, its accounted on the + // receiver side. + nni_aio_list_remove(wr); + nni_aio_set_msg(wr, NULL); + nni_aio_finish( + wr, 0, nni_msg_len(msg) + nni_msg_header_len(msg)); + + // TODO: We could check the max receive size here. + + // Now the receive side. We need to ensure that we have + // an exclusive copy of the message, and pull the header + // up into the body to match protocol expectations. + if ((pu = nni_msg_pull_up(msg)) == NULL) { + nni_msg_free(msg); + continue; + } + msg = pu; + + nni_aio_list_remove(rd); + nni_aio_set_msg(rd, msg); + nni_aio_finish(rd, 0, nni_msg_len(msg)); + } +} + +static void +inproc_queue_cancel(nni_aio *aio, void *arg, int rv) +{ + inproc_queue *queue = arg; + + nni_mtx_lock(&queue->lock); + if (nni_aio_list_active(aio)) { + nni_aio_list_remove(aio); + nni_aio_finish_error(aio, rv); + } + nni_mtx_unlock(&queue->lock); +} + +static void +inproc_pipe_send(void *arg, nni_aio *aio) +{ + inproc_pipe * pipe = arg; + inproc_queue *queue = pipe->send_queue; + int rv; + + if (nni_aio_begin(aio) != 0) { + return; + } + + nni_mtx_lock(&queue->lock); + if ((rv = nni_aio_schedule(aio, inproc_queue_cancel, queue)) != 0) { + nni_mtx_unlock(&queue->lock); + nni_aio_finish_error(aio, rv); + return; + } + nni_aio_list_append(&queue->writers, aio); + inproc_queue_run(queue); + nni_mtx_unlock(&queue->lock); +} + +static void +inproc_pipe_recv(void *arg, nni_aio *aio) +{ + inproc_pipe * pipe = arg; + inproc_queue *queue = pipe->recv_queue; + int rv; + + if (nni_aio_begin(aio) != 0) { + return; + } + + nni_mtx_lock(&queue->lock); + if ((rv = nni_aio_schedule(aio, inproc_queue_cancel, queue)) != 0) { + nni_mtx_unlock(&queue->lock); + nni_aio_finish_error(aio, rv); + return; + } + nni_aio_list_append(&queue->readers, aio); + inproc_queue_run(queue); + nni_mtx_unlock(&queue->lock); +} + +static void +inproc_pipe_close(void *arg) +{ + inproc_pipe *pipe = arg; + inproc_pair *pair = pipe->pair; + + for (int i = 0; i < 2; i++) { + inproc_queue *queue = &pair->queues[i]; + nni_mtx_lock(&queue->lock); + queue->closed = true; + inproc_queue_run_closed(queue); + nni_mtx_unlock(&queue->lock); + } +} + +static uint16_t +inproc_pipe_peer(void *arg) +{ + inproc_pipe *pipe = arg; + + return (pipe->peer); +} + +static int +inproc_pipe_get_addr(void *arg, void *buf, size_t *szp, nni_opt_type t) +{ + inproc_pipe *p = arg; + nni_sockaddr sa; + + memset(&sa, 0, sizeof(sa)); + sa.s_inproc.sa_family = NNG_AF_INPROC; + nni_strlcpy(sa.s_inproc.sa_name, p->addr, sizeof(sa.s_inproc.sa_name)); + return (nni_copyout_sockaddr(&sa, buf, szp, t)); +} + +static int +inproc_dialer_init(void **epp, nni_url *url, nni_dialer *ndialer) +{ + inproc_ep *ep; + nni_sock * sock = nni_dialer_sock(ndialer); + + if ((ep = NNI_ALLOC_STRUCT(ep)) == NULL) { + return (NNG_ENOMEM); + } + nni_mtx_init(&ep->mtx); + + ep->listener = false; + ep->proto = nni_sock_proto_id(sock); + ep->rcvmax = 0; + NNI_LIST_INIT(&ep->clients, inproc_ep, node); + nni_aio_list_init(&ep->aios); + + ep->addr = url->u_rawurl; // we match on the full URL. + + *epp = ep; + return (0); +} + +static int +inproc_listener_init(void **epp, nni_url *url, nni_listener *nlistener) +{ + inproc_ep *ep; + nni_sock * sock = nni_listener_sock(nlistener); + + if ((ep = NNI_ALLOC_STRUCT(ep)) == NULL) { + return (NNG_ENOMEM); + } + nni_mtx_init(&ep->mtx); + + ep->listener = true; + ep->proto = nni_sock_proto_id(sock); + ep->rcvmax = 0; + NNI_LIST_INIT(&ep->clients, inproc_ep, node); + nni_aio_list_init(&ep->aios); + + ep->addr = url->u_rawurl; // we match on the full URL. + + *epp = ep; + return (0); +} + +static void +inproc_ep_fini(void *arg) +{ + inproc_ep *ep = arg; + nni_mtx_fini(&ep->mtx); + NNI_FREE_STRUCT(ep); +} + +static void +inproc_conn_finish(nni_aio *aio, int rv, inproc_ep *ep, inproc_pipe *pipe) +{ + nni_aio_list_remove(aio); + + if ((!ep->listener) && nni_list_empty(&ep->aios)) { + nni_list_node_remove(&ep->node); + } + + if (rv == 0) { + nni_aio_set_output(aio, 0, pipe); + nni_aio_finish(aio, 0, 0); + } else { + NNI_ASSERT(pipe == NULL); + nni_aio_finish_error(aio, rv); + } +} + +static void +inproc_ep_close(void *arg) +{ + inproc_ep *ep = arg; + inproc_ep *client; + nni_aio * aio; + + nni_mtx_lock(&nni_inproc.mx); + if (nni_list_active(&nni_inproc.servers, ep)) { + nni_list_remove(&nni_inproc.servers, ep); + } + // Notify any waiting clients that we are closed. + while ((client = nni_list_first(&ep->clients)) != NULL) { + while ((aio = nni_list_first(&client->aios)) != NULL) { + inproc_conn_finish(aio, NNG_ECONNREFUSED, ep, NULL); + } + nni_list_remove(&ep->clients, client); + } + while ((aio = nni_list_first(&ep->aios)) != NULL) { + inproc_conn_finish(aio, NNG_ECLOSED, ep, NULL); + } + nni_mtx_unlock(&nni_inproc.mx); +} + +static void +inproc_accept_clients(inproc_ep *srv) +{ + inproc_ep *cli, *nclient; + + nclient = nni_list_first(&srv->clients); + while ((cli = nclient) != NULL) { + nni_aio *caio; + nclient = nni_list_next(&srv->clients, nclient); + NNI_LIST_FOREACH (&cli->aios, caio) { + + inproc_pipe *cpipe; + inproc_pipe *spipe; + inproc_pair *pair; + nni_aio * saio; + int rv; + + if ((saio = nni_list_first(&srv->aios)) == NULL) { + // No outstanding accept() calls. + break; + } + + if ((pair = NNI_ALLOC_STRUCT(pair)) == NULL) { + inproc_conn_finish( + caio, NNG_ENOMEM, cli, NULL); + inproc_conn_finish( + saio, NNG_ENOMEM, srv, NULL); + continue; + } + for (int i = 0; i < 2; i++) { + nni_aio_list_init(&pair->queues[i].readers); + nni_aio_list_init(&pair->queues[i].writers); + nni_mtx_init(&pair->queues[i].lock); + } + nni_atomic_init(&pair->ref); + nni_atomic_set(&pair->ref, 2); + + spipe = cpipe = NULL; + if (((rv = inproc_pipe_alloc(&cpipe, cli)) != 0) || + ((rv = inproc_pipe_alloc(&spipe, srv)) != 0)) { + + if (cpipe != NULL) { + inproc_pipe_fini(cpipe); + } + if (spipe != NULL) { + inproc_pipe_fini(spipe); + } + inproc_conn_finish(caio, rv, cli, NULL); + inproc_conn_finish(saio, rv, srv, NULL); + inproc_pair_destroy(pair); + continue; + } + + cpipe->peer = spipe->proto; + spipe->peer = cpipe->proto; + cpipe->pair = pair; + spipe->pair = pair; + cpipe->send_queue = &pair->queues[0]; + cpipe->recv_queue = &pair->queues[1]; + spipe->send_queue = &pair->queues[1]; + spipe->recv_queue = &pair->queues[0]; + + inproc_conn_finish(caio, 0, cli, cpipe); + inproc_conn_finish(saio, 0, srv, spipe); + } + + if (nni_list_first(&cli->aios) == NULL) { + // No more outstanding client connects. + // Normally there should only be one. + if (nni_list_active(&srv->clients, cli)) { + nni_list_remove(&srv->clients, cli); + } + } + } +} + +static void +inproc_ep_cancel(nni_aio *aio, void *arg, int rv) +{ + inproc_ep *ep = arg; + + nni_mtx_lock(&nni_inproc.mx); + if (nni_aio_list_active(aio)) { + nni_aio_list_remove(aio); + nni_list_node_remove(&ep->node); + nni_aio_finish_error(aio, rv); + } + nni_mtx_unlock(&nni_inproc.mx); +} + +static void +inproc_ep_connect(void *arg, nni_aio *aio) +{ + inproc_ep *ep = arg; + inproc_ep *server; + int rv; + + if (nni_aio_begin(aio) != 0) { + return; + } + + nni_mtx_lock(&nni_inproc.mx); + + // Find a server. + NNI_LIST_FOREACH (&nni_inproc.servers, server) { + if (strcmp(server->addr, ep->addr) == 0) { + break; + } + } + if (server == NULL) { + nni_mtx_unlock(&nni_inproc.mx); + nni_aio_finish_error(aio, NNG_ECONNREFUSED); + return; + } + + // We don't have to worry about the case where a zero timeout + // on connect was specified, as there is no option to specify + // that in the upper API. + if ((rv = nni_aio_schedule(aio, inproc_ep_cancel, ep)) != 0) { + nni_mtx_unlock(&nni_inproc.mx); + nni_aio_finish_error(aio, rv); + return; + } + + nni_list_append(&server->clients, ep); + nni_aio_list_append(&ep->aios, aio); + + inproc_accept_clients(server); + nni_mtx_unlock(&nni_inproc.mx); +} + +static int +inproc_ep_bind(void *arg) +{ + inproc_ep *ep = arg; + inproc_ep *srch; + nni_list * list = &nni_inproc.servers; + + nni_mtx_lock(&nni_inproc.mx); + NNI_LIST_FOREACH (list, srch) { + if (strcmp(srch->addr, ep->addr) == 0) { + nni_mtx_unlock(&nni_inproc.mx); + return (NNG_EADDRINUSE); + } + } + nni_list_append(list, ep); + nni_mtx_unlock(&nni_inproc.mx); + return (0); +} + +static void +inproc_ep_accept(void *arg, nni_aio *aio) +{ + inproc_ep *ep = arg; + int rv; + + if (nni_aio_begin(aio) != 0) { + return; + } + + nni_mtx_lock(&nni_inproc.mx); + + // We need not worry about the case where a non-blocking + // accept was tried -- there is no API to do such a thing. + if ((rv = nni_aio_schedule(aio, inproc_ep_cancel, ep)) != 0) { + nni_mtx_unlock(&nni_inproc.mx); + nni_aio_finish_error(aio, rv); + return; + } + + // We are already on the master list of servers, thanks to bind. + // Insert us into pending server aios, and then run accept list. + nni_aio_list_append(&ep->aios, aio); + inproc_accept_clients(ep); + nni_mtx_unlock(&nni_inproc.mx); +} + +static int +inproc_ep_get_recvmaxsz(void *arg, void *v, size_t *szp, nni_opt_type t) +{ + inproc_ep *ep = arg; + int rv; + nni_mtx_lock(&ep->mtx); + rv = nni_copyout_size(ep->rcvmax, v, szp, t); + nni_mtx_unlock(&ep->mtx); + return (rv); +} + +static int +inproc_ep_set_recvmaxsz(void *arg, const void *v, size_t sz, nni_opt_type t) +{ + inproc_ep *ep = arg; + size_t val; + int rv; + if ((rv = nni_copyin_size(&val, v, sz, 0, NNI_MAXSZ, t)) == 0) { + nni_mtx_lock(&ep->mtx); + ep->rcvmax = val; + nni_mtx_unlock(&ep->mtx); + } + return (rv); +} + +static int +inproc_ep_get_addr(void *arg, void *v, size_t *szp, nni_opt_type t) +{ + inproc_ep * ep = arg; + nng_sockaddr sa; + sa.s_inproc.sa_family = NNG_AF_INPROC; + nni_strlcpy( + sa.s_inproc.sa_name, ep->addr, sizeof(sa.s_inproc.sa_name)); + return (nni_copyout_sockaddr(&sa, v, szp, t)); +} + +static const nni_option inproc_pipe_options[] = { + { + .o_name = NNG_OPT_LOCADDR, + .o_get = inproc_pipe_get_addr, + }, + { + .o_name = NNG_OPT_REMADDR, + .o_get = inproc_pipe_get_addr, + }, + // terminate list + { + .o_name = NULL, + }, +}; + +static int +inproc_pipe_getopt( + void *arg, const char *name, void *v, size_t *szp, nni_type t) +{ + return (nni_getopt(inproc_pipe_options, name, arg, v, szp, t)); +} + +static nni_tran_pipe_ops inproc_pipe_ops = { + .p_init = inproc_pipe_init, + .p_fini = inproc_pipe_fini, + .p_send = inproc_pipe_send, + .p_recv = inproc_pipe_recv, + .p_close = inproc_pipe_close, + .p_peer = inproc_pipe_peer, + .p_getopt = inproc_pipe_getopt, +}; + +static const nni_option inproc_ep_options[] = { + { + .o_name = NNG_OPT_RECVMAXSZ, + .o_get = inproc_ep_get_recvmaxsz, + .o_set = inproc_ep_set_recvmaxsz, + }, + { + .o_name = NNG_OPT_LOCADDR, + .o_get = inproc_ep_get_addr, + }, + { + .o_name = NNG_OPT_REMADDR, + .o_get = inproc_ep_get_addr, + }, + // terminate list + { + .o_name = NULL, + }, +}; + +static int +inproc_ep_getopt(void *arg, const char *name, void *v, size_t *szp, nni_type t) +{ + return (nni_getopt(inproc_ep_options, name, arg, v, szp, t)); +} + +static int +inproc_ep_setopt( + void *arg, const char *name, const void *v, size_t sz, nni_type t) +{ + return (nni_setopt(inproc_ep_options, name, arg, v, sz, t)); +} + +static nni_tran_dialer_ops inproc_dialer_ops = { + .d_init = inproc_dialer_init, + .d_fini = inproc_ep_fini, + .d_connect = inproc_ep_connect, + .d_close = inproc_ep_close, + .d_getopt = inproc_ep_getopt, + .d_setopt = inproc_ep_setopt, +}; + +static nni_tran_listener_ops inproc_listener_ops = { + .l_init = inproc_listener_init, + .l_fini = inproc_ep_fini, + .l_bind = inproc_ep_bind, + .l_accept = inproc_ep_accept, + .l_close = inproc_ep_close, + .l_getopt = inproc_ep_getopt, + .l_setopt = inproc_ep_setopt, +}; + +// This is the inproc transport linkage, and should be the only global +// symbol in this entire file. +struct nni_tran nni_inproc_tran = { + .tran_version = NNI_TRANSPORT_VERSION, + .tran_scheme = "inproc", + .tran_dialer = &inproc_dialer_ops, + .tran_listener = &inproc_listener_ops, + .tran_pipe = &inproc_pipe_ops, + .tran_init = inproc_init, + .tran_fini = inproc_fini, +}; + +int +nng_inproc_register(void) +{ + return (nni_tran_register(&nni_inproc_tran)); +} diff --git a/src/sp/transport/ipc/CMakeLists.txt b/src/sp/transport/ipc/CMakeLists.txt new file mode 100644 index 00000000..c9927f75 --- /dev/null +++ b/src/sp/transport/ipc/CMakeLists.txt @@ -0,0 +1,17 @@ +# +# Copyright 2020 Staysail Systems, Inc. +# Copyright 2018 Capitar IT Group BV +# +# This software is supplied under the terms of the MIT License, a +# copy of which should be located in the distribution where this +# file was obtained (LICENSE.txt). A copy of the license may also be +# found online at https://opensource.org/licenses/MIT. +# + +# ipc protocol +nng_directory(ipc) + +nng_sources_if(NNG_TRANSPORT_IPC ipc.c) +nng_headers_if(NNG_TRANSPORT_IPC nng/transport/ipc/ipc.h) +nng_defines_if(NNG_TRANSPORT_IPC NNG_TRANSPORT_IPC) +nng_test_if(NNG_TRANSPORT_IPC ipc_test) \ No newline at end of file diff --git a/src/sp/transport/ipc/ipc.c b/src/sp/transport/ipc/ipc.c new file mode 100644 index 00000000..efaa823c --- /dev/null +++ b/src/sp/transport/ipc/ipc.c @@ -0,0 +1,1171 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// Copyright 2018 Capitar IT Group BV +// Copyright 2019 Devolutions +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include +#include + +#include "core/nng_impl.h" + +#include + +// IPC transport. Platform specific IPC operations must be +// supplied as well. Normally the IPC is UNIX domain sockets or +// Windows named pipes. Other platforms could use other mechanisms, +// but all implementations on the platform must use the same mechanism. + +typedef struct ipc_pipe ipc_pipe; +typedef struct ipc_ep ipc_ep; + +// ipc_pipe is one end of an IPC connection. +struct ipc_pipe { + nng_stream * conn; + uint16_t peer; + uint16_t proto; + size_t rcv_max; + bool closed; + nni_sockaddr sa; + ipc_ep * ep; + nni_pipe * pipe; + nni_list_node node; + nni_atomic_flag reaped; + nni_reap_node reap; + uint8_t tx_head[1 + sizeof(uint64_t)]; + uint8_t rx_head[1 + sizeof(uint64_t)]; + size_t got_tx_head; + size_t got_rx_head; + size_t want_tx_head; + size_t want_rx_head; + nni_list recv_q; + nni_list send_q; + nni_aio tx_aio; + nni_aio rx_aio; + nni_aio neg_aio; + nni_msg * rx_msg; + nni_mtx mtx; +}; + +struct ipc_ep { + nni_mtx mtx; + nni_sockaddr sa; + size_t rcv_max; + uint16_t proto; + bool started; + bool closed; + bool fini; + int ref_cnt; + nng_stream_dialer * dialer; + nng_stream_listener *listener; + nni_aio * user_aio; + nni_aio * conn_aio; + nni_aio * time_aio; + nni_list busy_pipes; // busy pipes -- ones passed to socket + nni_list wait_pipes; // pipes waiting to match to socket + nni_list neg_pipes; // pipes busy negotiating + nni_reap_node reap; +#ifdef NNG_ENABLE_STATS + nni_stat_item st_rcv_max; +#endif +}; + +static void ipc_pipe_send_start(ipc_pipe *p); +static void ipc_pipe_recv_start(ipc_pipe *p); +static void ipc_pipe_send_cb(void *); +static void ipc_pipe_recv_cb(void *); +static void ipc_pipe_neg_cb(void *); +static void ipc_pipe_fini(void *); +static void ipc_ep_fini(void *); + +static nni_reap_list ipc_ep_reap_list = { + .rl_offset = offsetof(ipc_ep, reap), + .rl_func = ipc_ep_fini, +}; + +static nni_reap_list ipc_pipe_reap_list = { + .rl_offset = offsetof(ipc_pipe, reap), + .rl_func = ipc_pipe_fini, +}; + +static int +ipc_tran_init(void) +{ + return (0); +} + +static void +ipc_tran_fini(void) +{ +} + +static void +ipc_pipe_close(void *arg) +{ + ipc_pipe *p = arg; + + nni_mtx_lock(&p->mtx); + p->closed = true; + nni_mtx_unlock(&p->mtx); + + nni_aio_close(&p->rx_aio); + nni_aio_close(&p->tx_aio); + nni_aio_close(&p->neg_aio); + + nng_stream_close(p->conn); +} + +static void +ipc_pipe_stop(void *arg) +{ + ipc_pipe *p = arg; + + nni_aio_stop(&p->rx_aio); + nni_aio_stop(&p->tx_aio); + nni_aio_stop(&p->neg_aio); +} + +static int +ipc_pipe_init(void *arg, nni_pipe *pipe) +{ + ipc_pipe *p = arg; + p->pipe = pipe; + return (0); +} + +static void +ipc_pipe_fini(void *arg) +{ + ipc_pipe *p = arg; + ipc_ep * ep; + + ipc_pipe_stop(p); + if ((ep = p->ep) != NULL) { + nni_mtx_lock(&ep->mtx); + nni_list_node_remove(&p->node); + ep->ref_cnt--; + if (ep->fini && (ep->ref_cnt == 0)) { + nni_reap(&ipc_ep_reap_list, ep); + } + nni_mtx_unlock(&ep->mtx); + } + nni_aio_fini(&p->rx_aio); + nni_aio_fini(&p->tx_aio); + nni_aio_fini(&p->neg_aio); + nng_stream_free(p->conn); + if (p->rx_msg) { + nni_msg_free(p->rx_msg); + } + nni_mtx_fini(&p->mtx); + NNI_FREE_STRUCT(p); +} + +static void +ipc_pipe_reap(ipc_pipe *p) +{ + if (!nni_atomic_flag_test_and_set(&p->reaped)) { + if (p->conn != NULL) { + nng_stream_close(p->conn); + } + nni_reap(&ipc_pipe_reap_list, p); + } +} + +static int +ipc_pipe_alloc(ipc_pipe **pipe_p) +{ + ipc_pipe *p; + + if ((p = NNI_ALLOC_STRUCT(p)) == NULL) { + return (NNG_ENOMEM); + } + nni_mtx_init(&p->mtx); + nni_aio_init(&p->tx_aio, ipc_pipe_send_cb, p); + nni_aio_init(&p->rx_aio, ipc_pipe_recv_cb, p); + nni_aio_init(&p->neg_aio, ipc_pipe_neg_cb, p); + nni_aio_list_init(&p->send_q); + nni_aio_list_init(&p->recv_q); + nni_atomic_flag_reset(&p->reaped); + *pipe_p = p; + return (0); +} + +static void +ipc_ep_match(ipc_ep *ep) +{ + nni_aio * aio; + ipc_pipe *p; + + if (((aio = ep->user_aio) == NULL) || + ((p = nni_list_first(&ep->wait_pipes)) == NULL)) { + return; + } + nni_list_remove(&ep->wait_pipes, p); + nni_list_append(&ep->busy_pipes, p); + ep->user_aio = NULL; + p->rcv_max = ep->rcv_max; + nni_aio_set_output(aio, 0, p); + nni_aio_finish(aio, 0, 0); +} + +static void +ipc_pipe_neg_cb(void *arg) +{ + ipc_pipe *p = arg; + ipc_ep * ep = p->ep; + nni_aio * aio = &p->neg_aio; + nni_aio * user_aio; + int rv; + + nni_mtx_lock(&ep->mtx); + if ((rv = nni_aio_result(aio)) != 0) { + goto error; + } + + // We start transmitting before we receive. + if (p->got_tx_head < p->want_tx_head) { + p->got_tx_head += nni_aio_count(aio); + } else if (p->got_rx_head < p->want_rx_head) { + p->got_rx_head += nni_aio_count(aio); + } + if (p->got_tx_head < p->want_tx_head) { + nni_iov iov; + iov.iov_len = p->want_tx_head - p->got_tx_head; + iov.iov_buf = &p->tx_head[p->got_tx_head]; + nni_aio_set_iov(aio, 1, &iov); + // send it down... + nng_stream_send(p->conn, aio); + nni_mtx_unlock(&p->ep->mtx); + return; + } + if (p->got_rx_head < p->want_rx_head) { + nni_iov iov; + iov.iov_len = p->want_rx_head - p->got_rx_head; + iov.iov_buf = &p->rx_head[p->got_rx_head]; + nni_aio_set_iov(aio, 1, &iov); + nng_stream_recv(p->conn, aio); + nni_mtx_unlock(&p->ep->mtx); + return; + } + // We have both sent and received the headers. Lets check the + // receive side header. + if ((p->rx_head[0] != 0) || (p->rx_head[1] != 'S') || + (p->rx_head[2] != 'P') || (p->rx_head[3] != 0) || + (p->rx_head[6] != 0) || (p->rx_head[7] != 0)) { + rv = NNG_EPROTO; + goto error; + } + + NNI_GET16(&p->rx_head[4], p->peer); + + // We are all ready now. We put this in the wait list, and + // then try to run the matcher. + nni_list_remove(&ep->neg_pipes, p); + nni_list_append(&ep->wait_pipes, p); + + ipc_ep_match(ep); + nni_mtx_unlock(&ep->mtx); + return; + +error: + + nng_stream_close(p->conn); + // If we are waiting to negotiate on a client side, then a failure + // here has to be passed to the user app. + if ((user_aio = ep->user_aio) != NULL) { + ep->user_aio = NULL; + nni_aio_finish_error(user_aio, rv); + } + nni_mtx_unlock(&ep->mtx); + ipc_pipe_reap(p); +} + +static void +ipc_pipe_send_cb(void *arg) +{ + ipc_pipe *p = arg; + int rv; + nni_aio * aio; + size_t n; + nni_msg * msg; + nni_aio * tx_aio = &p->tx_aio; + + nni_mtx_lock(&p->mtx); + if ((rv = nni_aio_result(tx_aio)) != 0) { + nni_pipe_bump_error(p->pipe, rv); + // Intentionally we do not queue up another transfer. + // There's an excellent chance that the pipe is no longer + // usable, with a partial transfer. + // The protocol should see this error, and close the + // pipe itself, we hope. + + while ((aio = nni_list_first(&p->send_q)) != NULL) { + nni_aio_list_remove(aio); + nni_aio_finish_error(aio, rv); + } + nni_mtx_unlock(&p->mtx); + return; + } + + n = nni_aio_count(tx_aio); + nni_aio_iov_advance(tx_aio, n); + if (nni_aio_iov_count(tx_aio) != 0) { + nng_stream_send(p->conn, tx_aio); + nni_mtx_unlock(&p->mtx); + return; + } + + aio = nni_list_first(&p->send_q); + nni_aio_list_remove(aio); + ipc_pipe_send_start(p); + + msg = nni_aio_get_msg(aio); + n = nni_msg_len(msg); + nni_pipe_bump_tx(p->pipe, n); + nni_mtx_unlock(&p->mtx); + + nni_aio_set_msg(aio, NULL); + nni_msg_free(msg); + nni_aio_finish_sync(aio, 0, n); +} + +static void +ipc_pipe_recv_cb(void *arg) +{ + ipc_pipe *p = arg; + nni_aio * aio; + int rv; + size_t n; + nni_msg * msg; + nni_aio * rx_aio = &p->rx_aio; + + nni_mtx_lock(&p->mtx); + + if ((rv = nni_aio_result(rx_aio)) != 0) { + // Error on receive. This has to cause an error back + // to the user. Also, if we had allocated an rx_msg, lets + // toss it. + goto error; + } + + n = nni_aio_count(rx_aio); + nni_aio_iov_advance(rx_aio, n); + if (nni_aio_iov_count(rx_aio) != 0) { + // Was this a partial read? If so then resubmit for the rest. + nng_stream_recv(p->conn, rx_aio); + nni_mtx_unlock(&p->mtx); + return; + } + + // If we don't have a message yet, we were reading the message + // header, which is just the length. This tells us the size of the + // message to allocate and how much more to expect. + if (p->rx_msg == NULL) { + uint64_t len; + + // Check to make sure we got msg type 1. + if (p->rx_head[0] != 1) { + rv = NNG_EPROTO; + goto error; + } + + // We should have gotten a message header. + NNI_GET64(p->rx_head + 1, len); + + // Make sure the message payload is not too big. If it is + // the caller will shut down the pipe. + if ((len > p->rcv_max) && (p->rcv_max > 0)) { + rv = NNG_EMSGSIZE; + goto error; + } + + // Note that all IO on this pipe is blocked behind this + // allocation. We could possibly look at using a separate + // lock for the read side in the future, so that we allow + // transmits to proceed normally. In practice this is + // unlikely to be much of an issue though. + if ((rv = nni_msg_alloc(&p->rx_msg, (size_t) len)) != 0) { + goto error; + } + + if (len != 0) { + nni_iov iov; + // Submit the rest of the data for a read -- we want to + // read the entire message now. + iov.iov_buf = nni_msg_body(p->rx_msg); + iov.iov_len = (size_t) len; + + nni_aio_set_iov(rx_aio, 1, &iov); + nng_stream_recv(p->conn, rx_aio); + nni_mtx_unlock(&p->mtx); + return; + } + } + + // Otherwise we got a message read completely. Let the user know the + // good news. + + aio = nni_list_first(&p->recv_q); + nni_aio_list_remove(aio); + msg = p->rx_msg; + p->rx_msg = NULL; + n = nni_msg_len(msg); + nni_pipe_bump_rx(p->pipe, n); + ipc_pipe_recv_start(p); + nni_mtx_unlock(&p->mtx); + + nni_aio_set_msg(aio, msg); + nni_aio_finish_sync(aio, 0, n); + return; + +error: + while ((aio = nni_list_first(&p->recv_q)) != NULL) { + nni_aio_list_remove(aio); + nni_aio_finish_error(aio, rv); + } + msg = p->rx_msg; + p->rx_msg = NULL; + nni_pipe_bump_error(p->pipe, rv); + // Intentionally, we do not queue up another receive. + // The protocol should notice this error and close the pipe. + nni_mtx_unlock(&p->mtx); + + nni_msg_free(msg); +} + +static void +ipc_pipe_send_cancel(nni_aio *aio, void *arg, int rv) +{ + ipc_pipe *p = arg; + + nni_mtx_lock(&p->mtx); + if (!nni_aio_list_active(aio)) { + nni_mtx_unlock(&p->mtx); + return; + } + // If this is being sent, then cancel the pending transfer. + // The callback on the tx_aio will cause the user aio to + // be canceled too. + if (nni_list_first(&p->send_q) == aio) { + nni_aio_abort(&p->tx_aio, rv); + nni_mtx_unlock(&p->mtx); + return; + } + nni_aio_list_remove(aio); + nni_mtx_unlock(&p->mtx); + + nni_aio_finish_error(aio, rv); +} + +static void +ipc_pipe_send_start(ipc_pipe *p) +{ + nni_aio *aio; + nni_msg *msg; + int nio; + nni_iov iov[3]; + uint64_t len; + + if (p->closed) { + while ((aio = nni_list_first(&p->send_q)) != NULL) { + nni_list_remove(&p->send_q, aio); + nni_aio_finish_error(aio, NNG_ECLOSED); + } + return; + } + if ((aio = nni_list_first(&p->send_q)) == NULL) { + return; + } + + // This runs to send the message. + msg = nni_aio_get_msg(aio); + len = nni_msg_len(msg) + nni_msg_header_len(msg); + + p->tx_head[0] = 1; // message type, 1. + NNI_PUT64(p->tx_head + 1, len); + + nio = 0; + iov[0].iov_buf = p->tx_head; + iov[0].iov_len = sizeof(p->tx_head); + nio++; + if (nni_msg_header_len(msg) > 0) { + iov[nio].iov_buf = nni_msg_header(msg); + iov[nio].iov_len = nni_msg_header_len(msg); + nio++; + } + if (nni_msg_len(msg) > 0) { + iov[nio].iov_buf = nni_msg_body(msg); + iov[nio].iov_len = nni_msg_len(msg); + nio++; + } + nni_aio_set_iov(&p->tx_aio, nio, iov); + nng_stream_send(p->conn, &p->tx_aio); +} + +static void +ipc_pipe_send(void *arg, nni_aio *aio) +{ + ipc_pipe *p = arg; + int rv; + + if (nni_aio_begin(aio) != 0) { + return; + } + nni_mtx_lock(&p->mtx); + if ((rv = nni_aio_schedule(aio, ipc_pipe_send_cancel, p)) != 0) { + nni_mtx_unlock(&p->mtx); + nni_aio_finish_error(aio, rv); + return; + } + nni_list_append(&p->send_q, aio); + if (nni_list_first(&p->send_q) == aio) { + ipc_pipe_send_start(p); + } + nni_mtx_unlock(&p->mtx); +} + +static void +ipc_pipe_recv_cancel(nni_aio *aio, void *arg, int rv) +{ + ipc_pipe *p = arg; + + nni_mtx_lock(&p->mtx); + if (!nni_aio_list_active(aio)) { + nni_mtx_unlock(&p->mtx); + return; + } + // If receive in progress, then cancel the pending transfer. + // The callback on the rx_aio will cause the user aio to + // be canceled too. + if (nni_list_first(&p->recv_q) == aio) { + nni_aio_abort(&p->rx_aio, rv); + nni_mtx_unlock(&p->mtx); + return; + } + nni_aio_list_remove(aio); + nni_mtx_unlock(&p->mtx); + nni_aio_finish_error(aio, rv); +} + +static void +ipc_pipe_recv_start(ipc_pipe *p) +{ + nni_iov iov; + NNI_ASSERT(p->rx_msg == NULL); + + if (p->closed) { + nni_aio *aio; + while ((aio = nni_list_first(&p->recv_q)) != NULL) { + nni_list_remove(&p->recv_q, aio); + nni_aio_finish_error(aio, NNG_ECLOSED); + } + return; + } + if (nni_list_empty(&p->recv_q)) { + return; + } + + // Schedule a read of the IPC header. + iov.iov_buf = p->rx_head; + iov.iov_len = sizeof(p->rx_head); + nni_aio_set_iov(&p->rx_aio, 1, &iov); + + nng_stream_recv(p->conn, &p->rx_aio); +} + +static void +ipc_pipe_recv(void *arg, nni_aio *aio) +{ + ipc_pipe *p = arg; + int rv; + + if (nni_aio_begin(aio) != 0) { + return; + } + nni_mtx_lock(&p->mtx); + if (p->closed) { + nni_mtx_unlock(&p->mtx); + nni_aio_finish_error(aio, NNG_ECLOSED); + return; + } + if ((rv = nni_aio_schedule(aio, ipc_pipe_recv_cancel, p)) != 0) { + nni_mtx_unlock(&p->mtx); + nni_aio_finish_error(aio, rv); + return; + } + + nni_list_append(&p->recv_q, aio); + if (nni_list_first(&p->recv_q) == aio) { + ipc_pipe_recv_start(p); + } + nni_mtx_unlock(&p->mtx); +} + +static uint16_t +ipc_pipe_peer(void *arg) +{ + ipc_pipe *p = arg; + + return (p->peer); +} + +static void +ipc_pipe_start(ipc_pipe *p, nng_stream *conn, ipc_ep *ep) +{ + nni_iov iov; + + ep->ref_cnt++; + + p->conn = conn; + p->ep = ep; + p->proto = ep->proto; + + p->tx_head[0] = 0; + p->tx_head[1] = 'S'; + p->tx_head[2] = 'P'; + p->tx_head[3] = 0; + NNI_PUT16(&p->tx_head[4], p->proto); + NNI_PUT16(&p->tx_head[6], 0); + + p->got_rx_head = 0; + p->got_tx_head = 0; + p->want_rx_head = 8; + p->want_tx_head = 8; + iov.iov_len = 8; + iov.iov_buf = &p->tx_head[0]; + nni_aio_set_iov(&p->neg_aio, 1, &iov); + nni_list_append(&ep->neg_pipes, p); + + nni_aio_set_timeout(&p->neg_aio, 10000); // 10 sec timeout to negotiate + nng_stream_send(p->conn, &p->neg_aio); +} + +static void +ipc_ep_close(void *arg) +{ + ipc_ep * ep = arg; + ipc_pipe *p; + + nni_mtx_lock(&ep->mtx); + ep->closed = true; + nni_aio_close(ep->time_aio); + if (ep->dialer != NULL) { + nng_stream_dialer_close(ep->dialer); + } + if (ep->listener != NULL) { + nng_stream_listener_close(ep->listener); + } + NNI_LIST_FOREACH (&ep->neg_pipes, p) { + ipc_pipe_close(p); + } + NNI_LIST_FOREACH (&ep->wait_pipes, p) { + ipc_pipe_close(p); + } + NNI_LIST_FOREACH (&ep->busy_pipes, p) { + ipc_pipe_close(p); + } + if (ep->user_aio != NULL) { + nni_aio_finish_error(ep->user_aio, NNG_ECLOSED); + ep->user_aio = NULL; + } + nni_mtx_unlock(&ep->mtx); +} + +static void +ipc_ep_fini(void *arg) +{ + ipc_ep *ep = arg; + + nni_mtx_lock(&ep->mtx); + ep->fini = true; + if (ep->ref_cnt != 0) { + nni_mtx_unlock(&ep->mtx); + return; + } + nni_mtx_unlock(&ep->mtx); + nni_aio_stop(ep->time_aio); + nni_aio_stop(ep->conn_aio); + nng_stream_dialer_free(ep->dialer); + nng_stream_listener_free(ep->listener); + nni_aio_free(ep->time_aio); + nni_aio_free(ep->conn_aio); + nni_mtx_fini(&ep->mtx); + NNI_FREE_STRUCT(ep); +} + +static void +ipc_ep_timer_cb(void *arg) +{ + ipc_ep *ep = arg; + nni_mtx_lock(&ep->mtx); + if (nni_aio_result(ep->time_aio) == 0) { + nng_stream_listener_accept(ep->listener, ep->conn_aio); + } + nni_mtx_unlock(&ep->mtx); +} + +static void +ipc_ep_accept_cb(void *arg) +{ + ipc_ep * ep = arg; + nni_aio * aio = ep->conn_aio; + ipc_pipe * p; + int rv; + nng_stream *conn; + + nni_mtx_lock(&ep->mtx); + if ((rv = nni_aio_result(aio)) != 0) { + goto error; + } + + conn = nni_aio_get_output(aio, 0); + if ((rv = ipc_pipe_alloc(&p)) != 0) { + nng_stream_free(conn); + goto error; + } + if (ep->closed) { + ipc_pipe_fini(p); + nng_stream_free(conn); + rv = NNG_ECLOSED; + goto error; + } + ipc_pipe_start(p, conn, ep); + nng_stream_listener_accept(ep->listener, ep->conn_aio); + nni_mtx_unlock(&ep->mtx); + return; + +error: + // When an error here occurs, let's send a notice up to the consumer. + // That way it can be reported properly. + if ((aio = ep->user_aio) != NULL) { + ep->user_aio = NULL; + nni_aio_finish_error(aio, rv); + } + + switch (rv) { + + case NNG_ENOMEM: + case NNG_ENOFILES: + nng_sleep_aio(10, ep->time_aio); + break; + + default: + if (!ep->closed) { + nng_stream_listener_accept(ep->listener, ep->conn_aio); + } + break; + } + nni_mtx_unlock(&ep->mtx); +} + +static void +ipc_ep_dial_cb(void *arg) +{ + ipc_ep * ep = arg; + nni_aio * aio = ep->conn_aio; + ipc_pipe * p; + int rv; + nng_stream *conn; + + if ((rv = nni_aio_result(aio)) != 0) { + goto error; + } + + conn = nni_aio_get_output(aio, 0); + if ((rv = ipc_pipe_alloc(&p)) != 0) { + nng_stream_free(conn); + goto error; + } + nni_mtx_lock(&ep->mtx); + if (ep->closed) { + ipc_pipe_fini(p); + nng_stream_free(conn); + rv = NNG_ECLOSED; + nni_mtx_unlock(&ep->mtx); + goto error; + } else { + ipc_pipe_start(p, conn, ep); + } + nni_mtx_unlock(&ep->mtx); + return; + +error: + // Error connecting. We need to pass this straight back + // to the user. + nni_mtx_lock(&ep->mtx); + if ((aio = ep->user_aio) != NULL) { + ep->user_aio = NULL; + nni_aio_finish_error(aio, rv); + } + nni_mtx_unlock(&ep->mtx); +} + +static int +ipc_ep_init(ipc_ep **epp, nni_sock *sock) +{ + ipc_ep *ep; + + if ((ep = NNI_ALLOC_STRUCT(ep)) == NULL) { + return (NNG_ENOMEM); + } + nni_mtx_init(&ep->mtx); + NNI_LIST_INIT(&ep->busy_pipes, ipc_pipe, node); + NNI_LIST_INIT(&ep->wait_pipes, ipc_pipe, node); + NNI_LIST_INIT(&ep->neg_pipes, ipc_pipe, node); + + ep->proto = nni_sock_proto_id(sock); + +#ifdef NNG_ENABLE_STATS + static const nni_stat_info rcv_max_info = { + .si_name = "rcv_max", + .si_desc = "maximum receive size", + .si_type = NNG_STAT_LEVEL, + .si_unit = NNG_UNIT_BYTES, + .si_atomic = true, + }; + nni_stat_init(&ep->st_rcv_max, &rcv_max_info); +#endif + + *epp = ep; + return (0); +} + +static int +ipc_ep_init_dialer(void **dp, nni_url *url, nni_dialer *dialer) +{ + ipc_ep * ep; + int rv; + nni_sock *sock = nni_dialer_sock(dialer); + + if ((rv = ipc_ep_init(&ep, sock)) != 0) { + return (rv); + } + + if (((rv = nni_aio_alloc(&ep->conn_aio, ipc_ep_dial_cb, ep)) != 0) || + ((rv = nng_stream_dialer_alloc_url(&ep->dialer, url)) != 0)) { + ipc_ep_fini(ep); + return (rv); + } +#ifdef NNG_ENABLE_STATS + nni_dialer_add_stat(dialer, &ep->st_rcv_max); +#endif + *dp = ep; + return (0); +} + +static int +ipc_ep_init_listener(void **dp, nni_url *url, nni_listener *listener) +{ + ipc_ep * ep; + int rv; + nni_sock *sock = nni_listener_sock(listener); + + if ((rv = ipc_ep_init(&ep, sock)) != 0) { + return (rv); + } + + if (((rv = nni_aio_alloc(&ep->conn_aio, ipc_ep_accept_cb, ep)) != 0) || + ((rv = nni_aio_alloc(&ep->time_aio, ipc_ep_timer_cb, ep)) != 0) || + ((rv = nng_stream_listener_alloc_url(&ep->listener, url)) != 0)) { + ipc_ep_fini(ep); + return (rv); + } + +#ifdef NNG_ENABLE_STATS + nni_listener_add_stat(listener, &ep->st_rcv_max); +#endif + *dp = ep; + return (0); +} + +static void +ipc_ep_cancel(nni_aio *aio, void *arg, int rv) +{ + ipc_ep *ep = arg; + nni_mtx_lock(&ep->mtx); + if (aio == ep->user_aio) { + ep->user_aio = NULL; + nni_aio_finish_error(aio, rv); + } + nni_mtx_unlock(&ep->mtx); +} + +static void +ipc_ep_connect(void *arg, nni_aio *aio) +{ + ipc_ep *ep = arg; + int rv; + + if (nni_aio_begin(aio) != 0) { + return; + } + nni_mtx_lock(&ep->mtx); + if (ep->closed) { + nni_mtx_unlock(&ep->mtx); + nni_aio_finish_error(aio, NNG_ECLOSED); + return; + } + if (ep->user_aio != NULL) { + nni_mtx_unlock(&ep->mtx); + nni_aio_finish_error(aio, NNG_EBUSY); + return; + } + + if ((rv = nni_aio_schedule(aio, ipc_ep_cancel, ep)) != 0) { + nni_mtx_unlock(&ep->mtx); + nni_aio_finish_error(aio, rv); + return; + } + ep->user_aio = aio; + nng_stream_dialer_dial(ep->dialer, ep->conn_aio); + nni_mtx_unlock(&ep->mtx); +} + +static int +ipc_ep_get_recv_max_sz(void *arg, void *v, size_t *szp, nni_type t) +{ + ipc_ep *ep = arg; + int rv; + nni_mtx_lock(&ep->mtx); + rv = nni_copyout_size(ep->rcv_max, v, szp, t); + nni_mtx_unlock(&ep->mtx); + return (rv); +} + +static int +ipc_ep_set_recv_max_sz(void *arg, const void *v, size_t sz, nni_type t) +{ + ipc_ep *ep = arg; + size_t val; + int rv; + if ((rv = nni_copyin_size(&val, v, sz, 0, NNI_MAXSZ, t)) == 0) { + + ipc_pipe *p; + nni_mtx_lock(&ep->mtx); + ep->rcv_max = val; + NNI_LIST_FOREACH (&ep->wait_pipes, p) { + p->rcv_max = val; + } + NNI_LIST_FOREACH (&ep->neg_pipes, p) { + p->rcv_max = val; + } + NNI_LIST_FOREACH (&ep->busy_pipes, p) { + p->rcv_max = val; + } + nni_mtx_unlock(&ep->mtx); +#ifdef NNG_ENABLE_STATS + nni_stat_set_value(&ep->st_rcv_max, val); +#endif + } + return (rv); +} + +static int +ipc_ep_bind(void *arg) +{ + ipc_ep *ep = arg; + int rv; + + nni_mtx_lock(&ep->mtx); + rv = nng_stream_listener_listen(ep->listener); + nni_mtx_unlock(&ep->mtx); + return (rv); +} + +static void +ipc_ep_accept(void *arg, nni_aio *aio) +{ + ipc_ep *ep = arg; + int rv; + + if (nni_aio_begin(aio) != 0) { + return; + } + nni_mtx_lock(&ep->mtx); + if (ep->closed) { + nni_aio_finish_error(aio, NNG_ECLOSED); + nni_mtx_unlock(&ep->mtx); + return; + } + if (ep->user_aio != NULL) { + nni_aio_finish_error(aio, NNG_EBUSY); + nni_mtx_unlock(&ep->mtx); + return; + } + if ((rv = nni_aio_schedule(aio, ipc_ep_cancel, ep)) != 0) { + nni_mtx_unlock(&ep->mtx); + nni_aio_finish_error(aio, rv); + return; + } + ep->user_aio = aio; + if (!ep->started) { + ep->started = true; + nng_stream_listener_accept(ep->listener, ep->conn_aio); + } else { + ipc_ep_match(ep); + } + + nni_mtx_unlock(&ep->mtx); +} + +static int +ipc_pipe_get(void *arg, const char *name, void *buf, size_t *szp, nni_type t) +{ + ipc_pipe *p = arg; + + return (nni_stream_get(p->conn, name, buf, szp, t)); +} + +static nni_tran_pipe_ops ipc_tran_pipe_ops = { + .p_init = ipc_pipe_init, + .p_fini = ipc_pipe_fini, + .p_stop = ipc_pipe_stop, + .p_send = ipc_pipe_send, + .p_recv = ipc_pipe_recv, + .p_close = ipc_pipe_close, + .p_peer = ipc_pipe_peer, + .p_getopt = ipc_pipe_get, +}; + +static const nni_option ipc_ep_options[] = { + { + .o_name = NNG_OPT_RECVMAXSZ, + .o_get = ipc_ep_get_recv_max_sz, + .o_set = ipc_ep_set_recv_max_sz, + }, + // terminate list + { + .o_name = NULL, + }, +}; + +static int +ipc_dialer_get(void *arg, const char *name, void *buf, size_t *szp, nni_type t) +{ + ipc_ep *ep = arg; + int rv; + + rv = nni_getopt(ipc_ep_options, name, ep, buf, szp, t); + if (rv == NNG_ENOTSUP) { + rv = nni_stream_dialer_get(ep->dialer, name, buf, szp, t); + } + return (rv); +} + +static int +ipc_dialer_set( + void *arg, const char *name, const void *buf, size_t sz, nni_type t) +{ + ipc_ep *ep = arg; + int rv; + + rv = nni_setopt(ipc_ep_options, name, ep, buf, sz, t); + if (rv == NNG_ENOTSUP) { + rv = nni_stream_dialer_set(ep->dialer, name, buf, sz, t); + } + return (rv); +} + +static int +ipc_listener_get( + void *arg, const char *name, void *buf, size_t *szp, nni_type t) +{ + ipc_ep *ep = arg; + int rv; + + rv = nni_getopt(ipc_ep_options, name, ep, buf, szp, t); + if (rv == NNG_ENOTSUP) { + rv = nni_stream_listener_get(ep->listener, name, buf, szp, t); + } + return (rv); +} + +static int +ipc_listener_set( + void *arg, const char *name, const void *buf, size_t sz, nni_type t) +{ + ipc_ep *ep = arg; + int rv; + + rv = nni_setopt(ipc_ep_options, name, ep, buf, sz, t); + if (rv == NNG_ENOTSUP) { + rv = nni_stream_listener_set(ep->listener, name, buf, sz, t); + } + return (rv); +} + +static nni_tran_dialer_ops ipc_dialer_ops = { + .d_init = ipc_ep_init_dialer, + .d_fini = ipc_ep_fini, + .d_connect = ipc_ep_connect, + .d_close = ipc_ep_close, + .d_getopt = ipc_dialer_get, + .d_setopt = ipc_dialer_set, +}; + +static nni_tran_listener_ops ipc_listener_ops = { + .l_init = ipc_ep_init_listener, + .l_fini = ipc_ep_fini, + .l_bind = ipc_ep_bind, + .l_accept = ipc_ep_accept, + .l_close = ipc_ep_close, + .l_getopt = ipc_listener_get, + .l_setopt = ipc_listener_set, +}; + +static nni_tran ipc_tran = { + .tran_version = NNI_TRANSPORT_VERSION, + .tran_scheme = "ipc", + .tran_dialer = &ipc_dialer_ops, + .tran_listener = &ipc_listener_ops, + .tran_pipe = &ipc_tran_pipe_ops, + .tran_init = ipc_tran_init, + .tran_fini = ipc_tran_fini, +}; + +#ifdef NNG_PLATFORM_POSIX +static nni_tran ipc_tran_unix = { + .tran_version = NNI_TRANSPORT_VERSION, + .tran_scheme = "unix", + .tran_dialer = &ipc_dialer_ops, + .tran_listener = &ipc_listener_ops, + .tran_pipe = &ipc_tran_pipe_ops, + .tran_init = ipc_tran_init, + .tran_fini = ipc_tran_fini, +}; +#endif + +#ifdef NNG_HAVE_ABSTRACT_SOCKETS +static nni_tran ipc_tran_abstract = { + .tran_version = NNI_TRANSPORT_VERSION, + .tran_scheme = "abstract", + .tran_dialer = &ipc_dialer_ops, + .tran_listener = &ipc_listener_ops, + .tran_pipe = &ipc_tran_pipe_ops, + .tran_init = ipc_tran_init, + .tran_fini = ipc_tran_fini, +}; +#endif + +int +nng_ipc_register(void) +{ + int rv; + if (((rv = nni_tran_register(&ipc_tran)) != 0) +#ifdef NNG_PLATFORM_POSIX + || ((rv = nni_tran_register(&ipc_tran_unix)) != 0) +#endif +#ifdef NNG_HAVE_ABSTRACT_SOCKETS + || ((rv = nni_tran_register(&ipc_tran_abstract)) != 0) +#endif + ) { + return (rv); + } + + return (0); +} diff --git a/src/sp/transport/ipc/ipc_test.c b/src/sp/transport/ipc/ipc_test.c new file mode 100644 index 00000000..2fb4afa3 --- /dev/null +++ b/src/sp/transport/ipc/ipc_test.c @@ -0,0 +1,395 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// Copyright 2018 Cody Piersall +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include + +#ifdef NNG_PLATFORM_POSIX +#include +#include +#include +#endif + +void +test_path_too_long(void) +{ + nng_socket s1; + char addr[256]; + + // All our names have to be less than 128 bytes. + memset(addr, 'a', 255); + addr[255] = 0; + memcpy(addr, "ipc://", strlen("ipc://")); + + NUTS_ASSERT(strlen(addr) == 255); + NUTS_OPEN(s1); + NUTS_PASS(nng_socket_set_ms(s1, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(s1, NNG_OPT_RECVTIMEO, 1000)); + NUTS_FAIL(nng_listen(s1, addr, NULL, 0), NNG_EADDRINVAL); + NUTS_FAIL(nng_dial(s1, addr, NULL, NNG_FLAG_NONBLOCK), NNG_EADDRINVAL); + + NUTS_CLOSE(s1); +} + +void +test_ipc_dialer_perms(void) +{ + nng_socket s; + nng_dialer d; + char * addr; + + NUTS_ADDR(addr, "ipc"); + NUTS_OPEN(s); + NUTS_PASS(nng_dialer_create(&d, s, addr)); + NUTS_FAIL( + nng_dialer_set_int(d, NNG_OPT_IPC_PERMISSIONS, 0444), NNG_ENOTSUP); + NUTS_CLOSE(s); +} + +void +test_ipc_dialer_properties(void) +{ + nng_socket s; + nng_dialer d; + nng_sockaddr sa; + size_t z; + char *addr; + + NUTS_ADDR(addr, "ipc"); + NUTS_OPEN(s); + NUTS_PASS(nng_dial(s, addr, &d, NNG_FLAG_NONBLOCK)); + // Dialers don't have local addresses. + NUTS_FAIL(nng_dialer_get_addr(d, NNG_OPT_LOCADDR, &sa), NNG_ENOTSUP); + + NUTS_FAIL( + nng_dialer_set(d, NNG_OPT_LOCADDR, &sa, sizeof(sa)), NNG_ENOTSUP); + + z = 8192; + NUTS_PASS(nng_dialer_set_size(d, NNG_OPT_RECVMAXSZ, z)); + z = 0; + NUTS_PASS(nng_dialer_get_size(d, NNG_OPT_RECVMAXSZ, &z)); + NUTS_TRUE(z == 8192); + NUTS_FAIL(nng_dialer_set_bool(d, NNG_OPT_RAW, true), NNG_ENOTSUP); + NUTS_CLOSE(s); +} + +void +test_ipc_listener_perms(void) +{ + nng_socket s; + nng_listener l; + char *addr; + +#ifndef _WIN32 + char * path; + struct stat st; +#endif + + NUTS_ADDR(addr, "ipc"); + NUTS_OPEN(s); + NUTS_PASS(nng_listener_create(&l, s, addr)); + +#ifdef _WIN32 + NUTS_FAIL(nng_listener_set_int(l, NNG_OPT_IPC_PERMISSIONS, 0444), + NNG_ENOTSUP); +#else + path = &addr[strlen("ipc://")]; + + // Attempt to set invalid permissions fails. + NUTS_FAIL(nng_listener_set_int(l, NNG_OPT_IPC_PERMISSIONS, S_IFREG), + NNG_EINVAL); + + NUTS_PASS(nng_listener_set_int(l, NNG_OPT_IPC_PERMISSIONS, 0444)); + NUTS_PASS(nng_listener_start(l, 0)); + NUTS_TRUE(stat(path, &st) == 0); + NUTS_TRUE((st.st_mode & 0777) == 0444); + + // Now that it's running, we cannot set it. + NUTS_FAIL( + nng_listener_set_int(l, NNG_OPT_IPC_PERMISSIONS, 0644), NNG_EBUSY); +#endif + + NUTS_CLOSE(s); +} + +void +test_ipc_listener_properties(void) +{ + nng_socket s; + nng_listener l; + nng_sockaddr sa; + size_t z; + char *addr; + + NUTS_ADDR(addr, "ipc"); + NUTS_OPEN(s); + NUTS_PASS(nng_listen(s, addr, &l, 0)); + NUTS_PASS(nng_listener_get_addr(l, NNG_OPT_LOCADDR, &sa)); + NUTS_TRUE(sa.s_ipc.sa_family == NNG_AF_IPC); + NUTS_MATCH(sa.s_ipc.sa_path, addr + strlen("ipc://")); + + NUTS_FAIL(nng_listener_set(l, NNG_OPT_LOCADDR, &sa, sizeof(sa)), + NNG_EREADONLY); + z = 8192; + NUTS_PASS(nng_listener_set_size(l, NNG_OPT_RECVMAXSZ, z)); + z = 0; + NUTS_PASS(nng_listener_get_size(l, NNG_OPT_RECVMAXSZ, &z)); + NUTS_TRUE(z == 8192); + NUTS_FAIL(nng_listener_set_bool(l, NNG_OPT_RAW, true), NNG_ENOTSUP); + NUTS_CLOSE(s); +} + +void +test_ipc_recv_max(void) +{ + char msg[256]; + char rcvbuf[256]; + nng_socket s0; + nng_socket s1; + nng_listener l; + size_t sz; + char *addr; + + NUTS_ADDR(addr, "ipc"); + NUTS_OPEN(s0); + NUTS_PASS(nng_socket_set_ms(s0, NNG_OPT_RECVTIMEO, 100)); + NUTS_PASS(nng_socket_set_size(s0, NNG_OPT_RECVMAXSZ, 200)); + NUTS_PASS(nng_listener_create(&l, s0, addr)); + NUTS_PASS(nng_socket_get_size(s0, NNG_OPT_RECVMAXSZ, &sz)); + NUTS_TRUE(sz == 200); + NUTS_PASS(nng_listener_set_size(l, NNG_OPT_RECVMAXSZ, 100)); + NUTS_PASS(nng_listener_start(l, 0)); + + NUTS_OPEN(s1); + NUTS_PASS(nng_dial(s1, addr, NULL, 0)); + NUTS_PASS(nng_send(s1, msg, 95, 0)); + NUTS_PASS(nng_socket_set_ms(s1, NNG_OPT_SENDTIMEO, 100)); + NUTS_PASS(nng_recv(s0, rcvbuf, &sz, 0)); + NUTS_TRUE(sz == 95); + NUTS_PASS(nng_send(s1, msg, 150, 0)); + NUTS_FAIL(nng_recv(s0, rcvbuf, &sz, 0), NNG_ETIMEDOUT); + NUTS_CLOSE(s0); + NUTS_CLOSE(s1); +} + +void +test_abstract_sockets(void) +{ +#ifdef NNG_HAVE_ABSTRACT_SOCKETS + nng_socket s1; + nng_socket s2; + char *addr; + nng_pipe p1; + nng_pipe p2; + nng_sockaddr sa1; + nng_sockaddr sa2; + char * prefix = "abstract://"; + + NUTS_ADDR(addr, "abstract"); + NUTS_OPEN(s1); + NUTS_OPEN(s2); + NUTS_MARRY_EX(s1, s2, addr, &p1, &p2); + NUTS_PASS(nng_pipe_get_addr(p1, NNG_OPT_REMADDR, &sa1)); + NUTS_PASS(nng_pipe_get_addr(p2, NNG_OPT_LOCADDR, &sa2)); + NUTS_TRUE(sa1.s_family == sa2.s_family); + NUTS_TRUE(sa1.s_family == NNG_AF_ABSTRACT); + NUTS_TRUE(sa1.s_abstract.sa_len == strlen(addr) - strlen(prefix)); + NUTS_TRUE(sa2.s_abstract.sa_len == strlen(addr) - strlen(prefix)); + NUTS_SEND(s1, "ping"); + NUTS_RECV(s2, "ping"); + NUTS_CLOSE(s1); + NUTS_CLOSE(s2); +#endif +} + +void +test_abstract_auto_bind(void) +{ +#ifdef NNG_HAVE_ABSTRACT_SOCKETS + nng_socket s1; + nng_socket s2; + char addr[40]; + char name[12]; + nng_sockaddr sa; + nng_listener l; + size_t len; + + snprintf(addr, sizeof(addr), "abstract://"); + + NUTS_OPEN(s1); + NUTS_OPEN(s2); + NUTS_PASS(nng_socket_set_ms(s1, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(s2, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(s1, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(s2, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_listen(s1, addr, &l, 0)); + + NUTS_PASS(nng_listener_get_addr(l, NNG_OPT_LOCADDR, &sa)); + // Under linux there are either 8 or 5 hex characters. + NUTS_TRUE(sa.s_family == NNG_AF_ABSTRACT); + NUTS_TRUE(sa.s_abstract.sa_len < 10); + + len = sa.s_abstract.sa_len; + memcpy(name, sa.s_abstract.sa_name, len); + name[len] = '\0'; + NUTS_TRUE(strlen(name) == len); + + (void) snprintf(addr, sizeof(addr), "abstract://%s", name); + NUTS_PASS(nng_dial(s2, addr, NULL, 0)); + + // first send the ping + NUTS_SEND(s1, "ping"); + NUTS_RECV(s2, "ping"); + + NUTS_SEND(s2, "pong"); + NUTS_RECV(s1, "pong"); + + NUTS_CLOSE(s1); + NUTS_CLOSE(s2); +#endif +} + +void +test_abstract_too_long(void) +{ +#ifdef NNG_HAVE_ABSTRACT_SOCKETS + nng_socket s1; + char addr[256]; + + // All our names have to be less than 128 bytes. + memset(addr, 'a', 255); + addr[255] = 0; + memcpy(addr, "abstract://", strlen("abstract://")); + + NUTS_ASSERT(strlen(addr) == 255); + NUTS_OPEN(s1); + NUTS_PASS(nng_socket_set_ms(s1, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(s1, NNG_OPT_RECVTIMEO, 1000)); + NUTS_FAIL(nng_listen(s1, addr, NULL, 0), NNG_EADDRINVAL); + NUTS_FAIL(nng_dial(s1, addr, NULL, NNG_FLAG_NONBLOCK), NNG_EADDRINVAL); + + NUTS_CLOSE(s1); +#endif +} + +void +test_abstract_null(void) +{ +#ifdef NNG_HAVE_ABSTRACT_SOCKETS + nng_socket s1; + nng_socket s2; + char addr[64]; + char name[40]; + char rng[20]; + + nng_sockaddr sa; + nng_listener l; + size_t len; + + snprintf(rng, sizeof(rng), "%08x%08x", nng_random(), nng_random()); + snprintf(name, sizeof(name), "a%%00b_%s", rng); + snprintf(addr, sizeof(addr), "abstract://%s", name); + + NUTS_OPEN(s1); + NUTS_OPEN(s2); + NUTS_PASS(nng_socket_set_ms(s1, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(s2, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(s1, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(s2, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_listen(s1, addr, &l, 0)); + + NUTS_PASS(nng_listener_get_addr(l, NNG_OPT_LOCADDR, &sa)); + // Under linux there are either 8 or 5 hex characters. + NUTS_TRUE(sa.s_family == NNG_AF_ABSTRACT); + NUTS_TRUE(sa.s_abstract.sa_len < 32); + len = sa.s_abstract.sa_len; + NUTS_TRUE(len == 20); + NUTS_TRUE(sa.s_abstract.sa_name[0] == 'a'); + NUTS_TRUE(sa.s_abstract.sa_name[1] == '\0'); + NUTS_TRUE(sa.s_abstract.sa_name[2] == 'b'); + NUTS_TRUE(sa.s_abstract.sa_name[3] == '_'); + NUTS_TRUE(memcmp(&sa.s_abstract.sa_name[4], rng, 16) == 0); + + NUTS_PASS(nng_dial(s2, addr, NULL, 0)); + + // first send the ping + NUTS_SEND(s1, "1234"); + NUTS_RECV(s2, "1234"); + + NUTS_SEND(s2, "5678"); + NUTS_RECV(s1, "5678"); + + NUTS_CLOSE(s1); + NUTS_CLOSE(s2); +#endif +} + +void +test_unix_alias(void) +{ +#ifdef NNG_PLATFORM_POSIX + nng_socket s1; + nng_socket s2; + char addr1[32]; + char addr2[32]; + char rng[20]; + nng_sockaddr sa1; + nng_sockaddr sa2; + nng_msg * msg; + nng_pipe p; + + // Presumes /tmp. + + (void) snprintf( + rng, sizeof(rng), "%08x%08x", nng_random(), nng_random()); + snprintf(addr1, sizeof(addr1), "ipc:///tmp/%s", rng); + snprintf(addr2, sizeof(addr2), "unix:///tmp/%s", rng); + + NUTS_OPEN(s1); + NUTS_OPEN(s2); + NUTS_PASS(nng_socket_set_ms(s1, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(s2, NNG_OPT_SENDTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(s1, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_socket_set_ms(s2, NNG_OPT_RECVTIMEO, 1000)); + NUTS_PASS(nng_listen(s1, addr1, NULL, 0)); + NUTS_PASS(nng_dial(s2, addr2, NULL, 0)); + + // first send the ping + NUTS_SEND(s1, "ping"); + NUTS_PASS(nng_recvmsg(s2, &msg, 0)); + NUTS_ASSERT(msg != NULL); + NUTS_TRUE(nng_msg_len(msg) == 5); + NUTS_MATCH(nng_msg_body(msg), "ping"); + p = nng_msg_get_pipe(msg); + NUTS_PASS(nng_pipe_get_addr(p, NNG_OPT_REMADDR, &sa1)); + NUTS_PASS(nng_pipe_get_addr(p, NNG_OPT_REMADDR, &sa2)); + NUTS_TRUE(sa1.s_family == sa2.s_family); + NUTS_TRUE(sa1.s_family == NNG_AF_IPC); + NUTS_MATCH(sa1.s_ipc.sa_path, sa2.s_ipc.sa_path); + nng_msg_free(msg); + + NUTS_CLOSE(s1); + NUTS_CLOSE(s2); +#endif +} + +TEST_LIST = { + { "ipc path too long", test_path_too_long }, + { "ipc dialer perms", test_ipc_dialer_perms }, + { "ipc dialer props", test_ipc_dialer_properties }, + { "ipc listener perms", test_ipc_listener_perms }, + { "ipc listener props", test_ipc_listener_properties }, + { "ipc recv max", test_ipc_recv_max }, + { "ipc abstract sockets", test_abstract_sockets }, + { "ipc abstract auto bind", test_abstract_auto_bind }, + { "ipc abstract name too long", test_abstract_too_long }, + { "ipc abstract embedded null", test_abstract_null }, + { "ipc unix alias", test_unix_alias }, + { NULL, NULL }, +}; \ No newline at end of file diff --git a/src/sp/transport/tcp/CMakeLists.txt b/src/sp/transport/tcp/CMakeLists.txt new file mode 100644 index 00000000..d6022329 --- /dev/null +++ b/src/sp/transport/tcp/CMakeLists.txt @@ -0,0 +1,17 @@ +# +# Copyright 2020 Staysail Systems, Inc. +# Copyright 2018 Capitar IT Group BV +# +# This software is supplied under the terms of the MIT License, a +# copy of which should be located in the distribution where this +# file was obtained (LICENSE.txt). A copy of the license may also be +# found online at https://opensource.org/licenses/MIT. +# + +# TCP protocol +nng_directory(tcp) + +nng_sources_if(NNG_TRANSPORT_TCP tcp.c) +nng_headers_if(NNG_TRANSPORT_TCP nng/transport/tcp/tcp.h) +nng_defines_if(NNG_TRANSPORT_TCP NNG_TRANSPORT_TCP) +nng_test(tcp_test) \ No newline at end of file diff --git a/src/sp/transport/tcp/tcp.c b/src/sp/transport/tcp/tcp.c new file mode 100644 index 00000000..524c6988 --- /dev/null +++ b/src/sp/transport/tcp/tcp.c @@ -0,0 +1,1263 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// Copyright 2018 Capitar IT Group BV +// Copyright 2019 Devolutions +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include +#include +#include + +#include "core/nng_impl.h" + +// TCP transport. Platform specific TCP operations must be +// supplied as well. + +typedef struct tcptran_pipe tcptran_pipe; +typedef struct tcptran_ep tcptran_ep; + +// tcp_pipe is one end of a TCP connection. +struct tcptran_pipe { + nng_stream * conn; + nni_pipe * npipe; + uint16_t peer; + uint16_t proto; + size_t rcvmax; + bool closed; + nni_list_node node; + tcptran_ep * ep; + nni_atomic_flag reaped; + nni_reap_node reap; + uint8_t txlen[sizeof(uint64_t)]; + uint8_t rxlen[sizeof(uint64_t)]; + size_t gottxhead; + size_t gotrxhead; + size_t wanttxhead; + size_t wantrxhead; + nni_list recvq; + nni_list sendq; + nni_aio * txaio; + nni_aio * rxaio; + nni_aio * negoaio; + nni_msg * rxmsg; + nni_mtx mtx; +}; + +struct tcptran_ep { + nni_mtx mtx; + uint16_t proto; + size_t rcvmax; + bool fini; + bool started; + bool closed; + nng_url * url; + const char * host; // for dialers + nng_sockaddr src; + int refcnt; // active pipes + nni_aio * useraio; + nni_aio * connaio; + nni_aio * timeaio; + nni_list busypipes; // busy pipes -- ones passed to socket + nni_list waitpipes; // pipes waiting to match to socket + nni_list negopipes; // pipes busy negotiating + nni_reap_node reap; + nng_stream_dialer * dialer; + nng_stream_listener *listener; + +#ifdef NNG_ENABLE_STATS + nni_stat_item st_rcv_max; +#endif +}; + +static void tcptran_pipe_send_start(tcptran_pipe *); +static void tcptran_pipe_recv_start(tcptran_pipe *); +static void tcptran_pipe_send_cb(void *); +static void tcptran_pipe_recv_cb(void *); +static void tcptran_pipe_nego_cb(void *); +static void tcptran_ep_fini(void *); +static void tcptran_pipe_fini(void *); + +static nni_reap_list tcptran_ep_reap_list = { + .rl_offset = offsetof(tcptran_ep, reap), + .rl_func = tcptran_ep_fini, +}; + +static nni_reap_list tcptran_pipe_reap_list = { + .rl_offset = offsetof (tcptran_pipe, reap), + .rl_func = tcptran_pipe_fini, +}; + +static int +tcptran_init(void) +{ + return (0); +} + +static void +tcptran_fini(void) +{ +} + +static void +tcptran_pipe_close(void *arg) +{ + tcptran_pipe *p = arg; + + nni_mtx_lock(&p->mtx); + p->closed = true; + nni_mtx_unlock(&p->mtx); + + nni_aio_close(p->rxaio); + nni_aio_close(p->txaio); + nni_aio_close(p->negoaio); + + nng_stream_close(p->conn); +} + +static void +tcptran_pipe_stop(void *arg) +{ + tcptran_pipe *p = arg; + + nni_aio_stop(p->rxaio); + nni_aio_stop(p->txaio); + nni_aio_stop(p->negoaio); +} + +static int +tcptran_pipe_init(void *arg, nni_pipe *npipe) +{ + tcptran_pipe *p = arg; + p->npipe = npipe; + + return (0); +} + +static void +tcptran_pipe_fini(void *arg) +{ + tcptran_pipe *p = arg; + tcptran_ep * ep; + + tcptran_pipe_stop(p); + if ((ep = p->ep) != NULL) { + nni_mtx_lock(&ep->mtx); + nni_list_node_remove(&p->node); + ep->refcnt--; + if (ep->fini && (ep->refcnt == 0)) { + nni_reap(&tcptran_ep_reap_list, ep); + } + nni_mtx_unlock(&ep->mtx); + } + + nni_aio_free(p->rxaio); + nni_aio_free(p->txaio); + nni_aio_free(p->negoaio); + nng_stream_free(p->conn); + nni_msg_free(p->rxmsg); + nni_mtx_fini(&p->mtx); + NNI_FREE_STRUCT(p); +} + +static void +tcptran_pipe_reap(tcptran_pipe *p) +{ + if (!nni_atomic_flag_test_and_set(&p->reaped)) { + if (p->conn != NULL) { + nng_stream_close(p->conn); + } + nni_reap(&tcptran_pipe_reap_list, p); + } +} + +static int +tcptran_pipe_alloc(tcptran_pipe **pipep) +{ + tcptran_pipe *p; + int rv; + + if ((p = NNI_ALLOC_STRUCT(p)) == NULL) { + return (NNG_ENOMEM); + } + nni_mtx_init(&p->mtx); + if (((rv = nni_aio_alloc(&p->txaio, tcptran_pipe_send_cb, p)) != 0) || + ((rv = nni_aio_alloc(&p->rxaio, tcptran_pipe_recv_cb, p)) != 0) || + ((rv = nni_aio_alloc(&p->negoaio, tcptran_pipe_nego_cb, p)) != + 0)) { + tcptran_pipe_fini(p); + return (rv); + } + nni_aio_list_init(&p->recvq); + nni_aio_list_init(&p->sendq); + nni_atomic_flag_reset(&p->reaped); + + *pipep = p; + + return (0); +} + +static void +tcptran_ep_match(tcptran_ep *ep) +{ + nni_aio * aio; + tcptran_pipe *p; + + if (((aio = ep->useraio) == NULL) || + ((p = nni_list_first(&ep->waitpipes)) == NULL)) { + return; + } + nni_list_remove(&ep->waitpipes, p); + nni_list_append(&ep->busypipes, p); + ep->useraio = NULL; + p->rcvmax = ep->rcvmax; + nni_aio_set_output(aio, 0, p); + nni_aio_finish(aio, 0, 0); +} + +static void +tcptran_pipe_nego_cb(void *arg) +{ + tcptran_pipe *p = arg; + tcptran_ep * ep = p->ep; + nni_aio * aio = p->negoaio; + nni_aio * uaio; + int rv; + + nni_mtx_lock(&ep->mtx); + + if ((rv = nni_aio_result(aio)) != 0) { + goto error; + } + + // We start transmitting before we receive. + if (p->gottxhead < p->wanttxhead) { + p->gottxhead += nni_aio_count(aio); + } else if (p->gotrxhead < p->wantrxhead) { + p->gotrxhead += nni_aio_count(aio); + } + + if (p->gottxhead < p->wanttxhead) { + nni_iov iov; + iov.iov_len = p->wanttxhead - p->gottxhead; + iov.iov_buf = &p->txlen[p->gottxhead]; + // send it down... + nni_aio_set_iov(aio, 1, &iov); + nng_stream_send(p->conn, aio); + nni_mtx_unlock(&ep->mtx); + return; + } + if (p->gotrxhead < p->wantrxhead) { + nni_iov iov; + iov.iov_len = p->wantrxhead - p->gotrxhead; + iov.iov_buf = &p->rxlen[p->gotrxhead]; + nni_aio_set_iov(aio, 1, &iov); + nng_stream_recv(p->conn, aio); + nni_mtx_unlock(&ep->mtx); + return; + } + // We have both sent and received the headers. Lets check the + // receive side header. + if ((p->rxlen[0] != 0) || (p->rxlen[1] != 'S') || + (p->rxlen[2] != 'P') || (p->rxlen[3] != 0) || (p->rxlen[6] != 0) || + (p->rxlen[7] != 0)) { + rv = NNG_EPROTO; + goto error; + } + + NNI_GET16(&p->rxlen[4], p->peer); + + // We are all ready now. We put this in the wait list, and + // then try to run the matcher. + nni_list_remove(&ep->negopipes, p); + nni_list_append(&ep->waitpipes, p); + + tcptran_ep_match(ep); + nni_mtx_unlock(&ep->mtx); + + return; + +error: + nng_stream_close(p->conn); + + if ((uaio = ep->useraio) != NULL) { + ep->useraio = NULL; + nni_aio_finish_error(uaio, rv); + } + nni_mtx_unlock(&ep->mtx); + tcptran_pipe_reap(p); +} + +static void +tcptran_pipe_send_cb(void *arg) +{ + tcptran_pipe *p = arg; + int rv; + nni_aio * aio; + size_t n; + nni_msg * msg; + nni_aio * txaio = p->txaio; + + nni_mtx_lock(&p->mtx); + aio = nni_list_first(&p->sendq); + + if ((rv = nni_aio_result(txaio)) != 0) { + nni_pipe_bump_error(p->npipe, rv); + // Intentionally we do not queue up another transfer. + // There's an excellent chance that the pipe is no longer + // usable, with a partial transfer. + // The protocol should see this error, and close the + // pipe itself, we hope. + nni_aio_list_remove(aio); + nni_mtx_unlock(&p->mtx); + nni_aio_finish_error(aio, rv); + return; + } + + n = nni_aio_count(txaio); + nni_aio_iov_advance(txaio, n); + if (nni_aio_iov_count(txaio) > 0) { + nng_stream_send(p->conn, txaio); + nni_mtx_unlock(&p->mtx); + return; + } + + nni_aio_list_remove(aio); + tcptran_pipe_send_start(p); + + msg = nni_aio_get_msg(aio); + n = nni_msg_len(msg); + nni_pipe_bump_tx(p->npipe, n); + nni_mtx_unlock(&p->mtx); + + nni_aio_set_msg(aio, NULL); + nni_msg_free(msg); + nni_aio_finish_sync(aio, 0, n); +} + +static void +tcptran_pipe_recv_cb(void *arg) +{ + tcptran_pipe *p = arg; + nni_aio * aio; + int rv; + size_t n; + nni_msg * msg; + nni_aio * rxaio = p->rxaio; + + nni_mtx_lock(&p->mtx); + aio = nni_list_first(&p->recvq); + + if ((rv = nni_aio_result(rxaio)) != 0) { + goto recv_error; + } + + n = nni_aio_count(rxaio); + nni_aio_iov_advance(rxaio, n); + if (nni_aio_iov_count(rxaio) > 0) { + nng_stream_recv(p->conn, rxaio); + nni_mtx_unlock(&p->mtx); + return; + } + + // If we don't have a message yet, we were reading the TCP message + // header, which is just the length. This tells us the size of the + // message to allocate and how much more to expect. + if (p->rxmsg == NULL) { + uint64_t len; + // We should have gotten a message header. + NNI_GET64(p->rxlen, len); + + // Make sure the message payload is not too big. If it is + // the caller will shut down the pipe. + if ((len > p->rcvmax) && (p->rcvmax > 0)) { + rv = NNG_EMSGSIZE; + goto recv_error; + } + + if ((rv = nni_msg_alloc(&p->rxmsg, (size_t) len)) != 0) { + goto recv_error; + } + + // Submit the rest of the data for a read -- we want to + // read the entire message now. + if (len != 0) { + nni_iov iov; + iov.iov_buf = nni_msg_body(p->rxmsg); + iov.iov_len = (size_t) len; + + nni_aio_set_iov(rxaio, 1, &iov); + nng_stream_recv(p->conn, rxaio); + nni_mtx_unlock(&p->mtx); + return; + } + } + + // We read a message completely. Let the user know the good news. + nni_aio_list_remove(aio); + msg = p->rxmsg; + p->rxmsg = NULL; + n = nni_msg_len(msg); + + nni_pipe_bump_rx(p->npipe, n); + tcptran_pipe_recv_start(p); + nni_mtx_unlock(&p->mtx); + + nni_aio_set_msg(aio, msg); + nni_aio_finish_sync(aio, 0, n); + return; + +recv_error: + nni_aio_list_remove(aio); + msg = p->rxmsg; + p->rxmsg = NULL; + nni_pipe_bump_error(p->npipe, rv); + // Intentionally, we do not queue up another receive. + // The protocol should notice this error and close the pipe. + nni_mtx_unlock(&p->mtx); + + nni_msg_free(msg); + nni_aio_finish_error(aio, rv); +} + +static void +tcptran_pipe_send_cancel(nni_aio *aio, void *arg, int rv) +{ + tcptran_pipe *p = arg; + + nni_mtx_lock(&p->mtx); + if (!nni_aio_list_active(aio)) { + nni_mtx_unlock(&p->mtx); + return; + } + // If this is being sent, then cancel the pending transfer. + // The callback on the txaio will cause the user aio to + // be canceled too. + if (nni_list_first(&p->sendq) == aio) { + nni_aio_abort(p->txaio, rv); + nni_mtx_unlock(&p->mtx); + return; + } + nni_aio_list_remove(aio); + nni_mtx_unlock(&p->mtx); + + nni_aio_finish_error(aio, rv); +} + +static void +tcptran_pipe_send_start(tcptran_pipe *p) +{ + nni_aio *aio; + nni_aio *txaio; + nni_msg *msg; + int niov; + nni_iov iov[3]; + uint64_t len; + + if (p->closed) { + while ((aio = nni_list_first(&p->sendq)) != NULL) { + nni_list_remove(&p->sendq, aio); + nni_aio_finish_error(aio, NNG_ECLOSED); + } + return; + } + + if ((aio = nni_list_first(&p->sendq)) == NULL) { + return; + } + + // This runs to send the message. + msg = nni_aio_get_msg(aio); + len = nni_msg_len(msg) + nni_msg_header_len(msg); + + NNI_PUT64(p->txlen, len); + + txaio = p->txaio; + niov = 0; + iov[0].iov_buf = p->txlen; + iov[0].iov_len = sizeof(p->txlen); + niov++; + if (nni_msg_header_len(msg) > 0) { + iov[niov].iov_buf = nni_msg_header(msg); + iov[niov].iov_len = nni_msg_header_len(msg); + niov++; + } + if (nni_msg_len(msg) > 0) { + iov[niov].iov_buf = nni_msg_body(msg); + iov[niov].iov_len = nni_msg_len(msg); + niov++; + } + nni_aio_set_iov(txaio, niov, iov); + nng_stream_send(p->conn, txaio); +} + +static void +tcptran_pipe_send(void *arg, nni_aio *aio) +{ + tcptran_pipe *p = arg; + int rv; + + if (nni_aio_begin(aio) != 0) { + return; + } + nni_mtx_lock(&p->mtx); + if ((rv = nni_aio_schedule(aio, tcptran_pipe_send_cancel, p)) != 0) { + nni_mtx_unlock(&p->mtx); + nni_aio_finish_error(aio, rv); + return; + } + nni_list_append(&p->sendq, aio); + if (nni_list_first(&p->sendq) == aio) { + tcptran_pipe_send_start(p); + } + nni_mtx_unlock(&p->mtx); +} + +static void +tcptran_pipe_recv_cancel(nni_aio *aio, void *arg, int rv) +{ + tcptran_pipe *p = arg; + + nni_mtx_lock(&p->mtx); + if (!nni_aio_list_active(aio)) { + nni_mtx_unlock(&p->mtx); + return; + } + // If receive in progress, then cancel the pending transfer. + // The callback on the rxaio will cause the user aio to + // be canceled too. + if (nni_list_first(&p->recvq) == aio) { + nni_aio_abort(p->rxaio, rv); + nni_mtx_unlock(&p->mtx); + return; + } + nni_aio_list_remove(aio); + nni_mtx_unlock(&p->mtx); + nni_aio_finish_error(aio, rv); +} + +static void +tcptran_pipe_recv_start(tcptran_pipe *p) +{ + nni_aio *rxaio; + nni_iov iov; + NNI_ASSERT(p->rxmsg == NULL); + + if (p->closed) { + nni_aio *aio; + while ((aio = nni_list_first(&p->recvq)) != NULL) { + nni_list_remove(&p->recvq, aio); + nni_aio_finish_error(aio, NNG_ECLOSED); + } + return; + } + if (nni_list_empty(&p->recvq)) { + return; + } + + // Schedule a read of the header. + rxaio = p->rxaio; + iov.iov_buf = p->rxlen; + iov.iov_len = sizeof(p->rxlen); + nni_aio_set_iov(rxaio, 1, &iov); + + nng_stream_recv(p->conn, rxaio); +} + +static void +tcptran_pipe_recv(void *arg, nni_aio *aio) +{ + tcptran_pipe *p = arg; + int rv; + + if (nni_aio_begin(aio) != 0) { + return; + } + nni_mtx_lock(&p->mtx); + if ((rv = nni_aio_schedule(aio, tcptran_pipe_recv_cancel, p)) != 0) { + nni_mtx_unlock(&p->mtx); + nni_aio_finish_error(aio, rv); + return; + } + + nni_list_append(&p->recvq, aio); + if (nni_list_first(&p->recvq) == aio) { + tcptran_pipe_recv_start(p); + } + nni_mtx_unlock(&p->mtx); +} + +static uint16_t +tcptran_pipe_peer(void *arg) +{ + tcptran_pipe *p = arg; + + return (p->peer); +} + +static int +tcptran_pipe_getopt( + void *arg, const char *name, void *buf, size_t *szp, nni_type t) +{ + tcptran_pipe *p = arg; + return (nni_stream_get(p->conn, name, buf, szp, t)); +} + +static void +tcptran_pipe_start(tcptran_pipe *p, nng_stream *conn, tcptran_ep *ep) +{ + nni_iov iov; + + ep->refcnt++; + + p->conn = conn; + p->ep = ep; + p->proto = ep->proto; + + p->txlen[0] = 0; + p->txlen[1] = 'S'; + p->txlen[2] = 'P'; + p->txlen[3] = 0; + NNI_PUT16(&p->txlen[4], p->proto); + NNI_PUT16(&p->txlen[6], 0); + + p->gotrxhead = 0; + p->gottxhead = 0; + p->wantrxhead = 8; + p->wanttxhead = 8; + iov.iov_len = 8; + iov.iov_buf = &p->txlen[0]; + nni_aio_set_iov(p->negoaio, 1, &iov); + nni_list_append(&ep->negopipes, p); + + nni_aio_set_timeout(p->negoaio, 10000); // 10 sec timeout to negotiate + nng_stream_send(p->conn, p->negoaio); +} + +static void +tcptran_ep_fini(void *arg) +{ + tcptran_ep *ep = arg; + + nni_mtx_lock(&ep->mtx); + ep->fini = true; + if (ep->refcnt != 0) { + nni_mtx_unlock(&ep->mtx); + return; + } + nni_mtx_unlock(&ep->mtx); + nni_aio_stop(ep->timeaio); + nni_aio_stop(ep->connaio); + nng_stream_dialer_free(ep->dialer); + nng_stream_listener_free(ep->listener); + nni_aio_free(ep->timeaio); + nni_aio_free(ep->connaio); + + nni_mtx_fini(&ep->mtx); + NNI_FREE_STRUCT(ep); +} + +static void +tcptran_ep_close(void *arg) +{ + tcptran_ep * ep = arg; + tcptran_pipe *p; + + nni_mtx_lock(&ep->mtx); + + ep->closed = true; + nni_aio_close(ep->timeaio); + if (ep->dialer != NULL) { + nng_stream_dialer_close(ep->dialer); + } + if (ep->listener != NULL) { + nng_stream_listener_close(ep->listener); + } + NNI_LIST_FOREACH (&ep->negopipes, p) { + tcptran_pipe_close(p); + } + NNI_LIST_FOREACH (&ep->waitpipes, p) { + tcptran_pipe_close(p); + } + NNI_LIST_FOREACH (&ep->busypipes, p) { + tcptran_pipe_close(p); + } + if (ep->useraio != NULL) { + nni_aio_finish_error(ep->useraio, NNG_ECLOSED); + ep->useraio = NULL; + } + + nni_mtx_unlock(&ep->mtx); +} + +// This parses off the optional source address that this transport uses. +// The special handling of this URL format is quite honestly an historical +// mistake, which we would remove if we could. +static int +tcptran_url_parse_source(nng_url *url, nng_sockaddr *sa, const nng_url *surl) +{ + int af; + char * semi; + char * src; + size_t len; + int rv; + nni_aio *aio; + + // We modify the URL. This relies on the fact that the underlying + // transport does not free this, so we can just use references. + + url->u_scheme = surl->u_scheme; + url->u_port = surl->u_port; + url->u_hostname = surl->u_hostname; + + if ((semi = strchr(url->u_hostname, ';')) == NULL) { + memset(sa, 0, sizeof(*sa)); + return (0); + } + + len = (size_t)(semi - url->u_hostname); + url->u_hostname = semi + 1; + + if (strcmp(surl->u_scheme, "tcp") == 0) { + af = NNG_AF_UNSPEC; + } else if (strcmp(surl->u_scheme, "tcp4") == 0) { + af = NNG_AF_INET; + } else if (strcmp(surl->u_scheme, "tcp6") == 0) { + af = NNG_AF_INET6; + } else { + return (NNG_EADDRINVAL); + } + + if ((src = nni_alloc(len + 1)) == NULL) { + return (NNG_ENOMEM); + } + memcpy(src, surl->u_hostname, len); + src[len] = '\0'; + + if ((rv = nni_aio_alloc(&aio, NULL, NULL)) != 0) { + nni_free(src, len + 1); + return (rv); + } + + nni_resolv_ip(src, "0", af, true, sa, aio); + nni_aio_wait(aio); + nni_aio_free(aio); + nni_free(src, len + 1); + return (rv); +} + +static void +tcptran_timer_cb(void *arg) +{ + tcptran_ep *ep = arg; + if (nni_aio_result(ep->timeaio) == 0) { + nng_stream_listener_accept(ep->listener, ep->connaio); + } +} + +static void +tcptran_accept_cb(void *arg) +{ + tcptran_ep * ep = arg; + nni_aio * aio = ep->connaio; + tcptran_pipe *p; + int rv; + nng_stream * conn; + + nni_mtx_lock(&ep->mtx); + + if ((rv = nni_aio_result(aio)) != 0) { + goto error; + } + + conn = nni_aio_get_output(aio, 0); + if ((rv = tcptran_pipe_alloc(&p)) != 0) { + nng_stream_free(conn); + goto error; + } + + if (ep->closed) { + tcptran_pipe_fini(p); + nng_stream_free(conn); + rv = NNG_ECLOSED; + goto error; + } + tcptran_pipe_start(p, conn, ep); + nng_stream_listener_accept(ep->listener, ep->connaio); + nni_mtx_unlock(&ep->mtx); + return; + +error: + // When an error here occurs, let's send a notice up to the consumer. + // That way it can be reported properly. + if ((aio = ep->useraio) != NULL) { + ep->useraio = NULL; + nni_aio_finish_error(aio, rv); + } + switch (rv) { + + case NNG_ENOMEM: + case NNG_ENOFILES: + nng_sleep_aio(10, ep->timeaio); + break; + + default: + if (!ep->closed) { + nng_stream_listener_accept(ep->listener, ep->connaio); + } + break; + } + nni_mtx_unlock(&ep->mtx); +} + +static void +tcptran_dial_cb(void *arg) +{ + tcptran_ep * ep = arg; + nni_aio * aio = ep->connaio; + tcptran_pipe *p; + int rv; + nng_stream * conn; + + if ((rv = nni_aio_result(aio)) != 0) { + goto error; + } + + conn = nni_aio_get_output(aio, 0); + if ((rv = tcptran_pipe_alloc(&p)) != 0) { + nng_stream_free(conn); + goto error; + } + nni_mtx_lock(&ep->mtx); + if (ep->closed) { + tcptran_pipe_fini(p); + nng_stream_free(conn); + rv = NNG_ECLOSED; + nni_mtx_unlock(&ep->mtx); + goto error; + } else { + tcptran_pipe_start(p, conn, ep); + } + nni_mtx_unlock(&ep->mtx); + return; + +error: + // Error connecting. We need to pass this straight back + // to the user. + nni_mtx_lock(&ep->mtx); + if ((aio = ep->useraio) != NULL) { + ep->useraio = NULL; + nni_aio_finish_error(aio, rv); + } + nni_mtx_unlock(&ep->mtx); +} + +static int +tcptran_ep_init(tcptran_ep **epp, nng_url *url, nni_sock *sock) +{ + tcptran_ep *ep; + + if ((ep = NNI_ALLOC_STRUCT(ep)) == NULL) { + return (NNG_ENOMEM); + } + nni_mtx_init(&ep->mtx); + NNI_LIST_INIT(&ep->busypipes, tcptran_pipe, node); + NNI_LIST_INIT(&ep->waitpipes, tcptran_pipe, node); + NNI_LIST_INIT(&ep->negopipes, tcptran_pipe, node); + + ep->proto = nni_sock_proto_id(sock); + ep->url = url; + +#ifdef NNG_ENABLE_STATS + static const nni_stat_info rcv_max_info = { + .si_name = "rcv_max", + .si_desc = "maximum receive size", + .si_type = NNG_STAT_LEVEL, + .si_unit = NNG_UNIT_BYTES, + .si_atomic = true, + }; + nni_stat_init(&ep->st_rcv_max, &rcv_max_info); +#endif + + *epp = ep; + return (0); +} + +static int +tcptran_dialer_init(void **dp, nng_url *url, nni_dialer *ndialer) +{ + tcptran_ep * ep; + int rv; + nng_sockaddr srcsa; + nni_sock * sock = nni_dialer_sock(ndialer); + nng_url myurl; + + // Check for invalid URL components. + if ((strlen(url->u_path) != 0) && (strcmp(url->u_path, "/") != 0)) { + return (NNG_EADDRINVAL); + } + if ((url->u_fragment != NULL) || (url->u_userinfo != NULL) || + (url->u_query != NULL) || (strlen(url->u_hostname) == 0) || + (strlen(url->u_port) == 0)) { + return (NNG_EADDRINVAL); + } + + if ((rv = tcptran_url_parse_source(&myurl, &srcsa, url)) != 0) { + return (rv); + } + + if ((rv = tcptran_ep_init(&ep, url, sock)) != 0) { + return (rv); + } + + if ((rv != 0) || + ((rv = nni_aio_alloc(&ep->connaio, tcptran_dial_cb, ep)) != 0) || + ((rv = nng_stream_dialer_alloc_url(&ep->dialer, &myurl)) != 0)) { + tcptran_ep_fini(ep); + return (rv); + } + if ((srcsa.s_family != NNG_AF_UNSPEC) && + ((rv = nni_stream_dialer_set(ep->dialer, NNG_OPT_LOCADDR, &srcsa, + sizeof(srcsa), NNI_TYPE_SOCKADDR)) != 0)) { + tcptran_ep_fini(ep); + return (rv); + } + +#ifdef NNG_ENABLE_STATS + nni_dialer_add_stat(ndialer, &ep->st_rcv_max); +#endif + *dp = ep; + return (0); +} + +static int +tcptran_listener_init(void **lp, nng_url *url, nni_listener *nlistener) +{ + tcptran_ep *ep; + int rv; + nni_sock * sock = nni_listener_sock(nlistener); + + // Check for invalid URL components. + if ((strlen(url->u_path) != 0) && (strcmp(url->u_path, "/") != 0)) { + return (NNG_EADDRINVAL); + } + if ((url->u_fragment != NULL) || (url->u_userinfo != NULL) || + (url->u_query != NULL)) { + return (NNG_EADDRINVAL); + } + + if ((rv = tcptran_ep_init(&ep, url, sock)) != 0) { + return (rv); + } + + if (((rv = nni_aio_alloc(&ep->connaio, tcptran_accept_cb, ep)) != 0) || + ((rv = nni_aio_alloc(&ep->timeaio, tcptran_timer_cb, ep)) != 0) || + ((rv = nng_stream_listener_alloc_url(&ep->listener, url)) != 0)) { + tcptran_ep_fini(ep); + return (rv); + } +#ifdef NNG_ENABLE_STATS + nni_listener_add_stat(nlistener, &ep->st_rcv_max); +#endif + + *lp = ep; + return (0); +} + +static void +tcptran_ep_cancel(nni_aio *aio, void *arg, int rv) +{ + tcptran_ep *ep = arg; + nni_mtx_lock(&ep->mtx); + if (ep->useraio == aio) { + ep->useraio = NULL; + nni_aio_finish_error(aio, rv); + } + nni_mtx_unlock(&ep->mtx); +} + +static void +tcptran_ep_connect(void *arg, nni_aio *aio) +{ + tcptran_ep *ep = arg; + int rv; + + if (nni_aio_begin(aio) != 0) { + return; + } + nni_mtx_lock(&ep->mtx); + if (ep->closed) { + nni_mtx_unlock(&ep->mtx); + nni_aio_finish_error(aio, NNG_ECLOSED); + return; + } + if (ep->useraio != NULL) { + nni_mtx_unlock(&ep->mtx); + nni_aio_finish_error(aio, NNG_EBUSY); + return; + } + if ((rv = nni_aio_schedule(aio, tcptran_ep_cancel, ep)) != 0) { + nni_mtx_unlock(&ep->mtx); + nni_aio_finish_error(aio, rv); + return; + } + ep->useraio = aio; + + nng_stream_dialer_dial(ep->dialer, ep->connaio); + nni_mtx_unlock(&ep->mtx); +} + +static int +tcptran_ep_get_url(void *arg, void *v, size_t *szp, nni_opt_type t) +{ + tcptran_ep *ep = arg; + char * s; + int rv; + int port = 0; + + if (ep->listener != NULL) { + (void) nng_stream_listener_get_int( + ep->listener, NNG_OPT_TCP_BOUND_PORT, &port); + } + + if ((rv = nni_url_asprintf_port(&s, ep->url, port)) == 0) { + rv = nni_copyout_str(s, v, szp, t); + nni_strfree(s); + } + return (rv); +} + +static int +tcptran_ep_get_recvmaxsz(void *arg, void *v, size_t *szp, nni_opt_type t) +{ + tcptran_ep *ep = arg; + int rv; + + nni_mtx_lock(&ep->mtx); + rv = nni_copyout_size(ep->rcvmax, v, szp, t); + nni_mtx_unlock(&ep->mtx); + return (rv); +} + +static int +tcptran_ep_set_recvmaxsz(void *arg, const void *v, size_t sz, nni_opt_type t) +{ + tcptran_ep *ep = arg; + size_t val; + int rv; + if ((rv = nni_copyin_size(&val, v, sz, 0, NNI_MAXSZ, t)) == 0) { + tcptran_pipe *p; + nni_mtx_lock(&ep->mtx); + ep->rcvmax = val; + NNI_LIST_FOREACH (&ep->waitpipes, p) { + p->rcvmax = val; + } + NNI_LIST_FOREACH (&ep->negopipes, p) { + p->rcvmax = val; + } + NNI_LIST_FOREACH (&ep->busypipes, p) { + p->rcvmax = val; + } + nni_mtx_unlock(&ep->mtx); +#ifdef NNG_ENABLE_STATS + nni_stat_set_value(&ep->st_rcv_max, val); +#endif + } + return (rv); +} + +static int +tcptran_ep_bind(void *arg) +{ + tcptran_ep *ep = arg; + int rv; + + nni_mtx_lock(&ep->mtx); + rv = nng_stream_listener_listen(ep->listener); + nni_mtx_unlock(&ep->mtx); + + return (rv); +} + +static void +tcptran_ep_accept(void *arg, nni_aio *aio) +{ + tcptran_ep *ep = arg; + int rv; + + if (nni_aio_begin(aio) != 0) { + return; + } + nni_mtx_lock(&ep->mtx); + if (ep->closed) { + nni_mtx_unlock(&ep->mtx); + nni_aio_finish_error(aio, NNG_ECLOSED); + return; + } + if (ep->useraio != NULL) { + nni_mtx_unlock(&ep->mtx); + nni_aio_finish_error(aio, NNG_EBUSY); + return; + } + if ((rv = nni_aio_schedule(aio, tcptran_ep_cancel, ep)) != 0) { + nni_mtx_unlock(&ep->mtx); + nni_aio_finish_error(aio, rv); + return; + } + ep->useraio = aio; + if (!ep->started) { + ep->started = true; + nng_stream_listener_accept(ep->listener, ep->connaio); + } else { + tcptran_ep_match(ep); + } + nni_mtx_unlock(&ep->mtx); +} + +static nni_tran_pipe_ops tcptran_pipe_ops = { + .p_init = tcptran_pipe_init, + .p_fini = tcptran_pipe_fini, + .p_stop = tcptran_pipe_stop, + .p_send = tcptran_pipe_send, + .p_recv = tcptran_pipe_recv, + .p_close = tcptran_pipe_close, + .p_peer = tcptran_pipe_peer, + .p_getopt = tcptran_pipe_getopt, +}; + +static const nni_option tcptran_ep_opts[] = { + { + .o_name = NNG_OPT_RECVMAXSZ, + .o_get = tcptran_ep_get_recvmaxsz, + .o_set = tcptran_ep_set_recvmaxsz, + }, + { + .o_name = NNG_OPT_URL, + .o_get = tcptran_ep_get_url, + }, + // terminate list + { + .o_name = NULL, + }, +}; + +static int +tcptran_dialer_getopt( + void *arg, const char *name, void *buf, size_t *szp, nni_type t) +{ + tcptran_ep *ep = arg; + int rv; + + rv = nni_stream_dialer_get(ep->dialer, name, buf, szp, t); + if (rv == NNG_ENOTSUP) { + rv = nni_getopt(tcptran_ep_opts, name, ep, buf, szp, t); + } + return (rv); +} + +static int +tcptran_dialer_setopt( + void *arg, const char *name, const void *buf, size_t sz, nni_type t) +{ + tcptran_ep *ep = arg; + int rv; + + rv = nni_stream_dialer_set(ep->dialer, name, buf, sz, t); + if (rv == NNG_ENOTSUP) { + rv = nni_setopt(tcptran_ep_opts, name, ep, buf, sz, t); + } + return (rv); +} + +static int +tcptran_listener_getopt( + void *arg, const char *name, void *buf, size_t *szp, nni_type t) +{ + tcptran_ep *ep = arg; + int rv; + + rv = nni_stream_listener_get(ep->listener, name, buf, szp, t); + if (rv == NNG_ENOTSUP) { + rv = nni_getopt(tcptran_ep_opts, name, ep, buf, szp, t); + } + return (rv); +} + +static int +tcptran_listener_setopt( + void *arg, const char *name, const void *buf, size_t sz, nni_type t) +{ + tcptran_ep *ep = arg; + int rv; + + rv = nni_stream_listener_set(ep->listener, name, buf, sz, t); + if (rv == NNG_ENOTSUP) { + rv = nni_setopt(tcptran_ep_opts, name, ep, buf, sz, t); + } + return (rv); +} + +static nni_tran_dialer_ops tcptran_dialer_ops = { + .d_init = tcptran_dialer_init, + .d_fini = tcptran_ep_fini, + .d_connect = tcptran_ep_connect, + .d_close = tcptran_ep_close, + .d_getopt = tcptran_dialer_getopt, + .d_setopt = tcptran_dialer_setopt, +}; + +static nni_tran_listener_ops tcptran_listener_ops = { + .l_init = tcptran_listener_init, + .l_fini = tcptran_ep_fini, + .l_bind = tcptran_ep_bind, + .l_accept = tcptran_ep_accept, + .l_close = tcptran_ep_close, + .l_getopt = tcptran_listener_getopt, + .l_setopt = tcptran_listener_setopt, +}; + +static nni_tran tcp_tran = { + .tran_version = NNI_TRANSPORT_VERSION, + .tran_scheme = "tcp", + .tran_dialer = &tcptran_dialer_ops, + .tran_listener = &tcptran_listener_ops, + .tran_pipe = &tcptran_pipe_ops, + .tran_init = tcptran_init, + .tran_fini = tcptran_fini, +}; + +static nni_tran tcp4_tran = { + .tran_version = NNI_TRANSPORT_VERSION, + .tran_scheme = "tcp4", + .tran_dialer = &tcptran_dialer_ops, + .tran_listener = &tcptran_listener_ops, + .tran_pipe = &tcptran_pipe_ops, + .tran_init = tcptran_init, + .tran_fini = tcptran_fini, +}; + +static nni_tran tcp6_tran = { + .tran_version = NNI_TRANSPORT_VERSION, + .tran_scheme = "tcp6", + .tran_dialer = &tcptran_dialer_ops, + .tran_listener = &tcptran_listener_ops, + .tran_pipe = &tcptran_pipe_ops, + .tran_init = tcptran_init, + .tran_fini = tcptran_fini, +}; + +int +nng_tcp_register(void) +{ + int rv; + if (((rv = nni_tran_register(&tcp_tran)) != 0) || + ((rv = nni_tran_register(&tcp4_tran)) != 0) || + ((rv = nni_tran_register(&tcp6_tran)) != 0)) { + return (rv); + } + return (0); +} diff --git a/src/sp/transport/tcp/tcp_test.c b/src/sp/transport/tcp/tcp_test.c new file mode 100644 index 00000000..d23227d7 --- /dev/null +++ b/src/sp/transport/tcp/tcp_test.c @@ -0,0 +1,297 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// Copyright 2018 Capitar IT Group BV +// Copyright 2018 Devolutions +// Copyright 2018 Cody Piersall +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + + +#include + +// TCP tests. + +static void +test_tcp_wild_card_connect_fail(void) +{ + nng_socket s; + char addr[NNG_MAXADDRLEN]; + + NUTS_OPEN(s); + (void) snprintf(addr, sizeof(addr), "tcp://*:%u", nuts_next_port()); + NUTS_FAIL(nng_dial(s, addr, NULL, 0), NNG_EADDRINVAL); + NUTS_CLOSE(s); +} + +void +test_tcp_wild_card_bind(void) +{ + nng_socket s1; + nng_socket s2; + char addr[NNG_MAXADDRLEN]; + uint16_t port; + + port = nuts_next_port(); + + NUTS_OPEN(s1); + NUTS_OPEN(s2); + (void) snprintf(addr, sizeof(addr), "tcp4://*:%u", port); + NUTS_PASS(nng_listen(s1, addr, NULL, 0)); + (void) snprintf(addr, sizeof(addr), "tcp://127.0.0.1:%u", port); + NUTS_PASS(nng_dial(s2, addr, NULL, 0)); + NUTS_CLOSE(s2); + NUTS_CLOSE(s1); +} + +void +test_tcp_local_address_connect(void) +{ + + nng_socket s1; + nng_socket s2; + char addr[NNG_MAXADDRLEN]; + uint16_t port; + + NUTS_OPEN(s1); + NUTS_OPEN(s2); + port = nuts_next_port(); + (void) snprintf(addr, sizeof(addr), "tcp://127.0.0.1:%u", port); + NUTS_PASS(nng_listen(s1, addr, NULL, 0)); + (void) snprintf( + addr, sizeof(addr), "tcp://127.0.0.1;127.0.0.1:%u", port); + NUTS_PASS(nng_dial(s2, addr, NULL, 0)); + NUTS_CLOSE(s2); + NUTS_CLOSE(s1); +} + +void +test_tcp_port_zero_bind(void) +{ + nng_socket s1; + nng_socket s2; + nng_sockaddr sa; + nng_listener l; + char * addr; + + NUTS_OPEN(s1); + NUTS_OPEN(s2); + NUTS_PASS(nng_listen(s1, "tcp://127.0.0.1:0", &l, 0)); + NUTS_PASS(nng_listener_get_string(l, NNG_OPT_URL, &addr)); + NUTS_TRUE(memcmp(addr, "tcp://", 6) == 0); + NUTS_PASS(nng_listener_get_addr(l, NNG_OPT_LOCADDR, &sa)); + NUTS_TRUE(sa.s_in.sa_family == NNG_AF_INET); + NUTS_TRUE(sa.s_in.sa_port != 0); + NUTS_TRUE(sa.s_in.sa_addr = nuts_be32(0x7f000001)); + NUTS_PASS(nng_dial(s2, addr, NULL, 0)); + nng_strfree(addr); + NUTS_CLOSE(s2); + NUTS_CLOSE(s1); +} + +void +test_tcp_bad_local_interface(void) +{ + nng_socket s1; + + NUTS_OPEN(s1); + NUTS_FAIL(nng_dial(s1, "tcp://bogus1;127.0.0.1:80", NULL, 0), + NNG_EADDRINVAL); + NUTS_CLOSE(s1); +} + +void +test_tcp_non_local_address(void) +{ + nng_socket s1; + + NUTS_OPEN(s1); + NUTS_FAIL(nng_dial(s1, "tcp://8.8.8.8;127.0.0.1:80", NULL, 0), + NNG_EADDRINVAL); + NUTS_CLOSE(s1); +} + +void +test_tcp_malformed_address(void) +{ + nng_socket s1; + + NUTS_OPEN(s1); + NUTS_FAIL( + nng_dial(s1, "tcp://127.0.0.1", NULL, 0), NNG_EADDRINVAL); + NUTS_FAIL( + nng_dial(s1, "tcp://127.0.0.1.32", NULL, 0), NNG_EADDRINVAL); + NUTS_FAIL( + nng_dial(s1, "tcp://127.0.x.1.32", NULL, 0), NNG_EADDRINVAL); + NUTS_FAIL( + nng_listen(s1, "tcp://127.0.0.1.32", NULL, 0), NNG_EADDRINVAL); + NUTS_FAIL( + nng_listen(s1, "tcp://127.0.x.1.32", NULL, 0), NNG_EADDRINVAL); + NUTS_CLOSE(s1); +} + +void +test_tcp_no_delay_option(void) +{ + nng_socket s; + nng_dialer d; + nng_listener l; + bool v; + int x; + char *addr; + + NUTS_ADDR(addr, "tcp"); + + NUTS_OPEN(s); +#ifndef NNG_ELIDE_DEPRECATED + NUTS_PASS(nng_socket_get_bool(s, NNG_OPT_TCP_NODELAY, &v)); + NUTS_TRUE(v); +#endif + NUTS_PASS(nng_dialer_create(&d, s, addr)); + NUTS_PASS(nng_dialer_get_bool(d, NNG_OPT_TCP_NODELAY, &v)); + NUTS_TRUE(v); + NUTS_PASS(nng_dialer_set_bool(d, NNG_OPT_TCP_NODELAY, false)); + NUTS_PASS(nng_dialer_get_bool(d, NNG_OPT_TCP_NODELAY, &v)); + NUTS_TRUE(v == false); + NUTS_FAIL( + nng_dialer_get_int(d, NNG_OPT_TCP_NODELAY, &x), NNG_EBADTYPE); + x = 0; + NUTS_FAIL( + nng_dialer_set_int(d, NNG_OPT_TCP_NODELAY, x), NNG_EBADTYPE); + // This assumes sizeof (bool) != sizeof (int) + if (sizeof(bool) != sizeof(int)) { + NUTS_FAIL( + nng_dialer_set(d, NNG_OPT_TCP_NODELAY, &x, sizeof(x)), + NNG_EINVAL); + } + + NUTS_PASS(nng_listener_create(&l, s, addr)); + NUTS_PASS(nng_listener_get_bool(l, NNG_OPT_TCP_NODELAY, &v)); + NUTS_TRUE(v == true); + x = 0; + NUTS_FAIL( + nng_listener_set_int(l, NNG_OPT_TCP_NODELAY, x), NNG_EBADTYPE); + // This assumes sizeof (bool) != sizeof (int) + NUTS_FAIL(nng_listener_set(l, NNG_OPT_TCP_NODELAY, &x, sizeof(x)), + NNG_EINVAL); + + NUTS_PASS(nng_dialer_close(d)); + NUTS_PASS(nng_listener_close(l)); + + // Make sure socket wide defaults apply. +#ifndef NNG_ELIDE_DEPRECATED + NUTS_PASS(nng_socket_set_bool(s, NNG_OPT_TCP_NODELAY, true)); + v = false; + NUTS_PASS(nng_socket_get_bool(s, NNG_OPT_TCP_NODELAY, &v)); + NUTS_TRUE(v); + NUTS_PASS(nng_socket_set_bool(s, NNG_OPT_TCP_NODELAY, false)); + NUTS_PASS(nng_dialer_create(&d, s, addr)); + NUTS_PASS(nng_dialer_get_bool(d, NNG_OPT_TCP_NODELAY, &v)); + NUTS_TRUE(v == false); +#endif + NUTS_CLOSE(s); +} + +void +test_tcp_keep_alive_option(void) +{ + nng_socket s; + nng_dialer d; + nng_listener l; + bool v; + int x; + char *addr; + + NUTS_ADDR(addr, "tcp"); + NUTS_OPEN(s); +#ifndef NNG_ELIDE_DEPRECATED + NUTS_PASS(nng_socket_get_bool(s, NNG_OPT_TCP_KEEPALIVE, &v)); + NUTS_TRUE(v == false); +#endif + NUTS_PASS(nng_dialer_create(&d, s, addr)); + NUTS_PASS(nng_dialer_get_bool(d, NNG_OPT_TCP_KEEPALIVE, &v)); + NUTS_TRUE(v == false); + NUTS_PASS(nng_dialer_set_bool(d, NNG_OPT_TCP_KEEPALIVE, true)); + NUTS_PASS(nng_dialer_get_bool(d, NNG_OPT_TCP_KEEPALIVE, &v)); + NUTS_TRUE(v); + NUTS_FAIL( + nng_dialer_get_int(d, NNG_OPT_TCP_KEEPALIVE, &x), NNG_EBADTYPE); + x = 1; + NUTS_FAIL( + nng_dialer_set_int(d, NNG_OPT_TCP_KEEPALIVE, x), NNG_EBADTYPE); + + NUTS_PASS(nng_listener_create(&l, s, addr)); + NUTS_PASS(nng_listener_get_bool(l, NNG_OPT_TCP_KEEPALIVE, &v)); + NUTS_TRUE(v == false); + x = 1; + NUTS_FAIL( + nng_listener_set_int(l, NNG_OPT_TCP_KEEPALIVE, x), NNG_EBADTYPE); + + NUTS_PASS(nng_dialer_close(d)); + NUTS_PASS(nng_listener_close(l)); + + // Make sure socket wide defaults apply. +#ifndef NNG_ELIDE_DEPRECATED + NUTS_PASS(nng_socket_set_bool(s, NNG_OPT_TCP_KEEPALIVE, false)); + v = true; + NUTS_PASS(nng_socket_get_bool(s, NNG_OPT_TCP_KEEPALIVE, &v)); + NUTS_TRUE(v == false); + NUTS_PASS(nng_socket_set_bool(s, NNG_OPT_TCP_KEEPALIVE, true)); + NUTS_PASS(nng_dialer_create(&d, s, addr)); + NUTS_PASS(nng_dialer_get_bool(d, NNG_OPT_TCP_KEEPALIVE, &v)); + NUTS_TRUE(v); +#endif + NUTS_CLOSE(s); +} + +void +test_tcp_recv_max(void) +{ + char msg[256]; + char buf[256]; + nng_socket s0; + nng_socket s1; + nng_listener l; + size_t sz; + char *addr; + + NUTS_ADDR(addr, "tcp"); + + NUTS_OPEN(s0); + NUTS_PASS(nng_socket_set_ms(s0, NNG_OPT_RECVTIMEO, 100)); + NUTS_PASS(nng_socket_set_size(s0, NNG_OPT_RECVMAXSZ, 200)); + NUTS_PASS(nng_listener_create(&l, s0, addr)); + NUTS_PASS(nng_socket_get_size(s0, NNG_OPT_RECVMAXSZ, &sz)); + NUTS_TRUE(sz == 200); + NUTS_PASS(nng_listener_set_size(l, NNG_OPT_RECVMAXSZ, 100)); + NUTS_PASS(nng_listener_start(l, 0)); + + NUTS_OPEN(s1); + NUTS_PASS(nng_dial(s1, addr, NULL, 0)); + NUTS_PASS(nng_send(s1, msg, 95, 0)); + NUTS_PASS(nng_socket_set_ms(s1, NNG_OPT_SENDTIMEO, 100)); + NUTS_PASS(nng_recv(s0, buf, &sz, 0)); + NUTS_TRUE(sz == 95); + NUTS_PASS(nng_send(s1, msg, 150, 0)); + NUTS_FAIL(nng_recv(s0, buf, &sz, 0), NNG_ETIMEDOUT); + NUTS_PASS(nng_close(s0)); + NUTS_CLOSE(s1); +} + +NUTS_TESTS = { + + { "tcp wild card connect fail", test_tcp_wild_card_connect_fail }, + { "tcp wild card bind", test_tcp_wild_card_bind }, + { "tcp port zero bind", test_tcp_port_zero_bind }, + { "tcp local address connect", test_tcp_local_address_connect }, + { "tcp bad local interface", test_tcp_bad_local_interface }, + { "tcp non-local address", test_tcp_non_local_address }, + { "tcp malformed address", test_tcp_malformed_address }, + { "tcp no delay option", test_tcp_no_delay_option }, + { "tcp keep alive option", test_tcp_keep_alive_option }, + { "tcp recv max", test_tcp_recv_max }, + { NULL, NULL }, +}; \ No newline at end of file diff --git a/src/sp/transport/tls/CMakeLists.txt b/src/sp/transport/tls/CMakeLists.txt new file mode 100644 index 00000000..82f24c79 --- /dev/null +++ b/src/sp/transport/tls/CMakeLists.txt @@ -0,0 +1,16 @@ +# +# Copyright 2020 Staysail Systems, Inc. +# Copyright 2018 Capitar IT Group BV +# +# This software is supplied under the terms of the MIT License, a +# copy of which should be located in the distribution where this +# file was obtained (LICENSE.txt). A copy of the license may also be +# found online at https://opensource.org/licenses/MIT. +# + +# TLS transport +nng_directory(tls) + +nng_sources_if(NNG_TRANSPORT_TLS tls.c) +nng_headers_if(NNG_TRANSPORT_TLS nng/transport/tls/tls.h) +nng_defines_if(NNG_TRANSPORT_TLS NNG_TRANSPORT_TLS) \ No newline at end of file diff --git a/src/sp/transport/tls/tls.c b/src/sp/transport/tls/tls.c new file mode 100644 index 00000000..b6623733 --- /dev/null +++ b/src/sp/transport/tls/tls.c @@ -0,0 +1,1292 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// Copyright 2018 Capitar IT Group BV +// Copyright 2019 Devolutions +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include +#include + +#include "core/nng_impl.h" + +#include "nng/supplemental/tls/tls.h" +#include "nng/transport/tls/tls.h" + +// TLS over TCP transport. Platform specific TCP operations must be +// supplied as well, and uses the supplemental TLS v1.2 code. It is not +// an accident that this very closely resembles the TCP transport itself. + +typedef struct tlstran_ep tlstran_ep; +typedef struct tlstran_dialer tlstran_dialer; +typedef struct tlstran_listener tlstran_listener; +typedef struct tlstran_pipe tlstran_pipe; + +// tlstran_pipe is one end of a TLS connection. +struct tlstran_pipe { + nng_stream * tls; + nni_pipe * npipe; + uint16_t peer; + uint16_t proto; + size_t rcvmax; + bool closed; + nni_list_node node; + nni_list sendq; + nni_list recvq; + tlstran_ep * ep; + nni_sockaddr sa; + nni_atomic_flag reaped; + nni_reap_node reap; + uint8_t txlen[sizeof(uint64_t)]; + uint8_t rxlen[sizeof(uint64_t)]; + size_t gottxhead; + size_t gotrxhead; + size_t wanttxhead; + size_t wantrxhead; + nni_aio * txaio; + nni_aio * rxaio; + nni_aio * negoaio; + nni_msg * rxmsg; + nni_mtx mtx; +}; + +// Stuff that is common to both dialers and listeners. +struct tlstran_ep { + nni_mtx mtx; + uint16_t proto; + size_t rcvmax; + bool started; + bool closed; + bool fini; + int refcnt; + int authmode; + nni_url * url; + nni_list pipes; + nni_reap_node reap; + nng_stream_dialer * dialer; + nng_stream_listener *listener; + nni_aio * useraio; + nni_aio * connaio; + nni_aio * timeaio; + nni_list busypipes; // busy pipes -- ones passed to socket + nni_list waitpipes; // pipes waiting to match to socket + nni_list negopipes; // pipes busy negotiating + const char * host; + nng_sockaddr src; + nng_sockaddr sa; + nni_stat_item st_rcv_max; +}; + +static void tlstran_pipe_send_start(tlstran_pipe *); +static void tlstran_pipe_recv_start(tlstran_pipe *); +static void tlstran_pipe_send_cb(void *); +static void tlstran_pipe_recv_cb(void *); +static void tlstran_pipe_nego_cb(void *); +static void tlstran_ep_fini(void *); +static void tlstran_pipe_fini(void *); + +static nni_reap_list tlstran_ep_reap_list = { + .rl_offset = offsetof(tlstran_ep, reap), + .rl_func = tlstran_ep_fini, +}; + +static nni_reap_list tlstran_pipe_reap_list = { + .rl_offset = offsetof(tlstran_pipe, reap), + .rl_func = tlstran_pipe_fini, +}; + +static int +tlstran_init(void) +{ + return (0); +} + +static void +tlstran_fini(void) +{ +} + +static void +tlstran_pipe_close(void *arg) +{ + tlstran_pipe *p = arg; + + nni_aio_close(p->rxaio); + nni_aio_close(p->txaio); + nni_aio_close(p->negoaio); + + nng_stream_close(p->tls); +} + +static void +tlstran_pipe_stop(void *arg) +{ + tlstran_pipe *p = arg; + + nni_aio_stop(p->rxaio); + nni_aio_stop(p->txaio); + nni_aio_stop(p->negoaio); +} + +static int +tlstran_pipe_init(void *arg, nni_pipe *npipe) +{ + tlstran_pipe *p = arg; + p->npipe = npipe; + return (0); +} + +static void +tlstran_pipe_fini(void *arg) +{ + tlstran_pipe *p = arg; + tlstran_ep * ep; + + tlstran_pipe_stop(p); + if ((ep = p->ep) != NULL) { + nni_mtx_lock(&ep->mtx); + nni_list_node_remove(&p->node); + ep->refcnt--; + if (ep->fini && (ep->refcnt == 0)) { + nni_reap(&tlstran_ep_reap_list, ep); + } + nni_mtx_unlock(&ep->mtx); + } + nni_aio_free(p->rxaio); + nni_aio_free(p->txaio); + nni_aio_free(p->negoaio); + nng_stream_free(p->tls); + nni_msg_free(p->rxmsg); + NNI_FREE_STRUCT(p); +} + +static int +tlstran_pipe_alloc(tlstran_pipe **pipep) +{ + tlstran_pipe *p; + int rv; + + if ((p = NNI_ALLOC_STRUCT(p)) == NULL) { + return (NNG_ENOMEM); + } + nni_mtx_init(&p->mtx); + + if (((rv = nni_aio_alloc(&p->txaio, tlstran_pipe_send_cb, p)) != 0) || + ((rv = nni_aio_alloc(&p->rxaio, tlstran_pipe_recv_cb, p)) != 0) || + ((rv = nni_aio_alloc(&p->negoaio, tlstran_pipe_nego_cb, p)) != + 0)) { + tlstran_pipe_fini(p); + return (rv); + } + nni_aio_list_init(&p->recvq); + nni_aio_list_init(&p->sendq); + nni_atomic_flag_reset(&p->reaped); + + *pipep = p; + return (0); +} + +static void +tlstran_pipe_reap(tlstran_pipe *p) +{ + if (!nni_atomic_flag_test_and_set(&p->reaped)) { + if (p->tls != NULL) { + nng_stream_close(p->tls); + } + nni_reap(&tlstran_pipe_reap_list, p); + } +} + +static void +tlstran_ep_match(tlstran_ep *ep) +{ + nni_aio * aio; + tlstran_pipe *p; + + if (((aio = ep->useraio) == NULL) || + ((p = nni_list_first(&ep->waitpipes)) == NULL)) { + return; + } + nni_list_remove(&ep->waitpipes, p); + nni_list_append(&ep->busypipes, p); + ep->useraio = NULL; + p->rcvmax = ep->rcvmax; + nni_aio_set_output(aio, 0, p); + nni_aio_finish(aio, 0, 0); +} + +static void +tlstran_pipe_nego_cb(void *arg) +{ + tlstran_pipe *p = arg; + tlstran_ep * ep = p->ep; + nni_aio * aio = p->negoaio; + nni_aio * uaio; + int rv; + + nni_mtx_lock(&ep->mtx); + if ((rv = nni_aio_result(aio)) != 0) { + goto error; + } + + // We start transmitting before we receive. + if (p->gottxhead < p->wanttxhead) { + p->gottxhead += nni_aio_count(aio); + } else if (p->gotrxhead < p->wantrxhead) { + p->gotrxhead += nni_aio_count(aio); + } + + if (p->gottxhead < p->wanttxhead) { + nni_iov iov; + iov.iov_len = p->wanttxhead - p->gottxhead; + iov.iov_buf = &p->txlen[p->gottxhead]; + nni_aio_set_iov(aio, 1, &iov); + // send it down... + nng_stream_send(p->tls, aio); + nni_mtx_unlock(&ep->mtx); + return; + } + if (p->gotrxhead < p->wantrxhead) { + nni_iov iov; + iov.iov_len = p->wantrxhead - p->gotrxhead; + iov.iov_buf = &p->rxlen[p->gotrxhead]; + nni_aio_set_iov(aio, 1, &iov); + nng_stream_recv(p->tls, aio); + nni_mtx_unlock(&ep->mtx); + return; + } + // We have both sent and received the headers. Lets check the + // receive side header. + if ((p->rxlen[0] != 0) || (p->rxlen[1] != 'S') || + (p->rxlen[2] != 'P') || (p->rxlen[3] != 0) || (p->rxlen[6] != 0) || + (p->rxlen[7] != 0)) { + rv = NNG_EPROTO; + goto error; + } + + NNI_GET16(&p->rxlen[4], p->peer); + + // We are all ready now. We put this in the wait list, and + // then try to run the matcher. + nni_list_remove(&ep->negopipes, p); + nni_list_append(&ep->waitpipes, p); + + tlstran_ep_match(ep); + nni_mtx_unlock(&ep->mtx); + + return; + +error: + nng_stream_close(p->tls); + + if ((uaio = ep->useraio) != NULL) { + ep->useraio = NULL; + nni_aio_finish_error(uaio, rv); + } + nni_mtx_unlock(&ep->mtx); + tlstran_pipe_reap(p); +} + +static void +tlstran_pipe_send_cb(void *arg) +{ + tlstran_pipe *p = arg; + int rv; + nni_aio * aio; + size_t n; + nni_msg * msg; + nni_aio * txaio = p->txaio; + + nni_mtx_lock(&p->mtx); + aio = nni_list_first(&p->sendq); + + if ((rv = nni_aio_result(txaio)) != 0) { + // Intentionally we do not queue up another transfer. + // There's an excellent chance that the pipe is no longer + // usable, with a partial transfer. + // The protocol should see this error, and close the + // pipe itself, we hope. + nni_aio_list_remove(aio); + nni_mtx_unlock(&p->mtx); + nni_aio_finish_error(aio, rv); + nni_pipe_bump_error(p->npipe, rv); + return; + } + + n = nni_aio_count(txaio); + nni_aio_iov_advance(txaio, n); + if (nni_aio_iov_count(txaio) > 0) { + nng_stream_send(p->tls, txaio); + nni_mtx_unlock(&p->mtx); + return; + } + nni_aio_list_remove(aio); + tlstran_pipe_send_start(p); + + msg = nni_aio_get_msg(aio); + n = nni_msg_len(msg); + nni_pipe_bump_tx(p->npipe, n); + nni_mtx_unlock(&p->mtx); + nni_aio_set_msg(aio, NULL); + nni_msg_free(msg); + nni_aio_finish_sync(aio, 0, n); +} + +static void +tlstran_pipe_recv_cb(void *arg) +{ + tlstran_pipe *p = arg; + nni_aio * aio; + int rv; + size_t n; + nni_msg * msg; + nni_aio * rxaio = p->rxaio; + + nni_mtx_lock(&p->mtx); + aio = nni_list_first(&p->recvq); + + if ((rv = nni_aio_result(p->rxaio)) != 0) { + goto recv_error; + } + + n = nni_aio_count(rxaio); + nni_aio_iov_advance(rxaio, n); + if (nni_aio_iov_count(rxaio) > 0) { + // Was this a partial read? If so then resubmit for the rest. + nng_stream_recv(p->tls, rxaio); + nni_mtx_unlock(&p->mtx); + return; + } + + // If we don't have a message yet, we were reading the TCP message + // header, which is just the length. This tells us the size of the + // message to allocate and how much more to expect. + if (p->rxmsg == NULL) { + uint64_t len; + // We should have gotten a message header. + NNI_GET64(p->rxlen, len); + + // Make sure the message payload is not too big. If it is + // the caller will shut down the pipe. + if ((len > p->rcvmax) && (p->rcvmax > 0)) { + rv = NNG_EMSGSIZE; + goto recv_error; + } + + if ((rv = nni_msg_alloc(&p->rxmsg, (size_t) len)) != 0) { + goto recv_error; + } + + // Submit the rest of the data for a read -- we want to + // read the entire message now. + if (len != 0) { + nni_iov iov; + iov.iov_buf = nni_msg_body(p->rxmsg); + iov.iov_len = (size_t) len; + nni_aio_set_iov(rxaio, 1, &iov); + + nng_stream_recv(p->tls, rxaio); + nni_mtx_unlock(&p->mtx); + return; + } + } + + // We read a message completely. Let the user know the good news. + nni_aio_list_remove(aio); + msg = p->rxmsg; + p->rxmsg = NULL; + n = nni_msg_len(msg); + if (!nni_list_empty(&p->recvq)) { + tlstran_pipe_recv_start(p); + } + nni_pipe_bump_rx(p->npipe, n); + nni_mtx_unlock(&p->mtx); + + nni_aio_set_msg(aio, msg); + nni_aio_finish_sync(aio, 0, n); + return; + +recv_error: + nni_aio_list_remove(aio); + msg = p->rxmsg; + p->rxmsg = NULL; + nni_pipe_bump_error(p->npipe, rv); + // Intentionally, we do not queue up another receive. + // The protocol should notice this error and close the pipe. + nni_mtx_unlock(&p->mtx); + nni_msg_free(msg); + nni_aio_finish_error(aio, rv); +} + +static void +tlstran_pipe_send_cancel(nni_aio *aio, void *arg, int rv) +{ + tlstran_pipe *p = arg; + + nni_mtx_lock(&p->mtx); + if (!nni_aio_list_active(aio)) { + nni_mtx_unlock(&p->mtx); + return; + } + // If this is being sent, then cancel the pending transfer. + // The callback on the txaio will cause the user aio to + // be canceled too. + if (nni_list_first(&p->sendq) == aio) { + nni_aio_abort(p->txaio, rv); + nni_mtx_unlock(&p->mtx); + return; + } + nni_aio_list_remove(aio); + nni_mtx_unlock(&p->mtx); + + nni_aio_finish_error(aio, rv); +} + +static void +tlstran_pipe_send_start(tlstran_pipe *p) +{ + nni_aio *txaio; + nni_aio *aio; + nni_msg *msg; + int niov; + nni_iov iov[3]; + uint64_t len; + + if ((aio = nni_list_first(&p->sendq)) == NULL) { + return; + } + + msg = nni_aio_get_msg(aio); + len = nni_msg_len(msg) + nni_msg_header_len(msg); + + NNI_PUT64(p->txlen, len); + + txaio = p->txaio; + niov = 0; + iov[niov].iov_buf = p->txlen; + iov[niov].iov_len = sizeof(p->txlen); + niov++; + if (nni_msg_header_len(msg) > 0) { + iov[niov].iov_buf = nni_msg_header(msg); + iov[niov].iov_len = nni_msg_header_len(msg); + niov++; + } + if (nni_msg_len(msg) > 0) { + iov[niov].iov_buf = nni_msg_body(msg); + iov[niov].iov_len = nni_msg_len(msg); + niov++; + } + + nni_aio_set_iov(txaio, niov, iov); + nng_stream_send(p->tls, txaio); +} + +static void +tlstran_pipe_send(void *arg, nni_aio *aio) +{ + tlstran_pipe *p = arg; + int rv; + + if (nni_aio_begin(aio) != 0) { + return; + } + nni_mtx_lock(&p->mtx); + if ((rv = nni_aio_schedule(aio, tlstran_pipe_send_cancel, p)) != 0) { + nni_mtx_unlock(&p->mtx); + nni_aio_finish_error(aio, rv); + return; + } + nni_list_append(&p->sendq, aio); + if (nni_list_first(&p->sendq) == aio) { + tlstran_pipe_send_start(p); + } + nni_mtx_unlock(&p->mtx); +} + +static void +tlstran_pipe_recv_cancel(nni_aio *aio, void *arg, int rv) +{ + tlstran_pipe *p = arg; + + nni_mtx_lock(&p->mtx); + if (!nni_aio_list_active(aio)) { + nni_mtx_unlock(&p->mtx); + return; + } + // If receive in progress, then cancel the pending transfer. + // The callback on the rxaio will cause the user aio to + // be canceled too. + if (nni_list_first(&p->recvq) == aio) { + nni_aio_abort(p->rxaio, rv); + nni_mtx_unlock(&p->mtx); + return; + } + nni_aio_list_remove(aio); + nni_mtx_unlock(&p->mtx); + nni_aio_finish_error(aio, rv); +} + +static void +tlstran_pipe_recv_start(tlstran_pipe *p) +{ + nni_aio *aio; + nni_iov iov; + NNI_ASSERT(p->rxmsg == NULL); + + // Schedule a read of the IPC header. + aio = p->rxaio; + iov.iov_buf = p->rxlen; + iov.iov_len = sizeof(p->rxlen); + nni_aio_set_iov(aio, 1, &iov); + + nng_stream_recv(p->tls, aio); +} + +static void +tlstran_pipe_recv(void *arg, nni_aio *aio) +{ + tlstran_pipe *p = arg; + int rv; + + if (nni_aio_begin(aio) != 0) { + return; + } + nni_mtx_lock(&p->mtx); + if ((rv = nni_aio_schedule(aio, tlstran_pipe_recv_cancel, p)) != 0) { + nni_mtx_unlock(&p->mtx); + nni_aio_finish_error(aio, rv); + return; + } + + nni_aio_list_append(&p->recvq, aio); + if (nni_list_first(&p->recvq) == aio) { + tlstran_pipe_recv_start(p); + } + nni_mtx_unlock(&p->mtx); +} + +static uint16_t +tlstran_pipe_peer(void *arg) +{ + tlstran_pipe *p = arg; + + return (p->peer); +} + +static void +tlstran_pipe_start(tlstran_pipe *p, nng_stream *conn, tlstran_ep *ep) +{ + nni_iov iov; + + ep->refcnt++; + + p->tls = conn; + p->ep = ep; + p->proto = ep->proto; + + p->txlen[0] = 0; + p->txlen[1] = 'S'; + p->txlen[2] = 'P'; + p->txlen[3] = 0; + NNI_PUT16(&p->txlen[4], p->proto); + NNI_PUT16(&p->txlen[6], 0); + + p->gotrxhead = 0; + p->gottxhead = 0; + p->wantrxhead = 8; + p->wanttxhead = 8; + iov.iov_len = 8; + iov.iov_buf = &p->txlen[0]; + nni_aio_set_iov(p->negoaio, 1, &iov); + nni_list_append(&ep->negopipes, p); + + nni_aio_set_timeout(p->negoaio, 10000); // 10 sec timeout to negotiate + nng_stream_send(p->tls, p->negoaio); +} + +static void +tlstran_ep_fini(void *arg) +{ + tlstran_ep *ep = arg; + + nni_mtx_lock(&ep->mtx); + ep->fini = true; + if (ep->refcnt != 0) { + nni_mtx_unlock(&ep->mtx); + return; + } + nni_mtx_unlock(&ep->mtx); + nni_aio_stop(ep->timeaio); + nni_aio_stop(ep->connaio); + nng_stream_dialer_free(ep->dialer); + nng_stream_listener_free(ep->listener); + nni_aio_free(ep->timeaio); + nni_aio_free(ep->connaio); + + nni_mtx_fini(&ep->mtx); + NNI_FREE_STRUCT(ep); +} + +static void +tlstran_ep_close(void *arg) +{ + tlstran_ep * ep = arg; + tlstran_pipe *p; + + nni_mtx_lock(&ep->mtx); + ep->closed = true; + nni_aio_close(ep->timeaio); + + if (ep->dialer != NULL) { + nng_stream_dialer_close(ep->dialer); + } + if (ep->listener != NULL) { + nng_stream_listener_close(ep->listener); + } + NNI_LIST_FOREACH (&ep->negopipes, p) { + tlstran_pipe_close(p); + } + NNI_LIST_FOREACH (&ep->waitpipes, p) { + tlstran_pipe_close(p); + } + NNI_LIST_FOREACH (&ep->busypipes, p) { + tlstran_pipe_close(p); + } + if (ep->useraio != NULL) { + nni_aio_finish_error(ep->useraio, NNG_ECLOSED); + ep->useraio = NULL; + } + nni_mtx_unlock(&ep->mtx); +} + +// This parses off the optional source address that this transport uses. +// The special handling of this URL format is quite honestly an historical +// mistake, which we would remove if we could. +static int +tlstran_url_parse_source(nni_url *url, nng_sockaddr *sa, const nni_url *surl) +{ + int af; + char * semi; + char * src; + size_t len; + int rv; + nni_aio *aio; + + // We modify the URL. This relies on the fact that the underlying + // transport does not free this, so we can just use references. + + url->u_scheme = surl->u_scheme; + url->u_port = surl->u_port; + url->u_hostname = surl->u_hostname; + + if ((semi = strchr(url->u_hostname, ';')) == NULL) { + memset(sa, 0, sizeof(*sa)); + return (0); + } + + len = (size_t)(semi - url->u_hostname); + url->u_hostname = semi + 1; + + if (strcmp(surl->u_scheme, "tls+tcp") == 0) { + af = NNG_AF_UNSPEC; + } else if (strcmp(surl->u_scheme, "tls+tcp4") == 0) { + af = NNG_AF_INET; + } else if (strcmp(surl->u_scheme, "tls+tcp6") == 0) { + af = NNG_AF_INET6; + } else { + return (NNG_EADDRINVAL); + } + + if ((src = nni_alloc(len + 1)) == NULL) { + return (NNG_ENOMEM); + } + memcpy(src, surl->u_hostname, len); + src[len] = '\0'; + + if ((rv = nni_aio_alloc(&aio, NULL, NULL)) != 0) { + nni_free(src, len + 1); + return (rv); + } + + nni_resolv_ip(src, "0", af, 1, sa, aio); + nni_aio_wait(aio); + nni_aio_free(aio); + nni_free(src, len + 1); + return (rv); +} + +static void +tlstran_timer_cb(void *arg) +{ + tlstran_ep *ep = arg; + if (nni_aio_result(ep->timeaio) == 0) { + nng_stream_listener_accept(ep->listener, ep->connaio); + } +} + +static void +tlstran_accept_cb(void *arg) +{ + tlstran_ep * ep = arg; + nni_aio * aio = ep->connaio; + tlstran_pipe *p; + int rv; + nng_stream * conn; + + nni_mtx_lock(&ep->mtx); + + if ((rv = nni_aio_result(aio)) != 0) { + goto error; + } + + conn = nni_aio_get_output(aio, 0); + if ((rv = tlstran_pipe_alloc(&p)) != 0) { + nng_stream_free(conn); + goto error; + } + + if (ep->closed) { + tlstran_pipe_fini(p); + nng_stream_free(conn); + rv = NNG_ECLOSED; + goto error; + } + tlstran_pipe_start(p, conn, ep); + nng_stream_listener_accept(ep->listener, ep->connaio); + nni_mtx_unlock(&ep->mtx); + return; + +error: + // When an error here occurs, let's send a notice up to the consumer. + // That way it can be reported properly. + if ((aio = ep->useraio) != NULL) { + ep->useraio = NULL; + nni_aio_finish_error(aio, rv); + } + switch (rv) { + + case NNG_ENOMEM: + case NNG_ENOFILES: + // We need to cool down here, to avoid spinning. + nng_sleep_aio(10, ep->timeaio); + break; + + default: + // Start another accept. This is done because we want to + // ensure that TLS negotiations are disconnected from + // the upper layer accept logic. + if (!ep->closed) { + nng_stream_listener_accept(ep->listener, ep->connaio); + } + break; + } + nni_mtx_unlock(&ep->mtx); +} + +static void +tlstran_dial_cb(void *arg) +{ + tlstran_ep * ep = arg; + nni_aio * aio = ep->connaio; + tlstran_pipe *p; + int rv; + nng_stream * conn; + + if ((rv = nni_aio_result(aio)) != 0) { + goto error; + } + + conn = nni_aio_get_output(aio, 0); + if ((rv = tlstran_pipe_alloc(&p)) != 0) { + nng_stream_free(conn); + goto error; + } + nni_mtx_lock(&ep->mtx); + if (ep->closed) { + tlstran_pipe_fini(p); + nng_stream_free(conn); + rv = NNG_ECLOSED; + nni_mtx_unlock(&ep->mtx); + goto error; + } else { + tlstran_pipe_start(p, conn, ep); + } + nni_mtx_unlock(&ep->mtx); + return; + +error: + // Error connecting. We need to pass this straight back to the user. + nni_mtx_lock(&ep->mtx); + if ((aio = ep->useraio) != NULL) { + ep->useraio = NULL; + nni_aio_finish_error(aio, rv); + } + nni_mtx_unlock(&ep->mtx); +} + +static int +tlstran_ep_init(tlstran_ep **epp, nng_url *url, nni_sock *sock) +{ + tlstran_ep *ep; + + if ((ep = NNI_ALLOC_STRUCT(ep)) == NULL) { + return (NNG_ENOMEM); + } + nni_mtx_init(&ep->mtx); + NNI_LIST_INIT(&ep->busypipes, tlstran_pipe, node); + NNI_LIST_INIT(&ep->waitpipes, tlstran_pipe, node); + NNI_LIST_INIT(&ep->negopipes, tlstran_pipe, node); + + ep->proto = nni_sock_proto_id(sock); + ep->url = url; + +#ifdef NNG_ENABLE_STATS + static const nni_stat_info rcv_max_info = { + .si_name = "rcv_max", + .si_desc = "maximum receive size", + .si_type = NNG_STAT_LEVEL, + .si_unit = NNG_UNIT_BYTES, + .si_atomic = true, + }; + nni_stat_init(&ep->st_rcv_max, &rcv_max_info); +#endif + + *epp = ep; + return (0); +} + +static int +tlstran_ep_init_dialer(void **dp, nni_url *url, nni_dialer *ndialer) +{ + tlstran_ep * ep; + int rv; + nng_sockaddr srcsa; + nni_sock * sock = nni_dialer_sock(ndialer); + nni_url myurl; + + // Check for invalid URL components. + if ((strlen(url->u_path) != 0) && (strcmp(url->u_path, "/") != 0)) { + return (NNG_EADDRINVAL); + } + if ((url->u_fragment != NULL) || (url->u_userinfo != NULL) || + (url->u_query != NULL) || (strlen(url->u_hostname) == 0) || + (strlen(url->u_port) == 0)) { + return (NNG_EADDRINVAL); + } + + if ((rv = tlstran_url_parse_source(&myurl, &srcsa, url)) != 0) { + return (rv); + } + + if (((rv = tlstran_ep_init(&ep, url, sock)) != 0) || + ((rv = nni_aio_alloc(&ep->connaio, tlstran_dial_cb, ep)) != 0)) { + return (rv); + } + ep->authmode = NNG_TLS_AUTH_MODE_REQUIRED; + + if ((rv != 0) || + ((rv = nng_stream_dialer_alloc_url(&ep->dialer, &myurl)) != 0)) { + tlstran_ep_fini(ep); + return (rv); + } + if ((srcsa.s_family != NNG_AF_UNSPEC) && + ((rv = nni_stream_dialer_set(ep->dialer, NNG_OPT_LOCADDR, &srcsa, + sizeof(srcsa), NNI_TYPE_SOCKADDR)) != 0)) { + tlstran_ep_fini(ep); + return (rv); + } +#ifdef NNG_ENABLE_STATS + nni_dialer_add_stat(ndialer, &ep->st_rcv_max); +#endif + *dp = ep; + return (0); +} + +static int +tlstran_ep_init_listener(void **lp, nni_url *url, nni_listener *nlistener) +{ + tlstran_ep *ep; + int rv; + uint16_t af; + char * host = url->u_hostname; + nni_aio * aio; + nni_sock * sock = nni_listener_sock(nlistener); + + if (strcmp(url->u_scheme, "tls+tcp") == 0) { + af = NNG_AF_UNSPEC; + } else if (strcmp(url->u_scheme, "tls+tcp4") == 0) { + af = NNG_AF_INET; + } else if (strcmp(url->u_scheme, "tls+tcp6") == 0) { + af = NNG_AF_INET6; + } else { + return (NNG_EADDRINVAL); + } + + // Check for invalid URL components. + if ((strlen(url->u_path) != 0) && (strcmp(url->u_path, "/") != 0)) { + return (NNG_EADDRINVAL); + } + if ((url->u_fragment != NULL) || (url->u_userinfo != NULL) || + (url->u_query != NULL)) { + return (NNG_EADDRINVAL); + } + if (((rv = tlstran_ep_init(&ep, url, sock)) != 0) || + ((rv = nni_aio_alloc(&ep->connaio, tlstran_accept_cb, ep)) != 0) || + ((rv = nni_aio_alloc(&ep->timeaio, tlstran_timer_cb, ep)) != 0)) { + return (rv); + } + + ep->authmode = NNG_TLS_AUTH_MODE_NONE; + + if (strlen(host) == 0) { + host = NULL; + } + + // XXX: We are doing lookup at listener initialization. There is + // a valid argument that this should be done at bind time, but that + // would require making bind asynchronous. In some ways this would + // be worse than the cost of just waiting here. We always recommend + // using local IP addresses rather than names when possible. + + if ((rv = nni_aio_alloc(&aio, NULL, NULL)) != 0) { + tlstran_ep_fini(ep); + return (rv); + } + nni_resolv_ip(host, url->u_port, af, true, &ep->sa, aio); + nni_aio_wait(aio); + rv = nni_aio_result(aio); + nni_aio_free(aio); + + if ((rv != 0) || + ((rv = nng_stream_listener_alloc_url(&ep->listener, url)) != 0) || + ((rv = nni_stream_listener_set(ep->listener, NNG_OPT_TLS_AUTH_MODE, + &ep->authmode, sizeof(ep->authmode), NNI_TYPE_INT32)) != + 0)) { + tlstran_ep_fini(ep); + return (rv); + } +#ifdef NNG_ENABLE_STATS + nni_listener_add_stat(nlistener, &ep->st_rcv_max); +#endif + *lp = ep; + return (0); +} + +static void +tlstran_ep_cancel(nni_aio *aio, void *arg, int rv) +{ + tlstran_ep *ep = arg; + nni_mtx_lock(&ep->mtx); + if (ep->useraio == aio) { + ep->useraio = NULL; + nni_aio_finish_error(aio, rv); + } + nni_mtx_unlock(&ep->mtx); +} + +static void +tlstran_ep_connect(void *arg, nni_aio *aio) +{ + tlstran_ep *ep = arg; + int rv; + + if (nni_aio_begin(aio) != 0) { + return; + } + + nni_mtx_lock(&ep->mtx); + if (ep->closed) { + nni_mtx_unlock(&ep->mtx); + nni_aio_finish_error(aio, NNG_ECLOSED); + return; + } + if (ep->useraio != NULL) { + nni_mtx_unlock(&ep->mtx); + nni_aio_finish_error(aio, NNG_EBUSY); + return; + } + if ((rv = nni_aio_schedule(aio, tlstran_ep_cancel, ep)) != 0) { + nni_mtx_unlock(&ep->mtx); + nni_aio_finish_error(aio, rv); + return; + } + ep->useraio = aio; + + nng_stream_dialer_dial(ep->dialer, ep->connaio); + nni_mtx_unlock(&ep->mtx); +} + +static int +tlstran_ep_bind(void *arg) +{ + tlstran_ep *ep = arg; + int rv; + + nni_mtx_lock(&ep->mtx); + rv = nng_stream_listener_listen(ep->listener); + nni_mtx_unlock(&ep->mtx); + + return (rv); +} + +static void +tlstran_ep_accept(void *arg, nni_aio *aio) +{ + tlstran_ep *ep = arg; + int rv; + + if (nni_aio_begin(aio) != 0) { + return; + } + nni_mtx_lock(&ep->mtx); + if (ep->closed) { + nni_mtx_unlock(&ep->mtx); + nni_aio_finish_error(aio, NNG_ECLOSED); + return; + } + if (ep->useraio != NULL) { + nni_mtx_unlock(&ep->mtx); + nni_aio_finish_error(aio, NNG_EBUSY); + return; + } + if ((rv = nni_aio_schedule(aio, tlstran_ep_cancel, ep)) != 0) { + nni_mtx_unlock(&ep->mtx); + nni_aio_finish_error(aio, rv); + return; + } + ep->useraio = aio; + if (!ep->started) { + ep->started = true; + nng_stream_listener_accept(ep->listener, ep->connaio); + } else { + tlstran_ep_match(ep); + } + nni_mtx_unlock(&ep->mtx); +} + +static int +tlstran_ep_set_recvmaxsz(void *arg, const void *v, size_t sz, nni_type t) +{ + tlstran_ep *ep = arg; + size_t val; + int rv; + if ((rv = nni_copyin_size(&val, v, sz, 0, NNI_MAXSZ, t)) == 0) { + tlstran_pipe *p; + nni_mtx_lock(&ep->mtx); + ep->rcvmax = val; + NNI_LIST_FOREACH (&ep->waitpipes, p) { + p->rcvmax = val; + } + NNI_LIST_FOREACH (&ep->negopipes, p) { + p->rcvmax = val; + } + NNI_LIST_FOREACH (&ep->busypipes, p) { + p->rcvmax = val; + } + nni_mtx_unlock(&ep->mtx); +#ifdef NNG_ENABLE_STATS + nni_stat_set_value(&ep->st_rcv_max, val); +#endif + } + return (rv); +} + +static int +tlstran_ep_get_recvmaxsz(void *arg, void *v, size_t *szp, nni_type t) +{ + tlstran_ep *ep = arg; + int rv; + nni_mtx_lock(&ep->mtx); + rv = nni_copyout_size(ep->rcvmax, v, szp, t); + nni_mtx_unlock(&ep->mtx); + return (rv); +} + +static int +tlstran_ep_get_url(void *arg, void *v, size_t *szp, nni_type t) +{ + tlstran_ep *ep = arg; + char * s; + int rv; + int port = 0; + + if (ep->listener != NULL) { + (void) nng_stream_listener_get_int( + ep->listener, NNG_OPT_TCP_BOUND_PORT, &port); + } + if ((rv = nni_url_asprintf_port(&s, ep->url, port)) == 0) { + rv = nni_copyout_str(s, v, szp, t); + nni_strfree(s); + } + return (rv); +} + +static const nni_option tlstran_pipe_opts[] = { + // terminate list + { + .o_name = NULL, + }, +}; + +static int +tlstran_pipe_getopt( + void *arg, const char *name, void *buf, size_t *szp, nni_type t) +{ + tlstran_pipe *p = arg; + int rv; + + if ((rv = nni_stream_get(p->tls, name, buf, szp, t)) == NNG_ENOTSUP) { + rv = nni_getopt(tlstran_pipe_opts, name, p, buf, szp, t); + } + return (rv); +} + +static nni_tran_pipe_ops tlstran_pipe_ops = { + .p_init = tlstran_pipe_init, + .p_fini = tlstran_pipe_fini, + .p_stop = tlstran_pipe_stop, + .p_send = tlstran_pipe_send, + .p_recv = tlstran_pipe_recv, + .p_close = tlstran_pipe_close, + .p_peer = tlstran_pipe_peer, + .p_getopt = tlstran_pipe_getopt, +}; + +static nni_option tlstran_ep_options[] = { + { + .o_name = NNG_OPT_RECVMAXSZ, + .o_get = tlstran_ep_get_recvmaxsz, + .o_set = tlstran_ep_set_recvmaxsz, + }, + { + .o_name = NNG_OPT_URL, + .o_get = tlstran_ep_get_url, + }, + // terminate list + { + .o_name = NULL, + }, +}; + +static int +tlstran_dialer_getopt( + void *arg, const char *name, void *buf, size_t *szp, nni_type t) +{ + int rv; + tlstran_ep *ep = arg; + + rv = nni_stream_dialer_get(ep->dialer, name, buf, szp, t); + if (rv == NNG_ENOTSUP) { + rv = nni_getopt(tlstran_ep_options, name, ep, buf, szp, t); + } + return (rv); +} + +static int +tlstran_dialer_setopt( + void *arg, const char *name, const void *buf, size_t sz, nni_type t) +{ + int rv; + tlstran_ep *ep = arg; + + rv = nni_stream_dialer_set( + ep != NULL ? ep->dialer : NULL, name, buf, sz, t); + if (rv == NNG_ENOTSUP) { + rv = nni_setopt(tlstran_ep_options, name, ep, buf, sz, t); + } + return (rv); +} + +static int +tlstran_listener_get( + void *arg, const char *name, void *buf, size_t *szp, nni_type t) +{ + int rv; + tlstran_ep *ep = arg; + + rv = nni_stream_listener_get(ep->listener, name, buf, szp, t); + if (rv == NNG_ENOTSUP) { + rv = nni_getopt(tlstran_ep_options, name, ep, buf, szp, t); + } + return (rv); +} + +static int +tlstran_listener_set( + void *arg, const char *name, const void *buf, size_t sz, nni_type t) +{ + int rv; + tlstran_ep *ep = arg; + + rv = nni_stream_listener_set( + ep != NULL ? ep->listener : NULL, name, buf, sz, t); + if (rv == NNG_ENOTSUP) { + rv = nni_setopt(tlstran_ep_options, name, ep, buf, sz, t); + } + return (rv); +} + +static nni_tran_dialer_ops tlstran_dialer_ops = { + .d_init = tlstran_ep_init_dialer, + .d_fini = tlstran_ep_fini, + .d_connect = tlstran_ep_connect, + .d_close = tlstran_ep_close, + .d_getopt = tlstran_dialer_getopt, + .d_setopt = tlstran_dialer_setopt, +}; + +static nni_tran_listener_ops tlstran_listener_ops = { + .l_init = tlstran_ep_init_listener, + .l_fini = tlstran_ep_fini, + .l_bind = tlstran_ep_bind, + .l_accept = tlstran_ep_accept, + .l_close = tlstran_ep_close, + .l_getopt = tlstran_listener_get, + .l_setopt = tlstran_listener_set, +}; + +static nni_tran tls_tran = { + .tran_version = NNI_TRANSPORT_VERSION, + .tran_scheme = "tls+tcp", + .tran_dialer = &tlstran_dialer_ops, + .tran_listener = &tlstran_listener_ops, + .tran_pipe = &tlstran_pipe_ops, + .tran_init = tlstran_init, + .tran_fini = tlstran_fini, +}; + +static nni_tran tls4_tran = { + .tran_version = NNI_TRANSPORT_VERSION, + .tran_scheme = "tls+tcp4", + .tran_dialer = &tlstran_dialer_ops, + .tran_listener = &tlstran_listener_ops, + .tran_pipe = &tlstran_pipe_ops, + .tran_init = tlstran_init, + .tran_fini = tlstran_fini, +}; + +static nni_tran tls6_tran = { + .tran_version = NNI_TRANSPORT_VERSION, + .tran_scheme = "tls+tcp6", + .tran_dialer = &tlstran_dialer_ops, + .tran_listener = &tlstran_listener_ops, + .tran_pipe = &tlstran_pipe_ops, + .tran_init = tlstran_init, + .tran_fini = tlstran_fini, +}; + +int +nng_tls_register(void) +{ + int rv; + if (((rv = nni_tran_register(&tls_tran)) != 0) || + ((rv = nni_tran_register(&tls4_tran)) != 0) || + ((rv = nni_tran_register(&tls6_tran)) != 0)) { + return (rv); + } + return (0); +} diff --git a/src/sp/transport/ws/CMakeLists.txt b/src/sp/transport/ws/CMakeLists.txt new file mode 100644 index 00000000..6e409b43 --- /dev/null +++ b/src/sp/transport/ws/CMakeLists.txt @@ -0,0 +1,24 @@ +# +# Copyright 2020 Staysail Systems, Inc. +# Copyright 2018 Capitar IT Group BV +# +# This software is supplied under the terms of the MIT License, a +# copy of which should be located in the distribution where this +# file was obtained (LICENSE.txt). A copy of the license may also be +# found online at https://opensource.org/licenses/MIT. +# + +# WebSocket transport +nng_directory(ws) + +if (NNG_TRANSPORT_WS OR NNG_TRANSPORT_WSS) + set(WS_ON ON) +endif() + +nng_defines_if(NNG_TRANSPORT_WS NNG_TRANSPORT_WS) +nng_defines_if(NNG_TRANSPORT_WSS NNG_TRANSPORT_WSS) +nng_sources_if(WS_ON websocket.c) +nng_headers_if(WS_ON nng/transport/ws/websocket.h) +nng_test_if(WS_ON ws_test) + + diff --git a/src/sp/transport/ws/README.adoc b/src/sp/transport/ws/README.adoc new file mode 100644 index 00000000..e3101297 --- /dev/null +++ b/src/sp/transport/ws/README.adoc @@ -0,0 +1,38 @@ += websocket transport + +This transport provides support for SP over websocket using TCP or TLS. +When using TCP, it is compatible with the libnanomsg legacy transport. +It also is compatible with mangos (both TCP and TLS). + +TLS support requires the mbedTLS library. + +We set the "protocol" such as "pair.sp.nanomsg.org" in the +Sec-WebSocket-Protocol field -- the client sets to the the server's +protocol - i.e. the protocol that the server speaks. For example, +if the the server is a REP, then a REQ client would send "rep.sp.nanomsg.org". + +The server sends the same value (it's own), per the WebSocket specs. (Note +that the client's protocol is never sent, but assumed to be complementary +to the protocol in the Sec-WebSocket-Protocol field.) + +Each SP message is a WebSocket message. + +WebSocket is defined in RFC 6455. + +== Design + +We unfortunately need to implement our own design for this -- the only +reasonable client library would be libcurl, and there is a dearth of +suitable server libraries. Since we don't have to support full HTTP, but +just the initial handshake, this isn't too tragic. + +== Multiple Server Sockets + +In order to support Multiple Server sockets listening on the same port, +the application must be long lived. We will set up a listener on the +configured TCP (or TLS) port, and examine the PATH supplied in the GET. +This will be used to match against the URL requested, and if the URL +matches we will create the appropriate pipe. + +If no server endpoint at that address can be found, we return an +HTTP error, and close the socket. diff --git a/src/sp/transport/ws/websocket.c b/src/sp/transport/ws/websocket.c new file mode 100644 index 00000000..3f73f47f --- /dev/null +++ b/src/sp/transport/ws/websocket.c @@ -0,0 +1,740 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// Copyright 2018 Capitar IT Group BV +// Copyright 2019 Devolutions +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include +#include +#include + +#include "core/nng_impl.h" +#include "supplemental/websocket/websocket.h" + +#include +#include + +typedef struct ws_dialer ws_dialer; +typedef struct ws_listener ws_listener; +typedef struct ws_pipe ws_pipe; + +struct ws_dialer { + uint16_t peer; // remote protocol + nni_list aios; + nni_mtx mtx; + nni_aio * connaio; + nng_stream_dialer *dialer; + bool started; +}; + +struct ws_listener { + uint16_t peer; // remote protocol + nni_list aios; + nni_mtx mtx; + nni_aio * accaio; + nng_stream_listener *listener; + bool started; +}; + +struct ws_pipe { + nni_mtx mtx; + bool closed; + uint16_t peer; + nni_aio * user_txaio; + nni_aio * user_rxaio; + nni_aio * txaio; + nni_aio * rxaio; + nng_stream *ws; +}; + +static void +wstran_pipe_send_cb(void *arg) +{ + ws_pipe *p = arg; + nni_aio *taio; + nni_aio *uaio; + + nni_mtx_lock(&p->mtx); + taio = p->txaio; + uaio = p->user_txaio; + p->user_txaio = NULL; + + if (uaio != NULL) { + int rv; + if ((rv = nni_aio_result(taio)) != 0) { + nni_aio_finish_error(uaio, rv); + } else { + nni_aio_finish(uaio, 0, 0); + } + } + nni_mtx_unlock(&p->mtx); +} + +static void +wstran_pipe_recv_cb(void *arg) +{ + ws_pipe *p = arg; + nni_aio *raio = p->rxaio; + nni_aio *uaio; + int rv; + + nni_mtx_lock(&p->mtx); + uaio = p->user_rxaio; + p->user_rxaio = NULL; + if ((rv = nni_aio_result(raio)) != 0) { + if (uaio != NULL) { + nni_aio_finish_error(uaio, rv); + } + } else { + nni_msg *msg = nni_aio_get_msg(raio); + if (uaio != NULL) { + nni_aio_finish_msg(uaio, msg); + } else { + nni_msg_free(msg); + } + } + nni_mtx_unlock(&p->mtx); +} + +static void +wstran_pipe_recv_cancel(nni_aio *aio, void *arg, int rv) +{ + ws_pipe *p = arg; + nni_mtx_lock(&p->mtx); + if (p->user_rxaio != aio) { + nni_mtx_unlock(&p->mtx); + return; + } + p->user_rxaio = NULL; + nni_aio_abort(p->rxaio, rv); + nni_aio_finish_error(aio, rv); + nni_mtx_unlock(&p->mtx); +} + +static void +wstran_pipe_recv(void *arg, nni_aio *aio) +{ + ws_pipe *p = arg; + int rv; + + if (nni_aio_begin(aio) != 0) { + return; + } + nni_mtx_lock(&p->mtx); + if ((rv = nni_aio_schedule(aio, wstran_pipe_recv_cancel, p)) != 0) { + nni_mtx_unlock(&p->mtx); + nni_aio_finish_error(aio, rv); + return; + } + p->user_rxaio = aio; + nng_stream_recv(p->ws, p->rxaio); + nni_mtx_unlock(&p->mtx); +} + +static void +wstran_pipe_send_cancel(nni_aio *aio, void *arg, int rv) +{ + ws_pipe *p = arg; + nni_mtx_lock(&p->mtx); + if (p->user_txaio != aio) { + nni_mtx_unlock(&p->mtx); + return; + } + p->user_txaio = NULL; + nni_aio_abort(p->txaio, rv); + nni_aio_finish_error(aio, rv); + nni_mtx_unlock(&p->mtx); +} + +static void +wstran_pipe_send(void *arg, nni_aio *aio) +{ + ws_pipe *p = arg; + int rv; + + if (nni_aio_begin(aio) != 0) { + return; + } + nni_mtx_lock(&p->mtx); + if ((rv = nni_aio_schedule(aio, wstran_pipe_send_cancel, p)) != 0) { + nni_mtx_unlock(&p->mtx); + nni_aio_finish_error(aio, rv); + return; + } + p->user_txaio = aio; + nni_aio_set_msg(p->txaio, nni_aio_get_msg(aio)); + nni_aio_set_msg(aio, NULL); + + nng_stream_send(p->ws, p->txaio); + nni_mtx_unlock(&p->mtx); +} + +static void +wstran_pipe_stop(void *arg) +{ + ws_pipe *p = arg; + + nni_aio_stop(p->rxaio); + nni_aio_stop(p->txaio); +} + +static int +wstran_pipe_init(void *arg, nni_pipe *pipe) +{ + NNI_ARG_UNUSED(arg); + NNI_ARG_UNUSED(pipe); + return (0); +} + +static void +wstran_pipe_fini(void *arg) +{ + ws_pipe *p = arg; + + nni_aio_free(p->rxaio); + nni_aio_free(p->txaio); + + nng_stream_free(p->ws); + nni_mtx_fini(&p->mtx); + NNI_FREE_STRUCT(p); +} + +static void +wstran_pipe_close(void *arg) +{ + ws_pipe *p = arg; + + nni_aio_close(p->rxaio); + nni_aio_close(p->txaio); + + nni_mtx_lock(&p->mtx); + nng_stream_close(p->ws); + nni_mtx_unlock(&p->mtx); +} + +static int +wstran_pipe_alloc(ws_pipe **pipep, void *ws) +{ + ws_pipe *p; + int rv; + + if ((p = NNI_ALLOC_STRUCT(p)) == NULL) { + return (NNG_ENOMEM); + } + nni_mtx_init(&p->mtx); + + // Initialize AIOs. + if (((rv = nni_aio_alloc(&p->txaio, wstran_pipe_send_cb, p)) != 0) || + ((rv = nni_aio_alloc(&p->rxaio, wstran_pipe_recv_cb, p)) != 0)) { + wstran_pipe_fini(p); + return (rv); + } + p->ws = ws; + + *pipep = p; + return (0); +} + +static uint16_t +wstran_pipe_peer(void *arg) +{ + ws_pipe *p = arg; + + return (p->peer); +} + +static int +ws_listener_bind(void *arg) +{ + ws_listener *l = arg; + int rv; + + if ((rv = nng_stream_listener_listen(l->listener)) == 0) { + l->started = true; + } + return (rv); +} + +static void +ws_listener_cancel(nni_aio *aio, void *arg, int rv) +{ + ws_listener *l = arg; + + nni_mtx_lock(&l->mtx); + if (nni_aio_list_active(aio)) { + nni_aio_list_remove(aio); + nni_aio_finish_error(aio, rv); + } + nni_mtx_unlock(&l->mtx); +} + +static void +wstran_listener_accept(void *arg, nni_aio *aio) +{ + ws_listener *l = arg; + int rv; + + // We already bound, so we just need to look for an available + // pipe (created by the handler), and match it. + // Otherwise we stick the AIO in the accept list. + if (nni_aio_begin(aio) != 0) { + return; + } + nni_mtx_lock(&l->mtx); + if ((rv = nni_aio_schedule(aio, ws_listener_cancel, l)) != 0) { + nni_mtx_unlock(&l->mtx); + nni_aio_finish_error(aio, rv); + return; + } + nni_list_append(&l->aios, aio); + if (aio == nni_list_first(&l->aios)) { + nng_stream_listener_accept(l->listener, l->accaio); + } + nni_mtx_unlock(&l->mtx); +} + +static void +wstran_dialer_cancel(nni_aio *aio, void *arg, int rv) +{ + ws_dialer *d = arg; + + nni_mtx_lock(&d->mtx); + if (nni_aio_list_active(aio)) { + nni_aio_list_remove(aio); + nni_aio_finish_error(aio, rv); + } + nni_mtx_unlock(&d->mtx); +} + +static void +wstran_dialer_connect(void *arg, nni_aio *aio) +{ + ws_dialer *d = arg; + int rv; + + if (nni_aio_begin(aio) != 0) { + return; + } + + nni_mtx_lock(&d->mtx); + if ((rv = nni_aio_schedule(aio, wstran_dialer_cancel, d)) != 0) { + nni_mtx_unlock(&d->mtx); + nni_aio_finish_error(aio, rv); + return; + } + NNI_ASSERT(nni_list_empty(&d->aios)); + d->started = true; + nni_list_append(&d->aios, aio); + nng_stream_dialer_dial(d->dialer, d->connaio); + nni_mtx_unlock(&d->mtx); +} + +static const nni_option ws_pipe_options[] = { + // terminate list + { + .o_name = NULL, + } +}; + +static int +wstran_pipe_getopt( + void *arg, const char *name, void *buf, size_t *szp, nni_type t) +{ + ws_pipe *p = arg; + int rv; + + if ((rv = nni_stream_get(p->ws, name, buf, szp, t)) == NNG_ENOTSUP) { + rv = nni_getopt(ws_pipe_options, name, p, buf, szp, t); + } + return (rv); +} + +static nni_tran_pipe_ops ws_pipe_ops = { + .p_init = wstran_pipe_init, + .p_fini = wstran_pipe_fini, + .p_stop = wstran_pipe_stop, + .p_send = wstran_pipe_send, + .p_recv = wstran_pipe_recv, + .p_close = wstran_pipe_close, + .p_peer = wstran_pipe_peer, + .p_getopt = wstran_pipe_getopt, +}; + +static void +wstran_dialer_fini(void *arg) +{ + ws_dialer *d = arg; + + nni_aio_stop(d->connaio); + nng_stream_dialer_free(d->dialer); + nni_aio_free(d->connaio); + nni_mtx_fini(&d->mtx); + NNI_FREE_STRUCT(d); +} + +static void +wstran_listener_fini(void *arg) +{ + ws_listener *l = arg; + + nni_aio_stop(l->accaio); + nng_stream_listener_free(l->listener); + nni_aio_free(l->accaio); + nni_mtx_fini(&l->mtx); + NNI_FREE_STRUCT(l); +} + +static void +wstran_connect_cb(void *arg) +{ + ws_dialer * d = arg; + ws_pipe * p; + nni_aio * caio = d->connaio; + nni_aio * uaio; + int rv; + nng_stream *ws = NULL; + + nni_mtx_lock(&d->mtx); + if (nni_aio_result(caio) == 0) { + ws = nni_aio_get_output(caio, 0); + } + if ((uaio = nni_list_first(&d->aios)) == NULL) { + // The client stopped caring about this! + nng_stream_free(ws); + nni_mtx_unlock(&d->mtx); + return; + } + nni_aio_list_remove(uaio); + NNI_ASSERT(nni_list_empty(&d->aios)); + if ((rv = nni_aio_result(caio)) != 0) { + nni_aio_finish_error(uaio, rv); + } else if ((rv = wstran_pipe_alloc(&p, ws)) != 0) { + nng_stream_free(ws); + nni_aio_finish_error(uaio, rv); + } else { + p->peer = d->peer; + + nni_aio_set_output(uaio, 0, p); + nni_aio_finish(uaio, 0, 0); + } + nni_mtx_unlock(&d->mtx); +} + +static void +wstran_dialer_close(void *arg) +{ + ws_dialer *d = arg; + + nni_aio_close(d->connaio); + nng_stream_dialer_close(d->dialer); +} + +static void +wstran_listener_close(void *arg) +{ + ws_listener *l = arg; + + nni_aio_close(l->accaio); + nng_stream_listener_close(l->listener); +} + +static void +wstran_accept_cb(void *arg) +{ + ws_listener *l = arg; + nni_aio * aaio = l->accaio; + nni_aio * uaio; + int rv; + + nni_mtx_lock(&l->mtx); + uaio = nni_list_first(&l->aios); + if ((rv = nni_aio_result(aaio)) != 0) { + if (uaio != NULL) { + nni_aio_list_remove(uaio); + nni_aio_finish_error(uaio, rv); + } + } else { + nng_stream *ws = nni_aio_get_output(aaio, 0); + if (uaio != NULL) { + ws_pipe *p; + // Make a pipe + nni_aio_list_remove(uaio); + if ((rv = wstran_pipe_alloc(&p, ws)) != 0) { + nng_stream_close(ws); + nni_aio_finish_error(uaio, rv); + } else { + p->peer = l->peer; + + nni_aio_set_output(uaio, 0, p); + nni_aio_finish(uaio, 0, 0); + } + } + } + if (!nni_list_empty(&l->aios)) { + nng_stream_listener_accept(l->listener, aaio); + } + nni_mtx_unlock(&l->mtx); +} + +static int +wstran_dialer_init(void **dp, nng_url *url, nni_dialer *ndialer) +{ + ws_dialer *d; + nni_sock * s = nni_dialer_sock(ndialer); + int rv; + char name[64]; + + if ((d = NNI_ALLOC_STRUCT(d)) == NULL) { + return (NNG_ENOMEM); + } + nni_mtx_init(&d->mtx); + + nni_aio_list_init(&d->aios); + + d->peer = nni_sock_peer_id(s); + + snprintf( + name, sizeof(name), "%s.sp.nanomsg.org", nni_sock_peer_name(s)); + + if (((rv = nni_ws_dialer_alloc(&d->dialer, url)) != 0) || + ((rv = nni_aio_alloc(&d->connaio, wstran_connect_cb, d)) != 0) || + ((rv = nng_stream_dialer_set_bool( + d->dialer, NNI_OPT_WS_MSGMODE, true)) != 0) || + ((rv = nng_stream_dialer_set_string( + d->dialer, NNG_OPT_WS_PROTOCOL, name)) != 0)) { + wstran_dialer_fini(d); + return (rv); + } + + *dp = d; + return (0); +} + +static int +wstran_listener_init(void **lp, nng_url *url, nni_listener *listener) +{ + ws_listener *l; + int rv; + nni_sock * s = nni_listener_sock(listener); + char name[64]; + + if ((l = NNI_ALLOC_STRUCT(l)) == NULL) { + return (NNG_ENOMEM); + } + nni_mtx_init(&l->mtx); + + nni_aio_list_init(&l->aios); + + l->peer = nni_sock_peer_id(s); + + snprintf( + name, sizeof(name), "%s.sp.nanomsg.org", nni_sock_proto_name(s)); + + if (((rv = nni_ws_listener_alloc(&l->listener, url)) != 0) || + ((rv = nni_aio_alloc(&l->accaio, wstran_accept_cb, l)) != 0) || + ((rv = nng_stream_listener_set_bool( + l->listener, NNI_OPT_WS_MSGMODE, true)) != 0) || + ((rv = nng_stream_listener_set_string( + l->listener, NNG_OPT_WS_PROTOCOL, name)) != 0)) { + wstran_listener_fini(l); + return (rv); + } + *lp = l; + return (0); +} + +static int +wstran_init(void) +{ + return (0); +} + +static void +wstran_fini(void) +{ +} + +static const nni_option wstran_ep_opts[] = { + // terminate list + { + .o_name = NULL, + }, +}; + +static int +wstran_dialer_getopt( + void *arg, const char *name, void *buf, size_t *szp, nni_type t) +{ + ws_dialer *d = arg; + int rv; + + rv = nni_stream_dialer_get(d->dialer, name, buf, szp, t); + if (rv == NNG_ENOTSUP) { + rv = nni_getopt(wstran_ep_opts, name, d, buf, szp, t); + } + return (rv); +} + +static int +wstran_dialer_setopt( + void *arg, const char *name, const void *buf, size_t sz, nni_type t) +{ + ws_dialer *d = arg; + int rv; + + rv = nni_stream_dialer_set(d->dialer, name, buf, sz, t); + if (rv == NNG_ENOTSUP) { + rv = nni_setopt(wstran_ep_opts, name, d, buf, sz, t); + } + return (rv); +} + +static int +wstran_listener_get( + void *arg, const char *name, void *buf, size_t *szp, nni_type t) +{ + ws_listener *l = arg; + int rv; + + rv = nni_stream_listener_get(l->listener, name, buf, szp, t); + if (rv == NNG_ENOTSUP) { + rv = nni_getopt(wstran_ep_opts, name, l, buf, szp, t); + } + return (rv); +} + +static int +wstran_listener_set( + void *arg, const char *name, const void *buf, size_t sz, nni_type t) +{ + ws_listener *l = arg; + int rv; + + rv = nni_stream_listener_set(l->listener, name, buf, sz, t); + if (rv == NNG_ENOTSUP) { + rv = nni_setopt(wstran_ep_opts, name, l, buf, sz, t); + } + return (rv); +} + +static nni_tran_dialer_ops ws_dialer_ops = { + .d_init = wstran_dialer_init, + .d_fini = wstran_dialer_fini, + .d_connect = wstran_dialer_connect, + .d_close = wstran_dialer_close, + .d_setopt = wstran_dialer_setopt, + .d_getopt = wstran_dialer_getopt, +}; + +static nni_tran_listener_ops ws_listener_ops = { + .l_init = wstran_listener_init, + .l_fini = wstran_listener_fini, + .l_bind = ws_listener_bind, + .l_accept = wstran_listener_accept, + .l_close = wstran_listener_close, + .l_setopt = wstran_listener_set, + .l_getopt = wstran_listener_get, +}; + +static nni_tran ws_tran = { + .tran_version = NNI_TRANSPORT_VERSION, + .tran_scheme = "ws", + .tran_dialer = &ws_dialer_ops, + .tran_listener = &ws_listener_ops, + .tran_pipe = &ws_pipe_ops, + .tran_init = wstran_init, + .tran_fini = wstran_fini, +}; + +static nni_tran ws4_tran = { + .tran_version = NNI_TRANSPORT_VERSION, + .tran_scheme = "ws4", + .tran_dialer = &ws_dialer_ops, + .tran_listener = &ws_listener_ops, + .tran_pipe = &ws_pipe_ops, + .tran_init = wstran_init, + .tran_fini = wstran_fini, +}; + +static nni_tran ws6_tran = { + .tran_version = NNI_TRANSPORT_VERSION, + .tran_scheme = "ws6", + .tran_dialer = &ws_dialer_ops, + .tran_listener = &ws_listener_ops, + .tran_pipe = &ws_pipe_ops, + .tran_init = wstran_init, + .tran_fini = wstran_fini, +}; + +int +nng_ws_register(void) +{ + int rv; + if (((rv = nni_tran_register(&ws_tran)) != 0) || + ((rv = nni_tran_register(&ws4_tran)) != 0) || + ((rv = nni_tran_register(&ws6_tran)) != 0)) { + return (rv); + } + + return (0); +} + +#ifdef NNG_TRANSPORT_WSS + +static nni_tran wss_tran = { + .tran_version = NNI_TRANSPORT_VERSION, + .tran_scheme = "wss", + .tran_dialer = &ws_dialer_ops, + .tran_listener = &ws_listener_ops, + .tran_pipe = &ws_pipe_ops, + .tran_init = wstran_init, + .tran_fini = wstran_fini, +}; + +static nni_tran wss4_tran = { + .tran_version = NNI_TRANSPORT_VERSION, + .tran_scheme = "wss4", + .tran_dialer = &ws_dialer_ops, + .tran_listener = &ws_listener_ops, + .tran_pipe = &ws_pipe_ops, + .tran_init = wstran_init, + .tran_fini = wstran_fini, +}; + +static nni_tran wss6_tran = { + .tran_version = NNI_TRANSPORT_VERSION, + .tran_scheme = "wss6", + .tran_dialer = &ws_dialer_ops, + .tran_listener = &ws_listener_ops, + .tran_pipe = &ws_pipe_ops, + .tran_init = wstran_init, + .tran_fini = wstran_fini, +}; + +int +nng_wss_register(void) +{ + int rv; + if (((rv = nni_tran_register(&wss_tran)) != 0) || + ((rv = nni_tran_register(&wss4_tran)) != 0) || + ((rv = nni_tran_register(&wss6_tran)) != 0)) { + return (rv); + } + + return (0); +} + +#else + +int +nng_wss_register(void) +{ + return (0); +} + +#endif // NNG_TRANSPORT_WSS diff --git a/src/sp/transport/ws/ws_test.c b/src/sp/transport/ws/ws_test.c new file mode 100644 index 00000000..7cbcd9d7 --- /dev/null +++ b/src/sp/transport/ws/ws_test.c @@ -0,0 +1,181 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// Copyright 2018 Cody Piersall +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include + +static void +test_ws_url_path_filters(void) +{ + nng_socket s1; + nng_socket s2; + char addr[NNG_MAXADDRLEN]; + + NUTS_OPEN(s1); + NUTS_OPEN(s2); + + nuts_scratch_addr("ws", sizeof(addr), addr); + NUTS_PASS(nng_listen(s1, addr, NULL, 0)); + + // Now try we just remove the last character for now. + // This will make the path different. + addr[strlen(addr) - 1] = '\0'; + NUTS_FAIL(nng_dial(s2, addr, NULL, 0), NNG_ECONNREFUSED); + + NUTS_CLOSE(s1); + NUTS_CLOSE(s2); +} + +static void +test_wild_card_port(void) +{ + nng_socket s1; + nng_socket s2; + nng_socket s3; + nng_socket s4; + nng_socket s5; + nng_socket s6; + + nng_listener l1; + nng_listener l2; + nng_listener l3; + int port1; + int port2; + int port3; + char ws_url[128]; + NUTS_OPEN(s1); + NUTS_OPEN(s2); + NUTS_OPEN(s3); + NUTS_OPEN(s4); + NUTS_OPEN(s5); + NUTS_OPEN(s6); + NUTS_PASS(nng_listen(s1, "ws://127.0.0.1:0/one", &l1, 0)); + NUTS_PASS( + nng_listener_get_int(l1, NNG_OPT_TCP_BOUND_PORT, &port1)); + NUTS_TRUE(port1 != 0); + snprintf(ws_url, sizeof(ws_url), "ws4://127.0.0.1:%d/two", port1); + NUTS_PASS(nng_listen(s2, ws_url, &l2, 0)); + NUTS_PASS( + nng_listener_get_int(l2, NNG_OPT_TCP_BOUND_PORT, &port2)); + NUTS_TRUE(port1 != 0); + NUTS_TRUE(port1 == port2); + // Now try a different wild card port. + NUTS_PASS(nng_listen(s3, "ws4://127.0.0.1:0/three", &l3, 0)); + NUTS_PASS( + nng_listener_get_int(l3, NNG_OPT_TCP_BOUND_PORT, &port3)); + NUTS_TRUE(port3 != 0); + NUTS_TRUE(port3 != port1); + + // Let's make sure can dial to each. + snprintf(ws_url, sizeof(ws_url), "ws://127.0.0.1:%d/one", port1); + NUTS_PASS(nng_dial(s4, ws_url, NULL, 0)); + snprintf(ws_url, sizeof(ws_url), "ws://127.0.0.1:%d/two", port2); + NUTS_PASS(nng_dial(s6, ws_url, NULL, 0)); + snprintf(ws_url, sizeof(ws_url), "ws://127.0.0.1:%d/three", port3); + NUTS_PASS(nng_dial(s6, ws_url, NULL, 0)); + + NUTS_CLOSE(s1); + NUTS_CLOSE(s2); + NUTS_CLOSE(s3); + NUTS_CLOSE(s4); + NUTS_CLOSE(s5); + NUTS_CLOSE(s6); +} + +static void +test_wild_card_host(void) +{ + nng_socket s1; + nng_socket s2; + char addr[NNG_MAXADDRLEN]; + uint16_t port; + + NUTS_OPEN(s1); + NUTS_OPEN(s2); + + port = nuts_next_port(); + + // we use ws4 to ensure 127.0.0.1 binding + snprintf(addr, sizeof(addr), "ws4://*:%u/test", port); + NUTS_PASS(nng_listen(s1, addr, NULL, 0)); + nng_msleep(100); + + snprintf(addr, sizeof(addr), "ws://127.0.0.1:%u/test", port); + NUTS_PASS(nng_dial(s2, addr, NULL, 0)); + + NUTS_CLOSE(s1); + NUTS_CLOSE(s2); +} + +static void +test_empty_host(void) +{ + nng_socket s1; + nng_socket s2; + char addr[NNG_MAXADDRLEN]; + uint16_t port; + + NUTS_OPEN(s1); + NUTS_OPEN(s2); + + port = nuts_next_port(); + + // we use ws4 to ensure 127.0.0.1 binding + snprintf(addr, sizeof(addr), "ws4://:%u/test", port); + NUTS_PASS(nng_listen(s1, addr, NULL, 0)); + nng_msleep(100); + + snprintf(addr, sizeof(addr), "ws://127.0.0.1:%u/test", port); + NUTS_PASS(nng_dial(s2, addr, NULL, 0)); + + NUTS_CLOSE(s1); + NUTS_CLOSE(s2); +} + +void +test_ws_recv_max(void) +{ + char msg[256]; + char buf[256]; + nng_socket s0; + nng_socket s1; + nng_listener l; + size_t sz; + char *addr; + + NUTS_ADDR(addr, "ws"); + NUTS_OPEN(s0); + NUTS_PASS(nng_socket_set_ms(s0, NNG_OPT_RECVTIMEO, 100)); + NUTS_PASS(nng_socket_set_size(s0, NNG_OPT_RECVMAXSZ, 200)); + NUTS_PASS(nng_listener_create(&l, s0, addr)); + NUTS_PASS(nng_socket_get_size(s0, NNG_OPT_RECVMAXSZ, &sz)); + NUTS_TRUE(sz == 200); + NUTS_PASS(nng_listener_set_size(l, NNG_OPT_RECVMAXSZ, 100)); + NUTS_PASS(nng_listener_start(l, 0)); + + NUTS_OPEN(s1); + NUTS_PASS(nng_dial(s1, addr, NULL, 0)); + NUTS_PASS(nng_send(s1, msg, 95, 0)); + NUTS_PASS(nng_socket_set_ms(s1, NNG_OPT_SENDTIMEO, 100)); + NUTS_PASS(nng_recv(s0, buf, &sz, 0)); + NUTS_TRUE(sz == 95); + NUTS_PASS(nng_send(s1, msg, 150, 0)); + NUTS_FAIL(nng_recv(s0, buf, &sz, 0), NNG_ETIMEDOUT); + NUTS_CLOSE(s0); + NUTS_CLOSE(s1); +} + +TEST_LIST = { + { "ws url path filters", test_ws_url_path_filters }, + { "ws wild card port", test_wild_card_port }, + { "ws wild card host", test_wild_card_host }, + { "ws empty host", test_empty_host }, + { "ws recv max", test_ws_recv_max }, + { NULL, NULL }, +}; \ No newline at end of file diff --git a/src/sp/transport/zerotier/CMakeLists.txt b/src/sp/transport/zerotier/CMakeLists.txt new file mode 100644 index 00000000..903b7f56 --- /dev/null +++ b/src/sp/transport/zerotier/CMakeLists.txt @@ -0,0 +1,37 @@ +# +# Copyright 2020 Staysail Systems, Inc. +# Copyright 2018 Capitar IT Group BV +# +# This software is supplied under the terms of the MIT License, a +# copy of which should be located in the distribution where this +# file was obtained (LICENSE.txt). A copy of the license may also be +# found online at https://opensource.org/licenses/MIT. +# + +# ZeroTier protocol + +option (NNG_TRANSPORT_ZEROTIER "Enable ZeroTier transport (requires libzerotiercore)." OFF) +mark_as_advanced(NNG_TRANSPORT_ZEROTIER) + +nng_directory(zerotier) + +if (NNG_TRANSPORT_ZEROTIER) + + # NB: As we wind up linking libzerotiercore.a into the application, + # this means that your application will *also* need to either be licensed + # under the GPLv3, or you will need to have a commercial license from + # ZeroTier permitting its use elsewhere. + + message(WARNING " + ************************************************************ + Linking against zerotiercore changes license terms. + Consult a lawyer and the license files for details. + ************************************************************") + + find_package(zerotiercore REQUIRED) + + nng_link_libraries(zerotiercore::zerotiercore) + nng_defines(NNG_TRANSPORT_ZEROTIER) + nng_sources(zerotier.c zthash.c) + nng_headers(nng/transport/zerotier/zerotier.h) +endif() diff --git a/src/sp/transport/zerotier/zerotier.c b/src/sp/transport/zerotier/zerotier.c new file mode 100644 index 00000000..896add29 --- /dev/null +++ b/src/sp/transport/zerotier/zerotier.c @@ -0,0 +1,3241 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// Copyright 2018 Capitar IT Group BV +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include +#include +#include +#include + +#include "core/nng_impl.h" +#include "zthash.h" + +#include "nng/transport/zerotier/zerotier.h" + +#include + +// ZeroTier Transport. This sits on the ZeroTier L2 network, which itself +// is implemented on top of UDP. This requires the 3rd party +// libzerotiercore library (which is GPLv3!) and platform specific UDP +// functionality to be built in. Note that care must be taken to link +// dynamically if one wishes to avoid making your entire application GPL3. +// (Alternatively ZeroTier offers commercial licenses which may prevent +// this particular problem.) This implementation does not make use of +// certain advanced capabilities in ZeroTier such as more sophisticated +// route management and TCP fallback. You need to have connectivity +// to the Internet to use this. (Or at least to your Planetary root.) +// +// Because ZeroTier takes a while to establish connectivity, it is even +// more important that applications using the ZeroTier transport not +// assume that a connection will be immediately available. It can take +// quite a few seconds for peer-to-peer connectivity to be established. +// +// The ZeroTier transport was funded by Capitar IT Group, BV. +// +// This transport is highly experimental. + +// ZeroTier and UDP are connectionless, but nng is designed around +// connection oriented paradigms. An "unreliable" connection is created +// on top using our own network protocol. The details of this are +// documented in the RFC. + +// Every participant has an "address", which is a 64-bit value constructed +// using the ZT node number in the upper 40-bits, and a 24-bit port number +// in the lower bits. We elect to operate primarily on these addresses, +// but the wire protocol relies on just conveying the 24-bit port along +// with the MAC address (from which the ZT node number can be derived, +// given the network ID.) + +typedef struct zt_pipe zt_pipe; +typedef struct zt_ep zt_ep; +typedef struct zt_node zt_node; +typedef struct zt_frag zt_frag; +typedef struct zt_fraglist zt_fraglist; + +// Port numbers are stored as 24-bit values in network byte order. +#define ZT_GET24(ptr, v) \ + v = (((uint32_t)((uint8_t)(ptr)[0])) << 16) + \ + (((uint32_t)((uint8_t)(ptr)[1])) << 8) + \ + (((uint32_t)(uint8_t)(ptr)[2])) + +#define ZT_PUT24(ptr, u) \ + do { \ + (ptr)[0] = (uint8_t)(((uint32_t)(u)) >> 16); \ + (ptr)[1] = (uint8_t)(((uint32_t)(u)) >> 8); \ + (ptr)[2] = (uint8_t)((uint32_t)(u)); \ + } while (0) + +static const uint16_t zt_ethertype = 0x901; +static const uint8_t zt_version = 0x01; +static const uint32_t zt_ephemeral = 0x800000u; // start of ephemeral ports +static const uint32_t zt_max_port = 0xffffffu; // largest port +static const uint32_t zt_port_mask = 0xffffffu; // mask of valid ports +static const uint32_t zt_port_shift = 24; +static const int zt_conn_tries = 240; // max connect attempts +static const nng_duration zt_conn_time = 500; // between attempts (msec) +static const int zt_ping_tries = 10; // max keepalive attempts +static const nng_duration zt_ping_time = 60000; // keepalive time (msec) + +// These are compile time tunables for now. +enum zt_tunables { + zt_listenq = 128, // backlog queue length + zt_listen_expire = 10000, // maximum time in backlog (msec) + zt_rcv_bufsize = 4096, // max UDP recv + zt_udp_sendq = 16, // outgoing UDP queue length + zt_recvq = 2, // max pending recv (per pipe) + zt_recv_stale = 1000, // frags older than are stale (msec) +}; + +enum zt_op_codes { + zt_op_data = 0x00, // data, final fragment + zt_op_conn_req = 0x10, // connect request + zt_op_conn_ack = 0x12, // connect accepted + zt_op_disc_req = 0x20, // disconnect request (no ack) + zt_op_ping = 0x30, // ping request + zt_op_pong = 0x32, // ping response + zt_op_error = 0x40, // error response +}; + +enum zt_offsets { + zt_offset_op = 0x00, + zt_offset_flags = 0x01, + zt_offset_version = 0x02, // protocol version number (2 bytes) + zt_offset_zero1 = 0x04, // reserved, must be zero (1 byte) + zt_offset_dst_port = 0x05, // destination port (3 bytes) + zt_offset_zero2 = 0x08, // reserved, must be zero (1 byte) + zt_offset_src_port = 0x09, // source port number (3 bytes) + zt_offset_creq_proto = 0x0C, // SP protocol number (2 bytes) + zt_offset_cack_proto = 0x0C, // SP protocol number (2 bytes) + zt_offset_err_code = 0x0C, // error code (1 byte) + zt_offset_err_msg = 0x0D, // error message (string) + zt_offset_data_id = 0x0C, // message ID (2 bytes) + zt_offset_data_fragsz = 0x0E, // fragment size + zt_offset_data_frag = 0x10, // fragment number, first is 1 (2 bytes) + zt_offset_data_nfrag = 0x12, // total fragments (2 bytes) + zt_offset_data_data = 0x14, // user payload + zt_size_headers = 0x0C, // size of headers + zt_size_conn_req = 0x0E, // size of conn_req (connect request) + zt_size_conn_ack = 0x0E, // size of conn_ack (connect reply) + zt_size_disc_req = 0x0C, // size of disc_req (disconnect) + zt_size_ping = 0x0C, // size of ping request + zt_size_pong = 0x0C, // size of ping reply + zt_size_data = 0x14, // size of data message (w/o payload) +}; + +enum zt_errors { + zt_err_refused = 0x01, // Connection refused + zt_err_notconn = 0x02, // Connection does not exit + zt_err_wrongsp = 0x03, // SP protocol mismatch + zt_err_proto = 0x04, // Other protocol error + zt_err_msgsize = 0x05, // Message to large + zt_err_unknown = 0x06, // Other errors +}; + +// This node structure is wrapped around the ZT_node; this allows us to +// have multiple endpoints referencing the same ZT_node, but also to +// support different nodes (identities) based on different home dirs. +// This means we need to stick these on a global linked list, manage +// them with a reference count, and uniquely identify them using the +// homedir. +struct zt_node { + char zn_path[NNG_MAXADDRLEN]; // ought to be sufficient + nni_file_lockh *zn_flock; + ZT_Node * zn_znode; + uint64_t zn_self; + nni_list_node zn_link; + bool zn_closed; + nni_plat_udp * zn_udp4; + nni_plat_udp * zn_udp6; + nni_list zn_eplist; + nni_list zn_plist; + zt_hash * zn_ports; + zt_hash * zn_eps; + zt_hash * zn_lpipes; + zt_hash * zn_rpipes; + nni_aio * zn_rcv4_aio; + uint8_t * zn_rcv4_buf; + nng_sockaddr zn_rcv4_addr; + nni_aio * zn_rcv6_aio; + uint8_t * zn_rcv6_buf; + nng_sockaddr zn_rcv6_addr; + nni_thr zn_bgthr; + int64_t zn_bgtime; + nni_cv zn_bgcv; + nni_cv zn_snd6_cv; +}; + +// The fragment list is used to keep track of incoming received +// fragments for reassembly into a complete message. +struct zt_fraglist { + nni_time fl_time; // time first frag was received + uint32_t fl_msgid; // message id + int fl_ready; // we have all messages + size_t fl_fragsz; + unsigned int fl_nfrags; + uint8_t * fl_missing; + size_t fl_missingsz; + nni_msg * fl_msg; +}; + +struct zt_pipe { + nni_list_node zp_link; + zt_node * zp_ztn; + nni_pipe * zp_npipe; + uint64_t zp_nwid; + uint64_t zp_laddr; + uint64_t zp_raddr; + uint16_t zp_peer; + uint16_t zp_proto; + uint16_t zp_next_msgid; + size_t zp_rcvmax; + size_t zp_mtu; + nni_aio * zp_user_rxaio; + nni_time zp_last_recv; + zt_fraglist zp_recvq[zt_recvq]; + int zp_ping_try; + int zp_ping_tries; + bool zp_closed; + nni_duration zp_ping_time; + nni_aio * zp_ping_aio; + uint8_t * zp_send_buf; + nni_atomic_flag zp_reaped; + nni_reap_item zp_reap; +}; + +typedef struct zt_creq zt_creq; +struct zt_creq { + uint64_t cr_expire; + uint64_t cr_raddr; + uint16_t cr_proto; +}; + +struct zt_ep { + nni_list_node ze_link; + char ze_home[NNG_MAXADDRLEN]; // should be enough + zt_node * ze_ztn; + uint64_t ze_nwid; + bool ze_running; + uint64_t ze_raddr; // remote node address + uint64_t ze_laddr; // local node address + uint16_t ze_proto; + size_t ze_rcvmax; + nni_aio * ze_aio; + nni_aio * ze_creq_aio; + bool ze_creq_active; + int ze_creq_try; + nni_list ze_aios; + int ze_mtu; + int ze_ping_tries; + nni_duration ze_ping_time; + nni_duration ze_conn_time; + int ze_conn_tries; + + // Incoming connection requests (server only). We only + // only have "accepted" requests -- that is we won't have an + // established connection/pipe unless the application calls + // accept. Since the "application" is our library, that should + // be pretty much as fast we can run. + zt_creq ze_creqs[zt_listenq]; + int ze_creq_head; + int ze_creq_tail; + nni_dialer * ze_ndialer; + nni_listener *ze_nlistener; +}; + +// Locking strategy. At present the ZeroTier core is not reentrant or fully +// threadsafe. (We expect this will be fixed.) Furthermore, there are +// some significant challenges in dealing with locks associated with the +// callbacks, etc. So we take a big-hammer approach, and just use a single +// global lock for everything. We hold this lock when calling into the +// ZeroTier framework. Since ZeroTier has no independent threads, that +// means that it will always hold this lock in its core, and the lock will +// also be held automatically in any of our callbacks. We never hold any +// other locks across ZeroTier core calls. We may not acquire the global +// lock in callbacks (they will already have it held). Any other locks +// can be acquired as long as they are not held during calls into ZeroTier. +// +// This will have a detrimental impact on performance, but to be completely +// honest we don't think anyone will be using the ZeroTier transport in +// performance critical applications; scalability may become a factor for +// large servers sitting in a ZeroTier hub situation. (Then again, since +// only the zerotier processing is single threaded, it may not +// be that much of a bottleneck -- really depends on how expensive these +// operations are. We can use lockstat or other lock-hotness tools to +// check for this later.) + +static nni_mtx zt_lk; +static nni_list zt_nodes; + +static void zt_ep_send_conn_req(zt_ep *); +static void zt_ep_conn_req_cb(void *); +static void zt_ep_doaccept(zt_ep *); +static void zt_pipe_dorecv(zt_pipe *); +static int zt_pipe_alloc(zt_pipe **, zt_ep *, uint64_t, uint64_t, bool); +static void zt_pipe_ping_cb(void *); +static void zt_fraglist_clear(zt_fraglist *); +static void zt_fraglist_free(zt_fraglist *); +static void zt_virtual_recv(ZT_Node *, void *, void *, uint64_t, void **, + uint64_t, uint64_t, unsigned int, unsigned int, const void *, + unsigned int); +static void zt_pipe_start_ping(zt_pipe *); + +static int64_t +zt_now(void) +{ + // We return msec + return ((int64_t) nni_clock()); +} + +static void +zt_bgthr(void *arg) +{ + zt_node *ztn = arg; + int64_t now; + + nni_mtx_lock(&zt_lk); + for (;;) { + now = zt_now(); + + if (ztn->zn_closed) { + break; + } + + if (now < ztn->zn_bgtime) { + nni_cv_until(&ztn->zn_bgcv, (nni_time) ztn->zn_bgtime); + continue; + } + + ztn->zn_bgtime = 0; + ZT_Node_processBackgroundTasks(ztn->zn_znode, NULL, now, &now); + + ztn->zn_bgtime = now; + } + nni_mtx_unlock(&zt_lk); +} + +static void +zt_node_resched(zt_node *ztn, int64_t msec) +{ + if (msec > ztn->zn_bgtime && ztn->zn_bgtime != 0) { + return; + } + ztn->zn_bgtime = msec; + nni_cv_wake1(&ztn->zn_bgcv); +} + +static void +zt_node_rcv4_cb(void *arg) +{ + zt_node * ztn = arg; + nni_aio * aio = ztn->zn_rcv4_aio; + struct sockaddr_storage sa; + struct sockaddr_in * sin; + nng_sockaddr_in * nsin; + int64_t now; + + if (nni_aio_result(aio) != 0) { + // Outside of memory exhaustion, we can't really think + // of any reason for this to legitimately fail. + // Arguably we should inject a fallback delay, but for + // now we just carry on. + return; + } + + memset(&sa, 0, sizeof(sa)); + sin = (void *) &sa; + nsin = &ztn->zn_rcv4_addr.s_in; + sin->sin_family = AF_INET; + sin->sin_port = nsin->sa_port; + sin->sin_addr.s_addr = nsin->sa_addr; + + nni_mtx_lock(&zt_lk); + now = zt_now(); + + // We are not going to perform any validation of the data; we + // just pass this straight into the ZeroTier core. + // XXX: CHECK THIS, if it fails then we have a fatal error with + // the znode, and have to shut everything down. + ZT_Node_processWirePacket(ztn->zn_znode, NULL, now, 0, (void *) &sa, + ztn->zn_rcv4_buf, nni_aio_count(aio), &now); + + // Schedule background work + zt_node_resched(ztn, now); + + // Schedule another receive. + if (ztn->zn_udp4 != NULL) { + nni_iov iov; + iov.iov_buf = ztn->zn_rcv4_buf; + iov.iov_len = zt_rcv_bufsize; + nni_aio_set_iov(aio, 1, &iov); + + nni_aio_set_input(aio, 0, &ztn->zn_rcv4_addr); + + nni_plat_udp_recv(ztn->zn_udp4, aio); + } + nni_mtx_unlock(&zt_lk); +} + +static void +zt_node_rcv6_cb(void *arg) +{ + zt_node * ztn = arg; + nni_aio * aio = ztn->zn_rcv6_aio; + struct sockaddr_storage sa; + struct sockaddr_in6 * sin6; + struct nng_sockaddr_in6 *nsin6; + int64_t now; + + if (nni_aio_result(aio) != 0) { + // Outside of memory exhaustion, we can't really think + // of any reason for this to legitimately fail. + // Arguably we should inject a fallback delay, but for + // now we just carry on. + return; + } + + memset(&sa, 0, sizeof(sa)); + sin6 = (void *) &sa; + nsin6 = &ztn->zn_rcv6_addr.s_in6; + sin6->sin6_family = AF_INET6; + sin6->sin6_port = nsin6->sa_port; + memcpy(&sin6->sin6_addr, nsin6->sa_addr, 16); + + nni_mtx_lock(&zt_lk); + now = (uint64_t) zt_now(); // msec + + // We are not going to perform any validation of the data; we + // just pass this straight into the ZeroTier core. + ZT_Node_processWirePacket(ztn->zn_znode, NULL, now, 0, (void *) &sa, + ztn->zn_rcv6_buf, nni_aio_count(aio), &now); + + // Schedule background work + zt_node_resched(ztn, now); + + // Schedule another receive. + if (ztn->zn_udp6 != NULL) { + nni_iov iov; + iov.iov_buf = ztn->zn_rcv6_buf; + iov.iov_len = zt_rcv_bufsize; + nni_aio_set_iov(aio, 1, &iov); + nni_aio_set_input(aio, 0, &ztn->zn_rcv6_addr); + nni_plat_udp_recv(ztn->zn_udp6, aio); + } + nni_mtx_unlock(&zt_lk); +} + +static uint64_t +zt_mac_to_node(uint64_t mac, uint64_t nwid) +{ + uint64_t node; + // This extracts a node address from a mac address. The + // network ID is mixed in, and has to be extricated. We + // the node ID is located in the lower 40 bits, and scrambled + // against the nwid. + node = mac & 0xffffffffffull; + node ^= ((nwid >> 8) & 0xff) << 32; + node ^= ((nwid >> 16) & 0xff) << 24; + node ^= ((nwid >> 24) & 0xff) << 16; + node ^= ((nwid >> 32) & 0xff) << 8; + node ^= (nwid >> 40) & 0xff; + return (node); +} + +static uint64_t +zt_node_to_mac(uint64_t node, uint64_t nwid) +{ + uint64_t mac; + // We use LSB of network ID, and make sure that we clear + // multicast and set local administration -- this is the first + // octet of the 48 bit mac address. We also avoid 0x52, which + // is known to be used in KVM, libvirt, etc. + mac = ((uint8_t)(nwid & 0xfe) | 0x02); + if (mac == 0x52) { + mac = 0x32; + } + mac <<= 40; + mac |= node; + // The rest of the network ID is XOR'd in, in reverse byte + // order. + mac ^= ((nwid >> 8) & 0xff) << 32; + mac ^= ((nwid >> 16) & 0xff) << 24; + mac ^= ((nwid >> 24) & 0xff) << 16; + mac ^= ((nwid >> 32) & 0xff) << 8; + mac ^= (nwid >> 40) & 0xff; + return (mac); +} + +static int +zt_result(enum ZT_ResultCode rv) +{ + switch (rv) { + case ZT_RESULT_OK: + return (0); + case ZT_RESULT_OK_IGNORED: + return (0); + case ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY: + return (NNG_ENOMEM); + case ZT_RESULT_FATAL_ERROR_DATA_STORE_FAILED: + return (NNG_EPERM); + case ZT_RESULT_FATAL_ERROR_INTERNAL: + return (NNG_EINTERNAL); + case ZT_RESULT_ERROR_NETWORK_NOT_FOUND: + return (NNG_EADDRINVAL); + case ZT_RESULT_ERROR_UNSUPPORTED_OPERATION: + return (NNG_ENOTSUP); + case ZT_RESULT_ERROR_BAD_PARAMETER: + return (NNG_EINVAL); + default: + return (NNG_ETRANERR + (int) rv); + } +} + +// ZeroTier Node API callbacks +static int +zt_virtual_config(ZT_Node *node, void *userptr, void *thr, uint64_t nwid, + void **netptr, enum ZT_VirtualNetworkConfigOperation op, + const ZT_VirtualNetworkConfig *config) +{ + zt_node *ztn = userptr; + zt_ep * ep; + + NNI_ARG_UNUSED(thr); + NNI_ARG_UNUSED(netptr); + + NNI_ASSERT(node == ztn->zn_znode); + + // Maybe we don't have to create taps or anything like that. + // We do get our mac and MTUs from this, so there's that. + switch (op) { + case ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_UP: + case ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_CONFIG_UPDATE: + + // We only really care about changes to the MTU. From + // an API perspective the MAC could change, but that + // cannot really happen because the node identity and + // the nwid are fixed. + NNI_LIST_FOREACH (&ztn->zn_eplist, ep) { + NNI_ASSERT(nwid == config->nwid); + if (ep->ze_nwid != config->nwid) { + continue; + } + ep->ze_mtu = config->mtu; + } + break; + case ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY: + case ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DOWN: + // XXX: tear down endpoints? + default: + break; + } + return (0); +} + +// zt_send modifies the start of the supplied buffer to update the +// message headers with protocol specific details (version, port numbers, +// etc.) and then sends it over the virtual network. +static void +zt_send(zt_node *ztn, uint64_t nwid, uint8_t op, uint64_t raddr, + uint64_t laddr, uint8_t *data, size_t len) +{ + uint64_t srcmac = zt_node_to_mac(laddr >> 24, nwid); + uint64_t dstmac = zt_node_to_mac(raddr >> 24, nwid); + int64_t now = zt_now(); + + NNI_ASSERT(len >= zt_size_headers); + data[zt_offset_op] = op; + data[zt_offset_flags] = 0; + data[zt_offset_zero1] = 0; + data[zt_offset_zero2] = 0; + NNI_PUT16(data + zt_offset_version, zt_version); + ZT_PUT24(data + zt_offset_dst_port, raddr & zt_port_mask); + ZT_PUT24(data + zt_offset_src_port, laddr & zt_port_mask); + + (void) ZT_Node_processVirtualNetworkFrame(ztn->zn_znode, NULL, now, + nwid, srcmac, dstmac, zt_ethertype, 0, data, len, &now); + + zt_node_resched(ztn, now); +} + +static void +zt_send_err(zt_node *ztn, uint64_t nwid, uint64_t raddr, uint64_t laddr, + uint8_t err, const char *msg) +{ + uint8_t data[128]; + + NNI_ASSERT((strlen(msg) + zt_offset_err_msg) < sizeof(data)); + + data[zt_offset_err_code] = err; + nni_strlcpy((char *) data + zt_offset_err_msg, msg, + sizeof(data) - zt_offset_err_msg); + + zt_send(ztn, nwid, zt_op_error, raddr, laddr, data, + strlen(msg) + zt_offset_err_msg); +} + +static void +zt_pipe_send_err(zt_pipe *p, uint8_t err, const char *msg) +{ + zt_send_err(p->zp_ztn, p->zp_nwid, p->zp_raddr, p->zp_laddr, err, msg); +} + +static void +zt_pipe_send_disc_req(zt_pipe *p) +{ + uint8_t data[zt_size_disc_req]; + + zt_send(p->zp_ztn, p->zp_nwid, zt_op_disc_req, p->zp_raddr, + p->zp_laddr, data, sizeof(data)); +} + +static void +zt_pipe_send_ping(zt_pipe *p) +{ + uint8_t data[zt_size_ping]; + + zt_send(p->zp_ztn, p->zp_nwid, zt_op_ping, p->zp_raddr, p->zp_laddr, + data, sizeof(data)); +} + +static void +zt_pipe_send_pong(zt_pipe *p) +{ + uint8_t data[zt_size_ping]; + + zt_send(p->zp_ztn, p->zp_nwid, zt_op_pong, p->zp_raddr, p->zp_laddr, + data, sizeof(data)); +} + +static void +zt_pipe_send_conn_ack(zt_pipe *p) +{ + uint8_t data[zt_size_conn_ack]; + + NNI_PUT16(data + zt_offset_cack_proto, p->zp_proto); + zt_send(p->zp_ztn, p->zp_nwid, zt_op_conn_ack, p->zp_raddr, + p->zp_laddr, data, sizeof(data)); +} + +static void +zt_ep_send_conn_req(zt_ep *ep) +{ + uint8_t data[zt_size_conn_req]; + + NNI_PUT16(data + zt_offset_creq_proto, ep->ze_proto); + zt_send(ep->ze_ztn, ep->ze_nwid, zt_op_conn_req, ep->ze_raddr, + ep->ze_laddr, data, sizeof(data)); +} + +static void +zt_ep_recv_conn_ack(zt_ep *ep, uint64_t raddr, const uint8_t *data, size_t len) +{ + zt_node *ztn = ep->ze_ztn; + nni_aio *aio = ep->ze_creq_aio; + zt_pipe *p; + int rv; + + if (ep->ze_ndialer == NULL) { + zt_send_err(ztn, ep->ze_nwid, raddr, ep->ze_laddr, + zt_err_proto, "Inappropriate operation"); + return; + } + + if (len != zt_size_conn_ack) { + zt_send_err(ztn, ep->ze_nwid, raddr, ep->ze_laddr, + zt_err_proto, "Bad message length"); + return; + } + + if (ep->ze_creq_try == 0) { + return; + } + + // Do we already have a matching pipe? If so, we can discard + // the operation. This should not happen, since we normally, + // deregister the endpoint when we create the pipe. + if ((zt_hash_find(ztn->zn_lpipes, ep->ze_laddr, (void **) &p)) == 0) { + return; + } + + if ((rv = zt_pipe_alloc(&p, ep, raddr, ep->ze_laddr, false)) != 0) { + // We couldn't create the pipe, just drop it. + nni_aio_finish_error(aio, rv); + return; + } + NNI_GET16(data + zt_offset_cack_proto, p->zp_peer); + + // Reset the address of the endpoint, so that the next call to + // ep_connect will bind a new one -- we are using this one for the + // pipe. + zt_hash_remove(ztn->zn_eps, ep->ze_laddr); + ep->ze_laddr = 0; + + nni_aio_set_output(aio, 0, p); + nni_aio_finish(aio, 0, 0); +} + +static void +zt_ep_recv_conn_req(zt_ep *ep, uint64_t raddr, const uint8_t *data, size_t len) +{ + zt_node *ztn = ep->ze_ztn; + zt_pipe *p; + int i; + + if (ep->ze_nlistener == NULL) { + zt_send_err(ztn, ep->ze_nwid, raddr, ep->ze_laddr, + zt_err_proto, "Inappropriate operation"); + return; + } + if (len != zt_size_conn_req) { + zt_send_err(ztn, ep->ze_nwid, raddr, ep->ze_laddr, + zt_err_proto, "Bad message length"); + return; + } + + // If we already have created a pipe for this connection + // then just reply the conn ack. + if ((zt_hash_find(ztn->zn_rpipes, raddr, (void **) &p)) == 0) { + zt_pipe_send_conn_ack(p); + return; + } + + // We may already have a connection request queued (if this was + // a resend for example); if that's the case we just ignore + // this one. + for (i = ep->ze_creq_tail; i != ep->ze_creq_head; i++) { + if (ep->ze_creqs[i % zt_listenq].cr_raddr == raddr) { + return; + } + } + // We may already have filled our listenq, in which case we just drop. + if ((ep->ze_creq_tail + zt_listenq) == ep->ze_creq_head) { + // We have taken as many as we can, so just drop it. + return; + } + + // Record the connection request, and then process any + // pending acceptors. + i = ep->ze_creq_head % zt_listenq; + + NNI_GET16(data + zt_offset_creq_proto, ep->ze_creqs[i].cr_proto); + ep->ze_creqs[i].cr_raddr = raddr; + ep->ze_creqs[i].cr_expire = nni_clock() + zt_listen_expire; + ep->ze_creq_head++; + + zt_ep_doaccept(ep); +} + +static void +zt_ep_recv_error(zt_ep *ep, const uint8_t *data, size_t len) +{ + int code; + + // Most of the time we don't care about errors. The exception here + // is that when we have an outstanding CON_REQ, we would like to + // process that appropriately. + + if (ep->ze_ndialer == NULL) { + // Not a dialer. Drop it. + return; + } + + if (len < zt_offset_err_msg) { + // Malformed error frame. + return; + } + + code = data[zt_offset_err_code]; + switch (code) { + case zt_err_refused: + code = NNG_ECONNREFUSED; + break; + case zt_err_notconn: + code = NNG_ECLOSED; + break; + case zt_err_wrongsp: + code = NNG_EPROTO; + break; + default: + code = NNG_ETRANERR; + break; + } + + if (ep->ze_creq_active) { + ep->ze_creq_try = 0; + ep->ze_creq_active = 0; + nni_aio_finish_error(ep->ze_creq_aio, code); + } +} + +static void +zt_ep_virtual_recv( + zt_ep *ep, uint8_t op, uint64_t raddr, const uint8_t *data, size_t len) +{ + // Only listeners should be receiving. Dialers receive on the pipe, + // rather than the endpoint. The only message that endpoints can + // receive are connection requests. + switch (op) { + case zt_op_conn_req: + zt_ep_recv_conn_req(ep, raddr, data, len); + return; + case zt_op_conn_ack: + zt_ep_recv_conn_ack(ep, raddr, data, len); + return; + case zt_op_error: + zt_ep_recv_error(ep, data, len); + return; + default: + zt_send_err(ep->ze_ztn, ep->ze_nwid, raddr, ep->ze_laddr, + zt_err_proto, "Bad operation"); + return; + } +} + +static void +zt_pipe_close_err(zt_pipe *p, int err, uint8_t code, const char *msg) +{ + nni_aio *aio; + if ((aio = p->zp_user_rxaio) != NULL) { + p->zp_user_rxaio = NULL; + nni_aio_finish_error(aio, err); + } + nni_aio_close(p->zp_ping_aio); + p->zp_closed = true; + if (msg != NULL) { + zt_pipe_send_err(p, code, msg); + } +} + +static void +zt_pipe_recv_data(zt_pipe *p, const uint8_t *data, size_t len) +{ + uint16_t msgid; + uint16_t fragno; + uint16_t nfrags; + size_t fragsz; + zt_fraglist *fl; + int i; + int slot; + uint8_t bit; + uint8_t * body; + + if (len < zt_size_data) { + // Runt frame. Drop it and close pipe with a protocol error. + zt_pipe_close_err(p, NNG_EPROTO, zt_err_proto, "Runt frame"); + return; + } + + NNI_GET16(data + zt_offset_data_id, msgid); + NNI_GET16(data + zt_offset_data_fragsz, fragsz); + NNI_GET16(data + zt_offset_data_frag, fragno); + NNI_GET16(data + zt_offset_data_nfrag, nfrags); + len -= zt_offset_data_data; + data += zt_offset_data_data; + + // Check for cases where message size is clearly too large. Note + // that we only can catch the case where a message is larger by + // more than a fragment, since the final fragment may be shorter, + // and we won't know that until we receive it. + if ((p->zp_rcvmax > 0) && + ((nfrags * fragsz) >= (p->zp_rcvmax + fragsz))) { + // Discard, as the forwarder might be on the other side + // of a device. This is gentler than just shutting the pipe + // down. Sending a remote error might be polite, but since + // most peers will close the pipe on such an error, we + // simply silently discard it. + return; + } + + // We run the recv logic once, to clear stale fragment entries. + zt_pipe_dorecv(p); + + // Find a suitable fragment slot. + slot = -1; + for (i = 0; i < zt_recvq; i++) { + fl = &p->zp_recvq[i]; + // This was our message ID, we always use it. + if (msgid == fl->fl_msgid) { + slot = i; + break; + } + + if (slot < 0) { + slot = i; + } else if (fl->fl_time < p->zp_recvq[slot].fl_time) { + // This has an earlier expiration, so lets choose it. + slot = i; + } + } + + NNI_ASSERT(slot >= 0); + + fl = &p->zp_recvq[slot]; + if (fl->fl_msgid != msgid) { + // First fragment we've received for this message (but might + // not be first fragment for message!) + zt_fraglist_clear(fl); + + if (nni_msg_alloc(&fl->fl_msg, nfrags * fragsz) != 0) { + // Out of memory. We don't close the pipe, but + // just fail to receive the message. Bump a stat? + return; + } + + fl->fl_nfrags = nfrags; + fl->fl_fragsz = fragsz; + fl->fl_msgid = msgid; + fl->fl_time = nni_clock(); + + // Set the missing mask. + memset(fl->fl_missing, 0xff, nfrags / 8); + fl->fl_missing[nfrags / 8] |= ((1 << (nfrags % 8)) - 1); + } + if ((nfrags != fl->fl_nfrags) || (fragsz != fl->fl_fragsz) || + (fragno >= nfrags) || (fragsz == 0) || (nfrags == 0) || + ((fragno != (nfrags - 1)) && (len != fragsz))) { + // Protocol error, message parameters changed. + zt_pipe_close_err( + p, NNG_EPROTO, zt_err_proto, "Invalid message parameters"); + zt_fraglist_clear(fl); + return; + } + + bit = (uint8_t)(1 << (fragno % 8)); + if ((fl->fl_missing[fragno / 8] & bit) == 0) { + // We've already got this fragment, ignore it. We don't + // bother to check for changed data. + return; + } + + fl->fl_missing[fragno / 8] &= ~(bit); + body = nni_msg_body(fl->fl_msg); + body += fragno * fragsz; + memcpy(body, data, len); + if (fragno == (nfrags - 1)) { + // Last frag, maybe shorten the message. + nni_msg_chop(fl->fl_msg, (fragsz - len)); + if ((nni_msg_len(fl->fl_msg) > p->zp_rcvmax) && + (p->zp_rcvmax > 0)) { + // Strict enforcement of max recv. + zt_fraglist_clear(fl); + // Just discard the message. + return; + } + } + + for (i = 0; i < ((nfrags + 7) / 8); i++) { + if (fl->fl_missing[i]) { + return; + } + } + + // We got all fragments... try to send it up. + fl->fl_ready = 1; + zt_pipe_dorecv(p); +} + +static void +zt_pipe_recv_ping(zt_pipe *p, const uint8_t *data, size_t len) +{ + NNI_ARG_UNUSED(data); + + if (len != zt_size_ping) { + zt_pipe_send_err(p, zt_err_proto, "Incorrect ping size"); + return; + } + zt_pipe_send_pong(p); +} + +static void +zt_pipe_recv_pong(zt_pipe *p, const uint8_t *data, size_t len) +{ + NNI_ARG_UNUSED(data); + + if (len != zt_size_pong) { + zt_pipe_send_err(p, zt_err_proto, "Incorrect pong size"); + } +} + +static void +zt_pipe_recv_disc_req(zt_pipe *p, const uint8_t *data, size_t len) +{ + nni_aio *aio; + NNI_ARG_UNUSED(data); + NNI_ARG_UNUSED(len); + + // NB: lock held already. + // Don't bother to check the length, going to disconnect anyway. + if ((aio = p->zp_user_rxaio) != NULL) { + p->zp_user_rxaio = NULL; + p->zp_closed = true; + nni_aio_finish_error(aio, NNG_ECLOSED); + } +} + +static void +zt_pipe_recv_error(zt_pipe *p, const uint8_t *data, size_t len) +{ + nni_aio *aio; + NNI_ARG_UNUSED(data); + NNI_ARG_UNUSED(len); + + // Perhaps we should log an error message, but at the end of + // the day, the details are just not that interesting. + if ((aio = p->zp_user_rxaio) != NULL) { + p->zp_user_rxaio = NULL; + p->zp_closed = true; + nni_aio_finish_error(aio, NNG_ETRANERR); + } +} + +// This function is called when we have determined that a frame has +// arrived for a pipe. The remote and local addresses were both +// matched by the caller. +static void +zt_pipe_virtual_recv(zt_pipe *p, uint8_t op, const uint8_t *data, size_t len) +{ + // We got data, so update our recv time. + p->zp_last_recv = nni_clock(); + p->zp_ping_try = 0; + + switch (op) { + case zt_op_data: + zt_pipe_recv_data(p, data, len); + return; + case zt_op_disc_req: + zt_pipe_recv_disc_req(p, data, len); + return; + case zt_op_ping: + zt_pipe_recv_ping(p, data, len); + return; + case zt_op_pong: + zt_pipe_recv_pong(p, data, len); + return; + case zt_op_error: + zt_pipe_recv_error(p, data, len); + return; + case zt_op_conn_req: + zt_pipe_send_conn_ack(p); + return; + } +} + +// This function is called when a frame arrives on the +// *virtual* network. +static void +zt_virtual_recv(ZT_Node *node, void *userptr, void *thr, uint64_t nwid, + void **netptr, uint64_t srcmac, uint64_t dstmac, unsigned int ethertype, + unsigned int vlanid, const void *payload, unsigned int len) +{ + zt_node * ztn = userptr; + uint8_t op; + const uint8_t *data = payload; + uint16_t version; + uint32_t rport; + uint32_t lport; + zt_ep * ep; + zt_pipe * p; + uint64_t raddr; + uint64_t laddr; + + NNI_ARG_UNUSED(node); + NNI_ARG_UNUSED(thr); + NNI_ARG_UNUSED(netptr); + + if ((ethertype != zt_ethertype) || (len < zt_size_headers) || + (data[zt_offset_flags] != 0) || (data[zt_offset_zero1] != 0) || + (data[zt_offset_zero2] != 0)) { + return; + } + NNI_GET16(data + zt_offset_version, version); + if (version != zt_version) { + return; + } + if (vlanid != 0) { // for now we only use vlan 0. + return; + } + + op = data[zt_offset_op]; + + ZT_GET24(data + zt_offset_dst_port, lport); + ZT_GET24(data + zt_offset_src_port, rport); + + raddr = zt_mac_to_node(srcmac, nwid); + raddr <<= 24; + raddr |= rport; + + laddr = zt_mac_to_node(dstmac, nwid); + laddr <<= 24; + laddr |= lport; + + // NB: We are holding the zt_lock. + + // Look up a pipe, but also we use this chance to check that + // the source address matches what the pipe was established with. + // If the pipe does not match then we nak it. Note that pipes can + // appear on the znode twice (loopback), so we have to be careful + // to check the entire set of parameters, and to check for server + // vs. client pipes separately. + + // If its a local address match on a client pipe, process it. + if ((zt_hash_find(ztn->zn_lpipes, laddr, (void *) &p) == 0) && + (p->zp_nwid == nwid) && (p->zp_raddr == raddr)) { + zt_pipe_virtual_recv(p, op, data, len); + return; + } + + // If its a remote address match on a server pipe, process it. + if ((zt_hash_find(ztn->zn_rpipes, raddr, (void *) &p) == 0) && + (p->zp_nwid == nwid) && (p->zp_laddr == laddr)) { + zt_pipe_virtual_recv(p, op, data, len); + return; + } + + // No pipe, so look for an endpoint. + if ((zt_hash_find(ztn->zn_eps, laddr, (void **) &ep) == 0) && + (ep->ze_nwid == nwid)) { + // direct this to an endpoint. + zt_ep_virtual_recv(ep, op, raddr, data, len); + return; + } + + // We have a request for which we have no listener, and no + // pipe. For some of these we send back a NAK, but for others + // we just drop the frame. + switch (op) { + case zt_op_conn_req: + // No listener. Connection refused. + zt_send_err(ztn, nwid, raddr, laddr, zt_err_refused, + "Connection refused"); + return; + case zt_op_data: + case zt_op_ping: + case zt_op_conn_ack: + zt_send_err(ztn, nwid, raddr, laddr, zt_err_notconn, + "Connection not found"); + break; + case zt_op_error: + case zt_op_pong: + case zt_op_disc_req: + default: + // Just drop these. + break; + } +} + +static void +zt_event_cb(ZT_Node *node, void *userptr, void *thr, enum ZT_Event event, + const void *payload) +{ + NNI_ARG_UNUSED(node); + NNI_ARG_UNUSED(userptr); + NNI_ARG_UNUSED(thr); + NNI_ARG_UNUSED(payload); + + switch (event) { + case ZT_EVENT_ONLINE: // Connected to the virtual net. + case ZT_EVENT_UP: // Node initialized (may not be connected). + case ZT_EVENT_DOWN: // Teardown of the node. + case ZT_EVENT_OFFLINE: // Removal of the node from the net. + case ZT_EVENT_TRACE: // Local trace events. + // printf("TRACE: %s\n", (const char *) payload); + break; + case ZT_EVENT_REMOTE_TRACE: // Remote trace, not supported. + default: + break; + } +} + +static const char *zt_files[] = { + // clang-format off + NULL, // none, i.e. not used at all + "identity.public", + "identity.secret", + "planet", + "moon.%llx", + NULL, // peer, e.g. peers.d/ -- we don't persist this + "network.%llx", + // clang-format on +}; + +static struct { + size_t len; + void * data; +} zt_ephemeral_state[ZT_STATE_OBJECT_NETWORK_CONFIG + 1]; + +static void +zt_state_put(ZT_Node *node, void *userptr, void *thr, + enum ZT_StateObjectType objtype, const uint64_t objid[2], const void *data, + int len) +{ + zt_node *ztn = userptr; + char * path; + const char *template; + char fname[32]; + + NNI_ARG_UNUSED(node); + NNI_ARG_UNUSED(thr); + NNI_ARG_UNUSED(objid); // only use global files + + if ((objtype > ZT_STATE_OBJECT_NETWORK_CONFIG) || + ((template = zt_files[(int) objtype]) == NULL)) { + return; + } + + (void) snprintf(fname, sizeof(fname), template, + (unsigned long long) objid[0], (unsigned long long) objid[1]); + + // If we have no valid path, then we just use ephemeral data. + // Note that for moons, and so forth, we wind up just storing them + // all in the same place, but it does not matter since we don't + // really persist them anyway. + if (strlen(ztn->zn_path) == 0) { + void * ndata = NULL; + void * odata = zt_ephemeral_state[objtype].data; + size_t olen = zt_ephemeral_state[objtype].len; + if ((len >= 0) && ((ndata = nni_alloc(len)) != NULL)) { + memcpy(ndata, data, len); + zt_ephemeral_state[objtype].data = ndata; + zt_ephemeral_state[objtype].len = len; + } else if (len < 0) { + zt_ephemeral_state[objtype].data = NULL; + zt_ephemeral_state[objtype].len = 0; + } + + if (olen > 0) { + nni_free(odata, olen); + } + return; + } + + if ((path = nni_file_join(ztn->zn_path, fname)) == NULL) { + return; + } + + if (len < 0) { + (void) nni_file_delete(path); + } else { + (void) nni_file_put(path, data, len); + } + nni_strfree(path); +} + +static int +zt_state_get(ZT_Node *node, void *userptr, void *thr, + enum ZT_StateObjectType objtype, const uint64_t objid[2], void *data, + unsigned int len) +{ + zt_node *ztn = userptr; + char * path; + char fname[32]; + const char *template; + size_t sz; + void * buf; + + NNI_ARG_UNUSED(node); + NNI_ARG_UNUSED(thr); + NNI_ARG_UNUSED(objid); // we only use global files + + if ((objtype > ZT_STATE_OBJECT_NETWORK_CONFIG) || + ((template = zt_files[(int) objtype]) == NULL)) { + return (-1); + } + snprintf(fname, sizeof(fname), template, objid[0], objid[1]); + + // If no base directory, we are using ephemeral data. + if (strlen(ztn->zn_path) == 0) { + if (zt_ephemeral_state[objtype].data == NULL) { + return (-1); + } + if (zt_ephemeral_state[objtype].len > len) { + return (-1); + } + len = zt_ephemeral_state[objtype].len; + memcpy(data, zt_ephemeral_state[objtype].data, len); + return (len); + } + + if ((path = nni_file_join(ztn->zn_path, fname)) == NULL) { + return (-1); + } + + if (nni_file_get(path, &buf, &sz) != 0) { + nni_strfree(path); + return (-1); + } + nni_strfree(path); + if (sz > len) { + nni_free(buf, sz); + return (-1); + } + memcpy(data, buf, sz); + nni_free(buf, sz); + return ((int) sz); +} + +typedef struct zt_send_hdr { + nni_sockaddr sa; + size_t len; +} zt_send_hdr; + +// This function is called when ZeroTier desires to send a +// physical frame. The data is a UDP payload, the rest of the +// payload should be set over vanilla UDP. +static int +zt_wire_packet_send(ZT_Node *node, void *userptr, void *thr, int64_t socket, + const struct sockaddr_storage *remaddr, const void *data, unsigned int len, + unsigned int ttl) +{ + nni_aio * aio; + nni_sockaddr addr; + struct sockaddr_in * sin = (void *) remaddr; + struct sockaddr_in6 *sin6 = (void *) remaddr; + zt_node * ztn = userptr; + nni_plat_udp * udp; + uint8_t * buf; + zt_send_hdr * hdr; + nni_iov iov; + + NNI_ARG_UNUSED(node); + NNI_ARG_UNUSED(thr); + NNI_ARG_UNUSED(socket); + NNI_ARG_UNUSED(ttl); + + // Kind of unfortunate, but we have to convert the + // sockaddr to a neutral form, and then back again in + // the platform layer. + switch (sin->sin_family) { + case AF_INET: + addr.s_in.sa_family = NNG_AF_INET; + addr.s_in.sa_port = sin->sin_port; + addr.s_in.sa_addr = sin->sin_addr.s_addr; + udp = ztn->zn_udp4; + break; + case AF_INET6: + addr.s_in6.sa_family = NNG_AF_INET6; + addr.s_in6.sa_port = sin6->sin6_port; + udp = ztn->zn_udp6; + memcpy(addr.s_in6.sa_addr, sin6->sin6_addr.s6_addr, 16); + break; + default: + // No way to understand the address. + return (-1); + } + + if (nni_aio_alloc(&aio, NULL, NULL) != 0) { + // Out of memory + return (-1); + } + if ((buf = nni_alloc(sizeof(*hdr) + len)) == NULL) { + nni_aio_free(aio); + return (-1); + } + + hdr = (void *) buf; + buf += sizeof(*hdr); + + memcpy(buf, data, len); + nni_aio_set_data(aio, 0, hdr); + hdr->sa = addr; + hdr->len = len; + nni_aio_set_input(aio, 0, &hdr->sa); + + iov.iov_buf = buf; + iov.iov_len = len; + nni_aio_set_iov(aio, 1, &iov); + + // This should be non-blocking/best-effort, so while + // not great that we're holding the lock, also not tragic. + nni_plat_udp_send(udp, aio); + + // UDP sending is "fast" on all platforms -- given that its + // best effort only, this will complete immediately, resulting + // in either a message on the wire, or a discarded frame. We don't + // care which. (There may be a few thread context switches, but + // none of them are going to have to wait for some unbounded time.) + nni_aio_wait(aio); + nni_aio_free(aio); + nni_free(hdr, hdr->len + sizeof(*hdr)); + + return (0); +} + +static struct ZT_Node_Callbacks zt_callbacks = { + .version = 0, + .statePutFunction = zt_state_put, + .stateGetFunction = zt_state_get, + .wirePacketSendFunction = zt_wire_packet_send, + .virtualNetworkFrameFunction = zt_virtual_recv, + .virtualNetworkConfigFunction = zt_virtual_config, + .eventCallback = zt_event_cb, + .pathCheckFunction = NULL, + .pathLookupFunction = NULL, +}; + +static void +zt_node_destroy(zt_node *ztn) +{ + nni_aio_stop(ztn->zn_rcv4_aio); + nni_aio_stop(ztn->zn_rcv6_aio); + + // Wait for background thread to exit! + nni_thr_fini(&ztn->zn_bgthr); + + if (ztn->zn_znode != NULL) { + ZT_Node_delete(ztn->zn_znode); + } + + if (ztn->zn_udp4 != NULL) { + nni_plat_udp_close(ztn->zn_udp4); + } + if (ztn->zn_udp6 != NULL) { + nni_plat_udp_close(ztn->zn_udp6); + } + + if (ztn->zn_rcv4_buf != NULL) { + nni_free(ztn->zn_rcv4_buf, zt_rcv_bufsize); + } + if (ztn->zn_rcv6_buf != NULL) { + nni_free(ztn->zn_rcv6_buf, zt_rcv_bufsize); + } + if (ztn->zn_flock != NULL) { + nni_file_unlock(ztn->zn_flock); + } + nni_aio_free(ztn->zn_rcv4_aio); + nni_aio_free(ztn->zn_rcv6_aio); + zt_hash_fini(ztn->zn_eps); + zt_hash_fini(ztn->zn_lpipes); + zt_hash_fini(ztn->zn_rpipes); + nni_cv_fini(&ztn->zn_bgcv); + NNI_FREE_STRUCT(ztn); +} + +static int +zt_node_create(zt_node **ztnp, const char *path) +{ + zt_node * ztn; + nng_sockaddr sa4; + nng_sockaddr sa6; + int rv; + enum ZT_ResultCode zrv; + nni_iov iov; + + // XXX: Right now we depend on having both IPv6 and IPv4 available. + // Probably we should support coping with the lack of either of them. + + // We want to bind to any address we can (for now). + memset(&sa4, 0, sizeof(sa4)); + sa4.s_in.sa_family = NNG_AF_INET; + memset(&sa6, 0, sizeof(sa6)); + sa6.s_in6.sa_family = NNG_AF_INET6; + + if ((ztn = NNI_ALLOC_STRUCT(ztn)) == NULL) { + return (NNG_ENOMEM); + } + NNI_LIST_INIT(&ztn->zn_eplist, zt_ep, ze_link); + NNI_LIST_INIT(&ztn->zn_plist, zt_pipe, zp_link); + nni_cv_init(&ztn->zn_bgcv, &zt_lk); + nni_aio_alloc(&ztn->zn_rcv4_aio, zt_node_rcv4_cb, ztn); + nni_aio_alloc(&ztn->zn_rcv6_aio, zt_node_rcv6_cb, ztn); + + if (((ztn->zn_rcv4_buf = nni_alloc(zt_rcv_bufsize)) == NULL) || + ((ztn->zn_rcv6_buf = nni_alloc(zt_rcv_bufsize)) == NULL)) { + zt_node_destroy(ztn); + return (NNG_ENOMEM); + } + if (((rv = zt_hash_init(&ztn->zn_ports)) != 0) || + ((rv = zt_hash_init(&ztn->zn_eps)) != 0) || + ((rv = zt_hash_init(&ztn->zn_lpipes)) != 0) || + ((rv = zt_hash_init(&ztn->zn_rpipes)) != 0) || + ((rv = nni_thr_init(&ztn->zn_bgthr, zt_bgthr, ztn)) != 0) || + ((rv = nni_plat_udp_open(&ztn->zn_udp4, &sa4)) != 0) || + ((rv = nni_plat_udp_open(&ztn->zn_udp6, &sa6)) != 0)) { + zt_node_destroy(ztn); + return (rv); + } + nni_thr_set_name(&ztn->zn_bgthr, "nng:zt"); + + if (strlen(path) > 0) { + char *lkfile; + if ((lkfile = nni_file_join(path, "lock")) == NULL) { + zt_node_destroy(ztn); + return (NNG_ENOMEM); + } + + if ((rv = nni_file_lock(lkfile, &ztn->zn_flock)) != 0) { + zt_node_destroy(ztn); + nni_strfree(lkfile); + return (rv); + } + nni_strfree(lkfile); + } + + // Setup for dynamic ephemeral port allocations. We + // set the range to allow for ephemeral ports, but not + // higher than the max port, and starting with an + // initial random value. Note that this should give us + // about 8 million possible ephemeral ports. + zt_hash_limits(ztn->zn_ports, zt_ephemeral, zt_max_port, + (nni_random() % (zt_max_port - zt_ephemeral)) + zt_ephemeral); + + nni_strlcpy(ztn->zn_path, path, sizeof(ztn->zn_path)); + zrv = ZT_Node_new(&ztn->zn_znode, ztn, NULL, &zt_callbacks, zt_now()); + if (zrv != ZT_RESULT_OK) { + zt_node_destroy(ztn); + return (zt_result(zrv)); + } + + nni_list_append(&zt_nodes, ztn); + + ztn->zn_self = ZT_Node_address(ztn->zn_znode); + + nni_thr_run(&ztn->zn_bgthr); + + // Schedule an initial background run. + zt_node_resched(ztn, 1); + + // Schedule receive + iov.iov_buf = ztn->zn_rcv4_buf; + iov.iov_len = zt_rcv_bufsize; + nni_aio_set_iov(ztn->zn_rcv4_aio, 1, &iov); + nni_aio_set_input(ztn->zn_rcv4_aio, 0, &ztn->zn_rcv4_addr); + iov.iov_buf = ztn->zn_rcv6_buf; + iov.iov_len = zt_rcv_bufsize; + nni_aio_set_iov(ztn->zn_rcv6_aio, 1, &iov); + nni_aio_set_input(ztn->zn_rcv6_aio, 0, &ztn->zn_rcv6_addr); + + nni_plat_udp_recv(ztn->zn_udp4, ztn->zn_rcv4_aio); + nni_plat_udp_recv(ztn->zn_udp6, ztn->zn_rcv6_aio); + + *ztnp = ztn; + return (0); +} + +static int +zt_walk_moons(const char *path, void *arg) +{ + zt_node * ztn = arg; + const char *bn = nni_file_basename(path); + char * end; + uint64_t moonid; + + if (strncmp(bn, "moon.", 5) != 0) { + return (NNI_FILE_WALK_CONTINUE); + } + if (((moonid = (uint64_t) strtoull(bn + 5, &end, 16)) != 0) && + (*end == '\0')) { + ZT_Node_orbit(ztn->zn_znode, NULL, moonid, 0); + } + return (NNI_FILE_WALK_CONTINUE); +} + +static int +zt_node_find(zt_ep *ep) +{ + zt_node * ztn; + int rv; + ZT_VirtualNetworkConfig *cf; + + NNI_LIST_FOREACH (&zt_nodes, ztn) { + if (strcmp(ep->ze_home, ztn->zn_path) == 0) { + goto done; + } + } + + // We didn't find a node, so make one. And try to + // initialize it. + if ((rv = zt_node_create(&ztn, ep->ze_home)) != 0) { + return (rv); + } + + // Load moons + if (strlen(ep->ze_home) != 0) { + (void) nni_file_walk(ep->ze_home, zt_walk_moons, ztn, + NNI_FILE_WALK_FILES_ONLY | NNI_FILE_WALK_SHALLOW); + } + +done: + + ep->ze_ztn = ztn; + if (nni_list_node_active(&ep->ze_link)) { + nni_list_node_remove(&ep->ze_link); + } + nni_list_append(&ztn->zn_eplist, ep); + + (void) ZT_Node_join(ztn->zn_znode, ep->ze_nwid, ztn, NULL); + + if ((cf = ZT_Node_networkConfig(ztn->zn_znode, ep->ze_nwid)) != NULL) { + NNI_ASSERT(cf->nwid == ep->ze_nwid); + ep->ze_mtu = cf->mtu; + ZT_Node_freeQueryResult(ztn->zn_znode, cf); + } + + return (0); +} + +static int +zt_tran_init(void) +{ + nni_mtx_init(&zt_lk); + NNI_LIST_INIT(&zt_nodes, zt_node, zn_link); + return (0); +} + +static void +zt_tran_fini(void) +{ + zt_node *ztn; + + nni_mtx_lock(&zt_lk); + while ((ztn = nni_list_first(&zt_nodes)) != 0) { + nni_list_remove(&zt_nodes, ztn); + ztn->zn_closed = true; + nni_cv_wake(&ztn->zn_bgcv); + nni_mtx_unlock(&zt_lk); + + zt_node_destroy(ztn); + + nni_mtx_lock(&zt_lk); + } + nni_mtx_unlock(&zt_lk); + + for (int i = 0; i <= ZT_STATE_OBJECT_NETWORK_CONFIG; i++) { + if (zt_ephemeral_state[i].len > 0) { + nni_free(zt_ephemeral_state[i].data, + zt_ephemeral_state[i].len); + } + } + NNI_ASSERT(nni_list_empty(&zt_nodes)); + nni_mtx_fini(&zt_lk); +} + +static int +zt_check_recvmaxsz(const void *v, size_t sz, nni_type t) +{ + return (nni_copyin_size(NULL, v, sz, 0, NNI_MAXSZ, t)); +} + +static int +zt_check_orbit(const void *v, size_t sz, nni_type t) +{ + NNI_ARG_UNUSED(v); + if ((t != NNI_TYPE_UINT64) && (t != NNI_TYPE_OPAQUE)) { + return (NNG_EBADTYPE); + } + if (sz != sizeof(uint64_t) && sz != sizeof(uint64_t) * 2) { + return (NNG_EINVAL); + } + return (0); +} + +static int +zt_check_deorbit(const void *v, size_t sz, nni_type t) +{ + return (nni_copyin_u64(NULL, v, sz, t)); +} + +static int +zt_check_string(const void *v, size_t sz, nni_type t) +{ + size_t len; + + if ((t != NNI_TYPE_OPAQUE) && (t != NNI_TYPE_STRING)) { + return (NNG_EBADTYPE); + } + len = nni_strnlen(v, sz); + if ((len >= sz) || (len >= NNG_MAXADDRLEN)) { + return (NNG_EINVAL); + } + return (0); +} + +static int +zt_check_time(const void *v, size_t sz, nni_type t) +{ + return (nni_copyin_ms(NULL, v, sz, t)); +} + +static int +zt_check_tries(const void *v, size_t sz, nni_type t) +{ + return (nni_copyin_int(NULL, v, sz, 0, 1000000, t)); +} + +static void +zt_pipe_close(void *arg) +{ + zt_pipe *p = arg; + nni_aio *aio; + + nni_mtx_lock(&zt_lk); + p->zp_closed = true; + nni_aio_close(p->zp_ping_aio); + if ((aio = p->zp_user_rxaio) != NULL) { + p->zp_user_rxaio = NULL; + nni_aio_finish_error(aio, NNG_ECLOSED); + } + zt_pipe_send_disc_req(p); + nni_mtx_unlock(&zt_lk); +} + +static int +zt_pipe_init(void *arg, nni_pipe *npipe) +{ + zt_pipe *p = arg; + p->zp_npipe = npipe; + return (0); +} + +static void +zt_pipe_fini(void *arg) +{ + zt_pipe *p = arg; + zt_node *ztn = p->zp_ztn; + + nni_aio_free(p->zp_ping_aio); + + // This tosses the connection details and all state. + nni_mtx_lock(&zt_lk); + zt_hash_remove(ztn->zn_ports, p->zp_laddr & zt_port_mask); + zt_hash_remove(ztn->zn_lpipes, p->zp_laddr); + zt_hash_remove(ztn->zn_rpipes, p->zp_raddr); + nni_mtx_unlock(&zt_lk); + + for (int i = 0; i < zt_recvq; i++) { + zt_fraglist_free(&p->zp_recvq[i]); + } + nni_free(p->zp_send_buf, ZT_MAX_MTU); + NNI_FREE_STRUCT(p); +} + +static void +zt_pipe_reap(zt_pipe *p) +{ + if (!nni_atomic_flag_test_and_set(&p->zp_reaped)) { + nni_reap(&p->zp_reap, zt_pipe_fini, p); + } +} + +static int +zt_pipe_alloc( + zt_pipe **pipep, zt_ep *ep, uint64_t raddr, uint64_t laddr, bool listener) +{ + zt_pipe *p; + int rv; + zt_node *ztn = ep->ze_ztn; + int i; + size_t maxfrag; + size_t maxfrags = 0; + + if ((p = NNI_ALLOC_STRUCT(p)) == NULL) { + return (NNG_ENOMEM); + } + if ((p->zp_send_buf = nni_alloc(ZT_MAX_MTU)) == NULL) { + NNI_FREE_STRUCT(p); + return (NNG_ENOMEM); + } + p->zp_ztn = ztn; + p->zp_raddr = raddr; + p->zp_laddr = laddr; + p->zp_proto = ep->ze_proto; + p->zp_nwid = ep->ze_nwid; + p->zp_mtu = ep->ze_mtu; + p->zp_rcvmax = ep->ze_rcvmax; + p->zp_ping_tries = ep->ze_ping_tries; + p->zp_ping_time = ep->ze_ping_time; + p->zp_next_msgid = (uint16_t) nni_random(); + p->zp_ping_try = 0; + nni_atomic_flag_reset(&p->zp_reaped); + + if (listener) { + // listener + rv = zt_hash_insert(ztn->zn_rpipes, raddr, p); + } else { + // dialer + rv = zt_hash_insert(ztn->zn_lpipes, laddr, p); + } + if ((rv != 0) || + ((rv = nni_aio_alloc(&p->zp_ping_aio, zt_pipe_ping_cb, p)) != 0)) { + zt_pipe_reap(p); + return (rv); + } + + // The largest fragment we can accept on this pipe. The MTU is + // configurable by the network administrator. Probably ZT would + // pass a larger one (up to MAXMTU), but we honor the network + // administration's configuration. + maxfrag = p->zp_mtu - zt_offset_data_data; + + // The largest fragment count we can accept on this pipe. + // This is rounded up to account for alignment. + if (p->zp_rcvmax > 0) { + maxfrags = (p->zp_rcvmax + (maxfrag - 1)) / maxfrag; + } + + if ((maxfrags > 0xffff) || (maxfrags == 0)) { + maxfrags = 0xffff; + } + + for (i = 0; i < zt_recvq; i++) { + zt_fraglist *fl = &p->zp_recvq[i]; + fl->fl_time = NNI_TIME_ZERO; + fl->fl_msgid = 0; + fl->fl_ready = 0; + fl->fl_missingsz = (maxfrags + 7) / 8; + fl->fl_missing = nni_alloc(fl->fl_missingsz); + if (fl->fl_missing == NULL) { + zt_pipe_reap(p); + return (NNG_ENOMEM); + } + } + + *pipep = p; + return (0); +} + +static void +zt_pipe_send(void *arg, nni_aio *aio) +{ + // As we are sending UDP, and there is no callback to worry + // about, we just go ahead and send out a stream of messages + // synchronously. + zt_pipe *p = arg; + uint8_t *data = p->zp_send_buf; + size_t offset; + uint16_t id; + uint16_t nfrags; + uint16_t fragno; + size_t fragsz; + size_t bytes; + nni_msg *m; + + if (nni_aio_begin(aio) != 0) { + return; + } + if ((m = nni_aio_get_msg(aio)) == NULL) { + nni_aio_finish_error(aio, NNG_EINVAL); + return; + } + + nni_mtx_lock(&zt_lk); + + if (p->zp_closed) { + nni_mtx_unlock(&zt_lk); + nni_aio_finish_error(aio, NNG_ECLOSED); + return; + } + + fragsz = p->zp_mtu - zt_offset_data_data; + NNI_ASSERT(fragsz < 0x10000); // Because zp_mtu is 16 bits + + bytes = nni_msg_header_len(m) + nni_msg_len(m); + if (bytes >= (0xfffe * fragsz)) { + nni_aio_finish_error(aio, NNG_EMSGSIZE); + nni_mtx_unlock(&zt_lk); + return; + } + // above check means nfrags will fit in 16-bits. + nfrags = (uint16_t)((bytes + (fragsz - 1)) / fragsz); + + // get the next message ID, but skip 0 + if ((id = p->zp_next_msgid++) == 0) { + id = p->zp_next_msgid++; + } + + offset = 0; + fragno = 0; + do { + uint8_t *dest = data + zt_offset_data_data; + size_t room = fragsz; + size_t fraglen = 0; + size_t len; + + // Prepend the header first. + if ((len = nni_msg_header_len(m)) > 0) { + if (len > fragsz) { + // This shouldn't happen! SP headers are + // supposed to be quite small. + nni_aio_finish_error(aio, NNG_EMSGSIZE); + nni_mtx_unlock(&zt_lk); + return; + } + memcpy(dest, nni_msg_header(m), len); + dest += len; + room -= len; + offset += len; + fraglen += len; + nni_msg_header_clear(m); + } + + len = nni_msg_len(m); + if (len > room) { + len = room; + } + memcpy(dest, nni_msg_body(m), len); + + nng_msg_trim(m, len); + NNI_PUT16(data + zt_offset_data_id, id); + NNI_PUT16(data + zt_offset_data_fragsz, (uint16_t) fragsz); + NNI_PUT16(data + zt_offset_data_frag, fragno); + NNI_PUT16(data + zt_offset_data_nfrag, nfrags); + offset += len; + fraglen += len; + fragno++; + zt_send(p->zp_ztn, p->zp_nwid, zt_op_data, p->zp_raddr, + p->zp_laddr, data, fraglen + zt_offset_data_data); + } while (nni_msg_len(m) != 0); + nni_mtx_unlock(&zt_lk); + + // NB, We never bothered to call nn_aio_sched, because we run this + // synchronously, relying on UDP to simply discard messages if we + // cannot deliver them. This means that pipe send operations with + // this transport are not cancellable. + + nni_aio_set_msg(aio, NULL); + nni_msg_free(m); + nni_aio_finish(aio, 0, offset); +} + +static void +zt_pipe_cancel_recv(nni_aio *aio, void *arg, int rv) +{ + zt_pipe *p = arg; + nni_mtx_lock(&zt_lk); + if (p->zp_user_rxaio == aio) { + p->zp_user_rxaio = NULL; + nni_aio_finish_error(aio, rv); + } + nni_mtx_unlock(&zt_lk); +} + +static void +zt_fraglist_clear(zt_fraglist *fl) +{ + nni_msg *msg; + + fl->fl_ready = 0; + fl->fl_msgid = 0; + fl->fl_time = NNI_TIME_ZERO; + if ((msg = fl->fl_msg) != NULL) { + fl->fl_msg = NULL; + nni_msg_free(msg); + } + memset(fl->fl_missing, 0, fl->fl_missingsz); +} + +static void +zt_fraglist_free(zt_fraglist *fl) +{ + zt_fraglist_clear(fl); + nni_free(fl->fl_missing, fl->fl_missingsz); + fl->fl_missing = NULL; +} + +static void +zt_pipe_dorecv(zt_pipe *p) +{ + nni_aio *aio = p->zp_user_rxaio; + nni_time now = nni_clock(); + + if (aio == NULL) { + return; + } + + for (int i = 0; i < zt_recvq; i++) { + zt_fraglist *fl = &p->zp_recvq[i]; + nni_msg * msg; + + if (now > (fl->fl_time + zt_recv_stale)) { + // fragment list is stale, clean it. + zt_fraglist_clear(fl); + continue; + } + if (!fl->fl_ready) { + continue; + } + + // Got data. Let's pass it up. + msg = fl->fl_msg; + fl->fl_msg = NULL; + NNI_ASSERT(msg != NULL); + + p->zp_user_rxaio = NULL; + nni_aio_finish_msg(aio, msg); + zt_fraglist_clear(fl); + return; + } +} + +static void +zt_pipe_recv(void *arg, nni_aio *aio) +{ + zt_pipe *p = arg; + int rv; + + if (nni_aio_begin(aio) != 0) { + return; + } + nni_mtx_lock(&zt_lk); + if (p->zp_closed) { + nni_mtx_unlock(&zt_lk); + nni_aio_finish_error(aio, NNG_ECLOSED); + return; + } + if ((rv = nni_aio_schedule(aio, zt_pipe_cancel_recv, p)) != 0) { + nni_mtx_unlock(&zt_lk); + nni_aio_finish_error(aio, rv); + return; + } + p->zp_user_rxaio = aio; + zt_pipe_dorecv(p); + nni_mtx_unlock(&zt_lk); +} + +static uint16_t +zt_pipe_peer(void *arg) +{ + zt_pipe *pipe = arg; + + return (pipe->zp_peer); +} + +static int +zt_get_nw_status(zt_node *ztn, uint64_t nwid, int *statusp) +{ + ZT_VirtualNetworkConfig *vcfg; + int status; + + vcfg = ZT_Node_networkConfig(ztn->zn_znode, nwid); + if (vcfg == NULL) { + return (NNG_ECLOSED); + } + switch (vcfg->status) { + case ZT_NETWORK_STATUS_REQUESTING_CONFIGURATION: + status = NNG_ZT_STATUS_CONFIG; + break; + case ZT_NETWORK_STATUS_OK: + status = NNG_ZT_STATUS_UP; + break; + case ZT_NETWORK_STATUS_ACCESS_DENIED: + status = NNG_ZT_STATUS_DENIED; + break; + case ZT_NETWORK_STATUS_NOT_FOUND: + status = NNG_ZT_STATUS_NOTFOUND; + break; + case ZT_NETWORK_STATUS_PORT_ERROR: + status = NNG_ZT_STATUS_ERROR; + break; + case ZT_NETWORK_STATUS_CLIENT_TOO_OLD: + status = NNG_ZT_STATUS_OBSOLETE; + break; + default: + status = NNG_ZT_STATUS_UNKNOWN; + break; + } + ZT_Node_freeQueryResult(ztn->zn_znode, vcfg); + + *statusp = status; + return (0); +} + +static int +zt_get_nw_name(zt_node *ztn, uint64_t nwid, void *buf, size_t *szp, nni_type t) +{ + ZT_VirtualNetworkConfig *vcfg; + int rv; + + vcfg = ZT_Node_networkConfig(ztn->zn_znode, nwid); + if (vcfg == NULL) { + return (NNG_ECLOSED); + } + + rv = nni_copyout_str(vcfg->name, buf, szp, t); + ZT_Node_freeQueryResult(ztn->zn_znode, vcfg); + + return (rv); +} + +static int +zt_pipe_get_recvmaxsz(void *arg, void *buf, size_t *szp, nni_type t) +{ + zt_pipe *p = arg; + return (nni_copyout_size(p->zp_rcvmax, buf, szp, t)); +} + +static int +zt_pipe_get_nwid(void *arg, void *buf, size_t *szp, nni_type t) +{ + zt_pipe *p = arg; + return (nni_copyout_u64(p->zp_nwid, buf, szp, t)); +} + +static int +zt_pipe_get_node(void *arg, void *buf, size_t *szp, nni_type t) +{ + zt_pipe *p = arg; + return (nni_copyout_u64(p->zp_laddr >> 24, buf, szp, t)); +} + +static void +zt_pipe_ping_cb(void *arg) +{ + zt_pipe *p = arg; + nni_aio *aio = p->zp_ping_aio; + int rv; + + if ((rv = nni_aio_result(aio)) != 0) { + // We were canceled. That means we're done. + return; + } + nni_mtx_lock(&zt_lk); + if (p->zp_closed || aio == NULL || (p->zp_ping_tries == 0) || + (p->zp_ping_time == NNG_DURATION_INFINITE) || + (p->zp_ping_time == NNG_DURATION_ZERO)) { + nni_mtx_unlock(&zt_lk); + return; + } + if (p->zp_ping_try > p->zp_ping_tries) { + // Ping count exceeded; the other side is AFK. + // Close the pipe, but no need to send a reason to the peer. + zt_pipe_close_err(p, NNG_ECLOSED, 0, NULL); + nni_mtx_unlock(&zt_lk); + return; + } + + if (nni_clock() > (p->zp_last_recv + p->zp_ping_time)) { + p->zp_ping_try++; + zt_pipe_send_ping(p); + } + + nni_sleep_aio(p->zp_ping_time, aio); // Schedule a recheck. + nni_mtx_unlock(&zt_lk); +} + +static void +zt_pipe_start_ping(zt_pipe *p) +{ + // send a gratuitous ping, and start the ping interval timer. + if ((p->zp_ping_tries > 0) && (p->zp_ping_time != NNG_DURATION_ZERO) && + (p->zp_ping_time != NNG_DURATION_INFINITE)) { + p->zp_ping_try = 0; + zt_pipe_send_ping(p); + nni_sleep_aio(p->zp_ping_time, p->zp_ping_aio); + } +} + +static void +zt_ep_fini(void *arg) +{ + zt_ep *ep = arg; + nni_aio_stop(ep->ze_creq_aio); + nni_aio_free(ep->ze_creq_aio); + NNI_FREE_STRUCT(ep); +} + +static int +zt_parsehex(const char **sp, uint64_t *valp, bool wildok) +{ + int n; + const char *s = *sp; + char c; + uint64_t v; + + if (wildok && *s == '*') { + *valp = 0; + s++; + *sp = s; + return (0); + } + + for (v = 0, n = 0; (n < 16) && isxdigit(c = tolower(*s)); n++, s++) { + v *= 16; + if (isdigit(c)) { + v += (c - '0'); + } else { + v += ((c - 'a') + 10); + } + } + + *sp = s; + *valp = v; + return (n ? 0 : NNG_EINVAL); +} + +static int +zt_parsedec(const char **sp, uint64_t *valp) +{ + int n; + const char *s = *sp; + char c; + uint64_t v; + + for (v = 0, n = 0; (n < 20) && isdigit(c = *s); n++, s++) { + v *= 10; + v += (c - '0'); + } + *sp = s; + *valp = v; + return (n ? 0 : NNG_EINVAL); +} + +static int +zt_ep_init(void **epp, nni_url *url, nni_sock *sock, nni_dialer *ndialer, + nni_listener *nlistener) +{ + zt_ep * ep; + uint64_t node; + uint64_t port; + int rv; + const char *h; + + if ((ep = NNI_ALLOC_STRUCT(ep)) == NULL) { + return (NNG_ENOMEM); + } + + ep->ze_mtu = ZT_MIN_MTU; + ep->ze_aio = NULL; + ep->ze_ping_tries = zt_ping_tries; + ep->ze_ping_time = zt_ping_time; + ep->ze_conn_time = zt_conn_time; + ep->ze_conn_tries = zt_conn_tries; + ep->ze_proto = nni_sock_proto_id(sock); + ep->ze_ndialer = ndialer; + ep->ze_nlistener = nlistener; + + nni_aio_list_init(&ep->ze_aios); + + rv = nni_aio_alloc(&ep->ze_creq_aio, zt_ep_conn_req_cb, ep); + if (rv != 0) { + zt_ep_fini(ep); + return (rv); + } + + // Our URL format is: + // + // zt://.: + // + // The port must be specified, but may be zero. The nodeid + // may be '*' to refer to ourself. There may be a trailing slash + // which will be ignored. + + h = url->u_hostname; + if (((strlen(url->u_path) == 1) && (url->u_path[0] != '/')) || + (strlen(url->u_path) > 1) || (url->u_fragment != NULL) || + (url->u_query != NULL) || (url->u_userinfo != NULL) || + (zt_parsehex(&h, &node, true) != 0) || (*h++ != '.') || + (zt_parsehex(&h, &ep->ze_nwid, false) != 0) || + (node > 0xffffffffffull)) { + return (NNG_EADDRINVAL); + } + h = url->u_port; + if ((zt_parsedec(&h, &port) != 0) || (port > zt_max_port)) { + return (NNG_EADDRINVAL); + } + + // Parse the URL. + if (nlistener != NULL) { + // listener + ep->ze_laddr = node; + ep->ze_laddr <<= 24; + ep->ze_laddr |= port; + ep->ze_raddr = 0; + ep->ze_nlistener = nlistener; + } else { + // dialer + if (port == 0) { + return (NNG_EADDRINVAL); + } + ep->ze_raddr = node; + ep->ze_raddr <<= 24; + ep->ze_raddr |= port; + ep->ze_laddr = 0; + ep->ze_ndialer = ndialer; + } + + *epp = ep; + return (0); +} + +static int +zt_dialer_init(void **epp, nni_url *url, nni_dialer *d) +{ + return (zt_ep_init(epp, url, nni_dialer_sock(d), d, NULL)); +} + +static int +zt_listener_init(void **epp, nni_url *url, nni_listener *l) +{ + return (zt_ep_init(epp, url, nni_listener_sock(l), NULL, l)); +} + +static void +zt_ep_close(void *arg) +{ + zt_ep * ep = arg; + zt_node *ztn; + nni_aio *aio; + + nni_aio_abort(ep->ze_creq_aio, NNG_ECLOSED); + + // Cancel any outstanding user operation(s) - they should have + // been aborted by the above cancellation, but we need to be + // sure, as the cancellation callback may not have run yet. + + nni_mtx_lock(&zt_lk); + while ((aio = nni_list_first(&ep->ze_aios)) != NULL) { + nni_aio_list_remove(aio); + nni_aio_finish_error(aio, NNG_ECLOSED); + } + + // Endpoint framework guarantees to only call us once, + // and to not call other things while we are closed. + ztn = ep->ze_ztn; + // If we're on the ztn node list, pull us off. + if (ztn != NULL) { + nni_list_node_remove(&ep->ze_link); + zt_hash_remove(ztn->zn_ports, ep->ze_laddr & zt_port_mask); + zt_hash_remove(ztn->zn_eps, ep->ze_laddr); + } + + nni_mtx_unlock(&zt_lk); +} + +static int +zt_ep_bind_locked(zt_ep *ep) +{ + int rv; + uint64_t port; + uint64_t node; + zt_node *ztn; + + // If we haven't already got a ZT node, get one. + if ((ztn = ep->ze_ztn) == NULL) { + if ((rv = zt_node_find(ep)) != 0) { + return (rv); + } + ztn = ep->ze_ztn; + } + + node = ep->ze_laddr >> 24; + if ((node != 0) && (node != ztn->zn_self)) { + // User requested node id, but it doesn't match our + // own. + return (NNG_EADDRINVAL); + } + + if ((ep->ze_laddr & zt_port_mask) == 0) { + // ask for an ephemeral port + if ((rv = zt_hash_alloc(ztn->zn_ports, &port, ep)) != 0) { + return (rv); + } + NNI_ASSERT(port & zt_ephemeral); + } else { + void *conflict; + // make sure port requested is free. + port = ep->ze_laddr & zt_port_mask; + + if (zt_hash_find(ztn->zn_ports, port, &conflict) == 0) { + return (NNG_EADDRINUSE); + } + if ((rv = zt_hash_insert(ztn->zn_ports, port, ep)) != 0) { + return (rv); + } + } + NNI_ASSERT(port <= zt_max_port); + NNI_ASSERT(port > 0); + + ep->ze_laddr = ztn->zn_self; + ep->ze_laddr <<= 24; + ep->ze_laddr |= port; + ep->ze_running = true; + + if ((rv = zt_hash_insert(ztn->zn_eps, ep->ze_laddr, ep)) != 0) { + zt_hash_remove(ztn->zn_ports, port); + return (rv); + } + + return (0); +} + +static int +zt_ep_bind(void *arg) +{ + int rv; + zt_ep *ep = arg; + + nni_mtx_lock(&zt_lk); + rv = zt_ep_bind_locked(ep); + nni_mtx_unlock(&zt_lk); + + return (rv); +} + +static void +zt_ep_cancel(nni_aio *aio, void *arg, int rv) +{ + zt_ep *ep = arg; + + nni_mtx_lock(&zt_lk); + if (nni_aio_list_active(aio)) { + if (ep->ze_aio != NULL) { + nni_aio_abort(ep->ze_aio, rv); + } + nni_aio_list_remove(aio); + nni_aio_finish_error(aio, rv); + } + nni_mtx_unlock(&zt_lk); +} + +static void +zt_ep_doaccept(zt_ep *ep) +{ + // Call with ep lock held. + nni_time now; + zt_pipe *p; + int rv; + + now = nni_clock(); + // Consume any timedout connect requests. + while (ep->ze_creq_tail != ep->ze_creq_head) { + zt_creq creq; + nni_aio *aio; + + creq = ep->ze_creqs[ep->ze_creq_tail % zt_listenq]; + // Discard old connection requests. + if (creq.cr_expire < now) { + ep->ze_creq_tail++; + continue; + } + + if ((aio = nni_list_first(&ep->ze_aios)) == NULL) { + // No outstanding accept. We're done. + break; + } + + // We have both conn request, and a place to accept it. + + // Advance the tail. + ep->ze_creq_tail++; + + // We remove this AIO. This keeps it from being canceled. + nni_aio_list_remove(aio); + + rv = zt_pipe_alloc(&p, ep, creq.cr_raddr, ep->ze_laddr, true); + if (rv != 0) { + zt_send_err(ep->ze_ztn, ep->ze_nwid, creq.cr_raddr, + ep->ze_laddr, zt_err_unknown, + "Failed creating pipe"); + nni_aio_finish_error(aio, rv); + continue; + } + p->zp_peer = creq.cr_proto; + zt_pipe_send_conn_ack(p); + zt_pipe_start_ping(p); + nni_aio_set_output(aio, 0, p); + nni_aio_finish(aio, 0, 0); + } +} + +static void +zt_ep_accept(void *arg, nni_aio *aio) +{ + zt_ep *ep = arg; + int rv; + + if (nni_aio_begin(aio) != 0) { + return; + } + nni_mtx_lock(&zt_lk); + if ((rv = nni_aio_schedule(aio, zt_ep_cancel, ep)) != 0) { + nni_mtx_unlock(&zt_lk); + nni_aio_finish_error(aio, rv); + return; + } + nni_aio_list_append(&ep->ze_aios, aio); + zt_ep_doaccept(ep); + nni_mtx_unlock(&zt_lk); +} + +static void +zt_ep_conn_req_cancel(nni_aio *aio, void *arg, int rv) +{ + zt_ep *ep = arg; + // We don't have much to do here. The AIO will have been + // canceled as a result of the "parent" AIO canceling. + nni_mtx_lock(&zt_lk); + if (ep->ze_creq_active) { + ep->ze_creq_active = false; + nni_aio_finish_error(aio, rv); + } + nni_mtx_unlock(&zt_lk); +} + +static void +zt_ep_conn_req_cb(void *arg) +{ + zt_ep * ep = arg; + zt_pipe *p; + nni_aio *aio = ep->ze_creq_aio; + nni_aio *uaio; + int rv; + + nni_mtx_lock(&zt_lk); + + ep->ze_creq_active = false; + switch ((rv = nni_aio_result(aio))) { + case 0: + p = nni_aio_get_output(aio, 0); + // Already canceled, or already handled? + if ((uaio = nni_list_first(&ep->ze_aios)) != NULL) { + nni_aio_list_remove(uaio); + zt_pipe_start_ping(p); + nni_aio_set_output(uaio, 0, p); + nni_aio_finish(uaio, 0, 0); + } else { + // We have a pipe, but nowhere to stick it. + // Just discard it. + zt_pipe_fini(p); + } + ep->ze_creq_try = 0; + break; + + case NNG_ETIMEDOUT: + if ((ep->ze_creq_try > ep->ze_conn_tries) && + (ep->ze_conn_tries > 0)) { + // Final timeout attempt. + if ((uaio = nni_list_first(&ep->ze_aios)) != NULL) { + nni_aio_list_remove(uaio); + nni_aio_finish_error(uaio, rv); + // reset the counter. + ep->ze_creq_try = 0; + } + } + break; + + default: + // Failed hard? + if ((uaio = nni_list_first(&ep->ze_aios)) != NULL) { + nni_aio_list_remove(uaio); + nni_aio_finish_error(uaio, rv); + } + ep->ze_creq_try = 0; + break; + } + + if (nni_list_first(&ep->ze_aios) != NULL) { + nni_aio_set_timeout(aio, ep->ze_conn_time); + if (nni_aio_begin(aio) == 0) { + rv = nni_aio_schedule(aio, zt_ep_conn_req_cancel, ep); + if (rv != 0) { + nni_aio_finish_error(aio, rv); + } else { + ep->ze_creq_active = true; + ep->ze_creq_try++; + zt_ep_send_conn_req(ep); + } + } + } + + nni_mtx_unlock(&zt_lk); +} + +static void +zt_ep_connect(void *arg, nni_aio *aio) +{ + zt_ep *ep = arg; + int rv; + + if (nni_aio_begin(aio) != 0) { + return; + } + // We bind locally. We'll use the address later when we give + // it to the pipe, but this allows us to receive the initial + // ack back from the server. (This gives us an ephemeral + // address to work with.) + nni_mtx_lock(&zt_lk); + + // Clear the port so we get an ephemeral port. + ep->ze_laddr &= ~((uint64_t) zt_port_mask); + + if ((rv = zt_ep_bind_locked(ep)) != 0) { + nni_aio_finish_error(aio, rv); + nni_mtx_unlock(&zt_lk); + return; + } + + if ((ep->ze_raddr >> 24) == 0) { + ep->ze_raddr |= (ep->ze_ztn->zn_self << zt_port_shift); + } + if ((rv = nni_aio_schedule(aio, zt_ep_cancel, ep)) != 0) { + nni_aio_finish_error(aio, rv); + nni_mtx_unlock(&zt_lk); + return; + } + nni_aio_list_append(&ep->ze_aios, aio); + ep->ze_running = true; + + nni_aio_set_timeout(ep->ze_creq_aio, ep->ze_conn_time); + if (nni_aio_begin(ep->ze_creq_aio) == 0) { + rv = nni_aio_schedule( + ep->ze_creq_aio, zt_ep_conn_req_cancel, ep); + if (rv != 0) { + nni_aio_finish_error(ep->ze_creq_aio, rv); + } else { + // Send out the first connect message; if not + // yet attached to network message will be dropped. + ep->ze_creq_try = 1; + ep->ze_creq_active = true; + zt_ep_send_conn_req(ep); + } + } + nni_mtx_unlock(&zt_lk); +} + +static int +zt_ep_set_recvmaxsz(void *arg, const void *data, size_t sz, nni_type t) +{ + zt_ep *ep = arg; + size_t val; + int rv; + + if ((rv = nni_copyin_size(&val, data, sz, 0, NNI_MAXSZ, t)) == 0) { + nni_mtx_lock(&zt_lk); + ep->ze_rcvmax = val; + nni_mtx_unlock(&zt_lk); + } + return (rv); +} + +static int +zt_ep_get_recvmaxsz(void *arg, void *data, size_t *szp, nni_type t) +{ + zt_ep *ep = arg; + int rv; + nni_mtx_lock(&zt_lk); + rv = nni_copyout_size(ep->ze_rcvmax, data, szp, t); + nni_mtx_unlock(&zt_lk); + return (rv); +} + +static int +zt_ep_set_home(void *arg, const void *data, size_t sz, nni_type t) +{ + int rv; + zt_ep *ep = arg; + + if ((rv = zt_check_string(data, sz, t)) == 0) { + nni_mtx_lock(&zt_lk); + if (ep->ze_running) { + rv = NNG_ESTATE; + } else { + nni_strlcpy(ep->ze_home, data, sizeof(ep->ze_home)); + if ((rv = zt_node_find(ep)) != 0) { + ep->ze_ztn = NULL; + } + } + nni_mtx_unlock(&zt_lk); + } + + return (rv); +} + +static int +zt_ep_get_home(void *arg, void *data, size_t *szp, nni_type t) +{ + zt_ep *ep = arg; + int rv; + + nni_mtx_lock(&zt_lk); + rv = nni_copyout_str(ep->ze_home, data, szp, t); + nni_mtx_unlock(&zt_lk); + return (rv); +} + +static int +zt_ep_get_url(void *arg, void *data, size_t *szp, nni_type t) +{ + char ustr[64]; // more than plenty + zt_ep * ep = arg; + uint64_t addr; + + nni_mtx_lock(&zt_lk); + addr = ep->ze_nlistener != NULL ? ep->ze_laddr : ep->ze_raddr; + snprintf(ustr, sizeof(ustr), "zt://%llx.%llx:%u", + (unsigned long long) addr >> zt_port_shift, + (unsigned long long) ep->ze_nwid, + (unsigned) (addr & zt_port_mask)); + nni_mtx_unlock(&zt_lk); + return (nni_copyout_str(ustr, data, szp, t)); +} + +static int +zt_ep_set_orbit(void *arg, const void *data, size_t sz, nni_type t) +{ + uint64_t moonid; + uint64_t peerid; + zt_ep * ep = arg; + int rv; + enum ZT_ResultCode zrv; + + if ((t != NNI_TYPE_UINT64) && (t != NNI_TYPE_OPAQUE)) { + return (NNG_EBADTYPE); + } + if (sz == sizeof(uint64_t)) { + memcpy(&moonid, data, sizeof(moonid)); + peerid = 0; + } else if (sz == sizeof(uint64_t) * 2) { + memcpy(&moonid, data, sizeof(moonid)); + memcpy(&peerid, ((char *) data) + sizeof(uint64_t), + sizeof(peerid)); + } else { + return (NNG_EINVAL); + } + + nni_mtx_lock(&zt_lk); + if ((ep->ze_ztn == NULL) && ((rv = zt_node_find(ep)) != 0)) { + nni_mtx_unlock(&zt_lk); + return (rv); + } + zrv = ZT_Node_orbit(ep->ze_ztn->zn_znode, NULL, moonid, peerid); + nni_mtx_unlock(&zt_lk); + + return (zt_result(zrv)); +} + +static int +zt_ep_set_deorbit(void *arg, const void *data, size_t sz, nni_type t) +{ + uint64_t moonid; + zt_ep * ep = arg; + int rv; + + if ((rv = nni_copyin_u64(&moonid, data, sz, t)) == 0) { + enum ZT_ResultCode zrv; + + nni_mtx_lock(&zt_lk); + if ((ep->ze_ztn == NULL) && ((rv = zt_node_find(ep)) != 0)) { + nni_mtx_unlock(&zt_lk); + return (rv); + } + zrv = ZT_Node_deorbit(ep->ze_ztn->zn_znode, NULL, moonid); + nni_mtx_unlock(&zt_lk); + rv = zt_result(zrv); + } + return (rv); +} + +static int +zt_ep_set_add_local_addr(void *arg, const void *data, size_t sz, nni_type t) +{ + nng_sockaddr sa; + zt_ep * ep = arg; + int rv; + + if ((rv = nni_copyin_sockaddr(&sa, data, sz, t)) == 0) { + enum ZT_ResultCode zrv; + zt_node * ztn; + struct sockaddr_storage ss; + struct sockaddr_in * sin; + struct sockaddr_in6 * sin6; + + memset(&ss, 0, sizeof(ss)); + switch (sa.s_family) { + case NNG_AF_INET: + sin = (void *) &ss; + sin->sin_family = AF_INET; + sin->sin_addr.s_addr = sa.s_in.sa_addr; + sin->sin_port = 0; + break; + case NNG_AF_INET6: + sin6 = (void *) &ss; + sin6->sin6_family = AF_INET6; + sin6->sin6_port = 0; + memcpy(&sin6->sin6_addr, sa.s_in6.sa_addr, 16); + break; + default: + return (NNG_EINVAL); + } + + nni_mtx_lock(&zt_lk); + if ((ep->ze_ztn == NULL) && ((rv = zt_node_find(ep)) != 0)) { + nni_mtx_unlock(&zt_lk); + return (rv); + } + ztn = ep->ze_ztn; + zrv = ZT_Node_addLocalInterfaceAddress(ztn->zn_znode, &ss); + nni_mtx_unlock(&zt_lk); + rv = zt_result(zrv); + } + return (rv); +} + +static int +zt_ep_set_clear_local_addrs(void *arg, const void *data, size_t sz, nni_type t) +{ + zt_ep *ep = arg; + int rv; + NNI_ARG_UNUSED(data); + NNI_ARG_UNUSED(sz); + NNI_ARG_UNUSED(t); + + ZT_Node *zn; + nni_mtx_lock(&zt_lk); + if ((ep->ze_ztn == NULL) && ((rv = zt_node_find(ep)) != 0)) { + nni_mtx_unlock(&zt_lk); + return (rv); + } + zn = ep->ze_ztn; + ZT_Node_clearLocalInterfaceAddresses(zn); + nni_mtx_unlock(&zt_lk); + return (0); +} + +static int +zt_ep_get_node(void *arg, void *data, size_t *szp, nni_type t) +{ + zt_ep *ep = arg; + int rv; + + nni_mtx_lock(&zt_lk); + if ((ep->ze_ztn == NULL) && ((rv = zt_node_find(ep)) != 0)) { + nni_mtx_unlock(&zt_lk); + return (rv); + } + + rv = nni_copyout_u64(ep->ze_ztn->zn_self, data, szp, t); + + nni_mtx_unlock(&zt_lk); + return (rv); +} + +static int +zt_ep_get_nwid(void *arg, void *data, size_t *szp, nni_type t) +{ + zt_ep *ep = arg; + int rv; + + nni_mtx_lock(&zt_lk); + if ((ep->ze_ztn == NULL) && ((rv = zt_node_find(ep)) != 0)) { + nni_mtx_unlock(&zt_lk); + return (rv); + } + rv = nni_copyout_u64(ep->ze_nwid, data, szp, t); + nni_mtx_unlock(&zt_lk); + return (rv); +} + +static int +zt_ep_get_nw_name(void *arg, void *buf, size_t *szp, nni_type t) +{ + zt_ep *ep = arg; + int rv; + + nni_mtx_lock(&zt_lk); + if ((ep->ze_ztn == NULL) && ((rv = zt_node_find(ep)) != 0)) { + nni_mtx_unlock(&zt_lk); + return (rv); + } + rv = zt_get_nw_name(ep->ze_ztn, ep->ze_nwid, buf, szp, t); + nni_mtx_unlock(&zt_lk); + return (rv); +} + +static int +zt_ep_get_nw_status(void *arg, void *buf, size_t *szp, nni_type t) +{ + zt_ep *ep = arg; + int rv; + int status; + + nni_mtx_lock(&zt_lk); + if ((ep->ze_ztn == NULL) && ((rv = zt_node_find(ep)) != 0)) { + nni_mtx_unlock(&zt_lk); + return (rv); + } + if ((rv = zt_get_nw_status(ep->ze_ztn, ep->ze_nwid, &status)) != 0) { + nni_mtx_unlock(&zt_lk); + return (rv); + } + nni_mtx_unlock(&zt_lk); + return (nni_copyout_int(status, buf, szp, t)); +} + +static int +zt_ep_set_ping_time(void *arg, const void *data, size_t sz, nni_type t) +{ + zt_ep * ep = arg; + nng_duration val; + int rv; + + if ((rv = nni_copyin_ms(&val, data, sz, t)) == 0) { + nni_mtx_lock(&zt_lk); + ep->ze_ping_time = val; + nni_mtx_unlock(&zt_lk); + } + return (rv); +} + +static int +zt_ep_get_ping_time(void *arg, void *data, size_t *szp, nni_type t) +{ + zt_ep *ep = arg; + int rv; + + nni_mtx_lock(&zt_lk); + rv = nni_copyout_ms(ep->ze_ping_time, data, szp, t); + nni_mtx_unlock(&zt_lk); + return (rv); +} + +static int +zt_ep_set_ping_tries(void *arg, const void *data, size_t sz, nni_type t) +{ + zt_ep *ep = arg; + int val; + int rv; + + if ((rv = nni_copyin_int(&val, data, sz, 0, 1000000, t)) == 0) { + nni_mtx_lock(&zt_lk); + ep->ze_ping_tries = val; + nni_mtx_unlock(&zt_lk); + } + return (rv); +} + +static int +zt_ep_get_ping_tries(void *arg, void *data, size_t *szp, nni_type t) +{ + zt_ep *ep = arg; + int rv; + + nni_mtx_lock(&zt_lk); + rv = nni_copyout_int(ep->ze_ping_tries, data, szp, t); + nni_mtx_unlock(&zt_lk); + return (rv); +} + +static int +zt_ep_set_conn_time(void *arg, const void *data, size_t sz, nni_type t) +{ + zt_ep * ep = arg; + nng_duration val; + int rv; + + if ((rv = nni_copyin_ms(&val, data, sz, t)) == 0) { + nni_mtx_lock(&zt_lk); + ep->ze_conn_time = val; + nni_mtx_unlock(&zt_lk); + } + return (rv); +} + +static int +zt_ep_get_conn_time(void *arg, void *data, size_t *szp, nni_type t) +{ + zt_ep *ep = arg; + int rv; + + nni_mtx_lock(&zt_lk); + rv = nni_copyout_ms(ep->ze_conn_time, data, szp, t); + nni_mtx_unlock(&zt_lk); + return (rv); +} + +static int +zt_ep_set_conn_tries(void *arg, const void *data, size_t sz, nni_type t) +{ + zt_ep *ep = arg; + int val; + int rv; + + if ((rv = nni_copyin_int(&val, data, sz, 0, 1000000, t)) == 0) { + nni_mtx_lock(&zt_lk); + ep->ze_conn_tries = val; + nni_mtx_unlock(&zt_lk); + } + return (rv); +} + +static int +zt_ep_get_conn_tries(void *arg, void *data, size_t *szp, nni_type t) +{ + zt_ep *ep = arg; + int rv; + + nni_mtx_lock(&zt_lk); + rv = nni_copyout_int(ep->ze_conn_tries, data, szp, t); + nni_mtx_unlock(&zt_lk); + return (rv); +} + +static int +zt_ep_get_locaddr(void *arg, void *data, size_t *szp, nni_type t) +{ + zt_ep * ep = arg; + nng_sockaddr sa; + + memset(&sa, 0, sizeof(sa)); + sa.s_zt.sa_family = NNG_AF_ZT; + nni_mtx_lock(&zt_lk); + sa.s_zt.sa_nwid = ep->ze_nwid; + sa.s_zt.sa_nodeid = ep->ze_laddr >> zt_port_shift; + sa.s_zt.sa_port = ep->ze_laddr & zt_port_mask; + nni_mtx_unlock(&zt_lk); + return (nni_copyout_sockaddr(&sa, data, szp, t)); +} + +static int +zt_pipe_get_locaddr(void *arg, void *data, size_t *szp, nni_type t) +{ + zt_pipe * p = arg; + nng_sockaddr sa; + + memset(&sa, 0, sizeof(sa)); + sa.s_zt.sa_family = NNG_AF_ZT; + sa.s_zt.sa_nwid = p->zp_nwid; + sa.s_zt.sa_nodeid = p->zp_laddr >> zt_port_shift; + sa.s_zt.sa_port = p->zp_laddr & zt_port_mask; + return (nni_copyout_sockaddr(&sa, data, szp, t)); +} + +static int +zt_pipe_get_remaddr(void *arg, void *data, size_t *szp, nni_type t) +{ + zt_pipe * p = arg; + nng_sockaddr sa; + + memset(&sa, 0, sizeof(sa)); + sa.s_zt.sa_family = NNG_AF_ZT; + sa.s_zt.sa_nwid = p->zp_nwid; + sa.s_zt.sa_nodeid = p->zp_raddr >> zt_port_shift; + sa.s_zt.sa_port = p->zp_raddr & zt_port_mask; + return (nni_copyout_sockaddr(&sa, data, szp, t)); +} + +static int +zt_pipe_get_mtu(void *arg, void *data, size_t *szp, nni_type t) +{ + zt_pipe *p = arg; + return (nni_copyout_size(p->zp_mtu, data, szp, t)); +} + +static const nni_option zt_pipe_options[] = { + { + .o_name = NNG_OPT_LOCADDR, + .o_get = zt_pipe_get_locaddr, + }, + { + .o_name = NNG_OPT_REMADDR, + .o_get = zt_pipe_get_remaddr, + }, + { + .o_name = NNG_OPT_ZT_MTU, + .o_get = zt_pipe_get_mtu, + }, + { + .o_name = NNG_OPT_ZT_NWID, + .o_get = zt_pipe_get_nwid, + }, + { + .o_name = NNG_OPT_ZT_NODE, + .o_get = zt_pipe_get_node, + }, + { + .o_name = NNG_OPT_RECVMAXSZ, + .o_get = zt_pipe_get_recvmaxsz, + }, + // terminate list + { + .o_name = NULL, + }, +}; + +static int +zt_pipe_getopt(void *arg, const char *name, void *buf, size_t *szp, nni_type t) +{ + zt_pipe *p = arg; + return (nni_getopt(zt_pipe_options, name, p, buf, szp, t)); +} + +static nni_tran_pipe_ops zt_pipe_ops = { + .p_init = zt_pipe_init, + .p_fini = zt_pipe_fini, + .p_send = zt_pipe_send, + .p_recv = zt_pipe_recv, + .p_close = zt_pipe_close, + .p_peer = zt_pipe_peer, + .p_getopt = zt_pipe_getopt, +}; + +static nni_option zt_dialer_options[] = { + { + .o_name = NNG_OPT_RECVMAXSZ, + .o_get = zt_ep_get_recvmaxsz, + .o_set = zt_ep_set_recvmaxsz, + }, + { + .o_name = NNG_OPT_URL, + .o_get = zt_ep_get_url, + }, + { + .o_name = NNG_OPT_ZT_HOME, + .o_get = zt_ep_get_home, + .o_set = zt_ep_set_home, + }, + { + .o_name = NNG_OPT_ZT_NODE, + .o_get = zt_ep_get_node, + }, + { + .o_name = NNG_OPT_ZT_NWID, + .o_get = zt_ep_get_nwid, + }, + { + .o_name = NNG_OPT_ZT_NETWORK_STATUS, + .o_get = zt_ep_get_nw_status, + }, + { + .o_name = NNG_OPT_ZT_NETWORK_NAME, + .o_get = zt_ep_get_nw_name, + }, + { + .o_name = NNG_OPT_ZT_PING_TIME, + .o_get = zt_ep_get_ping_time, + .o_set = zt_ep_set_ping_time, + }, + { + .o_name = NNG_OPT_ZT_PING_TRIES, + .o_get = zt_ep_get_ping_tries, + .o_set = zt_ep_set_ping_tries, + }, + { + .o_name = NNG_OPT_ZT_CONN_TIME, + .o_get = zt_ep_get_conn_time, + .o_set = zt_ep_set_conn_time, + }, + { + .o_name = NNG_OPT_ZT_CONN_TRIES, + .o_get = zt_ep_get_conn_tries, + .o_set = zt_ep_set_conn_tries, + }, + { + .o_name = NNG_OPT_ZT_ORBIT, + .o_set = zt_ep_set_orbit, + }, + { + .o_name = NNG_OPT_ZT_DEORBIT, + .o_set = zt_ep_set_deorbit, + }, + { + .o_name = NNG_OPT_ZT_ADD_LOCAL_ADDR, + .o_set = zt_ep_set_add_local_addr, + }, + { + .o_name = NNG_OPT_ZT_CLEAR_LOCAL_ADDRS, + .o_set = zt_ep_set_clear_local_addrs, + }, + + // terminate list + { + .o_name = NULL, + }, +}; + +static nni_option zt_listener_options[] = { + { + .o_name = NNG_OPT_RECVMAXSZ, + .o_get = zt_ep_get_recvmaxsz, + .o_set = zt_ep_set_recvmaxsz, + }, + { + .o_name = NNG_OPT_URL, + .o_get = zt_ep_get_url, + }, + { + .o_name = NNG_OPT_ZT_HOME, + .o_get = zt_ep_get_home, + .o_set = zt_ep_set_home, + }, + { + .o_name = NNG_OPT_ZT_NODE, + .o_get = zt_ep_get_node, + }, + { + .o_name = NNG_OPT_ZT_NWID, + .o_get = zt_ep_get_nwid, + }, + { + .o_name = NNG_OPT_ZT_NETWORK_STATUS, + .o_get = zt_ep_get_nw_status, + }, + { + .o_name = NNG_OPT_ZT_NETWORK_NAME, + .o_get = zt_ep_get_nw_name, + }, + { + .o_name = NNG_OPT_ZT_PING_TIME, + .o_get = zt_ep_get_ping_time, + .o_set = zt_ep_set_ping_time, + }, + { + .o_name = NNG_OPT_ZT_PING_TRIES, + .o_get = zt_ep_get_ping_tries, + .o_set = zt_ep_set_ping_tries, + }, + { + .o_name = NNG_OPT_ZT_ORBIT, + .o_set = zt_ep_set_orbit, + }, + { + .o_name = NNG_OPT_ZT_DEORBIT, + .o_set = zt_ep_set_deorbit, + }, + { + .o_name = NNG_OPT_LOCADDR, + .o_get = zt_ep_get_locaddr, + }, + // terminate list + { + .o_name = NULL, + }, +}; + +static nni_tran_dialer_ops zt_dialer_ops = { + .d_init = zt_dialer_init, + .d_fini = zt_ep_fini, + .d_connect = zt_ep_connect, + .d_close = zt_ep_close, + .d_options = zt_dialer_options, +}; + +static nni_tran_listener_ops zt_listener_ops = { + .l_init = zt_listener_init, + .l_fini = zt_ep_fini, + .l_bind = zt_ep_bind, + .l_accept = zt_ep_accept, + .l_close = zt_ep_close, + .l_options = zt_listener_options, +}; + +// This is the ZeroTier transport linkage, and should be the +// only global symbol in this entire file. +static struct nni_tran zt_tran = { + .tran_version = NNI_TRANSPORT_VERSION, + .tran_scheme = "zt", + .tran_dialer = &zt_dialer_ops, + .tran_listener = &zt_listener_ops, + .tran_pipe = &zt_pipe_ops, + .tran_init = zt_tran_init, + .tran_fini = zt_tran_fini, +}; + +int +nng_zt_register(void) +{ + return (nni_tran_register(&zt_tran)); +} diff --git a/src/sp/transport/zerotier/zthash.c b/src/sp/transport/zerotier/zthash.c new file mode 100644 index 00000000..ca46b373 --- /dev/null +++ b/src/sp/transport/zerotier/zthash.c @@ -0,0 +1,302 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// Copyright 2018 Capitar IT Group BV +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include "core/nng_impl.h" +#include "zthash.h" + +struct zt_hash_entry { + uint64_t key; + void * val; + uint32_t skips; +}; + +int +zt_hash_init(zt_hash **hp) +{ + zt_hash *h; + + if ((h = NNI_ALLOC_STRUCT(h)) == NULL) { + return (NNG_ENOMEM); + } + h->ih_entries = NULL; + h->ih_count = 0; + h->ih_load = 0; + h->ih_cap = 0; + h->ih_maxload = 0; + h->ih_minload = 0; // never shrink below this + h->ih_minval = 0; + h->ih_maxval = 0xffffffff; + h->ih_dynval = 0; + + *hp = h; + return (0); +} + +void +zt_hash_fini(zt_hash *h) +{ + if (h != NULL) { + if (h->ih_entries != NULL) { + NNI_FREE_STRUCTS(h->ih_entries, h->ih_cap); + h->ih_entries = NULL; + h->ih_cap = h->ih_count = 0; + h->ih_load = h->ih_minload = h->ih_maxload = 0; + } + + NNI_FREE_STRUCT(h); + } +} + +void +zt_hash_limits(zt_hash *h, uint64_t minval, uint64_t maxval, uint64_t start) +{ + if (start < minval) { + start = minval; + } + if (start > maxval) { + start = maxval; + } + + h->ih_minval = minval; + h->ih_maxval = maxval; + h->ih_dynval = start; + NNI_ASSERT(minval < maxval); + NNI_ASSERT(start >= minval); + NNI_ASSERT(start <= maxval); +} + +// Inspired by Python dict implementation. This probe will visit every +// cell. We always hash consecutively assigned IDs. +#define ZT_HASH_NEXT(h, j) ((((j) *5) + 1) & (h->ih_cap - 1)) +#define ZT_HASH_INDEX(h, j) ((j) & (h->ih_cap - 1)) + +static size_t +zt_hash_find_index(zt_hash *h, uint64_t id) +{ + size_t index; + size_t start; + if (h->ih_count == 0) { + return ((size_t) -1); + } + + index = ZT_HASH_INDEX(h, id); + start = index; + for (;;) { + // The value of ihe_key is only valid if ihe_val is not NULL. + if ((h->ih_entries[index].key == id) && + (h->ih_entries[index].val != NULL)) { + return (index); + } + if (h->ih_entries[index].skips == 0) { + return ((size_t) -1); + } + index = ZT_HASH_NEXT(h, index); + + if (index == start) { + break; + } + } + + return ((size_t) -1); +} + +int +zt_hash_find(zt_hash *h, uint64_t id, void **vp) +{ + size_t index; + if ((index = zt_hash_find_index(h, id)) == (size_t) -1) { + return (NNG_ENOENT); + } + *vp = h->ih_entries[index].val; + return (0); +} + +static int +zt_hash_resize(zt_hash *h) +{ + size_t newsize; + size_t oldsize; + zt_hash_entry *newents; + zt_hash_entry *oldents; + uint32_t i; + + if ((h->ih_load < h->ih_maxload) && (h->ih_load >= h->ih_minload)) { + // No resize needed. + return (0); + } + + oldsize = h->ih_cap; + + newsize = 8; + while (newsize < (h->ih_count * 2)) { + newsize *= 2; + } + if (newsize == oldsize) { + // Same size. + return (0); + } + + oldents = h->ih_entries; + newents = NNI_ALLOC_STRUCTS(newents, newsize); + if (newents == NULL) { + return (NNG_ENOMEM); + } + + h->ih_entries = newents; + h->ih_cap = newsize; + h->ih_load = 0; + if (newsize > 8) { + h->ih_minload = newsize / 8; + h->ih_maxload = newsize * 2 / 3; + } else { + h->ih_minload = 0; + h->ih_maxload = 5; + } + for (i = 0; i < oldsize; i++) { + size_t index; + if (oldents[i].val == NULL) { + continue; + } + index = oldents[i].key & (newsize - 1); + for (;;) { + // Increment the load unconditionally. It counts + // once for every item stored, plus once for each + // hashing operation we use to store the item (i.e. + // one for the item, plus once for each rehash.) + h->ih_load++; + if (newents[index].val == NULL) { + // As we are hitting this entry for the first + // time, it won't have any skips. + NNI_ASSERT(newents[index].skips == 0); + newents[index].val = oldents[i].val; + newents[index].key = oldents[i].key; + break; + } + newents[index].skips++; + index = ZT_HASH_NEXT(h, index); + } + } + if (oldsize != 0) { + NNI_FREE_STRUCTS(oldents, oldsize); + } + return (0); +} + +int +zt_hash_remove(zt_hash *h, uint64_t id) +{ + size_t index; + size_t probe; + + if ((index = zt_hash_find_index(h, id)) == (size_t) -1) { + return (NNG_ENOENT); + } + + // Now we have found the index where the object exists. We are going + // to restart the search, until the index matches, to decrement the + // skips counter. + probe = (int) ZT_HASH_INDEX(h, id); + + for (;;) { + zt_hash_entry *entry; + // The load was increased once each hashing operation we used + // to place the the item. Decrement it accordingly. + h->ih_load--; + entry = &h->ih_entries[probe]; + if (probe == index) { + entry->val = NULL; + entry->key = 0; + break; + } + NNI_ASSERT(entry->skips > 0); + entry->skips--; + probe = ZT_HASH_NEXT(h, probe); + } + + h->ih_count--; + + // Shrink -- but it's ok if we can't. + (void) zt_hash_resize(h); + + return (0); +} + +int +zt_hash_insert(zt_hash *h, uint64_t id, void *val) +{ + size_t index; + zt_hash_entry *ent; + + // Try to resize -- if we don't need to, this will be a no-op. + if (zt_hash_resize(h) != 0) { + return (NNG_ENOMEM); + } + + // If it already exists, just overwrite the old value. + if ((index = zt_hash_find_index(h, id)) != (size_t) -1) { + ent = &h->ih_entries[index]; + ent->val = val; + return (0); + } + + index = ZT_HASH_INDEX(h, id); + for (;;) { + ent = &h->ih_entries[index]; + + // Increment the load count. We do this each time time we + // rehash. This may over-count items that collide on the + // same rehashing, but this should just cause a table to + // grow sooner, which is probably a good thing. + h->ih_load++; + if (ent->val == NULL) { + h->ih_count++; + ent->key = id; + ent->val = val; + return (0); + } + // Record the skip count. This being non-zero informs + // that a rehash will be necessary. Without this we + // would need to scan the entire hash for the match. + ent->skips++; + index = ZT_HASH_NEXT(h, index); + } +} + +int +zt_hash_alloc(zt_hash *h, uint64_t *idp, void *val) +{ + uint64_t id; + int rv; + + NNI_ASSERT(val != NULL); + + if (h->ih_count > (h->ih_maxval - h->ih_minval)) { + // Really more like ENOSPC.. the table is filled to max. + return (NNG_ENOMEM); + } + + for (;;) { + id = h->ih_dynval; + h->ih_dynval++; + if (h->ih_dynval > h->ih_maxval) { + h->ih_dynval = h->ih_minval; + } + + if (zt_hash_find_index(h, id) == (size_t) -1) { + break; + } + } + + rv = zt_hash_insert(h, id, val); + if (rv == 0) { + *idp = id; + } + return (rv); +} diff --git a/src/sp/transport/zerotier/zthash.h b/src/sp/transport/zerotier/zthash.h new file mode 100644 index 00000000..249eabbf --- /dev/null +++ b/src/sp/transport/zerotier/zthash.h @@ -0,0 +1,43 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// Copyright 2018 Capitar IT Group BV +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#ifndef ZT_HASH_H +#define ZT_HASH_H + +#include + +// This code is derived from id hash, but supports 64-bit IDs. + +typedef struct zt_hash zt_hash; +typedef struct zt_hash_entry zt_hash_entry; + +// NB: These details are entirely private to the hash implementation. +// They are provided here to facilitate inlining in structures. +struct zt_hash { + size_t ih_cap; + size_t ih_count; + size_t ih_load; + size_t ih_minload; // considers placeholders + size_t ih_maxload; + uint64_t ih_minval; + uint64_t ih_maxval; + uint64_t ih_dynval; + zt_hash_entry *ih_entries; +}; + +extern int zt_hash_init(zt_hash **); +extern void zt_hash_fini(zt_hash *); +extern void zt_hash_limits(zt_hash *, uint64_t, uint64_t, uint64_t); +extern int zt_hash_find(zt_hash *, uint64_t, void **); +extern int zt_hash_remove(zt_hash *, uint64_t); +extern int zt_hash_insert(zt_hash *, uint64_t, void *); +extern int zt_hash_alloc(zt_hash *, uint64_t *, void *); + +#endif // CORE_IDHASH_H diff --git a/src/supplemental/CMakeLists.txt b/src/supplemental/CMakeLists.txt new file mode 100644 index 00000000..5be439b1 --- /dev/null +++ b/src/supplemental/CMakeLists.txt @@ -0,0 +1,17 @@ +# +# Copyright 2021 Staysail Systems, Inc. +# +# This software is supplied under the terms of the MIT License, a +# copy of which should be located in the distribution where this +# file was obtained (LICENSE.txt). A copy of the license may also be +# found online at https://opensource.org/licenses/MIT. +# + +nng_directory(supplemental) + +add_subdirectory(base64) +add_subdirectory(http) +add_subdirectory(sha1) +add_subdirectory(tls) +add_subdirectory(util) +add_subdirectory(websocket) \ No newline at end of file diff --git a/src/tools/CMakeLists.txt b/src/tools/CMakeLists.txt new file mode 100644 index 00000000..0487d9ec --- /dev/null +++ b/src/tools/CMakeLists.txt @@ -0,0 +1,13 @@ +# +# Copyright 2021 Staysail Systems, Inc. +# +# This software is supplied under the terms of the MIT License, a +# copy of which should be located in the distribution where this +# file was obtained (LICENSE.txt). A copy of the license may also be +# found online at https://opensource.org/licenses/MIT. +# + +nng_directory(tools) + +add_subdirectory(nngcat) +add_subdirectory(perf) diff --git a/src/tools/nngcat/CMakeLists.txt b/src/tools/nngcat/CMakeLists.txt new file mode 100644 index 00000000..d522a045 --- /dev/null +++ b/src/tools/nngcat/CMakeLists.txt @@ -0,0 +1,41 @@ +# +# Copyright 2021 Staysail Systems, Inc. +# Copyright 2018 Capitar IT Group BV +# +# This software is supplied under the terms of the MIT License, a +# copy of which should be located in the distribution where this +# file was obtained (LICENSE.txt). A copy of the license may also be +# found online at https://opensource.org/licenses/MIT. +# + +nng_directory(nngcat) + +if (NNG_ENABLE_NNGCAT) + add_executable(nngcat nngcat.c) + target_include_directories(nngcat PUBLIC ${PROJECT_SOURCE_DIR}/src) + target_link_libraries(nngcat nng nng_private) + install(TARGETS nngcat RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} + COMPONENT Tools) + + if (NNG_TESTS AND CMAKE_SYSTEM_NAME MATCHES "Linux") + include(FindUnixCommands) + endif () + # TODO: This should be refactored to use a test driver. + # We only run the tests on Linux for now, because the Darwin CI/CD is too brittle. + if (NNG_TESTS AND BASH AND CMAKE_SYSTEM_NAME MATCHES "Linux") + macro(add_nngcat_test NAME TIMEOUT) + add_test(NAME nng.${NAME} COMMAND ${BASH} ${CMAKE_CURRENT_SOURCE_DIR}/${NAME}_test.sh $) + set_tests_properties(nng.${NAME} PROPERTIES TIMEOUT ${TIMEOUT}) + endmacro() + add_nngcat_test(nngcat_async 10) + add_nngcat_test(nngcat_ambiguous 10) + add_nngcat_test(nngcat_need_proto 10) + add_nngcat_test(nngcat_dup_proto 10) + add_nngcat_test(nngcat_help 10) + add_nngcat_test(nngcat_incompat 10) + add_nngcat_test(nngcat_pubsub 20) + add_nngcat_test(nngcat_recvmaxsz 20) + add_nngcat_test(nngcat_unlimited 20) + add_nngcat_test(nngcat_stdin_pipe 20) + endif () +endif () diff --git a/src/tools/nngcat/nngcat.c b/src/tools/nngcat/nngcat.c new file mode 100644 index 00000000..9e106069 --- /dev/null +++ b/src/tools/nngcat/nngcat.c @@ -0,0 +1,1217 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// Copyright 2018 Capitar IT Group BV +// Copyright 2020 Lager Data, Inc. +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// Globals. We need this to avoid passing around everything. +int format = 0; +int proto = 0; +int verbose = 0; +int delay = 0; +nng_duration interval = NNG_DURATION_INFINITE; +nng_duration sendtimeo = NNG_DURATION_INFINITE; +nng_duration recvtimeo = NNG_DURATION_INFINITE; +void * data = NULL; +size_t datalen = 0; +int compat = 0; +int async = 0; +int insecure = 0; +void * cacert = NULL; +size_t cacertlen = 0; +void * keyfile = NULL; +size_t keylen = 0; +void * certfile = NULL; +size_t certlen = 0; +const char * zthome = NULL; +int count = 0; +int recvmaxsz = -1; + +// Options, must start at 1 because zero is sentinel. +enum options { + OPT_HELP = 1, + OPT_VERBOSE, + OPT_SILENT, + OPT_REP0, + OPT_REQ0, + OPT_PUSH0, + OPT_PULL0, + OPT_PUB0, + OPT_SUB0, + OPT_SURVEY0, + OPT_RESPOND0, + OPT_PAIR0, + OPT_PAIR1, + OPT_PAIR, + OPT_BUS0, + OPT_DIAL, + OPT_LISTEN, + OPT_DATA, + OPT_FILE, + OPT_SUBSCRIBE, + OPT_INTERVAL, + OPT_DELAY, + OPT_COUNT, + OPT_FORMAT, + OPT_RAW, + OPT_ASCII, + OPT_QUOTED, + OPT_MSGPACK, + OPT_HEX, + OPT_BLANK, // no printing, not an actual option. + OPT_COMPAT, + OPT_ASYNC, + OPT_RCV_TIMEO, + OPT_SND_TIMEO, + OPT_SOCK_NAME, + OPT_LISTEN_IPC, + OPT_DIAL_IPC, + OPT_LISTEN_LOCAL, + OPT_DIAL_LOCAL, + OPT_INSECURE, + OPT_CACERT, + OPT_KEYFILE, + OPT_CERTFILE, + OPT_VERSION, + OPT_RECVMAXSZ, + OPT_ZTHOME, +}; + +static nng_optspec opts[] = { + { .o_name = "help", .o_short = 'h', .o_val = OPT_HELP }, + { .o_name = "verbose", .o_short = 'v', .o_val = OPT_VERBOSE }, + { .o_name = "silent", .o_short = 'q', .o_val = OPT_SILENT }, + { .o_name = "req0", .o_val = OPT_REQ0 }, + { .o_name = "rep0", .o_val = OPT_REP0 }, + { .o_name = "push0", .o_val = OPT_PUSH0 }, + { .o_name = "pull0", .o_val = OPT_PULL0 }, + { .o_name = "pub0", .o_val = OPT_PUB0 }, + { .o_name = "sub", .o_val = OPT_SUB0 }, // explicit for alias + { .o_name = "sub0", .o_val = OPT_SUB0 }, + { .o_name = "surveyor0", .o_val = OPT_SURVEY0 }, + { .o_name = "respondent0", .o_val = OPT_RESPOND0 }, + { .o_name = "pair", .o_val = OPT_PAIR }, + { .o_name = "pair0", .o_val = OPT_PAIR0 }, + { .o_name = "pair1", .o_val = OPT_PAIR1 }, + { .o_name = "bus0", .o_val = OPT_BUS0 }, + { .o_name = "dial", .o_val = OPT_DIAL, .o_arg = true }, + { .o_name = "listen", .o_val = OPT_LISTEN, .o_arg = true }, + { .o_name = "data", .o_short = 'D', .o_val = OPT_DATA, .o_arg = true }, + { .o_name = "file", .o_short = 'F', .o_val = OPT_FILE, .o_arg = true }, + { .o_name = "subscribe", .o_val = OPT_SUBSCRIBE, .o_arg = true }, + { .o_name = "format", .o_val = OPT_FORMAT, .o_arg = true }, + { .o_name = "ascii", .o_short = 'A', .o_val = OPT_ASCII }, + { .o_name = "quoted", .o_short = 'Q', .o_val = OPT_QUOTED }, + { .o_name = "raw", .o_val = OPT_RAW }, + { .o_name = "hex", .o_val = OPT_HEX }, + { .o_name = "compat", .o_val = OPT_COMPAT }, + { .o_name = "async", .o_val = OPT_ASYNC }, + { .o_name = "msgpack", .o_val = OPT_MSGPACK }, + + { + .o_name = "recv-maxsz", + .o_short = 'Z', + .o_val = OPT_RECVMAXSZ, + .o_arg = true, + }, + { + .o_name = "count", + .o_short = 'C', + .o_val = OPT_COUNT, + .o_arg = true, + }, + { + .o_name = "delay", + .o_short = 'd', + .o_val = OPT_DELAY, + .o_arg = true, + }, + { + .o_name = "interval", + .o_short = 'i', + .o_val = OPT_INTERVAL, + .o_arg = true, + }, + { .o_name = "recv-timeout", .o_val = OPT_RCV_TIMEO, .o_arg = true }, + { .o_name = "send-timeout", .o_val = OPT_SND_TIMEO, .o_arg = true }, + { .o_name = "socket-name", .o_val = OPT_SOCK_NAME, .o_arg = true }, + + // Legacy compatibility with nanocat. + { .o_name = "bind", .o_val = OPT_LISTEN, .o_arg = true }, + { .o_name = "connect", .o_val = OPT_DIAL, .o_arg = true }, + { + .o_name = "bind-ipc", + .o_short = 'X', + .o_val = OPT_LISTEN_IPC, + .o_arg = true, + }, + { + .o_name = "connect-ipc", + .o_short = 'x', + .o_val = OPT_DIAL_IPC, + .o_arg = true, + }, + { + .o_name = "bind-local", + .o_short = 'L', + .o_val = OPT_LISTEN_LOCAL, + .o_arg = true, + }, + { + .o_name = "connect-local", + .o_short = 'l', + .o_val = OPT_DIAL_LOCAL, + .o_arg = true, + }, + { .o_name = "insecure", .o_short = 'k', .o_val = OPT_INSECURE }, + { .o_name = "cacert", .o_val = OPT_CACERT, .o_arg = true }, + { .o_name = "key", .o_val = OPT_KEYFILE, .o_arg = true }, + { + .o_name = "cert", + .o_short = 'E', + .o_val = OPT_CERTFILE, + .o_arg = true, + }, + { + .o_name = "zt-home", + .o_val = OPT_ZTHOME, + .o_arg = true, + }, + { .o_name = "version", .o_short = 'V', .o_val = OPT_VERSION }, + + // Sentinel. + { .o_name = NULL, .o_val = 0 }, +}; + +static void +fatal(const char *msg, ...) +{ + va_list ap; + va_start(ap, msg); + vfprintf(stderr, msg, ap); + va_end(ap); + fprintf(stderr, "\n"); + exit(1); +} + +static void +help(void) +{ + printf( + "Usage: nngcat ... [] [...] []\n\n"); + printf(" must be one of:\n"); + printf(" --req0 (or alias --req)\n"); + printf(" --rep0 (or alias --rep)\n"); + printf(" --pub0 (or alias --pub)\n"); + printf(" --sub0 (or alias --sub)\n"); + printf(" --push0 (or alias --push)\n"); + printf(" --pull0 (or alias --pull)\n"); + printf(" --surveyor0 (or alias --surveyor)\n"); + printf(" --respondent0 (or alias --respondent)\n"); + printf(" --pair0\n"); + printf(" --pair1\n"); + printf(" --pair (alias for either pair0 or pair1)\n"); + printf("\n must be one or more of:\n"); + printf(" --dial (or alias --connect )\n"); + printf(" --listen (or alias --bind )\n"); + printf(" --bind-local (or alias -L )\n"); + printf(" --connect-local (or alias -l )\n"); + printf(" --bind-ipc (or alias -X )\n"); + printf(" --connect-ipc (or alias -x )\n"); + printf("\n may be one or none of:\n"); + printf(" --format \n"); + printf(" --ascii (or alias -A)\n"); + printf(" --quoted (or alias -Q)\n"); + printf(" --hex\n"); + printf(" --msgpack\n"); + printf(" --raw\n"); + printf("\n may be any of:\n"); + printf(" --subscribe (only with --sub protocol)\n"); + printf(" --silent (or alias -q)\n"); + printf(" --verbose (or alias -v)\n"); + printf(" --count (or alias -C )\n"); + printf(" --delay (or alias -d )\n"); + printf(" --interval (or alias -i )\n"); + printf(" --recv-timeout \n"); + printf(" --send-timeout \n"); + printf(" --recv-maxsz (or alias -Z )\n"); + printf(" --compat\n"); + printf(" --async\n"); + printf(" --insecure (or alias -k)\n"); + printf(" --cacert \n"); + printf(" --cert (or alias -E )\n"); + printf(" --key \n"); + printf(" --zt-home \n"); + printf("\n may be one of:\n"); + printf(" --file (or alias -F ). " + "Use - for standard input.\n"); + printf(" --data (or alias -D )\n"); + exit(1); +} + +static int +intarg(const char *val, int maxv) +{ + int v = 0; + + if (val[0] == '\0') { + fatal("Empty integer argument."); + } + while (*val != '\0') { + if (!isdigit(*val)) { + fatal("Integer argument expected."); + } + v *= 10; + v += ((*val) - '0'); + val++; + if (v > maxv) { + fatal("Integer argument too large."); + } + } + if (v < 0) { + fatal("Integer argument overflow."); + } + return (v); +} + +// This reads a file into memory. Care is taken to ensure that +// the buffer is one byte larger and contains a terminating +// NUL. (Useful for key files and such.) +static void +loadfile(const char *path, void **datap, size_t *lenp) +{ + FILE * f; + size_t total_read = 0; + size_t allocation_size = BUFSIZ; + char * fdata; + char * realloc_result; + + if (strcmp(path, "-") == 0) { + f = stdin; + } else { + if ((f = fopen(path, "rb")) == NULL) { + fatal( + "Cannot open file %s: %s", path, strerror(errno)); + } + } + + if ((fdata = malloc(allocation_size + 1)) == NULL) { + fatal("Out of memory."); + } + + while (1) { + total_read += fread( + fdata + total_read, 1, allocation_size - total_read, f); + if (ferror(f)) { + if (errno == EINTR) { + continue; + } + fatal( + "Read from %s failed: %s", path, strerror(errno)); + } + if (feof(f)) { + break; + } + if (total_read == allocation_size) { + if (allocation_size > SIZE_MAX / 2) { + fatal("Out of memory."); + } + allocation_size *= 2; + if ((realloc_result = realloc( + fdata, allocation_size + 1)) == NULL) { + free(fdata); + fatal("Out of memory."); + } + fdata = realloc_result; + } + } + if (f != stdin) { + fclose(f); + } + fdata[total_read] = '\0'; + *datap = fdata; + *lenp = total_read; +} + +static void +configtls(nng_tls_config *tls) +{ + int rv = 0; + if (insecure) { + rv = nng_tls_config_auth_mode(tls, NNG_TLS_AUTH_MODE_NONE); + } + if ((rv == 0) && (certfile != NULL)) { + keyfile = keyfile ? keyfile : certfile; + rv = nng_tls_config_own_cert(tls, certfile, keyfile, NULL); + } + if ((rv == 0) && (cacert != NULL)) { + rv = nng_tls_config_ca_chain(tls, cacert, NULL); + } + if (rv != 0) { + fatal("Unable to configure TLS: %s", nng_strerror(rv)); + } +} + +struct addr { + struct addr *next; + int mode; + char * val; +}; + +struct addr ** +addaddr(struct addr **endp, int mode, const char *a) +{ + struct addr *na; + + if (((na = malloc(sizeof(*na))) == NULL) || + ((na->val = malloc(strlen(a) + 1)) == NULL)) { + fatal("Out of memory."); + } + na->mode = mode; + memcpy(na->val, a, strlen(a) + 1); + na->next = NULL; + *endp = na; + return (&na->next); +} + +struct topic { + struct topic *next; + char * val; +}; + +struct topic ** +addtopic(struct topic **endp, const char *s) +{ + struct topic *t; + + if (((t = malloc(sizeof(*t))) == NULL) || + ((t->val = malloc(strlen(s) + 1)) == NULL)) { + fatal("Out of memory."); + } + memcpy(t->val, s, strlen(s) + 1); + t->next = NULL; + *endp = t; + return (&t->next); +} + +void +printmsg(char *buf, size_t len) +{ + switch (format) { + case OPT_BLANK: // Suppress contents. + return; + case OPT_RAW: // Just emit raw contents. + fwrite(buf, 1, len, stdout); + break; + case OPT_ASCII: // ASCII, but non-ASCII substituted with '.' + for (size_t i = 0; i < len; i++) { + if (isprint(buf[i])) { + putchar(buf[i]); + } else { + putchar('.'); + } + } + break; + case OPT_QUOTED: // C style quoted strings. + putchar('"'); + for (size_t i = 0; i < len; i++) { + switch (buf[i]) { + case '\n': + putchar('\\'); + putchar('n'); + break; + case '\r': + putchar('\\'); + putchar('r'); + break; + case '\t': + putchar('\\'); + putchar('t'); + break; + case '"': + case '\\': + putchar('\\'); + putchar(buf[i]); + break; + default: + if (isprint(buf[i])) { + fputc(buf[i], stdout); + } else { + printf("\\x%02x", (uint8_t) buf[i]); + } + } + } + putchar('"'); + putchar('\n'); + break; + case OPT_MSGPACK: // MsgPack, we just encode prefix + len, then + // raw. + if (len < 256) { + putchar('\xc4'); + putchar(len & 0xff); + } else if (len < 65536) { + putchar('\xc5'); + putchar((len >> 8) & 0xff); + putchar(len & 0xff); + } else { + putchar('\xc6'); + putchar((len >> 24) & 0xff); + putchar((len >> 16) & 0xff); + putchar((len >> 8) & 0xff); + putchar(len & 0xff); + } + fwrite(buf, 1, len, stdout); + break; + case OPT_HEX: // Hex, quoted C string encoded with hex + // literals. + putchar('"'); + for (size_t i = 0; i < len; i++) { + printf("\\x%02x", (uint8_t) buf[i]); + } + putchar('"'); + putchar('\n'); + break; + } + fflush(stdout); +} + +void +recvloop(nng_socket sock) +{ + int iters = 0; + for (;;) { + int rv; + nng_msg *msg; + + if ((count > 0) && (iters >= count)) { + break; + } + rv = nng_recvmsg(sock, &msg, 0); + iters++; + switch (rv) { + case NNG_ETIMEDOUT: + case NNG_ESTATE: + // Either a regular timeout, or we reached the + // end of an event like a survey completing. + return; + case 0: + printmsg(nng_msg_body(msg), nng_msg_len(msg)); + nng_msg_free(msg); + continue; + default: + fatal("Receive error: %s", nng_strerror(rv)); + } + } +} + +void +resploop(nng_socket sock) +{ + int iters = 0; + for (;;) { + int rv; + nng_msg *msg; + + rv = nng_recvmsg(sock, &msg, 0); + if (rv != 0) { + fatal("Receive error: %s", nng_strerror(rv)); + } + printmsg(nng_msg_body(msg), nng_msg_len(msg)); + nng_msg_clear(msg); + if ((rv = nng_msg_append(msg, data, datalen)) != 0) { + fatal(nng_strerror(rv)); + } + if ((rv = nng_sendmsg(sock, msg, 0)) != 0) { + fatal("Send error: %s", nng_strerror(rv)); + } + + iters++; + if ((count > 0) && (iters >= count)) { + break; + } + } + + nng_msleep(200); +} + +void +sendloop(nng_socket sock) +{ + int iters = 0; + + if (data == NULL) { + fatal("No data to send (specify with --data or --file)"); + } + if (delay > 0) { + nng_msleep(delay); + } + + for (;;) { + int rv; + nng_msg * msg; + nng_time start; + nng_time end; + nng_duration delta; + + start = nng_clock(); + if (((rv = nng_msg_alloc(&msg, 0)) != 0) || + ((rv = nng_msg_append(msg, data, datalen)) != 0)) { + fatal(nng_strerror(rv)); + } + if ((rv = nng_sendmsg(sock, msg, 0)) != 0) { + fatal("Send error: %s", nng_strerror(rv)); + } + end = nng_clock(); + delta = (nng_duration)(end - start); + + iters++; + // By default, we don't loop. + if (((interval < 0) && (count == 0)) || + ((count > 0) && (iters >= count))) { + break; + } + + // We sleep, but we account for time spent, so that our + // interval appears more or less constant. Of course + // if we took more than the interval here, then we skip + // the sleep altogether. + if ((delta >= 0) && (delta < interval)) { + nng_msleep(interval - delta); + } + } + // Wait a bit to give queues a chance to drain. + nng_msleep(200); +} + +void +sendrecv(nng_socket sock) +{ + int iters = 0; + + if (data == NULL) { + fatal("No data to send (specify with --data or --file)"); + } + if (delay > 0) { + nng_msleep(delay); + } + + // We start by sending a message, then we receive iteratively + // but we set a concrete timeout if interval is set, to ensure + // that we exit the receive loop, and can continue. + for (;;) { + int rv; + nng_msg * msg; + nng_time start; + nng_time end; + nng_duration delta; + + start = nng_clock(); + if (((rv = nng_msg_alloc(&msg, 0)) != 0) || + ((rv = nng_msg_append(msg, data, datalen)) != 0)) { + fatal(nng_strerror(rv)); + } + if ((rv = nng_sendmsg(sock, msg, 0)) != 0) { + fatal("Send error: %s", nng_strerror(rv)); + } + if ((interval < 0) && (count == 0)) { + // Only one iteration through. + recvloop(sock); + break; + } + + // We would like to use recvloop, but we need to reset + // our timeout each time, as the timer counts down + // towards zero. Furthermore, with survey, we don't + // want to increment the iteration count. + + for (;;) { + delta = (nng_duration)(nng_clock() - start); + + nng_duration expire = interval - delta; + if ((recvtimeo >= 0) && (expire > recvtimeo)) { + expire = recvtimeo; + } + rv = + nng_socket_set_ms(sock, NNG_OPT_RECVTIMEO, expire); + if (rv != 0) { + fatal("Cannot set recv timeout: %s", + nng_strerror(rv)); + } + + rv = nng_recvmsg(sock, &msg, 0); + switch (rv) { + case 0: + printmsg(nng_msg_body(msg), nng_msg_len(msg)); + nng_msg_free(msg); + continue; + case NNG_ETIMEDOUT: + case NNG_ESTATE: + // We're done receiving + break; + default: + fatal("Cannot receive: %s", nng_strerror(rv)); + break; + } + + // We're done receiving, break out. + break; + } + + end = nng_clock(); + delta = (nng_duration)(end - start); + + iters++; + if ((count > 0) && (iters >= count)) { + break; + } + + // We sleep, but we account for time spent, so that our + // interval appears more or less constant. Of course + // if we took more than the interval here, then we skip + // the sleep altogether. + if ((delta >= 0) && (delta < interval)) { + nng_msleep(interval - delta); + } + } +} + +int +main(int ac, char **av) +{ + int idx; + char * arg; + int val; + int rv; + char scratch[512]; + struct addr * addrs = NULL; + struct addr ** addrend; + struct topic * topics = NULL; + struct topic **topicend; + nng_socket sock; + int port; + + idx = 1; + addrend = &addrs; + topicend = &topics; + + while ((rv = nng_opts_parse(ac, av, opts, &val, &arg, &idx)) == 0) { + switch (val) { + case OPT_HELP: + help(); + break; + case OPT_REQ0: + case OPT_REP0: + case OPT_SUB0: + case OPT_PUB0: + case OPT_BUS0: + case OPT_SURVEY0: + case OPT_RESPOND0: + case OPT_PAIR0: + case OPT_PAIR1: + case OPT_PUSH0: + case OPT_PULL0: + if (proto != 0) { + fatal("Only one protocol may be " + "specified."); + } + proto = val; + break; + case OPT_DIAL: + case OPT_LISTEN: + addrend = addaddr(addrend, val, arg); + break; + case OPT_DIAL_LOCAL: + case OPT_LISTEN_LOCAL: + port = intarg(arg, 65536); + snprintf(scratch, sizeof(scratch), + "tcp://127.0.0.1:%d", port); + addrend = addaddr(addrend, val, scratch); + break; + case OPT_DIAL_IPC: + case OPT_LISTEN_IPC: + snprintf(scratch, sizeof(scratch), "ipc:///%s", arg); + addrend = addaddr(addrend, val, scratch); + break; + case OPT_COUNT: + count = intarg(arg, 0x7fffffff); + break; + case OPT_SUBSCRIBE: + topicend = addtopic(topicend, arg); + break; + case OPT_VERBOSE: + case OPT_SILENT: + verbose = val; + break; + case OPT_DELAY: + delay = intarg(arg, 86400) * 1000; // max 1 day + break; + case OPT_INTERVAL: + interval = intarg(arg, 86400) * 1000; // max 1 day + break; + case OPT_SND_TIMEO: + sendtimeo = intarg(arg, 86400) * 1000; // max 1 day + break; + case OPT_RCV_TIMEO: + recvtimeo = intarg(arg, 86400) * 1000; // max 1 day + break; + case OPT_RECVMAXSZ: + recvmaxsz = intarg(arg, 0x7fffffff); + if (recvmaxsz == 0) { + recvmaxsz = 0x7fffffff; + } + break; + case OPT_COMPAT: + compat = 1; + break; + case OPT_ASYNC: + async = NNG_FLAG_NONBLOCK; + break; + case OPT_ASCII: + case OPT_RAW: + case OPT_QUOTED: + case OPT_MSGPACK: + case OPT_HEX: + if (format != 0) { + fatal("Format may be specified only " + "once."); + } + format = val; + break; + case OPT_FORMAT: + if (format != 0) { + fatal("Format may be specified only " + "once."); + } + if (strcmp(arg, "no") == 0) { + format = OPT_BLANK; + } else if (strcmp(arg, "ascii") == 0) { + format = OPT_ASCII; + } else if (strcmp(arg, "hex") == 0) { + format = OPT_HEX; + } else if (strcmp(arg, "quoted") == 0) { + format = OPT_QUOTED; + } else if (strcmp(arg, "raw") == 0) { + format = OPT_RAW; + } else if (strcmp(arg, "msgpack") == 0) { + format = OPT_MSGPACK; + } else { + fatal("Invalid format specified."); + } + break; + case OPT_FILE: + if (data != NULL) { + fatal("Data (--file, --data) may be " + "specified " + "only once."); + } + loadfile(arg, &data, &datalen); + break; + case OPT_DATA: + if (data != NULL) { + fatal("Data (--file, --data) may be " + "specified " + "only once."); + } + if ((data = malloc(strlen(arg) + 1)) == NULL) { + fatal("Out of memory."); + } + memcpy(data, arg, strlen(arg) + 1); + datalen = strlen(arg); + break; + case OPT_CACERT: + if (cacert != NULL) { + fatal("CA Certificate (--cacert) may be " + "specified only once."); + } + loadfile(arg, &cacert, &cacertlen); + break; + case OPT_KEYFILE: + if (keyfile != NULL) { + fatal( + "Key (--key) may be specified only once."); + } + loadfile(arg, &keyfile, &keylen); + break; + case OPT_CERTFILE: + if (certfile != NULL) { + fatal("Cert (--cert) may be specified " + "only " + "once."); + } + loadfile(arg, &certfile, &certlen); + break; + case OPT_ZTHOME: + zthome = arg; + break; + case OPT_INSECURE: + insecure = 1; + break; + case OPT_VERSION: + printf("%s\n", nng_version()); + exit(0); + } + } + switch (rv) { + case NNG_EINVAL: + fatal("Option %s is invalid.", av[idx]); + break; + case NNG_EAMBIGUOUS: + fatal("Option %s is ambiguous (specify in full).", av[idx]); + break; + case NNG_ENOARG: + fatal("Option %s requires argument.", av[idx]); + break; + default: + break; + } + + if (addrs == NULL) { + fatal("No address specified."); + } + + if (compat) { + if (async != 0) { + fatal("Options --async and --compat are " + "incompatible."); + } + if (count != 0) { + fatal("Options --count and --compat are " + "incompatible."); + } + if (proto == OPT_PAIR) { + proto = OPT_PAIR0; + } + if (proto == OPT_PAIR1) { + fatal("Protocol does not support --compat."); + } + async = NNG_FLAG_NONBLOCK; + } else { + if (proto == OPT_PAIR) { + proto = OPT_PAIR1; + } + } + if (proto == OPT_SUB0) { + if (topics == NULL) { + (void) addtopic(topicend, ""); // subscribe to all + } + } else { + if (topics != NULL) { + fatal("Protocol does not support --subscribe."); + } + } + + switch (proto) { + case OPT_PULL0: + case OPT_SUB0: + if (data != NULL) { + fatal("Protocol does not support --file or " + "--data."); + } + if (interval >= 0) { + fatal("Protocol does not support --interval."); + } + break; + case OPT_PUSH0: + case OPT_PUB0: + if (format != 0) { + fatal("Protocol does not support --format " + "options."); + } + if (data == NULL) { + fatal("Protocol requires either --file or " + "--data."); + } + break; + case OPT_SURVEY0: + case OPT_REQ0: + if (data == NULL) { + fatal("Protocol requires either --file or " + "--data."); + } + break; + case OPT_REP0: + case OPT_RESPOND0: + if (interval >= 0) { + fatal("Protocol does not support --interval."); + } + break; + case OPT_PAIR0: + case OPT_PAIR1: + case OPT_BUS0: + if ((interval >= 0) && (data == NULL)) { + fatal("Option --interval requires either " + "--file or --data."); + } + break; + default: + // Will be caught in next switch statement. + break; + } + + switch (proto) { + case OPT_REQ0: +#ifdef NNG_HAVE_REQ0 + rv = nng_req0_open(&sock); +#else + fatal("Protocol not supported."); +#endif + break; + case OPT_REP0: +#ifdef NNG_HAVE_REP0 + rv = nng_rep0_open(&sock); +#else + fatal("Protocol not supported."); +#endif + break; + case OPT_SUB0: +#ifdef NNG_HAVE_SUB0 + rv = nng_sub0_open(&sock); +#else + fatal("Protocol not supported."); +#endif + break; + case OPT_PUB0: +#ifdef NNG_HAVE_PUB0 + rv = nng_pub0_open(&sock); +#else + fatal("Protocol not supported."); +#endif + break; + case OPT_PAIR0: +#ifdef NNG_HAVE_PAIR0 + rv = nng_pair0_open(&sock); +#else + fatal("Protocol not supported."); +#endif + break; + case OPT_PAIR1: +#ifdef NNG_HAVE_PAIR1 + rv = nng_pair1_open(&sock); +#else + fatal("Protocol not supported"); +#endif + break; + case OPT_BUS0: +#ifdef NNG_HAVE_BUS0 + rv = nng_bus0_open(&sock); +#else + fatal("Protocol not supported."); +#endif + break; + case OPT_PUSH0: +#ifdef NNG_HAVE_PUSH0 + rv = nng_push0_open(&sock); +#else + fatal("Protocol not supported."); +#endif + break; + case OPT_PULL0: +#ifdef NNG_HAVE_PULL0 + rv = nng_pull0_open(&sock); +#else + fatal("Protocol not supported."); +#endif + break; + case OPT_SURVEY0: +#ifdef NNG_HAVE_SURVEYOR0 + rv = nng_surveyor0_open(&sock); +#else + fatal("Protocol not supported."); +#endif + break; + case OPT_RESPOND0: +#ifdef NNG_HAVE_RESPONDENT0 + rv = nng_respondent0_open(&sock); +#else + fatal("Protocol not supported"); +#endif + break; + case 0: + default: + fatal("No protocol specified."); + break; + } + if (rv != 0) { + fatal("Unable to open socket: %s", nng_strerror(rv)); + } + + for (struct topic *t = topics; t != NULL; t = t->next) { + rv = nng_socket_set( + sock, NNG_OPT_SUB_SUBSCRIBE, t->val, strlen(t->val)); + if (rv != 0) { + fatal("Unable to subscribe to topic %s: %s", t->val, + nng_strerror(rv)); + } + } + + if ((sendtimeo > 0) && + ((rv = nng_socket_set_ms(sock, NNG_OPT_SENDTIMEO, sendtimeo)) != + 0)) { + fatal("Unable to set send timeout: %s", nng_strerror(rv)); + } + if ((recvtimeo > 0) && + ((rv = nng_socket_set_ms(sock, NNG_OPT_RECVTIMEO, recvtimeo)) != + 0)) { + fatal("Unable to set send timeout: %s", nng_strerror(rv)); + } + + if ((recvmaxsz >= 0) && + ((rv = nng_socket_set_size(sock, NNG_OPT_RECVMAXSZ, recvmaxsz)) != + 0)) { + fatal("Unable to set max receive size: %s", nng_strerror(rv)); + } + + for (struct addr *a = addrs; a != NULL; a = a->next) { + char * act; + nng_listener l; + nng_dialer d; + nng_tls_config *tls; + switch (a->mode) { + case OPT_DIAL: + case OPT_DIAL_IPC: + case OPT_DIAL_LOCAL: + rv = nng_dialer_create(&d, sock, a->val); + if (rv != 0) { + fatal("Unable to create dialer for %s: %s", + a->val, nng_strerror(rv)); + } + rv = nng_dialer_getopt_ptr( + d, NNG_OPT_TLS_CONFIG, (void **) &tls); + if (rv == 0) { + configtls(tls); + } else if (rv != NNG_ENOTSUP) { + fatal("Unable to get TLS config: %s", + nng_strerror(rv)); + } + if (zthome != NULL) { + rv = nng_dialer_set(d, NNG_OPT_ZT_HOME, + zthome, strlen(zthome) + 1); + if ((rv != 0) && (rv != NNG_ENOTSUP)) { + fatal("Unable to set ZT home: %s", + nng_strerror(rv)); + } + } + rv = nng_dialer_start(d, async); + act = "dial"; + if ((rv == 0) && (verbose == OPT_VERBOSE)) { + char ustr[256]; + size_t sz; + sz = sizeof(ustr); + if (nng_dialer_getopt( + d, NNG_OPT_URL, ustr, &sz) == 0) { + printf("Connected to: %s\n", ustr); + } + } + break; + case OPT_LISTEN: + case OPT_LISTEN_IPC: + case OPT_LISTEN_LOCAL: + rv = nng_listener_create(&l, sock, a->val); + if (rv != 0) { + fatal("Unable to create listener for %s: %s", + a->val, nng_strerror(rv)); + } + rv = nng_listener_getopt_ptr( + l, NNG_OPT_TLS_CONFIG, (void **) &tls); + if (rv == 0) { + configtls(tls); + } else if (rv != NNG_ENOTSUP) { + fatal("Unable to get TLS config: %s", + nng_strerror(rv)); + } + if (zthome != NULL) { + rv = nng_listener_set(l, NNG_OPT_ZT_HOME, + zthome, strlen(zthome) + 1); + if ((rv != 0) && (rv != NNG_ENOTSUP)) { + fatal("Unable to set ZT home: %s", + nng_strerror(rv)); + } + } + rv = nng_listener_start(l, async); + act = "listen"; + if ((rv == 0) && (verbose == OPT_VERBOSE)) { + char ustr[256]; + size_t sz; + sz = sizeof(ustr); + if (nng_listener_getopt( + l, NNG_OPT_URL, ustr, &sz) == 0) { + printf("Listening at: %s\n", ustr); + } + } + break; + default: + fatal("Invalid address mode! (Bug!)"); + } + + if (rv != 0) { + fatal("Unable to %s on %s: %s", act, a->val, + nng_strerror(rv)); + } + } + + switch (proto) { + case OPT_SUB0: + case OPT_PULL0: + recvloop(sock); + break; + case OPT_REP0: + case OPT_RESPOND0: + if (data == NULL) { + recvloop(sock); + } else { + resploop(sock); + } + break; + case OPT_PUSH0: + case OPT_PUB0: + sendloop(sock); + break; + case OPT_BUS0: + case OPT_PAIR0: + case OPT_PAIR1: + if (data != NULL) { + sendrecv(sock); + } else { + recvloop(sock); + } + break; + case OPT_REQ0: + case OPT_SURVEY0: + sendrecv(sock); + break; + default: + fatal("Protocol handling unimplemented."); + } + + exit(0); +} diff --git a/src/tools/nngcat/nngcat_ambiguous_test.sh b/src/tools/nngcat/nngcat_ambiguous_test.sh new file mode 100755 index 00000000..414b6d19 --- /dev/null +++ b/src/tools/nngcat/nngcat_ambiguous_test.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +# +# Copyright 2018 Staysail Systems, Inc. +# Copyright 2018 Capitar IT Group BV +# +# This software is supplied under the terms of the MIT License, a +# copy of which should be located in the distribution where this +# file was obtained (LICENSE.txt). A copy of the license may also be +# found online at https://opensource.org/licenses/MIT. +# + +NNGCAT=${NNGCAT:=$1} +NNGCAT=${NNGCAT:-./nngcat} +CMD="${NNGCAT} --re --dial=tcp://127.0.0.1:27272" + +echo -n "Verify ambiguous options fail: " +if ${CMD} >/dev/null 2>&1 +then + echo "Failed: ambigous accepted" + exit 1 +fi +x=$(${CMD} 2>&1) +if [[ ${x} =~ "ambiguous" ]] +then + echo "pass" + exit 0 +fi + +echo "Failed: error did not match" +exit 1 diff --git a/src/tools/nngcat/nngcat_async_test.sh b/src/tools/nngcat/nngcat_async_test.sh new file mode 100755 index 00000000..2b03e522 --- /dev/null +++ b/src/tools/nngcat/nngcat_async_test.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# +# Copyright 2018 Staysail Systems, Inc. +# Copyright 2018 Capitar IT Group BV +# +# This software is supplied under the terms of the MIT License, a +# copy of which should be located in the distribution where this +# file was obtained (LICENSE.txt). A copy of the license may also be +# found online at https://opensource.org/licenses/MIT. +# + +NNGCAT=${NNGCAT:=$1} +NNGCAT=${NNGCAT:-./nngcat} +ADDR="ipc:///tmp/nngcat_async_test" + +echo -n "Verify async connect: " + +${NNGCAT} --async -d 1 --connect ${ADDR} --req0 -D "ping" & + + +answer=$( ${NNGCAT} --rep0 --recv-timeout=3 --listen ${ADDR} -D "pong" --ascii 2>/dev/null ) + +if [[ ${answer} == "ping" ]] +then + echo "pass" + exit 0 +fi + +echo "Failed: req did not match" +echo "RES: $answer" +exit 1 diff --git a/src/tools/nngcat/nngcat_dup_proto_test.sh b/src/tools/nngcat/nngcat_dup_proto_test.sh new file mode 100755 index 00000000..1513d01c --- /dev/null +++ b/src/tools/nngcat/nngcat_dup_proto_test.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# +# Copyright 2018 Staysail Systems, Inc. +# Copyright 2018 Capitar IT Group BV +# +# This software is supplied under the terms of the MIT License, a +# copy of which should be located in the distribution where this +# file was obtained (LICENSE.txt). A copy of the license may also be +# found online at https://opensource.org/licenses/MIT. +# + +NNGCAT=${NNGCAT:=$1} +NNGCAT=${NNGCAT:-./nngcat} + +echo -n "Verify only a single protocol is allowed: " +if ${NNGCAT} --pub0 --sub0 --dial=tcp://127.0.0.1:8989 >/dev/null 2>&1 +then + echo "Failed: duplicate protocols accepted" + exit 1 +fi +echo "pass" +exit 0 diff --git a/src/tools/nngcat/nngcat_help_test.sh b/src/tools/nngcat/nngcat_help_test.sh new file mode 100755 index 00000000..95ed9e3e --- /dev/null +++ b/src/tools/nngcat/nngcat_help_test.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# +# Copyright 2018 Staysail Systems, Inc. +# Copyright 2018 Capitar IT Group BV +# +# This software is supplied under the terms of the MIT License, a +# copy of which should be located in the distribution where this +# file was obtained (LICENSE.txt). A copy of the license may also be +# found online at https://opensource.org/licenses/MIT. +# + +NNGCAT=${NNGCAT:=$1} +NNGCAT=${NNGCAT:-./nngcat} + +echo -n "Verify nngcat help: " +if ${NNGCAT} --help >/dev/null 2>&1 +then + echo "Failed: help didn't return 1" + exit 1 +fi +x=$(${NNGCAT} --help 2>&1) +if [[ ${x} =~ "Usage:" ]] +then + echo "pass" + exit 0 +fi + +echo "Failed: usage did not match" +echo "Output:" +echo "$x" +exit 1 diff --git a/src/tools/nngcat/nngcat_incompat_test.sh b/src/tools/nngcat/nngcat_incompat_test.sh new file mode 100755 index 00000000..128b57ba --- /dev/null +++ b/src/tools/nngcat/nngcat_incompat_test.sh @@ -0,0 +1,73 @@ +#!/bin/sh + +# +# Copyright 2018 Staysail Systems, Inc. +# Copyright 2018 Capitar IT Group BV +# +# This software is supplied under the terms of the MIT License, a +# copy of which should be located in the distribution where this +# file was obtained (LICENSE.txt). A copy of the license may also be +# found online at https://opensource.org/licenses/MIT. +# + +NNGCAT=${NNGCAT:=$1} +NNGCAT=${NNGCAT:-./nngcat} + +echo "Verify incompatible options: " + +# Just bind something to this so other ones connect +${NNGCAT} --pull0 --ascii -X /tmp/bogusipc & +pid=$! + +trap "kill $pid && wait $pid 2>/dev/null" 0 + +echo -n " --subscribe doesn't work with non-sub" +if ${NNGCAT} --req0 -x /tmp/bogusipc --subscribe=oops >/dev/null 2>&1 +then + echo "fail" + exit 1 +fi +echo "pass" + +echo -n " --interval doesn't work with recv only: " +if ${NNGCAT} --interval 1 --pull -x /tmp/bogusipc >/dev/null 2>&1 +then + echo "fail" + exit 1 +fi +echo "pass" + +echo -n " --pair1 doesn't work with --compat: " +if ${NNGCAT} --compat --pair1 -x /tmp/bogusipc >/dev/null 2>&1 +then + echo "fail" + exit 1 +fi +echo "pass" + +echo -n " --count doesn't work with --compat: " +if ${NNGCAT} --compat --count=1 --pair0 -x /tmp/bogusipc >/dev/null 2>&1 +then + echo "fail" + exit 1 +fi +echo "pass" + +echo -n " --count fails with non-integer: " +if ${NNGCAT} --count=xyz --pair0 -x /tmp/bogusipc >/dev/null 2>&1 +then + echo "fail" + exit 1 +fi +echo "pass" + +echo -n " --file fails with non-existing file: " +if ${NNGCAT} --async --file=/nosuchfilehere --push0 -x /tmp/bogusipc >/dev/null 2>&1 +then + echo "fail" + exit 1 +fi +echo "pass" + +echo "PASS." +exit 0 diff --git a/src/tools/nngcat/nngcat_need_proto_test.sh b/src/tools/nngcat/nngcat_need_proto_test.sh new file mode 100755 index 00000000..d6733cad --- /dev/null +++ b/src/tools/nngcat/nngcat_need_proto_test.sh @@ -0,0 +1,23 @@ +#!/bin/sh + +# +# Copyright 2018 Staysail Systems, Inc. +# Copyright 2018 Capitar IT Group BV +# +# This software is supplied under the terms of the MIT License, a +# copy of which should be located in the distribution where this +# file was obtained (LICENSE.txt). A copy of the license may also be +# found online at https://opensource.org/licenses/MIT. +# + +NNGCAT=${NNGCAT:=$1} +NNGCAT=${NNGCAT:-./nngcat} + +echo -n "Verify a protocol is needed: " +if ${NNGCAT} --dial=tcp://127.0.0.1:8989 >/dev/null 2>&1 +then + echo "Failed: protocol should be required" + exit 1 +fi +echo "pass" +exit 0 diff --git a/src/tools/nngcat/nngcat_pubsub_test.sh b/src/tools/nngcat/nngcat_pubsub_test.sh new file mode 100755 index 00000000..b9ba90ed --- /dev/null +++ b/src/tools/nngcat/nngcat_pubsub_test.sh @@ -0,0 +1,45 @@ +#!/bin/bash + +# +# Copyright 2018 Staysail Systems, Inc. +# Copyright 2018 Capitar IT Group BV +# +# This software is supplied under the terms of the MIT License, a +# copy of which should be located in the distribution where this +# file was obtained (LICENSE.txt). A copy of the license may also be +# found online at https://opensource.org/licenses/MIT. +# + +NNGCAT=${NNGCAT:=$1} +NNGCAT=${NNGCAT:-./nngcat} +ADDR="ipc:///tmp/nngcat_pub_sub_test" +OUTPUT=/tmp/nngcat_pubsub_test.$$.out + +echo -n "Verify pub sub: " + +trap "rm $OUTPUT" 0 + +${NNGCAT} --listen ${ADDR} --count=3 --recv-timeout=20 --sub0 --subscribe=one --subscribe=two --quoted > $OUTPUT 2>/dev/null & +sleep 1 +# for speed of execution, run these in the background, they should be ignored +${NNGCAT} -d 1 --connect ${ADDR} --pub0 --data "xyz" & +${NNGCAT} -d 1 --connect ${ADDR} --pub0 -D "none swam" & +# these we care about, due to ordering (checksum) so run them serially +${NNGCAT} -d 1 --connect ${ADDR} --pub0 -D "one flew" +${NNGCAT} -d 1 --connect ${ADDR} --pub0 --data "twofer test" +${NNGCAT} -d 1 --connect ${ADDR} --pub0 --data "one more" + +wait $bgid 2>/dev/null + +sum=$(cksum ${OUTPUT}) +sum=${sum%% *} +if [[ ${sum} == 3929078614 ]] +then + echo "pass" + exit 0 +fi +echo "FAIL: Checksum failed (Wanted 3929078614 got ${sum})" +echo "OUTPUT:" +cat ${OUTPUT} + +exit 1 diff --git a/src/tools/nngcat/nngcat_recvmaxsz_test.sh b/src/tools/nngcat/nngcat_recvmaxsz_test.sh new file mode 100755 index 00000000..b5d4ff4a --- /dev/null +++ b/src/tools/nngcat/nngcat_recvmaxsz_test.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# +# Copyright 2018 Staysail Systems, Inc. +# Copyright 2018 Capitar IT Group BV +# +# This software is supplied under the terms of the MIT License, a +# copy of which should be located in the distribution where this +# file was obtained (LICENSE.txt). A copy of the license may also be +# found online at https://opensource.org/licenses/MIT. +# + +NNGCAT=${NNGCAT:=$1} +NNGCAT=${NNGCAT:-./nngcat} +ADDR="ipc:///tmp/nngcat_recvmaxsz_test" +OUTPUT=/tmp/nngcat_recvmaxsz_test.$$.out + +echo -n "Verify maximum receive size: " + +trap "rm $OUTPUT" 0 + +${NNGCAT} --listen ${ADDR} --count=3 --recv-maxsz=5 --pull0 --quoted > $OUTPUT 2>/dev/null & +sleep 1 +# for speed of execution, run these in the background, they should be ignored +${NNGCAT} --connect ${ADDR} --push0 --data "one" +${NNGCAT} --connect ${ADDR} --push0 --data "55555" +${NNGCAT} --connect ${ADDR} --push0 --data "666666" +${NNGCAT} --connect ${ADDR} --push0 --data "7777777" +${NNGCAT} --connect ${ADDR} --push0 --data "88888" + +wait $bgid 2>/dev/null + +sum=$(cksum ${OUTPUT}) +sum=${sum%% *} + +# This matches 3 lines of "one", "55555", "88888". +if [[ ${sum} == 4122906158 ]] +then + echo "pass" + exit 0 +fi +echo "FAIL: Checksum failed (Wanted 3929078614 got ${sum})" +echo "OUTPUT:" +cat ${OUTPUT} + +exit 1 diff --git a/src/tools/nngcat/nngcat_stdin_pipe_test.sh b/src/tools/nngcat/nngcat_stdin_pipe_test.sh new file mode 100755 index 00000000..5fec0ab7 --- /dev/null +++ b/src/tools/nngcat/nngcat_stdin_pipe_test.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# +# Copyright 2018 Staysail Systems, Inc. +# Copyright 2018 Capitar IT Group BV +# Copyright 2020 Lager Data, Inc. +# +# This software is supplied under the terms of the MIT License, a +# copy of which should be located in the distribution where this +# file was obtained (LICENSE.txt). A copy of the license may also be +# found online at https://opensource.org/licenses/MIT. +# + +NNGCAT=${NNGCAT:=$1} +NNGCAT=${NNGCAT:-./nngcat} +ADDR="ipc:///tmp/nngcat_stdin_pipe_test" +OUTPUT=/tmp/nngcat_stdin_pipe_test.$$.out + +echo -n "Verify reading from stdin pipe: " + +trap "rm $OUTPUT" 0 + +${NNGCAT} --listen ${ADDR} --count=1 --recv-timeout=3 --recv-maxsz=0 --pull0 --raw > $OUTPUT 2>/dev/null & +bgid=$! + +sleep 1 +# for speed of execution, run these in the background, they should be ignored +echo "hello world" | ${NNGCAT} --connect ${ADDR} --delay=1 --push0 --file - +wait "$bgid" 2>/dev/null + +sum=$(cksum ${OUTPUT}) +sum=${sum%% *} + +# This matches "hello world\n" since echo adds a trailing newline +if [[ ${sum} == 3733384285 ]] +then + echo "pass" + exit 0 +fi +echo "FAIL: Checksum failed (Wanted 3733384285 got ${sum})" +echo "OUTPUT:" +ls -la ${OUTPUT} + +exit 1 diff --git a/src/tools/nngcat/nngcat_unlimited_test.sh b/src/tools/nngcat/nngcat_unlimited_test.sh new file mode 100755 index 00000000..0486b9b1 --- /dev/null +++ b/src/tools/nngcat/nngcat_unlimited_test.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# +# Copyright 2018 Staysail Systems, Inc. +# Copyright 2018 Capitar IT Group BV +# +# This software is supplied under the terms of the MIT License, a +# copy of which should be located in the distribution where this +# file was obtained (LICENSE.txt). A copy of the license may also be +# found online at https://opensource.org/licenses/MIT. +# + +NNGCAT=${NNGCAT:=$1} +NNGCAT=${NNGCAT:-./nngcat} +ADDR="ipc:///tmp/nngcat_unlimited_test" +INPUT=/tmp/nngcat_unlimited_test.$$.in +OUTPUT=/tmp/nngcat_unlimited_test.$$.out + +echo -n "Verify unlimited receive size: " + +trap "rm $OUTPUT $INPUT" 0 + +# 4 MB +dd if=/dev/urandom of=${INPUT} bs=1024 count=4096 >/dev/null 2>&1 +goodsum=$(cksum ${INPUT}) +goodsum=${goodsum%% *} + +${NNGCAT} --listen ${ADDR} --count=1 --recv-maxsz=0 --pull0 --raw > $OUTPUT 2>/dev/null & +sleep 1 +# for speed of execution, run these in the background, they should be ignored +${NNGCAT} --connect ${ADDR} --delay=1 --push0 --file ${INPUT} +wait $bgid 2>/dev/null + +sum=$(cksum ${OUTPUT}) +sum=${sum%% *} + +if [[ ${sum} == ${goodsum} ]] +then + echo "pass" + exit 0 +fi +echo "FAIL: Checksum failed (Wanted ${goodsum} got ${sum})" +echo "OUTPUT:" +ls -la ${OUTPUT} + +exit 1 diff --git a/src/tools/perf/CMakeLists.txt b/src/tools/perf/CMakeLists.txt new file mode 100644 index 00000000..135544bb --- /dev/null +++ b/src/tools/perf/CMakeLists.txt @@ -0,0 +1,35 @@ +# +# Copyright 2021 Staysail Systems, Inc. +# +# This software is supplied under the terms of the MIT License, a +# copy of which should be located in the distribution where this +# file was obtained (LICENSE.txt). A copy of the license may also be +# found online at https://opensource.org/licenses/MIT. +# + +# Build performance tests. + +nng_directory(perf) + +if (NNG_TESTS) + macro (add_nng_perf NAME) + add_executable (${NAME} perf.c) + target_link_libraries (${NAME} nng nng_private) + endmacro (add_nng_perf) + + add_nng_perf(remote_lat) + add_nng_perf(local_lat) + add_nng_perf(local_thr) + add_nng_perf(remote_thr) + add_nng_perf(inproc_thr) + add_nng_perf(inproc_lat) + + add_test (NAME nng.inproc_lat COMMAND inproc_lat 64 10000) + set_tests_properties (nng.inproc_lat PROPERTIES TIMEOUT 30) + + add_test (NAME nng.inproc_thr COMMAND inproc_thr 1400 10000) + set_tests_properties (nng.inproc_thr PROPERTIES TIMEOUT 30) + + add_executable (pubdrop pubdrop.c) + target_link_libraries(pubdrop nng nng_private) +endif () diff --git a/src/tools/perf/perf.c b/src/tools/perf/perf.c new file mode 100644 index 00000000..accac621 --- /dev/null +++ b/src/tools/perf/perf.c @@ -0,0 +1,662 @@ +// +// Copyright 2020 Staysail Systems, Inc. +// Copyright 2018 Capitar IT Group BV +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +static void die(const char *, ...); +static int +no_open(nng_socket *arg) +{ + (void) arg; + die("Protocol not supported in this build!"); + return (NNG_ENOTSUP); +} + +typedef int (*open_func)(nng_socket *); + +static open_func open_server = no_open; +static open_func open_client = no_open; + +#if defined(NNG_HAVE_PAIR1) +#include +#else +#define nng_pair1_open no_open +#endif + +#if defined(NNG_HAVE_PAIR0) +#include +#else +#define nng_pair0_open no_open +#endif + +#if defined(NNG_HAVE_REQ0) +#include +#else +#define nng_req0_open no_open +#endif + +#if defined(NNG_HAVE_REP0) +#include +#else +#define nng_rep0_open no_open +#endif + +#if defined(NNG_HAVE_BUS0) +#include +#else +#define nng_bus0_open no_open +#endif + +#if defined(NNG_HAVE_PULL0) +#include +#else +#define nng_pull0_open no_open +#endif + +#if defined(NNG_HAVE_PUSH0) +#include +#else +#define nng_push0_open no_open +#endif + +#if defined(NNG_HAVE_PUB0) +#include +#else +#define nng_pub0_open no_open +#endif + +#if defined(NNG_HAVE_SUB0) +#include +#else +#define nng_sub0_open no_open +#endif + +enum options { + OPT_PAIR0 = 1, + OPT_PAIR1, + OPT_REQREP0, + OPT_PUBSUB0, + OPT_PIPELINE0, + OPT_SURVEY0, + OPT_BUS0, + OPT_URL, +}; + +// These are not universally supported by the variants yet. +static nng_optspec opts[] = { + { .o_name = "pair1", .o_val = OPT_PAIR1 }, + { .o_name = "pair0", .o_val = OPT_PAIR0 }, + { .o_name = "reqrep0", .o_val = OPT_REQREP0 }, + { .o_name = "bus0", .o_val = OPT_BUS0 }, + { .o_name = "pubsub0", .o_val = OPT_PUBSUB0 }, + { .o_name = "pipeline0", .o_val = OPT_PIPELINE0 }, + { .o_name = "url", .o_val = OPT_URL, .o_arg = true }, + { .o_name = NULL, .o_val = 0 }, +}; + +static void latency_client(const char *, size_t, int); +static void latency_server(const char *, size_t, int); +static void throughput_client(const char *, size_t, int); +static void throughput_server(const char *, size_t, int); +static void do_remote_lat(int argc, char **argv); +static void do_local_lat(int argc, char **argv); +static void do_remote_thr(int argc, char **argv); +static void do_local_thr(int argc, char **argv); +static void do_inproc_thr(int argc, char **argv); +static void do_inproc_lat(int argc, char **argv); +static void die(const char *, ...); + +// perf implements the same performance tests found in the standard +// nanomsg & mangos performance tests. As with mangos, the decision +// about which test to run is determined by the program name (ARGV[0}]) +// that it is run under. +// +// Options are: +// +// - remote_lat - remote latency side (client, aka latency_client) +// - local_lat - local latency side (server, aka latency_server) +// - local_thr - local throughput side +// - remote_thr - remote throughput side +// - inproc_lat - inproc latency +// - inproc_thr - inproc throughput +// + +bool +matches(const char *arg, const char *name) +{ + const char *ptr = arg; + const char *x; + + while (((x = strchr(ptr, '/')) != NULL) || + ((x = strchr(ptr, '\\')) != NULL) || + ((x = strchr(ptr, ':')) != NULL)) { + ptr = x + 1; + } + for (;;) { + if (*name == '\0') { + break; + } + if (tolower(*ptr) != *name) { + return (false); + } + ptr++; + name++; + } + + switch (*ptr) { + case '\0': + /* FALLTHROUGH*/ + case '.': // extension; ignore it. + return (true); + default: // some other trailing bit. + return (false); + } +} + +const int PAIR0 = 0; +const int PAIR1 = 1; +const int REQREP = 2; + +int +main(int argc, char **argv) +{ + char *prog; + +#if defined(NNG_HAVE_PAIR1) + open_server = nng_pair1_open; + open_client = nng_pair1_open; +#elif defined(NNG_HAVE_PAIR0) + open_server = nng_pair0_open; + open_client = nng_pair0_open; +#endif + + // Allow -m or whatever to override argv[0]. + if ((argc >= 3) && (strcmp(argv[1], "-m") == 0)) { + prog = argv[2]; + argv += 3; + argc -= 3; + } else { + prog = argv[0]; + argc--; + argv++; + } + if (matches(prog, "remote_lat") || matches(prog, "latency_client")) { + do_remote_lat(argc, argv); + } else if (matches(prog, "local_lat") || + matches(prog, "latency_server")) { + do_local_lat(argc, argv); + } else if (matches(prog, "local_thr") || + matches(prog, "throughput_server")) { + do_local_thr(argc, argv); + } else if (matches(prog, "remote_thr") || + matches(prog, "throughput_client")) { + do_remote_thr(argc, argv); + } else if (matches(prog, "inproc_thr")) { + do_inproc_thr(argc, argv); + } else if (matches(prog, "inproc_lat")) { + do_inproc_lat(argc, argv); + } else { + die("Unknown program mode? Use -m ."); + } +} + +static void +die(const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); + fprintf(stderr, "\n"); + exit(2); +} + +static int +parse_int(const char *arg, const char *what) +{ + long val; + char *eptr; + + val = strtol(arg, &eptr, 10); + // Must be a positive number less than around a billion. + if ((val < 0) || (val > 1000000000) || (*eptr != 0) || (eptr == arg)) { + die("Invalid %s", what); + } + return ((int) val); +} + +void +do_local_lat(int argc, char **argv) +{ + long int msgsize; + long int trips; + + if (argc != 3) { + die("Usage: local_lat "); + } + + msgsize = parse_int(argv[1], "message size"); + trips = parse_int(argv[2], "round-trips"); + + latency_server(argv[0], msgsize, trips); +} + +void +do_remote_lat(int argc, char **argv) +{ + int msgsize; + int trips; + + if (argc != 3) { + die("Usage: remote_lat "); + } + + msgsize = parse_int(argv[1], "message size"); + trips = parse_int(argv[2], "round-trips"); + + latency_client(argv[0], msgsize, trips); +} + +void +do_local_thr(int argc, char **argv) +{ + int msgsize; + int trips; + + if (argc != 3) { + die("Usage: local_thr "); + } + + msgsize = parse_int(argv[1], "message size"); + trips = parse_int(argv[2], "count"); + + throughput_server(argv[0], msgsize, trips); +} + +void +do_remote_thr(int argc, char **argv) +{ + int msgsize; + int trips; + + if (argc != 3) { + die("Usage: remote_thr "); + } + + msgsize = parse_int(argv[1], "message size"); + trips = parse_int(argv[2], "count"); + + throughput_client(argv[0], msgsize, trips); +} + +struct inproc_args { + int count; + int msgsize; + const char *addr; + void (*func)(const char *, size_t, int); +}; + +static void +do_inproc(void *args) +{ + struct inproc_args *ia = args; + + ia->func(ia->addr, ia->msgsize, ia->count); +} + +void +do_inproc_lat(int argc, char **argv) +{ + nng_thread * thr; + struct inproc_args ia; + int rv; + int val; + int optidx; + char * arg; + char * addr; + + addr = "inproc://latency_test"; + + optidx = 0; + while ((rv = nng_opts_parse(argc, argv, opts, &val, &arg, &optidx)) == + 0) { + switch (val) { + case OPT_REQREP0: + open_client = nng_req0_open; + open_server = nng_rep0_open; + break; + case OPT_PAIR0: + open_client = nng_pair0_open; + open_server = nng_pair0_open; + break; + case OPT_PAIR1: + open_client = nng_pair1_open; + open_server = nng_pair1_open; + break; + case OPT_BUS0: + open_client = nng_bus0_open; + open_server = nng_bus0_open; + break; + + case OPT_URL: + addr = arg; + break; + default: + die("bad option"); + } + } + argc -= optidx; + argv += optidx; + + if (argc != 2) { + die("Usage: inproc_lat "); + } + + ia.addr = addr; + ia.msgsize = parse_int(argv[0], "message size"); + ia.count = parse_int(argv[1], "count"); + ia.func = latency_server; + + if ((rv = nng_thread_create(&thr, do_inproc, &ia)) != 0) { + die("Cannot create thread: %s", nng_strerror(rv)); + } + + // Sleep a bit. + nng_msleep(100); + + latency_client(addr, ia.msgsize, ia.count); + nng_thread_destroy(thr); +} + +void +do_inproc_thr(int argc, char **argv) +{ + nng_thread * thr; + struct inproc_args ia; + int rv; + int optidx; + int val; + char * arg; + char * addr = "inproc://throughput-test"; + + optidx = 0; + while ((rv = nng_opts_parse(argc, argv, opts, &val, &arg, &optidx)) == + 0) { + switch (val) { +#if 0 + // For now these protocols simply do not work with + // throughput -- they don't work with backpressure properly. + // In the future we should support synchronizing in the same + // process, and alerting the sender both on completion of + // a single message, and on completion of all messages. + case OPT_REQREP0: + open_client = nng_req0_open; + open_server = nng_rep0_open; + break; +#endif + case OPT_PAIR0: + open_client = nng_pair0_open; + open_server = nng_pair0_open; + break; + case OPT_PAIR1: + open_client = nng_pair1_open; + open_server = nng_pair1_open; + break; + case OPT_PIPELINE0: + open_client = nng_pull0_open; + open_server = nng_push0_open; + break; + case OPT_URL: + addr = arg; + break; + default: + die("bad option"); + } + } + argc -= optidx; + argv += optidx; + + if (argc != 2) { + die("Usage: inproc_thr "); + } + + ia.addr = addr; + ia.msgsize = parse_int(argv[0], "message size"); + ia.count = parse_int(argv[1], "count"); + ia.func = throughput_server; + + if ((rv = nng_thread_create(&thr, do_inproc, &ia)) != 0) { + die("Cannot create thread: %s", nng_strerror(rv)); + } + + // Sleep a bit. + nng_msleep(100); + + throughput_client(addr, ia.msgsize, ia.count); + nng_thread_destroy(thr); +} + +void +latency_client(const char *addr, size_t msgsize, int trips) +{ + nng_socket s; + nng_msg * msg; + nng_time start, end; + int rv; + int i; + float total; + float latency; + if ((rv = open_client(&s)) != 0) { + die("nng_socket: %s", nng_strerror(rv)); + } + + // XXX: set no delay + // XXX: other options (TLS in the future?, Linger?) + + if ((rv = nng_dial(s, addr, NULL, 0)) != 0) { + die("nng_dial: %s", nng_strerror(rv)); + } + + nng_msleep(100); + + if (nng_msg_alloc(&msg, msgsize) != 0) { + die("nng_msg_alloc: %s", nng_strerror(rv)); + } + + start = nng_clock(); + for (i = 0; i < trips; i++) { + if ((rv = nng_sendmsg(s, msg, 0)) != 0) { + die("nng_sendmsg: %s", nng_strerror(rv)); + } + + if ((rv = nng_recvmsg(s, &msg, 0)) != 0) { + die("nng_recvmsg: %s", nng_strerror(rv)); + } + } + end = nng_clock(); + + nng_msg_free(msg); + nng_close(s); + + total = (float) ((end - start)) / 1000; + latency = ((float) ((total * 1000000)) / (float) (trips * 2)); + printf("total time: %.3f [s]\n", total); + printf("message size: %d [B]\n", (int) msgsize); + printf("round trip count: %d\n", trips); + printf("average latency: %.3f [us]\n", latency); +} + +void +latency_server(const char *addr, size_t msgsize, int trips) +{ + nng_socket s; + nng_msg * msg; + int rv; + int i; + + if ((rv = open_server(&s)) != 0) { + die("nng_socket: %s", nng_strerror(rv)); + } + + // XXX: set no delay + // XXX: other options (TLS in the future?, Linger?) + + if ((rv = nng_listen(s, addr, NULL, 0)) != 0) { + die("nng_listen: %s", nng_strerror(rv)); + } + + for (i = 0; i < trips; i++) { + if ((rv = nng_recvmsg(s, &msg, 0)) != 0) { + die("nng_recvmsg: %s", nng_strerror(rv)); + } + if (nng_msg_len(msg) != msgsize) { + die("wrong message size: %lu != %lu", nng_msg_len(msg), + msgsize); + } + if ((rv = nng_sendmsg(s, msg, 0)) != 0) { + die("nng_sendmsg: %s", nng_strerror(rv)); + } + } + + // Wait a bit for things to drain... linger should do this. + // 100ms ought to be enough. + nng_msleep(100); + nng_close(s); +} + +// Our throughput story is quite a mess. Mostly I think because of the poor +// caching and message reuse. We should probably implement a message pooling +// API somewhere. + +void +throughput_server(const char *addr, size_t msgsize, int count) +{ + nng_socket s; + nng_msg * msg; + int rv; + int i; + uint64_t start, end; + float msgpersec, mbps, total; + + if ((rv = nng_pair_open(&s)) != 0) { + die("nng_socket: %s", nng_strerror(rv)); + } + rv = nng_socket_set_int(s, NNG_OPT_RECVBUF, 128); + if (rv != 0) { + die("nng_socket_set(nng_opt_recvbuf): %s", nng_strerror(rv)); + } + + // XXX: set no delay + // XXX: other options (TLS in the future?, Linger?) + + if ((rv = nng_listen(s, addr, NULL, 0)) != 0) { + die("nng_listen: %s", nng_strerror(rv)); + } + + // Receive first synchronization message. + if ((rv = nng_recvmsg(s, &msg, 0)) != 0) { + die("nng_recvmsg: %s", nng_strerror(rv)); + } + nng_msg_free(msg); + start = nng_clock(); + + for (i = 0; i < count; i++) { + if ((rv = nng_recvmsg(s, &msg, 0)) != 0) { + die("nng_recvmsg: %s", nng_strerror(rv)); + } + if (nng_msg_len(msg) != msgsize) { + die("wrong message size: %lu != %lu", nng_msg_len(msg), + msgsize); + } + nng_msg_free(msg); + } + end = nng_clock(); + // Send a synchronization message (empty) to the other side, + // and wait a bit to make sure it goes out the wire. + nng_send(s, "", 0, 0); + nng_msleep(200); + nng_close(s); + total = (float) ((end - start)) / 1000; + msgpersec = (float) (count) / total; + mbps = (float) (msgpersec * 8 * msgsize) / (1024 * 1024); + printf("total time: %.3f [s]\n", total); + printf("message size: %d [B]\n", (int) msgsize); + printf("message count: %d\n", count); + printf("throughput: %.f [msg/s]\n", msgpersec); + printf("throughput: %.3f [Mb/s]\n", mbps); +} + +void +throughput_client(const char *addr, size_t msgsize, int count) +{ + nng_socket s; + nng_msg * msg; + int rv; + int i; + + // We send one extra zero length message to start the timer. + count++; + + if ((rv = nng_pair_open(&s)) != 0) { + die("nng_socket: %s", nng_strerror(rv)); + } + + // XXX: set no delay + // XXX: other options (TLS in the future?, Linger?) + + rv = nng_socket_set_int(s, NNG_OPT_SENDBUF, 128); + if (rv != 0) { + die("nng_socket_set(nng_opt_sendbuf): %s", nng_strerror(rv)); + } + + rv = nng_socket_set_ms(s, NNG_OPT_RECVTIMEO, 5000); + if (rv != 0) { + die("nng_socket_set(nng_opt_recvtimeo): %s", nng_strerror(rv)); + } + + if ((rv = nng_dial(s, addr, NULL, 0)) != 0) { + die("nng_dial: %s", nng_strerror(rv)); + } + + if ((rv = nng_msg_alloc(&msg, 0)) != 0) { + die("nng_msg_alloc: %s", nng_strerror(rv)); + } + if ((rv = nng_sendmsg(s, msg, 0)) != 0) { + die("nng_sendmsg: %s", nng_strerror(rv)); + } + + for (i = 0; i < count; i++) { + if ((rv = nng_msg_alloc(&msg, msgsize)) != 0) { + die("nng_msg_alloc: %s", nng_strerror(rv)); + } + + if ((rv = nng_sendmsg(s, msg, 0)) != 0) { + die("nng_sendmsg: %s", nng_strerror(rv)); + } + } + + // Attempt to get the completion indication from the other + // side. + if (nng_recvmsg(s, &msg, 0) == 0) { + nng_msg_free(msg); + } + + nng_close(s); +} diff --git a/src/tools/perf/pubdrop.c b/src/tools/perf/pubdrop.c new file mode 100644 index 00000000..be51bb2f --- /dev/null +++ b/src/tools/perf/pubdrop.c @@ -0,0 +1,325 @@ +// +// Copyright 2019 Staysail Systems, Inc. +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include +#include +#include +#include +#include +#include + +#include +#include + +// pubdrop - this is a simple testing utility that lets us measure PUB/SUB +// performance, including dropped messages, delivery across multiple threads, +// etc. It actually uses a wild card subscription for now. + +#if defined(NNG_HAVE_PUB0) && defined(NNG_HAVE_SUB0) +#include +#include + +#else + +static void die(const char *, ...); + +static int +nng_pub0_open(nng_socket *arg) +{ + (void) arg; + die("Pub protocol enabled in this build!"); + return (NNG_ENOTSUP); +} + +static int +nng_sub0_open(nng_socket *arg) +{ + (void) arg; + die("Sub protocol enabled in this build!"); + return (NNG_ENOTSUP); +} + +#endif // NNG_HAVE_PUB0.... + +static void die(const char *, ...); +static void do_pubdrop(int argc, char **argv); +static uint64_t nperusec; +static volatile int x; + +void +work(void) +{ + x = rand(); +} + +void +usdelay(unsigned long long nusec) +{ + nusec *= nperusec; + while (nusec > 0) { + work(); + nusec--; + } +} + +int +main(int argc, char **argv) +{ + argc--; + argv++; + + // We calculate a delay factor to roughly delay 1 usec. We don't + // need this to be perfect, just reproducible on the same host. + unsigned long cnt = 1000000; + + nng_time beg = nng_clock(); + for (unsigned long i = 0; i < cnt; i++) { + work(); + } + nng_time end = nng_clock(); + nperusec = cnt / (1000 * (end - beg)); + + do_pubdrop(argc, argv); +} + +static void +die(const char *fmt, ...) +{ + va_list ap; + + va_start(ap, fmt); + vfprintf(stderr, fmt, ap); + va_end(ap); + fprintf(stderr, "\n"); + exit(2); +} + +static int +parse_int(const char *arg, const char *what) +{ + long val; + char *eptr; + + val = strtol(arg, &eptr, 10); + // Must be a postive number less than around a billion. + if ((val < 0) || (val > (1 << 30)) || (*eptr != 0) || (eptr == arg)) { + die("Invalid %s", what); + } + return ((int) val); +} + +struct pubdrop_args { + const char * addr; + bool start; + unsigned long long msgsize; + unsigned long long count; + unsigned long long intvl; + unsigned long long drops; + unsigned long long gaps; + unsigned long long errs; + unsigned long long recvs; + nng_time beg; + nng_time end; + nng_mtx * mtx; + nng_cv * cv; +}; + +static void +pub_server(void *arg) +{ + struct pubdrop_args *pa = arg; + nng_socket sock; + int rv; + nng_msg * msg; + nng_time start; + nng_time end; + + if ((rv = nng_pub0_open(&sock)) != 0) { + die("Cannot open sub: %s", nng_strerror(rv)); + } + if ((rv = nng_listen(sock, pa->addr, NULL, 0)) != 0) { + die("Cannot listen: %s", nng_strerror(rv)); + } + + nng_mtx_lock(pa->mtx); + while (!pa->start) { + nng_cv_wait(pa->cv); + } + nng_mtx_unlock(pa->mtx); + + start = nng_clock(); + for (uint64_t i = 0; i < pa->count; i++) { + // Unfortunately we need to allocate messages dynamically as we + // go. The other option would be to allocate them all up front, + // but that could be a rather excessive amount of memory. + if ((rv = nng_msg_alloc(&msg, (size_t) pa->msgsize)) != 0) { + die("Message alloc failed"); + } + memcpy(nng_msg_body(msg), &i, sizeof(i)); + if ((rv = nng_sendmsg(sock, msg, 0)) != 0) { + die("Sendmsg: %s", nng_strerror(rv)); + } + // It sure would be nice if we had a usec granularity option + // here. + if (pa->intvl > 0) { + usdelay((unsigned long long) pa->intvl); + } + } + + end = nng_clock(); + + nng_msleep(1000); // drain the queue + nng_close(sock); + + nng_mtx_lock(pa->mtx); + pa->beg = start; + pa->end = end; + nng_mtx_unlock(pa->mtx); +} + +static void +sub_client(void *arg) +{ + struct pubdrop_args *pa = arg; + nng_socket sock; + int rv; + nng_msg * msg; + unsigned long long recvs; + unsigned long long drops; + unsigned long long gaps; + unsigned long long errs; + unsigned long long expect; + + if ((rv = nng_sub0_open(&sock)) != 0) { + die("Cannot open sub: %s", nng_strerror(rv)); + } + if ((rv = nng_dial(sock, pa->addr, NULL, 0)) != 0) { + die("Cannot listen: %s", nng_strerror(rv)); + } + if ((rv = nng_setopt_ms(sock, NNG_OPT_RECONNMINT, 51)) != 0) { + die("setopt: %s", nng_strerror(rv)); + } + if ((rv = nng_setopt(sock, NNG_OPT_SUB_SUBSCRIBE, "", 0)) != 0) { + die("setopt: %s", nng_strerror(rv)); + } + if ((rv = nng_setopt_ms(sock, NNG_OPT_RECVTIMEO, 10000)) != 0) { + die("setopt: %s", nng_strerror(rv)); + } + + expect = 0; + recvs = drops = gaps = errs = 0; + + while (expect < pa->count) { + uint64_t got; + if ((rv = nng_recvmsg(sock, &msg, 0)) != 0) { + if ((rv == NNG_ECLOSED) || (rv == NNG_ETIMEDOUT)) { + // Closed without receiving the last message + drops += (pa->count - expect); + gaps++; + break; + } + printf("ERROR: %s\n", nng_strerror(rv)); + errs++; + break; + } + recvs++; + memcpy(&got, nng_msg_body(msg), sizeof(got)); + nng_msg_free(msg); + if (got != expect) { + gaps++; + if (got > expect) { + drops += (got - expect); + } else { + die("Misordered delivery"); + } + } + expect = got + 1; + } + + nng_mtx_lock(pa->mtx); + pa->drops += drops; + pa->errs += errs; + pa->recvs += recvs; + pa->gaps += gaps; + nng_mtx_unlock(pa->mtx); +} + +static void +do_pubdrop(int argc, char **argv) +{ + nng_thread ** thrs; + struct pubdrop_args pa; + int rv; + int nsubs; + + if (argc != 5) { + die("Usage: pubdrop " + ""); + } + + memset(&pa, 0, sizeof(pa)); + pa.addr = argv[0]; + pa.msgsize = parse_int(argv[1], "message size"); + pa.count = parse_int(argv[2], "count"); + pa.intvl = parse_int(argv[4], "interval"); + nsubs = parse_int(argv[3], "#subscribers"); + + if (pa.msgsize < sizeof(uint64_t)) { + die("Message size too small."); + } + + thrs = calloc(sizeof(nng_thread *), (size_t) pa.count + 1); + if (((rv = nng_mtx_alloc(&pa.mtx)) != 0) || + ((nng_cv_alloc(&pa.cv, pa.mtx)) != 0)) { + die("Startup: %s\n", nng_strerror(rv)); + }; + + if ((rv = nng_thread_create(&thrs[0], pub_server, &pa)) != 0) { + die("Cannot create pub thread: %s", nng_strerror(rv)); + } + + nng_msleep(100); // give time for listener to start... + + for (int i = 0; i < nsubs; i++) { + if ((rv = nng_thread_create(&thrs[i + 1], sub_client, &pa)) != + 0) { + die("Cannot create sub thread: %s", nng_strerror(rv)); + } + } + + // Sleep a bit for conns to establish. + nng_msleep(2000); + nng_mtx_lock(pa.mtx); + pa.start = true; + nng_cv_wake(pa.cv); + nng_mtx_unlock(pa.mtx); + + for (int i = 0; i < nsubs + 1; i++) { + nng_thread_destroy(thrs[i]); + } + + nng_mtx_lock(pa.mtx); + + unsigned long long expect = nsubs * pa.count; + unsigned long long missing = nsubs ? expect - pa.recvs : 0; + double dur = (pa.end - pa.beg) / 1000.0; + + printf("Sub Sent %llu messages in %.3f sec (%.2f msgs/sec)\n", + pa.count, dur, pa.count / dur); + printf("Expected %llu messages total\n", expect); + printf("Received %llu messages total\n", pa.recvs); + printf("Effective rate %.2f msgs/sec\n", pa.recvs / dur); + printf("Errors %llu total\n", pa.errs); + printf("Reported %llu dropped messages in %llu gaps\n", pa.drops, + pa.gaps); + printf("Dropped %llu messages (missing)\n", missing); + printf("Drop rate %.2f%%\n", expect ? 100.0 * missing / expect : 0); + + nng_mtx_unlock(pa.mtx); +} diff --git a/src/transport/CMakeLists.txt b/src/transport/CMakeLists.txt deleted file mode 100644 index add8a9c9..00000000 --- a/src/transport/CMakeLists.txt +++ /dev/null @@ -1,19 +0,0 @@ -# -# Copyright 2020 Staysail Systems, Inc. -# -# This software is supplied under the terms of the MIT License, a -# copy of which should be located in the distribution where this -# file was obtained (LICENSE.txt). A copy of the license may also be -# found online at https://opensource.org/licenses/MIT. -# - -# Transports. -nng_directory(transport) - -add_subdirectory(inproc) -add_subdirectory(ipc) -add_subdirectory(tcp) -add_subdirectory(tls) -add_subdirectory(ws) -add_subdirectory(zerotier) - diff --git a/src/transport/inproc/CMakeLists.txt b/src/transport/inproc/CMakeLists.txt deleted file mode 100644 index 317686bb..00000000 --- a/src/transport/inproc/CMakeLists.txt +++ /dev/null @@ -1,16 +0,0 @@ -# -# Copyright 2020 Staysail Systems, Inc. -# Copyright 2018 Capitar IT Group BV -# -# This software is supplied under the terms of the MIT License, a -# copy of which should be located in the distribution where this -# file was obtained (LICENSE.txt). A copy of the license may also be -# found online at https://opensource.org/licenses/MIT. -# - -# inproc protocol -nng_directory(inproc) - -nng_sources_if(NNG_TRANSPORT_INPROC inproc.c) -nng_headers_if(NNG_TRANSPORT_INPROC nng/transport/inproc/inproc.h) -nng_defines_if(NNG_TRANSPORT_INPROC NNG_TRANSPORT_INPROC) \ No newline at end of file diff --git a/src/transport/inproc/inproc.c b/src/transport/inproc/inproc.c deleted file mode 100644 index 84e2c625..00000000 --- a/src/transport/inproc/inproc.c +++ /dev/null @@ -1,692 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// Copyright 2018 Capitar IT Group BV -// Copyright 2018 Devolutions -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - -#include - -#include "core/nng_impl.h" - -// Inproc transport. This just transports messages from one -// peer to another. The inproc transport is only valid within the same -// process. - -typedef struct inproc_pair inproc_pair; -typedef struct inproc_pipe inproc_pipe; -typedef struct inproc_ep inproc_ep; -typedef struct inproc_queue inproc_queue; - -typedef struct { - nni_mtx mx; - nni_list servers; -} inproc_global; - -// inproc_pipe represents one half of a connection. -struct inproc_pipe { - const char * addr; - inproc_pair * pair; - inproc_queue *recv_queue; - inproc_queue *send_queue; - uint16_t peer; - uint16_t proto; -}; - -struct inproc_queue { - nni_list readers; - nni_list writers; - nni_mtx lock; - bool closed; -}; - -// inproc_pair represents a pair of pipes. Because we control both -// sides of the pipes, we can allocate and free this in one structure. -struct inproc_pair { - nni_atomic_int ref; - inproc_queue queues[2]; -}; - -struct inproc_ep { - const char * addr; - bool listener; - nni_list_node node; - uint16_t proto; - nni_cv cv; - nni_list clients; - nni_list aios; - size_t rcvmax; - nni_mtx mtx; -}; - -// nni_inproc is our global state - this contains the list of active endpoints -// which we use for coordinating rendezvous. -static inproc_global nni_inproc; - -static int -inproc_init(void) -{ - NNI_LIST_INIT(&nni_inproc.servers, inproc_ep, node); - - nni_mtx_init(&nni_inproc.mx); - return (0); -} - -static void -inproc_fini(void) -{ - nni_mtx_fini(&nni_inproc.mx); -} - -// inproc_pair destroy is called when both pipe-ends of the pipe -// have been destroyed. -static void -inproc_pair_destroy(inproc_pair *pair) -{ - for (int i = 0; i < 2; i++) { - nni_mtx_fini(&pair->queues[i].lock); - } - NNI_FREE_STRUCT(pair); -} - -static int -inproc_pipe_alloc(inproc_pipe **pipep, inproc_ep *ep) -{ - inproc_pipe *pipe; - - if ((pipe = NNI_ALLOC_STRUCT(pipe)) == NULL) { - return (NNG_ENOMEM); - } - - pipe->proto = ep->proto; - pipe->addr = ep->addr; - *pipep = pipe; - return (0); -} - -static int -inproc_pipe_init(void *arg, nni_pipe *p) -{ - NNI_ARG_UNUSED(arg); - NNI_ARG_UNUSED(p); - return (0); -} - -static void -inproc_pipe_fini(void *arg) -{ - inproc_pipe *pipe = arg; - inproc_pair *pair; - - if ((pair = pipe->pair) != NULL) { - // If we are the last peer, then toss the pair structure. - if (nni_atomic_dec_nv(&pair->ref) == 0) { - inproc_pair_destroy(pair); - } - } - - NNI_FREE_STRUCT(pipe); -} - -static void -inproc_queue_run_closed(inproc_queue *queue) -{ - nni_aio *aio; - while (((aio = nni_list_first(&queue->readers)) != NULL) || - ((aio = nni_list_first(&queue->writers)) != NULL)) { - nni_aio_list_remove(aio); - nni_aio_finish_error(aio, NNG_ECLOSED); - } -} - -static void -inproc_queue_run(inproc_queue *queue) -{ - if (queue->closed) { - inproc_queue_run_closed(queue); - } - for (;;) { - nni_aio *rd; - nni_aio *wr; - nni_msg *msg; - nni_msg *pu; - - if (((rd = nni_list_first(&queue->readers)) == NULL) || - ((wr = nni_list_first(&queue->writers)) == NULL)) { - return; - } - - msg = nni_aio_get_msg(wr); - NNI_ASSERT(msg != NULL); - - // At this point, we pass success back to the caller. If - // we drop the message for any reason, its accounted on the - // receiver side. - nni_aio_list_remove(wr); - nni_aio_set_msg(wr, NULL); - nni_aio_finish( - wr, 0, nni_msg_len(msg) + nni_msg_header_len(msg)); - - // TODO: We could check the max receive size here. - - // Now the receive side. We need to ensure that we have - // an exclusive copy of the message, and pull the header - // up into the body to match protocol expectations. - if ((pu = nni_msg_pull_up(msg)) == NULL) { - nni_msg_free(msg); - continue; - } - msg = pu; - - nni_aio_list_remove(rd); - nni_aio_set_msg(rd, msg); - nni_aio_finish(rd, 0, nni_msg_len(msg)); - } -} - -static void -inproc_queue_cancel(nni_aio *aio, void *arg, int rv) -{ - inproc_queue *queue = arg; - - nni_mtx_lock(&queue->lock); - if (nni_aio_list_active(aio)) { - nni_aio_list_remove(aio); - nni_aio_finish_error(aio, rv); - } - nni_mtx_unlock(&queue->lock); -} - -static void -inproc_pipe_send(void *arg, nni_aio *aio) -{ - inproc_pipe * pipe = arg; - inproc_queue *queue = pipe->send_queue; - int rv; - - if (nni_aio_begin(aio) != 0) { - return; - } - - nni_mtx_lock(&queue->lock); - if ((rv = nni_aio_schedule(aio, inproc_queue_cancel, queue)) != 0) { - nni_mtx_unlock(&queue->lock); - nni_aio_finish_error(aio, rv); - return; - } - nni_aio_list_append(&queue->writers, aio); - inproc_queue_run(queue); - nni_mtx_unlock(&queue->lock); -} - -static void -inproc_pipe_recv(void *arg, nni_aio *aio) -{ - inproc_pipe * pipe = arg; - inproc_queue *queue = pipe->recv_queue; - int rv; - - if (nni_aio_begin(aio) != 0) { - return; - } - - nni_mtx_lock(&queue->lock); - if ((rv = nni_aio_schedule(aio, inproc_queue_cancel, queue)) != 0) { - nni_mtx_unlock(&queue->lock); - nni_aio_finish_error(aio, rv); - return; - } - nni_aio_list_append(&queue->readers, aio); - inproc_queue_run(queue); - nni_mtx_unlock(&queue->lock); -} - -static void -inproc_pipe_close(void *arg) -{ - inproc_pipe *pipe = arg; - inproc_pair *pair = pipe->pair; - - for (int i = 0; i < 2; i++) { - inproc_queue *queue = &pair->queues[i]; - nni_mtx_lock(&queue->lock); - queue->closed = true; - inproc_queue_run_closed(queue); - nni_mtx_unlock(&queue->lock); - } -} - -static uint16_t -inproc_pipe_peer(void *arg) -{ - inproc_pipe *pipe = arg; - - return (pipe->peer); -} - -static int -inproc_pipe_get_addr(void *arg, void *buf, size_t *szp, nni_opt_type t) -{ - inproc_pipe *p = arg; - nni_sockaddr sa; - - memset(&sa, 0, sizeof(sa)); - sa.s_inproc.sa_family = NNG_AF_INPROC; - nni_strlcpy(sa.s_inproc.sa_name, p->addr, sizeof(sa.s_inproc.sa_name)); - return (nni_copyout_sockaddr(&sa, buf, szp, t)); -} - -static int -inproc_dialer_init(void **epp, nni_url *url, nni_dialer *ndialer) -{ - inproc_ep *ep; - nni_sock * sock = nni_dialer_sock(ndialer); - - if ((ep = NNI_ALLOC_STRUCT(ep)) == NULL) { - return (NNG_ENOMEM); - } - nni_mtx_init(&ep->mtx); - - ep->listener = false; - ep->proto = nni_sock_proto_id(sock); - ep->rcvmax = 0; - NNI_LIST_INIT(&ep->clients, inproc_ep, node); - nni_aio_list_init(&ep->aios); - - ep->addr = url->u_rawurl; // we match on the full URL. - - *epp = ep; - return (0); -} - -static int -inproc_listener_init(void **epp, nni_url *url, nni_listener *nlistener) -{ - inproc_ep *ep; - nni_sock * sock = nni_listener_sock(nlistener); - - if ((ep = NNI_ALLOC_STRUCT(ep)) == NULL) { - return (NNG_ENOMEM); - } - nni_mtx_init(&ep->mtx); - - ep->listener = true; - ep->proto = nni_sock_proto_id(sock); - ep->rcvmax = 0; - NNI_LIST_INIT(&ep->clients, inproc_ep, node); - nni_aio_list_init(&ep->aios); - - ep->addr = url->u_rawurl; // we match on the full URL. - - *epp = ep; - return (0); -} - -static void -inproc_ep_fini(void *arg) -{ - inproc_ep *ep = arg; - nni_mtx_fini(&ep->mtx); - NNI_FREE_STRUCT(ep); -} - -static void -inproc_conn_finish(nni_aio *aio, int rv, inproc_ep *ep, inproc_pipe *pipe) -{ - nni_aio_list_remove(aio); - - if ((!ep->listener) && nni_list_empty(&ep->aios)) { - nni_list_node_remove(&ep->node); - } - - if (rv == 0) { - nni_aio_set_output(aio, 0, pipe); - nni_aio_finish(aio, 0, 0); - } else { - NNI_ASSERT(pipe == NULL); - nni_aio_finish_error(aio, rv); - } -} - -static void -inproc_ep_close(void *arg) -{ - inproc_ep *ep = arg; - inproc_ep *client; - nni_aio * aio; - - nni_mtx_lock(&nni_inproc.mx); - if (nni_list_active(&nni_inproc.servers, ep)) { - nni_list_remove(&nni_inproc.servers, ep); - } - // Notify any waiting clients that we are closed. - while ((client = nni_list_first(&ep->clients)) != NULL) { - while ((aio = nni_list_first(&client->aios)) != NULL) { - inproc_conn_finish(aio, NNG_ECONNREFUSED, ep, NULL); - } - nni_list_remove(&ep->clients, client); - } - while ((aio = nni_list_first(&ep->aios)) != NULL) { - inproc_conn_finish(aio, NNG_ECLOSED, ep, NULL); - } - nni_mtx_unlock(&nni_inproc.mx); -} - -static void -inproc_accept_clients(inproc_ep *srv) -{ - inproc_ep *cli, *nclient; - - nclient = nni_list_first(&srv->clients); - while ((cli = nclient) != NULL) { - nni_aio *caio; - nclient = nni_list_next(&srv->clients, nclient); - NNI_LIST_FOREACH (&cli->aios, caio) { - - inproc_pipe *cpipe; - inproc_pipe *spipe; - inproc_pair *pair; - nni_aio * saio; - int rv; - - if ((saio = nni_list_first(&srv->aios)) == NULL) { - // No outstanding accept() calls. - break; - } - - if ((pair = NNI_ALLOC_STRUCT(pair)) == NULL) { - inproc_conn_finish( - caio, NNG_ENOMEM, cli, NULL); - inproc_conn_finish( - saio, NNG_ENOMEM, srv, NULL); - continue; - } - for (int i = 0; i < 2; i++) { - nni_aio_list_init(&pair->queues[i].readers); - nni_aio_list_init(&pair->queues[i].writers); - nni_mtx_init(&pair->queues[i].lock); - } - nni_atomic_init(&pair->ref); - nni_atomic_set(&pair->ref, 2); - - spipe = cpipe = NULL; - if (((rv = inproc_pipe_alloc(&cpipe, cli)) != 0) || - ((rv = inproc_pipe_alloc(&spipe, srv)) != 0)) { - - if (cpipe != NULL) { - inproc_pipe_fini(cpipe); - } - if (spipe != NULL) { - inproc_pipe_fini(spipe); - } - inproc_conn_finish(caio, rv, cli, NULL); - inproc_conn_finish(saio, rv, srv, NULL); - inproc_pair_destroy(pair); - continue; - } - - cpipe->peer = spipe->proto; - spipe->peer = cpipe->proto; - cpipe->pair = pair; - spipe->pair = pair; - cpipe->send_queue = &pair->queues[0]; - cpipe->recv_queue = &pair->queues[1]; - spipe->send_queue = &pair->queues[1]; - spipe->recv_queue = &pair->queues[0]; - - inproc_conn_finish(caio, 0, cli, cpipe); - inproc_conn_finish(saio, 0, srv, spipe); - } - - if (nni_list_first(&cli->aios) == NULL) { - // No more outstanding client connects. - // Normally there should only be one. - if (nni_list_active(&srv->clients, cli)) { - nni_list_remove(&srv->clients, cli); - } - } - } -} - -static void -inproc_ep_cancel(nni_aio *aio, void *arg, int rv) -{ - inproc_ep *ep = arg; - - nni_mtx_lock(&nni_inproc.mx); - if (nni_aio_list_active(aio)) { - nni_aio_list_remove(aio); - nni_list_node_remove(&ep->node); - nni_aio_finish_error(aio, rv); - } - nni_mtx_unlock(&nni_inproc.mx); -} - -static void -inproc_ep_connect(void *arg, nni_aio *aio) -{ - inproc_ep *ep = arg; - inproc_ep *server; - int rv; - - if (nni_aio_begin(aio) != 0) { - return; - } - - nni_mtx_lock(&nni_inproc.mx); - - // Find a server. - NNI_LIST_FOREACH (&nni_inproc.servers, server) { - if (strcmp(server->addr, ep->addr) == 0) { - break; - } - } - if (server == NULL) { - nni_mtx_unlock(&nni_inproc.mx); - nni_aio_finish_error(aio, NNG_ECONNREFUSED); - return; - } - - // We don't have to worry about the case where a zero timeout - // on connect was specified, as there is no option to specify - // that in the upper API. - if ((rv = nni_aio_schedule(aio, inproc_ep_cancel, ep)) != 0) { - nni_mtx_unlock(&nni_inproc.mx); - nni_aio_finish_error(aio, rv); - return; - } - - nni_list_append(&server->clients, ep); - nni_aio_list_append(&ep->aios, aio); - - inproc_accept_clients(server); - nni_mtx_unlock(&nni_inproc.mx); -} - -static int -inproc_ep_bind(void *arg) -{ - inproc_ep *ep = arg; - inproc_ep *srch; - nni_list * list = &nni_inproc.servers; - - nni_mtx_lock(&nni_inproc.mx); - NNI_LIST_FOREACH (list, srch) { - if (strcmp(srch->addr, ep->addr) == 0) { - nni_mtx_unlock(&nni_inproc.mx); - return (NNG_EADDRINUSE); - } - } - nni_list_append(list, ep); - nni_mtx_unlock(&nni_inproc.mx); - return (0); -} - -static void -inproc_ep_accept(void *arg, nni_aio *aio) -{ - inproc_ep *ep = arg; - int rv; - - if (nni_aio_begin(aio) != 0) { - return; - } - - nni_mtx_lock(&nni_inproc.mx); - - // We need not worry about the case where a non-blocking - // accept was tried -- there is no API to do such a thing. - if ((rv = nni_aio_schedule(aio, inproc_ep_cancel, ep)) != 0) { - nni_mtx_unlock(&nni_inproc.mx); - nni_aio_finish_error(aio, rv); - return; - } - - // We are already on the master list of servers, thanks to bind. - // Insert us into pending server aios, and then run accept list. - nni_aio_list_append(&ep->aios, aio); - inproc_accept_clients(ep); - nni_mtx_unlock(&nni_inproc.mx); -} - -static int -inproc_ep_get_recvmaxsz(void *arg, void *v, size_t *szp, nni_opt_type t) -{ - inproc_ep *ep = arg; - int rv; - nni_mtx_lock(&ep->mtx); - rv = nni_copyout_size(ep->rcvmax, v, szp, t); - nni_mtx_unlock(&ep->mtx); - return (rv); -} - -static int -inproc_ep_set_recvmaxsz(void *arg, const void *v, size_t sz, nni_opt_type t) -{ - inproc_ep *ep = arg; - size_t val; - int rv; - if ((rv = nni_copyin_size(&val, v, sz, 0, NNI_MAXSZ, t)) == 0) { - nni_mtx_lock(&ep->mtx); - ep->rcvmax = val; - nni_mtx_unlock(&ep->mtx); - } - return (rv); -} - -static int -inproc_ep_get_addr(void *arg, void *v, size_t *szp, nni_opt_type t) -{ - inproc_ep * ep = arg; - nng_sockaddr sa; - sa.s_inproc.sa_family = NNG_AF_INPROC; - nni_strlcpy( - sa.s_inproc.sa_name, ep->addr, sizeof(sa.s_inproc.sa_name)); - return (nni_copyout_sockaddr(&sa, v, szp, t)); -} - -static const nni_option inproc_pipe_options[] = { - { - .o_name = NNG_OPT_LOCADDR, - .o_get = inproc_pipe_get_addr, - }, - { - .o_name = NNG_OPT_REMADDR, - .o_get = inproc_pipe_get_addr, - }, - // terminate list - { - .o_name = NULL, - }, -}; - -static int -inproc_pipe_getopt( - void *arg, const char *name, void *v, size_t *szp, nni_type t) -{ - return (nni_getopt(inproc_pipe_options, name, arg, v, szp, t)); -} - -static nni_tran_pipe_ops inproc_pipe_ops = { - .p_init = inproc_pipe_init, - .p_fini = inproc_pipe_fini, - .p_send = inproc_pipe_send, - .p_recv = inproc_pipe_recv, - .p_close = inproc_pipe_close, - .p_peer = inproc_pipe_peer, - .p_getopt = inproc_pipe_getopt, -}; - -static const nni_option inproc_ep_options[] = { - { - .o_name = NNG_OPT_RECVMAXSZ, - .o_get = inproc_ep_get_recvmaxsz, - .o_set = inproc_ep_set_recvmaxsz, - }, - { - .o_name = NNG_OPT_LOCADDR, - .o_get = inproc_ep_get_addr, - }, - { - .o_name = NNG_OPT_REMADDR, - .o_get = inproc_ep_get_addr, - }, - // terminate list - { - .o_name = NULL, - }, -}; - -static int -inproc_ep_getopt(void *arg, const char *name, void *v, size_t *szp, nni_type t) -{ - return (nni_getopt(inproc_ep_options, name, arg, v, szp, t)); -} - -static int -inproc_ep_setopt( - void *arg, const char *name, const void *v, size_t sz, nni_type t) -{ - return (nni_setopt(inproc_ep_options, name, arg, v, sz, t)); -} - -static nni_tran_dialer_ops inproc_dialer_ops = { - .d_init = inproc_dialer_init, - .d_fini = inproc_ep_fini, - .d_connect = inproc_ep_connect, - .d_close = inproc_ep_close, - .d_getopt = inproc_ep_getopt, - .d_setopt = inproc_ep_setopt, -}; - -static nni_tran_listener_ops inproc_listener_ops = { - .l_init = inproc_listener_init, - .l_fini = inproc_ep_fini, - .l_bind = inproc_ep_bind, - .l_accept = inproc_ep_accept, - .l_close = inproc_ep_close, - .l_getopt = inproc_ep_getopt, - .l_setopt = inproc_ep_setopt, -}; - -// This is the inproc transport linkage, and should be the only global -// symbol in this entire file. -struct nni_tran nni_inproc_tran = { - .tran_version = NNI_TRANSPORT_VERSION, - .tran_scheme = "inproc", - .tran_dialer = &inproc_dialer_ops, - .tran_listener = &inproc_listener_ops, - .tran_pipe = &inproc_pipe_ops, - .tran_init = inproc_init, - .tran_fini = inproc_fini, -}; - -int -nng_inproc_register(void) -{ - return (nni_tran_register(&nni_inproc_tran)); -} diff --git a/src/transport/ipc/CMakeLists.txt b/src/transport/ipc/CMakeLists.txt deleted file mode 100644 index c9927f75..00000000 --- a/src/transport/ipc/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -# -# Copyright 2020 Staysail Systems, Inc. -# Copyright 2018 Capitar IT Group BV -# -# This software is supplied under the terms of the MIT License, a -# copy of which should be located in the distribution where this -# file was obtained (LICENSE.txt). A copy of the license may also be -# found online at https://opensource.org/licenses/MIT. -# - -# ipc protocol -nng_directory(ipc) - -nng_sources_if(NNG_TRANSPORT_IPC ipc.c) -nng_headers_if(NNG_TRANSPORT_IPC nng/transport/ipc/ipc.h) -nng_defines_if(NNG_TRANSPORT_IPC NNG_TRANSPORT_IPC) -nng_test_if(NNG_TRANSPORT_IPC ipc_test) \ No newline at end of file diff --git a/src/transport/ipc/ipc.c b/src/transport/ipc/ipc.c deleted file mode 100644 index efaa823c..00000000 --- a/src/transport/ipc/ipc.c +++ /dev/null @@ -1,1171 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// Copyright 2018 Capitar IT Group BV -// Copyright 2019 Devolutions -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - -#include -#include - -#include "core/nng_impl.h" - -#include - -// IPC transport. Platform specific IPC operations must be -// supplied as well. Normally the IPC is UNIX domain sockets or -// Windows named pipes. Other platforms could use other mechanisms, -// but all implementations on the platform must use the same mechanism. - -typedef struct ipc_pipe ipc_pipe; -typedef struct ipc_ep ipc_ep; - -// ipc_pipe is one end of an IPC connection. -struct ipc_pipe { - nng_stream * conn; - uint16_t peer; - uint16_t proto; - size_t rcv_max; - bool closed; - nni_sockaddr sa; - ipc_ep * ep; - nni_pipe * pipe; - nni_list_node node; - nni_atomic_flag reaped; - nni_reap_node reap; - uint8_t tx_head[1 + sizeof(uint64_t)]; - uint8_t rx_head[1 + sizeof(uint64_t)]; - size_t got_tx_head; - size_t got_rx_head; - size_t want_tx_head; - size_t want_rx_head; - nni_list recv_q; - nni_list send_q; - nni_aio tx_aio; - nni_aio rx_aio; - nni_aio neg_aio; - nni_msg * rx_msg; - nni_mtx mtx; -}; - -struct ipc_ep { - nni_mtx mtx; - nni_sockaddr sa; - size_t rcv_max; - uint16_t proto; - bool started; - bool closed; - bool fini; - int ref_cnt; - nng_stream_dialer * dialer; - nng_stream_listener *listener; - nni_aio * user_aio; - nni_aio * conn_aio; - nni_aio * time_aio; - nni_list busy_pipes; // busy pipes -- ones passed to socket - nni_list wait_pipes; // pipes waiting to match to socket - nni_list neg_pipes; // pipes busy negotiating - nni_reap_node reap; -#ifdef NNG_ENABLE_STATS - nni_stat_item st_rcv_max; -#endif -}; - -static void ipc_pipe_send_start(ipc_pipe *p); -static void ipc_pipe_recv_start(ipc_pipe *p); -static void ipc_pipe_send_cb(void *); -static void ipc_pipe_recv_cb(void *); -static void ipc_pipe_neg_cb(void *); -static void ipc_pipe_fini(void *); -static void ipc_ep_fini(void *); - -static nni_reap_list ipc_ep_reap_list = { - .rl_offset = offsetof(ipc_ep, reap), - .rl_func = ipc_ep_fini, -}; - -static nni_reap_list ipc_pipe_reap_list = { - .rl_offset = offsetof(ipc_pipe, reap), - .rl_func = ipc_pipe_fini, -}; - -static int -ipc_tran_init(void) -{ - return (0); -} - -static void -ipc_tran_fini(void) -{ -} - -static void -ipc_pipe_close(void *arg) -{ - ipc_pipe *p = arg; - - nni_mtx_lock(&p->mtx); - p->closed = true; - nni_mtx_unlock(&p->mtx); - - nni_aio_close(&p->rx_aio); - nni_aio_close(&p->tx_aio); - nni_aio_close(&p->neg_aio); - - nng_stream_close(p->conn); -} - -static void -ipc_pipe_stop(void *arg) -{ - ipc_pipe *p = arg; - - nni_aio_stop(&p->rx_aio); - nni_aio_stop(&p->tx_aio); - nni_aio_stop(&p->neg_aio); -} - -static int -ipc_pipe_init(void *arg, nni_pipe *pipe) -{ - ipc_pipe *p = arg; - p->pipe = pipe; - return (0); -} - -static void -ipc_pipe_fini(void *arg) -{ - ipc_pipe *p = arg; - ipc_ep * ep; - - ipc_pipe_stop(p); - if ((ep = p->ep) != NULL) { - nni_mtx_lock(&ep->mtx); - nni_list_node_remove(&p->node); - ep->ref_cnt--; - if (ep->fini && (ep->ref_cnt == 0)) { - nni_reap(&ipc_ep_reap_list, ep); - } - nni_mtx_unlock(&ep->mtx); - } - nni_aio_fini(&p->rx_aio); - nni_aio_fini(&p->tx_aio); - nni_aio_fini(&p->neg_aio); - nng_stream_free(p->conn); - if (p->rx_msg) { - nni_msg_free(p->rx_msg); - } - nni_mtx_fini(&p->mtx); - NNI_FREE_STRUCT(p); -} - -static void -ipc_pipe_reap(ipc_pipe *p) -{ - if (!nni_atomic_flag_test_and_set(&p->reaped)) { - if (p->conn != NULL) { - nng_stream_close(p->conn); - } - nni_reap(&ipc_pipe_reap_list, p); - } -} - -static int -ipc_pipe_alloc(ipc_pipe **pipe_p) -{ - ipc_pipe *p; - - if ((p = NNI_ALLOC_STRUCT(p)) == NULL) { - return (NNG_ENOMEM); - } - nni_mtx_init(&p->mtx); - nni_aio_init(&p->tx_aio, ipc_pipe_send_cb, p); - nni_aio_init(&p->rx_aio, ipc_pipe_recv_cb, p); - nni_aio_init(&p->neg_aio, ipc_pipe_neg_cb, p); - nni_aio_list_init(&p->send_q); - nni_aio_list_init(&p->recv_q); - nni_atomic_flag_reset(&p->reaped); - *pipe_p = p; - return (0); -} - -static void -ipc_ep_match(ipc_ep *ep) -{ - nni_aio * aio; - ipc_pipe *p; - - if (((aio = ep->user_aio) == NULL) || - ((p = nni_list_first(&ep->wait_pipes)) == NULL)) { - return; - } - nni_list_remove(&ep->wait_pipes, p); - nni_list_append(&ep->busy_pipes, p); - ep->user_aio = NULL; - p->rcv_max = ep->rcv_max; - nni_aio_set_output(aio, 0, p); - nni_aio_finish(aio, 0, 0); -} - -static void -ipc_pipe_neg_cb(void *arg) -{ - ipc_pipe *p = arg; - ipc_ep * ep = p->ep; - nni_aio * aio = &p->neg_aio; - nni_aio * user_aio; - int rv; - - nni_mtx_lock(&ep->mtx); - if ((rv = nni_aio_result(aio)) != 0) { - goto error; - } - - // We start transmitting before we receive. - if (p->got_tx_head < p->want_tx_head) { - p->got_tx_head += nni_aio_count(aio); - } else if (p->got_rx_head < p->want_rx_head) { - p->got_rx_head += nni_aio_count(aio); - } - if (p->got_tx_head < p->want_tx_head) { - nni_iov iov; - iov.iov_len = p->want_tx_head - p->got_tx_head; - iov.iov_buf = &p->tx_head[p->got_tx_head]; - nni_aio_set_iov(aio, 1, &iov); - // send it down... - nng_stream_send(p->conn, aio); - nni_mtx_unlock(&p->ep->mtx); - return; - } - if (p->got_rx_head < p->want_rx_head) { - nni_iov iov; - iov.iov_len = p->want_rx_head - p->got_rx_head; - iov.iov_buf = &p->rx_head[p->got_rx_head]; - nni_aio_set_iov(aio, 1, &iov); - nng_stream_recv(p->conn, aio); - nni_mtx_unlock(&p->ep->mtx); - return; - } - // We have both sent and received the headers. Lets check the - // receive side header. - if ((p->rx_head[0] != 0) || (p->rx_head[1] != 'S') || - (p->rx_head[2] != 'P') || (p->rx_head[3] != 0) || - (p->rx_head[6] != 0) || (p->rx_head[7] != 0)) { - rv = NNG_EPROTO; - goto error; - } - - NNI_GET16(&p->rx_head[4], p->peer); - - // We are all ready now. We put this in the wait list, and - // then try to run the matcher. - nni_list_remove(&ep->neg_pipes, p); - nni_list_append(&ep->wait_pipes, p); - - ipc_ep_match(ep); - nni_mtx_unlock(&ep->mtx); - return; - -error: - - nng_stream_close(p->conn); - // If we are waiting to negotiate on a client side, then a failure - // here has to be passed to the user app. - if ((user_aio = ep->user_aio) != NULL) { - ep->user_aio = NULL; - nni_aio_finish_error(user_aio, rv); - } - nni_mtx_unlock(&ep->mtx); - ipc_pipe_reap(p); -} - -static void -ipc_pipe_send_cb(void *arg) -{ - ipc_pipe *p = arg; - int rv; - nni_aio * aio; - size_t n; - nni_msg * msg; - nni_aio * tx_aio = &p->tx_aio; - - nni_mtx_lock(&p->mtx); - if ((rv = nni_aio_result(tx_aio)) != 0) { - nni_pipe_bump_error(p->pipe, rv); - // Intentionally we do not queue up another transfer. - // There's an excellent chance that the pipe is no longer - // usable, with a partial transfer. - // The protocol should see this error, and close the - // pipe itself, we hope. - - while ((aio = nni_list_first(&p->send_q)) != NULL) { - nni_aio_list_remove(aio); - nni_aio_finish_error(aio, rv); - } - nni_mtx_unlock(&p->mtx); - return; - } - - n = nni_aio_count(tx_aio); - nni_aio_iov_advance(tx_aio, n); - if (nni_aio_iov_count(tx_aio) != 0) { - nng_stream_send(p->conn, tx_aio); - nni_mtx_unlock(&p->mtx); - return; - } - - aio = nni_list_first(&p->send_q); - nni_aio_list_remove(aio); - ipc_pipe_send_start(p); - - msg = nni_aio_get_msg(aio); - n = nni_msg_len(msg); - nni_pipe_bump_tx(p->pipe, n); - nni_mtx_unlock(&p->mtx); - - nni_aio_set_msg(aio, NULL); - nni_msg_free(msg); - nni_aio_finish_sync(aio, 0, n); -} - -static void -ipc_pipe_recv_cb(void *arg) -{ - ipc_pipe *p = arg; - nni_aio * aio; - int rv; - size_t n; - nni_msg * msg; - nni_aio * rx_aio = &p->rx_aio; - - nni_mtx_lock(&p->mtx); - - if ((rv = nni_aio_result(rx_aio)) != 0) { - // Error on receive. This has to cause an error back - // to the user. Also, if we had allocated an rx_msg, lets - // toss it. - goto error; - } - - n = nni_aio_count(rx_aio); - nni_aio_iov_advance(rx_aio, n); - if (nni_aio_iov_count(rx_aio) != 0) { - // Was this a partial read? If so then resubmit for the rest. - nng_stream_recv(p->conn, rx_aio); - nni_mtx_unlock(&p->mtx); - return; - } - - // If we don't have a message yet, we were reading the message - // header, which is just the length. This tells us the size of the - // message to allocate and how much more to expect. - if (p->rx_msg == NULL) { - uint64_t len; - - // Check to make sure we got msg type 1. - if (p->rx_head[0] != 1) { - rv = NNG_EPROTO; - goto error; - } - - // We should have gotten a message header. - NNI_GET64(p->rx_head + 1, len); - - // Make sure the message payload is not too big. If it is - // the caller will shut down the pipe. - if ((len > p->rcv_max) && (p->rcv_max > 0)) { - rv = NNG_EMSGSIZE; - goto error; - } - - // Note that all IO on this pipe is blocked behind this - // allocation. We could possibly look at using a separate - // lock for the read side in the future, so that we allow - // transmits to proceed normally. In practice this is - // unlikely to be much of an issue though. - if ((rv = nni_msg_alloc(&p->rx_msg, (size_t) len)) != 0) { - goto error; - } - - if (len != 0) { - nni_iov iov; - // Submit the rest of the data for a read -- we want to - // read the entire message now. - iov.iov_buf = nni_msg_body(p->rx_msg); - iov.iov_len = (size_t) len; - - nni_aio_set_iov(rx_aio, 1, &iov); - nng_stream_recv(p->conn, rx_aio); - nni_mtx_unlock(&p->mtx); - return; - } - } - - // Otherwise we got a message read completely. Let the user know the - // good news. - - aio = nni_list_first(&p->recv_q); - nni_aio_list_remove(aio); - msg = p->rx_msg; - p->rx_msg = NULL; - n = nni_msg_len(msg); - nni_pipe_bump_rx(p->pipe, n); - ipc_pipe_recv_start(p); - nni_mtx_unlock(&p->mtx); - - nni_aio_set_msg(aio, msg); - nni_aio_finish_sync(aio, 0, n); - return; - -error: - while ((aio = nni_list_first(&p->recv_q)) != NULL) { - nni_aio_list_remove(aio); - nni_aio_finish_error(aio, rv); - } - msg = p->rx_msg; - p->rx_msg = NULL; - nni_pipe_bump_error(p->pipe, rv); - // Intentionally, we do not queue up another receive. - // The protocol should notice this error and close the pipe. - nni_mtx_unlock(&p->mtx); - - nni_msg_free(msg); -} - -static void -ipc_pipe_send_cancel(nni_aio *aio, void *arg, int rv) -{ - ipc_pipe *p = arg; - - nni_mtx_lock(&p->mtx); - if (!nni_aio_list_active(aio)) { - nni_mtx_unlock(&p->mtx); - return; - } - // If this is being sent, then cancel the pending transfer. - // The callback on the tx_aio will cause the user aio to - // be canceled too. - if (nni_list_first(&p->send_q) == aio) { - nni_aio_abort(&p->tx_aio, rv); - nni_mtx_unlock(&p->mtx); - return; - } - nni_aio_list_remove(aio); - nni_mtx_unlock(&p->mtx); - - nni_aio_finish_error(aio, rv); -} - -static void -ipc_pipe_send_start(ipc_pipe *p) -{ - nni_aio *aio; - nni_msg *msg; - int nio; - nni_iov iov[3]; - uint64_t len; - - if (p->closed) { - while ((aio = nni_list_first(&p->send_q)) != NULL) { - nni_list_remove(&p->send_q, aio); - nni_aio_finish_error(aio, NNG_ECLOSED); - } - return; - } - if ((aio = nni_list_first(&p->send_q)) == NULL) { - return; - } - - // This runs to send the message. - msg = nni_aio_get_msg(aio); - len = nni_msg_len(msg) + nni_msg_header_len(msg); - - p->tx_head[0] = 1; // message type, 1. - NNI_PUT64(p->tx_head + 1, len); - - nio = 0; - iov[0].iov_buf = p->tx_head; - iov[0].iov_len = sizeof(p->tx_head); - nio++; - if (nni_msg_header_len(msg) > 0) { - iov[nio].iov_buf = nni_msg_header(msg); - iov[nio].iov_len = nni_msg_header_len(msg); - nio++; - } - if (nni_msg_len(msg) > 0) { - iov[nio].iov_buf = nni_msg_body(msg); - iov[nio].iov_len = nni_msg_len(msg); - nio++; - } - nni_aio_set_iov(&p->tx_aio, nio, iov); - nng_stream_send(p->conn, &p->tx_aio); -} - -static void -ipc_pipe_send(void *arg, nni_aio *aio) -{ - ipc_pipe *p = arg; - int rv; - - if (nni_aio_begin(aio) != 0) { - return; - } - nni_mtx_lock(&p->mtx); - if ((rv = nni_aio_schedule(aio, ipc_pipe_send_cancel, p)) != 0) { - nni_mtx_unlock(&p->mtx); - nni_aio_finish_error(aio, rv); - return; - } - nni_list_append(&p->send_q, aio); - if (nni_list_first(&p->send_q) == aio) { - ipc_pipe_send_start(p); - } - nni_mtx_unlock(&p->mtx); -} - -static void -ipc_pipe_recv_cancel(nni_aio *aio, void *arg, int rv) -{ - ipc_pipe *p = arg; - - nni_mtx_lock(&p->mtx); - if (!nni_aio_list_active(aio)) { - nni_mtx_unlock(&p->mtx); - return; - } - // If receive in progress, then cancel the pending transfer. - // The callback on the rx_aio will cause the user aio to - // be canceled too. - if (nni_list_first(&p->recv_q) == aio) { - nni_aio_abort(&p->rx_aio, rv); - nni_mtx_unlock(&p->mtx); - return; - } - nni_aio_list_remove(aio); - nni_mtx_unlock(&p->mtx); - nni_aio_finish_error(aio, rv); -} - -static void -ipc_pipe_recv_start(ipc_pipe *p) -{ - nni_iov iov; - NNI_ASSERT(p->rx_msg == NULL); - - if (p->closed) { - nni_aio *aio; - while ((aio = nni_list_first(&p->recv_q)) != NULL) { - nni_list_remove(&p->recv_q, aio); - nni_aio_finish_error(aio, NNG_ECLOSED); - } - return; - } - if (nni_list_empty(&p->recv_q)) { - return; - } - - // Schedule a read of the IPC header. - iov.iov_buf = p->rx_head; - iov.iov_len = sizeof(p->rx_head); - nni_aio_set_iov(&p->rx_aio, 1, &iov); - - nng_stream_recv(p->conn, &p->rx_aio); -} - -static void -ipc_pipe_recv(void *arg, nni_aio *aio) -{ - ipc_pipe *p = arg; - int rv; - - if (nni_aio_begin(aio) != 0) { - return; - } - nni_mtx_lock(&p->mtx); - if (p->closed) { - nni_mtx_unlock(&p->mtx); - nni_aio_finish_error(aio, NNG_ECLOSED); - return; - } - if ((rv = nni_aio_schedule(aio, ipc_pipe_recv_cancel, p)) != 0) { - nni_mtx_unlock(&p->mtx); - nni_aio_finish_error(aio, rv); - return; - } - - nni_list_append(&p->recv_q, aio); - if (nni_list_first(&p->recv_q) == aio) { - ipc_pipe_recv_start(p); - } - nni_mtx_unlock(&p->mtx); -} - -static uint16_t -ipc_pipe_peer(void *arg) -{ - ipc_pipe *p = arg; - - return (p->peer); -} - -static void -ipc_pipe_start(ipc_pipe *p, nng_stream *conn, ipc_ep *ep) -{ - nni_iov iov; - - ep->ref_cnt++; - - p->conn = conn; - p->ep = ep; - p->proto = ep->proto; - - p->tx_head[0] = 0; - p->tx_head[1] = 'S'; - p->tx_head[2] = 'P'; - p->tx_head[3] = 0; - NNI_PUT16(&p->tx_head[4], p->proto); - NNI_PUT16(&p->tx_head[6], 0); - - p->got_rx_head = 0; - p->got_tx_head = 0; - p->want_rx_head = 8; - p->want_tx_head = 8; - iov.iov_len = 8; - iov.iov_buf = &p->tx_head[0]; - nni_aio_set_iov(&p->neg_aio, 1, &iov); - nni_list_append(&ep->neg_pipes, p); - - nni_aio_set_timeout(&p->neg_aio, 10000); // 10 sec timeout to negotiate - nng_stream_send(p->conn, &p->neg_aio); -} - -static void -ipc_ep_close(void *arg) -{ - ipc_ep * ep = arg; - ipc_pipe *p; - - nni_mtx_lock(&ep->mtx); - ep->closed = true; - nni_aio_close(ep->time_aio); - if (ep->dialer != NULL) { - nng_stream_dialer_close(ep->dialer); - } - if (ep->listener != NULL) { - nng_stream_listener_close(ep->listener); - } - NNI_LIST_FOREACH (&ep->neg_pipes, p) { - ipc_pipe_close(p); - } - NNI_LIST_FOREACH (&ep->wait_pipes, p) { - ipc_pipe_close(p); - } - NNI_LIST_FOREACH (&ep->busy_pipes, p) { - ipc_pipe_close(p); - } - if (ep->user_aio != NULL) { - nni_aio_finish_error(ep->user_aio, NNG_ECLOSED); - ep->user_aio = NULL; - } - nni_mtx_unlock(&ep->mtx); -} - -static void -ipc_ep_fini(void *arg) -{ - ipc_ep *ep = arg; - - nni_mtx_lock(&ep->mtx); - ep->fini = true; - if (ep->ref_cnt != 0) { - nni_mtx_unlock(&ep->mtx); - return; - } - nni_mtx_unlock(&ep->mtx); - nni_aio_stop(ep->time_aio); - nni_aio_stop(ep->conn_aio); - nng_stream_dialer_free(ep->dialer); - nng_stream_listener_free(ep->listener); - nni_aio_free(ep->time_aio); - nni_aio_free(ep->conn_aio); - nni_mtx_fini(&ep->mtx); - NNI_FREE_STRUCT(ep); -} - -static void -ipc_ep_timer_cb(void *arg) -{ - ipc_ep *ep = arg; - nni_mtx_lock(&ep->mtx); - if (nni_aio_result(ep->time_aio) == 0) { - nng_stream_listener_accept(ep->listener, ep->conn_aio); - } - nni_mtx_unlock(&ep->mtx); -} - -static void -ipc_ep_accept_cb(void *arg) -{ - ipc_ep * ep = arg; - nni_aio * aio = ep->conn_aio; - ipc_pipe * p; - int rv; - nng_stream *conn; - - nni_mtx_lock(&ep->mtx); - if ((rv = nni_aio_result(aio)) != 0) { - goto error; - } - - conn = nni_aio_get_output(aio, 0); - if ((rv = ipc_pipe_alloc(&p)) != 0) { - nng_stream_free(conn); - goto error; - } - if (ep->closed) { - ipc_pipe_fini(p); - nng_stream_free(conn); - rv = NNG_ECLOSED; - goto error; - } - ipc_pipe_start(p, conn, ep); - nng_stream_listener_accept(ep->listener, ep->conn_aio); - nni_mtx_unlock(&ep->mtx); - return; - -error: - // When an error here occurs, let's send a notice up to the consumer. - // That way it can be reported properly. - if ((aio = ep->user_aio) != NULL) { - ep->user_aio = NULL; - nni_aio_finish_error(aio, rv); - } - - switch (rv) { - - case NNG_ENOMEM: - case NNG_ENOFILES: - nng_sleep_aio(10, ep->time_aio); - break; - - default: - if (!ep->closed) { - nng_stream_listener_accept(ep->listener, ep->conn_aio); - } - break; - } - nni_mtx_unlock(&ep->mtx); -} - -static void -ipc_ep_dial_cb(void *arg) -{ - ipc_ep * ep = arg; - nni_aio * aio = ep->conn_aio; - ipc_pipe * p; - int rv; - nng_stream *conn; - - if ((rv = nni_aio_result(aio)) != 0) { - goto error; - } - - conn = nni_aio_get_output(aio, 0); - if ((rv = ipc_pipe_alloc(&p)) != 0) { - nng_stream_free(conn); - goto error; - } - nni_mtx_lock(&ep->mtx); - if (ep->closed) { - ipc_pipe_fini(p); - nng_stream_free(conn); - rv = NNG_ECLOSED; - nni_mtx_unlock(&ep->mtx); - goto error; - } else { - ipc_pipe_start(p, conn, ep); - } - nni_mtx_unlock(&ep->mtx); - return; - -error: - // Error connecting. We need to pass this straight back - // to the user. - nni_mtx_lock(&ep->mtx); - if ((aio = ep->user_aio) != NULL) { - ep->user_aio = NULL; - nni_aio_finish_error(aio, rv); - } - nni_mtx_unlock(&ep->mtx); -} - -static int -ipc_ep_init(ipc_ep **epp, nni_sock *sock) -{ - ipc_ep *ep; - - if ((ep = NNI_ALLOC_STRUCT(ep)) == NULL) { - return (NNG_ENOMEM); - } - nni_mtx_init(&ep->mtx); - NNI_LIST_INIT(&ep->busy_pipes, ipc_pipe, node); - NNI_LIST_INIT(&ep->wait_pipes, ipc_pipe, node); - NNI_LIST_INIT(&ep->neg_pipes, ipc_pipe, node); - - ep->proto = nni_sock_proto_id(sock); - -#ifdef NNG_ENABLE_STATS - static const nni_stat_info rcv_max_info = { - .si_name = "rcv_max", - .si_desc = "maximum receive size", - .si_type = NNG_STAT_LEVEL, - .si_unit = NNG_UNIT_BYTES, - .si_atomic = true, - }; - nni_stat_init(&ep->st_rcv_max, &rcv_max_info); -#endif - - *epp = ep; - return (0); -} - -static int -ipc_ep_init_dialer(void **dp, nni_url *url, nni_dialer *dialer) -{ - ipc_ep * ep; - int rv; - nni_sock *sock = nni_dialer_sock(dialer); - - if ((rv = ipc_ep_init(&ep, sock)) != 0) { - return (rv); - } - - if (((rv = nni_aio_alloc(&ep->conn_aio, ipc_ep_dial_cb, ep)) != 0) || - ((rv = nng_stream_dialer_alloc_url(&ep->dialer, url)) != 0)) { - ipc_ep_fini(ep); - return (rv); - } -#ifdef NNG_ENABLE_STATS - nni_dialer_add_stat(dialer, &ep->st_rcv_max); -#endif - *dp = ep; - return (0); -} - -static int -ipc_ep_init_listener(void **dp, nni_url *url, nni_listener *listener) -{ - ipc_ep * ep; - int rv; - nni_sock *sock = nni_listener_sock(listener); - - if ((rv = ipc_ep_init(&ep, sock)) != 0) { - return (rv); - } - - if (((rv = nni_aio_alloc(&ep->conn_aio, ipc_ep_accept_cb, ep)) != 0) || - ((rv = nni_aio_alloc(&ep->time_aio, ipc_ep_timer_cb, ep)) != 0) || - ((rv = nng_stream_listener_alloc_url(&ep->listener, url)) != 0)) { - ipc_ep_fini(ep); - return (rv); - } - -#ifdef NNG_ENABLE_STATS - nni_listener_add_stat(listener, &ep->st_rcv_max); -#endif - *dp = ep; - return (0); -} - -static void -ipc_ep_cancel(nni_aio *aio, void *arg, int rv) -{ - ipc_ep *ep = arg; - nni_mtx_lock(&ep->mtx); - if (aio == ep->user_aio) { - ep->user_aio = NULL; - nni_aio_finish_error(aio, rv); - } - nni_mtx_unlock(&ep->mtx); -} - -static void -ipc_ep_connect(void *arg, nni_aio *aio) -{ - ipc_ep *ep = arg; - int rv; - - if (nni_aio_begin(aio) != 0) { - return; - } - nni_mtx_lock(&ep->mtx); - if (ep->closed) { - nni_mtx_unlock(&ep->mtx); - nni_aio_finish_error(aio, NNG_ECLOSED); - return; - } - if (ep->user_aio != NULL) { - nni_mtx_unlock(&ep->mtx); - nni_aio_finish_error(aio, NNG_EBUSY); - return; - } - - if ((rv = nni_aio_schedule(aio, ipc_ep_cancel, ep)) != 0) { - nni_mtx_unlock(&ep->mtx); - nni_aio_finish_error(aio, rv); - return; - } - ep->user_aio = aio; - nng_stream_dialer_dial(ep->dialer, ep->conn_aio); - nni_mtx_unlock(&ep->mtx); -} - -static int -ipc_ep_get_recv_max_sz(void *arg, void *v, size_t *szp, nni_type t) -{ - ipc_ep *ep = arg; - int rv; - nni_mtx_lock(&ep->mtx); - rv = nni_copyout_size(ep->rcv_max, v, szp, t); - nni_mtx_unlock(&ep->mtx); - return (rv); -} - -static int -ipc_ep_set_recv_max_sz(void *arg, const void *v, size_t sz, nni_type t) -{ - ipc_ep *ep = arg; - size_t val; - int rv; - if ((rv = nni_copyin_size(&val, v, sz, 0, NNI_MAXSZ, t)) == 0) { - - ipc_pipe *p; - nni_mtx_lock(&ep->mtx); - ep->rcv_max = val; - NNI_LIST_FOREACH (&ep->wait_pipes, p) { - p->rcv_max = val; - } - NNI_LIST_FOREACH (&ep->neg_pipes, p) { - p->rcv_max = val; - } - NNI_LIST_FOREACH (&ep->busy_pipes, p) { - p->rcv_max = val; - } - nni_mtx_unlock(&ep->mtx); -#ifdef NNG_ENABLE_STATS - nni_stat_set_value(&ep->st_rcv_max, val); -#endif - } - return (rv); -} - -static int -ipc_ep_bind(void *arg) -{ - ipc_ep *ep = arg; - int rv; - - nni_mtx_lock(&ep->mtx); - rv = nng_stream_listener_listen(ep->listener); - nni_mtx_unlock(&ep->mtx); - return (rv); -} - -static void -ipc_ep_accept(void *arg, nni_aio *aio) -{ - ipc_ep *ep = arg; - int rv; - - if (nni_aio_begin(aio) != 0) { - return; - } - nni_mtx_lock(&ep->mtx); - if (ep->closed) { - nni_aio_finish_error(aio, NNG_ECLOSED); - nni_mtx_unlock(&ep->mtx); - return; - } - if (ep->user_aio != NULL) { - nni_aio_finish_error(aio, NNG_EBUSY); - nni_mtx_unlock(&ep->mtx); - return; - } - if ((rv = nni_aio_schedule(aio, ipc_ep_cancel, ep)) != 0) { - nni_mtx_unlock(&ep->mtx); - nni_aio_finish_error(aio, rv); - return; - } - ep->user_aio = aio; - if (!ep->started) { - ep->started = true; - nng_stream_listener_accept(ep->listener, ep->conn_aio); - } else { - ipc_ep_match(ep); - } - - nni_mtx_unlock(&ep->mtx); -} - -static int -ipc_pipe_get(void *arg, const char *name, void *buf, size_t *szp, nni_type t) -{ - ipc_pipe *p = arg; - - return (nni_stream_get(p->conn, name, buf, szp, t)); -} - -static nni_tran_pipe_ops ipc_tran_pipe_ops = { - .p_init = ipc_pipe_init, - .p_fini = ipc_pipe_fini, - .p_stop = ipc_pipe_stop, - .p_send = ipc_pipe_send, - .p_recv = ipc_pipe_recv, - .p_close = ipc_pipe_close, - .p_peer = ipc_pipe_peer, - .p_getopt = ipc_pipe_get, -}; - -static const nni_option ipc_ep_options[] = { - { - .o_name = NNG_OPT_RECVMAXSZ, - .o_get = ipc_ep_get_recv_max_sz, - .o_set = ipc_ep_set_recv_max_sz, - }, - // terminate list - { - .o_name = NULL, - }, -}; - -static int -ipc_dialer_get(void *arg, const char *name, void *buf, size_t *szp, nni_type t) -{ - ipc_ep *ep = arg; - int rv; - - rv = nni_getopt(ipc_ep_options, name, ep, buf, szp, t); - if (rv == NNG_ENOTSUP) { - rv = nni_stream_dialer_get(ep->dialer, name, buf, szp, t); - } - return (rv); -} - -static int -ipc_dialer_set( - void *arg, const char *name, const void *buf, size_t sz, nni_type t) -{ - ipc_ep *ep = arg; - int rv; - - rv = nni_setopt(ipc_ep_options, name, ep, buf, sz, t); - if (rv == NNG_ENOTSUP) { - rv = nni_stream_dialer_set(ep->dialer, name, buf, sz, t); - } - return (rv); -} - -static int -ipc_listener_get( - void *arg, const char *name, void *buf, size_t *szp, nni_type t) -{ - ipc_ep *ep = arg; - int rv; - - rv = nni_getopt(ipc_ep_options, name, ep, buf, szp, t); - if (rv == NNG_ENOTSUP) { - rv = nni_stream_listener_get(ep->listener, name, buf, szp, t); - } - return (rv); -} - -static int -ipc_listener_set( - void *arg, const char *name, const void *buf, size_t sz, nni_type t) -{ - ipc_ep *ep = arg; - int rv; - - rv = nni_setopt(ipc_ep_options, name, ep, buf, sz, t); - if (rv == NNG_ENOTSUP) { - rv = nni_stream_listener_set(ep->listener, name, buf, sz, t); - } - return (rv); -} - -static nni_tran_dialer_ops ipc_dialer_ops = { - .d_init = ipc_ep_init_dialer, - .d_fini = ipc_ep_fini, - .d_connect = ipc_ep_connect, - .d_close = ipc_ep_close, - .d_getopt = ipc_dialer_get, - .d_setopt = ipc_dialer_set, -}; - -static nni_tran_listener_ops ipc_listener_ops = { - .l_init = ipc_ep_init_listener, - .l_fini = ipc_ep_fini, - .l_bind = ipc_ep_bind, - .l_accept = ipc_ep_accept, - .l_close = ipc_ep_close, - .l_getopt = ipc_listener_get, - .l_setopt = ipc_listener_set, -}; - -static nni_tran ipc_tran = { - .tran_version = NNI_TRANSPORT_VERSION, - .tran_scheme = "ipc", - .tran_dialer = &ipc_dialer_ops, - .tran_listener = &ipc_listener_ops, - .tran_pipe = &ipc_tran_pipe_ops, - .tran_init = ipc_tran_init, - .tran_fini = ipc_tran_fini, -}; - -#ifdef NNG_PLATFORM_POSIX -static nni_tran ipc_tran_unix = { - .tran_version = NNI_TRANSPORT_VERSION, - .tran_scheme = "unix", - .tran_dialer = &ipc_dialer_ops, - .tran_listener = &ipc_listener_ops, - .tran_pipe = &ipc_tran_pipe_ops, - .tran_init = ipc_tran_init, - .tran_fini = ipc_tran_fini, -}; -#endif - -#ifdef NNG_HAVE_ABSTRACT_SOCKETS -static nni_tran ipc_tran_abstract = { - .tran_version = NNI_TRANSPORT_VERSION, - .tran_scheme = "abstract", - .tran_dialer = &ipc_dialer_ops, - .tran_listener = &ipc_listener_ops, - .tran_pipe = &ipc_tran_pipe_ops, - .tran_init = ipc_tran_init, - .tran_fini = ipc_tran_fini, -}; -#endif - -int -nng_ipc_register(void) -{ - int rv; - if (((rv = nni_tran_register(&ipc_tran)) != 0) -#ifdef NNG_PLATFORM_POSIX - || ((rv = nni_tran_register(&ipc_tran_unix)) != 0) -#endif -#ifdef NNG_HAVE_ABSTRACT_SOCKETS - || ((rv = nni_tran_register(&ipc_tran_abstract)) != 0) -#endif - ) { - return (rv); - } - - return (0); -} diff --git a/src/transport/ipc/ipc_test.c b/src/transport/ipc/ipc_test.c deleted file mode 100644 index 2fb4afa3..00000000 --- a/src/transport/ipc/ipc_test.c +++ /dev/null @@ -1,395 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// Copyright 2018 Cody Piersall -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - -#include - -#ifdef NNG_PLATFORM_POSIX -#include -#include -#include -#endif - -void -test_path_too_long(void) -{ - nng_socket s1; - char addr[256]; - - // All our names have to be less than 128 bytes. - memset(addr, 'a', 255); - addr[255] = 0; - memcpy(addr, "ipc://", strlen("ipc://")); - - NUTS_ASSERT(strlen(addr) == 255); - NUTS_OPEN(s1); - NUTS_PASS(nng_socket_set_ms(s1, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(s1, NNG_OPT_RECVTIMEO, 1000)); - NUTS_FAIL(nng_listen(s1, addr, NULL, 0), NNG_EADDRINVAL); - NUTS_FAIL(nng_dial(s1, addr, NULL, NNG_FLAG_NONBLOCK), NNG_EADDRINVAL); - - NUTS_CLOSE(s1); -} - -void -test_ipc_dialer_perms(void) -{ - nng_socket s; - nng_dialer d; - char * addr; - - NUTS_ADDR(addr, "ipc"); - NUTS_OPEN(s); - NUTS_PASS(nng_dialer_create(&d, s, addr)); - NUTS_FAIL( - nng_dialer_set_int(d, NNG_OPT_IPC_PERMISSIONS, 0444), NNG_ENOTSUP); - NUTS_CLOSE(s); -} - -void -test_ipc_dialer_properties(void) -{ - nng_socket s; - nng_dialer d; - nng_sockaddr sa; - size_t z; - char *addr; - - NUTS_ADDR(addr, "ipc"); - NUTS_OPEN(s); - NUTS_PASS(nng_dial(s, addr, &d, NNG_FLAG_NONBLOCK)); - // Dialers don't have local addresses. - NUTS_FAIL(nng_dialer_get_addr(d, NNG_OPT_LOCADDR, &sa), NNG_ENOTSUP); - - NUTS_FAIL( - nng_dialer_set(d, NNG_OPT_LOCADDR, &sa, sizeof(sa)), NNG_ENOTSUP); - - z = 8192; - NUTS_PASS(nng_dialer_set_size(d, NNG_OPT_RECVMAXSZ, z)); - z = 0; - NUTS_PASS(nng_dialer_get_size(d, NNG_OPT_RECVMAXSZ, &z)); - NUTS_TRUE(z == 8192); - NUTS_FAIL(nng_dialer_set_bool(d, NNG_OPT_RAW, true), NNG_ENOTSUP); - NUTS_CLOSE(s); -} - -void -test_ipc_listener_perms(void) -{ - nng_socket s; - nng_listener l; - char *addr; - -#ifndef _WIN32 - char * path; - struct stat st; -#endif - - NUTS_ADDR(addr, "ipc"); - NUTS_OPEN(s); - NUTS_PASS(nng_listener_create(&l, s, addr)); - -#ifdef _WIN32 - NUTS_FAIL(nng_listener_set_int(l, NNG_OPT_IPC_PERMISSIONS, 0444), - NNG_ENOTSUP); -#else - path = &addr[strlen("ipc://")]; - - // Attempt to set invalid permissions fails. - NUTS_FAIL(nng_listener_set_int(l, NNG_OPT_IPC_PERMISSIONS, S_IFREG), - NNG_EINVAL); - - NUTS_PASS(nng_listener_set_int(l, NNG_OPT_IPC_PERMISSIONS, 0444)); - NUTS_PASS(nng_listener_start(l, 0)); - NUTS_TRUE(stat(path, &st) == 0); - NUTS_TRUE((st.st_mode & 0777) == 0444); - - // Now that it's running, we cannot set it. - NUTS_FAIL( - nng_listener_set_int(l, NNG_OPT_IPC_PERMISSIONS, 0644), NNG_EBUSY); -#endif - - NUTS_CLOSE(s); -} - -void -test_ipc_listener_properties(void) -{ - nng_socket s; - nng_listener l; - nng_sockaddr sa; - size_t z; - char *addr; - - NUTS_ADDR(addr, "ipc"); - NUTS_OPEN(s); - NUTS_PASS(nng_listen(s, addr, &l, 0)); - NUTS_PASS(nng_listener_get_addr(l, NNG_OPT_LOCADDR, &sa)); - NUTS_TRUE(sa.s_ipc.sa_family == NNG_AF_IPC); - NUTS_MATCH(sa.s_ipc.sa_path, addr + strlen("ipc://")); - - NUTS_FAIL(nng_listener_set(l, NNG_OPT_LOCADDR, &sa, sizeof(sa)), - NNG_EREADONLY); - z = 8192; - NUTS_PASS(nng_listener_set_size(l, NNG_OPT_RECVMAXSZ, z)); - z = 0; - NUTS_PASS(nng_listener_get_size(l, NNG_OPT_RECVMAXSZ, &z)); - NUTS_TRUE(z == 8192); - NUTS_FAIL(nng_listener_set_bool(l, NNG_OPT_RAW, true), NNG_ENOTSUP); - NUTS_CLOSE(s); -} - -void -test_ipc_recv_max(void) -{ - char msg[256]; - char rcvbuf[256]; - nng_socket s0; - nng_socket s1; - nng_listener l; - size_t sz; - char *addr; - - NUTS_ADDR(addr, "ipc"); - NUTS_OPEN(s0); - NUTS_PASS(nng_socket_set_ms(s0, NNG_OPT_RECVTIMEO, 100)); - NUTS_PASS(nng_socket_set_size(s0, NNG_OPT_RECVMAXSZ, 200)); - NUTS_PASS(nng_listener_create(&l, s0, addr)); - NUTS_PASS(nng_socket_get_size(s0, NNG_OPT_RECVMAXSZ, &sz)); - NUTS_TRUE(sz == 200); - NUTS_PASS(nng_listener_set_size(l, NNG_OPT_RECVMAXSZ, 100)); - NUTS_PASS(nng_listener_start(l, 0)); - - NUTS_OPEN(s1); - NUTS_PASS(nng_dial(s1, addr, NULL, 0)); - NUTS_PASS(nng_send(s1, msg, 95, 0)); - NUTS_PASS(nng_socket_set_ms(s1, NNG_OPT_SENDTIMEO, 100)); - NUTS_PASS(nng_recv(s0, rcvbuf, &sz, 0)); - NUTS_TRUE(sz == 95); - NUTS_PASS(nng_send(s1, msg, 150, 0)); - NUTS_FAIL(nng_recv(s0, rcvbuf, &sz, 0), NNG_ETIMEDOUT); - NUTS_CLOSE(s0); - NUTS_CLOSE(s1); -} - -void -test_abstract_sockets(void) -{ -#ifdef NNG_HAVE_ABSTRACT_SOCKETS - nng_socket s1; - nng_socket s2; - char *addr; - nng_pipe p1; - nng_pipe p2; - nng_sockaddr sa1; - nng_sockaddr sa2; - char * prefix = "abstract://"; - - NUTS_ADDR(addr, "abstract"); - NUTS_OPEN(s1); - NUTS_OPEN(s2); - NUTS_MARRY_EX(s1, s2, addr, &p1, &p2); - NUTS_PASS(nng_pipe_get_addr(p1, NNG_OPT_REMADDR, &sa1)); - NUTS_PASS(nng_pipe_get_addr(p2, NNG_OPT_LOCADDR, &sa2)); - NUTS_TRUE(sa1.s_family == sa2.s_family); - NUTS_TRUE(sa1.s_family == NNG_AF_ABSTRACT); - NUTS_TRUE(sa1.s_abstract.sa_len == strlen(addr) - strlen(prefix)); - NUTS_TRUE(sa2.s_abstract.sa_len == strlen(addr) - strlen(prefix)); - NUTS_SEND(s1, "ping"); - NUTS_RECV(s2, "ping"); - NUTS_CLOSE(s1); - NUTS_CLOSE(s2); -#endif -} - -void -test_abstract_auto_bind(void) -{ -#ifdef NNG_HAVE_ABSTRACT_SOCKETS - nng_socket s1; - nng_socket s2; - char addr[40]; - char name[12]; - nng_sockaddr sa; - nng_listener l; - size_t len; - - snprintf(addr, sizeof(addr), "abstract://"); - - NUTS_OPEN(s1); - NUTS_OPEN(s2); - NUTS_PASS(nng_socket_set_ms(s1, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(s2, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(s1, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(s2, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_listen(s1, addr, &l, 0)); - - NUTS_PASS(nng_listener_get_addr(l, NNG_OPT_LOCADDR, &sa)); - // Under linux there are either 8 or 5 hex characters. - NUTS_TRUE(sa.s_family == NNG_AF_ABSTRACT); - NUTS_TRUE(sa.s_abstract.sa_len < 10); - - len = sa.s_abstract.sa_len; - memcpy(name, sa.s_abstract.sa_name, len); - name[len] = '\0'; - NUTS_TRUE(strlen(name) == len); - - (void) snprintf(addr, sizeof(addr), "abstract://%s", name); - NUTS_PASS(nng_dial(s2, addr, NULL, 0)); - - // first send the ping - NUTS_SEND(s1, "ping"); - NUTS_RECV(s2, "ping"); - - NUTS_SEND(s2, "pong"); - NUTS_RECV(s1, "pong"); - - NUTS_CLOSE(s1); - NUTS_CLOSE(s2); -#endif -} - -void -test_abstract_too_long(void) -{ -#ifdef NNG_HAVE_ABSTRACT_SOCKETS - nng_socket s1; - char addr[256]; - - // All our names have to be less than 128 bytes. - memset(addr, 'a', 255); - addr[255] = 0; - memcpy(addr, "abstract://", strlen("abstract://")); - - NUTS_ASSERT(strlen(addr) == 255); - NUTS_OPEN(s1); - NUTS_PASS(nng_socket_set_ms(s1, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(s1, NNG_OPT_RECVTIMEO, 1000)); - NUTS_FAIL(nng_listen(s1, addr, NULL, 0), NNG_EADDRINVAL); - NUTS_FAIL(nng_dial(s1, addr, NULL, NNG_FLAG_NONBLOCK), NNG_EADDRINVAL); - - NUTS_CLOSE(s1); -#endif -} - -void -test_abstract_null(void) -{ -#ifdef NNG_HAVE_ABSTRACT_SOCKETS - nng_socket s1; - nng_socket s2; - char addr[64]; - char name[40]; - char rng[20]; - - nng_sockaddr sa; - nng_listener l; - size_t len; - - snprintf(rng, sizeof(rng), "%08x%08x", nng_random(), nng_random()); - snprintf(name, sizeof(name), "a%%00b_%s", rng); - snprintf(addr, sizeof(addr), "abstract://%s", name); - - NUTS_OPEN(s1); - NUTS_OPEN(s2); - NUTS_PASS(nng_socket_set_ms(s1, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(s2, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(s1, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(s2, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_listen(s1, addr, &l, 0)); - - NUTS_PASS(nng_listener_get_addr(l, NNG_OPT_LOCADDR, &sa)); - // Under linux there are either 8 or 5 hex characters. - NUTS_TRUE(sa.s_family == NNG_AF_ABSTRACT); - NUTS_TRUE(sa.s_abstract.sa_len < 32); - len = sa.s_abstract.sa_len; - NUTS_TRUE(len == 20); - NUTS_TRUE(sa.s_abstract.sa_name[0] == 'a'); - NUTS_TRUE(sa.s_abstract.sa_name[1] == '\0'); - NUTS_TRUE(sa.s_abstract.sa_name[2] == 'b'); - NUTS_TRUE(sa.s_abstract.sa_name[3] == '_'); - NUTS_TRUE(memcmp(&sa.s_abstract.sa_name[4], rng, 16) == 0); - - NUTS_PASS(nng_dial(s2, addr, NULL, 0)); - - // first send the ping - NUTS_SEND(s1, "1234"); - NUTS_RECV(s2, "1234"); - - NUTS_SEND(s2, "5678"); - NUTS_RECV(s1, "5678"); - - NUTS_CLOSE(s1); - NUTS_CLOSE(s2); -#endif -} - -void -test_unix_alias(void) -{ -#ifdef NNG_PLATFORM_POSIX - nng_socket s1; - nng_socket s2; - char addr1[32]; - char addr2[32]; - char rng[20]; - nng_sockaddr sa1; - nng_sockaddr sa2; - nng_msg * msg; - nng_pipe p; - - // Presumes /tmp. - - (void) snprintf( - rng, sizeof(rng), "%08x%08x", nng_random(), nng_random()); - snprintf(addr1, sizeof(addr1), "ipc:///tmp/%s", rng); - snprintf(addr2, sizeof(addr2), "unix:///tmp/%s", rng); - - NUTS_OPEN(s1); - NUTS_OPEN(s2); - NUTS_PASS(nng_socket_set_ms(s1, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(s2, NNG_OPT_SENDTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(s1, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_socket_set_ms(s2, NNG_OPT_RECVTIMEO, 1000)); - NUTS_PASS(nng_listen(s1, addr1, NULL, 0)); - NUTS_PASS(nng_dial(s2, addr2, NULL, 0)); - - // first send the ping - NUTS_SEND(s1, "ping"); - NUTS_PASS(nng_recvmsg(s2, &msg, 0)); - NUTS_ASSERT(msg != NULL); - NUTS_TRUE(nng_msg_len(msg) == 5); - NUTS_MATCH(nng_msg_body(msg), "ping"); - p = nng_msg_get_pipe(msg); - NUTS_PASS(nng_pipe_get_addr(p, NNG_OPT_REMADDR, &sa1)); - NUTS_PASS(nng_pipe_get_addr(p, NNG_OPT_REMADDR, &sa2)); - NUTS_TRUE(sa1.s_family == sa2.s_family); - NUTS_TRUE(sa1.s_family == NNG_AF_IPC); - NUTS_MATCH(sa1.s_ipc.sa_path, sa2.s_ipc.sa_path); - nng_msg_free(msg); - - NUTS_CLOSE(s1); - NUTS_CLOSE(s2); -#endif -} - -TEST_LIST = { - { "ipc path too long", test_path_too_long }, - { "ipc dialer perms", test_ipc_dialer_perms }, - { "ipc dialer props", test_ipc_dialer_properties }, - { "ipc listener perms", test_ipc_listener_perms }, - { "ipc listener props", test_ipc_listener_properties }, - { "ipc recv max", test_ipc_recv_max }, - { "ipc abstract sockets", test_abstract_sockets }, - { "ipc abstract auto bind", test_abstract_auto_bind }, - { "ipc abstract name too long", test_abstract_too_long }, - { "ipc abstract embedded null", test_abstract_null }, - { "ipc unix alias", test_unix_alias }, - { NULL, NULL }, -}; \ No newline at end of file diff --git a/src/transport/tcp/CMakeLists.txt b/src/transport/tcp/CMakeLists.txt deleted file mode 100644 index d6022329..00000000 --- a/src/transport/tcp/CMakeLists.txt +++ /dev/null @@ -1,17 +0,0 @@ -# -# Copyright 2020 Staysail Systems, Inc. -# Copyright 2018 Capitar IT Group BV -# -# This software is supplied under the terms of the MIT License, a -# copy of which should be located in the distribution where this -# file was obtained (LICENSE.txt). A copy of the license may also be -# found online at https://opensource.org/licenses/MIT. -# - -# TCP protocol -nng_directory(tcp) - -nng_sources_if(NNG_TRANSPORT_TCP tcp.c) -nng_headers_if(NNG_TRANSPORT_TCP nng/transport/tcp/tcp.h) -nng_defines_if(NNG_TRANSPORT_TCP NNG_TRANSPORT_TCP) -nng_test(tcp_test) \ No newline at end of file diff --git a/src/transport/tcp/tcp.c b/src/transport/tcp/tcp.c deleted file mode 100644 index 524c6988..00000000 --- a/src/transport/tcp/tcp.c +++ /dev/null @@ -1,1263 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// Copyright 2018 Capitar IT Group BV -// Copyright 2019 Devolutions -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - -#include -#include -#include - -#include "core/nng_impl.h" - -// TCP transport. Platform specific TCP operations must be -// supplied as well. - -typedef struct tcptran_pipe tcptran_pipe; -typedef struct tcptran_ep tcptran_ep; - -// tcp_pipe is one end of a TCP connection. -struct tcptran_pipe { - nng_stream * conn; - nni_pipe * npipe; - uint16_t peer; - uint16_t proto; - size_t rcvmax; - bool closed; - nni_list_node node; - tcptran_ep * ep; - nni_atomic_flag reaped; - nni_reap_node reap; - uint8_t txlen[sizeof(uint64_t)]; - uint8_t rxlen[sizeof(uint64_t)]; - size_t gottxhead; - size_t gotrxhead; - size_t wanttxhead; - size_t wantrxhead; - nni_list recvq; - nni_list sendq; - nni_aio * txaio; - nni_aio * rxaio; - nni_aio * negoaio; - nni_msg * rxmsg; - nni_mtx mtx; -}; - -struct tcptran_ep { - nni_mtx mtx; - uint16_t proto; - size_t rcvmax; - bool fini; - bool started; - bool closed; - nng_url * url; - const char * host; // for dialers - nng_sockaddr src; - int refcnt; // active pipes - nni_aio * useraio; - nni_aio * connaio; - nni_aio * timeaio; - nni_list busypipes; // busy pipes -- ones passed to socket - nni_list waitpipes; // pipes waiting to match to socket - nni_list negopipes; // pipes busy negotiating - nni_reap_node reap; - nng_stream_dialer * dialer; - nng_stream_listener *listener; - -#ifdef NNG_ENABLE_STATS - nni_stat_item st_rcv_max; -#endif -}; - -static void tcptran_pipe_send_start(tcptran_pipe *); -static void tcptran_pipe_recv_start(tcptran_pipe *); -static void tcptran_pipe_send_cb(void *); -static void tcptran_pipe_recv_cb(void *); -static void tcptran_pipe_nego_cb(void *); -static void tcptran_ep_fini(void *); -static void tcptran_pipe_fini(void *); - -static nni_reap_list tcptran_ep_reap_list = { - .rl_offset = offsetof(tcptran_ep, reap), - .rl_func = tcptran_ep_fini, -}; - -static nni_reap_list tcptran_pipe_reap_list = { - .rl_offset = offsetof (tcptran_pipe, reap), - .rl_func = tcptran_pipe_fini, -}; - -static int -tcptran_init(void) -{ - return (0); -} - -static void -tcptran_fini(void) -{ -} - -static void -tcptran_pipe_close(void *arg) -{ - tcptran_pipe *p = arg; - - nni_mtx_lock(&p->mtx); - p->closed = true; - nni_mtx_unlock(&p->mtx); - - nni_aio_close(p->rxaio); - nni_aio_close(p->txaio); - nni_aio_close(p->negoaio); - - nng_stream_close(p->conn); -} - -static void -tcptran_pipe_stop(void *arg) -{ - tcptran_pipe *p = arg; - - nni_aio_stop(p->rxaio); - nni_aio_stop(p->txaio); - nni_aio_stop(p->negoaio); -} - -static int -tcptran_pipe_init(void *arg, nni_pipe *npipe) -{ - tcptran_pipe *p = arg; - p->npipe = npipe; - - return (0); -} - -static void -tcptran_pipe_fini(void *arg) -{ - tcptran_pipe *p = arg; - tcptran_ep * ep; - - tcptran_pipe_stop(p); - if ((ep = p->ep) != NULL) { - nni_mtx_lock(&ep->mtx); - nni_list_node_remove(&p->node); - ep->refcnt--; - if (ep->fini && (ep->refcnt == 0)) { - nni_reap(&tcptran_ep_reap_list, ep); - } - nni_mtx_unlock(&ep->mtx); - } - - nni_aio_free(p->rxaio); - nni_aio_free(p->txaio); - nni_aio_free(p->negoaio); - nng_stream_free(p->conn); - nni_msg_free(p->rxmsg); - nni_mtx_fini(&p->mtx); - NNI_FREE_STRUCT(p); -} - -static void -tcptran_pipe_reap(tcptran_pipe *p) -{ - if (!nni_atomic_flag_test_and_set(&p->reaped)) { - if (p->conn != NULL) { - nng_stream_close(p->conn); - } - nni_reap(&tcptran_pipe_reap_list, p); - } -} - -static int -tcptran_pipe_alloc(tcptran_pipe **pipep) -{ - tcptran_pipe *p; - int rv; - - if ((p = NNI_ALLOC_STRUCT(p)) == NULL) { - return (NNG_ENOMEM); - } - nni_mtx_init(&p->mtx); - if (((rv = nni_aio_alloc(&p->txaio, tcptran_pipe_send_cb, p)) != 0) || - ((rv = nni_aio_alloc(&p->rxaio, tcptran_pipe_recv_cb, p)) != 0) || - ((rv = nni_aio_alloc(&p->negoaio, tcptran_pipe_nego_cb, p)) != - 0)) { - tcptran_pipe_fini(p); - return (rv); - } - nni_aio_list_init(&p->recvq); - nni_aio_list_init(&p->sendq); - nni_atomic_flag_reset(&p->reaped); - - *pipep = p; - - return (0); -} - -static void -tcptran_ep_match(tcptran_ep *ep) -{ - nni_aio * aio; - tcptran_pipe *p; - - if (((aio = ep->useraio) == NULL) || - ((p = nni_list_first(&ep->waitpipes)) == NULL)) { - return; - } - nni_list_remove(&ep->waitpipes, p); - nni_list_append(&ep->busypipes, p); - ep->useraio = NULL; - p->rcvmax = ep->rcvmax; - nni_aio_set_output(aio, 0, p); - nni_aio_finish(aio, 0, 0); -} - -static void -tcptran_pipe_nego_cb(void *arg) -{ - tcptran_pipe *p = arg; - tcptran_ep * ep = p->ep; - nni_aio * aio = p->negoaio; - nni_aio * uaio; - int rv; - - nni_mtx_lock(&ep->mtx); - - if ((rv = nni_aio_result(aio)) != 0) { - goto error; - } - - // We start transmitting before we receive. - if (p->gottxhead < p->wanttxhead) { - p->gottxhead += nni_aio_count(aio); - } else if (p->gotrxhead < p->wantrxhead) { - p->gotrxhead += nni_aio_count(aio); - } - - if (p->gottxhead < p->wanttxhead) { - nni_iov iov; - iov.iov_len = p->wanttxhead - p->gottxhead; - iov.iov_buf = &p->txlen[p->gottxhead]; - // send it down... - nni_aio_set_iov(aio, 1, &iov); - nng_stream_send(p->conn, aio); - nni_mtx_unlock(&ep->mtx); - return; - } - if (p->gotrxhead < p->wantrxhead) { - nni_iov iov; - iov.iov_len = p->wantrxhead - p->gotrxhead; - iov.iov_buf = &p->rxlen[p->gotrxhead]; - nni_aio_set_iov(aio, 1, &iov); - nng_stream_recv(p->conn, aio); - nni_mtx_unlock(&ep->mtx); - return; - } - // We have both sent and received the headers. Lets check the - // receive side header. - if ((p->rxlen[0] != 0) || (p->rxlen[1] != 'S') || - (p->rxlen[2] != 'P') || (p->rxlen[3] != 0) || (p->rxlen[6] != 0) || - (p->rxlen[7] != 0)) { - rv = NNG_EPROTO; - goto error; - } - - NNI_GET16(&p->rxlen[4], p->peer); - - // We are all ready now. We put this in the wait list, and - // then try to run the matcher. - nni_list_remove(&ep->negopipes, p); - nni_list_append(&ep->waitpipes, p); - - tcptran_ep_match(ep); - nni_mtx_unlock(&ep->mtx); - - return; - -error: - nng_stream_close(p->conn); - - if ((uaio = ep->useraio) != NULL) { - ep->useraio = NULL; - nni_aio_finish_error(uaio, rv); - } - nni_mtx_unlock(&ep->mtx); - tcptran_pipe_reap(p); -} - -static void -tcptran_pipe_send_cb(void *arg) -{ - tcptran_pipe *p = arg; - int rv; - nni_aio * aio; - size_t n; - nni_msg * msg; - nni_aio * txaio = p->txaio; - - nni_mtx_lock(&p->mtx); - aio = nni_list_first(&p->sendq); - - if ((rv = nni_aio_result(txaio)) != 0) { - nni_pipe_bump_error(p->npipe, rv); - // Intentionally we do not queue up another transfer. - // There's an excellent chance that the pipe is no longer - // usable, with a partial transfer. - // The protocol should see this error, and close the - // pipe itself, we hope. - nni_aio_list_remove(aio); - nni_mtx_unlock(&p->mtx); - nni_aio_finish_error(aio, rv); - return; - } - - n = nni_aio_count(txaio); - nni_aio_iov_advance(txaio, n); - if (nni_aio_iov_count(txaio) > 0) { - nng_stream_send(p->conn, txaio); - nni_mtx_unlock(&p->mtx); - return; - } - - nni_aio_list_remove(aio); - tcptran_pipe_send_start(p); - - msg = nni_aio_get_msg(aio); - n = nni_msg_len(msg); - nni_pipe_bump_tx(p->npipe, n); - nni_mtx_unlock(&p->mtx); - - nni_aio_set_msg(aio, NULL); - nni_msg_free(msg); - nni_aio_finish_sync(aio, 0, n); -} - -static void -tcptran_pipe_recv_cb(void *arg) -{ - tcptran_pipe *p = arg; - nni_aio * aio; - int rv; - size_t n; - nni_msg * msg; - nni_aio * rxaio = p->rxaio; - - nni_mtx_lock(&p->mtx); - aio = nni_list_first(&p->recvq); - - if ((rv = nni_aio_result(rxaio)) != 0) { - goto recv_error; - } - - n = nni_aio_count(rxaio); - nni_aio_iov_advance(rxaio, n); - if (nni_aio_iov_count(rxaio) > 0) { - nng_stream_recv(p->conn, rxaio); - nni_mtx_unlock(&p->mtx); - return; - } - - // If we don't have a message yet, we were reading the TCP message - // header, which is just the length. This tells us the size of the - // message to allocate and how much more to expect. - if (p->rxmsg == NULL) { - uint64_t len; - // We should have gotten a message header. - NNI_GET64(p->rxlen, len); - - // Make sure the message payload is not too big. If it is - // the caller will shut down the pipe. - if ((len > p->rcvmax) && (p->rcvmax > 0)) { - rv = NNG_EMSGSIZE; - goto recv_error; - } - - if ((rv = nni_msg_alloc(&p->rxmsg, (size_t) len)) != 0) { - goto recv_error; - } - - // Submit the rest of the data for a read -- we want to - // read the entire message now. - if (len != 0) { - nni_iov iov; - iov.iov_buf = nni_msg_body(p->rxmsg); - iov.iov_len = (size_t) len; - - nni_aio_set_iov(rxaio, 1, &iov); - nng_stream_recv(p->conn, rxaio); - nni_mtx_unlock(&p->mtx); - return; - } - } - - // We read a message completely. Let the user know the good news. - nni_aio_list_remove(aio); - msg = p->rxmsg; - p->rxmsg = NULL; - n = nni_msg_len(msg); - - nni_pipe_bump_rx(p->npipe, n); - tcptran_pipe_recv_start(p); - nni_mtx_unlock(&p->mtx); - - nni_aio_set_msg(aio, msg); - nni_aio_finish_sync(aio, 0, n); - return; - -recv_error: - nni_aio_list_remove(aio); - msg = p->rxmsg; - p->rxmsg = NULL; - nni_pipe_bump_error(p->npipe, rv); - // Intentionally, we do not queue up another receive. - // The protocol should notice this error and close the pipe. - nni_mtx_unlock(&p->mtx); - - nni_msg_free(msg); - nni_aio_finish_error(aio, rv); -} - -static void -tcptran_pipe_send_cancel(nni_aio *aio, void *arg, int rv) -{ - tcptran_pipe *p = arg; - - nni_mtx_lock(&p->mtx); - if (!nni_aio_list_active(aio)) { - nni_mtx_unlock(&p->mtx); - return; - } - // If this is being sent, then cancel the pending transfer. - // The callback on the txaio will cause the user aio to - // be canceled too. - if (nni_list_first(&p->sendq) == aio) { - nni_aio_abort(p->txaio, rv); - nni_mtx_unlock(&p->mtx); - return; - } - nni_aio_list_remove(aio); - nni_mtx_unlock(&p->mtx); - - nni_aio_finish_error(aio, rv); -} - -static void -tcptran_pipe_send_start(tcptran_pipe *p) -{ - nni_aio *aio; - nni_aio *txaio; - nni_msg *msg; - int niov; - nni_iov iov[3]; - uint64_t len; - - if (p->closed) { - while ((aio = nni_list_first(&p->sendq)) != NULL) { - nni_list_remove(&p->sendq, aio); - nni_aio_finish_error(aio, NNG_ECLOSED); - } - return; - } - - if ((aio = nni_list_first(&p->sendq)) == NULL) { - return; - } - - // This runs to send the message. - msg = nni_aio_get_msg(aio); - len = nni_msg_len(msg) + nni_msg_header_len(msg); - - NNI_PUT64(p->txlen, len); - - txaio = p->txaio; - niov = 0; - iov[0].iov_buf = p->txlen; - iov[0].iov_len = sizeof(p->txlen); - niov++; - if (nni_msg_header_len(msg) > 0) { - iov[niov].iov_buf = nni_msg_header(msg); - iov[niov].iov_len = nni_msg_header_len(msg); - niov++; - } - if (nni_msg_len(msg) > 0) { - iov[niov].iov_buf = nni_msg_body(msg); - iov[niov].iov_len = nni_msg_len(msg); - niov++; - } - nni_aio_set_iov(txaio, niov, iov); - nng_stream_send(p->conn, txaio); -} - -static void -tcptran_pipe_send(void *arg, nni_aio *aio) -{ - tcptran_pipe *p = arg; - int rv; - - if (nni_aio_begin(aio) != 0) { - return; - } - nni_mtx_lock(&p->mtx); - if ((rv = nni_aio_schedule(aio, tcptran_pipe_send_cancel, p)) != 0) { - nni_mtx_unlock(&p->mtx); - nni_aio_finish_error(aio, rv); - return; - } - nni_list_append(&p->sendq, aio); - if (nni_list_first(&p->sendq) == aio) { - tcptran_pipe_send_start(p); - } - nni_mtx_unlock(&p->mtx); -} - -static void -tcptran_pipe_recv_cancel(nni_aio *aio, void *arg, int rv) -{ - tcptran_pipe *p = arg; - - nni_mtx_lock(&p->mtx); - if (!nni_aio_list_active(aio)) { - nni_mtx_unlock(&p->mtx); - return; - } - // If receive in progress, then cancel the pending transfer. - // The callback on the rxaio will cause the user aio to - // be canceled too. - if (nni_list_first(&p->recvq) == aio) { - nni_aio_abort(p->rxaio, rv); - nni_mtx_unlock(&p->mtx); - return; - } - nni_aio_list_remove(aio); - nni_mtx_unlock(&p->mtx); - nni_aio_finish_error(aio, rv); -} - -static void -tcptran_pipe_recv_start(tcptran_pipe *p) -{ - nni_aio *rxaio; - nni_iov iov; - NNI_ASSERT(p->rxmsg == NULL); - - if (p->closed) { - nni_aio *aio; - while ((aio = nni_list_first(&p->recvq)) != NULL) { - nni_list_remove(&p->recvq, aio); - nni_aio_finish_error(aio, NNG_ECLOSED); - } - return; - } - if (nni_list_empty(&p->recvq)) { - return; - } - - // Schedule a read of the header. - rxaio = p->rxaio; - iov.iov_buf = p->rxlen; - iov.iov_len = sizeof(p->rxlen); - nni_aio_set_iov(rxaio, 1, &iov); - - nng_stream_recv(p->conn, rxaio); -} - -static void -tcptran_pipe_recv(void *arg, nni_aio *aio) -{ - tcptran_pipe *p = arg; - int rv; - - if (nni_aio_begin(aio) != 0) { - return; - } - nni_mtx_lock(&p->mtx); - if ((rv = nni_aio_schedule(aio, tcptran_pipe_recv_cancel, p)) != 0) { - nni_mtx_unlock(&p->mtx); - nni_aio_finish_error(aio, rv); - return; - } - - nni_list_append(&p->recvq, aio); - if (nni_list_first(&p->recvq) == aio) { - tcptran_pipe_recv_start(p); - } - nni_mtx_unlock(&p->mtx); -} - -static uint16_t -tcptran_pipe_peer(void *arg) -{ - tcptran_pipe *p = arg; - - return (p->peer); -} - -static int -tcptran_pipe_getopt( - void *arg, const char *name, void *buf, size_t *szp, nni_type t) -{ - tcptran_pipe *p = arg; - return (nni_stream_get(p->conn, name, buf, szp, t)); -} - -static void -tcptran_pipe_start(tcptran_pipe *p, nng_stream *conn, tcptran_ep *ep) -{ - nni_iov iov; - - ep->refcnt++; - - p->conn = conn; - p->ep = ep; - p->proto = ep->proto; - - p->txlen[0] = 0; - p->txlen[1] = 'S'; - p->txlen[2] = 'P'; - p->txlen[3] = 0; - NNI_PUT16(&p->txlen[4], p->proto); - NNI_PUT16(&p->txlen[6], 0); - - p->gotrxhead = 0; - p->gottxhead = 0; - p->wantrxhead = 8; - p->wanttxhead = 8; - iov.iov_len = 8; - iov.iov_buf = &p->txlen[0]; - nni_aio_set_iov(p->negoaio, 1, &iov); - nni_list_append(&ep->negopipes, p); - - nni_aio_set_timeout(p->negoaio, 10000); // 10 sec timeout to negotiate - nng_stream_send(p->conn, p->negoaio); -} - -static void -tcptran_ep_fini(void *arg) -{ - tcptran_ep *ep = arg; - - nni_mtx_lock(&ep->mtx); - ep->fini = true; - if (ep->refcnt != 0) { - nni_mtx_unlock(&ep->mtx); - return; - } - nni_mtx_unlock(&ep->mtx); - nni_aio_stop(ep->timeaio); - nni_aio_stop(ep->connaio); - nng_stream_dialer_free(ep->dialer); - nng_stream_listener_free(ep->listener); - nni_aio_free(ep->timeaio); - nni_aio_free(ep->connaio); - - nni_mtx_fini(&ep->mtx); - NNI_FREE_STRUCT(ep); -} - -static void -tcptran_ep_close(void *arg) -{ - tcptran_ep * ep = arg; - tcptran_pipe *p; - - nni_mtx_lock(&ep->mtx); - - ep->closed = true; - nni_aio_close(ep->timeaio); - if (ep->dialer != NULL) { - nng_stream_dialer_close(ep->dialer); - } - if (ep->listener != NULL) { - nng_stream_listener_close(ep->listener); - } - NNI_LIST_FOREACH (&ep->negopipes, p) { - tcptran_pipe_close(p); - } - NNI_LIST_FOREACH (&ep->waitpipes, p) { - tcptran_pipe_close(p); - } - NNI_LIST_FOREACH (&ep->busypipes, p) { - tcptran_pipe_close(p); - } - if (ep->useraio != NULL) { - nni_aio_finish_error(ep->useraio, NNG_ECLOSED); - ep->useraio = NULL; - } - - nni_mtx_unlock(&ep->mtx); -} - -// This parses off the optional source address that this transport uses. -// The special handling of this URL format is quite honestly an historical -// mistake, which we would remove if we could. -static int -tcptran_url_parse_source(nng_url *url, nng_sockaddr *sa, const nng_url *surl) -{ - int af; - char * semi; - char * src; - size_t len; - int rv; - nni_aio *aio; - - // We modify the URL. This relies on the fact that the underlying - // transport does not free this, so we can just use references. - - url->u_scheme = surl->u_scheme; - url->u_port = surl->u_port; - url->u_hostname = surl->u_hostname; - - if ((semi = strchr(url->u_hostname, ';')) == NULL) { - memset(sa, 0, sizeof(*sa)); - return (0); - } - - len = (size_t)(semi - url->u_hostname); - url->u_hostname = semi + 1; - - if (strcmp(surl->u_scheme, "tcp") == 0) { - af = NNG_AF_UNSPEC; - } else if (strcmp(surl->u_scheme, "tcp4") == 0) { - af = NNG_AF_INET; - } else if (strcmp(surl->u_scheme, "tcp6") == 0) { - af = NNG_AF_INET6; - } else { - return (NNG_EADDRINVAL); - } - - if ((src = nni_alloc(len + 1)) == NULL) { - return (NNG_ENOMEM); - } - memcpy(src, surl->u_hostname, len); - src[len] = '\0'; - - if ((rv = nni_aio_alloc(&aio, NULL, NULL)) != 0) { - nni_free(src, len + 1); - return (rv); - } - - nni_resolv_ip(src, "0", af, true, sa, aio); - nni_aio_wait(aio); - nni_aio_free(aio); - nni_free(src, len + 1); - return (rv); -} - -static void -tcptran_timer_cb(void *arg) -{ - tcptran_ep *ep = arg; - if (nni_aio_result(ep->timeaio) == 0) { - nng_stream_listener_accept(ep->listener, ep->connaio); - } -} - -static void -tcptran_accept_cb(void *arg) -{ - tcptran_ep * ep = arg; - nni_aio * aio = ep->connaio; - tcptran_pipe *p; - int rv; - nng_stream * conn; - - nni_mtx_lock(&ep->mtx); - - if ((rv = nni_aio_result(aio)) != 0) { - goto error; - } - - conn = nni_aio_get_output(aio, 0); - if ((rv = tcptran_pipe_alloc(&p)) != 0) { - nng_stream_free(conn); - goto error; - } - - if (ep->closed) { - tcptran_pipe_fini(p); - nng_stream_free(conn); - rv = NNG_ECLOSED; - goto error; - } - tcptran_pipe_start(p, conn, ep); - nng_stream_listener_accept(ep->listener, ep->connaio); - nni_mtx_unlock(&ep->mtx); - return; - -error: - // When an error here occurs, let's send a notice up to the consumer. - // That way it can be reported properly. - if ((aio = ep->useraio) != NULL) { - ep->useraio = NULL; - nni_aio_finish_error(aio, rv); - } - switch (rv) { - - case NNG_ENOMEM: - case NNG_ENOFILES: - nng_sleep_aio(10, ep->timeaio); - break; - - default: - if (!ep->closed) { - nng_stream_listener_accept(ep->listener, ep->connaio); - } - break; - } - nni_mtx_unlock(&ep->mtx); -} - -static void -tcptran_dial_cb(void *arg) -{ - tcptran_ep * ep = arg; - nni_aio * aio = ep->connaio; - tcptran_pipe *p; - int rv; - nng_stream * conn; - - if ((rv = nni_aio_result(aio)) != 0) { - goto error; - } - - conn = nni_aio_get_output(aio, 0); - if ((rv = tcptran_pipe_alloc(&p)) != 0) { - nng_stream_free(conn); - goto error; - } - nni_mtx_lock(&ep->mtx); - if (ep->closed) { - tcptran_pipe_fini(p); - nng_stream_free(conn); - rv = NNG_ECLOSED; - nni_mtx_unlock(&ep->mtx); - goto error; - } else { - tcptran_pipe_start(p, conn, ep); - } - nni_mtx_unlock(&ep->mtx); - return; - -error: - // Error connecting. We need to pass this straight back - // to the user. - nni_mtx_lock(&ep->mtx); - if ((aio = ep->useraio) != NULL) { - ep->useraio = NULL; - nni_aio_finish_error(aio, rv); - } - nni_mtx_unlock(&ep->mtx); -} - -static int -tcptran_ep_init(tcptran_ep **epp, nng_url *url, nni_sock *sock) -{ - tcptran_ep *ep; - - if ((ep = NNI_ALLOC_STRUCT(ep)) == NULL) { - return (NNG_ENOMEM); - } - nni_mtx_init(&ep->mtx); - NNI_LIST_INIT(&ep->busypipes, tcptran_pipe, node); - NNI_LIST_INIT(&ep->waitpipes, tcptran_pipe, node); - NNI_LIST_INIT(&ep->negopipes, tcptran_pipe, node); - - ep->proto = nni_sock_proto_id(sock); - ep->url = url; - -#ifdef NNG_ENABLE_STATS - static const nni_stat_info rcv_max_info = { - .si_name = "rcv_max", - .si_desc = "maximum receive size", - .si_type = NNG_STAT_LEVEL, - .si_unit = NNG_UNIT_BYTES, - .si_atomic = true, - }; - nni_stat_init(&ep->st_rcv_max, &rcv_max_info); -#endif - - *epp = ep; - return (0); -} - -static int -tcptran_dialer_init(void **dp, nng_url *url, nni_dialer *ndialer) -{ - tcptran_ep * ep; - int rv; - nng_sockaddr srcsa; - nni_sock * sock = nni_dialer_sock(ndialer); - nng_url myurl; - - // Check for invalid URL components. - if ((strlen(url->u_path) != 0) && (strcmp(url->u_path, "/") != 0)) { - return (NNG_EADDRINVAL); - } - if ((url->u_fragment != NULL) || (url->u_userinfo != NULL) || - (url->u_query != NULL) || (strlen(url->u_hostname) == 0) || - (strlen(url->u_port) == 0)) { - return (NNG_EADDRINVAL); - } - - if ((rv = tcptran_url_parse_source(&myurl, &srcsa, url)) != 0) { - return (rv); - } - - if ((rv = tcptran_ep_init(&ep, url, sock)) != 0) { - return (rv); - } - - if ((rv != 0) || - ((rv = nni_aio_alloc(&ep->connaio, tcptran_dial_cb, ep)) != 0) || - ((rv = nng_stream_dialer_alloc_url(&ep->dialer, &myurl)) != 0)) { - tcptran_ep_fini(ep); - return (rv); - } - if ((srcsa.s_family != NNG_AF_UNSPEC) && - ((rv = nni_stream_dialer_set(ep->dialer, NNG_OPT_LOCADDR, &srcsa, - sizeof(srcsa), NNI_TYPE_SOCKADDR)) != 0)) { - tcptran_ep_fini(ep); - return (rv); - } - -#ifdef NNG_ENABLE_STATS - nni_dialer_add_stat(ndialer, &ep->st_rcv_max); -#endif - *dp = ep; - return (0); -} - -static int -tcptran_listener_init(void **lp, nng_url *url, nni_listener *nlistener) -{ - tcptran_ep *ep; - int rv; - nni_sock * sock = nni_listener_sock(nlistener); - - // Check for invalid URL components. - if ((strlen(url->u_path) != 0) && (strcmp(url->u_path, "/") != 0)) { - return (NNG_EADDRINVAL); - } - if ((url->u_fragment != NULL) || (url->u_userinfo != NULL) || - (url->u_query != NULL)) { - return (NNG_EADDRINVAL); - } - - if ((rv = tcptran_ep_init(&ep, url, sock)) != 0) { - return (rv); - } - - if (((rv = nni_aio_alloc(&ep->connaio, tcptran_accept_cb, ep)) != 0) || - ((rv = nni_aio_alloc(&ep->timeaio, tcptran_timer_cb, ep)) != 0) || - ((rv = nng_stream_listener_alloc_url(&ep->listener, url)) != 0)) { - tcptran_ep_fini(ep); - return (rv); - } -#ifdef NNG_ENABLE_STATS - nni_listener_add_stat(nlistener, &ep->st_rcv_max); -#endif - - *lp = ep; - return (0); -} - -static void -tcptran_ep_cancel(nni_aio *aio, void *arg, int rv) -{ - tcptran_ep *ep = arg; - nni_mtx_lock(&ep->mtx); - if (ep->useraio == aio) { - ep->useraio = NULL; - nni_aio_finish_error(aio, rv); - } - nni_mtx_unlock(&ep->mtx); -} - -static void -tcptran_ep_connect(void *arg, nni_aio *aio) -{ - tcptran_ep *ep = arg; - int rv; - - if (nni_aio_begin(aio) != 0) { - return; - } - nni_mtx_lock(&ep->mtx); - if (ep->closed) { - nni_mtx_unlock(&ep->mtx); - nni_aio_finish_error(aio, NNG_ECLOSED); - return; - } - if (ep->useraio != NULL) { - nni_mtx_unlock(&ep->mtx); - nni_aio_finish_error(aio, NNG_EBUSY); - return; - } - if ((rv = nni_aio_schedule(aio, tcptran_ep_cancel, ep)) != 0) { - nni_mtx_unlock(&ep->mtx); - nni_aio_finish_error(aio, rv); - return; - } - ep->useraio = aio; - - nng_stream_dialer_dial(ep->dialer, ep->connaio); - nni_mtx_unlock(&ep->mtx); -} - -static int -tcptran_ep_get_url(void *arg, void *v, size_t *szp, nni_opt_type t) -{ - tcptran_ep *ep = arg; - char * s; - int rv; - int port = 0; - - if (ep->listener != NULL) { - (void) nng_stream_listener_get_int( - ep->listener, NNG_OPT_TCP_BOUND_PORT, &port); - } - - if ((rv = nni_url_asprintf_port(&s, ep->url, port)) == 0) { - rv = nni_copyout_str(s, v, szp, t); - nni_strfree(s); - } - return (rv); -} - -static int -tcptran_ep_get_recvmaxsz(void *arg, void *v, size_t *szp, nni_opt_type t) -{ - tcptran_ep *ep = arg; - int rv; - - nni_mtx_lock(&ep->mtx); - rv = nni_copyout_size(ep->rcvmax, v, szp, t); - nni_mtx_unlock(&ep->mtx); - return (rv); -} - -static int -tcptran_ep_set_recvmaxsz(void *arg, const void *v, size_t sz, nni_opt_type t) -{ - tcptran_ep *ep = arg; - size_t val; - int rv; - if ((rv = nni_copyin_size(&val, v, sz, 0, NNI_MAXSZ, t)) == 0) { - tcptran_pipe *p; - nni_mtx_lock(&ep->mtx); - ep->rcvmax = val; - NNI_LIST_FOREACH (&ep->waitpipes, p) { - p->rcvmax = val; - } - NNI_LIST_FOREACH (&ep->negopipes, p) { - p->rcvmax = val; - } - NNI_LIST_FOREACH (&ep->busypipes, p) { - p->rcvmax = val; - } - nni_mtx_unlock(&ep->mtx); -#ifdef NNG_ENABLE_STATS - nni_stat_set_value(&ep->st_rcv_max, val); -#endif - } - return (rv); -} - -static int -tcptran_ep_bind(void *arg) -{ - tcptran_ep *ep = arg; - int rv; - - nni_mtx_lock(&ep->mtx); - rv = nng_stream_listener_listen(ep->listener); - nni_mtx_unlock(&ep->mtx); - - return (rv); -} - -static void -tcptran_ep_accept(void *arg, nni_aio *aio) -{ - tcptran_ep *ep = arg; - int rv; - - if (nni_aio_begin(aio) != 0) { - return; - } - nni_mtx_lock(&ep->mtx); - if (ep->closed) { - nni_mtx_unlock(&ep->mtx); - nni_aio_finish_error(aio, NNG_ECLOSED); - return; - } - if (ep->useraio != NULL) { - nni_mtx_unlock(&ep->mtx); - nni_aio_finish_error(aio, NNG_EBUSY); - return; - } - if ((rv = nni_aio_schedule(aio, tcptran_ep_cancel, ep)) != 0) { - nni_mtx_unlock(&ep->mtx); - nni_aio_finish_error(aio, rv); - return; - } - ep->useraio = aio; - if (!ep->started) { - ep->started = true; - nng_stream_listener_accept(ep->listener, ep->connaio); - } else { - tcptran_ep_match(ep); - } - nni_mtx_unlock(&ep->mtx); -} - -static nni_tran_pipe_ops tcptran_pipe_ops = { - .p_init = tcptran_pipe_init, - .p_fini = tcptran_pipe_fini, - .p_stop = tcptran_pipe_stop, - .p_send = tcptran_pipe_send, - .p_recv = tcptran_pipe_recv, - .p_close = tcptran_pipe_close, - .p_peer = tcptran_pipe_peer, - .p_getopt = tcptran_pipe_getopt, -}; - -static const nni_option tcptran_ep_opts[] = { - { - .o_name = NNG_OPT_RECVMAXSZ, - .o_get = tcptran_ep_get_recvmaxsz, - .o_set = tcptran_ep_set_recvmaxsz, - }, - { - .o_name = NNG_OPT_URL, - .o_get = tcptran_ep_get_url, - }, - // terminate list - { - .o_name = NULL, - }, -}; - -static int -tcptran_dialer_getopt( - void *arg, const char *name, void *buf, size_t *szp, nni_type t) -{ - tcptran_ep *ep = arg; - int rv; - - rv = nni_stream_dialer_get(ep->dialer, name, buf, szp, t); - if (rv == NNG_ENOTSUP) { - rv = nni_getopt(tcptran_ep_opts, name, ep, buf, szp, t); - } - return (rv); -} - -static int -tcptran_dialer_setopt( - void *arg, const char *name, const void *buf, size_t sz, nni_type t) -{ - tcptran_ep *ep = arg; - int rv; - - rv = nni_stream_dialer_set(ep->dialer, name, buf, sz, t); - if (rv == NNG_ENOTSUP) { - rv = nni_setopt(tcptran_ep_opts, name, ep, buf, sz, t); - } - return (rv); -} - -static int -tcptran_listener_getopt( - void *arg, const char *name, void *buf, size_t *szp, nni_type t) -{ - tcptran_ep *ep = arg; - int rv; - - rv = nni_stream_listener_get(ep->listener, name, buf, szp, t); - if (rv == NNG_ENOTSUP) { - rv = nni_getopt(tcptran_ep_opts, name, ep, buf, szp, t); - } - return (rv); -} - -static int -tcptran_listener_setopt( - void *arg, const char *name, const void *buf, size_t sz, nni_type t) -{ - tcptran_ep *ep = arg; - int rv; - - rv = nni_stream_listener_set(ep->listener, name, buf, sz, t); - if (rv == NNG_ENOTSUP) { - rv = nni_setopt(tcptran_ep_opts, name, ep, buf, sz, t); - } - return (rv); -} - -static nni_tran_dialer_ops tcptran_dialer_ops = { - .d_init = tcptran_dialer_init, - .d_fini = tcptran_ep_fini, - .d_connect = tcptran_ep_connect, - .d_close = tcptran_ep_close, - .d_getopt = tcptran_dialer_getopt, - .d_setopt = tcptran_dialer_setopt, -}; - -static nni_tran_listener_ops tcptran_listener_ops = { - .l_init = tcptran_listener_init, - .l_fini = tcptran_ep_fini, - .l_bind = tcptran_ep_bind, - .l_accept = tcptran_ep_accept, - .l_close = tcptran_ep_close, - .l_getopt = tcptran_listener_getopt, - .l_setopt = tcptran_listener_setopt, -}; - -static nni_tran tcp_tran = { - .tran_version = NNI_TRANSPORT_VERSION, - .tran_scheme = "tcp", - .tran_dialer = &tcptran_dialer_ops, - .tran_listener = &tcptran_listener_ops, - .tran_pipe = &tcptran_pipe_ops, - .tran_init = tcptran_init, - .tran_fini = tcptran_fini, -}; - -static nni_tran tcp4_tran = { - .tran_version = NNI_TRANSPORT_VERSION, - .tran_scheme = "tcp4", - .tran_dialer = &tcptran_dialer_ops, - .tran_listener = &tcptran_listener_ops, - .tran_pipe = &tcptran_pipe_ops, - .tran_init = tcptran_init, - .tran_fini = tcptran_fini, -}; - -static nni_tran tcp6_tran = { - .tran_version = NNI_TRANSPORT_VERSION, - .tran_scheme = "tcp6", - .tran_dialer = &tcptran_dialer_ops, - .tran_listener = &tcptran_listener_ops, - .tran_pipe = &tcptran_pipe_ops, - .tran_init = tcptran_init, - .tran_fini = tcptran_fini, -}; - -int -nng_tcp_register(void) -{ - int rv; - if (((rv = nni_tran_register(&tcp_tran)) != 0) || - ((rv = nni_tran_register(&tcp4_tran)) != 0) || - ((rv = nni_tran_register(&tcp6_tran)) != 0)) { - return (rv); - } - return (0); -} diff --git a/src/transport/tcp/tcp_test.c b/src/transport/tcp/tcp_test.c deleted file mode 100644 index d23227d7..00000000 --- a/src/transport/tcp/tcp_test.c +++ /dev/null @@ -1,297 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// Copyright 2018 Capitar IT Group BV -// Copyright 2018 Devolutions -// Copyright 2018 Cody Piersall -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - - -#include - -// TCP tests. - -static void -test_tcp_wild_card_connect_fail(void) -{ - nng_socket s; - char addr[NNG_MAXADDRLEN]; - - NUTS_OPEN(s); - (void) snprintf(addr, sizeof(addr), "tcp://*:%u", nuts_next_port()); - NUTS_FAIL(nng_dial(s, addr, NULL, 0), NNG_EADDRINVAL); - NUTS_CLOSE(s); -} - -void -test_tcp_wild_card_bind(void) -{ - nng_socket s1; - nng_socket s2; - char addr[NNG_MAXADDRLEN]; - uint16_t port; - - port = nuts_next_port(); - - NUTS_OPEN(s1); - NUTS_OPEN(s2); - (void) snprintf(addr, sizeof(addr), "tcp4://*:%u", port); - NUTS_PASS(nng_listen(s1, addr, NULL, 0)); - (void) snprintf(addr, sizeof(addr), "tcp://127.0.0.1:%u", port); - NUTS_PASS(nng_dial(s2, addr, NULL, 0)); - NUTS_CLOSE(s2); - NUTS_CLOSE(s1); -} - -void -test_tcp_local_address_connect(void) -{ - - nng_socket s1; - nng_socket s2; - char addr[NNG_MAXADDRLEN]; - uint16_t port; - - NUTS_OPEN(s1); - NUTS_OPEN(s2); - port = nuts_next_port(); - (void) snprintf(addr, sizeof(addr), "tcp://127.0.0.1:%u", port); - NUTS_PASS(nng_listen(s1, addr, NULL, 0)); - (void) snprintf( - addr, sizeof(addr), "tcp://127.0.0.1;127.0.0.1:%u", port); - NUTS_PASS(nng_dial(s2, addr, NULL, 0)); - NUTS_CLOSE(s2); - NUTS_CLOSE(s1); -} - -void -test_tcp_port_zero_bind(void) -{ - nng_socket s1; - nng_socket s2; - nng_sockaddr sa; - nng_listener l; - char * addr; - - NUTS_OPEN(s1); - NUTS_OPEN(s2); - NUTS_PASS(nng_listen(s1, "tcp://127.0.0.1:0", &l, 0)); - NUTS_PASS(nng_listener_get_string(l, NNG_OPT_URL, &addr)); - NUTS_TRUE(memcmp(addr, "tcp://", 6) == 0); - NUTS_PASS(nng_listener_get_addr(l, NNG_OPT_LOCADDR, &sa)); - NUTS_TRUE(sa.s_in.sa_family == NNG_AF_INET); - NUTS_TRUE(sa.s_in.sa_port != 0); - NUTS_TRUE(sa.s_in.sa_addr = nuts_be32(0x7f000001)); - NUTS_PASS(nng_dial(s2, addr, NULL, 0)); - nng_strfree(addr); - NUTS_CLOSE(s2); - NUTS_CLOSE(s1); -} - -void -test_tcp_bad_local_interface(void) -{ - nng_socket s1; - - NUTS_OPEN(s1); - NUTS_FAIL(nng_dial(s1, "tcp://bogus1;127.0.0.1:80", NULL, 0), - NNG_EADDRINVAL); - NUTS_CLOSE(s1); -} - -void -test_tcp_non_local_address(void) -{ - nng_socket s1; - - NUTS_OPEN(s1); - NUTS_FAIL(nng_dial(s1, "tcp://8.8.8.8;127.0.0.1:80", NULL, 0), - NNG_EADDRINVAL); - NUTS_CLOSE(s1); -} - -void -test_tcp_malformed_address(void) -{ - nng_socket s1; - - NUTS_OPEN(s1); - NUTS_FAIL( - nng_dial(s1, "tcp://127.0.0.1", NULL, 0), NNG_EADDRINVAL); - NUTS_FAIL( - nng_dial(s1, "tcp://127.0.0.1.32", NULL, 0), NNG_EADDRINVAL); - NUTS_FAIL( - nng_dial(s1, "tcp://127.0.x.1.32", NULL, 0), NNG_EADDRINVAL); - NUTS_FAIL( - nng_listen(s1, "tcp://127.0.0.1.32", NULL, 0), NNG_EADDRINVAL); - NUTS_FAIL( - nng_listen(s1, "tcp://127.0.x.1.32", NULL, 0), NNG_EADDRINVAL); - NUTS_CLOSE(s1); -} - -void -test_tcp_no_delay_option(void) -{ - nng_socket s; - nng_dialer d; - nng_listener l; - bool v; - int x; - char *addr; - - NUTS_ADDR(addr, "tcp"); - - NUTS_OPEN(s); -#ifndef NNG_ELIDE_DEPRECATED - NUTS_PASS(nng_socket_get_bool(s, NNG_OPT_TCP_NODELAY, &v)); - NUTS_TRUE(v); -#endif - NUTS_PASS(nng_dialer_create(&d, s, addr)); - NUTS_PASS(nng_dialer_get_bool(d, NNG_OPT_TCP_NODELAY, &v)); - NUTS_TRUE(v); - NUTS_PASS(nng_dialer_set_bool(d, NNG_OPT_TCP_NODELAY, false)); - NUTS_PASS(nng_dialer_get_bool(d, NNG_OPT_TCP_NODELAY, &v)); - NUTS_TRUE(v == false); - NUTS_FAIL( - nng_dialer_get_int(d, NNG_OPT_TCP_NODELAY, &x), NNG_EBADTYPE); - x = 0; - NUTS_FAIL( - nng_dialer_set_int(d, NNG_OPT_TCP_NODELAY, x), NNG_EBADTYPE); - // This assumes sizeof (bool) != sizeof (int) - if (sizeof(bool) != sizeof(int)) { - NUTS_FAIL( - nng_dialer_set(d, NNG_OPT_TCP_NODELAY, &x, sizeof(x)), - NNG_EINVAL); - } - - NUTS_PASS(nng_listener_create(&l, s, addr)); - NUTS_PASS(nng_listener_get_bool(l, NNG_OPT_TCP_NODELAY, &v)); - NUTS_TRUE(v == true); - x = 0; - NUTS_FAIL( - nng_listener_set_int(l, NNG_OPT_TCP_NODELAY, x), NNG_EBADTYPE); - // This assumes sizeof (bool) != sizeof (int) - NUTS_FAIL(nng_listener_set(l, NNG_OPT_TCP_NODELAY, &x, sizeof(x)), - NNG_EINVAL); - - NUTS_PASS(nng_dialer_close(d)); - NUTS_PASS(nng_listener_close(l)); - - // Make sure socket wide defaults apply. -#ifndef NNG_ELIDE_DEPRECATED - NUTS_PASS(nng_socket_set_bool(s, NNG_OPT_TCP_NODELAY, true)); - v = false; - NUTS_PASS(nng_socket_get_bool(s, NNG_OPT_TCP_NODELAY, &v)); - NUTS_TRUE(v); - NUTS_PASS(nng_socket_set_bool(s, NNG_OPT_TCP_NODELAY, false)); - NUTS_PASS(nng_dialer_create(&d, s, addr)); - NUTS_PASS(nng_dialer_get_bool(d, NNG_OPT_TCP_NODELAY, &v)); - NUTS_TRUE(v == false); -#endif - NUTS_CLOSE(s); -} - -void -test_tcp_keep_alive_option(void) -{ - nng_socket s; - nng_dialer d; - nng_listener l; - bool v; - int x; - char *addr; - - NUTS_ADDR(addr, "tcp"); - NUTS_OPEN(s); -#ifndef NNG_ELIDE_DEPRECATED - NUTS_PASS(nng_socket_get_bool(s, NNG_OPT_TCP_KEEPALIVE, &v)); - NUTS_TRUE(v == false); -#endif - NUTS_PASS(nng_dialer_create(&d, s, addr)); - NUTS_PASS(nng_dialer_get_bool(d, NNG_OPT_TCP_KEEPALIVE, &v)); - NUTS_TRUE(v == false); - NUTS_PASS(nng_dialer_set_bool(d, NNG_OPT_TCP_KEEPALIVE, true)); - NUTS_PASS(nng_dialer_get_bool(d, NNG_OPT_TCP_KEEPALIVE, &v)); - NUTS_TRUE(v); - NUTS_FAIL( - nng_dialer_get_int(d, NNG_OPT_TCP_KEEPALIVE, &x), NNG_EBADTYPE); - x = 1; - NUTS_FAIL( - nng_dialer_set_int(d, NNG_OPT_TCP_KEEPALIVE, x), NNG_EBADTYPE); - - NUTS_PASS(nng_listener_create(&l, s, addr)); - NUTS_PASS(nng_listener_get_bool(l, NNG_OPT_TCP_KEEPALIVE, &v)); - NUTS_TRUE(v == false); - x = 1; - NUTS_FAIL( - nng_listener_set_int(l, NNG_OPT_TCP_KEEPALIVE, x), NNG_EBADTYPE); - - NUTS_PASS(nng_dialer_close(d)); - NUTS_PASS(nng_listener_close(l)); - - // Make sure socket wide defaults apply. -#ifndef NNG_ELIDE_DEPRECATED - NUTS_PASS(nng_socket_set_bool(s, NNG_OPT_TCP_KEEPALIVE, false)); - v = true; - NUTS_PASS(nng_socket_get_bool(s, NNG_OPT_TCP_KEEPALIVE, &v)); - NUTS_TRUE(v == false); - NUTS_PASS(nng_socket_set_bool(s, NNG_OPT_TCP_KEEPALIVE, true)); - NUTS_PASS(nng_dialer_create(&d, s, addr)); - NUTS_PASS(nng_dialer_get_bool(d, NNG_OPT_TCP_KEEPALIVE, &v)); - NUTS_TRUE(v); -#endif - NUTS_CLOSE(s); -} - -void -test_tcp_recv_max(void) -{ - char msg[256]; - char buf[256]; - nng_socket s0; - nng_socket s1; - nng_listener l; - size_t sz; - char *addr; - - NUTS_ADDR(addr, "tcp"); - - NUTS_OPEN(s0); - NUTS_PASS(nng_socket_set_ms(s0, NNG_OPT_RECVTIMEO, 100)); - NUTS_PASS(nng_socket_set_size(s0, NNG_OPT_RECVMAXSZ, 200)); - NUTS_PASS(nng_listener_create(&l, s0, addr)); - NUTS_PASS(nng_socket_get_size(s0, NNG_OPT_RECVMAXSZ, &sz)); - NUTS_TRUE(sz == 200); - NUTS_PASS(nng_listener_set_size(l, NNG_OPT_RECVMAXSZ, 100)); - NUTS_PASS(nng_listener_start(l, 0)); - - NUTS_OPEN(s1); - NUTS_PASS(nng_dial(s1, addr, NULL, 0)); - NUTS_PASS(nng_send(s1, msg, 95, 0)); - NUTS_PASS(nng_socket_set_ms(s1, NNG_OPT_SENDTIMEO, 100)); - NUTS_PASS(nng_recv(s0, buf, &sz, 0)); - NUTS_TRUE(sz == 95); - NUTS_PASS(nng_send(s1, msg, 150, 0)); - NUTS_FAIL(nng_recv(s0, buf, &sz, 0), NNG_ETIMEDOUT); - NUTS_PASS(nng_close(s0)); - NUTS_CLOSE(s1); -} - -NUTS_TESTS = { - - { "tcp wild card connect fail", test_tcp_wild_card_connect_fail }, - { "tcp wild card bind", test_tcp_wild_card_bind }, - { "tcp port zero bind", test_tcp_port_zero_bind }, - { "tcp local address connect", test_tcp_local_address_connect }, - { "tcp bad local interface", test_tcp_bad_local_interface }, - { "tcp non-local address", test_tcp_non_local_address }, - { "tcp malformed address", test_tcp_malformed_address }, - { "tcp no delay option", test_tcp_no_delay_option }, - { "tcp keep alive option", test_tcp_keep_alive_option }, - { "tcp recv max", test_tcp_recv_max }, - { NULL, NULL }, -}; \ No newline at end of file diff --git a/src/transport/tls/CMakeLists.txt b/src/transport/tls/CMakeLists.txt deleted file mode 100644 index 82f24c79..00000000 --- a/src/transport/tls/CMakeLists.txt +++ /dev/null @@ -1,16 +0,0 @@ -# -# Copyright 2020 Staysail Systems, Inc. -# Copyright 2018 Capitar IT Group BV -# -# This software is supplied under the terms of the MIT License, a -# copy of which should be located in the distribution where this -# file was obtained (LICENSE.txt). A copy of the license may also be -# found online at https://opensource.org/licenses/MIT. -# - -# TLS transport -nng_directory(tls) - -nng_sources_if(NNG_TRANSPORT_TLS tls.c) -nng_headers_if(NNG_TRANSPORT_TLS nng/transport/tls/tls.h) -nng_defines_if(NNG_TRANSPORT_TLS NNG_TRANSPORT_TLS) \ No newline at end of file diff --git a/src/transport/tls/tls.c b/src/transport/tls/tls.c deleted file mode 100644 index b6623733..00000000 --- a/src/transport/tls/tls.c +++ /dev/null @@ -1,1292 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// Copyright 2018 Capitar IT Group BV -// Copyright 2019 Devolutions -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - -#include -#include - -#include "core/nng_impl.h" - -#include "nng/supplemental/tls/tls.h" -#include "nng/transport/tls/tls.h" - -// TLS over TCP transport. Platform specific TCP operations must be -// supplied as well, and uses the supplemental TLS v1.2 code. It is not -// an accident that this very closely resembles the TCP transport itself. - -typedef struct tlstran_ep tlstran_ep; -typedef struct tlstran_dialer tlstran_dialer; -typedef struct tlstran_listener tlstran_listener; -typedef struct tlstran_pipe tlstran_pipe; - -// tlstran_pipe is one end of a TLS connection. -struct tlstran_pipe { - nng_stream * tls; - nni_pipe * npipe; - uint16_t peer; - uint16_t proto; - size_t rcvmax; - bool closed; - nni_list_node node; - nni_list sendq; - nni_list recvq; - tlstran_ep * ep; - nni_sockaddr sa; - nni_atomic_flag reaped; - nni_reap_node reap; - uint8_t txlen[sizeof(uint64_t)]; - uint8_t rxlen[sizeof(uint64_t)]; - size_t gottxhead; - size_t gotrxhead; - size_t wanttxhead; - size_t wantrxhead; - nni_aio * txaio; - nni_aio * rxaio; - nni_aio * negoaio; - nni_msg * rxmsg; - nni_mtx mtx; -}; - -// Stuff that is common to both dialers and listeners. -struct tlstran_ep { - nni_mtx mtx; - uint16_t proto; - size_t rcvmax; - bool started; - bool closed; - bool fini; - int refcnt; - int authmode; - nni_url * url; - nni_list pipes; - nni_reap_node reap; - nng_stream_dialer * dialer; - nng_stream_listener *listener; - nni_aio * useraio; - nni_aio * connaio; - nni_aio * timeaio; - nni_list busypipes; // busy pipes -- ones passed to socket - nni_list waitpipes; // pipes waiting to match to socket - nni_list negopipes; // pipes busy negotiating - const char * host; - nng_sockaddr src; - nng_sockaddr sa; - nni_stat_item st_rcv_max; -}; - -static void tlstran_pipe_send_start(tlstran_pipe *); -static void tlstran_pipe_recv_start(tlstran_pipe *); -static void tlstran_pipe_send_cb(void *); -static void tlstran_pipe_recv_cb(void *); -static void tlstran_pipe_nego_cb(void *); -static void tlstran_ep_fini(void *); -static void tlstran_pipe_fini(void *); - -static nni_reap_list tlstran_ep_reap_list = { - .rl_offset = offsetof(tlstran_ep, reap), - .rl_func = tlstran_ep_fini, -}; - -static nni_reap_list tlstran_pipe_reap_list = { - .rl_offset = offsetof(tlstran_pipe, reap), - .rl_func = tlstran_pipe_fini, -}; - -static int -tlstran_init(void) -{ - return (0); -} - -static void -tlstran_fini(void) -{ -} - -static void -tlstran_pipe_close(void *arg) -{ - tlstran_pipe *p = arg; - - nni_aio_close(p->rxaio); - nni_aio_close(p->txaio); - nni_aio_close(p->negoaio); - - nng_stream_close(p->tls); -} - -static void -tlstran_pipe_stop(void *arg) -{ - tlstran_pipe *p = arg; - - nni_aio_stop(p->rxaio); - nni_aio_stop(p->txaio); - nni_aio_stop(p->negoaio); -} - -static int -tlstran_pipe_init(void *arg, nni_pipe *npipe) -{ - tlstran_pipe *p = arg; - p->npipe = npipe; - return (0); -} - -static void -tlstran_pipe_fini(void *arg) -{ - tlstran_pipe *p = arg; - tlstran_ep * ep; - - tlstran_pipe_stop(p); - if ((ep = p->ep) != NULL) { - nni_mtx_lock(&ep->mtx); - nni_list_node_remove(&p->node); - ep->refcnt--; - if (ep->fini && (ep->refcnt == 0)) { - nni_reap(&tlstran_ep_reap_list, ep); - } - nni_mtx_unlock(&ep->mtx); - } - nni_aio_free(p->rxaio); - nni_aio_free(p->txaio); - nni_aio_free(p->negoaio); - nng_stream_free(p->tls); - nni_msg_free(p->rxmsg); - NNI_FREE_STRUCT(p); -} - -static int -tlstran_pipe_alloc(tlstran_pipe **pipep) -{ - tlstran_pipe *p; - int rv; - - if ((p = NNI_ALLOC_STRUCT(p)) == NULL) { - return (NNG_ENOMEM); - } - nni_mtx_init(&p->mtx); - - if (((rv = nni_aio_alloc(&p->txaio, tlstran_pipe_send_cb, p)) != 0) || - ((rv = nni_aio_alloc(&p->rxaio, tlstran_pipe_recv_cb, p)) != 0) || - ((rv = nni_aio_alloc(&p->negoaio, tlstran_pipe_nego_cb, p)) != - 0)) { - tlstran_pipe_fini(p); - return (rv); - } - nni_aio_list_init(&p->recvq); - nni_aio_list_init(&p->sendq); - nni_atomic_flag_reset(&p->reaped); - - *pipep = p; - return (0); -} - -static void -tlstran_pipe_reap(tlstran_pipe *p) -{ - if (!nni_atomic_flag_test_and_set(&p->reaped)) { - if (p->tls != NULL) { - nng_stream_close(p->tls); - } - nni_reap(&tlstran_pipe_reap_list, p); - } -} - -static void -tlstran_ep_match(tlstran_ep *ep) -{ - nni_aio * aio; - tlstran_pipe *p; - - if (((aio = ep->useraio) == NULL) || - ((p = nni_list_first(&ep->waitpipes)) == NULL)) { - return; - } - nni_list_remove(&ep->waitpipes, p); - nni_list_append(&ep->busypipes, p); - ep->useraio = NULL; - p->rcvmax = ep->rcvmax; - nni_aio_set_output(aio, 0, p); - nni_aio_finish(aio, 0, 0); -} - -static void -tlstran_pipe_nego_cb(void *arg) -{ - tlstran_pipe *p = arg; - tlstran_ep * ep = p->ep; - nni_aio * aio = p->negoaio; - nni_aio * uaio; - int rv; - - nni_mtx_lock(&ep->mtx); - if ((rv = nni_aio_result(aio)) != 0) { - goto error; - } - - // We start transmitting before we receive. - if (p->gottxhead < p->wanttxhead) { - p->gottxhead += nni_aio_count(aio); - } else if (p->gotrxhead < p->wantrxhead) { - p->gotrxhead += nni_aio_count(aio); - } - - if (p->gottxhead < p->wanttxhead) { - nni_iov iov; - iov.iov_len = p->wanttxhead - p->gottxhead; - iov.iov_buf = &p->txlen[p->gottxhead]; - nni_aio_set_iov(aio, 1, &iov); - // send it down... - nng_stream_send(p->tls, aio); - nni_mtx_unlock(&ep->mtx); - return; - } - if (p->gotrxhead < p->wantrxhead) { - nni_iov iov; - iov.iov_len = p->wantrxhead - p->gotrxhead; - iov.iov_buf = &p->rxlen[p->gotrxhead]; - nni_aio_set_iov(aio, 1, &iov); - nng_stream_recv(p->tls, aio); - nni_mtx_unlock(&ep->mtx); - return; - } - // We have both sent and received the headers. Lets check the - // receive side header. - if ((p->rxlen[0] != 0) || (p->rxlen[1] != 'S') || - (p->rxlen[2] != 'P') || (p->rxlen[3] != 0) || (p->rxlen[6] != 0) || - (p->rxlen[7] != 0)) { - rv = NNG_EPROTO; - goto error; - } - - NNI_GET16(&p->rxlen[4], p->peer); - - // We are all ready now. We put this in the wait list, and - // then try to run the matcher. - nni_list_remove(&ep->negopipes, p); - nni_list_append(&ep->waitpipes, p); - - tlstran_ep_match(ep); - nni_mtx_unlock(&ep->mtx); - - return; - -error: - nng_stream_close(p->tls); - - if ((uaio = ep->useraio) != NULL) { - ep->useraio = NULL; - nni_aio_finish_error(uaio, rv); - } - nni_mtx_unlock(&ep->mtx); - tlstran_pipe_reap(p); -} - -static void -tlstran_pipe_send_cb(void *arg) -{ - tlstran_pipe *p = arg; - int rv; - nni_aio * aio; - size_t n; - nni_msg * msg; - nni_aio * txaio = p->txaio; - - nni_mtx_lock(&p->mtx); - aio = nni_list_first(&p->sendq); - - if ((rv = nni_aio_result(txaio)) != 0) { - // Intentionally we do not queue up another transfer. - // There's an excellent chance that the pipe is no longer - // usable, with a partial transfer. - // The protocol should see this error, and close the - // pipe itself, we hope. - nni_aio_list_remove(aio); - nni_mtx_unlock(&p->mtx); - nni_aio_finish_error(aio, rv); - nni_pipe_bump_error(p->npipe, rv); - return; - } - - n = nni_aio_count(txaio); - nni_aio_iov_advance(txaio, n); - if (nni_aio_iov_count(txaio) > 0) { - nng_stream_send(p->tls, txaio); - nni_mtx_unlock(&p->mtx); - return; - } - nni_aio_list_remove(aio); - tlstran_pipe_send_start(p); - - msg = nni_aio_get_msg(aio); - n = nni_msg_len(msg); - nni_pipe_bump_tx(p->npipe, n); - nni_mtx_unlock(&p->mtx); - nni_aio_set_msg(aio, NULL); - nni_msg_free(msg); - nni_aio_finish_sync(aio, 0, n); -} - -static void -tlstran_pipe_recv_cb(void *arg) -{ - tlstran_pipe *p = arg; - nni_aio * aio; - int rv; - size_t n; - nni_msg * msg; - nni_aio * rxaio = p->rxaio; - - nni_mtx_lock(&p->mtx); - aio = nni_list_first(&p->recvq); - - if ((rv = nni_aio_result(p->rxaio)) != 0) { - goto recv_error; - } - - n = nni_aio_count(rxaio); - nni_aio_iov_advance(rxaio, n); - if (nni_aio_iov_count(rxaio) > 0) { - // Was this a partial read? If so then resubmit for the rest. - nng_stream_recv(p->tls, rxaio); - nni_mtx_unlock(&p->mtx); - return; - } - - // If we don't have a message yet, we were reading the TCP message - // header, which is just the length. This tells us the size of the - // message to allocate and how much more to expect. - if (p->rxmsg == NULL) { - uint64_t len; - // We should have gotten a message header. - NNI_GET64(p->rxlen, len); - - // Make sure the message payload is not too big. If it is - // the caller will shut down the pipe. - if ((len > p->rcvmax) && (p->rcvmax > 0)) { - rv = NNG_EMSGSIZE; - goto recv_error; - } - - if ((rv = nni_msg_alloc(&p->rxmsg, (size_t) len)) != 0) { - goto recv_error; - } - - // Submit the rest of the data for a read -- we want to - // read the entire message now. - if (len != 0) { - nni_iov iov; - iov.iov_buf = nni_msg_body(p->rxmsg); - iov.iov_len = (size_t) len; - nni_aio_set_iov(rxaio, 1, &iov); - - nng_stream_recv(p->tls, rxaio); - nni_mtx_unlock(&p->mtx); - return; - } - } - - // We read a message completely. Let the user know the good news. - nni_aio_list_remove(aio); - msg = p->rxmsg; - p->rxmsg = NULL; - n = nni_msg_len(msg); - if (!nni_list_empty(&p->recvq)) { - tlstran_pipe_recv_start(p); - } - nni_pipe_bump_rx(p->npipe, n); - nni_mtx_unlock(&p->mtx); - - nni_aio_set_msg(aio, msg); - nni_aio_finish_sync(aio, 0, n); - return; - -recv_error: - nni_aio_list_remove(aio); - msg = p->rxmsg; - p->rxmsg = NULL; - nni_pipe_bump_error(p->npipe, rv); - // Intentionally, we do not queue up another receive. - // The protocol should notice this error and close the pipe. - nni_mtx_unlock(&p->mtx); - nni_msg_free(msg); - nni_aio_finish_error(aio, rv); -} - -static void -tlstran_pipe_send_cancel(nni_aio *aio, void *arg, int rv) -{ - tlstran_pipe *p = arg; - - nni_mtx_lock(&p->mtx); - if (!nni_aio_list_active(aio)) { - nni_mtx_unlock(&p->mtx); - return; - } - // If this is being sent, then cancel the pending transfer. - // The callback on the txaio will cause the user aio to - // be canceled too. - if (nni_list_first(&p->sendq) == aio) { - nni_aio_abort(p->txaio, rv); - nni_mtx_unlock(&p->mtx); - return; - } - nni_aio_list_remove(aio); - nni_mtx_unlock(&p->mtx); - - nni_aio_finish_error(aio, rv); -} - -static void -tlstran_pipe_send_start(tlstran_pipe *p) -{ - nni_aio *txaio; - nni_aio *aio; - nni_msg *msg; - int niov; - nni_iov iov[3]; - uint64_t len; - - if ((aio = nni_list_first(&p->sendq)) == NULL) { - return; - } - - msg = nni_aio_get_msg(aio); - len = nni_msg_len(msg) + nni_msg_header_len(msg); - - NNI_PUT64(p->txlen, len); - - txaio = p->txaio; - niov = 0; - iov[niov].iov_buf = p->txlen; - iov[niov].iov_len = sizeof(p->txlen); - niov++; - if (nni_msg_header_len(msg) > 0) { - iov[niov].iov_buf = nni_msg_header(msg); - iov[niov].iov_len = nni_msg_header_len(msg); - niov++; - } - if (nni_msg_len(msg) > 0) { - iov[niov].iov_buf = nni_msg_body(msg); - iov[niov].iov_len = nni_msg_len(msg); - niov++; - } - - nni_aio_set_iov(txaio, niov, iov); - nng_stream_send(p->tls, txaio); -} - -static void -tlstran_pipe_send(void *arg, nni_aio *aio) -{ - tlstran_pipe *p = arg; - int rv; - - if (nni_aio_begin(aio) != 0) { - return; - } - nni_mtx_lock(&p->mtx); - if ((rv = nni_aio_schedule(aio, tlstran_pipe_send_cancel, p)) != 0) { - nni_mtx_unlock(&p->mtx); - nni_aio_finish_error(aio, rv); - return; - } - nni_list_append(&p->sendq, aio); - if (nni_list_first(&p->sendq) == aio) { - tlstran_pipe_send_start(p); - } - nni_mtx_unlock(&p->mtx); -} - -static void -tlstran_pipe_recv_cancel(nni_aio *aio, void *arg, int rv) -{ - tlstran_pipe *p = arg; - - nni_mtx_lock(&p->mtx); - if (!nni_aio_list_active(aio)) { - nni_mtx_unlock(&p->mtx); - return; - } - // If receive in progress, then cancel the pending transfer. - // The callback on the rxaio will cause the user aio to - // be canceled too. - if (nni_list_first(&p->recvq) == aio) { - nni_aio_abort(p->rxaio, rv); - nni_mtx_unlock(&p->mtx); - return; - } - nni_aio_list_remove(aio); - nni_mtx_unlock(&p->mtx); - nni_aio_finish_error(aio, rv); -} - -static void -tlstran_pipe_recv_start(tlstran_pipe *p) -{ - nni_aio *aio; - nni_iov iov; - NNI_ASSERT(p->rxmsg == NULL); - - // Schedule a read of the IPC header. - aio = p->rxaio; - iov.iov_buf = p->rxlen; - iov.iov_len = sizeof(p->rxlen); - nni_aio_set_iov(aio, 1, &iov); - - nng_stream_recv(p->tls, aio); -} - -static void -tlstran_pipe_recv(void *arg, nni_aio *aio) -{ - tlstran_pipe *p = arg; - int rv; - - if (nni_aio_begin(aio) != 0) { - return; - } - nni_mtx_lock(&p->mtx); - if ((rv = nni_aio_schedule(aio, tlstran_pipe_recv_cancel, p)) != 0) { - nni_mtx_unlock(&p->mtx); - nni_aio_finish_error(aio, rv); - return; - } - - nni_aio_list_append(&p->recvq, aio); - if (nni_list_first(&p->recvq) == aio) { - tlstran_pipe_recv_start(p); - } - nni_mtx_unlock(&p->mtx); -} - -static uint16_t -tlstran_pipe_peer(void *arg) -{ - tlstran_pipe *p = arg; - - return (p->peer); -} - -static void -tlstran_pipe_start(tlstran_pipe *p, nng_stream *conn, tlstran_ep *ep) -{ - nni_iov iov; - - ep->refcnt++; - - p->tls = conn; - p->ep = ep; - p->proto = ep->proto; - - p->txlen[0] = 0; - p->txlen[1] = 'S'; - p->txlen[2] = 'P'; - p->txlen[3] = 0; - NNI_PUT16(&p->txlen[4], p->proto); - NNI_PUT16(&p->txlen[6], 0); - - p->gotrxhead = 0; - p->gottxhead = 0; - p->wantrxhead = 8; - p->wanttxhead = 8; - iov.iov_len = 8; - iov.iov_buf = &p->txlen[0]; - nni_aio_set_iov(p->negoaio, 1, &iov); - nni_list_append(&ep->negopipes, p); - - nni_aio_set_timeout(p->negoaio, 10000); // 10 sec timeout to negotiate - nng_stream_send(p->tls, p->negoaio); -} - -static void -tlstran_ep_fini(void *arg) -{ - tlstran_ep *ep = arg; - - nni_mtx_lock(&ep->mtx); - ep->fini = true; - if (ep->refcnt != 0) { - nni_mtx_unlock(&ep->mtx); - return; - } - nni_mtx_unlock(&ep->mtx); - nni_aio_stop(ep->timeaio); - nni_aio_stop(ep->connaio); - nng_stream_dialer_free(ep->dialer); - nng_stream_listener_free(ep->listener); - nni_aio_free(ep->timeaio); - nni_aio_free(ep->connaio); - - nni_mtx_fini(&ep->mtx); - NNI_FREE_STRUCT(ep); -} - -static void -tlstran_ep_close(void *arg) -{ - tlstran_ep * ep = arg; - tlstran_pipe *p; - - nni_mtx_lock(&ep->mtx); - ep->closed = true; - nni_aio_close(ep->timeaio); - - if (ep->dialer != NULL) { - nng_stream_dialer_close(ep->dialer); - } - if (ep->listener != NULL) { - nng_stream_listener_close(ep->listener); - } - NNI_LIST_FOREACH (&ep->negopipes, p) { - tlstran_pipe_close(p); - } - NNI_LIST_FOREACH (&ep->waitpipes, p) { - tlstran_pipe_close(p); - } - NNI_LIST_FOREACH (&ep->busypipes, p) { - tlstran_pipe_close(p); - } - if (ep->useraio != NULL) { - nni_aio_finish_error(ep->useraio, NNG_ECLOSED); - ep->useraio = NULL; - } - nni_mtx_unlock(&ep->mtx); -} - -// This parses off the optional source address that this transport uses. -// The special handling of this URL format is quite honestly an historical -// mistake, which we would remove if we could. -static int -tlstran_url_parse_source(nni_url *url, nng_sockaddr *sa, const nni_url *surl) -{ - int af; - char * semi; - char * src; - size_t len; - int rv; - nni_aio *aio; - - // We modify the URL. This relies on the fact that the underlying - // transport does not free this, so we can just use references. - - url->u_scheme = surl->u_scheme; - url->u_port = surl->u_port; - url->u_hostname = surl->u_hostname; - - if ((semi = strchr(url->u_hostname, ';')) == NULL) { - memset(sa, 0, sizeof(*sa)); - return (0); - } - - len = (size_t)(semi - url->u_hostname); - url->u_hostname = semi + 1; - - if (strcmp(surl->u_scheme, "tls+tcp") == 0) { - af = NNG_AF_UNSPEC; - } else if (strcmp(surl->u_scheme, "tls+tcp4") == 0) { - af = NNG_AF_INET; - } else if (strcmp(surl->u_scheme, "tls+tcp6") == 0) { - af = NNG_AF_INET6; - } else { - return (NNG_EADDRINVAL); - } - - if ((src = nni_alloc(len + 1)) == NULL) { - return (NNG_ENOMEM); - } - memcpy(src, surl->u_hostname, len); - src[len] = '\0'; - - if ((rv = nni_aio_alloc(&aio, NULL, NULL)) != 0) { - nni_free(src, len + 1); - return (rv); - } - - nni_resolv_ip(src, "0", af, 1, sa, aio); - nni_aio_wait(aio); - nni_aio_free(aio); - nni_free(src, len + 1); - return (rv); -} - -static void -tlstran_timer_cb(void *arg) -{ - tlstran_ep *ep = arg; - if (nni_aio_result(ep->timeaio) == 0) { - nng_stream_listener_accept(ep->listener, ep->connaio); - } -} - -static void -tlstran_accept_cb(void *arg) -{ - tlstran_ep * ep = arg; - nni_aio * aio = ep->connaio; - tlstran_pipe *p; - int rv; - nng_stream * conn; - - nni_mtx_lock(&ep->mtx); - - if ((rv = nni_aio_result(aio)) != 0) { - goto error; - } - - conn = nni_aio_get_output(aio, 0); - if ((rv = tlstran_pipe_alloc(&p)) != 0) { - nng_stream_free(conn); - goto error; - } - - if (ep->closed) { - tlstran_pipe_fini(p); - nng_stream_free(conn); - rv = NNG_ECLOSED; - goto error; - } - tlstran_pipe_start(p, conn, ep); - nng_stream_listener_accept(ep->listener, ep->connaio); - nni_mtx_unlock(&ep->mtx); - return; - -error: - // When an error here occurs, let's send a notice up to the consumer. - // That way it can be reported properly. - if ((aio = ep->useraio) != NULL) { - ep->useraio = NULL; - nni_aio_finish_error(aio, rv); - } - switch (rv) { - - case NNG_ENOMEM: - case NNG_ENOFILES: - // We need to cool down here, to avoid spinning. - nng_sleep_aio(10, ep->timeaio); - break; - - default: - // Start another accept. This is done because we want to - // ensure that TLS negotiations are disconnected from - // the upper layer accept logic. - if (!ep->closed) { - nng_stream_listener_accept(ep->listener, ep->connaio); - } - break; - } - nni_mtx_unlock(&ep->mtx); -} - -static void -tlstran_dial_cb(void *arg) -{ - tlstran_ep * ep = arg; - nni_aio * aio = ep->connaio; - tlstran_pipe *p; - int rv; - nng_stream * conn; - - if ((rv = nni_aio_result(aio)) != 0) { - goto error; - } - - conn = nni_aio_get_output(aio, 0); - if ((rv = tlstran_pipe_alloc(&p)) != 0) { - nng_stream_free(conn); - goto error; - } - nni_mtx_lock(&ep->mtx); - if (ep->closed) { - tlstran_pipe_fini(p); - nng_stream_free(conn); - rv = NNG_ECLOSED; - nni_mtx_unlock(&ep->mtx); - goto error; - } else { - tlstran_pipe_start(p, conn, ep); - } - nni_mtx_unlock(&ep->mtx); - return; - -error: - // Error connecting. We need to pass this straight back to the user. - nni_mtx_lock(&ep->mtx); - if ((aio = ep->useraio) != NULL) { - ep->useraio = NULL; - nni_aio_finish_error(aio, rv); - } - nni_mtx_unlock(&ep->mtx); -} - -static int -tlstran_ep_init(tlstran_ep **epp, nng_url *url, nni_sock *sock) -{ - tlstran_ep *ep; - - if ((ep = NNI_ALLOC_STRUCT(ep)) == NULL) { - return (NNG_ENOMEM); - } - nni_mtx_init(&ep->mtx); - NNI_LIST_INIT(&ep->busypipes, tlstran_pipe, node); - NNI_LIST_INIT(&ep->waitpipes, tlstran_pipe, node); - NNI_LIST_INIT(&ep->negopipes, tlstran_pipe, node); - - ep->proto = nni_sock_proto_id(sock); - ep->url = url; - -#ifdef NNG_ENABLE_STATS - static const nni_stat_info rcv_max_info = { - .si_name = "rcv_max", - .si_desc = "maximum receive size", - .si_type = NNG_STAT_LEVEL, - .si_unit = NNG_UNIT_BYTES, - .si_atomic = true, - }; - nni_stat_init(&ep->st_rcv_max, &rcv_max_info); -#endif - - *epp = ep; - return (0); -} - -static int -tlstran_ep_init_dialer(void **dp, nni_url *url, nni_dialer *ndialer) -{ - tlstran_ep * ep; - int rv; - nng_sockaddr srcsa; - nni_sock * sock = nni_dialer_sock(ndialer); - nni_url myurl; - - // Check for invalid URL components. - if ((strlen(url->u_path) != 0) && (strcmp(url->u_path, "/") != 0)) { - return (NNG_EADDRINVAL); - } - if ((url->u_fragment != NULL) || (url->u_userinfo != NULL) || - (url->u_query != NULL) || (strlen(url->u_hostname) == 0) || - (strlen(url->u_port) == 0)) { - return (NNG_EADDRINVAL); - } - - if ((rv = tlstran_url_parse_source(&myurl, &srcsa, url)) != 0) { - return (rv); - } - - if (((rv = tlstran_ep_init(&ep, url, sock)) != 0) || - ((rv = nni_aio_alloc(&ep->connaio, tlstran_dial_cb, ep)) != 0)) { - return (rv); - } - ep->authmode = NNG_TLS_AUTH_MODE_REQUIRED; - - if ((rv != 0) || - ((rv = nng_stream_dialer_alloc_url(&ep->dialer, &myurl)) != 0)) { - tlstran_ep_fini(ep); - return (rv); - } - if ((srcsa.s_family != NNG_AF_UNSPEC) && - ((rv = nni_stream_dialer_set(ep->dialer, NNG_OPT_LOCADDR, &srcsa, - sizeof(srcsa), NNI_TYPE_SOCKADDR)) != 0)) { - tlstran_ep_fini(ep); - return (rv); - } -#ifdef NNG_ENABLE_STATS - nni_dialer_add_stat(ndialer, &ep->st_rcv_max); -#endif - *dp = ep; - return (0); -} - -static int -tlstran_ep_init_listener(void **lp, nni_url *url, nni_listener *nlistener) -{ - tlstran_ep *ep; - int rv; - uint16_t af; - char * host = url->u_hostname; - nni_aio * aio; - nni_sock * sock = nni_listener_sock(nlistener); - - if (strcmp(url->u_scheme, "tls+tcp") == 0) { - af = NNG_AF_UNSPEC; - } else if (strcmp(url->u_scheme, "tls+tcp4") == 0) { - af = NNG_AF_INET; - } else if (strcmp(url->u_scheme, "tls+tcp6") == 0) { - af = NNG_AF_INET6; - } else { - return (NNG_EADDRINVAL); - } - - // Check for invalid URL components. - if ((strlen(url->u_path) != 0) && (strcmp(url->u_path, "/") != 0)) { - return (NNG_EADDRINVAL); - } - if ((url->u_fragment != NULL) || (url->u_userinfo != NULL) || - (url->u_query != NULL)) { - return (NNG_EADDRINVAL); - } - if (((rv = tlstran_ep_init(&ep, url, sock)) != 0) || - ((rv = nni_aio_alloc(&ep->connaio, tlstran_accept_cb, ep)) != 0) || - ((rv = nni_aio_alloc(&ep->timeaio, tlstran_timer_cb, ep)) != 0)) { - return (rv); - } - - ep->authmode = NNG_TLS_AUTH_MODE_NONE; - - if (strlen(host) == 0) { - host = NULL; - } - - // XXX: We are doing lookup at listener initialization. There is - // a valid argument that this should be done at bind time, but that - // would require making bind asynchronous. In some ways this would - // be worse than the cost of just waiting here. We always recommend - // using local IP addresses rather than names when possible. - - if ((rv = nni_aio_alloc(&aio, NULL, NULL)) != 0) { - tlstran_ep_fini(ep); - return (rv); - } - nni_resolv_ip(host, url->u_port, af, true, &ep->sa, aio); - nni_aio_wait(aio); - rv = nni_aio_result(aio); - nni_aio_free(aio); - - if ((rv != 0) || - ((rv = nng_stream_listener_alloc_url(&ep->listener, url)) != 0) || - ((rv = nni_stream_listener_set(ep->listener, NNG_OPT_TLS_AUTH_MODE, - &ep->authmode, sizeof(ep->authmode), NNI_TYPE_INT32)) != - 0)) { - tlstran_ep_fini(ep); - return (rv); - } -#ifdef NNG_ENABLE_STATS - nni_listener_add_stat(nlistener, &ep->st_rcv_max); -#endif - *lp = ep; - return (0); -} - -static void -tlstran_ep_cancel(nni_aio *aio, void *arg, int rv) -{ - tlstran_ep *ep = arg; - nni_mtx_lock(&ep->mtx); - if (ep->useraio == aio) { - ep->useraio = NULL; - nni_aio_finish_error(aio, rv); - } - nni_mtx_unlock(&ep->mtx); -} - -static void -tlstran_ep_connect(void *arg, nni_aio *aio) -{ - tlstran_ep *ep = arg; - int rv; - - if (nni_aio_begin(aio) != 0) { - return; - } - - nni_mtx_lock(&ep->mtx); - if (ep->closed) { - nni_mtx_unlock(&ep->mtx); - nni_aio_finish_error(aio, NNG_ECLOSED); - return; - } - if (ep->useraio != NULL) { - nni_mtx_unlock(&ep->mtx); - nni_aio_finish_error(aio, NNG_EBUSY); - return; - } - if ((rv = nni_aio_schedule(aio, tlstran_ep_cancel, ep)) != 0) { - nni_mtx_unlock(&ep->mtx); - nni_aio_finish_error(aio, rv); - return; - } - ep->useraio = aio; - - nng_stream_dialer_dial(ep->dialer, ep->connaio); - nni_mtx_unlock(&ep->mtx); -} - -static int -tlstran_ep_bind(void *arg) -{ - tlstran_ep *ep = arg; - int rv; - - nni_mtx_lock(&ep->mtx); - rv = nng_stream_listener_listen(ep->listener); - nni_mtx_unlock(&ep->mtx); - - return (rv); -} - -static void -tlstran_ep_accept(void *arg, nni_aio *aio) -{ - tlstran_ep *ep = arg; - int rv; - - if (nni_aio_begin(aio) != 0) { - return; - } - nni_mtx_lock(&ep->mtx); - if (ep->closed) { - nni_mtx_unlock(&ep->mtx); - nni_aio_finish_error(aio, NNG_ECLOSED); - return; - } - if (ep->useraio != NULL) { - nni_mtx_unlock(&ep->mtx); - nni_aio_finish_error(aio, NNG_EBUSY); - return; - } - if ((rv = nni_aio_schedule(aio, tlstran_ep_cancel, ep)) != 0) { - nni_mtx_unlock(&ep->mtx); - nni_aio_finish_error(aio, rv); - return; - } - ep->useraio = aio; - if (!ep->started) { - ep->started = true; - nng_stream_listener_accept(ep->listener, ep->connaio); - } else { - tlstran_ep_match(ep); - } - nni_mtx_unlock(&ep->mtx); -} - -static int -tlstran_ep_set_recvmaxsz(void *arg, const void *v, size_t sz, nni_type t) -{ - tlstran_ep *ep = arg; - size_t val; - int rv; - if ((rv = nni_copyin_size(&val, v, sz, 0, NNI_MAXSZ, t)) == 0) { - tlstran_pipe *p; - nni_mtx_lock(&ep->mtx); - ep->rcvmax = val; - NNI_LIST_FOREACH (&ep->waitpipes, p) { - p->rcvmax = val; - } - NNI_LIST_FOREACH (&ep->negopipes, p) { - p->rcvmax = val; - } - NNI_LIST_FOREACH (&ep->busypipes, p) { - p->rcvmax = val; - } - nni_mtx_unlock(&ep->mtx); -#ifdef NNG_ENABLE_STATS - nni_stat_set_value(&ep->st_rcv_max, val); -#endif - } - return (rv); -} - -static int -tlstran_ep_get_recvmaxsz(void *arg, void *v, size_t *szp, nni_type t) -{ - tlstran_ep *ep = arg; - int rv; - nni_mtx_lock(&ep->mtx); - rv = nni_copyout_size(ep->rcvmax, v, szp, t); - nni_mtx_unlock(&ep->mtx); - return (rv); -} - -static int -tlstran_ep_get_url(void *arg, void *v, size_t *szp, nni_type t) -{ - tlstran_ep *ep = arg; - char * s; - int rv; - int port = 0; - - if (ep->listener != NULL) { - (void) nng_stream_listener_get_int( - ep->listener, NNG_OPT_TCP_BOUND_PORT, &port); - } - if ((rv = nni_url_asprintf_port(&s, ep->url, port)) == 0) { - rv = nni_copyout_str(s, v, szp, t); - nni_strfree(s); - } - return (rv); -} - -static const nni_option tlstran_pipe_opts[] = { - // terminate list - { - .o_name = NULL, - }, -}; - -static int -tlstran_pipe_getopt( - void *arg, const char *name, void *buf, size_t *szp, nni_type t) -{ - tlstran_pipe *p = arg; - int rv; - - if ((rv = nni_stream_get(p->tls, name, buf, szp, t)) == NNG_ENOTSUP) { - rv = nni_getopt(tlstran_pipe_opts, name, p, buf, szp, t); - } - return (rv); -} - -static nni_tran_pipe_ops tlstran_pipe_ops = { - .p_init = tlstran_pipe_init, - .p_fini = tlstran_pipe_fini, - .p_stop = tlstran_pipe_stop, - .p_send = tlstran_pipe_send, - .p_recv = tlstran_pipe_recv, - .p_close = tlstran_pipe_close, - .p_peer = tlstran_pipe_peer, - .p_getopt = tlstran_pipe_getopt, -}; - -static nni_option tlstran_ep_options[] = { - { - .o_name = NNG_OPT_RECVMAXSZ, - .o_get = tlstran_ep_get_recvmaxsz, - .o_set = tlstran_ep_set_recvmaxsz, - }, - { - .o_name = NNG_OPT_URL, - .o_get = tlstran_ep_get_url, - }, - // terminate list - { - .o_name = NULL, - }, -}; - -static int -tlstran_dialer_getopt( - void *arg, const char *name, void *buf, size_t *szp, nni_type t) -{ - int rv; - tlstran_ep *ep = arg; - - rv = nni_stream_dialer_get(ep->dialer, name, buf, szp, t); - if (rv == NNG_ENOTSUP) { - rv = nni_getopt(tlstran_ep_options, name, ep, buf, szp, t); - } - return (rv); -} - -static int -tlstran_dialer_setopt( - void *arg, const char *name, const void *buf, size_t sz, nni_type t) -{ - int rv; - tlstran_ep *ep = arg; - - rv = nni_stream_dialer_set( - ep != NULL ? ep->dialer : NULL, name, buf, sz, t); - if (rv == NNG_ENOTSUP) { - rv = nni_setopt(tlstran_ep_options, name, ep, buf, sz, t); - } - return (rv); -} - -static int -tlstran_listener_get( - void *arg, const char *name, void *buf, size_t *szp, nni_type t) -{ - int rv; - tlstran_ep *ep = arg; - - rv = nni_stream_listener_get(ep->listener, name, buf, szp, t); - if (rv == NNG_ENOTSUP) { - rv = nni_getopt(tlstran_ep_options, name, ep, buf, szp, t); - } - return (rv); -} - -static int -tlstran_listener_set( - void *arg, const char *name, const void *buf, size_t sz, nni_type t) -{ - int rv; - tlstran_ep *ep = arg; - - rv = nni_stream_listener_set( - ep != NULL ? ep->listener : NULL, name, buf, sz, t); - if (rv == NNG_ENOTSUP) { - rv = nni_setopt(tlstran_ep_options, name, ep, buf, sz, t); - } - return (rv); -} - -static nni_tran_dialer_ops tlstran_dialer_ops = { - .d_init = tlstran_ep_init_dialer, - .d_fini = tlstran_ep_fini, - .d_connect = tlstran_ep_connect, - .d_close = tlstran_ep_close, - .d_getopt = tlstran_dialer_getopt, - .d_setopt = tlstran_dialer_setopt, -}; - -static nni_tran_listener_ops tlstran_listener_ops = { - .l_init = tlstran_ep_init_listener, - .l_fini = tlstran_ep_fini, - .l_bind = tlstran_ep_bind, - .l_accept = tlstran_ep_accept, - .l_close = tlstran_ep_close, - .l_getopt = tlstran_listener_get, - .l_setopt = tlstran_listener_set, -}; - -static nni_tran tls_tran = { - .tran_version = NNI_TRANSPORT_VERSION, - .tran_scheme = "tls+tcp", - .tran_dialer = &tlstran_dialer_ops, - .tran_listener = &tlstran_listener_ops, - .tran_pipe = &tlstran_pipe_ops, - .tran_init = tlstran_init, - .tran_fini = tlstran_fini, -}; - -static nni_tran tls4_tran = { - .tran_version = NNI_TRANSPORT_VERSION, - .tran_scheme = "tls+tcp4", - .tran_dialer = &tlstran_dialer_ops, - .tran_listener = &tlstran_listener_ops, - .tran_pipe = &tlstran_pipe_ops, - .tran_init = tlstran_init, - .tran_fini = tlstran_fini, -}; - -static nni_tran tls6_tran = { - .tran_version = NNI_TRANSPORT_VERSION, - .tran_scheme = "tls+tcp6", - .tran_dialer = &tlstran_dialer_ops, - .tran_listener = &tlstran_listener_ops, - .tran_pipe = &tlstran_pipe_ops, - .tran_init = tlstran_init, - .tran_fini = tlstran_fini, -}; - -int -nng_tls_register(void) -{ - int rv; - if (((rv = nni_tran_register(&tls_tran)) != 0) || - ((rv = nni_tran_register(&tls4_tran)) != 0) || - ((rv = nni_tran_register(&tls6_tran)) != 0)) { - return (rv); - } - return (0); -} diff --git a/src/transport/ws/CMakeLists.txt b/src/transport/ws/CMakeLists.txt deleted file mode 100644 index 6e409b43..00000000 --- a/src/transport/ws/CMakeLists.txt +++ /dev/null @@ -1,24 +0,0 @@ -# -# Copyright 2020 Staysail Systems, Inc. -# Copyright 2018 Capitar IT Group BV -# -# This software is supplied under the terms of the MIT License, a -# copy of which should be located in the distribution where this -# file was obtained (LICENSE.txt). A copy of the license may also be -# found online at https://opensource.org/licenses/MIT. -# - -# WebSocket transport -nng_directory(ws) - -if (NNG_TRANSPORT_WS OR NNG_TRANSPORT_WSS) - set(WS_ON ON) -endif() - -nng_defines_if(NNG_TRANSPORT_WS NNG_TRANSPORT_WS) -nng_defines_if(NNG_TRANSPORT_WSS NNG_TRANSPORT_WSS) -nng_sources_if(WS_ON websocket.c) -nng_headers_if(WS_ON nng/transport/ws/websocket.h) -nng_test_if(WS_ON ws_test) - - diff --git a/src/transport/ws/README.adoc b/src/transport/ws/README.adoc deleted file mode 100644 index e3101297..00000000 --- a/src/transport/ws/README.adoc +++ /dev/null @@ -1,38 +0,0 @@ -= websocket transport - -This transport provides support for SP over websocket using TCP or TLS. -When using TCP, it is compatible with the libnanomsg legacy transport. -It also is compatible with mangos (both TCP and TLS). - -TLS support requires the mbedTLS library. - -We set the "protocol" such as "pair.sp.nanomsg.org" in the -Sec-WebSocket-Protocol field -- the client sets to the the server's -protocol - i.e. the protocol that the server speaks. For example, -if the the server is a REP, then a REQ client would send "rep.sp.nanomsg.org". - -The server sends the same value (it's own), per the WebSocket specs. (Note -that the client's protocol is never sent, but assumed to be complementary -to the protocol in the Sec-WebSocket-Protocol field.) - -Each SP message is a WebSocket message. - -WebSocket is defined in RFC 6455. - -== Design - -We unfortunately need to implement our own design for this -- the only -reasonable client library would be libcurl, and there is a dearth of -suitable server libraries. Since we don't have to support full HTTP, but -just the initial handshake, this isn't too tragic. - -== Multiple Server Sockets - -In order to support Multiple Server sockets listening on the same port, -the application must be long lived. We will set up a listener on the -configured TCP (or TLS) port, and examine the PATH supplied in the GET. -This will be used to match against the URL requested, and if the URL -matches we will create the appropriate pipe. - -If no server endpoint at that address can be found, we return an -HTTP error, and close the socket. diff --git a/src/transport/ws/websocket.c b/src/transport/ws/websocket.c deleted file mode 100644 index 3f73f47f..00000000 --- a/src/transport/ws/websocket.c +++ /dev/null @@ -1,740 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// Copyright 2018 Capitar IT Group BV -// Copyright 2019 Devolutions -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - -#include -#include -#include - -#include "core/nng_impl.h" -#include "supplemental/websocket/websocket.h" - -#include -#include - -typedef struct ws_dialer ws_dialer; -typedef struct ws_listener ws_listener; -typedef struct ws_pipe ws_pipe; - -struct ws_dialer { - uint16_t peer; // remote protocol - nni_list aios; - nni_mtx mtx; - nni_aio * connaio; - nng_stream_dialer *dialer; - bool started; -}; - -struct ws_listener { - uint16_t peer; // remote protocol - nni_list aios; - nni_mtx mtx; - nni_aio * accaio; - nng_stream_listener *listener; - bool started; -}; - -struct ws_pipe { - nni_mtx mtx; - bool closed; - uint16_t peer; - nni_aio * user_txaio; - nni_aio * user_rxaio; - nni_aio * txaio; - nni_aio * rxaio; - nng_stream *ws; -}; - -static void -wstran_pipe_send_cb(void *arg) -{ - ws_pipe *p = arg; - nni_aio *taio; - nni_aio *uaio; - - nni_mtx_lock(&p->mtx); - taio = p->txaio; - uaio = p->user_txaio; - p->user_txaio = NULL; - - if (uaio != NULL) { - int rv; - if ((rv = nni_aio_result(taio)) != 0) { - nni_aio_finish_error(uaio, rv); - } else { - nni_aio_finish(uaio, 0, 0); - } - } - nni_mtx_unlock(&p->mtx); -} - -static void -wstran_pipe_recv_cb(void *arg) -{ - ws_pipe *p = arg; - nni_aio *raio = p->rxaio; - nni_aio *uaio; - int rv; - - nni_mtx_lock(&p->mtx); - uaio = p->user_rxaio; - p->user_rxaio = NULL; - if ((rv = nni_aio_result(raio)) != 0) { - if (uaio != NULL) { - nni_aio_finish_error(uaio, rv); - } - } else { - nni_msg *msg = nni_aio_get_msg(raio); - if (uaio != NULL) { - nni_aio_finish_msg(uaio, msg); - } else { - nni_msg_free(msg); - } - } - nni_mtx_unlock(&p->mtx); -} - -static void -wstran_pipe_recv_cancel(nni_aio *aio, void *arg, int rv) -{ - ws_pipe *p = arg; - nni_mtx_lock(&p->mtx); - if (p->user_rxaio != aio) { - nni_mtx_unlock(&p->mtx); - return; - } - p->user_rxaio = NULL; - nni_aio_abort(p->rxaio, rv); - nni_aio_finish_error(aio, rv); - nni_mtx_unlock(&p->mtx); -} - -static void -wstran_pipe_recv(void *arg, nni_aio *aio) -{ - ws_pipe *p = arg; - int rv; - - if (nni_aio_begin(aio) != 0) { - return; - } - nni_mtx_lock(&p->mtx); - if ((rv = nni_aio_schedule(aio, wstran_pipe_recv_cancel, p)) != 0) { - nni_mtx_unlock(&p->mtx); - nni_aio_finish_error(aio, rv); - return; - } - p->user_rxaio = aio; - nng_stream_recv(p->ws, p->rxaio); - nni_mtx_unlock(&p->mtx); -} - -static void -wstran_pipe_send_cancel(nni_aio *aio, void *arg, int rv) -{ - ws_pipe *p = arg; - nni_mtx_lock(&p->mtx); - if (p->user_txaio != aio) { - nni_mtx_unlock(&p->mtx); - return; - } - p->user_txaio = NULL; - nni_aio_abort(p->txaio, rv); - nni_aio_finish_error(aio, rv); - nni_mtx_unlock(&p->mtx); -} - -static void -wstran_pipe_send(void *arg, nni_aio *aio) -{ - ws_pipe *p = arg; - int rv; - - if (nni_aio_begin(aio) != 0) { - return; - } - nni_mtx_lock(&p->mtx); - if ((rv = nni_aio_schedule(aio, wstran_pipe_send_cancel, p)) != 0) { - nni_mtx_unlock(&p->mtx); - nni_aio_finish_error(aio, rv); - return; - } - p->user_txaio = aio; - nni_aio_set_msg(p->txaio, nni_aio_get_msg(aio)); - nni_aio_set_msg(aio, NULL); - - nng_stream_send(p->ws, p->txaio); - nni_mtx_unlock(&p->mtx); -} - -static void -wstran_pipe_stop(void *arg) -{ - ws_pipe *p = arg; - - nni_aio_stop(p->rxaio); - nni_aio_stop(p->txaio); -} - -static int -wstran_pipe_init(void *arg, nni_pipe *pipe) -{ - NNI_ARG_UNUSED(arg); - NNI_ARG_UNUSED(pipe); - return (0); -} - -static void -wstran_pipe_fini(void *arg) -{ - ws_pipe *p = arg; - - nni_aio_free(p->rxaio); - nni_aio_free(p->txaio); - - nng_stream_free(p->ws); - nni_mtx_fini(&p->mtx); - NNI_FREE_STRUCT(p); -} - -static void -wstran_pipe_close(void *arg) -{ - ws_pipe *p = arg; - - nni_aio_close(p->rxaio); - nni_aio_close(p->txaio); - - nni_mtx_lock(&p->mtx); - nng_stream_close(p->ws); - nni_mtx_unlock(&p->mtx); -} - -static int -wstran_pipe_alloc(ws_pipe **pipep, void *ws) -{ - ws_pipe *p; - int rv; - - if ((p = NNI_ALLOC_STRUCT(p)) == NULL) { - return (NNG_ENOMEM); - } - nni_mtx_init(&p->mtx); - - // Initialize AIOs. - if (((rv = nni_aio_alloc(&p->txaio, wstran_pipe_send_cb, p)) != 0) || - ((rv = nni_aio_alloc(&p->rxaio, wstran_pipe_recv_cb, p)) != 0)) { - wstran_pipe_fini(p); - return (rv); - } - p->ws = ws; - - *pipep = p; - return (0); -} - -static uint16_t -wstran_pipe_peer(void *arg) -{ - ws_pipe *p = arg; - - return (p->peer); -} - -static int -ws_listener_bind(void *arg) -{ - ws_listener *l = arg; - int rv; - - if ((rv = nng_stream_listener_listen(l->listener)) == 0) { - l->started = true; - } - return (rv); -} - -static void -ws_listener_cancel(nni_aio *aio, void *arg, int rv) -{ - ws_listener *l = arg; - - nni_mtx_lock(&l->mtx); - if (nni_aio_list_active(aio)) { - nni_aio_list_remove(aio); - nni_aio_finish_error(aio, rv); - } - nni_mtx_unlock(&l->mtx); -} - -static void -wstran_listener_accept(void *arg, nni_aio *aio) -{ - ws_listener *l = arg; - int rv; - - // We already bound, so we just need to look for an available - // pipe (created by the handler), and match it. - // Otherwise we stick the AIO in the accept list. - if (nni_aio_begin(aio) != 0) { - return; - } - nni_mtx_lock(&l->mtx); - if ((rv = nni_aio_schedule(aio, ws_listener_cancel, l)) != 0) { - nni_mtx_unlock(&l->mtx); - nni_aio_finish_error(aio, rv); - return; - } - nni_list_append(&l->aios, aio); - if (aio == nni_list_first(&l->aios)) { - nng_stream_listener_accept(l->listener, l->accaio); - } - nni_mtx_unlock(&l->mtx); -} - -static void -wstran_dialer_cancel(nni_aio *aio, void *arg, int rv) -{ - ws_dialer *d = arg; - - nni_mtx_lock(&d->mtx); - if (nni_aio_list_active(aio)) { - nni_aio_list_remove(aio); - nni_aio_finish_error(aio, rv); - } - nni_mtx_unlock(&d->mtx); -} - -static void -wstran_dialer_connect(void *arg, nni_aio *aio) -{ - ws_dialer *d = arg; - int rv; - - if (nni_aio_begin(aio) != 0) { - return; - } - - nni_mtx_lock(&d->mtx); - if ((rv = nni_aio_schedule(aio, wstran_dialer_cancel, d)) != 0) { - nni_mtx_unlock(&d->mtx); - nni_aio_finish_error(aio, rv); - return; - } - NNI_ASSERT(nni_list_empty(&d->aios)); - d->started = true; - nni_list_append(&d->aios, aio); - nng_stream_dialer_dial(d->dialer, d->connaio); - nni_mtx_unlock(&d->mtx); -} - -static const nni_option ws_pipe_options[] = { - // terminate list - { - .o_name = NULL, - } -}; - -static int -wstran_pipe_getopt( - void *arg, const char *name, void *buf, size_t *szp, nni_type t) -{ - ws_pipe *p = arg; - int rv; - - if ((rv = nni_stream_get(p->ws, name, buf, szp, t)) == NNG_ENOTSUP) { - rv = nni_getopt(ws_pipe_options, name, p, buf, szp, t); - } - return (rv); -} - -static nni_tran_pipe_ops ws_pipe_ops = { - .p_init = wstran_pipe_init, - .p_fini = wstran_pipe_fini, - .p_stop = wstran_pipe_stop, - .p_send = wstran_pipe_send, - .p_recv = wstran_pipe_recv, - .p_close = wstran_pipe_close, - .p_peer = wstran_pipe_peer, - .p_getopt = wstran_pipe_getopt, -}; - -static void -wstran_dialer_fini(void *arg) -{ - ws_dialer *d = arg; - - nni_aio_stop(d->connaio); - nng_stream_dialer_free(d->dialer); - nni_aio_free(d->connaio); - nni_mtx_fini(&d->mtx); - NNI_FREE_STRUCT(d); -} - -static void -wstran_listener_fini(void *arg) -{ - ws_listener *l = arg; - - nni_aio_stop(l->accaio); - nng_stream_listener_free(l->listener); - nni_aio_free(l->accaio); - nni_mtx_fini(&l->mtx); - NNI_FREE_STRUCT(l); -} - -static void -wstran_connect_cb(void *arg) -{ - ws_dialer * d = arg; - ws_pipe * p; - nni_aio * caio = d->connaio; - nni_aio * uaio; - int rv; - nng_stream *ws = NULL; - - nni_mtx_lock(&d->mtx); - if (nni_aio_result(caio) == 0) { - ws = nni_aio_get_output(caio, 0); - } - if ((uaio = nni_list_first(&d->aios)) == NULL) { - // The client stopped caring about this! - nng_stream_free(ws); - nni_mtx_unlock(&d->mtx); - return; - } - nni_aio_list_remove(uaio); - NNI_ASSERT(nni_list_empty(&d->aios)); - if ((rv = nni_aio_result(caio)) != 0) { - nni_aio_finish_error(uaio, rv); - } else if ((rv = wstran_pipe_alloc(&p, ws)) != 0) { - nng_stream_free(ws); - nni_aio_finish_error(uaio, rv); - } else { - p->peer = d->peer; - - nni_aio_set_output(uaio, 0, p); - nni_aio_finish(uaio, 0, 0); - } - nni_mtx_unlock(&d->mtx); -} - -static void -wstran_dialer_close(void *arg) -{ - ws_dialer *d = arg; - - nni_aio_close(d->connaio); - nng_stream_dialer_close(d->dialer); -} - -static void -wstran_listener_close(void *arg) -{ - ws_listener *l = arg; - - nni_aio_close(l->accaio); - nng_stream_listener_close(l->listener); -} - -static void -wstran_accept_cb(void *arg) -{ - ws_listener *l = arg; - nni_aio * aaio = l->accaio; - nni_aio * uaio; - int rv; - - nni_mtx_lock(&l->mtx); - uaio = nni_list_first(&l->aios); - if ((rv = nni_aio_result(aaio)) != 0) { - if (uaio != NULL) { - nni_aio_list_remove(uaio); - nni_aio_finish_error(uaio, rv); - } - } else { - nng_stream *ws = nni_aio_get_output(aaio, 0); - if (uaio != NULL) { - ws_pipe *p; - // Make a pipe - nni_aio_list_remove(uaio); - if ((rv = wstran_pipe_alloc(&p, ws)) != 0) { - nng_stream_close(ws); - nni_aio_finish_error(uaio, rv); - } else { - p->peer = l->peer; - - nni_aio_set_output(uaio, 0, p); - nni_aio_finish(uaio, 0, 0); - } - } - } - if (!nni_list_empty(&l->aios)) { - nng_stream_listener_accept(l->listener, aaio); - } - nni_mtx_unlock(&l->mtx); -} - -static int -wstran_dialer_init(void **dp, nng_url *url, nni_dialer *ndialer) -{ - ws_dialer *d; - nni_sock * s = nni_dialer_sock(ndialer); - int rv; - char name[64]; - - if ((d = NNI_ALLOC_STRUCT(d)) == NULL) { - return (NNG_ENOMEM); - } - nni_mtx_init(&d->mtx); - - nni_aio_list_init(&d->aios); - - d->peer = nni_sock_peer_id(s); - - snprintf( - name, sizeof(name), "%s.sp.nanomsg.org", nni_sock_peer_name(s)); - - if (((rv = nni_ws_dialer_alloc(&d->dialer, url)) != 0) || - ((rv = nni_aio_alloc(&d->connaio, wstran_connect_cb, d)) != 0) || - ((rv = nng_stream_dialer_set_bool( - d->dialer, NNI_OPT_WS_MSGMODE, true)) != 0) || - ((rv = nng_stream_dialer_set_string( - d->dialer, NNG_OPT_WS_PROTOCOL, name)) != 0)) { - wstran_dialer_fini(d); - return (rv); - } - - *dp = d; - return (0); -} - -static int -wstran_listener_init(void **lp, nng_url *url, nni_listener *listener) -{ - ws_listener *l; - int rv; - nni_sock * s = nni_listener_sock(listener); - char name[64]; - - if ((l = NNI_ALLOC_STRUCT(l)) == NULL) { - return (NNG_ENOMEM); - } - nni_mtx_init(&l->mtx); - - nni_aio_list_init(&l->aios); - - l->peer = nni_sock_peer_id(s); - - snprintf( - name, sizeof(name), "%s.sp.nanomsg.org", nni_sock_proto_name(s)); - - if (((rv = nni_ws_listener_alloc(&l->listener, url)) != 0) || - ((rv = nni_aio_alloc(&l->accaio, wstran_accept_cb, l)) != 0) || - ((rv = nng_stream_listener_set_bool( - l->listener, NNI_OPT_WS_MSGMODE, true)) != 0) || - ((rv = nng_stream_listener_set_string( - l->listener, NNG_OPT_WS_PROTOCOL, name)) != 0)) { - wstran_listener_fini(l); - return (rv); - } - *lp = l; - return (0); -} - -static int -wstran_init(void) -{ - return (0); -} - -static void -wstran_fini(void) -{ -} - -static const nni_option wstran_ep_opts[] = { - // terminate list - { - .o_name = NULL, - }, -}; - -static int -wstran_dialer_getopt( - void *arg, const char *name, void *buf, size_t *szp, nni_type t) -{ - ws_dialer *d = arg; - int rv; - - rv = nni_stream_dialer_get(d->dialer, name, buf, szp, t); - if (rv == NNG_ENOTSUP) { - rv = nni_getopt(wstran_ep_opts, name, d, buf, szp, t); - } - return (rv); -} - -static int -wstran_dialer_setopt( - void *arg, const char *name, const void *buf, size_t sz, nni_type t) -{ - ws_dialer *d = arg; - int rv; - - rv = nni_stream_dialer_set(d->dialer, name, buf, sz, t); - if (rv == NNG_ENOTSUP) { - rv = nni_setopt(wstran_ep_opts, name, d, buf, sz, t); - } - return (rv); -} - -static int -wstran_listener_get( - void *arg, const char *name, void *buf, size_t *szp, nni_type t) -{ - ws_listener *l = arg; - int rv; - - rv = nni_stream_listener_get(l->listener, name, buf, szp, t); - if (rv == NNG_ENOTSUP) { - rv = nni_getopt(wstran_ep_opts, name, l, buf, szp, t); - } - return (rv); -} - -static int -wstran_listener_set( - void *arg, const char *name, const void *buf, size_t sz, nni_type t) -{ - ws_listener *l = arg; - int rv; - - rv = nni_stream_listener_set(l->listener, name, buf, sz, t); - if (rv == NNG_ENOTSUP) { - rv = nni_setopt(wstran_ep_opts, name, l, buf, sz, t); - } - return (rv); -} - -static nni_tran_dialer_ops ws_dialer_ops = { - .d_init = wstran_dialer_init, - .d_fini = wstran_dialer_fini, - .d_connect = wstran_dialer_connect, - .d_close = wstran_dialer_close, - .d_setopt = wstran_dialer_setopt, - .d_getopt = wstran_dialer_getopt, -}; - -static nni_tran_listener_ops ws_listener_ops = { - .l_init = wstran_listener_init, - .l_fini = wstran_listener_fini, - .l_bind = ws_listener_bind, - .l_accept = wstran_listener_accept, - .l_close = wstran_listener_close, - .l_setopt = wstran_listener_set, - .l_getopt = wstran_listener_get, -}; - -static nni_tran ws_tran = { - .tran_version = NNI_TRANSPORT_VERSION, - .tran_scheme = "ws", - .tran_dialer = &ws_dialer_ops, - .tran_listener = &ws_listener_ops, - .tran_pipe = &ws_pipe_ops, - .tran_init = wstran_init, - .tran_fini = wstran_fini, -}; - -static nni_tran ws4_tran = { - .tran_version = NNI_TRANSPORT_VERSION, - .tran_scheme = "ws4", - .tran_dialer = &ws_dialer_ops, - .tran_listener = &ws_listener_ops, - .tran_pipe = &ws_pipe_ops, - .tran_init = wstran_init, - .tran_fini = wstran_fini, -}; - -static nni_tran ws6_tran = { - .tran_version = NNI_TRANSPORT_VERSION, - .tran_scheme = "ws6", - .tran_dialer = &ws_dialer_ops, - .tran_listener = &ws_listener_ops, - .tran_pipe = &ws_pipe_ops, - .tran_init = wstran_init, - .tran_fini = wstran_fini, -}; - -int -nng_ws_register(void) -{ - int rv; - if (((rv = nni_tran_register(&ws_tran)) != 0) || - ((rv = nni_tran_register(&ws4_tran)) != 0) || - ((rv = nni_tran_register(&ws6_tran)) != 0)) { - return (rv); - } - - return (0); -} - -#ifdef NNG_TRANSPORT_WSS - -static nni_tran wss_tran = { - .tran_version = NNI_TRANSPORT_VERSION, - .tran_scheme = "wss", - .tran_dialer = &ws_dialer_ops, - .tran_listener = &ws_listener_ops, - .tran_pipe = &ws_pipe_ops, - .tran_init = wstran_init, - .tran_fini = wstran_fini, -}; - -static nni_tran wss4_tran = { - .tran_version = NNI_TRANSPORT_VERSION, - .tran_scheme = "wss4", - .tran_dialer = &ws_dialer_ops, - .tran_listener = &ws_listener_ops, - .tran_pipe = &ws_pipe_ops, - .tran_init = wstran_init, - .tran_fini = wstran_fini, -}; - -static nni_tran wss6_tran = { - .tran_version = NNI_TRANSPORT_VERSION, - .tran_scheme = "wss6", - .tran_dialer = &ws_dialer_ops, - .tran_listener = &ws_listener_ops, - .tran_pipe = &ws_pipe_ops, - .tran_init = wstran_init, - .tran_fini = wstran_fini, -}; - -int -nng_wss_register(void) -{ - int rv; - if (((rv = nni_tran_register(&wss_tran)) != 0) || - ((rv = nni_tran_register(&wss4_tran)) != 0) || - ((rv = nni_tran_register(&wss6_tran)) != 0)) { - return (rv); - } - - return (0); -} - -#else - -int -nng_wss_register(void) -{ - return (0); -} - -#endif // NNG_TRANSPORT_WSS diff --git a/src/transport/ws/ws_test.c b/src/transport/ws/ws_test.c deleted file mode 100644 index 7cbcd9d7..00000000 --- a/src/transport/ws/ws_test.c +++ /dev/null @@ -1,181 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// Copyright 2018 Cody Piersall -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - -#include - -static void -test_ws_url_path_filters(void) -{ - nng_socket s1; - nng_socket s2; - char addr[NNG_MAXADDRLEN]; - - NUTS_OPEN(s1); - NUTS_OPEN(s2); - - nuts_scratch_addr("ws", sizeof(addr), addr); - NUTS_PASS(nng_listen(s1, addr, NULL, 0)); - - // Now try we just remove the last character for now. - // This will make the path different. - addr[strlen(addr) - 1] = '\0'; - NUTS_FAIL(nng_dial(s2, addr, NULL, 0), NNG_ECONNREFUSED); - - NUTS_CLOSE(s1); - NUTS_CLOSE(s2); -} - -static void -test_wild_card_port(void) -{ - nng_socket s1; - nng_socket s2; - nng_socket s3; - nng_socket s4; - nng_socket s5; - nng_socket s6; - - nng_listener l1; - nng_listener l2; - nng_listener l3; - int port1; - int port2; - int port3; - char ws_url[128]; - NUTS_OPEN(s1); - NUTS_OPEN(s2); - NUTS_OPEN(s3); - NUTS_OPEN(s4); - NUTS_OPEN(s5); - NUTS_OPEN(s6); - NUTS_PASS(nng_listen(s1, "ws://127.0.0.1:0/one", &l1, 0)); - NUTS_PASS( - nng_listener_get_int(l1, NNG_OPT_TCP_BOUND_PORT, &port1)); - NUTS_TRUE(port1 != 0); - snprintf(ws_url, sizeof(ws_url), "ws4://127.0.0.1:%d/two", port1); - NUTS_PASS(nng_listen(s2, ws_url, &l2, 0)); - NUTS_PASS( - nng_listener_get_int(l2, NNG_OPT_TCP_BOUND_PORT, &port2)); - NUTS_TRUE(port1 != 0); - NUTS_TRUE(port1 == port2); - // Now try a different wild card port. - NUTS_PASS(nng_listen(s3, "ws4://127.0.0.1:0/three", &l3, 0)); - NUTS_PASS( - nng_listener_get_int(l3, NNG_OPT_TCP_BOUND_PORT, &port3)); - NUTS_TRUE(port3 != 0); - NUTS_TRUE(port3 != port1); - - // Let's make sure can dial to each. - snprintf(ws_url, sizeof(ws_url), "ws://127.0.0.1:%d/one", port1); - NUTS_PASS(nng_dial(s4, ws_url, NULL, 0)); - snprintf(ws_url, sizeof(ws_url), "ws://127.0.0.1:%d/two", port2); - NUTS_PASS(nng_dial(s6, ws_url, NULL, 0)); - snprintf(ws_url, sizeof(ws_url), "ws://127.0.0.1:%d/three", port3); - NUTS_PASS(nng_dial(s6, ws_url, NULL, 0)); - - NUTS_CLOSE(s1); - NUTS_CLOSE(s2); - NUTS_CLOSE(s3); - NUTS_CLOSE(s4); - NUTS_CLOSE(s5); - NUTS_CLOSE(s6); -} - -static void -test_wild_card_host(void) -{ - nng_socket s1; - nng_socket s2; - char addr[NNG_MAXADDRLEN]; - uint16_t port; - - NUTS_OPEN(s1); - NUTS_OPEN(s2); - - port = nuts_next_port(); - - // we use ws4 to ensure 127.0.0.1 binding - snprintf(addr, sizeof(addr), "ws4://*:%u/test", port); - NUTS_PASS(nng_listen(s1, addr, NULL, 0)); - nng_msleep(100); - - snprintf(addr, sizeof(addr), "ws://127.0.0.1:%u/test", port); - NUTS_PASS(nng_dial(s2, addr, NULL, 0)); - - NUTS_CLOSE(s1); - NUTS_CLOSE(s2); -} - -static void -test_empty_host(void) -{ - nng_socket s1; - nng_socket s2; - char addr[NNG_MAXADDRLEN]; - uint16_t port; - - NUTS_OPEN(s1); - NUTS_OPEN(s2); - - port = nuts_next_port(); - - // we use ws4 to ensure 127.0.0.1 binding - snprintf(addr, sizeof(addr), "ws4://:%u/test", port); - NUTS_PASS(nng_listen(s1, addr, NULL, 0)); - nng_msleep(100); - - snprintf(addr, sizeof(addr), "ws://127.0.0.1:%u/test", port); - NUTS_PASS(nng_dial(s2, addr, NULL, 0)); - - NUTS_CLOSE(s1); - NUTS_CLOSE(s2); -} - -void -test_ws_recv_max(void) -{ - char msg[256]; - char buf[256]; - nng_socket s0; - nng_socket s1; - nng_listener l; - size_t sz; - char *addr; - - NUTS_ADDR(addr, "ws"); - NUTS_OPEN(s0); - NUTS_PASS(nng_socket_set_ms(s0, NNG_OPT_RECVTIMEO, 100)); - NUTS_PASS(nng_socket_set_size(s0, NNG_OPT_RECVMAXSZ, 200)); - NUTS_PASS(nng_listener_create(&l, s0, addr)); - NUTS_PASS(nng_socket_get_size(s0, NNG_OPT_RECVMAXSZ, &sz)); - NUTS_TRUE(sz == 200); - NUTS_PASS(nng_listener_set_size(l, NNG_OPT_RECVMAXSZ, 100)); - NUTS_PASS(nng_listener_start(l, 0)); - - NUTS_OPEN(s1); - NUTS_PASS(nng_dial(s1, addr, NULL, 0)); - NUTS_PASS(nng_send(s1, msg, 95, 0)); - NUTS_PASS(nng_socket_set_ms(s1, NNG_OPT_SENDTIMEO, 100)); - NUTS_PASS(nng_recv(s0, buf, &sz, 0)); - NUTS_TRUE(sz == 95); - NUTS_PASS(nng_send(s1, msg, 150, 0)); - NUTS_FAIL(nng_recv(s0, buf, &sz, 0), NNG_ETIMEDOUT); - NUTS_CLOSE(s0); - NUTS_CLOSE(s1); -} - -TEST_LIST = { - { "ws url path filters", test_ws_url_path_filters }, - { "ws wild card port", test_wild_card_port }, - { "ws wild card host", test_wild_card_host }, - { "ws empty host", test_empty_host }, - { "ws recv max", test_ws_recv_max }, - { NULL, NULL }, -}; \ No newline at end of file diff --git a/src/transport/zerotier/CMakeLists.txt b/src/transport/zerotier/CMakeLists.txt deleted file mode 100644 index 903b7f56..00000000 --- a/src/transport/zerotier/CMakeLists.txt +++ /dev/null @@ -1,37 +0,0 @@ -# -# Copyright 2020 Staysail Systems, Inc. -# Copyright 2018 Capitar IT Group BV -# -# This software is supplied under the terms of the MIT License, a -# copy of which should be located in the distribution where this -# file was obtained (LICENSE.txt). A copy of the license may also be -# found online at https://opensource.org/licenses/MIT. -# - -# ZeroTier protocol - -option (NNG_TRANSPORT_ZEROTIER "Enable ZeroTier transport (requires libzerotiercore)." OFF) -mark_as_advanced(NNG_TRANSPORT_ZEROTIER) - -nng_directory(zerotier) - -if (NNG_TRANSPORT_ZEROTIER) - - # NB: As we wind up linking libzerotiercore.a into the application, - # this means that your application will *also* need to either be licensed - # under the GPLv3, or you will need to have a commercial license from - # ZeroTier permitting its use elsewhere. - - message(WARNING " - ************************************************************ - Linking against zerotiercore changes license terms. - Consult a lawyer and the license files for details. - ************************************************************") - - find_package(zerotiercore REQUIRED) - - nng_link_libraries(zerotiercore::zerotiercore) - nng_defines(NNG_TRANSPORT_ZEROTIER) - nng_sources(zerotier.c zthash.c) - nng_headers(nng/transport/zerotier/zerotier.h) -endif() diff --git a/src/transport/zerotier/zerotier.c b/src/transport/zerotier/zerotier.c deleted file mode 100644 index 896add29..00000000 --- a/src/transport/zerotier/zerotier.c +++ /dev/null @@ -1,3241 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// Copyright 2018 Capitar IT Group BV -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - -#include -#include -#include -#include - -#include "core/nng_impl.h" -#include "zthash.h" - -#include "nng/transport/zerotier/zerotier.h" - -#include - -// ZeroTier Transport. This sits on the ZeroTier L2 network, which itself -// is implemented on top of UDP. This requires the 3rd party -// libzerotiercore library (which is GPLv3!) and platform specific UDP -// functionality to be built in. Note that care must be taken to link -// dynamically if one wishes to avoid making your entire application GPL3. -// (Alternatively ZeroTier offers commercial licenses which may prevent -// this particular problem.) This implementation does not make use of -// certain advanced capabilities in ZeroTier such as more sophisticated -// route management and TCP fallback. You need to have connectivity -// to the Internet to use this. (Or at least to your Planetary root.) -// -// Because ZeroTier takes a while to establish connectivity, it is even -// more important that applications using the ZeroTier transport not -// assume that a connection will be immediately available. It can take -// quite a few seconds for peer-to-peer connectivity to be established. -// -// The ZeroTier transport was funded by Capitar IT Group, BV. -// -// This transport is highly experimental. - -// ZeroTier and UDP are connectionless, but nng is designed around -// connection oriented paradigms. An "unreliable" connection is created -// on top using our own network protocol. The details of this are -// documented in the RFC. - -// Every participant has an "address", which is a 64-bit value constructed -// using the ZT node number in the upper 40-bits, and a 24-bit port number -// in the lower bits. We elect to operate primarily on these addresses, -// but the wire protocol relies on just conveying the 24-bit port along -// with the MAC address (from which the ZT node number can be derived, -// given the network ID.) - -typedef struct zt_pipe zt_pipe; -typedef struct zt_ep zt_ep; -typedef struct zt_node zt_node; -typedef struct zt_frag zt_frag; -typedef struct zt_fraglist zt_fraglist; - -// Port numbers are stored as 24-bit values in network byte order. -#define ZT_GET24(ptr, v) \ - v = (((uint32_t)((uint8_t)(ptr)[0])) << 16) + \ - (((uint32_t)((uint8_t)(ptr)[1])) << 8) + \ - (((uint32_t)(uint8_t)(ptr)[2])) - -#define ZT_PUT24(ptr, u) \ - do { \ - (ptr)[0] = (uint8_t)(((uint32_t)(u)) >> 16); \ - (ptr)[1] = (uint8_t)(((uint32_t)(u)) >> 8); \ - (ptr)[2] = (uint8_t)((uint32_t)(u)); \ - } while (0) - -static const uint16_t zt_ethertype = 0x901; -static const uint8_t zt_version = 0x01; -static const uint32_t zt_ephemeral = 0x800000u; // start of ephemeral ports -static const uint32_t zt_max_port = 0xffffffu; // largest port -static const uint32_t zt_port_mask = 0xffffffu; // mask of valid ports -static const uint32_t zt_port_shift = 24; -static const int zt_conn_tries = 240; // max connect attempts -static const nng_duration zt_conn_time = 500; // between attempts (msec) -static const int zt_ping_tries = 10; // max keepalive attempts -static const nng_duration zt_ping_time = 60000; // keepalive time (msec) - -// These are compile time tunables for now. -enum zt_tunables { - zt_listenq = 128, // backlog queue length - zt_listen_expire = 10000, // maximum time in backlog (msec) - zt_rcv_bufsize = 4096, // max UDP recv - zt_udp_sendq = 16, // outgoing UDP queue length - zt_recvq = 2, // max pending recv (per pipe) - zt_recv_stale = 1000, // frags older than are stale (msec) -}; - -enum zt_op_codes { - zt_op_data = 0x00, // data, final fragment - zt_op_conn_req = 0x10, // connect request - zt_op_conn_ack = 0x12, // connect accepted - zt_op_disc_req = 0x20, // disconnect request (no ack) - zt_op_ping = 0x30, // ping request - zt_op_pong = 0x32, // ping response - zt_op_error = 0x40, // error response -}; - -enum zt_offsets { - zt_offset_op = 0x00, - zt_offset_flags = 0x01, - zt_offset_version = 0x02, // protocol version number (2 bytes) - zt_offset_zero1 = 0x04, // reserved, must be zero (1 byte) - zt_offset_dst_port = 0x05, // destination port (3 bytes) - zt_offset_zero2 = 0x08, // reserved, must be zero (1 byte) - zt_offset_src_port = 0x09, // source port number (3 bytes) - zt_offset_creq_proto = 0x0C, // SP protocol number (2 bytes) - zt_offset_cack_proto = 0x0C, // SP protocol number (2 bytes) - zt_offset_err_code = 0x0C, // error code (1 byte) - zt_offset_err_msg = 0x0D, // error message (string) - zt_offset_data_id = 0x0C, // message ID (2 bytes) - zt_offset_data_fragsz = 0x0E, // fragment size - zt_offset_data_frag = 0x10, // fragment number, first is 1 (2 bytes) - zt_offset_data_nfrag = 0x12, // total fragments (2 bytes) - zt_offset_data_data = 0x14, // user payload - zt_size_headers = 0x0C, // size of headers - zt_size_conn_req = 0x0E, // size of conn_req (connect request) - zt_size_conn_ack = 0x0E, // size of conn_ack (connect reply) - zt_size_disc_req = 0x0C, // size of disc_req (disconnect) - zt_size_ping = 0x0C, // size of ping request - zt_size_pong = 0x0C, // size of ping reply - zt_size_data = 0x14, // size of data message (w/o payload) -}; - -enum zt_errors { - zt_err_refused = 0x01, // Connection refused - zt_err_notconn = 0x02, // Connection does not exit - zt_err_wrongsp = 0x03, // SP protocol mismatch - zt_err_proto = 0x04, // Other protocol error - zt_err_msgsize = 0x05, // Message to large - zt_err_unknown = 0x06, // Other errors -}; - -// This node structure is wrapped around the ZT_node; this allows us to -// have multiple endpoints referencing the same ZT_node, but also to -// support different nodes (identities) based on different home dirs. -// This means we need to stick these on a global linked list, manage -// them with a reference count, and uniquely identify them using the -// homedir. -struct zt_node { - char zn_path[NNG_MAXADDRLEN]; // ought to be sufficient - nni_file_lockh *zn_flock; - ZT_Node * zn_znode; - uint64_t zn_self; - nni_list_node zn_link; - bool zn_closed; - nni_plat_udp * zn_udp4; - nni_plat_udp * zn_udp6; - nni_list zn_eplist; - nni_list zn_plist; - zt_hash * zn_ports; - zt_hash * zn_eps; - zt_hash * zn_lpipes; - zt_hash * zn_rpipes; - nni_aio * zn_rcv4_aio; - uint8_t * zn_rcv4_buf; - nng_sockaddr zn_rcv4_addr; - nni_aio * zn_rcv6_aio; - uint8_t * zn_rcv6_buf; - nng_sockaddr zn_rcv6_addr; - nni_thr zn_bgthr; - int64_t zn_bgtime; - nni_cv zn_bgcv; - nni_cv zn_snd6_cv; -}; - -// The fragment list is used to keep track of incoming received -// fragments for reassembly into a complete message. -struct zt_fraglist { - nni_time fl_time; // time first frag was received - uint32_t fl_msgid; // message id - int fl_ready; // we have all messages - size_t fl_fragsz; - unsigned int fl_nfrags; - uint8_t * fl_missing; - size_t fl_missingsz; - nni_msg * fl_msg; -}; - -struct zt_pipe { - nni_list_node zp_link; - zt_node * zp_ztn; - nni_pipe * zp_npipe; - uint64_t zp_nwid; - uint64_t zp_laddr; - uint64_t zp_raddr; - uint16_t zp_peer; - uint16_t zp_proto; - uint16_t zp_next_msgid; - size_t zp_rcvmax; - size_t zp_mtu; - nni_aio * zp_user_rxaio; - nni_time zp_last_recv; - zt_fraglist zp_recvq[zt_recvq]; - int zp_ping_try; - int zp_ping_tries; - bool zp_closed; - nni_duration zp_ping_time; - nni_aio * zp_ping_aio; - uint8_t * zp_send_buf; - nni_atomic_flag zp_reaped; - nni_reap_item zp_reap; -}; - -typedef struct zt_creq zt_creq; -struct zt_creq { - uint64_t cr_expire; - uint64_t cr_raddr; - uint16_t cr_proto; -}; - -struct zt_ep { - nni_list_node ze_link; - char ze_home[NNG_MAXADDRLEN]; // should be enough - zt_node * ze_ztn; - uint64_t ze_nwid; - bool ze_running; - uint64_t ze_raddr; // remote node address - uint64_t ze_laddr; // local node address - uint16_t ze_proto; - size_t ze_rcvmax; - nni_aio * ze_aio; - nni_aio * ze_creq_aio; - bool ze_creq_active; - int ze_creq_try; - nni_list ze_aios; - int ze_mtu; - int ze_ping_tries; - nni_duration ze_ping_time; - nni_duration ze_conn_time; - int ze_conn_tries; - - // Incoming connection requests (server only). We only - // only have "accepted" requests -- that is we won't have an - // established connection/pipe unless the application calls - // accept. Since the "application" is our library, that should - // be pretty much as fast we can run. - zt_creq ze_creqs[zt_listenq]; - int ze_creq_head; - int ze_creq_tail; - nni_dialer * ze_ndialer; - nni_listener *ze_nlistener; -}; - -// Locking strategy. At present the ZeroTier core is not reentrant or fully -// threadsafe. (We expect this will be fixed.) Furthermore, there are -// some significant challenges in dealing with locks associated with the -// callbacks, etc. So we take a big-hammer approach, and just use a single -// global lock for everything. We hold this lock when calling into the -// ZeroTier framework. Since ZeroTier has no independent threads, that -// means that it will always hold this lock in its core, and the lock will -// also be held automatically in any of our callbacks. We never hold any -// other locks across ZeroTier core calls. We may not acquire the global -// lock in callbacks (they will already have it held). Any other locks -// can be acquired as long as they are not held during calls into ZeroTier. -// -// This will have a detrimental impact on performance, but to be completely -// honest we don't think anyone will be using the ZeroTier transport in -// performance critical applications; scalability may become a factor for -// large servers sitting in a ZeroTier hub situation. (Then again, since -// only the zerotier processing is single threaded, it may not -// be that much of a bottleneck -- really depends on how expensive these -// operations are. We can use lockstat or other lock-hotness tools to -// check for this later.) - -static nni_mtx zt_lk; -static nni_list zt_nodes; - -static void zt_ep_send_conn_req(zt_ep *); -static void zt_ep_conn_req_cb(void *); -static void zt_ep_doaccept(zt_ep *); -static void zt_pipe_dorecv(zt_pipe *); -static int zt_pipe_alloc(zt_pipe **, zt_ep *, uint64_t, uint64_t, bool); -static void zt_pipe_ping_cb(void *); -static void zt_fraglist_clear(zt_fraglist *); -static void zt_fraglist_free(zt_fraglist *); -static void zt_virtual_recv(ZT_Node *, void *, void *, uint64_t, void **, - uint64_t, uint64_t, unsigned int, unsigned int, const void *, - unsigned int); -static void zt_pipe_start_ping(zt_pipe *); - -static int64_t -zt_now(void) -{ - // We return msec - return ((int64_t) nni_clock()); -} - -static void -zt_bgthr(void *arg) -{ - zt_node *ztn = arg; - int64_t now; - - nni_mtx_lock(&zt_lk); - for (;;) { - now = zt_now(); - - if (ztn->zn_closed) { - break; - } - - if (now < ztn->zn_bgtime) { - nni_cv_until(&ztn->zn_bgcv, (nni_time) ztn->zn_bgtime); - continue; - } - - ztn->zn_bgtime = 0; - ZT_Node_processBackgroundTasks(ztn->zn_znode, NULL, now, &now); - - ztn->zn_bgtime = now; - } - nni_mtx_unlock(&zt_lk); -} - -static void -zt_node_resched(zt_node *ztn, int64_t msec) -{ - if (msec > ztn->zn_bgtime && ztn->zn_bgtime != 0) { - return; - } - ztn->zn_bgtime = msec; - nni_cv_wake1(&ztn->zn_bgcv); -} - -static void -zt_node_rcv4_cb(void *arg) -{ - zt_node * ztn = arg; - nni_aio * aio = ztn->zn_rcv4_aio; - struct sockaddr_storage sa; - struct sockaddr_in * sin; - nng_sockaddr_in * nsin; - int64_t now; - - if (nni_aio_result(aio) != 0) { - // Outside of memory exhaustion, we can't really think - // of any reason for this to legitimately fail. - // Arguably we should inject a fallback delay, but for - // now we just carry on. - return; - } - - memset(&sa, 0, sizeof(sa)); - sin = (void *) &sa; - nsin = &ztn->zn_rcv4_addr.s_in; - sin->sin_family = AF_INET; - sin->sin_port = nsin->sa_port; - sin->sin_addr.s_addr = nsin->sa_addr; - - nni_mtx_lock(&zt_lk); - now = zt_now(); - - // We are not going to perform any validation of the data; we - // just pass this straight into the ZeroTier core. - // XXX: CHECK THIS, if it fails then we have a fatal error with - // the znode, and have to shut everything down. - ZT_Node_processWirePacket(ztn->zn_znode, NULL, now, 0, (void *) &sa, - ztn->zn_rcv4_buf, nni_aio_count(aio), &now); - - // Schedule background work - zt_node_resched(ztn, now); - - // Schedule another receive. - if (ztn->zn_udp4 != NULL) { - nni_iov iov; - iov.iov_buf = ztn->zn_rcv4_buf; - iov.iov_len = zt_rcv_bufsize; - nni_aio_set_iov(aio, 1, &iov); - - nni_aio_set_input(aio, 0, &ztn->zn_rcv4_addr); - - nni_plat_udp_recv(ztn->zn_udp4, aio); - } - nni_mtx_unlock(&zt_lk); -} - -static void -zt_node_rcv6_cb(void *arg) -{ - zt_node * ztn = arg; - nni_aio * aio = ztn->zn_rcv6_aio; - struct sockaddr_storage sa; - struct sockaddr_in6 * sin6; - struct nng_sockaddr_in6 *nsin6; - int64_t now; - - if (nni_aio_result(aio) != 0) { - // Outside of memory exhaustion, we can't really think - // of any reason for this to legitimately fail. - // Arguably we should inject a fallback delay, but for - // now we just carry on. - return; - } - - memset(&sa, 0, sizeof(sa)); - sin6 = (void *) &sa; - nsin6 = &ztn->zn_rcv6_addr.s_in6; - sin6->sin6_family = AF_INET6; - sin6->sin6_port = nsin6->sa_port; - memcpy(&sin6->sin6_addr, nsin6->sa_addr, 16); - - nni_mtx_lock(&zt_lk); - now = (uint64_t) zt_now(); // msec - - // We are not going to perform any validation of the data; we - // just pass this straight into the ZeroTier core. - ZT_Node_processWirePacket(ztn->zn_znode, NULL, now, 0, (void *) &sa, - ztn->zn_rcv6_buf, nni_aio_count(aio), &now); - - // Schedule background work - zt_node_resched(ztn, now); - - // Schedule another receive. - if (ztn->zn_udp6 != NULL) { - nni_iov iov; - iov.iov_buf = ztn->zn_rcv6_buf; - iov.iov_len = zt_rcv_bufsize; - nni_aio_set_iov(aio, 1, &iov); - nni_aio_set_input(aio, 0, &ztn->zn_rcv6_addr); - nni_plat_udp_recv(ztn->zn_udp6, aio); - } - nni_mtx_unlock(&zt_lk); -} - -static uint64_t -zt_mac_to_node(uint64_t mac, uint64_t nwid) -{ - uint64_t node; - // This extracts a node address from a mac address. The - // network ID is mixed in, and has to be extricated. We - // the node ID is located in the lower 40 bits, and scrambled - // against the nwid. - node = mac & 0xffffffffffull; - node ^= ((nwid >> 8) & 0xff) << 32; - node ^= ((nwid >> 16) & 0xff) << 24; - node ^= ((nwid >> 24) & 0xff) << 16; - node ^= ((nwid >> 32) & 0xff) << 8; - node ^= (nwid >> 40) & 0xff; - return (node); -} - -static uint64_t -zt_node_to_mac(uint64_t node, uint64_t nwid) -{ - uint64_t mac; - // We use LSB of network ID, and make sure that we clear - // multicast and set local administration -- this is the first - // octet of the 48 bit mac address. We also avoid 0x52, which - // is known to be used in KVM, libvirt, etc. - mac = ((uint8_t)(nwid & 0xfe) | 0x02); - if (mac == 0x52) { - mac = 0x32; - } - mac <<= 40; - mac |= node; - // The rest of the network ID is XOR'd in, in reverse byte - // order. - mac ^= ((nwid >> 8) & 0xff) << 32; - mac ^= ((nwid >> 16) & 0xff) << 24; - mac ^= ((nwid >> 24) & 0xff) << 16; - mac ^= ((nwid >> 32) & 0xff) << 8; - mac ^= (nwid >> 40) & 0xff; - return (mac); -} - -static int -zt_result(enum ZT_ResultCode rv) -{ - switch (rv) { - case ZT_RESULT_OK: - return (0); - case ZT_RESULT_OK_IGNORED: - return (0); - case ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY: - return (NNG_ENOMEM); - case ZT_RESULT_FATAL_ERROR_DATA_STORE_FAILED: - return (NNG_EPERM); - case ZT_RESULT_FATAL_ERROR_INTERNAL: - return (NNG_EINTERNAL); - case ZT_RESULT_ERROR_NETWORK_NOT_FOUND: - return (NNG_EADDRINVAL); - case ZT_RESULT_ERROR_UNSUPPORTED_OPERATION: - return (NNG_ENOTSUP); - case ZT_RESULT_ERROR_BAD_PARAMETER: - return (NNG_EINVAL); - default: - return (NNG_ETRANERR + (int) rv); - } -} - -// ZeroTier Node API callbacks -static int -zt_virtual_config(ZT_Node *node, void *userptr, void *thr, uint64_t nwid, - void **netptr, enum ZT_VirtualNetworkConfigOperation op, - const ZT_VirtualNetworkConfig *config) -{ - zt_node *ztn = userptr; - zt_ep * ep; - - NNI_ARG_UNUSED(thr); - NNI_ARG_UNUSED(netptr); - - NNI_ASSERT(node == ztn->zn_znode); - - // Maybe we don't have to create taps or anything like that. - // We do get our mac and MTUs from this, so there's that. - switch (op) { - case ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_UP: - case ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_CONFIG_UPDATE: - - // We only really care about changes to the MTU. From - // an API perspective the MAC could change, but that - // cannot really happen because the node identity and - // the nwid are fixed. - NNI_LIST_FOREACH (&ztn->zn_eplist, ep) { - NNI_ASSERT(nwid == config->nwid); - if (ep->ze_nwid != config->nwid) { - continue; - } - ep->ze_mtu = config->mtu; - } - break; - case ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY: - case ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DOWN: - // XXX: tear down endpoints? - default: - break; - } - return (0); -} - -// zt_send modifies the start of the supplied buffer to update the -// message headers with protocol specific details (version, port numbers, -// etc.) and then sends it over the virtual network. -static void -zt_send(zt_node *ztn, uint64_t nwid, uint8_t op, uint64_t raddr, - uint64_t laddr, uint8_t *data, size_t len) -{ - uint64_t srcmac = zt_node_to_mac(laddr >> 24, nwid); - uint64_t dstmac = zt_node_to_mac(raddr >> 24, nwid); - int64_t now = zt_now(); - - NNI_ASSERT(len >= zt_size_headers); - data[zt_offset_op] = op; - data[zt_offset_flags] = 0; - data[zt_offset_zero1] = 0; - data[zt_offset_zero2] = 0; - NNI_PUT16(data + zt_offset_version, zt_version); - ZT_PUT24(data + zt_offset_dst_port, raddr & zt_port_mask); - ZT_PUT24(data + zt_offset_src_port, laddr & zt_port_mask); - - (void) ZT_Node_processVirtualNetworkFrame(ztn->zn_znode, NULL, now, - nwid, srcmac, dstmac, zt_ethertype, 0, data, len, &now); - - zt_node_resched(ztn, now); -} - -static void -zt_send_err(zt_node *ztn, uint64_t nwid, uint64_t raddr, uint64_t laddr, - uint8_t err, const char *msg) -{ - uint8_t data[128]; - - NNI_ASSERT((strlen(msg) + zt_offset_err_msg) < sizeof(data)); - - data[zt_offset_err_code] = err; - nni_strlcpy((char *) data + zt_offset_err_msg, msg, - sizeof(data) - zt_offset_err_msg); - - zt_send(ztn, nwid, zt_op_error, raddr, laddr, data, - strlen(msg) + zt_offset_err_msg); -} - -static void -zt_pipe_send_err(zt_pipe *p, uint8_t err, const char *msg) -{ - zt_send_err(p->zp_ztn, p->zp_nwid, p->zp_raddr, p->zp_laddr, err, msg); -} - -static void -zt_pipe_send_disc_req(zt_pipe *p) -{ - uint8_t data[zt_size_disc_req]; - - zt_send(p->zp_ztn, p->zp_nwid, zt_op_disc_req, p->zp_raddr, - p->zp_laddr, data, sizeof(data)); -} - -static void -zt_pipe_send_ping(zt_pipe *p) -{ - uint8_t data[zt_size_ping]; - - zt_send(p->zp_ztn, p->zp_nwid, zt_op_ping, p->zp_raddr, p->zp_laddr, - data, sizeof(data)); -} - -static void -zt_pipe_send_pong(zt_pipe *p) -{ - uint8_t data[zt_size_ping]; - - zt_send(p->zp_ztn, p->zp_nwid, zt_op_pong, p->zp_raddr, p->zp_laddr, - data, sizeof(data)); -} - -static void -zt_pipe_send_conn_ack(zt_pipe *p) -{ - uint8_t data[zt_size_conn_ack]; - - NNI_PUT16(data + zt_offset_cack_proto, p->zp_proto); - zt_send(p->zp_ztn, p->zp_nwid, zt_op_conn_ack, p->zp_raddr, - p->zp_laddr, data, sizeof(data)); -} - -static void -zt_ep_send_conn_req(zt_ep *ep) -{ - uint8_t data[zt_size_conn_req]; - - NNI_PUT16(data + zt_offset_creq_proto, ep->ze_proto); - zt_send(ep->ze_ztn, ep->ze_nwid, zt_op_conn_req, ep->ze_raddr, - ep->ze_laddr, data, sizeof(data)); -} - -static void -zt_ep_recv_conn_ack(zt_ep *ep, uint64_t raddr, const uint8_t *data, size_t len) -{ - zt_node *ztn = ep->ze_ztn; - nni_aio *aio = ep->ze_creq_aio; - zt_pipe *p; - int rv; - - if (ep->ze_ndialer == NULL) { - zt_send_err(ztn, ep->ze_nwid, raddr, ep->ze_laddr, - zt_err_proto, "Inappropriate operation"); - return; - } - - if (len != zt_size_conn_ack) { - zt_send_err(ztn, ep->ze_nwid, raddr, ep->ze_laddr, - zt_err_proto, "Bad message length"); - return; - } - - if (ep->ze_creq_try == 0) { - return; - } - - // Do we already have a matching pipe? If so, we can discard - // the operation. This should not happen, since we normally, - // deregister the endpoint when we create the pipe. - if ((zt_hash_find(ztn->zn_lpipes, ep->ze_laddr, (void **) &p)) == 0) { - return; - } - - if ((rv = zt_pipe_alloc(&p, ep, raddr, ep->ze_laddr, false)) != 0) { - // We couldn't create the pipe, just drop it. - nni_aio_finish_error(aio, rv); - return; - } - NNI_GET16(data + zt_offset_cack_proto, p->zp_peer); - - // Reset the address of the endpoint, so that the next call to - // ep_connect will bind a new one -- we are using this one for the - // pipe. - zt_hash_remove(ztn->zn_eps, ep->ze_laddr); - ep->ze_laddr = 0; - - nni_aio_set_output(aio, 0, p); - nni_aio_finish(aio, 0, 0); -} - -static void -zt_ep_recv_conn_req(zt_ep *ep, uint64_t raddr, const uint8_t *data, size_t len) -{ - zt_node *ztn = ep->ze_ztn; - zt_pipe *p; - int i; - - if (ep->ze_nlistener == NULL) { - zt_send_err(ztn, ep->ze_nwid, raddr, ep->ze_laddr, - zt_err_proto, "Inappropriate operation"); - return; - } - if (len != zt_size_conn_req) { - zt_send_err(ztn, ep->ze_nwid, raddr, ep->ze_laddr, - zt_err_proto, "Bad message length"); - return; - } - - // If we already have created a pipe for this connection - // then just reply the conn ack. - if ((zt_hash_find(ztn->zn_rpipes, raddr, (void **) &p)) == 0) { - zt_pipe_send_conn_ack(p); - return; - } - - // We may already have a connection request queued (if this was - // a resend for example); if that's the case we just ignore - // this one. - for (i = ep->ze_creq_tail; i != ep->ze_creq_head; i++) { - if (ep->ze_creqs[i % zt_listenq].cr_raddr == raddr) { - return; - } - } - // We may already have filled our listenq, in which case we just drop. - if ((ep->ze_creq_tail + zt_listenq) == ep->ze_creq_head) { - // We have taken as many as we can, so just drop it. - return; - } - - // Record the connection request, and then process any - // pending acceptors. - i = ep->ze_creq_head % zt_listenq; - - NNI_GET16(data + zt_offset_creq_proto, ep->ze_creqs[i].cr_proto); - ep->ze_creqs[i].cr_raddr = raddr; - ep->ze_creqs[i].cr_expire = nni_clock() + zt_listen_expire; - ep->ze_creq_head++; - - zt_ep_doaccept(ep); -} - -static void -zt_ep_recv_error(zt_ep *ep, const uint8_t *data, size_t len) -{ - int code; - - // Most of the time we don't care about errors. The exception here - // is that when we have an outstanding CON_REQ, we would like to - // process that appropriately. - - if (ep->ze_ndialer == NULL) { - // Not a dialer. Drop it. - return; - } - - if (len < zt_offset_err_msg) { - // Malformed error frame. - return; - } - - code = data[zt_offset_err_code]; - switch (code) { - case zt_err_refused: - code = NNG_ECONNREFUSED; - break; - case zt_err_notconn: - code = NNG_ECLOSED; - break; - case zt_err_wrongsp: - code = NNG_EPROTO; - break; - default: - code = NNG_ETRANERR; - break; - } - - if (ep->ze_creq_active) { - ep->ze_creq_try = 0; - ep->ze_creq_active = 0; - nni_aio_finish_error(ep->ze_creq_aio, code); - } -} - -static void -zt_ep_virtual_recv( - zt_ep *ep, uint8_t op, uint64_t raddr, const uint8_t *data, size_t len) -{ - // Only listeners should be receiving. Dialers receive on the pipe, - // rather than the endpoint. The only message that endpoints can - // receive are connection requests. - switch (op) { - case zt_op_conn_req: - zt_ep_recv_conn_req(ep, raddr, data, len); - return; - case zt_op_conn_ack: - zt_ep_recv_conn_ack(ep, raddr, data, len); - return; - case zt_op_error: - zt_ep_recv_error(ep, data, len); - return; - default: - zt_send_err(ep->ze_ztn, ep->ze_nwid, raddr, ep->ze_laddr, - zt_err_proto, "Bad operation"); - return; - } -} - -static void -zt_pipe_close_err(zt_pipe *p, int err, uint8_t code, const char *msg) -{ - nni_aio *aio; - if ((aio = p->zp_user_rxaio) != NULL) { - p->zp_user_rxaio = NULL; - nni_aio_finish_error(aio, err); - } - nni_aio_close(p->zp_ping_aio); - p->zp_closed = true; - if (msg != NULL) { - zt_pipe_send_err(p, code, msg); - } -} - -static void -zt_pipe_recv_data(zt_pipe *p, const uint8_t *data, size_t len) -{ - uint16_t msgid; - uint16_t fragno; - uint16_t nfrags; - size_t fragsz; - zt_fraglist *fl; - int i; - int slot; - uint8_t bit; - uint8_t * body; - - if (len < zt_size_data) { - // Runt frame. Drop it and close pipe with a protocol error. - zt_pipe_close_err(p, NNG_EPROTO, zt_err_proto, "Runt frame"); - return; - } - - NNI_GET16(data + zt_offset_data_id, msgid); - NNI_GET16(data + zt_offset_data_fragsz, fragsz); - NNI_GET16(data + zt_offset_data_frag, fragno); - NNI_GET16(data + zt_offset_data_nfrag, nfrags); - len -= zt_offset_data_data; - data += zt_offset_data_data; - - // Check for cases where message size is clearly too large. Note - // that we only can catch the case where a message is larger by - // more than a fragment, since the final fragment may be shorter, - // and we won't know that until we receive it. - if ((p->zp_rcvmax > 0) && - ((nfrags * fragsz) >= (p->zp_rcvmax + fragsz))) { - // Discard, as the forwarder might be on the other side - // of a device. This is gentler than just shutting the pipe - // down. Sending a remote error might be polite, but since - // most peers will close the pipe on such an error, we - // simply silently discard it. - return; - } - - // We run the recv logic once, to clear stale fragment entries. - zt_pipe_dorecv(p); - - // Find a suitable fragment slot. - slot = -1; - for (i = 0; i < zt_recvq; i++) { - fl = &p->zp_recvq[i]; - // This was our message ID, we always use it. - if (msgid == fl->fl_msgid) { - slot = i; - break; - } - - if (slot < 0) { - slot = i; - } else if (fl->fl_time < p->zp_recvq[slot].fl_time) { - // This has an earlier expiration, so lets choose it. - slot = i; - } - } - - NNI_ASSERT(slot >= 0); - - fl = &p->zp_recvq[slot]; - if (fl->fl_msgid != msgid) { - // First fragment we've received for this message (but might - // not be first fragment for message!) - zt_fraglist_clear(fl); - - if (nni_msg_alloc(&fl->fl_msg, nfrags * fragsz) != 0) { - // Out of memory. We don't close the pipe, but - // just fail to receive the message. Bump a stat? - return; - } - - fl->fl_nfrags = nfrags; - fl->fl_fragsz = fragsz; - fl->fl_msgid = msgid; - fl->fl_time = nni_clock(); - - // Set the missing mask. - memset(fl->fl_missing, 0xff, nfrags / 8); - fl->fl_missing[nfrags / 8] |= ((1 << (nfrags % 8)) - 1); - } - if ((nfrags != fl->fl_nfrags) || (fragsz != fl->fl_fragsz) || - (fragno >= nfrags) || (fragsz == 0) || (nfrags == 0) || - ((fragno != (nfrags - 1)) && (len != fragsz))) { - // Protocol error, message parameters changed. - zt_pipe_close_err( - p, NNG_EPROTO, zt_err_proto, "Invalid message parameters"); - zt_fraglist_clear(fl); - return; - } - - bit = (uint8_t)(1 << (fragno % 8)); - if ((fl->fl_missing[fragno / 8] & bit) == 0) { - // We've already got this fragment, ignore it. We don't - // bother to check for changed data. - return; - } - - fl->fl_missing[fragno / 8] &= ~(bit); - body = nni_msg_body(fl->fl_msg); - body += fragno * fragsz; - memcpy(body, data, len); - if (fragno == (nfrags - 1)) { - // Last frag, maybe shorten the message. - nni_msg_chop(fl->fl_msg, (fragsz - len)); - if ((nni_msg_len(fl->fl_msg) > p->zp_rcvmax) && - (p->zp_rcvmax > 0)) { - // Strict enforcement of max recv. - zt_fraglist_clear(fl); - // Just discard the message. - return; - } - } - - for (i = 0; i < ((nfrags + 7) / 8); i++) { - if (fl->fl_missing[i]) { - return; - } - } - - // We got all fragments... try to send it up. - fl->fl_ready = 1; - zt_pipe_dorecv(p); -} - -static void -zt_pipe_recv_ping(zt_pipe *p, const uint8_t *data, size_t len) -{ - NNI_ARG_UNUSED(data); - - if (len != zt_size_ping) { - zt_pipe_send_err(p, zt_err_proto, "Incorrect ping size"); - return; - } - zt_pipe_send_pong(p); -} - -static void -zt_pipe_recv_pong(zt_pipe *p, const uint8_t *data, size_t len) -{ - NNI_ARG_UNUSED(data); - - if (len != zt_size_pong) { - zt_pipe_send_err(p, zt_err_proto, "Incorrect pong size"); - } -} - -static void -zt_pipe_recv_disc_req(zt_pipe *p, const uint8_t *data, size_t len) -{ - nni_aio *aio; - NNI_ARG_UNUSED(data); - NNI_ARG_UNUSED(len); - - // NB: lock held already. - // Don't bother to check the length, going to disconnect anyway. - if ((aio = p->zp_user_rxaio) != NULL) { - p->zp_user_rxaio = NULL; - p->zp_closed = true; - nni_aio_finish_error(aio, NNG_ECLOSED); - } -} - -static void -zt_pipe_recv_error(zt_pipe *p, const uint8_t *data, size_t len) -{ - nni_aio *aio; - NNI_ARG_UNUSED(data); - NNI_ARG_UNUSED(len); - - // Perhaps we should log an error message, but at the end of - // the day, the details are just not that interesting. - if ((aio = p->zp_user_rxaio) != NULL) { - p->zp_user_rxaio = NULL; - p->zp_closed = true; - nni_aio_finish_error(aio, NNG_ETRANERR); - } -} - -// This function is called when we have determined that a frame has -// arrived for a pipe. The remote and local addresses were both -// matched by the caller. -static void -zt_pipe_virtual_recv(zt_pipe *p, uint8_t op, const uint8_t *data, size_t len) -{ - // We got data, so update our recv time. - p->zp_last_recv = nni_clock(); - p->zp_ping_try = 0; - - switch (op) { - case zt_op_data: - zt_pipe_recv_data(p, data, len); - return; - case zt_op_disc_req: - zt_pipe_recv_disc_req(p, data, len); - return; - case zt_op_ping: - zt_pipe_recv_ping(p, data, len); - return; - case zt_op_pong: - zt_pipe_recv_pong(p, data, len); - return; - case zt_op_error: - zt_pipe_recv_error(p, data, len); - return; - case zt_op_conn_req: - zt_pipe_send_conn_ack(p); - return; - } -} - -// This function is called when a frame arrives on the -// *virtual* network. -static void -zt_virtual_recv(ZT_Node *node, void *userptr, void *thr, uint64_t nwid, - void **netptr, uint64_t srcmac, uint64_t dstmac, unsigned int ethertype, - unsigned int vlanid, const void *payload, unsigned int len) -{ - zt_node * ztn = userptr; - uint8_t op; - const uint8_t *data = payload; - uint16_t version; - uint32_t rport; - uint32_t lport; - zt_ep * ep; - zt_pipe * p; - uint64_t raddr; - uint64_t laddr; - - NNI_ARG_UNUSED(node); - NNI_ARG_UNUSED(thr); - NNI_ARG_UNUSED(netptr); - - if ((ethertype != zt_ethertype) || (len < zt_size_headers) || - (data[zt_offset_flags] != 0) || (data[zt_offset_zero1] != 0) || - (data[zt_offset_zero2] != 0)) { - return; - } - NNI_GET16(data + zt_offset_version, version); - if (version != zt_version) { - return; - } - if (vlanid != 0) { // for now we only use vlan 0. - return; - } - - op = data[zt_offset_op]; - - ZT_GET24(data + zt_offset_dst_port, lport); - ZT_GET24(data + zt_offset_src_port, rport); - - raddr = zt_mac_to_node(srcmac, nwid); - raddr <<= 24; - raddr |= rport; - - laddr = zt_mac_to_node(dstmac, nwid); - laddr <<= 24; - laddr |= lport; - - // NB: We are holding the zt_lock. - - // Look up a pipe, but also we use this chance to check that - // the source address matches what the pipe was established with. - // If the pipe does not match then we nak it. Note that pipes can - // appear on the znode twice (loopback), so we have to be careful - // to check the entire set of parameters, and to check for server - // vs. client pipes separately. - - // If its a local address match on a client pipe, process it. - if ((zt_hash_find(ztn->zn_lpipes, laddr, (void *) &p) == 0) && - (p->zp_nwid == nwid) && (p->zp_raddr == raddr)) { - zt_pipe_virtual_recv(p, op, data, len); - return; - } - - // If its a remote address match on a server pipe, process it. - if ((zt_hash_find(ztn->zn_rpipes, raddr, (void *) &p) == 0) && - (p->zp_nwid == nwid) && (p->zp_laddr == laddr)) { - zt_pipe_virtual_recv(p, op, data, len); - return; - } - - // No pipe, so look for an endpoint. - if ((zt_hash_find(ztn->zn_eps, laddr, (void **) &ep) == 0) && - (ep->ze_nwid == nwid)) { - // direct this to an endpoint. - zt_ep_virtual_recv(ep, op, raddr, data, len); - return; - } - - // We have a request for which we have no listener, and no - // pipe. For some of these we send back a NAK, but for others - // we just drop the frame. - switch (op) { - case zt_op_conn_req: - // No listener. Connection refused. - zt_send_err(ztn, nwid, raddr, laddr, zt_err_refused, - "Connection refused"); - return; - case zt_op_data: - case zt_op_ping: - case zt_op_conn_ack: - zt_send_err(ztn, nwid, raddr, laddr, zt_err_notconn, - "Connection not found"); - break; - case zt_op_error: - case zt_op_pong: - case zt_op_disc_req: - default: - // Just drop these. - break; - } -} - -static void -zt_event_cb(ZT_Node *node, void *userptr, void *thr, enum ZT_Event event, - const void *payload) -{ - NNI_ARG_UNUSED(node); - NNI_ARG_UNUSED(userptr); - NNI_ARG_UNUSED(thr); - NNI_ARG_UNUSED(payload); - - switch (event) { - case ZT_EVENT_ONLINE: // Connected to the virtual net. - case ZT_EVENT_UP: // Node initialized (may not be connected). - case ZT_EVENT_DOWN: // Teardown of the node. - case ZT_EVENT_OFFLINE: // Removal of the node from the net. - case ZT_EVENT_TRACE: // Local trace events. - // printf("TRACE: %s\n", (const char *) payload); - break; - case ZT_EVENT_REMOTE_TRACE: // Remote trace, not supported. - default: - break; - } -} - -static const char *zt_files[] = { - // clang-format off - NULL, // none, i.e. not used at all - "identity.public", - "identity.secret", - "planet", - "moon.%llx", - NULL, // peer, e.g. peers.d/ -- we don't persist this - "network.%llx", - // clang-format on -}; - -static struct { - size_t len; - void * data; -} zt_ephemeral_state[ZT_STATE_OBJECT_NETWORK_CONFIG + 1]; - -static void -zt_state_put(ZT_Node *node, void *userptr, void *thr, - enum ZT_StateObjectType objtype, const uint64_t objid[2], const void *data, - int len) -{ - zt_node *ztn = userptr; - char * path; - const char *template; - char fname[32]; - - NNI_ARG_UNUSED(node); - NNI_ARG_UNUSED(thr); - NNI_ARG_UNUSED(objid); // only use global files - - if ((objtype > ZT_STATE_OBJECT_NETWORK_CONFIG) || - ((template = zt_files[(int) objtype]) == NULL)) { - return; - } - - (void) snprintf(fname, sizeof(fname), template, - (unsigned long long) objid[0], (unsigned long long) objid[1]); - - // If we have no valid path, then we just use ephemeral data. - // Note that for moons, and so forth, we wind up just storing them - // all in the same place, but it does not matter since we don't - // really persist them anyway. - if (strlen(ztn->zn_path) == 0) { - void * ndata = NULL; - void * odata = zt_ephemeral_state[objtype].data; - size_t olen = zt_ephemeral_state[objtype].len; - if ((len >= 0) && ((ndata = nni_alloc(len)) != NULL)) { - memcpy(ndata, data, len); - zt_ephemeral_state[objtype].data = ndata; - zt_ephemeral_state[objtype].len = len; - } else if (len < 0) { - zt_ephemeral_state[objtype].data = NULL; - zt_ephemeral_state[objtype].len = 0; - } - - if (olen > 0) { - nni_free(odata, olen); - } - return; - } - - if ((path = nni_file_join(ztn->zn_path, fname)) == NULL) { - return; - } - - if (len < 0) { - (void) nni_file_delete(path); - } else { - (void) nni_file_put(path, data, len); - } - nni_strfree(path); -} - -static int -zt_state_get(ZT_Node *node, void *userptr, void *thr, - enum ZT_StateObjectType objtype, const uint64_t objid[2], void *data, - unsigned int len) -{ - zt_node *ztn = userptr; - char * path; - char fname[32]; - const char *template; - size_t sz; - void * buf; - - NNI_ARG_UNUSED(node); - NNI_ARG_UNUSED(thr); - NNI_ARG_UNUSED(objid); // we only use global files - - if ((objtype > ZT_STATE_OBJECT_NETWORK_CONFIG) || - ((template = zt_files[(int) objtype]) == NULL)) { - return (-1); - } - snprintf(fname, sizeof(fname), template, objid[0], objid[1]); - - // If no base directory, we are using ephemeral data. - if (strlen(ztn->zn_path) == 0) { - if (zt_ephemeral_state[objtype].data == NULL) { - return (-1); - } - if (zt_ephemeral_state[objtype].len > len) { - return (-1); - } - len = zt_ephemeral_state[objtype].len; - memcpy(data, zt_ephemeral_state[objtype].data, len); - return (len); - } - - if ((path = nni_file_join(ztn->zn_path, fname)) == NULL) { - return (-1); - } - - if (nni_file_get(path, &buf, &sz) != 0) { - nni_strfree(path); - return (-1); - } - nni_strfree(path); - if (sz > len) { - nni_free(buf, sz); - return (-1); - } - memcpy(data, buf, sz); - nni_free(buf, sz); - return ((int) sz); -} - -typedef struct zt_send_hdr { - nni_sockaddr sa; - size_t len; -} zt_send_hdr; - -// This function is called when ZeroTier desires to send a -// physical frame. The data is a UDP payload, the rest of the -// payload should be set over vanilla UDP. -static int -zt_wire_packet_send(ZT_Node *node, void *userptr, void *thr, int64_t socket, - const struct sockaddr_storage *remaddr, const void *data, unsigned int len, - unsigned int ttl) -{ - nni_aio * aio; - nni_sockaddr addr; - struct sockaddr_in * sin = (void *) remaddr; - struct sockaddr_in6 *sin6 = (void *) remaddr; - zt_node * ztn = userptr; - nni_plat_udp * udp; - uint8_t * buf; - zt_send_hdr * hdr; - nni_iov iov; - - NNI_ARG_UNUSED(node); - NNI_ARG_UNUSED(thr); - NNI_ARG_UNUSED(socket); - NNI_ARG_UNUSED(ttl); - - // Kind of unfortunate, but we have to convert the - // sockaddr to a neutral form, and then back again in - // the platform layer. - switch (sin->sin_family) { - case AF_INET: - addr.s_in.sa_family = NNG_AF_INET; - addr.s_in.sa_port = sin->sin_port; - addr.s_in.sa_addr = sin->sin_addr.s_addr; - udp = ztn->zn_udp4; - break; - case AF_INET6: - addr.s_in6.sa_family = NNG_AF_INET6; - addr.s_in6.sa_port = sin6->sin6_port; - udp = ztn->zn_udp6; - memcpy(addr.s_in6.sa_addr, sin6->sin6_addr.s6_addr, 16); - break; - default: - // No way to understand the address. - return (-1); - } - - if (nni_aio_alloc(&aio, NULL, NULL) != 0) { - // Out of memory - return (-1); - } - if ((buf = nni_alloc(sizeof(*hdr) + len)) == NULL) { - nni_aio_free(aio); - return (-1); - } - - hdr = (void *) buf; - buf += sizeof(*hdr); - - memcpy(buf, data, len); - nni_aio_set_data(aio, 0, hdr); - hdr->sa = addr; - hdr->len = len; - nni_aio_set_input(aio, 0, &hdr->sa); - - iov.iov_buf = buf; - iov.iov_len = len; - nni_aio_set_iov(aio, 1, &iov); - - // This should be non-blocking/best-effort, so while - // not great that we're holding the lock, also not tragic. - nni_plat_udp_send(udp, aio); - - // UDP sending is "fast" on all platforms -- given that its - // best effort only, this will complete immediately, resulting - // in either a message on the wire, or a discarded frame. We don't - // care which. (There may be a few thread context switches, but - // none of them are going to have to wait for some unbounded time.) - nni_aio_wait(aio); - nni_aio_free(aio); - nni_free(hdr, hdr->len + sizeof(*hdr)); - - return (0); -} - -static struct ZT_Node_Callbacks zt_callbacks = { - .version = 0, - .statePutFunction = zt_state_put, - .stateGetFunction = zt_state_get, - .wirePacketSendFunction = zt_wire_packet_send, - .virtualNetworkFrameFunction = zt_virtual_recv, - .virtualNetworkConfigFunction = zt_virtual_config, - .eventCallback = zt_event_cb, - .pathCheckFunction = NULL, - .pathLookupFunction = NULL, -}; - -static void -zt_node_destroy(zt_node *ztn) -{ - nni_aio_stop(ztn->zn_rcv4_aio); - nni_aio_stop(ztn->zn_rcv6_aio); - - // Wait for background thread to exit! - nni_thr_fini(&ztn->zn_bgthr); - - if (ztn->zn_znode != NULL) { - ZT_Node_delete(ztn->zn_znode); - } - - if (ztn->zn_udp4 != NULL) { - nni_plat_udp_close(ztn->zn_udp4); - } - if (ztn->zn_udp6 != NULL) { - nni_plat_udp_close(ztn->zn_udp6); - } - - if (ztn->zn_rcv4_buf != NULL) { - nni_free(ztn->zn_rcv4_buf, zt_rcv_bufsize); - } - if (ztn->zn_rcv6_buf != NULL) { - nni_free(ztn->zn_rcv6_buf, zt_rcv_bufsize); - } - if (ztn->zn_flock != NULL) { - nni_file_unlock(ztn->zn_flock); - } - nni_aio_free(ztn->zn_rcv4_aio); - nni_aio_free(ztn->zn_rcv6_aio); - zt_hash_fini(ztn->zn_eps); - zt_hash_fini(ztn->zn_lpipes); - zt_hash_fini(ztn->zn_rpipes); - nni_cv_fini(&ztn->zn_bgcv); - NNI_FREE_STRUCT(ztn); -} - -static int -zt_node_create(zt_node **ztnp, const char *path) -{ - zt_node * ztn; - nng_sockaddr sa4; - nng_sockaddr sa6; - int rv; - enum ZT_ResultCode zrv; - nni_iov iov; - - // XXX: Right now we depend on having both IPv6 and IPv4 available. - // Probably we should support coping with the lack of either of them. - - // We want to bind to any address we can (for now). - memset(&sa4, 0, sizeof(sa4)); - sa4.s_in.sa_family = NNG_AF_INET; - memset(&sa6, 0, sizeof(sa6)); - sa6.s_in6.sa_family = NNG_AF_INET6; - - if ((ztn = NNI_ALLOC_STRUCT(ztn)) == NULL) { - return (NNG_ENOMEM); - } - NNI_LIST_INIT(&ztn->zn_eplist, zt_ep, ze_link); - NNI_LIST_INIT(&ztn->zn_plist, zt_pipe, zp_link); - nni_cv_init(&ztn->zn_bgcv, &zt_lk); - nni_aio_alloc(&ztn->zn_rcv4_aio, zt_node_rcv4_cb, ztn); - nni_aio_alloc(&ztn->zn_rcv6_aio, zt_node_rcv6_cb, ztn); - - if (((ztn->zn_rcv4_buf = nni_alloc(zt_rcv_bufsize)) == NULL) || - ((ztn->zn_rcv6_buf = nni_alloc(zt_rcv_bufsize)) == NULL)) { - zt_node_destroy(ztn); - return (NNG_ENOMEM); - } - if (((rv = zt_hash_init(&ztn->zn_ports)) != 0) || - ((rv = zt_hash_init(&ztn->zn_eps)) != 0) || - ((rv = zt_hash_init(&ztn->zn_lpipes)) != 0) || - ((rv = zt_hash_init(&ztn->zn_rpipes)) != 0) || - ((rv = nni_thr_init(&ztn->zn_bgthr, zt_bgthr, ztn)) != 0) || - ((rv = nni_plat_udp_open(&ztn->zn_udp4, &sa4)) != 0) || - ((rv = nni_plat_udp_open(&ztn->zn_udp6, &sa6)) != 0)) { - zt_node_destroy(ztn); - return (rv); - } - nni_thr_set_name(&ztn->zn_bgthr, "nng:zt"); - - if (strlen(path) > 0) { - char *lkfile; - if ((lkfile = nni_file_join(path, "lock")) == NULL) { - zt_node_destroy(ztn); - return (NNG_ENOMEM); - } - - if ((rv = nni_file_lock(lkfile, &ztn->zn_flock)) != 0) { - zt_node_destroy(ztn); - nni_strfree(lkfile); - return (rv); - } - nni_strfree(lkfile); - } - - // Setup for dynamic ephemeral port allocations. We - // set the range to allow for ephemeral ports, but not - // higher than the max port, and starting with an - // initial random value. Note that this should give us - // about 8 million possible ephemeral ports. - zt_hash_limits(ztn->zn_ports, zt_ephemeral, zt_max_port, - (nni_random() % (zt_max_port - zt_ephemeral)) + zt_ephemeral); - - nni_strlcpy(ztn->zn_path, path, sizeof(ztn->zn_path)); - zrv = ZT_Node_new(&ztn->zn_znode, ztn, NULL, &zt_callbacks, zt_now()); - if (zrv != ZT_RESULT_OK) { - zt_node_destroy(ztn); - return (zt_result(zrv)); - } - - nni_list_append(&zt_nodes, ztn); - - ztn->zn_self = ZT_Node_address(ztn->zn_znode); - - nni_thr_run(&ztn->zn_bgthr); - - // Schedule an initial background run. - zt_node_resched(ztn, 1); - - // Schedule receive - iov.iov_buf = ztn->zn_rcv4_buf; - iov.iov_len = zt_rcv_bufsize; - nni_aio_set_iov(ztn->zn_rcv4_aio, 1, &iov); - nni_aio_set_input(ztn->zn_rcv4_aio, 0, &ztn->zn_rcv4_addr); - iov.iov_buf = ztn->zn_rcv6_buf; - iov.iov_len = zt_rcv_bufsize; - nni_aio_set_iov(ztn->zn_rcv6_aio, 1, &iov); - nni_aio_set_input(ztn->zn_rcv6_aio, 0, &ztn->zn_rcv6_addr); - - nni_plat_udp_recv(ztn->zn_udp4, ztn->zn_rcv4_aio); - nni_plat_udp_recv(ztn->zn_udp6, ztn->zn_rcv6_aio); - - *ztnp = ztn; - return (0); -} - -static int -zt_walk_moons(const char *path, void *arg) -{ - zt_node * ztn = arg; - const char *bn = nni_file_basename(path); - char * end; - uint64_t moonid; - - if (strncmp(bn, "moon.", 5) != 0) { - return (NNI_FILE_WALK_CONTINUE); - } - if (((moonid = (uint64_t) strtoull(bn + 5, &end, 16)) != 0) && - (*end == '\0')) { - ZT_Node_orbit(ztn->zn_znode, NULL, moonid, 0); - } - return (NNI_FILE_WALK_CONTINUE); -} - -static int -zt_node_find(zt_ep *ep) -{ - zt_node * ztn; - int rv; - ZT_VirtualNetworkConfig *cf; - - NNI_LIST_FOREACH (&zt_nodes, ztn) { - if (strcmp(ep->ze_home, ztn->zn_path) == 0) { - goto done; - } - } - - // We didn't find a node, so make one. And try to - // initialize it. - if ((rv = zt_node_create(&ztn, ep->ze_home)) != 0) { - return (rv); - } - - // Load moons - if (strlen(ep->ze_home) != 0) { - (void) nni_file_walk(ep->ze_home, zt_walk_moons, ztn, - NNI_FILE_WALK_FILES_ONLY | NNI_FILE_WALK_SHALLOW); - } - -done: - - ep->ze_ztn = ztn; - if (nni_list_node_active(&ep->ze_link)) { - nni_list_node_remove(&ep->ze_link); - } - nni_list_append(&ztn->zn_eplist, ep); - - (void) ZT_Node_join(ztn->zn_znode, ep->ze_nwid, ztn, NULL); - - if ((cf = ZT_Node_networkConfig(ztn->zn_znode, ep->ze_nwid)) != NULL) { - NNI_ASSERT(cf->nwid == ep->ze_nwid); - ep->ze_mtu = cf->mtu; - ZT_Node_freeQueryResult(ztn->zn_znode, cf); - } - - return (0); -} - -static int -zt_tran_init(void) -{ - nni_mtx_init(&zt_lk); - NNI_LIST_INIT(&zt_nodes, zt_node, zn_link); - return (0); -} - -static void -zt_tran_fini(void) -{ - zt_node *ztn; - - nni_mtx_lock(&zt_lk); - while ((ztn = nni_list_first(&zt_nodes)) != 0) { - nni_list_remove(&zt_nodes, ztn); - ztn->zn_closed = true; - nni_cv_wake(&ztn->zn_bgcv); - nni_mtx_unlock(&zt_lk); - - zt_node_destroy(ztn); - - nni_mtx_lock(&zt_lk); - } - nni_mtx_unlock(&zt_lk); - - for (int i = 0; i <= ZT_STATE_OBJECT_NETWORK_CONFIG; i++) { - if (zt_ephemeral_state[i].len > 0) { - nni_free(zt_ephemeral_state[i].data, - zt_ephemeral_state[i].len); - } - } - NNI_ASSERT(nni_list_empty(&zt_nodes)); - nni_mtx_fini(&zt_lk); -} - -static int -zt_check_recvmaxsz(const void *v, size_t sz, nni_type t) -{ - return (nni_copyin_size(NULL, v, sz, 0, NNI_MAXSZ, t)); -} - -static int -zt_check_orbit(const void *v, size_t sz, nni_type t) -{ - NNI_ARG_UNUSED(v); - if ((t != NNI_TYPE_UINT64) && (t != NNI_TYPE_OPAQUE)) { - return (NNG_EBADTYPE); - } - if (sz != sizeof(uint64_t) && sz != sizeof(uint64_t) * 2) { - return (NNG_EINVAL); - } - return (0); -} - -static int -zt_check_deorbit(const void *v, size_t sz, nni_type t) -{ - return (nni_copyin_u64(NULL, v, sz, t)); -} - -static int -zt_check_string(const void *v, size_t sz, nni_type t) -{ - size_t len; - - if ((t != NNI_TYPE_OPAQUE) && (t != NNI_TYPE_STRING)) { - return (NNG_EBADTYPE); - } - len = nni_strnlen(v, sz); - if ((len >= sz) || (len >= NNG_MAXADDRLEN)) { - return (NNG_EINVAL); - } - return (0); -} - -static int -zt_check_time(const void *v, size_t sz, nni_type t) -{ - return (nni_copyin_ms(NULL, v, sz, t)); -} - -static int -zt_check_tries(const void *v, size_t sz, nni_type t) -{ - return (nni_copyin_int(NULL, v, sz, 0, 1000000, t)); -} - -static void -zt_pipe_close(void *arg) -{ - zt_pipe *p = arg; - nni_aio *aio; - - nni_mtx_lock(&zt_lk); - p->zp_closed = true; - nni_aio_close(p->zp_ping_aio); - if ((aio = p->zp_user_rxaio) != NULL) { - p->zp_user_rxaio = NULL; - nni_aio_finish_error(aio, NNG_ECLOSED); - } - zt_pipe_send_disc_req(p); - nni_mtx_unlock(&zt_lk); -} - -static int -zt_pipe_init(void *arg, nni_pipe *npipe) -{ - zt_pipe *p = arg; - p->zp_npipe = npipe; - return (0); -} - -static void -zt_pipe_fini(void *arg) -{ - zt_pipe *p = arg; - zt_node *ztn = p->zp_ztn; - - nni_aio_free(p->zp_ping_aio); - - // This tosses the connection details and all state. - nni_mtx_lock(&zt_lk); - zt_hash_remove(ztn->zn_ports, p->zp_laddr & zt_port_mask); - zt_hash_remove(ztn->zn_lpipes, p->zp_laddr); - zt_hash_remove(ztn->zn_rpipes, p->zp_raddr); - nni_mtx_unlock(&zt_lk); - - for (int i = 0; i < zt_recvq; i++) { - zt_fraglist_free(&p->zp_recvq[i]); - } - nni_free(p->zp_send_buf, ZT_MAX_MTU); - NNI_FREE_STRUCT(p); -} - -static void -zt_pipe_reap(zt_pipe *p) -{ - if (!nni_atomic_flag_test_and_set(&p->zp_reaped)) { - nni_reap(&p->zp_reap, zt_pipe_fini, p); - } -} - -static int -zt_pipe_alloc( - zt_pipe **pipep, zt_ep *ep, uint64_t raddr, uint64_t laddr, bool listener) -{ - zt_pipe *p; - int rv; - zt_node *ztn = ep->ze_ztn; - int i; - size_t maxfrag; - size_t maxfrags = 0; - - if ((p = NNI_ALLOC_STRUCT(p)) == NULL) { - return (NNG_ENOMEM); - } - if ((p->zp_send_buf = nni_alloc(ZT_MAX_MTU)) == NULL) { - NNI_FREE_STRUCT(p); - return (NNG_ENOMEM); - } - p->zp_ztn = ztn; - p->zp_raddr = raddr; - p->zp_laddr = laddr; - p->zp_proto = ep->ze_proto; - p->zp_nwid = ep->ze_nwid; - p->zp_mtu = ep->ze_mtu; - p->zp_rcvmax = ep->ze_rcvmax; - p->zp_ping_tries = ep->ze_ping_tries; - p->zp_ping_time = ep->ze_ping_time; - p->zp_next_msgid = (uint16_t) nni_random(); - p->zp_ping_try = 0; - nni_atomic_flag_reset(&p->zp_reaped); - - if (listener) { - // listener - rv = zt_hash_insert(ztn->zn_rpipes, raddr, p); - } else { - // dialer - rv = zt_hash_insert(ztn->zn_lpipes, laddr, p); - } - if ((rv != 0) || - ((rv = nni_aio_alloc(&p->zp_ping_aio, zt_pipe_ping_cb, p)) != 0)) { - zt_pipe_reap(p); - return (rv); - } - - // The largest fragment we can accept on this pipe. The MTU is - // configurable by the network administrator. Probably ZT would - // pass a larger one (up to MAXMTU), but we honor the network - // administration's configuration. - maxfrag = p->zp_mtu - zt_offset_data_data; - - // The largest fragment count we can accept on this pipe. - // This is rounded up to account for alignment. - if (p->zp_rcvmax > 0) { - maxfrags = (p->zp_rcvmax + (maxfrag - 1)) / maxfrag; - } - - if ((maxfrags > 0xffff) || (maxfrags == 0)) { - maxfrags = 0xffff; - } - - for (i = 0; i < zt_recvq; i++) { - zt_fraglist *fl = &p->zp_recvq[i]; - fl->fl_time = NNI_TIME_ZERO; - fl->fl_msgid = 0; - fl->fl_ready = 0; - fl->fl_missingsz = (maxfrags + 7) / 8; - fl->fl_missing = nni_alloc(fl->fl_missingsz); - if (fl->fl_missing == NULL) { - zt_pipe_reap(p); - return (NNG_ENOMEM); - } - } - - *pipep = p; - return (0); -} - -static void -zt_pipe_send(void *arg, nni_aio *aio) -{ - // As we are sending UDP, and there is no callback to worry - // about, we just go ahead and send out a stream of messages - // synchronously. - zt_pipe *p = arg; - uint8_t *data = p->zp_send_buf; - size_t offset; - uint16_t id; - uint16_t nfrags; - uint16_t fragno; - size_t fragsz; - size_t bytes; - nni_msg *m; - - if (nni_aio_begin(aio) != 0) { - return; - } - if ((m = nni_aio_get_msg(aio)) == NULL) { - nni_aio_finish_error(aio, NNG_EINVAL); - return; - } - - nni_mtx_lock(&zt_lk); - - if (p->zp_closed) { - nni_mtx_unlock(&zt_lk); - nni_aio_finish_error(aio, NNG_ECLOSED); - return; - } - - fragsz = p->zp_mtu - zt_offset_data_data; - NNI_ASSERT(fragsz < 0x10000); // Because zp_mtu is 16 bits - - bytes = nni_msg_header_len(m) + nni_msg_len(m); - if (bytes >= (0xfffe * fragsz)) { - nni_aio_finish_error(aio, NNG_EMSGSIZE); - nni_mtx_unlock(&zt_lk); - return; - } - // above check means nfrags will fit in 16-bits. - nfrags = (uint16_t)((bytes + (fragsz - 1)) / fragsz); - - // get the next message ID, but skip 0 - if ((id = p->zp_next_msgid++) == 0) { - id = p->zp_next_msgid++; - } - - offset = 0; - fragno = 0; - do { - uint8_t *dest = data + zt_offset_data_data; - size_t room = fragsz; - size_t fraglen = 0; - size_t len; - - // Prepend the header first. - if ((len = nni_msg_header_len(m)) > 0) { - if (len > fragsz) { - // This shouldn't happen! SP headers are - // supposed to be quite small. - nni_aio_finish_error(aio, NNG_EMSGSIZE); - nni_mtx_unlock(&zt_lk); - return; - } - memcpy(dest, nni_msg_header(m), len); - dest += len; - room -= len; - offset += len; - fraglen += len; - nni_msg_header_clear(m); - } - - len = nni_msg_len(m); - if (len > room) { - len = room; - } - memcpy(dest, nni_msg_body(m), len); - - nng_msg_trim(m, len); - NNI_PUT16(data + zt_offset_data_id, id); - NNI_PUT16(data + zt_offset_data_fragsz, (uint16_t) fragsz); - NNI_PUT16(data + zt_offset_data_frag, fragno); - NNI_PUT16(data + zt_offset_data_nfrag, nfrags); - offset += len; - fraglen += len; - fragno++; - zt_send(p->zp_ztn, p->zp_nwid, zt_op_data, p->zp_raddr, - p->zp_laddr, data, fraglen + zt_offset_data_data); - } while (nni_msg_len(m) != 0); - nni_mtx_unlock(&zt_lk); - - // NB, We never bothered to call nn_aio_sched, because we run this - // synchronously, relying on UDP to simply discard messages if we - // cannot deliver them. This means that pipe send operations with - // this transport are not cancellable. - - nni_aio_set_msg(aio, NULL); - nni_msg_free(m); - nni_aio_finish(aio, 0, offset); -} - -static void -zt_pipe_cancel_recv(nni_aio *aio, void *arg, int rv) -{ - zt_pipe *p = arg; - nni_mtx_lock(&zt_lk); - if (p->zp_user_rxaio == aio) { - p->zp_user_rxaio = NULL; - nni_aio_finish_error(aio, rv); - } - nni_mtx_unlock(&zt_lk); -} - -static void -zt_fraglist_clear(zt_fraglist *fl) -{ - nni_msg *msg; - - fl->fl_ready = 0; - fl->fl_msgid = 0; - fl->fl_time = NNI_TIME_ZERO; - if ((msg = fl->fl_msg) != NULL) { - fl->fl_msg = NULL; - nni_msg_free(msg); - } - memset(fl->fl_missing, 0, fl->fl_missingsz); -} - -static void -zt_fraglist_free(zt_fraglist *fl) -{ - zt_fraglist_clear(fl); - nni_free(fl->fl_missing, fl->fl_missingsz); - fl->fl_missing = NULL; -} - -static void -zt_pipe_dorecv(zt_pipe *p) -{ - nni_aio *aio = p->zp_user_rxaio; - nni_time now = nni_clock(); - - if (aio == NULL) { - return; - } - - for (int i = 0; i < zt_recvq; i++) { - zt_fraglist *fl = &p->zp_recvq[i]; - nni_msg * msg; - - if (now > (fl->fl_time + zt_recv_stale)) { - // fragment list is stale, clean it. - zt_fraglist_clear(fl); - continue; - } - if (!fl->fl_ready) { - continue; - } - - // Got data. Let's pass it up. - msg = fl->fl_msg; - fl->fl_msg = NULL; - NNI_ASSERT(msg != NULL); - - p->zp_user_rxaio = NULL; - nni_aio_finish_msg(aio, msg); - zt_fraglist_clear(fl); - return; - } -} - -static void -zt_pipe_recv(void *arg, nni_aio *aio) -{ - zt_pipe *p = arg; - int rv; - - if (nni_aio_begin(aio) != 0) { - return; - } - nni_mtx_lock(&zt_lk); - if (p->zp_closed) { - nni_mtx_unlock(&zt_lk); - nni_aio_finish_error(aio, NNG_ECLOSED); - return; - } - if ((rv = nni_aio_schedule(aio, zt_pipe_cancel_recv, p)) != 0) { - nni_mtx_unlock(&zt_lk); - nni_aio_finish_error(aio, rv); - return; - } - p->zp_user_rxaio = aio; - zt_pipe_dorecv(p); - nni_mtx_unlock(&zt_lk); -} - -static uint16_t -zt_pipe_peer(void *arg) -{ - zt_pipe *pipe = arg; - - return (pipe->zp_peer); -} - -static int -zt_get_nw_status(zt_node *ztn, uint64_t nwid, int *statusp) -{ - ZT_VirtualNetworkConfig *vcfg; - int status; - - vcfg = ZT_Node_networkConfig(ztn->zn_znode, nwid); - if (vcfg == NULL) { - return (NNG_ECLOSED); - } - switch (vcfg->status) { - case ZT_NETWORK_STATUS_REQUESTING_CONFIGURATION: - status = NNG_ZT_STATUS_CONFIG; - break; - case ZT_NETWORK_STATUS_OK: - status = NNG_ZT_STATUS_UP; - break; - case ZT_NETWORK_STATUS_ACCESS_DENIED: - status = NNG_ZT_STATUS_DENIED; - break; - case ZT_NETWORK_STATUS_NOT_FOUND: - status = NNG_ZT_STATUS_NOTFOUND; - break; - case ZT_NETWORK_STATUS_PORT_ERROR: - status = NNG_ZT_STATUS_ERROR; - break; - case ZT_NETWORK_STATUS_CLIENT_TOO_OLD: - status = NNG_ZT_STATUS_OBSOLETE; - break; - default: - status = NNG_ZT_STATUS_UNKNOWN; - break; - } - ZT_Node_freeQueryResult(ztn->zn_znode, vcfg); - - *statusp = status; - return (0); -} - -static int -zt_get_nw_name(zt_node *ztn, uint64_t nwid, void *buf, size_t *szp, nni_type t) -{ - ZT_VirtualNetworkConfig *vcfg; - int rv; - - vcfg = ZT_Node_networkConfig(ztn->zn_znode, nwid); - if (vcfg == NULL) { - return (NNG_ECLOSED); - } - - rv = nni_copyout_str(vcfg->name, buf, szp, t); - ZT_Node_freeQueryResult(ztn->zn_znode, vcfg); - - return (rv); -} - -static int -zt_pipe_get_recvmaxsz(void *arg, void *buf, size_t *szp, nni_type t) -{ - zt_pipe *p = arg; - return (nni_copyout_size(p->zp_rcvmax, buf, szp, t)); -} - -static int -zt_pipe_get_nwid(void *arg, void *buf, size_t *szp, nni_type t) -{ - zt_pipe *p = arg; - return (nni_copyout_u64(p->zp_nwid, buf, szp, t)); -} - -static int -zt_pipe_get_node(void *arg, void *buf, size_t *szp, nni_type t) -{ - zt_pipe *p = arg; - return (nni_copyout_u64(p->zp_laddr >> 24, buf, szp, t)); -} - -static void -zt_pipe_ping_cb(void *arg) -{ - zt_pipe *p = arg; - nni_aio *aio = p->zp_ping_aio; - int rv; - - if ((rv = nni_aio_result(aio)) != 0) { - // We were canceled. That means we're done. - return; - } - nni_mtx_lock(&zt_lk); - if (p->zp_closed || aio == NULL || (p->zp_ping_tries == 0) || - (p->zp_ping_time == NNG_DURATION_INFINITE) || - (p->zp_ping_time == NNG_DURATION_ZERO)) { - nni_mtx_unlock(&zt_lk); - return; - } - if (p->zp_ping_try > p->zp_ping_tries) { - // Ping count exceeded; the other side is AFK. - // Close the pipe, but no need to send a reason to the peer. - zt_pipe_close_err(p, NNG_ECLOSED, 0, NULL); - nni_mtx_unlock(&zt_lk); - return; - } - - if (nni_clock() > (p->zp_last_recv + p->zp_ping_time)) { - p->zp_ping_try++; - zt_pipe_send_ping(p); - } - - nni_sleep_aio(p->zp_ping_time, aio); // Schedule a recheck. - nni_mtx_unlock(&zt_lk); -} - -static void -zt_pipe_start_ping(zt_pipe *p) -{ - // send a gratuitous ping, and start the ping interval timer. - if ((p->zp_ping_tries > 0) && (p->zp_ping_time != NNG_DURATION_ZERO) && - (p->zp_ping_time != NNG_DURATION_INFINITE)) { - p->zp_ping_try = 0; - zt_pipe_send_ping(p); - nni_sleep_aio(p->zp_ping_time, p->zp_ping_aio); - } -} - -static void -zt_ep_fini(void *arg) -{ - zt_ep *ep = arg; - nni_aio_stop(ep->ze_creq_aio); - nni_aio_free(ep->ze_creq_aio); - NNI_FREE_STRUCT(ep); -} - -static int -zt_parsehex(const char **sp, uint64_t *valp, bool wildok) -{ - int n; - const char *s = *sp; - char c; - uint64_t v; - - if (wildok && *s == '*') { - *valp = 0; - s++; - *sp = s; - return (0); - } - - for (v = 0, n = 0; (n < 16) && isxdigit(c = tolower(*s)); n++, s++) { - v *= 16; - if (isdigit(c)) { - v += (c - '0'); - } else { - v += ((c - 'a') + 10); - } - } - - *sp = s; - *valp = v; - return (n ? 0 : NNG_EINVAL); -} - -static int -zt_parsedec(const char **sp, uint64_t *valp) -{ - int n; - const char *s = *sp; - char c; - uint64_t v; - - for (v = 0, n = 0; (n < 20) && isdigit(c = *s); n++, s++) { - v *= 10; - v += (c - '0'); - } - *sp = s; - *valp = v; - return (n ? 0 : NNG_EINVAL); -} - -static int -zt_ep_init(void **epp, nni_url *url, nni_sock *sock, nni_dialer *ndialer, - nni_listener *nlistener) -{ - zt_ep * ep; - uint64_t node; - uint64_t port; - int rv; - const char *h; - - if ((ep = NNI_ALLOC_STRUCT(ep)) == NULL) { - return (NNG_ENOMEM); - } - - ep->ze_mtu = ZT_MIN_MTU; - ep->ze_aio = NULL; - ep->ze_ping_tries = zt_ping_tries; - ep->ze_ping_time = zt_ping_time; - ep->ze_conn_time = zt_conn_time; - ep->ze_conn_tries = zt_conn_tries; - ep->ze_proto = nni_sock_proto_id(sock); - ep->ze_ndialer = ndialer; - ep->ze_nlistener = nlistener; - - nni_aio_list_init(&ep->ze_aios); - - rv = nni_aio_alloc(&ep->ze_creq_aio, zt_ep_conn_req_cb, ep); - if (rv != 0) { - zt_ep_fini(ep); - return (rv); - } - - // Our URL format is: - // - // zt://.: - // - // The port must be specified, but may be zero. The nodeid - // may be '*' to refer to ourself. There may be a trailing slash - // which will be ignored. - - h = url->u_hostname; - if (((strlen(url->u_path) == 1) && (url->u_path[0] != '/')) || - (strlen(url->u_path) > 1) || (url->u_fragment != NULL) || - (url->u_query != NULL) || (url->u_userinfo != NULL) || - (zt_parsehex(&h, &node, true) != 0) || (*h++ != '.') || - (zt_parsehex(&h, &ep->ze_nwid, false) != 0) || - (node > 0xffffffffffull)) { - return (NNG_EADDRINVAL); - } - h = url->u_port; - if ((zt_parsedec(&h, &port) != 0) || (port > zt_max_port)) { - return (NNG_EADDRINVAL); - } - - // Parse the URL. - if (nlistener != NULL) { - // listener - ep->ze_laddr = node; - ep->ze_laddr <<= 24; - ep->ze_laddr |= port; - ep->ze_raddr = 0; - ep->ze_nlistener = nlistener; - } else { - // dialer - if (port == 0) { - return (NNG_EADDRINVAL); - } - ep->ze_raddr = node; - ep->ze_raddr <<= 24; - ep->ze_raddr |= port; - ep->ze_laddr = 0; - ep->ze_ndialer = ndialer; - } - - *epp = ep; - return (0); -} - -static int -zt_dialer_init(void **epp, nni_url *url, nni_dialer *d) -{ - return (zt_ep_init(epp, url, nni_dialer_sock(d), d, NULL)); -} - -static int -zt_listener_init(void **epp, nni_url *url, nni_listener *l) -{ - return (zt_ep_init(epp, url, nni_listener_sock(l), NULL, l)); -} - -static void -zt_ep_close(void *arg) -{ - zt_ep * ep = arg; - zt_node *ztn; - nni_aio *aio; - - nni_aio_abort(ep->ze_creq_aio, NNG_ECLOSED); - - // Cancel any outstanding user operation(s) - they should have - // been aborted by the above cancellation, but we need to be - // sure, as the cancellation callback may not have run yet. - - nni_mtx_lock(&zt_lk); - while ((aio = nni_list_first(&ep->ze_aios)) != NULL) { - nni_aio_list_remove(aio); - nni_aio_finish_error(aio, NNG_ECLOSED); - } - - // Endpoint framework guarantees to only call us once, - // and to not call other things while we are closed. - ztn = ep->ze_ztn; - // If we're on the ztn node list, pull us off. - if (ztn != NULL) { - nni_list_node_remove(&ep->ze_link); - zt_hash_remove(ztn->zn_ports, ep->ze_laddr & zt_port_mask); - zt_hash_remove(ztn->zn_eps, ep->ze_laddr); - } - - nni_mtx_unlock(&zt_lk); -} - -static int -zt_ep_bind_locked(zt_ep *ep) -{ - int rv; - uint64_t port; - uint64_t node; - zt_node *ztn; - - // If we haven't already got a ZT node, get one. - if ((ztn = ep->ze_ztn) == NULL) { - if ((rv = zt_node_find(ep)) != 0) { - return (rv); - } - ztn = ep->ze_ztn; - } - - node = ep->ze_laddr >> 24; - if ((node != 0) && (node != ztn->zn_self)) { - // User requested node id, but it doesn't match our - // own. - return (NNG_EADDRINVAL); - } - - if ((ep->ze_laddr & zt_port_mask) == 0) { - // ask for an ephemeral port - if ((rv = zt_hash_alloc(ztn->zn_ports, &port, ep)) != 0) { - return (rv); - } - NNI_ASSERT(port & zt_ephemeral); - } else { - void *conflict; - // make sure port requested is free. - port = ep->ze_laddr & zt_port_mask; - - if (zt_hash_find(ztn->zn_ports, port, &conflict) == 0) { - return (NNG_EADDRINUSE); - } - if ((rv = zt_hash_insert(ztn->zn_ports, port, ep)) != 0) { - return (rv); - } - } - NNI_ASSERT(port <= zt_max_port); - NNI_ASSERT(port > 0); - - ep->ze_laddr = ztn->zn_self; - ep->ze_laddr <<= 24; - ep->ze_laddr |= port; - ep->ze_running = true; - - if ((rv = zt_hash_insert(ztn->zn_eps, ep->ze_laddr, ep)) != 0) { - zt_hash_remove(ztn->zn_ports, port); - return (rv); - } - - return (0); -} - -static int -zt_ep_bind(void *arg) -{ - int rv; - zt_ep *ep = arg; - - nni_mtx_lock(&zt_lk); - rv = zt_ep_bind_locked(ep); - nni_mtx_unlock(&zt_lk); - - return (rv); -} - -static void -zt_ep_cancel(nni_aio *aio, void *arg, int rv) -{ - zt_ep *ep = arg; - - nni_mtx_lock(&zt_lk); - if (nni_aio_list_active(aio)) { - if (ep->ze_aio != NULL) { - nni_aio_abort(ep->ze_aio, rv); - } - nni_aio_list_remove(aio); - nni_aio_finish_error(aio, rv); - } - nni_mtx_unlock(&zt_lk); -} - -static void -zt_ep_doaccept(zt_ep *ep) -{ - // Call with ep lock held. - nni_time now; - zt_pipe *p; - int rv; - - now = nni_clock(); - // Consume any timedout connect requests. - while (ep->ze_creq_tail != ep->ze_creq_head) { - zt_creq creq; - nni_aio *aio; - - creq = ep->ze_creqs[ep->ze_creq_tail % zt_listenq]; - // Discard old connection requests. - if (creq.cr_expire < now) { - ep->ze_creq_tail++; - continue; - } - - if ((aio = nni_list_first(&ep->ze_aios)) == NULL) { - // No outstanding accept. We're done. - break; - } - - // We have both conn request, and a place to accept it. - - // Advance the tail. - ep->ze_creq_tail++; - - // We remove this AIO. This keeps it from being canceled. - nni_aio_list_remove(aio); - - rv = zt_pipe_alloc(&p, ep, creq.cr_raddr, ep->ze_laddr, true); - if (rv != 0) { - zt_send_err(ep->ze_ztn, ep->ze_nwid, creq.cr_raddr, - ep->ze_laddr, zt_err_unknown, - "Failed creating pipe"); - nni_aio_finish_error(aio, rv); - continue; - } - p->zp_peer = creq.cr_proto; - zt_pipe_send_conn_ack(p); - zt_pipe_start_ping(p); - nni_aio_set_output(aio, 0, p); - nni_aio_finish(aio, 0, 0); - } -} - -static void -zt_ep_accept(void *arg, nni_aio *aio) -{ - zt_ep *ep = arg; - int rv; - - if (nni_aio_begin(aio) != 0) { - return; - } - nni_mtx_lock(&zt_lk); - if ((rv = nni_aio_schedule(aio, zt_ep_cancel, ep)) != 0) { - nni_mtx_unlock(&zt_lk); - nni_aio_finish_error(aio, rv); - return; - } - nni_aio_list_append(&ep->ze_aios, aio); - zt_ep_doaccept(ep); - nni_mtx_unlock(&zt_lk); -} - -static void -zt_ep_conn_req_cancel(nni_aio *aio, void *arg, int rv) -{ - zt_ep *ep = arg; - // We don't have much to do here. The AIO will have been - // canceled as a result of the "parent" AIO canceling. - nni_mtx_lock(&zt_lk); - if (ep->ze_creq_active) { - ep->ze_creq_active = false; - nni_aio_finish_error(aio, rv); - } - nni_mtx_unlock(&zt_lk); -} - -static void -zt_ep_conn_req_cb(void *arg) -{ - zt_ep * ep = arg; - zt_pipe *p; - nni_aio *aio = ep->ze_creq_aio; - nni_aio *uaio; - int rv; - - nni_mtx_lock(&zt_lk); - - ep->ze_creq_active = false; - switch ((rv = nni_aio_result(aio))) { - case 0: - p = nni_aio_get_output(aio, 0); - // Already canceled, or already handled? - if ((uaio = nni_list_first(&ep->ze_aios)) != NULL) { - nni_aio_list_remove(uaio); - zt_pipe_start_ping(p); - nni_aio_set_output(uaio, 0, p); - nni_aio_finish(uaio, 0, 0); - } else { - // We have a pipe, but nowhere to stick it. - // Just discard it. - zt_pipe_fini(p); - } - ep->ze_creq_try = 0; - break; - - case NNG_ETIMEDOUT: - if ((ep->ze_creq_try > ep->ze_conn_tries) && - (ep->ze_conn_tries > 0)) { - // Final timeout attempt. - if ((uaio = nni_list_first(&ep->ze_aios)) != NULL) { - nni_aio_list_remove(uaio); - nni_aio_finish_error(uaio, rv); - // reset the counter. - ep->ze_creq_try = 0; - } - } - break; - - default: - // Failed hard? - if ((uaio = nni_list_first(&ep->ze_aios)) != NULL) { - nni_aio_list_remove(uaio); - nni_aio_finish_error(uaio, rv); - } - ep->ze_creq_try = 0; - break; - } - - if (nni_list_first(&ep->ze_aios) != NULL) { - nni_aio_set_timeout(aio, ep->ze_conn_time); - if (nni_aio_begin(aio) == 0) { - rv = nni_aio_schedule(aio, zt_ep_conn_req_cancel, ep); - if (rv != 0) { - nni_aio_finish_error(aio, rv); - } else { - ep->ze_creq_active = true; - ep->ze_creq_try++; - zt_ep_send_conn_req(ep); - } - } - } - - nni_mtx_unlock(&zt_lk); -} - -static void -zt_ep_connect(void *arg, nni_aio *aio) -{ - zt_ep *ep = arg; - int rv; - - if (nni_aio_begin(aio) != 0) { - return; - } - // We bind locally. We'll use the address later when we give - // it to the pipe, but this allows us to receive the initial - // ack back from the server. (This gives us an ephemeral - // address to work with.) - nni_mtx_lock(&zt_lk); - - // Clear the port so we get an ephemeral port. - ep->ze_laddr &= ~((uint64_t) zt_port_mask); - - if ((rv = zt_ep_bind_locked(ep)) != 0) { - nni_aio_finish_error(aio, rv); - nni_mtx_unlock(&zt_lk); - return; - } - - if ((ep->ze_raddr >> 24) == 0) { - ep->ze_raddr |= (ep->ze_ztn->zn_self << zt_port_shift); - } - if ((rv = nni_aio_schedule(aio, zt_ep_cancel, ep)) != 0) { - nni_aio_finish_error(aio, rv); - nni_mtx_unlock(&zt_lk); - return; - } - nni_aio_list_append(&ep->ze_aios, aio); - ep->ze_running = true; - - nni_aio_set_timeout(ep->ze_creq_aio, ep->ze_conn_time); - if (nni_aio_begin(ep->ze_creq_aio) == 0) { - rv = nni_aio_schedule( - ep->ze_creq_aio, zt_ep_conn_req_cancel, ep); - if (rv != 0) { - nni_aio_finish_error(ep->ze_creq_aio, rv); - } else { - // Send out the first connect message; if not - // yet attached to network message will be dropped. - ep->ze_creq_try = 1; - ep->ze_creq_active = true; - zt_ep_send_conn_req(ep); - } - } - nni_mtx_unlock(&zt_lk); -} - -static int -zt_ep_set_recvmaxsz(void *arg, const void *data, size_t sz, nni_type t) -{ - zt_ep *ep = arg; - size_t val; - int rv; - - if ((rv = nni_copyin_size(&val, data, sz, 0, NNI_MAXSZ, t)) == 0) { - nni_mtx_lock(&zt_lk); - ep->ze_rcvmax = val; - nni_mtx_unlock(&zt_lk); - } - return (rv); -} - -static int -zt_ep_get_recvmaxsz(void *arg, void *data, size_t *szp, nni_type t) -{ - zt_ep *ep = arg; - int rv; - nni_mtx_lock(&zt_lk); - rv = nni_copyout_size(ep->ze_rcvmax, data, szp, t); - nni_mtx_unlock(&zt_lk); - return (rv); -} - -static int -zt_ep_set_home(void *arg, const void *data, size_t sz, nni_type t) -{ - int rv; - zt_ep *ep = arg; - - if ((rv = zt_check_string(data, sz, t)) == 0) { - nni_mtx_lock(&zt_lk); - if (ep->ze_running) { - rv = NNG_ESTATE; - } else { - nni_strlcpy(ep->ze_home, data, sizeof(ep->ze_home)); - if ((rv = zt_node_find(ep)) != 0) { - ep->ze_ztn = NULL; - } - } - nni_mtx_unlock(&zt_lk); - } - - return (rv); -} - -static int -zt_ep_get_home(void *arg, void *data, size_t *szp, nni_type t) -{ - zt_ep *ep = arg; - int rv; - - nni_mtx_lock(&zt_lk); - rv = nni_copyout_str(ep->ze_home, data, szp, t); - nni_mtx_unlock(&zt_lk); - return (rv); -} - -static int -zt_ep_get_url(void *arg, void *data, size_t *szp, nni_type t) -{ - char ustr[64]; // more than plenty - zt_ep * ep = arg; - uint64_t addr; - - nni_mtx_lock(&zt_lk); - addr = ep->ze_nlistener != NULL ? ep->ze_laddr : ep->ze_raddr; - snprintf(ustr, sizeof(ustr), "zt://%llx.%llx:%u", - (unsigned long long) addr >> zt_port_shift, - (unsigned long long) ep->ze_nwid, - (unsigned) (addr & zt_port_mask)); - nni_mtx_unlock(&zt_lk); - return (nni_copyout_str(ustr, data, szp, t)); -} - -static int -zt_ep_set_orbit(void *arg, const void *data, size_t sz, nni_type t) -{ - uint64_t moonid; - uint64_t peerid; - zt_ep * ep = arg; - int rv; - enum ZT_ResultCode zrv; - - if ((t != NNI_TYPE_UINT64) && (t != NNI_TYPE_OPAQUE)) { - return (NNG_EBADTYPE); - } - if (sz == sizeof(uint64_t)) { - memcpy(&moonid, data, sizeof(moonid)); - peerid = 0; - } else if (sz == sizeof(uint64_t) * 2) { - memcpy(&moonid, data, sizeof(moonid)); - memcpy(&peerid, ((char *) data) + sizeof(uint64_t), - sizeof(peerid)); - } else { - return (NNG_EINVAL); - } - - nni_mtx_lock(&zt_lk); - if ((ep->ze_ztn == NULL) && ((rv = zt_node_find(ep)) != 0)) { - nni_mtx_unlock(&zt_lk); - return (rv); - } - zrv = ZT_Node_orbit(ep->ze_ztn->zn_znode, NULL, moonid, peerid); - nni_mtx_unlock(&zt_lk); - - return (zt_result(zrv)); -} - -static int -zt_ep_set_deorbit(void *arg, const void *data, size_t sz, nni_type t) -{ - uint64_t moonid; - zt_ep * ep = arg; - int rv; - - if ((rv = nni_copyin_u64(&moonid, data, sz, t)) == 0) { - enum ZT_ResultCode zrv; - - nni_mtx_lock(&zt_lk); - if ((ep->ze_ztn == NULL) && ((rv = zt_node_find(ep)) != 0)) { - nni_mtx_unlock(&zt_lk); - return (rv); - } - zrv = ZT_Node_deorbit(ep->ze_ztn->zn_znode, NULL, moonid); - nni_mtx_unlock(&zt_lk); - rv = zt_result(zrv); - } - return (rv); -} - -static int -zt_ep_set_add_local_addr(void *arg, const void *data, size_t sz, nni_type t) -{ - nng_sockaddr sa; - zt_ep * ep = arg; - int rv; - - if ((rv = nni_copyin_sockaddr(&sa, data, sz, t)) == 0) { - enum ZT_ResultCode zrv; - zt_node * ztn; - struct sockaddr_storage ss; - struct sockaddr_in * sin; - struct sockaddr_in6 * sin6; - - memset(&ss, 0, sizeof(ss)); - switch (sa.s_family) { - case NNG_AF_INET: - sin = (void *) &ss; - sin->sin_family = AF_INET; - sin->sin_addr.s_addr = sa.s_in.sa_addr; - sin->sin_port = 0; - break; - case NNG_AF_INET6: - sin6 = (void *) &ss; - sin6->sin6_family = AF_INET6; - sin6->sin6_port = 0; - memcpy(&sin6->sin6_addr, sa.s_in6.sa_addr, 16); - break; - default: - return (NNG_EINVAL); - } - - nni_mtx_lock(&zt_lk); - if ((ep->ze_ztn == NULL) && ((rv = zt_node_find(ep)) != 0)) { - nni_mtx_unlock(&zt_lk); - return (rv); - } - ztn = ep->ze_ztn; - zrv = ZT_Node_addLocalInterfaceAddress(ztn->zn_znode, &ss); - nni_mtx_unlock(&zt_lk); - rv = zt_result(zrv); - } - return (rv); -} - -static int -zt_ep_set_clear_local_addrs(void *arg, const void *data, size_t sz, nni_type t) -{ - zt_ep *ep = arg; - int rv; - NNI_ARG_UNUSED(data); - NNI_ARG_UNUSED(sz); - NNI_ARG_UNUSED(t); - - ZT_Node *zn; - nni_mtx_lock(&zt_lk); - if ((ep->ze_ztn == NULL) && ((rv = zt_node_find(ep)) != 0)) { - nni_mtx_unlock(&zt_lk); - return (rv); - } - zn = ep->ze_ztn; - ZT_Node_clearLocalInterfaceAddresses(zn); - nni_mtx_unlock(&zt_lk); - return (0); -} - -static int -zt_ep_get_node(void *arg, void *data, size_t *szp, nni_type t) -{ - zt_ep *ep = arg; - int rv; - - nni_mtx_lock(&zt_lk); - if ((ep->ze_ztn == NULL) && ((rv = zt_node_find(ep)) != 0)) { - nni_mtx_unlock(&zt_lk); - return (rv); - } - - rv = nni_copyout_u64(ep->ze_ztn->zn_self, data, szp, t); - - nni_mtx_unlock(&zt_lk); - return (rv); -} - -static int -zt_ep_get_nwid(void *arg, void *data, size_t *szp, nni_type t) -{ - zt_ep *ep = arg; - int rv; - - nni_mtx_lock(&zt_lk); - if ((ep->ze_ztn == NULL) && ((rv = zt_node_find(ep)) != 0)) { - nni_mtx_unlock(&zt_lk); - return (rv); - } - rv = nni_copyout_u64(ep->ze_nwid, data, szp, t); - nni_mtx_unlock(&zt_lk); - return (rv); -} - -static int -zt_ep_get_nw_name(void *arg, void *buf, size_t *szp, nni_type t) -{ - zt_ep *ep = arg; - int rv; - - nni_mtx_lock(&zt_lk); - if ((ep->ze_ztn == NULL) && ((rv = zt_node_find(ep)) != 0)) { - nni_mtx_unlock(&zt_lk); - return (rv); - } - rv = zt_get_nw_name(ep->ze_ztn, ep->ze_nwid, buf, szp, t); - nni_mtx_unlock(&zt_lk); - return (rv); -} - -static int -zt_ep_get_nw_status(void *arg, void *buf, size_t *szp, nni_type t) -{ - zt_ep *ep = arg; - int rv; - int status; - - nni_mtx_lock(&zt_lk); - if ((ep->ze_ztn == NULL) && ((rv = zt_node_find(ep)) != 0)) { - nni_mtx_unlock(&zt_lk); - return (rv); - } - if ((rv = zt_get_nw_status(ep->ze_ztn, ep->ze_nwid, &status)) != 0) { - nni_mtx_unlock(&zt_lk); - return (rv); - } - nni_mtx_unlock(&zt_lk); - return (nni_copyout_int(status, buf, szp, t)); -} - -static int -zt_ep_set_ping_time(void *arg, const void *data, size_t sz, nni_type t) -{ - zt_ep * ep = arg; - nng_duration val; - int rv; - - if ((rv = nni_copyin_ms(&val, data, sz, t)) == 0) { - nni_mtx_lock(&zt_lk); - ep->ze_ping_time = val; - nni_mtx_unlock(&zt_lk); - } - return (rv); -} - -static int -zt_ep_get_ping_time(void *arg, void *data, size_t *szp, nni_type t) -{ - zt_ep *ep = arg; - int rv; - - nni_mtx_lock(&zt_lk); - rv = nni_copyout_ms(ep->ze_ping_time, data, szp, t); - nni_mtx_unlock(&zt_lk); - return (rv); -} - -static int -zt_ep_set_ping_tries(void *arg, const void *data, size_t sz, nni_type t) -{ - zt_ep *ep = arg; - int val; - int rv; - - if ((rv = nni_copyin_int(&val, data, sz, 0, 1000000, t)) == 0) { - nni_mtx_lock(&zt_lk); - ep->ze_ping_tries = val; - nni_mtx_unlock(&zt_lk); - } - return (rv); -} - -static int -zt_ep_get_ping_tries(void *arg, void *data, size_t *szp, nni_type t) -{ - zt_ep *ep = arg; - int rv; - - nni_mtx_lock(&zt_lk); - rv = nni_copyout_int(ep->ze_ping_tries, data, szp, t); - nni_mtx_unlock(&zt_lk); - return (rv); -} - -static int -zt_ep_set_conn_time(void *arg, const void *data, size_t sz, nni_type t) -{ - zt_ep * ep = arg; - nng_duration val; - int rv; - - if ((rv = nni_copyin_ms(&val, data, sz, t)) == 0) { - nni_mtx_lock(&zt_lk); - ep->ze_conn_time = val; - nni_mtx_unlock(&zt_lk); - } - return (rv); -} - -static int -zt_ep_get_conn_time(void *arg, void *data, size_t *szp, nni_type t) -{ - zt_ep *ep = arg; - int rv; - - nni_mtx_lock(&zt_lk); - rv = nni_copyout_ms(ep->ze_conn_time, data, szp, t); - nni_mtx_unlock(&zt_lk); - return (rv); -} - -static int -zt_ep_set_conn_tries(void *arg, const void *data, size_t sz, nni_type t) -{ - zt_ep *ep = arg; - int val; - int rv; - - if ((rv = nni_copyin_int(&val, data, sz, 0, 1000000, t)) == 0) { - nni_mtx_lock(&zt_lk); - ep->ze_conn_tries = val; - nni_mtx_unlock(&zt_lk); - } - return (rv); -} - -static int -zt_ep_get_conn_tries(void *arg, void *data, size_t *szp, nni_type t) -{ - zt_ep *ep = arg; - int rv; - - nni_mtx_lock(&zt_lk); - rv = nni_copyout_int(ep->ze_conn_tries, data, szp, t); - nni_mtx_unlock(&zt_lk); - return (rv); -} - -static int -zt_ep_get_locaddr(void *arg, void *data, size_t *szp, nni_type t) -{ - zt_ep * ep = arg; - nng_sockaddr sa; - - memset(&sa, 0, sizeof(sa)); - sa.s_zt.sa_family = NNG_AF_ZT; - nni_mtx_lock(&zt_lk); - sa.s_zt.sa_nwid = ep->ze_nwid; - sa.s_zt.sa_nodeid = ep->ze_laddr >> zt_port_shift; - sa.s_zt.sa_port = ep->ze_laddr & zt_port_mask; - nni_mtx_unlock(&zt_lk); - return (nni_copyout_sockaddr(&sa, data, szp, t)); -} - -static int -zt_pipe_get_locaddr(void *arg, void *data, size_t *szp, nni_type t) -{ - zt_pipe * p = arg; - nng_sockaddr sa; - - memset(&sa, 0, sizeof(sa)); - sa.s_zt.sa_family = NNG_AF_ZT; - sa.s_zt.sa_nwid = p->zp_nwid; - sa.s_zt.sa_nodeid = p->zp_laddr >> zt_port_shift; - sa.s_zt.sa_port = p->zp_laddr & zt_port_mask; - return (nni_copyout_sockaddr(&sa, data, szp, t)); -} - -static int -zt_pipe_get_remaddr(void *arg, void *data, size_t *szp, nni_type t) -{ - zt_pipe * p = arg; - nng_sockaddr sa; - - memset(&sa, 0, sizeof(sa)); - sa.s_zt.sa_family = NNG_AF_ZT; - sa.s_zt.sa_nwid = p->zp_nwid; - sa.s_zt.sa_nodeid = p->zp_raddr >> zt_port_shift; - sa.s_zt.sa_port = p->zp_raddr & zt_port_mask; - return (nni_copyout_sockaddr(&sa, data, szp, t)); -} - -static int -zt_pipe_get_mtu(void *arg, void *data, size_t *szp, nni_type t) -{ - zt_pipe *p = arg; - return (nni_copyout_size(p->zp_mtu, data, szp, t)); -} - -static const nni_option zt_pipe_options[] = { - { - .o_name = NNG_OPT_LOCADDR, - .o_get = zt_pipe_get_locaddr, - }, - { - .o_name = NNG_OPT_REMADDR, - .o_get = zt_pipe_get_remaddr, - }, - { - .o_name = NNG_OPT_ZT_MTU, - .o_get = zt_pipe_get_mtu, - }, - { - .o_name = NNG_OPT_ZT_NWID, - .o_get = zt_pipe_get_nwid, - }, - { - .o_name = NNG_OPT_ZT_NODE, - .o_get = zt_pipe_get_node, - }, - { - .o_name = NNG_OPT_RECVMAXSZ, - .o_get = zt_pipe_get_recvmaxsz, - }, - // terminate list - { - .o_name = NULL, - }, -}; - -static int -zt_pipe_getopt(void *arg, const char *name, void *buf, size_t *szp, nni_type t) -{ - zt_pipe *p = arg; - return (nni_getopt(zt_pipe_options, name, p, buf, szp, t)); -} - -static nni_tran_pipe_ops zt_pipe_ops = { - .p_init = zt_pipe_init, - .p_fini = zt_pipe_fini, - .p_send = zt_pipe_send, - .p_recv = zt_pipe_recv, - .p_close = zt_pipe_close, - .p_peer = zt_pipe_peer, - .p_getopt = zt_pipe_getopt, -}; - -static nni_option zt_dialer_options[] = { - { - .o_name = NNG_OPT_RECVMAXSZ, - .o_get = zt_ep_get_recvmaxsz, - .o_set = zt_ep_set_recvmaxsz, - }, - { - .o_name = NNG_OPT_URL, - .o_get = zt_ep_get_url, - }, - { - .o_name = NNG_OPT_ZT_HOME, - .o_get = zt_ep_get_home, - .o_set = zt_ep_set_home, - }, - { - .o_name = NNG_OPT_ZT_NODE, - .o_get = zt_ep_get_node, - }, - { - .o_name = NNG_OPT_ZT_NWID, - .o_get = zt_ep_get_nwid, - }, - { - .o_name = NNG_OPT_ZT_NETWORK_STATUS, - .o_get = zt_ep_get_nw_status, - }, - { - .o_name = NNG_OPT_ZT_NETWORK_NAME, - .o_get = zt_ep_get_nw_name, - }, - { - .o_name = NNG_OPT_ZT_PING_TIME, - .o_get = zt_ep_get_ping_time, - .o_set = zt_ep_set_ping_time, - }, - { - .o_name = NNG_OPT_ZT_PING_TRIES, - .o_get = zt_ep_get_ping_tries, - .o_set = zt_ep_set_ping_tries, - }, - { - .o_name = NNG_OPT_ZT_CONN_TIME, - .o_get = zt_ep_get_conn_time, - .o_set = zt_ep_set_conn_time, - }, - { - .o_name = NNG_OPT_ZT_CONN_TRIES, - .o_get = zt_ep_get_conn_tries, - .o_set = zt_ep_set_conn_tries, - }, - { - .o_name = NNG_OPT_ZT_ORBIT, - .o_set = zt_ep_set_orbit, - }, - { - .o_name = NNG_OPT_ZT_DEORBIT, - .o_set = zt_ep_set_deorbit, - }, - { - .o_name = NNG_OPT_ZT_ADD_LOCAL_ADDR, - .o_set = zt_ep_set_add_local_addr, - }, - { - .o_name = NNG_OPT_ZT_CLEAR_LOCAL_ADDRS, - .o_set = zt_ep_set_clear_local_addrs, - }, - - // terminate list - { - .o_name = NULL, - }, -}; - -static nni_option zt_listener_options[] = { - { - .o_name = NNG_OPT_RECVMAXSZ, - .o_get = zt_ep_get_recvmaxsz, - .o_set = zt_ep_set_recvmaxsz, - }, - { - .o_name = NNG_OPT_URL, - .o_get = zt_ep_get_url, - }, - { - .o_name = NNG_OPT_ZT_HOME, - .o_get = zt_ep_get_home, - .o_set = zt_ep_set_home, - }, - { - .o_name = NNG_OPT_ZT_NODE, - .o_get = zt_ep_get_node, - }, - { - .o_name = NNG_OPT_ZT_NWID, - .o_get = zt_ep_get_nwid, - }, - { - .o_name = NNG_OPT_ZT_NETWORK_STATUS, - .o_get = zt_ep_get_nw_status, - }, - { - .o_name = NNG_OPT_ZT_NETWORK_NAME, - .o_get = zt_ep_get_nw_name, - }, - { - .o_name = NNG_OPT_ZT_PING_TIME, - .o_get = zt_ep_get_ping_time, - .o_set = zt_ep_set_ping_time, - }, - { - .o_name = NNG_OPT_ZT_PING_TRIES, - .o_get = zt_ep_get_ping_tries, - .o_set = zt_ep_set_ping_tries, - }, - { - .o_name = NNG_OPT_ZT_ORBIT, - .o_set = zt_ep_set_orbit, - }, - { - .o_name = NNG_OPT_ZT_DEORBIT, - .o_set = zt_ep_set_deorbit, - }, - { - .o_name = NNG_OPT_LOCADDR, - .o_get = zt_ep_get_locaddr, - }, - // terminate list - { - .o_name = NULL, - }, -}; - -static nni_tran_dialer_ops zt_dialer_ops = { - .d_init = zt_dialer_init, - .d_fini = zt_ep_fini, - .d_connect = zt_ep_connect, - .d_close = zt_ep_close, - .d_options = zt_dialer_options, -}; - -static nni_tran_listener_ops zt_listener_ops = { - .l_init = zt_listener_init, - .l_fini = zt_ep_fini, - .l_bind = zt_ep_bind, - .l_accept = zt_ep_accept, - .l_close = zt_ep_close, - .l_options = zt_listener_options, -}; - -// This is the ZeroTier transport linkage, and should be the -// only global symbol in this entire file. -static struct nni_tran zt_tran = { - .tran_version = NNI_TRANSPORT_VERSION, - .tran_scheme = "zt", - .tran_dialer = &zt_dialer_ops, - .tran_listener = &zt_listener_ops, - .tran_pipe = &zt_pipe_ops, - .tran_init = zt_tran_init, - .tran_fini = zt_tran_fini, -}; - -int -nng_zt_register(void) -{ - return (nni_tran_register(&zt_tran)); -} diff --git a/src/transport/zerotier/zthash.c b/src/transport/zerotier/zthash.c deleted file mode 100644 index ca46b373..00000000 --- a/src/transport/zerotier/zthash.c +++ /dev/null @@ -1,302 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// Copyright 2018 Capitar IT Group BV -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - -#include "core/nng_impl.h" -#include "zthash.h" - -struct zt_hash_entry { - uint64_t key; - void * val; - uint32_t skips; -}; - -int -zt_hash_init(zt_hash **hp) -{ - zt_hash *h; - - if ((h = NNI_ALLOC_STRUCT(h)) == NULL) { - return (NNG_ENOMEM); - } - h->ih_entries = NULL; - h->ih_count = 0; - h->ih_load = 0; - h->ih_cap = 0; - h->ih_maxload = 0; - h->ih_minload = 0; // never shrink below this - h->ih_minval = 0; - h->ih_maxval = 0xffffffff; - h->ih_dynval = 0; - - *hp = h; - return (0); -} - -void -zt_hash_fini(zt_hash *h) -{ - if (h != NULL) { - if (h->ih_entries != NULL) { - NNI_FREE_STRUCTS(h->ih_entries, h->ih_cap); - h->ih_entries = NULL; - h->ih_cap = h->ih_count = 0; - h->ih_load = h->ih_minload = h->ih_maxload = 0; - } - - NNI_FREE_STRUCT(h); - } -} - -void -zt_hash_limits(zt_hash *h, uint64_t minval, uint64_t maxval, uint64_t start) -{ - if (start < minval) { - start = minval; - } - if (start > maxval) { - start = maxval; - } - - h->ih_minval = minval; - h->ih_maxval = maxval; - h->ih_dynval = start; - NNI_ASSERT(minval < maxval); - NNI_ASSERT(start >= minval); - NNI_ASSERT(start <= maxval); -} - -// Inspired by Python dict implementation. This probe will visit every -// cell. We always hash consecutively assigned IDs. -#define ZT_HASH_NEXT(h, j) ((((j) *5) + 1) & (h->ih_cap - 1)) -#define ZT_HASH_INDEX(h, j) ((j) & (h->ih_cap - 1)) - -static size_t -zt_hash_find_index(zt_hash *h, uint64_t id) -{ - size_t index; - size_t start; - if (h->ih_count == 0) { - return ((size_t) -1); - } - - index = ZT_HASH_INDEX(h, id); - start = index; - for (;;) { - // The value of ihe_key is only valid if ihe_val is not NULL. - if ((h->ih_entries[index].key == id) && - (h->ih_entries[index].val != NULL)) { - return (index); - } - if (h->ih_entries[index].skips == 0) { - return ((size_t) -1); - } - index = ZT_HASH_NEXT(h, index); - - if (index == start) { - break; - } - } - - return ((size_t) -1); -} - -int -zt_hash_find(zt_hash *h, uint64_t id, void **vp) -{ - size_t index; - if ((index = zt_hash_find_index(h, id)) == (size_t) -1) { - return (NNG_ENOENT); - } - *vp = h->ih_entries[index].val; - return (0); -} - -static int -zt_hash_resize(zt_hash *h) -{ - size_t newsize; - size_t oldsize; - zt_hash_entry *newents; - zt_hash_entry *oldents; - uint32_t i; - - if ((h->ih_load < h->ih_maxload) && (h->ih_load >= h->ih_minload)) { - // No resize needed. - return (0); - } - - oldsize = h->ih_cap; - - newsize = 8; - while (newsize < (h->ih_count * 2)) { - newsize *= 2; - } - if (newsize == oldsize) { - // Same size. - return (0); - } - - oldents = h->ih_entries; - newents = NNI_ALLOC_STRUCTS(newents, newsize); - if (newents == NULL) { - return (NNG_ENOMEM); - } - - h->ih_entries = newents; - h->ih_cap = newsize; - h->ih_load = 0; - if (newsize > 8) { - h->ih_minload = newsize / 8; - h->ih_maxload = newsize * 2 / 3; - } else { - h->ih_minload = 0; - h->ih_maxload = 5; - } - for (i = 0; i < oldsize; i++) { - size_t index; - if (oldents[i].val == NULL) { - continue; - } - index = oldents[i].key & (newsize - 1); - for (;;) { - // Increment the load unconditionally. It counts - // once for every item stored, plus once for each - // hashing operation we use to store the item (i.e. - // one for the item, plus once for each rehash.) - h->ih_load++; - if (newents[index].val == NULL) { - // As we are hitting this entry for the first - // time, it won't have any skips. - NNI_ASSERT(newents[index].skips == 0); - newents[index].val = oldents[i].val; - newents[index].key = oldents[i].key; - break; - } - newents[index].skips++; - index = ZT_HASH_NEXT(h, index); - } - } - if (oldsize != 0) { - NNI_FREE_STRUCTS(oldents, oldsize); - } - return (0); -} - -int -zt_hash_remove(zt_hash *h, uint64_t id) -{ - size_t index; - size_t probe; - - if ((index = zt_hash_find_index(h, id)) == (size_t) -1) { - return (NNG_ENOENT); - } - - // Now we have found the index where the object exists. We are going - // to restart the search, until the index matches, to decrement the - // skips counter. - probe = (int) ZT_HASH_INDEX(h, id); - - for (;;) { - zt_hash_entry *entry; - // The load was increased once each hashing operation we used - // to place the the item. Decrement it accordingly. - h->ih_load--; - entry = &h->ih_entries[probe]; - if (probe == index) { - entry->val = NULL; - entry->key = 0; - break; - } - NNI_ASSERT(entry->skips > 0); - entry->skips--; - probe = ZT_HASH_NEXT(h, probe); - } - - h->ih_count--; - - // Shrink -- but it's ok if we can't. - (void) zt_hash_resize(h); - - return (0); -} - -int -zt_hash_insert(zt_hash *h, uint64_t id, void *val) -{ - size_t index; - zt_hash_entry *ent; - - // Try to resize -- if we don't need to, this will be a no-op. - if (zt_hash_resize(h) != 0) { - return (NNG_ENOMEM); - } - - // If it already exists, just overwrite the old value. - if ((index = zt_hash_find_index(h, id)) != (size_t) -1) { - ent = &h->ih_entries[index]; - ent->val = val; - return (0); - } - - index = ZT_HASH_INDEX(h, id); - for (;;) { - ent = &h->ih_entries[index]; - - // Increment the load count. We do this each time time we - // rehash. This may over-count items that collide on the - // same rehashing, but this should just cause a table to - // grow sooner, which is probably a good thing. - h->ih_load++; - if (ent->val == NULL) { - h->ih_count++; - ent->key = id; - ent->val = val; - return (0); - } - // Record the skip count. This being non-zero informs - // that a rehash will be necessary. Without this we - // would need to scan the entire hash for the match. - ent->skips++; - index = ZT_HASH_NEXT(h, index); - } -} - -int -zt_hash_alloc(zt_hash *h, uint64_t *idp, void *val) -{ - uint64_t id; - int rv; - - NNI_ASSERT(val != NULL); - - if (h->ih_count > (h->ih_maxval - h->ih_minval)) { - // Really more like ENOSPC.. the table is filled to max. - return (NNG_ENOMEM); - } - - for (;;) { - id = h->ih_dynval; - h->ih_dynval++; - if (h->ih_dynval > h->ih_maxval) { - h->ih_dynval = h->ih_minval; - } - - if (zt_hash_find_index(h, id) == (size_t) -1) { - break; - } - } - - rv = zt_hash_insert(h, id, val); - if (rv == 0) { - *idp = id; - } - return (rv); -} diff --git a/src/transport/zerotier/zthash.h b/src/transport/zerotier/zthash.h deleted file mode 100644 index 249eabbf..00000000 --- a/src/transport/zerotier/zthash.h +++ /dev/null @@ -1,43 +0,0 @@ -// -// Copyright 2020 Staysail Systems, Inc. -// Copyright 2018 Capitar IT Group BV -// -// This software is supplied under the terms of the MIT License, a -// copy of which should be located in the distribution where this -// file was obtained (LICENSE.txt). A copy of the license may also be -// found online at https://opensource.org/licenses/MIT. -// - -#ifndef ZT_HASH_H -#define ZT_HASH_H - -#include - -// This code is derived from id hash, but supports 64-bit IDs. - -typedef struct zt_hash zt_hash; -typedef struct zt_hash_entry zt_hash_entry; - -// NB: These details are entirely private to the hash implementation. -// They are provided here to facilitate inlining in structures. -struct zt_hash { - size_t ih_cap; - size_t ih_count; - size_t ih_load; - size_t ih_minload; // considers placeholders - size_t ih_maxload; - uint64_t ih_minval; - uint64_t ih_maxval; - uint64_t ih_dynval; - zt_hash_entry *ih_entries; -}; - -extern int zt_hash_init(zt_hash **); -extern void zt_hash_fini(zt_hash *); -extern void zt_hash_limits(zt_hash *, uint64_t, uint64_t, uint64_t); -extern int zt_hash_find(zt_hash *, uint64_t, void **); -extern int zt_hash_remove(zt_hash *, uint64_t); -extern int zt_hash_insert(zt_hash *, uint64_t, void *); -extern int zt_hash_alloc(zt_hash *, uint64_t *, void *); - -#endif // CORE_IDHASH_H -- cgit v1.2.3-70-g09d2