fixes #1104 move allocation of protocol objects to common core

fixes #1103 respondent could inline backtrace

5 files changed, 72 insertions, 40 deletions

diff --git a/src/core/defs.h b/src/core/defs.h
index 9b7abafa..0e0deac2 100644
--- a/src/core/defs.h
+++ b/src/core/defs.h
@@ -134,6 +134,12 @@ typedef struct {
 // This increments a pointer a fixed number of byte cells.
 #define NNI_INCPTR(ptr, n) ((ptr) = (void *) ((char *) (ptr) + (n)))
 
+// Alignment -- this is used when allocating adjacent objects to ensure
+// that each object begins on a natural alignment boundary.
+#define NNI_ALIGN_SIZE sizeof(void *)
+#define NNI_ALIGN_MASK (NNI_ALIGN_SIZE - 1)
+#define NNI_ALIGN_UP(sz) (((sz) + NNI_ALIGN_MASK) & ~NNI_ALIGN_MASK)
+
 // A few assorted other items.
 #define NNI_FLAG_IPV4ONLY 1
 
diff --git a/src/core/pipe.c b/src/core/pipe.c
index f7269eb4..4076b62c 100644
--- a/src/core/pipe.c
+++ b/src/core/pipe.c
@@ -1,5 +1,5 @@
 //
-// Copyright 2019 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2020 Staysail Systems, Inc. <info@staysail.tech>
 // Copyright 2018 Capitar IT Group BV <info@capitar.com>
 // Copyright 2018 Devolutions <info@devolutions.net>
 //
@@ -95,7 +95,7 @@ pipe_destroy(nni_pipe *p)
 	}
 	nni_cv_fini(&p->p_cv);
 	nni_mtx_fini(&p->p_mtx);
-	NNI_FREE_STRUCT(p);
+	nni_free(p, p->p_size);
 }
 
 int
@@ -188,17 +188,21 @@ pipe_create(nni_pipe **pp, nni_sock *sock, nni_tran *tran, void *tdata)
 	void *              sdata = nni_sock_proto_data(sock);
 	nni_proto_pipe_ops *pops  = nni_sock_proto_pipe_ops(sock);
 	nni_pipe_stats *    st;
+	size_t sz;
 
-	if ((p = NNI_ALLOC_STRUCT(p)) == NULL) {
+	sz = NNI_ALIGN_UP(sizeof (*p)) + pops->pipe_size;
+
+	if ((p = nni_zalloc(sz)) == NULL) {
 		// In this case we just toss the pipe...
 		tran->tran_pipe->p_fini(tdata);
 		return (NNG_ENOMEM);
 	}
 
+	p->p_size = sz;
+	p->p_proto_data = p + 1;
 	p->p_tran_ops   = *tran->tran_pipe;
 	p->p_tran_data  = tdata;
 	p->p_proto_ops  = *pops;
-	p->p_proto_data = NULL;
 	p->p_sock       = sock;
 	p->p_closed     = false;
 	p->p_cbs        = false;
@@ -242,7 +246,7 @@ pipe_create(nni_pipe **pp, nni_sock *sock, nni_tran *tran, void *tdata)
 	nni_stat_add(&st->s_root, &st->s_txbytes);
 
 	if ((rv != 0) || ((rv = p->p_tran_ops.p_init(tdata, p)) != 0) ||
-	    ((rv = pops->pipe_init(&p->p_proto_data, p, sdata)) != 0)) {
+	    ((rv = pops->pipe_init(p->p_proto_data, p, sdata)) != 0)) {
 		nni_pipe_close(p);
 		nni_pipe_rele(p);
 		return (rv);
diff --git a/src/core/protocol.h b/src/core/protocol.h
index 005e34fd..77ccdb08 100644
--- a/src/core/protocol.h
+++ b/src/core/protocol.h
@@ -1,5 +1,5 @@
 //
-// Copyright 2019 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2020 Staysail Systems, Inc. <info@staysail.tech>
 // Copyright 2018 Capitar IT Group BV <info@capitar.com>
 //
 // This software is supplied under the terms of the MIT License, a
@@ -25,9 +25,13 @@
 
 // nni_proto_pipe contains protocol-specific per-pipe operations.
 struct nni_proto_pipe_ops {
-	// pipe_init creates the protocol-specific per pipe data structure.
+	// pipe_size is the size of a protocol pipe object.  The common
+	// code allocates this memory for the protocol private state.
+	size_t pipe_size;
+
+	// pipe_init2 initializes the protocol-specific pipe data structure.
 	// The last argument is the per-socket protocol private data.
-	int (*pipe_init)(void **, nni_pipe *, void *);
+	int (*pipe_init)(void *, nni_pipe *, void *);
 
 	// pipe_fini releases any pipe data structures.  This is called after
 	// the pipe has been removed from the protocol, and the generic
@@ -53,9 +57,13 @@ struct nni_proto_pipe_ops {
 };
 
 struct nni_proto_ctx_ops {
-	// ctx_init creates a new context. The second argument is the
+	// ctx_size is the size of a protocol context object.  The common
+	// code allocates this memory for the protocol private state.
+	size_t ctx_size;
+
+	// ctx_init initializes a new context. The second argument is the
 	// protocol specific socket structure.
-	int (*ctx_init)(void **, void *);
+	int (*ctx_init)(void *, void *);
 
 	// ctx_fini destroys a context.
 	void (*ctx_fini)(void *);
@@ -79,10 +87,13 @@ struct nni_proto_ctx_ops {
 };
 
 struct nni_proto_sock_ops {
-	// sock_init creates the protocol instance, which will be stored on
-	// the socket. This is run without the sock lock held, and allocates
-	// storage or other resources for the socket.
-	int (*sock_init)(void **, nni_sock *);
+	// ctx_size is the size of a protocol socket object.  The common
+	// code allocates this memory for the protocol private state.
+	size_t sock_size;
+
+	// sock_init2 initializes the protocol instance, which will be stored
+	// on the socket. This is run without the sock lock held.
+	int (*sock_init)(void *, nni_sock *);
 
 	// sock_fini destroys the protocol instance.  This is run without the
 	// socket lock held, and is intended to release resources.  It may
@@ -141,8 +152,9 @@ struct nni_proto {
 // during the life of the project.  If we add a new version, please keep
 // the old version around -- it may be possible to automatically convert
 // older versions in the future.
-#define NNI_PROTOCOL_V0 0x50520000 // "pr\0\0"
-#define NNI_PROTOCOL_VERSION NNI_PROTOCOL_V0
+#define NNI_PROTOCOL_V0 0x50520000u // "pr\0\0"
+#define NNI_PROTOCOL_V1 0x50520001u // "pr\0\0"
+#define NNI_PROTOCOL_VERSION NNI_PROTOCOL_V1
 
 // These flags determine which operations make sense.  We use them so that
 // we can reject attempts to create notification fds for operations that make
@@ -150,11 +162,11 @@ struct nni_proto {
 // that at the socket layer (NNG_PROTO_FLAG_RAW).  Finally, we provide the
 // NNI_PROTO_FLAG_NOMSGQ flag for protocols that do not use the upper write
 // or upper read queues.
-#define NNI_PROTO_FLAG_RCV 1    // Protocol can receive
-#define NNI_PROTO_FLAG_SND 2    // Protocol can send
-#define NNI_PROTO_FLAG_SNDRCV 3 // Protocol can both send & recv
-#define NNI_PROTO_FLAG_RAW 4    // Protocol is raw
-#define NNI_PROTO_FLAG_NOMSGQ 8 // Protocol bypasses the upper queues
+#define NNI_PROTO_FLAG_RCV 1u    // Protocol can receive
+#define NNI_PROTO_FLAG_SND 2u    // Protocol can send
+#define NNI_PROTO_FLAG_SNDRCV 3u // Protocol can both send & recv
+#define NNI_PROTO_FLAG_RAW 4u    // Protocol is raw
+#define NNI_PROTO_FLAG_NOMSGQ 8u // Protocol bypasses the upper queues
 
 // nni_proto_open is called by the protocol to create a socket instance
 // with its ops vector.  The intent is that applications will only see
diff --git a/src/core/socket.c b/src/core/socket.c
index 13d88253..bffe0f67 100644
--- a/src/core/socket.c
+++ b/src/core/socket.c
@@ -1,5 +1,5 @@
 //
-// Copyright 2019 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2020 Staysail Systems, Inc. <info@staysail.tech>
 // Copyright 2018 Capitar IT Group BV <info@capitar.com>
 //
 // This software is supplied under the terms of the MIT License, a
@@ -26,6 +26,7 @@ struct nni_ctx {
 	nni_sock *        c_sock;
 	nni_proto_ctx_ops c_ops;
 	void *            c_data;
+	size_t            c_size;
 	bool              c_closed;
 	unsigned          c_refcnt; // protected by global lock
 	uint32_t          c_id;
@@ -71,6 +72,7 @@ struct nni_socket {
 	uint32_t s_flags;
 	unsigned s_refcnt; // protected by global lock
 	void *   s_data;   // Protocol private
+	size_t   s_size;
 
 	nni_msgq *s_uwq; // Upper write queue
 	nni_msgq *s_urq; // Upper read queue
@@ -496,7 +498,7 @@ sock_destroy(nni_sock *s)
 	nni_cv_fini(&s->s_cv);
 	nni_mtx_fini(&s->s_mx);
 	nni_mtx_fini(&s->s_pipe_cbs_mtx);
-	NNI_FREE_STRUCT(s);
+	nni_free(s, s->s_size);
 }
 
 static int
@@ -505,10 +507,13 @@ nni_sock_create(nni_sock **sp, const nni_proto *proto)
 	int       rv;
 	nni_sock *s;
 	bool      on;
+	size_t    sz;
 
-	if ((s = NNI_ALLOC_STRUCT(s)) == NULL) {
+	sz = NNI_ALIGN_UP(sizeof(*s)) + proto->proto_sock_ops->sock_size;
+	if ((s = nni_zalloc(sz)) == NULL) {
 		return (NNG_ENOMEM);
 	}
+	s->s_data      = s + 1;
 	s->s_sndtimeo  = -1;
 	s->s_rcvtimeo  = -1;
 	s->s_reconn    = NNI_SECOND;
@@ -545,7 +550,7 @@ nni_sock_create(nni_sock **sp, const nni_proto *proto)
 
 	if (((rv = nni_msgq_init(&s->s_uwq, 0)) != 0) ||
 	    ((rv = nni_msgq_init(&s->s_urq, 1)) != 0) ||
-	    ((rv = s->s_sock_ops.sock_init(&s->s_data, s)) != 0) ||
+	    ((rv = s->s_sock_ops.sock_init(s->s_data, s)) != 0) ||
 	    ((rv = nni_sock_setopt(s, NNG_OPT_SENDTIMEO, &s->s_sndtimeo,
 	          sizeof(nni_duration), NNI_TYPE_DURATION)) != 0) ||
 	    ((rv = nni_sock_setopt(s, NNG_OPT_RECVTIMEO, &s->s_rcvtimeo,
@@ -989,7 +994,7 @@ nni_sock_setopt(
 		return (rv);
 	}
 
-	// Prepare a copy of the sockoption.
+	// Prepare a copy of the socket option.
 	if ((optv = NNI_ALLOC_STRUCT(optv)) == NULL) {
 		return (NNG_ENOMEM);
 	}
@@ -1187,7 +1192,7 @@ nni_ctx_destroy(nni_ctx *ctx)
 	}
 
 	// Let the socket go, our hold on it is done.
-	NNI_FREE_STRUCT(ctx);
+	nni_free(ctx, ctx->c_size);
 }
 
 void
@@ -1221,40 +1226,44 @@ nni_ctx_open(nni_ctx **ctxp, nni_sock *sock)
 {
 	nni_ctx *ctx;
 	int      rv;
+	size_t   sz;
 
 	if (sock->s_ctx_ops.ctx_init == NULL) {
 		return (NNG_ENOTSUP);
 	}
-	if ((ctx = NNI_ALLOC_STRUCT(ctx)) == NULL) {
+
+	sz = NNI_ALIGN_UP(sizeof(*ctx)) + sock->s_ctx_ops.ctx_size;
+	if ((ctx = nni_zalloc(sz)) == NULL) {
 		return (NNG_ENOMEM);
 	}
+	ctx->c_size = sz;
+	ctx->c_data = ctx + 1;
+	ctx->c_closed   = false;
+	ctx->c_refcnt   = 1; // Caller implicitly gets a reference.
+	ctx->c_sock     = sock;
+	ctx->c_ops      = sock->s_ctx_ops;
+	ctx->c_rcvtimeo = sock->s_rcvtimeo;
+	ctx->c_sndtimeo = sock->s_sndtimeo;
 
 	nni_mtx_lock(&sock_lk);
 	if (sock->s_closed) {
 		nni_mtx_unlock(&sock_lk);
-		NNI_FREE_STRUCT(ctx);
+		nni_free(ctx, ctx->c_size);
 		return (NNG_ECLOSED);
 	}
 	if ((rv = nni_idhash_alloc32(ctx_hash, &ctx->c_id, ctx)) != 0) {
 		nni_mtx_unlock(&sock_lk);
-		NNI_FREE_STRUCT(ctx);
+		nni_free(ctx, ctx->c_size);
 		return (rv);
 	}
 
-	if ((rv = sock->s_ctx_ops.ctx_init(&ctx->c_data, sock->s_data)) != 0) {
+	if ((rv = sock->s_ctx_ops.ctx_init(ctx->c_data, sock->s_data)) != 0) {
 		nni_idhash_remove(ctx_hash, ctx->c_id);
 		nni_mtx_unlock(&sock_lk);
-		NNI_FREE_STRUCT(ctx);
+		nni_free(ctx, ctx->c_size);
 		return (rv);
 	}
 
-	ctx->c_closed   = false;
-	ctx->c_refcnt   = 1; // Caller implicitly gets a reference.
-	ctx->c_sock     = sock;
-	ctx->c_ops      = sock->s_ctx_ops;
-	ctx->c_rcvtimeo = sock->s_rcvtimeo;
-	ctx->c_sndtimeo = sock->s_sndtimeo;
-
 	nni_list_append(&sock->s_ctxs, ctx);
 	nni_mtx_unlock(&sock_lk);
 
diff --git a/src/core/sockimpl.h b/src/core/sockimpl.h
index ffe6c6e8..16596c63 100644
--- a/src/core/sockimpl.h
+++ b/src/core/sockimpl.h
@@ -1,5 +1,5 @@
 //
-// Copyright 2019 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2020 Staysail Systems, Inc. <info@staysail.tech>
 // Copyright 2018 Capitar IT Group BV <info@capitar.com>
 //
 // This software is supplied under the terms of the MIT License, a
@@ -113,6 +113,7 @@ struct nni_pipe {
 	uint32_t           p_id;
 	nni_tran_pipe_ops  p_tran_ops;
 	nni_proto_pipe_ops p_proto_ops;
+	size_t             p_size;
 	void *             p_tran_data;
 	void *             p_proto_data;
 	nni_list_node      p_sock_node;