fixes #166 Websocket TLS mapping

This introduces the wss:// scheme, which is available and works like
the ws:// scheme if TLS is enabled in the library.

The library modularization is refactored somewhat, to make it easier
to use.  There is now a single NNG_ENABLE_TLS that enables TLS support
under the hood.

This also adds a new option for the TLS transport, NNG_OPT_TLS_CONFIG
(and a similar one for WSS, NNG_OPT_TLS_WSS_CONFIG) that offer access
to the underlying TLS configuration object, which now has a public API
to go with it as well.

Note that it is also possible to use pure HTTPS using the *private*
API, which will be exposed in a public form soon.

4 files changed, 193 insertions, 74 deletions

diff --git a/src/supplemental/http/client.c b/src/supplemental/http/client.c
index 8c082a17..4bb517ce 100644
--- a/src/supplemental/http/client.c
+++ b/src/supplemental/http/client.c
@@ -14,6 +14,8 @@
 #include <string.h>
 
 #include "core/nng_impl.h"
+#include "supplemental/tls/tls.h"
+
 #include "http.h"
 
 struct nni_http_client {
@@ -21,7 +23,7 @@ struct nni_http_client {
 	nni_list         aios;
 	nni_mtx          mtx;
 	bool             closed;
-	bool             tls;
+	nng_tls_config * tls;
 	nni_aio *        connaio;
 	nni_plat_tcp_ep *tep;
 };
@@ -39,7 +41,6 @@ http_conn_done(void *arg)
 	nni_aio *          aio;
 	int                rv;
 	nni_plat_tcp_pipe *p;
-	nni_http_tran      t;
 	nni_http *         http;
 
 	nni_mtx_lock(&c->mtx);
@@ -60,17 +61,13 @@ http_conn_done(void *arg)
 		return;
 	}
 
-	t.h_data      = p;
-	t.h_write     = (void *) nni_plat_tcp_pipe_send;
-	t.h_read      = (void *) nni_plat_tcp_pipe_recv;
-	t.h_close     = (void *) nni_plat_tcp_pipe_close;
-	t.h_sock_addr = (void *) nni_plat_tcp_pipe_sockname;
-	t.h_peer_addr = (void *) nni_plat_tcp_pipe_peername;
-	t.h_fini      = (void *) nni_plat_tcp_pipe_fini;
-
-	if ((rv = nni_http_init(&http, &t)) != 0) {
+	if (c->tls != NULL) {
+		rv = nni_http_init_tls(&http, c->tls, p);
+	} else {
+		rv = nni_http_init_tcp(&http, p);
+	}
+	if (rv != 0) {
 		nni_aio_finish_error(aio, rv);
-		nni_plat_tcp_pipe_fini(p);
 		nni_mtx_unlock(&c->mtx);
 		return;
 	}
@@ -90,6 +87,11 @@ nni_http_client_fini(nni_http_client *c)
 	nni_aio_fini(c->connaio);
 	nni_plat_tcp_ep_fini(c->tep);
 	nni_mtx_fini(&c->mtx);
+#ifdef NNG_SUPP_TLS
+	if (c->tls != NULL) {
+		nng_tls_config_fini(c->tls);
+	}
+#endif
 	NNI_FREE_STRUCT(c);
 }
 
@@ -119,6 +121,25 @@ nni_http_client_init(nni_http_client **cp, nng_sockaddr *sa)
 	return (0);
 }
 
+#ifdef NNG_SUPP_TLS
+int
+nni_http_client_set_tls(nni_http_client *c, nng_tls_config *tls)
+{
+	nng_tls_config *old;
+	nni_mtx_lock(&c->mtx);
+	old    = c->tls;
+	c->tls = tls;
+	if (tls != NULL) {
+		nni_tls_config_hold(tls);
+	}
+	nni_mtx_unlock(&c->mtx);
+	if (old != NULL) {
+		nng_tls_config_fini(old);
+	}
+	return (0);
+}
+#endif
+
 static void
 http_connect_cancel(nni_aio *aio, int rv)
 {
diff --git a/src/supplemental/http/http.c b/src/supplemental/http/http.c
index f414c5c7..df5c7588 100644
--- a/src/supplemental/http/http.c
+++ b/src/supplemental/http/http.c
@@ -12,6 +12,8 @@
 #include <string.h>
 
 #include "core/nng_impl.h"
+#include "supplemental/tls/tls.h"
+
 #include "http.h"
 
 // We insist that individual headers fit in 8K.
@@ -33,6 +35,15 @@ enum write_flavor {
 	HTTP_WR_RES,
 };
 
+typedef struct nni_http_tran {
+	void (*h_read)(void *, nni_aio *);
+	void (*h_write)(void *, nni_aio *);
+	int (*h_sock_addr)(void *, nni_sockaddr *);
+	int (*h_peer_addr)(void *, nni_sockaddr *);
+	void (*h_close)(void *);
+	void (*h_fini)(void *);
+} nni_http_tran;
+
 #define SET_RD_FLAVOR(aio, f) (aio)->a_prov_extra[0] = ((void *) (intptr_t)(f))
 #define GET_RD_FLAVOR(aio) (int) ((intptr_t) aio->a_prov_extra[0])
 #define SET_WR_FLAVOR(aio, f) (aio)->a_prov_extra[0] = ((void *) (intptr_t)(f))
@@ -76,14 +87,17 @@ http_close(nni_http *http)
 	http->closed = true;
 	if (nni_list_first(&http->wrq)) {
 		nni_aio_cancel(http->wr_aio, NNG_ECLOSED);
-		while ((aio = nni_list_first(&http->wrq)) != NULL) {
+		// Abort all operations except the one in flight.
+		while ((aio = nni_list_last(&http->wrq)) !=
+		    nni_list_first(&http->wrq)) {
 			nni_aio_list_remove(aio);
 			nni_aio_finish_error(aio, NNG_ECLOSED);
 		}
 	}
 	if (nni_list_first(&http->rdq)) {
 		nni_aio_cancel(http->rd_aio, NNG_ECLOSED);
-		while ((aio = nni_list_first(&http->rdq)) != NULL) {
+		while ((aio = nni_list_last(&http->rdq)) !=
+		    nni_list_first(&http->rdq)) {
 			nni_aio_list_remove(aio);
 			nni_aio_finish_error(aio, NNG_ECLOSED);
 		}
@@ -591,8 +605,8 @@ nni_http_fini(nni_http *http)
 	NNI_FREE_STRUCT(http);
 }
 
-int
-nni_http_init(nni_http **httpp, nni_http_tran *tran)
+static int
+http_init(nni_http **httpp, nni_http_tran *tran, void *data)
 {
 	nni_http *http;
 	int       rv;
@@ -609,21 +623,71 @@ nni_http_init(nni_http **httpp, nni_http_tran *tran)
 	nni_aio_list_init(&http->rdq);
 	nni_aio_list_init(&http->wrq);
 
-	if (((rv = nni_aio_init(&http->wr_aio, http_wr_cb, http)) != 0) ||
-	    ((rv = nni_aio_init(&http->rd_aio, http_rd_cb, http)) != 0)) {
-		nni_http_fini(http);
-		return (rv);
-	}
+	http->sock      = data;
 	http->rd_bufsz  = HTTP_BUFSIZE;
 	http->rd        = tran->h_read;
 	http->wr        = tran->h_write;
 	http->close     = tran->h_close;
 	http->fini      = tran->h_fini;
-	http->sock      = tran->h_data;
 	http->sock_addr = tran->h_sock_addr;
 	http->peer_addr = tran->h_peer_addr;
 
+	if (((rv = nni_aio_init(&http->wr_aio, http_wr_cb, http)) != 0) ||
+	    ((rv = nni_aio_init(&http->rd_aio, http_rd_cb, http)) != 0)) {
+		nni_http_fini(http);
+		return (rv);
+	}
+
 	*httpp = http;
 
 	return (0);
 }
+
+static nni_http_tran http_tcp_ops = {
+	.h_read      = (void *) nni_plat_tcp_pipe_recv,
+	.h_write     = (void *) nni_plat_tcp_pipe_send,
+	.h_close     = (void *) nni_plat_tcp_pipe_close,
+	.h_fini      = (void *) nni_plat_tcp_pipe_fini,
+	.h_sock_addr = (void *) nni_plat_tcp_pipe_sockname,
+	.h_peer_addr = (void *) nni_plat_tcp_pipe_peername,
+};
+
+int
+nni_http_init_tcp(nni_http **hpp, void *tcp)
+{
+	return (http_init(hpp, &http_tcp_ops, tcp));
+}
+
+#ifdef NNG_SUPP_TLS
+static nni_http_tran http_tls_ops = {
+	.h_read      = (void *) nni_tls_recv,
+	.h_write     = (void *) nni_tls_send,
+	.h_close     = (void *) nni_tls_close,
+	.h_fini      = (void *) nni_tls_fini,
+	.h_sock_addr = (void *) nni_tls_sockname,
+	.h_peer_addr = (void *) nni_tls_peername,
+};
+
+int
+nni_http_init_tls(nni_http **hpp, nng_tls_config *cfg, void *tcp)
+{
+	nni_tls *tls;
+	int      rv;
+
+	if ((rv = nni_tls_init(&tls, cfg, tcp)) != 0) {
+		nni_plat_tcp_pipe_fini(tcp);
+		return (rv);
+	}
+
+	return (http_init(hpp, &http_tls_ops, tls));
+}
+#else
+int
+nni_http_init_tls(nni_http **hpp, nng_tls_config *cfg, void *tcp)
+{
+	NNI_ARG_UNUSED(hpp);
+	NNI_ARG_UNUSED(cfg);
+	nni_plat_tcp_pipe_fini(tcp);
+	return (NNG_ENOTSUP);
+}
+#endif // NNG_SUPP_TLS
\ No newline at end of file
diff --git a/src/supplemental/http/http.h b/src/supplemental/http/http.h
index 3ecce4c8..d817ba37 100644
--- a/src/supplemental/http/http.h
+++ b/src/supplemental/http/http.h
@@ -18,16 +18,6 @@ typedef struct nni_http_msg    nni_http_msg;
 typedef struct nni_http_res    nni_http_res;
 typedef struct nni_http_entity nni_http_entity;
 
-typedef struct nni_http_tran {
-	void *h_data;
-	void (*h_read)(void *, nni_aio *);
-	void (*h_write)(void *, nni_aio *);
-	int (*h_sock_addr)(void *, nni_sockaddr *);
-	int (*h_peer_addr)(void *, nni_sockaddr *);
-	void (*h_close)(void *);
-	void (*h_fini)(void *);
-} nni_http_tran;
-
 typedef struct nni_http_req nni_http_req;
 
 extern int  nni_http_req_init(nni_http_req **);
@@ -146,7 +136,8 @@ enum { NNI_HTTP_STATUS_CONTINUE                  = 100,
 // the connection.
 typedef struct nni_http nni_http;
 
-extern int  nni_http_init(nni_http **, nni_http_tran *);
+extern int  nni_http_init_tcp(nni_http **, void *);
+extern int  nni_http_init_tls(nni_http **, nng_tls_config *, void *);
 extern void nni_http_close(nni_http *);
 extern void nni_http_fini(nni_http *);
 
@@ -254,6 +245,11 @@ extern int nni_http_server_add_handler(
 
 extern void nni_http_server_del_handler(nni_http_server *, void *);
 
+// nni_http_server_set_tls adds a TLS configuration to the server,
+// and enables the use of it.  This returns NNG_EBUSY if the server is
+// already started.
+extern int nni_http_server_set_tls(nni_http_server *, nng_tls_config *);
+
 // nni_http_server_start starts listening on the supplied port.
 extern int nni_http_server_start(nni_http_server *);
 
@@ -281,16 +277,13 @@ extern int nni_http_server_add_static(nni_http_server *, const char *host,
 extern int nni_http_server_add_file(nni_http_server *, const char *host,
     const char *ctype, const char *uri, const char *path);
 
-// TLS will use
-// extern int nni_http_server_start_tls(nni_http_server *, nng_sockaddr *,
-//     nni_tls_config *);
-
 // Client stuff.
 
 typedef struct nni_http_client nni_http_client;
 
 extern int  nni_http_client_init(nni_http_client **, nng_sockaddr *);
 extern void nni_http_client_fini(nni_http_client *);
+extern int  nni_http_client_set_tls(nni_http_client *, nng_tls_config *);
 extern void nni_http_client_connect(nni_http_client *, nni_aio *);
 
 #endif // NNG_SUPPLEMENTAL_HTTP_HTTP_H
diff --git a/src/supplemental/http/server.c b/src/supplemental/http/server.c
index ea7f15ce..ba74a138 100644
--- a/src/supplemental/http/server.c
+++ b/src/supplemental/http/server.c
@@ -14,6 +14,8 @@
 #include <string.h>
 
 #include "core/nng_impl.h"
+#include "supplemental/tls/tls.h"
+
 #include "http.h"
 
 static int  http_server_sys_init(void);
@@ -50,7 +52,6 @@ typedef struct http_sconn {
 	nni_aio *        rxaio;
 	nni_aio *        txaio;
 	nni_aio *        txdataio;
-	nni_http_tran    tran;
 	nni_reap_item    reap;
 } http_sconn;
 
@@ -64,7 +65,7 @@ struct nni_http_server {
 	nni_mtx          mtx;
 	nni_cv           cv;
 	bool             closed;
-	bool             tls;
+	nng_tls_config * tls;
 	nni_aio *        accaio;
 	nni_plat_tcp_ep *tep;
 };
@@ -105,28 +106,48 @@ http_sconn_fini(http_sconn *sc)
 }
 
 static void
-http_sconn_close(http_sconn *sc)
+http_sconn_close_locked(http_sconn *sc)
 {
 	nni_http_server *s;
 	s = sc->server;
+	nni_http *h;
 
+	if (sc->closed) {
+		return;
+	}
 	NNI_ASSERT(!sc->finished);
-	nni_mtx_lock(&s->mtx);
-	if (!sc->closed) {
-		nni_http *h;
-		sc->closed = true;
-		// Close the underlying transport.
-		if (nni_list_node_active(&sc->node)) {
-			nni_list_remove(&s->conns, sc);
-			if (nni_list_empty(&s->conns)) {
-				nni_cv_wake(&s->cv);
-			}
-		}
-		if ((h = sc->http) != NULL) {
-			nni_http_close(h);
+
+	sc->closed = true;
+	// Close the underlying transport.
+	if (nni_list_node_active(&sc->node)) {
+		nni_list_remove(&s->conns, sc);
+		if (nni_list_empty(&s->conns)) {
+			nni_cv_wake(&s->cv);
 		}
-		http_sconn_fini(sc);
 	}
+	nni_aio_cancel(sc->rxaio, NNG_ECLOSED);
+	nni_aio_cancel(sc->txaio, NNG_ECLOSED);
+	nni_aio_cancel(sc->txdataio, NNG_ECLOSED);
+	nni_aio_cancel(sc->cbaio, NNG_ECLOSED);
+
+	if ((h = sc->http) != NULL) {
+		nni_http_close(h);
+	}
+	http_sconn_fini(sc);
+}
+
+static void
+http_sconn_close(http_sconn *sc)
+{
+	nni_http_server *s;
+	s = sc->server;
+
+	if (sc->closed) {
+		return;
+	}
+
+	nni_mtx_lock(&s->mtx);
+	http_sconn_close_locked(sc);
 	nni_mtx_unlock(&s->mtx);
 }
 
@@ -484,14 +505,16 @@ http_sconn_cbdone(void *arg)
 }
 
 static int
-http_sconn_init(http_sconn **scp, nni_plat_tcp_pipe *tcp)
+http_sconn_init(http_sconn **scp, nni_http_server *s, nni_plat_tcp_pipe *tcp)
 {
 	http_sconn *sc;
 	int         rv;
 
 	if ((sc = NNI_ALLOC_STRUCT(sc)) == NULL) {
+		nni_plat_tcp_pipe_fini(tcp);
 		return (NNG_ENOMEM);
 	}
+
 	if (((rv = nni_http_req_init(&sc->req)) != 0) ||
 	    ((rv = nni_aio_init(&sc->rxaio, http_sconn_rxdone, sc)) != 0) ||
 	    ((rv = nni_aio_init(&sc->txaio, http_sconn_txdone, sc)) != 0) ||
@@ -502,18 +525,13 @@ http_sconn_init(http_sconn **scp, nni_plat_tcp_pipe *tcp)
 		http_sconn_close(sc);
 		return (rv);
 	}
-	// XXX: for HTTPS we would then try to do the TLS negotiation here.
-	// That would use a different set of tran values.
 
-	sc->tran.h_data  = tcp;
-	sc->tran.h_read  = (void *) nni_plat_tcp_pipe_recv;
-	sc->tran.h_write = (void *) nni_plat_tcp_pipe_send;
-	sc->tran.h_close = (void *) nni_plat_tcp_pipe_close; // close implied
-	sc->tran.h_fini  = (void *) nni_plat_tcp_pipe_fini;
-	sc->tran.h_sock_addr = (void *) nni_plat_tcp_pipe_sockname;
-	sc->tran.h_peer_addr = (void *) nni_plat_tcp_pipe_peername;
-
-	if ((rv = nni_http_init(&sc->http, &sc->tran)) != 0) {
+	if (s->tls != NULL) {
+		rv = nni_http_init_tls(&sc->http, s->tls, tcp);
+	} else {
+		rv = nni_http_init_tcp(&sc->http, tcp);
+	}
+	if (rv != 0) {
 		http_sconn_close(sc);
 		return (rv);
 	}
@@ -539,9 +557,8 @@ http_server_acccb(void *arg)
 		return;
 	}
 	tcp = nni_aio_get_pipe(aio);
-	if (http_sconn_init(&sc, tcp) != 0) {
-		nni_plat_tcp_pipe_close(tcp);
-		nni_plat_tcp_pipe_fini(tcp);
+	if (http_sconn_init(&sc, s, tcp) != 0) {
+		// The TCP structure is already cleaned up.
 
 		nni_plat_tcp_ep_accept(s->tep, s->accaio);
 		return;
@@ -590,6 +607,11 @@ http_server_fini(nni_http_server *s)
 		http_handler_fini(h);
 	}
 	nni_mtx_unlock(&s->mtx);
+#ifdef NNG_SUPP_TLS
+	if (s->tls != NULL) {
+		nng_tls_config_fini(s->tls);
+	}
+#endif
 	nni_aio_fini(s->accaio);
 	nni_cv_fini(&s->cv);
 	nni_mtx_fini(&s->mtx);
@@ -726,11 +748,7 @@ http_server_stop(nni_http_server *s)
 	// Stopping the server is a hard stop -- it aborts any work being
 	// done by clients.  (No graceful shutdown).
 	NNI_LIST_FOREACH (&s->conns, sc) {
-		nni_list_remove(&s->conns, sc);
-		if (sc->http != NULL) {
-			nni_http_close(sc->http);
-		}
-		http_sconn_fini(sc);
+		http_sconn_close_locked(sc);
 	}
 	nni_cv_wake(&s->cv);
 }
@@ -1073,6 +1091,29 @@ nni_http_server_add_static(nni_http_server *s, const char *host,
 	return (0);
 }
 
+#ifdef NNG_SUPP_TLS
+int
+nni_http_server_set_tls(nni_http_server *s, nng_tls_config *tcfg)
+{
+	nng_tls_config *old;
+	nni_mtx_lock(&s->mtx);
+	if (s->starts) {
+		nni_mtx_unlock(&s->mtx);
+		return (NNG_EBUSY);
+	}
+	old    = s->tls;
+	s->tls = tcfg;
+	if (tcfg) {
+		nni_tls_config_hold(tcfg);
+	}
+	nni_mtx_unlock(&s->mtx);
+	if (old) {
+		nng_tls_config_fini(old);
+	}
+	return (0);
+}
+#endif
+
 static int
 http_server_sys_init(void)
 {