fixes #682 Support for Chunked Transfer Coding

This is the client side only, although the work is structured to
support server applications.  The chunked API is for now private,
although the intent to is to make it public for applications who
really want to use it.

Note that chunked transfer encoding puts data through extra copies.
First it copies through the buffering area (because I have to be able
to extract variable length strings from inside the data stream), and then
again to reassemble the chunks into a single unified object.

We do assume that the user wants the entire thing as a single object.
This means that using this to pull unbounded data will just silently
consume all memory. Use caution!

5 files changed, 461 insertions, 28 deletions

diff --git a/src/supplemental/http/CMakeLists.txt b/src/supplemental/http/CMakeLists.txt
index 98334109..ed7d5ae2 100644
--- a/src/supplemental/http/CMakeLists.txt
+++ b/src/supplemental/http/CMakeLists.txt
@@ -16,6 +16,7 @@ if (NNG_SUPP_HTTP)
                 supplemental/http/http.h
                 supplemental/http/http_api.h
                 supplemental/http/http_client.c
+                supplemental/http/http_chunk.c
                 supplemental/http/http_conn.c
                 supplemental/http/http_msg.c
                 supplemental/http/http_public.c
diff --git a/src/supplemental/http/http_api.h b/src/supplemental/http/http_api.h
index 14e842be..fdee70e9 100644
--- a/src/supplemental/http/http_api.h
+++ b/src/supplemental/http/http_api.h
@@ -27,6 +27,8 @@ typedef struct nng_http_conn    nni_http_conn;
 typedef struct nng_http_handler nni_http_handler;
 typedef struct nng_http_server  nni_http_server;
 typedef struct nng_http_client  nni_http_client;
+typedef struct nng_http_chunk   nni_http_chunk;
+typedef struct nng_http_chunks  nni_http_chunks;
 
 // These functions are private to the internal framework, and really should
 // not be used elsewhere.
@@ -46,6 +48,33 @@ extern int   nni_http_res_parse(nni_http_res *, void *, size_t, size_t *);
 extern void  nni_http_res_get_data(nni_http_res *, void **, size_t *);
 extern char *nni_http_res_headers(nni_http_res *);
 
+// Chunked transfer encoding.  For the moment this is not part of our public
+// API.  We can change that later.
+
+// nni_http_chunk_list_init creates a list of chunks, which shall not exceed
+// the specified overall size.  (Size 0 means no limit.)
+extern int nni_http_chunks_init(nni_http_chunks **, size_t);
+
+extern void nni_http_chunks_free(nni_http_chunks *);
+
+// nni_http_chunk_iter iterates over all chunks in the list.
+// Pass NULL for the last chunk to start at the head.  Returns NULL when done.
+extern nni_http_chunk *nni_http_chunks_iter(
+    nni_http_chunks *, nni_http_chunk *);
+
+// nni_http_chunk_list_size returns the combined size of all chunks in list.
+extern size_t nni_http_chunks_size(nni_http_chunks *);
+
+// nni_http_chunk_size returns the size of given chunk.
+extern size_t nni_http_chunk_size(nni_http_chunk *);
+// nni_http_chunk_data returns a pointer to the data.
+extern void *nni_http_chunk_data(nni_http_chunk *);
+
+extern int nni_http_chunks_parse(nni_http_chunks *, void *, size_t, size_t *);
+
+extern void nni_http_read_chunks(
+    nni_http_conn *, nni_http_chunks *, nni_aio *);
+
 // Private to the server. (Used to support session hijacking.)
 extern void  nni_http_conn_set_ctx(nni_http_conn *, void *);
 extern void *nni_http_conn_get_ctx(nni_http_conn *);
diff --git a/src/supplemental/http/http_chunk.c b/src/supplemental/http/http_chunk.c
new file mode 100644
index 00000000..9333548e
--- /dev/null
+++ b/src/supplemental/http/http_chunk.c
@@ -0,0 +1,341 @@
+//
+// Copyright 2018 Staysail Systems, Inc. <info@staysail.tech>
+// Copyright 2018 Capitar IT Group BV <info@capitar.com>
+//
+// This software is supplied under the terms of the MIT License, a
+// copy of which should be located in the distribution where this
+// file was obtained (LICENSE.txt).  A copy of the license may also be
+// found online at https://opensource.org/licenses/MIT.
+//
+
+#include <ctype.h>
+#include <stdbool.h>
+#include <string.h>
+
+#include "core/nng_impl.h"
+
+#include "http_api.h"
+
+// Chunked transfer encoding support.
+
+// Note that HTTP/1.1 chunked transfer encoding is horrible, and should
+// be avoided if at all possible.  It necessarily creates extra need for
+// data copies, creates a lot of extra back and forth complexity.  If you're
+// stuck in this code, we feel great sympathy for you.
+//
+// We feel strongly enough about this that we refuse to provide any
+// method to automatically generate chunked transfers.  If you think
+// you need to send chunked transfers (because you have no idea how
+// much data you will send, such as a streaming workload), consider a
+// different method such as WebSocket to send your data.  Unbounded
+// entity body data is just impolite.
+
+enum chunk_state {
+	CS_INIT,   // initial state
+	CS_LEN,    // length
+	CS_EXT,    // random extension text (we ignore)
+	CS_CR,     // carriage return after length (and extensions)
+	CS_DATA,   // actual data
+	CS_TRLR,   // trailer
+	CS_TRLRCR, // CRLF at end of trailer
+	CS_DONE,
+};
+
+struct nng_http_chunks {
+	nni_list         cl_chunks;
+	size_t           cl_maxsz;
+	size_t           cl_size; // parsed size (so far)
+	size_t           cl_line; // bytes since last newline
+	enum chunk_state cl_state;
+};
+
+struct nng_http_chunk {
+	nni_list_node c_node;
+	size_t        c_size;
+	size_t        c_alloc;
+	size_t        c_resid; // residual data to transfer
+	char *        c_data;
+};
+
+int
+nni_http_chunks_init(nni_http_chunks **clp, size_t maxsz)
+{
+	nni_http_chunks *cl;
+
+	if ((cl = NNI_ALLOC_STRUCT(cl)) == NULL) {
+		return (NNG_ENOMEM);
+	}
+	NNI_LIST_INIT(&cl->cl_chunks, nni_http_chunk, c_node);
+	cl->cl_maxsz = maxsz;
+	*clp         = cl;
+	return (0);
+}
+
+void
+nni_http_chunks_free(nni_http_chunks *cl)
+{
+	nni_http_chunk *ch;
+	if (cl == NULL) {
+		return;
+	}
+	while ((ch = nni_list_first(&cl->cl_chunks)) != NULL) {
+		nni_list_remove(&cl->cl_chunks, ch);
+		if (ch->c_data != NULL) {
+			nni_free(ch->c_data, ch->c_alloc);
+		}
+		NNI_FREE_STRUCT(ch);
+	}
+	NNI_FREE_STRUCT(cl);
+}
+
+nni_http_chunk *
+nni_http_chunks_iter(nni_http_chunks *cl, nni_http_chunk *last)
+{
+	if (last == NULL) {
+		return (nni_list_first(&cl->cl_chunks));
+	}
+	return (nni_list_next(&cl->cl_chunks, last));
+}
+
+size_t
+nni_http_chunks_size(nni_http_chunks *cl)
+{
+	size_t          tot = 0;
+	nni_http_chunk *ch;
+	NNI_LIST_FOREACH (&cl->cl_chunks, ch) {
+		tot += ch->c_size;
+	}
+	return (tot);
+}
+
+size_t
+nni_http_chunk_size(nni_http_chunk *ch)
+{
+	return (ch->c_size);
+}
+
+void *
+nni_http_chunk_data(nni_http_chunk *ch)
+{
+	return (ch->c_data);
+}
+
+static int
+chunk_ingest_len(nni_http_chunks *cl, char c)
+{
+	if (isdigit(c)) {
+		cl->cl_size *= 16;
+		cl->cl_size += (c - '0');
+	} else if ((c >= 'A') && (c <= 'F')) {
+		cl->cl_size *= 16;
+		cl->cl_size += (c - 'A') + 10;
+	} else if ((c >= 'a') && (c <= 'f')) {
+		cl->cl_size *= 16;
+		cl->cl_size += (c - 'a') + 10;
+	} else if (c == ';') {
+		cl->cl_state = CS_EXT;
+	} else if (c == '\r') {
+		cl->cl_state = CS_CR;
+	} else {
+		return (NNG_EPROTO);
+	}
+	return (0);
+}
+
+static int
+chunk_ingest_ext(nni_http_chunks *cl, char c)
+{
+	if (c == '\r') {
+		cl->cl_state = CS_CR;
+	} else if (!isprint(c)) {
+		return (NNG_EPROTO);
+	}
+	return (0);
+}
+
+static int
+chunk_ingest_newline(nni_http_chunks *cl, char c)
+{
+	nni_http_chunk *chunk;
+
+	if (c != '\n') {
+		return (NNG_EPROTO);
+	}
+	if (cl->cl_size == 0) {
+		cl->cl_line  = 0;
+		cl->cl_state = CS_TRLR;
+		return (0);
+	}
+	if ((cl->cl_maxsz > 0) &&
+	    ((nni_http_chunks_size(cl) + cl->cl_size) > cl->cl_maxsz)) {
+		return (NNG_EMSGSIZE);
+	}
+	if ((chunk = NNI_ALLOC_STRUCT(chunk)) == NULL) {
+		return (NNG_ENOMEM);
+	}
+	// two extra bytes to accommodate trailing CRLF
+	if ((chunk->c_data = nni_alloc(cl->cl_size + 2)) == NULL) {
+		NNI_FREE_STRUCT(chunk);
+		return (NNG_ENOMEM);
+	}
+
+	// Data, so allocate a new chunk, stick it on the end of the list,
+	// and note that we have residual data needs.  The residual is
+	// to allow for the trailing CRLF to be consumed.
+	cl->cl_state   = CS_DATA;
+	chunk->c_size  = cl->cl_size;
+	chunk->c_alloc = cl->cl_size + 2;
+	chunk->c_resid = chunk->c_alloc;
+	nni_list_append(&cl->cl_chunks, chunk);
+
+	return (0);
+}
+
+static int
+chunk_ingest_trailer(nni_http_chunks *cl, char c)
+{
+	if (c == '\r') {
+		cl->cl_state = CS_TRLRCR;
+		return (0);
+	}
+	if (!isprint(c)) {
+		return (NNG_EPROTO);
+	}
+	cl->cl_line++;
+	return (0);
+}
+
+static int
+chunk_ingest_trailercr(nni_http_chunks *cl, char c)
+{
+	if (c != '\n') {
+		return (NNG_EPROTO);
+	}
+	if (cl->cl_line == 0) {
+		cl->cl_state = CS_DONE;
+		return (0);
+	}
+	cl->cl_line  = 0;
+	cl->cl_state = CS_TRLR;
+	return (0);
+}
+
+static int
+chunk_ingest_char(nni_http_chunks *cl, char c)
+{
+	int rv;
+	switch (cl->cl_state) {
+	case CS_INIT:
+		if (!isalnum(c)) {
+			rv = NNG_EPROTO;
+			break;
+		}
+		cl->cl_state = CS_LEN;
+		// fallthrough
+	case CS_LEN:
+		rv = chunk_ingest_len(cl, c);
+		break;
+	case CS_EXT:
+		rv = chunk_ingest_ext(cl, c);
+		break;
+	case CS_CR:
+		rv = chunk_ingest_newline(cl, c);
+		break;
+	case CS_TRLR:
+		rv = chunk_ingest_trailer(cl, c);
+		break;
+	case CS_TRLRCR:
+		rv = chunk_ingest_trailercr(cl, c);
+		break;
+	default:
+		// NB: No support for CS_DATA here, as that is handled
+		// in the caller for reasons of efficiency.
+		rv = NNG_EPROTO;
+		break;
+	}
+
+	return (rv);
+}
+
+static int
+chunk_ingest_data(nni_http_chunks *cl, char *buf, size_t n, size_t *lenp)
+{
+	nni_http_chunk *chunk;
+	size_t          offset;
+	char *          dest;
+
+	chunk = nni_list_last(&cl->cl_chunks);
+
+	NNI_ASSERT(chunk != NULL);
+	NNI_ASSERT(cl->cl_state == CS_DATA);
+	NNI_ASSERT(chunk->c_resid <= chunk->c_alloc);
+	NNI_ASSERT(chunk->c_alloc > 2); // not be zero, plus newlines
+
+	dest   = chunk->c_data;
+	offset = chunk->c_alloc - chunk->c_resid;
+	dest += offset;
+
+	if (n >= chunk->c_resid) {
+		n = chunk->c_resid;
+		memcpy(dest, buf, n);
+
+		if ((chunk->c_data[chunk->c_size] != '\r') ||
+		    (chunk->c_data[chunk->c_size + 1] != '\n')) {
+			return (NNG_EPROTO);
+		}
+		chunk->c_resid = 0;
+		cl->cl_state   = CS_INIT;
+		cl->cl_size    = 0;
+		cl->cl_line    = 0;
+		*lenp          = n;
+		return (0);
+	}
+
+	memcpy(dest, buf, n);
+	chunk->c_resid -= n;
+	*lenp = n;
+	return (0);
+}
+
+int
+nni_http_chunks_parse(nni_http_chunks *cl, void *buf, size_t n, size_t *lenp)
+{
+	size_t i   = 0;
+	char * src = buf;
+
+	// Format of this data is <hexdigits> [ ; <ascii> CRLF ]
+	// The <ascii> are chunk extensions, and we don't support any.
+
+	while ((cl->cl_state != CS_DONE) && (i < n)) {
+		int    rv;
+		size_t cnt;
+		switch (cl->cl_state) {
+		case CS_DONE:
+			// Completed parse!
+			break;
+
+		case CS_DATA:
+			if ((rv = chunk_ingest_data(cl, src + i, n, &cnt)) !=
+			    0) {
+				return (rv);
+			}
+			i += cnt;
+			break;
+
+		default:
+			// All others character by character parse through
+			// the state machine grinder.
+			if ((rv = chunk_ingest_char(cl, src[i])) != 0) {
+				return (rv);
+			}
+			i++;
+			break;
+		}
+	}
+
+	*lenp = i;
+	if (cl->cl_state != CS_DONE) {
+		return (NNG_EAGAIN);
+	}
+	return (0);
+}
diff --git a/src/supplemental/http/http_client.c b/src/supplemental/http/http_client.c
index a61f7884..a8260705 100644
--- a/src/supplemental/http/http_client.c
+++ b/src/supplemental/http/http_client.c
@@ -282,6 +282,7 @@ typedef enum http_txn_state {
 	HTTP_SENDING,
 	HTTP_RECVING,
 	HTTP_RECVING_BODY,
+	HTTP_RECVING_CHUNKS,
 } http_txn_state;
 
 typedef struct http_txn {
@@ -291,6 +292,7 @@ typedef struct http_txn {
 	nni_http_conn *  conn;
 	nni_http_req *   req;
 	nni_http_res *   res;
+	nni_http_chunks *chunks;
 	http_txn_state   state;
 	nni_reap_item    reap;
 } http_txn;
@@ -306,26 +308,36 @@ http_txn_reap(void *arg)
 			txn->conn = NULL;
 		}
 	}
+	nni_http_chunks_free(txn->chunks);
 	nni_aio_fini(txn->aio);
 	NNI_FREE_STRUCT(txn);
 }
 
 static void
+http_txn_finish_aios(http_txn *txn, int rv)
+{
+	nni_aio *aio;
+	while ((aio = nni_list_first(&txn->aios)) != NULL) {
+		nni_list_remove(&txn->aios, aio);
+		nni_aio_finish_error(aio, rv);
+	}
+}
+
+static void
 http_txn_cb(void *arg)
 {
-	http_txn *  txn = arg;
-	const char *str;
-	nni_aio *   aio;
-	int         rv;
-	uint64_t    len;
-	nni_iov     iov;
+	http_txn *      txn = arg;
+	const char *    str;
+	int             rv;
+	uint64_t        len;
+	nni_iov         iov;
+	char *          dst;
+	size_t          sz;
+	nni_http_chunk *chunk = NULL;
 
 	nni_mtx_lock(&http_txn_lk);
 	if ((rv = nni_aio_result(txn->aio)) != 0) {
-		while ((aio = nni_list_first(&txn->aios)) != NULL) {
-			nni_list_remove(&txn->aios, aio);
-			nni_aio_finish_error(aio, rv);
-		}
+		http_txn_finish_aios(txn, rv);
 		nni_mtx_unlock(&http_txn_lk);
 		nni_reap(&txn->reap, http_txn_reap, txn);
 		return;
@@ -345,22 +357,22 @@ http_txn_cb(void *arg)
 		return;
 
 	case HTTP_RECVING:
+
+		// Detect chunked encoding.  You poor bastard.
 		if (((str = nni_http_res_get_header(
 		          txn->res, "Transfer-Encoding")) != NULL) &&
 		    (strstr(str, "chunked") != NULL)) {
-			// We refuse to receive chunked encoding data.
-			// This is an implementation limitation, but as HTTP/2
-			// has eliminated this encoding, maybe it's not that
-			// big of a deal.  We forcibly close this.
-			while ((aio = nni_list_first(&txn->aios)) != NULL) {
-				nni_list_remove(&txn->aios, aio);
-				nni_aio_finish_error(aio, NNG_ENOTSUP);
+
+			if ((rv = nni_http_chunks_init(&txn->chunks, 0)) !=
+			    0) {
+				goto error;
 			}
-			nni_http_conn_close(txn->conn);
+			txn->state = HTTP_RECVING_CHUNKS;
+			nni_http_read_chunks(txn->conn, txn->chunks, txn->aio);
 			nni_mtx_unlock(&http_txn_lk);
-			nni_reap(&txn->reap, http_txn_reap, txn);
 			return;
 		}
+
 		str = nni_http_req_get_method(txn->req);
 		if ((nni_strcasecmp(str, "HEAD") == 0) ||
 		    ((str = nni_http_res_get_header(
@@ -368,16 +380,16 @@ http_txn_cb(void *arg)
 		    (nni_strtou64(str, &len) != 0) || (len == 0)) {
 			// If no content-length, or HEAD (which per RFC
 			// never transfers data), then we are done.
-			while ((aio = nni_list_first(&txn->aios)) != NULL) {
-				nni_list_remove(&txn->aios, aio);
-				nni_aio_finish(aio, 0, 0);
-			}
+			http_txn_finish_aios(txn, 0);
 			nni_mtx_unlock(&http_txn_lk);
 			nni_reap(&txn->reap, http_txn_reap, txn);
 			return;
 		}
 
-		nni_http_res_alloc_data(txn->res, (size_t) len);
+		if ((rv = nni_http_res_alloc_data(txn->res, (size_t) len)) !=
+		    0) {
+			goto error;
+		}
 		nni_http_res_get_data(txn->res, &iov.iov_buf, &iov.iov_len);
 		nni_aio_set_iov(txn->aio, 1, &iov);
 		txn->state = HTTP_RECVING_BODY;
@@ -387,16 +399,36 @@ http_txn_cb(void *arg)
 
 	case HTTP_RECVING_BODY:
 		// All done!
-		while ((aio = nni_list_first(&txn->aios)) != NULL) {
-			nni_list_remove(&txn->aios, aio);
-			nni_aio_finish(aio, 0, 0);
+		http_txn_finish_aios(txn, 0);
+		nni_mtx_unlock(&http_txn_lk);
+		nni_reap(&txn->reap, http_txn_reap, txn);
+		return;
+
+	case HTTP_RECVING_CHUNKS:
+		// All done, but now we need to coalesce the chunks, for
+		// yet *another* copy.  Chunked transfers are such crap.
+		sz = nni_http_chunks_size(txn->chunks);
+		if ((rv = nni_http_res_alloc_data(txn->res, sz)) != 0) {
+			goto error;
 		}
+		nni_http_res_get_data(txn->res, (void **) &dst, &sz);
+		while ((chunk = nni_http_chunks_iter(txn->chunks, chunk)) !=
+		    NULL) {
+			memcpy(dst, nni_http_chunk_data(chunk),
+			    nni_http_chunk_size(chunk));
+			dst += nni_http_chunk_size(chunk);
+		}
+		http_txn_finish_aios(txn, 0);
 		nni_mtx_unlock(&http_txn_lk);
 		nni_reap(&txn->reap, http_txn_reap, txn);
 		return;
 	}
 
-	NNI_ASSERT(0); // Unknown state!
+error:
+	http_txn_finish_aios(txn, rv);
+	nni_http_conn_close(txn->conn);
+	nni_mtx_unlock(&http_txn_lk);
+	nni_reap(&txn->reap, http_txn_reap, txn);
 }
 
 static void
diff --git a/src/supplemental/http/http_conn.c b/src/supplemental/http/http_conn.c
index b17b02cf..d00bd910 100644
--- a/src/supplemental/http/http_conn.c
+++ b/src/supplemental/http/http_conn.c
@@ -8,6 +8,7 @@
 // found online at https://opensource.org/licenses/MIT.
 //
 
+#include <ctype.h>
 #include <stdbool.h>
 #include <string.h>
 
@@ -27,6 +28,7 @@ enum read_flavor {
 	HTTP_RD_FULL,
 	HTTP_RD_REQ,
 	HTTP_RD_RES,
+	HTTP_RD_CHUNK,
 };
 
 enum write_flavor {
@@ -235,6 +237,23 @@ http_rd_buf(nni_http_conn *conn, nni_aio *aio)
 			conn->rd(conn->sock, conn->rd_aio);
 		}
 		return (rv);
+
+	case HTTP_RD_CHUNK:
+		rv = nni_http_chunks_parse(
+		    nni_aio_get_prov_extra(aio, 1), rbuf, cnt, &n);
+		conn->rd_get += n;
+		if (conn->rd_get == conn->rd_put) {
+			conn->rd_get = conn->rd_put = 0;
+		}
+		if (rv == NNG_EAGAIN) {
+			nni_iov iov1;
+			iov1.iov_buf = conn->rd_buf + conn->rd_put;
+			iov1.iov_len = conn->rd_bufsz - conn->rd_put;
+			nni_aio_set_iov(conn->rd_aio, 1, &iov1);
+			nni_aio_set_data(conn->rd_aio, 1, aio);
+			conn->rd(conn->sock, conn->rd_aio);
+		}
+		return (rv);
 	}
 	return (NNG_EINVAL);
 }
@@ -531,6 +550,17 @@ nni_http_read_res(nni_http_conn *conn, nni_http_res *res, nni_aio *aio)
 }
 
 void
+nni_http_read_chunks(nni_http_conn *conn, nni_http_chunks *cl, nni_aio *aio)
+{
+	SET_RD_FLAVOR(aio, HTTP_RD_CHUNK);
+	nni_aio_set_prov_extra(aio, 1, cl);
+
+	nni_mtx_lock(&conn->mtx);
+	http_rd_submit(conn, aio);
+	nni_mtx_unlock(&conn->mtx);
+}
+
+void
 nni_http_read_full(nni_http_conn *conn, nni_aio *aio)
 {
 	SET_RD_FLAVOR(aio, HTTP_RD_FULL);