diff options
| -rw-r--r-- | docs/man/nng_http_conn_transact.3http.adoc | 7 | ||||
| -rw-r--r-- | docs/man/nng_http_handler_collect_body.3http.adoc | 2 | ||||
| -rw-r--r-- | src/supplemental/http/CMakeLists.txt | 1 | ||||
| -rw-r--r-- | src/supplemental/http/http_api.h | 29 | ||||
| -rw-r--r-- | src/supplemental/http/http_chunk.c | 341 | ||||
| -rw-r--r-- | src/supplemental/http/http_client.c | 88 | ||||
| -rw-r--r-- | src/supplemental/http/http_conn.c | 30 | ||||
| -rw-r--r-- | tests/httpclient.c | 49 |
8 files changed, 512 insertions, 35 deletions
diff --git a/docs/man/nng_http_conn_transact.3http.adoc b/docs/man/nng_http_conn_transact.3http.adoc index bd41f659..a38592b6 100644 --- a/docs/man/nng_http_conn_transact.3http.adoc +++ b/docs/man/nng_http_conn_transact.3http.adoc @@ -48,11 +48,6 @@ exists. That function behaves similarily, but creates a connection on demand for the transaction, and disposes of it when finished. -NOTE: This function does not support reading data sent using chunked -transfer encoding, and if the server attempts to do so, the underlying -connection will be closed and an `NNG_ENOTSUP` error will be returned. -This limitation is considered a bug, and a fix is planned for the future. - WARNING: If the remote server tries to send an extremely large buffer, then a corresponding allocation will be made, which can lead to denial of service attacks. @@ -79,7 +74,7 @@ None. `NNG_ECLOSED`:: The connection was closed. `NNG_ECONNRESET`:: The peer closed the connection. `NNG_ENOMEM`:: Insufficient free memory to perform the operation. -`NNG_ENOTSUP`:: HTTP operations are not supported, or peer sent chunked encoding. +`NNG_ENOTSUP`:: HTTP operations are not supported. `NNG_EPROTO`:: An HTTP protocol error occurred. `NNG_ETIMEDOUT`:: Timeout waiting for data from the connection. diff --git a/docs/man/nng_http_handler_collect_body.3http.adoc b/docs/man/nng_http_handler_collect_body.3http.adoc index 875f32c4..273f8dc6 100644 --- a/docs/man/nng_http_handler_collect_body.3http.adoc +++ b/docs/man/nng_http_handler_collect_body.3http.adoc @@ -58,7 +58,7 @@ If this header is absent, the request is assumed not to contain any data. NOTE: This specifically does not support the `Chunked` transfer-encoding. This is considered a bug, and is a deficiency for full HTTP/1.1 compliance. However, few clients send data in this format, so in practice this should -not create few limitations. +create few limitations. == RETURN VALUES diff --git a/src/supplemental/http/CMakeLists.txt b/src/supplemental/http/CMakeLists.txt index 98334109..ed7d5ae2 100644 --- a/src/supplemental/http/CMakeLists.txt +++ b/src/supplemental/http/CMakeLists.txt @@ -16,6 +16,7 @@ if (NNG_SUPP_HTTP) supplemental/http/http.h supplemental/http/http_api.h supplemental/http/http_client.c + supplemental/http/http_chunk.c supplemental/http/http_conn.c supplemental/http/http_msg.c supplemental/http/http_public.c diff --git a/src/supplemental/http/http_api.h b/src/supplemental/http/http_api.h index 14e842be..fdee70e9 100644 --- a/src/supplemental/http/http_api.h +++ b/src/supplemental/http/http_api.h @@ -27,6 +27,8 @@ typedef struct nng_http_conn nni_http_conn; typedef struct nng_http_handler nni_http_handler; typedef struct nng_http_server nni_http_server; typedef struct nng_http_client nni_http_client; +typedef struct nng_http_chunk nni_http_chunk; +typedef struct nng_http_chunks nni_http_chunks; // These functions are private to the internal framework, and really should // not be used elsewhere. @@ -46,6 +48,33 @@ extern int nni_http_res_parse(nni_http_res *, void *, size_t, size_t *); extern void nni_http_res_get_data(nni_http_res *, void **, size_t *); extern char *nni_http_res_headers(nni_http_res *); +// Chunked transfer encoding. For the moment this is not part of our public +// API. We can change that later. + +// nni_http_chunk_list_init creates a list of chunks, which shall not exceed +// the specified overall size. (Size 0 means no limit.) +extern int nni_http_chunks_init(nni_http_chunks **, size_t); + +extern void nni_http_chunks_free(nni_http_chunks *); + +// nni_http_chunk_iter iterates over all chunks in the list. +// Pass NULL for the last chunk to start at the head. Returns NULL when done. +extern nni_http_chunk *nni_http_chunks_iter( + nni_http_chunks *, nni_http_chunk *); + +// nni_http_chunk_list_size returns the combined size of all chunks in list. +extern size_t nni_http_chunks_size(nni_http_chunks *); + +// nni_http_chunk_size returns the size of given chunk. +extern size_t nni_http_chunk_size(nni_http_chunk *); +// nni_http_chunk_data returns a pointer to the data. +extern void *nni_http_chunk_data(nni_http_chunk *); + +extern int nni_http_chunks_parse(nni_http_chunks *, void *, size_t, size_t *); + +extern void nni_http_read_chunks( + nni_http_conn *, nni_http_chunks *, nni_aio *); + // Private to the server. (Used to support session hijacking.) extern void nni_http_conn_set_ctx(nni_http_conn *, void *); extern void *nni_http_conn_get_ctx(nni_http_conn *); diff --git a/src/supplemental/http/http_chunk.c b/src/supplemental/http/http_chunk.c new file mode 100644 index 00000000..9333548e --- /dev/null +++ b/src/supplemental/http/http_chunk.c @@ -0,0 +1,341 @@ +// +// Copyright 2018 Staysail Systems, Inc. <info@staysail.tech> +// Copyright 2018 Capitar IT Group BV <info@capitar.com> +// +// This software is supplied under the terms of the MIT License, a +// copy of which should be located in the distribution where this +// file was obtained (LICENSE.txt). A copy of the license may also be +// found online at https://opensource.org/licenses/MIT. +// + +#include <ctype.h> +#include <stdbool.h> +#include <string.h> + +#include "core/nng_impl.h" + +#include "http_api.h" + +// Chunked transfer encoding support. + +// Note that HTTP/1.1 chunked transfer encoding is horrible, and should +// be avoided if at all possible. It necessarily creates extra need for +// data copies, creates a lot of extra back and forth complexity. If you're +// stuck in this code, we feel great sympathy for you. +// +// We feel strongly enough about this that we refuse to provide any +// method to automatically generate chunked transfers. If you think +// you need to send chunked transfers (because you have no idea how +// much data you will send, such as a streaming workload), consider a +// different method such as WebSocket to send your data. Unbounded +// entity body data is just impolite. + +enum chunk_state { + CS_INIT, // initial state + CS_LEN, // length + CS_EXT, // random extension text (we ignore) + CS_CR, // carriage return after length (and extensions) + CS_DATA, // actual data + CS_TRLR, // trailer + CS_TRLRCR, // CRLF at end of trailer + CS_DONE, +}; + +struct nng_http_chunks { + nni_list cl_chunks; + size_t cl_maxsz; + size_t cl_size; // parsed size (so far) + size_t cl_line; // bytes since last newline + enum chunk_state cl_state; +}; + +struct nng_http_chunk { + nni_list_node c_node; + size_t c_size; + size_t c_alloc; + size_t c_resid; // residual data to transfer + char * c_data; +}; + +int +nni_http_chunks_init(nni_http_chunks **clp, size_t maxsz) +{ + nni_http_chunks *cl; + + if ((cl = NNI_ALLOC_STRUCT(cl)) == NULL) { + return (NNG_ENOMEM); + } + NNI_LIST_INIT(&cl->cl_chunks, nni_http_chunk, c_node); + cl->cl_maxsz = maxsz; + *clp = cl; + return (0); +} + +void +nni_http_chunks_free(nni_http_chunks *cl) +{ + nni_http_chunk *ch; + if (cl == NULL) { + return; + } + while ((ch = nni_list_first(&cl->cl_chunks)) != NULL) { + nni_list_remove(&cl->cl_chunks, ch); + if (ch->c_data != NULL) { + nni_free(ch->c_data, ch->c_alloc); + } + NNI_FREE_STRUCT(ch); + } + NNI_FREE_STRUCT(cl); +} + +nni_http_chunk * +nni_http_chunks_iter(nni_http_chunks *cl, nni_http_chunk *last) +{ + if (last == NULL) { + return (nni_list_first(&cl->cl_chunks)); + } + return (nni_list_next(&cl->cl_chunks, last)); +} + +size_t +nni_http_chunks_size(nni_http_chunks *cl) +{ + size_t tot = 0; + nni_http_chunk *ch; + NNI_LIST_FOREACH (&cl->cl_chunks, ch) { + tot += ch->c_size; + } + return (tot); +} + +size_t +nni_http_chunk_size(nni_http_chunk *ch) +{ + return (ch->c_size); +} + +void * +nni_http_chunk_data(nni_http_chunk *ch) +{ + return (ch->c_data); +} + +static int +chunk_ingest_len(nni_http_chunks *cl, char c) +{ + if (isdigit(c)) { + cl->cl_size *= 16; + cl->cl_size += (c - '0'); + } else if ((c >= 'A') && (c <= 'F')) { + cl->cl_size *= 16; + cl->cl_size += (c - 'A') + 10; + } else if ((c >= 'a') && (c <= 'f')) { + cl->cl_size *= 16; + cl->cl_size += (c - 'a') + 10; + } else if (c == ';') { + cl->cl_state = CS_EXT; + } else if (c == '\r') { + cl->cl_state = CS_CR; + } else { + return (NNG_EPROTO); + } + return (0); +} + +static int +chunk_ingest_ext(nni_http_chunks *cl, char c) +{ + if (c == '\r') { + cl->cl_state = CS_CR; + } else if (!isprint(c)) { + return (NNG_EPROTO); + } + return (0); +} + +static int +chunk_ingest_newline(nni_http_chunks *cl, char c) +{ + nni_http_chunk *chunk; + + if (c != '\n') { + return (NNG_EPROTO); + } + if (cl->cl_size == 0) { + cl->cl_line = 0; + cl->cl_state = CS_TRLR; + return (0); + } + if ((cl->cl_maxsz > 0) && + ((nni_http_chunks_size(cl) + cl->cl_size) > cl->cl_maxsz)) { + return (NNG_EMSGSIZE); + } + if ((chunk = NNI_ALLOC_STRUCT(chunk)) == NULL) { + return (NNG_ENOMEM); + } + // two extra bytes to accommodate trailing CRLF + if ((chunk->c_data = nni_alloc(cl->cl_size + 2)) == NULL) { + NNI_FREE_STRUCT(chunk); + return (NNG_ENOMEM); + } + + // Data, so allocate a new chunk, stick it on the end of the list, + // and note that we have residual data needs. The residual is + // to allow for the trailing CRLF to be consumed. + cl->cl_state = CS_DATA; + chunk->c_size = cl->cl_size; + chunk->c_alloc = cl->cl_size + 2; + chunk->c_resid = chunk->c_alloc; + nni_list_append(&cl->cl_chunks, chunk); + + return (0); +} + +static int +chunk_ingest_trailer(nni_http_chunks *cl, char c) +{ + if (c == '\r') { + cl->cl_state = CS_TRLRCR; + return (0); + } + if (!isprint(c)) { + return (NNG_EPROTO); + } + cl->cl_line++; + return (0); +} + +static int +chunk_ingest_trailercr(nni_http_chunks *cl, char c) +{ + if (c != '\n') { + return (NNG_EPROTO); + } + if (cl->cl_line == 0) { + cl->cl_state = CS_DONE; + return (0); + } + cl->cl_line = 0; + cl->cl_state = CS_TRLR; + return (0); +} + +static int +chunk_ingest_char(nni_http_chunks *cl, char c) +{ + int rv; + switch (cl->cl_state) { + case CS_INIT: + if (!isalnum(c)) { + rv = NNG_EPROTO; + break; + } + cl->cl_state = CS_LEN; + // fallthrough + case CS_LEN: + rv = chunk_ingest_len(cl, c); + break; + case CS_EXT: + rv = chunk_ingest_ext(cl, c); + break; + case CS_CR: + rv = chunk_ingest_newline(cl, c); + break; + case CS_TRLR: + rv = chunk_ingest_trailer(cl, c); + break; + case CS_TRLRCR: + rv = chunk_ingest_trailercr(cl, c); + break; + default: + // NB: No support for CS_DATA here, as that is handled + // in the caller for reasons of efficiency. + rv = NNG_EPROTO; + break; + } + + return (rv); +} + +static int +chunk_ingest_data(nni_http_chunks *cl, char *buf, size_t n, size_t *lenp) +{ + nni_http_chunk *chunk; + size_t offset; + char * dest; + + chunk = nni_list_last(&cl->cl_chunks); + + NNI_ASSERT(chunk != NULL); + NNI_ASSERT(cl->cl_state == CS_DATA); + NNI_ASSERT(chunk->c_resid <= chunk->c_alloc); + NNI_ASSERT(chunk->c_alloc > 2); // not be zero, plus newlines + + dest = chunk->c_data; + offset = chunk->c_alloc - chunk->c_resid; + dest += offset; + + if (n >= chunk->c_resid) { + n = chunk->c_resid; + memcpy(dest, buf, n); + + if ((chunk->c_data[chunk->c_size] != '\r') || + (chunk->c_data[chunk->c_size + 1] != '\n')) { + return (NNG_EPROTO); + } + chunk->c_resid = 0; + cl->cl_state = CS_INIT; + cl->cl_size = 0; + cl->cl_line = 0; + *lenp = n; + return (0); + } + + memcpy(dest, buf, n); + chunk->c_resid -= n; + *lenp = n; + return (0); +} + +int +nni_http_chunks_parse(nni_http_chunks *cl, void *buf, size_t n, size_t *lenp) +{ + size_t i = 0; + char * src = buf; + + // Format of this data is <hexdigits> [ ; <ascii> CRLF ] + // The <ascii> are chunk extensions, and we don't support any. + + while ((cl->cl_state != CS_DONE) && (i < n)) { + int rv; + size_t cnt; + switch (cl->cl_state) { + case CS_DONE: + // Completed parse! + break; + + case CS_DATA: + if ((rv = chunk_ingest_data(cl, src + i, n, &cnt)) != + 0) { + return (rv); + } + i += cnt; + break; + + default: + // All others character by character parse through + // the state machine grinder. + if ((rv = chunk_ingest_char(cl, src[i])) != 0) { + return (rv); + } + i++; + break; + } + } + + *lenp = i; + if (cl->cl_state != CS_DONE) { + return (NNG_EAGAIN); + } + return (0); +} diff --git a/src/supplemental/http/http_client.c b/src/supplemental/http/http_client.c index a61f7884..a8260705 100644 --- a/src/supplemental/http/http_client.c +++ b/src/supplemental/http/http_client.c @@ -282,6 +282,7 @@ typedef enum http_txn_state { HTTP_SENDING, HTTP_RECVING, HTTP_RECVING_BODY, + HTTP_RECVING_CHUNKS, } http_txn_state; typedef struct http_txn { @@ -291,6 +292,7 @@ typedef struct http_txn { nni_http_conn * conn; nni_http_req * req; nni_http_res * res; + nni_http_chunks *chunks; http_txn_state state; nni_reap_item reap; } http_txn; @@ -306,26 +308,36 @@ http_txn_reap(void *arg) txn->conn = NULL; } } + nni_http_chunks_free(txn->chunks); nni_aio_fini(txn->aio); NNI_FREE_STRUCT(txn); } static void +http_txn_finish_aios(http_txn *txn, int rv) +{ + nni_aio *aio; + while ((aio = nni_list_first(&txn->aios)) != NULL) { + nni_list_remove(&txn->aios, aio); + nni_aio_finish_error(aio, rv); + } +} + +static void http_txn_cb(void *arg) { - http_txn * txn = arg; - const char *str; - nni_aio * aio; - int rv; - uint64_t len; - nni_iov iov; + http_txn * txn = arg; + const char * str; + int rv; + uint64_t len; + nni_iov iov; + char * dst; + size_t sz; + nni_http_chunk *chunk = NULL; nni_mtx_lock(&http_txn_lk); if ((rv = nni_aio_result(txn->aio)) != 0) { - while ((aio = nni_list_first(&txn->aios)) != NULL) { - nni_list_remove(&txn->aios, aio); - nni_aio_finish_error(aio, rv); - } + http_txn_finish_aios(txn, rv); nni_mtx_unlock(&http_txn_lk); nni_reap(&txn->reap, http_txn_reap, txn); return; @@ -345,22 +357,22 @@ http_txn_cb(void *arg) return; case HTTP_RECVING: + + // Detect chunked encoding. You poor bastard. if (((str = nni_http_res_get_header( txn->res, "Transfer-Encoding")) != NULL) && (strstr(str, "chunked") != NULL)) { - // We refuse to receive chunked encoding data. - // This is an implementation limitation, but as HTTP/2 - // has eliminated this encoding, maybe it's not that - // big of a deal. We forcibly close this. - while ((aio = nni_list_first(&txn->aios)) != NULL) { - nni_list_remove(&txn->aios, aio); - nni_aio_finish_error(aio, NNG_ENOTSUP); + + if ((rv = nni_http_chunks_init(&txn->chunks, 0)) != + 0) { + goto error; } - nni_http_conn_close(txn->conn); + txn->state = HTTP_RECVING_CHUNKS; + nni_http_read_chunks(txn->conn, txn->chunks, txn->aio); nni_mtx_unlock(&http_txn_lk); - nni_reap(&txn->reap, http_txn_reap, txn); return; } + str = nni_http_req_get_method(txn->req); if ((nni_strcasecmp(str, "HEAD") == 0) || ((str = nni_http_res_get_header( @@ -368,16 +380,16 @@ http_txn_cb(void *arg) (nni_strtou64(str, &len) != 0) || (len == 0)) { // If no content-length, or HEAD (which per RFC // never transfers data), then we are done. - while ((aio = nni_list_first(&txn->aios)) != NULL) { - nni_list_remove(&txn->aios, aio); - nni_aio_finish(aio, 0, 0); - } + http_txn_finish_aios(txn, 0); nni_mtx_unlock(&http_txn_lk); nni_reap(&txn->reap, http_txn_reap, txn); return; } - nni_http_res_alloc_data(txn->res, (size_t) len); + if ((rv = nni_http_res_alloc_data(txn->res, (size_t) len)) != + 0) { + goto error; + } nni_http_res_get_data(txn->res, &iov.iov_buf, &iov.iov_len); nni_aio_set_iov(txn->aio, 1, &iov); txn->state = HTTP_RECVING_BODY; @@ -387,16 +399,36 @@ http_txn_cb(void *arg) case HTTP_RECVING_BODY: // All done! - while ((aio = nni_list_first(&txn->aios)) != NULL) { - nni_list_remove(&txn->aios, aio); - nni_aio_finish(aio, 0, 0); + http_txn_finish_aios(txn, 0); + nni_mtx_unlock(&http_txn_lk); + nni_reap(&txn->reap, http_txn_reap, txn); + return; + + case HTTP_RECVING_CHUNKS: + // All done, but now we need to coalesce the chunks, for + // yet *another* copy. Chunked transfers are such crap. + sz = nni_http_chunks_size(txn->chunks); + if ((rv = nni_http_res_alloc_data(txn->res, sz)) != 0) { + goto error; } + nni_http_res_get_data(txn->res, (void **) &dst, &sz); + while ((chunk = nni_http_chunks_iter(txn->chunks, chunk)) != + NULL) { + memcpy(dst, nni_http_chunk_data(chunk), + nni_http_chunk_size(chunk)); + dst += nni_http_chunk_size(chunk); + } + http_txn_finish_aios(txn, 0); nni_mtx_unlock(&http_txn_lk); nni_reap(&txn->reap, http_txn_reap, txn); return; } - NNI_ASSERT(0); // Unknown state! +error: + http_txn_finish_aios(txn, rv); + nni_http_conn_close(txn->conn); + nni_mtx_unlock(&http_txn_lk); + nni_reap(&txn->reap, http_txn_reap, txn); } static void diff --git a/src/supplemental/http/http_conn.c b/src/supplemental/http/http_conn.c index b17b02cf..d00bd910 100644 --- a/src/supplemental/http/http_conn.c +++ b/src/supplemental/http/http_conn.c @@ -8,6 +8,7 @@ // found online at https://opensource.org/licenses/MIT. // +#include <ctype.h> #include <stdbool.h> #include <string.h> @@ -27,6 +28,7 @@ enum read_flavor { HTTP_RD_FULL, HTTP_RD_REQ, HTTP_RD_RES, + HTTP_RD_CHUNK, }; enum write_flavor { @@ -235,6 +237,23 @@ http_rd_buf(nni_http_conn *conn, nni_aio *aio) conn->rd(conn->sock, conn->rd_aio); } return (rv); + + case HTTP_RD_CHUNK: + rv = nni_http_chunks_parse( + nni_aio_get_prov_extra(aio, 1), rbuf, cnt, &n); + conn->rd_get += n; + if (conn->rd_get == conn->rd_put) { + conn->rd_get = conn->rd_put = 0; + } + if (rv == NNG_EAGAIN) { + nni_iov iov1; + iov1.iov_buf = conn->rd_buf + conn->rd_put; + iov1.iov_len = conn->rd_bufsz - conn->rd_put; + nni_aio_set_iov(conn->rd_aio, 1, &iov1); + nni_aio_set_data(conn->rd_aio, 1, aio); + conn->rd(conn->sock, conn->rd_aio); + } + return (rv); } return (NNG_EINVAL); } @@ -531,6 +550,17 @@ nni_http_read_res(nni_http_conn *conn, nni_http_res *res, nni_aio *aio) } void +nni_http_read_chunks(nni_http_conn *conn, nni_http_chunks *cl, nni_aio *aio) +{ + SET_RD_FLAVOR(aio, HTTP_RD_CHUNK); + nni_aio_set_prov_extra(aio, 1, cl); + + nni_mtx_lock(&conn->mtx); + http_rd_submit(conn, aio); + nni_mtx_unlock(&conn->mtx); +} + +void nni_http_read_full(nni_http_conn *conn, nni_aio *aio) { SET_RD_FLAVOR(aio, HTTP_RD_FULL); diff --git a/tests/httpclient.c b/tests/httpclient.c index 6964bcc2..75ecbae9 100644 --- a/tests/httpclient.c +++ b/tests/httpclient.c @@ -26,6 +26,10 @@ const uint8_t example_sum[20] = { 0x0e, 0x97, 0x3b, 0x59, 0xf4, 0x76, 0x00, 0x7f, 0xd1, 0x0f, 0x87, 0xf3, 0x47, 0xc3, 0x95, 0x60, 0x65, 0x51, 0x6f, 0xc0 }; +const uint8_t chunked_sum[20] = { 0x9b, 0x06, 0xfb, 0xee, 0x51, 0xc6, 0x42, + 0x69, 0x1c, 0xb3, 0xaa, 0x38, 0xce, 0xb8, 0x0b, 0x3a, 0xc8, 0x3b, 0x96, + 0x68 }; + TestMain("HTTP Client", { atexit(nng_fini); @@ -208,4 +212,49 @@ TestMain("HTTP Client", { So(memcmp(digest, example_sum, 20) == 0); }); }); + + Convey("Given a client (chunked)", { + nng_aio * aio; + nng_http_client *cli; + nng_url * url; + + So(nng_aio_alloc(&aio, NULL, NULL) == 0); + + So(nng_url_parse(&url, + "http://anglesharp.azurewebsites.net/Chunked") == 0); + // "https://jigsaw.w3.org/HTTP/ChunkedScript") + //== 0); + + So(nng_http_client_alloc(&cli, url) == 0); + nng_aio_set_timeout(aio, 10000); // 10 sec timeout + + Reset({ + nng_http_client_free(cli); + nng_url_free(url); + nng_aio_free(aio); + }); + + Convey("One off exchange works", { + nng_http_req *req; + nng_http_res *res; + void * data; + size_t len; + uint8_t digest[20]; + + So(nng_http_req_alloc(&req, url) == 0); + So(nng_http_res_alloc(&res) == 0); + Reset({ + nng_http_req_free(req); + nng_http_res_free(res); + }); + + nng_http_client_transact(cli, req, res, aio); + nng_aio_wait(aio); + So(nng_aio_result(aio) == 0); + So(nng_http_res_get_status(res) == 200); + nng_http_res_get_data(res, &data, &len); + nni_sha1(data, len, digest); + So(memcmp(digest, chunked_sum, 20) == 0); + }); + }); }) |