http: The big HTTP API refactoring of January 2025.v2.0.0-alpha.3http-client-trans

This represents a major change in the HTTP code base, consisting
of a complete revamp of the HTTP API. The changes here are too
numerous to mention, but the end result should be a vastly
simpler API for both server and client applications.

Many needless allocations were removed by providing fixed buffers
for various parameters and headers when possible.

A few bugs were fixed. Most especially we have fixed some bugs
around very large URIs and headers, and we have also addressed
conformance bugs to more closely conform to RFCs 9110 and 9112.

As part of this work, the APIs for WebSockets changed slightly
as well.  In particular the properties available for accessing
headers have changed.

There is still documentation conversion work to do, and additional
functionality (such as proper support for chunked transfers), but
this is a big step in the right direction.

1 files changed, 55 insertions, 84 deletions

diff --git a/src/supplemental/http/http_client.c b/src/supplemental/http/http_client.c
index 8701a96b..7062ae3c 100644
--- a/src/supplemental/http/http_client.c
+++ b/src/supplemental/http/http_client.c
@@ -10,12 +10,14 @@
 //
 
 #include <stdbool.h>
+#include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
 #include "core/nng_impl.h"
 
 #include "http_api.h"
+#include "http_msg.h"
 
 static nni_mtx http_txn_lk = NNI_MTX_INITIALIZER;
 
@@ -24,6 +26,7 @@ struct nng_http_client {
 	nni_mtx            mtx;
 	bool               closed;
 	nni_aio            aio;
+	char               host[260];
 	nng_stream_dialer *dialer;
 };
 
@@ -70,7 +73,9 @@ http_dial_cb(void *arg)
 	stream = nni_aio_get_output(&c->aio, 0);
 	NNI_ASSERT(stream != NULL);
 
-	rv = nni_http_conn_init(&conn, stream);
+	rv = nni_http_conn_init(&conn, stream, true);
+
+	// set up the host header
 	http_dial_start(c);
 	nni_mtx_unlock(&c->mtx);
 
@@ -79,7 +84,7 @@ http_dial_cb(void *arg)
 		nni_aio_finish_error(aio, rv);
 		return;
 	}
-
+	nni_http_set_host(conn, c->host);
 	nni_aio_set_output(aio, 0, conn);
 	nni_aio_finish(aio, 0, 0);
 }
@@ -110,7 +115,8 @@ nni_http_client_init(nni_http_client **cp, const nng_url *url)
 	memcpy(&my_url, url, sizeof(my_url));
 	my_url.u_scheme = (char *) scheme;
 
-	if (strlen(url->u_hostname) == 0) {
+	if ((strlen(url->u_hostname) == 0) ||
+	    (strlen(url->u_hostname) > 253)) {
 		// We require a valid hostname.
 		return (NNG_EADDRINVAL);
 	}
@@ -122,6 +128,16 @@ nni_http_client_init(nni_http_client **cp, const nng_url *url)
 	nni_aio_list_init(&c->aios);
 	nni_aio_init(&c->aio, http_dial_cb, c);
 
+	if (nni_url_default_port(url->u_scheme) == url->u_port) {
+		snprintf(c->host, sizeof(c->host), "%s", url->u_hostname);
+	} else if (strchr(url->u_hostname, ':') != NULL) {
+		// IPv6 address, needs [wrapping]
+		snprintf(c->host, sizeof(c->host), "[%s]:%d", url->u_hostname,
+		    url->u_port);
+	} else {
+		snprintf(c->host, sizeof(c->host), "%s:%d", url->u_hostname,
+		    url->u_port);
+	}
 	if ((rv = nng_stream_dialer_alloc_url(&c->dialer, &my_url)) != 0) {
 		nni_http_client_fini(c);
 		return (rv);
@@ -190,7 +206,6 @@ nni_http_client_connect(nni_http_client *c, nni_aio *aio)
 }
 
 typedef enum http_txn_state {
-	HTTP_CONNECTING,
 	HTTP_SENDING,
 	HTTP_RECVING,
 	HTTP_RECVING_BODY,
@@ -198,7 +213,7 @@ typedef enum http_txn_state {
 } http_txn_state;
 
 typedef struct http_txn {
-	nni_aio         *aio;  // lower level aio
+	nni_aio          aio;  // lower level aio
 	nni_list         aios; // upper level aio(s) -- maximum one
 	nni_http_client *client;
 	nni_http_conn   *conn;
@@ -206,12 +221,15 @@ typedef struct http_txn {
 	nni_http_res    *res;
 	nni_http_chunks *chunks;
 	http_txn_state   state;
+	nni_reap_node    reap;
 } http_txn;
 
 static void
-http_txn_fini(void *arg)
+http_txn_reap(void *arg)
 {
 	http_txn *txn = arg;
+
+	nni_aio_stop(&txn->aio);
 	if (txn->client != NULL) {
 		// We only close the connection if we created it.
 		if (txn->conn != NULL) {
@@ -220,10 +238,21 @@ http_txn_fini(void *arg)
 		}
 	}
 	nni_http_chunks_free(txn->chunks);
-	nni_aio_reap(txn->aio);
+	nni_aio_fini(&txn->aio);
 	NNI_FREE_STRUCT(txn);
 }
 
+static nni_reap_list http_txn_reaplist = {
+	.rl_offset = offsetof(http_txn, reap),
+	.rl_func   = (nni_cb) http_txn_reap,
+};
+
+static void
+http_txn_fini(http_txn *txn)
+{
+	nni_reap(&http_txn_reaplist, txn);
+}
+
 static void
 http_txn_finish_aios(http_txn *txn, int rv)
 {
@@ -248,31 +277,26 @@ http_txn_cb(void *arg)
 	nni_http_chunk *chunk = NULL;
 
 	nni_mtx_lock(&http_txn_lk);
-	if ((rv = nni_aio_result(txn->aio)) != 0) {
+	if ((rv = nni_aio_result(&txn->aio)) != 0) {
 		http_txn_finish_aios(txn, rv);
 		nni_mtx_unlock(&http_txn_lk);
 		http_txn_fini(txn);
 		return;
 	}
 	switch (txn->state) {
-	case HTTP_CONNECTING:
-		txn->conn  = nni_aio_get_output(txn->aio, 0);
-		txn->state = HTTP_SENDING;
-		nni_http_write_req(txn->conn, txn->req, txn->aio);
-		nni_mtx_unlock(&http_txn_lk);
-		return;
-
 	case HTTP_SENDING:
 		txn->state = HTTP_RECVING;
-		nni_http_read_res(txn->conn, txn->res, txn->aio);
+		nni_http_read_res(txn->conn, &txn->aio);
 		nni_mtx_unlock(&http_txn_lk);
 		return;
 
 	case HTTP_RECVING:
 
-		// Detect chunked encoding.  You poor bastard.
-		if (((str = nni_http_res_get_header(
-		          txn->res, "Transfer-Encoding")) != NULL) &&
+		// Detect chunked encoding.  You poor bastard.  (Only if not
+		// HEAD.)
+		if ((strcmp(nni_http_get_method(txn->conn), "HEAD") != 0) &&
+		    ((str = nni_http_get_header(
+		          txn->conn, "Transfer-Encoding")) != NULL) &&
 		    (strstr(str, "chunked") != NULL)) {
 
 			if ((rv = nni_http_chunks_init(&txn->chunks, 0)) !=
@@ -280,15 +304,15 @@ http_txn_cb(void *arg)
 				goto error;
 			}
 			txn->state = HTTP_RECVING_CHUNKS;
-			nni_http_read_chunks(txn->conn, txn->chunks, txn->aio);
+			nni_http_read_chunks(
+			    txn->conn, txn->chunks, &txn->aio);
 			nni_mtx_unlock(&http_txn_lk);
 			return;
 		}
 
-		str = nni_http_req_get_method(txn->req);
-		if ((nni_strcasecmp(str, "HEAD") == 0) ||
-		    ((str = nni_http_res_get_header(
-		          txn->res, "Content-Length")) == NULL) ||
+		if ((strcmp(nni_http_get_method(txn->conn), "HEAD") == 0) ||
+		    ((str = nni_http_get_header(
+		          txn->conn, "Content-Length")) == NULL) ||
 		    ((len = (uint64_t) strtoull(str, &end, 10)) == 0) ||
 		    (end == NULL) || (*end != '\0')) {
 			// If no content-length, or HEAD (which per RFC
@@ -303,10 +327,10 @@ http_txn_cb(void *arg)
 		    0) {
 			goto error;
 		}
-		nni_http_res_get_data(txn->res, &iov.iov_buf, &iov.iov_len);
-		nni_aio_set_iov(txn->aio, 1, &iov);
+		nni_http_get_body(txn->conn, &iov.iov_buf, &iov.iov_len);
+		nni_aio_set_iov(&txn->aio, 1, &iov);
 		txn->state = HTTP_RECVING_BODY;
-		nni_http_read_full(txn->conn, txn->aio);
+		nni_http_read_full(txn->conn, &txn->aio);
 		nni_mtx_unlock(&http_txn_lk);
 		return;
 
@@ -324,7 +348,7 @@ http_txn_cb(void *arg)
 		if ((rv = nni_http_res_alloc_data(txn->res, sz)) != 0) {
 			goto error;
 		}
-		nni_http_res_get_data(txn->res, (void **) &dst, &sz);
+		nni_http_get_body(txn->conn, (void **) &dst, &sz);
 		while ((chunk = nni_http_chunks_iter(txn->chunks, chunk)) !=
 		    NULL) {
 			memcpy(dst, nni_http_chunk_data(chunk),
@@ -350,7 +374,7 @@ http_txn_cancel(nni_aio *aio, void *arg, int rv)
 	http_txn *txn = arg;
 	nni_mtx_lock(&http_txn_lk);
 	if (nni_aio_list_active(aio)) {
-		nni_aio_abort(txn->aio, rv);
+		nni_aio_abort(&txn->aio, rv);
 	}
 	nni_mtx_unlock(&http_txn_lk);
 }
@@ -364,18 +388,13 @@ void
 nni_http_transact_conn(nni_http_conn *conn, nni_aio *aio)
 {
 	http_txn *txn;
-	int       rv;
 
 	nni_aio_reset(aio);
 	if ((txn = NNI_ALLOC_STRUCT(txn)) == NULL) {
 		nni_aio_finish_error(aio, NNG_ENOMEM);
 		return;
 	}
-	if ((rv = nni_aio_alloc(&txn->aio, http_txn_cb, txn)) != 0) {
-		NNI_FREE_STRUCT(txn);
-		nni_aio_finish_error(aio, rv);
-		return;
-	}
+	nni_aio_init(&txn->aio, http_txn_cb, txn);
 	nni_aio_list_init(&txn->aios);
 	txn->client = NULL;
 	txn->conn   = conn;
@@ -391,55 +410,7 @@ nni_http_transact_conn(nni_http_conn *conn, nni_aio *aio)
 		http_txn_fini(txn);
 		return;
 	}
-	nni_http_res_reset(txn->res);
-	nni_list_append(&txn->aios, aio);
-	nni_http_write_req(conn, txn->req, txn->aio);
-	nni_mtx_unlock(&http_txn_lk);
-}
-
-// nni_http_transact_simple does a single transaction, creating a connection
-// just for the purpose, and closing it when done.  (No connection caching.)
-// The reason we require a client to be created first is to deal with TLS
-// settings.  A single global client (per server) may be used.
-void
-nni_http_transact(nni_http_client *client, nni_http_req *req,
-    nni_http_res *res, nni_aio *aio)
-{
-	http_txn *txn;
-	int       rv;
-
-	nni_aio_reset(aio);
-	if ((txn = NNI_ALLOC_STRUCT(txn)) == NULL) {
-		nni_aio_finish_error(aio, NNG_ENOMEM);
-		return;
-	}
-	if ((rv = nni_aio_alloc(&txn->aio, http_txn_cb, txn)) != 0) {
-		NNI_FREE_STRUCT(txn);
-		nni_aio_finish_error(aio, rv);
-		return;
-	}
-
-	if ((rv = nni_http_req_set_header(req, "Connection", "close")) != 0) {
-		nni_aio_finish_error(aio, rv);
-		http_txn_fini(txn);
-		return;
-	}
-
-	nni_aio_list_init(&txn->aios);
-	txn->client = client;
-	txn->conn   = NULL;
-	txn->req    = req;
-	txn->res    = res;
-	txn->state  = HTTP_CONNECTING;
-
-	nni_mtx_lock(&http_txn_lk);
-	if (!nni_aio_start(aio, http_txn_cancel, txn)) {
-		nni_mtx_unlock(&http_txn_lk);
-		http_txn_fini(txn);
-		return;
-	}
-	nni_http_res_reset(txn->res);
 	nni_list_append(&txn->aios, aio);
-	nni_http_client_connect(client, txn->aio);
+	nni_http_write_req(conn, &txn->aio);
 	nni_mtx_unlock(&http_txn_lk);
 }