add knob to toggle use of zero copy

include/h2o.h

@@ -330,6 +330,17 @@ typedef enum h2o_send_informational_mode {
     H2O_SEND_INFORMATIONAL_MODE_ALL
 } h2o_send_informational_mode_t;
 
+/**
+ * If zero copy should be used. "Always" indicates to the proxy handler that pipe-backed vectors should be used even when the http
+ * protocol handler does not support zero-copy. This mode delays the load of content to userspace, at the cost of moving around
+ * memory page between the socket connected to the origin and the pipe.
+ */
+typedef enum h2o_proxy_zero_copy_mode {
+    H2O_PROXY_ZERO_COPY_DISABLED,
+    H2O_PROXY_ZERO_COPY_ENABLED,
+    H2O_PROXY_ZERO_COPY_ALWAYS
+} h2o_proxy_zero_copy_mode_t;
+
 struct st_h2o_globalconf_t {
     /**
      * a NULL-terminated list of host contexts (h2o_hostconf_t)
@@ -512,6 +523,10 @@ struct st_h2o_globalconf_t {
          * maximum size to buffer for the response
          */
         size_t max_buffer_size;
+        /**
+         * a boolean flag if set to true, instructs to use zero copy (i.e., splice to pipe then splice to socket) if possible
+         */
+        h2o_proxy_zero_copy_mode_t zero_copy;
 
         struct {
             uint32_t max_concurrent_streams;
@@ -900,6 +915,10 @@ typedef struct st_h2o_conn_callbacks_t {
      * yet available.
      */
     int64_t (*get_rtt)(h2o_conn_t *conn);
+    /**
+     * optional callback that returns if zero copy is supported by the HTTP handler
+     */
+    int (*can_zero_copy)(h2o_conn_t *conn);
     /**
      * logging callbacks (all of them are optional)
      */

include/h2o/httpclient.h

@@ -61,6 +61,10 @@ typedef struct st_h2o_httpclient_properties_t {
      * value of the connection header field to be sent to the server. This can be used for upgrading an HTTP/1.1 connection.
      */
     h2o_iovec_t *connection_header;
+    /**
+     * defaults to false
+     */
+    unsigned prefer_pipe_reader : 1;
 } h2o_httpclient_properties_t;
 
 typedef struct st_h2o_httpclient_pipe_reader_t h2o_httpclient_pipe_reader_t;

include/h2o/socket.h

@@ -356,6 +356,10 @@ void h2o_socket_setpeername(h2o_socket_t *sock, struct sockaddr *sa, socklen_t l
  *
  */
 ptls_t *h2o_socket_get_ptls(h2o_socket_t *sock);
+/**
+ *
+ */
+int h2o_socket_can_tls_offload(h2o_socket_t *sock);
 /**
  *
  */

lib/common/http1client.c

@@ -85,6 +85,7 @@ struct st_h2o_http1client_t {
     unsigned _is_chunked : 1;
     unsigned _seen_at_least_one_chunk : 1;
     unsigned _delay_free : 1;
+    unsigned _app_prefers_pipe_reader : 1;
 };
 
 static void on_body_to_pipe(h2o_socket_t *_sock, const char *err);
@@ -402,9 +403,10 @@ static void on_head(h2o_socket_t *sock, const char *err)
         return;
     }
 
-#if !H2O_USE_LIBUV
     /* revert max read size to 1MB now that we have received the first chunk, presumably carrying all the response headers */
-    h2o_evloop_socket_set_max_read_size(client->sock, h2o_evloop_socket_max_read_size);
+#if USE_PIPE_READER
+    if (client->_app_prefers_pipe_reader)
+        h2o_evloop_socket_set_max_read_size(client->sock, h2o_evloop_socket_max_read_size);
 #endif
 
     client->super._timeout.cb = on_head_timeout;
@@ -527,9 +529,12 @@ static void on_head(h2o_socket_t *sock, const char *err)
         .header_requires_dup = 1,
     };
 #if USE_PIPE_READER
-    /* If there is no less than 64KB of data to be read from the socket, offer the application the opportunity to use pipe for
-     * transferring the content zero-copy (TODO fine tune the threshold). */
-    if (reader == on_body_content_length && client->sock->input->size + 65536 <= client->_body_decoder.content_length.bytesleft)
+    /* If there is no less than 16KB of data to be read from the socket, offer the application the opportunity to use pipe for
+     * transferring the content zero-copy. While we might want to further reduce the threshold (from 16KB), relative cost of
+     * skipping copy becomes small as the cost of request / response header processing becomes significant. Or so is the belief
+     * without any benchmark that proves the argument. */
+    if (client->_app_prefers_pipe_reader && reader == on_body_content_length &&
+        client->sock->input->size + 16384 <= client->_body_decoder.content_length.bytesleft)
         on_head.pipe_reader = &client->pipe_reader;
 #endif
 
@@ -816,11 +821,13 @@ static void start_request(struct st_h2o_http1client_t *client, h2o_iovec_t metho
     client->state.req = STREAM_STATE_BODY;
     client->super.timings.request_begin_at = h2o_gettimeofday(client->super.ctx->loop);
 
-#if !H2O_USE_LIBUV
-    /* Reduce max read size before fetching headers. The intent here is to not do a full-sized read of 1MB when we have the chance
-     * of passing data zero-copy through pipe. 16KB has been chosen so that an almost full-sized HTTP/2 frame / TLS record can be
-     * generated for the first chunk of data that we pass through memory. */
-    h2o_evloop_socket_set_max_read_size(client->sock, 16384);
+    /* If there's possibility of using a pipe for forwarding the content, reduce maximum read size before fetching headers. The
+     * intent here is to not do a full-sized read of 1MB. 16KB has been chosen so that all HTTP response headers would be available,
+     * and that an almost full-sized HTTP/2 frame / TLS record can be generated for the first chunk of data that we pass through
+     * memory. */
+#if USE_PIPE_READER
+    if (client->_app_prefers_pipe_reader)
+        h2o_evloop_socket_set_max_read_size(client->sock, 16384);
 #endif
 
     h2o_socket_read_start(client->sock, on_head);
@@ -844,6 +851,7 @@ static void on_connection_ready(struct st_h2o_http1client_t *client)
 
     client->super._cb.on_head = client->super._cb.on_connect(&client->super, NULL, &method, &url, (const h2o_header_t **)&headers,
                                                              &num_headers, &body, &client->proceed_req, &props, client->_origin);
+    client->_app_prefers_pipe_reader = props.prefer_pipe_reader;
 
     if (client->super._cb.on_head == NULL) {
         close_client(client);

lib/common/socket.c

@@ -1044,6 +1044,18 @@ const char *h2o_socket_get_ssl_server_name(const h2o_socket_t *sock)
     return NULL;
 }
 
+int h2o_socket_can_tls_offload(h2o_socket_t *sock)
+{
+    if (sock->ssl == NULL)
+        return 0;
+
+#if H2O_USE_LIBUV
+    return 0;
+#else
+    return can_tls_offload(sock);
+#endif
+}
+
 h2o_iovec_t h2o_socket_log_tcp_congestion_controller(h2o_socket_t *sock, h2o_mem_pool_t *pool)
 {
 #if defined(TCP_CONGESTION)

lib/common/socket/evloop.c.h

@@ -383,6 +383,24 @@ Schedule_Write:
     link_to_statechanged(sock);
 }
 
+static int can_tls_offload(h2o_socket_t *sock)
+{
+#ifdef __linux__
+    if (sock->ssl->ptls != NULL) {
+        ptls_cipher_suite_t *cipher = ptls_get_cipher(sock->ssl->ptls);
+        switch (cipher->id) {
+        case PTLS_CIPHER_SUITE_AES_128_GCM_SHA256:
+        case PTLS_CIPHER_SUITE_AES_256_GCM_SHA384:
+            return 1;
+        default:
+            break;
+        }
+    }
+#endif
+
+    return 0;
+}
+
 #ifdef __linux__
 static void switch_to_ktls(struct st_h2o_evloop_socket_t *sock)
 {

lib/core/proxy.c

@@ -774,6 +774,21 @@ static h2o_httpclient_head_cb on_connect(h2o_httpclient_t *client, const char *e
 
     client->get_conn_properties(client, &req->proxy_stats.conn);
 
+    { /* indicate to httpclient if use of pipe is preferred */
+        h2o_conn_t *conn = self->src_req->conn;
+        switch (conn->ctx->globalconf->proxy.zero_copy) {
+        case H2O_PROXY_ZERO_COPY_ALWAYS:
+            props->prefer_pipe_reader = 1;
+            break;
+        case H2O_PROXY_ZERO_COPY_ENABLED:
+            if (conn->callbacks->can_zero_copy != NULL && conn->callbacks->can_zero_copy(conn))
+                props->prefer_pipe_reader = 1;
+            break;
+        default:
+            break;
+        }
+    }
+
     return on_head;
 }
 

lib/handler/configurator/proxy.c

@@ -483,6 +483,25 @@ static int on_config_emit_missing_date_header(h2o_configurator_command_t *cmd, h
     return 0;
 }
 
+static int on_config_zero_copy(h2o_configurator_command_t *cmd, h2o_configurator_context_t *ctx, yoml_t *node)
+{
+    ssize_t ret = h2o_configurator_get_one_of(cmd, node, "OFF,ON,ALWAYS");
+    switch (ret) {
+    case 0:
+        ctx->globalconf->proxy.zero_copy = H2O_PROXY_ZERO_COPY_DISABLED;
+        break;
+    case 1:
+        ctx->globalconf->proxy.zero_copy = H2O_PROXY_ZERO_COPY_ENABLED;
+        break;
+    case 2:
+        ctx->globalconf->proxy.zero_copy = H2O_PROXY_ZERO_COPY_ALWAYS;
+        break;
+    default:
+        return -1;
+    }
+    return 0;
+}
+
 static int on_config_preserve_x_forwarded_proto(h2o_configurator_command_t *cmd, h2o_configurator_context_t *ctx, yoml_t *node)
 {
     ssize_t ret = h2o_configurator_get_one_of(cmd, node, "OFF,ON");
@@ -694,6 +713,8 @@ void h2o_proxy_register_configurator(h2o_globalconf_t *conf)
     h2o_configurator_define_command(&c->super, "proxy.emit-missing-date-header",
                                     H2O_CONFIGURATOR_FLAG_GLOBAL | H2O_CONFIGURATOR_FLAG_EXPECT_SCALAR,
                                     on_config_emit_missing_date_header);
+    h2o_configurator_define_command(&c->super, "proxy.zero-copy",
+                                    H2O_CONFIGURATOR_FLAG_GLOBAL | H2O_CONFIGURATOR_FLAG_EXPECT_SCALAR, on_config_zero_copy);
     h2o_configurator_define_headers_commands(conf, &c->super, "proxy.header", get_headers_commands);
     h2o_configurator_define_command(&c->super, "proxy.max-buffer-size",
                                     H2O_CONFIGURATOR_FLAG_ALL_LEVELS | H2O_CONFIGURATOR_FLAG_EXPECT_SCALAR,

lib/http1.c

@@ -1131,6 +1131,12 @@ static int skip_tracing(h2o_conn_t *_conn)
     return h2o_socket_skip_tracing(conn->sock);
 }
 
+static int can_zero_copy(h2o_conn_t *_conn)
+{
+    struct st_h2o_http1_conn_t *conn = (void *)_conn;
+    return conn->sock->ssl == NULL || h2o_socket_can_tls_offload(conn->sock);
+}
+
 #define DEFINE_LOGGER(name)                                                                                                        \
     static h2o_iovec_t log_##name(h2o_req_t *req)                                                                                  \
     {                                                                                                                              \
@@ -1176,6 +1182,7 @@ static const h2o_conn_callbacks_t h1_callbacks = {
     .get_peername = get_peername,
     .get_ptls = get_ptls,
     .skip_tracing = skip_tracing,
+    .can_zero_copy = can_zero_copy,
     .log_ = {{
         .transport =
             {