diff --git a/CMakeLists.txt b/CMakeLists.txt index 64bc61482..4ce672076 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -794,7 +794,7 @@ ENDIF () IF (WITH_IO_URING) # io_uring is enabled only for standalone server, as it is impossible to cleanly shutdown synchronously SET(STANDALONE_COMPILE_FLAGS "${STANDALONE_COMPILE_FLAGS} -DH2O_USE_IO_URING=1") - LIST(APPEND STANDALONE_SOURCE_FILES lib/common/async_io.c) + LIST(APPEND STANDALONE_SOURCE_FILES lib/common/io_uring.c) ENDIF () ADD_EXECUTABLE(h2o ${STANDALONE_SOURCE_FILES}) SET_TARGET_PROPERTIES(h2o PROPERTIES COMPILE_FLAGS "${STANDALONE_COMPILE_FLAGS}") diff --git a/h2o.xcodeproj/project.pbxproj b/h2o.xcodeproj/project.pbxproj index e497fe4fb..cbf6930ad 100644 --- a/h2o.xcodeproj/project.pbxproj +++ b/h2o.xcodeproj/project.pbxproj @@ -64,7 +64,7 @@ 08F320E01E7A9CBD0038FA5A /* net.c in Sources */ = {isa = PBXBuildFile; fileRef = 08790DD91D80153600A04BC1 /* net.c */; settings = {COMPILER_FLAGS = "-Wno-conversion"; }; }; 08F320E11E7A9CBF0038FA5A /* read.c in Sources */ = {isa = PBXBuildFile; fileRef = 0812AB291D7FD54700004F23 /* read.c */; settings = {COMPILER_FLAGS = "-Wno-conversion"; }; }; 08F320E21E7A9CC20038FA5A /* sds.c in Sources */ = {isa = PBXBuildFile; fileRef = 08790DDB1D80154C00A04BC1 /* sds.c */; settings = {COMPILER_FLAGS = "-Wno-conversion"; }; }; - 08FE0365299F9F8B00141FA2 /* async_io.h in Headers */ = {isa = PBXBuildFile; fileRef = 08FE0364299F9F8B00141FA2 /* async_io.h */; }; + 08FE0365299F9F8B00141FA2 /* io_uring.h in Headers */ = {isa = PBXBuildFile; fileRef = 08FE0364299F9F8B00141FA2 /* io_uring.h */; }; 100A55101C2BB15600C4E3E0 /* http_request.c in Sources */ = {isa = PBXBuildFile; fileRef = 100A550E1C2BB15100C4E3E0 /* http_request.c */; }; 101788B219B561AA0084C6D8 /* socket.c in Sources */ = {isa = PBXBuildFile; fileRef = 101788B119B561AA0084C6D8 /* socket.c */; }; 101B670C19ADD3380084A351 /* access_log.c in Sources */ = {isa = PBXBuildFile; fileRef = 101B670B19ADD3380084A351 /* access_log.c */; }; @@ -783,8 +783,8 @@ 08E9CC4D1E41F6660049DD26 /* embedded.c.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = embedded.c.h; sourceTree = ""; }; 08EFC4CF27FD8FAA004C532C /* 50file-shrink.t */ = {isa = PBXFileReference; lastKnownFileType = text; path = "50file-shrink.t"; sourceTree = ""; xcLanguageSpecificationIdentifier = xcode.lang.perl; }; 08EFC4DF27FE9F96004C532C /* throttle_resp.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; path = throttle_resp.c; sourceTree = ""; }; - 08FE0364299F9F8B00141FA2 /* async_io.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = async_io.h; sourceTree = ""; }; - 08FE0366299FA1FE00141FA2 /* async_io.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; path = async_io.c; sourceTree = ""; }; + 08FE0364299F9F8B00141FA2 /* io_uring.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = io_uring.h; sourceTree = ""; }; + 08FE0366299FA1FE00141FA2 /* io_uring.c */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.c; path = io_uring.c; sourceTree = ""; }; 100A550C1C22857B00C4E3E0 /* 50mruby-htpasswd.t */ = {isa = PBXFileReference; lastKnownFileType = text; path = "50mruby-htpasswd.t"; sourceTree = ""; xcLanguageSpecificationIdentifier = xcode.lang.perl; }; 100A550E1C2BB15100C4E3E0 /* http_request.c */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.c; path = http_request.c; sourceTree = ""; }; 100A55131C2E5FAC00C4E3E0 /* 50mruby-http-request.t */ = {isa = PBXFileReference; lastKnownFileType = text; path = "50mruby-http-request.t"; sourceTree = ""; xcLanguageSpecificationIdentifier = xcode.lang.perl; }; @@ -1578,7 +1578,6 @@ isa = PBXGroup; children = ( E96A24CE23E6743600CA970A /* absprio.c */, - 08FE0366299FA1FE00141FA2 /* async_io.c */, D08137371FD400F4004679DF /* balancer */, 10DA969D1CD2BFAC00679165 /* cache.c */, 107D4D541B5B30EE004A9B21 /* file.c */, @@ -1588,6 +1587,7 @@ 10D0905419F102C70043D458 /* http1client.c */, 7D0341FA1FE4D5B60052E0A1 /* http2client.c */, E9E50472214A5B8A004DC170 /* http3client.c */, + 08FE0366299FA1FE00141FA2 /* io_uring.c */, 10A3D3D11B4CDBDC00327CF9 /* memcached.c */, 08CEA9D126701D7600B4BB6B /* rand.c */, 08790DE31D8276EA00A04BC1 /* redis.c */, @@ -2001,7 +2001,6 @@ isa = PBXGroup; children = ( E96A24CC23E673F400CA970A /* absprio.h */, - 08FE0364299F9F8B00141FA2 /* async_io.h */, D08137411FD408C1004679DF /* balancer.h */, 10DA969B1CD2BF9000679165 /* cache.h */, 1024A3FD1D22546800EB13F1 /* cache_digests.h */, @@ -2022,6 +2021,7 @@ E9E50475214B3D28004DC170 /* http3_common.h */, E9757708221BB78C00D1EF74 /* http3_internal.h */, E9E5049321501CA5004DC170 /* http3_server.h */, + 08FE0364299F9F8B00141FA2 /* io_uring.h */, 10A3D3D01B4CDBC700327CF9 /* memcached.h */, 10D0904219F0BA780043D458 /* linklist.h */, 10D0903919F0A51C0043D458 /* memory.h */, @@ -2983,7 +2983,7 @@ 0812AB211D7FCFEB00004F23 /* async.h in Headers */, 10D0903A19F0A51C0043D458 /* memory.h in Headers */, 7DA3F5AF20E0B0400000222F /* token_table.h in Headers */, - 08FE0365299F9F8B00141FA2 /* async_io.h in Headers */, + 08FE0365299F9F8B00141FA2 /* io_uring.h in Headers */, 104B9A2D1A4BE029009EEE64 /* version.h in Headers */, E980423B22463044008B9745 /* cc.h in Headers */, E9E5049421501CA5004DC170 /* http3_server.h in Headers */, diff --git a/include/h2o/async_io.h b/include/h2o/io_uring.h similarity index 73% rename from include/h2o/async_io.h rename to include/h2o/io_uring.h index 7be49f5a5..7c80ffbd3 100644 --- a/include/h2o/async_io.h +++ b/include/h2o/io_uring.h @@ -19,23 +19,29 @@ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. */ -#ifndef h2o__async_io_h -#define h2o__async_io_h +#ifndef h2o__io_uring_h +#define h2o__io_uring_h #if !H2O_USE_IO_URING #error "this file may be included only when io_uring support is available" #endif -typedef struct st_h2o_async_io_t h2o_async_io_t; -typedef struct st_h2o_async_io_cmd_t h2o_async_io_cmd_t; -typedef void (*h2o_async_io_cb)(h2o_async_io_cmd_t *); +typedef struct st_h2o_io_uring_t h2o_io_uring_t; +typedef struct st_h2o_io_uring_cmd_t h2o_io_uring_cmd_t; +typedef void (*h2o_io_uring_cb)(h2o_io_uring_cmd_t *); -struct st_h2o_async_io_cmd_t { +struct st_h2o_io_uring_cmd_t { struct { - h2o_async_io_cb func; + h2o_io_uring_cb func; void *data; } cb; int result; + struct st_h2o_io_uring_cmd_t *next; + struct { + int filefd, pipefd; + uint64_t offset; + size_t len; + } splice_; }; extern size_t h2o_io_uring_batch_size; @@ -43,11 +49,11 @@ extern size_t h2o_io_uring_batch_size; /** * initializes structure related to async I/O of `loop` */ -void h2o_async_io_setup(h2o_loop_t *loop); +void h2o_io_uring_setup(h2o_loop_t *loop); /** * Runs `splice` on linux. Buffer size of `outfd` must be large enough to contain `len` bytes. */ -void h2o_async_io_splice_file(h2o_async_io_cmd_t **cmd, h2o_loop_t *loop, int filefd, uint64_t offset, int pipefd, size_t len, - h2o_async_io_cb cb, void *data); +void h2o_io_uring_splice_file(h2o_io_uring_cmd_t **cmd, h2o_loop_t *loop, int filefd, uint64_t offset, int pipefd, size_t len, + h2o_io_uring_cb cb, void *data); #endif diff --git a/include/h2o/socket/evloop.h b/include/h2o/socket/evloop.h index 01ffb317a..3c9408dc9 100644 --- a/include/h2o/socket/evloop.h +++ b/include/h2o/socket/evloop.h @@ -41,8 +41,8 @@ #define H2O_SOCKET_FLAG__EPOLL_IS_REGISTERED 0x1000 #if H2O_USE_IO_URING -/* forward declaration of async_io.h */ -struct st_h2o_async_io_t; +/* forward declaration of h2o/io_uring.h */ +struct st_h2o_io_uring_t; #endif typedef struct st_h2o_evloop_t { @@ -59,7 +59,7 @@ typedef struct st_h2o_evloop_t { h2o_sliding_counter_t exec_time_nanosec_counter; uint64_t run_count; #if H2O_USE_IO_URING - struct st_h2o_async_io_t *_async_io; + struct st_h2o_io_uring_t *_io_uring; #endif } h2o_evloop_t; diff --git a/lib/common/async_io.c b/lib/common/io_uring.c similarity index 67% rename from lib/common/async_io.c rename to lib/common/io_uring.c index 63a409de5..96d05c839 100644 --- a/lib/common/async_io.c +++ b/lib/common/io_uring.c @@ -26,26 +26,16 @@ #include #include #include "h2o/socket.h" -#include "h2o/async_io.h" +#include "h2o/io_uring.h" #include "../probes_.h" -struct st_h2o_io_uring_cmd_t { - h2o_async_io_cmd_t super; - struct { - int filefd, pipefd; - uint64_t offset; - size_t len; - } splice_; - struct st_h2o_io_uring_cmd_t *next; -}; - struct st_h2o_io_uring_queue_t { struct st_h2o_io_uring_cmd_t *head; struct st_h2o_io_uring_cmd_t **tail; size_t size; }; -struct st_h2o_async_io_t { +struct st_h2o_io_uring_t { h2o_loop_t *loop; struct io_uring uring; h2o_socket_t *sock_notify; @@ -89,16 +79,16 @@ static struct st_h2o_io_uring_cmd_t *pop_queue(struct st_h2o_io_uring_queue_t *q static int submit_commands(h2o_loop_t *loop, int can_delay) { - if (can_delay && loop->_async_io->submission.size < h2o_io_uring_batch_size) + if (can_delay && loop->_io_uring->submission.size < h2o_io_uring_batch_size) return 0; int made_progress = 0; - while (loop->_async_io->submission.head != NULL) { + while (loop->_io_uring->submission.head != NULL) { struct io_uring_sqe *sqe; - if ((sqe = io_uring_get_sqe(&loop->_async_io->uring)) == NULL) + if ((sqe = io_uring_get_sqe(&loop->_io_uring->uring)) == NULL) break; - struct st_h2o_io_uring_cmd_t *cmd = pop_queue(&loop->_async_io->submission); + struct st_h2o_io_uring_cmd_t *cmd = pop_queue(&loop->_io_uring->submission); assert(cmd != NULL); io_uring_prep_splice(sqe, cmd->splice_.filefd, cmd->splice_.offset, cmd->splice_.pipefd, -1, cmd->splice_.len, 0); sqe->user_data = (uint64_t)cmd; @@ -107,7 +97,7 @@ static int submit_commands(h2o_loop_t *loop, int can_delay) if (made_progress) { int ret; - while ((ret = io_uring_submit(&loop->_async_io->uring)) == -EINTR) + while ((ret = io_uring_submit(&loop->_io_uring->uring)) == -EINTR) ; if (ret < 0) h2o_fatal("io_uring_submit:%s", strerror(-ret)); @@ -125,20 +115,20 @@ static int check_completion(h2o_loop_t *loop, struct st_h2o_io_uring_cmd_t *cmd_ { /* obtain completed command and its result */ struct io_uring_cqe *cqe; - while ((ret = io_uring_peek_cqe(&loop->_async_io->uring, &cqe)) == -EINTR) + while ((ret = io_uring_peek_cqe(&loop->_io_uring->uring, &cqe)) == -EINTR) ; if (ret != 0) break; cmd = (struct st_h2o_io_uring_cmd_t *)cqe->user_data; - cmd->super.result = cqe->res; - io_uring_cqe_seen(&loop->_async_io->uring, cqe); + cmd->result = cqe->res; + io_uring_cqe_seen(&loop->_io_uring->uring, cqe); } /* link to completion list or indicate to the caller that `cmd_sync` has completed */ if (cmd == cmd_sync) { cmd_sync_done = 1; } else { - insert_queue(&loop->_async_io->completion, cmd); + insert_queue(&loop->_io_uring->completion, cmd); } } @@ -147,15 +137,15 @@ static int check_completion(h2o_loop_t *loop, struct st_h2o_io_uring_cmd_t *cmd_ static int dispatch_completed(h2o_loop_t *loop) { - if (loop->_async_io->completion.head == NULL) + if (loop->_io_uring->completion.head == NULL) return 0; do { - struct st_h2o_io_uring_cmd_t *cmd = pop_queue(&loop->_async_io->completion); - H2O_PROBE(ASYNC_IO_END, cmd); - cmd->super.cb.func(&cmd->super); + struct st_h2o_io_uring_cmd_t *cmd = pop_queue(&loop->_io_uring->completion); + H2O_PROBE(IO_URING_END, cmd); + cmd->cb.func(cmd); free(cmd); - } while (loop->_async_io->completion.head != NULL); + } while (loop->_io_uring->completion.head != NULL); return 1; } @@ -163,53 +153,52 @@ static struct st_h2o_io_uring_cmd_t *start_command(h2o_loop_t *loop, struct st_h { int needs_timer = 0; - insert_queue(&loop->_async_io->submission, cmd); + insert_queue(&loop->_io_uring->submission, cmd); submit_commands(loop, 1); /* if we have submitted all commands up to the current one, fetch completion events in hope that we might be able to complete * the current one synchronously (as doing so improves locality) */ - if (loop->_async_io->submission.head == NULL) { + if (loop->_io_uring->submission.head == NULL) { if (check_completion(loop, cmd)) { - cmd->super.cb.func(&cmd->super); + cmd->cb.func(cmd); free(cmd); cmd = NULL; } - if (loop->_async_io->completion.head != NULL) + if (loop->_io_uring->completion.head != NULL) needs_timer = 1; } else { needs_timer = 1; } - if (needs_timer && !h2o_timer_is_linked(&loop->_async_io->delayed)) - h2o_timer_link(loop, 0, &loop->_async_io->delayed); + if (needs_timer && !h2o_timer_is_linked(&loop->_io_uring->delayed)) + h2o_timer_link(loop, 0, &loop->_io_uring->delayed); return cmd; } -void h2o_async_io_splice_file(h2o_async_io_cmd_t **_cmd, h2o_loop_t *loop, int _filefd, uint64_t _offset, int _pipefd, size_t _len, - h2o_async_io_cb _cb, void *_data) +void h2o_io_uring_splice_file(h2o_io_uring_cmd_t **_cmd, h2o_loop_t *loop, int _filefd, uint64_t _offset, int _pipefd, size_t _len, + h2o_io_uring_cb _cb, void *_data) { /* build command */ struct st_h2o_io_uring_cmd_t *cmd = h2o_mem_alloc(sizeof(*cmd)); *cmd = (struct st_h2o_io_uring_cmd_t){ - .super = {.cb = {.func = _cb, .data = _data}}, - .splice_ = { - .filefd = _filefd, - .pipefd = _pipefd, - .offset = _offset, - .len = _len, - }, + .cb.func = _cb, + .cb.data = _data, + .splice_.filefd = _filefd, + .splice_.pipefd = _pipefd, + .splice_.offset = _offset, + .splice_.len = _len, }; cmd = start_command(loop, cmd); - *_cmd = &cmd->super; + *_cmd = cmd; if (cmd != NULL) - H2O_PROBE(ASYNC_IO_START_SPLICE_FILE, cmd); + H2O_PROBE(IO_URING_START_SPLICE_FILE, cmd); } -int h2o_async_io_can_splice(void) +int h2o_io_uring_can_splice(void) { return 1; } @@ -221,7 +210,7 @@ static void run_uring(h2o_loop_t *loop) check_completion(loop, NULL); } while (dispatch_completed(loop) || submit_commands(loop, 0)); - assert(loop->_async_io->completion.head == NULL); + assert(loop->_io_uring->completion.head == NULL); } void on_notify(h2o_socket_t *sock, const char *err) @@ -234,25 +223,25 @@ void on_notify(h2o_socket_t *sock, const char *err) void on_delayed(h2o_timer_t *_timer) { - struct st_h2o_async_io_t *async_io = H2O_STRUCT_FROM_MEMBER(struct st_h2o_async_io_t, delayed, _timer); - h2o_loop_t *loop = async_io->loop; + struct st_h2o_io_uring_t *io_uring = H2O_STRUCT_FROM_MEMBER(struct st_h2o_io_uring_t, delayed, _timer); + h2o_loop_t *loop = io_uring->loop; run_uring(loop); } -void h2o_async_io_setup(h2o_loop_t *loop) +void h2o_io_uring_setup(h2o_loop_t *loop) { - loop->_async_io = h2o_mem_alloc(sizeof(*loop->_async_io)); - loop->_async_io->loop = loop; + loop->_io_uring = h2o_mem_alloc(sizeof(*loop->_io_uring)); + loop->_io_uring->loop = loop; int ret; - if ((ret = io_uring_queue_init(16, &loop->_async_io->uring, 0)) != 0) + if ((ret = io_uring_queue_init(16, &loop->_io_uring->uring, 0)) != 0) h2o_fatal("io_uring_queue_init:%s", strerror(-ret)); - loop->_async_io->sock_notify = h2o_evloop_socket_create(loop, loop->_async_io->uring.ring_fd, H2O_SOCKET_FLAG_DONT_READ); - h2o_socket_read_start(loop->_async_io->sock_notify, on_notify); + loop->_io_uring->sock_notify = h2o_evloop_socket_create(loop, loop->_io_uring->uring.ring_fd, H2O_SOCKET_FLAG_DONT_READ); + h2o_socket_read_start(loop->_io_uring->sock_notify, on_notify); - init_queue(&loop->_async_io->submission); - init_queue(&loop->_async_io->completion); - h2o_timer_init(&loop->_async_io->delayed, on_delayed); + init_queue(&loop->_io_uring->submission); + init_queue(&loop->_io_uring->completion); + h2o_timer_init(&loop->_io_uring->delayed, on_delayed); } diff --git a/lib/common/socket/evloop.c.h b/lib/common/socket/evloop.c.h index 28ba4367b..475dc4294 100644 --- a/lib/common/socket/evloop.c.h +++ b/lib/common/socket/evloop.c.h @@ -31,7 +31,7 @@ #include "cloexec.h" #include "h2o/linklist.h" #if H2O_USE_IO_URING -#include "h2o/async_io.h" +#include "h2o/io_uring.h" #endif #if !defined(H2O_USE_ACCEPT4) @@ -797,7 +797,7 @@ h2o_evloop_t *create_evloop(size_t sz) loop->_timeouts = h2o_timerwheel_create(3, loop->_now_millisec); #if H2O_USE_IO_URING - h2o_async_io_setup(loop); + h2o_io_uring_setup(loop); #endif return loop; diff --git a/lib/handler/file.c b/lib/handler/file.c index 8458b1783..fd3afdf0e 100644 --- a/lib/handler/file.c +++ b/lib/handler/file.c @@ -36,7 +36,7 @@ #include #include "h2o.h" #if H2O_USE_IO_URING -#include "h2o/async_io.h" +#include "h2o/io_uring.h" #endif #define MAX_BUF_SIZE 65000 @@ -156,7 +156,7 @@ static void do_stop_async_splice(h2o_generator_t *_self, h2o_req_t *req) self->src_req = NULL; } -static void do_proceed_on_splice_complete(h2o_async_io_cmd_t *cmd) +static void do_proceed_on_splice_complete(h2o_io_uring_cmd_t *cmd) { struct st_h2o_sendfile_generator_t *self = cmd->cb.data; @@ -257,12 +257,12 @@ static void do_proceed(h2o_generator_t *_self, h2o_req_t *req) struct st_h2o_sendfile_generator_t *self = (void *)_self; size_t bytes_to_send = self->bytesleft < H2O_PULL_SENDVEC_MAX_SIZE ? self->bytesleft : H2O_PULL_SENDVEC_MAX_SIZE; - /* if io_uring is to be used, addref so that the self would not be released, then call `h2o_async_io_splice_file` */ + /* if io_uring is to be used, addref so that the self would not be released, then call `h2o_io_uring_splice_file` */ #if H2O_USE_IO_URING if (self->splice_fds[0] != -1) { - h2o_async_io_cmd_t *cmd; + h2o_io_uring_cmd_t *cmd; h2o_mem_addref_shared(self); - h2o_async_io_splice_file(&cmd, self->src_req->conn->ctx->loop, self->file.ref->fd, self->file.off, self->splice_fds[1], + h2o_io_uring_splice_file(&cmd, self->src_req->conn->ctx->loop, self->file.ref->fd, self->file.off, self->splice_fds[1], bytes_to_send, do_proceed_on_splice_complete, self); return; } diff --git a/src/main.c b/src/main.c index 47542c97d..63f8541a8 100644 --- a/src/main.c +++ b/src/main.c @@ -96,7 +96,7 @@ #include "h2o/mruby_.h" #endif #if H2O_USE_IO_URING -#include "h2o/async_io.h" +#include "h2o/io_uring.h" #endif #include "standalone.h" #include "../lib/probes_.h"