superrob1500
/
mirror-linux
mirror of https://github.com/torvalds/linux
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

io_uring/kbuf: introduce struct io_br_sel 

Rather than return addresses directly from buffer selection, add a
struct around it. No functional changes in this patch, it's in
preparation for storing more buffer related information locally, rather
than in struct io_kiocb.

Link: https://lore.kernel.org/r/20250821020750.598432-7-axboe@kernel.dk
Signed-off-by: Jens Axboe <axboe@kernel.dk>
pull/1354/merge
Jens Axboe 2025-08-20 20:03:34 -06:00
parent 1b5add75d7
commit ab6559bdbb
4 changed files with 59 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
io_uring/kbuf.c
View File

@ -151,18 +151,18 @@ static int io_provided_buffers_select(struct io_kiocb *req, size_t *len,
return 1;
}
static void __user *io_ring_buffer_select(struct io_kiocb *req, size_t *len,
struct io_buffer_list *bl,
unsigned int issue_flags)
static struct io_br_sel io_ring_buffer_select(struct io_kiocb *req, size_t *len,
struct io_buffer_list *bl,
unsigned int issue_flags)
{
struct io_uring_buf_ring *br = bl->buf_ring;
__u16 tail, head = bl->head;
struct io_br_sel sel = { };
struct io_uring_buf *buf;
void __user *ret;
tail = smp_load_acquire(&br->tail);
if (unlikely(tail == head))
return NULL;
return sel;
if (head + 1 == tail)
req->flags |= REQ_F_BL_EMPTY;
@ -173,7 +173,7 @@ static void __user *io_ring_buffer_select(struct io_kiocb *req, size_t *len,
req->flags |= REQ_F_BUFFER_RING | REQ_F_BUFFERS_COMMIT;
req->buf_list = bl;
req->buf_index = buf->bid;
ret = u64_to_user_ptr(buf->addr);
sel.addr = u64_to_user_ptr(buf->addr);
if (issue_flags & IO_URING_F_UNLOCKED || !io_file_can_poll(req)) {
/*
@ -189,27 +189,27 @@ static void __user *io_ring_buffer_select(struct io_kiocb *req, size_t *len,
io_kbuf_commit(req, bl, *len, 1);
req->buf_list = NULL;
}
return ret;
return sel;
}
void __user *io_buffer_select(struct io_kiocb *req, size_t *len,
unsigned buf_group, unsigned int issue_flags)
struct io_br_sel io_buffer_select(struct io_kiocb *req, size_t *len,
unsigned buf_group, unsigned int issue_flags)
{
struct io_ring_ctx *ctx = req->ctx;
struct io_br_sel sel = { };
struct io_buffer_list *bl;
void __user *ret = NULL;
io_ring_submit_lock(req->ctx, issue_flags);
bl = io_buffer_get_list(ctx, buf_group);
if (likely(bl)) {
if (bl->flags & IOBL_BUF_RING)
ret = io_ring_buffer_select(req, len, bl, issue_flags);
sel = io_ring_buffer_select(req, len, bl, issue_flags);
else
ret = io_provided_buffer_select(req, len, bl);
sel.addr = io_provided_buffer_select(req, len, bl);
}
io_ring_submit_unlock(req->ctx, issue_flags);
return ret;
return sel;
}
/* cap it at a reasonable 256, will be one page even for 4K */

19
io_uring/kbuf.h
View File

@ -62,8 +62,23 @@ struct buf_sel_arg {
unsigned short partial_map;
};
void __user *io_buffer_select(struct io_kiocb *req, size_t *len,
unsigned buf_group, unsigned int issue_flags);
/*
* Return value from io_buffer_list selection. Just returns the error or
* user address for now, will be extended to return the buffer list in the
* future.
*/
struct io_br_sel {
/*
* Some selection parts return the user address, others return an error.
*/
union {
void __user *addr;
ssize_t val;
};
};
struct io_br_sel io_buffer_select(struct io_kiocb *req, size_t *len,
unsigned buf_group, unsigned int issue_flags);
int io_buffers_select(struct io_kiocb *req, struct buf_sel_arg *arg,
unsigned int issue_flags);
int io_buffers_peek(struct io_kiocb *req, struct buf_sel_arg *arg);

18
io_uring/net.c
View File

@ -1035,22 +1035,22 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags)
retry_multishot:
if (io_do_buffer_select(req)) {
void __user *buf;
struct io_br_sel sel;
size_t len = sr->len;
buf = io_buffer_select(req, &len, sr->buf_group, issue_flags);
if (!buf)
sel = io_buffer_select(req, &len, sr->buf_group, issue_flags);
if (!sel.addr)
return -ENOBUFS;
if (req->flags & REQ_F_APOLL_MULTISHOT) {
ret = io_recvmsg_prep_multishot(kmsg, sr, &buf, &len);
ret = io_recvmsg_prep_multishot(kmsg, sr, &sel.addr, &len);
if (ret) {
io_kbuf_recycle(req, req->buf_list, issue_flags);
return ret;
}
}
iov_iter_ubuf(&kmsg->msg.msg_iter, ITER_DEST, buf, len);
iov_iter_ubuf(&kmsg->msg.msg_iter, ITER_DEST, sel.addr, len);
}
kmsg->msg.msg_get_inq = 1;
@ -1153,13 +1153,13 @@ static int io_recv_buf_select(struct io_kiocb *req, struct io_async_msghdr *kmsg
iov_iter_init(&kmsg->msg.msg_iter, ITER_DEST, arg.iovs, ret,
arg.out_len);
} else {
void __user *buf;
struct io_br_sel sel;
*len = sr->len;
buf = io_buffer_select(req, len, sr->buf_group, issue_flags);
if (!buf)
sel = io_buffer_select(req, len, sr->buf_group, issue_flags);
if (!sel.addr)
return -ENOBUFS;
sr->buf = buf;
sr->buf = sel.addr;
sr->len = *len;
map_ubuf:
ret = import_ubuf(ITER_DEST, sr->buf, sr->len,

34
io_uring/rw.c
View File

@ -107,34 +107,35 @@ static int io_import_vec(int ddir, struct io_kiocb *req,
}
static int __io_import_rw_buffer(int ddir, struct io_kiocb *req,
struct io_async_rw *io,
unsigned int issue_flags)
struct io_async_rw *io, struct io_br_sel *sel,
unsigned int issue_flags)
{
const struct io_issue_def *def = &io_issue_defs[req->opcode];
struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
void __user *buf = u64_to_user_ptr(rw->addr);
size_t sqe_len = rw->len;
sel->addr = u64_to_user_ptr(rw->addr);
if (def->vectored && !(req->flags & REQ_F_BUFFER_SELECT))
return io_import_vec(ddir, req, io, buf, sqe_len);
return io_import_vec(ddir, req, io, sel->addr, sqe_len);
if (io_do_buffer_select(req)) {
buf = io_buffer_select(req, &sqe_len, io->buf_group, issue_flags);
if (!buf)
*sel = io_buffer_select(req, &sqe_len, io->buf_group, issue_flags);
if (!sel->addr)
return -ENOBUFS;
rw->addr = (unsigned long) buf;
rw->addr = (unsigned long) sel->addr;
rw->len = sqe_len;
}
return import_ubuf(ddir, buf, sqe_len, &io->iter);
return import_ubuf(ddir, sel->addr, sqe_len, &io->iter);
}
static inline int io_import_rw_buffer(int rw, struct io_kiocb *req,
struct io_async_rw *io,
struct io_br_sel *sel,
unsigned int issue_flags)
{
int ret;
ret = __io_import_rw_buffer(rw, req, io, issue_flags);
ret = __io_import_rw_buffer(rw, req, io, sel, issue_flags);
if (unlikely(ret < 0))
return ret;
@ -306,10 +307,12 @@ static int __io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
static int io_rw_do_import(struct io_kiocb *req, int ddir)
{
struct io_br_sel sel = { };
if (io_do_buffer_select(req))
return 0;
return io_import_rw_buffer(ddir, req, req->async_data, 0);
return io_import_rw_buffer(ddir, req, req->async_data, &sel, 0);
}
static int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe,
@ -899,7 +902,8 @@ static int io_rw_init_file(struct io_kiocb *req, fmode_t mode, int rw_type)
return 0;
}
static int __io_read(struct io_kiocb *req, unsigned int issue_flags)
static int __io_read(struct io_kiocb *req, struct io_br_sel *sel,
unsigned int issue_flags)
{
bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK;
struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
@ -913,7 +917,7 @@ static int __io_read(struct io_kiocb *req, unsigned int issue_flags)
if (unlikely(ret))
return ret;
} else if (io_do_buffer_select(req)) {
ret = io_import_rw_buffer(ITER_DEST, req, io, issue_flags);
ret = io_import_rw_buffer(ITER_DEST, req, io, sel, issue_flags);
if (unlikely(ret < 0))
return ret;
}
@ -1015,9 +1019,10 @@ done:
int io_read(struct io_kiocb *req, unsigned int issue_flags)
{
struct io_br_sel sel = { };
int ret;
ret = __io_read(req, issue_flags);
ret = __io_read(req, &sel, issue_flags);
if (ret >= 0)
return kiocb_done(req, ret, issue_flags);
@ -1027,6 +1032,7 @@ int io_read(struct io_kiocb *req, unsigned int issue_flags)
int io_read_mshot(struct io_kiocb *req, unsigned int issue_flags)
{
struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw);
struct io_br_sel sel = { };
unsigned int cflags = 0;
int ret;
@ -1038,7 +1044,7 @@ int io_read_mshot(struct io_kiocb *req, unsigned int issue_flags)
/* make it sync, multishot doesn't support async execution */
rw->kiocb.ki_complete = NULL;
ret = __io_read(req, issue_flags);
ret = __io_read(req, &sel, issue_flags);
/*
* If we get -EAGAIN, recycle our buffer and just let normal poll