superrob1500
/
mirror-linux
mirror of https://github.com/torvalds/linux
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

io_uring: keep ring laoyut in a structure 

Add a structure keeping SQ/CQ sizes and offsets. For now it only records
data previously returned from rings_size and the SQ size.

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Signed-off-by: Jens Axboe <axboe@kernel.dk>
pull/1354/merge
Pavel Begunkov 2025-11-12 12:45:57 +00:00 committed by Jens Axboe
parent 0f4b537363
commit 001b76b7e7
3 changed files with 45 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

54
io_uring/io_uring.c
View File

@ -2757,47 +2757,57 @@ static void io_rings_free(struct io_ring_ctx *ctx)
ctx->sq_sqes = NULL; ctx->sq_sqes = NULL;
} }
unsigned long rings_size(unsigned int flags, unsigned int sq_entries, int rings_size(unsigned int flags, unsigned int sq_entries,
unsigned int cq_entries, size_t *sq_offset) unsigned int cq_entries, struct io_rings_layout *rl)
{ {
struct io_rings *rings; struct io_rings *rings;
size_t sqe_size;
size_t off; size_t off;
*sq_offset = SIZE_MAX;
if (flags & IORING_SETUP_CQE_MIXED) { if (flags & IORING_SETUP_CQE_MIXED) {
if (cq_entries < 2) if (cq_entries < 2)
return SIZE_MAX; return -EOVERFLOW;
} }
if (flags & IORING_SETUP_SQE_MIXED) { if (flags & IORING_SETUP_SQE_MIXED) {
if (sq_entries < 2) if (sq_entries < 2)
return SIZE_MAX; return -EOVERFLOW;
} }
rl->sq_array_offset = SIZE_MAX;
sqe_size = sizeof(struct io_uring_sqe);
if (flags & IORING_SETUP_SQE128)
sqe_size *= 2;
rl->sq_size = array_size(sqe_size, sq_entries);
if (rl->sq_size == SIZE_MAX)
return -EOVERFLOW;
off = struct_size(rings, cqes, cq_entries); off = struct_size(rings, cqes, cq_entries);
if (flags & IORING_SETUP_CQE32) if (flags & IORING_SETUP_CQE32)
off = size_mul(off, 2); off = size_mul(off, 2);
if (off == SIZE_MAX) if (off == SIZE_MAX)
return SIZE_MAX; return -EOVERFLOW;
#ifdef CONFIG_SMP #ifdef CONFIG_SMP
off = ALIGN(off, SMP_CACHE_BYTES); off = ALIGN(off, SMP_CACHE_BYTES);
if (off == 0) if (off == 0)
return SIZE_MAX; return -EOVERFLOW;
#endif #endif
if (!(flags & IORING_SETUP_NO_SQARRAY)) { if (!(flags & IORING_SETUP_NO_SQARRAY)) {
size_t sq_array_size; size_t sq_array_size;
*sq_offset = off; rl->sq_array_offset = off;
sq_array_size = array_size(sizeof(u32), sq_entries); sq_array_size = array_size(sizeof(u32), sq_entries);
off = size_add(off, sq_array_size); off = size_add(off, sq_array_size);
if (off == SIZE_MAX) if (off == SIZE_MAX)
return SIZE_MAX; return -EOVERFLOW;
} }
return off; rl->rings_size = off;
return 0;
} }
static __cold void __io_req_caches_free(struct io_ring_ctx *ctx) static __cold void __io_req_caches_free(struct io_ring_ctx *ctx)
@ -3346,28 +3356,20 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,
struct io_uring_params *p) struct io_uring_params *p)
{ {
struct io_uring_region_desc rd; struct io_uring_region_desc rd;
struct io_rings_layout __rl, *rl = &__rl;
struct io_rings *rings; struct io_rings *rings;
size_t sq_array_offset;
size_t sq_size, cq_size, sqe_size;
int ret; int ret;
/* make sure these are sane, as we already accounted them */ /* make sure these are sane, as we already accounted them */
ctx->sq_entries = p->sq_entries; ctx->sq_entries = p->sq_entries;
ctx->cq_entries = p->cq_entries; ctx->cq_entries = p->cq_entries;
sqe_size = sizeof(struct io_uring_sqe); ret = rings_size(ctx->flags, p->sq_entries, p->cq_entries, rl);
if (p->flags & IORING_SETUP_SQE128) if (ret)
sqe_size *= 2; return ret;
sq_size = array_size(sqe_size, p->sq_entries);
if (sq_size == SIZE_MAX)
return -EOVERFLOW;
cq_size = rings_size(ctx->flags, p->sq_entries, p->cq_entries,
&sq_array_offset);
if (cq_size == SIZE_MAX)
return -EOVERFLOW;
memset(&rd, 0, sizeof(rd)); memset(&rd, 0, sizeof(rd));
rd.size = PAGE_ALIGN(cq_size); rd.size = PAGE_ALIGN(rl->rings_size);
if (ctx->flags & IORING_SETUP_NO_MMAP) { if (ctx->flags & IORING_SETUP_NO_MMAP) {
rd.user_addr = p->cq_off.user_addr; rd.user_addr = p->cq_off.user_addr;
rd.flags |= IORING_MEM_REGION_TYPE_USER; rd.flags |= IORING_MEM_REGION_TYPE_USER;
@ -3378,10 +3380,10 @@ static __cold int io_allocate_scq_urings(struct io_ring_ctx *ctx,
ctx->rings = rings = io_region_get_ptr(&ctx->ring_region); ctx->rings = rings = io_region_get_ptr(&ctx->ring_region);
if (!(ctx->flags & IORING_SETUP_NO_SQARRAY)) if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))
ctx->sq_array = (u32 *)((char *)rings + sq_array_offset); ctx->sq_array = (u32 *)((char *)rings + rl->sq_array_offset);
memset(&rd, 0, sizeof(rd)); memset(&rd, 0, sizeof(rd));
rd.size = PAGE_ALIGN(sq_size); rd.size = PAGE_ALIGN(rl->sq_size);
if (ctx->flags & IORING_SETUP_NO_MMAP) { if (ctx->flags & IORING_SETUP_NO_MMAP) {
rd.user_addr = p->sq_off.user_addr; rd.user_addr = p->sq_off.user_addr;
rd.flags |= IORING_MEM_REGION_TYPE_USER; rd.flags |= IORING_MEM_REGION_TYPE_USER;

12
io_uring/io_uring.h
View File

@ -17,6 +17,14 @@
#include <trace/events/io_uring.h> #include <trace/events/io_uring.h>
#endif #endif
struct io_rings_layout {
/* size of CQ + headers + SQ offset array */
size_t rings_size;
size_t sq_size;
size_t sq_array_offset;
};
struct io_ctx_config { struct io_ctx_config {
struct io_uring_params p; struct io_uring_params p;
struct io_uring_params __user *uptr; struct io_uring_params __user *uptr;
@ -139,8 +147,8 @@ static inline bool io_should_wake(struct io_wait_queue *iowq)
#define IORING_MAX_ENTRIES 32768 #define IORING_MAX_ENTRIES 32768
#define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES) #define IORING_MAX_CQ_ENTRIES (2 * IORING_MAX_ENTRIES)
unsigned long rings_size(unsigned int flags, unsigned int sq_entries, int rings_size(unsigned int flags, unsigned int sq_entries,
unsigned int cq_entries, size_t *sq_offset); unsigned int cq_entries, struct io_rings_layout *rl);
int io_prepare_config(struct io_ctx_config *config); int io_prepare_config(struct io_ctx_config *config);
bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow, bool cqe32); bool io_cqe_cache_refill(struct io_ring_ctx *ctx, bool overflow, bool cqe32);

24
io_uring/register.c
View File

@ -401,9 +401,9 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
struct io_ctx_config config; struct io_ctx_config config;
struct io_uring_region_desc rd; struct io_uring_region_desc rd;
struct io_ring_ctx_rings o = { }, n = { }, *to_free = NULL; struct io_ring_ctx_rings o = { }, n = { }, *to_free = NULL;
size_t size, sq_array_offset;
unsigned i, tail, old_head; unsigned i, tail, old_head;
struct io_uring_params *p = &config.p; struct io_uring_params *p = &config.p;
struct io_rings_layout __rl, *rl = &__rl;
int ret; int ret;
memset(&config, 0, sizeof(config)); memset(&config, 0, sizeof(config));
@ -423,13 +423,12 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
if (unlikely(ret)) if (unlikely(ret))
return ret; return ret;
size = rings_size(p->flags, p->sq_entries, p->cq_entries, ret = rings_size(p->flags, p->sq_entries, p->cq_entries, rl);
&sq_array_offset); if (ret)
if (size == SIZE_MAX) return ret;
return -EOVERFLOW;
memset(&rd, 0, sizeof(rd)); memset(&rd, 0, sizeof(rd));
rd.size = PAGE_ALIGN(size); rd.size = PAGE_ALIGN(rl->rings_size);
if (p->flags & IORING_SETUP_NO_MMAP) { if (p->flags & IORING_SETUP_NO_MMAP) {
rd.user_addr = p->cq_off.user_addr; rd.user_addr = p->cq_off.user_addr;
rd.flags |= IORING_MEM_REGION_TYPE_USER; rd.flags |= IORING_MEM_REGION_TYPE_USER;
@ -458,17 +457,8 @@ static int io_register_resize_rings(struct io_ring_ctx *ctx, void __user *arg)
return -EFAULT; return -EFAULT;
} }
if (p->flags & IORING_SETUP_SQE128)
size = array_size(2 * sizeof(struct io_uring_sqe), p->sq_entries);
else
size = array_size(sizeof(struct io_uring_sqe), p->sq_entries);
if (size == SIZE_MAX) {
io_register_free_rings(ctx, &n);
return -EOVERFLOW;
}
memset(&rd, 0, sizeof(rd)); memset(&rd, 0, sizeof(rd));
rd.size = PAGE_ALIGN(size); rd.size = PAGE_ALIGN(rl->sq_size);
if (p->flags & IORING_SETUP_NO_MMAP) { if (p->flags & IORING_SETUP_NO_MMAP) {
rd.user_addr = p->sq_off.user_addr; rd.user_addr = p->sq_off.user_addr;
rd.flags |= IORING_MEM_REGION_TYPE_USER; rd.flags |= IORING_MEM_REGION_TYPE_USER;
@ -551,7 +541,7 @@ overflow:
/* all done, store old pointers and assign new ones */ /* all done, store old pointers and assign new ones */
if (!(ctx->flags & IORING_SETUP_NO_SQARRAY)) if (!(ctx->flags & IORING_SETUP_NO_SQARRAY))
ctx->sq_array = (u32 *)((char *)n.rings + sq_array_offset); ctx->sq_array = (u32 *)((char *)n.rings + rl->sq_array_offset);
ctx->sq_entries = p->sq_entries; ctx->sq_entries = p->sq_entries;
ctx->cq_entries = p->cq_entries; ctx->cq_entries = p->cq_entries;