superrob1500
/
mirror-linux
mirror of https://github.com/torvalds/linux
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

io_uring/rsrc: send exact nr_segs for fixed buffer 

Sending exact nr_segs, avoids bio split check and processing in
block layer, which takes around 5%[1] of overall CPU utilization.

In our setup, we see overall improvement of IOPS from 7.15M to 7.65M [2]
and 5% less CPU utilization.

[1]
     3.52%  io_uring         [kernel.kallsyms]     [k] bio_split_rw_at
     1.42%  io_uring         [kernel.kallsyms]     [k] bio_split_rw
     0.62%  io_uring         [kernel.kallsyms]     [k] bio_submit_split

[2]
sudo taskset -c 0,1 ./t/io_uring -b512 -d128 -c32 -s32 -p1 -F1 -B1 -n2
-r4 /dev/nvme0n1 /dev/nvme1n1

Signed-off-by: Nitesh Shetty <nj.shetty@samsung.com>
[Pavel: fixed for kbuf, rebased and reworked on top of cleanups]
Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/7a1a49a8d053bd617c244291d63dbfbc07afde36.1744882081.git.asml.silence@gmail.com
[axboe: fold in fix factoring in buf reg offset]
Signed-off-by: Jens Axboe <axboe@kernel.dk>
pull/1205/head
Nitesh Shetty 2025-04-17 10:32:34 +01:00 committed by Jens Axboe
parent 59852ebad9
commit 80c7378f94
1 changed files with 4 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
io_uring/rsrc.c
View File

@ -1037,6 +1037,7 @@ static int io_import_fixed(int ddir, struct iov_iter *iter,
u64 buf_addr, size_t len) u64 buf_addr, size_t len)
{ {
const struct bio_vec *bvec; const struct bio_vec *bvec;
size_t folio_mask;
unsigned nr_segs; unsigned nr_segs;
size_t offset; size_t offset;
int ret; int ret;
@ -1067,6 +1068,7 @@ static int io_import_fixed(int ddir, struct iov_iter *iter,
* 2) all bvecs are the same in size, except potentially the * 2) all bvecs are the same in size, except potentially the
* first and last bvec * first and last bvec
*/ */
folio_mask = (1UL << imu->folio_shift) - 1;
bvec = imu->bvec; bvec = imu->bvec;
if (offset >= bvec->bv_len) { if (offset >= bvec->bv_len) {
unsigned long seg_skip; unsigned long seg_skip;
@ -1075,10 +1077,9 @@ static int io_import_fixed(int ddir, struct iov_iter *iter,
offset -= bvec->bv_len; offset -= bvec->bv_len;
seg_skip = 1 + (offset >> imu->folio_shift); seg_skip = 1 + (offset >> imu->folio_shift);
bvec += seg_skip; bvec += seg_skip;
offset &= (1UL << imu->folio_shift) - 1; offset &= folio_mask;
} }
nr_segs = (offset + len + bvec->bv_offset + folio_mask) >> imu->folio_shift;
nr_segs = imu->nr_bvecs - (bvec - imu->bvec);
iov_iter_bvec(iter, ddir, bvec, nr_segs, len); iov_iter_bvec(iter, ddir, bvec, nr_segs, len);
iter->iov_offset = offset; iter->iov_offset = offset;
return 0; return 0;