superrob1500
/
mirror-linux
mirror of https://github.com/torvalds/linux
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

io_uring/wq: avoid indirect do_work/free_work calls 

struct io_wq stores do_work and free_work function pointers which are
called on each work item. But these function pointers are always set to
io_wq_submit_work and io_wq_free_work, respectively. So remove these
function pointers and just call the functions directly.

Signed-off-by: Caleb Sander Mateos <csander@purestorage.com>
Link: https://lore.kernel.org/r/20250329161527.3281314-1-csander@purestorage.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
pull/1250/head
Caleb Sander Mateos 2025-03-29 10:15:24 -06:00 committed by Jens Axboe
parent 9d7a0577c9
commit 9fe99eed91
4 changed files with 5 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
io_uring/io-wq.c
View File

@ -114,9 +114,6 @@ enum {
struct io_wq { struct io_wq {
unsigned long state; unsigned long state;
free_work_fn *free_work;
io_wq_work_fn *do_work;
struct io_wq_hash *hash; struct io_wq_hash *hash;
atomic_t worker_refs; atomic_t worker_refs;
@ -612,10 +609,10 @@ static void io_worker_handle_work(struct io_wq_acct *acct,
if (do_kill && if (do_kill &&
(work_flags & IO_WQ_WORK_UNBOUND)) (work_flags & IO_WQ_WORK_UNBOUND))
atomic_or(IO_WQ_WORK_CANCEL, &work->flags); atomic_or(IO_WQ_WORK_CANCEL, &work->flags);
wq->do_work(work); io_wq_submit_work(work);
io_assign_current_work(worker, NULL); io_assign_current_work(worker, NULL);
linked = wq->free_work(work); linked = io_wq_free_work(work);
work = next_hashed; work = next_hashed;
if (!work && linked && !io_wq_is_hashed(linked)) { if (!work && linked && !io_wq_is_hashed(linked)) {
work = linked; work = linked;
@ -934,8 +931,8 @@ static void io_run_cancel(struct io_wq_work *work, struct io_wq *wq)
{ {
do { do {
atomic_or(IO_WQ_WORK_CANCEL, &work->flags); atomic_or(IO_WQ_WORK_CANCEL, &work->flags);
wq->do_work(work); io_wq_submit_work(work);
work = wq->free_work(work); work = io_wq_free_work(work);
} while (work); } while (work);
} }
@ -1195,8 +1192,6 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data)
int ret, i; int ret, i;
struct io_wq *wq; struct io_wq *wq;
if (WARN_ON_ONCE(!data->free_work || !data->do_work))
return ERR_PTR(-EINVAL);
if (WARN_ON_ONCE(!bounded)) if (WARN_ON_ONCE(!bounded))
return ERR_PTR(-EINVAL); return ERR_PTR(-EINVAL);
@ -1206,8 +1201,6 @@ struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data)
refcount_inc(&data->hash->refs); refcount_inc(&data->hash->refs);
wq->hash = data->hash; wq->hash = data->hash;
wq->free_work = data->free_work;
wq->do_work = data->do_work;
ret = -ENOMEM; ret = -ENOMEM;

5
io_uring/io-wq.h
View File

@ -21,9 +21,6 @@ enum io_wq_cancel {
IO_WQ_CANCEL_NOTFOUND, /* work not found */ IO_WQ_CANCEL_NOTFOUND, /* work not found */
}; };
typedef struct io_wq_work *(free_work_fn)(struct io_wq_work *);
typedef void (io_wq_work_fn)(struct io_wq_work *);
struct io_wq_hash { struct io_wq_hash {
refcount_t refs; refcount_t refs;
unsigned long map; unsigned long map;
@ -39,8 +36,6 @@ static inline void io_wq_put_hash(struct io_wq_hash *hash)
struct io_wq_data { struct io_wq_data {
struct io_wq_hash *hash; struct io_wq_hash *hash;
struct task_struct *task; struct task_struct *task;
io_wq_work_fn *do_work;
free_work_fn *free_work;
}; };
struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data); struct io_wq *io_wq_create(unsigned bounded, struct io_wq_data *data);

2
io_uring/io_uring.c
View File

@ -1812,7 +1812,7 @@ void io_wq_submit_work(struct io_wq_work *work)
bool needs_poll = false; bool needs_poll = false;
int ret = 0, err = -ECANCELED; int ret = 0, err = -ECANCELED;
/* one will be dropped by ->io_wq_free_work() after returning to io-wq */ /* one will be dropped by io_wq_free_work() after returning to io-wq */
if (!(req->flags & REQ_F_REFCOUNT)) if (!(req->flags & REQ_F_REFCOUNT))
__io_req_set_refcount(req, 2); __io_req_set_refcount(req, 2);
else else

2
io_uring/tctx.c
View File

@ -35,8 +35,6 @@ static struct io_wq *io_init_wq_offload(struct io_ring_ctx *ctx,
data.hash = hash; data.hash = hash;
data.task = task; data.task = task;
data.free_work = io_wq_free_work;
data.do_work = io_wq_submit_work;
/* Do QD, or 4 * CPUS, whatever is smallest */ /* Do QD, or 4 * CPUS, whatever is smallest */
concurrency = min(ctx->sq_entries, 4 * num_online_cpus()); concurrency = min(ctx->sq_entries, 4 * num_online_cpus());