diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h index 0651865a4564..5b004b95648b 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -1037,6 +1037,8 @@ static inline u64 cgroup_id_from_mm(struct mm_struct *mm) return id; } +void mem_cgroup_flush_workqueue(void); + extern int mem_cgroup_init(void); #else /* CONFIG_MEMCG */ @@ -1436,6 +1438,8 @@ static inline u64 cgroup_id_from_mm(struct mm_struct *mm) return 0; } +static inline void mem_cgroup_flush_workqueue(void) { } + static inline int mem_cgroup_init(void) { return 0; } #endif /* CONFIG_MEMCG */ diff --git a/kernel/sched/isolation.c b/kernel/sched/isolation.c index 674a02891b38..f4053ebf4027 100644 --- a/kernel/sched/isolation.c +++ b/kernel/sched/isolation.c @@ -144,6 +144,8 @@ int housekeeping_update(struct cpumask *isol_mask) synchronize_rcu(); + mem_cgroup_flush_workqueue(); + kfree(old); return 0; diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h index 653e898a996a..65dfa48e54b7 100644 --- a/kernel/sched/sched.h +++ b/kernel/sched/sched.h @@ -44,6 +44,7 @@ #include #include #include +#include #include #include #include diff --git a/mm/memcontrol.c b/mm/memcontrol.c index 2289a0299331..b3ca241bb1d6 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -96,6 +96,8 @@ static bool cgroup_memory_nokmem __ro_after_init; /* BPF memory accounting disabled? */ static bool cgroup_memory_nobpf __ro_after_init; +static struct workqueue_struct *memcg_wq __ro_after_init; + static struct kmem_cache *memcg_cachep; static struct kmem_cache *memcg_pn_cachep; @@ -2013,7 +2015,7 @@ static void schedule_drain_work(int cpu, struct work_struct *work) */ guard(rcu)(); if (!cpu_is_isolated(cpu)) - schedule_work_on(cpu, work); + queue_work_on(cpu, memcg_wq, work); } /* @@ -5125,6 +5127,11 @@ void mem_cgroup_sk_uncharge(const struct sock *sk, unsigned int nr_pages) refill_stock(memcg, nr_pages); } +void mem_cgroup_flush_workqueue(void) +{ + flush_workqueue(memcg_wq); +} + static int __init cgroup_memory(char *s) { char *token; @@ -5167,6 +5174,9 @@ int __init mem_cgroup_init(void) cpuhp_setup_state_nocalls(CPUHP_MM_MEMCQ_DEAD, "mm/memctrl:dead", NULL, memcg_hotplug_cpu_dead); + memcg_wq = alloc_workqueue("memcg", WQ_PERCPU, 0); + WARN_ON(!memcg_wq); + for_each_possible_cpu(cpu) { INIT_WORK(&per_cpu_ptr(&memcg_stock, cpu)->work, drain_local_memcg_stock);