fs/erofs/zutil.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2018 HUAWEI, Inc.
   4  *             https://www.huawei.com/
   5  * Copyright (C) 2024 Alibaba Cloud
   6  */
   7 #include "internal.h"
   8
   9 struct z_erofs_gbuf {
  10         spinlock_t lock;
  11         void *ptr;
  12         struct page **pages;
  13         unsigned int nrpages;
  14 };
  15
  16 static struct z_erofs_gbuf *z_erofs_gbufpool, *z_erofs_rsvbuf;
  17 static unsigned int z_erofs_gbuf_count, z_erofs_gbuf_nrpages,
  18                 z_erofs_rsv_nrpages;
  19
  20 module_param_named(global_buffers, z_erofs_gbuf_count, uint, 0444);
  21 module_param_named(reserved_pages, z_erofs_rsv_nrpages, uint, 0444);
  22
  23 atomic_long_t erofs_global_shrink_cnt;  /* for all mounted instances */
  24
  25 /* protects `erofs_sb_list_lock` and the mounted `erofs_sb_list` */
  26 static DEFINE_SPINLOCK(erofs_sb_list_lock);
  27 static LIST_HEAD(erofs_sb_list);
  28 static unsigned int shrinker_run_no;
  29 static struct shrinker *erofs_shrinker_info;
  30
  31 static unsigned int z_erofs_gbuf_id(void)
  32 {
  33         return raw_smp_processor_id() % z_erofs_gbuf_count;
  34 }
  35
  36 void *z_erofs_get_gbuf(unsigned int requiredpages)
  37         __acquires(gbuf->lock)
  38 {
  39         struct z_erofs_gbuf *gbuf;
  40
  41         migrate_disable();
  42         gbuf = &z_erofs_gbufpool[z_erofs_gbuf_id()];
  43         spin_lock(&gbuf->lock);
  44         /* check if the buffer is too small */
  45         if (requiredpages > gbuf->nrpages) {
  46                 spin_unlock(&gbuf->lock);
  47                 migrate_enable();
  48                 /* (for sparse checker) pretend gbuf->lock is still taken */
  49                 __acquire(gbuf->lock);
  50                 return NULL;
  51         }
  52         return gbuf->ptr;
  53 }
  54
  55 void z_erofs_put_gbuf(void *ptr) __releases(gbuf->lock)
  56 {
  57         struct z_erofs_gbuf *gbuf;
  58
  59         gbuf = &z_erofs_gbufpool[z_erofs_gbuf_id()];
  60         DBG_BUGON(gbuf->ptr != ptr);
  61         spin_unlock(&gbuf->lock);
  62         migrate_enable();
  63 }
  64
  65 int z_erofs_gbuf_growsize(unsigned int nrpages)
  66 {
  67         static DEFINE_MUTEX(gbuf_resize_mutex);
  68         struct page **tmp_pages = NULL;
  69         struct z_erofs_gbuf *gbuf;
  70         void *ptr, *old_ptr;
  71         int last, i, j;
  72
  73         mutex_lock(&gbuf_resize_mutex);
  74         /* avoid shrinking gbufs, since no idea how many fses rely on */
  75         if (nrpages <= z_erofs_gbuf_nrpages) {
  76                 mutex_unlock(&gbuf_resize_mutex);
  77                 return 0;
  78         }
  79
  80         for (i = 0; i < z_erofs_gbuf_count; ++i) {
  81                 gbuf = &z_erofs_gbufpool[i];
  82                 tmp_pages = kcalloc(nrpages, sizeof(*tmp_pages), GFP_KERNEL);
  83                 if (!tmp_pages)
  84                         goto out;
  85
  86                 for (j = 0; j < gbuf->nrpages; ++j)
  87                         tmp_pages[j] = gbuf->pages[j];
  88                 do {
  89                         last = j;
  90                         j = alloc_pages_bulk_array(GFP_KERNEL, nrpages,
  91                                                    tmp_pages);
  92                         if (last == j)
  93                                 goto out;
  94                 } while (j != nrpages);
  95
  96                 ptr = vmap(tmp_pages, nrpages, VM_MAP, PAGE_KERNEL);
  97                 if (!ptr)
  98                         goto out;
  99
 100                 spin_lock(&gbuf->lock);
 101                 kfree(gbuf->pages);
 102                 gbuf->pages = tmp_pages;
 103                 old_ptr = gbuf->ptr;
 104                 gbuf->ptr = ptr;
 105                 gbuf->nrpages = nrpages;
 106                 spin_unlock(&gbuf->lock);
 107                 if (old_ptr)
 108                         vunmap(old_ptr);
 109         }
 110         z_erofs_gbuf_nrpages = nrpages;
 111 out:
 112         if (i < z_erofs_gbuf_count && tmp_pages) {
 113                 for (j = 0; j < nrpages; ++j)
 114                         if (tmp_pages[j] && (j >= gbuf->nrpages ||
 115                                              tmp_pages[j] != gbuf->pages[j]))
 116                                 __free_page(tmp_pages[j]);
 117                 kfree(tmp_pages);
 118         }
 119         mutex_unlock(&gbuf_resize_mutex);
 120         return i < z_erofs_gbuf_count ? -ENOMEM : 0;
 121 }
 122
 123 int __init z_erofs_gbuf_init(void)
 124 {
 125         unsigned int i, total = num_possible_cpus();
 126
 127         if (z_erofs_gbuf_count)
 128                 total = min(z_erofs_gbuf_count, total);
 129         z_erofs_gbuf_count = total;
 130
 131         /* The last (special) global buffer is the reserved buffer */
 132         total += !!z_erofs_rsv_nrpages;
 133
 134         z_erofs_gbufpool = kcalloc(total, sizeof(*z_erofs_gbufpool),
 135                                    GFP_KERNEL);
 136         if (!z_erofs_gbufpool)
 137                 return -ENOMEM;
 138
 139         if (z_erofs_rsv_nrpages) {
 140                 z_erofs_rsvbuf = &z_erofs_gbufpool[total - 1];
 141                 z_erofs_rsvbuf->pages = kcalloc(z_erofs_rsv_nrpages,
 142                                 sizeof(*z_erofs_rsvbuf->pages), GFP_KERNEL);
 143                 if (!z_erofs_rsvbuf->pages) {
 144                         z_erofs_rsvbuf = NULL;
 145                         z_erofs_rsv_nrpages = 0;
 146                 }
 147         }
 148         for (i = 0; i < total; ++i)
 149                 spin_lock_init(&z_erofs_gbufpool[i].lock);
 150         return 0;
 151 }
 152
 153 void z_erofs_gbuf_exit(void)
 154 {
 155         int i, j;
 156
 157         for (i = 0; i < z_erofs_gbuf_count + (!!z_erofs_rsvbuf); ++i) {
 158                 struct z_erofs_gbuf *gbuf = &z_erofs_gbufpool[i];
 159
 160                 if (gbuf->ptr) {
 161                         vunmap(gbuf->ptr);
 162                         gbuf->ptr = NULL;
 163                 }
 164
 165                 if (!gbuf->pages)
 166                         continue;
 167
 168                 for (j = 0; j < gbuf->nrpages; ++j)
 169                         if (gbuf->pages[j])
 170                                 put_page(gbuf->pages[j]);
 171                 kfree(gbuf->pages);
 172                 gbuf->pages = NULL;
 173         }
 174         kfree(z_erofs_gbufpool);
 175 }
 176
 177 struct page *__erofs_allocpage(struct page **pagepool, gfp_t gfp, bool tryrsv)
 178 {
 179         struct page *page = *pagepool;
 180
 181         if (page) {
 182                 *pagepool = (struct page *)page_private(page);
 183         } else if (tryrsv && z_erofs_rsvbuf && z_erofs_rsvbuf->nrpages) {
 184                 spin_lock(&z_erofs_rsvbuf->lock);
 185                 if (z_erofs_rsvbuf->nrpages)
 186                         page = z_erofs_rsvbuf->pages[--z_erofs_rsvbuf->nrpages];
 187                 spin_unlock(&z_erofs_rsvbuf->lock);
 188         }
 189         if (!page)
 190                 page = alloc_page(gfp);
 191         DBG_BUGON(page && page_ref_count(page) != 1);
 192         return page;
 193 }
 194
 195 void erofs_release_pages(struct page **pagepool)
 196 {
 197         while (*pagepool) {
 198                 struct page *page = *pagepool;
 199
 200                 *pagepool = (struct page *)page_private(page);
 201                 /* try to fill reserved global pool first */
 202                 if (z_erofs_rsvbuf && z_erofs_rsvbuf->nrpages <
 203                                 z_erofs_rsv_nrpages) {
 204                         spin_lock(&z_erofs_rsvbuf->lock);
 205                         if (z_erofs_rsvbuf->nrpages < z_erofs_rsv_nrpages) {
 206                                 z_erofs_rsvbuf->pages[z_erofs_rsvbuf->nrpages++]
 207                                                 = page;
 208                                 spin_unlock(&z_erofs_rsvbuf->lock);
 209                                 continue;
 210                         }
 211                         spin_unlock(&z_erofs_rsvbuf->lock);
 212                 }
 213                 put_page(page);
 214         }
 215 }
 216
 217 void erofs_shrinker_register(struct super_block *sb)
 218 {
 219         struct erofs_sb_info *sbi = EROFS_SB(sb);
 220
 221         mutex_init(&sbi->umount_mutex);
 222
 223         spin_lock(&erofs_sb_list_lock);
 224         list_add(&sbi->list, &erofs_sb_list);
 225         spin_unlock(&erofs_sb_list_lock);
 226 }
 227
 228 void erofs_shrinker_unregister(struct super_block *sb)
 229 {
 230         struct erofs_sb_info *const sbi = EROFS_SB(sb);
 231
 232         mutex_lock(&sbi->umount_mutex);
 233         while (!xa_empty(&sbi->managed_pslots)) {
 234                 z_erofs_shrink_scan(sbi, ~0UL);
 235                 cond_resched();
 236         }
 237         spin_lock(&erofs_sb_list_lock);
 238         list_del(&sbi->list);
 239         spin_unlock(&erofs_sb_list_lock);
 240         mutex_unlock(&sbi->umount_mutex);
 241 }
 242
 243 static unsigned long erofs_shrink_count(struct shrinker *shrink,
 244                                         struct shrink_control *sc)
 245 {
 246         return atomic_long_read(&erofs_global_shrink_cnt);
 247 }
 248
 249 static unsigned long erofs_shrink_scan(struct shrinker *shrink,
 250                                        struct shrink_control *sc)
 251 {
 252         struct erofs_sb_info *sbi;
 253         struct list_head *p;
 254
 255         unsigned long nr = sc->nr_to_scan;
 256         unsigned int run_no;
 257         unsigned long freed = 0;
 258
 259         spin_lock(&erofs_sb_list_lock);
 260         do {
 261                 run_no = ++shrinker_run_no;
 262         } while (run_no == 0);
 263
 264         /* Iterate over all mounted superblocks and try to shrink them */
 265         p = erofs_sb_list.next;
 266         while (p != &erofs_sb_list) {
 267                 sbi = list_entry(p, struct erofs_sb_info, list);
 268
 269                 /*
 270                  * We move the ones we do to the end of the list, so we stop
 271                  * when we see one we have already done.
 272                  */
 273                 if (sbi->shrinker_run_no == run_no)
 274                         break;
 275
 276                 if (!mutex_trylock(&sbi->umount_mutex)) {
 277                         p = p->next;
 278                         continue;
 279                 }
 280
 281                 spin_unlock(&erofs_sb_list_lock);
 282                 sbi->shrinker_run_no = run_no;
 283                 freed += z_erofs_shrink_scan(sbi, nr - freed);
 284                 spin_lock(&erofs_sb_list_lock);
 285                 /* Get the next list element before we move this one */
 286                 p = p->next;
 287
 288                 /*
 289                  * Move this one to the end of the list to provide some
 290                  * fairness.
 291                  */
 292                 list_move_tail(&sbi->list, &erofs_sb_list);
 293                 mutex_unlock(&sbi->umount_mutex);
 294
 295                 if (freed >= nr)
 296                         break;
 297         }
 298         spin_unlock(&erofs_sb_list_lock);
 299         return freed;
 300 }
 301
 302 int __init erofs_init_shrinker(void)
 303 {
 304         erofs_shrinker_info = shrinker_alloc(0, "erofs-shrinker");
 305         if (!erofs_shrinker_info)
 306                 return -ENOMEM;
 307
 308         erofs_shrinker_info->count_objects = erofs_shrink_count;
 309         erofs_shrinker_info->scan_objects = erofs_shrink_scan;
 310         shrinker_register(erofs_shrinker_info);
 311         return 0;
 312 }
 313
 314 void erofs_exit_shrinker(void)
 315 {
 316         shrinker_free(erofs_shrinker_info);
 317 }