fs/f2fs/segment.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * fs/f2fs/segment.c
   4  *
   5  * Copyright (c) 2012 Samsung Electronics Co., Ltd.
   6  *             http://www.samsung.com/
   7  */
   8 #include <linux/fs.h>
   9 #include <linux/f2fs_fs.h>
  10 #include <linux/bio.h>
  11 #include <linux/blkdev.h>
  12 #include <linux/sched/mm.h>
  13 #include <linux/prefetch.h>
  14 #include <linux/kthread.h>
  15 #include <linux/swap.h>
  16 #include <linux/timer.h>
  17 #include <linux/freezer.h>
  18 #include <linux/sched/signal.h>
  19 #include <linux/random.h>
  20
  21 #include "f2fs.h"
  22 #include "segment.h"
  23 #include "node.h"
  24 #include "gc.h"
  25 #include "iostat.h"
  26 #include <trace/events/f2fs.h>
  27
  28 #define __reverse_ffz(x) __reverse_ffs(~(x))
  29
  30 static struct kmem_cache *discard_entry_slab;
  31 static struct kmem_cache *discard_cmd_slab;
  32 static struct kmem_cache *sit_entry_set_slab;
  33 static struct kmem_cache *revoke_entry_slab;
  34
  35 static unsigned long __reverse_ulong(unsigned char *str)
  36 {
  37         unsigned long tmp = 0;
  38         int shift = 24, idx = 0;
  39
  40 #if BITS_PER_LONG == 64
  41         shift = 56;
  42 #endif
  43         while (shift >= 0) {
  44                 tmp |= (unsigned long)str[idx++] << shift;
  45                 shift -= BITS_PER_BYTE;
  46         }
  47         return tmp;
  48 }
  49
  50 /*
  51  * __reverse_ffs is copied from include/asm-generic/bitops/__ffs.h since
  52  * MSB and LSB are reversed in a byte by f2fs_set_bit.
  53  */
  54 static inline unsigned long __reverse_ffs(unsigned long word)
  55 {
  56         int num = 0;
  57
  58 #if BITS_PER_LONG == 64
  59         if ((word & 0xffffffff00000000UL) == 0)
  60                 num += 32;
  61         else
  62                 word >>= 32;
  63 #endif
  64         if ((word & 0xffff0000) == 0)
  65                 num += 16;
  66         else
  67                 word >>= 16;
  68
  69         if ((word & 0xff00) == 0)
  70                 num += 8;
  71         else
  72                 word >>= 8;
  73
  74         if ((word & 0xf0) == 0)
  75                 num += 4;
  76         else
  77                 word >>= 4;
  78
  79         if ((word & 0xc) == 0)
  80                 num += 2;
  81         else
  82                 word >>= 2;
  83
  84         if ((word & 0x2) == 0)
  85                 num += 1;
  86         return num;
  87 }
  88
  89 /*
  90  * __find_rev_next(_zero)_bit is copied from lib/find_next_bit.c because
  91  * f2fs_set_bit makes MSB and LSB reversed in a byte.
  92  * @size must be integral times of unsigned long.
  93  * Example:
  94  *                             MSB <--> LSB
  95  *   f2fs_set_bit(0, bitmap) => 1000 0000
  96  *   f2fs_set_bit(7, bitmap) => 0000 0001
  97  */
  98 static unsigned long __find_rev_next_bit(const unsigned long *addr,
  99                         unsigned long size, unsigned long offset)
 100 {
 101         const unsigned long *p = addr + BIT_WORD(offset);
 102         unsigned long result = size;
 103         unsigned long tmp;
 104
 105         if (offset >= size)
 106                 return size;
 107
 108         size -= (offset & ~(BITS_PER_LONG - 1));
 109         offset %= BITS_PER_LONG;
 110
 111         while (1) {
 112                 if (*p == 0)
 113                         goto pass;
 114
 115                 tmp = __reverse_ulong((unsigned char *)p);
 116
 117                 tmp &= ~0UL >> offset;
 118                 if (size < BITS_PER_LONG)
 119                         tmp &= (~0UL << (BITS_PER_LONG - size));
 120                 if (tmp)
 121                         goto found;
 122 pass:
 123                 if (size <= BITS_PER_LONG)
 124                         break;
 125                 size -= BITS_PER_LONG;
 126                 offset = 0;
 127                 p++;
 128         }
 129         return result;
 130 found:
 131         return result - size + __reverse_ffs(tmp);
 132 }
 133
 134 static unsigned long __find_rev_next_zero_bit(const unsigned long *addr,
 135                         unsigned long size, unsigned long offset)
 136 {
 137         const unsigned long *p = addr + BIT_WORD(offset);
 138         unsigned long result = size;
 139         unsigned long tmp;
 140
 141         if (offset >= size)
 142                 return size;
 143
 144         size -= (offset & ~(BITS_PER_LONG - 1));
 145         offset %= BITS_PER_LONG;
 146
 147         while (1) {
 148                 if (*p == ~0UL)
 149                         goto pass;
 150
 151                 tmp = __reverse_ulong((unsigned char *)p);
 152
 153                 if (offset)
 154                         tmp |= ~0UL << (BITS_PER_LONG - offset);
 155                 if (size < BITS_PER_LONG)
 156                         tmp |= ~0UL >> size;
 157                 if (tmp != ~0UL)
 158                         goto found;
 159 pass:
 160                 if (size <= BITS_PER_LONG)
 161                         break;
 162                 size -= BITS_PER_LONG;
 163                 offset = 0;
 164                 p++;
 165         }
 166         return result;
 167 found:
 168         return result - size + __reverse_ffz(tmp);
 169 }
 170
 171 bool f2fs_need_SSR(struct f2fs_sb_info *sbi)
 172 {
 173         int node_secs = get_blocktype_secs(sbi, F2FS_DIRTY_NODES);
 174         int dent_secs = get_blocktype_secs(sbi, F2FS_DIRTY_DENTS);
 175         int imeta_secs = get_blocktype_secs(sbi, F2FS_DIRTY_IMETA);
 176
 177         if (f2fs_lfs_mode(sbi))
 178                 return false;
 179         if (sbi->gc_mode == GC_URGENT_HIGH)
 180                 return true;
 181         if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
 182                 return true;
 183
 184         return free_sections(sbi) <= (node_secs + 2 * dent_secs + imeta_secs +
 185                         SM_I(sbi)->min_ssr_sections + reserved_sections(sbi));
 186 }
 187
 188 void f2fs_abort_atomic_write(struct inode *inode, bool clean)
 189 {
 190         struct f2fs_inode_info *fi = F2FS_I(inode);
 191
 192         if (!f2fs_is_atomic_file(inode))
 193                 return;
 194
 195         if (clean)
 196                 truncate_inode_pages_final(inode->i_mapping);
 197
 198         release_atomic_write_cnt(inode);
 199         clear_inode_flag(inode, FI_ATOMIC_COMMITTED);
 200         clear_inode_flag(inode, FI_ATOMIC_REPLACE);
 201         clear_inode_flag(inode, FI_ATOMIC_FILE);
 202         if (is_inode_flag_set(inode, FI_ATOMIC_DIRTIED)) {
 203                 clear_inode_flag(inode, FI_ATOMIC_DIRTIED);
 204                 /*
 205                  * The vfs inode keeps clean during commit, but the f2fs inode
 206                  * doesn't. So clear the dirty state after commit and let
 207                  * f2fs_mark_inode_dirty_sync ensure a consistent dirty state.
 208                  */
 209                 f2fs_inode_synced(inode);
 210                 f2fs_mark_inode_dirty_sync(inode, true);
 211         }
 212         stat_dec_atomic_inode(inode);
 213
 214         F2FS_I(inode)->atomic_write_task = NULL;
 215
 216         if (clean) {
 217                 f2fs_i_size_write(inode, fi->original_i_size);
 218                 fi->original_i_size = 0;
 219         }
 220         /* avoid stale dirty inode during eviction */
 221         sync_inode_metadata(inode, 0);
 222 }
 223
 224 static int __replace_atomic_write_block(struct inode *inode, pgoff_t index,
 225                         block_t new_addr, block_t *old_addr, bool recover)
 226 {
 227         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 228         struct dnode_of_data dn;
 229         struct node_info ni;
 230         int err;
 231
 232 retry:
 233         set_new_dnode(&dn, inode, NULL, NULL, 0);
 234         err = f2fs_get_dnode_of_data(&dn, index, ALLOC_NODE);
 235         if (err) {
 236                 if (err == -ENOMEM) {
 237                         f2fs_io_schedule_timeout(DEFAULT_IO_TIMEOUT);
 238                         goto retry;
 239                 }
 240                 return err;
 241         }
 242
 243         err = f2fs_get_node_info(sbi, dn.nid, &ni, false);
 244         if (err) {
 245                 f2fs_put_dnode(&dn);
 246                 return err;
 247         }
 248
 249         if (recover) {
 250                 /* dn.data_blkaddr is always valid */
 251                 if (!__is_valid_data_blkaddr(new_addr)) {
 252                         if (new_addr == NULL_ADDR)
 253                                 dec_valid_block_count(sbi, inode, 1);
 254                         f2fs_invalidate_blocks(sbi, dn.data_blkaddr, 1);
 255                         f2fs_update_data_blkaddr(&dn, new_addr);
 256                 } else {
 257                         f2fs_replace_block(sbi, &dn, dn.data_blkaddr,
 258                                 new_addr, ni.version, true, true);
 259                 }
 260         } else {
 261                 blkcnt_t count = 1;
 262
 263                 err = inc_valid_block_count(sbi, inode, &count, true);
 264                 if (err) {
 265                         f2fs_put_dnode(&dn);
 266                         return err;
 267                 }
 268
 269                 *old_addr = dn.data_blkaddr;
 270                 f2fs_truncate_data_blocks_range(&dn, 1);
 271                 dec_valid_block_count(sbi, F2FS_I(inode)->cow_inode, count);
 272
 273                 f2fs_replace_block(sbi, &dn, dn.data_blkaddr, new_addr,
 274                                         ni.version, true, false);
 275         }
 276
 277         f2fs_put_dnode(&dn);
 278
 279         trace_f2fs_replace_atomic_write_block(inode, F2FS_I(inode)->cow_inode,
 280                         index, old_addr ? *old_addr : 0, new_addr, recover);
 281         return 0;
 282 }
 283
 284 static void __complete_revoke_list(struct inode *inode, struct list_head *head,
 285                                         bool revoke)
 286 {
 287         struct revoke_entry *cur, *tmp;
 288         pgoff_t start_index = 0;
 289         bool truncate = is_inode_flag_set(inode, FI_ATOMIC_REPLACE);
 290
 291         list_for_each_entry_safe(cur, tmp, head, list) {
 292                 if (revoke) {
 293                         __replace_atomic_write_block(inode, cur->index,
 294                                                 cur->old_addr, NULL, true);
 295                 } else if (truncate) {
 296                         f2fs_truncate_hole(inode, start_index, cur->index);
 297                         start_index = cur->index + 1;
 298                 }
 299
 300                 list_del(&cur->list);
 301                 kmem_cache_free(revoke_entry_slab, cur);
 302         }
 303
 304         if (!revoke && truncate)
 305                 f2fs_do_truncate_blocks(inode, start_index * PAGE_SIZE, false);
 306 }
 307
 308 static int __f2fs_commit_atomic_write(struct inode *inode)
 309 {
 310         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 311         struct f2fs_inode_info *fi = F2FS_I(inode);
 312         struct inode *cow_inode = fi->cow_inode;
 313         struct revoke_entry *new;
 314         struct list_head revoke_list;
 315         block_t blkaddr;
 316         struct dnode_of_data dn;
 317         pgoff_t len = DIV_ROUND_UP(i_size_read(inode), PAGE_SIZE);
 318         pgoff_t off = 0, blen, index;
 319         int ret = 0, i;
 320
 321         INIT_LIST_HEAD(&revoke_list);
 322
 323         while (len) {
 324                 blen = min_t(pgoff_t, ADDRS_PER_BLOCK(cow_inode), len);
 325
 326                 set_new_dnode(&dn, cow_inode, NULL, NULL, 0);
 327                 ret = f2fs_get_dnode_of_data(&dn, off, LOOKUP_NODE_RA);
 328                 if (ret && ret != -ENOENT) {
 329                         goto out;
 330                 } else if (ret == -ENOENT) {
 331                         ret = 0;
 332                         if (dn.max_level == 0)
 333                                 goto out;
 334                         goto next;
 335                 }
 336
 337                 blen = min((pgoff_t)ADDRS_PER_PAGE(dn.node_page, cow_inode),
 338                                 len);
 339                 index = off;
 340                 for (i = 0; i < blen; i++, dn.ofs_in_node++, index++) {
 341                         blkaddr = f2fs_data_blkaddr(&dn);
 342
 343                         if (!__is_valid_data_blkaddr(blkaddr)) {
 344                                 continue;
 345                         } else if (!f2fs_is_valid_blkaddr(sbi, blkaddr,
 346                                         DATA_GENERIC_ENHANCE)) {
 347                                 f2fs_put_dnode(&dn);
 348                                 ret = -EFSCORRUPTED;
 349                                 goto out;
 350                         }
 351
 352                         new = f2fs_kmem_cache_alloc(revoke_entry_slab, GFP_NOFS,
 353                                                         true, NULL);
 354
 355                         ret = __replace_atomic_write_block(inode, index, blkaddr,
 356                                                         &new->old_addr, false);
 357                         if (ret) {
 358                                 f2fs_put_dnode(&dn);
 359                                 kmem_cache_free(revoke_entry_slab, new);
 360                                 goto out;
 361                         }
 362
 363                         f2fs_update_data_blkaddr(&dn, NULL_ADDR);
 364                         new->index = index;
 365                         list_add_tail(&new->list, &revoke_list);
 366                 }
 367                 f2fs_put_dnode(&dn);
 368 next:
 369                 off += blen;
 370                 len -= blen;
 371         }
 372
 373 out:
 374         if (ret) {
 375                 sbi->revoked_atomic_block += fi->atomic_write_cnt;
 376         } else {
 377                 sbi->committed_atomic_block += fi->atomic_write_cnt;
 378                 set_inode_flag(inode, FI_ATOMIC_COMMITTED);
 379                 if (is_inode_flag_set(inode, FI_ATOMIC_DIRTIED)) {
 380                         clear_inode_flag(inode, FI_ATOMIC_DIRTIED);
 381                         f2fs_mark_inode_dirty_sync(inode, true);
 382                 }
 383         }
 384
 385         __complete_revoke_list(inode, &revoke_list, ret ? true : false);
 386
 387         return ret;
 388 }
 389
 390 int f2fs_commit_atomic_write(struct inode *inode)
 391 {
 392         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
 393         struct f2fs_inode_info *fi = F2FS_I(inode);
 394         int err;
 395
 396         err = filemap_write_and_wait_range(inode->i_mapping, 0, LLONG_MAX);
 397         if (err)
 398                 return err;
 399
 400         f2fs_down_write(&fi->i_gc_rwsem[WRITE]);
 401         f2fs_lock_op(sbi);
 402
 403         err = __f2fs_commit_atomic_write(inode);
 404
 405         f2fs_unlock_op(sbi);
 406         f2fs_up_write(&fi->i_gc_rwsem[WRITE]);
 407
 408         return err;
 409 }
 410
 411 /*
 412  * This function balances dirty node and dentry pages.
 413  * In addition, it controls garbage collection.
 414  */
 415 void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need)
 416 {
 417         if (f2fs_cp_error(sbi))
 418                 return;
 419
 420         if (time_to_inject(sbi, FAULT_CHECKPOINT))
 421                 f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_FAULT_INJECT);
 422
 423         /* balance_fs_bg is able to be pending */
 424         if (need && excess_cached_nats(sbi))
 425                 f2fs_balance_fs_bg(sbi, false);
 426
 427         if (!f2fs_is_checkpoint_ready(sbi))
 428                 return;
 429
 430         /*
 431          * We should do GC or end up with checkpoint, if there are so many dirty
 432          * dir/node pages without enough free segments.
 433          */
 434         if (has_enough_free_secs(sbi, 0, 0))
 435                 return;
 436
 437         if (test_opt(sbi, GC_MERGE) && sbi->gc_thread &&
 438                                 sbi->gc_thread->f2fs_gc_task) {
 439                 DEFINE_WAIT(wait);
 440
 441                 prepare_to_wait(&sbi->gc_thread->fggc_wq, &wait,
 442                                         TASK_UNINTERRUPTIBLE);
 443                 wake_up(&sbi->gc_thread->gc_wait_queue_head);
 444                 io_schedule();
 445                 finish_wait(&sbi->gc_thread->fggc_wq, &wait);
 446         } else {
 447                 struct f2fs_gc_control gc_control = {
 448                         .victim_segno = NULL_SEGNO,
 449                         .init_gc_type = BG_GC,
 450                         .no_bg_gc = true,
 451                         .should_migrate_blocks = false,
 452                         .err_gc_skipped = false,
 453                         .nr_free_secs = 1 };
 454                 f2fs_down_write(&sbi->gc_lock);
 455                 stat_inc_gc_call_count(sbi, FOREGROUND);
 456                 f2fs_gc(sbi, &gc_control);
 457         }
 458 }
 459
 460 static inline bool excess_dirty_threshold(struct f2fs_sb_info *sbi)
 461 {
 462         int factor = f2fs_rwsem_is_locked(&sbi->cp_rwsem) ? 3 : 2;
 463         unsigned int dents = get_pages(sbi, F2FS_DIRTY_DENTS);
 464         unsigned int qdata = get_pages(sbi, F2FS_DIRTY_QDATA);
 465         unsigned int nodes = get_pages(sbi, F2FS_DIRTY_NODES);
 466         unsigned int meta = get_pages(sbi, F2FS_DIRTY_META);
 467         unsigned int imeta = get_pages(sbi, F2FS_DIRTY_IMETA);
 468         unsigned int threshold =
 469                 SEGS_TO_BLKS(sbi, (factor * DEFAULT_DIRTY_THRESHOLD));
 470         unsigned int global_threshold = threshold * 3 / 2;
 471
 472         if (dents >= threshold || qdata >= threshold ||
 473                 nodes >= threshold || meta >= threshold ||
 474                 imeta >= threshold)
 475                 return true;
 476         return dents + qdata + nodes + meta + imeta >  global_threshold;
 477 }
 478
 479 void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi, bool from_bg)
 480 {
 481         if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
 482                 return;
 483
 484         /* try to shrink extent cache when there is no enough memory */
 485         if (!f2fs_available_free_memory(sbi, READ_EXTENT_CACHE))
 486                 f2fs_shrink_read_extent_tree(sbi,
 487                                 READ_EXTENT_CACHE_SHRINK_NUMBER);
 488
 489         /* try to shrink age extent cache when there is no enough memory */
 490         if (!f2fs_available_free_memory(sbi, AGE_EXTENT_CACHE))
 491                 f2fs_shrink_age_extent_tree(sbi,
 492                                 AGE_EXTENT_CACHE_SHRINK_NUMBER);
 493
 494         /* check the # of cached NAT entries */
 495         if (!f2fs_available_free_memory(sbi, NAT_ENTRIES))
 496                 f2fs_try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK);
 497
 498         if (!f2fs_available_free_memory(sbi, FREE_NIDS))
 499                 f2fs_try_to_free_nids(sbi, MAX_FREE_NIDS);
 500         else
 501                 f2fs_build_free_nids(sbi, false, false);
 502
 503         if (excess_dirty_nats(sbi) || excess_dirty_threshold(sbi) ||
 504                 excess_prefree_segs(sbi) || !f2fs_space_for_roll_forward(sbi))
 505                 goto do_sync;
 506
 507         /* there is background inflight IO or foreground operation recently */
 508         if (is_inflight_io(sbi, REQ_TIME) ||
 509                 (!f2fs_time_over(sbi, REQ_TIME) && f2fs_rwsem_is_locked(&sbi->cp_rwsem)))
 510                 return;
 511
 512         /* exceed periodical checkpoint timeout threshold */
 513         if (f2fs_time_over(sbi, CP_TIME))
 514                 goto do_sync;
 515
 516         /* checkpoint is the only way to shrink partial cached entries */
 517         if (f2fs_available_free_memory(sbi, NAT_ENTRIES) &&
 518                 f2fs_available_free_memory(sbi, INO_ENTRIES))
 519                 return;
 520
 521 do_sync:
 522         if (test_opt(sbi, DATA_FLUSH) && from_bg) {
 523                 struct blk_plug plug;
 524
 525                 mutex_lock(&sbi->flush_lock);
 526
 527                 blk_start_plug(&plug);
 528                 f2fs_sync_dirty_inodes(sbi, FILE_INODE, false);
 529                 blk_finish_plug(&plug);
 530
 531                 mutex_unlock(&sbi->flush_lock);
 532         }
 533         stat_inc_cp_call_count(sbi, BACKGROUND);
 534         f2fs_sync_fs(sbi->sb, 1);
 535 }
 536
 537 static int __submit_flush_wait(struct f2fs_sb_info *sbi,
 538                                 struct block_device *bdev)
 539 {
 540         int ret = blkdev_issue_flush(bdev);
 541
 542         trace_f2fs_issue_flush(bdev, test_opt(sbi, NOBARRIER),
 543                                 test_opt(sbi, FLUSH_MERGE), ret);
 544         if (!ret)
 545                 f2fs_update_iostat(sbi, NULL, FS_FLUSH_IO, 0);
 546         return ret;
 547 }
 548
 549 static int submit_flush_wait(struct f2fs_sb_info *sbi, nid_t ino)
 550 {
 551         int ret = 0;
 552         int i;
 553
 554         if (!f2fs_is_multi_device(sbi))
 555                 return __submit_flush_wait(sbi, sbi->sb->s_bdev);
 556
 557         for (i = 0; i < sbi->s_ndevs; i++) {
 558                 if (!f2fs_is_dirty_device(sbi, ino, i, FLUSH_INO))
 559                         continue;
 560                 ret = __submit_flush_wait(sbi, FDEV(i).bdev);
 561                 if (ret)
 562                         break;
 563         }
 564         return ret;
 565 }
 566
 567 static int issue_flush_thread(void *data)
 568 {
 569         struct f2fs_sb_info *sbi = data;
 570         struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;
 571         wait_queue_head_t *q = &fcc->flush_wait_queue;
 572 repeat:
 573         if (kthread_should_stop())
 574                 return 0;
 575
 576         if (!llist_empty(&fcc->issue_list)) {
 577                 struct flush_cmd *cmd, *next;
 578                 int ret;
 579
 580                 fcc->dispatch_list = llist_del_all(&fcc->issue_list);
 581                 fcc->dispatch_list = llist_reverse_order(fcc->dispatch_list);
 582
 583                 cmd = llist_entry(fcc->dispatch_list, struct flush_cmd, llnode);
 584
 585                 ret = submit_flush_wait(sbi, cmd->ino);
 586                 atomic_inc(&fcc->issued_flush);
 587
 588                 llist_for_each_entry_safe(cmd, next,
 589                                           fcc->dispatch_list, llnode) {
 590                         cmd->ret = ret;
 591                         complete(&cmd->wait);
 592                 }
 593                 fcc->dispatch_list = NULL;
 594         }
 595
 596         wait_event_interruptible(*q,
 597                 kthread_should_stop() || !llist_empty(&fcc->issue_list));
 598         goto repeat;
 599 }
 600
 601 int f2fs_issue_flush(struct f2fs_sb_info *sbi, nid_t ino)
 602 {
 603         struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;
 604         struct flush_cmd cmd;
 605         int ret;
 606
 607         if (test_opt(sbi, NOBARRIER))
 608                 return 0;
 609
 610         if (!test_opt(sbi, FLUSH_MERGE)) {
 611                 atomic_inc(&fcc->queued_flush);
 612                 ret = submit_flush_wait(sbi, ino);
 613                 atomic_dec(&fcc->queued_flush);
 614                 atomic_inc(&fcc->issued_flush);
 615                 return ret;
 616         }
 617
 618         if (atomic_inc_return(&fcc->queued_flush) == 1 ||
 619             f2fs_is_multi_device(sbi)) {
 620                 ret = submit_flush_wait(sbi, ino);
 621                 atomic_dec(&fcc->queued_flush);
 622
 623                 atomic_inc(&fcc->issued_flush);
 624                 return ret;
 625         }
 626
 627         cmd.ino = ino;
 628         init_completion(&cmd.wait);
 629
 630         llist_add(&cmd.llnode, &fcc->issue_list);
 631
 632         /*
 633          * update issue_list before we wake up issue_flush thread, this
 634          * smp_mb() pairs with another barrier in ___wait_event(), see
 635          * more details in comments of waitqueue_active().
 636          */
 637         smp_mb();
 638
 639         if (waitqueue_active(&fcc->flush_wait_queue))
 640                 wake_up(&fcc->flush_wait_queue);
 641
 642         if (fcc->f2fs_issue_flush) {
 643                 wait_for_completion(&cmd.wait);
 644                 atomic_dec(&fcc->queued_flush);
 645         } else {
 646                 struct llist_node *list;
 647
 648                 list = llist_del_all(&fcc->issue_list);
 649                 if (!list) {
 650                         wait_for_completion(&cmd.wait);
 651                         atomic_dec(&fcc->queued_flush);
 652                 } else {
 653                         struct flush_cmd *tmp, *next;
 654
 655                         ret = submit_flush_wait(sbi, ino);
 656
 657                         llist_for_each_entry_safe(tmp, next, list, llnode) {
 658                                 if (tmp == &cmd) {
 659                                         cmd.ret = ret;
 660                                         atomic_dec(&fcc->queued_flush);
 661                                         continue;
 662                                 }
 663                                 tmp->ret = ret;
 664                                 complete(&tmp->wait);
 665                         }
 666                 }
 667         }
 668
 669         return cmd.ret;
 670 }
 671
 672 int f2fs_create_flush_cmd_control(struct f2fs_sb_info *sbi)
 673 {
 674         dev_t dev = sbi->sb->s_bdev->bd_dev;
 675         struct flush_cmd_control *fcc;
 676
 677         if (SM_I(sbi)->fcc_info) {
 678                 fcc = SM_I(sbi)->fcc_info;
 679                 if (fcc->f2fs_issue_flush)
 680                         return 0;
 681                 goto init_thread;
 682         }
 683
 684         fcc = f2fs_kzalloc(sbi, sizeof(struct flush_cmd_control), GFP_KERNEL);
 685         if (!fcc)
 686                 return -ENOMEM;
 687         atomic_set(&fcc->issued_flush, 0);
 688         atomic_set(&fcc->queued_flush, 0);
 689         init_waitqueue_head(&fcc->flush_wait_queue);
 690         init_llist_head(&fcc->issue_list);
 691         SM_I(sbi)->fcc_info = fcc;
 692         if (!test_opt(sbi, FLUSH_MERGE))
 693                 return 0;
 694
 695 init_thread:
 696         fcc->f2fs_issue_flush = kthread_run(issue_flush_thread, sbi,
 697                                 "f2fs_flush-%u:%u", MAJOR(dev), MINOR(dev));
 698         if (IS_ERR(fcc->f2fs_issue_flush)) {
 699                 int err = PTR_ERR(fcc->f2fs_issue_flush);
 700
 701                 fcc->f2fs_issue_flush = NULL;
 702                 return err;
 703         }
 704
 705         return 0;
 706 }
 707
 708 void f2fs_destroy_flush_cmd_control(struct f2fs_sb_info *sbi, bool free)
 709 {
 710         struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;
 711
 712         if (fcc && fcc->f2fs_issue_flush) {
 713                 struct task_struct *flush_thread = fcc->f2fs_issue_flush;
 714
 715                 fcc->f2fs_issue_flush = NULL;
 716                 kthread_stop(flush_thread);
 717         }
 718         if (free) {
 719                 kfree(fcc);
 720                 SM_I(sbi)->fcc_info = NULL;
 721         }
 722 }
 723
 724 int f2fs_flush_device_cache(struct f2fs_sb_info *sbi)
 725 {
 726         int ret = 0, i;
 727
 728         if (!f2fs_is_multi_device(sbi))
 729                 return 0;
 730
 731         if (test_opt(sbi, NOBARRIER))
 732                 return 0;
 733
 734         for (i = 1; i < sbi->s_ndevs; i++) {
 735                 int count = DEFAULT_RETRY_IO_COUNT;
 736
 737                 if (!f2fs_test_bit(i, (char *)&sbi->dirty_device))
 738                         continue;
 739
 740                 do {
 741                         ret = __submit_flush_wait(sbi, FDEV(i).bdev);
 742                         if (ret)
 743                                 f2fs_io_schedule_timeout(DEFAULT_IO_TIMEOUT);
 744                 } while (ret && --count);
 745
 746                 if (ret) {
 747                         f2fs_stop_checkpoint(sbi, false,
 748                                         STOP_CP_REASON_FLUSH_FAIL);
 749                         break;
 750                 }
 751
 752                 spin_lock(&sbi->dev_lock);
 753                 f2fs_clear_bit(i, (char *)&sbi->dirty_device);
 754                 spin_unlock(&sbi->dev_lock);
 755         }
 756
 757         return ret;
 758 }
 759
 760 static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
 761                 enum dirty_type dirty_type)
 762 {
 763         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
 764
 765         /* need not be added */
 766         if (IS_CURSEG(sbi, segno))
 767                 return;
 768
 769         if (!test_and_set_bit(segno, dirty_i->dirty_segmap[dirty_type]))
 770                 dirty_i->nr_dirty[dirty_type]++;
 771
 772         if (dirty_type == DIRTY) {
 773                 struct seg_entry *sentry = get_seg_entry(sbi, segno);
 774                 enum dirty_type t = sentry->type;
 775
 776                 if (unlikely(t >= DIRTY)) {
 777                         f2fs_bug_on(sbi, 1);
 778                         return;
 779                 }
 780                 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[t]))
 781                         dirty_i->nr_dirty[t]++;
 782
 783                 if (__is_large_section(sbi)) {
 784                         unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);
 785                         block_t valid_blocks =
 786                                 get_valid_blocks(sbi, segno, true);
 787
 788                         f2fs_bug_on(sbi,
 789                                 (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) &&
 790                                 !valid_blocks) ||
 791                                 valid_blocks == CAP_BLKS_PER_SEC(sbi));
 792
 793                         if (!IS_CURSEC(sbi, secno))
 794                                 set_bit(secno, dirty_i->dirty_secmap);
 795                 }
 796         }
 797 }
 798
 799 static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
 800                 enum dirty_type dirty_type)
 801 {
 802         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
 803         block_t valid_blocks;
 804
 805         if (test_and_clear_bit(segno, dirty_i->dirty_segmap[dirty_type]))
 806                 dirty_i->nr_dirty[dirty_type]--;
 807
 808         if (dirty_type == DIRTY) {
 809                 struct seg_entry *sentry = get_seg_entry(sbi, segno);
 810                 enum dirty_type t = sentry->type;
 811
 812                 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[t]))
 813                         dirty_i->nr_dirty[t]--;
 814
 815                 valid_blocks = get_valid_blocks(sbi, segno, true);
 816                 if (valid_blocks == 0) {
 817                         clear_bit(GET_SEC_FROM_SEG(sbi, segno),
 818                                                 dirty_i->victim_secmap);
 819 #ifdef CONFIG_F2FS_CHECK_FS
 820                         clear_bit(segno, SIT_I(sbi)->invalid_segmap);
 821 #endif
 822                 }
 823                 if (__is_large_section(sbi)) {
 824                         unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);
 825
 826                         if (!valid_blocks ||
 827                                         valid_blocks == CAP_BLKS_PER_SEC(sbi)) {
 828                                 clear_bit(secno, dirty_i->dirty_secmap);
 829                                 return;
 830                         }
 831
 832                         if (!IS_CURSEC(sbi, secno))
 833                                 set_bit(secno, dirty_i->dirty_secmap);
 834                 }
 835         }
 836 }
 837
 838 /*
 839  * Should not occur error such as -ENOMEM.
 840  * Adding dirty entry into seglist is not critical operation.
 841  * If a given segment is one of current working segments, it won't be added.
 842  */
 843 static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno)
 844 {
 845         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
 846         unsigned short valid_blocks, ckpt_valid_blocks;
 847         unsigned int usable_blocks;
 848
 849         if (segno == NULL_SEGNO || IS_CURSEG(sbi, segno))
 850                 return;
 851
 852         usable_blocks = f2fs_usable_blks_in_seg(sbi, segno);
 853         mutex_lock(&dirty_i->seglist_lock);
 854
 855         valid_blocks = get_valid_blocks(sbi, segno, false);
 856         ckpt_valid_blocks = get_ckpt_valid_blocks(sbi, segno, false);
 857
 858         if (valid_blocks == 0 && (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) ||
 859                 ckpt_valid_blocks == usable_blocks)) {
 860                 __locate_dirty_segment(sbi, segno, PRE);
 861                 __remove_dirty_segment(sbi, segno, DIRTY);
 862         } else if (valid_blocks < usable_blocks) {
 863                 __locate_dirty_segment(sbi, segno, DIRTY);
 864         } else {
 865                 /* Recovery routine with SSR needs this */
 866                 __remove_dirty_segment(sbi, segno, DIRTY);
 867         }
 868
 869         mutex_unlock(&dirty_i->seglist_lock);
 870 }
 871
 872 /* This moves currently empty dirty blocks to prefree. Must hold seglist_lock */
 873 void f2fs_dirty_to_prefree(struct f2fs_sb_info *sbi)
 874 {
 875         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
 876         unsigned int segno;
 877
 878         mutex_lock(&dirty_i->seglist_lock);
 879         for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
 880                 if (get_valid_blocks(sbi, segno, false))
 881                         continue;
 882                 if (IS_CURSEG(sbi, segno))
 883                         continue;
 884                 __locate_dirty_segment(sbi, segno, PRE);
 885                 __remove_dirty_segment(sbi, segno, DIRTY);
 886         }
 887         mutex_unlock(&dirty_i->seglist_lock);
 888 }
 889
 890 block_t f2fs_get_unusable_blocks(struct f2fs_sb_info *sbi)
 891 {
 892         int ovp_hole_segs =
 893                 (overprovision_segments(sbi) - reserved_segments(sbi));
 894         block_t ovp_holes = SEGS_TO_BLKS(sbi, ovp_hole_segs);
 895         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
 896         block_t holes[2] = {0, 0};      /* DATA and NODE */
 897         block_t unusable;
 898         struct seg_entry *se;
 899         unsigned int segno;
 900
 901         mutex_lock(&dirty_i->seglist_lock);
 902         for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
 903                 se = get_seg_entry(sbi, segno);
 904                 if (IS_NODESEG(se->type))
 905                         holes[NODE] += f2fs_usable_blks_in_seg(sbi, segno) -
 906                                                         se->valid_blocks;
 907                 else
 908                         holes[DATA] += f2fs_usable_blks_in_seg(sbi, segno) -
 909                                                         se->valid_blocks;
 910         }
 911         mutex_unlock(&dirty_i->seglist_lock);
 912
 913         unusable = max(holes[DATA], holes[NODE]);
 914         if (unusable > ovp_holes)
 915                 return unusable - ovp_holes;
 916         return 0;
 917 }
 918
 919 int f2fs_disable_cp_again(struct f2fs_sb_info *sbi, block_t unusable)
 920 {
 921         int ovp_hole_segs =
 922                 (overprovision_segments(sbi) - reserved_segments(sbi));
 923
 924         if (F2FS_OPTION(sbi).unusable_cap_perc == 100)
 925                 return 0;
 926         if (unusable > F2FS_OPTION(sbi).unusable_cap)
 927                 return -EAGAIN;
 928         if (is_sbi_flag_set(sbi, SBI_CP_DISABLED_QUICK) &&
 929                 dirty_segments(sbi) > ovp_hole_segs)
 930                 return -EAGAIN;
 931         if (has_not_enough_free_secs(sbi, 0, 0))
 932                 return -EAGAIN;
 933         return 0;
 934 }
 935
 936 /* This is only used by SBI_CP_DISABLED */
 937 static unsigned int get_free_segment(struct f2fs_sb_info *sbi)
 938 {
 939         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
 940         unsigned int segno = 0;
 941
 942         mutex_lock(&dirty_i->seglist_lock);
 943         for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
 944                 if (get_valid_blocks(sbi, segno, false))
 945                         continue;
 946                 if (get_ckpt_valid_blocks(sbi, segno, false))
 947                         continue;
 948                 mutex_unlock(&dirty_i->seglist_lock);
 949                 return segno;
 950         }
 951         mutex_unlock(&dirty_i->seglist_lock);
 952         return NULL_SEGNO;
 953 }
 954
 955 static struct discard_cmd *__create_discard_cmd(struct f2fs_sb_info *sbi,
 956                 struct block_device *bdev, block_t lstart,
 957                 block_t start, block_t len)
 958 {
 959         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
 960         struct list_head *pend_list;
 961         struct discard_cmd *dc;
 962
 963         f2fs_bug_on(sbi, !len);
 964
 965         pend_list = &dcc->pend_list[plist_idx(len)];
 966
 967         dc = f2fs_kmem_cache_alloc(discard_cmd_slab, GFP_NOFS, true, NULL);
 968         INIT_LIST_HEAD(&dc->list);
 969         dc->bdev = bdev;
 970         dc->di.lstart = lstart;
 971         dc->di.start = start;
 972         dc->di.len = len;
 973         dc->ref = 0;
 974         dc->state = D_PREP;
 975         dc->queued = 0;
 976         dc->error = 0;
 977         init_completion(&dc->wait);
 978         list_add_tail(&dc->list, pend_list);
 979         spin_lock_init(&dc->lock);
 980         dc->bio_ref = 0;
 981         atomic_inc(&dcc->discard_cmd_cnt);
 982         dcc->undiscard_blks += len;
 983
 984         return dc;
 985 }
 986
 987 static bool f2fs_check_discard_tree(struct f2fs_sb_info *sbi)
 988 {
 989 #ifdef CONFIG_F2FS_CHECK_FS
 990         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
 991         struct rb_node *cur = rb_first_cached(&dcc->root), *next;
 992         struct discard_cmd *cur_dc, *next_dc;
 993
 994         while (cur) {
 995                 next = rb_next(cur);
 996                 if (!next)
 997                         return true;
 998
 999                 cur_dc = rb_entry(cur, struct discard_cmd, rb_node);
1000                 next_dc = rb_entry(next, struct discard_cmd, rb_node);
1001
1002                 if (cur_dc->di.lstart + cur_dc->di.len > next_dc->di.lstart) {
1003                         f2fs_info(sbi, "broken discard_rbtree, "
1004                                 "cur(%u, %u) next(%u, %u)",
1005                                 cur_dc->di.lstart, cur_dc->di.len,
1006                                 next_dc->di.lstart, next_dc->di.len);
1007                         return false;
1008                 }
1009                 cur = next;
1010         }
1011 #endif
1012         return true;
1013 }
1014
1015 static struct discard_cmd *__lookup_discard_cmd(struct f2fs_sb_info *sbi,
1016                                                 block_t blkaddr)
1017 {
1018         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1019         struct rb_node *node = dcc->root.rb_root.rb_node;
1020         struct discard_cmd *dc;
1021
1022         while (node) {
1023                 dc = rb_entry(node, struct discard_cmd, rb_node);
1024
1025                 if (blkaddr < dc->di.lstart)
1026                         node = node->rb_left;
1027                 else if (blkaddr >= dc->di.lstart + dc->di.len)
1028                         node = node->rb_right;
1029                 else
1030                         return dc;
1031         }
1032         return NULL;
1033 }
1034
1035 static struct discard_cmd *__lookup_discard_cmd_ret(struct rb_root_cached *root,
1036                                 block_t blkaddr,
1037                                 struct discard_cmd **prev_entry,
1038                                 struct discard_cmd **next_entry,
1039                                 struct rb_node ***insert_p,
1040                                 struct rb_node **insert_parent)
1041 {
1042         struct rb_node **pnode = &root->rb_root.rb_node;
1043         struct rb_node *parent = NULL, *tmp_node;
1044         struct discard_cmd *dc;
1045
1046         *insert_p = NULL;
1047         *insert_parent = NULL;
1048         *prev_entry = NULL;
1049         *next_entry = NULL;
1050
1051         if (RB_EMPTY_ROOT(&root->rb_root))
1052                 return NULL;
1053
1054         while (*pnode) {
1055                 parent = *pnode;
1056                 dc = rb_entry(*pnode, struct discard_cmd, rb_node);
1057
1058                 if (blkaddr < dc->di.lstart)
1059                         pnode = &(*pnode)->rb_left;
1060                 else if (blkaddr >= dc->di.lstart + dc->di.len)
1061                         pnode = &(*pnode)->rb_right;
1062                 else
1063                         goto lookup_neighbors;
1064         }
1065
1066         *insert_p = pnode;
1067         *insert_parent = parent;
1068
1069         dc = rb_entry(parent, struct discard_cmd, rb_node);
1070         tmp_node = parent;
1071         if (parent && blkaddr > dc->di.lstart)
1072                 tmp_node = rb_next(parent);
1073         *next_entry = rb_entry_safe(tmp_node, struct discard_cmd, rb_node);
1074
1075         tmp_node = parent;
1076         if (parent && blkaddr < dc->di.lstart)
1077                 tmp_node = rb_prev(parent);
1078         *prev_entry = rb_entry_safe(tmp_node, struct discard_cmd, rb_node);
1079         return NULL;
1080
1081 lookup_neighbors:
1082         /* lookup prev node for merging backward later */
1083         tmp_node = rb_prev(&dc->rb_node);
1084         *prev_entry = rb_entry_safe(tmp_node, struct discard_cmd, rb_node);
1085
1086         /* lookup next node for merging frontward later */
1087         tmp_node = rb_next(&dc->rb_node);
1088         *next_entry = rb_entry_safe(tmp_node, struct discard_cmd, rb_node);
1089         return dc;
1090 }
1091
1092 static void __detach_discard_cmd(struct discard_cmd_control *dcc,
1093                                                         struct discard_cmd *dc)
1094 {
1095         if (dc->state == D_DONE)
1096                 atomic_sub(dc->queued, &dcc->queued_discard);
1097
1098         list_del(&dc->list);
1099         rb_erase_cached(&dc->rb_node, &dcc->root);
1100         dcc->undiscard_blks -= dc->di.len;
1101
1102         kmem_cache_free(discard_cmd_slab, dc);
1103
1104         atomic_dec(&dcc->discard_cmd_cnt);
1105 }
1106
1107 static void __remove_discard_cmd(struct f2fs_sb_info *sbi,
1108                                                         struct discard_cmd *dc)
1109 {
1110         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1111         unsigned long flags;
1112
1113         trace_f2fs_remove_discard(dc->bdev, dc->di.start, dc->di.len);
1114
1115         spin_lock_irqsave(&dc->lock, flags);
1116         if (dc->bio_ref) {
1117                 spin_unlock_irqrestore(&dc->lock, flags);
1118                 return;
1119         }
1120         spin_unlock_irqrestore(&dc->lock, flags);
1121
1122         f2fs_bug_on(sbi, dc->ref);
1123
1124         if (dc->error == -EOPNOTSUPP)
1125                 dc->error = 0;
1126
1127         if (dc->error)
1128                 f2fs_info_ratelimited(sbi,
1129                         "Issue discard(%u, %u, %u) failed, ret: %d",
1130                         dc->di.lstart, dc->di.start, dc->di.len, dc->error);
1131         __detach_discard_cmd(dcc, dc);
1132 }
1133
1134 static void f2fs_submit_discard_endio(struct bio *bio)
1135 {
1136         struct discard_cmd *dc = (struct discard_cmd *)bio->bi_private;
1137         unsigned long flags;
1138
1139         spin_lock_irqsave(&dc->lock, flags);
1140         if (!dc->error)
1141                 dc->error = blk_status_to_errno(bio->bi_status);
1142         dc->bio_ref--;
1143         if (!dc->bio_ref && dc->state == D_SUBMIT) {
1144                 dc->state = D_DONE;
1145                 complete_all(&dc->wait);
1146         }
1147         spin_unlock_irqrestore(&dc->lock, flags);
1148         bio_put(bio);
1149 }
1150
1151 static void __check_sit_bitmap(struct f2fs_sb_info *sbi,
1152                                 block_t start, block_t end)
1153 {
1154 #ifdef CONFIG_F2FS_CHECK_FS
1155         struct seg_entry *sentry;
1156         unsigned int segno;
1157         block_t blk = start;
1158         unsigned long offset, size, *map;
1159
1160         while (blk < end) {
1161                 segno = GET_SEGNO(sbi, blk);
1162                 sentry = get_seg_entry(sbi, segno);
1163                 offset = GET_BLKOFF_FROM_SEG0(sbi, blk);
1164
1165                 if (end < START_BLOCK(sbi, segno + 1))
1166                         size = GET_BLKOFF_FROM_SEG0(sbi, end);
1167                 else
1168                         size = BLKS_PER_SEG(sbi);
1169                 map = (unsigned long *)(sentry->cur_valid_map);
1170                 offset = __find_rev_next_bit(map, size, offset);
1171                 f2fs_bug_on(sbi, offset != size);
1172                 blk = START_BLOCK(sbi, segno + 1);
1173         }
1174 #endif
1175 }
1176
1177 static void __init_discard_policy(struct f2fs_sb_info *sbi,
1178                                 struct discard_policy *dpolicy,
1179                                 int discard_type, unsigned int granularity)
1180 {
1181         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1182
1183         /* common policy */
1184         dpolicy->type = discard_type;
1185         dpolicy->sync = true;
1186         dpolicy->ordered = false;
1187         dpolicy->granularity = granularity;
1188
1189         dpolicy->max_requests = dcc->max_discard_request;
1190         dpolicy->io_aware_gran = dcc->discard_io_aware_gran;
1191         dpolicy->timeout = false;
1192
1193         if (discard_type == DPOLICY_BG) {
1194                 dpolicy->min_interval = dcc->min_discard_issue_time;
1195                 dpolicy->mid_interval = dcc->mid_discard_issue_time;
1196                 dpolicy->max_interval = dcc->max_discard_issue_time;
1197                 if (dcc->discard_io_aware == DPOLICY_IO_AWARE_ENABLE)
1198                         dpolicy->io_aware = true;
1199                 else if (dcc->discard_io_aware == DPOLICY_IO_AWARE_DISABLE)
1200                         dpolicy->io_aware = false;
1201                 dpolicy->sync = false;
1202                 dpolicy->ordered = true;
1203                 if (utilization(sbi) > dcc->discard_urgent_util) {
1204                         dpolicy->granularity = MIN_DISCARD_GRANULARITY;
1205                         if (atomic_read(&dcc->discard_cmd_cnt))
1206                                 dpolicy->max_interval =
1207                                         dcc->min_discard_issue_time;
1208                 }
1209         } else if (discard_type == DPOLICY_FORCE) {
1210                 dpolicy->min_interval = dcc->min_discard_issue_time;
1211                 dpolicy->mid_interval = dcc->mid_discard_issue_time;
1212                 dpolicy->max_interval = dcc->max_discard_issue_time;
1213                 dpolicy->io_aware = false;
1214         } else if (discard_type == DPOLICY_FSTRIM) {
1215                 dpolicy->io_aware = false;
1216         } else if (discard_type == DPOLICY_UMOUNT) {
1217                 dpolicy->io_aware = false;
1218                 /* we need to issue all to keep CP_TRIMMED_FLAG */
1219                 dpolicy->granularity = MIN_DISCARD_GRANULARITY;
1220                 dpolicy->timeout = true;
1221         }
1222 }
1223
1224 static void __update_discard_tree_range(struct f2fs_sb_info *sbi,
1225                                 struct block_device *bdev, block_t lstart,
1226                                 block_t start, block_t len);
1227
1228 #ifdef CONFIG_BLK_DEV_ZONED
1229 static void __submit_zone_reset_cmd(struct f2fs_sb_info *sbi,
1230                                    struct discard_cmd *dc, blk_opf_t flag,
1231                                    struct list_head *wait_list,
1232                                    unsigned int *issued)
1233 {
1234         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1235         struct block_device *bdev = dc->bdev;
1236         struct bio *bio = bio_alloc(bdev, 0, REQ_OP_ZONE_RESET | flag, GFP_NOFS);
1237         unsigned long flags;
1238
1239         trace_f2fs_issue_reset_zone(bdev, dc->di.start);
1240
1241         spin_lock_irqsave(&dc->lock, flags);
1242         dc->state = D_SUBMIT;
1243         dc->bio_ref++;
1244         spin_unlock_irqrestore(&dc->lock, flags);
1245
1246         if (issued)
1247                 (*issued)++;
1248
1249         atomic_inc(&dcc->queued_discard);
1250         dc->queued++;
1251         list_move_tail(&dc->list, wait_list);
1252
1253         /* sanity check on discard range */
1254         __check_sit_bitmap(sbi, dc->di.lstart, dc->di.lstart + dc->di.len);
1255
1256         bio->bi_iter.bi_sector = SECTOR_FROM_BLOCK(dc->di.start);
1257         bio->bi_private = dc;
1258         bio->bi_end_io = f2fs_submit_discard_endio;
1259         submit_bio(bio);
1260
1261         atomic_inc(&dcc->issued_discard);
1262         f2fs_update_iostat(sbi, NULL, FS_ZONE_RESET_IO, dc->di.len * F2FS_BLKSIZE);
1263 }
1264 #endif
1265
1266 /* this function is copied from blkdev_issue_discard from block/blk-lib.c */
1267 static int __submit_discard_cmd(struct f2fs_sb_info *sbi,
1268                                 struct discard_policy *dpolicy,
1269                                 struct discard_cmd *dc, int *issued)
1270 {
1271         struct block_device *bdev = dc->bdev;
1272         unsigned int max_discard_blocks =
1273                         SECTOR_TO_BLOCK(bdev_max_discard_sectors(bdev));
1274         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1275         struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ?
1276                                         &(dcc->fstrim_list) : &(dcc->wait_list);
1277         blk_opf_t flag = dpolicy->sync ? REQ_SYNC : 0;
1278         block_t lstart, start, len, total_len;
1279         int err = 0;
1280
1281         if (dc->state != D_PREP)
1282                 return 0;
1283
1284         if (is_sbi_flag_set(sbi, SBI_NEED_FSCK))
1285                 return 0;
1286
1287 #ifdef CONFIG_BLK_DEV_ZONED
1288         if (f2fs_sb_has_blkzoned(sbi) && bdev_is_zoned(bdev)) {
1289                 int devi = f2fs_bdev_index(sbi, bdev);
1290
1291                 if (devi < 0)
1292                         return -EINVAL;
1293
1294                 if (f2fs_blkz_is_seq(sbi, devi, dc->di.start)) {
1295                         __submit_zone_reset_cmd(sbi, dc, flag,
1296                                                 wait_list, issued);
1297                         return 0;
1298                 }
1299         }
1300 #endif
1301
1302         /*
1303          * stop issuing discard for any of below cases:
1304          * 1. device is conventional zone, but it doesn't support discard.
1305          * 2. device is regulare device, after snapshot it doesn't support
1306          * discard.
1307          */
1308         if (!bdev_max_discard_sectors(bdev))
1309                 return -EOPNOTSUPP;
1310
1311         trace_f2fs_issue_discard(bdev, dc->di.start, dc->di.len);
1312
1313         lstart = dc->di.lstart;
1314         start = dc->di.start;
1315         len = dc->di.len;
1316         total_len = len;
1317
1318         dc->di.len = 0;
1319
1320         while (total_len && *issued < dpolicy->max_requests && !err) {
1321                 struct bio *bio = NULL;
1322                 unsigned long flags;
1323                 bool last = true;
1324
1325                 if (len > max_discard_blocks) {
1326                         len = max_discard_blocks;
1327                         last = false;
1328                 }
1329
1330                 (*issued)++;
1331                 if (*issued == dpolicy->max_requests)
1332                         last = true;
1333
1334                 dc->di.len += len;
1335
1336                 if (time_to_inject(sbi, FAULT_DISCARD)) {
1337                         err = -EIO;
1338                 } else {
1339                         err = __blkdev_issue_discard(bdev,
1340                                         SECTOR_FROM_BLOCK(start),
1341                                         SECTOR_FROM_BLOCK(len),
1342                                         GFP_NOFS, &bio);
1343                 }
1344                 if (err) {
1345                         spin_lock_irqsave(&dc->lock, flags);
1346                         if (dc->state == D_PARTIAL)
1347                                 dc->state = D_SUBMIT;
1348                         spin_unlock_irqrestore(&dc->lock, flags);
1349
1350                         break;
1351                 }
1352
1353                 f2fs_bug_on(sbi, !bio);
1354
1355                 /*
1356                  * should keep before submission to avoid D_DONE
1357                  * right away
1358                  */
1359                 spin_lock_irqsave(&dc->lock, flags);
1360                 if (last)
1361                         dc->state = D_SUBMIT;
1362                 else
1363                         dc->state = D_PARTIAL;
1364                 dc->bio_ref++;
1365                 spin_unlock_irqrestore(&dc->lock, flags);
1366
1367                 atomic_inc(&dcc->queued_discard);
1368                 dc->queued++;
1369                 list_move_tail(&dc->list, wait_list);
1370
1371                 /* sanity check on discard range */
1372                 __check_sit_bitmap(sbi, lstart, lstart + len);
1373
1374                 bio->bi_private = dc;
1375                 bio->bi_end_io = f2fs_submit_discard_endio;
1376                 bio->bi_opf |= flag;
1377                 submit_bio(bio);
1378
1379                 atomic_inc(&dcc->issued_discard);
1380
1381                 f2fs_update_iostat(sbi, NULL, FS_DISCARD_IO, len * F2FS_BLKSIZE);
1382
1383                 lstart += len;
1384                 start += len;
1385                 total_len -= len;
1386                 len = total_len;
1387         }
1388
1389         if (!err && len) {
1390                 dcc->undiscard_blks -= len;
1391                 __update_discard_tree_range(sbi, bdev, lstart, start, len);
1392         }
1393         return err;
1394 }
1395
1396 static void __insert_discard_cmd(struct f2fs_sb_info *sbi,
1397                                 struct block_device *bdev, block_t lstart,
1398                                 block_t start, block_t len)
1399 {
1400         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1401         struct rb_node **p = &dcc->root.rb_root.rb_node;
1402         struct rb_node *parent = NULL;
1403         struct discard_cmd *dc;
1404         bool leftmost = true;
1405
1406         /* look up rb tree to find parent node */
1407         while (*p) {
1408                 parent = *p;
1409                 dc = rb_entry(parent, struct discard_cmd, rb_node);
1410
1411                 if (lstart < dc->di.lstart) {
1412                         p = &(*p)->rb_left;
1413                 } else if (lstart >= dc->di.lstart + dc->di.len) {
1414                         p = &(*p)->rb_right;
1415                         leftmost = false;
1416                 } else {
1417                         /* Let's skip to add, if exists */
1418                         return;
1419                 }
1420         }
1421
1422         dc = __create_discard_cmd(sbi, bdev, lstart, start, len);
1423
1424         rb_link_node(&dc->rb_node, parent, p);
1425         rb_insert_color_cached(&dc->rb_node, &dcc->root, leftmost);
1426 }
1427
1428 static void __relocate_discard_cmd(struct discard_cmd_control *dcc,
1429                                                 struct discard_cmd *dc)
1430 {
1431         list_move_tail(&dc->list, &dcc->pend_list[plist_idx(dc->di.len)]);
1432 }
1433
1434 static void __punch_discard_cmd(struct f2fs_sb_info *sbi,
1435                                 struct discard_cmd *dc, block_t blkaddr)
1436 {
1437         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1438         struct discard_info di = dc->di;
1439         bool modified = false;
1440
1441         if (dc->state == D_DONE || dc->di.len == 1) {
1442                 __remove_discard_cmd(sbi, dc);
1443                 return;
1444         }
1445
1446         dcc->undiscard_blks -= di.len;
1447
1448         if (blkaddr > di.lstart) {
1449                 dc->di.len = blkaddr - dc->di.lstart;
1450                 dcc->undiscard_blks += dc->di.len;
1451                 __relocate_discard_cmd(dcc, dc);
1452                 modified = true;
1453         }
1454
1455         if (blkaddr < di.lstart + di.len - 1) {
1456                 if (modified) {
1457                         __insert_discard_cmd(sbi, dc->bdev, blkaddr + 1,
1458                                         di.start + blkaddr + 1 - di.lstart,
1459                                         di.lstart + di.len - 1 - blkaddr);
1460                 } else {
1461                         dc->di.lstart++;
1462                         dc->di.len--;
1463                         dc->di.start++;
1464                         dcc->undiscard_blks += dc->di.len;
1465                         __relocate_discard_cmd(dcc, dc);
1466                 }
1467         }
1468 }
1469
1470 static void __update_discard_tree_range(struct f2fs_sb_info *sbi,
1471                                 struct block_device *bdev, block_t lstart,
1472                                 block_t start, block_t len)
1473 {
1474         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1475         struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
1476         struct discard_cmd *dc;
1477         struct discard_info di = {0};
1478         struct rb_node **insert_p = NULL, *insert_parent = NULL;
1479         unsigned int max_discard_blocks =
1480                         SECTOR_TO_BLOCK(bdev_max_discard_sectors(bdev));
1481         block_t end = lstart + len;
1482
1483         dc = __lookup_discard_cmd_ret(&dcc->root, lstart,
1484                                 &prev_dc, &next_dc, &insert_p, &insert_parent);
1485         if (dc)
1486                 prev_dc = dc;
1487
1488         if (!prev_dc) {
1489                 di.lstart = lstart;
1490                 di.len = next_dc ? next_dc->di.lstart - lstart : len;
1491                 di.len = min(di.len, len);
1492                 di.start = start;
1493         }
1494
1495         while (1) {
1496                 struct rb_node *node;
1497                 bool merged = false;
1498                 struct discard_cmd *tdc = NULL;
1499
1500                 if (prev_dc) {
1501                         di.lstart = prev_dc->di.lstart + prev_dc->di.len;
1502                         if (di.lstart < lstart)
1503                                 di.lstart = lstart;
1504                         if (di.lstart >= end)
1505                                 break;
1506
1507                         if (!next_dc || next_dc->di.lstart > end)
1508                                 di.len = end - di.lstart;
1509                         else
1510                                 di.len = next_dc->di.lstart - di.lstart;
1511                         di.start = start + di.lstart - lstart;
1512                 }
1513
1514                 if (!di.len)
1515                         goto next;
1516
1517                 if (prev_dc && prev_dc->state == D_PREP &&
1518                         prev_dc->bdev == bdev &&
1519                         __is_discard_back_mergeable(&di, &prev_dc->di,
1520                                                         max_discard_blocks)) {
1521                         prev_dc->di.len += di.len;
1522                         dcc->undiscard_blks += di.len;
1523                         __relocate_discard_cmd(dcc, prev_dc);
1524                         di = prev_dc->di;
1525                         tdc = prev_dc;
1526                         merged = true;
1527                 }
1528
1529                 if (next_dc && next_dc->state == D_PREP &&
1530                         next_dc->bdev == bdev &&
1531                         __is_discard_front_mergeable(&di, &next_dc->di,
1532                                                         max_discard_blocks)) {
1533                         next_dc->di.lstart = di.lstart;
1534                         next_dc->di.len += di.len;
1535                         next_dc->di.start = di.start;
1536                         dcc->undiscard_blks += di.len;
1537                         __relocate_discard_cmd(dcc, next_dc);
1538                         if (tdc)
1539                                 __remove_discard_cmd(sbi, tdc);
1540                         merged = true;
1541                 }
1542
1543                 if (!merged)
1544                         __insert_discard_cmd(sbi, bdev,
1545                                                 di.lstart, di.start, di.len);
1546  next:
1547                 prev_dc = next_dc;
1548                 if (!prev_dc)
1549                         break;
1550
1551                 node = rb_next(&prev_dc->rb_node);
1552                 next_dc = rb_entry_safe(node, struct discard_cmd, rb_node);
1553         }
1554 }
1555
1556 #ifdef CONFIG_BLK_DEV_ZONED
1557 static void __queue_zone_reset_cmd(struct f2fs_sb_info *sbi,
1558                 struct block_device *bdev, block_t blkstart, block_t lblkstart,
1559                 block_t blklen)
1560 {
1561         trace_f2fs_queue_reset_zone(bdev, blkstart);
1562
1563         mutex_lock(&SM_I(sbi)->dcc_info->cmd_lock);
1564         __insert_discard_cmd(sbi, bdev, lblkstart, blkstart, blklen);
1565         mutex_unlock(&SM_I(sbi)->dcc_info->cmd_lock);
1566 }
1567 #endif
1568
1569 static void __queue_discard_cmd(struct f2fs_sb_info *sbi,
1570                 struct block_device *bdev, block_t blkstart, block_t blklen)
1571 {
1572         block_t lblkstart = blkstart;
1573
1574         if (!f2fs_bdev_support_discard(bdev))
1575                 return;
1576
1577         trace_f2fs_queue_discard(bdev, blkstart, blklen);
1578
1579         if (f2fs_is_multi_device(sbi)) {
1580                 int devi = f2fs_target_device_index(sbi, blkstart);
1581
1582                 blkstart -= FDEV(devi).start_blk;
1583         }
1584         mutex_lock(&SM_I(sbi)->dcc_info->cmd_lock);
1585         __update_discard_tree_range(sbi, bdev, lblkstart, blkstart, blklen);
1586         mutex_unlock(&SM_I(sbi)->dcc_info->cmd_lock);
1587 }
1588
1589 static void __issue_discard_cmd_orderly(struct f2fs_sb_info *sbi,
1590                 struct discard_policy *dpolicy, int *issued)
1591 {
1592         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1593         struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
1594         struct rb_node **insert_p = NULL, *insert_parent = NULL;
1595         struct discard_cmd *dc;
1596         struct blk_plug plug;
1597         bool io_interrupted = false;
1598
1599         mutex_lock(&dcc->cmd_lock);
1600         dc = __lookup_discard_cmd_ret(&dcc->root, dcc->next_pos,
1601                                 &prev_dc, &next_dc, &insert_p, &insert_parent);
1602         if (!dc)
1603                 dc = next_dc;
1604
1605         blk_start_plug(&plug);
1606
1607         while (dc) {
1608                 struct rb_node *node;
1609                 int err = 0;
1610
1611                 if (dc->state != D_PREP)
1612                         goto next;
1613
1614                 if (dpolicy->io_aware && !is_idle(sbi, DISCARD_TIME)) {
1615                         io_interrupted = true;
1616                         break;
1617                 }
1618
1619                 dcc->next_pos = dc->di.lstart + dc->di.len;
1620                 err = __submit_discard_cmd(sbi, dpolicy, dc, issued);
1621
1622                 if (*issued >= dpolicy->max_requests)
1623                         break;
1624 next:
1625                 node = rb_next(&dc->rb_node);
1626                 if (err)
1627                         __remove_discard_cmd(sbi, dc);
1628                 dc = rb_entry_safe(node, struct discard_cmd, rb_node);
1629         }
1630
1631         blk_finish_plug(&plug);
1632
1633         if (!dc)
1634                 dcc->next_pos = 0;
1635
1636         mutex_unlock(&dcc->cmd_lock);
1637
1638         if (!(*issued) && io_interrupted)
1639                 *issued = -1;
1640 }
1641 static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi,
1642                                         struct discard_policy *dpolicy);
1643
1644 static int __issue_discard_cmd(struct f2fs_sb_info *sbi,
1645                                         struct discard_policy *dpolicy)
1646 {
1647         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1648         struct list_head *pend_list;
1649         struct discard_cmd *dc, *tmp;
1650         struct blk_plug plug;
1651         int i, issued;
1652         bool io_interrupted = false;
1653
1654         if (dpolicy->timeout)
1655                 f2fs_update_time(sbi, UMOUNT_DISCARD_TIMEOUT);
1656
1657 retry:
1658         issued = 0;
1659         for (i = MAX_PLIST_NUM - 1; i >= 0; i--) {
1660                 if (dpolicy->timeout &&
1661                                 f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT))
1662                         break;
1663
1664                 if (i + 1 < dpolicy->granularity)
1665                         break;
1666
1667                 if (i + 1 < dcc->max_ordered_discard && dpolicy->ordered) {
1668                         __issue_discard_cmd_orderly(sbi, dpolicy, &issued);
1669                         return issued;
1670                 }
1671
1672                 pend_list = &dcc->pend_list[i];
1673
1674                 mutex_lock(&dcc->cmd_lock);
1675                 if (list_empty(pend_list))
1676                         goto next;
1677                 if (unlikely(dcc->rbtree_check))
1678                         f2fs_bug_on(sbi, !f2fs_check_discard_tree(sbi));
1679                 blk_start_plug(&plug);
1680                 list_for_each_entry_safe(dc, tmp, pend_list, list) {
1681                         f2fs_bug_on(sbi, dc->state != D_PREP);
1682
1683                         if (dpolicy->timeout &&
1684                                 f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT))
1685                                 break;
1686
1687                         if (dpolicy->io_aware && i < dpolicy->io_aware_gran &&
1688                                                 !is_idle(sbi, DISCARD_TIME)) {
1689                                 io_interrupted = true;
1690                                 break;
1691                         }
1692
1693                         __submit_discard_cmd(sbi, dpolicy, dc, &issued);
1694
1695                         if (issued >= dpolicy->max_requests)
1696                                 break;
1697                 }
1698                 blk_finish_plug(&plug);
1699 next:
1700                 mutex_unlock(&dcc->cmd_lock);
1701
1702                 if (issued >= dpolicy->max_requests || io_interrupted)
1703                         break;
1704         }
1705
1706         if (dpolicy->type == DPOLICY_UMOUNT && issued) {
1707                 __wait_all_discard_cmd(sbi, dpolicy);
1708                 goto retry;
1709         }
1710
1711         if (!issued && io_interrupted)
1712                 issued = -1;
1713
1714         return issued;
1715 }
1716
1717 static bool __drop_discard_cmd(struct f2fs_sb_info *sbi)
1718 {
1719         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1720         struct list_head *pend_list;
1721         struct discard_cmd *dc, *tmp;
1722         int i;
1723         bool dropped = false;
1724
1725         mutex_lock(&dcc->cmd_lock);
1726         for (i = MAX_PLIST_NUM - 1; i >= 0; i--) {
1727                 pend_list = &dcc->pend_list[i];
1728                 list_for_each_entry_safe(dc, tmp, pend_list, list) {
1729                         f2fs_bug_on(sbi, dc->state != D_PREP);
1730                         __remove_discard_cmd(sbi, dc);
1731                         dropped = true;
1732                 }
1733         }
1734         mutex_unlock(&dcc->cmd_lock);
1735
1736         return dropped;
1737 }
1738
1739 void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi)
1740 {
1741         __drop_discard_cmd(sbi);
1742 }
1743
1744 static unsigned int __wait_one_discard_bio(struct f2fs_sb_info *sbi,
1745                                                         struct discard_cmd *dc)
1746 {
1747         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1748         unsigned int len = 0;
1749
1750         wait_for_completion_io(&dc->wait);
1751         mutex_lock(&dcc->cmd_lock);
1752         f2fs_bug_on(sbi, dc->state != D_DONE);
1753         dc->ref--;
1754         if (!dc->ref) {
1755                 if (!dc->error)
1756                         len = dc->di.len;
1757                 __remove_discard_cmd(sbi, dc);
1758         }
1759         mutex_unlock(&dcc->cmd_lock);
1760
1761         return len;
1762 }
1763
1764 static unsigned int __wait_discard_cmd_range(struct f2fs_sb_info *sbi,
1765                                                 struct discard_policy *dpolicy,
1766                                                 block_t start, block_t end)
1767 {
1768         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1769         struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ?
1770                                         &(dcc->fstrim_list) : &(dcc->wait_list);
1771         struct discard_cmd *dc = NULL, *iter, *tmp;
1772         unsigned int trimmed = 0;
1773
1774 next:
1775         dc = NULL;
1776
1777         mutex_lock(&dcc->cmd_lock);
1778         list_for_each_entry_safe(iter, tmp, wait_list, list) {
1779                 if (iter->di.lstart + iter->di.len <= start ||
1780                                         end <= iter->di.lstart)
1781                         continue;
1782                 if (iter->di.len < dpolicy->granularity)
1783                         continue;
1784                 if (iter->state == D_DONE && !iter->ref) {
1785                         wait_for_completion_io(&iter->wait);
1786                         if (!iter->error)
1787                                 trimmed += iter->di.len;
1788                         __remove_discard_cmd(sbi, iter);
1789                 } else {
1790                         iter->ref++;
1791                         dc = iter;
1792                         break;
1793                 }
1794         }
1795         mutex_unlock(&dcc->cmd_lock);
1796
1797         if (dc) {
1798                 trimmed += __wait_one_discard_bio(sbi, dc);
1799                 goto next;
1800         }
1801
1802         return trimmed;
1803 }
1804
1805 static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi,
1806                                                 struct discard_policy *dpolicy)
1807 {
1808         struct discard_policy dp;
1809         unsigned int discard_blks;
1810
1811         if (dpolicy)
1812                 return __wait_discard_cmd_range(sbi, dpolicy, 0, UINT_MAX);
1813
1814         /* wait all */
1815         __init_discard_policy(sbi, &dp, DPOLICY_FSTRIM, MIN_DISCARD_GRANULARITY);
1816         discard_blks = __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX);
1817         __init_discard_policy(sbi, &dp, DPOLICY_UMOUNT, MIN_DISCARD_GRANULARITY);
1818         discard_blks += __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX);
1819
1820         return discard_blks;
1821 }
1822
1823 /* This should be covered by global mutex, &sit_i->sentry_lock */
1824 static void f2fs_wait_discard_bio(struct f2fs_sb_info *sbi, block_t blkaddr)
1825 {
1826         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1827         struct discard_cmd *dc;
1828         bool need_wait = false;
1829
1830         mutex_lock(&dcc->cmd_lock);
1831         dc = __lookup_discard_cmd(sbi, blkaddr);
1832 #ifdef CONFIG_BLK_DEV_ZONED
1833         if (dc && f2fs_sb_has_blkzoned(sbi) && bdev_is_zoned(dc->bdev)) {
1834                 int devi = f2fs_bdev_index(sbi, dc->bdev);
1835
1836                 if (devi < 0) {
1837                         mutex_unlock(&dcc->cmd_lock);
1838                         return;
1839                 }
1840
1841                 if (f2fs_blkz_is_seq(sbi, devi, dc->di.start)) {
1842                         /* force submit zone reset */
1843                         if (dc->state == D_PREP)
1844                                 __submit_zone_reset_cmd(sbi, dc, REQ_SYNC,
1845                                                         &dcc->wait_list, NULL);
1846                         dc->ref++;
1847                         mutex_unlock(&dcc->cmd_lock);
1848                         /* wait zone reset */
1849                         __wait_one_discard_bio(sbi, dc);
1850                         return;
1851                 }
1852         }
1853 #endif
1854         if (dc) {
1855                 if (dc->state == D_PREP) {
1856                         __punch_discard_cmd(sbi, dc, blkaddr);
1857                 } else {
1858                         dc->ref++;
1859                         need_wait = true;
1860                 }
1861         }
1862         mutex_unlock(&dcc->cmd_lock);
1863
1864         if (need_wait)
1865                 __wait_one_discard_bio(sbi, dc);
1866 }
1867
1868 void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi)
1869 {
1870         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1871
1872         if (dcc && dcc->f2fs_issue_discard) {
1873                 struct task_struct *discard_thread = dcc->f2fs_issue_discard;
1874
1875                 dcc->f2fs_issue_discard = NULL;
1876                 kthread_stop(discard_thread);
1877         }
1878 }
1879
1880 /**
1881  * f2fs_issue_discard_timeout() - Issue all discard cmd within UMOUNT_DISCARD_TIMEOUT
1882  * @sbi: the f2fs_sb_info data for discard cmd to issue
1883  *
1884  * When UMOUNT_DISCARD_TIMEOUT is exceeded, all remaining discard commands will be dropped
1885  *
1886  * Return true if issued all discard cmd or no discard cmd need issue, otherwise return false.
1887  */
1888 bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi)
1889 {
1890         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1891         struct discard_policy dpolicy;
1892         bool dropped;
1893
1894         if (!atomic_read(&dcc->discard_cmd_cnt))
1895                 return true;
1896
1897         __init_discard_policy(sbi, &dpolicy, DPOLICY_UMOUNT,
1898                                         dcc->discard_granularity);
1899         __issue_discard_cmd(sbi, &dpolicy);
1900         dropped = __drop_discard_cmd(sbi);
1901
1902         /* just to make sure there is no pending discard commands */
1903         __wait_all_discard_cmd(sbi, NULL);
1904
1905         f2fs_bug_on(sbi, atomic_read(&dcc->discard_cmd_cnt));
1906         return !dropped;
1907 }
1908
1909 static int issue_discard_thread(void *data)
1910 {
1911         struct f2fs_sb_info *sbi = data;
1912         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1913         wait_queue_head_t *q = &dcc->discard_wait_queue;
1914         struct discard_policy dpolicy;
1915         unsigned int wait_ms = dcc->min_discard_issue_time;
1916         int issued;
1917
1918         set_freezable();
1919
1920         do {
1921                 wait_event_freezable_timeout(*q,
1922                                 kthread_should_stop() || dcc->discard_wake,
1923                                 msecs_to_jiffies(wait_ms));
1924
1925                 if (sbi->gc_mode == GC_URGENT_HIGH ||
1926                         !f2fs_available_free_memory(sbi, DISCARD_CACHE))
1927                         __init_discard_policy(sbi, &dpolicy, DPOLICY_FORCE,
1928                                                 MIN_DISCARD_GRANULARITY);
1929                 else
1930                         __init_discard_policy(sbi, &dpolicy, DPOLICY_BG,
1931                                                 dcc->discard_granularity);
1932
1933                 if (dcc->discard_wake)
1934                         dcc->discard_wake = false;
1935
1936                 /* clean up pending candidates before going to sleep */
1937                 if (atomic_read(&dcc->queued_discard))
1938                         __wait_all_discard_cmd(sbi, NULL);
1939
1940                 if (f2fs_readonly(sbi->sb))
1941                         continue;
1942                 if (kthread_should_stop())
1943                         return 0;
1944                 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK) ||
1945                         !atomic_read(&dcc->discard_cmd_cnt)) {
1946                         wait_ms = dpolicy.max_interval;
1947                         continue;
1948                 }
1949
1950                 sb_start_intwrite(sbi->sb);
1951
1952                 issued = __issue_discard_cmd(sbi, &dpolicy);
1953                 if (issued > 0) {
1954                         __wait_all_discard_cmd(sbi, &dpolicy);
1955                         wait_ms = dpolicy.min_interval;
1956                 } else if (issued == -1) {
1957                         wait_ms = f2fs_time_to_wait(sbi, DISCARD_TIME);
1958                         if (!wait_ms)
1959                                 wait_ms = dpolicy.mid_interval;
1960                 } else {
1961                         wait_ms = dpolicy.max_interval;
1962                 }
1963                 if (!atomic_read(&dcc->discard_cmd_cnt))
1964                         wait_ms = dpolicy.max_interval;
1965
1966                 sb_end_intwrite(sbi->sb);
1967
1968         } while (!kthread_should_stop());
1969         return 0;
1970 }
1971
1972 #ifdef CONFIG_BLK_DEV_ZONED
1973 static int __f2fs_issue_discard_zone(struct f2fs_sb_info *sbi,
1974                 struct block_device *bdev, block_t blkstart, block_t blklen)
1975 {
1976         sector_t sector, nr_sects;
1977         block_t lblkstart = blkstart;
1978         int devi = 0;
1979         u64 remainder = 0;
1980
1981         if (f2fs_is_multi_device(sbi)) {
1982                 devi = f2fs_target_device_index(sbi, blkstart);
1983                 if (blkstart < FDEV(devi).start_blk ||
1984                     blkstart > FDEV(devi).end_blk) {
1985                         f2fs_err(sbi, "Invalid block %x", blkstart);
1986                         return -EIO;
1987                 }
1988                 blkstart -= FDEV(devi).start_blk;
1989         }
1990
1991         /* For sequential zones, reset the zone write pointer */
1992         if (f2fs_blkz_is_seq(sbi, devi, blkstart)) {
1993                 sector = SECTOR_FROM_BLOCK(blkstart);
1994                 nr_sects = SECTOR_FROM_BLOCK(blklen);
1995                 div64_u64_rem(sector, bdev_zone_sectors(bdev), &remainder);
1996
1997                 if (remainder || nr_sects != bdev_zone_sectors(bdev)) {
1998                         f2fs_err(sbi, "(%d) %s: Unaligned zone reset attempted (block %x + %x)",
1999                                  devi, sbi->s_ndevs ? FDEV(devi).path : "",
2000                                  blkstart, blklen);
2001                         return -EIO;
2002                 }
2003
2004                 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING))) {
2005                         unsigned int nofs_flags;
2006                         int ret;
2007
2008                         trace_f2fs_issue_reset_zone(bdev, blkstart);
2009                         nofs_flags = memalloc_nofs_save();
2010                         ret = blkdev_zone_mgmt(bdev, REQ_OP_ZONE_RESET,
2011                                                 sector, nr_sects);
2012                         memalloc_nofs_restore(nofs_flags);
2013                         return ret;
2014                 }
2015
2016                 __queue_zone_reset_cmd(sbi, bdev, blkstart, lblkstart, blklen);
2017                 return 0;
2018         }
2019
2020         /* For conventional zones, use regular discard if supported */
2021         __queue_discard_cmd(sbi, bdev, lblkstart, blklen);
2022         return 0;
2023 }
2024 #endif
2025
2026 static int __issue_discard_async(struct f2fs_sb_info *sbi,
2027                 struct block_device *bdev, block_t blkstart, block_t blklen)
2028 {
2029 #ifdef CONFIG_BLK_DEV_ZONED
2030         if (f2fs_sb_has_blkzoned(sbi) && bdev_is_zoned(bdev))
2031                 return __f2fs_issue_discard_zone(sbi, bdev, blkstart, blklen);
2032 #endif
2033         __queue_discard_cmd(sbi, bdev, blkstart, blklen);
2034         return 0;
2035 }
2036
2037 static int f2fs_issue_discard(struct f2fs_sb_info *sbi,
2038                                 block_t blkstart, block_t blklen)
2039 {
2040         sector_t start = blkstart, len = 0;
2041         struct block_device *bdev;
2042         struct seg_entry *se;
2043         unsigned int offset;
2044         block_t i;
2045         int err = 0;
2046
2047         bdev = f2fs_target_device(sbi, blkstart, NULL);
2048
2049         for (i = blkstart; i < blkstart + blklen; i++, len++) {
2050                 if (i != start) {
2051                         struct block_device *bdev2 =
2052                                 f2fs_target_device(sbi, i, NULL);
2053
2054                         if (bdev2 != bdev) {
2055                                 err = __issue_discard_async(sbi, bdev,
2056                                                 start, len);
2057                                 if (err)
2058                                         return err;
2059                                 bdev = bdev2;
2060                                 start = i;
2061                                 len = 0;
2062                         }
2063                 }
2064
2065                 se = get_seg_entry(sbi, GET_SEGNO(sbi, i));
2066                 offset = GET_BLKOFF_FROM_SEG0(sbi, i);
2067
2068                 if (f2fs_block_unit_discard(sbi) &&
2069                                 !f2fs_test_and_set_bit(offset, se->discard_map))
2070                         sbi->discard_blks--;
2071         }
2072
2073         if (len)
2074                 err = __issue_discard_async(sbi, bdev, start, len);
2075         return err;
2076 }
2077
2078 static bool add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc,
2079                                                         bool check_only)
2080 {
2081         int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
2082         struct seg_entry *se = get_seg_entry(sbi, cpc->trim_start);
2083         unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
2084         unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
2085         unsigned long *discard_map = (unsigned long *)se->discard_map;
2086         unsigned long *dmap = SIT_I(sbi)->tmp_map;
2087         unsigned int start = 0, end = -1;
2088         bool force = (cpc->reason & CP_DISCARD);
2089         struct discard_entry *de = NULL;
2090         struct list_head *head = &SM_I(sbi)->dcc_info->entry_list;
2091         int i;
2092
2093         if (se->valid_blocks == BLKS_PER_SEG(sbi) ||
2094             !f2fs_hw_support_discard(sbi) ||
2095             !f2fs_block_unit_discard(sbi))
2096                 return false;
2097
2098         if (!force) {
2099                 if (!f2fs_realtime_discard_enable(sbi) || !se->valid_blocks ||
2100                         SM_I(sbi)->dcc_info->nr_discards >=
2101                                 SM_I(sbi)->dcc_info->max_discards)
2102                         return false;
2103         }
2104
2105         /* SIT_VBLOCK_MAP_SIZE should be multiple of sizeof(unsigned long) */
2106         for (i = 0; i < entries; i++)
2107                 dmap[i] = force ? ~ckpt_map[i] & ~discard_map[i] :
2108                                 (cur_map[i] ^ ckpt_map[i]) & ckpt_map[i];
2109
2110         while (force || SM_I(sbi)->dcc_info->nr_discards <=
2111                                 SM_I(sbi)->dcc_info->max_discards) {
2112                 start = __find_rev_next_bit(dmap, BLKS_PER_SEG(sbi), end + 1);
2113                 if (start >= BLKS_PER_SEG(sbi))
2114                         break;
2115
2116                 end = __find_rev_next_zero_bit(dmap,
2117                                                 BLKS_PER_SEG(sbi), start + 1);
2118                 if (force && start && end != BLKS_PER_SEG(sbi) &&
2119                     (end - start) < cpc->trim_minlen)
2120                         continue;
2121
2122                 if (check_only)
2123                         return true;
2124
2125                 if (!de) {
2126                         de = f2fs_kmem_cache_alloc(discard_entry_slab,
2127                                                 GFP_F2FS_ZERO, true, NULL);
2128                         de->start_blkaddr = START_BLOCK(sbi, cpc->trim_start);
2129                         list_add_tail(&de->list, head);
2130                 }
2131
2132                 for (i = start; i < end; i++)
2133                         __set_bit_le(i, (void *)de->discard_map);
2134
2135                 SM_I(sbi)->dcc_info->nr_discards += end - start;
2136         }
2137         return false;
2138 }
2139
2140 static void release_discard_addr(struct discard_entry *entry)
2141 {
2142         list_del(&entry->list);
2143         kmem_cache_free(discard_entry_slab, entry);
2144 }
2145
2146 void f2fs_release_discard_addrs(struct f2fs_sb_info *sbi)
2147 {
2148         struct list_head *head = &(SM_I(sbi)->dcc_info->entry_list);
2149         struct discard_entry *entry, *this;
2150
2151         /* drop caches */
2152         list_for_each_entry_safe(entry, this, head, list)
2153                 release_discard_addr(entry);
2154 }
2155
2156 /*
2157  * Should call f2fs_clear_prefree_segments after checkpoint is done.
2158  */
2159 static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi)
2160 {
2161         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2162         unsigned int segno;
2163
2164         mutex_lock(&dirty_i->seglist_lock);
2165         for_each_set_bit(segno, dirty_i->dirty_segmap[PRE], MAIN_SEGS(sbi))
2166                 __set_test_and_free(sbi, segno, false);
2167         mutex_unlock(&dirty_i->seglist_lock);
2168 }
2169
2170 void f2fs_clear_prefree_segments(struct f2fs_sb_info *sbi,
2171                                                 struct cp_control *cpc)
2172 {
2173         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
2174         struct list_head *head = &dcc->entry_list;
2175         struct discard_entry *entry, *this;
2176         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2177         unsigned long *prefree_map = dirty_i->dirty_segmap[PRE];
2178         unsigned int start = 0, end = -1;
2179         unsigned int secno, start_segno;
2180         bool force = (cpc->reason & CP_DISCARD);
2181         bool section_alignment = F2FS_OPTION(sbi).discard_unit ==
2182                                                 DISCARD_UNIT_SECTION;
2183
2184         if (f2fs_lfs_mode(sbi) && __is_large_section(sbi))
2185                 section_alignment = true;
2186
2187         mutex_lock(&dirty_i->seglist_lock);
2188
2189         while (1) {
2190                 int i;
2191
2192                 if (section_alignment && end != -1)
2193                         end--;
2194                 start = find_next_bit(prefree_map, MAIN_SEGS(sbi), end + 1);
2195                 if (start >= MAIN_SEGS(sbi))
2196                         break;
2197                 end = find_next_zero_bit(prefree_map, MAIN_SEGS(sbi),
2198                                                                 start + 1);
2199
2200                 if (section_alignment) {
2201                         start = rounddown(start, SEGS_PER_SEC(sbi));
2202                         end = roundup(end, SEGS_PER_SEC(sbi));
2203                 }
2204
2205                 for (i = start; i < end; i++) {
2206                         if (test_and_clear_bit(i, prefree_map))
2207                                 dirty_i->nr_dirty[PRE]--;
2208                 }
2209
2210                 if (!f2fs_realtime_discard_enable(sbi))
2211                         continue;
2212
2213                 if (force && start >= cpc->trim_start &&
2214                                         (end - 1) <= cpc->trim_end)
2215                         continue;
2216
2217                 /* Should cover 2MB zoned device for zone-based reset */
2218                 if (!f2fs_sb_has_blkzoned(sbi) &&
2219                     (!f2fs_lfs_mode(sbi) || !__is_large_section(sbi))) {
2220                         f2fs_issue_discard(sbi, START_BLOCK(sbi, start),
2221                                 SEGS_TO_BLKS(sbi, end - start));
2222                         continue;
2223                 }
2224 next:
2225                 secno = GET_SEC_FROM_SEG(sbi, start);
2226                 start_segno = GET_SEG_FROM_SEC(sbi, secno);
2227                 if (!IS_CURSEC(sbi, secno) &&
2228                         !get_valid_blocks(sbi, start, true))
2229                         f2fs_issue_discard(sbi, START_BLOCK(sbi, start_segno),
2230                                                 BLKS_PER_SEC(sbi));
2231
2232                 start = start_segno + SEGS_PER_SEC(sbi);
2233                 if (start < end)
2234                         goto next;
2235                 else
2236                         end = start - 1;
2237         }
2238         mutex_unlock(&dirty_i->seglist_lock);
2239
2240         if (!f2fs_block_unit_discard(sbi))
2241                 goto wakeup;
2242
2243         /* send small discards */
2244         list_for_each_entry_safe(entry, this, head, list) {
2245                 unsigned int cur_pos = 0, next_pos, len, total_len = 0;
2246                 bool is_valid = test_bit_le(0, entry->discard_map);
2247
2248 find_next:
2249                 if (is_valid) {
2250                         next_pos = find_next_zero_bit_le(entry->discard_map,
2251                                                 BLKS_PER_SEG(sbi), cur_pos);
2252                         len = next_pos - cur_pos;
2253
2254                         if (f2fs_sb_has_blkzoned(sbi) ||
2255                             (force && len < cpc->trim_minlen))
2256                                 goto skip;
2257
2258                         f2fs_issue_discard(sbi, entry->start_blkaddr + cur_pos,
2259                                                                         len);
2260                         total_len += len;
2261                 } else {
2262                         next_pos = find_next_bit_le(entry->discard_map,
2263                                                 BLKS_PER_SEG(sbi), cur_pos);
2264                 }
2265 skip:
2266                 cur_pos = next_pos;
2267                 is_valid = !is_valid;
2268
2269                 if (cur_pos < BLKS_PER_SEG(sbi))
2270                         goto find_next;
2271
2272                 release_discard_addr(entry);
2273                 dcc->nr_discards -= total_len;
2274         }
2275
2276 wakeup:
2277         wake_up_discard_thread(sbi, false);
2278 }
2279
2280 int f2fs_start_discard_thread(struct f2fs_sb_info *sbi)
2281 {
2282         dev_t dev = sbi->sb->s_bdev->bd_dev;
2283         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
2284         int err = 0;
2285
2286         if (f2fs_sb_has_readonly(sbi)) {
2287                 f2fs_info(sbi,
2288                         "Skip to start discard thread for readonly image");
2289                 return 0;
2290         }
2291
2292         if (!f2fs_realtime_discard_enable(sbi))
2293                 return 0;
2294
2295         dcc->f2fs_issue_discard = kthread_run(issue_discard_thread, sbi,
2296                                 "f2fs_discard-%u:%u", MAJOR(dev), MINOR(dev));
2297         if (IS_ERR(dcc->f2fs_issue_discard)) {
2298                 err = PTR_ERR(dcc->f2fs_issue_discard);
2299                 dcc->f2fs_issue_discard = NULL;
2300         }
2301
2302         return err;
2303 }
2304
2305 static int create_discard_cmd_control(struct f2fs_sb_info *sbi)
2306 {
2307         struct discard_cmd_control *dcc;
2308         int err = 0, i;
2309
2310         if (SM_I(sbi)->dcc_info) {
2311                 dcc = SM_I(sbi)->dcc_info;
2312                 goto init_thread;
2313         }
2314
2315         dcc = f2fs_kzalloc(sbi, sizeof(struct discard_cmd_control), GFP_KERNEL);
2316         if (!dcc)
2317                 return -ENOMEM;
2318
2319         dcc->discard_io_aware_gran = MAX_PLIST_NUM;
2320         dcc->discard_granularity = DEFAULT_DISCARD_GRANULARITY;
2321         dcc->max_ordered_discard = DEFAULT_MAX_ORDERED_DISCARD_GRANULARITY;
2322         dcc->discard_io_aware = DPOLICY_IO_AWARE_ENABLE;
2323         if (F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SEGMENT)
2324                 dcc->discard_granularity = BLKS_PER_SEG(sbi);
2325         else if (F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SECTION)
2326                 dcc->discard_granularity = BLKS_PER_SEC(sbi);
2327
2328         INIT_LIST_HEAD(&dcc->entry_list);
2329         for (i = 0; i < MAX_PLIST_NUM; i++)
2330                 INIT_LIST_HEAD(&dcc->pend_list[i]);
2331         INIT_LIST_HEAD(&dcc->wait_list);
2332         INIT_LIST_HEAD(&dcc->fstrim_list);
2333         mutex_init(&dcc->cmd_lock);
2334         atomic_set(&dcc->issued_discard, 0);
2335         atomic_set(&dcc->queued_discard, 0);
2336         atomic_set(&dcc->discard_cmd_cnt, 0);
2337         dcc->nr_discards = 0;
2338         dcc->max_discards = SEGS_TO_BLKS(sbi, MAIN_SEGS(sbi));
2339         dcc->max_discard_request = DEF_MAX_DISCARD_REQUEST;
2340         dcc->min_discard_issue_time = DEF_MIN_DISCARD_ISSUE_TIME;
2341         dcc->mid_discard_issue_time = DEF_MID_DISCARD_ISSUE_TIME;
2342         dcc->max_discard_issue_time = DEF_MAX_DISCARD_ISSUE_TIME;
2343         dcc->discard_urgent_util = DEF_DISCARD_URGENT_UTIL;
2344         dcc->undiscard_blks = 0;
2345         dcc->next_pos = 0;
2346         dcc->root = RB_ROOT_CACHED;
2347         dcc->rbtree_check = false;
2348
2349         init_waitqueue_head(&dcc->discard_wait_queue);
2350         SM_I(sbi)->dcc_info = dcc;
2351 init_thread:
2352         err = f2fs_start_discard_thread(sbi);
2353         if (err) {
2354                 kfree(dcc);
2355                 SM_I(sbi)->dcc_info = NULL;
2356         }
2357
2358         return err;
2359 }
2360
2361 static void destroy_discard_cmd_control(struct f2fs_sb_info *sbi)
2362 {
2363         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
2364
2365         if (!dcc)
2366                 return;
2367
2368         f2fs_stop_discard_thread(sbi);
2369
2370         /*
2371          * Recovery can cache discard commands, so in error path of
2372          * fill_super(), it needs to give a chance to handle them.
2373          */
2374         f2fs_issue_discard_timeout(sbi);
2375
2376         kfree(dcc);
2377         SM_I(sbi)->dcc_info = NULL;
2378 }
2379
2380 static bool __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno)
2381 {
2382         struct sit_info *sit_i = SIT_I(sbi);
2383
2384         if (!__test_and_set_bit(segno, sit_i->dirty_sentries_bitmap)) {
2385                 sit_i->dirty_sentries++;
2386                 return false;
2387         }
2388
2389         return true;
2390 }
2391
2392 static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type,
2393                                         unsigned int segno, int modified)
2394 {
2395         struct seg_entry *se = get_seg_entry(sbi, segno);
2396
2397         se->type = type;
2398         if (modified)
2399                 __mark_sit_entry_dirty(sbi, segno);
2400 }
2401
2402 static inline unsigned long long get_segment_mtime(struct f2fs_sb_info *sbi,
2403                                                                 block_t blkaddr)
2404 {
2405         unsigned int segno = GET_SEGNO(sbi, blkaddr);
2406
2407         if (segno == NULL_SEGNO)
2408                 return 0;
2409         return get_seg_entry(sbi, segno)->mtime;
2410 }
2411
2412 static void update_segment_mtime(struct f2fs_sb_info *sbi, block_t blkaddr,
2413                                                 unsigned long long old_mtime)
2414 {
2415         struct seg_entry *se;
2416         unsigned int segno = GET_SEGNO(sbi, blkaddr);
2417         unsigned long long ctime = get_mtime(sbi, false);
2418         unsigned long long mtime = old_mtime ? old_mtime : ctime;
2419
2420         if (segno == NULL_SEGNO)
2421                 return;
2422
2423         se = get_seg_entry(sbi, segno);
2424
2425         if (!se->mtime)
2426                 se->mtime = mtime;
2427         else
2428                 se->mtime = div_u64(se->mtime * se->valid_blocks + mtime,
2429                                                 se->valid_blocks + 1);
2430
2431         if (ctime > SIT_I(sbi)->max_mtime)
2432                 SIT_I(sbi)->max_mtime = ctime;
2433 }
2434
2435 /*
2436  * NOTE: when updating multiple blocks at the same time, please ensure
2437  * that the consecutive input blocks belong to the same segment.
2438  */
2439 static int update_sit_entry_for_release(struct f2fs_sb_info *sbi, struct seg_entry *se,
2440                                 block_t blkaddr, unsigned int offset, int del)
2441 {
2442         bool exist;
2443 #ifdef CONFIG_F2FS_CHECK_FS
2444         bool mir_exist;
2445 #endif
2446         int i;
2447         int del_count = -del;
2448
2449         f2fs_bug_on(sbi, GET_SEGNO(sbi, blkaddr) != GET_SEGNO(sbi, blkaddr + del_count - 1));
2450
2451         for (i = 0; i < del_count; i++) {
2452                 exist = f2fs_test_and_clear_bit(offset + i, se->cur_valid_map);
2453 #ifdef CONFIG_F2FS_CHECK_FS
2454                 mir_exist = f2fs_test_and_clear_bit(offset + i,
2455                                                 se->cur_valid_map_mir);
2456                 if (unlikely(exist != mir_exist)) {
2457                         f2fs_err(sbi, "Inconsistent error when clearing bitmap, blk:%u, old bit:%d",
2458                                 blkaddr + i, exist);
2459                         f2fs_bug_on(sbi, 1);
2460                 }
2461 #endif
2462                 if (unlikely(!exist)) {
2463                         f2fs_err(sbi, "Bitmap was wrongly cleared, blk:%u", blkaddr + i);
2464                         f2fs_bug_on(sbi, 1);
2465                         se->valid_blocks++;
2466                         del += 1;
2467                 } else if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
2468                         /*
2469                          * If checkpoints are off, we must not reuse data that
2470                          * was used in the previous checkpoint. If it was used
2471                          * before, we must track that to know how much space we
2472                          * really have.
2473                          */
2474                         if (f2fs_test_bit(offset + i, se->ckpt_valid_map)) {
2475                                 spin_lock(&sbi->stat_lock);
2476                                 sbi->unusable_block_count++;
2477                                 spin_unlock(&sbi->stat_lock);
2478                         }
2479                 }
2480
2481                 if (f2fs_block_unit_discard(sbi) &&
2482                                 f2fs_test_and_clear_bit(offset + i, se->discard_map))
2483                         sbi->discard_blks++;
2484
2485                 if (!f2fs_test_bit(offset + i, se->ckpt_valid_map))
2486                         se->ckpt_valid_blocks -= 1;
2487         }
2488
2489         return del;
2490 }
2491
2492 static int update_sit_entry_for_alloc(struct f2fs_sb_info *sbi, struct seg_entry *se,
2493                                 block_t blkaddr, unsigned int offset, int del)
2494 {
2495         bool exist;
2496 #ifdef CONFIG_F2FS_CHECK_FS
2497         bool mir_exist;
2498 #endif
2499
2500         exist = f2fs_test_and_set_bit(offset, se->cur_valid_map);
2501 #ifdef CONFIG_F2FS_CHECK_FS
2502         mir_exist = f2fs_test_and_set_bit(offset,
2503                                         se->cur_valid_map_mir);
2504         if (unlikely(exist != mir_exist)) {
2505                 f2fs_err(sbi, "Inconsistent error when setting bitmap, blk:%u, old bit:%d",
2506                         blkaddr, exist);
2507                 f2fs_bug_on(sbi, 1);
2508         }
2509 #endif
2510         if (unlikely(exist)) {
2511                 f2fs_err(sbi, "Bitmap was wrongly set, blk:%u", blkaddr);
2512                 f2fs_bug_on(sbi, 1);
2513                 se->valid_blocks--;
2514                 del = 0;
2515         }
2516
2517         if (f2fs_block_unit_discard(sbi) &&
2518                         !f2fs_test_and_set_bit(offset, se->discard_map))
2519                 sbi->discard_blks--;
2520
2521         /*
2522          * SSR should never reuse block which is checkpointed
2523          * or newly invalidated.
2524          */
2525         if (!is_sbi_flag_set(sbi, SBI_CP_DISABLED)) {
2526                 if (!f2fs_test_and_set_bit(offset, se->ckpt_valid_map))
2527                         se->ckpt_valid_blocks++;
2528         }
2529
2530         if (!f2fs_test_bit(offset, se->ckpt_valid_map))
2531                 se->ckpt_valid_blocks += del;
2532
2533         return del;
2534 }
2535
2536 /*
2537  * If releasing blocks, this function supports updating multiple consecutive blocks
2538  * at one time, but please note that these consecutive blocks need to belong to the
2539  * same segment.
2540  */
2541 static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del)
2542 {
2543         struct seg_entry *se;
2544         unsigned int segno, offset;
2545         long int new_vblocks;
2546
2547         segno = GET_SEGNO(sbi, blkaddr);
2548         if (segno == NULL_SEGNO)
2549                 return;
2550
2551         se = get_seg_entry(sbi, segno);
2552         new_vblocks = se->valid_blocks + del;
2553         offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
2554
2555         f2fs_bug_on(sbi, (new_vblocks < 0 ||
2556                         (new_vblocks > f2fs_usable_blks_in_seg(sbi, segno))));
2557
2558         se->valid_blocks = new_vblocks;
2559
2560         /* Update valid block bitmap */
2561         if (del > 0) {
2562                 del = update_sit_entry_for_alloc(sbi, se, blkaddr, offset, del);
2563         } else {
2564                 del = update_sit_entry_for_release(sbi, se, blkaddr, offset, del);
2565         }
2566
2567         __mark_sit_entry_dirty(sbi, segno);
2568
2569         /* update total number of valid blocks to be written in ckpt area */
2570         SIT_I(sbi)->written_valid_blocks += del;
2571
2572         if (__is_large_section(sbi))
2573                 get_sec_entry(sbi, segno)->valid_blocks += del;
2574 }
2575
2576 void f2fs_invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr,
2577                                 unsigned int len)
2578 {
2579         unsigned int segno = GET_SEGNO(sbi, addr);
2580         struct sit_info *sit_i = SIT_I(sbi);
2581         block_t addr_start = addr, addr_end = addr + len - 1;
2582         unsigned int seg_num = GET_SEGNO(sbi, addr_end) - segno + 1;
2583         unsigned int i = 1, max_blocks = sbi->blocks_per_seg, cnt;
2584
2585         f2fs_bug_on(sbi, addr == NULL_ADDR);
2586         if (addr == NEW_ADDR || addr == COMPRESS_ADDR)
2587                 return;
2588
2589         f2fs_invalidate_internal_cache(sbi, addr, len);
2590
2591         /* add it into sit main buffer */
2592         down_write(&sit_i->sentry_lock);
2593
2594         if (seg_num == 1)
2595                 cnt = len;
2596         else
2597                 cnt = max_blocks - GET_BLKOFF_FROM_SEG0(sbi, addr);
2598
2599         do {
2600                 update_segment_mtime(sbi, addr_start, 0);
2601                 update_sit_entry(sbi, addr_start, -cnt);
2602
2603                 /* add it into dirty seglist */
2604                 locate_dirty_segment(sbi, segno);
2605
2606                 /* update @addr_start and @cnt and @segno */
2607                 addr_start = START_BLOCK(sbi, ++segno);
2608                 if (++i == seg_num)
2609                         cnt = GET_BLKOFF_FROM_SEG0(sbi, addr_end) + 1;
2610                 else
2611                         cnt = max_blocks;
2612         } while (i <= seg_num);
2613
2614         up_write(&sit_i->sentry_lock);
2615 }
2616
2617 bool f2fs_is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr)
2618 {
2619         struct sit_info *sit_i = SIT_I(sbi);
2620         unsigned int segno, offset;
2621         struct seg_entry *se;
2622         bool is_cp = false;
2623
2624         if (!__is_valid_data_blkaddr(blkaddr))
2625                 return true;
2626
2627         down_read(&sit_i->sentry_lock);
2628
2629         segno = GET_SEGNO(sbi, blkaddr);
2630         se = get_seg_entry(sbi, segno);
2631         offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
2632
2633         if (f2fs_test_bit(offset, se->ckpt_valid_map))
2634                 is_cp = true;
2635
2636         up_read(&sit_i->sentry_lock);
2637
2638         return is_cp;
2639 }
2640
2641 static unsigned short f2fs_curseg_valid_blocks(struct f2fs_sb_info *sbi, int type)
2642 {
2643         struct curseg_info *curseg = CURSEG_I(sbi, type);
2644
2645         if (sbi->ckpt->alloc_type[type] == SSR)
2646                 return BLKS_PER_SEG(sbi);
2647         return curseg->next_blkoff;
2648 }
2649
2650 /*
2651  * Calculate the number of current summary pages for writing
2652  */
2653 int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra)
2654 {
2655         int valid_sum_count = 0;
2656         int i, sum_in_page;
2657
2658         for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
2659                 if (sbi->ckpt->alloc_type[i] != SSR && for_ra)
2660                         valid_sum_count +=
2661                                 le16_to_cpu(F2FS_CKPT(sbi)->cur_data_blkoff[i]);
2662                 else
2663                         valid_sum_count += f2fs_curseg_valid_blocks(sbi, i);
2664         }
2665
2666         sum_in_page = (PAGE_SIZE - 2 * SUM_JOURNAL_SIZE -
2667                         SUM_FOOTER_SIZE) / SUMMARY_SIZE;
2668         if (valid_sum_count <= sum_in_page)
2669                 return 1;
2670         else if ((valid_sum_count - sum_in_page) <=
2671                 (PAGE_SIZE - SUM_FOOTER_SIZE) / SUMMARY_SIZE)
2672                 return 2;
2673         return 3;
2674 }
2675
2676 /*
2677  * Caller should put this summary page
2678  */
2679 struct page *f2fs_get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno)
2680 {
2681         if (unlikely(f2fs_cp_error(sbi)))
2682                 return ERR_PTR(-EIO);
2683         return f2fs_get_meta_page_retry(sbi, GET_SUM_BLOCK(sbi, segno));
2684 }
2685
2686 void f2fs_update_meta_page(struct f2fs_sb_info *sbi,
2687                                         void *src, block_t blk_addr)
2688 {
2689         struct page *page = f2fs_grab_meta_page(sbi, blk_addr);
2690
2691         memcpy(page_address(page), src, PAGE_SIZE);
2692         set_page_dirty(page);
2693         f2fs_put_page(page, 1);
2694 }
2695
2696 static void write_sum_page(struct f2fs_sb_info *sbi,
2697                         struct f2fs_summary_block *sum_blk, block_t blk_addr)
2698 {
2699         f2fs_update_meta_page(sbi, (void *)sum_blk, blk_addr);
2700 }
2701
2702 static void write_current_sum_page(struct f2fs_sb_info *sbi,
2703                                                 int type, block_t blk_addr)
2704 {
2705         struct curseg_info *curseg = CURSEG_I(sbi, type);
2706         struct page *page = f2fs_grab_meta_page(sbi, blk_addr);
2707         struct f2fs_summary_block *src = curseg->sum_blk;
2708         struct f2fs_summary_block *dst;
2709
2710         dst = (struct f2fs_summary_block *)page_address(page);
2711         memset(dst, 0, PAGE_SIZE);
2712
2713         mutex_lock(&curseg->curseg_mutex);
2714
2715         down_read(&curseg->journal_rwsem);
2716         memcpy(&dst->journal, curseg->journal, SUM_JOURNAL_SIZE);
2717         up_read(&curseg->journal_rwsem);
2718
2719         memcpy(dst->entries, src->entries, SUM_ENTRY_SIZE);
2720         memcpy(&dst->footer, &src->footer, SUM_FOOTER_SIZE);
2721
2722         mutex_unlock(&curseg->curseg_mutex);
2723
2724         set_page_dirty(page);
2725         f2fs_put_page(page, 1);
2726 }
2727
2728 static int is_next_segment_free(struct f2fs_sb_info *sbi,
2729                                 struct curseg_info *curseg)
2730 {
2731         unsigned int segno = curseg->segno + 1;
2732         struct free_segmap_info *free_i = FREE_I(sbi);
2733
2734         if (segno < MAIN_SEGS(sbi) && segno % SEGS_PER_SEC(sbi))
2735                 return !test_bit(segno, free_i->free_segmap);
2736         return 0;
2737 }
2738
2739 /*
2740  * Find a new segment from the free segments bitmap to right order
2741  * This function should be returned with success, otherwise BUG
2742  */
2743 static int get_new_segment(struct f2fs_sb_info *sbi,
2744                         unsigned int *newseg, bool new_sec, bool pinning)
2745 {
2746         struct free_segmap_info *free_i = FREE_I(sbi);
2747         unsigned int segno, secno, zoneno;
2748         unsigned int total_zones = MAIN_SECS(sbi) / sbi->secs_per_zone;
2749         unsigned int hint = GET_SEC_FROM_SEG(sbi, *newseg);
2750         unsigned int old_zoneno = GET_ZONE_FROM_SEG(sbi, *newseg);
2751         bool init = true;
2752         int i;
2753         int ret = 0;
2754
2755         spin_lock(&free_i->segmap_lock);
2756
2757         if (time_to_inject(sbi, FAULT_NO_SEGMENT)) {
2758                 ret = -ENOSPC;
2759                 goto out_unlock;
2760         }
2761
2762         if (!new_sec && ((*newseg + 1) % SEGS_PER_SEC(sbi))) {
2763                 segno = find_next_zero_bit(free_i->free_segmap,
2764                         GET_SEG_FROM_SEC(sbi, hint + 1), *newseg + 1);
2765                 if (segno < GET_SEG_FROM_SEC(sbi, hint + 1))
2766                         goto got_it;
2767         }
2768
2769 #ifdef CONFIG_BLK_DEV_ZONED
2770         /*
2771          * If we format f2fs on zoned storage, let's try to get pinned sections
2772          * from beginning of the storage, which should be a conventional one.
2773          */
2774         if (f2fs_sb_has_blkzoned(sbi)) {
2775                 /* Prioritize writing to conventional zones */
2776                 if (sbi->blkzone_alloc_policy == BLKZONE_ALLOC_PRIOR_CONV || pinning)
2777                         segno = 0;
2778                 else
2779                         segno = max(sbi->first_zoned_segno, *newseg);
2780                 hint = GET_SEC_FROM_SEG(sbi, segno);
2781         }
2782 #endif
2783
2784 find_other_zone:
2785         secno = find_next_zero_bit(free_i->free_secmap, MAIN_SECS(sbi), hint);
2786
2787 #ifdef CONFIG_BLK_DEV_ZONED
2788         if (secno >= MAIN_SECS(sbi) && f2fs_sb_has_blkzoned(sbi)) {
2789                 /* Write only to sequential zones */
2790                 if (sbi->blkzone_alloc_policy == BLKZONE_ALLOC_ONLY_SEQ) {
2791                         hint = GET_SEC_FROM_SEG(sbi, sbi->first_zoned_segno);
2792                         secno = find_next_zero_bit(free_i->free_secmap, MAIN_SECS(sbi), hint);
2793                 } else
2794                         secno = find_first_zero_bit(free_i->free_secmap,
2795                                                                 MAIN_SECS(sbi));
2796                 if (secno >= MAIN_SECS(sbi)) {
2797                         ret = -ENOSPC;
2798                         f2fs_bug_on(sbi, 1);
2799                         goto out_unlock;
2800                 }
2801         }
2802 #endif
2803
2804         if (secno >= MAIN_SECS(sbi)) {
2805                 secno = find_first_zero_bit(free_i->free_secmap,
2806                                                         MAIN_SECS(sbi));
2807                 if (secno >= MAIN_SECS(sbi)) {
2808                         ret = -ENOSPC;
2809                         f2fs_bug_on(sbi, 1);
2810                         goto out_unlock;
2811                 }
2812         }
2813         segno = GET_SEG_FROM_SEC(sbi, secno);
2814         zoneno = GET_ZONE_FROM_SEC(sbi, secno);
2815
2816         /* give up on finding another zone */
2817         if (!init)
2818                 goto got_it;
2819         if (sbi->secs_per_zone == 1)
2820                 goto got_it;
2821         if (zoneno == old_zoneno)
2822                 goto got_it;
2823         for (i = 0; i < NR_CURSEG_TYPE; i++)
2824                 if (CURSEG_I(sbi, i)->zone == zoneno)
2825                         break;
2826
2827         if (i < NR_CURSEG_TYPE) {
2828                 /* zone is in user, try another */
2829                 if (zoneno + 1 >= total_zones)
2830                         hint = 0;
2831                 else
2832                         hint = (zoneno + 1) * sbi->secs_per_zone;
2833                 init = false;
2834                 goto find_other_zone;
2835         }
2836 got_it:
2837         /* set it as dirty segment in free segmap */
2838         f2fs_bug_on(sbi, test_bit(segno, free_i->free_segmap));
2839
2840         /* no free section in conventional zone */
2841         if (new_sec && pinning &&
2842                 !f2fs_valid_pinned_area(sbi, START_BLOCK(sbi, segno))) {
2843                 ret = -EAGAIN;
2844                 goto out_unlock;
2845         }
2846         __set_inuse(sbi, segno);
2847         *newseg = segno;
2848 out_unlock:
2849         spin_unlock(&free_i->segmap_lock);
2850
2851         if (ret == -ENOSPC)
2852                 f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_NO_SEGMENT);
2853         return ret;
2854 }
2855
2856 static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified)
2857 {
2858         struct curseg_info *curseg = CURSEG_I(sbi, type);
2859         struct summary_footer *sum_footer;
2860         unsigned short seg_type = curseg->seg_type;
2861
2862         /* only happen when get_new_segment() fails */
2863         if (curseg->next_segno == NULL_SEGNO)
2864                 return;
2865
2866         curseg->inited = true;
2867         curseg->segno = curseg->next_segno;
2868         curseg->zone = GET_ZONE_FROM_SEG(sbi, curseg->segno);
2869         curseg->next_blkoff = 0;
2870         curseg->next_segno = NULL_SEGNO;
2871
2872         sum_footer = &(curseg->sum_blk->footer);
2873         memset(sum_footer, 0, sizeof(struct summary_footer));
2874
2875         sanity_check_seg_type(sbi, seg_type);
2876
2877         if (IS_DATASEG(seg_type))
2878                 SET_SUM_TYPE(sum_footer, SUM_TYPE_DATA);
2879         if (IS_NODESEG(seg_type))
2880                 SET_SUM_TYPE(sum_footer, SUM_TYPE_NODE);
2881         __set_sit_entry_type(sbi, seg_type, curseg->segno, modified);
2882 }
2883
2884 static unsigned int __get_next_segno(struct f2fs_sb_info *sbi, int type)
2885 {
2886         struct curseg_info *curseg = CURSEG_I(sbi, type);
2887         unsigned short seg_type = curseg->seg_type;
2888
2889         sanity_check_seg_type(sbi, seg_type);
2890         if (__is_large_section(sbi)) {
2891                 if (f2fs_need_rand_seg(sbi)) {
2892                         unsigned int hint = GET_SEC_FROM_SEG(sbi, curseg->segno);
2893
2894                         if (GET_SEC_FROM_SEG(sbi, curseg->segno + 1) != hint)
2895                                 return curseg->segno;
2896                         return get_random_u32_inclusive(curseg->segno + 1,
2897                                         GET_SEG_FROM_SEC(sbi, hint + 1) - 1);
2898                 }
2899                 return curseg->segno;
2900         } else if (f2fs_need_rand_seg(sbi)) {
2901                 return get_random_u32_below(MAIN_SECS(sbi) * SEGS_PER_SEC(sbi));
2902         }
2903
2904         /* inmem log may not locate on any segment after mount */
2905         if (!curseg->inited)
2906                 return 0;
2907
2908         if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
2909                 return 0;
2910
2911         if (seg_type == CURSEG_HOT_DATA || IS_NODESEG(seg_type))
2912                 return 0;
2913
2914         if (SIT_I(sbi)->last_victim[ALLOC_NEXT])
2915                 return SIT_I(sbi)->last_victim[ALLOC_NEXT];
2916
2917         /* find segments from 0 to reuse freed segments */
2918         if (F2FS_OPTION(sbi).alloc_mode == ALLOC_MODE_REUSE)
2919                 return 0;
2920
2921         return curseg->segno;
2922 }
2923
2924 /*
2925  * Allocate a current working segment.
2926  * This function always allocates a free segment in LFS manner.
2927  */
2928 static int new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec)
2929 {
2930         struct curseg_info *curseg = CURSEG_I(sbi, type);
2931         unsigned int segno = curseg->segno;
2932         bool pinning = type == CURSEG_COLD_DATA_PINNED;
2933         int ret;
2934
2935         if (curseg->inited)
2936                 write_sum_page(sbi, curseg->sum_blk, GET_SUM_BLOCK(sbi, segno));
2937
2938         segno = __get_next_segno(sbi, type);
2939         ret = get_new_segment(sbi, &segno, new_sec, pinning);
2940         if (ret) {
2941                 if (ret == -ENOSPC)
2942                         curseg->segno = NULL_SEGNO;
2943                 return ret;
2944         }
2945
2946         curseg->next_segno = segno;
2947         reset_curseg(sbi, type, 1);
2948         curseg->alloc_type = LFS;
2949         if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK)
2950                 curseg->fragment_remained_chunk =
2951                                 get_random_u32_inclusive(1, sbi->max_fragment_chunk);
2952         return 0;
2953 }
2954
2955 static int __next_free_blkoff(struct f2fs_sb_info *sbi,
2956                                         int segno, block_t start)
2957 {
2958         struct seg_entry *se = get_seg_entry(sbi, segno);
2959         int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
2960         unsigned long *target_map = SIT_I(sbi)->tmp_map;
2961         unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
2962         unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
2963         int i;
2964
2965         for (i = 0; i < entries; i++)
2966                 target_map[i] = ckpt_map[i] | cur_map[i];
2967
2968         return __find_rev_next_zero_bit(target_map, BLKS_PER_SEG(sbi), start);
2969 }
2970
2971 static int f2fs_find_next_ssr_block(struct f2fs_sb_info *sbi,
2972                 struct curseg_info *seg)
2973 {
2974         return __next_free_blkoff(sbi, seg->segno, seg->next_blkoff + 1);
2975 }
2976
2977 bool f2fs_segment_has_free_slot(struct f2fs_sb_info *sbi, int segno)
2978 {
2979         return __next_free_blkoff(sbi, segno, 0) < BLKS_PER_SEG(sbi);
2980 }
2981
2982 /*
2983  * This function always allocates a used segment(from dirty seglist) by SSR
2984  * manner, so it should recover the existing segment information of valid blocks
2985  */
2986 static int change_curseg(struct f2fs_sb_info *sbi, int type)
2987 {
2988         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2989         struct curseg_info *curseg = CURSEG_I(sbi, type);
2990         unsigned int new_segno = curseg->next_segno;
2991         struct f2fs_summary_block *sum_node;
2992         struct page *sum_page;
2993
2994         if (curseg->inited)
2995                 write_sum_page(sbi, curseg->sum_blk, GET_SUM_BLOCK(sbi, curseg->segno));
2996
2997         __set_test_and_inuse(sbi, new_segno);
2998
2999         mutex_lock(&dirty_i->seglist_lock);
3000         __remove_dirty_segment(sbi, new_segno, PRE);
3001         __remove_dirty_segment(sbi, new_segno, DIRTY);
3002         mutex_unlock(&dirty_i->seglist_lock);
3003
3004         reset_curseg(sbi, type, 1);
3005         curseg->alloc_type = SSR;
3006         curseg->next_blkoff = __next_free_blkoff(sbi, curseg->segno, 0);
3007
3008         sum_page = f2fs_get_sum_page(sbi, new_segno);
3009         if (IS_ERR(sum_page)) {
3010                 /* GC won't be able to use stale summary pages by cp_error */
3011                 memset(curseg->sum_blk, 0, SUM_ENTRY_SIZE);
3012                 return PTR_ERR(sum_page);
3013         }
3014         sum_node = (struct f2fs_summary_block *)page_address(sum_page);
3015         memcpy(curseg->sum_blk, sum_node, SUM_ENTRY_SIZE);
3016         f2fs_put_page(sum_page, 1);
3017         return 0;
3018 }
3019
3020 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type,
3021                                 int alloc_mode, unsigned long long age);
3022
3023 static int get_atssr_segment(struct f2fs_sb_info *sbi, int type,
3024                                         int target_type, int alloc_mode,
3025                                         unsigned long long age)
3026 {
3027         struct curseg_info *curseg = CURSEG_I(sbi, type);
3028         int ret = 0;
3029
3030         curseg->seg_type = target_type;
3031
3032         if (get_ssr_segment(sbi, type, alloc_mode, age)) {
3033                 struct seg_entry *se = get_seg_entry(sbi, curseg->next_segno);
3034
3035                 curseg->seg_type = se->type;
3036                 ret = change_curseg(sbi, type);
3037         } else {
3038                 /* allocate cold segment by default */
3039                 curseg->seg_type = CURSEG_COLD_DATA;
3040                 ret = new_curseg(sbi, type, true);
3041         }
3042         stat_inc_seg_type(sbi, curseg);
3043         return ret;
3044 }
3045
3046 static int __f2fs_init_atgc_curseg(struct f2fs_sb_info *sbi, bool force)
3047 {
3048         struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_ALL_DATA_ATGC);
3049         int ret = 0;
3050
3051         if (!sbi->am.atgc_enabled && !force)
3052                 return 0;
3053
3054         f2fs_down_read(&SM_I(sbi)->curseg_lock);
3055
3056         mutex_lock(&curseg->curseg_mutex);
3057         down_write(&SIT_I(sbi)->sentry_lock);
3058
3059         ret = get_atssr_segment(sbi, CURSEG_ALL_DATA_ATGC,
3060                                         CURSEG_COLD_DATA, SSR, 0);
3061
3062         up_write(&SIT_I(sbi)->sentry_lock);
3063         mutex_unlock(&curseg->curseg_mutex);
3064
3065         f2fs_up_read(&SM_I(sbi)->curseg_lock);
3066         return ret;
3067 }
3068
3069 int f2fs_init_inmem_curseg(struct f2fs_sb_info *sbi)
3070 {
3071         return __f2fs_init_atgc_curseg(sbi, false);
3072 }
3073
3074 int f2fs_reinit_atgc_curseg(struct f2fs_sb_info *sbi)
3075 {
3076         int ret;
3077
3078         if (!test_opt(sbi, ATGC))
3079                 return 0;
3080         if (sbi->am.atgc_enabled)
3081                 return 0;
3082         if (le64_to_cpu(F2FS_CKPT(sbi)->elapsed_time) <
3083                         sbi->am.age_threshold)
3084                 return 0;
3085
3086         ret = __f2fs_init_atgc_curseg(sbi, true);
3087         if (!ret) {
3088                 sbi->am.atgc_enabled = true;
3089                 f2fs_info(sbi, "reenabled age threshold GC");
3090         }
3091         return ret;
3092 }
3093
3094 static void __f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi, int type)
3095 {
3096         struct curseg_info *curseg = CURSEG_I(sbi, type);
3097
3098         mutex_lock(&curseg->curseg_mutex);
3099         if (!curseg->inited)
3100                 goto out;
3101
3102         if (get_valid_blocks(sbi, curseg->segno, false)) {
3103                 write_sum_page(sbi, curseg->sum_blk,
3104                                 GET_SUM_BLOCK(sbi, curseg->segno));
3105         } else {
3106                 mutex_lock(&DIRTY_I(sbi)->seglist_lock);
3107                 __set_test_and_free(sbi, curseg->segno, true);
3108                 mutex_unlock(&DIRTY_I(sbi)->seglist_lock);
3109         }
3110 out:
3111         mutex_unlock(&curseg->curseg_mutex);
3112 }
3113
3114 void f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi)
3115 {
3116         __f2fs_save_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED);
3117
3118         if (sbi->am.atgc_enabled)
3119                 __f2fs_save_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC);
3120 }
3121
3122 static void __f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi, int type)
3123 {
3124         struct curseg_info *curseg = CURSEG_I(sbi, type);
3125
3126         mutex_lock(&curseg->curseg_mutex);
3127         if (!curseg->inited)
3128                 goto out;
3129         if (get_valid_blocks(sbi, curseg->segno, false))
3130                 goto out;
3131
3132         mutex_lock(&DIRTY_I(sbi)->seglist_lock);
3133         __set_test_and_inuse(sbi, curseg->segno);
3134         mutex_unlock(&DIRTY_I(sbi)->seglist_lock);
3135 out:
3136         mutex_unlock(&curseg->curseg_mutex);
3137 }
3138
3139 void f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi)
3140 {
3141         __f2fs_restore_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED);
3142
3143         if (sbi->am.atgc_enabled)
3144                 __f2fs_restore_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC);
3145 }
3146
3147 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type,
3148                                 int alloc_mode, unsigned long long age)
3149 {
3150         struct curseg_info *curseg = CURSEG_I(sbi, type);
3151         unsigned segno = NULL_SEGNO;
3152         unsigned short seg_type = curseg->seg_type;
3153         int i, cnt;
3154         bool reversed = false;
3155
3156         sanity_check_seg_type(sbi, seg_type);
3157
3158         /* f2fs_need_SSR() already forces to do this */
3159         if (!f2fs_get_victim(sbi, &segno, BG_GC, seg_type,
3160                                 alloc_mode, age, false)) {
3161                 curseg->next_segno = segno;
3162                 return 1;
3163         }
3164
3165         /* For node segments, let's do SSR more intensively */
3166         if (IS_NODESEG(seg_type)) {
3167                 if (seg_type >= CURSEG_WARM_NODE) {
3168                         reversed = true;
3169                         i = CURSEG_COLD_NODE;
3170                 } else {
3171                         i = CURSEG_HOT_NODE;
3172                 }
3173                 cnt = NR_CURSEG_NODE_TYPE;
3174         } else {
3175                 if (seg_type >= CURSEG_WARM_DATA) {
3176                         reversed = true;
3177                         i = CURSEG_COLD_DATA;
3178                 } else {
3179                         i = CURSEG_HOT_DATA;
3180                 }
3181                 cnt = NR_CURSEG_DATA_TYPE;
3182         }
3183
3184         for (; cnt-- > 0; reversed ? i-- : i++) {
3185                 if (i == seg_type)
3186                         continue;
3187                 if (!f2fs_get_victim(sbi, &segno, BG_GC, i,
3188                                         alloc_mode, age, false)) {
3189                         curseg->next_segno = segno;
3190                         return 1;
3191                 }
3192         }
3193
3194         /* find valid_blocks=0 in dirty list */
3195         if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
3196                 segno = get_free_segment(sbi);
3197                 if (segno != NULL_SEGNO) {
3198                         curseg->next_segno = segno;
3199                         return 1;
3200                 }
3201         }
3202         return 0;
3203 }
3204
3205 static bool need_new_seg(struct f2fs_sb_info *sbi, int type)
3206 {
3207         struct curseg_info *curseg = CURSEG_I(sbi, type);
3208
3209         if (!is_set_ckpt_flags(sbi, CP_CRC_RECOVERY_FLAG) &&
3210             curseg->seg_type == CURSEG_WARM_NODE)
3211                 return true;
3212         if (curseg->alloc_type == LFS && is_next_segment_free(sbi, curseg) &&
3213             likely(!is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
3214                 return true;
3215         if (!f2fs_need_SSR(sbi) || !get_ssr_segment(sbi, type, SSR, 0))
3216                 return true;
3217         return false;
3218 }
3219
3220 int f2fs_allocate_segment_for_resize(struct f2fs_sb_info *sbi, int type,
3221                                         unsigned int start, unsigned int end)
3222 {
3223         struct curseg_info *curseg = CURSEG_I(sbi, type);
3224         unsigned int segno;
3225         int ret = 0;
3226
3227         f2fs_down_read(&SM_I(sbi)->curseg_lock);
3228         mutex_lock(&curseg->curseg_mutex);
3229         down_write(&SIT_I(sbi)->sentry_lock);
3230
3231         segno = CURSEG_I(sbi, type)->segno;
3232         if (segno < start || segno > end)
3233                 goto unlock;
3234
3235         if (f2fs_need_SSR(sbi) && get_ssr_segment(sbi, type, SSR, 0))
3236                 ret = change_curseg(sbi, type);
3237         else
3238                 ret = new_curseg(sbi, type, true);
3239
3240         stat_inc_seg_type(sbi, curseg);
3241
3242         locate_dirty_segment(sbi, segno);
3243 unlock:
3244         up_write(&SIT_I(sbi)->sentry_lock);
3245
3246         if (segno != curseg->segno)
3247                 f2fs_notice(sbi, "For resize: curseg of type %d: %u ==> %u",
3248                             type, segno, curseg->segno);
3249
3250         mutex_unlock(&curseg->curseg_mutex);
3251         f2fs_up_read(&SM_I(sbi)->curseg_lock);
3252         return ret;
3253 }
3254
3255 static int __allocate_new_segment(struct f2fs_sb_info *sbi, int type,
3256                                                 bool new_sec, bool force)
3257 {
3258         struct curseg_info *curseg = CURSEG_I(sbi, type);
3259         unsigned int old_segno;
3260         int err = 0;
3261
3262         if (type == CURSEG_COLD_DATA_PINNED && !curseg->inited)
3263                 goto allocate;
3264
3265         if (!force && curseg->inited &&
3266             !curseg->next_blkoff &&
3267             !get_valid_blocks(sbi, curseg->segno, new_sec) &&
3268             !get_ckpt_valid_blocks(sbi, curseg->segno, new_sec))
3269                 return 0;
3270
3271 allocate:
3272         old_segno = curseg->segno;
3273         err = new_curseg(sbi, type, true);
3274         if (err)
3275                 return err;
3276         stat_inc_seg_type(sbi, curseg);
3277         locate_dirty_segment(sbi, old_segno);
3278         return 0;
3279 }
3280
3281 int f2fs_allocate_new_section(struct f2fs_sb_info *sbi, int type, bool force)
3282 {
3283         int ret;
3284
3285         f2fs_down_read(&SM_I(sbi)->curseg_lock);
3286         down_write(&SIT_I(sbi)->sentry_lock);
3287         ret = __allocate_new_segment(sbi, type, true, force);
3288         up_write(&SIT_I(sbi)->sentry_lock);
3289         f2fs_up_read(&SM_I(sbi)->curseg_lock);
3290
3291         return ret;
3292 }
3293
3294 int f2fs_allocate_pinning_section(struct f2fs_sb_info *sbi)
3295 {
3296         int err;
3297         bool gc_required = true;
3298
3299 retry:
3300         f2fs_lock_op(sbi);
3301         err = f2fs_allocate_new_section(sbi, CURSEG_COLD_DATA_PINNED, false);
3302         f2fs_unlock_op(sbi);
3303
3304         if (f2fs_sb_has_blkzoned(sbi) && err == -EAGAIN && gc_required) {
3305                 f2fs_down_write(&sbi->gc_lock);
3306                 err = f2fs_gc_range(sbi, 0, GET_SEGNO(sbi, FDEV(0).end_blk),
3307                                 true, ZONED_PIN_SEC_REQUIRED_COUNT);
3308                 f2fs_up_write(&sbi->gc_lock);
3309
3310                 gc_required = false;
3311                 if (!err)
3312                         goto retry;
3313         }
3314
3315         return err;
3316 }
3317
3318 int f2fs_allocate_new_segments(struct f2fs_sb_info *sbi)
3319 {
3320         int i;
3321         int err = 0;
3322
3323         f2fs_down_read(&SM_I(sbi)->curseg_lock);
3324         down_write(&SIT_I(sbi)->sentry_lock);
3325         for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++)
3326                 err += __allocate_new_segment(sbi, i, false, false);
3327         up_write(&SIT_I(sbi)->sentry_lock);
3328         f2fs_up_read(&SM_I(sbi)->curseg_lock);
3329
3330         return err;
3331 }
3332
3333 bool f2fs_exist_trim_candidates(struct f2fs_sb_info *sbi,
3334                                                 struct cp_control *cpc)
3335 {
3336         __u64 trim_start = cpc->trim_start;
3337         bool has_candidate = false;
3338
3339         down_write(&SIT_I(sbi)->sentry_lock);
3340         for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++) {
3341                 if (add_discard_addrs(sbi, cpc, true)) {
3342                         has_candidate = true;
3343                         break;
3344                 }
3345         }
3346         up_write(&SIT_I(sbi)->sentry_lock);
3347
3348         cpc->trim_start = trim_start;
3349         return has_candidate;
3350 }
3351
3352 static unsigned int __issue_discard_cmd_range(struct f2fs_sb_info *sbi,
3353                                         struct discard_policy *dpolicy,
3354                                         unsigned int start, unsigned int end)
3355 {
3356         struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
3357         struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
3358         struct rb_node **insert_p = NULL, *insert_parent = NULL;
3359         struct discard_cmd *dc;
3360         struct blk_plug plug;
3361         int issued;
3362         unsigned int trimmed = 0;
3363
3364 next:
3365         issued = 0;
3366
3367         mutex_lock(&dcc->cmd_lock);
3368         if (unlikely(dcc->rbtree_check))
3369                 f2fs_bug_on(sbi, !f2fs_check_discard_tree(sbi));
3370
3371         dc = __lookup_discard_cmd_ret(&dcc->root, start,
3372                                 &prev_dc, &next_dc, &insert_p, &insert_parent);
3373         if (!dc)
3374                 dc = next_dc;
3375
3376         blk_start_plug(&plug);
3377
3378         while (dc && dc->di.lstart <= end) {
3379                 struct rb_node *node;
3380                 int err = 0;
3381
3382                 if (dc->di.len < dpolicy->granularity)
3383                         goto skip;
3384
3385                 if (dc->state != D_PREP) {
3386                         list_move_tail(&dc->list, &dcc->fstrim_list);
3387                         goto skip;
3388                 }
3389
3390                 err = __submit_discard_cmd(sbi, dpolicy, dc, &issued);
3391
3392                 if (issued >= dpolicy->max_requests) {
3393                         start = dc->di.lstart + dc->di.len;
3394
3395                         if (err)
3396                                 __remove_discard_cmd(sbi, dc);
3397
3398                         blk_finish_plug(&plug);
3399                         mutex_unlock(&dcc->cmd_lock);
3400                         trimmed += __wait_all_discard_cmd(sbi, NULL);
3401                         f2fs_io_schedule_timeout(DEFAULT_IO_TIMEOUT);
3402                         goto next;
3403                 }
3404 skip:
3405                 node = rb_next(&dc->rb_node);
3406                 if (err)
3407                         __remove_discard_cmd(sbi, dc);
3408                 dc = rb_entry_safe(node, struct discard_cmd, rb_node);
3409
3410                 if (fatal_signal_pending(current))
3411                         break;
3412         }
3413
3414         blk_finish_plug(&plug);
3415         mutex_unlock(&dcc->cmd_lock);
3416
3417         return trimmed;
3418 }
3419
3420 int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range)
3421 {
3422         __u64 start = F2FS_BYTES_TO_BLK(range->start);
3423         __u64 end = start + F2FS_BYTES_TO_BLK(range->len) - 1;
3424         unsigned int start_segno, end_segno;
3425         block_t start_block, end_block;
3426         struct cp_control cpc;
3427         struct discard_policy dpolicy;
3428         unsigned long long trimmed = 0;
3429         int err = 0;
3430         bool need_align = f2fs_lfs_mode(sbi) && __is_large_section(sbi);
3431
3432         if (start >= MAX_BLKADDR(sbi) || range->len < sbi->blocksize)
3433                 return -EINVAL;
3434
3435         if (end < MAIN_BLKADDR(sbi))
3436                 goto out;
3437
3438         if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) {
3439                 f2fs_warn(sbi, "Found FS corruption, run fsck to fix.");
3440                 return -EFSCORRUPTED;
3441         }
3442
3443         /* start/end segment number in main_area */
3444         start_segno = (start <= MAIN_BLKADDR(sbi)) ? 0 : GET_SEGNO(sbi, start);
3445         end_segno = (end >= MAX_BLKADDR(sbi)) ? MAIN_SEGS(sbi) - 1 :
3446                                                 GET_SEGNO(sbi, end);
3447         if (need_align) {
3448                 start_segno = rounddown(start_segno, SEGS_PER_SEC(sbi));
3449                 end_segno = roundup(end_segno + 1, SEGS_PER_SEC(sbi)) - 1;
3450         }
3451
3452         cpc.reason = CP_DISCARD;
3453         cpc.trim_minlen = max_t(__u64, 1, F2FS_BYTES_TO_BLK(range->minlen));
3454         cpc.trim_start = start_segno;
3455         cpc.trim_end = end_segno;
3456
3457         if (sbi->discard_blks == 0)
3458                 goto out;
3459
3460         f2fs_down_write(&sbi->gc_lock);
3461         stat_inc_cp_call_count(sbi, TOTAL_CALL);
3462         err = f2fs_write_checkpoint(sbi, &cpc);
3463         f2fs_up_write(&sbi->gc_lock);
3464         if (err)
3465                 goto out;
3466
3467         /*
3468          * We filed discard candidates, but actually we don't need to wait for
3469          * all of them, since they'll be issued in idle time along with runtime
3470          * discard option. User configuration looks like using runtime discard
3471          * or periodic fstrim instead of it.
3472          */
3473         if (f2fs_realtime_discard_enable(sbi))
3474                 goto out;
3475
3476         start_block = START_BLOCK(sbi, start_segno);
3477         end_block = START_BLOCK(sbi, end_segno + 1);
3478
3479         __init_discard_policy(sbi, &dpolicy, DPOLICY_FSTRIM, cpc.trim_minlen);
3480         trimmed = __issue_discard_cmd_range(sbi, &dpolicy,
3481                                         start_block, end_block);
3482
3483         trimmed += __wait_discard_cmd_range(sbi, &dpolicy,
3484                                         start_block, end_block);
3485 out:
3486         if (!err)
3487                 range->len = F2FS_BLK_TO_BYTES(trimmed);
3488         return err;
3489 }
3490
3491 int f2fs_rw_hint_to_seg_type(struct f2fs_sb_info *sbi, enum rw_hint hint)
3492 {
3493         if (F2FS_OPTION(sbi).active_logs == 2)
3494                 return CURSEG_HOT_DATA;
3495         else if (F2FS_OPTION(sbi).active_logs == 4)
3496                 return CURSEG_COLD_DATA;
3497
3498         /* active_log == 6 */
3499         switch (hint) {
3500         case WRITE_LIFE_SHORT:
3501                 return CURSEG_HOT_DATA;
3502         case WRITE_LIFE_EXTREME:
3503                 return CURSEG_COLD_DATA;
3504         default:
3505                 return CURSEG_WARM_DATA;
3506         }
3507 }
3508
3509 /*
3510  * This returns write hints for each segment type. This hints will be
3511  * passed down to block layer as below by default.
3512  *
3513  * User                  F2FS                     Block
3514  * ----                  ----                     -----
3515  *                       META                     WRITE_LIFE_NONE|REQ_META
3516  *                       HOT_NODE                 WRITE_LIFE_NONE
3517  *                       WARM_NODE                WRITE_LIFE_MEDIUM
3518  *                       COLD_NODE                WRITE_LIFE_LONG
3519  * ioctl(COLD)           COLD_DATA                WRITE_LIFE_EXTREME
3520  * extension list        "                        "
3521  *
3522  * -- buffered io
3523  *                       COLD_DATA                WRITE_LIFE_EXTREME
3524  *                       HOT_DATA                 WRITE_LIFE_SHORT
3525  *                       WARM_DATA                WRITE_LIFE_NOT_SET
3526  *
3527  * -- direct io
3528  * WRITE_LIFE_EXTREME    COLD_DATA                WRITE_LIFE_EXTREME
3529  * WRITE_LIFE_SHORT      HOT_DATA                 WRITE_LIFE_SHORT
3530  * WRITE_LIFE_NOT_SET    WARM_DATA                WRITE_LIFE_NOT_SET
3531  * WRITE_LIFE_NONE       "                        WRITE_LIFE_NONE
3532  * WRITE_LIFE_MEDIUM     "                        WRITE_LIFE_MEDIUM
3533  * WRITE_LIFE_LONG       "                        WRITE_LIFE_LONG
3534  */
3535 enum rw_hint f2fs_io_type_to_rw_hint(struct f2fs_sb_info *sbi,
3536                                 enum page_type type, enum temp_type temp)
3537 {
3538         switch (type) {
3539         case DATA:
3540                 switch (temp) {
3541                 case WARM:
3542                         return WRITE_LIFE_NOT_SET;
3543                 case HOT:
3544                         return WRITE_LIFE_SHORT;
3545                 case COLD:
3546                         return WRITE_LIFE_EXTREME;
3547                 default:
3548                         return WRITE_LIFE_NONE;
3549                 }
3550         case NODE:
3551                 switch (temp) {
3552                 case WARM:
3553                         return WRITE_LIFE_MEDIUM;
3554                 case HOT:
3555                         return WRITE_LIFE_NONE;
3556                 case COLD:
3557                         return WRITE_LIFE_LONG;
3558                 default:
3559                         return WRITE_LIFE_NONE;
3560                 }
3561         case META:
3562                 return WRITE_LIFE_NONE;
3563         default:
3564                 return WRITE_LIFE_NONE;
3565         }
3566 }
3567
3568 static int __get_segment_type_2(struct f2fs_io_info *fio)
3569 {
3570         if (fio->type == DATA)
3571                 return CURSEG_HOT_DATA;
3572         else
3573                 return CURSEG_HOT_NODE;
3574 }
3575
3576 static int __get_segment_type_4(struct f2fs_io_info *fio)
3577 {
3578         if (fio->type == DATA) {
3579                 struct inode *inode = fio->page->mapping->host;
3580
3581                 if (S_ISDIR(inode->i_mode))
3582                         return CURSEG_HOT_DATA;
3583                 else
3584                         return CURSEG_COLD_DATA;
3585         } else {
3586                 if (IS_DNODE(fio->page) && is_cold_node(fio->page))
3587                         return CURSEG_WARM_NODE;
3588                 else
3589                         return CURSEG_COLD_NODE;
3590         }
3591 }
3592
3593 static int __get_age_segment_type(struct inode *inode, pgoff_t pgofs)
3594 {
3595         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3596         struct extent_info ei = {};
3597
3598         if (f2fs_lookup_age_extent_cache(inode, pgofs, &ei)) {
3599                 if (!ei.age)
3600                         return NO_CHECK_TYPE;
3601                 if (ei.age <= sbi->hot_data_age_threshold)
3602                         return CURSEG_HOT_DATA;
3603                 if (ei.age <= sbi->warm_data_age_threshold)
3604                         return CURSEG_WARM_DATA;
3605                 return CURSEG_COLD_DATA;
3606         }
3607         return NO_CHECK_TYPE;
3608 }
3609
3610 static int __get_segment_type_6(struct f2fs_io_info *fio)
3611 {
3612         if (fio->type == DATA) {
3613                 struct inode *inode = fio->page->mapping->host;
3614                 int type;
3615
3616                 if (is_inode_flag_set(inode, FI_ALIGNED_WRITE))
3617                         return CURSEG_COLD_DATA_PINNED;
3618
3619                 if (page_private_gcing(fio->page)) {
3620                         if (fio->sbi->am.atgc_enabled &&
3621                                 (fio->io_type == FS_DATA_IO) &&
3622                                 (fio->sbi->gc_mode != GC_URGENT_HIGH) &&
3623                                 __is_valid_data_blkaddr(fio->old_blkaddr) &&
3624                                 !is_inode_flag_set(inode, FI_OPU_WRITE))
3625                                 return CURSEG_ALL_DATA_ATGC;
3626                         else
3627                                 return CURSEG_COLD_DATA;
3628                 }
3629                 if (file_is_cold(inode) || f2fs_need_compress_data(inode))
3630                         return CURSEG_COLD_DATA;
3631
3632                 type = __get_age_segment_type(inode,
3633                                 page_folio(fio->page)->index);
3634                 if (type != NO_CHECK_TYPE)
3635                         return type;
3636
3637                 if (file_is_hot(inode) ||
3638                                 is_inode_flag_set(inode, FI_HOT_DATA) ||
3639                                 f2fs_is_cow_file(inode))
3640                         return CURSEG_HOT_DATA;
3641                 return f2fs_rw_hint_to_seg_type(F2FS_I_SB(inode),
3642                                                 inode->i_write_hint);
3643         } else {
3644                 if (IS_DNODE(fio->page))
3645                         return is_cold_node(fio->page) ? CURSEG_WARM_NODE :
3646                                                 CURSEG_HOT_NODE;
3647                 return CURSEG_COLD_NODE;
3648         }
3649 }
3650
3651 enum temp_type f2fs_get_segment_temp(struct f2fs_sb_info *sbi,
3652                                                 enum log_type type)
3653 {
3654         struct curseg_info *curseg = CURSEG_I(sbi, type);
3655         enum temp_type temp = COLD;
3656
3657         switch (curseg->seg_type) {
3658         case CURSEG_HOT_NODE:
3659         case CURSEG_HOT_DATA:
3660                 temp = HOT;
3661                 break;
3662         case CURSEG_WARM_NODE:
3663         case CURSEG_WARM_DATA:
3664                 temp = WARM;
3665                 break;
3666         case CURSEG_COLD_NODE:
3667         case CURSEG_COLD_DATA:
3668                 temp = COLD;
3669                 break;
3670         default:
3671                 f2fs_bug_on(sbi, 1);
3672         }
3673
3674         return temp;
3675 }
3676
3677 static int __get_segment_type(struct f2fs_io_info *fio)
3678 {
3679         enum log_type type = CURSEG_HOT_DATA;
3680
3681         switch (F2FS_OPTION(fio->sbi).active_logs) {
3682         case 2:
3683                 type = __get_segment_type_2(fio);
3684                 break;
3685         case 4:
3686                 type = __get_segment_type_4(fio);
3687                 break;
3688         case 6:
3689                 type = __get_segment_type_6(fio);
3690                 break;
3691         default:
3692                 f2fs_bug_on(fio->sbi, true);
3693         }
3694
3695         fio->temp = f2fs_get_segment_temp(fio->sbi, type);
3696
3697         return type;
3698 }
3699
3700 static void f2fs_randomize_chunk(struct f2fs_sb_info *sbi,
3701                 struct curseg_info *seg)
3702 {
3703         /* To allocate block chunks in different sizes, use random number */
3704         if (--seg->fragment_remained_chunk > 0)
3705                 return;
3706
3707         seg->fragment_remained_chunk =
3708                 get_random_u32_inclusive(1, sbi->max_fragment_chunk);
3709         seg->next_blkoff +=
3710                 get_random_u32_inclusive(1, sbi->max_fragment_hole);
3711 }
3712
3713 static void reset_curseg_fields(struct curseg_info *curseg)
3714 {
3715         curseg->inited = false;
3716         curseg->segno = NULL_SEGNO;
3717         curseg->next_segno = 0;
3718 }
3719
3720 int f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,
3721                 block_t old_blkaddr, block_t *new_blkaddr,
3722                 struct f2fs_summary *sum, int type,
3723                 struct f2fs_io_info *fio)
3724 {
3725         struct sit_info *sit_i = SIT_I(sbi);
3726         struct curseg_info *curseg = CURSEG_I(sbi, type);
3727         unsigned long long old_mtime;
3728         bool from_gc = (type == CURSEG_ALL_DATA_ATGC);
3729         struct seg_entry *se = NULL;
3730         bool segment_full = false;
3731         int ret = 0;
3732
3733         f2fs_down_read(&SM_I(sbi)->curseg_lock);
3734
3735         mutex_lock(&curseg->curseg_mutex);
3736         down_write(&sit_i->sentry_lock);
3737
3738         if (curseg->segno == NULL_SEGNO) {
3739                 ret = -ENOSPC;
3740                 goto out_err;
3741         }
3742
3743         if (from_gc) {
3744                 f2fs_bug_on(sbi, GET_SEGNO(sbi, old_blkaddr) == NULL_SEGNO);
3745                 se = get_seg_entry(sbi, GET_SEGNO(sbi, old_blkaddr));
3746                 sanity_check_seg_type(sbi, se->type);
3747                 f2fs_bug_on(sbi, IS_NODESEG(se->type));
3748         }
3749         *new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg);
3750
3751         f2fs_bug_on(sbi, curseg->next_blkoff >= BLKS_PER_SEG(sbi));
3752
3753         f2fs_wait_discard_bio(sbi, *new_blkaddr);
3754
3755         curseg->sum_blk->entries[curseg->next_blkoff] = *sum;
3756         if (curseg->alloc_type == SSR) {
3757                 curseg->next_blkoff = f2fs_find_next_ssr_block(sbi, curseg);
3758         } else {
3759                 curseg->next_blkoff++;
3760                 if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK)
3761                         f2fs_randomize_chunk(sbi, curseg);
3762         }
3763         if (curseg->next_blkoff >= f2fs_usable_blks_in_seg(sbi, curseg->segno))
3764                 segment_full = true;
3765         stat_inc_block_count(sbi, curseg);
3766
3767         if (from_gc) {
3768                 old_mtime = get_segment_mtime(sbi, old_blkaddr);
3769         } else {
3770                 update_segment_mtime(sbi, old_blkaddr, 0);
3771                 old_mtime = 0;
3772         }
3773         update_segment_mtime(sbi, *new_blkaddr, old_mtime);
3774
3775         /*
3776          * SIT information should be updated before segment allocation,
3777          * since SSR needs latest valid block information.
3778          */
3779         update_sit_entry(sbi, *new_blkaddr, 1);
3780         update_sit_entry(sbi, old_blkaddr, -1);
3781
3782         /*
3783          * If the current segment is full, flush it out and replace it with a
3784          * new segment.
3785          */
3786         if (segment_full) {
3787                 if (type == CURSEG_COLD_DATA_PINNED &&
3788                     !((curseg->segno + 1) % sbi->segs_per_sec)) {
3789                         write_sum_page(sbi, curseg->sum_blk,
3790                                         GET_SUM_BLOCK(sbi, curseg->segno));
3791                         reset_curseg_fields(curseg);
3792                         goto skip_new_segment;
3793                 }
3794
3795                 if (from_gc) {
3796                         ret = get_atssr_segment(sbi, type, se->type,
3797                                                 AT_SSR, se->mtime);
3798                 } else {
3799                         if (need_new_seg(sbi, type))
3800                                 ret = new_curseg(sbi, type, false);
3801                         else
3802                                 ret = change_curseg(sbi, type);
3803                         stat_inc_seg_type(sbi, curseg);
3804                 }
3805
3806                 if (ret)
3807                         goto out_err;
3808         }
3809
3810 skip_new_segment:
3811         /*
3812          * segment dirty status should be updated after segment allocation,
3813          * so we just need to update status only one time after previous
3814          * segment being closed.
3815          */
3816         locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));
3817         locate_dirty_segment(sbi, GET_SEGNO(sbi, *new_blkaddr));
3818
3819         if (IS_DATASEG(curseg->seg_type))
3820                 atomic64_inc(&sbi->allocated_data_blocks);
3821
3822         up_write(&sit_i->sentry_lock);
3823
3824         if (page && IS_NODESEG(curseg->seg_type)) {
3825                 fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg));
3826
3827                 f2fs_inode_chksum_set(sbi, page);
3828         }
3829
3830         if (fio) {
3831                 struct f2fs_bio_info *io;
3832
3833                 INIT_LIST_HEAD(&fio->list);
3834                 fio->in_list = 1;
3835                 io = sbi->write_io[fio->type] + fio->temp;
3836                 spin_lock(&io->io_lock);
3837                 list_add_tail(&fio->list, &io->io_list);
3838                 spin_unlock(&io->io_lock);
3839         }
3840
3841         mutex_unlock(&curseg->curseg_mutex);
3842         f2fs_up_read(&SM_I(sbi)->curseg_lock);
3843         return 0;
3844
3845 out_err:
3846         *new_blkaddr = NULL_ADDR;
3847         up_write(&sit_i->sentry_lock);
3848         mutex_unlock(&curseg->curseg_mutex);
3849         f2fs_up_read(&SM_I(sbi)->curseg_lock);
3850         return ret;
3851 }
3852
3853 void f2fs_update_device_state(struct f2fs_sb_info *sbi, nid_t ino,
3854                                         block_t blkaddr, unsigned int blkcnt)
3855 {
3856         if (!f2fs_is_multi_device(sbi))
3857                 return;
3858
3859         while (1) {
3860                 unsigned int devidx = f2fs_target_device_index(sbi, blkaddr);
3861                 unsigned int blks = FDEV(devidx).end_blk - blkaddr + 1;
3862
3863                 /* update device state for fsync */
3864                 f2fs_set_dirty_device(sbi, ino, devidx, FLUSH_INO);
3865
3866                 /* update device state for checkpoint */
3867                 if (!f2fs_test_bit(devidx, (char *)&sbi->dirty_device)) {
3868                         spin_lock(&sbi->dev_lock);
3869                         f2fs_set_bit(devidx, (char *)&sbi->dirty_device);
3870                         spin_unlock(&sbi->dev_lock);
3871                 }
3872
3873                 if (blkcnt <= blks)
3874                         break;
3875                 blkcnt -= blks;
3876                 blkaddr += blks;
3877         }
3878 }
3879
3880 static int log_type_to_seg_type(enum log_type type)
3881 {
3882         int seg_type = CURSEG_COLD_DATA;
3883
3884         switch (type) {
3885         case CURSEG_HOT_DATA:
3886         case CURSEG_WARM_DATA:
3887         case CURSEG_COLD_DATA:
3888         case CURSEG_HOT_NODE:
3889         case CURSEG_WARM_NODE:
3890         case CURSEG_COLD_NODE:
3891                 seg_type = (int)type;
3892                 break;
3893         case CURSEG_COLD_DATA_PINNED:
3894         case CURSEG_ALL_DATA_ATGC:
3895                 seg_type = CURSEG_COLD_DATA;
3896                 break;
3897         default:
3898                 break;
3899         }
3900         return seg_type;
3901 }
3902
3903 static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio)
3904 {
3905         enum log_type type = __get_segment_type(fio);
3906         int seg_type = log_type_to_seg_type(type);
3907         bool keep_order = (f2fs_lfs_mode(fio->sbi) &&
3908                                 seg_type == CURSEG_COLD_DATA);
3909
3910         if (keep_order)
3911                 f2fs_down_read(&fio->sbi->io_order_lock);
3912
3913         if (f2fs_allocate_data_block(fio->sbi, fio->page, fio->old_blkaddr,
3914                         &fio->new_blkaddr, sum, type, fio)) {
3915                 if (fscrypt_inode_uses_fs_layer_crypto(fio->page->mapping->host))
3916                         fscrypt_finalize_bounce_page(&fio->encrypted_page);
3917                 end_page_writeback(fio->page);
3918                 if (f2fs_in_warm_node_list(fio->sbi, fio->page))
3919                         f2fs_del_fsync_node_entry(fio->sbi, fio->page);
3920                 goto out;
3921         }
3922         if (GET_SEGNO(fio->sbi, fio->old_blkaddr) != NULL_SEGNO)
3923                 f2fs_invalidate_internal_cache(fio->sbi, fio->old_blkaddr, 1);
3924
3925         /* writeout dirty page into bdev */
3926         f2fs_submit_page_write(fio);
3927
3928         f2fs_update_device_state(fio->sbi, fio->ino, fio->new_blkaddr, 1);
3929 out:
3930         if (keep_order)
3931                 f2fs_up_read(&fio->sbi->io_order_lock);
3932 }
3933
3934 void f2fs_do_write_meta_page(struct f2fs_sb_info *sbi, struct folio *folio,
3935                                         enum iostat_type io_type)
3936 {
3937         struct f2fs_io_info fio = {
3938                 .sbi = sbi,
3939                 .type = META,
3940                 .temp = HOT,
3941                 .op = REQ_OP_WRITE,
3942                 .op_flags = REQ_SYNC | REQ_META | REQ_PRIO,
3943                 .old_blkaddr = folio->index,
3944                 .new_blkaddr = folio->index,
3945                 .page = folio_page(folio, 0),
3946                 .encrypted_page = NULL,
3947                 .in_list = 0,
3948         };
3949
3950         if (unlikely(folio->index >= MAIN_BLKADDR(sbi)))
3951                 fio.op_flags &= ~REQ_META;
3952
3953         folio_start_writeback(folio);
3954         f2fs_submit_page_write(&fio);
3955
3956         stat_inc_meta_count(sbi, folio->index);
3957         f2fs_update_iostat(sbi, NULL, io_type, F2FS_BLKSIZE);
3958 }
3959
3960 void f2fs_do_write_node_page(unsigned int nid, struct f2fs_io_info *fio)
3961 {
3962         struct f2fs_summary sum;
3963
3964         set_summary(&sum, nid, 0, 0);
3965         do_write_page(&sum, fio);
3966
3967         f2fs_update_iostat(fio->sbi, NULL, fio->io_type, F2FS_BLKSIZE);
3968 }
3969
3970 void f2fs_outplace_write_data(struct dnode_of_data *dn,
3971                                         struct f2fs_io_info *fio)
3972 {
3973         struct f2fs_sb_info *sbi = fio->sbi;
3974         struct f2fs_summary sum;
3975
3976         f2fs_bug_on(sbi, dn->data_blkaddr == NULL_ADDR);
3977         if (fio->io_type == FS_DATA_IO || fio->io_type == FS_CP_DATA_IO)
3978                 f2fs_update_age_extent_cache(dn);
3979         set_summary(&sum, dn->nid, dn->ofs_in_node, fio->version);
3980         do_write_page(&sum, fio);
3981         f2fs_update_data_blkaddr(dn, fio->new_blkaddr);
3982
3983         f2fs_update_iostat(sbi, dn->inode, fio->io_type, F2FS_BLKSIZE);
3984 }
3985
3986 int f2fs_inplace_write_data(struct f2fs_io_info *fio)
3987 {
3988         int err;
3989         struct f2fs_sb_info *sbi = fio->sbi;
3990         unsigned int segno;
3991
3992         fio->new_blkaddr = fio->old_blkaddr;
3993         /* i/o temperature is needed for passing down write hints */
3994         __get_segment_type(fio);
3995
3996         segno = GET_SEGNO(sbi, fio->new_blkaddr);
3997
3998         if (!IS_DATASEG(get_seg_entry(sbi, segno)->type)) {
3999                 set_sbi_flag(sbi, SBI_NEED_FSCK);
4000                 f2fs_warn(sbi, "%s: incorrect segment(%u) type, run fsck to fix.",
4001                           __func__, segno);
4002                 err = -EFSCORRUPTED;
4003                 f2fs_handle_error(sbi, ERROR_INCONSISTENT_SUM_TYPE);
4004                 goto drop_bio;
4005         }
4006
4007         if (f2fs_cp_error(sbi)) {
4008                 err = -EIO;
4009                 goto drop_bio;
4010         }
4011
4012         if (fio->meta_gc)
4013                 f2fs_truncate_meta_inode_pages(sbi, fio->new_blkaddr, 1);
4014
4015         stat_inc_inplace_blocks(fio->sbi);
4016
4017         if (fio->bio && !IS_F2FS_IPU_NOCACHE(sbi))
4018                 err = f2fs_merge_page_bio(fio);
4019         else
4020                 err = f2fs_submit_page_bio(fio);
4021         if (!err) {
4022                 f2fs_update_device_state(fio->sbi, fio->ino,
4023                                                 fio->new_blkaddr, 1);
4024                 f2fs_update_iostat(fio->sbi, fio->page->mapping->host,
4025                                                 fio->io_type, F2FS_BLKSIZE);
4026         }
4027
4028         return err;
4029 drop_bio:
4030         if (fio->bio && *(fio->bio)) {
4031                 struct bio *bio = *(fio->bio);
4032
4033                 bio->bi_status = BLK_STS_IOERR;
4034                 bio_endio(bio);
4035                 *(fio->bio) = NULL;
4036         }
4037         return err;
4038 }
4039
4040 static inline int __f2fs_get_curseg(struct f2fs_sb_info *sbi,
4041                                                 unsigned int segno)
4042 {
4043         int i;
4044
4045         for (i = CURSEG_HOT_DATA; i < NO_CHECK_TYPE; i++) {
4046                 if (CURSEG_I(sbi, i)->segno == segno)
4047                         break;
4048         }
4049         return i;
4050 }
4051
4052 void f2fs_do_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,
4053                                 block_t old_blkaddr, block_t new_blkaddr,
4054                                 bool recover_curseg, bool recover_newaddr,
4055                                 bool from_gc)
4056 {
4057         struct sit_info *sit_i = SIT_I(sbi);
4058         struct curseg_info *curseg;
4059         unsigned int segno, old_cursegno;
4060         struct seg_entry *se;
4061         int type;
4062         unsigned short old_blkoff;
4063         unsigned char old_alloc_type;
4064
4065         segno = GET_SEGNO(sbi, new_blkaddr);
4066         se = get_seg_entry(sbi, segno);
4067         type = se->type;
4068
4069         f2fs_down_write(&SM_I(sbi)->curseg_lock);
4070
4071         if (!recover_curseg) {
4072                 /* for recovery flow */
4073                 if (se->valid_blocks == 0 && !IS_CURSEG(sbi, segno)) {
4074                         if (old_blkaddr == NULL_ADDR)
4075                                 type = CURSEG_COLD_DATA;
4076                         else
4077                                 type = CURSEG_WARM_DATA;
4078                 }
4079         } else {
4080                 if (IS_CURSEG(sbi, segno)) {
4081                         /* se->type is volatile as SSR allocation */
4082                         type = __f2fs_get_curseg(sbi, segno);
4083                         f2fs_bug_on(sbi, type == NO_CHECK_TYPE);
4084                 } else {
4085                         type = CURSEG_WARM_DATA;
4086                 }
4087         }
4088
4089         curseg = CURSEG_I(sbi, type);
4090         f2fs_bug_on(sbi, !IS_DATASEG(curseg->seg_type));
4091
4092         mutex_lock(&curseg->curseg_mutex);
4093         down_write(&sit_i->sentry_lock);
4094
4095         old_cursegno = curseg->segno;
4096         old_blkoff = curseg->next_blkoff;
4097         old_alloc_type = curseg->alloc_type;
4098
4099         /* change the current segment */
4100         if (segno != curseg->segno) {
4101                 curseg->next_segno = segno;
4102                 if (change_curseg(sbi, type))
4103                         goto out_unlock;
4104         }
4105
4106         curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr);
4107         curseg->sum_blk->entries[curseg->next_blkoff] = *sum;
4108
4109         if (!recover_curseg || recover_newaddr) {
4110                 if (!from_gc)
4111                         update_segment_mtime(sbi, new_blkaddr, 0);
4112                 update_sit_entry(sbi, new_blkaddr, 1);
4113         }
4114         if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) {
4115                 f2fs_invalidate_internal_cache(sbi, old_blkaddr, 1);
4116                 if (!from_gc)
4117                         update_segment_mtime(sbi, old_blkaddr, 0);
4118                 update_sit_entry(sbi, old_blkaddr, -1);
4119         }
4120
4121         locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));
4122         locate_dirty_segment(sbi, GET_SEGNO(sbi, new_blkaddr));
4123
4124         locate_dirty_segment(sbi, old_cursegno);
4125
4126         if (recover_curseg) {
4127                 if (old_cursegno != curseg->segno) {
4128                         curseg->next_segno = old_cursegno;
4129                         if (change_curseg(sbi, type))
4130                                 goto out_unlock;
4131                 }
4132                 curseg->next_blkoff = old_blkoff;
4133                 curseg->alloc_type = old_alloc_type;
4134         }
4135
4136 out_unlock:
4137         up_write(&sit_i->sentry_lock);
4138         mutex_unlock(&curseg->curseg_mutex);
4139         f2fs_up_write(&SM_I(sbi)->curseg_lock);
4140 }
4141
4142 void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn,
4143                                 block_t old_addr, block_t new_addr,
4144                                 unsigned char version, bool recover_curseg,
4145                                 bool recover_newaddr)
4146 {
4147         struct f2fs_summary sum;
4148
4149         set_summary(&sum, dn->nid, dn->ofs_in_node, version);
4150
4151         f2fs_do_replace_block(sbi, &sum, old_addr, new_addr,
4152                                         recover_curseg, recover_newaddr, false);
4153
4154         f2fs_update_data_blkaddr(dn, new_addr);
4155 }
4156
4157 void f2fs_wait_on_page_writeback(struct page *page,
4158                                 enum page_type type, bool ordered, bool locked)
4159 {
4160         if (folio_test_writeback(page_folio(page))) {
4161                 struct f2fs_sb_info *sbi = F2FS_P_SB(page);
4162
4163                 /* submit cached LFS IO */
4164                 f2fs_submit_merged_write_cond(sbi, NULL, page, 0, type);
4165                 /* submit cached IPU IO */
4166                 f2fs_submit_merged_ipu_write(sbi, NULL, page);
4167                 if (ordered) {
4168                         wait_on_page_writeback(page);
4169                         f2fs_bug_on(sbi, locked &&
4170                                 folio_test_writeback(page_folio(page)));
4171                 } else {
4172                         wait_for_stable_page(page);
4173                 }
4174         }
4175 }
4176
4177 void f2fs_wait_on_block_writeback(struct inode *inode, block_t blkaddr)
4178 {
4179         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
4180         struct page *cpage;
4181
4182         if (!f2fs_meta_inode_gc_required(inode))
4183                 return;
4184
4185         if (!__is_valid_data_blkaddr(blkaddr))
4186                 return;
4187
4188         cpage = find_lock_page(META_MAPPING(sbi), blkaddr);
4189         if (cpage) {
4190                 f2fs_wait_on_page_writeback(cpage, DATA, true, true);
4191                 f2fs_put_page(cpage, 1);
4192         }
4193 }
4194
4195 void f2fs_wait_on_block_writeback_range(struct inode *inode, block_t blkaddr,
4196                                                                 block_t len)
4197 {
4198         struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
4199         block_t i;
4200
4201         if (!f2fs_meta_inode_gc_required(inode))
4202                 return;
4203
4204         for (i = 0; i < len; i++)
4205                 f2fs_wait_on_block_writeback(inode, blkaddr + i);
4206
4207         f2fs_truncate_meta_inode_pages(sbi, blkaddr, len);
4208 }
4209
4210 static int read_compacted_summaries(struct f2fs_sb_info *sbi)
4211 {
4212         struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
4213         struct curseg_info *seg_i;
4214         unsigned char *kaddr;
4215         struct page *page;
4216         block_t start;
4217         int i, j, offset;
4218
4219         start = start_sum_block(sbi);
4220
4221         page = f2fs_get_meta_page(sbi, start++);
4222         if (IS_ERR(page))
4223                 return PTR_ERR(page);
4224         kaddr = (unsigned char *)page_address(page);
4225
4226         /* Step 1: restore nat cache */
4227         seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
4228         memcpy(seg_i->journal, kaddr, SUM_JOURNAL_SIZE);
4229
4230         /* Step 2: restore sit cache */
4231         seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
4232         memcpy(seg_i->journal, kaddr + SUM_JOURNAL_SIZE, SUM_JOURNAL_SIZE);
4233         offset = 2 * SUM_JOURNAL_SIZE;
4234
4235         /* Step 3: restore summary entries */
4236         for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
4237                 unsigned short blk_off;
4238                 unsigned int segno;
4239
4240                 seg_i = CURSEG_I(sbi, i);
4241                 segno = le32_to_cpu(ckpt->cur_data_segno[i]);
4242                 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[i]);
4243                 seg_i->next_segno = segno;
4244                 reset_curseg(sbi, i, 0);
4245                 seg_i->alloc_type = ckpt->alloc_type[i];
4246                 seg_i->next_blkoff = blk_off;
4247
4248                 if (seg_i->alloc_type == SSR)
4249                         blk_off = BLKS_PER_SEG(sbi);
4250
4251                 for (j = 0; j < blk_off; j++) {
4252                         struct f2fs_summary *s;
4253
4254                         s = (struct f2fs_summary *)(kaddr + offset);
4255                         seg_i->sum_blk->entries[j] = *s;
4256                         offset += SUMMARY_SIZE;
4257                         if (offset + SUMMARY_SIZE <= PAGE_SIZE -
4258                                                 SUM_FOOTER_SIZE)
4259                                 continue;
4260
4261                         f2fs_put_page(page, 1);
4262                         page = NULL;
4263
4264                         page = f2fs_get_meta_page(sbi, start++);
4265                         if (IS_ERR(page))
4266                                 return PTR_ERR(page);
4267                         kaddr = (unsigned char *)page_address(page);
4268                         offset = 0;
4269                 }
4270         }
4271         f2fs_put_page(page, 1);
4272         return 0;
4273 }
4274
4275 static int read_normal_summaries(struct f2fs_sb_info *sbi, int type)
4276 {
4277         struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
4278         struct f2fs_summary_block *sum;
4279         struct curseg_info *curseg;
4280         struct page *new;
4281         unsigned short blk_off;
4282         unsigned int segno = 0;
4283         block_t blk_addr = 0;
4284         int err = 0;
4285
4286         /* get segment number and block addr */
4287         if (IS_DATASEG(type)) {
4288                 segno = le32_to_cpu(ckpt->cur_data_segno[type]);
4289                 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[type -
4290                                                         CURSEG_HOT_DATA]);
4291                 if (__exist_node_summaries(sbi))
4292                         blk_addr = sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type);
4293                 else
4294                         blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type);
4295         } else {
4296                 segno = le32_to_cpu(ckpt->cur_node_segno[type -
4297                                                         CURSEG_HOT_NODE]);
4298                 blk_off = le16_to_cpu(ckpt->cur_node_blkoff[type -
4299                                                         CURSEG_HOT_NODE]);
4300                 if (__exist_node_summaries(sbi))
4301                         blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE,
4302                                                         type - CURSEG_HOT_NODE);
4303                 else
4304                         blk_addr = GET_SUM_BLOCK(sbi, segno);
4305         }
4306
4307         new = f2fs_get_meta_page(sbi, blk_addr);
4308         if (IS_ERR(new))
4309                 return PTR_ERR(new);
4310         sum = (struct f2fs_summary_block *)page_address(new);
4311
4312         if (IS_NODESEG(type)) {
4313                 if (__exist_node_summaries(sbi)) {
4314                         struct f2fs_summary *ns = &sum->entries[0];
4315                         int i;
4316
4317                         for (i = 0; i < BLKS_PER_SEG(sbi); i++, ns++) {
4318                                 ns->version = 0;
4319                                 ns->ofs_in_node = 0;
4320                         }
4321                 } else {
4322                         err = f2fs_restore_node_summary(sbi, segno, sum);
4323                         if (err)
4324                                 goto out;
4325                 }
4326         }
4327
4328         /* set uncompleted segment to curseg */
4329         curseg = CURSEG_I(sbi, type);
4330         mutex_lock(&curseg->curseg_mutex);
4331
4332         /* update journal info */
4333         down_write(&curseg->journal_rwsem);
4334         memcpy(curseg->journal, &sum->journal, SUM_JOURNAL_SIZE);
4335         up_write(&curseg->journal_rwsem);
4336
4337         memcpy(curseg->sum_blk->entries, sum->entries, SUM_ENTRY_SIZE);
4338         memcpy(&curseg->sum_blk->footer, &sum->footer, SUM_FOOTER_SIZE);
4339         curseg->next_segno = segno;
4340         reset_curseg(sbi, type, 0);
4341         curseg->alloc_type = ckpt->alloc_type[type];
4342         curseg->next_blkoff = blk_off;
4343         mutex_unlock(&curseg->curseg_mutex);
4344 out:
4345         f2fs_put_page(new, 1);
4346         return err;
4347 }
4348
4349 static int restore_curseg_summaries(struct f2fs_sb_info *sbi)
4350 {
4351         struct f2fs_journal *sit_j = CURSEG_I(sbi, CURSEG_COLD_DATA)->journal;
4352         struct f2fs_journal *nat_j = CURSEG_I(sbi, CURSEG_HOT_DATA)->journal;
4353         int type = CURSEG_HOT_DATA;
4354         int err;
4355
4356         if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG)) {
4357                 int npages = f2fs_npages_for_summary_flush(sbi, true);
4358
4359                 if (npages >= 2)
4360                         f2fs_ra_meta_pages(sbi, start_sum_block(sbi), npages,
4361                                                         META_CP, true);
4362
4363                 /* restore for compacted data summary */
4364                 err = read_compacted_summaries(sbi);
4365                 if (err)
4366                         return err;
4367                 type = CURSEG_HOT_NODE;
4368         }
4369
4370         if (__exist_node_summaries(sbi))
4371                 f2fs_ra_meta_pages(sbi,
4372                                 sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type),
4373                                 NR_CURSEG_PERSIST_TYPE - type, META_CP, true);
4374
4375         for (; type <= CURSEG_COLD_NODE; type++) {
4376                 err = read_normal_summaries(sbi, type);
4377                 if (err)
4378                         return err;
4379         }
4380
4381         /* sanity check for summary blocks */
4382         if (nats_in_cursum(nat_j) > NAT_JOURNAL_ENTRIES ||
4383                         sits_in_cursum(sit_j) > SIT_JOURNAL_ENTRIES) {
4384                 f2fs_err(sbi, "invalid journal entries nats %u sits %u",
4385                          nats_in_cursum(nat_j), sits_in_cursum(sit_j));
4386                 return -EINVAL;
4387         }
4388
4389         return 0;
4390 }
4391
4392 static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr)
4393 {
4394         struct page *page;
4395         unsigned char *kaddr;
4396         struct f2fs_summary *summary;
4397         struct curseg_info *seg_i;
4398         int written_size = 0;
4399         int i, j;
4400
4401         page = f2fs_grab_meta_page(sbi, blkaddr++);
4402         kaddr = (unsigned char *)page_address(page);
4403         memset(kaddr, 0, PAGE_SIZE);
4404
4405         /* Step 1: write nat cache */
4406         seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
4407         memcpy(kaddr, seg_i->journal, SUM_JOURNAL_SIZE);
4408         written_size += SUM_JOURNAL_SIZE;
4409
4410         /* Step 2: write sit cache */
4411         seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
4412         memcpy(kaddr + written_size, seg_i->journal, SUM_JOURNAL_SIZE);
4413         written_size += SUM_JOURNAL_SIZE;
4414
4415         /* Step 3: write summary entries */
4416         for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
4417                 seg_i = CURSEG_I(sbi, i);
4418                 for (j = 0; j < f2fs_curseg_valid_blocks(sbi, i); j++) {
4419                         if (!page) {
4420                                 page = f2fs_grab_meta_page(sbi, blkaddr++);
4421                                 kaddr = (unsigned char *)page_address(page);
4422                                 memset(kaddr, 0, PAGE_SIZE);
4423                                 written_size = 0;
4424                         }
4425                         summary = (struct f2fs_summary *)(kaddr + written_size);
4426                         *summary = seg_i->sum_blk->entries[j];
4427                         written_size += SUMMARY_SIZE;
4428
4429                         if (written_size + SUMMARY_SIZE <= PAGE_SIZE -
4430                                                         SUM_FOOTER_SIZE)
4431                                 continue;
4432
4433                         set_page_dirty(page);
4434                         f2fs_put_page(page, 1);
4435                         page = NULL;
4436                 }
4437         }
4438         if (page) {
4439                 set_page_dirty(page);
4440                 f2fs_put_page(page, 1);
4441         }
4442 }
4443
4444 static void write_normal_summaries(struct f2fs_sb_info *sbi,
4445                                         block_t blkaddr, int type)
4446 {
4447         int i, end;
4448
4449         if (IS_DATASEG(type))
4450                 end = type + NR_CURSEG_DATA_TYPE;
4451         else
4452                 end = type + NR_CURSEG_NODE_TYPE;
4453
4454         for (i = type; i < end; i++)
4455                 write_current_sum_page(sbi, i, blkaddr + (i - type));
4456 }
4457
4458 void f2fs_write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
4459 {
4460         if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG))
4461                 write_compacted_summaries(sbi, start_blk);
4462         else
4463                 write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA);
4464 }
4465
4466 void f2fs_write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
4467 {
4468         write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE);
4469 }
4470
4471 int f2fs_lookup_journal_in_cursum(struct f2fs_journal *journal, int type,
4472                                         unsigned int val, int alloc)
4473 {
4474         int i;
4475
4476         if (type == NAT_JOURNAL) {
4477                 for (i = 0; i < nats_in_cursum(journal); i++) {
4478                         if (le32_to_cpu(nid_in_journal(journal, i)) == val)
4479                                 return i;
4480                 }
4481                 if (alloc && __has_cursum_space(journal, 1, NAT_JOURNAL))
4482                         return update_nats_in_cursum(journal, 1);
4483         } else if (type == SIT_JOURNAL) {
4484                 for (i = 0; i < sits_in_cursum(journal); i++)
4485                         if (le32_to_cpu(segno_in_journal(journal, i)) == val)
4486                                 return i;
4487                 if (alloc && __has_cursum_space(journal, 1, SIT_JOURNAL))
4488                         return update_sits_in_cursum(journal, 1);
4489         }
4490         return -1;
4491 }
4492
4493 static struct page *get_current_sit_page(struct f2fs_sb_info *sbi,
4494                                         unsigned int segno)
4495 {
4496         return f2fs_get_meta_page(sbi, current_sit_addr(sbi, segno));
4497 }
4498
4499 static struct page *get_next_sit_page(struct f2fs_sb_info *sbi,
4500                                         unsigned int start)
4501 {
4502         struct sit_info *sit_i = SIT_I(sbi);
4503         struct page *page;
4504         pgoff_t src_off, dst_off;
4505
4506         src_off = current_sit_addr(sbi, start);
4507         dst_off = next_sit_addr(sbi, src_off);
4508
4509         page = f2fs_grab_meta_page(sbi, dst_off);
4510         seg_info_to_sit_page(sbi, page, start);
4511
4512         set_page_dirty(page);
4513         set_to_next_sit(sit_i, start);
4514
4515         return page;
4516 }
4517
4518 static struct sit_entry_set *grab_sit_entry_set(void)
4519 {
4520         struct sit_entry_set *ses =
4521                         f2fs_kmem_cache_alloc(sit_entry_set_slab,
4522                                                 GFP_NOFS, true, NULL);
4523
4524         ses->entry_cnt = 0;
4525         INIT_LIST_HEAD(&ses->set_list);
4526         return ses;
4527 }
4528
4529 static void release_sit_entry_set(struct sit_entry_set *ses)
4530 {
4531         list_del(&ses->set_list);
4532         kmem_cache_free(sit_entry_set_slab, ses);
4533 }
4534
4535 static void adjust_sit_entry_set(struct sit_entry_set *ses,
4536                                                 struct list_head *head)
4537 {
4538         struct sit_entry_set *next = ses;
4539
4540         if (list_is_last(&ses->set_list, head))
4541                 return;
4542
4543         list_for_each_entry_continue(next, head, set_list)
4544                 if (ses->entry_cnt <= next->entry_cnt) {
4545                         list_move_tail(&ses->set_list, &next->set_list);
4546                         return;
4547                 }
4548
4549         list_move_tail(&ses->set_list, head);
4550 }
4551
4552 static void add_sit_entry(unsigned int segno, struct list_head *head)
4553 {
4554         struct sit_entry_set *ses;
4555         unsigned int start_segno = START_SEGNO(segno);
4556
4557         list_for_each_entry(ses, head, set_list) {
4558                 if (ses->start_segno == start_segno) {
4559                         ses->entry_cnt++;
4560                         adjust_sit_entry_set(ses, head);
4561                         return;
4562                 }
4563         }
4564
4565         ses = grab_sit_entry_set();
4566
4567         ses->start_segno = start_segno;
4568         ses->entry_cnt++;
4569         list_add(&ses->set_list, head);
4570 }
4571
4572 static void add_sits_in_set(struct f2fs_sb_info *sbi)
4573 {
4574         struct f2fs_sm_info *sm_info = SM_I(sbi);
4575         struct list_head *set_list = &sm_info->sit_entry_set;
4576         unsigned long *bitmap = SIT_I(sbi)->dirty_sentries_bitmap;
4577         unsigned int segno;
4578
4579         for_each_set_bit(segno, bitmap, MAIN_SEGS(sbi))
4580                 add_sit_entry(segno, set_list);
4581 }
4582
4583 static void remove_sits_in_journal(struct f2fs_sb_info *sbi)
4584 {
4585         struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
4586         struct f2fs_journal *journal = curseg->journal;
4587         int i;
4588
4589         down_write(&curseg->journal_rwsem);
4590         for (i = 0; i < sits_in_cursum(journal); i++) {
4591                 unsigned int segno;
4592                 bool dirtied;
4593
4594                 segno = le32_to_cpu(segno_in_journal(journal, i));
4595                 dirtied = __mark_sit_entry_dirty(sbi, segno);
4596
4597                 if (!dirtied)
4598                         add_sit_entry(segno, &SM_I(sbi)->sit_entry_set);
4599         }
4600         update_sits_in_cursum(journal, -i);
4601         up_write(&curseg->journal_rwsem);
4602 }
4603
4604 /*
4605  * CP calls this function, which flushes SIT entries including sit_journal,
4606  * and moves prefree segs to free segs.
4607  */
4608 void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc)
4609 {
4610         struct sit_info *sit_i = SIT_I(sbi);
4611         unsigned long *bitmap = sit_i->dirty_sentries_bitmap;
4612         struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
4613         struct f2fs_journal *journal = curseg->journal;
4614         struct sit_entry_set *ses, *tmp;
4615         struct list_head *head = &SM_I(sbi)->sit_entry_set;
4616         bool to_journal = !is_sbi_flag_set(sbi, SBI_IS_RESIZEFS);
4617         struct seg_entry *se;
4618
4619         down_write(&sit_i->sentry_lock);
4620
4621         if (!sit_i->dirty_sentries)
4622                 goto out;
4623
4624         /*
4625          * add and account sit entries of dirty bitmap in sit entry
4626          * set temporarily
4627          */
4628         add_sits_in_set(sbi);
4629
4630         /*
4631          * if there are no enough space in journal to store dirty sit
4632          * entries, remove all entries from journal and add and account
4633          * them in sit entry set.
4634          */
4635         if (!__has_cursum_space(journal, sit_i->dirty_sentries, SIT_JOURNAL) ||
4636                                                                 !to_journal)
4637                 remove_sits_in_journal(sbi);
4638
4639         /*
4640          * there are two steps to flush sit entries:
4641          * #1, flush sit entries to journal in current cold data summary block.
4642          * #2, flush sit entries to sit page.
4643          */
4644         list_for_each_entry_safe(ses, tmp, head, set_list) {
4645                 struct page *page = NULL;
4646                 struct f2fs_sit_block *raw_sit = NULL;
4647                 unsigned int start_segno = ses->start_segno;
4648                 unsigned int end = min(start_segno + SIT_ENTRY_PER_BLOCK,
4649                                                 (unsigned long)MAIN_SEGS(sbi));
4650                 unsigned int segno = start_segno;
4651
4652                 if (to_journal &&
4653                         !__has_cursum_space(journal, ses->entry_cnt, SIT_JOURNAL))
4654                         to_journal = false;
4655
4656                 if (to_journal) {
4657                         down_write(&curseg->journal_rwsem);
4658                 } else {
4659                         page = get_next_sit_page(sbi, start_segno);
4660                         raw_sit = page_address(page);
4661                 }
4662
4663                 /* flush dirty sit entries in region of current sit set */
4664                 for_each_set_bit_from(segno, bitmap, end) {
4665                         int offset, sit_offset;
4666
4667                         se = get_seg_entry(sbi, segno);
4668 #ifdef CONFIG_F2FS_CHECK_FS
4669                         if (memcmp(se->cur_valid_map, se->cur_valid_map_mir,
4670                                                 SIT_VBLOCK_MAP_SIZE))
4671                                 f2fs_bug_on(sbi, 1);
4672 #endif
4673
4674                         /* add discard candidates */
4675                         if (!(cpc->reason & CP_DISCARD)) {
4676                                 cpc->trim_start = segno;
4677                                 add_discard_addrs(sbi, cpc, false);
4678                         }
4679
4680                         if (to_journal) {
4681                                 offset = f2fs_lookup_journal_in_cursum(journal,
4682                                                         SIT_JOURNAL, segno, 1);
4683                                 f2fs_bug_on(sbi, offset < 0);
4684                                 segno_in_journal(journal, offset) =
4685                                                         cpu_to_le32(segno);
4686                                 seg_info_to_raw_sit(se,
4687                                         &sit_in_journal(journal, offset));
4688                                 check_block_count(sbi, segno,
4689                                         &sit_in_journal(journal, offset));
4690                         } else {
4691                                 sit_offset = SIT_ENTRY_OFFSET(sit_i, segno);
4692                                 seg_info_to_raw_sit(se,
4693                                                 &raw_sit->entries[sit_offset]);
4694                                 check_block_count(sbi, segno,
4695                                                 &raw_sit->entries[sit_offset]);
4696                         }
4697
4698                         __clear_bit(segno, bitmap);
4699                         sit_i->dirty_sentries--;
4700                         ses->entry_cnt--;
4701                 }
4702
4703                 if (to_journal)
4704                         up_write(&curseg->journal_rwsem);
4705                 else
4706                         f2fs_put_page(page, 1);
4707
4708                 f2fs_bug_on(sbi, ses->entry_cnt);
4709                 release_sit_entry_set(ses);
4710         }
4711
4712         f2fs_bug_on(sbi, !list_empty(head));
4713         f2fs_bug_on(sbi, sit_i->dirty_sentries);
4714 out:
4715         if (cpc->reason & CP_DISCARD) {
4716                 __u64 trim_start = cpc->trim_start;
4717
4718                 for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++)
4719                         add_discard_addrs(sbi, cpc, false);
4720
4721                 cpc->trim_start = trim_start;
4722         }
4723         up_write(&sit_i->sentry_lock);
4724
4725         set_prefree_as_free_segments(sbi);
4726 }
4727
4728 static int build_sit_info(struct f2fs_sb_info *sbi)
4729 {
4730         struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
4731         struct sit_info *sit_i;
4732         unsigned int sit_segs, start;
4733         char *src_bitmap, *bitmap;
4734         unsigned int bitmap_size, main_bitmap_size, sit_bitmap_size;
4735         unsigned int discard_map = f2fs_block_unit_discard(sbi) ? 1 : 0;
4736
4737         /* allocate memory for SIT information */
4738         sit_i = f2fs_kzalloc(sbi, sizeof(struct sit_info), GFP_KERNEL);
4739         if (!sit_i)
4740                 return -ENOMEM;
4741
4742         SM_I(sbi)->sit_info = sit_i;
4743
4744         sit_i->sentries =
4745                 f2fs_kvzalloc(sbi, array_size(sizeof(struct seg_entry),
4746                                               MAIN_SEGS(sbi)),
4747                               GFP_KERNEL);
4748         if (!sit_i->sentries)
4749                 return -ENOMEM;
4750
4751         main_bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
4752         sit_i->dirty_sentries_bitmap = f2fs_kvzalloc(sbi, main_bitmap_size,
4753                                                                 GFP_KERNEL);
4754         if (!sit_i->dirty_sentries_bitmap)
4755                 return -ENOMEM;
4756
4757 #ifdef CONFIG_F2FS_CHECK_FS
4758         bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * (3 + discard_map);
4759 #else
4760         bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * (2 + discard_map);
4761 #endif
4762         sit_i->bitmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);
4763         if (!sit_i->bitmap)
4764                 return -ENOMEM;
4765
4766         bitmap = sit_i->bitmap;
4767
4768         for (start = 0; start < MAIN_SEGS(sbi); start++) {
4769                 sit_i->sentries[start].cur_valid_map = bitmap;
4770                 bitmap += SIT_VBLOCK_MAP_SIZE;
4771
4772                 sit_i->sentries[start].ckpt_valid_map = bitmap;
4773                 bitmap += SIT_VBLOCK_MAP_SIZE;
4774
4775 #ifdef CONFIG_F2FS_CHECK_FS
4776                 sit_i->sentries[start].cur_valid_map_mir = bitmap;
4777                 bitmap += SIT_VBLOCK_MAP_SIZE;
4778 #endif
4779
4780                 if (discard_map) {
4781                         sit_i->sentries[start].discard_map = bitmap;
4782                         bitmap += SIT_VBLOCK_MAP_SIZE;
4783                 }
4784         }
4785
4786         sit_i->tmp_map = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
4787         if (!sit_i->tmp_map)
4788                 return -ENOMEM;
4789
4790         if (__is_large_section(sbi)) {
4791                 sit_i->sec_entries =
4792                         f2fs_kvzalloc(sbi, array_size(sizeof(struct sec_entry),
4793                                                       MAIN_SECS(sbi)),
4794                                       GFP_KERNEL);
4795                 if (!sit_i->sec_entries)
4796                         return -ENOMEM;
4797         }
4798
4799         /* get information related with SIT */
4800         sit_segs = le32_to_cpu(raw_super->segment_count_sit) >> 1;
4801
4802         /* setup SIT bitmap from ckeckpoint pack */
4803         sit_bitmap_size = __bitmap_size(sbi, SIT_BITMAP);
4804         src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP);
4805
4806         sit_i->sit_bitmap = kmemdup(src_bitmap, sit_bitmap_size, GFP_KERNEL);
4807         if (!sit_i->sit_bitmap)
4808                 return -ENOMEM;
4809
4810 #ifdef CONFIG_F2FS_CHECK_FS
4811         sit_i->sit_bitmap_mir = kmemdup(src_bitmap,
4812                                         sit_bitmap_size, GFP_KERNEL);
4813         if (!sit_i->sit_bitmap_mir)
4814                 return -ENOMEM;
4815
4816         sit_i->invalid_segmap = f2fs_kvzalloc(sbi,
4817                                         main_bitmap_size, GFP_KERNEL);
4818         if (!sit_i->invalid_segmap)
4819                 return -ENOMEM;
4820 #endif
4821
4822         sit_i->sit_base_addr = le32_to_cpu(raw_super->sit_blkaddr);
4823         sit_i->sit_blocks = SEGS_TO_BLKS(sbi, sit_segs);
4824         sit_i->written_valid_blocks = 0;
4825         sit_i->bitmap_size = sit_bitmap_size;
4826         sit_i->dirty_sentries = 0;
4827         sit_i->sents_per_block = SIT_ENTRY_PER_BLOCK;
4828         sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time);
4829         sit_i->mounted_time = ktime_get_boottime_seconds();
4830         init_rwsem(&sit_i->sentry_lock);
4831         return 0;
4832 }
4833
4834 static int build_free_segmap(struct f2fs_sb_info *sbi)
4835 {
4836         struct free_segmap_info *free_i;
4837         unsigned int bitmap_size, sec_bitmap_size;
4838
4839         /* allocate memory for free segmap information */
4840         free_i = f2fs_kzalloc(sbi, sizeof(struct free_segmap_info), GFP_KERNEL);
4841         if (!free_i)
4842                 return -ENOMEM;
4843
4844         SM_I(sbi)->free_info = free_i;
4845
4846         bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
4847         free_i->free_segmap = f2fs_kvmalloc(sbi, bitmap_size, GFP_KERNEL);
4848         if (!free_i->free_segmap)
4849                 return -ENOMEM;
4850
4851         sec_bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
4852         free_i->free_secmap = f2fs_kvmalloc(sbi, sec_bitmap_size, GFP_KERNEL);
4853         if (!free_i->free_secmap)
4854                 return -ENOMEM;
4855
4856         /* set all segments as dirty temporarily */
4857         memset(free_i->free_segmap, 0xff, bitmap_size);
4858         memset(free_i->free_secmap, 0xff, sec_bitmap_size);
4859
4860         /* init free segmap information */
4861         free_i->start_segno = GET_SEGNO_FROM_SEG0(sbi, MAIN_BLKADDR(sbi));
4862         free_i->free_segments = 0;
4863         free_i->free_sections = 0;
4864         spin_lock_init(&free_i->segmap_lock);
4865         return 0;
4866 }
4867
4868 static int build_curseg(struct f2fs_sb_info *sbi)
4869 {
4870         struct curseg_info *array;
4871         int i;
4872
4873         array = f2fs_kzalloc(sbi, array_size(NR_CURSEG_TYPE,
4874                                         sizeof(*array)), GFP_KERNEL);
4875         if (!array)
4876                 return -ENOMEM;
4877
4878         SM_I(sbi)->curseg_array = array;
4879
4880         for (i = 0; i < NO_CHECK_TYPE; i++) {
4881                 mutex_init(&array[i].curseg_mutex);
4882                 array[i].sum_blk = f2fs_kzalloc(sbi, PAGE_SIZE, GFP_KERNEL);
4883                 if (!array[i].sum_blk)
4884                         return -ENOMEM;
4885                 init_rwsem(&array[i].journal_rwsem);
4886                 array[i].journal = f2fs_kzalloc(sbi,
4887                                 sizeof(struct f2fs_journal), GFP_KERNEL);
4888                 if (!array[i].journal)
4889                         return -ENOMEM;
4890                 array[i].seg_type = log_type_to_seg_type(i);
4891                 reset_curseg_fields(&array[i]);
4892         }
4893         return restore_curseg_summaries(sbi);
4894 }
4895
4896 static int build_sit_entries(struct f2fs_sb_info *sbi)
4897 {
4898         struct sit_info *sit_i = SIT_I(sbi);
4899         struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
4900         struct f2fs_journal *journal = curseg->journal;
4901         struct seg_entry *se;
4902         struct f2fs_sit_entry sit;
4903         int sit_blk_cnt = SIT_BLK_CNT(sbi);
4904         unsigned int i, start, end;
4905         unsigned int readed, start_blk = 0;
4906         int err = 0;
4907         block_t sit_valid_blocks[2] = {0, 0};
4908
4909         do {
4910                 readed = f2fs_ra_meta_pages(sbi, start_blk, BIO_MAX_VECS,
4911                                                         META_SIT, true);
4912
4913                 start = start_blk * sit_i->sents_per_block;
4914                 end = (start_blk + readed) * sit_i->sents_per_block;
4915
4916                 for (; start < end && start < MAIN_SEGS(sbi); start++) {
4917                         struct f2fs_sit_block *sit_blk;
4918                         struct page *page;
4919
4920                         se = &sit_i->sentries[start];
4921                         page = get_current_sit_page(sbi, start);
4922                         if (IS_ERR(page))
4923                                 return PTR_ERR(page);
4924                         sit_blk = (struct f2fs_sit_block *)page_address(page);
4925                         sit = sit_blk->entries[SIT_ENTRY_OFFSET(sit_i, start)];
4926                         f2fs_put_page(page, 1);
4927
4928                         err = check_block_count(sbi, start, &sit);
4929                         if (err)
4930                                 return err;
4931                         seg_info_from_raw_sit(se, &sit);
4932
4933                         if (se->type >= NR_PERSISTENT_LOG) {
4934                                 f2fs_err(sbi, "Invalid segment type: %u, segno: %u",
4935                                                         se->type, start);
4936                                 f2fs_handle_error(sbi,
4937                                                 ERROR_INCONSISTENT_SUM_TYPE);
4938                                 return -EFSCORRUPTED;
4939                         }
4940
4941                         sit_valid_blocks[SE_PAGETYPE(se)] += se->valid_blocks;
4942
4943                         if (!f2fs_block_unit_discard(sbi))
4944                                 goto init_discard_map_done;
4945
4946                         /* build discard map only one time */
4947                         if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) {
4948                                 memset(se->discard_map, 0xff,
4949                                                 SIT_VBLOCK_MAP_SIZE);
4950                                 goto init_discard_map_done;
4951                         }
4952                         memcpy(se->discard_map, se->cur_valid_map,
4953                                                 SIT_VBLOCK_MAP_SIZE);
4954                         sbi->discard_blks += BLKS_PER_SEG(sbi) -
4955                                                 se->valid_blocks;
4956 init_discard_map_done:
4957                         if (__is_large_section(sbi))
4958                                 get_sec_entry(sbi, start)->valid_blocks +=
4959                                                         se->valid_blocks;
4960                 }
4961                 start_blk += readed;
4962         } while (start_blk < sit_blk_cnt);
4963
4964         down_read(&curseg->journal_rwsem);
4965         for (i = 0; i < sits_in_cursum(journal); i++) {
4966                 unsigned int old_valid_blocks;
4967
4968                 start = le32_to_cpu(segno_in_journal(journal, i));
4969                 if (start >= MAIN_SEGS(sbi)) {
4970                         f2fs_err(sbi, "Wrong journal entry on segno %u",
4971                                  start);
4972                         err = -EFSCORRUPTED;
4973                         f2fs_handle_error(sbi, ERROR_CORRUPTED_JOURNAL);
4974                         break;
4975                 }
4976
4977                 se = &sit_i->sentries[start];
4978                 sit = sit_in_journal(journal, i);
4979
4980                 old_valid_blocks = se->valid_blocks;
4981
4982                 sit_valid_blocks[SE_PAGETYPE(se)] -= old_valid_blocks;
4983
4984                 err = check_block_count(sbi, start, &sit);
4985                 if (err)
4986                         break;
4987                 seg_info_from_raw_sit(se, &sit);
4988
4989                 if (se->type >= NR_PERSISTENT_LOG) {
4990                         f2fs_err(sbi, "Invalid segment type: %u, segno: %u",
4991                                                         se->type, start);
4992                         err = -EFSCORRUPTED;
4993                         f2fs_handle_error(sbi, ERROR_INCONSISTENT_SUM_TYPE);
4994                         break;
4995                 }
4996
4997                 sit_valid_blocks[SE_PAGETYPE(se)] += se->valid_blocks;
4998
4999                 if (f2fs_block_unit_discard(sbi)) {
5000                         if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) {
5001                                 memset(se->discard_map, 0xff, SIT_VBLOCK_MAP_SIZE);
5002                         } else {
5003                                 memcpy(se->discard_map, se->cur_valid_map,
5004                                                         SIT_VBLOCK_MAP_SIZE);
5005                                 sbi->discard_blks += old_valid_blocks;
5006                                 sbi->discard_blks -= se->valid_blocks;
5007                         }
5008                 }
5009
5010                 if (__is_large_section(sbi)) {
5011                         get_sec_entry(sbi, start)->valid_blocks +=
5012                                                         se->valid_blocks;
5013                         get_sec_entry(sbi, start)->valid_blocks -=
5014                                                         old_valid_blocks;
5015                 }
5016         }
5017         up_read(&curseg->journal_rwsem);
5018
5019         if (err)
5020                 return err;
5021
5022         if (sit_valid_blocks[NODE] != valid_node_count(sbi)) {
5023                 f2fs_err(sbi, "SIT is corrupted node# %u vs %u",
5024                          sit_valid_blocks[NODE], valid_node_count(sbi));
5025                 f2fs_handle_error(sbi, ERROR_INCONSISTENT_NODE_COUNT);
5026                 return -EFSCORRUPTED;
5027         }
5028
5029         if (sit_valid_blocks[DATA] + sit_valid_blocks[NODE] >
5030                                 valid_user_blocks(sbi)) {
5031                 f2fs_err(sbi, "SIT is corrupted data# %u %u vs %u",
5032                          sit_valid_blocks[DATA], sit_valid_blocks[NODE],
5033                          valid_user_blocks(sbi));
5034                 f2fs_handle_error(sbi, ERROR_INCONSISTENT_BLOCK_COUNT);
5035                 return -EFSCORRUPTED;
5036         }
5037
5038         return 0;
5039 }
5040
5041 static void init_free_segmap(struct f2fs_sb_info *sbi)
5042 {
5043         unsigned int start;
5044         int type;
5045         struct seg_entry *sentry;
5046
5047         for (start = 0; start < MAIN_SEGS(sbi); start++) {
5048                 if (f2fs_usable_blks_in_seg(sbi, start) == 0)
5049                         continue;
5050                 sentry = get_seg_entry(sbi, start);
5051                 if (!sentry->valid_blocks)
5052                         __set_free(sbi, start);
5053                 else
5054                         SIT_I(sbi)->written_valid_blocks +=
5055                                                 sentry->valid_blocks;
5056         }
5057
5058         /* set use the current segments */
5059         for (type = CURSEG_HOT_DATA; type <= CURSEG_COLD_NODE; type++) {
5060                 struct curseg_info *curseg_t = CURSEG_I(sbi, type);
5061
5062                 __set_test_and_inuse(sbi, curseg_t->segno);
5063         }
5064 }
5065
5066 static void init_dirty_segmap(struct f2fs_sb_info *sbi)
5067 {
5068         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
5069         struct free_segmap_info *free_i = FREE_I(sbi);
5070         unsigned int segno = 0, offset = 0, secno;
5071         block_t valid_blocks, usable_blks_in_seg;
5072
5073         while (1) {
5074                 /* find dirty segment based on free segmap */
5075                 segno = find_next_inuse(free_i, MAIN_SEGS(sbi), offset);
5076                 if (segno >= MAIN_SEGS(sbi))
5077                         break;
5078                 offset = segno + 1;
5079                 valid_blocks = get_valid_blocks(sbi, segno, false);
5080                 usable_blks_in_seg = f2fs_usable_blks_in_seg(sbi, segno);
5081                 if (valid_blocks == usable_blks_in_seg || !valid_blocks)
5082                         continue;
5083                 if (valid_blocks > usable_blks_in_seg) {
5084                         f2fs_bug_on(sbi, 1);
5085                         continue;
5086                 }
5087                 mutex_lock(&dirty_i->seglist_lock);
5088                 __locate_dirty_segment(sbi, segno, DIRTY);
5089                 mutex_unlock(&dirty_i->seglist_lock);
5090         }
5091
5092         if (!__is_large_section(sbi))
5093                 return;
5094
5095         mutex_lock(&dirty_i->seglist_lock);
5096         for (segno = 0; segno < MAIN_SEGS(sbi); segno += SEGS_PER_SEC(sbi)) {
5097                 valid_blocks = get_valid_blocks(sbi, segno, true);
5098                 secno = GET_SEC_FROM_SEG(sbi, segno);
5099
5100                 if (!valid_blocks || valid_blocks == CAP_BLKS_PER_SEC(sbi))
5101                         continue;
5102                 if (IS_CURSEC(sbi, secno))
5103                         continue;
5104                 set_bit(secno, dirty_i->dirty_secmap);
5105         }
5106         mutex_unlock(&dirty_i->seglist_lock);
5107 }
5108
5109 static int init_victim_secmap(struct f2fs_sb_info *sbi)
5110 {
5111         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
5112         unsigned int bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
5113
5114         dirty_i->victim_secmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);
5115         if (!dirty_i->victim_secmap)
5116                 return -ENOMEM;
5117
5118         dirty_i->pinned_secmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);
5119         if (!dirty_i->pinned_secmap)
5120                 return -ENOMEM;
5121
5122         dirty_i->pinned_secmap_cnt = 0;
5123         dirty_i->enable_pin_section = true;
5124         return 0;
5125 }
5126
5127 static int build_dirty_segmap(struct f2fs_sb_info *sbi)
5128 {
5129         struct dirty_seglist_info *dirty_i;
5130         unsigned int bitmap_size, i;
5131
5132         /* allocate memory for dirty segments list information */
5133         dirty_i = f2fs_kzalloc(sbi, sizeof(struct dirty_seglist_info),
5134                                                                 GFP_KERNEL);
5135         if (!dirty_i)
5136                 return -ENOMEM;
5137
5138         SM_I(sbi)->dirty_info = dirty_i;
5139         mutex_init(&dirty_i->seglist_lock);
5140
5141         bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
5142
5143         for (i = 0; i < NR_DIRTY_TYPE; i++) {
5144                 dirty_i->dirty_segmap[i] = f2fs_kvzalloc(sbi, bitmap_size,
5145                                                                 GFP_KERNEL);
5146                 if (!dirty_i->dirty_segmap[i])
5147                         return -ENOMEM;
5148         }
5149
5150         if (__is_large_section(sbi)) {
5151                 bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
5152                 dirty_i->dirty_secmap = f2fs_kvzalloc(sbi,
5153                                                 bitmap_size, GFP_KERNEL);
5154                 if (!dirty_i->dirty_secmap)
5155                         return -ENOMEM;
5156         }
5157
5158         init_dirty_segmap(sbi);
5159         return init_victim_secmap(sbi);
5160 }
5161
5162 static int sanity_check_curseg(struct f2fs_sb_info *sbi)
5163 {
5164         int i;
5165
5166         /*
5167          * In LFS/SSR curseg, .next_blkoff should point to an unused blkaddr;
5168          * In LFS curseg, all blkaddr after .next_blkoff should be unused.
5169          */
5170         for (i = 0; i < NR_PERSISTENT_LOG; i++) {
5171                 struct curseg_info *curseg = CURSEG_I(sbi, i);
5172                 struct seg_entry *se = get_seg_entry(sbi, curseg->segno);
5173                 unsigned int blkofs = curseg->next_blkoff;
5174
5175                 if (f2fs_sb_has_readonly(sbi) &&
5176                         i != CURSEG_HOT_DATA && i != CURSEG_HOT_NODE)
5177                         continue;
5178
5179                 sanity_check_seg_type(sbi, curseg->seg_type);
5180
5181                 if (curseg->alloc_type != LFS && curseg->alloc_type != SSR) {
5182                         f2fs_err(sbi,
5183                                  "Current segment has invalid alloc_type:%d",
5184                                  curseg->alloc_type);
5185                         f2fs_handle_error(sbi, ERROR_INVALID_CURSEG);
5186                         return -EFSCORRUPTED;
5187                 }
5188
5189                 if (f2fs_test_bit(blkofs, se->cur_valid_map))
5190                         goto out;
5191
5192                 if (curseg->alloc_type == SSR)
5193                         continue;
5194
5195                 for (blkofs += 1; blkofs < BLKS_PER_SEG(sbi); blkofs++) {
5196                         if (!f2fs_test_bit(blkofs, se->cur_valid_map))
5197                                 continue;
5198 out:
5199                         f2fs_err(sbi,
5200                                  "Current segment's next free block offset is inconsistent with bitmap, logtype:%u, segno:%u, type:%u, next_blkoff:%u, blkofs:%u",
5201                                  i, curseg->segno, curseg->alloc_type,
5202                                  curseg->next_blkoff, blkofs);
5203                         f2fs_handle_error(sbi, ERROR_INVALID_CURSEG);
5204                         return -EFSCORRUPTED;
5205                 }
5206         }
5207         return 0;
5208 }
5209
5210 #ifdef CONFIG_BLK_DEV_ZONED
5211 static int check_zone_write_pointer(struct f2fs_sb_info *sbi,
5212                                     struct f2fs_dev_info *fdev,
5213                                     struct blk_zone *zone)
5214 {
5215         unsigned int zone_segno;
5216         block_t zone_block, valid_block_cnt;
5217         unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT;
5218         int ret;
5219         unsigned int nofs_flags;
5220
5221         if (zone->type != BLK_ZONE_TYPE_SEQWRITE_REQ)
5222                 return 0;
5223
5224         zone_block = fdev->start_blk + (zone->start >> log_sectors_per_block);
5225         zone_segno = GET_SEGNO(sbi, zone_block);
5226
5227         /*
5228          * Skip check of zones cursegs point to, since
5229          * fix_curseg_write_pointer() checks them.
5230          */
5231         if (zone_segno >= MAIN_SEGS(sbi))
5232                 return 0;
5233
5234         /*
5235          * Get # of valid block of the zone.
5236          */
5237         valid_block_cnt = get_valid_blocks(sbi, zone_segno, true);
5238         if (IS_CURSEC(sbi, GET_SEC_FROM_SEG(sbi, zone_segno))) {
5239                 f2fs_notice(sbi, "Open zones: valid block[0x%x,0x%x] cond[%s]",
5240                                 zone_segno, valid_block_cnt,
5241                                 blk_zone_cond_str(zone->cond));
5242                 return 0;
5243         }
5244
5245         if ((!valid_block_cnt && zone->cond == BLK_ZONE_COND_EMPTY) ||
5246             (valid_block_cnt && zone->cond == BLK_ZONE_COND_FULL))
5247                 return 0;
5248
5249         if (!valid_block_cnt) {
5250                 f2fs_notice(sbi, "Zone without valid block has non-zero write "
5251                             "pointer. Reset the write pointer: cond[%s]",
5252                             blk_zone_cond_str(zone->cond));
5253                 ret = __f2fs_issue_discard_zone(sbi, fdev->bdev, zone_block,
5254                                         zone->len >> log_sectors_per_block);
5255                 if (ret)
5256                         f2fs_err(sbi, "Discard zone failed: %s (errno=%d)",
5257                                  fdev->path, ret);
5258                 return ret;
5259         }
5260
5261         /*
5262          * If there are valid blocks and the write pointer doesn't match
5263          * with them, we need to report the inconsistency and fill
5264          * the zone till the end to close the zone. This inconsistency
5265          * does not cause write error because the zone will not be
5266          * selected for write operation until it get discarded.
5267          */
5268         f2fs_notice(sbi, "Valid blocks are not aligned with write "
5269                     "pointer: valid block[0x%x,0x%x] cond[%s]",
5270                     zone_segno, valid_block_cnt, blk_zone_cond_str(zone->cond));
5271
5272         nofs_flags = memalloc_nofs_save();
5273         ret = blkdev_zone_mgmt(fdev->bdev, REQ_OP_ZONE_FINISH,
5274                                 zone->start, zone->len);
5275         memalloc_nofs_restore(nofs_flags);
5276         if (ret == -EOPNOTSUPP) {
5277                 ret = blkdev_issue_zeroout(fdev->bdev, zone->wp,
5278                                         zone->len - (zone->wp - zone->start),
5279                                         GFP_NOFS, 0);
5280                 if (ret)
5281                         f2fs_err(sbi, "Fill up zone failed: %s (errno=%d)",
5282                                         fdev->path, ret);
5283         } else if (ret) {
5284                 f2fs_err(sbi, "Finishing zone failed: %s (errno=%d)",
5285                                 fdev->path, ret);
5286         }
5287
5288         return ret;
5289 }
5290
5291 static struct f2fs_dev_info *get_target_zoned_dev(struct f2fs_sb_info *sbi,
5292                                                   block_t zone_blkaddr)
5293 {
5294         int i;
5295
5296         for (i = 0; i < sbi->s_ndevs; i++) {
5297                 if (!bdev_is_zoned(FDEV(i).bdev))
5298                         continue;
5299                 if (sbi->s_ndevs == 1 || (FDEV(i).start_blk <= zone_blkaddr &&
5300                                 zone_blkaddr <= FDEV(i).end_blk))
5301                         return &FDEV(i);
5302         }
5303
5304         return NULL;
5305 }
5306
5307 static int report_one_zone_cb(struct blk_zone *zone, unsigned int idx,
5308                               void *data)
5309 {
5310         memcpy(data, zone, sizeof(struct blk_zone));
5311         return 0;
5312 }
5313
5314 static int do_fix_curseg_write_pointer(struct f2fs_sb_info *sbi, int type)
5315 {
5316         struct curseg_info *cs = CURSEG_I(sbi, type);
5317         struct f2fs_dev_info *zbd;
5318         struct blk_zone zone;
5319         unsigned int cs_section, wp_segno, wp_blkoff, wp_sector_off;
5320         block_t cs_zone_block, wp_block;
5321         unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT;
5322         sector_t zone_sector;
5323         int err;
5324
5325         cs_section = GET_SEC_FROM_SEG(sbi, cs->segno);
5326         cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section));
5327
5328         zbd = get_target_zoned_dev(sbi, cs_zone_block);
5329         if (!zbd)
5330                 return 0;
5331
5332         /* report zone for the sector the curseg points to */
5333         zone_sector = (sector_t)(cs_zone_block - zbd->start_blk)
5334                 << log_sectors_per_block;
5335         err = blkdev_report_zones(zbd->bdev, zone_sector, 1,
5336                                   report_one_zone_cb, &zone);
5337         if (err != 1) {
5338                 f2fs_err(sbi, "Report zone failed: %s errno=(%d)",
5339                          zbd->path, err);
5340                 return err;
5341         }
5342
5343         if (zone.type != BLK_ZONE_TYPE_SEQWRITE_REQ)
5344                 return 0;
5345
5346         /*
5347          * When safely unmounted in the previous mount, we could use current
5348          * segments. Otherwise, allocate new sections.
5349          */
5350         if (is_set_ckpt_flags(sbi, CP_UMOUNT_FLAG)) {
5351                 wp_block = zbd->start_blk + (zone.wp >> log_sectors_per_block);
5352                 wp_segno = GET_SEGNO(sbi, wp_block);
5353                 wp_blkoff = wp_block - START_BLOCK(sbi, wp_segno);
5354                 wp_sector_off = zone.wp & GENMASK(log_sectors_per_block - 1, 0);
5355
5356                 if (cs->segno == wp_segno && cs->next_blkoff == wp_blkoff &&
5357                                 wp_sector_off == 0)
5358                         return 0;
5359
5360                 f2fs_notice(sbi, "Unaligned curseg[%d] with write pointer: "
5361                             "curseg[0x%x,0x%x] wp[0x%x,0x%x]", type, cs->segno,
5362                             cs->next_blkoff, wp_segno, wp_blkoff);
5363         }
5364
5365         /* Allocate a new section if it's not new. */
5366         if (cs->next_blkoff ||
5367             cs->segno != GET_SEG_FROM_SEC(sbi, GET_ZONE_FROM_SEC(sbi, cs_section))) {
5368                 unsigned int old_segno = cs->segno, old_blkoff = cs->next_blkoff;
5369
5370                 f2fs_allocate_new_section(sbi, type, true);
5371                 f2fs_notice(sbi, "Assign new section to curseg[%d]: "
5372                                 "[0x%x,0x%x] -> [0x%x,0x%x]",
5373                                 type, old_segno, old_blkoff,
5374                                 cs->segno, cs->next_blkoff);
5375         }
5376
5377         /* check consistency of the zone curseg pointed to */
5378         if (check_zone_write_pointer(sbi, zbd, &zone))
5379                 return -EIO;
5380
5381         /* check newly assigned zone */
5382         cs_section = GET_SEC_FROM_SEG(sbi, cs->segno);
5383         cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section));
5384
5385         zbd = get_target_zoned_dev(sbi, cs_zone_block);
5386         if (!zbd)
5387                 return 0;
5388
5389         zone_sector = (sector_t)(cs_zone_block - zbd->start_blk)
5390                 << log_sectors_per_block;
5391         err = blkdev_report_zones(zbd->bdev, zone_sector, 1,
5392                                   report_one_zone_cb, &zone);
5393         if (err != 1) {
5394                 f2fs_err(sbi, "Report zone failed: %s errno=(%d)",
5395                          zbd->path, err);
5396                 return err;
5397         }
5398
5399         if (zone.type != BLK_ZONE_TYPE_SEQWRITE_REQ)
5400                 return 0;
5401
5402         if (zone.wp != zone.start) {
5403                 f2fs_notice(sbi,
5404                             "New zone for curseg[%d] is not yet discarded. "
5405                             "Reset the zone: curseg[0x%x,0x%x]",
5406                             type, cs->segno, cs->next_blkoff);
5407                 err = __f2fs_issue_discard_zone(sbi, zbd->bdev, cs_zone_block,
5408                                         zone.len >> log_sectors_per_block);
5409                 if (err) {
5410                         f2fs_err(sbi, "Discard zone failed: %s (errno=%d)",
5411                                  zbd->path, err);
5412                         return err;
5413                 }
5414         }
5415
5416         return 0;
5417 }
5418
5419 static int fix_curseg_write_pointer(struct f2fs_sb_info *sbi)
5420 {
5421         int i, ret;
5422
5423         for (i = 0; i < NR_PERSISTENT_LOG; i++) {
5424                 ret = do_fix_curseg_write_pointer(sbi, i);
5425                 if (ret)
5426                         return ret;
5427         }
5428
5429         return 0;
5430 }
5431
5432 struct check_zone_write_pointer_args {
5433         struct f2fs_sb_info *sbi;
5434         struct f2fs_dev_info *fdev;
5435 };
5436
5437 static int check_zone_write_pointer_cb(struct blk_zone *zone, unsigned int idx,
5438                                       void *data)
5439 {
5440         struct check_zone_write_pointer_args *args;
5441
5442         args = (struct check_zone_write_pointer_args *)data;
5443
5444         return check_zone_write_pointer(args->sbi, args->fdev, zone);
5445 }
5446
5447 static int check_write_pointer(struct f2fs_sb_info *sbi)
5448 {
5449         int i, ret;
5450         struct check_zone_write_pointer_args args;
5451
5452         for (i = 0; i < sbi->s_ndevs; i++) {
5453                 if (!bdev_is_zoned(FDEV(i).bdev))
5454                         continue;
5455
5456                 args.sbi = sbi;
5457                 args.fdev = &FDEV(i);
5458                 ret = blkdev_report_zones(FDEV(i).bdev, 0, BLK_ALL_ZONES,
5459                                           check_zone_write_pointer_cb, &args);
5460                 if (ret < 0)
5461                         return ret;
5462         }
5463
5464         return 0;
5465 }
5466
5467 int f2fs_check_and_fix_write_pointer(struct f2fs_sb_info *sbi)
5468 {
5469         int ret;
5470
5471         if (!f2fs_sb_has_blkzoned(sbi) || f2fs_readonly(sbi->sb) ||
5472             f2fs_hw_is_readonly(sbi))
5473                 return 0;
5474
5475         f2fs_notice(sbi, "Checking entire write pointers");
5476         ret = fix_curseg_write_pointer(sbi);
5477         if (!ret)
5478                 ret = check_write_pointer(sbi);
5479         return ret;
5480 }
5481
5482 /*
5483  * Return the number of usable blocks in a segment. The number of blocks
5484  * returned is always equal to the number of blocks in a segment for
5485  * segments fully contained within a sequential zone capacity or a
5486  * conventional zone. For segments partially contained in a sequential
5487  * zone capacity, the number of usable blocks up to the zone capacity
5488  * is returned. 0 is returned in all other cases.
5489  */
5490 static inline unsigned int f2fs_usable_zone_blks_in_seg(
5491                         struct f2fs_sb_info *sbi, unsigned int segno)
5492 {
5493         block_t seg_start, sec_start_blkaddr, sec_cap_blkaddr;
5494         unsigned int secno;
5495
5496         if (!sbi->unusable_blocks_per_sec)
5497                 return BLKS_PER_SEG(sbi);
5498
5499         secno = GET_SEC_FROM_SEG(sbi, segno);
5500         seg_start = START_BLOCK(sbi, segno);
5501         sec_start_blkaddr = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, secno));
5502         sec_cap_blkaddr = sec_start_blkaddr + CAP_BLKS_PER_SEC(sbi);
5503
5504         /*
5505          * If segment starts before zone capacity and spans beyond
5506          * zone capacity, then usable blocks are from seg start to
5507          * zone capacity. If the segment starts after the zone capacity,
5508          * then there are no usable blocks.
5509          */
5510         if (seg_start >= sec_cap_blkaddr)
5511                 return 0;
5512         if (seg_start + BLKS_PER_SEG(sbi) > sec_cap_blkaddr)
5513                 return sec_cap_blkaddr - seg_start;
5514
5515         return BLKS_PER_SEG(sbi);
5516 }
5517 #else
5518 int f2fs_check_and_fix_write_pointer(struct f2fs_sb_info *sbi)
5519 {
5520         return 0;
5521 }
5522
5523 static inline unsigned int f2fs_usable_zone_blks_in_seg(struct f2fs_sb_info *sbi,
5524                                                         unsigned int segno)
5525 {
5526         return 0;
5527 }
5528
5529 #endif
5530 unsigned int f2fs_usable_blks_in_seg(struct f2fs_sb_info *sbi,
5531                                         unsigned int segno)
5532 {
5533         if (f2fs_sb_has_blkzoned(sbi))
5534                 return f2fs_usable_zone_blks_in_seg(sbi, segno);
5535
5536         return BLKS_PER_SEG(sbi);
5537 }
5538
5539 unsigned int f2fs_usable_segs_in_sec(struct f2fs_sb_info *sbi)
5540 {
5541         if (f2fs_sb_has_blkzoned(sbi))
5542                 return CAP_SEGS_PER_SEC(sbi);
5543
5544         return SEGS_PER_SEC(sbi);
5545 }
5546
5547 unsigned long long f2fs_get_section_mtime(struct f2fs_sb_info *sbi,
5548         unsigned int segno)
5549 {
5550         unsigned int usable_segs_per_sec = f2fs_usable_segs_in_sec(sbi);
5551         unsigned int secno = 0, start = 0;
5552         unsigned int total_valid_blocks = 0;
5553         unsigned long long mtime = 0;
5554         unsigned int i = 0;
5555
5556         secno = GET_SEC_FROM_SEG(sbi, segno);
5557         start = GET_SEG_FROM_SEC(sbi, secno);
5558
5559         if (!__is_large_section(sbi)) {
5560                 mtime = get_seg_entry(sbi, start + i)->mtime;
5561                 goto out;
5562         }
5563
5564         for (i = 0; i < usable_segs_per_sec; i++) {
5565                 /* for large section, only check the mtime of valid segments */
5566                 struct seg_entry *se = get_seg_entry(sbi, start+i);
5567
5568                 mtime += se->mtime * se->valid_blocks;
5569                 total_valid_blocks += se->valid_blocks;
5570         }
5571
5572         if (total_valid_blocks == 0)
5573                 return INVALID_MTIME;
5574
5575         mtime = div_u64(mtime, total_valid_blocks);
5576 out:
5577         if (unlikely(mtime == INVALID_MTIME))
5578                 mtime -= 1;
5579         return mtime;
5580 }
5581
5582 /*
5583  * Update min, max modified time for cost-benefit GC algorithm
5584  */
5585 static void init_min_max_mtime(struct f2fs_sb_info *sbi)
5586 {
5587         struct sit_info *sit_i = SIT_I(sbi);
5588         unsigned int segno;
5589
5590         down_write(&sit_i->sentry_lock);
5591
5592         sit_i->min_mtime = ULLONG_MAX;
5593
5594         for (segno = 0; segno < MAIN_SEGS(sbi); segno += SEGS_PER_SEC(sbi)) {
5595                 unsigned long long mtime = 0;
5596
5597                 mtime = f2fs_get_section_mtime(sbi, segno);
5598
5599                 if (sit_i->min_mtime > mtime)
5600                         sit_i->min_mtime = mtime;
5601         }
5602         sit_i->max_mtime = get_mtime(sbi, false);
5603         sit_i->dirty_max_mtime = 0;
5604         up_write(&sit_i->sentry_lock);
5605 }
5606
5607 int f2fs_build_segment_manager(struct f2fs_sb_info *sbi)
5608 {
5609         struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
5610         struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
5611         struct f2fs_sm_info *sm_info;
5612         int err;
5613
5614         sm_info = f2fs_kzalloc(sbi, sizeof(struct f2fs_sm_info), GFP_KERNEL);
5615         if (!sm_info)
5616                 return -ENOMEM;
5617
5618         /* init sm info */
5619         sbi->sm_info = sm_info;
5620         sm_info->seg0_blkaddr = le32_to_cpu(raw_super->segment0_blkaddr);
5621         sm_info->main_blkaddr = le32_to_cpu(raw_super->main_blkaddr);
5622         sm_info->segment_count = le32_to_cpu(raw_super->segment_count);
5623         sm_info->reserved_segments = le32_to_cpu(ckpt->rsvd_segment_count);
5624         sm_info->ovp_segments = le32_to_cpu(ckpt->overprov_segment_count);
5625         sm_info->main_segments = le32_to_cpu(raw_super->segment_count_main);
5626         sm_info->ssa_blkaddr = le32_to_cpu(raw_super->ssa_blkaddr);
5627         sm_info->rec_prefree_segments = sm_info->main_segments *
5628                                         DEF_RECLAIM_PREFREE_SEGMENTS / 100;
5629         if (sm_info->rec_prefree_segments > DEF_MAX_RECLAIM_PREFREE_SEGMENTS)
5630                 sm_info->rec_prefree_segments = DEF_MAX_RECLAIM_PREFREE_SEGMENTS;
5631
5632         if (!f2fs_lfs_mode(sbi))
5633                 sm_info->ipu_policy = BIT(F2FS_IPU_FSYNC);
5634         sm_info->min_ipu_util = DEF_MIN_IPU_UTIL;
5635         sm_info->min_fsync_blocks = DEF_MIN_FSYNC_BLOCKS;
5636         sm_info->min_seq_blocks = BLKS_PER_SEG(sbi);
5637         sm_info->min_hot_blocks = DEF_MIN_HOT_BLOCKS;
5638         sm_info->min_ssr_sections = reserved_sections(sbi);
5639
5640         INIT_LIST_HEAD(&sm_info->sit_entry_set);
5641
5642         init_f2fs_rwsem(&sm_info->curseg_lock);
5643
5644         err = f2fs_create_flush_cmd_control(sbi);
5645         if (err)
5646                 return err;
5647
5648         err = create_discard_cmd_control(sbi);
5649         if (err)
5650                 return err;
5651
5652         err = build_sit_info(sbi);
5653         if (err)
5654                 return err;
5655         err = build_free_segmap(sbi);
5656         if (err)
5657                 return err;
5658         err = build_curseg(sbi);
5659         if (err)
5660                 return err;
5661
5662         /* reinit free segmap based on SIT */
5663         err = build_sit_entries(sbi);
5664         if (err)
5665                 return err;
5666
5667         init_free_segmap(sbi);
5668         err = build_dirty_segmap(sbi);
5669         if (err)
5670                 return err;
5671
5672         err = sanity_check_curseg(sbi);
5673         if (err)
5674                 return err;
5675
5676         init_min_max_mtime(sbi);
5677         return 0;
5678 }
5679
5680 static void discard_dirty_segmap(struct f2fs_sb_info *sbi,
5681                 enum dirty_type dirty_type)
5682 {
5683         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
5684
5685         mutex_lock(&dirty_i->seglist_lock);
5686         kvfree(dirty_i->dirty_segmap[dirty_type]);
5687         dirty_i->nr_dirty[dirty_type] = 0;
5688         mutex_unlock(&dirty_i->seglist_lock);
5689 }
5690
5691 static void destroy_victim_secmap(struct f2fs_sb_info *sbi)
5692 {
5693         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
5694
5695         kvfree(dirty_i->pinned_secmap);
5696         kvfree(dirty_i->victim_secmap);
5697 }
5698
5699 static void destroy_dirty_segmap(struct f2fs_sb_info *sbi)
5700 {
5701         struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
5702         int i;
5703
5704         if (!dirty_i)
5705                 return;
5706
5707         /* discard pre-free/dirty segments list */
5708         for (i = 0; i < NR_DIRTY_TYPE; i++)
5709                 discard_dirty_segmap(sbi, i);
5710
5711         if (__is_large_section(sbi)) {
5712                 mutex_lock(&dirty_i->seglist_lock);
5713                 kvfree(dirty_i->dirty_secmap);
5714                 mutex_unlock(&dirty_i->seglist_lock);
5715         }
5716
5717         destroy_victim_secmap(sbi);
5718         SM_I(sbi)->dirty_info = NULL;
5719         kfree(dirty_i);
5720 }
5721
5722 static void destroy_curseg(struct f2fs_sb_info *sbi)
5723 {
5724         struct curseg_info *array = SM_I(sbi)->curseg_array;
5725         int i;
5726
5727         if (!array)
5728                 return;
5729         SM_I(sbi)->curseg_array = NULL;
5730         for (i = 0; i < NR_CURSEG_TYPE; i++) {
5731                 kfree(array[i].sum_blk);
5732                 kfree(array[i].journal);
5733         }
5734         kfree(array);
5735 }
5736
5737 static void destroy_free_segmap(struct f2fs_sb_info *sbi)
5738 {
5739         struct free_segmap_info *free_i = SM_I(sbi)->free_info;
5740
5741         if (!free_i)
5742                 return;
5743         SM_I(sbi)->free_info = NULL;
5744         kvfree(free_i->free_segmap);
5745         kvfree(free_i->free_secmap);
5746         kfree(free_i);
5747 }
5748
5749 static void destroy_sit_info(struct f2fs_sb_info *sbi)
5750 {
5751         struct sit_info *sit_i = SIT_I(sbi);
5752
5753         if (!sit_i)
5754                 return;
5755
5756         if (sit_i->sentries)
5757                 kvfree(sit_i->bitmap);
5758         kfree(sit_i->tmp_map);
5759
5760         kvfree(sit_i->sentries);
5761         kvfree(sit_i->sec_entries);
5762         kvfree(sit_i->dirty_sentries_bitmap);
5763
5764         SM_I(sbi)->sit_info = NULL;
5765         kvfree(sit_i->sit_bitmap);
5766 #ifdef CONFIG_F2FS_CHECK_FS
5767         kvfree(sit_i->sit_bitmap_mir);
5768         kvfree(sit_i->invalid_segmap);
5769 #endif
5770         kfree(sit_i);
5771 }
5772
5773 void f2fs_destroy_segment_manager(struct f2fs_sb_info *sbi)
5774 {
5775         struct f2fs_sm_info *sm_info = SM_I(sbi);
5776
5777         if (!sm_info)
5778                 return;
5779         f2fs_destroy_flush_cmd_control(sbi, true);
5780         destroy_discard_cmd_control(sbi);
5781         destroy_dirty_segmap(sbi);
5782         destroy_curseg(sbi);
5783         destroy_free_segmap(sbi);
5784         destroy_sit_info(sbi);
5785         sbi->sm_info = NULL;
5786         kfree(sm_info);
5787 }
5788
5789 int __init f2fs_create_segment_manager_caches(void)
5790 {
5791         discard_entry_slab = f2fs_kmem_cache_create("f2fs_discard_entry",
5792                         sizeof(struct discard_entry));
5793         if (!discard_entry_slab)
5794                 goto fail;
5795
5796         discard_cmd_slab = f2fs_kmem_cache_create("f2fs_discard_cmd",
5797                         sizeof(struct discard_cmd));
5798         if (!discard_cmd_slab)
5799                 goto destroy_discard_entry;
5800
5801         sit_entry_set_slab = f2fs_kmem_cache_create("f2fs_sit_entry_set",
5802                         sizeof(struct sit_entry_set));
5803         if (!sit_entry_set_slab)
5804                 goto destroy_discard_cmd;
5805
5806         revoke_entry_slab = f2fs_kmem_cache_create("f2fs_revoke_entry",
5807                         sizeof(struct revoke_entry));
5808         if (!revoke_entry_slab)
5809                 goto destroy_sit_entry_set;
5810         return 0;
5811
5812 destroy_sit_entry_set:
5813         kmem_cache_destroy(sit_entry_set_slab);
5814 destroy_discard_cmd:
5815         kmem_cache_destroy(discard_cmd_slab);
5816 destroy_discard_entry:
5817         kmem_cache_destroy(discard_entry_slab);
5818 fail:
5819         return -ENOMEM;
5820 }
5821
5822 void f2fs_destroy_segment_manager_caches(void)
5823 {
5824         kmem_cache_destroy(sit_entry_set_slab);
5825         kmem_cache_destroy(discard_cmd_slab);
5826         kmem_cache_destroy(discard_entry_slab);
5827         kmem_cache_destroy(revoke_entry_slab);
5828 }