fs/bcachefs/recovery.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 #include "bcachefs.h"
   4 #include "alloc_background.h"
   5 #include "bkey_buf.h"
   6 #include "btree_journal_iter.h"
   7 #include "btree_node_scan.h"
   8 #include "btree_update.h"
   9 #include "btree_update_interior.h"
  10 #include "btree_io.h"
  11 #include "buckets.h"
  12 #include "dirent.h"
  13 #include "disk_accounting.h"
  14 #include "errcode.h"
  15 #include "error.h"
  16 #include "fs-common.h"
  17 #include "journal_io.h"
  18 #include "journal_reclaim.h"
  19 #include "journal_seq_blacklist.h"
  20 #include "logged_ops.h"
  21 #include "move.h"
  22 #include "quota.h"
  23 #include "rebalance.h"
  24 #include "recovery.h"
  25 #include "recovery_passes.h"
  26 #include "replicas.h"
  27 #include "sb-clean.h"
  28 #include "sb-downgrade.h"
  29 #include "snapshot.h"
  30 #include "super-io.h"
  31
  32 #include <linux/sort.h>
  33 #include <linux/stat.h>
  34
  35 #define QSTR(n) { { { .len = strlen(n) } }, .name = n }
  36
  37 void bch2_btree_lost_data(struct bch_fs *c, enum btree_id btree)
  38 {
  39         if (btree >= BTREE_ID_NR_MAX)
  40                 return;
  41
  42         u64 b = BIT_ULL(btree);
  43
  44         if (!(c->sb.btrees_lost_data & b)) {
  45                 bch_err(c, "flagging btree %s lost data", bch2_btree_id_str(btree));
  46
  47                 mutex_lock(&c->sb_lock);
  48                 bch2_sb_field_get(c->disk_sb.sb, ext)->btrees_lost_data |= cpu_to_le64(b);
  49                 bch2_write_super(c);
  50                 mutex_unlock(&c->sb_lock);
  51         }
  52 }
  53
  54 /* for -o reconstruct_alloc: */
  55 static void bch2_reconstruct_alloc(struct bch_fs *c)
  56 {
  57         bch2_journal_log_msg(c, "dropping alloc info");
  58         bch_info(c, "dropping and reconstructing all alloc info");
  59
  60         mutex_lock(&c->sb_lock);
  61         struct bch_sb_field_ext *ext = bch2_sb_field_get(c->disk_sb.sb, ext);
  62
  63         __set_bit_le64(BCH_RECOVERY_PASS_STABLE_check_allocations, ext->recovery_passes_required);
  64         __set_bit_le64(BCH_RECOVERY_PASS_STABLE_check_alloc_info, ext->recovery_passes_required);
  65         __set_bit_le64(BCH_RECOVERY_PASS_STABLE_check_lrus, ext->recovery_passes_required);
  66         __set_bit_le64(BCH_RECOVERY_PASS_STABLE_check_extents_to_backpointers, ext->recovery_passes_required);
  67         __set_bit_le64(BCH_RECOVERY_PASS_STABLE_check_alloc_to_lru_refs, ext->recovery_passes_required);
  68
  69         __set_bit_le64(BCH_FSCK_ERR_ptr_to_missing_alloc_key, ext->errors_silent);
  70         __set_bit_le64(BCH_FSCK_ERR_ptr_gen_newer_than_bucket_gen, ext->errors_silent);
  71         __set_bit_le64(BCH_FSCK_ERR_stale_dirty_ptr, ext->errors_silent);
  72
  73         __set_bit_le64(BCH_FSCK_ERR_dev_usage_buckets_wrong, ext->errors_silent);
  74         __set_bit_le64(BCH_FSCK_ERR_dev_usage_sectors_wrong, ext->errors_silent);
  75         __set_bit_le64(BCH_FSCK_ERR_dev_usage_fragmented_wrong, ext->errors_silent);
  76
  77         __set_bit_le64(BCH_FSCK_ERR_fs_usage_btree_wrong, ext->errors_silent);
  78         __set_bit_le64(BCH_FSCK_ERR_fs_usage_cached_wrong, ext->errors_silent);
  79         __set_bit_le64(BCH_FSCK_ERR_fs_usage_persistent_reserved_wrong, ext->errors_silent);
  80         __set_bit_le64(BCH_FSCK_ERR_fs_usage_replicas_wrong, ext->errors_silent);
  81
  82         __set_bit_le64(BCH_FSCK_ERR_alloc_key_data_type_wrong, ext->errors_silent);
  83         __set_bit_le64(BCH_FSCK_ERR_alloc_key_gen_wrong, ext->errors_silent);
  84         __set_bit_le64(BCH_FSCK_ERR_alloc_key_dirty_sectors_wrong, ext->errors_silent);
  85         __set_bit_le64(BCH_FSCK_ERR_alloc_key_cached_sectors_wrong, ext->errors_silent);
  86         __set_bit_le64(BCH_FSCK_ERR_alloc_key_stripe_wrong, ext->errors_silent);
  87         __set_bit_le64(BCH_FSCK_ERR_alloc_key_stripe_redundancy_wrong, ext->errors_silent);
  88         __set_bit_le64(BCH_FSCK_ERR_need_discard_key_wrong, ext->errors_silent);
  89         __set_bit_le64(BCH_FSCK_ERR_freespace_key_wrong, ext->errors_silent);
  90         __set_bit_le64(BCH_FSCK_ERR_bucket_gens_key_wrong, ext->errors_silent);
  91         __set_bit_le64(BCH_FSCK_ERR_freespace_hole_missing, ext->errors_silent);
  92         __set_bit_le64(BCH_FSCK_ERR_ptr_to_missing_backpointer, ext->errors_silent);
  93         __set_bit_le64(BCH_FSCK_ERR_lru_entry_bad, ext->errors_silent);
  94         __set_bit_le64(BCH_FSCK_ERR_accounting_mismatch, ext->errors_silent);
  95         c->sb.compat &= ~(1ULL << BCH_COMPAT_alloc_info);
  96
  97         c->opts.recovery_passes |= bch2_recovery_passes_from_stable(le64_to_cpu(ext->recovery_passes_required[0]));
  98
  99         bch2_write_super(c);
 100         mutex_unlock(&c->sb_lock);
 101
 102         bch2_shoot_down_journal_keys(c, BTREE_ID_alloc,
 103                                      0, BTREE_MAX_DEPTH, POS_MIN, SPOS_MAX);
 104         bch2_shoot_down_journal_keys(c, BTREE_ID_backpointers,
 105                                      0, BTREE_MAX_DEPTH, POS_MIN, SPOS_MAX);
 106         bch2_shoot_down_journal_keys(c, BTREE_ID_need_discard,
 107                                      0, BTREE_MAX_DEPTH, POS_MIN, SPOS_MAX);
 108         bch2_shoot_down_journal_keys(c, BTREE_ID_freespace,
 109                                      0, BTREE_MAX_DEPTH, POS_MIN, SPOS_MAX);
 110         bch2_shoot_down_journal_keys(c, BTREE_ID_bucket_gens,
 111                                      0, BTREE_MAX_DEPTH, POS_MIN, SPOS_MAX);
 112 }
 113
 114 /*
 115  * Btree node pointers have a field to stack a pointer to the in memory btree
 116  * node; we need to zero out this field when reading in btree nodes, or when
 117  * reading in keys from the journal:
 118  */
 119 static void zero_out_btree_mem_ptr(struct journal_keys *keys)
 120 {
 121         darray_for_each(*keys, i)
 122                 if (i->k->k.type == KEY_TYPE_btree_ptr_v2)
 123                         bkey_i_to_btree_ptr_v2(i->k)->v.mem_ptr = 0;
 124 }
 125
 126 /* journal replay: */
 127
 128 static void replay_now_at(struct journal *j, u64 seq)
 129 {
 130         BUG_ON(seq < j->replay_journal_seq);
 131
 132         seq = min(seq, j->replay_journal_seq_end);
 133
 134         while (j->replay_journal_seq < seq)
 135                 bch2_journal_pin_put(j, j->replay_journal_seq++);
 136 }
 137
 138 static int bch2_journal_replay_accounting_key(struct btree_trans *trans,
 139                                               struct journal_key *k)
 140 {
 141         struct btree_iter iter;
 142         bch2_trans_node_iter_init(trans, &iter, k->btree_id, k->k->k.p,
 143                                   BTREE_MAX_DEPTH, k->level,
 144                                   BTREE_ITER_intent);
 145         int ret = bch2_btree_iter_traverse(&iter);
 146         if (ret)
 147                 goto out;
 148
 149         struct bkey u;
 150         struct bkey_s_c old = bch2_btree_path_peek_slot(btree_iter_path(trans, &iter), &u);
 151
 152         /* Has this delta already been applied to the btree? */
 153         if (bversion_cmp(old.k->bversion, k->k->k.bversion) >= 0) {
 154                 ret = 0;
 155                 goto out;
 156         }
 157
 158         struct bkey_i *new = k->k;
 159         if (old.k->type == KEY_TYPE_accounting) {
 160                 new = bch2_bkey_make_mut_noupdate(trans, bkey_i_to_s_c(k->k));
 161                 ret = PTR_ERR_OR_ZERO(new);
 162                 if (ret)
 163                         goto out;
 164
 165                 bch2_accounting_accumulate(bkey_i_to_accounting(new),
 166                                            bkey_s_c_to_accounting(old));
 167         }
 168
 169         trans->journal_res.seq = k->journal_seq;
 170
 171         ret = bch2_trans_update(trans, &iter, new, BTREE_TRIGGER_norun);
 172 out:
 173         bch2_trans_iter_exit(trans, &iter);
 174         return ret;
 175 }
 176
 177 static int bch2_journal_replay_key(struct btree_trans *trans,
 178                                    struct journal_key *k)
 179 {
 180         struct btree_iter iter;
 181         unsigned iter_flags =
 182                 BTREE_ITER_intent|
 183                 BTREE_ITER_not_extents;
 184         unsigned update_flags = BTREE_TRIGGER_norun;
 185         int ret;
 186
 187         if (k->overwritten)
 188                 return 0;
 189
 190         trans->journal_res.seq = k->journal_seq;
 191
 192         /*
 193          * BTREE_UPDATE_key_cache_reclaim disables key cache lookup/update to
 194          * keep the key cache coherent with the underlying btree. Nothing
 195          * besides the allocator is doing updates yet so we don't need key cache
 196          * coherency for non-alloc btrees, and key cache fills for snapshots
 197          * btrees use BTREE_ITER_filter_snapshots, which isn't available until
 198          * the snapshots recovery pass runs.
 199          */
 200         if (!k->level && k->btree_id == BTREE_ID_alloc)
 201                 iter_flags |= BTREE_ITER_cached;
 202         else
 203                 update_flags |= BTREE_UPDATE_key_cache_reclaim;
 204
 205         bch2_trans_node_iter_init(trans, &iter, k->btree_id, k->k->k.p,
 206                                   BTREE_MAX_DEPTH, k->level,
 207                                   iter_flags);
 208         ret = bch2_btree_iter_traverse(&iter);
 209         if (ret)
 210                 goto out;
 211
 212         struct btree_path *path = btree_iter_path(trans, &iter);
 213         if (unlikely(!btree_path_node(path, k->level))) {
 214                 bch2_trans_iter_exit(trans, &iter);
 215                 bch2_trans_node_iter_init(trans, &iter, k->btree_id, k->k->k.p,
 216                                           BTREE_MAX_DEPTH, 0, iter_flags);
 217                 ret =   bch2_btree_iter_traverse(&iter) ?:
 218                         bch2_btree_increase_depth(trans, iter.path, 0) ?:
 219                         -BCH_ERR_transaction_restart_nested;
 220                 goto out;
 221         }
 222
 223         /* Must be checked with btree locked: */
 224         if (k->overwritten)
 225                 goto out;
 226
 227         if (k->k->k.type == KEY_TYPE_accounting) {
 228                 ret = bch2_trans_update_buffered(trans, BTREE_ID_accounting, k->k);
 229                 goto out;
 230         }
 231
 232         ret = bch2_trans_update(trans, &iter, k->k, update_flags);
 233 out:
 234         bch2_trans_iter_exit(trans, &iter);
 235         return ret;
 236 }
 237
 238 static int journal_sort_seq_cmp(const void *_l, const void *_r)
 239 {
 240         const struct journal_key *l = *((const struct journal_key **)_l);
 241         const struct journal_key *r = *((const struct journal_key **)_r);
 242
 243         /*
 244          * Map 0 to U64_MAX, so that keys with journal_seq === 0 come last
 245          *
 246          * journal_seq == 0 means that the key comes from early repair, and
 247          * should be inserted last so as to avoid overflowing the journal
 248          */
 249         return cmp_int(l->journal_seq - 1, r->journal_seq - 1);
 250 }
 251
 252 int bch2_journal_replay(struct bch_fs *c)
 253 {
 254         struct journal_keys *keys = &c->journal_keys;
 255         DARRAY(struct journal_key *) keys_sorted = { 0 };
 256         struct journal *j = &c->journal;
 257         u64 start_seq   = c->journal_replay_seq_start;
 258         u64 end_seq     = c->journal_replay_seq_start;
 259         struct btree_trans *trans = NULL;
 260         bool immediate_flush = false;
 261         int ret = 0;
 262
 263         if (keys->nr) {
 264                 ret = bch2_journal_log_msg(c, "Starting journal replay (%zu keys in entries %llu-%llu)",
 265                                            keys->nr, start_seq, end_seq);
 266                 if (ret)
 267                         goto err;
 268         }
 269
 270         BUG_ON(!atomic_read(&keys->ref));
 271
 272         move_gap(keys, keys->nr);
 273         trans = bch2_trans_get(c);
 274
 275         /*
 276          * Replay accounting keys first: we can't allow the write buffer to
 277          * flush accounting keys until we're done
 278          */
 279         darray_for_each(*keys, k) {
 280                 if (!(k->k->k.type == KEY_TYPE_accounting && !k->allocated))
 281                         continue;
 282
 283                 cond_resched();
 284
 285                 ret = commit_do(trans, NULL, NULL,
 286                                 BCH_TRANS_COMMIT_no_enospc|
 287                                 BCH_TRANS_COMMIT_journal_reclaim|
 288                                 BCH_TRANS_COMMIT_skip_accounting_apply|
 289                                 BCH_TRANS_COMMIT_no_journal_res|
 290                                 BCH_WATERMARK_reclaim,
 291                              bch2_journal_replay_accounting_key(trans, k));
 292                 if (bch2_fs_fatal_err_on(ret, c, "error replaying accounting; %s", bch2_err_str(ret)))
 293                         goto err;
 294
 295                 k->overwritten = true;
 296         }
 297
 298         set_bit(BCH_FS_accounting_replay_done, &c->flags);
 299
 300         /*
 301          * First, attempt to replay keys in sorted order. This is more
 302          * efficient - better locality of btree access -  but some might fail if
 303          * that would cause a journal deadlock.
 304          */
 305         darray_for_each(*keys, k) {
 306                 cond_resched();
 307
 308                 /*
 309                  * k->allocated means the key wasn't read in from the journal,
 310                  * rather it was from early repair code
 311                  */
 312                 if (k->allocated)
 313                         immediate_flush = true;
 314
 315                 /* Skip fastpath if we're low on space in the journal */
 316                 ret = c->journal.watermark ? -1 :
 317                         commit_do(trans, NULL, NULL,
 318                                   BCH_TRANS_COMMIT_no_enospc|
 319                                   BCH_TRANS_COMMIT_journal_reclaim|
 320                                   BCH_TRANS_COMMIT_skip_accounting_apply|
 321                                   (!k->allocated ? BCH_TRANS_COMMIT_no_journal_res : 0),
 322                              bch2_journal_replay_key(trans, k));
 323                 BUG_ON(!ret && !k->overwritten && k->k->k.type != KEY_TYPE_accounting);
 324                 if (ret) {
 325                         ret = darray_push(&keys_sorted, k);
 326                         if (ret)
 327                                 goto err;
 328                 }
 329         }
 330
 331         bch2_trans_unlock_long(trans);
 332         /*
 333          * Now, replay any remaining keys in the order in which they appear in
 334          * the journal, unpinning those journal entries as we go:
 335          */
 336         sort(keys_sorted.data, keys_sorted.nr,
 337              sizeof(keys_sorted.data[0]),
 338              journal_sort_seq_cmp, NULL);
 339
 340         darray_for_each(keys_sorted, kp) {
 341                 cond_resched();
 342
 343                 struct journal_key *k = *kp;
 344
 345                 if (k->journal_seq)
 346                         replay_now_at(j, k->journal_seq);
 347                 else
 348                         replay_now_at(j, j->replay_journal_seq_end);
 349
 350                 ret = commit_do(trans, NULL, NULL,
 351                                 BCH_TRANS_COMMIT_no_enospc|
 352                                 BCH_TRANS_COMMIT_skip_accounting_apply|
 353                                 (!k->allocated
 354                                  ? BCH_TRANS_COMMIT_no_journal_res|BCH_WATERMARK_reclaim
 355                                  : 0),
 356                              bch2_journal_replay_key(trans, k));
 357                 bch_err_msg(c, ret, "while replaying key at btree %s level %u:",
 358                             bch2_btree_id_str(k->btree_id), k->level);
 359                 if (ret)
 360                         goto err;
 361
 362                 BUG_ON(k->btree_id != BTREE_ID_accounting && !k->overwritten);
 363         }
 364
 365         /*
 366          * We need to put our btree_trans before calling flush_all_pins(), since
 367          * that will use a btree_trans internally
 368          */
 369         bch2_trans_put(trans);
 370         trans = NULL;
 371
 372         if (!c->opts.retain_recovery_info &&
 373             c->recovery_pass_done >= BCH_RECOVERY_PASS_journal_replay)
 374                 bch2_journal_keys_put_initial(c);
 375
 376         replay_now_at(j, j->replay_journal_seq_end);
 377         j->replay_journal_seq = 0;
 378
 379         bch2_journal_set_replay_done(j);
 380
 381         /* if we did any repair, flush it immediately */
 382         if (immediate_flush) {
 383                 bch2_journal_flush_all_pins(&c->journal);
 384                 ret = bch2_journal_meta(&c->journal);
 385         }
 386
 387         if (keys->nr)
 388                 bch2_journal_log_msg(c, "journal replay finished");
 389 err:
 390         if (trans)
 391                 bch2_trans_put(trans);
 392         darray_exit(&keys_sorted);
 393         bch_err_fn(c, ret);
 394         return ret;
 395 }
 396
 397 /* journal replay early: */
 398
 399 static int journal_replay_entry_early(struct bch_fs *c,
 400                                       struct jset_entry *entry)
 401 {
 402         int ret = 0;
 403
 404         switch (entry->type) {
 405         case BCH_JSET_ENTRY_btree_root: {
 406                 struct btree_root *r;
 407
 408                 if (fsck_err_on(entry->btree_id >= BTREE_ID_NR_MAX,
 409                                 c, invalid_btree_id,
 410                                 "invalid btree id %u (max %u)",
 411                                 entry->btree_id, BTREE_ID_NR_MAX))
 412                         return 0;
 413
 414                 while (entry->btree_id >= c->btree_roots_extra.nr + BTREE_ID_NR) {
 415                         ret = darray_push(&c->btree_roots_extra, (struct btree_root) { NULL });
 416                         if (ret)
 417                                 return ret;
 418                 }
 419
 420                 r = bch2_btree_id_root(c, entry->btree_id);
 421
 422                 if (entry->u64s) {
 423                         r->level = entry->level;
 424                         bkey_copy(&r->key, (struct bkey_i *) entry->start);
 425                         r->error = 0;
 426                 } else {
 427                         r->error = -BCH_ERR_btree_node_read_error;
 428                 }
 429                 r->alive = true;
 430                 break;
 431         }
 432         case BCH_JSET_ENTRY_usage: {
 433                 struct jset_entry_usage *u =
 434                         container_of(entry, struct jset_entry_usage, entry);
 435
 436                 switch (entry->btree_id) {
 437                 case BCH_FS_USAGE_key_version:
 438                         atomic64_set(&c->key_version, le64_to_cpu(u->v));
 439                         break;
 440                 }
 441                 break;
 442         }
 443         case BCH_JSET_ENTRY_blacklist: {
 444                 struct jset_entry_blacklist *bl_entry =
 445                         container_of(entry, struct jset_entry_blacklist, entry);
 446
 447                 ret = bch2_journal_seq_blacklist_add(c,
 448                                 le64_to_cpu(bl_entry->seq),
 449                                 le64_to_cpu(bl_entry->seq) + 1);
 450                 break;
 451         }
 452         case BCH_JSET_ENTRY_blacklist_v2: {
 453                 struct jset_entry_blacklist_v2 *bl_entry =
 454                         container_of(entry, struct jset_entry_blacklist_v2, entry);
 455
 456                 ret = bch2_journal_seq_blacklist_add(c,
 457                                 le64_to_cpu(bl_entry->start),
 458                                 le64_to_cpu(bl_entry->end) + 1);
 459                 break;
 460         }
 461         case BCH_JSET_ENTRY_clock: {
 462                 struct jset_entry_clock *clock =
 463                         container_of(entry, struct jset_entry_clock, entry);
 464
 465                 atomic64_set(&c->io_clock[clock->rw].now, le64_to_cpu(clock->time));
 466         }
 467         }
 468 fsck_err:
 469         return ret;
 470 }
 471
 472 static int journal_replay_early(struct bch_fs *c,
 473                                 struct bch_sb_field_clean *clean)
 474 {
 475         if (clean) {
 476                 for (struct jset_entry *entry = clean->start;
 477                      entry != vstruct_end(&clean->field);
 478                      entry = vstruct_next(entry)) {
 479                         int ret = journal_replay_entry_early(c, entry);
 480                         if (ret)
 481                                 return ret;
 482                 }
 483         } else {
 484                 struct genradix_iter iter;
 485                 struct journal_replay *i, **_i;
 486
 487                 genradix_for_each(&c->journal_entries, iter, _i) {
 488                         i = *_i;
 489
 490                         if (journal_replay_ignore(i))
 491                                 continue;
 492
 493                         vstruct_for_each(&i->j, entry) {
 494                                 int ret = journal_replay_entry_early(c, entry);
 495                                 if (ret)
 496                                         return ret;
 497                         }
 498                 }
 499         }
 500
 501         return 0;
 502 }
 503
 504 /* sb clean section: */
 505
 506 static int read_btree_roots(struct bch_fs *c)
 507 {
 508         int ret = 0;
 509
 510         for (unsigned i = 0; i < btree_id_nr_alive(c); i++) {
 511                 struct btree_root *r = bch2_btree_id_root(c, i);
 512
 513                 if (!r->alive)
 514                         continue;
 515
 516                 if (btree_id_is_alloc(i) && c->opts.reconstruct_alloc)
 517                         continue;
 518
 519                 if (mustfix_fsck_err_on((ret = r->error),
 520                                         c, btree_root_bkey_invalid,
 521                                         "invalid btree root %s",
 522                                         bch2_btree_id_str(i)) ||
 523                     mustfix_fsck_err_on((ret = r->error = bch2_btree_root_read(c, i, &r->key, r->level)),
 524                                         c, btree_root_read_error,
 525                                         "error reading btree root %s l=%u: %s",
 526                                         bch2_btree_id_str(i), r->level, bch2_err_str(ret))) {
 527                         if (btree_id_is_alloc(i)) {
 528                                 c->opts.recovery_passes |= BIT_ULL(BCH_RECOVERY_PASS_check_allocations);
 529                                 c->opts.recovery_passes |= BIT_ULL(BCH_RECOVERY_PASS_check_alloc_info);
 530                                 c->opts.recovery_passes |= BIT_ULL(BCH_RECOVERY_PASS_check_lrus);
 531                                 c->opts.recovery_passes |= BIT_ULL(BCH_RECOVERY_PASS_check_extents_to_backpointers);
 532                                 c->opts.recovery_passes |= BIT_ULL(BCH_RECOVERY_PASS_check_alloc_to_lru_refs);
 533                                 c->sb.compat &= ~(1ULL << BCH_COMPAT_alloc_info);
 534                                 r->error = 0;
 535                         } else if (!(c->opts.recovery_passes & BIT_ULL(BCH_RECOVERY_PASS_scan_for_btree_nodes))) {
 536                                 bch_info(c, "will run btree node scan");
 537                                 c->opts.recovery_passes |= BIT_ULL(BCH_RECOVERY_PASS_scan_for_btree_nodes);
 538                                 c->opts.recovery_passes |= BIT_ULL(BCH_RECOVERY_PASS_check_topology);
 539                         }
 540
 541                         ret = 0;
 542                         bch2_btree_lost_data(c, i);
 543                 }
 544         }
 545
 546         for (unsigned i = 0; i < BTREE_ID_NR; i++) {
 547                 struct btree_root *r = bch2_btree_id_root(c, i);
 548
 549                 if (!r->b && !r->error) {
 550                         r->alive = false;
 551                         r->level = 0;
 552                         bch2_btree_root_alloc_fake(c, i, 0);
 553                 }
 554         }
 555 fsck_err:
 556         return ret;
 557 }
 558
 559 static bool check_version_upgrade(struct bch_fs *c)
 560 {
 561         unsigned latest_version = bcachefs_metadata_version_current;
 562         unsigned latest_compatible = min(latest_version,
 563                                          bch2_latest_compatible_version(c->sb.version));
 564         unsigned old_version = c->sb.version_upgrade_complete ?: c->sb.version;
 565         unsigned new_version = 0;
 566
 567         if (old_version < bcachefs_metadata_required_upgrade_below) {
 568                 if (c->opts.version_upgrade == BCH_VERSION_UPGRADE_incompatible ||
 569                     latest_compatible < bcachefs_metadata_required_upgrade_below)
 570                         new_version = latest_version;
 571                 else
 572                         new_version = latest_compatible;
 573         } else {
 574                 switch (c->opts.version_upgrade) {
 575                 case BCH_VERSION_UPGRADE_compatible:
 576                         new_version = latest_compatible;
 577                         break;
 578                 case BCH_VERSION_UPGRADE_incompatible:
 579                         new_version = latest_version;
 580                         break;
 581                 case BCH_VERSION_UPGRADE_none:
 582                         new_version = min(old_version, latest_version);
 583                         break;
 584                 }
 585         }
 586
 587         if (new_version > old_version) {
 588                 struct printbuf buf = PRINTBUF;
 589
 590                 if (old_version < bcachefs_metadata_required_upgrade_below)
 591                         prt_str(&buf, "Version upgrade required:\n");
 592
 593                 if (old_version != c->sb.version) {
 594                         prt_str(&buf, "Version upgrade from ");
 595                         bch2_version_to_text(&buf, c->sb.version_upgrade_complete);
 596                         prt_str(&buf, " to ");
 597                         bch2_version_to_text(&buf, c->sb.version);
 598                         prt_str(&buf, " incomplete\n");
 599                 }
 600
 601                 prt_printf(&buf, "Doing %s version upgrade from ",
 602                            BCH_VERSION_MAJOR(old_version) != BCH_VERSION_MAJOR(new_version)
 603                            ? "incompatible" : "compatible");
 604                 bch2_version_to_text(&buf, old_version);
 605                 prt_str(&buf, " to ");
 606                 bch2_version_to_text(&buf, new_version);
 607                 prt_newline(&buf);
 608
 609                 struct bch_sb_field_ext *ext = bch2_sb_field_get(c->disk_sb.sb, ext);
 610                 __le64 passes = ext->recovery_passes_required[0];
 611                 bch2_sb_set_upgrade(c, old_version, new_version);
 612                 passes = ext->recovery_passes_required[0] & ~passes;
 613
 614                 if (passes) {
 615                         prt_str(&buf, "  running recovery passes: ");
 616                         prt_bitflags(&buf, bch2_recovery_passes,
 617                                      bch2_recovery_passes_from_stable(le64_to_cpu(passes)));
 618                 }
 619
 620                 bch_info(c, "%s", buf.buf);
 621
 622                 bch2_sb_upgrade(c, new_version);
 623
 624                 printbuf_exit(&buf);
 625                 return true;
 626         }
 627
 628         return false;
 629 }
 630
 631 int bch2_fs_recovery(struct bch_fs *c)
 632 {
 633         struct bch_sb_field_clean *clean = NULL;
 634         struct jset *last_journal_entry = NULL;
 635         u64 last_seq = 0, blacklist_seq, journal_seq;
 636         int ret = 0;
 637
 638         if (c->sb.clean) {
 639                 clean = bch2_read_superblock_clean(c);
 640                 ret = PTR_ERR_OR_ZERO(clean);
 641                 if (ret)
 642                         goto err;
 643
 644                 bch_info(c, "recovering from clean shutdown, journal seq %llu",
 645                          le64_to_cpu(clean->journal_seq));
 646         } else {
 647                 bch_info(c, "recovering from unclean shutdown");
 648         }
 649
 650         if (!(c->sb.features & (1ULL << BCH_FEATURE_new_extent_overwrite))) {
 651                 bch_err(c, "feature new_extent_overwrite not set, filesystem no longer supported");
 652                 ret = -EINVAL;
 653                 goto err;
 654         }
 655
 656         if (!c->sb.clean &&
 657             !(c->sb.features & (1ULL << BCH_FEATURE_extents_above_btree_updates))) {
 658                 bch_err(c, "filesystem needs recovery from older version; run fsck from older bcachefs-tools to fix");
 659                 ret = -EINVAL;
 660                 goto err;
 661         }
 662
 663         if (c->opts.norecovery)
 664                 c->opts.recovery_pass_last = BCH_RECOVERY_PASS_journal_replay - 1;
 665
 666         mutex_lock(&c->sb_lock);
 667         struct bch_sb_field_ext *ext = bch2_sb_field_get(c->disk_sb.sb, ext);
 668         bool write_sb = false;
 669
 670         if (BCH_SB_HAS_TOPOLOGY_ERRORS(c->disk_sb.sb)) {
 671                 ext->recovery_passes_required[0] |=
 672                         cpu_to_le64(bch2_recovery_passes_to_stable(BIT_ULL(BCH_RECOVERY_PASS_check_topology)));
 673                 write_sb = true;
 674         }
 675
 676         u64 sb_passes = bch2_recovery_passes_from_stable(le64_to_cpu(ext->recovery_passes_required[0]));
 677         if (sb_passes) {
 678                 struct printbuf buf = PRINTBUF;
 679                 prt_str(&buf, "superblock requires following recovery passes to be run:\n  ");
 680                 prt_bitflags(&buf, bch2_recovery_passes, sb_passes);
 681                 bch_info(c, "%s", buf.buf);
 682                 printbuf_exit(&buf);
 683         }
 684
 685         if (bch2_check_version_downgrade(c)) {
 686                 struct printbuf buf = PRINTBUF;
 687
 688                 prt_str(&buf, "Version downgrade required:");
 689
 690                 __le64 passes = ext->recovery_passes_required[0];
 691                 bch2_sb_set_downgrade(c,
 692                                       BCH_VERSION_MINOR(bcachefs_metadata_version_current),
 693                                       BCH_VERSION_MINOR(c->sb.version));
 694                 passes = ext->recovery_passes_required[0] & ~passes;
 695                 if (passes) {
 696                         prt_str(&buf, "\n  running recovery passes: ");
 697                         prt_bitflags(&buf, bch2_recovery_passes,
 698                                      bch2_recovery_passes_from_stable(le64_to_cpu(passes)));
 699                 }
 700
 701                 bch_info(c, "%s", buf.buf);
 702                 printbuf_exit(&buf);
 703                 write_sb = true;
 704         }
 705
 706         if (check_version_upgrade(c))
 707                 write_sb = true;
 708
 709         c->opts.recovery_passes |= bch2_recovery_passes_from_stable(le64_to_cpu(ext->recovery_passes_required[0]));
 710
 711         if (write_sb)
 712                 bch2_write_super(c);
 713         mutex_unlock(&c->sb_lock);
 714
 715         if (c->opts.fsck && IS_ENABLED(CONFIG_BCACHEFS_DEBUG))
 716                 c->opts.recovery_passes |= BIT_ULL(BCH_RECOVERY_PASS_check_topology);
 717
 718         if (c->opts.fsck)
 719                 set_bit(BCH_FS_fsck_running, &c->flags);
 720         if (c->sb.clean)
 721                 set_bit(BCH_FS_clean_recovery, &c->flags);
 722
 723         ret = bch2_blacklist_table_initialize(c);
 724         if (ret) {
 725                 bch_err(c, "error initializing blacklist table");
 726                 goto err;
 727         }
 728
 729         bch2_journal_pos_from_member_info_resume(c);
 730
 731         if (!c->sb.clean || c->opts.retain_recovery_info) {
 732                 struct genradix_iter iter;
 733                 struct journal_replay **i;
 734
 735                 bch_verbose(c, "starting journal read");
 736                 ret = bch2_journal_read(c, &last_seq, &blacklist_seq, &journal_seq);
 737                 if (ret)
 738                         goto err;
 739
 740                 /*
 741                  * note: cmd_list_journal needs the blacklist table fully up to date so
 742                  * it can asterisk ignored journal entries:
 743                  */
 744                 if (c->opts.read_journal_only)
 745                         goto out;
 746
 747                 genradix_for_each_reverse(&c->journal_entries, iter, i)
 748                         if (!journal_replay_ignore(*i)) {
 749                                 last_journal_entry = &(*i)->j;
 750                                 break;
 751                         }
 752
 753                 if (mustfix_fsck_err_on(c->sb.clean &&
 754                                         last_journal_entry &&
 755                                         !journal_entry_empty(last_journal_entry), c,
 756                                 clean_but_journal_not_empty,
 757                                 "filesystem marked clean but journal not empty")) {
 758                         c->sb.compat &= ~(1ULL << BCH_COMPAT_alloc_info);
 759                         SET_BCH_SB_CLEAN(c->disk_sb.sb, false);
 760                         c->sb.clean = false;
 761                 }
 762
 763                 if (!last_journal_entry) {
 764                         fsck_err_on(!c->sb.clean, c,
 765                                     dirty_but_no_journal_entries,
 766                                     "no journal entries found");
 767                         if (clean)
 768                                 goto use_clean;
 769
 770                         genradix_for_each_reverse(&c->journal_entries, iter, i)
 771                                 if (*i) {
 772                                         last_journal_entry = &(*i)->j;
 773                                         (*i)->ignore_blacklisted = false;
 774                                         (*i)->ignore_not_dirty= false;
 775                                         /*
 776                                          * This was probably a NO_FLUSH entry,
 777                                          * so last_seq was garbage - but we know
 778                                          * we're only using a single journal
 779                                          * entry, set it here:
 780                                          */
 781                                         (*i)->j.last_seq = (*i)->j.seq;
 782                                         break;
 783                                 }
 784                 }
 785
 786                 ret = bch2_journal_keys_sort(c);
 787                 if (ret)
 788                         goto err;
 789
 790                 if (c->sb.clean && last_journal_entry) {
 791                         ret = bch2_verify_superblock_clean(c, &clean,
 792                                                       last_journal_entry);
 793                         if (ret)
 794                                 goto err;
 795                 }
 796         } else {
 797 use_clean:
 798                 if (!clean) {
 799                         bch_err(c, "no superblock clean section found");
 800                         ret = -BCH_ERR_fsck_repair_impossible;
 801                         goto err;
 802
 803                 }
 804                 blacklist_seq = journal_seq = le64_to_cpu(clean->journal_seq) + 1;
 805         }
 806
 807         c->journal_replay_seq_start     = last_seq;
 808         c->journal_replay_seq_end       = blacklist_seq - 1;
 809
 810         if (c->opts.reconstruct_alloc)
 811                 bch2_reconstruct_alloc(c);
 812
 813         zero_out_btree_mem_ptr(&c->journal_keys);
 814
 815         ret = journal_replay_early(c, clean);
 816         if (ret)
 817                 goto err;
 818
 819         /*
 820          * After an unclean shutdown, skip then next few journal sequence
 821          * numbers as they may have been referenced by btree writes that
 822          * happened before their corresponding journal writes - those btree
 823          * writes need to be ignored, by skipping and blacklisting the next few
 824          * journal sequence numbers:
 825          */
 826         if (!c->sb.clean)
 827                 journal_seq += 8;
 828
 829         if (blacklist_seq != journal_seq) {
 830                 ret =   bch2_journal_log_msg(c, "blacklisting entries %llu-%llu",
 831                                              blacklist_seq, journal_seq) ?:
 832                         bch2_journal_seq_blacklist_add(c,
 833                                         blacklist_seq, journal_seq);
 834                 if (ret) {
 835                         bch_err_msg(c, ret, "error creating new journal seq blacklist entry");
 836                         goto err;
 837                 }
 838         }
 839
 840         ret =   bch2_journal_log_msg(c, "starting journal at entry %llu, replaying %llu-%llu",
 841                                      journal_seq, last_seq, blacklist_seq - 1) ?:
 842                 bch2_fs_journal_start(&c->journal, journal_seq);
 843         if (ret)
 844                 goto err;
 845
 846         /*
 847          * Skip past versions that might have possibly been used (as nonces),
 848          * but hadn't had their pointers written:
 849          */
 850         if (c->sb.encryption_type && !c->sb.clean)
 851                 atomic64_add(1 << 16, &c->key_version);
 852
 853         ret = read_btree_roots(c);
 854         if (ret)
 855                 goto err;
 856
 857         set_bit(BCH_FS_btree_running, &c->flags);
 858
 859         ret = bch2_sb_set_upgrade_extra(c);
 860
 861         ret = bch2_run_recovery_passes(c);
 862         if (ret)
 863                 goto err;
 864
 865         /*
 866          * Normally set by the appropriate recovery pass: when cleared, this
 867          * indicates we're in early recovery and btree updates should be done by
 868          * being applied to the journal replay keys. _Must_ be cleared before
 869          * multithreaded use:
 870          */
 871         set_bit(BCH_FS_may_go_rw, &c->flags);
 872         clear_bit(BCH_FS_fsck_running, &c->flags);
 873
 874         /* in case we don't run journal replay, i.e. norecovery mode */
 875         set_bit(BCH_FS_accounting_replay_done, &c->flags);
 876
 877         /* fsync if we fixed errors */
 878         if (test_bit(BCH_FS_errors_fixed, &c->flags) &&
 879             bch2_write_ref_tryget(c, BCH_WRITE_REF_fsync)) {
 880                 bch2_journal_flush_all_pins(&c->journal);
 881                 bch2_journal_meta(&c->journal);
 882                 bch2_write_ref_put(c, BCH_WRITE_REF_fsync);
 883         }
 884
 885         /* If we fixed errors, verify that fs is actually clean now: */
 886         if (IS_ENABLED(CONFIG_BCACHEFS_DEBUG) &&
 887             test_bit(BCH_FS_errors_fixed, &c->flags) &&
 888             !test_bit(BCH_FS_errors_not_fixed, &c->flags) &&
 889             !test_bit(BCH_FS_error, &c->flags)) {
 890                 bch2_flush_fsck_errs(c);
 891
 892                 bch_info(c, "Fixed errors, running fsck a second time to verify fs is clean");
 893                 clear_bit(BCH_FS_errors_fixed, &c->flags);
 894
 895                 c->curr_recovery_pass = BCH_RECOVERY_PASS_check_alloc_info;
 896
 897                 ret = bch2_run_recovery_passes(c);
 898                 if (ret)
 899                         goto err;
 900
 901                 if (test_bit(BCH_FS_errors_fixed, &c->flags) ||
 902                     test_bit(BCH_FS_errors_not_fixed, &c->flags)) {
 903                         bch_err(c, "Second fsck run was not clean");
 904                         set_bit(BCH_FS_errors_not_fixed, &c->flags);
 905                 }
 906
 907                 set_bit(BCH_FS_errors_fixed, &c->flags);
 908         }
 909
 910         if (enabled_qtypes(c)) {
 911                 bch_verbose(c, "reading quotas");
 912                 ret = bch2_fs_quota_read(c);
 913                 if (ret)
 914                         goto err;
 915                 bch_verbose(c, "quotas done");
 916         }
 917
 918         mutex_lock(&c->sb_lock);
 919         ext = bch2_sb_field_get(c->disk_sb.sb, ext);
 920         write_sb = false;
 921
 922         if (BCH_SB_VERSION_UPGRADE_COMPLETE(c->disk_sb.sb) != le16_to_cpu(c->disk_sb.sb->version)) {
 923                 SET_BCH_SB_VERSION_UPGRADE_COMPLETE(c->disk_sb.sb, le16_to_cpu(c->disk_sb.sb->version));
 924                 write_sb = true;
 925         }
 926
 927         if (!test_bit(BCH_FS_error, &c->flags) &&
 928             !(c->disk_sb.sb->compat[0] & cpu_to_le64(1ULL << BCH_COMPAT_alloc_info))) {
 929                 c->disk_sb.sb->compat[0] |= cpu_to_le64(1ULL << BCH_COMPAT_alloc_info);
 930                 write_sb = true;
 931         }
 932
 933         if (!test_bit(BCH_FS_error, &c->flags) &&
 934             !bch2_is_zero(ext->errors_silent, sizeof(ext->errors_silent))) {
 935                 memset(ext->errors_silent, 0, sizeof(ext->errors_silent));
 936                 write_sb = true;
 937         }
 938
 939         if (c->opts.fsck &&
 940             !test_bit(BCH_FS_error, &c->flags) &&
 941             c->recovery_pass_done == BCH_RECOVERY_PASS_NR - 1 &&
 942             ext->btrees_lost_data) {
 943                 ext->btrees_lost_data = 0;
 944                 write_sb = true;
 945         }
 946
 947         if (c->opts.fsck &&
 948             !test_bit(BCH_FS_error, &c->flags) &&
 949             !test_bit(BCH_FS_errors_not_fixed, &c->flags)) {
 950                 SET_BCH_SB_HAS_ERRORS(c->disk_sb.sb, 0);
 951                 SET_BCH_SB_HAS_TOPOLOGY_ERRORS(c->disk_sb.sb, 0);
 952                 write_sb = true;
 953         }
 954
 955         if (bch2_blacklist_entries_gc(c))
 956                 write_sb = true;
 957
 958         if (write_sb)
 959                 bch2_write_super(c);
 960         mutex_unlock(&c->sb_lock);
 961
 962         if (!(c->sb.compat & (1ULL << BCH_COMPAT_extents_above_btree_updates_done)) ||
 963             c->sb.version_min < bcachefs_metadata_version_btree_ptr_sectors_written) {
 964                 struct bch_move_stats stats;
 965
 966                 bch2_move_stats_init(&stats, "recovery");
 967
 968                 struct printbuf buf = PRINTBUF;
 969                 bch2_version_to_text(&buf, c->sb.version_min);
 970                 bch_info(c, "scanning for old btree nodes: min_version %s", buf.buf);
 971                 printbuf_exit(&buf);
 972
 973                 ret =   bch2_fs_read_write_early(c) ?:
 974                         bch2_scan_old_btree_nodes(c, &stats);
 975                 if (ret)
 976                         goto err;
 977                 bch_info(c, "scanning for old btree nodes done");
 978         }
 979
 980         ret = 0;
 981 out:
 982         bch2_flush_fsck_errs(c);
 983
 984         if (!c->opts.retain_recovery_info) {
 985                 bch2_journal_keys_put_initial(c);
 986                 bch2_find_btree_nodes_exit(&c->found_btree_nodes);
 987         }
 988         if (!IS_ERR(clean))
 989                 kfree(clean);
 990
 991         if (!ret &&
 992             test_bit(BCH_FS_need_delete_dead_snapshots, &c->flags) &&
 993             !c->opts.nochanges) {
 994                 bch2_fs_read_write_early(c);
 995                 bch2_delete_dead_snapshots_async(c);
 996         }
 997
 998         bch_err_fn(c, ret);
 999         return ret;
1000 err:
1001 fsck_err:
1002         bch2_fs_emergency_read_only(c);
1003         goto out;
1004 }
1005
1006 int bch2_fs_initialize(struct bch_fs *c)
1007 {
1008         struct bch_inode_unpacked root_inode, lostfound_inode;
1009         struct bkey_inode_buf packed_inode;
1010         struct qstr lostfound = QSTR("lost+found");
1011         struct bch_member *m;
1012         int ret;
1013
1014         bch_notice(c, "initializing new filesystem");
1015         set_bit(BCH_FS_new_fs, &c->flags);
1016
1017         mutex_lock(&c->sb_lock);
1018         c->disk_sb.sb->compat[0] |= cpu_to_le64(1ULL << BCH_COMPAT_extents_above_btree_updates_done);
1019         c->disk_sb.sb->compat[0] |= cpu_to_le64(1ULL << BCH_COMPAT_bformat_overflow_done);
1020
1021         bch2_check_version_downgrade(c);
1022
1023         if (c->opts.version_upgrade != BCH_VERSION_UPGRADE_none) {
1024                 bch2_sb_upgrade(c, bcachefs_metadata_version_current);
1025                 SET_BCH_SB_VERSION_UPGRADE_COMPLETE(c->disk_sb.sb, bcachefs_metadata_version_current);
1026                 bch2_write_super(c);
1027         }
1028
1029         for_each_member_device(c, ca) {
1030                 m = bch2_members_v2_get_mut(c->disk_sb.sb, ca->dev_idx);
1031                 SET_BCH_MEMBER_FREESPACE_INITIALIZED(m, false);
1032                 ca->mi = bch2_mi_to_cpu(m);
1033         }
1034
1035         bch2_write_super(c);
1036         mutex_unlock(&c->sb_lock);
1037
1038         c->curr_recovery_pass = BCH_RECOVERY_PASS_NR;
1039         set_bit(BCH_FS_btree_running, &c->flags);
1040         set_bit(BCH_FS_may_go_rw, &c->flags);
1041
1042         for (unsigned i = 0; i < BTREE_ID_NR; i++)
1043                 bch2_btree_root_alloc_fake(c, i, 0);
1044
1045         ret = bch2_fs_journal_alloc(c);
1046         if (ret)
1047                 goto err;
1048
1049         /*
1050          * journal_res_get() will crash if called before this has
1051          * set up the journal.pin FIFO and journal.cur pointer:
1052          */
1053         bch2_fs_journal_start(&c->journal, 1);
1054         set_bit(BCH_FS_accounting_replay_done, &c->flags);
1055         bch2_journal_set_replay_done(&c->journal);
1056
1057         ret = bch2_fs_read_write_early(c);
1058         if (ret)
1059                 goto err;
1060
1061         for_each_member_device(c, ca) {
1062                 ret = bch2_dev_usage_init(ca, false);
1063                 if (ret) {
1064                         bch2_dev_put(ca);
1065                         goto err;
1066                 }
1067         }
1068
1069         /*
1070          * Write out the superblock and journal buckets, now that we can do
1071          * btree updates
1072          */
1073         bch_verbose(c, "marking superblocks");
1074         ret = bch2_trans_mark_dev_sbs(c);
1075         bch_err_msg(c, ret, "marking superblocks");
1076         if (ret)
1077                 goto err;
1078
1079         for_each_online_member(c, ca)
1080                 ca->new_fs_bucket_idx = 0;
1081
1082         ret = bch2_fs_freespace_init(c);
1083         if (ret)
1084                 goto err;
1085
1086         ret = bch2_initialize_subvolumes(c);
1087         if (ret)
1088                 goto err;
1089
1090         bch_verbose(c, "reading snapshots table");
1091         ret = bch2_snapshots_read(c);
1092         if (ret)
1093                 goto err;
1094         bch_verbose(c, "reading snapshots done");
1095
1096         bch2_inode_init(c, &root_inode, 0, 0, S_IFDIR|0755, 0, NULL);
1097         root_inode.bi_inum      = BCACHEFS_ROOT_INO;
1098         root_inode.bi_subvol    = BCACHEFS_ROOT_SUBVOL;
1099         bch2_inode_pack(&packed_inode, &root_inode);
1100         packed_inode.inode.k.p.snapshot = U32_MAX;
1101
1102         ret = bch2_btree_insert(c, BTREE_ID_inodes, &packed_inode.inode.k_i, NULL, 0, 0);
1103         bch_err_msg(c, ret, "creating root directory");
1104         if (ret)
1105                 goto err;
1106
1107         bch2_inode_init_early(c, &lostfound_inode);
1108
1109         ret = bch2_trans_commit_do(c, NULL, NULL, 0,
1110                 bch2_create_trans(trans,
1111                                   BCACHEFS_ROOT_SUBVOL_INUM,
1112                                   &root_inode, &lostfound_inode,
1113                                   &lostfound,
1114                                   0, 0, S_IFDIR|0700, 0,
1115                                   NULL, NULL, (subvol_inum) { 0 }, 0));
1116         bch_err_msg(c, ret, "creating lost+found");
1117         if (ret)
1118                 goto err;
1119
1120         c->recovery_pass_done = BCH_RECOVERY_PASS_NR - 1;
1121
1122         if (enabled_qtypes(c)) {
1123                 ret = bch2_fs_quota_read(c);
1124                 if (ret)
1125                         goto err;
1126         }
1127
1128         ret = bch2_journal_flush(&c->journal);
1129         bch_err_msg(c, ret, "writing first journal entry");
1130         if (ret)
1131                 goto err;
1132
1133         mutex_lock(&c->sb_lock);
1134         SET_BCH_SB_INITIALIZED(c->disk_sb.sb, true);
1135         SET_BCH_SB_CLEAN(c->disk_sb.sb, false);
1136
1137         bch2_write_super(c);
1138         mutex_unlock(&c->sb_lock);
1139
1140         return 0;
1141 err:
1142         bch_err_fn(c, ret);
1143         return ret;
1144 }