fs/bcachefs/inode.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 #include "bcachefs.h"
   4 #include "btree_key_cache.h"
   5 #include "btree_write_buffer.h"
   6 #include "bkey_methods.h"
   7 #include "btree_update.h"
   8 #include "buckets.h"
   9 #include "compress.h"
  10 #include "dirent.h"
  11 #include "error.h"
  12 #include "extents.h"
  13 #include "extent_update.h"
  14 #include "inode.h"
  15 #include "str_hash.h"
  16 #include "snapshot.h"
  17 #include "subvolume.h"
  18 #include "varint.h"
  19
  20 #include <linux/random.h>
  21
  22 #include <asm/unaligned.h>
  23
  24 #define x(name, ...)    #name,
  25 const char * const bch2_inode_opts[] = {
  26         BCH_INODE_OPTS()
  27         NULL,
  28 };
  29
  30 static const char * const bch2_inode_flag_strs[] = {
  31         BCH_INODE_FLAGS()
  32         NULL
  33 };
  34 #undef  x
  35
  36 static const u8 byte_table[8] = { 1, 2, 3, 4, 6, 8, 10, 13 };
  37
  38 static int inode_decode_field(const u8 *in, const u8 *end,
  39                               u64 out[2], unsigned *out_bits)
  40 {
  41         __be64 be[2] = { 0, 0 };
  42         unsigned bytes, shift;
  43         u8 *p;
  44
  45         if (in >= end)
  46                 return -1;
  47
  48         if (!*in)
  49                 return -1;
  50
  51         /*
  52          * position of highest set bit indicates number of bytes:
  53          * shift = number of bits to remove in high byte:
  54          */
  55         shift   = 8 - __fls(*in); /* 1 <= shift <= 8 */
  56         bytes   = byte_table[shift - 1];
  57
  58         if (in + bytes > end)
  59                 return -1;
  60
  61         p = (u8 *) be + 16 - bytes;
  62         memcpy(p, in, bytes);
  63         *p ^= (1 << 8) >> shift;
  64
  65         out[0] = be64_to_cpu(be[0]);
  66         out[1] = be64_to_cpu(be[1]);
  67         *out_bits = out[0] ? 64 + fls64(out[0]) : fls64(out[1]);
  68
  69         return bytes;
  70 }
  71
  72 static inline void bch2_inode_pack_inlined(struct bkey_inode_buf *packed,
  73                                            const struct bch_inode_unpacked *inode)
  74 {
  75         struct bkey_i_inode_v3 *k = &packed->inode;
  76         u8 *out = k->v.fields;
  77         u8 *end = (void *) &packed[1];
  78         u8 *last_nonzero_field = out;
  79         unsigned nr_fields = 0, last_nonzero_fieldnr = 0;
  80         unsigned bytes;
  81         int ret;
  82
  83         bkey_inode_v3_init(&packed->inode.k_i);
  84         packed->inode.k.p.offset        = inode->bi_inum;
  85         packed->inode.v.bi_journal_seq  = cpu_to_le64(inode->bi_journal_seq);
  86         packed->inode.v.bi_hash_seed    = inode->bi_hash_seed;
  87         packed->inode.v.bi_flags        = cpu_to_le64(inode->bi_flags);
  88         packed->inode.v.bi_sectors      = cpu_to_le64(inode->bi_sectors);
  89         packed->inode.v.bi_size         = cpu_to_le64(inode->bi_size);
  90         packed->inode.v.bi_version      = cpu_to_le64(inode->bi_version);
  91         SET_INODEv3_MODE(&packed->inode.v, inode->bi_mode);
  92         SET_INODEv3_FIELDS_START(&packed->inode.v, INODEv3_FIELDS_START_CUR);
  93
  94
  95 #define x(_name, _bits)                                                 \
  96         nr_fields++;                                                    \
  97                                                                         \
  98         if (inode->_name) {                                             \
  99                 ret = bch2_varint_encode_fast(out, inode->_name);       \
 100                 out += ret;                                             \
 101                                                                         \
 102                 if (_bits > 64)                                         \
 103                         *out++ = 0;                                     \
 104                                                                         \
 105                 last_nonzero_field = out;                               \
 106                 last_nonzero_fieldnr = nr_fields;                       \
 107         } else {                                                        \
 108                 *out++ = 0;                                             \
 109                                                                         \
 110                 if (_bits > 64)                                         \
 111                         *out++ = 0;                                     \
 112         }
 113
 114         BCH_INODE_FIELDS_v3()
 115 #undef  x
 116         BUG_ON(out > end);
 117
 118         out = last_nonzero_field;
 119         nr_fields = last_nonzero_fieldnr;
 120
 121         bytes = out - (u8 *) &packed->inode.v;
 122         set_bkey_val_bytes(&packed->inode.k, bytes);
 123         memset_u64s_tail(&packed->inode.v, 0, bytes);
 124
 125         SET_INODEv3_NR_FIELDS(&k->v, nr_fields);
 126
 127         if (IS_ENABLED(CONFIG_BCACHEFS_DEBUG)) {
 128                 struct bch_inode_unpacked unpacked;
 129
 130                 ret = bch2_inode_unpack(bkey_i_to_s_c(&packed->inode.k_i), &unpacked);
 131                 BUG_ON(ret);
 132                 BUG_ON(unpacked.bi_inum         != inode->bi_inum);
 133                 BUG_ON(unpacked.bi_hash_seed    != inode->bi_hash_seed);
 134                 BUG_ON(unpacked.bi_sectors      != inode->bi_sectors);
 135                 BUG_ON(unpacked.bi_size         != inode->bi_size);
 136                 BUG_ON(unpacked.bi_version      != inode->bi_version);
 137                 BUG_ON(unpacked.bi_mode         != inode->bi_mode);
 138
 139 #define x(_name, _bits) if (unpacked._name != inode->_name)             \
 140                         panic("unpacked %llu should be %llu",           \
 141                               (u64) unpacked._name, (u64) inode->_name);
 142                 BCH_INODE_FIELDS_v3()
 143 #undef  x
 144         }
 145 }
 146
 147 void bch2_inode_pack(struct bkey_inode_buf *packed,
 148                      const struct bch_inode_unpacked *inode)
 149 {
 150         bch2_inode_pack_inlined(packed, inode);
 151 }
 152
 153 static noinline int bch2_inode_unpack_v1(struct bkey_s_c_inode inode,
 154                                 struct bch_inode_unpacked *unpacked)
 155 {
 156         const u8 *in = inode.v->fields;
 157         const u8 *end = bkey_val_end(inode);
 158         u64 field[2];
 159         unsigned fieldnr = 0, field_bits;
 160         int ret;
 161
 162 #define x(_name, _bits)                                 \
 163         if (fieldnr++ == INODE_NR_FIELDS(inode.v)) {                    \
 164                 unsigned offset = offsetof(struct bch_inode_unpacked, _name);\
 165                 memset((void *) unpacked + offset, 0,                   \
 166                        sizeof(*unpacked) - offset);                     \
 167                 return 0;                                               \
 168         }                                                               \
 169                                                                         \
 170         ret = inode_decode_field(in, end, field, &field_bits);          \
 171         if (ret < 0)                                                    \
 172                 return ret;                                             \
 173                                                                         \
 174         if (field_bits > sizeof(unpacked->_name) * 8)                   \
 175                 return -1;                                              \
 176                                                                         \
 177         unpacked->_name = field[1];                                     \
 178         in += ret;
 179
 180         BCH_INODE_FIELDS_v2()
 181 #undef  x
 182
 183         /* XXX: signal if there were more fields than expected? */
 184         return 0;
 185 }
 186
 187 static int bch2_inode_unpack_v2(struct bch_inode_unpacked *unpacked,
 188                                 const u8 *in, const u8 *end,
 189                                 unsigned nr_fields)
 190 {
 191         unsigned fieldnr = 0;
 192         int ret;
 193         u64 v[2];
 194
 195 #define x(_name, _bits)                                                 \
 196         if (fieldnr < nr_fields) {                                      \
 197                 ret = bch2_varint_decode_fast(in, end, &v[0]);          \
 198                 if (ret < 0)                                            \
 199                         return ret;                                     \
 200                 in += ret;                                              \
 201                                                                         \
 202                 if (_bits > 64) {                                       \
 203                         ret = bch2_varint_decode_fast(in, end, &v[1]);  \
 204                         if (ret < 0)                                    \
 205                                 return ret;                             \
 206                         in += ret;                                      \
 207                 } else {                                                \
 208                         v[1] = 0;                                       \
 209                 }                                                       \
 210         } else {                                                        \
 211                 v[0] = v[1] = 0;                                        \
 212         }                                                               \
 213                                                                         \
 214         unpacked->_name = v[0];                                         \
 215         if (v[1] || v[0] != unpacked->_name)                            \
 216                 return -1;                                              \
 217         fieldnr++;
 218
 219         BCH_INODE_FIELDS_v2()
 220 #undef  x
 221
 222         /* XXX: signal if there were more fields than expected? */
 223         return 0;
 224 }
 225
 226 static int bch2_inode_unpack_v3(struct bkey_s_c k,
 227                                 struct bch_inode_unpacked *unpacked)
 228 {
 229         struct bkey_s_c_inode_v3 inode = bkey_s_c_to_inode_v3(k);
 230         const u8 *in = inode.v->fields;
 231         const u8 *end = bkey_val_end(inode);
 232         unsigned nr_fields = INODEv3_NR_FIELDS(inode.v);
 233         unsigned fieldnr = 0;
 234         int ret;
 235         u64 v[2];
 236
 237         unpacked->bi_inum       = inode.k->p.offset;
 238         unpacked->bi_journal_seq= le64_to_cpu(inode.v->bi_journal_seq);
 239         unpacked->bi_hash_seed  = inode.v->bi_hash_seed;
 240         unpacked->bi_flags      = le64_to_cpu(inode.v->bi_flags);
 241         unpacked->bi_sectors    = le64_to_cpu(inode.v->bi_sectors);
 242         unpacked->bi_size       = le64_to_cpu(inode.v->bi_size);
 243         unpacked->bi_version    = le64_to_cpu(inode.v->bi_version);
 244         unpacked->bi_mode       = INODEv3_MODE(inode.v);
 245
 246 #define x(_name, _bits)                                                 \
 247         if (fieldnr < nr_fields) {                                      \
 248                 ret = bch2_varint_decode_fast(in, end, &v[0]);          \
 249                 if (ret < 0)                                            \
 250                         return ret;                                     \
 251                 in += ret;                                              \
 252                                                                         \
 253                 if (_bits > 64) {                                       \
 254                         ret = bch2_varint_decode_fast(in, end, &v[1]);  \
 255                         if (ret < 0)                                    \
 256                                 return ret;                             \
 257                         in += ret;                                      \
 258                 } else {                                                \
 259                         v[1] = 0;                                       \
 260                 }                                                       \
 261         } else {                                                        \
 262                 v[0] = v[1] = 0;                                        \
 263         }                                                               \
 264                                                                         \
 265         unpacked->_name = v[0];                                         \
 266         if (v[1] || v[0] != unpacked->_name)                            \
 267                 return -1;                                              \
 268         fieldnr++;
 269
 270         BCH_INODE_FIELDS_v3()
 271 #undef  x
 272
 273         /* XXX: signal if there were more fields than expected? */
 274         return 0;
 275 }
 276
 277 static noinline int bch2_inode_unpack_slowpath(struct bkey_s_c k,
 278                                                struct bch_inode_unpacked *unpacked)
 279 {
 280         memset(unpacked, 0, sizeof(*unpacked));
 281
 282         switch (k.k->type) {
 283         case KEY_TYPE_inode: {
 284                 struct bkey_s_c_inode inode = bkey_s_c_to_inode(k);
 285
 286                 unpacked->bi_inum       = inode.k->p.offset;
 287                 unpacked->bi_journal_seq= 0;
 288                 unpacked->bi_hash_seed  = inode.v->bi_hash_seed;
 289                 unpacked->bi_flags      = le32_to_cpu(inode.v->bi_flags);
 290                 unpacked->bi_mode       = le16_to_cpu(inode.v->bi_mode);
 291
 292                 if (INODE_NEW_VARINT(inode.v)) {
 293                         return bch2_inode_unpack_v2(unpacked, inode.v->fields,
 294                                                     bkey_val_end(inode),
 295                                                     INODE_NR_FIELDS(inode.v));
 296                 } else {
 297                         return bch2_inode_unpack_v1(inode, unpacked);
 298                 }
 299                 break;
 300         }
 301         case KEY_TYPE_inode_v2: {
 302                 struct bkey_s_c_inode_v2 inode = bkey_s_c_to_inode_v2(k);
 303
 304                 unpacked->bi_inum       = inode.k->p.offset;
 305                 unpacked->bi_journal_seq= le64_to_cpu(inode.v->bi_journal_seq);
 306                 unpacked->bi_hash_seed  = inode.v->bi_hash_seed;
 307                 unpacked->bi_flags      = le64_to_cpu(inode.v->bi_flags);
 308                 unpacked->bi_mode       = le16_to_cpu(inode.v->bi_mode);
 309
 310                 return bch2_inode_unpack_v2(unpacked, inode.v->fields,
 311                                             bkey_val_end(inode),
 312                                             INODEv2_NR_FIELDS(inode.v));
 313         }
 314         default:
 315                 BUG();
 316         }
 317 }
 318
 319 int bch2_inode_unpack(struct bkey_s_c k,
 320                       struct bch_inode_unpacked *unpacked)
 321 {
 322         if (likely(k.k->type == KEY_TYPE_inode_v3))
 323                 return bch2_inode_unpack_v3(k, unpacked);
 324         return bch2_inode_unpack_slowpath(k, unpacked);
 325 }
 326
 327 int bch2_inode_peek_nowarn(struct btree_trans *trans,
 328                     struct btree_iter *iter,
 329                     struct bch_inode_unpacked *inode,
 330                     subvol_inum inum, unsigned flags)
 331 {
 332         struct bkey_s_c k;
 333         u32 snapshot;
 334         int ret;
 335
 336         ret = bch2_subvolume_get_snapshot(trans, inum.subvol, &snapshot);
 337         if (ret)
 338                 return ret;
 339
 340         k = bch2_bkey_get_iter(trans, iter, BTREE_ID_inodes,
 341                                SPOS(0, inum.inum, snapshot),
 342                                flags|BTREE_ITER_cached);
 343         ret = bkey_err(k);
 344         if (ret)
 345                 return ret;
 346
 347         ret = bkey_is_inode(k.k) ? 0 : -BCH_ERR_ENOENT_inode;
 348         if (ret)
 349                 goto err;
 350
 351         ret = bch2_inode_unpack(k, inode);
 352         if (ret)
 353                 goto err;
 354
 355         return 0;
 356 err:
 357         bch2_trans_iter_exit(trans, iter);
 358         return ret;
 359 }
 360
 361 int bch2_inode_peek(struct btree_trans *trans,
 362                     struct btree_iter *iter,
 363                     struct bch_inode_unpacked *inode,
 364                     subvol_inum inum, unsigned flags)
 365 {
 366         int ret = bch2_inode_peek_nowarn(trans, iter, inode, inum, flags);
 367         bch_err_msg(trans->c, ret, "looking up inum %u:%llu:", inum.subvol, inum.inum);
 368         return ret;
 369 }
 370
 371 int bch2_inode_write_flags(struct btree_trans *trans,
 372                      struct btree_iter *iter,
 373                      struct bch_inode_unpacked *inode,
 374                      enum btree_iter_update_trigger_flags flags)
 375 {
 376         struct bkey_inode_buf *inode_p;
 377
 378         inode_p = bch2_trans_kmalloc(trans, sizeof(*inode_p));
 379         if (IS_ERR(inode_p))
 380                 return PTR_ERR(inode_p);
 381
 382         bch2_inode_pack_inlined(inode_p, inode);
 383         inode_p->inode.k.p.snapshot = iter->snapshot;
 384         return bch2_trans_update(trans, iter, &inode_p->inode.k_i, flags);
 385 }
 386
 387 int __bch2_fsck_write_inode(struct btree_trans *trans,
 388                          struct bch_inode_unpacked *inode,
 389                          u32 snapshot)
 390 {
 391         struct bkey_inode_buf *inode_p =
 392                 bch2_trans_kmalloc(trans, sizeof(*inode_p));
 393
 394         if (IS_ERR(inode_p))
 395                 return PTR_ERR(inode_p);
 396
 397         bch2_inode_pack(inode_p, inode);
 398         inode_p->inode.k.p.snapshot = snapshot;
 399
 400         return bch2_btree_insert_nonextent(trans, BTREE_ID_inodes,
 401                                 &inode_p->inode.k_i,
 402                                 BTREE_UPDATE_internal_snapshot_node);
 403 }
 404
 405 int bch2_fsck_write_inode(struct btree_trans *trans,
 406                             struct bch_inode_unpacked *inode,
 407                             u32 snapshot)
 408 {
 409         int ret = commit_do(trans, NULL, NULL, BCH_TRANS_COMMIT_no_enospc,
 410                             __bch2_fsck_write_inode(trans, inode, snapshot));
 411         bch_err_fn(trans->c, ret);
 412         return ret;
 413 }
 414
 415 struct bkey_i *bch2_inode_to_v3(struct btree_trans *trans, struct bkey_i *k)
 416 {
 417         struct bch_inode_unpacked u;
 418         struct bkey_inode_buf *inode_p;
 419         int ret;
 420
 421         if (!bkey_is_inode(&k->k))
 422                 return ERR_PTR(-ENOENT);
 423
 424         inode_p = bch2_trans_kmalloc(trans, sizeof(*inode_p));
 425         if (IS_ERR(inode_p))
 426                 return ERR_CAST(inode_p);
 427
 428         ret = bch2_inode_unpack(bkey_i_to_s_c(k), &u);
 429         if (ret)
 430                 return ERR_PTR(ret);
 431
 432         bch2_inode_pack(inode_p, &u);
 433         return &inode_p->inode.k_i;
 434 }
 435
 436 static int __bch2_inode_invalid(struct bch_fs *c, struct bkey_s_c k, struct printbuf *err)
 437 {
 438         struct bch_inode_unpacked unpacked;
 439         int ret = 0;
 440
 441         bkey_fsck_err_on(k.k->p.inode, c, err,
 442                          inode_pos_inode_nonzero,
 443                          "nonzero k.p.inode");
 444
 445         bkey_fsck_err_on(k.k->p.offset < BLOCKDEV_INODE_MAX, c, err,
 446                          inode_pos_blockdev_range,
 447                          "fs inode in blockdev range");
 448
 449         bkey_fsck_err_on(bch2_inode_unpack(k, &unpacked), c, err,
 450                          inode_unpack_error,
 451                          "invalid variable length fields");
 452
 453         bkey_fsck_err_on(unpacked.bi_data_checksum >= BCH_CSUM_OPT_NR + 1, c, err,
 454                          inode_checksum_type_invalid,
 455                          "invalid data checksum type (%u >= %u",
 456                          unpacked.bi_data_checksum, BCH_CSUM_OPT_NR + 1);
 457
 458         bkey_fsck_err_on(unpacked.bi_compression &&
 459                          !bch2_compression_opt_valid(unpacked.bi_compression - 1), c, err,
 460                          inode_compression_type_invalid,
 461                          "invalid compression opt %u", unpacked.bi_compression - 1);
 462
 463         bkey_fsck_err_on((unpacked.bi_flags & BCH_INODE_unlinked) &&
 464                          unpacked.bi_nlink != 0, c, err,
 465                          inode_unlinked_but_nlink_nonzero,
 466                          "flagged as unlinked but bi_nlink != 0");
 467
 468         bkey_fsck_err_on(unpacked.bi_subvol && !S_ISDIR(unpacked.bi_mode), c, err,
 469                          inode_subvol_root_but_not_dir,
 470                          "subvolume root but not a directory");
 471 fsck_err:
 472         return ret;
 473 }
 474
 475 int bch2_inode_invalid(struct bch_fs *c, struct bkey_s_c k,
 476                        enum bch_validate_flags flags,
 477                        struct printbuf *err)
 478 {
 479         struct bkey_s_c_inode inode = bkey_s_c_to_inode(k);
 480         int ret = 0;
 481
 482         bkey_fsck_err_on(INODE_STR_HASH(inode.v) >= BCH_STR_HASH_NR, c, err,
 483                          inode_str_hash_invalid,
 484                          "invalid str hash type (%llu >= %u)",
 485                          INODE_STR_HASH(inode.v), BCH_STR_HASH_NR);
 486
 487         ret = __bch2_inode_invalid(c, k, err);
 488 fsck_err:
 489         return ret;
 490 }
 491
 492 int bch2_inode_v2_invalid(struct bch_fs *c, struct bkey_s_c k,
 493                           enum bch_validate_flags flags,
 494                           struct printbuf *err)
 495 {
 496         struct bkey_s_c_inode_v2 inode = bkey_s_c_to_inode_v2(k);
 497         int ret = 0;
 498
 499         bkey_fsck_err_on(INODEv2_STR_HASH(inode.v) >= BCH_STR_HASH_NR, c, err,
 500                          inode_str_hash_invalid,
 501                          "invalid str hash type (%llu >= %u)",
 502                          INODEv2_STR_HASH(inode.v), BCH_STR_HASH_NR);
 503
 504         ret = __bch2_inode_invalid(c, k, err);
 505 fsck_err:
 506         return ret;
 507 }
 508
 509 int bch2_inode_v3_invalid(struct bch_fs *c, struct bkey_s_c k,
 510                           enum bch_validate_flags flags,
 511                           struct printbuf *err)
 512 {
 513         struct bkey_s_c_inode_v3 inode = bkey_s_c_to_inode_v3(k);
 514         int ret = 0;
 515
 516         bkey_fsck_err_on(INODEv3_FIELDS_START(inode.v) < INODEv3_FIELDS_START_INITIAL ||
 517                          INODEv3_FIELDS_START(inode.v) > bkey_val_u64s(inode.k), c, err,
 518                          inode_v3_fields_start_bad,
 519                          "invalid fields_start (got %llu, min %u max %zu)",
 520                          INODEv3_FIELDS_START(inode.v),
 521                          INODEv3_FIELDS_START_INITIAL,
 522                          bkey_val_u64s(inode.k));
 523
 524         bkey_fsck_err_on(INODEv3_STR_HASH(inode.v) >= BCH_STR_HASH_NR, c, err,
 525                          inode_str_hash_invalid,
 526                          "invalid str hash type (%llu >= %u)",
 527                          INODEv3_STR_HASH(inode.v), BCH_STR_HASH_NR);
 528
 529         ret = __bch2_inode_invalid(c, k, err);
 530 fsck_err:
 531         return ret;
 532 }
 533
 534 static void __bch2_inode_unpacked_to_text(struct printbuf *out,
 535                                           struct bch_inode_unpacked *inode)
 536 {
 537         printbuf_indent_add(out, 2);
 538         prt_printf(out, "mode=%o\n", inode->bi_mode);
 539
 540         prt_str(out, "flags=");
 541         prt_bitflags(out, bch2_inode_flag_strs, inode->bi_flags & ((1U << 20) - 1));
 542         prt_printf(out, " (%x)\n", inode->bi_flags);
 543
 544         prt_printf(out, "journal_seq=%llu\n",   inode->bi_journal_seq);
 545         prt_printf(out, "bi_size=%llu\n",       inode->bi_size);
 546         prt_printf(out, "bi_sectors=%llu\n",    inode->bi_sectors);
 547         prt_printf(out, "bi_version=%llu\n",    inode->bi_version);
 548
 549 #define x(_name, _bits)                                         \
 550         prt_printf(out, #_name "=%llu\n", (u64) inode->_name);
 551         BCH_INODE_FIELDS_v3()
 552 #undef  x
 553         printbuf_indent_sub(out, 2);
 554 }
 555
 556 void bch2_inode_unpacked_to_text(struct printbuf *out, struct bch_inode_unpacked *inode)
 557 {
 558         prt_printf(out, "inum: %llu ", inode->bi_inum);
 559         __bch2_inode_unpacked_to_text(out, inode);
 560 }
 561
 562 void bch2_inode_to_text(struct printbuf *out, struct bch_fs *c, struct bkey_s_c k)
 563 {
 564         struct bch_inode_unpacked inode;
 565
 566         if (bch2_inode_unpack(k, &inode)) {
 567                 prt_printf(out, "(unpack error)");
 568                 return;
 569         }
 570
 571         __bch2_inode_unpacked_to_text(out, &inode);
 572 }
 573
 574 static inline u64 bkey_inode_flags(struct bkey_s_c k)
 575 {
 576         switch (k.k->type) {
 577         case KEY_TYPE_inode:
 578                 return le32_to_cpu(bkey_s_c_to_inode(k).v->bi_flags);
 579         case KEY_TYPE_inode_v2:
 580                 return le64_to_cpu(bkey_s_c_to_inode_v2(k).v->bi_flags);
 581         case KEY_TYPE_inode_v3:
 582                 return le64_to_cpu(bkey_s_c_to_inode_v3(k).v->bi_flags);
 583         default:
 584                 return 0;
 585         }
 586 }
 587
 588 static inline bool bkey_is_deleted_inode(struct bkey_s_c k)
 589 {
 590         return bkey_inode_flags(k) & BCH_INODE_unlinked;
 591 }
 592
 593 int bch2_trigger_inode(struct btree_trans *trans,
 594                        enum btree_id btree_id, unsigned level,
 595                        struct bkey_s_c old,
 596                        struct bkey_s new,
 597                        enum btree_iter_update_trigger_flags flags)
 598 {
 599         s64 nr = (s64) bkey_is_inode(new.k) - (s64) bkey_is_inode(old.k);
 600
 601         if (flags & BTREE_TRIGGER_transactional) {
 602                 if (nr) {
 603                         int ret = bch2_replicas_deltas_realloc(trans, 0);
 604                         if (ret)
 605                                 return ret;
 606
 607                         trans->fs_usage_deltas->nr_inodes += nr;
 608                 }
 609
 610                 bool old_deleted = bkey_is_deleted_inode(old);
 611                 bool new_deleted = bkey_is_deleted_inode(new.s_c);
 612                 if (old_deleted != new_deleted) {
 613                         int ret = bch2_btree_bit_mod_buffered(trans, BTREE_ID_deleted_inodes,
 614                                                               new.k->p, new_deleted);
 615                         if (ret)
 616                                 return ret;
 617                 }
 618         }
 619
 620         if ((flags & BTREE_TRIGGER_atomic) && (flags & BTREE_TRIGGER_insert)) {
 621                 BUG_ON(!trans->journal_res.seq);
 622
 623                 bkey_s_to_inode_v3(new).v->bi_journal_seq = cpu_to_le64(trans->journal_res.seq);
 624         }
 625
 626         if (flags & BTREE_TRIGGER_gc) {
 627                 struct bch_fs *c = trans->c;
 628
 629                 percpu_down_read(&c->mark_lock);
 630                 this_cpu_add(c->usage_gc->b.nr_inodes, nr);
 631                 percpu_up_read(&c->mark_lock);
 632         }
 633
 634         return 0;
 635 }
 636
 637 int bch2_inode_generation_invalid(struct bch_fs *c, struct bkey_s_c k,
 638                                   enum bch_validate_flags flags,
 639                                   struct printbuf *err)
 640 {
 641         int ret = 0;
 642
 643         bkey_fsck_err_on(k.k->p.inode, c, err,
 644                          inode_pos_inode_nonzero,
 645                          "nonzero k.p.inode");
 646 fsck_err:
 647         return ret;
 648 }
 649
 650 void bch2_inode_generation_to_text(struct printbuf *out, struct bch_fs *c,
 651                                    struct bkey_s_c k)
 652 {
 653         struct bkey_s_c_inode_generation gen = bkey_s_c_to_inode_generation(k);
 654
 655         prt_printf(out, "generation: %u", le32_to_cpu(gen.v->bi_generation));
 656 }
 657
 658 void bch2_inode_init_early(struct bch_fs *c,
 659                            struct bch_inode_unpacked *inode_u)
 660 {
 661         enum bch_str_hash_type str_hash =
 662                 bch2_str_hash_opt_to_type(c, c->opts.str_hash);
 663
 664         memset(inode_u, 0, sizeof(*inode_u));
 665
 666         /* ick */
 667         inode_u->bi_flags |= str_hash << INODE_STR_HASH_OFFSET;
 668         get_random_bytes(&inode_u->bi_hash_seed,
 669                          sizeof(inode_u->bi_hash_seed));
 670 }
 671
 672 void bch2_inode_init_late(struct bch_inode_unpacked *inode_u, u64 now,
 673                           uid_t uid, gid_t gid, umode_t mode, dev_t rdev,
 674                           struct bch_inode_unpacked *parent)
 675 {
 676         inode_u->bi_mode        = mode;
 677         inode_u->bi_uid         = uid;
 678         inode_u->bi_gid         = gid;
 679         inode_u->bi_dev         = rdev;
 680         inode_u->bi_atime       = now;
 681         inode_u->bi_mtime       = now;
 682         inode_u->bi_ctime       = now;
 683         inode_u->bi_otime       = now;
 684
 685         if (parent && parent->bi_mode & S_ISGID) {
 686                 inode_u->bi_gid = parent->bi_gid;
 687                 if (S_ISDIR(mode))
 688                         inode_u->bi_mode |= S_ISGID;
 689         }
 690
 691         if (parent) {
 692 #define x(_name, ...)   inode_u->bi_##_name = parent->bi_##_name;
 693                 BCH_INODE_OPTS()
 694 #undef x
 695         }
 696 }
 697
 698 void bch2_inode_init(struct bch_fs *c, struct bch_inode_unpacked *inode_u,
 699                      uid_t uid, gid_t gid, umode_t mode, dev_t rdev,
 700                      struct bch_inode_unpacked *parent)
 701 {
 702         bch2_inode_init_early(c, inode_u);
 703         bch2_inode_init_late(inode_u, bch2_current_time(c),
 704                              uid, gid, mode, rdev, parent);
 705 }
 706
 707 static inline u32 bkey_generation(struct bkey_s_c k)
 708 {
 709         switch (k.k->type) {
 710         case KEY_TYPE_inode:
 711         case KEY_TYPE_inode_v2:
 712                 BUG();
 713         case KEY_TYPE_inode_generation:
 714                 return le32_to_cpu(bkey_s_c_to_inode_generation(k).v->bi_generation);
 715         default:
 716                 return 0;
 717         }
 718 }
 719
 720 /*
 721  * This just finds an empty slot:
 722  */
 723 int bch2_inode_create(struct btree_trans *trans,
 724                       struct btree_iter *iter,
 725                       struct bch_inode_unpacked *inode_u,
 726                       u32 snapshot, u64 cpu)
 727 {
 728         struct bch_fs *c = trans->c;
 729         struct bkey_s_c k;
 730         u64 min, max, start, pos, *hint;
 731         int ret = 0;
 732         unsigned bits = (c->opts.inodes_32bit ? 31 : 63);
 733
 734         if (c->opts.shard_inode_numbers) {
 735                 bits -= c->inode_shard_bits;
 736
 737                 min = (cpu << bits);
 738                 max = (cpu << bits) | ~(ULLONG_MAX << bits);
 739
 740                 min = max_t(u64, min, BLOCKDEV_INODE_MAX);
 741                 hint = c->unused_inode_hints + cpu;
 742         } else {
 743                 min = BLOCKDEV_INODE_MAX;
 744                 max = ~(ULLONG_MAX << bits);
 745                 hint = c->unused_inode_hints;
 746         }
 747
 748         start = READ_ONCE(*hint);
 749
 750         if (start >= max || start < min)
 751                 start = min;
 752
 753         pos = start;
 754         bch2_trans_iter_init(trans, iter, BTREE_ID_inodes, POS(0, pos),
 755                              BTREE_ITER_all_snapshots|
 756                              BTREE_ITER_intent);
 757 again:
 758         while ((k = bch2_btree_iter_peek(iter)).k &&
 759                !(ret = bkey_err(k)) &&
 760                bkey_lt(k.k->p, POS(0, max))) {
 761                 if (pos < iter->pos.offset)
 762                         goto found_slot;
 763
 764                 /*
 765                  * We don't need to iterate over keys in every snapshot once
 766                  * we've found just one:
 767                  */
 768                 pos = iter->pos.offset + 1;
 769                 bch2_btree_iter_set_pos(iter, POS(0, pos));
 770         }
 771
 772         if (!ret && pos < max)
 773                 goto found_slot;
 774
 775         if (!ret && start == min)
 776                 ret = -BCH_ERR_ENOSPC_inode_create;
 777
 778         if (ret) {
 779                 bch2_trans_iter_exit(trans, iter);
 780                 return ret;
 781         }
 782
 783         /* Retry from start */
 784         pos = start = min;
 785         bch2_btree_iter_set_pos(iter, POS(0, pos));
 786         goto again;
 787 found_slot:
 788         bch2_btree_iter_set_pos(iter, SPOS(0, pos, snapshot));
 789         k = bch2_btree_iter_peek_slot(iter);
 790         ret = bkey_err(k);
 791         if (ret) {
 792                 bch2_trans_iter_exit(trans, iter);
 793                 return ret;
 794         }
 795
 796         *hint                   = k.k->p.offset;
 797         inode_u->bi_inum        = k.k->p.offset;
 798         inode_u->bi_generation  = bkey_generation(k);
 799         return 0;
 800 }
 801
 802 static int bch2_inode_delete_keys(struct btree_trans *trans,
 803                                   subvol_inum inum, enum btree_id id)
 804 {
 805         struct btree_iter iter;
 806         struct bkey_s_c k;
 807         struct bkey_i delete;
 808         struct bpos end = POS(inum.inum, U64_MAX);
 809         u32 snapshot;
 810         int ret = 0;
 811
 812         /*
 813          * We're never going to be deleting partial extents, no need to use an
 814          * extent iterator:
 815          */
 816         bch2_trans_iter_init(trans, &iter, id, POS(inum.inum, 0),
 817                              BTREE_ITER_intent);
 818
 819         while (1) {
 820                 bch2_trans_begin(trans);
 821
 822                 ret = bch2_subvolume_get_snapshot(trans, inum.subvol, &snapshot);
 823                 if (ret)
 824                         goto err;
 825
 826                 bch2_btree_iter_set_snapshot(&iter, snapshot);
 827
 828                 k = bch2_btree_iter_peek_upto(&iter, end);
 829                 ret = bkey_err(k);
 830                 if (ret)
 831                         goto err;
 832
 833                 if (!k.k)
 834                         break;
 835
 836                 bkey_init(&delete.k);
 837                 delete.k.p = iter.pos;
 838
 839                 if (iter.flags & BTREE_ITER_is_extents)
 840                         bch2_key_resize(&delete.k,
 841                                         bpos_min(end, k.k->p).offset -
 842                                         iter.pos.offset);
 843
 844                 ret = bch2_trans_update(trans, &iter, &delete, 0) ?:
 845                       bch2_trans_commit(trans, NULL, NULL,
 846                                         BCH_TRANS_COMMIT_no_enospc);
 847 err:
 848                 if (ret && !bch2_err_matches(ret, BCH_ERR_transaction_restart))
 849                         break;
 850         }
 851
 852         bch2_trans_iter_exit(trans, &iter);
 853         return ret;
 854 }
 855
 856 int bch2_inode_rm(struct bch_fs *c, subvol_inum inum)
 857 {
 858         struct btree_trans *trans = bch2_trans_get(c);
 859         struct btree_iter iter = { NULL };
 860         struct bkey_i_inode_generation delete;
 861         struct bch_inode_unpacked inode_u;
 862         struct bkey_s_c k;
 863         u32 snapshot;
 864         int ret;
 865
 866         /*
 867          * If this was a directory, there shouldn't be any real dirents left -
 868          * but there could be whiteouts (from hash collisions) that we should
 869          * delete:
 870          *
 871          * XXX: the dirent could ideally would delete whiteouts when they're no
 872          * longer needed
 873          */
 874         ret   = bch2_inode_delete_keys(trans, inum, BTREE_ID_extents) ?:
 875                 bch2_inode_delete_keys(trans, inum, BTREE_ID_xattrs) ?:
 876                 bch2_inode_delete_keys(trans, inum, BTREE_ID_dirents);
 877         if (ret)
 878                 goto err;
 879 retry:
 880         bch2_trans_begin(trans);
 881
 882         ret = bch2_subvolume_get_snapshot(trans, inum.subvol, &snapshot);
 883         if (ret)
 884                 goto err;
 885
 886         k = bch2_bkey_get_iter(trans, &iter, BTREE_ID_inodes,
 887                                SPOS(0, inum.inum, snapshot),
 888                                BTREE_ITER_intent|BTREE_ITER_cached);
 889         ret = bkey_err(k);
 890         if (ret)
 891                 goto err;
 892
 893         if (!bkey_is_inode(k.k)) {
 894                 bch2_fs_inconsistent(c,
 895                                      "inode %llu:%u not found when deleting",
 896                                      inum.inum, snapshot);
 897                 ret = -EIO;
 898                 goto err;
 899         }
 900
 901         bch2_inode_unpack(k, &inode_u);
 902
 903         bkey_inode_generation_init(&delete.k_i);
 904         delete.k.p = iter.pos;
 905         delete.v.bi_generation = cpu_to_le32(inode_u.bi_generation + 1);
 906
 907         ret   = bch2_trans_update(trans, &iter, &delete.k_i, 0) ?:
 908                 bch2_trans_commit(trans, NULL, NULL,
 909                                 BCH_TRANS_COMMIT_no_enospc);
 910 err:
 911         bch2_trans_iter_exit(trans, &iter);
 912         if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
 913                 goto retry;
 914
 915         bch2_trans_put(trans);
 916         return ret;
 917 }
 918
 919 int bch2_inode_find_by_inum_nowarn_trans(struct btree_trans *trans,
 920                                   subvol_inum inum,
 921                                   struct bch_inode_unpacked *inode)
 922 {
 923         struct btree_iter iter;
 924         int ret;
 925
 926         ret = bch2_inode_peek_nowarn(trans, &iter, inode, inum, 0);
 927         if (!ret)
 928                 bch2_trans_iter_exit(trans, &iter);
 929         return ret;
 930 }
 931
 932 int bch2_inode_find_by_inum_trans(struct btree_trans *trans,
 933                                   subvol_inum inum,
 934                                   struct bch_inode_unpacked *inode)
 935 {
 936         struct btree_iter iter;
 937         int ret;
 938
 939         ret = bch2_inode_peek(trans, &iter, inode, inum, 0);
 940         if (!ret)
 941                 bch2_trans_iter_exit(trans, &iter);
 942         return ret;
 943 }
 944
 945 int bch2_inode_find_by_inum(struct bch_fs *c, subvol_inum inum,
 946                             struct bch_inode_unpacked *inode)
 947 {
 948         return bch2_trans_do(c, NULL, NULL, 0,
 949                 bch2_inode_find_by_inum_trans(trans, inum, inode));
 950 }
 951
 952 int bch2_inode_nlink_inc(struct bch_inode_unpacked *bi)
 953 {
 954         if (bi->bi_flags & BCH_INODE_unlinked)
 955                 bi->bi_flags &= ~BCH_INODE_unlinked;
 956         else {
 957                 if (bi->bi_nlink == U32_MAX)
 958                         return -EINVAL;
 959
 960                 bi->bi_nlink++;
 961         }
 962
 963         return 0;
 964 }
 965
 966 void bch2_inode_nlink_dec(struct btree_trans *trans, struct bch_inode_unpacked *bi)
 967 {
 968         if (bi->bi_nlink && (bi->bi_flags & BCH_INODE_unlinked)) {
 969                 bch2_trans_inconsistent(trans, "inode %llu unlinked but link count nonzero",
 970                                         bi->bi_inum);
 971                 return;
 972         }
 973
 974         if (bi->bi_flags & BCH_INODE_unlinked) {
 975                 bch2_trans_inconsistent(trans, "inode %llu link count underflow", bi->bi_inum);
 976                 return;
 977         }
 978
 979         if (bi->bi_nlink)
 980                 bi->bi_nlink--;
 981         else
 982                 bi->bi_flags |= BCH_INODE_unlinked;
 983 }
 984
 985 struct bch_opts bch2_inode_opts_to_opts(struct bch_inode_unpacked *inode)
 986 {
 987         struct bch_opts ret = { 0 };
 988 #define x(_name, _bits)                                                 \
 989         if (inode->bi_##_name)                                          \
 990                 opt_set(ret, _name, inode->bi_##_name - 1);
 991         BCH_INODE_OPTS()
 992 #undef x
 993         return ret;
 994 }
 995
 996 void bch2_inode_opts_get(struct bch_io_opts *opts, struct bch_fs *c,
 997                          struct bch_inode_unpacked *inode)
 998 {
 999 #define x(_name, _bits)         opts->_name = inode_opt_get(c, inode, _name);
1000         BCH_INODE_OPTS()
1001 #undef x
1002
1003         if (opts->nocow)
1004                 opts->compression = opts->background_compression = opts->data_checksum = opts->erasure_code = 0;
1005 }
1006
1007 int bch2_inum_opts_get(struct btree_trans *trans, subvol_inum inum, struct bch_io_opts *opts)
1008 {
1009         struct bch_inode_unpacked inode;
1010         int ret = lockrestart_do(trans, bch2_inode_find_by_inum_trans(trans, inum, &inode));
1011
1012         if (ret)
1013                 return ret;
1014
1015         bch2_inode_opts_get(opts, trans->c, &inode);
1016         return 0;
1017 }
1018
1019 int bch2_inode_rm_snapshot(struct btree_trans *trans, u64 inum, u32 snapshot)
1020 {
1021         struct bch_fs *c = trans->c;
1022         struct btree_iter iter = { NULL };
1023         struct bkey_i_inode_generation delete;
1024         struct bch_inode_unpacked inode_u;
1025         struct bkey_s_c k;
1026         int ret;
1027
1028         do {
1029                 ret   = bch2_btree_delete_range_trans(trans, BTREE_ID_extents,
1030                                                       SPOS(inum, 0, snapshot),
1031                                                       SPOS(inum, U64_MAX, snapshot),
1032                                                       0, NULL) ?:
1033                         bch2_btree_delete_range_trans(trans, BTREE_ID_dirents,
1034                                                       SPOS(inum, 0, snapshot),
1035                                                       SPOS(inum, U64_MAX, snapshot),
1036                                                       0, NULL) ?:
1037                         bch2_btree_delete_range_trans(trans, BTREE_ID_xattrs,
1038                                                       SPOS(inum, 0, snapshot),
1039                                                       SPOS(inum, U64_MAX, snapshot),
1040                                                       0, NULL);
1041         } while (ret == -BCH_ERR_transaction_restart_nested);
1042         if (ret)
1043                 goto err;
1044 retry:
1045         bch2_trans_begin(trans);
1046
1047         k = bch2_bkey_get_iter(trans, &iter, BTREE_ID_inodes,
1048                                SPOS(0, inum, snapshot), BTREE_ITER_intent);
1049         ret = bkey_err(k);
1050         if (ret)
1051                 goto err;
1052
1053         if (!bkey_is_inode(k.k)) {
1054                 bch2_fs_inconsistent(c,
1055                                      "inode %llu:%u not found when deleting",
1056                                      inum, snapshot);
1057                 ret = -EIO;
1058                 goto err;
1059         }
1060
1061         bch2_inode_unpack(k, &inode_u);
1062
1063         /* Subvolume root? */
1064         if (inode_u.bi_subvol)
1065                 bch_warn(c, "deleting inode %llu marked as unlinked, but also a subvolume root!?", inode_u.bi_inum);
1066
1067         bkey_inode_generation_init(&delete.k_i);
1068         delete.k.p = iter.pos;
1069         delete.v.bi_generation = cpu_to_le32(inode_u.bi_generation + 1);
1070
1071         ret   = bch2_trans_update(trans, &iter, &delete.k_i, 0) ?:
1072                 bch2_trans_commit(trans, NULL, NULL,
1073                                 BCH_TRANS_COMMIT_no_enospc);
1074 err:
1075         bch2_trans_iter_exit(trans, &iter);
1076         if (bch2_err_matches(ret, BCH_ERR_transaction_restart))
1077                 goto retry;
1078
1079         return ret ?: -BCH_ERR_transaction_restart_nested;
1080 }
1081
1082 static int may_delete_deleted_inode(struct btree_trans *trans,
1083                                     struct btree_iter *iter,
1084                                     struct bpos pos,
1085                                     bool *need_another_pass)
1086 {
1087         struct bch_fs *c = trans->c;
1088         struct btree_iter inode_iter;
1089         struct bkey_s_c k;
1090         struct bch_inode_unpacked inode;
1091         int ret;
1092
1093         k = bch2_bkey_get_iter(trans, &inode_iter, BTREE_ID_inodes, pos, BTREE_ITER_cached);
1094         ret = bkey_err(k);
1095         if (ret)
1096                 return ret;
1097
1098         ret = bkey_is_inode(k.k) ? 0 : -BCH_ERR_ENOENT_inode;
1099         if (fsck_err_on(!bkey_is_inode(k.k), c,
1100                         deleted_inode_missing,
1101                         "nonexistent inode %llu:%u in deleted_inodes btree",
1102                         pos.offset, pos.snapshot))
1103                 goto delete;
1104
1105         ret = bch2_inode_unpack(k, &inode);
1106         if (ret)
1107                 goto out;
1108
1109         if (S_ISDIR(inode.bi_mode)) {
1110                 ret = bch2_empty_dir_snapshot(trans, pos.offset, 0, pos.snapshot);
1111                 if (fsck_err_on(bch2_err_matches(ret, ENOTEMPTY),
1112                                 c, deleted_inode_is_dir,
1113                                 "non empty directory %llu:%u in deleted_inodes btree",
1114                                 pos.offset, pos.snapshot))
1115                         goto delete;
1116                 if (ret)
1117                         goto out;
1118         }
1119
1120         if (fsck_err_on(!(inode.bi_flags & BCH_INODE_unlinked), c,
1121                         deleted_inode_not_unlinked,
1122                         "non-deleted inode %llu:%u in deleted_inodes btree",
1123                         pos.offset, pos.snapshot))
1124                 goto delete;
1125
1126         if (c->sb.clean &&
1127             !fsck_err(c,
1128                       deleted_inode_but_clean,
1129                       "filesystem marked as clean but have deleted inode %llu:%u",
1130                       pos.offset, pos.snapshot)) {
1131                 ret = 0;
1132                 goto out;
1133         }
1134
1135         if (bch2_snapshot_is_internal_node(c, pos.snapshot)) {
1136                 struct bpos new_min_pos;
1137
1138                 ret = bch2_propagate_key_to_snapshot_leaves(trans, inode_iter.btree_id, k, &new_min_pos);
1139                 if (ret)
1140                         goto out;
1141
1142                 inode.bi_flags &= ~BCH_INODE_unlinked;
1143
1144                 ret = bch2_inode_write_flags(trans, &inode_iter, &inode,
1145                                              BTREE_UPDATE_internal_snapshot_node);
1146                 bch_err_msg(c, ret, "clearing inode unlinked flag");
1147                 if (ret)
1148                         goto out;
1149
1150                 /*
1151                  * We'll need another write buffer flush to pick up the new
1152                  * unlinked inodes in the snapshot leaves:
1153                  */
1154                 *need_another_pass = true;
1155                 goto out;
1156         }
1157
1158         ret = 1;
1159 out:
1160 fsck_err:
1161         bch2_trans_iter_exit(trans, &inode_iter);
1162         return ret;
1163 delete:
1164         ret = bch2_btree_bit_mod_buffered(trans, BTREE_ID_deleted_inodes, pos, false);
1165         goto out;
1166 }
1167
1168 int bch2_delete_dead_inodes(struct bch_fs *c)
1169 {
1170         struct btree_trans *trans = bch2_trans_get(c);
1171         bool need_another_pass;
1172         int ret;
1173 again:
1174         /*
1175          * if we ran check_inodes() unlinked inodes will have already been
1176          * cleaned up but the write buffer will be out of sync; therefore we
1177          * alway need a write buffer flush
1178          */
1179         ret = bch2_btree_write_buffer_flush_sync(trans);
1180         if (ret)
1181                 goto err;
1182
1183         need_another_pass = false;
1184
1185         /*
1186          * Weird transaction restart handling here because on successful delete,
1187          * bch2_inode_rm_snapshot() will return a nested transaction restart,
1188          * but we can't retry because the btree write buffer won't have been
1189          * flushed and we'd spin:
1190          */
1191         ret = for_each_btree_key_commit(trans, iter, BTREE_ID_deleted_inodes, POS_MIN,
1192                                         BTREE_ITER_prefetch|BTREE_ITER_all_snapshots, k,
1193                                         NULL, NULL, BCH_TRANS_COMMIT_no_enospc, ({
1194                 ret = may_delete_deleted_inode(trans, &iter, k.k->p, &need_another_pass);
1195                 if (ret > 0) {
1196                         bch_verbose(c, "deleting unlinked inode %llu:%u", k.k->p.offset, k.k->p.snapshot);
1197
1198                         ret = bch2_inode_rm_snapshot(trans, k.k->p.offset, k.k->p.snapshot);
1199                         /*
1200                          * We don't want to loop here: a transaction restart
1201                          * error here means we handled a transaction restart and
1202                          * we're actually done, but if we loop we'll retry the
1203                          * same key because the write buffer hasn't been flushed
1204                          * yet
1205                          */
1206                         if (bch2_err_matches(ret, BCH_ERR_transaction_restart)) {
1207                                 ret = 0;
1208                                 continue;
1209                         }
1210                 }
1211
1212                 ret;
1213         }));
1214
1215         if (!ret && need_another_pass)
1216                 goto again;
1217 err:
1218         bch2_trans_put(trans);
1219         return ret;
1220 }