fs/bcachefs/alloc_foreground.h

   1 /* SPDX-License-Identifier: GPL-2.0 */
   2 #ifndef _BCACHEFS_ALLOC_FOREGROUND_H
   3 #define _BCACHEFS_ALLOC_FOREGROUND_H
   4
   5 #include "bcachefs.h"
   6 #include "alloc_types.h"
   7 #include "extents.h"
   8 #include "sb-members.h"
   9
  10 #include <linux/hash.h>
  11
  12 struct bkey;
  13 struct bch_dev;
  14 struct bch_fs;
  15 struct bch_devs_List;
  16
  17 extern const char * const bch2_watermarks[];
  18
  19 void bch2_reset_alloc_cursors(struct bch_fs *);
  20
  21 struct dev_alloc_list {
  22         unsigned        nr;
  23         u8              data[BCH_SB_MEMBERS_MAX];
  24 };
  25
  26 struct dev_alloc_list bch2_dev_alloc_list(struct bch_fs *,
  27                                           struct dev_stripe_state *,
  28                                           struct bch_devs_mask *);
  29 void bch2_dev_stripe_increment(struct bch_dev *, struct dev_stripe_state *);
  30
  31 static inline struct bch_dev *ob_dev(struct bch_fs *c, struct open_bucket *ob)
  32 {
  33         return bch2_dev_have_ref(c, ob->dev);
  34 }
  35
  36 struct open_bucket *bch2_bucket_alloc(struct bch_fs *, struct bch_dev *,
  37                                       enum bch_watermark, enum bch_data_type,
  38                                       struct closure *);
  39
  40 static inline void ob_push(struct bch_fs *c, struct open_buckets *obs,
  41                            struct open_bucket *ob)
  42 {
  43         BUG_ON(obs->nr >= ARRAY_SIZE(obs->v));
  44
  45         obs->v[obs->nr++] = ob - c->open_buckets;
  46 }
  47
  48 #define open_bucket_for_each(_c, _obs, _ob, _i)                         \
  49         for ((_i) = 0;                                                  \
  50              (_i) < (_obs)->nr &&                                       \
  51              ((_ob) = (_c)->open_buckets + (_obs)->v[_i], true);        \
  52              (_i)++)
  53
  54 static inline struct open_bucket *ec_open_bucket(struct bch_fs *c,
  55                                                  struct open_buckets *obs)
  56 {
  57         struct open_bucket *ob;
  58         unsigned i;
  59
  60         open_bucket_for_each(c, obs, ob, i)
  61                 if (ob->ec)
  62                         return ob;
  63
  64         return NULL;
  65 }
  66
  67 void bch2_open_bucket_write_error(struct bch_fs *,
  68                         struct open_buckets *, unsigned);
  69
  70 void __bch2_open_bucket_put(struct bch_fs *, struct open_bucket *);
  71
  72 static inline void bch2_open_bucket_put(struct bch_fs *c, struct open_bucket *ob)
  73 {
  74         if (atomic_dec_and_test(&ob->pin))
  75                 __bch2_open_bucket_put(c, ob);
  76 }
  77
  78 static inline void bch2_open_buckets_put(struct bch_fs *c,
  79                                          struct open_buckets *ptrs)
  80 {
  81         struct open_bucket *ob;
  82         unsigned i;
  83
  84         open_bucket_for_each(c, ptrs, ob, i)
  85                 bch2_open_bucket_put(c, ob);
  86         ptrs->nr = 0;
  87 }
  88
  89 static inline void bch2_alloc_sectors_done_inlined(struct bch_fs *c, struct write_point *wp)
  90 {
  91         struct open_buckets ptrs = { .nr = 0 }, keep = { .nr = 0 };
  92         struct open_bucket *ob;
  93         unsigned i;
  94
  95         open_bucket_for_each(c, &wp->ptrs, ob, i)
  96                 ob_push(c, !ob->sectors_free ? &ptrs : &keep, ob);
  97         wp->ptrs = keep;
  98
  99         mutex_unlock(&wp->lock);
 100
 101         bch2_open_buckets_put(c, &ptrs);
 102 }
 103
 104 static inline void bch2_open_bucket_get(struct bch_fs *c,
 105                                         struct write_point *wp,
 106                                         struct open_buckets *ptrs)
 107 {
 108         struct open_bucket *ob;
 109         unsigned i;
 110
 111         open_bucket_for_each(c, &wp->ptrs, ob, i) {
 112                 ob->data_type = wp->data_type;
 113                 atomic_inc(&ob->pin);
 114                 ob_push(c, ptrs, ob);
 115         }
 116 }
 117
 118 static inline open_bucket_idx_t *open_bucket_hashslot(struct bch_fs *c,
 119                                                   unsigned dev, u64 bucket)
 120 {
 121         return c->open_buckets_hash +
 122                 (jhash_3words(dev, bucket, bucket >> 32, 0) &
 123                  (OPEN_BUCKETS_COUNT - 1));
 124 }
 125
 126 static inline bool bch2_bucket_is_open(struct bch_fs *c, unsigned dev, u64 bucket)
 127 {
 128         open_bucket_idx_t slot = *open_bucket_hashslot(c, dev, bucket);
 129
 130         while (slot) {
 131                 struct open_bucket *ob = &c->open_buckets[slot];
 132
 133                 if (ob->dev == dev && ob->bucket == bucket)
 134                         return true;
 135
 136                 slot = ob->hash;
 137         }
 138
 139         return false;
 140 }
 141
 142 static inline bool bch2_bucket_is_open_safe(struct bch_fs *c, unsigned dev, u64 bucket)
 143 {
 144         bool ret;
 145
 146         if (bch2_bucket_is_open(c, dev, bucket))
 147                 return true;
 148
 149         spin_lock(&c->freelist_lock);
 150         ret = bch2_bucket_is_open(c, dev, bucket);
 151         spin_unlock(&c->freelist_lock);
 152
 153         return ret;
 154 }
 155
 156 enum bch_write_flags;
 157 int bch2_bucket_alloc_set_trans(struct btree_trans *, struct open_buckets *,
 158                       struct dev_stripe_state *, struct bch_devs_mask *,
 159                       unsigned, unsigned *, bool *, enum bch_write_flags,
 160                       enum bch_data_type, enum bch_watermark,
 161                       struct closure *);
 162
 163 int bch2_alloc_sectors_start_trans(struct btree_trans *,
 164                                    unsigned, unsigned,
 165                                    struct write_point_specifier,
 166                                    struct bch_devs_list *,
 167                                    unsigned, unsigned,
 168                                    enum bch_watermark,
 169                                    enum bch_write_flags,
 170                                    struct closure *,
 171                                    struct write_point **);
 172
 173 struct bch_extent_ptr bch2_ob_ptr(struct bch_fs *, struct open_bucket *);
 174
 175 /*
 176  * Append pointers to the space we just allocated to @k, and mark @sectors space
 177  * as allocated out of @ob
 178  */
 179 static inline void
 180 bch2_alloc_sectors_append_ptrs_inlined(struct bch_fs *c, struct write_point *wp,
 181                                        struct bkey_i *k, unsigned sectors,
 182                                        bool cached)
 183 {
 184         struct open_bucket *ob;
 185         unsigned i;
 186
 187         BUG_ON(sectors > wp->sectors_free);
 188         wp->sectors_free        -= sectors;
 189         wp->sectors_allocated   += sectors;
 190
 191         open_bucket_for_each(c, &wp->ptrs, ob, i) {
 192                 struct bch_dev *ca = ob_dev(c, ob);
 193                 struct bch_extent_ptr ptr = bch2_ob_ptr(c, ob);
 194
 195                 ptr.cached = cached ||
 196                         (!ca->mi.durability &&
 197                          wp->data_type == BCH_DATA_user);
 198
 199                 bch2_bkey_append_ptr(k, ptr);
 200
 201                 BUG_ON(sectors > ob->sectors_free);
 202                 ob->sectors_free -= sectors;
 203         }
 204 }
 205
 206 void bch2_alloc_sectors_append_ptrs(struct bch_fs *, struct write_point *,
 207                                     struct bkey_i *, unsigned, bool);
 208 void bch2_alloc_sectors_done(struct bch_fs *, struct write_point *);
 209
 210 void bch2_open_buckets_stop(struct bch_fs *c, struct bch_dev *, bool);
 211
 212 static inline struct write_point_specifier writepoint_hashed(unsigned long v)
 213 {
 214         return (struct write_point_specifier) { .v = v | 1 };
 215 }
 216
 217 static inline struct write_point_specifier writepoint_ptr(struct write_point *wp)
 218 {
 219         return (struct write_point_specifier) { .v = (unsigned long) wp };
 220 }
 221
 222 void bch2_fs_allocator_foreground_init(struct bch_fs *);
 223
 224 void bch2_open_bucket_to_text(struct printbuf *, struct bch_fs *, struct open_bucket *);
 225 void bch2_open_buckets_to_text(struct printbuf *, struct bch_fs *, struct bch_dev *);
 226 void bch2_open_buckets_partial_to_text(struct printbuf *, struct bch_fs *);
 227
 228 void bch2_write_points_to_text(struct printbuf *, struct bch_fs *);
 229
 230 void bch2_fs_alloc_debug_to_text(struct printbuf *, struct bch_fs *);
 231 void bch2_dev_alloc_debug_to_text(struct printbuf *, struct bch_dev *);
 232
 233 void __bch2_wait_on_allocator(struct bch_fs *, struct closure *);
 234 static inline void bch2_wait_on_allocator(struct bch_fs *c, struct closure *cl)
 235 {
 236         if (cl->closure_get_happened)
 237                 __bch2_wait_on_allocator(c, cl);
 238 }
 239
 240 #endif /* _BCACHEFS_ALLOC_FOREGROUND_H */