fs/btrfs/zstd.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (c) 2016-present, Facebook, Inc.
   4  * All rights reserved.
   5  *
   6  */
   7
   8 #include <linux/bio.h>
   9 #include <linux/bitmap.h>
  10 #include <linux/err.h>
  11 #include <linux/init.h>
  12 #include <linux/kernel.h>
  13 #include <linux/mm.h>
  14 #include <linux/sched/mm.h>
  15 #include <linux/pagemap.h>
  16 #include <linux/refcount.h>
  17 #include <linux/sched.h>
  18 #include <linux/slab.h>
  19 #include <linux/zstd.h>
  20 #include "misc.h"
  21 #include "fs.h"
  22 #include "compression.h"
  23 #include "super.h"
  24
  25 #define ZSTD_BTRFS_MAX_WINDOWLOG 17
  26 #define ZSTD_BTRFS_MAX_INPUT (1 << ZSTD_BTRFS_MAX_WINDOWLOG)
  27 #define ZSTD_BTRFS_DEFAULT_LEVEL 3
  28 #define ZSTD_BTRFS_MAX_LEVEL 15
  29 /* 307s to avoid pathologically clashing with transaction commit */
  30 #define ZSTD_BTRFS_RECLAIM_JIFFIES (307 * HZ)
  31
  32 static zstd_parameters zstd_get_btrfs_parameters(unsigned int level,
  33                                                  size_t src_len)
  34 {
  35         zstd_parameters params = zstd_get_params(level, src_len);
  36
  37         if (params.cParams.windowLog > ZSTD_BTRFS_MAX_WINDOWLOG)
  38                 params.cParams.windowLog = ZSTD_BTRFS_MAX_WINDOWLOG;
  39         WARN_ON(src_len > ZSTD_BTRFS_MAX_INPUT);
  40         return params;
  41 }
  42
  43 struct workspace {
  44         void *mem;
  45         size_t size;
  46         char *buf;
  47         unsigned int level;
  48         unsigned int req_level;
  49         unsigned long last_used; /* jiffies */
  50         struct list_head list;
  51         struct list_head lru_list;
  52         zstd_in_buffer in_buf;
  53         zstd_out_buffer out_buf;
  54 };
  55
  56 /*
  57  * Zstd Workspace Management
  58  *
  59  * Zstd workspaces have different memory requirements depending on the level.
  60  * The zstd workspaces are managed by having individual lists for each level
  61  * and a global lru.  Forward progress is maintained by protecting a max level
  62  * workspace.
  63  *
  64  * Getting a workspace is done by using the bitmap to identify the levels that
  65  * have available workspaces and scans up.  This lets us recycle higher level
  66  * workspaces because of the monotonic memory guarantee.  A workspace's
  67  * last_used is only updated if it is being used by the corresponding memory
  68  * level.  Putting a workspace involves adding it back to the appropriate places
  69  * and adding it back to the lru if necessary.
  70  *
  71  * A timer is used to reclaim workspaces if they have not been used for
  72  * ZSTD_BTRFS_RECLAIM_JIFFIES.  This helps keep only active workspaces around.
  73  * The upper bound is provided by the workqueue limit which is 2 (percpu limit).
  74  */
  75
  76 struct zstd_workspace_manager {
  77         const struct btrfs_compress_op *ops;
  78         spinlock_t lock;
  79         struct list_head lru_list;
  80         struct list_head idle_ws[ZSTD_BTRFS_MAX_LEVEL];
  81         unsigned long active_map;
  82         wait_queue_head_t wait;
  83         struct timer_list timer;
  84 };
  85
  86 static struct zstd_workspace_manager wsm;
  87
  88 static size_t zstd_ws_mem_sizes[ZSTD_BTRFS_MAX_LEVEL];
  89
  90 static inline struct workspace *list_to_workspace(struct list_head *list)
  91 {
  92         return container_of(list, struct workspace, list);
  93 }
  94
  95 void zstd_free_workspace(struct list_head *ws);
  96 struct list_head *zstd_alloc_workspace(unsigned int level);
  97
  98 /*
  99  * Timer callback to free unused workspaces.
 100  *
 101  * @t: timer
 102  *
 103  * This scans the lru_list and attempts to reclaim any workspace that hasn't
 104  * been used for ZSTD_BTRFS_RECLAIM_JIFFIES.
 105  *
 106  * The context is softirq and does not need the _bh locking primitives.
 107  */
 108 static void zstd_reclaim_timer_fn(struct timer_list *timer)
 109 {
 110         unsigned long reclaim_threshold = jiffies - ZSTD_BTRFS_RECLAIM_JIFFIES;
 111         struct list_head *pos, *next;
 112
 113         spin_lock(&wsm.lock);
 114
 115         if (list_empty(&wsm.lru_list)) {
 116                 spin_unlock(&wsm.lock);
 117                 return;
 118         }
 119
 120         list_for_each_prev_safe(pos, next, &wsm.lru_list) {
 121                 struct workspace *victim = container_of(pos, struct workspace,
 122                                                         lru_list);
 123                 unsigned int level;
 124
 125                 if (time_after(victim->last_used, reclaim_threshold))
 126                         break;
 127
 128                 /* workspace is in use */
 129                 if (victim->req_level)
 130                         continue;
 131
 132                 level = victim->level;
 133                 list_del(&victim->lru_list);
 134                 list_del(&victim->list);
 135                 zstd_free_workspace(&victim->list);
 136
 137                 if (list_empty(&wsm.idle_ws[level - 1]))
 138                         clear_bit(level - 1, &wsm.active_map);
 139
 140         }
 141
 142         if (!list_empty(&wsm.lru_list))
 143                 mod_timer(&wsm.timer, jiffies + ZSTD_BTRFS_RECLAIM_JIFFIES);
 144
 145         spin_unlock(&wsm.lock);
 146 }
 147
 148 /*
 149  * Calculate monotonic memory bounds.
 150  *
 151  * It is possible based on the level configurations that a higher level
 152  * workspace uses less memory than a lower level workspace.  In order to reuse
 153  * workspaces, this must be made a monotonic relationship.  This precomputes
 154  * the required memory for each level and enforces the monotonicity between
 155  * level and memory required.
 156  */
 157 static void zstd_calc_ws_mem_sizes(void)
 158 {
 159         size_t max_size = 0;
 160         unsigned int level;
 161
 162         for (level = 1; level <= ZSTD_BTRFS_MAX_LEVEL; level++) {
 163                 zstd_parameters params =
 164                         zstd_get_btrfs_parameters(level, ZSTD_BTRFS_MAX_INPUT);
 165                 size_t level_size =
 166                         max_t(size_t,
 167                               zstd_cstream_workspace_bound(&params.cParams),
 168                               zstd_dstream_workspace_bound(ZSTD_BTRFS_MAX_INPUT));
 169
 170                 max_size = max_t(size_t, max_size, level_size);
 171                 zstd_ws_mem_sizes[level - 1] = max_size;
 172         }
 173 }
 174
 175 void zstd_init_workspace_manager(void)
 176 {
 177         struct list_head *ws;
 178         int i;
 179
 180         zstd_calc_ws_mem_sizes();
 181
 182         wsm.ops = &btrfs_zstd_compress;
 183         spin_lock_init(&wsm.lock);
 184         init_waitqueue_head(&wsm.wait);
 185         timer_setup(&wsm.timer, zstd_reclaim_timer_fn, 0);
 186
 187         INIT_LIST_HEAD(&wsm.lru_list);
 188         for (i = 0; i < ZSTD_BTRFS_MAX_LEVEL; i++)
 189                 INIT_LIST_HEAD(&wsm.idle_ws[i]);
 190
 191         ws = zstd_alloc_workspace(ZSTD_BTRFS_MAX_LEVEL);
 192         if (IS_ERR(ws)) {
 193                 pr_warn(
 194                 "BTRFS: cannot preallocate zstd compression workspace\n");
 195         } else {
 196                 set_bit(ZSTD_BTRFS_MAX_LEVEL - 1, &wsm.active_map);
 197                 list_add(ws, &wsm.idle_ws[ZSTD_BTRFS_MAX_LEVEL - 1]);
 198         }
 199 }
 200
 201 void zstd_cleanup_workspace_manager(void)
 202 {
 203         struct workspace *workspace;
 204         int i;
 205
 206         spin_lock_bh(&wsm.lock);
 207         for (i = 0; i < ZSTD_BTRFS_MAX_LEVEL; i++) {
 208                 while (!list_empty(&wsm.idle_ws[i])) {
 209                         workspace = container_of(wsm.idle_ws[i].next,
 210                                                  struct workspace, list);
 211                         list_del(&workspace->list);
 212                         list_del(&workspace->lru_list);
 213                         zstd_free_workspace(&workspace->list);
 214                 }
 215         }
 216         spin_unlock_bh(&wsm.lock);
 217
 218         del_timer_sync(&wsm.timer);
 219 }
 220
 221 /*
 222  * Find workspace for given level.
 223  *
 224  * @level: compression level
 225  *
 226  * This iterates over the set bits in the active_map beginning at the requested
 227  * compression level.  This lets us utilize already allocated workspaces before
 228  * allocating a new one.  If the workspace is of a larger size, it is used, but
 229  * the place in the lru_list and last_used times are not updated.  This is to
 230  * offer the opportunity to reclaim the workspace in favor of allocating an
 231  * appropriately sized one in the future.
 232  */
 233 static struct list_head *zstd_find_workspace(unsigned int level)
 234 {
 235         struct list_head *ws;
 236         struct workspace *workspace;
 237         int i = level - 1;
 238
 239         spin_lock_bh(&wsm.lock);
 240         for_each_set_bit_from(i, &wsm.active_map, ZSTD_BTRFS_MAX_LEVEL) {
 241                 if (!list_empty(&wsm.idle_ws[i])) {
 242                         ws = wsm.idle_ws[i].next;
 243                         workspace = list_to_workspace(ws);
 244                         list_del_init(ws);
 245                         /* keep its place if it's a lower level using this */
 246                         workspace->req_level = level;
 247                         if (level == workspace->level)
 248                                 list_del(&workspace->lru_list);
 249                         if (list_empty(&wsm.idle_ws[i]))
 250                                 clear_bit(i, &wsm.active_map);
 251                         spin_unlock_bh(&wsm.lock);
 252                         return ws;
 253                 }
 254         }
 255         spin_unlock_bh(&wsm.lock);
 256
 257         return NULL;
 258 }
 259
 260 /*
 261  * Zstd get_workspace for level.
 262  *
 263  * @level: compression level
 264  *
 265  * If @level is 0, then any compression level can be used.  Therefore, we begin
 266  * scanning from 1.  We first scan through possible workspaces and then after
 267  * attempt to allocate a new workspace.  If we fail to allocate one due to
 268  * memory pressure, go to sleep waiting for the max level workspace to free up.
 269  */
 270 struct list_head *zstd_get_workspace(unsigned int level)
 271 {
 272         struct list_head *ws;
 273         unsigned int nofs_flag;
 274
 275         /* level == 0 means we can use any workspace */
 276         if (!level)
 277                 level = 1;
 278
 279 again:
 280         ws = zstd_find_workspace(level);
 281         if (ws)
 282                 return ws;
 283
 284         nofs_flag = memalloc_nofs_save();
 285         ws = zstd_alloc_workspace(level);
 286         memalloc_nofs_restore(nofs_flag);
 287
 288         if (IS_ERR(ws)) {
 289                 DEFINE_WAIT(wait);
 290
 291                 prepare_to_wait(&wsm.wait, &wait, TASK_UNINTERRUPTIBLE);
 292                 schedule();
 293                 finish_wait(&wsm.wait, &wait);
 294
 295                 goto again;
 296         }
 297
 298         return ws;
 299 }
 300
 301 /*
 302  * Zstd put_workspace.
 303  *
 304  * @ws: list_head for the workspace
 305  *
 306  * When putting back a workspace, we only need to update the LRU if we are of
 307  * the requested compression level.  Here is where we continue to protect the
 308  * max level workspace or update last_used accordingly.  If the reclaim timer
 309  * isn't set, it is also set here.  Only the max level workspace tries and wakes
 310  * up waiting workspaces.
 311  */
 312 void zstd_put_workspace(struct list_head *ws)
 313 {
 314         struct workspace *workspace = list_to_workspace(ws);
 315
 316         spin_lock_bh(&wsm.lock);
 317
 318         /* A node is only taken off the lru if we are the corresponding level */
 319         if (workspace->req_level == workspace->level) {
 320                 /* Hide a max level workspace from reclaim */
 321                 if (list_empty(&wsm.idle_ws[ZSTD_BTRFS_MAX_LEVEL - 1])) {
 322                         INIT_LIST_HEAD(&workspace->lru_list);
 323                 } else {
 324                         workspace->last_used = jiffies;
 325                         list_add(&workspace->lru_list, &wsm.lru_list);
 326                         if (!timer_pending(&wsm.timer))
 327                                 mod_timer(&wsm.timer,
 328                                           jiffies + ZSTD_BTRFS_RECLAIM_JIFFIES);
 329                 }
 330         }
 331
 332         set_bit(workspace->level - 1, &wsm.active_map);
 333         list_add(&workspace->list, &wsm.idle_ws[workspace->level - 1]);
 334         workspace->req_level = 0;
 335
 336         spin_unlock_bh(&wsm.lock);
 337
 338         if (workspace->level == ZSTD_BTRFS_MAX_LEVEL)
 339                 cond_wake_up(&wsm.wait);
 340 }
 341
 342 void zstd_free_workspace(struct list_head *ws)
 343 {
 344         struct workspace *workspace = list_entry(ws, struct workspace, list);
 345
 346         kvfree(workspace->mem);
 347         kfree(workspace->buf);
 348         kfree(workspace);
 349 }
 350
 351 struct list_head *zstd_alloc_workspace(unsigned int level)
 352 {
 353         struct workspace *workspace;
 354
 355         workspace = kzalloc(sizeof(*workspace), GFP_KERNEL);
 356         if (!workspace)
 357                 return ERR_PTR(-ENOMEM);
 358
 359         workspace->size = zstd_ws_mem_sizes[level - 1];
 360         workspace->level = level;
 361         workspace->req_level = level;
 362         workspace->last_used = jiffies;
 363         workspace->mem = kvmalloc(workspace->size, GFP_KERNEL | __GFP_NOWARN);
 364         workspace->buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
 365         if (!workspace->mem || !workspace->buf)
 366                 goto fail;
 367
 368         INIT_LIST_HEAD(&workspace->list);
 369         INIT_LIST_HEAD(&workspace->lru_list);
 370
 371         return &workspace->list;
 372 fail:
 373         zstd_free_workspace(&workspace->list);
 374         return ERR_PTR(-ENOMEM);
 375 }
 376
 377 int zstd_compress_folios(struct list_head *ws, struct address_space *mapping,
 378                          u64 start, struct folio **folios, unsigned long *out_folios,
 379                          unsigned long *total_in, unsigned long *total_out)
 380 {
 381         struct workspace *workspace = list_entry(ws, struct workspace, list);
 382         zstd_cstream *stream;
 383         int ret = 0;
 384         int nr_folios = 0;
 385         struct folio *in_folio = NULL;  /* The current folio to read. */
 386         struct folio *out_folio = NULL; /* The current folio to write to. */
 387         unsigned long tot_in = 0;
 388         unsigned long tot_out = 0;
 389         unsigned long len = *total_out;
 390         const unsigned long nr_dest_folios = *out_folios;
 391         unsigned long max_out = nr_dest_folios * PAGE_SIZE;
 392         zstd_parameters params = zstd_get_btrfs_parameters(workspace->req_level,
 393                                                            len);
 394
 395         *out_folios = 0;
 396         *total_out = 0;
 397         *total_in = 0;
 398
 399         /* Initialize the stream */
 400         stream = zstd_init_cstream(&params, len, workspace->mem,
 401                         workspace->size);
 402         if (!stream) {
 403                 pr_warn("BTRFS: zstd_init_cstream failed\n");
 404                 ret = -EIO;
 405                 goto out;
 406         }
 407
 408         /* map in the first page of input data */
 409         ret = btrfs_compress_filemap_get_folio(mapping, start, &in_folio);
 410         if (ret < 0)
 411                 goto out;
 412         workspace->in_buf.src = kmap_local_folio(in_folio, 0);
 413         workspace->in_buf.pos = 0;
 414         workspace->in_buf.size = min_t(size_t, len, PAGE_SIZE);
 415
 416         /* Allocate and map in the output buffer */
 417         out_folio = btrfs_alloc_compr_folio();
 418         if (out_folio == NULL) {
 419                 ret = -ENOMEM;
 420                 goto out;
 421         }
 422         folios[nr_folios++] = out_folio;
 423         workspace->out_buf.dst = folio_address(out_folio);
 424         workspace->out_buf.pos = 0;
 425         workspace->out_buf.size = min_t(size_t, max_out, PAGE_SIZE);
 426
 427         while (1) {
 428                 size_t ret2;
 429
 430                 ret2 = zstd_compress_stream(stream, &workspace->out_buf,
 431                                 &workspace->in_buf);
 432                 if (zstd_is_error(ret2)) {
 433                         pr_debug("BTRFS: zstd_compress_stream returned %d\n",
 434                                         zstd_get_error_code(ret2));
 435                         ret = -EIO;
 436                         goto out;
 437                 }
 438
 439                 /* Check to see if we are making it bigger */
 440                 if (tot_in + workspace->in_buf.pos > 8192 &&
 441                                 tot_in + workspace->in_buf.pos <
 442                                 tot_out + workspace->out_buf.pos) {
 443                         ret = -E2BIG;
 444                         goto out;
 445                 }
 446
 447                 /* We've reached the end of our output range */
 448                 if (workspace->out_buf.pos >= max_out) {
 449                         tot_out += workspace->out_buf.pos;
 450                         ret = -E2BIG;
 451                         goto out;
 452                 }
 453
 454                 /* Check if we need more output space */
 455                 if (workspace->out_buf.pos == workspace->out_buf.size) {
 456                         tot_out += PAGE_SIZE;
 457                         max_out -= PAGE_SIZE;
 458                         if (nr_folios == nr_dest_folios) {
 459                                 ret = -E2BIG;
 460                                 goto out;
 461                         }
 462                         out_folio = btrfs_alloc_compr_folio();
 463                         if (out_folio == NULL) {
 464                                 ret = -ENOMEM;
 465                                 goto out;
 466                         }
 467                         folios[nr_folios++] = out_folio;
 468                         workspace->out_buf.dst = folio_address(out_folio);
 469                         workspace->out_buf.pos = 0;
 470                         workspace->out_buf.size = min_t(size_t, max_out,
 471                                                         PAGE_SIZE);
 472                 }
 473
 474                 /* We've reached the end of the input */
 475                 if (workspace->in_buf.pos >= len) {
 476                         tot_in += workspace->in_buf.pos;
 477                         break;
 478                 }
 479
 480                 /* Check if we need more input */
 481                 if (workspace->in_buf.pos == workspace->in_buf.size) {
 482                         tot_in += PAGE_SIZE;
 483                         kunmap_local(workspace->in_buf.src);
 484                         workspace->in_buf.src = NULL;
 485                         folio_put(in_folio);
 486                         start += PAGE_SIZE;
 487                         len -= PAGE_SIZE;
 488                         ret = btrfs_compress_filemap_get_folio(mapping, start, &in_folio);
 489                         if (ret < 0)
 490                                 goto out;
 491                         workspace->in_buf.src = kmap_local_folio(in_folio, 0);
 492                         workspace->in_buf.pos = 0;
 493                         workspace->in_buf.size = min_t(size_t, len, PAGE_SIZE);
 494                 }
 495         }
 496         while (1) {
 497                 size_t ret2;
 498
 499                 ret2 = zstd_end_stream(stream, &workspace->out_buf);
 500                 if (zstd_is_error(ret2)) {
 501                         pr_debug("BTRFS: zstd_end_stream returned %d\n",
 502                                         zstd_get_error_code(ret2));
 503                         ret = -EIO;
 504                         goto out;
 505                 }
 506                 if (ret2 == 0) {
 507                         tot_out += workspace->out_buf.pos;
 508                         break;
 509                 }
 510                 if (workspace->out_buf.pos >= max_out) {
 511                         tot_out += workspace->out_buf.pos;
 512                         ret = -E2BIG;
 513                         goto out;
 514                 }
 515
 516                 tot_out += PAGE_SIZE;
 517                 max_out -= PAGE_SIZE;
 518                 if (nr_folios == nr_dest_folios) {
 519                         ret = -E2BIG;
 520                         goto out;
 521                 }
 522                 out_folio = btrfs_alloc_compr_folio();
 523                 if (out_folio == NULL) {
 524                         ret = -ENOMEM;
 525                         goto out;
 526                 }
 527                 folios[nr_folios++] = out_folio;
 528                 workspace->out_buf.dst = folio_address(out_folio);
 529                 workspace->out_buf.pos = 0;
 530                 workspace->out_buf.size = min_t(size_t, max_out, PAGE_SIZE);
 531         }
 532
 533         if (tot_out >= tot_in) {
 534                 ret = -E2BIG;
 535                 goto out;
 536         }
 537
 538         ret = 0;
 539         *total_in = tot_in;
 540         *total_out = tot_out;
 541 out:
 542         *out_folios = nr_folios;
 543         if (workspace->in_buf.src) {
 544                 kunmap_local(workspace->in_buf.src);
 545                 folio_put(in_folio);
 546         }
 547         return ret;
 548 }
 549
 550 int zstd_decompress_bio(struct list_head *ws, struct compressed_bio *cb)
 551 {
 552         struct workspace *workspace = list_entry(ws, struct workspace, list);
 553         struct folio **folios_in = cb->compressed_folios;
 554         size_t srclen = cb->compressed_len;
 555         zstd_dstream *stream;
 556         int ret = 0;
 557         unsigned long folio_in_index = 0;
 558         unsigned long total_folios_in = DIV_ROUND_UP(srclen, PAGE_SIZE);
 559         unsigned long buf_start;
 560         unsigned long total_out = 0;
 561
 562         stream = zstd_init_dstream(
 563                         ZSTD_BTRFS_MAX_INPUT, workspace->mem, workspace->size);
 564         if (!stream) {
 565                 pr_debug("BTRFS: zstd_init_dstream failed\n");
 566                 ret = -EIO;
 567                 goto done;
 568         }
 569
 570         workspace->in_buf.src = kmap_local_folio(folios_in[folio_in_index], 0);
 571         workspace->in_buf.pos = 0;
 572         workspace->in_buf.size = min_t(size_t, srclen, PAGE_SIZE);
 573
 574         workspace->out_buf.dst = workspace->buf;
 575         workspace->out_buf.pos = 0;
 576         workspace->out_buf.size = PAGE_SIZE;
 577
 578         while (1) {
 579                 size_t ret2;
 580
 581                 ret2 = zstd_decompress_stream(stream, &workspace->out_buf,
 582                                 &workspace->in_buf);
 583                 if (zstd_is_error(ret2)) {
 584                         pr_debug("BTRFS: zstd_decompress_stream returned %d\n",
 585                                         zstd_get_error_code(ret2));
 586                         ret = -EIO;
 587                         goto done;
 588                 }
 589                 buf_start = total_out;
 590                 total_out += workspace->out_buf.pos;
 591                 workspace->out_buf.pos = 0;
 592
 593                 ret = btrfs_decompress_buf2page(workspace->out_buf.dst,
 594                                 total_out - buf_start, cb, buf_start);
 595                 if (ret == 0)
 596                         break;
 597
 598                 if (workspace->in_buf.pos >= srclen)
 599                         break;
 600
 601                 /* Check if we've hit the end of a frame */
 602                 if (ret2 == 0)
 603                         break;
 604
 605                 if (workspace->in_buf.pos == workspace->in_buf.size) {
 606                         kunmap_local(workspace->in_buf.src);
 607                         folio_in_index++;
 608                         if (folio_in_index >= total_folios_in) {
 609                                 workspace->in_buf.src = NULL;
 610                                 ret = -EIO;
 611                                 goto done;
 612                         }
 613                         srclen -= PAGE_SIZE;
 614                         workspace->in_buf.src =
 615                                 kmap_local_folio(folios_in[folio_in_index], 0);
 616                         workspace->in_buf.pos = 0;
 617                         workspace->in_buf.size = min_t(size_t, srclen, PAGE_SIZE);
 618                 }
 619         }
 620         ret = 0;
 621 done:
 622         if (workspace->in_buf.src)
 623                 kunmap_local(workspace->in_buf.src);
 624         return ret;
 625 }
 626
 627 int zstd_decompress(struct list_head *ws, const u8 *data_in,
 628                 struct page *dest_page, unsigned long dest_pgoff, size_t srclen,
 629                 size_t destlen)
 630 {
 631         struct workspace *workspace = list_entry(ws, struct workspace, list);
 632         struct btrfs_fs_info *fs_info = btrfs_sb(dest_page->mapping->host->i_sb);
 633         const u32 sectorsize = fs_info->sectorsize;
 634         zstd_dstream *stream;
 635         int ret = 0;
 636         unsigned long to_copy = 0;
 637
 638         stream = zstd_init_dstream(
 639                         ZSTD_BTRFS_MAX_INPUT, workspace->mem, workspace->size);
 640         if (!stream) {
 641                 pr_warn("BTRFS: zstd_init_dstream failed\n");
 642                 goto finish;
 643         }
 644
 645         workspace->in_buf.src = data_in;
 646         workspace->in_buf.pos = 0;
 647         workspace->in_buf.size = srclen;
 648
 649         workspace->out_buf.dst = workspace->buf;
 650         workspace->out_buf.pos = 0;
 651         workspace->out_buf.size = sectorsize;
 652
 653         /*
 654          * Since both input and output buffers should not exceed one sector,
 655          * one call should end the decompression.
 656          */
 657         ret = zstd_decompress_stream(stream, &workspace->out_buf, &workspace->in_buf);
 658         if (zstd_is_error(ret)) {
 659                 pr_warn_ratelimited("BTRFS: zstd_decompress_stream return %d\n",
 660                                     zstd_get_error_code(ret));
 661                 goto finish;
 662         }
 663         to_copy = workspace->out_buf.pos;
 664         memcpy_to_page(dest_page, dest_pgoff, workspace->out_buf.dst, to_copy);
 665 finish:
 666         /* Error or early end. */
 667         if (unlikely(to_copy < destlen)) {
 668                 ret = -EIO;
 669                 memzero_page(dest_page, dest_pgoff + to_copy, destlen - to_copy);
 670         }
 671         return ret;
 672 }
 673
 674 const struct btrfs_compress_op btrfs_zstd_compress = {
 675         /* ZSTD uses own workspace manager */
 676         .workspace_manager = NULL,
 677         .max_level      = ZSTD_BTRFS_MAX_LEVEL,
 678         .default_level  = ZSTD_BTRFS_DEFAULT_LEVEL,
 679 };