fs/btrfs/zlib.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2008 Oracle.  All rights reserved.
   4  *
   5  * Based on jffs2 zlib code:
   6  * Copyright © 2001-2007 Red Hat, Inc.
   7  * Created by David Woodhouse <[email protected]>
   8  */
   9
  10 #include <linux/kernel.h>
  11 #include <linux/slab.h>
  12 #include <linux/zlib.h>
  13 #include <linux/zutil.h>
  14 #include <linux/mm.h>
  15 #include <linux/init.h>
  16 #include <linux/err.h>
  17 #include <linux/sched.h>
  18 #include <linux/pagemap.h>
  19 #include <linux/bio.h>
  20 #include <linux/refcount.h>
  21 #include "btrfs_inode.h"
  22 #include "compression.h"
  23 #include "fs.h"
  24 #include "subpage.h"
  25
  26 /* workspace buffer size for s390 zlib hardware support */
  27 #define ZLIB_DFLTCC_BUF_SIZE    (4 * PAGE_SIZE)
  28
  29 struct workspace {
  30         z_stream strm;
  31         char *buf;
  32         unsigned int buf_size;
  33         struct list_head list;
  34         int level;
  35 };
  36
  37 static struct workspace_manager wsm;
  38
  39 struct list_head *zlib_get_workspace(unsigned int level)
  40 {
  41         struct list_head *ws = btrfs_get_workspace(BTRFS_COMPRESS_ZLIB, level);
  42         struct workspace *workspace = list_entry(ws, struct workspace, list);
  43
  44         workspace->level = level;
  45
  46         return ws;
  47 }
  48
  49 void zlib_free_workspace(struct list_head *ws)
  50 {
  51         struct workspace *workspace = list_entry(ws, struct workspace, list);
  52
  53         kvfree(workspace->strm.workspace);
  54         kfree(workspace->buf);
  55         kfree(workspace);
  56 }
  57
  58 struct list_head *zlib_alloc_workspace(unsigned int level)
  59 {
  60         struct workspace *workspace;
  61         int workspacesize;
  62
  63         workspace = kzalloc(sizeof(*workspace), GFP_KERNEL);
  64         if (!workspace)
  65                 return ERR_PTR(-ENOMEM);
  66
  67         workspacesize = max(zlib_deflate_workspacesize(MAX_WBITS, MAX_MEM_LEVEL),
  68                         zlib_inflate_workspacesize());
  69         workspace->strm.workspace = kvzalloc(workspacesize, GFP_KERNEL | __GFP_NOWARN);
  70         workspace->level = level;
  71         workspace->buf = NULL;
  72         /*
  73          * In case of s390 zlib hardware support, allocate lager workspace
  74          * buffer. If allocator fails, fall back to a single page buffer.
  75          */
  76         if (zlib_deflate_dfltcc_enabled()) {
  77                 workspace->buf = kmalloc(ZLIB_DFLTCC_BUF_SIZE,
  78                                          __GFP_NOMEMALLOC | __GFP_NORETRY |
  79                                          __GFP_NOWARN | GFP_NOIO);
  80                 workspace->buf_size = ZLIB_DFLTCC_BUF_SIZE;
  81         }
  82         if (!workspace->buf) {
  83                 workspace->buf = kmalloc(PAGE_SIZE, GFP_KERNEL);
  84                 workspace->buf_size = PAGE_SIZE;
  85         }
  86         if (!workspace->strm.workspace || !workspace->buf)
  87                 goto fail;
  88
  89         INIT_LIST_HEAD(&workspace->list);
  90
  91         return &workspace->list;
  92 fail:
  93         zlib_free_workspace(&workspace->list);
  94         return ERR_PTR(-ENOMEM);
  95 }
  96
  97 int zlib_compress_folios(struct list_head *ws, struct address_space *mapping,
  98                          u64 start, struct folio **folios, unsigned long *out_folios,
  99                          unsigned long *total_in, unsigned long *total_out)
 100 {
 101         struct workspace *workspace = list_entry(ws, struct workspace, list);
 102         int ret;
 103         char *data_in = NULL;
 104         char *cfolio_out;
 105         int nr_folios = 0;
 106         struct folio *in_folio = NULL;
 107         struct folio *out_folio = NULL;
 108         unsigned long bytes_left;
 109         unsigned int in_buf_folios;
 110         unsigned long len = *total_out;
 111         unsigned long nr_dest_folios = *out_folios;
 112         const unsigned long max_out = nr_dest_folios * PAGE_SIZE;
 113         const u64 orig_end = start + len;
 114
 115         *out_folios = 0;
 116         *total_out = 0;
 117         *total_in = 0;
 118
 119         ret = zlib_deflateInit(&workspace->strm, workspace->level);
 120         if (unlikely(ret != Z_OK)) {
 121                 struct btrfs_inode *inode = BTRFS_I(mapping->host);
 122
 123                 btrfs_err(inode->root->fs_info,
 124         "zlib compression init failed, error %d root %llu inode %llu offset %llu",
 125                           ret, btrfs_root_id(inode->root), btrfs_ino(inode), start);
 126                 ret = -EIO;
 127                 goto out;
 128         }
 129
 130         workspace->strm.total_in = 0;
 131         workspace->strm.total_out = 0;
 132
 133         out_folio = btrfs_alloc_compr_folio();
 134         if (out_folio == NULL) {
 135                 ret = -ENOMEM;
 136                 goto out;
 137         }
 138         cfolio_out = folio_address(out_folio);
 139         folios[0] = out_folio;
 140         nr_folios = 1;
 141
 142         workspace->strm.next_in = workspace->buf;
 143         workspace->strm.avail_in = 0;
 144         workspace->strm.next_out = cfolio_out;
 145         workspace->strm.avail_out = PAGE_SIZE;
 146
 147         while (workspace->strm.total_in < len) {
 148                 /*
 149                  * Get next input pages and copy the contents to
 150                  * the workspace buffer if required.
 151                  */
 152                 if (workspace->strm.avail_in == 0) {
 153                         bytes_left = len - workspace->strm.total_in;
 154                         in_buf_folios = min(DIV_ROUND_UP(bytes_left, PAGE_SIZE),
 155                                             workspace->buf_size / PAGE_SIZE);
 156                         if (in_buf_folios > 1) {
 157                                 int i;
 158
 159                                 /* S390 hardware acceleration path, not subpage. */
 160                                 ASSERT(!btrfs_is_subpage(
 161                                                 inode_to_fs_info(mapping->host),
 162                                                 mapping));
 163                                 for (i = 0; i < in_buf_folios; i++) {
 164                                         if (data_in) {
 165                                                 kunmap_local(data_in);
 166                                                 folio_put(in_folio);
 167                                                 data_in = NULL;
 168                                         }
 169                                         ret = btrfs_compress_filemap_get_folio(mapping,
 170                                                         start, &in_folio);
 171                                         if (ret < 0)
 172                                                 goto out;
 173                                         data_in = kmap_local_folio(in_folio, 0);
 174                                         copy_page(workspace->buf + i * PAGE_SIZE,
 175                                                   data_in);
 176                                         start += PAGE_SIZE;
 177                                         workspace->strm.avail_in =
 178                                                 (in_buf_folios << PAGE_SHIFT);
 179                                 }
 180                                 workspace->strm.next_in = workspace->buf;
 181                         } else {
 182                                 unsigned int pg_off;
 183                                 unsigned int cur_len;
 184
 185                                 if (data_in) {
 186                                         kunmap_local(data_in);
 187                                         folio_put(in_folio);
 188                                         data_in = NULL;
 189                                 }
 190                                 ret = btrfs_compress_filemap_get_folio(mapping,
 191                                                 start, &in_folio);
 192                                 if (ret < 0)
 193                                         goto out;
 194                                 pg_off = offset_in_page(start);
 195                                 cur_len = btrfs_calc_input_length(orig_end, start);
 196                                 data_in = kmap_local_folio(in_folio, pg_off);
 197                                 start += PAGE_SIZE;
 198                                 workspace->strm.next_in = data_in;
 199                                 workspace->strm.avail_in = cur_len;
 200                         }
 201                 }
 202
 203                 ret = zlib_deflate(&workspace->strm, Z_SYNC_FLUSH);
 204                 if (unlikely(ret != Z_OK)) {
 205                         struct btrfs_inode *inode = BTRFS_I(mapping->host);
 206
 207                         btrfs_warn(inode->root->fs_info,
 208                 "zlib compression failed, error %d root %llu inode %llu offset %llu",
 209                                    ret, btrfs_root_id(inode->root), btrfs_ino(inode),
 210                                    start);
 211                         zlib_deflateEnd(&workspace->strm);
 212                         ret = -EIO;
 213                         goto out;
 214                 }
 215
 216                 /* we're making it bigger, give up */
 217                 if (workspace->strm.total_in > 8192 &&
 218                     workspace->strm.total_in <
 219                     workspace->strm.total_out) {
 220                         ret = -E2BIG;
 221                         goto out;
 222                 }
 223                 /* we need another page for writing out.  Test this
 224                  * before the total_in so we will pull in a new page for
 225                  * the stream end if required
 226                  */
 227                 if (workspace->strm.avail_out == 0) {
 228                         if (nr_folios == nr_dest_folios) {
 229                                 ret = -E2BIG;
 230                                 goto out;
 231                         }
 232                         out_folio = btrfs_alloc_compr_folio();
 233                         if (out_folio == NULL) {
 234                                 ret = -ENOMEM;
 235                                 goto out;
 236                         }
 237                         cfolio_out = folio_address(out_folio);
 238                         folios[nr_folios] = out_folio;
 239                         nr_folios++;
 240                         workspace->strm.avail_out = PAGE_SIZE;
 241                         workspace->strm.next_out = cfolio_out;
 242                 }
 243                 /* we're all done */
 244                 if (workspace->strm.total_in >= len)
 245                         break;
 246                 if (workspace->strm.total_out > max_out)
 247                         break;
 248         }
 249         workspace->strm.avail_in = 0;
 250         /*
 251          * Call deflate with Z_FINISH flush parameter providing more output
 252          * space but no more input data, until it returns with Z_STREAM_END.
 253          */
 254         while (ret != Z_STREAM_END) {
 255                 ret = zlib_deflate(&workspace->strm, Z_FINISH);
 256                 if (ret == Z_STREAM_END)
 257                         break;
 258                 if (ret != Z_OK && ret != Z_BUF_ERROR) {
 259                         zlib_deflateEnd(&workspace->strm);
 260                         ret = -EIO;
 261                         goto out;
 262                 } else if (workspace->strm.avail_out == 0) {
 263                         /* Get another folio for the stream end. */
 264                         if (nr_folios == nr_dest_folios) {
 265                                 ret = -E2BIG;
 266                                 goto out;
 267                         }
 268                         out_folio = btrfs_alloc_compr_folio();
 269                         if (out_folio == NULL) {
 270                                 ret = -ENOMEM;
 271                                 goto out;
 272                         }
 273                         cfolio_out = folio_address(out_folio);
 274                         folios[nr_folios] = out_folio;
 275                         nr_folios++;
 276                         workspace->strm.avail_out = PAGE_SIZE;
 277                         workspace->strm.next_out = cfolio_out;
 278                 }
 279         }
 280         zlib_deflateEnd(&workspace->strm);
 281
 282         if (workspace->strm.total_out >= workspace->strm.total_in) {
 283                 ret = -E2BIG;
 284                 goto out;
 285         }
 286
 287         ret = 0;
 288         *total_out = workspace->strm.total_out;
 289         *total_in = workspace->strm.total_in;
 290 out:
 291         *out_folios = nr_folios;
 292         if (data_in) {
 293                 kunmap_local(data_in);
 294                 folio_put(in_folio);
 295         }
 296
 297         return ret;
 298 }
 299
 300 int zlib_decompress_bio(struct list_head *ws, struct compressed_bio *cb)
 301 {
 302         struct workspace *workspace = list_entry(ws, struct workspace, list);
 303         int ret = 0, ret2;
 304         int wbits = MAX_WBITS;
 305         char *data_in;
 306         size_t total_out = 0;
 307         unsigned long folio_in_index = 0;
 308         size_t srclen = cb->compressed_len;
 309         unsigned long total_folios_in = DIV_ROUND_UP(srclen, PAGE_SIZE);
 310         unsigned long buf_start;
 311         struct folio **folios_in = cb->compressed_folios;
 312
 313         data_in = kmap_local_folio(folios_in[folio_in_index], 0);
 314         workspace->strm.next_in = data_in;
 315         workspace->strm.avail_in = min_t(size_t, srclen, PAGE_SIZE);
 316         workspace->strm.total_in = 0;
 317
 318         workspace->strm.total_out = 0;
 319         workspace->strm.next_out = workspace->buf;
 320         workspace->strm.avail_out = workspace->buf_size;
 321
 322         /* If it's deflate, and it's got no preset dictionary, then
 323            we can tell zlib to skip the adler32 check. */
 324         if (srclen > 2 && !(data_in[1] & PRESET_DICT) &&
 325             ((data_in[0] & 0x0f) == Z_DEFLATED) &&
 326             !(((data_in[0]<<8) + data_in[1]) % 31)) {
 327
 328                 wbits = -((data_in[0] >> 4) + 8);
 329                 workspace->strm.next_in += 2;
 330                 workspace->strm.avail_in -= 2;
 331         }
 332
 333         ret = zlib_inflateInit2(&workspace->strm, wbits);
 334         if (unlikely(ret != Z_OK)) {
 335                 struct btrfs_inode *inode = cb->bbio.inode;
 336
 337                 kunmap_local(data_in);
 338                 btrfs_err(inode->root->fs_info,
 339         "zlib decompression init failed, error %d root %llu inode %llu offset %llu",
 340                           ret, btrfs_root_id(inode->root), btrfs_ino(inode), cb->start);
 341                 return -EIO;
 342         }
 343         while (workspace->strm.total_in < srclen) {
 344                 ret = zlib_inflate(&workspace->strm, Z_NO_FLUSH);
 345                 if (ret != Z_OK && ret != Z_STREAM_END)
 346                         break;
 347
 348                 buf_start = total_out;
 349                 total_out = workspace->strm.total_out;
 350
 351                 /* we didn't make progress in this inflate call, we're done */
 352                 if (buf_start == total_out)
 353                         break;
 354
 355                 ret2 = btrfs_decompress_buf2page(workspace->buf,
 356                                 total_out - buf_start, cb, buf_start);
 357                 if (ret2 == 0) {
 358                         ret = 0;
 359                         goto done;
 360                 }
 361
 362                 workspace->strm.next_out = workspace->buf;
 363                 workspace->strm.avail_out = workspace->buf_size;
 364
 365                 if (workspace->strm.avail_in == 0) {
 366                         unsigned long tmp;
 367                         kunmap_local(data_in);
 368                         folio_in_index++;
 369                         if (folio_in_index >= total_folios_in) {
 370                                 data_in = NULL;
 371                                 break;
 372                         }
 373                         data_in = kmap_local_folio(folios_in[folio_in_index], 0);
 374                         workspace->strm.next_in = data_in;
 375                         tmp = srclen - workspace->strm.total_in;
 376                         workspace->strm.avail_in = min(tmp, PAGE_SIZE);
 377                 }
 378         }
 379         if (unlikely(ret != Z_STREAM_END)) {
 380                 btrfs_err(cb->bbio.inode->root->fs_info,
 381                 "zlib decompression failed, error %d root %llu inode %llu offset %llu",
 382                           ret, btrfs_root_id(cb->bbio.inode->root),
 383                           btrfs_ino(cb->bbio.inode), cb->start);
 384                 ret = -EIO;
 385         } else {
 386                 ret = 0;
 387         }
 388 done:
 389         zlib_inflateEnd(&workspace->strm);
 390         if (data_in)
 391                 kunmap_local(data_in);
 392         return ret;
 393 }
 394
 395 int zlib_decompress(struct list_head *ws, const u8 *data_in,
 396                 struct folio *dest_folio, unsigned long dest_pgoff, size_t srclen,
 397                 size_t destlen)
 398 {
 399         struct workspace *workspace = list_entry(ws, struct workspace, list);
 400         int ret = 0;
 401         int wbits = MAX_WBITS;
 402         unsigned long to_copy;
 403
 404         workspace->strm.next_in = data_in;
 405         workspace->strm.avail_in = srclen;
 406         workspace->strm.total_in = 0;
 407
 408         workspace->strm.next_out = workspace->buf;
 409         workspace->strm.avail_out = workspace->buf_size;
 410         workspace->strm.total_out = 0;
 411         /* If it's deflate, and it's got no preset dictionary, then
 412            we can tell zlib to skip the adler32 check. */
 413         if (srclen > 2 && !(data_in[1] & PRESET_DICT) &&
 414             ((data_in[0] & 0x0f) == Z_DEFLATED) &&
 415             !(((data_in[0]<<8) + data_in[1]) % 31)) {
 416
 417                 wbits = -((data_in[0] >> 4) + 8);
 418                 workspace->strm.next_in += 2;
 419                 workspace->strm.avail_in -= 2;
 420         }
 421
 422         ret = zlib_inflateInit2(&workspace->strm, wbits);
 423         if (unlikely(ret != Z_OK)) {
 424                 struct btrfs_inode *inode = folio_to_inode(dest_folio);
 425
 426                 btrfs_err(inode->root->fs_info,
 427                 "zlib decompression init failed, error %d root %llu inode %llu offset %llu",
 428                           ret, btrfs_root_id(inode->root), btrfs_ino(inode),
 429                           folio_pos(dest_folio));
 430                 return -EIO;
 431         }
 432
 433         /*
 434          * Everything (in/out buf) should be at most one sector, there should
 435          * be no need to switch any input/output buffer.
 436          */
 437         ret = zlib_inflate(&workspace->strm, Z_FINISH);
 438         to_copy = min(workspace->strm.total_out, destlen);
 439         if (ret != Z_STREAM_END)
 440                 goto out;
 441
 442         memcpy_to_folio(dest_folio, dest_pgoff, workspace->buf, to_copy);
 443
 444 out:
 445         if (unlikely(to_copy != destlen)) {
 446                 struct btrfs_inode *inode = folio_to_inode(dest_folio);
 447
 448                 btrfs_err(inode->root->fs_info,
 449 "zlib decompression failed, error %d root %llu inode %llu offset %llu decompressed %lu expected %zu",
 450                           ret, btrfs_root_id(inode->root), btrfs_ino(inode),
 451                           folio_pos(dest_folio), to_copy, destlen);
 452                 ret = -EIO;
 453         } else {
 454                 ret = 0;
 455         }
 456
 457         zlib_inflateEnd(&workspace->strm);
 458
 459         if (unlikely(to_copy < destlen))
 460                 folio_zero_range(dest_folio, dest_pgoff + to_copy, destlen - to_copy);
 461         return ret;
 462 }
 463
 464 const struct btrfs_compress_op btrfs_zlib_compress = {
 465         .workspace_manager      = &wsm,
 466         .max_level              = 9,
 467         .default_level          = BTRFS_ZLIB_DEFAULT_LEVEL,
 468 };