fs/erofs/decompressor_lzma.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 #include <linux/xz.h>
   3 #include "compress.h"
   4
   5 struct z_erofs_lzma {
   6         struct z_erofs_lzma *next;
   7         struct xz_dec_microlzma *state;
   8         struct xz_buf buf;
   9         u8 bounce[PAGE_SIZE];
  10 };
  11
  12 /* considering the LZMA performance, no need to use a lockless list for now */
  13 static DEFINE_SPINLOCK(z_erofs_lzma_lock);
  14 static unsigned int z_erofs_lzma_max_dictsize;
  15 static unsigned int z_erofs_lzma_nstrms, z_erofs_lzma_avail_strms;
  16 static struct z_erofs_lzma *z_erofs_lzma_head;
  17 static DECLARE_WAIT_QUEUE_HEAD(z_erofs_lzma_wq);
  18
  19 module_param_named(lzma_streams, z_erofs_lzma_nstrms, uint, 0444);
  20
  21 void z_erofs_lzma_exit(void)
  22 {
  23         /* there should be no running fs instance */
  24         while (z_erofs_lzma_avail_strms) {
  25                 struct z_erofs_lzma *strm;
  26
  27                 spin_lock(&z_erofs_lzma_lock);
  28                 strm = z_erofs_lzma_head;
  29                 if (!strm) {
  30                         spin_unlock(&z_erofs_lzma_lock);
  31                         DBG_BUGON(1);
  32                         return;
  33                 }
  34                 z_erofs_lzma_head = NULL;
  35                 spin_unlock(&z_erofs_lzma_lock);
  36
  37                 while (strm) {
  38                         struct z_erofs_lzma *n = strm->next;
  39
  40                         if (strm->state)
  41                                 xz_dec_microlzma_end(strm->state);
  42                         kfree(strm);
  43                         --z_erofs_lzma_avail_strms;
  44                         strm = n;
  45                 }
  46         }
  47 }
  48
  49 int __init z_erofs_lzma_init(void)
  50 {
  51         unsigned int i;
  52
  53         /* by default, use # of possible CPUs instead */
  54         if (!z_erofs_lzma_nstrms)
  55                 z_erofs_lzma_nstrms = num_possible_cpus();
  56
  57         for (i = 0; i < z_erofs_lzma_nstrms; ++i) {
  58                 struct z_erofs_lzma *strm = kzalloc(sizeof(*strm), GFP_KERNEL);
  59
  60                 if (!strm) {
  61                         z_erofs_lzma_exit();
  62                         return -ENOMEM;
  63                 }
  64                 spin_lock(&z_erofs_lzma_lock);
  65                 strm->next = z_erofs_lzma_head;
  66                 z_erofs_lzma_head = strm;
  67                 spin_unlock(&z_erofs_lzma_lock);
  68                 ++z_erofs_lzma_avail_strms;
  69         }
  70         return 0;
  71 }
  72
  73 int z_erofs_load_lzma_config(struct super_block *sb,
  74                         struct erofs_super_block *dsb, void *data, int size)
  75 {
  76         static DEFINE_MUTEX(lzma_resize_mutex);
  77         struct z_erofs_lzma_cfgs *lzma = data;
  78         unsigned int dict_size, i;
  79         struct z_erofs_lzma *strm, *head = NULL;
  80         int err;
  81
  82         if (!lzma || size < sizeof(struct z_erofs_lzma_cfgs)) {
  83                 erofs_err(sb, "invalid lzma cfgs, size=%u", size);
  84                 return -EINVAL;
  85         }
  86         if (lzma->format) {
  87                 erofs_err(sb, "unidentified lzma format %x, please check kernel version",
  88                           le16_to_cpu(lzma->format));
  89                 return -EINVAL;
  90         }
  91         dict_size = le32_to_cpu(lzma->dict_size);
  92         if (dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE || dict_size < 4096) {
  93                 erofs_err(sb, "unsupported lzma dictionary size %u",
  94                           dict_size);
  95                 return -EINVAL;
  96         }
  97
  98         /* in case 2 z_erofs_load_lzma_config() race to avoid deadlock */
  99         mutex_lock(&lzma_resize_mutex);
 100
 101         if (z_erofs_lzma_max_dictsize >= dict_size) {
 102                 mutex_unlock(&lzma_resize_mutex);
 103                 return 0;
 104         }
 105
 106         /* 1. collect/isolate all streams for the following check */
 107         for (i = 0; i < z_erofs_lzma_avail_strms; ++i) {
 108                 struct z_erofs_lzma *last;
 109
 110 again:
 111                 spin_lock(&z_erofs_lzma_lock);
 112                 strm = z_erofs_lzma_head;
 113                 if (!strm) {
 114                         spin_unlock(&z_erofs_lzma_lock);
 115                         wait_event(z_erofs_lzma_wq,
 116                                    READ_ONCE(z_erofs_lzma_head));
 117                         goto again;
 118                 }
 119                 z_erofs_lzma_head = NULL;
 120                 spin_unlock(&z_erofs_lzma_lock);
 121
 122                 for (last = strm; last->next; last = last->next)
 123                         ++i;
 124                 last->next = head;
 125                 head = strm;
 126         }
 127
 128         err = 0;
 129         /* 2. walk each isolated stream and grow max dict_size if needed */
 130         for (strm = head; strm; strm = strm->next) {
 131                 if (strm->state)
 132                         xz_dec_microlzma_end(strm->state);
 133                 strm->state = xz_dec_microlzma_alloc(XZ_PREALLOC, dict_size);
 134                 if (!strm->state)
 135                         err = -ENOMEM;
 136         }
 137
 138         /* 3. push back all to the global list and update max dict_size */
 139         spin_lock(&z_erofs_lzma_lock);
 140         DBG_BUGON(z_erofs_lzma_head);
 141         z_erofs_lzma_head = head;
 142         spin_unlock(&z_erofs_lzma_lock);
 143         wake_up_all(&z_erofs_lzma_wq);
 144
 145         z_erofs_lzma_max_dictsize = dict_size;
 146         mutex_unlock(&lzma_resize_mutex);
 147         return err;
 148 }
 149
 150 int z_erofs_lzma_decompress(struct z_erofs_decompress_req *rq,
 151                             struct page **pagepool)
 152 {
 153         const unsigned int nrpages_out =
 154                 PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT;
 155         const unsigned int nrpages_in =
 156                 PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT;
 157         unsigned int inlen, outlen, pageofs;
 158         struct z_erofs_lzma *strm;
 159         u8 *kin;
 160         bool bounced = false;
 161         int no, ni, j, err = 0;
 162
 163         /* 1. get the exact LZMA compressed size */
 164         kin = kmap(*rq->in);
 165         err = z_erofs_fixup_insize(rq, kin + rq->pageofs_in,
 166                         min_t(unsigned int, rq->inputsize,
 167                               rq->sb->s_blocksize - rq->pageofs_in));
 168         if (err) {
 169                 kunmap(*rq->in);
 170                 return err;
 171         }
 172
 173         /* 2. get an available lzma context */
 174 again:
 175         spin_lock(&z_erofs_lzma_lock);
 176         strm = z_erofs_lzma_head;
 177         if (!strm) {
 178                 spin_unlock(&z_erofs_lzma_lock);
 179                 wait_event(z_erofs_lzma_wq, READ_ONCE(z_erofs_lzma_head));
 180                 goto again;
 181         }
 182         z_erofs_lzma_head = strm->next;
 183         spin_unlock(&z_erofs_lzma_lock);
 184
 185         /* 3. multi-call decompress */
 186         inlen = rq->inputsize;
 187         outlen = rq->outputsize;
 188         xz_dec_microlzma_reset(strm->state, inlen, outlen,
 189                                !rq->partial_decoding);
 190         pageofs = rq->pageofs_out;
 191         strm->buf.in = kin + rq->pageofs_in;
 192         strm->buf.in_pos = 0;
 193         strm->buf.in_size = min_t(u32, inlen, PAGE_SIZE - rq->pageofs_in);
 194         inlen -= strm->buf.in_size;
 195         strm->buf.out = NULL;
 196         strm->buf.out_pos = 0;
 197         strm->buf.out_size = 0;
 198
 199         for (ni = 0, no = -1;;) {
 200                 enum xz_ret xz_err;
 201
 202                 if (strm->buf.out_pos == strm->buf.out_size) {
 203                         if (strm->buf.out) {
 204                                 kunmap(rq->out[no]);
 205                                 strm->buf.out = NULL;
 206                         }
 207
 208                         if (++no >= nrpages_out || !outlen) {
 209                                 erofs_err(rq->sb, "decompressed buf out of bound");
 210                                 err = -EFSCORRUPTED;
 211                                 break;
 212                         }
 213                         strm->buf.out_pos = 0;
 214                         strm->buf.out_size = min_t(u32, outlen,
 215                                                    PAGE_SIZE - pageofs);
 216                         outlen -= strm->buf.out_size;
 217                         if (!rq->out[no] && rq->fillgaps) {     /* deduped */
 218                                 rq->out[no] = erofs_allocpage(pagepool,
 219                                                 GFP_KERNEL | __GFP_NOFAIL);
 220                                 set_page_private(rq->out[no],
 221                                                  Z_EROFS_SHORTLIVED_PAGE);
 222                         }
 223                         if (rq->out[no])
 224                                 strm->buf.out = kmap(rq->out[no]) + pageofs;
 225                         pageofs = 0;
 226                 } else if (strm->buf.in_pos == strm->buf.in_size) {
 227                         kunmap(rq->in[ni]);
 228
 229                         if (++ni >= nrpages_in || !inlen) {
 230                                 erofs_err(rq->sb, "compressed buf out of bound");
 231                                 err = -EFSCORRUPTED;
 232                                 break;
 233                         }
 234                         strm->buf.in_pos = 0;
 235                         strm->buf.in_size = min_t(u32, inlen, PAGE_SIZE);
 236                         inlen -= strm->buf.in_size;
 237                         kin = kmap(rq->in[ni]);
 238                         strm->buf.in = kin;
 239                         bounced = false;
 240                 }
 241
 242                 /*
 243                  * Handle overlapping: Use bounced buffer if the compressed
 244                  * data is under processing; Otherwise, Use short-lived pages
 245                  * from the on-stack pagepool where pages share with the same
 246                  * request.
 247                  */
 248                 if (!bounced && rq->out[no] == rq->in[ni]) {
 249                         memcpy(strm->bounce, strm->buf.in, strm->buf.in_size);
 250                         strm->buf.in = strm->bounce;
 251                         bounced = true;
 252                 }
 253                 for (j = ni + 1; j < nrpages_in; ++j) {
 254                         struct page *tmppage;
 255
 256                         if (rq->out[no] != rq->in[j])
 257                                 continue;
 258
 259                         DBG_BUGON(erofs_page_is_managed(EROFS_SB(rq->sb),
 260                                                         rq->in[j]));
 261                         tmppage = erofs_allocpage(pagepool,
 262                                                   GFP_KERNEL | __GFP_NOFAIL);
 263                         set_page_private(tmppage, Z_EROFS_SHORTLIVED_PAGE);
 264                         copy_highpage(tmppage, rq->in[j]);
 265                         rq->in[j] = tmppage;
 266                 }
 267                 xz_err = xz_dec_microlzma_run(strm->state, &strm->buf);
 268                 DBG_BUGON(strm->buf.out_pos > strm->buf.out_size);
 269                 DBG_BUGON(strm->buf.in_pos > strm->buf.in_size);
 270
 271                 if (xz_err != XZ_OK) {
 272                         if (xz_err == XZ_STREAM_END && !outlen)
 273                                 break;
 274                         erofs_err(rq->sb, "failed to decompress %d in[%u] out[%u]",
 275                                   xz_err, rq->inputsize, rq->outputsize);
 276                         err = -EFSCORRUPTED;
 277                         break;
 278                 }
 279         }
 280         if (no < nrpages_out && strm->buf.out)
 281                 kunmap(rq->out[no]);
 282         if (ni < nrpages_in)
 283                 kunmap(rq->in[ni]);
 284         /* 4. push back LZMA stream context to the global list */
 285         spin_lock(&z_erofs_lzma_lock);
 286         strm->next = z_erofs_lzma_head;
 287         z_erofs_lzma_head = strm;
 288         spin_unlock(&z_erofs_lzma_lock);
 289         wake_up(&z_erofs_lzma_wq);
 290         return err;
 291 }