fs/erofs/decompressor_lzma.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 #include <linux/xz.h>
   3 #include <linux/module.h>
   4 #include "compress.h"
   5
   6 struct z_erofs_lzma {
   7         struct z_erofs_lzma *next;
   8         struct xz_dec_microlzma *state;
   9         struct xz_buf buf;
  10         u8 bounce[PAGE_SIZE];
  11 };
  12
  13 /* considering the LZMA performance, no need to use a lockless list for now */
  14 static DEFINE_SPINLOCK(z_erofs_lzma_lock);
  15 static unsigned int z_erofs_lzma_max_dictsize;
  16 static unsigned int z_erofs_lzma_nstrms, z_erofs_lzma_avail_strms;
  17 static struct z_erofs_lzma *z_erofs_lzma_head;
  18 static DECLARE_WAIT_QUEUE_HEAD(z_erofs_lzma_wq);
  19
  20 module_param_named(lzma_streams, z_erofs_lzma_nstrms, uint, 0444);
  21
  22 void z_erofs_lzma_exit(void)
  23 {
  24         /* there should be no running fs instance */
  25         while (z_erofs_lzma_avail_strms) {
  26                 struct z_erofs_lzma *strm;
  27
  28                 spin_lock(&z_erofs_lzma_lock);
  29                 strm = z_erofs_lzma_head;
  30                 if (!strm) {
  31                         spin_unlock(&z_erofs_lzma_lock);
  32                         DBG_BUGON(1);
  33                         return;
  34                 }
  35                 z_erofs_lzma_head = NULL;
  36                 spin_unlock(&z_erofs_lzma_lock);
  37
  38                 while (strm) {
  39                         struct z_erofs_lzma *n = strm->next;
  40
  41                         if (strm->state)
  42                                 xz_dec_microlzma_end(strm->state);
  43                         kfree(strm);
  44                         --z_erofs_lzma_avail_strms;
  45                         strm = n;
  46                 }
  47         }
  48 }
  49
  50 int __init z_erofs_lzma_init(void)
  51 {
  52         unsigned int i;
  53
  54         /* by default, use # of possible CPUs instead */
  55         if (!z_erofs_lzma_nstrms)
  56                 z_erofs_lzma_nstrms = num_possible_cpus();
  57
  58         for (i = 0; i < z_erofs_lzma_nstrms; ++i) {
  59                 struct z_erofs_lzma *strm = kzalloc(sizeof(*strm), GFP_KERNEL);
  60
  61                 if (!strm) {
  62                         z_erofs_lzma_exit();
  63                         return -ENOMEM;
  64                 }
  65                 spin_lock(&z_erofs_lzma_lock);
  66                 strm->next = z_erofs_lzma_head;
  67                 z_erofs_lzma_head = strm;
  68                 spin_unlock(&z_erofs_lzma_lock);
  69                 ++z_erofs_lzma_avail_strms;
  70         }
  71         return 0;
  72 }
  73
  74 int z_erofs_load_lzma_config(struct super_block *sb,
  75                              struct erofs_super_block *dsb,
  76                              struct z_erofs_lzma_cfgs *lzma, int size)
  77 {
  78         static DEFINE_MUTEX(lzma_resize_mutex);
  79         unsigned int dict_size, i;
  80         struct z_erofs_lzma *strm, *head = NULL;
  81         int err;
  82
  83         if (!lzma || size < sizeof(struct z_erofs_lzma_cfgs)) {
  84                 erofs_err(sb, "invalid lzma cfgs, size=%u", size);
  85                 return -EINVAL;
  86         }
  87         if (lzma->format) {
  88                 erofs_err(sb, "unidentified lzma format %x, please check kernel version",
  89                           le16_to_cpu(lzma->format));
  90                 return -EINVAL;
  91         }
  92         dict_size = le32_to_cpu(lzma->dict_size);
  93         if (dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE || dict_size < 4096) {
  94                 erofs_err(sb, "unsupported lzma dictionary size %u",
  95                           dict_size);
  96                 return -EINVAL;
  97         }
  98
  99         erofs_info(sb, "EXPERIMENTAL MicroLZMA in use. Use at your own risk!");
 100
 101         /* in case 2 z_erofs_load_lzma_config() race to avoid deadlock */
 102         mutex_lock(&lzma_resize_mutex);
 103
 104         if (z_erofs_lzma_max_dictsize >= dict_size) {
 105                 mutex_unlock(&lzma_resize_mutex);
 106                 return 0;
 107         }
 108
 109         /* 1. collect/isolate all streams for the following check */
 110         for (i = 0; i < z_erofs_lzma_avail_strms; ++i) {
 111                 struct z_erofs_lzma *last;
 112
 113 again:
 114                 spin_lock(&z_erofs_lzma_lock);
 115                 strm = z_erofs_lzma_head;
 116                 if (!strm) {
 117                         spin_unlock(&z_erofs_lzma_lock);
 118                         wait_event(z_erofs_lzma_wq,
 119                                    READ_ONCE(z_erofs_lzma_head));
 120                         goto again;
 121                 }
 122                 z_erofs_lzma_head = NULL;
 123                 spin_unlock(&z_erofs_lzma_lock);
 124
 125                 for (last = strm; last->next; last = last->next)
 126                         ++i;
 127                 last->next = head;
 128                 head = strm;
 129         }
 130
 131         err = 0;
 132         /* 2. walk each isolated stream and grow max dict_size if needed */
 133         for (strm = head; strm; strm = strm->next) {
 134                 if (strm->state)
 135                         xz_dec_microlzma_end(strm->state);
 136                 strm->state = xz_dec_microlzma_alloc(XZ_PREALLOC, dict_size);
 137                 if (!strm->state)
 138                         err = -ENOMEM;
 139         }
 140
 141         /* 3. push back all to the global list and update max dict_size */
 142         spin_lock(&z_erofs_lzma_lock);
 143         DBG_BUGON(z_erofs_lzma_head);
 144         z_erofs_lzma_head = head;
 145         spin_unlock(&z_erofs_lzma_lock);
 146         wake_up_all(&z_erofs_lzma_wq);
 147
 148         z_erofs_lzma_max_dictsize = dict_size;
 149         mutex_unlock(&lzma_resize_mutex);
 150         return err;
 151 }
 152
 153 int z_erofs_lzma_decompress(struct z_erofs_decompress_req *rq,
 154                             struct page **pagepool)
 155 {
 156         const unsigned int nrpages_out =
 157                 PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT;
 158         const unsigned int nrpages_in =
 159                 PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT;
 160         unsigned int inlen, outlen, pageofs;
 161         struct z_erofs_lzma *strm;
 162         u8 *kin;
 163         bool bounced = false;
 164         int no, ni, j, err = 0;
 165
 166         /* 1. get the exact LZMA compressed size */
 167         kin = kmap(*rq->in);
 168         err = z_erofs_fixup_insize(rq, kin + rq->pageofs_in,
 169                         min_t(unsigned int, rq->inputsize,
 170                               rq->sb->s_blocksize - rq->pageofs_in));
 171         if (err) {
 172                 kunmap(*rq->in);
 173                 return err;
 174         }
 175
 176         /* 2. get an available lzma context */
 177 again:
 178         spin_lock(&z_erofs_lzma_lock);
 179         strm = z_erofs_lzma_head;
 180         if (!strm) {
 181                 spin_unlock(&z_erofs_lzma_lock);
 182                 wait_event(z_erofs_lzma_wq, READ_ONCE(z_erofs_lzma_head));
 183                 goto again;
 184         }
 185         z_erofs_lzma_head = strm->next;
 186         spin_unlock(&z_erofs_lzma_lock);
 187
 188         /* 3. multi-call decompress */
 189         inlen = rq->inputsize;
 190         outlen = rq->outputsize;
 191         xz_dec_microlzma_reset(strm->state, inlen, outlen,
 192                                !rq->partial_decoding);
 193         pageofs = rq->pageofs_out;
 194         strm->buf.in = kin + rq->pageofs_in;
 195         strm->buf.in_pos = 0;
 196         strm->buf.in_size = min_t(u32, inlen, PAGE_SIZE - rq->pageofs_in);
 197         inlen -= strm->buf.in_size;
 198         strm->buf.out = NULL;
 199         strm->buf.out_pos = 0;
 200         strm->buf.out_size = 0;
 201
 202         for (ni = 0, no = -1;;) {
 203                 enum xz_ret xz_err;
 204
 205                 if (strm->buf.out_pos == strm->buf.out_size) {
 206                         if (strm->buf.out) {
 207                                 kunmap(rq->out[no]);
 208                                 strm->buf.out = NULL;
 209                         }
 210
 211                         if (++no >= nrpages_out || !outlen) {
 212                                 erofs_err(rq->sb, "decompressed buf out of bound");
 213                                 err = -EFSCORRUPTED;
 214                                 break;
 215                         }
 216                         strm->buf.out_pos = 0;
 217                         strm->buf.out_size = min_t(u32, outlen,
 218                                                    PAGE_SIZE - pageofs);
 219                         outlen -= strm->buf.out_size;
 220                         if (!rq->out[no] && rq->fillgaps) {     /* deduped */
 221                                 rq->out[no] = erofs_allocpage(pagepool,
 222                                                 GFP_KERNEL | __GFP_NOFAIL);
 223                                 set_page_private(rq->out[no],
 224                                                  Z_EROFS_SHORTLIVED_PAGE);
 225                         }
 226                         if (rq->out[no])
 227                                 strm->buf.out = kmap(rq->out[no]) + pageofs;
 228                         pageofs = 0;
 229                 } else if (strm->buf.in_pos == strm->buf.in_size) {
 230                         kunmap(rq->in[ni]);
 231
 232                         if (++ni >= nrpages_in || !inlen) {
 233                                 erofs_err(rq->sb, "compressed buf out of bound");
 234                                 err = -EFSCORRUPTED;
 235                                 break;
 236                         }
 237                         strm->buf.in_pos = 0;
 238                         strm->buf.in_size = min_t(u32, inlen, PAGE_SIZE);
 239                         inlen -= strm->buf.in_size;
 240                         kin = kmap(rq->in[ni]);
 241                         strm->buf.in = kin;
 242                         bounced = false;
 243                 }
 244
 245                 /*
 246                  * Handle overlapping: Use bounced buffer if the compressed
 247                  * data is under processing; Otherwise, Use short-lived pages
 248                  * from the on-stack pagepool where pages share with the same
 249                  * request.
 250                  */
 251                 if (!bounced && rq->out[no] == rq->in[ni]) {
 252                         memcpy(strm->bounce, strm->buf.in, strm->buf.in_size);
 253                         strm->buf.in = strm->bounce;
 254                         bounced = true;
 255                 }
 256                 for (j = ni + 1; j < nrpages_in; ++j) {
 257                         struct page *tmppage;
 258
 259                         if (rq->out[no] != rq->in[j])
 260                                 continue;
 261
 262                         DBG_BUGON(erofs_page_is_managed(EROFS_SB(rq->sb),
 263                                                         rq->in[j]));
 264                         tmppage = erofs_allocpage(pagepool,
 265                                                   GFP_KERNEL | __GFP_NOFAIL);
 266                         set_page_private(tmppage, Z_EROFS_SHORTLIVED_PAGE);
 267                         copy_highpage(tmppage, rq->in[j]);
 268                         rq->in[j] = tmppage;
 269                 }
 270                 xz_err = xz_dec_microlzma_run(strm->state, &strm->buf);
 271                 DBG_BUGON(strm->buf.out_pos > strm->buf.out_size);
 272                 DBG_BUGON(strm->buf.in_pos > strm->buf.in_size);
 273
 274                 if (xz_err != XZ_OK) {
 275                         if (xz_err == XZ_STREAM_END && !outlen)
 276                                 break;
 277                         erofs_err(rq->sb, "failed to decompress %d in[%u] out[%u]",
 278                                   xz_err, rq->inputsize, rq->outputsize);
 279                         err = -EFSCORRUPTED;
 280                         break;
 281                 }
 282         }
 283         if (no < nrpages_out && strm->buf.out)
 284                 kunmap(rq->out[no]);
 285         if (ni < nrpages_in)
 286                 kunmap(rq->in[ni]);
 287         /* 4. push back LZMA stream context to the global list */
 288         spin_lock(&z_erofs_lzma_lock);
 289         strm->next = z_erofs_lzma_head;
 290         z_erofs_lzma_head = strm;
 291         spin_unlock(&z_erofs_lzma_lock);
 292         wake_up(&z_erofs_lzma_wq);
 293         return err;
 294 }