fs/erofs/zmap.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2018-2019 HUAWEI, Inc.
   4  *             https://www.huawei.com/
   5  */
   6 #include "internal.h"
   7 #include <asm/unaligned.h>
   8 #include <trace/events/erofs.h>
   9
  10 int z_erofs_fill_inode(struct inode *inode)
  11 {
  12         struct erofs_inode *const vi = EROFS_I(inode);
  13         struct erofs_sb_info *sbi = EROFS_SB(inode->i_sb);
  14
  15         if (!erofs_sb_has_big_pcluster(sbi) &&
  16             !erofs_sb_has_ztailpacking(sbi) && !erofs_sb_has_fragments(sbi) &&
  17             vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY) {
  18                 vi->z_advise = 0;
  19                 vi->z_algorithmtype[0] = 0;
  20                 vi->z_algorithmtype[1] = 0;
  21                 vi->z_logical_clusterbits = LOG_BLOCK_SIZE;
  22                 set_bit(EROFS_I_Z_INITED_BIT, &vi->flags);
  23         }
  24         inode->i_mapping->a_ops = &z_erofs_aops;
  25         return 0;
  26 }
  27
  28 struct z_erofs_maprecorder {
  29         struct inode *inode;
  30         struct erofs_map_blocks *map;
  31         void *kaddr;
  32
  33         unsigned long lcn;
  34         /* compression extent information gathered */
  35         u8  type, headtype;
  36         u16 clusterofs;
  37         u16 delta[2];
  38         erofs_blk_t pblk, compressedblks;
  39         erofs_off_t nextpackoff;
  40         bool partialref;
  41 };
  42
  43 static int legacy_load_cluster_from_disk(struct z_erofs_maprecorder *m,
  44                                          unsigned long lcn)
  45 {
  46         struct inode *const inode = m->inode;
  47         struct erofs_inode *const vi = EROFS_I(inode);
  48         const erofs_off_t pos =
  49                 Z_EROFS_VLE_LEGACY_INDEX_ALIGN(erofs_iloc(inode) +
  50                                 vi->inode_isize + vi->xattr_isize) +
  51                 lcn * sizeof(struct z_erofs_vle_decompressed_index);
  52         struct z_erofs_vle_decompressed_index *di;
  53         unsigned int advise, type;
  54
  55         m->kaddr = erofs_read_metabuf(&m->map->buf, inode->i_sb,
  56                                       erofs_blknr(pos), EROFS_KMAP);
  57         if (IS_ERR(m->kaddr))
  58                 return PTR_ERR(m->kaddr);
  59
  60         m->nextpackoff = pos + sizeof(struct z_erofs_vle_decompressed_index);
  61         m->lcn = lcn;
  62         di = m->kaddr + erofs_blkoff(pos);
  63
  64         advise = le16_to_cpu(di->di_advise);
  65         type = (advise >> Z_EROFS_VLE_DI_CLUSTER_TYPE_BIT) &
  66                 ((1 << Z_EROFS_VLE_DI_CLUSTER_TYPE_BITS) - 1);
  67         switch (type) {
  68         case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD:
  69                 m->clusterofs = 1 << vi->z_logical_clusterbits;
  70                 m->delta[0] = le16_to_cpu(di->di_u.delta[0]);
  71                 if (m->delta[0] & Z_EROFS_VLE_DI_D0_CBLKCNT) {
  72                         if (!(vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 |
  73                                         Z_EROFS_ADVISE_BIG_PCLUSTER_2))) {
  74                                 DBG_BUGON(1);
  75                                 return -EFSCORRUPTED;
  76                         }
  77                         m->compressedblks = m->delta[0] &
  78                                 ~Z_EROFS_VLE_DI_D0_CBLKCNT;
  79                         m->delta[0] = 1;
  80                 }
  81                 m->delta[1] = le16_to_cpu(di->di_u.delta[1]);
  82                 break;
  83         case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN:
  84         case Z_EROFS_VLE_CLUSTER_TYPE_HEAD1:
  85         case Z_EROFS_VLE_CLUSTER_TYPE_HEAD2:
  86                 if (advise & Z_EROFS_VLE_DI_PARTIAL_REF)
  87                         m->partialref = true;
  88                 m->clusterofs = le16_to_cpu(di->di_clusterofs);
  89                 m->pblk = le32_to_cpu(di->di_u.blkaddr);
  90                 break;
  91         default:
  92                 DBG_BUGON(1);
  93                 return -EOPNOTSUPP;
  94         }
  95         m->type = type;
  96         return 0;
  97 }
  98
  99 static unsigned int decode_compactedbits(unsigned int lobits,
 100                                          unsigned int lomask,
 101                                          u8 *in, unsigned int pos, u8 *type)
 102 {
 103         const unsigned int v = get_unaligned_le32(in + pos / 8) >> (pos & 7);
 104         const unsigned int lo = v & lomask;
 105
 106         *type = (v >> lobits) & 3;
 107         return lo;
 108 }
 109
 110 static int get_compacted_la_distance(unsigned int lclusterbits,
 111                                      unsigned int encodebits,
 112                                      unsigned int vcnt, u8 *in, int i)
 113 {
 114         const unsigned int lomask = (1 << lclusterbits) - 1;
 115         unsigned int lo, d1 = 0;
 116         u8 type;
 117
 118         DBG_BUGON(i >= vcnt);
 119
 120         do {
 121                 lo = decode_compactedbits(lclusterbits, lomask,
 122                                           in, encodebits * i, &type);
 123
 124                 if (type != Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD)
 125                         return d1;
 126                 ++d1;
 127         } while (++i < vcnt);
 128
 129         /* vcnt - 1 (Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) item */
 130         if (!(lo & Z_EROFS_VLE_DI_D0_CBLKCNT))
 131                 d1 += lo - 1;
 132         return d1;
 133 }
 134
 135 static int unpack_compacted_index(struct z_erofs_maprecorder *m,
 136                                   unsigned int amortizedshift,
 137                                   erofs_off_t pos, bool lookahead)
 138 {
 139         struct erofs_inode *const vi = EROFS_I(m->inode);
 140         const unsigned int lclusterbits = vi->z_logical_clusterbits;
 141         const unsigned int lomask = (1 << lclusterbits) - 1;
 142         unsigned int vcnt, base, lo, encodebits, nblk, eofs;
 143         int i;
 144         u8 *in, type;
 145         bool big_pcluster;
 146
 147         if (1 << amortizedshift == 4)
 148                 vcnt = 2;
 149         else if (1 << amortizedshift == 2 && lclusterbits == 12)
 150                 vcnt = 16;
 151         else
 152                 return -EOPNOTSUPP;
 153
 154         /* it doesn't equal to round_up(..) */
 155         m->nextpackoff = round_down(pos, vcnt << amortizedshift) +
 156                          (vcnt << amortizedshift);
 157         big_pcluster = vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1;
 158         encodebits = ((vcnt << amortizedshift) - sizeof(__le32)) * 8 / vcnt;
 159         eofs = erofs_blkoff(pos);
 160         base = round_down(eofs, vcnt << amortizedshift);
 161         in = m->kaddr + base;
 162
 163         i = (eofs - base) >> amortizedshift;
 164
 165         lo = decode_compactedbits(lclusterbits, lomask,
 166                                   in, encodebits * i, &type);
 167         m->type = type;
 168         if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) {
 169                 m->clusterofs = 1 << lclusterbits;
 170
 171                 /* figure out lookahead_distance: delta[1] if needed */
 172                 if (lookahead)
 173                         m->delta[1] = get_compacted_la_distance(lclusterbits,
 174                                                 encodebits, vcnt, in, i);
 175                 if (lo & Z_EROFS_VLE_DI_D0_CBLKCNT) {
 176                         if (!big_pcluster) {
 177                                 DBG_BUGON(1);
 178                                 return -EFSCORRUPTED;
 179                         }
 180                         m->compressedblks = lo & ~Z_EROFS_VLE_DI_D0_CBLKCNT;
 181                         m->delta[0] = 1;
 182                         return 0;
 183                 } else if (i + 1 != (int)vcnt) {
 184                         m->delta[0] = lo;
 185                         return 0;
 186                 }
 187                 /*
 188                  * since the last lcluster in the pack is special,
 189                  * of which lo saves delta[1] rather than delta[0].
 190                  * Hence, get delta[0] by the previous lcluster indirectly.
 191                  */
 192                 lo = decode_compactedbits(lclusterbits, lomask,
 193                                           in, encodebits * (i - 1), &type);
 194                 if (type != Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD)
 195                         lo = 0;
 196                 else if (lo & Z_EROFS_VLE_DI_D0_CBLKCNT)
 197                         lo = 1;
 198                 m->delta[0] = lo + 1;
 199                 return 0;
 200         }
 201         m->clusterofs = lo;
 202         m->delta[0] = 0;
 203         /* figout out blkaddr (pblk) for HEAD lclusters */
 204         if (!big_pcluster) {
 205                 nblk = 1;
 206                 while (i > 0) {
 207                         --i;
 208                         lo = decode_compactedbits(lclusterbits, lomask,
 209                                                   in, encodebits * i, &type);
 210                         if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD)
 211                                 i -= lo;
 212
 213                         if (i >= 0)
 214                                 ++nblk;
 215                 }
 216         } else {
 217                 nblk = 0;
 218                 while (i > 0) {
 219                         --i;
 220                         lo = decode_compactedbits(lclusterbits, lomask,
 221                                                   in, encodebits * i, &type);
 222                         if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) {
 223                                 if (lo & Z_EROFS_VLE_DI_D0_CBLKCNT) {
 224                                         --i;
 225                                         nblk += lo & ~Z_EROFS_VLE_DI_D0_CBLKCNT;
 226                                         continue;
 227                                 }
 228                                 /* bigpcluster shouldn't have plain d0 == 1 */
 229                                 if (lo <= 1) {
 230                                         DBG_BUGON(1);
 231                                         return -EFSCORRUPTED;
 232                                 }
 233                                 i -= lo - 2;
 234                                 continue;
 235                         }
 236                         ++nblk;
 237                 }
 238         }
 239         in += (vcnt << amortizedshift) - sizeof(__le32);
 240         m->pblk = le32_to_cpu(*(__le32 *)in) + nblk;
 241         return 0;
 242 }
 243
 244 static int compacted_load_cluster_from_disk(struct z_erofs_maprecorder *m,
 245                                             unsigned long lcn, bool lookahead)
 246 {
 247         struct inode *const inode = m->inode;
 248         struct erofs_inode *const vi = EROFS_I(inode);
 249         const unsigned int lclusterbits = vi->z_logical_clusterbits;
 250         const erofs_off_t ebase = sizeof(struct z_erofs_map_header) +
 251                 ALIGN(erofs_iloc(inode) + vi->inode_isize + vi->xattr_isize, 8);
 252         const unsigned int totalidx = DIV_ROUND_UP(inode->i_size, EROFS_BLKSIZ);
 253         unsigned int compacted_4b_initial, compacted_2b;
 254         unsigned int amortizedshift;
 255         erofs_off_t pos;
 256
 257         if (lclusterbits != 12)
 258                 return -EOPNOTSUPP;
 259
 260         if (lcn >= totalidx)
 261                 return -EINVAL;
 262
 263         m->lcn = lcn;
 264         /* used to align to 32-byte (compacted_2b) alignment */
 265         compacted_4b_initial = (32 - ebase % 32) / 4;
 266         if (compacted_4b_initial == 32 / 4)
 267                 compacted_4b_initial = 0;
 268
 269         if ((vi->z_advise & Z_EROFS_ADVISE_COMPACTED_2B) &&
 270             compacted_4b_initial < totalidx)
 271                 compacted_2b = rounddown(totalidx - compacted_4b_initial, 16);
 272         else
 273                 compacted_2b = 0;
 274
 275         pos = ebase;
 276         if (lcn < compacted_4b_initial) {
 277                 amortizedshift = 2;
 278                 goto out;
 279         }
 280         pos += compacted_4b_initial * 4;
 281         lcn -= compacted_4b_initial;
 282
 283         if (lcn < compacted_2b) {
 284                 amortizedshift = 1;
 285                 goto out;
 286         }
 287         pos += compacted_2b * 2;
 288         lcn -= compacted_2b;
 289         amortizedshift = 2;
 290 out:
 291         pos += lcn * (1 << amortizedshift);
 292         m->kaddr = erofs_read_metabuf(&m->map->buf, inode->i_sb,
 293                                       erofs_blknr(pos), EROFS_KMAP);
 294         if (IS_ERR(m->kaddr))
 295                 return PTR_ERR(m->kaddr);
 296         return unpack_compacted_index(m, amortizedshift, pos, lookahead);
 297 }
 298
 299 static int z_erofs_load_cluster_from_disk(struct z_erofs_maprecorder *m,
 300                                           unsigned int lcn, bool lookahead)
 301 {
 302         const unsigned int datamode = EROFS_I(m->inode)->datalayout;
 303
 304         if (datamode == EROFS_INODE_FLAT_COMPRESSION_LEGACY)
 305                 return legacy_load_cluster_from_disk(m, lcn);
 306
 307         if (datamode == EROFS_INODE_FLAT_COMPRESSION)
 308                 return compacted_load_cluster_from_disk(m, lcn, lookahead);
 309
 310         return -EINVAL;
 311 }
 312
 313 static int z_erofs_extent_lookback(struct z_erofs_maprecorder *m,
 314                                    unsigned int lookback_distance)
 315 {
 316         struct erofs_inode *const vi = EROFS_I(m->inode);
 317         const unsigned int lclusterbits = vi->z_logical_clusterbits;
 318
 319         while (m->lcn >= lookback_distance) {
 320                 unsigned long lcn = m->lcn - lookback_distance;
 321                 int err;
 322
 323                 /* load extent head logical cluster if needed */
 324                 err = z_erofs_load_cluster_from_disk(m, lcn, false);
 325                 if (err)
 326                         return err;
 327
 328                 switch (m->type) {
 329                 case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD:
 330                         if (!m->delta[0]) {
 331                                 erofs_err(m->inode->i_sb,
 332                                           "invalid lookback distance 0 @ nid %llu",
 333                                           vi->nid);
 334                                 DBG_BUGON(1);
 335                                 return -EFSCORRUPTED;
 336                         }
 337                         lookback_distance = m->delta[0];
 338                         continue;
 339                 case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN:
 340                 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD1:
 341                 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD2:
 342                         m->headtype = m->type;
 343                         m->map->m_la = (lcn << lclusterbits) | m->clusterofs;
 344                         return 0;
 345                 default:
 346                         erofs_err(m->inode->i_sb,
 347                                   "unknown type %u @ lcn %lu of nid %llu",
 348                                   m->type, lcn, vi->nid);
 349                         DBG_BUGON(1);
 350                         return -EOPNOTSUPP;
 351                 }
 352         }
 353
 354         erofs_err(m->inode->i_sb, "bogus lookback distance @ nid %llu",
 355                   vi->nid);
 356         DBG_BUGON(1);
 357         return -EFSCORRUPTED;
 358 }
 359
 360 static int z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m,
 361                                             unsigned int initial_lcn)
 362 {
 363         struct erofs_inode *const vi = EROFS_I(m->inode);
 364         struct erofs_map_blocks *const map = m->map;
 365         const unsigned int lclusterbits = vi->z_logical_clusterbits;
 366         unsigned long lcn;
 367         int err;
 368
 369         DBG_BUGON(m->type != Z_EROFS_VLE_CLUSTER_TYPE_PLAIN &&
 370                   m->type != Z_EROFS_VLE_CLUSTER_TYPE_HEAD1 &&
 371                   m->type != Z_EROFS_VLE_CLUSTER_TYPE_HEAD2);
 372         DBG_BUGON(m->type != m->headtype);
 373
 374         if (m->headtype == Z_EROFS_VLE_CLUSTER_TYPE_PLAIN ||
 375             ((m->headtype == Z_EROFS_VLE_CLUSTER_TYPE_HEAD1) &&
 376              !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1)) ||
 377             ((m->headtype == Z_EROFS_VLE_CLUSTER_TYPE_HEAD2) &&
 378              !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2))) {
 379                 map->m_plen = 1ULL << lclusterbits;
 380                 return 0;
 381         }
 382         lcn = m->lcn + 1;
 383         if (m->compressedblks)
 384                 goto out;
 385
 386         err = z_erofs_load_cluster_from_disk(m, lcn, false);
 387         if (err)
 388                 return err;
 389
 390         /*
 391          * If the 1st NONHEAD lcluster has already been handled initially w/o
 392          * valid compressedblks, which means at least it mustn't be CBLKCNT, or
 393          * an internal implemenatation error is detected.
 394          *
 395          * The following code can also handle it properly anyway, but let's
 396          * BUG_ON in the debugging mode only for developers to notice that.
 397          */
 398         DBG_BUGON(lcn == initial_lcn &&
 399                   m->type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD);
 400
 401         switch (m->type) {
 402         case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN:
 403         case Z_EROFS_VLE_CLUSTER_TYPE_HEAD1:
 404         case Z_EROFS_VLE_CLUSTER_TYPE_HEAD2:
 405                 /*
 406                  * if the 1st NONHEAD lcluster is actually PLAIN or HEAD type
 407                  * rather than CBLKCNT, it's a 1 lcluster-sized pcluster.
 408                  */
 409                 m->compressedblks = 1 << (lclusterbits - LOG_BLOCK_SIZE);
 410                 break;
 411         case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD:
 412                 if (m->delta[0] != 1)
 413                         goto err_bonus_cblkcnt;
 414                 if (m->compressedblks)
 415                         break;
 416                 fallthrough;
 417         default:
 418                 erofs_err(m->inode->i_sb,
 419                           "cannot found CBLKCNT @ lcn %lu of nid %llu",
 420                           lcn, vi->nid);
 421                 DBG_BUGON(1);
 422                 return -EFSCORRUPTED;
 423         }
 424 out:
 425         map->m_plen = (u64)m->compressedblks << LOG_BLOCK_SIZE;
 426         return 0;
 427 err_bonus_cblkcnt:
 428         erofs_err(m->inode->i_sb,
 429                   "bogus CBLKCNT @ lcn %lu of nid %llu",
 430                   lcn, vi->nid);
 431         DBG_BUGON(1);
 432         return -EFSCORRUPTED;
 433 }
 434
 435 static int z_erofs_get_extent_decompressedlen(struct z_erofs_maprecorder *m)
 436 {
 437         struct inode *inode = m->inode;
 438         struct erofs_inode *vi = EROFS_I(inode);
 439         struct erofs_map_blocks *map = m->map;
 440         unsigned int lclusterbits = vi->z_logical_clusterbits;
 441         u64 lcn = m->lcn, headlcn = map->m_la >> lclusterbits;
 442         int err;
 443
 444         do {
 445                 /* handle the last EOF pcluster (no next HEAD lcluster) */
 446                 if ((lcn << lclusterbits) >= inode->i_size) {
 447                         map->m_llen = inode->i_size - map->m_la;
 448                         return 0;
 449                 }
 450
 451                 err = z_erofs_load_cluster_from_disk(m, lcn, true);
 452                 if (err)
 453                         return err;
 454
 455                 if (m->type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) {
 456                         DBG_BUGON(!m->delta[1] &&
 457                                   m->clusterofs != 1 << lclusterbits);
 458                 } else if (m->type == Z_EROFS_VLE_CLUSTER_TYPE_PLAIN ||
 459                            m->type == Z_EROFS_VLE_CLUSTER_TYPE_HEAD1 ||
 460                            m->type == Z_EROFS_VLE_CLUSTER_TYPE_HEAD2) {
 461                         /* go on until the next HEAD lcluster */
 462                         if (lcn != headlcn)
 463                                 break;
 464                         m->delta[1] = 1;
 465                 } else {
 466                         erofs_err(inode->i_sb, "unknown type %u @ lcn %llu of nid %llu",
 467                                   m->type, lcn, vi->nid);
 468                         DBG_BUGON(1);
 469                         return -EOPNOTSUPP;
 470                 }
 471                 lcn += m->delta[1];
 472         } while (m->delta[1]);
 473
 474         map->m_llen = (lcn << lclusterbits) + m->clusterofs - map->m_la;
 475         return 0;
 476 }
 477
 478 static int z_erofs_do_map_blocks(struct inode *inode,
 479                                  struct erofs_map_blocks *map,
 480                                  int flags)
 481 {
 482         struct erofs_inode *const vi = EROFS_I(inode);
 483         bool ztailpacking = vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER;
 484         bool fragment = vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER;
 485         struct z_erofs_maprecorder m = {
 486                 .inode = inode,
 487                 .map = map,
 488         };
 489         int err = 0;
 490         unsigned int lclusterbits, endoff;
 491         unsigned long initial_lcn;
 492         unsigned long long ofs, end;
 493
 494         lclusterbits = vi->z_logical_clusterbits;
 495         ofs = flags & EROFS_GET_BLOCKS_FINDTAIL ? inode->i_size - 1 : map->m_la;
 496         initial_lcn = ofs >> lclusterbits;
 497         endoff = ofs & ((1 << lclusterbits) - 1);
 498
 499         err = z_erofs_load_cluster_from_disk(&m, initial_lcn, false);
 500         if (err)
 501                 goto unmap_out;
 502
 503         if (ztailpacking && (flags & EROFS_GET_BLOCKS_FINDTAIL))
 504                 vi->z_idataoff = m.nextpackoff;
 505
 506         map->m_flags = EROFS_MAP_MAPPED | EROFS_MAP_ENCODED;
 507         end = (m.lcn + 1ULL) << lclusterbits;
 508
 509         switch (m.type) {
 510         case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN:
 511         case Z_EROFS_VLE_CLUSTER_TYPE_HEAD1:
 512         case Z_EROFS_VLE_CLUSTER_TYPE_HEAD2:
 513                 if (endoff >= m.clusterofs) {
 514                         m.headtype = m.type;
 515                         map->m_la = (m.lcn << lclusterbits) | m.clusterofs;
 516                         /*
 517                          * For ztailpacking files, in order to inline data more
 518                          * effectively, special EOF lclusters are now supported
 519                          * which can have three parts at most.
 520                          */
 521                         if (ztailpacking && end > inode->i_size)
 522                                 end = inode->i_size;
 523                         break;
 524                 }
 525                 /* m.lcn should be >= 1 if endoff < m.clusterofs */
 526                 if (!m.lcn) {
 527                         erofs_err(inode->i_sb,
 528                                   "invalid logical cluster 0 at nid %llu",
 529                                   vi->nid);
 530                         err = -EFSCORRUPTED;
 531                         goto unmap_out;
 532                 }
 533                 end = (m.lcn << lclusterbits) | m.clusterofs;
 534                 map->m_flags |= EROFS_MAP_FULL_MAPPED;
 535                 m.delta[0] = 1;
 536                 fallthrough;
 537         case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD:
 538                 /* get the corresponding first chunk */
 539                 err = z_erofs_extent_lookback(&m, m.delta[0]);
 540                 if (err)
 541                         goto unmap_out;
 542                 break;
 543         default:
 544                 erofs_err(inode->i_sb,
 545                           "unknown type %u @ offset %llu of nid %llu",
 546                           m.type, ofs, vi->nid);
 547                 err = -EOPNOTSUPP;
 548                 goto unmap_out;
 549         }
 550         if (m.partialref)
 551                 map->m_flags |= EROFS_MAP_PARTIAL_REF;
 552         map->m_llen = end - map->m_la;
 553
 554         if (flags & EROFS_GET_BLOCKS_FINDTAIL) {
 555                 vi->z_tailextent_headlcn = m.lcn;
 556                 /* for non-compact indexes, fragmentoff is 64 bits */
 557                 if (fragment &&
 558                     vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY)
 559                         vi->z_fragmentoff |= (u64)m.pblk << 32;
 560         }
 561         if (ztailpacking && m.lcn == vi->z_tailextent_headlcn) {
 562                 map->m_flags |= EROFS_MAP_META;
 563                 map->m_pa = vi->z_idataoff;
 564                 map->m_plen = vi->z_idata_size;
 565         } else if (fragment && m.lcn == vi->z_tailextent_headlcn) {
 566                 map->m_flags |= EROFS_MAP_FRAGMENT;
 567         } else {
 568                 map->m_pa = blknr_to_addr(m.pblk);
 569                 err = z_erofs_get_extent_compressedlen(&m, initial_lcn);
 570                 if (err)
 571                         goto unmap_out;
 572         }
 573
 574         if (m.headtype == Z_EROFS_VLE_CLUSTER_TYPE_PLAIN) {
 575                 if (map->m_llen > map->m_plen) {
 576                         DBG_BUGON(1);
 577                         err = -EFSCORRUPTED;
 578                         goto unmap_out;
 579                 }
 580                 if (vi->z_advise & Z_EROFS_ADVISE_INTERLACED_PCLUSTER)
 581                         map->m_algorithmformat =
 582                                 Z_EROFS_COMPRESSION_INTERLACED;
 583                 else
 584                         map->m_algorithmformat =
 585                                 Z_EROFS_COMPRESSION_SHIFTED;
 586         } else if (m.headtype == Z_EROFS_VLE_CLUSTER_TYPE_HEAD2) {
 587                 map->m_algorithmformat = vi->z_algorithmtype[1];
 588         } else {
 589                 map->m_algorithmformat = vi->z_algorithmtype[0];
 590         }
 591
 592         if ((flags & EROFS_GET_BLOCKS_FIEMAP) ||
 593             ((flags & EROFS_GET_BLOCKS_READMORE) &&
 594              map->m_algorithmformat == Z_EROFS_COMPRESSION_LZMA &&
 595              map->m_llen >= EROFS_BLKSIZ)) {
 596                 err = z_erofs_get_extent_decompressedlen(&m);
 597                 if (!err)
 598                         map->m_flags |= EROFS_MAP_FULL_MAPPED;
 599         }
 600
 601 unmap_out:
 602         erofs_unmap_metabuf(&m.map->buf);
 603         erofs_dbg("%s, m_la %llu m_pa %llu m_llen %llu m_plen %llu m_flags 0%o",
 604                   __func__, map->m_la, map->m_pa,
 605                   map->m_llen, map->m_plen, map->m_flags);
 606         return err;
 607 }
 608
 609 static int z_erofs_fill_inode_lazy(struct inode *inode)
 610 {
 611         struct erofs_inode *const vi = EROFS_I(inode);
 612         struct super_block *const sb = inode->i_sb;
 613         int err, headnr;
 614         erofs_off_t pos;
 615         struct erofs_buf buf = __EROFS_BUF_INITIALIZER;
 616         void *kaddr;
 617         struct z_erofs_map_header *h;
 618
 619         if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags)) {
 620                 /*
 621                  * paired with smp_mb() at the end of the function to ensure
 622                  * fields will only be observed after the bit is set.
 623                  */
 624                 smp_mb();
 625                 return 0;
 626         }
 627
 628         if (wait_on_bit_lock(&vi->flags, EROFS_I_BL_Z_BIT, TASK_KILLABLE))
 629                 return -ERESTARTSYS;
 630
 631         err = 0;
 632         if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags))
 633                 goto out_unlock;
 634
 635         pos = ALIGN(erofs_iloc(inode) + vi->inode_isize + vi->xattr_isize, 8);
 636         kaddr = erofs_read_metabuf(&buf, sb, erofs_blknr(pos), EROFS_KMAP);
 637         if (IS_ERR(kaddr)) {
 638                 err = PTR_ERR(kaddr);
 639                 goto out_unlock;
 640         }
 641
 642         h = kaddr + erofs_blkoff(pos);
 643         /*
 644          * if the highest bit of the 8-byte map header is set, the whole file
 645          * is stored in the packed inode. The rest bits keeps z_fragmentoff.
 646          */
 647         if (h->h_clusterbits >> Z_EROFS_FRAGMENT_INODE_BIT) {
 648                 vi->z_advise = Z_EROFS_ADVISE_FRAGMENT_PCLUSTER;
 649                 vi->z_fragmentoff = le64_to_cpu(*(__le64 *)h) ^ (1ULL << 63);
 650                 vi->z_tailextent_headlcn = 0;
 651                 goto done;
 652         }
 653         vi->z_advise = le16_to_cpu(h->h_advise);
 654         vi->z_algorithmtype[0] = h->h_algorithmtype & 15;
 655         vi->z_algorithmtype[1] = h->h_algorithmtype >> 4;
 656
 657         headnr = 0;
 658         if (vi->z_algorithmtype[0] >= Z_EROFS_COMPRESSION_MAX ||
 659             vi->z_algorithmtype[++headnr] >= Z_EROFS_COMPRESSION_MAX) {
 660                 erofs_err(sb, "unknown HEAD%u format %u for nid %llu, please upgrade kernel",
 661                           headnr + 1, vi->z_algorithmtype[headnr], vi->nid);
 662                 err = -EOPNOTSUPP;
 663                 goto out_put_metabuf;
 664         }
 665
 666         vi->z_logical_clusterbits = LOG_BLOCK_SIZE + (h->h_clusterbits & 7);
 667         if (!erofs_sb_has_big_pcluster(EROFS_SB(sb)) &&
 668             vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 |
 669                             Z_EROFS_ADVISE_BIG_PCLUSTER_2)) {
 670                 erofs_err(sb, "per-inode big pcluster without sb feature for nid %llu",
 671                           vi->nid);
 672                 err = -EFSCORRUPTED;
 673                 goto out_put_metabuf;
 674         }
 675         if (vi->datalayout == EROFS_INODE_FLAT_COMPRESSION &&
 676             !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) ^
 677             !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2)) {
 678                 erofs_err(sb, "big pcluster head1/2 of compact indexes should be consistent for nid %llu",
 679                           vi->nid);
 680                 err = -EFSCORRUPTED;
 681                 goto out_put_metabuf;
 682         }
 683
 684         if (vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER) {
 685                 struct erofs_map_blocks map = {
 686                         .buf = __EROFS_BUF_INITIALIZER
 687                 };
 688
 689                 vi->z_idata_size = le16_to_cpu(h->h_idata_size);
 690                 err = z_erofs_do_map_blocks(inode, &map,
 691                                             EROFS_GET_BLOCKS_FINDTAIL);
 692                 erofs_put_metabuf(&map.buf);
 693
 694                 if (!map.m_plen ||
 695                     erofs_blkoff(map.m_pa) + map.m_plen > EROFS_BLKSIZ) {
 696                         erofs_err(sb, "invalid tail-packing pclustersize %llu",
 697                                   map.m_plen);
 698                         err = -EFSCORRUPTED;
 699                 }
 700                 if (err < 0)
 701                         goto out_put_metabuf;
 702         }
 703
 704         if (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER &&
 705             !(h->h_clusterbits >> Z_EROFS_FRAGMENT_INODE_BIT)) {
 706                 struct erofs_map_blocks map = {
 707                         .buf = __EROFS_BUF_INITIALIZER
 708                 };
 709
 710                 vi->z_fragmentoff = le32_to_cpu(h->h_fragmentoff);
 711                 err = z_erofs_do_map_blocks(inode, &map,
 712                                             EROFS_GET_BLOCKS_FINDTAIL);
 713                 erofs_put_metabuf(&map.buf);
 714                 if (err < 0)
 715                         goto out_put_metabuf;
 716         }
 717 done:
 718         /* paired with smp_mb() at the beginning of the function */
 719         smp_mb();
 720         set_bit(EROFS_I_Z_INITED_BIT, &vi->flags);
 721 out_put_metabuf:
 722         erofs_put_metabuf(&buf);
 723 out_unlock:
 724         clear_and_wake_up_bit(EROFS_I_BL_Z_BIT, &vi->flags);
 725         return err;
 726 }
 727
 728 int z_erofs_map_blocks_iter(struct inode *inode, struct erofs_map_blocks *map,
 729                             int flags)
 730 {
 731         struct erofs_inode *const vi = EROFS_I(inode);
 732         int err = 0;
 733
 734         trace_z_erofs_map_blocks_iter_enter(inode, map, flags);
 735
 736         /* when trying to read beyond EOF, leave it unmapped */
 737         if (map->m_la >= inode->i_size) {
 738                 map->m_llen = map->m_la + 1 - inode->i_size;
 739                 map->m_la = inode->i_size;
 740                 map->m_flags = 0;
 741                 goto out;
 742         }
 743
 744         err = z_erofs_fill_inode_lazy(inode);
 745         if (err)
 746                 goto out;
 747
 748         if ((vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) &&
 749             !vi->z_tailextent_headlcn) {
 750                 map->m_la = 0;
 751                 map->m_llen = inode->i_size;
 752                 map->m_flags = EROFS_MAP_MAPPED | EROFS_MAP_FULL_MAPPED |
 753                                 EROFS_MAP_FRAGMENT;
 754                 goto out;
 755         }
 756
 757         err = z_erofs_do_map_blocks(inode, map, flags);
 758 out:
 759         trace_z_erofs_map_blocks_iter_exit(inode, map, flags, err);
 760
 761         /* aggressively BUG_ON iff CONFIG_EROFS_FS_DEBUG is on */
 762         DBG_BUGON(err < 0 && err != -ENOMEM);
 763         return err;
 764 }
 765
 766 static int z_erofs_iomap_begin_report(struct inode *inode, loff_t offset,
 767                                 loff_t length, unsigned int flags,
 768                                 struct iomap *iomap, struct iomap *srcmap)
 769 {
 770         int ret;
 771         struct erofs_map_blocks map = { .m_la = offset };
 772
 773         ret = z_erofs_map_blocks_iter(inode, &map, EROFS_GET_BLOCKS_FIEMAP);
 774         erofs_put_metabuf(&map.buf);
 775         if (ret < 0)
 776                 return ret;
 777
 778         iomap->bdev = inode->i_sb->s_bdev;
 779         iomap->offset = map.m_la;
 780         iomap->length = map.m_llen;
 781         if (map.m_flags & EROFS_MAP_MAPPED) {
 782                 iomap->type = IOMAP_MAPPED;
 783                 iomap->addr = map.m_flags & EROFS_MAP_FRAGMENT ?
 784                               IOMAP_NULL_ADDR : map.m_pa;
 785         } else {
 786                 iomap->type = IOMAP_HOLE;
 787                 iomap->addr = IOMAP_NULL_ADDR;
 788                 /*
 789                  * No strict rule on how to describe extents for post EOF, yet
 790                  * we need to do like below. Otherwise, iomap itself will get
 791                  * into an endless loop on post EOF.
 792                  *
 793                  * Calculate the effective offset by subtracting extent start
 794                  * (map.m_la) from the requested offset, and add it to length.
 795                  * (NB: offset >= map.m_la always)
 796                  */
 797                 if (iomap->offset >= inode->i_size)
 798                         iomap->length = length + offset - map.m_la;
 799         }
 800         iomap->flags = 0;
 801         return 0;
 802 }
 803
 804 const struct iomap_ops z_erofs_iomap_report_ops = {
 805         .iomap_begin = z_erofs_iomap_begin_report,
 806 };