1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2018-2019 HUAWEI, Inc.
4 * https://www.huawei.com/
5 */
6 #include "internal.h"
7 #include <asm/unaligned.h>
8 #include <trace/events/erofs.h>
9
z_erofs_fill_inode(struct inode * inode)10 int z_erofs_fill_inode(struct inode *inode)
11 {
12 struct erofs_inode *const vi = EROFS_I(inode);
13 struct erofs_sb_info *sbi = EROFS_SB(inode->i_sb);
14
15 if (!erofs_sb_has_big_pcluster(sbi) &&
16 !erofs_sb_has_ztailpacking(sbi) && !erofs_sb_has_fragments(sbi) &&
17 vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY) {
18 vi->z_advise = 0;
19 vi->z_algorithmtype[0] = 0;
20 vi->z_algorithmtype[1] = 0;
21 vi->z_logical_clusterbits = inode->i_sb->s_blocksize_bits;
22 set_bit(EROFS_I_Z_INITED_BIT, &vi->flags);
23 }
24 inode->i_mapping->a_ops = &z_erofs_aops;
25 return 0;
26 }
27
28 struct z_erofs_maprecorder {
29 struct inode *inode;
30 struct erofs_map_blocks *map;
31 void *kaddr;
32
33 unsigned long lcn;
34 /* compression extent information gathered */
35 u8 type, headtype;
36 u16 clusterofs;
37 u16 delta[2];
38 erofs_blk_t pblk, compressedblks;
39 erofs_off_t nextpackoff;
40 bool partialref;
41 };
42
legacy_load_cluster_from_disk(struct z_erofs_maprecorder * m,unsigned long lcn)43 static int legacy_load_cluster_from_disk(struct z_erofs_maprecorder *m,
44 unsigned long lcn)
45 {
46 struct inode *const inode = m->inode;
47 struct erofs_inode *const vi = EROFS_I(inode);
48 const erofs_off_t pos =
49 Z_EROFS_VLE_LEGACY_INDEX_ALIGN(erofs_iloc(inode) +
50 vi->inode_isize + vi->xattr_isize) +
51 lcn * sizeof(struct z_erofs_vle_decompressed_index);
52 struct z_erofs_vle_decompressed_index *di;
53 unsigned int advise, type;
54
55 m->kaddr = erofs_read_metabuf(&m->map->buf, inode->i_sb,
56 erofs_blknr(inode->i_sb, pos), EROFS_KMAP_ATOMIC);
57 if (IS_ERR(m->kaddr))
58 return PTR_ERR(m->kaddr);
59
60 m->nextpackoff = pos + sizeof(struct z_erofs_vle_decompressed_index);
61 m->lcn = lcn;
62 di = m->kaddr + erofs_blkoff(inode->i_sb, pos);
63
64 advise = le16_to_cpu(di->di_advise);
65 type = (advise >> Z_EROFS_VLE_DI_CLUSTER_TYPE_BIT) &
66 ((1 << Z_EROFS_VLE_DI_CLUSTER_TYPE_BITS) - 1);
67 switch (type) {
68 case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD:
69 m->clusterofs = 1 << vi->z_logical_clusterbits;
70 m->delta[0] = le16_to_cpu(di->di_u.delta[0]);
71 if (m->delta[0] & Z_EROFS_VLE_DI_D0_CBLKCNT) {
72 if (!(vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 |
73 Z_EROFS_ADVISE_BIG_PCLUSTER_2))) {
74 DBG_BUGON(1);
75 return -EFSCORRUPTED;
76 }
77 m->compressedblks = m->delta[0] &
78 ~Z_EROFS_VLE_DI_D0_CBLKCNT;
79 m->delta[0] = 1;
80 }
81 m->delta[1] = le16_to_cpu(di->di_u.delta[1]);
82 break;
83 case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN:
84 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD1:
85 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD2:
86 if (advise & Z_EROFS_VLE_DI_PARTIAL_REF)
87 m->partialref = true;
88 m->clusterofs = le16_to_cpu(di->di_clusterofs);
89 if (m->clusterofs >= 1 << vi->z_logical_clusterbits) {
90 DBG_BUGON(1);
91 return -EFSCORRUPTED;
92 }
93 m->pblk = le32_to_cpu(di->di_u.blkaddr);
94 break;
95 default:
96 DBG_BUGON(1);
97 return -EOPNOTSUPP;
98 }
99 m->type = type;
100 return 0;
101 }
102
decode_compactedbits(unsigned int lobits,u8 * in,unsigned int pos,u8 * type)103 static unsigned int decode_compactedbits(unsigned int lobits,
104 u8 *in, unsigned int pos, u8 *type)
105 {
106 const unsigned int v = get_unaligned_le32(in + pos / 8) >> (pos & 7);
107 const unsigned int lo = v & ((1 << lobits) - 1);
108
109 *type = (v >> lobits) & 3;
110 return lo;
111 }
112
get_compacted_la_distance(unsigned int lobits,unsigned int encodebits,unsigned int vcnt,u8 * in,int i)113 static int get_compacted_la_distance(unsigned int lobits,
114 unsigned int encodebits,
115 unsigned int vcnt, u8 *in, int i)
116 {
117 unsigned int lo, d1 = 0;
118 u8 type;
119
120 DBG_BUGON(i >= vcnt);
121
122 do {
123 lo = decode_compactedbits(lobits, in, encodebits * i, &type);
124
125 if (type != Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD)
126 return d1;
127 ++d1;
128 } while (++i < vcnt);
129
130 /* vcnt - 1 (Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) item */
131 if (!(lo & Z_EROFS_VLE_DI_D0_CBLKCNT))
132 d1 += lo - 1;
133 return d1;
134 }
135
unpack_compacted_index(struct z_erofs_maprecorder * m,unsigned int amortizedshift,erofs_off_t pos,bool lookahead)136 static int unpack_compacted_index(struct z_erofs_maprecorder *m,
137 unsigned int amortizedshift,
138 erofs_off_t pos, bool lookahead)
139 {
140 struct erofs_inode *const vi = EROFS_I(m->inode);
141 const unsigned int lclusterbits = vi->z_logical_clusterbits;
142 unsigned int vcnt, base, lo, lobits, encodebits, nblk, eofs;
143 int i;
144 u8 *in, type;
145 bool big_pcluster;
146
147 if (1 << amortizedshift == 4 && lclusterbits <= 14)
148 vcnt = 2;
149 else if (1 << amortizedshift == 2 && lclusterbits <= 12)
150 vcnt = 16;
151 else
152 return -EOPNOTSUPP;
153
154 /* it doesn't equal to round_up(..) */
155 m->nextpackoff = round_down(pos, vcnt << amortizedshift) +
156 (vcnt << amortizedshift);
157 big_pcluster = vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1;
158 lobits = max(lclusterbits, ilog2(Z_EROFS_VLE_DI_D0_CBLKCNT) + 1U);
159 encodebits = ((vcnt << amortizedshift) - sizeof(__le32)) * 8 / vcnt;
160 eofs = erofs_blkoff(m->inode->i_sb, pos);
161 base = round_down(eofs, vcnt << amortizedshift);
162 in = m->kaddr + base;
163
164 i = (eofs - base) >> amortizedshift;
165
166 lo = decode_compactedbits(lobits, in, encodebits * i, &type);
167 m->type = type;
168 if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) {
169 m->clusterofs = 1 << lclusterbits;
170
171 /* figure out lookahead_distance: delta[1] if needed */
172 if (lookahead)
173 m->delta[1] = get_compacted_la_distance(lobits,
174 encodebits, vcnt, in, i);
175 if (lo & Z_EROFS_VLE_DI_D0_CBLKCNT) {
176 if (!big_pcluster) {
177 DBG_BUGON(1);
178 return -EFSCORRUPTED;
179 }
180 m->compressedblks = lo & ~Z_EROFS_VLE_DI_D0_CBLKCNT;
181 m->delta[0] = 1;
182 return 0;
183 } else if (i + 1 != (int)vcnt) {
184 m->delta[0] = lo;
185 return 0;
186 }
187 /*
188 * since the last lcluster in the pack is special,
189 * of which lo saves delta[1] rather than delta[0].
190 * Hence, get delta[0] by the previous lcluster indirectly.
191 */
192 lo = decode_compactedbits(lobits, in,
193 encodebits * (i - 1), &type);
194 if (type != Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD)
195 lo = 0;
196 else if (lo & Z_EROFS_VLE_DI_D0_CBLKCNT)
197 lo = 1;
198 m->delta[0] = lo + 1;
199 return 0;
200 }
201 m->clusterofs = lo;
202 m->delta[0] = 0;
203 /* figout out blkaddr (pblk) for HEAD lclusters */
204 if (!big_pcluster) {
205 nblk = 1;
206 while (i > 0) {
207 --i;
208 lo = decode_compactedbits(lobits, in,
209 encodebits * i, &type);
210 if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD)
211 i -= lo;
212
213 if (i >= 0)
214 ++nblk;
215 }
216 } else {
217 nblk = 0;
218 while (i > 0) {
219 --i;
220 lo = decode_compactedbits(lobits, in,
221 encodebits * i, &type);
222 if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) {
223 if (lo & Z_EROFS_VLE_DI_D0_CBLKCNT) {
224 --i;
225 nblk += lo & ~Z_EROFS_VLE_DI_D0_CBLKCNT;
226 continue;
227 }
228 /* bigpcluster shouldn't have plain d0 == 1 */
229 if (lo <= 1) {
230 DBG_BUGON(1);
231 return -EFSCORRUPTED;
232 }
233 i -= lo - 2;
234 continue;
235 }
236 ++nblk;
237 }
238 }
239 in += (vcnt << amortizedshift) - sizeof(__le32);
240 m->pblk = le32_to_cpu(*(__le32 *)in) + nblk;
241 return 0;
242 }
243
compacted_load_cluster_from_disk(struct z_erofs_maprecorder * m,unsigned long lcn,bool lookahead)244 static int compacted_load_cluster_from_disk(struct z_erofs_maprecorder *m,
245 unsigned long lcn, bool lookahead)
246 {
247 struct inode *const inode = m->inode;
248 struct erofs_inode *const vi = EROFS_I(inode);
249 const erofs_off_t ebase = sizeof(struct z_erofs_map_header) +
250 ALIGN(erofs_iloc(inode) + vi->inode_isize + vi->xattr_isize, 8);
251 unsigned int totalidx = erofs_iblks(inode);
252 unsigned int compacted_4b_initial, compacted_2b;
253 unsigned int amortizedshift;
254 erofs_off_t pos;
255
256 if (lcn >= totalidx)
257 return -EINVAL;
258
259 m->lcn = lcn;
260 /* used to align to 32-byte (compacted_2b) alignment */
261 compacted_4b_initial = (32 - ebase % 32) / 4;
262 if (compacted_4b_initial == 32 / 4)
263 compacted_4b_initial = 0;
264
265 if ((vi->z_advise & Z_EROFS_ADVISE_COMPACTED_2B) &&
266 compacted_4b_initial < totalidx)
267 compacted_2b = rounddown(totalidx - compacted_4b_initial, 16);
268 else
269 compacted_2b = 0;
270
271 pos = ebase;
272 if (lcn < compacted_4b_initial) {
273 amortizedshift = 2;
274 goto out;
275 }
276 pos += compacted_4b_initial * 4;
277 lcn -= compacted_4b_initial;
278
279 if (lcn < compacted_2b) {
280 amortizedshift = 1;
281 goto out;
282 }
283 pos += compacted_2b * 2;
284 lcn -= compacted_2b;
285 amortizedshift = 2;
286 out:
287 pos += lcn * (1 << amortizedshift);
288 m->kaddr = erofs_read_metabuf(&m->map->buf, inode->i_sb,
289 erofs_blknr(inode->i_sb, pos), EROFS_KMAP_ATOMIC);
290 if (IS_ERR(m->kaddr))
291 return PTR_ERR(m->kaddr);
292 return unpack_compacted_index(m, amortizedshift, pos, lookahead);
293 }
294
z_erofs_load_cluster_from_disk(struct z_erofs_maprecorder * m,unsigned int lcn,bool lookahead)295 static int z_erofs_load_cluster_from_disk(struct z_erofs_maprecorder *m,
296 unsigned int lcn, bool lookahead)
297 {
298 const unsigned int datamode = EROFS_I(m->inode)->datalayout;
299
300 if (datamode == EROFS_INODE_FLAT_COMPRESSION_LEGACY)
301 return legacy_load_cluster_from_disk(m, lcn);
302
303 if (datamode == EROFS_INODE_FLAT_COMPRESSION)
304 return compacted_load_cluster_from_disk(m, lcn, lookahead);
305
306 return -EINVAL;
307 }
308
z_erofs_extent_lookback(struct z_erofs_maprecorder * m,unsigned int lookback_distance)309 static int z_erofs_extent_lookback(struct z_erofs_maprecorder *m,
310 unsigned int lookback_distance)
311 {
312 struct erofs_inode *const vi = EROFS_I(m->inode);
313 const unsigned int lclusterbits = vi->z_logical_clusterbits;
314
315 while (m->lcn >= lookback_distance) {
316 unsigned long lcn = m->lcn - lookback_distance;
317 int err;
318
319 /* load extent head logical cluster if needed */
320 err = z_erofs_load_cluster_from_disk(m, lcn, false);
321 if (err)
322 return err;
323
324 switch (m->type) {
325 case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD:
326 if (!m->delta[0]) {
327 erofs_err(m->inode->i_sb,
328 "invalid lookback distance 0 @ nid %llu",
329 vi->nid);
330 DBG_BUGON(1);
331 return -EFSCORRUPTED;
332 }
333 lookback_distance = m->delta[0];
334 continue;
335 case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN:
336 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD1:
337 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD2:
338 m->headtype = m->type;
339 m->map->m_la = (lcn << lclusterbits) | m->clusterofs;
340 return 0;
341 default:
342 erofs_err(m->inode->i_sb,
343 "unknown type %u @ lcn %lu of nid %llu",
344 m->type, lcn, vi->nid);
345 DBG_BUGON(1);
346 return -EOPNOTSUPP;
347 }
348 }
349
350 erofs_err(m->inode->i_sb, "bogus lookback distance @ nid %llu",
351 vi->nid);
352 DBG_BUGON(1);
353 return -EFSCORRUPTED;
354 }
355
z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder * m,unsigned int initial_lcn)356 static int z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m,
357 unsigned int initial_lcn)
358 {
359 struct super_block *sb = m->inode->i_sb;
360 struct erofs_inode *const vi = EROFS_I(m->inode);
361 struct erofs_map_blocks *const map = m->map;
362 const unsigned int lclusterbits = vi->z_logical_clusterbits;
363 unsigned long lcn;
364 int err;
365
366 DBG_BUGON(m->type != Z_EROFS_VLE_CLUSTER_TYPE_PLAIN &&
367 m->type != Z_EROFS_VLE_CLUSTER_TYPE_HEAD1 &&
368 m->type != Z_EROFS_VLE_CLUSTER_TYPE_HEAD2);
369 DBG_BUGON(m->type != m->headtype);
370
371 if (m->headtype == Z_EROFS_VLE_CLUSTER_TYPE_PLAIN ||
372 ((m->headtype == Z_EROFS_VLE_CLUSTER_TYPE_HEAD1) &&
373 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1)) ||
374 ((m->headtype == Z_EROFS_VLE_CLUSTER_TYPE_HEAD2) &&
375 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2))) {
376 map->m_plen = 1ULL << lclusterbits;
377 return 0;
378 }
379 lcn = m->lcn + 1;
380 if (m->compressedblks)
381 goto out;
382
383 err = z_erofs_load_cluster_from_disk(m, lcn, false);
384 if (err)
385 return err;
386
387 /*
388 * If the 1st NONHEAD lcluster has already been handled initially w/o
389 * valid compressedblks, which means at least it mustn't be CBLKCNT, or
390 * an internal implemenatation error is detected.
391 *
392 * The following code can also handle it properly anyway, but let's
393 * BUG_ON in the debugging mode only for developers to notice that.
394 */
395 DBG_BUGON(lcn == initial_lcn &&
396 m->type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD);
397
398 switch (m->type) {
399 case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN:
400 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD1:
401 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD2:
402 /*
403 * if the 1st NONHEAD lcluster is actually PLAIN or HEAD type
404 * rather than CBLKCNT, it's a 1 lcluster-sized pcluster.
405 */
406 m->compressedblks = 1 << (lclusterbits - sb->s_blocksize_bits);
407 break;
408 case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD:
409 if (m->delta[0] != 1)
410 goto err_bonus_cblkcnt;
411 if (m->compressedblks)
412 break;
413 fallthrough;
414 default:
415 erofs_err(m->inode->i_sb,
416 "cannot found CBLKCNT @ lcn %lu of nid %llu",
417 lcn, vi->nid);
418 DBG_BUGON(1);
419 return -EFSCORRUPTED;
420 }
421 out:
422 map->m_plen = erofs_pos(sb, m->compressedblks);
423 return 0;
424 err_bonus_cblkcnt:
425 erofs_err(m->inode->i_sb,
426 "bogus CBLKCNT @ lcn %lu of nid %llu",
427 lcn, vi->nid);
428 DBG_BUGON(1);
429 return -EFSCORRUPTED;
430 }
431
z_erofs_get_extent_decompressedlen(struct z_erofs_maprecorder * m)432 static int z_erofs_get_extent_decompressedlen(struct z_erofs_maprecorder *m)
433 {
434 struct inode *inode = m->inode;
435 struct erofs_inode *vi = EROFS_I(inode);
436 struct erofs_map_blocks *map = m->map;
437 unsigned int lclusterbits = vi->z_logical_clusterbits;
438 u64 lcn = m->lcn, headlcn = map->m_la >> lclusterbits;
439 int err;
440
441 do {
442 /* handle the last EOF pcluster (no next HEAD lcluster) */
443 if ((lcn << lclusterbits) >= inode->i_size) {
444 map->m_llen = inode->i_size - map->m_la;
445 return 0;
446 }
447
448 err = z_erofs_load_cluster_from_disk(m, lcn, true);
449 if (err)
450 return err;
451
452 if (m->type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) {
453 DBG_BUGON(!m->delta[1] &&
454 m->clusterofs != 1 << lclusterbits);
455 } else if (m->type == Z_EROFS_VLE_CLUSTER_TYPE_PLAIN ||
456 m->type == Z_EROFS_VLE_CLUSTER_TYPE_HEAD1 ||
457 m->type == Z_EROFS_VLE_CLUSTER_TYPE_HEAD2) {
458 /* go on until the next HEAD lcluster */
459 if (lcn != headlcn)
460 break;
461 m->delta[1] = 1;
462 } else {
463 erofs_err(inode->i_sb, "unknown type %u @ lcn %llu of nid %llu",
464 m->type, lcn, vi->nid);
465 DBG_BUGON(1);
466 return -EOPNOTSUPP;
467 }
468 lcn += m->delta[1];
469 } while (m->delta[1]);
470
471 map->m_llen = (lcn << lclusterbits) + m->clusterofs - map->m_la;
472 return 0;
473 }
474
z_erofs_do_map_blocks(struct inode * inode,struct erofs_map_blocks * map,int flags)475 static int z_erofs_do_map_blocks(struct inode *inode,
476 struct erofs_map_blocks *map,
477 int flags)
478 {
479 struct erofs_inode *const vi = EROFS_I(inode);
480 bool ztailpacking = vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER;
481 bool fragment = vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER;
482 struct z_erofs_maprecorder m = {
483 .inode = inode,
484 .map = map,
485 };
486 int err = 0;
487 unsigned int lclusterbits, endoff, afmt;
488 unsigned long initial_lcn;
489 unsigned long long ofs, end;
490
491 lclusterbits = vi->z_logical_clusterbits;
492 ofs = flags & EROFS_GET_BLOCKS_FINDTAIL ? inode->i_size - 1 : map->m_la;
493 initial_lcn = ofs >> lclusterbits;
494 endoff = ofs & ((1 << lclusterbits) - 1);
495
496 err = z_erofs_load_cluster_from_disk(&m, initial_lcn, false);
497 if (err)
498 goto unmap_out;
499
500 if (ztailpacking && (flags & EROFS_GET_BLOCKS_FINDTAIL))
501 vi->z_idataoff = m.nextpackoff;
502
503 map->m_flags = EROFS_MAP_MAPPED | EROFS_MAP_ENCODED;
504 end = (m.lcn + 1ULL) << lclusterbits;
505
506 switch (m.type) {
507 case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN:
508 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD1:
509 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD2:
510 if (endoff >= m.clusterofs) {
511 m.headtype = m.type;
512 map->m_la = (m.lcn << lclusterbits) | m.clusterofs;
513 /*
514 * For ztailpacking files, in order to inline data more
515 * effectively, special EOF lclusters are now supported
516 * which can have three parts at most.
517 */
518 if (ztailpacking && end > inode->i_size)
519 end = inode->i_size;
520 break;
521 }
522 /* m.lcn should be >= 1 if endoff < m.clusterofs */
523 if (!m.lcn) {
524 erofs_err(inode->i_sb,
525 "invalid logical cluster 0 at nid %llu",
526 vi->nid);
527 err = -EFSCORRUPTED;
528 goto unmap_out;
529 }
530 end = (m.lcn << lclusterbits) | m.clusterofs;
531 map->m_flags |= EROFS_MAP_FULL_MAPPED;
532 m.delta[0] = 1;
533 fallthrough;
534 case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD:
535 /* get the corresponding first chunk */
536 err = z_erofs_extent_lookback(&m, m.delta[0]);
537 if (err)
538 goto unmap_out;
539 break;
540 default:
541 erofs_err(inode->i_sb,
542 "unknown type %u @ offset %llu of nid %llu",
543 m.type, ofs, vi->nid);
544 err = -EOPNOTSUPP;
545 goto unmap_out;
546 }
547 if (m.partialref)
548 map->m_flags |= EROFS_MAP_PARTIAL_REF;
549 map->m_llen = end - map->m_la;
550
551 if (flags & EROFS_GET_BLOCKS_FINDTAIL) {
552 vi->z_tailextent_headlcn = m.lcn;
553 /* for non-compact indexes, fragmentoff is 64 bits */
554 if (fragment &&
555 vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY)
556 vi->z_fragmentoff |= (u64)m.pblk << 32;
557 }
558 if (ztailpacking && m.lcn == vi->z_tailextent_headlcn) {
559 map->m_flags |= EROFS_MAP_META;
560 map->m_pa = vi->z_idataoff;
561 map->m_plen = vi->z_idata_size;
562 } else if (fragment && m.lcn == vi->z_tailextent_headlcn) {
563 map->m_flags |= EROFS_MAP_FRAGMENT;
564 } else {
565 map->m_pa = erofs_pos(inode->i_sb, m.pblk);
566 err = z_erofs_get_extent_compressedlen(&m, initial_lcn);
567 if (err)
568 goto unmap_out;
569 }
570
571 if (m.headtype == Z_EROFS_VLE_CLUSTER_TYPE_PLAIN) {
572 if (map->m_llen > map->m_plen) {
573 DBG_BUGON(1);
574 err = -EFSCORRUPTED;
575 goto unmap_out;
576 }
577 afmt = vi->z_advise & Z_EROFS_ADVISE_INTERLACED_PCLUSTER ?
578 Z_EROFS_COMPRESSION_INTERLACED :
579 Z_EROFS_COMPRESSION_SHIFTED;
580 } else {
581 afmt = m.headtype == Z_EROFS_VLE_CLUSTER_TYPE_HEAD2 ?
582 vi->z_algorithmtype[1] : vi->z_algorithmtype[0];
583 if (!(EROFS_I_SB(inode)->available_compr_algs & (1 << afmt))) {
584 erofs_err(inode->i_sb, "inconsistent algorithmtype %u for nid %llu",
585 afmt, vi->nid);
586 err = -EFSCORRUPTED;
587 goto unmap_out;
588 }
589 }
590 map->m_algorithmformat = afmt;
591
592 if ((flags & EROFS_GET_BLOCKS_FIEMAP) ||
593 ((flags & EROFS_GET_BLOCKS_READMORE) &&
594 map->m_algorithmformat == Z_EROFS_COMPRESSION_LZMA &&
595 map->m_llen >= i_blocksize(inode))) {
596 err = z_erofs_get_extent_decompressedlen(&m);
597 if (!err)
598 map->m_flags |= EROFS_MAP_FULL_MAPPED;
599 }
600
601 unmap_out:
602 erofs_unmap_metabuf(&m.map->buf);
603 erofs_dbg("%s, m_la %llu m_pa %llu m_llen %llu m_plen %llu m_flags 0%o",
604 __func__, map->m_la, map->m_pa,
605 map->m_llen, map->m_plen, map->m_flags);
606 return err;
607 }
608
z_erofs_fill_inode_lazy(struct inode * inode)609 static int z_erofs_fill_inode_lazy(struct inode *inode)
610 {
611 struct erofs_inode *const vi = EROFS_I(inode);
612 struct super_block *const sb = inode->i_sb;
613 int err, headnr;
614 erofs_off_t pos;
615 struct erofs_buf buf = __EROFS_BUF_INITIALIZER;
616 void *kaddr;
617 struct z_erofs_map_header *h;
618
619 if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags)) {
620 /*
621 * paired with smp_mb() at the end of the function to ensure
622 * fields will only be observed after the bit is set.
623 */
624 smp_mb();
625 return 0;
626 }
627
628 if (wait_on_bit_lock(&vi->flags, EROFS_I_BL_Z_BIT, TASK_KILLABLE))
629 return -ERESTARTSYS;
630
631 err = 0;
632 if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags))
633 goto out_unlock;
634
635 pos = ALIGN(erofs_iloc(inode) + vi->inode_isize + vi->xattr_isize, 8);
636 kaddr = erofs_read_metabuf(&buf, sb, erofs_blknr(sb, pos), EROFS_KMAP);
637 if (IS_ERR(kaddr)) {
638 err = PTR_ERR(kaddr);
639 goto out_unlock;
640 }
641
642 h = kaddr + erofs_blkoff(sb, pos);
643 /*
644 * if the highest bit of the 8-byte map header is set, the whole file
645 * is stored in the packed inode. The rest bits keeps z_fragmentoff.
646 */
647 if (h->h_clusterbits >> Z_EROFS_FRAGMENT_INODE_BIT) {
648 vi->z_advise = Z_EROFS_ADVISE_FRAGMENT_PCLUSTER;
649 vi->z_fragmentoff = le64_to_cpu(*(__le64 *)h) ^ (1ULL << 63);
650 vi->z_tailextent_headlcn = 0;
651 goto done;
652 }
653 vi->z_advise = le16_to_cpu(h->h_advise);
654 vi->z_algorithmtype[0] = h->h_algorithmtype & 15;
655 vi->z_algorithmtype[1] = h->h_algorithmtype >> 4;
656
657 headnr = 0;
658 if (vi->z_algorithmtype[0] >= Z_EROFS_COMPRESSION_MAX ||
659 vi->z_algorithmtype[++headnr] >= Z_EROFS_COMPRESSION_MAX) {
660 erofs_err(sb, "unknown HEAD%u format %u for nid %llu, please upgrade kernel",
661 headnr + 1, vi->z_algorithmtype[headnr], vi->nid);
662 err = -EOPNOTSUPP;
663 goto out_put_metabuf;
664 }
665
666 vi->z_logical_clusterbits = sb->s_blocksize_bits + (h->h_clusterbits & 7);
667 if (!erofs_sb_has_big_pcluster(EROFS_SB(sb)) &&
668 vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 |
669 Z_EROFS_ADVISE_BIG_PCLUSTER_2)) {
670 erofs_err(sb, "per-inode big pcluster without sb feature for nid %llu",
671 vi->nid);
672 err = -EFSCORRUPTED;
673 goto out_put_metabuf;
674 }
675 if (vi->datalayout == EROFS_INODE_FLAT_COMPRESSION &&
676 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) ^
677 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2)) {
678 erofs_err(sb, "big pcluster head1/2 of compact indexes should be consistent for nid %llu",
679 vi->nid);
680 err = -EFSCORRUPTED;
681 goto out_put_metabuf;
682 }
683
684 if (vi->z_advise & Z_EROFS_ADVISE_INLINE_PCLUSTER) {
685 struct erofs_map_blocks map = {
686 .buf = __EROFS_BUF_INITIALIZER
687 };
688
689 vi->z_idata_size = le16_to_cpu(h->h_idata_size);
690 err = z_erofs_do_map_blocks(inode, &map,
691 EROFS_GET_BLOCKS_FINDTAIL);
692 erofs_put_metabuf(&map.buf);
693
694 if (!map.m_plen ||
695 erofs_blkoff(sb, map.m_pa) + map.m_plen > sb->s_blocksize) {
696 erofs_err(sb, "invalid tail-packing pclustersize %llu",
697 map.m_plen);
698 err = -EFSCORRUPTED;
699 }
700 if (err < 0)
701 goto out_put_metabuf;
702 }
703
704 if (vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER &&
705 !(h->h_clusterbits >> Z_EROFS_FRAGMENT_INODE_BIT)) {
706 struct erofs_map_blocks map = {
707 .buf = __EROFS_BUF_INITIALIZER
708 };
709
710 vi->z_fragmentoff = le32_to_cpu(h->h_fragmentoff);
711 err = z_erofs_do_map_blocks(inode, &map,
712 EROFS_GET_BLOCKS_FINDTAIL);
713 erofs_put_metabuf(&map.buf);
714 if (err < 0)
715 goto out_put_metabuf;
716 }
717 done:
718 /* paired with smp_mb() at the beginning of the function */
719 smp_mb();
720 set_bit(EROFS_I_Z_INITED_BIT, &vi->flags);
721 out_put_metabuf:
722 erofs_put_metabuf(&buf);
723 out_unlock:
724 clear_and_wake_up_bit(EROFS_I_BL_Z_BIT, &vi->flags);
725 return err;
726 }
727
z_erofs_map_blocks_iter(struct inode * inode,struct erofs_map_blocks * map,int flags)728 int z_erofs_map_blocks_iter(struct inode *inode, struct erofs_map_blocks *map,
729 int flags)
730 {
731 struct erofs_inode *const vi = EROFS_I(inode);
732 int err = 0;
733
734 trace_z_erofs_map_blocks_iter_enter(inode, map, flags);
735
736 /* when trying to read beyond EOF, leave it unmapped */
737 if (map->m_la >= inode->i_size) {
738 map->m_llen = map->m_la + 1 - inode->i_size;
739 map->m_la = inode->i_size;
740 map->m_flags = 0;
741 goto out;
742 }
743
744 err = z_erofs_fill_inode_lazy(inode);
745 if (err)
746 goto out;
747
748 if ((vi->z_advise & Z_EROFS_ADVISE_FRAGMENT_PCLUSTER) &&
749 !vi->z_tailextent_headlcn) {
750 map->m_la = 0;
751 map->m_llen = inode->i_size;
752 map->m_flags = EROFS_MAP_MAPPED | EROFS_MAP_FULL_MAPPED |
753 EROFS_MAP_FRAGMENT;
754 goto out;
755 }
756
757 err = z_erofs_do_map_blocks(inode, map, flags);
758 out:
759 trace_z_erofs_map_blocks_iter_exit(inode, map, flags, err);
760
761 /* aggressively BUG_ON iff CONFIG_EROFS_FS_DEBUG is on */
762 DBG_BUGON(err < 0 && err != -ENOMEM);
763 return err;
764 }
765
z_erofs_iomap_begin_report(struct inode * inode,loff_t offset,loff_t length,unsigned int flags,struct iomap * iomap,struct iomap * srcmap)766 static int z_erofs_iomap_begin_report(struct inode *inode, loff_t offset,
767 loff_t length, unsigned int flags,
768 struct iomap *iomap, struct iomap *srcmap)
769 {
770 int ret;
771 struct erofs_map_blocks map = { .m_la = offset };
772
773 ret = z_erofs_map_blocks_iter(inode, &map, EROFS_GET_BLOCKS_FIEMAP);
774 erofs_put_metabuf(&map.buf);
775 if (ret < 0)
776 return ret;
777
778 iomap->bdev = inode->i_sb->s_bdev;
779 iomap->offset = map.m_la;
780 iomap->length = map.m_llen;
781 if (map.m_flags & EROFS_MAP_MAPPED) {
782 iomap->type = IOMAP_MAPPED;
783 iomap->addr = map.m_flags & EROFS_MAP_FRAGMENT ?
784 IOMAP_NULL_ADDR : map.m_pa;
785 } else {
786 iomap->type = IOMAP_HOLE;
787 iomap->addr = IOMAP_NULL_ADDR;
788 /*
789 * No strict rule on how to describe extents for post EOF, yet
790 * we need to do like below. Otherwise, iomap itself will get
791 * into an endless loop on post EOF.
792 *
793 * Calculate the effective offset by subtracting extent start
794 * (map.m_la) from the requested offset, and add it to length.
795 * (NB: offset >= map.m_la always)
796 */
797 if (iomap->offset >= inode->i_size)
798 iomap->length = length + offset - map.m_la;
799 }
800 iomap->flags = 0;
801 return 0;
802 }
803
804 const struct iomap_ops z_erofs_iomap_report_ops = {
805 .iomap_begin = z_erofs_iomap_begin_report,
806 };
807