1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2018-2019 HUAWEI, Inc.
4 * https://www.huawei.com/
5 */
6 #include "internal.h"
7 #include <asm/unaligned.h>
8 #include <trace/events/erofs.h>
9
z_erofs_fill_inode(struct inode * inode)10 int z_erofs_fill_inode(struct inode *inode)
11 {
12 struct erofs_inode *const vi = EROFS_I(inode);
13 struct erofs_sb_info *sbi = EROFS_SB(inode->i_sb);
14
15 if (!erofs_sb_has_big_pcluster(sbi) &&
16 vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY) {
17 vi->z_advise = 0;
18 vi->z_algorithmtype[0] = 0;
19 vi->z_algorithmtype[1] = 0;
20 vi->z_logical_clusterbits = LOG_BLOCK_SIZE;
21 set_bit(EROFS_I_Z_INITED_BIT, &vi->flags);
22 }
23 inode->i_mapping->a_ops = &z_erofs_aops;
24 return 0;
25 }
26
z_erofs_fill_inode_lazy(struct inode * inode)27 static int z_erofs_fill_inode_lazy(struct inode *inode)
28 {
29 struct erofs_inode *const vi = EROFS_I(inode);
30 struct super_block *const sb = inode->i_sb;
31 int err, headnr;
32 erofs_off_t pos;
33 struct page *page;
34 void *kaddr;
35 struct z_erofs_map_header *h;
36
37 if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags)) {
38 /*
39 * paired with smp_mb() at the end of the function to ensure
40 * fields will only be observed after the bit is set.
41 */
42 smp_mb();
43 return 0;
44 }
45
46 if (wait_on_bit_lock(&vi->flags, EROFS_I_BL_Z_BIT, TASK_KILLABLE))
47 return -ERESTARTSYS;
48
49 err = 0;
50 if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags))
51 goto out_unlock;
52
53 DBG_BUGON(!erofs_sb_has_big_pcluster(EROFS_SB(sb)) &&
54 vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY);
55
56 pos = ALIGN(iloc(EROFS_SB(sb), vi->nid) + vi->inode_isize +
57 vi->xattr_isize, 8);
58 page = erofs_get_meta_page(sb, erofs_blknr(pos));
59 if (IS_ERR(page)) {
60 err = PTR_ERR(page);
61 goto out_unlock;
62 }
63
64 kaddr = kmap_atomic(page);
65
66 h = kaddr + erofs_blkoff(pos);
67 vi->z_advise = le16_to_cpu(h->h_advise);
68 vi->z_algorithmtype[0] = h->h_algorithmtype & 15;
69 vi->z_algorithmtype[1] = h->h_algorithmtype >> 4;
70
71 headnr = 0;
72 if (vi->z_algorithmtype[0] >= Z_EROFS_COMPRESSION_MAX ||
73 vi->z_algorithmtype[++headnr] >= Z_EROFS_COMPRESSION_MAX) {
74 erofs_err(sb, "unknown HEAD%u format %u for nid %llu, please upgrade kernel",
75 headnr + 1, vi->z_algorithmtype[headnr], vi->nid);
76 err = -EOPNOTSUPP;
77 goto unmap_done;
78 }
79
80 vi->z_logical_clusterbits = LOG_BLOCK_SIZE + (h->h_clusterbits & 7);
81 if (!erofs_sb_has_big_pcluster(EROFS_SB(sb)) &&
82 vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 |
83 Z_EROFS_ADVISE_BIG_PCLUSTER_2)) {
84 erofs_err(sb, "per-inode big pcluster without sb feature for nid %llu",
85 vi->nid);
86 err = -EFSCORRUPTED;
87 goto unmap_done;
88 }
89 if (vi->datalayout == EROFS_INODE_FLAT_COMPRESSION &&
90 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) ^
91 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2)) {
92 erofs_err(sb, "big pcluster head1/2 of compact indexes should be consistent for nid %llu",
93 vi->nid);
94 err = -EFSCORRUPTED;
95 goto unmap_done;
96 }
97 /* paired with smp_mb() at the beginning of the function */
98 smp_mb();
99 set_bit(EROFS_I_Z_INITED_BIT, &vi->flags);
100 unmap_done:
101 kunmap_atomic(kaddr);
102 unlock_page(page);
103 put_page(page);
104 out_unlock:
105 clear_and_wake_up_bit(EROFS_I_BL_Z_BIT, &vi->flags);
106 return err;
107 }
108
109 struct z_erofs_maprecorder {
110 struct inode *inode;
111 struct erofs_map_blocks *map;
112 void *kaddr;
113
114 unsigned long lcn;
115 /* compression extent information gathered */
116 u8 type, headtype;
117 u16 clusterofs;
118 u16 delta[2];
119 erofs_blk_t pblk, compressedlcs;
120 };
121
z_erofs_reload_indexes(struct z_erofs_maprecorder * m,erofs_blk_t eblk)122 static int z_erofs_reload_indexes(struct z_erofs_maprecorder *m,
123 erofs_blk_t eblk)
124 {
125 struct super_block *const sb = m->inode->i_sb;
126 struct erofs_map_blocks *const map = m->map;
127 struct page *mpage = map->mpage;
128
129 if (mpage) {
130 if (mpage->index == eblk) {
131 if (!m->kaddr)
132 m->kaddr = kmap_atomic(mpage);
133 return 0;
134 }
135
136 if (m->kaddr) {
137 kunmap_atomic(m->kaddr);
138 m->kaddr = NULL;
139 }
140 put_page(mpage);
141 }
142
143 mpage = erofs_get_meta_page(sb, eblk);
144 if (IS_ERR(mpage)) {
145 map->mpage = NULL;
146 return PTR_ERR(mpage);
147 }
148 m->kaddr = kmap_atomic(mpage);
149 unlock_page(mpage);
150 map->mpage = mpage;
151 return 0;
152 }
153
legacy_load_cluster_from_disk(struct z_erofs_maprecorder * m,unsigned long lcn)154 static int legacy_load_cluster_from_disk(struct z_erofs_maprecorder *m,
155 unsigned long lcn)
156 {
157 struct inode *const inode = m->inode;
158 struct erofs_inode *const vi = EROFS_I(inode);
159 const erofs_off_t ibase = iloc(EROFS_I_SB(inode), vi->nid);
160 const erofs_off_t pos =
161 Z_EROFS_VLE_LEGACY_INDEX_ALIGN(ibase + vi->inode_isize +
162 vi->xattr_isize) +
163 lcn * sizeof(struct z_erofs_vle_decompressed_index);
164 struct z_erofs_vle_decompressed_index *di;
165 unsigned int advise, type;
166 int err;
167
168 err = z_erofs_reload_indexes(m, erofs_blknr(pos));
169 if (err)
170 return err;
171
172 m->lcn = lcn;
173 di = m->kaddr + erofs_blkoff(pos);
174
175 advise = le16_to_cpu(di->di_advise);
176 type = (advise >> Z_EROFS_VLE_DI_CLUSTER_TYPE_BIT) &
177 ((1 << Z_EROFS_VLE_DI_CLUSTER_TYPE_BITS) - 1);
178 switch (type) {
179 case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD:
180 m->clusterofs = 1 << vi->z_logical_clusterbits;
181 m->delta[0] = le16_to_cpu(di->di_u.delta[0]);
182 if (m->delta[0] & Z_EROFS_VLE_DI_D0_CBLKCNT) {
183 if (!(vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 |
184 Z_EROFS_ADVISE_BIG_PCLUSTER_2))) {
185 DBG_BUGON(1);
186 return -EFSCORRUPTED;
187 }
188 m->compressedlcs = m->delta[0] &
189 ~Z_EROFS_VLE_DI_D0_CBLKCNT;
190 m->delta[0] = 1;
191 }
192 m->delta[1] = le16_to_cpu(di->di_u.delta[1]);
193 break;
194 case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN:
195 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD1:
196 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD2:
197 m->clusterofs = le16_to_cpu(di->di_clusterofs);
198 if (m->clusterofs >= 1 << vi->z_logical_clusterbits) {
199 DBG_BUGON(1);
200 return -EFSCORRUPTED;
201 }
202 m->pblk = le32_to_cpu(di->di_u.blkaddr);
203 break;
204 default:
205 DBG_BUGON(1);
206 return -EOPNOTSUPP;
207 }
208 m->type = type;
209 return 0;
210 }
211
decode_compactedbits(unsigned int lobits,unsigned int lomask,u8 * in,unsigned int pos,u8 * type)212 static unsigned int decode_compactedbits(unsigned int lobits,
213 unsigned int lomask,
214 u8 *in, unsigned int pos, u8 *type)
215 {
216 const unsigned int v = get_unaligned_le32(in + pos / 8) >> (pos & 7);
217 const unsigned int lo = v & lomask;
218
219 *type = (v >> lobits) & 3;
220 return lo;
221 }
222
get_compacted_la_distance(unsigned int lclusterbits,unsigned int encodebits,unsigned int vcnt,u8 * in,int i)223 static int get_compacted_la_distance(unsigned int lclusterbits,
224 unsigned int encodebits,
225 unsigned int vcnt, u8 *in, int i)
226 {
227 const unsigned int lomask = (1 << lclusterbits) - 1;
228 unsigned int lo, d1 = 0;
229 u8 type;
230
231 DBG_BUGON(i >= vcnt);
232
233 do {
234 lo = decode_compactedbits(lclusterbits, lomask,
235 in, encodebits * i, &type);
236
237 if (type != Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD)
238 return d1;
239 ++d1;
240 } while (++i < vcnt);
241
242 /* vcnt - 1 (Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) item */
243 if (!(lo & Z_EROFS_VLE_DI_D0_CBLKCNT))
244 d1 += lo - 1;
245 return d1;
246 }
247
unpack_compacted_index(struct z_erofs_maprecorder * m,unsigned int amortizedshift,unsigned int eofs,bool lookahead)248 static int unpack_compacted_index(struct z_erofs_maprecorder *m,
249 unsigned int amortizedshift,
250 unsigned int eofs, bool lookahead)
251 {
252 struct erofs_inode *const vi = EROFS_I(m->inode);
253 const unsigned int lclusterbits = vi->z_logical_clusterbits;
254 const unsigned int lomask = (1 << lclusterbits) - 1;
255 unsigned int vcnt, base, lo, encodebits, nblk;
256 int i;
257 u8 *in, type;
258 bool big_pcluster;
259
260 if (1 << amortizedshift == 4 && lclusterbits <= 14)
261 vcnt = 2;
262 else if (1 << amortizedshift == 2 && lclusterbits == 12)
263 vcnt = 16;
264 else
265 return -EOPNOTSUPP;
266
267 big_pcluster = vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1;
268 encodebits = ((vcnt << amortizedshift) - sizeof(__le32)) * 8 / vcnt;
269 base = round_down(eofs, vcnt << amortizedshift);
270 in = m->kaddr + base;
271
272 i = (eofs - base) >> amortizedshift;
273
274 lo = decode_compactedbits(lclusterbits, lomask,
275 in, encodebits * i, &type);
276 m->type = type;
277 if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) {
278 m->clusterofs = 1 << lclusterbits;
279
280 /* figure out lookahead_distance: delta[1] if needed */
281 if (lookahead)
282 m->delta[1] = get_compacted_la_distance(lclusterbits,
283 encodebits, vcnt, in, i);
284 if (lo & Z_EROFS_VLE_DI_D0_CBLKCNT) {
285 if (!big_pcluster) {
286 DBG_BUGON(1);
287 return -EFSCORRUPTED;
288 }
289 m->compressedlcs = lo & ~Z_EROFS_VLE_DI_D0_CBLKCNT;
290 m->delta[0] = 1;
291 return 0;
292 } else if (i + 1 != (int)vcnt) {
293 m->delta[0] = lo;
294 return 0;
295 }
296 /*
297 * since the last lcluster in the pack is special,
298 * of which lo saves delta[1] rather than delta[0].
299 * Hence, get delta[0] by the previous lcluster indirectly.
300 */
301 lo = decode_compactedbits(lclusterbits, lomask,
302 in, encodebits * (i - 1), &type);
303 if (type != Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD)
304 lo = 0;
305 else if (lo & Z_EROFS_VLE_DI_D0_CBLKCNT)
306 lo = 1;
307 m->delta[0] = lo + 1;
308 return 0;
309 }
310 m->clusterofs = lo;
311 m->delta[0] = 0;
312 /* figout out blkaddr (pblk) for HEAD lclusters */
313 if (!big_pcluster) {
314 nblk = 1;
315 while (i > 0) {
316 --i;
317 lo = decode_compactedbits(lclusterbits, lomask,
318 in, encodebits * i, &type);
319 if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD)
320 i -= lo;
321
322 if (i >= 0)
323 ++nblk;
324 }
325 } else {
326 nblk = 0;
327 while (i > 0) {
328 --i;
329 lo = decode_compactedbits(lclusterbits, lomask,
330 in, encodebits * i, &type);
331 if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) {
332 if (lo & Z_EROFS_VLE_DI_D0_CBLKCNT) {
333 --i;
334 nblk += lo & ~Z_EROFS_VLE_DI_D0_CBLKCNT;
335 continue;
336 }
337 /* bigpcluster shouldn't have plain d0 == 1 */
338 if (lo <= 1) {
339 DBG_BUGON(1);
340 return -EFSCORRUPTED;
341 }
342 i -= lo - 2;
343 continue;
344 }
345 ++nblk;
346 }
347 }
348 in += (vcnt << amortizedshift) - sizeof(__le32);
349 m->pblk = le32_to_cpu(*(__le32 *)in) + nblk;
350 return 0;
351 }
352
compacted_load_cluster_from_disk(struct z_erofs_maprecorder * m,unsigned long lcn,bool lookahead)353 static int compacted_load_cluster_from_disk(struct z_erofs_maprecorder *m,
354 unsigned long lcn, bool lookahead)
355 {
356 struct inode *const inode = m->inode;
357 struct erofs_inode *const vi = EROFS_I(inode);
358 const erofs_off_t ebase = ALIGN(iloc(EROFS_I_SB(inode), vi->nid) +
359 vi->inode_isize + vi->xattr_isize, 8) +
360 sizeof(struct z_erofs_map_header);
361 const unsigned int totalidx = DIV_ROUND_UP(inode->i_size, EROFS_BLKSIZ);
362 unsigned int compacted_4b_initial, compacted_2b;
363 unsigned int amortizedshift;
364 erofs_off_t pos;
365 int err;
366
367 if (lcn >= totalidx)
368 return -EINVAL;
369
370 m->lcn = lcn;
371 /* used to align to 32-byte (compacted_2b) alignment */
372 compacted_4b_initial = (32 - ebase % 32) / 4;
373 if (compacted_4b_initial == 32 / 4)
374 compacted_4b_initial = 0;
375
376 if (vi->z_advise & Z_EROFS_ADVISE_COMPACTED_2B)
377 compacted_2b = rounddown(totalidx - compacted_4b_initial, 16);
378 else
379 compacted_2b = 0;
380
381 pos = ebase;
382 if (lcn < compacted_4b_initial) {
383 amortizedshift = 2;
384 goto out;
385 }
386 pos += compacted_4b_initial * 4;
387 lcn -= compacted_4b_initial;
388
389 if (lcn < compacted_2b) {
390 amortizedshift = 1;
391 goto out;
392 }
393 pos += compacted_2b * 2;
394 lcn -= compacted_2b;
395 amortizedshift = 2;
396 out:
397 pos += lcn * (1 << amortizedshift);
398 err = z_erofs_reload_indexes(m, erofs_blknr(pos));
399 if (err)
400 return err;
401 return unpack_compacted_index(m, amortizedshift, erofs_blkoff(pos),
402 lookahead);
403 }
404
z_erofs_load_cluster_from_disk(struct z_erofs_maprecorder * m,unsigned int lcn,bool lookahead)405 static int z_erofs_load_cluster_from_disk(struct z_erofs_maprecorder *m,
406 unsigned int lcn, bool lookahead)
407 {
408 const unsigned int datamode = EROFS_I(m->inode)->datalayout;
409
410 if (datamode == EROFS_INODE_FLAT_COMPRESSION_LEGACY)
411 return legacy_load_cluster_from_disk(m, lcn);
412
413 if (datamode == EROFS_INODE_FLAT_COMPRESSION)
414 return compacted_load_cluster_from_disk(m, lcn, lookahead);
415
416 return -EINVAL;
417 }
418
z_erofs_extent_lookback(struct z_erofs_maprecorder * m,unsigned int lookback_distance)419 static int z_erofs_extent_lookback(struct z_erofs_maprecorder *m,
420 unsigned int lookback_distance)
421 {
422 struct erofs_inode *const vi = EROFS_I(m->inode);
423 struct erofs_map_blocks *const map = m->map;
424 const unsigned int lclusterbits = vi->z_logical_clusterbits;
425 unsigned long lcn = m->lcn;
426 int err;
427
428 if (lcn < lookback_distance) {
429 erofs_err(m->inode->i_sb,
430 "bogus lookback distance @ nid %llu", vi->nid);
431 DBG_BUGON(1);
432 return -EFSCORRUPTED;
433 }
434
435 /* load extent head logical cluster if needed */
436 lcn -= lookback_distance;
437 err = z_erofs_load_cluster_from_disk(m, lcn, false);
438 if (err)
439 return err;
440
441 switch (m->type) {
442 case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD:
443 if (!m->delta[0]) {
444 erofs_err(m->inode->i_sb,
445 "invalid lookback distance 0 @ nid %llu",
446 vi->nid);
447 DBG_BUGON(1);
448 return -EFSCORRUPTED;
449 }
450 return z_erofs_extent_lookback(m, m->delta[0]);
451 case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN:
452 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD1:
453 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD2:
454 m->headtype = m->type;
455 map->m_la = (lcn << lclusterbits) | m->clusterofs;
456 break;
457 default:
458 erofs_err(m->inode->i_sb,
459 "unknown type %u @ lcn %lu of nid %llu",
460 m->type, lcn, vi->nid);
461 DBG_BUGON(1);
462 return -EOPNOTSUPP;
463 }
464 return 0;
465 }
466
z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder * m,unsigned int initial_lcn)467 static int z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m,
468 unsigned int initial_lcn)
469 {
470 struct erofs_inode *const vi = EROFS_I(m->inode);
471 struct erofs_map_blocks *const map = m->map;
472 const unsigned int lclusterbits = vi->z_logical_clusterbits;
473 unsigned long lcn;
474 int err;
475
476 DBG_BUGON(m->type != Z_EROFS_VLE_CLUSTER_TYPE_PLAIN &&
477 m->type != Z_EROFS_VLE_CLUSTER_TYPE_HEAD1 &&
478 m->type != Z_EROFS_VLE_CLUSTER_TYPE_HEAD2);
479 DBG_BUGON(m->type != m->headtype);
480
481 if (m->headtype == Z_EROFS_VLE_CLUSTER_TYPE_PLAIN ||
482 ((m->headtype == Z_EROFS_VLE_CLUSTER_TYPE_HEAD1) &&
483 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1)) ||
484 ((m->headtype == Z_EROFS_VLE_CLUSTER_TYPE_HEAD2) &&
485 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2))) {
486 map->m_plen = 1 << lclusterbits;
487 return 0;
488 }
489 lcn = m->lcn + 1;
490 if (m->compressedlcs)
491 goto out;
492
493 err = z_erofs_load_cluster_from_disk(m, lcn, false);
494 if (err)
495 return err;
496
497 /*
498 * If the 1st NONHEAD lcluster has already been handled initially w/o
499 * valid compressedlcs, which means at least it mustn't be CBLKCNT, or
500 * an internal implemenatation error is detected.
501 *
502 * The following code can also handle it properly anyway, but let's
503 * BUG_ON in the debugging mode only for developers to notice that.
504 */
505 DBG_BUGON(lcn == initial_lcn &&
506 m->type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD);
507
508 switch (m->type) {
509 case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN:
510 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD1:
511 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD2:
512 /*
513 * if the 1st NONHEAD lcluster is actually PLAIN or HEAD type
514 * rather than CBLKCNT, it's a 1 lcluster-sized pcluster.
515 */
516 m->compressedlcs = 1;
517 break;
518 case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD:
519 if (m->delta[0] != 1)
520 goto err_bonus_cblkcnt;
521 if (m->compressedlcs)
522 break;
523 fallthrough;
524 default:
525 erofs_err(m->inode->i_sb,
526 "cannot found CBLKCNT @ lcn %lu of nid %llu",
527 lcn, vi->nid);
528 DBG_BUGON(1);
529 return -EFSCORRUPTED;
530 }
531 out:
532 map->m_plen = m->compressedlcs << lclusterbits;
533 return 0;
534 err_bonus_cblkcnt:
535 erofs_err(m->inode->i_sb,
536 "bogus CBLKCNT @ lcn %lu of nid %llu",
537 lcn, vi->nid);
538 DBG_BUGON(1);
539 return -EFSCORRUPTED;
540 }
541
z_erofs_get_extent_decompressedlen(struct z_erofs_maprecorder * m)542 static int z_erofs_get_extent_decompressedlen(struct z_erofs_maprecorder *m)
543 {
544 struct inode *inode = m->inode;
545 struct erofs_inode *vi = EROFS_I(inode);
546 struct erofs_map_blocks *map = m->map;
547 unsigned int lclusterbits = vi->z_logical_clusterbits;
548 u64 lcn = m->lcn, headlcn = map->m_la >> lclusterbits;
549 int err;
550
551 do {
552 /* handle the last EOF pcluster (no next HEAD lcluster) */
553 if ((lcn << lclusterbits) >= inode->i_size) {
554 map->m_llen = inode->i_size - map->m_la;
555 return 0;
556 }
557
558 err = z_erofs_load_cluster_from_disk(m, lcn, true);
559 if (err)
560 return err;
561
562 if (m->type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) {
563 DBG_BUGON(!m->delta[1] &&
564 m->clusterofs != 1 << lclusterbits);
565 } else if (m->type == Z_EROFS_VLE_CLUSTER_TYPE_PLAIN ||
566 m->type == Z_EROFS_VLE_CLUSTER_TYPE_HEAD1 ||
567 m->type == Z_EROFS_VLE_CLUSTER_TYPE_HEAD2) {
568 /* go on until the next HEAD lcluster */
569 if (lcn != headlcn)
570 break;
571 m->delta[1] = 1;
572 } else {
573 erofs_err(inode->i_sb, "unknown type %u @ lcn %llu of nid %llu",
574 m->type, lcn, vi->nid);
575 DBG_BUGON(1);
576 return -EOPNOTSUPP;
577 }
578 lcn += m->delta[1];
579 } while (m->delta[1]);
580
581 map->m_llen = (lcn << lclusterbits) + m->clusterofs - map->m_la;
582 return 0;
583 }
584
z_erofs_map_blocks_iter(struct inode * inode,struct erofs_map_blocks * map,int flags)585 int z_erofs_map_blocks_iter(struct inode *inode,
586 struct erofs_map_blocks *map,
587 int flags)
588 {
589 struct erofs_inode *const vi = EROFS_I(inode);
590 struct z_erofs_maprecorder m = {
591 .inode = inode,
592 .map = map,
593 };
594 int err = 0;
595 unsigned int lclusterbits, endoff;
596 unsigned long initial_lcn;
597 unsigned long long ofs, end;
598
599 trace_z_erofs_map_blocks_iter_enter(inode, map, flags);
600
601 /* when trying to read beyond EOF, leave it unmapped */
602 if (map->m_la >= inode->i_size) {
603 map->m_llen = map->m_la + 1 - inode->i_size;
604 map->m_la = inode->i_size;
605 map->m_flags = 0;
606 goto out;
607 }
608
609 err = z_erofs_fill_inode_lazy(inode);
610 if (err)
611 goto out;
612
613 lclusterbits = vi->z_logical_clusterbits;
614 ofs = map->m_la;
615 initial_lcn = ofs >> lclusterbits;
616 endoff = ofs & ((1 << lclusterbits) - 1);
617
618 err = z_erofs_load_cluster_from_disk(&m, initial_lcn, false);
619 if (err)
620 goto unmap_out;
621
622 map->m_flags = EROFS_MAP_MAPPED | EROFS_MAP_ENCODED;
623 end = (m.lcn + 1ULL) << lclusterbits;
624
625 switch (m.type) {
626 case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN:
627 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD1:
628 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD2:
629 if (endoff >= m.clusterofs) {
630 m.headtype = m.type;
631 map->m_la = (m.lcn << lclusterbits) | m.clusterofs;
632 break;
633 }
634 /* m.lcn should be >= 1 if endoff < m.clusterofs */
635 if (!m.lcn) {
636 erofs_err(inode->i_sb,
637 "invalid logical cluster 0 at nid %llu",
638 vi->nid);
639 err = -EFSCORRUPTED;
640 goto unmap_out;
641 }
642 end = (m.lcn << lclusterbits) | m.clusterofs;
643 map->m_flags |= EROFS_MAP_FULL_MAPPED;
644 m.delta[0] = 1;
645 fallthrough;
646 case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD:
647 /* get the corresponding first chunk */
648 err = z_erofs_extent_lookback(&m, m.delta[0]);
649 if (err)
650 goto unmap_out;
651 break;
652 default:
653 erofs_err(inode->i_sb,
654 "unknown type %u @ offset %llu of nid %llu",
655 m.type, ofs, vi->nid);
656 err = -EOPNOTSUPP;
657 goto unmap_out;
658 }
659
660 map->m_llen = end - map->m_la;
661 map->m_pa = blknr_to_addr(m.pblk);
662
663 err = z_erofs_get_extent_compressedlen(&m, initial_lcn);
664 if (err)
665 goto out;
666
667 if (m.headtype == Z_EROFS_VLE_CLUSTER_TYPE_PLAIN)
668 map->m_algorithmformat = Z_EROFS_COMPRESSION_SHIFTED;
669 else if (m.headtype == Z_EROFS_VLE_CLUSTER_TYPE_HEAD2)
670 map->m_algorithmformat = vi->z_algorithmtype[1];
671 else
672 map->m_algorithmformat = vi->z_algorithmtype[0];
673
674 if ((flags & EROFS_GET_BLOCKS_FIEMAP) ||
675 ((flags & EROFS_GET_BLOCKS_READMORE) &&
676 map->m_algorithmformat == Z_EROFS_COMPRESSION_LZMA &&
677 map->m_llen >= EROFS_BLKSIZ)) {
678 err = z_erofs_get_extent_decompressedlen(&m);
679 if (!err)
680 map->m_flags |= EROFS_MAP_FULL_MAPPED;
681 }
682 unmap_out:
683 if (m.kaddr)
684 kunmap_atomic(m.kaddr);
685
686 out:
687 erofs_dbg("%s, m_la %llu m_pa %llu m_llen %llu m_plen %llu m_flags 0%o",
688 __func__, map->m_la, map->m_pa,
689 map->m_llen, map->m_plen, map->m_flags);
690
691 trace_z_erofs_map_blocks_iter_exit(inode, map, flags, err);
692
693 /* aggressively BUG_ON iff CONFIG_EROFS_FS_DEBUG is on */
694 DBG_BUGON(err < 0 && err != -ENOMEM);
695 return err;
696 }
697
z_erofs_iomap_begin_report(struct inode * inode,loff_t offset,loff_t length,unsigned int flags,struct iomap * iomap,struct iomap * srcmap)698 static int z_erofs_iomap_begin_report(struct inode *inode, loff_t offset,
699 loff_t length, unsigned int flags,
700 struct iomap *iomap, struct iomap *srcmap)
701 {
702 int ret;
703 struct erofs_map_blocks map = { .m_la = offset };
704
705 ret = z_erofs_map_blocks_iter(inode, &map, EROFS_GET_BLOCKS_FIEMAP);
706 if (map.mpage)
707 put_page(map.mpage);
708 if (ret < 0)
709 return ret;
710
711 iomap->bdev = inode->i_sb->s_bdev;
712 iomap->offset = map.m_la;
713 iomap->length = map.m_llen;
714 if (map.m_flags & EROFS_MAP_MAPPED) {
715 iomap->type = IOMAP_MAPPED;
716 iomap->addr = map.m_pa;
717 } else {
718 iomap->type = IOMAP_HOLE;
719 iomap->addr = IOMAP_NULL_ADDR;
720 /*
721 * No strict rule how to describe extents for post EOF, yet
722 * we need do like below. Otherwise, iomap itself will get
723 * into an endless loop on post EOF.
724 */
725 if (iomap->offset >= inode->i_size)
726 iomap->length = length + map.m_la - offset;
727 }
728 iomap->flags = 0;
729 return 0;
730 }
731
732 const struct iomap_ops z_erofs_iomap_report_ops = {
733 .iomap_begin = z_erofs_iomap_begin_report,
734 };
735