1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2018-2019 HUAWEI, Inc.
4 * https://www.huawei.com/
5 * Created by Gao Xiang <gaoxiang25@huawei.com>
6 */
7 #include "internal.h"
8 #include <asm/unaligned.h>
9 #include <trace/events/erofs.h>
10
z_erofs_fill_inode(struct inode * inode)11 int z_erofs_fill_inode(struct inode *inode)
12 {
13 struct erofs_inode *const vi = EROFS_I(inode);
14
15 if (vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY) {
16 vi->z_advise = 0;
17 vi->z_algorithmtype[0] = 0;
18 vi->z_algorithmtype[1] = 0;
19 vi->z_logical_clusterbits = LOG_BLOCK_SIZE;
20 vi->z_physical_clusterbits[0] = vi->z_logical_clusterbits;
21 vi->z_physical_clusterbits[1] = vi->z_logical_clusterbits;
22 set_bit(EROFS_I_Z_INITED_BIT, &vi->flags);
23 }
24
25 inode->i_mapping->a_ops = &z_erofs_aops;
26 return 0;
27 }
28
z_erofs_fill_inode_lazy(struct inode * inode)29 static int z_erofs_fill_inode_lazy(struct inode *inode)
30 {
31 struct erofs_inode *const vi = EROFS_I(inode);
32 struct super_block *const sb = inode->i_sb;
33 int err;
34 erofs_off_t pos;
35 struct page *page;
36 void *kaddr;
37 struct z_erofs_map_header *h;
38
39 if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags)) {
40 /*
41 * paired with smp_mb() at the end of the function to ensure
42 * fields will only be observed after the bit is set.
43 */
44 smp_mb();
45 return 0;
46 }
47
48 if (wait_on_bit_lock(&vi->flags, EROFS_I_BL_Z_BIT, TASK_KILLABLE))
49 return -ERESTARTSYS;
50
51 err = 0;
52 if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags))
53 goto out_unlock;
54
55 DBG_BUGON(vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY);
56
57 pos = ALIGN(iloc(EROFS_SB(sb), vi->nid) + vi->inode_isize +
58 vi->xattr_isize, 8);
59 page = erofs_get_meta_page(sb, erofs_blknr(pos));
60 if (IS_ERR(page)) {
61 err = PTR_ERR(page);
62 goto out_unlock;
63 }
64
65 kaddr = kmap_atomic(page);
66
67 h = kaddr + erofs_blkoff(pos);
68 vi->z_advise = le16_to_cpu(h->h_advise);
69 vi->z_algorithmtype[0] = h->h_algorithmtype & 15;
70 vi->z_algorithmtype[1] = h->h_algorithmtype >> 4;
71
72 if (vi->z_algorithmtype[0] >= Z_EROFS_COMPRESSION_MAX) {
73 erofs_err(sb, "unknown compression format %u for nid %llu, please upgrade kernel",
74 vi->z_algorithmtype[0], vi->nid);
75 err = -EOPNOTSUPP;
76 goto unmap_done;
77 }
78
79 vi->z_logical_clusterbits = LOG_BLOCK_SIZE + (h->h_clusterbits & 7);
80 vi->z_physical_clusterbits[0] = vi->z_logical_clusterbits +
81 ((h->h_clusterbits >> 3) & 3);
82
83 if (vi->z_physical_clusterbits[0] != LOG_BLOCK_SIZE) {
84 erofs_err(sb, "unsupported physical clusterbits %u for nid %llu, please upgrade kernel",
85 vi->z_physical_clusterbits[0], vi->nid);
86 err = -EOPNOTSUPP;
87 goto unmap_done;
88 }
89
90 vi->z_physical_clusterbits[1] = vi->z_logical_clusterbits +
91 ((h->h_clusterbits >> 5) & 7);
92 /* paired with smp_mb() at the beginning of the function */
93 smp_mb();
94 set_bit(EROFS_I_Z_INITED_BIT, &vi->flags);
95 unmap_done:
96 kunmap_atomic(kaddr);
97 unlock_page(page);
98 put_page(page);
99 out_unlock:
100 clear_and_wake_up_bit(EROFS_I_BL_Z_BIT, &vi->flags);
101 return err;
102 }
103
104 struct z_erofs_maprecorder {
105 struct inode *inode;
106 struct erofs_map_blocks *map;
107 void *kaddr;
108
109 unsigned long lcn;
110 /* compression extent information gathered */
111 u8 type;
112 u16 clusterofs;
113 u16 delta[2];
114 erofs_blk_t pblk;
115 };
116
z_erofs_reload_indexes(struct z_erofs_maprecorder * m,erofs_blk_t eblk)117 static int z_erofs_reload_indexes(struct z_erofs_maprecorder *m,
118 erofs_blk_t eblk)
119 {
120 struct super_block *const sb = m->inode->i_sb;
121 struct erofs_map_blocks *const map = m->map;
122 struct page *mpage = map->mpage;
123
124 if (mpage) {
125 if (mpage->index == eblk) {
126 if (!m->kaddr)
127 m->kaddr = kmap_atomic(mpage);
128 return 0;
129 }
130
131 if (m->kaddr) {
132 kunmap_atomic(m->kaddr);
133 m->kaddr = NULL;
134 }
135 put_page(mpage);
136 }
137
138 mpage = erofs_get_meta_page(sb, eblk);
139 if (IS_ERR(mpage)) {
140 map->mpage = NULL;
141 return PTR_ERR(mpage);
142 }
143 m->kaddr = kmap_atomic(mpage);
144 unlock_page(mpage);
145 map->mpage = mpage;
146 return 0;
147 }
148
legacy_load_cluster_from_disk(struct z_erofs_maprecorder * m,unsigned long lcn)149 static int legacy_load_cluster_from_disk(struct z_erofs_maprecorder *m,
150 unsigned long lcn)
151 {
152 struct inode *const inode = m->inode;
153 struct erofs_inode *const vi = EROFS_I(inode);
154 const erofs_off_t ibase = iloc(EROFS_I_SB(inode), vi->nid);
155 const erofs_off_t pos =
156 Z_EROFS_VLE_LEGACY_INDEX_ALIGN(ibase + vi->inode_isize +
157 vi->xattr_isize) +
158 lcn * sizeof(struct z_erofs_vle_decompressed_index);
159 struct z_erofs_vle_decompressed_index *di;
160 unsigned int advise, type;
161 int err;
162
163 err = z_erofs_reload_indexes(m, erofs_blknr(pos));
164 if (err)
165 return err;
166
167 m->lcn = lcn;
168 di = m->kaddr + erofs_blkoff(pos);
169
170 advise = le16_to_cpu(di->di_advise);
171 type = (advise >> Z_EROFS_VLE_DI_CLUSTER_TYPE_BIT) &
172 ((1 << Z_EROFS_VLE_DI_CLUSTER_TYPE_BITS) - 1);
173 switch (type) {
174 case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD:
175 m->clusterofs = 1 << vi->z_logical_clusterbits;
176 m->delta[0] = le16_to_cpu(di->di_u.delta[0]);
177 m->delta[1] = le16_to_cpu(di->di_u.delta[1]);
178 break;
179 case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN:
180 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD:
181 m->clusterofs = le16_to_cpu(di->di_clusterofs);
182 m->pblk = le32_to_cpu(di->di_u.blkaddr);
183 break;
184 default:
185 DBG_BUGON(1);
186 return -EOPNOTSUPP;
187 }
188 m->type = type;
189 return 0;
190 }
191
decode_compactedbits(unsigned int lobits,unsigned int lomask,u8 * in,unsigned int pos,u8 * type)192 static unsigned int decode_compactedbits(unsigned int lobits,
193 unsigned int lomask,
194 u8 *in, unsigned int pos, u8 *type)
195 {
196 const unsigned int v = get_unaligned_le32(in + pos / 8) >> (pos & 7);
197 const unsigned int lo = v & lomask;
198
199 *type = (v >> lobits) & 3;
200 return lo;
201 }
202
unpack_compacted_index(struct z_erofs_maprecorder * m,unsigned int amortizedshift,unsigned int eofs)203 static int unpack_compacted_index(struct z_erofs_maprecorder *m,
204 unsigned int amortizedshift,
205 unsigned int eofs)
206 {
207 struct erofs_inode *const vi = EROFS_I(m->inode);
208 const unsigned int lclusterbits = vi->z_logical_clusterbits;
209 const unsigned int lomask = (1 << lclusterbits) - 1;
210 unsigned int vcnt, base, lo, encodebits, nblk;
211 int i;
212 u8 *in, type;
213
214 if (1 << amortizedshift == 4)
215 vcnt = 2;
216 else if (1 << amortizedshift == 2 && lclusterbits == 12)
217 vcnt = 16;
218 else
219 return -EOPNOTSUPP;
220
221 encodebits = ((vcnt << amortizedshift) - sizeof(__le32)) * 8 / vcnt;
222 base = round_down(eofs, vcnt << amortizedshift);
223 in = m->kaddr + base;
224
225 i = (eofs - base) >> amortizedshift;
226
227 lo = decode_compactedbits(lclusterbits, lomask,
228 in, encodebits * i, &type);
229 m->type = type;
230 if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) {
231 m->clusterofs = 1 << lclusterbits;
232 if (i + 1 != vcnt) {
233 m->delta[0] = lo;
234 return 0;
235 }
236 /*
237 * since the last lcluster in the pack is special,
238 * of which lo saves delta[1] rather than delta[0].
239 * Hence, get delta[0] by the previous lcluster indirectly.
240 */
241 lo = decode_compactedbits(lclusterbits, lomask,
242 in, encodebits * (i - 1), &type);
243 if (type != Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD)
244 lo = 0;
245 m->delta[0] = lo + 1;
246 return 0;
247 }
248 m->clusterofs = lo;
249 m->delta[0] = 0;
250 /* figout out blkaddr (pblk) for HEAD lclusters */
251 nblk = 1;
252 while (i > 0) {
253 --i;
254 lo = decode_compactedbits(lclusterbits, lomask,
255 in, encodebits * i, &type);
256 if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD)
257 i -= lo;
258
259 if (i >= 0)
260 ++nblk;
261 }
262 in += (vcnt << amortizedshift) - sizeof(__le32);
263 m->pblk = le32_to_cpu(*(__le32 *)in) + nblk;
264 return 0;
265 }
266
compacted_load_cluster_from_disk(struct z_erofs_maprecorder * m,unsigned long lcn)267 static int compacted_load_cluster_from_disk(struct z_erofs_maprecorder *m,
268 unsigned long lcn)
269 {
270 struct inode *const inode = m->inode;
271 struct erofs_inode *const vi = EROFS_I(inode);
272 const unsigned int lclusterbits = vi->z_logical_clusterbits;
273 const erofs_off_t ebase = ALIGN(iloc(EROFS_I_SB(inode), vi->nid) +
274 vi->inode_isize + vi->xattr_isize, 8) +
275 sizeof(struct z_erofs_map_header);
276 const unsigned int totalidx = DIV_ROUND_UP(inode->i_size, EROFS_BLKSIZ);
277 unsigned int compacted_4b_initial, compacted_2b;
278 unsigned int amortizedshift;
279 erofs_off_t pos;
280 int err;
281
282 if (lclusterbits != 12)
283 return -EOPNOTSUPP;
284
285 if (lcn >= totalidx)
286 return -EINVAL;
287
288 m->lcn = lcn;
289 /* used to align to 32-byte (compacted_2b) alignment */
290 compacted_4b_initial = (32 - ebase % 32) / 4;
291 if (compacted_4b_initial == 32 / 4)
292 compacted_4b_initial = 0;
293
294 if (vi->z_advise & Z_EROFS_ADVISE_COMPACTED_2B)
295 compacted_2b = rounddown(totalidx - compacted_4b_initial, 16);
296 else
297 compacted_2b = 0;
298
299 pos = ebase;
300 if (lcn < compacted_4b_initial) {
301 amortizedshift = 2;
302 goto out;
303 }
304 pos += compacted_4b_initial * 4;
305 lcn -= compacted_4b_initial;
306
307 if (lcn < compacted_2b) {
308 amortizedshift = 1;
309 goto out;
310 }
311 pos += compacted_2b * 2;
312 lcn -= compacted_2b;
313 amortizedshift = 2;
314 out:
315 pos += lcn * (1 << amortizedshift);
316 err = z_erofs_reload_indexes(m, erofs_blknr(pos));
317 if (err)
318 return err;
319 return unpack_compacted_index(m, amortizedshift, erofs_blkoff(pos));
320 }
321
z_erofs_load_cluster_from_disk(struct z_erofs_maprecorder * m,unsigned int lcn)322 static int z_erofs_load_cluster_from_disk(struct z_erofs_maprecorder *m,
323 unsigned int lcn)
324 {
325 const unsigned int datamode = EROFS_I(m->inode)->datalayout;
326
327 if (datamode == EROFS_INODE_FLAT_COMPRESSION_LEGACY)
328 return legacy_load_cluster_from_disk(m, lcn);
329
330 if (datamode == EROFS_INODE_FLAT_COMPRESSION)
331 return compacted_load_cluster_from_disk(m, lcn);
332
333 return -EINVAL;
334 }
335
z_erofs_extent_lookback(struct z_erofs_maprecorder * m,unsigned int lookback_distance)336 static int z_erofs_extent_lookback(struct z_erofs_maprecorder *m,
337 unsigned int lookback_distance)
338 {
339 struct erofs_inode *const vi = EROFS_I(m->inode);
340 struct erofs_map_blocks *const map = m->map;
341 const unsigned int lclusterbits = vi->z_logical_clusterbits;
342 unsigned long lcn = m->lcn;
343 int err;
344
345 if (lcn < lookback_distance) {
346 erofs_err(m->inode->i_sb,
347 "bogus lookback distance @ nid %llu", vi->nid);
348 DBG_BUGON(1);
349 return -EFSCORRUPTED;
350 }
351
352 /* load extent head logical cluster if needed */
353 lcn -= lookback_distance;
354 err = z_erofs_load_cluster_from_disk(m, lcn);
355 if (err)
356 return err;
357
358 switch (m->type) {
359 case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD:
360 if (!m->delta[0]) {
361 erofs_err(m->inode->i_sb,
362 "invalid lookback distance 0 @ nid %llu",
363 vi->nid);
364 DBG_BUGON(1);
365 return -EFSCORRUPTED;
366 }
367 return z_erofs_extent_lookback(m, m->delta[0]);
368 case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN:
369 map->m_flags &= ~EROFS_MAP_ZIPPED;
370 fallthrough;
371 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD:
372 map->m_la = (lcn << lclusterbits) | m->clusterofs;
373 break;
374 default:
375 erofs_err(m->inode->i_sb,
376 "unknown type %u @ lcn %lu of nid %llu",
377 m->type, lcn, vi->nid);
378 DBG_BUGON(1);
379 return -EOPNOTSUPP;
380 }
381 return 0;
382 }
383
z_erofs_map_blocks_iter(struct inode * inode,struct erofs_map_blocks * map,int flags)384 int z_erofs_map_blocks_iter(struct inode *inode,
385 struct erofs_map_blocks *map,
386 int flags)
387 {
388 struct erofs_inode *const vi = EROFS_I(inode);
389 struct z_erofs_maprecorder m = {
390 .inode = inode,
391 .map = map,
392 };
393 int err = 0;
394 unsigned int lclusterbits, endoff;
395 unsigned long long ofs, end;
396
397 trace_z_erofs_map_blocks_iter_enter(inode, map, flags);
398
399 /* when trying to read beyond EOF, leave it unmapped */
400 if (map->m_la >= inode->i_size) {
401 map->m_llen = map->m_la + 1 - inode->i_size;
402 map->m_la = inode->i_size;
403 map->m_flags = 0;
404 goto out;
405 }
406
407 err = z_erofs_fill_inode_lazy(inode);
408 if (err)
409 goto out;
410
411 lclusterbits = vi->z_logical_clusterbits;
412 ofs = map->m_la;
413 m.lcn = ofs >> lclusterbits;
414 endoff = ofs & ((1 << lclusterbits) - 1);
415
416 err = z_erofs_load_cluster_from_disk(&m, m.lcn);
417 if (err)
418 goto unmap_out;
419
420 map->m_flags = EROFS_MAP_ZIPPED; /* by default, compressed */
421 end = (m.lcn + 1ULL) << lclusterbits;
422
423 switch (m.type) {
424 case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN:
425 if (endoff >= m.clusterofs)
426 map->m_flags &= ~EROFS_MAP_ZIPPED;
427 fallthrough;
428 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD:
429 if (endoff >= m.clusterofs) {
430 map->m_la = (m.lcn << lclusterbits) | m.clusterofs;
431 break;
432 }
433 /* m.lcn should be >= 1 if endoff < m.clusterofs */
434 if (!m.lcn) {
435 erofs_err(inode->i_sb,
436 "invalid logical cluster 0 at nid %llu",
437 vi->nid);
438 err = -EFSCORRUPTED;
439 goto unmap_out;
440 }
441 end = (m.lcn << lclusterbits) | m.clusterofs;
442 map->m_flags |= EROFS_MAP_FULL_MAPPED;
443 m.delta[0] = 1;
444 fallthrough;
445 case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD:
446 /* get the correspoinding first chunk */
447 err = z_erofs_extent_lookback(&m, m.delta[0]);
448 if (err)
449 goto unmap_out;
450 break;
451 default:
452 erofs_err(inode->i_sb,
453 "unknown type %u @ offset %llu of nid %llu",
454 m.type, ofs, vi->nid);
455 err = -EOPNOTSUPP;
456 goto unmap_out;
457 }
458
459 map->m_llen = end - map->m_la;
460 map->m_plen = 1 << lclusterbits;
461 map->m_pa = blknr_to_addr(m.pblk);
462 map->m_flags |= EROFS_MAP_MAPPED;
463
464 unmap_out:
465 if (m.kaddr)
466 kunmap_atomic(m.kaddr);
467
468 out:
469 erofs_dbg("%s, m_la %llu m_pa %llu m_llen %llu m_plen %llu m_flags 0%o",
470 __func__, map->m_la, map->m_pa,
471 map->m_llen, map->m_plen, map->m_flags);
472
473 trace_z_erofs_map_blocks_iter_exit(inode, map, flags, err);
474
475 /* aggressively BUG_ON iff CONFIG_EROFS_FS_DEBUG is on */
476 DBG_BUGON(err < 0 && err != -ENOMEM);
477 return err;
478 }
479
480