1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2018 HUAWEI, Inc.
4 * http://www.huawei.com/
5 * Created by Gao Xiang <gaoxiang25@huawei.com>
6 */
7 #include "zdata.h"
8 #include "compress.h"
9 #include <linux/prefetch.h>
10
11 #include <trace/events/erofs.h>
12
13 /*
14 * a compressed_pages[] placeholder in order to avoid
15 * being filled with file pages for in-place decompression.
16 */
17 #define PAGE_UNALLOCATED ((void *)0x5F0E4B1D)
18
19 /* how to allocate cached pages for a pcluster */
20 enum z_erofs_cache_alloctype {
21 DONTALLOC, /* don't allocate any cached pages */
22 DELAYEDALLOC, /* delayed allocation (at the time of submitting io) */
23 };
24
25 /*
26 * tagged pointer with 1-bit tag for all compressed pages
27 * tag 0 - the page is just found with an extra page reference
28 */
29 typedef tagptr1_t compressed_page_t;
30
31 #define tag_compressed_page_justfound(page) \
32 tagptr_fold(compressed_page_t, page, 1)
33
34 static struct workqueue_struct *z_erofs_workqueue __read_mostly;
35 static struct kmem_cache *pcluster_cachep __read_mostly;
36
z_erofs_exit_zip_subsystem(void)37 void z_erofs_exit_zip_subsystem(void)
38 {
39 destroy_workqueue(z_erofs_workqueue);
40 kmem_cache_destroy(pcluster_cachep);
41 }
42
z_erofs_init_workqueue(void)43 static inline int z_erofs_init_workqueue(void)
44 {
45 const unsigned int onlinecpus = num_possible_cpus();
46 const unsigned int flags = WQ_UNBOUND | WQ_HIGHPRI | WQ_CPU_INTENSIVE;
47
48 /*
49 * no need to spawn too many threads, limiting threads could minimum
50 * scheduling overhead, perhaps per-CPU threads should be better?
51 */
52 z_erofs_workqueue = alloc_workqueue("erofs_unzipd", flags,
53 onlinecpus + onlinecpus / 4);
54 return z_erofs_workqueue ? 0 : -ENOMEM;
55 }
56
z_erofs_pcluster_init_once(void * ptr)57 static void z_erofs_pcluster_init_once(void *ptr)
58 {
59 struct z_erofs_pcluster *pcl = ptr;
60 struct z_erofs_collection *cl = z_erofs_primarycollection(pcl);
61 unsigned int i;
62
63 mutex_init(&cl->lock);
64 cl->nr_pages = 0;
65 cl->vcnt = 0;
66 for (i = 0; i < Z_EROFS_CLUSTER_MAX_PAGES; ++i)
67 pcl->compressed_pages[i] = NULL;
68 }
69
z_erofs_pcluster_init_always(struct z_erofs_pcluster * pcl)70 static void z_erofs_pcluster_init_always(struct z_erofs_pcluster *pcl)
71 {
72 struct z_erofs_collection *cl = z_erofs_primarycollection(pcl);
73
74 atomic_set(&pcl->obj.refcount, 1);
75
76 DBG_BUGON(cl->nr_pages);
77 DBG_BUGON(cl->vcnt);
78 }
79
z_erofs_init_zip_subsystem(void)80 int __init z_erofs_init_zip_subsystem(void)
81 {
82 pcluster_cachep = kmem_cache_create("erofs_compress",
83 Z_EROFS_WORKGROUP_SIZE, 0,
84 SLAB_RECLAIM_ACCOUNT,
85 z_erofs_pcluster_init_once);
86 if (pcluster_cachep) {
87 if (!z_erofs_init_workqueue())
88 return 0;
89
90 kmem_cache_destroy(pcluster_cachep);
91 }
92 return -ENOMEM;
93 }
94
95 enum z_erofs_collectmode {
96 COLLECT_SECONDARY,
97 COLLECT_PRIMARY,
98 /*
99 * The current collection was the tail of an exist chain, in addition
100 * that the previous processed chained collections are all decided to
101 * be hooked up to it.
102 * A new chain will be created for the remaining collections which are
103 * not processed yet, therefore different from COLLECT_PRIMARY_FOLLOWED,
104 * the next collection cannot reuse the whole page safely in
105 * the following scenario:
106 * ________________________________________________________________
107 * | tail (partial) page | head (partial) page |
108 * | (belongs to the next cl) | (belongs to the current cl) |
109 * |_______PRIMARY_FOLLOWED_______|________PRIMARY_HOOKED___________|
110 */
111 COLLECT_PRIMARY_HOOKED,
112 COLLECT_PRIMARY_FOLLOWED_NOINPLACE,
113 /*
114 * The current collection has been linked with the owned chain, and
115 * could also be linked with the remaining collections, which means
116 * if the processing page is the tail page of the collection, thus
117 * the current collection can safely use the whole page (since
118 * the previous collection is under control) for in-place I/O, as
119 * illustrated below:
120 * ________________________________________________________________
121 * | tail (partial) page | head (partial) page |
122 * | (of the current cl) | (of the previous collection) |
123 * | PRIMARY_FOLLOWED or | |
124 * |_____PRIMARY_HOOKED___|____________PRIMARY_FOLLOWED____________|
125 *
126 * [ (*) the above page can be used as inplace I/O. ]
127 */
128 COLLECT_PRIMARY_FOLLOWED,
129 };
130
131 struct z_erofs_collector {
132 struct z_erofs_pagevec_ctor vector;
133
134 struct z_erofs_pcluster *pcl, *tailpcl;
135 struct z_erofs_collection *cl;
136 struct page **compressedpages;
137 z_erofs_next_pcluster_t owned_head;
138
139 enum z_erofs_collectmode mode;
140 };
141
142 struct z_erofs_decompress_frontend {
143 struct inode *const inode;
144
145 struct z_erofs_collector clt;
146 struct erofs_map_blocks map;
147
148 /* used for applying cache strategy on the fly */
149 bool backmost;
150 erofs_off_t headoffset;
151 };
152
153 #define COLLECTOR_INIT() { \
154 .owned_head = Z_EROFS_PCLUSTER_TAIL, \
155 .mode = COLLECT_PRIMARY_FOLLOWED }
156
157 #define DECOMPRESS_FRONTEND_INIT(__i) { \
158 .inode = __i, .clt = COLLECTOR_INIT(), \
159 .backmost = true, }
160
161 static struct page *z_pagemap_global[Z_EROFS_VMAP_GLOBAL_PAGES];
162 static DEFINE_MUTEX(z_pagemap_global_lock);
163
preload_compressed_pages(struct z_erofs_collector * clt,struct address_space * mc,enum z_erofs_cache_alloctype type,struct list_head * pagepool)164 static void preload_compressed_pages(struct z_erofs_collector *clt,
165 struct address_space *mc,
166 enum z_erofs_cache_alloctype type,
167 struct list_head *pagepool)
168 {
169 const struct z_erofs_pcluster *pcl = clt->pcl;
170 const unsigned int clusterpages = BIT(pcl->clusterbits);
171 struct page **pages = clt->compressedpages;
172 pgoff_t index = pcl->obj.index + (pages - pcl->compressed_pages);
173 bool standalone = true;
174
175 if (clt->mode < COLLECT_PRIMARY_FOLLOWED)
176 return;
177
178 for (; pages < pcl->compressed_pages + clusterpages; ++pages) {
179 struct page *page;
180 compressed_page_t t;
181
182 /* the compressed page was loaded before */
183 if (READ_ONCE(*pages))
184 continue;
185
186 page = find_get_page(mc, index);
187
188 if (page) {
189 t = tag_compressed_page_justfound(page);
190 } else if (type == DELAYEDALLOC) {
191 t = tagptr_init(compressed_page_t, PAGE_UNALLOCATED);
192 } else { /* DONTALLOC */
193 if (standalone)
194 clt->compressedpages = pages;
195 standalone = false;
196 continue;
197 }
198
199 if (!cmpxchg_relaxed(pages, NULL, tagptr_cast_ptr(t)))
200 continue;
201
202 if (page)
203 put_page(page);
204 }
205
206 if (standalone) /* downgrade to PRIMARY_FOLLOWED_NOINPLACE */
207 clt->mode = COLLECT_PRIMARY_FOLLOWED_NOINPLACE;
208 }
209
210 /* called by erofs_shrinker to get rid of all compressed_pages */
erofs_try_to_free_all_cached_pages(struct erofs_sb_info * sbi,struct erofs_workgroup * grp)211 int erofs_try_to_free_all_cached_pages(struct erofs_sb_info *sbi,
212 struct erofs_workgroup *grp)
213 {
214 struct z_erofs_pcluster *const pcl =
215 container_of(grp, struct z_erofs_pcluster, obj);
216 struct address_space *const mapping = MNGD_MAPPING(sbi);
217 const unsigned int clusterpages = BIT(pcl->clusterbits);
218 int i;
219
220 /*
221 * refcount of workgroup is now freezed as 1,
222 * therefore no need to worry about available decompression users.
223 */
224 for (i = 0; i < clusterpages; ++i) {
225 struct page *page = pcl->compressed_pages[i];
226
227 if (!page)
228 continue;
229
230 /* block other users from reclaiming or migrating the page */
231 if (!trylock_page(page))
232 return -EBUSY;
233
234 if (page->mapping != mapping)
235 continue;
236
237 /* barrier is implied in the following 'unlock_page' */
238 WRITE_ONCE(pcl->compressed_pages[i], NULL);
239 set_page_private(page, 0);
240 ClearPagePrivate(page);
241
242 unlock_page(page);
243 put_page(page);
244 }
245 return 0;
246 }
247
erofs_try_to_free_cached_page(struct address_space * mapping,struct page * page)248 int erofs_try_to_free_cached_page(struct address_space *mapping,
249 struct page *page)
250 {
251 struct z_erofs_pcluster *const pcl = (void *)page_private(page);
252 const unsigned int clusterpages = BIT(pcl->clusterbits);
253 int ret = 0; /* 0 - busy */
254
255 if (erofs_workgroup_try_to_freeze(&pcl->obj, 1)) {
256 unsigned int i;
257
258 for (i = 0; i < clusterpages; ++i) {
259 if (pcl->compressed_pages[i] == page) {
260 WRITE_ONCE(pcl->compressed_pages[i], NULL);
261 ret = 1;
262 break;
263 }
264 }
265 erofs_workgroup_unfreeze(&pcl->obj, 1);
266
267 if (ret) {
268 ClearPagePrivate(page);
269 put_page(page);
270 }
271 }
272 return ret;
273 }
274
275 /* page_type must be Z_EROFS_PAGE_TYPE_EXCLUSIVE */
z_erofs_try_inplace_io(struct z_erofs_collector * clt,struct page * page)276 static inline bool z_erofs_try_inplace_io(struct z_erofs_collector *clt,
277 struct page *page)
278 {
279 struct z_erofs_pcluster *const pcl = clt->pcl;
280 const unsigned int clusterpages = BIT(pcl->clusterbits);
281
282 while (clt->compressedpages < pcl->compressed_pages + clusterpages) {
283 if (!cmpxchg(clt->compressedpages++, NULL, page))
284 return true;
285 }
286 return false;
287 }
288
289 /* callers must be with collection lock held */
z_erofs_attach_page(struct z_erofs_collector * clt,struct page * page,enum z_erofs_page_type type)290 static int z_erofs_attach_page(struct z_erofs_collector *clt,
291 struct page *page,
292 enum z_erofs_page_type type)
293 {
294 int ret;
295 bool occupied;
296
297 /* give priority for inplaceio */
298 if (clt->mode >= COLLECT_PRIMARY &&
299 type == Z_EROFS_PAGE_TYPE_EXCLUSIVE &&
300 z_erofs_try_inplace_io(clt, page))
301 return 0;
302
303 ret = z_erofs_pagevec_enqueue(&clt->vector,
304 page, type, &occupied);
305 clt->cl->vcnt += (unsigned int)ret;
306
307 return ret ? 0 : -EAGAIN;
308 }
309
310 static enum z_erofs_collectmode
try_to_claim_pcluster(struct z_erofs_pcluster * pcl,z_erofs_next_pcluster_t * owned_head)311 try_to_claim_pcluster(struct z_erofs_pcluster *pcl,
312 z_erofs_next_pcluster_t *owned_head)
313 {
314 /* let's claim these following types of pclusters */
315 retry:
316 if (pcl->next == Z_EROFS_PCLUSTER_NIL) {
317 /* type 1, nil pcluster */
318 if (cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_NIL,
319 *owned_head) != Z_EROFS_PCLUSTER_NIL)
320 goto retry;
321
322 *owned_head = &pcl->next;
323 /* lucky, I am the followee :) */
324 return COLLECT_PRIMARY_FOLLOWED;
325 } else if (pcl->next == Z_EROFS_PCLUSTER_TAIL) {
326 /*
327 * type 2, link to the end of a existing open chain,
328 * be careful that its submission itself is governed
329 * by the original owned chain.
330 */
331 if (cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_TAIL,
332 *owned_head) != Z_EROFS_PCLUSTER_TAIL)
333 goto retry;
334 *owned_head = Z_EROFS_PCLUSTER_TAIL;
335 return COLLECT_PRIMARY_HOOKED;
336 }
337 return COLLECT_PRIMARY; /* :( better luck next time */
338 }
339
cllookup(struct z_erofs_collector * clt,struct inode * inode,struct erofs_map_blocks * map)340 static struct z_erofs_collection *cllookup(struct z_erofs_collector *clt,
341 struct inode *inode,
342 struct erofs_map_blocks *map)
343 {
344 struct erofs_workgroup *grp;
345 struct z_erofs_pcluster *pcl;
346 struct z_erofs_collection *cl;
347 unsigned int length;
348 bool tag;
349
350 grp = erofs_find_workgroup(inode->i_sb, map->m_pa >> PAGE_SHIFT, &tag);
351 if (!grp)
352 return NULL;
353
354 pcl = container_of(grp, struct z_erofs_pcluster, obj);
355 if (clt->owned_head == &pcl->next || pcl == clt->tailpcl) {
356 DBG_BUGON(1);
357 erofs_workgroup_put(grp);
358 return ERR_PTR(-EFSCORRUPTED);
359 }
360
361 cl = z_erofs_primarycollection(pcl);
362 if (cl->pageofs != (map->m_la & ~PAGE_MASK)) {
363 DBG_BUGON(1);
364 erofs_workgroup_put(grp);
365 return ERR_PTR(-EFSCORRUPTED);
366 }
367
368 length = READ_ONCE(pcl->length);
369 if (length & Z_EROFS_PCLUSTER_FULL_LENGTH) {
370 if ((map->m_llen << Z_EROFS_PCLUSTER_LENGTH_BIT) > length) {
371 DBG_BUGON(1);
372 erofs_workgroup_put(grp);
373 return ERR_PTR(-EFSCORRUPTED);
374 }
375 } else {
376 unsigned int llen = map->m_llen << Z_EROFS_PCLUSTER_LENGTH_BIT;
377
378 if (map->m_flags & EROFS_MAP_FULL_MAPPED)
379 llen |= Z_EROFS_PCLUSTER_FULL_LENGTH;
380
381 while (llen > length &&
382 length != cmpxchg_relaxed(&pcl->length, length, llen)) {
383 cpu_relax();
384 length = READ_ONCE(pcl->length);
385 }
386 }
387 mutex_lock(&cl->lock);
388 /* used to check tail merging loop due to corrupted images */
389 if (clt->owned_head == Z_EROFS_PCLUSTER_TAIL)
390 clt->tailpcl = pcl;
391 clt->mode = try_to_claim_pcluster(pcl, &clt->owned_head);
392 /* clean tailpcl if the current owned_head is Z_EROFS_PCLUSTER_TAIL */
393 if (clt->owned_head == Z_EROFS_PCLUSTER_TAIL)
394 clt->tailpcl = NULL;
395 clt->pcl = pcl;
396 clt->cl = cl;
397 return cl;
398 }
399
clregister(struct z_erofs_collector * clt,struct inode * inode,struct erofs_map_blocks * map)400 static struct z_erofs_collection *clregister(struct z_erofs_collector *clt,
401 struct inode *inode,
402 struct erofs_map_blocks *map)
403 {
404 struct z_erofs_pcluster *pcl;
405 struct z_erofs_collection *cl;
406 int err;
407
408 /* no available workgroup, let's allocate one */
409 pcl = kmem_cache_alloc(pcluster_cachep, GFP_NOFS);
410 if (!pcl)
411 return ERR_PTR(-ENOMEM);
412
413 z_erofs_pcluster_init_always(pcl);
414 pcl->obj.index = map->m_pa >> PAGE_SHIFT;
415
416 pcl->length = (map->m_llen << Z_EROFS_PCLUSTER_LENGTH_BIT) |
417 (map->m_flags & EROFS_MAP_FULL_MAPPED ?
418 Z_EROFS_PCLUSTER_FULL_LENGTH : 0);
419
420 if (map->m_flags & EROFS_MAP_ZIPPED)
421 pcl->algorithmformat = Z_EROFS_COMPRESSION_LZ4;
422 else
423 pcl->algorithmformat = Z_EROFS_COMPRESSION_SHIFTED;
424
425 pcl->clusterbits = EROFS_I(inode)->z_physical_clusterbits[0];
426 pcl->clusterbits -= PAGE_SHIFT;
427
428 /* new pclusters should be claimed as type 1, primary and followed */
429 pcl->next = clt->owned_head;
430 clt->mode = COLLECT_PRIMARY_FOLLOWED;
431
432 cl = z_erofs_primarycollection(pcl);
433 cl->pageofs = map->m_la & ~PAGE_MASK;
434
435 /*
436 * lock all primary followed works before visible to others
437 * and mutex_trylock *never* fails for a new pcluster.
438 */
439 mutex_trylock(&cl->lock);
440
441 err = erofs_register_workgroup(inode->i_sb, &pcl->obj, 0);
442 if (err) {
443 mutex_unlock(&cl->lock);
444 kmem_cache_free(pcluster_cachep, pcl);
445 return ERR_PTR(-EAGAIN);
446 }
447 /* used to check tail merging loop due to corrupted images */
448 if (clt->owned_head == Z_EROFS_PCLUSTER_TAIL)
449 clt->tailpcl = pcl;
450 clt->owned_head = &pcl->next;
451 clt->pcl = pcl;
452 clt->cl = cl;
453 return cl;
454 }
455
z_erofs_collector_begin(struct z_erofs_collector * clt,struct inode * inode,struct erofs_map_blocks * map)456 static int z_erofs_collector_begin(struct z_erofs_collector *clt,
457 struct inode *inode,
458 struct erofs_map_blocks *map)
459 {
460 struct z_erofs_collection *cl;
461
462 DBG_BUGON(clt->cl);
463
464 /* must be Z_EROFS_PCLUSTER_TAIL or pointed to previous collection */
465 DBG_BUGON(clt->owned_head == Z_EROFS_PCLUSTER_NIL);
466 DBG_BUGON(clt->owned_head == Z_EROFS_PCLUSTER_TAIL_CLOSED);
467
468 if (!PAGE_ALIGNED(map->m_pa)) {
469 DBG_BUGON(1);
470 return -EINVAL;
471 }
472
473 repeat:
474 cl = cllookup(clt, inode, map);
475 if (!cl) {
476 cl = clregister(clt, inode, map);
477
478 if (cl == ERR_PTR(-EAGAIN))
479 goto repeat;
480 }
481
482 if (IS_ERR(cl))
483 return PTR_ERR(cl);
484
485 z_erofs_pagevec_ctor_init(&clt->vector, Z_EROFS_NR_INLINE_PAGEVECS,
486 cl->pagevec, cl->vcnt);
487
488 clt->compressedpages = clt->pcl->compressed_pages;
489 if (clt->mode <= COLLECT_PRIMARY) /* cannot do in-place I/O */
490 clt->compressedpages += Z_EROFS_CLUSTER_MAX_PAGES;
491 return 0;
492 }
493
494 /*
495 * keep in mind that no referenced pclusters will be freed
496 * only after a RCU grace period.
497 */
z_erofs_rcu_callback(struct rcu_head * head)498 static void z_erofs_rcu_callback(struct rcu_head *head)
499 {
500 struct z_erofs_collection *const cl =
501 container_of(head, struct z_erofs_collection, rcu);
502
503 kmem_cache_free(pcluster_cachep,
504 container_of(cl, struct z_erofs_pcluster,
505 primary_collection));
506 }
507
erofs_workgroup_free_rcu(struct erofs_workgroup * grp)508 void erofs_workgroup_free_rcu(struct erofs_workgroup *grp)
509 {
510 struct z_erofs_pcluster *const pcl =
511 container_of(grp, struct z_erofs_pcluster, obj);
512 struct z_erofs_collection *const cl = z_erofs_primarycollection(pcl);
513
514 call_rcu(&cl->rcu, z_erofs_rcu_callback);
515 }
516
z_erofs_collection_put(struct z_erofs_collection * cl)517 static void z_erofs_collection_put(struct z_erofs_collection *cl)
518 {
519 struct z_erofs_pcluster *const pcl =
520 container_of(cl, struct z_erofs_pcluster, primary_collection);
521
522 erofs_workgroup_put(&pcl->obj);
523 }
524
z_erofs_collector_end(struct z_erofs_collector * clt)525 static bool z_erofs_collector_end(struct z_erofs_collector *clt)
526 {
527 struct z_erofs_collection *cl = clt->cl;
528
529 if (!cl)
530 return false;
531
532 z_erofs_pagevec_ctor_exit(&clt->vector, false);
533 mutex_unlock(&cl->lock);
534
535 /*
536 * if all pending pages are added, don't hold its reference
537 * any longer if the pcluster isn't hosted by ourselves.
538 */
539 if (clt->mode < COLLECT_PRIMARY_FOLLOWED_NOINPLACE)
540 z_erofs_collection_put(cl);
541
542 clt->cl = NULL;
543 return true;
544 }
545
__stagingpage_alloc(struct list_head * pagepool,gfp_t gfp)546 static inline struct page *__stagingpage_alloc(struct list_head *pagepool,
547 gfp_t gfp)
548 {
549 struct page *page = erofs_allocpage(pagepool, gfp, true);
550
551 page->mapping = Z_EROFS_MAPPING_STAGING;
552 return page;
553 }
554
should_alloc_managed_pages(struct z_erofs_decompress_frontend * fe,unsigned int cachestrategy,erofs_off_t la)555 static bool should_alloc_managed_pages(struct z_erofs_decompress_frontend *fe,
556 unsigned int cachestrategy,
557 erofs_off_t la)
558 {
559 if (cachestrategy <= EROFS_ZIP_CACHE_DISABLED)
560 return false;
561
562 if (fe->backmost)
563 return true;
564
565 return cachestrategy >= EROFS_ZIP_CACHE_READAROUND &&
566 la < fe->headoffset;
567 }
568
z_erofs_do_read_page(struct z_erofs_decompress_frontend * fe,struct page * page,struct list_head * pagepool)569 static int z_erofs_do_read_page(struct z_erofs_decompress_frontend *fe,
570 struct page *page,
571 struct list_head *pagepool)
572 {
573 struct inode *const inode = fe->inode;
574 struct erofs_sb_info *const sbi __maybe_unused = EROFS_I_SB(inode);
575 struct erofs_map_blocks *const map = &fe->map;
576 struct z_erofs_collector *const clt = &fe->clt;
577 const loff_t offset = page_offset(page);
578 bool tight = true;
579
580 enum z_erofs_cache_alloctype cache_strategy;
581 enum z_erofs_page_type page_type;
582 unsigned int cur, end, spiltted, index;
583 int err = 0;
584
585 /* register locked file pages as online pages in pack */
586 z_erofs_onlinepage_init(page);
587
588 spiltted = 0;
589 end = PAGE_SIZE;
590 repeat:
591 cur = end - 1;
592
593 /* lucky, within the range of the current map_blocks */
594 if (offset + cur >= map->m_la &&
595 offset + cur < map->m_la + map->m_llen) {
596 /* didn't get a valid collection previously (very rare) */
597 if (!clt->cl)
598 goto restart_now;
599 goto hitted;
600 }
601
602 /* go ahead the next map_blocks */
603 erofs_dbg("%s: [out-of-range] pos %llu", __func__, offset + cur);
604
605 if (z_erofs_collector_end(clt))
606 fe->backmost = false;
607
608 map->m_la = offset + cur;
609 map->m_llen = 0;
610 err = z_erofs_map_blocks_iter(inode, map, 0);
611 if (err)
612 goto err_out;
613
614 restart_now:
615 if (!(map->m_flags & EROFS_MAP_MAPPED))
616 goto hitted;
617
618 err = z_erofs_collector_begin(clt, inode, map);
619 if (err)
620 goto err_out;
621
622 /* preload all compressed pages (maybe downgrade role if necessary) */
623 if (should_alloc_managed_pages(fe, sbi->cache_strategy, map->m_la))
624 cache_strategy = DELAYEDALLOC;
625 else
626 cache_strategy = DONTALLOC;
627
628 preload_compressed_pages(clt, MNGD_MAPPING(sbi),
629 cache_strategy, pagepool);
630
631 hitted:
632 /*
633 * Ensure the current partial page belongs to this submit chain rather
634 * than other concurrent submit chains or the noio(bypass) chain since
635 * those chains are handled asynchronously thus the page cannot be used
636 * for inplace I/O or pagevec (should be processed in strict order.)
637 */
638 tight &= (clt->mode >= COLLECT_PRIMARY_HOOKED &&
639 clt->mode != COLLECT_PRIMARY_FOLLOWED_NOINPLACE);
640
641 cur = end - min_t(unsigned int, offset + end - map->m_la, end);
642 if (!(map->m_flags & EROFS_MAP_MAPPED)) {
643 zero_user_segment(page, cur, end);
644 goto next_part;
645 }
646
647 /* let's derive page type */
648 page_type = cur ? Z_EROFS_VLE_PAGE_TYPE_HEAD :
649 (!spiltted ? Z_EROFS_PAGE_TYPE_EXCLUSIVE :
650 (tight ? Z_EROFS_PAGE_TYPE_EXCLUSIVE :
651 Z_EROFS_VLE_PAGE_TYPE_TAIL_SHARED));
652
653 if (cur)
654 tight &= (clt->mode >= COLLECT_PRIMARY_FOLLOWED);
655
656 retry:
657 err = z_erofs_attach_page(clt, page, page_type);
658 /* should allocate an additional staging page for pagevec */
659 if (err == -EAGAIN) {
660 struct page *const newpage =
661 __stagingpage_alloc(pagepool, GFP_NOFS);
662
663 err = z_erofs_attach_page(clt, newpage,
664 Z_EROFS_PAGE_TYPE_EXCLUSIVE);
665 if (!err)
666 goto retry;
667 }
668
669 if (err)
670 goto err_out;
671
672 index = page->index - (map->m_la >> PAGE_SHIFT);
673
674 z_erofs_onlinepage_fixup(page, index, true);
675
676 /* bump up the number of spiltted parts of a page */
677 ++spiltted;
678 /* also update nr_pages */
679 clt->cl->nr_pages = max_t(pgoff_t, clt->cl->nr_pages, index + 1);
680 next_part:
681 /* can be used for verification */
682 map->m_llen = offset + cur - map->m_la;
683
684 end = cur;
685 if (end > 0)
686 goto repeat;
687
688 out:
689 z_erofs_onlinepage_endio(page);
690
691 erofs_dbg("%s, finish page: %pK spiltted: %u map->m_llen %llu",
692 __func__, page, spiltted, map->m_llen);
693 return err;
694
695 /* if some error occurred while processing this page */
696 err_out:
697 SetPageError(page);
698 goto out;
699 }
700
z_erofs_vle_unzip_kickoff(void * ptr,int bios)701 static void z_erofs_vle_unzip_kickoff(void *ptr, int bios)
702 {
703 tagptr1_t t = tagptr_init(tagptr1_t, ptr);
704 struct z_erofs_unzip_io *io = tagptr_unfold_ptr(t);
705 bool background = tagptr_unfold_tags(t);
706
707 if (!background) {
708 unsigned long flags;
709
710 spin_lock_irqsave(&io->u.wait.lock, flags);
711 if (!atomic_add_return(bios, &io->pending_bios))
712 wake_up_locked(&io->u.wait);
713 spin_unlock_irqrestore(&io->u.wait.lock, flags);
714 return;
715 }
716
717 if (!atomic_add_return(bios, &io->pending_bios))
718 queue_work(z_erofs_workqueue, &io->u.work);
719 }
720
z_erofs_vle_read_endio(struct bio * bio)721 static inline void z_erofs_vle_read_endio(struct bio *bio)
722 {
723 struct erofs_sb_info *sbi = NULL;
724 blk_status_t err = bio->bi_status;
725 struct bio_vec *bvec;
726 struct bvec_iter_all iter_all;
727
728 bio_for_each_segment_all(bvec, bio, iter_all) {
729 struct page *page = bvec->bv_page;
730 bool cachemngd = false;
731
732 DBG_BUGON(PageUptodate(page));
733 DBG_BUGON(!page->mapping);
734
735 if (!sbi && !z_erofs_page_is_staging(page))
736 sbi = EROFS_SB(page->mapping->host->i_sb);
737
738 /* sbi should already be gotten if the page is managed */
739 if (sbi)
740 cachemngd = erofs_page_is_managed(sbi, page);
741
742 if (err)
743 SetPageError(page);
744 else if (cachemngd)
745 SetPageUptodate(page);
746
747 if (cachemngd)
748 unlock_page(page);
749 }
750
751 z_erofs_vle_unzip_kickoff(bio->bi_private, -1);
752 bio_put(bio);
753 }
754
z_erofs_decompress_pcluster(struct super_block * sb,struct z_erofs_pcluster * pcl,struct list_head * pagepool)755 static int z_erofs_decompress_pcluster(struct super_block *sb,
756 struct z_erofs_pcluster *pcl,
757 struct list_head *pagepool)
758 {
759 struct erofs_sb_info *const sbi = EROFS_SB(sb);
760 const unsigned int clusterpages = BIT(pcl->clusterbits);
761 struct z_erofs_pagevec_ctor ctor;
762 unsigned int i, outputsize, llen, nr_pages;
763 struct page *pages_onstack[Z_EROFS_VMAP_ONSTACK_PAGES];
764 struct page **pages, **compressed_pages, *page;
765
766 enum z_erofs_page_type page_type;
767 bool overlapped, partial;
768 struct z_erofs_collection *cl;
769 int err;
770
771 might_sleep();
772 cl = z_erofs_primarycollection(pcl);
773 DBG_BUGON(!READ_ONCE(cl->nr_pages));
774
775 mutex_lock(&cl->lock);
776 nr_pages = cl->nr_pages;
777
778 if (nr_pages <= Z_EROFS_VMAP_ONSTACK_PAGES) {
779 pages = pages_onstack;
780 } else if (nr_pages <= Z_EROFS_VMAP_GLOBAL_PAGES &&
781 mutex_trylock(&z_pagemap_global_lock)) {
782 pages = z_pagemap_global;
783 } else {
784 gfp_t gfp_flags = GFP_KERNEL;
785
786 if (nr_pages > Z_EROFS_VMAP_GLOBAL_PAGES)
787 gfp_flags |= __GFP_NOFAIL;
788
789 pages = kvmalloc_array(nr_pages, sizeof(struct page *),
790 gfp_flags);
791
792 /* fallback to global pagemap for the lowmem scenario */
793 if (!pages) {
794 mutex_lock(&z_pagemap_global_lock);
795 pages = z_pagemap_global;
796 }
797 }
798
799 for (i = 0; i < nr_pages; ++i)
800 pages[i] = NULL;
801
802 err = 0;
803 z_erofs_pagevec_ctor_init(&ctor, Z_EROFS_NR_INLINE_PAGEVECS,
804 cl->pagevec, 0);
805
806 for (i = 0; i < cl->vcnt; ++i) {
807 unsigned int pagenr;
808
809 page = z_erofs_pagevec_dequeue(&ctor, &page_type);
810
811 /* all pages in pagevec ought to be valid */
812 DBG_BUGON(!page);
813 DBG_BUGON(!page->mapping);
814
815 if (z_erofs_put_stagingpage(pagepool, page))
816 continue;
817
818 if (page_type == Z_EROFS_VLE_PAGE_TYPE_HEAD)
819 pagenr = 0;
820 else
821 pagenr = z_erofs_onlinepage_index(page);
822
823 DBG_BUGON(pagenr >= nr_pages);
824
825 /*
826 * currently EROFS doesn't support multiref(dedup),
827 * so here erroring out one multiref page.
828 */
829 if (pages[pagenr]) {
830 DBG_BUGON(1);
831 SetPageError(pages[pagenr]);
832 z_erofs_onlinepage_endio(pages[pagenr]);
833 err = -EFSCORRUPTED;
834 }
835 pages[pagenr] = page;
836 }
837 z_erofs_pagevec_ctor_exit(&ctor, true);
838
839 overlapped = false;
840 compressed_pages = pcl->compressed_pages;
841
842 for (i = 0; i < clusterpages; ++i) {
843 unsigned int pagenr;
844
845 page = compressed_pages[i];
846
847 /* all compressed pages ought to be valid */
848 DBG_BUGON(!page);
849 DBG_BUGON(!page->mapping);
850
851 if (!z_erofs_page_is_staging(page)) {
852 if (erofs_page_is_managed(sbi, page)) {
853 if (!PageUptodate(page))
854 err = -EIO;
855 continue;
856 }
857
858 /*
859 * only if non-head page can be selected
860 * for inplace decompression
861 */
862 pagenr = z_erofs_onlinepage_index(page);
863
864 DBG_BUGON(pagenr >= nr_pages);
865 if (pages[pagenr]) {
866 DBG_BUGON(1);
867 SetPageError(pages[pagenr]);
868 z_erofs_onlinepage_endio(pages[pagenr]);
869 err = -EFSCORRUPTED;
870 }
871 pages[pagenr] = page;
872
873 overlapped = true;
874 }
875
876 /* PG_error needs checking for inplaced and staging pages */
877 if (PageError(page)) {
878 DBG_BUGON(PageUptodate(page));
879 err = -EIO;
880 }
881 }
882
883 if (err)
884 goto out;
885
886 llen = pcl->length >> Z_EROFS_PCLUSTER_LENGTH_BIT;
887 if (nr_pages << PAGE_SHIFT >= cl->pageofs + llen) {
888 outputsize = llen;
889 partial = !(pcl->length & Z_EROFS_PCLUSTER_FULL_LENGTH);
890 } else {
891 outputsize = (nr_pages << PAGE_SHIFT) - cl->pageofs;
892 partial = true;
893 }
894
895 err = z_erofs_decompress(&(struct z_erofs_decompress_req) {
896 .sb = sb,
897 .in = compressed_pages,
898 .out = pages,
899 .pageofs_out = cl->pageofs,
900 .inputsize = PAGE_SIZE,
901 .outputsize = outputsize,
902 .alg = pcl->algorithmformat,
903 .inplace_io = overlapped,
904 .partial_decoding = partial
905 }, pagepool);
906
907 out:
908 /* must handle all compressed pages before endding pages */
909 for (i = 0; i < clusterpages; ++i) {
910 page = compressed_pages[i];
911
912 if (erofs_page_is_managed(sbi, page))
913 continue;
914
915 /* recycle all individual staging pages */
916 (void)z_erofs_put_stagingpage(pagepool, page);
917
918 WRITE_ONCE(compressed_pages[i], NULL);
919 }
920
921 for (i = 0; i < nr_pages; ++i) {
922 page = pages[i];
923 if (!page)
924 continue;
925
926 DBG_BUGON(!page->mapping);
927
928 /* recycle all individual staging pages */
929 if (z_erofs_put_stagingpage(pagepool, page))
930 continue;
931
932 if (err < 0)
933 SetPageError(page);
934
935 z_erofs_onlinepage_endio(page);
936 }
937
938 if (pages == z_pagemap_global)
939 mutex_unlock(&z_pagemap_global_lock);
940 else if (pages != pages_onstack)
941 kvfree(pages);
942
943 cl->nr_pages = 0;
944 cl->vcnt = 0;
945
946 /* all cl locks MUST be taken before the following line */
947 WRITE_ONCE(pcl->next, Z_EROFS_PCLUSTER_NIL);
948
949 /* all cl locks SHOULD be released right now */
950 mutex_unlock(&cl->lock);
951
952 z_erofs_collection_put(cl);
953 return err;
954 }
955
z_erofs_vle_unzip_all(struct super_block * sb,struct z_erofs_unzip_io * io,struct list_head * pagepool)956 static void z_erofs_vle_unzip_all(struct super_block *sb,
957 struct z_erofs_unzip_io *io,
958 struct list_head *pagepool)
959 {
960 z_erofs_next_pcluster_t owned = io->head;
961
962 while (owned != Z_EROFS_PCLUSTER_TAIL_CLOSED) {
963 struct z_erofs_pcluster *pcl;
964
965 /* no possible that 'owned' equals Z_EROFS_WORK_TPTR_TAIL */
966 DBG_BUGON(owned == Z_EROFS_PCLUSTER_TAIL);
967
968 /* no possible that 'owned' equals NULL */
969 DBG_BUGON(owned == Z_EROFS_PCLUSTER_NIL);
970
971 pcl = container_of(owned, struct z_erofs_pcluster, next);
972 owned = READ_ONCE(pcl->next);
973
974 z_erofs_decompress_pcluster(sb, pcl, pagepool);
975 }
976 }
977
z_erofs_vle_unzip_wq(struct work_struct * work)978 static void z_erofs_vle_unzip_wq(struct work_struct *work)
979 {
980 struct z_erofs_unzip_io_sb *iosb =
981 container_of(work, struct z_erofs_unzip_io_sb, io.u.work);
982 LIST_HEAD(pagepool);
983
984 DBG_BUGON(iosb->io.head == Z_EROFS_PCLUSTER_TAIL_CLOSED);
985 z_erofs_vle_unzip_all(iosb->sb, &iosb->io, &pagepool);
986
987 put_pages_list(&pagepool);
988 kvfree(iosb);
989 }
990
pickup_page_for_submission(struct z_erofs_pcluster * pcl,unsigned int nr,struct list_head * pagepool,struct address_space * mc,gfp_t gfp)991 static struct page *pickup_page_for_submission(struct z_erofs_pcluster *pcl,
992 unsigned int nr,
993 struct list_head *pagepool,
994 struct address_space *mc,
995 gfp_t gfp)
996 {
997 /* determined at compile time to avoid too many #ifdefs */
998 const bool nocache = __builtin_constant_p(mc) ? !mc : false;
999 const pgoff_t index = pcl->obj.index;
1000 bool tocache = false;
1001
1002 struct address_space *mapping;
1003 struct page *oldpage, *page;
1004
1005 compressed_page_t t;
1006 int justfound;
1007
1008 repeat:
1009 page = READ_ONCE(pcl->compressed_pages[nr]);
1010 oldpage = page;
1011
1012 if (!page)
1013 goto out_allocpage;
1014
1015 /*
1016 * the cached page has not been allocated and
1017 * an placeholder is out there, prepare it now.
1018 */
1019 if (!nocache && page == PAGE_UNALLOCATED) {
1020 tocache = true;
1021 goto out_allocpage;
1022 }
1023
1024 /* process the target tagged pointer */
1025 t = tagptr_init(compressed_page_t, page);
1026 justfound = tagptr_unfold_tags(t);
1027 page = tagptr_unfold_ptr(t);
1028
1029 mapping = READ_ONCE(page->mapping);
1030
1031 /*
1032 * if managed cache is disabled, it's no way to
1033 * get such a cached-like page.
1034 */
1035 if (nocache) {
1036 /* if managed cache is disabled, it is impossible `justfound' */
1037 DBG_BUGON(justfound);
1038
1039 /* and it should be locked, not uptodate, and not truncated */
1040 DBG_BUGON(!PageLocked(page));
1041 DBG_BUGON(PageUptodate(page));
1042 DBG_BUGON(!mapping);
1043 goto out;
1044 }
1045
1046 /*
1047 * unmanaged (file) pages are all locked solidly,
1048 * therefore it is impossible for `mapping' to be NULL.
1049 */
1050 if (mapping && mapping != mc)
1051 /* ought to be unmanaged pages */
1052 goto out;
1053
1054 lock_page(page);
1055
1056 /* only true if page reclaim goes wrong, should never happen */
1057 DBG_BUGON(justfound && PagePrivate(page));
1058
1059 /* the page is still in manage cache */
1060 if (page->mapping == mc) {
1061 WRITE_ONCE(pcl->compressed_pages[nr], page);
1062
1063 ClearPageError(page);
1064 if (!PagePrivate(page)) {
1065 /*
1066 * impossible to be !PagePrivate(page) for
1067 * the current restriction as well if
1068 * the page is already in compressed_pages[].
1069 */
1070 DBG_BUGON(!justfound);
1071
1072 justfound = 0;
1073 set_page_private(page, (unsigned long)pcl);
1074 SetPagePrivate(page);
1075 }
1076
1077 /* no need to submit io if it is already up-to-date */
1078 if (PageUptodate(page)) {
1079 unlock_page(page);
1080 page = NULL;
1081 }
1082 goto out;
1083 }
1084
1085 /*
1086 * the managed page has been truncated, it's unsafe to
1087 * reuse this one, let's allocate a new cache-managed page.
1088 */
1089 DBG_BUGON(page->mapping);
1090 DBG_BUGON(!justfound);
1091
1092 tocache = true;
1093 unlock_page(page);
1094 put_page(page);
1095 out_allocpage:
1096 page = __stagingpage_alloc(pagepool, gfp);
1097 if (oldpage != cmpxchg(&pcl->compressed_pages[nr], oldpage, page)) {
1098 list_add(&page->lru, pagepool);
1099 cpu_relax();
1100 goto repeat;
1101 }
1102 if (nocache || !tocache)
1103 goto out;
1104 if (add_to_page_cache_lru(page, mc, index + nr, gfp)) {
1105 page->mapping = Z_EROFS_MAPPING_STAGING;
1106 goto out;
1107 }
1108
1109 set_page_private(page, (unsigned long)pcl);
1110 SetPagePrivate(page);
1111 out: /* the only exit (for tracing and debugging) */
1112 return page;
1113 }
1114
jobqueue_init(struct super_block * sb,struct z_erofs_unzip_io * io,bool foreground)1115 static struct z_erofs_unzip_io *jobqueue_init(struct super_block *sb,
1116 struct z_erofs_unzip_io *io,
1117 bool foreground)
1118 {
1119 struct z_erofs_unzip_io_sb *iosb;
1120
1121 if (foreground) {
1122 /* waitqueue available for foreground io */
1123 DBG_BUGON(!io);
1124
1125 init_waitqueue_head(&io->u.wait);
1126 atomic_set(&io->pending_bios, 0);
1127 goto out;
1128 }
1129
1130 iosb = kvzalloc(sizeof(*iosb), GFP_KERNEL | __GFP_NOFAIL);
1131 DBG_BUGON(!iosb);
1132
1133 /* initialize fields in the allocated descriptor */
1134 io = &iosb->io;
1135 iosb->sb = sb;
1136 INIT_WORK(&io->u.work, z_erofs_vle_unzip_wq);
1137 out:
1138 io->head = Z_EROFS_PCLUSTER_TAIL_CLOSED;
1139 return io;
1140 }
1141
1142 /* define decompression jobqueue types */
1143 enum {
1144 JQ_BYPASS,
1145 JQ_SUBMIT,
1146 NR_JOBQUEUES,
1147 };
1148
jobqueueset_init(struct super_block * sb,z_erofs_next_pcluster_t qtail[],struct z_erofs_unzip_io * q[],struct z_erofs_unzip_io * fgq,bool forcefg)1149 static void *jobqueueset_init(struct super_block *sb,
1150 z_erofs_next_pcluster_t qtail[],
1151 struct z_erofs_unzip_io *q[],
1152 struct z_erofs_unzip_io *fgq,
1153 bool forcefg)
1154 {
1155 /*
1156 * if managed cache is enabled, bypass jobqueue is needed,
1157 * no need to read from device for all pclusters in this queue.
1158 */
1159 q[JQ_BYPASS] = jobqueue_init(sb, fgq + JQ_BYPASS, true);
1160 qtail[JQ_BYPASS] = &q[JQ_BYPASS]->head;
1161
1162 q[JQ_SUBMIT] = jobqueue_init(sb, fgq + JQ_SUBMIT, forcefg);
1163 qtail[JQ_SUBMIT] = &q[JQ_SUBMIT]->head;
1164
1165 return tagptr_cast_ptr(tagptr_fold(tagptr1_t, q[JQ_SUBMIT], !forcefg));
1166 }
1167
move_to_bypass_jobqueue(struct z_erofs_pcluster * pcl,z_erofs_next_pcluster_t qtail[],z_erofs_next_pcluster_t owned_head)1168 static void move_to_bypass_jobqueue(struct z_erofs_pcluster *pcl,
1169 z_erofs_next_pcluster_t qtail[],
1170 z_erofs_next_pcluster_t owned_head)
1171 {
1172 z_erofs_next_pcluster_t *const submit_qtail = qtail[JQ_SUBMIT];
1173 z_erofs_next_pcluster_t *const bypass_qtail = qtail[JQ_BYPASS];
1174
1175 DBG_BUGON(owned_head == Z_EROFS_PCLUSTER_TAIL_CLOSED);
1176 if (owned_head == Z_EROFS_PCLUSTER_TAIL)
1177 owned_head = Z_EROFS_PCLUSTER_TAIL_CLOSED;
1178
1179 WRITE_ONCE(pcl->next, Z_EROFS_PCLUSTER_TAIL_CLOSED);
1180
1181 WRITE_ONCE(*submit_qtail, owned_head);
1182 WRITE_ONCE(*bypass_qtail, &pcl->next);
1183
1184 qtail[JQ_BYPASS] = &pcl->next;
1185 }
1186
postsubmit_is_all_bypassed(struct z_erofs_unzip_io * q[],unsigned int nr_bios,bool force_fg)1187 static bool postsubmit_is_all_bypassed(struct z_erofs_unzip_io *q[],
1188 unsigned int nr_bios,
1189 bool force_fg)
1190 {
1191 /*
1192 * although background is preferred, no one is pending for submission.
1193 * don't issue workqueue for decompression but drop it directly instead.
1194 */
1195 if (force_fg || nr_bios)
1196 return false;
1197
1198 kvfree(container_of(q[JQ_SUBMIT], struct z_erofs_unzip_io_sb, io));
1199 return true;
1200 }
1201
z_erofs_vle_submit_all(struct super_block * sb,z_erofs_next_pcluster_t owned_head,struct list_head * pagepool,struct z_erofs_unzip_io * fgq,bool force_fg)1202 static bool z_erofs_vle_submit_all(struct super_block *sb,
1203 z_erofs_next_pcluster_t owned_head,
1204 struct list_head *pagepool,
1205 struct z_erofs_unzip_io *fgq,
1206 bool force_fg)
1207 {
1208 struct erofs_sb_info *const sbi __maybe_unused = EROFS_SB(sb);
1209 z_erofs_next_pcluster_t qtail[NR_JOBQUEUES];
1210 struct z_erofs_unzip_io *q[NR_JOBQUEUES];
1211 struct bio *bio;
1212 void *bi_private;
1213 /* since bio will be NULL, no need to initialize last_index */
1214 pgoff_t uninitialized_var(last_index);
1215 bool force_submit = false;
1216 unsigned int nr_bios;
1217
1218 if (owned_head == Z_EROFS_PCLUSTER_TAIL)
1219 return false;
1220
1221 force_submit = false;
1222 bio = NULL;
1223 nr_bios = 0;
1224 bi_private = jobqueueset_init(sb, qtail, q, fgq, force_fg);
1225
1226 /* by default, all need io submission */
1227 q[JQ_SUBMIT]->head = owned_head;
1228
1229 do {
1230 struct z_erofs_pcluster *pcl;
1231 unsigned int clusterpages;
1232 pgoff_t first_index;
1233 struct page *page;
1234 unsigned int i = 0, bypass = 0;
1235 int err;
1236
1237 /* no possible 'owned_head' equals the following */
1238 DBG_BUGON(owned_head == Z_EROFS_PCLUSTER_TAIL_CLOSED);
1239 DBG_BUGON(owned_head == Z_EROFS_PCLUSTER_NIL);
1240
1241 pcl = container_of(owned_head, struct z_erofs_pcluster, next);
1242
1243 clusterpages = BIT(pcl->clusterbits);
1244
1245 /* close the main owned chain at first */
1246 owned_head = cmpxchg(&pcl->next, Z_EROFS_PCLUSTER_TAIL,
1247 Z_EROFS_PCLUSTER_TAIL_CLOSED);
1248
1249 first_index = pcl->obj.index;
1250 force_submit |= (first_index != last_index + 1);
1251
1252 repeat:
1253 page = pickup_page_for_submission(pcl, i, pagepool,
1254 MNGD_MAPPING(sbi),
1255 GFP_NOFS);
1256 if (!page) {
1257 force_submit = true;
1258 ++bypass;
1259 goto skippage;
1260 }
1261
1262 if (bio && force_submit) {
1263 submit_bio_retry:
1264 submit_bio(bio);
1265 bio = NULL;
1266 }
1267
1268 if (!bio) {
1269 bio = bio_alloc(GFP_NOIO, BIO_MAX_PAGES);
1270
1271 bio->bi_end_io = z_erofs_vle_read_endio;
1272 bio_set_dev(bio, sb->s_bdev);
1273 bio->bi_iter.bi_sector = (sector_t)(first_index + i) <<
1274 LOG_SECTORS_PER_BLOCK;
1275 bio->bi_private = bi_private;
1276 bio->bi_opf = REQ_OP_READ;
1277
1278 ++nr_bios;
1279 }
1280
1281 err = bio_add_page(bio, page, PAGE_SIZE, 0);
1282 if (err < PAGE_SIZE)
1283 goto submit_bio_retry;
1284
1285 force_submit = false;
1286 last_index = first_index + i;
1287 skippage:
1288 if (++i < clusterpages)
1289 goto repeat;
1290
1291 if (bypass < clusterpages)
1292 qtail[JQ_SUBMIT] = &pcl->next;
1293 else
1294 move_to_bypass_jobqueue(pcl, qtail, owned_head);
1295 } while (owned_head != Z_EROFS_PCLUSTER_TAIL);
1296
1297 if (bio)
1298 submit_bio(bio);
1299
1300 if (postsubmit_is_all_bypassed(q, nr_bios, force_fg))
1301 return true;
1302
1303 z_erofs_vle_unzip_kickoff(bi_private, nr_bios);
1304 return true;
1305 }
1306
z_erofs_submit_and_unzip(struct super_block * sb,struct z_erofs_collector * clt,struct list_head * pagepool,bool force_fg)1307 static void z_erofs_submit_and_unzip(struct super_block *sb,
1308 struct z_erofs_collector *clt,
1309 struct list_head *pagepool,
1310 bool force_fg)
1311 {
1312 struct z_erofs_unzip_io io[NR_JOBQUEUES];
1313
1314 if (!z_erofs_vle_submit_all(sb, clt->owned_head,
1315 pagepool, io, force_fg))
1316 return;
1317
1318 /* decompress no I/O pclusters immediately */
1319 z_erofs_vle_unzip_all(sb, &io[JQ_BYPASS], pagepool);
1320
1321 if (!force_fg)
1322 return;
1323
1324 /* wait until all bios are completed */
1325 wait_event(io[JQ_SUBMIT].u.wait,
1326 !atomic_read(&io[JQ_SUBMIT].pending_bios));
1327
1328 /* let's synchronous decompression */
1329 z_erofs_vle_unzip_all(sb, &io[JQ_SUBMIT], pagepool);
1330 }
1331
z_erofs_vle_normalaccess_readpage(struct file * file,struct page * page)1332 static int z_erofs_vle_normalaccess_readpage(struct file *file,
1333 struct page *page)
1334 {
1335 struct inode *const inode = page->mapping->host;
1336 struct z_erofs_decompress_frontend f = DECOMPRESS_FRONTEND_INIT(inode);
1337 int err;
1338 LIST_HEAD(pagepool);
1339
1340 trace_erofs_readpage(page, false);
1341
1342 f.headoffset = (erofs_off_t)page->index << PAGE_SHIFT;
1343
1344 err = z_erofs_do_read_page(&f, page, &pagepool);
1345 (void)z_erofs_collector_end(&f.clt);
1346
1347 /* if some compressed cluster ready, need submit them anyway */
1348 z_erofs_submit_and_unzip(inode->i_sb, &f.clt, &pagepool, true);
1349
1350 if (err)
1351 erofs_err(inode->i_sb, "failed to read, err [%d]", err);
1352
1353 if (f.map.mpage)
1354 put_page(f.map.mpage);
1355
1356 /* clean up the remaining free pages */
1357 put_pages_list(&pagepool);
1358 return err;
1359 }
1360
should_decompress_synchronously(struct erofs_sb_info * sbi,unsigned int nr)1361 static bool should_decompress_synchronously(struct erofs_sb_info *sbi,
1362 unsigned int nr)
1363 {
1364 return nr <= sbi->max_sync_decompress_pages;
1365 }
1366
z_erofs_vle_normalaccess_readpages(struct file * filp,struct address_space * mapping,struct list_head * pages,unsigned int nr_pages)1367 static int z_erofs_vle_normalaccess_readpages(struct file *filp,
1368 struct address_space *mapping,
1369 struct list_head *pages,
1370 unsigned int nr_pages)
1371 {
1372 struct inode *const inode = mapping->host;
1373 struct erofs_sb_info *const sbi = EROFS_I_SB(inode);
1374
1375 bool sync = should_decompress_synchronously(sbi, nr_pages);
1376 struct z_erofs_decompress_frontend f = DECOMPRESS_FRONTEND_INIT(inode);
1377 gfp_t gfp = mapping_gfp_constraint(mapping, GFP_KERNEL);
1378 struct page *head = NULL;
1379 LIST_HEAD(pagepool);
1380
1381 trace_erofs_readpages(mapping->host, lru_to_page(pages),
1382 nr_pages, false);
1383
1384 f.headoffset = (erofs_off_t)lru_to_page(pages)->index << PAGE_SHIFT;
1385
1386 for (; nr_pages; --nr_pages) {
1387 struct page *page = lru_to_page(pages);
1388
1389 prefetchw(&page->flags);
1390 list_del(&page->lru);
1391
1392 /*
1393 * A pure asynchronous readahead is indicated if
1394 * a PG_readahead marked page is hitted at first.
1395 * Let's also do asynchronous decompression for this case.
1396 */
1397 sync &= !(PageReadahead(page) && !head);
1398
1399 if (add_to_page_cache_lru(page, mapping, page->index, gfp)) {
1400 list_add(&page->lru, &pagepool);
1401 continue;
1402 }
1403
1404 set_page_private(page, (unsigned long)head);
1405 head = page;
1406 }
1407
1408 while (head) {
1409 struct page *page = head;
1410 int err;
1411
1412 /* traversal in reverse order */
1413 head = (void *)page_private(page);
1414
1415 err = z_erofs_do_read_page(&f, page, &pagepool);
1416 if (err)
1417 erofs_err(inode->i_sb,
1418 "readahead error at page %lu @ nid %llu",
1419 page->index, EROFS_I(inode)->nid);
1420 put_page(page);
1421 }
1422
1423 (void)z_erofs_collector_end(&f.clt);
1424
1425 z_erofs_submit_and_unzip(inode->i_sb, &f.clt, &pagepool, sync);
1426
1427 if (f.map.mpage)
1428 put_page(f.map.mpage);
1429
1430 /* clean up the remaining free pages */
1431 put_pages_list(&pagepool);
1432 return 0;
1433 }
1434
1435 const struct address_space_operations z_erofs_vle_normalaccess_aops = {
1436 .readpage = z_erofs_vle_normalaccess_readpage,
1437 .readpages = z_erofs_vle_normalaccess_readpages,
1438 };
1439
1440