1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) Gao Xiang <xiang@kernel.org>
4 *
5 * For low-latency decompression algorithms (e.g. lz4), reserve consecutive
6 * per-CPU virtual memory (in pages) in advance to store such inplace I/O
7 * data if inplace decompression is failed (due to unmet inplace margin for
8 * example).
9 */
10 #include "internal.h"
11
12 struct erofs_pcpubuf {
13 raw_spinlock_t lock;
14 void *ptr;
15 struct page **pages;
16 unsigned int nrpages;
17 };
18
19 static DEFINE_PER_CPU(struct erofs_pcpubuf, erofs_pcb);
20
erofs_get_pcpubuf(unsigned int requiredpages)21 void *erofs_get_pcpubuf(unsigned int requiredpages)
22 __acquires(pcb->lock)
23 {
24 struct erofs_pcpubuf *pcb = &get_cpu_var(erofs_pcb);
25
26 raw_spin_lock(&pcb->lock);
27 /* check if the per-CPU buffer is too small */
28 if (requiredpages > pcb->nrpages) {
29 raw_spin_unlock(&pcb->lock);
30 put_cpu_var(erofs_pcb);
31 /* (for sparse checker) pretend pcb->lock is still taken */
32 __acquire(pcb->lock);
33 return NULL;
34 }
35 return pcb->ptr;
36 }
37
erofs_put_pcpubuf(void * ptr)38 void erofs_put_pcpubuf(void *ptr) __releases(pcb->lock)
39 {
40 struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, smp_processor_id());
41
42 DBG_BUGON(pcb->ptr != ptr);
43 raw_spin_unlock(&pcb->lock);
44 put_cpu_var(erofs_pcb);
45 }
46
47 /* the next step: support per-CPU page buffers hotplug */
erofs_pcpubuf_growsize(unsigned int nrpages)48 int erofs_pcpubuf_growsize(unsigned int nrpages)
49 {
50 static DEFINE_MUTEX(pcb_resize_mutex);
51 static unsigned int pcb_nrpages;
52 struct page *pagepool = NULL;
53 int delta, cpu, ret, i;
54
55 mutex_lock(&pcb_resize_mutex);
56 delta = nrpages - pcb_nrpages;
57 ret = 0;
58 /* avoid shrinking pcpubuf, since no idea how many fses rely on */
59 if (delta <= 0)
60 goto out;
61
62 for_each_possible_cpu(cpu) {
63 struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu);
64 struct page **pages, **oldpages;
65 void *ptr, *old_ptr;
66
67 pages = kmalloc_array(nrpages, sizeof(*pages), GFP_KERNEL);
68 if (!pages) {
69 ret = -ENOMEM;
70 break;
71 }
72
73 for (i = 0; i < nrpages; ++i) {
74 pages[i] = erofs_allocpage(&pagepool, GFP_KERNEL);
75 if (!pages[i]) {
76 ret = -ENOMEM;
77 oldpages = pages;
78 goto free_pagearray;
79 }
80 }
81 ptr = vmap(pages, nrpages, VM_MAP, PAGE_KERNEL);
82 if (!ptr) {
83 ret = -ENOMEM;
84 oldpages = pages;
85 goto free_pagearray;
86 }
87 raw_spin_lock(&pcb->lock);
88 old_ptr = pcb->ptr;
89 pcb->ptr = ptr;
90 oldpages = pcb->pages;
91 pcb->pages = pages;
92 i = pcb->nrpages;
93 pcb->nrpages = nrpages;
94 raw_spin_unlock(&pcb->lock);
95
96 if (!oldpages) {
97 DBG_BUGON(old_ptr);
98 continue;
99 }
100
101 if (old_ptr)
102 vunmap(old_ptr);
103 free_pagearray:
104 while (i)
105 erofs_pagepool_add(&pagepool, oldpages[--i]);
106 kfree(oldpages);
107 if (ret)
108 break;
109 }
110 pcb_nrpages = nrpages;
111 erofs_release_pages(&pagepool);
112 out:
113 mutex_unlock(&pcb_resize_mutex);
114 return ret;
115 }
116
erofs_pcpubuf_init(void)117 void erofs_pcpubuf_init(void)
118 {
119 int cpu;
120
121 for_each_possible_cpu(cpu) {
122 struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu);
123
124 raw_spin_lock_init(&pcb->lock);
125 }
126 }
127
erofs_pcpubuf_exit(void)128 void erofs_pcpubuf_exit(void)
129 {
130 int cpu, i;
131
132 for_each_possible_cpu(cpu) {
133 struct erofs_pcpubuf *pcb = &per_cpu(erofs_pcb, cpu);
134
135 if (pcb->ptr) {
136 vunmap(pcb->ptr);
137 pcb->ptr = NULL;
138 }
139 if (!pcb->pages)
140 continue;
141
142 for (i = 0; i < pcb->nrpages; ++i)
143 if (pcb->pages[i])
144 put_page(pcb->pages[i]);
145 kfree(pcb->pages);
146 pcb->pages = NULL;
147 }
148 }
149