1 // SPDX-License-Identifier: GPL-2.0-or-later
2 #include <linux/xz.h>
3 #include <linux/module.h>
4 #include "compress.h"
5
6 struct z_erofs_lzma {
7 struct z_erofs_lzma *next;
8 struct xz_dec_microlzma *state;
9 struct xz_buf buf;
10 u8 bounce[PAGE_SIZE];
11 };
12
13 /* considering the LZMA performance, no need to use a lockless list for now */
14 static DEFINE_SPINLOCK(z_erofs_lzma_lock);
15 static unsigned int z_erofs_lzma_max_dictsize;
16 static unsigned int z_erofs_lzma_nstrms, z_erofs_lzma_avail_strms;
17 static struct z_erofs_lzma *z_erofs_lzma_head;
18 static DECLARE_WAIT_QUEUE_HEAD(z_erofs_lzma_wq);
19
20 module_param_named(lzma_streams, z_erofs_lzma_nstrms, uint, 0444);
21
z_erofs_lzma_exit(void)22 void z_erofs_lzma_exit(void)
23 {
24 /* there should be no running fs instance */
25 while (z_erofs_lzma_avail_strms) {
26 struct z_erofs_lzma *strm;
27
28 spin_lock(&z_erofs_lzma_lock);
29 strm = z_erofs_lzma_head;
30 if (!strm) {
31 spin_unlock(&z_erofs_lzma_lock);
32 DBG_BUGON(1);
33 return;
34 }
35 z_erofs_lzma_head = NULL;
36 spin_unlock(&z_erofs_lzma_lock);
37
38 while (strm) {
39 struct z_erofs_lzma *n = strm->next;
40
41 if (strm->state)
42 xz_dec_microlzma_end(strm->state);
43 kfree(strm);
44 --z_erofs_lzma_avail_strms;
45 strm = n;
46 }
47 }
48 }
49
z_erofs_lzma_init(void)50 int z_erofs_lzma_init(void)
51 {
52 unsigned int i;
53
54 /* by default, use # of possible CPUs instead */
55 if (!z_erofs_lzma_nstrms)
56 z_erofs_lzma_nstrms = num_possible_cpus();
57
58 for (i = 0; i < z_erofs_lzma_nstrms; ++i) {
59 struct z_erofs_lzma *strm = kzalloc(sizeof(*strm), GFP_KERNEL);
60
61 if (!strm) {
62 z_erofs_lzma_exit();
63 return -ENOMEM;
64 }
65 spin_lock(&z_erofs_lzma_lock);
66 strm->next = z_erofs_lzma_head;
67 z_erofs_lzma_head = strm;
68 spin_unlock(&z_erofs_lzma_lock);
69 ++z_erofs_lzma_avail_strms;
70 }
71 return 0;
72 }
73
z_erofs_load_lzma_config(struct super_block * sb,struct erofs_super_block * dsb,struct z_erofs_lzma_cfgs * lzma,int size)74 int z_erofs_load_lzma_config(struct super_block *sb,
75 struct erofs_super_block *dsb,
76 struct z_erofs_lzma_cfgs *lzma, int size)
77 {
78 static DEFINE_MUTEX(lzma_resize_mutex);
79 unsigned int dict_size, i;
80 struct z_erofs_lzma *strm, *head = NULL;
81 int err;
82
83 if (!lzma || size < sizeof(struct z_erofs_lzma_cfgs)) {
84 erofs_err(sb, "invalid lzma cfgs, size=%u", size);
85 return -EINVAL;
86 }
87 if (lzma->format) {
88 erofs_err(sb, "unidentified lzma format %x, please check kernel version",
89 le16_to_cpu(lzma->format));
90 return -EINVAL;
91 }
92 dict_size = le32_to_cpu(lzma->dict_size);
93 if (dict_size > Z_EROFS_LZMA_MAX_DICT_SIZE || dict_size < 4096) {
94 erofs_err(sb, "unsupported lzma dictionary size %u",
95 dict_size);
96 return -EINVAL;
97 }
98
99 erofs_info(sb, "EXPERIMENTAL MicroLZMA in use. Use at your own risk!");
100
101 /* in case 2 z_erofs_load_lzma_config() race to avoid deadlock */
102 mutex_lock(&lzma_resize_mutex);
103
104 if (z_erofs_lzma_max_dictsize >= dict_size) {
105 mutex_unlock(&lzma_resize_mutex);
106 return 0;
107 }
108
109 /* 1. collect/isolate all streams for the following check */
110 for (i = 0; i < z_erofs_lzma_avail_strms; ++i) {
111 struct z_erofs_lzma *last;
112
113 again:
114 spin_lock(&z_erofs_lzma_lock);
115 strm = z_erofs_lzma_head;
116 if (!strm) {
117 spin_unlock(&z_erofs_lzma_lock);
118 wait_event(z_erofs_lzma_wq,
119 READ_ONCE(z_erofs_lzma_head));
120 goto again;
121 }
122 z_erofs_lzma_head = NULL;
123 spin_unlock(&z_erofs_lzma_lock);
124
125 for (last = strm; last->next; last = last->next)
126 ++i;
127 last->next = head;
128 head = strm;
129 }
130
131 err = 0;
132 /* 2. walk each isolated stream and grow max dict_size if needed */
133 for (strm = head; strm; strm = strm->next) {
134 if (strm->state)
135 xz_dec_microlzma_end(strm->state);
136 strm->state = xz_dec_microlzma_alloc(XZ_PREALLOC, dict_size);
137 if (!strm->state)
138 err = -ENOMEM;
139 }
140
141 /* 3. push back all to the global list and update max dict_size */
142 spin_lock(&z_erofs_lzma_lock);
143 DBG_BUGON(z_erofs_lzma_head);
144 z_erofs_lzma_head = head;
145 spin_unlock(&z_erofs_lzma_lock);
146
147 z_erofs_lzma_max_dictsize = dict_size;
148 mutex_unlock(&lzma_resize_mutex);
149 return err;
150 }
151
z_erofs_lzma_decompress(struct z_erofs_decompress_req * rq,struct page ** pagepool)152 int z_erofs_lzma_decompress(struct z_erofs_decompress_req *rq,
153 struct page **pagepool)
154 {
155 const unsigned int nrpages_out =
156 PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT;
157 const unsigned int nrpages_in =
158 PAGE_ALIGN(rq->inputsize) >> PAGE_SHIFT;
159 unsigned int inputmargin, inlen, outlen, pageofs;
160 struct z_erofs_lzma *strm;
161 u8 *kin;
162 bool bounced = false;
163 int no, ni, j, err = 0;
164
165 /* 1. get the exact LZMA compressed size */
166 kin = kmap(*rq->in);
167 inputmargin = 0;
168 while (!kin[inputmargin & ~PAGE_MASK])
169 if (!(++inputmargin & ~PAGE_MASK))
170 break;
171
172 if (inputmargin >= PAGE_SIZE) {
173 kunmap(*rq->in);
174 return -EFSCORRUPTED;
175 }
176 rq->inputsize -= inputmargin;
177
178 /* 2. get an available lzma context */
179 again:
180 spin_lock(&z_erofs_lzma_lock);
181 strm = z_erofs_lzma_head;
182 if (!strm) {
183 spin_unlock(&z_erofs_lzma_lock);
184 wait_event(z_erofs_lzma_wq, READ_ONCE(z_erofs_lzma_head));
185 goto again;
186 }
187 z_erofs_lzma_head = strm->next;
188 spin_unlock(&z_erofs_lzma_lock);
189
190 /* 3. multi-call decompress */
191 inlen = rq->inputsize;
192 outlen = rq->outputsize;
193 xz_dec_microlzma_reset(strm->state, inlen, outlen,
194 !rq->partial_decoding);
195 pageofs = rq->pageofs_out;
196 strm->buf.in = kin + inputmargin;
197 strm->buf.in_pos = 0;
198 strm->buf.in_size = min_t(u32, inlen, PAGE_SIZE - inputmargin);
199 inlen -= strm->buf.in_size;
200 strm->buf.out = NULL;
201 strm->buf.out_pos = 0;
202 strm->buf.out_size = 0;
203
204 for (ni = 0, no = -1;;) {
205 enum xz_ret xz_err;
206
207 if (strm->buf.out_pos == strm->buf.out_size) {
208 if (strm->buf.out) {
209 kunmap(rq->out[no]);
210 strm->buf.out = NULL;
211 }
212
213 if (++no >= nrpages_out || !outlen) {
214 erofs_err(rq->sb, "decompressed buf out of bound");
215 err = -EFSCORRUPTED;
216 break;
217 }
218 strm->buf.out_pos = 0;
219 strm->buf.out_size = min_t(u32, outlen,
220 PAGE_SIZE - pageofs);
221 outlen -= strm->buf.out_size;
222 if (rq->out[no])
223 strm->buf.out = kmap(rq->out[no]) + pageofs;
224 pageofs = 0;
225 } else if (strm->buf.in_pos == strm->buf.in_size) {
226 kunmap(rq->in[ni]);
227
228 if (++ni >= nrpages_in || !inlen) {
229 erofs_err(rq->sb, "compressed buf out of bound");
230 err = -EFSCORRUPTED;
231 break;
232 }
233 strm->buf.in_pos = 0;
234 strm->buf.in_size = min_t(u32, inlen, PAGE_SIZE);
235 inlen -= strm->buf.in_size;
236 kin = kmap(rq->in[ni]);
237 strm->buf.in = kin;
238 bounced = false;
239 }
240
241 /*
242 * Handle overlapping: Use bounced buffer if the compressed
243 * data is under processing; Otherwise, Use short-lived pages
244 * from the on-stack pagepool where pages share with the same
245 * request.
246 */
247 if (!bounced && rq->out[no] == rq->in[ni]) {
248 memcpy(strm->bounce, strm->buf.in, strm->buf.in_size);
249 strm->buf.in = strm->bounce;
250 bounced = true;
251 }
252 for (j = ni + 1; j < nrpages_in; ++j) {
253 struct page *tmppage;
254
255 if (rq->out[no] != rq->in[j])
256 continue;
257
258 DBG_BUGON(erofs_page_is_managed(EROFS_SB(rq->sb),
259 rq->in[j]));
260 tmppage = erofs_allocpage(pagepool,
261 GFP_KERNEL | __GFP_NOFAIL);
262 set_page_private(tmppage, Z_EROFS_SHORTLIVED_PAGE);
263 copy_highpage(tmppage, rq->in[j]);
264 rq->in[j] = tmppage;
265 }
266 xz_err = xz_dec_microlzma_run(strm->state, &strm->buf);
267 DBG_BUGON(strm->buf.out_pos > strm->buf.out_size);
268 DBG_BUGON(strm->buf.in_pos > strm->buf.in_size);
269
270 if (xz_err != XZ_OK) {
271 if (xz_err == XZ_STREAM_END && !outlen)
272 break;
273 erofs_err(rq->sb, "failed to decompress %d in[%u] out[%u]",
274 xz_err, rq->inputsize, rq->outputsize);
275 err = -EFSCORRUPTED;
276 break;
277 }
278 }
279 if (no < nrpages_out && strm->buf.out)
280 kunmap(rq->in[no]);
281 if (ni < nrpages_in)
282 kunmap(rq->in[ni]);
283 /* 4. push back LZMA stream context to the global list */
284 spin_lock(&z_erofs_lzma_lock);
285 strm->next = z_erofs_lzma_head;
286 z_erofs_lzma_head = strm;
287 spin_unlock(&z_erofs_lzma_lock);
288 wake_up(&z_erofs_lzma_wq);
289 return err;
290 }
291