25 	int (*prepare_destpages)(struct z_erofs_decompress_req *rq,
27 	int (*decompress)(struct z_erofs_decompress_req *rq, u8 *out);
31 static int z_erofs_lz4_prepare_destpages(struct z_erofs_decompress_req *rq,  in z_erofs_lz4_prepare_destpages()  argument
35 		PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT;  in z_erofs_lz4_prepare_destpages()
44 		struct page *const page = rq->out[i];  in z_erofs_lz4_prepare_destpages()
54 			availables[top++] = rq->out[i - LZ4_MAX_DISTANCE_PAGES];  in z_erofs_lz4_prepare_destpages()
85 		rq->out[i] = victim;  in z_erofs_lz4_prepare_destpages()
90 static void *generic_copy_inplace_data(struct z_erofs_decompress_req *rq,  in generic_copy_inplace_data()  argument
97 	struct page **in = rq->in;  in generic_copy_inplace_data()
100 	unsigned int inlen = rq->inputsize - pageofs_in;  in generic_copy_inplace_data()
117 static int z_erofs_lz4_decompress(struct z_erofs_decompress_req *rq, u8 *out)  in z_erofs_lz4_decompress()  argument
124 	if (rq->inputsize > PAGE_SIZE)  in z_erofs_lz4_decompress()
127 	src = kmap_atomic(*rq->in);  in z_erofs_lz4_decompress()
132 	if (EROFS_SB(rq->sb)->feature_incompat &  in z_erofs_lz4_decompress()
140 		if (inputmargin >= rq->inputsize) {  in z_erofs_lz4_decompress()
147 	inlen = rq->inputsize - inputmargin;  in z_erofs_lz4_decompress()
148 	if (rq->inplace_io) {  in z_erofs_lz4_decompress()
149 		const uint oend = (rq->pageofs_out +  in z_erofs_lz4_decompress()
150 				   rq->outputsize) & ~PAGE_MASK;  in z_erofs_lz4_decompress()
151 		const uint nr = PAGE_ALIGN(rq->pageofs_out +  in z_erofs_lz4_decompress()
152 					   rq->outputsize) >> PAGE_SHIFT;  in z_erofs_lz4_decompress()
154 		if (rq->partial_decoding || !support_0padding ||  in z_erofs_lz4_decompress()
155 		    rq->out[nr - 1] != rq->in[0] ||  in z_erofs_lz4_decompress()
156 		    rq->inputsize - oend <  in z_erofs_lz4_decompress()
158 			src = generic_copy_inplace_data(rq, src, inputmargin);  in z_erofs_lz4_decompress()
165 	if (rq->partial_decoding || !support_0padding)  in z_erofs_lz4_decompress()
167 						  inlen, rq->outputsize,  in z_erofs_lz4_decompress()
168 						  rq->outputsize);  in z_erofs_lz4_decompress()
171 					  inlen, rq->outputsize);  in z_erofs_lz4_decompress()
173 	if (ret != rq->outputsize) {  in z_erofs_lz4_decompress()
174 		erofs_err(rq->sb, "failed to decompress %d in[%u, %u] out[%u]",  in z_erofs_lz4_decompress()
175 			  ret, inlen, inputmargin, rq->outputsize);  in z_erofs_lz4_decompress()
180 			       16, 1, out, rq->outputsize, true);  in z_erofs_lz4_decompress()
183 			memset(out + ret, 0, rq->outputsize - ret);  in z_erofs_lz4_decompress()
232 static int z_erofs_decompress_generic(struct z_erofs_decompress_req *rq,  in z_erofs_decompress_generic()  argument
236 		PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT;  in z_erofs_decompress_generic()
237 	const struct z_erofs_decompressor *alg = decompressors + rq->alg;  in z_erofs_decompress_generic()
242 	if (nrpages_out == 1 && !rq->inplace_io) {  in z_erofs_decompress_generic()
243 		DBG_BUGON(!*rq->out);  in z_erofs_decompress_generic()
244 		dst = kmap_atomic(*rq->out);  in z_erofs_decompress_generic()
254 	if (rq->outputsize <= PAGE_SIZE * 7 / 8) {  in z_erofs_decompress_generic()
259 		rq->inplace_io = false;  in z_erofs_decompress_generic()
260 		ret = alg->decompress(rq, dst);  in z_erofs_decompress_generic()
262 			copy_from_pcpubuf(rq->out, dst, rq->pageofs_out,  in z_erofs_decompress_generic()
263 					  rq->outputsize);  in z_erofs_decompress_generic()
269 	ret = alg->prepare_destpages(rq, pagepool);  in z_erofs_decompress_generic()
273 		dst = page_address(*rq->out);  in z_erofs_decompress_generic()
280 		dst = vm_map_ram(rq->out, nrpages_out, -1);  in z_erofs_decompress_generic()
294 	ret = alg->decompress(rq, dst + rq->pageofs_out);  in z_erofs_decompress_generic()
303 static int z_erofs_shifted_transform(const struct z_erofs_decompress_req *rq,  in z_erofs_shifted_transform()  argument
307 		PAGE_ALIGN(rq->pageofs_out + rq->outputsize) >> PAGE_SHIFT;  in z_erofs_shifted_transform()
308 	const unsigned int righthalf = PAGE_SIZE - rq->pageofs_out;  in z_erofs_shifted_transform()
316 	if (rq->out[0] == *rq->in) {  in z_erofs_shifted_transform()
321 	src = kmap_atomic(*rq->in);  in z_erofs_shifted_transform()
322 	if (rq->out[0]) {  in z_erofs_shifted_transform()
323 		dst = kmap_atomic(rq->out[0]);  in z_erofs_shifted_transform()
324 		memcpy(dst + rq->pageofs_out, src, righthalf);  in z_erofs_shifted_transform()
329 		DBG_BUGON(!rq->out[1]);  in z_erofs_shifted_transform()
330 		if (rq->out[1] == *rq->in) {  in z_erofs_shifted_transform()
331 			memmove(src, src + righthalf, rq->pageofs_out);  in z_erofs_shifted_transform()
333 			dst = kmap_atomic(rq->out[1]);  in z_erofs_shifted_transform()
334 			memcpy(dst, src + righthalf, rq->pageofs_out);  in z_erofs_shifted_transform()
342 int z_erofs_decompress(struct z_erofs_decompress_req *rq,  in z_erofs_decompress()  argument
345 	if (rq->alg == Z_EROFS_COMPRESSION_SHIFTED)  in z_erofs_decompress()
346 		return z_erofs_shifted_transform(rq, pagepool);  in z_erofs_decompress()
347 	return z_erofs_decompress_generic(rq, pagepool);  in z_erofs_decompress()