• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1/* SPDX-License-Identifier: GPL-2.0-only */
2/*
3 * Copyright 2008 Vitaly Mayatskikh <vmayatsk@redhat.com>
4 * Copyright 2002 Andi Kleen, SuSE Labs.
5 *
6 * Functions to copy from and to user space.
7 */
8
9#include <linux/linkage.h>
10#include <asm/current.h>
11#include <asm/asm-offsets.h>
12#include <asm/thread_info.h>
13#include <asm/cpufeatures.h>
14#include <asm/alternative.h>
15#include <asm/asm.h>
16#include <asm/smap.h>
17#include <asm/export.h>
18#include <asm/trapnr.h>
19
20.macro ALIGN_DESTINATION
21	/* check for bad alignment of destination */
22	movl %edi,%ecx
23	andl $7,%ecx
24	jz 102f				/* already aligned */
25	subl $8,%ecx
26	negl %ecx
27	subl %ecx,%edx
28100:	movb (%rsi),%al
29101:	movb %al,(%rdi)
30	incq %rsi
31	incq %rdi
32	decl %ecx
33	jnz 100b
34102:
35	.section .fixup,"ax"
36103:	addl %ecx,%edx			/* ecx is zerorest also */
37	jmp .Lcopy_user_handle_tail
38	.previous
39
40	_ASM_EXTABLE_CPY(100b, 103b)
41	_ASM_EXTABLE_CPY(101b, 103b)
42	.endm
43
44/*
45 * copy_user_generic_unrolled - memory copy with exception handling.
46 * This version is for CPUs like P4 that don't have efficient micro
47 * code for rep movsq
48 *
49 * Input:
50 * rdi destination
51 * rsi source
52 * rdx count
53 *
54 * Output:
55 * eax uncopied bytes or 0 if successful.
56 */
57SYM_FUNC_START(copy_user_generic_unrolled)
58	ASM_STAC
59	cmpl $8,%edx
60	jb 20f		/* less then 8 bytes, go to byte copy loop */
61	ALIGN_DESTINATION
62	movl %edx,%ecx
63	andl $63,%edx
64	shrl $6,%ecx
65	jz .L_copy_short_string
661:	movq (%rsi),%r8
672:	movq 1*8(%rsi),%r9
683:	movq 2*8(%rsi),%r10
694:	movq 3*8(%rsi),%r11
705:	movq %r8,(%rdi)
716:	movq %r9,1*8(%rdi)
727:	movq %r10,2*8(%rdi)
738:	movq %r11,3*8(%rdi)
749:	movq 4*8(%rsi),%r8
7510:	movq 5*8(%rsi),%r9
7611:	movq 6*8(%rsi),%r10
7712:	movq 7*8(%rsi),%r11
7813:	movq %r8,4*8(%rdi)
7914:	movq %r9,5*8(%rdi)
8015:	movq %r10,6*8(%rdi)
8116:	movq %r11,7*8(%rdi)
82	leaq 64(%rsi),%rsi
83	leaq 64(%rdi),%rdi
84	decl %ecx
85	jnz 1b
86.L_copy_short_string:
87	movl %edx,%ecx
88	andl $7,%edx
89	shrl $3,%ecx
90	jz 20f
9118:	movq (%rsi),%r8
9219:	movq %r8,(%rdi)
93	leaq 8(%rsi),%rsi
94	leaq 8(%rdi),%rdi
95	decl %ecx
96	jnz 18b
9720:	andl %edx,%edx
98	jz 23f
99	movl %edx,%ecx
10021:	movb (%rsi),%al
10122:	movb %al,(%rdi)
102	incq %rsi
103	incq %rdi
104	decl %ecx
105	jnz 21b
10623:	xor %eax,%eax
107	ASM_CLAC
108	RET
109
110	.section .fixup,"ax"
11130:	shll $6,%ecx
112	addl %ecx,%edx
113	jmp 60f
11440:	leal (%rdx,%rcx,8),%edx
115	jmp 60f
11650:	movl %ecx,%edx
11760:	jmp .Lcopy_user_handle_tail /* ecx is zerorest also */
118	.previous
119
120	_ASM_EXTABLE_CPY(1b, 30b)
121	_ASM_EXTABLE_CPY(2b, 30b)
122	_ASM_EXTABLE_CPY(3b, 30b)
123	_ASM_EXTABLE_CPY(4b, 30b)
124	_ASM_EXTABLE_CPY(5b, 30b)
125	_ASM_EXTABLE_CPY(6b, 30b)
126	_ASM_EXTABLE_CPY(7b, 30b)
127	_ASM_EXTABLE_CPY(8b, 30b)
128	_ASM_EXTABLE_CPY(9b, 30b)
129	_ASM_EXTABLE_CPY(10b, 30b)
130	_ASM_EXTABLE_CPY(11b, 30b)
131	_ASM_EXTABLE_CPY(12b, 30b)
132	_ASM_EXTABLE_CPY(13b, 30b)
133	_ASM_EXTABLE_CPY(14b, 30b)
134	_ASM_EXTABLE_CPY(15b, 30b)
135	_ASM_EXTABLE_CPY(16b, 30b)
136	_ASM_EXTABLE_CPY(18b, 40b)
137	_ASM_EXTABLE_CPY(19b, 40b)
138	_ASM_EXTABLE_CPY(21b, 50b)
139	_ASM_EXTABLE_CPY(22b, 50b)
140SYM_FUNC_END(copy_user_generic_unrolled)
141EXPORT_SYMBOL(copy_user_generic_unrolled)
142
143/* Some CPUs run faster using the string copy instructions.
144 * This is also a lot simpler. Use them when possible.
145 *
146 * Only 4GB of copy is supported. This shouldn't be a problem
147 * because the kernel normally only writes from/to page sized chunks
148 * even if user space passed a longer buffer.
149 * And more would be dangerous because both Intel and AMD have
150 * errata with rep movsq > 4GB. If someone feels the need to fix
151 * this please consider this.
152 *
153 * Input:
154 * rdi destination
155 * rsi source
156 * rdx count
157 *
158 * Output:
159 * eax uncopied bytes or 0 if successful.
160 */
161SYM_FUNC_START(copy_user_generic_string)
162	ASM_STAC
163	cmpl $8,%edx
164	jb 2f		/* less than 8 bytes, go to byte copy loop */
165	ALIGN_DESTINATION
166	movl %edx,%ecx
167	shrl $3,%ecx
168	andl $7,%edx
1691:	rep
170	movsq
1712:	movl %edx,%ecx
1723:	rep
173	movsb
174	xorl %eax,%eax
175	ASM_CLAC
176	RET
177
178	.section .fixup,"ax"
17911:	leal (%rdx,%rcx,8),%ecx
18012:	movl %ecx,%edx		/* ecx is zerorest also */
181	jmp .Lcopy_user_handle_tail
182	.previous
183
184	_ASM_EXTABLE_CPY(1b, 11b)
185	_ASM_EXTABLE_CPY(3b, 12b)
186SYM_FUNC_END(copy_user_generic_string)
187EXPORT_SYMBOL(copy_user_generic_string)
188
189/*
190 * Some CPUs are adding enhanced REP MOVSB/STOSB instructions.
191 * It's recommended to use enhanced REP MOVSB/STOSB if it's enabled.
192 *
193 * Input:
194 * rdi destination
195 * rsi source
196 * rdx count
197 *
198 * Output:
199 * eax uncopied bytes or 0 if successful.
200 */
201SYM_FUNC_START(copy_user_enhanced_fast_string)
202	ASM_STAC
203	cmpl $64,%edx
204	jb .L_copy_short_string	/* less then 64 bytes, avoid the costly 'rep' */
205	movl %edx,%ecx
2061:	rep
207	movsb
208	xorl %eax,%eax
209	ASM_CLAC
210	RET
211
212	.section .fixup,"ax"
21312:	movl %ecx,%edx		/* ecx is zerorest also */
214	jmp .Lcopy_user_handle_tail
215	.previous
216
217	_ASM_EXTABLE_CPY(1b, 12b)
218SYM_FUNC_END(copy_user_enhanced_fast_string)
219EXPORT_SYMBOL(copy_user_enhanced_fast_string)
220
221/*
222 * Try to copy last bytes and clear the rest if needed.
223 * Since protection fault in copy_from/to_user is not a normal situation,
224 * it is not necessary to optimize tail handling.
225 * Don't try to copy the tail if machine check happened
226 *
227 * Input:
228 * rdi destination
229 * rsi source
230 * rdx count
231 *
232 * Output:
233 * eax uncopied bytes or 0 if successful.
234 */
235SYM_CODE_START_LOCAL(.Lcopy_user_handle_tail)
236	movl %edx,%ecx
237	cmp $X86_TRAP_MC,%eax		/* check if X86_TRAP_MC */
238	je 3f
2391:	rep movsb
2402:	mov %ecx,%eax
241	ASM_CLAC
242	RET
243
244	/*
245	 * Return zero to pretend that this copy succeeded. This
246	 * is counter-intuitive, but needed to prevent the code
247	 * in lib/iov_iter.c from retrying and running back into
248	 * the poison cache line again. The machine check handler
249	 * will ensure that a SIGBUS is sent to the task.
250	 */
2513:	xorl %eax,%eax
252	ASM_CLAC
253	RET
254
255	_ASM_EXTABLE_CPY(1b, 2b)
256SYM_CODE_END(.Lcopy_user_handle_tail)
257
258/*
259 * copy_user_nocache - Uncached memory copy with exception handling
260 * This will force destination out of cache for more performance.
261 *
262 * Note: Cached memory copy is used when destination or size is not
263 * naturally aligned. That is:
264 *  - Require 8-byte alignment when size is 8 bytes or larger.
265 *  - Require 4-byte alignment when size is 4 bytes.
266 */
267SYM_FUNC_START(__copy_user_nocache)
268	ASM_STAC
269
270	/* If size is less than 8 bytes, go to 4-byte copy */
271	cmpl $8,%edx
272	jb .L_4b_nocache_copy_entry
273
274	/* If destination is not 8-byte aligned, "cache" copy to align it */
275	ALIGN_DESTINATION
276
277	/* Set 4x8-byte copy count and remainder */
278	movl %edx,%ecx
279	andl $63,%edx
280	shrl $6,%ecx
281	jz .L_8b_nocache_copy_entry	/* jump if count is 0 */
282
283	/* Perform 4x8-byte nocache loop-copy */
284.L_4x8b_nocache_copy_loop:
2851:	movq (%rsi),%r8
2862:	movq 1*8(%rsi),%r9
2873:	movq 2*8(%rsi),%r10
2884:	movq 3*8(%rsi),%r11
2895:	movnti %r8,(%rdi)
2906:	movnti %r9,1*8(%rdi)
2917:	movnti %r10,2*8(%rdi)
2928:	movnti %r11,3*8(%rdi)
2939:	movq 4*8(%rsi),%r8
29410:	movq 5*8(%rsi),%r9
29511:	movq 6*8(%rsi),%r10
29612:	movq 7*8(%rsi),%r11
29713:	movnti %r8,4*8(%rdi)
29814:	movnti %r9,5*8(%rdi)
29915:	movnti %r10,6*8(%rdi)
30016:	movnti %r11,7*8(%rdi)
301	leaq 64(%rsi),%rsi
302	leaq 64(%rdi),%rdi
303	decl %ecx
304	jnz .L_4x8b_nocache_copy_loop
305
306	/* Set 8-byte copy count and remainder */
307.L_8b_nocache_copy_entry:
308	movl %edx,%ecx
309	andl $7,%edx
310	shrl $3,%ecx
311	jz .L_4b_nocache_copy_entry	/* jump if count is 0 */
312
313	/* Perform 8-byte nocache loop-copy */
314.L_8b_nocache_copy_loop:
31520:	movq (%rsi),%r8
31621:	movnti %r8,(%rdi)
317	leaq 8(%rsi),%rsi
318	leaq 8(%rdi),%rdi
319	decl %ecx
320	jnz .L_8b_nocache_copy_loop
321
322	/* If no byte left, we're done */
323.L_4b_nocache_copy_entry:
324	andl %edx,%edx
325	jz .L_finish_copy
326
327	/* If destination is not 4-byte aligned, go to byte copy: */
328	movl %edi,%ecx
329	andl $3,%ecx
330	jnz .L_1b_cache_copy_entry
331
332	/* Set 4-byte copy count (1 or 0) and remainder */
333	movl %edx,%ecx
334	andl $3,%edx
335	shrl $2,%ecx
336	jz .L_1b_cache_copy_entry	/* jump if count is 0 */
337
338	/* Perform 4-byte nocache copy: */
33930:	movl (%rsi),%r8d
34031:	movnti %r8d,(%rdi)
341	leaq 4(%rsi),%rsi
342	leaq 4(%rdi),%rdi
343
344	/* If no bytes left, we're done: */
345	andl %edx,%edx
346	jz .L_finish_copy
347
348	/* Perform byte "cache" loop-copy for the remainder */
349.L_1b_cache_copy_entry:
350	movl %edx,%ecx
351.L_1b_cache_copy_loop:
35240:	movb (%rsi),%al
35341:	movb %al,(%rdi)
354	incq %rsi
355	incq %rdi
356	decl %ecx
357	jnz .L_1b_cache_copy_loop
358
359	/* Finished copying; fence the prior stores */
360.L_finish_copy:
361	xorl %eax,%eax
362	ASM_CLAC
363	sfence
364	RET
365
366	.section .fixup,"ax"
367.L_fixup_4x8b_copy:
368	shll $6,%ecx
369	addl %ecx,%edx
370	jmp .L_fixup_handle_tail
371.L_fixup_8b_copy:
372	lea (%rdx,%rcx,8),%rdx
373	jmp .L_fixup_handle_tail
374.L_fixup_4b_copy:
375	lea (%rdx,%rcx,4),%rdx
376	jmp .L_fixup_handle_tail
377.L_fixup_1b_copy:
378	movl %ecx,%edx
379.L_fixup_handle_tail:
380	sfence
381	jmp .Lcopy_user_handle_tail
382	.previous
383
384	_ASM_EXTABLE_CPY(1b, .L_fixup_4x8b_copy)
385	_ASM_EXTABLE_CPY(2b, .L_fixup_4x8b_copy)
386	_ASM_EXTABLE_CPY(3b, .L_fixup_4x8b_copy)
387	_ASM_EXTABLE_CPY(4b, .L_fixup_4x8b_copy)
388	_ASM_EXTABLE_CPY(5b, .L_fixup_4x8b_copy)
389	_ASM_EXTABLE_CPY(6b, .L_fixup_4x8b_copy)
390	_ASM_EXTABLE_CPY(7b, .L_fixup_4x8b_copy)
391	_ASM_EXTABLE_CPY(8b, .L_fixup_4x8b_copy)
392	_ASM_EXTABLE_CPY(9b, .L_fixup_4x8b_copy)
393	_ASM_EXTABLE_CPY(10b, .L_fixup_4x8b_copy)
394	_ASM_EXTABLE_CPY(11b, .L_fixup_4x8b_copy)
395	_ASM_EXTABLE_CPY(12b, .L_fixup_4x8b_copy)
396	_ASM_EXTABLE_CPY(13b, .L_fixup_4x8b_copy)
397	_ASM_EXTABLE_CPY(14b, .L_fixup_4x8b_copy)
398	_ASM_EXTABLE_CPY(15b, .L_fixup_4x8b_copy)
399	_ASM_EXTABLE_CPY(16b, .L_fixup_4x8b_copy)
400	_ASM_EXTABLE_CPY(20b, .L_fixup_8b_copy)
401	_ASM_EXTABLE_CPY(21b, .L_fixup_8b_copy)
402	_ASM_EXTABLE_CPY(30b, .L_fixup_4b_copy)
403	_ASM_EXTABLE_CPY(31b, .L_fixup_4b_copy)
404	_ASM_EXTABLE_CPY(40b, .L_fixup_1b_copy)
405	_ASM_EXTABLE_CPY(41b, .L_fixup_1b_copy)
406SYM_FUNC_END(__copy_user_nocache)
407EXPORT_SYMBOL(__copy_user_nocache)
408