24 #define ALIGN_DEST_TO8_UP(d,s,n) \  argument
28 		*(char *) d = *(char *) s; \
29 		d++; s++; \
31 #define ALIGN_DEST_TO8_DN(d,s,n) \  argument
35 		d--; s--; \
36 		*(char *) d = *(char *) s; \
43 #define DO_REST_UP(d,s,n) \  argument
46 		*(char *) d = *(char *) s; \
47 		d++; s++; \
49 #define DO_REST_DN(d,s,n) \  argument
52 		d--; s--; \
53 		*(char *) d = *(char *) s; \
60 #define DO_REST_ALIGNED_UP(d,s,n) DO_REST_UP(d,s,n)  argument
61 #define DO_REST_ALIGNED_DN(d,s,n) DO_REST_DN(d,s,n)  argument
70 static inline void __memcpy_unaligned_up (unsigned long d, unsigned long s,  in __memcpy_unaligned_up()  argument
73 	ALIGN_DEST_TO8_UP(d,s,n);  in __memcpy_unaligned_up()
77 		__asm__("ldq_u %0,%1":"=r" (low_word):"m" (*(unsigned long *) s));  in __memcpy_unaligned_up()
80 			__asm__("ldq_u %0,%1":"=r" (high_word):"m" (*(unsigned long *)(s+8)));  in __memcpy_unaligned_up()
84 				:"r" (low_word), "r" (s));  in __memcpy_unaligned_up()
87 				:"r" (high_word), "r" (s));  in __memcpy_unaligned_up()
88 			s += 8;  in __memcpy_unaligned_up()
95 	DO_REST_UP(d,s,n);  in __memcpy_unaligned_up()
98 static inline void __memcpy_unaligned_dn (unsigned long d, unsigned long s,  in __memcpy_unaligned_dn()  argument
102 	s += n;  in __memcpy_unaligned_dn()
105 		* (char *) --d = * (char *) --s;  in __memcpy_unaligned_dn()
116 static inline void __memcpy_aligned_up (unsigned long d, unsigned long s,  in __memcpy_aligned_up()  argument
119 	ALIGN_DEST_TO8_UP(d,s,n);  in __memcpy_aligned_up()
123 		__asm__("ldq %0,%1":"=r" (tmp):"m" (*(unsigned long *) s));  in __memcpy_aligned_up()
125 		s += 8;  in __memcpy_aligned_up()
130 	DO_REST_ALIGNED_UP(d,s,n);  in __memcpy_aligned_up()
132 static inline void __memcpy_aligned_dn (unsigned long d, unsigned long s,  in __memcpy_aligned_dn()  argument
135 	s += n;  in __memcpy_aligned_dn()
137 	ALIGN_DEST_TO8_DN(d,s,n);  in __memcpy_aligned_dn()
141 		s -= 8;  in __memcpy_aligned_dn()
142 		__asm__("ldq %0,%1":"=r" (tmp):"m" (*(unsigned long *) s));  in __memcpy_aligned_dn()
148 	DO_REST_ALIGNED_DN(d,s,n);  in __memcpy_aligned_dn()