72 	      const void *from, size_t count)  in pio_copy()  argument
92 			writeq(*(u64 *)from, dest);  in pio_copy()
93 			from += sizeof(u64);  in pio_copy()
108 			writeq(*(u64 *)from, dest);  in pio_copy()
109 			from += sizeof(u64);  in pio_copy()
127 				writeq(*(u64 *)from, dest);  in pio_copy()
128 				from += sizeof(u64);  in pio_copy()
138 			writeq(*(u64 *)from, dest);  in pio_copy()
139 			from += sizeof(u64);  in pio_copy()
150 		val.val32[0] = *(u32 *)from;  in pio_copy()
225 static inline void read_low_bytes(struct pio_buf *pbuf, const void *from,  in read_low_bytes()  argument
229 	jcopy(&pbuf->carry.val8[0], from, nbytes);  in read_low_bytes()
242 				    const void *from, unsigned int nbytes)  in read_extra_bytes()  argument
244 	jcopy(&pbuf->carry.val8[pbuf->carry_bytes], from, nbytes);  in read_extra_bytes()
305 			const void *from, size_t nbytes)  in seg_pio_copy_start()  argument
324 			writeq(*(u64 *)from, dest);  in seg_pio_copy_start()
325 			from += sizeof(u64);  in seg_pio_copy_start()
340 			writeq(*(u64 *)from, dest);  in seg_pio_copy_start()
341 			from += sizeof(u64);  in seg_pio_copy_start()
359 				writeq(*(u64 *)from, dest);  in seg_pio_copy_start()
360 				from += sizeof(u64);  in seg_pio_copy_start()
370 			writeq(*(u64 *)from, dest);  in seg_pio_copy_start()
371 			from += sizeof(u64);  in seg_pio_copy_start()
380 	read_low_bytes(pbuf, from, nbytes & 0x7);  in seg_pio_copy_start()
397 static void mid_copy_mix(struct pio_buf *pbuf, const void *from, size_t nbytes)  in mid_copy_mix()  argument
429 			merge_write8(pbuf, dest, from);  in mid_copy_mix()
430 			from += sizeof(u64);  in mid_copy_mix()
456 			merge_write8(pbuf, dest, from);  in mid_copy_mix()
457 			from += sizeof(u64);  in mid_copy_mix()
467 		merge_write8(pbuf, dest, from);  in mid_copy_mix()
468 		from += sizeof(u64);  in mid_copy_mix()
480 		read_extra_bytes(pbuf, from, nread);  in mid_copy_mix()
506 		from += nread; /* from is now not aligned */  in mid_copy_mix()
507 		read_low_bytes(pbuf, from, bytes_left);  in mid_copy_mix()
510 		read_extra_bytes(pbuf, from, bytes_left);  in mid_copy_mix()
525 			      const void *from, size_t nbytes)  in mid_copy_straight()  argument
555 			writeq(*(u64 *)from, dest);  in mid_copy_straight()
556 			from += sizeof(u64);  in mid_copy_straight()
582 			writeq(*(u64 *)from, dest);  in mid_copy_straight()
583 			from += sizeof(u64);  in mid_copy_straight()
593 		writeq(*(u64 *)from, dest);  in mid_copy_straight()
594 		from += sizeof(u64);  in mid_copy_straight()
599 	read_low_bytes(pbuf, from, nbytes & 0x7);  in mid_copy_straight()
613 void seg_pio_copy_mid(struct pio_buf *pbuf, const void *from, size_t nbytes)  in seg_pio_copy_mid()  argument
615 	unsigned long from_align = (unsigned long)from & 0x7;  in seg_pio_copy_mid()
619 		read_extra_bytes(pbuf, from, nbytes);  in seg_pio_copy_mid()
638 			read_extra_bytes(pbuf, from, to_align);  in seg_pio_copy_mid()
639 			from += to_align;  in seg_pio_copy_mid()
649 			read_extra_bytes(pbuf, from, to_fill);  in seg_pio_copy_mid()
650 			from += to_fill;  in seg_pio_copy_mid()
679 			read_low_bytes(pbuf, from, extra);  in seg_pio_copy_mid()
680 			from += extra;  in seg_pio_copy_mid()
697 		mid_copy_mix(pbuf, from, nbytes);  in seg_pio_copy_mid()
699 		mid_copy_straight(pbuf, from, nbytes);  in seg_pio_copy_mid()