169  * Handle carry bytes using shifts and masks.
171  * NOTE: the value the unused portion of carry is expected to always be zero.
181  * "merge" shift - bit shift used to merge with carry bytes.  Input is
217  * of pbuf->carry.  Other bytes are left as-is.  Any previous
218  * value in pbuf->carry is lost.
227 	pbuf->carry.val64 = 0;  in read_low_bytes()
228 	jcopy(&pbuf->carry.val8[0], from, nbytes);  in read_low_bytes()
233  * Read nbytes bytes from "from" and put them at the end of pbuf->carry.
234  * It is expected that the extra read does not overfill carry.
243 	jcopy(&pbuf->carry.val8[pbuf->carry_bytes], from, nbytes);  in read_extra_bytes()
248  * Write a quad word using parts of pbuf->carry and the next 8 bytes of src.
250  * pbuf->carry with the upper bytes zeroed..
264 	temp = pbuf->carry.val64 | (new << mshift(pbuf->carry_bytes));  in merge_write8()
266 	pbuf->carry.val64 = new >> zshift(pbuf->carry_bytes);  in merge_write8()
270  * Write a quad word using all bytes of carry.
272 static inline void carry8_write8(union mix carry, void __iomem *dest)  in carry8_write8()  argument
274 	writeq(carry.val64, dest);  in carry8_write8()
278  * Write a quad word using all the valid bytes of carry.  If carry
286 		writeq(pbuf->carry.val64, dest);  in carry_write8()
385  * Mid copy helper, "mixed case" - source is 64-bit aligned but carry
473 	/* handle carry and left-over bytes */  in mid_copy_mix()
477 		/* there is enough to fill another qw - fill carry */  in mid_copy_mix()
499 		/* flush out full carry */  in mid_copy_mix()
500 		carry8_write8(pbuf->carry, dest);  in mid_copy_mix()
503 		/* now adjust and read the rest of the bytes into carry */  in mid_copy_mix()
508 		/* not enough to fill another qw, append the rest to carry */  in mid_copy_mix()
515  * with no carry bytes.
632 		 * if we are here, we already know that carry+nbytes will  in seg_pio_copy_mid()
641 			/* bytes to fill carry */  in seg_pio_copy_mid()
647 			/* fill carry... */  in seg_pio_copy_mid()
655 			/* ...now write carry */  in seg_pio_copy_mid()
673 			carry8_write8(pbuf->carry, dest);  in seg_pio_copy_mid()
677 			/* this will overwrite anything in pbuf->carry */  in seg_pio_copy_mid()
704  * Write any remainder (in pbuf->carry) and finish writing the whole block.