169  * Handle carry bytes using shifts and masks.
171  * NOTE: the value the unused portion of carry is expected to always be zero.
181  * "merge" shift - bit shift used to merge with carry bytes.  Input is
218  * of pbuf->carry.  Other bytes are left as-is.  Any previous
219  * value in pbuf->carry is lost.
228 	pbuf->carry.val64 = 0;  in read_low_bytes()
229 	jcopy(&pbuf->carry.val8[0], from, nbytes);  in read_low_bytes()
234  * Read nbytes bytes from "from" and put them at the end of pbuf->carry.
235  * It is expected that the extra read does not overfill carry.
244 	jcopy(&pbuf->carry.val8[pbuf->carry_bytes], from, nbytes);  in read_extra_bytes()
249  * Write a quad word using parts of pbuf->carry and the next 8 bytes of src.
251  * pbuf->carry with the upper bytes zeroed..
265 	temp = pbuf->carry.val64 | (new << mshift(pbuf->carry_bytes));  in merge_write8()
267 	pbuf->carry.val64 = new >> zshift(pbuf->carry_bytes);  in merge_write8()
271  * Write a quad word using all bytes of carry.
273 static inline void carry8_write8(union mix carry, void __iomem *dest)  in carry8_write8()  argument
275 	writeq(carry.val64, dest);  in carry8_write8()
279  * Write a quad word using all the valid bytes of carry.  If carry
287 		writeq(pbuf->carry.val64, dest);  in carry_write8()
386  * Mid copy helper, "mixed case" - source is 64-bit aligned but carry
474 	/* handle carry and left-over bytes */  in mid_copy_mix()
478 		/* there is enough to fill another qw - fill carry */  in mid_copy_mix()
500 		/* flush out full carry */  in mid_copy_mix()
501 		carry8_write8(pbuf->carry, dest);  in mid_copy_mix()
504 		/* now adjust and read the rest of the bytes into carry */  in mid_copy_mix()
509 		/* not enough to fill another qw, append the rest to carry */  in mid_copy_mix()
516  * with no carry bytes.
633 		 * if we are here, we already know that carry+nbytes will  in seg_pio_copy_mid()
642 			/* bytes to fill carry */  in seg_pio_copy_mid()
648 			/* fill carry... */  in seg_pio_copy_mid()
656 			/* ...now write carry */  in seg_pio_copy_mid()
674 			carry8_write8(pbuf->carry, dest);  in seg_pio_copy_mid()
678 			/* this will overwrite anything in pbuf->carry */  in seg_pio_copy_mid()
705  * Write any remainder (in pbuf->carry) and finish writing the whole block.