Lines Matching +full:three +full:- +full:conversion +full:- +full:cycles
2 # SPDX-License-Identifier: GPL-2.0
12 # Copyright (c) 2006-2017, CRYPTOGAMS by <appro@openssl.org>
58 # The module is endian-agnostic in sense that it supports both big-
59 # and little-endian cases. Data alignment in parallelizable modes is
64 # is aligned programmatically, which in turn guarantees exception-
72 # Add XTS subroutine, 9x on little- and 12x improvement on big-endian
76 # Current large-block performance in cycles per byte processed with
77 # 128-bit key (less is better).
79 # CBC en-/decrypt CTR XTS
106 ( $xlate="${dir}ppc-xlate.pl" and -f $xlate ) or
107 ( $xlate="${dir}../../perlasm/ppc-xlate.pl" and -f $xlate) or
108 die "can't locate ppc-xlate.pl";
139 addi $ptr,$ptr,-0x48
151 li $ptr,-1
153 beq- Lenc_key_abort # if ($inp==0) return -1;
155 beq- Lenc_key_abort # if ($out==0) return -1;
156 li $ptr,-2
158 blt- Lenc_key_abort
160 bgt- Lenc_key_abort
162 bne- Lenc_key_abort
189 vspltisb $outmask,-1
201 vperm $key,$in0,$in0,$mask # rotate-n-splat
221 vperm $key,$in0,$in0,$mask # rotate-n-splat
238 vperm $key,$in0,$in0,$mask # rotate-n-splat
279 vperm $key,$in1,$in1,$mask # roate-n-splat
299 vperm $key,$in1,$in1,$mask # rotate-n-splat
353 vperm $key,$in1,$in1,$mask # rotate-n-splat
405 .size .${prefix}_set_encrypt_key,.-.${prefix}_set_encrypt_key
408 $STU $sp,-$FRAME($sp)
415 bne- Ldec_key_abort
438 stw r9, -16($inp)
439 stw r10,-12($inp)
440 stw r11,-8($inp)
441 stw r12,-4($inp)
451 .size .${prefix}_set_decrypt_key,.-.${prefix}_set_decrypt_key
455 {{{ # Single block en- and decrypt procedures #
508 vspltisb v2,-1
526 .size .${prefix}_${dir}crypt,.-.${prefix}_${dir}crypt
533 {{{ # CBC en- and decrypt procedures #
541 bltlr-
568 vspltisb $outmask,-1
583 subi $len,$len,16 # len-=16
631 subi $len,$len,16 # len-=16
671 addi $out,$out,-1
679 vspltisb $outmask,-1
704 my $rndkey0="v23"; # v24-v25 rotating buffer for first found keys
705 # v26-v31 last 6 round keys
711 $STU $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
736 li r0,-1
737 stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave
753 subi $rounds,$rounds,3 # -4 in total
768 stvx v24,$x00,$key_ # off-load round[1]
771 stvx v25,$x10,$key_ # off-load round[2]
778 stvx v24,$x00,$key_ # off-load round[3]
781 stvx v25,$x10,$key_ # off-load round[4]
792 lvx v24,$x00,$key_ # pre-load round[1]
794 lvx v25,$x10,$key_ # pre-load round[2]
856 subic $len,$len,128 # $len-=128
866 subfe. r0,r0,r0 # borrow?-1:0
888 # loop inX-in7 are loaded
908 lvx v24,$x00,$key_ # re-pre-load round[1]
918 lvx v25,$x10,$key_ # re-pre-load round[2]
991 beq Loop_cbc_dec8x # did $len-=128 borrow?
1283 .size .${prefix}_cbc_encrypt,.-.${prefix}_cbc_encrypt
1292 # This code is written as 'ctr32', based on a 32-bit counter used
1293 # upstream. The kernel does *not* use a 32-bit counter. The kernel uses
1294 # a 128-bit counter.
1302 # 1d4aa0b4c181 ("crypto: vmx - Fixing AES-CTR counter bug")
1303 # 009b30ac7444 ("crypto: vmx - CTR: always increment IV as quadword")
1316 bltlr-
1351 vspltisb $outmask,-1
1378 vadduqm $ivec,$ivec,$one # Kernel change for 128-bit
1382 subic. $len,$len,1 # blocks--
1408 addi $out,$out,-1
1425 my $rndkey0="v23"; # v24-v25 rotating buffer for first found keys
1426 # v26-v31 last 6 round keys
1428 my ($two,$three,$four)=($outhead,$outperm,$outmask);
1433 $STU $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
1458 li r0,-1
1459 stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave
1475 subi $rounds,$rounds,3 # -4 in total
1489 stvx v24,$x00,$key_ # off-load round[1]
1492 stvx v25,$x10,$key_ # off-load round[2]
1499 stvx v24,$x00,$key_ # off-load round[3]
1502 stvx v25,$x10,$key_ # off-load round[4]
1513 lvx v24,$x00,$key_ # pre-load round[1]
1515 lvx v25,$x10,$key_ # pre-load round[2]
1522 vadduqm $out2,$ivec,$two # (do all ctr adds as 128-bit)
1569 subic r11,$len,256 # $len-256, borrow $key_
1579 subfe r0,r0,r0 # borrow?-1:0
1599 lvx v24,$x00,$key_ # re-pre-load round[1]
1601 subic $len,$len,129 # $len-=129
1603 addi $len,$len,1 # $len-=128 really
1611 lvx v25,$x10,$key_ # re-pre-load round[2]
1650 # loop inX-in7 are loaded
1652 subfe. r0,r0,r0 # borrow?-1:0
1670 bne Lctr32_enc8x_break # did $len-129 borrow?
1728 cmpwi $len,-0x60
1732 cmpwi $len,-0x40
1736 cmpwi $len,-0x20
1951 .size .${prefix}_ctr32_encrypt_blocks,.-.${prefix}_ctr32_encrypt_blocks
1977 li r3,-1
1979 bltlr-
2043 li $idx,-16
2146 vspltisb $tmp,-1
2182 .size .${prefix}_xts_encrypt,.-.${prefix}_xts_encrypt
2186 li r3,-1
2188 bltlr-
2358 vxor $inout,$inout,$tweak # :-(
2359 vxor $inout,$inout,$tweak1 # :-)
2396 vspltisb $tmp,-1
2436 .size .${prefix}_xts_decrypt,.-.${prefix}_xts_decrypt
2446 my $rndkey0="v23"; # v24-v25 rotating buffer for first found keys
2447 # v26-v31 last 6 round keys
2454 $STU $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
2481 li r0,-1
2482 stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave
2498 subi $rounds,$rounds,3 # -4 in total
2512 stvx v24,$x00,$key_ # off-load round[1]
2515 stvx v25,$x10,$key_ # off-load round[2]
2522 stvx v24,$x00,$key_ # off-load round[3]
2525 stvx v25,$x10,$key_ # off-load round[4]
2536 lvx v24,$x00,$key_ # pre-load round[1]
2538 lvx v25,$x10,$key_ # pre-load round[2]
2628 subic $len,$len,96 # $len-=96
2641 subfe. r0,r0,r0 # borrow?-1:0
2668 # loop inX-in5 are loaded
2696 lvx v24,$x00,$key_ # re-pre-load round[1]
2709 lvx v25,$x10,$key_ # re-pre-load round[2]
2772 beq Loop_xts_enc6x # did $len-=96 borrow?
2904 lvx v24,$x00,$key_ # re-pre-load round[1]
2907 lvx v25,$x10,$key_ # re-pre-load round[2]
2939 vspltisb $out1,-1
3076 lvx v24,$x00,$key_ # re-pre-load round[1]
3085 lvx v25,$x10,$key_ # re-pre-load round[2]
3106 $STU $sp,-`($FRAME+21*16+6*$SIZE_T)`($sp)
3133 li r0,-1
3134 stw $vrsave,`$FRAME+21*16-4`($sp) # save vrsave
3150 subi $rounds,$rounds,3 # -4 in total
3164 stvx v24,$x00,$key_ # off-load round[1]
3167 stvx v25,$x10,$key_ # off-load round[2]
3174 stvx v24,$x00,$key_ # off-load round[3]
3177 stvx v25,$x10,$key_ # off-load round[4]
3188 lvx v24,$x00,$key_ # pre-load round[1]
3190 lvx v25,$x10,$key_ # pre-load round[2]
3280 subic $len,$len,96 # $len-=96
3293 subfe. r0,r0,r0 # borrow?-1:0
3320 # loop inX-in5 are loaded
3348 lvx v24,$x00,$key_ # re-pre-load round[1]
3361 lvx v25,$x10,$key_ # re-pre-load round[2]
3422 beq Loop_xts_dec6x # did $len-=96 borrow?
3558 lvx v24,$x00,$key_ # re-pre-load round[1]
3561 lvx v25,$x10,$key_ # re-pre-load round[2]
3610 lvx v24,$x00,$key_ # re-pre-load round[1]
3613 lvx v25,$x10,$key_ # re-pre-load round[2]
3627 vspltisb $out1,-1
3764 lvx v24,$x00,$key_ # re-pre-load round[1]
3773 lvx v25,$x10,$key_ # re-pre-load round[2]
3798 # constants table endian-specific conversion
3799 if ($consts && m/\.(long|byte)\s+(.+)\s+(\?[a-z]*)$/o) {
3803 # convert to endian-agnostic format
3813 # little-endian conversion
3827 # instructions prefixed with '?' are endian-specific and need
3829 if ($flavour =~ /le$/o) { # little-endian
3834 s/\?(vperm\s+v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+,\s*)(v[0-9]+)/$1$3$2$4/o or
3835 s/\?(vsldoi\s+v[0-9]+,\s*)(v[0-9]+,)\s*(v[0-9]+,\s*)([0-9]+)/$1$3$2 16-$4/o or
3836 s/\?(vspltw\s+v[0-9]+,\s*)(v[0-9]+,)\s*([0-9])/$1$2 3-$3/o;
3837 } else { # big-endian
3840 s/\?([a-z]+)/$1/o;