Lines Matching refs:T1
291 .macro GHASH_MUL_AVX GH HK T1 T2 T3 T4 T5
298 vpclmulqdq $0x11, \HK, \GH, \T1 # T1 = a1*b1
302 vpxor \T1, \T2,\T2 # T2 = a0*b1+a1*b0
307 vpxor \T2, \T1, \T1 # <T1:GH> = GH x HK
332 vpxor \T1, \GH, \GH # the result is in GH
337 .macro PRECOMPUTE_AVX HK T1 T2 T3 T4 T5 T6
342 vpshufd $0b01001110, \T5, \T1
343 vpxor \T5, \T1, \T1
344 vmovdqa \T1, HashKey_k(arg1)
346 GHASH_MUL_AVX \T5, \HK, \T1, \T3, \T4, \T6, \T2 # T5 = HashKey^2<<1 mod poly
348 vpshufd $0b01001110, \T5, \T1
349 vpxor \T5, \T1, \T1
350 vmovdqa \T1, HashKey_2_k(arg1)
352 GHASH_MUL_AVX \T5, \HK, \T1, \T3, \T4, \T6, \T2 # T5 = HashKey^3<<1 mod poly
354 vpshufd $0b01001110, \T5, \T1
355 vpxor \T5, \T1, \T1
356 vmovdqa \T1, HashKey_3_k(arg1)
358 GHASH_MUL_AVX \T5, \HK, \T1, \T3, \T4, \T6, \T2 # T5 = HashKey^4<<1 mod poly
360 vpshufd $0b01001110, \T5, \T1
361 vpxor \T5, \T1, \T1
362 vmovdqa \T1, HashKey_4_k(arg1)
364 GHASH_MUL_AVX \T5, \HK, \T1, \T3, \T4, \T6, \T2 # T5 = HashKey^5<<1 mod poly
366 vpshufd $0b01001110, \T5, \T1
367 vpxor \T5, \T1, \T1
368 vmovdqa \T1, HashKey_5_k(arg1)
370 GHASH_MUL_AVX \T5, \HK, \T1, \T3, \T4, \T6, \T2 # T5 = HashKey^6<<1 mod poly
372 vpshufd $0b01001110, \T5, \T1
373 vpxor \T5, \T1, \T1
374 vmovdqa \T1, HashKey_6_k(arg1)
376 GHASH_MUL_AVX \T5, \HK, \T1, \T3, \T4, \T6, \T2 # T5 = HashKey^7<<1 mod poly
378 vpshufd $0b01001110, \T5, \T1
379 vpxor \T5, \T1, \T1
380 vmovdqa \T1, HashKey_7_k(arg1)
382 GHASH_MUL_AVX \T5, \HK, \T1, \T3, \T4, \T6, \T2 # T5 = HashKey^8<<1 mod poly
384 vpshufd $0b01001110, \T5, \T1
385 vpxor \T5, \T1, \T1
386 vmovdqa \T1, HashKey_8_k(arg1)
397 .macro INITIAL_BLOCKS_AVX num_initial_blocks T1 T2 T3 T4 T5 CTR XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 …
416 GHASH_MUL_AVX reg_j, \T2, \T1, \T3, \T4, \T5, \T6
434 movq (%r10), \T1
437 vpslldq $8, \T1, \T1
439 vpxor \T1, reg_i, reg_i
445 movq %rax, \T1
448 vpslldq $12, \T1, \T1
450 vpxor \T1, reg_i, reg_i
457 movdqu aad_shift_arr(%r11), \T1
458 vpshufb \T1, reg_i, reg_i
462 GHASH_MUL_AVX reg_i, \T2, \T1, \T3, \T4, \T5, \T6
522 vmovdqu (arg3, %r11), \T1
523 vpxor \T1, reg_i, reg_i
527 vmovdqa \T1, reg_i
541 … GHASH_MUL_AVX reg_j, \T2, \T1, \T3, \T4, \T5, \T6 # apply GHASH on num_initial_blocks blocks
625 vmovdqu (arg3, %r11), \T1
626 vpxor \T1, \XMM1, \XMM1
629 vmovdqa \T1, \XMM1
632 vmovdqu 16*1(arg3, %r11), \T1
633 vpxor \T1, \XMM2, \XMM2
636 vmovdqa \T1, \XMM2
639 vmovdqu 16*2(arg3, %r11), \T1
640 vpxor \T1, \XMM3, \XMM3
643 vmovdqa \T1, \XMM3
646 vmovdqu 16*3(arg3, %r11), \T1
647 vpxor \T1, \XMM4, \XMM4
650 vmovdqa \T1, \XMM4
653 vmovdqu 16*4(arg3, %r11), \T1
654 vpxor \T1, \XMM5, \XMM5
657 vmovdqa \T1, \XMM5
660 vmovdqu 16*5(arg3, %r11), \T1
661 vpxor \T1, \XMM6, \XMM6
664 vmovdqa \T1, \XMM6
667 vmovdqu 16*6(arg3, %r11), \T1
668 vpxor \T1, \XMM7, \XMM7
671 vmovdqa \T1, \XMM7
674 vmovdqu 16*7(arg3, %r11), \T1
675 vpxor \T1, \XMM8, \XMM8
678 vmovdqa \T1, \XMM8
703 .macro GHASH_8_ENCRYPT_8_PARALLEL_AVX T1 T2 T3 T4 T5 T6 CTR XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8…
748 vmovdqu (arg1), \T1
749 vpxor \T1, \XMM1, \XMM1
750 vpxor \T1, \XMM2, \XMM2
751 vpxor \T1, \XMM3, \XMM3
752 vpxor \T1, \XMM4, \XMM4
753 vpxor \T1, \XMM5, \XMM5
754 vpxor \T1, \XMM6, \XMM6
755 vpxor \T1, \XMM7, \XMM7
756 vpxor \T1, \XMM8, \XMM8
764 vmovdqu 16*1(arg1), \T1
765 vaesenc \T1, \XMM1, \XMM1
766 vaesenc \T1, \XMM2, \XMM2
767 vaesenc \T1, \XMM3, \XMM3
768 vaesenc \T1, \XMM4, \XMM4
769 vaesenc \T1, \XMM5, \XMM5
770 vaesenc \T1, \XMM6, \XMM6
771 vaesenc \T1, \XMM7, \XMM7
772 vaesenc \T1, \XMM8, \XMM8
774 vmovdqu 16*2(arg1), \T1
775 vaesenc \T1, \XMM1, \XMM1
776 vaesenc \T1, \XMM2, \XMM2
777 vaesenc \T1, \XMM3, \XMM3
778 vaesenc \T1, \XMM4, \XMM4
779 vaesenc \T1, \XMM5, \XMM5
780 vaesenc \T1, \XMM6, \XMM6
781 vaesenc \T1, \XMM7, \XMM7
782 vaesenc \T1, \XMM8, \XMM8
797 vmovdqu 16*3(arg1), \T1
798 vaesenc \T1, \XMM1, \XMM1
799 vaesenc \T1, \XMM2, \XMM2
800 vaesenc \T1, \XMM3, \XMM3
801 vaesenc \T1, \XMM4, \XMM4
802 vaesenc \T1, \XMM5, \XMM5
803 vaesenc \T1, \XMM6, \XMM6
804 vaesenc \T1, \XMM7, \XMM7
805 vaesenc \T1, \XMM8, \XMM8
807 vmovdqa TMP2(%rsp), \T1
809 vpclmulqdq $0x11, \T5, \T1, \T3
811 vpclmulqdq $0x00, \T5, \T1, \T3
814 vpshufd $0b01001110, \T1, \T3
815 vpxor \T1, \T3, \T3
820 vmovdqu 16*4(arg1), \T1
821 vaesenc \T1, \XMM1, \XMM1
822 vaesenc \T1, \XMM2, \XMM2
823 vaesenc \T1, \XMM3, \XMM3
824 vaesenc \T1, \XMM4, \XMM4
825 vaesenc \T1, \XMM5, \XMM5
826 vaesenc \T1, \XMM6, \XMM6
827 vaesenc \T1, \XMM7, \XMM7
828 vaesenc \T1, \XMM8, \XMM8
832 vmovdqa TMP3(%rsp), \T1
834 vpclmulqdq $0x11, \T5, \T1, \T3
836 vpclmulqdq $0x00, \T5, \T1, \T3
839 vpshufd $0b01001110, \T1, \T3
840 vpxor \T1, \T3, \T3
845 vmovdqu 16*5(arg1), \T1
846 vaesenc \T1, \XMM1, \XMM1
847 vaesenc \T1, \XMM2, \XMM2
848 vaesenc \T1, \XMM3, \XMM3
849 vaesenc \T1, \XMM4, \XMM4
850 vaesenc \T1, \XMM5, \XMM5
851 vaesenc \T1, \XMM6, \XMM6
852 vaesenc \T1, \XMM7, \XMM7
853 vaesenc \T1, \XMM8, \XMM8
855 vmovdqa TMP4(%rsp), \T1
857 vpclmulqdq $0x11, \T5, \T1, \T3
859 vpclmulqdq $0x00, \T5, \T1, \T3
862 vpshufd $0b01001110, \T1, \T3
863 vpxor \T1, \T3, \T3
868 vmovdqu 16*6(arg1), \T1
869 vaesenc \T1, \XMM1, \XMM1
870 vaesenc \T1, \XMM2, \XMM2
871 vaesenc \T1, \XMM3, \XMM3
872 vaesenc \T1, \XMM4, \XMM4
873 vaesenc \T1, \XMM5, \XMM5
874 vaesenc \T1, \XMM6, \XMM6
875 vaesenc \T1, \XMM7, \XMM7
876 vaesenc \T1, \XMM8, \XMM8
879 vmovdqa TMP5(%rsp), \T1
881 vpclmulqdq $0x11, \T5, \T1, \T3
883 vpclmulqdq $0x00, \T5, \T1, \T3
886 vpshufd $0b01001110, \T1, \T3
887 vpxor \T1, \T3, \T3
892 vmovdqu 16*7(arg1), \T1
893 vaesenc \T1, \XMM1, \XMM1
894 vaesenc \T1, \XMM2, \XMM2
895 vaesenc \T1, \XMM3, \XMM3
896 vaesenc \T1, \XMM4, \XMM4
897 vaesenc \T1, \XMM5, \XMM5
898 vaesenc \T1, \XMM6, \XMM6
899 vaesenc \T1, \XMM7, \XMM7
900 vaesenc \T1, \XMM8, \XMM8
902 vmovdqa TMP6(%rsp), \T1
904 vpclmulqdq $0x11, \T5, \T1, \T3
906 vpclmulqdq $0x00, \T5, \T1, \T3
909 vpshufd $0b01001110, \T1, \T3
910 vpxor \T1, \T3, \T3
916 vmovdqu 16*8(arg1), \T1
917 vaesenc \T1, \XMM1, \XMM1
918 vaesenc \T1, \XMM2, \XMM2
919 vaesenc \T1, \XMM3, \XMM3
920 vaesenc \T1, \XMM4, \XMM4
921 vaesenc \T1, \XMM5, \XMM5
922 vaesenc \T1, \XMM6, \XMM6
923 vaesenc \T1, \XMM7, \XMM7
924 vaesenc \T1, \XMM8, \XMM8
926 vmovdqa TMP7(%rsp), \T1
928 vpclmulqdq $0x11, \T5, \T1, \T3
930 vpclmulqdq $0x00, \T5, \T1, \T3
933 vpshufd $0b01001110, \T1, \T3
934 vpxor \T1, \T3, \T3
951 vmovdqa TMP8(%rsp), \T1
953 vpclmulqdq $0x11, \T5, \T1, \T3
955 vpclmulqdq $0x00, \T5, \T1, \T3
958 vpshufd $0b01001110, \T1, \T3
959 vpxor \T1, \T3, \T3
1005 vpsrldq $4, \T2, \T1 # shift-R T1 1 DW
1029 vpxor \T1, \T2, \T2
1052 .macro GHASH_LAST_8_AVX T1 T2 T3 T4 T5 T6 T7 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8
1193 vpsrldq $4, \T2, \T1 # shift-R T1 1 DW
1207 vpxor \T1, \T2, \T2
1630 .macro GHASH_MUL_AVX2 GH HK T1 T2 T3 T4 T5
1632 vpclmulqdq $0x11,\HK,\GH,\T1 # T1 = a1*b1
1642 vpxor \T3, \T1, \T1
1663 vpxor \T1, \GH, \GH # the result is in GH
1668 .macro PRECOMPUTE_AVX2 HK T1 T2 T3 T4 T5 T6
1672 GHASH_MUL_AVX2 \T5, \HK, \T1, \T3, \T4, \T6, \T2 # T5 = HashKey^2<<1 mod poly
1675 GHASH_MUL_AVX2 \T5, \HK, \T1, \T3, \T4, \T6, \T2 # T5 = HashKey^3<<1 mod poly
1678 GHASH_MUL_AVX2 \T5, \HK, \T1, \T3, \T4, \T6, \T2 # T5 = HashKey^4<<1 mod poly
1681 GHASH_MUL_AVX2 \T5, \HK, \T1, \T3, \T4, \T6, \T2 # T5 = HashKey^5<<1 mod poly
1684 GHASH_MUL_AVX2 \T5, \HK, \T1, \T3, \T4, \T6, \T2 # T5 = HashKey^6<<1 mod poly
1687 GHASH_MUL_AVX2 \T5, \HK, \T1, \T3, \T4, \T6, \T2 # T5 = HashKey^7<<1 mod poly
1690 GHASH_MUL_AVX2 \T5, \HK, \T1, \T3, \T4, \T6, \T2 # T5 = HashKey^8<<1 mod poly
1703 .macro INITIAL_BLOCKS_AVX2 num_initial_blocks T1 T2 T3 T4 T5 CTR XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7…
1723 GHASH_MUL_AVX2 reg_j, \T2, \T1, \T3, \T4, \T5, \T6
1741 movq (%r10), \T1
1744 vpslldq $8, \T1, \T1
1746 vpxor \T1, reg_i, reg_i
1752 movq %rax, \T1
1755 vpslldq $12, \T1, \T1
1757 vpxor \T1, reg_i, reg_i
1764 movdqu aad_shift_arr(%r11), \T1
1765 vpshufb \T1, reg_i, reg_i
1769 GHASH_MUL_AVX2 reg_i, \T2, \T1, \T3, \T4, \T5, \T6
1829 vmovdqu (arg3, %r11), \T1
1830 vpxor \T1, reg_i, reg_i
1835 vmovdqa \T1, reg_i
1849 …GHASH_MUL_AVX2 reg_j, \T2, \T1, \T3, \T4, \T5, \T6 # apply GHASH on num_initial_blocks bloc…
1933 vmovdqu (arg3, %r11), \T1
1934 vpxor \T1, \XMM1, \XMM1
1937 vmovdqa \T1, \XMM1
1940 vmovdqu 16*1(arg3, %r11), \T1
1941 vpxor \T1, \XMM2, \XMM2
1944 vmovdqa \T1, \XMM2
1947 vmovdqu 16*2(arg3, %r11), \T1
1948 vpxor \T1, \XMM3, \XMM3
1951 vmovdqa \T1, \XMM3
1954 vmovdqu 16*3(arg3, %r11), \T1
1955 vpxor \T1, \XMM4, \XMM4
1958 vmovdqa \T1, \XMM4
1961 vmovdqu 16*4(arg3, %r11), \T1
1962 vpxor \T1, \XMM5, \XMM5
1965 vmovdqa \T1, \XMM5
1968 vmovdqu 16*5(arg3, %r11), \T1
1969 vpxor \T1, \XMM6, \XMM6
1972 vmovdqa \T1, \XMM6
1975 vmovdqu 16*6(arg3, %r11), \T1
1976 vpxor \T1, \XMM7, \XMM7
1979 vmovdqa \T1, \XMM7
1982 vmovdqu 16*7(arg3, %r11), \T1
1983 vpxor \T1, \XMM8, \XMM8
1986 vmovdqa \T1, \XMM8
2015 .macro GHASH_8_ENCRYPT_8_PARALLEL_AVX2 T1 T2 T3 T4 T5 T6 CTR XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM…
2060 vmovdqu (arg1), \T1
2061 vpxor \T1, \XMM1, \XMM1
2062 vpxor \T1, \XMM2, \XMM2
2063 vpxor \T1, \XMM3, \XMM3
2064 vpxor \T1, \XMM4, \XMM4
2065 vpxor \T1, \XMM5, \XMM5
2066 vpxor \T1, \XMM6, \XMM6
2067 vpxor \T1, \XMM7, \XMM7
2068 vpxor \T1, \XMM8, \XMM8
2076 vmovdqu 16*1(arg1), \T1
2077 vaesenc \T1, \XMM1, \XMM1
2078 vaesenc \T1, \XMM2, \XMM2
2079 vaesenc \T1, \XMM3, \XMM3
2080 vaesenc \T1, \XMM4, \XMM4
2081 vaesenc \T1, \XMM5, \XMM5
2082 vaesenc \T1, \XMM6, \XMM6
2083 vaesenc \T1, \XMM7, \XMM7
2084 vaesenc \T1, \XMM8, \XMM8
2086 vmovdqu 16*2(arg1), \T1
2087 vaesenc \T1, \XMM1, \XMM1
2088 vaesenc \T1, \XMM2, \XMM2
2089 vaesenc \T1, \XMM3, \XMM3
2090 vaesenc \T1, \XMM4, \XMM4
2091 vaesenc \T1, \XMM5, \XMM5
2092 vaesenc \T1, \XMM6, \XMM6
2093 vaesenc \T1, \XMM7, \XMM7
2094 vaesenc \T1, \XMM8, \XMM8
2106 vmovdqu 16*3(arg1), \T1
2107 vaesenc \T1, \XMM1, \XMM1
2108 vaesenc \T1, \XMM2, \XMM2
2109 vaesenc \T1, \XMM3, \XMM3
2110 vaesenc \T1, \XMM4, \XMM4
2111 vaesenc \T1, \XMM5, \XMM5
2112 vaesenc \T1, \XMM6, \XMM6
2113 vaesenc \T1, \XMM7, \XMM7
2114 vaesenc \T1, \XMM8, \XMM8
2116 vmovdqa TMP2(%rsp), \T1
2118 vpclmulqdq $0x11, \T5, \T1, \T3
2121 vpclmulqdq $0x00, \T5, \T1, \T3
2124 vpclmulqdq $0x01, \T5, \T1, \T3
2127 vpclmulqdq $0x10, \T5, \T1, \T3
2130 vmovdqu 16*4(arg1), \T1
2131 vaesenc \T1, \XMM1, \XMM1
2132 vaesenc \T1, \XMM2, \XMM2
2133 vaesenc \T1, \XMM3, \XMM3
2134 vaesenc \T1, \XMM4, \XMM4
2135 vaesenc \T1, \XMM5, \XMM5
2136 vaesenc \T1, \XMM6, \XMM6
2137 vaesenc \T1, \XMM7, \XMM7
2138 vaesenc \T1, \XMM8, \XMM8
2142 vmovdqa TMP3(%rsp), \T1
2144 vpclmulqdq $0x11, \T5, \T1, \T3
2147 vpclmulqdq $0x00, \T5, \T1, \T3
2150 vpclmulqdq $0x01, \T5, \T1, \T3
2153 vpclmulqdq $0x10, \T5, \T1, \T3
2156 vmovdqu 16*5(arg1), \T1
2157 vaesenc \T1, \XMM1, \XMM1
2158 vaesenc \T1, \XMM2, \XMM2
2159 vaesenc \T1, \XMM3, \XMM3
2160 vaesenc \T1, \XMM4, \XMM4
2161 vaesenc \T1, \XMM5, \XMM5
2162 vaesenc \T1, \XMM6, \XMM6
2163 vaesenc \T1, \XMM7, \XMM7
2164 vaesenc \T1, \XMM8, \XMM8
2166 vmovdqa TMP4(%rsp), \T1
2168 vpclmulqdq $0x11, \T5, \T1, \T3
2171 vpclmulqdq $0x00, \T5, \T1, \T3
2174 vpclmulqdq $0x01, \T5, \T1, \T3
2177 vpclmulqdq $0x10, \T5, \T1, \T3
2180 vmovdqu 16*6(arg1), \T1
2181 vaesenc \T1, \XMM1, \XMM1
2182 vaesenc \T1, \XMM2, \XMM2
2183 vaesenc \T1, \XMM3, \XMM3
2184 vaesenc \T1, \XMM4, \XMM4
2185 vaesenc \T1, \XMM5, \XMM5
2186 vaesenc \T1, \XMM6, \XMM6
2187 vaesenc \T1, \XMM7, \XMM7
2188 vaesenc \T1, \XMM8, \XMM8
2191 vmovdqa TMP5(%rsp), \T1
2193 vpclmulqdq $0x11, \T5, \T1, \T3
2196 vpclmulqdq $0x00, \T5, \T1, \T3
2199 vpclmulqdq $0x01, \T5, \T1, \T3
2202 vpclmulqdq $0x10, \T5, \T1, \T3
2205 vmovdqu 16*7(arg1), \T1
2206 vaesenc \T1, \XMM1, \XMM1
2207 vaesenc \T1, \XMM2, \XMM2
2208 vaesenc \T1, \XMM3, \XMM3
2209 vaesenc \T1, \XMM4, \XMM4
2210 vaesenc \T1, \XMM5, \XMM5
2211 vaesenc \T1, \XMM6, \XMM6
2212 vaesenc \T1, \XMM7, \XMM7
2213 vaesenc \T1, \XMM8, \XMM8
2215 vmovdqa TMP6(%rsp), \T1
2217 vpclmulqdq $0x11, \T5, \T1, \T3
2220 vpclmulqdq $0x00, \T5, \T1, \T3
2223 vpclmulqdq $0x01, \T5, \T1, \T3
2226 vpclmulqdq $0x10, \T5, \T1, \T3
2229 vmovdqu 16*8(arg1), \T1
2230 vaesenc \T1, \XMM1, \XMM1
2231 vaesenc \T1, \XMM2, \XMM2
2232 vaesenc \T1, \XMM3, \XMM3
2233 vaesenc \T1, \XMM4, \XMM4
2234 vaesenc \T1, \XMM5, \XMM5
2235 vaesenc \T1, \XMM6, \XMM6
2236 vaesenc \T1, \XMM7, \XMM7
2237 vaesenc \T1, \XMM8, \XMM8
2239 vmovdqa TMP7(%rsp), \T1
2241 vpclmulqdq $0x11, \T5, \T1, \T3
2244 vpclmulqdq $0x00, \T5, \T1, \T3
2247 vpclmulqdq $0x01, \T5, \T1, \T3
2250 vpclmulqdq $0x10, \T5, \T1, \T3
2266 vmovdqa TMP8(%rsp), \T1
2269 vpclmulqdq $0x00, \T5, \T1, \T3
2272 vpclmulqdq $0x01, \T5, \T1, \T3
2275 vpclmulqdq $0x10, \T5, \T1, \T3
2278 vpclmulqdq $0x11, \T5, \T1, \T3
2279 vpxor \T3, \T4, \T1
2306 vpxor \T6, \T1, \T1 # accumulate the results in T1:T7
2340 vpxor \T4, \T1, \T1 # the result is in T1
2352 vpxor \T1, \XMM1, \XMM1
2360 .macro GHASH_LAST_8_AVX2 T1 T2 T3 T4 T5 T6 T7 XMM1 XMM2 XMM3 XMM4 XMM5 XMM6 XMM7 XMM8