1 // Copyright 2019 Google LLC
2 //
3 // This source code is licensed under the BSD-style license found in the
4 // LICENSE file in the root directory of this source tree.
5
6 #include <assert.h>
7
8 #include <immintrin.h>
9
10 #include <xnnpack/common.h>
11 #include <xnnpack/intrinsics-polyfill.h>
12 #include <xnnpack/vscale.h>
13
14
xnn_f32_vscale_ukernel__avx512f_x64(size_t n,const float * x,float * y,float c)15 void xnn_f32_vscale_ukernel__avx512f_x64(
16 size_t n,
17 const float* x,
18 float* y,
19 float c)
20 {
21 assert(n != 0);
22 assert(n % sizeof(float) == 0);
23
24 __m512 vc = _mm512_set1_ps(c);
25 for (; n >= 64 * sizeof(float); n -= 64 * sizeof(float)) {
26 const __m512 vx0 = _mm512_loadu_ps(x);
27 const __m512 vx1 = _mm512_loadu_ps(x + 16);
28 const __m512 vx2 = _mm512_loadu_ps(x + 32);
29 const __m512 vx3 = _mm512_loadu_ps(x + 48);
30 x += 64;
31
32 const __m512 vy0 = _mm512_mul_ps(vx0, vc);
33 const __m512 vy1 = _mm512_mul_ps(vx1, vc);
34 const __m512 vy2 = _mm512_mul_ps(vx2, vc);
35 const __m512 vy3 = _mm512_mul_ps(vx3, vc);
36
37 _mm512_storeu_ps(y, vy0);
38 _mm512_storeu_ps(y + 16, vy1);
39 _mm512_storeu_ps(y + 32, vy2);
40 _mm512_storeu_ps(y + 48, vy3);
41 y += 64;
42 }
43 for (; n >= 16 * sizeof(float); n -= 16 * sizeof(float)) {
44 const __m512 vx = _mm512_loadu_ps(x);
45 x += 16;
46
47 const __m512 vy = _mm512_mul_ps(vx, vc);
48
49 _mm512_storeu_ps(y, vy);
50 y += 16;
51 }
52 if XNN_UNLIKELY(n != 0) {
53 n >>= 2 /* log2(sizeof(float)) */;
54 const __mmask16 vmask = _cvtu32_mask16((uint16_t) ((uint32_t) (UINT32_C(1) << n) - UINT32_C(1)));
55
56 const __m512 vx = _mm512_maskz_loadu_ps(vmask, x);
57 const __m512 vy = _mm512_mul_ps(vx, vc);
58 _mm512_mask_storeu_ps(y, vmask, vy);
59 }
60 }
61