1 // Copyright 2019 Google LLC
2 //
3 // This source code is licensed under the BSD-style license found in the
4 // LICENSE file in the root directory of this source tree.
5
6 #include <assert.h>
7
8 #include <immintrin.h>
9
10 #include <xnnpack/intrinsics-polyfill.h>
11 #include <xnnpack/rmax.h>
12
13
xnn_f32_rmax_ukernel__avx512f(size_t n,const float * x,float * y)14 void xnn_f32_rmax_ukernel__avx512f(
15 size_t n,
16 const float* x,
17 float* y)
18 {
19 assert(n != 0);
20 assert(n % sizeof(float) == 0);
21
22 __m512 vmax0 = _mm512_broadcastss_ps(_mm_load_ss(x));
23 __m512 vmax1 = vmax0;
24 __m512 vmax2 = vmax0;
25 __m512 vmax3 = vmax0;
26 for (; n >= 256; n -= 256) {
27 const __m512 vx0 = _mm512_loadu_ps(x);
28 const __m512 vx1 = _mm512_loadu_ps(x + 16);
29 const __m512 vx2 = _mm512_loadu_ps(x + 32);
30 const __m512 vx3 = _mm512_loadu_ps(x + 48);
31 x += 64;
32
33 vmax0 = _mm512_max_ps(vmax0, vx0);
34 vmax1 = _mm512_max_ps(vmax1, vx1);
35 vmax2 = _mm512_max_ps(vmax2, vx2);
36 vmax3 = _mm512_max_ps(vmax3, vx3);
37 }
38 __m512 vmax = _mm512_max_ps(_mm512_max_ps(vmax0, vmax1), _mm512_max_ps(vmax2, vmax3));
39 for (; n >= 64; n -= 64) {
40 const __m512 vx = _mm512_loadu_ps(x);
41 vmax = _mm512_max_ps(vmax, vx);
42 x += 16;
43 }
44 __m256 vmax_lo = _mm256_max_ps(_mm512_castps512_ps256(vmax), _mm512_castps512_ps256(_mm512_shuffle_f32x4(vmax, vmax, _MM_SHUFFLE(3, 2, 3, 2))));
45 __m128 vmax_ll = _mm_max_ps(_mm256_castps256_ps128(vmax_lo), _mm256_extractf128_ps(vmax_lo, 1));
46 for (; n >= 16; n -= 16) {
47 const __m128 vx = _mm_loadu_ps(x);
48 vmax_ll = _mm_max_ps(vmax_ll, vx);
49 x += 4;
50 }
51 vmax_ll = _mm_max_ps(vmax_ll, _mm_movehl_ps(vmax_ll, vmax_ll));
52 vmax_ll = _mm_max_ss(vmax_ll, _mm_shuffle_ps(vmax_ll, vmax_ll, _MM_SHUFFLE(3, 3, 1, 1)));
53 if XNN_UNLIKELY(n != 0) {
54 do {
55 vmax_ll = _mm_max_ss(vmax_ll, _mm_load_ss(x));
56 x += 1;
57 n -= 4;
58 } while (n != 0);
59 }
60 _mm_store_ss(y, vmax_ll);
61 }
62