1 // Copyright 2021 Google LLC
2 //
3 // This source code is licensed under the BSD-style license found in the
4 // LICENSE file in the root directory of this source tree.
5
6 #include <assert.h>
7
8 #include <emmintrin.h>
9
10 #include <xnnpack/vunary.h>
11
12
xnn_u8_vclamp_ukernel__sse2_x64(size_t n,const uint8_t * x,uint8_t * y,const union xnn_u8_minmax_params params[restrict XNN_MIN_ELEMENTS (1)])13 void xnn_u8_vclamp_ukernel__sse2_x64(
14 size_t n,
15 const uint8_t* x,
16 uint8_t* y,
17 const union xnn_u8_minmax_params params[restrict XNN_MIN_ELEMENTS(1)]) XNN_OOB_READS
18 {
19 assert(n != 0);
20
21 const __m128i voutput_max = _mm_load_si128((const __m128i*) params->sse2.max);
22 const __m128i voutput_min = _mm_load_si128((const __m128i*) params->sse2.min);
23 for (; n >= 64; n -= 64) {
24 __m128i vacc0 = _mm_loadu_si128((const __m128i*) x);
25 __m128i vacc1 = _mm_loadu_si128((const __m128i*) x + 1);
26 __m128i vacc2 = _mm_loadu_si128((const __m128i*) x + 2);
27 __m128i vacc3 = _mm_loadu_si128((const __m128i*) x + 3);
28 x += 64;
29
30 vacc0 = _mm_max_epu8(vacc0, voutput_min);
31 vacc1 = _mm_max_epu8(vacc1, voutput_min);
32 vacc2 = _mm_max_epu8(vacc2, voutput_min);
33 vacc3 = _mm_max_epu8(vacc3, voutput_min);
34
35 vacc0 = _mm_min_epu8(vacc0, voutput_max);
36 vacc1 = _mm_min_epu8(vacc1, voutput_max);
37 vacc2 = _mm_min_epu8(vacc2, voutput_max);
38 vacc3 = _mm_min_epu8(vacc3, voutput_max);
39
40 _mm_storeu_si128((__m128i*) y, vacc0);
41 _mm_storeu_si128((__m128i*) y + 1, vacc1);
42 _mm_storeu_si128((__m128i*) y + 2, vacc2);
43 _mm_storeu_si128((__m128i*) y + 3, vacc3);
44 y += 64;
45 }
46 for (; n >= 16; n -= 16) {
47 __m128i vacc = _mm_loadu_si128((const __m128i*) x);
48 x += 16;
49
50 vacc = _mm_min_epu8(vacc, voutput_max);
51 vacc = _mm_max_epu8(vacc, voutput_min);
52
53 _mm_storeu_si128((__m128i*) y, vacc);
54 y += 16;
55 }
56 if XNN_UNLIKELY(n != 0) {
57 __m128i vacc = _mm_loadu_si128((const __m128i*) x);
58
59 vacc = _mm_min_epu8(vacc, voutput_max);
60 vacc = _mm_max_epu8(vacc, voutput_min);
61
62 if (n & 8) {
63 _mm_storel_epi64((__m128i*) y, vacc);
64 y += 8;
65 vacc = _mm_unpackhi_epi64(vacc, vacc);
66 }
67 if (n & 4) {
68 *((uint32_t*) y) = (uint32_t) _mm_cvtsi128_si32(vacc);
69 y += 4;
70 vacc = _mm_srli_epi64(vacc, 32);
71 }
72 if (n & 2) {
73 *((uint16_t*) y) = (uint16_t) _mm_cvtsi128_si32(vacc);
74 y += 2;
75 vacc = _mm_srli_epi32(vacc, 16);
76 }
77 if (n & 1) {
78 *y = (uint8_t) _mm_cvtsi128_si32(vacc);
79 }
80 }
81 }
82