1 // Copyright 2019 Google LLC
2 //
3 // This source code is licensed under the BSD-style license found in the
4 // LICENSE file in the root directory of this source tree.
5
6 #include <assert.h>
7
8 #include <emmintrin.h>
9
10 #include <xnnpack/pad.h>
11
12
xnn_xx_pad_ukernel__sse2(size_t rows,size_t channels,size_t pre_padding,size_t post_padding,const void * input,size_t input_stride,void * output,size_t output_stride,const uint32_t fill_pattern)13 void xnn_xx_pad_ukernel__sse2(
14 size_t rows,
15 size_t channels,
16 size_t pre_padding,
17 size_t post_padding,
18 const void* input,
19 size_t input_stride,
20 void* output,
21 size_t output_stride,
22 const uint32_t fill_pattern) XNN_OOB_READS
23 {
24 const size_t input_increment = input_stride - channels;
25 const size_t output_increment = output_stride - (pre_padding + channels + post_padding);
26
27 const __m128i vfill_pattern = _mm_shuffle_epi32(_mm_cvtsi32_si128((int) fill_pattern), _MM_SHUFFLE(0, 0, 0, 0));
28 do {
29 // Pre-pad input channels.
30 size_t l = pre_padding;
31 if XNN_LIKELY(l != 0) {
32 for (; l >= 16 * sizeof(uint8_t); l -= 16 * sizeof(uint8_t)) {
33 _mm_storeu_si128((__m128i*) output, vfill_pattern);
34 output = (uint8_t*) output + 16;
35 }
36 if (l & (8 * sizeof(uint8_t))) {
37 _mm_storel_epi64((__m128i*) output, vfill_pattern);
38 output = (uint8_t*) output + 8;
39 }
40 uint32_t vfill_subpattern = fill_pattern;
41 if (l & (4 * sizeof(uint8_t))) {
42 *((uint32_t*) output) = vfill_subpattern;
43 output = (uint8_t*) output + 4;
44 }
45 if (l & (2 * sizeof(uint8_t))) {
46 *((uint16_t*) output) = (uint16_t) vfill_subpattern;
47 vfill_subpattern >>= 16;
48 output = (uint8_t*) output + 2;
49 }
50 if (l & (1 * sizeof(uint8_t))) {
51 *((uint8_t*) output) = (uint8_t) vfill_subpattern;
52 output = (uint8_t*) output + 1;
53 }
54 }
55
56 // Copy input channels.
57 size_t c = channels;
58 for (; c >= 16 * sizeof(uint8_t); c -= 16 * sizeof(uint8_t)) {
59 const __m128i vdata = _mm_loadu_si128((const __m128i*) input);
60 input = (const uint8_t*) input + 16;
61
62 _mm_storeu_si128((__m128i*) output, vdata);
63 output = (uint8_t*) output + 16;
64 }
65 if XNN_UNLIKELY(c != 0) {
66 __m128i vdata = _mm_loadu_si128((const __m128i*) input);
67 input = (const void*) ((uintptr_t) input + c);
68 if (c & (8 * sizeof(uint8_t))) {
69 _mm_storel_epi64((__m128i*) output, vdata);
70 vdata = _mm_unpackhi_epi64(vdata, vdata);
71 output = (uint8_t*) output + 8;
72 }
73 if (c & (4 * sizeof(uint8_t))) {
74 *((uint32_t*) output) = (uint32_t) _mm_cvtsi128_si32(vdata);
75 vdata = _mm_srli_epi64(vdata, 32);
76 output = (uint8_t*) output + 4;
77 }
78 uint32_t vsubdata = (uint32_t) _mm_cvtsi128_si32(vdata);
79 if (c & (2 * sizeof(uint8_t))) {
80 *((uint16_t*) output) = (uint16_t) vsubdata;
81 vsubdata >>= 16;
82 output = (uint8_t*) output + 2;
83 }
84 if (c & (1 * sizeof(uint8_t))) {
85 *((uint8_t*) output) = (uint8_t) vsubdata;
86 output = (uint8_t*) output + 1;
87 }
88 }
89
90 // Post-pad input channels.
91 size_t r = post_padding;
92 if XNN_LIKELY(r != 0) {
93 for (; r >= 16 * sizeof(uint8_t); r -= 16 * sizeof(uint8_t)) {
94 _mm_storeu_si128((__m128i*) output, vfill_pattern);
95 output = (uint8_t*) output + 16;
96 }
97 if (r & (8 * sizeof(uint8_t))) {
98 _mm_storel_epi64((__m128i*) output, vfill_pattern);
99 output = (uint8_t*) output + 8;
100 }
101 uint32_t vfill_subpattern = fill_pattern;
102 if (r & (4 * sizeof(uint8_t))) {
103 *((uint32_t*) output) = vfill_subpattern;
104 output = (uint8_t*) output + 4;
105 }
106 if (r & (2 * sizeof(uint8_t))) {
107 *((uint16_t*) output) = (uint16_t) vfill_subpattern;
108 vfill_subpattern >>= 16;
109 output = (uint8_t*) output + 2;
110 }
111 if (r & (1 * sizeof(uint8_t))) {
112 *((uint8_t*) output) = (uint8_t) vfill_subpattern;
113 output = (uint8_t*) output + 1;
114 }
115 }
116
117 input = (const void*) ((uintptr_t) input + input_increment);
118 output = (void*) ((uintptr_t) output + output_increment);
119 } while (--rows != 0);
120 }
121