1 // Copyright 2019 Google LLC
2 //
3 // This source code is licensed under the BSD-style license found in the
4 // LICENSE file in the root directory of this source tree.
5
6 #include <assert.h>
7
8 #include <xnnpack/scalar-utils.h>
9 #include <xnnpack/gavgpool.h>
10
11
xnn_q8_gavgpool_ukernel_mp7p7q__scalar(size_t m,size_t n,const uint8_t * input,size_t input_stride,const uint8_t * zero,int32_t * buffer,uint8_t * output,const union xnn_q8_avgpool_params params[restrict static1])12 void xnn_q8_gavgpool_ukernel_mp7p7q__scalar(
13 size_t m,
14 size_t n,
15 const uint8_t* input,
16 size_t input_stride,
17 const uint8_t* zero,
18 int32_t* buffer,
19 uint8_t* output,
20 const union xnn_q8_avgpool_params params[restrict static 1])
21 {
22 assert(m > 7);
23 assert(n != 0);
24
25 const uint8_t* i0 = input;
26 const uint8_t* i1 = (const uint8_t*) ((uintptr_t) i0 + input_stride);
27 const uint8_t* i2 = (const uint8_t*) ((uintptr_t) i1 + input_stride);
28 const uint8_t* i3 = (const uint8_t*) ((uintptr_t) i2 + input_stride);
29 const uint8_t* i4 = (const uint8_t*) ((uintptr_t) i3 + input_stride);
30 const uint8_t* i5 = (const uint8_t*) ((uintptr_t) i4 + input_stride);
31 const uint8_t* i6 = (const uint8_t*) ((uintptr_t) i5 + input_stride);
32 const size_t input_increment = 7 * input_stride - n;
33
34 // First pass.
35 {
36 const int32_t vbias = params->scalar.bias;
37
38 int32_t* b = buffer;
39 size_t k = n;
40 do {
41 const uint32_t vi0 = (uint32_t) *i0++;
42 const uint32_t vi1 = (uint32_t) *i1++;
43 const uint32_t vi2 = (uint32_t) *i2++;
44 const uint32_t vi3 = (uint32_t) *i3++;
45 const uint32_t vi4 = (uint32_t) *i4++;
46 const uint32_t vi5 = (uint32_t) *i5++;
47 const uint32_t vi6 = (uint32_t) *i6++;
48
49 const uint32_t vsum01 = vi0 + vi1;
50 const uint32_t vsum23 = vi2 + vi3;
51 const uint32_t vsum45 = vi4 + vi5;
52
53 const uint32_t vsum016 = vsum01 + vi6;
54 const uint32_t vsum2345 = vsum23 + vsum45;
55
56 const uint32_t vsum = vsum016 + vsum2345;
57 const int32_t vacc = vbias + (int32_t) vsum;
58
59 *b++ = vacc;
60 } while (--k != 0);
61 }
62 // Intermediate passes.
63 for (m -= 7; m > 7; m -= 7) {
64 i0 = (const uint8_t*) ((uintptr_t) i0 + input_increment);
65 i1 = (const uint8_t*) ((uintptr_t) i1 + input_increment);
66 i2 = (const uint8_t*) ((uintptr_t) i2 + input_increment);
67 i3 = (const uint8_t*) ((uintptr_t) i3 + input_increment);
68 i4 = (const uint8_t*) ((uintptr_t) i4 + input_increment);
69 i5 = (const uint8_t*) ((uintptr_t) i5 + input_increment);
70 i6 = (const uint8_t*) ((uintptr_t) i6 + input_increment);
71
72 int32_t* b = buffer;
73 size_t k = n;
74 do {
75 const uint32_t vi0 = (uint32_t) *i0++;
76 const uint32_t vi1 = (uint32_t) *i1++;
77 const uint32_t vi2 = (uint32_t) *i2++;
78 const uint32_t vi3 = (uint32_t) *i3++;
79 const uint32_t vi4 = (uint32_t) *i4++;
80 const uint32_t vi5 = (uint32_t) *i5++;
81 const uint32_t vi6 = (uint32_t) *i6++;
82
83 const uint32_t vsum01 = vi0 + vi1;
84 const uint32_t vsum23 = vi2 + vi3;
85 const uint32_t vsum45 = vi4 + vi5;
86
87 const uint32_t vsum016 = vsum01 + vi6;
88 const uint32_t vsum2345 = vsum23 + vsum45;
89
90 const uint32_t vsum = vsum016 + vsum2345;
91
92 *b++ += (int32_t) vsum;
93 } while (--k != 0);
94 }
95
96 // Last pass.
97 {
98 const int32_t vmultiplier = params->scalar.multiplier;
99 const int64_t vrounding = params->scalar.rounding;
100 const uint32_t vshift = params->scalar.right_shift;
101 const int32_t voutput_min = params->scalar.output_min_less_zero_point;
102 const int32_t voutput_max = params->scalar.output_max_less_zero_point;
103 const int32_t voutput_zero_point = params->scalar.output_zero_point;
104
105 i0 = (const uint8_t*) ((uintptr_t) i0 + input_increment);
106 i1 = (const uint8_t*) ((uintptr_t) i1 + input_increment);
107 if (m < 2) {
108 i1 = zero;
109 }
110 i2 = (const uint8_t*) ((uintptr_t) i2 + input_increment);
111 if (m <= 2) {
112 i2 = zero;
113 }
114 i3 = (const uint8_t*) ((uintptr_t) i3 + input_increment);
115 if (m < 4) {
116 i3 = zero;
117 }
118 i4 = (const uint8_t*) ((uintptr_t) i4 + input_increment);
119 if (m <= 4) {
120 i4 = zero;
121 }
122 i5 = (const uint8_t*) ((uintptr_t) i5 + input_increment);
123 if (m < 6) {
124 i5 = zero;
125 }
126 i6 = (const uint8_t*) ((uintptr_t) i6 + input_increment);
127 if (m <= 6) {
128 i6 = zero;
129 }
130
131 int32_t* b = buffer;
132 size_t k = n;
133 do {
134 int32_t vacc = *b++;
135 const uint32_t vi0 = (uint32_t) *i0++;
136 const uint32_t vi1 = (uint32_t) *i1++;
137 const uint32_t vi2 = (uint32_t) *i2++;
138 const uint32_t vi3 = (uint32_t) *i3++;
139 const uint32_t vi4 = (uint32_t) *i4++;
140 const uint32_t vi5 = (uint32_t) *i5++;
141 const uint32_t vi6 = (uint32_t) *i6++;
142
143 const uint32_t vsum01 = vi0 + vi1;
144 const uint32_t vsum23 = vi2 + vi3;
145 const uint32_t vsum45 = vi4 + vi5;
146
147 const uint32_t vsum016 = vsum01 + vi6;
148 const uint32_t vsum2345 = vsum23 + vsum45;
149
150 const uint32_t vsum = vsum016 + vsum2345;
151 vacc += (int32_t) vsum;
152
153 const int64_t vproduct = (int64_t) vacc * (int64_t) vmultiplier;
154 const int64_t vadjusted_product = vproduct - (int64_t) (vacc < 0);
155 int32_t vout = (int32_t) asr_s64(vadjusted_product + vrounding, vshift);
156 vout = vout < voutput_min ? voutput_min : vout;
157 vout = vout > voutput_max ? voutput_max : vout;
158 vout += voutput_zero_point;
159
160 *output++ = (uint8_t) vout;
161 } while (--k != 0);
162 }
163 }
164