1 // Copyright 2020 Google LLC
2 //
3 // This source code is licensed under the BSD-style license found in the
4 // LICENSE file in the root directory of this source tree.
5
6 #include <math.h>
7 #include <stddef.h>
8 #include <stdint.h>
9
10 #include <xnnpack.h>
11 #include <xnnpack/log.h>
12 #include <xnnpack/params.h>
13 #include <xnnpack/subgraph.h>
14
15
create_max_pooling_operator(const struct xnn_node * node,const struct xnn_value * values,size_t num_values,struct xnn_operator_data * opdata)16 static enum xnn_status create_max_pooling_operator(
17 const struct xnn_node* node,
18 const struct xnn_value* values,
19 size_t num_values,
20 struct xnn_operator_data* opdata)
21 {
22 assert(node->num_inputs == 1);
23 const uint32_t input_id = node->inputs[0];
24 assert(input_id != XNN_INVALID_VALUE_ID);
25 assert(input_id < num_values);
26
27 assert(node->num_outputs == 1);
28 const uint32_t output_id = node->outputs[0];
29 assert(output_id != XNN_INVALID_VALUE_ID);
30 assert(output_id < num_values);
31
32 const size_t channel_dim = values[input_id].shape.dim[3];
33 assert(channel_dim == values[output_id].shape.dim[3]);
34
35 enum xnn_status status;
36 switch (node->compute_type) {
37 #ifndef XNN_NO_F16_OPERATORS
38 case xnn_compute_type_fp16:
39 status = xnn_create_max_pooling2d_nhwc_f16(
40 node->params.pooling_2d.padding_top,
41 node->params.pooling_2d.padding_right,
42 node->params.pooling_2d.padding_bottom,
43 node->params.pooling_2d.padding_left,
44 node->params.pooling_2d.pooling_height,
45 node->params.pooling_2d.pooling_width,
46 node->params.pooling_2d.stride_height,
47 node->params.pooling_2d.stride_width,
48 node->params.pooling_2d.dilation_height,
49 node->params.pooling_2d.dilation_width,
50 channel_dim /* channels */, channel_dim /* input stride */, channel_dim /* output stride */,
51 node->activation.output_min,
52 node->activation.output_max,
53 node->flags,
54 &opdata->operator_object);
55 break;
56 #endif // !defined(XNN_NO_F16_OPERATORS)
57 case xnn_compute_type_fp32:
58 status = xnn_create_max_pooling2d_nhwc_f32(
59 node->params.pooling_2d.padding_top,
60 node->params.pooling_2d.padding_right,
61 node->params.pooling_2d.padding_bottom,
62 node->params.pooling_2d.padding_left,
63 node->params.pooling_2d.pooling_height,
64 node->params.pooling_2d.pooling_width,
65 node->params.pooling_2d.stride_height,
66 node->params.pooling_2d.stride_width,
67 node->params.pooling_2d.dilation_height,
68 node->params.pooling_2d.dilation_width,
69 channel_dim /* channels */, channel_dim /* input stride */, channel_dim /* output stride */,
70 node->activation.output_min,
71 node->activation.output_max,
72 node->flags,
73 &opdata->operator_object);
74 break;
75 #ifndef XNN_NO_S8_OPERATORS
76 case xnn_compute_type_qs8:
77 {
78 const float output_scale = values[output_id].quantization.scale;
79 const int32_t output_zero_point = values[output_id].quantization.zero_point;
80 const int8_t output_min =
81 (int8_t) lrintf(fminf(fmaxf(node->activation.output_min / output_scale + (float) output_zero_point, -128.0f), 127.0f));
82 const int8_t output_max =
83 (int8_t) lrintf(fminf(fmaxf(node->activation.output_max / output_scale + (float) output_zero_point, -128.0f), 127.0f));
84 status = xnn_create_max_pooling2d_nhwc_s8(
85 node->params.pooling_2d.padding_top,
86 node->params.pooling_2d.padding_right,
87 node->params.pooling_2d.padding_bottom,
88 node->params.pooling_2d.padding_left,
89 node->params.pooling_2d.pooling_height,
90 node->params.pooling_2d.pooling_width,
91 node->params.pooling_2d.stride_height,
92 node->params.pooling_2d.stride_width,
93 node->params.pooling_2d.dilation_height,
94 node->params.pooling_2d.dilation_width,
95 channel_dim /* channels */, channel_dim /* input stride */, channel_dim /* output stride */,
96 output_min,
97 output_max,
98 node->flags,
99 &opdata->operator_object);
100 break;
101 }
102 #endif // !defined(XNN_NO_S8_OPERATORS)
103 #ifndef XNN_NO_U8_OPERATORS
104 case xnn_compute_type_qu8:
105 {
106 const float output_scale = values[output_id].quantization.scale;
107 const int32_t output_zero_point = values[output_id].quantization.zero_point;
108 const uint8_t output_min =
109 (uint8_t) lrintf(fminf(fmaxf(node->activation.output_min / output_scale + (float) output_zero_point, 0.0f), 255.0f));
110 const uint8_t output_max =
111 (uint8_t) lrintf(fminf(fmaxf(node->activation.output_max / output_scale + (float) output_zero_point, 0.0f), 255.0f));
112 status = xnn_create_max_pooling2d_nhwc_u8(
113 node->params.pooling_2d.padding_top,
114 node->params.pooling_2d.padding_right,
115 node->params.pooling_2d.padding_bottom,
116 node->params.pooling_2d.padding_left,
117 node->params.pooling_2d.pooling_height,
118 node->params.pooling_2d.pooling_width,
119 node->params.pooling_2d.stride_height,
120 node->params.pooling_2d.stride_width,
121 node->params.pooling_2d.dilation_height,
122 node->params.pooling_2d.dilation_width,
123 channel_dim /* channels */, channel_dim /* input stride */, channel_dim /* output stride */,
124 output_min,
125 output_max,
126 node->flags,
127 &opdata->operator_object);
128 break;
129 }
130 #endif // !defined(XNN_NO_U8_OPERATORS)
131 default:
132 XNN_UNREACHABLE;
133 }
134 if (status == xnn_status_success) {
135 opdata->batch_size = values[input_id].shape.dim[0];
136 opdata->input_height = values[input_id].shape.dim[1];
137 opdata->input_width = values[input_id].shape.dim[2];
138 opdata->inputs[0] = input_id;
139 opdata->outputs[0] = output_id;
140 }
141 return status;
142 }
143
setup_max_pooling_operator(const struct xnn_operator_data * opdata,const struct xnn_blob * blobs,size_t num_blobs,pthreadpool_t threadpool)144 static enum xnn_status setup_max_pooling_operator(
145 const struct xnn_operator_data* opdata,
146 const struct xnn_blob* blobs,
147 size_t num_blobs,
148 pthreadpool_t threadpool)
149 {
150 const uint32_t input_id = opdata->inputs[0];
151 assert(input_id != XNN_INVALID_VALUE_ID);
152 assert(input_id < num_blobs);
153
154 const uint32_t output_id = opdata->outputs[0];
155 assert(output_id != XNN_INVALID_VALUE_ID);
156 assert(output_id < num_blobs);
157
158 const struct xnn_blob* input_blob = blobs + input_id;
159 const void* input_data = input_blob->data;
160 assert(input_data != NULL);
161
162 const struct xnn_blob* output_blob = blobs + output_id;
163 void* output_data = output_blob->data;
164 assert(output_data != NULL);
165
166 switch (opdata->operator_object->type) {
167 #ifndef XNN_NO_F16_OPERATORS
168 case xnn_operator_type_max_pooling_nhwc_f16:
169 return xnn_setup_max_pooling2d_nhwc_f16(
170 opdata->operator_object,
171 opdata->batch_size,
172 opdata->input_height,
173 opdata->input_width,
174 input_data,
175 output_data,
176 threadpool);
177 #endif // !defined(XNN_NO_F16_OPERATORS)
178 case xnn_operator_type_max_pooling_nhwc_f32:
179 return xnn_setup_max_pooling2d_nhwc_f32(
180 opdata->operator_object,
181 opdata->batch_size,
182 opdata->input_height,
183 opdata->input_width,
184 input_data,
185 output_data,
186 threadpool);
187 #ifndef XNN_NO_S8_OPERATORS
188 case xnn_operator_type_max_pooling_nhwc_s8:
189 return xnn_setup_max_pooling2d_nhwc_s8(
190 opdata->operator_object,
191 opdata->batch_size,
192 opdata->input_height,
193 opdata->input_width,
194 input_data,
195 output_data,
196 threadpool);
197 #endif // !defined(XNN_NO_S8_OPERATORS)
198 #ifndef XNN_NO_U8_OPERATORS
199 case xnn_operator_type_max_pooling_nhwc_u8:
200 return xnn_setup_max_pooling2d_nhwc_u8(
201 opdata->operator_object,
202 opdata->batch_size,
203 opdata->input_height,
204 opdata->input_width,
205 input_data,
206 output_data,
207 threadpool);
208 #endif // !defined(XNN_NO_U8_OPERATORS)
209 default:
210 XNN_UNREACHABLE;
211 }
212 }
213
xnn_define_max_pooling_2d(xnn_subgraph_t subgraph,uint32_t input_padding_top,uint32_t input_padding_right,uint32_t input_padding_bottom,uint32_t input_padding_left,uint32_t pooling_height,uint32_t pooling_width,uint32_t stride_height,uint32_t stride_width,uint32_t dilation_height,uint32_t dilation_width,float output_min,float output_max,uint32_t input_id,uint32_t output_id,uint32_t flags)214 enum xnn_status xnn_define_max_pooling_2d(
215 xnn_subgraph_t subgraph,
216 uint32_t input_padding_top,
217 uint32_t input_padding_right,
218 uint32_t input_padding_bottom,
219 uint32_t input_padding_left,
220 uint32_t pooling_height,
221 uint32_t pooling_width,
222 uint32_t stride_height,
223 uint32_t stride_width,
224 uint32_t dilation_height,
225 uint32_t dilation_width,
226 float output_min,
227 float output_max,
228 uint32_t input_id,
229 uint32_t output_id,
230 uint32_t flags)
231 {
232 if ((xnn_params.init_flags & XNN_INIT_FLAG_XNNPACK) == 0) {
233 xnn_log_error("failed to define %s operator: XNNPACK is not initialized",
234 xnn_node_type_to_string(xnn_node_type_max_pooling_2d));
235 return xnn_status_uninitialized;
236 }
237
238 const uint32_t pooling_size = pooling_height * pooling_width;
239 if (pooling_size == 0) {
240 xnn_log_error(
241 "failed to define %s operator with %" PRIu32 "x%" PRIu32 " pooling size: "
242 "pooling size dimensions must be non-zero",
243 xnn_node_type_to_string(xnn_node_type_max_pooling_2d), pooling_width, pooling_height);
244 return xnn_status_invalid_parameter;
245 }
246
247 if (pooling_size == 1) {
248 xnn_log_error(
249 "failed to define %s operator with 1 pooling element: 1x1 pooling is meaningless",
250 xnn_node_type_to_string(xnn_node_type_max_pooling_2d));
251 return xnn_status_invalid_parameter;
252 }
253
254 if (stride_height == 0 || stride_width == 0) {
255 xnn_log_error(
256 "failed to define %s operator with %" PRIu32 "x%" PRIu32 " stride: stride dimensions must be non-zero",
257 xnn_node_type_to_string(xnn_node_type_max_pooling_2d), stride_width, stride_height);
258 return xnn_status_invalid_parameter;
259 }
260
261 if (dilation_height == 0 || dilation_width == 0) {
262 xnn_log_error(
263 "failed to define %s operator with %" PRIu32 "x%" PRIu32 " dilation: dilation dimensions must be non-zero",
264 xnn_node_type_to_string(xnn_node_type_max_pooling_2d), dilation_width, dilation_height);
265 return xnn_status_invalid_parameter;
266 }
267
268 if (isnan(output_min)) {
269 xnn_log_error(
270 "failed to define %s with NaN output lower bound: lower bound must be non-NaN",
271 xnn_node_type_to_string(xnn_node_type_max_pooling_2d));
272 return xnn_status_invalid_parameter;
273 }
274
275 if (isnan(output_max)) {
276 xnn_log_error(
277 "failed to define %s with NaN output upper bound: upper bound must be non-NaN",
278 xnn_node_type_to_string(xnn_node_type_max_pooling_2d));
279 return xnn_status_invalid_parameter;
280 }
281
282 if (output_min >= output_max) {
283 xnn_log_error(
284 "failed to define %s with [%.7g, %.7g] output range: lower bound must be below upper bound",
285 xnn_node_type_to_string(xnn_node_type_max_pooling_2d), output_min, output_max);
286 return xnn_status_invalid_parameter;
287 }
288
289 const bool any_padding = (input_padding_left | input_padding_top | input_padding_right | input_padding_bottom) != 0;
290 if ((flags & XNN_FLAG_TENSORFLOW_SAME_PADDING) != 0) {
291 if (any_padding) {
292 xnn_log_error(
293 "failed to define %s operator with %" PRIu32 "+%" PRIu32 "x%" PRIu32 "+%" PRIu32" padding: "
294 "TensorFlow SAME padding can't be combined with explicit padding specification",
295 xnn_node_type_to_string(xnn_node_type_max_pooling_2d),
296 input_padding_top, input_padding_left, input_padding_bottom, input_padding_right);
297 return xnn_status_invalid_parameter;
298 }
299 }
300
301 if (input_id >= subgraph->num_values) {
302 xnn_log_error(
303 "failed to define %s operator with input ID #%" PRIu32 ": invalid Value ID",
304 xnn_node_type_to_string(xnn_node_type_max_pooling_2d), input_id);
305 return xnn_status_invalid_parameter;
306 }
307
308 const struct xnn_value* input_value = &subgraph->values[input_id];
309 if (input_value->type != xnn_value_type_dense_tensor) {
310 xnn_log_error(
311 "failed to define %s operator with input ID #%" PRIu32 ": unsupported Value type %d (expected dense tensor)",
312 xnn_node_type_to_string(xnn_node_type_max_pooling_2d), input_id, input_value->type);
313 return xnn_status_invalid_parameter;
314 }
315
316 switch (input_value->datatype) {
317 case xnn_datatype_fp32:
318 #ifndef XNN_NO_S8_OPERATORS
319 case xnn_datatype_qint8:
320 #endif // !defined(XNN_NO_S8_OPERATORS)
321 #ifndef XNN_NO_U8_OPERATORS
322 case xnn_datatype_quint8:
323 #endif // !defined(XNN_NO_U8_OPERATORS)
324 break;
325 default:
326 xnn_log_error(
327 "failed to define %s operator with input ID #%" PRIu32 ": unsupported Value datatype %s (%d)",
328 xnn_node_type_to_string(xnn_node_type_max_pooling_2d), input_id,
329 xnn_datatype_to_string(input_value->datatype), input_value->datatype);
330 return xnn_status_invalid_parameter;
331 }
332
333 if (output_id >= subgraph->num_values) {
334 xnn_log_error(
335 "failed to define %s operator with output ID #%" PRIu32 ": invalid Value ID",
336 xnn_node_type_to_string(xnn_node_type_max_pooling_2d), output_id);
337 return xnn_status_invalid_parameter;
338 }
339
340 const struct xnn_value* output_value = &subgraph->values[output_id];
341 if (output_value->type != xnn_value_type_dense_tensor) {
342 xnn_log_error(
343 "failed to define %s operator with output ID #%" PRIu32 ": unsupported Value type %d (expected dense tensor)",
344 xnn_node_type_to_string(xnn_node_type_max_pooling_2d), output_id, output_value->type);
345 return xnn_status_invalid_parameter;
346 }
347
348 enum xnn_compute_type compute_type = xnn_compute_type_invalid;
349 switch (output_value->datatype) {
350 case xnn_datatype_fp32:
351 compute_type = xnn_compute_type_fp32;
352 break;
353 #ifndef XNN_NO_S8_OPERATORS
354 case xnn_datatype_qint8:
355 compute_type = xnn_compute_type_qs8;
356 break;
357 #endif // !defined(XNN_NO_S8_OPERATORS)
358 #ifndef XNN_NO_U8_OPERATORS
359 case xnn_datatype_quint8:
360 compute_type = xnn_compute_type_qu8;
361 break;
362 #endif // !defined(XNN_NO_U8_OPERATORS)
363 default:
364 xnn_log_error(
365 "failed to define %s operator with output ID #%" PRIu32 ": unsupported Value datatype %s (%d)",
366 xnn_node_type_to_string(xnn_node_type_max_pooling_2d), output_id,
367 xnn_datatype_to_string(output_value->datatype), output_value->datatype);
368 return xnn_status_invalid_parameter;
369 }
370
371 if (input_value->datatype != output_value->datatype) {
372 xnn_log_error(
373 "failed to define %s operator with input ID #%" PRIu32 " and output ID #%" PRIu32
374 ": mismatching datatypes across input (%s) and output (%s)",
375 xnn_node_type_to_string(xnn_node_type_max_pooling_2d), input_id, output_id,
376 xnn_datatype_to_string(input_value->datatype),
377 xnn_datatype_to_string(output_value->datatype));
378 return xnn_status_invalid_parameter;
379 }
380
381 #if !defined(XNN_NO_S8_OPERATORS) || !defined(XNN_NO_U8_OPERATORS)
382 if (output_value->datatype == xnn_datatype_qint8 || output_value->datatype == xnn_datatype_quint8) {
383 if (input_value->quantization.zero_point != output_value->quantization.zero_point) {
384 xnn_log_error(
385 "failed to define %s operator with input ID #%" PRIu32 " and output ID #%" PRIu32
386 ": mismatching zero point quantization parameter across input (%"PRId32") and output (%"PRId32")",
387 xnn_node_type_to_string(xnn_node_type_max_pooling_2d), input_id, output_id,
388 input_value->quantization.zero_point, output_value->quantization.zero_point);
389 return xnn_status_invalid_parameter;
390 }
391 if (input_value->quantization.scale != output_value->quantization.scale) {
392 xnn_log_error(
393 "failed to define %s operator with input ID #%" PRIu32 " and output ID #%" PRIu32
394 ": mismatching zero point quantization parameter across input (%.7g) and output (%.7g)",
395 xnn_node_type_to_string(xnn_node_type_max_pooling_2d), input_id, output_id,
396 input_value->quantization.scale, output_value->quantization.scale);
397 return xnn_status_invalid_parameter;
398 }
399 }
400 #endif // !defined(XNN_NO_S8_OPERATORS) || !defined(XNN_NO_U8_OPERATORS)
401
402 struct xnn_node* node = xnn_subgraph_new_node(subgraph);
403 if (node == NULL) {
404 return xnn_status_out_of_memory;
405 }
406
407 node->type = xnn_node_type_max_pooling_2d;
408 node->compute_type = compute_type;
409 node->params.pooling_2d.padding_top = input_padding_top;
410 node->params.pooling_2d.padding_right = input_padding_right;
411 node->params.pooling_2d.padding_bottom = input_padding_bottom;
412 node->params.pooling_2d.padding_left = input_padding_left;
413 node->params.pooling_2d.pooling_height = pooling_height;
414 node->params.pooling_2d.pooling_width = pooling_width;
415 node->params.pooling_2d.stride_height = stride_height;
416 node->params.pooling_2d.stride_width = stride_width;
417 node->params.pooling_2d.dilation_height = dilation_height;
418 node->params.pooling_2d.dilation_width = dilation_width;
419 node->activation.output_min = output_min;
420 node->activation.output_max = output_max;
421 node->num_inputs = 1;
422 node->inputs[0] = input_id;
423 node->num_outputs = 1;
424 node->outputs[0] = output_id;
425 node->flags = flags;
426
427 node->create = create_max_pooling_operator;
428 node->setup = setup_max_pooling_operator;
429
430 return xnn_status_success;
431 }
432