1 // Copyright 2020 Google LLC
2 //
3 // This source code is licensed under the BSD-style license found in the
4 // LICENSE file in the root directory of this source tree.
5
6 #include <assert.h>
7 #include <math.h>
8 #include <stdbool.h>
9 #include <stddef.h>
10 #include <stdint.h>
11 #include <stdlib.h>
12 #include <string.h>
13
14 #include <xnnpack.h>
15 #include <xnnpack/allocator.h>
16 #include <xnnpack/operator.h>
17 #include <xnnpack/log.h>
18 #include <xnnpack/common.h>
19 #include <xnnpack/math.h>
20 #include <xnnpack/params.h>
21 #include <xnnpack/indirection.h>
22
xnn_create_resize_bilinear2d_nchw_f32(size_t channels,size_t input_pixel_stride,size_t output_pixel_stride,uint32_t flags,xnn_operator_t * resize_op_out)23 enum xnn_status xnn_create_resize_bilinear2d_nchw_f32(
24 size_t channels,
25 size_t input_pixel_stride,
26 size_t output_pixel_stride,
27 uint32_t flags,
28 xnn_operator_t* resize_op_out)
29 {
30 xnn_operator_t resize_op = NULL;
31 enum xnn_status status = xnn_status_uninitialized;
32
33 if ((xnn_params.init_flags & XNN_INIT_FLAG_XNNPACK) == 0) {
34 xnn_log_error("failed to create %s operator: XNNPACK is not initialized",
35 xnn_operator_type_to_string(xnn_operator_type_resize_bilinear_nchw_f32));
36 goto error;
37 }
38
39 status = xnn_status_invalid_parameter;
40
41 if (channels == 0) {
42 xnn_log_error(
43 "failed to create %s operator with %zu channels: number of channels must be non-zero",
44 xnn_operator_type_to_string(xnn_operator_type_resize_bilinear_nchw_f32), channels);
45 goto error;
46 }
47
48 if (input_pixel_stride < channels) {
49 xnn_log_error(
50 "failed to create %s operator with input pixel stride of %zu: "
51 "stride must be at least as large as the number of channels (%zu)",
52 xnn_operator_type_to_string(xnn_operator_type_resize_bilinear_nchw_f32), input_pixel_stride, channels);
53 goto error;
54 }
55
56 if (output_pixel_stride < channels) {
57 xnn_log_error(
58 "failed to create %s operator with output pixel stride of %zu: "
59 "stride must be at least as large as the number of channels (%zu)",
60 xnn_operator_type_to_string(xnn_operator_type_resize_bilinear_nchw_f32), output_pixel_stride, channels);
61 goto error;
62 }
63
64 status = xnn_status_out_of_memory;
65
66 resize_op = xnn_allocate_zero_simd_memory(sizeof(struct xnn_operator));
67 if (resize_op == NULL) {
68 xnn_log_error(
69 "failed to allocate %zu bytes for %s operator descriptor",
70 sizeof(struct xnn_operator), xnn_operator_type_to_string(xnn_operator_type_resize_bilinear_nchw_f32));
71 goto error;
72 }
73
74 resize_op->channels = channels;
75 resize_op->input_pixel_stride = input_pixel_stride;
76 resize_op->output_pixel_stride = output_pixel_stride;
77
78 resize_op->type = xnn_operator_type_resize_bilinear_nchw_f32;
79 resize_op->flags = flags;
80
81 resize_op->state = xnn_run_state_invalid;
82
83 *resize_op_out = resize_op;
84 return xnn_status_success;
85
86 error:
87 xnn_delete_operator(resize_op);
88 return status;
89 }
90
xnn_setup_resize_bilinear2d_nchw_f32(xnn_operator_t resize_op,size_t batch_size,size_t input_height,size_t input_width,size_t output_height,size_t output_width,const float * input,float * output,pthreadpool_t threadpool)91 enum xnn_status xnn_setup_resize_bilinear2d_nchw_f32(
92 xnn_operator_t resize_op,
93 size_t batch_size,
94 size_t input_height,
95 size_t input_width,
96 size_t output_height,
97 size_t output_width,
98 const float* input,
99 float* output,
100 pthreadpool_t threadpool)
101 {
102 if (resize_op->type != xnn_operator_type_resize_bilinear_nchw_f32) {
103 xnn_log_error("failed to setup operator: operator type mismatch (expected %s, got %s)",
104 xnn_operator_type_to_string(xnn_operator_type_resize_bilinear_nchw_f32),
105 xnn_operator_type_to_string(resize_op->type));
106 return xnn_status_invalid_parameter;
107 }
108 resize_op->state = xnn_run_state_invalid;
109
110 if ((xnn_params.init_flags & XNN_INIT_FLAG_XNNPACK) == 0) {
111 xnn_log_error("failed to setup %s operator: XNNPACK is not initialized",
112 xnn_operator_type_to_string(xnn_operator_type_resize_bilinear_nchw_f32));
113 return xnn_status_uninitialized;
114 }
115
116 if (input_width <= 1 || input_height <= 1) {
117 xnn_log_error(
118 "failed to setup %s operator with %zux%zu input: input dimensions must be greater than 1",
119 xnn_operator_type_to_string(xnn_operator_type_resize_bilinear_nchw_f32), input_width, input_height);
120 return xnn_status_invalid_parameter;
121 }
122
123 if (max(input_width, input_height) >= 16777216) {
124 xnn_log_error(
125 "failed to setup %s operator with %zux%zu input: input dimensions must be below 2**24",
126 xnn_operator_type_to_string(xnn_operator_type_resize_bilinear_nchw_f32), input_width, input_height);
127 return xnn_status_unsupported_parameter;
128 }
129
130 if (output_width == 0 || output_height == 0) {
131 xnn_log_error(
132 "failed to setup %s operator with %zux%zu output: output dimensions must be non-zero",
133 xnn_operator_type_to_string(xnn_operator_type_resize_bilinear_nchw_f32), output_width, output_height);
134 return xnn_status_invalid_parameter;
135 }
136
137 if (max(output_width, output_height) >= 16777216) {
138 xnn_log_error(
139 "failed to setup %s operator with %zux%zu output: output dimensions must be below 2**24",
140 xnn_operator_type_to_string(xnn_operator_type_resize_bilinear_nchw_f32), output_width, output_height);
141 return xnn_status_unsupported_parameter;
142 }
143
144 if (batch_size == 0) {
145 resize_op->state = xnn_run_state_skip;
146 return xnn_status_success;
147 }
148
149 if (output_height * output_width != resize_op->last_output_height * resize_op->last_output_width) {
150 const size_t indirection_buffer_size = sizeof(void*) * (output_height * output_width * 4);
151 const size_t packed_weights_size = sizeof(float) * (output_height * output_width * 2);
152
153 const void** indirection_buffer = (const void**) xnn_reallocate_memory(resize_op->indirection_buffer, indirection_buffer_size);
154 if (indirection_buffer == NULL) {
155 xnn_log_error(
156 "failed to allocate %zu bytes for %s operator indirection buffer",
157 indirection_buffer_size, xnn_operator_type_to_string(xnn_operator_type_resize_bilinear_nchw_f32));
158 return xnn_status_out_of_memory;
159 }
160 resize_op->indirection_buffer = indirection_buffer;
161
162 // Note: packed weights must be SIMD-aligned, so we can't use xnn_reallocate_memory
163 xnn_release_simd_memory(resize_op->packed_weights);
164 resize_op->packed_weights = xnn_allocate_simd_memory(packed_weights_size);
165 if (resize_op->packed_weights == NULL) {
166 xnn_log_error(
167 "failed to allocate %zu bytes for %s operator packed weights",
168 packed_weights_size, xnn_operator_type_to_string(xnn_operator_type_resize_bilinear_nchw_f32));
169 return xnn_status_out_of_memory;
170 }
171 }
172
173 const size_t input_pixel_stride_in_bytes = sizeof(float); // Since the layout in CHW the pixels
174 if (input_height != resize_op->last_input_height ||
175 input_width != resize_op->last_input_width ||
176 output_height != resize_op->last_output_height ||
177 output_width != resize_op->last_output_width)
178 {
179 const uint32_t flags = resize_op->flags;
180 xnn_indirection_init_resize_bilinear2d_chw_f32(
181 input_pixel_stride_in_bytes,
182 input_height, input_width,
183 output_height, output_width,
184 input, resize_op->indirection_buffer, resize_op->packed_weights,
185 !!(flags & XNN_FLAG_ALIGN_CORNERS),
186 !!(flags & XNN_FLAG_TENSORFLOW_LEGACY_MODE));
187
188 resize_op->last_input = input;
189 resize_op->last_input_height = input_height;
190 resize_op->last_input_width = input_width;
191 resize_op->last_output_height = output_height;
192 resize_op->last_output_width = output_width;
193 }
194
195 resize_op->context.resize_bilinear_chw = (struct resize_bilinear_chw_context) {
196 .output_pixels = output_height * output_width,
197 .channels = resize_op->channels,
198 .input_channel_stride = input_height * input_width * sizeof(float),
199 .indirect_input = resize_op->indirection_buffer,
200 .input_offset = (size_t) ((uintptr_t) input - (uintptr_t) resize_op->last_input),
201 .input_batch_stride = resize_op->input_pixel_stride * input_height * input_width * sizeof(float),
202 .packed_weights = resize_op->packed_weights,
203 .output = output,
204 .output_batch_stride = resize_op->output_pixel_stride * output_height * output_width * sizeof(float),
205 .output_channel_stride = output_height * output_width * sizeof(float),
206 .ukernel = xnn_params.f32.ibilinear_chw.ukernel,
207 };
208
209 const size_t num_threads = pthreadpool_get_threads_count(threadpool);
210 size_t output_channel_tile = resize_op->channels;
211 if (num_threads > 1) {
212 const size_t target_tiles_per_thread = 4;
213 const size_t max_channel_tile = divide_round_up(output_channel_tile, num_threads * target_tiles_per_thread);
214 if (max_channel_tile < output_channel_tile) {
215 const uint32_t output_channel_subtile = xnn_params.f32.ibilinear_chw.channel_tile;
216 output_channel_tile =
217 min(output_channel_tile,
218 divide_round_up(output_channel_tile, max_channel_tile * output_channel_subtile) * output_channel_subtile);
219 }
220 }
221 resize_op->compute.type = xnn_parallelization_type_2d_tile_1d;
222 resize_op->compute.task_2d_tile_1d = (pthreadpool_task_2d_tile_1d_t) xnn_compute_resize_bilinear_chw;
223 resize_op->compute.range[0] = batch_size;
224 resize_op->compute.range[1] = resize_op->channels;
225 resize_op->compute.tile[0] = output_channel_tile;
226 resize_op->state = xnn_run_state_ready;
227
228 return xnn_status_success;
229 }
230