1 // Copyright 2019 Google LLC
2 //
3 // This source code is licensed under the BSD-style license found in the
4 // LICENSE file in the root directory of this source tree.
5
6 #include <math.h>
7 #include <stddef.h>
8 #include <stdint.h>
9 #include <stdlib.h>
10 #include <string.h>
11
12 #include <xnnpack.h>
13 #include <xnnpack/allocator.h>
14 #include <xnnpack/log.h>
15 #include <xnnpack/operator.h>
16 #include <xnnpack/pack.h>
17 #include <xnnpack/params-init.h>
18 #include <xnnpack/params.h>
19
20
create_prelu_nc(size_t channels,size_t input_stride,size_t output_stride,const void * negative_slope,uint32_t flags,uint32_t log2_weights_element_size,xnn_pack_prelu_w_function pack_prelu_w,uint32_t datatype_init_flags,enum xnn_operator_type operator_type,xnn_operator_t * prelu_op_out)21 static enum xnn_status create_prelu_nc(
22 size_t channels,
23 size_t input_stride,
24 size_t output_stride,
25 const void* negative_slope,
26 uint32_t flags,
27 uint32_t log2_weights_element_size,
28 xnn_pack_prelu_w_function pack_prelu_w,
29 uint32_t datatype_init_flags,
30 enum xnn_operator_type operator_type,
31 xnn_operator_t* prelu_op_out)
32 {
33 xnn_operator_t prelu_op = NULL;
34 enum xnn_status status = xnn_status_uninitialized;
35
36 if ((xnn_params.init_flags & XNN_INIT_FLAG_XNNPACK) == 0) {
37 xnn_log_error("failed to setup %s operator: XNNPACK is not initialized",
38 xnn_operator_type_to_string(operator_type));
39 return xnn_status_uninitialized;
40 }
41
42 status = xnn_status_unsupported_hardware;
43
44 if ((xnn_params.init_flags & datatype_init_flags) != datatype_init_flags) {
45 xnn_log_error(
46 "failed to create %s operator: operations on data type are not supported",
47 xnn_operator_type_to_string(operator_type));
48 goto error;
49 }
50
51 status = xnn_status_invalid_parameter;
52
53 if (channels == 0) {
54 xnn_log_error(
55 "failed to create %s operator with %zu channels: number of channels must be non-zero",
56 xnn_operator_type_to_string(operator_type), channels);
57 goto error;
58 }
59
60 if (input_stride < channels) {
61 xnn_log_error(
62 "failed to create %s operator with input element stride of %zu: "
63 "stride must be at least as large as the number of channels (%zu)",
64 xnn_operator_type_to_string(operator_type), input_stride, channels);
65 goto error;
66 }
67
68 if (output_stride < channels) {
69 xnn_log_error(
70 "failed to create %s operator with output element stride of %zu: "
71 "stride must be at least as large as the number of channels (%zu)",
72 xnn_operator_type_to_string(operator_type), output_stride, channels);
73 goto error;
74 }
75
76 status = xnn_status_out_of_memory;
77
78 prelu_op = xnn_allocate_zero_simd_memory(sizeof(struct xnn_operator));
79 if (prelu_op == NULL) {
80 xnn_log_error(
81 "failed to allocate %zu bytes for %s operator descriptor",
82 sizeof(struct xnn_operator), xnn_operator_type_to_string(operator_type));
83 goto error;
84 }
85
86 const size_t packed_weights_size = (channels << log2_weights_element_size) + XNN_EXTRA_BYTES;
87 prelu_op->packed_weights = xnn_allocate_simd_memory(packed_weights_size);
88 if (prelu_op->packed_weights == NULL) {
89 xnn_log_error(
90 "failed to allocate %zu bytes for %s operator packed weights",
91 packed_weights_size, xnn_operator_type_to_string(operator_type));
92 goto error;
93 }
94 pack_prelu_w(channels, negative_slope, prelu_op->packed_weights);
95
96 prelu_op->channels = channels;
97 prelu_op->input_pixel_stride = input_stride;
98 prelu_op->output_pixel_stride = output_stride;
99
100 prelu_op->type = operator_type;
101 prelu_op->flags = flags;
102
103 prelu_op->state = xnn_run_state_invalid;
104
105 *prelu_op_out = prelu_op;
106 return xnn_status_success;
107
108 error:
109 xnn_delete_operator(prelu_op);
110 return status;
111 }
112
113
xnn_create_prelu_nc_f16(size_t channels,size_t input_stride,size_t output_stride,const void * negative_slope,uint32_t flags,xnn_operator_t * prelu_op_out)114 enum xnn_status xnn_create_prelu_nc_f16(
115 size_t channels,
116 size_t input_stride,
117 size_t output_stride,
118 const void* negative_slope,
119 uint32_t flags,
120 xnn_operator_t* prelu_op_out)
121 {
122 xnn_pack_prelu_w_function pack_prelu_w = (xnn_pack_prelu_w_function) xnn_pack_f16_prelu_w;
123 if (flags & XNN_FLAG_FP32_STATIC_WEIGHTS) {
124 pack_prelu_w = (xnn_pack_prelu_w_function) xnn_pack_f32_to_f16_prelu_w;
125 }
126
127 return create_prelu_nc(
128 channels, input_stride, output_stride,
129 negative_slope, flags,
130 1 /* log2(sizeof(uint16_t)) */,
131 pack_prelu_w,
132 XNN_INIT_FLAG_F16, xnn_operator_type_prelu_nc_f16,
133 prelu_op_out);
134 }
135
xnn_create_prelu_nc_f32(size_t channels,size_t input_stride,size_t output_stride,const float * negative_slope,uint32_t flags,xnn_operator_t * prelu_op_out)136 enum xnn_status xnn_create_prelu_nc_f32(
137 size_t channels,
138 size_t input_stride,
139 size_t output_stride,
140 const float* negative_slope,
141 uint32_t flags,
142 xnn_operator_t* prelu_op_out)
143 {
144 return create_prelu_nc(
145 channels, input_stride, output_stride,
146 negative_slope, flags,
147 2 /* log2(sizeof(float)) */,
148 (xnn_pack_prelu_w_function) xnn_pack_f32_prelu_w,
149 XNN_INIT_FLAG_F32, xnn_operator_type_prelu_nc_f32,
150 prelu_op_out);
151 }
152
setup_prelu_nc(xnn_operator_t prelu_op,enum xnn_operator_type expected_operator_type,size_t batch_size,const float * input,float * output,uint32_t datatype_init_flags,uint32_t log2_element_size,const struct prelu_parameters prelu[restrict XNN_MIN_ELEMENTS (1)],size_t num_threads)153 static enum xnn_status setup_prelu_nc(
154 xnn_operator_t prelu_op,
155 enum xnn_operator_type expected_operator_type,
156 size_t batch_size,
157 const float* input,
158 float* output,
159 uint32_t datatype_init_flags,
160 uint32_t log2_element_size,
161 const struct prelu_parameters prelu[restrict XNN_MIN_ELEMENTS(1)],
162 size_t num_threads)
163 {
164 if (prelu_op->type != expected_operator_type) {
165 xnn_log_error("failed to setup operator: operator type mismatch (expected %s, got %s)",
166 xnn_operator_type_to_string(expected_operator_type),
167 xnn_operator_type_to_string(prelu_op->type));
168 return xnn_status_invalid_parameter;
169 }
170 prelu_op->state = xnn_run_state_invalid;
171
172 if ((xnn_params.init_flags & XNN_INIT_FLAG_XNNPACK) == 0) {
173 xnn_log_error("failed to setup %s operator: XNNPACK is not initialized",
174 xnn_operator_type_to_string(expected_operator_type));
175 return xnn_status_uninitialized;
176 }
177
178 if ((xnn_params.init_flags & datatype_init_flags) != datatype_init_flags) {
179 xnn_log_error("failed to setup %s operator: operations on data type are not supported",
180 xnn_operator_type_to_string(expected_operator_type));
181 return xnn_status_unsupported_hardware;
182 }
183
184 if (batch_size == 0) {
185 prelu_op->state = xnn_run_state_skip;
186 return xnn_status_success;
187 }
188
189 const size_t channels = prelu_op->channels;
190 prelu_op->context.prelu = (struct prelu_context) {
191 .n = channels << log2_element_size,
192 .x = input,
193 .x_stride = prelu_op->input_pixel_stride << log2_element_size,
194 .w = prelu_op->packed_weights,
195 .y = output,
196 .y_stride = prelu_op->output_pixel_stride << log2_element_size,
197 .ukernel = prelu->ukernel,
198 };
199
200 size_t batch_tile = batch_size;
201 if (num_threads > 1) {
202 const size_t target_tiles_per_thread = 5;
203 const size_t max_batch_tile = divide_round_up(batch_size, num_threads * target_tiles_per_thread);
204 if (max_batch_tile < batch_tile) {
205 const uint32_t row_tile = prelu->row_tile;
206 batch_tile = min(batch_tile, divide_round_up(batch_tile, max_batch_tile * row_tile) * row_tile);
207 }
208 }
209 prelu_op->compute.type = xnn_parallelization_type_1d_tile_1d;
210 prelu_op->compute.task_1d_tile_1d = (pthreadpool_task_1d_tile_1d_t) xnn_compute_prelu;
211 prelu_op->compute.range[0] = batch_size;
212 prelu_op->compute.tile[0] = batch_tile;
213 prelu_op->state = xnn_run_state_ready;
214
215 return xnn_status_success;
216 }
217
xnn_setup_prelu_nc_f16(xnn_operator_t prelu_op,size_t batch_size,const void * input,void * output,pthreadpool_t threadpool)218 enum xnn_status xnn_setup_prelu_nc_f16(
219 xnn_operator_t prelu_op,
220 size_t batch_size,
221 const void* input,
222 void* output,
223 pthreadpool_t threadpool)
224 {
225 return setup_prelu_nc(
226 prelu_op, xnn_operator_type_prelu_nc_f16,
227 batch_size, input, output,
228 XNN_INIT_FLAG_F16,
229 1 /* log2(sizeof(uint16_t)) */,
230 &xnn_params.f16.prelu,
231 pthreadpool_get_threads_count(threadpool));
232 }
233
xnn_setup_prelu_nc_f32(xnn_operator_t prelu_op,size_t batch_size,const float * input,float * output,pthreadpool_t threadpool)234 enum xnn_status xnn_setup_prelu_nc_f32(
235 xnn_operator_t prelu_op,
236 size_t batch_size,
237 const float* input,
238 float* output,
239 pthreadpool_t threadpool)
240 {
241 return setup_prelu_nc(
242 prelu_op, xnn_operator_type_prelu_nc_f32,
243 batch_size, input, output,
244 XNN_INIT_FLAG_F32,
245 2 /* log2(sizeof(float)) */,
246 &xnn_params.f32.prelu,
247 pthreadpool_get_threads_count(threadpool));
248 }
249