• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // Copyright 2019 Google LLC
2 //
3 // This source code is licensed under the BSD-style license found in the
4 // LICENSE file in the root directory of this source tree.
5 
6 #include <math.h>
7 #include <stddef.h>
8 #include <stdint.h>
9 #include <stdlib.h>
10 #include <string.h>
11 
12 #include <xnnpack.h>
13 #include <xnnpack/allocator.h>
14 #include <xnnpack/log.h>
15 #include <xnnpack/operator.h>
16 #include <xnnpack/pack.h>
17 #include <xnnpack/params-init.h>
18 #include <xnnpack/params.h>
19 
20 
create_prelu_nc(size_t channels,size_t input_stride,size_t output_stride,const void * negative_slope,uint32_t flags,uint32_t log2_weights_element_size,xnn_pack_prelu_w_function pack_prelu_w,uint32_t datatype_init_flags,enum xnn_operator_type operator_type,xnn_operator_t * prelu_op_out)21 static enum xnn_status create_prelu_nc(
22     size_t channels,
23     size_t input_stride,
24     size_t output_stride,
25     const void* negative_slope,
26     uint32_t flags,
27     uint32_t log2_weights_element_size,
28     xnn_pack_prelu_w_function pack_prelu_w,
29     uint32_t datatype_init_flags,
30     enum xnn_operator_type operator_type,
31     xnn_operator_t* prelu_op_out)
32 {
33   xnn_operator_t prelu_op = NULL;
34   enum xnn_status status = xnn_status_uninitialized;
35 
36   if ((xnn_params.init_flags & XNN_INIT_FLAG_XNNPACK) == 0) {
37     xnn_log_error("failed to setup %s operator: XNNPACK is not initialized",
38       xnn_operator_type_to_string(operator_type));
39     return xnn_status_uninitialized;
40   }
41 
42   status = xnn_status_unsupported_hardware;
43 
44   if ((xnn_params.init_flags & datatype_init_flags) != datatype_init_flags) {
45     xnn_log_error(
46       "failed to create %s operator: operations on data type are not supported",
47       xnn_operator_type_to_string(operator_type));
48     goto error;
49   }
50 
51   status = xnn_status_invalid_parameter;
52 
53   if (channels == 0) {
54     xnn_log_error(
55       "failed to create %s operator with %zu channels: number of channels must be non-zero",
56       xnn_operator_type_to_string(operator_type), channels);
57     goto error;
58   }
59 
60   if (input_stride < channels) {
61     xnn_log_error(
62       "failed to create %s operator with input element stride of %zu: "
63       "stride must be at least as large as the number of channels (%zu)",
64       xnn_operator_type_to_string(operator_type), input_stride, channels);
65     goto error;
66   }
67 
68   if (output_stride < channels) {
69     xnn_log_error(
70       "failed to create %s operator with output element stride of %zu: "
71       "stride must be at least as large as the number of channels (%zu)",
72       xnn_operator_type_to_string(operator_type), output_stride, channels);
73     goto error;
74   }
75 
76   status = xnn_status_out_of_memory;
77 
78   prelu_op = xnn_allocate_zero_simd_memory(sizeof(struct xnn_operator));
79   if (prelu_op == NULL) {
80     xnn_log_error(
81       "failed to allocate %zu bytes for %s operator descriptor",
82       sizeof(struct xnn_operator), xnn_operator_type_to_string(operator_type));
83     goto error;
84   }
85 
86   const size_t packed_weights_size = (channels << log2_weights_element_size) + XNN_EXTRA_BYTES;
87   prelu_op->packed_weights = xnn_allocate_simd_memory(packed_weights_size);
88   if (prelu_op->packed_weights == NULL) {
89     xnn_log_error(
90       "failed to allocate %zu bytes for %s operator packed weights",
91       packed_weights_size, xnn_operator_type_to_string(operator_type));
92     goto error;
93   }
94   pack_prelu_w(channels, negative_slope, prelu_op->packed_weights);
95 
96   prelu_op->channels = channels;
97   prelu_op->input_pixel_stride = input_stride;
98   prelu_op->output_pixel_stride = output_stride;
99 
100   prelu_op->type = operator_type;
101   prelu_op->flags = flags;
102 
103   prelu_op->state = xnn_run_state_invalid;
104 
105   *prelu_op_out = prelu_op;
106   return xnn_status_success;
107 
108 error:
109   xnn_delete_operator(prelu_op);
110   return status;
111 }
112 
113 
xnn_create_prelu_nc_f16(size_t channels,size_t input_stride,size_t output_stride,const void * negative_slope,uint32_t flags,xnn_operator_t * prelu_op_out)114 enum xnn_status xnn_create_prelu_nc_f16(
115     size_t channels,
116     size_t input_stride,
117     size_t output_stride,
118     const void* negative_slope,
119     uint32_t flags,
120     xnn_operator_t* prelu_op_out)
121 {
122   xnn_pack_prelu_w_function pack_prelu_w = (xnn_pack_prelu_w_function) xnn_pack_f16_prelu_w;
123   if (flags & XNN_FLAG_FP32_STATIC_WEIGHTS) {
124     pack_prelu_w = (xnn_pack_prelu_w_function) xnn_pack_f32_to_f16_prelu_w;
125   }
126 
127   return create_prelu_nc(
128     channels, input_stride, output_stride,
129     negative_slope, flags,
130     1 /* log2(sizeof(uint16_t)) */,
131     pack_prelu_w,
132     XNN_INIT_FLAG_F16, xnn_operator_type_prelu_nc_f16,
133     prelu_op_out);
134 }
135 
xnn_create_prelu_nc_f32(size_t channels,size_t input_stride,size_t output_stride,const float * negative_slope,uint32_t flags,xnn_operator_t * prelu_op_out)136 enum xnn_status xnn_create_prelu_nc_f32(
137     size_t channels,
138     size_t input_stride,
139     size_t output_stride,
140     const float* negative_slope,
141     uint32_t flags,
142     xnn_operator_t* prelu_op_out)
143 {
144   return create_prelu_nc(
145     channels, input_stride, output_stride,
146     negative_slope, flags,
147     2 /* log2(sizeof(float)) */,
148     (xnn_pack_prelu_w_function) xnn_pack_f32_prelu_w,
149     XNN_INIT_FLAG_F32, xnn_operator_type_prelu_nc_f32,
150     prelu_op_out);
151 }
152 
setup_prelu_nc(xnn_operator_t prelu_op,enum xnn_operator_type expected_operator_type,size_t batch_size,const float * input,float * output,uint32_t datatype_init_flags,uint32_t log2_element_size,const struct prelu_parameters prelu[restrict XNN_MIN_ELEMENTS (1)],size_t num_threads)153 static enum xnn_status setup_prelu_nc(
154     xnn_operator_t prelu_op,
155     enum xnn_operator_type expected_operator_type,
156     size_t batch_size,
157     const float* input,
158     float* output,
159     uint32_t datatype_init_flags,
160     uint32_t log2_element_size,
161     const struct prelu_parameters prelu[restrict XNN_MIN_ELEMENTS(1)],
162     size_t num_threads)
163 {
164   if (prelu_op->type != expected_operator_type) {
165     xnn_log_error("failed to setup operator: operator type mismatch (expected %s, got %s)",
166       xnn_operator_type_to_string(expected_operator_type),
167       xnn_operator_type_to_string(prelu_op->type));
168     return xnn_status_invalid_parameter;
169   }
170   prelu_op->state = xnn_run_state_invalid;
171 
172   if ((xnn_params.init_flags & XNN_INIT_FLAG_XNNPACK) == 0) {
173     xnn_log_error("failed to setup %s operator: XNNPACK is not initialized",
174       xnn_operator_type_to_string(expected_operator_type));
175     return xnn_status_uninitialized;
176   }
177 
178   if ((xnn_params.init_flags & datatype_init_flags) != datatype_init_flags) {
179     xnn_log_error("failed to setup %s operator: operations on data type are not supported",
180       xnn_operator_type_to_string(expected_operator_type));
181     return xnn_status_unsupported_hardware;
182   }
183 
184   if (batch_size == 0) {
185     prelu_op->state = xnn_run_state_skip;
186     return xnn_status_success;
187   }
188 
189   const size_t channels = prelu_op->channels;
190   prelu_op->context.prelu = (struct prelu_context) {
191     .n = channels << log2_element_size,
192     .x = input,
193     .x_stride = prelu_op->input_pixel_stride << log2_element_size,
194     .w = prelu_op->packed_weights,
195     .y = output,
196     .y_stride = prelu_op->output_pixel_stride << log2_element_size,
197     .ukernel = prelu->ukernel,
198   };
199 
200   size_t batch_tile = batch_size;
201   if (num_threads > 1) {
202     const size_t target_tiles_per_thread = 5;
203     const size_t max_batch_tile = divide_round_up(batch_size, num_threads * target_tiles_per_thread);
204     if (max_batch_tile < batch_tile) {
205       const uint32_t row_tile = prelu->row_tile;
206       batch_tile = min(batch_tile, divide_round_up(batch_tile, max_batch_tile * row_tile) * row_tile);
207     }
208   }
209   prelu_op->compute.type = xnn_parallelization_type_1d_tile_1d;
210   prelu_op->compute.task_1d_tile_1d = (pthreadpool_task_1d_tile_1d_t) xnn_compute_prelu;
211   prelu_op->compute.range[0] = batch_size;
212   prelu_op->compute.tile[0] = batch_tile;
213   prelu_op->state = xnn_run_state_ready;
214 
215   return xnn_status_success;
216 }
217 
xnn_setup_prelu_nc_f16(xnn_operator_t prelu_op,size_t batch_size,const void * input,void * output,pthreadpool_t threadpool)218 enum xnn_status xnn_setup_prelu_nc_f16(
219     xnn_operator_t prelu_op,
220     size_t batch_size,
221     const void* input,
222     void* output,
223     pthreadpool_t threadpool)
224 {
225   return setup_prelu_nc(
226     prelu_op, xnn_operator_type_prelu_nc_f16,
227     batch_size, input, output,
228     XNN_INIT_FLAG_F16,
229     1 /* log2(sizeof(uint16_t)) */,
230     &xnn_params.f16.prelu,
231     pthreadpool_get_threads_count(threadpool));
232 }
233 
xnn_setup_prelu_nc_f32(xnn_operator_t prelu_op,size_t batch_size,const float * input,float * output,pthreadpool_t threadpool)234 enum xnn_status xnn_setup_prelu_nc_f32(
235     xnn_operator_t prelu_op,
236     size_t batch_size,
237     const float* input,
238     float* output,
239     pthreadpool_t threadpool)
240 {
241   return setup_prelu_nc(
242     prelu_op, xnn_operator_type_prelu_nc_f32,
243     batch_size, input, output,
244     XNN_INIT_FLAG_F32,
245     2 /* log2(sizeof(float)) */,
246     &xnn_params.f32.prelu,
247     pthreadpool_get_threads_count(threadpool));
248 }
249