1 // Copyright (c) Facebook, Inc. and its affiliates.
2 // All rights reserved.
3 //
4 // Copyright 2019 Google LLC
5 //
6 // This source code is licensed under the BSD-style license found in the
7 // LICENSE file in the root directory of this source tree.
8
9 #include <assert.h>
10 #include <math.h>
11 #include <stddef.h>
12 #include <stdint.h>
13 #include <stdlib.h>
14
15 #include <xnnpack.h>
16 #include <xnnpack/allocator.h>
17 #include <xnnpack/operator.h>
18 #include <xnnpack/log.h>
19 #include <xnnpack/params-init.h>
20
21
xnn_create_softmax_nc_qu8(size_t channels,size_t input_stride,size_t output_stride,float input_scale,uint8_t output_zero_point,float output_scale,uint32_t flags,xnn_operator_t * softmax_op_out)22 enum xnn_status xnn_create_softmax_nc_qu8(
23 size_t channels,
24 size_t input_stride,
25 size_t output_stride,
26 float input_scale,
27 uint8_t output_zero_point,
28 float output_scale,
29 uint32_t flags,
30 xnn_operator_t* softmax_op_out)
31 {
32 xnn_operator_t softmax_op = NULL;
33 enum xnn_status status = xnn_status_uninitialized;
34
35 if ((xnn_params.init_flags & XNN_INIT_FLAG_XNNPACK) == 0) {
36 xnn_log_error("failed to create %s operator: XNNPACK is not initialized",
37 xnn_operator_type_to_string(xnn_operator_type_sigmoid_nc_qu8));
38 goto error;
39 }
40
41 status = xnn_status_invalid_parameter;
42
43 if (channels == 0) {
44 xnn_log_error(
45 "failed to create %s operator with %zu channels: number of channels must be non-zero",
46 xnn_operator_type_to_string(xnn_operator_type_sigmoid_nc_qu8), channels);
47 goto error;
48 }
49
50 if (input_stride < channels) {
51 xnn_log_error(
52 "failed to create %s operator with input element stride of %zu: "
53 "stride must be at least as large as the number of channels (%zu)",
54 xnn_operator_type_to_string(xnn_operator_type_sigmoid_nc_qu8), input_stride, channels);
55 goto error;
56 }
57
58 if (output_stride < channels) {
59 xnn_log_error(
60 "failed to create %s operator with output element stride of %zu: "
61 "stride must be at least as large as the number of channels (%zu)",
62 xnn_operator_type_to_string(xnn_operator_type_sigmoid_nc_qu8), output_stride, channels);
63 goto error;
64 }
65
66 if (input_scale <= 0.0f || !isnormal(input_scale)) {
67 xnn_log_error(
68 "failed to create %s operator with %.7g input scale: scale must be finite, normalized, and positive",
69 xnn_operator_type_to_string(xnn_operator_type_sigmoid_nc_qu8), input_scale);
70 goto error;
71 }
72
73 if (output_scale <= 0.0f || !isnormal(output_scale)) {
74 xnn_log_error(
75 "failed to create %s operator with %.7g output scale: scale must be finite, normalized, and positive",
76 xnn_operator_type_to_string(xnn_operator_type_sigmoid_nc_qu8), output_scale);
77 goto error;
78 }
79
80 status = xnn_status_unsupported_parameter;
81
82 if (output_scale != 0x1.0p-8f) {
83 xnn_log_error(
84 "failed to create %s operator with %.7g output scale: only output scale of 1/256 is supported",
85 xnn_operator_type_to_string(xnn_operator_type_sigmoid_nc_qu8), output_scale);
86 goto error;
87 }
88
89 if (output_zero_point != 0) {
90 xnn_log_error(
91 "failed to create %s operator with %" PRIu8 " output zero point: only output zero point of 0 is supported",
92 xnn_operator_type_to_string(xnn_operator_type_sigmoid_nc_qu8), output_zero_point);
93 goto error;
94 }
95
96 status = xnn_status_out_of_memory;
97
98 softmax_op = xnn_allocate_zero_simd_memory(sizeof(struct xnn_operator));
99 if (softmax_op == NULL) {
100 xnn_log_error(
101 "failed to allocate %zu bytes for %s operator descriptor",
102 sizeof(struct xnn_operator), xnn_operator_type_to_string(xnn_operator_type_sigmoid_nc_qu8));
103 goto error;
104 }
105
106 softmax_op->lookup_table = xnn_allocate_simd_memory(256 * sizeof(uint32_t));
107 if (softmax_op->lookup_table == NULL) {
108 xnn_log_error(
109 "failed to allocate 256 bytes for %s operator lookup table",
110 xnn_operator_type_to_string(xnn_operator_type_sigmoid_nc_qu8));
111 goto error;
112 }
113
114 uint32_t* lookup_table = softmax_op->lookup_table;
115 const double qscale = fmin(((double) UINT32_MAX) / (double) channels, 8388607.0);
116 for (int32_t i = 0; i < 256; i++) {
117 const double scaled_exp_xi = qscale * exp((double) (i - 255) * (double) input_scale);
118 lookup_table[(uint32_t) i] = (uint32_t) lrint(scaled_exp_xi);
119 }
120
121 softmax_op->channels = channels;
122 softmax_op->input_pixel_stride = input_stride;
123 softmax_op->output_pixel_stride = output_stride;
124
125 softmax_op->type = xnn_operator_type_softmax_nc_qu8;
126 softmax_op->flags = flags;
127
128 softmax_op->state = xnn_run_state_invalid;
129
130 *softmax_op_out = softmax_op;
131 return xnn_status_success;
132
133 error:
134 xnn_delete_operator(softmax_op);
135 return status;
136 }
137
xnn_setup_softmax_nc_qu8(xnn_operator_t softmax_op,size_t batch_size,const uint8_t * input,uint8_t * output,pthreadpool_t threadpool)138 enum xnn_status xnn_setup_softmax_nc_qu8(
139 xnn_operator_t softmax_op,
140 size_t batch_size,
141 const uint8_t* input,
142 uint8_t* output,
143 pthreadpool_t threadpool)
144 {
145 if (softmax_op->type != xnn_operator_type_softmax_nc_qu8) {
146 xnn_log_error("failed to setup operator: operator type mismatch (expected %s, got %s)",
147 xnn_operator_type_to_string(xnn_operator_type_softmax_nc_qu8),
148 xnn_operator_type_to_string(softmax_op->type));
149 return xnn_status_invalid_parameter;
150 }
151 softmax_op->state = xnn_run_state_invalid;
152
153 if ((xnn_params.init_flags & XNN_INIT_FLAG_XNNPACK) == 0) {
154 xnn_log_error("failed to setup %s operator: XNNPACK is not initialized",
155 xnn_operator_type_to_string(xnn_operator_type_sigmoid_nc_qu8));
156 return xnn_status_uninitialized;
157 }
158
159 if (batch_size == 0) {
160 softmax_op->state = xnn_run_state_skip;
161 return xnn_status_success;
162 }
163
164 softmax_op->batch_size = batch_size;
165 softmax_op->input = input;
166 softmax_op->output = output;
167
168 softmax_op->context.u8_softmax = (struct u8_softmax_context) {
169 .n = softmax_op->channels,
170 .x = input,
171 .x_stride = softmax_op->input_pixel_stride * sizeof(uint8_t),
172 .t = softmax_op->lookup_table,
173 .y = output,
174 .y_stride = softmax_op->output_pixel_stride * sizeof(uint8_t),
175 .rmax_ukernel = xnn_params.u8.rmax,
176 .lut_norm_ukernel = xnn_params.u8.lut32norm,
177 };
178 softmax_op->compute.type = xnn_parallelization_type_1d;
179 softmax_op->compute.task_1d = (pthreadpool_task_1d_t) xnn_compute_u8_softmax;
180 softmax_op->compute.range[0] = batch_size;
181 softmax_op->state = xnn_run_state_ready;
182
183 return xnn_status_success;
184 }
185
xnn_create_softmax_nc_f32(size_t channels,size_t input_stride,size_t output_stride,uint32_t flags,xnn_operator_t * softmax_op_out)186 enum xnn_status xnn_create_softmax_nc_f32(
187 size_t channels,
188 size_t input_stride,
189 size_t output_stride,
190 uint32_t flags,
191 xnn_operator_t* softmax_op_out)
192 {
193 xnn_operator_t softmax_op = NULL;
194 enum xnn_status status = xnn_status_uninitialized;
195
196 if ((xnn_params.init_flags & XNN_INIT_FLAG_XNNPACK) == 0) {
197 xnn_log_error("failed to create %s operator: XNNPACK is not initialized",
198 xnn_operator_type_to_string(xnn_operator_type_sigmoid_nc_f32));
199 goto error;
200 }
201
202 status = xnn_status_invalid_parameter;
203
204 if (channels == 0) {
205 xnn_log_error(
206 "failed to create %s operator with %zu channels: number of channels must be non-zero",
207 xnn_operator_type_to_string(xnn_operator_type_sigmoid_nc_f32), channels);
208 goto error;
209 }
210
211 if (input_stride < channels) {
212 xnn_log_error(
213 "failed to create %s operator with input element stride of %zu: "
214 "stride must be at least as large as the number of channels (%zu)",
215 xnn_operator_type_to_string(xnn_operator_type_sigmoid_nc_f32), input_stride, channels);
216 goto error;
217 }
218
219 if (output_stride < channels) {
220 xnn_log_error(
221 "failed to create %s operator with output element stride of %zu: "
222 "stride must be at least as large as the number of channels (%zu)",
223 xnn_operator_type_to_string(xnn_operator_type_sigmoid_nc_f32), output_stride, channels);
224 goto error;
225 }
226
227 status = xnn_status_out_of_memory;
228
229 softmax_op = xnn_allocate_zero_simd_memory(sizeof(struct xnn_operator));
230 if (softmax_op == NULL) {
231 xnn_log_error(
232 "failed to allocate %zu bytes for %s operator descriptor",
233 sizeof(struct xnn_operator), xnn_operator_type_to_string(xnn_operator_type_sigmoid_nc_f32));
234 goto error;
235 }
236
237 softmax_op->channels = channels;
238 softmax_op->input_pixel_stride = input_stride;
239 softmax_op->output_pixel_stride = output_stride;
240
241 softmax_op->type = xnn_operator_type_softmax_nc_f32;
242 softmax_op->flags = flags;
243
244 softmax_op->state = xnn_run_state_invalid;
245
246 *softmax_op_out = softmax_op;
247 return xnn_status_success;
248
249 error:
250 xnn_delete_operator(softmax_op);
251 return status;
252 }
253
xnn_setup_softmax_nc_f32(xnn_operator_t softmax_op,size_t batch_size,const float * input,float * output,pthreadpool_t threadpool)254 enum xnn_status xnn_setup_softmax_nc_f32(
255 xnn_operator_t softmax_op,
256 size_t batch_size,
257 const float* input,
258 float* output,
259 pthreadpool_t threadpool)
260 {
261 if (softmax_op->type != xnn_operator_type_softmax_nc_f32) {
262 xnn_log_error("failed to setup operator: operator type mismatch (expected %s, got %s)",
263 xnn_operator_type_to_string(xnn_operator_type_softmax_nc_f32),
264 xnn_operator_type_to_string(softmax_op->type));
265 return xnn_status_invalid_parameter;
266 }
267 softmax_op->state = xnn_run_state_invalid;
268
269 if ((xnn_params.init_flags & XNN_INIT_FLAG_XNNPACK) == 0) {
270 xnn_log_error("failed to setup %s operator: XNNPACK is not initialized",
271 xnn_operator_type_to_string(xnn_operator_type_sigmoid_nc_f32));
272 return xnn_status_uninitialized;
273 }
274
275 if (batch_size == 0) {
276 softmax_op->state = xnn_run_state_skip;
277 return xnn_status_success;
278 }
279
280 softmax_op->batch_size = batch_size;
281 softmax_op->input = input;
282 softmax_op->output = output;
283
284 softmax_op->context.f32_three_pass_softmax = (struct f32_three_pass_softmax_context) {
285 .n = softmax_op->channels * sizeof(float),
286 .x = input,
287 .x_stride = softmax_op->input_pixel_stride * sizeof(float),
288 .y = output,
289 .y_stride = softmax_op->output_pixel_stride * sizeof(float),
290 .rmax_ukernel = xnn_params.f32.rmax,
291 .raddstoreexpminusmax_ukernel = xnn_params.f32.raddstoreexpminusmax.ukernel,
292 .vmulc_ukernel = xnn_params.f32.vmul.minmax.opc_ukernel,
293 };
294 if (xnn_params.f32.vmul.linear.opc_ukernel != NULL) {
295 softmax_op->context.f32_three_pass_softmax.vmulc_ukernel = xnn_params.f32.vmul.linear.opc_ukernel;
296 };
297 xnn_params.f32.vmul.init.f32_minmax(&softmax_op->context.f32_three_pass_softmax.minmax_params, -INFINITY, INFINITY);
298 xnn_params.f32.raddstoreexpminusmax.init(&softmax_op->context.f32_three_pass_softmax.expminus_params);
299 softmax_op->compute.type = xnn_parallelization_type_1d;
300 softmax_op->compute.task_1d = (pthreadpool_task_1d_t) xnn_compute_f32_three_pass_softmax;
301 softmax_op->compute.range[0] = batch_size;
302 softmax_op->state = xnn_run_state_ready;
303
304 return xnn_status_success;
305 }
306