1 /*
2 * Copyright (c) 2017-2020 Arm Limited.
3 *
4 * SPDX-License-Identifier: MIT
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to
8 * deal in the Software without restriction, including without limitation the
9 * rights to use, copy, modify, merge, publish, distribute, sublicense, and/or
10 * sell copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in all
14 * copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22 * SOFTWARE.
23 */
24 #include "src/core/NEON/kernels/NEQuantizationLayerKernel.h"
25
26 #include "arm_compute/core/Error.h"
27 #include "arm_compute/core/Helpers.h"
28 #include "arm_compute/core/Utils.h"
29 #include "arm_compute/core/Validate.h"
30 #include "arm_compute/core/Window.h"
31 #include "src/core/NEON/NEAsymm.h"
32 #include "src/core/NEON/NEMath.h"
33 #include "src/core/NEON/wrapper/wrapper.h"
34 #include "src/core/helpers/AutoConfiguration.h"
35 #include "src/core/helpers/WindowHelpers.h"
36
37 #include "src/core/CPP/Validate.h"
38
39 #include <arm_neon.h>
40 #include <map>
41
42 namespace arm_compute
43 {
44 namespace
45 {
46 constexpr auto window_step = 16;
47
validate_arguments(const ITensorInfo * input,const ITensorInfo * output)48 Status validate_arguments(const ITensorInfo *input, const ITensorInfo *output)
49 {
50 ARM_COMPUTE_RETURN_ERROR_ON_NULLPTR(input, output);
51 ARM_COMPUTE_RETURN_ERROR_ON_CPU_F16_UNSUPPORTED(input);
52 ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(input, 1, DataType::QASYMM8, DataType::QASYMM8_SIGNED, DataType::F16, DataType::F32);
53 ARM_COMPUTE_RETURN_ERROR_ON(output->tensor_shape().total_size() == 0);
54 ARM_COMPUTE_RETURN_ERROR_ON_DATA_TYPE_CHANNEL_NOT_IN(output, 1, DataType::QASYMM8, DataType::QASYMM8_SIGNED, DataType::QASYMM16);
55 ARM_COMPUTE_RETURN_ERROR_ON_MISMATCHING_SHAPES(input, output);
56
57 return Status{};
58 }
59
60 template <typename T>
load_value(const T * input_ptr)61 inline float32x4x4_t load_value(const T *input_ptr)
62 {
63 using Tx16_t = typename wrapper::traits::neon_vector<T, 16>::type;
64 return arm_compute::convert_to_float32x4x4<Tx16_t>(wrapper::vloadq(input_ptr));
65 }
66
67 template <>
load_value(const float * input_ptr)68 inline float32x4x4_t load_value(const float *input_ptr)
69 {
70 return { wrapper::vloadq(input_ptr),
71 wrapper::vloadq(input_ptr + 4),
72 wrapper::vloadq(input_ptr + 8),
73 wrapper::vloadq(input_ptr + 12) };
74 }
75 #ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
76 template <>
load_value(const float16_t * input_ptr)77 inline float32x4x4_t load_value(const float16_t *input_ptr)
78 {
79 return { vcvt_f32_f16(wrapper::vload(input_ptr)),
80 vcvt_f32_f16(wrapper::vload(input_ptr + 4)),
81 vcvt_f32_f16(wrapper::vload(input_ptr + 8)),
82 vcvt_f32_f16(wrapper::vload(input_ptr + 12)) };
83 }
84
85 #endif // __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
86
87 template <typename element_type>
88 using vector_type = wrapper::traits::neon_vector_t<element_type, window_step>;
89
90 template <typename quantized_type>
91 vector_type<quantized_type> vquantize_qasymm8(const float32x4x4_t &qv, const UniformQuantizationInfo &qi);
92
93 template <>
vquantize_qasymm8(const float32x4x4_t & qv,const UniformQuantizationInfo & qi)94 vector_type<uint8_t> vquantize_qasymm8<uint8_t>(const float32x4x4_t &qv, const UniformQuantizationInfo &qi)
95 {
96 return vquantize(qv, qi);
97 }
98
99 template <>
vquantize_qasymm8(const float32x4x4_t & qv,const UniformQuantizationInfo & qi)100 vector_type<int8_t> vquantize_qasymm8<int8_t>(const float32x4x4_t &qv, const UniformQuantizationInfo &qi)
101 {
102 return vquantize_signed(qv, qi);
103 }
104
105 } // namespace
106
NEQuantizationLayerKernel()107 NEQuantizationLayerKernel::NEQuantizationLayerKernel()
108 : _input(nullptr), _output(nullptr), _func(nullptr)
109 {
110 }
111
configure(const ITensor * input,ITensor * output)112 void NEQuantizationLayerKernel::configure(const ITensor *input, ITensor *output)
113 {
114 ARM_COMPUTE_ERROR_ON_NULLPTR(input, output);
115 ARM_COMPUTE_ERROR_THROW_ON(validate_arguments(input->info(), output->info()));
116
117 _input = input;
118 _output = output;
119
120 static const std::map<std::string, QuantizationFunctionExecutorPtr> quant_map =
121 {
122 { "op_QASYMM8_QASYMM8", &NEQuantizationLayerKernel::run_quantize_qasymm8<uint8_t, uint8_t> },
123 { "op_QASYMM8_QASYMM8_SIGNED", &NEQuantizationLayerKernel::run_quantize_qasymm8<uint8_t, int8_t> },
124 { "op_QASYMM8_QASYMM16", &NEQuantizationLayerKernel::run_quantize_qasymm16<uint8_t> },
125
126 { "op_QASYMM8_SIGNED_QASYMM8", &NEQuantizationLayerKernel::run_quantize_qasymm8<int8_t, uint8_t> },
127 { "op_QASYMM8_SIGNED_QASYMM8_SIGNED", &NEQuantizationLayerKernel::run_quantize_qasymm8<int8_t, int8_t> },
128 { "op_QASYMM8_SIGNED_QASYMM16", &NEQuantizationLayerKernel::run_quantize_qasymm16<int8_t> },
129
130 { "op_F32_QASYMM8", &NEQuantizationLayerKernel::run_quantize_qasymm8<float, uint8_t> },
131 { "op_F32_QASYMM8_SIGNED", &NEQuantizationLayerKernel::run_quantize_qasymm8<float, int8_t> },
132 { "op_F32_QASYMM16", &NEQuantizationLayerKernel::run_quantize_qasymm16<float> },
133
134 #ifdef __ARM_FEATURE_FP16_VECTOR_ARITHMETIC
135 { "op_F16_QASYMM8", &NEQuantizationLayerKernel::run_quantize_qasymm8<float16_t, uint8_t> },
136 { "op_F16_QASYMM8_SIGNED", &NEQuantizationLayerKernel::run_quantize_qasymm8<float16_t, int8_t> },
137 { "op_F16_QASYMM16", &NEQuantizationLayerKernel::run_quantize_qasymm16<float16_t> },
138 #endif /* __ARM_FEATURE_FP16_VECTOR_ARITHMETIC*/
139 };
140
141 std::string function_to_call("op_");
142 function_to_call += string_from_data_type(_input->info()->data_type()) + "_";
143 function_to_call += string_from_data_type(_output->info()->data_type());
144
145 auto it = quant_map.find(function_to_call);
146
147 if(it == quant_map.end())
148 {
149 ARM_COMPUTE_ERROR("Unsupported combination of input and output data types");
150 }
151 _func = it->second;
152
153 // Configure kernel window
154 Window win_config = calculate_max_window(*input->info(), Steps());
155
156 Coordinates coord;
157 coord.set_num_dimensions(output->info()->num_dimensions());
158 output->info()->set_valid_region(ValidRegion(coord, output->info()->tensor_shape()));
159
160 INEKernel::configure(win_config);
161 }
162
validate(const ITensorInfo * input,const ITensorInfo * output)163 Status NEQuantizationLayerKernel::validate(const ITensorInfo *input, const ITensorInfo *output)
164 {
165 ARM_COMPUTE_RETURN_ON_ERROR(validate_arguments(input, output));
166 return Status{};
167 }
168
169 template <typename TIn, typename TOut>
run_quantize_qasymm8(const Window & window)170 void NEQuantizationLayerKernel::run_quantize_qasymm8(const Window &window)
171 {
172 const auto window_start_x = static_cast<int>(window.x().start());
173 const auto window_end_x = static_cast<int>(window.x().end());
174
175 const UniformQuantizationInfo uqinfo_in = _input->info()->quantization_info().uniform();
176 UniformQuantizationInfo uqinfo = _output->info()->quantization_info().uniform();
177 if(is_data_type_quantized_asymmetric(_input->info()->data_type()))
178 {
179 uqinfo = compute_requantization_scale_offset(uqinfo_in, uqinfo);
180 }
181 #ifdef __aarch64__
182 constexpr RoundingPolicy rounding_policy = RoundingPolicy::TO_NEAREST_EVEN;
183 #else //__aarch64__
184 constexpr RoundingPolicy rounding_policy = RoundingPolicy::TO_ZERO;
185 #endif //__aarch64__
186
187 // Collapse window and reset first dimension to handle tail calculations manually
188 Window win_collapsed = window.collapse_if_possible(window, Window::DimZ);
189 win_collapsed.set(Window::DimX, Window::Dimension(0, 1, 1));
190
191 Iterator input(_input, win_collapsed);
192 Iterator output(_output, win_collapsed);
193 execute_window_loop(win_collapsed, [&](const Coordinates &)
194 {
195 auto input_ptr = reinterpret_cast<const TIn *>(input.ptr());
196 auto output_ptr = reinterpret_cast<TOut *>(output.ptr());
197
198 int x = window_start_x;
199 for(; x <= (window_end_x - window_step); x += window_step)
200 {
201 wrapper::vstore(&output_ptr[x], vquantize_qasymm8<TOut>(load_value(&input_ptr[x]), uqinfo));
202 }
203 // Compute left-over elements
204 for(; x < window_end_x; ++x)
205 {
206 output_ptr[x] = Qasymm8QuantizationHelper<TOut>::quantize(input_ptr[x], uqinfo, rounding_policy);
207 }
208 },
209 input, output);
210 }
211
212 template <typename T>
run_quantize_qasymm16(const Window & window)213 void NEQuantizationLayerKernel::run_quantize_qasymm16(const Window &window)
214 {
215 const auto window_start_x = static_cast<int>(window.x().start());
216 const auto window_end_x = static_cast<int>(window.x().end());
217
218 const UniformQuantizationInfo uqinfo_in = _input->info()->quantization_info().uniform();
219 UniformQuantizationInfo uqinfo = _output->info()->quantization_info().uniform();
220 if(is_data_type_quantized_asymmetric(_input->info()->data_type()))
221 {
222 uqinfo = compute_requantization_scale_offset(uqinfo_in, uqinfo);
223 }
224 #ifdef __aarch64__
225 constexpr RoundingPolicy rounding_policy = RoundingPolicy::TO_NEAREST_EVEN;
226 #else //__aarch64__
227 constexpr RoundingPolicy rounding_policy = RoundingPolicy::TO_ZERO;
228 #endif //__aarch64__
229
230 // Collapse window and reset first dimension to handle tail calculations manually
231 Window win_collapsed = window.collapse_if_possible(window, Window::DimZ);
232 win_collapsed.set(Window::DimX, Window::Dimension(0, 1, 1));
233
234 Iterator input(_input, win_collapsed);
235 Iterator output(_output, win_collapsed);
236 execute_window_loop(win_collapsed, [&](const Coordinates &)
237 {
238 auto input_ptr = reinterpret_cast<const T *>(input.ptr());
239 auto output_ptr = reinterpret_cast<uint16_t *>(output.ptr());
240
241 int x = window_start_x;
242 for(; x <= (window_end_x - window_step); x += window_step)
243 {
244 uint16x8x2_t tmp = vquantize_qasymm16(load_value(&input_ptr[x]), uqinfo);
245 vst1q_u16(&output_ptr[x], tmp.val[0]);
246 vst1q_u16(&output_ptr[x + 8], tmp.val[1]);
247 }
248 // Compute left-over elements
249 for(; x < window_end_x; ++x)
250 {
251 output_ptr[x] = quantize_qasymm16(input_ptr[x], uqinfo, rounding_policy);
252 }
253 },
254 input, output);
255 }
256
run(const Window & window,const ThreadInfo & info)257 void NEQuantizationLayerKernel::run(const Window &window, const ThreadInfo &info)
258 {
259 ARM_COMPUTE_UNUSED(info);
260 ARM_COMPUTE_ERROR_ON_UNCONFIGURED_KERNEL(this);
261 ARM_COMPUTE_ERROR_ON_INVALID_SUBWINDOW(INEKernel::window(), window);
262 ARM_COMPUTE_ERROR_ON(_func == nullptr);
263
264 (this->*_func)(window);
265 }
266 } // namespace arm_compute
267