Lines Matching refs:wrapper
56 using q8x8_t = typename wrapper::traits::neon_vector<T, 8>::type; in poolingMxN_q8_neon_nhwc()
57 using q8x16_t = typename wrapper::traits::neon_vector<T, 16>::type; in poolingMxN_q8_neon_nhwc()
58 using q16_t = typename wrapper::traits::promote_t<T>; in poolingMxN_q8_neon_nhwc()
59 using q16x8_t = typename wrapper::traits::neon_vector<q16_t, 8>::type; in poolingMxN_q8_neon_nhwc()
60 using q32_t = typename wrapper::traits::promote_t<q16_t>; in poolingMxN_q8_neon_nhwc()
61 using q32x4_t = typename wrapper::traits::neon_vector<q32_t, 4>::type; in poolingMxN_q8_neon_nhwc()
106 … q32x4_t vres1 = wrapper::vdup_n(static_cast<q32_t>(0.f), wrapper::traits::vector_128_tag{}); in poolingMxN_q8_neon_nhwc()
107 … q32x4_t vres2 = wrapper::vdup_n(static_cast<q32_t>(0.f), wrapper::traits::vector_128_tag{}); in poolingMxN_q8_neon_nhwc()
108 … q32x4_t vres3 = wrapper::vdup_n(static_cast<q32_t>(0.f), wrapper::traits::vector_128_tag{}); in poolingMxN_q8_neon_nhwc()
109 … q32x4_t vres4 = wrapper::vdup_n(static_cast<q32_t>(0.f), wrapper::traits::vector_128_tag{}); in poolingMxN_q8_neon_nhwc()
120 …const q8x16_t data = wrapper::vloadq(reinterpret_cast<const T *>(in.ptr() + (x - pool_pad_left) * … in poolingMxN_q8_neon_nhwc()
123 const q16x8_t data_q16 = wrapper::vmovl(wrapper::vgetlow(data)); in poolingMxN_q8_neon_nhwc()
124 const q16x8_t data2_q16 = wrapper::vmovl(wrapper::vgethigh(data)); in poolingMxN_q8_neon_nhwc()
125 … vres1 = wrapper::vadd(vres1, wrapper::vmovl(wrapper::vgetlow(data_q16))); in poolingMxN_q8_neon_nhwc()
126 … vres2 = wrapper::vadd(vres2, wrapper::vmovl(wrapper::vgethigh(data_q16))); in poolingMxN_q8_neon_nhwc()
127 … vres3 = wrapper::vadd(vres3, wrapper::vmovl(wrapper::vgetlow(data2_q16))); in poolingMxN_q8_neon_nhwc()
128 … vres4 = wrapper::vadd(vres4, wrapper::vmovl(wrapper::vgethigh(data2_q16))); in poolingMxN_q8_neon_nhwc()
145 … wrapper::vstore(reinterpret_cast<T *>(out.ptr()) + x_off, wrapper::vgetlow(requantized_dst)); in poolingMxN_q8_neon_nhwc()
146 … wrapper::vstore(reinterpret_cast<T *>(out.ptr()) + x_off + 8, wrapper::vgethigh(requantized_dst)); in poolingMxN_q8_neon_nhwc()
152 … vres1 = vcvtq_q32_f32<q32x4_t>(wrapper::vmla(half_scale_v, vcvtq_f32_q32(vres1), scale_v)); in poolingMxN_q8_neon_nhwc()
153 … vres2 = vcvtq_q32_f32<q32x4_t>(wrapper::vmla(half_scale_v, vcvtq_f32_q32(vres2), scale_v)); in poolingMxN_q8_neon_nhwc()
154 … vres3 = vcvtq_q32_f32<q32x4_t>(wrapper::vmla(half_scale_v, vcvtq_f32_q32(vres3), scale_v)); in poolingMxN_q8_neon_nhwc()
155 … vres4 = vcvtq_q32_f32<q32x4_t>(wrapper::vmla(half_scale_v, vcvtq_f32_q32(vres4), scale_v)); in poolingMxN_q8_neon_nhwc()
157 …const q8x8_t res1 = wrapper::vmovn(wrapper::vcombine(wrapper::vmovn(vres1), wrapper::vmovn(vres2))… in poolingMxN_q8_neon_nhwc()
158 …const q8x8_t res2 = wrapper::vmovn(wrapper::vcombine(wrapper::vmovn(vres3), wrapper::vmovn(vres4))… in poolingMxN_q8_neon_nhwc()
160 wrapper::vstore(reinterpret_cast<T *>(out.ptr()) + x_off, res1); in poolingMxN_q8_neon_nhwc()
161 wrapper::vstore(reinterpret_cast<T *>(out.ptr()) + x_off + 8, res2); in poolingMxN_q8_neon_nhwc()
166 … q8x16_t vres = wrapper::vdup_n(std::numeric_limits<T>::min(), wrapper::traits::vector_128_tag{}); in poolingMxN_q8_neon_nhwc()
172 …const q8x16_t data = wrapper::vloadq(reinterpret_cast<const T *>(in.ptr() + (x - pool_pad_left) * … in poolingMxN_q8_neon_nhwc()
174 vres = wrapper::vmax(vres, data); in poolingMxN_q8_neon_nhwc()
179 …wrapper::vstore(reinterpret_cast<T *>(out.ptr()) + x_off, (src_qinfo != dst_qinfo) ? vrequantize_p… in poolingMxN_q8_neon_nhwc()
189 … q8x8_t vres = wrapper::vdup_n(std::numeric_limits<T>::min(), wrapper::traits::vector_64_tag{}); in poolingMxN_q8_neon_nhwc()
194 …const q8x8_t data = wrapper::vload(reinterpret_cast<const T *>(in.ptr() + (x - pool_pad_left) * st… in poolingMxN_q8_neon_nhwc()
196 vres = wrapper::vmax(vres, data); in poolingMxN_q8_neon_nhwc()
201 wrapper::vstore(reinterpret_cast<T *>(out.ptr()) + x_off, in poolingMxN_q8_neon_nhwc()
294 wrapper::vgetlane(v, 0), in scale_vector_q16x8()
295 wrapper::vgetlane(v, 1), in scale_vector_q16x8()
296 wrapper::vgetlane(v, 2), in scale_vector_q16x8()
297 wrapper::vgetlane(v, 3), in scale_vector_q16x8()
298 wrapper::vgetlane(v, 4), in scale_vector_q16x8()
299 wrapper::vgetlane(v, 5), in scale_vector_q16x8()
300 wrapper::vgetlane(v, 6), in scale_vector_q16x8()
301 wrapper::vgetlane(v, 7), in scale_vector_q16x8()
318 v = wrapper::vsetlane(elems[0], v, 0); in scale_vector_q16x8()
319 v = wrapper::vsetlane(elems[1], v, 1); in scale_vector_q16x8()
320 v = wrapper::vsetlane(elems[2], v, 2); in scale_vector_q16x8()
321 v = wrapper::vsetlane(elems[3], v, 3); in scale_vector_q16x8()
322 v = wrapper::vsetlane(elems[4], v, 4); in scale_vector_q16x8()
323 v = wrapper::vsetlane(elems[5], v, 5); in scale_vector_q16x8()
324 v = wrapper::vsetlane(elems[6], v, 6); in scale_vector_q16x8()
325 v = wrapper::vsetlane(elems[7], v, 7); in scale_vector_q16x8()
346 return wrapper::vloadq(vec); in load16_boundary_aware()
393 using q8x8_t = typename wrapper::traits::neon_vector<T, 8>::type; in pooling2_quantized_neon_nchw()
394 using q8x16_t = typename wrapper::traits::neon_vector<T, 16>::type; in pooling2_quantized_neon_nchw()
395 using q16_t = typename wrapper::traits::promote_t<T>; in pooling2_quantized_neon_nchw()
396 using q16x4_t = typename wrapper::traits::neon_vector<q16_t, 4>::type; in pooling2_quantized_neon_nchw()
397 using q16x8_t = typename wrapper::traits::neon_vector<q16_t, 8>::type; in pooling2_quantized_neon_nchw()
398 using q16x8x2_t = typename wrapper::traits::neon_vector<q16_t, 16>::type; in pooling2_quantized_neon_nchw()
442 …const q16x8x2_t top_data_q16 = { { wrapper::vmovl(wrapper::vgetlow(top_data)), wrapper::vmovl(w… in pooling2_quantized_neon_nchw()
443 …const q16x8x2_t bottom_data_q16 = { { wrapper::vmovl(wrapper::vgetlow(bottom_data)), wrapper::vmov… in pooling2_quantized_neon_nchw()
449 wrapper::vadd(top_data_q16.val[0], bottom_data_q16.val[0]), in pooling2_quantized_neon_nchw()
450 wrapper::vadd(top_data_q16.val[1], bottom_data_q16.val[1]), in pooling2_quantized_neon_nchw()
455 …const q16x4_t vpsum_1 = wrapper::vpadd(wrapper::vgetlow(vrsum.val[0]), wrapper::vgethigh(vrsum.val… in pooling2_quantized_neon_nchw()
456 …const q16x4_t vpsum_2 = wrapper::vpadd(wrapper::vgetlow(vrsum.val[1]), wrapper::vgethigh(vrsum.val… in pooling2_quantized_neon_nchw()
458 q16x8_t res_lower = wrapper::vcombine(vpsum_1, vpsum_2); in pooling2_quantized_neon_nchw()
464 lower_res = wrapper::vmovn(res_lower); in pooling2_quantized_neon_nchw()
473 wrapper::vext_1(vrsum.val[0], vrsum.val[1]), in pooling2_quantized_neon_nchw()
474 wrapper::vext_1(vrsum.val[1], vrsum.val[1]) in pooling2_quantized_neon_nchw()
479 q16x8_t res_upper = wrapper::vcombine( in pooling2_quantized_neon_nchw()
480 … wrapper::vpadd(wrapper::vgetlow(vrsum_shifted.val[0]), wrapper::vgethigh(vrsum_shifted.val[0])), in pooling2_quantized_neon_nchw()
481 … wrapper::vpadd(wrapper::vgetlow(vrsum_shifted.val[1]), wrapper::vgethigh(vrsum_shifted.val[1]))); in pooling2_quantized_neon_nchw()
487 upper_res = wrapper::vmovn(res_upper); in pooling2_quantized_neon_nchw()
492 const q8x16_t max_data = wrapper::vmax(top_data, bottom_data); in pooling2_quantized_neon_nchw()
493 … lower_res = wrapper::vpmax(wrapper::vgetlow(max_data), wrapper::vgethigh(max_data)); in pooling2_quantized_neon_nchw()
496 const q8x16_t max_data_shifted = wrapper::vext_1(max_data, max_data); in pooling2_quantized_neon_nchw()
497 …upper_res = wrapper::vpmax(wrapper::vgetlow(max_data_shifted), wrapper::vgeth… in pooling2_quantized_neon_nchw()
504 lower_res = wrapper::vgetlow(requantized_dst); in pooling2_quantized_neon_nchw()
505 upper_res = wrapper::vgethigh(requantized_dst); in pooling2_quantized_neon_nchw()
529 using q8x8_t = typename wrapper::traits::neon_vector<T, 8>::type; in pooling3_quantized_neon_nchw()
530 using q8x16_t = typename wrapper::traits::neon_vector<T, 16>::type; in pooling3_quantized_neon_nchw()
532 using q16_t = typename wrapper::traits::promote_t<T>; in pooling3_quantized_neon_nchw()
533 using q16x8_t = typename wrapper::traits::neon_vector<q16_t, 8>::type; in pooling3_quantized_neon_nchw()
534 using q16x8x2_t = typename wrapper::traits::neon_vector<q16_t, 16>::type; in pooling3_quantized_neon_nchw()
583 …const q16x8x2_t top_data_q16 = { { wrapper::vmovl(wrapper::vgetlow(top_data)), wrapper::vmovl(w… in pooling3_quantized_neon_nchw()
584 …const q16x8x2_t middle_data_q16 = { { wrapper::vmovl(wrapper::vgetlow(middle_data)), wrapper::vmov… in pooling3_quantized_neon_nchw()
585 …const q16x8x2_t bottom_data_q16 = { { wrapper::vmovl(wrapper::vgetlow(bottom_data)), wrapper::vmov… in pooling3_quantized_neon_nchw()
591 … wrapper::vadd(wrapper::vadd(top_data_q16.val[0], bottom_data_q16.val[0]), middle_data_q16.val[0]), in pooling3_quantized_neon_nchw()
592 … wrapper::vadd(wrapper::vadd(top_data_q16.val[1], bottom_data_q16.val[1]), middle_data_q16.val[1]), in pooling3_quantized_neon_nchw()
598 wrapper::vext_1(vrsum.val[0], vrsum.val[1]), in pooling3_quantized_neon_nchw()
599 wrapper::vext_1(vrsum.val[1], vrsum.val[1]) in pooling3_quantized_neon_nchw()
605 wrapper::vext_2(vrsum.val[0], vrsum.val[1]), in pooling3_quantized_neon_nchw()
606 wrapper::vext_2(vrsum.val[1], vrsum.val[1]) in pooling3_quantized_neon_nchw()
613 … wrapper::vadd(wrapper::vadd(vrsum.val[0], vrsum_shifted_1.val[0]), vrsum_shifted_2.val[0]), in pooling3_quantized_neon_nchw()
614 … wrapper::vadd(wrapper::vadd(vrsum.val[1], vrsum_shifted_1.val[1]), vrsum_shifted_2.val[1]), in pooling3_quantized_neon_nchw()
621 wrapper::vgetlane(final_sum.val[0], 0), in pooling3_quantized_neon_nchw()
622 wrapper::vgetlane(final_sum.val[0], 2), in pooling3_quantized_neon_nchw()
623 wrapper::vgetlane(final_sum.val[0], 4), in pooling3_quantized_neon_nchw()
624 wrapper::vgetlane(final_sum.val[0], 6), in pooling3_quantized_neon_nchw()
625 wrapper::vgetlane(final_sum.val[1], 0), in pooling3_quantized_neon_nchw()
626 wrapper::vgetlane(final_sum.val[1], 2), in pooling3_quantized_neon_nchw()
627 wrapper::vgetlane(final_sum.val[1], 4), in pooling3_quantized_neon_nchw()
628 wrapper::vgetlane(final_sum.val[1], 6), in pooling3_quantized_neon_nchw()
634 fres = wrapper::vmovn(res); in pooling3_quantized_neon_nchw()
646 … fqres = wrapper::vcombine(wrapper::vmovn(final_sum.val[0]), wrapper::vmovn(final_sum.val[1])); in pooling3_quantized_neon_nchw()
651 … const q8x16_t max_data = wrapper::vmax(wrapper::vmax(top_data, bottom_data), middle_data); in pooling3_quantized_neon_nchw()
652 const q8x16_t max_data_shift1 = wrapper::vext_1(max_data, max_data); in pooling3_quantized_neon_nchw()
653 const q8x16_t max_data_shift2 = wrapper::vext_2(max_data, max_data); in pooling3_quantized_neon_nchw()
654 …const q8x16_t final_max = wrapper::vmax(wrapper::vmax(max_data, max_data_shift1), max_data_s… in pooling3_quantized_neon_nchw()
658 …const q8x8x2_t table = { { wrapper::vgetlow(final_max), wrapper::vgethigh(final_max) } }; in pooling3_quantized_neon_nchw()
660 fres = wrapper::vtbl(table, lookup_val); in pooling3_quantized_neon_nchw()
673 …fqres = vrequantize_pooling<q8x8_t, q8x16_t>(wrapper::vgetlow(fqres), wrapper::vgethigh(fqres), re… in pooling3_quantized_neon_nchw()
675 …write16_boundary_aware<T, q8x8_t, false>(id.x(), dst_w, wrapper::vgetlow(fqres), wrapper::vgethigh… in pooling3_quantized_neon_nchw()
697 using q16_t = typename wrapper::traits::promote_t<T>; in poolingMxN_quantized_neon_nchw()
698 using q32_t = typename wrapper::traits::promote_t<q16_t>; in poolingMxN_quantized_neon_nchw()