1 /*
2 *
3 * Copyright (c) 2018, Alliance for Open Media. All rights reserved
4 *
5 * This source code is subject to the terms of the BSD 2 Clause License and
6 * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
7 * was not distributed with this source code in the LICENSE file, you can
8 * obtain it at www.aomedia.org/license/software. If the Alliance for Open
9 * Media Patent License 1.0 was not distributed with this source code in the
10 * PATENTS file, you can obtain it at www.aomedia.org/license/patent.
11 */
12
13 #include <arm_neon.h>
14 #include <assert.h>
15
16 #include "aom/aom_integer.h"
17 #include "aom_dsp/aom_dsp_common.h"
18 #include "aom_dsp/blend.h"
19 #include "aom_dsp/arm/mem_neon.h"
20 #include "aom_ports/mem.h"
21 #include "config/aom_dsp_rtcd.h"
22
aom_blend_a64_vmask_neon(uint8_t * dst,uint32_t dst_stride,const uint8_t * src0,uint32_t src0_stride,const uint8_t * src1,uint32_t src1_stride,const uint8_t * mask,int w,int h)23 void aom_blend_a64_vmask_neon(uint8_t *dst, uint32_t dst_stride,
24 const uint8_t *src0, uint32_t src0_stride,
25 const uint8_t *src1, uint32_t src1_stride,
26 const uint8_t *mask, int w, int h) {
27 uint8x8_t tmp0, tmp1;
28 uint8x16_t tmp0_q, tmp1_q, res_q;
29 uint16x8_t res, res_low, res_high;
30 uint32x2_t tmp0_32 = vdup_n_u32(0), tmp1_32 = vdup_n_u32(0);
31 uint16x4_t tmp0_16 = vdup_n_u16(0), tmp1_16 = vdup_n_u16(0);
32 assert(IMPLIES(src0 == dst, src0_stride == dst_stride));
33 assert(IMPLIES(src1 == dst, src1_stride == dst_stride));
34
35 assert(h >= 2);
36 assert(w >= 2);
37 assert(IS_POWER_OF_TWO(h));
38 assert(IS_POWER_OF_TWO(w));
39
40 if (w >= 16) {
41 for (int i = 0; i < h; ++i) {
42 const uint8x8_t m = vdup_n_u8((uint8_t)mask[i]);
43 const uint8x8_t max_minus_m = vdup_n_u8(64 - (uint8_t)mask[i]);
44 for (int j = 0; j < w; j += 16) {
45 __builtin_prefetch(src0);
46 __builtin_prefetch(src1);
47 tmp0_q = vld1q_u8(src0);
48 tmp1_q = vld1q_u8(src1);
49 res_low = vmull_u8(m, vget_low_u8(tmp0_q));
50 res_low = vmlal_u8(res_low, max_minus_m, vget_low_u8(tmp1_q));
51 res_high = vmull_u8(m, vget_high_u8(tmp0_q));
52 res_high = vmlal_u8(res_high, max_minus_m, vget_high_u8(tmp1_q));
53 res_q = vcombine_u8(vrshrn_n_u16(res_low, AOM_BLEND_A64_ROUND_BITS),
54 vrshrn_n_u16(res_high, AOM_BLEND_A64_ROUND_BITS));
55 vst1q_u8(dst, res_q);
56 src0 += 16;
57 src1 += 16;
58 dst += 16;
59 }
60 src0 += src0_stride - w;
61 src1 += src1_stride - w;
62 dst += dst_stride - w;
63 }
64 } else if (w == 8) {
65 for (int i = 0; i < h; ++i) {
66 __builtin_prefetch(src0);
67 __builtin_prefetch(src1);
68 const uint8x8_t m = vdup_n_u8((uint8_t)mask[i]);
69 const uint8x8_t max_minus_m = vdup_n_u8(64 - (uint8_t)mask[i]);
70 tmp0 = vld1_u8(src0);
71 tmp1 = vld1_u8(src1);
72 res = vmull_u8(m, tmp0);
73 res = vmlal_u8(res, max_minus_m, tmp1);
74 vst1_u8(dst, vrshrn_n_u16(res, AOM_BLEND_A64_ROUND_BITS));
75 src0 += src0_stride;
76 src1 += src1_stride;
77 dst += dst_stride;
78 }
79 } else if (w == 4) {
80 for (int i = 0; i < h; i += 2) {
81 __builtin_prefetch(src0 + 0 * src0_stride);
82 __builtin_prefetch(src0 + 1 * src0_stride);
83 __builtin_prefetch(src1 + 0 * src1_stride);
84 __builtin_prefetch(src1 + 1 * src1_stride);
85 const uint16x4_t m1 = vdup_n_u16((uint16_t)mask[i]);
86 const uint16x4_t m2 = vdup_n_u16((uint16_t)mask[i + 1]);
87 const uint8x8_t m = vmovn_u16(vcombine_u16(m1, m2));
88 const uint16x4_t max_minus_m1 = vdup_n_u16(64 - (uint16_t)mask[i]);
89 const uint16x4_t max_minus_m2 = vdup_n_u16(64 - (uint16_t)mask[i + 1]);
90 const uint8x8_t max_minus_m =
91 vmovn_u16(vcombine_u16(max_minus_m1, max_minus_m2));
92 load_unaligned_u8_4x2(src0, src0_stride, &tmp0_32);
93 tmp0 = vreinterpret_u8_u32(tmp0_32);
94 load_unaligned_u8_4x2(src1, src1_stride, &tmp1_32);
95 tmp1 = vreinterpret_u8_u32(tmp1_32);
96 res = vmull_u8(m, tmp0);
97 res = vmlal_u8(res, max_minus_m, tmp1);
98 const uint8x8_t result = vrshrn_n_u16(res, AOM_BLEND_A64_ROUND_BITS);
99 store_unaligned_u8_4x1(dst + 0 * dst_stride, result, 0);
100 store_unaligned_u8_4x1(dst + 1 * dst_stride, result, 1);
101 src0 += (2 * src0_stride);
102 src1 += (2 * src1_stride);
103 dst += (2 * dst_stride);
104 }
105 } else if (w == 2) {
106 for (int i = 0; i < h; i += 2) {
107 __builtin_prefetch(src0 + 0 * src0_stride);
108 __builtin_prefetch(src0 + 1 * src0_stride);
109 __builtin_prefetch(src1 + 0 * src1_stride);
110 __builtin_prefetch(src1 + 1 * src1_stride);
111 const uint8x8_t m1 = vdup_n_u8(mask[i]);
112 const uint8x8_t m2 = vdup_n_u8(mask[i + 1]);
113 const uint16x4x2_t m_trn =
114 vtrn_u16(vreinterpret_u16_u8(m1), vreinterpret_u16_u8(m2));
115 const uint8x8_t m = vreinterpret_u8_u16(m_trn.val[0]);
116 const uint8x8_t max_minus_m1 = vdup_n_u8(64 - mask[i]);
117 const uint8x8_t max_minus_m2 = vdup_n_u8(64 - mask[i + 1]);
118 const uint16x4x2_t max_minus_m_trn = vtrn_u16(
119 vreinterpret_u16_u8(max_minus_m1), vreinterpret_u16_u8(max_minus_m2));
120 const uint8x8_t max_minus_m = vreinterpret_u8_u16(max_minus_m_trn.val[0]);
121 load_unaligned_u8_2x2(src0, src0_stride, &tmp0_16);
122 tmp0 = vreinterpret_u8_u16(tmp0_16);
123 load_unaligned_u8_2x2(src1, src1_stride, &tmp1_16);
124 tmp1 = vreinterpret_u8_u16(tmp1_16);
125 res = vmull_u8(m, tmp0);
126 res = vmlal_u8(res, max_minus_m, tmp1);
127 const uint8x8_t result = vrshrn_n_u16(res, AOM_BLEND_A64_ROUND_BITS);
128 store_unaligned_u8_2x1(dst + 0 * dst_stride, result, 0);
129 store_unaligned_u8_2x1(dst + 1 * dst_stride, result, 1);
130 src0 += (2 * src0_stride);
131 src1 += (2 * src1_stride);
132 dst += (2 * dst_stride);
133 }
134 }
135 }
136