/* * Copyright (C) 2013 The Android Open Source Project * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #ifndef ANDROID_AUDIO_RESAMPLER_FIR_PROCESS_NEON_H #define ANDROID_AUDIO_RESAMPLER_FIR_PROCESS_NEON_H namespace android { // depends on AudioResamplerFirOps.h, AudioResamplerFirProcess.h #if USE_NEON // use intrinsics if inline arm32 assembly is not possible #if !USE_INLINE_ASSEMBLY #define USE_INTRINSIC #endif // following intrinsics available only on ARM 64 bit ACLE #ifndef __aarch64__ #undef vld1q_f32_x2 #undef vld1q_s32_x2 #endif #define TO_STRING2(x) #x #define TO_STRING(x) TO_STRING2(x) // uncomment to print GCC version, may be relevant for intrinsic optimizations /* #pragma message ("GCC version: " TO_STRING(__GNUC__) \ "." TO_STRING(__GNUC_MINOR__) \ "." TO_STRING(__GNUC_PATCHLEVEL__)) */ // // NEON specializations are enabled for Process() and ProcessL() in AudioResamplerFirProcess.h // // Two variants are presented here: // ARM NEON inline assembly which appears up to 10-15% faster than intrinsics (gcc 4.9) for arm32. // ARM NEON intrinsics which can also be used by arm64 and x86/64 with NEON header. // // Macros to save a mono/stereo accumulator sample in q0 (and q4) as stereo out. // These are only used for inline assembly. #define ASSEMBLY_ACCUMULATE_MONO \ "vld1.s32 {d2}, [%[vLR]:64] \n"/* (1) load volumes */\ "vld1.s32 {d3}, %[out] \n"/* (2) unaligned load the output */\ "vpadd.s32 d0, d0, d1 \n"/* (1) add all 4 partial sums */\ "vpadd.s32 d0, d0, d0 \n"/* (1+4d) and replicate L/R */\ "vqrdmulh.s32 d0, d0, d2 \n"/* (2+3d) apply volume */\ "vqadd.s32 d3, d3, d0 \n"/* (1+4d) accumulate result (saturating) */\ "vst1.s32 {d3}, %[out] \n"/* (2+2d) store result */ #define ASSEMBLY_ACCUMULATE_STEREO \ "vld1.s32 {d2}, [%[vLR]:64] \n"/* (1) load volumes*/\ "vld1.s32 {d3}, %[out] \n"/* (2) unaligned load the output*/\ "vpadd.s32 d0, d0, d1 \n"/* (1) add all 4 partial sums from q0*/\ "vpadd.s32 d8, d8, d9 \n"/* (1) add all 4 partial sums from q4*/\ "vpadd.s32 d0, d0, d8 \n"/* (1+4d) combine into L/R*/\ "vqrdmulh.s32 d0, d0, d2 \n"/* (2+3d) apply volume*/\ "vqadd.s32 d3, d3, d0 \n"/* (1+4d) accumulate result (saturating)*/\ "vst1.s32 {d3}, %[out] \n"/* (2+2d)store result*/ template static inline void ProcessNeonIntrinsic(int32_t* out, int count, const int16_t* coefsP, const int16_t* coefsN, const int16_t* sP, const int16_t* sN, const int32_t* volumeLR, uint32_t lerpP, const int16_t* coefsP1, const int16_t* coefsN1) { ALOG_ASSERT(count > 0 && (count & 7) == 0); // multiple of 8 static_assert(CHANNELS == 1 || CHANNELS == 2, "CHANNELS must be 1 or 2"); sP -= CHANNELS*((STRIDE>>1)-1); coefsP = (const int16_t*)__builtin_assume_aligned(coefsP, 16); coefsN = (const int16_t*)__builtin_assume_aligned(coefsN, 16); int16x4_t interp; if (!FIXED) { interp = vdup_n_s16(lerpP); //interp = (int16x4_t)vset_lane_s32 ((int32x2_t)lerpP, interp, 0); coefsP1 = (const int16_t*)__builtin_assume_aligned(coefsP1, 16); coefsN1 = (const int16_t*)__builtin_assume_aligned(coefsN1, 16); } int32x4_t accum, accum2; // warning uninitialized if we use veorq_s32 // (alternative to below) accum = veorq_s32(accum, accum); accum = vdupq_n_s32(0); if (CHANNELS == 2) { // (alternative to below) accum2 = veorq_s32(accum2, accum2); accum2 = vdupq_n_s32(0); } do { int16x8_t posCoef = vld1q_s16(coefsP); coefsP += 8; int16x8_t negCoef = vld1q_s16(coefsN); coefsN += 8; if (!FIXED) { // interpolate int16x8_t posCoef1 = vld1q_s16(coefsP1); coefsP1 += 8; int16x8_t negCoef1 = vld1q_s16(coefsN1); coefsN1 += 8; posCoef1 = vsubq_s16(posCoef1, posCoef); negCoef = vsubq_s16(negCoef, negCoef1); posCoef1 = vqrdmulhq_lane_s16(posCoef1, interp, 0); negCoef = vqrdmulhq_lane_s16(negCoef, interp, 0); posCoef = vaddq_s16(posCoef, posCoef1); negCoef = vaddq_s16(negCoef, negCoef1); } switch (CHANNELS) { case 1: { int16x8_t posSamp = vld1q_s16(sP); int16x8_t negSamp = vld1q_s16(sN); sN += 8; posSamp = vrev64q_s16(posSamp); // dot product accum = vmlal_s16(accum, vget_low_s16(posSamp), vget_high_s16(posCoef)); // reversed accum = vmlal_s16(accum, vget_high_s16(posSamp), vget_low_s16(posCoef)); // reversed accum = vmlal_s16(accum, vget_low_s16(negSamp), vget_low_s16(negCoef)); accum = vmlal_s16(accum, vget_high_s16(negSamp), vget_high_s16(negCoef)); sP -= 8; } break; case 2: { int16x8x2_t posSamp = vld2q_s16(sP); int16x8x2_t negSamp = vld2q_s16(sN); sN += 16; posSamp.val[0] = vrev64q_s16(posSamp.val[0]); posSamp.val[1] = vrev64q_s16(posSamp.val[1]); // dot product accum = vmlal_s16(accum, vget_low_s16(posSamp.val[0]), vget_high_s16(posCoef)); // r accum = vmlal_s16(accum, vget_high_s16(posSamp.val[0]), vget_low_s16(posCoef)); // r accum2 = vmlal_s16(accum2, vget_low_s16(posSamp.val[1]), vget_high_s16(posCoef)); // r accum2 = vmlal_s16(accum2, vget_high_s16(posSamp.val[1]), vget_low_s16(posCoef)); // r accum = vmlal_s16(accum, vget_low_s16(negSamp.val[0]), vget_low_s16(negCoef)); accum = vmlal_s16(accum, vget_high_s16(negSamp.val[0]), vget_high_s16(negCoef)); accum2 = vmlal_s16(accum2, vget_low_s16(negSamp.val[1]), vget_low_s16(negCoef)); accum2 = vmlal_s16(accum2, vget_high_s16(negSamp.val[1]), vget_high_s16(negCoef)); sP -= 16; } break; } } while (count -= 8); // multiply by volume and save volumeLR = (const int32_t*)__builtin_assume_aligned(volumeLR, 8); int32x2_t vLR = vld1_s32(volumeLR); int32x2_t outSamp = vld1_s32(out); // combine and funnel down accumulator int32x2_t outAccum = vpadd_s32(vget_low_s32(accum), vget_high_s32(accum)); if (CHANNELS == 1) { // duplicate accum to both L and R outAccum = vpadd_s32(outAccum, outAccum); } else if (CHANNELS == 2) { // accum2 contains R, fold in int32x2_t outAccum2 = vpadd_s32(vget_low_s32(accum2), vget_high_s32(accum2)); outAccum = vpadd_s32(outAccum, outAccum2); } outAccum = vqrdmulh_s32(outAccum, vLR); outSamp = vqadd_s32(outSamp, outAccum); vst1_s32(out, outSamp); } template static inline void ProcessNeonIntrinsic(int32_t* out, int count, const int32_t* coefsP, const int32_t* coefsN, const int16_t* sP, const int16_t* sN, const int32_t* volumeLR, uint32_t lerpP, const int32_t* coefsP1, const int32_t* coefsN1) { ALOG_ASSERT(count > 0 && (count & 7) == 0); // multiple of 8 static_assert(CHANNELS == 1 || CHANNELS == 2, "CHANNELS must be 1 or 2"); sP -= CHANNELS*((STRIDE>>1)-1); coefsP = (const int32_t*)__builtin_assume_aligned(coefsP, 16); coefsN = (const int32_t*)__builtin_assume_aligned(coefsN, 16); int32x2_t interp; if (!FIXED) { interp = vdup_n_s32(lerpP); coefsP1 = (const int32_t*)__builtin_assume_aligned(coefsP1, 16); coefsN1 = (const int32_t*)__builtin_assume_aligned(coefsN1, 16); } int32x4_t accum, accum2; // warning uninitialized if we use veorq_s32 // (alternative to below) accum = veorq_s32(accum, accum); accum = vdupq_n_s32(0); if (CHANNELS == 2) { // (alternative to below) accum2 = veorq_s32(accum2, accum2); accum2 = vdupq_n_s32(0); } do { #ifdef vld1q_s32_x2 int32x4x2_t posCoef = vld1q_s32_x2(coefsP); coefsP += 8; int32x4x2_t negCoef = vld1q_s32_x2(coefsN); coefsN += 8; #else int32x4x2_t posCoef; posCoef.val[0] = vld1q_s32(coefsP); coefsP += 4; posCoef.val[1] = vld1q_s32(coefsP); coefsP += 4; int32x4x2_t negCoef; negCoef.val[0] = vld1q_s32(coefsN); coefsN += 4; negCoef.val[1] = vld1q_s32(coefsN); coefsN += 4; #endif if (!FIXED) { // interpolate #ifdef vld1q_s32_x2 int32x4x2_t posCoef1 = vld1q_s32_x2(coefsP1); coefsP1 += 8; int32x4x2_t negCoef1 = vld1q_s32_x2(coefsN1); coefsN1 += 8; #else int32x4x2_t posCoef1; posCoef1.val[0] = vld1q_s32(coefsP1); coefsP1 += 4; posCoef1.val[1] = vld1q_s32(coefsP1); coefsP1 += 4; int32x4x2_t negCoef1; negCoef1.val[0] = vld1q_s32(coefsN1); coefsN1 += 4; negCoef1.val[1] = vld1q_s32(coefsN1); coefsN1 += 4; #endif posCoef1.val[0] = vsubq_s32(posCoef1.val[0], posCoef.val[0]); posCoef1.val[1] = vsubq_s32(posCoef1.val[1], posCoef.val[1]); negCoef.val[0] = vsubq_s32(negCoef.val[0], negCoef1.val[0]); negCoef.val[1] = vsubq_s32(negCoef.val[1], negCoef1.val[1]); posCoef1.val[0] = vqrdmulhq_lane_s32(posCoef1.val[0], interp, 0); posCoef1.val[1] = vqrdmulhq_lane_s32(posCoef1.val[1], interp, 0); negCoef.val[0] = vqrdmulhq_lane_s32(negCoef.val[0], interp, 0); negCoef.val[1] = vqrdmulhq_lane_s32(negCoef.val[1], interp, 0); posCoef.val[0] = vaddq_s32(posCoef.val[0], posCoef1.val[0]); posCoef.val[1] = vaddq_s32(posCoef.val[1], posCoef1.val[1]); negCoef.val[0] = vaddq_s32(negCoef.val[0], negCoef1.val[0]); negCoef.val[1] = vaddq_s32(negCoef.val[1], negCoef1.val[1]); } switch (CHANNELS) { case 1: { int16x8_t posSamp = vld1q_s16(sP); int16x8_t negSamp = vld1q_s16(sN); sN += 8; posSamp = vrev64q_s16(posSamp); int32x4_t posSamp0 = vshll_n_s16(vget_low_s16(posSamp), 15); int32x4_t posSamp1 = vshll_n_s16(vget_high_s16(posSamp), 15); int32x4_t negSamp0 = vshll_n_s16(vget_low_s16(negSamp), 15); int32x4_t negSamp1 = vshll_n_s16(vget_high_s16(negSamp), 15); // dot product posSamp0 = vqrdmulhq_s32(posSamp0, posCoef.val[1]); // reversed posSamp1 = vqrdmulhq_s32(posSamp1, posCoef.val[0]); // reversed negSamp0 = vqrdmulhq_s32(negSamp0, negCoef.val[0]); negSamp1 = vqrdmulhq_s32(negSamp1, negCoef.val[1]); accum = vaddq_s32(accum, posSamp0); negSamp0 = vaddq_s32(negSamp0, negSamp1); accum = vaddq_s32(accum, posSamp1); accum = vaddq_s32(accum, negSamp0); sP -= 8; } break; case 2: { int16x8x2_t posSamp = vld2q_s16(sP); int16x8x2_t negSamp = vld2q_s16(sN); sN += 16; posSamp.val[0] = vrev64q_s16(posSamp.val[0]); posSamp.val[1] = vrev64q_s16(posSamp.val[1]); // left int32x4_t posSamp0 = vshll_n_s16(vget_low_s16(posSamp.val[0]), 15); int32x4_t posSamp1 = vshll_n_s16(vget_high_s16(posSamp.val[0]), 15); int32x4_t negSamp0 = vshll_n_s16(vget_low_s16(negSamp.val[0]), 15); int32x4_t negSamp1 = vshll_n_s16(vget_high_s16(negSamp.val[0]), 15); // dot product posSamp0 = vqrdmulhq_s32(posSamp0, posCoef.val[1]); // reversed posSamp1 = vqrdmulhq_s32(posSamp1, posCoef.val[0]); // reversed negSamp0 = vqrdmulhq_s32(negSamp0, negCoef.val[0]); negSamp1 = vqrdmulhq_s32(negSamp1, negCoef.val[1]); accum = vaddq_s32(accum, posSamp0); negSamp0 = vaddq_s32(negSamp0, negSamp1); accum = vaddq_s32(accum, posSamp1); accum = vaddq_s32(accum, negSamp0); // right posSamp0 = vshll_n_s16(vget_low_s16(posSamp.val[1]), 15); posSamp1 = vshll_n_s16(vget_high_s16(posSamp.val[1]), 15); negSamp0 = vshll_n_s16(vget_low_s16(negSamp.val[1]), 15); negSamp1 = vshll_n_s16(vget_high_s16(negSamp.val[1]), 15); // dot product posSamp0 = vqrdmulhq_s32(posSamp0, posCoef.val[1]); // reversed posSamp1 = vqrdmulhq_s32(posSamp1, posCoef.val[0]); // reversed negSamp0 = vqrdmulhq_s32(negSamp0, negCoef.val[0]); negSamp1 = vqrdmulhq_s32(negSamp1, negCoef.val[1]); accum2 = vaddq_s32(accum2, posSamp0); negSamp0 = vaddq_s32(negSamp0, negSamp1); accum2 = vaddq_s32(accum2, posSamp1); accum2 = vaddq_s32(accum2, negSamp0); sP -= 16; } break; } } while (count -= 8); // multiply by volume and save volumeLR = (const int32_t*)__builtin_assume_aligned(volumeLR, 8); int32x2_t vLR = vld1_s32(volumeLR); int32x2_t outSamp = vld1_s32(out); // combine and funnel down accumulator int32x2_t outAccum = vpadd_s32(vget_low_s32(accum), vget_high_s32(accum)); if (CHANNELS == 1) { // duplicate accum to both L and R outAccum = vpadd_s32(outAccum, outAccum); } else if (CHANNELS == 2) { // accum2 contains R, fold in int32x2_t outAccum2 = vpadd_s32(vget_low_s32(accum2), vget_high_s32(accum2)); outAccum = vpadd_s32(outAccum, outAccum2); } outAccum = vqrdmulh_s32(outAccum, vLR); outSamp = vqadd_s32(outSamp, outAccum); vst1_s32(out, outSamp); } template static inline void ProcessNeonIntrinsic(float* out, int count, const float* coefsP, const float* coefsN, const float* sP, const float* sN, const float* volumeLR, float lerpP, const float* coefsP1, const float* coefsN1) { ALOG_ASSERT(count > 0 && (count & 7) == 0); // multiple of 8 static_assert(CHANNELS == 1 || CHANNELS == 2, "CHANNELS must be 1 or 2"); sP -= CHANNELS*((STRIDE>>1)-1); coefsP = (const float*)__builtin_assume_aligned(coefsP, 16); coefsN = (const float*)__builtin_assume_aligned(coefsN, 16); float32x2_t interp; if (!FIXED) { interp = vdup_n_f32(lerpP); coefsP1 = (const float*)__builtin_assume_aligned(coefsP1, 16); coefsN1 = (const float*)__builtin_assume_aligned(coefsN1, 16); } float32x4_t accum, accum2; // warning uninitialized if we use veorq_s32 // (alternative to below) accum = veorq_s32(accum, accum); accum = vdupq_n_f32(0); if (CHANNELS == 2) { // (alternative to below) accum2 = veorq_s32(accum2, accum2); accum2 = vdupq_n_f32(0); } do { #ifdef vld1q_f32_x2 float32x4x2_t posCoef = vld1q_f32_x2(coefsP); coefsP += 8; float32x4x2_t negCoef = vld1q_f32_x2(coefsN); coefsN += 8; #else float32x4x2_t posCoef; posCoef.val[0] = vld1q_f32(coefsP); coefsP += 4; posCoef.val[1] = vld1q_f32(coefsP); coefsP += 4; float32x4x2_t negCoef; negCoef.val[0] = vld1q_f32(coefsN); coefsN += 4; negCoef.val[1] = vld1q_f32(coefsN); coefsN += 4; #endif if (!FIXED) { // interpolate #ifdef vld1q_f32_x2 float32x4x2_t posCoef1 = vld1q_f32_x2(coefsP1); coefsP1 += 8; float32x4x2_t negCoef1 = vld1q_f32_x2(coefsN1); coefsN1 += 8; #else float32x4x2_t posCoef1; posCoef1.val[0] = vld1q_f32(coefsP1); coefsP1 += 4; posCoef1.val[1] = vld1q_f32(coefsP1); coefsP1 += 4; float32x4x2_t negCoef1; negCoef1.val[0] = vld1q_f32(coefsN1); coefsN1 += 4; negCoef1.val[1] = vld1q_f32(coefsN1); coefsN1 += 4; #endif posCoef1.val[0] = vsubq_f32(posCoef1.val[0], posCoef.val[0]); posCoef1.val[1] = vsubq_f32(posCoef1.val[1], posCoef.val[1]); negCoef.val[0] = vsubq_f32(negCoef.val[0], negCoef1.val[0]); negCoef.val[1] = vsubq_f32(negCoef.val[1], negCoef1.val[1]); posCoef.val[0] = vmlaq_lane_f32(posCoef.val[0], posCoef1.val[0], interp, 0); posCoef.val[1] = vmlaq_lane_f32(posCoef.val[1], posCoef1.val[1], interp, 0); negCoef.val[0] = vmlaq_lane_f32(negCoef1.val[0], negCoef.val[0], interp, 0); // rev negCoef.val[1] = vmlaq_lane_f32(negCoef1.val[1], negCoef.val[1], interp, 0); // rev } switch (CHANNELS) { case 1: { #ifdef vld1q_f32_x2 float32x4x2_t posSamp = vld1q_f32_x2(sP); float32x4x2_t negSamp = vld1q_f32_x2(sN); sN += 8; sP -= 8; #else float32x4x2_t posSamp; posSamp.val[0] = vld1q_f32(sP); sP += 4; posSamp.val[1] = vld1q_f32(sP); sP -= 12; float32x4x2_t negSamp; negSamp.val[0] = vld1q_f32(sN); sN += 4; negSamp.val[1] = vld1q_f32(sN); sN += 4; #endif // effectively we want a vrev128q_f32() posSamp.val[0] = vrev64q_f32(posSamp.val[0]); posSamp.val[1] = vrev64q_f32(posSamp.val[1]); posSamp.val[0] = vcombine_f32( vget_high_f32(posSamp.val[0]), vget_low_f32(posSamp.val[0])); posSamp.val[1] = vcombine_f32( vget_high_f32(posSamp.val[1]), vget_low_f32(posSamp.val[1])); accum = vmlaq_f32(accum, posSamp.val[0], posCoef.val[1]); accum = vmlaq_f32(accum, posSamp.val[1], posCoef.val[0]); accum = vmlaq_f32(accum, negSamp.val[0], negCoef.val[0]); accum = vmlaq_f32(accum, negSamp.val[1], negCoef.val[1]); } break; case 2: { float32x4x2_t posSamp0 = vld2q_f32(sP); sP += 8; float32x4x2_t negSamp0 = vld2q_f32(sN); sN += 8; posSamp0.val[0] = vrev64q_f32(posSamp0.val[0]); posSamp0.val[1] = vrev64q_f32(posSamp0.val[1]); posSamp0.val[0] = vcombine_f32( vget_high_f32(posSamp0.val[0]), vget_low_f32(posSamp0.val[0])); posSamp0.val[1] = vcombine_f32( vget_high_f32(posSamp0.val[1]), vget_low_f32(posSamp0.val[1])); float32x4x2_t posSamp1 = vld2q_f32(sP); sP -= 24; float32x4x2_t negSamp1 = vld2q_f32(sN); sN += 8; posSamp1.val[0] = vrev64q_f32(posSamp1.val[0]); posSamp1.val[1] = vrev64q_f32(posSamp1.val[1]); posSamp1.val[0] = vcombine_f32( vget_high_f32(posSamp1.val[0]), vget_low_f32(posSamp1.val[0])); posSamp1.val[1] = vcombine_f32( vget_high_f32(posSamp1.val[1]), vget_low_f32(posSamp1.val[1])); // Note: speed is affected by accumulation order. // Also, speed appears slower using vmul/vadd instead of vmla for // stereo case, comparable for mono. accum = vmlaq_f32(accum, negSamp0.val[0], negCoef.val[0]); accum = vmlaq_f32(accum, negSamp1.val[0], negCoef.val[1]); accum2 = vmlaq_f32(accum2, negSamp0.val[1], negCoef.val[0]); accum2 = vmlaq_f32(accum2, negSamp1.val[1], negCoef.val[1]); accum = vmlaq_f32(accum, posSamp0.val[0], posCoef.val[1]); // reversed accum = vmlaq_f32(accum, posSamp1.val[0], posCoef.val[0]); // reversed accum2 = vmlaq_f32(accum2, posSamp0.val[1], posCoef.val[1]); // reversed accum2 = vmlaq_f32(accum2, posSamp1.val[1], posCoef.val[0]); // reversed } break; } } while (count -= 8); // multiply by volume and save volumeLR = (const float*)__builtin_assume_aligned(volumeLR, 8); float32x2_t vLR = vld1_f32(volumeLR); float32x2_t outSamp = vld1_f32(out); // combine and funnel down accumulator float32x2_t outAccum = vpadd_f32(vget_low_f32(accum), vget_high_f32(accum)); if (CHANNELS == 1) { // duplicate accum to both L and R outAccum = vpadd_f32(outAccum, outAccum); } else if (CHANNELS == 2) { // accum2 contains R, fold in float32x2_t outAccum2 = vpadd_f32(vget_low_f32(accum2), vget_high_f32(accum2)); outAccum = vpadd_f32(outAccum, outAccum2); } outSamp = vmla_f32(outSamp, outAccum, vLR); vst1_f32(out, outSamp); } template <> inline void ProcessL<1, 16>(int32_t* const out, int count, const int16_t* coefsP, const int16_t* coefsN, const int16_t* sP, const int16_t* sN, const int32_t* const volumeLR) { #ifdef USE_INTRINSIC ProcessNeonIntrinsic<1, 16, true>(out, count, coefsP, coefsN, sP, sN, volumeLR, 0 /*lerpP*/, NULL /*coefsP1*/, NULL /*coefsN1*/); #else const int CHANNELS = 1; // template specialization does not preserve params const int STRIDE = 16; sP -= CHANNELS*((STRIDE>>1)-1); asm ( "veor q0, q0, q0 \n"// (0 - combines+) accumulator = 0 "1: \n" "vld1.16 {q2}, [%[sP]] \n"// (2+0d) load 8 16-bits mono samples "vld1.16 {q3}, [%[sN]]! \n"// (2) load 8 16-bits mono samples "vld1.16 {q8}, [%[coefsP0]:128]! \n"// (1) load 8 16-bits coefs "vld1.16 {q10}, [%[coefsN0]:128]! \n"// (1) load 8 16-bits coefs "vrev64.16 q2, q2 \n"// (1) reverse s3, s2, s1, s0, s7, s6, s5, s4 // reordering the vmal to do d6, d7 before d4, d5 is slower(?) "vmlal.s16 q0, d4, d17 \n"// (1+0d) multiply (reversed)samples by coef "vmlal.s16 q0, d5, d16 \n"// (1) multiply (reversed)samples by coef "vmlal.s16 q0, d6, d20 \n"// (1) multiply neg samples "vmlal.s16 q0, d7, d21 \n"// (1) multiply neg samples // moving these ARM instructions before neon above seems to be slower "subs %[count], %[count], #8 \n"// (1) update loop counter "sub %[sP], %[sP], #16 \n"// (0) move pointer to next set of samples // sP used after branch (warning) "bne 1b \n"// loop ASSEMBLY_ACCUMULATE_MONO : [out] "=Uv" (out[0]), [count] "+r" (count), [coefsP0] "+r" (coefsP), [coefsN0] "+r" (coefsN), [sP] "+r" (sP), [sN] "+r" (sN) : [vLR] "r" (volumeLR) : "cc", "memory", "q0", "q1", "q2", "q3", "q8", "q10" ); #endif } template <> inline void ProcessL<2, 16>(int32_t* const out, int count, const int16_t* coefsP, const int16_t* coefsN, const int16_t* sP, const int16_t* sN, const int32_t* const volumeLR) { #ifdef USE_INTRINSIC ProcessNeonIntrinsic<2, 16, true>(out, count, coefsP, coefsN, sP, sN, volumeLR, 0 /*lerpP*/, NULL /*coefsP1*/, NULL /*coefsN1*/); #else const int CHANNELS = 2; // template specialization does not preserve params const int STRIDE = 16; sP -= CHANNELS*((STRIDE>>1)-1); asm ( "veor q0, q0, q0 \n"// (1) acc_L = 0 "veor q4, q4, q4 \n"// (0 combines+) acc_R = 0 "1: \n" "vld2.16 {q2, q3}, [%[sP]] \n"// (3+0d) load 8 16-bits stereo frames "vld2.16 {q5, q6}, [%[sN]]! \n"// (3) load 8 16-bits stereo frames "vld1.16 {q8}, [%[coefsP0]:128]! \n"// (1) load 8 16-bits coefs "vld1.16 {q10}, [%[coefsN0]:128]! \n"// (1) load 8 16-bits coefs "vrev64.16 q2, q2 \n"// (1) reverse 8 samples of positive left "vrev64.16 q3, q3 \n"// (0 combines+) reverse positive right "vmlal.s16 q0, d4, d17 \n"// (1) multiply (reversed) samples left "vmlal.s16 q0, d5, d16 \n"// (1) multiply (reversed) samples left "vmlal.s16 q4, d6, d17 \n"// (1) multiply (reversed) samples right "vmlal.s16 q4, d7, d16 \n"// (1) multiply (reversed) samples right "vmlal.s16 q0, d10, d20 \n"// (1) multiply samples left "vmlal.s16 q0, d11, d21 \n"// (1) multiply samples left "vmlal.s16 q4, d12, d20 \n"// (1) multiply samples right "vmlal.s16 q4, d13, d21 \n"// (1) multiply samples right // moving these ARM before neon seems to be slower "subs %[count], %[count], #8 \n"// (1) update loop counter "sub %[sP], %[sP], #32 \n"// (0) move pointer to next set of samples // sP used after branch (warning) "bne 1b \n"// loop ASSEMBLY_ACCUMULATE_STEREO : [out] "=Uv" (out[0]), [count] "+r" (count), [coefsP0] "+r" (coefsP), [coefsN0] "+r" (coefsN), [sP] "+r" (sP), [sN] "+r" (sN) : [vLR] "r" (volumeLR) : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q8", "q10" ); #endif } template <> inline void Process<1, 16>(int32_t* const out, int count, const int16_t* coefsP, const int16_t* coefsN, const int16_t* coefsP1, const int16_t* coefsN1, const int16_t* sP, const int16_t* sN, uint32_t lerpP, const int32_t* const volumeLR) { #ifdef USE_INTRINSIC ProcessNeonIntrinsic<1, 16, false>(out, count, coefsP, coefsN, sP, sN, volumeLR, lerpP, coefsP1, coefsN1); #else const int CHANNELS = 1; // template specialization does not preserve params const int STRIDE = 16; sP -= CHANNELS*((STRIDE>>1)-1); asm ( "vmov.32 d2[0], %[lerpP] \n"// load the positive phase S32 Q15 "veor q0, q0, q0 \n"// (0 - combines+) accumulator = 0 "1: \n" "vld1.16 {q2}, [%[sP]] \n"// (2+0d) load 8 16-bits mono samples "vld1.16 {q3}, [%[sN]]! \n"// (2) load 8 16-bits mono samples "vld1.16 {q8}, [%[coefsP0]:128]! \n"// (1) load 8 16-bits coefs "vld1.16 {q9}, [%[coefsP1]:128]! \n"// (1) load 8 16-bits coefs for interpolation "vld1.16 {q10}, [%[coefsN1]:128]! \n"// (1) load 8 16-bits coefs "vld1.16 {q11}, [%[coefsN0]:128]! \n"// (1) load 8 16-bits coefs for interpolation "vsub.s16 q9, q9, q8 \n"// (1) interpolate (step1) 1st set of coefs "vsub.s16 q11, q11, q10 \n"// (1) interpolate (step1) 2nd set of coets "vqrdmulh.s16 q9, q9, d2[0] \n"// (2) interpolate (step2) 1st set of coefs "vqrdmulh.s16 q11, q11, d2[0] \n"// (2) interpolate (step2) 2nd set of coefs "vrev64.16 q2, q2 \n"// (1) reverse s3, s2, s1, s0, s7, s6, s5, s4 "vadd.s16 q8, q8, q9 \n"// (1+2d) interpolate (step3) 1st set "vadd.s16 q10, q10, q11 \n"// (1+1d) interpolate (step3) 2nd set // reordering the vmal to do d6, d7 before d4, d5 is slower(?) "vmlal.s16 q0, d4, d17 \n"// (1+0d) multiply reversed samples by coef "vmlal.s16 q0, d5, d16 \n"// (1) multiply reversed samples by coef "vmlal.s16 q0, d6, d20 \n"// (1) multiply neg samples "vmlal.s16 q0, d7, d21 \n"// (1) multiply neg samples // moving these ARM instructions before neon above seems to be slower "subs %[count], %[count], #8 \n"// (1) update loop counter "sub %[sP], %[sP], #16 \n"// (0) move pointer to next set of samples // sP used after branch (warning) "bne 1b \n"// loop ASSEMBLY_ACCUMULATE_MONO : [out] "=Uv" (out[0]), [count] "+r" (count), [coefsP0] "+r" (coefsP), [coefsN0] "+r" (coefsN), [coefsP1] "+r" (coefsP1), [coefsN1] "+r" (coefsN1), [sP] "+r" (sP), [sN] "+r" (sN) : [lerpP] "r" (lerpP), [vLR] "r" (volumeLR) : "cc", "memory", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11" ); #endif } template <> inline void Process<2, 16>(int32_t* const out, int count, const int16_t* coefsP, const int16_t* coefsN, const int16_t* coefsP1, const int16_t* coefsN1, const int16_t* sP, const int16_t* sN, uint32_t lerpP, const int32_t* const volumeLR) { #ifdef USE_INTRINSIC ProcessNeonIntrinsic<2, 16, false>(out, count, coefsP, coefsN, sP, sN, volumeLR, lerpP, coefsP1, coefsN1); #else const int CHANNELS = 2; // template specialization does not preserve params const int STRIDE = 16; sP -= CHANNELS*((STRIDE>>1)-1); asm ( "vmov.32 d2[0], %[lerpP] \n"// load the positive phase "veor q0, q0, q0 \n"// (1) acc_L = 0 "veor q4, q4, q4 \n"// (0 combines+) acc_R = 0 "1: \n" "vld2.16 {q2, q3}, [%[sP]] \n"// (3+0d) load 8 16-bits stereo frames "vld2.16 {q5, q6}, [%[sN]]! \n"// (3) load 8 16-bits stereo frames "vld1.16 {q8}, [%[coefsP0]:128]! \n"// (1) load 8 16-bits coefs "vld1.16 {q9}, [%[coefsP1]:128]! \n"// (1) load 8 16-bits coefs for interpolation "vld1.16 {q10}, [%[coefsN1]:128]! \n"// (1) load 8 16-bits coefs "vld1.16 {q11}, [%[coefsN0]:128]! \n"// (1) load 8 16-bits coefs for interpolation "vsub.s16 q9, q9, q8 \n"// (1) interpolate (step1) 1st set of coefs "vsub.s16 q11, q11, q10 \n"// (1) interpolate (step1) 2nd set of coets "vqrdmulh.s16 q9, q9, d2[0] \n"// (2) interpolate (step2) 1st set of coefs "vqrdmulh.s16 q11, q11, d2[0] \n"// (2) interpolate (step2) 2nd set of coefs "vrev64.16 q2, q2 \n"// (1) reverse 8 samples of positive left "vrev64.16 q3, q3 \n"// (1) reverse 8 samples of positive right "vadd.s16 q8, q8, q9 \n"// (1+1d) interpolate (step3) 1st set "vadd.s16 q10, q10, q11 \n"// (1+1d) interpolate (step3) 2nd set "vmlal.s16 q0, d4, d17 \n"// (1) multiply reversed samples left "vmlal.s16 q0, d5, d16 \n"// (1) multiply reversed samples left "vmlal.s16 q4, d6, d17 \n"// (1) multiply reversed samples right "vmlal.s16 q4, d7, d16 \n"// (1) multiply reversed samples right "vmlal.s16 q0, d10, d20 \n"// (1) multiply samples left "vmlal.s16 q0, d11, d21 \n"// (1) multiply samples left "vmlal.s16 q4, d12, d20 \n"// (1) multiply samples right "vmlal.s16 q4, d13, d21 \n"// (1) multiply samples right // moving these ARM before neon seems to be slower "subs %[count], %[count], #8 \n"// (1) update loop counter "sub %[sP], %[sP], #32 \n"// (0) move pointer to next set of samples // sP used after branch (warning) "bne 1b \n"// loop ASSEMBLY_ACCUMULATE_STEREO : [out] "=Uv" (out[0]), [count] "+r" (count), [coefsP0] "+r" (coefsP), [coefsN0] "+r" (coefsN), [coefsP1] "+r" (coefsP1), [coefsN1] "+r" (coefsN1), [sP] "+r" (sP), [sN] "+r" (sN) : [lerpP] "r" (lerpP), [vLR] "r" (volumeLR) : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q8", "q9", "q10", "q11" ); #endif } template <> inline void ProcessL<1, 16>(int32_t* const out, int count, const int32_t* coefsP, const int32_t* coefsN, const int16_t* sP, const int16_t* sN, const int32_t* const volumeLR) { #ifdef USE_INTRINSIC ProcessNeonIntrinsic<1, 16, true>(out, count, coefsP, coefsN, sP, sN, volumeLR, 0 /*lerpP*/, NULL /*coefsP1*/, NULL /*coefsN1*/); #else const int CHANNELS = 1; // template specialization does not preserve params const int STRIDE = 16; sP -= CHANNELS*((STRIDE>>1)-1); asm ( "veor q0, q0, q0 \n"// result, initialize to 0 "1: \n" "vld1.16 {q2}, [%[sP]] \n"// load 8 16-bits mono samples "vld1.16 {q3}, [%[sN]]! \n"// load 8 16-bits mono samples "vld1.32 {q8, q9}, [%[coefsP0]:128]! \n"// load 8 32-bits coefs "vld1.32 {q10, q11}, [%[coefsN0]:128]! \n"// load 8 32-bits coefs "vrev64.16 q2, q2 \n"// reverse 8 samples of the positive side "vshll.s16 q12, d4, #15 \n"// extend samples to 31 bits "vshll.s16 q13, d5, #15 \n"// extend samples to 31 bits "vshll.s16 q14, d6, #15 \n"// extend samples to 31 bits "vshll.s16 q15, d7, #15 \n"// extend samples to 31 bits "vqrdmulh.s32 q12, q12, q9 \n"// multiply samples "vqrdmulh.s32 q13, q13, q8 \n"// multiply samples "vqrdmulh.s32 q14, q14, q10 \n"// multiply samples "vqrdmulh.s32 q15, q15, q11 \n"// multiply samples "vadd.s32 q0, q0, q12 \n"// accumulate result "vadd.s32 q13, q13, q14 \n"// accumulate result "vadd.s32 q0, q0, q15 \n"// accumulate result "vadd.s32 q0, q0, q13 \n"// accumulate result "sub %[sP], %[sP], #16 \n"// move pointer to next set of samples "subs %[count], %[count], #8 \n"// update loop counter "bne 1b \n"// loop ASSEMBLY_ACCUMULATE_MONO : [out] "=Uv" (out[0]), [count] "+r" (count), [coefsP0] "+r" (coefsP), [coefsN0] "+r" (coefsN), [sP] "+r" (sP), [sN] "+r" (sN) : [vLR] "r" (volumeLR) : "cc", "memory", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15" ); #endif } template <> inline void ProcessL<2, 16>(int32_t* const out, int count, const int32_t* coefsP, const int32_t* coefsN, const int16_t* sP, const int16_t* sN, const int32_t* const volumeLR) { #ifdef USE_INTRINSIC ProcessNeonIntrinsic<2, 16, true>(out, count, coefsP, coefsN, sP, sN, volumeLR, 0 /*lerpP*/, NULL /*coefsP1*/, NULL /*coefsN1*/); #else const int CHANNELS = 2; // template specialization does not preserve params const int STRIDE = 16; sP -= CHANNELS*((STRIDE>>1)-1); asm ( "veor q0, q0, q0 \n"// result, initialize to 0 "veor q4, q4, q4 \n"// result, initialize to 0 "1: \n" "vld2.16 {q2, q3}, [%[sP]] \n"// load 8 16-bits stereo frames "vld2.16 {q5, q6}, [%[sN]]! \n"// load 8 16-bits stereo frames "vld1.32 {q8, q9}, [%[coefsP0]:128]! \n"// load 8 32-bits coefs "vld1.32 {q10, q11}, [%[coefsN0]:128]! \n"// load 8 32-bits coefs "vrev64.16 q2, q2 \n"// reverse 8 samples of positive left "vrev64.16 q3, q3 \n"// reverse 8 samples of positive right "vshll.s16 q12, d4, #15 \n"// extend samples to 31 bits "vshll.s16 q13, d5, #15 \n"// extend samples to 31 bits "vshll.s16 q14, d10, #15 \n"// extend samples to 31 bits "vshll.s16 q15, d11, #15 \n"// extend samples to 31 bits "vqrdmulh.s32 q12, q12, q9 \n"// multiply samples by coef "vqrdmulh.s32 q13, q13, q8 \n"// multiply samples by coef "vqrdmulh.s32 q14, q14, q10 \n"// multiply samples by coef "vqrdmulh.s32 q15, q15, q11 \n"// multiply samples by coef "vadd.s32 q0, q0, q12 \n"// accumulate result "vadd.s32 q13, q13, q14 \n"// accumulate result "vadd.s32 q0, q0, q15 \n"// accumulate result "vadd.s32 q0, q0, q13 \n"// accumulate result "vshll.s16 q12, d6, #15 \n"// extend samples to 31 bits "vshll.s16 q13, d7, #15 \n"// extend samples to 31 bits "vshll.s16 q14, d12, #15 \n"// extend samples to 31 bits "vshll.s16 q15, d13, #15 \n"// extend samples to 31 bits "vqrdmulh.s32 q12, q12, q9 \n"// multiply samples by coef "vqrdmulh.s32 q13, q13, q8 \n"// multiply samples by coef "vqrdmulh.s32 q14, q14, q10 \n"// multiply samples by coef "vqrdmulh.s32 q15, q15, q11 \n"// multiply samples by coef "vadd.s32 q4, q4, q12 \n"// accumulate result "vadd.s32 q13, q13, q14 \n"// accumulate result "vadd.s32 q4, q4, q15 \n"// accumulate result "vadd.s32 q4, q4, q13 \n"// accumulate result "subs %[count], %[count], #8 \n"// update loop counter "sub %[sP], %[sP], #32 \n"// move pointer to next set of samples "bne 1b \n"// loop ASSEMBLY_ACCUMULATE_STEREO : [out] "=Uv" (out[0]), [count] "+r" (count), [coefsP0] "+r" (coefsP), [coefsN0] "+r" (coefsN), [sP] "+r" (sP), [sN] "+r" (sN) : [vLR] "r" (volumeLR) : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15" ); #endif } template <> inline void Process<1, 16>(int32_t* const out, int count, const int32_t* coefsP, const int32_t* coefsN, const int32_t* coefsP1, const int32_t* coefsN1, const int16_t* sP, const int16_t* sN, uint32_t lerpP, const int32_t* const volumeLR) { #ifdef USE_INTRINSIC ProcessNeonIntrinsic<1, 16, false>(out, count, coefsP, coefsN, sP, sN, volumeLR, lerpP, coefsP1, coefsN1); #else const int CHANNELS = 1; // template specialization does not preserve params const int STRIDE = 16; sP -= CHANNELS*((STRIDE>>1)-1); asm ( "vmov.32 d2[0], %[lerpP] \n"// load the positive phase "veor q0, q0, q0 \n"// result, initialize to 0 "1: \n" "vld1.16 {q2}, [%[sP]] \n"// load 8 16-bits mono samples "vld1.16 {q3}, [%[sN]]! \n"// load 8 16-bits mono samples "vld1.32 {q8, q9}, [%[coefsP0]:128]! \n"// load 8 32-bits coefs "vld1.32 {q12, q13}, [%[coefsP1]:128]! \n"// load 8 32-bits coefs "vld1.32 {q10, q11}, [%[coefsN1]:128]! \n"// load 8 32-bits coefs "vld1.32 {q14, q15}, [%[coefsN0]:128]! \n"// load 8 32-bits coefs "vsub.s32 q12, q12, q8 \n"// interpolate (step1) "vsub.s32 q13, q13, q9 \n"// interpolate (step1) "vsub.s32 q14, q14, q10 \n"// interpolate (step1) "vsub.s32 q15, q15, q11 \n"// interpolate (step1) "vqrdmulh.s32 q12, q12, d2[0] \n"// interpolate (step2) "vqrdmulh.s32 q13, q13, d2[0] \n"// interpolate (step2) "vqrdmulh.s32 q14, q14, d2[0] \n"// interpolate (step2) "vqrdmulh.s32 q15, q15, d2[0] \n"// interpolate (step2) "vadd.s32 q8, q8, q12 \n"// interpolate (step3) "vadd.s32 q9, q9, q13 \n"// interpolate (step3) "vadd.s32 q10, q10, q14 \n"// interpolate (step3) "vadd.s32 q11, q11, q15 \n"// interpolate (step3) "vrev64.16 q2, q2 \n"// reverse 8 samples of the positive side "vshll.s16 q12, d4, #15 \n"// extend samples to 31 bits "vshll.s16 q13, d5, #15 \n"// extend samples to 31 bits "vshll.s16 q14, d6, #15 \n"// extend samples to 31 bits "vshll.s16 q15, d7, #15 \n"// extend samples to 31 bits "vqrdmulh.s32 q12, q12, q9 \n"// multiply samples by interpolated coef "vqrdmulh.s32 q13, q13, q8 \n"// multiply samples by interpolated coef "vqrdmulh.s32 q14, q14, q10 \n"// multiply samples by interpolated coef "vqrdmulh.s32 q15, q15, q11 \n"// multiply samples by interpolated coef "vadd.s32 q0, q0, q12 \n"// accumulate result "vadd.s32 q13, q13, q14 \n"// accumulate result "vadd.s32 q0, q0, q15 \n"// accumulate result "vadd.s32 q0, q0, q13 \n"// accumulate result "sub %[sP], %[sP], #16 \n"// move pointer to next set of samples "subs %[count], %[count], #8 \n"// update loop counter "bne 1b \n"// loop ASSEMBLY_ACCUMULATE_MONO : [out] "=Uv" (out[0]), [count] "+r" (count), [coefsP0] "+r" (coefsP), [coefsN0] "+r" (coefsN), [coefsP1] "+r" (coefsP1), [coefsN1] "+r" (coefsN1), [sP] "+r" (sP), [sN] "+r" (sN) : [lerpP] "r" (lerpP), [vLR] "r" (volumeLR) : "cc", "memory", "q0", "q1", "q2", "q3", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15" ); #endif } template <> inline void Process<2, 16>(int32_t* const out, int count, const int32_t* coefsP, const int32_t* coefsN, const int32_t* coefsP1, const int32_t* coefsN1, const int16_t* sP, const int16_t* sN, uint32_t lerpP, const int32_t* const volumeLR) { #ifdef USE_INTRINSIC ProcessNeonIntrinsic<2, 16, false>(out, count, coefsP, coefsN, sP, sN, volumeLR, lerpP, coefsP1, coefsN1); #else const int CHANNELS = 2; // template specialization does not preserve params const int STRIDE = 16; sP -= CHANNELS*((STRIDE>>1)-1); asm ( "vmov.32 d2[0], %[lerpP] \n"// load the positive phase "veor q0, q0, q0 \n"// result, initialize to 0 "veor q4, q4, q4 \n"// result, initialize to 0 "1: \n" "vld2.16 {q2, q3}, [%[sP]] \n"// load 8 16-bits stereo frames "vld2.16 {q5, q6}, [%[sN]]! \n"// load 8 16-bits stereo frames "vld1.32 {q8, q9}, [%[coefsP0]:128]! \n"// load 8 32-bits coefs "vld1.32 {q12, q13}, [%[coefsP1]:128]! \n"// load 8 32-bits coefs "vld1.32 {q10, q11}, [%[coefsN1]:128]! \n"// load 8 32-bits coefs "vld1.32 {q14, q15}, [%[coefsN0]:128]! \n"// load 8 32-bits coefs "vsub.s32 q12, q12, q8 \n"// interpolate (step1) "vsub.s32 q13, q13, q9 \n"// interpolate (step1) "vsub.s32 q14, q14, q10 \n"// interpolate (step1) "vsub.s32 q15, q15, q11 \n"// interpolate (step1) "vqrdmulh.s32 q12, q12, d2[0] \n"// interpolate (step2) "vqrdmulh.s32 q13, q13, d2[0] \n"// interpolate (step2) "vqrdmulh.s32 q14, q14, d2[0] \n"// interpolate (step2) "vqrdmulh.s32 q15, q15, d2[0] \n"// interpolate (step2) "vadd.s32 q8, q8, q12 \n"// interpolate (step3) "vadd.s32 q9, q9, q13 \n"// interpolate (step3) "vadd.s32 q10, q10, q14 \n"// interpolate (step3) "vadd.s32 q11, q11, q15 \n"// interpolate (step3) "vrev64.16 q2, q2 \n"// reverse 8 samples of positive left "vrev64.16 q3, q3 \n"// reverse 8 samples of positive right "vshll.s16 q12, d4, #15 \n"// extend samples to 31 bits "vshll.s16 q13, d5, #15 \n"// extend samples to 31 bits "vshll.s16 q14, d10, #15 \n"// extend samples to 31 bits "vshll.s16 q15, d11, #15 \n"// extend samples to 31 bits "vqrdmulh.s32 q12, q12, q9 \n"// multiply samples by interpolated coef "vqrdmulh.s32 q13, q13, q8 \n"// multiply samples by interpolated coef "vqrdmulh.s32 q14, q14, q10 \n"// multiply samples by interpolated coef "vqrdmulh.s32 q15, q15, q11 \n"// multiply samples by interpolated coef "vadd.s32 q0, q0, q12 \n"// accumulate result "vadd.s32 q13, q13, q14 \n"// accumulate result "vadd.s32 q0, q0, q15 \n"// accumulate result "vadd.s32 q0, q0, q13 \n"// accumulate result "vshll.s16 q12, d6, #15 \n"// extend samples to 31 bits "vshll.s16 q13, d7, #15 \n"// extend samples to 31 bits "vshll.s16 q14, d12, #15 \n"// extend samples to 31 bits "vshll.s16 q15, d13, #15 \n"// extend samples to 31 bits "vqrdmulh.s32 q12, q12, q9 \n"// multiply samples by interpolated coef "vqrdmulh.s32 q13, q13, q8 \n"// multiply samples by interpolated coef "vqrdmulh.s32 q14, q14, q10 \n"// multiply samples by interpolated coef "vqrdmulh.s32 q15, q15, q11 \n"// multiply samples by interpolated coef "vadd.s32 q4, q4, q12 \n"// accumulate result "vadd.s32 q13, q13, q14 \n"// accumulate result "vadd.s32 q4, q4, q15 \n"// accumulate result "vadd.s32 q4, q4, q13 \n"// accumulate result "subs %[count], %[count], #8 \n"// update loop counter "sub %[sP], %[sP], #32 \n"// move pointer to next set of samples "bne 1b \n"// loop ASSEMBLY_ACCUMULATE_STEREO : [out] "=Uv" (out[0]), [count] "+r" (count), [coefsP0] "+r" (coefsP), [coefsN0] "+r" (coefsN), [coefsP1] "+r" (coefsP1), [coefsN1] "+r" (coefsN1), [sP] "+r" (sP), [sN] "+r" (sN) : [lerpP] "r" (lerpP), [vLR] "r" (volumeLR) : "cc", "memory", "q0", "q1", "q2", "q3", "q4", "q5", "q6", "q8", "q9", "q10", "q11", "q12", "q13", "q14", "q15" ); #endif } template<> inline void ProcessL<1, 16>(float* const out, int count, const float* coefsP, const float* coefsN, const float* sP, const float* sN, const float* const volumeLR) { ProcessNeonIntrinsic<1, 16, true>(out, count, coefsP, coefsN, sP, sN, volumeLR, 0 /*lerpP*/, NULL /*coefsP1*/, NULL /*coefsN1*/); } template<> inline void ProcessL<2, 16>(float* const out, int count, const float* coefsP, const float* coefsN, const float* sP, const float* sN, const float* const volumeLR) { ProcessNeonIntrinsic<2, 16, true>(out, count, coefsP, coefsN, sP, sN, volumeLR, 0 /*lerpP*/, NULL /*coefsP1*/, NULL /*coefsN1*/); } template<> inline void Process<1, 16>(float* const out, int count, const float* coefsP, const float* coefsN, const float* coefsP1, const float* coefsN1, const float* sP, const float* sN, float lerpP, const float* const volumeLR) { ProcessNeonIntrinsic<1, 16, false>(out, count, coefsP, coefsN, sP, sN, volumeLR, lerpP, coefsP1, coefsN1); } template<> inline void Process<2, 16>(float* const out, int count, const float* coefsP, const float* coefsN, const float* coefsP1, const float* coefsN1, const float* sP, const float* sN, float lerpP, const float* const volumeLR) { ProcessNeonIntrinsic<2, 16, false>(out, count, coefsP, coefsN, sP, sN, volumeLR, lerpP, coefsP1, coefsN1); } #endif //USE_NEON } // namespace android #endif /*ANDROID_AUDIO_RESAMPLER_FIR_PROCESS_NEON_H*/