/****************************************************************************** * * Copyright (C) 2012 Ittiam Systems Pvt Ltd, Bangalore * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at: * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * ******************************************************************************/ /** ******************************************************************************* * @file * ihevc_weighted_pred_x86_intr.c * * @brief * Contains function definitions for weighted prediction used in inter * prediction * * @author * * * @par List of Functions: * - ihevc_weighted_pred_uni_sse42() * - ihevc_weighted_pred_bi_sse42() * - ihevc_weighted_pred_bi_default_sse42() * - ihevc_weighted_pred_chroma_uni_sse42() * - ihevc_weighted_pred_chroma_bi_sse42() * * @remarks * None * ******************************************************************************* */ /*****************************************************************************/ /* File Includes */ /*****************************************************************************/ #include #include #include "ihevc_debug.h" #include "ihevc_typedefs.h" #include "ihevc_macros.h" #include "ihevc_platform_macros.h" #include "ihevc_func_selector.h" #include "ihevc_defs.h" #include "ihevc_weighted_pred.h" #include "ihevc_inter_pred.h" #include /** ******************************************************************************* * * @brief * Does uni-weighted prediction on the array pointed by pi2_src and stores * it at the location pointed by pi2_dst * * @par Description: * dst = ( (src + lvl_shift) * wgt0 + (1 << (shift - 1)) ) >> shift + * offset * * @param[in] pi2_src * Pointer to the source * * @param[out] pu1_dst * Pointer to the destination * * @param[in] src_strd * Source stride * * @param[in] dst_strd * Destination stride * * @param[in] wgt0 * weight to be multiplied to the source * * @param[in] off0 * offset to be added after rounding and * * @param[in] shifting * * * @param[in] shift * (14 Bit depth) + log2_weight_denominator * * @param[in] lvl_shift * added before shift and offset * * @param[in] ht * height of the source * * @param[in] wd * width of the source * * @returns * * @remarks * None * ******************************************************************************* */ void ihevc_weighted_pred_uni_sse42(WORD16 *pi2_src, UWORD8 *pu1_dst, WORD32 src_strd, WORD32 dst_strd, WORD32 wgt0, WORD32 off0, WORD32 shift, WORD32 lvl_shift, WORD32 ht, WORD32 wd) { WORD32 row, col, temp; WORD32 dst0, dst1, dst2, dst3; /* all 128 bit registers are named with a suffix mxnb, where m is the */ /* number of n bits packed in the register */ __m128i src_temp0_4x32b, src_temp1_4x32b, src_temp2_4x32b, src_temp3_4x32b; __m128i const_temp_4x32b, lvl_shift_4x32b, wgt0_4x32b, off0_4x32b; ASSERT(wd % 4 == 0); /* checking assumption*/ ASSERT(ht % 4 == 0); /* checking assumption*/ temp = 1 << (shift - 1); // seting values in register const_temp_4x32b = _mm_set1_epi32(temp); lvl_shift_4x32b = _mm_set1_epi32(lvl_shift); wgt0_4x32b = _mm_set1_epi32(wgt0); off0_4x32b = _mm_set1_epi32(off0); if(0 == (wd & 7)) /* wd multiple of 8 case */ { __m128i src_temp4_4x32b, src_temp5_4x32b, src_temp6_4x32b, src_temp7_4x32b; /* outer for loop starts from here */ for(row = 0; row < ht; row += 4) { for(col = 0; col < wd; col += 8) { /* for row =0 ,1,2,3*/ /* row = 0 */ /*load 8 pixel values from 7:0 pos. relative to cur. pos.*/ src_temp0_4x32b = _mm_loadu_si128((__m128i *)(pi2_src)); /* row = 1 */ src_temp1_4x32b = _mm_loadu_si128((__m128i *)(pi2_src + src_strd)); /* row = 2 */ src_temp2_4x32b = _mm_loadu_si128((__m128i *)(pi2_src + 2 * src_strd)); /* row = 3 */ src_temp3_4x32b = _mm_loadu_si128((__m128i *)(pi2_src + 3 * src_strd)); /* row = 0 */ /* Last 4 pixels */ src_temp4_4x32b = _mm_loadu_si128((__m128i *)(pi2_src + 4)); /* row = 1 */ src_temp5_4x32b = _mm_loadu_si128((__m128i *)(pi2_src + src_strd + 4)); /* row = 2 */ src_temp6_4x32b = _mm_loadu_si128((__m128i *)(pi2_src + 2 * src_strd + 4)); /* row = 3 */ src_temp7_4x32b = _mm_loadu_si128((__m128i *)(pi2_src + 3 * src_strd + 4)); /* considering pix. 4:0 by converting 16-into 32 bit */ /* First 4 pixels */ src_temp0_4x32b = _mm_cvtepi16_epi32(src_temp0_4x32b); src_temp1_4x32b = _mm_cvtepi16_epi32(src_temp1_4x32b); src_temp2_4x32b = _mm_cvtepi16_epi32(src_temp2_4x32b); src_temp3_4x32b = _mm_cvtepi16_epi32(src_temp3_4x32b); /* (pi2_src[col] + lvl_shift)*/ /* First 4 pixels */ src_temp0_4x32b = _mm_add_epi32(src_temp0_4x32b, lvl_shift_4x32b); src_temp1_4x32b = _mm_add_epi32(src_temp1_4x32b, lvl_shift_4x32b); src_temp2_4x32b = _mm_add_epi32(src_temp2_4x32b, lvl_shift_4x32b); src_temp3_4x32b = _mm_add_epi32(src_temp3_4x32b, lvl_shift_4x32b); /*i4_tmp = (pi2_src[col] + lvl_shift) * wgt0*/ /* First 4 pixels */ src_temp0_4x32b = _mm_mullo_epi32(src_temp0_4x32b, wgt0_4x32b); src_temp1_4x32b = _mm_mullo_epi32(src_temp1_4x32b, wgt0_4x32b); src_temp2_4x32b = _mm_mullo_epi32(src_temp2_4x32b, wgt0_4x32b); src_temp3_4x32b = _mm_mullo_epi32(src_temp3_4x32b, wgt0_4x32b); /* considering pix. 4:0 by converting 16-into 32 bit */ /* Last 4 pixels */ src_temp4_4x32b = _mm_cvtepi16_epi32(src_temp4_4x32b); src_temp5_4x32b = _mm_cvtepi16_epi32(src_temp5_4x32b); src_temp6_4x32b = _mm_cvtepi16_epi32(src_temp6_4x32b); src_temp7_4x32b = _mm_cvtepi16_epi32(src_temp7_4x32b); /* (pi2_src[col] + lvl_shift)*/ /* Last 4 pixels */ src_temp4_4x32b = _mm_add_epi32(src_temp4_4x32b, lvl_shift_4x32b); src_temp5_4x32b = _mm_add_epi32(src_temp5_4x32b, lvl_shift_4x32b); src_temp6_4x32b = _mm_add_epi32(src_temp6_4x32b, lvl_shift_4x32b); src_temp7_4x32b = _mm_add_epi32(src_temp7_4x32b, lvl_shift_4x32b); /*i4_tmp = (pi2_src[col] + lvl_shift) * wgt0*/ /* Last 4 pixels */ src_temp4_4x32b = _mm_mullo_epi32(src_temp4_4x32b, wgt0_4x32b); src_temp5_4x32b = _mm_mullo_epi32(src_temp5_4x32b, wgt0_4x32b); src_temp6_4x32b = _mm_mullo_epi32(src_temp6_4x32b, wgt0_4x32b); src_temp7_4x32b = _mm_mullo_epi32(src_temp7_4x32b, wgt0_4x32b); /* i4_tmp += 1 << (shift - 1) */ /* First 4 pixels */ src_temp0_4x32b = _mm_add_epi32(src_temp0_4x32b, const_temp_4x32b); src_temp1_4x32b = _mm_add_epi32(src_temp1_4x32b, const_temp_4x32b); src_temp2_4x32b = _mm_add_epi32(src_temp2_4x32b, const_temp_4x32b); src_temp3_4x32b = _mm_add_epi32(src_temp3_4x32b, const_temp_4x32b); /* (i4_tmp >> shift) */ /* First 4 pixels */ src_temp0_4x32b = _mm_srai_epi32(src_temp0_4x32b, shift); src_temp1_4x32b = _mm_srai_epi32(src_temp1_4x32b, shift); src_temp2_4x32b = _mm_srai_epi32(src_temp2_4x32b, shift); src_temp3_4x32b = _mm_srai_epi32(src_temp3_4x32b, shift); /* i4_tmp += 1 << (shift - 1) */ /* Last 4 pixels */ src_temp4_4x32b = _mm_add_epi32(src_temp4_4x32b, const_temp_4x32b); src_temp5_4x32b = _mm_add_epi32(src_temp5_4x32b, const_temp_4x32b); src_temp6_4x32b = _mm_add_epi32(src_temp6_4x32b, const_temp_4x32b); src_temp7_4x32b = _mm_add_epi32(src_temp7_4x32b, const_temp_4x32b); /* (i4_tmp >> shift) */ /* Last 4 pixels */ src_temp4_4x32b = _mm_srai_epi32(src_temp4_4x32b, shift); src_temp5_4x32b = _mm_srai_epi32(src_temp5_4x32b, shift); src_temp6_4x32b = _mm_srai_epi32(src_temp6_4x32b, shift); src_temp7_4x32b = _mm_srai_epi32(src_temp7_4x32b, shift); /*i4_tmp = (i4_tmp >> shift) + off0; */ /* First 4 pixels */ src_temp0_4x32b = _mm_add_epi32(src_temp0_4x32b, off0_4x32b); src_temp1_4x32b = _mm_add_epi32(src_temp1_4x32b, off0_4x32b); src_temp2_4x32b = _mm_add_epi32(src_temp2_4x32b, off0_4x32b); src_temp3_4x32b = _mm_add_epi32(src_temp3_4x32b, off0_4x32b); /*i4_tmp = (i4_tmp >> shift) + off0; */ /* Last 4 pixels */ src_temp4_4x32b = _mm_add_epi32(src_temp4_4x32b, off0_4x32b); src_temp5_4x32b = _mm_add_epi32(src_temp5_4x32b, off0_4x32b); src_temp6_4x32b = _mm_add_epi32(src_temp6_4x32b, off0_4x32b); src_temp7_4x32b = _mm_add_epi32(src_temp7_4x32b, off0_4x32b); src_temp0_4x32b = _mm_packs_epi32(src_temp0_4x32b, src_temp4_4x32b); src_temp1_4x32b = _mm_packs_epi32(src_temp1_4x32b, src_temp5_4x32b); src_temp2_4x32b = _mm_packs_epi32(src_temp2_4x32b, src_temp6_4x32b); src_temp3_4x32b = _mm_packs_epi32(src_temp3_4x32b, src_temp7_4x32b); /* pu1_dst[col] = CLIP_U8(i4_tmp); */ src_temp0_4x32b = _mm_packus_epi16(src_temp0_4x32b, src_temp0_4x32b); src_temp1_4x32b = _mm_packus_epi16(src_temp1_4x32b, src_temp1_4x32b); src_temp2_4x32b = _mm_packus_epi16(src_temp2_4x32b, src_temp2_4x32b); src_temp3_4x32b = _mm_packus_epi16(src_temp3_4x32b, src_temp3_4x32b); /* store four 8-bit output values */ _mm_storel_epi64((__m128i *)(pu1_dst + 0 * dst_strd), src_temp0_4x32b); /* row = 0*/ _mm_storel_epi64((__m128i *)(pu1_dst + 1 * dst_strd), src_temp1_4x32b); /* row = 2*/ _mm_storel_epi64((__m128i *)(pu1_dst + 2 * dst_strd), src_temp2_4x32b); /* row = 1*/ _mm_storel_epi64((__m128i *)(pu1_dst + 3 * dst_strd), src_temp3_4x32b); /* row = 3*/ /* To update pointer */ pi2_src += 8; pu1_dst += 8; } /* inner loop ends here(4-output values in single iteration) */ pi2_src = pi2_src - wd + 4 * src_strd; /* Pointer update */ pu1_dst = pu1_dst - wd + 4 * dst_strd; /* Pointer update */ } } else /* wd multiple of 4 case */ { /* outer for loop starts from here */ for(row = 0; row < ht; row += 4) { for(col = 0; col < wd; col += 4) { /* for row =0 ,1,2,3*/ /* row = 0 */ /*load 8 pixel values from 7:0 pos. relative to cur. pos.*/ src_temp0_4x32b = _mm_loadu_si128((__m128i *)(pi2_src)); /* row = 1 */ src_temp1_4x32b = _mm_loadu_si128((__m128i *)(pi2_src + src_strd)); /* row = 2 */ src_temp2_4x32b = _mm_loadu_si128((__m128i *)(pi2_src + 2 * src_strd)); /* row = 3 */ src_temp3_4x32b = _mm_loadu_si128((__m128i *)(pi2_src + 3 * src_strd)); /* considering pix. 4:0 by converting 16-into 32 bit */ src_temp0_4x32b = _mm_cvtepi16_epi32(src_temp0_4x32b); src_temp1_4x32b = _mm_cvtepi16_epi32(src_temp1_4x32b); src_temp2_4x32b = _mm_cvtepi16_epi32(src_temp2_4x32b); src_temp3_4x32b = _mm_cvtepi16_epi32(src_temp3_4x32b); /* (pi2_src[col] + lvl_shift)*/ src_temp0_4x32b = _mm_add_epi32(src_temp0_4x32b, lvl_shift_4x32b); src_temp1_4x32b = _mm_add_epi32(src_temp1_4x32b, lvl_shift_4x32b); src_temp2_4x32b = _mm_add_epi32(src_temp2_4x32b, lvl_shift_4x32b); src_temp3_4x32b = _mm_add_epi32(src_temp3_4x32b, lvl_shift_4x32b); /*i4_tmp = (pi2_src[col] + lvl_shift) * wgt0*/ src_temp0_4x32b = _mm_mullo_epi32(src_temp0_4x32b, wgt0_4x32b); src_temp1_4x32b = _mm_mullo_epi32(src_temp1_4x32b, wgt0_4x32b); src_temp2_4x32b = _mm_mullo_epi32(src_temp2_4x32b, wgt0_4x32b); src_temp3_4x32b = _mm_mullo_epi32(src_temp3_4x32b, wgt0_4x32b); /* i4_tmp += 1 << (shift - 1) */ src_temp0_4x32b = _mm_add_epi32(src_temp0_4x32b, const_temp_4x32b); src_temp1_4x32b = _mm_add_epi32(src_temp1_4x32b, const_temp_4x32b); src_temp2_4x32b = _mm_add_epi32(src_temp2_4x32b, const_temp_4x32b); src_temp3_4x32b = _mm_add_epi32(src_temp3_4x32b, const_temp_4x32b); /* (i4_tmp >> shift) */ src_temp0_4x32b = _mm_srai_epi32(src_temp0_4x32b, shift); src_temp1_4x32b = _mm_srai_epi32(src_temp1_4x32b, shift); src_temp2_4x32b = _mm_srai_epi32(src_temp2_4x32b, shift); src_temp3_4x32b = _mm_srai_epi32(src_temp3_4x32b, shift); /*i4_tmp = (i4_tmp >> shift) + off0; */ src_temp0_4x32b = _mm_add_epi32(src_temp0_4x32b, off0_4x32b); src_temp1_4x32b = _mm_add_epi32(src_temp1_4x32b, off0_4x32b); src_temp2_4x32b = _mm_add_epi32(src_temp2_4x32b, off0_4x32b); src_temp3_4x32b = _mm_add_epi32(src_temp3_4x32b, off0_4x32b); src_temp0_4x32b = _mm_packs_epi32(src_temp0_4x32b, src_temp1_4x32b); src_temp2_4x32b = _mm_packs_epi32(src_temp2_4x32b, src_temp3_4x32b); /* pu1_dst[col] = CLIP_U8(i4_tmp); */ src_temp0_4x32b = _mm_packus_epi16(src_temp0_4x32b, src_temp2_4x32b); dst0 = _mm_cvtsi128_si32(src_temp0_4x32b); /* dst row = 1 to 3 */ src_temp1_4x32b = _mm_shuffle_epi32(src_temp0_4x32b, 1); src_temp2_4x32b = _mm_shuffle_epi32(src_temp0_4x32b, 2); src_temp3_4x32b = _mm_shuffle_epi32(src_temp0_4x32b, 3); /* store four 8-bit output values */ *(WORD32 *)(&pu1_dst[0 * dst_strd]) = dst0; dst1 = _mm_cvtsi128_si32(src_temp1_4x32b); dst2 = _mm_cvtsi128_si32(src_temp2_4x32b); dst3 = _mm_cvtsi128_si32(src_temp3_4x32b); /* row = 1 to row = 3 */ *(WORD32 *)(&pu1_dst[1 * dst_strd]) = dst1; *(WORD32 *)(&pu1_dst[2 * dst_strd]) = dst2; *(WORD32 *)(&pu1_dst[3 * dst_strd]) = dst3; /* To update pointer */ pi2_src += 4; pu1_dst += 4; } /* inner loop ends here(4-output values in single iteration) */ pi2_src = pi2_src - wd + 4 * src_strd; /* Pointer update */ pu1_dst = pu1_dst - wd + 4 * dst_strd; /* Pointer update */ } } } /** ******************************************************************************* * * @brief * Does chroma uni-weighted prediction on array pointed by pi2_src and stores * it at the location pointed by pi2_dst * * @par Description: * dst = ( (src + lvl_shift) * wgt0 + (1 << (shift - 1)) ) >> shift + * offset * * @param[in] pi2_src * Pointer to the source * * @param[out] pu1_dst * Pointer to the destination * * @param[in] src_strd * Source stride * * @param[in] dst_strd * Destination stride * * @param[in] wgt0 * weight to be multiplied to the source * * @param[in] off0 * offset to be added after rounding and * * @param[in] shifting * * * @param[in] shift * (14 Bit depth) + log2_weight_denominator * * @param[in] lvl_shift * added before shift and offset * * @param[in] ht * height of the source * * @param[in] wd * width of the source (each colour component) * * @returns * * @remarks * None * ******************************************************************************* */ void ihevc_weighted_pred_chroma_uni_sse42(WORD16 *pi2_src, UWORD8 *pu1_dst, WORD32 src_strd, WORD32 dst_strd, WORD32 wgt0_cb, WORD32 wgt0_cr, WORD32 off0_cb, WORD32 off0_cr, WORD32 shift, WORD32 lvl_shift, WORD32 ht, WORD32 wd) { WORD32 row, col, temp, wdx2; /* all 128 bit registers are named with a suffix mxnb, where m is the */ /* number of n bits packed in the register */ __m128i src_temp0_4x32b, src_temp1_4x32b; __m128i const_temp_4x32b, lvl_shift_4x32b, wgt0_4x32b, off0_4x32b; ASSERT(wd % 2 == 0); /* checking assumption*/ ASSERT(ht % 2 == 0); /* checking assumption*/ temp = 1 << (shift - 1); wdx2 = 2 * wd; // seting values in register const_temp_4x32b = _mm_set1_epi32(temp); lvl_shift_4x32b = _mm_set1_epi32(lvl_shift); wgt0_4x32b = _mm_set_epi32(wgt0_cr, wgt0_cb, wgt0_cr, wgt0_cb); off0_4x32b = _mm_set_epi32(off0_cr, off0_cb, off0_cr, off0_cb); #if 0 /* Enable this for ht%4=0 case. But was degrading performance for lower sizes and improving for higher sizes!!! */ if( 0 == (ht & 3)) /* ht multiple of 4 case */ { if( 0 == (wdx2 & 15)) /* 2*wd multiple of 168 case */ { __m128i src_temp2_4x32b, src_temp3_4x32b; __m128i src_temp4_4x32b, src_temp5_4x32b, src_temp6_4x32b, src_temp7_4x32b; __m128i src_temp8_4x32b, src_temp9_4x32b, src_temp10_4x32b, src_temp11_4x32b; __m128i src_temp12_4x32b, src_temp13_4x32b, src_temp14_4x32b, src_temp15_4x32b; /* outer for loop starts from here */ for(row = 0; row < ht; row +=4) { for(col = 0; col < wdx2; col +=16) { /* row = 0 */ /*load 8 pixel values from 7:0 pos. relative to cur. pos.*/ src_temp0_4x32b = _mm_loadu_si128((__m128i*)(pi2_src)); /* row = 1 */ src_temp1_4x32b = _mm_loadu_si128((__m128i*)(pi2_src+src_strd)); /* row = 0 */ /* Second 4 pixels */ src_temp2_4x32b = _mm_loadu_si128((__m128i*)(pi2_src+4)); /* row = 1 */ src_temp3_4x32b = _mm_loadu_si128((__m128i*)(pi2_src+src_strd+4)); /* row = 0 */ /* Third 4 pixels */ src_temp4_4x32b = _mm_loadu_si128((__m128i*)(pi2_src+8)); /* row = 1 */ src_temp5_4x32b = _mm_loadu_si128((__m128i*)(pi2_src+src_strd+8)); /* row = 0 */ /* Last 4 pixels */ src_temp6_4x32b = _mm_loadu_si128((__m128i*)(pi2_src+12)); /* row = 1 */ src_temp7_4x32b = _mm_loadu_si128((__m128i*)(pi2_src+src_strd+12)); /* considering pix. 4:0 by converting 16-into 32 bit */ src_temp0_4x32b = _mm_cvtepi16_epi32(src_temp0_4x32b); src_temp1_4x32b = _mm_cvtepi16_epi32(src_temp1_4x32b); /* (pi2_src[col] + lvl_shift)*/ src_temp0_4x32b = _mm_add_epi32 (src_temp0_4x32b, lvl_shift_4x32b); src_temp1_4x32b = _mm_add_epi32 (src_temp1_4x32b, lvl_shift_4x32b); /*i4_tmp = (pi2_src[col] + lvl_shift) * wgt0*/ src_temp0_4x32b = _mm_mullo_epi32 (src_temp0_4x32b, wgt0_4x32b); src_temp1_4x32b = _mm_mullo_epi32 (src_temp1_4x32b, wgt0_4x32b); /* considering pix. 4:0 by converting 16-into 32 bit */ /* Second 4 pixels */ src_temp2_4x32b = _mm_cvtepi16_epi32(src_temp2_4x32b); src_temp3_4x32b = _mm_cvtepi16_epi32(src_temp3_4x32b); /* (pi2_src[col] + lvl_shift)*/ src_temp2_4x32b = _mm_add_epi32 (src_temp2_4x32b, lvl_shift_4x32b); src_temp3_4x32b = _mm_add_epi32 (src_temp3_4x32b, lvl_shift_4x32b); /*i4_tmp = (pi2_src[col] + lvl_shift) * wgt0*/ src_temp2_4x32b = _mm_mullo_epi32 (src_temp2_4x32b, wgt0_4x32b); src_temp3_4x32b = _mm_mullo_epi32 (src_temp3_4x32b, wgt0_4x32b); /* considering pix. 4:0 by converting 16-into 32 bit */ /* Third 4 pixels */ src_temp4_4x32b = _mm_cvtepi16_epi32(src_temp4_4x32b); src_temp5_4x32b = _mm_cvtepi16_epi32(src_temp5_4x32b); /* (pi2_src[col] + lvl_shift)*/ src_temp4_4x32b = _mm_add_epi32 (src_temp4_4x32b, lvl_shift_4x32b); src_temp5_4x32b = _mm_add_epi32 (src_temp5_4x32b, lvl_shift_4x32b); /*i4_tmp = (pi2_src[col] + lvl_shift) * wgt0*/ src_temp4_4x32b = _mm_mullo_epi32 (src_temp4_4x32b, wgt0_4x32b); src_temp5_4x32b = _mm_mullo_epi32 (src_temp5_4x32b, wgt0_4x32b); /* considering pix. 4:0 by converting 16-into 32 bit */ /* Last 4 pixels */ src_temp6_4x32b = _mm_cvtepi16_epi32(src_temp6_4x32b); src_temp7_4x32b = _mm_cvtepi16_epi32(src_temp7_4x32b); /* (pi2_src[col] + lvl_shift)*/ src_temp6_4x32b = _mm_add_epi32 (src_temp6_4x32b, lvl_shift_4x32b); src_temp7_4x32b = _mm_add_epi32 (src_temp7_4x32b, lvl_shift_4x32b); /*i4_tmp = (pi2_src[col] + lvl_shift) * wgt0*/ src_temp6_4x32b = _mm_mullo_epi32 (src_temp6_4x32b, wgt0_4x32b); src_temp7_4x32b = _mm_mullo_epi32 (src_temp7_4x32b, wgt0_4x32b); /* i4_tmp += 1 << (shift - 1) */ src_temp0_4x32b = _mm_add_epi32 (src_temp0_4x32b, const_temp_4x32b); src_temp1_4x32b = _mm_add_epi32 (src_temp1_4x32b, const_temp_4x32b); /* (i4_tmp >> shift) */ src_temp0_4x32b = _mm_srai_epi32(src_temp0_4x32b, shift); src_temp1_4x32b = _mm_srai_epi32(src_temp1_4x32b, shift); /* i4_tmp += 1 << (shift - 1) */ /* Second 4 pixels */ src_temp2_4x32b = _mm_add_epi32 (src_temp2_4x32b, const_temp_4x32b); src_temp3_4x32b = _mm_add_epi32 (src_temp3_4x32b, const_temp_4x32b); /* (i4_tmp >> shift) */ src_temp2_4x32b = _mm_srai_epi32(src_temp2_4x32b, shift); src_temp3_4x32b = _mm_srai_epi32(src_temp3_4x32b, shift); /* i4_tmp += 1 << (shift - 1) */ /* Third 4 pixels */ src_temp4_4x32b = _mm_add_epi32 (src_temp4_4x32b, const_temp_4x32b); src_temp5_4x32b = _mm_add_epi32 (src_temp5_4x32b, const_temp_4x32b); /* (i4_tmp >> shift) */ src_temp4_4x32b = _mm_srai_epi32(src_temp4_4x32b, shift); src_temp5_4x32b = _mm_srai_epi32(src_temp5_4x32b, shift); /* i4_tmp += 1 << (shift - 1) */ /* Last 4 pixels */ src_temp6_4x32b = _mm_add_epi32 (src_temp6_4x32b, const_temp_4x32b); src_temp7_4x32b = _mm_add_epi32 (src_temp7_4x32b, const_temp_4x32b); /* (i4_tmp >> shift) */ src_temp6_4x32b = _mm_srai_epi32(src_temp6_4x32b, shift); src_temp7_4x32b = _mm_srai_epi32(src_temp7_4x32b, shift); /*i4_tmp = (i4_tmp >> shift) + off0; */ src_temp0_4x32b = _mm_add_epi32 (src_temp0_4x32b, off0_4x32b); src_temp1_4x32b = _mm_add_epi32 (src_temp1_4x32b, off0_4x32b); /*i4_tmp = (i4_tmp >> shift) + off0; */ /* Second 4 pixels */ src_temp2_4x32b = _mm_add_epi32 (src_temp2_4x32b, off0_4x32b); src_temp3_4x32b = _mm_add_epi32 (src_temp3_4x32b, off0_4x32b); /*i4_tmp = (i4_tmp >> shift) + off0; */ /* Third 4 pixels */ src_temp4_4x32b = _mm_add_epi32 (src_temp4_4x32b, off0_4x32b); src_temp5_4x32b = _mm_add_epi32 (src_temp5_4x32b, off0_4x32b); /*i4_tmp = (i4_tmp >> shift) + off0; */ /* Last 4 pixels */ src_temp6_4x32b = _mm_add_epi32 (src_temp6_4x32b, off0_4x32b); src_temp7_4x32b = _mm_add_epi32 (src_temp7_4x32b, off0_4x32b); src_temp0_4x32b = _mm_packs_epi32 (src_temp0_4x32b, src_temp2_4x32b); src_temp1_4x32b = _mm_packs_epi32 (src_temp1_4x32b, src_temp3_4x32b); src_temp4_4x32b = _mm_packs_epi32 (src_temp4_4x32b, src_temp6_4x32b); src_temp5_4x32b = _mm_packs_epi32 (src_temp5_4x32b, src_temp7_4x32b); /* pu1_dst[col] = CLIP_U8(i4_tmp); */ src_temp0_4x32b = _mm_packus_epi16 (src_temp0_4x32b, src_temp4_4x32b); src_temp1_4x32b = _mm_packus_epi16 (src_temp1_4x32b, src_temp5_4x32b); /* store 16 8-bit output values */ _mm_storeu_si128((__m128i*)(pu1_dst+0*dst_strd), src_temp0_4x32b); /* row = 0*/ _mm_storeu_si128((__m128i*)(pu1_dst+1*dst_strd), src_temp1_4x32b); /* row = 1*/ /* row = 2 */ /*load 8 pixel values from 7:0 pos. relative to cur. pos.*/ src_temp8_4x32b = _mm_loadu_si128((__m128i*)(pi2_src+2*src_strd)); /* row = 3 */ src_temp9_4x32b = _mm_loadu_si128((__m128i*)(pi2_src+3*src_strd)); /* row = 2 */ /* Second 4 pixels */ src_temp10_4x32b = _mm_loadu_si128((__m128i*)(pi2_src+2*src_strd+4)); /* row = 3 */ src_temp11_4x32b = _mm_loadu_si128((__m128i*)(pi2_src+3*src_strd+4)); /* row = 2 */ /* Third 4 pixels */ src_temp12_4x32b = _mm_loadu_si128((__m128i*)(pi2_src+2*src_strd+8)); /* row = 3 */ src_temp13_4x32b = _mm_loadu_si128((__m128i*)(pi2_src+3*src_strd+8)); /* row = 2 */ /* Last 4 pixels */ src_temp14_4x32b = _mm_loadu_si128((__m128i*)(pi2_src+2*src_strd+12)); /* row = 3 */ src_temp15_4x32b = _mm_loadu_si128((__m128i*)(pi2_src+3*src_strd+12)); /* considering pix. 4:0 by converting 16-into 32 bit */ src_temp8_4x32b = _mm_cvtepi16_epi32(src_temp8_4x32b); src_temp9_4x32b = _mm_cvtepi16_epi32(src_temp9_4x32b); /* (pi2_src[col] + lvl_shift)*/ src_temp8_4x32b = _mm_add_epi32 (src_temp8_4x32b, lvl_shift_4x32b); src_temp9_4x32b = _mm_add_epi32 (src_temp9_4x32b, lvl_shift_4x32b); /*i4_tmp = (pi2_src[col] + lvl_shift) * wgt0*/ src_temp8_4x32b = _mm_mullo_epi32 (src_temp8_4x32b, wgt0_4x32b); src_temp9_4x32b = _mm_mullo_epi32 (src_temp9_4x32b, wgt0_4x32b); /* considering pix. 4:0 by converting 16-into 32 bit */ /* Second 4 pixels */ src_temp10_4x32b = _mm_cvtepi16_epi32(src_temp10_4x32b); src_temp11_4x32b = _mm_cvtepi16_epi32(src_temp11_4x32b); /* (pi2_src[col] + lvl_shift)*/ src_temp10_4x32b = _mm_add_epi32 (src_temp10_4x32b, lvl_shift_4x32b); src_temp11_4x32b = _mm_add_epi32 (src_temp11_4x32b, lvl_shift_4x32b); /*i4_tmp = (pi2_src[col] + lvl_shift) * wgt0*/ src_temp10_4x32b = _mm_mullo_epi32 (src_temp10_4x32b, wgt0_4x32b); src_temp11_4x32b = _mm_mullo_epi32 (src_temp11_4x32b, wgt0_4x32b); /* considering pix. 4:0 by converting 16-into 32 bit */ /* Third 4 pixels */ src_temp12_4x32b = _mm_cvtepi16_epi32(src_temp12_4x32b); src_temp13_4x32b = _mm_cvtepi16_epi32(src_temp13_4x32b); /* (pi2_src[col] + lvl_shift)*/ src_temp12_4x32b = _mm_add_epi32 (src_temp12_4x32b, lvl_shift_4x32b); src_temp13_4x32b = _mm_add_epi32 (src_temp13_4x32b, lvl_shift_4x32b); /*i4_tmp = (pi2_src[col] + lvl_shift) * wgt0*/ src_temp12_4x32b = _mm_mullo_epi32 (src_temp12_4x32b, wgt0_4x32b); src_temp13_4x32b = _mm_mullo_epi32 (src_temp13_4x32b, wgt0_4x32b); /* considering pix. 4:0 by converting 16-into 32 bit */ /* Last 4 pixels */ src_temp14_4x32b = _mm_cvtepi16_epi32(src_temp14_4x32b); src_temp15_4x32b = _mm_cvtepi16_epi32(src_temp15_4x32b); /* (pi2_src[col] + lvl_shift)*/ src_temp14_4x32b = _mm_add_epi32 (src_temp14_4x32b, lvl_shift_4x32b); src_temp15_4x32b = _mm_add_epi32 (src_temp15_4x32b, lvl_shift_4x32b); /*i4_tmp = (pi2_src[col] + lvl_shift) * wgt0*/ src_temp14_4x32b = _mm_mullo_epi32 (src_temp14_4x32b, wgt0_4x32b); src_temp15_4x32b = _mm_mullo_epi32 (src_temp15_4x32b, wgt0_4x32b); /* i4_tmp += 1 << (shift - 1) */ src_temp8_4x32b = _mm_add_epi32 (src_temp8_4x32b, const_temp_4x32b); src_temp9_4x32b = _mm_add_epi32 (src_temp9_4x32b, const_temp_4x32b); /* (i4_tmp >> shift) */ src_temp8_4x32b = _mm_srai_epi32(src_temp8_4x32b, shift); src_temp9_4x32b = _mm_srai_epi32(src_temp9_4x32b, shift); /* i4_tmp += 1 << (shift - 1) */ /* Second 4 pixels */ src_temp10_4x32b = _mm_add_epi32 (src_temp10_4x32b, const_temp_4x32b); src_temp11_4x32b = _mm_add_epi32 (src_temp11_4x32b, const_temp_4x32b); /* (i4_tmp >> shift) */ src_temp10_4x32b = _mm_srai_epi32(src_temp10_4x32b, shift); src_temp11_4x32b = _mm_srai_epi32(src_temp11_4x32b, shift); /* i4_tmp += 1 << (shift - 1) */ /* Third 4 pixels */ src_temp12_4x32b = _mm_add_epi32 (src_temp12_4x32b, const_temp_4x32b); src_temp13_4x32b = _mm_add_epi32 (src_temp13_4x32b, const_temp_4x32b); /* (i4_tmp >> shift) */ src_temp12_4x32b = _mm_srai_epi32(src_temp12_4x32b, shift); src_temp13_4x32b = _mm_srai_epi32(src_temp13_4x32b, shift); /* i4_tmp += 1 << (shift - 1) */ /* Last 4 pixels */ src_temp14_4x32b = _mm_add_epi32 (src_temp14_4x32b, const_temp_4x32b); src_temp15_4x32b = _mm_add_epi32 (src_temp15_4x32b, const_temp_4x32b); /* (i4_tmp >> shift) */ src_temp14_4x32b = _mm_srai_epi32(src_temp14_4x32b, shift); src_temp15_4x32b = _mm_srai_epi32(src_temp15_4x32b, shift); /*i4_tmp = (i4_tmp >> shift) + off0; */ src_temp8_4x32b = _mm_add_epi32 (src_temp8_4x32b, off0_4x32b); src_temp9_4x32b = _mm_add_epi32 (src_temp9_4x32b, off0_4x32b); /*i4_tmp = (i4_tmp >> shift) + off0; */ /* Second 4 pixels */ src_temp10_4x32b = _mm_add_epi32 (src_temp10_4x32b, off0_4x32b); src_temp11_4x32b = _mm_add_epi32 (src_temp11_4x32b, off0_4x32b); /*i4_tmp = (i4_tmp >> shift) + off0; */ /* Third 4 pixels */ src_temp12_4x32b = _mm_add_epi32 (src_temp12_4x32b, off0_4x32b); src_temp13_4x32b = _mm_add_epi32 (src_temp13_4x32b, off0_4x32b); /*i4_tmp = (i4_tmp >> shift) + off0; */ /* Last 4 pixels */ src_temp14_4x32b = _mm_add_epi32 (src_temp14_4x32b, off0_4x32b); src_temp15_4x32b = _mm_add_epi32 (src_temp15_4x32b, off0_4x32b); src_temp8_4x32b = _mm_packs_epi32 (src_temp8_4x32b, src_temp10_4x32b); src_temp9_4x32b = _mm_packs_epi32 (src_temp9_4x32b, src_temp11_4x32b); src_temp12_4x32b = _mm_packs_epi32 (src_temp12_4x32b, src_temp14_4x32b); src_temp13_4x32b = _mm_packs_epi32 (src_temp13_4x32b, src_temp15_4x32b); /* pu1_dst[col] = CLIP_U8(i4_tmp); */ src_temp8_4x32b = _mm_packus_epi16 (src_temp8_4x32b, src_temp12_4x32b); src_temp9_4x32b = _mm_packus_epi16 (src_temp9_4x32b, src_temp13_4x32b); /* store 16 8-bit output values */ _mm_storeu_si128((__m128i*)(pu1_dst+2*dst_strd), src_temp8_4x32b); /* row = 2*/ _mm_storeu_si128((__m128i*)(pu1_dst+3*dst_strd), src_temp9_4x32b); /* row = 3*/ pi2_src += 16; /* Pointer update */ pu1_dst += 16; /* Pointer update */ } /* inner loop ends here(4-output values in single iteration) */ pi2_src = pi2_src - wdx2 + 4*src_strd; /* Pointer update */ pu1_dst = pu1_dst - wdx2 + 4*dst_strd; /* Pointer update */ } } else if( 0 == (wdx2 & 7)) /* 2*wd multiple of 8 case */ { __m128i src_temp2_4x32b,src_temp3_4x32b; __m128i src_temp4_4x32b, src_temp5_4x32b, src_temp6_4x32b, src_temp7_4x32b; /* outer for loop starts from here */ for(row = 0; row < ht; row +=4) { for(col = 0; col < wdx2; col +=8) { /* row = 0 */ /*load 8 pixel values from 7:0 pos. relative to cur. pos.*/ src_temp0_4x32b = _mm_loadu_si128((__m128i*)(pi2_src)); /* row = 1 */ src_temp1_4x32b = _mm_loadu_si128((__m128i*)(pi2_src+src_strd)); /* row = 2 */ src_temp2_4x32b = _mm_loadu_si128((__m128i*)(pi2_src+2*src_strd)); /* row = 3 */ src_temp3_4x32b = _mm_loadu_si128((__m128i*)(pi2_src+3*src_strd)); /* row = 0 */ /* Last 4 pixels */ src_temp4_4x32b = _mm_loadu_si128((__m128i*)(pi2_src+4)); /* row = 1 */ src_temp5_4x32b = _mm_loadu_si128((__m128i*)(pi2_src+src_strd+4)); /* row = 2 */ src_temp6_4x32b = _mm_loadu_si128((__m128i*)(pi2_src+2*src_strd+4)); /* row = 3 */ src_temp7_4x32b = _mm_loadu_si128((__m128i*)(pi2_src+3*src_strd+4)); /* considering pix. 4:0 by converting 16-into 32 bit */ src_temp0_4x32b = _mm_cvtepi16_epi32(src_temp0_4x32b); src_temp1_4x32b = _mm_cvtepi16_epi32(src_temp1_4x32b); /* (pi2_src[col] + lvl_shift)*/ src_temp0_4x32b = _mm_add_epi32 (src_temp0_4x32b, lvl_shift_4x32b); src_temp1_4x32b = _mm_add_epi32 (src_temp1_4x32b, lvl_shift_4x32b); /*i4_tmp = (pi2_src[col] + lvl_shift) * wgt0*/ src_temp0_4x32b = _mm_mullo_epi32 (src_temp0_4x32b, wgt0_4x32b); src_temp1_4x32b = _mm_mullo_epi32 (src_temp1_4x32b, wgt0_4x32b); /* considering pix. 4:0 by converting 16-into 32 bit */ /* Last 4 pixels */ src_temp2_4x32b = _mm_cvtepi16_epi32(src_temp2_4x32b); src_temp3_4x32b = _mm_cvtepi16_epi32(src_temp3_4x32b); /* (pi2_src[col] + lvl_shift)*/ src_temp2_4x32b = _mm_add_epi32 (src_temp2_4x32b, lvl_shift_4x32b); src_temp3_4x32b = _mm_add_epi32 (src_temp3_4x32b, lvl_shift_4x32b); /*i4_tmp = (pi2_src[col] + lvl_shift) * wgt0*/ src_temp2_4x32b = _mm_mullo_epi32 (src_temp2_4x32b, wgt0_4x32b); src_temp3_4x32b = _mm_mullo_epi32 (src_temp3_4x32b, wgt0_4x32b); /* considering pix. 4:0 by converting 16-into 32 bit */ src_temp4_4x32b = _mm_cvtepi16_epi32(src_temp4_4x32b); src_temp5_4x32b = _mm_cvtepi16_epi32(src_temp5_4x32b); /* (pi2_src[col] + lvl_shift)*/ src_temp4_4x32b = _mm_add_epi32 (src_temp4_4x32b, lvl_shift_4x32b); src_temp5_4x32b = _mm_add_epi32 (src_temp5_4x32b, lvl_shift_4x32b); /*i4_tmp = (pi2_src[col] + lvl_shift) * wgt0*/ src_temp4_4x32b = _mm_mullo_epi32 (src_temp4_4x32b, wgt0_4x32b); src_temp5_4x32b = _mm_mullo_epi32 (src_temp5_4x32b, wgt0_4x32b); /* considering pix. 4:0 by converting 16-into 32 bit */ src_temp6_4x32b = _mm_cvtepi16_epi32(src_temp6_4x32b); src_temp7_4x32b = _mm_cvtepi16_epi32(src_temp7_4x32b); /* (pi2_src[col] + lvl_shift)*/ src_temp6_4x32b = _mm_add_epi32 (src_temp6_4x32b, lvl_shift_4x32b); src_temp7_4x32b = _mm_add_epi32 (src_temp7_4x32b, lvl_shift_4x32b); /*i4_tmp = (pi2_src[col] + lvl_shift) * wgt0*/ src_temp6_4x32b = _mm_mullo_epi32 (src_temp6_4x32b, wgt0_4x32b); src_temp7_4x32b = _mm_mullo_epi32 (src_temp7_4x32b, wgt0_4x32b); /* i4_tmp += 1 << (shift - 1) */ src_temp0_4x32b = _mm_add_epi32 (src_temp0_4x32b, const_temp_4x32b); src_temp1_4x32b = _mm_add_epi32 (src_temp1_4x32b, const_temp_4x32b); /* (i4_tmp >> shift) */ src_temp0_4x32b = _mm_srai_epi32(src_temp0_4x32b, shift); src_temp1_4x32b = _mm_srai_epi32(src_temp1_4x32b, shift); /* i4_tmp += 1 << (shift - 1) */ /* Last 4 pixels */ src_temp2_4x32b = _mm_add_epi32 (src_temp2_4x32b, const_temp_4x32b); src_temp3_4x32b = _mm_add_epi32 (src_temp3_4x32b, const_temp_4x32b); /* (i4_tmp >> shift) */ src_temp2_4x32b = _mm_srai_epi32(src_temp2_4x32b, shift); src_temp3_4x32b = _mm_srai_epi32(src_temp3_4x32b, shift); /* i4_tmp += 1 << (shift - 1) */ src_temp4_4x32b = _mm_add_epi32 (src_temp4_4x32b, const_temp_4x32b); src_temp5_4x32b = _mm_add_epi32 (src_temp5_4x32b, const_temp_4x32b); /* (i4_tmp >> shift) */ src_temp4_4x32b = _mm_srai_epi32(src_temp4_4x32b, shift); src_temp5_4x32b = _mm_srai_epi32(src_temp5_4x32b, shift); /* i4_tmp += 1 << (shift - 1) */ src_temp6_4x32b = _mm_add_epi32 (src_temp6_4x32b, const_temp_4x32b); src_temp7_4x32b = _mm_add_epi32 (src_temp7_4x32b, const_temp_4x32b); /* (i4_tmp >> shift) */ src_temp6_4x32b = _mm_srai_epi32(src_temp6_4x32b, shift); src_temp7_4x32b = _mm_srai_epi32(src_temp7_4x32b, shift); /*i4_tmp = (i4_tmp >> shift) + off0; */ src_temp0_4x32b = _mm_add_epi32 (src_temp0_4x32b, off0_4x32b); src_temp1_4x32b = _mm_add_epi32 (src_temp1_4x32b, off0_4x32b); /*i4_tmp = (i4_tmp >> shift) + off0; */ /* Last 4 pixels */ src_temp2_4x32b = _mm_add_epi32 (src_temp2_4x32b, off0_4x32b); src_temp3_4x32b = _mm_add_epi32 (src_temp3_4x32b, off0_4x32b); /*i4_tmp = (i4_tmp >> shift) + off0; */ src_temp4_4x32b = _mm_add_epi32 (src_temp4_4x32b, off0_4x32b); src_temp5_4x32b = _mm_add_epi32 (src_temp5_4x32b, off0_4x32b); /*i4_tmp = (i4_tmp >> shift) + off0; */ src_temp6_4x32b = _mm_add_epi32 (src_temp6_4x32b, off0_4x32b); src_temp7_4x32b = _mm_add_epi32 (src_temp7_4x32b, off0_4x32b); src_temp0_4x32b = _mm_packs_epi32 (src_temp0_4x32b, src_temp4_4x32b); src_temp1_4x32b = _mm_packs_epi32 (src_temp1_4x32b, src_temp5_4x32b); src_temp2_4x32b = _mm_packs_epi32 (src_temp2_4x32b, src_temp6_4x32b); src_temp3_4x32b = _mm_packs_epi32 (src_temp3_4x32b, src_temp7_4x32b); /* pu1_dst[col] = CLIP_U8(i4_tmp); */ src_temp0_4x32b = _mm_packus_epi16 (src_temp0_4x32b, src_temp0_4x32b); src_temp1_4x32b = _mm_packus_epi16 (src_temp1_4x32b, src_temp1_4x32b); src_temp2_4x32b = _mm_packus_epi16 (src_temp2_4x32b, src_temp2_4x32b); src_temp3_4x32b = _mm_packus_epi16 (src_temp3_4x32b, src_temp3_4x32b); /* store four 8-bit output values */ _mm_storel_epi64((__m128i*)(pu1_dst+0*dst_strd), src_temp0_4x32b); /* row = 0*/ _mm_storel_epi64((__m128i*)(pu1_dst+1*dst_strd), src_temp1_4x32b); /* row = 1*/ _mm_storel_epi64((__m128i*)(pu1_dst+2*dst_strd), src_temp2_4x32b); /* row = 0*/ _mm_storel_epi64((__m128i*)(pu1_dst+3*dst_strd), src_temp3_4x32b); /* row = 1*/ pi2_src += 8; /* Pointer update */ pu1_dst += 8; /* Pointer update */ } /* inner loop ends here(4-output values in single iteration) */ pi2_src = pi2_src - wdx2 + 4*src_strd; /* Pointer update */ pu1_dst = pu1_dst - wdx2 + 4*dst_strd; /* Pointer update */ } } else /* 2*wd multiple of 4 case */ { WORD32 dst0, dst1, dst2, dst3; __m128i src_temp2_4x32b,src_temp3_4x32b; /* outer for loop starts from here */ for(row = 0; row < ht; row +=4) { for(col = 0; col < wdx2; col +=4) { /* row = 0 */ /*load 8 pixel values from 7:0 pos. relative to cur. pos.*/ src_temp0_4x32b = _mm_loadu_si128((__m128i*)(pi2_src)); /* row = 1 */ src_temp1_4x32b = _mm_loadu_si128((__m128i*)(pi2_src+1*src_strd)); /* row = 2 */ src_temp2_4x32b = _mm_loadu_si128((__m128i*)(pi2_src+2*src_strd)); /* row = 3 */ src_temp3_4x32b = _mm_loadu_si128((__m128i*)(pi2_src+3*src_strd)); /* considering pix. 4:0 by converting 16-into 32 bit */ src_temp0_4x32b = _mm_cvtepi16_epi32(src_temp0_4x32b); src_temp1_4x32b = _mm_cvtepi16_epi32(src_temp1_4x32b); src_temp2_4x32b = _mm_cvtepi16_epi32(src_temp2_4x32b); src_temp3_4x32b = _mm_cvtepi16_epi32(src_temp3_4x32b); /* (pi2_src[col] + lvl_shift)*/ src_temp0_4x32b = _mm_add_epi32 (src_temp0_4x32b, lvl_shift_4x32b); src_temp1_4x32b = _mm_add_epi32 (src_temp1_4x32b, lvl_shift_4x32b); /*i4_tmp = (pi2_src[col] + lvl_shift) * wgt0*/ src_temp0_4x32b = _mm_mullo_epi32 (src_temp0_4x32b, wgt0_4x32b); src_temp1_4x32b = _mm_mullo_epi32 (src_temp1_4x32b, wgt0_4x32b); /* (pi2_src[col] + lvl_shift)*/ src_temp2_4x32b = _mm_add_epi32 (src_temp2_4x32b, lvl_shift_4x32b); src_temp3_4x32b = _mm_add_epi32 (src_temp3_4x32b, lvl_shift_4x32b); /*i4_tmp = (pi2_src[col] + lvl_shift) * wgt0*/ src_temp2_4x32b = _mm_mullo_epi32 (src_temp2_4x32b, wgt0_4x32b); src_temp3_4x32b = _mm_mullo_epi32 (src_temp3_4x32b, wgt0_4x32b); /* i4_tmp += 1 << (shift - 1) */ src_temp0_4x32b = _mm_add_epi32 (src_temp0_4x32b, const_temp_4x32b); src_temp1_4x32b = _mm_add_epi32 (src_temp1_4x32b, const_temp_4x32b); /* (i4_tmp >> shift) */ src_temp0_4x32b = _mm_srai_epi32(src_temp0_4x32b, shift); src_temp1_4x32b = _mm_srai_epi32(src_temp1_4x32b, shift); /*i4_tmp = (i4_tmp >> shift) + off0; */ src_temp0_4x32b = _mm_add_epi32 (src_temp0_4x32b, off0_4x32b); src_temp1_4x32b = _mm_add_epi32 (src_temp1_4x32b, off0_4x32b); /* i4_tmp += 1 << (shift - 1) */ src_temp2_4x32b = _mm_add_epi32 (src_temp2_4x32b, const_temp_4x32b); src_temp3_4x32b = _mm_add_epi32 (src_temp3_4x32b, const_temp_4x32b); /* (i4_tmp >> shift) */ src_temp2_4x32b = _mm_srai_epi32(src_temp2_4x32b, shift); src_temp3_4x32b = _mm_srai_epi32(src_temp3_4x32b, shift); /*i4_tmp = (i4_tmp >> shift) + off0; */ src_temp2_4x32b = _mm_add_epi32 (src_temp2_4x32b, off0_4x32b); src_temp3_4x32b = _mm_add_epi32 (src_temp3_4x32b, off0_4x32b); src_temp0_4x32b = _mm_packs_epi32 (src_temp0_4x32b, src_temp1_4x32b); src_temp2_4x32b = _mm_packs_epi32 (src_temp2_4x32b, src_temp3_4x32b); /* pu1_dst[col] = CLIP_U8(i4_tmp); */ src_temp0_4x32b = _mm_packus_epi16 (src_temp0_4x32b, src_temp2_4x32b); dst0 = _mm_cvtsi128_si32(src_temp0_4x32b); /* dst row = 1 to 3 */ src_temp1_4x32b = _mm_shuffle_epi32 (src_temp0_4x32b, 1); src_temp2_4x32b = _mm_shuffle_epi32 (src_temp0_4x32b, 2); src_temp3_4x32b = _mm_shuffle_epi32 (src_temp0_4x32b, 3); /* store four 8-bit output values */ *(WORD32 *) (&pu1_dst[0*dst_strd]) = dst0; dst1 = _mm_cvtsi128_si32(src_temp1_4x32b); dst2 = _mm_cvtsi128_si32(src_temp2_4x32b); dst3 = _mm_cvtsi128_si32(src_temp3_4x32b); /* row = 1 */ *(WORD32 *) (&pu1_dst[1*dst_strd]) = dst1; /* row = 2 */ *(WORD32 *) (&pu1_dst[2*dst_strd]) = dst2; /* row = 3 */ *(WORD32 *) (&pu1_dst[3*dst_strd]) = dst3; pi2_src += 4; /* Pointer update */ pu1_dst += 4; /* Pointer update */ } /* inner loop ends here(4-output values in single iteration) */ pi2_src = pi2_src - wdx2 + 4*src_strd; /* Pointer update */ pu1_dst = pu1_dst - wdx2 + 4*dst_strd; /* Pointer update */ } } } else /* ht multiple of 2 case */ #endif { if(0 == (wdx2 & 15)) /* 2*wd multiple of 168 case */ { __m128i src_temp2_4x32b, src_temp3_4x32b; __m128i src_temp4_4x32b, src_temp5_4x32b, src_temp6_4x32b, src_temp7_4x32b; /* outer for loop starts from here */ for(row = 0; row < ht; row += 2) { for(col = 0; col < wdx2; col += 16) { /* row = 0 */ /*load 8 pixel values from 7:0 pos. relative to cur. pos.*/ src_temp0_4x32b = _mm_loadu_si128((__m128i *)(pi2_src)); /* row = 1 */ src_temp1_4x32b = _mm_loadu_si128((__m128i *)(pi2_src + src_strd)); /* row = 0 */ /* Second 4 pixels */ src_temp2_4x32b = _mm_loadu_si128((__m128i *)(pi2_src + 4)); /* row = 1 */ src_temp3_4x32b = _mm_loadu_si128((__m128i *)(pi2_src + src_strd + 4)); /* row = 0 */ /* Third 4 pixels */ src_temp4_4x32b = _mm_loadu_si128((__m128i *)(pi2_src + 8)); /* row = 1 */ src_temp5_4x32b = _mm_loadu_si128((__m128i *)(pi2_src + src_strd + 8)); /* row = 0 */ /* Last 4 pixels */ src_temp6_4x32b = _mm_loadu_si128((__m128i *)(pi2_src + 12)); /* row = 1 */ src_temp7_4x32b = _mm_loadu_si128((__m128i *)(pi2_src + src_strd + 12)); /* considering pix. 4:0 by converting 16-into 32 bit */ src_temp0_4x32b = _mm_cvtepi16_epi32(src_temp0_4x32b); src_temp1_4x32b = _mm_cvtepi16_epi32(src_temp1_4x32b); /* (pi2_src[col] + lvl_shift)*/ src_temp0_4x32b = _mm_add_epi32(src_temp0_4x32b, lvl_shift_4x32b); src_temp1_4x32b = _mm_add_epi32(src_temp1_4x32b, lvl_shift_4x32b); /*i4_tmp = (pi2_src[col] + lvl_shift) * wgt0*/ src_temp0_4x32b = _mm_mullo_epi32(src_temp0_4x32b, wgt0_4x32b); src_temp1_4x32b = _mm_mullo_epi32(src_temp1_4x32b, wgt0_4x32b); /* considering pix. 4:0 by converting 16-into 32 bit */ /* Second 4 pixels */ src_temp2_4x32b = _mm_cvtepi16_epi32(src_temp2_4x32b); src_temp3_4x32b = _mm_cvtepi16_epi32(src_temp3_4x32b); /* (pi2_src[col] + lvl_shift)*/ src_temp2_4x32b = _mm_add_epi32(src_temp2_4x32b, lvl_shift_4x32b); src_temp3_4x32b = _mm_add_epi32(src_temp3_4x32b, lvl_shift_4x32b); /*i4_tmp = (pi2_src[col] + lvl_shift) * wgt0*/ src_temp2_4x32b = _mm_mullo_epi32(src_temp2_4x32b, wgt0_4x32b); src_temp3_4x32b = _mm_mullo_epi32(src_temp3_4x32b, wgt0_4x32b); /* considering pix. 4:0 by converting 16-into 32 bit */ /* Third 4 pixels */ src_temp4_4x32b = _mm_cvtepi16_epi32(src_temp4_4x32b); src_temp5_4x32b = _mm_cvtepi16_epi32(src_temp5_4x32b); /* (pi2_src[col] + lvl_shift)*/ src_temp4_4x32b = _mm_add_epi32(src_temp4_4x32b, lvl_shift_4x32b); src_temp5_4x32b = _mm_add_epi32(src_temp5_4x32b, lvl_shift_4x32b); /*i4_tmp = (pi2_src[col] + lvl_shift) * wgt0*/ src_temp4_4x32b = _mm_mullo_epi32(src_temp4_4x32b, wgt0_4x32b); src_temp5_4x32b = _mm_mullo_epi32(src_temp5_4x32b, wgt0_4x32b); /* considering pix. 4:0 by converting 16-into 32 bit */ /* Last 4 pixels */ src_temp6_4x32b = _mm_cvtepi16_epi32(src_temp6_4x32b); src_temp7_4x32b = _mm_cvtepi16_epi32(src_temp7_4x32b); /* (pi2_src[col] + lvl_shift)*/ src_temp6_4x32b = _mm_add_epi32(src_temp6_4x32b, lvl_shift_4x32b); src_temp7_4x32b = _mm_add_epi32(src_temp7_4x32b, lvl_shift_4x32b); /*i4_tmp = (pi2_src[col] + lvl_shift) * wgt0*/ src_temp6_4x32b = _mm_mullo_epi32(src_temp6_4x32b, wgt0_4x32b); src_temp7_4x32b = _mm_mullo_epi32(src_temp7_4x32b, wgt0_4x32b); /* i4_tmp += 1 << (shift - 1) */ src_temp0_4x32b = _mm_add_epi32(src_temp0_4x32b, const_temp_4x32b); src_temp1_4x32b = _mm_add_epi32(src_temp1_4x32b, const_temp_4x32b); /* (i4_tmp >> shift) */ src_temp0_4x32b = _mm_srai_epi32(src_temp0_4x32b, shift); src_temp1_4x32b = _mm_srai_epi32(src_temp1_4x32b, shift); /* i4_tmp += 1 << (shift - 1) */ /* Second 4 pixels */ src_temp2_4x32b = _mm_add_epi32(src_temp2_4x32b, const_temp_4x32b); src_temp3_4x32b = _mm_add_epi32(src_temp3_4x32b, const_temp_4x32b); /* (i4_tmp >> shift) */ src_temp2_4x32b = _mm_srai_epi32(src_temp2_4x32b, shift); src_temp3_4x32b = _mm_srai_epi32(src_temp3_4x32b, shift); /* i4_tmp += 1 << (shift - 1) */ /* Third 4 pixels */ src_temp4_4x32b = _mm_add_epi32(src_temp4_4x32b, const_temp_4x32b); src_temp5_4x32b = _mm_add_epi32(src_temp5_4x32b, const_temp_4x32b); /* (i4_tmp >> shift) */ src_temp4_4x32b = _mm_srai_epi32(src_temp4_4x32b, shift); src_temp5_4x32b = _mm_srai_epi32(src_temp5_4x32b, shift); /* i4_tmp += 1 << (shift - 1) */ /* Last 4 pixels */ src_temp6_4x32b = _mm_add_epi32(src_temp6_4x32b, const_temp_4x32b); src_temp7_4x32b = _mm_add_epi32(src_temp7_4x32b, const_temp_4x32b); /* (i4_tmp >> shift) */ src_temp6_4x32b = _mm_srai_epi32(src_temp6_4x32b, shift); src_temp7_4x32b = _mm_srai_epi32(src_temp7_4x32b, shift); /*i4_tmp = (i4_tmp >> shift) + off0; */ src_temp0_4x32b = _mm_add_epi32(src_temp0_4x32b, off0_4x32b); src_temp1_4x32b = _mm_add_epi32(src_temp1_4x32b, off0_4x32b); /*i4_tmp = (i4_tmp >> shift) + off0; */ /* Second 4 pixels */ src_temp2_4x32b = _mm_add_epi32(src_temp2_4x32b, off0_4x32b); src_temp3_4x32b = _mm_add_epi32(src_temp3_4x32b, off0_4x32b); /*i4_tmp = (i4_tmp >> shift) + off0; */ /* Third 4 pixels */ src_temp4_4x32b = _mm_add_epi32(src_temp4_4x32b, off0_4x32b); src_temp5_4x32b = _mm_add_epi32(src_temp5_4x32b, off0_4x32b); /*i4_tmp = (i4_tmp >> shift) + off0; */ /* Last 4 pixels */ src_temp6_4x32b = _mm_add_epi32(src_temp6_4x32b, off0_4x32b); src_temp7_4x32b = _mm_add_epi32(src_temp7_4x32b, off0_4x32b); src_temp0_4x32b = _mm_packs_epi32(src_temp0_4x32b, src_temp2_4x32b); src_temp1_4x32b = _mm_packs_epi32(src_temp1_4x32b, src_temp3_4x32b); src_temp4_4x32b = _mm_packs_epi32(src_temp4_4x32b, src_temp6_4x32b); src_temp5_4x32b = _mm_packs_epi32(src_temp5_4x32b, src_temp7_4x32b); /* pu1_dst[col] = CLIP_U8(i4_tmp); */ src_temp0_4x32b = _mm_packus_epi16(src_temp0_4x32b, src_temp4_4x32b); src_temp1_4x32b = _mm_packus_epi16(src_temp1_4x32b, src_temp5_4x32b); /* store 16 8-bit output values */ _mm_storeu_si128((__m128i *)(pu1_dst + 0 * dst_strd), src_temp0_4x32b); /* row = 0*/ _mm_storeu_si128((__m128i *)(pu1_dst + 1 * dst_strd), src_temp1_4x32b); /* row = 1*/ pi2_src += 16; /* Pointer update */ pu1_dst += 16; /* Pointer update */ } /* inner loop ends here(4-output values in single iteration) */ pi2_src = pi2_src - wdx2 + 2 * src_strd; /* Pointer update */ pu1_dst = pu1_dst - wdx2 + 2 * dst_strd; /* Pointer update */ } } else if(0 == (wdx2 & 7)) /* 2*wd multiple of 8 case */ { __m128i src_temp2_4x32b, src_temp3_4x32b; /* outer for loop starts from here */ for(row = 0; row < ht; row += 2) { for(col = 0; col < wdx2; col += 8) { /* row = 0 */ /*load 8 pixel values from 7:0 pos. relative to cur. pos.*/ src_temp0_4x32b = _mm_loadu_si128((__m128i *)(pi2_src)); /* row = 1 */ src_temp1_4x32b = _mm_loadu_si128((__m128i *)(pi2_src + src_strd)); /* row = 0 */ /* Last 4 pixels */ src_temp2_4x32b = _mm_loadu_si128((__m128i *)(pi2_src + 4)); /* row = 1 */ src_temp3_4x32b = _mm_loadu_si128((__m128i *)(pi2_src + src_strd + 4)); /* considering pix. 4:0 by converting 16-into 32 bit */ src_temp0_4x32b = _mm_cvtepi16_epi32(src_temp0_4x32b); src_temp1_4x32b = _mm_cvtepi16_epi32(src_temp1_4x32b); /* (pi2_src[col] + lvl_shift)*/ src_temp0_4x32b = _mm_add_epi32(src_temp0_4x32b, lvl_shift_4x32b); src_temp1_4x32b = _mm_add_epi32(src_temp1_4x32b, lvl_shift_4x32b); /*i4_tmp = (pi2_src[col] + lvl_shift) * wgt0*/ src_temp0_4x32b = _mm_mullo_epi32(src_temp0_4x32b, wgt0_4x32b); src_temp1_4x32b = _mm_mullo_epi32(src_temp1_4x32b, wgt0_4x32b); /* considering pix. 4:0 by converting 16-into 32 bit */ /* Last 4 pixels */ src_temp2_4x32b = _mm_cvtepi16_epi32(src_temp2_4x32b); src_temp3_4x32b = _mm_cvtepi16_epi32(src_temp3_4x32b); /* (pi2_src[col] + lvl_shift)*/ src_temp2_4x32b = _mm_add_epi32(src_temp2_4x32b, lvl_shift_4x32b); src_temp3_4x32b = _mm_add_epi32(src_temp3_4x32b, lvl_shift_4x32b); /*i4_tmp = (pi2_src[col] + lvl_shift) * wgt0*/ src_temp2_4x32b = _mm_mullo_epi32(src_temp2_4x32b, wgt0_4x32b); src_temp3_4x32b = _mm_mullo_epi32(src_temp3_4x32b, wgt0_4x32b); /* i4_tmp += 1 << (shift - 1) */ src_temp0_4x32b = _mm_add_epi32(src_temp0_4x32b, const_temp_4x32b); src_temp1_4x32b = _mm_add_epi32(src_temp1_4x32b, const_temp_4x32b); /* (i4_tmp >> shift) */ src_temp0_4x32b = _mm_srai_epi32(src_temp0_4x32b, shift); src_temp1_4x32b = _mm_srai_epi32(src_temp1_4x32b, shift); /* i4_tmp += 1 << (shift - 1) */ /* Last 4 pixels */ src_temp2_4x32b = _mm_add_epi32(src_temp2_4x32b, const_temp_4x32b); src_temp3_4x32b = _mm_add_epi32(src_temp3_4x32b, const_temp_4x32b); /* (i4_tmp >> shift) */ src_temp2_4x32b = _mm_srai_epi32(src_temp2_4x32b, shift); src_temp3_4x32b = _mm_srai_epi32(src_temp3_4x32b, shift); /*i4_tmp = (i4_tmp >> shift) + off0; */ src_temp0_4x32b = _mm_add_epi32(src_temp0_4x32b, off0_4x32b); src_temp1_4x32b = _mm_add_epi32(src_temp1_4x32b, off0_4x32b); /*i4_tmp = (i4_tmp >> shift) + off0; */ /* Last 4 pixels */ src_temp2_4x32b = _mm_add_epi32(src_temp2_4x32b, off0_4x32b); src_temp3_4x32b = _mm_add_epi32(src_temp3_4x32b, off0_4x32b); src_temp0_4x32b = _mm_packs_epi32(src_temp0_4x32b, src_temp2_4x32b); src_temp1_4x32b = _mm_packs_epi32(src_temp1_4x32b, src_temp3_4x32b); /* pu1_dst[col] = CLIP_U8(i4_tmp); */ src_temp0_4x32b = _mm_packus_epi16(src_temp0_4x32b, src_temp0_4x32b); src_temp1_4x32b = _mm_packus_epi16(src_temp1_4x32b, src_temp1_4x32b); /* store four 8-bit output values */ _mm_storel_epi64((__m128i *)(pu1_dst + 0 * dst_strd), src_temp0_4x32b); /* row = 0*/ _mm_storel_epi64((__m128i *)(pu1_dst + 1 * dst_strd), src_temp1_4x32b); /* row = 1*/ pi2_src += 8; /* Pointer update */ pu1_dst += 8; /* Pointer update */ } /* inner loop ends here(4-output values in single iteration) */ pi2_src = pi2_src - wdx2 + 2 * src_strd; /* Pointer update */ pu1_dst = pu1_dst - wdx2 + 2 * dst_strd; /* Pointer update */ } } else /* 2*wd multiple of 4 case */ { WORD32 dst0, dst1; /* outer for loop starts from here */ for(row = 0; row < ht; row += 2) { for(col = 0; col < wdx2; col += 4) { /* row = 0 */ /*load 8 pixel values from 7:0 pos. relative to cur. pos.*/ src_temp0_4x32b = _mm_loadu_si128((__m128i *)(pi2_src)); /* row = 1 */ src_temp1_4x32b = _mm_loadu_si128((__m128i *)(pi2_src + src_strd)); /* considering pix. 4:0 by converting 16-into 32 bit */ src_temp0_4x32b = _mm_cvtepi16_epi32(src_temp0_4x32b); src_temp1_4x32b = _mm_cvtepi16_epi32(src_temp1_4x32b); /* (pi2_src[col] + lvl_shift)*/ src_temp0_4x32b = _mm_add_epi32(src_temp0_4x32b, lvl_shift_4x32b); src_temp1_4x32b = _mm_add_epi32(src_temp1_4x32b, lvl_shift_4x32b); /*i4_tmp = (pi2_src[col] + lvl_shift) * wgt0*/ src_temp0_4x32b = _mm_mullo_epi32(src_temp0_4x32b, wgt0_4x32b); src_temp1_4x32b = _mm_mullo_epi32(src_temp1_4x32b, wgt0_4x32b); /* i4_tmp += 1 << (shift - 1) */ src_temp0_4x32b = _mm_add_epi32(src_temp0_4x32b, const_temp_4x32b); src_temp1_4x32b = _mm_add_epi32(src_temp1_4x32b, const_temp_4x32b); /* (i4_tmp >> shift) */ src_temp0_4x32b = _mm_srai_epi32(src_temp0_4x32b, shift); src_temp1_4x32b = _mm_srai_epi32(src_temp1_4x32b, shift); /*i4_tmp = (i4_tmp >> shift) + off0; */ src_temp0_4x32b = _mm_add_epi32(src_temp0_4x32b, off0_4x32b); src_temp1_4x32b = _mm_add_epi32(src_temp1_4x32b, off0_4x32b); src_temp0_4x32b = _mm_packs_epi32(src_temp0_4x32b, src_temp1_4x32b); /* pu1_dst[col] = CLIP_U8(i4_tmp); */ src_temp0_4x32b = _mm_packus_epi16(src_temp0_4x32b, src_temp0_4x32b); dst0 = _mm_cvtsi128_si32(src_temp0_4x32b); /* dst row = 1 to 3 */ src_temp1_4x32b = _mm_shuffle_epi32(src_temp0_4x32b, 1); /* store four 8-bit output values */ *(WORD32 *)(&pu1_dst[0 * dst_strd]) = dst0; dst1 = _mm_cvtsi128_si32(src_temp1_4x32b); /* row = 1 */ *(WORD32 *)(&pu1_dst[1 * dst_strd]) = dst1; pi2_src += 4; /* Pointer update */ pu1_dst += 4; /* Pointer update */ } /* inner loop ends here(4-output values in single iteration) */ pi2_src = pi2_src - wdx2 + 2 * src_strd; /* Pointer update */ pu1_dst = pu1_dst - wdx2 + 2 * dst_strd; /* Pointer update */ } } } } /** ******************************************************************************* * * @brief * Does bi-weighted prediction on the arrays pointed by pi2_src1 and * pi2_src2 and stores it at location pointed by pi2_dst * * @par Description: * dst = ( (src1 + lvl_shift1)*wgt0 + (src2 + lvl_shift2)*wgt1 + (off0 + * off1 + 1) << (shift - 1) ) >> shift * * @param[in] pi2_src1 * Pointer to source 1 * * @param[in] pi2_src2 * Pointer to source 2 * * @param[out] pu1_dst * Pointer to destination * * @param[in] src_strd1 * Source stride 1 * * @param[in] src_strd2 * Source stride 2 * * @param[in] dst_strd * Destination stride * * @param[in] wgt0 * weight to be multiplied to source 1 * * @param[in] off0 * offset 0 * * @param[in] wgt1 * weight to be multiplied to source 2 * * @param[in] off1 * offset 1 * * @param[in] shift * (14 Bit depth) + log2_weight_denominator * * @param[in] lvl_shift1 * added before shift and offset * * @param[in] lvl_shift2 * added before shift and offset * * @param[in] ht * height of the source * * @param[in] wd * width of the source * * @returns * * @remarks * None * ******************************************************************************* */ void ihevc_weighted_pred_bi_sse42(WORD16 *pi2_src1, WORD16 *pi2_src2, UWORD8 *pu1_dst, WORD32 src_strd1, WORD32 src_strd2, WORD32 dst_strd, WORD32 wgt0, WORD32 off0, WORD32 wgt1, WORD32 off1, WORD32 shift, WORD32 lvl_shift1, WORD32 lvl_shift2, WORD32 ht, WORD32 wd) { WORD32 row, col, temp; __m128i src_temp1_4x32b, src_temp2_4x32b, src_temp3_4x32b, src_temp4_4x32b; __m128i const_temp_4x32b, lvl_shift1_4x32b, lvl_shift2_4x32b, wgt0_4x32b, wgt1_4x32b; ASSERT(wd % 4 == 0); /* checking assumption*/ ASSERT(ht % 2 == 0); /* checking assumption*/ temp = (off0 + off1 + 1) << (shift - 1); // seting values in register const_temp_4x32b = _mm_set1_epi32(temp); lvl_shift1_4x32b = _mm_set1_epi32(lvl_shift1); lvl_shift2_4x32b = _mm_set1_epi32(lvl_shift2); wgt0_4x32b = _mm_set1_epi32(wgt0); wgt1_4x32b = _mm_set1_epi32(wgt1); if(0 == (wd & 7)) /* wd multiple of 8 case */ { __m128i src_temp5_4x32b, src_temp6_4x32b, src_temp7_4x32b, src_temp8_4x32b; /* outer for loop starts from here */ for(row = 0; row < ht; row += 2) { for(col = 0; col < wd; col += 8) { /*load 8 pixel values from 7:0 pos. relative to cur. pos.*/ src_temp1_4x32b = _mm_loadu_si128((__m128i *)(pi2_src1)); /* row = 0 */ src_temp2_4x32b = _mm_loadu_si128((__m128i *)(pi2_src2)); /* row = 0 */ src_temp3_4x32b = _mm_loadu_si128((__m128i *)(pi2_src1 + 1 * src_strd1)); /* row = 1 */ src_temp4_4x32b = _mm_loadu_si128((__m128i *)(pi2_src2 + 1 * src_strd2)); /* row = 1 */ /* Next 4 pixels */ src_temp5_4x32b = _mm_loadu_si128((__m128i *)(pi2_src1 + 4)); /* row = 0 */ src_temp6_4x32b = _mm_loadu_si128((__m128i *)(pi2_src2 + 4)); /* row = 0 */ src_temp7_4x32b = _mm_loadu_si128((__m128i *)(pi2_src1 + 1 * src_strd1 + 4)); /* row = 1 */ src_temp8_4x32b = _mm_loadu_si128((__m128i *)(pi2_src2 + 1 * src_strd2 + 4)); /* row = 1 */ /* considering pix. 4:0 by converting 16-into 32 bit */ src_temp1_4x32b = _mm_cvtepi16_epi32(src_temp1_4x32b); src_temp2_4x32b = _mm_cvtepi16_epi32(src_temp2_4x32b); /* (pi2_src1[col] + lvl_shift1) */ src_temp1_4x32b = _mm_add_epi32(src_temp1_4x32b, lvl_shift1_4x32b); /* (pi2_src2[col] + lvl_shift2) */ src_temp2_4x32b = _mm_add_epi32(src_temp2_4x32b, lvl_shift2_4x32b); /*i4_tmp = (pi2_src1[col] + lvl_shift1) * wgt0 */ src_temp1_4x32b = _mm_mullo_epi32(src_temp1_4x32b, wgt0_4x32b); /*(pi2_src2[col] + lvl_shift2) * wgt1 */ src_temp2_4x32b = _mm_mullo_epi32(src_temp2_4x32b, wgt1_4x32b); src_temp3_4x32b = _mm_cvtepi16_epi32(src_temp3_4x32b); src_temp4_4x32b = _mm_cvtepi16_epi32(src_temp4_4x32b); src_temp3_4x32b = _mm_add_epi32(src_temp3_4x32b, lvl_shift1_4x32b); src_temp4_4x32b = _mm_add_epi32(src_temp4_4x32b, lvl_shift2_4x32b); src_temp3_4x32b = _mm_mullo_epi32(src_temp3_4x32b, wgt0_4x32b); src_temp4_4x32b = _mm_mullo_epi32(src_temp4_4x32b, wgt1_4x32b); /* Next 4 Pixels */ src_temp5_4x32b = _mm_cvtepi16_epi32(src_temp5_4x32b); src_temp6_4x32b = _mm_cvtepi16_epi32(src_temp6_4x32b); src_temp5_4x32b = _mm_add_epi32(src_temp5_4x32b, lvl_shift1_4x32b); src_temp6_4x32b = _mm_add_epi32(src_temp6_4x32b, lvl_shift2_4x32b); src_temp5_4x32b = _mm_mullo_epi32(src_temp5_4x32b, wgt0_4x32b); src_temp6_4x32b = _mm_mullo_epi32(src_temp6_4x32b, wgt1_4x32b); src_temp7_4x32b = _mm_cvtepi16_epi32(src_temp7_4x32b); src_temp8_4x32b = _mm_cvtepi16_epi32(src_temp8_4x32b); src_temp7_4x32b = _mm_add_epi32(src_temp7_4x32b, lvl_shift1_4x32b); src_temp8_4x32b = _mm_add_epi32(src_temp8_4x32b, lvl_shift2_4x32b); src_temp7_4x32b = _mm_mullo_epi32(src_temp7_4x32b, wgt0_4x32b); src_temp8_4x32b = _mm_mullo_epi32(src_temp8_4x32b, wgt1_4x32b); /* (pi2_src1[col] + lvl_shift1) * wgt0 + (pi2_src2[col] + lvl_shift2) * wgt1 */ src_temp1_4x32b = _mm_add_epi32(src_temp1_4x32b, src_temp2_4x32b); src_temp3_4x32b = _mm_add_epi32(src_temp3_4x32b, src_temp4_4x32b); /* i4_tmp += (off0 + off1 + 1) << (shift - 1); */ src_temp1_4x32b = _mm_add_epi32(src_temp1_4x32b, const_temp_4x32b); src_temp3_4x32b = _mm_add_epi32(src_temp3_4x32b, const_temp_4x32b); /* (i4_tmp >> shift) */ src_temp1_4x32b = _mm_srai_epi32(src_temp1_4x32b, shift); src_temp3_4x32b = _mm_srai_epi32(src_temp3_4x32b, shift); /* Next 4 Pixels */ src_temp5_4x32b = _mm_add_epi32(src_temp5_4x32b, src_temp6_4x32b); src_temp7_4x32b = _mm_add_epi32(src_temp7_4x32b, src_temp8_4x32b); src_temp5_4x32b = _mm_add_epi32(src_temp5_4x32b, const_temp_4x32b); src_temp7_4x32b = _mm_add_epi32(src_temp7_4x32b, const_temp_4x32b); src_temp5_4x32b = _mm_srai_epi32(src_temp5_4x32b, shift); src_temp7_4x32b = _mm_srai_epi32(src_temp7_4x32b, shift); src_temp1_4x32b = _mm_packs_epi32(src_temp1_4x32b, src_temp5_4x32b); src_temp3_4x32b = _mm_packs_epi32(src_temp3_4x32b, src_temp7_4x32b); /* pu1_dst[col] = CLIP_U8(i4_tmp >> shift); */ src_temp1_4x32b = _mm_packus_epi16(src_temp1_4x32b, src_temp1_4x32b); src_temp3_4x32b = _mm_packus_epi16(src_temp3_4x32b, src_temp3_4x32b); /* store four 8-bit output values */ _mm_storel_epi64((__m128i *)(pu1_dst + 0 * dst_strd), src_temp1_4x32b); /* row = 0*/ _mm_storel_epi64((__m128i *)(pu1_dst + 1 * dst_strd), src_temp3_4x32b); /* row = 1*/ pi2_src1 += 8; /* Pointer update */ pi2_src2 += 8; /* Pointer update */ pu1_dst += 8; /* Pointer update */ } /* inner loop ends here(4-output values in single iteration) */ pi2_src1 = pi2_src1 - wd + 2 * src_strd1; /* Pointer update */ pi2_src2 = pi2_src2 - wd + 2 * src_strd2; /* Pointer update */ pu1_dst = pu1_dst - wd + 2 * dst_strd; /* Pointer update */ } /* outer loop ends */ } else /* wd multiple of 4 case */ { WORD32 dst0, dst1; /* outer for loop starts from here */ for(row = 0; row < ht; row += 2) { for(col = 0; col < wd; col += 4) { /*load 8 pixel values from 7:0 pos. relative to cur. pos.*/ src_temp1_4x32b = _mm_loadu_si128((__m128i *)(pi2_src1)); /* row = 0 */ src_temp2_4x32b = _mm_loadu_si128((__m128i *)(pi2_src2)); /* row = 0 */ src_temp3_4x32b = _mm_loadu_si128((__m128i *)(pi2_src1 + 1 * src_strd1)); /* row = 1 */ src_temp4_4x32b = _mm_loadu_si128((__m128i *)(pi2_src2 + 1 * src_strd2)); /* row = 1 */ /* considering pix. 4:0 by converting 16-into 32 bit */ src_temp1_4x32b = _mm_cvtepi16_epi32(src_temp1_4x32b); src_temp2_4x32b = _mm_cvtepi16_epi32(src_temp2_4x32b); /* (pi2_src1[col] + lvl_shift1) */ src_temp1_4x32b = _mm_add_epi32(src_temp1_4x32b, lvl_shift1_4x32b); /* (pi2_src2[col] + lvl_shift2) */ src_temp2_4x32b = _mm_add_epi32(src_temp2_4x32b, lvl_shift2_4x32b); /*i4_tmp = (pi2_src1[col] + lvl_shift1) * wgt0 */ src_temp1_4x32b = _mm_mullo_epi32(src_temp1_4x32b, wgt0_4x32b); /*(pi2_src2[col] + lvl_shift2) * wgt1 */ src_temp2_4x32b = _mm_mullo_epi32(src_temp2_4x32b, wgt1_4x32b); src_temp3_4x32b = _mm_cvtepi16_epi32(src_temp3_4x32b); src_temp4_4x32b = _mm_cvtepi16_epi32(src_temp4_4x32b); src_temp3_4x32b = _mm_add_epi32(src_temp3_4x32b, lvl_shift1_4x32b); src_temp4_4x32b = _mm_add_epi32(src_temp4_4x32b, lvl_shift2_4x32b); src_temp3_4x32b = _mm_mullo_epi32(src_temp3_4x32b, wgt0_4x32b); src_temp4_4x32b = _mm_mullo_epi32(src_temp4_4x32b, wgt1_4x32b); /* (pi2_src1[col] + lvl_shift1) * wgt0 + (pi2_src2[col] + lvl_shift2) * wgt1 */ src_temp1_4x32b = _mm_add_epi32(src_temp1_4x32b, src_temp2_4x32b); src_temp3_4x32b = _mm_add_epi32(src_temp3_4x32b, src_temp4_4x32b); /* i4_tmp += (off0 + off1 + 1) << (shift - 1); */ src_temp1_4x32b = _mm_add_epi32(src_temp1_4x32b, const_temp_4x32b); src_temp3_4x32b = _mm_add_epi32(src_temp3_4x32b, const_temp_4x32b); /* (i4_tmp >> shift) */ src_temp1_4x32b = _mm_srai_epi32(src_temp1_4x32b, shift); src_temp3_4x32b = _mm_srai_epi32(src_temp3_4x32b, shift); src_temp1_4x32b = _mm_packs_epi32(src_temp1_4x32b, src_temp3_4x32b); /* pu1_dst[col] = CLIP_U8(i4_tmp >> shift); */ src_temp1_4x32b = _mm_packus_epi16(src_temp1_4x32b, src_temp1_4x32b); dst0 = _mm_cvtsi128_si32(src_temp1_4x32b); /* dst row = 1 to 3 */ src_temp2_4x32b = _mm_shuffle_epi32(src_temp1_4x32b, 1); /* store four 8-bit output values */ *(WORD32 *)(&pu1_dst[0 * dst_strd]) = dst0; dst1 = _mm_cvtsi128_si32(src_temp2_4x32b); /* row = 1 to 3 */ *(WORD32 *)(&pu1_dst[1 * dst_strd]) = dst1; pi2_src1 += 4; /* Pointer update */ pi2_src2 += 4; /* Pointer update */ pu1_dst += 4; /* Pointer update */ } /* inner loop ends here(4-output values in single iteration) */ pi2_src1 = pi2_src1 - wd + 2 * src_strd1; /* Pointer update */ pi2_src2 = pi2_src2 - wd + 2 * src_strd2; /* Pointer update */ pu1_dst = pu1_dst - wd + 2 * dst_strd; /* Pointer update */ } /* outer loop ends */ } } /** ******************************************************************************* * * @brief * Does chroma bi-weighted prediction on the arrays pointed by pi2_src1 and * pi2_src2 and stores it at location pointed by pi2_dst * * @par Description: * dst = ( (src1 + lvl_shift1)*wgt0 + (src2 + lvl_shift2)*wgt1 + (off0 + * off1 + 1) << (shift - 1) ) >> shift * * @param[in] pi2_src1 * Pointer to source 1 * * @param[in] pi2_src2 * Pointer to source 2 * * @param[out] pu1_dst * Pointer to destination * * @param[in] src_strd1 * Source stride 1 * * @param[in] src_strd2 * Source stride 2 * * @param[in] dst_strd * Destination stride * * @param[in] wgt0 * weight to be multiplied to source 1 * * @param[in] off0 * offset 0 * * @param[in] wgt1 * weight to be multiplied to source 2 * * @param[in] off1 * offset 1 * * @param[in] shift * (14 Bit depth) + log2_weight_denominator * * @param[in] lvl_shift1 * added before shift and offset * * @param[in] lvl_shift2 * added before shift and offset * * @param[in] ht * height of the source * * @param[in] wd * width of the source (each colour component) * * @returns * * @remarks * None * ******************************************************************************* */ void ihevc_weighted_pred_chroma_bi_sse42(WORD16 *pi2_src1, WORD16 *pi2_src2, UWORD8 *pu1_dst, WORD32 src_strd1, WORD32 src_strd2, WORD32 dst_strd, WORD32 wgt0_cb, WORD32 wgt0_cr, WORD32 off0_cb, WORD32 off0_cr, WORD32 wgt1_cb, WORD32 wgt1_cr, WORD32 off1_cb, WORD32 off1_cr, WORD32 shift, WORD32 lvl_shift1, WORD32 lvl_shift2, WORD32 ht, WORD32 wd) { WORD32 row, col, temp1, temp2; WORD32 wdx2; __m128i src_temp1_4x32b, src_temp2_4x32b, src_temp3_4x32b, src_temp4_4x32b; __m128i const_temp_4x32b, lvl_shift1_4x32b, lvl_shift2_4x32b, wgt0_4x32b, wgt1_4x32b; ASSERT(wd % 2 == 0); /* checking assumption*/ ASSERT(ht % 2 == 0); /* checking assumption*/ temp1 = (off0_cb + off1_cb + 1) << (shift - 1); temp2 = (off0_cr + off1_cr + 1) << (shift - 1); // seting values in register const_temp_4x32b = _mm_set_epi32(temp2, temp1, temp2, temp1); lvl_shift1_4x32b = _mm_set1_epi32(lvl_shift1); lvl_shift2_4x32b = _mm_set1_epi32(lvl_shift2); wgt0_4x32b = _mm_set_epi32(wgt0_cr, wgt0_cb, wgt0_cr, wgt0_cb); wgt1_4x32b = _mm_set_epi32(wgt1_cr, wgt1_cb, wgt1_cr, wgt1_cb); wdx2 = wd * 2; if(0 == (wdx2 & 7)) /* wdx2 multiple of 8 case */ { __m128i src_temp5_4x32b, src_temp6_4x32b, src_temp7_4x32b, src_temp8_4x32b; /* outer for loop starts from here */ for(row = 0; row < ht; row += 2) { for(col = 0; col < wdx2; col += 8) { /*load 8 pixel values from 7:0 pos. relative to cur. pos.*/ src_temp1_4x32b = _mm_loadu_si128((__m128i *)(pi2_src1)); /* row = 0 */ src_temp2_4x32b = _mm_loadu_si128((__m128i *)(pi2_src2)); /* row = 0 */ src_temp3_4x32b = _mm_loadu_si128((__m128i *)(pi2_src1 + 1 * src_strd1)); /* row = 1 */ src_temp4_4x32b = _mm_loadu_si128((__m128i *)(pi2_src2 + 1 * src_strd2)); /* row = 1 */ /* Next 4 pixels */ src_temp5_4x32b = _mm_loadu_si128((__m128i *)(pi2_src1 + 4)); /* row = 0 */ src_temp6_4x32b = _mm_loadu_si128((__m128i *)(pi2_src2 + 4)); /* row = 0 */ src_temp7_4x32b = _mm_loadu_si128((__m128i *)(pi2_src1 + 1 * src_strd1 + 4)); /* row = 1 */ src_temp8_4x32b = _mm_loadu_si128((__m128i *)(pi2_src2 + 1 * src_strd2 + 4)); /* row = 1 */ /* considering pix. 4:0 by converting 16-into 32 bit */ src_temp1_4x32b = _mm_cvtepi16_epi32(src_temp1_4x32b); src_temp2_4x32b = _mm_cvtepi16_epi32(src_temp2_4x32b); /* (pi2_src1[col] + lvl_shift1) */ src_temp1_4x32b = _mm_add_epi32(src_temp1_4x32b, lvl_shift1_4x32b); /* (pi2_src2[col] + lvl_shift2) */ src_temp2_4x32b = _mm_add_epi32(src_temp2_4x32b, lvl_shift2_4x32b); /*i4_tmp = (pi2_src1[col] + lvl_shift1) * wgt0 */ src_temp1_4x32b = _mm_mullo_epi32(src_temp1_4x32b, wgt0_4x32b); /*(pi2_src2[col] + lvl_shift2) * wgt1 */ src_temp2_4x32b = _mm_mullo_epi32(src_temp2_4x32b, wgt1_4x32b); src_temp3_4x32b = _mm_cvtepi16_epi32(src_temp3_4x32b); src_temp4_4x32b = _mm_cvtepi16_epi32(src_temp4_4x32b); src_temp3_4x32b = _mm_add_epi32(src_temp3_4x32b, lvl_shift1_4x32b); src_temp4_4x32b = _mm_add_epi32(src_temp4_4x32b, lvl_shift2_4x32b); src_temp3_4x32b = _mm_mullo_epi32(src_temp3_4x32b, wgt0_4x32b); src_temp4_4x32b = _mm_mullo_epi32(src_temp4_4x32b, wgt1_4x32b); /* Next 4 Pixels */ src_temp5_4x32b = _mm_cvtepi16_epi32(src_temp5_4x32b); src_temp6_4x32b = _mm_cvtepi16_epi32(src_temp6_4x32b); src_temp5_4x32b = _mm_add_epi32(src_temp5_4x32b, lvl_shift1_4x32b); src_temp6_4x32b = _mm_add_epi32(src_temp6_4x32b, lvl_shift2_4x32b); src_temp5_4x32b = _mm_mullo_epi32(src_temp5_4x32b, wgt0_4x32b); src_temp6_4x32b = _mm_mullo_epi32(src_temp6_4x32b, wgt1_4x32b); src_temp7_4x32b = _mm_cvtepi16_epi32(src_temp7_4x32b); src_temp8_4x32b = _mm_cvtepi16_epi32(src_temp8_4x32b); src_temp7_4x32b = _mm_add_epi32(src_temp7_4x32b, lvl_shift1_4x32b); src_temp8_4x32b = _mm_add_epi32(src_temp8_4x32b, lvl_shift2_4x32b); src_temp7_4x32b = _mm_mullo_epi32(src_temp7_4x32b, wgt0_4x32b); src_temp8_4x32b = _mm_mullo_epi32(src_temp8_4x32b, wgt1_4x32b); /* (pi2_src1[col] + lvl_shift1) * wgt0 + (pi2_src2[col] + lvl_shift2) * wgt1 */ src_temp1_4x32b = _mm_add_epi32(src_temp1_4x32b, src_temp2_4x32b); src_temp3_4x32b = _mm_add_epi32(src_temp3_4x32b, src_temp4_4x32b); /* i4_tmp += (off0 + off1 + 1) << (shift - 1); */ src_temp1_4x32b = _mm_add_epi32(src_temp1_4x32b, const_temp_4x32b); src_temp3_4x32b = _mm_add_epi32(src_temp3_4x32b, const_temp_4x32b); /* (i4_tmp >> shift) */ src_temp1_4x32b = _mm_srai_epi32(src_temp1_4x32b, shift); src_temp3_4x32b = _mm_srai_epi32(src_temp3_4x32b, shift); /* Next 4 Pixels */ src_temp5_4x32b = _mm_add_epi32(src_temp5_4x32b, src_temp6_4x32b); src_temp7_4x32b = _mm_add_epi32(src_temp7_4x32b, src_temp8_4x32b); src_temp5_4x32b = _mm_add_epi32(src_temp5_4x32b, const_temp_4x32b); src_temp7_4x32b = _mm_add_epi32(src_temp7_4x32b, const_temp_4x32b); src_temp5_4x32b = _mm_srai_epi32(src_temp5_4x32b, shift); src_temp7_4x32b = _mm_srai_epi32(src_temp7_4x32b, shift); src_temp1_4x32b = _mm_packs_epi32(src_temp1_4x32b, src_temp5_4x32b); src_temp3_4x32b = _mm_packs_epi32(src_temp3_4x32b, src_temp7_4x32b); /* pu1_dst[col] = CLIP_U8(i4_tmp >> shift); */ src_temp1_4x32b = _mm_packus_epi16(src_temp1_4x32b, src_temp1_4x32b); src_temp3_4x32b = _mm_packus_epi16(src_temp3_4x32b, src_temp3_4x32b); /* store four 8-bit output values */ _mm_storel_epi64((__m128i *)(pu1_dst + 0 * dst_strd), src_temp1_4x32b); /* row = 0*/ _mm_storel_epi64((__m128i *)(pu1_dst + 1 * dst_strd), src_temp3_4x32b); /* row = 1*/ pi2_src1 += 8; /* Pointer update */ pi2_src2 += 8; /* Pointer update */ pu1_dst += 8; /* Pointer update */ } /* inner loop ends here(4-output values in single iteration) */ pi2_src1 = pi2_src1 - wdx2 + 2 * src_strd1; /* Pointer update */ pi2_src2 = pi2_src2 - wdx2 + 2 * src_strd2; /* Pointer update */ pu1_dst = pu1_dst - wdx2 + 2 * dst_strd; /* Pointer update */ } /* outer loop ends */ } else /* wdx2 multiple of 4 case */ { WORD32 dst0, dst1; /* outer for loop starts from here */ for(row = 0; row < ht; row += 2) { for(col = 0; col < wdx2; col += 4) { /*load 8 pixel values from 7:0 pos. relative to cur. pos.*/ src_temp1_4x32b = _mm_loadu_si128((__m128i *)(pi2_src1)); /* row = 0 */ src_temp2_4x32b = _mm_loadu_si128((__m128i *)(pi2_src2)); /* row = 0 */ src_temp3_4x32b = _mm_loadu_si128((__m128i *)(pi2_src1 + 1 * src_strd1)); /* row = 1 */ src_temp4_4x32b = _mm_loadu_si128((__m128i *)(pi2_src2 + 1 * src_strd2)); /* row = 1 */ /* considering pix. 4:0 by converting 16-into 32 bit */ src_temp1_4x32b = _mm_cvtepi16_epi32(src_temp1_4x32b); src_temp2_4x32b = _mm_cvtepi16_epi32(src_temp2_4x32b); /* (pi2_src1[col] + lvl_shift1) */ src_temp1_4x32b = _mm_add_epi32(src_temp1_4x32b, lvl_shift1_4x32b); /* (pi2_src2[col] + lvl_shift2) */ src_temp2_4x32b = _mm_add_epi32(src_temp2_4x32b, lvl_shift2_4x32b); /*i4_tmp = (pi2_src1[col] + lvl_shift1) * wgt0 */ src_temp1_4x32b = _mm_mullo_epi32(src_temp1_4x32b, wgt0_4x32b); /*(pi2_src2[col] + lvl_shift2) * wgt1 */ src_temp2_4x32b = _mm_mullo_epi32(src_temp2_4x32b, wgt1_4x32b); src_temp3_4x32b = _mm_cvtepi16_epi32(src_temp3_4x32b); src_temp4_4x32b = _mm_cvtepi16_epi32(src_temp4_4x32b); src_temp3_4x32b = _mm_add_epi32(src_temp3_4x32b, lvl_shift1_4x32b); src_temp4_4x32b = _mm_add_epi32(src_temp4_4x32b, lvl_shift2_4x32b); src_temp3_4x32b = _mm_mullo_epi32(src_temp3_4x32b, wgt0_4x32b); src_temp4_4x32b = _mm_mullo_epi32(src_temp4_4x32b, wgt1_4x32b); /* (pi2_src1[col] + lvl_shift1) * wgt0 + (pi2_src2[col] + lvl_shift2) * wgt1 */ src_temp1_4x32b = _mm_add_epi32(src_temp1_4x32b, src_temp2_4x32b); src_temp3_4x32b = _mm_add_epi32(src_temp3_4x32b, src_temp4_4x32b); /* i4_tmp += (off0 + off1 + 1) << (shift - 1); */ src_temp1_4x32b = _mm_add_epi32(src_temp1_4x32b, const_temp_4x32b); src_temp3_4x32b = _mm_add_epi32(src_temp3_4x32b, const_temp_4x32b); /* (i4_tmp >> shift) */ src_temp1_4x32b = _mm_srai_epi32(src_temp1_4x32b, shift); src_temp3_4x32b = _mm_srai_epi32(src_temp3_4x32b, shift); src_temp1_4x32b = _mm_packs_epi32(src_temp1_4x32b, src_temp3_4x32b); /* pu1_dst[col] = CLIP_U8(i4_tmp >> shift); */ src_temp1_4x32b = _mm_packus_epi16(src_temp1_4x32b, src_temp1_4x32b); dst0 = _mm_cvtsi128_si32(src_temp1_4x32b); /* dst row = 1 to 3 */ src_temp2_4x32b = _mm_shuffle_epi32(src_temp1_4x32b, 1); /* store four 8-bit output values */ *(WORD32 *)(&pu1_dst[0 * dst_strd]) = dst0; dst1 = _mm_cvtsi128_si32(src_temp2_4x32b); /* row = 1 to 3 */ *(WORD32 *)(&pu1_dst[1 * dst_strd]) = dst1; pi2_src1 += 4; /* Pointer update */ pi2_src2 += 4; /* Pointer update */ pu1_dst += 4; /* Pointer update */ } /* inner loop ends here(4-output values in single iteration) */ pi2_src1 = pi2_src1 - wdx2 + 2 * src_strd1; /* Pointer update */ pi2_src2 = pi2_src2 - wdx2 + 2 * src_strd2; /* Pointer update */ pu1_dst = pu1_dst - wdx2 + 2 * dst_strd; /* Pointer update */ } } } /** ******************************************************************************* * * @brief * Does default bi-weighted prediction on the arrays pointed by pi2_src1 and * pi2_src2 and stores it at location pointed by pi2_dst * * @par Description: * dst = ( (src1 + lvl_shift1) + (src2 + lvl_shift2) + 1 << (shift - 1) ) * >> shift where shift = 15 - BitDepth * * @param[in] pi2_src1 * Pointer to source 1 * * @param[in] pi2_src2 * Pointer to source 2 * * @param[out] pu1_dst * Pointer to destination * * @param[in] src_strd1 * Source stride 1 * * @param[in] src_strd2 * Source stride 2 * * @param[in] dst_strd * Destination stride * * @param[in] lvl_shift1 * added before shift and offset * * @param[in] lvl_shift2 * added before shift and offset * * @param[in] ht * height of the source * * @param[in] wd * width of the source * * @returns * * @remarks * None * * Assumption : ht%4 == 0, wd%4 == 0 * shift == 7, (lvl_shift1+lvl_shift2) can take {0, 8K, 16K}. In that case, * final result will match even if intermediate precision is in 16 bit. * ******************************************************************************* */ void ihevc_weighted_pred_bi_default_sse42(WORD16 *pi2_src1, WORD16 *pi2_src2, UWORD8 *pu1_dst, WORD32 src_strd1, WORD32 src_strd2, WORD32 dst_strd, WORD32 lvl_shift1, WORD32 lvl_shift2, WORD32 ht, WORD32 wd) { WORD32 row, col, temp; WORD32 shift; __m128i src_temp1_8x16b, src_temp2_8x16b, src_temp3_8x16b, src_temp4_8x16b; __m128i const_temp_8x16b, lvl_shift1_8x16b, lvl_shift2_8x16b; __m128i src_temp5_8x16b, src_temp6_8x16b, src_temp7_8x16b, src_temp8_8x16b; ASSERT(wd % 4 == 0); /* checking assumption*/ ASSERT(ht % 2 == 0); /* checking assumption*/ shift = SHIFT_14_MINUS_BIT_DEPTH + 1; temp = 1 << (shift - 1); // seting values in register lvl_shift1_8x16b = _mm_set1_epi16(lvl_shift1); lvl_shift2_8x16b = _mm_set1_epi16(lvl_shift2); const_temp_8x16b = _mm_set1_epi16(temp); lvl_shift1_8x16b = _mm_adds_epi16(lvl_shift1_8x16b, lvl_shift2_8x16b); lvl_shift1_8x16b = _mm_adds_epi16(lvl_shift1_8x16b, const_temp_8x16b); if(0 == (ht & 3)) /* ht multiple of 4*/ { if(0 == (wd & 15)) /* wd multiple of 16 case */ { __m128i src_temp9_8x16b, src_temp10_8x16b, src_temp11_8x16b, src_temp12_8x16b; __m128i src_temp13_8x16b, src_temp14_8x16b, src_temp15_8x16b, src_temp16_8x16b; /* outer for loop starts from here */ for(row = 0; row < ht; row += 4) { for(col = 0; col < wd; col += 16) { /*load 8 pixel values */ /* First 8 Values */ src_temp1_8x16b = _mm_loadu_si128((__m128i *)(pi2_src1)); src_temp2_8x16b = _mm_loadu_si128((__m128i *)(pi2_src2)); /* row = 1 */ src_temp3_8x16b = _mm_loadu_si128((__m128i *)(pi2_src1 + src_strd1)); src_temp4_8x16b = _mm_loadu_si128((__m128i *)(pi2_src2 + src_strd2)); /* row = 2 */ src_temp5_8x16b = _mm_loadu_si128((__m128i *)(pi2_src1 + 2 * src_strd1)); src_temp6_8x16b = _mm_loadu_si128((__m128i *)(pi2_src2 + 2 * src_strd2)); /* row = 3 */ src_temp7_8x16b = _mm_loadu_si128((__m128i *)(pi2_src1 + 3 * src_strd1)); src_temp8_8x16b = _mm_loadu_si128((__m128i *)(pi2_src2 + 3 * src_strd2)); /*load 8 pixel values */ /* Second 8 Values */ src_temp9_8x16b = _mm_loadu_si128((__m128i *)(pi2_src1 + 8)); src_temp10_8x16b = _mm_loadu_si128((__m128i *)(pi2_src2 + 8)); /* row = 1 */ src_temp11_8x16b = _mm_loadu_si128((__m128i *)(pi2_src1 + src_strd1 + 8)); src_temp12_8x16b = _mm_loadu_si128((__m128i *)(pi2_src2 + src_strd2 + 8)); /* row = 2 */ src_temp13_8x16b = _mm_loadu_si128((__m128i *)(pi2_src1 + 2 * src_strd1 + 8)); src_temp14_8x16b = _mm_loadu_si128((__m128i *)(pi2_src2 + 2 * src_strd2 + 8)); /* (pi2_src1[col] + pi2_src2[col]) */ /* First 8 Values */ src_temp1_8x16b = _mm_adds_epi16(src_temp1_8x16b, src_temp2_8x16b); src_temp3_8x16b = _mm_adds_epi16(src_temp3_8x16b, src_temp4_8x16b); src_temp5_8x16b = _mm_adds_epi16(src_temp5_8x16b, src_temp6_8x16b); src_temp7_8x16b = _mm_adds_epi16(src_temp7_8x16b, src_temp8_8x16b); /*load 8 pixel values */ /* Second 8 Values */ /* row = 3 */ src_temp15_8x16b = _mm_loadu_si128((__m128i *)(pi2_src1 + 3 * src_strd1 + 8)); src_temp16_8x16b = _mm_loadu_si128((__m128i *)(pi2_src2 + 3 * src_strd2 + 8)); /* i4_tmp = (pi2_src1[col] + pi2_src2[col] + lvl_shift1 + lvl_shift2 + shift_value) */ /* First 8 Values */ src_temp1_8x16b = _mm_adds_epi16(src_temp1_8x16b, lvl_shift1_8x16b); src_temp3_8x16b = _mm_adds_epi16(src_temp3_8x16b, lvl_shift1_8x16b); src_temp5_8x16b = _mm_adds_epi16(src_temp5_8x16b, lvl_shift1_8x16b); src_temp7_8x16b = _mm_adds_epi16(src_temp7_8x16b, lvl_shift1_8x16b); /* (pi2_src1[col] + pi2_src2[col]) */ /* Second 8 Values */ src_temp9_8x16b = _mm_adds_epi16(src_temp9_8x16b, src_temp10_8x16b); src_temp11_8x16b = _mm_adds_epi16(src_temp11_8x16b, src_temp12_8x16b); src_temp13_8x16b = _mm_adds_epi16(src_temp13_8x16b, src_temp14_8x16b); src_temp15_8x16b = _mm_adds_epi16(src_temp15_8x16b, src_temp16_8x16b); /* (i4_tmp >> shift) */ /* First 8 Values */ src_temp1_8x16b = _mm_srai_epi16(src_temp1_8x16b, shift); src_temp3_8x16b = _mm_srai_epi16(src_temp3_8x16b, shift); src_temp5_8x16b = _mm_srai_epi16(src_temp5_8x16b, shift); src_temp7_8x16b = _mm_srai_epi16(src_temp7_8x16b, shift); /* i4_tmp = (pi2_src1[col] + pi2_src2[col] + lvl_shift1 + lvl_shift2 + shift_value) */ /* Second 8 Values */ src_temp9_8x16b = _mm_adds_epi16(src_temp9_8x16b, lvl_shift1_8x16b); src_temp11_8x16b = _mm_adds_epi16(src_temp11_8x16b, lvl_shift1_8x16b); src_temp13_8x16b = _mm_adds_epi16(src_temp13_8x16b, lvl_shift1_8x16b); src_temp15_8x16b = _mm_adds_epi16(src_temp15_8x16b, lvl_shift1_8x16b); /* (i4_tmp >> shift) */ /* Second 8 Values */ src_temp9_8x16b = _mm_srai_epi16(src_temp9_8x16b, shift); src_temp11_8x16b = _mm_srai_epi16(src_temp11_8x16b, shift); src_temp13_8x16b = _mm_srai_epi16(src_temp13_8x16b, shift); src_temp15_8x16b = _mm_srai_epi16(src_temp15_8x16b, shift); /* pu1_dst[col] = CLIP_U8(i4_tmp >> shift); */ /* 16 8 Values */ src_temp1_8x16b = _mm_packus_epi16(src_temp1_8x16b, src_temp9_8x16b); src_temp3_8x16b = _mm_packus_epi16(src_temp3_8x16b, src_temp11_8x16b); src_temp5_8x16b = _mm_packus_epi16(src_temp5_8x16b, src_temp13_8x16b); src_temp7_8x16b = _mm_packus_epi16(src_temp7_8x16b, src_temp15_8x16b); /* store four 8-bit output values */ /* 16 8 Values */ _mm_storeu_si128((__m128i *)(pu1_dst + 0 * dst_strd), src_temp1_8x16b); /* row = 0*/ _mm_storeu_si128((__m128i *)(pu1_dst + 1 * dst_strd), src_temp3_8x16b); /* row = 2*/ _mm_storeu_si128((__m128i *)(pu1_dst + 2 * dst_strd), src_temp5_8x16b); /* row = 1*/ _mm_storeu_si128((__m128i *)(pu1_dst + 3 * dst_strd), src_temp7_8x16b); /* row = 3*/ /* To update pointer */ pi2_src1 += 16; pi2_src2 += 16; pu1_dst += 16; } /* inner loop ends here(8-output values in single iteration) */ pi2_src1 = pi2_src1 - wd + 4 * src_strd1; /* Pointer update */ pi2_src2 = pi2_src2 - wd + 4 * src_strd2; /* Pointer update */ pu1_dst = pu1_dst - wd + 4 * dst_strd; /* Pointer update */ } } else if(0 == (wd & 7)) /* multiple of 8 case */ { /* outer for loop starts from here */ for(row = 0; row < ht; row += 4) { for(col = 0; col < wd; col += 8) { /*load 8 pixel values */ src_temp1_8x16b = _mm_loadu_si128((__m128i *)(pi2_src1)); src_temp2_8x16b = _mm_loadu_si128((__m128i *)(pi2_src2)); /* row = 1 */ src_temp3_8x16b = _mm_loadu_si128((__m128i *)(pi2_src1 + src_strd1)); src_temp4_8x16b = _mm_loadu_si128((__m128i *)(pi2_src2 + src_strd2)); /* row = 2 */ src_temp5_8x16b = _mm_loadu_si128((__m128i *)(pi2_src1 + 2 * src_strd1)); src_temp6_8x16b = _mm_loadu_si128((__m128i *)(pi2_src2 + 2 * src_strd2)); /* row = 3 */ src_temp7_8x16b = _mm_loadu_si128((__m128i *)(pi2_src1 + 3 * src_strd1)); src_temp8_8x16b = _mm_loadu_si128((__m128i *)(pi2_src2 + 3 * src_strd2)); /* (pi2_src1[col] + pi2_src2[col]) */ src_temp1_8x16b = _mm_adds_epi16(src_temp1_8x16b, src_temp2_8x16b); src_temp3_8x16b = _mm_adds_epi16(src_temp3_8x16b, src_temp4_8x16b); src_temp5_8x16b = _mm_adds_epi16(src_temp5_8x16b, src_temp6_8x16b); src_temp7_8x16b = _mm_adds_epi16(src_temp7_8x16b, src_temp8_8x16b); /* i4_tmp = (pi2_src1[col] + pi2_src2[col] + lvl_shift1 + lvl_shift2 + shift_value) */ src_temp1_8x16b = _mm_adds_epi16(src_temp1_8x16b, lvl_shift1_8x16b); src_temp3_8x16b = _mm_adds_epi16(src_temp3_8x16b, lvl_shift1_8x16b); src_temp5_8x16b = _mm_adds_epi16(src_temp5_8x16b, lvl_shift1_8x16b); src_temp7_8x16b = _mm_adds_epi16(src_temp7_8x16b, lvl_shift1_8x16b); /* (i4_tmp >> shift) */ src_temp1_8x16b = _mm_srai_epi16(src_temp1_8x16b, shift); src_temp3_8x16b = _mm_srai_epi16(src_temp3_8x16b, shift); src_temp5_8x16b = _mm_srai_epi16(src_temp5_8x16b, shift); src_temp7_8x16b = _mm_srai_epi16(src_temp7_8x16b, shift); /* pu1_dst[col] = CLIP_U8(i4_tmp >> shift); */ src_temp1_8x16b = _mm_packus_epi16(src_temp1_8x16b, src_temp1_8x16b); src_temp3_8x16b = _mm_packus_epi16(src_temp3_8x16b, src_temp3_8x16b); src_temp5_8x16b = _mm_packus_epi16(src_temp5_8x16b, src_temp5_8x16b); src_temp7_8x16b = _mm_packus_epi16(src_temp7_8x16b, src_temp7_8x16b); /* store four 8-bit output values */ _mm_storel_epi64((__m128i *)(pu1_dst + 0 * dst_strd), src_temp1_8x16b); /* row = 0*/ _mm_storel_epi64((__m128i *)(pu1_dst + 1 * dst_strd), src_temp3_8x16b); /* row = 2*/ _mm_storel_epi64((__m128i *)(pu1_dst + 2 * dst_strd), src_temp5_8x16b); /* row = 1*/ _mm_storel_epi64((__m128i *)(pu1_dst + 3 * dst_strd), src_temp7_8x16b); /* row = 3*/ /* To update pointer */ pi2_src1 += 8; pi2_src2 += 8; pu1_dst += 8; } /* inner loop ends here(8-output values in single iteration) */ pi2_src1 = pi2_src1 - wd + 4 * src_strd1; /* Pointer update */ pi2_src2 = pi2_src2 - wd + 4 * src_strd2; /* Pointer update */ pu1_dst = pu1_dst - wd + 4 * dst_strd; /* Pointer update */ } } else /* wd multiple of 4 case*/ { WORD32 dst0, dst1, dst2, dst3; /* outer for loop starts from here */ for(row = 0; row < ht; row += 4) { for(col = 0; col < wd; col += 4) { /*load 4 pixel values from 7:0 pos. relative to cur. pos.*/ src_temp1_8x16b = _mm_loadl_epi64((__m128i *)(pi2_src1)); /*load 8 pixel values from 7:0 pos. relative to cur. pos.*/ src_temp2_8x16b = _mm_loadl_epi64((__m128i *)(pi2_src2)); /* row = 1 */ src_temp3_8x16b = _mm_loadl_epi64((__m128i *)(pi2_src1 + src_strd1)); src_temp4_8x16b = _mm_loadl_epi64((__m128i *)(pi2_src2 + src_strd2)); /* row = 2 */ src_temp5_8x16b = _mm_loadl_epi64((__m128i *)(pi2_src1 + 2 * src_strd1)); src_temp6_8x16b = _mm_loadl_epi64((__m128i *)(pi2_src2 + 2 * src_strd2)); /* row = 3 */ src_temp7_8x16b = _mm_loadl_epi64((__m128i *)(pi2_src1 + 3 * src_strd1)); src_temp8_8x16b = _mm_loadl_epi64((__m128i *)(pi2_src2 + 3 * src_strd2)); /* Pack two rows together */ src_temp1_8x16b = _mm_unpacklo_epi64(src_temp1_8x16b, src_temp3_8x16b); src_temp2_8x16b = _mm_unpacklo_epi64(src_temp2_8x16b, src_temp4_8x16b); src_temp5_8x16b = _mm_unpacklo_epi64(src_temp5_8x16b, src_temp7_8x16b); src_temp6_8x16b = _mm_unpacklo_epi64(src_temp6_8x16b, src_temp8_8x16b); /* (pi2_src1[col] + pi2_src2[col]) */ src_temp1_8x16b = _mm_adds_epi16(src_temp1_8x16b, src_temp2_8x16b); src_temp5_8x16b = _mm_adds_epi16(src_temp5_8x16b, src_temp6_8x16b); /* i4_tmp = (pi2_src1[col] + pi2_src2[col] + lvl_shift1 + lvl_shift2 + shift_value) */ src_temp1_8x16b = _mm_adds_epi16(src_temp1_8x16b, lvl_shift1_8x16b); src_temp5_8x16b = _mm_adds_epi16(src_temp5_8x16b, lvl_shift1_8x16b); /* (i4_tmp >> shift) */ src_temp1_8x16b = _mm_srai_epi16(src_temp1_8x16b, shift); src_temp5_8x16b = _mm_srai_epi16(src_temp5_8x16b, shift); /* pu1_dst[col] = CLIP_U8(i4_tmp >> shift); */ src_temp1_8x16b = _mm_packus_epi16(src_temp1_8x16b, src_temp1_8x16b); src_temp5_8x16b = _mm_packus_epi16(src_temp5_8x16b, src_temp5_8x16b); dst0 = _mm_cvtsi128_si32(src_temp1_8x16b); /* dst row = 1 to 3 */ src_temp2_8x16b = _mm_shuffle_epi32(src_temp1_8x16b, 1); src_temp4_8x16b = _mm_shuffle_epi32(src_temp5_8x16b, 1); /* store four 8-bit output values */ *(WORD32 *)(&pu1_dst[0 * dst_strd]) = dst0; dst1 = _mm_cvtsi128_si32(src_temp2_8x16b); dst2 = _mm_cvtsi128_si32(src_temp5_8x16b); dst3 = _mm_cvtsi128_si32(src_temp4_8x16b); /* row = 1 to row = 3 */ *(WORD32 *)(&pu1_dst[1 * dst_strd]) = dst1; *(WORD32 *)(&pu1_dst[2 * dst_strd]) = dst2; *(WORD32 *)(&pu1_dst[3 * dst_strd]) = dst3; /* To update pointer */ pi2_src1 += 4; pi2_src2 += 4; pu1_dst += 4; } /* inner loop ends here(4-output values in single iteration) */ pi2_src1 = pi2_src1 - wd + 4 * src_strd1; /* Pointer update */ pi2_src2 = pi2_src2 - wd + 4 * src_strd2; /* Pointer update */ pu1_dst = pu1_dst - wd + 4 * dst_strd; /* Pointer update */ } } } else /* ht multiple of 2 case and wd multiple of 4 case*/ { WORD32 dst0, dst1; /* outer for loop starts from here */ for(row = 0; row < ht; row += 2) { for(col = 0; col < wd; col += 4) { /*load 4 pixel values from 7:0 pos. relative to cur. pos.*/ src_temp1_8x16b = _mm_loadl_epi64((__m128i *)(pi2_src1)); /*load 8 pixel values from 7:0 pos. relative to cur. pos.*/ src_temp2_8x16b = _mm_loadl_epi64((__m128i *)(pi2_src2)); /* row = 1 */ src_temp3_8x16b = _mm_loadl_epi64((__m128i *)(pi2_src1 + src_strd1)); src_temp4_8x16b = _mm_loadl_epi64((__m128i *)(pi2_src2 + src_strd2)); /* Pack two rows together */ src_temp1_8x16b = _mm_unpacklo_epi64(src_temp1_8x16b, src_temp3_8x16b); src_temp2_8x16b = _mm_unpacklo_epi64(src_temp2_8x16b, src_temp4_8x16b); /* (pi2_src1[col] + pi2_src2[col]) */ src_temp1_8x16b = _mm_adds_epi16(src_temp1_8x16b, src_temp2_8x16b); /* i4_tmp = (pi2_src1[col] + pi2_src2[col] + lvl_shift1 + lvl_shift2 + shift_value) */ src_temp1_8x16b = _mm_adds_epi16(src_temp1_8x16b, lvl_shift1_8x16b); /* (i4_tmp >> shift) */ src_temp1_8x16b = _mm_srai_epi16(src_temp1_8x16b, shift); /* pu1_dst[col] = CLIP_U8(i4_tmp >> shift); */ src_temp1_8x16b = _mm_packus_epi16(src_temp1_8x16b, src_temp1_8x16b); dst0 = _mm_cvtsi128_si32(src_temp1_8x16b); /* dst row = 1 to 3 */ src_temp2_8x16b = _mm_shuffle_epi32(src_temp1_8x16b, 1); /* store four 8-bit output values */ *(WORD32 *)(&pu1_dst[0 * dst_strd]) = dst0; dst1 = _mm_cvtsi128_si32(src_temp2_8x16b); /* row = 1 to row = 3 */ *(WORD32 *)(&pu1_dst[1 * dst_strd]) = dst1; /* To update pointer */ pi2_src1 += 4; pi2_src2 += 4; pu1_dst += 4; } /* inner loop ends here(4-output values in single iteration) */ pi2_src1 = pi2_src1 - wd + 2 * src_strd1; /* Pointer update */ pi2_src2 = pi2_src2 - wd + 2 * src_strd2; /* Pointer update */ pu1_dst = pu1_dst - wd + 2 * dst_strd; /* Pointer update */ } } }