1// Copyright 2019 Google LLC 2// 3// This source code is licensed under the BSD-style license found in the 4// LICENSE file in the root directory of this source tree. 5 6$assert CHANNEL_TILE % 4 == 0 7$assert CHANNEL_TILE >= 4 8$assert PIXEL_TILE == 1 9$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" 10#include <assert.h> 11 12#include <immintrin.h> 13 14#include <xnnpack/ibilinear.h> 15 16 17void xnn_f32_ibilinear_ukernel__sse_c${CHANNEL_TILE}${"" if PIXEL_TILE == 1 else "x%d" % PIXEL_TILE}( 18 size_t output_pixels, 19 size_t channels, 20 const float**restrict input, 21 size_t input_offset, 22 const float*restrict weights, 23 float*restrict output, 24 size_t output_increment) XNN_DISABLE_TSAN 25{ 26 assert(output_pixels != 0); 27 assert(channels != 0); 28 assert(channels % sizeof(float) == 0); 29 30 do { 31 const float* i0 = (const float*) ((uintptr_t) input[0] + input_offset); 32 const float* i1 = (const float*) ((uintptr_t) input[1] + input_offset); 33 const float* i2 = (const float*) ((uintptr_t) input[2] + input_offset); 34 const float* i3 = (const float*) ((uintptr_t) input[3] + input_offset); 35 input += 4; 36 37 __m128 valphahv = _mm_loadl_pi(_mm_undefined_ps(), (const __m64*) weights); 38 valphahv = _mm_unpacklo_ps(valphahv, valphahv); 39 const __m128 valphah = _mm_movelh_ps(valphahv, valphahv); 40 const __m128 valphav = _mm_movehl_ps(valphahv, valphahv); 41 weights += 2; 42 43 size_t c = channels; 44 for (; c >= ${CHANNEL_TILE} * sizeof(float); c -= ${CHANNEL_TILE} * sizeof(float)) { 45 const __m128 vtl${ABC[0:4]} = _mm_loadu_ps(i0); 46 const __m128 vtr${ABC[0:4]} = _mm_loadu_ps(i1); 47 const __m128 vbl${ABC[0:4]} = _mm_loadu_ps(i2); 48 const __m128 vbr${ABC[0:4]} = _mm_loadu_ps(i3); 49 $for C in range(4, CHANNEL_TILE, 4): 50 const __m128 vtl${ABC[C:C+4]} = _mm_loadu_ps(i0 + ${C}); 51 const __m128 vtr${ABC[C:C+4]} = _mm_loadu_ps(i1 + ${C}); 52 const __m128 vbl${ABC[C:C+4]} = _mm_loadu_ps(i2 + ${C}); 53 const __m128 vbr${ABC[C:C+4]} = _mm_loadu_ps(i3 + ${C}); 54 i0 += ${CHANNEL_TILE}; 55 i1 += ${CHANNEL_TILE}; 56 i2 += ${CHANNEL_TILE}; 57 i3 += ${CHANNEL_TILE}; 58 59 $for C in range(0, CHANNEL_TILE, 4): 60 const __m128 vtd${ABC[C:C+4]} = _mm_sub_ps(vtr${ABC[C:C+4]}, vtl${ABC[C:C+4]}); 61 const __m128 vbd${ABC[C:C+4]} = _mm_sub_ps(vbr${ABC[C:C+4]}, vbl${ABC[C:C+4]}); 62 63 $for C in range(0, CHANNEL_TILE, 4): 64 const __m128 vt${ABC[C:C+4]} = _mm_add_ps(vtl${ABC[C:C+4]}, _mm_mul_ps(vtd${ABC[C:C+4]}, valphah)); 65 const __m128 vb${ABC[C:C+4]} = _mm_add_ps(vbl${ABC[C:C+4]}, _mm_mul_ps(vbd${ABC[C:C+4]}, valphah)); 66 67 $for C in range(0, CHANNEL_TILE, 4): 68 const __m128 vd${ABC[C:C+4]} = _mm_sub_ps(vb${ABC[C:C+4]}, vt${ABC[C:C+4]}); 69 70 $for C in range(0, CHANNEL_TILE, 4): 71 const __m128 vo${ABC[C:C+4]} = _mm_add_ps(vt${ABC[C:C+4]}, _mm_mul_ps(vd${ABC[C:C+4]}, valphav)); 72 73 _mm_storeu_ps(output, vo${ABC[0:4]}); 74 $for C in range(4, CHANNEL_TILE, 4): 75 _mm_storeu_ps(output + ${C}, vo${ABC[C:C+4]}); 76 output += ${CHANNEL_TILE}; 77 } 78 $if CHANNEL_TILE > 4: 79 for (; c >= 4 * sizeof(float); c -= 4 * sizeof(float)) { 80 const __m128 vtl0123 = _mm_loadu_ps(i0); 81 const __m128 vtr0123 = _mm_loadu_ps(i1); 82 const __m128 vbl0123 = _mm_loadu_ps(i2); 83 const __m128 vbr0123 = _mm_loadu_ps(i3); 84 i0 += 4; 85 i1 += 4; 86 i2 += 4; 87 i3 += 4; 88 89 const __m128 vtd0123 = _mm_sub_ps(vtr0123, vtl0123); 90 const __m128 vbd0123 = _mm_sub_ps(vbr0123, vbl0123); 91 92 const __m128 vt0123 = _mm_add_ps(vtl0123, _mm_mul_ps(vtd0123, valphah)); 93 const __m128 vb0123 = _mm_add_ps(vbl0123, _mm_mul_ps(vbd0123, valphah)); 94 95 const __m128 vd0123 = _mm_sub_ps(vb0123, vt0123); 96 97 const __m128 vo0123 = _mm_add_ps(vt0123, _mm_mul_ps(vd0123, valphav)); 98 99 _mm_storeu_ps(output, vo0123); 100 output += 4; 101 } 102 if XNN_UNLIKELY(c != 0) { 103 const __m128 vtl0123 = _mm_loadu_ps(i0); 104 const __m128 vtr0123 = _mm_loadu_ps(i1); 105 const __m128 vbl0123 = _mm_loadu_ps(i2); 106 const __m128 vbr0123 = _mm_loadu_ps(i3); 107 108 const __m128 vtd0123 = _mm_sub_ps(vtr0123, vtl0123); 109 const __m128 vbd0123 = _mm_sub_ps(vbr0123, vbl0123); 110 111 const __m128 vt0123 = _mm_add_ps(vtl0123, _mm_mul_ps(vtd0123, valphah)); 112 const __m128 vb0123 = _mm_add_ps(vbl0123, _mm_mul_ps(vbd0123, valphah)); 113 114 const __m128 vd0123 = _mm_sub_ps(vb0123, vt0123); 115 116 __m128 vo0123 = _mm_add_ps(vt0123, _mm_mul_ps(vd0123, valphav)); 117 118 if (c & (2 * sizeof(float))) { 119 _mm_storel_pi((__m64*) output, vo0123); 120 vo0123 = _mm_movehl_ps(vo0123, vo0123); 121 output += 2; 122 } 123 if (c & (1 * sizeof(float))) { 124 _mm_store_ss(output, vo0123); 125 output += 1; 126 } 127 } 128 129 output = (float*) ((uintptr_t) output + output_increment); 130 } while (--output_pixels != 0); 131} 132