1// Copyright 2019 Google LLC 2// 3// This source code is licensed under the BSD-style license found in the 4// LICENSE file in the root directory of this source tree. 5 6$assert BATCH_TILE >= 1 7$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" 8#include <assert.h> 9#include <math.h> 10 11#include <xnnpack/common.h> 12#include <xnnpack/vunary.h> 13 14#include <fp16/bitcasts.h> 15 16 17// Note redefine as uint32[] to avoid redundant bitcasts. 18extern XNN_INTERNAL const uint32_t xnn_table_exp2minus_k_over_2048[2048]; 19 20void xnn_f32_vsigmoid_ukernel__scalar_rr2_lut2048_p1_div_x${BATCH_TILE}( 21 size_t n, 22 const float* x, 23 float* y, 24 const union xnn_f32_sigmoid_params params[restrict XNN_MIN_ELEMENTS(1)]) 25{ 26 assert(n % sizeof(float) == 0); 27 28 const float vmagic_bias = params->scalar_rr2_lut2048_p1.magic_bias; 29 const float vminus_log2e = params->scalar_rr2_lut2048_p1.minus_log2e; 30 const uint32_t vindex_mask = UINT32_C(0x7FF); 31 const float vln2_hi = params->scalar_rr2_lut2048_p1.ln2_hi; 32 const float vln2_lo = params->scalar_rr2_lut2048_p1.ln2_lo; 33 const float vc1 = params->scalar_rr2_lut2048_p1.c1; 34 const float vone = params->scalar_rr2_lut2048_p1.one; 35 const float vdenorm_cutoff = params->scalar_rr2_lut2048_p1.denorm_cutoff; 36 37 $if BATCH_TILE > 1: 38 for (; n >= ${BATCH_TILE} * sizeof(float); n -= ${BATCH_TILE} * sizeof(float)) { 39 $for N in range(BATCH_TILE): 40 const float vx${N} = x[${N}]; 41 x += ${BATCH_TILE}; 42 43 $for N in range(BATCH_TILE): 44 const float vz${N} = fabsf(vx${N}); 45 46 $for N in range(BATCH_TILE): 47 float vn${N} = vz${N} * vminus_log2e + vmagic_bias; 48 49 $for N in range(BATCH_TILE): 50 const uint32_t ve${N} = fp32_to_bits(vn${N}) << 12; 51 52 $for N in range(BATCH_TILE): 53 const uint32_t vidx${N} = fp32_to_bits(vn${N}) & vindex_mask; 54 const float vs${N} = fp32_from_bits(xnn_table_exp2minus_k_over_2048[vidx${N}] + ve${N}); 55 56 $for N in range(BATCH_TILE): 57 vn${N} -= vmagic_bias; 58 59 $for N in range(BATCH_TILE): 60 float vt${N} = vn${N} * vln2_hi + vz${N}; 61 62 $for N in range(BATCH_TILE): 63 vt${N} = vn${N} * vln2_lo + vt${N}; 64 65 $for N in range(BATCH_TILE): 66 const float vp${N} = vt${N} * vc1; 67 68 $for N in range(BATCH_TILE): 69 const float vy${N} = vp${N} * vs${N} + vs${N}; 70 71 $for N in range(BATCH_TILE): 72 const float vd${N} = vy${N} + vone; 73 74 $for N in range(BATCH_TILE): 75 float vf${N} = vy${N} / vd${N}; 76 77 $for N in range(BATCH_TILE): 78 if XNN_UNPREDICTABLE(vz${N} > vdenorm_cutoff) { 79 vf${N} = 0.0f; 80 } 81 82 $for N in range(BATCH_TILE): 83 if XNN_UNPREDICTABLE(vx${N} > 0.0f) { 84 vf${N} = vone - vf${N}; 85 } 86 87 $for N in range(BATCH_TILE): 88 y[${N}] = vf${N}; 89 y += ${BATCH_TILE}; 90 } 91 $if BATCH_TILE == 1: 92 do { 93 const float vx = *x++; 94 95 const float vz = fabsf(vx); 96 97 float vn = vz * vminus_log2e + vmagic_bias; 98 const uint32_t ve = fp32_to_bits(vn) << 12; 99 const uint32_t vidx = fp32_to_bits(vn) & vindex_mask; 100 const float vs = fp32_from_bits(xnn_table_exp2minus_k_over_2048[vidx] + ve); 101 vn -= vmagic_bias; 102 103 float vt = vn * vln2_hi + vz; 104 vt = vn * vln2_lo + vt; 105 106 const float vp = vt * vc1; 107 const float vy = vp * vs + vs; 108 109 const float vd = vy + vone; 110 float vf = vy / vd; 111 if XNN_UNPREDICTABLE(vz > vdenorm_cutoff) { 112 vf = 0.0f; 113 } 114 if XNN_UNPREDICTABLE(vx > 0.0f) { 115 vf = vone - vf; 116 } 117 118 *y++ = vf; 119 120 n -= sizeof(float); 121 } while (n != 0); 122 $elif BATCH_TILE == 2: 123 if XNN_UNLIKELY(n != 0) { 124 const float vx = *x; 125 126 const float vz = fabsf(vx); 127 128 float vn = vz * vminus_log2e + vmagic_bias; 129 const uint32_t ve = fp32_to_bits(vn) << 12; 130 const uint32_t vidx = fp32_to_bits(vn) & vindex_mask; 131 const float vs = fp32_from_bits(xnn_table_exp2minus_k_over_2048[vidx] + ve); 132 vn -= vmagic_bias; 133 134 float vt = vn * vln2_hi + vz; 135 vt = vn * vln2_lo + vt; 136 137 const float vp = vt * vc1; 138 const float vy = vp * vs + vs; 139 140 const float vd = vy + vone; 141 float vf = vy / vd; 142 if XNN_UNPREDICTABLE(vz > vdenorm_cutoff) { 143 vf = 0.0f; 144 } 145 if XNN_UNPREDICTABLE(vx > 0.0f) { 146 vf = vone - vf; 147 } 148 149 *y = vf; 150 } 151 $else: 152 if XNN_UNLIKELY(n != 0) { 153 do { 154 const float vx = *x++; 155 156 const float vz = fabsf(vx); 157 158 float vn = vz * vminus_log2e + vmagic_bias; 159 const uint32_t ve = fp32_to_bits(vn) << 12; 160 const uint32_t vidx = fp32_to_bits(vn) & vindex_mask; 161 const float vs = fp32_from_bits(xnn_table_exp2minus_k_over_2048[vidx] + ve); 162 vn -= vmagic_bias; 163 164 float vt = vn * vln2_hi + vz; 165 vt = vn * vln2_lo + vt; 166 167 const float vp = vt * vc1; 168 const float vy = vp * vs + vs; 169 170 const float vd = vy + vone; 171 float vf = vy / vd; 172 if XNN_UNPREDICTABLE(vz > vdenorm_cutoff) { 173 vf = 0.0f; 174 } 175 if XNN_UNPREDICTABLE(vx > 0.0f) { 176 vf = vone - vf; 177 } 178 179 *y++ = vf; 180 181 n -= sizeof(float); 182 } while (n != 0); 183 } 184} 185