• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1// Copyright 2020 Google LLC
2//
3// This source code is licensed under the BSD-style license found in the
4// LICENSE file in the root directory of this source tree.
5
6$assert BATCH_TILE >= 1
7#include <assert.h>
8#include <math.h>
9
10#include <xnnpack/common.h>
11#include <xnnpack/vunary.h>
12
13#include <fp16/bitcasts.h>
14
15
16void xnn_f32_velu_ukernel__${"wasm" if WASM else "scalar"}_rr2_p6_x${BATCH_TILE}(
17    size_t n,
18    const float* x,
19    float* y,
20    const union xnn_f32_elu_params params[restrict XNN_MIN_ELEMENTS(1)])
21{
22  assert(n % sizeof(float) == 0);
23
24  const float vprescale = params->scalar_rr2_p6.prescale;
25  const float valpha = params->scalar_rr2_p6.alpha;
26  const float vbeta = params->scalar_rr2_p6.beta;
27  const float vmagic_bias = params->scalar_rr2_p6.magic_bias;
28  const float vlog2e = params->scalar_rr2_p6.log2e;
29  const float vsat_cutoff = params->scalar_rr2_p6.sat_cutoff;
30  const float vminus_ln2_hi = params->scalar_rr2_p6.minus_ln2_hi;
31  const float vminus_ln2_lo = params->scalar_rr2_p6.minus_ln2_lo;
32  const float vc6 = params->scalar_rr2_p6.c6;
33  const float vc5 = params->scalar_rr2_p6.c5;
34  const float vc4 = params->scalar_rr2_p6.c4;
35  const float vc3 = params->scalar_rr2_p6.c3;
36  const float vc2 = params->scalar_rr2_p6.c2;
37  const float vone = params->scalar_rr2_p6.one;
38
39  $if BATCH_TILE > 1:
40    for (; n >= ${BATCH_TILE} * sizeof(float); n -= ${BATCH_TILE} * sizeof(float)) {
41      $for N in range(BATCH_TILE):
42        float vx${N} = x[${N}];
43      x += ${BATCH_TILE};
44
45      $for N in range(BATCH_TILE):
46        $if WASM:
47          const float vz${N} = __builtin_wasm_min_f32(__builtin_wasm_max_f32(vx${N} * vprescale, vsat_cutoff), 0.0f);
48        $else:
49          const float vz${N} = vx${N} * vprescale;
50
51      $for N in range(BATCH_TILE):
52        float vn${N} = vz${N} * vlog2e + vmagic_bias;
53
54      $for N in range(BATCH_TILE):
55        float vs${N} = fp32_from_bits(fp32_to_bits(vn${N}) << 23);
56        vn${N} -= vmagic_bias;
57
58      $for N in range(BATCH_TILE):
59        float vt${N} = vn${N} * vminus_ln2_hi + vz${N};
60
61      $for N in range(BATCH_TILE):
62        vt${N} = vn${N} * vminus_ln2_lo + vt${N};
63
64      $if not WASM:
65        $for N in range(BATCH_TILE):
66          if XNN_UNPREDICTABLE(vz${N} <= vsat_cutoff) {
67            vs${N} = 0.0f;
68            vt${N} = 0.0f;
69          }
70
71      $for N in range(BATCH_TILE):
72        float vp${N} = vc6 * vt${N} + vc5;
73
74      $for N in range(BATCH_TILE):
75        vp${N} = vp${N} * vt${N} + vc4;
76
77      $for N in range(BATCH_TILE):
78        vp${N} = vp${N} * vt${N} + vc3;
79
80      $for N in range(BATCH_TILE):
81        vp${N} = vp${N} * vt${N} + vc2;
82
83      $for N in range(BATCH_TILE):
84        vp${N} *= vt${N};
85
86      $for N in range(BATCH_TILE):
87        vt${N} *= vs${N};
88        vs${N} -= vone;
89
90      $for N in range(BATCH_TILE):
91        vp${N} = vp${N} * vt${N} + vt${N};
92
93      $for N in range(BATCH_TILE):
94        const float ve${N} = (vp${N} + vs${N}) * valpha;
95        $if WASM:
96          float vy${N} = __builtin_wasm_max_f32(vx${N} * vbeta, 0.0f);
97        $else:
98          float vy${N} = vx${N} * vbeta;
99
100      $if WASM:
101        $for N in range(BATCH_TILE):
102          vy${N} += __builtin_wasm_min_f32(ve${N}, 0.0f);
103      $else:
104        $for N in range(BATCH_TILE):
105          if XNN_UNPREDICTABLE(vx${N} < 0.0f) {
106            vy${N} = ve${N};
107          }
108
109      $for N in range(BATCH_TILE):
110        y[${N}] = vy${N};
111      y += ${BATCH_TILE};
112    }
113  $if BATCH_TILE == 1:
114    do {
115      float vx = *x++;
116
117      $if WASM:
118        const float vz = __builtin_wasm_min_f32(__builtin_wasm_max_f32(vx * vprescale, vsat_cutoff), 0.0f);
119      $else:
120        const float vz = vx * vprescale;
121
122      float vn = vz * vlog2e + vmagic_bias;
123      float vs = fp32_from_bits(fp32_to_bits(vn) << 23);
124      vn -= vmagic_bias;
125
126      float vt = vn * vminus_ln2_hi + vz;
127      vt = vn * vminus_ln2_lo + vt;
128
129      $if not WASM:
130        if XNN_UNPREDICTABLE(vz <= vsat_cutoff) {
131          vs = 0.0f;
132          vt = 0.0f;
133        }
134
135      float vp = vc6 * vt + vc5;
136      vp = vp * vt + vc4;
137      vp = vp * vt + vc3;
138      vp = vp * vt + vc2;
139      vp *= vt;
140
141      vt *= vs;
142      vs -= vone;
143      vp = vp * vt + vt;
144      const float ve = (vp + vs) * valpha;
145
146      $if WASM:
147        float vy = __builtin_wasm_max_f32(vx * vbeta, 0.0f);
148        vy += __builtin_wasm_min_f32(ve, 0.0f);
149      $else:
150        float vy = vx * vbeta;
151        if XNN_UNPREDICTABLE(vx < 0.0f) {
152          vy = ve;
153        }
154
155      *y++ = vy;
156
157      n -= sizeof(float);
158    } while (n != 0);
159  $elif BATCH_TILE == 2:
160    if XNN_UNLIKELY(n != 0) {
161      float vx = *x;
162
163      $if WASM:
164        const float vz = __builtin_wasm_min_f32(__builtin_wasm_max_f32(vx * vprescale, vsat_cutoff), 0.0f);
165      $else:
166        const float vz = vx * vprescale;
167
168      float vn = vz * vlog2e + vmagic_bias;
169      float vs = fp32_from_bits(fp32_to_bits(vn) << 23);
170      vn -= vmagic_bias;
171
172      float vt = vn * vminus_ln2_hi + vz;
173      vt = vn * vminus_ln2_lo + vt;
174
175      $if not WASM:
176        if XNN_UNPREDICTABLE(vz <= vsat_cutoff) {
177          vs = 0.0f;
178          vt = 0.0f;
179        }
180
181      float vp = vc6 * vt + vc5;
182      vp = vp * vt + vc4;
183      vp = vp * vt + vc3;
184      vp = vp * vt + vc2;
185      vp *= vt;
186
187      vt *= vs;
188      vs -= vone;
189      vp = vp * vt + vt;
190      const float ve = (vp + vs) * valpha;
191
192      $if WASM:
193        float vy = __builtin_wasm_max_f32(vx * vbeta, 0.0f);
194        vy += __builtin_wasm_min_f32(ve, 0.0f);
195      $else:
196        float vy = vx * vbeta;
197        if XNN_UNPREDICTABLE(vx < 0.0f) {
198          vy = ve;
199        }
200
201      *y = vy;
202    }
203  $else:
204    if XNN_UNLIKELY(n != 0) {
205      do {
206        float vx = *x++;
207
208        $if WASM:
209          const float vz = __builtin_wasm_min_f32(__builtin_wasm_max_f32(vx * vprescale, vsat_cutoff), 0.0f);
210        $else:
211          const float vz = vx * vprescale;
212
213        float vn = vz * vlog2e + vmagic_bias;
214        float vs = fp32_from_bits(fp32_to_bits(vn) << 23);
215        vn -= vmagic_bias;
216
217        float vt = vn * vminus_ln2_hi + vz;
218        vt = vn * vminus_ln2_lo + vt;
219
220        $if not WASM:
221          if XNN_UNPREDICTABLE(vz <= vsat_cutoff) {
222            vs = 0.0f;
223            vt = 0.0f;
224          }
225
226        float vp = vc6 * vt + vc5;
227        vp = vp * vt + vc4;
228        vp = vp * vt + vc3;
229        vp = vp * vt + vc2;
230        vp *= vt;
231
232        vt *= vs;
233        vs -= vone;
234        vp = vp * vt + vt;
235        const float ve = (vp + vs) * valpha;
236
237        $if WASM:
238          float vy = __builtin_wasm_max_f32(vx * vbeta, 0.0f);
239          vy += __builtin_wasm_min_f32(ve, 0.0f);
240        $else:
241          float vy = vx * vbeta;
242          if XNN_UNPREDICTABLE(vx < 0.0f) {
243            vy = ve;
244          }
245
246        *y++ = vy;
247
248        n -= sizeof(float);
249      } while (n != 0);
250    }
251}
252