src/f32-vsigmoid/scalar-rr2-lut64-p2-div.c.in

// Copyright 2019 Google LLC
//
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree.

$assert BATCH_TILE >= 1
$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
#include <assert.h>
#include <math.h>

#include <xnnpack/common.h>
#include <xnnpack/math.h>
#include <xnnpack/vunary.h>


// Note redefine as uint32[] to avoid redundant bitcasts.
extern XNN_INTERNAL const uint32_t xnn_table_exp2minus_k_over_64[64];

void xnn_f32_vsigmoid_ukernel__scalar_rr2_lut64_p2_div_x${BATCH_TILE}(
    size_t n,
    const float* x,
    float* y,
    const union xnn_f32_sigmoid_params params[restrict XNN_MIN_ELEMENTS(1)])
{
  assert(n % sizeof(float) == 0);

  const float vmagic_bias = params->scalar_rr2_lut64_p2.magic_bias;
  const float vminus_log2e = params->scalar_rr2_lut64_p2.minus_log2e;
  const uint32_t vindex_mask = UINT32_C(0x3F);
  const float vln2_hi = params->scalar_rr2_lut64_p2.ln2_hi;
  const float vln2_lo = params->scalar_rr2_lut64_p2.ln2_lo;
  const float vc2 = params->scalar_rr2_lut64_p2.c2;
  const float vone = params->scalar_rr2_lut64_p2.one;
  const float vdenorm_cutoff = params->scalar_rr2_lut64_p2.denorm_cutoff;

  $if BATCH_TILE > 1:
    for (; n >= ${BATCH_TILE} * sizeof(float); n -= ${BATCH_TILE} * sizeof(float)) {
      $for N in range(BATCH_TILE):
        const float vx${N} = x[${N}];
      x += ${BATCH_TILE};

      $for N in range(BATCH_TILE):
        const float vz${N} = fabsf(vx${N});

      $for N in range(BATCH_TILE):
        float vn${N} = vz${N} * vminus_log2e + vmagic_bias;

      $for N in range(BATCH_TILE):
        const uint32_t ve${N} = float_as_uint32(vn${N}) << 17;

      $for N in range(BATCH_TILE):
        const uint32_t vidx${N} = float_as_uint32(vn${N}) & vindex_mask;
        const float vs${N} = uint32_as_float(xnn_table_exp2minus_k_over_64[vidx${N}] + ve${N});

      $for N in range(BATCH_TILE):
        vn${N} -= vmagic_bias;

      $for N in range(BATCH_TILE):
        float vt${N} = vn${N} * vln2_hi + vz${N};

      $for N in range(BATCH_TILE):
        vt${N} = vn${N} * vln2_lo + vt${N};

      $for N in range(BATCH_TILE):
        float vp${N} = vt${N} * vc2;

      $for N in range(BATCH_TILE):
        vp${N} = vt${N} - vp${N} * vt${N};

      $for N in range(BATCH_TILE):
        const float vy${N} = vs${N} - vs${N} * vp${N};

      $for N in range(BATCH_TILE):
        const float vd${N} = vy${N} + vone;

      $for N in range(BATCH_TILE):
        float vf${N} = vy${N} / vd${N};

      $for N in range(BATCH_TILE):
        if XNN_UNPREDICTABLE(vz${N} > vdenorm_cutoff) {
          vf${N} = 0.0f;
        }

      $for N in range(BATCH_TILE):
        if XNN_UNPREDICTABLE(vx${N} > 0.0f) {
          vf${N} = vone - vf${N};
        }

      $for N in range(BATCH_TILE):
        y[${N}] = vf${N};
      y += ${BATCH_TILE};
    }
  $if BATCH_TILE == 1:
    do {
      const float vx = *x++;

      const float vz = fabsf(vx);

      float vn = vz * vminus_log2e + vmagic_bias;
      const uint32_t ve = float_as_uint32(vn) << 17;
      const uint32_t vidx = float_as_uint32(vn) & vindex_mask;
      const float vs = uint32_as_float(xnn_table_exp2minus_k_over_64[vidx] + ve);
      vn -= vmagic_bias;

      float vt = vn * vln2_hi + vz;
      vt = vn * vln2_lo + vt;

      float vp = vt * vc2;
      vp = vt - vp * vt;

      const float vy = vs - vs * vp;
      const float vd = vy + vone;

      float vf = vy / vd;
      if XNN_UNPREDICTABLE(vz > vdenorm_cutoff) {
        vf = 0.0f;
      }
      if XNN_UNPREDICTABLE(vx > 0.0f) {
        vf = vone - vf;
      }

      *y++ = vf;

      n -= sizeof(float);
    } while (n != 0);
  $elif BATCH_TILE == 2:
    if XNN_UNLIKELY(n != 0) {
      const float vx = *x;

      const float vz = fabsf(vx);

      float vn = vz * vminus_log2e + vmagic_bias;
      const uint32_t ve = float_as_uint32(vn) << 17;
      const uint32_t vidx = float_as_uint32(vn) & vindex_mask;
      const float vs = uint32_as_float(xnn_table_exp2minus_k_over_64[vidx] + ve);
      vn -= vmagic_bias;

      float vt = vn * vln2_hi + vz;
      vt = vn * vln2_lo + vt;

      float vp = vt * vc2;
      vp = vt - vp * vt;

      const float vy = vs - vs * vp;
      const float vd = vy + vone;

      float vf = vy / vd;
      if XNN_UNPREDICTABLE(vz > vdenorm_cutoff) {
        vf = 0.0f;
      }
      if XNN_UNPREDICTABLE(vx > 0.0f) {
        vf = vone - vf;
      }

      *y = vf;
    }
  $else:
    if XNN_UNLIKELY(n != 0) {
      do {
        const float vx = *x++;

        const float vz = fabsf(vx);

        float vn = vz * vminus_log2e + vmagic_bias;
        const uint32_t ve = float_as_uint32(vn) << 17;
        const uint32_t vidx = float_as_uint32(vn) & vindex_mask;
        const float vs = uint32_as_float(xnn_table_exp2minus_k_over_64[vidx] + ve);
        vn -= vmagic_bias;

        float vt = vn * vln2_hi + vz;
        vt = vn * vln2_lo + vt;

        float vp = vt * vc2;
        vp = vt - vp * vt;

        const float vy = vs - vs * vp;
        const float vd = vy + vone;

        float vf = vy / vd;
        if XNN_UNPREDICTABLE(vz > vdenorm_cutoff) {
          vf = 0.0f;
        }
        if XNN_UNPREDICTABLE(vx > 0.0f) {
          vf = vone - vf;
        }

        *y++ = vf;

        n -= sizeof(float);
      } while (n != 0);
    }
}