src/f32-raddstoreexpminusmax/scalar-p5.c.in

// Copyright 2020 Google LLC
//
// This source code is licensed under the BSD-style license found in the
// LICENSE file in the root directory of this source tree.

$assert ELEMENTS_TILE >= 1
$ABC = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
#include <assert.h>

#include <xnnpack/common.h>
#include <xnnpack/raddstoreexpminusmax.h>

#include <fp16/bitcasts.h>


void xnn_f32_raddstoreexpminusmax_ukernel__scalar_p5_x${ELEMENTS_TILE}${"" if ACCUMULATORS == 1 else "_acc%d" % ACCUMULATORS}(
    size_t elements,
    const float* input,
    float* output,
    float* sum,
    float vi_max)
{
  assert(elements % sizeof(float) == 0);

  const float vmagic_bias = 0x1.8000FEp23f;
  // The smallest x for which expf(x) is normalized.
  const float vdenorm_cutoff = -0x1.5D589Ep6f;
  const float vlog2e = 0x1.715476p+0f;
  // Last 7 bits are zeroes
  const float vminus_ln2_hi = -0x1.62E400p-1f;
  const float vminus_ln2_lo = -0x1.7F7D1Cp-20f;

  const float vc1 = 0x1.FFFFF6p-1f;
  const float vc2 = 0x1.FFFDC6p-2f;
  const float vc3 = 0x1.555A80p-3f;
  const float vc4 = 0x1.573A1Ap-5f;
  const float vc5 = 0x1.0F9F9Cp-7f;

  $if ELEMENTS_TILE > 1:
    $for K in range(ACCUMULATORS):
      float vacc${K} = 0.0f;
    for (; elements >= ${ELEMENTS_TILE} * sizeof(float); elements -= ${ELEMENTS_TILE} * sizeof(float)) {
      // Load ${ELEMENTS_TILE} inputs at a time.
      $for N in range(ELEMENTS_TILE):
        const float vi${N} = input[${N}];
      input += ${ELEMENTS_TILE};

      // Subtract maximum input x := i - i_max. This implies x <= 0.
      $for N in range(ELEMENTS_TILE):
        const float vx${N} = vi${N} - vi_max;

      // Compute reduced argument n := round(x / log(2)).
      // We do it by adding a large number (magic bias) to the product x * (1/log(2)), which cause rounding of the result
      // to an integer, then subtracing the large number back. The trick with adding large number is valid only within
      // certain bounds (|x| <= 2**22), but thats ok, because inputs outside of [-87.336540, 0.0] underflow expf(x)
      // anyway. We fixup the result for such inputs at the very end of the algorithm.
      $for N in range(ELEMENTS_TILE):
        float vn${N} = vx${N} * vlog2e + vmagic_bias;

      // Create a floating-point number s (scale) such that s == 2**n for inputs which don't cause underflow, i.e.
      // -87.33642 <= x <= 0.0, and -126 <= n <= 0 accordingly.
      $for N in range(ELEMENTS_TILE):
        const float vs${N} = fp32_from_bits(fp32_to_bits(vn${N}) << 23);

      // Subtract the large number back to get final n := round(x / log(2)).
      $for N in range(ELEMENTS_TILE):
        vn${N} -= vmagic_bias;

      // Compute reduced argument t := x - n * log(2).
      // Use Cody-Waite range reduction method (note two constants to represent log(2)) to improve accuracy.
      $for N in range(ELEMENTS_TILE):
        float vt${N} = vn${N} * vminus_ln2_hi + vx${N};

      $for N in range(ELEMENTS_TILE):
        vt${N} = vn${N} * vminus_ln2_lo + vt${N};

      // Compute degree-5 polynomial approximation for exp(t) on [-log(2)/2, log(2)/2].
      $for N in range(ELEMENTS_TILE):
        float vp${N} = vc5 * vt${N} + vc4;

      $for N in range(ELEMENTS_TILE):
        vp${N} = vp${N} * vt${N} + vc3;

      $for N in range(ELEMENTS_TILE):
        vp${N} = vp${N} * vt${N} + vc2;

      $for N in range(ELEMENTS_TILE):
        vp${N} = vp${N} * vt${N} + vc1;

      // Reconstruct the final f value:
      //   f = s * (1 + t * (c1 + t * (c2 + t * (c3 + t * (c4 + t * c5)))))
      //     = s + (t * s) * (c1 + t * (c2 + t * (c3 + t * (c4 + t * c5))))
      //     = s + (t * s) * p
      $for N in range(ELEMENTS_TILE):
        vt${N} *= vs${N};

      $for N in range(ELEMENTS_TILE):
        float vf${N} = vt${N} * vp${N} + vs${N};

      // For inputs below denormal cutoff, replace output with +0.0f.
      // Note that for NaN inputs, comparison result is false, and outputs are left unchanged.
      $for N in range(ELEMENTS_TILE):
        if XNN_UNPREDICTABLE(vx${N} < vdenorm_cutoff) {
          vf${N} = 0.0f;
        }

      // Store ${ELEMENTS_TILE} outputs at a time.
      $for N in range(ELEMENTS_TILE):
        output[${N}] = vf${N};
      output += ${ELEMENTS_TILE};

      // Accumulate computed exponents.
      $for N in range(ELEMENTS_TILE):
        vacc${N % ACCUMULATORS} += vf${N};
    }
    $if ACCUMULATORS > 1:
      // Add up all accumulators to vacc0
      $ACC_SLICE = 1
      $while ACC_SLICE < ACCUMULATORS:
        $for A in range(0, ACCUMULATORS, ACC_SLICE * 2):
          $if A + ACC_SLICE < ACCUMULATORS:
            vacc${A} += vacc${A + ACC_SLICE};
        $ACC_SLICE *= 2

    float vacc = vacc0;
  $else:
    float vacc = 0.0f;
  for (; elements >= sizeof(float); elements -= sizeof(float)) {
    // Load 1 input at a time.
    const float vi = *input++;

    // Subtract maximum input x := i - i_max. This implies x <= 0.
    const float vx = vi - vi_max;

    // Compute reduced argument n := round(x / log(2)).
    // We do it by adding a large number (magic bias) to the product x * (1/log(2)), which cause rounding of the result
    // to an integer, then subtracing the large number back. The trick with adding large number is valid only within
    // certain bounds (|x| <= 2**22), but thats ok, because inputs outside of [-87.336540, 0.0] underflow expf(x)
    // anyway. We fixup the result for such inputs at the very end of the algorithm.
    float vn = vx * vlog2e + vmagic_bias;

    // Create a floating-point number s (scale) such that s == 2**n for inputs which don't cause underflow, i.e.
    // -87.33642 <= x <= 0.0, and -126 <= n <= 0 accordingly.
    const float vs = fp32_from_bits(fp32_to_bits(vn) << 23);

    // Subtract the large number back to get final n := round(x / log(2)).
    vn -= vmagic_bias;

    // Compute reduced argument t := x - n * log(2).
    // Use Cody-Waite range reduction method (note two constants to represent log(2)) to improve accuracy.
    float vt = vn * vminus_ln2_hi + vx;
    vt = vn * vminus_ln2_lo + vt;

    // Compute degree-5 polynomial approximation for exp(t) on [-log(2)/2, log(2)/2].
    float vp = vc5 * vt + vc4;
    vp = vp * vt + vc3;
    vp = vp * vt + vc2;
    vp = vp * vt + vc1;

    // Reconstruct the final f value:
    //   f = s * (1 + t * (c1 + t * (c2 + t * (c3 + t * (c4 + t * c5)))))
    //     = s + (t * s) * (c1 + t * (c2 + t * (c3 + t * (c4 + t * c5))))
    //     = s + (t * s) * p
    vt *= vs;
    float vf = vt * vp + vs;

    // For inputs below denormal cutoff, replace output with +0.0f.
    // Note that for NaN inputs, comparison result is false, and outputs are left unchanged.
    if XNN_UNPREDICTABLE(vx < vdenorm_cutoff) {
      vf = 0.0f;
    }

    // Store 1 output at a time.
    *output++ = vf;

    // Accumulate computed exponents.
    vacc += vf;
  }
  *sum = vacc;
}