1 /*
2 * Double-precision vector e^x function.
3 *
4 * Copyright (c) 2019-2022, Arm Limited.
5 * SPDX-License-Identifier: MIT OR Apache-2.0 WITH LLVM-exception
6 */
7
8 #include "mathlib.h"
9 #include "v_math.h"
10 #if V_SUPPORTED
11 #include "v_exp.h"
12
13 #if V_EXP_TABLE_BITS == 7
14 /* maxerr: 1.88 +0.5 ulp
15 rel error: 1.4337*2^-53
16 abs error: 1.4299*2^-53 in [ -ln2/256, ln2/256 ]. */
17 #define C1 v_f64 (0x1.ffffffffffd43p-2)
18 #define C2 v_f64 (0x1.55555c75adbb2p-3)
19 #define C3 v_f64 (0x1.55555da646206p-5)
20 #define InvLn2 v_f64 (0x1.71547652b82fep7) /* N/ln2. */
21 #define Ln2hi v_f64 (0x1.62e42fefa39efp-8) /* ln2/N. */
22 #define Ln2lo v_f64 (0x1.abc9e3b39803f3p-63)
23 #elif V_EXP_TABLE_BITS == 8
24 /* maxerr: 0.54 +0.5 ulp
25 rel error: 1.4318*2^-58
26 abs error: 1.4299*2^-58 in [ -ln2/512, ln2/512 ]. */
27 #define C1 v_f64 (0x1.fffffffffffd4p-2)
28 #define C2 v_f64 (0x1.5555571d6b68cp-3)
29 #define C3 v_f64 (0x1.5555576a59599p-5)
30 #define InvLn2 v_f64 (0x1.71547652b82fep8)
31 #define Ln2hi v_f64 (0x1.62e42fefa39efp-9)
32 #define Ln2lo v_f64 (0x1.abc9e3b39803f3p-64)
33 #endif
34
35 #define N (1 << V_EXP_TABLE_BITS)
36 #define Tab __v_exp_data
37 #define IndexMask v_u64 (N - 1)
38 #define Shift v_f64 (0x1.8p+52)
39
40 #if WANT_SIMD_EXCEPT
41
42 #define TinyBound 0x200 /* top12 (asuint64 (0x1p-511)). */
43 #define BigBound 0x408 /* top12 (asuint64 (0x1p9)). */
44
45 VPCS_ATTR static NOINLINE v_f64_t
specialcase(v_f64_t x,v_f64_t y,v_u64_t cmp)46 specialcase (v_f64_t x, v_f64_t y, v_u64_t cmp)
47 {
48 /* If fenv exceptions are to be triggered correctly, fall back to the scalar
49 routine to special lanes. */
50 return v_call_f64 (exp, x, y, cmp);
51 }
52
53 #else
54
55 #define Thres v_f64 (704.0)
56
57 VPCS_ATTR
58 static v_f64_t
specialcase(v_f64_t s,v_f64_t y,v_f64_t n)59 specialcase (v_f64_t s, v_f64_t y, v_f64_t n)
60 {
61 v_f64_t absn = v_abs_f64 (n);
62
63 /* 2^(n/N) may overflow, break it up into s1*s2. */
64 v_u64_t b = v_cond_u64 (n <= v_f64 (0.0)) & v_u64 (0x6000000000000000);
65 v_f64_t s1 = v_as_f64_u64 (v_u64 (0x7000000000000000) - b);
66 v_f64_t s2 = v_as_f64_u64 (v_as_u64_f64 (s) - v_u64 (0x3010000000000000) + b);
67 v_u64_t cmp = v_cond_u64 (absn > v_f64 (1280.0 * N));
68 v_f64_t r1 = s1 * s1;
69 v_f64_t r0 = v_fma_f64 (y, s2, s2) * s1;
70 return v_as_f64_u64 ((cmp & v_as_u64_f64 (r1)) | (~cmp & v_as_u64_f64 (r0)));
71 }
72
73 #endif
74
75 VPCS_ATTR
76 v_f64_t
V_NAME(exp)77 V_NAME(exp) (v_f64_t x)
78 {
79 v_f64_t n, r, r2, s, y, z;
80 v_u64_t cmp, u, e, i;
81
82 #if WANT_SIMD_EXCEPT
83 /* If any lanes are special, mask them with 1 and retain a copy of x to allow
84 specialcase to fix special lanes later. This is only necessary if fenv
85 exceptions are to be triggered correctly. */
86 v_f64_t xm = x;
87 cmp = v_cond_u64 ((v_as_u64_f64 (v_abs_f64 (x)) >> 52) - TinyBound
88 >= BigBound - TinyBound);
89 if (unlikely (v_any_u64 (cmp)))
90 x = v_sel_f64 (cmp, v_f64 (1), x);
91 #else
92 cmp = v_cond_u64 (v_abs_f64 (x) > Thres);
93 #endif
94
95 /* n = round(x/(ln2/N)). */
96 z = v_fma_f64 (x, InvLn2, Shift);
97 u = v_as_u64_f64 (z);
98 n = z - Shift;
99
100 /* r = x - n*ln2/N. */
101 r = x;
102 r = v_fma_f64 (-Ln2hi, n, r);
103 r = v_fma_f64 (-Ln2lo, n, r);
104
105 e = u << (52 - V_EXP_TABLE_BITS);
106 i = u & IndexMask;
107
108 /* y = exp(r) - 1 ~= r + C1 r^2 + C2 r^3 + C3 r^4. */
109 r2 = r * r;
110 y = v_fma_f64 (C2, r, C1);
111 y = v_fma_f64 (C3, r2, y);
112 y = v_fma_f64 (y, r2, r);
113
114 /* s = 2^(n/N). */
115 u = v_lookup_u64 (Tab, i);
116 s = v_as_f64_u64 (u + e);
117
118 if (unlikely (v_any_u64 (cmp)))
119 #if WANT_SIMD_EXCEPT
120 return specialcase (xm, v_fma_f64 (y, s, s), cmp);
121 #else
122 return specialcase (s, y, n);
123 #endif
124
125 return v_fma_f64 (y, s, s);
126 }
127 VPCS_ALIAS
128 #endif
129