1 // Copyright (c) Facebook, Inc. and its affiliates.
2 // All rights reserved.
3 //
4 // Copyright 2019 Google LLC
5 //
6 // This source code is licensed under the BSD-style license found in the
7 // LICENSE file in the root directory of this source tree.
8
9 #pragma once
10
11 #include <stdbool.h>
12 #include <stddef.h>
13 #include <stdint.h>
14 #include <assert.h>
15
16 #ifdef _MSC_VER
17 #include <intrin.h>
18 #endif
19
20 #include <xnnpack/common.h>
21
22
23 // stdlib.h from Windows 10 SDK defines min & max macros.
24 // Undefine them before defining the corresponding functions.
25 #ifdef min
26 #undef min
27 #endif
28 #ifdef max
29 #undef max
30 #endif
31
32
min(size_t a,size_t b)33 inline static size_t min(size_t a, size_t b) {
34 return XNN_UNPREDICTABLE(b < a) ? b : a;
35 }
36
max(size_t a,size_t b)37 inline static size_t max(size_t a, size_t b) {
38 return XNN_UNPREDICTABLE(b < a) ? a : b;
39 }
40
doz(size_t a,size_t b)41 inline static size_t doz(size_t a, size_t b) {
42 return XNN_UNPREDICTABLE(b < a) ? a - b : 0;
43 }
44
divide_round_up(size_t n,size_t q)45 inline static size_t divide_round_up(size_t n, size_t q) {
46 return XNN_UNPREDICTABLE(n % q == 0) ? n / q : n / q + 1;
47 }
48
round_up(size_t n,size_t q)49 inline static size_t round_up(size_t n, size_t q) {
50 return divide_round_up(n, q) * q;
51 }
52
is_po2(size_t n)53 inline static bool is_po2(size_t n) {
54 return (n != 0) && ((n & (n - 1)) == 0);
55 }
round_down_po2(size_t n,size_t q)56 inline static size_t round_down_po2(size_t n, size_t q) {
57 assert(is_po2(q));
58 return n & -q;
59 }
60
round_up_po2(size_t n,size_t q)61 inline static size_t round_up_po2(size_t n, size_t q) {
62 return round_down_po2(n + q - 1, q);
63 }
64
subtract_modulo(size_t a,size_t b,size_t m)65 inline static size_t subtract_modulo(size_t a, size_t b, size_t m) {
66 assert(a < m);
67 assert(b < m);
68 return XNN_UNPREDICTABLE(a >= b) ? a - b : a - b + m;
69 }
70
math_min_s32(int32_t a,int32_t b)71 inline static int32_t math_min_s32(int32_t a, int32_t b) {
72 return XNN_UNPREDICTABLE(a < b) ? a : b;
73 }
74
math_max_s32(int32_t a,int32_t b)75 inline static int32_t math_max_s32(int32_t a, int32_t b) {
76 return XNN_UNPREDICTABLE(a > b) ? a : b;
77 }
78
math_min_u32(uint32_t a,uint32_t b)79 inline static uint32_t math_min_u32(uint32_t a, uint32_t b) {
80 return XNN_UNPREDICTABLE(a < b) ? a : b;
81 }
82
math_max_u32(uint32_t a,uint32_t b)83 inline static uint32_t math_max_u32(uint32_t a, uint32_t b) {
84 return XNN_UNPREDICTABLE(a > b) ? a : b;
85 }
86
math_muladd_f32(float x,float y,float acc)87 inline static float math_muladd_f32(float x, float y, float acc) {
88 #if defined(__GNUC__) && defined(__FP_FAST_FMAF)
89 return __builtin_fmaf(x, y, acc);
90 #elif defined(__clang__) && defined(__riscv)
91 return __builtin_fmaf(x, y, acc);
92 #else
93 return x * y + acc;
94 #endif
95 }
96
math_min_f32(float a,float b)97 inline static float math_min_f32(float a, float b) {
98 #if defined(__GNUC__) && defined(__ARM_ARCH) && (__ARM_ARCH >= 8)
99 return __builtin_fminf(a, b);
100 #elif defined(__clang__) && defined(__riscv)
101 return __builtin_fminf(a, b);
102 #else
103 return XNN_UNPREDICTABLE(b < a) ? b : a;
104 #endif
105 }
106
math_max_f32(float a,float b)107 inline static float math_max_f32(float a, float b) {
108 #if defined(__GNUC__) && defined(__ARM_ARCH) && (__ARM_ARCH >= 8)
109 return __builtin_fmaxf(a, b);
110 #elif defined(__clang__) && defined(__riscv)
111 return __builtin_fmaxf(a, b);
112 #else
113 return XNN_UNPREDICTABLE(b < a) ? a : b;
114 #endif
115 }
116
math_nonsign_mask_f32()117 inline static float math_nonsign_mask_f32() {
118 #if defined(__INTEL_COMPILER)
119 // Surprisingly, Intel compiler ignores __builtin_nanf payload
120 return _castu32_f32(0x7FFFFFFF);
121 #elif defined(__GNUC__)
122 return __builtin_nanf("0x7FFFFF");
123 #else
124 union {
125 uint32_t as_word;
126 float as_float;
127 } f;
128 f.as_word = 0x7FFFFFFF;
129 return f.as_float;
130 #endif
131 }
132
133
134 #if defined(__clang__)
135 #if __clang_major__ == 3 && __clang_minor__ >= 7 || __clang_major__ > 3
136 #define XNN_IGNORE_SHIFT_BASE_UB __attribute__((__no_sanitize__("shift-base")))
137 #else
138 #define XNN_IGNORE_SHIFT_BASE_UB
139 #endif
140 #elif defined(__GNUC__)
141 #if __GNUC__ >= 8
142 #define XNN_IGNORE_SHIFT_BASE_UB __attribute__((__no_sanitize__("shift-base")))
143 #elif __GNUC__ == 4 && __GNUC_MINOR__ >= 9 || __GNUC__ > 4
144 // 4.9 <= gcc < 8 support ubsan, but doesn't support no_sanitize attribute
145 #define XNN_IGNORE_SHIFT_BASE_UB
146 #ifndef XNN_USE_SHIFT_BASE_UB_WORKAROUND
147 #define XNN_USE_SHIFT_BASE_UB_WORKAROUND 1
148 #endif
149 #else
150 #define XNN_IGNORE_SHIFT_BASE_UB
151 #endif
152 #else
153 #define XNN_IGNORE_SHIFT_BASE_UB
154 #endif
155
156 XNN_IGNORE_SHIFT_BASE_UB
asr_s32(int32_t x,uint32_t n)157 inline static int32_t asr_s32(int32_t x, uint32_t n) {
158 #ifdef XNN_USE_SHIFT_BASE_UB_WORKAROUND
159 #if XNN_ARCH_X86_64 || XNN_ARCH_ARM64
160 return (int32_t) ((uint64_t) (int64_t) x >> n);
161 #else
162 return x >= 0 ? x >> n : ~(~x >> n);
163 #endif
164 #else
165 return x >> n;
166 #endif
167 }
168
169 XNN_IGNORE_SHIFT_BASE_UB
asr_s64(int64_t x,uint32_t n)170 inline static int64_t asr_s64(int64_t x, uint32_t n) {
171 #ifdef XNN_USE_SHIFT_BASE_UB_WORKAROUND
172 return x >= 0 ? x >> n : ~(~x >> n);
173 #else
174 return x >> n;
175 #endif
176 }
177
ctz(uint32_t x)178 inline static uint32_t ctz(uint32_t x) {
179 #ifdef _MSC_VER
180 unsigned long index;
181 _BitScanForward(&index, (unsigned long) x);
182 return (uint32_t) index;
183 #else
184 return __builtin_ctz(x);
185 #endif
186 }
187