1 /* 2 * Copyright 2019 Google Inc. 3 * 4 * Use of this source code is governed by a BSD-style license that can be 5 * found in the LICENSE file. 6 */ 7 8 #ifndef SKVX_DEFINED 9 #define SKVX_DEFINED 10 11 // skvx::Vec<N,T> are SIMD vectors of N T's, a v1.5 successor to SkNx<N,T>. 12 // 13 // This time we're leaning a bit less on platform-specific intrinsics and a bit 14 // more on Clang/GCC vector extensions, but still keeping the option open to 15 // drop in platform-specific intrinsics, actually more easily than before. 16 // 17 // We've also fixed a few of the caveats that used to make SkNx awkward to work 18 // with across translation units. skvx::Vec<N,T> always has N*sizeof(T) size 19 // and alignment[1][2] and is safe to use across translation units freely. 20 // 21 // [1] Ideally we'd only align to T, but that tanks ARMv7 NEON codegen. 22 // [2] Some compilers barf if we try to use N*sizeof(T), so instead we leave them at T. 23 24 // Please try to keep this file independent of Skia headers. 25 #include <algorithm> // std::min, std::max 26 #include <cmath> // std::ceil, std::floor, std::trunc, std::round, std::sqrt, etc. 27 #include <cstdint> // intXX_t 28 #include <cstring> // memcpy() 29 #include <initializer_list> // std::initializer_list 30 31 #if defined(__SSE__) 32 #include <immintrin.h> 33 #elif defined(__ARM_NEON) 34 #include <arm_neon.h> 35 #endif 36 37 #if !defined(__clang__) && defined(__GNUC__) && defined(__mips64) 38 // GCC 7 hits an internal compiler error when targeting MIPS64. 39 #define SKVX_ALIGNMENT 40 #elif !defined(__clang__) && defined(_MSC_VER) && defined(_M_IX86) 41 // Our SkVx unit tests fail when built by MSVC for 32-bit x86. 42 #define SKVX_ALIGNMENT 43 #else 44 #define SKVX_ALIGNMENT alignas(N * sizeof(T)) 45 #endif 46 47 #if defined(__GNUC__) && !defined(__clang__) && defined(__SSE__) 48 // GCC warns about ABI changes when returning >= 32 byte vectors when -mavx is not enabled. 49 // This only happens for types like VExt whose ABI we don't care about, not for Vec itself. 50 #pragma GCC diagnostic ignored "-Wpsabi" 51 #endif 52 53 // To avoid ODR violations, all methods must be force-inlined, 54 // and all standalone functions must be static, perhaps using these helpers. 55 #if defined(_MSC_VER) 56 #define SKVX_ALWAYS_INLINE __forceinline 57 #else 58 #define SKVX_ALWAYS_INLINE __attribute__((always_inline)) 59 #endif 60 61 #define SIT template < typename T> static inline 62 #define SINT template <int N, typename T> static inline 63 #define SINTU template <int N, typename T, typename U, \ 64 typename=typename std::enable_if<std::is_convertible<U,T>::value>::type> \ 65 static inline 66 67 namespace skvx { 68 69 // All Vec have the same simple memory layout, the same as `T vec[N]`. 70 template <int N, typename T> 71 struct SKVX_ALIGNMENT Vec { 72 static_assert((N & (N-1)) == 0, "N must be a power of 2."); 73 static_assert(sizeof(T) >= alignof(T), "What kind of crazy T is this?"); 74 75 Vec<N/2,T> lo, hi; 76 77 // Methods belong here in the class declaration of Vec only if: 78 // - they must be here, like constructors or operator[]; 79 // - they'll definitely never want a specialized implementation. 80 // Other operations on Vec should be defined outside the type. 81 82 SKVX_ALWAYS_INLINE Vec() = default; 83 84 template <typename U, 85 typename=typename std::enable_if<std::is_convertible<U,T>::value>::type> 86 SKVX_ALWAYS_INLINE VecVec87 Vec(U x) : lo(x), hi(x) {} 88 VecVec89 SKVX_ALWAYS_INLINE Vec(std::initializer_list<T> xs) { 90 T vals[N] = {0}; 91 memcpy(vals, xs.begin(), std::min(xs.size(), (size_t)N)*sizeof(T)); 92 93 lo = Vec<N/2,T>::Load(vals + 0); 94 hi = Vec<N/2,T>::Load(vals + N/2); 95 } 96 97 SKVX_ALWAYS_INLINE T operator[](int i) const { return i < N/2 ? lo[i] : hi[i-N/2]; } 98 SKVX_ALWAYS_INLINE T& operator[](int i) { return i < N/2 ? lo[i] : hi[i-N/2]; } 99 LoadVec100 SKVX_ALWAYS_INLINE static Vec Load(const void* ptr) { 101 Vec v; 102 memcpy(&v, ptr, sizeof(Vec)); 103 return v; 104 } storeVec105 SKVX_ALWAYS_INLINE void store(void* ptr) const { 106 memcpy(ptr, this, sizeof(Vec)); 107 } 108 }; 109 110 template <typename T> 111 struct Vec<1,T> { 112 T val; 113 114 SKVX_ALWAYS_INLINE Vec() = default; 115 116 template <typename U, 117 typename=typename std::enable_if<std::is_convertible<U,T>::value>::type> 118 SKVX_ALWAYS_INLINE 119 Vec(U x) : val(x) {} 120 121 SKVX_ALWAYS_INLINE Vec(std::initializer_list<T> xs) : val(xs.size() ? *xs.begin() : 0) {} 122 123 SKVX_ALWAYS_INLINE T operator[](int) const { return val; } 124 SKVX_ALWAYS_INLINE T& operator[](int) { return val; } 125 126 SKVX_ALWAYS_INLINE static Vec Load(const void* ptr) { 127 Vec v; 128 memcpy(&v, ptr, sizeof(Vec)); 129 return v; 130 } 131 SKVX_ALWAYS_INLINE void store(void* ptr) const { 132 memcpy(ptr, this, sizeof(Vec)); 133 } 134 }; 135 136 template <typename D, typename S> 137 static inline D bit_pun(const S& s) { 138 static_assert(sizeof(D) == sizeof(S), ""); 139 D d; 140 memcpy(&d, &s, sizeof(D)); 141 return d; 142 } 143 144 // Translate from a value type T to its corresponding Mask, the result of a comparison. 145 template <typename T> struct Mask { using type = T; }; 146 template <> struct Mask<float > { using type = int32_t; }; 147 template <> struct Mask<double> { using type = int64_t; }; 148 template <typename T> using M = typename Mask<T>::type; 149 150 // Join two Vec<N,T> into one Vec<2N,T>. 151 SINT Vec<2*N,T> join(const Vec<N,T>& lo, const Vec<N,T>& hi) { 152 Vec<2*N,T> v; 153 v.lo = lo; 154 v.hi = hi; 155 return v; 156 } 157 158 // We have two default strategies for implementing most operations: 159 // 1) lean on Clang/GCC vector extensions when available; 160 // 2) recurse to scalar portable implementations when not. 161 // At the end we can drop in platform-specific implementations that override either default. 162 163 #if !defined(SKNX_NO_SIMD) && (defined(__clang__) || defined(__GNUC__)) 164 165 // VExt<N,T> types have the same size as Vec<N,T> and support most operations directly. 166 // N.B. VExt<N,T> alignment is N*alignof(T), stricter than Vec<N,T>'s alignof(T). 167 #if defined(__clang__) 168 template <int N, typename T> 169 using VExt = T __attribute__((ext_vector_type(N))); 170 171 #elif defined(__GNUC__) 172 template <int N, typename T> 173 struct VExtHelper { 174 typedef T __attribute__((vector_size(N*sizeof(T)))) type; 175 }; 176 177 template <int N, typename T> 178 using VExt = typename VExtHelper<N,T>::type; 179 180 // For some reason some (new!) versions of GCC cannot seem to deduce N in the generic 181 // to_vec<N,T>() below for N=4 and T=float. This workaround seems to help... 182 static inline Vec<4,float> to_vec(VExt<4,float> v) { return bit_pun<Vec<4,float>>(v); } 183 #endif 184 185 SINT VExt<N,T> to_vext(const Vec<N,T>& v) { return bit_pun<VExt<N,T>>(v); } 186 SINT Vec <N,T> to_vec(const VExt<N,T>& v) { return bit_pun<Vec <N,T>>(v); } 187 188 SINT Vec<N,T> operator+(const Vec<N,T>& x, const Vec<N,T>& y) { return to_vec<N,T>(to_vext(x) + to_vext(y)); } 189 SINT Vec<N,T> operator-(const Vec<N,T>& x, const Vec<N,T>& y) { return to_vec<N,T>(to_vext(x) - to_vext(y)); } 190 SINT Vec<N,T> operator*(const Vec<N,T>& x, const Vec<N,T>& y) { return to_vec<N,T>(to_vext(x) * to_vext(y)); } 191 SINT Vec<N,T> operator/(const Vec<N,T>& x, const Vec<N,T>& y) { return to_vec<N,T>(to_vext(x) / to_vext(y)); } 192 193 SINT Vec<N,T> operator^(const Vec<N,T>& x, const Vec<N,T>& y) { return to_vec<N,T>(to_vext(x) ^ to_vext(y)); } 194 SINT Vec<N,T> operator&(const Vec<N,T>& x, const Vec<N,T>& y) { return to_vec<N,T>(to_vext(x) & to_vext(y)); } 195 SINT Vec<N,T> operator|(const Vec<N,T>& x, const Vec<N,T>& y) { return to_vec<N,T>(to_vext(x) | to_vext(y)); } 196 197 SINT Vec<N,T> operator!(const Vec<N,T>& x) { return to_vec<N,T>(!to_vext(x)); } 198 SINT Vec<N,T> operator-(const Vec<N,T>& x) { return to_vec<N,T>(-to_vext(x)); } 199 SINT Vec<N,T> operator~(const Vec<N,T>& x) { return to_vec<N,T>(~to_vext(x)); } 200 201 SINT Vec<N,T> operator<<(const Vec<N,T>& x, int bits) { return to_vec<N,T>(to_vext(x) << bits); } 202 SINT Vec<N,T> operator>>(const Vec<N,T>& x, int bits) { return to_vec<N,T>(to_vext(x) >> bits); } 203 204 SINT Vec<N,M<T>> operator==(const Vec<N,T>& x, const Vec<N,T>& y) { return bit_pun<Vec<N,M<T>>>(to_vext(x) == to_vext(y)); } 205 SINT Vec<N,M<T>> operator!=(const Vec<N,T>& x, const Vec<N,T>& y) { return bit_pun<Vec<N,M<T>>>(to_vext(x) != to_vext(y)); } 206 SINT Vec<N,M<T>> operator<=(const Vec<N,T>& x, const Vec<N,T>& y) { return bit_pun<Vec<N,M<T>>>(to_vext(x) <= to_vext(y)); } 207 SINT Vec<N,M<T>> operator>=(const Vec<N,T>& x, const Vec<N,T>& y) { return bit_pun<Vec<N,M<T>>>(to_vext(x) >= to_vext(y)); } 208 SINT Vec<N,M<T>> operator< (const Vec<N,T>& x, const Vec<N,T>& y) { return bit_pun<Vec<N,M<T>>>(to_vext(x) < to_vext(y)); } 209 SINT Vec<N,M<T>> operator> (const Vec<N,T>& x, const Vec<N,T>& y) { return bit_pun<Vec<N,M<T>>>(to_vext(x) > to_vext(y)); } 210 211 #else 212 213 // Either SKNX_NO_SIMD is defined, or Clang/GCC vector extensions are not available. 214 // We'll implement things portably, in a way that should be easily autovectorizable. 215 216 // N == 1 scalar implementations. 217 SIT Vec<1,T> operator+(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val + y.val; } 218 SIT Vec<1,T> operator-(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val - y.val; } 219 SIT Vec<1,T> operator*(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val * y.val; } 220 SIT Vec<1,T> operator/(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val / y.val; } 221 222 SIT Vec<1,T> operator^(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val ^ y.val; } 223 SIT Vec<1,T> operator&(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val & y.val; } 224 SIT Vec<1,T> operator|(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val | y.val; } 225 226 SIT Vec<1,T> operator!(const Vec<1,T>& x) { return !x.val; } 227 SIT Vec<1,T> operator-(const Vec<1,T>& x) { return -x.val; } 228 SIT Vec<1,T> operator~(const Vec<1,T>& x) { return ~x.val; } 229 230 SIT Vec<1,T> operator<<(const Vec<1,T>& x, int bits) { return x.val << bits; } 231 SIT Vec<1,T> operator>>(const Vec<1,T>& x, int bits) { return x.val >> bits; } 232 233 SIT Vec<1,M<T>> operator==(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val == y.val ? ~0 : 0; } 234 SIT Vec<1,M<T>> operator!=(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val != y.val ? ~0 : 0; } 235 SIT Vec<1,M<T>> operator<=(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val <= y.val ? ~0 : 0; } 236 SIT Vec<1,M<T>> operator>=(const Vec<1,T>& x, const Vec<1,T>& y) { return x.val >= y.val ? ~0 : 0; } 237 SIT Vec<1,M<T>> operator< (const Vec<1,T>& x, const Vec<1,T>& y) { return x.val < y.val ? ~0 : 0; } 238 SIT Vec<1,M<T>> operator> (const Vec<1,T>& x, const Vec<1,T>& y) { return x.val > y.val ? ~0 : 0; } 239 240 // All default N != 1 implementations just recurse on lo and hi halves. 241 SINT Vec<N,T> operator+(const Vec<N,T>& x, const Vec<N,T>& y) { return join(x.lo + y.lo, x.hi + y.hi); } 242 SINT Vec<N,T> operator-(const Vec<N,T>& x, const Vec<N,T>& y) { return join(x.lo - y.lo, x.hi - y.hi); } 243 SINT Vec<N,T> operator*(const Vec<N,T>& x, const Vec<N,T>& y) { return join(x.lo * y.lo, x.hi * y.hi); } 244 SINT Vec<N,T> operator/(const Vec<N,T>& x, const Vec<N,T>& y) { return join(x.lo / y.lo, x.hi / y.hi); } 245 246 SINT Vec<N,T> operator^(const Vec<N,T>& x, const Vec<N,T>& y) { return join(x.lo ^ y.lo, x.hi ^ y.hi); } 247 SINT Vec<N,T> operator&(const Vec<N,T>& x, const Vec<N,T>& y) { return join(x.lo & y.lo, x.hi & y.hi); } 248 SINT Vec<N,T> operator|(const Vec<N,T>& x, const Vec<N,T>& y) { return join(x.lo | y.lo, x.hi | y.hi); } 249 250 SINT Vec<N,T> operator!(const Vec<N,T>& x) { return join(!x.lo, !x.hi); } 251 SINT Vec<N,T> operator-(const Vec<N,T>& x) { return join(-x.lo, -x.hi); } 252 SINT Vec<N,T> operator~(const Vec<N,T>& x) { return join(~x.lo, ~x.hi); } 253 254 SINT Vec<N,T> operator<<(const Vec<N,T>& x, int bits) { return join(x.lo << bits, x.hi << bits); } 255 SINT Vec<N,T> operator>>(const Vec<N,T>& x, int bits) { return join(x.lo >> bits, x.hi >> bits); } 256 257 SINT Vec<N,M<T>> operator==(const Vec<N,T>& x, const Vec<N,T>& y) { return join(x.lo == y.lo, x.hi == y.hi); } 258 SINT Vec<N,M<T>> operator!=(const Vec<N,T>& x, const Vec<N,T>& y) { return join(x.lo != y.lo, x.hi != y.hi); } 259 SINT Vec<N,M<T>> operator<=(const Vec<N,T>& x, const Vec<N,T>& y) { return join(x.lo <= y.lo, x.hi <= y.hi); } 260 SINT Vec<N,M<T>> operator>=(const Vec<N,T>& x, const Vec<N,T>& y) { return join(x.lo >= y.lo, x.hi >= y.hi); } 261 SINT Vec<N,M<T>> operator< (const Vec<N,T>& x, const Vec<N,T>& y) { return join(x.lo < y.lo, x.hi < y.hi); } 262 SINT Vec<N,M<T>> operator> (const Vec<N,T>& x, const Vec<N,T>& y) { return join(x.lo > y.lo, x.hi > y.hi); } 263 #endif 264 265 // Some operations we want are not expressible with Clang/GCC vector 266 // extensions, so we implement them using the recursive approach. 267 268 // N == 1 scalar implementations. 269 SIT Vec<1,T> if_then_else(const Vec<1,M<T>>& cond, const Vec<1,T>& t, const Vec<1,T>& e) { 270 auto t_bits = bit_pun<M<T>>(t), 271 e_bits = bit_pun<M<T>>(e); 272 return bit_pun<T>( (cond.val & t_bits) | (~cond.val & e_bits) ); 273 } 274 275 SIT bool any(const Vec<1,T>& x) { return x.val != 0; } 276 SIT bool all(const Vec<1,T>& x) { return x.val != 0; } 277 278 SIT T min(const Vec<1,T>& x) { return x.val; } 279 SIT T max(const Vec<1,T>& x) { return x.val; } 280 281 SIT Vec<1,T> min(const Vec<1,T>& x, const Vec<1,T>& y) { return std::min(x.val, y.val); } 282 SIT Vec<1,T> max(const Vec<1,T>& x, const Vec<1,T>& y) { return std::max(x.val, y.val); } 283 284 SIT Vec<1,T> ceil(const Vec<1,T>& x) { return std:: ceil(x.val); } 285 SIT Vec<1,T> floor(const Vec<1,T>& x) { return std::floor(x.val); } 286 SIT Vec<1,T> trunc(const Vec<1,T>& x) { return std::trunc(x.val); } 287 SIT Vec<1,T> round(const Vec<1,T>& x) { return std::round(x.val); } 288 SIT Vec<1,T> sqrt(const Vec<1,T>& x) { return std:: sqrt(x.val); } 289 SIT Vec<1,T> abs(const Vec<1,T>& x) { return std:: abs(x.val); } 290 291 SIT Vec<1,T> rcp(const Vec<1,T>& x) { return 1 / x.val; } 292 SIT Vec<1,T> rsqrt(const Vec<1,T>& x) { return rcp(sqrt(x)); } 293 SIT Vec<1,T> mad(const Vec<1,T>& f, 294 const Vec<1,T>& m, 295 const Vec<1,T>& a) { return f*m+a; } 296 297 // All default N != 1 implementations just recurse on lo and hi halves. 298 SINT Vec<N,T> if_then_else(const Vec<N,M<T>>& cond, const Vec<N,T>& t, const Vec<N,T>& e) { 299 return join(if_then_else(cond.lo, t.lo, e.lo), 300 if_then_else(cond.hi, t.hi, e.hi)); 301 } 302 303 SINT bool any(const Vec<N,T>& x) { return any(x.lo) || any(x.hi); } 304 SINT bool all(const Vec<N,T>& x) { return all(x.lo) && all(x.hi); } 305 306 SINT T min(const Vec<N,T>& x) { return std::min(min(x.lo), min(x.hi)); } 307 SINT T max(const Vec<N,T>& x) { return std::max(max(x.lo), max(x.hi)); } 308 309 SINT Vec<N,T> min(const Vec<N,T>& x, const Vec<N,T>& y) { return join(min(x.lo, y.lo), min(x.hi, y.hi)); } 310 SINT Vec<N,T> max(const Vec<N,T>& x, const Vec<N,T>& y) { return join(max(x.lo, y.lo), max(x.hi, y.hi)); } 311 312 SINT Vec<N,T> ceil(const Vec<N,T>& x) { return join( ceil(x.lo), ceil(x.hi)); } 313 SINT Vec<N,T> floor(const Vec<N,T>& x) { return join(floor(x.lo), floor(x.hi)); } 314 SINT Vec<N,T> trunc(const Vec<N,T>& x) { return join(trunc(x.lo), trunc(x.hi)); } 315 SINT Vec<N,T> round(const Vec<N,T>& x) { return join(round(x.lo), round(x.hi)); } 316 SINT Vec<N,T> sqrt(const Vec<N,T>& x) { return join( sqrt(x.lo), sqrt(x.hi)); } 317 SINT Vec<N,T> abs(const Vec<N,T>& x) { return join( abs(x.lo), abs(x.hi)); } 318 319 SINT Vec<N,T> rcp(const Vec<N,T>& x) { return join( rcp(x.lo), rcp(x.hi)); } 320 SINT Vec<N,T> rsqrt(const Vec<N,T>& x) { return join(rsqrt(x.lo), rsqrt(x.hi)); } 321 SINT Vec<N,T> mad(const Vec<N,T>& f, 322 const Vec<N,T>& m, 323 const Vec<N,T>& a) { return join(mad(f.lo, m.lo, a.lo), mad(f.hi, m.hi, a.hi)); } 324 325 326 // Scalar/vector operations just splat the scalar to a vector... 327 SINTU Vec<N,T> operator+ (U x, const Vec<N,T>& y) { return Vec<N,T>(x) + y; } 328 SINTU Vec<N,T> operator- (U x, const Vec<N,T>& y) { return Vec<N,T>(x) - y; } 329 SINTU Vec<N,T> operator* (U x, const Vec<N,T>& y) { return Vec<N,T>(x) * y; } 330 SINTU Vec<N,T> operator/ (U x, const Vec<N,T>& y) { return Vec<N,T>(x) / y; } 331 SINTU Vec<N,T> operator^ (U x, const Vec<N,T>& y) { return Vec<N,T>(x) ^ y; } 332 SINTU Vec<N,T> operator& (U x, const Vec<N,T>& y) { return Vec<N,T>(x) & y; } 333 SINTU Vec<N,T> operator| (U x, const Vec<N,T>& y) { return Vec<N,T>(x) | y; } 334 SINTU Vec<N,M<T>> operator==(U x, const Vec<N,T>& y) { return Vec<N,T>(x) == y; } 335 SINTU Vec<N,M<T>> operator!=(U x, const Vec<N,T>& y) { return Vec<N,T>(x) != y; } 336 SINTU Vec<N,M<T>> operator<=(U x, const Vec<N,T>& y) { return Vec<N,T>(x) <= y; } 337 SINTU Vec<N,M<T>> operator>=(U x, const Vec<N,T>& y) { return Vec<N,T>(x) >= y; } 338 SINTU Vec<N,M<T>> operator< (U x, const Vec<N,T>& y) { return Vec<N,T>(x) < y; } 339 SINTU Vec<N,M<T>> operator> (U x, const Vec<N,T>& y) { return Vec<N,T>(x) > y; } 340 SINTU Vec<N,T> min(U x, const Vec<N,T>& y) { return min(Vec<N,T>(x), y); } 341 SINTU Vec<N,T> max(U x, const Vec<N,T>& y) { return max(Vec<N,T>(x), y); } 342 343 // ... and same deal for vector/scalar operations. 344 SINTU Vec<N,T> operator+ (const Vec<N,T>& x, U y) { return x + Vec<N,T>(y); } 345 SINTU Vec<N,T> operator- (const Vec<N,T>& x, U y) { return x - Vec<N,T>(y); } 346 SINTU Vec<N,T> operator* (const Vec<N,T>& x, U y) { return x * Vec<N,T>(y); } 347 SINTU Vec<N,T> operator/ (const Vec<N,T>& x, U y) { return x / Vec<N,T>(y); } 348 SINTU Vec<N,T> operator^ (const Vec<N,T>& x, U y) { return x ^ Vec<N,T>(y); } 349 SINTU Vec<N,T> operator& (const Vec<N,T>& x, U y) { return x & Vec<N,T>(y); } 350 SINTU Vec<N,T> operator| (const Vec<N,T>& x, U y) { return x | Vec<N,T>(y); } 351 SINTU Vec<N,M<T>> operator==(const Vec<N,T>& x, U y) { return x == Vec<N,T>(y); } 352 SINTU Vec<N,M<T>> operator!=(const Vec<N,T>& x, U y) { return x != Vec<N,T>(y); } 353 SINTU Vec<N,M<T>> operator<=(const Vec<N,T>& x, U y) { return x <= Vec<N,T>(y); } 354 SINTU Vec<N,M<T>> operator>=(const Vec<N,T>& x, U y) { return x >= Vec<N,T>(y); } 355 SINTU Vec<N,M<T>> operator< (const Vec<N,T>& x, U y) { return x < Vec<N,T>(y); } 356 SINTU Vec<N,M<T>> operator> (const Vec<N,T>& x, U y) { return x > Vec<N,T>(y); } 357 SINTU Vec<N,T> min(const Vec<N,T>& x, U y) { return min(x, Vec<N,T>(y)); } 358 SINTU Vec<N,T> max(const Vec<N,T>& x, U y) { return max(x, Vec<N,T>(y)); } 359 360 // All vector/scalar combinations for mad() with at least one vector. 361 SINTU Vec<N,T> mad(U f, const Vec<N,T>& m, const Vec<N,T>& a) { return Vec<N,T>(f)*m + a; } 362 SINTU Vec<N,T> mad(const Vec<N,T>& f, U m, const Vec<N,T>& a) { return f*Vec<N,T>(m) + a; } 363 SINTU Vec<N,T> mad(const Vec<N,T>& f, const Vec<N,T>& m, U a) { return f*m + Vec<N,T>(a); } 364 SINTU Vec<N,T> mad(const Vec<N,T>& f, U m, U a) { return f*Vec<N,T>(m) + Vec<N,T>(a); } 365 SINTU Vec<N,T> mad(U f, const Vec<N,T>& m, U a) { return Vec<N,T>(f)*m + Vec<N,T>(a); } 366 SINTU Vec<N,T> mad(U f, U m, const Vec<N,T>& a) { return Vec<N,T>(f)*Vec<N,T>(m) + a; } 367 368 // The various op= operators, for vectors... 369 SINT Vec<N,T>& operator+=(Vec<N,T>& x, const Vec<N,T>& y) { return (x = x + y); } 370 SINT Vec<N,T>& operator-=(Vec<N,T>& x, const Vec<N,T>& y) { return (x = x - y); } 371 SINT Vec<N,T>& operator*=(Vec<N,T>& x, const Vec<N,T>& y) { return (x = x * y); } 372 SINT Vec<N,T>& operator/=(Vec<N,T>& x, const Vec<N,T>& y) { return (x = x / y); } 373 SINT Vec<N,T>& operator^=(Vec<N,T>& x, const Vec<N,T>& y) { return (x = x ^ y); } 374 SINT Vec<N,T>& operator&=(Vec<N,T>& x, const Vec<N,T>& y) { return (x = x & y); } 375 SINT Vec<N,T>& operator|=(Vec<N,T>& x, const Vec<N,T>& y) { return (x = x | y); } 376 377 // ... for scalars... 378 SINTU Vec<N,T>& operator+=(Vec<N,T>& x, U y) { return (x = x + Vec<N,T>(y)); } 379 SINTU Vec<N,T>& operator-=(Vec<N,T>& x, U y) { return (x = x - Vec<N,T>(y)); } 380 SINTU Vec<N,T>& operator*=(Vec<N,T>& x, U y) { return (x = x * Vec<N,T>(y)); } 381 SINTU Vec<N,T>& operator/=(Vec<N,T>& x, U y) { return (x = x / Vec<N,T>(y)); } 382 SINTU Vec<N,T>& operator^=(Vec<N,T>& x, U y) { return (x = x ^ Vec<N,T>(y)); } 383 SINTU Vec<N,T>& operator&=(Vec<N,T>& x, U y) { return (x = x & Vec<N,T>(y)); } 384 SINTU Vec<N,T>& operator|=(Vec<N,T>& x, U y) { return (x = x | Vec<N,T>(y)); } 385 386 // ... and for shifts. 387 SINT Vec<N,T>& operator<<=(Vec<N,T>& x, int bits) { return (x = x << bits); } 388 SINT Vec<N,T>& operator>>=(Vec<N,T>& x, int bits) { return (x = x >> bits); } 389 390 // cast() Vec<N,S> to Vec<N,D>, as if applying a C-cast to each lane. 391 template <typename D, typename S> 392 static inline Vec<1,D> cast(const Vec<1,S>& src) { return (D)src.val; } 393 394 template <typename D, int N, typename S> 395 static inline Vec<N,D> cast(const Vec<N,S>& src) { 396 #if !defined(SKNX_NO_SIMD) && defined(__clang__) 397 return to_vec(__builtin_convertvector(to_vext(src), VExt<N,D>)); 398 #else 399 return join(cast<D>(src.lo), cast<D>(src.hi)); 400 #endif 401 } 402 403 // Shuffle values from a vector pretty arbitrarily: 404 // skvx::Vec<4,float> rgba = {R,G,B,A}; 405 // shuffle<2,1,0,3> (rgba) ~> {B,G,R,A} 406 // shuffle<2,1> (rgba) ~> {B,G} 407 // shuffle<2,1,2,1,2,1,2,1>(rgba) ~> {B,G,B,G,B,G,B,G} 408 // shuffle<3,3,3,3> (rgba) ~> {A,A,A,A} 409 // The only real restriction is that the output also be a legal N=power-of-two sknx::Vec. 410 template <int... Ix, int N, typename T> 411 static inline Vec<sizeof...(Ix),T> shuffle(const Vec<N,T>& x) { 412 #if !defined(SKNX_NO_SIMD) && defined(__clang__) 413 return to_vec<sizeof...(Ix),T>(__builtin_shufflevector(to_vext(x), to_vext(x), Ix...)); 414 #else 415 return { x[Ix]... }; 416 #endif 417 } 418 419 // div255(x) = (x + 127) / 255 is a bit-exact rounding divide-by-255, packing down to 8-bit. 420 template <int N> 421 static inline Vec<N,uint8_t> div255(const Vec<N,uint16_t>& x) { 422 return cast<uint8_t>( (x+127)/255 ); 423 } 424 425 // approx_scale(x,y) approximates div255(cast<uint16_t>(x)*cast<uint16_t>(y)) within a bit, 426 // and is always perfect when x or y is 0 or 255. 427 template <int N> 428 static inline Vec<N,uint8_t> approx_scale(const Vec<N,uint8_t>& x, const Vec<N,uint8_t>& y) { 429 // All of (x*y+x)/256, (x*y+y)/256, and (x*y+255)/256 meet the criteria above. 430 // We happen to have historically picked (x*y+x)/256. 431 auto X = cast<uint16_t>(x), 432 Y = cast<uint16_t>(y); 433 return cast<uint8_t>( (X*Y+X)/256 ); 434 } 435 436 #if !defined(SKNX_NO_SIMD) && defined(__ARM_NEON) 437 // With NEON we can do eight u8*u8 -> u16 in one instruction, vmull_u8 (read, mul-long). 438 static inline Vec<8,uint16_t> mull(const Vec<8,uint8_t>& x, 439 const Vec<8,uint8_t>& y) { 440 return to_vec<8,uint16_t>(vmull_u8(to_vext(x), 441 to_vext(y))); 442 } 443 444 template <int N> 445 static inline typename std::enable_if<(N < 8), 446 Vec<N,uint16_t>>::type mull(const Vec<N,uint8_t>& x, 447 const Vec<N,uint8_t>& y) { 448 // N < 8 --> double up data until N == 8, returning the part we need. 449 return mull(join(x,x), 450 join(y,y)).lo; 451 } 452 453 template <int N> 454 static inline typename std::enable_if<(N > 8), 455 Vec<N,uint16_t>>::type mull(const Vec<N,uint8_t>& x, 456 const Vec<N,uint8_t>& y) { 457 // N > 8 --> usual join(lo,hi) strategy to recurse down to N == 8. 458 return join(mull(x.lo, y.lo), 459 mull(x.hi, y.hi)); 460 } 461 #else 462 // Nothing special when we don't have NEON... just cast up to 16-bit and multiply. 463 template <int N> 464 static inline Vec<N,uint16_t> mull(const Vec<N,uint8_t>& x, 465 const Vec<N,uint8_t>& y) { 466 return cast<uint16_t>(x) 467 * cast<uint16_t>(y); 468 } 469 #endif 470 471 #if !defined(SKNX_NO_SIMD) 472 473 // Platform-specific specializations and overloads can now drop in here. 474 475 #if defined(__SSE__) 476 static inline Vec<4,float> sqrt(const Vec<4,float>& x) { 477 return bit_pun<Vec<4,float>>(_mm_sqrt_ps(bit_pun<__m128>(x))); 478 } 479 static inline Vec<4,float> rsqrt(const Vec<4,float>& x) { 480 return bit_pun<Vec<4,float>>(_mm_rsqrt_ps(bit_pun<__m128>(x))); 481 } 482 static inline Vec<4,float> rcp(const Vec<4,float>& x) { 483 return bit_pun<Vec<4,float>>(_mm_rcp_ps(bit_pun<__m128>(x))); 484 } 485 486 static inline Vec<2,float> sqrt(const Vec<2,float>& x) { 487 return shuffle<0,1>( sqrt(shuffle<0,1,0,1>(x))); 488 } 489 static inline Vec<2,float> rsqrt(const Vec<2,float>& x) { 490 return shuffle<0,1>(rsqrt(shuffle<0,1,0,1>(x))); 491 } 492 static inline Vec<2,float> rcp(const Vec<2,float>& x) { 493 return shuffle<0,1>( rcp(shuffle<0,1,0,1>(x))); 494 } 495 #endif 496 497 #if defined(__SSE4_1__) 498 static inline Vec<4,float> if_then_else(const Vec<4,int >& c, 499 const Vec<4,float>& t, 500 const Vec<4,float>& e) { 501 return bit_pun<Vec<4,float>>(_mm_blendv_ps(bit_pun<__m128>(e), 502 bit_pun<__m128>(t), 503 bit_pun<__m128>(c))); 504 } 505 #elif defined(__SSE__) 506 static inline Vec<4,float> if_then_else(const Vec<4,int >& c, 507 const Vec<4,float>& t, 508 const Vec<4,float>& e) { 509 return bit_pun<Vec<4,float>>(_mm_or_ps(_mm_and_ps (bit_pun<__m128>(c), 510 bit_pun<__m128>(t)), 511 _mm_andnot_ps(bit_pun<__m128>(c), 512 bit_pun<__m128>(e)))); 513 } 514 #elif defined(__ARM_NEON) 515 static inline Vec<4,float> if_then_else(const Vec<4,int >& c, 516 const Vec<4,float>& t, 517 const Vec<4,float>& e) { 518 return bit_pun<Vec<4,float>>(vbslq_f32(bit_pun<uint32x4_t> (c), 519 bit_pun<float32x4_t>(t), 520 bit_pun<float32x4_t>(e))); 521 } 522 #endif 523 524 #endif // !defined(SKNX_NO_SIMD) 525 526 } // namespace skvx 527 528 #undef SINTU 529 #undef SINT 530 #undef SIT 531 #undef SKVX_ALIGNMENT 532 533 #endif//SKVX_DEFINED 534