1 /* 2 * Copyright 2019 Google Inc. 3 * 4 * Use of this source code is governed by a BSD-style license that can be 5 * found in the LICENSE file. 6 */ 7 8 #ifndef SKVX_DEFINED 9 #define SKVX_DEFINED 10 11 // skvx::Vec<N,T> are SIMD vectors of N T's, a v1.5 successor to SkNx<N,T>. 12 // 13 // This time we're leaning a bit less on platform-specific intrinsics and a bit 14 // more on Clang/GCC vector extensions, but still keeping the option open to 15 // drop in platform-specific intrinsics, actually more easily than before. 16 // 17 // We've also fixed a few of the caveats that used to make SkNx awkward to work 18 // with across translation units. skvx::Vec<N,T> always has N*sizeof(T) size 19 // and alignof(T) alignment and is safe to use across translation units freely. 20 21 #include "SkTypes.h" // SK_CPU_SSE_LEVEL*, etc. 22 #include <algorithm> // std::min, std::max 23 #include <cmath> // std::ceil, std::floor, std::trunc, std::round, std::sqrt, etc. 24 #include <cstdint> // intXX_t 25 #include <cstring> // memcpy() 26 #include <initializer_list> // std::initializer_list 27 28 #if SK_CPU_SSE_LEVEL >= SK_CPU_SSE_LEVEL_SSE1 29 #include <immintrin.h> 30 #elif defined(SK_ARM_HAS_NEON) 31 #include <arm_neon.h> 32 #endif 33 34 35 namespace skvx { 36 37 // All Vec have the same simple memory layout, the same as `T vec[N]`. 38 // This gives Vec a consistent ABI, letting them pass between files compiled with 39 // different instruction sets (e.g. SSE2 and AVX2) without fear of ODR violation. 40 template <int N, typename T> 41 struct Vec { 42 static_assert((N & (N-1)) == 0, "N must be a power of 2."); 43 44 Vec<N/2,T> lo, hi; 45 46 // Methods belong here in the class declaration of Vec only if: 47 // - they must be here, like constructors or operator[]; 48 // - they'll definitely never want a specialized implementation. 49 // Other operations on Vec should be defined outside the type. 50 51 Vec() = default; VecVec52 Vec(T x) : lo(x), hi(x) {} 53 VecVec54 Vec(std::initializer_list<T> xs) { 55 T vals[N] = {0}; 56 memcpy(vals, xs.begin(), std::min(xs.size(), (size_t)N)*sizeof(T)); 57 58 lo = Vec<N/2,T>::Load(vals + 0); 59 hi = Vec<N/2,T>::Load(vals + N/2); 60 } 61 62 T operator[](int i) const { return i < N/2 ? lo[i] : hi[i-N/2]; } 63 T& operator[](int i) { return i < N/2 ? lo[i] : hi[i-N/2]; } 64 LoadVec65 static Vec Load(const void* ptr) { 66 Vec v; 67 memcpy(&v, ptr, sizeof(Vec)); 68 return v; 69 } storeVec70 void store(void* ptr) const { 71 memcpy(ptr, this, sizeof(Vec)); 72 } 73 }; 74 75 template <typename T> 76 struct Vec<1,T> { 77 T val; 78 79 Vec() = default; 80 Vec(T x) : val(x) {} 81 82 Vec(std::initializer_list<T> xs) : val(xs.size() ? *xs.begin() : 0) {} 83 84 T operator[](int) const { return val; } 85 T& operator[](int) { return val; } 86 87 static Vec Load(const void* ptr) { 88 Vec v; 89 memcpy(&v, ptr, sizeof(Vec)); 90 return v; 91 } 92 void store(void* ptr) const { 93 memcpy(ptr, this, sizeof(Vec)); 94 } 95 }; 96 97 #if defined(__GNUC__) && !defined(__clang__) && SK_CPU_SSE_LEVEL >= SK_CPU_SSE_LEVEL_SSE1 98 // GCC warns about ABI changes when returning >= 32 byte vectors when -mavx is not enabled. 99 // This only happens for types like VExt whose ABI we don't care about, not for Vec itself. 100 #pragma GCC diagnostic ignored "-Wpsabi" 101 #endif 102 103 // Helps tamp down on the repetitive boilerplate. 104 #define SINT template <int N, typename T> static inline 105 #define SIT template < typename T> static inline 106 #define SI static inline 107 108 template <typename D, typename S> 109 SI D bit_pun(S s) { 110 static_assert(sizeof(D) == sizeof(S), ""); 111 D d; 112 memcpy(&d, &s, sizeof(D)); 113 return d; 114 } 115 116 // Translate from a value type T to its corresponding Mask, the result of a comparison. 117 template <typename T> struct Mask { using type = T; }; 118 template <> struct Mask<float > { using type = int32_t; }; 119 template <> struct Mask<double> { using type = int64_t; }; 120 template <typename T> using M = typename Mask<T>::type; 121 122 // Join two Vec<N,T> into one Vec<2N,T>. 123 SINT Vec<2*N,T> join(Vec<N,T> lo, Vec<N,T> hi) { 124 Vec<2*N,T> v; 125 v.lo = lo; 126 v.hi = hi; 127 return v; 128 } 129 130 // We have two default strategies for implementing most operations: 131 // 1) lean on Clang/GCC vector extensions when available; 132 // 2) recurse to scalar portable implementations when not. 133 // At the end we can drop in platform-specific implementations that override either default. 134 135 #if !defined(SKNX_NO_SIMD) && (defined(__clang__) || defined(__GNUC__)) 136 137 // VExt<N,T> types have the same size as Vec<N,T> and support most operations directly. 138 // N.B. VExt<N,T> alignment is N*alignof(T), stricter than Vec<N,T>'s alignof(T). 139 #if defined(__clang__) 140 template <int N, typename T> 141 using VExt = T __attribute__((ext_vector_type(N))); 142 143 #elif defined(__GNUC__) 144 template <int N, typename T> 145 struct VExtHelper { 146 typedef T __attribute__((vector_size(N*sizeof(T)))) type; 147 }; 148 149 template <int N, typename T> 150 using VExt = typename VExtHelper<N,T>::type; 151 152 // For some reason some (new!) versions of GCC cannot seem to deduce N in the generic 153 // to_vec<N,T>() below for N=4 and T=float. This workaround seems to help... 154 SI Vec<4,float> to_vec(VExt<4,float> v) { return bit_pun<Vec<4,float>>(v); } 155 #endif 156 157 SINT VExt<N,T> to_vext(Vec<N,T> v) { return bit_pun<VExt<N,T>>(v); } 158 SINT Vec <N,T> to_vec(VExt<N,T> v) { return bit_pun<Vec <N,T>>(v); } 159 160 SINT Vec<N,T> operator+(Vec<N,T> x, Vec<N,T> y) { return to_vec(to_vext(x) + to_vext(y)); } 161 SINT Vec<N,T> operator-(Vec<N,T> x, Vec<N,T> y) { return to_vec(to_vext(x) - to_vext(y)); } 162 SINT Vec<N,T> operator*(Vec<N,T> x, Vec<N,T> y) { return to_vec(to_vext(x) * to_vext(y)); } 163 SINT Vec<N,T> operator/(Vec<N,T> x, Vec<N,T> y) { return to_vec(to_vext(x) / to_vext(y)); } 164 165 SINT Vec<N,T> operator^(Vec<N,T> x, Vec<N,T> y) { return to_vec(to_vext(x) ^ to_vext(y)); } 166 SINT Vec<N,T> operator&(Vec<N,T> x, Vec<N,T> y) { return to_vec(to_vext(x) & to_vext(y)); } 167 SINT Vec<N,T> operator|(Vec<N,T> x, Vec<N,T> y) { return to_vec(to_vext(x) | to_vext(y)); } 168 169 SINT Vec<N,T> operator!(Vec<N,T> x) { return to_vec(!to_vext(x)); } 170 SINT Vec<N,T> operator-(Vec<N,T> x) { return to_vec(-to_vext(x)); } 171 SINT Vec<N,T> operator~(Vec<N,T> x) { return to_vec(~to_vext(x)); } 172 173 SINT Vec<N,T> operator<<(Vec<N,T> x, int bits) { return to_vec(to_vext(x) << bits); } 174 SINT Vec<N,T> operator>>(Vec<N,T> x, int bits) { return to_vec(to_vext(x) >> bits); } 175 176 SINT Vec<N,M<T>> operator==(Vec<N,T> x, Vec<N,T> y) { return bit_pun<Vec<N,M<T>>>(to_vext(x) == to_vext(y)); } 177 SINT Vec<N,M<T>> operator!=(Vec<N,T> x, Vec<N,T> y) { return bit_pun<Vec<N,M<T>>>(to_vext(x) != to_vext(y)); } 178 SINT Vec<N,M<T>> operator<=(Vec<N,T> x, Vec<N,T> y) { return bit_pun<Vec<N,M<T>>>(to_vext(x) <= to_vext(y)); } 179 SINT Vec<N,M<T>> operator>=(Vec<N,T> x, Vec<N,T> y) { return bit_pun<Vec<N,M<T>>>(to_vext(x) >= to_vext(y)); } 180 SINT Vec<N,M<T>> operator< (Vec<N,T> x, Vec<N,T> y) { return bit_pun<Vec<N,M<T>>>(to_vext(x) < to_vext(y)); } 181 SINT Vec<N,M<T>> operator> (Vec<N,T> x, Vec<N,T> y) { return bit_pun<Vec<N,M<T>>>(to_vext(x) > to_vext(y)); } 182 183 #else 184 185 // Either SKNX_NO_SIMD is defined, or Clang/GCC vector extensions are not available. 186 // We'll implement things portably, in a way that should be easily autovectorizable. 187 188 // N == 1 scalar implementations. 189 SIT Vec<1,T> operator+(Vec<1,T> x, Vec<1,T> y) { return x.val + y.val; } 190 SIT Vec<1,T> operator-(Vec<1,T> x, Vec<1,T> y) { return x.val - y.val; } 191 SIT Vec<1,T> operator*(Vec<1,T> x, Vec<1,T> y) { return x.val * y.val; } 192 SIT Vec<1,T> operator/(Vec<1,T> x, Vec<1,T> y) { return x.val / y.val; } 193 194 SIT Vec<1,T> operator^(Vec<1,T> x, Vec<1,T> y) { return x.val ^ y.val; } 195 SIT Vec<1,T> operator&(Vec<1,T> x, Vec<1,T> y) { return x.val & y.val; } 196 SIT Vec<1,T> operator|(Vec<1,T> x, Vec<1,T> y) { return x.val | y.val; } 197 198 SIT Vec<1,T> operator!(Vec<1,T> x) { return !x.val; } 199 SIT Vec<1,T> operator-(Vec<1,T> x) { return -x.val; } 200 SIT Vec<1,T> operator~(Vec<1,T> x) { return ~x.val; } 201 202 SIT Vec<1,T> operator<<(Vec<1,T> x, int bits) { return x.val << bits; } 203 SIT Vec<1,T> operator>>(Vec<1,T> x, int bits) { return x.val >> bits; } 204 205 SIT Vec<1,M<T>> operator==(Vec<1,T> x, Vec<1,T> y) { return x.val == y.val ? ~0 : 0; } 206 SIT Vec<1,M<T>> operator!=(Vec<1,T> x, Vec<1,T> y) { return x.val != y.val ? ~0 : 0; } 207 SIT Vec<1,M<T>> operator<=(Vec<1,T> x, Vec<1,T> y) { return x.val <= y.val ? ~0 : 0; } 208 SIT Vec<1,M<T>> operator>=(Vec<1,T> x, Vec<1,T> y) { return x.val >= y.val ? ~0 : 0; } 209 SIT Vec<1,M<T>> operator< (Vec<1,T> x, Vec<1,T> y) { return x.val < y.val ? ~0 : 0; } 210 SIT Vec<1,M<T>> operator> (Vec<1,T> x, Vec<1,T> y) { return x.val > y.val ? ~0 : 0; } 211 212 // All default N != 1 implementations just recurse on lo and hi halves. 213 SINT Vec<N,T> operator+(Vec<N,T> x, Vec<N,T> y) { return join(x.lo + y.lo, x.hi + y.hi); } 214 SINT Vec<N,T> operator-(Vec<N,T> x, Vec<N,T> y) { return join(x.lo - y.lo, x.hi - y.hi); } 215 SINT Vec<N,T> operator*(Vec<N,T> x, Vec<N,T> y) { return join(x.lo * y.lo, x.hi * y.hi); } 216 SINT Vec<N,T> operator/(Vec<N,T> x, Vec<N,T> y) { return join(x.lo / y.lo, x.hi / y.hi); } 217 218 SINT Vec<N,T> operator^(Vec<N,T> x, Vec<N,T> y) { return join(x.lo ^ y.lo, x.hi ^ y.hi); } 219 SINT Vec<N,T> operator&(Vec<N,T> x, Vec<N,T> y) { return join(x.lo & y.lo, x.hi & y.hi); } 220 SINT Vec<N,T> operator|(Vec<N,T> x, Vec<N,T> y) { return join(x.lo | y.lo, x.hi | y.hi); } 221 222 SINT Vec<N,T> operator!(Vec<N,T> x) { return join(!x.lo, !x.hi); } 223 SINT Vec<N,T> operator-(Vec<N,T> x) { return join(-x.lo, -x.hi); } 224 SINT Vec<N,T> operator~(Vec<N,T> x) { return join(~x.lo, ~x.hi); } 225 226 SINT Vec<N,T> operator<<(Vec<N,T> x, int bits) { return join(x.lo << bits, x.hi << bits); } 227 SINT Vec<N,T> operator>>(Vec<N,T> x, int bits) { return join(x.lo >> bits, x.hi >> bits); } 228 229 SINT Vec<N,M<T>> operator==(Vec<N,T> x, Vec<N,T> y) { return join(x.lo == y.lo, x.hi == y.hi); } 230 SINT Vec<N,M<T>> operator!=(Vec<N,T> x, Vec<N,T> y) { return join(x.lo != y.lo, x.hi != y.hi); } 231 SINT Vec<N,M<T>> operator<=(Vec<N,T> x, Vec<N,T> y) { return join(x.lo <= y.lo, x.hi <= y.hi); } 232 SINT Vec<N,M<T>> operator>=(Vec<N,T> x, Vec<N,T> y) { return join(x.lo >= y.lo, x.hi >= y.hi); } 233 SINT Vec<N,M<T>> operator< (Vec<N,T> x, Vec<N,T> y) { return join(x.lo < y.lo, x.hi < y.hi); } 234 SINT Vec<N,M<T>> operator> (Vec<N,T> x, Vec<N,T> y) { return join(x.lo > y.lo, x.hi > y.hi); } 235 #endif 236 237 // Some operations we want are not expressible with Clang/GCC vector 238 // extensions, so we implement them using the recursive approach. 239 240 // N == 1 scalar implementations. 241 SIT Vec<1,T> if_then_else(Vec<1,M<T>> cond, Vec<1,T> t, Vec<1,T> e) { 242 auto t_bits = bit_pun<M<T>>(t), 243 e_bits = bit_pun<M<T>>(e); 244 return bit_pun<T>( (cond.val & t_bits) | (~cond.val & e_bits) ); 245 } 246 247 SIT bool any(Vec<1,T> x) { return x.val != 0; } 248 SIT bool all(Vec<1,T> x) { return x.val != 0; } 249 250 SIT T min(Vec<1,T> x) { return x.val; } 251 SIT T max(Vec<1,T> x) { return x.val; } 252 253 SIT Vec<1,T> min(Vec<1,T> x, Vec<1,T> y) { return std::min(x.val, y.val); } 254 SIT Vec<1,T> max(Vec<1,T> x, Vec<1,T> y) { return std::max(x.val, y.val); } 255 256 SIT Vec<1,T> ceil(Vec<1,T> x) { return std:: ceil(x.val); } 257 SIT Vec<1,T> floor(Vec<1,T> x) { return std::floor(x.val); } 258 SIT Vec<1,T> trunc(Vec<1,T> x) { return std::trunc(x.val); } 259 SIT Vec<1,T> round(Vec<1,T> x) { return std::round(x.val); } 260 SIT Vec<1,T> sqrt(Vec<1,T> x) { return std:: sqrt(x.val); } 261 SIT Vec<1,T> abs(Vec<1,T> x) { return std:: abs(x.val); } 262 263 SIT Vec<1,T> rcp(Vec<1,T> x) { return 1 / x.val; } 264 SIT Vec<1,T> rsqrt(Vec<1,T> x) { return rcp(sqrt(x)); } 265 SIT Vec<1,T> mad(Vec<1,T> f, 266 Vec<1,T> m, 267 Vec<1,T> a) { return f*m+a; } 268 269 // All default N != 1 implementations just recurse on lo and hi halves. 270 SINT Vec<N,T> if_then_else(Vec<N,M<T>> cond, Vec<N,T> t, Vec<N,T> e) { 271 return join(if_then_else(cond.lo, t.lo, e.lo), 272 if_then_else(cond.hi, t.hi, e.hi)); 273 } 274 275 SINT bool any(Vec<N,T> x) { return any(x.lo) || any(x.hi); } 276 SINT bool all(Vec<N,T> x) { return all(x.lo) && all(x.hi); } 277 278 SINT T min(Vec<N,T> x) { return std::min(min(x.lo), min(x.hi)); } 279 SINT T max(Vec<N,T> x) { return std::max(max(x.lo), max(x.hi)); } 280 281 SINT Vec<N,T> min(Vec<N,T> x, Vec<N,T> y) { return join(min(x.lo, y.lo), min(x.hi, y.hi)); } 282 SINT Vec<N,T> max(Vec<N,T> x, Vec<N,T> y) { return join(max(x.lo, y.lo), max(x.hi, y.hi)); } 283 284 SINT Vec<N,T> ceil(Vec<N,T> x) { return join( ceil(x.lo), ceil(x.hi)); } 285 SINT Vec<N,T> floor(Vec<N,T> x) { return join(floor(x.lo), floor(x.hi)); } 286 SINT Vec<N,T> trunc(Vec<N,T> x) { return join(trunc(x.lo), trunc(x.hi)); } 287 SINT Vec<N,T> round(Vec<N,T> x) { return join(round(x.lo), round(x.hi)); } 288 SINT Vec<N,T> sqrt(Vec<N,T> x) { return join( sqrt(x.lo), sqrt(x.hi)); } 289 SINT Vec<N,T> abs(Vec<N,T> x) { return join( abs(x.lo), abs(x.hi)); } 290 291 SINT Vec<N,T> rcp(Vec<N,T> x) { return join( rcp(x.lo), rcp(x.hi)); } 292 SINT Vec<N,T> rsqrt(Vec<N,T> x) { return join(rsqrt(x.lo), rsqrt(x.hi)); } 293 SINT Vec<N,T> mad(Vec<N,T> f, 294 Vec<N,T> m, 295 Vec<N,T> a) { return join(mad(f.lo, m.lo, a.lo), mad(f.hi, m.hi, a.hi)); } 296 297 298 // Scalar/vector operations just splat the scalar to a vector... 299 SINT Vec<N,T> operator+ (T x, Vec<N,T> y) { return Vec<N,T>(x) + y; } 300 SINT Vec<N,T> operator- (T x, Vec<N,T> y) { return Vec<N,T>(x) - y; } 301 SINT Vec<N,T> operator* (T x, Vec<N,T> y) { return Vec<N,T>(x) * y; } 302 SINT Vec<N,T> operator/ (T x, Vec<N,T> y) { return Vec<N,T>(x) / y; } 303 SINT Vec<N,T> operator^ (T x, Vec<N,T> y) { return Vec<N,T>(x) ^ y; } 304 SINT Vec<N,T> operator& (T x, Vec<N,T> y) { return Vec<N,T>(x) & y; } 305 SINT Vec<N,T> operator| (T x, Vec<N,T> y) { return Vec<N,T>(x) | y; } 306 SINT Vec<N,M<T>> operator==(T x, Vec<N,T> y) { return Vec<N,T>(x) == y; } 307 SINT Vec<N,M<T>> operator!=(T x, Vec<N,T> y) { return Vec<N,T>(x) != y; } 308 SINT Vec<N,M<T>> operator<=(T x, Vec<N,T> y) { return Vec<N,T>(x) <= y; } 309 SINT Vec<N,M<T>> operator>=(T x, Vec<N,T> y) { return Vec<N,T>(x) >= y; } 310 SINT Vec<N,M<T>> operator< (T x, Vec<N,T> y) { return Vec<N,T>(x) < y; } 311 SINT Vec<N,M<T>> operator> (T x, Vec<N,T> y) { return Vec<N,T>(x) > y; } 312 SINT Vec<N,T> min(T x, Vec<N,T> y) { return min(Vec<N,T>(x), y); } 313 SINT Vec<N,T> max(T x, Vec<N,T> y) { return max(Vec<N,T>(x), y); } 314 315 // ... and same deal for vector/scalar operations. 316 SINT Vec<N,T> operator+ (Vec<N,T> x, T y) { return x + Vec<N,T>(y); } 317 SINT Vec<N,T> operator- (Vec<N,T> x, T y) { return x - Vec<N,T>(y); } 318 SINT Vec<N,T> operator* (Vec<N,T> x, T y) { return x * Vec<N,T>(y); } 319 SINT Vec<N,T> operator/ (Vec<N,T> x, T y) { return x / Vec<N,T>(y); } 320 SINT Vec<N,T> operator^ (Vec<N,T> x, T y) { return x ^ Vec<N,T>(y); } 321 SINT Vec<N,T> operator& (Vec<N,T> x, T y) { return x & Vec<N,T>(y); } 322 SINT Vec<N,T> operator| (Vec<N,T> x, T y) { return x | Vec<N,T>(y); } 323 SINT Vec<N,M<T>> operator==(Vec<N,T> x, T y) { return x == Vec<N,T>(y); } 324 SINT Vec<N,M<T>> operator!=(Vec<N,T> x, T y) { return x != Vec<N,T>(y); } 325 SINT Vec<N,M<T>> operator<=(Vec<N,T> x, T y) { return x <= Vec<N,T>(y); } 326 SINT Vec<N,M<T>> operator>=(Vec<N,T> x, T y) { return x >= Vec<N,T>(y); } 327 SINT Vec<N,M<T>> operator< (Vec<N,T> x, T y) { return x < Vec<N,T>(y); } 328 SINT Vec<N,M<T>> operator> (Vec<N,T> x, T y) { return x > Vec<N,T>(y); } 329 SINT Vec<N,T> min(Vec<N,T> x, T y) { return min(x, Vec<N,T>(y)); } 330 SINT Vec<N,T> max(Vec<N,T> x, T y) { return max(x, Vec<N,T>(y)); } 331 332 // All vector/scalar combinations for mad() with at least one vector. 333 SINT Vec<N,T> mad(T f, Vec<N,T> m, Vec<N,T> a) { return Vec<N,T>(f)*m + a; } 334 SINT Vec<N,T> mad(Vec<N,T> f, T m, Vec<N,T> a) { return f*Vec<N,T>(m) + a; } 335 SINT Vec<N,T> mad(Vec<N,T> f, Vec<N,T> m, T a) { return f*m + Vec<N,T>(a); } 336 SINT Vec<N,T> mad(Vec<N,T> f, T m, T a) { return f*Vec<N,T>(m) + Vec<N,T>(a); } 337 SINT Vec<N,T> mad(T f, Vec<N,T> m, T a) { return Vec<N,T>(f)*m + Vec<N,T>(a); } 338 SINT Vec<N,T> mad(T f, T m, Vec<N,T> a) { return Vec<N,T>(f)*Vec<N,T>(m) + a; } 339 340 // The various op= operators, for vectors... 341 SINT Vec<N,T>& operator+=(Vec<N,T>& x, Vec<N,T> y) { return (x = x + y); } 342 SINT Vec<N,T>& operator-=(Vec<N,T>& x, Vec<N,T> y) { return (x = x - y); } 343 SINT Vec<N,T>& operator*=(Vec<N,T>& x, Vec<N,T> y) { return (x = x * y); } 344 SINT Vec<N,T>& operator/=(Vec<N,T>& x, Vec<N,T> y) { return (x = x / y); } 345 SINT Vec<N,T>& operator^=(Vec<N,T>& x, Vec<N,T> y) { return (x = x ^ y); } 346 SINT Vec<N,T>& operator&=(Vec<N,T>& x, Vec<N,T> y) { return (x = x & y); } 347 SINT Vec<N,T>& operator|=(Vec<N,T>& x, Vec<N,T> y) { return (x = x | y); } 348 349 // ... for scalars... 350 SINT Vec<N,T>& operator+=(Vec<N,T>& x, T y) { return (x = x + Vec<N,T>(y)); } 351 SINT Vec<N,T>& operator-=(Vec<N,T>& x, T y) { return (x = x - Vec<N,T>(y)); } 352 SINT Vec<N,T>& operator*=(Vec<N,T>& x, T y) { return (x = x * Vec<N,T>(y)); } 353 SINT Vec<N,T>& operator/=(Vec<N,T>& x, T y) { return (x = x / Vec<N,T>(y)); } 354 SINT Vec<N,T>& operator^=(Vec<N,T>& x, T y) { return (x = x ^ Vec<N,T>(y)); } 355 SINT Vec<N,T>& operator&=(Vec<N,T>& x, T y) { return (x = x & Vec<N,T>(y)); } 356 SINT Vec<N,T>& operator|=(Vec<N,T>& x, T y) { return (x = x | Vec<N,T>(y)); } 357 358 // ... and for shifts. 359 SINT Vec<N,T>& operator<<=(Vec<N,T>& x, int bits) { return (x = x << bits); } 360 SINT Vec<N,T>& operator>>=(Vec<N,T>& x, int bits) { return (x = x >> bits); } 361 362 } // namespace skvx 363 364 // These next few routines take extra template arguments that prevent 365 // argument-dependent lookup. They must live outside the skvx namespace, 366 // but since they operate only on skvx::Vec, that shouldn't be a big deal. 367 368 // cast() Vec<N,S> to Vec<N,D>, as if applying a C-cast to each lane. 369 template <typename D, typename S> 370 SI skvx::Vec<1,D> cast(skvx::Vec<1,S> src) { return (D)src.val; } 371 372 template <typename D, int N, typename S> 373 SI skvx::Vec<N,D> cast(skvx::Vec<N,S> src) { 374 #if !defined(SKNX_NO_SIMD) && defined(__clang__) 375 return skvx::to_vec(__builtin_convertvector(skvx::to_vext(src), skvx::VExt<N,D>)); 376 #else 377 return join(cast<D>(src.lo), cast<D>(src.hi)); 378 #endif 379 } 380 381 // Shuffle values from a vector pretty arbitrarily: 382 // skvx::Vec<4,float> rgba = {R,G,B,A}; 383 // shuffle<2,1,0,3> (rgba) ~> {B,G,R,A} 384 // shuffle<2,1> (rgba) ~> {B,G} 385 // shuffle<2,1,2,1,2,1,2,1>(rgba) ~> {B,G,B,G,B,G,B,G} 386 // shuffle<3,3,3,3> (rgba) ~> {A,A,A,A} 387 // The only real restriction is that the output also be a legal N=power-of-two sknx::Vec. 388 template <int... Ix, int N, typename T> 389 SI skvx::Vec<sizeof...(Ix),T> shuffle(skvx::Vec<N,T> x) { 390 return { x[Ix]... }; 391 } 392 393 #if !defined(SKNX_NO_SIMD) 394 namespace skvx { 395 // Platform-specific specializations and overloads can now drop in here. 396 397 #if SK_CPU_SSE_LEVEL >= SK_CPU_SSE_LEVEL_SSE1 398 SI Vec<4,float> sqrt(Vec<4,float> x) { 399 return bit_pun<Vec<4,float>>(_mm_sqrt_ps(bit_pun<__m128>(x))); 400 } 401 SI Vec<4,float> rsqrt(Vec<4,float> x) { 402 return bit_pun<Vec<4,float>>(_mm_rsqrt_ps(bit_pun<__m128>(x))); 403 } 404 SI Vec<4,float> rcp(Vec<4,float> x) { 405 return bit_pun<Vec<4,float>>(_mm_rcp_ps(bit_pun<__m128>(x))); 406 } 407 408 SI Vec<2,float> sqrt(Vec<2,float> x) { return shuffle<0,1>( sqrt(shuffle<0,1,0,1>(x))); } 409 SI Vec<2,float> rsqrt(Vec<2,float> x) { return shuffle<0,1>(rsqrt(shuffle<0,1,0,1>(x))); } 410 SI Vec<2,float> rcp(Vec<2,float> x) { return shuffle<0,1>( rcp(shuffle<0,1,0,1>(x))); } 411 #endif 412 413 #if SK_CPU_SSE_LEVEL >= SK_CPU_SSE_LEVEL_SSE41 414 SI Vec<4,float> if_then_else(Vec<4,int> c, Vec<4,float> t, Vec<4,float> e) { 415 return bit_pun<Vec<4,float>>(_mm_blendv_ps(bit_pun<__m128>(e), 416 bit_pun<__m128>(t), 417 bit_pun<__m128>(c))); 418 } 419 #elif SK_CPU_SSE_LEVEL >= SK_CPU_SSE_LEVEL_SSE1 420 SI Vec<4,float> if_then_else(Vec<4,int> c, Vec<4,float> t, Vec<4,float> e) { 421 return bit_pun<Vec<4,float>>(_mm_or_ps(_mm_and_ps (bit_pun<__m128>(c), 422 bit_pun<__m128>(t)), 423 _mm_andnot_ps(bit_pun<__m128>(c), 424 bit_pun<__m128>(e)))); 425 } 426 #elif defined(SK_ARM_HAS_NEON) 427 SI Vec<4,float> if_then_else(Vec<4,int> c, Vec<4,float> t, Vec<4,float> e) { 428 return bit_pun<Vec<4,float>>(vbslq_f32(bit_pun<uint32x4_t> (c), 429 bit_pun<float32x4_t>(t), 430 bit_pun<float32x4_t>(e))); 431 } 432 #endif 433 434 } // namespace skvx 435 #endif // !defined(SKNX_NO_SIMD) 436 437 #undef SINT 438 #undef SIT 439 #undef SI 440 441 #endif//SKVX_DEFINED 442