1 /*
2 * Copyright 2015 Google Inc.
3 *
4 * Use of this source code is governed by a BSD-style license that can be
5 * found in the LICENSE file.
6 */
7
8 #ifndef SkNx_neon_DEFINED
9 #define SkNx_neon_DEFINED
10
11 #include <arm_neon.h>
12
13 namespace {
14
15 // ARMv8 has vrndmq_f32 to floor 4 floats. Here we emulate it:
16 // - roundtrip through integers via truncation
17 // - subtract 1 if that's too big (possible for negative values).
18 // This restricts the domain of our inputs to a maximum somehwere around 2^31. Seems plenty big.
armv7_vrndmq_f32(float32x4_t v)19 AI static float32x4_t armv7_vrndmq_f32(float32x4_t v) {
20 auto roundtrip = vcvtq_f32_s32(vcvtq_s32_f32(v));
21 auto too_big = vcgtq_f32(roundtrip, v);
22 return vsubq_f32(roundtrip, (float32x4_t)vandq_u32(too_big, (uint32x4_t)vdupq_n_f32(1)));
23 }
24
25 template <>
26 class SkNx<2, float> {
27 public:
SkNx(float32x2_t vec)28 AI SkNx(float32x2_t vec) : fVec(vec) {}
29
SkNx()30 AI SkNx() {}
SkNx(float val)31 AI SkNx(float val) : fVec(vdup_n_f32(val)) {}
SkNx(float a,float b)32 AI SkNx(float a, float b) { fVec = (float32x2_t) { a, b }; }
33
Load(const void * ptr)34 AI static SkNx Load(const void* ptr) { return vld1_f32((const float*)ptr); }
store(void * ptr)35 AI void store(void* ptr) const { vst1_f32((float*)ptr, fVec); }
36
invert()37 AI SkNx invert() const {
38 float32x2_t est0 = vrecpe_f32(fVec),
39 est1 = vmul_f32(vrecps_f32(est0, fVec), est0);
40 return est1;
41 }
42
43 AI SkNx operator + (const SkNx& o) const { return vadd_f32(fVec, o.fVec); }
44 AI SkNx operator - (const SkNx& o) const { return vsub_f32(fVec, o.fVec); }
45 AI SkNx operator * (const SkNx& o) const { return vmul_f32(fVec, o.fVec); }
46 AI SkNx operator / (const SkNx& o) const {
47 #if defined(SK_CPU_ARM64)
48 return vdiv_f32(fVec, o.fVec);
49 #else
50 float32x2_t est0 = vrecpe_f32(o.fVec),
51 est1 = vmul_f32(vrecps_f32(est0, o.fVec), est0),
52 est2 = vmul_f32(vrecps_f32(est1, o.fVec), est1);
53 return vmul_f32(fVec, est2);
54 #endif
55 }
56
57 AI SkNx operator==(const SkNx& o) const { return vreinterpret_f32_u32(vceq_f32(fVec, o.fVec)); }
58 AI SkNx operator <(const SkNx& o) const { return vreinterpret_f32_u32(vclt_f32(fVec, o.fVec)); }
59 AI SkNx operator >(const SkNx& o) const { return vreinterpret_f32_u32(vcgt_f32(fVec, o.fVec)); }
60 AI SkNx operator<=(const SkNx& o) const { return vreinterpret_f32_u32(vcle_f32(fVec, o.fVec)); }
61 AI SkNx operator>=(const SkNx& o) const { return vreinterpret_f32_u32(vcge_f32(fVec, o.fVec)); }
62 AI SkNx operator!=(const SkNx& o) const {
63 return vreinterpret_f32_u32(vmvn_u32(vceq_f32(fVec, o.fVec)));
64 }
65
Min(const SkNx & l,const SkNx & r)66 AI static SkNx Min(const SkNx& l, const SkNx& r) { return vmin_f32(l.fVec, r.fVec); }
Max(const SkNx & l,const SkNx & r)67 AI static SkNx Max(const SkNx& l, const SkNx& r) { return vmax_f32(l.fVec, r.fVec); }
68
rsqrt()69 AI SkNx rsqrt() const {
70 float32x2_t est0 = vrsqrte_f32(fVec);
71 return vmul_f32(vrsqrts_f32(fVec, vmul_f32(est0, est0)), est0);
72 }
73
sqrt()74 AI SkNx sqrt() const {
75 #if defined(SK_CPU_ARM64)
76 return vsqrt_f32(fVec);
77 #else
78 float32x2_t est0 = vrsqrte_f32(fVec),
79 est1 = vmul_f32(vrsqrts_f32(fVec, vmul_f32(est0, est0)), est0),
80 est2 = vmul_f32(vrsqrts_f32(fVec, vmul_f32(est1, est1)), est1);
81 return vmul_f32(fVec, est2);
82 #endif
83 }
84
85 AI float operator[](int k) const {
86 SkASSERT(0 <= k && k < 2);
87 union { float32x2_t v; float fs[2]; } pun = {fVec};
88 return pun.fs[k&1];
89 }
90
allTrue()91 AI bool allTrue() const {
92 auto v = vreinterpret_u32_f32(fVec);
93 return vget_lane_u32(v,0) && vget_lane_u32(v,1);
94 }
anyTrue()95 AI bool anyTrue() const {
96 auto v = vreinterpret_u32_f32(fVec);
97 return vget_lane_u32(v,0) || vget_lane_u32(v,1);
98 }
99
100 float32x2_t fVec;
101 };
102
103 template <>
104 class SkNx<4, float> {
105 public:
SkNx(float32x4_t vec)106 AI SkNx(float32x4_t vec) : fVec(vec) {}
107
SkNx()108 AI SkNx() {}
SkNx(float val)109 AI SkNx(float val) : fVec(vdupq_n_f32(val)) {}
SkNx(float a,float b,float c,float d)110 AI SkNx(float a, float b, float c, float d) { fVec = (float32x4_t) { a, b, c, d }; }
111
Load(const void * ptr)112 AI static SkNx Load(const void* ptr) { return vld1q_f32((const float*)ptr); }
store(void * ptr)113 AI void store(void* ptr) const { vst1q_f32((float*)ptr, fVec); }
114
Load4(const void * ptr,SkNx * r,SkNx * g,SkNx * b,SkNx * a)115 AI static void Load4(const void* ptr, SkNx* r, SkNx* g, SkNx* b, SkNx* a) {
116 float32x4x4_t rgba = vld4q_f32((const float*) ptr);
117 *r = rgba.val[0];
118 *g = rgba.val[1];
119 *b = rgba.val[2];
120 *a = rgba.val[3];
121 }
Store4(void * dst,const SkNx & r,const SkNx & g,const SkNx & b,const SkNx & a)122 AI static void Store4(void* dst, const SkNx& r, const SkNx& g, const SkNx& b, const SkNx& a) {
123 float32x4x4_t rgba = {{
124 r.fVec,
125 g.fVec,
126 b.fVec,
127 a.fVec,
128 }};
129 vst4q_f32((float*) dst, rgba);
130 }
131
invert()132 AI SkNx invert() const {
133 float32x4_t est0 = vrecpeq_f32(fVec),
134 est1 = vmulq_f32(vrecpsq_f32(est0, fVec), est0);
135 return est1;
136 }
137
138 AI SkNx operator + (const SkNx& o) const { return vaddq_f32(fVec, o.fVec); }
139 AI SkNx operator - (const SkNx& o) const { return vsubq_f32(fVec, o.fVec); }
140 AI SkNx operator * (const SkNx& o) const { return vmulq_f32(fVec, o.fVec); }
141 AI SkNx operator / (const SkNx& o) const {
142 #if defined(SK_CPU_ARM64)
143 return vdivq_f32(fVec, o.fVec);
144 #else
145 float32x4_t est0 = vrecpeq_f32(o.fVec),
146 est1 = vmulq_f32(vrecpsq_f32(est0, o.fVec), est0),
147 est2 = vmulq_f32(vrecpsq_f32(est1, o.fVec), est1);
148 return vmulq_f32(fVec, est2);
149 #endif
150 }
151
152 AI SkNx operator==(const SkNx& o) const {return vreinterpretq_f32_u32(vceqq_f32(fVec, o.fVec));}
153 AI SkNx operator <(const SkNx& o) const {return vreinterpretq_f32_u32(vcltq_f32(fVec, o.fVec));}
154 AI SkNx operator >(const SkNx& o) const {return vreinterpretq_f32_u32(vcgtq_f32(fVec, o.fVec));}
155 AI SkNx operator<=(const SkNx& o) const {return vreinterpretq_f32_u32(vcleq_f32(fVec, o.fVec));}
156 AI SkNx operator>=(const SkNx& o) const {return vreinterpretq_f32_u32(vcgeq_f32(fVec, o.fVec));}
157 AI SkNx operator!=(const SkNx& o) const {
158 return vreinterpretq_f32_u32(vmvnq_u32(vceqq_f32(fVec, o.fVec)));
159 }
160
Min(const SkNx & l,const SkNx & r)161 AI static SkNx Min(const SkNx& l, const SkNx& r) { return vminq_f32(l.fVec, r.fVec); }
Max(const SkNx & l,const SkNx & r)162 AI static SkNx Max(const SkNx& l, const SkNx& r) { return vmaxq_f32(l.fVec, r.fVec); }
163
abs()164 AI SkNx abs() const { return vabsq_f32(fVec); }
floor()165 AI SkNx floor() const {
166 #if defined(SK_CPU_ARM64)
167 return vrndmq_f32(fVec);
168 #else
169 return armv7_vrndmq_f32(fVec);
170 #endif
171 }
172
173
rsqrt()174 AI SkNx rsqrt() const {
175 float32x4_t est0 = vrsqrteq_f32(fVec);
176 return vmulq_f32(vrsqrtsq_f32(fVec, vmulq_f32(est0, est0)), est0);
177 }
178
sqrt()179 AI SkNx sqrt() const {
180 #if defined(SK_CPU_ARM64)
181 return vsqrtq_f32(fVec);
182 #else
183 float32x4_t est0 = vrsqrteq_f32(fVec),
184 est1 = vmulq_f32(vrsqrtsq_f32(fVec, vmulq_f32(est0, est0)), est0),
185 est2 = vmulq_f32(vrsqrtsq_f32(fVec, vmulq_f32(est1, est1)), est1);
186 return vmulq_f32(fVec, est2);
187 #endif
188 }
189
190 AI float operator[](int k) const {
191 SkASSERT(0 <= k && k < 4);
192 union { float32x4_t v; float fs[4]; } pun = {fVec};
193 return pun.fs[k&3];
194 }
195
allTrue()196 AI bool allTrue() const {
197 auto v = vreinterpretq_u32_f32(fVec);
198 return vgetq_lane_u32(v,0) && vgetq_lane_u32(v,1)
199 && vgetq_lane_u32(v,2) && vgetq_lane_u32(v,3);
200 }
anyTrue()201 AI bool anyTrue() const {
202 auto v = vreinterpretq_u32_f32(fVec);
203 return vgetq_lane_u32(v,0) || vgetq_lane_u32(v,1)
204 || vgetq_lane_u32(v,2) || vgetq_lane_u32(v,3);
205 }
206
thenElse(const SkNx & t,const SkNx & e)207 AI SkNx thenElse(const SkNx& t, const SkNx& e) const {
208 return vbslq_f32(vreinterpretq_u32_f32(fVec), t.fVec, e.fVec);
209 }
210
211 float32x4_t fVec;
212 };
213
214 #if defined(SK_CPU_ARM64)
SkNx_fma(const Sk4f & f,const Sk4f & m,const Sk4f & a)215 AI static Sk4f SkNx_fma(const Sk4f& f, const Sk4f& m, const Sk4f& a) {
216 return vfmaq_f32(a.fVec, f.fVec, m.fVec);
217 }
218 #endif
219
220 // It's possible that for our current use cases, representing this as
221 // half a uint16x8_t might be better than representing it as a uint16x4_t.
222 // It'd make conversion to Sk4b one step simpler.
223 template <>
224 class SkNx<4, uint16_t> {
225 public:
SkNx(const uint16x4_t & vec)226 AI SkNx(const uint16x4_t& vec) : fVec(vec) {}
227
SkNx()228 AI SkNx() {}
SkNx(uint16_t val)229 AI SkNx(uint16_t val) : fVec(vdup_n_u16(val)) {}
SkNx(uint16_t a,uint16_t b,uint16_t c,uint16_t d)230 AI SkNx(uint16_t a, uint16_t b, uint16_t c, uint16_t d) {
231 fVec = (uint16x4_t) { a,b,c,d };
232 }
233
Load(const void * ptr)234 AI static SkNx Load(const void* ptr) { return vld1_u16((const uint16_t*)ptr); }
store(void * ptr)235 AI void store(void* ptr) const { vst1_u16((uint16_t*)ptr, fVec); }
236
Load4(const void * ptr,SkNx * r,SkNx * g,SkNx * b,SkNx * a)237 AI static void Load4(const void* ptr, SkNx* r, SkNx* g, SkNx* b, SkNx* a) {
238 uint16x4x4_t rgba = vld4_u16((const uint16_t*)ptr);
239 *r = rgba.val[0];
240 *g = rgba.val[1];
241 *b = rgba.val[2];
242 *a = rgba.val[3];
243 }
Load3(const void * ptr,SkNx * r,SkNx * g,SkNx * b)244 AI static void Load3(const void* ptr, SkNx* r, SkNx* g, SkNx* b) {
245 uint16x4x3_t rgba = vld3_u16((const uint16_t*)ptr);
246 *r = rgba.val[0];
247 *g = rgba.val[1];
248 *b = rgba.val[2];
249 }
Store4(void * dst,const SkNx & r,const SkNx & g,const SkNx & b,const SkNx & a)250 AI static void Store4(void* dst, const SkNx& r, const SkNx& g, const SkNx& b, const SkNx& a) {
251 uint16x4x4_t rgba = {{
252 r.fVec,
253 g.fVec,
254 b.fVec,
255 a.fVec,
256 }};
257 vst4_u16((uint16_t*) dst, rgba);
258 }
259
260 AI SkNx operator + (const SkNx& o) const { return vadd_u16(fVec, o.fVec); }
261 AI SkNx operator - (const SkNx& o) const { return vsub_u16(fVec, o.fVec); }
262 AI SkNx operator * (const SkNx& o) const { return vmul_u16(fVec, o.fVec); }
263 AI SkNx operator & (const SkNx& o) const { return vand_u16(fVec, o.fVec); }
264 AI SkNx operator | (const SkNx& o) const { return vorr_u16(fVec, o.fVec); }
265
266 AI SkNx operator << (int bits) const { return fVec << SkNx(bits).fVec; }
267 AI SkNx operator >> (int bits) const { return fVec >> SkNx(bits).fVec; }
268
Min(const SkNx & a,const SkNx & b)269 AI static SkNx Min(const SkNx& a, const SkNx& b) { return vmin_u16(a.fVec, b.fVec); }
270
271 AI uint16_t operator[](int k) const {
272 SkASSERT(0 <= k && k < 4);
273 union { uint16x4_t v; uint16_t us[4]; } pun = {fVec};
274 return pun.us[k&3];
275 }
276
thenElse(const SkNx & t,const SkNx & e)277 AI SkNx thenElse(const SkNx& t, const SkNx& e) const {
278 return vbsl_u16(fVec, t.fVec, e.fVec);
279 }
280
281 uint16x4_t fVec;
282 };
283
284 template <>
285 class SkNx<8, uint16_t> {
286 public:
SkNx(const uint16x8_t & vec)287 AI SkNx(const uint16x8_t& vec) : fVec(vec) {}
288
SkNx()289 AI SkNx() {}
SkNx(uint16_t val)290 AI SkNx(uint16_t val) : fVec(vdupq_n_u16(val)) {}
Load(const void * ptr)291 AI static SkNx Load(const void* ptr) { return vld1q_u16((const uint16_t*)ptr); }
292
SkNx(uint16_t a,uint16_t b,uint16_t c,uint16_t d,uint16_t e,uint16_t f,uint16_t g,uint16_t h)293 AI SkNx(uint16_t a, uint16_t b, uint16_t c, uint16_t d,
294 uint16_t e, uint16_t f, uint16_t g, uint16_t h) {
295 fVec = (uint16x8_t) { a,b,c,d, e,f,g,h };
296 }
297
store(void * ptr)298 AI void store(void* ptr) const { vst1q_u16((uint16_t*)ptr, fVec); }
299
300 AI SkNx operator + (const SkNx& o) const { return vaddq_u16(fVec, o.fVec); }
301 AI SkNx operator - (const SkNx& o) const { return vsubq_u16(fVec, o.fVec); }
302 AI SkNx operator * (const SkNx& o) const { return vmulq_u16(fVec, o.fVec); }
303 AI SkNx operator & (const SkNx& o) const { return vandq_u16(fVec, o.fVec); }
304 AI SkNx operator | (const SkNx& o) const { return vorrq_u16(fVec, o.fVec); }
305
306 AI SkNx operator << (int bits) const { return fVec << SkNx(bits).fVec; }
307 AI SkNx operator >> (int bits) const { return fVec >> SkNx(bits).fVec; }
308
Min(const SkNx & a,const SkNx & b)309 AI static SkNx Min(const SkNx& a, const SkNx& b) { return vminq_u16(a.fVec, b.fVec); }
310
311 AI uint16_t operator[](int k) const {
312 SkASSERT(0 <= k && k < 8);
313 union { uint16x8_t v; uint16_t us[8]; } pun = {fVec};
314 return pun.us[k&7];
315 }
316
thenElse(const SkNx & t,const SkNx & e)317 AI SkNx thenElse(const SkNx& t, const SkNx& e) const {
318 return vbslq_u16(fVec, t.fVec, e.fVec);
319 }
320
321 uint16x8_t fVec;
322 };
323
324 template <>
325 class SkNx<4, uint8_t> {
326 public:
327 typedef uint32_t __attribute__((aligned(1))) unaligned_uint32_t;
328
SkNx(const uint8x8_t & vec)329 AI SkNx(const uint8x8_t& vec) : fVec(vec) {}
330
SkNx()331 AI SkNx() {}
SkNx(uint8_t a,uint8_t b,uint8_t c,uint8_t d)332 AI SkNx(uint8_t a, uint8_t b, uint8_t c, uint8_t d) {
333 fVec = (uint8x8_t){a,b,c,d, 0,0,0,0};
334 }
Load(const void * ptr)335 AI static SkNx Load(const void* ptr) {
336 return (uint8x8_t)vld1_dup_u32((const unaligned_uint32_t*)ptr);
337 }
store(void * ptr)338 AI void store(void* ptr) const {
339 return vst1_lane_u32((unaligned_uint32_t*)ptr, (uint32x2_t)fVec, 0);
340 }
341 AI uint8_t operator[](int k) const {
342 SkASSERT(0 <= k && k < 4);
343 union { uint8x8_t v; uint8_t us[8]; } pun = {fVec};
344 return pun.us[k&3];
345 }
346
347 // TODO as needed
348
349 uint8x8_t fVec;
350 };
351
352 template <>
353 class SkNx<16, uint8_t> {
354 public:
SkNx(const uint8x16_t & vec)355 AI SkNx(const uint8x16_t& vec) : fVec(vec) {}
356
SkNx()357 AI SkNx() {}
SkNx(uint8_t val)358 AI SkNx(uint8_t val) : fVec(vdupq_n_u8(val)) {}
SkNx(uint8_t a,uint8_t b,uint8_t c,uint8_t d,uint8_t e,uint8_t f,uint8_t g,uint8_t h,uint8_t i,uint8_t j,uint8_t k,uint8_t l,uint8_t m,uint8_t n,uint8_t o,uint8_t p)359 AI SkNx(uint8_t a, uint8_t b, uint8_t c, uint8_t d,
360 uint8_t e, uint8_t f, uint8_t g, uint8_t h,
361 uint8_t i, uint8_t j, uint8_t k, uint8_t l,
362 uint8_t m, uint8_t n, uint8_t o, uint8_t p) {
363 fVec = (uint8x16_t) { a,b,c,d, e,f,g,h, i,j,k,l, m,n,o,p };
364 }
365
Load(const void * ptr)366 AI static SkNx Load(const void* ptr) { return vld1q_u8((const uint8_t*)ptr); }
store(void * ptr)367 AI void store(void* ptr) const { vst1q_u8((uint8_t*)ptr, fVec); }
368
saturatedAdd(const SkNx & o)369 AI SkNx saturatedAdd(const SkNx& o) const { return vqaddq_u8(fVec, o.fVec); }
370
371 AI SkNx operator + (const SkNx& o) const { return vaddq_u8(fVec, o.fVec); }
372 AI SkNx operator - (const SkNx& o) const { return vsubq_u8(fVec, o.fVec); }
373
Min(const SkNx & a,const SkNx & b)374 AI static SkNx Min(const SkNx& a, const SkNx& b) { return vminq_u8(a.fVec, b.fVec); }
375 AI SkNx operator < (const SkNx& o) const { return vcltq_u8(fVec, o.fVec); }
376
377 AI uint8_t operator[](int k) const {
378 SkASSERT(0 <= k && k < 16);
379 union { uint8x16_t v; uint8_t us[16]; } pun = {fVec};
380 return pun.us[k&15];
381 }
382
thenElse(const SkNx & t,const SkNx & e)383 AI SkNx thenElse(const SkNx& t, const SkNx& e) const {
384 return vbslq_u8(fVec, t.fVec, e.fVec);
385 }
386
387 uint8x16_t fVec;
388 };
389
390 template <>
391 class SkNx<4, int32_t> {
392 public:
SkNx(const int32x4_t & vec)393 AI SkNx(const int32x4_t& vec) : fVec(vec) {}
394
SkNx()395 AI SkNx() {}
SkNx(int32_t v)396 AI SkNx(int32_t v) {
397 fVec = vdupq_n_s32(v);
398 }
SkNx(int32_t a,int32_t b,int32_t c,int32_t d)399 AI SkNx(int32_t a, int32_t b, int32_t c, int32_t d) {
400 fVec = (int32x4_t){a,b,c,d};
401 }
Load(const void * ptr)402 AI static SkNx Load(const void* ptr) {
403 return vld1q_s32((const int32_t*)ptr);
404 }
store(void * ptr)405 AI void store(void* ptr) const {
406 return vst1q_s32((int32_t*)ptr, fVec);
407 }
408 AI int32_t operator[](int k) const {
409 SkASSERT(0 <= k && k < 4);
410 union { int32x4_t v; int32_t is[4]; } pun = {fVec};
411 return pun.is[k&3];
412 }
413
414 AI SkNx operator + (const SkNx& o) const { return vaddq_s32(fVec, o.fVec); }
415 AI SkNx operator - (const SkNx& o) const { return vsubq_s32(fVec, o.fVec); }
416 AI SkNx operator * (const SkNx& o) const { return vmulq_s32(fVec, o.fVec); }
417
418 AI SkNx operator & (const SkNx& o) const { return vandq_s32(fVec, o.fVec); }
419 AI SkNx operator | (const SkNx& o) const { return vorrq_s32(fVec, o.fVec); }
420 AI SkNx operator ^ (const SkNx& o) const { return veorq_s32(fVec, o.fVec); }
421
422 AI SkNx operator << (int bits) const { return fVec << SkNx(bits).fVec; }
423 AI SkNx operator >> (int bits) const { return fVec >> SkNx(bits).fVec; }
424
425 AI SkNx operator == (const SkNx& o) const {
426 return vreinterpretq_s32_u32(vceqq_s32(fVec, o.fVec));
427 }
428 AI SkNx operator < (const SkNx& o) const {
429 return vreinterpretq_s32_u32(vcltq_s32(fVec, o.fVec));
430 }
431 AI SkNx operator > (const SkNx& o) const {
432 return vreinterpretq_s32_u32(vcgtq_s32(fVec, o.fVec));
433 }
434
Min(const SkNx & a,const SkNx & b)435 AI static SkNx Min(const SkNx& a, const SkNx& b) { return vminq_s32(a.fVec, b.fVec); }
Max(const SkNx & a,const SkNx & b)436 AI static SkNx Max(const SkNx& a, const SkNx& b) { return vmaxq_s32(a.fVec, b.fVec); }
437 // TODO as needed
438
thenElse(const SkNx & t,const SkNx & e)439 AI SkNx thenElse(const SkNx& t, const SkNx& e) const {
440 return vbslq_s32(vreinterpretq_u32_s32(fVec), t.fVec, e.fVec);
441 }
442
abs()443 AI SkNx abs() const { return vabsq_s32(fVec); }
444
445 int32x4_t fVec;
446 };
447
448 template <>
449 class SkNx<4, uint32_t> {
450 public:
SkNx(const uint32x4_t & vec)451 AI SkNx(const uint32x4_t& vec) : fVec(vec) {}
452
SkNx()453 AI SkNx() {}
SkNx(uint32_t v)454 AI SkNx(uint32_t v) {
455 fVec = vdupq_n_u32(v);
456 }
SkNx(uint32_t a,uint32_t b,uint32_t c,uint32_t d)457 AI SkNx(uint32_t a, uint32_t b, uint32_t c, uint32_t d) {
458 fVec = (uint32x4_t){a,b,c,d};
459 }
Load(const void * ptr)460 AI static SkNx Load(const void* ptr) {
461 return vld1q_u32((const uint32_t*)ptr);
462 }
store(void * ptr)463 AI void store(void* ptr) const {
464 return vst1q_u32((uint32_t*)ptr, fVec);
465 }
466 AI uint32_t operator[](int k) const {
467 SkASSERT(0 <= k && k < 4);
468 union { uint32x4_t v; uint32_t us[4]; } pun = {fVec};
469 return pun.us[k&3];
470 }
471
472 AI SkNx operator + (const SkNx& o) const { return vaddq_u32(fVec, o.fVec); }
473 AI SkNx operator - (const SkNx& o) const { return vsubq_u32(fVec, o.fVec); }
474 AI SkNx operator * (const SkNx& o) const { return vmulq_u32(fVec, o.fVec); }
475
476 AI SkNx operator & (const SkNx& o) const { return vandq_u32(fVec, o.fVec); }
477 AI SkNx operator | (const SkNx& o) const { return vorrq_u32(fVec, o.fVec); }
478 AI SkNx operator ^ (const SkNx& o) const { return veorq_u32(fVec, o.fVec); }
479
480 AI SkNx operator << (int bits) const { return fVec << SkNx(bits).fVec; }
481 AI SkNx operator >> (int bits) const { return fVec >> SkNx(bits).fVec; }
482
483 AI SkNx operator == (const SkNx& o) const { return vceqq_u32(fVec, o.fVec); }
484 AI SkNx operator < (const SkNx& o) const { return vcltq_u32(fVec, o.fVec); }
485 AI SkNx operator > (const SkNx& o) const { return vcgtq_u32(fVec, o.fVec); }
486
Min(const SkNx & a,const SkNx & b)487 AI static SkNx Min(const SkNx& a, const SkNx& b) { return vminq_u32(a.fVec, b.fVec); }
488 // TODO as needed
489
thenElse(const SkNx & t,const SkNx & e)490 AI SkNx thenElse(const SkNx& t, const SkNx& e) const {
491 return vbslq_u32(fVec, t.fVec, e.fVec);
492 }
493
494 uint32x4_t fVec;
495 };
496
497 template<> AI /*static*/ Sk4i SkNx_cast<int32_t, float>(const Sk4f& src) {
498 return vcvtq_s32_f32(src.fVec);
499
500 }
501 template<> AI /*static*/ Sk4f SkNx_cast<float, int32_t>(const Sk4i& src) {
502 return vcvtq_f32_s32(src.fVec);
503 }
504 template<> AI /*static*/ Sk4f SkNx_cast<float, uint32_t>(const Sk4u& src) {
505 return SkNx_cast<float>(Sk4i::Load(&src));
506 }
507
508 template<> AI /*static*/ Sk4h SkNx_cast<uint16_t, float>(const Sk4f& src) {
509 return vqmovn_u32(vcvtq_u32_f32(src.fVec));
510 }
511
512 template<> AI /*static*/ Sk4f SkNx_cast<float, uint16_t>(const Sk4h& src) {
513 return vcvtq_f32_u32(vmovl_u16(src.fVec));
514 }
515
516 template<> AI /*static*/ Sk4b SkNx_cast<uint8_t, float>(const Sk4f& src) {
517 uint32x4_t _32 = vcvtq_u32_f32(src.fVec);
518 uint16x4_t _16 = vqmovn_u32(_32);
519 return vqmovn_u16(vcombine_u16(_16, _16));
520 }
521
522 template<> AI /*static*/ Sk4i SkNx_cast<int32_t, uint8_t>(const Sk4b& src) {
523 uint16x8_t _16 = vmovl_u8(src.fVec);
524 return vreinterpretq_s32_u32(vmovl_u16(vget_low_u16(_16)));
525 }
526
527 template<> AI /*static*/ Sk4f SkNx_cast<float, uint8_t>(const Sk4b& src) {
528 return vcvtq_f32_s32(SkNx_cast<int32_t>(src).fVec);
529 }
530
531 template<> AI /*static*/ Sk16b SkNx_cast<uint8_t, float>(const Sk16f& src) {
532 Sk8f ab, cd;
533 SkNx_split(src, &ab, &cd);
534
535 Sk4f a,b,c,d;
536 SkNx_split(ab, &a, &b);
537 SkNx_split(cd, &c, &d);
538 return vuzpq_u8(vuzpq_u8((uint8x16_t)vcvtq_u32_f32(a.fVec),
539 (uint8x16_t)vcvtq_u32_f32(b.fVec)).val[0],
540 vuzpq_u8((uint8x16_t)vcvtq_u32_f32(c.fVec),
541 (uint8x16_t)vcvtq_u32_f32(d.fVec)).val[0]).val[0];
542 }
543
544 template<> AI /*static*/ Sk4h SkNx_cast<uint16_t, uint8_t>(const Sk4b& src) {
545 return vget_low_u16(vmovl_u8(src.fVec));
546 }
547
548 template<> AI /*static*/ Sk4b SkNx_cast<uint8_t, uint16_t>(const Sk4h& src) {
549 return vmovn_u16(vcombine_u16(src.fVec, src.fVec));
550 }
551
552 template<> AI /*static*/ Sk4b SkNx_cast<uint8_t, int32_t>(const Sk4i& src) {
553 uint16x4_t _16 = vqmovun_s32(src.fVec);
554 return vqmovn_u16(vcombine_u16(_16, _16));
555 }
556
557 template<> AI /*static*/ Sk4i SkNx_cast<int32_t, uint16_t>(const Sk4h& src) {
558 return vreinterpretq_s32_u32(vmovl_u16(src.fVec));
559 }
560
561 template<> AI /*static*/ Sk4h SkNx_cast<uint16_t, int32_t>(const Sk4i& src) {
562 return vmovn_u32(vreinterpretq_u32_s32(src.fVec));
563 }
564
565 template<> AI /*static*/ Sk4i SkNx_cast<int32_t, uint32_t>(const Sk4u& src) {
566 return vreinterpretq_s32_u32(src.fVec);
567 }
568
Sk4f_round(const Sk4f & x)569 AI static Sk4i Sk4f_round(const Sk4f& x) {
570 return vcvtq_s32_f32((x + 0.5f).fVec);
571 }
572
573 } // namespace
574
575 #endif//SkNx_neon_DEFINED
576