1 // SPDX-License-Identifier: Apache-2.0
2 // ----------------------------------------------------------------------------
3 // Copyright 2019-2022 Arm Limited
4 //
5 // Licensed under the Apache License, Version 2.0 (the "License"); you may not
6 // use this file except in compliance with the License. You may obtain a copy
7 // of the License at:
8 //
9 // http://www.apache.org/licenses/LICENSE-2.0
10 //
11 // Unless required by applicable law or agreed to in writing, software
12 // distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13 // WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14 // License for the specific language governing permissions and limitations
15 // under the License.
16 // ----------------------------------------------------------------------------
17
18 /**
19 * @brief 4x32-bit vectors, implemented using SSE.
20 *
21 * This module implements 4-wide 32-bit float, int, and mask vectors for x86
22 * SSE. The implementation requires at least SSE2, but higher levels of SSE can
23 * be selected at compile time to improve performance.
24 *
25 * There is a baseline level of functionality provided by all vector widths and
26 * implementations. This is implemented using identical function signatures,
27 * modulo data type, so we can use them as substitutable implementations in VLA
28 * code.
29 *
30 * The 4-wide vectors are also used as a fixed-width type, and significantly
31 * extend the functionality above that available to VLA code.
32 */
33
34 #ifndef ASTC_VECMATHLIB_SSE_4_H_INCLUDED
35 #define ASTC_VECMATHLIB_SSE_4_H_INCLUDED
36
37 #ifndef ASTCENC_SIMD_INLINE
38 #error "Include astcenc_vecmathlib.h, do not include directly"
39 #endif
40
41 #include <cstdio>
42
43 // ============================================================================
44 // vfloat4 data type
45 // ============================================================================
46
47 /**
48 * @brief Data type for 4-wide floats.
49 */
50 struct vfloat4
51 {
52 /**
53 * @brief Construct from zero-initialized value.
54 */
55 ASTCENC_SIMD_INLINE vfloat4() = default;
56
57 /**
58 * @brief Construct from 4 values loaded from an unaligned address.
59 *
60 * Consider using loada() which is better with vectors if data is aligned
61 * to vector length.
62 */
vfloat4vfloat463 ASTCENC_SIMD_INLINE explicit vfloat4(const float *p)
64 {
65 m = _mm_loadu_ps(p);
66 }
67
68 /**
69 * @brief Construct from 1 scalar value replicated across all lanes.
70 *
71 * Consider using zero() for constexpr zeros.
72 */
vfloat4vfloat473 ASTCENC_SIMD_INLINE explicit vfloat4(float a)
74 {
75 m = _mm_set1_ps(a);
76 }
77
78 /**
79 * @brief Construct from 4 scalar values.
80 *
81 * The value of @c a is stored to lane 0 (LSB) in the SIMD register.
82 */
vfloat4vfloat483 ASTCENC_SIMD_INLINE explicit vfloat4(float a, float b, float c, float d)
84 {
85 m = _mm_set_ps(d, c, b, a);
86 }
87
88 /**
89 * @brief Construct from an existing SIMD register.
90 */
vfloat4vfloat491 ASTCENC_SIMD_INLINE explicit vfloat4(__m128 a)
92 {
93 m = a;
94 }
95
96 /**
97 * @brief Get the scalar value of a single lane.
98 */
lanevfloat499 template <int l> ASTCENC_SIMD_INLINE float lane() const
100 {
101 return _mm_cvtss_f32(_mm_shuffle_ps(m, m, l));
102 }
103
104 /**
105 * @brief Set the scalar value of a single lane.
106 */
set_lanevfloat4107 template <int l> ASTCENC_SIMD_INLINE void set_lane(float a)
108 {
109 #if ASTCENC_SSE >= 41
110 __m128 v = _mm_set1_ps(a);
111 m = _mm_insert_ps(m, v, l << 6 | l << 4);
112 #else
113 alignas(16) float idx[4];
114 _mm_store_ps(idx, m);
115 idx[l] = a;
116 m = _mm_load_ps(idx);
117 #endif
118 }
119
120 /**
121 * @brief Factory that returns a vector of zeros.
122 */
zerovfloat4123 static ASTCENC_SIMD_INLINE vfloat4 zero()
124 {
125 return vfloat4(_mm_setzero_ps());
126 }
127
128 /**
129 * @brief Factory that returns a replicated scalar loaded from memory.
130 */
load1vfloat4131 static ASTCENC_SIMD_INLINE vfloat4 load1(const float* p)
132 {
133 return vfloat4(_mm_load_ps1(p));
134 }
135
136 /**
137 * @brief Factory that returns a vector loaded from 16B aligned memory.
138 */
loadavfloat4139 static ASTCENC_SIMD_INLINE vfloat4 loada(const float* p)
140 {
141 return vfloat4(_mm_load_ps(p));
142 }
143
144 /**
145 * @brief Factory that returns a vector containing the lane IDs.
146 */
lane_idvfloat4147 static ASTCENC_SIMD_INLINE vfloat4 lane_id()
148 {
149 return vfloat4(_mm_set_ps(3, 2, 1, 0));
150 }
151
152 /**
153 * @brief Return a swizzled float 2.
154 */
swzvfloat4155 template <int l0, int l1> ASTCENC_SIMD_INLINE vfloat4 swz() const
156 {
157 vfloat4 result(_mm_shuffle_ps(m, m, l0 | l1 << 2));
158 result.set_lane<2>(0.0f);
159 result.set_lane<3>(0.0f);
160 return result;
161 }
162
163 /**
164 * @brief Return a swizzled float 3.
165 */
swzvfloat4166 template <int l0, int l1, int l2> ASTCENC_SIMD_INLINE vfloat4 swz() const
167 {
168 vfloat4 result(_mm_shuffle_ps(m, m, l0 | l1 << 2 | l2 << 4));
169 result.set_lane<3>(0.0f);
170 return result;
171 }
172
173 /**
174 * @brief Return a swizzled float 4.
175 */
swzvfloat4176 template <int l0, int l1, int l2, int l3> ASTCENC_SIMD_INLINE vfloat4 swz() const
177 {
178 return vfloat4(_mm_shuffle_ps(m, m, l0 | l1 << 2 | l2 << 4 | l3 << 6));
179 }
180
181 /**
182 * @brief The vector ...
183 */
184 __m128 m;
185 };
186
187 // ============================================================================
188 // vint4 data type
189 // ============================================================================
190
191 /**
192 * @brief Data type for 4-wide ints.
193 */
194 struct vint4
195 {
196 /**
197 * @brief Construct from zero-initialized value.
198 */
199 ASTCENC_SIMD_INLINE vint4() = default;
200
201 /**
202 * @brief Construct from 4 values loaded from an unaligned address.
203 *
204 * Consider using loada() which is better with vectors if data is aligned
205 * to vector length.
206 */
vint4vint4207 ASTCENC_SIMD_INLINE explicit vint4(const int *p)
208 {
209 m = _mm_loadu_si128(reinterpret_cast<const __m128i*>(p));
210 }
211
212 /**
213 * @brief Construct from 4 uint8_t loaded from an unaligned address.
214 */
vint4vint4215 ASTCENC_SIMD_INLINE explicit vint4(const uint8_t *p)
216 {
217 // _mm_loadu_si32 would be nicer syntax, but missing on older GCC
218 __m128i t = _mm_cvtsi32_si128(*reinterpret_cast<const int*>(p));
219
220 #if ASTCENC_SSE >= 41
221 m = _mm_cvtepu8_epi32(t);
222 #else
223 t = _mm_unpacklo_epi8(t, _mm_setzero_si128());
224 m = _mm_unpacklo_epi16(t, _mm_setzero_si128());
225 #endif
226 }
227
228 /**
229 * @brief Construct from 1 scalar value replicated across all lanes.
230 *
231 * Consider using vfloat4::zero() for constexpr zeros.
232 */
vint4vint4233 ASTCENC_SIMD_INLINE explicit vint4(int a)
234 {
235 m = _mm_set1_epi32(a);
236 }
237
238 /**
239 * @brief Construct from 4 scalar values.
240 *
241 * The value of @c a is stored to lane 0 (LSB) in the SIMD register.
242 */
vint4vint4243 ASTCENC_SIMD_INLINE explicit vint4(int a, int b, int c, int d)
244 {
245 m = _mm_set_epi32(d, c, b, a);
246 }
247
248 /**
249 * @brief Construct from an existing SIMD register.
250 */
vint4vint4251 ASTCENC_SIMD_INLINE explicit vint4(__m128i a)
252 {
253 m = a;
254 }
255
256 /**
257 * @brief Get the scalar from a single lane.
258 */
lanevint4259 template <int l> ASTCENC_SIMD_INLINE int lane() const
260 {
261 return _mm_cvtsi128_si32(_mm_shuffle_epi32(m, l));
262 }
263
264 /**
265 * @brief Set the scalar value of a single lane.
266 */
set_lanevint4267 template <int l> ASTCENC_SIMD_INLINE void set_lane(int a)
268 {
269 #if ASTCENC_SSE >= 41
270 m = _mm_insert_epi32(m, a, l);
271 #else
272 alignas(16) int idx[4];
273 _mm_store_si128(reinterpret_cast<__m128i*>(idx), m);
274 idx[l] = a;
275 m = _mm_load_si128(reinterpret_cast<const __m128i*>(idx));
276 #endif
277 }
278
279 /**
280 * @brief Factory that returns a vector of zeros.
281 */
zerovint4282 static ASTCENC_SIMD_INLINE vint4 zero()
283 {
284 return vint4(_mm_setzero_si128());
285 }
286
287 /**
288 * @brief Factory that returns a replicated scalar loaded from memory.
289 */
load1vint4290 static ASTCENC_SIMD_INLINE vint4 load1(const int* p)
291 {
292 return vint4(*p);
293 }
294
295 /**
296 * @brief Factory that returns a vector loaded from 16B aligned memory.
297 */
loadavint4298 static ASTCENC_SIMD_INLINE vint4 loada(const int* p)
299 {
300 return vint4(_mm_load_si128(reinterpret_cast<const __m128i*>(p)));
301 }
302
303 /**
304 * @brief Factory that returns a vector containing the lane IDs.
305 */
lane_idvint4306 static ASTCENC_SIMD_INLINE vint4 lane_id()
307 {
308 return vint4(_mm_set_epi32(3, 2, 1, 0));
309 }
310
311 /**
312 * @brief The vector ...
313 */
314 __m128i m;
315 };
316
317 // ============================================================================
318 // vmask4 data type
319 // ============================================================================
320
321 /**
322 * @brief Data type for 4-wide control plane masks.
323 */
324 struct vmask4
325 {
326 /**
327 * @brief Construct from an existing SIMD register.
328 */
vmask4vmask4329 ASTCENC_SIMD_INLINE explicit vmask4(__m128 a)
330 {
331 m = a;
332 }
333
334 /**
335 * @brief Construct from an existing SIMD register.
336 */
vmask4vmask4337 ASTCENC_SIMD_INLINE explicit vmask4(__m128i a)
338 {
339 m = _mm_castsi128_ps(a);
340 }
341
342 /**
343 * @brief Construct from 1 scalar value.
344 */
vmask4vmask4345 ASTCENC_SIMD_INLINE explicit vmask4(bool a)
346 {
347 vint4 mask(a == false ? 0 : -1);
348 m = _mm_castsi128_ps(mask.m);
349 }
350
351 /**
352 * @brief Construct from 4 scalar values.
353 *
354 * The value of @c a is stored to lane 0 (LSB) in the SIMD register.
355 */
vmask4vmask4356 ASTCENC_SIMD_INLINE explicit vmask4(bool a, bool b, bool c, bool d)
357 {
358 vint4 mask(a == false ? 0 : -1,
359 b == false ? 0 : -1,
360 c == false ? 0 : -1,
361 d == false ? 0 : -1);
362
363 m = _mm_castsi128_ps(mask.m);
364 }
365
366 /**
367 * @brief Get the scalar value of a single lane.
368 */
lanevmask4369 template <int l> ASTCENC_SIMD_INLINE float lane() const
370 {
371 return _mm_cvtss_f32(_mm_shuffle_ps(m, m, l));
372 }
373
374 /**
375 * @brief The vector ...
376 */
377 __m128 m;
378 };
379
380 // ============================================================================
381 // vmask4 operators and functions
382 // ============================================================================
383
384 /**
385 * @brief Overload: mask union (or).
386 */
387 ASTCENC_SIMD_INLINE vmask4 operator|(vmask4 a, vmask4 b)
388 {
389 return vmask4(_mm_or_ps(a.m, b.m));
390 }
391
392 /**
393 * @brief Overload: mask intersect (and).
394 */
395 ASTCENC_SIMD_INLINE vmask4 operator&(vmask4 a, vmask4 b)
396 {
397 return vmask4(_mm_and_ps(a.m, b.m));
398 }
399
400 /**
401 * @brief Overload: mask difference (xor).
402 */
403 ASTCENC_SIMD_INLINE vmask4 operator^(vmask4 a, vmask4 b)
404 {
405 return vmask4(_mm_xor_ps(a.m, b.m));
406 }
407
408 /**
409 * @brief Overload: mask invert (not).
410 */
411 ASTCENC_SIMD_INLINE vmask4 operator~(vmask4 a)
412 {
413 return vmask4(_mm_xor_si128(_mm_castps_si128(a.m), _mm_set1_epi32(-1)));
414 }
415
416 /**
417 * @brief Return a 4-bit mask code indicating mask status.
418 *
419 * bit0 = lane 0
420 */
mask(vmask4 a)421 ASTCENC_SIMD_INLINE unsigned int mask(vmask4 a)
422 {
423 return static_cast<unsigned int>(_mm_movemask_ps(a.m));
424 }
425
426 // ============================================================================
427 // vint4 operators and functions
428 // ============================================================================
429
430 /**
431 * @brief Overload: vector by vector addition.
432 */
433 ASTCENC_SIMD_INLINE vint4 operator+(vint4 a, vint4 b)
434 {
435 return vint4(_mm_add_epi32(a.m, b.m));
436 }
437
438 /**
439 * @brief Overload: vector by vector subtraction.
440 */
441 ASTCENC_SIMD_INLINE vint4 operator-(vint4 a, vint4 b)
442 {
443 return vint4(_mm_sub_epi32(a.m, b.m));
444 }
445
446 /**
447 * @brief Overload: vector by vector multiplication.
448 */
449 ASTCENC_SIMD_INLINE vint4 operator*(vint4 a, vint4 b)
450 {
451 #if ASTCENC_SSE >= 41
452 return vint4(_mm_mullo_epi32 (a.m, b.m));
453 #else
454 __m128i t1 = _mm_mul_epu32(a.m, b.m);
455 __m128i t2 = _mm_mul_epu32(
456 _mm_srli_si128(a.m, 4),
457 _mm_srli_si128(b.m, 4));
458 __m128i r = _mm_unpacklo_epi32(
459 _mm_shuffle_epi32(t1, _MM_SHUFFLE (0, 0, 2, 0)),
460 _mm_shuffle_epi32(t2, _MM_SHUFFLE (0, 0, 2, 0)));
461 return vint4(r);
462 #endif
463 }
464
465 /**
466 * @brief Overload: vector bit invert.
467 */
468 ASTCENC_SIMD_INLINE vint4 operator~(vint4 a)
469 {
470 return vint4(_mm_xor_si128(a.m, _mm_set1_epi32(-1)));
471 }
472
473 /**
474 * @brief Overload: vector by vector bitwise or.
475 */
476 ASTCENC_SIMD_INLINE vint4 operator|(vint4 a, vint4 b)
477 {
478 return vint4(_mm_or_si128(a.m, b.m));
479 }
480
481 /**
482 * @brief Overload: vector by vector bitwise and.
483 */
484 ASTCENC_SIMD_INLINE vint4 operator&(vint4 a, vint4 b)
485 {
486 return vint4(_mm_and_si128(a.m, b.m));
487 }
488
489 /**
490 * @brief Overload: vector by vector bitwise xor.
491 */
492 ASTCENC_SIMD_INLINE vint4 operator^(vint4 a, vint4 b)
493 {
494 return vint4(_mm_xor_si128(a.m, b.m));
495 }
496
497 /**
498 * @brief Overload: vector by vector equality.
499 */
500 ASTCENC_SIMD_INLINE vmask4 operator==(vint4 a, vint4 b)
501 {
502 return vmask4(_mm_cmpeq_epi32(a.m, b.m));
503 }
504
505 /**
506 * @brief Overload: vector by vector inequality.
507 */
508 ASTCENC_SIMD_INLINE vmask4 operator!=(vint4 a, vint4 b)
509 {
510 return ~vmask4(_mm_cmpeq_epi32(a.m, b.m));
511 }
512
513 /**
514 * @brief Overload: vector by vector less than.
515 */
516 ASTCENC_SIMD_INLINE vmask4 operator<(vint4 a, vint4 b)
517 {
518 return vmask4(_mm_cmplt_epi32(a.m, b.m));
519 }
520
521 /**
522 * @brief Overload: vector by vector greater than.
523 */
524 ASTCENC_SIMD_INLINE vmask4 operator>(vint4 a, vint4 b)
525 {
526 return vmask4(_mm_cmpgt_epi32(a.m, b.m));
527 }
528
529 /**
530 * @brief Logical shift left.
531 */
lsl(vint4 a)532 template <int s> ASTCENC_SIMD_INLINE vint4 lsl(vint4 a)
533 {
534 return vint4(_mm_slli_epi32(a.m, s));
535 }
536
537 /**
538 * @brief Logical shift right.
539 */
lsr(vint4 a)540 template <int s> ASTCENC_SIMD_INLINE vint4 lsr(vint4 a)
541 {
542 return vint4(_mm_srli_epi32(a.m, s));
543 }
544
545 /**
546 * @brief Arithmetic shift right.
547 */
asr(vint4 a)548 template <int s> ASTCENC_SIMD_INLINE vint4 asr(vint4 a)
549 {
550 return vint4(_mm_srai_epi32(a.m, s));
551 }
552
553 /**
554 * @brief Return the min vector of two vectors.
555 */
min(vint4 a,vint4 b)556 ASTCENC_SIMD_INLINE vint4 min(vint4 a, vint4 b)
557 {
558 #if ASTCENC_SSE >= 41
559 return vint4(_mm_min_epi32(a.m, b.m));
560 #else
561 vmask4 d = a < b;
562 __m128i ap = _mm_and_si128(_mm_castps_si128(d.m), a.m);
563 __m128i bp = _mm_andnot_si128(_mm_castps_si128(d.m), b.m);
564 return vint4(_mm_or_si128(ap,bp));
565 #endif
566 }
567
568 /**
569 * @brief Return the max vector of two vectors.
570 */
max(vint4 a,vint4 b)571 ASTCENC_SIMD_INLINE vint4 max(vint4 a, vint4 b)
572 {
573 #if ASTCENC_SSE >= 41
574 return vint4(_mm_max_epi32(a.m, b.m));
575 #else
576 vmask4 d = a > b;
577 __m128i ap = _mm_and_si128(_mm_castps_si128(d.m), a.m);
578 __m128i bp = _mm_andnot_si128(_mm_castps_si128(d.m), b.m);
579 return vint4(_mm_or_si128(ap,bp));
580 #endif
581 }
582
583 /**
584 * @brief Return the horizontal minimum of a vector.
585 */
hmin(vint4 a)586 ASTCENC_SIMD_INLINE vint4 hmin(vint4 a)
587 {
588 a = min(a, vint4(_mm_shuffle_epi32(a.m, _MM_SHUFFLE(0, 0, 3, 2))));
589 a = min(a, vint4(_mm_shuffle_epi32(a.m, _MM_SHUFFLE(0, 0, 0, 1))));
590 return vint4(_mm_shuffle_epi32(a.m, _MM_SHUFFLE(0, 0, 0, 0)));
591 }
592
593 /*
594 * @brief Return the horizontal maximum of a vector.
595 */
hmax(vint4 a)596 ASTCENC_SIMD_INLINE vint4 hmax(vint4 a)
597 {
598 a = max(a, vint4(_mm_shuffle_epi32(a.m, _MM_SHUFFLE(0, 0, 3, 2))));
599 a = max(a, vint4(_mm_shuffle_epi32(a.m, _MM_SHUFFLE(0, 0, 0, 1))));
600 return vint4(_mm_shuffle_epi32(a.m, _MM_SHUFFLE(0, 0, 0, 0)));
601 }
602
603 /**
604 * @brief Return the horizontal sum of a vector as a scalar.
605 */
hadd_s(vint4 a)606 ASTCENC_SIMD_INLINE int hadd_s(vint4 a)
607 {
608 // Add top and bottom halves, lane 1/0
609 __m128i fold = _mm_castps_si128(_mm_movehl_ps(_mm_castsi128_ps(a.m),
610 _mm_castsi128_ps(a.m)));
611 __m128i t = _mm_add_epi32(a.m, fold);
612
613 // Add top and bottom halves, lane 0 (_mm_hadd_ps exists but slow)
614 t = _mm_add_epi32(t, _mm_shuffle_epi32(t, 0x55));
615
616 return _mm_cvtsi128_si32(t);
617 }
618
619 /**
620 * @brief Store a vector to a 16B aligned memory address.
621 */
storea(vint4 a,int * p)622 ASTCENC_SIMD_INLINE void storea(vint4 a, int* p)
623 {
624 _mm_store_si128(reinterpret_cast<__m128i*>(p), a.m);
625 }
626
627 /**
628 * @brief Store a vector to an unaligned memory address.
629 */
store(vint4 a,int * p)630 ASTCENC_SIMD_INLINE void store(vint4 a, int* p)
631 {
632 // Cast due to missing intrinsics
633 _mm_storeu_ps(reinterpret_cast<float*>(p), _mm_castsi128_ps(a.m));
634 }
635
636 /**
637 * @brief Store lowest N (vector width) bytes into an unaligned address.
638 */
store_nbytes(vint4 a,uint8_t * p)639 ASTCENC_SIMD_INLINE void store_nbytes(vint4 a, uint8_t* p)
640 {
641 // Cast due to missing intrinsics
642 _mm_store_ss(reinterpret_cast<float*>(p), _mm_castsi128_ps(a.m));
643 }
644
645 /**
646 * @brief Gather N (vector width) indices from the array.
647 */
gatheri(const int * base,vint4 indices)648 ASTCENC_SIMD_INLINE vint4 gatheri(const int* base, vint4 indices)
649 {
650 #if ASTCENC_AVX >= 2
651 return vint4(_mm_i32gather_epi32(base, indices.m, 4));
652 #else
653 alignas(16) int idx[4];
654 storea(indices, idx);
655 return vint4(base[idx[0]], base[idx[1]], base[idx[2]], base[idx[3]]);
656 #endif
657 }
658
659 /**
660 * @brief Pack low 8 bits of N (vector width) lanes into bottom of vector.
661 */
pack_low_bytes(vint4 a)662 ASTCENC_SIMD_INLINE vint4 pack_low_bytes(vint4 a)
663 {
664 #if ASTCENC_SSE >= 41
665 __m128i shuf = _mm_set_epi8(0,0,0,0, 0,0,0,0, 0,0,0,0, 12,8,4,0);
666 return vint4(_mm_shuffle_epi8(a.m, shuf));
667 #else
668 __m128i va = _mm_unpacklo_epi8(a.m, _mm_shuffle_epi32(a.m, _MM_SHUFFLE(1,1,1,1)));
669 __m128i vb = _mm_unpackhi_epi8(a.m, _mm_shuffle_epi32(a.m, _MM_SHUFFLE(3,3,3,3)));
670 return vint4(_mm_unpacklo_epi16(va, vb));
671 #endif
672 }
673
674 /**
675 * @brief Return lanes from @c b if @c cond is set, else @c a.
676 */
select(vint4 a,vint4 b,vmask4 cond)677 ASTCENC_SIMD_INLINE vint4 select(vint4 a, vint4 b, vmask4 cond)
678 {
679 __m128i condi = _mm_castps_si128(cond.m);
680
681 #if ASTCENC_SSE >= 41
682 return vint4(_mm_blendv_epi8(a.m, b.m, condi));
683 #else
684 return vint4(_mm_or_si128(_mm_and_si128(condi, b.m), _mm_andnot_si128(condi, a.m)));
685 #endif
686 }
687
688 // ============================================================================
689 // vfloat4 operators and functions
690 // ============================================================================
691
692 /**
693 * @brief Overload: vector by vector addition.
694 */
695 ASTCENC_SIMD_INLINE vfloat4 operator+(vfloat4 a, vfloat4 b)
696 {
697 return vfloat4(_mm_add_ps(a.m, b.m));
698 }
699
700 /**
701 * @brief Overload: vector by vector subtraction.
702 */
703 ASTCENC_SIMD_INLINE vfloat4 operator-(vfloat4 a, vfloat4 b)
704 {
705 return vfloat4(_mm_sub_ps(a.m, b.m));
706 }
707
708 /**
709 * @brief Overload: vector by vector multiplication.
710 */
711 ASTCENC_SIMD_INLINE vfloat4 operator*(vfloat4 a, vfloat4 b)
712 {
713 return vfloat4(_mm_mul_ps(a.m, b.m));
714 }
715
716 /**
717 * @brief Overload: vector by vector division.
718 */
719 ASTCENC_SIMD_INLINE vfloat4 operator/(vfloat4 a, vfloat4 b)
720 {
721 return vfloat4(_mm_div_ps(a.m, b.m));
722 }
723
724 /**
725 * @brief Overload: vector by vector equality.
726 */
727 ASTCENC_SIMD_INLINE vmask4 operator==(vfloat4 a, vfloat4 b)
728 {
729 return vmask4(_mm_cmpeq_ps(a.m, b.m));
730 }
731
732 /**
733 * @brief Overload: vector by vector inequality.
734 */
735 ASTCENC_SIMD_INLINE vmask4 operator!=(vfloat4 a, vfloat4 b)
736 {
737 return vmask4(_mm_cmpneq_ps(a.m, b.m));
738 }
739
740 /**
741 * @brief Overload: vector by vector less than.
742 */
743 ASTCENC_SIMD_INLINE vmask4 operator<(vfloat4 a, vfloat4 b)
744 {
745 return vmask4(_mm_cmplt_ps(a.m, b.m));
746 }
747
748 /**
749 * @brief Overload: vector by vector greater than.
750 */
751 ASTCENC_SIMD_INLINE vmask4 operator>(vfloat4 a, vfloat4 b)
752 {
753 return vmask4(_mm_cmpgt_ps(a.m, b.m));
754 }
755
756 /**
757 * @brief Overload: vector by vector less than or equal.
758 */
759 ASTCENC_SIMD_INLINE vmask4 operator<=(vfloat4 a, vfloat4 b)
760 {
761 return vmask4(_mm_cmple_ps(a.m, b.m));
762 }
763
764 /**
765 * @brief Overload: vector by vector greater than or equal.
766 */
767 ASTCENC_SIMD_INLINE vmask4 operator>=(vfloat4 a, vfloat4 b)
768 {
769 return vmask4(_mm_cmpge_ps(a.m, b.m));
770 }
771
772 /**
773 * @brief Return the min vector of two vectors.
774 *
775 * If either lane value is NaN, @c b will be returned for that lane.
776 */
min(vfloat4 a,vfloat4 b)777 ASTCENC_SIMD_INLINE vfloat4 min(vfloat4 a, vfloat4 b)
778 {
779 // Do not reorder - second operand will return if either is NaN
780 return vfloat4(_mm_min_ps(a.m, b.m));
781 }
782
783 /**
784 * @brief Return the max vector of two vectors.
785 *
786 * If either lane value is NaN, @c b will be returned for that lane.
787 */
max(vfloat4 a,vfloat4 b)788 ASTCENC_SIMD_INLINE vfloat4 max(vfloat4 a, vfloat4 b)
789 {
790 // Do not reorder - second operand will return if either is NaN
791 return vfloat4(_mm_max_ps(a.m, b.m));
792 }
793
794 /**
795 * @brief Return the absolute value of the float vector.
796 */
abs(vfloat4 a)797 ASTCENC_SIMD_INLINE vfloat4 abs(vfloat4 a)
798 {
799 return vfloat4(_mm_max_ps(_mm_sub_ps(_mm_setzero_ps(), a.m), a.m));
800 }
801
802 /**
803 * @brief Return a float rounded to the nearest integer value.
804 */
round(vfloat4 a)805 ASTCENC_SIMD_INLINE vfloat4 round(vfloat4 a)
806 {
807 #if ASTCENC_SSE >= 41
808 constexpr int flags = _MM_FROUND_TO_NEAREST_INT | _MM_FROUND_NO_EXC;
809 return vfloat4(_mm_round_ps(a.m, flags));
810 #else
811 __m128 v = a.m;
812 __m128 neg_zero = _mm_castsi128_ps(_mm_set1_epi32(static_cast<int>(0x80000000)));
813 __m128 no_fraction = _mm_set1_ps(8388608.0f);
814 __m128 abs_mask = _mm_castsi128_ps(_mm_set1_epi32(0x7FFFFFFF));
815 __m128 sign = _mm_and_ps(v, neg_zero);
816 __m128 s_magic = _mm_or_ps(no_fraction, sign);
817 __m128 r1 = _mm_add_ps(v, s_magic);
818 r1 = _mm_sub_ps(r1, s_magic);
819 __m128 r2 = _mm_and_ps(v, abs_mask);
820 __m128 mask = _mm_cmple_ps(r2, no_fraction);
821 r2 = _mm_andnot_ps(mask, v);
822 r1 = _mm_and_ps(r1, mask);
823 return vfloat4(_mm_xor_ps(r1, r2));
824 #endif
825 }
826
827 /**
828 * @brief Return the horizontal minimum of a vector.
829 */
hmin(vfloat4 a)830 ASTCENC_SIMD_INLINE vfloat4 hmin(vfloat4 a)
831 {
832 a = min(a, vfloat4(_mm_shuffle_ps(a.m, a.m, _MM_SHUFFLE(0, 0, 3, 2))));
833 a = min(a, vfloat4(_mm_shuffle_ps(a.m, a.m, _MM_SHUFFLE(0, 0, 0, 1))));
834 return vfloat4(_mm_shuffle_ps(a.m, a.m, _MM_SHUFFLE(0, 0, 0, 0)));
835 }
836
837 /**
838 * @brief Return the horizontal maximum of a vector.
839 */
hmax(vfloat4 a)840 ASTCENC_SIMD_INLINE vfloat4 hmax(vfloat4 a)
841 {
842 a = max(a, vfloat4(_mm_shuffle_ps(a.m, a.m, _MM_SHUFFLE(0, 0, 3, 2))));
843 a = max(a, vfloat4(_mm_shuffle_ps(a.m, a.m, _MM_SHUFFLE(0, 0, 0, 1))));
844 return vfloat4(_mm_shuffle_ps(a.m, a.m, _MM_SHUFFLE(0, 0, 0, 0)));
845 }
846
847 /**
848 * @brief Return the horizontal sum of a vector as a scalar.
849 */
hadd_s(vfloat4 a)850 ASTCENC_SIMD_INLINE float hadd_s(vfloat4 a)
851 {
852 // Add top and bottom halves, lane 1/0
853 __m128 t = _mm_add_ps(a.m, _mm_movehl_ps(a.m, a.m));
854
855 // Add top and bottom halves, lane 0 (_mm_hadd_ps exists but slow)
856 t = _mm_add_ss(t, _mm_shuffle_ps(t, t, 0x55));
857
858 return _mm_cvtss_f32(t);
859 }
860
861 /**
862 * @brief Return the sqrt of the lanes in the vector.
863 */
sqrt(vfloat4 a)864 ASTCENC_SIMD_INLINE vfloat4 sqrt(vfloat4 a)
865 {
866 return vfloat4(_mm_sqrt_ps(a.m));
867 }
868
869 /**
870 * @brief Return lanes from @c b if @c cond is set, else @c a.
871 */
select(vfloat4 a,vfloat4 b,vmask4 cond)872 ASTCENC_SIMD_INLINE vfloat4 select(vfloat4 a, vfloat4 b, vmask4 cond)
873 {
874 #if ASTCENC_SSE >= 41
875 return vfloat4(_mm_blendv_ps(a.m, b.m, cond.m));
876 #else
877 return vfloat4(_mm_or_ps(_mm_and_ps(cond.m, b.m), _mm_andnot_ps(cond.m, a.m)));
878 #endif
879 }
880
881 /**
882 * @brief Return lanes from @c b if MSB of @c cond is set, else @c a.
883 */
select_msb(vfloat4 a,vfloat4 b,vmask4 cond)884 ASTCENC_SIMD_INLINE vfloat4 select_msb(vfloat4 a, vfloat4 b, vmask4 cond)
885 {
886 #if ASTCENC_SSE >= 41
887 return vfloat4(_mm_blendv_ps(a.m, b.m, cond.m));
888 #else
889 __m128 d = _mm_castsi128_ps(_mm_srai_epi32(_mm_castps_si128(cond.m), 31));
890 return vfloat4(_mm_or_ps(_mm_and_ps(d, b.m), _mm_andnot_ps(d, a.m)));
891 #endif
892 }
893
894 /**
895 * @brief Load a vector of gathered results from an array;
896 */
gatherf(const float * base,vint4 indices)897 ASTCENC_SIMD_INLINE vfloat4 gatherf(const float* base, vint4 indices)
898 {
899 #if ASTCENC_AVX >= 2
900 return vfloat4(_mm_i32gather_ps(base, indices.m, 4));
901 #else
902 alignas(16) int idx[4];
903 storea(indices, idx);
904 return vfloat4(base[idx[0]], base[idx[1]], base[idx[2]], base[idx[3]]);
905 #endif
906 }
907
908 /**
909 * @brief Store a vector to an unaligned memory address.
910 */
store(vfloat4 a,float * p)911 ASTCENC_SIMD_INLINE void store(vfloat4 a, float* p)
912 {
913 _mm_storeu_ps(p, a.m);
914 }
915
916 /**
917 * @brief Store a vector to a 16B aligned memory address.
918 */
storea(vfloat4 a,float * p)919 ASTCENC_SIMD_INLINE void storea(vfloat4 a, float* p)
920 {
921 _mm_store_ps(p, a.m);
922 }
923
924 /**
925 * @brief Return a integer value for a float vector, using truncation.
926 */
float_to_int(vfloat4 a)927 ASTCENC_SIMD_INLINE vint4 float_to_int(vfloat4 a)
928 {
929 return vint4(_mm_cvttps_epi32(a.m));
930 }
931
932 /**
933 * @brief Return a integer value for a float vector, using round-to-nearest.
934 */
float_to_int_rtn(vfloat4 a)935 ASTCENC_SIMD_INLINE vint4 float_to_int_rtn(vfloat4 a)
936 {
937 a = round(a);
938 return vint4(_mm_cvttps_epi32(a.m));
939 }
940
941 /**
942 * @brief Return a float value for an integer vector.
943 */
int_to_float(vint4 a)944 ASTCENC_SIMD_INLINE vfloat4 int_to_float(vint4 a)
945 {
946 return vfloat4(_mm_cvtepi32_ps(a.m));
947 }
948
949 /**
950 * @brief Return a float16 value for a float vector, using round-to-nearest.
951 */
float_to_float16(vfloat4 a)952 ASTCENC_SIMD_INLINE vint4 float_to_float16(vfloat4 a)
953 {
954 #if ASTCENC_F16C >= 1
955 __m128i packedf16 = _mm_cvtps_ph(a.m, 0);
956 __m128i f16 = _mm_cvtepu16_epi32(packedf16);
957 return vint4(f16);
958 #else
959 return vint4(
960 float_to_sf16(a.lane<0>()),
961 float_to_sf16(a.lane<1>()),
962 float_to_sf16(a.lane<2>()),
963 float_to_sf16(a.lane<3>()));
964 #endif
965 }
966
967 /**
968 * @brief Return a float16 value for a float scalar, using round-to-nearest.
969 */
float_to_float16(float a)970 static inline uint16_t float_to_float16(float a)
971 {
972 #if ASTCENC_F16C >= 1
973 __m128i f16 = _mm_cvtps_ph(_mm_set1_ps(a), 0);
974 return static_cast<uint16_t>(_mm_cvtsi128_si32(f16));
975 #else
976 return float_to_sf16(a);
977 #endif
978 }
979
980 /**
981 * @brief Return a float value for a float16 vector.
982 */
float16_to_float(vint4 a)983 ASTCENC_SIMD_INLINE vfloat4 float16_to_float(vint4 a)
984 {
985 #if ASTCENC_F16C >= 1
986 __m128i packed = _mm_packs_epi32(a.m, a.m);
987 __m128 f32 = _mm_cvtph_ps(packed);
988 return vfloat4(f32);
989 #else
990 return vfloat4(
991 sf16_to_float(static_cast<uint16_t>(a.lane<0>())),
992 sf16_to_float(static_cast<uint16_t>(a.lane<1>())),
993 sf16_to_float(static_cast<uint16_t>(a.lane<2>())),
994 sf16_to_float(static_cast<uint16_t>(a.lane<3>())));
995 #endif
996 }
997
998 /**
999 * @brief Return a float value for a float16 scalar.
1000 */
float16_to_float(uint16_t a)1001 ASTCENC_SIMD_INLINE float float16_to_float(uint16_t a)
1002 {
1003 #if ASTCENC_F16C >= 1
1004 __m128i packed = _mm_set1_epi16(static_cast<short>(a));
1005 __m128 f32 = _mm_cvtph_ps(packed);
1006 return _mm_cvtss_f32(f32);
1007 #else
1008 return sf16_to_float(a);
1009 #endif
1010 }
1011
1012 /**
1013 * @brief Return a float value as an integer bit pattern (i.e. no conversion).
1014 *
1015 * It is a common trick to convert floats into integer bit patterns, perform
1016 * some bit hackery based on knowledge they are IEEE 754 layout, and then
1017 * convert them back again. This is the first half of that flip.
1018 */
float_as_int(vfloat4 a)1019 ASTCENC_SIMD_INLINE vint4 float_as_int(vfloat4 a)
1020 {
1021 return vint4(_mm_castps_si128(a.m));
1022 }
1023
1024 /**
1025 * @brief Return a integer value as a float bit pattern (i.e. no conversion).
1026 *
1027 * It is a common trick to convert floats into integer bit patterns, perform
1028 * some bit hackery based on knowledge they are IEEE 754 layout, and then
1029 * convert them back again. This is the second half of that flip.
1030 */
int_as_float(vint4 v)1031 ASTCENC_SIMD_INLINE vfloat4 int_as_float(vint4 v)
1032 {
1033 return vfloat4(_mm_castsi128_ps(v.m));
1034 }
1035
1036 /**
1037 * @brief Prepare a vtable lookup table for use with the native SIMD size.
1038 */
vtable_prepare(vint4 t0,vint4 & t0p)1039 ASTCENC_SIMD_INLINE void vtable_prepare(vint4 t0, vint4& t0p)
1040 {
1041 t0p = t0;
1042 }
1043
1044 /**
1045 * @brief Prepare a vtable lookup table for use with the native SIMD size.
1046 */
vtable_prepare(vint4 t0,vint4 t1,vint4 & t0p,vint4 & t1p)1047 ASTCENC_SIMD_INLINE void vtable_prepare(vint4 t0, vint4 t1, vint4& t0p, vint4& t1p)
1048 {
1049 #if ASTCENC_SSE >= 30
1050 t0p = t0;
1051 t1p = t0 ^ t1;
1052 #else
1053 t0p = t0;
1054 t1p = t1;
1055 #endif
1056 }
1057
1058 /**
1059 * @brief Prepare a vtable lookup table for use with the native SIMD size.
1060 */
vtable_prepare(vint4 t0,vint4 t1,vint4 t2,vint4 t3,vint4 & t0p,vint4 & t1p,vint4 & t2p,vint4 & t3p)1061 ASTCENC_SIMD_INLINE void vtable_prepare(
1062 vint4 t0, vint4 t1, vint4 t2, vint4 t3,
1063 vint4& t0p, vint4& t1p, vint4& t2p, vint4& t3p)
1064 {
1065 #if ASTCENC_SSE >= 30
1066 t0p = t0;
1067 t1p = t0 ^ t1;
1068 t2p = t1 ^ t2;
1069 t3p = t2 ^ t3;
1070 #else
1071 t0p = t0;
1072 t1p = t1;
1073 t2p = t2;
1074 t3p = t3;
1075 #endif
1076 }
1077
1078 /**
1079 * @brief Perform an 8-bit 16-entry table lookup, with 32-bit indexes.
1080 */
vtable_8bt_32bi(vint4 t0,vint4 idx)1081 ASTCENC_SIMD_INLINE vint4 vtable_8bt_32bi(vint4 t0, vint4 idx)
1082 {
1083 #if ASTCENC_SSE >= 30
1084 // Set index byte MSB to 1 for unused bytes so shuffle returns zero
1085 __m128i idxx = _mm_or_si128(idx.m, _mm_set1_epi32(static_cast<int>(0xFFFFFF00)));
1086
1087 __m128i result = _mm_shuffle_epi8(t0.m, idxx);
1088 return vint4(result);
1089 #else
1090 alignas(ASTCENC_VECALIGN) uint8_t table[16];
1091 storea(t0, reinterpret_cast<int*>(table + 0));
1092
1093 return vint4(table[idx.lane<0>()],
1094 table[idx.lane<1>()],
1095 table[idx.lane<2>()],
1096 table[idx.lane<3>()]);
1097 #endif
1098 }
1099
1100 /**
1101 * @brief Perform an 8-bit 32-entry table lookup, with 32-bit indexes.
1102 */
vtable_8bt_32bi(vint4 t0,vint4 t1,vint4 idx)1103 ASTCENC_SIMD_INLINE vint4 vtable_8bt_32bi(vint4 t0, vint4 t1, vint4 idx)
1104 {
1105 #if ASTCENC_SSE >= 30
1106 // Set index byte MSB to 1 for unused bytes so shuffle returns zero
1107 __m128i idxx = _mm_or_si128(idx.m, _mm_set1_epi32(static_cast<int>(0xFFFFFF00)));
1108
1109 __m128i result = _mm_shuffle_epi8(t0.m, idxx);
1110 idxx = _mm_sub_epi8(idxx, _mm_set1_epi8(16));
1111
1112 __m128i result2 = _mm_shuffle_epi8(t1.m, idxx);
1113 result = _mm_xor_si128(result, result2);
1114
1115 return vint4(result);
1116 #else
1117 alignas(ASTCENC_VECALIGN) uint8_t table[32];
1118 storea(t0, reinterpret_cast<int*>(table + 0));
1119 storea(t1, reinterpret_cast<int*>(table + 16));
1120
1121 return vint4(table[idx.lane<0>()],
1122 table[idx.lane<1>()],
1123 table[idx.lane<2>()],
1124 table[idx.lane<3>()]);
1125 #endif
1126 }
1127
1128 /**
1129 * @brief Perform an 8-bit 64-entry table lookup, with 32-bit indexes.
1130 */
vtable_8bt_32bi(vint4 t0,vint4 t1,vint4 t2,vint4 t3,vint4 idx)1131 ASTCENC_SIMD_INLINE vint4 vtable_8bt_32bi(vint4 t0, vint4 t1, vint4 t2, vint4 t3, vint4 idx)
1132 {
1133 #if ASTCENC_SSE >= 30
1134 // Set index byte MSB to 1 for unused bytes so shuffle returns zero
1135 __m128i idxx = _mm_or_si128(idx.m, _mm_set1_epi32(static_cast<int>(0xFFFFFF00)));
1136
1137 __m128i result = _mm_shuffle_epi8(t0.m, idxx);
1138 idxx = _mm_sub_epi8(idxx, _mm_set1_epi8(16));
1139
1140 __m128i result2 = _mm_shuffle_epi8(t1.m, idxx);
1141 result = _mm_xor_si128(result, result2);
1142 idxx = _mm_sub_epi8(idxx, _mm_set1_epi8(16));
1143
1144 result2 = _mm_shuffle_epi8(t2.m, idxx);
1145 result = _mm_xor_si128(result, result2);
1146 idxx = _mm_sub_epi8(idxx, _mm_set1_epi8(16));
1147
1148 result2 = _mm_shuffle_epi8(t3.m, idxx);
1149 result = _mm_xor_si128(result, result2);
1150
1151 return vint4(result);
1152 #else
1153 alignas(ASTCENC_VECALIGN) uint8_t table[64];
1154 storea(t0, reinterpret_cast<int*>(table + 0));
1155 storea(t1, reinterpret_cast<int*>(table + 16));
1156 storea(t2, reinterpret_cast<int*>(table + 32));
1157 storea(t3, reinterpret_cast<int*>(table + 48));
1158
1159 return vint4(table[idx.lane<0>()],
1160 table[idx.lane<1>()],
1161 table[idx.lane<2>()],
1162 table[idx.lane<3>()]);
1163 #endif
1164 }
1165
1166 /**
1167 * @brief Return a vector of interleaved RGBA data.
1168 *
1169 * Input vectors have the value stored in the bottom 8 bits of each lane,
1170 * with high bits set to zero.
1171 *
1172 * Output vector stores a single RGBA texel packed in each lane.
1173 */
interleave_rgba8(vint4 r,vint4 g,vint4 b,vint4 a)1174 ASTCENC_SIMD_INLINE vint4 interleave_rgba8(vint4 r, vint4 g, vint4 b, vint4 a)
1175 {
1176 // Workaround an XCode compiler internal fault; note is slower than slli_epi32
1177 // so we should revert this when we get the opportunity
1178 #if defined(__APPLE__)
1179 __m128i value = r.m;
1180 value = _mm_add_epi32(value, _mm_bslli_si128(g.m, 1));
1181 value = _mm_add_epi32(value, _mm_bslli_si128(b.m, 2));
1182 value = _mm_add_epi32(value, _mm_bslli_si128(a.m, 3));
1183 return vint4(value);
1184 #else
1185 __m128i value = r.m;
1186 value = _mm_add_epi32(value, _mm_slli_epi32(g.m, 8));
1187 value = _mm_add_epi32(value, _mm_slli_epi32(b.m, 16));
1188 value = _mm_add_epi32(value, _mm_slli_epi32(a.m, 24));
1189 return vint4(value);
1190 #endif
1191 }
1192
1193 /**
1194 * @brief Store a vector, skipping masked lanes.
1195 *
1196 * All masked lanes must be at the end of vector, after all non-masked lanes.
1197 */
store_lanes_masked(int * base,vint4 data,vmask4 mask)1198 ASTCENC_SIMD_INLINE void store_lanes_masked(int* base, vint4 data, vmask4 mask)
1199 {
1200 #if ASTCENC_AVX >= 2
1201 _mm_maskstore_epi32(base, _mm_castps_si128(mask.m), data.m);
1202 #else
1203 // Note - we cannot use _mm_maskmoveu_si128 as the underlying hardware doesn't guarantee
1204 // fault suppression on masked lanes so we can get page faults at the end of an image.
1205 if (mask.lane<3>() != 0.0f)
1206 {
1207 store(data, base);
1208 }
1209 else if (mask.lane<2>() != 0.0f)
1210 {
1211 base[0] = data.lane<0>();
1212 base[1] = data.lane<1>();
1213 base[2] = data.lane<2>();
1214 }
1215 else if (mask.lane<1>() != 0.0f)
1216 {
1217 base[0] = data.lane<0>();
1218 base[1] = data.lane<1>();
1219 }
1220 else if (mask.lane<0>() != 0.0f)
1221 {
1222 base[0] = data.lane<0>();
1223 }
1224 #endif
1225 }
1226
1227 #if defined(ASTCENC_NO_INVARIANCE) && (ASTCENC_SSE >= 41)
1228
1229 #define ASTCENC_USE_NATIVE_DOT_PRODUCT 1
1230
1231 /**
1232 * @brief Return the dot product for the full 4 lanes, returning scalar.
1233 */
dot_s(vfloat4 a,vfloat4 b)1234 ASTCENC_SIMD_INLINE float dot_s(vfloat4 a, vfloat4 b)
1235 {
1236 return _mm_cvtss_f32(_mm_dp_ps(a.m, b.m, 0xFF));
1237 }
1238
1239 /**
1240 * @brief Return the dot product for the full 4 lanes, returning vector.
1241 */
dot(vfloat4 a,vfloat4 b)1242 ASTCENC_SIMD_INLINE vfloat4 dot(vfloat4 a, vfloat4 b)
1243 {
1244 return vfloat4(_mm_dp_ps(a.m, b.m, 0xFF));
1245 }
1246
1247 /**
1248 * @brief Return the dot product for the bottom 3 lanes, returning scalar.
1249 */
dot3_s(vfloat4 a,vfloat4 b)1250 ASTCENC_SIMD_INLINE float dot3_s(vfloat4 a, vfloat4 b)
1251 {
1252 return _mm_cvtss_f32(_mm_dp_ps(a.m, b.m, 0x77));
1253 }
1254
1255 /**
1256 * @brief Return the dot product for the bottom 3 lanes, returning vector.
1257 */
dot3(vfloat4 a,vfloat4 b)1258 ASTCENC_SIMD_INLINE vfloat4 dot3(vfloat4 a, vfloat4 b)
1259 {
1260 return vfloat4(_mm_dp_ps(a.m, b.m, 0x77));
1261 }
1262
1263 #endif // #if defined(ASTCENC_NO_INVARIANCE) && (ASTCENC_SSE >= 41)
1264
1265 #if ASTCENC_POPCNT >= 1
1266
1267 #define ASTCENC_USE_NATIVE_POPCOUNT 1
1268
1269 /**
1270 * @brief Population bit count.
1271 *
1272 * @param v The value to population count.
1273 *
1274 * @return The number of 1 bits.
1275 */
popcount(uint64_t v)1276 ASTCENC_SIMD_INLINE int popcount(uint64_t v)
1277 {
1278 return static_cast<int>(_mm_popcnt_u64(v));
1279 }
1280
1281 #endif // ASTCENC_POPCNT >= 1
1282
1283 #endif // #ifndef ASTC_VECMATHLIB_SSE_4_H_INCLUDED
1284