1 /*
2 * Armv7-A specific checksum implementation using NEON
3 *
4 * Copyright (c) 2020, Arm Limited.
5 * SPDX-License-Identifier: MIT
6 */
7
8 #include "networking.h"
9 #include "../chksum_common.h"
10
11 #ifndef __ARM_NEON
12 #pragma GCC target("+simd")
13 #endif
14
15 #include <arm_neon.h>
16
17 unsigned short
__chksum_arm_simd(const void * ptr,unsigned int nbytes)18 __chksum_arm_simd(const void *ptr, unsigned int nbytes)
19 {
20 bool swap = (uintptr_t) ptr & 1;
21 uint64x1_t vsum = { 0 };
22
23 if (unlikely(nbytes < 40))
24 {
25 uint64_t sum = slurp_small(ptr, nbytes);
26 return fold_and_swap(sum, false);
27 }
28
29 /* 8-byte align pointer */
30 /* Inline slurp_head-like code since we use NEON here */
31 Assert(nbytes >= 8);
32 uint32_t off = (uintptr_t) ptr & 7;
33 if (likely(off != 0))
34 {
35 const uint64_t *may_alias ptr64 = align_ptr(ptr, 8);
36 uint64x1_t vword64 = vld1_u64(ptr64);
37 /* Get rid of bytes 0..off-1 */
38 uint64x1_t vmask = vdup_n_u64(ALL_ONES);
39 int64x1_t vshiftl = vdup_n_s64(CHAR_BIT * off);
40 vmask = vshl_u64(vmask, vshiftl);
41 vword64 = vand_u64(vword64, vmask);
42 uint32x2_t vtmp = vreinterpret_u32_u64(vword64);
43 /* Set accumulator */
44 vsum = vpaddl_u32(vtmp);
45 /* Update pointer and remaining size */
46 ptr = (char *) ptr64 + 8;
47 nbytes -= 8 - off;
48 }
49 Assert(((uintptr_t) ptr & 7) == 0);
50
51 /* Sum groups of 64 bytes */
52 uint64x2_t vsum0 = { 0, 0 };
53 uint64x2_t vsum1 = { 0, 0 };
54 uint64x2_t vsum2 = { 0, 0 };
55 uint64x2_t vsum3 = { 0, 0 };
56 const uint32_t *may_alias ptr32 = ptr;
57 for (uint32_t i = 0; i < nbytes / 64; i++)
58 {
59 uint32x4_t vtmp0 = vld1q_u32(ptr32);
60 uint32x4_t vtmp1 = vld1q_u32(ptr32 + 4);
61 uint32x4_t vtmp2 = vld1q_u32(ptr32 + 8);
62 uint32x4_t vtmp3 = vld1q_u32(ptr32 + 12);
63 vsum0 = vpadalq_u32(vsum0, vtmp0);
64 vsum1 = vpadalq_u32(vsum1, vtmp1);
65 vsum2 = vpadalq_u32(vsum2, vtmp2);
66 vsum3 = vpadalq_u32(vsum3, vtmp3);
67 ptr32 += 16;
68 }
69 nbytes %= 64;
70
71 /* Fold vsum1/vsum2/vsum3 into vsum0 */
72 vsum0 = vpadalq_u32(vsum0, vreinterpretq_u32_u64(vsum2));
73 vsum1 = vpadalq_u32(vsum1, vreinterpretq_u32_u64(vsum3));
74 vsum0 = vpadalq_u32(vsum0, vreinterpretq_u32_u64(vsum1));
75
76 /* Add any trailing 16-byte groups */
77 while (likely(nbytes >= 16))
78 {
79 uint32x4_t vtmp0 = vld1q_u32(ptr32);
80 vsum0 = vpadalq_u32(vsum0, vtmp0);
81 ptr32 += 4;
82 nbytes -= 16;
83 }
84 Assert(nbytes < 16);
85
86 /* Fold vsum0 into vsum */
87 {
88 /* 4xu32 (4x32b) -> 2xu64 (2x33b) */
89 vsum0 = vpaddlq_u32(vreinterpretq_u32_u64(vsum0));
90 /* 4xu32 (2x(1b+32b)) -> 2xu64 (2x(0b+32b)) */
91 vsum0 = vpaddlq_u32(vreinterpretq_u32_u64(vsum0));
92 /* 4xu32 (4x32b) -> 2xu64 (2x33b) */
93 Assert((vgetq_lane_u64(vsum0, 0) >> 32) == 0);
94 Assert((vgetq_lane_u64(vsum0, 1) >> 32) == 0);
95 uint32x2_t vtmp = vmovn_u64(vsum0);
96 /* Add to accumulator */
97 vsum = vpadal_u32(vsum, vtmp);
98 }
99
100 /* Add any trailing group of 8 bytes */
101 if (nbytes & 8)
102 {
103 uint32x2_t vtmp = vld1_u32(ptr32);
104 /* Add to accumulator */
105 vsum = vpadal_u32(vsum, vtmp);
106 ptr32 += 2;
107 nbytes -= 8;
108 }
109 Assert(nbytes < 8);
110
111 /* Handle any trailing 1..7 bytes */
112 if (likely(nbytes != 0))
113 {
114 Assert(((uintptr_t) ptr32 & 7) == 0);
115 Assert(nbytes < 8);
116 uint64x1_t vword64 = vld1_u64((const uint64_t *) ptr32);
117 /* Get rid of bytes 7..nbytes */
118 uint64x1_t vmask = vdup_n_u64(ALL_ONES);
119 int64x1_t vshiftr = vdup_n_s64(-CHAR_BIT * (8 - nbytes));
120 vmask = vshl_u64(vmask, vshiftr);/* Shift right */
121 vword64 = vand_u64(vword64, vmask);
122 /* Fold 64-bit sum to 33 bits */
123 vword64 = vpaddl_u32(vreinterpret_u32_u64(vword64));
124 /* Add to accumulator */
125 vsum = vpadal_u32(vsum, vreinterpret_u32_u64(vword64));
126 }
127
128 /* Fold 64-bit vsum to 32 bits */
129 vsum = vpaddl_u32(vreinterpret_u32_u64(vsum));
130 vsum = vpaddl_u32(vreinterpret_u32_u64(vsum));
131 Assert(vget_lane_u32(vreinterpret_u32_u64(vsum), 1) == 0);
132
133 /* Fold 32-bit vsum to 16 bits */
134 uint32x2_t vsum32 = vreinterpret_u32_u64(vsum);
135 vsum32 = vpaddl_u16(vreinterpret_u16_u32(vsum32));
136 vsum32 = vpaddl_u16(vreinterpret_u16_u32(vsum32));
137 Assert(vget_lane_u16(vreinterpret_u16_u32(vsum32), 1) == 0);
138 Assert(vget_lane_u16(vreinterpret_u16_u32(vsum32), 2) == 0);
139 Assert(vget_lane_u16(vreinterpret_u16_u32(vsum32), 3) == 0);
140
141 /* Convert to 16-bit scalar */
142 uint16_t sum = vget_lane_u16(vreinterpret_u16_u32(vsum32), 0);
143
144 if (unlikely(swap))/* Odd base pointer is unexpected */
145 {
146 sum = bswap16(sum);
147 }
148 return sum;
149 }
150