1 /* 7zCrc.c -- CRC32 init
2 2021-04-01 : Igor Pavlov : Public domain */
3
4 #include "Precomp.h"
5
6 #include "7zCrc.h"
7 #include "CpuArch.h"
8
9 #define kCrcPoly 0xEDB88320
10
11 #ifdef MY_CPU_LE
12 #define CRC_NUM_TABLES 8
13 #else
14 #define CRC_NUM_TABLES 9
15
16 #define CRC_UINT32_SWAP(v) ((v >> 24) | ((v >> 8) & 0xFF00) | ((v << 8) & 0xFF0000) | (v << 24))
17
18 UInt32 MY_FAST_CALL CrcUpdateT1_BeT4(UInt32 v, const void *data, size_t size, const UInt32 *table);
19 UInt32 MY_FAST_CALL CrcUpdateT1_BeT8(UInt32 v, const void *data, size_t size, const UInt32 *table);
20 #endif
21
22 #ifndef MY_CPU_BE
23 UInt32 MY_FAST_CALL CrcUpdateT4(UInt32 v, const void *data, size_t size, const UInt32 *table);
24 UInt32 MY_FAST_CALL CrcUpdateT8(UInt32 v, const void *data, size_t size, const UInt32 *table);
25 #endif
26
27 typedef UInt32 (MY_FAST_CALL *CRC_FUNC)(UInt32 v, const void *data, size_t size, const UInt32 *table);
28
29 extern
30 CRC_FUNC g_CrcUpdateT4;
31 CRC_FUNC g_CrcUpdateT4;
32 extern
33 CRC_FUNC g_CrcUpdateT8;
34 CRC_FUNC g_CrcUpdateT8;
35 extern
36 CRC_FUNC g_CrcUpdateT0_32;
37 CRC_FUNC g_CrcUpdateT0_32;
38 extern
39 CRC_FUNC g_CrcUpdateT0_64;
40 CRC_FUNC g_CrcUpdateT0_64;
41 extern
42 CRC_FUNC g_CrcUpdate;
43 CRC_FUNC g_CrcUpdate;
44
45 UInt32 g_CrcTable[256 * CRC_NUM_TABLES];
46
CrcUpdate(UInt32 v,const void * data,size_t size)47 UInt32 MY_FAST_CALL CrcUpdate(UInt32 v, const void *data, size_t size)
48 {
49 return g_CrcUpdate(v, data, size, g_CrcTable);
50 }
51
CrcCalc(const void * data,size_t size)52 UInt32 MY_FAST_CALL CrcCalc(const void *data, size_t size)
53 {
54 return g_CrcUpdate(CRC_INIT_VAL, data, size, g_CrcTable) ^ CRC_INIT_VAL;
55 }
56
57 #define CRC_UPDATE_BYTE_2(crc, b) (table[((crc) ^ (b)) & 0xFF] ^ ((crc) >> 8))
58
59 UInt32 MY_FAST_CALL CrcUpdateT1(UInt32 v, const void *data, size_t size, const UInt32 *table);
CrcUpdateT1(UInt32 v,const void * data,size_t size,const UInt32 * table)60 UInt32 MY_FAST_CALL CrcUpdateT1(UInt32 v, const void *data, size_t size, const UInt32 *table)
61 {
62 const Byte *p = (const Byte *)data;
63 const Byte *pEnd = p + size;
64 for (; p != pEnd; p++)
65 v = CRC_UPDATE_BYTE_2(v, *p);
66 return v;
67 }
68
69
70 /* ---------- hardware CRC ---------- */
71
72 #ifdef MY_CPU_LE
73
74 #if defined(MY_CPU_ARM_OR_ARM64)
75
76 // #pragma message("ARM*")
77
78 #if defined(_MSC_VER)
79 #if defined(MY_CPU_ARM64)
80 #if (_MSC_VER >= 1910)
81 #define USE_ARM64_CRC
82 #endif
83 #endif
84 #elif (defined(__clang__) && (__clang_major__ >= 16)) \
85 || (defined(__GNUC__) && (__GNUC__ > 4))
86 #if !defined(__ARM_FEATURE_CRC32)
87 #define __ARM_FEATURE_CRC32 1
88 #if (!defined(__clang__) || (__clang_major__ > 16)) // fix these numbers
89 #define ATTRIB_CRC __attribute__((__target__("arch=armv8-a+crc")))
90 #endif
91 #endif
92 #if defined(__ARM_FEATURE_CRC32)
93 #define USE_ARM64_CRC
94 #include <arm_acle.h>
95 #endif
96 #endif
97
98 #else
99
100 // no hardware CRC
101
102 // #define USE_CRC_EMU
103
104 #ifdef USE_CRC_EMU
105
106 #pragma message("ARM64 CRC emulation")
107
108 MY_FORCE_INLINE
__crc32b(UInt32 v,UInt32 data)109 UInt32 __crc32b(UInt32 v, UInt32 data)
110 {
111 const UInt32 *table = g_CrcTable;
112 v = CRC_UPDATE_BYTE_2(v, (Byte)data);
113 return v;
114 }
115
116 MY_FORCE_INLINE
__crc32w(UInt32 v,UInt32 data)117 UInt32 __crc32w(UInt32 v, UInt32 data)
118 {
119 const UInt32 *table = g_CrcTable;
120 v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8;
121 v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8;
122 v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8;
123 v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8;
124 return v;
125 }
126
127 MY_FORCE_INLINE
__crc32d(UInt32 v,UInt64 data)128 UInt32 __crc32d(UInt32 v, UInt64 data)
129 {
130 const UInt32 *table = g_CrcTable;
131 v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8;
132 v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8;
133 v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8;
134 v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8;
135 v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8;
136 v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8;
137 v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8;
138 v = CRC_UPDATE_BYTE_2(v, (Byte)data); data >>= 8;
139 return v;
140 }
141
142 #endif // USE_CRC_EMU
143
144 #endif // defined(MY_CPU_ARM64) && defined(MY_CPU_LE)
145
146
147
148 #if defined(USE_ARM64_CRC) || defined(USE_CRC_EMU)
149
150 #define T0_32_UNROLL_BYTES (4 * 4)
151 #define T0_64_UNROLL_BYTES (4 * 8)
152
153 #ifndef ATTRIB_CRC
154 #define ATTRIB_CRC
155 #endif
156 // #pragma message("USE ARM HW CRC")
157
158 ATTRIB_CRC
159 UInt32 MY_FAST_CALL CrcUpdateT0_32(UInt32 v, const void *data, size_t size, const UInt32 *table);
160 ATTRIB_CRC
CrcUpdateT0_32(UInt32 v,const void * data,size_t size,const UInt32 * table)161 UInt32 MY_FAST_CALL CrcUpdateT0_32(UInt32 v, const void *data, size_t size, const UInt32 *table)
162 {
163 const Byte *p = (const Byte *)data;
164 UNUSED_VAR(table);
165
166 for (; size != 0 && ((unsigned)(ptrdiff_t)p & (T0_32_UNROLL_BYTES - 1)) != 0; size--)
167 v = __crc32b(v, *p++);
168
169 if (size >= T0_32_UNROLL_BYTES)
170 {
171 const Byte *lim = p + size;
172 size &= (T0_32_UNROLL_BYTES - 1);
173 lim -= size;
174 do
175 {
176 v = __crc32w(v, *(const UInt32 *)(const void *)(p));
177 v = __crc32w(v, *(const UInt32 *)(const void *)(p + 4)); p += 2 * 4;
178 v = __crc32w(v, *(const UInt32 *)(const void *)(p));
179 v = __crc32w(v, *(const UInt32 *)(const void *)(p + 4)); p += 2 * 4;
180 }
181 while (p != lim);
182 }
183
184 for (; size != 0; size--)
185 v = __crc32b(v, *p++);
186
187 return v;
188 }
189
190 ATTRIB_CRC
191 UInt32 MY_FAST_CALL CrcUpdateT0_64(UInt32 v, const void *data, size_t size, const UInt32 *table);
192 ATTRIB_CRC
CrcUpdateT0_64(UInt32 v,const void * data,size_t size,const UInt32 * table)193 UInt32 MY_FAST_CALL CrcUpdateT0_64(UInt32 v, const void *data, size_t size, const UInt32 *table)
194 {
195 const Byte *p = (const Byte *)data;
196 UNUSED_VAR(table);
197
198 for (; size != 0 && ((unsigned)(ptrdiff_t)p & (T0_64_UNROLL_BYTES - 1)) != 0; size--)
199 v = __crc32b(v, *p++);
200
201 if (size >= T0_64_UNROLL_BYTES)
202 {
203 const Byte *lim = p + size;
204 size &= (T0_64_UNROLL_BYTES - 1);
205 lim -= size;
206 do
207 {
208 v = __crc32d(v, *(const UInt64 *)(const void *)(p));
209 v = __crc32d(v, *(const UInt64 *)(const void *)(p + 8)); p += 2 * 8;
210 v = __crc32d(v, *(const UInt64 *)(const void *)(p));
211 v = __crc32d(v, *(const UInt64 *)(const void *)(p + 8)); p += 2 * 8;
212 }
213 while (p != lim);
214 }
215
216 for (; size != 0; size--)
217 v = __crc32b(v, *p++);
218
219 return v;
220 }
221
222 #endif // defined(USE_ARM64_CRC) || defined(USE_CRC_EMU)
223
224 #endif // MY_CPU_LE
225
226
227
228
CrcGenerateTable()229 void MY_FAST_CALL CrcGenerateTable()
230 {
231 UInt32 i;
232 for (i = 0; i < 256; i++)
233 {
234 UInt32 r = i;
235 unsigned j;
236 for (j = 0; j < 8; j++)
237 r = (r >> 1) ^ (kCrcPoly & ((UInt32)0 - (r & 1)));
238 g_CrcTable[i] = r;
239 }
240 for (i = 256; i < 256 * CRC_NUM_TABLES; i++)
241 {
242 UInt32 r = g_CrcTable[(size_t)i - 256];
243 g_CrcTable[i] = g_CrcTable[r & 0xFF] ^ (r >> 8);
244 }
245
246 #if CRC_NUM_TABLES < 4
247
248 g_CrcUpdate = CrcUpdateT1;
249
250 #else
251
252 #ifdef MY_CPU_LE
253
254 g_CrcUpdateT4 = CrcUpdateT4;
255 g_CrcUpdate = CrcUpdateT4;
256
257 #if CRC_NUM_TABLES >= 8
258 g_CrcUpdateT8 = CrcUpdateT8;
259
260 #ifdef MY_CPU_X86_OR_AMD64
261 if (!CPU_Is_InOrder())
262 #endif
263 g_CrcUpdate = CrcUpdateT8;
264 #endif
265
266 #else
267 {
268 #ifndef MY_CPU_BE
269 UInt32 k = 0x01020304;
270 const Byte *p = (const Byte *)&k;
271 if (p[0] == 4 && p[1] == 3)
272 {
273 g_CrcUpdateT4 = CrcUpdateT4;
274 g_CrcUpdate = CrcUpdateT4;
275 #if CRC_NUM_TABLES >= 8
276 g_CrcUpdateT8 = CrcUpdateT8;
277 g_CrcUpdate = CrcUpdateT8;
278 #endif
279 }
280 else if (p[0] != 1 || p[1] != 2)
281 g_CrcUpdate = CrcUpdateT1;
282 else
283 #endif
284 {
285 for (i = 256 * CRC_NUM_TABLES - 1; i >= 256; i--)
286 {
287 UInt32 x = g_CrcTable[(size_t)i - 256];
288 g_CrcTable[i] = CRC_UINT32_SWAP(x);
289 }
290 g_CrcUpdateT4 = CrcUpdateT1_BeT4;
291 g_CrcUpdate = CrcUpdateT1_BeT4;
292 #if CRC_NUM_TABLES >= 8
293 g_CrcUpdateT8 = CrcUpdateT1_BeT8;
294 g_CrcUpdate = CrcUpdateT1_BeT8;
295 #endif
296 }
297 }
298 #endif
299 #endif
300
301 #ifdef MY_CPU_LE
302 #ifdef USE_ARM64_CRC
303 if (CPU_IsSupported_CRC32())
304 {
305 g_CrcUpdateT0_32 = CrcUpdateT0_32;
306 g_CrcUpdateT0_64 = CrcUpdateT0_64;
307 g_CrcUpdate =
308 #if defined(MY_CPU_ARM)
309 CrcUpdateT0_32;
310 #else
311 CrcUpdateT0_64;
312 #endif
313 }
314 #endif
315
316 #ifdef USE_CRC_EMU
317 g_CrcUpdateT0_32 = CrcUpdateT0_32;
318 g_CrcUpdateT0_64 = CrcUpdateT0_64;
319 g_CrcUpdate = CrcUpdateT0_64;
320 #endif
321 #endif
322 }
323