1 /*
2 * xxHash - Extremely Fast Hash algorithm
3 * Header File
4 * Copyright (c) Yann Collet - Meta Platforms, Inc
5 *
6 * This source code is licensed under both the BSD-style license (found in the
7 * LICENSE file in the root directory of this source tree) and the GPLv2 (found
8 * in the COPYING file in the root directory of this source tree).
9 * You may select, at your option, one of the above-listed licenses.
10 */
11
12 /* Local adaptations for Zstandard */
13
14 #ifndef XXH_NO_XXH3
15 # define XXH_NO_XXH3
16 #endif
17
18 #ifndef XXH_NAMESPACE
19 # define XXH_NAMESPACE ZSTD_
20 #endif
21
22 /*!
23 * @mainpage xxHash
24 *
25 * xxHash is an extremely fast non-cryptographic hash algorithm, working at RAM speed
26 * limits.
27 *
28 * It is proposed in four flavors, in three families:
29 * 1. @ref XXH32_family
30 * - Classic 32-bit hash function. Simple, compact, and runs on almost all
31 * 32-bit and 64-bit systems.
32 * 2. @ref XXH64_family
33 * - Classic 64-bit adaptation of XXH32. Just as simple, and runs well on most
34 * 64-bit systems (but _not_ 32-bit systems).
35 * 3. @ref XXH3_family
36 * - Modern 64-bit and 128-bit hash function family which features improved
37 * strength and performance across the board, especially on smaller data.
38 * It benefits greatly from SIMD and 64-bit without requiring it.
39 *
40 * Benchmarks
41 * ---
42 * The reference system uses an Intel i7-9700K CPU, and runs Ubuntu x64 20.04.
43 * The open source benchmark program is compiled with clang v10.0 using -O3 flag.
44 *
45 * | Hash Name | ISA ext | Width | Large Data Speed | Small Data Velocity |
46 * | -------------------- | ------- | ----: | ---------------: | ------------------: |
47 * | XXH3_64bits() | @b AVX2 | 64 | 59.4 GB/s | 133.1 |
48 * | MeowHash | AES-NI | 128 | 58.2 GB/s | 52.5 |
49 * | XXH3_128bits() | @b AVX2 | 128 | 57.9 GB/s | 118.1 |
50 * | CLHash | PCLMUL | 64 | 37.1 GB/s | 58.1 |
51 * | XXH3_64bits() | @b SSE2 | 64 | 31.5 GB/s | 133.1 |
52 * | XXH3_128bits() | @b SSE2 | 128 | 29.6 GB/s | 118.1 |
53 * | RAM sequential read | | N/A | 28.0 GB/s | N/A |
54 * | ahash | AES-NI | 64 | 22.5 GB/s | 107.2 |
55 * | City64 | | 64 | 22.0 GB/s | 76.6 |
56 * | T1ha2 | | 64 | 22.0 GB/s | 99.0 |
57 * | City128 | | 128 | 21.7 GB/s | 57.7 |
58 * | FarmHash | AES-NI | 64 | 21.3 GB/s | 71.9 |
59 * | XXH64() | | 64 | 19.4 GB/s | 71.0 |
60 * | SpookyHash | | 64 | 19.3 GB/s | 53.2 |
61 * | Mum | | 64 | 18.0 GB/s | 67.0 |
62 * | CRC32C | SSE4.2 | 32 | 13.0 GB/s | 57.9 |
63 * | XXH32() | | 32 | 9.7 GB/s | 71.9 |
64 * | City32 | | 32 | 9.1 GB/s | 66.0 |
65 * | Blake3* | @b AVX2 | 256 | 4.4 GB/s | 8.1 |
66 * | Murmur3 | | 32 | 3.9 GB/s | 56.1 |
67 * | SipHash* | | 64 | 3.0 GB/s | 43.2 |
68 * | Blake3* | @b SSE2 | 256 | 2.4 GB/s | 8.1 |
69 * | HighwayHash | | 64 | 1.4 GB/s | 6.0 |
70 * | FNV64 | | 64 | 1.2 GB/s | 62.7 |
71 * | Blake2* | | 256 | 1.1 GB/s | 5.1 |
72 * | SHA1* | | 160 | 0.8 GB/s | 5.6 |
73 * | MD5* | | 128 | 0.6 GB/s | 7.8 |
74 * @note
75 * - Hashes which require a specific ISA extension are noted. SSE2 is also noted,
76 * even though it is mandatory on x64.
77 * - Hashes with an asterisk are cryptographic. Note that MD5 is non-cryptographic
78 * by modern standards.
79 * - Small data velocity is a rough average of algorithm's efficiency for small
80 * data. For more accurate information, see the wiki.
81 * - More benchmarks and strength tests are found on the wiki:
82 * https://github.com/Cyan4973/xxHash/wiki
83 *
84 * Usage
85 * ------
86 * All xxHash variants use a similar API. Changing the algorithm is a trivial
87 * substitution.
88 *
89 * @pre
90 * For functions which take an input and length parameter, the following
91 * requirements are assumed:
92 * - The range from [`input`, `input + length`) is valid, readable memory.
93 * - The only exception is if the `length` is `0`, `input` may be `NULL`.
94 * - For C++, the objects must have the *TriviallyCopyable* property, as the
95 * functions access bytes directly as if it was an array of `unsigned char`.
96 *
97 * @anchor single_shot_example
98 * **Single Shot**
99 *
100 * These functions are stateless functions which hash a contiguous block of memory,
101 * immediately returning the result. They are the easiest and usually the fastest
102 * option.
103 *
104 * XXH32(), XXH64(), XXH3_64bits(), XXH3_128bits()
105 *
106 * @code{.c}
107 * #include <string.h>
108 * #include "xxhash.h"
109 *
110 * // Example for a function which hashes a null terminated string with XXH32().
111 * XXH32_hash_t hash_string(const char* string, XXH32_hash_t seed)
112 * {
113 * // NULL pointers are only valid if the length is zero
114 * size_t length = (string == NULL) ? 0 : strlen(string);
115 * return XXH32(string, length, seed);
116 * }
117 * @endcode
118 *
119 *
120 * @anchor streaming_example
121 * **Streaming**
122 *
123 * These groups of functions allow incremental hashing of unknown size, even
124 * more than what would fit in a size_t.
125 *
126 * XXH32_reset(), XXH64_reset(), XXH3_64bits_reset(), XXH3_128bits_reset()
127 *
128 * @code{.c}
129 * #include <stdio.h>
130 * #include <assert.h>
131 * #include "xxhash.h"
132 * // Example for a function which hashes a FILE incrementally with XXH3_64bits().
133 * XXH64_hash_t hashFile(FILE* f)
134 * {
135 * // Allocate a state struct. Do not just use malloc() or new.
136 * XXH3_state_t* state = XXH3_createState();
137 * assert(state != NULL && "Out of memory!");
138 * // Reset the state to start a new hashing session.
139 * XXH3_64bits_reset(state);
140 * char buffer[4096];
141 * size_t count;
142 * // Read the file in chunks
143 * while ((count = fread(buffer, 1, sizeof(buffer), f)) != 0) {
144 * // Run update() as many times as necessary to process the data
145 * XXH3_64bits_update(state, buffer, count);
146 * }
147 * // Retrieve the finalized hash. This will not change the state.
148 * XXH64_hash_t result = XXH3_64bits_digest(state);
149 * // Free the state. Do not use free().
150 * XXH3_freeState(state);
151 * return result;
152 * }
153 * @endcode
154 *
155 * Streaming functions generate the xxHash value from an incremental input.
156 * This method is slower than single-call functions, due to state management.
157 * For small inputs, prefer `XXH32()` and `XXH64()`, which are better optimized.
158 *
159 * An XXH state must first be allocated using `XXH*_createState()`.
160 *
161 * Start a new hash by initializing the state with a seed using `XXH*_reset()`.
162 *
163 * Then, feed the hash state by calling `XXH*_update()` as many times as necessary.
164 *
165 * The function returns an error code, with 0 meaning OK, and any other value
166 * meaning there is an error.
167 *
168 * Finally, a hash value can be produced anytime, by using `XXH*_digest()`.
169 * This function returns the nn-bits hash as an int or long long.
170 *
171 * It's still possible to continue inserting input into the hash state after a
172 * digest, and generate new hash values later on by invoking `XXH*_digest()`.
173 *
174 * When done, release the state using `XXH*_freeState()`.
175 *
176 *
177 * @anchor canonical_representation_example
178 * **Canonical Representation**
179 *
180 * The default return values from XXH functions are unsigned 32, 64 and 128 bit
181 * integers.
182 * This the simplest and fastest format for further post-processing.
183 *
184 * However, this leaves open the question of what is the order on the byte level,
185 * since little and big endian conventions will store the same number differently.
186 *
187 * The canonical representation settles this issue by mandating big-endian
188 * convention, the same convention as human-readable numbers (large digits first).
189 *
190 * When writing hash values to storage, sending them over a network, or printing
191 * them, it's highly recommended to use the canonical representation to ensure
192 * portability across a wider range of systems, present and future.
193 *
194 * The following functions allow transformation of hash values to and from
195 * canonical format.
196 *
197 * XXH32_canonicalFromHash(), XXH32_hashFromCanonical(),
198 * XXH64_canonicalFromHash(), XXH64_hashFromCanonical(),
199 * XXH128_canonicalFromHash(), XXH128_hashFromCanonical(),
200 *
201 * @code{.c}
202 * #include <stdio.h>
203 * #include "xxhash.h"
204 *
205 * // Example for a function which prints XXH32_hash_t in human readable format
206 * void printXxh32(XXH32_hash_t hash)
207 * {
208 * XXH32_canonical_t cano;
209 * XXH32_canonicalFromHash(&cano, hash);
210 * size_t i;
211 * for(i = 0; i < sizeof(cano.digest); ++i) {
212 * printf("%02x", cano.digest[i]);
213 * }
214 * printf("\n");
215 * }
216 *
217 * // Example for a function which converts XXH32_canonical_t to XXH32_hash_t
218 * XXH32_hash_t convertCanonicalToXxh32(XXH32_canonical_t cano)
219 * {
220 * XXH32_hash_t hash = XXH32_hashFromCanonical(&cano);
221 * return hash;
222 * }
223 * @endcode
224 *
225 *
226 * @file xxhash.h
227 * xxHash prototypes and implementation
228 */
229
230 /* ****************************
231 * INLINE mode
232 ******************************/
233 /*!
234 * @defgroup public Public API
235 * Contains details on the public xxHash functions.
236 * @{
237 */
238 #ifdef XXH_DOXYGEN
239 /*!
240 * @brief Gives access to internal state declaration, required for static allocation.
241 *
242 * Incompatible with dynamic linking, due to risks of ABI changes.
243 *
244 * Usage:
245 * @code{.c}
246 * #define XXH_STATIC_LINKING_ONLY
247 * #include "xxhash.h"
248 * @endcode
249 */
250 # define XXH_STATIC_LINKING_ONLY
251 /* Do not undef XXH_STATIC_LINKING_ONLY for Doxygen */
252
253 /*!
254 * @brief Gives access to internal definitions.
255 *
256 * Usage:
257 * @code{.c}
258 * #define XXH_STATIC_LINKING_ONLY
259 * #define XXH_IMPLEMENTATION
260 * #include "xxhash.h"
261 * @endcode
262 */
263 # define XXH_IMPLEMENTATION
264 /* Do not undef XXH_IMPLEMENTATION for Doxygen */
265
266 /*!
267 * @brief Exposes the implementation and marks all functions as `inline`.
268 *
269 * Use these build macros to inline xxhash into the target unit.
270 * Inlining improves performance on small inputs, especially when the length is
271 * expressed as a compile-time constant:
272 *
273 * https://fastcompression.blogspot.com/2018/03/xxhash-for-small-keys-impressive-power.html
274 *
275 * It also keeps xxHash symbols private to the unit, so they are not exported.
276 *
277 * Usage:
278 * @code{.c}
279 * #define XXH_INLINE_ALL
280 * #include "xxhash.h"
281 * @endcode
282 * Do not compile and link xxhash.o as a separate object, as it is not useful.
283 */
284 # define XXH_INLINE_ALL
285 # undef XXH_INLINE_ALL
286 /*!
287 * @brief Exposes the implementation without marking functions as inline.
288 */
289 # define XXH_PRIVATE_API
290 # undef XXH_PRIVATE_API
291 /*!
292 * @brief Emulate a namespace by transparently prefixing all symbols.
293 *
294 * If you want to include _and expose_ xxHash functions from within your own
295 * library, but also want to avoid symbol collisions with other libraries which
296 * may also include xxHash, you can use @ref XXH_NAMESPACE to automatically prefix
297 * any public symbol from xxhash library with the value of @ref XXH_NAMESPACE
298 * (therefore, avoid empty or numeric values).
299 *
300 * Note that no change is required within the calling program as long as it
301 * includes `xxhash.h`: Regular symbol names will be automatically translated
302 * by this header.
303 */
304 # define XXH_NAMESPACE /* YOUR NAME HERE */
305 # undef XXH_NAMESPACE
306 #endif
307
308 #if (defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)) \
309 && !defined(XXH_INLINE_ALL_31684351384)
310 /* this section should be traversed only once */
311 # define XXH_INLINE_ALL_31684351384
312 /* give access to the advanced API, required to compile implementations */
313 # undef XXH_STATIC_LINKING_ONLY /* avoid macro redef */
314 # define XXH_STATIC_LINKING_ONLY
315 /* make all functions private */
316 # undef XXH_PUBLIC_API
317 # if defined(__GNUC__)
318 # define XXH_PUBLIC_API static __inline __attribute__((unused))
319 # elif defined (__cplusplus) || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */)
320 # define XXH_PUBLIC_API static inline
321 # elif defined(_MSC_VER)
322 # define XXH_PUBLIC_API static __inline
323 # else
324 /* note: this version may generate warnings for unused static functions */
325 # define XXH_PUBLIC_API static
326 # endif
327
328 /*
329 * This part deals with the special case where a unit wants to inline xxHash,
330 * but "xxhash.h" has previously been included without XXH_INLINE_ALL,
331 * such as part of some previously included *.h header file.
332 * Without further action, the new include would just be ignored,
333 * and functions would effectively _not_ be inlined (silent failure).
334 * The following macros solve this situation by prefixing all inlined names,
335 * avoiding naming collision with previous inclusions.
336 */
337 /* Before that, we unconditionally #undef all symbols,
338 * in case they were already defined with XXH_NAMESPACE.
339 * They will then be redefined for XXH_INLINE_ALL
340 */
341 # undef XXH_versionNumber
342 /* XXH32 */
343 # undef XXH32
344 # undef XXH32_createState
345 # undef XXH32_freeState
346 # undef XXH32_reset
347 # undef XXH32_update
348 # undef XXH32_digest
349 # undef XXH32_copyState
350 # undef XXH32_canonicalFromHash
351 # undef XXH32_hashFromCanonical
352 /* XXH64 */
353 # undef XXH64
354 # undef XXH64_createState
355 # undef XXH64_freeState
356 # undef XXH64_reset
357 # undef XXH64_update
358 # undef XXH64_digest
359 # undef XXH64_copyState
360 # undef XXH64_canonicalFromHash
361 # undef XXH64_hashFromCanonical
362 /* XXH3_64bits */
363 # undef XXH3_64bits
364 # undef XXH3_64bits_withSecret
365 # undef XXH3_64bits_withSeed
366 # undef XXH3_64bits_withSecretandSeed
367 # undef XXH3_createState
368 # undef XXH3_freeState
369 # undef XXH3_copyState
370 # undef XXH3_64bits_reset
371 # undef XXH3_64bits_reset_withSeed
372 # undef XXH3_64bits_reset_withSecret
373 # undef XXH3_64bits_update
374 # undef XXH3_64bits_digest
375 # undef XXH3_generateSecret
376 /* XXH3_128bits */
377 # undef XXH128
378 # undef XXH3_128bits
379 # undef XXH3_128bits_withSeed
380 # undef XXH3_128bits_withSecret
381 # undef XXH3_128bits_reset
382 # undef XXH3_128bits_reset_withSeed
383 # undef XXH3_128bits_reset_withSecret
384 # undef XXH3_128bits_reset_withSecretandSeed
385 # undef XXH3_128bits_update
386 # undef XXH3_128bits_digest
387 # undef XXH128_isEqual
388 # undef XXH128_cmp
389 # undef XXH128_canonicalFromHash
390 # undef XXH128_hashFromCanonical
391 /* Finally, free the namespace itself */
392 # undef XXH_NAMESPACE
393
394 /* employ the namespace for XXH_INLINE_ALL */
395 # define XXH_NAMESPACE XXH_INLINE_
396 /*
397 * Some identifiers (enums, type names) are not symbols,
398 * but they must nonetheless be renamed to avoid redeclaration.
399 * Alternative solution: do not redeclare them.
400 * However, this requires some #ifdefs, and has a more dispersed impact.
401 * Meanwhile, renaming can be achieved in a single place.
402 */
403 # define XXH_IPREF(Id) XXH_NAMESPACE ## Id
404 # define XXH_OK XXH_IPREF(XXH_OK)
405 # define XXH_ERROR XXH_IPREF(XXH_ERROR)
406 # define XXH_errorcode XXH_IPREF(XXH_errorcode)
407 # define XXH32_canonical_t XXH_IPREF(XXH32_canonical_t)
408 # define XXH64_canonical_t XXH_IPREF(XXH64_canonical_t)
409 # define XXH128_canonical_t XXH_IPREF(XXH128_canonical_t)
410 # define XXH32_state_s XXH_IPREF(XXH32_state_s)
411 # define XXH32_state_t XXH_IPREF(XXH32_state_t)
412 # define XXH64_state_s XXH_IPREF(XXH64_state_s)
413 # define XXH64_state_t XXH_IPREF(XXH64_state_t)
414 # define XXH3_state_s XXH_IPREF(XXH3_state_s)
415 # define XXH3_state_t XXH_IPREF(XXH3_state_t)
416 # define XXH128_hash_t XXH_IPREF(XXH128_hash_t)
417 /* Ensure the header is parsed again, even if it was previously included */
418 # undef XXHASH_H_5627135585666179
419 # undef XXHASH_H_STATIC_13879238742
420 #endif /* XXH_INLINE_ALL || XXH_PRIVATE_API */
421
422 /* ****************************************************************
423 * Stable API
424 *****************************************************************/
425 #ifndef XXHASH_H_5627135585666179
426 #define XXHASH_H_5627135585666179 1
427
428 /*! @brief Marks a global symbol. */
429 #if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API)
430 # if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT))
431 # ifdef XXH_EXPORT
432 # define XXH_PUBLIC_API __declspec(dllexport)
433 # elif XXH_IMPORT
434 # define XXH_PUBLIC_API __declspec(dllimport)
435 # endif
436 # else
437 # define XXH_PUBLIC_API /* do nothing */
438 # endif
439 #endif
440
441 #ifdef XXH_NAMESPACE
442 # define XXH_CAT(A,B) A##B
443 # define XXH_NAME2(A,B) XXH_CAT(A,B)
444 # define XXH_versionNumber XXH_NAME2(XXH_NAMESPACE, XXH_versionNumber)
445 /* XXH32 */
446 # define XXH32 XXH_NAME2(XXH_NAMESPACE, XXH32)
447 # define XXH32_createState XXH_NAME2(XXH_NAMESPACE, XXH32_createState)
448 # define XXH32_freeState XXH_NAME2(XXH_NAMESPACE, XXH32_freeState)
449 # define XXH32_reset XXH_NAME2(XXH_NAMESPACE, XXH32_reset)
450 # define XXH32_update XXH_NAME2(XXH_NAMESPACE, XXH32_update)
451 # define XXH32_digest XXH_NAME2(XXH_NAMESPACE, XXH32_digest)
452 # define XXH32_copyState XXH_NAME2(XXH_NAMESPACE, XXH32_copyState)
453 # define XXH32_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH32_canonicalFromHash)
454 # define XXH32_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH32_hashFromCanonical)
455 /* XXH64 */
456 # define XXH64 XXH_NAME2(XXH_NAMESPACE, XXH64)
457 # define XXH64_createState XXH_NAME2(XXH_NAMESPACE, XXH64_createState)
458 # define XXH64_freeState XXH_NAME2(XXH_NAMESPACE, XXH64_freeState)
459 # define XXH64_reset XXH_NAME2(XXH_NAMESPACE, XXH64_reset)
460 # define XXH64_update XXH_NAME2(XXH_NAMESPACE, XXH64_update)
461 # define XXH64_digest XXH_NAME2(XXH_NAMESPACE, XXH64_digest)
462 # define XXH64_copyState XXH_NAME2(XXH_NAMESPACE, XXH64_copyState)
463 # define XXH64_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH64_canonicalFromHash)
464 # define XXH64_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH64_hashFromCanonical)
465 /* XXH3_64bits */
466 # define XXH3_64bits XXH_NAME2(XXH_NAMESPACE, XXH3_64bits)
467 # define XXH3_64bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecret)
468 # define XXH3_64bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSeed)
469 # define XXH3_64bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_withSecretandSeed)
470 # define XXH3_createState XXH_NAME2(XXH_NAMESPACE, XXH3_createState)
471 # define XXH3_freeState XXH_NAME2(XXH_NAMESPACE, XXH3_freeState)
472 # define XXH3_copyState XXH_NAME2(XXH_NAMESPACE, XXH3_copyState)
473 # define XXH3_64bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset)
474 # define XXH3_64bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSeed)
475 # define XXH3_64bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecret)
476 # define XXH3_64bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_reset_withSecretandSeed)
477 # define XXH3_64bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_update)
478 # define XXH3_64bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_64bits_digest)
479 # define XXH3_generateSecret XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret)
480 # define XXH3_generateSecret_fromSeed XXH_NAME2(XXH_NAMESPACE, XXH3_generateSecret_fromSeed)
481 /* XXH3_128bits */
482 # define XXH128 XXH_NAME2(XXH_NAMESPACE, XXH128)
483 # define XXH3_128bits XXH_NAME2(XXH_NAMESPACE, XXH3_128bits)
484 # define XXH3_128bits_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSeed)
485 # define XXH3_128bits_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecret)
486 # define XXH3_128bits_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_withSecretandSeed)
487 # define XXH3_128bits_reset XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset)
488 # define XXH3_128bits_reset_withSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSeed)
489 # define XXH3_128bits_reset_withSecret XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecret)
490 # define XXH3_128bits_reset_withSecretandSeed XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_reset_withSecretandSeed)
491 # define XXH3_128bits_update XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_update)
492 # define XXH3_128bits_digest XXH_NAME2(XXH_NAMESPACE, XXH3_128bits_digest)
493 # define XXH128_isEqual XXH_NAME2(XXH_NAMESPACE, XXH128_isEqual)
494 # define XXH128_cmp XXH_NAME2(XXH_NAMESPACE, XXH128_cmp)
495 # define XXH128_canonicalFromHash XXH_NAME2(XXH_NAMESPACE, XXH128_canonicalFromHash)
496 # define XXH128_hashFromCanonical XXH_NAME2(XXH_NAMESPACE, XXH128_hashFromCanonical)
497 #endif
498
499
500 /* *************************************
501 * Compiler specifics
502 ***************************************/
503
504 /* specific declaration modes for Windows */
505 #if !defined(XXH_INLINE_ALL) && !defined(XXH_PRIVATE_API)
506 # if defined(WIN32) && defined(_MSC_VER) && (defined(XXH_IMPORT) || defined(XXH_EXPORT))
507 # ifdef XXH_EXPORT
508 # define XXH_PUBLIC_API __declspec(dllexport)
509 # elif XXH_IMPORT
510 # define XXH_PUBLIC_API __declspec(dllimport)
511 # endif
512 # else
513 # define XXH_PUBLIC_API /* do nothing */
514 # endif
515 #endif
516
517 #if defined (__GNUC__)
518 # define XXH_CONSTF __attribute__((const))
519 # define XXH_PUREF __attribute__((pure))
520 # define XXH_MALLOCF __attribute__((malloc))
521 #else
522 # define XXH_CONSTF /* disable */
523 # define XXH_PUREF
524 # define XXH_MALLOCF
525 #endif
526
527 /* *************************************
528 * Version
529 ***************************************/
530 #define XXH_VERSION_MAJOR 0
531 #define XXH_VERSION_MINOR 8
532 #define XXH_VERSION_RELEASE 2
533 /*! @brief Version number, encoded as two digits each */
534 #define XXH_VERSION_NUMBER (XXH_VERSION_MAJOR *100*100 + XXH_VERSION_MINOR *100 + XXH_VERSION_RELEASE)
535
536 #if defined (__cplusplus)
537 extern "C" {
538 #endif
539 /*!
540 * @brief Obtains the xxHash version.
541 *
542 * This is mostly useful when xxHash is compiled as a shared library,
543 * since the returned value comes from the library, as opposed to header file.
544 *
545 * @return @ref XXH_VERSION_NUMBER of the invoked library.
546 */
547 XXH_PUBLIC_API XXH_CONSTF unsigned XXH_versionNumber (void);
548
549 #if defined (__cplusplus)
550 }
551 #endif
552
553 /* ****************************
554 * Common basic types
555 ******************************/
556 #include <stddef.h> /* size_t */
557 /*!
558 * @brief Exit code for the streaming API.
559 */
560 typedef enum {
561 XXH_OK = 0, /*!< OK */
562 XXH_ERROR /*!< Error */
563 } XXH_errorcode;
564
565
566 /*-**********************************************************************
567 * 32-bit hash
568 ************************************************************************/
569 #if defined(XXH_DOXYGEN) /* Don't show <stdint.h> include */
570 /*!
571 * @brief An unsigned 32-bit integer.
572 *
573 * Not necessarily defined to `uint32_t` but functionally equivalent.
574 */
575 typedef uint32_t XXH32_hash_t;
576
577 #elif !defined (__VMS) \
578 && (defined (__cplusplus) \
579 || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
580 # ifdef _AIX
581 # include <inttypes.h>
582 # else
583 # include <stdint.h>
584 # endif
585 typedef uint32_t XXH32_hash_t;
586
587 #else
588 # include <limits.h>
589 # if UINT_MAX == 0xFFFFFFFFUL
590 typedef unsigned int XXH32_hash_t;
591 # elif ULONG_MAX == 0xFFFFFFFFUL
592 typedef unsigned long XXH32_hash_t;
593 # else
594 # error "unsupported platform: need a 32-bit type"
595 # endif
596 #endif
597
598 #if defined (__cplusplus)
599 extern "C" {
600 #endif
601
602 /*!
603 * @}
604 *
605 * @defgroup XXH32_family XXH32 family
606 * @ingroup public
607 * Contains functions used in the classic 32-bit xxHash algorithm.
608 *
609 * @note
610 * XXH32 is useful for older platforms, with no or poor 64-bit performance.
611 * Note that the @ref XXH3_family provides competitive speed for both 32-bit
612 * and 64-bit systems, and offers true 64/128 bit hash results.
613 *
614 * @see @ref XXH64_family, @ref XXH3_family : Other xxHash families
615 * @see @ref XXH32_impl for implementation details
616 * @{
617 */
618
619 /*!
620 * @brief Calculates the 32-bit hash of @p input using xxHash32.
621 *
622 * @param input The block of data to be hashed, at least @p length bytes in size.
623 * @param length The length of @p input, in bytes.
624 * @param seed The 32-bit seed to alter the hash's output predictably.
625 *
626 * @pre
627 * The memory between @p input and @p input + @p length must be valid,
628 * readable, contiguous memory. However, if @p length is `0`, @p input may be
629 * `NULL`. In C++, this also must be *TriviallyCopyable*.
630 *
631 * @return The calculated 32-bit xxHash32 value.
632 *
633 * @see @ref single_shot_example "Single Shot Example" for an example.
634 */
635 XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32 (const void* input, size_t length, XXH32_hash_t seed);
636
637 #ifndef XXH_NO_STREAM
638 /*!
639 * @typedef struct XXH32_state_s XXH32_state_t
640 * @brief The opaque state struct for the XXH32 streaming API.
641 *
642 * @see XXH32_state_s for details.
643 */
644 typedef struct XXH32_state_s XXH32_state_t;
645
646 /*!
647 * @brief Allocates an @ref XXH32_state_t.
648 *
649 * @return An allocated pointer of @ref XXH32_state_t on success.
650 * @return `NULL` on failure.
651 *
652 * @note Must be freed with XXH32_freeState().
653 */
654 XXH_PUBLIC_API XXH_MALLOCF XXH32_state_t* XXH32_createState(void);
655 /*!
656 * @brief Frees an @ref XXH32_state_t.
657 *
658 * @param statePtr A pointer to an @ref XXH32_state_t allocated with @ref XXH32_createState().
659 *
660 * @return @ref XXH_OK.
661 *
662 * @note @p statePtr must be allocated with XXH32_createState().
663 *
664 */
665 XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr);
666 /*!
667 * @brief Copies one @ref XXH32_state_t to another.
668 *
669 * @param dst_state The state to copy to.
670 * @param src_state The state to copy from.
671 * @pre
672 * @p dst_state and @p src_state must not be `NULL` and must not overlap.
673 */
674 XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dst_state, const XXH32_state_t* src_state);
675
676 /*!
677 * @brief Resets an @ref XXH32_state_t to begin a new hash.
678 *
679 * @param statePtr The state struct to reset.
680 * @param seed The 32-bit seed to alter the hash result predictably.
681 *
682 * @pre
683 * @p statePtr must not be `NULL`.
684 *
685 * @return @ref XXH_OK on success.
686 * @return @ref XXH_ERROR on failure.
687 *
688 * @note This function resets and seeds a state. Call it before @ref XXH32_update().
689 */
690 XXH_PUBLIC_API XXH_errorcode XXH32_reset (XXH32_state_t* statePtr, XXH32_hash_t seed);
691
692 /*!
693 * @brief Consumes a block of @p input to an @ref XXH32_state_t.
694 *
695 * @param statePtr The state struct to update.
696 * @param input The block of data to be hashed, at least @p length bytes in size.
697 * @param length The length of @p input, in bytes.
698 *
699 * @pre
700 * @p statePtr must not be `NULL`.
701 * @pre
702 * The memory between @p input and @p input + @p length must be valid,
703 * readable, contiguous memory. However, if @p length is `0`, @p input may be
704 * `NULL`. In C++, this also must be *TriviallyCopyable*.
705 *
706 * @return @ref XXH_OK on success.
707 * @return @ref XXH_ERROR on failure.
708 *
709 * @note Call this to incrementally consume blocks of data.
710 */
711 XXH_PUBLIC_API XXH_errorcode XXH32_update (XXH32_state_t* statePtr, const void* input, size_t length);
712
713 /*!
714 * @brief Returns the calculated hash value from an @ref XXH32_state_t.
715 *
716 * @param statePtr The state struct to calculate the hash from.
717 *
718 * @pre
719 * @p statePtr must not be `NULL`.
720 *
721 * @return The calculated 32-bit xxHash32 value from that state.
722 *
723 * @note
724 * Calling XXH32_digest() will not affect @p statePtr, so you can update,
725 * digest, and update again.
726 */
727 XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_digest (const XXH32_state_t* statePtr);
728 #endif /* !XXH_NO_STREAM */
729
730 /******* Canonical representation *******/
731
732 /*!
733 * @brief Canonical (big endian) representation of @ref XXH32_hash_t.
734 */
735 typedef struct {
736 unsigned char digest[4]; /*!< Hash bytes, big endian */
737 } XXH32_canonical_t;
738
739 /*!
740 * @brief Converts an @ref XXH32_hash_t to a big endian @ref XXH32_canonical_t.
741 *
742 * @param dst The @ref XXH32_canonical_t pointer to be stored to.
743 * @param hash The @ref XXH32_hash_t to be converted.
744 *
745 * @pre
746 * @p dst must not be `NULL`.
747 *
748 * @see @ref canonical_representation_example "Canonical Representation Example"
749 */
750 XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash);
751
752 /*!
753 * @brief Converts an @ref XXH32_canonical_t to a native @ref XXH32_hash_t.
754 *
755 * @param src The @ref XXH32_canonical_t to convert.
756 *
757 * @pre
758 * @p src must not be `NULL`.
759 *
760 * @return The converted hash.
761 *
762 * @see @ref canonical_representation_example "Canonical Representation Example"
763 */
764 XXH_PUBLIC_API XXH_PUREF XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src);
765
766
767 /*! @cond Doxygen ignores this part */
768 #ifdef __has_attribute
769 # define XXH_HAS_ATTRIBUTE(x) __has_attribute(x)
770 #else
771 # define XXH_HAS_ATTRIBUTE(x) 0
772 #endif
773 /*! @endcond */
774
775 /*! @cond Doxygen ignores this part */
776 /*
777 * C23 __STDC_VERSION__ number hasn't been specified yet. For now
778 * leave as `201711L` (C17 + 1).
779 * TODO: Update to correct value when its been specified.
780 */
781 #define XXH_C23_VN 201711L
782 /*! @endcond */
783
784 /*! @cond Doxygen ignores this part */
785 /* C-language Attributes are added in C23. */
786 #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= XXH_C23_VN) && defined(__has_c_attribute)
787 # define XXH_HAS_C_ATTRIBUTE(x) __has_c_attribute(x)
788 #else
789 # define XXH_HAS_C_ATTRIBUTE(x) 0
790 #endif
791 /*! @endcond */
792
793 /*! @cond Doxygen ignores this part */
794 #if defined(__cplusplus) && defined(__has_cpp_attribute)
795 # define XXH_HAS_CPP_ATTRIBUTE(x) __has_cpp_attribute(x)
796 #else
797 # define XXH_HAS_CPP_ATTRIBUTE(x) 0
798 #endif
799 /*! @endcond */
800
801 /*! @cond Doxygen ignores this part */
802 /*
803 * Define XXH_FALLTHROUGH macro for annotating switch case with the 'fallthrough' attribute
804 * introduced in CPP17 and C23.
805 * CPP17 : https://en.cppreference.com/w/cpp/language/attributes/fallthrough
806 * C23 : https://en.cppreference.com/w/c/language/attributes/fallthrough
807 */
808 #if XXH_HAS_C_ATTRIBUTE(fallthrough) || XXH_HAS_CPP_ATTRIBUTE(fallthrough)
809 # define XXH_FALLTHROUGH [[fallthrough]]
810 #elif XXH_HAS_ATTRIBUTE(__fallthrough__)
811 # define XXH_FALLTHROUGH __attribute__ ((__fallthrough__))
812 #else
813 # define XXH_FALLTHROUGH /* fallthrough */
814 #endif
815 /*! @endcond */
816
817 /*! @cond Doxygen ignores this part */
818 /*
819 * Define XXH_NOESCAPE for annotated pointers in public API.
820 * https://clang.llvm.org/docs/AttributeReference.html#noescape
821 * As of writing this, only supported by clang.
822 */
823 #if XXH_HAS_ATTRIBUTE(noescape)
824 # define XXH_NOESCAPE __attribute__((noescape))
825 #else
826 # define XXH_NOESCAPE
827 #endif
828 /*! @endcond */
829
830 #if defined (__cplusplus)
831 } /* end of extern "C" */
832 #endif
833
834 /*!
835 * @}
836 * @ingroup public
837 * @{
838 */
839
840 #ifndef XXH_NO_LONG_LONG
841 /*-**********************************************************************
842 * 64-bit hash
843 ************************************************************************/
844 #if defined(XXH_DOXYGEN) /* don't include <stdint.h> */
845 /*!
846 * @brief An unsigned 64-bit integer.
847 *
848 * Not necessarily defined to `uint64_t` but functionally equivalent.
849 */
850 typedef uint64_t XXH64_hash_t;
851 #elif !defined (__VMS) \
852 && (defined (__cplusplus) \
853 || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
854 # ifdef _AIX
855 # include <inttypes.h>
856 # else
857 # include <stdint.h>
858 # endif
859 typedef uint64_t XXH64_hash_t;
860 #else
861 # include <limits.h>
862 # if defined(__LP64__) && ULONG_MAX == 0xFFFFFFFFFFFFFFFFULL
863 /* LP64 ABI says uint64_t is unsigned long */
864 typedef unsigned long XXH64_hash_t;
865 # else
866 /* the following type must have a width of 64-bit */
867 typedef unsigned long long XXH64_hash_t;
868 # endif
869 #endif
870
871 #if defined (__cplusplus)
872 extern "C" {
873 #endif
874 /*!
875 * @}
876 *
877 * @defgroup XXH64_family XXH64 family
878 * @ingroup public
879 * @{
880 * Contains functions used in the classic 64-bit xxHash algorithm.
881 *
882 * @note
883 * XXH3 provides competitive speed for both 32-bit and 64-bit systems,
884 * and offers true 64/128 bit hash results.
885 * It provides better speed for systems with vector processing capabilities.
886 */
887
888 /*!
889 * @brief Calculates the 64-bit hash of @p input using xxHash64.
890 *
891 * @param input The block of data to be hashed, at least @p length bytes in size.
892 * @param length The length of @p input, in bytes.
893 * @param seed The 64-bit seed to alter the hash's output predictably.
894 *
895 * @pre
896 * The memory between @p input and @p input + @p length must be valid,
897 * readable, contiguous memory. However, if @p length is `0`, @p input may be
898 * `NULL`. In C++, this also must be *TriviallyCopyable*.
899 *
900 * @return The calculated 64-bit xxHash64 value.
901 *
902 * @see @ref single_shot_example "Single Shot Example" for an example.
903 */
904 XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed);
905
906 /******* Streaming *******/
907 #ifndef XXH_NO_STREAM
908 /*!
909 * @brief The opaque state struct for the XXH64 streaming API.
910 *
911 * @see XXH64_state_s for details.
912 */
913 typedef struct XXH64_state_s XXH64_state_t; /* incomplete type */
914
915 /*!
916 * @brief Allocates an @ref XXH64_state_t.
917 *
918 * @return An allocated pointer of @ref XXH64_state_t on success.
919 * @return `NULL` on failure.
920 *
921 * @note Must be freed with XXH64_freeState().
922 */
923 XXH_PUBLIC_API XXH_MALLOCF XXH64_state_t* XXH64_createState(void);
924
925 /*!
926 * @brief Frees an @ref XXH64_state_t.
927 *
928 * @param statePtr A pointer to an @ref XXH64_state_t allocated with @ref XXH64_createState().
929 *
930 * @return @ref XXH_OK.
931 *
932 * @note @p statePtr must be allocated with XXH64_createState().
933 */
934 XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr);
935
936 /*!
937 * @brief Copies one @ref XXH64_state_t to another.
938 *
939 * @param dst_state The state to copy to.
940 * @param src_state The state to copy from.
941 * @pre
942 * @p dst_state and @p src_state must not be `NULL` and must not overlap.
943 */
944 XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dst_state, const XXH64_state_t* src_state);
945
946 /*!
947 * @brief Resets an @ref XXH64_state_t to begin a new hash.
948 *
949 * @param statePtr The state struct to reset.
950 * @param seed The 64-bit seed to alter the hash result predictably.
951 *
952 * @pre
953 * @p statePtr must not be `NULL`.
954 *
955 * @return @ref XXH_OK on success.
956 * @return @ref XXH_ERROR on failure.
957 *
958 * @note This function resets and seeds a state. Call it before @ref XXH64_update().
959 */
960 XXH_PUBLIC_API XXH_errorcode XXH64_reset (XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed);
961
962 /*!
963 * @brief Consumes a block of @p input to an @ref XXH64_state_t.
964 *
965 * @param statePtr The state struct to update.
966 * @param input The block of data to be hashed, at least @p length bytes in size.
967 * @param length The length of @p input, in bytes.
968 *
969 * @pre
970 * @p statePtr must not be `NULL`.
971 * @pre
972 * The memory between @p input and @p input + @p length must be valid,
973 * readable, contiguous memory. However, if @p length is `0`, @p input may be
974 * `NULL`. In C++, this also must be *TriviallyCopyable*.
975 *
976 * @return @ref XXH_OK on success.
977 * @return @ref XXH_ERROR on failure.
978 *
979 * @note Call this to incrementally consume blocks of data.
980 */
981 XXH_PUBLIC_API XXH_errorcode XXH64_update (XXH_NOESCAPE XXH64_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length);
982
983 /*!
984 * @brief Returns the calculated hash value from an @ref XXH64_state_t.
985 *
986 * @param statePtr The state struct to calculate the hash from.
987 *
988 * @pre
989 * @p statePtr must not be `NULL`.
990 *
991 * @return The calculated 64-bit xxHash64 value from that state.
992 *
993 * @note
994 * Calling XXH64_digest() will not affect @p statePtr, so you can update,
995 * digest, and update again.
996 */
997 XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_digest (XXH_NOESCAPE const XXH64_state_t* statePtr);
998 #endif /* !XXH_NO_STREAM */
999 /******* Canonical representation *******/
1000
1001 /*!
1002 * @brief Canonical (big endian) representation of @ref XXH64_hash_t.
1003 */
1004 typedef struct { unsigned char digest[sizeof(XXH64_hash_t)]; } XXH64_canonical_t;
1005
1006 /*!
1007 * @brief Converts an @ref XXH64_hash_t to a big endian @ref XXH64_canonical_t.
1008 *
1009 * @param dst The @ref XXH64_canonical_t pointer to be stored to.
1010 * @param hash The @ref XXH64_hash_t to be converted.
1011 *
1012 * @pre
1013 * @p dst must not be `NULL`.
1014 *
1015 * @see @ref canonical_representation_example "Canonical Representation Example"
1016 */
1017 XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash);
1018
1019 /*!
1020 * @brief Converts an @ref XXH64_canonical_t to a native @ref XXH64_hash_t.
1021 *
1022 * @param src The @ref XXH64_canonical_t to convert.
1023 *
1024 * @pre
1025 * @p src must not be `NULL`.
1026 *
1027 * @return The converted hash.
1028 *
1029 * @see @ref canonical_representation_example "Canonical Representation Example"
1030 */
1031 XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src);
1032
1033 #ifndef XXH_NO_XXH3
1034
1035 /*!
1036 * @}
1037 * ************************************************************************
1038 * @defgroup XXH3_family XXH3 family
1039 * @ingroup public
1040 * @{
1041 *
1042 * XXH3 is a more recent hash algorithm featuring:
1043 * - Improved speed for both small and large inputs
1044 * - True 64-bit and 128-bit outputs
1045 * - SIMD acceleration
1046 * - Improved 32-bit viability
1047 *
1048 * Speed analysis methodology is explained here:
1049 *
1050 * https://fastcompression.blogspot.com/2019/03/presenting-xxh3.html
1051 *
1052 * Compared to XXH64, expect XXH3 to run approximately
1053 * ~2x faster on large inputs and >3x faster on small ones,
1054 * exact differences vary depending on platform.
1055 *
1056 * XXH3's speed benefits greatly from SIMD and 64-bit arithmetic,
1057 * but does not require it.
1058 * Most 32-bit and 64-bit targets that can run XXH32 smoothly can run XXH3
1059 * at competitive speeds, even without vector support. Further details are
1060 * explained in the implementation.
1061 *
1062 * XXH3 has a fast scalar implementation, but it also includes accelerated SIMD
1063 * implementations for many common platforms:
1064 * - AVX512
1065 * - AVX2
1066 * - SSE2
1067 * - ARM NEON
1068 * - WebAssembly SIMD128
1069 * - POWER8 VSX
1070 * - s390x ZVector
1071 * This can be controlled via the @ref XXH_VECTOR macro, but it automatically
1072 * selects the best version according to predefined macros. For the x86 family, an
1073 * automatic runtime dispatcher is included separately in @ref xxh_x86dispatch.c.
1074 *
1075 * XXH3 implementation is portable:
1076 * it has a generic C90 formulation that can be compiled on any platform,
1077 * all implementations generate exactly the same hash value on all platforms.
1078 * Starting from v0.8.0, it's also labelled "stable", meaning that
1079 * any future version will also generate the same hash value.
1080 *
1081 * XXH3 offers 2 variants, _64bits and _128bits.
1082 *
1083 * When only 64 bits are needed, prefer invoking the _64bits variant, as it
1084 * reduces the amount of mixing, resulting in faster speed on small inputs.
1085 * It's also generally simpler to manipulate a scalar return type than a struct.
1086 *
1087 * The API supports one-shot hashing, streaming mode, and custom secrets.
1088 */
1089 /*-**********************************************************************
1090 * XXH3 64-bit variant
1091 ************************************************************************/
1092
1093 /*!
1094 * @brief Calculates 64-bit unseeded variant of XXH3 hash of @p input.
1095 *
1096 * @param input The block of data to be hashed, at least @p length bytes in size.
1097 * @param length The length of @p input, in bytes.
1098 *
1099 * @pre
1100 * The memory between @p input and @p input + @p length must be valid,
1101 * readable, contiguous memory. However, if @p length is `0`, @p input may be
1102 * `NULL`. In C++, this also must be *TriviallyCopyable*.
1103 *
1104 * @return The calculated 64-bit XXH3 hash value.
1105 *
1106 * @note
1107 * This is equivalent to @ref XXH3_64bits_withSeed() with a seed of `0`, however
1108 * it may have slightly better performance due to constant propagation of the
1109 * defaults.
1110 *
1111 * @see
1112 * XXH3_64bits_withSeed(), XXH3_64bits_withSecret(): other seeding variants
1113 * @see @ref single_shot_example "Single Shot Example" for an example.
1114 */
1115 XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length);
1116
1117 /*!
1118 * @brief Calculates 64-bit seeded variant of XXH3 hash of @p input.
1119 *
1120 * @param input The block of data to be hashed, at least @p length bytes in size.
1121 * @param length The length of @p input, in bytes.
1122 * @param seed The 64-bit seed to alter the hash result predictably.
1123 *
1124 * @pre
1125 * The memory between @p input and @p input + @p length must be valid,
1126 * readable, contiguous memory. However, if @p length is `0`, @p input may be
1127 * `NULL`. In C++, this also must be *TriviallyCopyable*.
1128 *
1129 * @return The calculated 64-bit XXH3 hash value.
1130 *
1131 * @note
1132 * seed == 0 produces the same results as @ref XXH3_64bits().
1133 *
1134 * This variant generates a custom secret on the fly based on default secret
1135 * altered using the @p seed value.
1136 *
1137 * While this operation is decently fast, note that it's not completely free.
1138 *
1139 * @see @ref single_shot_example "Single Shot Example" for an example.
1140 */
1141 XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSeed(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed);
1142
1143 /*!
1144 * The bare minimum size for a custom secret.
1145 *
1146 * @see
1147 * XXH3_64bits_withSecret(), XXH3_64bits_reset_withSecret(),
1148 * XXH3_128bits_withSecret(), XXH3_128bits_reset_withSecret().
1149 */
1150 #define XXH3_SECRET_SIZE_MIN 136
1151
1152 /*!
1153 * @brief Calculates 64-bit variant of XXH3 with a custom "secret".
1154 *
1155 * @param data The block of data to be hashed, at least @p len bytes in size.
1156 * @param len The length of @p data, in bytes.
1157 * @param secret The secret data.
1158 * @param secretSize The length of @p secret, in bytes.
1159 *
1160 * @return The calculated 64-bit XXH3 hash value.
1161 *
1162 * @pre
1163 * The memory between @p data and @p data + @p len must be valid,
1164 * readable, contiguous memory. However, if @p length is `0`, @p data may be
1165 * `NULL`. In C++, this also must be *TriviallyCopyable*.
1166 *
1167 * It's possible to provide any blob of bytes as a "secret" to generate the hash.
1168 * This makes it more difficult for an external actor to prepare an intentional collision.
1169 * The main condition is that @p secretSize *must* be large enough (>= @ref XXH3_SECRET_SIZE_MIN).
1170 * However, the quality of the secret impacts the dispersion of the hash algorithm.
1171 * Therefore, the secret _must_ look like a bunch of random bytes.
1172 * Avoid "trivial" or structured data such as repeated sequences or a text document.
1173 * Whenever in doubt about the "randomness" of the blob of bytes,
1174 * consider employing @ref XXH3_generateSecret() instead (see below).
1175 * It will generate a proper high entropy secret derived from the blob of bytes.
1176 * Another advantage of using XXH3_generateSecret() is that
1177 * it guarantees that all bits within the initial blob of bytes
1178 * will impact every bit of the output.
1179 * This is not necessarily the case when using the blob of bytes directly
1180 * because, when hashing _small_ inputs, only a portion of the secret is employed.
1181 *
1182 * @see @ref single_shot_example "Single Shot Example" for an example.
1183 */
1184 XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize);
1185
1186
1187 /******* Streaming *******/
1188 #ifndef XXH_NO_STREAM
1189 /*
1190 * Streaming requires state maintenance.
1191 * This operation costs memory and CPU.
1192 * As a consequence, streaming is slower than one-shot hashing.
1193 * For better performance, prefer one-shot functions whenever applicable.
1194 */
1195
1196 /*!
1197 * @brief The opaque state struct for the XXH3 streaming API.
1198 *
1199 * @see XXH3_state_s for details.
1200 */
1201 typedef struct XXH3_state_s XXH3_state_t;
1202 XXH_PUBLIC_API XXH_MALLOCF XXH3_state_t* XXH3_createState(void);
1203 XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr);
1204
1205 /*!
1206 * @brief Copies one @ref XXH3_state_t to another.
1207 *
1208 * @param dst_state The state to copy to.
1209 * @param src_state The state to copy from.
1210 * @pre
1211 * @p dst_state and @p src_state must not be `NULL` and must not overlap.
1212 */
1213 XXH_PUBLIC_API void XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOESCAPE const XXH3_state_t* src_state);
1214
1215 /*!
1216 * @brief Resets an @ref XXH3_state_t to begin a new hash.
1217 *
1218 * @param statePtr The state struct to reset.
1219 *
1220 * @pre
1221 * @p statePtr must not be `NULL`.
1222 *
1223 * @return @ref XXH_OK on success.
1224 * @return @ref XXH_ERROR on failure.
1225 *
1226 * @note
1227 * - This function resets `statePtr` and generate a secret with default parameters.
1228 * - Call this function before @ref XXH3_64bits_update().
1229 * - Digest will be equivalent to `XXH3_64bits()`.
1230 *
1231 */
1232 XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr);
1233
1234 /*!
1235 * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash.
1236 *
1237 * @param statePtr The state struct to reset.
1238 * @param seed The 64-bit seed to alter the hash result predictably.
1239 *
1240 * @pre
1241 * @p statePtr must not be `NULL`.
1242 *
1243 * @return @ref XXH_OK on success.
1244 * @return @ref XXH_ERROR on failure.
1245 *
1246 * @note
1247 * - This function resets `statePtr` and generate a secret from `seed`.
1248 * - Call this function before @ref XXH3_64bits_update().
1249 * - Digest will be equivalent to `XXH3_64bits_withSeed()`.
1250 *
1251 */
1252 XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed);
1253
1254 /*!
1255 * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash.
1256 *
1257 * @param statePtr The state struct to reset.
1258 * @param secret The secret data.
1259 * @param secretSize The length of @p secret, in bytes.
1260 *
1261 * @pre
1262 * @p statePtr must not be `NULL`.
1263 *
1264 * @return @ref XXH_OK on success.
1265 * @return @ref XXH_ERROR on failure.
1266 *
1267 * @note
1268 * `secret` is referenced, it _must outlive_ the hash streaming session.
1269 *
1270 * Similar to one-shot API, `secretSize` must be >= @ref XXH3_SECRET_SIZE_MIN,
1271 * and the quality of produced hash values depends on secret's entropy
1272 * (secret's content should look like a bunch of random bytes).
1273 * When in doubt about the randomness of a candidate `secret`,
1274 * consider employing `XXH3_generateSecret()` instead (see below).
1275 */
1276 XXH_PUBLIC_API XXH_errorcode XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize);
1277
1278 /*!
1279 * @brief Consumes a block of @p input to an @ref XXH3_state_t.
1280 *
1281 * @param statePtr The state struct to update.
1282 * @param input The block of data to be hashed, at least @p length bytes in size.
1283 * @param length The length of @p input, in bytes.
1284 *
1285 * @pre
1286 * @p statePtr must not be `NULL`.
1287 * @pre
1288 * The memory between @p input and @p input + @p length must be valid,
1289 * readable, contiguous memory. However, if @p length is `0`, @p input may be
1290 * `NULL`. In C++, this also must be *TriviallyCopyable*.
1291 *
1292 * @return @ref XXH_OK on success.
1293 * @return @ref XXH_ERROR on failure.
1294 *
1295 * @note Call this to incrementally consume blocks of data.
1296 */
1297 XXH_PUBLIC_API XXH_errorcode XXH3_64bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length);
1298
1299 /*!
1300 * @brief Returns the calculated XXH3 64-bit hash value from an @ref XXH3_state_t.
1301 *
1302 * @param statePtr The state struct to calculate the hash from.
1303 *
1304 * @pre
1305 * @p statePtr must not be `NULL`.
1306 *
1307 * @return The calculated XXH3 64-bit hash value from that state.
1308 *
1309 * @note
1310 * Calling XXH3_64bits_digest() will not affect @p statePtr, so you can update,
1311 * digest, and update again.
1312 */
1313 XXH_PUBLIC_API XXH_PUREF XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr);
1314 #endif /* !XXH_NO_STREAM */
1315
1316 /* note : canonical representation of XXH3 is the same as XXH64
1317 * since they both produce XXH64_hash_t values */
1318
1319
1320 /*-**********************************************************************
1321 * XXH3 128-bit variant
1322 ************************************************************************/
1323
1324 /*!
1325 * @brief The return value from 128-bit hashes.
1326 *
1327 * Stored in little endian order, although the fields themselves are in native
1328 * endianness.
1329 */
1330 typedef struct {
1331 XXH64_hash_t low64; /*!< `value & 0xFFFFFFFFFFFFFFFF` */
1332 XXH64_hash_t high64; /*!< `value >> 64` */
1333 } XXH128_hash_t;
1334
1335 /*!
1336 * @brief Calculates 128-bit unseeded variant of XXH3 of @p data.
1337 *
1338 * @param data The block of data to be hashed, at least @p length bytes in size.
1339 * @param len The length of @p data, in bytes.
1340 *
1341 * @return The calculated 128-bit variant of XXH3 value.
1342 *
1343 * The 128-bit variant of XXH3 has more strength, but it has a bit of overhead
1344 * for shorter inputs.
1345 *
1346 * This is equivalent to @ref XXH3_128bits_withSeed() with a seed of `0`, however
1347 * it may have slightly better performance due to constant propagation of the
1348 * defaults.
1349 *
1350 * @see XXH3_128bits_withSeed(), XXH3_128bits_withSecret(): other seeding variants
1351 * @see @ref single_shot_example "Single Shot Example" for an example.
1352 */
1353 XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void* data, size_t len);
1354 /*! @brief Calculates 128-bit seeded variant of XXH3 hash of @p data.
1355 *
1356 * @param data The block of data to be hashed, at least @p length bytes in size.
1357 * @param len The length of @p data, in bytes.
1358 * @param seed The 64-bit seed to alter the hash result predictably.
1359 *
1360 * @return The calculated 128-bit variant of XXH3 value.
1361 *
1362 * @note
1363 * seed == 0 produces the same results as @ref XXH3_64bits().
1364 *
1365 * This variant generates a custom secret on the fly based on default secret
1366 * altered using the @p seed value.
1367 *
1368 * While this operation is decently fast, note that it's not completely free.
1369 *
1370 * @see XXH3_128bits(), XXH3_128bits_withSecret(): other seeding variants
1371 * @see @ref single_shot_example "Single Shot Example" for an example.
1372 */
1373 XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSeed(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed);
1374 /*!
1375 * @brief Calculates 128-bit variant of XXH3 with a custom "secret".
1376 *
1377 * @param data The block of data to be hashed, at least @p len bytes in size.
1378 * @param len The length of @p data, in bytes.
1379 * @param secret The secret data.
1380 * @param secretSize The length of @p secret, in bytes.
1381 *
1382 * @return The calculated 128-bit variant of XXH3 value.
1383 *
1384 * It's possible to provide any blob of bytes as a "secret" to generate the hash.
1385 * This makes it more difficult for an external actor to prepare an intentional collision.
1386 * The main condition is that @p secretSize *must* be large enough (>= @ref XXH3_SECRET_SIZE_MIN).
1387 * However, the quality of the secret impacts the dispersion of the hash algorithm.
1388 * Therefore, the secret _must_ look like a bunch of random bytes.
1389 * Avoid "trivial" or structured data such as repeated sequences or a text document.
1390 * Whenever in doubt about the "randomness" of the blob of bytes,
1391 * consider employing @ref XXH3_generateSecret() instead (see below).
1392 * It will generate a proper high entropy secret derived from the blob of bytes.
1393 * Another advantage of using XXH3_generateSecret() is that
1394 * it guarantees that all bits within the initial blob of bytes
1395 * will impact every bit of the output.
1396 * This is not necessarily the case when using the blob of bytes directly
1397 * because, when hashing _small_ inputs, only a portion of the secret is employed.
1398 *
1399 * @see @ref single_shot_example "Single Shot Example" for an example.
1400 */
1401 XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_withSecret(XXH_NOESCAPE const void* data, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize);
1402
1403 /******* Streaming *******/
1404 #ifndef XXH_NO_STREAM
1405 /*
1406 * Streaming requires state maintenance.
1407 * This operation costs memory and CPU.
1408 * As a consequence, streaming is slower than one-shot hashing.
1409 * For better performance, prefer one-shot functions whenever applicable.
1410 *
1411 * XXH3_128bits uses the same XXH3_state_t as XXH3_64bits().
1412 * Use already declared XXH3_createState() and XXH3_freeState().
1413 *
1414 * All reset and streaming functions have same meaning as their 64-bit counterpart.
1415 */
1416
1417 /*!
1418 * @brief Resets an @ref XXH3_state_t to begin a new hash.
1419 *
1420 * @param statePtr The state struct to reset.
1421 *
1422 * @pre
1423 * @p statePtr must not be `NULL`.
1424 *
1425 * @return @ref XXH_OK on success.
1426 * @return @ref XXH_ERROR on failure.
1427 *
1428 * @note
1429 * - This function resets `statePtr` and generate a secret with default parameters.
1430 * - Call it before @ref XXH3_128bits_update().
1431 * - Digest will be equivalent to `XXH3_128bits()`.
1432 */
1433 XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr);
1434
1435 /*!
1436 * @brief Resets an @ref XXH3_state_t with 64-bit seed to begin a new hash.
1437 *
1438 * @param statePtr The state struct to reset.
1439 * @param seed The 64-bit seed to alter the hash result predictably.
1440 *
1441 * @pre
1442 * @p statePtr must not be `NULL`.
1443 *
1444 * @return @ref XXH_OK on success.
1445 * @return @ref XXH_ERROR on failure.
1446 *
1447 * @note
1448 * - This function resets `statePtr` and generate a secret from `seed`.
1449 * - Call it before @ref XXH3_128bits_update().
1450 * - Digest will be equivalent to `XXH3_128bits_withSeed()`.
1451 */
1452 XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed);
1453 /*!
1454 * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash.
1455 *
1456 * @param statePtr The state struct to reset.
1457 * @param secret The secret data.
1458 * @param secretSize The length of @p secret, in bytes.
1459 *
1460 * @pre
1461 * @p statePtr must not be `NULL`.
1462 *
1463 * @return @ref XXH_OK on success.
1464 * @return @ref XXH_ERROR on failure.
1465 *
1466 * `secret` is referenced, it _must outlive_ the hash streaming session.
1467 * Similar to one-shot API, `secretSize` must be >= @ref XXH3_SECRET_SIZE_MIN,
1468 * and the quality of produced hash values depends on secret's entropy
1469 * (secret's content should look like a bunch of random bytes).
1470 * When in doubt about the randomness of a candidate `secret`,
1471 * consider employing `XXH3_generateSecret()` instead (see below).
1472 */
1473 XXH_PUBLIC_API XXH_errorcode XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize);
1474
1475 /*!
1476 * @brief Consumes a block of @p input to an @ref XXH3_state_t.
1477 *
1478 * Call this to incrementally consume blocks of data.
1479 *
1480 * @param statePtr The state struct to update.
1481 * @param input The block of data to be hashed, at least @p length bytes in size.
1482 * @param length The length of @p input, in bytes.
1483 *
1484 * @pre
1485 * @p statePtr must not be `NULL`.
1486 *
1487 * @return @ref XXH_OK on success.
1488 * @return @ref XXH_ERROR on failure.
1489 *
1490 * @note
1491 * The memory between @p input and @p input + @p length must be valid,
1492 * readable, contiguous memory. However, if @p length is `0`, @p input may be
1493 * `NULL`. In C++, this also must be *TriviallyCopyable*.
1494 *
1495 */
1496 XXH_PUBLIC_API XXH_errorcode XXH3_128bits_update (XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* input, size_t length);
1497
1498 /*!
1499 * @brief Returns the calculated XXH3 128-bit hash value from an @ref XXH3_state_t.
1500 *
1501 * @param statePtr The state struct to calculate the hash from.
1502 *
1503 * @pre
1504 * @p statePtr must not be `NULL`.
1505 *
1506 * @return The calculated XXH3 128-bit hash value from that state.
1507 *
1508 * @note
1509 * Calling XXH3_128bits_digest() will not affect @p statePtr, so you can update,
1510 * digest, and update again.
1511 *
1512 */
1513 XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_t* statePtr);
1514 #endif /* !XXH_NO_STREAM */
1515
1516 /* Following helper functions make it possible to compare XXH128_hast_t values.
1517 * Since XXH128_hash_t is a structure, this capability is not offered by the language.
1518 * Note: For better performance, these functions can be inlined using XXH_INLINE_ALL */
1519
1520 /*!
1521 * @brief Check equality of two XXH128_hash_t values
1522 *
1523 * @param h1 The 128-bit hash value.
1524 * @param h2 Another 128-bit hash value.
1525 *
1526 * @return `1` if `h1` and `h2` are equal.
1527 * @return `0` if they are not.
1528 */
1529 XXH_PUBLIC_API XXH_PUREF int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2);
1530
1531 /*!
1532 * @brief Compares two @ref XXH128_hash_t
1533 *
1534 * This comparator is compatible with stdlib's `qsort()`/`bsearch()`.
1535 *
1536 * @param h128_1 Left-hand side value
1537 * @param h128_2 Right-hand side value
1538 *
1539 * @return >0 if @p h128_1 > @p h128_2
1540 * @return =0 if @p h128_1 == @p h128_2
1541 * @return <0 if @p h128_1 < @p h128_2
1542 */
1543 XXH_PUBLIC_API XXH_PUREF int XXH128_cmp(XXH_NOESCAPE const void* h128_1, XXH_NOESCAPE const void* h128_2);
1544
1545
1546 /******* Canonical representation *******/
1547 typedef struct { unsigned char digest[sizeof(XXH128_hash_t)]; } XXH128_canonical_t;
1548
1549
1550 /*!
1551 * @brief Converts an @ref XXH128_hash_t to a big endian @ref XXH128_canonical_t.
1552 *
1553 * @param dst The @ref XXH128_canonical_t pointer to be stored to.
1554 * @param hash The @ref XXH128_hash_t to be converted.
1555 *
1556 * @pre
1557 * @p dst must not be `NULL`.
1558 * @see @ref canonical_representation_example "Canonical Representation Example"
1559 */
1560 XXH_PUBLIC_API void XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* dst, XXH128_hash_t hash);
1561
1562 /*!
1563 * @brief Converts an @ref XXH128_canonical_t to a native @ref XXH128_hash_t.
1564 *
1565 * @param src The @ref XXH128_canonical_t to convert.
1566 *
1567 * @pre
1568 * @p src must not be `NULL`.
1569 *
1570 * @return The converted hash.
1571 * @see @ref canonical_representation_example "Canonical Representation Example"
1572 */
1573 XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t* src);
1574
1575
1576 #endif /* !XXH_NO_XXH3 */
1577
1578 #if defined (__cplusplus)
1579 } /* extern "C" */
1580 #endif
1581
1582 #endif /* XXH_NO_LONG_LONG */
1583
1584 /*!
1585 * @}
1586 */
1587 #endif /* XXHASH_H_5627135585666179 */
1588
1589
1590
1591 #if defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742)
1592 #define XXHASH_H_STATIC_13879238742
1593 /* ****************************************************************************
1594 * This section contains declarations which are not guaranteed to remain stable.
1595 * They may change in future versions, becoming incompatible with a different
1596 * version of the library.
1597 * These declarations should only be used with static linking.
1598 * Never use them in association with dynamic linking!
1599 ***************************************************************************** */
1600
1601 /*
1602 * These definitions are only present to allow static allocation
1603 * of XXH states, on stack or in a struct, for example.
1604 * Never **ever** access their members directly.
1605 */
1606
1607 /*!
1608 * @internal
1609 * @brief Structure for XXH32 streaming API.
1610 *
1611 * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY,
1612 * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is
1613 * an opaque type. This allows fields to safely be changed.
1614 *
1615 * Typedef'd to @ref XXH32_state_t.
1616 * Do not access the members of this struct directly.
1617 * @see XXH64_state_s, XXH3_state_s
1618 */
1619 struct XXH32_state_s {
1620 XXH32_hash_t total_len_32; /*!< Total length hashed, modulo 2^32 */
1621 XXH32_hash_t large_len; /*!< Whether the hash is >= 16 (handles @ref total_len_32 overflow) */
1622 XXH32_hash_t v[4]; /*!< Accumulator lanes */
1623 XXH32_hash_t mem32[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[16]. */
1624 XXH32_hash_t memsize; /*!< Amount of data in @ref mem32 */
1625 XXH32_hash_t reserved; /*!< Reserved field. Do not read nor write to it. */
1626 }; /* typedef'd to XXH32_state_t */
1627
1628
1629 #ifndef XXH_NO_LONG_LONG /* defined when there is no 64-bit support */
1630
1631 /*!
1632 * @internal
1633 * @brief Structure for XXH64 streaming API.
1634 *
1635 * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY,
1636 * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined. Otherwise it is
1637 * an opaque type. This allows fields to safely be changed.
1638 *
1639 * Typedef'd to @ref XXH64_state_t.
1640 * Do not access the members of this struct directly.
1641 * @see XXH32_state_s, XXH3_state_s
1642 */
1643 struct XXH64_state_s {
1644 XXH64_hash_t total_len; /*!< Total length hashed. This is always 64-bit. */
1645 XXH64_hash_t v[4]; /*!< Accumulator lanes */
1646 XXH64_hash_t mem64[4]; /*!< Internal buffer for partial reads. Treated as unsigned char[32]. */
1647 XXH32_hash_t memsize; /*!< Amount of data in @ref mem64 */
1648 XXH32_hash_t reserved32; /*!< Reserved field, needed for padding anyways*/
1649 XXH64_hash_t reserved64; /*!< Reserved field. Do not read or write to it. */
1650 }; /* typedef'd to XXH64_state_t */
1651
1652 #ifndef XXH_NO_XXH3
1653
1654 #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* >= C11 */
1655 # include <stdalign.h>
1656 # define XXH_ALIGN(n) alignas(n)
1657 #elif defined(__cplusplus) && (__cplusplus >= 201103L) /* >= C++11 */
1658 /* In C++ alignas() is a keyword */
1659 # define XXH_ALIGN(n) alignas(n)
1660 #elif defined(__GNUC__)
1661 # define XXH_ALIGN(n) __attribute__ ((aligned(n)))
1662 #elif defined(_MSC_VER)
1663 # define XXH_ALIGN(n) __declspec(align(n))
1664 #else
1665 # define XXH_ALIGN(n) /* disabled */
1666 #endif
1667
1668 /* Old GCC versions only accept the attribute after the type in structures. */
1669 #if !(defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L)) /* C11+ */ \
1670 && ! (defined(__cplusplus) && (__cplusplus >= 201103L)) /* >= C++11 */ \
1671 && defined(__GNUC__)
1672 # define XXH_ALIGN_MEMBER(align, type) type XXH_ALIGN(align)
1673 #else
1674 # define XXH_ALIGN_MEMBER(align, type) XXH_ALIGN(align) type
1675 #endif
1676
1677 /*!
1678 * @brief The size of the internal XXH3 buffer.
1679 *
1680 * This is the optimal update size for incremental hashing.
1681 *
1682 * @see XXH3_64b_update(), XXH3_128b_update().
1683 */
1684 #define XXH3_INTERNALBUFFER_SIZE 256
1685
1686 /*!
1687 * @internal
1688 * @brief Default size of the secret buffer (and @ref XXH3_kSecret).
1689 *
1690 * This is the size used in @ref XXH3_kSecret and the seeded functions.
1691 *
1692 * Not to be confused with @ref XXH3_SECRET_SIZE_MIN.
1693 */
1694 #define XXH3_SECRET_DEFAULT_SIZE 192
1695
1696 /*!
1697 * @internal
1698 * @brief Structure for XXH3 streaming API.
1699 *
1700 * @note This is only defined when @ref XXH_STATIC_LINKING_ONLY,
1701 * @ref XXH_INLINE_ALL, or @ref XXH_IMPLEMENTATION is defined.
1702 * Otherwise it is an opaque type.
1703 * Never use this definition in combination with dynamic library.
1704 * This allows fields to safely be changed in the future.
1705 *
1706 * @note ** This structure has a strict alignment requirement of 64 bytes!! **
1707 * Do not allocate this with `malloc()` or `new`,
1708 * it will not be sufficiently aligned.
1709 * Use @ref XXH3_createState() and @ref XXH3_freeState(), or stack allocation.
1710 *
1711 * Typedef'd to @ref XXH3_state_t.
1712 * Do never access the members of this struct directly.
1713 *
1714 * @see XXH3_INITSTATE() for stack initialization.
1715 * @see XXH3_createState(), XXH3_freeState().
1716 * @see XXH32_state_s, XXH64_state_s
1717 */
1718 struct XXH3_state_s {
1719 XXH_ALIGN_MEMBER(64, XXH64_hash_t acc[8]);
1720 /*!< The 8 accumulators. See @ref XXH32_state_s::v and @ref XXH64_state_s::v */
1721 XXH_ALIGN_MEMBER(64, unsigned char customSecret[XXH3_SECRET_DEFAULT_SIZE]);
1722 /*!< Used to store a custom secret generated from a seed. */
1723 XXH_ALIGN_MEMBER(64, unsigned char buffer[XXH3_INTERNALBUFFER_SIZE]);
1724 /*!< The internal buffer. @see XXH32_state_s::mem32 */
1725 XXH32_hash_t bufferedSize;
1726 /*!< The amount of memory in @ref buffer, @see XXH32_state_s::memsize */
1727 XXH32_hash_t useSeed;
1728 /*!< Reserved field. Needed for padding on 64-bit. */
1729 size_t nbStripesSoFar;
1730 /*!< Number or stripes processed. */
1731 XXH64_hash_t totalLen;
1732 /*!< Total length hashed. 64-bit even on 32-bit targets. */
1733 size_t nbStripesPerBlock;
1734 /*!< Number of stripes per block. */
1735 size_t secretLimit;
1736 /*!< Size of @ref customSecret or @ref extSecret */
1737 XXH64_hash_t seed;
1738 /*!< Seed for _withSeed variants. Must be zero otherwise, @see XXH3_INITSTATE() */
1739 XXH64_hash_t reserved64;
1740 /*!< Reserved field. */
1741 const unsigned char* extSecret;
1742 /*!< Reference to an external secret for the _withSecret variants, NULL
1743 * for other variants. */
1744 /* note: there may be some padding at the end due to alignment on 64 bytes */
1745 }; /* typedef'd to XXH3_state_t */
1746
1747 #undef XXH_ALIGN_MEMBER
1748
1749 /*!
1750 * @brief Initializes a stack-allocated `XXH3_state_s`.
1751 *
1752 * When the @ref XXH3_state_t structure is merely emplaced on stack,
1753 * it should be initialized with XXH3_INITSTATE() or a memset()
1754 * in case its first reset uses XXH3_NNbits_reset_withSeed().
1755 * This init can be omitted if the first reset uses default or _withSecret mode.
1756 * This operation isn't necessary when the state is created with XXH3_createState().
1757 * Note that this doesn't prepare the state for a streaming operation,
1758 * it's still necessary to use XXH3_NNbits_reset*() afterwards.
1759 */
1760 #define XXH3_INITSTATE(XXH3_state_ptr) \
1761 do { \
1762 XXH3_state_t* tmp_xxh3_state_ptr = (XXH3_state_ptr); \
1763 tmp_xxh3_state_ptr->seed = 0; \
1764 tmp_xxh3_state_ptr->extSecret = NULL; \
1765 } while(0)
1766
1767
1768 #if defined (__cplusplus)
1769 extern "C" {
1770 #endif
1771
1772 /*!
1773 * @brief Calculates the 128-bit hash of @p data using XXH3.
1774 *
1775 * @param data The block of data to be hashed, at least @p len bytes in size.
1776 * @param len The length of @p data, in bytes.
1777 * @param seed The 64-bit seed to alter the hash's output predictably.
1778 *
1779 * @pre
1780 * The memory between @p data and @p data + @p len must be valid,
1781 * readable, contiguous memory. However, if @p len is `0`, @p data may be
1782 * `NULL`. In C++, this also must be *TriviallyCopyable*.
1783 *
1784 * @return The calculated 128-bit XXH3 value.
1785 *
1786 * @see @ref single_shot_example "Single Shot Example" for an example.
1787 */
1788 XXH_PUBLIC_API XXH_PUREF XXH128_hash_t XXH128(XXH_NOESCAPE const void* data, size_t len, XXH64_hash_t seed);
1789
1790
1791 /* === Experimental API === */
1792 /* Symbols defined below must be considered tied to a specific library version. */
1793
1794 /*!
1795 * @brief Derive a high-entropy secret from any user-defined content, named customSeed.
1796 *
1797 * @param secretBuffer A writable buffer for derived high-entropy secret data.
1798 * @param secretSize Size of secretBuffer, in bytes. Must be >= XXH3_SECRET_DEFAULT_SIZE.
1799 * @param customSeed A user-defined content.
1800 * @param customSeedSize Size of customSeed, in bytes.
1801 *
1802 * @return @ref XXH_OK on success.
1803 * @return @ref XXH_ERROR on failure.
1804 *
1805 * The generated secret can be used in combination with `*_withSecret()` functions.
1806 * The `_withSecret()` variants are useful to provide a higher level of protection
1807 * than 64-bit seed, as it becomes much more difficult for an external actor to
1808 * guess how to impact the calculation logic.
1809 *
1810 * The function accepts as input a custom seed of any length and any content,
1811 * and derives from it a high-entropy secret of length @p secretSize into an
1812 * already allocated buffer @p secretBuffer.
1813 *
1814 * The generated secret can then be used with any `*_withSecret()` variant.
1815 * The functions @ref XXH3_128bits_withSecret(), @ref XXH3_64bits_withSecret(),
1816 * @ref XXH3_128bits_reset_withSecret() and @ref XXH3_64bits_reset_withSecret()
1817 * are part of this list. They all accept a `secret` parameter
1818 * which must be large enough for implementation reasons (>= @ref XXH3_SECRET_SIZE_MIN)
1819 * _and_ feature very high entropy (consist of random-looking bytes).
1820 * These conditions can be a high bar to meet, so @ref XXH3_generateSecret() can
1821 * be employed to ensure proper quality.
1822 *
1823 * @p customSeed can be anything. It can have any size, even small ones,
1824 * and its content can be anything, even "poor entropy" sources such as a bunch
1825 * of zeroes. The resulting `secret` will nonetheless provide all required qualities.
1826 *
1827 * @pre
1828 * - @p secretSize must be >= @ref XXH3_SECRET_SIZE_MIN
1829 * - When @p customSeedSize > 0, supplying NULL as customSeed is undefined behavior.
1830 *
1831 * Example code:
1832 * @code{.c}
1833 * #include <stdio.h>
1834 * #include <stdlib.h>
1835 * #include <string.h>
1836 * #define XXH_STATIC_LINKING_ONLY // expose unstable API
1837 * #include "xxhash.h"
1838 * // Hashes argv[2] using the entropy from argv[1].
1839 * int main(int argc, char* argv[])
1840 * {
1841 * char secret[XXH3_SECRET_SIZE_MIN];
1842 * if (argv != 3) { return 1; }
1843 * XXH3_generateSecret(secret, sizeof(secret), argv[1], strlen(argv[1]));
1844 * XXH64_hash_t h = XXH3_64bits_withSecret(
1845 * argv[2], strlen(argv[2]),
1846 * secret, sizeof(secret)
1847 * );
1848 * printf("%016llx\n", (unsigned long long) h);
1849 * }
1850 * @endcode
1851 */
1852 XXH_PUBLIC_API XXH_errorcode XXH3_generateSecret(XXH_NOESCAPE void* secretBuffer, size_t secretSize, XXH_NOESCAPE const void* customSeed, size_t customSeedSize);
1853
1854 /*!
1855 * @brief Generate the same secret as the _withSeed() variants.
1856 *
1857 * @param secretBuffer A writable buffer of @ref XXH3_SECRET_SIZE_MIN bytes
1858 * @param seed The 64-bit seed to alter the hash result predictably.
1859 *
1860 * The generated secret can be used in combination with
1861 *`*_withSecret()` and `_withSecretandSeed()` variants.
1862 *
1863 * Example C++ `std::string` hash class:
1864 * @code{.cpp}
1865 * #include <string>
1866 * #define XXH_STATIC_LINKING_ONLY // expose unstable API
1867 * #include "xxhash.h"
1868 * // Slow, seeds each time
1869 * class HashSlow {
1870 * XXH64_hash_t seed;
1871 * public:
1872 * HashSlow(XXH64_hash_t s) : seed{s} {}
1873 * size_t operator()(const std::string& x) const {
1874 * return size_t{XXH3_64bits_withSeed(x.c_str(), x.length(), seed)};
1875 * }
1876 * };
1877 * // Fast, caches the seeded secret for future uses.
1878 * class HashFast {
1879 * unsigned char secret[XXH3_SECRET_SIZE_MIN];
1880 * public:
1881 * HashFast(XXH64_hash_t s) {
1882 * XXH3_generateSecret_fromSeed(secret, seed);
1883 * }
1884 * size_t operator()(const std::string& x) const {
1885 * return size_t{
1886 * XXH3_64bits_withSecret(x.c_str(), x.length(), secret, sizeof(secret))
1887 * };
1888 * }
1889 * };
1890 * @endcode
1891 */
1892 XXH_PUBLIC_API void XXH3_generateSecret_fromSeed(XXH_NOESCAPE void* secretBuffer, XXH64_hash_t seed);
1893
1894 /*!
1895 * @brief Calculates 64/128-bit seeded variant of XXH3 hash of @p data.
1896 *
1897 * @param data The block of data to be hashed, at least @p len bytes in size.
1898 * @param len The length of @p data, in bytes.
1899 * @param secret The secret data.
1900 * @param secretSize The length of @p secret, in bytes.
1901 * @param seed The 64-bit seed to alter the hash result predictably.
1902 *
1903 * These variants generate hash values using either
1904 * @p seed for "short" keys (< @ref XXH3_MIDSIZE_MAX = 240 bytes)
1905 * or @p secret for "large" keys (>= @ref XXH3_MIDSIZE_MAX).
1906 *
1907 * This generally benefits speed, compared to `_withSeed()` or `_withSecret()`.
1908 * `_withSeed()` has to generate the secret on the fly for "large" keys.
1909 * It's fast, but can be perceptible for "not so large" keys (< 1 KB).
1910 * `_withSecret()` has to generate the masks on the fly for "small" keys,
1911 * which requires more instructions than _withSeed() variants.
1912 * Therefore, _withSecretandSeed variant combines the best of both worlds.
1913 *
1914 * When @p secret has been generated by XXH3_generateSecret_fromSeed(),
1915 * this variant produces *exactly* the same results as `_withSeed()` variant,
1916 * hence offering only a pure speed benefit on "large" input,
1917 * by skipping the need to regenerate the secret for every large input.
1918 *
1919 * Another usage scenario is to hash the secret to a 64-bit hash value,
1920 * for example with XXH3_64bits(), which then becomes the seed,
1921 * and then employ both the seed and the secret in _withSecretandSeed().
1922 * On top of speed, an added benefit is that each bit in the secret
1923 * has a 50% chance to swap each bit in the output, via its impact to the seed.
1924 *
1925 * This is not guaranteed when using the secret directly in "small data" scenarios,
1926 * because only portions of the secret are employed for small data.
1927 */
1928 XXH_PUBLIC_API XXH_PUREF XXH64_hash_t
1929 XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* data, size_t len,
1930 XXH_NOESCAPE const void* secret, size_t secretSize,
1931 XXH64_hash_t seed);
1932 /*!
1933 * @brief Calculates 128-bit seeded variant of XXH3 hash of @p data.
1934 *
1935 * @param input The block of data to be hashed, at least @p len bytes in size.
1936 * @param length The length of @p data, in bytes.
1937 * @param secret The secret data.
1938 * @param secretSize The length of @p secret, in bytes.
1939 * @param seed64 The 64-bit seed to alter the hash result predictably.
1940 *
1941 * @return @ref XXH_OK on success.
1942 * @return @ref XXH_ERROR on failure.
1943 *
1944 * @see XXH3_64bits_withSecretandSeed()
1945 */
1946 XXH_PUBLIC_API XXH_PUREF XXH128_hash_t
1947 XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length,
1948 XXH_NOESCAPE const void* secret, size_t secretSize,
1949 XXH64_hash_t seed64);
1950 #ifndef XXH_NO_STREAM
1951 /*!
1952 * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash.
1953 *
1954 * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState().
1955 * @param secret The secret data.
1956 * @param secretSize The length of @p secret, in bytes.
1957 * @param seed64 The 64-bit seed to alter the hash result predictably.
1958 *
1959 * @return @ref XXH_OK on success.
1960 * @return @ref XXH_ERROR on failure.
1961 *
1962 * @see XXH3_64bits_withSecretandSeed()
1963 */
1964 XXH_PUBLIC_API XXH_errorcode
1965 XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr,
1966 XXH_NOESCAPE const void* secret, size_t secretSize,
1967 XXH64_hash_t seed64);
1968 /*!
1969 * @brief Resets an @ref XXH3_state_t with secret data to begin a new hash.
1970 *
1971 * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState().
1972 * @param secret The secret data.
1973 * @param secretSize The length of @p secret, in bytes.
1974 * @param seed64 The 64-bit seed to alter the hash result predictably.
1975 *
1976 * @return @ref XXH_OK on success.
1977 * @return @ref XXH_ERROR on failure.
1978 *
1979 * @see XXH3_64bits_withSecretandSeed()
1980 */
1981 XXH_PUBLIC_API XXH_errorcode
1982 XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr,
1983 XXH_NOESCAPE const void* secret, size_t secretSize,
1984 XXH64_hash_t seed64);
1985 #endif /* !XXH_NO_STREAM */
1986
1987 #if defined (__cplusplus)
1988 } /* extern "C" */
1989 #endif
1990
1991 #endif /* !XXH_NO_XXH3 */
1992 #endif /* XXH_NO_LONG_LONG */
1993
1994 #if defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API)
1995 # define XXH_IMPLEMENTATION
1996 #endif
1997
1998 #endif /* defined(XXH_STATIC_LINKING_ONLY) && !defined(XXHASH_H_STATIC_13879238742) */
1999
2000
2001 /* ======================================================================== */
2002 /* ======================================================================== */
2003 /* ======================================================================== */
2004
2005
2006 /*-**********************************************************************
2007 * xxHash implementation
2008 *-**********************************************************************
2009 * xxHash's implementation used to be hosted inside xxhash.c.
2010 *
2011 * However, inlining requires implementation to be visible to the compiler,
2012 * hence be included alongside the header.
2013 * Previously, implementation was hosted inside xxhash.c,
2014 * which was then #included when inlining was activated.
2015 * This construction created issues with a few build and install systems,
2016 * as it required xxhash.c to be stored in /include directory.
2017 *
2018 * xxHash implementation is now directly integrated within xxhash.h.
2019 * As a consequence, xxhash.c is no longer needed in /include.
2020 *
2021 * xxhash.c is still available and is still useful.
2022 * In a "normal" setup, when xxhash is not inlined,
2023 * xxhash.h only exposes the prototypes and public symbols,
2024 * while xxhash.c can be built into an object file xxhash.o
2025 * which can then be linked into the final binary.
2026 ************************************************************************/
2027
2028 #if ( defined(XXH_INLINE_ALL) || defined(XXH_PRIVATE_API) \
2029 || defined(XXH_IMPLEMENTATION) ) && !defined(XXH_IMPLEM_13a8737387)
2030 # define XXH_IMPLEM_13a8737387
2031
2032 /* *************************************
2033 * Tuning parameters
2034 ***************************************/
2035
2036 /*!
2037 * @defgroup tuning Tuning parameters
2038 * @{
2039 *
2040 * Various macros to control xxHash's behavior.
2041 */
2042 #ifdef XXH_DOXYGEN
2043 /*!
2044 * @brief Define this to disable 64-bit code.
2045 *
2046 * Useful if only using the @ref XXH32_family and you have a strict C90 compiler.
2047 */
2048 # define XXH_NO_LONG_LONG
2049 # undef XXH_NO_LONG_LONG /* don't actually */
2050 /*!
2051 * @brief Controls how unaligned memory is accessed.
2052 *
2053 * By default, access to unaligned memory is controlled by `memcpy()`, which is
2054 * safe and portable.
2055 *
2056 * Unfortunately, on some target/compiler combinations, the generated assembly
2057 * is sub-optimal.
2058 *
2059 * The below switch allow selection of a different access method
2060 * in the search for improved performance.
2061 *
2062 * @par Possible options:
2063 *
2064 * - `XXH_FORCE_MEMORY_ACCESS=0` (default): `memcpy`
2065 * @par
2066 * Use `memcpy()`. Safe and portable. Note that most modern compilers will
2067 * eliminate the function call and treat it as an unaligned access.
2068 *
2069 * - `XXH_FORCE_MEMORY_ACCESS=1`: `__attribute__((aligned(1)))`
2070 * @par
2071 * Depends on compiler extensions and is therefore not portable.
2072 * This method is safe _if_ your compiler supports it,
2073 * and *generally* as fast or faster than `memcpy`.
2074 *
2075 * - `XXH_FORCE_MEMORY_ACCESS=2`: Direct cast
2076 * @par
2077 * Casts directly and dereferences. This method doesn't depend on the
2078 * compiler, but it violates the C standard as it directly dereferences an
2079 * unaligned pointer. It can generate buggy code on targets which do not
2080 * support unaligned memory accesses, but in some circumstances, it's the
2081 * only known way to get the most performance.
2082 *
2083 * - `XXH_FORCE_MEMORY_ACCESS=3`: Byteshift
2084 * @par
2085 * Also portable. This can generate the best code on old compilers which don't
2086 * inline small `memcpy()` calls, and it might also be faster on big-endian
2087 * systems which lack a native byteswap instruction. However, some compilers
2088 * will emit literal byteshifts even if the target supports unaligned access.
2089 *
2090 *
2091 * @warning
2092 * Methods 1 and 2 rely on implementation-defined behavior. Use these with
2093 * care, as what works on one compiler/platform/optimization level may cause
2094 * another to read garbage data or even crash.
2095 *
2096 * See https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html for details.
2097 *
2098 * Prefer these methods in priority order (0 > 3 > 1 > 2)
2099 */
2100 # define XXH_FORCE_MEMORY_ACCESS 0
2101
2102 /*!
2103 * @def XXH_SIZE_OPT
2104 * @brief Controls how much xxHash optimizes for size.
2105 *
2106 * xxHash, when compiled, tends to result in a rather large binary size. This
2107 * is mostly due to heavy usage to forced inlining and constant folding of the
2108 * @ref XXH3_family to increase performance.
2109 *
2110 * However, some developers prefer size over speed. This option can
2111 * significantly reduce the size of the generated code. When using the `-Os`
2112 * or `-Oz` options on GCC or Clang, this is defined to 1 by default,
2113 * otherwise it is defined to 0.
2114 *
2115 * Most of these size optimizations can be controlled manually.
2116 *
2117 * This is a number from 0-2.
2118 * - `XXH_SIZE_OPT` == 0: Default. xxHash makes no size optimizations. Speed
2119 * comes first.
2120 * - `XXH_SIZE_OPT` == 1: Default for `-Os` and `-Oz`. xxHash is more
2121 * conservative and disables hacks that increase code size. It implies the
2122 * options @ref XXH_NO_INLINE_HINTS == 1, @ref XXH_FORCE_ALIGN_CHECK == 0,
2123 * and @ref XXH3_NEON_LANES == 8 if they are not already defined.
2124 * - `XXH_SIZE_OPT` == 2: xxHash tries to make itself as small as possible.
2125 * Performance may cry. For example, the single shot functions just use the
2126 * streaming API.
2127 */
2128 # define XXH_SIZE_OPT 0
2129
2130 /*!
2131 * @def XXH_FORCE_ALIGN_CHECK
2132 * @brief If defined to non-zero, adds a special path for aligned inputs (XXH32()
2133 * and XXH64() only).
2134 *
2135 * This is an important performance trick for architectures without decent
2136 * unaligned memory access performance.
2137 *
2138 * It checks for input alignment, and when conditions are met, uses a "fast
2139 * path" employing direct 32-bit/64-bit reads, resulting in _dramatically
2140 * faster_ read speed.
2141 *
2142 * The check costs one initial branch per hash, which is generally negligible,
2143 * but not zero.
2144 *
2145 * Moreover, it's not useful to generate an additional code path if memory
2146 * access uses the same instruction for both aligned and unaligned
2147 * addresses (e.g. x86 and aarch64).
2148 *
2149 * In these cases, the alignment check can be removed by setting this macro to 0.
2150 * Then the code will always use unaligned memory access.
2151 * Align check is automatically disabled on x86, x64, ARM64, and some ARM chips
2152 * which are platforms known to offer good unaligned memory accesses performance.
2153 *
2154 * It is also disabled by default when @ref XXH_SIZE_OPT >= 1.
2155 *
2156 * This option does not affect XXH3 (only XXH32 and XXH64).
2157 */
2158 # define XXH_FORCE_ALIGN_CHECK 0
2159
2160 /*!
2161 * @def XXH_NO_INLINE_HINTS
2162 * @brief When non-zero, sets all functions to `static`.
2163 *
2164 * By default, xxHash tries to force the compiler to inline almost all internal
2165 * functions.
2166 *
2167 * This can usually improve performance due to reduced jumping and improved
2168 * constant folding, but significantly increases the size of the binary which
2169 * might not be favorable.
2170 *
2171 * Additionally, sometimes the forced inlining can be detrimental to performance,
2172 * depending on the architecture.
2173 *
2174 * XXH_NO_INLINE_HINTS marks all internal functions as static, giving the
2175 * compiler full control on whether to inline or not.
2176 *
2177 * When not optimizing (-O0), using `-fno-inline` with GCC or Clang, or if
2178 * @ref XXH_SIZE_OPT >= 1, this will automatically be defined.
2179 */
2180 # define XXH_NO_INLINE_HINTS 0
2181
2182 /*!
2183 * @def XXH3_INLINE_SECRET
2184 * @brief Determines whether to inline the XXH3 withSecret code.
2185 *
2186 * When the secret size is known, the compiler can improve the performance
2187 * of XXH3_64bits_withSecret() and XXH3_128bits_withSecret().
2188 *
2189 * However, if the secret size is not known, it doesn't have any benefit. This
2190 * happens when xxHash is compiled into a global symbol. Therefore, if
2191 * @ref XXH_INLINE_ALL is *not* defined, this will be defined to 0.
2192 *
2193 * Additionally, this defaults to 0 on GCC 12+, which has an issue with function pointers
2194 * that are *sometimes* force inline on -Og, and it is impossible to automatically
2195 * detect this optimization level.
2196 */
2197 # define XXH3_INLINE_SECRET 0
2198
2199 /*!
2200 * @def XXH32_ENDJMP
2201 * @brief Whether to use a jump for `XXH32_finalize`.
2202 *
2203 * For performance, `XXH32_finalize` uses multiple branches in the finalizer.
2204 * This is generally preferable for performance,
2205 * but depending on exact architecture, a jmp may be preferable.
2206 *
2207 * This setting is only possibly making a difference for very small inputs.
2208 */
2209 # define XXH32_ENDJMP 0
2210
2211 /*!
2212 * @internal
2213 * @brief Redefines old internal names.
2214 *
2215 * For compatibility with code that uses xxHash's internals before the names
2216 * were changed to improve namespacing. There is no other reason to use this.
2217 */
2218 # define XXH_OLD_NAMES
2219 # undef XXH_OLD_NAMES /* don't actually use, it is ugly. */
2220
2221 /*!
2222 * @def XXH_NO_STREAM
2223 * @brief Disables the streaming API.
2224 *
2225 * When xxHash is not inlined and the streaming functions are not used, disabling
2226 * the streaming functions can improve code size significantly, especially with
2227 * the @ref XXH3_family which tends to make constant folded copies of itself.
2228 */
2229 # define XXH_NO_STREAM
2230 # undef XXH_NO_STREAM /* don't actually */
2231 #endif /* XXH_DOXYGEN */
2232 /*!
2233 * @}
2234 */
2235
2236 #ifndef XXH_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */
2237 /* prefer __packed__ structures (method 1) for GCC
2238 * < ARMv7 with unaligned access (e.g. Raspbian armhf) still uses byte shifting, so we use memcpy
2239 * which for some reason does unaligned loads. */
2240 # if defined(__GNUC__) && !(defined(__ARM_ARCH) && __ARM_ARCH < 7 && defined(__ARM_FEATURE_UNALIGNED))
2241 # define XXH_FORCE_MEMORY_ACCESS 1
2242 # endif
2243 #endif
2244
2245 #ifndef XXH_SIZE_OPT
2246 /* default to 1 for -Os or -Oz */
2247 # if (defined(__GNUC__) || defined(__clang__)) && defined(__OPTIMIZE_SIZE__)
2248 # define XXH_SIZE_OPT 1
2249 # else
2250 # define XXH_SIZE_OPT 0
2251 # endif
2252 #endif
2253
2254 #ifndef XXH_FORCE_ALIGN_CHECK /* can be defined externally */
2255 /* don't check on sizeopt, x86, aarch64, or arm when unaligned access is available */
2256 # if XXH_SIZE_OPT >= 1 || \
2257 defined(__i386) || defined(__x86_64__) || defined(__aarch64__) || defined(__ARM_FEATURE_UNALIGNED) \
2258 || defined(_M_IX86) || defined(_M_X64) || defined(_M_ARM64) || defined(_M_ARM) /* visual */
2259 # define XXH_FORCE_ALIGN_CHECK 0
2260 # else
2261 # define XXH_FORCE_ALIGN_CHECK 1
2262 # endif
2263 #endif
2264
2265 #ifndef XXH_NO_INLINE_HINTS
2266 # if XXH_SIZE_OPT >= 1 || defined(__NO_INLINE__) /* -O0, -fno-inline */
2267 # define XXH_NO_INLINE_HINTS 1
2268 # else
2269 # define XXH_NO_INLINE_HINTS 0
2270 # endif
2271 #endif
2272
2273 #ifndef XXH3_INLINE_SECRET
2274 # if (defined(__GNUC__) && !defined(__clang__) && __GNUC__ >= 12) \
2275 || !defined(XXH_INLINE_ALL)
2276 # define XXH3_INLINE_SECRET 0
2277 # else
2278 # define XXH3_INLINE_SECRET 1
2279 # endif
2280 #endif
2281
2282 #ifndef XXH32_ENDJMP
2283 /* generally preferable for performance */
2284 # define XXH32_ENDJMP 0
2285 #endif
2286
2287 /*!
2288 * @defgroup impl Implementation
2289 * @{
2290 */
2291
2292 /* *************************************
2293 * Includes & Memory related functions
2294 ***************************************/
2295 #include <string.h> /* memcmp, memcpy */
2296 #include <limits.h> /* ULLONG_MAX */
2297
2298 #if defined(XXH_NO_STREAM)
2299 /* nothing */
2300 #elif defined(XXH_NO_STDLIB)
2301
2302 /* When requesting to disable any mention of stdlib,
2303 * the library loses the ability to invoked malloc / free.
2304 * In practice, it means that functions like `XXH*_createState()`
2305 * will always fail, and return NULL.
2306 * This flag is useful in situations where
2307 * xxhash.h is integrated into some kernel, embedded or limited environment
2308 * without access to dynamic allocation.
2309 */
2310
2311 #if defined (__cplusplus)
2312 extern "C" {
2313 #endif
2314
XXH_malloc(size_t s)2315 static XXH_CONSTF void* XXH_malloc(size_t s) { (void)s; return NULL; }
XXH_free(void * p)2316 static void XXH_free(void* p) { (void)p; }
2317
2318 #if defined (__cplusplus)
2319 } /* extern "C" */
2320 #endif
2321
2322 #else
2323
2324 /*
2325 * Modify the local functions below should you wish to use
2326 * different memory routines for malloc() and free()
2327 */
2328 #include <stdlib.h>
2329
2330 #if defined (__cplusplus)
2331 extern "C" {
2332 #endif
2333 /*!
2334 * @internal
2335 * @brief Modify this function to use a different routine than malloc().
2336 */
XXH_malloc(size_t s)2337 static XXH_MALLOCF void* XXH_malloc(size_t s) { return malloc(s); }
2338
2339 /*!
2340 * @internal
2341 * @brief Modify this function to use a different routine than free().
2342 */
XXH_free(void * p)2343 static void XXH_free(void* p) { free(p); }
2344
2345 #if defined (__cplusplus)
2346 } /* extern "C" */
2347 #endif
2348
2349 #endif /* XXH_NO_STDLIB */
2350
2351 #if defined (__cplusplus)
2352 extern "C" {
2353 #endif
2354 /*!
2355 * @internal
2356 * @brief Modify this function to use a different routine than memcpy().
2357 */
XXH_memcpy(void * dest,const void * src,size_t size)2358 static void* XXH_memcpy(void* dest, const void* src, size_t size)
2359 {
2360 return memcpy(dest,src,size);
2361 }
2362
2363 #if defined (__cplusplus)
2364 } /* extern "C" */
2365 #endif
2366
2367 /* *************************************
2368 * Compiler Specific Options
2369 ***************************************/
2370 #ifdef _MSC_VER /* Visual Studio warning fix */
2371 # pragma warning(disable : 4127) /* disable: C4127: conditional expression is constant */
2372 #endif
2373
2374 #if XXH_NO_INLINE_HINTS /* disable inlining hints */
2375 # if defined(__GNUC__) || defined(__clang__)
2376 # define XXH_FORCE_INLINE static __attribute__((unused))
2377 # else
2378 # define XXH_FORCE_INLINE static
2379 # endif
2380 # define XXH_NO_INLINE static
2381 /* enable inlining hints */
2382 #elif defined(__GNUC__) || defined(__clang__)
2383 # define XXH_FORCE_INLINE static __inline__ __attribute__((always_inline, unused))
2384 # define XXH_NO_INLINE static __attribute__((noinline))
2385 #elif defined(_MSC_VER) /* Visual Studio */
2386 # define XXH_FORCE_INLINE static __forceinline
2387 # define XXH_NO_INLINE static __declspec(noinline)
2388 #elif defined (__cplusplus) \
2389 || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L)) /* C99 */
2390 # define XXH_FORCE_INLINE static inline
2391 # define XXH_NO_INLINE static
2392 #else
2393 # define XXH_FORCE_INLINE static
2394 # define XXH_NO_INLINE static
2395 #endif
2396
2397 #if XXH3_INLINE_SECRET
2398 # define XXH3_WITH_SECRET_INLINE XXH_FORCE_INLINE
2399 #else
2400 # define XXH3_WITH_SECRET_INLINE XXH_NO_INLINE
2401 #endif
2402
2403
2404 /* *************************************
2405 * Debug
2406 ***************************************/
2407 /*!
2408 * @ingroup tuning
2409 * @def XXH_DEBUGLEVEL
2410 * @brief Sets the debugging level.
2411 *
2412 * XXH_DEBUGLEVEL is expected to be defined externally, typically via the
2413 * compiler's command line options. The value must be a number.
2414 */
2415 #ifndef XXH_DEBUGLEVEL
2416 # ifdef DEBUGLEVEL /* backwards compat */
2417 # define XXH_DEBUGLEVEL DEBUGLEVEL
2418 # else
2419 # define XXH_DEBUGLEVEL 0
2420 # endif
2421 #endif
2422
2423 #if (XXH_DEBUGLEVEL>=1)
2424 # include <assert.h> /* note: can still be disabled with NDEBUG */
2425 # define XXH_ASSERT(c) assert(c)
2426 #else
2427 # if defined(__INTEL_COMPILER)
2428 # define XXH_ASSERT(c) XXH_ASSUME((unsigned char) (c))
2429 # else
2430 # define XXH_ASSERT(c) XXH_ASSUME(c)
2431 # endif
2432 #endif
2433
2434 /* note: use after variable declarations */
2435 #ifndef XXH_STATIC_ASSERT
2436 # if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= 201112L) /* C11 */
2437 # define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { _Static_assert((c),m); } while(0)
2438 # elif defined(__cplusplus) && (__cplusplus >= 201103L) /* C++11 */
2439 # define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { static_assert((c),m); } while(0)
2440 # else
2441 # define XXH_STATIC_ASSERT_WITH_MESSAGE(c,m) do { struct xxh_sa { char x[(c) ? 1 : -1]; }; } while(0)
2442 # endif
2443 # define XXH_STATIC_ASSERT(c) XXH_STATIC_ASSERT_WITH_MESSAGE((c),#c)
2444 #endif
2445
2446 /*!
2447 * @internal
2448 * @def XXH_COMPILER_GUARD(var)
2449 * @brief Used to prevent unwanted optimizations for @p var.
2450 *
2451 * It uses an empty GCC inline assembly statement with a register constraint
2452 * which forces @p var into a general purpose register (eg eax, ebx, ecx
2453 * on x86) and marks it as modified.
2454 *
2455 * This is used in a few places to avoid unwanted autovectorization (e.g.
2456 * XXH32_round()). All vectorization we want is explicit via intrinsics,
2457 * and _usually_ isn't wanted elsewhere.
2458 *
2459 * We also use it to prevent unwanted constant folding for AArch64 in
2460 * XXH3_initCustomSecret_scalar().
2461 */
2462 #if defined(__GNUC__) || defined(__clang__)
2463 # define XXH_COMPILER_GUARD(var) __asm__("" : "+r" (var))
2464 #else
2465 # define XXH_COMPILER_GUARD(var) ((void)0)
2466 #endif
2467
2468 /* Specifically for NEON vectors which use the "w" constraint, on
2469 * Clang. */
2470 #if defined(__clang__) && defined(__ARM_ARCH) && !defined(__wasm__)
2471 # define XXH_COMPILER_GUARD_CLANG_NEON(var) __asm__("" : "+w" (var))
2472 #else
2473 # define XXH_COMPILER_GUARD_CLANG_NEON(var) ((void)0)
2474 #endif
2475
2476 /* *************************************
2477 * Basic Types
2478 ***************************************/
2479 #if !defined (__VMS) \
2480 && (defined (__cplusplus) \
2481 || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
2482 # ifdef _AIX
2483 # include <inttypes.h>
2484 # else
2485 # include <stdint.h>
2486 # endif
2487 typedef uint8_t xxh_u8;
2488 #else
2489 typedef unsigned char xxh_u8;
2490 #endif
2491 typedef XXH32_hash_t xxh_u32;
2492
2493 #ifdef XXH_OLD_NAMES
2494 # warning "XXH_OLD_NAMES is planned to be removed starting v0.9. If the program depends on it, consider moving away from it by employing newer type names directly"
2495 # define BYTE xxh_u8
2496 # define U8 xxh_u8
2497 # define U32 xxh_u32
2498 #endif
2499
2500 #if defined (__cplusplus)
2501 extern "C" {
2502 #endif
2503
2504 /* *** Memory access *** */
2505
2506 /*!
2507 * @internal
2508 * @fn xxh_u32 XXH_read32(const void* ptr)
2509 * @brief Reads an unaligned 32-bit integer from @p ptr in native endianness.
2510 *
2511 * Affected by @ref XXH_FORCE_MEMORY_ACCESS.
2512 *
2513 * @param ptr The pointer to read from.
2514 * @return The 32-bit native endian integer from the bytes at @p ptr.
2515 */
2516
2517 /*!
2518 * @internal
2519 * @fn xxh_u32 XXH_readLE32(const void* ptr)
2520 * @brief Reads an unaligned 32-bit little endian integer from @p ptr.
2521 *
2522 * Affected by @ref XXH_FORCE_MEMORY_ACCESS.
2523 *
2524 * @param ptr The pointer to read from.
2525 * @return The 32-bit little endian integer from the bytes at @p ptr.
2526 */
2527
2528 /*!
2529 * @internal
2530 * @fn xxh_u32 XXH_readBE32(const void* ptr)
2531 * @brief Reads an unaligned 32-bit big endian integer from @p ptr.
2532 *
2533 * Affected by @ref XXH_FORCE_MEMORY_ACCESS.
2534 *
2535 * @param ptr The pointer to read from.
2536 * @return The 32-bit big endian integer from the bytes at @p ptr.
2537 */
2538
2539 /*!
2540 * @internal
2541 * @fn xxh_u32 XXH_readLE32_align(const void* ptr, XXH_alignment align)
2542 * @brief Like @ref XXH_readLE32(), but has an option for aligned reads.
2543 *
2544 * Affected by @ref XXH_FORCE_MEMORY_ACCESS.
2545 * Note that when @ref XXH_FORCE_ALIGN_CHECK == 0, the @p align parameter is
2546 * always @ref XXH_alignment::XXH_unaligned.
2547 *
2548 * @param ptr The pointer to read from.
2549 * @param align Whether @p ptr is aligned.
2550 * @pre
2551 * If @p align == @ref XXH_alignment::XXH_aligned, @p ptr must be 4 byte
2552 * aligned.
2553 * @return The 32-bit little endian integer from the bytes at @p ptr.
2554 */
2555
2556 #if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))
2557 /*
2558 * Manual byteshift. Best for old compilers which don't inline memcpy.
2559 * We actually directly use XXH_readLE32 and XXH_readBE32.
2560 */
2561 #elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))
2562
2563 /*
2564 * Force direct memory access. Only works on CPU which support unaligned memory
2565 * access in hardware.
2566 */
2567 static xxh_u32 XXH_read32(const void* memPtr) { return *(const xxh_u32*) memPtr; }
2568
2569 #elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))
2570
2571 /*
2572 * __attribute__((aligned(1))) is supported by gcc and clang. Originally the
2573 * documentation claimed that it only increased the alignment, but actually it
2574 * can decrease it on gcc, clang, and icc:
2575 * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502,
2576 * https://gcc.godbolt.org/z/xYez1j67Y.
2577 */
2578 #ifdef XXH_OLD_NAMES
2579 typedef union { xxh_u32 u32; } __attribute__((packed)) unalign;
2580 #endif
2581 static xxh_u32 XXH_read32(const void* ptr)
2582 {
2583 typedef __attribute__((aligned(1))) xxh_u32 xxh_unalign32;
2584 return *((const xxh_unalign32*)ptr);
2585 }
2586
2587 #else
2588
2589 /*
2590 * Portable and safe solution. Generally efficient.
2591 * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html
2592 */
2593 static xxh_u32 XXH_read32(const void* memPtr)
2594 {
2595 xxh_u32 val;
2596 XXH_memcpy(&val, memPtr, sizeof(val));
2597 return val;
2598 }
2599
2600 #endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */
2601
2602
2603 /* *** Endianness *** */
2604
2605 /*!
2606 * @ingroup tuning
2607 * @def XXH_CPU_LITTLE_ENDIAN
2608 * @brief Whether the target is little endian.
2609 *
2610 * Defined to 1 if the target is little endian, or 0 if it is big endian.
2611 * It can be defined externally, for example on the compiler command line.
2612 *
2613 * If it is not defined,
2614 * a runtime check (which is usually constant folded) is used instead.
2615 *
2616 * @note
2617 * This is not necessarily defined to an integer constant.
2618 *
2619 * @see XXH_isLittleEndian() for the runtime check.
2620 */
2621 #ifndef XXH_CPU_LITTLE_ENDIAN
2622 /*
2623 * Try to detect endianness automatically, to avoid the nonstandard behavior
2624 * in `XXH_isLittleEndian()`
2625 */
2626 # if defined(_WIN32) /* Windows is always little endian */ \
2627 || defined(__LITTLE_ENDIAN__) \
2628 || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
2629 # define XXH_CPU_LITTLE_ENDIAN 1
2630 # elif defined(__BIG_ENDIAN__) \
2631 || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
2632 # define XXH_CPU_LITTLE_ENDIAN 0
2633 # else
2634 /*!
2635 * @internal
2636 * @brief Runtime check for @ref XXH_CPU_LITTLE_ENDIAN.
2637 *
2638 * Most compilers will constant fold this.
2639 */
2640 static int XXH_isLittleEndian(void)
2641 {
2642 /*
2643 * Portable and well-defined behavior.
2644 * Don't use static: it is detrimental to performance.
2645 */
2646 const union { xxh_u32 u; xxh_u8 c[4]; } one = { 1 };
2647 return one.c[0];
2648 }
2649 # define XXH_CPU_LITTLE_ENDIAN XXH_isLittleEndian()
2650 # endif
2651 #endif
2652
2653
2654
2655
2656 /* ****************************************
2657 * Compiler-specific Functions and Macros
2658 ******************************************/
2659 #define XXH_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__)
2660
2661 #ifdef __has_builtin
2662 # define XXH_HAS_BUILTIN(x) __has_builtin(x)
2663 #else
2664 # define XXH_HAS_BUILTIN(x) 0
2665 #endif
2666
2667
2668
2669 /*
2670 * C23 and future versions have standard "unreachable()".
2671 * Once it has been implemented reliably we can add it as an
2672 * additional case:
2673 *
2674 * ```
2675 * #if defined(__STDC_VERSION__) && (__STDC_VERSION__ >= XXH_C23_VN)
2676 * # include <stddef.h>
2677 * # ifdef unreachable
2678 * # define XXH_UNREACHABLE() unreachable()
2679 * # endif
2680 * #endif
2681 * ```
2682 *
2683 * Note C++23 also has std::unreachable() which can be detected
2684 * as follows:
2685 * ```
2686 * #if defined(__cpp_lib_unreachable) && (__cpp_lib_unreachable >= 202202L)
2687 * # include <utility>
2688 * # define XXH_UNREACHABLE() std::unreachable()
2689 * #endif
2690 * ```
2691 * NB: `__cpp_lib_unreachable` is defined in the `<version>` header.
2692 * We don't use that as including `<utility>` in `extern "C"` blocks
2693 * doesn't work on GCC12
2694 */
2695
2696 #if XXH_HAS_BUILTIN(__builtin_unreachable)
2697 # define XXH_UNREACHABLE() __builtin_unreachable()
2698
2699 #elif defined(_MSC_VER)
2700 # define XXH_UNREACHABLE() __assume(0)
2701
2702 #else
2703 # define XXH_UNREACHABLE()
2704 #endif
2705
2706 #if XXH_HAS_BUILTIN(__builtin_assume)
2707 # define XXH_ASSUME(c) __builtin_assume(c)
2708 #else
2709 # define XXH_ASSUME(c) if (!(c)) { XXH_UNREACHABLE(); }
2710 #endif
2711
2712 /*!
2713 * @internal
2714 * @def XXH_rotl32(x,r)
2715 * @brief 32-bit rotate left.
2716 *
2717 * @param x The 32-bit integer to be rotated.
2718 * @param r The number of bits to rotate.
2719 * @pre
2720 * @p r > 0 && @p r < 32
2721 * @note
2722 * @p x and @p r may be evaluated multiple times.
2723 * @return The rotated result.
2724 */
2725 #if !defined(NO_CLANG_BUILTIN) && XXH_HAS_BUILTIN(__builtin_rotateleft32) \
2726 && XXH_HAS_BUILTIN(__builtin_rotateleft64)
2727 # define XXH_rotl32 __builtin_rotateleft32
2728 # define XXH_rotl64 __builtin_rotateleft64
2729 /* Note: although _rotl exists for minGW (GCC under windows), performance seems poor */
2730 #elif defined(_MSC_VER)
2731 # define XXH_rotl32(x,r) _rotl(x,r)
2732 # define XXH_rotl64(x,r) _rotl64(x,r)
2733 #else
2734 # define XXH_rotl32(x,r) (((x) << (r)) | ((x) >> (32 - (r))))
2735 # define XXH_rotl64(x,r) (((x) << (r)) | ((x) >> (64 - (r))))
2736 #endif
2737
2738 /*!
2739 * @internal
2740 * @fn xxh_u32 XXH_swap32(xxh_u32 x)
2741 * @brief A 32-bit byteswap.
2742 *
2743 * @param x The 32-bit integer to byteswap.
2744 * @return @p x, byteswapped.
2745 */
2746 #if defined(_MSC_VER) /* Visual Studio */
2747 # define XXH_swap32 _byteswap_ulong
2748 #elif XXH_GCC_VERSION >= 403
2749 # define XXH_swap32 __builtin_bswap32
2750 #else
2751 static xxh_u32 XXH_swap32 (xxh_u32 x)
2752 {
2753 return ((x << 24) & 0xff000000 ) |
2754 ((x << 8) & 0x00ff0000 ) |
2755 ((x >> 8) & 0x0000ff00 ) |
2756 ((x >> 24) & 0x000000ff );
2757 }
2758 #endif
2759
2760
2761 /* ***************************
2762 * Memory reads
2763 *****************************/
2764
2765 /*!
2766 * @internal
2767 * @brief Enum to indicate whether a pointer is aligned.
2768 */
2769 typedef enum {
2770 XXH_aligned, /*!< Aligned */
2771 XXH_unaligned /*!< Possibly unaligned */
2772 } XXH_alignment;
2773
2774 /*
2775 * XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load.
2776 *
2777 * This is ideal for older compilers which don't inline memcpy.
2778 */
2779 #if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))
2780
XXH_readLE32(const void * memPtr)2781 XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* memPtr)
2782 {
2783 const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
2784 return bytePtr[0]
2785 | ((xxh_u32)bytePtr[1] << 8)
2786 | ((xxh_u32)bytePtr[2] << 16)
2787 | ((xxh_u32)bytePtr[3] << 24);
2788 }
2789
XXH_readBE32(const void * memPtr)2790 XXH_FORCE_INLINE xxh_u32 XXH_readBE32(const void* memPtr)
2791 {
2792 const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
2793 return bytePtr[3]
2794 | ((xxh_u32)bytePtr[2] << 8)
2795 | ((xxh_u32)bytePtr[1] << 16)
2796 | ((xxh_u32)bytePtr[0] << 24);
2797 }
2798
2799 #else
XXH_readLE32(const void * ptr)2800 XXH_FORCE_INLINE xxh_u32 XXH_readLE32(const void* ptr)
2801 {
2802 return XXH_CPU_LITTLE_ENDIAN ? XXH_read32(ptr) : XXH_swap32(XXH_read32(ptr));
2803 }
2804
XXH_readBE32(const void * ptr)2805 static xxh_u32 XXH_readBE32(const void* ptr)
2806 {
2807 return XXH_CPU_LITTLE_ENDIAN ? XXH_swap32(XXH_read32(ptr)) : XXH_read32(ptr);
2808 }
2809 #endif
2810
2811 XXH_FORCE_INLINE xxh_u32
XXH_readLE32_align(const void * ptr,XXH_alignment align)2812 XXH_readLE32_align(const void* ptr, XXH_alignment align)
2813 {
2814 if (align==XXH_unaligned) {
2815 return XXH_readLE32(ptr);
2816 } else {
2817 return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u32*)ptr : XXH_swap32(*(const xxh_u32*)ptr);
2818 }
2819 }
2820
2821
2822 /* *************************************
2823 * Misc
2824 ***************************************/
2825 /*! @ingroup public */
XXH_versionNumber(void)2826 XXH_PUBLIC_API unsigned XXH_versionNumber (void) { return XXH_VERSION_NUMBER; }
2827
2828
2829 /* *******************************************************************
2830 * 32-bit hash functions
2831 *********************************************************************/
2832 /*!
2833 * @}
2834 * @defgroup XXH32_impl XXH32 implementation
2835 * @ingroup impl
2836 *
2837 * Details on the XXH32 implementation.
2838 * @{
2839 */
2840 /* #define instead of static const, to be used as initializers */
2841 #define XXH_PRIME32_1 0x9E3779B1U /*!< 0b10011110001101110111100110110001 */
2842 #define XXH_PRIME32_2 0x85EBCA77U /*!< 0b10000101111010111100101001110111 */
2843 #define XXH_PRIME32_3 0xC2B2AE3DU /*!< 0b11000010101100101010111000111101 */
2844 #define XXH_PRIME32_4 0x27D4EB2FU /*!< 0b00100111110101001110101100101111 */
2845 #define XXH_PRIME32_5 0x165667B1U /*!< 0b00010110010101100110011110110001 */
2846
2847 #ifdef XXH_OLD_NAMES
2848 # define PRIME32_1 XXH_PRIME32_1
2849 # define PRIME32_2 XXH_PRIME32_2
2850 # define PRIME32_3 XXH_PRIME32_3
2851 # define PRIME32_4 XXH_PRIME32_4
2852 # define PRIME32_5 XXH_PRIME32_5
2853 #endif
2854
2855 /*!
2856 * @internal
2857 * @brief Normal stripe processing routine.
2858 *
2859 * This shuffles the bits so that any bit from @p input impacts several bits in
2860 * @p acc.
2861 *
2862 * @param acc The accumulator lane.
2863 * @param input The stripe of input to mix.
2864 * @return The mixed accumulator lane.
2865 */
XXH32_round(xxh_u32 acc,xxh_u32 input)2866 static xxh_u32 XXH32_round(xxh_u32 acc, xxh_u32 input)
2867 {
2868 acc += input * XXH_PRIME32_2;
2869 acc = XXH_rotl32(acc, 13);
2870 acc *= XXH_PRIME32_1;
2871 #if (defined(__SSE4_1__) || defined(__aarch64__) || defined(__wasm_simd128__)) && !defined(XXH_ENABLE_AUTOVECTORIZE)
2872 /*
2873 * UGLY HACK:
2874 * A compiler fence is the only thing that prevents GCC and Clang from
2875 * autovectorizing the XXH32 loop (pragmas and attributes don't work for some
2876 * reason) without globally disabling SSE4.1.
2877 *
2878 * The reason we want to avoid vectorization is because despite working on
2879 * 4 integers at a time, there are multiple factors slowing XXH32 down on
2880 * SSE4:
2881 * - There's a ridiculous amount of lag from pmulld (10 cycles of latency on
2882 * newer chips!) making it slightly slower to multiply four integers at
2883 * once compared to four integers independently. Even when pmulld was
2884 * fastest, Sandy/Ivy Bridge, it is still not worth it to go into SSE
2885 * just to multiply unless doing a long operation.
2886 *
2887 * - Four instructions are required to rotate,
2888 * movqda tmp, v // not required with VEX encoding
2889 * pslld tmp, 13 // tmp <<= 13
2890 * psrld v, 19 // x >>= 19
2891 * por v, tmp // x |= tmp
2892 * compared to one for scalar:
2893 * roll v, 13 // reliably fast across the board
2894 * shldl v, v, 13 // Sandy Bridge and later prefer this for some reason
2895 *
2896 * - Instruction level parallelism is actually more beneficial here because
2897 * the SIMD actually serializes this operation: While v1 is rotating, v2
2898 * can load data, while v3 can multiply. SSE forces them to operate
2899 * together.
2900 *
2901 * This is also enabled on AArch64, as Clang is *very aggressive* in vectorizing
2902 * the loop. NEON is only faster on the A53, and with the newer cores, it is less
2903 * than half the speed.
2904 *
2905 * Additionally, this is used on WASM SIMD128 because it JITs to the same
2906 * SIMD instructions and has the same issue.
2907 */
2908 XXH_COMPILER_GUARD(acc);
2909 #endif
2910 return acc;
2911 }
2912
2913 /*!
2914 * @internal
2915 * @brief Mixes all bits to finalize the hash.
2916 *
2917 * The final mix ensures that all input bits have a chance to impact any bit in
2918 * the output digest, resulting in an unbiased distribution.
2919 *
2920 * @param hash The hash to avalanche.
2921 * @return The avalanched hash.
2922 */
XXH32_avalanche(xxh_u32 hash)2923 static xxh_u32 XXH32_avalanche(xxh_u32 hash)
2924 {
2925 hash ^= hash >> 15;
2926 hash *= XXH_PRIME32_2;
2927 hash ^= hash >> 13;
2928 hash *= XXH_PRIME32_3;
2929 hash ^= hash >> 16;
2930 return hash;
2931 }
2932
2933 #define XXH_get32bits(p) XXH_readLE32_align(p, align)
2934
2935 /*!
2936 * @internal
2937 * @brief Processes the last 0-15 bytes of @p ptr.
2938 *
2939 * There may be up to 15 bytes remaining to consume from the input.
2940 * This final stage will digest them to ensure that all input bytes are present
2941 * in the final mix.
2942 *
2943 * @param hash The hash to finalize.
2944 * @param ptr The pointer to the remaining input.
2945 * @param len The remaining length, modulo 16.
2946 * @param align Whether @p ptr is aligned.
2947 * @return The finalized hash.
2948 * @see XXH64_finalize().
2949 */
2950 static XXH_PUREF xxh_u32
XXH32_finalize(xxh_u32 hash,const xxh_u8 * ptr,size_t len,XXH_alignment align)2951 XXH32_finalize(xxh_u32 hash, const xxh_u8* ptr, size_t len, XXH_alignment align)
2952 {
2953 #define XXH_PROCESS1 do { \
2954 hash += (*ptr++) * XXH_PRIME32_5; \
2955 hash = XXH_rotl32(hash, 11) * XXH_PRIME32_1; \
2956 } while (0)
2957
2958 #define XXH_PROCESS4 do { \
2959 hash += XXH_get32bits(ptr) * XXH_PRIME32_3; \
2960 ptr += 4; \
2961 hash = XXH_rotl32(hash, 17) * XXH_PRIME32_4; \
2962 } while (0)
2963
2964 if (ptr==NULL) XXH_ASSERT(len == 0);
2965
2966 /* Compact rerolled version; generally faster */
2967 if (!XXH32_ENDJMP) {
2968 len &= 15;
2969 while (len >= 4) {
2970 XXH_PROCESS4;
2971 len -= 4;
2972 }
2973 while (len > 0) {
2974 XXH_PROCESS1;
2975 --len;
2976 }
2977 return XXH32_avalanche(hash);
2978 } else {
2979 switch(len&15) /* or switch(bEnd - p) */ {
2980 case 12: XXH_PROCESS4;
2981 XXH_FALLTHROUGH; /* fallthrough */
2982 case 8: XXH_PROCESS4;
2983 XXH_FALLTHROUGH; /* fallthrough */
2984 case 4: XXH_PROCESS4;
2985 return XXH32_avalanche(hash);
2986
2987 case 13: XXH_PROCESS4;
2988 XXH_FALLTHROUGH; /* fallthrough */
2989 case 9: XXH_PROCESS4;
2990 XXH_FALLTHROUGH; /* fallthrough */
2991 case 5: XXH_PROCESS4;
2992 XXH_PROCESS1;
2993 return XXH32_avalanche(hash);
2994
2995 case 14: XXH_PROCESS4;
2996 XXH_FALLTHROUGH; /* fallthrough */
2997 case 10: XXH_PROCESS4;
2998 XXH_FALLTHROUGH; /* fallthrough */
2999 case 6: XXH_PROCESS4;
3000 XXH_PROCESS1;
3001 XXH_PROCESS1;
3002 return XXH32_avalanche(hash);
3003
3004 case 15: XXH_PROCESS4;
3005 XXH_FALLTHROUGH; /* fallthrough */
3006 case 11: XXH_PROCESS4;
3007 XXH_FALLTHROUGH; /* fallthrough */
3008 case 7: XXH_PROCESS4;
3009 XXH_FALLTHROUGH; /* fallthrough */
3010 case 3: XXH_PROCESS1;
3011 XXH_FALLTHROUGH; /* fallthrough */
3012 case 2: XXH_PROCESS1;
3013 XXH_FALLTHROUGH; /* fallthrough */
3014 case 1: XXH_PROCESS1;
3015 XXH_FALLTHROUGH; /* fallthrough */
3016 case 0: return XXH32_avalanche(hash);
3017 }
3018 XXH_ASSERT(0);
3019 return hash; /* reaching this point is deemed impossible */
3020 }
3021 }
3022
3023 #ifdef XXH_OLD_NAMES
3024 # define PROCESS1 XXH_PROCESS1
3025 # define PROCESS4 XXH_PROCESS4
3026 #else
3027 # undef XXH_PROCESS1
3028 # undef XXH_PROCESS4
3029 #endif
3030
3031 /*!
3032 * @internal
3033 * @brief The implementation for @ref XXH32().
3034 *
3035 * @param input , len , seed Directly passed from @ref XXH32().
3036 * @param align Whether @p input is aligned.
3037 * @return The calculated hash.
3038 */
3039 XXH_FORCE_INLINE XXH_PUREF xxh_u32
XXH32_endian_align(const xxh_u8 * input,size_t len,xxh_u32 seed,XXH_alignment align)3040 XXH32_endian_align(const xxh_u8* input, size_t len, xxh_u32 seed, XXH_alignment align)
3041 {
3042 xxh_u32 h32;
3043
3044 if (input==NULL) XXH_ASSERT(len == 0);
3045
3046 if (len>=16) {
3047 const xxh_u8* const bEnd = input + len;
3048 const xxh_u8* const limit = bEnd - 15;
3049 xxh_u32 v1 = seed + XXH_PRIME32_1 + XXH_PRIME32_2;
3050 xxh_u32 v2 = seed + XXH_PRIME32_2;
3051 xxh_u32 v3 = seed + 0;
3052 xxh_u32 v4 = seed - XXH_PRIME32_1;
3053
3054 do {
3055 v1 = XXH32_round(v1, XXH_get32bits(input)); input += 4;
3056 v2 = XXH32_round(v2, XXH_get32bits(input)); input += 4;
3057 v3 = XXH32_round(v3, XXH_get32bits(input)); input += 4;
3058 v4 = XXH32_round(v4, XXH_get32bits(input)); input += 4;
3059 } while (input < limit);
3060
3061 h32 = XXH_rotl32(v1, 1) + XXH_rotl32(v2, 7)
3062 + XXH_rotl32(v3, 12) + XXH_rotl32(v4, 18);
3063 } else {
3064 h32 = seed + XXH_PRIME32_5;
3065 }
3066
3067 h32 += (xxh_u32)len;
3068
3069 return XXH32_finalize(h32, input, len&15, align);
3070 }
3071
3072 /*! @ingroup XXH32_family */
XXH32(const void * input,size_t len,XXH32_hash_t seed)3073 XXH_PUBLIC_API XXH32_hash_t XXH32 (const void* input, size_t len, XXH32_hash_t seed)
3074 {
3075 #if !defined(XXH_NO_STREAM) && XXH_SIZE_OPT >= 2
3076 /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
3077 XXH32_state_t state;
3078 XXH32_reset(&state, seed);
3079 XXH32_update(&state, (const xxh_u8*)input, len);
3080 return XXH32_digest(&state);
3081 #else
3082 if (XXH_FORCE_ALIGN_CHECK) {
3083 if ((((size_t)input) & 3) == 0) { /* Input is 4-bytes aligned, leverage the speed benefit */
3084 return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_aligned);
3085 } }
3086
3087 return XXH32_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned);
3088 #endif
3089 }
3090
3091
3092
3093 /******* Hash streaming *******/
3094 #ifndef XXH_NO_STREAM
3095 /*! @ingroup XXH32_family */
XXH32_createState(void)3096 XXH_PUBLIC_API XXH32_state_t* XXH32_createState(void)
3097 {
3098 return (XXH32_state_t*)XXH_malloc(sizeof(XXH32_state_t));
3099 }
3100 /*! @ingroup XXH32_family */
XXH32_freeState(XXH32_state_t * statePtr)3101 XXH_PUBLIC_API XXH_errorcode XXH32_freeState(XXH32_state_t* statePtr)
3102 {
3103 XXH_free(statePtr);
3104 return XXH_OK;
3105 }
3106
3107 /*! @ingroup XXH32_family */
XXH32_copyState(XXH32_state_t * dstState,const XXH32_state_t * srcState)3108 XXH_PUBLIC_API void XXH32_copyState(XXH32_state_t* dstState, const XXH32_state_t* srcState)
3109 {
3110 XXH_memcpy(dstState, srcState, sizeof(*dstState));
3111 }
3112
3113 /*! @ingroup XXH32_family */
XXH32_reset(XXH32_state_t * statePtr,XXH32_hash_t seed)3114 XXH_PUBLIC_API XXH_errorcode XXH32_reset(XXH32_state_t* statePtr, XXH32_hash_t seed)
3115 {
3116 XXH_ASSERT(statePtr != NULL);
3117 memset(statePtr, 0, sizeof(*statePtr));
3118 statePtr->v[0] = seed + XXH_PRIME32_1 + XXH_PRIME32_2;
3119 statePtr->v[1] = seed + XXH_PRIME32_2;
3120 statePtr->v[2] = seed + 0;
3121 statePtr->v[3] = seed - XXH_PRIME32_1;
3122 return XXH_OK;
3123 }
3124
3125
3126 /*! @ingroup XXH32_family */
3127 XXH_PUBLIC_API XXH_errorcode
XXH32_update(XXH32_state_t * state,const void * input,size_t len)3128 XXH32_update(XXH32_state_t* state, const void* input, size_t len)
3129 {
3130 if (input==NULL) {
3131 XXH_ASSERT(len == 0);
3132 return XXH_OK;
3133 }
3134
3135 { const xxh_u8* p = (const xxh_u8*)input;
3136 const xxh_u8* const bEnd = p + len;
3137
3138 state->total_len_32 += (XXH32_hash_t)len;
3139 state->large_len |= (XXH32_hash_t)((len>=16) | (state->total_len_32>=16));
3140
3141 if (state->memsize + len < 16) { /* fill in tmp buffer */
3142 XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, len);
3143 state->memsize += (XXH32_hash_t)len;
3144 return XXH_OK;
3145 }
3146
3147 if (state->memsize) { /* some data left from previous update */
3148 XXH_memcpy((xxh_u8*)(state->mem32) + state->memsize, input, 16-state->memsize);
3149 { const xxh_u32* p32 = state->mem32;
3150 state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p32)); p32++;
3151 state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p32)); p32++;
3152 state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p32)); p32++;
3153 state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p32));
3154 }
3155 p += 16-state->memsize;
3156 state->memsize = 0;
3157 }
3158
3159 if (p <= bEnd-16) {
3160 const xxh_u8* const limit = bEnd - 16;
3161
3162 do {
3163 state->v[0] = XXH32_round(state->v[0], XXH_readLE32(p)); p+=4;
3164 state->v[1] = XXH32_round(state->v[1], XXH_readLE32(p)); p+=4;
3165 state->v[2] = XXH32_round(state->v[2], XXH_readLE32(p)); p+=4;
3166 state->v[3] = XXH32_round(state->v[3], XXH_readLE32(p)); p+=4;
3167 } while (p<=limit);
3168
3169 }
3170
3171 if (p < bEnd) {
3172 XXH_memcpy(state->mem32, p, (size_t)(bEnd-p));
3173 state->memsize = (unsigned)(bEnd-p);
3174 }
3175 }
3176
3177 return XXH_OK;
3178 }
3179
3180
3181 /*! @ingroup XXH32_family */
XXH32_digest(const XXH32_state_t * state)3182 XXH_PUBLIC_API XXH32_hash_t XXH32_digest(const XXH32_state_t* state)
3183 {
3184 xxh_u32 h32;
3185
3186 if (state->large_len) {
3187 h32 = XXH_rotl32(state->v[0], 1)
3188 + XXH_rotl32(state->v[1], 7)
3189 + XXH_rotl32(state->v[2], 12)
3190 + XXH_rotl32(state->v[3], 18);
3191 } else {
3192 h32 = state->v[2] /* == seed */ + XXH_PRIME32_5;
3193 }
3194
3195 h32 += state->total_len_32;
3196
3197 return XXH32_finalize(h32, (const xxh_u8*)state->mem32, state->memsize, XXH_aligned);
3198 }
3199 #endif /* !XXH_NO_STREAM */
3200
3201 /******* Canonical representation *******/
3202
3203 /*! @ingroup XXH32_family */
XXH32_canonicalFromHash(XXH32_canonical_t * dst,XXH32_hash_t hash)3204 XXH_PUBLIC_API void XXH32_canonicalFromHash(XXH32_canonical_t* dst, XXH32_hash_t hash)
3205 {
3206 XXH_STATIC_ASSERT(sizeof(XXH32_canonical_t) == sizeof(XXH32_hash_t));
3207 if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap32(hash);
3208 XXH_memcpy(dst, &hash, sizeof(*dst));
3209 }
3210 /*! @ingroup XXH32_family */
XXH32_hashFromCanonical(const XXH32_canonical_t * src)3211 XXH_PUBLIC_API XXH32_hash_t XXH32_hashFromCanonical(const XXH32_canonical_t* src)
3212 {
3213 return XXH_readBE32(src);
3214 }
3215
3216
3217 #ifndef XXH_NO_LONG_LONG
3218
3219 /* *******************************************************************
3220 * 64-bit hash functions
3221 *********************************************************************/
3222 /*!
3223 * @}
3224 * @ingroup impl
3225 * @{
3226 */
3227 /******* Memory access *******/
3228
3229 typedef XXH64_hash_t xxh_u64;
3230
3231 #ifdef XXH_OLD_NAMES
3232 # define U64 xxh_u64
3233 #endif
3234
3235 #if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))
3236 /*
3237 * Manual byteshift. Best for old compilers which don't inline memcpy.
3238 * We actually directly use XXH_readLE64 and XXH_readBE64.
3239 */
3240 #elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==2))
3241
3242 /* Force direct memory access. Only works on CPU which support unaligned memory access in hardware */
XXH_read64(const void * memPtr)3243 static xxh_u64 XXH_read64(const void* memPtr)
3244 {
3245 return *(const xxh_u64*) memPtr;
3246 }
3247
3248 #elif (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==1))
3249
3250 /*
3251 * __attribute__((aligned(1))) is supported by gcc and clang. Originally the
3252 * documentation claimed that it only increased the alignment, but actually it
3253 * can decrease it on gcc, clang, and icc:
3254 * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=69502,
3255 * https://gcc.godbolt.org/z/xYez1j67Y.
3256 */
3257 #ifdef XXH_OLD_NAMES
3258 typedef union { xxh_u32 u32; xxh_u64 u64; } __attribute__((packed)) unalign64;
3259 #endif
XXH_read64(const void * ptr)3260 static xxh_u64 XXH_read64(const void* ptr)
3261 {
3262 typedef __attribute__((aligned(1))) xxh_u64 xxh_unalign64;
3263 return *((const xxh_unalign64*)ptr);
3264 }
3265
3266 #else
3267
3268 /*
3269 * Portable and safe solution. Generally efficient.
3270 * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html
3271 */
XXH_read64(const void * memPtr)3272 static xxh_u64 XXH_read64(const void* memPtr)
3273 {
3274 xxh_u64 val;
3275 XXH_memcpy(&val, memPtr, sizeof(val));
3276 return val;
3277 }
3278
3279 #endif /* XXH_FORCE_DIRECT_MEMORY_ACCESS */
3280
3281 #if defined(_MSC_VER) /* Visual Studio */
3282 # define XXH_swap64 _byteswap_uint64
3283 #elif XXH_GCC_VERSION >= 403
3284 # define XXH_swap64 __builtin_bswap64
3285 #else
XXH_swap64(xxh_u64 x)3286 static xxh_u64 XXH_swap64(xxh_u64 x)
3287 {
3288 return ((x << 56) & 0xff00000000000000ULL) |
3289 ((x << 40) & 0x00ff000000000000ULL) |
3290 ((x << 24) & 0x0000ff0000000000ULL) |
3291 ((x << 8) & 0x000000ff00000000ULL) |
3292 ((x >> 8) & 0x00000000ff000000ULL) |
3293 ((x >> 24) & 0x0000000000ff0000ULL) |
3294 ((x >> 40) & 0x000000000000ff00ULL) |
3295 ((x >> 56) & 0x00000000000000ffULL);
3296 }
3297 #endif
3298
3299
3300 /* XXH_FORCE_MEMORY_ACCESS==3 is an endian-independent byteshift load. */
3301 #if (defined(XXH_FORCE_MEMORY_ACCESS) && (XXH_FORCE_MEMORY_ACCESS==3))
3302
XXH_readLE64(const void * memPtr)3303 XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* memPtr)
3304 {
3305 const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
3306 return bytePtr[0]
3307 | ((xxh_u64)bytePtr[1] << 8)
3308 | ((xxh_u64)bytePtr[2] << 16)
3309 | ((xxh_u64)bytePtr[3] << 24)
3310 | ((xxh_u64)bytePtr[4] << 32)
3311 | ((xxh_u64)bytePtr[5] << 40)
3312 | ((xxh_u64)bytePtr[6] << 48)
3313 | ((xxh_u64)bytePtr[7] << 56);
3314 }
3315
XXH_readBE64(const void * memPtr)3316 XXH_FORCE_INLINE xxh_u64 XXH_readBE64(const void* memPtr)
3317 {
3318 const xxh_u8* bytePtr = (const xxh_u8 *)memPtr;
3319 return bytePtr[7]
3320 | ((xxh_u64)bytePtr[6] << 8)
3321 | ((xxh_u64)bytePtr[5] << 16)
3322 | ((xxh_u64)bytePtr[4] << 24)
3323 | ((xxh_u64)bytePtr[3] << 32)
3324 | ((xxh_u64)bytePtr[2] << 40)
3325 | ((xxh_u64)bytePtr[1] << 48)
3326 | ((xxh_u64)bytePtr[0] << 56);
3327 }
3328
3329 #else
XXH_readLE64(const void * ptr)3330 XXH_FORCE_INLINE xxh_u64 XXH_readLE64(const void* ptr)
3331 {
3332 return XXH_CPU_LITTLE_ENDIAN ? XXH_read64(ptr) : XXH_swap64(XXH_read64(ptr));
3333 }
3334
XXH_readBE64(const void * ptr)3335 static xxh_u64 XXH_readBE64(const void* ptr)
3336 {
3337 return XXH_CPU_LITTLE_ENDIAN ? XXH_swap64(XXH_read64(ptr)) : XXH_read64(ptr);
3338 }
3339 #endif
3340
3341 XXH_FORCE_INLINE xxh_u64
XXH_readLE64_align(const void * ptr,XXH_alignment align)3342 XXH_readLE64_align(const void* ptr, XXH_alignment align)
3343 {
3344 if (align==XXH_unaligned)
3345 return XXH_readLE64(ptr);
3346 else
3347 return XXH_CPU_LITTLE_ENDIAN ? *(const xxh_u64*)ptr : XXH_swap64(*(const xxh_u64*)ptr);
3348 }
3349
3350
3351 /******* xxh64 *******/
3352 /*!
3353 * @}
3354 * @defgroup XXH64_impl XXH64 implementation
3355 * @ingroup impl
3356 *
3357 * Details on the XXH64 implementation.
3358 * @{
3359 */
3360 /* #define rather that static const, to be used as initializers */
3361 #define XXH_PRIME64_1 0x9E3779B185EBCA87ULL /*!< 0b1001111000110111011110011011000110000101111010111100101010000111 */
3362 #define XXH_PRIME64_2 0xC2B2AE3D27D4EB4FULL /*!< 0b1100001010110010101011100011110100100111110101001110101101001111 */
3363 #define XXH_PRIME64_3 0x165667B19E3779F9ULL /*!< 0b0001011001010110011001111011000110011110001101110111100111111001 */
3364 #define XXH_PRIME64_4 0x85EBCA77C2B2AE63ULL /*!< 0b1000010111101011110010100111011111000010101100101010111001100011 */
3365 #define XXH_PRIME64_5 0x27D4EB2F165667C5ULL /*!< 0b0010011111010100111010110010111100010110010101100110011111000101 */
3366
3367 #ifdef XXH_OLD_NAMES
3368 # define PRIME64_1 XXH_PRIME64_1
3369 # define PRIME64_2 XXH_PRIME64_2
3370 # define PRIME64_3 XXH_PRIME64_3
3371 # define PRIME64_4 XXH_PRIME64_4
3372 # define PRIME64_5 XXH_PRIME64_5
3373 #endif
3374
3375 /*! @copydoc XXH32_round */
XXH64_round(xxh_u64 acc,xxh_u64 input)3376 static xxh_u64 XXH64_round(xxh_u64 acc, xxh_u64 input)
3377 {
3378 acc += input * XXH_PRIME64_2;
3379 acc = XXH_rotl64(acc, 31);
3380 acc *= XXH_PRIME64_1;
3381 #if (defined(__AVX512F__)) && !defined(XXH_ENABLE_AUTOVECTORIZE)
3382 /*
3383 * DISABLE AUTOVECTORIZATION:
3384 * A compiler fence is used to prevent GCC and Clang from
3385 * autovectorizing the XXH64 loop (pragmas and attributes don't work for some
3386 * reason) without globally disabling AVX512.
3387 *
3388 * Autovectorization of XXH64 tends to be detrimental,
3389 * though the exact outcome may change depending on exact cpu and compiler version.
3390 * For information, it has been reported as detrimental for Skylake-X,
3391 * but possibly beneficial for Zen4.
3392 *
3393 * The default is to disable auto-vectorization,
3394 * but you can select to enable it instead using `XXH_ENABLE_AUTOVECTORIZE` build variable.
3395 */
3396 XXH_COMPILER_GUARD(acc);
3397 #endif
3398 return acc;
3399 }
3400
XXH64_mergeRound(xxh_u64 acc,xxh_u64 val)3401 static xxh_u64 XXH64_mergeRound(xxh_u64 acc, xxh_u64 val)
3402 {
3403 val = XXH64_round(0, val);
3404 acc ^= val;
3405 acc = acc * XXH_PRIME64_1 + XXH_PRIME64_4;
3406 return acc;
3407 }
3408
3409 /*! @copydoc XXH32_avalanche */
XXH64_avalanche(xxh_u64 hash)3410 static xxh_u64 XXH64_avalanche(xxh_u64 hash)
3411 {
3412 hash ^= hash >> 33;
3413 hash *= XXH_PRIME64_2;
3414 hash ^= hash >> 29;
3415 hash *= XXH_PRIME64_3;
3416 hash ^= hash >> 32;
3417 return hash;
3418 }
3419
3420
3421 #define XXH_get64bits(p) XXH_readLE64_align(p, align)
3422
3423 /*!
3424 * @internal
3425 * @brief Processes the last 0-31 bytes of @p ptr.
3426 *
3427 * There may be up to 31 bytes remaining to consume from the input.
3428 * This final stage will digest them to ensure that all input bytes are present
3429 * in the final mix.
3430 *
3431 * @param hash The hash to finalize.
3432 * @param ptr The pointer to the remaining input.
3433 * @param len The remaining length, modulo 32.
3434 * @param align Whether @p ptr is aligned.
3435 * @return The finalized hash
3436 * @see XXH32_finalize().
3437 */
3438 static XXH_PUREF xxh_u64
XXH64_finalize(xxh_u64 hash,const xxh_u8 * ptr,size_t len,XXH_alignment align)3439 XXH64_finalize(xxh_u64 hash, const xxh_u8* ptr, size_t len, XXH_alignment align)
3440 {
3441 if (ptr==NULL) XXH_ASSERT(len == 0);
3442 len &= 31;
3443 while (len >= 8) {
3444 xxh_u64 const k1 = XXH64_round(0, XXH_get64bits(ptr));
3445 ptr += 8;
3446 hash ^= k1;
3447 hash = XXH_rotl64(hash,27) * XXH_PRIME64_1 + XXH_PRIME64_4;
3448 len -= 8;
3449 }
3450 if (len >= 4) {
3451 hash ^= (xxh_u64)(XXH_get32bits(ptr)) * XXH_PRIME64_1;
3452 ptr += 4;
3453 hash = XXH_rotl64(hash, 23) * XXH_PRIME64_2 + XXH_PRIME64_3;
3454 len -= 4;
3455 }
3456 while (len > 0) {
3457 hash ^= (*ptr++) * XXH_PRIME64_5;
3458 hash = XXH_rotl64(hash, 11) * XXH_PRIME64_1;
3459 --len;
3460 }
3461 return XXH64_avalanche(hash);
3462 }
3463
3464 #ifdef XXH_OLD_NAMES
3465 # define PROCESS1_64 XXH_PROCESS1_64
3466 # define PROCESS4_64 XXH_PROCESS4_64
3467 # define PROCESS8_64 XXH_PROCESS8_64
3468 #else
3469 # undef XXH_PROCESS1_64
3470 # undef XXH_PROCESS4_64
3471 # undef XXH_PROCESS8_64
3472 #endif
3473
3474 /*!
3475 * @internal
3476 * @brief The implementation for @ref XXH64().
3477 *
3478 * @param input , len , seed Directly passed from @ref XXH64().
3479 * @param align Whether @p input is aligned.
3480 * @return The calculated hash.
3481 */
3482 XXH_FORCE_INLINE XXH_PUREF xxh_u64
XXH64_endian_align(const xxh_u8 * input,size_t len,xxh_u64 seed,XXH_alignment align)3483 XXH64_endian_align(const xxh_u8* input, size_t len, xxh_u64 seed, XXH_alignment align)
3484 {
3485 xxh_u64 h64;
3486 if (input==NULL) XXH_ASSERT(len == 0);
3487
3488 if (len>=32) {
3489 const xxh_u8* const bEnd = input + len;
3490 const xxh_u8* const limit = bEnd - 31;
3491 xxh_u64 v1 = seed + XXH_PRIME64_1 + XXH_PRIME64_2;
3492 xxh_u64 v2 = seed + XXH_PRIME64_2;
3493 xxh_u64 v3 = seed + 0;
3494 xxh_u64 v4 = seed - XXH_PRIME64_1;
3495
3496 do {
3497 v1 = XXH64_round(v1, XXH_get64bits(input)); input+=8;
3498 v2 = XXH64_round(v2, XXH_get64bits(input)); input+=8;
3499 v3 = XXH64_round(v3, XXH_get64bits(input)); input+=8;
3500 v4 = XXH64_round(v4, XXH_get64bits(input)); input+=8;
3501 } while (input<limit);
3502
3503 h64 = XXH_rotl64(v1, 1) + XXH_rotl64(v2, 7) + XXH_rotl64(v3, 12) + XXH_rotl64(v4, 18);
3504 h64 = XXH64_mergeRound(h64, v1);
3505 h64 = XXH64_mergeRound(h64, v2);
3506 h64 = XXH64_mergeRound(h64, v3);
3507 h64 = XXH64_mergeRound(h64, v4);
3508
3509 } else {
3510 h64 = seed + XXH_PRIME64_5;
3511 }
3512
3513 h64 += (xxh_u64) len;
3514
3515 return XXH64_finalize(h64, input, len, align);
3516 }
3517
3518
3519 /*! @ingroup XXH64_family */
XXH64(XXH_NOESCAPE const void * input,size_t len,XXH64_hash_t seed)3520 XXH_PUBLIC_API XXH64_hash_t XXH64 (XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed)
3521 {
3522 #if !defined(XXH_NO_STREAM) && XXH_SIZE_OPT >= 2
3523 /* Simple version, good for code maintenance, but unfortunately slow for small inputs */
3524 XXH64_state_t state;
3525 XXH64_reset(&state, seed);
3526 XXH64_update(&state, (const xxh_u8*)input, len);
3527 return XXH64_digest(&state);
3528 #else
3529 if (XXH_FORCE_ALIGN_CHECK) {
3530 if ((((size_t)input) & 7)==0) { /* Input is aligned, let's leverage the speed advantage */
3531 return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_aligned);
3532 } }
3533
3534 return XXH64_endian_align((const xxh_u8*)input, len, seed, XXH_unaligned);
3535
3536 #endif
3537 }
3538
3539 /******* Hash Streaming *******/
3540 #ifndef XXH_NO_STREAM
3541 /*! @ingroup XXH64_family*/
XXH64_createState(void)3542 XXH_PUBLIC_API XXH64_state_t* XXH64_createState(void)
3543 {
3544 return (XXH64_state_t*)XXH_malloc(sizeof(XXH64_state_t));
3545 }
3546 /*! @ingroup XXH64_family */
XXH64_freeState(XXH64_state_t * statePtr)3547 XXH_PUBLIC_API XXH_errorcode XXH64_freeState(XXH64_state_t* statePtr)
3548 {
3549 XXH_free(statePtr);
3550 return XXH_OK;
3551 }
3552
3553 /*! @ingroup XXH64_family */
XXH64_copyState(XXH_NOESCAPE XXH64_state_t * dstState,const XXH64_state_t * srcState)3554 XXH_PUBLIC_API void XXH64_copyState(XXH_NOESCAPE XXH64_state_t* dstState, const XXH64_state_t* srcState)
3555 {
3556 XXH_memcpy(dstState, srcState, sizeof(*dstState));
3557 }
3558
3559 /*! @ingroup XXH64_family */
XXH64_reset(XXH_NOESCAPE XXH64_state_t * statePtr,XXH64_hash_t seed)3560 XXH_PUBLIC_API XXH_errorcode XXH64_reset(XXH_NOESCAPE XXH64_state_t* statePtr, XXH64_hash_t seed)
3561 {
3562 XXH_ASSERT(statePtr != NULL);
3563 memset(statePtr, 0, sizeof(*statePtr));
3564 statePtr->v[0] = seed + XXH_PRIME64_1 + XXH_PRIME64_2;
3565 statePtr->v[1] = seed + XXH_PRIME64_2;
3566 statePtr->v[2] = seed + 0;
3567 statePtr->v[3] = seed - XXH_PRIME64_1;
3568 return XXH_OK;
3569 }
3570
3571 /*! @ingroup XXH64_family */
3572 XXH_PUBLIC_API XXH_errorcode
XXH64_update(XXH_NOESCAPE XXH64_state_t * state,XXH_NOESCAPE const void * input,size_t len)3573 XXH64_update (XXH_NOESCAPE XXH64_state_t* state, XXH_NOESCAPE const void* input, size_t len)
3574 {
3575 if (input==NULL) {
3576 XXH_ASSERT(len == 0);
3577 return XXH_OK;
3578 }
3579
3580 { const xxh_u8* p = (const xxh_u8*)input;
3581 const xxh_u8* const bEnd = p + len;
3582
3583 state->total_len += len;
3584
3585 if (state->memsize + len < 32) { /* fill in tmp buffer */
3586 XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, len);
3587 state->memsize += (xxh_u32)len;
3588 return XXH_OK;
3589 }
3590
3591 if (state->memsize) { /* tmp buffer is full */
3592 XXH_memcpy(((xxh_u8*)state->mem64) + state->memsize, input, 32-state->memsize);
3593 state->v[0] = XXH64_round(state->v[0], XXH_readLE64(state->mem64+0));
3594 state->v[1] = XXH64_round(state->v[1], XXH_readLE64(state->mem64+1));
3595 state->v[2] = XXH64_round(state->v[2], XXH_readLE64(state->mem64+2));
3596 state->v[3] = XXH64_round(state->v[3], XXH_readLE64(state->mem64+3));
3597 p += 32 - state->memsize;
3598 state->memsize = 0;
3599 }
3600
3601 if (p+32 <= bEnd) {
3602 const xxh_u8* const limit = bEnd - 32;
3603
3604 do {
3605 state->v[0] = XXH64_round(state->v[0], XXH_readLE64(p)); p+=8;
3606 state->v[1] = XXH64_round(state->v[1], XXH_readLE64(p)); p+=8;
3607 state->v[2] = XXH64_round(state->v[2], XXH_readLE64(p)); p+=8;
3608 state->v[3] = XXH64_round(state->v[3], XXH_readLE64(p)); p+=8;
3609 } while (p<=limit);
3610
3611 }
3612
3613 if (p < bEnd) {
3614 XXH_memcpy(state->mem64, p, (size_t)(bEnd-p));
3615 state->memsize = (unsigned)(bEnd-p);
3616 }
3617 }
3618
3619 return XXH_OK;
3620 }
3621
3622
3623 /*! @ingroup XXH64_family */
XXH64_digest(XXH_NOESCAPE const XXH64_state_t * state)3624 XXH_PUBLIC_API XXH64_hash_t XXH64_digest(XXH_NOESCAPE const XXH64_state_t* state)
3625 {
3626 xxh_u64 h64;
3627
3628 if (state->total_len >= 32) {
3629 h64 = XXH_rotl64(state->v[0], 1) + XXH_rotl64(state->v[1], 7) + XXH_rotl64(state->v[2], 12) + XXH_rotl64(state->v[3], 18);
3630 h64 = XXH64_mergeRound(h64, state->v[0]);
3631 h64 = XXH64_mergeRound(h64, state->v[1]);
3632 h64 = XXH64_mergeRound(h64, state->v[2]);
3633 h64 = XXH64_mergeRound(h64, state->v[3]);
3634 } else {
3635 h64 = state->v[2] /*seed*/ + XXH_PRIME64_5;
3636 }
3637
3638 h64 += (xxh_u64) state->total_len;
3639
3640 return XXH64_finalize(h64, (const xxh_u8*)state->mem64, (size_t)state->total_len, XXH_aligned);
3641 }
3642 #endif /* !XXH_NO_STREAM */
3643
3644 /******* Canonical representation *******/
3645
3646 /*! @ingroup XXH64_family */
XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t * dst,XXH64_hash_t hash)3647 XXH_PUBLIC_API void XXH64_canonicalFromHash(XXH_NOESCAPE XXH64_canonical_t* dst, XXH64_hash_t hash)
3648 {
3649 XXH_STATIC_ASSERT(sizeof(XXH64_canonical_t) == sizeof(XXH64_hash_t));
3650 if (XXH_CPU_LITTLE_ENDIAN) hash = XXH_swap64(hash);
3651 XXH_memcpy(dst, &hash, sizeof(*dst));
3652 }
3653
3654 /*! @ingroup XXH64_family */
XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t * src)3655 XXH_PUBLIC_API XXH64_hash_t XXH64_hashFromCanonical(XXH_NOESCAPE const XXH64_canonical_t* src)
3656 {
3657 return XXH_readBE64(src);
3658 }
3659
3660 #if defined (__cplusplus)
3661 }
3662 #endif
3663
3664 #ifndef XXH_NO_XXH3
3665
3666 /* *********************************************************************
3667 * XXH3
3668 * New generation hash designed for speed on small keys and vectorization
3669 ************************************************************************ */
3670 /*!
3671 * @}
3672 * @defgroup XXH3_impl XXH3 implementation
3673 * @ingroup impl
3674 * @{
3675 */
3676
3677 /* === Compiler specifics === */
3678
3679 #if ((defined(sun) || defined(__sun)) && __cplusplus) /* Solaris includes __STDC_VERSION__ with C++. Tested with GCC 5.5 */
3680 # define XXH_RESTRICT /* disable */
3681 #elif defined (__STDC_VERSION__) && __STDC_VERSION__ >= 199901L /* >= C99 */
3682 # define XXH_RESTRICT restrict
3683 #elif (defined (__GNUC__) && ((__GNUC__ > 3) || (__GNUC__ == 3 && __GNUC_MINOR__ >= 1))) \
3684 || (defined (__clang__)) \
3685 || (defined (_MSC_VER) && (_MSC_VER >= 1400)) \
3686 || (defined (__INTEL_COMPILER) && (__INTEL_COMPILER >= 1300))
3687 /*
3688 * There are a LOT more compilers that recognize __restrict but this
3689 * covers the major ones.
3690 */
3691 # define XXH_RESTRICT __restrict
3692 #else
3693 # define XXH_RESTRICT /* disable */
3694 #endif
3695
3696 #if (defined(__GNUC__) && (__GNUC__ >= 3)) \
3697 || (defined(__INTEL_COMPILER) && (__INTEL_COMPILER >= 800)) \
3698 || defined(__clang__)
3699 # define XXH_likely(x) __builtin_expect(x, 1)
3700 # define XXH_unlikely(x) __builtin_expect(x, 0)
3701 #else
3702 # define XXH_likely(x) (x)
3703 # define XXH_unlikely(x) (x)
3704 #endif
3705
3706 #ifndef XXH_HAS_INCLUDE
3707 # ifdef __has_include
3708 /*
3709 * Not defined as XXH_HAS_INCLUDE(x) (function-like) because
3710 * this causes segfaults in Apple Clang 4.2 (on Mac OS X 10.7 Lion)
3711 */
3712 # define XXH_HAS_INCLUDE __has_include
3713 # else
3714 # define XXH_HAS_INCLUDE(x) 0
3715 # endif
3716 #endif
3717
3718 #if defined(__GNUC__) || defined(__clang__)
3719 # if defined(__ARM_FEATURE_SVE)
3720 # include <arm_sve.h>
3721 # endif
3722 # if defined(__ARM_NEON__) || defined(__ARM_NEON) \
3723 || (defined(_M_ARM) && _M_ARM >= 7) \
3724 || defined(_M_ARM64) || defined(_M_ARM64EC) \
3725 || (defined(__wasm_simd128__) && XXH_HAS_INCLUDE(<arm_neon.h>)) /* WASM SIMD128 via SIMDe */
3726 # define inline __inline__ /* circumvent a clang bug */
3727 # include <arm_neon.h>
3728 # undef inline
3729 # elif defined(__AVX2__)
3730 # include <immintrin.h>
3731 # elif defined(__SSE2__)
3732 # include <emmintrin.h>
3733 # endif
3734 #endif
3735
3736 #if defined(_MSC_VER)
3737 # include <intrin.h>
3738 #endif
3739
3740 /*
3741 * One goal of XXH3 is to make it fast on both 32-bit and 64-bit, while
3742 * remaining a true 64-bit/128-bit hash function.
3743 *
3744 * This is done by prioritizing a subset of 64-bit operations that can be
3745 * emulated without too many steps on the average 32-bit machine.
3746 *
3747 * For example, these two lines seem similar, and run equally fast on 64-bit:
3748 *
3749 * xxh_u64 x;
3750 * x ^= (x >> 47); // good
3751 * x ^= (x >> 13); // bad
3752 *
3753 * However, to a 32-bit machine, there is a major difference.
3754 *
3755 * x ^= (x >> 47) looks like this:
3756 *
3757 * x.lo ^= (x.hi >> (47 - 32));
3758 *
3759 * while x ^= (x >> 13) looks like this:
3760 *
3761 * // note: funnel shifts are not usually cheap.
3762 * x.lo ^= (x.lo >> 13) | (x.hi << (32 - 13));
3763 * x.hi ^= (x.hi >> 13);
3764 *
3765 * The first one is significantly faster than the second, simply because the
3766 * shift is larger than 32. This means:
3767 * - All the bits we need are in the upper 32 bits, so we can ignore the lower
3768 * 32 bits in the shift.
3769 * - The shift result will always fit in the lower 32 bits, and therefore,
3770 * we can ignore the upper 32 bits in the xor.
3771 *
3772 * Thanks to this optimization, XXH3 only requires these features to be efficient:
3773 *
3774 * - Usable unaligned access
3775 * - A 32-bit or 64-bit ALU
3776 * - If 32-bit, a decent ADC instruction
3777 * - A 32 or 64-bit multiply with a 64-bit result
3778 * - For the 128-bit variant, a decent byteswap helps short inputs.
3779 *
3780 * The first two are already required by XXH32, and almost all 32-bit and 64-bit
3781 * platforms which can run XXH32 can run XXH3 efficiently.
3782 *
3783 * Thumb-1, the classic 16-bit only subset of ARM's instruction set, is one
3784 * notable exception.
3785 *
3786 * First of all, Thumb-1 lacks support for the UMULL instruction which
3787 * performs the important long multiply. This means numerous __aeabi_lmul
3788 * calls.
3789 *
3790 * Second of all, the 8 functional registers are just not enough.
3791 * Setup for __aeabi_lmul, byteshift loads, pointers, and all arithmetic need
3792 * Lo registers, and this shuffling results in thousands more MOVs than A32.
3793 *
3794 * A32 and T32 don't have this limitation. They can access all 14 registers,
3795 * do a 32->64 multiply with UMULL, and the flexible operand allowing free
3796 * shifts is helpful, too.
3797 *
3798 * Therefore, we do a quick sanity check.
3799 *
3800 * If compiling Thumb-1 for a target which supports ARM instructions, we will
3801 * emit a warning, as it is not a "sane" platform to compile for.
3802 *
3803 * Usually, if this happens, it is because of an accident and you probably need
3804 * to specify -march, as you likely meant to compile for a newer architecture.
3805 *
3806 * Credit: large sections of the vectorial and asm source code paths
3807 * have been contributed by @easyaspi314
3808 */
3809 #if defined(__thumb__) && !defined(__thumb2__) && defined(__ARM_ARCH_ISA_ARM)
3810 # warning "XXH3 is highly inefficient without ARM or Thumb-2."
3811 #endif
3812
3813 /* ==========================================
3814 * Vectorization detection
3815 * ========================================== */
3816
3817 #ifdef XXH_DOXYGEN
3818 /*!
3819 * @ingroup tuning
3820 * @brief Overrides the vectorization implementation chosen for XXH3.
3821 *
3822 * Can be defined to 0 to disable SIMD or any of the values mentioned in
3823 * @ref XXH_VECTOR_TYPE.
3824 *
3825 * If this is not defined, it uses predefined macros to determine the best
3826 * implementation.
3827 */
3828 # define XXH_VECTOR XXH_SCALAR
3829 /*!
3830 * @ingroup tuning
3831 * @brief Possible values for @ref XXH_VECTOR.
3832 *
3833 * Note that these are actually implemented as macros.
3834 *
3835 * If this is not defined, it is detected automatically.
3836 * internal macro XXH_X86DISPATCH overrides this.
3837 */
3838 enum XXH_VECTOR_TYPE /* fake enum */ {
3839 XXH_SCALAR = 0, /*!< Portable scalar version */
3840 XXH_SSE2 = 1, /*!<
3841 * SSE2 for Pentium 4, Opteron, all x86_64.
3842 *
3843 * @note SSE2 is also guaranteed on Windows 10, macOS, and
3844 * Android x86.
3845 */
3846 XXH_AVX2 = 2, /*!< AVX2 for Haswell and Bulldozer */
3847 XXH_AVX512 = 3, /*!< AVX512 for Skylake and Icelake */
3848 XXH_NEON = 4, /*!<
3849 * NEON for most ARMv7-A, all AArch64, and WASM SIMD128
3850 * via the SIMDeverywhere polyfill provided with the
3851 * Emscripten SDK.
3852 */
3853 XXH_VSX = 5, /*!< VSX and ZVector for POWER8/z13 (64-bit) */
3854 XXH_SVE = 6, /*!< SVE for some ARMv8-A and ARMv9-A */
3855 };
3856 /*!
3857 * @ingroup tuning
3858 * @brief Selects the minimum alignment for XXH3's accumulators.
3859 *
3860 * When using SIMD, this should match the alignment required for said vector
3861 * type, so, for example, 32 for AVX2.
3862 *
3863 * Default: Auto detected.
3864 */
3865 # define XXH_ACC_ALIGN 8
3866 #endif
3867
3868 /* Actual definition */
3869 #ifndef XXH_DOXYGEN
3870 # define XXH_SCALAR 0
3871 # define XXH_SSE2 1
3872 # define XXH_AVX2 2
3873 # define XXH_AVX512 3
3874 # define XXH_NEON 4
3875 # define XXH_VSX 5
3876 # define XXH_SVE 6
3877 #endif
3878
3879 #ifndef XXH_VECTOR /* can be defined on command line */
3880 # if defined(__ARM_FEATURE_SVE)
3881 # define XXH_VECTOR XXH_SVE
3882 # elif ( \
3883 defined(__ARM_NEON__) || defined(__ARM_NEON) /* gcc */ \
3884 || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_ARM64EC) /* msvc */ \
3885 || (defined(__wasm_simd128__) && XXH_HAS_INCLUDE(<arm_neon.h>)) /* wasm simd128 via SIMDe */ \
3886 ) && ( \
3887 defined(_WIN32) || defined(__LITTLE_ENDIAN__) /* little endian only */ \
3888 || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__) \
3889 )
3890 # define XXH_VECTOR XXH_NEON
3891 # elif defined(__AVX512F__)
3892 # define XXH_VECTOR XXH_AVX512
3893 # elif defined(__AVX2__)
3894 # define XXH_VECTOR XXH_AVX2
3895 # elif defined(__SSE2__) || defined(_M_X64) || (defined(_M_IX86_FP) && (_M_IX86_FP == 2))
3896 # define XXH_VECTOR XXH_SSE2
3897 # elif (defined(__PPC64__) && defined(__POWER8_VECTOR__)) \
3898 || (defined(__s390x__) && defined(__VEC__)) \
3899 && defined(__GNUC__) /* TODO: IBM XL */
3900 # define XXH_VECTOR XXH_VSX
3901 # else
3902 # define XXH_VECTOR XXH_SCALAR
3903 # endif
3904 #endif
3905
3906 /* __ARM_FEATURE_SVE is only supported by GCC & Clang. */
3907 #if (XXH_VECTOR == XXH_SVE) && !defined(__ARM_FEATURE_SVE)
3908 # ifdef _MSC_VER
3909 # pragma warning(once : 4606)
3910 # else
3911 # warning "__ARM_FEATURE_SVE isn't supported. Use SCALAR instead."
3912 # endif
3913 # undef XXH_VECTOR
3914 # define XXH_VECTOR XXH_SCALAR
3915 #endif
3916
3917 /*
3918 * Controls the alignment of the accumulator,
3919 * for compatibility with aligned vector loads, which are usually faster.
3920 */
3921 #ifndef XXH_ACC_ALIGN
3922 # if defined(XXH_X86DISPATCH)
3923 # define XXH_ACC_ALIGN 64 /* for compatibility with avx512 */
3924 # elif XXH_VECTOR == XXH_SCALAR /* scalar */
3925 # define XXH_ACC_ALIGN 8
3926 # elif XXH_VECTOR == XXH_SSE2 /* sse2 */
3927 # define XXH_ACC_ALIGN 16
3928 # elif XXH_VECTOR == XXH_AVX2 /* avx2 */
3929 # define XXH_ACC_ALIGN 32
3930 # elif XXH_VECTOR == XXH_NEON /* neon */
3931 # define XXH_ACC_ALIGN 16
3932 # elif XXH_VECTOR == XXH_VSX /* vsx */
3933 # define XXH_ACC_ALIGN 16
3934 # elif XXH_VECTOR == XXH_AVX512 /* avx512 */
3935 # define XXH_ACC_ALIGN 64
3936 # elif XXH_VECTOR == XXH_SVE /* sve */
3937 # define XXH_ACC_ALIGN 64
3938 # endif
3939 #endif
3940
3941 #if defined(XXH_X86DISPATCH) || XXH_VECTOR == XXH_SSE2 \
3942 || XXH_VECTOR == XXH_AVX2 || XXH_VECTOR == XXH_AVX512
3943 # define XXH_SEC_ALIGN XXH_ACC_ALIGN
3944 #elif XXH_VECTOR == XXH_SVE
3945 # define XXH_SEC_ALIGN XXH_ACC_ALIGN
3946 #else
3947 # define XXH_SEC_ALIGN 8
3948 #endif
3949
3950 #if defined(__GNUC__) || defined(__clang__)
3951 # define XXH_ALIASING __attribute__((may_alias))
3952 #else
3953 # define XXH_ALIASING /* nothing */
3954 #endif
3955
3956 /*
3957 * UGLY HACK:
3958 * GCC usually generates the best code with -O3 for xxHash.
3959 *
3960 * However, when targeting AVX2, it is overzealous in its unrolling resulting
3961 * in code roughly 3/4 the speed of Clang.
3962 *
3963 * There are other issues, such as GCC splitting _mm256_loadu_si256 into
3964 * _mm_loadu_si128 + _mm256_inserti128_si256. This is an optimization which
3965 * only applies to Sandy and Ivy Bridge... which don't even support AVX2.
3966 *
3967 * That is why when compiling the AVX2 version, it is recommended to use either
3968 * -O2 -mavx2 -march=haswell
3969 * or
3970 * -O2 -mavx2 -mno-avx256-split-unaligned-load
3971 * for decent performance, or to use Clang instead.
3972 *
3973 * Fortunately, we can control the first one with a pragma that forces GCC into
3974 * -O2, but the other one we can't control without "failed to inline always
3975 * inline function due to target mismatch" warnings.
3976 */
3977 #if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \
3978 && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \
3979 && defined(__OPTIMIZE__) && XXH_SIZE_OPT <= 0 /* respect -O0 and -Os */
3980 # pragma GCC push_options
3981 # pragma GCC optimize("-O2")
3982 #endif
3983
3984 #if defined (__cplusplus)
3985 extern "C" {
3986 #endif
3987
3988 #if XXH_VECTOR == XXH_NEON
3989
3990 /*
3991 * UGLY HACK: While AArch64 GCC on Linux does not seem to care, on macOS, GCC -O3
3992 * optimizes out the entire hashLong loop because of the aliasing violation.
3993 *
3994 * However, GCC is also inefficient at load-store optimization with vld1q/vst1q,
3995 * so the only option is to mark it as aliasing.
3996 */
3997 typedef uint64x2_t xxh_aliasing_uint64x2_t XXH_ALIASING;
3998
3999 /*!
4000 * @internal
4001 * @brief `vld1q_u64` but faster and alignment-safe.
4002 *
4003 * On AArch64, unaligned access is always safe, but on ARMv7-a, it is only
4004 * *conditionally* safe (`vld1` has an alignment bit like `movdq[ua]` in x86).
4005 *
4006 * GCC for AArch64 sees `vld1q_u8` as an intrinsic instead of a load, so it
4007 * prohibits load-store optimizations. Therefore, a direct dereference is used.
4008 *
4009 * Otherwise, `vld1q_u8` is used with `vreinterpretq_u8_u64` to do a safe
4010 * unaligned load.
4011 */
4012 #if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__)
XXH_vld1q_u64(void const * ptr)4013 XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const* ptr) /* silence -Wcast-align */
4014 {
4015 return *(xxh_aliasing_uint64x2_t const *)ptr;
4016 }
4017 #else
XXH_vld1q_u64(void const * ptr)4018 XXH_FORCE_INLINE uint64x2_t XXH_vld1q_u64(void const* ptr)
4019 {
4020 return vreinterpretq_u64_u8(vld1q_u8((uint8_t const*)ptr));
4021 }
4022 #endif
4023
4024 /*!
4025 * @internal
4026 * @brief `vmlal_u32` on low and high halves of a vector.
4027 *
4028 * This is a workaround for AArch64 GCC < 11 which implemented arm_neon.h with
4029 * inline assembly and were therefore incapable of merging the `vget_{low, high}_u32`
4030 * with `vmlal_u32`.
4031 */
4032 #if defined(__aarch64__) && defined(__GNUC__) && !defined(__clang__) && __GNUC__ < 11
4033 XXH_FORCE_INLINE uint64x2_t
XXH_vmlal_low_u32(uint64x2_t acc,uint32x4_t lhs,uint32x4_t rhs)4034 XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs)
4035 {
4036 /* Inline assembly is the only way */
4037 __asm__("umlal %0.2d, %1.2s, %2.2s" : "+w" (acc) : "w" (lhs), "w" (rhs));
4038 return acc;
4039 }
4040 XXH_FORCE_INLINE uint64x2_t
XXH_vmlal_high_u32(uint64x2_t acc,uint32x4_t lhs,uint32x4_t rhs)4041 XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs)
4042 {
4043 /* This intrinsic works as expected */
4044 return vmlal_high_u32(acc, lhs, rhs);
4045 }
4046 #else
4047 /* Portable intrinsic versions */
4048 XXH_FORCE_INLINE uint64x2_t
XXH_vmlal_low_u32(uint64x2_t acc,uint32x4_t lhs,uint32x4_t rhs)4049 XXH_vmlal_low_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs)
4050 {
4051 return vmlal_u32(acc, vget_low_u32(lhs), vget_low_u32(rhs));
4052 }
4053 /*! @copydoc XXH_vmlal_low_u32
4054 * Assume the compiler converts this to vmlal_high_u32 on aarch64 */
4055 XXH_FORCE_INLINE uint64x2_t
XXH_vmlal_high_u32(uint64x2_t acc,uint32x4_t lhs,uint32x4_t rhs)4056 XXH_vmlal_high_u32(uint64x2_t acc, uint32x4_t lhs, uint32x4_t rhs)
4057 {
4058 return vmlal_u32(acc, vget_high_u32(lhs), vget_high_u32(rhs));
4059 }
4060 #endif
4061
4062 /*!
4063 * @ingroup tuning
4064 * @brief Controls the NEON to scalar ratio for XXH3
4065 *
4066 * This can be set to 2, 4, 6, or 8.
4067 *
4068 * ARM Cortex CPUs are _very_ sensitive to how their pipelines are used.
4069 *
4070 * For example, the Cortex-A73 can dispatch 3 micro-ops per cycle, but only 2 of those
4071 * can be NEON. If you are only using NEON instructions, you are only using 2/3 of the CPU
4072 * bandwidth.
4073 *
4074 * This is even more noticeable on the more advanced cores like the Cortex-A76 which
4075 * can dispatch 8 micro-ops per cycle, but still only 2 NEON micro-ops at once.
4076 *
4077 * Therefore, to make the most out of the pipeline, it is beneficial to run 6 NEON lanes
4078 * and 2 scalar lanes, which is chosen by default.
4079 *
4080 * This does not apply to Apple processors or 32-bit processors, which run better with
4081 * full NEON. These will default to 8. Additionally, size-optimized builds run 8 lanes.
4082 *
4083 * This change benefits CPUs with large micro-op buffers without negatively affecting
4084 * most other CPUs:
4085 *
4086 * | Chipset | Dispatch type | NEON only | 6:2 hybrid | Diff. |
4087 * |:----------------------|:--------------------|----------:|-----------:|------:|
4088 * | Snapdragon 730 (A76) | 2 NEON/8 micro-ops | 8.8 GB/s | 10.1 GB/s | ~16% |
4089 * | Snapdragon 835 (A73) | 2 NEON/3 micro-ops | 5.1 GB/s | 5.3 GB/s | ~5% |
4090 * | Marvell PXA1928 (A53) | In-order dual-issue | 1.9 GB/s | 1.9 GB/s | 0% |
4091 * | Apple M1 | 4 NEON/8 micro-ops | 37.3 GB/s | 36.1 GB/s | ~-3% |
4092 *
4093 * It also seems to fix some bad codegen on GCC, making it almost as fast as clang.
4094 *
4095 * When using WASM SIMD128, if this is 2 or 6, SIMDe will scalarize 2 of the lanes meaning
4096 * it effectively becomes worse 4.
4097 *
4098 * @see XXH3_accumulate_512_neon()
4099 */
4100 # ifndef XXH3_NEON_LANES
4101 # if (defined(__aarch64__) || defined(__arm64__) || defined(_M_ARM64) || defined(_M_ARM64EC)) \
4102 && !defined(__APPLE__) && XXH_SIZE_OPT <= 0
4103 # define XXH3_NEON_LANES 6
4104 # else
4105 # define XXH3_NEON_LANES XXH_ACC_NB
4106 # endif
4107 # endif
4108 #endif /* XXH_VECTOR == XXH_NEON */
4109
4110 #if defined (__cplusplus)
4111 } /* extern "C" */
4112 #endif
4113
4114 /*
4115 * VSX and Z Vector helpers.
4116 *
4117 * This is very messy, and any pull requests to clean this up are welcome.
4118 *
4119 * There are a lot of problems with supporting VSX and s390x, due to
4120 * inconsistent intrinsics, spotty coverage, and multiple endiannesses.
4121 */
4122 #if XXH_VECTOR == XXH_VSX
4123 /* Annoyingly, these headers _may_ define three macros: `bool`, `vector`,
4124 * and `pixel`. This is a problem for obvious reasons.
4125 *
4126 * These keywords are unnecessary; the spec literally says they are
4127 * equivalent to `__bool`, `__vector`, and `__pixel` and may be undef'd
4128 * after including the header.
4129 *
4130 * We use pragma push_macro/pop_macro to keep the namespace clean. */
4131 # pragma push_macro("bool")
4132 # pragma push_macro("vector")
4133 # pragma push_macro("pixel")
4134 /* silence potential macro redefined warnings */
4135 # undef bool
4136 # undef vector
4137 # undef pixel
4138
4139 # if defined(__s390x__)
4140 # include <s390intrin.h>
4141 # else
4142 # include <altivec.h>
4143 # endif
4144
4145 /* Restore the original macro values, if applicable. */
4146 # pragma pop_macro("pixel")
4147 # pragma pop_macro("vector")
4148 # pragma pop_macro("bool")
4149
4150 typedef __vector unsigned long long xxh_u64x2;
4151 typedef __vector unsigned char xxh_u8x16;
4152 typedef __vector unsigned xxh_u32x4;
4153
4154 /*
4155 * UGLY HACK: Similar to aarch64 macOS GCC, s390x GCC has the same aliasing issue.
4156 */
4157 typedef xxh_u64x2 xxh_aliasing_u64x2 XXH_ALIASING;
4158
4159 # ifndef XXH_VSX_BE
4160 # if defined(__BIG_ENDIAN__) \
4161 || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
4162 # define XXH_VSX_BE 1
4163 # elif defined(__VEC_ELEMENT_REG_ORDER__) && __VEC_ELEMENT_REG_ORDER__ == __ORDER_BIG_ENDIAN__
4164 # warning "-maltivec=be is not recommended. Please use native endianness."
4165 # define XXH_VSX_BE 1
4166 # else
4167 # define XXH_VSX_BE 0
4168 # endif
4169 # endif /* !defined(XXH_VSX_BE) */
4170
4171 # if XXH_VSX_BE
4172 # if defined(__POWER9_VECTOR__) || (defined(__clang__) && defined(__s390x__))
4173 # define XXH_vec_revb vec_revb
4174 # else
4175 #if defined (__cplusplus)
4176 extern "C" {
4177 #endif
4178 /*!
4179 * A polyfill for POWER9's vec_revb().
4180 */
XXH_vec_revb(xxh_u64x2 val)4181 XXH_FORCE_INLINE xxh_u64x2 XXH_vec_revb(xxh_u64x2 val)
4182 {
4183 xxh_u8x16 const vByteSwap = { 0x07, 0x06, 0x05, 0x04, 0x03, 0x02, 0x01, 0x00,
4184 0x0F, 0x0E, 0x0D, 0x0C, 0x0B, 0x0A, 0x09, 0x08 };
4185 return vec_perm(val, val, vByteSwap);
4186 }
4187 #if defined (__cplusplus)
4188 } /* extern "C" */
4189 #endif
4190 # endif
4191 # endif /* XXH_VSX_BE */
4192
4193 #if defined (__cplusplus)
4194 extern "C" {
4195 #endif
4196 /*!
4197 * Performs an unaligned vector load and byte swaps it on big endian.
4198 */
XXH_vec_loadu(const void * ptr)4199 XXH_FORCE_INLINE xxh_u64x2 XXH_vec_loadu(const void *ptr)
4200 {
4201 xxh_u64x2 ret;
4202 XXH_memcpy(&ret, ptr, sizeof(xxh_u64x2));
4203 # if XXH_VSX_BE
4204 ret = XXH_vec_revb(ret);
4205 # endif
4206 return ret;
4207 }
4208
4209 /*
4210 * vec_mulo and vec_mule are very problematic intrinsics on PowerPC
4211 *
4212 * These intrinsics weren't added until GCC 8, despite existing for a while,
4213 * and they are endian dependent. Also, their meaning swap depending on version.
4214 * */
4215 # if defined(__s390x__)
4216 /* s390x is always big endian, no issue on this platform */
4217 # define XXH_vec_mulo vec_mulo
4218 # define XXH_vec_mule vec_mule
4219 # elif defined(__clang__) && XXH_HAS_BUILTIN(__builtin_altivec_vmuleuw) && !defined(__ibmxl__)
4220 /* Clang has a better way to control this, we can just use the builtin which doesn't swap. */
4221 /* The IBM XL Compiler (which defined __clang__) only implements the vec_* operations */
4222 # define XXH_vec_mulo __builtin_altivec_vmulouw
4223 # define XXH_vec_mule __builtin_altivec_vmuleuw
4224 # else
4225 /* gcc needs inline assembly */
4226 /* Adapted from https://github.com/google/highwayhash/blob/master/highwayhash/hh_vsx.h. */
XXH_vec_mulo(xxh_u32x4 a,xxh_u32x4 b)4227 XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mulo(xxh_u32x4 a, xxh_u32x4 b)
4228 {
4229 xxh_u64x2 result;
4230 __asm__("vmulouw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b));
4231 return result;
4232 }
XXH_vec_mule(xxh_u32x4 a,xxh_u32x4 b)4233 XXH_FORCE_INLINE xxh_u64x2 XXH_vec_mule(xxh_u32x4 a, xxh_u32x4 b)
4234 {
4235 xxh_u64x2 result;
4236 __asm__("vmuleuw %0, %1, %2" : "=v" (result) : "v" (a), "v" (b));
4237 return result;
4238 }
4239 # endif /* XXH_vec_mulo, XXH_vec_mule */
4240
4241 #if defined (__cplusplus)
4242 } /* extern "C" */
4243 #endif
4244
4245 #endif /* XXH_VECTOR == XXH_VSX */
4246
4247 #if XXH_VECTOR == XXH_SVE
4248 #define ACCRND(acc, offset) \
4249 do { \
4250 svuint64_t input_vec = svld1_u64(mask, xinput + offset); \
4251 svuint64_t secret_vec = svld1_u64(mask, xsecret + offset); \
4252 svuint64_t mixed = sveor_u64_x(mask, secret_vec, input_vec); \
4253 svuint64_t swapped = svtbl_u64(input_vec, kSwap); \
4254 svuint64_t mixed_lo = svextw_u64_x(mask, mixed); \
4255 svuint64_t mixed_hi = svlsr_n_u64_x(mask, mixed, 32); \
4256 svuint64_t mul = svmad_u64_x(mask, mixed_lo, mixed_hi, swapped); \
4257 acc = svadd_u64_x(mask, acc, mul); \
4258 } while (0)
4259 #endif /* XXH_VECTOR == XXH_SVE */
4260
4261 /* prefetch
4262 * can be disabled, by declaring XXH_NO_PREFETCH build macro */
4263 #if defined(XXH_NO_PREFETCH)
4264 # define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */
4265 #else
4266 # if XXH_SIZE_OPT >= 1
4267 # define XXH_PREFETCH(ptr) (void)(ptr)
4268 # elif defined(_MSC_VER) && (defined(_M_X64) || defined(_M_IX86)) /* _mm_prefetch() not defined outside of x86/x64 */
4269 # include <mmintrin.h> /* https://msdn.microsoft.com/fr-fr/library/84szxsww(v=vs.90).aspx */
4270 # define XXH_PREFETCH(ptr) _mm_prefetch((const char*)(ptr), _MM_HINT_T0)
4271 # elif defined(__GNUC__) && ( (__GNUC__ >= 4) || ( (__GNUC__ == 3) && (__GNUC_MINOR__ >= 1) ) )
4272 # define XXH_PREFETCH(ptr) __builtin_prefetch((ptr), 0 /* rw==read */, 3 /* locality */)
4273 # else
4274 # define XXH_PREFETCH(ptr) (void)(ptr) /* disabled */
4275 # endif
4276 #endif /* XXH_NO_PREFETCH */
4277
4278 #if defined (__cplusplus)
4279 extern "C" {
4280 #endif
4281 /* ==========================================
4282 * XXH3 default settings
4283 * ========================================== */
4284
4285 #define XXH_SECRET_DEFAULT_SIZE 192 /* minimum XXH3_SECRET_SIZE_MIN */
4286
4287 #if (XXH_SECRET_DEFAULT_SIZE < XXH3_SECRET_SIZE_MIN)
4288 # error "default keyset is not large enough"
4289 #endif
4290
4291 /*! Pseudorandom secret taken directly from FARSH. */
4292 XXH_ALIGN(64) static const xxh_u8 XXH3_kSecret[XXH_SECRET_DEFAULT_SIZE] = {
4293 0xb8, 0xfe, 0x6c, 0x39, 0x23, 0xa4, 0x4b, 0xbe, 0x7c, 0x01, 0x81, 0x2c, 0xf7, 0x21, 0xad, 0x1c,
4294 0xde, 0xd4, 0x6d, 0xe9, 0x83, 0x90, 0x97, 0xdb, 0x72, 0x40, 0xa4, 0xa4, 0xb7, 0xb3, 0x67, 0x1f,
4295 0xcb, 0x79, 0xe6, 0x4e, 0xcc, 0xc0, 0xe5, 0x78, 0x82, 0x5a, 0xd0, 0x7d, 0xcc, 0xff, 0x72, 0x21,
4296 0xb8, 0x08, 0x46, 0x74, 0xf7, 0x43, 0x24, 0x8e, 0xe0, 0x35, 0x90, 0xe6, 0x81, 0x3a, 0x26, 0x4c,
4297 0x3c, 0x28, 0x52, 0xbb, 0x91, 0xc3, 0x00, 0xcb, 0x88, 0xd0, 0x65, 0x8b, 0x1b, 0x53, 0x2e, 0xa3,
4298 0x71, 0x64, 0x48, 0x97, 0xa2, 0x0d, 0xf9, 0x4e, 0x38, 0x19, 0xef, 0x46, 0xa9, 0xde, 0xac, 0xd8,
4299 0xa8, 0xfa, 0x76, 0x3f, 0xe3, 0x9c, 0x34, 0x3f, 0xf9, 0xdc, 0xbb, 0xc7, 0xc7, 0x0b, 0x4f, 0x1d,
4300 0x8a, 0x51, 0xe0, 0x4b, 0xcd, 0xb4, 0x59, 0x31, 0xc8, 0x9f, 0x7e, 0xc9, 0xd9, 0x78, 0x73, 0x64,
4301 0xea, 0xc5, 0xac, 0x83, 0x34, 0xd3, 0xeb, 0xc3, 0xc5, 0x81, 0xa0, 0xff, 0xfa, 0x13, 0x63, 0xeb,
4302 0x17, 0x0d, 0xdd, 0x51, 0xb7, 0xf0, 0xda, 0x49, 0xd3, 0x16, 0x55, 0x26, 0x29, 0xd4, 0x68, 0x9e,
4303 0x2b, 0x16, 0xbe, 0x58, 0x7d, 0x47, 0xa1, 0xfc, 0x8f, 0xf8, 0xb8, 0xd1, 0x7a, 0xd0, 0x31, 0xce,
4304 0x45, 0xcb, 0x3a, 0x8f, 0x95, 0x16, 0x04, 0x28, 0xaf, 0xd7, 0xfb, 0xca, 0xbb, 0x4b, 0x40, 0x7e,
4305 };
4306
4307 static const xxh_u64 PRIME_MX1 = 0x165667919E3779F9ULL; /*!< 0b0001011001010110011001111001000110011110001101110111100111111001 */
4308 static const xxh_u64 PRIME_MX2 = 0x9FB21C651E98DF25ULL; /*!< 0b1001111110110010000111000110010100011110100110001101111100100101 */
4309
4310 #ifdef XXH_OLD_NAMES
4311 # define kSecret XXH3_kSecret
4312 #endif
4313
4314 #ifdef XXH_DOXYGEN
4315 /*!
4316 * @brief Calculates a 32-bit to 64-bit long multiply.
4317 *
4318 * Implemented as a macro.
4319 *
4320 * Wraps `__emulu` on MSVC x86 because it tends to call `__allmul` when it doesn't
4321 * need to (but it shouldn't need to anyways, it is about 7 instructions to do
4322 * a 64x64 multiply...). Since we know that this will _always_ emit `MULL`, we
4323 * use that instead of the normal method.
4324 *
4325 * If you are compiling for platforms like Thumb-1 and don't have a better option,
4326 * you may also want to write your own long multiply routine here.
4327 *
4328 * @param x, y Numbers to be multiplied
4329 * @return 64-bit product of the low 32 bits of @p x and @p y.
4330 */
4331 XXH_FORCE_INLINE xxh_u64
XXH_mult32to64(xxh_u64 x,xxh_u64 y)4332 XXH_mult32to64(xxh_u64 x, xxh_u64 y)
4333 {
4334 return (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF);
4335 }
4336 #elif defined(_MSC_VER) && defined(_M_IX86)
4337 # define XXH_mult32to64(x, y) __emulu((unsigned)(x), (unsigned)(y))
4338 #else
4339 /*
4340 * Downcast + upcast is usually better than masking on older compilers like
4341 * GCC 4.2 (especially 32-bit ones), all without affecting newer compilers.
4342 *
4343 * The other method, (x & 0xFFFFFFFF) * (y & 0xFFFFFFFF), will AND both operands
4344 * and perform a full 64x64 multiply -- entirely redundant on 32-bit.
4345 */
4346 # define XXH_mult32to64(x, y) ((xxh_u64)(xxh_u32)(x) * (xxh_u64)(xxh_u32)(y))
4347 #endif
4348
4349 /*!
4350 * @brief Calculates a 64->128-bit long multiply.
4351 *
4352 * Uses `__uint128_t` and `_umul128` if available, otherwise uses a scalar
4353 * version.
4354 *
4355 * @param lhs , rhs The 64-bit integers to be multiplied
4356 * @return The 128-bit result represented in an @ref XXH128_hash_t.
4357 */
4358 static XXH128_hash_t
XXH_mult64to128(xxh_u64 lhs,xxh_u64 rhs)4359 XXH_mult64to128(xxh_u64 lhs, xxh_u64 rhs)
4360 {
4361 /*
4362 * GCC/Clang __uint128_t method.
4363 *
4364 * On most 64-bit targets, GCC and Clang define a __uint128_t type.
4365 * This is usually the best way as it usually uses a native long 64-bit
4366 * multiply, such as MULQ on x86_64 or MUL + UMULH on aarch64.
4367 *
4368 * Usually.
4369 *
4370 * Despite being a 32-bit platform, Clang (and emscripten) define this type
4371 * despite not having the arithmetic for it. This results in a laggy
4372 * compiler builtin call which calculates a full 128-bit multiply.
4373 * In that case it is best to use the portable one.
4374 * https://github.com/Cyan4973/xxHash/issues/211#issuecomment-515575677
4375 */
4376 #if (defined(__GNUC__) || defined(__clang__)) && !defined(__wasm__) \
4377 && defined(__SIZEOF_INT128__) \
4378 || (defined(_INTEGRAL_MAX_BITS) && _INTEGRAL_MAX_BITS >= 128)
4379
4380 __uint128_t const product = (__uint128_t)lhs * (__uint128_t)rhs;
4381 XXH128_hash_t r128;
4382 r128.low64 = (xxh_u64)(product);
4383 r128.high64 = (xxh_u64)(product >> 64);
4384 return r128;
4385
4386 /*
4387 * MSVC for x64's _umul128 method.
4388 *
4389 * xxh_u64 _umul128(xxh_u64 Multiplier, xxh_u64 Multiplicand, xxh_u64 *HighProduct);
4390 *
4391 * This compiles to single operand MUL on x64.
4392 */
4393 #elif (defined(_M_X64) || defined(_M_IA64)) && !defined(_M_ARM64EC)
4394
4395 #ifndef _MSC_VER
4396 # pragma intrinsic(_umul128)
4397 #endif
4398 xxh_u64 product_high;
4399 xxh_u64 const product_low = _umul128(lhs, rhs, &product_high);
4400 XXH128_hash_t r128;
4401 r128.low64 = product_low;
4402 r128.high64 = product_high;
4403 return r128;
4404
4405 /*
4406 * MSVC for ARM64's __umulh method.
4407 *
4408 * This compiles to the same MUL + UMULH as GCC/Clang's __uint128_t method.
4409 */
4410 #elif defined(_M_ARM64) || defined(_M_ARM64EC)
4411
4412 #ifndef _MSC_VER
4413 # pragma intrinsic(__umulh)
4414 #endif
4415 XXH128_hash_t r128;
4416 r128.low64 = lhs * rhs;
4417 r128.high64 = __umulh(lhs, rhs);
4418 return r128;
4419
4420 #else
4421 /*
4422 * Portable scalar method. Optimized for 32-bit and 64-bit ALUs.
4423 *
4424 * This is a fast and simple grade school multiply, which is shown below
4425 * with base 10 arithmetic instead of base 0x100000000.
4426 *
4427 * 9 3 // D2 lhs = 93
4428 * x 7 5 // D2 rhs = 75
4429 * ----------
4430 * 1 5 // D2 lo_lo = (93 % 10) * (75 % 10) = 15
4431 * 4 5 | // D2 hi_lo = (93 / 10) * (75 % 10) = 45
4432 * 2 1 | // D2 lo_hi = (93 % 10) * (75 / 10) = 21
4433 * + 6 3 | | // D2 hi_hi = (93 / 10) * (75 / 10) = 63
4434 * ---------
4435 * 2 7 | // D2 cross = (15 / 10) + (45 % 10) + 21 = 27
4436 * + 6 7 | | // D2 upper = (27 / 10) + (45 / 10) + 63 = 67
4437 * ---------
4438 * 6 9 7 5 // D4 res = (27 * 10) + (15 % 10) + (67 * 100) = 6975
4439 *
4440 * The reasons for adding the products like this are:
4441 * 1. It avoids manual carry tracking. Just like how
4442 * (9 * 9) + 9 + 9 = 99, the same applies with this for UINT64_MAX.
4443 * This avoids a lot of complexity.
4444 *
4445 * 2. It hints for, and on Clang, compiles to, the powerful UMAAL
4446 * instruction available in ARM's Digital Signal Processing extension
4447 * in 32-bit ARMv6 and later, which is shown below:
4448 *
4449 * void UMAAL(xxh_u32 *RdLo, xxh_u32 *RdHi, xxh_u32 Rn, xxh_u32 Rm)
4450 * {
4451 * xxh_u64 product = (xxh_u64)*RdLo * (xxh_u64)*RdHi + Rn + Rm;
4452 * *RdLo = (xxh_u32)(product & 0xFFFFFFFF);
4453 * *RdHi = (xxh_u32)(product >> 32);
4454 * }
4455 *
4456 * This instruction was designed for efficient long multiplication, and
4457 * allows this to be calculated in only 4 instructions at speeds
4458 * comparable to some 64-bit ALUs.
4459 *
4460 * 3. It isn't terrible on other platforms. Usually this will be a couple
4461 * of 32-bit ADD/ADCs.
4462 */
4463
4464 /* First calculate all of the cross products. */
4465 xxh_u64 const lo_lo = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs & 0xFFFFFFFF);
4466 xxh_u64 const hi_lo = XXH_mult32to64(lhs >> 32, rhs & 0xFFFFFFFF);
4467 xxh_u64 const lo_hi = XXH_mult32to64(lhs & 0xFFFFFFFF, rhs >> 32);
4468 xxh_u64 const hi_hi = XXH_mult32to64(lhs >> 32, rhs >> 32);
4469
4470 /* Now add the products together. These will never overflow. */
4471 xxh_u64 const cross = (lo_lo >> 32) + (hi_lo & 0xFFFFFFFF) + lo_hi;
4472 xxh_u64 const upper = (hi_lo >> 32) + (cross >> 32) + hi_hi;
4473 xxh_u64 const lower = (cross << 32) | (lo_lo & 0xFFFFFFFF);
4474
4475 XXH128_hash_t r128;
4476 r128.low64 = lower;
4477 r128.high64 = upper;
4478 return r128;
4479 #endif
4480 }
4481
4482 /*!
4483 * @brief Calculates a 64-bit to 128-bit multiply, then XOR folds it.
4484 *
4485 * The reason for the separate function is to prevent passing too many structs
4486 * around by value. This will hopefully inline the multiply, but we don't force it.
4487 *
4488 * @param lhs , rhs The 64-bit integers to multiply
4489 * @return The low 64 bits of the product XOR'd by the high 64 bits.
4490 * @see XXH_mult64to128()
4491 */
4492 static xxh_u64
XXH3_mul128_fold64(xxh_u64 lhs,xxh_u64 rhs)4493 XXH3_mul128_fold64(xxh_u64 lhs, xxh_u64 rhs)
4494 {
4495 XXH128_hash_t product = XXH_mult64to128(lhs, rhs);
4496 return product.low64 ^ product.high64;
4497 }
4498
4499 /*! Seems to produce slightly better code on GCC for some reason. */
XXH_xorshift64(xxh_u64 v64,int shift)4500 XXH_FORCE_INLINE XXH_CONSTF xxh_u64 XXH_xorshift64(xxh_u64 v64, int shift)
4501 {
4502 XXH_ASSERT(0 <= shift && shift < 64);
4503 return v64 ^ (v64 >> shift);
4504 }
4505
4506 /*
4507 * This is a fast avalanche stage,
4508 * suitable when input bits are already partially mixed
4509 */
XXH3_avalanche(xxh_u64 h64)4510 static XXH64_hash_t XXH3_avalanche(xxh_u64 h64)
4511 {
4512 h64 = XXH_xorshift64(h64, 37);
4513 h64 *= PRIME_MX1;
4514 h64 = XXH_xorshift64(h64, 32);
4515 return h64;
4516 }
4517
4518 /*
4519 * This is a stronger avalanche,
4520 * inspired by Pelle Evensen's rrmxmx
4521 * preferable when input has not been previously mixed
4522 */
XXH3_rrmxmx(xxh_u64 h64,xxh_u64 len)4523 static XXH64_hash_t XXH3_rrmxmx(xxh_u64 h64, xxh_u64 len)
4524 {
4525 /* this mix is inspired by Pelle Evensen's rrmxmx */
4526 h64 ^= XXH_rotl64(h64, 49) ^ XXH_rotl64(h64, 24);
4527 h64 *= PRIME_MX2;
4528 h64 ^= (h64 >> 35) + len ;
4529 h64 *= PRIME_MX2;
4530 return XXH_xorshift64(h64, 28);
4531 }
4532
4533
4534 /* ==========================================
4535 * Short keys
4536 * ==========================================
4537 * One of the shortcomings of XXH32 and XXH64 was that their performance was
4538 * sub-optimal on short lengths. It used an iterative algorithm which strongly
4539 * favored lengths that were a multiple of 4 or 8.
4540 *
4541 * Instead of iterating over individual inputs, we use a set of single shot
4542 * functions which piece together a range of lengths and operate in constant time.
4543 *
4544 * Additionally, the number of multiplies has been significantly reduced. This
4545 * reduces latency, especially when emulating 64-bit multiplies on 32-bit.
4546 *
4547 * Depending on the platform, this may or may not be faster than XXH32, but it
4548 * is almost guaranteed to be faster than XXH64.
4549 */
4550
4551 /*
4552 * At very short lengths, there isn't enough input to fully hide secrets, or use
4553 * the entire secret.
4554 *
4555 * There is also only a limited amount of mixing we can do before significantly
4556 * impacting performance.
4557 *
4558 * Therefore, we use different sections of the secret and always mix two secret
4559 * samples with an XOR. This should have no effect on performance on the
4560 * seedless or withSeed variants because everything _should_ be constant folded
4561 * by modern compilers.
4562 *
4563 * The XOR mixing hides individual parts of the secret and increases entropy.
4564 *
4565 * This adds an extra layer of strength for custom secrets.
4566 */
4567 XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t
XXH3_len_1to3_64b(const xxh_u8 * input,size_t len,const xxh_u8 * secret,XXH64_hash_t seed)4568 XXH3_len_1to3_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
4569 {
4570 XXH_ASSERT(input != NULL);
4571 XXH_ASSERT(1 <= len && len <= 3);
4572 XXH_ASSERT(secret != NULL);
4573 /*
4574 * len = 1: combined = { input[0], 0x01, input[0], input[0] }
4575 * len = 2: combined = { input[1], 0x02, input[0], input[1] }
4576 * len = 3: combined = { input[2], 0x03, input[0], input[1] }
4577 */
4578 { xxh_u8 const c1 = input[0];
4579 xxh_u8 const c2 = input[len >> 1];
4580 xxh_u8 const c3 = input[len - 1];
4581 xxh_u32 const combined = ((xxh_u32)c1 << 16) | ((xxh_u32)c2 << 24)
4582 | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8);
4583 xxh_u64 const bitflip = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed;
4584 xxh_u64 const keyed = (xxh_u64)combined ^ bitflip;
4585 return XXH64_avalanche(keyed);
4586 }
4587 }
4588
4589 XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t
XXH3_len_4to8_64b(const xxh_u8 * input,size_t len,const xxh_u8 * secret,XXH64_hash_t seed)4590 XXH3_len_4to8_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
4591 {
4592 XXH_ASSERT(input != NULL);
4593 XXH_ASSERT(secret != NULL);
4594 XXH_ASSERT(4 <= len && len <= 8);
4595 seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32;
4596 { xxh_u32 const input1 = XXH_readLE32(input);
4597 xxh_u32 const input2 = XXH_readLE32(input + len - 4);
4598 xxh_u64 const bitflip = (XXH_readLE64(secret+8) ^ XXH_readLE64(secret+16)) - seed;
4599 xxh_u64 const input64 = input2 + (((xxh_u64)input1) << 32);
4600 xxh_u64 const keyed = input64 ^ bitflip;
4601 return XXH3_rrmxmx(keyed, len);
4602 }
4603 }
4604
4605 XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t
XXH3_len_9to16_64b(const xxh_u8 * input,size_t len,const xxh_u8 * secret,XXH64_hash_t seed)4606 XXH3_len_9to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
4607 {
4608 XXH_ASSERT(input != NULL);
4609 XXH_ASSERT(secret != NULL);
4610 XXH_ASSERT(9 <= len && len <= 16);
4611 { xxh_u64 const bitflip1 = (XXH_readLE64(secret+24) ^ XXH_readLE64(secret+32)) + seed;
4612 xxh_u64 const bitflip2 = (XXH_readLE64(secret+40) ^ XXH_readLE64(secret+48)) - seed;
4613 xxh_u64 const input_lo = XXH_readLE64(input) ^ bitflip1;
4614 xxh_u64 const input_hi = XXH_readLE64(input + len - 8) ^ bitflip2;
4615 xxh_u64 const acc = len
4616 + XXH_swap64(input_lo) + input_hi
4617 + XXH3_mul128_fold64(input_lo, input_hi);
4618 return XXH3_avalanche(acc);
4619 }
4620 }
4621
4622 XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t
XXH3_len_0to16_64b(const xxh_u8 * input,size_t len,const xxh_u8 * secret,XXH64_hash_t seed)4623 XXH3_len_0to16_64b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
4624 {
4625 XXH_ASSERT(len <= 16);
4626 { if (XXH_likely(len > 8)) return XXH3_len_9to16_64b(input, len, secret, seed);
4627 if (XXH_likely(len >= 4)) return XXH3_len_4to8_64b(input, len, secret, seed);
4628 if (len) return XXH3_len_1to3_64b(input, len, secret, seed);
4629 return XXH64_avalanche(seed ^ (XXH_readLE64(secret+56) ^ XXH_readLE64(secret+64)));
4630 }
4631 }
4632
4633 /*
4634 * DISCLAIMER: There are known *seed-dependent* multicollisions here due to
4635 * multiplication by zero, affecting hashes of lengths 17 to 240.
4636 *
4637 * However, they are very unlikely.
4638 *
4639 * Keep this in mind when using the unseeded XXH3_64bits() variant: As with all
4640 * unseeded non-cryptographic hashes, it does not attempt to defend itself
4641 * against specially crafted inputs, only random inputs.
4642 *
4643 * Compared to classic UMAC where a 1 in 2^31 chance of 4 consecutive bytes
4644 * cancelling out the secret is taken an arbitrary number of times (addressed
4645 * in XXH3_accumulate_512), this collision is very unlikely with random inputs
4646 * and/or proper seeding:
4647 *
4648 * This only has a 1 in 2^63 chance of 8 consecutive bytes cancelling out, in a
4649 * function that is only called up to 16 times per hash with up to 240 bytes of
4650 * input.
4651 *
4652 * This is not too bad for a non-cryptographic hash function, especially with
4653 * only 64 bit outputs.
4654 *
4655 * The 128-bit variant (which trades some speed for strength) is NOT affected
4656 * by this, although it is always a good idea to use a proper seed if you care
4657 * about strength.
4658 */
XXH3_mix16B(const xxh_u8 * XXH_RESTRICT input,const xxh_u8 * XXH_RESTRICT secret,xxh_u64 seed64)4659 XXH_FORCE_INLINE xxh_u64 XXH3_mix16B(const xxh_u8* XXH_RESTRICT input,
4660 const xxh_u8* XXH_RESTRICT secret, xxh_u64 seed64)
4661 {
4662 #if defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \
4663 && defined(__i386__) && defined(__SSE2__) /* x86 + SSE2 */ \
4664 && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable like XXH32 hack */
4665 /*
4666 * UGLY HACK:
4667 * GCC for x86 tends to autovectorize the 128-bit multiply, resulting in
4668 * slower code.
4669 *
4670 * By forcing seed64 into a register, we disrupt the cost model and
4671 * cause it to scalarize. See `XXH32_round()`
4672 *
4673 * FIXME: Clang's output is still _much_ faster -- On an AMD Ryzen 3600,
4674 * XXH3_64bits @ len=240 runs at 4.6 GB/s with Clang 9, but 3.3 GB/s on
4675 * GCC 9.2, despite both emitting scalar code.
4676 *
4677 * GCC generates much better scalar code than Clang for the rest of XXH3,
4678 * which is why finding a more optimal codepath is an interest.
4679 */
4680 XXH_COMPILER_GUARD(seed64);
4681 #endif
4682 { xxh_u64 const input_lo = XXH_readLE64(input);
4683 xxh_u64 const input_hi = XXH_readLE64(input+8);
4684 return XXH3_mul128_fold64(
4685 input_lo ^ (XXH_readLE64(secret) + seed64),
4686 input_hi ^ (XXH_readLE64(secret+8) - seed64)
4687 );
4688 }
4689 }
4690
4691 /* For mid range keys, XXH3 uses a Mum-hash variant. */
4692 XXH_FORCE_INLINE XXH_PUREF XXH64_hash_t
XXH3_len_17to128_64b(const xxh_u8 * XXH_RESTRICT input,size_t len,const xxh_u8 * XXH_RESTRICT secret,size_t secretSize,XXH64_hash_t seed)4693 XXH3_len_17to128_64b(const xxh_u8* XXH_RESTRICT input, size_t len,
4694 const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
4695 XXH64_hash_t seed)
4696 {
4697 XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
4698 XXH_ASSERT(16 < len && len <= 128);
4699
4700 { xxh_u64 acc = len * XXH_PRIME64_1;
4701 #if XXH_SIZE_OPT >= 1
4702 /* Smaller and cleaner, but slightly slower. */
4703 unsigned int i = (unsigned int)(len - 1) / 32;
4704 do {
4705 acc += XXH3_mix16B(input+16 * i, secret+32*i, seed);
4706 acc += XXH3_mix16B(input+len-16*(i+1), secret+32*i+16, seed);
4707 } while (i-- != 0);
4708 #else
4709 if (len > 32) {
4710 if (len > 64) {
4711 if (len > 96) {
4712 acc += XXH3_mix16B(input+48, secret+96, seed);
4713 acc += XXH3_mix16B(input+len-64, secret+112, seed);
4714 }
4715 acc += XXH3_mix16B(input+32, secret+64, seed);
4716 acc += XXH3_mix16B(input+len-48, secret+80, seed);
4717 }
4718 acc += XXH3_mix16B(input+16, secret+32, seed);
4719 acc += XXH3_mix16B(input+len-32, secret+48, seed);
4720 }
4721 acc += XXH3_mix16B(input+0, secret+0, seed);
4722 acc += XXH3_mix16B(input+len-16, secret+16, seed);
4723 #endif
4724 return XXH3_avalanche(acc);
4725 }
4726 }
4727
4728 /*!
4729 * @brief Maximum size of "short" key in bytes.
4730 */
4731 #define XXH3_MIDSIZE_MAX 240
4732
4733 XXH_NO_INLINE XXH_PUREF XXH64_hash_t
XXH3_len_129to240_64b(const xxh_u8 * XXH_RESTRICT input,size_t len,const xxh_u8 * XXH_RESTRICT secret,size_t secretSize,XXH64_hash_t seed)4734 XXH3_len_129to240_64b(const xxh_u8* XXH_RESTRICT input, size_t len,
4735 const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
4736 XXH64_hash_t seed)
4737 {
4738 XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
4739 XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX);
4740
4741 #define XXH3_MIDSIZE_STARTOFFSET 3
4742 #define XXH3_MIDSIZE_LASTOFFSET 17
4743
4744 { xxh_u64 acc = len * XXH_PRIME64_1;
4745 xxh_u64 acc_end;
4746 unsigned int const nbRounds = (unsigned int)len / 16;
4747 unsigned int i;
4748 XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX);
4749 for (i=0; i<8; i++) {
4750 acc += XXH3_mix16B(input+(16*i), secret+(16*i), seed);
4751 }
4752 /* last bytes */
4753 acc_end = XXH3_mix16B(input + len - 16, secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET, seed);
4754 XXH_ASSERT(nbRounds >= 8);
4755 acc = XXH3_avalanche(acc);
4756 #if defined(__clang__) /* Clang */ \
4757 && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \
4758 && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */
4759 /*
4760 * UGLY HACK:
4761 * Clang for ARMv7-A tries to vectorize this loop, similar to GCC x86.
4762 * In everywhere else, it uses scalar code.
4763 *
4764 * For 64->128-bit multiplies, even if the NEON was 100% optimal, it
4765 * would still be slower than UMAAL (see XXH_mult64to128).
4766 *
4767 * Unfortunately, Clang doesn't handle the long multiplies properly and
4768 * converts them to the nonexistent "vmulq_u64" intrinsic, which is then
4769 * scalarized into an ugly mess of VMOV.32 instructions.
4770 *
4771 * This mess is difficult to avoid without turning autovectorization
4772 * off completely, but they are usually relatively minor and/or not
4773 * worth it to fix.
4774 *
4775 * This loop is the easiest to fix, as unlike XXH32, this pragma
4776 * _actually works_ because it is a loop vectorization instead of an
4777 * SLP vectorization.
4778 */
4779 #pragma clang loop vectorize(disable)
4780 #endif
4781 for (i=8 ; i < nbRounds; i++) {
4782 /*
4783 * Prevents clang for unrolling the acc loop and interleaving with this one.
4784 */
4785 XXH_COMPILER_GUARD(acc);
4786 acc_end += XXH3_mix16B(input+(16*i), secret+(16*(i-8)) + XXH3_MIDSIZE_STARTOFFSET, seed);
4787 }
4788 return XXH3_avalanche(acc + acc_end);
4789 }
4790 }
4791
4792
4793 /* ======= Long Keys ======= */
4794
4795 #define XXH_STRIPE_LEN 64
4796 #define XXH_SECRET_CONSUME_RATE 8 /* nb of secret bytes consumed at each accumulation */
4797 #define XXH_ACC_NB (XXH_STRIPE_LEN / sizeof(xxh_u64))
4798
4799 #ifdef XXH_OLD_NAMES
4800 # define STRIPE_LEN XXH_STRIPE_LEN
4801 # define ACC_NB XXH_ACC_NB
4802 #endif
4803
4804 #ifndef XXH_PREFETCH_DIST
4805 # ifdef __clang__
4806 # define XXH_PREFETCH_DIST 320
4807 # else
4808 # if (XXH_VECTOR == XXH_AVX512)
4809 # define XXH_PREFETCH_DIST 512
4810 # else
4811 # define XXH_PREFETCH_DIST 384
4812 # endif
4813 # endif /* __clang__ */
4814 #endif /* XXH_PREFETCH_DIST */
4815
4816 /*
4817 * These macros are to generate an XXH3_accumulate() function.
4818 * The two arguments select the name suffix and target attribute.
4819 *
4820 * The name of this symbol is XXH3_accumulate_<name>() and it calls
4821 * XXH3_accumulate_512_<name>().
4822 *
4823 * It may be useful to hand implement this function if the compiler fails to
4824 * optimize the inline function.
4825 */
4826 #define XXH3_ACCUMULATE_TEMPLATE(name) \
4827 void \
4828 XXH3_accumulate_##name(xxh_u64* XXH_RESTRICT acc, \
4829 const xxh_u8* XXH_RESTRICT input, \
4830 const xxh_u8* XXH_RESTRICT secret, \
4831 size_t nbStripes) \
4832 { \
4833 size_t n; \
4834 for (n = 0; n < nbStripes; n++ ) { \
4835 const xxh_u8* const in = input + n*XXH_STRIPE_LEN; \
4836 XXH_PREFETCH(in + XXH_PREFETCH_DIST); \
4837 XXH3_accumulate_512_##name( \
4838 acc, \
4839 in, \
4840 secret + n*XXH_SECRET_CONSUME_RATE); \
4841 } \
4842 }
4843
4844
XXH_writeLE64(void * dst,xxh_u64 v64)4845 XXH_FORCE_INLINE void XXH_writeLE64(void* dst, xxh_u64 v64)
4846 {
4847 if (!XXH_CPU_LITTLE_ENDIAN) v64 = XXH_swap64(v64);
4848 XXH_memcpy(dst, &v64, sizeof(v64));
4849 }
4850
4851 /* Several intrinsic functions below are supposed to accept __int64 as argument,
4852 * as documented in https://software.intel.com/sites/landingpage/IntrinsicsGuide/ .
4853 * However, several environments do not define __int64 type,
4854 * requiring a workaround.
4855 */
4856 #if !defined (__VMS) \
4857 && (defined (__cplusplus) \
4858 || (defined (__STDC_VERSION__) && (__STDC_VERSION__ >= 199901L) /* C99 */) )
4859 typedef int64_t xxh_i64;
4860 #else
4861 /* the following type must have a width of 64-bit */
4862 typedef long long xxh_i64;
4863 #endif
4864
4865
4866 /*
4867 * XXH3_accumulate_512 is the tightest loop for long inputs, and it is the most optimized.
4868 *
4869 * It is a hardened version of UMAC, based off of FARSH's implementation.
4870 *
4871 * This was chosen because it adapts quite well to 32-bit, 64-bit, and SIMD
4872 * implementations, and it is ridiculously fast.
4873 *
4874 * We harden it by mixing the original input to the accumulators as well as the product.
4875 *
4876 * This means that in the (relatively likely) case of a multiply by zero, the
4877 * original input is preserved.
4878 *
4879 * On 128-bit inputs, we swap 64-bit pairs when we add the input to improve
4880 * cross-pollination, as otherwise the upper and lower halves would be
4881 * essentially independent.
4882 *
4883 * This doesn't matter on 64-bit hashes since they all get merged together in
4884 * the end, so we skip the extra step.
4885 *
4886 * Both XXH3_64bits and XXH3_128bits use this subroutine.
4887 */
4888
4889 #if (XXH_VECTOR == XXH_AVX512) \
4890 || (defined(XXH_DISPATCH_AVX512) && XXH_DISPATCH_AVX512 != 0)
4891
4892 #ifndef XXH_TARGET_AVX512
4893 # define XXH_TARGET_AVX512 /* disable attribute target */
4894 #endif
4895
4896 XXH_FORCE_INLINE XXH_TARGET_AVX512 void
XXH3_accumulate_512_avx512(void * XXH_RESTRICT acc,const void * XXH_RESTRICT input,const void * XXH_RESTRICT secret)4897 XXH3_accumulate_512_avx512(void* XXH_RESTRICT acc,
4898 const void* XXH_RESTRICT input,
4899 const void* XXH_RESTRICT secret)
4900 {
4901 __m512i* const xacc = (__m512i *) acc;
4902 XXH_ASSERT((((size_t)acc) & 63) == 0);
4903 XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i));
4904
4905 {
4906 /* data_vec = input[0]; */
4907 __m512i const data_vec = _mm512_loadu_si512 (input);
4908 /* key_vec = secret[0]; */
4909 __m512i const key_vec = _mm512_loadu_si512 (secret);
4910 /* data_key = data_vec ^ key_vec; */
4911 __m512i const data_key = _mm512_xor_si512 (data_vec, key_vec);
4912 /* data_key_lo = data_key >> 32; */
4913 __m512i const data_key_lo = _mm512_srli_epi64 (data_key, 32);
4914 /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */
4915 __m512i const product = _mm512_mul_epu32 (data_key, data_key_lo);
4916 /* xacc[0] += swap(data_vec); */
4917 __m512i const data_swap = _mm512_shuffle_epi32(data_vec, (_MM_PERM_ENUM)_MM_SHUFFLE(1, 0, 3, 2));
4918 __m512i const sum = _mm512_add_epi64(*xacc, data_swap);
4919 /* xacc[0] += product; */
4920 *xacc = _mm512_add_epi64(product, sum);
4921 }
4922 }
XXH3_ACCUMULATE_TEMPLATE(avx512)4923 XXH_FORCE_INLINE XXH_TARGET_AVX512 XXH3_ACCUMULATE_TEMPLATE(avx512)
4924
4925 /*
4926 * XXH3_scrambleAcc: Scrambles the accumulators to improve mixing.
4927 *
4928 * Multiplication isn't perfect, as explained by Google in HighwayHash:
4929 *
4930 * // Multiplication mixes/scrambles bytes 0-7 of the 64-bit result to
4931 * // varying degrees. In descending order of goodness, bytes
4932 * // 3 4 2 5 1 6 0 7 have quality 228 224 164 160 100 96 36 32.
4933 * // As expected, the upper and lower bytes are much worse.
4934 *
4935 * Source: https://github.com/google/highwayhash/blob/0aaf66b/highwayhash/hh_avx2.h#L291
4936 *
4937 * Since our algorithm uses a pseudorandom secret to add some variance into the
4938 * mix, we don't need to (or want to) mix as often or as much as HighwayHash does.
4939 *
4940 * This isn't as tight as XXH3_accumulate, but still written in SIMD to avoid
4941 * extraction.
4942 *
4943 * Both XXH3_64bits and XXH3_128bits use this subroutine.
4944 */
4945
4946 XXH_FORCE_INLINE XXH_TARGET_AVX512 void
4947 XXH3_scrambleAcc_avx512(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
4948 {
4949 XXH_ASSERT((((size_t)acc) & 63) == 0);
4950 XXH_STATIC_ASSERT(XXH_STRIPE_LEN == sizeof(__m512i));
4951 { __m512i* const xacc = (__m512i*) acc;
4952 const __m512i prime32 = _mm512_set1_epi32((int)XXH_PRIME32_1);
4953
4954 /* xacc[0] ^= (xacc[0] >> 47) */
4955 __m512i const acc_vec = *xacc;
4956 __m512i const shifted = _mm512_srli_epi64 (acc_vec, 47);
4957 /* xacc[0] ^= secret; */
4958 __m512i const key_vec = _mm512_loadu_si512 (secret);
4959 __m512i const data_key = _mm512_ternarylogic_epi32(key_vec, acc_vec, shifted, 0x96 /* key_vec ^ acc_vec ^ shifted */);
4960
4961 /* xacc[0] *= XXH_PRIME32_1; */
4962 __m512i const data_key_hi = _mm512_srli_epi64 (data_key, 32);
4963 __m512i const prod_lo = _mm512_mul_epu32 (data_key, prime32);
4964 __m512i const prod_hi = _mm512_mul_epu32 (data_key_hi, prime32);
4965 *xacc = _mm512_add_epi64(prod_lo, _mm512_slli_epi64(prod_hi, 32));
4966 }
4967 }
4968
4969 XXH_FORCE_INLINE XXH_TARGET_AVX512 void
XXH3_initCustomSecret_avx512(void * XXH_RESTRICT customSecret,xxh_u64 seed64)4970 XXH3_initCustomSecret_avx512(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
4971 {
4972 XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 63) == 0);
4973 XXH_STATIC_ASSERT(XXH_SEC_ALIGN == 64);
4974 XXH_ASSERT(((size_t)customSecret & 63) == 0);
4975 (void)(&XXH_writeLE64);
4976 { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m512i);
4977 __m512i const seed_pos = _mm512_set1_epi64((xxh_i64)seed64);
4978 __m512i const seed = _mm512_mask_sub_epi64(seed_pos, 0xAA, _mm512_set1_epi8(0), seed_pos);
4979
4980 const __m512i* const src = (const __m512i*) ((const void*) XXH3_kSecret);
4981 __m512i* const dest = ( __m512i*) customSecret;
4982 int i;
4983 XXH_ASSERT(((size_t)src & 63) == 0); /* control alignment */
4984 XXH_ASSERT(((size_t)dest & 63) == 0);
4985 for (i=0; i < nbRounds; ++i) {
4986 dest[i] = _mm512_add_epi64(_mm512_load_si512(src + i), seed);
4987 } }
4988 }
4989
4990 #endif
4991
4992 #if (XXH_VECTOR == XXH_AVX2) \
4993 || (defined(XXH_DISPATCH_AVX2) && XXH_DISPATCH_AVX2 != 0)
4994
4995 #ifndef XXH_TARGET_AVX2
4996 # define XXH_TARGET_AVX2 /* disable attribute target */
4997 #endif
4998
4999 XXH_FORCE_INLINE XXH_TARGET_AVX2 void
XXH3_accumulate_512_avx2(void * XXH_RESTRICT acc,const void * XXH_RESTRICT input,const void * XXH_RESTRICT secret)5000 XXH3_accumulate_512_avx2( void* XXH_RESTRICT acc,
5001 const void* XXH_RESTRICT input,
5002 const void* XXH_RESTRICT secret)
5003 {
5004 XXH_ASSERT((((size_t)acc) & 31) == 0);
5005 { __m256i* const xacc = (__m256i *) acc;
5006 /* Unaligned. This is mainly for pointer arithmetic, and because
5007 * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */
5008 const __m256i* const xinput = (const __m256i *) input;
5009 /* Unaligned. This is mainly for pointer arithmetic, and because
5010 * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */
5011 const __m256i* const xsecret = (const __m256i *) secret;
5012
5013 size_t i;
5014 for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) {
5015 /* data_vec = xinput[i]; */
5016 __m256i const data_vec = _mm256_loadu_si256 (xinput+i);
5017 /* key_vec = xsecret[i]; */
5018 __m256i const key_vec = _mm256_loadu_si256 (xsecret+i);
5019 /* data_key = data_vec ^ key_vec; */
5020 __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec);
5021 /* data_key_lo = data_key >> 32; */
5022 __m256i const data_key_lo = _mm256_srli_epi64 (data_key, 32);
5023 /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */
5024 __m256i const product = _mm256_mul_epu32 (data_key, data_key_lo);
5025 /* xacc[i] += swap(data_vec); */
5026 __m256i const data_swap = _mm256_shuffle_epi32(data_vec, _MM_SHUFFLE(1, 0, 3, 2));
5027 __m256i const sum = _mm256_add_epi64(xacc[i], data_swap);
5028 /* xacc[i] += product; */
5029 xacc[i] = _mm256_add_epi64(product, sum);
5030 } }
5031 }
XXH3_ACCUMULATE_TEMPLATE(avx2)5032 XXH_FORCE_INLINE XXH_TARGET_AVX2 XXH3_ACCUMULATE_TEMPLATE(avx2)
5033
5034 XXH_FORCE_INLINE XXH_TARGET_AVX2 void
5035 XXH3_scrambleAcc_avx2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
5036 {
5037 XXH_ASSERT((((size_t)acc) & 31) == 0);
5038 { __m256i* const xacc = (__m256i*) acc;
5039 /* Unaligned. This is mainly for pointer arithmetic, and because
5040 * _mm256_loadu_si256 requires a const __m256i * pointer for some reason. */
5041 const __m256i* const xsecret = (const __m256i *) secret;
5042 const __m256i prime32 = _mm256_set1_epi32((int)XXH_PRIME32_1);
5043
5044 size_t i;
5045 for (i=0; i < XXH_STRIPE_LEN/sizeof(__m256i); i++) {
5046 /* xacc[i] ^= (xacc[i] >> 47) */
5047 __m256i const acc_vec = xacc[i];
5048 __m256i const shifted = _mm256_srli_epi64 (acc_vec, 47);
5049 __m256i const data_vec = _mm256_xor_si256 (acc_vec, shifted);
5050 /* xacc[i] ^= xsecret; */
5051 __m256i const key_vec = _mm256_loadu_si256 (xsecret+i);
5052 __m256i const data_key = _mm256_xor_si256 (data_vec, key_vec);
5053
5054 /* xacc[i] *= XXH_PRIME32_1; */
5055 __m256i const data_key_hi = _mm256_srli_epi64 (data_key, 32);
5056 __m256i const prod_lo = _mm256_mul_epu32 (data_key, prime32);
5057 __m256i const prod_hi = _mm256_mul_epu32 (data_key_hi, prime32);
5058 xacc[i] = _mm256_add_epi64(prod_lo, _mm256_slli_epi64(prod_hi, 32));
5059 }
5060 }
5061 }
5062
XXH3_initCustomSecret_avx2(void * XXH_RESTRICT customSecret,xxh_u64 seed64)5063 XXH_FORCE_INLINE XXH_TARGET_AVX2 void XXH3_initCustomSecret_avx2(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
5064 {
5065 XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 31) == 0);
5066 XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE / sizeof(__m256i)) == 6);
5067 XXH_STATIC_ASSERT(XXH_SEC_ALIGN <= 64);
5068 (void)(&XXH_writeLE64);
5069 XXH_PREFETCH(customSecret);
5070 { __m256i const seed = _mm256_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64, (xxh_i64)(0U - seed64), (xxh_i64)seed64);
5071
5072 const __m256i* const src = (const __m256i*) ((const void*) XXH3_kSecret);
5073 __m256i* dest = ( __m256i*) customSecret;
5074
5075 # if defined(__GNUC__) || defined(__clang__)
5076 /*
5077 * On GCC & Clang, marking 'dest' as modified will cause the compiler:
5078 * - do not extract the secret from sse registers in the internal loop
5079 * - use less common registers, and avoid pushing these reg into stack
5080 */
5081 XXH_COMPILER_GUARD(dest);
5082 # endif
5083 XXH_ASSERT(((size_t)src & 31) == 0); /* control alignment */
5084 XXH_ASSERT(((size_t)dest & 31) == 0);
5085
5086 /* GCC -O2 need unroll loop manually */
5087 dest[0] = _mm256_add_epi64(_mm256_load_si256(src+0), seed);
5088 dest[1] = _mm256_add_epi64(_mm256_load_si256(src+1), seed);
5089 dest[2] = _mm256_add_epi64(_mm256_load_si256(src+2), seed);
5090 dest[3] = _mm256_add_epi64(_mm256_load_si256(src+3), seed);
5091 dest[4] = _mm256_add_epi64(_mm256_load_si256(src+4), seed);
5092 dest[5] = _mm256_add_epi64(_mm256_load_si256(src+5), seed);
5093 }
5094 }
5095
5096 #endif
5097
5098 /* x86dispatch always generates SSE2 */
5099 #if (XXH_VECTOR == XXH_SSE2) || defined(XXH_X86DISPATCH)
5100
5101 #ifndef XXH_TARGET_SSE2
5102 # define XXH_TARGET_SSE2 /* disable attribute target */
5103 #endif
5104
5105 XXH_FORCE_INLINE XXH_TARGET_SSE2 void
XXH3_accumulate_512_sse2(void * XXH_RESTRICT acc,const void * XXH_RESTRICT input,const void * XXH_RESTRICT secret)5106 XXH3_accumulate_512_sse2( void* XXH_RESTRICT acc,
5107 const void* XXH_RESTRICT input,
5108 const void* XXH_RESTRICT secret)
5109 {
5110 /* SSE2 is just a half-scale version of the AVX2 version. */
5111 XXH_ASSERT((((size_t)acc) & 15) == 0);
5112 { __m128i* const xacc = (__m128i *) acc;
5113 /* Unaligned. This is mainly for pointer arithmetic, and because
5114 * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */
5115 const __m128i* const xinput = (const __m128i *) input;
5116 /* Unaligned. This is mainly for pointer arithmetic, and because
5117 * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */
5118 const __m128i* const xsecret = (const __m128i *) secret;
5119
5120 size_t i;
5121 for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) {
5122 /* data_vec = xinput[i]; */
5123 __m128i const data_vec = _mm_loadu_si128 (xinput+i);
5124 /* key_vec = xsecret[i]; */
5125 __m128i const key_vec = _mm_loadu_si128 (xsecret+i);
5126 /* data_key = data_vec ^ key_vec; */
5127 __m128i const data_key = _mm_xor_si128 (data_vec, key_vec);
5128 /* data_key_lo = data_key >> 32; */
5129 __m128i const data_key_lo = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1));
5130 /* product = (data_key & 0xffffffff) * (data_key_lo & 0xffffffff); */
5131 __m128i const product = _mm_mul_epu32 (data_key, data_key_lo);
5132 /* xacc[i] += swap(data_vec); */
5133 __m128i const data_swap = _mm_shuffle_epi32(data_vec, _MM_SHUFFLE(1,0,3,2));
5134 __m128i const sum = _mm_add_epi64(xacc[i], data_swap);
5135 /* xacc[i] += product; */
5136 xacc[i] = _mm_add_epi64(product, sum);
5137 } }
5138 }
XXH3_ACCUMULATE_TEMPLATE(sse2)5139 XXH_FORCE_INLINE XXH_TARGET_SSE2 XXH3_ACCUMULATE_TEMPLATE(sse2)
5140
5141 XXH_FORCE_INLINE XXH_TARGET_SSE2 void
5142 XXH3_scrambleAcc_sse2(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
5143 {
5144 XXH_ASSERT((((size_t)acc) & 15) == 0);
5145 { __m128i* const xacc = (__m128i*) acc;
5146 /* Unaligned. This is mainly for pointer arithmetic, and because
5147 * _mm_loadu_si128 requires a const __m128i * pointer for some reason. */
5148 const __m128i* const xsecret = (const __m128i *) secret;
5149 const __m128i prime32 = _mm_set1_epi32((int)XXH_PRIME32_1);
5150
5151 size_t i;
5152 for (i=0; i < XXH_STRIPE_LEN/sizeof(__m128i); i++) {
5153 /* xacc[i] ^= (xacc[i] >> 47) */
5154 __m128i const acc_vec = xacc[i];
5155 __m128i const shifted = _mm_srli_epi64 (acc_vec, 47);
5156 __m128i const data_vec = _mm_xor_si128 (acc_vec, shifted);
5157 /* xacc[i] ^= xsecret[i]; */
5158 __m128i const key_vec = _mm_loadu_si128 (xsecret+i);
5159 __m128i const data_key = _mm_xor_si128 (data_vec, key_vec);
5160
5161 /* xacc[i] *= XXH_PRIME32_1; */
5162 __m128i const data_key_hi = _mm_shuffle_epi32 (data_key, _MM_SHUFFLE(0, 3, 0, 1));
5163 __m128i const prod_lo = _mm_mul_epu32 (data_key, prime32);
5164 __m128i const prod_hi = _mm_mul_epu32 (data_key_hi, prime32);
5165 xacc[i] = _mm_add_epi64(prod_lo, _mm_slli_epi64(prod_hi, 32));
5166 }
5167 }
5168 }
5169
XXH3_initCustomSecret_sse2(void * XXH_RESTRICT customSecret,xxh_u64 seed64)5170 XXH_FORCE_INLINE XXH_TARGET_SSE2 void XXH3_initCustomSecret_sse2(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
5171 {
5172 XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0);
5173 (void)(&XXH_writeLE64);
5174 { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / sizeof(__m128i);
5175
5176 # if defined(_MSC_VER) && defined(_M_IX86) && _MSC_VER < 1900
5177 /* MSVC 32bit mode does not support _mm_set_epi64x before 2015 */
5178 XXH_ALIGN(16) const xxh_i64 seed64x2[2] = { (xxh_i64)seed64, (xxh_i64)(0U - seed64) };
5179 __m128i const seed = _mm_load_si128((__m128i const*)seed64x2);
5180 # else
5181 __m128i const seed = _mm_set_epi64x((xxh_i64)(0U - seed64), (xxh_i64)seed64);
5182 # endif
5183 int i;
5184
5185 const void* const src16 = XXH3_kSecret;
5186 __m128i* dst16 = (__m128i*) customSecret;
5187 # if defined(__GNUC__) || defined(__clang__)
5188 /*
5189 * On GCC & Clang, marking 'dest' as modified will cause the compiler:
5190 * - do not extract the secret from sse registers in the internal loop
5191 * - use less common registers, and avoid pushing these reg into stack
5192 */
5193 XXH_COMPILER_GUARD(dst16);
5194 # endif
5195 XXH_ASSERT(((size_t)src16 & 15) == 0); /* control alignment */
5196 XXH_ASSERT(((size_t)dst16 & 15) == 0);
5197
5198 for (i=0; i < nbRounds; ++i) {
5199 dst16[i] = _mm_add_epi64(_mm_load_si128((const __m128i *)src16+i), seed);
5200 } }
5201 }
5202
5203 #endif
5204
5205 #if (XXH_VECTOR == XXH_NEON)
5206
5207 /* forward declarations for the scalar routines */
5208 XXH_FORCE_INLINE void
5209 XXH3_scalarRound(void* XXH_RESTRICT acc, void const* XXH_RESTRICT input,
5210 void const* XXH_RESTRICT secret, size_t lane);
5211
5212 XXH_FORCE_INLINE void
5213 XXH3_scalarScrambleRound(void* XXH_RESTRICT acc,
5214 void const* XXH_RESTRICT secret, size_t lane);
5215
5216 /*!
5217 * @internal
5218 * @brief The bulk processing loop for NEON and WASM SIMD128.
5219 *
5220 * The NEON code path is actually partially scalar when running on AArch64. This
5221 * is to optimize the pipelining and can have up to 15% speedup depending on the
5222 * CPU, and it also mitigates some GCC codegen issues.
5223 *
5224 * @see XXH3_NEON_LANES for configuring this and details about this optimization.
5225 *
5226 * NEON's 32-bit to 64-bit long multiply takes a half vector of 32-bit
5227 * integers instead of the other platforms which mask full 64-bit vectors,
5228 * so the setup is more complicated than just shifting right.
5229 *
5230 * Additionally, there is an optimization for 4 lanes at once noted below.
5231 *
5232 * Since, as stated, the most optimal amount of lanes for Cortexes is 6,
5233 * there needs to be *three* versions of the accumulate operation used
5234 * for the remaining 2 lanes.
5235 *
5236 * WASM's SIMD128 uses SIMDe's arm_neon.h polyfill because the intrinsics overlap
5237 * nearly perfectly.
5238 */
5239
5240 XXH_FORCE_INLINE void
XXH3_accumulate_512_neon(void * XXH_RESTRICT acc,const void * XXH_RESTRICT input,const void * XXH_RESTRICT secret)5241 XXH3_accumulate_512_neon( void* XXH_RESTRICT acc,
5242 const void* XXH_RESTRICT input,
5243 const void* XXH_RESTRICT secret)
5244 {
5245 XXH_ASSERT((((size_t)acc) & 15) == 0);
5246 XXH_STATIC_ASSERT(XXH3_NEON_LANES > 0 && XXH3_NEON_LANES <= XXH_ACC_NB && XXH3_NEON_LANES % 2 == 0);
5247 { /* GCC for darwin arm64 does not like aliasing here */
5248 xxh_aliasing_uint64x2_t* const xacc = (xxh_aliasing_uint64x2_t*) acc;
5249 /* We don't use a uint32x4_t pointer because it causes bus errors on ARMv7. */
5250 uint8_t const* xinput = (const uint8_t *) input;
5251 uint8_t const* xsecret = (const uint8_t *) secret;
5252
5253 size_t i;
5254 #ifdef __wasm_simd128__
5255 /*
5256 * On WASM SIMD128, Clang emits direct address loads when XXH3_kSecret
5257 * is constant propagated, which results in it converting it to this
5258 * inside the loop:
5259 *
5260 * a = v128.load(XXH3_kSecret + 0 + $secret_offset, offset = 0)
5261 * b = v128.load(XXH3_kSecret + 16 + $secret_offset, offset = 0)
5262 * ...
5263 *
5264 * This requires a full 32-bit address immediate (and therefore a 6 byte
5265 * instruction) as well as an add for each offset.
5266 *
5267 * Putting an asm guard prevents it from folding (at the cost of losing
5268 * the alignment hint), and uses the free offset in `v128.load` instead
5269 * of adding secret_offset each time which overall reduces code size by
5270 * about a kilobyte and improves performance.
5271 */
5272 XXH_COMPILER_GUARD(xsecret);
5273 #endif
5274 /* Scalar lanes use the normal scalarRound routine */
5275 for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) {
5276 XXH3_scalarRound(acc, input, secret, i);
5277 }
5278 i = 0;
5279 /* 4 NEON lanes at a time. */
5280 for (; i+1 < XXH3_NEON_LANES / 2; i+=2) {
5281 /* data_vec = xinput[i]; */
5282 uint64x2_t data_vec_1 = XXH_vld1q_u64(xinput + (i * 16));
5283 uint64x2_t data_vec_2 = XXH_vld1q_u64(xinput + ((i+1) * 16));
5284 /* key_vec = xsecret[i]; */
5285 uint64x2_t key_vec_1 = XXH_vld1q_u64(xsecret + (i * 16));
5286 uint64x2_t key_vec_2 = XXH_vld1q_u64(xsecret + ((i+1) * 16));
5287 /* data_swap = swap(data_vec) */
5288 uint64x2_t data_swap_1 = vextq_u64(data_vec_1, data_vec_1, 1);
5289 uint64x2_t data_swap_2 = vextq_u64(data_vec_2, data_vec_2, 1);
5290 /* data_key = data_vec ^ key_vec; */
5291 uint64x2_t data_key_1 = veorq_u64(data_vec_1, key_vec_1);
5292 uint64x2_t data_key_2 = veorq_u64(data_vec_2, key_vec_2);
5293
5294 /*
5295 * If we reinterpret the 64x2 vectors as 32x4 vectors, we can use a
5296 * de-interleave operation for 4 lanes in 1 step with `vuzpq_u32` to
5297 * get one vector with the low 32 bits of each lane, and one vector
5298 * with the high 32 bits of each lane.
5299 *
5300 * The intrinsic returns a double vector because the original ARMv7-a
5301 * instruction modified both arguments in place. AArch64 and SIMD128 emit
5302 * two instructions from this intrinsic.
5303 *
5304 * [ dk11L | dk11H | dk12L | dk12H ] -> [ dk11L | dk12L | dk21L | dk22L ]
5305 * [ dk21L | dk21H | dk22L | dk22H ] -> [ dk11H | dk12H | dk21H | dk22H ]
5306 */
5307 uint32x4x2_t unzipped = vuzpq_u32(
5308 vreinterpretq_u32_u64(data_key_1),
5309 vreinterpretq_u32_u64(data_key_2)
5310 );
5311 /* data_key_lo = data_key & 0xFFFFFFFF */
5312 uint32x4_t data_key_lo = unzipped.val[0];
5313 /* data_key_hi = data_key >> 32 */
5314 uint32x4_t data_key_hi = unzipped.val[1];
5315 /*
5316 * Then, we can split the vectors horizontally and multiply which, as for most
5317 * widening intrinsics, have a variant that works on both high half vectors
5318 * for free on AArch64. A similar instruction is available on SIMD128.
5319 *
5320 * sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi
5321 */
5322 uint64x2_t sum_1 = XXH_vmlal_low_u32(data_swap_1, data_key_lo, data_key_hi);
5323 uint64x2_t sum_2 = XXH_vmlal_high_u32(data_swap_2, data_key_lo, data_key_hi);
5324 /*
5325 * Clang reorders
5326 * a += b * c; // umlal swap.2d, dkl.2s, dkh.2s
5327 * c += a; // add acc.2d, acc.2d, swap.2d
5328 * to
5329 * c += a; // add acc.2d, acc.2d, swap.2d
5330 * c += b * c; // umlal acc.2d, dkl.2s, dkh.2s
5331 *
5332 * While it would make sense in theory since the addition is faster,
5333 * for reasons likely related to umlal being limited to certain NEON
5334 * pipelines, this is worse. A compiler guard fixes this.
5335 */
5336 XXH_COMPILER_GUARD_CLANG_NEON(sum_1);
5337 XXH_COMPILER_GUARD_CLANG_NEON(sum_2);
5338 /* xacc[i] = acc_vec + sum; */
5339 xacc[i] = vaddq_u64(xacc[i], sum_1);
5340 xacc[i+1] = vaddq_u64(xacc[i+1], sum_2);
5341 }
5342 /* Operate on the remaining NEON lanes 2 at a time. */
5343 for (; i < XXH3_NEON_LANES / 2; i++) {
5344 /* data_vec = xinput[i]; */
5345 uint64x2_t data_vec = XXH_vld1q_u64(xinput + (i * 16));
5346 /* key_vec = xsecret[i]; */
5347 uint64x2_t key_vec = XXH_vld1q_u64(xsecret + (i * 16));
5348 /* acc_vec_2 = swap(data_vec) */
5349 uint64x2_t data_swap = vextq_u64(data_vec, data_vec, 1);
5350 /* data_key = data_vec ^ key_vec; */
5351 uint64x2_t data_key = veorq_u64(data_vec, key_vec);
5352 /* For two lanes, just use VMOVN and VSHRN. */
5353 /* data_key_lo = data_key & 0xFFFFFFFF; */
5354 uint32x2_t data_key_lo = vmovn_u64(data_key);
5355 /* data_key_hi = data_key >> 32; */
5356 uint32x2_t data_key_hi = vshrn_n_u64(data_key, 32);
5357 /* sum = data_swap + (u64x2) data_key_lo * (u64x2) data_key_hi; */
5358 uint64x2_t sum = vmlal_u32(data_swap, data_key_lo, data_key_hi);
5359 /* Same Clang workaround as before */
5360 XXH_COMPILER_GUARD_CLANG_NEON(sum);
5361 /* xacc[i] = acc_vec + sum; */
5362 xacc[i] = vaddq_u64 (xacc[i], sum);
5363 }
5364 }
5365 }
XXH3_ACCUMULATE_TEMPLATE(neon)5366 XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(neon)
5367
5368 XXH_FORCE_INLINE void
5369 XXH3_scrambleAcc_neon(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
5370 {
5371 XXH_ASSERT((((size_t)acc) & 15) == 0);
5372
5373 { xxh_aliasing_uint64x2_t* xacc = (xxh_aliasing_uint64x2_t*) acc;
5374 uint8_t const* xsecret = (uint8_t const*) secret;
5375
5376 size_t i;
5377 /* WASM uses operator overloads and doesn't need these. */
5378 #ifndef __wasm_simd128__
5379 /* { prime32_1, prime32_1 } */
5380 uint32x2_t const kPrimeLo = vdup_n_u32(XXH_PRIME32_1);
5381 /* { 0, prime32_1, 0, prime32_1 } */
5382 uint32x4_t const kPrimeHi = vreinterpretq_u32_u64(vdupq_n_u64((xxh_u64)XXH_PRIME32_1 << 32));
5383 #endif
5384
5385 /* AArch64 uses both scalar and neon at the same time */
5386 for (i = XXH3_NEON_LANES; i < XXH_ACC_NB; i++) {
5387 XXH3_scalarScrambleRound(acc, secret, i);
5388 }
5389 for (i=0; i < XXH3_NEON_LANES / 2; i++) {
5390 /* xacc[i] ^= (xacc[i] >> 47); */
5391 uint64x2_t acc_vec = xacc[i];
5392 uint64x2_t shifted = vshrq_n_u64(acc_vec, 47);
5393 uint64x2_t data_vec = veorq_u64(acc_vec, shifted);
5394
5395 /* xacc[i] ^= xsecret[i]; */
5396 uint64x2_t key_vec = XXH_vld1q_u64(xsecret + (i * 16));
5397 uint64x2_t data_key = veorq_u64(data_vec, key_vec);
5398 /* xacc[i] *= XXH_PRIME32_1 */
5399 #ifdef __wasm_simd128__
5400 /* SIMD128 has multiply by u64x2, use it instead of expanding and scalarizing */
5401 xacc[i] = data_key * XXH_PRIME32_1;
5402 #else
5403 /*
5404 * Expanded version with portable NEON intrinsics
5405 *
5406 * lo(x) * lo(y) + (hi(x) * lo(y) << 32)
5407 *
5408 * prod_hi = hi(data_key) * lo(prime) << 32
5409 *
5410 * Since we only need 32 bits of this multiply a trick can be used, reinterpreting the vector
5411 * as a uint32x4_t and multiplying by { 0, prime, 0, prime } to cancel out the unwanted bits
5412 * and avoid the shift.
5413 */
5414 uint32x4_t prod_hi = vmulq_u32 (vreinterpretq_u32_u64(data_key), kPrimeHi);
5415 /* Extract low bits for vmlal_u32 */
5416 uint32x2_t data_key_lo = vmovn_u64(data_key);
5417 /* xacc[i] = prod_hi + lo(data_key) * XXH_PRIME32_1; */
5418 xacc[i] = vmlal_u32(vreinterpretq_u64_u32(prod_hi), data_key_lo, kPrimeLo);
5419 #endif
5420 }
5421 }
5422 }
5423 #endif
5424
5425 #if (XXH_VECTOR == XXH_VSX)
5426
5427 XXH_FORCE_INLINE void
XXH3_accumulate_512_vsx(void * XXH_RESTRICT acc,const void * XXH_RESTRICT input,const void * XXH_RESTRICT secret)5428 XXH3_accumulate_512_vsx( void* XXH_RESTRICT acc,
5429 const void* XXH_RESTRICT input,
5430 const void* XXH_RESTRICT secret)
5431 {
5432 /* presumed aligned */
5433 xxh_aliasing_u64x2* const xacc = (xxh_aliasing_u64x2*) acc;
5434 xxh_u8 const* const xinput = (xxh_u8 const*) input; /* no alignment restriction */
5435 xxh_u8 const* const xsecret = (xxh_u8 const*) secret; /* no alignment restriction */
5436 xxh_u64x2 const v32 = { 32, 32 };
5437 size_t i;
5438 for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) {
5439 /* data_vec = xinput[i]; */
5440 xxh_u64x2 const data_vec = XXH_vec_loadu(xinput + 16*i);
5441 /* key_vec = xsecret[i]; */
5442 xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + 16*i);
5443 xxh_u64x2 const data_key = data_vec ^ key_vec;
5444 /* shuffled = (data_key << 32) | (data_key >> 32); */
5445 xxh_u32x4 const shuffled = (xxh_u32x4)vec_rl(data_key, v32);
5446 /* product = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)shuffled & 0xFFFFFFFF); */
5447 xxh_u64x2 const product = XXH_vec_mulo((xxh_u32x4)data_key, shuffled);
5448 /* acc_vec = xacc[i]; */
5449 xxh_u64x2 acc_vec = xacc[i];
5450 acc_vec += product;
5451
5452 /* swap high and low halves */
5453 #ifdef __s390x__
5454 acc_vec += vec_permi(data_vec, data_vec, 2);
5455 #else
5456 acc_vec += vec_xxpermdi(data_vec, data_vec, 2);
5457 #endif
5458 xacc[i] = acc_vec;
5459 }
5460 }
XXH3_ACCUMULATE_TEMPLATE(vsx)5461 XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(vsx)
5462
5463 XXH_FORCE_INLINE void
5464 XXH3_scrambleAcc_vsx(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
5465 {
5466 XXH_ASSERT((((size_t)acc) & 15) == 0);
5467
5468 { xxh_aliasing_u64x2* const xacc = (xxh_aliasing_u64x2*) acc;
5469 const xxh_u8* const xsecret = (const xxh_u8*) secret;
5470 /* constants */
5471 xxh_u64x2 const v32 = { 32, 32 };
5472 xxh_u64x2 const v47 = { 47, 47 };
5473 xxh_u32x4 const prime = { XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1, XXH_PRIME32_1 };
5474 size_t i;
5475 for (i = 0; i < XXH_STRIPE_LEN / sizeof(xxh_u64x2); i++) {
5476 /* xacc[i] ^= (xacc[i] >> 47); */
5477 xxh_u64x2 const acc_vec = xacc[i];
5478 xxh_u64x2 const data_vec = acc_vec ^ (acc_vec >> v47);
5479
5480 /* xacc[i] ^= xsecret[i]; */
5481 xxh_u64x2 const key_vec = XXH_vec_loadu(xsecret + 16*i);
5482 xxh_u64x2 const data_key = data_vec ^ key_vec;
5483
5484 /* xacc[i] *= XXH_PRIME32_1 */
5485 /* prod_lo = ((xxh_u64x2)data_key & 0xFFFFFFFF) * ((xxh_u64x2)prime & 0xFFFFFFFF); */
5486 xxh_u64x2 const prod_even = XXH_vec_mule((xxh_u32x4)data_key, prime);
5487 /* prod_hi = ((xxh_u64x2)data_key >> 32) * ((xxh_u64x2)prime >> 32); */
5488 xxh_u64x2 const prod_odd = XXH_vec_mulo((xxh_u32x4)data_key, prime);
5489 xacc[i] = prod_odd + (prod_even << v32);
5490 } }
5491 }
5492
5493 #endif
5494
5495 #if (XXH_VECTOR == XXH_SVE)
5496
5497 XXH_FORCE_INLINE void
XXH3_accumulate_512_sve(void * XXH_RESTRICT acc,const void * XXH_RESTRICT input,const void * XXH_RESTRICT secret)5498 XXH3_accumulate_512_sve( void* XXH_RESTRICT acc,
5499 const void* XXH_RESTRICT input,
5500 const void* XXH_RESTRICT secret)
5501 {
5502 uint64_t *xacc = (uint64_t *)acc;
5503 const uint64_t *xinput = (const uint64_t *)(const void *)input;
5504 const uint64_t *xsecret = (const uint64_t *)(const void *)secret;
5505 svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1);
5506 uint64_t element_count = svcntd();
5507 if (element_count >= 8) {
5508 svbool_t mask = svptrue_pat_b64(SV_VL8);
5509 svuint64_t vacc = svld1_u64(mask, xacc);
5510 ACCRND(vacc, 0);
5511 svst1_u64(mask, xacc, vacc);
5512 } else if (element_count == 2) { /* sve128 */
5513 svbool_t mask = svptrue_pat_b64(SV_VL2);
5514 svuint64_t acc0 = svld1_u64(mask, xacc + 0);
5515 svuint64_t acc1 = svld1_u64(mask, xacc + 2);
5516 svuint64_t acc2 = svld1_u64(mask, xacc + 4);
5517 svuint64_t acc3 = svld1_u64(mask, xacc + 6);
5518 ACCRND(acc0, 0);
5519 ACCRND(acc1, 2);
5520 ACCRND(acc2, 4);
5521 ACCRND(acc3, 6);
5522 svst1_u64(mask, xacc + 0, acc0);
5523 svst1_u64(mask, xacc + 2, acc1);
5524 svst1_u64(mask, xacc + 4, acc2);
5525 svst1_u64(mask, xacc + 6, acc3);
5526 } else {
5527 svbool_t mask = svptrue_pat_b64(SV_VL4);
5528 svuint64_t acc0 = svld1_u64(mask, xacc + 0);
5529 svuint64_t acc1 = svld1_u64(mask, xacc + 4);
5530 ACCRND(acc0, 0);
5531 ACCRND(acc1, 4);
5532 svst1_u64(mask, xacc + 0, acc0);
5533 svst1_u64(mask, xacc + 4, acc1);
5534 }
5535 }
5536
5537 XXH_FORCE_INLINE void
XXH3_accumulate_sve(xxh_u64 * XXH_RESTRICT acc,const xxh_u8 * XXH_RESTRICT input,const xxh_u8 * XXH_RESTRICT secret,size_t nbStripes)5538 XXH3_accumulate_sve(xxh_u64* XXH_RESTRICT acc,
5539 const xxh_u8* XXH_RESTRICT input,
5540 const xxh_u8* XXH_RESTRICT secret,
5541 size_t nbStripes)
5542 {
5543 if (nbStripes != 0) {
5544 uint64_t *xacc = (uint64_t *)acc;
5545 const uint64_t *xinput = (const uint64_t *)(const void *)input;
5546 const uint64_t *xsecret = (const uint64_t *)(const void *)secret;
5547 svuint64_t kSwap = sveor_n_u64_z(svptrue_b64(), svindex_u64(0, 1), 1);
5548 uint64_t element_count = svcntd();
5549 if (element_count >= 8) {
5550 svbool_t mask = svptrue_pat_b64(SV_VL8);
5551 svuint64_t vacc = svld1_u64(mask, xacc + 0);
5552 do {
5553 /* svprfd(svbool_t, void *, enum svfprop); */
5554 svprfd(mask, xinput + 128, SV_PLDL1STRM);
5555 ACCRND(vacc, 0);
5556 xinput += 8;
5557 xsecret += 1;
5558 nbStripes--;
5559 } while (nbStripes != 0);
5560
5561 svst1_u64(mask, xacc + 0, vacc);
5562 } else if (element_count == 2) { /* sve128 */
5563 svbool_t mask = svptrue_pat_b64(SV_VL2);
5564 svuint64_t acc0 = svld1_u64(mask, xacc + 0);
5565 svuint64_t acc1 = svld1_u64(mask, xacc + 2);
5566 svuint64_t acc2 = svld1_u64(mask, xacc + 4);
5567 svuint64_t acc3 = svld1_u64(mask, xacc + 6);
5568 do {
5569 svprfd(mask, xinput + 128, SV_PLDL1STRM);
5570 ACCRND(acc0, 0);
5571 ACCRND(acc1, 2);
5572 ACCRND(acc2, 4);
5573 ACCRND(acc3, 6);
5574 xinput += 8;
5575 xsecret += 1;
5576 nbStripes--;
5577 } while (nbStripes != 0);
5578
5579 svst1_u64(mask, xacc + 0, acc0);
5580 svst1_u64(mask, xacc + 2, acc1);
5581 svst1_u64(mask, xacc + 4, acc2);
5582 svst1_u64(mask, xacc + 6, acc3);
5583 } else {
5584 svbool_t mask = svptrue_pat_b64(SV_VL4);
5585 svuint64_t acc0 = svld1_u64(mask, xacc + 0);
5586 svuint64_t acc1 = svld1_u64(mask, xacc + 4);
5587 do {
5588 svprfd(mask, xinput + 128, SV_PLDL1STRM);
5589 ACCRND(acc0, 0);
5590 ACCRND(acc1, 4);
5591 xinput += 8;
5592 xsecret += 1;
5593 nbStripes--;
5594 } while (nbStripes != 0);
5595
5596 svst1_u64(mask, xacc + 0, acc0);
5597 svst1_u64(mask, xacc + 4, acc1);
5598 }
5599 }
5600 }
5601
5602 #endif
5603
5604 /* scalar variants - universal */
5605
5606 #if defined(__aarch64__) && (defined(__GNUC__) || defined(__clang__))
5607 /*
5608 * In XXH3_scalarRound(), GCC and Clang have a similar codegen issue, where they
5609 * emit an excess mask and a full 64-bit multiply-add (MADD X-form).
5610 *
5611 * While this might not seem like much, as AArch64 is a 64-bit architecture, only
5612 * big Cortex designs have a full 64-bit multiplier.
5613 *
5614 * On the little cores, the smaller 32-bit multiplier is used, and full 64-bit
5615 * multiplies expand to 2-3 multiplies in microcode. This has a major penalty
5616 * of up to 4 latency cycles and 2 stall cycles in the multiply pipeline.
5617 *
5618 * Thankfully, AArch64 still provides the 32-bit long multiply-add (UMADDL) which does
5619 * not have this penalty and does the mask automatically.
5620 */
5621 XXH_FORCE_INLINE xxh_u64
XXH_mult32to64_add64(xxh_u64 lhs,xxh_u64 rhs,xxh_u64 acc)5622 XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, xxh_u64 acc)
5623 {
5624 xxh_u64 ret;
5625 /* note: %x = 64-bit register, %w = 32-bit register */
5626 __asm__("umaddl %x0, %w1, %w2, %x3" : "=r" (ret) : "r" (lhs), "r" (rhs), "r" (acc));
5627 return ret;
5628 }
5629 #else
5630 XXH_FORCE_INLINE xxh_u64
XXH_mult32to64_add64(xxh_u64 lhs,xxh_u64 rhs,xxh_u64 acc)5631 XXH_mult32to64_add64(xxh_u64 lhs, xxh_u64 rhs, xxh_u64 acc)
5632 {
5633 return XXH_mult32to64((xxh_u32)lhs, (xxh_u32)rhs) + acc;
5634 }
5635 #endif
5636
5637 /*!
5638 * @internal
5639 * @brief Scalar round for @ref XXH3_accumulate_512_scalar().
5640 *
5641 * This is extracted to its own function because the NEON path uses a combination
5642 * of NEON and scalar.
5643 */
5644 XXH_FORCE_INLINE void
XXH3_scalarRound(void * XXH_RESTRICT acc,void const * XXH_RESTRICT input,void const * XXH_RESTRICT secret,size_t lane)5645 XXH3_scalarRound(void* XXH_RESTRICT acc,
5646 void const* XXH_RESTRICT input,
5647 void const* XXH_RESTRICT secret,
5648 size_t lane)
5649 {
5650 xxh_u64* xacc = (xxh_u64*) acc;
5651 xxh_u8 const* xinput = (xxh_u8 const*) input;
5652 xxh_u8 const* xsecret = (xxh_u8 const*) secret;
5653 XXH_ASSERT(lane < XXH_ACC_NB);
5654 XXH_ASSERT(((size_t)acc & (XXH_ACC_ALIGN-1)) == 0);
5655 {
5656 xxh_u64 const data_val = XXH_readLE64(xinput + lane * 8);
5657 xxh_u64 const data_key = data_val ^ XXH_readLE64(xsecret + lane * 8);
5658 xacc[lane ^ 1] += data_val; /* swap adjacent lanes */
5659 xacc[lane] = XXH_mult32to64_add64(data_key /* & 0xFFFFFFFF */, data_key >> 32, xacc[lane]);
5660 }
5661 }
5662
5663 /*!
5664 * @internal
5665 * @brief Processes a 64 byte block of data using the scalar path.
5666 */
5667 XXH_FORCE_INLINE void
XXH3_accumulate_512_scalar(void * XXH_RESTRICT acc,const void * XXH_RESTRICT input,const void * XXH_RESTRICT secret)5668 XXH3_accumulate_512_scalar(void* XXH_RESTRICT acc,
5669 const void* XXH_RESTRICT input,
5670 const void* XXH_RESTRICT secret)
5671 {
5672 size_t i;
5673 /* ARM GCC refuses to unroll this loop, resulting in a 24% slowdown on ARMv6. */
5674 #if defined(__GNUC__) && !defined(__clang__) \
5675 && (defined(__arm__) || defined(__thumb2__)) \
5676 && defined(__ARM_FEATURE_UNALIGNED) /* no unaligned access just wastes bytes */ \
5677 && XXH_SIZE_OPT <= 0
5678 # pragma GCC unroll 8
5679 #endif
5680 for (i=0; i < XXH_ACC_NB; i++) {
5681 XXH3_scalarRound(acc, input, secret, i);
5682 }
5683 }
XXH3_ACCUMULATE_TEMPLATE(scalar)5684 XXH_FORCE_INLINE XXH3_ACCUMULATE_TEMPLATE(scalar)
5685
5686 /*!
5687 * @internal
5688 * @brief Scalar scramble step for @ref XXH3_scrambleAcc_scalar().
5689 *
5690 * This is extracted to its own function because the NEON path uses a combination
5691 * of NEON and scalar.
5692 */
5693 XXH_FORCE_INLINE void
5694 XXH3_scalarScrambleRound(void* XXH_RESTRICT acc,
5695 void const* XXH_RESTRICT secret,
5696 size_t lane)
5697 {
5698 xxh_u64* const xacc = (xxh_u64*) acc; /* presumed aligned */
5699 const xxh_u8* const xsecret = (const xxh_u8*) secret; /* no alignment restriction */
5700 XXH_ASSERT((((size_t)acc) & (XXH_ACC_ALIGN-1)) == 0);
5701 XXH_ASSERT(lane < XXH_ACC_NB);
5702 {
5703 xxh_u64 const key64 = XXH_readLE64(xsecret + lane * 8);
5704 xxh_u64 acc64 = xacc[lane];
5705 acc64 = XXH_xorshift64(acc64, 47);
5706 acc64 ^= key64;
5707 acc64 *= XXH_PRIME32_1;
5708 xacc[lane] = acc64;
5709 }
5710 }
5711
5712 /*!
5713 * @internal
5714 * @brief Scrambles the accumulators after a large chunk has been read
5715 */
5716 XXH_FORCE_INLINE void
XXH3_scrambleAcc_scalar(void * XXH_RESTRICT acc,const void * XXH_RESTRICT secret)5717 XXH3_scrambleAcc_scalar(void* XXH_RESTRICT acc, const void* XXH_RESTRICT secret)
5718 {
5719 size_t i;
5720 for (i=0; i < XXH_ACC_NB; i++) {
5721 XXH3_scalarScrambleRound(acc, secret, i);
5722 }
5723 }
5724
5725 XXH_FORCE_INLINE void
XXH3_initCustomSecret_scalar(void * XXH_RESTRICT customSecret,xxh_u64 seed64)5726 XXH3_initCustomSecret_scalar(void* XXH_RESTRICT customSecret, xxh_u64 seed64)
5727 {
5728 /*
5729 * We need a separate pointer for the hack below,
5730 * which requires a non-const pointer.
5731 * Any decent compiler will optimize this out otherwise.
5732 */
5733 const xxh_u8* kSecretPtr = XXH3_kSecret;
5734 XXH_STATIC_ASSERT((XXH_SECRET_DEFAULT_SIZE & 15) == 0);
5735
5736 #if defined(__GNUC__) && defined(__aarch64__)
5737 /*
5738 * UGLY HACK:
5739 * GCC and Clang generate a bunch of MOV/MOVK pairs for aarch64, and they are
5740 * placed sequentially, in order, at the top of the unrolled loop.
5741 *
5742 * While MOVK is great for generating constants (2 cycles for a 64-bit
5743 * constant compared to 4 cycles for LDR), it fights for bandwidth with
5744 * the arithmetic instructions.
5745 *
5746 * I L S
5747 * MOVK
5748 * MOVK
5749 * MOVK
5750 * MOVK
5751 * ADD
5752 * SUB STR
5753 * STR
5754 * By forcing loads from memory (as the asm line causes the compiler to assume
5755 * that XXH3_kSecretPtr has been changed), the pipelines are used more
5756 * efficiently:
5757 * I L S
5758 * LDR
5759 * ADD LDR
5760 * SUB STR
5761 * STR
5762 *
5763 * See XXH3_NEON_LANES for details on the pipsline.
5764 *
5765 * XXH3_64bits_withSeed, len == 256, Snapdragon 835
5766 * without hack: 2654.4 MB/s
5767 * with hack: 3202.9 MB/s
5768 */
5769 XXH_COMPILER_GUARD(kSecretPtr);
5770 #endif
5771 { int const nbRounds = XXH_SECRET_DEFAULT_SIZE / 16;
5772 int i;
5773 for (i=0; i < nbRounds; i++) {
5774 /*
5775 * The asm hack causes the compiler to assume that kSecretPtr aliases with
5776 * customSecret, and on aarch64, this prevented LDP from merging two
5777 * loads together for free. Putting the loads together before the stores
5778 * properly generates LDP.
5779 */
5780 xxh_u64 lo = XXH_readLE64(kSecretPtr + 16*i) + seed64;
5781 xxh_u64 hi = XXH_readLE64(kSecretPtr + 16*i + 8) - seed64;
5782 XXH_writeLE64((xxh_u8*)customSecret + 16*i, lo);
5783 XXH_writeLE64((xxh_u8*)customSecret + 16*i + 8, hi);
5784 } }
5785 }
5786
5787
5788 typedef void (*XXH3_f_accumulate)(xxh_u64* XXH_RESTRICT, const xxh_u8* XXH_RESTRICT, const xxh_u8* XXH_RESTRICT, size_t);
5789 typedef void (*XXH3_f_scrambleAcc)(void* XXH_RESTRICT, const void*);
5790 typedef void (*XXH3_f_initCustomSecret)(void* XXH_RESTRICT, xxh_u64);
5791
5792
5793 #if (XXH_VECTOR == XXH_AVX512)
5794
5795 #define XXH3_accumulate_512 XXH3_accumulate_512_avx512
5796 #define XXH3_accumulate XXH3_accumulate_avx512
5797 #define XXH3_scrambleAcc XXH3_scrambleAcc_avx512
5798 #define XXH3_initCustomSecret XXH3_initCustomSecret_avx512
5799
5800 #elif (XXH_VECTOR == XXH_AVX2)
5801
5802 #define XXH3_accumulate_512 XXH3_accumulate_512_avx2
5803 #define XXH3_accumulate XXH3_accumulate_avx2
5804 #define XXH3_scrambleAcc XXH3_scrambleAcc_avx2
5805 #define XXH3_initCustomSecret XXH3_initCustomSecret_avx2
5806
5807 #elif (XXH_VECTOR == XXH_SSE2)
5808
5809 #define XXH3_accumulate_512 XXH3_accumulate_512_sse2
5810 #define XXH3_accumulate XXH3_accumulate_sse2
5811 #define XXH3_scrambleAcc XXH3_scrambleAcc_sse2
5812 #define XXH3_initCustomSecret XXH3_initCustomSecret_sse2
5813
5814 #elif (XXH_VECTOR == XXH_NEON)
5815
5816 #define XXH3_accumulate_512 XXH3_accumulate_512_neon
5817 #define XXH3_accumulate XXH3_accumulate_neon
5818 #define XXH3_scrambleAcc XXH3_scrambleAcc_neon
5819 #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar
5820
5821 #elif (XXH_VECTOR == XXH_VSX)
5822
5823 #define XXH3_accumulate_512 XXH3_accumulate_512_vsx
5824 #define XXH3_accumulate XXH3_accumulate_vsx
5825 #define XXH3_scrambleAcc XXH3_scrambleAcc_vsx
5826 #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar
5827
5828 #elif (XXH_VECTOR == XXH_SVE)
5829 #define XXH3_accumulate_512 XXH3_accumulate_512_sve
5830 #define XXH3_accumulate XXH3_accumulate_sve
5831 #define XXH3_scrambleAcc XXH3_scrambleAcc_scalar
5832 #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar
5833
5834 #else /* scalar */
5835
5836 #define XXH3_accumulate_512 XXH3_accumulate_512_scalar
5837 #define XXH3_accumulate XXH3_accumulate_scalar
5838 #define XXH3_scrambleAcc XXH3_scrambleAcc_scalar
5839 #define XXH3_initCustomSecret XXH3_initCustomSecret_scalar
5840
5841 #endif
5842
5843 #if XXH_SIZE_OPT >= 1 /* don't do SIMD for initialization */
5844 # undef XXH3_initCustomSecret
5845 # define XXH3_initCustomSecret XXH3_initCustomSecret_scalar
5846 #endif
5847
5848 XXH_FORCE_INLINE void
XXH3_hashLong_internal_loop(xxh_u64 * XXH_RESTRICT acc,const xxh_u8 * XXH_RESTRICT input,size_t len,const xxh_u8 * XXH_RESTRICT secret,size_t secretSize,XXH3_f_accumulate f_acc,XXH3_f_scrambleAcc f_scramble)5849 XXH3_hashLong_internal_loop(xxh_u64* XXH_RESTRICT acc,
5850 const xxh_u8* XXH_RESTRICT input, size_t len,
5851 const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
5852 XXH3_f_accumulate f_acc,
5853 XXH3_f_scrambleAcc f_scramble)
5854 {
5855 size_t const nbStripesPerBlock = (secretSize - XXH_STRIPE_LEN) / XXH_SECRET_CONSUME_RATE;
5856 size_t const block_len = XXH_STRIPE_LEN * nbStripesPerBlock;
5857 size_t const nb_blocks = (len - 1) / block_len;
5858
5859 size_t n;
5860
5861 XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN);
5862
5863 for (n = 0; n < nb_blocks; n++) {
5864 f_acc(acc, input + n*block_len, secret, nbStripesPerBlock);
5865 f_scramble(acc, secret + secretSize - XXH_STRIPE_LEN);
5866 }
5867
5868 /* last partial block */
5869 XXH_ASSERT(len > XXH_STRIPE_LEN);
5870 { size_t const nbStripes = ((len - 1) - (block_len * nb_blocks)) / XXH_STRIPE_LEN;
5871 XXH_ASSERT(nbStripes <= (secretSize / XXH_SECRET_CONSUME_RATE));
5872 f_acc(acc, input + nb_blocks*block_len, secret, nbStripes);
5873
5874 /* last stripe */
5875 { const xxh_u8* const p = input + len - XXH_STRIPE_LEN;
5876 #define XXH_SECRET_LASTACC_START 7 /* not aligned on 8, last secret is different from acc & scrambler */
5877 XXH3_accumulate_512(acc, p, secret + secretSize - XXH_STRIPE_LEN - XXH_SECRET_LASTACC_START);
5878 } }
5879 }
5880
5881 XXH_FORCE_INLINE xxh_u64
XXH3_mix2Accs(const xxh_u64 * XXH_RESTRICT acc,const xxh_u8 * XXH_RESTRICT secret)5882 XXH3_mix2Accs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret)
5883 {
5884 return XXH3_mul128_fold64(
5885 acc[0] ^ XXH_readLE64(secret),
5886 acc[1] ^ XXH_readLE64(secret+8) );
5887 }
5888
5889 static XXH64_hash_t
XXH3_mergeAccs(const xxh_u64 * XXH_RESTRICT acc,const xxh_u8 * XXH_RESTRICT secret,xxh_u64 start)5890 XXH3_mergeAccs(const xxh_u64* XXH_RESTRICT acc, const xxh_u8* XXH_RESTRICT secret, xxh_u64 start)
5891 {
5892 xxh_u64 result64 = start;
5893 size_t i = 0;
5894
5895 for (i = 0; i < 4; i++) {
5896 result64 += XXH3_mix2Accs(acc+2*i, secret + 16*i);
5897 #if defined(__clang__) /* Clang */ \
5898 && (defined(__arm__) || defined(__thumb__)) /* ARMv7 */ \
5899 && (defined(__ARM_NEON) || defined(__ARM_NEON__)) /* NEON */ \
5900 && !defined(XXH_ENABLE_AUTOVECTORIZE) /* Define to disable */
5901 /*
5902 * UGLY HACK:
5903 * Prevent autovectorization on Clang ARMv7-a. Exact same problem as
5904 * the one in XXH3_len_129to240_64b. Speeds up shorter keys > 240b.
5905 * XXH3_64bits, len == 256, Snapdragon 835:
5906 * without hack: 2063.7 MB/s
5907 * with hack: 2560.7 MB/s
5908 */
5909 XXH_COMPILER_GUARD(result64);
5910 #endif
5911 }
5912
5913 return XXH3_avalanche(result64);
5914 }
5915
5916 #define XXH3_INIT_ACC { XXH_PRIME32_3, XXH_PRIME64_1, XXH_PRIME64_2, XXH_PRIME64_3, \
5917 XXH_PRIME64_4, XXH_PRIME32_2, XXH_PRIME64_5, XXH_PRIME32_1 }
5918
5919 XXH_FORCE_INLINE XXH64_hash_t
XXH3_hashLong_64b_internal(const void * XXH_RESTRICT input,size_t len,const void * XXH_RESTRICT secret,size_t secretSize,XXH3_f_accumulate f_acc,XXH3_f_scrambleAcc f_scramble)5920 XXH3_hashLong_64b_internal(const void* XXH_RESTRICT input, size_t len,
5921 const void* XXH_RESTRICT secret, size_t secretSize,
5922 XXH3_f_accumulate f_acc,
5923 XXH3_f_scrambleAcc f_scramble)
5924 {
5925 XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC;
5926
5927 XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, (const xxh_u8*)secret, secretSize, f_acc, f_scramble);
5928
5929 /* converge into final hash */
5930 XXH_STATIC_ASSERT(sizeof(acc) == 64);
5931 /* do not align on 8, so that the secret is different from the accumulator */
5932 #define XXH_SECRET_MERGEACCS_START 11
5933 XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START);
5934 return XXH3_mergeAccs(acc, (const xxh_u8*)secret + XXH_SECRET_MERGEACCS_START, (xxh_u64)len * XXH_PRIME64_1);
5935 }
5936
5937 /*
5938 * It's important for performance to transmit secret's size (when it's static)
5939 * so that the compiler can properly optimize the vectorized loop.
5940 * This makes a big performance difference for "medium" keys (<1 KB) when using AVX instruction set.
5941 * When the secret size is unknown, or on GCC 12 where the mix of NO_INLINE and FORCE_INLINE
5942 * breaks -Og, this is XXH_NO_INLINE.
5943 */
5944 XXH3_WITH_SECRET_INLINE XXH64_hash_t
XXH3_hashLong_64b_withSecret(const void * XXH_RESTRICT input,size_t len,XXH64_hash_t seed64,const xxh_u8 * XXH_RESTRICT secret,size_t secretLen)5945 XXH3_hashLong_64b_withSecret(const void* XXH_RESTRICT input, size_t len,
5946 XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen)
5947 {
5948 (void)seed64;
5949 return XXH3_hashLong_64b_internal(input, len, secret, secretLen, XXH3_accumulate, XXH3_scrambleAcc);
5950 }
5951
5952 /*
5953 * It's preferable for performance that XXH3_hashLong is not inlined,
5954 * as it results in a smaller function for small data, easier to the instruction cache.
5955 * Note that inside this no_inline function, we do inline the internal loop,
5956 * and provide a statically defined secret size to allow optimization of vector loop.
5957 */
5958 XXH_NO_INLINE XXH_PUREF XXH64_hash_t
XXH3_hashLong_64b_default(const void * XXH_RESTRICT input,size_t len,XXH64_hash_t seed64,const xxh_u8 * XXH_RESTRICT secret,size_t secretLen)5959 XXH3_hashLong_64b_default(const void* XXH_RESTRICT input, size_t len,
5960 XXH64_hash_t seed64, const xxh_u8* XXH_RESTRICT secret, size_t secretLen)
5961 {
5962 (void)seed64; (void)secret; (void)secretLen;
5963 return XXH3_hashLong_64b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_accumulate, XXH3_scrambleAcc);
5964 }
5965
5966 /*
5967 * XXH3_hashLong_64b_withSeed():
5968 * Generate a custom key based on alteration of default XXH3_kSecret with the seed,
5969 * and then use this key for long mode hashing.
5970 *
5971 * This operation is decently fast but nonetheless costs a little bit of time.
5972 * Try to avoid it whenever possible (typically when seed==0).
5973 *
5974 * It's important for performance that XXH3_hashLong is not inlined. Not sure
5975 * why (uop cache maybe?), but the difference is large and easily measurable.
5976 */
5977 XXH_FORCE_INLINE XXH64_hash_t
XXH3_hashLong_64b_withSeed_internal(const void * input,size_t len,XXH64_hash_t seed,XXH3_f_accumulate f_acc,XXH3_f_scrambleAcc f_scramble,XXH3_f_initCustomSecret f_initSec)5978 XXH3_hashLong_64b_withSeed_internal(const void* input, size_t len,
5979 XXH64_hash_t seed,
5980 XXH3_f_accumulate f_acc,
5981 XXH3_f_scrambleAcc f_scramble,
5982 XXH3_f_initCustomSecret f_initSec)
5983 {
5984 #if XXH_SIZE_OPT <= 0
5985 if (seed == 0)
5986 return XXH3_hashLong_64b_internal(input, len,
5987 XXH3_kSecret, sizeof(XXH3_kSecret),
5988 f_acc, f_scramble);
5989 #endif
5990 { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE];
5991 f_initSec(secret, seed);
5992 return XXH3_hashLong_64b_internal(input, len, secret, sizeof(secret),
5993 f_acc, f_scramble);
5994 }
5995 }
5996
5997 /*
5998 * It's important for performance that XXH3_hashLong is not inlined.
5999 */
6000 XXH_NO_INLINE XXH64_hash_t
XXH3_hashLong_64b_withSeed(const void * XXH_RESTRICT input,size_t len,XXH64_hash_t seed,const xxh_u8 * XXH_RESTRICT secret,size_t secretLen)6001 XXH3_hashLong_64b_withSeed(const void* XXH_RESTRICT input, size_t len,
6002 XXH64_hash_t seed, const xxh_u8* XXH_RESTRICT secret, size_t secretLen)
6003 {
6004 (void)secret; (void)secretLen;
6005 return XXH3_hashLong_64b_withSeed_internal(input, len, seed,
6006 XXH3_accumulate, XXH3_scrambleAcc, XXH3_initCustomSecret);
6007 }
6008
6009
6010 typedef XXH64_hash_t (*XXH3_hashLong64_f)(const void* XXH_RESTRICT, size_t,
6011 XXH64_hash_t, const xxh_u8* XXH_RESTRICT, size_t);
6012
6013 XXH_FORCE_INLINE XXH64_hash_t
XXH3_64bits_internal(const void * XXH_RESTRICT input,size_t len,XXH64_hash_t seed64,const void * XXH_RESTRICT secret,size_t secretLen,XXH3_hashLong64_f f_hashLong)6014 XXH3_64bits_internal(const void* XXH_RESTRICT input, size_t len,
6015 XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen,
6016 XXH3_hashLong64_f f_hashLong)
6017 {
6018 XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN);
6019 /*
6020 * If an action is to be taken if `secretLen` condition is not respected,
6021 * it should be done here.
6022 * For now, it's a contract pre-condition.
6023 * Adding a check and a branch here would cost performance at every hash.
6024 * Also, note that function signature doesn't offer room to return an error.
6025 */
6026 if (len <= 16)
6027 return XXH3_len_0to16_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64);
6028 if (len <= 128)
6029 return XXH3_len_17to128_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64);
6030 if (len <= XXH3_MIDSIZE_MAX)
6031 return XXH3_len_129to240_64b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64);
6032 return f_hashLong(input, len, seed64, (const xxh_u8*)secret, secretLen);
6033 }
6034
6035
6036 /* === Public entry point === */
6037
6038 /*! @ingroup XXH3_family */
XXH3_64bits(XXH_NOESCAPE const void * input,size_t length)6039 XXH_PUBLIC_API XXH64_hash_t XXH3_64bits(XXH_NOESCAPE const void* input, size_t length)
6040 {
6041 return XXH3_64bits_internal(input, length, 0, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_default);
6042 }
6043
6044 /*! @ingroup XXH3_family */
6045 XXH_PUBLIC_API XXH64_hash_t
XXH3_64bits_withSecret(XXH_NOESCAPE const void * input,size_t length,XXH_NOESCAPE const void * secret,size_t secretSize)6046 XXH3_64bits_withSecret(XXH_NOESCAPE const void* input, size_t length, XXH_NOESCAPE const void* secret, size_t secretSize)
6047 {
6048 return XXH3_64bits_internal(input, length, 0, secret, secretSize, XXH3_hashLong_64b_withSecret);
6049 }
6050
6051 /*! @ingroup XXH3_family */
6052 XXH_PUBLIC_API XXH64_hash_t
XXH3_64bits_withSeed(XXH_NOESCAPE const void * input,size_t length,XXH64_hash_t seed)6053 XXH3_64bits_withSeed(XXH_NOESCAPE const void* input, size_t length, XXH64_hash_t seed)
6054 {
6055 return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, sizeof(XXH3_kSecret), XXH3_hashLong_64b_withSeed);
6056 }
6057
6058 XXH_PUBLIC_API XXH64_hash_t
XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void * input,size_t length,XXH_NOESCAPE const void * secret,size_t secretSize,XXH64_hash_t seed)6059 XXH3_64bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t length, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed)
6060 {
6061 if (length <= XXH3_MIDSIZE_MAX)
6062 return XXH3_64bits_internal(input, length, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL);
6063 return XXH3_hashLong_64b_withSecret(input, length, seed, (const xxh_u8*)secret, secretSize);
6064 }
6065
6066
6067 /* === XXH3 streaming === */
6068 #ifndef XXH_NO_STREAM
6069 /*
6070 * Malloc's a pointer that is always aligned to align.
6071 *
6072 * This must be freed with `XXH_alignedFree()`.
6073 *
6074 * malloc typically guarantees 16 byte alignment on 64-bit systems and 8 byte
6075 * alignment on 32-bit. This isn't enough for the 32 byte aligned loads in AVX2
6076 * or on 32-bit, the 16 byte aligned loads in SSE2 and NEON.
6077 *
6078 * This underalignment previously caused a rather obvious crash which went
6079 * completely unnoticed due to XXH3_createState() not actually being tested.
6080 * Credit to RedSpah for noticing this bug.
6081 *
6082 * The alignment is done manually: Functions like posix_memalign or _mm_malloc
6083 * are avoided: To maintain portability, we would have to write a fallback
6084 * like this anyways, and besides, testing for the existence of library
6085 * functions without relying on external build tools is impossible.
6086 *
6087 * The method is simple: Overallocate, manually align, and store the offset
6088 * to the original behind the returned pointer.
6089 *
6090 * Align must be a power of 2 and 8 <= align <= 128.
6091 */
XXH_alignedMalloc(size_t s,size_t align)6092 static XXH_MALLOCF void* XXH_alignedMalloc(size_t s, size_t align)
6093 {
6094 XXH_ASSERT(align <= 128 && align >= 8); /* range check */
6095 XXH_ASSERT((align & (align-1)) == 0); /* power of 2 */
6096 XXH_ASSERT(s != 0 && s < (s + align)); /* empty/overflow */
6097 { /* Overallocate to make room for manual realignment and an offset byte */
6098 xxh_u8* base = (xxh_u8*)XXH_malloc(s + align);
6099 if (base != NULL) {
6100 /*
6101 * Get the offset needed to align this pointer.
6102 *
6103 * Even if the returned pointer is aligned, there will always be
6104 * at least one byte to store the offset to the original pointer.
6105 */
6106 size_t offset = align - ((size_t)base & (align - 1)); /* base % align */
6107 /* Add the offset for the now-aligned pointer */
6108 xxh_u8* ptr = base + offset;
6109
6110 XXH_ASSERT((size_t)ptr % align == 0);
6111
6112 /* Store the offset immediately before the returned pointer. */
6113 ptr[-1] = (xxh_u8)offset;
6114 return ptr;
6115 }
6116 return NULL;
6117 }
6118 }
6119 /*
6120 * Frees an aligned pointer allocated by XXH_alignedMalloc(). Don't pass
6121 * normal malloc'd pointers, XXH_alignedMalloc has a specific data layout.
6122 */
XXH_alignedFree(void * p)6123 static void XXH_alignedFree(void* p)
6124 {
6125 if (p != NULL) {
6126 xxh_u8* ptr = (xxh_u8*)p;
6127 /* Get the offset byte we added in XXH_malloc. */
6128 xxh_u8 offset = ptr[-1];
6129 /* Free the original malloc'd pointer */
6130 xxh_u8* base = ptr - offset;
6131 XXH_free(base);
6132 }
6133 }
6134 /*! @ingroup XXH3_family */
6135 /*!
6136 * @brief Allocate an @ref XXH3_state_t.
6137 *
6138 * @return An allocated pointer of @ref XXH3_state_t on success.
6139 * @return `NULL` on failure.
6140 *
6141 * @note Must be freed with XXH3_freeState().
6142 */
XXH3_createState(void)6143 XXH_PUBLIC_API XXH3_state_t* XXH3_createState(void)
6144 {
6145 XXH3_state_t* const state = (XXH3_state_t*)XXH_alignedMalloc(sizeof(XXH3_state_t), 64);
6146 if (state==NULL) return NULL;
6147 XXH3_INITSTATE(state);
6148 return state;
6149 }
6150
6151 /*! @ingroup XXH3_family */
6152 /*!
6153 * @brief Frees an @ref XXH3_state_t.
6154 *
6155 * @param statePtr A pointer to an @ref XXH3_state_t allocated with @ref XXH3_createState().
6156 *
6157 * @return @ref XXH_OK.
6158 *
6159 * @note Must be allocated with XXH3_createState().
6160 */
XXH3_freeState(XXH3_state_t * statePtr)6161 XXH_PUBLIC_API XXH_errorcode XXH3_freeState(XXH3_state_t* statePtr)
6162 {
6163 XXH_alignedFree(statePtr);
6164 return XXH_OK;
6165 }
6166
6167 /*! @ingroup XXH3_family */
6168 XXH_PUBLIC_API void
XXH3_copyState(XXH_NOESCAPE XXH3_state_t * dst_state,XXH_NOESCAPE const XXH3_state_t * src_state)6169 XXH3_copyState(XXH_NOESCAPE XXH3_state_t* dst_state, XXH_NOESCAPE const XXH3_state_t* src_state)
6170 {
6171 XXH_memcpy(dst_state, src_state, sizeof(*dst_state));
6172 }
6173
6174 static void
XXH3_reset_internal(XXH3_state_t * statePtr,XXH64_hash_t seed,const void * secret,size_t secretSize)6175 XXH3_reset_internal(XXH3_state_t* statePtr,
6176 XXH64_hash_t seed,
6177 const void* secret, size_t secretSize)
6178 {
6179 size_t const initStart = offsetof(XXH3_state_t, bufferedSize);
6180 size_t const initLength = offsetof(XXH3_state_t, nbStripesPerBlock) - initStart;
6181 XXH_ASSERT(offsetof(XXH3_state_t, nbStripesPerBlock) > initStart);
6182 XXH_ASSERT(statePtr != NULL);
6183 /* set members from bufferedSize to nbStripesPerBlock (excluded) to 0 */
6184 memset((char*)statePtr + initStart, 0, initLength);
6185 statePtr->acc[0] = XXH_PRIME32_3;
6186 statePtr->acc[1] = XXH_PRIME64_1;
6187 statePtr->acc[2] = XXH_PRIME64_2;
6188 statePtr->acc[3] = XXH_PRIME64_3;
6189 statePtr->acc[4] = XXH_PRIME64_4;
6190 statePtr->acc[5] = XXH_PRIME32_2;
6191 statePtr->acc[6] = XXH_PRIME64_5;
6192 statePtr->acc[7] = XXH_PRIME32_1;
6193 statePtr->seed = seed;
6194 statePtr->useSeed = (seed != 0);
6195 statePtr->extSecret = (const unsigned char*)secret;
6196 XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN);
6197 statePtr->secretLimit = secretSize - XXH_STRIPE_LEN;
6198 statePtr->nbStripesPerBlock = statePtr->secretLimit / XXH_SECRET_CONSUME_RATE;
6199 }
6200
6201 /*! @ingroup XXH3_family */
6202 XXH_PUBLIC_API XXH_errorcode
XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t * statePtr)6203 XXH3_64bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr)
6204 {
6205 if (statePtr == NULL) return XXH_ERROR;
6206 XXH3_reset_internal(statePtr, 0, XXH3_kSecret, XXH_SECRET_DEFAULT_SIZE);
6207 return XXH_OK;
6208 }
6209
6210 /*! @ingroup XXH3_family */
6211 XXH_PUBLIC_API XXH_errorcode
XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t * statePtr,XXH_NOESCAPE const void * secret,size_t secretSize)6212 XXH3_64bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize)
6213 {
6214 if (statePtr == NULL) return XXH_ERROR;
6215 XXH3_reset_internal(statePtr, 0, secret, secretSize);
6216 if (secret == NULL) return XXH_ERROR;
6217 if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR;
6218 return XXH_OK;
6219 }
6220
6221 /*! @ingroup XXH3_family */
6222 XXH_PUBLIC_API XXH_errorcode
XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t * statePtr,XXH64_hash_t seed)6223 XXH3_64bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed)
6224 {
6225 if (statePtr == NULL) return XXH_ERROR;
6226 if (seed==0) return XXH3_64bits_reset(statePtr);
6227 if ((seed != statePtr->seed) || (statePtr->extSecret != NULL))
6228 XXH3_initCustomSecret(statePtr->customSecret, seed);
6229 XXH3_reset_internal(statePtr, seed, NULL, XXH_SECRET_DEFAULT_SIZE);
6230 return XXH_OK;
6231 }
6232
6233 /*! @ingroup XXH3_family */
6234 XXH_PUBLIC_API XXH_errorcode
XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t * statePtr,XXH_NOESCAPE const void * secret,size_t secretSize,XXH64_hash_t seed64)6235 XXH3_64bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed64)
6236 {
6237 if (statePtr == NULL) return XXH_ERROR;
6238 if (secret == NULL) return XXH_ERROR;
6239 if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR;
6240 XXH3_reset_internal(statePtr, seed64, secret, secretSize);
6241 statePtr->useSeed = 1; /* always, even if seed64==0 */
6242 return XXH_OK;
6243 }
6244
6245 /*!
6246 * @internal
6247 * @brief Processes a large input for XXH3_update() and XXH3_digest_long().
6248 *
6249 * Unlike XXH3_hashLong_internal_loop(), this can process data that overlaps a block.
6250 *
6251 * @param acc Pointer to the 8 accumulator lanes
6252 * @param nbStripesSoFarPtr In/out pointer to the number of leftover stripes in the block*
6253 * @param nbStripesPerBlock Number of stripes in a block
6254 * @param input Input pointer
6255 * @param nbStripes Number of stripes to process
6256 * @param secret Secret pointer
6257 * @param secretLimit Offset of the last block in @p secret
6258 * @param f_acc Pointer to an XXH3_accumulate implementation
6259 * @param f_scramble Pointer to an XXH3_scrambleAcc implementation
6260 * @return Pointer past the end of @p input after processing
6261 */
6262 XXH_FORCE_INLINE const xxh_u8 *
XXH3_consumeStripes(xxh_u64 * XXH_RESTRICT acc,size_t * XXH_RESTRICT nbStripesSoFarPtr,size_t nbStripesPerBlock,const xxh_u8 * XXH_RESTRICT input,size_t nbStripes,const xxh_u8 * XXH_RESTRICT secret,size_t secretLimit,XXH3_f_accumulate f_acc,XXH3_f_scrambleAcc f_scramble)6263 XXH3_consumeStripes(xxh_u64* XXH_RESTRICT acc,
6264 size_t* XXH_RESTRICT nbStripesSoFarPtr, size_t nbStripesPerBlock,
6265 const xxh_u8* XXH_RESTRICT input, size_t nbStripes,
6266 const xxh_u8* XXH_RESTRICT secret, size_t secretLimit,
6267 XXH3_f_accumulate f_acc,
6268 XXH3_f_scrambleAcc f_scramble)
6269 {
6270 const xxh_u8* initialSecret = secret + *nbStripesSoFarPtr * XXH_SECRET_CONSUME_RATE;
6271 /* Process full blocks */
6272 if (nbStripes >= (nbStripesPerBlock - *nbStripesSoFarPtr)) {
6273 /* Process the initial partial block... */
6274 size_t nbStripesThisIter = nbStripesPerBlock - *nbStripesSoFarPtr;
6275
6276 do {
6277 /* Accumulate and scramble */
6278 f_acc(acc, input, initialSecret, nbStripesThisIter);
6279 f_scramble(acc, secret + secretLimit);
6280 input += nbStripesThisIter * XXH_STRIPE_LEN;
6281 nbStripes -= nbStripesThisIter;
6282 /* Then continue the loop with the full block size */
6283 nbStripesThisIter = nbStripesPerBlock;
6284 initialSecret = secret;
6285 } while (nbStripes >= nbStripesPerBlock);
6286 *nbStripesSoFarPtr = 0;
6287 }
6288 /* Process a partial block */
6289 if (nbStripes > 0) {
6290 f_acc(acc, input, initialSecret, nbStripes);
6291 input += nbStripes * XXH_STRIPE_LEN;
6292 *nbStripesSoFarPtr += nbStripes;
6293 }
6294 /* Return end pointer */
6295 return input;
6296 }
6297
6298 #ifndef XXH3_STREAM_USE_STACK
6299 # if XXH_SIZE_OPT <= 0 && !defined(__clang__) /* clang doesn't need additional stack space */
6300 # define XXH3_STREAM_USE_STACK 1
6301 # endif
6302 #endif
6303 /*
6304 * Both XXH3_64bits_update and XXH3_128bits_update use this routine.
6305 */
6306 XXH_FORCE_INLINE XXH_errorcode
XXH3_update(XXH3_state_t * XXH_RESTRICT const state,const xxh_u8 * XXH_RESTRICT input,size_t len,XXH3_f_accumulate f_acc,XXH3_f_scrambleAcc f_scramble)6307 XXH3_update(XXH3_state_t* XXH_RESTRICT const state,
6308 const xxh_u8* XXH_RESTRICT input, size_t len,
6309 XXH3_f_accumulate f_acc,
6310 XXH3_f_scrambleAcc f_scramble)
6311 {
6312 if (input==NULL) {
6313 XXH_ASSERT(len == 0);
6314 return XXH_OK;
6315 }
6316
6317 XXH_ASSERT(state != NULL);
6318 { const xxh_u8* const bEnd = input + len;
6319 const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret;
6320 #if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1
6321 /* For some reason, gcc and MSVC seem to suffer greatly
6322 * when operating accumulators directly into state.
6323 * Operating into stack space seems to enable proper optimization.
6324 * clang, on the other hand, doesn't seem to need this trick */
6325 XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[8];
6326 XXH_memcpy(acc, state->acc, sizeof(acc));
6327 #else
6328 xxh_u64* XXH_RESTRICT const acc = state->acc;
6329 #endif
6330 state->totalLen += len;
6331 XXH_ASSERT(state->bufferedSize <= XXH3_INTERNALBUFFER_SIZE);
6332
6333 /* small input : just fill in tmp buffer */
6334 if (len <= XXH3_INTERNALBUFFER_SIZE - state->bufferedSize) {
6335 XXH_memcpy(state->buffer + state->bufferedSize, input, len);
6336 state->bufferedSize += (XXH32_hash_t)len;
6337 return XXH_OK;
6338 }
6339
6340 /* total input is now > XXH3_INTERNALBUFFER_SIZE */
6341 #define XXH3_INTERNALBUFFER_STRIPES (XXH3_INTERNALBUFFER_SIZE / XXH_STRIPE_LEN)
6342 XXH_STATIC_ASSERT(XXH3_INTERNALBUFFER_SIZE % XXH_STRIPE_LEN == 0); /* clean multiple */
6343
6344 /*
6345 * Internal buffer is partially filled (always, except at beginning)
6346 * Complete it, then consume it.
6347 */
6348 if (state->bufferedSize) {
6349 size_t const loadSize = XXH3_INTERNALBUFFER_SIZE - state->bufferedSize;
6350 XXH_memcpy(state->buffer + state->bufferedSize, input, loadSize);
6351 input += loadSize;
6352 XXH3_consumeStripes(acc,
6353 &state->nbStripesSoFar, state->nbStripesPerBlock,
6354 state->buffer, XXH3_INTERNALBUFFER_STRIPES,
6355 secret, state->secretLimit,
6356 f_acc, f_scramble);
6357 state->bufferedSize = 0;
6358 }
6359 XXH_ASSERT(input < bEnd);
6360 if (bEnd - input > XXH3_INTERNALBUFFER_SIZE) {
6361 size_t nbStripes = (size_t)(bEnd - 1 - input) / XXH_STRIPE_LEN;
6362 input = XXH3_consumeStripes(acc,
6363 &state->nbStripesSoFar, state->nbStripesPerBlock,
6364 input, nbStripes,
6365 secret, state->secretLimit,
6366 f_acc, f_scramble);
6367 XXH_memcpy(state->buffer + sizeof(state->buffer) - XXH_STRIPE_LEN, input - XXH_STRIPE_LEN, XXH_STRIPE_LEN);
6368
6369 }
6370 /* Some remaining input (always) : buffer it */
6371 XXH_ASSERT(input < bEnd);
6372 XXH_ASSERT(bEnd - input <= XXH3_INTERNALBUFFER_SIZE);
6373 XXH_ASSERT(state->bufferedSize == 0);
6374 XXH_memcpy(state->buffer, input, (size_t)(bEnd-input));
6375 state->bufferedSize = (XXH32_hash_t)(bEnd-input);
6376 #if defined(XXH3_STREAM_USE_STACK) && XXH3_STREAM_USE_STACK >= 1
6377 /* save stack accumulators into state */
6378 XXH_memcpy(state->acc, acc, sizeof(acc));
6379 #endif
6380 }
6381
6382 return XXH_OK;
6383 }
6384
6385 /*! @ingroup XXH3_family */
6386 XXH_PUBLIC_API XXH_errorcode
XXH3_64bits_update(XXH_NOESCAPE XXH3_state_t * state,XXH_NOESCAPE const void * input,size_t len)6387 XXH3_64bits_update(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len)
6388 {
6389 return XXH3_update(state, (const xxh_u8*)input, len,
6390 XXH3_accumulate, XXH3_scrambleAcc);
6391 }
6392
6393
6394 XXH_FORCE_INLINE void
XXH3_digest_long(XXH64_hash_t * acc,const XXH3_state_t * state,const unsigned char * secret)6395 XXH3_digest_long (XXH64_hash_t* acc,
6396 const XXH3_state_t* state,
6397 const unsigned char* secret)
6398 {
6399 xxh_u8 lastStripe[XXH_STRIPE_LEN];
6400 const xxh_u8* lastStripePtr;
6401
6402 /*
6403 * Digest on a local copy. This way, the state remains unaltered, and it can
6404 * continue ingesting more input afterwards.
6405 */
6406 XXH_memcpy(acc, state->acc, sizeof(state->acc));
6407 if (state->bufferedSize >= XXH_STRIPE_LEN) {
6408 /* Consume remaining stripes then point to remaining data in buffer */
6409 size_t const nbStripes = (state->bufferedSize - 1) / XXH_STRIPE_LEN;
6410 size_t nbStripesSoFar = state->nbStripesSoFar;
6411 XXH3_consumeStripes(acc,
6412 &nbStripesSoFar, state->nbStripesPerBlock,
6413 state->buffer, nbStripes,
6414 secret, state->secretLimit,
6415 XXH3_accumulate, XXH3_scrambleAcc);
6416 lastStripePtr = state->buffer + state->bufferedSize - XXH_STRIPE_LEN;
6417 } else { /* bufferedSize < XXH_STRIPE_LEN */
6418 /* Copy to temp buffer */
6419 size_t const catchupSize = XXH_STRIPE_LEN - state->bufferedSize;
6420 XXH_ASSERT(state->bufferedSize > 0); /* there is always some input buffered */
6421 XXH_memcpy(lastStripe, state->buffer + sizeof(state->buffer) - catchupSize, catchupSize);
6422 XXH_memcpy(lastStripe + catchupSize, state->buffer, state->bufferedSize);
6423 lastStripePtr = lastStripe;
6424 }
6425 /* Last stripe */
6426 XXH3_accumulate_512(acc,
6427 lastStripePtr,
6428 secret + state->secretLimit - XXH_SECRET_LASTACC_START);
6429 }
6430
6431 /*! @ingroup XXH3_family */
XXH3_64bits_digest(XXH_NOESCAPE const XXH3_state_t * state)6432 XXH_PUBLIC_API XXH64_hash_t XXH3_64bits_digest (XXH_NOESCAPE const XXH3_state_t* state)
6433 {
6434 const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret;
6435 if (state->totalLen > XXH3_MIDSIZE_MAX) {
6436 XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB];
6437 XXH3_digest_long(acc, state, secret);
6438 return XXH3_mergeAccs(acc,
6439 secret + XXH_SECRET_MERGEACCS_START,
6440 (xxh_u64)state->totalLen * XXH_PRIME64_1);
6441 }
6442 /* totalLen <= XXH3_MIDSIZE_MAX: digesting a short input */
6443 if (state->useSeed)
6444 return XXH3_64bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed);
6445 return XXH3_64bits_withSecret(state->buffer, (size_t)(state->totalLen),
6446 secret, state->secretLimit + XXH_STRIPE_LEN);
6447 }
6448 #endif /* !XXH_NO_STREAM */
6449
6450
6451 /* ==========================================
6452 * XXH3 128 bits (a.k.a XXH128)
6453 * ==========================================
6454 * XXH3's 128-bit variant has better mixing and strength than the 64-bit variant,
6455 * even without counting the significantly larger output size.
6456 *
6457 * For example, extra steps are taken to avoid the seed-dependent collisions
6458 * in 17-240 byte inputs (See XXH3_mix16B and XXH128_mix32B).
6459 *
6460 * This strength naturally comes at the cost of some speed, especially on short
6461 * lengths. Note that longer hashes are about as fast as the 64-bit version
6462 * due to it using only a slight modification of the 64-bit loop.
6463 *
6464 * XXH128 is also more oriented towards 64-bit machines. It is still extremely
6465 * fast for a _128-bit_ hash on 32-bit (it usually clears XXH64).
6466 */
6467
6468 XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t
XXH3_len_1to3_128b(const xxh_u8 * input,size_t len,const xxh_u8 * secret,XXH64_hash_t seed)6469 XXH3_len_1to3_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
6470 {
6471 /* A doubled version of 1to3_64b with different constants. */
6472 XXH_ASSERT(input != NULL);
6473 XXH_ASSERT(1 <= len && len <= 3);
6474 XXH_ASSERT(secret != NULL);
6475 /*
6476 * len = 1: combinedl = { input[0], 0x01, input[0], input[0] }
6477 * len = 2: combinedl = { input[1], 0x02, input[0], input[1] }
6478 * len = 3: combinedl = { input[2], 0x03, input[0], input[1] }
6479 */
6480 { xxh_u8 const c1 = input[0];
6481 xxh_u8 const c2 = input[len >> 1];
6482 xxh_u8 const c3 = input[len - 1];
6483 xxh_u32 const combinedl = ((xxh_u32)c1 <<16) | ((xxh_u32)c2 << 24)
6484 | ((xxh_u32)c3 << 0) | ((xxh_u32)len << 8);
6485 xxh_u32 const combinedh = XXH_rotl32(XXH_swap32(combinedl), 13);
6486 xxh_u64 const bitflipl = (XXH_readLE32(secret) ^ XXH_readLE32(secret+4)) + seed;
6487 xxh_u64 const bitfliph = (XXH_readLE32(secret+8) ^ XXH_readLE32(secret+12)) - seed;
6488 xxh_u64 const keyed_lo = (xxh_u64)combinedl ^ bitflipl;
6489 xxh_u64 const keyed_hi = (xxh_u64)combinedh ^ bitfliph;
6490 XXH128_hash_t h128;
6491 h128.low64 = XXH64_avalanche(keyed_lo);
6492 h128.high64 = XXH64_avalanche(keyed_hi);
6493 return h128;
6494 }
6495 }
6496
6497 XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t
XXH3_len_4to8_128b(const xxh_u8 * input,size_t len,const xxh_u8 * secret,XXH64_hash_t seed)6498 XXH3_len_4to8_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
6499 {
6500 XXH_ASSERT(input != NULL);
6501 XXH_ASSERT(secret != NULL);
6502 XXH_ASSERT(4 <= len && len <= 8);
6503 seed ^= (xxh_u64)XXH_swap32((xxh_u32)seed) << 32;
6504 { xxh_u32 const input_lo = XXH_readLE32(input);
6505 xxh_u32 const input_hi = XXH_readLE32(input + len - 4);
6506 xxh_u64 const input_64 = input_lo + ((xxh_u64)input_hi << 32);
6507 xxh_u64 const bitflip = (XXH_readLE64(secret+16) ^ XXH_readLE64(secret+24)) + seed;
6508 xxh_u64 const keyed = input_64 ^ bitflip;
6509
6510 /* Shift len to the left to ensure it is even, this avoids even multiplies. */
6511 XXH128_hash_t m128 = XXH_mult64to128(keyed, XXH_PRIME64_1 + (len << 2));
6512
6513 m128.high64 += (m128.low64 << 1);
6514 m128.low64 ^= (m128.high64 >> 3);
6515
6516 m128.low64 = XXH_xorshift64(m128.low64, 35);
6517 m128.low64 *= PRIME_MX2;
6518 m128.low64 = XXH_xorshift64(m128.low64, 28);
6519 m128.high64 = XXH3_avalanche(m128.high64);
6520 return m128;
6521 }
6522 }
6523
6524 XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t
XXH3_len_9to16_128b(const xxh_u8 * input,size_t len,const xxh_u8 * secret,XXH64_hash_t seed)6525 XXH3_len_9to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
6526 {
6527 XXH_ASSERT(input != NULL);
6528 XXH_ASSERT(secret != NULL);
6529 XXH_ASSERT(9 <= len && len <= 16);
6530 { xxh_u64 const bitflipl = (XXH_readLE64(secret+32) ^ XXH_readLE64(secret+40)) - seed;
6531 xxh_u64 const bitfliph = (XXH_readLE64(secret+48) ^ XXH_readLE64(secret+56)) + seed;
6532 xxh_u64 const input_lo = XXH_readLE64(input);
6533 xxh_u64 input_hi = XXH_readLE64(input + len - 8);
6534 XXH128_hash_t m128 = XXH_mult64to128(input_lo ^ input_hi ^ bitflipl, XXH_PRIME64_1);
6535 /*
6536 * Put len in the middle of m128 to ensure that the length gets mixed to
6537 * both the low and high bits in the 128x64 multiply below.
6538 */
6539 m128.low64 += (xxh_u64)(len - 1) << 54;
6540 input_hi ^= bitfliph;
6541 /*
6542 * Add the high 32 bits of input_hi to the high 32 bits of m128, then
6543 * add the long product of the low 32 bits of input_hi and XXH_PRIME32_2 to
6544 * the high 64 bits of m128.
6545 *
6546 * The best approach to this operation is different on 32-bit and 64-bit.
6547 */
6548 if (sizeof(void *) < sizeof(xxh_u64)) { /* 32-bit */
6549 /*
6550 * 32-bit optimized version, which is more readable.
6551 *
6552 * On 32-bit, it removes an ADC and delays a dependency between the two
6553 * halves of m128.high64, but it generates an extra mask on 64-bit.
6554 */
6555 m128.high64 += (input_hi & 0xFFFFFFFF00000000ULL) + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2);
6556 } else {
6557 /*
6558 * 64-bit optimized (albeit more confusing) version.
6559 *
6560 * Uses some properties of addition and multiplication to remove the mask:
6561 *
6562 * Let:
6563 * a = input_hi.lo = (input_hi & 0x00000000FFFFFFFF)
6564 * b = input_hi.hi = (input_hi & 0xFFFFFFFF00000000)
6565 * c = XXH_PRIME32_2
6566 *
6567 * a + (b * c)
6568 * Inverse Property: x + y - x == y
6569 * a + (b * (1 + c - 1))
6570 * Distributive Property: x * (y + z) == (x * y) + (x * z)
6571 * a + (b * 1) + (b * (c - 1))
6572 * Identity Property: x * 1 == x
6573 * a + b + (b * (c - 1))
6574 *
6575 * Substitute a, b, and c:
6576 * input_hi.hi + input_hi.lo + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1))
6577 *
6578 * Since input_hi.hi + input_hi.lo == input_hi, we get this:
6579 * input_hi + ((xxh_u64)input_hi.lo * (XXH_PRIME32_2 - 1))
6580 */
6581 m128.high64 += input_hi + XXH_mult32to64((xxh_u32)input_hi, XXH_PRIME32_2 - 1);
6582 }
6583 /* m128 ^= XXH_swap64(m128 >> 64); */
6584 m128.low64 ^= XXH_swap64(m128.high64);
6585
6586 { /* 128x64 multiply: h128 = m128 * XXH_PRIME64_2; */
6587 XXH128_hash_t h128 = XXH_mult64to128(m128.low64, XXH_PRIME64_2);
6588 h128.high64 += m128.high64 * XXH_PRIME64_2;
6589
6590 h128.low64 = XXH3_avalanche(h128.low64);
6591 h128.high64 = XXH3_avalanche(h128.high64);
6592 return h128;
6593 } }
6594 }
6595
6596 /*
6597 * Assumption: `secret` size is >= XXH3_SECRET_SIZE_MIN
6598 */
6599 XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t
XXH3_len_0to16_128b(const xxh_u8 * input,size_t len,const xxh_u8 * secret,XXH64_hash_t seed)6600 XXH3_len_0to16_128b(const xxh_u8* input, size_t len, const xxh_u8* secret, XXH64_hash_t seed)
6601 {
6602 XXH_ASSERT(len <= 16);
6603 { if (len > 8) return XXH3_len_9to16_128b(input, len, secret, seed);
6604 if (len >= 4) return XXH3_len_4to8_128b(input, len, secret, seed);
6605 if (len) return XXH3_len_1to3_128b(input, len, secret, seed);
6606 { XXH128_hash_t h128;
6607 xxh_u64 const bitflipl = XXH_readLE64(secret+64) ^ XXH_readLE64(secret+72);
6608 xxh_u64 const bitfliph = XXH_readLE64(secret+80) ^ XXH_readLE64(secret+88);
6609 h128.low64 = XXH64_avalanche(seed ^ bitflipl);
6610 h128.high64 = XXH64_avalanche( seed ^ bitfliph);
6611 return h128;
6612 } }
6613 }
6614
6615 /*
6616 * A bit slower than XXH3_mix16B, but handles multiply by zero better.
6617 */
6618 XXH_FORCE_INLINE XXH128_hash_t
XXH128_mix32B(XXH128_hash_t acc,const xxh_u8 * input_1,const xxh_u8 * input_2,const xxh_u8 * secret,XXH64_hash_t seed)6619 XXH128_mix32B(XXH128_hash_t acc, const xxh_u8* input_1, const xxh_u8* input_2,
6620 const xxh_u8* secret, XXH64_hash_t seed)
6621 {
6622 acc.low64 += XXH3_mix16B (input_1, secret+0, seed);
6623 acc.low64 ^= XXH_readLE64(input_2) + XXH_readLE64(input_2 + 8);
6624 acc.high64 += XXH3_mix16B (input_2, secret+16, seed);
6625 acc.high64 ^= XXH_readLE64(input_1) + XXH_readLE64(input_1 + 8);
6626 return acc;
6627 }
6628
6629
6630 XXH_FORCE_INLINE XXH_PUREF XXH128_hash_t
XXH3_len_17to128_128b(const xxh_u8 * XXH_RESTRICT input,size_t len,const xxh_u8 * XXH_RESTRICT secret,size_t secretSize,XXH64_hash_t seed)6631 XXH3_len_17to128_128b(const xxh_u8* XXH_RESTRICT input, size_t len,
6632 const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
6633 XXH64_hash_t seed)
6634 {
6635 XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
6636 XXH_ASSERT(16 < len && len <= 128);
6637
6638 { XXH128_hash_t acc;
6639 acc.low64 = len * XXH_PRIME64_1;
6640 acc.high64 = 0;
6641
6642 #if XXH_SIZE_OPT >= 1
6643 {
6644 /* Smaller, but slightly slower. */
6645 unsigned int i = (unsigned int)(len - 1) / 32;
6646 do {
6647 acc = XXH128_mix32B(acc, input+16*i, input+len-16*(i+1), secret+32*i, seed);
6648 } while (i-- != 0);
6649 }
6650 #else
6651 if (len > 32) {
6652 if (len > 64) {
6653 if (len > 96) {
6654 acc = XXH128_mix32B(acc, input+48, input+len-64, secret+96, seed);
6655 }
6656 acc = XXH128_mix32B(acc, input+32, input+len-48, secret+64, seed);
6657 }
6658 acc = XXH128_mix32B(acc, input+16, input+len-32, secret+32, seed);
6659 }
6660 acc = XXH128_mix32B(acc, input, input+len-16, secret, seed);
6661 #endif
6662 { XXH128_hash_t h128;
6663 h128.low64 = acc.low64 + acc.high64;
6664 h128.high64 = (acc.low64 * XXH_PRIME64_1)
6665 + (acc.high64 * XXH_PRIME64_4)
6666 + ((len - seed) * XXH_PRIME64_2);
6667 h128.low64 = XXH3_avalanche(h128.low64);
6668 h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64);
6669 return h128;
6670 }
6671 }
6672 }
6673
6674 XXH_NO_INLINE XXH_PUREF XXH128_hash_t
XXH3_len_129to240_128b(const xxh_u8 * XXH_RESTRICT input,size_t len,const xxh_u8 * XXH_RESTRICT secret,size_t secretSize,XXH64_hash_t seed)6675 XXH3_len_129to240_128b(const xxh_u8* XXH_RESTRICT input, size_t len,
6676 const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
6677 XXH64_hash_t seed)
6678 {
6679 XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN); (void)secretSize;
6680 XXH_ASSERT(128 < len && len <= XXH3_MIDSIZE_MAX);
6681
6682 { XXH128_hash_t acc;
6683 unsigned i;
6684 acc.low64 = len * XXH_PRIME64_1;
6685 acc.high64 = 0;
6686 /*
6687 * We set as `i` as offset + 32. We do this so that unchanged
6688 * `len` can be used as upper bound. This reaches a sweet spot
6689 * where both x86 and aarch64 get simple agen and good codegen
6690 * for the loop.
6691 */
6692 for (i = 32; i < 160; i += 32) {
6693 acc = XXH128_mix32B(acc,
6694 input + i - 32,
6695 input + i - 16,
6696 secret + i - 32,
6697 seed);
6698 }
6699 acc.low64 = XXH3_avalanche(acc.low64);
6700 acc.high64 = XXH3_avalanche(acc.high64);
6701 /*
6702 * NB: `i <= len` will duplicate the last 32-bytes if
6703 * len % 32 was zero. This is an unfortunate necessity to keep
6704 * the hash result stable.
6705 */
6706 for (i=160; i <= len; i += 32) {
6707 acc = XXH128_mix32B(acc,
6708 input + i - 32,
6709 input + i - 16,
6710 secret + XXH3_MIDSIZE_STARTOFFSET + i - 160,
6711 seed);
6712 }
6713 /* last bytes */
6714 acc = XXH128_mix32B(acc,
6715 input + len - 16,
6716 input + len - 32,
6717 secret + XXH3_SECRET_SIZE_MIN - XXH3_MIDSIZE_LASTOFFSET - 16,
6718 (XXH64_hash_t)0 - seed);
6719
6720 { XXH128_hash_t h128;
6721 h128.low64 = acc.low64 + acc.high64;
6722 h128.high64 = (acc.low64 * XXH_PRIME64_1)
6723 + (acc.high64 * XXH_PRIME64_4)
6724 + ((len - seed) * XXH_PRIME64_2);
6725 h128.low64 = XXH3_avalanche(h128.low64);
6726 h128.high64 = (XXH64_hash_t)0 - XXH3_avalanche(h128.high64);
6727 return h128;
6728 }
6729 }
6730 }
6731
6732 XXH_FORCE_INLINE XXH128_hash_t
XXH3_hashLong_128b_internal(const void * XXH_RESTRICT input,size_t len,const xxh_u8 * XXH_RESTRICT secret,size_t secretSize,XXH3_f_accumulate f_acc,XXH3_f_scrambleAcc f_scramble)6733 XXH3_hashLong_128b_internal(const void* XXH_RESTRICT input, size_t len,
6734 const xxh_u8* XXH_RESTRICT secret, size_t secretSize,
6735 XXH3_f_accumulate f_acc,
6736 XXH3_f_scrambleAcc f_scramble)
6737 {
6738 XXH_ALIGN(XXH_ACC_ALIGN) xxh_u64 acc[XXH_ACC_NB] = XXH3_INIT_ACC;
6739
6740 XXH3_hashLong_internal_loop(acc, (const xxh_u8*)input, len, secret, secretSize, f_acc, f_scramble);
6741
6742 /* converge into final hash */
6743 XXH_STATIC_ASSERT(sizeof(acc) == 64);
6744 XXH_ASSERT(secretSize >= sizeof(acc) + XXH_SECRET_MERGEACCS_START);
6745 { XXH128_hash_t h128;
6746 h128.low64 = XXH3_mergeAccs(acc,
6747 secret + XXH_SECRET_MERGEACCS_START,
6748 (xxh_u64)len * XXH_PRIME64_1);
6749 h128.high64 = XXH3_mergeAccs(acc,
6750 secret + secretSize
6751 - sizeof(acc) - XXH_SECRET_MERGEACCS_START,
6752 ~((xxh_u64)len * XXH_PRIME64_2));
6753 return h128;
6754 }
6755 }
6756
6757 /*
6758 * It's important for performance that XXH3_hashLong() is not inlined.
6759 */
6760 XXH_NO_INLINE XXH_PUREF XXH128_hash_t
XXH3_hashLong_128b_default(const void * XXH_RESTRICT input,size_t len,XXH64_hash_t seed64,const void * XXH_RESTRICT secret,size_t secretLen)6761 XXH3_hashLong_128b_default(const void* XXH_RESTRICT input, size_t len,
6762 XXH64_hash_t seed64,
6763 const void* XXH_RESTRICT secret, size_t secretLen)
6764 {
6765 (void)seed64; (void)secret; (void)secretLen;
6766 return XXH3_hashLong_128b_internal(input, len, XXH3_kSecret, sizeof(XXH3_kSecret),
6767 XXH3_accumulate, XXH3_scrambleAcc);
6768 }
6769
6770 /*
6771 * It's important for performance to pass @p secretLen (when it's static)
6772 * to the compiler, so that it can properly optimize the vectorized loop.
6773 *
6774 * When the secret size is unknown, or on GCC 12 where the mix of NO_INLINE and FORCE_INLINE
6775 * breaks -Og, this is XXH_NO_INLINE.
6776 */
6777 XXH3_WITH_SECRET_INLINE XXH128_hash_t
XXH3_hashLong_128b_withSecret(const void * XXH_RESTRICT input,size_t len,XXH64_hash_t seed64,const void * XXH_RESTRICT secret,size_t secretLen)6778 XXH3_hashLong_128b_withSecret(const void* XXH_RESTRICT input, size_t len,
6779 XXH64_hash_t seed64,
6780 const void* XXH_RESTRICT secret, size_t secretLen)
6781 {
6782 (void)seed64;
6783 return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, secretLen,
6784 XXH3_accumulate, XXH3_scrambleAcc);
6785 }
6786
6787 XXH_FORCE_INLINE XXH128_hash_t
XXH3_hashLong_128b_withSeed_internal(const void * XXH_RESTRICT input,size_t len,XXH64_hash_t seed64,XXH3_f_accumulate f_acc,XXH3_f_scrambleAcc f_scramble,XXH3_f_initCustomSecret f_initSec)6788 XXH3_hashLong_128b_withSeed_internal(const void* XXH_RESTRICT input, size_t len,
6789 XXH64_hash_t seed64,
6790 XXH3_f_accumulate f_acc,
6791 XXH3_f_scrambleAcc f_scramble,
6792 XXH3_f_initCustomSecret f_initSec)
6793 {
6794 if (seed64 == 0)
6795 return XXH3_hashLong_128b_internal(input, len,
6796 XXH3_kSecret, sizeof(XXH3_kSecret),
6797 f_acc, f_scramble);
6798 { XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE];
6799 f_initSec(secret, seed64);
6800 return XXH3_hashLong_128b_internal(input, len, (const xxh_u8*)secret, sizeof(secret),
6801 f_acc, f_scramble);
6802 }
6803 }
6804
6805 /*
6806 * It's important for performance that XXH3_hashLong is not inlined.
6807 */
6808 XXH_NO_INLINE XXH128_hash_t
XXH3_hashLong_128b_withSeed(const void * input,size_t len,XXH64_hash_t seed64,const void * XXH_RESTRICT secret,size_t secretLen)6809 XXH3_hashLong_128b_withSeed(const void* input, size_t len,
6810 XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen)
6811 {
6812 (void)secret; (void)secretLen;
6813 return XXH3_hashLong_128b_withSeed_internal(input, len, seed64,
6814 XXH3_accumulate, XXH3_scrambleAcc, XXH3_initCustomSecret);
6815 }
6816
6817 typedef XXH128_hash_t (*XXH3_hashLong128_f)(const void* XXH_RESTRICT, size_t,
6818 XXH64_hash_t, const void* XXH_RESTRICT, size_t);
6819
6820 XXH_FORCE_INLINE XXH128_hash_t
XXH3_128bits_internal(const void * input,size_t len,XXH64_hash_t seed64,const void * XXH_RESTRICT secret,size_t secretLen,XXH3_hashLong128_f f_hl128)6821 XXH3_128bits_internal(const void* input, size_t len,
6822 XXH64_hash_t seed64, const void* XXH_RESTRICT secret, size_t secretLen,
6823 XXH3_hashLong128_f f_hl128)
6824 {
6825 XXH_ASSERT(secretLen >= XXH3_SECRET_SIZE_MIN);
6826 /*
6827 * If an action is to be taken if `secret` conditions are not respected,
6828 * it should be done here.
6829 * For now, it's a contract pre-condition.
6830 * Adding a check and a branch here would cost performance at every hash.
6831 */
6832 if (len <= 16)
6833 return XXH3_len_0to16_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, seed64);
6834 if (len <= 128)
6835 return XXH3_len_17to128_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64);
6836 if (len <= XXH3_MIDSIZE_MAX)
6837 return XXH3_len_129to240_128b((const xxh_u8*)input, len, (const xxh_u8*)secret, secretLen, seed64);
6838 return f_hl128(input, len, seed64, secret, secretLen);
6839 }
6840
6841
6842 /* === Public XXH128 API === */
6843
6844 /*! @ingroup XXH3_family */
XXH3_128bits(XXH_NOESCAPE const void * input,size_t len)6845 XXH_PUBLIC_API XXH128_hash_t XXH3_128bits(XXH_NOESCAPE const void* input, size_t len)
6846 {
6847 return XXH3_128bits_internal(input, len, 0,
6848 XXH3_kSecret, sizeof(XXH3_kSecret),
6849 XXH3_hashLong_128b_default);
6850 }
6851
6852 /*! @ingroup XXH3_family */
6853 XXH_PUBLIC_API XXH128_hash_t
XXH3_128bits_withSecret(XXH_NOESCAPE const void * input,size_t len,XXH_NOESCAPE const void * secret,size_t secretSize)6854 XXH3_128bits_withSecret(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize)
6855 {
6856 return XXH3_128bits_internal(input, len, 0,
6857 (const xxh_u8*)secret, secretSize,
6858 XXH3_hashLong_128b_withSecret);
6859 }
6860
6861 /*! @ingroup XXH3_family */
6862 XXH_PUBLIC_API XXH128_hash_t
XXH3_128bits_withSeed(XXH_NOESCAPE const void * input,size_t len,XXH64_hash_t seed)6863 XXH3_128bits_withSeed(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed)
6864 {
6865 return XXH3_128bits_internal(input, len, seed,
6866 XXH3_kSecret, sizeof(XXH3_kSecret),
6867 XXH3_hashLong_128b_withSeed);
6868 }
6869
6870 /*! @ingroup XXH3_family */
6871 XXH_PUBLIC_API XXH128_hash_t
XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void * input,size_t len,XXH_NOESCAPE const void * secret,size_t secretSize,XXH64_hash_t seed)6872 XXH3_128bits_withSecretandSeed(XXH_NOESCAPE const void* input, size_t len, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed)
6873 {
6874 if (len <= XXH3_MIDSIZE_MAX)
6875 return XXH3_128bits_internal(input, len, seed, XXH3_kSecret, sizeof(XXH3_kSecret), NULL);
6876 return XXH3_hashLong_128b_withSecret(input, len, seed, secret, secretSize);
6877 }
6878
6879 /*! @ingroup XXH3_family */
6880 XXH_PUBLIC_API XXH128_hash_t
XXH128(XXH_NOESCAPE const void * input,size_t len,XXH64_hash_t seed)6881 XXH128(XXH_NOESCAPE const void* input, size_t len, XXH64_hash_t seed)
6882 {
6883 return XXH3_128bits_withSeed(input, len, seed);
6884 }
6885
6886
6887 /* === XXH3 128-bit streaming === */
6888 #ifndef XXH_NO_STREAM
6889 /*
6890 * All initialization and update functions are identical to 64-bit streaming variant.
6891 * The only difference is the finalization routine.
6892 */
6893
6894 /*! @ingroup XXH3_family */
6895 XXH_PUBLIC_API XXH_errorcode
XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t * statePtr)6896 XXH3_128bits_reset(XXH_NOESCAPE XXH3_state_t* statePtr)
6897 {
6898 return XXH3_64bits_reset(statePtr);
6899 }
6900
6901 /*! @ingroup XXH3_family */
6902 XXH_PUBLIC_API XXH_errorcode
XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t * statePtr,XXH_NOESCAPE const void * secret,size_t secretSize)6903 XXH3_128bits_reset_withSecret(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize)
6904 {
6905 return XXH3_64bits_reset_withSecret(statePtr, secret, secretSize);
6906 }
6907
6908 /*! @ingroup XXH3_family */
6909 XXH_PUBLIC_API XXH_errorcode
XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t * statePtr,XXH64_hash_t seed)6910 XXH3_128bits_reset_withSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH64_hash_t seed)
6911 {
6912 return XXH3_64bits_reset_withSeed(statePtr, seed);
6913 }
6914
6915 /*! @ingroup XXH3_family */
6916 XXH_PUBLIC_API XXH_errorcode
XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t * statePtr,XXH_NOESCAPE const void * secret,size_t secretSize,XXH64_hash_t seed)6917 XXH3_128bits_reset_withSecretandSeed(XXH_NOESCAPE XXH3_state_t* statePtr, XXH_NOESCAPE const void* secret, size_t secretSize, XXH64_hash_t seed)
6918 {
6919 return XXH3_64bits_reset_withSecretandSeed(statePtr, secret, secretSize, seed);
6920 }
6921
6922 /*! @ingroup XXH3_family */
6923 XXH_PUBLIC_API XXH_errorcode
XXH3_128bits_update(XXH_NOESCAPE XXH3_state_t * state,XXH_NOESCAPE const void * input,size_t len)6924 XXH3_128bits_update(XXH_NOESCAPE XXH3_state_t* state, XXH_NOESCAPE const void* input, size_t len)
6925 {
6926 return XXH3_64bits_update(state, input, len);
6927 }
6928
6929 /*! @ingroup XXH3_family */
XXH3_128bits_digest(XXH_NOESCAPE const XXH3_state_t * state)6930 XXH_PUBLIC_API XXH128_hash_t XXH3_128bits_digest (XXH_NOESCAPE const XXH3_state_t* state)
6931 {
6932 const unsigned char* const secret = (state->extSecret == NULL) ? state->customSecret : state->extSecret;
6933 if (state->totalLen > XXH3_MIDSIZE_MAX) {
6934 XXH_ALIGN(XXH_ACC_ALIGN) XXH64_hash_t acc[XXH_ACC_NB];
6935 XXH3_digest_long(acc, state, secret);
6936 XXH_ASSERT(state->secretLimit + XXH_STRIPE_LEN >= sizeof(acc) + XXH_SECRET_MERGEACCS_START);
6937 { XXH128_hash_t h128;
6938 h128.low64 = XXH3_mergeAccs(acc,
6939 secret + XXH_SECRET_MERGEACCS_START,
6940 (xxh_u64)state->totalLen * XXH_PRIME64_1);
6941 h128.high64 = XXH3_mergeAccs(acc,
6942 secret + state->secretLimit + XXH_STRIPE_LEN
6943 - sizeof(acc) - XXH_SECRET_MERGEACCS_START,
6944 ~((xxh_u64)state->totalLen * XXH_PRIME64_2));
6945 return h128;
6946 }
6947 }
6948 /* len <= XXH3_MIDSIZE_MAX : short code */
6949 if (state->seed)
6950 return XXH3_128bits_withSeed(state->buffer, (size_t)state->totalLen, state->seed);
6951 return XXH3_128bits_withSecret(state->buffer, (size_t)(state->totalLen),
6952 secret, state->secretLimit + XXH_STRIPE_LEN);
6953 }
6954 #endif /* !XXH_NO_STREAM */
6955 /* 128-bit utility functions */
6956
6957 /* return : 1 is equal, 0 if different */
6958 /*! @ingroup XXH3_family */
XXH128_isEqual(XXH128_hash_t h1,XXH128_hash_t h2)6959 XXH_PUBLIC_API int XXH128_isEqual(XXH128_hash_t h1, XXH128_hash_t h2)
6960 {
6961 /* note : XXH128_hash_t is compact, it has no padding byte */
6962 return !(memcmp(&h1, &h2, sizeof(h1)));
6963 }
6964
6965 /* This prototype is compatible with stdlib's qsort().
6966 * @return : >0 if *h128_1 > *h128_2
6967 * <0 if *h128_1 < *h128_2
6968 * =0 if *h128_1 == *h128_2 */
6969 /*! @ingroup XXH3_family */
XXH128_cmp(XXH_NOESCAPE const void * h128_1,XXH_NOESCAPE const void * h128_2)6970 XXH_PUBLIC_API int XXH128_cmp(XXH_NOESCAPE const void* h128_1, XXH_NOESCAPE const void* h128_2)
6971 {
6972 XXH128_hash_t const h1 = *(const XXH128_hash_t*)h128_1;
6973 XXH128_hash_t const h2 = *(const XXH128_hash_t*)h128_2;
6974 int const hcmp = (h1.high64 > h2.high64) - (h2.high64 > h1.high64);
6975 /* note : bets that, in most cases, hash values are different */
6976 if (hcmp) return hcmp;
6977 return (h1.low64 > h2.low64) - (h2.low64 > h1.low64);
6978 }
6979
6980
6981 /*====== Canonical representation ======*/
6982 /*! @ingroup XXH3_family */
6983 XXH_PUBLIC_API void
XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t * dst,XXH128_hash_t hash)6984 XXH128_canonicalFromHash(XXH_NOESCAPE XXH128_canonical_t* dst, XXH128_hash_t hash)
6985 {
6986 XXH_STATIC_ASSERT(sizeof(XXH128_canonical_t) == sizeof(XXH128_hash_t));
6987 if (XXH_CPU_LITTLE_ENDIAN) {
6988 hash.high64 = XXH_swap64(hash.high64);
6989 hash.low64 = XXH_swap64(hash.low64);
6990 }
6991 XXH_memcpy(dst, &hash.high64, sizeof(hash.high64));
6992 XXH_memcpy((char*)dst + sizeof(hash.high64), &hash.low64, sizeof(hash.low64));
6993 }
6994
6995 /*! @ingroup XXH3_family */
6996 XXH_PUBLIC_API XXH128_hash_t
XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t * src)6997 XXH128_hashFromCanonical(XXH_NOESCAPE const XXH128_canonical_t* src)
6998 {
6999 XXH128_hash_t h;
7000 h.high64 = XXH_readBE64(src);
7001 h.low64 = XXH_readBE64(src->digest + 8);
7002 return h;
7003 }
7004
7005
7006
7007 /* ==========================================
7008 * Secret generators
7009 * ==========================================
7010 */
7011 #define XXH_MIN(x, y) (((x) > (y)) ? (y) : (x))
7012
XXH3_combine16(void * dst,XXH128_hash_t h128)7013 XXH_FORCE_INLINE void XXH3_combine16(void* dst, XXH128_hash_t h128)
7014 {
7015 XXH_writeLE64( dst, XXH_readLE64(dst) ^ h128.low64 );
7016 XXH_writeLE64( (char*)dst+8, XXH_readLE64((char*)dst+8) ^ h128.high64 );
7017 }
7018
7019 /*! @ingroup XXH3_family */
7020 XXH_PUBLIC_API XXH_errorcode
XXH3_generateSecret(XXH_NOESCAPE void * secretBuffer,size_t secretSize,XXH_NOESCAPE const void * customSeed,size_t customSeedSize)7021 XXH3_generateSecret(XXH_NOESCAPE void* secretBuffer, size_t secretSize, XXH_NOESCAPE const void* customSeed, size_t customSeedSize)
7022 {
7023 #if (XXH_DEBUGLEVEL >= 1)
7024 XXH_ASSERT(secretBuffer != NULL);
7025 XXH_ASSERT(secretSize >= XXH3_SECRET_SIZE_MIN);
7026 #else
7027 /* production mode, assert() are disabled */
7028 if (secretBuffer == NULL) return XXH_ERROR;
7029 if (secretSize < XXH3_SECRET_SIZE_MIN) return XXH_ERROR;
7030 #endif
7031
7032 if (customSeedSize == 0) {
7033 customSeed = XXH3_kSecret;
7034 customSeedSize = XXH_SECRET_DEFAULT_SIZE;
7035 }
7036 #if (XXH_DEBUGLEVEL >= 1)
7037 XXH_ASSERT(customSeed != NULL);
7038 #else
7039 if (customSeed == NULL) return XXH_ERROR;
7040 #endif
7041
7042 /* Fill secretBuffer with a copy of customSeed - repeat as needed */
7043 { size_t pos = 0;
7044 while (pos < secretSize) {
7045 size_t const toCopy = XXH_MIN((secretSize - pos), customSeedSize);
7046 memcpy((char*)secretBuffer + pos, customSeed, toCopy);
7047 pos += toCopy;
7048 } }
7049
7050 { size_t const nbSeg16 = secretSize / 16;
7051 size_t n;
7052 XXH128_canonical_t scrambler;
7053 XXH128_canonicalFromHash(&scrambler, XXH128(customSeed, customSeedSize, 0));
7054 for (n=0; n<nbSeg16; n++) {
7055 XXH128_hash_t const h128 = XXH128(&scrambler, sizeof(scrambler), n);
7056 XXH3_combine16((char*)secretBuffer + n*16, h128);
7057 }
7058 /* last segment */
7059 XXH3_combine16((char*)secretBuffer + secretSize - 16, XXH128_hashFromCanonical(&scrambler));
7060 }
7061 return XXH_OK;
7062 }
7063
7064 /*! @ingroup XXH3_family */
7065 XXH_PUBLIC_API void
XXH3_generateSecret_fromSeed(XXH_NOESCAPE void * secretBuffer,XXH64_hash_t seed)7066 XXH3_generateSecret_fromSeed(XXH_NOESCAPE void* secretBuffer, XXH64_hash_t seed)
7067 {
7068 XXH_ALIGN(XXH_SEC_ALIGN) xxh_u8 secret[XXH_SECRET_DEFAULT_SIZE];
7069 XXH3_initCustomSecret(secret, seed);
7070 XXH_ASSERT(secretBuffer != NULL);
7071 memcpy(secretBuffer, secret, XXH_SECRET_DEFAULT_SIZE);
7072 }
7073
7074
7075
7076 /* Pop our optimization override from above */
7077 #if XXH_VECTOR == XXH_AVX2 /* AVX2 */ \
7078 && defined(__GNUC__) && !defined(__clang__) /* GCC, not Clang */ \
7079 && defined(__OPTIMIZE__) && XXH_SIZE_OPT <= 0 /* respect -O0 and -Os */
7080 # pragma GCC pop_options
7081 #endif
7082
7083
7084 #if defined (__cplusplus)
7085 } /* extern "C" */
7086 #endif
7087
7088 #endif /* XXH_NO_LONG_LONG */
7089 #endif /* XXH_NO_XXH3 */
7090
7091 /*!
7092 * @}
7093 */
7094 #endif /* XXH_IMPLEMENTATION */
7095