1 /* Copyright (C) 1995-2011, 2016 Mark Adler
2 * Copyright (C) 2017 ARM Holdings Inc.
3 * Author: Adenilson Cavalcanti <adenilson.cavalcanti@arm.com>
4 *
5 * For conditions of distribution and use, see copyright notice in zlib.h
6 */
7 #ifdef ARM_NEON_ADLER32
8 #ifdef _M_ARM64
9 # include <arm64_neon.h>
10 #else
11 # include <arm_neon.h>
12 #endif
13 #include "../../zutil.h"
14 #include "../../adler32_p.h"
15
NEON_accum32(uint32_t * s,const unsigned char * buf,size_t len)16 static void NEON_accum32(uint32_t *s, const unsigned char *buf, size_t len) {
17 static const uint8_t taps[32] = {
18 32, 31, 30, 29, 28, 27, 26, 25,
19 24, 23, 22, 21, 20, 19, 18, 17,
20 16, 15, 14, 13, 12, 11, 10, 9,
21 8, 7, 6, 5, 4, 3, 2, 1 };
22
23 uint32x2_t adacc2, s2acc2, as;
24 uint8x16_t t0 = vld1q_u8(taps), t1 = vld1q_u8(taps + 16);
25
26 uint32x4_t adacc = vdupq_n_u32(0), s2acc = vdupq_n_u32(0);
27 adacc = vsetq_lane_u32(s[0], adacc, 0);
28 s2acc = vsetq_lane_u32(s[1], s2acc, 0);
29
30 while (len >= 2) {
31 uint8x16_t d0 = vld1q_u8(buf), d1 = vld1q_u8(buf + 16);
32 uint16x8_t adler, sum2;
33 s2acc = vaddq_u32(s2acc, vshlq_n_u32(adacc, 5));
34 adler = vpaddlq_u8( d0);
35 adler = vpadalq_u8(adler, d1);
36 sum2 = vmull_u8( vget_low_u8(t0), vget_low_u8(d0));
37 sum2 = vmlal_u8(sum2, vget_high_u8(t0), vget_high_u8(d0));
38 sum2 = vmlal_u8(sum2, vget_low_u8(t1), vget_low_u8(d1));
39 sum2 = vmlal_u8(sum2, vget_high_u8(t1), vget_high_u8(d1));
40 adacc = vpadalq_u16(adacc, adler);
41 s2acc = vpadalq_u16(s2acc, sum2);
42 len -= 2;
43 buf += 32;
44 }
45
46 while (len > 0) {
47 uint8x16_t d0 = vld1q_u8(buf);
48 uint16x8_t adler, sum2;
49 s2acc = vaddq_u32(s2acc, vshlq_n_u32(adacc, 4));
50 adler = vpaddlq_u8(d0);
51 sum2 = vmull_u8( vget_low_u8(t1), vget_low_u8(d0));
52 sum2 = vmlal_u8(sum2, vget_high_u8(t1), vget_high_u8(d0));
53 adacc = vpadalq_u16(adacc, adler);
54 s2acc = vpadalq_u16(s2acc, sum2);
55 buf += 16;
56 len--;
57 }
58
59 adacc2 = vpadd_u32(vget_low_u32(adacc), vget_high_u32(adacc));
60 s2acc2 = vpadd_u32(vget_low_u32(s2acc), vget_high_u32(s2acc));
61 as = vpadd_u32(adacc2, s2acc2);
62 s[0] = vget_lane_u32(as, 0);
63 s[1] = vget_lane_u32(as, 1);
64 }
65
NEON_handle_tail(uint32_t * pair,const unsigned char * buf,size_t len)66 static void NEON_handle_tail(uint32_t *pair, const unsigned char *buf, size_t len) {
67 unsigned int i;
68 for (i = 0; i < len; ++i) {
69 pair[0] += buf[i];
70 pair[1] += pair[0];
71 }
72 }
73
adler32_neon(uint32_t adler,const unsigned char * buf,size_t len)74 uint32_t adler32_neon(uint32_t adler, const unsigned char *buf, size_t len) {
75 /* split Adler-32 into component sums */
76 uint32_t sum2 = (adler >> 16) & 0xffff;
77 adler &= 0xffff;
78
79 /* in case user likes doing a byte at a time, keep it fast */
80 if (len == 1)
81 return adler32_len_1(adler, buf, sum2);
82
83 /* initial Adler-32 value (deferred check for len == 1 speed) */
84 if (buf == NULL)
85 return 1L;
86
87 /* in case short lengths are provided, keep it somewhat fast */
88 if (len < 16)
89 return adler32_len_16(adler, buf, len, sum2);
90
91 uint32_t pair[2];
92 int n = NMAX;
93 unsigned int done = 0;
94 unsigned int i;
95
96 /* Split Adler-32 into component sums, it can be supplied by
97 * the caller sites (e.g. in a PNG file).
98 */
99 pair[0] = adler;
100 pair[1] = sum2;
101
102 for (i = 0; i < len; i += n) {
103 if ((i + n) > len)
104 n = (int)(len - i);
105
106 if (n < 16)
107 break;
108
109 NEON_accum32(pair, buf + i, n / 16);
110 pair[0] %= BASE;
111 pair[1] %= BASE;
112
113 done += (n / 16) * 16;
114 }
115
116 /* Handle the tail elements. */
117 if (done < len) {
118 NEON_handle_tail(pair, (buf + done), len - done);
119 pair[0] %= BASE;
120 pair[1] %= BASE;
121 }
122
123 /* D = B * 65536 + A, see: https://en.wikipedia.org/wiki/Adler-32. */
124 return (pair[1] << 16) | pair[0];
125 }
126 #endif
127