1 /*
2 * Fill Window with SSE2-optimized hash shifting
3 *
4 * Copyright (C) 2013 Intel Corporation
5 * Authors:
6 * Arjan van de Ven <arjan@linux.intel.com>
7 * Jim Kukunas <james.t.kukunas@linux.intel.com>
8 *
9 * For conditions of distribution and use, see copyright notice in zlib.h
10 */
11
12 #include "deflate.h"
13
14 #ifdef DEFLATE_FILL_WINDOW_SSE2
15
16 #define UPDATE_HASH(s,h,i) \
17 {\
18 if (s->level < 6) { \
19 h = (3483 * (s->window[i]) +\
20 23081* (s->window[i+1]) +\
21 6954 * (s->window[i+2]) +\
22 20947* (s->window[i+3])) & s->hash_mask;\
23 } else {\
24 h = (25881* (s->window[i]) +\
25 24674* (s->window[i+1]) +\
26 25811* (s->window[i+2])) & s->hash_mask;\
27 }\
28 }\
29
30 extern int deflate_read_buf OF((z_streamp strm, Bytef *buf, unsigned size));
31
32 #include <immintrin.h>
33
fill_window_sse(deflate_state * s)34 void fill_window_sse(deflate_state *s)
35 {
36 const __m128i xmm_wsize = _mm_set1_epi16(s->w_size);
37
38 register unsigned n;
39 register Posf *p;
40 unsigned more; /* Amount of free space at the end of the window. */
41 uInt wsize = s->w_size;
42
43 Assert(s->lookahead < MIN_LOOKAHEAD, "already enough lookahead");
44
45 do {
46 more = (unsigned)(s->window_size -(ulg)s->lookahead -(ulg)s->strstart);
47
48 /* Deal with !@#$% 64K limit: */
49 if (sizeof(int) <= 2) {
50 if (more == 0 && s->strstart == 0 && s->lookahead == 0) {
51 more = wsize;
52
53 } else if (more == (unsigned)(-1)) {
54 /* Very unlikely, but possible on 16 bit machine if
55 * strstart == 0 && lookahead == 1 (input done a byte at time)
56 */
57 more--;
58 }
59 }
60
61 /* If the window is almost full and there is insufficient lookahead,
62 * move the upper half to the lower one to make room in the upper half.
63 */
64 if (s->strstart >= wsize+MAX_DIST(s)) {
65
66 zmemcpy(s->window, s->window+wsize, (unsigned)wsize);
67 s->match_start -= wsize;
68 s->strstart -= wsize; /* we now have strstart >= MAX_DIST */
69 s->block_start -= (long) wsize;
70
71 /* Slide the hash table (could be avoided with 32 bit values
72 at the expense of memory usage). We slide even when level == 0
73 to keep the hash table consistent if we switch back to level > 0
74 later. (Using level 0 permanently is not an optimal usage of
75 zlib, so we don't care about this pathological case.)
76 */
77 n = s->hash_size;
78 p = &s->head[n];
79 p -= 8;
80 do {
81 __m128i value, result;
82
83 value = _mm_loadu_si128((__m128i *)p);
84 result = _mm_subs_epu16(value, xmm_wsize);
85 _mm_storeu_si128((__m128i *)p, result);
86
87 p -= 8;
88 n -= 8;
89 } while (n > 0);
90
91 n = wsize;
92 #ifndef FASTEST
93 p = &s->prev[n];
94 p -= 8;
95 do {
96 __m128i value, result;
97
98 value = _mm_loadu_si128((__m128i *)p);
99 result = _mm_subs_epu16(value, xmm_wsize);
100 _mm_storeu_si128((__m128i *)p, result);
101
102 p -= 8;
103 n -= 8;
104 } while (n > 0);
105 #endif
106 more += wsize;
107 }
108 if (s->strm->avail_in == 0) break;
109
110 /* If there was no sliding:
111 * strstart <= WSIZE+MAX_DIST-1 && lookahead <= MIN_LOOKAHEAD - 1 &&
112 * more == window_size - lookahead - strstart
113 * => more >= window_size - (MIN_LOOKAHEAD-1 + WSIZE + MAX_DIST-1)
114 * => more >= window_size - 2*WSIZE + 2
115 * In the BIG_MEM or MMAP case (not yet supported),
116 * window_size == input_size + MIN_LOOKAHEAD &&
117 * strstart + s->lookahead <= input_size => more >= MIN_LOOKAHEAD.
118 * Otherwise, window_size == 2*WSIZE so more >= 2.
119 * If there was sliding, more >= WSIZE. So in all cases, more >= 2.
120 */
121 Assert(more >= 2, "more < 2");
122
123 n = deflate_read_buf(s->strm,
124 s->window + s->strstart + s->lookahead,
125 more);
126 s->lookahead += n;
127
128 /* Initialize the hash value now that we have some input: */
129 if (s->lookahead >= MIN_MATCH) {
130 uInt str = s->strstart;
131 s->ins_h = s->window[str];
132 if (str >= 1)
133 UPDATE_HASH(s, s->ins_h, str + 1 - (MIN_MATCH-1));
134 #if MIN_MATCH != 3
135 Call UPDATE_HASH() MIN_MATCH-3 more times
136 #endif
137 }
138 /* If the whole input has less than MIN_MATCH bytes, ins_h is garbage,
139 * but this is not important since only literal bytes will be emitted.
140 */
141
142 } while (s->lookahead < MIN_LOOKAHEAD && s->strm->avail_in != 0);
143
144 /* If the WIN_INIT bytes after the end of the current data have never been
145 * written, then zero those bytes in order to avoid memory check reports of
146 * the use of uninitialized (or uninitialised as Julian writes) bytes by
147 * the longest match routines. Update the high water mark for the next
148 * time through here. WIN_INIT is set to MAX_MATCH since the longest match
149 * routines allow scanning to strstart + MAX_MATCH, ignoring lookahead.
150 */
151 if (s->high_water < s->window_size) {
152 ulg curr = s->strstart + (ulg)(s->lookahead);
153 ulg init;
154
155 if (s->high_water < curr) {
156 /* Previous high water mark below current data -- zero WIN_INIT
157 * bytes or up to end of window, whichever is less.
158 */
159 init = s->window_size - curr;
160 if (init > WIN_INIT)
161 init = WIN_INIT;
162 zmemzero(s->window + curr, (unsigned)init);
163 s->high_water = curr + init;
164 }
165 else if (s->high_water < (ulg)curr + WIN_INIT) {
166 /* High water mark at or above current data, but below current data
167 * plus WIN_INIT -- zero out to current data plus WIN_INIT, or up
168 * to end of window, whichever is less.
169 */
170 init = (ulg)curr + WIN_INIT - s->high_water;
171 if (init > s->window_size - s->high_water)
172 init = s->window_size - s->high_water;
173 zmemzero(s->window + s->high_water, (unsigned)init);
174 s->high_water += init;
175 }
176 }
177
178 Assert((ulg)s->strstart <= s->window_size - MIN_LOOKAHEAD,
179 "not enough room for search");
180 }
181
182 #endif /* DEFLATE_FILL_WINDOW_SSE2 */
183