1 use crate::iter::Bytes;
2
parse_uri_batch_16(bytes: &mut Bytes)3 pub unsafe fn parse_uri_batch_16(bytes: &mut Bytes) {
4 while bytes.as_ref().len() >= 16 {
5 let advance = match_url_char_16_sse(bytes.as_ref());
6 bytes.advance(advance);
7
8 if advance != 16 {
9 break;
10 }
11 }
12 }
13
14 #[target_feature(enable = "sse4.2")]
15 #[allow(non_snake_case, overflowing_literals)]
match_url_char_16_sse(buf: &[u8]) -> usize16 unsafe fn match_url_char_16_sse(buf: &[u8]) -> usize {
17 debug_assert!(buf.len() >= 16);
18
19 #[cfg(target_arch = "x86")]
20 use core::arch::x86::*;
21 #[cfg(target_arch = "x86_64")]
22 use core::arch::x86_64::*;
23
24 let ptr = buf.as_ptr();
25
26 let LSH: __m128i = _mm_set1_epi8(0x0f);
27
28 // The first 0xf8 corresponds to the 8 first rows of the first column
29 // of URI_MAP in the crate's root, with the first row corresponding to bit 0
30 // and the 8th row corresponding to bit 7.
31 // The 8 first rows give 0 0 0 1 1 1 1 1, which is 0xf8 (with least
32 // significant digit on the left).
33 //
34 // Another example just to drive the point home: in column 15, '>' is
35 // rejected, so the values are 0 0 1 0 1 1 1 1, which gives us 0xf4.
36 //
37 // Thanks to Vlad Krasnov for explaining this stuff to us mere mortals in
38 // a GitHub comment!
39 //
40 // https://github.com/seanmonstar/httparse/pull/89#issuecomment-807039219
41
42 let URI: __m128i = _mm_setr_epi8(
43 0xf8, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc, 0xfc,
44 0xfc, 0xfc, 0xfc, 0xfc, 0xf4, 0xfc, 0xf4, 0x7c,
45 );
46 let ARF: __m128i = _mm_setr_epi8(
47 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,
48 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
49 );
50
51 let data = _mm_lddqu_si128(ptr as *const _);
52 let rbms = _mm_shuffle_epi8(URI, data);
53 let cols = _mm_and_si128(LSH, _mm_srli_epi16(data, 4));
54 let bits = _mm_and_si128(_mm_shuffle_epi8(ARF, cols), rbms);
55
56 let v = _mm_cmpeq_epi8(bits, _mm_setzero_si128());
57 let r = 0xffff_0000 | _mm_movemask_epi8(v) as u32;
58
59 _tzcnt_u32(r) as usize
60 }
61
match_header_value_batch_16(bytes: &mut Bytes)62 pub unsafe fn match_header_value_batch_16(bytes: &mut Bytes) {
63 while bytes.as_ref().len() >= 16 {
64 let advance = match_header_value_char_16_sse(bytes.as_ref());
65 bytes.advance(advance);
66
67 if advance != 16 {
68 break;
69 }
70 }
71 }
72
73 #[target_feature(enable = "sse4.2")]
74 #[allow(non_snake_case)]
match_header_value_char_16_sse(buf: &[u8]) -> usize75 unsafe fn match_header_value_char_16_sse(buf: &[u8]) -> usize {
76 debug_assert!(buf.len() >= 16);
77
78 #[cfg(target_arch = "x86")]
79 use core::arch::x86::*;
80 #[cfg(target_arch = "x86_64")]
81 use core::arch::x86_64::*;
82
83 let ptr = buf.as_ptr();
84
85 // %x09 %x20-%x7e %x80-%xff
86 let TAB: __m128i = _mm_set1_epi8(0x09);
87 let DEL: __m128i = _mm_set1_epi8(0x7f);
88 let LOW: __m128i = _mm_set1_epi8(0x20);
89
90 let dat = _mm_lddqu_si128(ptr as *const _);
91 // unsigned comparison dat >= LOW
92 let low = _mm_cmpeq_epi8(_mm_max_epu8(dat, LOW), dat);
93 let tab = _mm_cmpeq_epi8(dat, TAB);
94 let del = _mm_cmpeq_epi8(dat, DEL);
95 let bit = _mm_andnot_si128(del, _mm_or_si128(low, tab));
96 let rev = _mm_cmpeq_epi8(bit, _mm_setzero_si128());
97 let res = 0xffff_0000 | _mm_movemask_epi8(rev) as u32;
98
99 _tzcnt_u32(res) as usize
100 }
101
102 #[test]
sse_code_matches_uri_chars_table()103 fn sse_code_matches_uri_chars_table() {
104 match super::detect() {
105 super::SSE_42 | super::AVX_2_AND_SSE_42 => {},
106 _ => return,
107 }
108
109 unsafe {
110 assert!(byte_is_allowed(b'_', parse_uri_batch_16));
111
112 for (b, allowed) in crate::URI_MAP.iter().cloned().enumerate() {
113 assert_eq!(
114 byte_is_allowed(b as u8, parse_uri_batch_16), allowed,
115 "byte_is_allowed({:?}) should be {:?}", b, allowed,
116 );
117 }
118 }
119 }
120
121 #[test]
sse_code_matches_header_value_chars_table()122 fn sse_code_matches_header_value_chars_table() {
123 match super::detect() {
124 super::SSE_42 | super::AVX_2_AND_SSE_42 => {},
125 _ => return,
126 }
127
128 unsafe {
129 assert!(byte_is_allowed(b'_', match_header_value_batch_16));
130
131 for (b, allowed) in crate::HEADER_VALUE_MAP.iter().cloned().enumerate() {
132 assert_eq!(
133 byte_is_allowed(b as u8, match_header_value_batch_16), allowed,
134 "byte_is_allowed({:?}) should be {:?}", b, allowed,
135 );
136 }
137 }
138 }
139
140 #[cfg(test)]
byte_is_allowed(byte: u8, f: unsafe fn(bytes: &mut Bytes<'_>)) -> bool141 unsafe fn byte_is_allowed(byte: u8, f: unsafe fn(bytes: &mut Bytes<'_>)) -> bool {
142 let slice = [
143 b'_', b'_', b'_', b'_',
144 b'_', b'_', b'_', b'_',
145 b'_', b'_', byte, b'_',
146 b'_', b'_', b'_', b'_',
147 ];
148 let mut bytes = Bytes::new(&slice);
149
150 f(&mut bytes);
151
152 match bytes.pos() {
153 16 => true,
154 10 => false,
155 _ => unreachable!(),
156 }
157 }
158