1 #[cfg(not(all( 2 httparse_simd, 3 any( 4 target_arch = "x86", 5 target_arch = "x86_64", 6 ), 7 )))] 8 mod fallback; 9 10 #[cfg(not(all( 11 httparse_simd, 12 any( 13 target_arch = "x86", 14 target_arch = "x86_64", 15 ), 16 )))] 17 pub use self::fallback::*; 18 19 #[cfg(all( 20 httparse_simd, 21 any( 22 target_arch = "x86", 23 target_arch = "x86_64", 24 ), 25 ))] 26 mod sse42; 27 28 #[cfg(all( 29 httparse_simd, 30 any( 31 httparse_simd_target_feature_avx2, 32 not(httparse_simd_target_feature_sse42), 33 ), 34 any( 35 target_arch = "x86", 36 target_arch = "x86_64", 37 ), 38 ))] 39 mod avx2; 40 41 #[cfg(all( 42 httparse_simd, 43 any( 44 target_arch = "x86", 45 target_arch = "x86_64", 46 ), 47 ))] 48 pub const SSE_42: usize = 1; 49 #[cfg(all( 50 httparse_simd, 51 any(not(httparse_simd_target_feature_sse42), httparse_simd_target_feature_avx2), 52 any( 53 target_arch = "x86", 54 target_arch = "x86_64", 55 ), 56 ))] 57 pub const AVX_2: usize = 2; 58 #[cfg(all( 59 httparse_simd, 60 any( 61 not(httparse_simd_target_feature_sse42), 62 httparse_simd_target_feature_avx2, 63 test, 64 ), 65 any( 66 target_arch = "x86", 67 target_arch = "x86_64", 68 ), 69 ))] 70 pub const AVX_2_AND_SSE_42: usize = 3; 71 72 #[cfg(all( 73 httparse_simd, 74 any( 75 target_arch = "x86", 76 target_arch = "x86_64", 77 ), 78 ))] 79 const NONE: usize = std::usize::MAX; 80 #[cfg(all( 81 httparse_simd, 82 not(any( 83 httparse_simd_target_feature_sse42, 84 httparse_simd_target_feature_avx2, 85 )), 86 any( 87 target_arch = "x86", 88 target_arch = "x86_64", 89 ), 90 ))] 91 mod runtime { 92 //! Runtime detection of simd features. Used when the build script 93 //! doesn't notice any target features at build time. 94 //! 95 //! While `is_x86_feature_detected!` has it's own caching built-in, 96 //! at least in 1.27.0, the functions don't inline, leaving using it 97 //! actually *slower* than just using the scalar fallback. 98 99 use core::sync::atomic::{AtomicUsize, Ordering}; 100 101 static FEATURE: AtomicUsize = AtomicUsize::new(0); 102 103 const INIT: usize = 0; 104 detect() -> usize105 pub fn detect() -> usize { 106 let feat = FEATURE.load(Ordering::Relaxed); 107 if feat == INIT { 108 if cfg!(target_arch = "x86_64") && is_x86_feature_detected!("avx2") { 109 if is_x86_feature_detected!("sse4.2") { 110 FEATURE.store(super::AVX_2_AND_SSE_42, Ordering::Relaxed); 111 return super::AVX_2_AND_SSE_42; 112 } else { 113 FEATURE.store(super::AVX_2, Ordering::Relaxed); 114 return super::AVX_2; 115 } 116 } else if is_x86_feature_detected!("sse4.2") { 117 FEATURE.store(super::SSE_42, Ordering::Relaxed); 118 return super::SSE_42; 119 } else { 120 FEATURE.store(super::NONE, Ordering::Relaxed); 121 } 122 } 123 feat 124 } 125 match_uri_vectored(bytes: &mut crate::iter::Bytes)126 pub fn match_uri_vectored(bytes: &mut crate::iter::Bytes) { 127 unsafe { 128 match detect() { 129 super::SSE_42 => super::sse42::parse_uri_batch_16(bytes), 130 super::AVX_2 => { super::avx2::parse_uri_batch_32(bytes); }, 131 super::AVX_2_AND_SSE_42 => { 132 if let super::avx2::Scan::Found = super::avx2::parse_uri_batch_32(bytes) { 133 return; 134 } 135 super::sse42::parse_uri_batch_16(bytes) 136 }, 137 _ => () 138 } 139 } 140 141 // else do nothing 142 } 143 match_header_value_vectored(bytes: &mut crate::iter::Bytes)144 pub fn match_header_value_vectored(bytes: &mut crate::iter::Bytes) { 145 unsafe { 146 match detect() { 147 super::SSE_42 => super::sse42::match_header_value_batch_16(bytes), 148 super::AVX_2 => { super::avx2::match_header_value_batch_32(bytes); }, 149 super::AVX_2_AND_SSE_42 => { 150 if let super::avx2::Scan::Found = super::avx2::match_header_value_batch_32(bytes) { 151 return; 152 } 153 super::sse42::match_header_value_batch_16(bytes) 154 }, 155 _ => () 156 } 157 } 158 159 // else do nothing 160 } 161 } 162 163 #[cfg(all( 164 httparse_simd, 165 not(any( 166 httparse_simd_target_feature_sse42, 167 httparse_simd_target_feature_avx2, 168 )), 169 any( 170 target_arch = "x86", 171 target_arch = "x86_64", 172 ), 173 ))] 174 pub use self::runtime::*; 175 176 #[cfg(all( 177 httparse_simd, 178 httparse_simd_target_feature_sse42, 179 not(httparse_simd_target_feature_avx2), 180 any( 181 target_arch = "x86", 182 target_arch = "x86_64", 183 ), 184 ))] 185 mod sse42_compile_time { match_uri_vectored(bytes: &mut crate::iter::Bytes)186 pub fn match_uri_vectored(bytes: &mut crate::iter::Bytes) { 187 if detect() == super::SSE_42 { 188 unsafe { 189 super::sse42::parse_uri_batch_16(bytes); 190 } 191 } 192 193 // else do nothing 194 } 195 match_header_value_vectored(bytes: &mut crate::iter::Bytes)196 pub fn match_header_value_vectored(bytes: &mut crate::iter::Bytes) { 197 if detect() == super::SSE_42 { 198 unsafe { 199 super::sse42::match_header_value_batch_16(bytes); 200 } 201 } 202 203 // else do nothing 204 } 205 detect() -> usize206 pub fn detect() -> usize { 207 if is_x86_feature_detected!("sse4.2") { 208 super::SSE_42 209 } else { 210 super::NONE 211 } 212 } 213 } 214 215 #[cfg(all( 216 httparse_simd, 217 httparse_simd_target_feature_sse42, 218 not(httparse_simd_target_feature_avx2), 219 any( 220 target_arch = "x86", 221 target_arch = "x86_64", 222 ), 223 ))] 224 pub use self::sse42_compile_time::*; 225 226 #[cfg(all( 227 httparse_simd, 228 httparse_simd_target_feature_avx2, 229 any( 230 target_arch = "x86", 231 target_arch = "x86_64", 232 ), 233 ))] 234 mod avx2_compile_time { match_uri_vectored(bytes: &mut crate::iter::Bytes)235 pub fn match_uri_vectored(bytes: &mut crate::iter::Bytes) { 236 // do both, since avx2 only works when bytes.len() >= 32 237 if detect() == super::AVX_2_AND_SSE_42 { 238 unsafe { 239 super::avx2::parse_uri_batch_32(bytes); 240 } 241 242 } 243 if detect() == super::SSE_42 { 244 unsafe { 245 super::sse42::parse_uri_batch_16(bytes); 246 } 247 } 248 249 // else do nothing 250 } 251 match_header_value_vectored(bytes: &mut crate::iter::Bytes)252 pub fn match_header_value_vectored(bytes: &mut crate::iter::Bytes) { 253 // do both, since avx2 only works when bytes.len() >= 32 254 if detect() == super::AVX_2_AND_SSE_42 { 255 let scanned = unsafe { 256 super::avx2::match_header_value_batch_32(bytes) 257 }; 258 259 if let super::avx2::Scan::Found = scanned { 260 return; 261 } 262 } 263 if detect() == super::SSE_42 { 264 unsafe { 265 super::sse42::match_header_value_batch_16(bytes); 266 } 267 } 268 269 // else do nothing 270 } 271 detect() -> usize272 pub fn detect() -> usize { 273 if cfg!(target_arch = "x86_64") && is_x86_feature_detected!("avx2") { 274 super::AVX_2_AND_SSE_42 275 } else if is_x86_feature_detected!("sse4.2") { 276 super::SSE_42 277 } else { 278 super::NONE 279 } 280 } 281 } 282 283 #[cfg(all( 284 httparse_simd, 285 httparse_simd_target_feature_avx2, 286 any( 287 target_arch = "x86", 288 target_arch = "x86_64", 289 ), 290 ))] 291 pub use self::avx2_compile_time::*; 292