1//===- IntrinsicsAARCH64.td - Defines AARCH64 intrinsics ---*- tablegen -*-===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file defines all of the AARCH64-specific intrinsics. 10// 11//===----------------------------------------------------------------------===// 12 13let TargetPrefix = "aarch64" in { 14 15def int_aarch64_ldxr : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty]>; 16def int_aarch64_ldaxr : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty]>; 17def int_aarch64_stxr : Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_anyptr_ty]>; 18def int_aarch64_stlxr : Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_anyptr_ty]>; 19 20def int_aarch64_ldxp : Intrinsic<[llvm_i64_ty, llvm_i64_ty], [llvm_ptr_ty]>; 21def int_aarch64_ldaxp : Intrinsic<[llvm_i64_ty, llvm_i64_ty], [llvm_ptr_ty]>; 22def int_aarch64_stxp : Intrinsic<[llvm_i32_ty], 23 [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty]>; 24def int_aarch64_stlxp : Intrinsic<[llvm_i32_ty], 25 [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty]>; 26 27def int_aarch64_clrex : Intrinsic<[]>; 28 29def int_aarch64_sdiv : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, 30 LLVMMatchType<0>], [IntrNoMem]>; 31def int_aarch64_udiv : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, 32 LLVMMatchType<0>], [IntrNoMem]>; 33 34def int_aarch64_fjcvtzs : Intrinsic<[llvm_i32_ty], [llvm_double_ty], [IntrNoMem]>; 35 36def int_aarch64_cls: Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem]>; 37def int_aarch64_cls64: Intrinsic<[llvm_i32_ty], [llvm_i64_ty], [IntrNoMem]>; 38 39//===----------------------------------------------------------------------===// 40// HINT 41 42def int_aarch64_hint : Intrinsic<[], [llvm_i32_ty]>; 43 44//===----------------------------------------------------------------------===// 45// Data Barrier Instructions 46 47def int_aarch64_dmb : GCCBuiltin<"__builtin_arm_dmb">, MSBuiltin<"__dmb">, Intrinsic<[], [llvm_i32_ty]>; 48def int_aarch64_dsb : GCCBuiltin<"__builtin_arm_dsb">, MSBuiltin<"__dsb">, Intrinsic<[], [llvm_i32_ty]>; 49def int_aarch64_isb : GCCBuiltin<"__builtin_arm_isb">, MSBuiltin<"__isb">, Intrinsic<[], [llvm_i32_ty]>; 50 51// A space-consuming intrinsic primarily for testing block and jump table 52// placements. The first argument is the number of bytes this "instruction" 53// takes up, the second and return value are essentially chains, used to force 54// ordering during ISel. 55def int_aarch64_space : Intrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_i64_ty], []>; 56 57} 58 59//===----------------------------------------------------------------------===// 60// Advanced SIMD (NEON) 61 62let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". 63 class AdvSIMD_2Scalar_Float_Intrinsic 64 : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>], 65 [IntrNoMem]>; 66 67 class AdvSIMD_FPToIntRounding_Intrinsic 68 : Intrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty], [IntrNoMem]>; 69 70 class AdvSIMD_1IntArg_Intrinsic 71 : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>], [IntrNoMem]>; 72 class AdvSIMD_1FloatArg_Intrinsic 73 : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem]>; 74 class AdvSIMD_1VectorArg_Intrinsic 75 : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>], [IntrNoMem]>; 76 class AdvSIMD_1VectorArg_Expand_Intrinsic 77 : Intrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty], [IntrNoMem]>; 78 class AdvSIMD_1VectorArg_Long_Intrinsic 79 : Intrinsic<[llvm_anyvector_ty], [LLVMTruncatedType<0>], [IntrNoMem]>; 80 class AdvSIMD_1IntArg_Narrow_Intrinsic 81 : Intrinsic<[llvm_anyint_ty], [llvm_anyint_ty], [IntrNoMem]>; 82 class AdvSIMD_1VectorArg_Narrow_Intrinsic 83 : Intrinsic<[llvm_anyint_ty], [LLVMExtendedType<0>], [IntrNoMem]>; 84 class AdvSIMD_1VectorArg_Int_Across_Intrinsic 85 : Intrinsic<[llvm_anyint_ty], [llvm_anyvector_ty], [IntrNoMem]>; 86 class AdvSIMD_1VectorArg_Float_Across_Intrinsic 87 : Intrinsic<[llvm_anyfloat_ty], [llvm_anyvector_ty], [IntrNoMem]>; 88 89 class AdvSIMD_2IntArg_Intrinsic 90 : Intrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>], 91 [IntrNoMem]>; 92 class AdvSIMD_2FloatArg_Intrinsic 93 : Intrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>], 94 [IntrNoMem]>; 95 class AdvSIMD_2VectorArg_Intrinsic 96 : Intrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>], 97 [IntrNoMem]>; 98 class AdvSIMD_2VectorArg_Compare_Intrinsic 99 : Intrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, LLVMMatchType<1>], 100 [IntrNoMem]>; 101 class AdvSIMD_2Arg_FloatCompare_Intrinsic 102 : Intrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty, LLVMMatchType<1>], 103 [IntrNoMem]>; 104 class AdvSIMD_2VectorArg_Long_Intrinsic 105 : Intrinsic<[llvm_anyvector_ty], 106 [LLVMTruncatedType<0>, LLVMTruncatedType<0>], 107 [IntrNoMem]>; 108 class AdvSIMD_2VectorArg_Wide_Intrinsic 109 : Intrinsic<[llvm_anyvector_ty], 110 [LLVMMatchType<0>, LLVMTruncatedType<0>], 111 [IntrNoMem]>; 112 class AdvSIMD_2VectorArg_Narrow_Intrinsic 113 : Intrinsic<[llvm_anyvector_ty], 114 [LLVMExtendedType<0>, LLVMExtendedType<0>], 115 [IntrNoMem]>; 116 class AdvSIMD_2Arg_Scalar_Narrow_Intrinsic 117 : Intrinsic<[llvm_anyint_ty], 118 [LLVMExtendedType<0>, llvm_i32_ty], 119 [IntrNoMem]>; 120 class AdvSIMD_2VectorArg_Scalar_Expand_BySize_Intrinsic 121 : Intrinsic<[llvm_anyvector_ty], 122 [llvm_anyvector_ty], 123 [IntrNoMem]>; 124 class AdvSIMD_2VectorArg_Scalar_Wide_BySize_Intrinsic 125 : Intrinsic<[llvm_anyvector_ty], 126 [LLVMTruncatedType<0>], 127 [IntrNoMem]>; 128 class AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic 129 : Intrinsic<[llvm_anyvector_ty], 130 [LLVMTruncatedType<0>, llvm_i32_ty], 131 [IntrNoMem]>; 132 class AdvSIMD_2VectorArg_Tied_Narrow_Intrinsic 133 : Intrinsic<[llvm_anyvector_ty], 134 [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty], 135 [IntrNoMem]>; 136 class AdvSIMD_2VectorArg_Lane_Intrinsic 137 : Intrinsic<[llvm_anyint_ty], 138 [LLVMMatchType<0>, llvm_anyint_ty, llvm_i32_ty], 139 [IntrNoMem]>; 140 141 class AdvSIMD_3VectorArg_Intrinsic 142 : Intrinsic<[llvm_anyvector_ty], 143 [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], 144 [IntrNoMem]>; 145 class AdvSIMD_3VectorArg_Scalar_Intrinsic 146 : Intrinsic<[llvm_anyvector_ty], 147 [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty], 148 [IntrNoMem]>; 149 class AdvSIMD_3VectorArg_Tied_Narrow_Intrinsic 150 : Intrinsic<[llvm_anyvector_ty], 151 [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty, 152 LLVMMatchType<1>], [IntrNoMem]>; 153 class AdvSIMD_3VectorArg_Scalar_Tied_Narrow_Intrinsic 154 : Intrinsic<[llvm_anyvector_ty], 155 [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty, llvm_i32_ty], 156 [IntrNoMem]>; 157 class AdvSIMD_CvtFxToFP_Intrinsic 158 : Intrinsic<[llvm_anyfloat_ty], [llvm_anyint_ty, llvm_i32_ty], 159 [IntrNoMem]>; 160 class AdvSIMD_CvtFPToFx_Intrinsic 161 : Intrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty, llvm_i32_ty], 162 [IntrNoMem]>; 163 164 class AdvSIMD_1Arg_Intrinsic 165 : Intrinsic<[llvm_any_ty], [LLVMMatchType<0>], [IntrNoMem]>; 166 167 class AdvSIMD_Dot_Intrinsic 168 : Intrinsic<[llvm_anyvector_ty], 169 [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>], 170 [IntrNoMem]>; 171 172 class AdvSIMD_FP16FML_Intrinsic 173 : Intrinsic<[llvm_anyvector_ty], 174 [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>], 175 [IntrNoMem]>; 176 177 class AdvSIMD_MatMul_Intrinsic 178 : Intrinsic<[llvm_anyvector_ty], 179 [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>], 180 [IntrNoMem]>; 181 182 class AdvSIMD_FML_Intrinsic 183 : Intrinsic<[llvm_anyvector_ty], 184 [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>], 185 [IntrNoMem]>; 186 187 class AdvSIMD_BF16FML_Intrinsic 188 : Intrinsic<[llvm_v4f32_ty], 189 [llvm_v4f32_ty, llvm_v8bf16_ty, llvm_v8bf16_ty], 190 [IntrNoMem]>; 191} 192 193// Arithmetic ops 194 195let TargetPrefix = "aarch64", IntrProperties = [IntrNoMem] in { 196 // Vector Add Across Lanes 197 def int_aarch64_neon_saddv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; 198 def int_aarch64_neon_uaddv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; 199 def int_aarch64_neon_faddv : AdvSIMD_1VectorArg_Float_Across_Intrinsic; 200 201 // Vector Long Add Across Lanes 202 def int_aarch64_neon_saddlv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; 203 def int_aarch64_neon_uaddlv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; 204 205 // Vector Halving Add 206 def int_aarch64_neon_shadd : AdvSIMD_2VectorArg_Intrinsic; 207 def int_aarch64_neon_uhadd : AdvSIMD_2VectorArg_Intrinsic; 208 209 // Vector Rounding Halving Add 210 def int_aarch64_neon_srhadd : AdvSIMD_2VectorArg_Intrinsic; 211 def int_aarch64_neon_urhadd : AdvSIMD_2VectorArg_Intrinsic; 212 213 // Vector Saturating Add 214 def int_aarch64_neon_sqadd : AdvSIMD_2IntArg_Intrinsic; 215 def int_aarch64_neon_suqadd : AdvSIMD_2IntArg_Intrinsic; 216 def int_aarch64_neon_usqadd : AdvSIMD_2IntArg_Intrinsic; 217 def int_aarch64_neon_uqadd : AdvSIMD_2IntArg_Intrinsic; 218 219 // Vector Add High-Half 220 // FIXME: this is a legacy intrinsic for aarch64_simd.h. Remove it when that 221 // header is no longer supported. 222 def int_aarch64_neon_addhn : AdvSIMD_2VectorArg_Narrow_Intrinsic; 223 224 // Vector Rounding Add High-Half 225 def int_aarch64_neon_raddhn : AdvSIMD_2VectorArg_Narrow_Intrinsic; 226 227 // Vector Saturating Doubling Multiply High 228 def int_aarch64_neon_sqdmulh : AdvSIMD_2IntArg_Intrinsic; 229 def int_aarch64_neon_sqdmulh_lane : AdvSIMD_2VectorArg_Lane_Intrinsic; 230 def int_aarch64_neon_sqdmulh_laneq : AdvSIMD_2VectorArg_Lane_Intrinsic; 231 232 // Vector Saturating Rounding Doubling Multiply High 233 def int_aarch64_neon_sqrdmulh : AdvSIMD_2IntArg_Intrinsic; 234 def int_aarch64_neon_sqrdmulh_lane : AdvSIMD_2VectorArg_Lane_Intrinsic; 235 def int_aarch64_neon_sqrdmulh_laneq : AdvSIMD_2VectorArg_Lane_Intrinsic; 236 237 // Vector Polynominal Multiply 238 def int_aarch64_neon_pmul : AdvSIMD_2VectorArg_Intrinsic; 239 240 // Vector Long Multiply 241 def int_aarch64_neon_smull : AdvSIMD_2VectorArg_Long_Intrinsic; 242 def int_aarch64_neon_umull : AdvSIMD_2VectorArg_Long_Intrinsic; 243 def int_aarch64_neon_pmull : AdvSIMD_2VectorArg_Long_Intrinsic; 244 245 // 64-bit polynomial multiply really returns an i128, which is not legal. Fake 246 // it with a v16i8. 247 def int_aarch64_neon_pmull64 : 248 Intrinsic<[llvm_v16i8_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>; 249 250 // Vector Extending Multiply 251 def int_aarch64_neon_fmulx : AdvSIMD_2FloatArg_Intrinsic { 252 let IntrProperties = [IntrNoMem, Commutative]; 253 } 254 255 // Vector Saturating Doubling Long Multiply 256 def int_aarch64_neon_sqdmull : AdvSIMD_2VectorArg_Long_Intrinsic; 257 def int_aarch64_neon_sqdmulls_scalar 258 : Intrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>; 259 260 // Vector Halving Subtract 261 def int_aarch64_neon_shsub : AdvSIMD_2VectorArg_Intrinsic; 262 def int_aarch64_neon_uhsub : AdvSIMD_2VectorArg_Intrinsic; 263 264 // Vector Saturating Subtract 265 def int_aarch64_neon_sqsub : AdvSIMD_2IntArg_Intrinsic; 266 def int_aarch64_neon_uqsub : AdvSIMD_2IntArg_Intrinsic; 267 268 // Vector Subtract High-Half 269 // FIXME: this is a legacy intrinsic for aarch64_simd.h. Remove it when that 270 // header is no longer supported. 271 def int_aarch64_neon_subhn : AdvSIMD_2VectorArg_Narrow_Intrinsic; 272 273 // Vector Rounding Subtract High-Half 274 def int_aarch64_neon_rsubhn : AdvSIMD_2VectorArg_Narrow_Intrinsic; 275 276 // Vector Compare Absolute Greater-than-or-equal 277 def int_aarch64_neon_facge : AdvSIMD_2Arg_FloatCompare_Intrinsic; 278 279 // Vector Compare Absolute Greater-than 280 def int_aarch64_neon_facgt : AdvSIMD_2Arg_FloatCompare_Intrinsic; 281 282 // Vector Absolute Difference 283 def int_aarch64_neon_sabd : AdvSIMD_2VectorArg_Intrinsic; 284 def int_aarch64_neon_uabd : AdvSIMD_2VectorArg_Intrinsic; 285 def int_aarch64_neon_fabd : AdvSIMD_2VectorArg_Intrinsic; 286 287 // Scalar Absolute Difference 288 def int_aarch64_sisd_fabd : AdvSIMD_2Scalar_Float_Intrinsic; 289 290 // Vector Max 291 def int_aarch64_neon_smax : AdvSIMD_2VectorArg_Intrinsic; 292 def int_aarch64_neon_umax : AdvSIMD_2VectorArg_Intrinsic; 293 def int_aarch64_neon_fmax : AdvSIMD_2FloatArg_Intrinsic; 294 def int_aarch64_neon_fmaxnmp : AdvSIMD_2VectorArg_Intrinsic; 295 296 // Vector Max Across Lanes 297 def int_aarch64_neon_smaxv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; 298 def int_aarch64_neon_umaxv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; 299 def int_aarch64_neon_fmaxv : AdvSIMD_1VectorArg_Float_Across_Intrinsic; 300 def int_aarch64_neon_fmaxnmv : AdvSIMD_1VectorArg_Float_Across_Intrinsic; 301 302 // Vector Min 303 def int_aarch64_neon_smin : AdvSIMD_2VectorArg_Intrinsic; 304 def int_aarch64_neon_umin : AdvSIMD_2VectorArg_Intrinsic; 305 def int_aarch64_neon_fmin : AdvSIMD_2FloatArg_Intrinsic; 306 def int_aarch64_neon_fminnmp : AdvSIMD_2VectorArg_Intrinsic; 307 308 // Vector Min/Max Number 309 def int_aarch64_neon_fminnm : AdvSIMD_2FloatArg_Intrinsic; 310 def int_aarch64_neon_fmaxnm : AdvSIMD_2FloatArg_Intrinsic; 311 312 // Vector Min Across Lanes 313 def int_aarch64_neon_sminv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; 314 def int_aarch64_neon_uminv : AdvSIMD_1VectorArg_Int_Across_Intrinsic; 315 def int_aarch64_neon_fminv : AdvSIMD_1VectorArg_Float_Across_Intrinsic; 316 def int_aarch64_neon_fminnmv : AdvSIMD_1VectorArg_Float_Across_Intrinsic; 317 318 // Pairwise Add 319 def int_aarch64_neon_addp : AdvSIMD_2VectorArg_Intrinsic; 320 def int_aarch64_neon_faddp : AdvSIMD_2VectorArg_Intrinsic; 321 322 // Long Pairwise Add 323 // FIXME: In theory, we shouldn't need intrinsics for saddlp or 324 // uaddlp, but tblgen's type inference currently can't handle the 325 // pattern fragments this ends up generating. 326 def int_aarch64_neon_saddlp : AdvSIMD_1VectorArg_Expand_Intrinsic; 327 def int_aarch64_neon_uaddlp : AdvSIMD_1VectorArg_Expand_Intrinsic; 328 329 // Folding Maximum 330 def int_aarch64_neon_smaxp : AdvSIMD_2VectorArg_Intrinsic; 331 def int_aarch64_neon_umaxp : AdvSIMD_2VectorArg_Intrinsic; 332 def int_aarch64_neon_fmaxp : AdvSIMD_2VectorArg_Intrinsic; 333 334 // Folding Minimum 335 def int_aarch64_neon_sminp : AdvSIMD_2VectorArg_Intrinsic; 336 def int_aarch64_neon_uminp : AdvSIMD_2VectorArg_Intrinsic; 337 def int_aarch64_neon_fminp : AdvSIMD_2VectorArg_Intrinsic; 338 339 // Reciprocal Estimate/Step 340 def int_aarch64_neon_frecps : AdvSIMD_2FloatArg_Intrinsic; 341 def int_aarch64_neon_frsqrts : AdvSIMD_2FloatArg_Intrinsic; 342 343 // Reciprocal Exponent 344 def int_aarch64_neon_frecpx : AdvSIMD_1FloatArg_Intrinsic; 345 346 // Vector Saturating Shift Left 347 def int_aarch64_neon_sqshl : AdvSIMD_2IntArg_Intrinsic; 348 def int_aarch64_neon_uqshl : AdvSIMD_2IntArg_Intrinsic; 349 350 // Vector Rounding Shift Left 351 def int_aarch64_neon_srshl : AdvSIMD_2IntArg_Intrinsic; 352 def int_aarch64_neon_urshl : AdvSIMD_2IntArg_Intrinsic; 353 354 // Vector Saturating Rounding Shift Left 355 def int_aarch64_neon_sqrshl : AdvSIMD_2IntArg_Intrinsic; 356 def int_aarch64_neon_uqrshl : AdvSIMD_2IntArg_Intrinsic; 357 358 // Vector Signed->Unsigned Shift Left by Constant 359 def int_aarch64_neon_sqshlu : AdvSIMD_2IntArg_Intrinsic; 360 361 // Vector Signed->Unsigned Narrowing Saturating Shift Right by Constant 362 def int_aarch64_neon_sqshrun : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic; 363 364 // Vector Signed->Unsigned Rounding Narrowing Saturating Shift Right by Const 365 def int_aarch64_neon_sqrshrun : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic; 366 367 // Vector Narrowing Shift Right by Constant 368 def int_aarch64_neon_sqshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic; 369 def int_aarch64_neon_uqshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic; 370 371 // Vector Rounding Narrowing Shift Right by Constant 372 def int_aarch64_neon_rshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic; 373 374 // Vector Rounding Narrowing Saturating Shift Right by Constant 375 def int_aarch64_neon_sqrshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic; 376 def int_aarch64_neon_uqrshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic; 377 378 // Vector Shift Left 379 def int_aarch64_neon_sshl : AdvSIMD_2IntArg_Intrinsic; 380 def int_aarch64_neon_ushl : AdvSIMD_2IntArg_Intrinsic; 381 382 // Vector Widening Shift Left by Constant 383 def int_aarch64_neon_shll : AdvSIMD_2VectorArg_Scalar_Wide_BySize_Intrinsic; 384 def int_aarch64_neon_sshll : AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic; 385 def int_aarch64_neon_ushll : AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic; 386 387 // Vector Shift Right by Constant and Insert 388 def int_aarch64_neon_vsri : AdvSIMD_3VectorArg_Scalar_Intrinsic; 389 390 // Vector Shift Left by Constant and Insert 391 def int_aarch64_neon_vsli : AdvSIMD_3VectorArg_Scalar_Intrinsic; 392 393 // Vector Saturating Narrow 394 def int_aarch64_neon_scalar_sqxtn: AdvSIMD_1IntArg_Narrow_Intrinsic; 395 def int_aarch64_neon_scalar_uqxtn : AdvSIMD_1IntArg_Narrow_Intrinsic; 396 def int_aarch64_neon_sqxtn : AdvSIMD_1VectorArg_Narrow_Intrinsic; 397 def int_aarch64_neon_uqxtn : AdvSIMD_1VectorArg_Narrow_Intrinsic; 398 399 // Vector Saturating Extract and Unsigned Narrow 400 def int_aarch64_neon_scalar_sqxtun : AdvSIMD_1IntArg_Narrow_Intrinsic; 401 def int_aarch64_neon_sqxtun : AdvSIMD_1VectorArg_Narrow_Intrinsic; 402 403 // Vector Absolute Value 404 def int_aarch64_neon_abs : AdvSIMD_1Arg_Intrinsic; 405 406 // Vector Saturating Absolute Value 407 def int_aarch64_neon_sqabs : AdvSIMD_1IntArg_Intrinsic; 408 409 // Vector Saturating Negation 410 def int_aarch64_neon_sqneg : AdvSIMD_1IntArg_Intrinsic; 411 412 // Vector Count Leading Sign Bits 413 def int_aarch64_neon_cls : AdvSIMD_1VectorArg_Intrinsic; 414 415 // Vector Reciprocal Estimate 416 def int_aarch64_neon_urecpe : AdvSIMD_1VectorArg_Intrinsic; 417 def int_aarch64_neon_frecpe : AdvSIMD_1FloatArg_Intrinsic; 418 419 // Vector Square Root Estimate 420 def int_aarch64_neon_ursqrte : AdvSIMD_1VectorArg_Intrinsic; 421 def int_aarch64_neon_frsqrte : AdvSIMD_1FloatArg_Intrinsic; 422 423 // Vector Bitwise Reverse 424 def int_aarch64_neon_rbit : AdvSIMD_1VectorArg_Intrinsic; 425 426 // Vector Conversions Between Half-Precision and Single-Precision. 427 def int_aarch64_neon_vcvtfp2hf 428 : Intrinsic<[llvm_v4i16_ty], [llvm_v4f32_ty], [IntrNoMem]>; 429 def int_aarch64_neon_vcvthf2fp 430 : Intrinsic<[llvm_v4f32_ty], [llvm_v4i16_ty], [IntrNoMem]>; 431 432 // Vector Conversions Between Floating-point and Fixed-point. 433 def int_aarch64_neon_vcvtfp2fxs : AdvSIMD_CvtFPToFx_Intrinsic; 434 def int_aarch64_neon_vcvtfp2fxu : AdvSIMD_CvtFPToFx_Intrinsic; 435 def int_aarch64_neon_vcvtfxs2fp : AdvSIMD_CvtFxToFP_Intrinsic; 436 def int_aarch64_neon_vcvtfxu2fp : AdvSIMD_CvtFxToFP_Intrinsic; 437 438 // Vector FP->Int Conversions 439 def int_aarch64_neon_fcvtas : AdvSIMD_FPToIntRounding_Intrinsic; 440 def int_aarch64_neon_fcvtau : AdvSIMD_FPToIntRounding_Intrinsic; 441 def int_aarch64_neon_fcvtms : AdvSIMD_FPToIntRounding_Intrinsic; 442 def int_aarch64_neon_fcvtmu : AdvSIMD_FPToIntRounding_Intrinsic; 443 def int_aarch64_neon_fcvtns : AdvSIMD_FPToIntRounding_Intrinsic; 444 def int_aarch64_neon_fcvtnu : AdvSIMD_FPToIntRounding_Intrinsic; 445 def int_aarch64_neon_fcvtps : AdvSIMD_FPToIntRounding_Intrinsic; 446 def int_aarch64_neon_fcvtpu : AdvSIMD_FPToIntRounding_Intrinsic; 447 def int_aarch64_neon_fcvtzs : AdvSIMD_FPToIntRounding_Intrinsic; 448 def int_aarch64_neon_fcvtzu : AdvSIMD_FPToIntRounding_Intrinsic; 449 450 // Vector FP Rounding: only ties to even is unrepresented by a normal 451 // intrinsic. 452 def int_aarch64_neon_frintn : AdvSIMD_1FloatArg_Intrinsic; 453 454 // Scalar FP->Int conversions 455 456 // Vector FP Inexact Narrowing 457 def int_aarch64_neon_fcvtxn : AdvSIMD_1VectorArg_Expand_Intrinsic; 458 459 // Scalar FP Inexact Narrowing 460 def int_aarch64_sisd_fcvtxn : Intrinsic<[llvm_float_ty], [llvm_double_ty], 461 [IntrNoMem]>; 462 463 // v8.2-A Dot Product 464 def int_aarch64_neon_udot : AdvSIMD_Dot_Intrinsic; 465 def int_aarch64_neon_sdot : AdvSIMD_Dot_Intrinsic; 466 467// v8.6-A Matrix Multiply Intrinsics 468 def int_aarch64_neon_ummla : AdvSIMD_MatMul_Intrinsic; 469 def int_aarch64_neon_smmla : AdvSIMD_MatMul_Intrinsic; 470 def int_aarch64_neon_usmmla : AdvSIMD_MatMul_Intrinsic; 471 def int_aarch64_neon_usdot : AdvSIMD_Dot_Intrinsic; 472 def int_aarch64_neon_bfdot : AdvSIMD_Dot_Intrinsic; 473 def int_aarch64_neon_bfmmla 474 : Intrinsic<[llvm_v4f32_ty], 475 [llvm_v4f32_ty, llvm_v8bf16_ty, llvm_v8bf16_ty], 476 [IntrNoMem]>; 477 def int_aarch64_neon_bfmlalb : AdvSIMD_BF16FML_Intrinsic; 478 def int_aarch64_neon_bfmlalt : AdvSIMD_BF16FML_Intrinsic; 479 480 481 // v8.6-A Bfloat Intrinsics 482 def int_aarch64_neon_bfcvt 483 : Intrinsic<[llvm_bfloat_ty], [llvm_float_ty], [IntrNoMem]>; 484 def int_aarch64_neon_bfcvtn 485 : Intrinsic<[llvm_v8bf16_ty], [llvm_v4f32_ty], [IntrNoMem]>; 486 def int_aarch64_neon_bfcvtn2 487 : Intrinsic<[llvm_v8bf16_ty], 488 [llvm_v8bf16_ty, llvm_v4f32_ty], 489 [IntrNoMem]>; 490 491 // v8.2-A FP16 Fused Multiply-Add Long 492 def int_aarch64_neon_fmlal : AdvSIMD_FP16FML_Intrinsic; 493 def int_aarch64_neon_fmlsl : AdvSIMD_FP16FML_Intrinsic; 494 def int_aarch64_neon_fmlal2 : AdvSIMD_FP16FML_Intrinsic; 495 def int_aarch64_neon_fmlsl2 : AdvSIMD_FP16FML_Intrinsic; 496 497 // v8.3-A Floating-point complex add 498 def int_aarch64_neon_vcadd_rot90 : AdvSIMD_2VectorArg_Intrinsic; 499 def int_aarch64_neon_vcadd_rot270 : AdvSIMD_2VectorArg_Intrinsic; 500} 501 502let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". 503 class AdvSIMD_2Vector2Index_Intrinsic 504 : Intrinsic<[llvm_anyvector_ty], 505 [llvm_anyvector_ty, llvm_i64_ty, LLVMMatchType<0>, llvm_i64_ty], 506 [IntrNoMem]>; 507} 508 509// Vector element to element moves 510def int_aarch64_neon_vcopy_lane: AdvSIMD_2Vector2Index_Intrinsic; 511 512let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". 513 class AdvSIMD_1Vec_Load_Intrinsic 514 : Intrinsic<[llvm_anyvector_ty], [LLVMAnyPointerType<LLVMMatchType<0>>], 515 [IntrReadMem, IntrArgMemOnly]>; 516 class AdvSIMD_1Vec_Store_Lane_Intrinsic 517 : Intrinsic<[], [llvm_anyvector_ty, llvm_i64_ty, llvm_anyptr_ty], 518 [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>; 519 520 class AdvSIMD_2Vec_Load_Intrinsic 521 : Intrinsic<[LLVMMatchType<0>, llvm_anyvector_ty], 522 [LLVMAnyPointerType<LLVMMatchType<0>>], 523 [IntrReadMem, IntrArgMemOnly]>; 524 class AdvSIMD_2Vec_Load_Lane_Intrinsic 525 : Intrinsic<[LLVMMatchType<0>, LLVMMatchType<0>], 526 [LLVMMatchType<0>, llvm_anyvector_ty, 527 llvm_i64_ty, llvm_anyptr_ty], 528 [IntrReadMem, IntrArgMemOnly]>; 529 class AdvSIMD_2Vec_Store_Intrinsic 530 : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>, 531 LLVMAnyPointerType<LLVMMatchType<0>>], 532 [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>; 533 class AdvSIMD_2Vec_Store_Lane_Intrinsic 534 : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>, 535 llvm_i64_ty, llvm_anyptr_ty], 536 [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>; 537 538 class AdvSIMD_3Vec_Load_Intrinsic 539 : Intrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty], 540 [LLVMAnyPointerType<LLVMMatchType<0>>], 541 [IntrReadMem, IntrArgMemOnly]>; 542 class AdvSIMD_3Vec_Load_Lane_Intrinsic 543 : Intrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>], 544 [LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty, 545 llvm_i64_ty, llvm_anyptr_ty], 546 [IntrReadMem, IntrArgMemOnly]>; 547 class AdvSIMD_3Vec_Store_Intrinsic 548 : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>, 549 LLVMMatchType<0>, LLVMAnyPointerType<LLVMMatchType<0>>], 550 [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>; 551 class AdvSIMD_3Vec_Store_Lane_Intrinsic 552 : Intrinsic<[], [llvm_anyvector_ty, 553 LLVMMatchType<0>, LLVMMatchType<0>, 554 llvm_i64_ty, llvm_anyptr_ty], 555 [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>; 556 557 class AdvSIMD_4Vec_Load_Intrinsic 558 : Intrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, 559 LLVMMatchType<0>, llvm_anyvector_ty], 560 [LLVMAnyPointerType<LLVMMatchType<0>>], 561 [IntrReadMem, IntrArgMemOnly]>; 562 class AdvSIMD_4Vec_Load_Lane_Intrinsic 563 : Intrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, 564 LLVMMatchType<0>, LLVMMatchType<0>], 565 [LLVMMatchType<0>, LLVMMatchType<0>, 566 LLVMMatchType<0>, llvm_anyvector_ty, 567 llvm_i64_ty, llvm_anyptr_ty], 568 [IntrReadMem, IntrArgMemOnly]>; 569 class AdvSIMD_4Vec_Store_Intrinsic 570 : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>, 571 LLVMMatchType<0>, LLVMMatchType<0>, 572 LLVMAnyPointerType<LLVMMatchType<0>>], 573 [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>; 574 class AdvSIMD_4Vec_Store_Lane_Intrinsic 575 : Intrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>, 576 LLVMMatchType<0>, LLVMMatchType<0>, 577 llvm_i64_ty, llvm_anyptr_ty], 578 [IntrArgMemOnly, NoCapture<ArgIndex<5>>]>; 579} 580 581// Memory ops 582 583def int_aarch64_neon_ld1x2 : AdvSIMD_2Vec_Load_Intrinsic; 584def int_aarch64_neon_ld1x3 : AdvSIMD_3Vec_Load_Intrinsic; 585def int_aarch64_neon_ld1x4 : AdvSIMD_4Vec_Load_Intrinsic; 586 587def int_aarch64_neon_st1x2 : AdvSIMD_2Vec_Store_Intrinsic; 588def int_aarch64_neon_st1x3 : AdvSIMD_3Vec_Store_Intrinsic; 589def int_aarch64_neon_st1x4 : AdvSIMD_4Vec_Store_Intrinsic; 590 591def int_aarch64_neon_ld2 : AdvSIMD_2Vec_Load_Intrinsic; 592def int_aarch64_neon_ld3 : AdvSIMD_3Vec_Load_Intrinsic; 593def int_aarch64_neon_ld4 : AdvSIMD_4Vec_Load_Intrinsic; 594 595def int_aarch64_neon_ld2lane : AdvSIMD_2Vec_Load_Lane_Intrinsic; 596def int_aarch64_neon_ld3lane : AdvSIMD_3Vec_Load_Lane_Intrinsic; 597def int_aarch64_neon_ld4lane : AdvSIMD_4Vec_Load_Lane_Intrinsic; 598 599def int_aarch64_neon_ld2r : AdvSIMD_2Vec_Load_Intrinsic; 600def int_aarch64_neon_ld3r : AdvSIMD_3Vec_Load_Intrinsic; 601def int_aarch64_neon_ld4r : AdvSIMD_4Vec_Load_Intrinsic; 602 603def int_aarch64_neon_st2 : AdvSIMD_2Vec_Store_Intrinsic; 604def int_aarch64_neon_st3 : AdvSIMD_3Vec_Store_Intrinsic; 605def int_aarch64_neon_st4 : AdvSIMD_4Vec_Store_Intrinsic; 606 607def int_aarch64_neon_st2lane : AdvSIMD_2Vec_Store_Lane_Intrinsic; 608def int_aarch64_neon_st3lane : AdvSIMD_3Vec_Store_Lane_Intrinsic; 609def int_aarch64_neon_st4lane : AdvSIMD_4Vec_Store_Lane_Intrinsic; 610 611let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". 612 class AdvSIMD_Tbl1_Intrinsic 613 : Intrinsic<[llvm_anyvector_ty], [llvm_v16i8_ty, LLVMMatchType<0>], 614 [IntrNoMem]>; 615 class AdvSIMD_Tbl2_Intrinsic 616 : Intrinsic<[llvm_anyvector_ty], 617 [llvm_v16i8_ty, llvm_v16i8_ty, LLVMMatchType<0>], [IntrNoMem]>; 618 class AdvSIMD_Tbl3_Intrinsic 619 : Intrinsic<[llvm_anyvector_ty], 620 [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty, 621 LLVMMatchType<0>], 622 [IntrNoMem]>; 623 class AdvSIMD_Tbl4_Intrinsic 624 : Intrinsic<[llvm_anyvector_ty], 625 [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty, 626 LLVMMatchType<0>], 627 [IntrNoMem]>; 628 629 class AdvSIMD_Tbx1_Intrinsic 630 : Intrinsic<[llvm_anyvector_ty], 631 [LLVMMatchType<0>, llvm_v16i8_ty, LLVMMatchType<0>], 632 [IntrNoMem]>; 633 class AdvSIMD_Tbx2_Intrinsic 634 : Intrinsic<[llvm_anyvector_ty], 635 [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty, 636 LLVMMatchType<0>], 637 [IntrNoMem]>; 638 class AdvSIMD_Tbx3_Intrinsic 639 : Intrinsic<[llvm_anyvector_ty], 640 [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty, 641 llvm_v16i8_ty, LLVMMatchType<0>], 642 [IntrNoMem]>; 643 class AdvSIMD_Tbx4_Intrinsic 644 : Intrinsic<[llvm_anyvector_ty], 645 [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty, 646 llvm_v16i8_ty, llvm_v16i8_ty, LLVMMatchType<0>], 647 [IntrNoMem]>; 648} 649def int_aarch64_neon_tbl1 : AdvSIMD_Tbl1_Intrinsic; 650def int_aarch64_neon_tbl2 : AdvSIMD_Tbl2_Intrinsic; 651def int_aarch64_neon_tbl3 : AdvSIMD_Tbl3_Intrinsic; 652def int_aarch64_neon_tbl4 : AdvSIMD_Tbl4_Intrinsic; 653 654def int_aarch64_neon_tbx1 : AdvSIMD_Tbx1_Intrinsic; 655def int_aarch64_neon_tbx2 : AdvSIMD_Tbx2_Intrinsic; 656def int_aarch64_neon_tbx3 : AdvSIMD_Tbx3_Intrinsic; 657def int_aarch64_neon_tbx4 : AdvSIMD_Tbx4_Intrinsic; 658 659let TargetPrefix = "aarch64" in { 660 class FPCR_Get_Intrinsic 661 : Intrinsic<[llvm_i64_ty], [], [IntrNoMem, IntrHasSideEffects]>; 662} 663 664// FPCR 665def int_aarch64_get_fpcr : FPCR_Get_Intrinsic; 666 667let TargetPrefix = "aarch64" in { 668 class Crypto_AES_DataKey_Intrinsic 669 : Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>; 670 671 class Crypto_AES_Data_Intrinsic 672 : Intrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty], [IntrNoMem]>; 673 674 // SHA intrinsic taking 5 words of the hash (v4i32, i32) and 4 of the schedule 675 // (v4i32). 676 class Crypto_SHA_5Hash4Schedule_Intrinsic 677 : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty, llvm_v4i32_ty], 678 [IntrNoMem]>; 679 680 // SHA intrinsic taking 5 words of the hash (v4i32, i32) and 4 of the schedule 681 // (v4i32). 682 class Crypto_SHA_1Hash_Intrinsic 683 : Intrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem]>; 684 685 // SHA intrinsic taking 8 words of the schedule 686 class Crypto_SHA_8Schedule_Intrinsic 687 : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>; 688 689 // SHA intrinsic taking 12 words of the schedule 690 class Crypto_SHA_12Schedule_Intrinsic 691 : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty], 692 [IntrNoMem]>; 693 694 // SHA intrinsic taking 8 words of the hash and 4 of the schedule. 695 class Crypto_SHA_8Hash4Schedule_Intrinsic 696 : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty], 697 [IntrNoMem]>; 698} 699 700// AES 701def int_aarch64_crypto_aese : Crypto_AES_DataKey_Intrinsic; 702def int_aarch64_crypto_aesd : Crypto_AES_DataKey_Intrinsic; 703def int_aarch64_crypto_aesmc : Crypto_AES_Data_Intrinsic; 704def int_aarch64_crypto_aesimc : Crypto_AES_Data_Intrinsic; 705 706// SHA1 707def int_aarch64_crypto_sha1c : Crypto_SHA_5Hash4Schedule_Intrinsic; 708def int_aarch64_crypto_sha1p : Crypto_SHA_5Hash4Schedule_Intrinsic; 709def int_aarch64_crypto_sha1m : Crypto_SHA_5Hash4Schedule_Intrinsic; 710def int_aarch64_crypto_sha1h : Crypto_SHA_1Hash_Intrinsic; 711 712def int_aarch64_crypto_sha1su0 : Crypto_SHA_12Schedule_Intrinsic; 713def int_aarch64_crypto_sha1su1 : Crypto_SHA_8Schedule_Intrinsic; 714 715// SHA256 716def int_aarch64_crypto_sha256h : Crypto_SHA_8Hash4Schedule_Intrinsic; 717def int_aarch64_crypto_sha256h2 : Crypto_SHA_8Hash4Schedule_Intrinsic; 718def int_aarch64_crypto_sha256su0 : Crypto_SHA_8Schedule_Intrinsic; 719def int_aarch64_crypto_sha256su1 : Crypto_SHA_12Schedule_Intrinsic; 720 721//===----------------------------------------------------------------------===// 722// CRC32 723 724let TargetPrefix = "aarch64" in { 725 726def int_aarch64_crc32b : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], 727 [IntrNoMem]>; 728def int_aarch64_crc32cb : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], 729 [IntrNoMem]>; 730def int_aarch64_crc32h : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], 731 [IntrNoMem]>; 732def int_aarch64_crc32ch : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], 733 [IntrNoMem]>; 734def int_aarch64_crc32w : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], 735 [IntrNoMem]>; 736def int_aarch64_crc32cw : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty], 737 [IntrNoMem]>; 738def int_aarch64_crc32x : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty], 739 [IntrNoMem]>; 740def int_aarch64_crc32cx : Intrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty], 741 [IntrNoMem]>; 742} 743 744//===----------------------------------------------------------------------===// 745// Memory Tagging Extensions (MTE) Intrinsics 746let TargetPrefix = "aarch64" in { 747def int_aarch64_irg : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i64_ty], 748 [IntrNoMem, IntrHasSideEffects]>; 749def int_aarch64_addg : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i64_ty], 750 [IntrNoMem]>; 751def int_aarch64_gmi : Intrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_i64_ty], 752 [IntrNoMem]>; 753def int_aarch64_ldg : Intrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_ptr_ty], 754 [IntrReadMem]>; 755def int_aarch64_stg : Intrinsic<[], [llvm_ptr_ty, llvm_ptr_ty], 756 [IntrWriteMem]>; 757def int_aarch64_subp : Intrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_ptr_ty], 758 [IntrNoMem]>; 759 760// The following are codegen-only intrinsics for stack instrumentation. 761 762// Generate a randomly tagged stack base pointer. 763def int_aarch64_irg_sp : Intrinsic<[llvm_ptr_ty], [llvm_i64_ty], 764 [IntrNoMem, IntrHasSideEffects]>; 765 766// Transfer pointer tag with offset. 767// ptr1 = tagp(ptr0, baseptr, tag_offset) returns a pointer where 768// * address is the address in ptr0 769// * tag is a function of (tag in baseptr, tag_offset). 770// ** Beware, this is not the same function as implemented by the ADDG instruction! 771// Backend optimizations may change tag_offset; the only guarantee is that calls 772// to tagp with the same pair of (baseptr, tag_offset) will produce pointers 773// with the same tag value, assuming the set of excluded tags has not changed. 774// Address bits in baseptr and tag bits in ptr0 are ignored. 775// When offset between ptr0 and baseptr is a compile time constant, this can be emitted as 776// ADDG ptr1, baseptr, (ptr0 - baseptr), tag_offset 777// It is intended that ptr0 is an alloca address, and baseptr is the direct output of llvm.aarch64.irg.sp. 778def int_aarch64_tagp : Intrinsic<[llvm_anyptr_ty], [LLVMMatchType<0>, llvm_ptr_ty, llvm_i64_ty], 779 [IntrNoMem, ImmArg<ArgIndex<2>>]>; 780 781// Update allocation tags for the memory range to match the tag in the pointer argument. 782def int_aarch64_settag : Intrinsic<[], [llvm_ptr_ty, llvm_i64_ty], 783 [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>; 784 785// Update allocation tags for the memory range to match the tag in the pointer argument, 786// and set memory contents to zero. 787def int_aarch64_settag_zero : Intrinsic<[], [llvm_ptr_ty, llvm_i64_ty], 788 [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>; 789 790// Update allocation tags for 16-aligned, 16-sized memory region, and store a pair 8-byte values. 791def int_aarch64_stgp : Intrinsic<[], [llvm_ptr_ty, llvm_i64_ty, llvm_i64_ty], 792 [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>; 793} 794 795// Transactional Memory Extension (TME) Intrinsics 796let TargetPrefix = "aarch64" in { 797def int_aarch64_tstart : GCCBuiltin<"__builtin_arm_tstart">, 798 Intrinsic<[llvm_i64_ty]>; 799 800def int_aarch64_tcommit : GCCBuiltin<"__builtin_arm_tcommit">, Intrinsic<[]>; 801 802def int_aarch64_tcancel : GCCBuiltin<"__builtin_arm_tcancel">, 803 Intrinsic<[], [llvm_i64_ty], [ImmArg<ArgIndex<0>>]>; 804 805def int_aarch64_ttest : GCCBuiltin<"__builtin_arm_ttest">, 806 Intrinsic<[llvm_i64_ty], [], 807 [IntrNoMem, IntrHasSideEffects]>; 808} 809 810def llvm_nxv2i1_ty : LLVMType<nxv2i1>; 811def llvm_nxv4i1_ty : LLVMType<nxv4i1>; 812def llvm_nxv8i1_ty : LLVMType<nxv8i1>; 813def llvm_nxv16i1_ty : LLVMType<nxv16i1>; 814def llvm_nxv16i8_ty : LLVMType<nxv16i8>; 815def llvm_nxv4i32_ty : LLVMType<nxv4i32>; 816def llvm_nxv2i64_ty : LLVMType<nxv2i64>; 817def llvm_nxv8f16_ty : LLVMType<nxv8f16>; 818def llvm_nxv8bf16_ty : LLVMType<nxv8bf16>; 819def llvm_nxv4f32_ty : LLVMType<nxv4f32>; 820def llvm_nxv2f64_ty : LLVMType<nxv2f64>; 821 822let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". 823 824 class AdvSIMD_SVE_Create_2Vector_Tuple 825 : Intrinsic<[llvm_anyvector_ty], 826 [llvm_anyvector_ty, LLVMMatchType<1>], 827 [IntrReadMem]>; 828 829 class AdvSIMD_SVE_Create_3Vector_Tuple 830 : Intrinsic<[llvm_anyvector_ty], 831 [llvm_anyvector_ty, LLVMMatchType<1>, LLVMMatchType<1>], 832 [IntrReadMem]>; 833 834 class AdvSIMD_SVE_Create_4Vector_Tuple 835 : Intrinsic<[llvm_anyvector_ty], 836 [llvm_anyvector_ty, LLVMMatchType<1>, LLVMMatchType<1>, 837 LLVMMatchType<1>], 838 [IntrReadMem]>; 839 840 class AdvSIMD_SVE_Set_Vector_Tuple 841 : Intrinsic<[llvm_anyvector_ty], 842 [LLVMMatchType<0>, llvm_i32_ty, llvm_anyvector_ty], 843 [IntrReadMem, ImmArg<ArgIndex<1>>]>; 844 845 class AdvSIMD_SVE_Get_Vector_Tuple 846 : Intrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, llvm_i32_ty], 847 [IntrReadMem, IntrArgMemOnly, ImmArg<ArgIndex<1>>]>; 848 849 class AdvSIMD_ManyVec_PredLoad_Intrinsic 850 : Intrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, LLVMPointerToElt<0>], 851 [IntrReadMem, IntrArgMemOnly]>; 852 853 class AdvSIMD_1Vec_PredLoad_Intrinsic 854 : Intrinsic<[llvm_anyvector_ty], 855 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 856 LLVMPointerToElt<0>], 857 [IntrReadMem, IntrArgMemOnly]>; 858 859 class AdvSIMD_1Vec_PredStore_Intrinsic 860 : Intrinsic<[], 861 [llvm_anyvector_ty, 862 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 863 LLVMPointerToElt<0>], 864 [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>; 865 866 class AdvSIMD_2Vec_PredStore_Intrinsic 867 : Intrinsic<[], 868 [llvm_anyvector_ty, LLVMMatchType<0>, 869 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMPointerToElt<0>], 870 [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>; 871 872 class AdvSIMD_3Vec_PredStore_Intrinsic 873 : Intrinsic<[], 874 [llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, 875 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMPointerToElt<0>], 876 [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>; 877 878 class AdvSIMD_4Vec_PredStore_Intrinsic 879 : Intrinsic<[], 880 [llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>, 881 LLVMMatchType<0>, 882 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMPointerToElt<0>], 883 [IntrArgMemOnly, NoCapture<ArgIndex<5>>]>; 884 885 class AdvSIMD_SVE_Index_Intrinsic 886 : Intrinsic<[llvm_anyvector_ty], 887 [LLVMVectorElementType<0>, 888 LLVMVectorElementType<0>], 889 [IntrNoMem]>; 890 891 class AdvSIMD_Merged1VectorArg_Intrinsic 892 : Intrinsic<[llvm_anyvector_ty], 893 [LLVMMatchType<0>, 894 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 895 LLVMMatchType<0>], 896 [IntrNoMem]>; 897 898 class AdvSIMD_2VectorArgIndexed_Intrinsic 899 : Intrinsic<[llvm_anyvector_ty], 900 [LLVMMatchType<0>, 901 LLVMMatchType<0>, 902 llvm_i32_ty], 903 [IntrNoMem, ImmArg<ArgIndex<2>>]>; 904 905 class AdvSIMD_3VectorArgIndexed_Intrinsic 906 : Intrinsic<[llvm_anyvector_ty], 907 [LLVMMatchType<0>, 908 LLVMMatchType<0>, 909 LLVMMatchType<0>, 910 llvm_i32_ty], 911 [IntrNoMem, ImmArg<ArgIndex<3>>]>; 912 913 class AdvSIMD_Pred1VectorArg_Intrinsic 914 : Intrinsic<[llvm_anyvector_ty], 915 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 916 LLVMMatchType<0>], 917 [IntrNoMem]>; 918 919 class AdvSIMD_Pred2VectorArg_Intrinsic 920 : Intrinsic<[llvm_anyvector_ty], 921 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 922 LLVMMatchType<0>, 923 LLVMMatchType<0>], 924 [IntrNoMem]>; 925 926 class AdvSIMD_Pred3VectorArg_Intrinsic 927 : Intrinsic<[llvm_anyvector_ty], 928 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 929 LLVMMatchType<0>, 930 LLVMMatchType<0>, 931 LLVMMatchType<0>], 932 [IntrNoMem]>; 933 934 class AdvSIMD_SVE_Compare_Intrinsic 935 : Intrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], 936 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 937 llvm_anyvector_ty, 938 LLVMMatchType<0>], 939 [IntrNoMem]>; 940 941 class AdvSIMD_SVE_CompareWide_Intrinsic 942 : Intrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], 943 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 944 llvm_anyvector_ty, 945 llvm_nxv2i64_ty], 946 [IntrNoMem]>; 947 948 class AdvSIMD_SVE_Saturating_Intrinsic 949 : Intrinsic<[llvm_anyvector_ty], 950 [LLVMMatchType<0>, 951 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>], 952 [IntrNoMem]>; 953 954 class AdvSIMD_SVE_SaturatingWithPattern_Intrinsic 955 : Intrinsic<[llvm_anyvector_ty], 956 [LLVMMatchType<0>, 957 llvm_i32_ty, 958 llvm_i32_ty], 959 [IntrNoMem, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>; 960 961 class AdvSIMD_SVE_Saturating_N_Intrinsic<LLVMType T> 962 : Intrinsic<[T], 963 [T, llvm_anyvector_ty], 964 [IntrNoMem]>; 965 966 class AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<LLVMType T> 967 : Intrinsic<[T], 968 [T, llvm_i32_ty, llvm_i32_ty], 969 [IntrNoMem, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>; 970 971 class AdvSIMD_SVE_CNT_Intrinsic 972 : Intrinsic<[LLVMVectorOfBitcastsToInt<0>], 973 [LLVMVectorOfBitcastsToInt<0>, 974 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 975 llvm_anyvector_ty], 976 [IntrNoMem]>; 977 978 class AdvSIMD_SVE_ReduceWithInit_Intrinsic 979 : Intrinsic<[LLVMVectorElementType<0>], 980 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 981 LLVMVectorElementType<0>, 982 llvm_anyvector_ty], 983 [IntrNoMem]>; 984 985 class AdvSIMD_SVE_ShiftByImm_Intrinsic 986 : Intrinsic<[llvm_anyvector_ty], 987 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 988 LLVMMatchType<0>, 989 llvm_i32_ty], 990 [IntrNoMem, ImmArg<ArgIndex<2>>]>; 991 992 class AdvSIMD_SVE_ShiftWide_Intrinsic 993 : Intrinsic<[llvm_anyvector_ty], 994 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 995 LLVMMatchType<0>, 996 llvm_nxv2i64_ty], 997 [IntrNoMem]>; 998 999 class AdvSIMD_SVE_Unpack_Intrinsic 1000 : Intrinsic<[llvm_anyvector_ty], 1001 [LLVMSubdivide2VectorType<0>], 1002 [IntrNoMem]>; 1003 1004 class AdvSIMD_SVE_CADD_Intrinsic 1005 : Intrinsic<[llvm_anyvector_ty], 1006 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1007 LLVMMatchType<0>, 1008 LLVMMatchType<0>, 1009 llvm_i32_ty], 1010 [IntrNoMem, ImmArg<ArgIndex<3>>]>; 1011 1012 class AdvSIMD_SVE_CMLA_Intrinsic 1013 : Intrinsic<[llvm_anyvector_ty], 1014 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1015 LLVMMatchType<0>, 1016 LLVMMatchType<0>, 1017 LLVMMatchType<0>, 1018 llvm_i32_ty], 1019 [IntrNoMem, ImmArg<ArgIndex<4>>]>; 1020 1021 class AdvSIMD_SVE_CMLA_LANE_Intrinsic 1022 : Intrinsic<[llvm_anyvector_ty], 1023 [LLVMMatchType<0>, 1024 LLVMMatchType<0>, 1025 LLVMMatchType<0>, 1026 llvm_i32_ty, 1027 llvm_i32_ty], 1028 [IntrNoMem, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>]>; 1029 1030 class AdvSIMD_SVE_DUP_Intrinsic 1031 : Intrinsic<[llvm_anyvector_ty], 1032 [LLVMMatchType<0>, 1033 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1034 LLVMVectorElementType<0>], 1035 [IntrNoMem]>; 1036 1037 class AdvSIMD_SVE_DUP_Unpred_Intrinsic 1038 : Intrinsic<[llvm_anyvector_ty], [LLVMVectorElementType<0>], 1039 [IntrNoMem]>; 1040 1041 class AdvSIMD_SVE_DUPQ_Intrinsic 1042 : Intrinsic<[llvm_anyvector_ty], 1043 [LLVMMatchType<0>, 1044 llvm_i64_ty], 1045 [IntrNoMem]>; 1046 1047 class AdvSIMD_SVE_EXPA_Intrinsic 1048 : Intrinsic<[llvm_anyvector_ty], 1049 [LLVMVectorOfBitcastsToInt<0>], 1050 [IntrNoMem]>; 1051 1052 class AdvSIMD_SVE_FCVT_Intrinsic 1053 : Intrinsic<[llvm_anyvector_ty], 1054 [LLVMMatchType<0>, 1055 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1056 llvm_anyvector_ty], 1057 [IntrNoMem]>; 1058 1059 class AdvSIMD_SVE_FCVTZS_Intrinsic 1060 : Intrinsic<[llvm_anyvector_ty], 1061 [LLVMVectorOfBitcastsToInt<0>, 1062 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1063 llvm_anyvector_ty], 1064 [IntrNoMem]>; 1065 1066 class AdvSIMD_SVE_INSR_Intrinsic 1067 : Intrinsic<[llvm_anyvector_ty], 1068 [LLVMMatchType<0>, 1069 LLVMVectorElementType<0>], 1070 [IntrNoMem]>; 1071 1072 class AdvSIMD_SVE_PTRUE_Intrinsic 1073 : Intrinsic<[llvm_anyvector_ty], 1074 [llvm_i32_ty], 1075 [IntrNoMem, ImmArg<ArgIndex<0>>]>; 1076 1077 class AdvSIMD_SVE_PUNPKHI_Intrinsic 1078 : Intrinsic<[LLVMHalfElementsVectorType<0>], 1079 [llvm_anyvector_ty], 1080 [IntrNoMem]>; 1081 1082 class AdvSIMD_SVE_SCALE_Intrinsic 1083 : Intrinsic<[llvm_anyvector_ty], 1084 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1085 LLVMMatchType<0>, 1086 LLVMVectorOfBitcastsToInt<0>], 1087 [IntrNoMem]>; 1088 1089 class AdvSIMD_SVE_SCVTF_Intrinsic 1090 : Intrinsic<[llvm_anyvector_ty], 1091 [LLVMMatchType<0>, 1092 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1093 llvm_anyvector_ty], 1094 [IntrNoMem]>; 1095 1096 class AdvSIMD_SVE_TSMUL_Intrinsic 1097 : Intrinsic<[llvm_anyvector_ty], 1098 [LLVMMatchType<0>, 1099 LLVMVectorOfBitcastsToInt<0>], 1100 [IntrNoMem]>; 1101 1102 class AdvSIMD_SVE_CNTB_Intrinsic 1103 : Intrinsic<[llvm_i64_ty], 1104 [llvm_i32_ty], 1105 [IntrNoMem, ImmArg<ArgIndex<0>>]>; 1106 1107 class AdvSIMD_SVE_CNTP_Intrinsic 1108 : Intrinsic<[llvm_i64_ty], 1109 [llvm_anyvector_ty, LLVMMatchType<0>], 1110 [IntrNoMem]>; 1111 1112 class AdvSIMD_SVE_DOT_Intrinsic 1113 : Intrinsic<[llvm_anyvector_ty], 1114 [LLVMMatchType<0>, 1115 LLVMSubdivide4VectorType<0>, 1116 LLVMSubdivide4VectorType<0>], 1117 [IntrNoMem]>; 1118 1119 class AdvSIMD_SVE_DOT_Indexed_Intrinsic 1120 : Intrinsic<[llvm_anyvector_ty], 1121 [LLVMMatchType<0>, 1122 LLVMSubdivide4VectorType<0>, 1123 LLVMSubdivide4VectorType<0>, 1124 llvm_i32_ty], 1125 [IntrNoMem, ImmArg<ArgIndex<3>>]>; 1126 1127 class AdvSIMD_SVE_PTEST_Intrinsic 1128 : Intrinsic<[llvm_i1_ty], 1129 [llvm_anyvector_ty, 1130 LLVMMatchType<0>], 1131 [IntrNoMem]>; 1132 1133 class AdvSIMD_SVE_TBL_Intrinsic 1134 : Intrinsic<[llvm_anyvector_ty], 1135 [LLVMMatchType<0>, 1136 LLVMVectorOfBitcastsToInt<0>], 1137 [IntrNoMem]>; 1138 1139 class AdvSIMD_SVE2_TBX_Intrinsic 1140 : Intrinsic<[llvm_anyvector_ty], 1141 [LLVMMatchType<0>, 1142 LLVMMatchType<0>, 1143 LLVMVectorOfBitcastsToInt<0>], 1144 [IntrNoMem]>; 1145 1146 class SVE2_1VectorArg_Long_Intrinsic 1147 : Intrinsic<[llvm_anyvector_ty], 1148 [LLVMSubdivide2VectorType<0>, 1149 llvm_i32_ty], 1150 [IntrNoMem, ImmArg<ArgIndex<1>>]>; 1151 1152 class SVE2_2VectorArg_Long_Intrinsic 1153 : Intrinsic<[llvm_anyvector_ty], 1154 [LLVMSubdivide2VectorType<0>, 1155 LLVMSubdivide2VectorType<0>], 1156 [IntrNoMem]>; 1157 1158 class SVE2_2VectorArgIndexed_Long_Intrinsic 1159 : Intrinsic<[llvm_anyvector_ty], 1160 [LLVMSubdivide2VectorType<0>, 1161 LLVMSubdivide2VectorType<0>, 1162 llvm_i32_ty], 1163 [IntrNoMem, ImmArg<ArgIndex<2>>]>; 1164 1165 class SVE2_2VectorArg_Wide_Intrinsic 1166 : Intrinsic<[llvm_anyvector_ty], 1167 [LLVMMatchType<0>, 1168 LLVMSubdivide2VectorType<0>], 1169 [IntrNoMem]>; 1170 1171 class SVE2_2VectorArg_Pred_Long_Intrinsic 1172 : Intrinsic<[llvm_anyvector_ty], 1173 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1174 LLVMMatchType<0>, 1175 LLVMSubdivide2VectorType<0>], 1176 [IntrNoMem]>; 1177 1178 class SVE2_3VectorArg_Long_Intrinsic 1179 : Intrinsic<[llvm_anyvector_ty], 1180 [LLVMMatchType<0>, 1181 LLVMSubdivide2VectorType<0>, 1182 LLVMSubdivide2VectorType<0>], 1183 [IntrNoMem]>; 1184 1185 class SVE2_3VectorArgIndexed_Long_Intrinsic 1186 : Intrinsic<[llvm_anyvector_ty], 1187 [LLVMMatchType<0>, 1188 LLVMSubdivide2VectorType<0>, 1189 LLVMSubdivide2VectorType<0>, 1190 llvm_i32_ty], 1191 [IntrNoMem, ImmArg<ArgIndex<3>>]>; 1192 1193 class SVE2_1VectorArg_Narrowing_Intrinsic 1194 : Intrinsic<[LLVMSubdivide2VectorType<0>], 1195 [llvm_anyvector_ty], 1196 [IntrNoMem]>; 1197 1198 class SVE2_Merged1VectorArg_Narrowing_Intrinsic 1199 : Intrinsic<[LLVMSubdivide2VectorType<0>], 1200 [LLVMSubdivide2VectorType<0>, 1201 llvm_anyvector_ty], 1202 [IntrNoMem]>; 1203 class SVE2_2VectorArg_Narrowing_Intrinsic 1204 : Intrinsic< 1205 [LLVMSubdivide2VectorType<0>], 1206 [llvm_anyvector_ty, LLVMMatchType<0>], 1207 [IntrNoMem]>; 1208 1209 class SVE2_Merged2VectorArg_Narrowing_Intrinsic 1210 : Intrinsic< 1211 [LLVMSubdivide2VectorType<0>], 1212 [LLVMSubdivide2VectorType<0>, llvm_anyvector_ty, LLVMMatchType<0>], 1213 [IntrNoMem]>; 1214 1215 class SVE2_1VectorArg_Imm_Narrowing_Intrinsic 1216 : Intrinsic<[LLVMSubdivide2VectorType<0>], 1217 [llvm_anyvector_ty, llvm_i32_ty], 1218 [IntrNoMem, ImmArg<ArgIndex<1>>]>; 1219 1220 class SVE2_2VectorArg_Imm_Narrowing_Intrinsic 1221 : Intrinsic<[LLVMSubdivide2VectorType<0>], 1222 [LLVMSubdivide2VectorType<0>, llvm_anyvector_ty, 1223 llvm_i32_ty], 1224 [IntrNoMem, ImmArg<ArgIndex<2>>]>; 1225 1226 class SVE2_CONFLICT_DETECT_Intrinsic 1227 : Intrinsic<[llvm_anyvector_ty], 1228 [LLVMAnyPointerType<llvm_any_ty>, 1229 LLVMMatchType<1>]>; 1230 1231 class SVE2_3VectorArg_Indexed_Intrinsic 1232 : Intrinsic<[llvm_anyvector_ty], 1233 [LLVMMatchType<0>, 1234 LLVMSubdivide2VectorType<0>, 1235 LLVMSubdivide2VectorType<0>, 1236 llvm_i32_ty], 1237 [IntrNoMem, ImmArg<ArgIndex<3>>]>; 1238 1239 class AdvSIMD_SVE_CDOT_LANE_Intrinsic 1240 : Intrinsic<[llvm_anyvector_ty], 1241 [LLVMMatchType<0>, 1242 LLVMSubdivide4VectorType<0>, 1243 LLVMSubdivide4VectorType<0>, 1244 llvm_i32_ty, 1245 llvm_i32_ty], 1246 [IntrNoMem, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>]>; 1247 1248 // NOTE: There is no relationship between these intrinsics beyond an attempt 1249 // to reuse currently identical class definitions. 1250 class AdvSIMD_SVE_LOGB_Intrinsic : AdvSIMD_SVE_CNT_Intrinsic; 1251 class AdvSIMD_SVE2_CADD_Intrinsic : AdvSIMD_2VectorArgIndexed_Intrinsic; 1252 class AdvSIMD_SVE2_CMLA_Intrinsic : AdvSIMD_3VectorArgIndexed_Intrinsic; 1253 1254 // This class of intrinsics are not intended to be useful within LLVM IR but 1255 // are instead here to support some of the more regid parts of the ACLE. 1256 class Builtin_SVCVT<string name, LLVMType OUT, LLVMType PRED, LLVMType IN> 1257 : Intrinsic<[OUT], [OUT, PRED, IN], [IntrNoMem]>; 1258} 1259 1260//===----------------------------------------------------------------------===// 1261// SVE 1262 1263let TargetPrefix = "aarch64" in { // All intrinsics start with "llvm.aarch64.". 1264 1265class AdvSIMD_SVE_Reduce_Intrinsic 1266 : Intrinsic<[LLVMVectorElementType<0>], 1267 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1268 llvm_anyvector_ty], 1269 [IntrNoMem]>; 1270 1271class AdvSIMD_SVE_SADDV_Reduce_Intrinsic 1272 : Intrinsic<[llvm_i64_ty], 1273 [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1274 llvm_anyvector_ty], 1275 [IntrNoMem]>; 1276 1277class AdvSIMD_SVE_WHILE_Intrinsic 1278 : Intrinsic<[llvm_anyvector_ty], 1279 [llvm_anyint_ty, LLVMMatchType<1>], 1280 [IntrNoMem]>; 1281 1282class AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic 1283 : Intrinsic<[llvm_anyvector_ty], 1284 [ 1285 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1286 LLVMPointerToElt<0>, 1287 LLVMScalarOrSameVectorWidth<0, llvm_i64_ty> 1288 ], 1289 [IntrReadMem, IntrArgMemOnly]>; 1290 1291class AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic 1292 : Intrinsic<[llvm_anyvector_ty], 1293 [ 1294 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1295 LLVMPointerToElt<0>, 1296 LLVMScalarOrSameVectorWidth<0, llvm_i32_ty> 1297 ], 1298 [IntrReadMem, IntrArgMemOnly]>; 1299 1300class AdvSIMD_GatherLoad_VS_Intrinsic 1301 : Intrinsic<[llvm_anyvector_ty], 1302 [ 1303 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1304 llvm_anyvector_ty, 1305 llvm_i64_ty 1306 ], 1307 [IntrReadMem]>; 1308 1309class AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic 1310 : Intrinsic<[], 1311 [ 1312 llvm_anyvector_ty, 1313 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1314 LLVMPointerToElt<0>, 1315 LLVMScalarOrSameVectorWidth<0, llvm_i64_ty> 1316 ], 1317 [IntrWriteMem, IntrArgMemOnly]>; 1318 1319class AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic 1320 : Intrinsic<[], 1321 [ 1322 llvm_anyvector_ty, 1323 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1324 LLVMPointerToElt<0>, 1325 LLVMScalarOrSameVectorWidth<0, llvm_i32_ty> 1326 ], 1327 [IntrWriteMem, IntrArgMemOnly]>; 1328 1329class AdvSIMD_ScatterStore_VS_Intrinsic 1330 : Intrinsic<[], 1331 [ 1332 llvm_anyvector_ty, 1333 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, 1334 llvm_anyvector_ty, llvm_i64_ty 1335 ], 1336 [IntrWriteMem]>; 1337 1338 1339class SVE_gather_prf_SV 1340 : Intrinsic<[], 1341 [ 1342 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, // Predicate 1343 llvm_ptr_ty, // Base address 1344 llvm_anyvector_ty, // Offsets 1345 llvm_i32_ty // Prfop 1346 ], 1347 [IntrInaccessibleMemOrArgMemOnly, NoCapture<ArgIndex<1>>, ImmArg<ArgIndex<3>>]>; 1348 1349class SVE_gather_prf_VS 1350 : Intrinsic<[], 1351 [ 1352 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, // Predicate 1353 llvm_anyvector_ty, // Base addresses 1354 llvm_i64_ty, // Scalar offset 1355 llvm_i32_ty // Prfop 1356 ], 1357 [IntrInaccessibleMemOrArgMemOnly, ImmArg<ArgIndex<3>>]>; 1358 1359class SVE_MatMul_Intrinsic 1360 : Intrinsic<[llvm_anyvector_ty], 1361 [LLVMMatchType<0>, LLVMSubdivide4VectorType<0>, LLVMSubdivide4VectorType<0>], 1362 [IntrNoMem]>; 1363 1364class SVE_4Vec_BF16 1365 : Intrinsic<[llvm_nxv4f32_ty], 1366 [llvm_nxv4f32_ty, llvm_nxv8bf16_ty, llvm_nxv8bf16_ty], 1367 [IntrNoMem]>; 1368 1369class SVE_4Vec_BF16_Indexed 1370 : Intrinsic<[llvm_nxv4f32_ty], 1371 [llvm_nxv4f32_ty, llvm_nxv8bf16_ty, llvm_nxv8bf16_ty, llvm_i64_ty], 1372 [IntrNoMem, ImmArg<ArgIndex<3>>]>; 1373 1374// 1375// Vector tuple creation intrinsics (ACLE) 1376// 1377 1378def int_aarch64_sve_tuple_create2 : AdvSIMD_SVE_Create_2Vector_Tuple; 1379def int_aarch64_sve_tuple_create3 : AdvSIMD_SVE_Create_3Vector_Tuple; 1380def int_aarch64_sve_tuple_create4 : AdvSIMD_SVE_Create_4Vector_Tuple; 1381 1382// 1383// Vector tuple insertion/extraction intrinsics (ACLE) 1384// 1385 1386def int_aarch64_sve_tuple_get : AdvSIMD_SVE_Get_Vector_Tuple; 1387def int_aarch64_sve_tuple_set : AdvSIMD_SVE_Set_Vector_Tuple; 1388 1389// 1390// Loads 1391// 1392 1393def int_aarch64_sve_ld1 : AdvSIMD_1Vec_PredLoad_Intrinsic; 1394 1395def int_aarch64_sve_ld2 : AdvSIMD_ManyVec_PredLoad_Intrinsic; 1396def int_aarch64_sve_ld3 : AdvSIMD_ManyVec_PredLoad_Intrinsic; 1397def int_aarch64_sve_ld4 : AdvSIMD_ManyVec_PredLoad_Intrinsic; 1398 1399def int_aarch64_sve_ldnt1 : AdvSIMD_1Vec_PredLoad_Intrinsic; 1400def int_aarch64_sve_ldnf1 : AdvSIMD_1Vec_PredLoad_Intrinsic; 1401def int_aarch64_sve_ldff1 : AdvSIMD_1Vec_PredLoad_Intrinsic; 1402 1403def int_aarch64_sve_ld1rq : AdvSIMD_1Vec_PredLoad_Intrinsic; 1404def int_aarch64_sve_ld1ro : AdvSIMD_1Vec_PredLoad_Intrinsic; 1405 1406// 1407// Stores 1408// 1409 1410def int_aarch64_sve_st1 : AdvSIMD_1Vec_PredStore_Intrinsic; 1411def int_aarch64_sve_st2 : AdvSIMD_2Vec_PredStore_Intrinsic; 1412def int_aarch64_sve_st3 : AdvSIMD_3Vec_PredStore_Intrinsic; 1413def int_aarch64_sve_st4 : AdvSIMD_4Vec_PredStore_Intrinsic; 1414 1415def int_aarch64_sve_stnt1 : AdvSIMD_1Vec_PredStore_Intrinsic; 1416 1417// 1418// Prefetches 1419// 1420 1421def int_aarch64_sve_prf 1422 : Intrinsic<[], [llvm_anyvector_ty, llvm_ptr_ty, llvm_i32_ty], 1423 [IntrArgMemOnly, ImmArg<ArgIndex<2>>]>; 1424 1425// Scalar + 32-bit scaled offset vector, zero extend, packed and 1426// unpacked. 1427def int_aarch64_sve_prfb_gather_uxtw_index : SVE_gather_prf_SV; 1428def int_aarch64_sve_prfh_gather_uxtw_index : SVE_gather_prf_SV; 1429def int_aarch64_sve_prfw_gather_uxtw_index : SVE_gather_prf_SV; 1430def int_aarch64_sve_prfd_gather_uxtw_index : SVE_gather_prf_SV; 1431 1432// Scalar + 32-bit scaled offset vector, sign extend, packed and 1433// unpacked. 1434def int_aarch64_sve_prfb_gather_sxtw_index : SVE_gather_prf_SV; 1435def int_aarch64_sve_prfw_gather_sxtw_index : SVE_gather_prf_SV; 1436def int_aarch64_sve_prfh_gather_sxtw_index : SVE_gather_prf_SV; 1437def int_aarch64_sve_prfd_gather_sxtw_index : SVE_gather_prf_SV; 1438 1439// Scalar + 64-bit scaled offset vector. 1440def int_aarch64_sve_prfb_gather_index : SVE_gather_prf_SV; 1441def int_aarch64_sve_prfh_gather_index : SVE_gather_prf_SV; 1442def int_aarch64_sve_prfw_gather_index : SVE_gather_prf_SV; 1443def int_aarch64_sve_prfd_gather_index : SVE_gather_prf_SV; 1444 1445// Vector + scalar. 1446def int_aarch64_sve_prfb_gather_scalar_offset : SVE_gather_prf_VS; 1447def int_aarch64_sve_prfh_gather_scalar_offset : SVE_gather_prf_VS; 1448def int_aarch64_sve_prfw_gather_scalar_offset : SVE_gather_prf_VS; 1449def int_aarch64_sve_prfd_gather_scalar_offset : SVE_gather_prf_VS; 1450 1451// 1452// Scalar to vector operations 1453// 1454 1455def int_aarch64_sve_dup : AdvSIMD_SVE_DUP_Intrinsic; 1456def int_aarch64_sve_dup_x : AdvSIMD_SVE_DUP_Unpred_Intrinsic; 1457 1458 1459def int_aarch64_sve_index : AdvSIMD_SVE_Index_Intrinsic; 1460 1461// 1462// Address calculation 1463// 1464 1465def int_aarch64_sve_adrb : AdvSIMD_2VectorArg_Intrinsic; 1466def int_aarch64_sve_adrh : AdvSIMD_2VectorArg_Intrinsic; 1467def int_aarch64_sve_adrw : AdvSIMD_2VectorArg_Intrinsic; 1468def int_aarch64_sve_adrd : AdvSIMD_2VectorArg_Intrinsic; 1469 1470// 1471// Integer arithmetic 1472// 1473 1474def int_aarch64_sve_add : AdvSIMD_Pred2VectorArg_Intrinsic; 1475def int_aarch64_sve_sub : AdvSIMD_Pred2VectorArg_Intrinsic; 1476def int_aarch64_sve_subr : AdvSIMD_Pred2VectorArg_Intrinsic; 1477 1478def int_aarch64_sve_pmul : AdvSIMD_2VectorArg_Intrinsic; 1479 1480def int_aarch64_sve_mul : AdvSIMD_Pred2VectorArg_Intrinsic; 1481def int_aarch64_sve_mul_lane : AdvSIMD_2VectorArgIndexed_Intrinsic; 1482def int_aarch64_sve_smulh : AdvSIMD_Pred2VectorArg_Intrinsic; 1483def int_aarch64_sve_umulh : AdvSIMD_Pred2VectorArg_Intrinsic; 1484 1485def int_aarch64_sve_sdiv : AdvSIMD_Pred2VectorArg_Intrinsic; 1486def int_aarch64_sve_udiv : AdvSIMD_Pred2VectorArg_Intrinsic; 1487def int_aarch64_sve_sdivr : AdvSIMD_Pred2VectorArg_Intrinsic; 1488def int_aarch64_sve_udivr : AdvSIMD_Pred2VectorArg_Intrinsic; 1489 1490def int_aarch64_sve_smax : AdvSIMD_Pred2VectorArg_Intrinsic; 1491def int_aarch64_sve_umax : AdvSIMD_Pred2VectorArg_Intrinsic; 1492def int_aarch64_sve_smin : AdvSIMD_Pred2VectorArg_Intrinsic; 1493def int_aarch64_sve_umin : AdvSIMD_Pred2VectorArg_Intrinsic; 1494def int_aarch64_sve_sabd : AdvSIMD_Pred2VectorArg_Intrinsic; 1495def int_aarch64_sve_uabd : AdvSIMD_Pred2VectorArg_Intrinsic; 1496 1497def int_aarch64_sve_mad : AdvSIMD_Pred3VectorArg_Intrinsic; 1498def int_aarch64_sve_msb : AdvSIMD_Pred3VectorArg_Intrinsic; 1499def int_aarch64_sve_mla : AdvSIMD_Pred3VectorArg_Intrinsic; 1500def int_aarch64_sve_mla_lane : AdvSIMD_3VectorArgIndexed_Intrinsic; 1501def int_aarch64_sve_mls : AdvSIMD_Pred3VectorArg_Intrinsic; 1502def int_aarch64_sve_mls_lane : AdvSIMD_3VectorArgIndexed_Intrinsic; 1503 1504def int_aarch64_sve_saddv : AdvSIMD_SVE_SADDV_Reduce_Intrinsic; 1505def int_aarch64_sve_uaddv : AdvSIMD_SVE_SADDV_Reduce_Intrinsic; 1506 1507def int_aarch64_sve_smaxv : AdvSIMD_SVE_Reduce_Intrinsic; 1508def int_aarch64_sve_umaxv : AdvSIMD_SVE_Reduce_Intrinsic; 1509def int_aarch64_sve_sminv : AdvSIMD_SVE_Reduce_Intrinsic; 1510def int_aarch64_sve_uminv : AdvSIMD_SVE_Reduce_Intrinsic; 1511 1512def int_aarch64_sve_orv : AdvSIMD_SVE_Reduce_Intrinsic; 1513def int_aarch64_sve_eorv : AdvSIMD_SVE_Reduce_Intrinsic; 1514def int_aarch64_sve_andv : AdvSIMD_SVE_Reduce_Intrinsic; 1515 1516def int_aarch64_sve_abs : AdvSIMD_Merged1VectorArg_Intrinsic; 1517def int_aarch64_sve_neg : AdvSIMD_Merged1VectorArg_Intrinsic; 1518 1519def int_aarch64_sve_sdot : AdvSIMD_SVE_DOT_Intrinsic; 1520def int_aarch64_sve_sdot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic; 1521 1522def int_aarch64_sve_udot : AdvSIMD_SVE_DOT_Intrinsic; 1523def int_aarch64_sve_udot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic; 1524 1525def int_aarch64_sve_sqadd_x : AdvSIMD_2VectorArg_Intrinsic; 1526def int_aarch64_sve_sqsub_x : AdvSIMD_2VectorArg_Intrinsic; 1527def int_aarch64_sve_uqadd_x : AdvSIMD_2VectorArg_Intrinsic; 1528def int_aarch64_sve_uqsub_x : AdvSIMD_2VectorArg_Intrinsic; 1529 1530// Shifts 1531 1532def int_aarch64_sve_asr : AdvSIMD_Pred2VectorArg_Intrinsic; 1533def int_aarch64_sve_asr_wide : AdvSIMD_SVE_ShiftWide_Intrinsic; 1534def int_aarch64_sve_asrd : AdvSIMD_SVE_ShiftByImm_Intrinsic; 1535def int_aarch64_sve_insr : AdvSIMD_SVE_INSR_Intrinsic; 1536def int_aarch64_sve_lsl : AdvSIMD_Pred2VectorArg_Intrinsic; 1537def int_aarch64_sve_lsl_wide : AdvSIMD_SVE_ShiftWide_Intrinsic; 1538def int_aarch64_sve_lsr : AdvSIMD_Pred2VectorArg_Intrinsic; 1539def int_aarch64_sve_lsr_wide : AdvSIMD_SVE_ShiftWide_Intrinsic; 1540 1541// 1542// Integer comparisons 1543// 1544 1545def int_aarch64_sve_cmpeq : AdvSIMD_SVE_Compare_Intrinsic; 1546def int_aarch64_sve_cmpge : AdvSIMD_SVE_Compare_Intrinsic; 1547def int_aarch64_sve_cmpgt : AdvSIMD_SVE_Compare_Intrinsic; 1548def int_aarch64_sve_cmphi : AdvSIMD_SVE_Compare_Intrinsic; 1549def int_aarch64_sve_cmphs : AdvSIMD_SVE_Compare_Intrinsic; 1550def int_aarch64_sve_cmpne : AdvSIMD_SVE_Compare_Intrinsic; 1551 1552def int_aarch64_sve_cmpeq_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1553def int_aarch64_sve_cmpge_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1554def int_aarch64_sve_cmpgt_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1555def int_aarch64_sve_cmphi_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1556def int_aarch64_sve_cmphs_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1557def int_aarch64_sve_cmple_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1558def int_aarch64_sve_cmplo_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1559def int_aarch64_sve_cmpls_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1560def int_aarch64_sve_cmplt_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1561def int_aarch64_sve_cmpne_wide : AdvSIMD_SVE_CompareWide_Intrinsic; 1562 1563// 1564// Counting bits 1565// 1566 1567def int_aarch64_sve_cls : AdvSIMD_Merged1VectorArg_Intrinsic; 1568def int_aarch64_sve_clz : AdvSIMD_Merged1VectorArg_Intrinsic; 1569def int_aarch64_sve_cnt : AdvSIMD_SVE_CNT_Intrinsic; 1570 1571// 1572// Counting elements 1573// 1574 1575def int_aarch64_sve_cntb : AdvSIMD_SVE_CNTB_Intrinsic; 1576def int_aarch64_sve_cnth : AdvSIMD_SVE_CNTB_Intrinsic; 1577def int_aarch64_sve_cntw : AdvSIMD_SVE_CNTB_Intrinsic; 1578def int_aarch64_sve_cntd : AdvSIMD_SVE_CNTB_Intrinsic; 1579 1580def int_aarch64_sve_cntp : AdvSIMD_SVE_CNTP_Intrinsic; 1581 1582// 1583// FFR manipulation 1584// 1585 1586def int_aarch64_sve_rdffr : GCCBuiltin<"__builtin_sve_svrdffr">, Intrinsic<[llvm_nxv16i1_ty], []>; 1587def int_aarch64_sve_rdffr_z : GCCBuiltin<"__builtin_sve_svrdffr_z">, Intrinsic<[llvm_nxv16i1_ty], [llvm_nxv16i1_ty]>; 1588def int_aarch64_sve_setffr : GCCBuiltin<"__builtin_sve_svsetffr">, Intrinsic<[], []>; 1589def int_aarch64_sve_wrffr : GCCBuiltin<"__builtin_sve_svwrffr">, Intrinsic<[], [llvm_nxv16i1_ty]>; 1590 1591// 1592// Saturating scalar arithmetic 1593// 1594 1595def int_aarch64_sve_sqdech : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1596def int_aarch64_sve_sqdecw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1597def int_aarch64_sve_sqdecd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1598def int_aarch64_sve_sqdecp : AdvSIMD_SVE_Saturating_Intrinsic; 1599 1600def int_aarch64_sve_sqdecb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1601def int_aarch64_sve_sqdecb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1602def int_aarch64_sve_sqdech_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1603def int_aarch64_sve_sqdech_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1604def int_aarch64_sve_sqdecw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1605def int_aarch64_sve_sqdecw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1606def int_aarch64_sve_sqdecd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1607def int_aarch64_sve_sqdecd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1608def int_aarch64_sve_sqdecp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>; 1609def int_aarch64_sve_sqdecp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>; 1610 1611def int_aarch64_sve_sqinch : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1612def int_aarch64_sve_sqincw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1613def int_aarch64_sve_sqincd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1614def int_aarch64_sve_sqincp : AdvSIMD_SVE_Saturating_Intrinsic; 1615 1616def int_aarch64_sve_sqincb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1617def int_aarch64_sve_sqincb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1618def int_aarch64_sve_sqinch_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1619def int_aarch64_sve_sqinch_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1620def int_aarch64_sve_sqincw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1621def int_aarch64_sve_sqincw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1622def int_aarch64_sve_sqincd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1623def int_aarch64_sve_sqincd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1624def int_aarch64_sve_sqincp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>; 1625def int_aarch64_sve_sqincp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>; 1626 1627def int_aarch64_sve_uqdech : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1628def int_aarch64_sve_uqdecw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1629def int_aarch64_sve_uqdecd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1630def int_aarch64_sve_uqdecp : AdvSIMD_SVE_Saturating_Intrinsic; 1631 1632def int_aarch64_sve_uqdecb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1633def int_aarch64_sve_uqdecb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1634def int_aarch64_sve_uqdech_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1635def int_aarch64_sve_uqdech_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1636def int_aarch64_sve_uqdecw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1637def int_aarch64_sve_uqdecw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1638def int_aarch64_sve_uqdecd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1639def int_aarch64_sve_uqdecd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1640def int_aarch64_sve_uqdecp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>; 1641def int_aarch64_sve_uqdecp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>; 1642 1643def int_aarch64_sve_uqinch : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1644def int_aarch64_sve_uqincw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1645def int_aarch64_sve_uqincd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic; 1646def int_aarch64_sve_uqincp : AdvSIMD_SVE_Saturating_Intrinsic; 1647 1648def int_aarch64_sve_uqincb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1649def int_aarch64_sve_uqincb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1650def int_aarch64_sve_uqinch_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1651def int_aarch64_sve_uqinch_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1652def int_aarch64_sve_uqincw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1653def int_aarch64_sve_uqincw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1654def int_aarch64_sve_uqincd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>; 1655def int_aarch64_sve_uqincd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>; 1656def int_aarch64_sve_uqincp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>; 1657def int_aarch64_sve_uqincp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>; 1658 1659// 1660// Reversal 1661// 1662 1663def int_aarch64_sve_rbit : AdvSIMD_Merged1VectorArg_Intrinsic; 1664def int_aarch64_sve_revb : AdvSIMD_Merged1VectorArg_Intrinsic; 1665def int_aarch64_sve_revh : AdvSIMD_Merged1VectorArg_Intrinsic; 1666def int_aarch64_sve_revw : AdvSIMD_Merged1VectorArg_Intrinsic; 1667 1668// 1669// Permutations and selection 1670// 1671 1672def int_aarch64_sve_clasta : AdvSIMD_Pred2VectorArg_Intrinsic; 1673def int_aarch64_sve_clasta_n : AdvSIMD_SVE_ReduceWithInit_Intrinsic; 1674def int_aarch64_sve_clastb : AdvSIMD_Pred2VectorArg_Intrinsic; 1675def int_aarch64_sve_clastb_n : AdvSIMD_SVE_ReduceWithInit_Intrinsic; 1676def int_aarch64_sve_compact : AdvSIMD_Pred1VectorArg_Intrinsic; 1677def int_aarch64_sve_dupq_lane : AdvSIMD_SVE_DUPQ_Intrinsic; 1678def int_aarch64_sve_ext : AdvSIMD_2VectorArgIndexed_Intrinsic; 1679def int_aarch64_sve_sel : AdvSIMD_Pred2VectorArg_Intrinsic; 1680def int_aarch64_sve_lasta : AdvSIMD_SVE_Reduce_Intrinsic; 1681def int_aarch64_sve_lastb : AdvSIMD_SVE_Reduce_Intrinsic; 1682def int_aarch64_sve_rev : AdvSIMD_1VectorArg_Intrinsic; 1683def int_aarch64_sve_splice : AdvSIMD_Pred2VectorArg_Intrinsic; 1684def int_aarch64_sve_sunpkhi : AdvSIMD_SVE_Unpack_Intrinsic; 1685def int_aarch64_sve_sunpklo : AdvSIMD_SVE_Unpack_Intrinsic; 1686def int_aarch64_sve_tbl : AdvSIMD_SVE_TBL_Intrinsic; 1687def int_aarch64_sve_trn1 : AdvSIMD_2VectorArg_Intrinsic; 1688def int_aarch64_sve_trn2 : AdvSIMD_2VectorArg_Intrinsic; 1689def int_aarch64_sve_trn1q : AdvSIMD_2VectorArg_Intrinsic; 1690def int_aarch64_sve_trn2q : AdvSIMD_2VectorArg_Intrinsic; 1691def int_aarch64_sve_uunpkhi : AdvSIMD_SVE_Unpack_Intrinsic; 1692def int_aarch64_sve_uunpklo : AdvSIMD_SVE_Unpack_Intrinsic; 1693def int_aarch64_sve_uzp1 : AdvSIMD_2VectorArg_Intrinsic; 1694def int_aarch64_sve_uzp2 : AdvSIMD_2VectorArg_Intrinsic; 1695def int_aarch64_sve_uzp1q : AdvSIMD_2VectorArg_Intrinsic; 1696def int_aarch64_sve_uzp2q : AdvSIMD_2VectorArg_Intrinsic; 1697def int_aarch64_sve_zip1 : AdvSIMD_2VectorArg_Intrinsic; 1698def int_aarch64_sve_zip2 : AdvSIMD_2VectorArg_Intrinsic; 1699def int_aarch64_sve_zip1q : AdvSIMD_2VectorArg_Intrinsic; 1700def int_aarch64_sve_zip2q : AdvSIMD_2VectorArg_Intrinsic; 1701 1702// 1703// Logical operations 1704// 1705 1706def int_aarch64_sve_and : AdvSIMD_Pred2VectorArg_Intrinsic; 1707def int_aarch64_sve_bic : AdvSIMD_Pred2VectorArg_Intrinsic; 1708def int_aarch64_sve_cnot : AdvSIMD_Merged1VectorArg_Intrinsic; 1709def int_aarch64_sve_eor : AdvSIMD_Pred2VectorArg_Intrinsic; 1710def int_aarch64_sve_not : AdvSIMD_Merged1VectorArg_Intrinsic; 1711def int_aarch64_sve_orr : AdvSIMD_Pred2VectorArg_Intrinsic; 1712 1713// 1714// Conversion 1715// 1716 1717def int_aarch64_sve_sxtb : AdvSIMD_Merged1VectorArg_Intrinsic; 1718def int_aarch64_sve_sxth : AdvSIMD_Merged1VectorArg_Intrinsic; 1719def int_aarch64_sve_sxtw : AdvSIMD_Merged1VectorArg_Intrinsic; 1720def int_aarch64_sve_uxtb : AdvSIMD_Merged1VectorArg_Intrinsic; 1721def int_aarch64_sve_uxth : AdvSIMD_Merged1VectorArg_Intrinsic; 1722def int_aarch64_sve_uxtw : AdvSIMD_Merged1VectorArg_Intrinsic; 1723 1724// 1725// While comparisons 1726// 1727 1728def int_aarch64_sve_whilele : AdvSIMD_SVE_WHILE_Intrinsic; 1729def int_aarch64_sve_whilelo : AdvSIMD_SVE_WHILE_Intrinsic; 1730def int_aarch64_sve_whilels : AdvSIMD_SVE_WHILE_Intrinsic; 1731def int_aarch64_sve_whilelt : AdvSIMD_SVE_WHILE_Intrinsic; 1732def int_aarch64_sve_whilege : AdvSIMD_SVE_WHILE_Intrinsic; 1733def int_aarch64_sve_whilegt : AdvSIMD_SVE_WHILE_Intrinsic; 1734def int_aarch64_sve_whilehs : AdvSIMD_SVE_WHILE_Intrinsic; 1735def int_aarch64_sve_whilehi : AdvSIMD_SVE_WHILE_Intrinsic; 1736 1737// 1738// Floating-point arithmetic 1739// 1740 1741def int_aarch64_sve_fabd : AdvSIMD_Pred2VectorArg_Intrinsic; 1742def int_aarch64_sve_fabs : AdvSIMD_Merged1VectorArg_Intrinsic; 1743def int_aarch64_sve_fadd : AdvSIMD_Pred2VectorArg_Intrinsic; 1744def int_aarch64_sve_fcadd : AdvSIMD_SVE_CADD_Intrinsic; 1745def int_aarch64_sve_fcmla : AdvSIMD_SVE_CMLA_Intrinsic; 1746def int_aarch64_sve_fcmla_lane : AdvSIMD_SVE_CMLA_LANE_Intrinsic; 1747def int_aarch64_sve_fdiv : AdvSIMD_Pred2VectorArg_Intrinsic; 1748def int_aarch64_sve_fdivr : AdvSIMD_Pred2VectorArg_Intrinsic; 1749def int_aarch64_sve_fexpa_x : AdvSIMD_SVE_EXPA_Intrinsic; 1750def int_aarch64_sve_fmad : AdvSIMD_Pred3VectorArg_Intrinsic; 1751def int_aarch64_sve_fmax : AdvSIMD_Pred2VectorArg_Intrinsic; 1752def int_aarch64_sve_fmaxnm : AdvSIMD_Pred2VectorArg_Intrinsic; 1753def int_aarch64_sve_fmin : AdvSIMD_Pred2VectorArg_Intrinsic; 1754def int_aarch64_sve_fminnm : AdvSIMD_Pred2VectorArg_Intrinsic; 1755def int_aarch64_sve_fmla : AdvSIMD_Pred3VectorArg_Intrinsic; 1756def int_aarch64_sve_fmla_lane : AdvSIMD_3VectorArgIndexed_Intrinsic; 1757def int_aarch64_sve_fmls : AdvSIMD_Pred3VectorArg_Intrinsic; 1758def int_aarch64_sve_fmls_lane : AdvSIMD_3VectorArgIndexed_Intrinsic; 1759def int_aarch64_sve_fmsb : AdvSIMD_Pred3VectorArg_Intrinsic; 1760def int_aarch64_sve_fmul : AdvSIMD_Pred2VectorArg_Intrinsic; 1761def int_aarch64_sve_fmulx : AdvSIMD_Pred2VectorArg_Intrinsic; 1762def int_aarch64_sve_fneg : AdvSIMD_Merged1VectorArg_Intrinsic; 1763def int_aarch64_sve_fmul_lane : AdvSIMD_2VectorArgIndexed_Intrinsic; 1764def int_aarch64_sve_fnmad : AdvSIMD_Pred3VectorArg_Intrinsic; 1765def int_aarch64_sve_fnmla : AdvSIMD_Pred3VectorArg_Intrinsic; 1766def int_aarch64_sve_fnmls : AdvSIMD_Pred3VectorArg_Intrinsic; 1767def int_aarch64_sve_fnmsb : AdvSIMD_Pred3VectorArg_Intrinsic; 1768def int_aarch64_sve_frecpe_x : AdvSIMD_1VectorArg_Intrinsic; 1769def int_aarch64_sve_frecps_x : AdvSIMD_2VectorArg_Intrinsic; 1770def int_aarch64_sve_frecpx : AdvSIMD_Merged1VectorArg_Intrinsic; 1771def int_aarch64_sve_frinta : AdvSIMD_Merged1VectorArg_Intrinsic; 1772def int_aarch64_sve_frinti : AdvSIMD_Merged1VectorArg_Intrinsic; 1773def int_aarch64_sve_frintm : AdvSIMD_Merged1VectorArg_Intrinsic; 1774def int_aarch64_sve_frintn : AdvSIMD_Merged1VectorArg_Intrinsic; 1775def int_aarch64_sve_frintp : AdvSIMD_Merged1VectorArg_Intrinsic; 1776def int_aarch64_sve_frintx : AdvSIMD_Merged1VectorArg_Intrinsic; 1777def int_aarch64_sve_frintz : AdvSIMD_Merged1VectorArg_Intrinsic; 1778def int_aarch64_sve_frsqrte_x : AdvSIMD_1VectorArg_Intrinsic; 1779def int_aarch64_sve_frsqrts_x : AdvSIMD_2VectorArg_Intrinsic; 1780def int_aarch64_sve_fscale : AdvSIMD_SVE_SCALE_Intrinsic; 1781def int_aarch64_sve_fsqrt : AdvSIMD_Merged1VectorArg_Intrinsic; 1782def int_aarch64_sve_fsub : AdvSIMD_Pred2VectorArg_Intrinsic; 1783def int_aarch64_sve_fsubr : AdvSIMD_Pred2VectorArg_Intrinsic; 1784def int_aarch64_sve_ftmad_x : AdvSIMD_2VectorArgIndexed_Intrinsic; 1785def int_aarch64_sve_ftsmul_x : AdvSIMD_SVE_TSMUL_Intrinsic; 1786def int_aarch64_sve_ftssel_x : AdvSIMD_SVE_TSMUL_Intrinsic; 1787 1788// 1789// Floating-point reductions 1790// 1791 1792def int_aarch64_sve_fadda : AdvSIMD_SVE_ReduceWithInit_Intrinsic; 1793def int_aarch64_sve_faddv : AdvSIMD_SVE_Reduce_Intrinsic; 1794def int_aarch64_sve_fmaxv : AdvSIMD_SVE_Reduce_Intrinsic; 1795def int_aarch64_sve_fmaxnmv : AdvSIMD_SVE_Reduce_Intrinsic; 1796def int_aarch64_sve_fminv : AdvSIMD_SVE_Reduce_Intrinsic; 1797def int_aarch64_sve_fminnmv : AdvSIMD_SVE_Reduce_Intrinsic; 1798 1799// 1800// Floating-point conversions 1801// 1802 1803def int_aarch64_sve_fcvt : AdvSIMD_SVE_FCVT_Intrinsic; 1804def int_aarch64_sve_fcvtzs : AdvSIMD_SVE_FCVTZS_Intrinsic; 1805def int_aarch64_sve_fcvtzu : AdvSIMD_SVE_FCVTZS_Intrinsic; 1806def int_aarch64_sve_scvtf : AdvSIMD_SVE_SCVTF_Intrinsic; 1807def int_aarch64_sve_ucvtf : AdvSIMD_SVE_SCVTF_Intrinsic; 1808 1809// 1810// Floating-point comparisons 1811// 1812 1813def int_aarch64_sve_facge : AdvSIMD_SVE_Compare_Intrinsic; 1814def int_aarch64_sve_facgt : AdvSIMD_SVE_Compare_Intrinsic; 1815 1816def int_aarch64_sve_fcmpeq : AdvSIMD_SVE_Compare_Intrinsic; 1817def int_aarch64_sve_fcmpge : AdvSIMD_SVE_Compare_Intrinsic; 1818def int_aarch64_sve_fcmpgt : AdvSIMD_SVE_Compare_Intrinsic; 1819def int_aarch64_sve_fcmpne : AdvSIMD_SVE_Compare_Intrinsic; 1820def int_aarch64_sve_fcmpuo : AdvSIMD_SVE_Compare_Intrinsic; 1821 1822def int_aarch64_sve_fcvtzs_i32f16 : Builtin_SVCVT<"svcvt_s32_f16_m", llvm_nxv4i32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>; 1823def int_aarch64_sve_fcvtzs_i32f64 : Builtin_SVCVT<"svcvt_s32_f64_m", llvm_nxv4i32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>; 1824def int_aarch64_sve_fcvtzs_i64f16 : Builtin_SVCVT<"svcvt_s64_f16_m", llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>; 1825def int_aarch64_sve_fcvtzs_i64f32 : Builtin_SVCVT<"svcvt_s64_f32_m", llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>; 1826 1827def int_aarch64_sve_fcvt_bf16f32 : Builtin_SVCVT<"svcvt_bf16_f32_m", llvm_nxv8bf16_ty, llvm_nxv8i1_ty, llvm_nxv4f32_ty>; 1828def int_aarch64_sve_fcvtnt_bf16f32 : Builtin_SVCVT<"svcvtnt_bf16_f32_m", llvm_nxv8bf16_ty, llvm_nxv8i1_ty, llvm_nxv4f32_ty>; 1829 1830def int_aarch64_sve_fcvtzu_i32f16 : Builtin_SVCVT<"svcvt_u32_f16_m", llvm_nxv4i32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>; 1831def int_aarch64_sve_fcvtzu_i32f64 : Builtin_SVCVT<"svcvt_u32_f64_m", llvm_nxv4i32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>; 1832def int_aarch64_sve_fcvtzu_i64f16 : Builtin_SVCVT<"svcvt_u64_f16_m", llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>; 1833def int_aarch64_sve_fcvtzu_i64f32 : Builtin_SVCVT<"svcvt_u64_f32_m", llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>; 1834 1835def int_aarch64_sve_fcvt_f16f32 : Builtin_SVCVT<"svcvt_f16_f32_m", llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4f32_ty>; 1836def int_aarch64_sve_fcvt_f16f64 : Builtin_SVCVT<"svcvt_f16_f64_m", llvm_nxv8f16_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>; 1837def int_aarch64_sve_fcvt_f32f64 : Builtin_SVCVT<"svcvt_f32_f64_m", llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>; 1838 1839def int_aarch64_sve_fcvt_f32f16 : Builtin_SVCVT<"svcvt_f32_f16_m", llvm_nxv4f32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>; 1840def int_aarch64_sve_fcvt_f64f16 : Builtin_SVCVT<"svcvt_f64_f16_m", llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>; 1841def int_aarch64_sve_fcvt_f64f32 : Builtin_SVCVT<"svcvt_f64_f32_m", llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>; 1842 1843def int_aarch64_sve_fcvtlt_f32f16 : Builtin_SVCVT<"svcvtlt_f32_f16_m", llvm_nxv4f32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>; 1844def int_aarch64_sve_fcvtlt_f64f32 : Builtin_SVCVT<"svcvtlt_f64_f32_m", llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>; 1845def int_aarch64_sve_fcvtnt_f16f32 : Builtin_SVCVT<"svcvtnt_f16_f32_m", llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4f32_ty>; 1846def int_aarch64_sve_fcvtnt_f32f64 : Builtin_SVCVT<"svcvtnt_f32_f64_m", llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>; 1847 1848def int_aarch64_sve_fcvtx_f32f64 : Builtin_SVCVT<"svcvtx_f32_f64_m", llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>; 1849def int_aarch64_sve_fcvtxnt_f32f64 : Builtin_SVCVT<"svcvtxnt_f32_f64_m", llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>; 1850 1851def int_aarch64_sve_scvtf_f16i32 : Builtin_SVCVT<"svcvt_f16_s32_m", llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4i32_ty>; 1852def int_aarch64_sve_scvtf_f16i64 : Builtin_SVCVT<"svcvt_f16_s64_m", llvm_nxv8f16_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>; 1853def int_aarch64_sve_scvtf_f32i64 : Builtin_SVCVT<"svcvt_f32_s64_m", llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>; 1854def int_aarch64_sve_scvtf_f64i32 : Builtin_SVCVT<"svcvt_f64_s32_m", llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4i32_ty>; 1855 1856def int_aarch64_sve_ucvtf_f16i32 : Builtin_SVCVT<"svcvt_f16_u32_m", llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4i32_ty>; 1857def int_aarch64_sve_ucvtf_f16i64 : Builtin_SVCVT<"svcvt_f16_u64_m", llvm_nxv8f16_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>; 1858def int_aarch64_sve_ucvtf_f32i64 : Builtin_SVCVT<"svcvt_f32_u64_m", llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>; 1859def int_aarch64_sve_ucvtf_f64i32 : Builtin_SVCVT<"svcvt_f64_u32_m", llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4i32_ty>; 1860 1861// 1862// Predicate creation 1863// 1864 1865def int_aarch64_sve_ptrue : AdvSIMD_SVE_PTRUE_Intrinsic; 1866 1867// 1868// Predicate operations 1869// 1870 1871def int_aarch64_sve_and_z : AdvSIMD_Pred2VectorArg_Intrinsic; 1872def int_aarch64_sve_bic_z : AdvSIMD_Pred2VectorArg_Intrinsic; 1873def int_aarch64_sve_brka : AdvSIMD_Merged1VectorArg_Intrinsic; 1874def int_aarch64_sve_brka_z : AdvSIMD_Pred1VectorArg_Intrinsic; 1875def int_aarch64_sve_brkb : AdvSIMD_Merged1VectorArg_Intrinsic; 1876def int_aarch64_sve_brkb_z : AdvSIMD_Pred1VectorArg_Intrinsic; 1877def int_aarch64_sve_brkn_z : AdvSIMD_Pred2VectorArg_Intrinsic; 1878def int_aarch64_sve_brkpa_z : AdvSIMD_Pred2VectorArg_Intrinsic; 1879def int_aarch64_sve_brkpb_z : AdvSIMD_Pred2VectorArg_Intrinsic; 1880def int_aarch64_sve_eor_z : AdvSIMD_Pred2VectorArg_Intrinsic; 1881def int_aarch64_sve_nand_z : AdvSIMD_Pred2VectorArg_Intrinsic; 1882def int_aarch64_sve_nor_z : AdvSIMD_Pred2VectorArg_Intrinsic; 1883def int_aarch64_sve_orn_z : AdvSIMD_Pred2VectorArg_Intrinsic; 1884def int_aarch64_sve_orr_z : AdvSIMD_Pred2VectorArg_Intrinsic; 1885def int_aarch64_sve_pfirst : AdvSIMD_Pred1VectorArg_Intrinsic; 1886def int_aarch64_sve_pnext : AdvSIMD_Pred1VectorArg_Intrinsic; 1887def int_aarch64_sve_punpkhi : AdvSIMD_SVE_PUNPKHI_Intrinsic; 1888def int_aarch64_sve_punpklo : AdvSIMD_SVE_PUNPKHI_Intrinsic; 1889 1890// 1891// Testing predicates 1892// 1893 1894def int_aarch64_sve_ptest_any : AdvSIMD_SVE_PTEST_Intrinsic; 1895def int_aarch64_sve_ptest_first : AdvSIMD_SVE_PTEST_Intrinsic; 1896def int_aarch64_sve_ptest_last : AdvSIMD_SVE_PTEST_Intrinsic; 1897 1898// 1899// Reinterpreting data 1900// 1901 1902def int_aarch64_sve_convert_from_svbool : Intrinsic<[llvm_anyvector_ty], 1903 [llvm_nxv16i1_ty], 1904 [IntrNoMem]>; 1905 1906def int_aarch64_sve_convert_to_svbool : Intrinsic<[llvm_nxv16i1_ty], 1907 [llvm_anyvector_ty], 1908 [IntrNoMem]>; 1909 1910// 1911// Gather loads: scalar base + vector offsets 1912// 1913 1914// 64 bit unscaled offsets 1915def int_aarch64_sve_ld1_gather : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic; 1916 1917// 64 bit scaled offsets 1918def int_aarch64_sve_ld1_gather_index : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic; 1919 1920// 32 bit unscaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits 1921def int_aarch64_sve_ld1_gather_sxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic; 1922def int_aarch64_sve_ld1_gather_uxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic; 1923 1924// 32 bit scaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits 1925def int_aarch64_sve_ld1_gather_sxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic; 1926def int_aarch64_sve_ld1_gather_uxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic; 1927 1928// 1929// Gather loads: vector base + scalar offset 1930// 1931 1932def int_aarch64_sve_ld1_gather_scalar_offset : AdvSIMD_GatherLoad_VS_Intrinsic; 1933 1934 1935// 1936// First-faulting gather loads: scalar base + vector offsets 1937// 1938 1939// 64 bit unscaled offsets 1940def int_aarch64_sve_ldff1_gather : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic; 1941 1942// 64 bit scaled offsets 1943def int_aarch64_sve_ldff1_gather_index : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic; 1944 1945// 32 bit unscaled offsets, sign (sxtw) or zero (uxtw) extended to 64 bits 1946def int_aarch64_sve_ldff1_gather_sxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic; 1947def int_aarch64_sve_ldff1_gather_uxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic; 1948 1949// 32 bit scaled offsets, sign (sxtw) or zero (uxtw) extended to 64 bits 1950def int_aarch64_sve_ldff1_gather_sxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic; 1951def int_aarch64_sve_ldff1_gather_uxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic; 1952 1953// 1954// First-faulting gather loads: vector base + scalar offset 1955// 1956 1957def int_aarch64_sve_ldff1_gather_scalar_offset : AdvSIMD_GatherLoad_VS_Intrinsic; 1958 1959 1960// 1961// Non-temporal gather loads: scalar base + vector offsets 1962// 1963 1964// 64 bit unscaled offsets 1965def int_aarch64_sve_ldnt1_gather : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic; 1966 1967// 64 bit indices 1968def int_aarch64_sve_ldnt1_gather_index : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic; 1969 1970// 32 bit unscaled offsets, zero (zxtw) extended to 64 bits 1971def int_aarch64_sve_ldnt1_gather_uxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic; 1972 1973// 1974// Non-temporal gather loads: vector base + scalar offset 1975// 1976 1977def int_aarch64_sve_ldnt1_gather_scalar_offset : AdvSIMD_GatherLoad_VS_Intrinsic; 1978 1979// 1980// Scatter stores: scalar base + vector offsets 1981// 1982 1983// 64 bit unscaled offsets 1984def int_aarch64_sve_st1_scatter : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic; 1985 1986// 64 bit scaled offsets 1987def int_aarch64_sve_st1_scatter_index 1988 : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic; 1989 1990// 32 bit unscaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits 1991def int_aarch64_sve_st1_scatter_sxtw 1992 : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic; 1993 1994def int_aarch64_sve_st1_scatter_uxtw 1995 : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic; 1996 1997// 32 bit scaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits 1998def int_aarch64_sve_st1_scatter_sxtw_index 1999 : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic; 2000 2001def int_aarch64_sve_st1_scatter_uxtw_index 2002 : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic; 2003 2004// 2005// Scatter stores: vector base + scalar offset 2006// 2007 2008def int_aarch64_sve_st1_scatter_scalar_offset : AdvSIMD_ScatterStore_VS_Intrinsic; 2009 2010// 2011// Non-temporal scatter stores: scalar base + vector offsets 2012// 2013 2014// 64 bit unscaled offsets 2015def int_aarch64_sve_stnt1_scatter : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic; 2016 2017// 64 bit indices 2018def int_aarch64_sve_stnt1_scatter_index 2019 : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic; 2020 2021// 32 bit unscaled offsets, zero (zxtw) extended to 64 bits 2022def int_aarch64_sve_stnt1_scatter_uxtw : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic; 2023 2024// 2025// Non-temporal scatter stores: vector base + scalar offset 2026// 2027 2028def int_aarch64_sve_stnt1_scatter_scalar_offset : AdvSIMD_ScatterStore_VS_Intrinsic; 2029 2030// 2031// SVE2 - Uniform DSP operations 2032// 2033 2034def int_aarch64_sve_saba : AdvSIMD_3VectorArg_Intrinsic; 2035def int_aarch64_sve_shadd : AdvSIMD_Pred2VectorArg_Intrinsic; 2036def int_aarch64_sve_shsub : AdvSIMD_Pred2VectorArg_Intrinsic; 2037def int_aarch64_sve_shsubr : AdvSIMD_Pred2VectorArg_Intrinsic; 2038def int_aarch64_sve_sli : AdvSIMD_2VectorArgIndexed_Intrinsic; 2039def int_aarch64_sve_sqabs : AdvSIMD_Merged1VectorArg_Intrinsic; 2040def int_aarch64_sve_sqadd : AdvSIMD_Pred2VectorArg_Intrinsic; 2041def int_aarch64_sve_sqdmulh : AdvSIMD_2VectorArg_Intrinsic; 2042def int_aarch64_sve_sqdmulh_lane : AdvSIMD_2VectorArgIndexed_Intrinsic; 2043def int_aarch64_sve_sqneg : AdvSIMD_Merged1VectorArg_Intrinsic; 2044def int_aarch64_sve_sqrdmlah : AdvSIMD_3VectorArg_Intrinsic; 2045def int_aarch64_sve_sqrdmlah_lane : AdvSIMD_3VectorArgIndexed_Intrinsic; 2046def int_aarch64_sve_sqrdmlsh : AdvSIMD_3VectorArg_Intrinsic; 2047def int_aarch64_sve_sqrdmlsh_lane : AdvSIMD_3VectorArgIndexed_Intrinsic; 2048def int_aarch64_sve_sqrdmulh : AdvSIMD_2VectorArg_Intrinsic; 2049def int_aarch64_sve_sqrdmulh_lane : AdvSIMD_2VectorArgIndexed_Intrinsic; 2050def int_aarch64_sve_sqrshl : AdvSIMD_Pred2VectorArg_Intrinsic; 2051def int_aarch64_sve_sqshl : AdvSIMD_Pred2VectorArg_Intrinsic; 2052def int_aarch64_sve_sqshlu : AdvSIMD_SVE_ShiftByImm_Intrinsic; 2053def int_aarch64_sve_sqsub : AdvSIMD_Pred2VectorArg_Intrinsic; 2054def int_aarch64_sve_sqsubr : AdvSIMD_Pred2VectorArg_Intrinsic; 2055def int_aarch64_sve_srhadd : AdvSIMD_Pred2VectorArg_Intrinsic; 2056def int_aarch64_sve_sri : AdvSIMD_2VectorArgIndexed_Intrinsic; 2057def int_aarch64_sve_srshl : AdvSIMD_Pred2VectorArg_Intrinsic; 2058def int_aarch64_sve_srshr : AdvSIMD_SVE_ShiftByImm_Intrinsic; 2059def int_aarch64_sve_srsra : AdvSIMD_2VectorArgIndexed_Intrinsic; 2060def int_aarch64_sve_ssra : AdvSIMD_2VectorArgIndexed_Intrinsic; 2061def int_aarch64_sve_suqadd : AdvSIMD_Pred2VectorArg_Intrinsic; 2062def int_aarch64_sve_uaba : AdvSIMD_3VectorArg_Intrinsic; 2063def int_aarch64_sve_uhadd : AdvSIMD_Pred2VectorArg_Intrinsic; 2064def int_aarch64_sve_uhsub : AdvSIMD_Pred2VectorArg_Intrinsic; 2065def int_aarch64_sve_uhsubr : AdvSIMD_Pred2VectorArg_Intrinsic; 2066def int_aarch64_sve_uqadd : AdvSIMD_Pred2VectorArg_Intrinsic; 2067def int_aarch64_sve_uqrshl : AdvSIMD_Pred2VectorArg_Intrinsic; 2068def int_aarch64_sve_uqshl : AdvSIMD_Pred2VectorArg_Intrinsic; 2069def int_aarch64_sve_uqsub : AdvSIMD_Pred2VectorArg_Intrinsic; 2070def int_aarch64_sve_uqsubr : AdvSIMD_Pred2VectorArg_Intrinsic; 2071def int_aarch64_sve_urecpe : AdvSIMD_Merged1VectorArg_Intrinsic; 2072def int_aarch64_sve_urhadd : AdvSIMD_Pred2VectorArg_Intrinsic; 2073def int_aarch64_sve_urshl : AdvSIMD_Pred2VectorArg_Intrinsic; 2074def int_aarch64_sve_urshr : AdvSIMD_SVE_ShiftByImm_Intrinsic; 2075def int_aarch64_sve_ursqrte : AdvSIMD_Merged1VectorArg_Intrinsic; 2076def int_aarch64_sve_ursra : AdvSIMD_2VectorArgIndexed_Intrinsic; 2077def int_aarch64_sve_usqadd : AdvSIMD_Pred2VectorArg_Intrinsic; 2078def int_aarch64_sve_usra : AdvSIMD_2VectorArgIndexed_Intrinsic; 2079 2080// 2081// SVE2 - Widening DSP operations 2082// 2083 2084def int_aarch64_sve_sabalb : SVE2_3VectorArg_Long_Intrinsic; 2085def int_aarch64_sve_sabalt : SVE2_3VectorArg_Long_Intrinsic; 2086def int_aarch64_sve_sabdlb : SVE2_2VectorArg_Long_Intrinsic; 2087def int_aarch64_sve_sabdlt : SVE2_2VectorArg_Long_Intrinsic; 2088def int_aarch64_sve_saddlb : SVE2_2VectorArg_Long_Intrinsic; 2089def int_aarch64_sve_saddlt : SVE2_2VectorArg_Long_Intrinsic; 2090def int_aarch64_sve_saddwb : SVE2_2VectorArg_Wide_Intrinsic; 2091def int_aarch64_sve_saddwt : SVE2_2VectorArg_Wide_Intrinsic; 2092def int_aarch64_sve_sshllb : SVE2_1VectorArg_Long_Intrinsic; 2093def int_aarch64_sve_sshllt : SVE2_1VectorArg_Long_Intrinsic; 2094def int_aarch64_sve_ssublb : SVE2_2VectorArg_Long_Intrinsic; 2095def int_aarch64_sve_ssublt : SVE2_2VectorArg_Long_Intrinsic; 2096def int_aarch64_sve_ssubwb : SVE2_2VectorArg_Wide_Intrinsic; 2097def int_aarch64_sve_ssubwt : SVE2_2VectorArg_Wide_Intrinsic; 2098def int_aarch64_sve_uabalb : SVE2_3VectorArg_Long_Intrinsic; 2099def int_aarch64_sve_uabalt : SVE2_3VectorArg_Long_Intrinsic; 2100def int_aarch64_sve_uabdlb : SVE2_2VectorArg_Long_Intrinsic; 2101def int_aarch64_sve_uabdlt : SVE2_2VectorArg_Long_Intrinsic; 2102def int_aarch64_sve_uaddlb : SVE2_2VectorArg_Long_Intrinsic; 2103def int_aarch64_sve_uaddlt : SVE2_2VectorArg_Long_Intrinsic; 2104def int_aarch64_sve_uaddwb : SVE2_2VectorArg_Wide_Intrinsic; 2105def int_aarch64_sve_uaddwt : SVE2_2VectorArg_Wide_Intrinsic; 2106def int_aarch64_sve_ushllb : SVE2_1VectorArg_Long_Intrinsic; 2107def int_aarch64_sve_ushllt : SVE2_1VectorArg_Long_Intrinsic; 2108def int_aarch64_sve_usublb : SVE2_2VectorArg_Long_Intrinsic; 2109def int_aarch64_sve_usublt : SVE2_2VectorArg_Long_Intrinsic; 2110def int_aarch64_sve_usubwb : SVE2_2VectorArg_Wide_Intrinsic; 2111def int_aarch64_sve_usubwt : SVE2_2VectorArg_Wide_Intrinsic; 2112 2113// 2114// SVE2 - Non-widening pairwise arithmetic 2115// 2116 2117def int_aarch64_sve_addp : AdvSIMD_Pred2VectorArg_Intrinsic; 2118def int_aarch64_sve_faddp : AdvSIMD_Pred2VectorArg_Intrinsic; 2119def int_aarch64_sve_fmaxp : AdvSIMD_Pred2VectorArg_Intrinsic; 2120def int_aarch64_sve_fmaxnmp : AdvSIMD_Pred2VectorArg_Intrinsic; 2121def int_aarch64_sve_fminp : AdvSIMD_Pred2VectorArg_Intrinsic; 2122def int_aarch64_sve_fminnmp : AdvSIMD_Pred2VectorArg_Intrinsic; 2123def int_aarch64_sve_smaxp : AdvSIMD_Pred2VectorArg_Intrinsic; 2124def int_aarch64_sve_sminp : AdvSIMD_Pred2VectorArg_Intrinsic; 2125def int_aarch64_sve_umaxp : AdvSIMD_Pred2VectorArg_Intrinsic; 2126def int_aarch64_sve_uminp : AdvSIMD_Pred2VectorArg_Intrinsic; 2127 2128// 2129// SVE2 - Widening pairwise arithmetic 2130// 2131 2132def int_aarch64_sve_sadalp : SVE2_2VectorArg_Pred_Long_Intrinsic; 2133def int_aarch64_sve_uadalp : SVE2_2VectorArg_Pred_Long_Intrinsic; 2134 2135// 2136// SVE2 - Uniform complex integer arithmetic 2137// 2138 2139def int_aarch64_sve_cadd_x : AdvSIMD_SVE2_CADD_Intrinsic; 2140def int_aarch64_sve_sqcadd_x : AdvSIMD_SVE2_CADD_Intrinsic; 2141def int_aarch64_sve_cmla_x : AdvSIMD_SVE2_CMLA_Intrinsic; 2142def int_aarch64_sve_cmla_lane_x : AdvSIMD_SVE_CMLA_LANE_Intrinsic; 2143def int_aarch64_sve_sqrdcmlah_x : AdvSIMD_SVE2_CMLA_Intrinsic; 2144def int_aarch64_sve_sqrdcmlah_lane_x : AdvSIMD_SVE_CMLA_LANE_Intrinsic; 2145 2146// 2147// SVE2 - Widening complex integer arithmetic 2148// 2149 2150def int_aarch64_sve_saddlbt : SVE2_2VectorArg_Long_Intrinsic; 2151def int_aarch64_sve_ssublbt : SVE2_2VectorArg_Long_Intrinsic; 2152def int_aarch64_sve_ssubltb : SVE2_2VectorArg_Long_Intrinsic; 2153 2154// 2155// SVE2 - Widening complex integer dot product 2156// 2157 2158def int_aarch64_sve_cdot : AdvSIMD_SVE_DOT_Indexed_Intrinsic; 2159def int_aarch64_sve_cdot_lane : AdvSIMD_SVE_CDOT_LANE_Intrinsic; 2160 2161// 2162// SVE2 - Floating-point widening multiply-accumulate 2163// 2164 2165def int_aarch64_sve_fmlalb : SVE2_3VectorArg_Long_Intrinsic; 2166def int_aarch64_sve_fmlalb_lane : SVE2_3VectorArgIndexed_Long_Intrinsic; 2167def int_aarch64_sve_fmlalt : SVE2_3VectorArg_Long_Intrinsic; 2168def int_aarch64_sve_fmlalt_lane : SVE2_3VectorArgIndexed_Long_Intrinsic; 2169def int_aarch64_sve_fmlslb : SVE2_3VectorArg_Long_Intrinsic; 2170def int_aarch64_sve_fmlslb_lane : SVE2_3VectorArgIndexed_Long_Intrinsic; 2171def int_aarch64_sve_fmlslt : SVE2_3VectorArg_Long_Intrinsic; 2172def int_aarch64_sve_fmlslt_lane : SVE2_3VectorArgIndexed_Long_Intrinsic; 2173 2174// 2175// SVE2 - Floating-point integer binary logarithm 2176// 2177 2178def int_aarch64_sve_flogb : AdvSIMD_SVE_LOGB_Intrinsic; 2179 2180// 2181// SVE2 - Vector histogram count 2182// 2183 2184def int_aarch64_sve_histcnt : AdvSIMD_Pred2VectorArg_Intrinsic; 2185def int_aarch64_sve_histseg : AdvSIMD_2VectorArg_Intrinsic; 2186 2187// 2188// SVE2 - Character match 2189// 2190 2191def int_aarch64_sve_match : AdvSIMD_SVE_Compare_Intrinsic; 2192def int_aarch64_sve_nmatch : AdvSIMD_SVE_Compare_Intrinsic; 2193 2194// 2195// SVE2 - Unary narrowing operations 2196// 2197 2198def int_aarch64_sve_sqxtnb : SVE2_1VectorArg_Narrowing_Intrinsic; 2199def int_aarch64_sve_sqxtnt : SVE2_Merged1VectorArg_Narrowing_Intrinsic; 2200def int_aarch64_sve_sqxtunb : SVE2_1VectorArg_Narrowing_Intrinsic; 2201def int_aarch64_sve_sqxtunt : SVE2_Merged1VectorArg_Narrowing_Intrinsic; 2202def int_aarch64_sve_uqxtnb : SVE2_1VectorArg_Narrowing_Intrinsic; 2203def int_aarch64_sve_uqxtnt : SVE2_Merged1VectorArg_Narrowing_Intrinsic; 2204 2205// 2206// SVE2 - Binary narrowing DSP operations 2207// 2208def int_aarch64_sve_addhnb : SVE2_2VectorArg_Narrowing_Intrinsic; 2209def int_aarch64_sve_addhnt : SVE2_Merged2VectorArg_Narrowing_Intrinsic; 2210 2211def int_aarch64_sve_raddhnb : SVE2_2VectorArg_Narrowing_Intrinsic; 2212def int_aarch64_sve_raddhnt : SVE2_Merged2VectorArg_Narrowing_Intrinsic; 2213 2214def int_aarch64_sve_subhnb : SVE2_2VectorArg_Narrowing_Intrinsic; 2215def int_aarch64_sve_subhnt : SVE2_Merged2VectorArg_Narrowing_Intrinsic; 2216 2217def int_aarch64_sve_rsubhnb : SVE2_2VectorArg_Narrowing_Intrinsic; 2218def int_aarch64_sve_rsubhnt : SVE2_Merged2VectorArg_Narrowing_Intrinsic; 2219 2220// Narrowing shift right 2221def int_aarch64_sve_shrnb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; 2222def int_aarch64_sve_shrnt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; 2223 2224def int_aarch64_sve_rshrnb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; 2225def int_aarch64_sve_rshrnt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; 2226 2227// Saturating shift right - signed input/output 2228def int_aarch64_sve_sqshrnb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; 2229def int_aarch64_sve_sqshrnt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; 2230 2231def int_aarch64_sve_sqrshrnb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; 2232def int_aarch64_sve_sqrshrnt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; 2233 2234// Saturating shift right - unsigned input/output 2235def int_aarch64_sve_uqshrnb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; 2236def int_aarch64_sve_uqshrnt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; 2237 2238def int_aarch64_sve_uqrshrnb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; 2239def int_aarch64_sve_uqrshrnt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; 2240 2241// Saturating shift right - signed input, unsigned output 2242def int_aarch64_sve_sqshrunb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; 2243def int_aarch64_sve_sqshrunt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; 2244 2245def int_aarch64_sve_sqrshrunb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic; 2246def int_aarch64_sve_sqrshrunt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic; 2247 2248// SVE2 MLA LANE. 2249def int_aarch64_sve_smlalb_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2250def int_aarch64_sve_smlalt_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2251def int_aarch64_sve_umlalb_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2252def int_aarch64_sve_umlalt_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2253def int_aarch64_sve_smlslb_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2254def int_aarch64_sve_smlslt_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2255def int_aarch64_sve_umlslb_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2256def int_aarch64_sve_umlslt_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2257def int_aarch64_sve_smullb_lane : SVE2_2VectorArgIndexed_Long_Intrinsic; 2258def int_aarch64_sve_smullt_lane : SVE2_2VectorArgIndexed_Long_Intrinsic; 2259def int_aarch64_sve_umullb_lane : SVE2_2VectorArgIndexed_Long_Intrinsic; 2260def int_aarch64_sve_umullt_lane : SVE2_2VectorArgIndexed_Long_Intrinsic; 2261def int_aarch64_sve_sqdmlalb_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2262def int_aarch64_sve_sqdmlalt_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2263def int_aarch64_sve_sqdmlslb_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2264def int_aarch64_sve_sqdmlslt_lane : SVE2_3VectorArg_Indexed_Intrinsic; 2265def int_aarch64_sve_sqdmullb_lane : SVE2_2VectorArgIndexed_Long_Intrinsic; 2266def int_aarch64_sve_sqdmullt_lane : SVE2_2VectorArgIndexed_Long_Intrinsic; 2267 2268// SVE2 MLA Unpredicated. 2269def int_aarch64_sve_smlalb : SVE2_3VectorArg_Long_Intrinsic; 2270def int_aarch64_sve_smlalt : SVE2_3VectorArg_Long_Intrinsic; 2271def int_aarch64_sve_umlalb : SVE2_3VectorArg_Long_Intrinsic; 2272def int_aarch64_sve_umlalt : SVE2_3VectorArg_Long_Intrinsic; 2273def int_aarch64_sve_smlslb : SVE2_3VectorArg_Long_Intrinsic; 2274def int_aarch64_sve_smlslt : SVE2_3VectorArg_Long_Intrinsic; 2275def int_aarch64_sve_umlslb : SVE2_3VectorArg_Long_Intrinsic; 2276def int_aarch64_sve_umlslt : SVE2_3VectorArg_Long_Intrinsic; 2277def int_aarch64_sve_smullb : SVE2_2VectorArg_Long_Intrinsic; 2278def int_aarch64_sve_smullt : SVE2_2VectorArg_Long_Intrinsic; 2279def int_aarch64_sve_umullb : SVE2_2VectorArg_Long_Intrinsic; 2280def int_aarch64_sve_umullt : SVE2_2VectorArg_Long_Intrinsic; 2281 2282def int_aarch64_sve_sqdmlalb : SVE2_3VectorArg_Long_Intrinsic; 2283def int_aarch64_sve_sqdmlalt : SVE2_3VectorArg_Long_Intrinsic; 2284def int_aarch64_sve_sqdmlslb : SVE2_3VectorArg_Long_Intrinsic; 2285def int_aarch64_sve_sqdmlslt : SVE2_3VectorArg_Long_Intrinsic; 2286def int_aarch64_sve_sqdmullb : SVE2_2VectorArg_Long_Intrinsic; 2287def int_aarch64_sve_sqdmullt : SVE2_2VectorArg_Long_Intrinsic; 2288def int_aarch64_sve_sqdmlalbt : SVE2_3VectorArg_Long_Intrinsic; 2289def int_aarch64_sve_sqdmlslbt : SVE2_3VectorArg_Long_Intrinsic; 2290 2291// SVE2 ADDSUB Long Unpredicated. 2292def int_aarch64_sve_adclb : AdvSIMD_3VectorArg_Intrinsic; 2293def int_aarch64_sve_adclt : AdvSIMD_3VectorArg_Intrinsic; 2294def int_aarch64_sve_sbclb : AdvSIMD_3VectorArg_Intrinsic; 2295def int_aarch64_sve_sbclt : AdvSIMD_3VectorArg_Intrinsic; 2296 2297// 2298// SVE2 - Polynomial arithmetic 2299// 2300def int_aarch64_sve_eorbt : AdvSIMD_3VectorArg_Intrinsic; 2301def int_aarch64_sve_eortb : AdvSIMD_3VectorArg_Intrinsic; 2302def int_aarch64_sve_pmullb_pair : AdvSIMD_2VectorArg_Intrinsic; 2303def int_aarch64_sve_pmullt_pair : AdvSIMD_2VectorArg_Intrinsic; 2304 2305// 2306// SVE2 bitwise ternary operations. 2307// 2308def int_aarch64_sve_eor3 : AdvSIMD_3VectorArg_Intrinsic; 2309def int_aarch64_sve_bcax : AdvSIMD_3VectorArg_Intrinsic; 2310def int_aarch64_sve_bsl : AdvSIMD_3VectorArg_Intrinsic; 2311def int_aarch64_sve_bsl1n : AdvSIMD_3VectorArg_Intrinsic; 2312def int_aarch64_sve_bsl2n : AdvSIMD_3VectorArg_Intrinsic; 2313def int_aarch64_sve_nbsl : AdvSIMD_3VectorArg_Intrinsic; 2314def int_aarch64_sve_xar : AdvSIMD_2VectorArgIndexed_Intrinsic; 2315 2316// 2317// SVE2 - Optional AES, SHA-3 and SM4 2318// 2319 2320def int_aarch64_sve_aesd : GCCBuiltin<"__builtin_sve_svaesd_u8">, 2321 Intrinsic<[llvm_nxv16i8_ty], 2322 [llvm_nxv16i8_ty, llvm_nxv16i8_ty], 2323 [IntrNoMem]>; 2324def int_aarch64_sve_aesimc : GCCBuiltin<"__builtin_sve_svaesimc_u8">, 2325 Intrinsic<[llvm_nxv16i8_ty], 2326 [llvm_nxv16i8_ty], 2327 [IntrNoMem]>; 2328def int_aarch64_sve_aese : GCCBuiltin<"__builtin_sve_svaese_u8">, 2329 Intrinsic<[llvm_nxv16i8_ty], 2330 [llvm_nxv16i8_ty, llvm_nxv16i8_ty], 2331 [IntrNoMem]>; 2332def int_aarch64_sve_aesmc : GCCBuiltin<"__builtin_sve_svaesmc_u8">, 2333 Intrinsic<[llvm_nxv16i8_ty], 2334 [llvm_nxv16i8_ty], 2335 [IntrNoMem]>; 2336def int_aarch64_sve_rax1 : GCCBuiltin<"__builtin_sve_svrax1_u64">, 2337 Intrinsic<[llvm_nxv2i64_ty], 2338 [llvm_nxv2i64_ty, llvm_nxv2i64_ty], 2339 [IntrNoMem]>; 2340def int_aarch64_sve_sm4e : GCCBuiltin<"__builtin_sve_svsm4e_u32">, 2341 Intrinsic<[llvm_nxv4i32_ty], 2342 [llvm_nxv4i32_ty, llvm_nxv4i32_ty], 2343 [IntrNoMem]>; 2344def int_aarch64_sve_sm4ekey : GCCBuiltin<"__builtin_sve_svsm4ekey_u32">, 2345 Intrinsic<[llvm_nxv4i32_ty], 2346 [llvm_nxv4i32_ty, llvm_nxv4i32_ty], 2347 [IntrNoMem]>; 2348// 2349// SVE2 - Extended table lookup/permute 2350// 2351 2352def int_aarch64_sve_tbl2 : AdvSIMD_SVE2_TBX_Intrinsic; 2353def int_aarch64_sve_tbx : AdvSIMD_SVE2_TBX_Intrinsic; 2354 2355// 2356// SVE2 - Optional bit permutation 2357// 2358 2359def int_aarch64_sve_bdep_x : AdvSIMD_2VectorArg_Intrinsic; 2360def int_aarch64_sve_bext_x : AdvSIMD_2VectorArg_Intrinsic; 2361def int_aarch64_sve_bgrp_x : AdvSIMD_2VectorArg_Intrinsic; 2362 2363 2364// 2365// SVE ACLE: 7.3. INT8 matrix multiply extensions 2366// 2367def int_aarch64_sve_ummla : SVE_MatMul_Intrinsic; 2368def int_aarch64_sve_smmla : SVE_MatMul_Intrinsic; 2369def int_aarch64_sve_usmmla : SVE_MatMul_Intrinsic; 2370 2371def int_aarch64_sve_usdot : AdvSIMD_SVE_DOT_Intrinsic; 2372def int_aarch64_sve_usdot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic; 2373def int_aarch64_sve_sudot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic; 2374 2375// 2376// SVE ACLE: 7.4/5. FP64/FP32 matrix multiply extensions 2377// 2378def int_aarch64_sve_fmmla : AdvSIMD_3VectorArg_Intrinsic; 2379 2380// 2381// SVE ACLE: 7.2. BFloat16 extensions 2382// 2383 2384def int_aarch64_sve_bfdot : SVE_4Vec_BF16; 2385def int_aarch64_sve_bfmlalb : SVE_4Vec_BF16; 2386def int_aarch64_sve_bfmlalt : SVE_4Vec_BF16; 2387 2388def int_aarch64_sve_bfmmla : SVE_4Vec_BF16; 2389 2390def int_aarch64_sve_bfdot_lane : SVE_4Vec_BF16_Indexed; 2391def int_aarch64_sve_bfmlalb_lane : SVE_4Vec_BF16_Indexed; 2392def int_aarch64_sve_bfmlalt_lane : SVE_4Vec_BF16_Indexed; 2393} 2394 2395// 2396// SVE2 - Contiguous conflict detection 2397// 2398 2399def int_aarch64_sve_whilerw_b : SVE2_CONFLICT_DETECT_Intrinsic; 2400def int_aarch64_sve_whilerw_h : SVE2_CONFLICT_DETECT_Intrinsic; 2401def int_aarch64_sve_whilerw_s : SVE2_CONFLICT_DETECT_Intrinsic; 2402def int_aarch64_sve_whilerw_d : SVE2_CONFLICT_DETECT_Intrinsic; 2403def int_aarch64_sve_whilewr_b : SVE2_CONFLICT_DETECT_Intrinsic; 2404def int_aarch64_sve_whilewr_h : SVE2_CONFLICT_DETECT_Intrinsic; 2405def int_aarch64_sve_whilewr_s : SVE2_CONFLICT_DETECT_Intrinsic; 2406def int_aarch64_sve_whilewr_d : SVE2_CONFLICT_DETECT_Intrinsic; 2407