1 /* 2 * Distributed under the Boost Software License, Version 1.0. 3 * (See accompanying file LICENSE_1_0.txt or copy at 4 * http://www.boost.org/LICENSE_1_0.txt) 5 * 6 * Copyright (c) 2020 Andrey Semashev 7 */ 8 /*! 9 * \file atomic/detail/extra_ops_gcc_aarch64.hpp 10 * 11 * This header contains implementation of the extra atomic operations for AArch64. 12 */ 13 14 #ifndef BOOST_ATOMIC_DETAIL_EXTRA_OPS_GCC_AARCH64_HPP_INCLUDED_ 15 #define BOOST_ATOMIC_DETAIL_EXTRA_OPS_GCC_AARCH64_HPP_INCLUDED_ 16 17 #include <cstddef> 18 #include <boost/cstdint.hpp> 19 #include <boost/memory_order.hpp> 20 #include <boost/atomic/detail/config.hpp> 21 #include <boost/atomic/detail/platform.hpp> 22 #include <boost/atomic/detail/storage_traits.hpp> 23 #include <boost/atomic/detail/extra_operations_fwd.hpp> 24 #include <boost/atomic/detail/extra_ops_generic.hpp> 25 #include <boost/atomic/detail/ops_gcc_aarch64_common.hpp> 26 #include <boost/atomic/detail/capabilities.hpp> 27 #include <boost/atomic/detail/header.hpp> 28 29 #ifdef BOOST_HAS_PRAGMA_ONCE 30 #pragma once 31 #endif 32 33 namespace boost { 34 namespace atomics { 35 namespace detail { 36 37 template< typename Base > 38 struct extra_operations_gcc_aarch64_common : 39 public Base 40 { 41 typedef Base base_type; 42 typedef typename base_type::storage_type storage_type; 43 44 // Note: For opaque operations prefer operations returning the resulting values instead of the original values 45 // as these operations require less registers. That is unless LSE is available, in which case 46 // it is better to use the dedicated atomic instructions. The LSE check is done in the base_type, 47 // where needed (e.g. for 128-bit operations there are no LSE instructions). opaque_negateboost::atomics::detail::extra_operations_gcc_aarch64_common48 static BOOST_FORCEINLINE void opaque_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 49 { 50 base_type::negate(storage, order); 51 } 52 opaque_complementboost::atomics::detail::extra_operations_gcc_aarch64_common53 static BOOST_FORCEINLINE void opaque_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 54 { 55 base_type::bitwise_complement(storage, order); 56 } 57 opaque_addboost::atomics::detail::extra_operations_gcc_aarch64_common58 static BOOST_FORCEINLINE void opaque_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 59 { 60 base_type::add(storage, v, order); 61 } 62 opaque_subboost::atomics::detail::extra_operations_gcc_aarch64_common63 static BOOST_FORCEINLINE void opaque_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 64 { 65 base_type::sub(storage, v, order); 66 } 67 opaque_andboost::atomics::detail::extra_operations_gcc_aarch64_common68 static BOOST_FORCEINLINE void opaque_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 69 { 70 base_type::bitwise_and(storage, v, order); 71 } 72 opaque_orboost::atomics::detail::extra_operations_gcc_aarch64_common73 static BOOST_FORCEINLINE void opaque_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 74 { 75 base_type::bitwise_or(storage, v, order); 76 } 77 opaque_xorboost::atomics::detail::extra_operations_gcc_aarch64_common78 static BOOST_FORCEINLINE void opaque_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 79 { 80 base_type::bitwise_xor(storage, v, order); 81 } 82 negate_and_testboost::atomics::detail::extra_operations_gcc_aarch64_common83 static BOOST_FORCEINLINE bool negate_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 84 { 85 return !!base_type::negate(storage, order); 86 } 87 add_and_testboost::atomics::detail::extra_operations_gcc_aarch64_common88 static BOOST_FORCEINLINE bool add_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 89 { 90 return !!base_type::add(storage, v, order); 91 } 92 sub_and_testboost::atomics::detail::extra_operations_gcc_aarch64_common93 static BOOST_FORCEINLINE bool sub_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 94 { 95 return !!base_type::sub(storage, v, order); 96 } 97 and_and_testboost::atomics::detail::extra_operations_gcc_aarch64_common98 static BOOST_FORCEINLINE bool and_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 99 { 100 return !!base_type::bitwise_and(storage, v, order); 101 } 102 or_and_testboost::atomics::detail::extra_operations_gcc_aarch64_common103 static BOOST_FORCEINLINE bool or_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 104 { 105 return !!base_type::bitwise_or(storage, v, order); 106 } 107 xor_and_testboost::atomics::detail::extra_operations_gcc_aarch64_common108 static BOOST_FORCEINLINE bool xor_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 109 { 110 return !!base_type::bitwise_xor(storage, v, order); 111 } 112 complement_and_testboost::atomics::detail::extra_operations_gcc_aarch64_common113 static BOOST_FORCEINLINE bool complement_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 114 { 115 return !!base_type::bitwise_complement(storage, order); 116 } 117 }; 118 119 template< typename Base, std::size_t Size, bool Signed > 120 struct extra_operations_gcc_aarch64; 121 122 template< typename Base, bool Signed > 123 struct extra_operations_gcc_aarch64< Base, 1u, Signed > : 124 public extra_operations_generic< Base, 1u, Signed > 125 { 126 typedef extra_operations_generic< Base, 1u, Signed > base_type; 127 typedef typename base_type::storage_type storage_type; 128 fetch_negateboost::atomics::detail::extra_operations_gcc_aarch64129 static BOOST_FORCEINLINE storage_type fetch_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 130 { 131 storage_type original, result; 132 uint32_t tmp; 133 134 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 135 __asm__ __volatile__\ 136 (\ 137 "1:\n\t"\ 138 "ld" ld_mo "xrb %w[original], %[storage]\n\t"\ 139 "neg %w[result], %w[original]\n\t"\ 140 "st" st_mo "xrb %w[tmp], %w[result], %[storage]\n\t"\ 141 "cbnz %w[tmp], 1b\n\t"\ 142 : [tmp] "=&r" (tmp), [result] "=&r" (result), [storage] "+Q" (storage), [original] "=&r" (original)\ 143 : \ 144 : "memory"\ 145 ); 146 147 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 148 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 149 150 return original; 151 } 152 negateboost::atomics::detail::extra_operations_gcc_aarch64153 static BOOST_FORCEINLINE storage_type negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 154 { 155 storage_type result; 156 uint32_t tmp; 157 158 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 159 __asm__ __volatile__\ 160 (\ 161 "1:\n\t"\ 162 "ld" ld_mo "xrb %w[result], %[storage]\n\t"\ 163 "neg %w[result], %w[result]\n\t"\ 164 "st" st_mo "xrb %w[tmp], %w[result], %[storage]\n\t"\ 165 "cbnz %w[tmp], 1b\n\t"\ 166 : [tmp] "=&r" (tmp), [storage] "+Q" (storage), [result] "=&r" (result)\ 167 : \ 168 : "memory"\ 169 ); 170 171 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 172 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 173 174 return result; 175 } 176 177 #if !defined(BOOST_ATOMIC_DETAIL_AARCH64_HAS_LSE) 178 addboost::atomics::detail::extra_operations_gcc_aarch64179 static BOOST_FORCEINLINE storage_type add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 180 { 181 storage_type result; 182 uint32_t tmp; 183 184 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 185 __asm__ __volatile__\ 186 (\ 187 "1:\n\t"\ 188 "ld" ld_mo "xrb %w[result], %[storage]\n\t"\ 189 "add %w[result], %w[result], %w[value]\n\t"\ 190 "st" st_mo "xrb %w[tmp], %w[result], %[storage]\n\t"\ 191 "cbnz %w[tmp], 1b\n\t"\ 192 : [tmp] "=&r" (tmp), [storage] "+Q" (storage), [result] "=&r" (result)\ 193 : [value] "Ir" (v)\ 194 : "memory"\ 195 ); 196 197 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 198 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 199 200 return result; 201 } 202 subboost::atomics::detail::extra_operations_gcc_aarch64203 static BOOST_FORCEINLINE storage_type sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 204 { 205 storage_type result; 206 uint32_t tmp; 207 208 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 209 __asm__ __volatile__\ 210 (\ 211 "1:\n\t"\ 212 "ld" ld_mo "xrb %w[result], %[storage]\n\t"\ 213 "sub %w[result], %w[result], %w[value]\n\t"\ 214 "st" st_mo "xrb %w[tmp], %w[result], %[storage]\n\t"\ 215 "cbnz %w[tmp], 1b\n\t"\ 216 : [tmp] "=&r" (tmp), [storage] "+Q" (storage), [result] "=&r" (result)\ 217 : [value] "Ir" (v)\ 218 : "memory"\ 219 ); 220 221 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 222 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 223 224 return result; 225 } 226 bitwise_andboost::atomics::detail::extra_operations_gcc_aarch64227 static BOOST_FORCEINLINE storage_type bitwise_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 228 { 229 storage_type result; 230 uint32_t tmp; 231 232 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 233 __asm__ __volatile__\ 234 (\ 235 "1:\n\t"\ 236 "ld" ld_mo "xrb %w[result], %[storage]\n\t"\ 237 "and %w[result], %w[result], %w[value]\n\t"\ 238 "st" st_mo "xrb %w[tmp], %w[result], %[storage]\n\t"\ 239 "cbnz %w[tmp], 1b\n\t"\ 240 : [tmp] "=&r" (tmp), [storage] "+Q" (storage), [result] "=&r" (result)\ 241 : [value] "Ir" (v)\ 242 : "memory"\ 243 ); 244 245 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 246 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 247 248 return result; 249 } 250 bitwise_orboost::atomics::detail::extra_operations_gcc_aarch64251 static BOOST_FORCEINLINE storage_type bitwise_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 252 { 253 storage_type result; 254 uint32_t tmp; 255 256 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 257 __asm__ __volatile__\ 258 (\ 259 "1:\n\t"\ 260 "ld" ld_mo "xrb %w[result], %[storage]\n\t"\ 261 "orr %w[result], %w[result], %w[value]\n\t"\ 262 "st" st_mo "xrb %w[tmp], %w[result], %[storage]\n\t"\ 263 "cbnz %w[tmp], 1b\n\t"\ 264 : [tmp] "=&r" (tmp), [storage] "+Q" (storage), [result] "=&r" (result)\ 265 : [value] "Ir" (v)\ 266 : "memory"\ 267 ); 268 269 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 270 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 271 272 return result; 273 } 274 bitwise_xorboost::atomics::detail::extra_operations_gcc_aarch64275 static BOOST_FORCEINLINE storage_type bitwise_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 276 { 277 storage_type result; 278 uint32_t tmp; 279 280 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 281 __asm__ __volatile__\ 282 (\ 283 "1:\n\t"\ 284 "ld" ld_mo "xrb %w[result], %[storage]\n\t"\ 285 "eor %w[result], %w[result], %w[value]\n\t"\ 286 "st" st_mo "xrb %w[tmp], %w[result], %[storage]\n\t"\ 287 "cbnz %w[tmp], 1b\n\t"\ 288 : [tmp] "=&r" (tmp), [storage] "+Q" (storage), [result] "=&r" (result)\ 289 : [value] "Ir" (v)\ 290 : "memory"\ 291 ); 292 293 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 294 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 295 296 return result; 297 } 298 fetch_complementboost::atomics::detail::extra_operations_gcc_aarch64299 static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 300 { 301 storage_type original, result; 302 uint32_t tmp; 303 304 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 305 __asm__ __volatile__\ 306 (\ 307 "1:\n\t"\ 308 "ld" ld_mo "xrb %w[original], %[storage]\n\t"\ 309 "mvn %w[result], %w[original]\n\t"\ 310 "st" st_mo "xrb %w[tmp], %w[result], %[storage]\n\t"\ 311 "cbnz %w[tmp], 1b\n\t"\ 312 : [tmp] "=&r" (tmp), [result] "=&r" (result), [storage] "+Q" (storage), [original] "=&r" (original)\ 313 : \ 314 : "memory"\ 315 ); 316 317 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 318 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 319 320 return original; 321 } 322 bitwise_complementboost::atomics::detail::extra_operations_gcc_aarch64323 static BOOST_FORCEINLINE storage_type bitwise_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 324 { 325 storage_type result; 326 uint32_t tmp; 327 328 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 329 __asm__ __volatile__\ 330 (\ 331 "1:\n\t"\ 332 "ld" ld_mo "xrb %w[result], %[storage]\n\t"\ 333 "mvn %w[result], %w[result]\n\t"\ 334 "st" st_mo "xrb %w[tmp], %w[result], %[storage]\n\t"\ 335 "cbnz %w[tmp], 1b\n\t"\ 336 : [tmp] "=&r" (tmp), [storage] "+Q" (storage), [result] "=&r" (result)\ 337 : \ 338 : "memory"\ 339 ); 340 341 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 342 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 343 344 return result; 345 } 346 347 #endif // !defined(BOOST_ATOMIC_DETAIL_AARCH64_HAS_LSE) 348 }; 349 350 template< typename Base, bool Signed > 351 struct extra_operations< Base, 1u, Signed, true > : 352 public extra_operations_gcc_aarch64_common< extra_operations_gcc_aarch64< Base, 1u, Signed > > 353 { 354 }; 355 356 357 template< typename Base, bool Signed > 358 struct extra_operations_gcc_aarch64< Base, 2u, Signed > : 359 public extra_operations_generic< Base, 2u, Signed > 360 { 361 typedef extra_operations_generic< Base, 2u, Signed > base_type; 362 typedef typename base_type::storage_type storage_type; 363 fetch_negateboost::atomics::detail::extra_operations_gcc_aarch64364 static BOOST_FORCEINLINE storage_type fetch_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 365 { 366 storage_type original, result; 367 uint32_t tmp; 368 369 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 370 __asm__ __volatile__\ 371 (\ 372 "1:\n\t"\ 373 "ld" ld_mo "xrh %w[original], %[storage]\n\t"\ 374 "neg %w[result], %w[original]\n\t"\ 375 "st" st_mo "xrh %w[tmp], %w[result], %[storage]\n\t"\ 376 "cbnz %w[tmp], 1b\n\t"\ 377 : [tmp] "=&r" (tmp), [result] "=&r" (result), [storage] "+Q" (storage), [original] "=&r" (original)\ 378 : \ 379 : "memory"\ 380 ); 381 382 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 383 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 384 385 return original; 386 } 387 negateboost::atomics::detail::extra_operations_gcc_aarch64388 static BOOST_FORCEINLINE storage_type negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 389 { 390 storage_type result; 391 uint32_t tmp; 392 393 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 394 __asm__ __volatile__\ 395 (\ 396 "1:\n\t"\ 397 "ld" ld_mo "xrh %w[result], %[storage]\n\t"\ 398 "neg %w[result], %w[result]\n\t"\ 399 "st" st_mo "xrh %w[tmp], %w[result], %[storage]\n\t"\ 400 "cbnz %w[tmp], 1b\n\t"\ 401 : [tmp] "=&r" (tmp), [storage] "+Q" (storage), [result] "=&r" (result)\ 402 : \ 403 : "memory"\ 404 ); 405 406 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 407 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 408 409 return result; 410 } 411 412 #if !defined(BOOST_ATOMIC_DETAIL_AARCH64_HAS_LSE) 413 addboost::atomics::detail::extra_operations_gcc_aarch64414 static BOOST_FORCEINLINE storage_type add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 415 { 416 storage_type result; 417 uint32_t tmp; 418 419 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 420 __asm__ __volatile__\ 421 (\ 422 "1:\n\t"\ 423 "ld" ld_mo "xrh %w[result], %[storage]\n\t"\ 424 "add %w[result], %w[result], %w[value]\n\t"\ 425 "st" st_mo "xrh %w[tmp], %w[result], %[storage]\n\t"\ 426 "cbnz %w[tmp], 1b\n\t"\ 427 : [tmp] "=&r" (tmp), [storage] "+Q" (storage), [result] "=&r" (result)\ 428 : [value] "Ir" (v)\ 429 : "memory"\ 430 ); 431 432 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 433 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 434 435 return result; 436 } 437 subboost::atomics::detail::extra_operations_gcc_aarch64438 static BOOST_FORCEINLINE storage_type sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 439 { 440 storage_type result; 441 uint32_t tmp; 442 443 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 444 __asm__ __volatile__\ 445 (\ 446 "1:\n\t"\ 447 "ld" ld_mo "xrh %w[result], %[storage]\n\t"\ 448 "sub %w[result], %w[result], %w[value]\n\t"\ 449 "st" st_mo "xrh %w[tmp], %w[result], %[storage]\n\t"\ 450 "cbnz %w[tmp], 1b\n\t"\ 451 : [tmp] "=&r" (tmp), [storage] "+Q" (storage), [result] "=&r" (result)\ 452 : [value] "Ir" (v)\ 453 : "memory"\ 454 ); 455 456 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 457 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 458 459 return result; 460 } 461 bitwise_andboost::atomics::detail::extra_operations_gcc_aarch64462 static BOOST_FORCEINLINE storage_type bitwise_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 463 { 464 storage_type result; 465 uint32_t tmp; 466 467 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 468 __asm__ __volatile__\ 469 (\ 470 "1:\n\t"\ 471 "ld" ld_mo "xrh %w[result], %[storage]\n\t"\ 472 "and %w[result], %w[result], %w[value]\n\t"\ 473 "st" st_mo "xrh %w[tmp], %w[result], %[storage]\n\t"\ 474 "cbnz %w[tmp], 1b\n\t"\ 475 : [tmp] "=&r" (tmp), [storage] "+Q" (storage), [result] "=&r" (result)\ 476 : [value] "Ir" (v)\ 477 : "memory"\ 478 ); 479 480 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 481 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 482 483 return result; 484 } 485 bitwise_orboost::atomics::detail::extra_operations_gcc_aarch64486 static BOOST_FORCEINLINE storage_type bitwise_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 487 { 488 storage_type result; 489 uint32_t tmp; 490 491 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 492 __asm__ __volatile__\ 493 (\ 494 "1:\n\t"\ 495 "ld" ld_mo "xrh %w[result], %[storage]\n\t"\ 496 "orr %w[result], %w[result], %w[value]\n\t"\ 497 "st" st_mo "xrh %w[tmp], %w[result], %[storage]\n\t"\ 498 "cbnz %w[tmp], 1b\n\t"\ 499 : [tmp] "=&r" (tmp), [storage] "+Q" (storage), [result] "=&r" (result)\ 500 : [value] "Ir" (v)\ 501 : "memory"\ 502 ); 503 504 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 505 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 506 507 return result; 508 } 509 bitwise_xorboost::atomics::detail::extra_operations_gcc_aarch64510 static BOOST_FORCEINLINE storage_type bitwise_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 511 { 512 storage_type result; 513 uint32_t tmp; 514 515 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 516 __asm__ __volatile__\ 517 (\ 518 "1:\n\t"\ 519 "ld" ld_mo "xrh %w[result], %[storage]\n\t"\ 520 "eor %w[result], %w[result], %w[value]\n\t"\ 521 "st" st_mo "xrh %w[tmp], %w[result], %[storage]\n\t"\ 522 "cbnz %w[tmp], 1b\n\t"\ 523 : [tmp] "=&r" (tmp), [storage] "+Q" (storage), [result] "=&r" (result)\ 524 : [value] "Ir" (v)\ 525 : "memory"\ 526 ); 527 528 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 529 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 530 531 return result; 532 } 533 fetch_complementboost::atomics::detail::extra_operations_gcc_aarch64534 static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 535 { 536 storage_type original, result; 537 uint32_t tmp; 538 539 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 540 __asm__ __volatile__\ 541 (\ 542 "1:\n\t"\ 543 "ld" ld_mo "xrh %w[original], %[storage]\n\t"\ 544 "mvn %w[result], %w[original]\n\t"\ 545 "st" st_mo "xrh %w[tmp], %w[result], %[storage]\n\t"\ 546 "cbnz %w[tmp], 1b\n\t"\ 547 : [tmp] "=&r" (tmp), [result] "=&r" (result), [storage] "+Q" (storage), [original] "=&r" (original)\ 548 : \ 549 : "memory"\ 550 ); 551 552 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 553 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 554 555 return original; 556 } 557 bitwise_complementboost::atomics::detail::extra_operations_gcc_aarch64558 static BOOST_FORCEINLINE storage_type bitwise_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 559 { 560 storage_type result; 561 uint32_t tmp; 562 563 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 564 __asm__ __volatile__\ 565 (\ 566 "1:\n\t"\ 567 "ld" ld_mo "xrh %w[result], %[storage]\n\t"\ 568 "mvn %w[result], %w[result]\n\t"\ 569 "st" st_mo "xrh %w[tmp], %w[result], %[storage]\n\t"\ 570 "cbnz %w[tmp], 1b\n\t"\ 571 : [tmp] "=&r" (tmp), [storage] "+Q" (storage), [result] "=&r" (result)\ 572 : \ 573 : "memory"\ 574 ); 575 576 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 577 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 578 579 return result; 580 } 581 582 #endif // !defined(BOOST_ATOMIC_DETAIL_AARCH64_HAS_LSE) 583 }; 584 585 template< typename Base, bool Signed > 586 struct extra_operations< Base, 2u, Signed, true > : 587 public extra_operations_gcc_aarch64_common< extra_operations_gcc_aarch64< Base, 2u, Signed > > 588 { 589 }; 590 591 592 template< typename Base, bool Signed > 593 struct extra_operations_gcc_aarch64< Base, 4u, Signed > : 594 public extra_operations_generic< Base, 4u, Signed > 595 { 596 typedef extra_operations_generic< Base, 4u, Signed > base_type; 597 typedef typename base_type::storage_type storage_type; 598 fetch_negateboost::atomics::detail::extra_operations_gcc_aarch64599 static BOOST_FORCEINLINE storage_type fetch_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 600 { 601 storage_type original, result; 602 uint32_t tmp; 603 604 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 605 __asm__ __volatile__\ 606 (\ 607 "1:\n\t"\ 608 "ld" ld_mo "xr %w[original], %[storage]\n\t"\ 609 "neg %w[result], %w[original]\n\t"\ 610 "st" st_mo "xr %w[tmp], %w[result], %[storage]\n\t"\ 611 "cbnz %w[tmp], 1b\n\t"\ 612 : [tmp] "=&r" (tmp), [result] "=&r" (result), [storage] "+Q" (storage), [original] "=&r" (original)\ 613 : \ 614 : "memory"\ 615 ); 616 617 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 618 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 619 620 return original; 621 } 622 negateboost::atomics::detail::extra_operations_gcc_aarch64623 static BOOST_FORCEINLINE storage_type negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 624 { 625 storage_type result; 626 uint32_t tmp; 627 628 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 629 __asm__ __volatile__\ 630 (\ 631 "1:\n\t"\ 632 "ld" ld_mo "xr %w[result], %[storage]\n\t"\ 633 "neg %w[result], %w[result]\n\t"\ 634 "st" st_mo "xr %w[tmp], %w[result], %[storage]\n\t"\ 635 "cbnz %w[tmp], 1b\n\t"\ 636 : [tmp] "=&r" (tmp), [storage] "+Q" (storage), [result] "=&r" (result)\ 637 : \ 638 : "memory"\ 639 ); 640 641 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 642 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 643 644 return result; 645 } 646 647 #if !defined(BOOST_ATOMIC_DETAIL_AARCH64_HAS_LSE) 648 addboost::atomics::detail::extra_operations_gcc_aarch64649 static BOOST_FORCEINLINE storage_type add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 650 { 651 storage_type result; 652 uint32_t tmp; 653 654 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 655 __asm__ __volatile__\ 656 (\ 657 "1:\n\t"\ 658 "ld" ld_mo "xr %w[result], %[storage]\n\t"\ 659 "add %w[result], %w[result], %w[value]\n\t"\ 660 "st" st_mo "xr %w[tmp], %w[result], %[storage]\n\t"\ 661 "cbnz %w[tmp], 1b\n\t"\ 662 : [tmp] "=&r" (tmp), [storage] "+Q" (storage), [result] "=&r" (result)\ 663 : [value] "Ir" (v)\ 664 : "memory"\ 665 ); 666 667 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 668 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 669 670 return result; 671 } 672 subboost::atomics::detail::extra_operations_gcc_aarch64673 static BOOST_FORCEINLINE storage_type sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 674 { 675 storage_type result; 676 uint32_t tmp; 677 678 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 679 __asm__ __volatile__\ 680 (\ 681 "1:\n\t"\ 682 "ld" ld_mo "xr %w[result], %[storage]\n\t"\ 683 "sub %w[result], %w[result], %w[value]\n\t"\ 684 "st" st_mo "xr %w[tmp], %w[result], %[storage]\n\t"\ 685 "cbnz %w[tmp], 1b\n\t"\ 686 : [tmp] "=&r" (tmp), [storage] "+Q" (storage), [result] "=&r" (result)\ 687 : [value] "Ir" (v)\ 688 : "memory"\ 689 ); 690 691 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 692 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 693 694 return result; 695 } 696 bitwise_andboost::atomics::detail::extra_operations_gcc_aarch64697 static BOOST_FORCEINLINE storage_type bitwise_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 698 { 699 storage_type result; 700 uint32_t tmp; 701 702 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 703 __asm__ __volatile__\ 704 (\ 705 "1:\n\t"\ 706 "ld" ld_mo "xr %w[result], %[storage]\n\t"\ 707 "and %w[result], %w[result], %w[value]\n\t"\ 708 "st" st_mo "xr %w[tmp], %w[result], %[storage]\n\t"\ 709 "cbnz %w[tmp], 1b\n\t"\ 710 : [tmp] "=&r" (tmp), [storage] "+Q" (storage), [result] "=&r" (result)\ 711 : [value] "Ir" (v)\ 712 : "memory"\ 713 ); 714 715 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 716 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 717 718 return result; 719 } 720 bitwise_orboost::atomics::detail::extra_operations_gcc_aarch64721 static BOOST_FORCEINLINE storage_type bitwise_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 722 { 723 storage_type result; 724 uint32_t tmp; 725 726 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 727 __asm__ __volatile__\ 728 (\ 729 "1:\n\t"\ 730 "ld" ld_mo "xr %w[result], %[storage]\n\t"\ 731 "orr %w[result], %w[result], %w[value]\n\t"\ 732 "st" st_mo "xr %w[tmp], %w[result], %[storage]\n\t"\ 733 "cbnz %w[tmp], 1b\n\t"\ 734 : [tmp] "=&r" (tmp), [storage] "+Q" (storage), [result] "=&r" (result)\ 735 : [value] "Ir" (v)\ 736 : "memory"\ 737 ); 738 739 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 740 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 741 742 return result; 743 } 744 bitwise_xorboost::atomics::detail::extra_operations_gcc_aarch64745 static BOOST_FORCEINLINE storage_type bitwise_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 746 { 747 storage_type result; 748 uint32_t tmp; 749 750 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 751 __asm__ __volatile__\ 752 (\ 753 "1:\n\t"\ 754 "ld" ld_mo "xr %w[result], %[storage]\n\t"\ 755 "eor %w[result], %w[result], %w[value]\n\t"\ 756 "st" st_mo "xr %w[tmp], %w[result], %[storage]\n\t"\ 757 "cbnz %w[tmp], 1b\n\t"\ 758 : [tmp] "=&r" (tmp), [storage] "+Q" (storage), [result] "=&r" (result)\ 759 : [value] "Ir" (v)\ 760 : "memory"\ 761 ); 762 763 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 764 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 765 766 return result; 767 } 768 fetch_complementboost::atomics::detail::extra_operations_gcc_aarch64769 static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 770 { 771 storage_type original, result; 772 uint32_t tmp; 773 774 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 775 __asm__ __volatile__\ 776 (\ 777 "1:\n\t"\ 778 "ld" ld_mo "xr %w[original], %[storage]\n\t"\ 779 "mvn %w[result], %w[original]\n\t"\ 780 "st" st_mo "xr %w[tmp], %w[result], %[storage]\n\t"\ 781 "cbnz %w[tmp], 1b\n\t"\ 782 : [tmp] "=&r" (tmp), [result] "=&r" (result), [storage] "+Q" (storage), [original] "=&r" (original)\ 783 : \ 784 : "memory"\ 785 ); 786 787 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 788 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 789 790 return original; 791 } 792 bitwise_complementboost::atomics::detail::extra_operations_gcc_aarch64793 static BOOST_FORCEINLINE storage_type bitwise_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 794 { 795 storage_type result; 796 uint32_t tmp; 797 798 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 799 __asm__ __volatile__\ 800 (\ 801 "1:\n\t"\ 802 "ld" ld_mo "xr %w[result], %[storage]\n\t"\ 803 "mvn %w[result], %w[result]\n\t"\ 804 "st" st_mo "xr %w[tmp], %w[result], %[storage]\n\t"\ 805 "cbnz %w[tmp], 1b\n\t"\ 806 : [tmp] "=&r" (tmp), [storage] "+Q" (storage), [result] "=&r" (result)\ 807 : \ 808 : "memory"\ 809 ); 810 811 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 812 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 813 814 return result; 815 } 816 817 #endif // !defined(BOOST_ATOMIC_DETAIL_AARCH64_HAS_LSE) 818 }; 819 820 template< typename Base, bool Signed > 821 struct extra_operations< Base, 4u, Signed, true > : 822 public extra_operations_gcc_aarch64_common< extra_operations_gcc_aarch64< Base, 4u, Signed > > 823 { 824 }; 825 826 827 template< typename Base, bool Signed > 828 struct extra_operations_gcc_aarch64< Base, 8u, Signed > : 829 public extra_operations_generic< Base, 8u, Signed > 830 { 831 typedef extra_operations_generic< Base, 8u, Signed > base_type; 832 typedef typename base_type::storage_type storage_type; 833 fetch_negateboost::atomics::detail::extra_operations_gcc_aarch64834 static BOOST_FORCEINLINE storage_type fetch_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 835 { 836 storage_type original, result; 837 uint32_t tmp; 838 839 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 840 __asm__ __volatile__\ 841 (\ 842 "1:\n\t"\ 843 "ld" ld_mo "xr %x[original], %[storage]\n\t"\ 844 "neg %x[result], %x[original]\n\t"\ 845 "st" st_mo "xr %w[tmp], %x[result], %[storage]\n\t"\ 846 "cbnz %w[tmp], 1b\n\t"\ 847 : [tmp] "=&r" (tmp), [result] "=&r" (result), [storage] "+Q" (storage), [original] "=&r" (original)\ 848 : \ 849 : "memory"\ 850 ); 851 852 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 853 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 854 855 return original; 856 } 857 negateboost::atomics::detail::extra_operations_gcc_aarch64858 static BOOST_FORCEINLINE storage_type negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 859 { 860 storage_type result; 861 uint32_t tmp; 862 863 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 864 __asm__ __volatile__\ 865 (\ 866 "1:\n\t"\ 867 "ld" ld_mo "xr %x[result], %[storage]\n\t"\ 868 "neg %x[result], %x[result]\n\t"\ 869 "st" st_mo "xr %w[tmp], %x[result], %[storage]\n\t"\ 870 "cbnz %w[tmp], 1b\n\t"\ 871 : [tmp] "=&r" (tmp), [storage] "+Q" (storage), [result] "=&r" (result)\ 872 : \ 873 : "memory"\ 874 ); 875 876 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 877 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 878 879 return result; 880 } 881 882 #if !defined(BOOST_ATOMIC_DETAIL_AARCH64_HAS_LSE) 883 addboost::atomics::detail::extra_operations_gcc_aarch64884 static BOOST_FORCEINLINE storage_type add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 885 { 886 storage_type result; 887 uint32_t tmp; 888 889 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 890 __asm__ __volatile__\ 891 (\ 892 "1:\n\t"\ 893 "ld" ld_mo "xr %x[result], %[storage]\n\t"\ 894 "add %x[result], %x[result], %x[value]\n\t"\ 895 "st" st_mo "xr %w[tmp], %x[result], %[storage]\n\t"\ 896 "cbnz %w[tmp], 1b\n\t"\ 897 : [tmp] "=&r" (tmp), [storage] "+Q" (storage), [result] "=&r" (result)\ 898 : [value] "Ir" (v)\ 899 : "memory"\ 900 ); 901 902 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 903 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 904 905 return result; 906 } 907 subboost::atomics::detail::extra_operations_gcc_aarch64908 static BOOST_FORCEINLINE storage_type sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 909 { 910 storage_type result; 911 uint32_t tmp; 912 913 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 914 __asm__ __volatile__\ 915 (\ 916 "1:\n\t"\ 917 "ld" ld_mo "xr %x[result], %[storage]\n\t"\ 918 "sub %x[result], %x[result], %x[value]\n\t"\ 919 "st" st_mo "xr %w[tmp], %x[result], %[storage]\n\t"\ 920 "cbnz %w[tmp], 1b\n\t"\ 921 : [tmp] "=&r" (tmp), [storage] "+Q" (storage), [result] "=&r" (result)\ 922 : [value] "Ir" (v)\ 923 : "memory"\ 924 ); 925 926 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 927 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 928 929 return result; 930 } 931 bitwise_andboost::atomics::detail::extra_operations_gcc_aarch64932 static BOOST_FORCEINLINE storage_type bitwise_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 933 { 934 storage_type result; 935 uint32_t tmp; 936 937 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 938 __asm__ __volatile__\ 939 (\ 940 "1:\n\t"\ 941 "ld" ld_mo "xr %x[result], %[storage]\n\t"\ 942 "and %x[result], %x[result], %x[value]\n\t"\ 943 "st" st_mo "xr %w[tmp], %x[result], %[storage]\n\t"\ 944 "cbnz %w[tmp], 1b\n\t"\ 945 : [tmp] "=&r" (tmp), [storage] "+Q" (storage), [result] "=&r" (result)\ 946 : [value] "Ir" (v)\ 947 : "memory"\ 948 ); 949 950 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 951 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 952 953 return result; 954 } 955 bitwise_orboost::atomics::detail::extra_operations_gcc_aarch64956 static BOOST_FORCEINLINE storage_type bitwise_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 957 { 958 storage_type result; 959 uint32_t tmp; 960 961 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 962 __asm__ __volatile__\ 963 (\ 964 "1:\n\t"\ 965 "ld" ld_mo "xr %x[result], %[storage]\n\t"\ 966 "orr %x[result], %x[result], %x[value]\n\t"\ 967 "st" st_mo "xr %w[tmp], %x[result], %[storage]\n\t"\ 968 "cbnz %w[tmp], 1b\n\t"\ 969 : [tmp] "=&r" (tmp), [storage] "+Q" (storage), [result] "=&r" (result)\ 970 : [value] "Ir" (v)\ 971 : "memory"\ 972 ); 973 974 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 975 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 976 977 return result; 978 } 979 bitwise_xorboost::atomics::detail::extra_operations_gcc_aarch64980 static BOOST_FORCEINLINE storage_type bitwise_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 981 { 982 storage_type result; 983 uint32_t tmp; 984 985 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 986 __asm__ __volatile__\ 987 (\ 988 "1:\n\t"\ 989 "ld" ld_mo "xr %x[result], %[storage]\n\t"\ 990 "eor %x[result], %x[result], %x[value]\n\t"\ 991 "st" st_mo "xr %w[tmp], %x[result], %[storage]\n\t"\ 992 "cbnz %w[tmp], 1b\n\t"\ 993 : [tmp] "=&r" (tmp), [storage] "+Q" (storage), [result] "=&r" (result)\ 994 : [value] "Ir" (v)\ 995 : "memory"\ 996 ); 997 998 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 999 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 1000 1001 return result; 1002 } 1003 fetch_complementboost::atomics::detail::extra_operations_gcc_aarch641004 static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 1005 { 1006 storage_type original, result; 1007 uint32_t tmp; 1008 1009 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 1010 __asm__ __volatile__\ 1011 (\ 1012 "1:\n\t"\ 1013 "ld" ld_mo "xr %x[original], %[storage]\n\t"\ 1014 "mvn %x[result], %x[original]\n\t"\ 1015 "st" st_mo "xr %w[tmp], %x[result], %[storage]\n\t"\ 1016 "cbnz %w[tmp], 1b\n\t"\ 1017 : [tmp] "=&r" (tmp), [result] "=&r" (result), [storage] "+Q" (storage), [original] "=&r" (original)\ 1018 : \ 1019 : "memory"\ 1020 ); 1021 1022 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 1023 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 1024 1025 return original; 1026 } 1027 bitwise_complementboost::atomics::detail::extra_operations_gcc_aarch641028 static BOOST_FORCEINLINE storage_type bitwise_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 1029 { 1030 storage_type result; 1031 uint32_t tmp; 1032 1033 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 1034 __asm__ __volatile__\ 1035 (\ 1036 "1:\n\t"\ 1037 "ld" ld_mo "xr %x[result], %[storage]\n\t"\ 1038 "mvn %x[result], %x[result]\n\t"\ 1039 "st" st_mo "xr %w[tmp], %x[result], %[storage]\n\t"\ 1040 "cbnz %w[tmp], 1b\n\t"\ 1041 : [tmp] "=&r" (tmp), [storage] "+Q" (storage), [result] "=&r" (result)\ 1042 : \ 1043 : "memory"\ 1044 ); 1045 1046 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 1047 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 1048 1049 return result; 1050 } 1051 1052 #endif // !defined(BOOST_ATOMIC_DETAIL_AARCH64_HAS_LSE) 1053 }; 1054 1055 template< typename Base, bool Signed > 1056 struct extra_operations< Base, 8u, Signed, true > : 1057 public extra_operations_gcc_aarch64_common< extra_operations_gcc_aarch64< Base, 8u, Signed > > 1058 { 1059 }; 1060 1061 1062 template< typename Base, bool Signed > 1063 struct extra_operations_gcc_aarch64< Base, 16u, Signed > : 1064 public extra_operations_generic< Base, 16u, Signed > 1065 { 1066 typedef extra_operations_generic< Base, 16u, Signed > base_type; 1067 typedef typename base_type::storage_type storage_type; 1068 typedef typename base_type::storage_union storage_union; 1069 fetch_negateboost::atomics::detail::extra_operations_gcc_aarch641070 static BOOST_FORCEINLINE storage_type fetch_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 1071 { 1072 storage_union original; 1073 storage_union result; 1074 uint32_t tmp; 1075 1076 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 1077 __asm__ __volatile__\ 1078 (\ 1079 "1:\n\t"\ 1080 "ld" ld_mo "xp %x[original_0], %x[original_1], %[storage]\n\t"\ 1081 "mvn %x[result_0], %x[original_0]\n\t"\ 1082 "mvn %x[result_1], %x[original_1]\n\t"\ 1083 "adds %x[result_" BOOST_ATOMIC_DETAIL_AARCH64_ASM_ARG_LO "], %x[result_" BOOST_ATOMIC_DETAIL_AARCH64_ASM_ARG_LO "], #1\n\t"\ 1084 "adc %x[result_" BOOST_ATOMIC_DETAIL_AARCH64_ASM_ARG_HI "], %x[result_" BOOST_ATOMIC_DETAIL_AARCH64_ASM_ARG_HI "], xzr\n\t"\ 1085 "st" st_mo "xp %w[tmp], %x[result_0], %x[result_1], %[storage]\n\t"\ 1086 "cbnz %w[tmp], 1b\n\t"\ 1087 : [tmp] "=&r" (tmp), [storage] "+Q" (storage),\ 1088 [original_0] "=&r" (original.as_uint64[0u]), [original_1] "=&r" (original.as_uint64[1u]),\ 1089 [result_0] "=&r" (result.as_uint64[0u]), [result_1] "=&r" (result.as_uint64[1u])\ 1090 : \ 1091 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\ 1092 ); 1093 1094 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 1095 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 1096 1097 return original.as_storage; 1098 } 1099 negateboost::atomics::detail::extra_operations_gcc_aarch641100 static BOOST_FORCEINLINE storage_type negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 1101 { 1102 storage_union result; 1103 uint32_t tmp; 1104 1105 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 1106 __asm__ __volatile__\ 1107 (\ 1108 "1:\n\t"\ 1109 "ld" ld_mo "xp %x[result_0], %x[result_1], %[storage]\n\t"\ 1110 "mvn %x[result_0], %x[result_0]\n\t"\ 1111 "mvn %x[result_1], %x[result_1]\n\t"\ 1112 "adds %x[result_" BOOST_ATOMIC_DETAIL_AARCH64_ASM_ARG_LO "], %x[result_" BOOST_ATOMIC_DETAIL_AARCH64_ASM_ARG_LO "], #1\n\t"\ 1113 "adc %x[result_" BOOST_ATOMIC_DETAIL_AARCH64_ASM_ARG_HI "], %x[result_" BOOST_ATOMIC_DETAIL_AARCH64_ASM_ARG_HI "], xzr\n\t"\ 1114 "st" st_mo "xp %w[tmp], %x[result_0], %x[result_1], %[storage]\n\t"\ 1115 "cbnz %w[tmp], 1b\n\t"\ 1116 : [tmp] "=&r" (tmp), [storage] "+Q" (storage),\ 1117 [result_0] "=&r" (result.as_uint64[0u]), [result_1] "=&r" (result.as_uint64[1u])\ 1118 : \ 1119 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\ 1120 ); 1121 1122 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 1123 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 1124 1125 return result.as_storage; 1126 } 1127 addboost::atomics::detail::extra_operations_gcc_aarch641128 static BOOST_FORCEINLINE storage_type add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 1129 { 1130 storage_union result; 1131 storage_union value = { v }; 1132 uint32_t tmp; 1133 1134 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 1135 __asm__ __volatile__\ 1136 (\ 1137 "1:\n\t"\ 1138 "ld" ld_mo "xp %x[result_0], %x[result_1], %[storage]\n\t"\ 1139 "adds %x[result_" BOOST_ATOMIC_DETAIL_AARCH64_ASM_ARG_LO "], %x[result_" BOOST_ATOMIC_DETAIL_AARCH64_ASM_ARG_LO "], %x[value_" BOOST_ATOMIC_DETAIL_AARCH64_ASM_ARG_LO "]\n\t"\ 1140 "adc %x[result_" BOOST_ATOMIC_DETAIL_AARCH64_ASM_ARG_HI "], %x[result_" BOOST_ATOMIC_DETAIL_AARCH64_ASM_ARG_HI "], %x[value_" BOOST_ATOMIC_DETAIL_AARCH64_ASM_ARG_HI "]\n\t"\ 1141 "st" st_mo "xp %w[tmp], %x[result_0], %x[result_1], %[storage]\n\t"\ 1142 "cbnz %w[tmp], 1b\n\t"\ 1143 : [tmp] "=&r" (tmp), [storage] "+Q" (storage),\ 1144 [result_0] "=&r" (result.as_uint64[0u]), [result_1] "=&r" (result.as_uint64[1u])\ 1145 : [value_0] "r" (value.as_uint64[0u]), [value_1] "r" (value.as_uint64[1u])\ 1146 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\ 1147 ); 1148 1149 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 1150 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 1151 1152 return result.as_storage; 1153 } 1154 subboost::atomics::detail::extra_operations_gcc_aarch641155 static BOOST_FORCEINLINE storage_type sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 1156 { 1157 storage_union result; 1158 storage_union value = { v }; 1159 uint32_t tmp; 1160 1161 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 1162 __asm__ __volatile__\ 1163 (\ 1164 "1:\n\t"\ 1165 "ld" ld_mo "xp %x[result_0], %x[result_1], %[storage]\n\t"\ 1166 "subs %x[result_" BOOST_ATOMIC_DETAIL_AARCH64_ASM_ARG_LO "], %x[result_" BOOST_ATOMIC_DETAIL_AARCH64_ASM_ARG_LO "], %x[value_" BOOST_ATOMIC_DETAIL_AARCH64_ASM_ARG_LO "]\n\t"\ 1167 "sbc %x[result_" BOOST_ATOMIC_DETAIL_AARCH64_ASM_ARG_HI "], %x[result_" BOOST_ATOMIC_DETAIL_AARCH64_ASM_ARG_HI "], %x[value_" BOOST_ATOMIC_DETAIL_AARCH64_ASM_ARG_HI "]\n\t"\ 1168 "st" st_mo "xp %w[tmp], %x[result_0], %x[result_1], %[storage]\n\t"\ 1169 "cbnz %w[tmp], 1b\n\t"\ 1170 : [tmp] "=&r" (tmp), [storage] "+Q" (storage),\ 1171 [result_0] "=&r" (result.as_uint64[0u]), [result_1] "=&r" (result.as_uint64[1u])\ 1172 : [value_0] "r" (value.as_uint64[0u]), [value_1] "r" (value.as_uint64[1u])\ 1173 : BOOST_ATOMIC_DETAIL_ASM_CLOBBER_CC_COMMA "memory"\ 1174 ); 1175 1176 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 1177 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 1178 1179 return result.as_storage; 1180 } 1181 bitwise_andboost::atomics::detail::extra_operations_gcc_aarch641182 static BOOST_FORCEINLINE storage_type bitwise_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 1183 { 1184 storage_union result; 1185 storage_union value = { v }; 1186 uint32_t tmp; 1187 1188 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 1189 __asm__ __volatile__\ 1190 (\ 1191 "1:\n\t"\ 1192 "ld" ld_mo "xp %x[result_0], %x[result_1], %[storage]\n\t"\ 1193 "and %x[result_0], %x[result_0], %x[value_0]\n\t"\ 1194 "and %x[result_1], %x[result_1], %x[value_1]\n\t"\ 1195 "st" st_mo "xp %w[tmp], %x[result_0], %x[result_1], %[storage]\n\t"\ 1196 "cbnz %w[tmp], 1b\n\t"\ 1197 : [tmp] "=&r" (tmp), [storage] "+Q" (storage),\ 1198 [result_0] "=&r" (result.as_uint64[0u]), [result_1] "=&r" (result.as_uint64[1u])\ 1199 : [value_0] "r" (value.as_uint64[0u]), [value_1] "r" (value.as_uint64[1u])\ 1200 : "memory"\ 1201 ); 1202 1203 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 1204 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 1205 1206 return result.as_storage; 1207 } 1208 bitwise_orboost::atomics::detail::extra_operations_gcc_aarch641209 static BOOST_FORCEINLINE storage_type bitwise_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 1210 { 1211 storage_union result; 1212 storage_union value = { v }; 1213 uint32_t tmp; 1214 1215 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 1216 __asm__ __volatile__\ 1217 (\ 1218 "1:\n\t"\ 1219 "ld" ld_mo "xp %x[result_0], %x[result_1], %[storage]\n\t"\ 1220 "orr %x[result_0], %x[result_0], %x[value_0]\n\t"\ 1221 "orr %x[result_1], %x[result_1], %x[value_1]\n\t"\ 1222 "st" st_mo "xp %w[tmp], %x[result_0], %x[result_1], %[storage]\n\t"\ 1223 "cbnz %w[tmp], 1b\n\t"\ 1224 : [tmp] "=&r" (tmp), [storage] "+Q" (storage),\ 1225 [result_0] "=&r" (result.as_uint64[0u]), [result_1] "=&r" (result.as_uint64[1u])\ 1226 : [value_0] "r" (value.as_uint64[0u]), [value_1] "r" (value.as_uint64[1u])\ 1227 : "memory"\ 1228 ); 1229 1230 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 1231 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 1232 1233 return result.as_storage; 1234 } 1235 bitwise_xorboost::atomics::detail::extra_operations_gcc_aarch641236 static BOOST_FORCEINLINE storage_type bitwise_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 1237 { 1238 storage_union result; 1239 storage_union value = { v }; 1240 uint32_t tmp; 1241 1242 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 1243 __asm__ __volatile__\ 1244 (\ 1245 "1:\n\t"\ 1246 "ld" ld_mo "xp %x[result_0], %x[result_1], %[storage]\n\t"\ 1247 "eor %x[result_0], %x[result_0], %x[value_0]\n\t"\ 1248 "eor %x[result_1], %x[result_1], %x[value_1]\n\t"\ 1249 "st" st_mo "xp %w[tmp], %x[result_0], %x[result_1], %[storage]\n\t"\ 1250 "cbnz %w[tmp], 1b\n\t"\ 1251 : [tmp] "=&r" (tmp), [storage] "+Q" (storage),\ 1252 [result_0] "=&r" (result.as_uint64[0u]), [result_1] "=&r" (result.as_uint64[1u])\ 1253 : [value_0] "r" (value.as_uint64[0u]), [value_1] "r" (value.as_uint64[1u])\ 1254 : "memory"\ 1255 ); 1256 1257 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 1258 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 1259 1260 return result.as_storage; 1261 } 1262 fetch_complementboost::atomics::detail::extra_operations_gcc_aarch641263 static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 1264 { 1265 storage_union original; 1266 storage_union result; 1267 uint32_t tmp; 1268 1269 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 1270 __asm__ __volatile__\ 1271 (\ 1272 "1:\n\t"\ 1273 "ld" ld_mo "xp %x[original_0], %x[original_1], %[storage]\n\t"\ 1274 "mvn %x[result_0], %x[original_0]\n\t"\ 1275 "mvn %x[result_1], %x[original_1]\n\t"\ 1276 "st" st_mo "xp %w[tmp], %x[result_0], %x[result_1], %[storage]\n\t"\ 1277 "cbnz %w[tmp], 1b\n\t"\ 1278 : [tmp] "=&r" (tmp), [storage] "+Q" (storage),\ 1279 [original_0] "=&r" (original.as_uint64[0u]), [original_1] "=&r" (original.as_uint64[1u]),\ 1280 [result_0] "=&r" (result.as_uint64[0u]), [result_1] "=&r" (result.as_uint64[1u])\ 1281 : \ 1282 : "memory"\ 1283 ); 1284 1285 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 1286 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 1287 1288 return original.as_storage; 1289 } 1290 bitwise_complementboost::atomics::detail::extra_operations_gcc_aarch641291 static BOOST_FORCEINLINE storage_type bitwise_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 1292 { 1293 storage_union result; 1294 uint32_t tmp; 1295 1296 #define BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN(ld_mo, st_mo)\ 1297 __asm__ __volatile__\ 1298 (\ 1299 "1:\n\t"\ 1300 "ld" ld_mo "xp %x[result_0], %x[result_1], %[storage]\n\t"\ 1301 "mvn %x[result_0], %x[result_0]\n\t"\ 1302 "mvn %x[result_1], %x[result_1]\n\t"\ 1303 "st" st_mo "xp %w[tmp], %x[result_0], %x[result_1], %[storage]\n\t"\ 1304 "cbnz %w[tmp], 1b\n\t"\ 1305 : [tmp] "=&r" (tmp), [storage] "+Q" (storage),\ 1306 [result_0] "=&r" (result.as_uint64[0u]), [result_1] "=&r" (result.as_uint64[1u])\ 1307 : \ 1308 : "memory"\ 1309 ); 1310 1311 BOOST_ATOMIC_DETAIL_AARCH64_MO_SWITCH(order) 1312 #undef BOOST_ATOMIC_DETAIL_AARCH64_MO_INSN 1313 1314 return result.as_storage; 1315 } 1316 }; 1317 1318 template< typename Base, bool Signed > 1319 struct extra_operations< Base, 16u, Signed, true > : 1320 public extra_operations_gcc_aarch64_common< extra_operations_gcc_aarch64< Base, 16u, Signed > > 1321 { 1322 }; 1323 1324 } // namespace detail 1325 } // namespace atomics 1326 } // namespace boost 1327 1328 #include <boost/atomic/detail/footer.hpp> 1329 1330 #endif // BOOST_ATOMIC_DETAIL_EXTRA_OPS_GCC_AARCH64_HPP_INCLUDED_ 1331