1 /* 2 * Distributed under the Boost Software License, Version 1.0. 3 * (See accompanying file LICENSE_1_0.txt or copy at 4 * http://www.boost.org/LICENSE_1_0.txt) 5 * 6 * Copyright (c) 2017 Andrey Semashev 7 */ 8 /*! 9 * \file atomic/detail/extra_ops_msvc_x86.hpp 10 * 11 * This header contains implementation of the extra atomic operations for x86. 12 */ 13 14 #ifndef BOOST_ATOMIC_DETAIL_EXTRA_OPS_MSVC_X86_HPP_INCLUDED_ 15 #define BOOST_ATOMIC_DETAIL_EXTRA_OPS_MSVC_X86_HPP_INCLUDED_ 16 17 #include <cstddef> 18 #include <boost/memory_order.hpp> 19 #include <boost/atomic/detail/config.hpp> 20 #include <boost/atomic/detail/interlocked.hpp> 21 #include <boost/atomic/detail/storage_traits.hpp> 22 #include <boost/atomic/detail/extra_operations_fwd.hpp> 23 #include <boost/atomic/detail/extra_ops_generic.hpp> 24 #include <boost/atomic/detail/header.hpp> 25 26 #ifdef BOOST_HAS_PRAGMA_ONCE 27 #pragma once 28 #endif 29 30 namespace boost { 31 namespace atomics { 32 namespace detail { 33 34 #if defined(_M_IX86) 35 36 template< typename Base, bool Signed > 37 struct extra_operations< Base, 1u, Signed, true > : 38 public extra_operations_generic< Base, 1u, Signed > 39 { 40 typedef extra_operations_generic< Base, 1u, Signed > base_type; 41 typedef typename base_type::storage_type storage_type; 42 fetch_negateboost::atomics::detail::extra_operations43 static BOOST_FORCEINLINE storage_type fetch_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 44 { 45 base_type::fence_before(order); 46 storage_type old_val; 47 __asm 48 { 49 mov ecx, storage 50 movzx eax, byte ptr [ecx] 51 align 16 52 again: 53 mov edx, eax 54 neg dl 55 lock cmpxchg byte ptr [ecx], dl 56 jne again 57 mov old_val, al 58 }; 59 base_type::fence_after(order); 60 return old_val; 61 } 62 negateboost::atomics::detail::extra_operations63 static BOOST_FORCEINLINE storage_type negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 64 { 65 base_type::fence_before(order); 66 storage_type new_val; 67 __asm 68 { 69 mov ecx, storage 70 movzx eax, byte ptr [ecx] 71 align 16 72 again: 73 mov edx, eax 74 neg dl 75 lock cmpxchg byte ptr [ecx], dl 76 jne again 77 mov new_val, dl 78 }; 79 base_type::fence_after(order); 80 return new_val; 81 } 82 negate_and_testboost::atomics::detail::extra_operations83 static BOOST_FORCEINLINE bool negate_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 84 { 85 base_type::fence_before(order); 86 bool result; 87 __asm 88 { 89 mov ecx, storage 90 movzx eax, byte ptr [ecx] 91 align 16 92 again: 93 mov edx, eax 94 neg dl 95 lock cmpxchg byte ptr [ecx], dl 96 jne again 97 test dl, dl 98 setnz result 99 }; 100 base_type::fence_after(order); 101 return result; 102 } 103 opaque_negateboost::atomics::detail::extra_operations104 static BOOST_FORCEINLINE void opaque_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 105 { 106 base_type::fence_before(order); 107 __asm 108 { 109 mov ecx, storage 110 movzx eax, byte ptr [ecx] 111 align 16 112 again: 113 mov edx, eax 114 neg dl 115 lock cmpxchg byte ptr [ecx], dl 116 jne again 117 }; 118 base_type::fence_after(order); 119 } 120 bitwise_andboost::atomics::detail::extra_operations121 static BOOST_FORCEINLINE storage_type bitwise_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 122 { 123 base_type::fence_before(order); 124 __asm 125 { 126 mov edi, storage 127 movzx ecx, v 128 xor edx, edx 129 movzx eax, byte ptr [edi] 130 align 16 131 again: 132 mov dl, al 133 and dl, cl 134 lock cmpxchg byte ptr [edi], dl 135 jne again 136 mov v, dl 137 }; 138 base_type::fence_after(order); 139 return v; 140 } 141 bitwise_orboost::atomics::detail::extra_operations142 static BOOST_FORCEINLINE storage_type bitwise_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 143 { 144 base_type::fence_before(order); 145 __asm 146 { 147 mov edi, storage 148 movzx ecx, v 149 xor edx, edx 150 movzx eax, byte ptr [edi] 151 align 16 152 again: 153 mov dl, al 154 or dl, cl 155 lock cmpxchg byte ptr [edi], dl 156 jne again 157 mov v, dl 158 }; 159 base_type::fence_after(order); 160 return v; 161 } 162 bitwise_xorboost::atomics::detail::extra_operations163 static BOOST_FORCEINLINE storage_type bitwise_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 164 { 165 base_type::fence_before(order); 166 __asm 167 { 168 mov edi, storage 169 movzx ecx, v 170 xor edx, edx 171 movzx eax, byte ptr [edi] 172 align 16 173 again: 174 mov dl, al 175 xor dl, cl 176 lock cmpxchg byte ptr [edi], dl 177 jne again 178 mov v, dl 179 }; 180 base_type::fence_after(order); 181 return v; 182 } 183 fetch_complementboost::atomics::detail::extra_operations184 static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 185 { 186 base_type::fence_before(order); 187 storage_type old_val; 188 __asm 189 { 190 mov ecx, storage 191 movzx eax, byte ptr [ecx] 192 align 16 193 again: 194 mov edx, eax 195 not dl 196 lock cmpxchg byte ptr [ecx], dl 197 jne again 198 mov old_val, al 199 }; 200 base_type::fence_after(order); 201 return old_val; 202 } 203 bitwise_complementboost::atomics::detail::extra_operations204 static BOOST_FORCEINLINE storage_type bitwise_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 205 { 206 base_type::fence_before(order); 207 storage_type new_val; 208 __asm 209 { 210 mov ecx, storage 211 movzx eax, byte ptr [ecx] 212 align 16 213 again: 214 mov edx, eax 215 not dl 216 lock cmpxchg byte ptr [ecx], dl 217 jne again 218 mov new_val, dl 219 }; 220 base_type::fence_after(order); 221 return new_val; 222 } 223 complement_and_testboost::atomics::detail::extra_operations224 static BOOST_FORCEINLINE bool complement_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 225 { 226 base_type::fence_before(order); 227 bool result; 228 __asm 229 { 230 mov ecx, storage 231 movzx eax, byte ptr [ecx] 232 align 16 233 again: 234 mov edx, eax 235 not dl 236 lock cmpxchg byte ptr [ecx], dl 237 jne again 238 test dl, dl 239 setnz result 240 }; 241 base_type::fence_after(order); 242 return result; 243 } 244 opaque_complementboost::atomics::detail::extra_operations245 static BOOST_FORCEINLINE void opaque_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 246 { 247 base_type::fence_before(order); 248 __asm 249 { 250 mov ecx, storage 251 movzx eax, byte ptr [ecx] 252 align 16 253 again: 254 mov edx, eax 255 not dl 256 lock cmpxchg byte ptr [ecx], dl 257 jne again 258 }; 259 base_type::fence_after(order); 260 } 261 opaque_addboost::atomics::detail::extra_operations262 static BOOST_FORCEINLINE void opaque_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 263 { 264 base_type::fence_before(order); 265 __asm 266 { 267 mov edx, storage 268 movzx eax, v 269 lock add byte ptr [edx], al 270 }; 271 base_type::fence_after(order); 272 } 273 opaque_subboost::atomics::detail::extra_operations274 static BOOST_FORCEINLINE void opaque_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 275 { 276 base_type::fence_before(order); 277 __asm 278 { 279 mov edx, storage 280 movzx eax, v 281 lock sub byte ptr [edx], al 282 }; 283 base_type::fence_after(order); 284 } 285 opaque_negateboost::atomics::detail::extra_operations286 static BOOST_FORCEINLINE void opaque_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 287 { 288 base_type::fence_before(order); 289 __asm 290 { 291 mov edx, storage 292 lock neg byte ptr [edx] 293 }; 294 base_type::fence_after(order); 295 } 296 opaque_andboost::atomics::detail::extra_operations297 static BOOST_FORCEINLINE void opaque_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 298 { 299 base_type::fence_before(order); 300 __asm 301 { 302 mov edx, storage 303 movzx eax, v 304 lock and byte ptr [edx], al 305 }; 306 base_type::fence_after(order); 307 } 308 opaque_orboost::atomics::detail::extra_operations309 static BOOST_FORCEINLINE void opaque_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 310 { 311 base_type::fence_before(order); 312 __asm 313 { 314 mov edx, storage 315 movzx eax, v 316 lock or byte ptr [edx], al 317 }; 318 base_type::fence_after(order); 319 } 320 opaque_xorboost::atomics::detail::extra_operations321 static BOOST_FORCEINLINE void opaque_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 322 { 323 base_type::fence_before(order); 324 __asm 325 { 326 mov edx, storage 327 movzx eax, v 328 lock xor byte ptr [edx], al 329 }; 330 base_type::fence_after(order); 331 } 332 opaque_complementboost::atomics::detail::extra_operations333 static BOOST_FORCEINLINE void opaque_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 334 { 335 base_type::fence_before(order); 336 __asm 337 { 338 mov edx, storage 339 lock not byte ptr [edx] 340 }; 341 base_type::fence_after(order); 342 } 343 add_and_testboost::atomics::detail::extra_operations344 static BOOST_FORCEINLINE bool add_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 345 { 346 base_type::fence_before(order); 347 bool result; 348 __asm 349 { 350 mov edx, storage 351 movzx eax, v 352 lock add byte ptr [edx], al 353 setnz result 354 }; 355 base_type::fence_after(order); 356 return result; 357 } 358 sub_and_testboost::atomics::detail::extra_operations359 static BOOST_FORCEINLINE bool sub_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 360 { 361 base_type::fence_before(order); 362 bool result; 363 __asm 364 { 365 mov edx, storage 366 movzx eax, v 367 lock sub byte ptr [edx], al 368 setnz result 369 }; 370 base_type::fence_after(order); 371 return result; 372 } 373 and_and_testboost::atomics::detail::extra_operations374 static BOOST_FORCEINLINE bool and_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 375 { 376 base_type::fence_before(order); 377 bool result; 378 __asm 379 { 380 mov edx, storage 381 movzx eax, v 382 lock and byte ptr [edx], al 383 setnz result 384 }; 385 base_type::fence_after(order); 386 return result; 387 } 388 or_and_testboost::atomics::detail::extra_operations389 static BOOST_FORCEINLINE bool or_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 390 { 391 base_type::fence_before(order); 392 bool result; 393 __asm 394 { 395 mov edx, storage 396 movzx eax, v 397 lock or byte ptr [edx], al 398 setnz result 399 }; 400 base_type::fence_after(order); 401 return result; 402 } 403 xor_and_testboost::atomics::detail::extra_operations404 static BOOST_FORCEINLINE bool xor_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 405 { 406 base_type::fence_before(order); 407 bool result; 408 __asm 409 { 410 mov edx, storage 411 movzx eax, v 412 lock xor byte ptr [edx], al 413 setnz result 414 }; 415 base_type::fence_after(order); 416 return result; 417 } 418 }; 419 420 template< typename Base, bool Signed > 421 struct extra_operations< Base, 2u, Signed, true > : 422 public extra_operations_generic< Base, 2u, Signed > 423 { 424 typedef extra_operations_generic< Base, 2u, Signed > base_type; 425 typedef typename base_type::storage_type storage_type; 426 fetch_negateboost::atomics::detail::extra_operations427 static BOOST_FORCEINLINE storage_type fetch_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 428 { 429 base_type::fence_before(order); 430 storage_type old_val; 431 __asm 432 { 433 mov ecx, storage 434 movzx eax, word ptr [ecx] 435 align 16 436 again: 437 mov edx, eax 438 neg dx 439 lock cmpxchg word ptr [ecx], dx 440 jne again 441 mov old_val, ax 442 }; 443 base_type::fence_after(order); 444 return old_val; 445 } 446 negateboost::atomics::detail::extra_operations447 static BOOST_FORCEINLINE storage_type negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 448 { 449 base_type::fence_before(order); 450 storage_type new_val; 451 __asm 452 { 453 mov ecx, storage 454 movzx eax, word ptr [ecx] 455 align 16 456 again: 457 mov edx, eax 458 neg dx 459 lock cmpxchg word ptr [ecx], dx 460 jne again 461 mov new_val, dx 462 }; 463 base_type::fence_after(order); 464 return new_val; 465 } 466 negate_and_testboost::atomics::detail::extra_operations467 static BOOST_FORCEINLINE bool negate_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 468 { 469 base_type::fence_before(order); 470 bool result; 471 __asm 472 { 473 mov ecx, storage 474 movzx eax, word ptr [ecx] 475 align 16 476 again: 477 mov edx, eax 478 neg dx 479 lock cmpxchg word ptr [ecx], dx 480 jne again 481 test dx, dx 482 setnz result 483 }; 484 base_type::fence_after(order); 485 return result; 486 } 487 opaque_negateboost::atomics::detail::extra_operations488 static BOOST_FORCEINLINE void opaque_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 489 { 490 base_type::fence_before(order); 491 __asm 492 { 493 mov ecx, storage 494 movzx eax, word ptr [ecx] 495 align 16 496 again: 497 mov edx, eax 498 neg dx 499 lock cmpxchg word ptr [ecx], dx 500 jne again 501 }; 502 base_type::fence_after(order); 503 } 504 bitwise_andboost::atomics::detail::extra_operations505 static BOOST_FORCEINLINE storage_type bitwise_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 506 { 507 base_type::fence_before(order); 508 __asm 509 { 510 mov edi, storage 511 movzx ecx, v 512 xor edx, edx 513 movzx eax, word ptr [edi] 514 align 16 515 again: 516 mov dx, ax 517 and dx, cx 518 lock cmpxchg word ptr [edi], dx 519 jne again 520 mov v, dx 521 }; 522 base_type::fence_after(order); 523 return v; 524 } 525 bitwise_orboost::atomics::detail::extra_operations526 static BOOST_FORCEINLINE storage_type bitwise_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 527 { 528 base_type::fence_before(order); 529 __asm 530 { 531 mov edi, storage 532 movzx ecx, v 533 xor edx, edx 534 movzx eax, word ptr [edi] 535 align 16 536 again: 537 mov dx, ax 538 or dx, cx 539 lock cmpxchg word ptr [edi], dx 540 jne again 541 mov v, dx 542 }; 543 base_type::fence_after(order); 544 return v; 545 } 546 bitwise_xorboost::atomics::detail::extra_operations547 static BOOST_FORCEINLINE storage_type bitwise_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 548 { 549 base_type::fence_before(order); 550 __asm 551 { 552 mov edi, storage 553 movzx ecx, v 554 xor edx, edx 555 movzx eax, word ptr [edi] 556 align 16 557 again: 558 mov dx, ax 559 xor dx, cx 560 lock cmpxchg word ptr [edi], dx 561 jne again 562 mov v, dx 563 }; 564 base_type::fence_after(order); 565 return v; 566 } 567 fetch_complementboost::atomics::detail::extra_operations568 static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 569 { 570 base_type::fence_before(order); 571 storage_type old_val; 572 __asm 573 { 574 mov ecx, storage 575 movzx eax, word ptr [ecx] 576 align 16 577 again: 578 mov edx, eax 579 not dx 580 lock cmpxchg word ptr [ecx], dx 581 jne again 582 mov old_val, ax 583 }; 584 base_type::fence_after(order); 585 return old_val; 586 } 587 bitwise_complementboost::atomics::detail::extra_operations588 static BOOST_FORCEINLINE storage_type bitwise_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 589 { 590 base_type::fence_before(order); 591 storage_type new_val; 592 __asm 593 { 594 mov ecx, storage 595 movzx eax, word ptr [ecx] 596 align 16 597 again: 598 mov edx, eax 599 not dx 600 lock cmpxchg word ptr [ecx], dx 601 jne again 602 mov new_val, dx 603 }; 604 base_type::fence_after(order); 605 return new_val; 606 } 607 complement_and_testboost::atomics::detail::extra_operations608 static BOOST_FORCEINLINE bool complement_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 609 { 610 base_type::fence_before(order); 611 bool result; 612 __asm 613 { 614 mov ecx, storage 615 movzx eax, word ptr [ecx] 616 align 16 617 again: 618 mov edx, eax 619 not dx 620 lock cmpxchg word ptr [ecx], dx 621 jne again 622 test dx, dx 623 setnz result 624 }; 625 base_type::fence_after(order); 626 return result; 627 } 628 opaque_complementboost::atomics::detail::extra_operations629 static BOOST_FORCEINLINE void opaque_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 630 { 631 base_type::fence_before(order); 632 __asm 633 { 634 mov ecx, storage 635 movzx eax, word ptr [ecx] 636 align 16 637 again: 638 mov edx, eax 639 not dx 640 lock cmpxchg word ptr [ecx], dx 641 jne again 642 }; 643 base_type::fence_after(order); 644 } 645 opaque_addboost::atomics::detail::extra_operations646 static BOOST_FORCEINLINE void opaque_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 647 { 648 base_type::fence_before(order); 649 __asm 650 { 651 mov edx, storage 652 movzx eax, v 653 lock add word ptr [edx], ax 654 }; 655 base_type::fence_after(order); 656 } 657 opaque_subboost::atomics::detail::extra_operations658 static BOOST_FORCEINLINE void opaque_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 659 { 660 base_type::fence_before(order); 661 __asm 662 { 663 mov edx, storage 664 movzx eax, v 665 lock sub word ptr [edx], ax 666 }; 667 base_type::fence_after(order); 668 } 669 opaque_negateboost::atomics::detail::extra_operations670 static BOOST_FORCEINLINE void opaque_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 671 { 672 base_type::fence_before(order); 673 __asm 674 { 675 mov edx, storage 676 lock neg word ptr [edx] 677 }; 678 base_type::fence_after(order); 679 } 680 opaque_andboost::atomics::detail::extra_operations681 static BOOST_FORCEINLINE void opaque_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 682 { 683 base_type::fence_before(order); 684 __asm 685 { 686 mov edx, storage 687 movzx eax, v 688 lock and word ptr [edx], ax 689 }; 690 base_type::fence_after(order); 691 } 692 opaque_orboost::atomics::detail::extra_operations693 static BOOST_FORCEINLINE void opaque_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 694 { 695 base_type::fence_before(order); 696 __asm 697 { 698 mov edx, storage 699 movzx eax, v 700 lock or word ptr [edx], ax 701 }; 702 base_type::fence_after(order); 703 } 704 opaque_xorboost::atomics::detail::extra_operations705 static BOOST_FORCEINLINE void opaque_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 706 { 707 base_type::fence_before(order); 708 __asm 709 { 710 mov edx, storage 711 movzx eax, v 712 lock xor word ptr [edx], ax 713 }; 714 base_type::fence_after(order); 715 } 716 opaque_complementboost::atomics::detail::extra_operations717 static BOOST_FORCEINLINE void opaque_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 718 { 719 base_type::fence_before(order); 720 __asm 721 { 722 mov edx, storage 723 lock not word ptr [edx] 724 }; 725 base_type::fence_after(order); 726 } 727 add_and_testboost::atomics::detail::extra_operations728 static BOOST_FORCEINLINE bool add_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 729 { 730 base_type::fence_before(order); 731 bool result; 732 __asm 733 { 734 mov edx, storage 735 movzx eax, v 736 lock add word ptr [edx], ax 737 setnz result 738 }; 739 base_type::fence_after(order); 740 return result; 741 } 742 sub_and_testboost::atomics::detail::extra_operations743 static BOOST_FORCEINLINE bool sub_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 744 { 745 base_type::fence_before(order); 746 bool result; 747 __asm 748 { 749 mov edx, storage 750 movzx eax, v 751 lock sub word ptr [edx], ax 752 setnz result 753 }; 754 base_type::fence_after(order); 755 return result; 756 } 757 and_and_testboost::atomics::detail::extra_operations758 static BOOST_FORCEINLINE bool and_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 759 { 760 base_type::fence_before(order); 761 bool result; 762 __asm 763 { 764 mov edx, storage 765 movzx eax, v 766 lock and word ptr [edx], ax 767 setnz result 768 }; 769 base_type::fence_after(order); 770 return result; 771 } 772 or_and_testboost::atomics::detail::extra_operations773 static BOOST_FORCEINLINE bool or_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 774 { 775 base_type::fence_before(order); 776 bool result; 777 __asm 778 { 779 mov edx, storage 780 movzx eax, v 781 lock or word ptr [edx], ax 782 setnz result 783 }; 784 base_type::fence_after(order); 785 return result; 786 } 787 xor_and_testboost::atomics::detail::extra_operations788 static BOOST_FORCEINLINE bool xor_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 789 { 790 base_type::fence_before(order); 791 bool result; 792 __asm 793 { 794 mov edx, storage 795 movzx eax, v 796 lock xor word ptr [edx], ax 797 setnz result 798 }; 799 base_type::fence_after(order); 800 return result; 801 } 802 bit_test_and_setboost::atomics::detail::extra_operations803 static BOOST_FORCEINLINE bool bit_test_and_set(storage_type volatile& storage, unsigned int bit_number, memory_order order) BOOST_NOEXCEPT 804 { 805 base_type::fence_before(order); 806 bool result; 807 __asm 808 { 809 mov edx, storage 810 mov eax, bit_number 811 lock bts word ptr [edx], ax 812 setc result 813 }; 814 base_type::fence_after(order); 815 return result; 816 } 817 bit_test_and_resetboost::atomics::detail::extra_operations818 static BOOST_FORCEINLINE bool bit_test_and_reset(storage_type volatile& storage, unsigned int bit_number, memory_order order) BOOST_NOEXCEPT 819 { 820 base_type::fence_before(order); 821 bool result; 822 __asm 823 { 824 mov edx, storage 825 mov eax, bit_number 826 lock btr word ptr [edx], ax 827 setc result 828 }; 829 base_type::fence_after(order); 830 return result; 831 } 832 bit_test_and_complementboost::atomics::detail::extra_operations833 static BOOST_FORCEINLINE bool bit_test_and_complement(storage_type volatile& storage, unsigned int bit_number, memory_order order) BOOST_NOEXCEPT 834 { 835 base_type::fence_before(order); 836 bool result; 837 __asm 838 { 839 mov edx, storage 840 mov eax, bit_number 841 lock btc word ptr [edx], ax 842 setc result 843 }; 844 base_type::fence_after(order); 845 return result; 846 } 847 }; 848 849 #endif // defined(_M_IX86) 850 851 #if defined(_M_IX86) || (defined(BOOST_ATOMIC_INTERLOCKED_BTS) && defined(BOOST_ATOMIC_INTERLOCKED_BTR)) 852 853 template< typename Base, bool Signed > 854 struct extra_operations< Base, 4u, Signed, true > : 855 public extra_operations_generic< Base, 4u, Signed > 856 { 857 typedef extra_operations_generic< Base, 4u, Signed > base_type; 858 typedef typename base_type::storage_type storage_type; 859 860 #if defined(_M_IX86) fetch_negateboost::atomics::detail::extra_operations861 static BOOST_FORCEINLINE storage_type fetch_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 862 { 863 base_type::fence_before(order); 864 storage_type old_val; 865 __asm 866 { 867 mov ecx, storage 868 mov eax, dword ptr [ecx] 869 align 16 870 again: 871 mov edx, eax 872 neg edx 873 lock cmpxchg dword ptr [ecx], edx 874 jne again 875 mov old_val, eax 876 }; 877 base_type::fence_after(order); 878 return old_val; 879 } 880 negateboost::atomics::detail::extra_operations881 static BOOST_FORCEINLINE storage_type negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 882 { 883 base_type::fence_before(order); 884 storage_type new_val; 885 __asm 886 { 887 mov ecx, storage 888 mov eax, dword ptr [ecx] 889 align 16 890 again: 891 mov edx, eax 892 neg edx 893 lock cmpxchg dword ptr [ecx], edx 894 jne again 895 mov new_val, edx 896 }; 897 base_type::fence_after(order); 898 return new_val; 899 } 900 negate_and_testboost::atomics::detail::extra_operations901 static BOOST_FORCEINLINE bool negate_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 902 { 903 base_type::fence_before(order); 904 bool result; 905 __asm 906 { 907 mov ecx, storage 908 mov eax, dword ptr [ecx] 909 align 16 910 again: 911 mov edx, eax 912 neg edx 913 lock cmpxchg dword ptr [ecx], edx 914 jne again 915 test edx, edx 916 setnz result 917 }; 918 base_type::fence_after(order); 919 return result; 920 } 921 opaque_negateboost::atomics::detail::extra_operations922 static BOOST_FORCEINLINE void opaque_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 923 { 924 base_type::fence_before(order); 925 __asm 926 { 927 mov ecx, storage 928 mov eax, dword ptr [ecx] 929 align 16 930 again: 931 mov edx, eax 932 neg edx 933 lock cmpxchg dword ptr [ecx], edx 934 jne again 935 }; 936 base_type::fence_after(order); 937 } 938 bitwise_andboost::atomics::detail::extra_operations939 static BOOST_FORCEINLINE storage_type bitwise_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 940 { 941 base_type::fence_before(order); 942 __asm 943 { 944 mov edi, storage 945 mov ecx, v 946 xor edx, edx 947 mov eax, dword ptr [edi] 948 align 16 949 again: 950 mov edx, eax 951 and edx, ecx 952 lock cmpxchg dword ptr [edi], edx 953 jne again 954 mov v, edx 955 }; 956 base_type::fence_after(order); 957 return v; 958 } 959 bitwise_orboost::atomics::detail::extra_operations960 static BOOST_FORCEINLINE storage_type bitwise_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 961 { 962 base_type::fence_before(order); 963 __asm 964 { 965 mov edi, storage 966 mov ecx, v 967 xor edx, edx 968 mov eax, dword ptr [edi] 969 align 16 970 again: 971 mov edx, eax 972 or edx, ecx 973 lock cmpxchg dword ptr [edi], edx 974 jne again 975 mov v, edx 976 }; 977 base_type::fence_after(order); 978 return v; 979 } 980 bitwise_xorboost::atomics::detail::extra_operations981 static BOOST_FORCEINLINE storage_type bitwise_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 982 { 983 base_type::fence_before(order); 984 __asm 985 { 986 mov edi, storage 987 mov ecx, v 988 xor edx, edx 989 mov eax, dword ptr [edi] 990 align 16 991 again: 992 mov edx, eax 993 xor edx, ecx 994 lock cmpxchg dword ptr [edi], edx 995 jne again 996 mov v, edx 997 }; 998 base_type::fence_after(order); 999 return v; 1000 } 1001 fetch_complementboost::atomics::detail::extra_operations1002 static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 1003 { 1004 base_type::fence_before(order); 1005 storage_type old_val; 1006 __asm 1007 { 1008 mov ecx, storage 1009 mov eax, dword ptr [ecx] 1010 align 16 1011 again: 1012 mov edx, eax 1013 not edx 1014 lock cmpxchg dword ptr [ecx], edx 1015 jne again 1016 mov old_val, eax 1017 }; 1018 base_type::fence_after(order); 1019 return old_val; 1020 } 1021 bitwise_complementboost::atomics::detail::extra_operations1022 static BOOST_FORCEINLINE storage_type bitwise_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 1023 { 1024 base_type::fence_before(order); 1025 storage_type new_val; 1026 __asm 1027 { 1028 mov ecx, storage 1029 mov eax, dword ptr [ecx] 1030 align 16 1031 again: 1032 mov edx, eax 1033 not edx 1034 lock cmpxchg dword ptr [ecx], edx 1035 jne again 1036 mov new_val, edx 1037 }; 1038 base_type::fence_after(order); 1039 return new_val; 1040 } 1041 complement_and_testboost::atomics::detail::extra_operations1042 static BOOST_FORCEINLINE bool complement_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 1043 { 1044 base_type::fence_before(order); 1045 bool result; 1046 __asm 1047 { 1048 mov ecx, storage 1049 mov eax, dword ptr [ecx] 1050 align 16 1051 again: 1052 mov edx, eax 1053 not edx 1054 lock cmpxchg dword ptr [ecx], edx 1055 jne again 1056 test edx, edx 1057 setnz result 1058 }; 1059 base_type::fence_after(order); 1060 return result; 1061 } 1062 opaque_complementboost::atomics::detail::extra_operations1063 static BOOST_FORCEINLINE void opaque_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 1064 { 1065 base_type::fence_before(order); 1066 __asm 1067 { 1068 mov ecx, storage 1069 mov eax, dword ptr [ecx] 1070 align 16 1071 again: 1072 mov edx, eax 1073 not edx 1074 lock cmpxchg dword ptr [ecx], edx 1075 jne again 1076 }; 1077 base_type::fence_after(order); 1078 } 1079 opaque_addboost::atomics::detail::extra_operations1080 static BOOST_FORCEINLINE void opaque_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 1081 { 1082 base_type::fence_before(order); 1083 __asm 1084 { 1085 mov edx, storage 1086 mov eax, v 1087 lock add dword ptr [edx], eax 1088 }; 1089 base_type::fence_after(order); 1090 } 1091 opaque_subboost::atomics::detail::extra_operations1092 static BOOST_FORCEINLINE void opaque_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 1093 { 1094 base_type::fence_before(order); 1095 __asm 1096 { 1097 mov edx, storage 1098 mov eax, v 1099 lock sub dword ptr [edx], eax 1100 }; 1101 base_type::fence_after(order); 1102 } 1103 opaque_negateboost::atomics::detail::extra_operations1104 static BOOST_FORCEINLINE void opaque_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 1105 { 1106 base_type::fence_before(order); 1107 __asm 1108 { 1109 mov edx, storage 1110 lock neg dword ptr [edx] 1111 }; 1112 base_type::fence_after(order); 1113 } 1114 opaque_andboost::atomics::detail::extra_operations1115 static BOOST_FORCEINLINE void opaque_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 1116 { 1117 base_type::fence_before(order); 1118 __asm 1119 { 1120 mov edx, storage 1121 mov eax, v 1122 lock and dword ptr [edx], eax 1123 }; 1124 base_type::fence_after(order); 1125 } 1126 opaque_orboost::atomics::detail::extra_operations1127 static BOOST_FORCEINLINE void opaque_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 1128 { 1129 base_type::fence_before(order); 1130 __asm 1131 { 1132 mov edx, storage 1133 mov eax, v 1134 lock or dword ptr [edx], eax 1135 }; 1136 base_type::fence_after(order); 1137 } 1138 opaque_xorboost::atomics::detail::extra_operations1139 static BOOST_FORCEINLINE void opaque_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 1140 { 1141 base_type::fence_before(order); 1142 __asm 1143 { 1144 mov edx, storage 1145 mov eax, v 1146 lock xor dword ptr [edx], eax 1147 }; 1148 base_type::fence_after(order); 1149 } 1150 opaque_complementboost::atomics::detail::extra_operations1151 static BOOST_FORCEINLINE void opaque_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT 1152 { 1153 base_type::fence_before(order); 1154 __asm 1155 { 1156 mov edx, storage 1157 lock not dword ptr [edx] 1158 }; 1159 base_type::fence_after(order); 1160 } 1161 add_and_testboost::atomics::detail::extra_operations1162 static BOOST_FORCEINLINE bool add_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 1163 { 1164 base_type::fence_before(order); 1165 bool result; 1166 __asm 1167 { 1168 mov edx, storage 1169 mov eax, v 1170 lock add dword ptr [edx], eax 1171 setnz result 1172 }; 1173 base_type::fence_after(order); 1174 return result; 1175 } 1176 sub_and_testboost::atomics::detail::extra_operations1177 static BOOST_FORCEINLINE bool sub_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 1178 { 1179 base_type::fence_before(order); 1180 bool result; 1181 __asm 1182 { 1183 mov edx, storage 1184 mov eax, v 1185 lock sub dword ptr [edx], eax 1186 setnz result 1187 }; 1188 base_type::fence_after(order); 1189 return result; 1190 } 1191 and_and_testboost::atomics::detail::extra_operations1192 static BOOST_FORCEINLINE bool and_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 1193 { 1194 base_type::fence_before(order); 1195 bool result; 1196 __asm 1197 { 1198 mov edx, storage 1199 mov eax, v 1200 lock and dword ptr [edx], eax 1201 setnz result 1202 }; 1203 base_type::fence_after(order); 1204 return result; 1205 } 1206 or_and_testboost::atomics::detail::extra_operations1207 static BOOST_FORCEINLINE bool or_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 1208 { 1209 base_type::fence_before(order); 1210 bool result; 1211 __asm 1212 { 1213 mov edx, storage 1214 mov eax, v 1215 lock or dword ptr [edx], eax 1216 setnz result 1217 }; 1218 base_type::fence_after(order); 1219 return result; 1220 } 1221 xor_and_testboost::atomics::detail::extra_operations1222 static BOOST_FORCEINLINE bool xor_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT 1223 { 1224 base_type::fence_before(order); 1225 bool result; 1226 __asm 1227 { 1228 mov edx, storage 1229 mov eax, v 1230 lock xor dword ptr [edx], eax 1231 setnz result 1232 }; 1233 base_type::fence_after(order); 1234 return result; 1235 } 1236 bit_test_and_complementboost::atomics::detail::extra_operations1237 static BOOST_FORCEINLINE bool bit_test_and_complement(storage_type volatile& storage, unsigned int bit_number, memory_order order) BOOST_NOEXCEPT 1238 { 1239 base_type::fence_before(order); 1240 bool result; 1241 __asm 1242 { 1243 mov edx, storage 1244 mov eax, bit_number 1245 lock btc dword ptr [edx], eax 1246 setc result 1247 }; 1248 base_type::fence_after(order); 1249 return result; 1250 } 1251 #endif // defined(_M_IX86) 1252 1253 #if defined(BOOST_ATOMIC_INTERLOCKED_BTS) bit_test_and_setboost::atomics::detail::extra_operations1254 static BOOST_FORCEINLINE bool bit_test_and_set(storage_type volatile& storage, unsigned int bit_number, memory_order) BOOST_NOEXCEPT 1255 { 1256 return !!BOOST_ATOMIC_INTERLOCKED_BTS(&storage, bit_number); 1257 } 1258 #elif defined(_M_IX86) bit_test_and_setboost::atomics::detail::extra_operations1259 static BOOST_FORCEINLINE bool bit_test_and_set(storage_type volatile& storage, unsigned int bit_number, memory_order order) BOOST_NOEXCEPT 1260 { 1261 base_type::fence_before(order); 1262 bool result; 1263 __asm 1264 { 1265 mov edx, storage 1266 mov eax, bit_number 1267 lock bts dword ptr [edx], eax 1268 setc result 1269 }; 1270 base_type::fence_after(order); 1271 return result; 1272 } 1273 #endif 1274 1275 #if defined(BOOST_ATOMIC_INTERLOCKED_BTR) bit_test_and_resetboost::atomics::detail::extra_operations1276 static BOOST_FORCEINLINE bool bit_test_and_reset(storage_type volatile& storage, unsigned int bit_number, memory_order) BOOST_NOEXCEPT 1277 { 1278 return !!BOOST_ATOMIC_INTERLOCKED_BTR(&storage, bit_number); 1279 } 1280 #elif defined(_M_IX86) bit_test_and_resetboost::atomics::detail::extra_operations1281 static BOOST_FORCEINLINE bool bit_test_and_reset(storage_type volatile& storage, unsigned int bit_number, memory_order order) BOOST_NOEXCEPT 1282 { 1283 base_type::fence_before(order); 1284 bool result; 1285 __asm 1286 { 1287 mov edx, storage 1288 mov eax, bit_number 1289 lock btr dword ptr [edx], eax 1290 setc result 1291 }; 1292 base_type::fence_after(order); 1293 return result; 1294 } 1295 #endif 1296 }; 1297 1298 #endif // defined(_M_IX86) || (defined(BOOST_ATOMIC_INTERLOCKED_BTS) && defined(BOOST_ATOMIC_INTERLOCKED_BTR)) 1299 1300 #if defined(BOOST_ATOMIC_INTERLOCKED_BTS64) && defined(BOOST_ATOMIC_INTERLOCKED_BTR64) 1301 1302 template< typename Base, bool Signed > 1303 struct extra_operations< Base, 8u, Signed, true > : 1304 public extra_operations_generic< Base, 8u, Signed > 1305 { 1306 typedef extra_operations_generic< Base, 8u, Signed > base_type; 1307 typedef typename base_type::storage_type storage_type; 1308 bit_test_and_setboost::atomics::detail::extra_operations1309 static BOOST_FORCEINLINE bool bit_test_and_set(storage_type volatile& storage, unsigned int bit_number, memory_order order) BOOST_NOEXCEPT 1310 { 1311 return !!BOOST_ATOMIC_INTERLOCKED_BTS64(&storage, bit_number); 1312 } 1313 bit_test_and_resetboost::atomics::detail::extra_operations1314 static BOOST_FORCEINLINE bool bit_test_and_reset(storage_type volatile& storage, unsigned int bit_number, memory_order order) BOOST_NOEXCEPT 1315 { 1316 return !!BOOST_ATOMIC_INTERLOCKED_BTR64(&storage, bit_number); 1317 } 1318 }; 1319 1320 #endif // defined(BOOST_ATOMIC_INTERLOCKED_BTS64) && defined(BOOST_ATOMIC_INTERLOCKED_BTR64) 1321 1322 } // namespace detail 1323 } // namespace atomics 1324 } // namespace boost 1325 1326 #include <boost/atomic/detail/footer.hpp> 1327 1328 #endif // BOOST_ATOMIC_DETAIL_EXTRA_OPS_MSVC_X86_HPP_INCLUDED_ 1329