1 /*
2 * Copyright (C) 2022 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #ifndef ART_RUNTIME_BASE_ATOMIC_PAIR_H_
18 #define ART_RUNTIME_BASE_ATOMIC_PAIR_H_
19
20 #include "base/macros.h"
21
22 #include <type_traits>
23
24 namespace art {
25
26 // std::pair<> is not trivially copyable and as such it is unsuitable for atomic operations.
27 template <typename IntType>
28 struct PACKED(2 * sizeof(IntType)) AtomicPair {
29 static_assert(std::is_integral_v<IntType>);
30
AtomicPairAtomicPair31 constexpr AtomicPair() : first(0), second(0) { }
AtomicPairAtomicPair32 AtomicPair(IntType f, IntType s) : first(f), second(s) { }
33 AtomicPair(const AtomicPair&) = default;
34 AtomicPair& operator=(const AtomicPair&) = default;
35
36 IntType first;
37 IntType second;
38 };
39
40 template <typename IntType>
AtomicPairLoadAcquire(std::atomic<AtomicPair<IntType>> * target)41 ALWAYS_INLINE static inline AtomicPair<IntType> AtomicPairLoadAcquire(
42 std::atomic<AtomicPair<IntType>>* target) {
43 static_assert(std::atomic<AtomicPair<IntType>>::is_always_lock_free);
44 return target->load(std::memory_order_acquire);
45 }
46
47 template <typename IntType>
AtomicPairStoreRelease(std::atomic<AtomicPair<IntType>> * target,AtomicPair<IntType> value)48 ALWAYS_INLINE static inline void AtomicPairStoreRelease(
49 std::atomic<AtomicPair<IntType>>* target, AtomicPair<IntType> value) {
50 static_assert(std::atomic<AtomicPair<IntType>>::is_always_lock_free);
51 target->store(value, std::memory_order_release);
52 }
53
54 // llvm does not implement 16-byte atomic operations on x86-64.
55 #if defined(__x86_64__)
AtomicPairLoadAcquire(std::atomic<AtomicPair<uint64_t>> * target)56 ALWAYS_INLINE static inline AtomicPair<uint64_t> AtomicPairLoadAcquire(
57 std::atomic<AtomicPair<uint64_t>>* target) {
58 uint64_t first, second;
59 __asm__ __volatile__(
60 "lock cmpxchg16b (%2)"
61 : "=&a"(first), "=&d"(second)
62 : "r"(target), "a"(0), "d"(0), "b"(0), "c"(0)
63 : "cc");
64 return {first, second};
65 }
66
AtomicPairStoreRelease(std::atomic<AtomicPair<uint64_t>> * target,AtomicPair<uint64_t> value)67 ALWAYS_INLINE static inline void AtomicPairStoreRelease(
68 std::atomic<AtomicPair<uint64_t>>* target, AtomicPair<uint64_t> value) {
69 uint64_t first, second;
70 __asm__ __volatile__ (
71 "movq (%2), %%rax\n\t"
72 "movq 8(%2), %%rdx\n\t"
73 "1:\n\t"
74 "lock cmpxchg16b (%2)\n\t"
75 "jnz 1b"
76 : "=&a"(first), "=&d"(second)
77 : "r"(target), "b"(value.first), "c"(value.second)
78 : "cc");
79 }
80 #endif // defined(__x86_64__)
81
82 } // namespace art
83
84 #endif // ART_RUNTIME_BASE_ATOMIC_PAIR_H_
85