1 /*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include "atomic.h"
18
19 #define NEED_SWAP_MUTEXES !defined(__arm__) && !defined(__i386__)
20
21 #if NEED_SWAP_MUTEXES
22 #include <vector>
23 #include "base/mutex.h"
24 #include "base/stl_util.h"
25 #include "base/stringprintf.h"
26 #include "thread.h"
27 #endif
28
29 namespace art {
30
31 #if NEED_SWAP_MUTEXES
32 // We stripe across a bunch of different mutexes to reduce contention.
33 static const size_t kSwapMutexCount = 32;
34 static std::vector<Mutex*>* gSwapMutexes;
35
GetSwapMutex(const volatile int64_t * addr)36 static Mutex& GetSwapMutex(const volatile int64_t* addr) {
37 return *(*gSwapMutexes)[(reinterpret_cast<unsigned>(addr) >> 3U) % kSwapMutexCount];
38 }
39 #endif
40
Startup()41 void QuasiAtomic::Startup() {
42 #if NEED_SWAP_MUTEXES
43 gSwapMutexes = new std::vector<Mutex*>;
44 for (size_t i = 0; i < kSwapMutexCount; ++i) {
45 gSwapMutexes->push_back(new Mutex("QuasiAtomic stripe"));
46 }
47 #endif
48 }
49
Shutdown()50 void QuasiAtomic::Shutdown() {
51 #if NEED_SWAP_MUTEXES
52 STLDeleteElements(gSwapMutexes);
53 delete gSwapMutexes;
54 #endif
55 }
56
Read64(volatile const int64_t * addr)57 int64_t QuasiAtomic::Read64(volatile const int64_t* addr) {
58 int64_t value;
59 #if NEED_SWAP_MUTEXES
60 MutexLock mu(Thread::Current(), GetSwapMutex(addr));
61 value = *addr;
62 #elif defined(__arm__)
63 // Exclusive loads are defined not to tear, clearing the exclusive state isn't necessary. If we
64 // have LPAE (such as Cortex-A15) then ldrd would suffice.
65 __asm__ __volatile__("@ QuasiAtomic::Read64\n"
66 "ldrexd %0, %H0, [%1]"
67 : "=&r" (value)
68 : "r" (addr));
69 #elif defined(__i386__)
70 __asm__ __volatile__(
71 "movq %1, %0\n"
72 : "=x" (value)
73 : "m" (*addr));
74 #else
75 #error Unexpected architecture
76 #endif
77 return value;
78 }
79
Write64(volatile int64_t * addr,int64_t value)80 void QuasiAtomic::Write64(volatile int64_t* addr, int64_t value) {
81 #if NEED_SWAP_MUTEXES
82 MutexLock mu(Thread::Current(), GetSwapMutex(addr));
83 *addr = value;
84 #elif defined(__arm__)
85 // The write is done as a swap so that the cache-line is in the exclusive state for the store. If
86 // we know that ARM architecture has LPAE (such as Cortex-A15) this isn't necessary and strd will
87 // suffice.
88 int64_t prev;
89 int status;
90 do {
91 __asm__ __volatile__("@ QuasiAtomic::Write64\n"
92 "ldrexd %0, %H0, [%3]\n"
93 "strexd %1, %4, %H4, [%3]"
94 : "=&r" (prev), "=&r" (status), "+m"(*addr)
95 : "r" (addr), "r" (value)
96 : "cc");
97 } while (__builtin_expect(status != 0, 0));
98 #elif defined(__i386__)
99 __asm__ __volatile__(
100 "movq %1, %0"
101 : "=m" (*addr)
102 : "x" (value));
103 #else
104 #error Unexpected architecture
105 #endif
106 }
107
108
Cas64(int64_t old_value,int64_t new_value,volatile int64_t * addr)109 bool QuasiAtomic::Cas64(int64_t old_value, int64_t new_value, volatile int64_t* addr) {
110 #if NEED_SWAP_MUTEXES
111 MutexLock mu(Thread::Current(), GetSwapMutex(addr));
112 if (*addr == old_value) {
113 *addr = new_value;
114 return true;
115 }
116 return false;
117 #elif defined(__arm__)
118 int64_t prev;
119 int status;
120 do {
121 __asm__ __volatile__("@ QuasiAtomic::Cas64\n"
122 "ldrexd %0, %H0, [%3]\n"
123 "mov %1, #0\n"
124 "teq %0, %4\n"
125 "teqeq %H0, %H4\n"
126 "strexdeq %1, %5, %H5, [%3]"
127 : "=&r" (prev), "=&r" (status), "+m"(*addr)
128 : "r" (addr), "Ir" (old_value), "r" (new_value)
129 : "cc");
130 } while (__builtin_expect(status != 0, 0));
131 return prev == old_value;
132 #elif defined(__i386__)
133 // The compiler does the right job and works better than inline assembly, especially with -O0
134 // compilation.
135 return __sync_bool_compare_and_swap(addr, old_value, new_value);
136 #else
137 #error Unexpected architecture
138 #endif
139 }
140
LongAtomicsUseMutexes()141 bool QuasiAtomic::LongAtomicsUseMutexes() {
142 #if NEED_SWAP_MUTEXES
143 return true;
144 #else
145 return false;
146 #endif
147 }
148
149 } // namespace art
150