1 /*
2 * Copyright (C) 2011 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #ifndef ART_RUNTIME_MIRROR_OBJECT_READBARRIER_INL_H_
18 #define ART_RUNTIME_MIRROR_OBJECT_READBARRIER_INL_H_
19
20 #include "object.h"
21
22 #include "base/atomic.h"
23 #include "heap_poisoning.h"
24 #include "lock_word-inl.h"
25 #include "object_reference-inl.h"
26 #include "read_barrier.h"
27 #include "runtime.h"
28
29 namespace art {
30 namespace mirror {
31
32 template<VerifyObjectFlags kVerifyFlags>
GetLockWord(bool as_volatile)33 inline LockWord Object::GetLockWord(bool as_volatile) {
34 if (as_volatile) {
35 return LockWord(GetField32Volatile<kVerifyFlags>(OFFSET_OF_OBJECT_MEMBER(Object, monitor_)));
36 }
37 return LockWord(GetField32<kVerifyFlags>(OFFSET_OF_OBJECT_MEMBER(Object, monitor_)));
38 }
39
40 template<bool kTransactionActive, bool kCheckTransaction, VerifyObjectFlags kVerifyFlags>
CasFieldWeakRelaxed32(MemberOffset field_offset,int32_t old_value,int32_t new_value)41 inline bool Object::CasFieldWeakRelaxed32(MemberOffset field_offset,
42 int32_t old_value, int32_t new_value) {
43 if (kCheckTransaction) {
44 DCHECK_EQ(kTransactionActive, Runtime::Current()->IsActiveTransaction());
45 }
46 if (kTransactionActive) {
47 Runtime::Current()->RecordWriteField32(this, field_offset, old_value, true);
48 }
49 if (kVerifyFlags & kVerifyThis) {
50 VerifyObject(this);
51 }
52 uint8_t* raw_addr = reinterpret_cast<uint8_t*>(this) + field_offset.Int32Value();
53 AtomicInteger* atomic_addr = reinterpret_cast<AtomicInteger*>(raw_addr);
54
55 return atomic_addr->CompareAndSetWeakRelaxed(old_value, new_value);
56 }
57
CasLockWordWeakRelaxed(LockWord old_val,LockWord new_val)58 inline bool Object::CasLockWordWeakRelaxed(LockWord old_val, LockWord new_val) {
59 // Force use of non-transactional mode and do not check.
60 return CasFieldWeakRelaxed32<false, false>(
61 OFFSET_OF_OBJECT_MEMBER(Object, monitor_), old_val.GetValue(), new_val.GetValue());
62 }
63
CasLockWordWeakRelease(LockWord old_val,LockWord new_val)64 inline bool Object::CasLockWordWeakRelease(LockWord old_val, LockWord new_val) {
65 // Force use of non-transactional mode and do not check.
66 return CasFieldWeakRelease32<false, false>(
67 OFFSET_OF_OBJECT_MEMBER(Object, monitor_), old_val.GetValue(), new_val.GetValue());
68 }
69
GetReadBarrierState(uintptr_t * fake_address_dependency)70 inline uint32_t Object::GetReadBarrierState(uintptr_t* fake_address_dependency) {
71 if (!kUseBakerReadBarrier) {
72 LOG(FATAL) << "Unreachable";
73 UNREACHABLE();
74 }
75 #if defined(__arm__)
76 uintptr_t obj = reinterpret_cast<uintptr_t>(this);
77 uintptr_t result;
78 DCHECK_EQ(OFFSETOF_MEMBER(Object, monitor_), 4U);
79 // Use inline assembly to prevent the compiler from optimizing away the false dependency.
80 __asm__ __volatile__(
81 "ldr %[result], [%[obj], #4]\n\t"
82 // This instruction is enough to "fool the compiler and the CPU" by having `fad` always be
83 // null, without them being able to assume that fact.
84 "eor %[fad], %[result], %[result]\n\t"
85 : [result] "+r" (result), [fad] "=r" (*fake_address_dependency)
86 : [obj] "r" (obj));
87 DCHECK_EQ(*fake_address_dependency, 0U);
88 LockWord lw(static_cast<uint32_t>(result));
89 uint32_t rb_state = lw.ReadBarrierState();
90 return rb_state;
91 #elif defined(__aarch64__)
92 uintptr_t obj = reinterpret_cast<uintptr_t>(this);
93 uintptr_t result;
94 DCHECK_EQ(OFFSETOF_MEMBER(Object, monitor_), 4U);
95 // Use inline assembly to prevent the compiler from optimizing away the false dependency.
96 __asm__ __volatile__(
97 "ldr %w[result], [%[obj], #4]\n\t"
98 // This instruction is enough to "fool the compiler and the CPU" by having `fad` always be
99 // null, without them being able to assume that fact.
100 "eor %[fad], %[result], %[result]\n\t"
101 : [result] "+r" (result), [fad] "=r" (*fake_address_dependency)
102 : [obj] "r" (obj));
103 DCHECK_EQ(*fake_address_dependency, 0U);
104 LockWord lw(static_cast<uint32_t>(result));
105 uint32_t rb_state = lw.ReadBarrierState();
106 return rb_state;
107 #elif defined(__i386__) || defined(__x86_64__)
108 LockWord lw = GetLockWord(false);
109 // i386/x86_64 don't need fake address dependency. Use a compiler fence to avoid compiler
110 // reordering.
111 *fake_address_dependency = 0;
112 std::atomic_signal_fence(std::memory_order_acquire);
113 uint32_t rb_state = lw.ReadBarrierState();
114 return rb_state;
115 #else
116 // MIPS32/MIPS64: use a memory barrier to prevent load-load reordering.
117 LockWord lw = GetLockWord(false);
118 *fake_address_dependency = 0;
119 std::atomic_thread_fence(std::memory_order_acquire);
120 uint32_t rb_state = lw.ReadBarrierState();
121 return rb_state;
122 #endif
123 }
124
GetReadBarrierState()125 inline uint32_t Object::GetReadBarrierState() {
126 if (!kUseBakerReadBarrier) {
127 LOG(FATAL) << "Unreachable";
128 UNREACHABLE();
129 }
130 DCHECK(kUseBakerReadBarrier);
131 LockWord lw(GetField<uint32_t, /*kIsVolatile*/false>(OFFSET_OF_OBJECT_MEMBER(Object, monitor_)));
132 uint32_t rb_state = lw.ReadBarrierState();
133 DCHECK(ReadBarrier::IsValidReadBarrierState(rb_state)) << rb_state;
134 return rb_state;
135 }
136
GetReadBarrierStateAcquire()137 inline uint32_t Object::GetReadBarrierStateAcquire() {
138 if (!kUseBakerReadBarrier) {
139 LOG(FATAL) << "Unreachable";
140 UNREACHABLE();
141 }
142 LockWord lw(GetFieldAcquire<uint32_t>(OFFSET_OF_OBJECT_MEMBER(Object, monitor_)));
143 uint32_t rb_state = lw.ReadBarrierState();
144 DCHECK(ReadBarrier::IsValidReadBarrierState(rb_state)) << rb_state;
145 return rb_state;
146 }
147
148 template<bool kCasRelease>
AtomicSetReadBarrierState(uint32_t expected_rb_state,uint32_t rb_state)149 inline bool Object::AtomicSetReadBarrierState(uint32_t expected_rb_state, uint32_t rb_state) {
150 if (!kUseBakerReadBarrier) {
151 LOG(FATAL) << "Unreachable";
152 UNREACHABLE();
153 }
154 DCHECK(ReadBarrier::IsValidReadBarrierState(expected_rb_state)) << expected_rb_state;
155 DCHECK(ReadBarrier::IsValidReadBarrierState(rb_state)) << rb_state;
156 LockWord expected_lw;
157 LockWord new_lw;
158 do {
159 LockWord lw = GetLockWord(false);
160 if (UNLIKELY(lw.ReadBarrierState() != expected_rb_state)) {
161 // Lost the race.
162 return false;
163 }
164 expected_lw = lw;
165 expected_lw.SetReadBarrierState(expected_rb_state);
166 new_lw = lw;
167 new_lw.SetReadBarrierState(rb_state);
168 // ConcurrentCopying::ProcessMarkStackRef uses this with kCasRelease == true.
169 // If kCasRelease == true, use a CAS release so that when GC updates all the fields of
170 // an object and then changes the object from gray to black, the field updates (stores) will be
171 // visible (won't be reordered after this CAS.)
172 } while (!(kCasRelease ?
173 CasLockWordWeakRelease(expected_lw, new_lw) :
174 CasLockWordWeakRelaxed(expected_lw, new_lw)));
175 return true;
176 }
177
AtomicSetMarkBit(uint32_t expected_mark_bit,uint32_t mark_bit)178 inline bool Object::AtomicSetMarkBit(uint32_t expected_mark_bit, uint32_t mark_bit) {
179 LockWord expected_lw;
180 LockWord new_lw;
181 do {
182 LockWord lw = GetLockWord(false);
183 if (UNLIKELY(lw.MarkBitState() != expected_mark_bit)) {
184 // Lost the race.
185 return false;
186 }
187 expected_lw = lw;
188 new_lw = lw;
189 new_lw.SetMarkBitState(mark_bit);
190 // Since this is only set from the mutator, we can use the non-release CAS.
191 } while (!CasLockWordWeakRelaxed(expected_lw, new_lw));
192 return true;
193 }
194
195 template<bool kTransactionActive, bool kCheckTransaction, VerifyObjectFlags kVerifyFlags>
CasFieldStrongRelaxedObjectWithoutWriteBarrier(MemberOffset field_offset,ObjPtr<Object> old_value,ObjPtr<Object> new_value)196 inline bool Object::CasFieldStrongRelaxedObjectWithoutWriteBarrier(
197 MemberOffset field_offset,
198 ObjPtr<Object> old_value,
199 ObjPtr<Object> new_value) {
200 if (kCheckTransaction) {
201 DCHECK_EQ(kTransactionActive, Runtime::Current()->IsActiveTransaction());
202 }
203 if (kVerifyFlags & kVerifyThis) {
204 VerifyObject(this);
205 }
206 if (kVerifyFlags & kVerifyWrites) {
207 VerifyObject(new_value);
208 }
209 if (kVerifyFlags & kVerifyReads) {
210 VerifyObject(old_value);
211 }
212 if (kTransactionActive) {
213 Runtime::Current()->RecordWriteFieldReference(this, field_offset, old_value, true);
214 }
215 uint32_t old_ref(PtrCompression<kPoisonHeapReferences, Object>::Compress(old_value));
216 uint32_t new_ref(PtrCompression<kPoisonHeapReferences, Object>::Compress(new_value));
217 uint8_t* raw_addr = reinterpret_cast<uint8_t*>(this) + field_offset.Int32Value();
218 Atomic<uint32_t>* atomic_addr = reinterpret_cast<Atomic<uint32_t>*>(raw_addr);
219
220 bool success = atomic_addr->CompareAndSetStrongRelaxed(old_ref, new_ref);
221 return success;
222 }
223
224 template<bool kTransactionActive, bool kCheckTransaction, VerifyObjectFlags kVerifyFlags>
CasFieldStrongReleaseObjectWithoutWriteBarrier(MemberOffset field_offset,ObjPtr<Object> old_value,ObjPtr<Object> new_value)225 inline bool Object::CasFieldStrongReleaseObjectWithoutWriteBarrier(
226 MemberOffset field_offset,
227 ObjPtr<Object> old_value,
228 ObjPtr<Object> new_value) {
229 if (kCheckTransaction) {
230 DCHECK_EQ(kTransactionActive, Runtime::Current()->IsActiveTransaction());
231 }
232 if (kVerifyFlags & kVerifyThis) {
233 VerifyObject(this);
234 }
235 if (kVerifyFlags & kVerifyWrites) {
236 VerifyObject(new_value);
237 }
238 if (kVerifyFlags & kVerifyReads) {
239 VerifyObject(old_value);
240 }
241 if (kTransactionActive) {
242 Runtime::Current()->RecordWriteFieldReference(this, field_offset, old_value, true);
243 }
244 uint32_t old_ref(PtrCompression<kPoisonHeapReferences, Object>::Compress(old_value));
245 uint32_t new_ref(PtrCompression<kPoisonHeapReferences, Object>::Compress(new_value));
246 uint8_t* raw_addr = reinterpret_cast<uint8_t*>(this) + field_offset.Int32Value();
247 Atomic<uint32_t>* atomic_addr = reinterpret_cast<Atomic<uint32_t>*>(raw_addr);
248
249 bool success = atomic_addr->CompareAndSetStrongRelease(old_ref, new_ref);
250 return success;
251 }
252
253 } // namespace mirror
254 } // namespace art
255
256 #endif // ART_RUNTIME_MIRROR_OBJECT_READBARRIER_INL_H_
257