1 /*
2 * Copyright (C) 2011 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #ifndef ART_RUNTIME_MIRROR_OBJECT_READBARRIER_INL_H_
18 #define ART_RUNTIME_MIRROR_OBJECT_READBARRIER_INL_H_
19
20 #include "object.h"
21
22 #include "atomic.h"
23 #include "lock_word-inl.h"
24 #include "object_reference-inl.h"
25 #include "read_barrier.h"
26 #include "runtime.h"
27
28 namespace art {
29 namespace mirror {
30
31 template<VerifyObjectFlags kVerifyFlags>
GetLockWord(bool as_volatile)32 inline LockWord Object::GetLockWord(bool as_volatile) {
33 if (as_volatile) {
34 return LockWord(GetField32Volatile<kVerifyFlags>(OFFSET_OF_OBJECT_MEMBER(Object, monitor_)));
35 }
36 return LockWord(GetField32<kVerifyFlags>(OFFSET_OF_OBJECT_MEMBER(Object, monitor_)));
37 }
38
39 template<bool kTransactionActive, bool kCheckTransaction, VerifyObjectFlags kVerifyFlags>
CasFieldWeakRelaxed32(MemberOffset field_offset,int32_t old_value,int32_t new_value)40 inline bool Object::CasFieldWeakRelaxed32(MemberOffset field_offset,
41 int32_t old_value, int32_t new_value) {
42 if (kCheckTransaction) {
43 DCHECK_EQ(kTransactionActive, Runtime::Current()->IsActiveTransaction());
44 }
45 if (kTransactionActive) {
46 Runtime::Current()->RecordWriteField32(this, field_offset, old_value, true);
47 }
48 if (kVerifyFlags & kVerifyThis) {
49 VerifyObject(this);
50 }
51 uint8_t* raw_addr = reinterpret_cast<uint8_t*>(this) + field_offset.Int32Value();
52 AtomicInteger* atomic_addr = reinterpret_cast<AtomicInteger*>(raw_addr);
53
54 return atomic_addr->CompareExchangeWeakRelaxed(old_value, new_value);
55 }
56
CasLockWordWeakRelaxed(LockWord old_val,LockWord new_val)57 inline bool Object::CasLockWordWeakRelaxed(LockWord old_val, LockWord new_val) {
58 // Force use of non-transactional mode and do not check.
59 return CasFieldWeakRelaxed32<false, false>(
60 OFFSET_OF_OBJECT_MEMBER(Object, monitor_), old_val.GetValue(), new_val.GetValue());
61 }
62
CasLockWordWeakRelease(LockWord old_val,LockWord new_val)63 inline bool Object::CasLockWordWeakRelease(LockWord old_val, LockWord new_val) {
64 // Force use of non-transactional mode and do not check.
65 return CasFieldWeakRelease32<false, false>(
66 OFFSET_OF_OBJECT_MEMBER(Object, monitor_), old_val.GetValue(), new_val.GetValue());
67 }
68
GetReadBarrierState(uintptr_t * fake_address_dependency)69 inline uint32_t Object::GetReadBarrierState(uintptr_t* fake_address_dependency) {
70 if (!kUseBakerReadBarrier) {
71 LOG(FATAL) << "Unreachable";
72 UNREACHABLE();
73 }
74 #if defined(__arm__)
75 uintptr_t obj = reinterpret_cast<uintptr_t>(this);
76 uintptr_t result;
77 DCHECK_EQ(OFFSETOF_MEMBER(Object, monitor_), 4U);
78 // Use inline assembly to prevent the compiler from optimizing away the false dependency.
79 __asm__ __volatile__(
80 "ldr %[result], [%[obj], #4]\n\t"
81 // This instruction is enough to "fool the compiler and the CPU" by having `fad` always be
82 // null, without them being able to assume that fact.
83 "eor %[fad], %[result], %[result]\n\t"
84 : [result] "+r" (result), [fad] "=r" (*fake_address_dependency)
85 : [obj] "r" (obj));
86 DCHECK_EQ(*fake_address_dependency, 0U);
87 LockWord lw(static_cast<uint32_t>(result));
88 uint32_t rb_state = lw.ReadBarrierState();
89 return rb_state;
90 #elif defined(__aarch64__)
91 uintptr_t obj = reinterpret_cast<uintptr_t>(this);
92 uintptr_t result;
93 DCHECK_EQ(OFFSETOF_MEMBER(Object, monitor_), 4U);
94 // Use inline assembly to prevent the compiler from optimizing away the false dependency.
95 __asm__ __volatile__(
96 "ldr %w[result], [%[obj], #4]\n\t"
97 // This instruction is enough to "fool the compiler and the CPU" by having `fad` always be
98 // null, without them being able to assume that fact.
99 "eor %[fad], %[result], %[result]\n\t"
100 : [result] "+r" (result), [fad] "=r" (*fake_address_dependency)
101 : [obj] "r" (obj));
102 DCHECK_EQ(*fake_address_dependency, 0U);
103 LockWord lw(static_cast<uint32_t>(result));
104 uint32_t rb_state = lw.ReadBarrierState();
105 return rb_state;
106 #elif defined(__i386__) || defined(__x86_64__)
107 LockWord lw = GetLockWord(false);
108 // i386/x86_64 don't need fake address dependency. Use a compiler fence to avoid compiler
109 // reordering.
110 *fake_address_dependency = 0;
111 std::atomic_signal_fence(std::memory_order_acquire);
112 uint32_t rb_state = lw.ReadBarrierState();
113 return rb_state;
114 #else
115 // MIPS32/MIPS64: use a memory barrier to prevent load-load reordering.
116 LockWord lw = GetLockWord(false);
117 *fake_address_dependency = 0;
118 std::atomic_thread_fence(std::memory_order_acquire);
119 uint32_t rb_state = lw.ReadBarrierState();
120 return rb_state;
121 #endif
122 }
123
GetReadBarrierState()124 inline uint32_t Object::GetReadBarrierState() {
125 if (!kUseBakerReadBarrier) {
126 LOG(FATAL) << "Unreachable";
127 UNREACHABLE();
128 }
129 DCHECK(kUseBakerReadBarrier);
130 LockWord lw(GetField<uint32_t, /*kIsVolatile*/false>(OFFSET_OF_OBJECT_MEMBER(Object, monitor_)));
131 uint32_t rb_state = lw.ReadBarrierState();
132 DCHECK(ReadBarrier::IsValidReadBarrierState(rb_state)) << rb_state;
133 return rb_state;
134 }
135
GetReadBarrierStateAcquire()136 inline uint32_t Object::GetReadBarrierStateAcquire() {
137 if (!kUseBakerReadBarrier) {
138 LOG(FATAL) << "Unreachable";
139 UNREACHABLE();
140 }
141 LockWord lw(GetFieldAcquire<uint32_t>(OFFSET_OF_OBJECT_MEMBER(Object, monitor_)));
142 uint32_t rb_state = lw.ReadBarrierState();
143 DCHECK(ReadBarrier::IsValidReadBarrierState(rb_state)) << rb_state;
144 return rb_state;
145 }
146
147 template<bool kCasRelease>
AtomicSetReadBarrierState(uint32_t expected_rb_state,uint32_t rb_state)148 inline bool Object::AtomicSetReadBarrierState(uint32_t expected_rb_state, uint32_t rb_state) {
149 if (!kUseBakerReadBarrier) {
150 LOG(FATAL) << "Unreachable";
151 UNREACHABLE();
152 }
153 DCHECK(ReadBarrier::IsValidReadBarrierState(expected_rb_state)) << expected_rb_state;
154 DCHECK(ReadBarrier::IsValidReadBarrierState(rb_state)) << rb_state;
155 LockWord expected_lw;
156 LockWord new_lw;
157 do {
158 LockWord lw = GetLockWord(false);
159 if (UNLIKELY(lw.ReadBarrierState() != expected_rb_state)) {
160 // Lost the race.
161 return false;
162 }
163 expected_lw = lw;
164 expected_lw.SetReadBarrierState(expected_rb_state);
165 new_lw = lw;
166 new_lw.SetReadBarrierState(rb_state);
167 // ConcurrentCopying::ProcessMarkStackRef uses this with kCasRelease == true.
168 // If kCasRelease == true, use a CAS release so that when GC updates all the fields of
169 // an object and then changes the object from gray to black, the field updates (stores) will be
170 // visible (won't be reordered after this CAS.)
171 } while (!(kCasRelease ?
172 CasLockWordWeakRelease(expected_lw, new_lw) :
173 CasLockWordWeakRelaxed(expected_lw, new_lw)));
174 return true;
175 }
176
AtomicSetMarkBit(uint32_t expected_mark_bit,uint32_t mark_bit)177 inline bool Object::AtomicSetMarkBit(uint32_t expected_mark_bit, uint32_t mark_bit) {
178 LockWord expected_lw;
179 LockWord new_lw;
180 do {
181 LockWord lw = GetLockWord(false);
182 if (UNLIKELY(lw.MarkBitState() != expected_mark_bit)) {
183 // Lost the race.
184 return false;
185 }
186 expected_lw = lw;
187 new_lw = lw;
188 new_lw.SetMarkBitState(mark_bit);
189 // Since this is only set from the mutator, we can use the non release Cas.
190 } while (!CasLockWordWeakRelaxed(expected_lw, new_lw));
191 return true;
192 }
193
194 template<bool kTransactionActive, bool kCheckTransaction, VerifyObjectFlags kVerifyFlags>
CasFieldStrongRelaxedObjectWithoutWriteBarrier(MemberOffset field_offset,ObjPtr<Object> old_value,ObjPtr<Object> new_value)195 inline bool Object::CasFieldStrongRelaxedObjectWithoutWriteBarrier(
196 MemberOffset field_offset,
197 ObjPtr<Object> old_value,
198 ObjPtr<Object> new_value) {
199 if (kCheckTransaction) {
200 DCHECK_EQ(kTransactionActive, Runtime::Current()->IsActiveTransaction());
201 }
202 if (kVerifyFlags & kVerifyThis) {
203 VerifyObject(this);
204 }
205 if (kVerifyFlags & kVerifyWrites) {
206 VerifyObject(new_value);
207 }
208 if (kVerifyFlags & kVerifyReads) {
209 VerifyObject(old_value);
210 }
211 if (kTransactionActive) {
212 Runtime::Current()->RecordWriteFieldReference(this, field_offset, old_value, true);
213 }
214 HeapReference<Object> old_ref(HeapReference<Object>::FromObjPtr(old_value));
215 HeapReference<Object> new_ref(HeapReference<Object>::FromObjPtr(new_value));
216 uint8_t* raw_addr = reinterpret_cast<uint8_t*>(this) + field_offset.Int32Value();
217 Atomic<uint32_t>* atomic_addr = reinterpret_cast<Atomic<uint32_t>*>(raw_addr);
218
219 bool success = atomic_addr->CompareExchangeStrongRelaxed(old_ref.reference_,
220 new_ref.reference_);
221 return success;
222 }
223
224 template<bool kTransactionActive, bool kCheckTransaction, VerifyObjectFlags kVerifyFlags>
CasFieldStrongReleaseObjectWithoutWriteBarrier(MemberOffset field_offset,ObjPtr<Object> old_value,ObjPtr<Object> new_value)225 inline bool Object::CasFieldStrongReleaseObjectWithoutWriteBarrier(
226 MemberOffset field_offset,
227 ObjPtr<Object> old_value,
228 ObjPtr<Object> new_value) {
229 if (kCheckTransaction) {
230 DCHECK_EQ(kTransactionActive, Runtime::Current()->IsActiveTransaction());
231 }
232 if (kVerifyFlags & kVerifyThis) {
233 VerifyObject(this);
234 }
235 if (kVerifyFlags & kVerifyWrites) {
236 VerifyObject(new_value);
237 }
238 if (kVerifyFlags & kVerifyReads) {
239 VerifyObject(old_value);
240 }
241 if (kTransactionActive) {
242 Runtime::Current()->RecordWriteFieldReference(this, field_offset, old_value, true);
243 }
244 HeapReference<Object> old_ref(HeapReference<Object>::FromObjPtr(old_value));
245 HeapReference<Object> new_ref(HeapReference<Object>::FromObjPtr(new_value));
246 uint8_t* raw_addr = reinterpret_cast<uint8_t*>(this) + field_offset.Int32Value();
247 Atomic<uint32_t>* atomic_addr = reinterpret_cast<Atomic<uint32_t>*>(raw_addr);
248
249 bool success = atomic_addr->CompareExchangeStrongRelease(old_ref.reference_,
250 new_ref.reference_);
251 return success;
252 }
253
254 } // namespace mirror
255 } // namespace art
256
257 #endif // ART_RUNTIME_MIRROR_OBJECT_READBARRIER_INL_H_
258