• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (C) 2011 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #ifndef ART_RUNTIME_MIRROR_OBJECT_READBARRIER_INL_H_
18 #define ART_RUNTIME_MIRROR_OBJECT_READBARRIER_INL_H_
19 
20 #include "object.h"
21 
22 #include "base/atomic.h"
23 #include "heap_poisoning.h"
24 #include "lock_word-inl.h"
25 #include "object_reference-inl.h"
26 #include "read_barrier.h"
27 #include "runtime.h"
28 
29 namespace art {
30 namespace mirror {
31 
32 template<VerifyObjectFlags kVerifyFlags>
GetLockWord(bool as_volatile)33 inline LockWord Object::GetLockWord(bool as_volatile) {
34   if (as_volatile) {
35     return LockWord(GetField32Volatile<kVerifyFlags>(MonitorOffset()));
36   }
37   return LockWord(GetField32<kVerifyFlags>(MonitorOffset()));
38 }
39 
40 template<bool kTransactionActive, bool kCheckTransaction, VerifyObjectFlags kVerifyFlags>
CasField32(MemberOffset field_offset,int32_t old_value,int32_t new_value,CASMode mode,std::memory_order memory_order)41 inline bool Object::CasField32(MemberOffset field_offset,
42                                int32_t old_value,
43                                int32_t new_value,
44                                CASMode mode,
45                                std::memory_order memory_order) {
46   if (kCheckTransaction) {
47     DCHECK_EQ(kTransactionActive, Runtime::Current()->IsActiveTransaction());
48   }
49   if (kTransactionActive) {
50     Runtime::Current()->RecordWriteField32(this, field_offset, old_value, true);
51   }
52   if (kVerifyFlags & kVerifyThis) {
53     VerifyObject(this);
54   }
55   uint8_t* raw_addr = reinterpret_cast<uint8_t*>(this) + field_offset.Int32Value();
56   AtomicInteger* atomic_addr = reinterpret_cast<AtomicInteger*>(raw_addr);
57 
58   return atomic_addr->CompareAndSet(old_value, new_value, mode, memory_order);
59 }
60 
CasLockWord(LockWord old_val,LockWord new_val,CASMode mode,std::memory_order memory_order)61 inline bool Object::CasLockWord(LockWord old_val,
62                                 LockWord new_val,
63                                 CASMode mode,
64                                 std::memory_order memory_order) {
65   // Force use of non-transactional mode and do not check.
66   return CasField32<false, false>(MonitorOffset(),
67                                   old_val.GetValue(),
68                                   new_val.GetValue(),
69                                   mode,
70                                   memory_order);
71 }
72 
GetReadBarrierState(uintptr_t * fake_address_dependency)73 inline uint32_t Object::GetReadBarrierState(uintptr_t* fake_address_dependency) {
74   if (!kUseBakerReadBarrier) {
75     LOG(FATAL) << "Unreachable";
76     UNREACHABLE();
77   }
78 #if defined(__arm__)
79   uintptr_t obj = reinterpret_cast<uintptr_t>(this);
80   uintptr_t result;
81   DCHECK_EQ(OFFSETOF_MEMBER(Object, monitor_), 4U);
82   // Use inline assembly to prevent the compiler from optimizing away the false dependency.
83   __asm__ __volatile__(
84       "ldr %[result], [%[obj], #4]\n\t"
85       // This instruction is enough to "fool the compiler and the CPU" by having `fad` always be
86       // null, without them being able to assume that fact.
87       "eor %[fad], %[result], %[result]\n\t"
88       : [result] "+r" (result), [fad] "=r" (*fake_address_dependency)
89       : [obj] "r" (obj));
90   DCHECK_EQ(*fake_address_dependency, 0U);
91   LockWord lw(static_cast<uint32_t>(result));
92   uint32_t rb_state = lw.ReadBarrierState();
93   return rb_state;
94 #elif defined(__aarch64__)
95   uintptr_t obj = reinterpret_cast<uintptr_t>(this);
96   uintptr_t result;
97   DCHECK_EQ(OFFSETOF_MEMBER(Object, monitor_), 4U);
98   // Use inline assembly to prevent the compiler from optimizing away the false dependency.
99   __asm__ __volatile__(
100       "ldr %w[result], [%[obj], #4]\n\t"
101       // This instruction is enough to "fool the compiler and the CPU" by having `fad` always be
102       // null, without them being able to assume that fact.
103       "eor %[fad], %[result], %[result]\n\t"
104       : [result] "+r" (result), [fad] "=r" (*fake_address_dependency)
105       : [obj] "r" (obj));
106   DCHECK_EQ(*fake_address_dependency, 0U);
107   LockWord lw(static_cast<uint32_t>(result));
108   uint32_t rb_state = lw.ReadBarrierState();
109   return rb_state;
110 #elif defined(__i386__) || defined(__x86_64__)
111   LockWord lw = GetLockWord(false);
112   // i386/x86_64 don't need fake address dependency. Use a compiler fence to avoid compiler
113   // reordering.
114   *fake_address_dependency = 0;
115   std::atomic_signal_fence(std::memory_order_acquire);
116   uint32_t rb_state = lw.ReadBarrierState();
117   return rb_state;
118 #else
119   // MIPS32/MIPS64: use a memory barrier to prevent load-load reordering.
120   LockWord lw = GetLockWord(false);
121   *fake_address_dependency = 0;
122   std::atomic_thread_fence(std::memory_order_acquire);
123   uint32_t rb_state = lw.ReadBarrierState();
124   return rb_state;
125 #endif
126 }
127 
GetReadBarrierState()128 inline uint32_t Object::GetReadBarrierState() {
129   if (!kUseBakerReadBarrier) {
130     LOG(FATAL) << "Unreachable";
131     UNREACHABLE();
132   }
133   DCHECK(kUseBakerReadBarrier);
134   LockWord lw(GetFieldPrimitive<uint32_t, /*kIsVolatile=*/false>(MonitorOffset()));
135   uint32_t rb_state = lw.ReadBarrierState();
136   DCHECK(ReadBarrier::IsValidReadBarrierState(rb_state)) << rb_state;
137   return rb_state;
138 }
139 
GetReadBarrierStateAcquire()140 inline uint32_t Object::GetReadBarrierStateAcquire() {
141   if (!kUseBakerReadBarrier) {
142     LOG(FATAL) << "Unreachable";
143     UNREACHABLE();
144   }
145   LockWord lw(GetFieldAcquire<uint32_t>(MonitorOffset()));
146   uint32_t rb_state = lw.ReadBarrierState();
147   DCHECK(ReadBarrier::IsValidReadBarrierState(rb_state)) << rb_state;
148   return rb_state;
149 }
150 
151 template<std::memory_order kMemoryOrder>
AtomicSetReadBarrierState(uint32_t expected_rb_state,uint32_t rb_state)152 inline bool Object::AtomicSetReadBarrierState(uint32_t expected_rb_state, uint32_t rb_state) {
153   if (!kUseBakerReadBarrier) {
154     LOG(FATAL) << "Unreachable";
155     UNREACHABLE();
156   }
157   DCHECK(ReadBarrier::IsValidReadBarrierState(expected_rb_state)) << expected_rb_state;
158   DCHECK(ReadBarrier::IsValidReadBarrierState(rb_state)) << rb_state;
159   LockWord expected_lw;
160   LockWord new_lw;
161   do {
162     LockWord lw = GetLockWord(false);
163     if (UNLIKELY(lw.ReadBarrierState() != expected_rb_state)) {
164       // Lost the race.
165       return false;
166     }
167     expected_lw = lw;
168     expected_lw.SetReadBarrierState(expected_rb_state);
169     new_lw = lw;
170     new_lw.SetReadBarrierState(rb_state);
171     // ConcurrentCopying::ProcessMarkStackRef uses this with
172     // `kMemoryOrder` == `std::memory_order_release`.
173     // If `kMemoryOrder` == `std::memory_order_release`, use a CAS release so that when GC updates
174     // all the fields of an object and then changes the object from gray to black (non-gray), the
175     // field updates (stores) will be visible (won't be reordered after this CAS.)
176   } while (!CasLockWord(expected_lw, new_lw, CASMode::kWeak, kMemoryOrder));
177   return true;
178 }
179 
AtomicSetMarkBit(uint32_t expected_mark_bit,uint32_t mark_bit)180 inline bool Object::AtomicSetMarkBit(uint32_t expected_mark_bit, uint32_t mark_bit) {
181   LockWord expected_lw;
182   LockWord new_lw;
183   do {
184     LockWord lw = GetLockWord(false);
185     if (UNLIKELY(lw.MarkBitState() != expected_mark_bit)) {
186       // Lost the race.
187       return false;
188     }
189     expected_lw = lw;
190     new_lw = lw;
191     new_lw.SetMarkBitState(mark_bit);
192     // Since this is only set from the mutator, we can use the non-release CAS.
193   } while (!CasLockWord(expected_lw, new_lw, CASMode::kWeak, std::memory_order_relaxed));
194   return true;
195 }
196 
197 }  // namespace mirror
198 }  // namespace art
199 
200 #endif  // ART_RUNTIME_MIRROR_OBJECT_READBARRIER_INL_H_
201