1 //===-- tsd_exclusive.h -----------------------------------------*- C++ -*-===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8
9 #ifndef SCUDO_TSD_EXCLUSIVE_H_
10 #define SCUDO_TSD_EXCLUSIVE_H_
11
12 #include "tsd.h"
13
14 #include "string_utils.h"
15
16 namespace scudo {
17
18 struct ThreadState {
19 bool DisableMemInit : 1;
20 enum : unsigned {
21 NotInitialized = 0,
22 Initialized,
23 TornDown,
24 } InitState : 2;
25 };
26
27 template <class Allocator> void teardownThread(void *Ptr);
28
29 template <class Allocator> struct TSDRegistryExT {
initTSDRegistryExT30 void init(Allocator *Instance) REQUIRES(Mutex) {
31 DCHECK(!Initialized);
32 Instance->init();
33 CHECK_EQ(pthread_key_create(&PThreadKey, teardownThread<Allocator>), 0);
34 FallbackTSD.init(Instance);
35 Initialized = true;
36 }
37
initOnceMaybeTSDRegistryExT38 void initOnceMaybe(Allocator *Instance) EXCLUDES(Mutex) {
39 ScopedLock L(Mutex);
40 if (LIKELY(Initialized))
41 return;
42 init(Instance); // Sets Initialized.
43 }
44
unmapTestOnlyTSDRegistryExT45 void unmapTestOnly(Allocator *Instance) EXCLUDES(Mutex) {
46 DCHECK(Instance);
47 if (reinterpret_cast<Allocator *>(pthread_getspecific(PThreadKey))) {
48 DCHECK_EQ(reinterpret_cast<Allocator *>(pthread_getspecific(PThreadKey)),
49 Instance);
50 ThreadTSD.commitBack(Instance);
51 ThreadTSD = {};
52 }
53 CHECK_EQ(pthread_key_delete(PThreadKey), 0);
54 PThreadKey = {};
55 FallbackTSD.commitBack(Instance);
56 FallbackTSD = {};
57 State = {};
58 ScopedLock L(Mutex);
59 Initialized = false;
60 }
61
drainCachesTSDRegistryExT62 void drainCaches(Allocator *Instance) {
63 // We don't have a way to iterate all thread local `ThreadTSD`s. Simply
64 // drain the `ThreadTSD` of current thread and `FallbackTSD`.
65 Instance->drainCache(&ThreadTSD);
66 FallbackTSD.lock();
67 Instance->drainCache(&FallbackTSD);
68 FallbackTSD.unlock();
69 }
70
initThreadMaybeTSDRegistryExT71 ALWAYS_INLINE void initThreadMaybe(Allocator *Instance, bool MinimalInit) {
72 if (LIKELY(State.InitState != ThreadState::NotInitialized))
73 return;
74 initThread(Instance, MinimalInit);
75 }
76
77 // TODO(chiahungduan): Consider removing the argument `UnlockRequired` by
78 // embedding the logic into TSD or always locking the TSD. It will enable us
79 // to properly mark thread annotation here and adding proper runtime
80 // assertions in the member functions of TSD. For example, assert the lock is
81 // acquired before calling TSD::commitBack().
82 ALWAYS_INLINE TSD<Allocator> *
getTSDAndLockTSDRegistryExT83 getTSDAndLock(bool *UnlockRequired) NO_THREAD_SAFETY_ANALYSIS {
84 if (LIKELY(State.InitState == ThreadState::Initialized &&
85 !atomic_load(&Disabled, memory_order_acquire))) {
86 *UnlockRequired = false;
87 return &ThreadTSD;
88 }
89 FallbackTSD.lock();
90 *UnlockRequired = true;
91 return &FallbackTSD;
92 }
93
94 // To disable the exclusive TSD registry, we effectively lock the fallback TSD
95 // and force all threads to attempt to use it instead of their local one.
disableTSDRegistryExT96 void disable() NO_THREAD_SAFETY_ANALYSIS {
97 Mutex.lock();
98 FallbackTSD.lock();
99 atomic_store(&Disabled, 1U, memory_order_release);
100 }
101
enableTSDRegistryExT102 void enable() NO_THREAD_SAFETY_ANALYSIS {
103 atomic_store(&Disabled, 0U, memory_order_release);
104 FallbackTSD.unlock();
105 Mutex.unlock();
106 }
107
setOptionTSDRegistryExT108 bool setOption(Option O, sptr Value) {
109 if (O == Option::ThreadDisableMemInit)
110 State.DisableMemInit = Value;
111 if (O == Option::MaxTSDsCount)
112 return false;
113 return true;
114 }
115
getDisableMemInitTSDRegistryExT116 bool getDisableMemInit() { return State.DisableMemInit; }
117
getStatsTSDRegistryExT118 void getStats(ScopedString *Str) {
119 // We don't have a way to iterate all thread local `ThreadTSD`s. Instead of
120 // printing only self `ThreadTSD` which may mislead the usage, we just skip
121 // it.
122 Str->append("Exclusive TSD don't support iterating each TSD\n");
123 }
124
125 private:
126 // Using minimal initialization allows for global initialization while keeping
127 // the thread specific structure untouched. The fallback structure will be
128 // used instead.
initThreadTSDRegistryExT129 NOINLINE void initThread(Allocator *Instance, bool MinimalInit) {
130 initOnceMaybe(Instance);
131 if (UNLIKELY(MinimalInit))
132 return;
133 CHECK_EQ(
134 pthread_setspecific(PThreadKey, reinterpret_cast<void *>(Instance)), 0);
135 ThreadTSD.init(Instance);
136 State.InitState = ThreadState::Initialized;
137 Instance->callPostInitCallback();
138 }
139
140 pthread_key_t PThreadKey = {};
141 bool Initialized GUARDED_BY(Mutex) = false;
142 atomic_u8 Disabled = {};
143 TSD<Allocator> FallbackTSD;
144 HybridMutex Mutex;
145 static thread_local ThreadState State;
146 static thread_local TSD<Allocator> ThreadTSD;
147
148 friend void teardownThread<Allocator>(void *Ptr);
149 };
150
151 template <class Allocator>
152 thread_local TSD<Allocator> TSDRegistryExT<Allocator>::ThreadTSD;
153 template <class Allocator>
154 thread_local ThreadState TSDRegistryExT<Allocator>::State;
155
156 template <class Allocator>
teardownThread(void * Ptr)157 void teardownThread(void *Ptr) NO_THREAD_SAFETY_ANALYSIS {
158 typedef TSDRegistryExT<Allocator> TSDRegistryT;
159 Allocator *Instance = reinterpret_cast<Allocator *>(Ptr);
160 // The glibc POSIX thread-local-storage deallocation routine calls user
161 // provided destructors in a loop of PTHREAD_DESTRUCTOR_ITERATIONS.
162 // We want to be called last since other destructors might call free and the
163 // like, so we wait until PTHREAD_DESTRUCTOR_ITERATIONS before draining the
164 // quarantine and swallowing the cache.
165 if (TSDRegistryT::ThreadTSD.DestructorIterations > 1) {
166 TSDRegistryT::ThreadTSD.DestructorIterations--;
167 // If pthread_setspecific fails, we will go ahead with the teardown.
168 if (LIKELY(pthread_setspecific(Instance->getTSDRegistry()->PThreadKey,
169 Ptr) == 0))
170 return;
171 }
172 TSDRegistryT::ThreadTSD.commitBack(Instance);
173 TSDRegistryT::State.InitState = ThreadState::TornDown;
174 }
175
176 } // namespace scudo
177
178 #endif // SCUDO_TSD_EXCLUSIVE_H_
179