1 /*
2 * Copyright (C) 2019 The Android Open Source Project
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * * Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * * Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in
12 * the documentation and/or other materials provided with the
13 * distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
16 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
17 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
18 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
19 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
21 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
22 * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
23 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
25 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 #pragma once
30
31 #include <link.h>
32 #include <pthread.h>
33 #include <stdatomic.h>
34 #include <stdint.h>
35 #include <sys/cdefs.h>
36
37 __LIBC_HIDDEN__ extern _Atomic(size_t) __libc_tls_generation_copy;
38
39 struct TlsSegment {
40 size_t size = 0;
41 size_t alignment = 1;
42 const void* init_ptr = ""; // Field is non-null even when init_size is 0.
43 size_t init_size = 0;
44 };
45
46 __LIBC_HIDDEN__ bool __bionic_get_tls_segment(const ElfW(Phdr)* phdr_table, size_t phdr_count,
47 ElfW(Addr) load_bias, TlsSegment* out);
48
49 __LIBC_HIDDEN__ bool __bionic_check_tls_alignment(size_t* alignment);
50
51 struct StaticTlsLayout {
StaticTlsLayoutStaticTlsLayout52 constexpr StaticTlsLayout() {}
53
54 private:
55 size_t offset_ = 0;
56 size_t alignment_ = 1;
57 bool overflowed_ = false;
58
59 // Offsets to various Bionic TLS structs from the beginning of static TLS.
60 size_t offset_bionic_tcb_ = SIZE_MAX;
61 size_t offset_bionic_tls_ = SIZE_MAX;
62
63 public:
offset_bionic_tcbStaticTlsLayout64 size_t offset_bionic_tcb() const { return offset_bionic_tcb_; }
offset_bionic_tlsStaticTlsLayout65 size_t offset_bionic_tls() const { return offset_bionic_tls_; }
66 size_t offset_thread_pointer() const;
67
sizeStaticTlsLayout68 size_t size() const { return offset_; }
alignmentStaticTlsLayout69 size_t alignment() const { return alignment_; }
overflowedStaticTlsLayout70 bool overflowed() const { return overflowed_; }
71
72 size_t reserve_exe_segment_and_tcb(const TlsSegment* exe_segment, const char* progname);
73 void reserve_bionic_tls();
reserve_solib_segmentStaticTlsLayout74 size_t reserve_solib_segment(const TlsSegment& segment) {
75 return reserve(segment.size, segment.alignment);
76 }
77 void finish_layout();
78
79 private:
80 size_t reserve(size_t size, size_t alignment);
81
reserve_typeStaticTlsLayout82 template <typename T> size_t reserve_type() {
83 return reserve(sizeof(T), alignof(T));
84 }
85
86 size_t round_up_with_overflow_check(size_t value, size_t alignment);
87 };
88
89 static constexpr size_t kTlsGenerationNone = 0;
90 static constexpr size_t kTlsGenerationFirst = 1;
91
92 // The first ELF TLS module has ID 1. Zero is reserved for the first word of
93 // the DTV, a generation count. Unresolved weak symbols also use module ID 0.
94 static constexpr size_t kTlsUninitializedModuleId = 0;
95
__tls_module_id_to_idx(size_t id)96 static inline size_t __tls_module_id_to_idx(size_t id) { return id - 1; }
__tls_module_idx_to_id(size_t idx)97 static inline size_t __tls_module_idx_to_id(size_t idx) { return idx + 1; }
98
99 // A descriptor for a single ELF TLS module.
100 struct TlsModule {
101 TlsSegment segment;
102
103 // Offset into the static TLS block or SIZE_MAX for a dynamic module.
104 size_t static_offset = SIZE_MAX;
105
106 // The generation in which this module was loaded. Dynamic TLS lookups use
107 // this field to detect when a module has been unloaded.
108 size_t first_generation = kTlsGenerationNone;
109
110 // Used by the dynamic linker to track the associated soinfo* object.
111 void* soinfo_ptr = nullptr;
112 };
113
114 // Table of the ELF TLS modules. Either the dynamic linker or the static
115 // initialization code prepares this table, and it's then used during thread
116 // creation and for dynamic TLS lookups.
117 struct TlsModules {
TlsModulesTlsModules118 constexpr TlsModules() {}
119
120 // A pointer to the TLS generation counter in libc.so. The counter is
121 // incremented each time an solib is loaded or unloaded.
122 _Atomic(size_t) generation = kTlsGenerationFirst;
123 _Atomic(size_t) *generation_libc_so = nullptr;
124
125 // Access to the TlsModule[] table requires taking this lock.
126 pthread_rwlock_t rwlock = PTHREAD_RWLOCK_INITIALIZER;
127
128 // Pointer to a block of TlsModule objects. The first module has ID 1 and
129 // is stored at index 0 in this table.
130 size_t module_count = 0;
131 TlsModule* module_table = nullptr;
132 };
133
134 void __init_static_tls(void* static_tls);
135
136 // Dynamic Thread Vector. Each thread has a different DTV. For each module
137 // (executable or solib), the DTV has a pointer to that module's TLS memory. The
138 // DTV is initially empty and is allocated on-demand. It grows as more modules
139 // are dlopen'ed. See https://www.akkadia.org/drepper/tls.pdf.
140 //
141 // The layout of the DTV is specified in various documents, but it is not part
142 // of Bionic's public ABI. A compiler can't generate code to access it directly,
143 // because it can't access libc's global generation counter.
144 struct TlsDtv {
145 // Number of elements in this object's modules field.
146 size_t count;
147
148 // A pointer to an older TlsDtv object that should be freed when the thread
149 // exits. The objects aren't immediately freed because a DTV could be
150 // reallocated by a signal handler that interrupted __tls_get_addr's fast
151 // path.
152 TlsDtv* next;
153
154 // The DTV slot points at this field, which allows omitting an add instruction
155 // on the fast path for a TLS lookup. The arm64 tlsdesc_resolver.S depends on
156 // the layout of fields past this point.
157 size_t generation;
158 void* modules[];
159 };
160
161 struct TlsIndex {
162 size_t module_id;
163 size_t offset;
164 };
165
166 #if defined(__i386__)
167 #define TLS_GET_ADDR_CCONV __attribute__((regparm(1)))
168 #define TLS_GET_ADDR ___tls_get_addr
169 #else
170 #define TLS_GET_ADDR_CCONV
171 #define TLS_GET_ADDR __tls_get_addr
172 #endif
173
174 extern "C" void* TLS_GET_ADDR(const TlsIndex* ti) TLS_GET_ADDR_CCONV;
175
176 struct bionic_tcb;
177 void __free_dynamic_tls(bionic_tcb* tcb);
178