1 /*
2 * Copyright (C) 2020 The Android Open Source Project
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * * Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * * Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in
12 * the documentation and/or other materials provided with the
13 * distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
16 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
17 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
18 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
19 * COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
21 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
22 * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
23 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
25 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 #include "atexit.h"
30
31 #include <errno.h>
32 #include <pthread.h>
33 #include <stdint.h>
34 #include <stdlib.h>
35 #include <string.h>
36 #include <sys/mman.h>
37 #include <sys/param.h>
38 #include <sys/prctl.h>
39
40 #include <async_safe/CHECK.h>
41 #include <async_safe/log.h>
42
43 #include "platform/bionic/page.h"
44
45 extern "C" void __libc_stdio_cleanup();
46 extern "C" void __unregister_atfork(void* dso);
47
48 namespace {
49
50 struct AtexitEntry {
51 void (*fn)(void*); // the __cxa_atexit callback
52 void* arg; // argument for `fn` callback
53 void* dso; // shared module handle
54 };
55
56 class AtexitArray {
57 public:
size() const58 size_t size() const { return size_; }
total_appends() const59 uint64_t total_appends() const { return total_appends_; }
operator [](size_t idx) const60 const AtexitEntry& operator[](size_t idx) const { return array_[idx]; }
61
62 bool append_entry(const AtexitEntry& entry);
63 AtexitEntry extract_entry(size_t idx);
64 void recompact();
65
66 private:
67 AtexitEntry* array_;
68 size_t size_;
69 size_t extracted_count_;
70 size_t capacity_;
71
72 // An entry can be appended by a __cxa_finalize callback. Track the number of appends so we
73 // restart concurrent __cxa_finalize passes.
74 uint64_t total_appends_;
75
page_start_of_index(size_t idx)76 static size_t page_start_of_index(size_t idx) { return PAGE_START(idx * sizeof(AtexitEntry)); }
page_end_of_index(size_t idx)77 static size_t page_end_of_index(size_t idx) { return PAGE_END(idx * sizeof(AtexitEntry)); }
78
79 // Recompact the array if it will save at least one page of memory at the end.
needs_recompaction() const80 bool needs_recompaction() const {
81 return page_end_of_index(size_ - extracted_count_) < page_end_of_index(size_);
82 }
83
84 void set_writable(bool writable, size_t start_idx, size_t num_entries);
85 static bool next_capacity(size_t capacity, size_t* result);
86 bool expand_capacity();
87 };
88
89 } // anonymous namespace
90
append_entry(const AtexitEntry & entry)91 bool AtexitArray::append_entry(const AtexitEntry& entry) {
92 if (size_ >= capacity_ && !expand_capacity()) return false;
93
94 size_t idx = size_++;
95
96 set_writable(true, idx, 1);
97 array_[idx] = entry;
98 ++total_appends_;
99 set_writable(false, idx, 1);
100
101 return true;
102 }
103
104 // Extract an entry and return it.
extract_entry(size_t idx)105 AtexitEntry AtexitArray::extract_entry(size_t idx) {
106 AtexitEntry result = array_[idx];
107
108 set_writable(true, idx, 1);
109 array_[idx] = {};
110 ++extracted_count_;
111 set_writable(false, idx, 1);
112
113 return result;
114 }
115
recompact()116 void AtexitArray::recompact() {
117 if (!needs_recompaction()) return;
118
119 set_writable(true, 0, size_);
120
121 // Optimization: quickly skip over the initial non-null entries.
122 size_t src = 0, dst = 0;
123 while (src < size_ && array_[src].fn != nullptr) {
124 ++src;
125 ++dst;
126 }
127
128 // Shift the non-null entries forward, and zero out the removed entries at the end of the array.
129 for (; src < size_; ++src) {
130 const AtexitEntry entry = array_[src];
131 array_[src] = {};
132 if (entry.fn != nullptr) {
133 array_[dst++] = entry;
134 }
135 }
136
137 // If the table uses fewer pages, clean the pages at the end.
138 size_t old_bytes = page_end_of_index(size_);
139 size_t new_bytes = page_end_of_index(dst);
140 if (new_bytes < old_bytes) {
141 madvise(reinterpret_cast<char*>(array_) + new_bytes, old_bytes - new_bytes, MADV_DONTNEED);
142 }
143
144 set_writable(false, 0, size_);
145
146 size_ = dst;
147 extracted_count_ = 0;
148 }
149
150 // Use mprotect to make the array writable or read-only. Returns true on success. Making the array
151 // read-only could protect against either unintentional or malicious corruption of the array.
set_writable(bool writable,size_t start_idx,size_t num_entries)152 void AtexitArray::set_writable(bool writable, size_t start_idx, size_t num_entries) {
153 if (array_ == nullptr) return;
154
155 const size_t start_byte = page_start_of_index(start_idx);
156 const size_t stop_byte = page_end_of_index(start_idx + num_entries);
157 const size_t byte_len = stop_byte - start_byte;
158
159 const int prot = PROT_READ | (writable ? PROT_WRITE : 0);
160 if (mprotect(reinterpret_cast<char*>(array_) + start_byte, byte_len, prot) != 0) {
161 async_safe_fatal("mprotect failed on atexit array: %s", strerror(errno));
162 }
163 }
164
165 // Approximately double the capacity. Returns true if successful (no overflow). AtexitEntry is
166 // smaller than a page, but this function should still be correct even if AtexitEntry were larger
167 // than one.
next_capacity(size_t capacity,size_t * result)168 bool AtexitArray::next_capacity(size_t capacity, size_t* result) {
169 if (capacity == 0) {
170 *result = PAGE_END(sizeof(AtexitEntry)) / sizeof(AtexitEntry);
171 return true;
172 }
173 size_t num_bytes;
174 if (__builtin_mul_overflow(page_end_of_index(capacity), 2, &num_bytes)) {
175 async_safe_format_log(ANDROID_LOG_WARN, "libc", "__cxa_atexit: capacity calculation overflow");
176 return false;
177 }
178 *result = num_bytes / sizeof(AtexitEntry);
179 return true;
180 }
181
expand_capacity()182 bool AtexitArray::expand_capacity() {
183 size_t new_capacity;
184 if (!next_capacity(capacity_, &new_capacity)) return false;
185 const size_t new_capacity_bytes = page_end_of_index(new_capacity);
186
187 set_writable(true, 0, capacity_);
188
189 bool result = false;
190 void* new_pages;
191 if (array_ == nullptr) {
192 new_pages = mmap(nullptr, new_capacity_bytes, PROT_READ | PROT_WRITE,
193 MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
194 } else {
195 // mremap fails if the source buffer crosses a boundary between two VMAs. When a single array
196 // element is modified, the kernel should split then rejoin the buffer's VMA.
197 new_pages = mremap(array_, page_end_of_index(capacity_), new_capacity_bytes, MREMAP_MAYMOVE);
198 }
199 if (new_pages == MAP_FAILED) {
200 async_safe_format_log(ANDROID_LOG_WARN, "libc",
201 "__cxa_atexit: mmap/mremap failed to allocate %zu bytes: %s",
202 new_capacity_bytes, strerror(errno));
203 } else {
204 result = true;
205 prctl(PR_SET_VMA, PR_SET_VMA_ANON_NAME, new_pages, new_capacity_bytes, "atexit handlers");
206 array_ = static_cast<AtexitEntry*>(new_pages);
207 capacity_ = new_capacity;
208 }
209 set_writable(false, 0, capacity_);
210 return result;
211 }
212
213 static AtexitArray g_array;
214 static pthread_mutex_t g_atexit_lock = PTHREAD_MUTEX_INITIALIZER;
215
atexit_lock()216 static inline void atexit_lock() {
217 pthread_mutex_lock(&g_atexit_lock);
218 }
219
atexit_unlock()220 static inline void atexit_unlock() {
221 pthread_mutex_unlock(&g_atexit_lock);
222 }
223
224 // Register a function to be called either when a library is unloaded (dso != nullptr), or when the
225 // program exits (dso == nullptr). The `dso` argument is typically the address of a hidden
226 // __dso_handle variable. This function is also used as the backend for the atexit function.
227 //
228 // See https://itanium-cxx-abi.github.io/cxx-abi/abi.html#dso-dtor.
229 //
__cxa_atexit(void (* func)(void *),void * arg,void * dso)230 int __cxa_atexit(void (*func)(void*), void* arg, void* dso) {
231 int result = -1;
232
233 if (func != nullptr) {
234 atexit_lock();
235 if (g_array.append_entry({.fn = func, .arg = arg, .dso = dso})) {
236 result = 0;
237 }
238 atexit_unlock();
239 }
240
241 return result;
242 }
243
__cxa_finalize(void * dso)244 void __cxa_finalize(void* dso) {
245 atexit_lock();
246
247 static uint32_t call_depth = 0;
248 ++call_depth;
249
250 restart:
251 const uint64_t total_appends = g_array.total_appends();
252
253 for (ssize_t i = g_array.size() - 1; i >= 0; --i) {
254 if (g_array[i].fn == nullptr || (dso != nullptr && g_array[i].dso != dso)) continue;
255
256 // Clear the entry in the array because its DSO handle will become invalid, and to avoid calling
257 // an entry again if __cxa_finalize is called recursively.
258 const AtexitEntry entry = g_array.extract_entry(i);
259
260 atexit_unlock();
261 entry.fn(entry.arg);
262 atexit_lock();
263
264 if (g_array.total_appends() != total_appends) goto restart;
265 }
266
267 // Avoid recompaction on recursive calls because it's unnecessary and would require earlier,
268 // concurrent __cxa_finalize calls to restart. Skip recompaction on program exit too
269 // (dso == nullptr), because the memory will be reclaimed soon anyway.
270 --call_depth;
271 if (call_depth == 0 && dso != nullptr) {
272 g_array.recompact();
273 }
274
275 atexit_unlock();
276
277 if (dso != nullptr) {
278 __unregister_atfork(dso);
279 } else {
280 // If called via exit(), flush output of all open files.
281 __libc_stdio_cleanup();
282 }
283 }
284