1 // Copyright 2017 The Abseil Authors.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // https://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 // Allow dynamic symbol lookup in the kernel VDSO page.
16 //
17 // VDSOSupport -- a class representing kernel VDSO (if present).
18
19 #include "absl/debugging/internal/vdso_support.h"
20
21 #ifdef ABSL_HAVE_VDSO_SUPPORT // defined in vdso_support.h
22
23 #if !defined(__has_include)
24 #define __has_include(header) 0
25 #endif
26
27 #include <errno.h>
28 #include <fcntl.h>
29 #if __has_include(<syscall.h>)
30 #include <syscall.h>
31 #elif __has_include(<sys/syscall.h>)
32 #include <sys/syscall.h>
33 #endif
34 #include <unistd.h>
35
36 #if defined(__GLIBC__) && \
37 (__GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 16))
38 #define ABSL_HAVE_GETAUXVAL
39 #endif
40
41 #ifdef ABSL_HAVE_GETAUXVAL
42 #include <sys/auxv.h>
43 #endif
44
45 #include "absl/base/dynamic_annotations.h"
46 #include "absl/base/internal/raw_logging.h"
47 #include "absl/base/port.h"
48
49 #ifndef AT_SYSINFO_EHDR
50 #define AT_SYSINFO_EHDR 33 // for crosstoolv10
51 #endif
52
53 #if defined(__FreeBSD__)
54 using Elf64_auxv_t = Elf64_Auxinfo;
55 using Elf32_auxv_t = Elf32_Auxinfo;
56 #endif
57
58 namespace absl {
59 ABSL_NAMESPACE_BEGIN
60 namespace debugging_internal {
61
62 ABSL_CONST_INIT
63 std::atomic<const void *> VDSOSupport::vdso_base_(
64 debugging_internal::ElfMemImage::kInvalidBase);
65
66 std::atomic<VDSOSupport::GetCpuFn> VDSOSupport::getcpu_fn_(&InitAndGetCPU);
VDSOSupport()67 VDSOSupport::VDSOSupport()
68 // If vdso_base_ is still set to kInvalidBase, we got here
69 // before VDSOSupport::Init has been called. Call it now.
70 : image_(vdso_base_.load(std::memory_order_relaxed) ==
71 debugging_internal::ElfMemImage::kInvalidBase
72 ? Init()
73 : vdso_base_.load(std::memory_order_relaxed)) {}
74
75 // NOTE: we can't use GoogleOnceInit() below, because we can be
76 // called by tcmalloc, and none of the *once* stuff may be functional yet.
77 //
78 // In addition, we hope that the VDSOSupportHelper constructor
79 // causes this code to run before there are any threads, and before
80 // InitGoogle() has executed any chroot or setuid calls.
81 //
82 // Finally, even if there is a race here, it is harmless, because
83 // the operation should be idempotent.
Init()84 const void *VDSOSupport::Init() {
85 const auto kInvalidBase = debugging_internal::ElfMemImage::kInvalidBase;
86 #ifdef ABSL_HAVE_GETAUXVAL
87 if (vdso_base_.load(std::memory_order_relaxed) == kInvalidBase) {
88 errno = 0;
89 const void *const sysinfo_ehdr =
90 reinterpret_cast<const void *>(getauxval(AT_SYSINFO_EHDR));
91 if (errno == 0) {
92 vdso_base_.store(sysinfo_ehdr, std::memory_order_relaxed);
93 }
94 }
95 #endif // ABSL_HAVE_GETAUXVAL
96 if (vdso_base_.load(std::memory_order_relaxed) == kInvalidBase) {
97 int fd = open("/proc/self/auxv", O_RDONLY);
98 if (fd == -1) {
99 // Kernel too old to have a VDSO.
100 vdso_base_.store(nullptr, std::memory_order_relaxed);
101 getcpu_fn_.store(&GetCPUViaSyscall, std::memory_order_relaxed);
102 return nullptr;
103 }
104 ElfW(auxv_t) aux;
105 while (read(fd, &aux, sizeof(aux)) == sizeof(aux)) {
106 if (aux.a_type == AT_SYSINFO_EHDR) {
107 vdso_base_.store(reinterpret_cast<void *>(aux.a_un.a_val),
108 std::memory_order_relaxed);
109 break;
110 }
111 }
112 close(fd);
113 if (vdso_base_.load(std::memory_order_relaxed) == kInvalidBase) {
114 // Didn't find AT_SYSINFO_EHDR in auxv[].
115 vdso_base_.store(nullptr, std::memory_order_relaxed);
116 }
117 }
118 GetCpuFn fn = &GetCPUViaSyscall; // default if VDSO not present.
119 if (vdso_base_.load(std::memory_order_relaxed)) {
120 VDSOSupport vdso;
121 SymbolInfo info;
122 if (vdso.LookupSymbol("__vdso_getcpu", "LINUX_2.6", STT_FUNC, &info)) {
123 fn = reinterpret_cast<GetCpuFn>(const_cast<void *>(info.address));
124 }
125 }
126 // Subtle: this code runs outside of any locks; prevent compiler
127 // from assigning to getcpu_fn_ more than once.
128 getcpu_fn_.store(fn, std::memory_order_relaxed);
129 return vdso_base_.load(std::memory_order_relaxed);
130 }
131
SetBase(const void * base)132 const void *VDSOSupport::SetBase(const void *base) {
133 ABSL_RAW_CHECK(base != debugging_internal::ElfMemImage::kInvalidBase,
134 "internal error");
135 const void *old_base = vdso_base_.load(std::memory_order_relaxed);
136 vdso_base_.store(base, std::memory_order_relaxed);
137 image_.Init(base);
138 // Also reset getcpu_fn_, so GetCPU could be tested with simulated VDSO.
139 getcpu_fn_.store(&InitAndGetCPU, std::memory_order_relaxed);
140 return old_base;
141 }
142
LookupSymbol(const char * name,const char * version,int type,SymbolInfo * info) const143 bool VDSOSupport::LookupSymbol(const char *name,
144 const char *version,
145 int type,
146 SymbolInfo *info) const {
147 return image_.LookupSymbol(name, version, type, info);
148 }
149
LookupSymbolByAddress(const void * address,SymbolInfo * info_out) const150 bool VDSOSupport::LookupSymbolByAddress(const void *address,
151 SymbolInfo *info_out) const {
152 return image_.LookupSymbolByAddress(address, info_out);
153 }
154
155 // NOLINT on 'long' because this routine mimics kernel api.
GetCPUViaSyscall(unsigned * cpu,void *,void *)156 long VDSOSupport::GetCPUViaSyscall(unsigned *cpu, // NOLINT(runtime/int)
157 void *, void *) {
158 #ifdef SYS_getcpu
159 return syscall(SYS_getcpu, cpu, nullptr, nullptr);
160 #else
161 // x86_64 never implemented sys_getcpu(), except as a VDSO call.
162 static_cast<void>(cpu); // Avoid an unused argument compiler warning.
163 errno = ENOSYS;
164 return -1;
165 #endif
166 }
167
168 // Use fast __vdso_getcpu if available.
InitAndGetCPU(unsigned * cpu,void * x,void * y)169 long VDSOSupport::InitAndGetCPU(unsigned *cpu, // NOLINT(runtime/int)
170 void *x, void *y) {
171 Init();
172 GetCpuFn fn = getcpu_fn_.load(std::memory_order_relaxed);
173 ABSL_RAW_CHECK(fn != &InitAndGetCPU, "Init() did not set getcpu_fn_");
174 return (*fn)(cpu, x, y);
175 }
176
177 // This function must be very fast, and may be called from very
178 // low level (e.g. tcmalloc). Hence I avoid things like
179 // GoogleOnceInit() and ::operator new.
180 ABSL_ATTRIBUTE_NO_SANITIZE_MEMORY
GetCPU()181 int GetCPU() {
182 unsigned cpu;
183 int ret_code = (*VDSOSupport::getcpu_fn_)(&cpu, nullptr, nullptr);
184 return ret_code == 0 ? cpu : ret_code;
185 }
186
187 } // namespace debugging_internal
188 ABSL_NAMESPACE_END
189 } // namespace absl
190
191 #endif // ABSL_HAVE_VDSO_SUPPORT
192