1 // Copyright 2017 The Abseil Authors.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // https://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 // Allow dynamic symbol lookup in the kernel VDSO page.
16 //
17 // VDSOSupport -- a class representing kernel VDSO (if present).
18
19 #include "absl/debugging/internal/vdso_support.h"
20
21 #ifdef ABSL_HAVE_VDSO_SUPPORT // defined in vdso_support.h
22
23 #include <errno.h>
24 #include <fcntl.h>
25 #include <sys/syscall.h>
26 #include <unistd.h>
27
28 #if __GLIBC_PREREQ(2, 16) // GLIBC-2.16 implements getauxval.
29 #include <sys/auxv.h>
30 #endif
31
32 #include "absl/base/dynamic_annotations.h"
33 #include "absl/base/internal/raw_logging.h"
34 #include "absl/base/port.h"
35
36 #ifndef AT_SYSINFO_EHDR
37 #define AT_SYSINFO_EHDR 33 // for crosstoolv10
38 #endif
39
40 namespace absl {
41 ABSL_NAMESPACE_BEGIN
42 namespace debugging_internal {
43
44 ABSL_CONST_INIT
45 std::atomic<const void *> VDSOSupport::vdso_base_(
46 debugging_internal::ElfMemImage::kInvalidBase);
47
48 std::atomic<VDSOSupport::GetCpuFn> VDSOSupport::getcpu_fn_(&InitAndGetCPU);
VDSOSupport()49 VDSOSupport::VDSOSupport()
50 // If vdso_base_ is still set to kInvalidBase, we got here
51 // before VDSOSupport::Init has been called. Call it now.
52 : image_(vdso_base_.load(std::memory_order_relaxed) ==
53 debugging_internal::ElfMemImage::kInvalidBase
54 ? Init()
55 : vdso_base_.load(std::memory_order_relaxed)) {}
56
57 // NOTE: we can't use GoogleOnceInit() below, because we can be
58 // called by tcmalloc, and none of the *once* stuff may be functional yet.
59 //
60 // In addition, we hope that the VDSOSupportHelper constructor
61 // causes this code to run before there are any threads, and before
62 // InitGoogle() has executed any chroot or setuid calls.
63 //
64 // Finally, even if there is a race here, it is harmless, because
65 // the operation should be idempotent.
Init()66 const void *VDSOSupport::Init() {
67 const auto kInvalidBase = debugging_internal::ElfMemImage::kInvalidBase;
68 #if __GLIBC_PREREQ(2, 16)
69 if (vdso_base_.load(std::memory_order_relaxed) == kInvalidBase) {
70 errno = 0;
71 const void *const sysinfo_ehdr =
72 reinterpret_cast<const void *>(getauxval(AT_SYSINFO_EHDR));
73 if (errno == 0) {
74 vdso_base_.store(sysinfo_ehdr, std::memory_order_relaxed);
75 }
76 }
77 #endif // __GLIBC_PREREQ(2, 16)
78 if (vdso_base_.load(std::memory_order_relaxed) == kInvalidBase) {
79 int fd = open("/proc/self/auxv", O_RDONLY);
80 if (fd == -1) {
81 // Kernel too old to have a VDSO.
82 vdso_base_.store(nullptr, std::memory_order_relaxed);
83 getcpu_fn_.store(&GetCPUViaSyscall, std::memory_order_relaxed);
84 return nullptr;
85 }
86 ElfW(auxv_t) aux;
87 while (read(fd, &aux, sizeof(aux)) == sizeof(aux)) {
88 if (aux.a_type == AT_SYSINFO_EHDR) {
89 vdso_base_.store(reinterpret_cast<void *>(aux.a_un.a_val),
90 std::memory_order_relaxed);
91 break;
92 }
93 }
94 close(fd);
95 if (vdso_base_.load(std::memory_order_relaxed) == kInvalidBase) {
96 // Didn't find AT_SYSINFO_EHDR in auxv[].
97 vdso_base_.store(nullptr, std::memory_order_relaxed);
98 }
99 }
100 GetCpuFn fn = &GetCPUViaSyscall; // default if VDSO not present.
101 if (vdso_base_.load(std::memory_order_relaxed)) {
102 VDSOSupport vdso;
103 SymbolInfo info;
104 if (vdso.LookupSymbol("__vdso_getcpu", "LINUX_2.6", STT_FUNC, &info)) {
105 fn = reinterpret_cast<GetCpuFn>(const_cast<void *>(info.address));
106 }
107 }
108 // Subtle: this code runs outside of any locks; prevent compiler
109 // from assigning to getcpu_fn_ more than once.
110 getcpu_fn_.store(fn, std::memory_order_relaxed);
111 return vdso_base_.load(std::memory_order_relaxed);
112 }
113
SetBase(const void * base)114 const void *VDSOSupport::SetBase(const void *base) {
115 ABSL_RAW_CHECK(base != debugging_internal::ElfMemImage::kInvalidBase,
116 "internal error");
117 const void *old_base = vdso_base_.load(std::memory_order_relaxed);
118 vdso_base_.store(base, std::memory_order_relaxed);
119 image_.Init(base);
120 // Also reset getcpu_fn_, so GetCPU could be tested with simulated VDSO.
121 getcpu_fn_.store(&InitAndGetCPU, std::memory_order_relaxed);
122 return old_base;
123 }
124
LookupSymbol(const char * name,const char * version,int type,SymbolInfo * info) const125 bool VDSOSupport::LookupSymbol(const char *name,
126 const char *version,
127 int type,
128 SymbolInfo *info) const {
129 return image_.LookupSymbol(name, version, type, info);
130 }
131
LookupSymbolByAddress(const void * address,SymbolInfo * info_out) const132 bool VDSOSupport::LookupSymbolByAddress(const void *address,
133 SymbolInfo *info_out) const {
134 return image_.LookupSymbolByAddress(address, info_out);
135 }
136
137 // NOLINT on 'long' because this routine mimics kernel api.
GetCPUViaSyscall(unsigned * cpu,void *,void *)138 long VDSOSupport::GetCPUViaSyscall(unsigned *cpu, // NOLINT(runtime/int)
139 void *, void *) {
140 #ifdef SYS_getcpu
141 return syscall(SYS_getcpu, cpu, nullptr, nullptr);
142 #else
143 // x86_64 never implemented sys_getcpu(), except as a VDSO call.
144 static_cast<void>(cpu); // Avoid an unused argument compiler warning.
145 errno = ENOSYS;
146 return -1;
147 #endif
148 }
149
150 // Use fast __vdso_getcpu if available.
InitAndGetCPU(unsigned * cpu,void * x,void * y)151 long VDSOSupport::InitAndGetCPU(unsigned *cpu, // NOLINT(runtime/int)
152 void *x, void *y) {
153 Init();
154 GetCpuFn fn = getcpu_fn_.load(std::memory_order_relaxed);
155 ABSL_RAW_CHECK(fn != &InitAndGetCPU, "Init() did not set getcpu_fn_");
156 return (*fn)(cpu, x, y);
157 }
158
159 // This function must be very fast, and may be called from very
160 // low level (e.g. tcmalloc). Hence I avoid things like
161 // GoogleOnceInit() and ::operator new.
162 ABSL_ATTRIBUTE_NO_SANITIZE_MEMORY
GetCPU()163 int GetCPU() {
164 unsigned cpu;
165 int ret_code = (*VDSOSupport::getcpu_fn_)(&cpu, nullptr, nullptr);
166 return ret_code == 0 ? cpu : ret_code;
167 }
168
169 } // namespace debugging_internal
170 ABSL_NAMESPACE_END
171 } // namespace absl
172
173 #endif // ABSL_HAVE_VDSO_SUPPORT
174