1 // SPDX-License-Identifier: LGPL-2.1
2 /*
3 * rseq.c
4 *
5 * Copyright (C) 2016 Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
6 *
7 * This library is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU Lesser General Public
9 * License as published by the Free Software Foundation; only
10 * version 2.1 of the License.
11 *
12 * This library is distributed in the hope that it will be useful,
13 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15 * Lesser General Public License for more details.
16 */
17
18 #define _GNU_SOURCE
19 #include <errno.h>
20 #include <sched.h>
21 #include <stdio.h>
22 #include <stdlib.h>
23 #include <string.h>
24 #include <unistd.h>
25 #include <syscall.h>
26 #include <assert.h>
27 #include <signal.h>
28 #include <limits.h>
29 #include <dlfcn.h>
30 #include <stddef.h>
31
32 #include <linux/compiler.h>
33
34 #include "../kselftest.h"
35 #include "rseq.h"
36
37 /*
38 * Define weak versions to play nice with binaries that are statically linked
39 * against a libc that doesn't support registering its own rseq.
40 */
41 __weak ptrdiff_t __rseq_offset;
42 __weak unsigned int __rseq_size;
43 __weak unsigned int __rseq_flags;
44
45 static const ptrdiff_t *libc_rseq_offset_p = &__rseq_offset;
46 static const unsigned int *libc_rseq_size_p = &__rseq_size;
47 static const unsigned int *libc_rseq_flags_p = &__rseq_flags;
48
49 /* Offset from the thread pointer to the rseq area. */
50 ptrdiff_t rseq_offset;
51
52 /* Size of the registered rseq area. 0 if the registration was
53 unsuccessful. */
54 unsigned int rseq_size = -1U;
55
56 /* Flags used during rseq registration. */
57 unsigned int rseq_flags;
58
59 static int rseq_ownership;
60
61 static
62 __thread struct rseq_abi __rseq_abi __attribute__((tls_model("initial-exec"))) = {
63 .cpu_id = RSEQ_ABI_CPU_ID_UNINITIALIZED,
64 };
65
sys_rseq(struct rseq_abi * rseq_abi,uint32_t rseq_len,int flags,uint32_t sig)66 static int sys_rseq(struct rseq_abi *rseq_abi, uint32_t rseq_len,
67 int flags, uint32_t sig)
68 {
69 return syscall(__NR_rseq, rseq_abi, rseq_len, flags, sig);
70 }
71
rseq_available(void)72 int rseq_available(void)
73 {
74 int rc;
75
76 rc = sys_rseq(NULL, 0, 0, 0);
77 if (rc != -1)
78 abort();
79 switch (errno) {
80 case ENOSYS:
81 return 0;
82 case EINVAL:
83 return 1;
84 default:
85 abort();
86 }
87 }
88
rseq_register_current_thread(void)89 int rseq_register_current_thread(void)
90 {
91 int rc;
92
93 if (!rseq_ownership) {
94 /* Treat libc's ownership as a successful registration. */
95 return 0;
96 }
97 rc = sys_rseq(&__rseq_abi, sizeof(struct rseq_abi), 0, RSEQ_SIG);
98 if (rc)
99 return -1;
100 assert(rseq_current_cpu_raw() >= 0);
101 return 0;
102 }
103
rseq_unregister_current_thread(void)104 int rseq_unregister_current_thread(void)
105 {
106 int rc;
107
108 if (!rseq_ownership) {
109 /* Treat libc's ownership as a successful unregistration. */
110 return 0;
111 }
112 rc = sys_rseq(&__rseq_abi, sizeof(struct rseq_abi), RSEQ_ABI_FLAG_UNREGISTER, RSEQ_SIG);
113 if (rc)
114 return -1;
115 return 0;
116 }
117
118 static __attribute__((constructor))
rseq_init(void)119 void rseq_init(void)
120 {
121 /*
122 * If the libc's registered rseq size isn't already valid, it may be
123 * because the binary is dynamically linked and not necessarily due to
124 * libc not having registered a restartable sequence. Try to find the
125 * symbols if that's the case.
126 */
127 if (!*libc_rseq_size_p) {
128 libc_rseq_offset_p = dlsym(RTLD_NEXT, "__rseq_offset");
129 libc_rseq_size_p = dlsym(RTLD_NEXT, "__rseq_size");
130 libc_rseq_flags_p = dlsym(RTLD_NEXT, "__rseq_flags");
131 }
132 if (libc_rseq_size_p && libc_rseq_offset_p && libc_rseq_flags_p &&
133 *libc_rseq_size_p != 0) {
134 /* rseq registration owned by glibc */
135 rseq_offset = *libc_rseq_offset_p;
136 rseq_size = *libc_rseq_size_p;
137 rseq_flags = *libc_rseq_flags_p;
138 return;
139 }
140 if (!rseq_available())
141 return;
142 rseq_ownership = 1;
143 rseq_offset = (void *)&__rseq_abi - rseq_thread_pointer();
144 rseq_size = sizeof(struct rseq_abi);
145 rseq_flags = 0;
146 }
147
148 static __attribute__((destructor))
rseq_exit(void)149 void rseq_exit(void)
150 {
151 if (!rseq_ownership)
152 return;
153 rseq_offset = 0;
154 rseq_size = -1U;
155 rseq_ownership = 0;
156 }
157
rseq_fallback_current_cpu(void)158 int32_t rseq_fallback_current_cpu(void)
159 {
160 int32_t cpu;
161
162 cpu = sched_getcpu();
163 if (cpu < 0) {
164 perror("sched_getcpu()");
165 abort();
166 }
167 return cpu;
168 }
169