1diff --git a/include/libunwind_i.h b/include/libunwind_i.h 2index 97422679..bc529810 100644 3--- a/include/libunwind_i.h 4+++ b/include/libunwind_i.h 5@@ -167,6 +167,7 @@ target_is_big_endian() 6 #define UNWI_ARCH_OBJ(fn) UNW_PASTE(UNW_PASTE(UNW_PASTE(_UI,UNW_TARGET),_), fn) 7 8 #define unwi_full_mask UNWI_ARCH_OBJ(full_mask) 9+#define NO_SANITIZE __attribute__((no_sanitize("address"), no_sanitize("hwaddress"))) 10 11 /* Type of a mask that can be used to inhibit preemption. At the 12 userlevel, preemption is caused by signals and hence sigset_t is 13diff --git a/include/tdep-aarch64/libunwind_i.h b/include/tdep-aarch64/libunwind_i.h 14index d96833a2..3d57587b 100644 15--- a/include/tdep-aarch64/libunwind_i.h 16+++ b/include/tdep-aarch64/libunwind_i.h 17@@ -148,8 +148,7 @@ dwarf_get (struct dwarf_cursor *c, dwarf_loc_t loc, unw_word_t *val) 18 { 19 if (!DWARF_GET_LOC (loc)) 20 return -1; 21- *val = *(unw_word_t *) DWARF_GET_LOC (loc); 22- return 0; 23+ return (*c->as->acc.access_mem) (c->as, DWARF_GET_LOC (loc), val, 0, c->as_arg); 24 } 25 26 static inline int 27@@ -157,8 +156,7 @@ dwarf_put (struct dwarf_cursor *c, dwarf_loc_t loc, unw_word_t val) 28 { 29 if (!DWARF_GET_LOC (loc)) 30 return -1; 31- *(unw_word_t *) DWARF_GET_LOC (loc) = val; 32- return 0; 33+ return (*c->as->acc.access_mem) (c->as, DWARF_GET_LOC (loc), &val, 1, c->as_arg); 34 } 35 36 #else /* !UNW_LOCAL_ONLY */ 37diff --git a/include/tdep-arm/libunwind_i.h b/include/tdep-arm/libunwind_i.h 38index 88ebfb06..c19ed523 100644 39--- a/include/tdep-arm/libunwind_i.h 40+++ b/include/tdep-arm/libunwind_i.h 41@@ -131,8 +131,7 @@ dwarf_get (struct dwarf_cursor *c, dwarf_loc_t loc, unw_word_t *val) 42 { 43 if (!DWARF_GET_LOC (loc)) 44 return -1; 45- *val = *(unw_word_t *) DWARF_GET_LOC (loc); 46- return 0; 47+ return (*c->as->acc.access_mem) (c->as, DWARF_GET_LOC (loc), val, 0, c->as_arg); 48 } 49 50 static inline int 51@@ -140,8 +139,7 @@ dwarf_put (struct dwarf_cursor *c, dwarf_loc_t loc, unw_word_t val) 52 { 53 if (!DWARF_GET_LOC (loc)) 54 return -1; 55- *(unw_word_t *) DWARF_GET_LOC (loc) = val; 56- return 0; 57+ return (*c->as->acc.access_mem) (c->as, DWARF_GET_LOC (loc), &val, 1, c->as_arg); 58 } 59 60 #else /* !UNW_LOCAL_ONLY */ 61@@ -254,6 +252,7 @@ dwarf_put (struct dwarf_cursor *c, dwarf_loc_t loc, unw_word_t val) 62 63 #define tdep_getcontext_trace unw_getcontext 64 #define tdep_init_done UNW_OBJ(init_done) 65+#define tdep_init_mem_validate UNW_OBJ(init_mem_validate) 66 #define tdep_init UNW_OBJ(init) 67 #define arm_find_proc_info UNW_OBJ(find_proc_info) 68 #define arm_put_unwind_info UNW_OBJ(put_unwind_info) 69@@ -294,6 +293,7 @@ dwarf_put (struct dwarf_cursor *c, dwarf_loc_t loc, unw_word_t val) 70 extern atomic_bool tdep_init_done; 71 72 extern void tdep_init (void); 73+extern void tdep_init_mem_validate (void); 74 extern int arm_find_proc_info (unw_addr_space_t as, unw_word_t ip, 75 unw_proc_info_t *pi, int need_unwind_info, 76 void *arg); 77diff --git a/src/aarch64/Ginit.c b/src/aarch64/Ginit.c 78index 2b08feb3..58ff370a 100644 79--- a/src/aarch64/Ginit.c 80+++ b/src/aarch64/Ginit.c 81@@ -25,9 +25,11 @@ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 82 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ 83 84 #include <errno.h> 85+#include <pthread.h> 86 #include <stdlib.h> 87 #include <string.h> 88 #include <sys/mman.h> 89+#include <sys/syscall.h> 90 #include <stdatomic.h> 91 92 #include "unwind_i.h" 93@@ -88,7 +90,7 @@ get_dyn_info_list_addr (unw_addr_space_t as, unw_word_t *dyn_info_list_addr, 94 #define PAGE_START(a) ((a) & ~(PAGE_SIZE-1)) 95 96 static int mem_validate_pipe[2] = {-1, -1}; 97- 98+pthread_mutex_t g_mutex; 99 #ifdef HAVE_PIPE2 100 static inline void 101 do_pipe2 (int pipefd[2]) 102@@ -129,13 +131,11 @@ open_pipe (void) 103 do_pipe2 (mem_validate_pipe); 104 } 105 106-ALWAYS_INLINE 107-static int 108-write_validate (void *addr) 109+ALWAYS_INLINE NO_SANITIZE static int write_validate (void *addr) 110 { 111 int ret = -1; 112 ssize_t bytes = 0; 113- 114+ pthread_mutex_lock(&g_mutex); 115 do 116 { 117 char buf; 118@@ -152,21 +152,17 @@ write_validate (void *addr) 119 120 do 121 { 122- ret = write (mem_validate_pipe[1], addr, 1); 123+ /* use syscall insteadof write() so that ASAN does not complain */ 124+ ret = syscall (SYS_write, mem_validate_pipe[1], addr, 1); 125 } 126 while ( errno == EINTR ); 127- 128+ pthread_mutex_unlock(&g_mutex); 129 return ret; 130 } 131 132 static int (*mem_validate_func) (void *addr, size_t len); 133 static int msync_validate (void *addr, size_t len) 134 { 135- if (msync (addr, len, MS_ASYNC) != 0) 136- { 137- return -1; 138- } 139- 140 return write_validate (addr); 141 } 142 143@@ -316,8 +312,7 @@ validate_mem (unw_word_t addr) 144 return 0; 145 } 146 147-static int 148-access_mem (unw_addr_space_t as, unw_word_t addr, unw_word_t *val, int write, 149+NO_SANITIZE static int access_mem (unw_addr_space_t as, unw_word_t addr, unw_word_t *val, int write, 150 void *arg) 151 { 152 if (unlikely (write)) 153@@ -329,8 +324,7 @@ access_mem (unw_addr_space_t as, unw_word_t addr, unw_word_t *val, int write, 154 { 155 /* validate address */ 156 const struct cursor *c = (const struct cursor *)arg; 157- if (likely (c != NULL) && unlikely (c->validate) 158- && unlikely (validate_mem (addr))) { 159+ if (unlikely (validate_mem (addr))) { 160 Debug (16, "mem[%016lx] -> invalid\n", addr); 161 return -1; 162 } 163diff --git a/src/arm/Gglobal.c b/src/arm/Gglobal.c 164index 0700f930..efb3cce5 100644 165--- a/src/arm/Gglobal.c 166+++ b/src/arm/Gglobal.c 167@@ -61,6 +61,7 @@ tdep_init (void) 168 dwarf_init (); 169 170 #ifndef UNW_REMOTE_ONLY 171+ tdep_init_mem_validate (); 172 arm_local_addr_space_init (); 173 #endif 174 atomic_store(&tdep_init_done, 1); /* signal that we're initialized... */ 175diff --git a/src/arm/Ginit.c b/src/arm/Ginit.c 176index 0bac0d72..0c67a62c 100644 177--- a/src/arm/Ginit.c 178+++ b/src/arm/Ginit.c 179@@ -22,8 +22,13 @@ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 180 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 181 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ 182 183+#include <errno.h> 184+#include <pthread.h> 185 #include <stdlib.h> 186 #include <string.h> 187+#include <sys/mman.h> 188+#include <sys/syscall.h> 189+#include <stdatomic.h> 190 191 #include "unwind_i.h" 192 193@@ -74,41 +79,161 @@ get_dyn_info_list_addr (unw_addr_space_t as, unw_word_t *dyn_info_list_addr, 194 #define PAGE_SIZE 4096 195 #define PAGE_START(a) ((a) & ~(PAGE_SIZE-1)) 196 197-/* Cache of already validated addresses */ 198-#define NLGA 4 199-static unw_word_t last_good_addr[NLGA]; 200-static int lga_victim; 201+static int mem_validate_pipe[2] = {-1, -1}; 202+pthread_mutex_t g_mutex; 203+#ifdef HAVE_PIPE2 204+static inline void 205+do_pipe2 (int pipefd[2]) 206+{ 207+ pipe2 (pipefd, O_CLOEXEC | O_NONBLOCK); 208+} 209+#else 210+static inline void 211+set_pipe_flags (int fd) 212+{ 213+ int fd_flags = fcntl (fd, F_GETFD, 0); 214+ int status_flags = fcntl (fd, F_GETFL, 0); 215 216-static int 217-validate_mem (unw_word_t addr) 218+ fd_flags |= FD_CLOEXEC; 219+ fcntl (fd, F_SETFD, fd_flags); 220+ 221+ status_flags |= O_NONBLOCK; 222+ fcntl (fd, F_SETFL, status_flags); 223+} 224+ 225+static inline void 226+do_pipe2 (int pipefd[2]) 227 { 228- int i, victim; 229- size_t len; 230+ pipe (pipefd); 231+ set_pipe_flags(pipefd[0]); 232+ set_pipe_flags(pipefd[1]); 233+} 234+#endif 235 236- if (PAGE_START(addr + sizeof (unw_word_t) - 1) == PAGE_START(addr)) 237- len = PAGE_SIZE; 238- else 239- len = PAGE_SIZE * 2; 240+static inline void 241+open_pipe (void) 242+{ 243+ if (mem_validate_pipe[0] != -1) 244+ close (mem_validate_pipe[0]); 245+ if (mem_validate_pipe[1] != -1) 246+ close (mem_validate_pipe[1]); 247 248- addr = PAGE_START(addr); 249+ do_pipe2 (mem_validate_pipe); 250+} 251 252- if (addr == 0) 253+ALWAYS_INLINE NO_SANITIZE static int write_validate (void *addr) 254+{ 255+ int ret = -1; 256+ ssize_t bytes = 0; 257+ pthread_mutex_lock(&g_mutex); 258+ 259+ do 260+ { 261+ char buf; 262+ bytes = syscall(SYS_read, mem_validate_pipe[0], &buf, 1); 263+ } 264+ while ( errno == EINTR ); 265+ 266+ int valid_read = (bytes > 0 || errno == EAGAIN || errno == EWOULDBLOCK); 267+ if (!valid_read) 268+ { 269+ // re-open closed pipe 270+ open_pipe (); 271+ } 272+ 273+ do 274+ { 275+ ret = syscall(SYS_write, mem_validate_pipe[1], addr, 1); 276+ } 277+ while ( errno == EINTR ); 278+ pthread_mutex_unlock(&g_mutex); 279+ return ret; 280+} 281+ 282+static int (*mem_validate_func) (void *addr, size_t len); 283+static int msync_validate (void *addr, size_t len) 284+{ 285+ if (msync (addr, len, MS_ASYNC) != 0) 286+ { 287+ return -1; 288+ } 289+ 290+ return write_validate (addr); 291+} 292+ 293+#ifdef HAVE_MINCORE 294+static int mincore_validate (void *addr, size_t len) 295+{ 296+ unsigned char mvec[2]; /* Unaligned access may cross page boundary */ 297+ 298+ /* mincore could fail with EAGAIN but we conservatively return -1 299+ instead of looping. */ 300+ if (mincore (addr, len, (unsigned char *)mvec) != 0) 301+ { 302 return -1; 303+ } 304 305+ return write_validate (addr); 306+} 307+#endif 308+ 309+/* Initialise memory validation method. On linux kernels <2.6.21, 310+ mincore() returns incorrect value for MAP_PRIVATE mappings, 311+ such as stacks. If mincore() was available at compile time, 312+ check if we can actually use it. If not, use msync() instead. */ 313+HIDDEN void 314+tdep_init_mem_validate (void) 315+{ 316+ open_pipe (); 317+ 318+#ifdef HAVE_MINCORE 319+ unsigned char present = 1; 320+ unw_word_t addr = PAGE_START((unw_word_t)&present); 321+ unsigned char mvec[1]; 322+ int ret; 323+ while ((ret = mincore ((void*)addr, PAGE_SIZE, (unsigned char *)mvec)) == -1 && 324+ errno == EAGAIN) {} 325+ if (ret == 0) 326+ { 327+ Debug(1, "using mincore to validate memory\n"); 328+ mem_validate_func = mincore_validate; 329+ } 330+ else 331+#endif 332+ { 333+ Debug(1, "using msync to validate memory\n"); 334+ mem_validate_func = msync_validate; 335+ } 336+} 337+ 338+/* Cache of already validated addresses */ 339+#define NLGA 4 340+#if defined(HAVE___CACHE_PER_THREAD) && HAVE___CACHE_PER_THREAD 341+// thread-local variant 342+static _Thread_local unw_word_t last_good_addr[NLGA]; 343+static _Thread_local int lga_victim; 344+ 345+static int 346+is_cached_valid_mem(unw_word_t addr) 347+{ 348+ int i; 349 for (i = 0; i < NLGA; i++) 350 { 351- if (last_good_addr[i] && (addr == last_good_addr[i])) 352+ if (addr == last_good_addr[i]) 353+ return 1; 354+ } 355 return 0; 356 } 357 358- if (msync ((void *) addr, len, MS_ASYNC) == -1) 359- return -1; 360- 361+static void 362+cache_valid_mem(unw_word_t addr) 363+{ 364+ int i, victim; 365 victim = lga_victim; 366 for (i = 0; i < NLGA; i++) { 367- if (!last_good_addr[victim]) { 368- last_good_addr[victim++] = addr; 369- return 0; 370+ if (last_good_addr[victim] == 0) { 371+ last_good_addr[victim] = addr; 372+ return; 373 } 374 victim = (victim + 1) % NLGA; 375 } 376@@ -117,12 +242,72 @@ validate_mem (unw_word_t addr) 377 last_good_addr[victim] = addr; 378 victim = (victim + 1) % NLGA; 379 lga_victim = victim; 380+} 381+ 382+#else 383+// global, thread safe variant 384+static _Atomic unw_word_t last_good_addr[NLGA]; 385+static _Atomic int lga_victim; 386 387+static int 388+is_cached_valid_mem(unw_word_t addr) 389+{ 390+ int i; 391+ for (i = 0; i < NLGA; i++) 392+ { 393+ if (addr == atomic_load(&last_good_addr[i])) 394+ return 1; 395+ } 396 return 0; 397 } 398 399+static void 400+cache_valid_mem(unw_word_t addr) 401+{ 402+ int i, victim; 403+ victim = atomic_load(&lga_victim); 404+ unw_word_t zero = 0; 405+ for (i = 0; i < NLGA; i++) { 406+ if (atomic_compare_exchange_strong(&last_good_addr[victim], &zero, addr)) { 407+ return; 408+ } 409+ victim = (victim + 1) % NLGA; 410+ } 411+ 412+ /* All slots full. Evict the victim. */ 413+ atomic_store(&last_good_addr[victim], addr); 414+ victim = (victim + 1) % NLGA; 415+ atomic_store(&lga_victim, victim); 416+} 417+#endif 418+ 419 static int 420-access_mem (unw_addr_space_t as, unw_word_t addr, unw_word_t *val, int write, 421+validate_mem (unw_word_t addr) 422+{ 423+ size_t len; 424+ 425+ if (PAGE_START(addr + sizeof (unw_word_t) - 1) == PAGE_START(addr)) 426+ len = PAGE_SIZE; 427+ else 428+ len = PAGE_SIZE * 2; 429+ 430+ addr = PAGE_START(addr); 431+ 432+ if (addr == 0) 433+ return -1; 434+ 435+ if (is_cached_valid_mem(addr)) 436+ return 0; 437+ 438+ if (mem_validate_func ((void *) addr, len) == -1) 439+ return -1; 440+ 441+ cache_valid_mem(addr); 442+ 443+ return 0; 444+} 445+ 446+NO_SANITIZE static int access_mem (unw_addr_space_t as, unw_word_t addr, unw_word_t *val, int write, 447 void *arg) 448 { 449 /* validate address */ 450diff --git a/src/arm/Gregs.c b/src/arm/Gregs.c 451index 0d52f0b2..5d704e94 100644 452--- a/src/arm/Gregs.c 453+++ b/src/arm/Gregs.c 454@@ -35,24 +35,39 @@ tdep_access_reg (struct cursor *c, unw_regnum_t reg, unw_word_t *valp, 455 case UNW_ARM_R15: 456 if (write) 457 c->dwarf.ip = *valp; /* update the IP cache */ 458+ /* fall-through */ 459 case UNW_ARM_R0: 460+ /* fall-through */ 461 case UNW_ARM_R1: 462+ /* fall-through */ 463 case UNW_ARM_R2: 464+ /* fall-through */ 465 case UNW_ARM_R3: 466+ /* fall-through */ 467 case UNW_ARM_R4: 468+ /* fall-through */ 469 case UNW_ARM_R5: 470+ /* fall-through */ 471 case UNW_ARM_R6: 472+ /* fall-through */ 473 case UNW_ARM_R7: 474+ /* fall-through */ 475 case UNW_ARM_R8: 476+ /* fall-through */ 477 case UNW_ARM_R9: 478+ /* fall-through */ 479 case UNW_ARM_R10: 480+ /* fall-through */ 481 case UNW_ARM_R11: 482+ /* fall-through */ 483 case UNW_ARM_R12: 484+ /* fall-through */ 485 case UNW_ARM_R14: 486 loc = c->dwarf.loc[reg - UNW_ARM_R0]; 487 break; 488 489 case UNW_ARM_R13: 490+ /* fall-through */ 491 case UNW_ARM_CFA: 492 if (write) 493 return -UNW_EREADONLYREG; 494diff --git a/src/arm/Gtrace.c b/src/arm/Gtrace.c 495index 51fc281d..fb6e9eb3 100644 496--- a/src/arm/Gtrace.c 497+++ b/src/arm/Gtrace.c 498@@ -479,6 +479,7 @@ tdep_trace (unw_cursor_t *cursor, void **buffer, int *size) 499 case UNW_ARM_FRAME_GUESSED: 500 /* Fall thru to standard processing after forcing validation. */ 501 c->validate = 1; 502+ /* fall-through */ 503 504 case UNW_ARM_FRAME_STANDARD: 505 /* Advance standard traceable frame. */ 506