1 /*
2 * Copyright (c) 2023 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include <unistd.h>
17 #ifndef _GNU_SOURCE
18 #define _GNU_SOURCE
19 #endif
20 #include <sys/time.h>
21 #include <sys/syscall.h>
22
23 #include <map>
24 #include <functional>
25 #include <linux/futex.h>
26 #include "sync.h"
27
28 #ifdef NS_PER_SEC
29 #undef NS_PER_SEC
30 #endif
31 namespace ffrt {
DelayedWakeup(const time_point_t & to,WaitEntry * we,const std::function<void (WaitEntry *)> & wakeup)32 bool DelayedWakeup(const time_point_t& to, WaitEntry* we, const std::function<void(WaitEntry*)>& wakeup)
33 {
34 static DelayedWorker w;
35 return w.dispatch(to, we, wakeup);
36 }
37
lock_contended()38 void spin_mutex::lock_contended()
39 {
40 int v = l.load(std::memory_order_relaxed);
41 do {
42 while (v != sync_detail::UNLOCK) {
43 std::this_thread::yield();
44 v = l.load(std::memory_order_relaxed);
45 }
46 } while (!l.compare_exchange_weak(v, sync_detail::LOCK, std::memory_order_acquire, std::memory_order_relaxed));
47 }
48
spin()49 static void spin()
50 {
51 #if defined(__x86_64__)
52 asm volatile("pause");
53 #elif defined(__aarch64__)
54 asm volatile("isb sy");
55 #elif defined(__arm__)
56 asm volatile("yield");
57 #elif defined(__riscv)
58 // https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/arch/riscv/include/asm/vdso/processor.h?h=v6.6#n25
59 asm volatile(".4byte 0x100000F");
60 #endif
61 }
62
lock_contended()63 void fast_mutex::lock_contended()
64 {
65 int v;
66 // lightly contended
67 for (uint32_t n = static_cast<uint32_t>(1 + rand() % 4); n <= 64; n <<= 1) {
68 for (uint32_t i = 0; i < n; ++i) {
69 spin();
70 }
71 v = __atomic_load_n(&l, __ATOMIC_RELAXED);
72 if (v == sync_detail::WAIT) {
73 break;
74 }
75 if (v == sync_detail::UNLOCK) {
76 if (__atomic_compare_exchange_n(&l, &v, sync_detail::LOCK, 0, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED)) {
77 return;
78 }
79 break;
80 }
81 }
82 // heavily contended
83 if (v == sync_detail::WAIT) {
84 syscall(SYS_futex, &l, FUTEX_WAIT_PRIVATE, sync_detail::WAIT, nullptr, nullptr, 0);
85 }
86 while (__atomic_exchange_n(&l, sync_detail::WAIT, __ATOMIC_ACQUIRE) != sync_detail::UNLOCK) {
87 syscall(SYS_futex, &l, FUTEX_WAIT_PRIVATE, sync_detail::WAIT, nullptr, nullptr, 0);
88 }
89 }
90 } // namespace ffrt
91