1 /*
2 * Copyright (c) 2023 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15 #ifndef BLOCKAWARE_H
16 #define BLOCKAWARE_H
17
18 #include <securec.h>
19 #include <cstdio>
20 #include <cstring>
21 #include <sys/prctl.h>
22 #include <cerrno>
23 #include "dfx/log/ffrt_log_api.h"
24
25 #ifdef __cplusplus
26 extern "C" {
27 #endif
28
29 #define BLOCKAWARE_DOMAIN_ID_MAX 15
30 #define HM_PR_SILK_BLOCKAWARE_OPS 0x534b4241
31 #define BLOCKAWARE_SUBOPS_INIT 0x1
32 #define BLOCKAWARE_SUBOPS_REG 0x2
33 #define BLOCKAWARE_SUBOPS_UNREG 0x3
34 #define BLOCKAWARE_SUBOPS_WAIT 0x4
35 #define BLOCKAWARE_SUBOPS_WAKE 0x5
36 #define BLOCKAWARE_SUBOPS_MONITORFD 0X6
37
38 struct BlockawareDomainInfo {
39 unsigned int nrRunning;
40 unsigned int nrSleeping;
41 unsigned int nrBlocked;
42 };
43 struct BlockawareDomainInfoArea {
44 struct BlockawareDomainInfo localinfo[BLOCKAWARE_DOMAIN_ID_MAX + 1];
45 struct BlockawareDomainInfo globalinfo;
46 };
47 struct BlockawareWatermark {
48 unsigned int low;
49 unsigned int high;
50 };
51 struct BlockawareWakeupCond {
52 struct BlockawareWatermark local[BLOCKAWARE_DOMAIN_ID_MAX + 1];
53 struct BlockawareWatermark global;
54 bool check_ahead;
55 };
56 struct BlockawareKinfoPageS {
57 uint32_t seq;
58 struct BlockawareDomainInfoArea infoArea;
59 };
60
61 static inline int BlockawareInit(unsigned long *keyPtr);
62 static inline int BlockawareRegister(unsigned int domain);
63 static inline int BlockawareUnregister(void);
64 static int BlockawareLoadSnapshot(unsigned long key, struct BlockawareDomainInfoArea *infoArea);
65 static inline int BlockawareEnterSleeping(void);
66 static inline int BlockawareLeaveSleeping(void);
67 static inline int BlockawareWaitCond(struct BlockawareWakeupCond *cond);
68 static inline int BlockawareWake(void);
69 static inline int BlockawareMonitorfd(int fd, struct BlockawareWakeupCond *cond);
70
71 #ifdef __aarch64__
CpuRelax(void)72 static inline void CpuRelax(void)
73 {
74 asm volatile("yield" ::: "memory");
75 }
76
SmpRmb(void)77 static inline void SmpRmb(void)
78 {
79 asm volatile("dmb ishld" ::: "memory");
80 }
81
GetTlsPtr(void)82 static inline unsigned long GetTlsPtr(void)
83 {
84 unsigned long tls = 0;
85 asm volatile ("mrs %0, tpidr_el0\n" : "=r" (tls));
86 return tls;
87 }
88
curr_thread_tls_blockaware_slot_of(void)89 static inline unsigned long *curr_thread_tls_blockaware_slot_of(void)
90 {
91 unsigned long tls = GetTlsPtr();
92 unsigned long slot_addr = tls - sizeof (unsigned long) * (2UL + 5UL);
93 return reinterpret_cast<unsigned long *>(slot_addr);
94 }
95
BlockawareEnterSleeping(void)96 static inline int BlockawareEnterSleeping(void)
97 {
98 unsigned long *slot_ptr = curr_thread_tls_blockaware_slot_of();
99 *slot_ptr += 1;
100 return 0;
101 }
102
BlockawareLeaveSleeping(void)103 static inline int BlockawareLeaveSleeping(void)
104 {
105 unsigned long *slot_ptr = curr_thread_tls_blockaware_slot_of();
106 int err = 0;
107
108 if (*slot_ptr == 0) {
109 err = -EINVAL;
110 } else {
111 *slot_ptr -= 1;
112 }
113
114 return err;
115 }
116 #elif defined(__arm__)
117
CpuRelax(void)118 static inline void CpuRelax(void)
119 {
120 asm volatile("yield" ::: "memory");
121 }
122
SmpRmb(void)123 static inline void SmpRmb(void)
124 {
125 asm volatile("dmb ish" ::: "memory");
126 }
127
GetTlsPtr(void)128 static inline unsigned long GetTlsPtr(void)
129 {
130 unsigned long tpid = 0;
131 asm volatile("mrc p15, 0, %0, c13, c0, 3" : "=r"(tpid));
132 return tpid;
133 }
134
curr_thread_tls_blockaware_slot_of(void)135 static inline unsigned long *curr_thread_tls_blockaware_slot_of(void)
136 {
137 unsigned long tls = GetTlsPtr();
138 unsigned long slot_addr = tls - sizeof (unsigned long) * (2UL + 5UL);
139 return (unsigned long *)slot_addr;
140 }
141
BlockawareEnterSleeping(void)142 static inline int BlockawareEnterSleeping(void)
143 {
144 unsigned long *slot_ptr = curr_thread_tls_blockaware_slot_of();
145 *slot_ptr += 1;
146 return 0;
147 }
148
BlockawareLeaveSleeping(void)149 static inline int BlockawareLeaveSleeping(void)
150 {
151 unsigned long *slot_ptr = curr_thread_tls_blockaware_slot_of();
152 int err = 0;
153
154 if (*slot_ptr == 0) {
155 err = -EINVAL;
156 } else {
157 *slot_ptr -= 1;
158 }
159
160 return err;
161 }
162 #else
CpuRelax(void)163 static inline void CpuRelax(void)
164 {
165 }
166
SmpRmb(void)167 static inline void SmpRmb(void)
168 {
169 }
170
GetTlsPtr(void)171 static inline unsigned long GetTlsPtr(void)
172 {
173 return 0;
174 }
175
BlockawareEnterSleeping(void)176 static inline int BlockawareEnterSleeping(void)
177 {
178 return 0;
179 }
180
BlockawareLeaveSleeping(void)181 static inline int BlockawareLeaveSleeping(void)
182 {
183 return 0;
184 }
185 #endif
186
BlockawareInit(unsigned long * keyPtr)187 static inline int BlockawareInit(unsigned long *keyPtr)
188 {
189 int rc = prctl(HM_PR_SILK_BLOCKAWARE_OPS, BLOCKAWARE_SUBOPS_INIT, reinterpret_cast<unsigned long>(keyPtr));
190 return (rc == 0) ? 0 : errno;
191 }
192
BlockawareRegister(unsigned int domain)193 static inline int BlockawareRegister(unsigned int domain)
194 {
195 /* Mention that it is kernel's responsibility to init tls slot to 0 */
196 int rc = prctl(HM_PR_SILK_BLOCKAWARE_OPS, BLOCKAWARE_SUBOPS_REG, static_cast<unsigned long>(domain));
197 return (rc == 0) ? 0 : errno;
198 }
199
BlockawareUnregister(void)200 static inline int BlockawareUnregister(void)
201 {
202 /* Mention that it is kernel's responsibility to reset tls slot to 0 */
203 int rc = prctl(HM_PR_SILK_BLOCKAWARE_OPS, BLOCKAWARE_SUBOPS_UNREG);
204 return (rc == 0) ? 0 : errno;
205 }
206
seqlock_start_read(const uint32_t * seq_ptr)207 static inline uint32_t seqlock_start_read(const uint32_t *seq_ptr)
208 {
209 uint32_t seq;
210 do {
211 seq = *reinterpret_cast<const volatile uint32_t *>(seq_ptr);
212 if ((seq & 1U) == 0U) {
213 break;
214 }
215 CpuRelax();
216 } while (true);
217 SmpRmb();
218 return seq;
219 }
220
seqlock_check(const uint32_t * seq_ptr,uint32_t seq_prev)221 static inline bool seqlock_check(const uint32_t *seq_ptr, uint32_t seq_prev)
222 {
223 SmpRmb();
224 return (*seq_ptr == seq_prev);
225 }
226
BlockawareLoadSnapshot(unsigned long key,struct BlockawareDomainInfoArea * infoArea)227 static int BlockawareLoadSnapshot(unsigned long key, struct BlockawareDomainInfoArea *infoArea)
228 {
229 struct BlockawareKinfoPageS *kinfoPage = reinterpret_cast<struct BlockawareKinfoPageS *>(key);
230 uint32_t seq;
231 int ret = 0;
232 do {
233 seq = seqlock_start_read(&kinfoPage->seq);
234 ret = memcpy_s(infoArea, sizeof(BlockawareDomainInfoArea),
235 &kinfoPage->infoArea, sizeof(BlockawareDomainInfoArea));
236 } while (!seqlock_check(&kinfoPage->seq, seq));
237 if (ret != EOK) {
238 FFRT_SYSEVENT_LOGE("The memcpy operation failed for the infoArea.");
239 }
240 return ret;
241 }
242
BlockawareLoadSnapshotNrRunningFast(unsigned long key,int domainId)243 static inline unsigned int BlockawareLoadSnapshotNrRunningFast(unsigned long key, int domainId)
244 {
245 BlockawareKinfoPageS* kinfoPage = reinterpret_cast<BlockawareKinfoPageS*>(key);
246 return kinfoPage->infoArea.localinfo[domainId].nrRunning;
247 }
248
BlockawareLoadSnapshotNrBlockedFast(unsigned long key,int domainId)249 static inline unsigned int BlockawareLoadSnapshotNrBlockedFast(unsigned long key, int domainId)
250 {
251 BlockawareKinfoPageS* kinfoPage = reinterpret_cast<BlockawareKinfoPageS*>(key);
252 return kinfoPage->infoArea.localinfo[domainId].nrBlocked;
253 }
254
BlockawareWaitCond(struct BlockawareWakeupCond * cond)255 static inline int BlockawareWaitCond(struct BlockawareWakeupCond *cond)
256 {
257 int rc = prctl(HM_PR_SILK_BLOCKAWARE_OPS, BLOCKAWARE_SUBOPS_WAIT, reinterpret_cast<unsigned long>(cond));
258 return (rc == 0) ? 0 : errno;
259 }
260
BlockawareWake(void)261 static inline int BlockawareWake(void)
262 {
263 int rc = prctl(HM_PR_SILK_BLOCKAWARE_OPS, BLOCKAWARE_SUBOPS_WAKE);
264 return (rc == 0) ? 0 : errno;
265 }
266
BlockawareMonitorfd(int fd,struct BlockawareWakeupCond * cond)267 static inline int BlockawareMonitorfd(int fd, struct BlockawareWakeupCond *cond)
268 {
269 int rc = prctl(HM_PR_SILK_BLOCKAWARE_OPS, BLOCKAWARE_SUBOPS_MONITORFD,
270 static_cast<unsigned long>(fd), reinterpret_cast<unsigned long>(cond));
271 return (rc >= 0) ? rc : -errno;
272 }
273
274 #ifdef __cplusplus
275 }
276 #endif
277 #endif /* BLOCKAWARE_H */