1 /*
2 * Copyright (c) 2023 Huawei Device Co., Ltd.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "co_routine.h"
17 #include <cstdio>
18 #include <cstdlib>
19 #include <cstring>
20 #include <string>
21 #include "dfx/trace/ffrt_trace.h"
22 #include "core/dependence_manager.h"
23 #include "core/entity.h"
24 #include "sched/scheduler.h"
25 #include "sync/sync.h"
26 #include "util/slab.h"
27 #ifndef _MSC_VER
28 #include <sys/mman.h>
29 #else
30 #define PROT_READ 0x1
31 #define PROT_WRITE 0x2
32 #endif
33 #include "sched/sched_deadline.h"
34 #include "sync/perf_counter.h"
35 #include "sync/io_poller.h"
36 #include "dfx/bbox/bbox.h"
37
38
39 static thread_local CoRoutineEnv* g_CoThreadEnv = nullptr;
40
CoSwitch(CoCtx * from,CoCtx * to)41 static inline void CoSwitch(CoCtx* from, CoCtx* to)
42 {
43 co2_switch_context(from, to);
44 }
45
CoExit(CoRoutine * co)46 static inline void CoExit(CoRoutine* co)
47 {
48 CoSwitch(&co->ctx, &co->thEnv->schCtx);
49 }
50
CoStartEntry(void * arg)51 static inline void CoStartEntry(void* arg)
52 {
53 CoRoutine* co = reinterpret_cast<CoRoutine*>(arg);
54 {
55 FFRT_LOGD("Execute func() task[%lu], name[%s]", co->task->gid, co->task->label.c_str());
56 auto f = reinterpret_cast<ffrt_function_header_t*>(co->task->func_storage);
57 auto exp = ffrt::SkipStatus::SUBMITTED;
58 if (likely(__atomic_compare_exchange_n(&co->task->skipped, &exp, ffrt::SkipStatus::EXECUTED, 0,
59 __ATOMIC_ACQUIRE, __ATOMIC_RELAXED))) {
60 f->exec(f);
61 }
62 f->destroy(f);
63 }
64 FFRT_TASKDONE_MARKER(co->task->gid);
65 co->task->UpdateState(ffrt::TaskState::EXITED);
66 co->status.store(static_cast<int>(CoStatus::CO_UNINITIALIZED));
67 CoExit(co);
68 }
69
CoInitThreadEnv(void)70 static inline void CoInitThreadEnv(void)
71 {
72 g_CoThreadEnv = reinterpret_cast<CoRoutineEnv*>(calloc(1, sizeof(CoRoutineEnv)));
73 CoRoutineEnv* t = g_CoThreadEnv;
74 if (!t) {
75 abort();
76 }
77 }
78
CoSetStackProt(CoRoutine * co,int prot)79 static void CoSetStackProt(CoRoutine* co, int prot)
80 {
81 /* set the attribute of the page table closest to the stack top in the user stack to read-only,
82 * and 1~2 page table space will be wasted
83 */
84 #ifndef _MSC_VER
85 size_t p_size = getpagesize();
86 #else
87 size_t p_size = 4 * 1024;
88 #endif
89 if (CoStackAttr::Instance()->size < p_size * 3) {
90 abort();
91 }
92
93 #ifndef _MSC_VER
94 uint64_t mp = reinterpret_cast<uint64_t>(co->stkMem.stk);
95 mp = (mp + p_size - 1) / p_size * p_size;
96 int ret = mprotect(reinterpret_cast<void *>(static_cast<uintptr_t>(mp)), p_size, PROT_READ);
97 if (ret < 0) {
98 printf("coroutine size:%lu, mp:0x%lx, page_size:%zu,result:%d,prot:%d, err:%d,%s.\n",
99 static_cast<unsigned long>(sizeof(struct CoRoutine)), static_cast<unsigned long>(mp),
100 p_size, ret, prot, errno, strerror(errno));
101 abort();
102 }
103 #endif
104 }
105
AllocNewCoRoutine(void)106 static inline CoRoutine* AllocNewCoRoutine(void)
107 {
108 std::size_t stack_size = CoStackAttr::Instance()->size + sizeof(CoRoutine) - 8;
109 CoRoutine* co = ffrt::QSimpleAllocator<CoRoutine>::allocMem(stack_size);
110 if (co == nullptr) {
111 abort();
112 }
113
114 co->stkMem.size = CoStackAttr::Instance()->size;
115 co->stkMem.magic = STACK_MAGIC;
116 if (CoStackAttr::Instance()->type == CoStackProtectType::CO_STACK_STRONG_PROTECT) {
117 CoSetStackProt(co, PROT_READ);
118 }
119 co->status.store(static_cast<int>(CoStatus::CO_UNINITIALIZED));
120 return co;
121 }
122
CoMemFree(CoRoutine * co)123 static inline void CoMemFree(CoRoutine* co)
124 {
125 if (CoStackAttr::Instance()->type == CoStackProtectType::CO_STACK_STRONG_PROTECT) {
126 CoSetStackProt(co, PROT_WRITE | PROT_READ);
127 }
128 ffrt::QSimpleAllocator<CoRoutine>::freeMem(co);
129 }
130
CoWorkerExit(void)131 void CoWorkerExit(void)
132 {
133 if (g_CoThreadEnv) {
134 if (g_CoThreadEnv->runningCo) {
135 CoMemFree(g_CoThreadEnv->runningCo);
136 g_CoThreadEnv->runningCo = nullptr;
137 }
138 ::free(g_CoThreadEnv);
139 g_CoThreadEnv = nullptr;
140 }
141 }
142
BindNewCoRoutione(ffrt::TaskCtx * task)143 static inline void BindNewCoRoutione(ffrt::TaskCtx* task)
144 {
145 task->coRoutine = g_CoThreadEnv->runningCo;
146 task->coRoutine->task = task;
147 task->coRoutine->thEnv = g_CoThreadEnv;
148 }
149
UnbindCoRoutione(ffrt::TaskCtx * task)150 static inline void UnbindCoRoutione(ffrt::TaskCtx* task)
151 {
152 task->coRoutine->task = nullptr;
153 task->coRoutine = nullptr;
154 }
155
CoAlloc(ffrt::TaskCtx * task)156 static inline int CoAlloc(ffrt::TaskCtx* task)
157 {
158 if (!g_CoThreadEnv) {
159 CoInitThreadEnv();
160 }
161 if (task->coRoutine) {
162 if (g_CoThreadEnv->runningCo) {
163 CoMemFree(g_CoThreadEnv->runningCo);
164 }
165 g_CoThreadEnv->runningCo = task->coRoutine;
166 } else {
167 if (!g_CoThreadEnv->runningCo) {
168 g_CoThreadEnv->runningCo = AllocNewCoRoutine();
169 }
170 }
171 BindNewCoRoutione(task);
172 return 0;
173 }
174
175 // call CoCreat when task creat
CoCreat(ffrt::TaskCtx * task)176 static inline int CoCreat(ffrt::TaskCtx* task)
177 {
178 CoAlloc(task);
179 auto co = task->coRoutine;
180 if (co->status.load() == static_cast<int>(CoStatus::CO_UNINITIALIZED)) {
181 co2_init_context(&co->ctx, CoStartEntry, static_cast<void*>(co), co->stkMem.stk, co->stkMem.size);
182 }
183 return 0;
184 }
185
CoStackCheck(CoRoutine * co)186 static inline void CoStackCheck(CoRoutine* co)
187 {
188 if (co->stkMem.magic != STACK_MAGIC) {
189 FFRT_LOGE("sp offset:%lu.\n", (uint64_t)co->stkMem.stk +
190 CoStackAttr::Instance()->size - co->ctx.regs[REG_SP]);
191 FFRT_LOGE("stack over flow, check local variable in you tasks or use api 'ffrt_set_co_stack_attribute'.\n");
192 abort();
193 }
194 }
195
CoSwitchInTrace(ffrt::TaskCtx * task)196 static inline void CoSwitchInTrace(ffrt::TaskCtx* task)
197 {
198 if (task->coRoutine->status == static_cast<int>(CoStatus::CO_NOT_FINISH)) {
199 for (auto name : task->traceTag) {
200 FFRT_TRACE_BEGIN(name.c_str());
201 }
202 }
203 FFRT_FAKE_TRACE_MARKER(task->gid);
204 }
205
CoSwitchOutTrace(ffrt::TaskCtx * task)206 static inline void CoSwitchOutTrace(ffrt::TaskCtx* task)
207 {
208 FFRT_FAKE_TRACE_MARKER(task->gid);
209 int traceTagNum = static_cast<int>(task->traceTag.size());
210 for (int i = 0; i < traceTagNum; ++i) {
211 FFRT_TRACE_END();
212 }
213 }
214
215 // called by thread work
CoStart(ffrt::TaskCtx * task)216 void CoStart(ffrt::TaskCtx* task)
217 {
218 if (task->coRoutine) {
219 int ret = task->coRoutine->status.exchange(static_cast<int>(CoStatus::CO_RUNNING));
220 if (ret == static_cast<int>(CoStatus::CO_RUNNING) && GetBboxEnableState() != 0) {
221 FFRT_BBOX_LOG("executed by worker suddenly, ignore backtrace");
222 return;
223 }
224 }
225 CoCreat(task);
226 auto co = task->coRoutine;
227
228 #ifdef FFRT_BBOX_ENABLE
229 TaskRunCounterInc();
230 #endif
231
232 for (;;) {
233 FFRT_LOGD("Costart task[%lu], name[%s]", task->gid, task->label.c_str());
234 ffrt::TaskLoadTracking::Begin(task);
235 FFRT_TASK_BEGIN(task->label, task->gid);
236 CoSwitchInTrace(task);
237
238 CoSwitch(&co->thEnv->schCtx, &co->ctx);
239 FFRT_TASK_END();
240 ffrt::TaskLoadTracking::End(task); // Todo: deal with CoWait()
241 CoStackCheck(co);
242 auto pending = g_CoThreadEnv->pending;
243 if (pending == nullptr) {
244 #ifdef FFRT_BBOX_ENABLE
245 TaskFinishCounterInc();
246 #endif
247 break;
248 }
249 g_CoThreadEnv->pending = nullptr;
250 // Fast path: skip state transition
251 if ((*pending)(task)) {
252 FFRT_LOGD("Cowait task[%lu], name[%s]", task->gid, task->label.c_str());
253 #ifdef FFRT_BBOX_ENABLE
254 TaskSwitchCounterInc();
255 #endif
256 return;
257 }
258 FFRT_WAKE_TRACER(task->gid); // fast path wk
259 g_CoThreadEnv->runningCo = co;
260 }
261 }
262
263 // called by thread work
CoYield(void)264 void CoYield(void)
265 {
266 CoRoutine* co = static_cast<CoRoutine*>(g_CoThreadEnv->runningCo);
267 co->status.store(static_cast<int>(CoStatus::CO_NOT_FINISH));
268 g_CoThreadEnv->runningCo = nullptr;
269 CoSwitchOutTrace(co->task);
270 FFRT_BLOCK_MARKER(co->task->gid);
271 CoSwitch(&co->ctx, &g_CoThreadEnv->schCtx);
272 while (GetBboxEnableState() != 0) {
273 if (GetBboxEnableState() != gettid()) {
274 BboxFreeze(); // freeze non-crash thread
275 return;
276 }
277 const int IGNORE_DEPTH = 3;
278 backtrace(IGNORE_DEPTH);
279 co->status.store(static_cast<int>(CoStatus::CO_NOT_FINISH)); // recovery to old state
280 CoExit(co);
281 }
282 }
283
CoWait(const std::function<bool (ffrt::TaskCtx *)> & pred)284 void CoWait(const std::function<bool(ffrt::TaskCtx*)>& pred)
285 {
286 g_CoThreadEnv->pending = &pred;
287 CoYield();
288 }
289
CoWake(ffrt::TaskCtx * task,bool timeOut)290 void CoWake(ffrt::TaskCtx* task, bool timeOut)
291 {
292 if (task == nullptr) {
293 FFRT_LOGE("task is nullptr");
294 return;
295 }
296 // Fast path: state transition without lock
297 FFRT_LOGD("Cowake task[%lu], name[%s], timeOut[%d]", task->gid, task->label.c_str(), timeOut);
298 task->wakeupTimeOut = timeOut;
299 FFRT_WAKE_TRACER(task->gid);
300 task->UpdateState(ffrt::TaskState::READY);
301 }
302