• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #include "pbreader_parser.h"
17 #include <unistd.h>
18 #include "app_start_filter.h"
19 #include "binder_filter.h"
20 #include "common_types.pbreader.h"
21 #include "cpu_filter.h"
22 #include "data_area.h"
23 #ifdef ENABLE_HTRACE
24 #include "ftrace_event.pbreader.h"
25 #include "trace_plugin_result.pbreader.h"
26 #endif
27 #ifdef ENABLE_MEMORY
28 #include "memory_plugin_result.pbreader.h"
29 #endif
30 #include "stat_filter.h"
31 #if IS_WASM
32 #include "wasm_func.h"
33 #endif
34 namespace SysTuning {
35 namespace TraceStreamer {
PbreaderParser(TraceDataCache * dataCache,const TraceStreamerFilters * filters)36 PbreaderParser::PbreaderParser(TraceDataCache *dataCache, const TraceStreamerFilters *filters)
37     : ParserBase(filters),
38       pbreaderClockDetailParser_(std::make_unique<PbreaderClockDetailParser>(dataCache, filters)),
39 #ifdef ENABLE_HTRACE
40       htraceCpuDetailParser_(std::make_unique<HtraceCpuDetailParser>(dataCache, filters)),
41       htraceSymbolsDetailParser_(std::make_unique<HtraceSymbolsDetailParser>(dataCache, filters)),
42 #endif
43 #ifdef ENABLE_FFRT
44       pbreaderFfrtParser_(
45           std::make_unique<PbreaderFfrtDetailParser>(dataCache, filters, htraceCpuDetailParser_->eventParser_.get())),
46 #endif
47 #ifdef ENABLE_MEMORY
48       pbreaderMemParser_(std::make_unique<PbreaderMemParser>(dataCache, filters)),
49 #endif
50 #ifdef ENABLE_HILOG
51       pbreaderHiLogParser_(std::make_unique<PbreaderHiLogParser>(dataCache, filters)),
52 #endif
53 #ifdef ENABLE_NATIVE_HOOK
54       pbreaderNativeHookParser_(std::make_unique<PbreaderNativeHookParser>(dataCache, filters)),
55 #endif
56 #ifdef ENABLE_HTDUMP
57       pbreaderHidumpParser_(std::make_unique<PbreaderHidumpParser>(dataCache, filters)),
58 #endif
59 #ifdef ENABLE_CPUDATA
60       cpuUsageParser_(std::make_unique<PbreaderCpuDataParser>(dataCache, filters)),
61 #endif
62 #ifdef ENABLE_NETWORK
63       networkParser_(std::make_unique<PbreaderNetworkParser>(dataCache, filters)),
64 #endif
65 #ifdef ENABLE_DISKIO
66       diskIOParser_(std::make_unique<PbreaderDiskIOParser>(dataCache, filters)),
67 #endif
68 #ifdef ENABLE_PROCESS
69       processParser_(std::make_unique<PbreaderProcessParser>(dataCache, filters)),
70 #endif
71 #ifdef ENABLE_HISYSEVENT
72       hisyseventParser_(std::make_unique<PbreaderHisyseventParser>(dataCache, filters)),
73 #endif
74 #ifdef ENABLE_ARKTS
75       jsMemoryParser_(std::make_unique<PbreaderJSMemoryParser>(dataCache, filters)),
76 #endif
77 #ifdef ENABLE_HIPERF
78       perfDataParser_(std::make_unique<PerfDataParser>(dataCache, filters)),
79 #endif
80 #ifdef ENABLE_EBPF
81       ebpfDataParser_(std::make_unique<EbpfDataParser>(dataCache, filters)),
82 #endif
83 #ifdef ENABLE_STREAM_EXTEND
84       pbreaderStreamParser_(std::make_unique<PbreaderStreamParser>(dataCache, filters)),
85 #endif
86       traceDataCache_(dataCache)
87 {
88     InitPluginNameIndex();
89     if (traceDataCache_->supportThread_) {
90         dataSegArray_ = std::make_unique<PbreaderDataSegment[]>(maxSegArraySize);
91     } else {
92         dataSegArray_ = std::make_unique<PbreaderDataSegment[]>(1);
93     }
94 }
InitHookPluginNameIndex()95 inline void PbreaderParser::InitHookPluginNameIndex()
96 {
97 #ifdef ENABLE_NATIVE_HOOK
98     nativeHookPluginIndex_.insert(traceDataCache_->GetDataIndex("nativehook"));
99     nativeHookPluginIndex_.insert(traceDataCache_->GetDataIndex("hookdaemon"));
100     nativeHookConfigIndex_ = traceDataCache_->GetDataIndex("nativehook_config");
101     supportPluginNameIndex_.insert(nativeHookPluginIndex_.begin(), nativeHookPluginIndex_.end());
102     supportPluginNameIndex_.insert(nativeHookConfigIndex_);
103 #endif
104 }
InitMemoryPluginNameIndex()105 inline void PbreaderParser::InitMemoryPluginNameIndex()
106 {
107 #ifdef ENABLE_MEMORY
108     memPluginIndex_ = traceDataCache_->GetDataIndex("memory-plugin");
109     memoryPluginConfigIndex_ = traceDataCache_->GetDataIndex("memory-plugin_config");
110     supportPluginNameIndex_.insert(memPluginIndex_);
111     supportPluginNameIndex_.insert(memoryPluginConfigIndex_);
112 #endif
113 }
InitHiPluginNameIndex()114 inline void PbreaderParser::InitHiPluginNameIndex()
115 {
116 #ifdef ENABLE_HTDUMP
117     hidumpPluginIndex_.insert(traceDataCache_->GetDataIndex("hidump-plugin"));
118     hidumpPluginIndex_.insert(traceDataCache_->GetDataIndex("/data/local/tmp/libhidumpplugin.z.so"));
119     supportPluginNameIndex_.insert(hidumpPluginIndex_.begin(), hidumpPluginIndex_.end());
120 #endif
121 #ifdef ENABLE_HILOG
122     hilogPluginIndex_.insert(traceDataCache_->GetDataIndex("hilog-plugin"));
123     hilogPluginIndex_.insert(traceDataCache_->GetDataIndex("/data/local/tmp/libhilogplugin.z.so"));
124     supportPluginNameIndex_.insert(hilogPluginIndex_.begin(), hilogPluginIndex_.end());
125 #endif
126 #ifdef ENABLE_HISYSEVENT
127     hisyseventPluginIndex_ = traceDataCache_->GetDataIndex("hisysevent-plugin");
128     hisyseventPluginConfigIndex_ = traceDataCache_->GetDataIndex("hisysevent-plugin_config");
129     supportPluginNameIndex_.insert(hisyseventPluginIndex_);
130     supportPluginNameIndex_.insert(hisyseventPluginConfigIndex_);
131 #endif
132 }
InitPluginNameIndex()133 void PbreaderParser::InitPluginNameIndex()
134 {
135 #ifdef ENABLE_PROCESS
136     processPluginIndex_ = traceDataCache_->GetDataIndex("process-plugin");
137     supportPluginNameIndex_.insert(processPluginIndex_);
138 #endif
139 #ifdef ENABLE_DISKIO
140     diskioPluginIndex_ = traceDataCache_->GetDataIndex("diskio-plugin");
141     supportPluginNameIndex_.insert(diskioPluginIndex_);
142 #endif
143     InitMemoryPluginNameIndex();
144     InitHiPluginNameIndex();
145 #ifdef ENABLE_CPUDATA
146     cpuPluginIndex_ = traceDataCache_->GetDataIndex("cpu-plugin");
147     supportPluginNameIndex_.insert(cpuPluginIndex_);
148 #endif
149 #ifdef ENABLE_NETWORK
150     networkPluginIndex_ = traceDataCache_->GetDataIndex("network-plugin");
151     supportPluginNameIndex_.insert(networkPluginIndex_);
152 #endif
153     InitHookPluginNameIndex();
154 #ifdef ENABLE_ARKTS
155     arktsPluginIndex_ = traceDataCache_->GetDataIndex("arkts-plugin");
156     arktsPluginConfigIndex_ = traceDataCache_->GetDataIndex("arkts-plugin_config");
157     supportPluginNameIndex_.insert(arktsPluginIndex_);
158     supportPluginNameIndex_.insert(arktsPluginConfigIndex_);
159 #endif
160 #ifdef ENABLE_HTRACE
161     ftracePluginIndex_.insert(traceDataCache_->GetDataIndex("ftrace-plugin"));
162     ftracePluginIndex_.insert(traceDataCache_->GetDataIndex("/data/local/tmp/libftrace_plugin.z.so"));
163     supportPluginNameIndex_.insert(ftracePluginIndex_.begin(), ftracePluginIndex_.end());
164 #endif
165 #ifdef ENABLE_FFRT
166     ffrtPluginIndex_ = traceDataCache_->GetDataIndex("ffrt-profiler");
167     ffrtPluginConfigIndex_ = traceDataCache_->GetDataIndex("ffrt-profiler_config");
168     supportPluginNameIndex_.insert(ffrtPluginIndex_);
169     supportPluginNameIndex_.insert(ffrtPluginConfigIndex_);
170 #endif
171 #ifdef ENABLE_STREAM_EXTEND
172     streamPluginIndex_ = traceDataCache_->GetDataIndex("stream-plugin");
173     supportPluginNameIndex_.insert(streamPluginIndex_);
174 #endif
175 }
176 
177 #if defined(ENABLE_HIPERF) || defined(ENABLE_NATIVE_HOOK) || defined(ENABLE_EBPF)
ParserFileSO(std::string & directory,const std::vector<std::string> & relativeFilePaths)178 void PbreaderParser::ParserFileSO(std::string &directory, const std::vector<std::string> &relativeFilePaths)
179 {
180     for (const auto &filePath : relativeFilePaths) {
181         auto absoluteFilePath = filePath.substr(directory.length());
182         auto symbolsFile =
183             OHOS::Developtools::HiPerf::SymbolsFile::CreateSymbolsFile(SYMBOL_ELF_FILE, absoluteFilePath);
184         symbolsFile->setSymbolsFilePath(directory);
185         symbolsFile->LoadSymbols(nullptr, absoluteFilePath);
186         symbolsFiles_.emplace_back(std::move(symbolsFile));
187     }
188 }
189 #endif
190 
~PbreaderParser()191 PbreaderParser::~PbreaderParser()
192 {
193     TS_LOGI("clockid 2 is for RealTime and 1 is for BootTime");
194 }
195 
ReparseSymbolFilesAndResymbolization(std::string & symbolsPath,std::vector<std::string> & symbolsPaths)196 bool PbreaderParser::ReparseSymbolFilesAndResymbolization(std::string &symbolsPath,
197                                                           std::vector<std::string> &symbolsPaths)
198 {
199     auto parseStatus = false;
200 #if defined(ENABLE_HIPERF) || defined(ENABLE_NATIVE_HOOK) || defined(ENABLE_EBPF)
201     ParserFileSO(symbolsPath, symbolsPaths);
202 #endif
203 #ifdef ENABLE_HIPERF
204     if (traceDataCache_->GetPerfFilesData()->Size() > 0) {
205         perfDataParser_->PerfReloadSymbolFiles(symbolsFiles_);
206         parseStatus = true;
207     }
208 #endif
209 #ifdef ENABLE_NATIVE_HOOK
210     if (traceDataCache_->GetNativeHookFrameData()->Size() > 0) {
211         pbreaderNativeHookParser_->NativeHookReloadElfSymbolTable(symbolsFiles_);
212         parseStatus = true;
213     }
214 #endif
215 #ifdef ENABLE_EBPF
216     if (traceDataCache_->GetEbpfCallStack()->Size() > 0) {
217         ebpfDataParser_->EBPFReloadElfSymbolTable(symbolsFiles_);
218         parseStatus = true;
219     }
220 #endif
221     symbolsFiles_.clear();
222     return parseStatus;
223 }
224 
WaitForHPluginParserEnd()225 inline void PbreaderParser::WaitForHPluginParserEnd()
226 {
227 #ifdef ENABLE_HTRACE
228     htraceCpuDetailParser_->FilterAllEvents();
229     traceDataCache_->GetDataSourceClockIdData()->SetDataSourceClockId(DATA_SOURCE_TYPE_TRACE,
230                                                                       dataSourceTypeTraceClockid_);
231 #endif
232 #ifdef ENABLE_HILOG
233     pbreaderHiLogParser_->Finish();
234     traceDataCache_->GetDataSourceClockIdData()->SetDataSourceClockId(DATA_SOURCE_TYPE_HILOG,
235                                                                       dataSourceTypeHilogClockid_);
236 #endif
237 #ifdef ENABLE_HTDUMP
238     pbreaderHidumpParser_->Finish();
239     traceDataCache_->GetDataSourceClockIdData()->SetDataSourceClockId(DATA_SOURCE_TYPE_FPS, dataSourceTypeFpsClockid_);
240 #endif
241 #ifdef ENABLE_HISYSEVENT
242     hisyseventParser_->Finish();
243     traceDataCache_->GetDataSourceClockIdData()->SetDataSourceClockId(DATA_SOURCE_TYPE_HISYSEVENT,
244                                                                       dataSourceTypeHisyseventClockid_);
245 #endif
246 }
247 
WaitForOtherPluginParserEnd()248 inline void PbreaderParser::WaitForOtherPluginParserEnd()
249 {
250 #ifdef ENABLE_NATIVE_HOOK
251     pbreaderNativeHookParser_->FinishParseNativeHookData();
252     pbreaderNativeHookParser_->Finish();
253     traceDataCache_->GetDataSourceClockIdData()->SetDataSourceClockId(DATA_SOURCE_TYPE_NATIVEHOOK,
254                                                                       dataSourceTypeNativeHookClockid_);
255 #endif
256 #ifdef ENABLE_CPUDATA
257     cpuUsageParser_->Finish();
258     traceDataCache_->GetDataSourceClockIdData()->SetDataSourceClockId(DATA_SOURCE_TYPE_CPU, dataSourceTypeCpuClockid_);
259 #endif
260 #ifdef ENABLE_NETWORK
261     networkParser_->Finish();
262     traceDataCache_->GetDataSourceClockIdData()->SetDataSourceClockId(DATA_SOURCE_TYPE_NETWORK,
263                                                                       dataSourceTypeNetworkClockid_);
264 #endif
265 #ifdef ENABLE_PROCESS
266     processParser_->Finish();
267     traceDataCache_->GetDataSourceClockIdData()->SetDataSourceClockId(DATA_SOURCE_TYPE_PROCESS,
268                                                                       dataSourceTypeProcessClockid_);
269 #endif
270 #ifdef ENABLE_DISKIO
271     diskIOParser_->Finish();
272     traceDataCache_->GetDataSourceClockIdData()->SetDataSourceClockId(DATA_SOURCE_TYPE_DISKIO,
273                                                                       dataSourceTypeDiskioClockid_);
274 #endif
275 #ifdef ENABLE_ARKTS
276     jsMemoryParser_->Finish();
277     traceDataCache_->GetDataSourceClockIdData()->SetDataSourceClockId(DATA_SOURCE_TYPE_JSMEMORY,
278                                                                       dataSourceTypeJSMemoryClockid_);
279 #endif
280 #ifdef ENABLE_EBPF
281     ebpfDataParser_->Finish(); // keep final upate perf and ebpf data time range
282 #endif
283 #ifdef ENABLE_HIPERF
284     perfDataParser_->Finish();
285 #endif
286 #ifdef ENABLE_MEMORY
287     pbreaderMemParser_->Finish();
288     traceDataCache_->GetDataSourceClockIdData()->SetDataSourceClockId(DATA_SOURCE_TYPE_MEM, dataSourceTypeMemClockid_);
289 #endif
290 }
291 
WaitForParserEnd()292 void PbreaderParser::WaitForParserEnd()
293 {
294     if (parseThreadStarted_ || filterThreadStarted_) {
295         toExit_ = true;
296         while (!exited_) {
297             usleep(sleepDur_ * sleepDur_);
298         }
299     }
300     hasGotHeader_ = false;
301     WaitForHPluginParserEnd();
302     WaitForOtherPluginParserEnd();
303 #if defined(ENABLE_HTRACE) && defined(ENABLE_NATIVE_HOOK) && defined(ENABLE_HIPERF)
304     ParseNapiAsync();
305 #endif
306     traceDataCache_->GetDataSourceClockIdData()->Finish();
307     dataSegArray_.reset();
308     processedDataLen_ = 0;
309 }
310 
ParseTraceDataItem(const std::string & buffer)311 void PbreaderParser::ParseTraceDataItem(const std::string &buffer)
312 {
313     int32_t head = rawDataHead_;
314     if (!traceDataCache_->supportThread_ || traceDataCache_->isSplitFile_) {
315         dataSegArray_[head].seg = std::make_shared<std::string>(std::move(buffer));
316         dataSegArray_[head].status = TS_PARSE_STATUS_SEPRATED;
317         ParserData(dataSegArray_[head], traceDataCache_->isSplitFile_);
318         return;
319     }
320     while (!toExit_) {
321         if (dataSegArray_[head].status.load() != TS_PARSE_STATUS_INIT) {
322             usleep(sleepDur_);
323             continue;
324         }
325         dataSegArray_[head].seg = std::make_shared<std::string>(std::move(buffer));
326         dataSegArray_[head].status = TS_PARSE_STATUS_SEPRATED;
327         rawDataHead_ = (rawDataHead_ + 1) % maxSegArraySize;
328         break;
329     }
330     if (!parseThreadStarted_) {
331         parseThreadStarted_ = true;
332         int32_t tmp = traceDataCache_->parserThreadNum_;
333         while (tmp--) {
334             parserThreadCount_++;
335             std::thread ParseTypeThread(&PbreaderParser::ParseThread, this);
336             ParseTypeThread.detach();
337             TS_LOGI("parser Thread:%d/%d start working ...\n", traceDataCache_->parserThreadNum_ - tmp,
338                     traceDataCache_->parserThreadNum_);
339         }
340     }
341     if (!filterThreadStarted_) {
342         filterThreadStarted_ = true;
343         std::thread FilterTypeThread(&PbreaderParser::FilterThread, this);
344         TS_LOGI("FilterThread start working ...");
345         FilterTypeThread.detach();
346     }
347 }
348 
349 #ifdef ENABLE_ARKTS
EnableFileSeparate(bool enabled)350 void PbreaderParser::EnableFileSeparate(bool enabled)
351 {
352     jsMemoryParser_->EnableSaveFile(enabled);
353 }
354 #endif
FilterData(PbreaderDataSegment & seg,bool isSplitFile)355 void PbreaderParser::FilterData(PbreaderDataSegment &seg, bool isSplitFile)
356 {
357     bool haveSplitSeg = false;
358     if (seg.dataType == DATA_SOURCE_TYPE_TRACE) {
359 #ifdef ENABLE_HTRACE
360         htraceCpuDetailParser_->FilterAllEventsReader();
361 #endif
362     }
363 #ifdef ENABLE_FFRT
364     else if (seg.dataType == DATA_SOURCE_TYPE_FFRT) {
365         pbreaderFfrtParser_->FilterAllEventsReader();
366     }
367 #endif
368 #ifdef ENABLE_NATIVE_HOOK
369     else if (seg.dataType == DATA_SOURCE_TYPE_NATIVEHOOK) {
370         pbreaderNativeHookParser_->Parse(seg, haveSplitSeg);
371     } else if (seg.dataType == DATA_SOURCE_TYPE_NATIVEHOOK_CONFIG) {
372         pbreaderNativeHookParser_->ParseConfigInfo(seg);
373     }
374 #endif
375 #ifdef ENABLE_MEMORY
376     else if (seg.dataType == DATA_SOURCE_TYPE_MEM) {
377         pbreaderMemParser_->Parse(seg, seg.timeStamp, seg.clockId);
378     } else if (seg.dataType == DATA_SOURCE_TYPE_MEM_CONFIG) {
379         pbreaderMemParser_->ParseMemoryConfig(seg);
380     }
381 #endif
382 #ifdef ENABLE_HILOG
383     else if (seg.dataType == DATA_SOURCE_TYPE_HILOG) {
384         pbreaderHiLogParser_->Parse(seg.protoData, haveSplitSeg);
385     }
386 #endif
387 #ifdef ENABLE_CPUDATA
388     else if (seg.dataType == DATA_SOURCE_TYPE_CPU) {
389         cpuUsageParser_->Parse(seg.protoData, seg.timeStamp);
390     }
391 #endif
392 #ifdef ENABLE_HTDUMP
393     else if (seg.dataType == DATA_SOURCE_TYPE_FPS) {
394         pbreaderHidumpParser_->Parse(seg.protoData);
395         dataSourceTypeFpsClockid_ = pbreaderHidumpParser_->ClockId();
396     }
397 #endif
398 #ifdef ENABLE_NETWORK
399     else if (seg.dataType == DATA_SOURCE_TYPE_NETWORK) {
400         networkParser_->Parse(seg.protoData, seg.timeStamp);
401     }
402 #endif
403 #ifdef ENABLE_PROCESS
404     else if (seg.dataType == DATA_SOURCE_TYPE_PROCESS) {
405         processParser_->Parse(seg.protoData, seg.timeStamp);
406     }
407 #endif
408 #ifdef ENABLE_DISKIO
409     else if (seg.dataType == DATA_SOURCE_TYPE_DISKIO) {
410         diskIOParser_->Parse(seg.protoData, seg.timeStamp);
411     }
412 #endif
413 #ifdef ENABLE_ARKTS
414     else if (seg.dataType == DATA_SOURCE_TYPE_JSMEMORY) {
415         jsMemoryParser_->Parse(seg.protoData, seg.timeStamp, traceDataCache_->SplitFileMinTime(),
416                                traceDataCache_->SplitFileMaxTime(), profilerPluginData_);
417     } else if (seg.dataType == DATA_SOURCE_TYPE_JSMEMORY_CONFIG) {
418         jsMemoryParser_->ParseJSMemoryConfig(seg.protoData);
419     }
420 #endif
421 #ifdef ENABLE_HISYSEVENT
422     else if (seg.dataType == DATA_SOURCE_TYPE_HISYSEVENT) {
423         ProtoReader::HisyseventInfo_Reader hisyseventInfo(seg.protoData.data_, seg.protoData.size_);
424         hisyseventParser_->Parse(&hisyseventInfo, seg.timeStamp, haveSplitSeg);
425     } else if (seg.dataType == DATA_SOURCE_TYPE_HISYSEVENT_CONFIG) {
426         ProtoReader::HisyseventConfig_Reader hisyseventConfig(seg.protoData.data_, seg.protoData.size_);
427         hisyseventParser_->Parse(&hisyseventConfig, seg.timeStamp);
428     }
429 #endif
430 #ifdef ENABLE_STREAM_EXTEND
431     else if (seg.dataType == DATA_SOURCE_TYPE_STREAM) {
432         pbreaderStreamParser_->Parse(seg);
433     }
434 #endif
435     if (traceDataCache_->isSplitFile_ && haveSplitSeg) {
436         mPbreaderSplitData_.emplace(splitFileOffset_, nextLength_ + packetSegLength_);
437     }
438     if (traceDataCache_->supportThread_ && !traceDataCache_->isSplitFile_) {
439         filterHead_ = (filterHead_ + 1) % maxSegArraySize;
440     }
441     seg.status = TS_PARSE_STATUS_INIT;
442 }
FilterThread()443 void PbreaderParser::FilterThread()
444 {
445     TS_LOGI("filter thread start work!");
446     while (true) {
447         PbreaderDataSegment &seg = dataSegArray_[filterHead_];
448         if (seg.status.load() == TS_PARSE_STATUS_INVALID) {
449             seg.status = TS_PARSE_STATUS_INIT;
450             filterHead_ = (filterHead_ + 1) % maxSegArraySize;
451             TS_LOGD("seprateHead_d:\t%d, parseHead_:\t%d, filterHead_:\t%d\n", rawDataHead_, parseHead_, filterHead_);
452             continue;
453         }
454         if (seg.status.load() != TS_PARSE_STATUS_PARSED) {
455             if (toExit_ && !parserThreadCount_) {
456                 TS_LOGI("exiting Filter Thread");
457                 exited_ = true;
458                 filterThreadStarted_ = false;
459                 TS_LOGI("seprateHead:\t%d, parseHead_:\t%d, filterHead_:\t%d, status:%d\n", rawDataHead_, parseHead_,
460                         filterHead_, seg.status.load());
461                 return;
462             }
463             TS_LOGD("seprateHead:\t%d, parseHead_:\t%d, filterHead_:\t%d, status:%d\n", rawDataHead_, parseHead_,
464                     filterHead_, seg.status.load());
465             usleep(sleepDur_);
466             continue;
467         }
468         FilterData(seg, false);
469     }
470 }
471 
SpliteConfigData(const std::string & pluginName,const PbreaderDataSegment & dataSeg)472 bool PbreaderParser::SpliteConfigData(const std::string &pluginName, const PbreaderDataSegment &dataSeg)
473 {
474     if (EndWith(pluginName, "arkts-plugin_config")) {
475         std::string dataString(dataSeg.seg->c_str(), dataSeg.seg->length());
476         arkTsConfigData_ = lenBuffer_ + dataString;
477         return true;
478     } else if (EndWith(pluginName, "config")) {
479         mPbreaderSplitData_.emplace(splitFileOffset_, nextLength_ + packetSegLength_);
480         return true;
481     }
482     return false;
483 }
484 
SpliteDataBySegment(DataIndex pluginNameIndex,PbreaderDataSegment & dataSeg)485 bool PbreaderParser::SpliteDataBySegment(DataIndex pluginNameIndex, PbreaderDataSegment &dataSeg)
486 {
487     bool isOtherPlugin = false;
488 #ifdef ENABLE_HTRACE
489     isOtherPlugin = isOtherPlugin || ftracePluginIndex_.count(pluginNameIndex);
490 #endif
491 #ifdef ENABLE_FFRT
492     isOtherPlugin = isOtherPlugin || (ffrtPluginIndex_ == pluginNameIndex);
493 #endif
494 #ifdef ENABLE_HISYSEVENT
495     isOtherPlugin = isOtherPlugin || (hisyseventPluginIndex_ == pluginNameIndex);
496 #endif
497 #ifdef ENABLE_NATIVE_HOOK
498     isOtherPlugin = isOtherPlugin || nativeHookPluginIndex_.count(pluginNameIndex);
499 #endif
500 #ifdef ENABLE_HILOG
501     isOtherPlugin = isOtherPlugin || hilogPluginIndex_.count(pluginNameIndex);
502 #endif
503     if (isOtherPlugin) {
504         return false;
505     }
506     // need convert to Primary Time Plugin
507 #ifdef ENABLE_MEMORY
508     if (pluginNameIndex == memPluginIndex_) {
509         dataSeg.timeStamp = streamFilters_->clockFilter_->ToPrimaryTraceTime(TS_CLOCK_REALTIME, dataSeg.timeStamp);
510         UpdatePluginTimeRange(TS_CLOCK_BOOTTIME, dataSeg.timeStamp, dataSeg.timeStamp);
511     }
512 #endif
513     if (dataSeg.timeStamp >= traceDataCache_->SplitFileMinTime() &&
514         dataSeg.timeStamp <= traceDataCache_->SplitFileMaxTime()) {
515         mPbreaderSplitData_.emplace(splitFileOffset_, nextLength_ + packetSegLength_);
516     }
517     if (pluginNameIndex == arktsPluginConfigIndex_ || pluginNameIndex == arktsPluginIndex_) {
518         return false;
519     }
520     return true;
521 }
ParseDataByPluginName(PbreaderDataSegment & dataSeg,DataIndex pulginNameIndex,const ProtoReader::ProfilerPluginData_Reader & pluginDataZero,bool isSplitFile)522 void PbreaderParser::ParseDataByPluginName(PbreaderDataSegment &dataSeg,
523                                            DataIndex pulginNameIndex,
524                                            const ProtoReader::ProfilerPluginData_Reader &pluginDataZero,
525                                            bool isSplitFile)
526 {
527     if (ftracePluginIndex_.count(pulginNameIndex)) { // ok
528 #ifdef ENABLE_HTRACE
529         ParseFtrace(dataSeg);
530 #endif
531     }
532 #ifdef ENABLE_FFRT
533     else if (ffrtPluginIndex_ == pulginNameIndex) {
534         ParseFfrt(dataSeg);
535     } else if (ffrtPluginConfigIndex_ == pulginNameIndex) {
536         ParseFfrtConfig(dataSeg);
537     }
538 #endif
539 #ifdef ENABLE_NATIVE_HOOK
540     else if (nativeHookPluginIndex_.count(pulginNameIndex)) {
541         ParseNativeHook(dataSeg, isSplitFile);
542     } else if (pulginNameIndex == nativeHookConfigIndex_) {
543         ParseNativeHookConfig(dataSeg);
544     }
545 #endif
546 #ifdef ENABLE_MEMORY
547     else if (pulginNameIndex == memPluginIndex_) {
548         ParseMemory(pluginDataZero, dataSeg);
549     } else if (pulginNameIndex == memoryPluginConfigIndex_) {
550         ParseMemoryConfig(dataSeg, pluginDataZero);
551     }
552 #endif
553 #ifdef ENABLE_HILOG
554     else if (hilogPluginIndex_.count(pulginNameIndex)) {
555         ParseHilog(dataSeg);
556     }
557 #endif
558 #ifdef ENABLE_HTDUMP
559     else if (hidumpPluginIndex_.count(pulginNameIndex)) {
560         ParseFPS(dataSeg);
561     }
562 #endif
563 #ifdef ENABLE_CPUDATA
564     else if (pulginNameIndex == cpuPluginIndex_) {
565         ParseCpuUsage(dataSeg);
566     }
567 #endif
568 #ifdef ENABLE_NETWORK
569     else if (pulginNameIndex == networkPluginIndex_) {
570         ParseNetwork(dataSeg);
571     }
572 #endif
573 #ifdef ENABLE_DISKIO
574     else if (pulginNameIndex == diskioPluginIndex_) {
575         ParseDiskIO(dataSeg);
576     }
577 #endif
578 #ifdef ENABLE_PROCESS
579     else if (pulginNameIndex == processPluginIndex_) {
580         ParseProcess(dataSeg);
581     }
582 #endif
583 #ifdef ENABLE_HISYSEVENT
584     else if (pulginNameIndex == hisyseventPluginIndex_) {
585         ParseHisysevent(dataSeg);
586     } else if (pulginNameIndex == hisyseventPluginConfigIndex_) {
587         ParseHisyseventConfig(dataSeg);
588     }
589 #endif
590 #ifdef ENABLE_ARKTS
591     else if (pulginNameIndex == arktsPluginIndex_) {
592         ParseJSMemory(pluginDataZero, dataSeg, isSplitFile);
593     } else if (pulginNameIndex == arktsPluginConfigIndex_) {
594         ParseJSMemoryConfig(dataSeg);
595     }
596 #endif
597 #ifdef ENABLE_STREAM_EXTEND
598     else if (pulginNameIndex == streamPluginIndex_) { // for trace extend demo
599         ParseStream(dataSeg);
600     }
601 #endif
602 }
603 
ParserData(PbreaderDataSegment & dataSeg,bool isSplitFile)604 void PbreaderParser::ParserData(PbreaderDataSegment &dataSeg, bool isSplitFile)
605 {
606     ProtoReader::ProfilerPluginData_Reader pluginDataZero(reinterpret_cast<const uint8_t *>(dataSeg.seg->c_str()),
607                                                           dataSeg.seg->length());
608     if (!pluginDataZero.has_name()) {
609         return;
610     }
611     auto pluginName = pluginDataZero.name().ToStdString();
612     auto pluginNameIndex = traceDataCache_->GetDataIndex(pluginName);
613     if (isSplitFile && SpliteConfigData(pluginName, dataSeg)) {
614         return;
615     }
616     if (pluginDataZero.has_tv_sec() && pluginDataZero.has_tv_nsec()) {
617         dataSeg.timeStamp = pluginDataZero.tv_sec() * SEC_TO_NS + pluginDataZero.tv_nsec();
618     }
619 
620     if (isSplitFile && SpliteDataBySegment(pluginNameIndex, dataSeg)) {
621         return;
622     }
623     if (supportPluginNameIndex_.count(pluginNameIndex)) {
624         dataSeg.protoData = pluginDataZero.data();
625         ParseDataByPluginName(dataSeg, pluginNameIndex, pluginDataZero, isSplitFile);
626     } else {
627 #if IS_WASM
628         TraceStreamerPluginOutFilter(reinterpret_cast<const char *>(pluginDataZero.data().data_),
629                                      pluginDataZero.data().size_, pluginName);
630 #endif
631         dataSeg.status = TS_PARSE_STATUS_INVALID;
632         streamFilters_->statFilter_->IncreaseStat(TRACE_EVENT_OTHER, STAT_EVENT_DATA_INVALID);
633         return;
634     }
635     if (!traceDataCache_->supportThread_ || traceDataCache_->isSplitFile_) {
636         FilterData(dataSeg, isSplitFile);
637     }
638 }
ParseThread()639 void PbreaderParser::ParseThread()
640 {
641     TS_LOGI("parser thread start work!\n");
642     while (true) {
643         int32_t head = GetNextSegment();
644         if (head < 0) {
645             if (head == ERROR_CODE_EXIT) {
646                 TS_LOGI("parse thread exit");
647                 return;
648             } else if (head == ERROR_CODE_NODATA) {
649                 continue;
650             }
651         }
652         PbreaderDataSegment &dataSeg = dataSegArray_[head];
653         ParserData(dataSeg, false);
654     }
655 }
656 
657 #ifdef ENABLE_MEMORY
ParseMemory(const ProtoReader::ProfilerPluginData_Reader & pluginDataZero,PbreaderDataSegment & dataSeg)658 void PbreaderParser::ParseMemory(const ProtoReader::ProfilerPluginData_Reader &pluginDataZero,
659                                  PbreaderDataSegment &dataSeg)
660 {
661     BuiltinClocks clockId = TS_CLOCK_REALTIME;
662     dataSourceTypeMemClockid_ = clockId;
663     dataSeg.dataType = DATA_SOURCE_TYPE_MEM;
664     dataSeg.clockId = clockId;
665     dataSeg.status = TS_PARSE_STATUS_PARSED;
666 }
ParseMemoryConfig(PbreaderDataSegment & dataSeg,const ProtoReader::ProfilerPluginData_Reader & pluginDataZero)667 void PbreaderParser::ParseMemoryConfig(PbreaderDataSegment &dataSeg,
668                                        const ProtoReader::ProfilerPluginData_Reader &pluginDataZero)
669 {
670     if (pluginDataZero.has_sample_interval()) {
671         uint32_t sampleInterval = pluginDataZero.sample_interval();
672         traceDataCache_->GetTraceConfigData()->AppendNewData("memory_config", "sample_interval",
673                                                              std::to_string(sampleInterval));
674     }
675     dataSeg.dataType = DATA_SOURCE_TYPE_MEM_CONFIG;
676     dataSeg.status = TS_PARSE_STATUS_PARSED;
677 }
678 #endif
679 #ifdef ENABLE_HILOG
ParseHilog(PbreaderDataSegment & dataSeg)680 void PbreaderParser::ParseHilog(PbreaderDataSegment &dataSeg)
681 {
682     dataSeg.dataType = DATA_SOURCE_TYPE_HILOG;
683     dataSourceTypeHilogClockid_ = TS_CLOCK_REALTIME;
684     dataSeg.status = TS_PARSE_STATUS_PARSED;
685 }
686 #endif
687 #ifdef ENABLE_NATIVE_HOOK
ParseNativeHookConfig(PbreaderDataSegment & dataSeg)688 void PbreaderParser::ParseNativeHookConfig(PbreaderDataSegment &dataSeg)
689 {
690     dataSeg.dataType = DATA_SOURCE_TYPE_NATIVEHOOK_CONFIG;
691     dataSeg.status = TS_PARSE_STATUS_PARSED;
692 }
ParseNativeHook(PbreaderDataSegment & dataSeg,bool isSplitFile)693 void PbreaderParser::ParseNativeHook(PbreaderDataSegment &dataSeg, bool isSplitFile)
694 {
695     dataSourceTypeNativeHookClockid_ = TS_CLOCK_REALTIME;
696     dataSeg.dataType = DATA_SOURCE_TYPE_NATIVEHOOK;
697     dataSeg.status = TS_PARSE_STATUS_PARSED;
698     if (isSplitFile) {
699         dataSourceType_ = DATA_SOURCE_TYPE_NATIVEHOOK;
700     }
701 }
702 #endif
703 
704 #ifdef ENABLE_HTRACE
ParseFtrace(PbreaderDataSegment & dataSeg)705 void PbreaderParser::ParseFtrace(PbreaderDataSegment &dataSeg)
706 {
707     dataSeg.dataType = DATA_SOURCE_TYPE_TRACE;
708     ProtoReader::TracePluginResult_Reader tracePluginResult(dataSeg.protoData);
709     if (tracePluginResult.has_ftrace_cpu_stats()) {
710         auto cpuStats = *tracePluginResult.ftrace_cpu_stats();
711         ProtoReader::FtraceCpuStatsMsg_Reader ftraceCpuStatsMsg(cpuStats.data_, cpuStats.size_);
712         auto s = *ftraceCpuStatsMsg.per_cpu_stats();
713         ProtoReader::PerCpuStatsMsg_Reader perCpuStatsMsg(s.data_, s.size_);
714         TS_LOGD("s.overrun():%" PRIu64 "", perCpuStatsMsg.overrun());
715         TS_LOGD("s.dropped_events():%" PRIu64 "", perCpuStatsMsg.dropped_events());
716         auto clock = ftraceCpuStatsMsg.trace_clock().ToStdString();
717         if (clock == "boot") {
718             clock_ = TS_CLOCK_BOOTTIME;
719         } else if (clock == "mono") {
720             clock_ = TS_MONOTONIC;
721         } else {
722             TS_LOGI("invalid clock:%s", clock.c_str());
723             dataSeg.status = TS_PARSE_STATUS_INVALID;
724             return;
725         }
726         dataSeg.clockId = clock_;
727         dataSourceTypeTraceClockid_ = clock_;
728         dataSeg.status = TS_PARSE_STATUS_PARSED;
729         return;
730     }
731     bool haveSplitSeg = false;
732     dataSeg.clockId = clock_;
733     if (tracePluginResult.has_ftrace_cpu_detail()) {
734         htraceCpuDetailParser_->Parse(dataSeg, tracePluginResult, haveSplitSeg);
735     }
736     if (tracePluginResult.has_symbols_detail()) {
737         htraceSymbolsDetailParser_->Parse(dataSeg.protoData); // has Event
738         haveSplitSeg = true;
739     }
740     if (tracePluginResult.has_clocks_detail()) {
741         pbreaderClockDetailParser_->Parse(dataSeg.protoData); // has Event
742         haveSplitSeg = true;
743     }
744     if (traceDataCache_->isSplitFile_ && haveSplitSeg) {
745         mPbreaderSplitData_.emplace(splitFileOffset_, nextLength_ + packetSegLength_);
746     }
747     if (tracePluginResult.has_ftrace_cpu_detail() || tracePluginResult.has_clocks_detail() ||
748         tracePluginResult.has_symbols_detail()) {
749         dataSeg.status = TS_PARSE_STATUS_PARSED;
750     } else {
751         dataSeg.status = TS_PARSE_STATUS_INVALID;
752     }
753 }
754 #endif
755 #ifdef ENABLE_FFRT
ParseFfrtConfig(PbreaderDataSegment & dataSeg)756 void PbreaderParser::ParseFfrtConfig(PbreaderDataSegment &dataSeg)
757 {
758     pbreaderFfrtParser_->SetFfrtSrcClockid(dataSeg);
759     dataSeg.dataType = DATA_SOURCE_TYPE_FFRT_CONFIG;
760     dataSeg.status = TS_PARSE_STATUS_PARSED;
761 }
ParseFfrt(PbreaderDataSegment & dataSeg)762 void PbreaderParser::ParseFfrt(PbreaderDataSegment &dataSeg)
763 {
764     bool haveSplitSeg = false;
765     pbreaderFfrtParser_->Parser(dataSeg, haveSplitSeg);
766     if (haveSplitSeg) {
767         mPbreaderSplitData_.emplace(splitFileOffset_, nextLength_ + packetSegLength_);
768     }
769     dataSeg.dataType = DATA_SOURCE_TYPE_FFRT;
770     dataSeg.status = TS_PARSE_STATUS_PARSED;
771 }
772 #endif
773 #ifdef ENABLE_HTDUMP
ParseFPS(PbreaderDataSegment & dataSeg)774 void PbreaderParser::ParseFPS(PbreaderDataSegment &dataSeg)
775 {
776     dataSeg.dataType = DATA_SOURCE_TYPE_FPS;
777     dataSeg.status = TS_PARSE_STATUS_PARSED;
778 }
779 #endif
780 
781 #ifdef ENABLE_CPUDATA
ParseCpuUsage(PbreaderDataSegment & dataSeg)782 void PbreaderParser::ParseCpuUsage(PbreaderDataSegment &dataSeg)
783 {
784     dataSourceTypeCpuClockid_ = TS_CLOCK_REALTIME;
785     dataSeg.dataType = DATA_SOURCE_TYPE_CPU;
786     dataSeg.status = TS_PARSE_STATUS_PARSED;
787 }
788 #endif
789 #ifdef ENABLE_NETWORK
ParseNetwork(PbreaderDataSegment & dataSeg)790 void PbreaderParser::ParseNetwork(PbreaderDataSegment &dataSeg)
791 {
792     dataSourceTypeNetworkClockid_ = TS_CLOCK_REALTIME;
793     dataSeg.dataType = DATA_SOURCE_TYPE_NETWORK;
794     dataSeg.status = TS_PARSE_STATUS_PARSED;
795 }
796 #endif
797 #ifdef ENABLE_DISKIO
ParseDiskIO(PbreaderDataSegment & dataSeg)798 void PbreaderParser::ParseDiskIO(PbreaderDataSegment &dataSeg)
799 {
800     dataSourceTypeDiskioClockid_ = TS_CLOCK_REALTIME;
801     dataSeg.dataType = DATA_SOURCE_TYPE_DISKIO;
802     dataSeg.status = TS_PARSE_STATUS_PARSED;
803 }
804 #endif
805 
806 #ifdef ENABLE_PROCESS
ParseProcess(PbreaderDataSegment & dataSeg)807 void PbreaderParser::ParseProcess(PbreaderDataSegment &dataSeg)
808 {
809     dataSourceTypeProcessClockid_ = TS_CLOCK_BOOTTIME;
810     dataSeg.dataType = DATA_SOURCE_TYPE_PROCESS;
811     dataSeg.status = TS_PARSE_STATUS_PARSED;
812 }
813 #endif
814 
815 #ifdef ENABLE_HISYSEVENT
ParseHisysevent(PbreaderDataSegment & dataSeg)816 void PbreaderParser::ParseHisysevent(PbreaderDataSegment &dataSeg)
817 {
818     dataSourceTypeHisyseventClockid_ = TS_CLOCK_REALTIME;
819     dataSeg.dataType = DATA_SOURCE_TYPE_HISYSEVENT;
820     dataSeg.status = TS_PARSE_STATUS_PARSED;
821 }
ParseHisyseventConfig(PbreaderDataSegment & dataSeg)822 void PbreaderParser::ParseHisyseventConfig(PbreaderDataSegment &dataSeg)
823 {
824     dataSourceTypeHisyseventClockid_ = TS_CLOCK_REALTIME;
825     dataSeg.dataType = DATA_SOURCE_TYPE_HISYSEVENT_CONFIG;
826     dataSeg.status = TS_PARSE_STATUS_PARSED;
827 }
828 #endif
829 
830 #ifdef ENABLE_ARKTS
ParseJSMemory(const ProtoReader::ProfilerPluginData_Reader & pluginDataZero,PbreaderDataSegment & dataSeg,bool isSplitFile)831 void PbreaderParser::ParseJSMemory(const ProtoReader::ProfilerPluginData_Reader &pluginDataZero,
832                                    PbreaderDataSegment &dataSeg,
833                                    bool isSplitFile)
834 {
835     if (isSplitFile) {
836         dataSourceType_ = DATA_SOURCE_TYPE_JSMEMORY;
837         profilerPluginData_.name = pluginDataZero.name().ToStdString();
838         profilerPluginData_.status = pluginDataZero.status();
839         profilerPluginData_.clockId = pluginDataZero.clock_id();
840         profilerPluginData_.tvSec = pluginDataZero.tv_sec();
841         profilerPluginData_.tvNsec = pluginDataZero.tv_nsec();
842         profilerPluginData_.version = pluginDataZero.version().ToStdString();
843         profilerPluginData_.sampleInterval = pluginDataZero.sample_interval();
844     }
845     dataSourceTypeJSMemoryClockid_ = TS_CLOCK_REALTIME;
846     dataSeg.dataType = DATA_SOURCE_TYPE_JSMEMORY;
847     dataSeg.status = TS_PARSE_STATUS_PARSED;
848 }
ParseJSMemoryConfig(PbreaderDataSegment & dataSeg)849 void PbreaderParser::ParseJSMemoryConfig(PbreaderDataSegment &dataSeg)
850 {
851     dataSourceTypeJSMemoryClockid_ = TS_CLOCK_REALTIME;
852     dataSeg.dataType = DATA_SOURCE_TYPE_JSMEMORY_CONFIG;
853     dataSeg.status = TS_PARSE_STATUS_PARSED;
854 }
855 #endif
856 
857 #ifdef ENABLE_STREAM_EXTEND
ParseStream(PbreaderDataSegment & dataSeg)858 void PbreaderParser::ParseStream(PbreaderDataSegment &dataSeg)
859 {
860     dataSeg.dataType = DATA_SOURCE_TYPE_STREAM;
861     dataSeg.status = TS_PARSE_STATUS_PARSED;
862 }
863 #endif
864 
GetNextSegment()865 int32_t PbreaderParser::GetNextSegment()
866 {
867     int32_t head;
868     std::lock_guard<std::mutex> muxLockGuard(pbreaderDataSegMux_);
869     head = parseHead_;
870     PbreaderDataSegment &pbreaderDataSegmentSeg = dataSegArray_[head];
871     if (pbreaderDataSegmentSeg.status.load() != TS_PARSE_STATUS_SEPRATED) {
872         if (toExit_) {
873             parserThreadCount_--;
874             TS_LOGI("exiting parser, parserThread Count:%d\n", parserThreadCount_);
875             TS_LOGI("seprateHead_x:\t%d, parseHead_:\t%d, filterHead_:\t%d status:%d\n", rawDataHead_, parseHead_,
876                     filterHead_, pbreaderDataSegmentSeg.status.load());
877             if (!parserThreadCount_ && !filterThreadStarted_) {
878                 exited_ = true;
879             }
880             return ERROR_CODE_EXIT;
881         }
882         usleep(sleepDur_);
883         return ERROR_CODE_NODATA;
884     }
885     parseHead_ = (parseHead_ + 1) % maxSegArraySize;
886     pbreaderDataSegmentSeg.status = TS_PARSE_STATUS_PARSING;
887     return head;
888 }
889 #ifdef ENABLE_EBPF
CalcEbpfCutOffset(std::deque<uint8_t>::iterator & packagesBegin,size_t & currentLength)890 bool PbreaderParser::CalcEbpfCutOffset(std::deque<uint8_t>::iterator &packagesBegin, size_t &currentLength)
891 {
892     auto standaloneDataLength = profilerDataLength_ - packetHeaderLength_;
893     if (traceDataCache_->isSplitFile_ && !parsedEbpfOver_) {
894         if (!hasInitEbpfPublicData_) {
895             // Record the offset of Hiperf's 1024-byte header relative to the entire file.
896             ebpfDataParser_->SetEbpfDataOffset(processedDataLen_);
897             ebpfDataParser_->SetSpliteTimeRange(traceDataCache_->SplitFileMinTime(),
898                                                 traceDataCache_->SplitFileMaxTime());
899             parsedFileOffset_ += profilerDataLength_ - packetHeaderLength_;
900             hasInitEbpfPublicData_ = true;
901         }
902         parsedEbpfOver_ = ebpfDataParser_->AddAndSplitEbpfData(packagesBuffer_);
903         if (parsedEbpfOver_) {
904             profilerDataType_ = ProfilerTraceFileHeader::UNKNOW_TYPE;
905             hasGotHeader_ = false;
906             processedDataLen_ += standaloneDataLength;
907         }
908         return false;
909     }
910     if (!traceDataCache_->isSplitFile_ && packagesBuffer_.size() >= standaloneDataLength) {
911         ebpfDataParser_->InitAndParseEbpfData(packagesBuffer_, standaloneDataLength);
912         currentLength -= standaloneDataLength;
913         packagesBegin += standaloneDataLength;
914         profilerDataType_ = ProfilerTraceFileHeader::UNKNOW_TYPE;
915         hasGotHeader_ = false;
916         return true;
917     }
918     return false;
919 }
920 #endif
921 
GetHeaderAndUpdateLengthMark(std::deque<uint8_t>::iterator & packagesBegin,size_t & currentLength)922 bool PbreaderParser::GetHeaderAndUpdateLengthMark(std::deque<uint8_t>::iterator &packagesBegin, size_t &currentLength)
923 {
924     if (!hasGotHeader_) {
925         if (!InitProfilerTraceFileHeader()) {
926             return false;
927         }
928         packagesBuffer_.erase(packagesBuffer_.begin(), packagesBuffer_.begin() + packetHeaderLength_);
929         processedDataLen_ += packetHeaderLength_;
930         currentLength -= packetHeaderLength_;
931         packagesBegin += packetHeaderLength_;
932         parsedFileOffset_ += packetHeaderLength_;
933         pbreaderCurentLength_ = profilerDataLength_ - packetHeaderLength_;
934         hasGotHeader_ = true;
935         if (!currentLength) {
936             return false;
937         }
938     }
939     return true;
940 }
941 #if IS_WASM
ParseSDKData()942 bool PbreaderParser::ParseSDKData()
943 {
944     if (packagesBuffer_.size() >= profilerDataLength_ - packetHeaderLength_) {
945         auto thirdPartySize = profilerDataLength_ - packetHeaderLength_;
946         auto buffer = std::make_unique<uint8_t[]>(thirdPartySize).get();
947         std::copy(packagesBuffer_.begin(), packagesBuffer_.begin() + thirdPartySize, buffer);
948         TraceStreamerPluginOutFilter(reinterpret_cast<const char *>(buffer), thirdPartySize, standalonePluginName_);
949         return true;
950     }
951     return false;
952 }
953 #endif
954 
ParseSegLengthAndEnsureSegDataEnough(std::deque<uint8_t>::iterator & packagesBegin,size_t & currentLength)955 bool PbreaderParser::ParseSegLengthAndEnsureSegDataEnough(std::deque<uint8_t>::iterator &packagesBegin,
956                                                           size_t &currentLength)
957 {
958     std::string bufferLine;
959     if (!hasGotSegLength_) {
960         if (currentLength < packetSegLength_) {
961             return false;
962         }
963         bufferLine.assign(packagesBegin, packagesBegin + packetSegLength_);
964         const uint32_t *len = reinterpret_cast<const uint32_t *>(bufferLine.data());
965         nextLength_ = *len;
966         lenBuffer_ = bufferLine;
967         pbreaderLength_ += nextLength_ + packetSegLength_;
968         hasGotSegLength_ = true;
969         currentLength -= packetSegLength_;
970         packagesBegin += packetSegLength_;
971         parsedFileOffset_ += packetSegLength_;
972         splitFileOffset_ = profilerDataLength_ - pbreaderCurentLength_;
973         pbreaderCurentLength_ -= packetSegLength_;
974     }
975     if (currentLength < nextLength_) {
976         return false;
977     }
978     return true;
979 }
ParseDataRecursively(std::deque<uint8_t>::iterator & packagesBegin,size_t & currentLength)980 bool PbreaderParser::ParseDataRecursively(std::deque<uint8_t>::iterator &packagesBegin, size_t &currentLength)
981 {
982     TS_CHECK_TRUE_RET(GetHeaderAndUpdateLengthMark(packagesBegin, currentLength), false);
983 #ifdef ENABLE_HIPERF
984     if (profilerDataType_ == ProfilerTraceFileHeader::HIPERF_DATA) {
985         return ParseHiperfData(packagesBegin, currentLength);
986     }
987 #endif
988     if (profilerDataType_ == ProfilerTraceFileHeader::STANDALONE_DATA) {
989         if (EBPF_PLUGIN_NAME.compare(standalonePluginName_) == 0) {
990 #ifdef ENABLE_EBPF
991             return CalcEbpfCutOffset(packagesBegin, currentLength);
992 #else
993             return false;
994 #endif
995         } else {
996 #if IS_WASM
997             TS_CHECK_TRUE_RET(ParseSDKData(), false); // 三方sdk逻辑待验证。
998 #endif
999         }
1000     }
1001     std::string bufferLine;
1002     while (true) {
1003         TS_CHECK_TRUE_RET(ParseSegLengthAndEnsureSegDataEnough(packagesBegin, currentLength), true);
1004         bufferLine.assign(packagesBegin, packagesBegin + nextLength_);
1005         ParseTraceDataItem(bufferLine);
1006         hasGotSegLength_ = false;
1007         packagesBegin += nextLength_;
1008         currentLength -= nextLength_;
1009         parsedFileOffset_ += nextLength_;
1010         if (nextLength_ > pbreaderCurentLength_) {
1011             TS_LOGE("fatal error, data length not match nextLength_:%u, pbreaderCurentLength_:%" PRIu64 "", nextLength_,
1012                     pbreaderCurentLength_);
1013         }
1014         pbreaderCurentLength_ -= nextLength_;
1015         if (pbreaderCurentLength_ == 0) {
1016             hasGotHeader_ = false;
1017             processedDataLen_ += packagesBegin - packagesBuffer_.begin();
1018             packagesBuffer_.erase(packagesBuffer_.begin(), packagesBegin);
1019             profilerDataType_ = ProfilerTraceFileHeader::UNKNOW_TYPE;
1020             TS_LOGD("read proto finished!");
1021             return ParseDataRecursively(packagesBegin, currentLength);
1022         }
1023     }
1024     return true;
1025 }
1026 
ParseTraceDataSegment(std::unique_ptr<uint8_t[]> bufferStr,size_t size,bool isFinish)1027 void PbreaderParser::ParseTraceDataSegment(std::unique_ptr<uint8_t[]> bufferStr, size_t size, bool isFinish)
1028 {
1029     packagesBuffer_.insert(packagesBuffer_.end(), &bufferStr[0], &bufferStr[size]);
1030     auto packagesBegin = packagesBuffer_.begin();
1031     auto currentLength = packagesBuffer_.size();
1032     if (ParseDataRecursively(packagesBegin, currentLength)) {
1033         processedDataLen_ += packagesBegin - packagesBuffer_.begin();
1034         packagesBuffer_.erase(packagesBuffer_.begin(), packagesBegin);
1035     }
1036     return;
1037 }
1038 #ifdef ENABLE_HIPERF
ParseHiperfData(std::deque<uint8_t>::iterator & packagesBegin,size_t & currentLength)1039 bool PbreaderParser::ParseHiperfData(std::deque<uint8_t>::iterator &packagesBegin, size_t &currentLength)
1040 {
1041     if (!traceDataCache_->isSplitFile_) {
1042         if (packagesBuffer_.size() >= profilerDataLength_ - packetHeaderLength_) {
1043             auto size = profilerDataLength_ - packetHeaderLength_;
1044             (void)perfDataParser_->InitPerfDataAndLoad(packagesBuffer_, size, processedDataLen_, false, true);
1045             currentLength -= size;
1046             packagesBegin += size;
1047             profilerDataType_ = ProfilerTraceFileHeader::UNKNOW_TYPE;
1048             hasGotHeader_ = false;
1049             return true;
1050         }
1051         return false;
1052     }
1053     bool isFinish = perfProcessedLen_ + packagesBuffer_.size() >= profilerDataLength_ - packetHeaderLength_;
1054     auto size = packagesBuffer_.size();
1055     if (isFinish) {
1056         size = profilerDataLength_ - packetHeaderLength_ - perfProcessedLen_;
1057     }
1058     auto ret = perfDataParser_->InitPerfDataAndLoad(packagesBuffer_, size, processedDataLen_, true, isFinish);
1059     perfProcessedLen_ += ret;
1060     currentLength -= ret;
1061     packagesBegin += ret;
1062     if (isFinish) {
1063         profilerDataType_ = ProfilerTraceFileHeader::UNKNOW_TYPE;
1064         hasGotHeader_ = false;
1065     }
1066     return true;
1067 }
StoreTraceDataSegment(std::unique_ptr<uint8_t[]> bufferStr,size_t size,int32_t isFinish)1068 void PbreaderParser::StoreTraceDataSegment(std::unique_ptr<uint8_t[]> bufferStr, size_t size, int32_t isFinish)
1069 {
1070     packagesBuffer_.insert(packagesBuffer_.end(), &bufferStr[0], &bufferStr[size]);
1071     if (!traceDataCache_->isSplitFile_) {
1072         return;
1073     }
1074 
1075     uint64_t length = packagesBuffer_.size();
1076     auto ret = perfDataParser_->InitPerfDataAndLoad(packagesBuffer_, length, 0, true, isFinish);
1077     perfProcessedLen_ += ret;
1078     processedDataLen_ += ret;
1079     packagesBuffer_.erase(packagesBuffer_.begin(), packagesBuffer_.begin() + ret);
1080     return;
1081 }
TraceDataSegmentEnd(bool isSplitFile)1082 void PbreaderParser::TraceDataSegmentEnd(bool isSplitFile)
1083 {
1084     perfDataParser_->InitPerfDataAndLoad(packagesBuffer_, packagesBuffer_.size(), 0, isSplitFile, true);
1085     packagesBuffer_.clear();
1086     return;
1087 }
1088 #endif
1089 
InitProfilerTraceFileHeader()1090 bool PbreaderParser::InitProfilerTraceFileHeader()
1091 {
1092     if (packagesBuffer_.size() < packetHeaderLength_) {
1093         TS_LOGI("buffer size less than profiler trace file header");
1094         return false;
1095     }
1096     uint8_t buffer[packetHeaderLength_];
1097     (void)memset_s(buffer, sizeof(buffer), 0, sizeof(buffer));
1098     int32_t i = 0;
1099     for (auto it = packagesBuffer_.begin(); it != packagesBuffer_.begin() + packetHeaderLength_; ++it, ++i) {
1100         buffer[i] = *it;
1101     }
1102     ProfilerTraceFileHeader *pHeader = reinterpret_cast<ProfilerTraceFileHeader *>(buffer);
1103     if (pHeader->data.length <= packetHeaderLength_ || pHeader->data.magic != ProfilerTraceFileHeader::HEADER_MAGIC) {
1104         TS_LOGE("Profiler Trace data is truncated or invalid magic! len = %" PRIu64 ", maigc = %" PRIx64 "",
1105                 pHeader->data.length, pHeader->data.magic);
1106         return false;
1107     }
1108     if (pHeader->data.dataType == ProfilerTraceFileHeader::HIPERF_DATA) {
1109 #ifdef ENABLE_HIPERF
1110         perfDataParser_->RecordPerfProfilerHeader(buffer, packetHeaderLength_);
1111 #endif
1112     } else if (pHeader->data.dataType == ProfilerTraceFileHeader::STANDALONE_DATA &&
1113                EBPF_PLUGIN_NAME.compare(pHeader->data.standalonePluginName) == 0) {
1114 #ifdef ENABLE_EBPF
1115         ebpfDataParser_->RecordEbpfProfilerHeader(buffer, packetHeaderLength_);
1116 #endif
1117     } else {
1118         auto ret = memcpy_s(&profilerTraceFileHeader_, sizeof(profilerTraceFileHeader_), buffer, packetHeaderLength_);
1119         if (ret == -1 || profilerTraceFileHeader_.data.magic != ProfilerTraceFileHeader::HEADER_MAGIC) {
1120             TS_LOGE("Get profiler trace file header failed! ret = %d, magic = %" PRIx64 "", ret,
1121                     profilerTraceFileHeader_.data.magic);
1122             return false;
1123         }
1124     }
1125     profilerDataLength_ = pHeader->data.length;
1126     profilerDataType_ = pHeader->data.dataType;
1127     memcpy_s(standalonePluginName_, sizeof(standalonePluginName_), pHeader->data.standalonePluginName,
1128              sizeof(standalonePluginName_));
1129 
1130     TS_LOGI("magic = %" PRIx64 ", length = %" PRIu64 ", dataType = %x, boottime = %" PRIu64 "", pHeader->data.magic,
1131             pHeader->data.length, pHeader->data.dataType, pHeader->data.boottime);
1132 #if IS_WASM
1133     const int32_t DATA_TYPE_CLOCK = 100;
1134     TraceStreamerPluginOutSendData(reinterpret_cast<char *>(buffer), packetHeaderLength_, DATA_TYPE_CLOCK);
1135 #endif
1136     pbreaderClockDetailParser_->Parse(pHeader);
1137     return true;
1138 }
1139 
1140 #if defined(ENABLE_HTRACE) && defined(ENABLE_NATIVE_HOOK) && defined(ENABLE_HIPERF)
ParseNapiAsync()1141 void PbreaderParser::ParseNapiAsync()
1142 {
1143     // 将native memory中存在的traceid取出, 并记录其对应的callstackid
1144     std::unordered_map<std::string, uint32_t> traceidToCallchainidMap;
1145     GetTraceidInfoFromNativeHook(traceidToCallchainidMap);
1146 
1147     // 从callstack表中获取所有的traceid, 根据其所属的itid将SliceInfo存入对应queue
1148     std::unordered_map<uint64_t, std::queue<SliceInfo>> itidToCallstackIdsMap;
1149     GetTraceidInfoFromCallstack(traceidToCallchainidMap, itidToCallstackIdsMap);
1150 
1151     // 筛选出包含NativeAsyncWork::AsyncWorkCallback的函数栈的callchainid, 将其存入callchainIdSet
1152     std::unordered_set<uint32_t> callchainIdSet;
1153     GetCallchainIdSetFromHiperf(callchainIdSet);
1154 
1155     DumpDataFromHiperf(traceidToCallchainidMap, callchainIdSet, itidToCallstackIdsMap);
1156 }
1157 
GetTraceidInfoFromNativeHook(std::unordered_map<std::string,uint32_t> & traceidToCallchainidMap)1158 void PbreaderParser::GetTraceidInfoFromNativeHook(std::unordered_map<std::string, uint32_t> &traceidToCallchainidMap)
1159 {
1160     auto nativeHook = traceDataCache_->GetConstNativeHookData();
1161     std::string preWord("napi:");
1162     for (int i = 0; i < nativeHook.Size(); i++) {
1163         auto subType = nativeHook.SubTypes()[i];
1164         if (subType == INVALID_UINT64) {
1165             continue;
1166         }
1167         auto subTypeStr = traceDataCache_->GetDataFromDict(subType);
1168         if (!StartWith(subTypeStr, preWord)) {
1169             continue;
1170         }
1171         auto pos = subTypeStr.find(preWord) + preWord.size();
1172         auto traceidStr = subTypeStr.substr(pos, subTypeStr.find_last_of(':') - pos);
1173         auto traceidIndex = traceDataCache_->GetDataIndex(traceidStr);
1174         traceidToCallchainidMap.emplace(std::move(traceidStr), nativeHook.CallChainIds()[i]);
1175     }
1176 }
1177 
GetTraceidInfoFromCallstack(const std::unordered_map<std::string,uint32_t> & traceidToCallchainidMap,std::unordered_map<uint64_t,std::queue<SliceInfo>> & itidToCallstackIdsMap)1178 void PbreaderParser::GetTraceidInfoFromCallstack(
1179     const std::unordered_map<std::string, uint32_t> &traceidToCallchainidMap,
1180     std::unordered_map<uint64_t, std::queue<SliceInfo>> &itidToCallstackIdsMap)
1181 {
1182     auto callStack = traceDataCache_->GetConstInternalSlicesData();
1183     std::string preWord("traceid:");
1184     std::string invalidTraceidStr("0x0");
1185     for (int i = 0; i < callStack.Size(); i++) {
1186         auto name = traceDataCache_->GetDataFromDict(callStack.NamesData()[i]);
1187         if (!StartWith(name, "H:Napi execute, name:") || callStack.DursData()[i] == INVALID_UINT64) {
1188             continue;
1189         }
1190         auto traceidStr = name.substr(name.find(preWord) + preWord.size());
1191         if (traceidStr == invalidTraceidStr) {
1192             continue;
1193         }
1194         if (traceidToCallchainidMap.find(traceidStr) == traceidToCallchainidMap.end()) {
1195             continue;
1196         }
1197         auto iter = itidToCallstackIdsMap.find(callStack.CallIds()[i]);
1198         if (iter == itidToCallstackIdsMap.end()) {
1199             itidToCallstackIdsMap.emplace(callStack.CallIds()[i], std::queue<SliceInfo>());
1200             iter = itidToCallstackIdsMap.find(callStack.CallIds()[i]);
1201         }
1202         iter->second.emplace(callStack.TimeStampData()[i], callStack.TimeStampData()[i] + callStack.DursData()[i],
1203                              traceidStr);
1204     }
1205 }
1206 
GetCallchainIdSetFromHiperf(std::unordered_set<uint32_t> & callchainIdSet)1207 void PbreaderParser::GetCallchainIdSetFromHiperf(std::unordered_set<uint32_t> &callchainIdSet)
1208 {
1209     auto perfCallChain = traceDataCache_->GetConstPerfCallChainData();
1210     std::string asyncWork("NativeAsyncWork::AsyncWorkCallback");
1211     for (int i = 0; i < perfCallChain.Size(); i++) {
1212         auto callchainId = perfCallChain.CallChainIds()[i];
1213         if (callchainIdSet.find(callchainId) != callchainIdSet.end()) {
1214             continue;
1215         }
1216         auto nameIndex = perfCallChain.Names()[i];
1217         if (nameIndex == INVALID_UINT64) {
1218             continue;
1219         }
1220         auto name = traceDataCache_->GetDataFromDict(nameIndex);
1221         if (name.find(asyncWork) != std::string::npos) {
1222             callchainIdSet.emplace(perfCallChain.CallChainIds()[i]);
1223         }
1224     }
1225 }
1226 
DumpDataFromHiperf(const std::unordered_map<std::string,uint32_t> & traceidToCallchainidMap,const std::unordered_set<uint32_t> & callchainIdSet,std::unordered_map<uint64_t,std::queue<SliceInfo>> & itidToCallstackIdsMap)1227 void PbreaderParser::DumpDataFromHiperf(const std::unordered_map<std::string, uint32_t> &traceidToCallchainidMap,
1228                                         const std::unordered_set<uint32_t> &callchainIdSet,
1229                                         std::unordered_map<uint64_t, std::queue<SliceInfo>> &itidToCallstackIdsMap)
1230 {
1231     auto perfThread = traceDataCache_->GetConstPerfThreadData();
1232     std::unordered_map<uint32_t, uint32_t> tidToPidMap;
1233     for (size_t i = 0; i < perfThread.Size(); i++) {
1234         tidToPidMap.emplace(perfThread.Tids()[i], perfThread.Pids()[i]);
1235     }
1236     auto perfSample = traceDataCache_->GetConstPerfSampleData();
1237     auto callStack = traceDataCache_->GetConstInternalSlicesData();
1238     for (size_t i = 0; i < perfSample.Size(); i++) {
1239         // callchainid未命中, 即当前栈不包含NativeAsyncWork::AsyncWorkCallback
1240         if (callchainIdSet.find(perfSample.SampleIds()[i]) == callchainIdSet.end()) {
1241             continue;
1242         }
1243         // 根据tid和tsPerfSample查询对应的SliceInfo
1244         auto itid = streamFilters_->processFilter_->GetInternalTid(perfSample.Tids()[i]);
1245         if (itidToCallstackIdsMap.find(itid) == itidToCallstackIdsMap.end()) {
1246             continue;
1247         }
1248         auto tsPerfSample = perfSample.TimestampTraces()[i];
1249         auto queue = itidToCallstackIdsMap.at(itid);
1250         while (!queue.empty() && queue.front().tsEnd_ < tsPerfSample) {
1251             queue.pop();
1252         }
1253         if (queue.empty() || tsPerfSample < queue.front().tsBegin_) {
1254             continue;
1255         }
1256         // 根据traceid查询native侧的callchainid
1257         auto iterNative = traceidToCallchainidMap.find(queue.front().traceid_);
1258         if (iterNative == traceidToCallchainidMap.end()) {
1259             continue;
1260         }
1261         // 根据tid查询pid
1262         auto iterPid = tidToPidMap.find(perfSample.Tids()[i]);
1263         if (iterPid == tidToPidMap.end()) {
1264             continue;
1265         }
1266         PerfNapiAsyncRow perfNapiAsyncRow{
1267             .timeStamp = tsPerfSample,
1268             .traceid = traceDataCache_->GetDataIndex(queue.front().traceid_),
1269             .cpuId = static_cast<uint8_t>(perfSample.CpuIds()[i]),
1270             .threadId = perfSample.Tids()[i],
1271             .processId = iterPid->second,
1272             .callerCallchainid = iterNative->second,
1273             .calleeCallchainid = perfSample.SampleIds()[i],
1274             .perfSampleId = i,
1275             .eventCount = perfSample.EventCounts()[i],
1276             .eventTypeId = perfSample.EventTypeIds()[i],
1277         };
1278         traceDataCache_->GetPerfNapiAsyncData()->AppendNewPerfNapiAsync(perfNapiAsyncRow);
1279     }
1280 }
1281 #endif
1282 } // namespace TraceStreamer
1283 } // namespace SysTuning
1284