• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #ifndef PBREADER_PARSER_H
17 #define PBREADER_PARSER_H
18 #include <cstdint>
19 #include <limits>
20 #include <map>
21 #include <queue>
22 #include <stdexcept>
23 #include <string>
24 #include <thread>
25 #include <unordered_map>
26 #include <unordered_set>
27 #include "common_types.h"
28 #include "common_types.pbreader.h"
29 #include "clock_filter_ex.h"
30 #ifdef ENABLE_EBPF
31 #include "ebpf_data_parser.h"
32 #endif
33 #include "file.h"
34 #ifdef ENABLE_HTRACE
35 #include "htrace_cpu_detail_parser.h"
36 #include "htrace_symbols_detail_parser.h"
37 #endif
38 #include "htrace_plugin_time_parser.h"
39 #include "parser_base.h"
40 #include "pbreader_clock_detail_parser.h"
41 #include "pbreader_file_header.h"
42 #ifdef ENABLE_FFRT
43 #include "pbreader_ffrt_parser.h"
44 #endif
45 #ifdef ENABLE_CPUDATA
46 #include "pbreader_cpu_data_parser.h"
47 #endif
48 #ifdef ENABLE_DISKIO
49 #include "pbreader_disk_io_parser.h"
50 #endif
51 #ifdef ENABLE_HTDUMP
52 #include "pbreader_hidump_parser.h"
53 #endif
54 #ifdef ENABLE_HILOG
55 #include "pbreader_hilog_parser.h"
56 #endif
57 #ifdef ENABLE_HISYSEVENT
58 #include "pbreader_hisysevent_parser.h"
59 #endif
60 #ifdef ENABLE_ARKTS
61 #include "pbreader_js_memory_parser.h"
62 #endif
63 #ifdef ENABLE_MEMORY
64 #include "pbreader_mem_parser.h"
65 #endif
66 #ifdef ENABLE_NATIVE_HOOK
67 #include "pbreader_native_hook_parser.h"
68 #endif
69 #ifdef ENABLE_NETWORK
70 #include "pbreader_network_parser.h"
71 #endif
72 #ifdef ENABLE_PROCESS
73 #include "pbreader_process_parser.h"
74 #endif
75 #ifdef ENABLE_STREAM_EXTEND
76 #include "pbreader_stream_parser.h"
77 #endif
78 #ifdef ENABLE_XPOWER
79 #include "pbreader_xpower_parser.h"
80 #endif
81 #ifdef ENABLE_HIPERF
82 #include "perf_data_parser.h"
83 #endif
84 #include "proto_reader_help.h"
85 #include "string_help.h"
86 #include "trace_data_cache.h"
87 #include "trace_streamer_filters.h"
88 #include "ts_common.h"
89 
90 namespace SysTuning {
91 namespace TraceStreamer {
92 using namespace SysTuning::base;
93 #if defined(ENABLE_HTRACE) && defined(ENABLE_NATIVE_HOOK) && defined(ENABLE_HIPERF)
94 struct SliceInfo {
SliceInfoSliceInfo95     SliceInfo(uint64_t tsBegin, uint64_t tsEnd, const std::string &traceid)
96         : tsBegin_(tsBegin), tsEnd_(tsEnd), traceid_(traceid){};
97     uint64_t tsBegin_ = INVALID_UINT64;
98     uint64_t tsEnd_ = INVALID_UINT64;
99     std::string traceid_;
100 };
101 #endif
102 class PbreaderParser : public ParserBase, public HtracePluginTimeParser {
103 public:
104     PbreaderParser(TraceDataCache *dataCache, const TraceStreamerFilters *filters);
105     ~PbreaderParser();
106     void ParseTraceDataSegment(std::unique_ptr<uint8_t[]> bufferStr, size_t size, bool isFinish = false) override;
107     bool ReparseSymbolFileAndResymbolization(const std::string &directory, const std::string &fileName);
108     void WaitForParserEnd();
109 #ifdef ENABLE_ARKTS
110     void EnableFileSeparate(bool enabled);
111 #endif
112 #if defined(ENABLE_HIPERF) || defined(ENABLE_NATIVE_HOOK) || defined(ENABLE_EBPF)
113     std::unique_ptr<SymbolsFile> ParseELF(const std::string &directory, const std::string &fileName);
114 #endif
115 #ifdef ENABLE_HIPERF
116     void TraceDataSegmentEnd(bool isSplitFile);
117     void StoreTraceDataSegment(std::unique_ptr<uint8_t[]> bufferStr, size_t size, int32_t isFinish);
118 #endif
GetPbreaderSplitData()119     const auto &GetPbreaderSplitData()
120     {
121         return mPbreaderSplitData_;
122     }
GetProfilerHeader()123     auto GetProfilerHeader()
124     {
125         return profilerTraceFileHeader_;
126     }
127 #ifdef ENABLE_NATIVE_HOOK
ClearNativehookData()128     auto ClearNativehookData()
129     {
130         pbreaderNativeHookParser_->FinishSplitNativeHook();
131     }
132 #endif
GetDataSourceType()133     auto GetDataSourceType()
134     {
135         return dataSourceType_;
136     }
137 #ifdef ENABLE_ARKTS
GetJsMemoryData()138     auto GetJsMemoryData()
139     {
140         return jsMemoryParser_.get();
141     }
GetArkTsConfigData()142     auto GetArkTsConfigData()
143     {
144         return arkTsConfigData_;
145     }
146 #endif
ClearPbreaderSplitData()147     auto ClearPbreaderSplitData()
148     {
149 #ifdef ENABLE_HIPERF
150         perfDataParser_->ClearPerfSplitResult();
151         perfProcessedLen_ = 0;
152 #endif
153 #ifdef ENABLE_EBPF
154         ebpfDataParser_->ClearEbpfSplitResult();
155         hasInitEbpfPublicData_ = false;
156         parsedEbpfOver_ = false;
157 #endif
158         processedDataLen_ = 0;
159         splitFileOffset_ = 0;
160         hasGotHeader_ = false;
161         return mPbreaderSplitData_.clear();
162     }
163 #ifdef ENABLE_HIPERF
GetPerfSplitResult()164     const auto &GetPerfSplitResult()
165     {
166         return perfDataParser_->GetPerfSplitResult();
167     }
168 #endif
169 #ifdef ENABLE_EBPF
GetEbpfDataParser()170     const auto &GetEbpfDataParser()
171     {
172         return ebpfDataParser_;
173     }
174 #endif
175 #ifdef ENABLE_HTRACE
EnableOnlyParseFtrace()176     void EnableOnlyParseFtrace()
177     {
178         onlyParseFtrace_ = true;
179     }
180 #endif
181 
182 private:
183     bool ParseDataRecursively(std::deque<uint8_t>::iterator &packagesBegin, size_t &currentLength);
184 #ifdef ENABLE_HIPERF
185     bool ParseHiperfData(std::deque<uint8_t>::iterator &packagesBegin, size_t &currentLength);
186 #endif
187     void ParseTraceDataItem(const std::string &buffer) override;
188     void FilterData(PbreaderDataSegment &seg, bool isSplitFile);
189     void ParserData(PbreaderDataSegment &dataSeg, bool isSplitFile);
190 
191 private:
192 #if IS_WASM
193     bool ParseSDKData();
194 #endif
195     void InitPluginNameIndex();
196     void InitHookPluginNameIndex();
197     void InitMemoryPluginNameIndex();
198     void InitHiPluginNameIndex();
199     void WaitForHPluginParserEnd();
200     void WaitForOtherPluginParserEnd();
201     bool GetHeaderAndUpdateLengthMark(std::deque<uint8_t>::iterator &packagesBegin, size_t &currentLength);
202     bool ParseSegLengthAndEnsureSegDataEnough(std::deque<uint8_t>::iterator &packagesBegin, size_t &currentLength);
203 #ifdef ENABLE_MEMORY
204     void ParseMemory(const ProtoReader::ProfilerPluginData_Reader &pluginDataZero, PbreaderDataSegment &dataSeg);
205     void ParseMemoryConfig(PbreaderDataSegment &dataSeg, const ProtoReader::ProfilerPluginData_Reader &pluginDataZero);
206 #endif
207 #ifdef ENABLE_HILOG
208     void ParseHilog(PbreaderDataSegment &dataSeg);
209 #endif
210 #ifdef ENABLE_HTRACE
211     void ParseFtrace(PbreaderDataSegment &dataSeg);
212 #endif
213 #ifdef ENABLE_FFRT
214     void ParseFfrtConfig(PbreaderDataSegment &dataSeg);
215     void ParseFfrt(PbreaderDataSegment &dataSeg);
216 #endif
217 #ifdef ENABLE_HTDUMP
218     void ParseFPS(PbreaderDataSegment &dataSeg);
219 #endif
220 #ifdef ENABLE_CPUDATA
221     void ParseCpuUsage(PbreaderDataSegment &dataSeg);
222 #endif
223 #ifdef ENABLE_NETWORK
224     void ParseNetwork(PbreaderDataSegment &dataSeg);
225 #endif
226 #ifdef ENABLE_XPOWER
227     void ParseXpower(PbreaderDataSegment &dataSeg);
228 #endif
229 #ifdef ENABLE_DISKIO
230     void ParseDiskIO(PbreaderDataSegment &dataSeg);
231 #endif
232 #ifdef ENABLE_PROCESS
233     void ParseProcess(PbreaderDataSegment &dataSeg);
234 #endif
235 #ifdef ENABLE_HISYSEVENT
236     void ParseHisysevent(PbreaderDataSegment &dataSeg);
237     void ParseHisyseventConfig(PbreaderDataSegment &dataSeg);
238 #endif
239 #ifdef ENABLE_NATIVE_HOOK
240     void ParseNativeHookConfig(PbreaderDataSegment &dataSeg);
241     void ParseNativeHook(PbreaderDataSegment &dataSeg, bool isSplitFile);
242 #endif
243 #ifdef ENABLE_ARKTS
244     void ParseJSMemory(const ProtoReader::ProfilerPluginData_Reader &pluginDataZero,
245                        PbreaderDataSegment &dataSeg,
246                        bool isSplitFile);
247     void ParseJSMemoryConfig(PbreaderDataSegment &dataSeg);
248 #endif
249 #ifdef ENABLE_STREAM_EXTEND
250     void ParseStream(PbreaderDataSegment &dataSeg);
251 #endif
252     void ParseThread();
253     int32_t GetNextSegment();
254     void FilterThread();
255 #ifdef ENABLE_EBPF
256     bool CalcEbpfCutOffset(std::deque<uint8_t>::iterator &packagesBegin, size_t &currentLength);
257 #endif
258     bool SpliteConfigData(const std::string &pluginName, const PbreaderDataSegment &dataSeg);
259     bool InitProfilerTraceFileHeader();
260     void ParseDataByPluginName(PbreaderDataSegment &dataSeg,
261                                DataIndex pulginNameIndex,
262                                const ProtoReader::ProfilerPluginData_Reader &pluginDataZero,
263                                bool isSplitFile);
264     bool SpliteDataBySegment(DataIndex pluginNameIndex, PbreaderDataSegment &dataSeg);
265 #if defined(ENABLE_HTRACE) && defined(ENABLE_NATIVE_HOOK) && defined(ENABLE_HIPERF)
266     void ParseNapiAsync();
267     void GetTraceidInfoFromCallstack(const std::unordered_map<std::string, uint32_t> &traceidToCallchainidMap,
268                                      std::unordered_map<uint64_t, std::queue<SliceInfo>> &itidToCallstackIdsMap);
269     void GetTraceidInfoFromNativeHook(std::unordered_map<std::string, uint32_t> &traceidToCallchainidMap);
270     void GetCallchainIdSetFromHiperf(std::unordered_set<uint32_t> &callchainIdSet);
271     void DumpDataFromHiperf(const std::unordered_map<std::string, uint32_t> &traceidToCallchainidMap,
272                             const std::unordered_set<uint32_t> &callchainIdSet,
273                             std::unordered_map<uint64_t, std::queue<SliceInfo>> &itidToCallstackIdsMap);
274 #endif
275     ProfilerTraceFileHeader profilerTraceFileHeader_;
276     uint32_t profilerDataType_ = ProfilerTraceFileHeader::UNKNOW_TYPE;
277     uint64_t profilerDataLength_ = 0;
278     ProfilerPluginDataHeader profilerPluginData_;
279     uint64_t pbreaderCurentLength_ = 0;
280     char standalonePluginName_[ProfilerTraceFileHeader::PLUGIN_MODULE_NAME_MAX + 1] = "";
281     bool hasGotSegLength_ = false;
282     bool hasGotHeader_ = false;
283     uint32_t nextLength_ = 0;
284     const size_t packetSegLength_ = 4;
285     const size_t packetHeaderLength_ = 1024;
286     TraceDataCache *traceDataCache_;
287     std::unique_ptr<PbreaderClockDetailParser> pbreaderClockDetailParser_;
288 #ifdef ENABLE_HTRACE
289     std::unique_ptr<HtraceCpuDetailParser> htraceCpuDetailParser_;
290     std::unique_ptr<HtraceSymbolsDetailParser> htraceSymbolsDetailParser_;
291     ClockId dataSourceTypeTraceClockid_ = TS_CLOCK_UNKNOW;
292     bool onlyParseFtrace_ = false;
293 #endif
294     std::set<DataIndex> ftracePluginIndex_ = {};
295 #ifdef ENABLE_FFRT
296     DataIndex ffrtPluginIndex_ = {};
297     DataIndex ffrtPluginConfigIndex_ = {};
298     std::unique_ptr<PbreaderFfrtDetailParser> pbreaderFfrtParser_;
299 #endif
300 #ifdef ENABLE_MEMORY
301     std::unique_ptr<PbreaderMemParser> pbreaderMemParser_;
302     ClockId dataSourceTypeMemClockid_ = TS_CLOCK_UNKNOW;
303     DataIndex memPluginIndex_;
304     DataIndex memoryPluginConfigIndex_;
305 #endif
306 #ifdef ENABLE_HTDUMP
307     std::unique_ptr<PbreaderHidumpParser> pbreaderHidumpParser_;
308     std::set<DataIndex> hidumpPluginIndex_ = {};
309     ClockId dataSourceTypeFpsClockid_ = TS_CLOCK_UNKNOW;
310 #endif
311 #ifdef ENABLE_CPUDATA
312     std::unique_ptr<PbreaderCpuDataParser> cpuUsageParser_;
313     ClockId dataSourceTypeCpuClockid_ = TS_CLOCK_UNKNOW;
314     DataIndex cpuPluginIndex_;
315 #endif
316 #ifdef ENABLE_NETWORK
317     std::unique_ptr<PbreaderNetworkParser> networkParser_;
318     ClockId dataSourceTypeNetworkClockid_ = TS_CLOCK_UNKNOW;
319     DataIndex networkPluginIndex_;
320 #endif
321 #ifdef ENABLE_DISKIO
322     std::unique_ptr<PbreaderDiskIOParser> diskIOParser_;
323     ClockId dataSourceTypeDiskioClockid_ = TS_CLOCK_UNKNOW;
324     DataIndex diskioPluginIndex_;
325 #endif
326 #ifdef ENABLE_PROCESS
327     std::unique_ptr<PbreaderProcessParser> processParser_;
328     ClockId dataSourceTypeProcessClockid_ = TS_CLOCK_UNKNOW;
329     DataIndex processPluginIndex_;
330 #endif
331 #ifdef ENABLE_HISYSEVENT
332     std::unique_ptr<PbreaderHisyseventParser> hisyseventParser_;
333     ClockId dataSourceTypeHisyseventClockid_ = TS_CLOCK_UNKNOW;
334     DataIndex hisyseventPluginIndex_;
335     DataIndex hisyseventPluginConfigIndex_;
336 #endif
337 #ifdef ENABLE_ARKTS
338     std::unique_ptr<PbreaderJSMemoryParser> jsMemoryParser_;
339     ClockId dataSourceTypeJSMemoryClockid_ = TS_CLOCK_UNKNOW;
340 #endif
341 #ifdef ENABLE_HIPERF
342     uint64_t perfProcessedLen_ = 0;
343     std::unique_ptr<PerfDataParser> perfDataParser_;
344 #endif
345 #ifdef ENABLE_EBPF
346     std::unique_ptr<EbpfDataParser> ebpfDataParser_;
347     bool hasInitEbpfPublicData_ = false;
348     bool parsedEbpfOver_ = false;
349 #endif
350 #ifdef ENABLE_NATIVE_HOOK
351     std::unique_ptr<PbreaderNativeHookParser> pbreaderNativeHookParser_;
352     ClockId dataSourceTypeNativeHookClockid_ = TS_CLOCK_UNKNOW;
353     std::set<DataIndex> nativeHookPluginIndex_ = {};
354     DataIndex nativeHookConfigIndex_;
355 #endif
356 #ifdef ENABLE_HILOG
357     std::set<DataIndex> hilogPluginIndex_ = {};
358     std::unique_ptr<PbreaderHiLogParser> pbreaderHiLogParser_;
359     ClockId dataSourceTypeHilogClockid_ = TS_CLOCK_UNKNOW;
360 #endif
361 #ifdef ENABLE_XPOWER
362     std::unique_ptr<PbreaderXpowerParser> xpowerParser_;
363     DataIndex xpowerPluginIndex_;
364 #endif
365 #ifdef ENABLE_STREAM_EXTEND
366     DataIndex streamPluginIndex_;
367     std::unique_ptr<PbreaderStreamParser> pbreaderStreamParser_;
368 #endif
369     std::unique_ptr<PbreaderDataSegment[]> dataSegArray_;
370     std::atomic<bool> filterThreadStarted_{false};
371     const int32_t maxSegArraySize = 10000;
372     int32_t rawDataHead_ = 0;
373     bool toExit_ = false;
374     bool exited_ = false;
375     int32_t filterHead_ = 0;
376     int32_t parseHead_ = 0;
377     size_t pbreaderLength_ = 1024;
378     const int32_t sleepDur_ = 100;
379     bool parseThreadStarted_ = false;
380     int32_t parserThreadCount_ = 0;
381     std::mutex pbreaderDataSegMux_ = {};
382     std::map<int32_t, int32_t> mPbreaderSplitData_ = {};
383     uint64_t splitFileOffset_ = 0;
384     uint64_t processedDataLen_ = 0;
385     uint64_t parsedFileOffset_ = 0;
386     uint32_t dataSourceType_ = INVALID_UINT32;
387     std::string arkTsConfigData_ = "";
388     std::string lenBuffer_ = "";
389     DataIndex arktsPluginIndex_;
390     DataIndex arktsPluginConfigIndex_;
391     std::set<DataIndex> supportPluginNameIndex_ = {};
392 };
393 } // namespace TraceStreamer
394 } // namespace SysTuning
395 
396 #endif // HTRACE_PARSER_H_
397