• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
3  * Licensed under the Apache License, Version 2.0 (the "License");
4  * you may not use this file except in compliance with the License.
5  * You may obtain a copy of the License at
6  *
7  * http://www.apache.org/licenses/LICENSE-2.0
8  *
9  * Unless required by applicable law or agreed to in writing, software
10  * distributed under the License is distributed on an "AS IS" BASIS,
11  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12  * See the License for the specific language governing permissions and
13  * limitations under the License.
14  */
15 
16 #ifndef PBREADER_PARSER_H
17 #define PBREADER_PARSER_H
18 #include <cstdint>
19 #include <limits>
20 #include <map>
21 #include <queue>
22 #include <stdexcept>
23 #include <string>
24 #include <thread>
25 #include <unordered_map>
26 #include <unordered_set>
27 #include "common_types.h"
28 #include "common_types.pbreader.h"
29 #include "clock_filter_ex.h"
30 #ifdef ENABLE_EBPF
31 #include "ebpf_data_parser.h"
32 #endif
33 #include "file.h"
34 #include "pbreader_clock_detail_parser.h"
35 #ifdef ENABLE_HTRACE
36 #include "htrace_cpu_detail_parser.h"
37 #include "htrace_symbols_detail_parser.h"
38 #endif
39 #ifdef ENABLE_FFRT
40 #include "pbreader_ffrt_parser.h"
41 #endif
42 #include "htrace_plugin_time_parser.h"
43 #ifdef ENABLE_CPUDATA
44 #include "pbreader_cpu_data_parser.h"
45 #endif
46 #ifdef ENABLE_DISKIO
47 #include "pbreader_disk_io_parser.h"
48 #endif
49 #ifdef ENABLE_HTDUMP
50 #include "pbreader_hidump_parser.h"
51 #endif
52 #ifdef ENABLE_HILOG
53 #include "pbreader_hilog_parser.h"
54 #endif
55 #ifdef ENABLE_HISYSEVENT
56 #include "pbreader_hisysevent_parser.h"
57 #endif
58 #ifdef ENABLE_ARKTS
59 #include "pbreader_js_memory_parser.h"
60 #endif
61 #ifdef ENABLE_MEMORY
62 #include "pbreader_mem_parser.h"
63 #endif
64 #ifdef ENABLE_NATIVE_HOOK
65 #include "pbreader_native_hook_parser.h"
66 #endif
67 #ifdef ENABLE_NETWORK
68 #include "pbreader_network_parser.h"
69 #endif
70 #ifdef ENABLE_PROCESS
71 #include "pbreader_process_parser.h"
72 #endif
73 #include "parser_base.h"
74 #include "pbreader_file_header.h"
75 #ifdef ENABLE_HIPERF
76 #include "perf_data_parser.h"
77 #endif
78 #include "proto_reader_help.h"
79 #include "string_help.h"
80 #include "symbols_file.h"
81 #include "trace_data_cache.h"
82 #include "trace_streamer_filters.h"
83 #include "ts_common.h"
84 #ifdef ENABLE_STREAM_EXTEND
85 #include "pbreader_stream_parser.h"
86 #endif
87 
88 namespace SysTuning {
89 namespace TraceStreamer {
90 using namespace SysTuning::base;
91 using namespace OHOS::Developtools::HiPerf;
92 #if defined(ENABLE_HTRACE) && defined(ENABLE_NATIVE_HOOK) && defined(ENABLE_HIPERF)
93 struct SliceInfo {
SliceInfoSliceInfo94     SliceInfo(uint64_t tsBegin, uint64_t tsEnd, const std::string &traceid)
95         : tsBegin_(tsBegin), tsEnd_(tsEnd), traceid_(traceid){};
96     uint64_t tsBegin_ = INVALID_UINT64;
97     uint64_t tsEnd_ = INVALID_UINT64;
98     std::string traceid_;
99 };
100 #endif
101 class PbreaderParser : public ParserBase, public HtracePluginTimeParser {
102 public:
103     PbreaderParser(TraceDataCache *dataCache, const TraceStreamerFilters *filters);
104     ~PbreaderParser();
105     void ParseTraceDataSegment(std::unique_ptr<uint8_t[]> bufferStr, size_t size, bool isFinish = false) override;
106     bool ReparseSymbolFilesAndResymbolization(std::string &symbolsPath, std::vector<std::string> &symbolsPaths);
107     void WaitForParserEnd();
108 #ifdef ENABLE_ARKTS
109     void EnableFileSeparate(bool enabled);
110 #endif
111 #if defined(ENABLE_HIPERF) || defined(ENABLE_NATIVE_HOOK) || defined(ENABLE_EBPF)
112     void ParserFileSO(std::string &directory, const std::vector<std::string> &relativeFilePaths);
113 #endif
114 #ifdef ENABLE_HIPERF
115     void TraceDataSegmentEnd(bool isSplitFile);
116     void StoreTraceDataSegment(std::unique_ptr<uint8_t[]> bufferStr, size_t size, int32_t isFinish);
117 #endif
GetPbreaderSplitData()118     const auto &GetPbreaderSplitData()
119     {
120         return mPbreaderSplitData_;
121     }
GetProfilerHeader()122     auto GetProfilerHeader()
123     {
124         return profilerTraceFileHeader_;
125     }
126 #ifdef ENABLE_NATIVE_HOOK
ClearNativehookData()127     auto ClearNativehookData()
128     {
129         pbreaderNativeHookParser_->FinishSplitNativeHook();
130     }
131 #endif
GetDataSourceType()132     auto GetDataSourceType()
133     {
134         return dataSourceType_;
135     }
136 #ifdef ENABLE_ARKTS
GetJsMemoryData()137     auto GetJsMemoryData()
138     {
139         return jsMemoryParser_.get();
140     }
GetArkTsConfigData()141     auto GetArkTsConfigData()
142     {
143         return arkTsConfigData_;
144     }
145 #endif
ClearPbreaderSplitData()146     auto ClearPbreaderSplitData()
147     {
148 #ifdef ENABLE_HIPERF
149         perfDataParser_->ClearPerfSplitResult();
150         perfProcessedLen_ = 0;
151 #endif
152 #ifdef ENABLE_EBPF
153         ebpfDataParser_->ClearEbpfSplitResult();
154         hasInitEbpfPublicData_ = false;
155         parsedEbpfOver_ = false;
156 #endif
157         processedDataLen_ = 0;
158         splitFileOffset_ = 0;
159         hasGotHeader_ = false;
160         return mPbreaderSplitData_.clear();
161     }
162 #ifdef ENABLE_HIPERF
GetPerfSplitResult()163     const auto &GetPerfSplitResult()
164     {
165         return perfDataParser_->GetPerfSplitResult();
166     }
167 #endif
168 #ifdef ENABLE_EBPF
GetEbpfDataParser()169     const auto &GetEbpfDataParser()
170     {
171         return ebpfDataParser_;
172     }
173 #endif
174 #ifdef ENABLE_HTRACE
EnableOnlyParseFtrace()175     void EnableOnlyParseFtrace()
176     {
177         onlyParseFtrace_ = true;
178     }
179 #endif
180 
181 private:
182     bool ParseDataRecursively(std::deque<uint8_t>::iterator &packagesBegin, size_t &currentLength);
183 #ifdef ENABLE_HIPERF
184     bool ParseHiperfData(std::deque<uint8_t>::iterator &packagesBegin, size_t &currentLength);
185 #endif
186     void ParseTraceDataItem(const std::string &buffer) override;
187     void FilterData(PbreaderDataSegment &seg, bool isSplitFile);
188     void ParserData(PbreaderDataSegment &dataSeg, bool isSplitFile);
189 
190 private:
191 #if IS_WASM
192     bool ParseSDKData();
193 #endif
194     void InitPluginNameIndex();
195     void InitHookPluginNameIndex();
196     void InitMemoryPluginNameIndex();
197     void InitHiPluginNameIndex();
198     void WaitForHPluginParserEnd();
199     void WaitForOtherPluginParserEnd();
200     bool GetHeaderAndUpdateLengthMark(std::deque<uint8_t>::iterator &packagesBegin, size_t &currentLength);
201     bool ParseSegLengthAndEnsureSegDataEnough(std::deque<uint8_t>::iterator &packagesBegin, size_t &currentLength);
202 #ifdef ENABLE_MEMORY
203     void ParseMemory(const ProtoReader::ProfilerPluginData_Reader &pluginDataZero, PbreaderDataSegment &dataSeg);
204     void ParseMemoryConfig(PbreaderDataSegment &dataSeg, const ProtoReader::ProfilerPluginData_Reader &pluginDataZero);
205 #endif
206 #ifdef ENABLE_HILOG
207     void ParseHilog(PbreaderDataSegment &dataSeg);
208 #endif
209 #ifdef ENABLE_HTRACE
210     void ParseFtrace(PbreaderDataSegment &dataSeg);
211 #endif
212 #ifdef ENABLE_FFRT
213     void ParseFfrtConfig(PbreaderDataSegment &dataSeg);
214     void ParseFfrt(PbreaderDataSegment &dataSeg);
215 #endif
216 #ifdef ENABLE_HTDUMP
217     void ParseFPS(PbreaderDataSegment &dataSeg);
218 #endif
219 #ifdef ENABLE_CPUDATA
220     void ParseCpuUsage(PbreaderDataSegment &dataSeg);
221 #endif
222 #ifdef ENABLE_NETWORK
223     void ParseNetwork(PbreaderDataSegment &dataSeg);
224 #endif
225 #ifdef ENABLE_DISKIO
226     void ParseDiskIO(PbreaderDataSegment &dataSeg);
227 #endif
228 #ifdef ENABLE_PROCESS
229     void ParseProcess(PbreaderDataSegment &dataSeg);
230 #endif
231 #ifdef ENABLE_HISYSEVENT
232     void ParseHisysevent(PbreaderDataSegment &dataSeg);
233     void ParseHisyseventConfig(PbreaderDataSegment &dataSeg);
234 #endif
235 #ifdef ENABLE_NATIVE_HOOK
236     void ParseNativeHookConfig(PbreaderDataSegment &dataSeg);
237     void ParseNativeHook(PbreaderDataSegment &dataSeg, bool isSplitFile);
238 #endif
239 #ifdef ENABLE_ARKTS
240     void ParseJSMemory(const ProtoReader::ProfilerPluginData_Reader &pluginDataZero,
241                        PbreaderDataSegment &dataSeg,
242                        bool isSplitFile);
243     void ParseJSMemoryConfig(PbreaderDataSegment &dataSeg);
244 #endif
245 #ifdef ENABLE_STREAM_EXTEND
246     void ParseStream(PbreaderDataSegment &dataSeg);
247 #endif
248     void ParseThread();
249     int32_t GetNextSegment();
250     void FilterThread();
251 #ifdef ENABLE_EBPF
252     bool CalcEbpfCutOffset(std::deque<uint8_t>::iterator &packagesBegin, size_t &currentLength);
253 #endif
254     bool SpliteConfigData(const std::string &pluginName, const PbreaderDataSegment &dataSeg);
255     bool InitProfilerTraceFileHeader();
256     void ParseDataByPluginName(PbreaderDataSegment &dataSeg,
257                                DataIndex pulginNameIndex,
258                                const ProtoReader::ProfilerPluginData_Reader &pluginDataZero,
259                                bool isSplitFile);
260     bool SpliteDataBySegment(DataIndex pluginNameIndex, PbreaderDataSegment &dataSeg);
261 #if defined(ENABLE_HTRACE) && defined(ENABLE_NATIVE_HOOK) && defined(ENABLE_HIPERF)
262     void ParseNapiAsync();
263     void GetTraceidInfoFromCallstack(const std::unordered_map<std::string, uint32_t> &traceidToCallchainidMap,
264                                      std::unordered_map<uint64_t, std::queue<SliceInfo>> &itidToCallstackIdsMap);
265     void GetTraceidInfoFromNativeHook(std::unordered_map<std::string, uint32_t> &traceidToCallchainidMap);
266     void GetCallchainIdSetFromHiperf(std::unordered_set<uint32_t> &callchainIdSet);
267     void DumpDataFromHiperf(const std::unordered_map<std::string, uint32_t> &traceidToCallchainidMap,
268                             const std::unordered_set<uint32_t> &callchainIdSet,
269                             std::unordered_map<uint64_t, std::queue<SliceInfo>> &itidToCallstackIdsMap);
270 #endif
271     ProfilerTraceFileHeader profilerTraceFileHeader_;
272     uint32_t profilerDataType_ = ProfilerTraceFileHeader::UNKNOW_TYPE;
273     uint64_t profilerDataLength_ = 0;
274     ProfilerPluginDataHeader profilerPluginData_;
275     uint64_t pbreaderCurentLength_ = 0;
276     char standalonePluginName_[ProfilerTraceFileHeader::PLUGIN_MODULE_NAME_MAX + 1] = "";
277     bool hasGotSegLength_ = false;
278     bool hasGotHeader_ = false;
279     uint32_t nextLength_ = 0;
280     const size_t packetSegLength_ = 4;
281     const size_t packetHeaderLength_ = 1024;
282     TraceDataCache *traceDataCache_;
283     std::unique_ptr<PbreaderClockDetailParser> pbreaderClockDetailParser_;
284 #ifdef ENABLE_HTRACE
285     std::unique_ptr<HtraceCpuDetailParser> htraceCpuDetailParser_;
286     std::unique_ptr<HtraceSymbolsDetailParser> htraceSymbolsDetailParser_;
287     ClockId dataSourceTypeTraceClockid_ = TS_CLOCK_UNKNOW;
288     bool onlyParseFtrace_ = false;
289 #endif
290     std::set<DataIndex> ftracePluginIndex_ = {};
291 #ifdef ENABLE_FFRT
292     DataIndex ffrtPluginIndex_ = {};
293     DataIndex ffrtPluginConfigIndex_ = {};
294     std::unique_ptr<PbreaderFfrtDetailParser> pbreaderFfrtParser_;
295 #endif
296 #ifdef ENABLE_MEMORY
297     std::unique_ptr<PbreaderMemParser> pbreaderMemParser_;
298     ClockId dataSourceTypeMemClockid_ = TS_CLOCK_UNKNOW;
299     DataIndex memPluginIndex_;
300     DataIndex memoryPluginConfigIndex_;
301 #endif
302 #ifdef ENABLE_HTDUMP
303     std::unique_ptr<PbreaderHidumpParser> pbreaderHidumpParser_;
304     std::set<DataIndex> hidumpPluginIndex_ = {};
305     ClockId dataSourceTypeFpsClockid_ = TS_CLOCK_UNKNOW;
306 #endif
307 #ifdef ENABLE_CPUDATA
308     std::unique_ptr<PbreaderCpuDataParser> cpuUsageParser_;
309     ClockId dataSourceTypeCpuClockid_ = TS_CLOCK_UNKNOW;
310     DataIndex cpuPluginIndex_;
311 #endif
312 #ifdef ENABLE_NETWORK
313     std::unique_ptr<PbreaderNetworkParser> networkParser_;
314     ClockId dataSourceTypeNetworkClockid_ = TS_CLOCK_UNKNOW;
315     DataIndex networkPluginIndex_;
316 #endif
317 #ifdef ENABLE_DISKIO
318     std::unique_ptr<PbreaderDiskIOParser> diskIOParser_;
319     ClockId dataSourceTypeDiskioClockid_ = TS_CLOCK_UNKNOW;
320     DataIndex diskioPluginIndex_;
321 #endif
322 #ifdef ENABLE_PROCESS
323     std::unique_ptr<PbreaderProcessParser> processParser_;
324     ClockId dataSourceTypeProcessClockid_ = TS_CLOCK_UNKNOW;
325     DataIndex processPluginIndex_;
326 #endif
327 #ifdef ENABLE_HISYSEVENT
328     std::unique_ptr<PbreaderHisyseventParser> hisyseventParser_;
329     ClockId dataSourceTypeHisyseventClockid_ = TS_CLOCK_UNKNOW;
330     DataIndex hisyseventPluginIndex_;
331     DataIndex hisyseventPluginConfigIndex_;
332 #endif
333 #ifdef ENABLE_ARKTS
334     std::unique_ptr<PbreaderJSMemoryParser> jsMemoryParser_;
335     ClockId dataSourceTypeJSMemoryClockid_ = TS_CLOCK_UNKNOW;
336 #endif
337 #ifdef ENABLE_HIPERF
338     uint64_t perfProcessedLen_ = 0;
339     std::unique_ptr<PerfDataParser> perfDataParser_;
340 #endif
341 #ifdef ENABLE_EBPF
342     std::unique_ptr<EbpfDataParser> ebpfDataParser_;
343     bool hasInitEbpfPublicData_ = false;
344     bool parsedEbpfOver_ = false;
345 #endif
346 #ifdef ENABLE_NATIVE_HOOK
347     std::unique_ptr<PbreaderNativeHookParser> pbreaderNativeHookParser_;
348     ClockId dataSourceTypeNativeHookClockid_ = TS_CLOCK_UNKNOW;
349     std::set<DataIndex> nativeHookPluginIndex_ = {};
350     DataIndex nativeHookConfigIndex_;
351 #endif
352 #ifdef ENABLE_HILOG
353     std::set<DataIndex> hilogPluginIndex_ = {};
354     std::unique_ptr<PbreaderHiLogParser> pbreaderHiLogParser_;
355     ClockId dataSourceTypeHilogClockid_ = TS_CLOCK_UNKNOW;
356 #endif
357 #ifdef ENABLE_STREAM_EXTEND
358     DataIndex streamPluginIndex_;
359     std::unique_ptr<PbreaderStreamParser> pbreaderStreamParser_;
360 #endif
361     std::unique_ptr<PbreaderDataSegment[]> dataSegArray_;
362     std::atomic<bool> filterThreadStarted_{false};
363     const int32_t maxSegArraySize = 10000;
364     int32_t rawDataHead_ = 0;
365     bool toExit_ = false;
366     bool exited_ = false;
367     int32_t filterHead_ = 0;
368     int32_t parseHead_ = 0;
369     size_t pbreaderLength_ = 1024;
370     const int32_t sleepDur_ = 100;
371     bool parseThreadStarted_ = false;
372     int32_t parserThreadCount_ = 0;
373     std::mutex pbreaderDataSegMux_ = {};
374     std::vector<std::unique_ptr<SymbolsFile>> symbolsFiles_;
375     std::map<int32_t, int32_t> mPbreaderSplitData_ = {};
376     uint64_t splitFileOffset_ = 0;
377     uint64_t processedDataLen_ = 0;
378     uint64_t parsedFileOffset_ = 0;
379     uint32_t dataSourceType_ = INVALID_UINT32;
380     std::string arkTsConfigData_ = "";
381     std::string lenBuffer_ = "";
382     DataIndex arktsPluginIndex_;
383     DataIndex arktsPluginConfigIndex_;
384     std::set<DataIndex> supportPluginNameIndex_ = {};
385 };
386 } // namespace TraceStreamer
387 } // namespace SysTuning
388 
389 #endif // HTRACE_PARSER_H_
390