1 /*
2 * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "rawtrace_parser.h"
17 #include <cinttypes>
18 #if IS_WASM
19 #include "wasm_func.h"
20 #endif
21 #include "string_help.h"
22 namespace SysTuning {
23 namespace TraceStreamer {
RawTraceParser(TraceDataCache * dataCache,const TraceStreamerFilters * filters)24 RawTraceParser::RawTraceParser(TraceDataCache *dataCache, const TraceStreamerFilters *filters)
25 : ParserBase(filters),
26 cpuDetail_(std::make_unique<FtraceCpuDetailMsg>()),
27 cpuDetailParser_(std::make_unique<CpuDetailParser>(dataCache, filters)),
28 ftraceProcessor_(std::make_unique<FtraceProcessor>(dataCache)),
29 ksymsProcessor_(std::make_unique<KernelSymbolsProcessor>(dataCache, filters)),
30 traceDataCache_(dataCache)
31 {
32 }
33
~RawTraceParser()34 RawTraceParser::~RawTraceParser() {}
ParseTraceDataItem(const std::string & buffer)35 void RawTraceParser::ParseTraceDataItem(const std::string &buffer) {}
WaitForParserEnd()36 void RawTraceParser::WaitForParserEnd()
37 {
38 cpuDetailParser_->FilterAllEvents(*cpuDetail_.get(), true);
39 cpuDetailParser_->FinishCpuDetailParser();
40 UpdateTraceMinRange();
41 restCommDataCnt_ = 0;
42 hasGotHeader_ = false;
43 curCpuCoreNum_ = 0;
44 auto tidToName = ftraceProcessor_->GetTidToName();
45 streamFilters_->processFilter_->UpdateProcessNameByNameToTid(tidToName);
46 ClearRawTraceData();
47 TS_LOGI("Parser raw trace end!");
48 }
UpdateTraceMinRange()49 void RawTraceParser::UpdateTraceMinRange()
50 {
51 if (!streamFilters_->configFilter_->GetSwitchConfig().RawTraceCutStartTsEnabled()) {
52 return;
53 }
54 auto schedSlice = traceDataCache_->GetConstSchedSliceData();
55 std::set<uint32_t> uniqueCpuIdSet;
56 uint64_t cpuRunningStatMinTime = INVALID_TIME;
57 for (size_t i = 0; i < schedSlice.Size() && uniqueCpuIdSet.size() <= curCpuCoreNum_; i++) {
58 auto iter = uniqueCpuIdSet.find(schedSlice.CpusData()[i]);
59 if (iter != uniqueCpuIdSet.end()) {
60 continue;
61 }
62 uniqueCpuIdSet.emplace(schedSlice.CpusData()[i]);
63 cpuRunningStatMinTime = schedSlice.TimeStampData()[i];
64 TS_LOGW("curCpuId=%u, cpuRunningStatMinTime=%" PRIu64 "", schedSlice.CpusData()[i], cpuRunningStatMinTime);
65 }
66 if (cpuRunningStatMinTime != INVALID_TIME) {
67 traceDataCache_->UpdateTraceMinTime(cpuRunningStatMinTime);
68 }
69 }
InitRawTraceFileHeader(std::deque<uint8_t>::iterator & packagesCurIter)70 bool RawTraceParser::InitRawTraceFileHeader(std::deque<uint8_t>::iterator &packagesCurIter)
71 {
72 TS_CHECK_TRUE(packagesBuffer_.size() >= sizeof(RawTraceFileHeader), false,
73 "buffer size less than rawtrace file header");
74 RawTraceFileHeader header;
75 std::copy(packagesBuffer_.begin(), packagesBuffer_.begin() + sizeof(RawTraceFileHeader),
76 reinterpret_cast<uint8_t *>(&header));
77 TS_LOGI("magicNumber=%d fileType=%d", header.magicNumber, header.fileType);
78
79 fileType_ = header.fileType;
80 if (traceDataCache_->isSplitFile_) {
81 // To resolve the second incoming file, it is necessary to reset the previously set variables to zero
82 ClearRawTraceData();
83 rawTraceSplitCommData_.emplace_back(SpliteDataInfo(curFileOffset_, sizeof(RawTraceFileHeader)));
84 curFileOffset_ += sizeof(RawTraceFileHeader);
85 }
86 packagesCurIter += sizeof(RawTraceFileHeader);
87 packagesCurIter = packagesBuffer_.erase(packagesBuffer_.begin(), packagesCurIter);
88 hasGotHeader_ = true;
89 return true;
90 }
InitEventFormats(const std::string & buffer)91 bool RawTraceParser::InitEventFormats(const std::string &buffer)
92 {
93 #ifdef IS_WASM
94 if (!isWasmReadFile_) {
95 restCommDataCnt_ = INVALID_UINT8; // ensure that the restCommData is parsed only once
96 }
97 #endif
98 std::string line;
99 std::istringstream iss(buffer);
100 std::stringstream eventFormat;
101 while (std::getline(iss, line)) {
102 eventFormat << line << '\n';
103 if (base::StartWith(line, eventEndCmd_)) {
104 ftraceProcessor_->SetupEvent(eventFormat.str());
105 eventFormat.str("");
106 }
107 }
108 return true;
109 }
UpdateCpuCoreMax(uint32_t cpuId)110 bool RawTraceParser::UpdateCpuCoreMax(uint32_t cpuId)
111 {
112 if (cpuId >= curCpuCoreNum_) {
113 curCpuCoreNum_++;
114 TS_LOGI("cpuId=%u, curCpuCoreNum_=%u", cpuId, curCpuCoreNum_);
115 return false;
116 }
117 if (cpuDetailParser_->cpuCoreMax_ == CPU_CORE_MAX) {
118 cpuDetailParser_->ResizeStandAloneCpuEventList(curCpuCoreNum_);
119 }
120 return true;
121 }
122
ParseCpuRawData(uint32_t cpuId,const std::string & buffer,uint32_t curType)123 bool RawTraceParser::ParseCpuRawData(uint32_t cpuId, const std::string &buffer, uint32_t curType)
124 {
125 UpdateCpuCoreMax(cpuId);
126 // splice the data curType adn size of each cup that matches the timestamp
127 uint32_t curFileOffset = curFileOffset_ + sizeof(curType) + sizeof(uint32_t);
128 uint32_t splitOffset = 0;
129 uint32_t splitSize = 0;
130 bool isSplitPosition = false;
131 if (0 == buffer.size() && traceDataCache_->isSplitFile_) {
132 // For rawtrace. fileType_=0, in order to count the number of CPUs and maintain the CPU data structure (which
133 // will also be passed to data types with CPU size 0), it is necessary to save the data during the cutting
134 // process and exit the buffer directly.
135 rawTraceSplitCpuData_.emplace_back(SpliteDataInfo(curFileOffset, 0, curType));
136 }
137 TS_CHECK_TRUE(buffer.size() > 0, true, "cur cpu(%u) raw data is null!", cpuId);
138 auto startPtr = reinterpret_cast<const uint8_t *>(buffer.c_str());
139 auto endPtr = startPtr + buffer.size();
140 cpuDetail_->set_cpu(cpuId);
141 for (uint8_t *page = const_cast<uint8_t *>(startPtr); page < endPtr; page += FTRACE_PAGE_SIZE) {
142 bool haveSplitSeg = false;
143 TS_CHECK_TRUE(ftraceProcessor_->HandlePage(*cpuDetail_.get(), *cpuDetailParser_.get(), page, haveSplitSeg),
144 false, "handle page failed!");
145 if (haveSplitSeg) {
146 splitSize += FTRACE_PAGE_SIZE;
147 if (!isSplitPosition) {
148 // splitOffset = first Save the migration amount of CPURAW that currently matches the timestamp
149 isSplitPosition = true;
150 splitOffset = curFileOffset;
151 }
152 }
153 curFileOffset += FTRACE_PAGE_SIZE;
154 }
155 if (traceDataCache_->isSplitFile_) {
156 // Skip parsing data for timestamp or non timestamp compliant data
157 if (splitSize > 0) {
158 rawTraceSplitCpuData_.emplace_back(SpliteDataInfo(splitOffset, splitSize, curType));
159 } else {
160 // For rawtrace. fileType_=0,In order to count the number of CPUs and maintain the CPU data structure (also
161 // through For CPU data types with a size of 0, it is necessary to set the size to 0 during the cutting
162 // process to save CPU data that does not meet the cutting event stamp
163 rawTraceSplitCpuData_.emplace_back(SpliteDataInfo(curFileOffset, 0, curType));
164 }
165 return true;
166 }
167 if (cpuDetailParser_->cpuCoreMax_ != CPU_CORE_MAX) {
168 cpuDetailParser_->FilterAllEvents(*cpuDetail_.get());
169 }
170 return true;
171 }
172
HmParseCpuRawData(const std::string & buffer,uint32_t curType)173 bool RawTraceParser::HmParseCpuRawData(const std::string &buffer, uint32_t curType)
174 {
175 TS_CHECK_TRUE(buffer.size() > 0, true, "hm raw data is null!");
176 auto startPtr = reinterpret_cast<const uint8_t *>(buffer.c_str());
177 auto endPtr = startPtr + buffer.size();
178 // splice the data curType adn size of each cup that matches the timestamp
179 uint32_t curFileOffset = curFileOffset_ + sizeof(curType) + sizeof(uint32_t);
180 uint32_t splitOffset = 0;
181 uint32_t splitSize = 0;
182 bool isSplitPosition = false;
183 for (uint8_t *data = const_cast<uint8_t *>(startPtr); data < endPtr; data += FTRACE_PAGE_SIZE) {
184 bool haveSplitSeg = false;
185 TS_CHECK_TRUE(ftraceProcessor_->HmParsePageData(*cpuDetail_.get(), *cpuDetailParser_.get(), data, haveSplitSeg),
186 false, "hm parse page failed!");
187 if (haveSplitSeg) {
188 splitSize += FTRACE_PAGE_SIZE;
189 if (!isSplitPosition) {
190 // splitOffset = first Save the migration amount of CPURAW that currently matches the timestamp
191 isSplitPosition = true;
192 splitOffset = curFileOffset;
193 }
194 }
195 if (!traceDataCache_->isSplitFile_) {
196 // No specific analysis is required for time cutting
197 cpuDetailParser_->FilterAllEvents(*cpuDetail_.get());
198 }
199 curFileOffset += FTRACE_PAGE_SIZE;
200 }
201 if (traceDataCache_->isSplitFile_ && splitSize > 0) {
202 rawTraceSplitCpuData_.emplace_back(SpliteDataInfo(splitOffset, splitSize, curType));
203 // For rawtrace. fileType_=1,There is no need to record the total number of CPUs, so for data that does not meet
204 // the cutting timestamp, there is no need to record and save it
205 return true;
206 }
207 TS_LOGD("mark.debug. HmParseCpuRawData end success");
208 return true;
209 }
210
ParseLastCommData(uint8_t type,const std::string & buffer)211 bool RawTraceParser::ParseLastCommData(uint8_t type, const std::string &buffer)
212 {
213 TS_CHECK_TRUE_RET(restCommDataCnt_ != INVALID_UINT8, false);
214 switch (type) {
215 case static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_CMDLINES):
216 TS_CHECK_TRUE(ftraceProcessor_->HandleCmdlines(buffer), false, "parse cmdlines failed");
217 break;
218 case static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_TGIDS):
219 TS_CHECK_TRUE(ftraceProcessor_->HandleTgids(buffer), false, "parse tgid failed");
220 break;
221 case static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_HEADER_PAGE):
222 TS_CHECK_TRUE(ftraceProcessor_->HandleHeaderPageFormat(buffer), false, "init header page failed");
223 break;
224 case static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_PRINTK_FORMATS):
225 TS_CHECK_TRUE(PrintkFormatsProcessor::GetInstance().HandlePrintkSyms(buffer), false,
226 "init printk_formats failed");
227 break;
228 case static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_KALLSYMS):
229 TS_CHECK_TRUE(ksymsProcessor_->HandleKallSyms(buffer), false, "init printk_formats failed");
230 break;
231 default:
232 #ifdef IS_WASM
233 if (!isWasmReadFile_) {
234 return false;
235 }
236 #endif
237 break;
238 }
239 ++restCommDataCnt_;
240 return true;
241 }
242
ParseTraceDataSegment(std::unique_ptr<uint8_t[]> bufferStr,size_t size,bool isFinish)243 void RawTraceParser::ParseTraceDataSegment(std::unique_ptr<uint8_t[]> bufferStr, size_t size, bool isFinish)
244 {
245 packagesBuffer_.insert(packagesBuffer_.end(), &bufferStr[0], &bufferStr[size]);
246 auto packagesCurIter = packagesBuffer_.begin();
247 if (ParseDataRecursively(packagesCurIter)) {
248 packagesCurIter = packagesBuffer_.erase(packagesBuffer_.begin(), packagesCurIter);
249 }
250 if (isFinish) {
251 restCommDataCnt_ = INVALID_UINT8;
252 hasGotHeader_ = false;
253 packagesBuffer_.clear();
254 }
255 return;
256 }
257
ProcessRawTraceContent(std::string & bufferLine,uint8_t curType)258 bool RawTraceParser::ProcessRawTraceContent(std::string &bufferLine, uint8_t curType)
259 {
260 if (curType >= static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_CPU_RAW) &&
261 curType < static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_HEADER_PAGE)) {
262 curType = static_cast<uint32_t>(curType);
263 if (fileType_ == static_cast<uint8_t>(RawTraceFileType::FILE_RAW_TRACE)) {
264 auto cpuId = curType - static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_CPU_RAW);
265 TS_CHECK_TRUE(ParseCpuRawData(cpuId, bufferLine, curType), false, "cpu raw parse failed");
266 } else if (fileType_ == static_cast<uint8_t>(RawTraceFileType::HM_FILE_RAW_TRACE)) {
267 TS_CHECK_TRUE(HmParseCpuRawData(bufferLine, curType), false, "hm raw trace parse failed");
268 }
269 if (traceDataCache_->isSplitFile_) {
270 curFileOffset_ += sizeof(uint32_t) + sizeof(uint32_t) + bufferLine.size();
271 }
272 } else if (curType == static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_EVENTS_FORMAT)) {
273 TS_CHECK_TRUE(InitEventFormats(bufferLine), false, "init event format failed");
274 } else {
275 TS_LOGW("Raw Trace Type(%d) Unknown or has been parsed.", curType);
276 }
277 return true;
278 }
ParseDataRecursively(std::deque<uint8_t>::iterator & packagesCurIter)279 bool RawTraceParser::ParseDataRecursively(std::deque<uint8_t>::iterator &packagesCurIter)
280 {
281 uint32_t type = 0;
282 uint32_t len = 0;
283 if (!hasGotHeader_) {
284 TS_CHECK_TRUE(InitRawTraceFileHeader(packagesCurIter), false, "get rawtrace file header failed");
285 }
286 while (true) {
287 std::copy(packagesCurIter, packagesCurIter + sizeof(type), reinterpret_cast<uint8_t *>(&type));
288 packagesCurIter += sizeof(type);
289 std::copy(packagesCurIter, packagesCurIter + sizeof(len), reinterpret_cast<uint8_t *>(&len));
290 packagesCurIter += sizeof(len);
291 uint32_t restDataLen = std::distance(packagesCurIter, packagesBuffer_.end());
292 TS_CHECK_TRUE_RET(len <= restDataLen && packagesBuffer_.size() > 0, false);
293 std::string bufferLine(packagesCurIter, packagesCurIter + len);
294 packagesCurIter += len;
295 packagesCurIter = packagesBuffer_.erase(packagesBuffer_.begin(), packagesCurIter);
296 uint8_t curType = static_cast<uint8_t>(type);
297 if (ParseLastCommData(curType, bufferLine)) {
298 continue;
299 }
300 // for jump first comm data
301 if (traceDataCache_->isSplitFile_ &&
302 (curType < static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_CPU_RAW) ||
303 curType >= static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_HEADER_PAGE))) {
304 uint32_t curSegSize = sizeof(type) + sizeof(len) + bufferLine.size();
305 rawTraceSplitCommData_.emplace_back(SpliteDataInfo(curFileOffset_, curSegSize));
306 curFileOffset_ += curSegSize;
307 if (curType == static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_EVENTS_FORMAT)) {
308 restCommDataCnt_ = INVALID_UINT8;
309 }
310 continue;
311 }
312 if (!ProcessRawTraceContent(bufferLine, curType)) {
313 return false;
314 }
315 }
316 return true;
317 }
318 } // namespace TraceStreamer
319 } // namespace SysTuning
320