1 /*
2 * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "rawtrace_parser.h"
17 #include <cinttypes>
18 #if IS_WASM
19 #include "wasm_func.h"
20 #endif
21 #include "string_help.h"
22 namespace SysTuning {
23 namespace TraceStreamer {
RawTraceParser(TraceDataCache * dataCache,const TraceStreamerFilters * filters)24 RawTraceParser::RawTraceParser(TraceDataCache *dataCache, const TraceStreamerFilters *filters)
25 : ParserBase(filters),
26 cpuDetail_(std::make_unique<FtraceCpuDetailMsg>()),
27 cpuDetailParser_(std::make_unique<CpuDetailParser>(dataCache, filters)),
28 ftraceProcessor_(std::make_unique<FtraceProcessor>(dataCache)),
29 ksymsProcessor_(std::make_unique<KernelSymbolsProcessor>(dataCache, filters)),
30 traceDataCache_(dataCache)
31 {
32 }
33
~RawTraceParser()34 RawTraceParser::~RawTraceParser() {}
ParseTraceDataItem(const std::string & buffer)35 void RawTraceParser::ParseTraceDataItem(const std::string &buffer) {}
WaitForParserEnd()36 void RawTraceParser::WaitForParserEnd()
37 {
38 cpuDetailParser_->FilterAllEvents(*cpuDetail_.get(), true);
39 cpuDetailParser_->FinishCpuDetailParser();
40 UpdateTraceMinRange();
41 restCommDataCnt_ = 0;
42 hasGotHeader_ = false;
43 curCpuCoreNum_ = 0;
44 ClearRawTraceData();
45 TS_LOGI("Parser raw trace end!");
46 }
UpdateTraceMinRange()47 void RawTraceParser::UpdateTraceMinRange()
48 {
49 if (!traceDataCache_->RawTraceCutStartTsEnabled()) {
50 return;
51 }
52 auto schedSlice = traceDataCache_->GetConstSchedSliceData();
53 std::set<uint32_t> uniqueCpuIdSet;
54 uint64_t cpuRunningStatMinTime = INVALID_TIME;
55 for (size_t i = 0; i < schedSlice.Size() && uniqueCpuIdSet.size() <= curCpuCoreNum_; i++) {
56 auto iter = uniqueCpuIdSet.find(schedSlice.CpusData()[i]);
57 if (iter != uniqueCpuIdSet.end()) {
58 continue;
59 }
60 uniqueCpuIdSet.emplace(schedSlice.CpusData()[i]);
61 cpuRunningStatMinTime = schedSlice.TimeStampData()[i];
62 TS_LOGW("curCpuId=%u, cpuRunningStatMinTime=%" PRIu64 "", schedSlice.CpusData()[i], cpuRunningStatMinTime);
63 }
64 if (cpuRunningStatMinTime != INVALID_TIME) {
65 traceDataCache_->UpdateTraceMinTime(cpuRunningStatMinTime);
66 }
67 }
InitRawTraceFileHeader(std::deque<uint8_t>::iterator & packagesCurIter)68 bool RawTraceParser::InitRawTraceFileHeader(std::deque<uint8_t>::iterator &packagesCurIter)
69 {
70 TS_CHECK_TRUE(packagesBuffer_.size() >= sizeof(RawTraceFileHeader), false,
71 "buffer size less than rawtrace file header");
72 RawTraceFileHeader header;
73 std::copy(packagesBuffer_.begin(), packagesBuffer_.begin() + sizeof(RawTraceFileHeader),
74 reinterpret_cast<uint8_t *>(&header));
75 TS_LOGI("magicNumber=%d fileType=%d", header.magicNumber, header.fileType);
76
77 fileType_ = header.fileType;
78 if (traceDataCache_->isSplitFile_) {
79 // To resolve the second incoming file, it is necessary to reset the previously set variables to zero
80 ClearRawTraceData();
81 rawTraceSplitCommData_.emplace_back(SpliteDataInfo(curFileOffset_, sizeof(RawTraceFileHeader)));
82 curFileOffset_ += sizeof(RawTraceFileHeader);
83 }
84 packagesCurIter += sizeof(RawTraceFileHeader);
85 packagesCurIter = packagesBuffer_.erase(packagesBuffer_.begin(), packagesCurIter);
86 hasGotHeader_ = true;
87 return true;
88 }
InitEventFormats(const std::string & buffer)89 bool RawTraceParser::InitEventFormats(const std::string &buffer)
90 {
91 #ifdef IS_WASM
92 if (!isWasmReadFile_) {
93 restCommDataCnt_ = INVALID_UINT8; // ensure that the restCommData is parsed only once
94 }
95 #endif
96 std::string line;
97 std::istringstream iss(buffer);
98 std::stringstream eventFormat;
99 while (std::getline(iss, line)) {
100 eventFormat << line << '\n';
101 if (base::StartWith(line, eventEndCmd_)) {
102 ftraceProcessor_->SetupEvent(eventFormat.str());
103 eventFormat.str("");
104 }
105 }
106 return true;
107 }
UpdateCpuCoreMax(uint32_t cpuId)108 bool RawTraceParser::UpdateCpuCoreMax(uint32_t cpuId)
109 {
110 if (cpuId >= curCpuCoreNum_) {
111 curCpuCoreNum_++;
112 TS_LOGI("cpuId=%u, curCpuCoreNum_=%u", cpuId, curCpuCoreNum_);
113 return false;
114 }
115 if (cpuDetailParser_->cpuCoreMax_ == CPU_CORE_MAX) {
116 cpuDetailParser_->ResizeStandAloneCpuEventList(curCpuCoreNum_);
117 }
118 return true;
119 }
120
ParseCpuRawData(uint32_t cpuId,const std::string & buffer,uint32_t curType)121 bool RawTraceParser::ParseCpuRawData(uint32_t cpuId, const std::string &buffer, uint32_t curType)
122 {
123 UpdateCpuCoreMax(cpuId);
124 // splice the data curType adn size of each cup that matches the timestamp
125 uint32_t curFileOffset = curFileOffset_ + sizeof(curType) + sizeof(uint32_t);
126 uint32_t splitOffset = 0;
127 uint32_t splitSize = 0;
128 bool isSplitPosition = false;
129 if (0 == buffer.size() && traceDataCache_->isSplitFile_) {
130 // For rawtrace. fileType_=0, in order to count the number of CPUs and maintain the CPU data structure (which
131 // will also be passed to data types with CPU size 0), it is necessary to save the data during the cutting
132 // process and exit the buffer directly.
133 rawTraceSplitCpuData_.emplace_back(SpliteDataInfo(curFileOffset, 0, curType));
134 }
135 TS_CHECK_TRUE(buffer.size() > 0, true, "cur cpu(%u) raw data is null!", cpuId);
136 auto startPtr = reinterpret_cast<const uint8_t *>(buffer.c_str());
137 auto endPtr = startPtr + buffer.size();
138 cpuDetail_->set_cpu(cpuId);
139 for (uint8_t *page = const_cast<uint8_t *>(startPtr); page < endPtr; page += FTRACE_PAGE_SIZE) {
140 bool haveSplitSeg = false;
141 TS_CHECK_TRUE(ftraceProcessor_->HandlePage(*cpuDetail_.get(), *cpuDetailParser_.get(), page, haveSplitSeg),
142 false, "handle page failed!");
143 if (haveSplitSeg) {
144 splitSize += FTRACE_PAGE_SIZE;
145 if (!isSplitPosition) {
146 // splitOffset = first Save the migration amount of CPURAW that currently matches the timestamp
147 isSplitPosition = true;
148 splitOffset = curFileOffset;
149 }
150 }
151 curFileOffset += FTRACE_PAGE_SIZE;
152 }
153 if (traceDataCache_->isSplitFile_) {
154 // Skip parsing data for timestamp or non timestamp compliant data
155 if (splitSize > 0) {
156 rawTraceSplitCpuData_.emplace_back(SpliteDataInfo(splitOffset, splitSize, curType));
157 } else {
158 // For rawtrace. fileType_=0,In order to count the number of CPUs and maintain the CPU data structure (also
159 // through For CPU data types with a size of 0, it is necessary to set the size to 0 during the cutting
160 // process to save CPU data that does not meet the cutting event stamp
161 rawTraceSplitCpuData_.emplace_back(SpliteDataInfo(curFileOffset, 0, curType));
162 }
163 return true;
164 }
165 if (cpuDetailParser_->cpuCoreMax_ != CPU_CORE_MAX) {
166 cpuDetailParser_->FilterAllEvents(*cpuDetail_.get());
167 }
168 return true;
169 }
170
HmParseCpuRawData(const std::string & buffer,uint32_t curType)171 bool RawTraceParser::HmParseCpuRawData(const std::string &buffer, uint32_t curType)
172 {
173 TS_CHECK_TRUE(buffer.size() > 0, true, "hm raw data is null!");
174 auto startPtr = reinterpret_cast<const uint8_t *>(buffer.c_str());
175 auto endPtr = startPtr + buffer.size();
176 // splice the data curType adn size of each cup that matches the timestamp
177 uint32_t curFileOffset = curFileOffset_ + sizeof(curType) + sizeof(uint32_t);
178 uint32_t splitOffset = 0;
179 uint32_t splitSize = 0;
180 bool isSplitPosition = false;
181 for (uint8_t *data = const_cast<uint8_t *>(startPtr); data < endPtr; data += FTRACE_PAGE_SIZE) {
182 bool haveSplitSeg = false;
183 TS_CHECK_TRUE(ftraceProcessor_->HmParsePageData(*cpuDetail_.get(), *cpuDetailParser_.get(), data, haveSplitSeg),
184 false, "hm parse page failed!");
185 if (haveSplitSeg) {
186 splitSize += FTRACE_PAGE_SIZE;
187 if (!isSplitPosition) {
188 // splitOffset = first Save the migration amount of CPURAW that currently matches the timestamp
189 isSplitPosition = true;
190 splitOffset = curFileOffset;
191 }
192 }
193 if (!traceDataCache_->isSplitFile_) {
194 // No specific analysis is required for time cutting
195 cpuDetailParser_->FilterAllEvents(*cpuDetail_.get());
196 }
197 curFileOffset += FTRACE_PAGE_SIZE;
198 }
199 if (traceDataCache_->isSplitFile_ && splitSize > 0) {
200 rawTraceSplitCpuData_.emplace_back(SpliteDataInfo(splitOffset, splitSize, curType));
201 // For rawtrace. fileType_=1,There is no need to record the total number of CPUs, so for data that does not meet
202 // the cutting timestamp, there is no need to record and save it
203 return true;
204 }
205 TS_LOGD("mark.debug. HmParseCpuRawData end success");
206 return true;
207 }
208
ParseLastCommData(uint8_t type,const std::string & buffer)209 bool RawTraceParser::ParseLastCommData(uint8_t type, const std::string &buffer)
210 {
211 TS_CHECK_TRUE_RET(restCommDataCnt_ != INVALID_UINT8, false);
212 switch (type) {
213 case static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_CMDLINES):
214 TS_CHECK_TRUE(ftraceProcessor_->HandleCmdlines(buffer), false, "parse cmdlines failed");
215 break;
216 case static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_TGIDS):
217 TS_CHECK_TRUE(ftraceProcessor_->HandleTgids(buffer), false, "parse tgid failed");
218 break;
219 case static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_HEADER_PAGE):
220 TS_CHECK_TRUE(ftraceProcessor_->HandleHeaderPageFormat(buffer), false, "init header page failed");
221 break;
222 case static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_PRINTK_FORMATS):
223 TS_CHECK_TRUE(PrintkFormatsProcessor::GetInstance().HandlePrintkSyms(buffer), false,
224 "init printk_formats failed");
225 break;
226 case static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_KALLSYMS):
227 TS_CHECK_TRUE(ksymsProcessor_->HandleKallSyms(buffer), false, "init printk_formats failed");
228 break;
229 default:
230 #ifdef IS_WASM
231 if (!isWasmReadFile_) {
232 return false;
233 }
234 #endif
235 break;
236 }
237 ++restCommDataCnt_;
238 return true;
239 }
240
ParseTraceDataSegment(std::unique_ptr<uint8_t[]> bufferStr,size_t size,bool isFinish)241 void RawTraceParser::ParseTraceDataSegment(std::unique_ptr<uint8_t[]> bufferStr, size_t size, bool isFinish)
242 {
243 packagesBuffer_.insert(packagesBuffer_.end(), &bufferStr[0], &bufferStr[size]);
244 auto packagesCurIter = packagesBuffer_.begin();
245 if (ParseDataRecursively(packagesCurIter)) {
246 packagesCurIter = packagesBuffer_.erase(packagesBuffer_.begin(), packagesCurIter);
247 }
248 if (isFinish) {
249 restCommDataCnt_ = INVALID_UINT8;
250 hasGotHeader_ = false;
251 packagesBuffer_.clear();
252 }
253 return;
254 }
255
ProcessRawTraceContent(std::string & bufferLine,uint8_t curType)256 bool RawTraceParser::ProcessRawTraceContent(std::string &bufferLine, uint8_t curType)
257 {
258 if (curType >= static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_CPU_RAW) &&
259 curType < static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_HEADER_PAGE)) {
260 curType = static_cast<uint32_t>(curType);
261 if (fileType_ == static_cast<uint8_t>(RawTraceFileType::FILE_RAW_TRACE)) {
262 auto cpuId = curType - static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_CPU_RAW);
263 TS_CHECK_TRUE(ParseCpuRawData(cpuId, bufferLine, curType), false, "cpu raw parse failed");
264 } else if (fileType_ == static_cast<uint8_t>(RawTraceFileType::HM_FILE_RAW_TRACE)) {
265 TS_CHECK_TRUE(HmParseCpuRawData(bufferLine, curType), false, "hm raw trace parse failed");
266 }
267 if (traceDataCache_->isSplitFile_) {
268 curFileOffset_ += sizeof(uint32_t) + sizeof(uint32_t) + bufferLine.size();
269 }
270 } else if (curType == static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_EVENTS_FORMAT)) {
271 TS_CHECK_TRUE(InitEventFormats(bufferLine), false, "init event format failed");
272 } else {
273 TS_LOGW("Raw Trace Type(%d) Unknown or has been parsed.", curType);
274 }
275 return true;
276 }
ParseDataRecursively(std::deque<uint8_t>::iterator & packagesCurIter)277 bool RawTraceParser::ParseDataRecursively(std::deque<uint8_t>::iterator &packagesCurIter)
278 {
279 uint32_t type = 0;
280 uint32_t len = 0;
281 if (!hasGotHeader_) {
282 TS_CHECK_TRUE(InitRawTraceFileHeader(packagesCurIter), false, "get rawtrace file header failed");
283 }
284 while (true) {
285 std::copy(packagesCurIter, packagesCurIter + sizeof(type), reinterpret_cast<uint8_t *>(&type));
286 packagesCurIter += sizeof(type);
287 std::copy(packagesCurIter, packagesCurIter + sizeof(len), reinterpret_cast<uint8_t *>(&len));
288 packagesCurIter += sizeof(len);
289 uint32_t restDataLen = std::distance(packagesCurIter, packagesBuffer_.end());
290 TS_CHECK_TRUE_RET(len <= restDataLen && packagesBuffer_.size() > 0, false);
291 std::string bufferLine(packagesCurIter, packagesCurIter + len);
292 packagesCurIter += len;
293 packagesCurIter = packagesBuffer_.erase(packagesBuffer_.begin(), packagesCurIter);
294 uint8_t curType = static_cast<uint8_t>(type);
295 if (ParseLastCommData(curType, bufferLine)) {
296 continue;
297 }
298 // for jump first comm data
299 if (traceDataCache_->isSplitFile_ &&
300 (curType < static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_CPU_RAW) ||
301 curType >= static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_HEADER_PAGE))) {
302 uint32_t curSegSize = sizeof(type) + sizeof(len) + bufferLine.size();
303 rawTraceSplitCommData_.emplace_back(SpliteDataInfo(curFileOffset_, curSegSize));
304 curFileOffset_ += curSegSize;
305 if (curType == static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_EVENTS_FORMAT)) {
306 restCommDataCnt_ = INVALID_UINT8;
307 }
308 continue;
309 }
310 if (!ProcessRawTraceContent(bufferLine, curType)) {
311 return false;
312 }
313 }
314 return true;
315 }
316 } // namespace TraceStreamer
317 } // namespace SysTuning
318