1 /*
2 * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "rawtrace_parser.h"
17 #include <cinttypes>
18 #if IS_WASM
19 #include "wasm_func.h"
20 #endif
21 #include "string_help.h"
22 namespace SysTuning {
23 namespace TraceStreamer {
RawTraceParser(TraceDataCache * dataCache,const TraceStreamerFilters * filters)24 RawTraceParser::RawTraceParser(TraceDataCache *dataCache, const TraceStreamerFilters *filters)
25 : ParserBase(filters),
26 cpuDetail_(std::make_unique<FtraceCpuDetailMsg>()),
27 cpuDetailParser_(std::make_unique<CpuDetailParser>(dataCache, filters)),
28 ftraceProcessor_(std::make_unique<FtraceProcessor>(dataCache)),
29 ksymsProcessor_(std::make_unique<KernelSymbolsProcessor>(dataCache, filters)),
30 traceDataCache_(dataCache)
31 {
32 }
33
~RawTraceParser()34 RawTraceParser::~RawTraceParser() {}
ParseTraceDataItem(const std::string & buffer)35 void RawTraceParser::ParseTraceDataItem(const std::string &buffer) {}
WaitForParserEnd()36 void RawTraceParser::WaitForParserEnd()
37 {
38 cpuDetailParser_->FilterAllEvents(*cpuDetail_.get(), true);
39 cpuDetailParser_->FinishCpuDetailParser();
40 UpdateTraceMinRange();
41 restCommDataCnt_ = 0;
42 hasGotHeader_ = false;
43 curCpuCoreNum_ = 0;
44 ClearRawTraceData();
45 TS_LOGI("Parser raw trace end!");
46 }
UpdateTraceMinRange()47 void RawTraceParser::UpdateTraceMinRange()
48 {
49 auto schedSlice = traceDataCache_->GetConstSchedSliceData();
50 std::set<uint32_t> uniqueCpuIdSet;
51 uint64_t cpuRunningStatMinTime = INVALID_TIME;
52 for (size_t i = 0; i < schedSlice.Size() && uniqueCpuIdSet.size() <= curCpuCoreNum_; i++) {
53 auto iter = uniqueCpuIdSet.find(schedSlice.CpusData()[i]);
54 if (iter != uniqueCpuIdSet.end()) {
55 continue;
56 }
57 uniqueCpuIdSet.emplace(schedSlice.CpusData()[i]);
58 cpuRunningStatMinTime = schedSlice.TimeStampData()[i];
59 TS_LOGW("curCpuId=%u, cpuRunningStatMinTime=%" PRIu64 "", schedSlice.CpusData()[i], cpuRunningStatMinTime);
60 }
61 if (cpuRunningStatMinTime != INVALID_TIME) {
62 traceDataCache_->UpdateTraceMinTime(cpuRunningStatMinTime);
63 }
64 }
InitRawTraceFileHeader(std::deque<uint8_t>::iterator & packagesCurIter)65 bool RawTraceParser::InitRawTraceFileHeader(std::deque<uint8_t>::iterator &packagesCurIter)
66 {
67 TS_CHECK_TRUE(packagesBuffer_.size() >= sizeof(RawTraceFileHeader), false,
68 "buffer size less than rawtrace file header");
69 RawTraceFileHeader header;
70 std::copy(packagesBuffer_.begin(), packagesBuffer_.begin() + sizeof(RawTraceFileHeader),
71 reinterpret_cast<uint8_t *>(&header));
72 TS_LOGI("magicNumber=%d fileType=%d", header.magicNumber, header.fileType);
73
74 fileType_ = header.fileType;
75 if (traceDataCache_->isSplitFile_) {
76 // To resolve the second incoming file, it is necessary to reset the previously set variables to zero
77 ClearRawTraceData();
78 rawTraceSplitCommData_.emplace_back(SpliteDataInfo(curFileOffset_, sizeof(RawTraceFileHeader)));
79 curFileOffset_ += sizeof(RawTraceFileHeader);
80 }
81 packagesCurIter += sizeof(RawTraceFileHeader);
82 packagesCurIter = packagesBuffer_.erase(packagesBuffer_.begin(), packagesCurIter);
83 hasGotHeader_ = true;
84 return true;
85 }
InitEventFormats(const std::string & buffer)86 bool RawTraceParser::InitEventFormats(const std::string &buffer)
87 {
88 #ifdef IS_WASM
89 restCommDataCnt_ = INVALID_UINT8; // ensure that the restCommData is parsed only once
90 #endif
91 std::string line;
92 std::istringstream iss(buffer);
93 std::stringstream eventFormat;
94 while (std::getline(iss, line)) {
95 eventFormat << line << '\n';
96 if (base::StartWith(line, eventEndCmd_)) {
97 ftraceProcessor_->SetupEvent(eventFormat.str());
98 eventFormat.str("");
99 }
100 }
101 return true;
102 }
UpdateCpuCoreMax(uint32_t cpuId)103 bool RawTraceParser::UpdateCpuCoreMax(uint32_t cpuId)
104 {
105 if (cpuId >= curCpuCoreNum_) {
106 curCpuCoreNum_++;
107 TS_LOGI("cpuId=%u, curCpuCoreNum_=%u", cpuId, curCpuCoreNum_);
108 return false;
109 }
110 if (cpuDetailParser_->cpuCoreMax_ == CPU_CORE_MAX) {
111 cpuDetailParser_->ResizeStandAloneCpuEventList(curCpuCoreNum_);
112 }
113 return true;
114 }
115
ParseCpuRawData(uint32_t cpuId,const std::string & buffer,uint32_t curType)116 bool RawTraceParser::ParseCpuRawData(uint32_t cpuId, const std::string &buffer, uint32_t curType)
117 {
118 UpdateCpuCoreMax(cpuId);
119 // splice the data curType adn size of each cup that matches the timestamp
120 uint32_t curFileOffset = curFileOffset_ + sizeof(curType) + sizeof(uint32_t);
121 uint32_t splitOffset = 0;
122 uint32_t splitSize = 0;
123 bool isSplitPosition = false;
124 if (0 == buffer.size() && traceDataCache_->isSplitFile_) {
125 // For rawtrace. fileType_=0, in order to count the number of CPUs and maintain the CPU data structure (which
126 // will also be passed to data types with CPU size 0), it is necessary to save the data during the cutting
127 // process and exit the buffer directly.
128 rawTraceSplitCpuData_.emplace_back(SpliteDataInfo(curFileOffset, 0, curType));
129 }
130 TS_CHECK_TRUE(buffer.size() > 0, true, "cur cpu(%u) raw data is null!", cpuId);
131 auto startPtr = reinterpret_cast<const uint8_t *>(buffer.c_str());
132 auto endPtr = startPtr + buffer.size();
133 cpuDetail_->set_cpu(cpuId);
134 for (uint8_t *page = const_cast<uint8_t *>(startPtr); page < endPtr; page += FTRACE_PAGE_SIZE) {
135 bool haveSplitSeg = false;
136 TS_CHECK_TRUE(ftraceProcessor_->HandlePage(*cpuDetail_.get(), *cpuDetailParser_.get(), page, haveSplitSeg),
137 false, "handle page failed!");
138 if (haveSplitSeg) {
139 splitSize += FTRACE_PAGE_SIZE;
140 if (!isSplitPosition) {
141 // splitOffset = first Save the migration amount of CPURAW that currently matches the timestamp
142 isSplitPosition = true;
143 splitOffset = curFileOffset;
144 }
145 }
146 curFileOffset += FTRACE_PAGE_SIZE;
147 }
148 if (traceDataCache_->isSplitFile_) {
149 // Skip parsing data for timestamp or non timestamp compliant data
150 if (splitSize > 0) {
151 rawTraceSplitCpuData_.emplace_back(SpliteDataInfo(splitOffset, splitSize, curType));
152 } else {
153 // For rawtrace. fileType_=0,In order to count the number of CPUs and maintain the CPU data structure (also
154 // through For CPU data types with a size of 0, it is necessary to set the size to 0 during the cutting
155 // process to save CPU data that does not meet the cutting event stamp
156 rawTraceSplitCpuData_.emplace_back(SpliteDataInfo(curFileOffset, 0, curType));
157 }
158 return true;
159 }
160 if (cpuDetailParser_->cpuCoreMax_ != CPU_CORE_MAX) {
161 cpuDetailParser_->FilterAllEvents(*cpuDetail_.get());
162 }
163 return true;
164 }
165
HmParseCpuRawData(const std::string & buffer,uint32_t curType)166 bool RawTraceParser::HmParseCpuRawData(const std::string &buffer, uint32_t curType)
167 {
168 TS_CHECK_TRUE(buffer.size() > 0, true, "hm raw data is null!");
169 auto startPtr = reinterpret_cast<const uint8_t *>(buffer.c_str());
170 auto endPtr = startPtr + buffer.size();
171 // splice the data curType adn size of each cup that matches the timestamp
172 uint32_t curFileOffset = curFileOffset_ + sizeof(curType) + sizeof(uint32_t);
173 uint32_t splitOffset = 0;
174 uint32_t splitSize = 0;
175 bool isSplitPosition = false;
176 for (uint8_t *data = const_cast<uint8_t *>(startPtr); data < endPtr; data += FTRACE_PAGE_SIZE) {
177 bool haveSplitSeg = false;
178 TS_CHECK_TRUE(ftraceProcessor_->HmParsePageData(*cpuDetail_.get(), *cpuDetailParser_.get(), data, haveSplitSeg),
179 false, "hm parse page failed!");
180 if (haveSplitSeg) {
181 splitSize += FTRACE_PAGE_SIZE;
182 if (!isSplitPosition) {
183 // splitOffset = first Save the migration amount of CPURAW that currently matches the timestamp
184 isSplitPosition = true;
185 splitOffset = curFileOffset;
186 }
187 }
188 if (!traceDataCache_->isSplitFile_) {
189 // No specific analysis is required for time cutting
190 cpuDetailParser_->FilterAllEvents(*cpuDetail_.get());
191 }
192 curFileOffset += FTRACE_PAGE_SIZE;
193 }
194 if (traceDataCache_->isSplitFile_ && splitSize > 0) {
195 rawTraceSplitCpuData_.emplace_back(SpliteDataInfo(splitOffset, splitSize, curType));
196 // For rawtrace. fileType_=1,There is no need to record the total number of CPUs, so for data that does not meet
197 // the cutting timestamp, there is no need to record and save it
198 return true;
199 }
200 TS_LOGD("mark.debug. HmParseCpuRawData end success");
201 return true;
202 }
203
ParseLastCommData(uint8_t type,const std::string & buffer)204 bool RawTraceParser::ParseLastCommData(uint8_t type, const std::string &buffer)
205 {
206 TS_CHECK_TRUE_RET(restCommDataCnt_ != INVALID_UINT8, false);
207 switch (type) {
208 case static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_CMDLINES):
209 TS_CHECK_TRUE(ftraceProcessor_->HandleCmdlines(buffer), false, "parse cmdlines failed");
210 break;
211 case static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_TGIDS):
212 TS_CHECK_TRUE(ftraceProcessor_->HandleTgids(buffer), false, "parse tgid failed");
213 break;
214 case static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_HEADER_PAGE):
215 TS_CHECK_TRUE(ftraceProcessor_->HandleHeaderPageFormat(buffer), false, "init header page failed");
216 break;
217 case static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_PRINTK_FORMATS):
218 TS_CHECK_TRUE(PrintkFormatsProcessor::GetInstance().HandlePrintkSyms(buffer), false,
219 "init printk_formats failed");
220 break;
221 case static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_KALLSYMS):
222 TS_CHECK_TRUE(ksymsProcessor_->HandleKallSyms(buffer), false, "init printk_formats failed");
223 break;
224 default:
225 #ifdef IS_WASM
226 return false;
227 #else
228 break;
229 #endif
230 }
231 ++restCommDataCnt_;
232 return true;
233 }
234
ParseTraceDataSegment(std::unique_ptr<uint8_t[]> bufferStr,size_t size,bool isFinish)235 void RawTraceParser::ParseTraceDataSegment(std::unique_ptr<uint8_t[]> bufferStr, size_t size, bool isFinish)
236 {
237 packagesBuffer_.insert(packagesBuffer_.end(), &bufferStr[0], &bufferStr[size]);
238 auto packagesCurIter = packagesBuffer_.begin();
239 if (ParseDataRecursively(packagesCurIter)) {
240 packagesCurIter = packagesBuffer_.erase(packagesBuffer_.begin(), packagesCurIter);
241 }
242 if (isFinish) {
243 restCommDataCnt_ = INVALID_UINT8;
244 hasGotHeader_ = false;
245 packagesBuffer_.clear();
246 }
247 return;
248 }
249
ProcessRawTraceContent(std::string & bufferLine,uint8_t curType)250 bool RawTraceParser::ProcessRawTraceContent(std::string &bufferLine, uint8_t curType)
251 {
252 if (curType >= static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_CPU_RAW) &&
253 curType < static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_HEADER_PAGE)) {
254 curType = static_cast<uint32_t>(curType);
255 if (fileType_ == static_cast<uint8_t>(RawTraceFileType::FILE_RAW_TRACE)) {
256 auto cpuId = curType - static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_CPU_RAW);
257 TS_CHECK_TRUE(ParseCpuRawData(cpuId, bufferLine, curType), false, "cpu raw parse failed");
258 } else if (fileType_ == static_cast<uint8_t>(RawTraceFileType::HM_FILE_RAW_TRACE)) {
259 TS_CHECK_TRUE(HmParseCpuRawData(bufferLine, curType), false, "hm raw trace parse failed");
260 }
261 if (traceDataCache_->isSplitFile_) {
262 curFileOffset_ += sizeof(uint32_t) + sizeof(uint32_t) + bufferLine.size();
263 }
264 } else if (curType == static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_EVENTS_FORMAT)) {
265 TS_CHECK_TRUE(InitEventFormats(bufferLine), false, "init event format failed");
266 } else {
267 TS_LOGW("Raw Trace Type(%d) Unknown or has been parsed.", curType);
268 }
269 return true;
270 }
ParseDataRecursively(std::deque<uint8_t>::iterator & packagesCurIter)271 bool RawTraceParser::ParseDataRecursively(std::deque<uint8_t>::iterator &packagesCurIter)
272 {
273 uint32_t type = 0;
274 uint32_t len = 0;
275 if (!hasGotHeader_) {
276 TS_CHECK_TRUE(InitRawTraceFileHeader(packagesCurIter), false, "get rawtrace file header failed");
277 }
278 while (true) {
279 std::copy(packagesCurIter, packagesCurIter + sizeof(type), reinterpret_cast<uint8_t *>(&type));
280 packagesCurIter += sizeof(type);
281 std::copy(packagesCurIter, packagesCurIter + sizeof(len), reinterpret_cast<uint8_t *>(&len));
282 packagesCurIter += sizeof(len);
283 uint32_t restDataLen = std::distance(packagesCurIter, packagesBuffer_.end());
284 TS_CHECK_TRUE_RET(len <= restDataLen && packagesBuffer_.size() > 0, false);
285 std::string bufferLine(packagesCurIter, packagesCurIter + len);
286 packagesCurIter += len;
287 packagesCurIter = packagesBuffer_.erase(packagesBuffer_.begin(), packagesCurIter);
288 uint8_t curType = static_cast<uint8_t>(type);
289 if (ParseLastCommData(curType, bufferLine)) {
290 continue;
291 }
292 // for jump first comm data
293 if (traceDataCache_->isSplitFile_ &&
294 (curType < static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_CPU_RAW) ||
295 curType >= static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_HEADER_PAGE))) {
296 uint32_t curSegSize = sizeof(type) + sizeof(len) + bufferLine.size();
297 rawTraceSplitCommData_.emplace_back(SpliteDataInfo(curFileOffset_, curSegSize));
298 curFileOffset_ += curSegSize;
299 if (curType == static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_EVENTS_FORMAT)) {
300 restCommDataCnt_ = INVALID_UINT8;
301 }
302 continue;
303 }
304 if (!ProcessRawTraceContent(bufferLine, curType)) {
305 return false;
306 }
307 }
308 return true;
309 }
310 } // namespace TraceStreamer
311 } // namespace SysTuning
312