1 /*
2 * Copyright (c) Huawei Technologies Co., Ltd. 2023. All rights reserved.
3 * Licensed under the Apache License, Version 2.0 (the "License");
4 * you may not use this file except in compliance with the License.
5 * You may obtain a copy of the License at
6 *
7 * http://www.apache.org/licenses/LICENSE-2.0
8 *
9 * Unless required by applicable law or agreed to in writing, software
10 * distributed under the License is distributed on an "AS IS" BASIS,
11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 * See the License for the specific language governing permissions and
13 * limitations under the License.
14 */
15
16 #include "rawtrace_parser.h"
17 #include <cinttypes>
18 #if IS_WASM
19 #include "wasm_func.h"
20 #endif
21 #include "string_help.h"
22 namespace SysTuning {
23 namespace TraceStreamer {
RawTraceParser(TraceDataCache * dataCache,const TraceStreamerFilters * filters)24 RawTraceParser::RawTraceParser(TraceDataCache *dataCache, const TraceStreamerFilters *filters)
25 : ParserBase(filters),
26 cpuDetail_(std::make_unique<FtraceCpuDetailMsg>()),
27 cpuDetailParser_(std::make_unique<CpuDetailParser>(dataCache, filters)),
28 ftraceProcessor_(std::make_unique<FtraceProcessor>(dataCache)),
29 ksymsProcessor_(std::make_unique<KernelSymbolsProcessor>(dataCache, filters)),
30 traceDataCache_(dataCache)
31 {
32 }
33
~RawTraceParser()34 RawTraceParser::~RawTraceParser() {}
ParseTraceDataItem(const std::string & buffer)35 void RawTraceParser::ParseTraceDataItem(const std::string &buffer) {}
WaitForParserEnd()36 void RawTraceParser::WaitForParserEnd()
37 {
38 cpuDetailParser_->FilterAllEvents(*cpuDetail_.get(), true);
39 cpuDetailParser_->FinishCpuDetailParser();
40 UpdateTraceMinRange();
41 restCommDataCnt_ = 0;
42 hasGotHeader_ = false;
43 curCpuCoreNum_ = 0;
44 ClearRawTraceData();
45 TS_LOGI("Parser raw trace end!");
46 }
UpdateTraceMinRange()47 void RawTraceParser::UpdateTraceMinRange()
48 {
49 if (!traceDataCache_->RawTraceCutStartTsEnabled()) {
50 return;
51 }
52 auto schedSlice = traceDataCache_->GetConstSchedSliceData();
53 std::set<uint32_t> uniqueCpuIdSet;
54 uint64_t cpuRunningStatMinTime = INVALID_TIME;
55 for (size_t i = 0; i < schedSlice.Size() && uniqueCpuIdSet.size() <= curCpuCoreNum_; i++) {
56 auto iter = uniqueCpuIdSet.find(schedSlice.CpusData()[i]);
57 if (iter != uniqueCpuIdSet.end()) {
58 continue;
59 }
60 uniqueCpuIdSet.emplace(schedSlice.CpusData()[i]);
61 cpuRunningStatMinTime = schedSlice.TimeStampData()[i];
62 TS_LOGW("curCpuId=%u, cpuRunningStatMinTime=%" PRIu64 "", schedSlice.CpusData()[i], cpuRunningStatMinTime);
63 }
64 if (cpuRunningStatMinTime != INVALID_TIME) {
65 traceDataCache_->UpdateTraceMinTime(cpuRunningStatMinTime);
66 }
67 }
InitRawTraceFileHeader(std::deque<uint8_t>::iterator & packagesCurIter)68 bool RawTraceParser::InitRawTraceFileHeader(std::deque<uint8_t>::iterator &packagesCurIter)
69 {
70 TS_CHECK_TRUE(packagesBuffer_.size() >= sizeof(RawTraceFileHeader), false,
71 "buffer size less than rawtrace file header");
72 RawTraceFileHeader header;
73 std::copy(packagesBuffer_.begin(), packagesBuffer_.begin() + sizeof(RawTraceFileHeader),
74 reinterpret_cast<uint8_t *>(&header));
75 TS_LOGI("magicNumber=%d fileType=%d", header.magicNumber, header.fileType);
76
77 fileType_ = header.fileType;
78 if (traceDataCache_->isSplitFile_) {
79 // To resolve the second incoming file, it is necessary to reset the previously set variables to zero
80 ClearRawTraceData();
81 rawTraceSplitCommData_.emplace_back(SpliteDataInfo(curFileOffset_, sizeof(RawTraceFileHeader)));
82 curFileOffset_ += sizeof(RawTraceFileHeader);
83 }
84 packagesCurIter += sizeof(RawTraceFileHeader);
85 packagesCurIter = packagesBuffer_.erase(packagesBuffer_.begin(), packagesCurIter);
86 hasGotHeader_ = true;
87 return true;
88 }
InitEventFormats(const std::string & buffer)89 bool RawTraceParser::InitEventFormats(const std::string &buffer)
90 {
91 #ifdef IS_WASM
92 restCommDataCnt_ = INVALID_UINT8; // ensure that the restCommData is parsed only once
93 #endif
94 std::string line;
95 std::istringstream iss(buffer);
96 std::stringstream eventFormat;
97 while (std::getline(iss, line)) {
98 eventFormat << line << '\n';
99 if (base::StartWith(line, eventEndCmd_)) {
100 ftraceProcessor_->SetupEvent(eventFormat.str());
101 eventFormat.str("");
102 }
103 }
104 return true;
105 }
UpdateCpuCoreMax(uint32_t cpuId)106 bool RawTraceParser::UpdateCpuCoreMax(uint32_t cpuId)
107 {
108 if (cpuId >= curCpuCoreNum_) {
109 curCpuCoreNum_++;
110 TS_LOGI("cpuId=%u, curCpuCoreNum_=%u", cpuId, curCpuCoreNum_);
111 return false;
112 }
113 if (cpuDetailParser_->cpuCoreMax_ == CPU_CORE_MAX) {
114 cpuDetailParser_->ResizeStandAloneCpuEventList(curCpuCoreNum_);
115 }
116 return true;
117 }
118
ParseCpuRawData(uint32_t cpuId,const std::string & buffer,uint32_t curType)119 bool RawTraceParser::ParseCpuRawData(uint32_t cpuId, const std::string &buffer, uint32_t curType)
120 {
121 UpdateCpuCoreMax(cpuId);
122 // splice the data curType adn size of each cup that matches the timestamp
123 uint32_t curFileOffset = curFileOffset_ + sizeof(curType) + sizeof(uint32_t);
124 uint32_t splitOffset = 0;
125 uint32_t splitSize = 0;
126 bool isSplitPosition = false;
127 if (0 == buffer.size() && traceDataCache_->isSplitFile_) {
128 // For rawtrace. fileType_=0, in order to count the number of CPUs and maintain the CPU data structure (which
129 // will also be passed to data types with CPU size 0), it is necessary to save the data during the cutting
130 // process and exit the buffer directly.
131 rawTraceSplitCpuData_.emplace_back(SpliteDataInfo(curFileOffset, 0, curType));
132 }
133 TS_CHECK_TRUE(buffer.size() > 0, true, "cur cpu(%u) raw data is null!", cpuId);
134 auto startPtr = reinterpret_cast<const uint8_t *>(buffer.c_str());
135 auto endPtr = startPtr + buffer.size();
136 cpuDetail_->set_cpu(cpuId);
137 for (uint8_t *page = const_cast<uint8_t *>(startPtr); page < endPtr; page += FTRACE_PAGE_SIZE) {
138 bool haveSplitSeg = false;
139 TS_CHECK_TRUE(ftraceProcessor_->HandlePage(*cpuDetail_.get(), *cpuDetailParser_.get(), page, haveSplitSeg),
140 false, "handle page failed!");
141 if (haveSplitSeg) {
142 splitSize += FTRACE_PAGE_SIZE;
143 if (!isSplitPosition) {
144 // splitOffset = first Save the migration amount of CPURAW that currently matches the timestamp
145 isSplitPosition = true;
146 splitOffset = curFileOffset;
147 }
148 }
149 curFileOffset += FTRACE_PAGE_SIZE;
150 }
151 if (traceDataCache_->isSplitFile_) {
152 // Skip parsing data for timestamp or non timestamp compliant data
153 if (splitSize > 0) {
154 rawTraceSplitCpuData_.emplace_back(SpliteDataInfo(splitOffset, splitSize, curType));
155 } else {
156 // For rawtrace. fileType_=0,In order to count the number of CPUs and maintain the CPU data structure (also
157 // through For CPU data types with a size of 0, it is necessary to set the size to 0 during the cutting
158 // process to save CPU data that does not meet the cutting event stamp
159 rawTraceSplitCpuData_.emplace_back(SpliteDataInfo(curFileOffset, 0, curType));
160 }
161 return true;
162 }
163 if (cpuDetailParser_->cpuCoreMax_ != CPU_CORE_MAX) {
164 cpuDetailParser_->FilterAllEvents(*cpuDetail_.get());
165 }
166 return true;
167 }
168
HmParseCpuRawData(const std::string & buffer,uint32_t curType)169 bool RawTraceParser::HmParseCpuRawData(const std::string &buffer, uint32_t curType)
170 {
171 TS_CHECK_TRUE(buffer.size() > 0, true, "hm raw data is null!");
172 auto startPtr = reinterpret_cast<const uint8_t *>(buffer.c_str());
173 auto endPtr = startPtr + buffer.size();
174 // splice the data curType adn size of each cup that matches the timestamp
175 uint32_t curFileOffset = curFileOffset_ + sizeof(curType) + sizeof(uint32_t);
176 uint32_t splitOffset = 0;
177 uint32_t splitSize = 0;
178 bool isSplitPosition = false;
179 for (uint8_t *data = const_cast<uint8_t *>(startPtr); data < endPtr; data += FTRACE_PAGE_SIZE) {
180 bool haveSplitSeg = false;
181 TS_CHECK_TRUE(ftraceProcessor_->HmParsePageData(*cpuDetail_.get(), *cpuDetailParser_.get(), data, haveSplitSeg),
182 false, "hm parse page failed!");
183 if (haveSplitSeg) {
184 splitSize += FTRACE_PAGE_SIZE;
185 if (!isSplitPosition) {
186 // splitOffset = first Save the migration amount of CPURAW that currently matches the timestamp
187 isSplitPosition = true;
188 splitOffset = curFileOffset;
189 }
190 }
191 if (!traceDataCache_->isSplitFile_) {
192 // No specific analysis is required for time cutting
193 cpuDetailParser_->FilterAllEvents(*cpuDetail_.get());
194 }
195 curFileOffset += FTRACE_PAGE_SIZE;
196 }
197 if (traceDataCache_->isSplitFile_ && splitSize > 0) {
198 rawTraceSplitCpuData_.emplace_back(SpliteDataInfo(splitOffset, splitSize, curType));
199 // For rawtrace. fileType_=1,There is no need to record the total number of CPUs, so for data that does not meet
200 // the cutting timestamp, there is no need to record and save it
201 return true;
202 }
203 TS_LOGD("mark.debug. HmParseCpuRawData end success");
204 return true;
205 }
206
ParseLastCommData(uint8_t type,const std::string & buffer)207 bool RawTraceParser::ParseLastCommData(uint8_t type, const std::string &buffer)
208 {
209 TS_CHECK_TRUE_RET(restCommDataCnt_ != INVALID_UINT8, false);
210 switch (type) {
211 case static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_CMDLINES):
212 TS_CHECK_TRUE(ftraceProcessor_->HandleCmdlines(buffer), false, "parse cmdlines failed");
213 break;
214 case static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_TGIDS):
215 TS_CHECK_TRUE(ftraceProcessor_->HandleTgids(buffer), false, "parse tgid failed");
216 break;
217 case static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_HEADER_PAGE):
218 TS_CHECK_TRUE(ftraceProcessor_->HandleHeaderPageFormat(buffer), false, "init header page failed");
219 break;
220 case static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_PRINTK_FORMATS):
221 TS_CHECK_TRUE(PrintkFormatsProcessor::GetInstance().HandlePrintkSyms(buffer), false,
222 "init printk_formats failed");
223 break;
224 case static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_KALLSYMS):
225 TS_CHECK_TRUE(ksymsProcessor_->HandleKallSyms(buffer), false, "init printk_formats failed");
226 break;
227 default:
228 #ifdef IS_WASM
229 return false;
230 #else
231 break;
232 #endif
233 }
234 ++restCommDataCnt_;
235 return true;
236 }
237
ParseTraceDataSegment(std::unique_ptr<uint8_t[]> bufferStr,size_t size,bool isFinish)238 void RawTraceParser::ParseTraceDataSegment(std::unique_ptr<uint8_t[]> bufferStr, size_t size, bool isFinish)
239 {
240 packagesBuffer_.insert(packagesBuffer_.end(), &bufferStr[0], &bufferStr[size]);
241 auto packagesCurIter = packagesBuffer_.begin();
242 if (ParseDataRecursively(packagesCurIter)) {
243 packagesCurIter = packagesBuffer_.erase(packagesBuffer_.begin(), packagesCurIter);
244 }
245 if (isFinish) {
246 restCommDataCnt_ = INVALID_UINT8;
247 hasGotHeader_ = false;
248 packagesBuffer_.clear();
249 }
250 return;
251 }
252
ProcessRawTraceContent(std::string & bufferLine,uint8_t curType)253 bool RawTraceParser::ProcessRawTraceContent(std::string &bufferLine, uint8_t curType)
254 {
255 if (curType >= static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_CPU_RAW) &&
256 curType < static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_HEADER_PAGE)) {
257 curType = static_cast<uint32_t>(curType);
258 if (fileType_ == static_cast<uint8_t>(RawTraceFileType::FILE_RAW_TRACE)) {
259 auto cpuId = curType - static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_CPU_RAW);
260 TS_CHECK_TRUE(ParseCpuRawData(cpuId, bufferLine, curType), false, "cpu raw parse failed");
261 } else if (fileType_ == static_cast<uint8_t>(RawTraceFileType::HM_FILE_RAW_TRACE)) {
262 TS_CHECK_TRUE(HmParseCpuRawData(bufferLine, curType), false, "hm raw trace parse failed");
263 }
264 if (traceDataCache_->isSplitFile_) {
265 curFileOffset_ += sizeof(uint32_t) + sizeof(uint32_t) + bufferLine.size();
266 }
267 } else if (curType == static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_EVENTS_FORMAT)) {
268 TS_CHECK_TRUE(InitEventFormats(bufferLine), false, "init event format failed");
269 } else {
270 TS_LOGW("Raw Trace Type(%d) Unknown or has been parsed.", curType);
271 }
272 return true;
273 }
ParseDataRecursively(std::deque<uint8_t>::iterator & packagesCurIter)274 bool RawTraceParser::ParseDataRecursively(std::deque<uint8_t>::iterator &packagesCurIter)
275 {
276 uint32_t type = 0;
277 uint32_t len = 0;
278 if (!hasGotHeader_) {
279 TS_CHECK_TRUE(InitRawTraceFileHeader(packagesCurIter), false, "get rawtrace file header failed");
280 }
281 while (true) {
282 std::copy(packagesCurIter, packagesCurIter + sizeof(type), reinterpret_cast<uint8_t *>(&type));
283 packagesCurIter += sizeof(type);
284 std::copy(packagesCurIter, packagesCurIter + sizeof(len), reinterpret_cast<uint8_t *>(&len));
285 packagesCurIter += sizeof(len);
286 uint32_t restDataLen = std::distance(packagesCurIter, packagesBuffer_.end());
287 TS_CHECK_TRUE_RET(len <= restDataLen && packagesBuffer_.size() > 0, false);
288 std::string bufferLine(packagesCurIter, packagesCurIter + len);
289 packagesCurIter += len;
290 packagesCurIter = packagesBuffer_.erase(packagesBuffer_.begin(), packagesCurIter);
291 uint8_t curType = static_cast<uint8_t>(type);
292 if (ParseLastCommData(curType, bufferLine)) {
293 continue;
294 }
295 // for jump first comm data
296 if (traceDataCache_->isSplitFile_ &&
297 (curType < static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_CPU_RAW) ||
298 curType >= static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_HEADER_PAGE))) {
299 uint32_t curSegSize = sizeof(type) + sizeof(len) + bufferLine.size();
300 rawTraceSplitCommData_.emplace_back(SpliteDataInfo(curFileOffset_, curSegSize));
301 curFileOffset_ += curSegSize;
302 if (curType == static_cast<uint8_t>(RawTraceContentType::CONTENT_TYPE_EVENTS_FORMAT)) {
303 restCommDataCnt_ = INVALID_UINT8;
304 }
305 continue;
306 }
307 if (!ProcessRawTraceContent(bufferLine, curType)) {
308 return false;
309 }
310 }
311 return true;
312 }
313 } // namespace TraceStreamer
314 } // namespace SysTuning
315