1 /* 2 * Copyright (c) 2021-2022 Huawei Device Co., Ltd. 3 * Licensed under the Apache License, Version 2.0 (the "License"); 4 * you may not use this file except in compliance with the License. 5 * You may obtain a copy of the License at 6 * 7 * http://www.apache.org/licenses/LICENSE-2.0 8 * 9 * Unless required by applicable law or agreed to in writing, software 10 * distributed under the License is distributed on an "AS IS" BASIS, 11 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 * See the License for the specific language governing permissions and 13 * limitations under the License. 14 */ 15 #ifndef HIPERF_PERF_EVENTS_H 16 #define HIPERF_PERF_EVENTS_H 17 18 #include <atomic> 19 #include <cassert> 20 #include <chrono> 21 #include <cinttypes> 22 #include <condition_variable> 23 #include <deque> 24 #include <map> 25 #include <memory> 26 #include <string> 27 #include <thread> 28 #include <variant> 29 #include <vector> 30 #if !is_mingw 31 #include <poll.h> 32 #endif 33 34 #include <sys/types.h> 35 #include <unique_fd.h> 36 #include <linux/perf_event.h> 37 38 #include "debug_logger.h" 39 #include "perf_event_record.h" 40 #include "ring_buffer.h" 41 #include "tracked_command.h" 42 #include "utilities.h" 43 #include "virtual_runtime.h" 44 45 // this for some performance debug 46 #define HIDEBUG_SKIP_CALLBACK 0 47 48 namespace OHOS { 49 namespace Developtools { 50 namespace HiPerf { 51 using ConfigTable = std::map<__u64, const std::string>; 52 using SharedConfigTable = std::unique_ptr<ConfigTable>; 53 54 static const std::string PERF_EVENT_PARANOID = "/proc/sys/kernel/perf_event_paranoid"; 55 static const std::string PERF_DISABLE_PARAM = "security.perf_harden"; 56 57 // define convert from linux/perf_event.h 58 // description from https://man7.org/linux/man-pages/man2/perf_event_open.2.html 59 60 static const ConfigTable PERF_HW_CONFIGS = { 61 {PERF_COUNT_HW_CPU_CYCLES, "hw-cpu-cycles"}, 62 {PERF_COUNT_HW_INSTRUCTIONS, "hw-instructions"}, 63 {PERF_COUNT_HW_CACHE_REFERENCES, "hw-cache-references"}, 64 {PERF_COUNT_HW_CACHE_MISSES, "hw-cache-misses"}, 65 {PERF_COUNT_HW_BRANCH_INSTRUCTIONS, "hw-branch-instructions"}, 66 {PERF_COUNT_HW_BRANCH_MISSES, "hw-branch-misses"}, 67 {PERF_COUNT_HW_BUS_CYCLES, "hw-bus-cycles"}, 68 {PERF_COUNT_HW_STALLED_CYCLES_FRONTEND, "hw-stalled-cycles-backend"}, 69 {PERF_COUNT_HW_STALLED_CYCLES_BACKEND, "hw-stalled-cycles-frontend"}, 70 {PERF_COUNT_HW_REF_CPU_CYCLES, "hw-ref-cpu-cycles"}, 71 }; 72 static const ConfigTable PERF_HW_CACHE_CONFIGS = { 73 {PERF_COUNT_HW_CACHE_L1D, "hw-cache-l1d"}, {PERF_COUNT_HW_CACHE_L1I, "hw-cache-l1i"}, 74 {PERF_COUNT_HW_CACHE_LL, "hw-cache-ll"}, {PERF_COUNT_HW_CACHE_DTLB, "hw-cache-dtlb"}, 75 {PERF_COUNT_HW_CACHE_ITLB, "hw-cache-itlb"}, {PERF_COUNT_HW_CACHE_BPU, "hw-cache-bpu"}, 76 {PERF_COUNT_HW_CACHE_NODE, "hw-cache-node"}, 77 }; 78 static const ConfigTable PERF_HW_CACHE_OP_CONFIGS = { 79 {PERF_COUNT_HW_CACHE_OP_READ, "hw-cache-op-read"}, 80 {PERF_COUNT_HW_CACHE_OP_WRITE, "hw-cache-op-write"}, 81 {PERF_COUNT_HW_CACHE_OP_PREFETCH, "hw-cache-op-prefetch"}, 82 }; 83 static const ConfigTable PERF_HW_CACHE_OP_RESULT_CONFIGS = { 84 {PERF_COUNT_HW_CACHE_RESULT_ACCESS, "hw-cache-result-access"}, 85 {PERF_COUNT_HW_CACHE_RESULT_MISS, "hw-cache-result-miss"}, 86 }; 87 static const ConfigTable PERF_SW_CONFIGS = { 88 {PERF_COUNT_SW_CPU_CLOCK, "sw-cpu-clock"}, 89 {PERF_COUNT_SW_TASK_CLOCK, "sw-task-clock"}, 90 {PERF_COUNT_SW_PAGE_FAULTS, "sw-page-faults"}, 91 {PERF_COUNT_SW_CONTEXT_SWITCHES, "sw-context-switches"}, 92 {PERF_COUNT_SW_CPU_MIGRATIONS, "sw-cpu-migrations"}, 93 {PERF_COUNT_SW_PAGE_FAULTS_MIN, "sw-page-faults-min"}, 94 {PERF_COUNT_SW_PAGE_FAULTS_MAJ, "sw-page-faults-maj"}, 95 {PERF_COUNT_SW_ALIGNMENT_FAULTS, "sw-alignment-faults"}, 96 {PERF_COUNT_SW_EMULATION_FAULTS, "sw-emulation-faults"}, 97 {PERF_COUNT_SW_DUMMY, "sw-dummy"}, 98 {PERF_COUNT_SW_BPF_OUTPUT, "sw-bpf-output"}, 99 }; 100 static const ConfigTable PERF_RAW_CONFIGS = { 101 {0x0, "raw-sw-incr"}, 102 {0x1, "raw-l1-icache-refill"}, 103 {0x2, "raw-l1-itlb-refill"}, 104 {0x3, "raw-l1-dcache-refill"}, 105 {0x4, "raw-l1-dcache"}, 106 {0x5, "raw-l1-dtlb-refill"}, 107 {0x6, "raw-load-retired"}, 108 {0x7, "raw-store-retired"}, 109 {0x8, "raw-instruction-retired"}, 110 {0x9, "raw-exception-taken"}, 111 {0xa, "raw-exception-return"}, 112 {0xb, "raw-cid-write-retired"}, 113 {0xc, "raw-pc-write-retired"}, 114 {0xd, "raw-br-immed-retired"}, 115 {0xe, "raw-br-return-retired"}, 116 {0xf, "raw-unaligned-ldst-retired"}, 117 {0x10, "raw-br-mis-pred"}, 118 {0x11, "raw-cpu-cycles"}, 119 {0x12, "raw-br-pred"}, 120 {0x13, "raw-mem-access"}, 121 {0x14, "raw-l1-icache"}, 122 {0x15, "raw-l1-dcache-wb"}, 123 {0x16, "raw-l2-dcache"}, 124 {0x17, "raw-l2-dcache-refill"}, 125 {0x18, "raw-l2-dcache-wb"}, 126 {0x19, "raw-bus-access"}, 127 {0x1a, "raw-memory-error"}, 128 {0x1b, "raw-inst-spec"}, 129 {0x1c, "raw-ttbr-write-retired"}, 130 {0x1d, "raw-bus-cycles"}, 131 {0x1f, "raw-l1-dcache-allocate"}, 132 {0x20, "raw-l2-dcache-allocate"}, 133 {0x21, "raw-br-retired"}, 134 {0x22, "raw-br-mis-pred-retired"}, 135 {0x23, "raw-stall-frontend"}, 136 {0x24, "raw-stall-backend"}, 137 {0x25, "raw-l1-dtlb"}, 138 {0x26, "raw-l1-itlb"}, 139 {0x27, "raw-l2-icache"}, 140 {0x28, "raw-l2-icache-refill"}, 141 {0x29, "raw-l3-dcache-allocate"}, 142 {0x2a, "raw-l3-dcache-refill"}, 143 {0x2b, "raw-l3-dcache"}, 144 {0x2c, "raw-l3-dcache-wb"}, 145 {0x2d, "raw-l2-dtlb-refill"}, 146 {0x2e, "raw-l2-itlb-refill"}, 147 {0x2f, "raw-l2-dtlb"}, 148 {0x30, "raw-l2-itlb"}, 149 }; 150 static ConfigTable PERF_TRACEPOINT_CONFIGS = { 151 152 }; 153 154 static const std::map<perf_type_id, std::string> PERF_TYPES = { 155 {PERF_TYPE_HARDWARE, "hardware"}, 156 {PERF_TYPE_SOFTWARE, "software"}, 157 {PERF_TYPE_TRACEPOINT, "tracepoint"}, 158 {PERF_TYPE_HW_CACHE, "hardware cache"}, 159 {PERF_TYPE_RAW, "raw"}, 160 }; 161 162 static std::map<perf_type_id, ConfigTable> TYPE_CONFIGS = { 163 {PERF_TYPE_HARDWARE, (PERF_HW_CONFIGS)}, {PERF_TYPE_SOFTWARE, (PERF_SW_CONFIGS)}, 164 {PERF_TYPE_HW_CACHE, (PERF_HW_CACHE_CONFIGS)}, {PERF_TYPE_RAW, (PERF_RAW_CONFIGS)}, 165 {PERF_TYPE_TRACEPOINT, (PERF_TRACEPOINT_CONFIGS)}, 166 }; 167 168 // default config 169 static const std::vector<__u64> DEFAULT_HW_CONFIGS = { 170 PERF_COUNT_HW_CPU_CYCLES, 171 #if defined(__aarch64__) 172 PERF_COUNT_HW_STALLED_CYCLES_FRONTEND, 173 PERF_COUNT_HW_STALLED_CYCLES_BACKEND, 174 #endif 175 PERF_COUNT_HW_INSTRUCTIONS, 176 PERF_COUNT_HW_BRANCH_INSTRUCTIONS, 177 PERF_COUNT_HW_BRANCH_MISSES, 178 }; 179 static const std::vector<__u64> DEFAULT_SW_CONFIGS = { 180 PERF_COUNT_SW_TASK_CLOCK, 181 PERF_COUNT_SW_CONTEXT_SWITCHES, 182 PERF_COUNT_SW_PAGE_FAULTS, 183 }; 184 static const std::map<perf_type_id, std::vector<__u64>> DEFAULT_TYPE_CONFIGS = { 185 {PERF_TYPE_HARDWARE, DEFAULT_HW_CONFIGS}, 186 {PERF_TYPE_SOFTWARE, DEFAULT_SW_CONFIGS}, 187 }; 188 189 struct read_format_event { 190 __u64 value; /* The value of the event */ 191 __u64 id; /* if PERF_FORMAT_ID */ 192 }; 193 194 struct read_format_group { 195 __u64 nr; /* The number of events */ 196 __u64 time_enabled; /* if PERF_FORMAT_TOTAL_TIME_ENABLED */ 197 __u64 time_running; /* if PERF_FORMAT_TOTAL_TIME_RUNNING */ 198 read_format_event events[1]; 199 }; 200 201 struct read_format_no_group { 202 __u64 value; /* The value of the event */ 203 __u64 time_enabled; /* if PERF_FORMAT_TOTAL_TIME_ENABLED */ 204 __u64 time_running; /* if PERF_FORMAT_TOTAL_TIME_RUNNING */ 205 __u64 id; /* if PERF_FORMAT_ID */ 206 }; 207 208 /* 209 2 allow only user-space measurements (default since 210 Linux 4.6). 211 1 allow both kernel and user measurements (default 212 before Linux 4.6). 213 0 allow access to CPU-specific data but not raw 214 tracepoint samples. 215 -1 no restrictions. 216 */ 217 enum PerfEventParanoid { 218 NOLIMIT = -1, 219 KERNEL_USER_CPU = 0, 220 KERNEL_USER = 1, 221 USER = 2, 222 UNKNOW = 99, 223 }; 224 225 class PerfEvents { 226 public: 227 static constexpr uint64_t DEFAULT_SAMPLE_FREQUNCY = 4000; 228 static constexpr uint64_t DEFAULT_SAMPLE_PERIOD = 1; 229 static constexpr uint64_t DEFAULT_TIMEOUT = 10 * 1000; 230 static constexpr size_t MIN_BUFFER_SIZE = 64 * 1024 * 1024; 231 #ifdef LITTLE_MEMORY 232 static constexpr size_t MAX_BUFFER_SIZE = 128 * 1024 * 1024; 233 #else 234 static constexpr size_t MAX_BUFFER_SIZE = 256 * 1024 * 1024; 235 #endif 236 static constexpr size_t BUFFER_LOW_LEVEL = 10 * 1024 * 1024; 237 static constexpr size_t BUFFER_CRITICAL_LEVEL = 5 * 1024 * 1024; 238 239 PerfEvents(); 240 ~PerfEvents(); 241 242 bool AddEvents(const std::vector<std::string> &eventStrings, bool group = false); 243 bool PrepareTracking(void); 244 bool StartTracking(bool immediately = true); 245 bool StopTracking(void); 246 bool PauseTracking(void); 247 bool ResumeTracking(void); 248 /* call sequence 249 1. setXXX 250 2. AddEvents() 251 3. PrepareTracking 252 4. StartTracking (blocking...) 253 */ 254 bool EnableTracking(); 255 bool IsTrackRunning(); 256 257 void SetSystemTarget(bool); 258 void SetCpu(const std::vector<pid_t> cpus); // cpu id must be [0~N] 259 void SetPid(const std::vector<pid_t> pids); // tis is same as pid in kernel 260 void SetTimeOut(float timeOut); 261 void SetTimeReport(int); 262 void SetVerboseReport(bool); 263 bool AddOffCpuEvent(); 264 SetTrackedCommand(const std::vector<std::string> & trackedCommand)265 inline void SetTrackedCommand(const std::vector<std::string> &trackedCommand) 266 { 267 if (!trackedCommand.empty()) { 268 trackedCommand_ = TrackedCommand::CreateInstance(trackedCommand); 269 } 270 } 271 272 void SetSampleFrequency(unsigned int frequency); 273 void SetSamplePeriod(unsigned int period); 274 275 enum SampleStackType { 276 NONE, 277 FP, 278 DWARF, 279 }; 280 void SetSampleStackType(SampleStackType type); 281 void SetDwarfSampleStackSize(uint32_t stackSize); 282 void SetMmapPages(size_t mmapPages); 283 std::vector<AttrWithId> GetAttrWithId() const; 284 SetInherit(bool inherit)285 void SetInherit(bool inherit) 286 { 287 inherit_ = inherit; 288 }; SetClockId(int clockId)289 void SetClockId(int clockId) 290 { 291 clockId_ = clockId; 292 }; 293 bool SetBranchSampleType(uint64_t value); 294 bool AddDefaultEvent(perf_type_id type); 295 296 std::map<__u64, std::string> GetSupportEvents(perf_type_id type); 297 298 struct CountEvent { 299 bool userOnly = false; 300 bool kernelOnly = false; 301 __u64 eventCount = 0; 302 __u64 time_enabled = 0; 303 __u64 time_running = 0; 304 __u64 id = 0; 305 double used_cpus = 0; 306 }; 307 using StatCallBack = 308 std::function<void(const std::map<std::string, std::unique_ptr<PerfEvents::CountEvent>> &)>; 309 using RecordCallBack = std::function<bool(std::unique_ptr<PerfEventRecord>)>; 310 311 void SetStatCallBack(StatCallBack reportCallBack); 312 void SetRecordCallBack(RecordCallBack recordCallBack); GetLostSamples(size_t & lostSamples,size_t & lostNonSamples)313 void GetLostSamples(size_t &lostSamples, size_t &lostNonSamples) 314 { 315 lostSamples = lostSamples_; 316 lostNonSamples = lostNonSamples_; 317 } 318 319 // review: remove this function. GetStaticConfigName(perf_type_id type_id,__u64 config_id)320 static const std::string GetStaticConfigName(perf_type_id type_id, __u64 config_id) 321 { 322 auto typeConfigs = TYPE_CONFIGS.find(type_id); 323 if (typeConfigs != TYPE_CONFIGS.end()) { 324 auto configs = typeConfigs->second; 325 auto config = configs.find(config_id); 326 if (config != configs.end()) { 327 return config->second; 328 } else { 329 HLOGW("config not found for %u:%lld in %zu:%zu", type_id, config_id, 330 TYPE_CONFIGS.size(), configs.size()); 331 // dump all config size 332 for (auto types : TYPE_CONFIGS) { 333 HLOGV("type id %d %zu", types.first, types.second.size()); 334 } 335 } 336 } else { 337 HLOGW("type not found for %d in %zu", type_id, TYPE_CONFIGS.size()); 338 } 339 return "<not found>"; 340 }; 341 GetTraceConfigName(__u64 config_id)342 const std::string GetTraceConfigName(__u64 config_id) 343 { 344 auto config = traceConfigTable.find(config_id); 345 if (config != traceConfigTable.end()) { 346 return config->second; 347 } else { 348 HLOGW("config not found for %lld in traceConfigTable.", config_id); 349 } 350 return "<not found>"; 351 }; 352 353 static const std::string GetTypeName(perf_type_id type_id); 354 bool ParseEventName(const std::string &nameStr, std::string &name, bool &excludeUser, 355 bool &excludeKernel, bool &isTracePoint); 356 357 // mmap one fd for each cpu 358 struct MmapFd { 359 int fd; 360 perf_event_mmap_page *mmapPage = nullptr; 361 uint8_t *buf = nullptr; 362 size_t bufSize = 0; 363 // for read and sort 364 size_t dataSize = 0; 365 perf_event_header header; 366 uint64_t timestamp = 0; 367 const perf_event_attr *attr = nullptr; 368 size_t posCallChain = 0; 369 }; 370 371 private: 372 size_t recordEventCount_ = 0; // only for debug time 373 #ifdef HIPERF_DEBUG_TIME 374 std::chrono::microseconds recordCallBackTime_ = std::chrono::microseconds::zero(); 375 std::chrono::microseconds recordWaitDataTime_ = std::chrono::microseconds::zero(); 376 std::chrono::microseconds recordSleepTime_ = std::chrono::microseconds::zero(); 377 std::chrono::microseconds recordKernelReadTime_ = std::chrono::microseconds::zero(); 378 #endif 379 size_t lostSamples_ = 0; 380 size_t lostNonSamples_ = 0; 381 382 std::unique_ptr<RingBuffer> recordBuf_ {nullptr}; 383 std::mutex mtxRrecordBuf_; 384 std::condition_variable cvRecordBuf_; 385 std::thread readRecordBufThread_; 386 std::atomic_bool readRecordThreadRunning_ = false; 387 bool startedTracking_ = false; 388 bool isLowPriorityThread_ = false; 389 void RecordLoop(); 390 void StatLoop(); 391 bool IsRecordInMmap(); 392 void ReadRecordsFromMmaps(); 393 bool GetRecordFromMmap(MmapFd &mmap); 394 void GetRecordFieldFromMmap(MmapFd &mmap, void *dest, size_t pos, size_t size); 395 void MoveRecordToBuf(MmapFd &mmap); 396 size_t GetCallChainPosInSampleRecord(const perf_event_attr &attr); 397 size_t GetStackSizePosInSampleRecord(MmapFd &mmap); 398 bool CutStackAndMove(MmapFd &mmap); 399 void ReadRecordFromBuf(); 400 size_t CalcBufferSize(); 401 bool PrepareRecordThread(); 402 void WaitRecordThread(); 403 bool HaveTargetsExit(const std::chrono::steady_clock::time_point &startTime); 404 void ExitReadRecordBufThread(); 405 406 enum EventSpaceType { 407 UNKNOW = 0, 408 USER = 1, 409 KERNEL = 2, 410 USER_KERNEL = 3, 411 }; 412 uint8_t eventSpaceType_ = EventSpaceType::UNKNOW; 413 414 PerfEventParanoid requestPermission_ = PerfEventParanoid::USER; 415 bool CheckPermissions(PerfEventParanoid request = KERNEL_USER_CPU); 416 bool CheckOhosPermissions(); 417 418 static PerfEventParanoid perfEventParanoid_; 419 420 bool inherit_ = false; 421 std::vector<pid_t> pids_; 422 std::vector<pid_t> cpus_; 423 std::vector<OHOS::UniqueFd> groups_; 424 std::chrono::milliseconds timeOut_; // milliseconds 425 std::chrono::milliseconds timeReport_; // means same as timeOut 426 bool verboseReport_ = false; 427 bool prepared_ = false; 428 ConfigTable traceConfigTable; 429 430 unsigned int samplePeriod_ = 0; 431 unsigned int sampleFreq_ = 0; 432 433 struct FdItem { 434 OHOS::UniqueFd fd; 435 int cpu; 436 pid_t pid; 437 __u64 eventCount; 438 mutable uint64_t perf_id_ = 0; GetPrefIdFdItem439 uint64_t GetPrefId() const 440 { 441 if (perf_id_ == 0) { 442 read_format_no_group readNoGroupValue; 443 if (read(fd, &readNoGroupValue, sizeof(readNoGroupValue)) > 0) { 444 perf_id_ = readNoGroupValue.id; 445 } else { 446 HLOGW("read failed with fd %d", fd.Get()); 447 } 448 } 449 return perf_id_; 450 } 451 }; 452 struct EventItem { 453 std::string typeName; 454 std::string configName; 455 perf_event_attr attr = {}; 456 std::vector<FdItem> fdItems; 457 }; 458 struct EventGroupItem { 459 std::vector<EventItem> eventItems; 460 }; 461 std::vector<EventGroupItem> eventGroupItem_; 462 463 std::map<int, MmapFd> cpuMmap_; 464 std::vector<MmapFd *> MmapRecordHeap_; 465 466 #if !is_mingw 467 std::vector<struct pollfd> pollFds_; 468 #endif 469 const int pollTimeOut_ = 100; // ms 470 size_t pageSize_ = 4096; 471 bool systemTarget_ = false; 472 bool excludeHiperf_ = false; 473 pid_t selfPid_ = -1; 474 unsigned int mmapPages_ = 0; 475 int clockId_ = -1; 476 uint64_t branchSampleType_ = 0; 477 478 SampleStackType sampleStackType_ = SampleStackType::NONE; 479 uint32_t dwarfSampleStackSize_ = MAX_SAMPLE_STACK_SIZE; 480 481 // read records from the ring buffer singleton 482 void ReadRecordFromBuffer(); 483 void ReadRecordFromBufferThread(); 484 485 std::unique_ptr<TrackedCommand> trackedCommand_ = {}; 486 487 StatCallBack reportCallBack_; 488 RecordCallBack recordCallBack_; 489 490 void LoadTracepointEventTypesFromSystem(); 491 bool PerfEventsEnable(bool); 492 bool AddEvent(perf_type_id type, __u64 config, bool excludeUser = false, 493 bool excludeKernel = false, bool followGroup = false); 494 bool AddEvent(const std::string &eventString, bool followGroup = false); 495 bool IsEventSupport(perf_type_id type, __u64 config); 496 bool IsEventAttrSupport(perf_event_attr &attr); 497 498 std::chrono::time_point<std::chrono::steady_clock> trackingStartTime_; 499 std::chrono::time_point<std::chrono::steady_clock> trackingEndTime_; 500 std::chrono::time_point<std::chrono::steady_clock> readingStartTime_; 501 502 std::map<std::string, std::unique_ptr<CountEvent>> countEvents_; 503 504 void PutAllCpus(); 505 bool PrepareFdEvents(); 506 bool CreateFdEvents(); 507 bool StatReport(const __u64 &durationInSec); 508 bool CreateMmap(const FdItem &item, const perf_event_attr &attr); 509 GetDefaultAttr()510 const perf_event_attr *GetDefaultAttr() 511 { 512 HLOG_ASSERT(eventGroupItem_.size() > 0); 513 HLOG_ASSERT(eventGroupItem_[0].eventItems.size() > 0); 514 return &(eventGroupItem_.at(0).eventItems.at(0).attr); 515 }; 516 517 OHOS::UniqueFd Open(perf_event_attr &attr, pid_t pid = 0, int cpu = -1, int group_fd = -1, 518 unsigned long flags = 0); 519 std::unique_ptr<perf_event_attr> CreateDefaultAttr(perf_type_id type, __u64 config); 520 }; 521 } // namespace HiPerf 522 } // namespace Developtools 523 } // namespace OHOS 524 #endif