/* * Copyright (c) 2016 Facebook, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #include #include #include #include #include #include #include #include #include #include "bcc_exception.h" #include "bcc_syms.h" #include "bpf_module.h" #include "libbpf.h" #include "perf_reader.h" #include "table_desc.h" #include "linux/bpf.h" namespace ebpf { template class BPFQueueStackTableBase { public: size_t capacity() const { return desc.max_entries; } StatusTuple string_to_leaf(const std::string& value_str, ValueType* value) { return desc.leaf_sscanf(value_str.c_str(), value); } StatusTuple leaf_to_string(const ValueType* value, std::string& value_str) { char buf[8 * desc.leaf_size]; StatusTuple rc = desc.leaf_snprintf(buf, sizeof(buf), value); if (rc.ok()) value_str.assign(buf); return rc; } int get_fd() { return desc.fd; } protected: explicit BPFQueueStackTableBase(const TableDesc& desc) : desc(desc) {} bool pop(void *value) { return bpf_lookup_and_delete(desc.fd, nullptr, value) >= 0; } // Flags are extremely useful, since they completely changes extraction behaviour // (eg. if flag BPF_EXIST, then if the queue/stack is full remove the oldest one) bool push(void *value, unsigned long long int flags) { return bpf_update_elem(desc.fd, nullptr, value, flags) >= 0; } bool peek(void *value) { return bpf_lookup_elem(desc.fd, nullptr, value) >= 0; } const TableDesc& desc; }; template class BPFTableBase { public: size_t capacity() { return desc.max_entries; } StatusTuple string_to_key(const std::string& key_str, KeyType* key) { return desc.key_sscanf(key_str.c_str(), key); } StatusTuple string_to_leaf(const std::string& value_str, ValueType* value) { return desc.leaf_sscanf(value_str.c_str(), value); } StatusTuple key_to_string(const KeyType* key, std::string& key_str) { char buf[8 * desc.key_size]; StatusTuple rc = desc.key_snprintf(buf, sizeof(buf), key); if (rc.ok()) key_str.assign(buf); return rc; } StatusTuple leaf_to_string(const ValueType* value, std::string& value_str) { char buf[8 * desc.leaf_size]; StatusTuple rc = desc.leaf_snprintf(buf, sizeof(buf), value); if (rc.ok()) value_str.assign(buf); return rc; } int get_fd() { return desc.fd; } protected: explicit BPFTableBase(const TableDesc& desc) : desc(desc) {} bool lookup(void* key, void* value) { return bpf_lookup_elem(desc.fd, key, value) >= 0; } bool first(void* key) { return bpf_get_first_key(desc.fd, key, desc.key_size) >= 0; } bool next(void* key, void* next_key) { return bpf_get_next_key(desc.fd, key, next_key) >= 0; } bool update(void* key, void* value) { return bpf_update_elem(desc.fd, key, value, 0) >= 0; } bool remove(void* key) { return bpf_delete_elem(desc.fd, key) >= 0; } const TableDesc& desc; }; class BPFTable : public BPFTableBase { public: BPFTable(const TableDesc& desc); StatusTuple get_value(const std::string& key_str, std::string& value); StatusTuple get_value(const std::string& key_str, std::vector& value); StatusTuple update_value(const std::string& key_str, const std::string& value_str); StatusTuple update_value(const std::string& key_str, const std::vector& value_str); StatusTuple remove_value(const std::string& key_str); StatusTuple clear_table_non_atomic(); StatusTuple get_table_offline(std::vector> &res); static size_t get_possible_cpu_count(); }; template void* get_value_addr(ValueType& t) { return &t; } template void* get_value_addr(std::vector& t) { return t.data(); } template class BPFQueueStackTable : public BPFQueueStackTableBase { public: explicit BPFQueueStackTable(const TableDesc& desc) : BPFQueueStackTableBase(desc) { if (desc.type != BPF_MAP_TYPE_QUEUE && desc.type != BPF_MAP_TYPE_STACK) throw std::invalid_argument("Table '" + desc.name + "' is not a queue/stack table"); } virtual StatusTuple pop_value(ValueType& value) { if (!this->pop(get_value_addr(value))) return StatusTuple(-1, "Error getting value: %s", std::strerror(errno)); return StatusTuple::OK(); } virtual StatusTuple push_value(const ValueType& value, unsigned long long int flags = 0) { if (!this->push(get_value_addr(const_cast(value)), flags)) return StatusTuple(-1, "Error updating value: %s", std::strerror(errno)); return StatusTuple::OK(); } virtual StatusTuple get_head(const ValueType& value) { if (!this->peek(get_value_addr(const_cast(value)))) return StatusTuple(-1, "Error peeking value: %s", std::strerror(errno)); return StatusTuple::OK(); } }; template class BPFArrayTable : public BPFTableBase { public: BPFArrayTable(const TableDesc& desc) : BPFTableBase(desc) { if (desc.type != BPF_MAP_TYPE_ARRAY && desc.type != BPF_MAP_TYPE_PERCPU_ARRAY) throw std::invalid_argument("Table '" + desc.name + "' is not an array table"); } virtual StatusTuple get_value(const int& index, ValueType& value) { if (!this->lookup(const_cast(&index), get_value_addr(value))) return StatusTuple(-1, "Error getting value: %s", std::strerror(errno)); return StatusTuple::OK(); } virtual StatusTuple update_value(const int& index, const ValueType& value) { if (!this->update(const_cast(&index), get_value_addr(const_cast(value)))) return StatusTuple(-1, "Error updating value: %s", std::strerror(errno)); return StatusTuple::OK(); } ValueType operator[](const int& key) { ValueType value; get_value(key, value); return value; } std::vector get_table_offline() { std::vector res(this->capacity()); for (int i = 0; i < (int)this->capacity(); i++) { get_value(i, res[i]); } return res; } }; template class BPFPercpuArrayTable : public BPFArrayTable> { public: BPFPercpuArrayTable(const TableDesc& desc) : BPFArrayTable>(desc), ncpus(BPFTable::get_possible_cpu_count()) { if (desc.type != BPF_MAP_TYPE_PERCPU_ARRAY) throw std::invalid_argument("Table '" + desc.name + "' is not a percpu array table"); // leaf structures have to be aligned to 8 bytes as hardcoded in the linux // kernel. if (sizeof(ValueType) % 8) throw std::invalid_argument("leaf must be aligned to 8 bytes"); } StatusTuple get_value(const int& index, std::vector& value) { value.resize(ncpus); return BPFArrayTable>::get_value(index, value); } StatusTuple update_value(const int& index, const std::vector& value) { if (value.size() != ncpus) return StatusTuple(-1, "bad value size"); return BPFArrayTable>::update_value(index, value); } private: unsigned int ncpus; }; template class BPFHashTable : public BPFTableBase { public: explicit BPFHashTable(const TableDesc& desc) : BPFTableBase(desc) { if (desc.type != BPF_MAP_TYPE_HASH && desc.type != BPF_MAP_TYPE_PERCPU_HASH && desc.type != BPF_MAP_TYPE_LRU_HASH && desc.type != BPF_MAP_TYPE_LRU_PERCPU_HASH) throw std::invalid_argument("Table '" + desc.name + "' is not a hash table"); } virtual StatusTuple get_value(const KeyType& key, ValueType& value) { if (!this->lookup(const_cast(&key), get_value_addr(value))) return StatusTuple(-1, "Error getting value: %s", std::strerror(errno)); return StatusTuple::OK(); } virtual StatusTuple update_value(const KeyType& key, const ValueType& value) { if (!this->update(const_cast(&key), get_value_addr(const_cast(value)))) return StatusTuple(-1, "Error updating value: %s", std::strerror(errno)); return StatusTuple::OK(); } virtual StatusTuple remove_value(const KeyType& key) { if (!this->remove(const_cast(&key))) return StatusTuple(-1, "Error removing value: %s", std::strerror(errno)); return StatusTuple::OK(); } ValueType operator[](const KeyType& key) { ValueType value; get_value(key, value); return value; } std::vector> get_table_offline() { std::vector> res; KeyType cur; ValueType value; StatusTuple r(0); if (!this->first(&cur)) return res; while (true) { r = get_value(cur, value); if (!r.ok()) break; res.emplace_back(cur, value); if (!this->next(&cur, &cur)) break; } return res; } StatusTuple clear_table_non_atomic() { KeyType cur; while (this->first(&cur)) TRY2(remove_value(cur)); return StatusTuple::OK(); } }; template class BPFPercpuHashTable : public BPFHashTable> { public: explicit BPFPercpuHashTable(const TableDesc& desc) : BPFHashTable>(desc), ncpus(BPFTable::get_possible_cpu_count()) { if (desc.type != BPF_MAP_TYPE_PERCPU_HASH && desc.type != BPF_MAP_TYPE_LRU_PERCPU_HASH) throw std::invalid_argument("Table '" + desc.name + "' is not a percpu hash table"); // leaf structures have to be aligned to 8 bytes as hardcoded in the linux // kernel. if (sizeof(ValueType) % 8) throw std::invalid_argument("leaf must be aligned to 8 bytes"); } StatusTuple get_value(const KeyType& key, std::vector& value) { value.resize(ncpus); return BPFHashTable>::get_value(key, value); } StatusTuple update_value(const KeyType& key, const std::vector& value) { if (value.size() != ncpus) return StatusTuple(-1, "bad value size"); return BPFHashTable>::update_value(key, value); } private: unsigned int ncpus; }; // From src/cc/export/helpers.h static const int BPF_MAX_STACK_DEPTH = 127; struct stacktrace_t { uintptr_t ip[BPF_MAX_STACK_DEPTH]; }; class BPFStackTable : public BPFTableBase { public: BPFStackTable(const TableDesc& desc, bool use_debug_file, bool check_debug_file_crc); BPFStackTable(BPFStackTable&& that); ~BPFStackTable(); void free_symcache(int pid); void clear_table_non_atomic(); std::vector get_stack_addr(int stack_id); std::vector get_stack_symbol(int stack_id, int pid); private: bcc_symbol_option symbol_option_; std::map pid_sym_; }; // from src/cc/export/helpers.h struct stacktrace_buildid_t { struct bpf_stack_build_id trace[BPF_MAX_STACK_DEPTH]; }; class BPFStackBuildIdTable : public BPFTableBase { public: BPFStackBuildIdTable(const TableDesc& desc, bool use_debug_file, bool check_debug_file_crc, void *bsymcache); ~BPFStackBuildIdTable() = default; void clear_table_non_atomic(); std::vector get_stack_addr(int stack_id); std::vector get_stack_symbol(int stack_id); private: void *bsymcache_; bcc_symbol_option symbol_option_; }; class BPFPerfBuffer : public BPFTableBase { public: BPFPerfBuffer(const TableDesc& desc); ~BPFPerfBuffer(); StatusTuple open_all_cpu(perf_reader_raw_cb cb, perf_reader_lost_cb lost_cb, void* cb_cookie, int page_cnt); StatusTuple open_all_cpu(perf_reader_raw_cb cb, perf_reader_lost_cb lost_cb, void* cb_cookie, int page_cnt, int wakeup_events); StatusTuple close_all_cpu(); int poll(int timeout_ms); int consume(); private: StatusTuple open_on_cpu(perf_reader_raw_cb cb, perf_reader_lost_cb lost_cb, void* cb_cookie, int page_cnt, struct bcc_perf_buffer_opts& opts); StatusTuple close_on_cpu(int cpu); std::map cpu_readers_; int epfd_; std::unique_ptr ep_events_; }; class BPFPerfEventArray : public BPFTableBase { public: BPFPerfEventArray(const TableDesc& desc); ~BPFPerfEventArray(); StatusTuple open_all_cpu(uint32_t type, uint64_t config); StatusTuple close_all_cpu(); private: StatusTuple open_on_cpu(int cpu, uint32_t type, uint64_t config); StatusTuple close_on_cpu(int cpu); std::map cpu_fds_; }; class BPFProgTable : public BPFTableBase { public: BPFProgTable(const TableDesc& desc); StatusTuple update_value(const int& index, const int& prog_fd); StatusTuple remove_value(const int& index); }; class BPFCgroupArray : public BPFTableBase { public: BPFCgroupArray(const TableDesc& desc); StatusTuple update_value(const int& index, const int& cgroup2_fd); StatusTuple update_value(const int& index, const std::string& cgroup2_path); StatusTuple remove_value(const int& index); }; class BPFDevmapTable : public BPFTableBase { public: BPFDevmapTable(const TableDesc& desc); StatusTuple update_value(const int& index, const int& value); StatusTuple get_value(const int& index, int& value); StatusTuple remove_value(const int& index); }; class BPFXskmapTable : public BPFTableBase { public: BPFXskmapTable(const TableDesc& desc); StatusTuple update_value(const int& index, const int& value); StatusTuple get_value(const int& index, int& value); StatusTuple remove_value(const int& index); }; template class BPFMapInMapTable : public BPFTableBase { public: BPFMapInMapTable(const TableDesc& desc) : BPFTableBase(desc) { if (desc.type != BPF_MAP_TYPE_ARRAY_OF_MAPS && desc.type != BPF_MAP_TYPE_HASH_OF_MAPS) throw std::invalid_argument("Table '" + desc.name + "' is not a map-in-map table"); } virtual StatusTuple update_value(const KeyType& key, const int& inner_map_fd) { if (!this->update(const_cast(&key), const_cast(&inner_map_fd))) return StatusTuple(-1, "Error updating value: %s", std::strerror(errno)); return StatusTuple::OK(); } virtual StatusTuple remove_value(const KeyType& key) { if (!this->remove(const_cast(&key))) return StatusTuple(-1, "Error removing value: %s", std::strerror(errno)); return StatusTuple::OK(); } }; class BPFSockmapTable : public BPFTableBase { public: BPFSockmapTable(const TableDesc& desc); StatusTuple update_value(const int& index, const int& value); StatusTuple remove_value(const int& index); }; class BPFSockhashTable : public BPFTableBase { public: BPFSockhashTable(const TableDesc& desc); StatusTuple update_value(const int& key, const int& value); StatusTuple remove_value(const int& key); }; template class BPFSkStorageTable : public BPFTableBase { public: BPFSkStorageTable(const TableDesc& desc) : BPFTableBase(desc) { if (desc.type != BPF_MAP_TYPE_SK_STORAGE) throw std::invalid_argument("Table '" + desc.name + "' is not a sk_storage table"); } virtual StatusTuple get_value(const int& sock_fd, ValueType& value) { if (!this->lookup(const_cast(&sock_fd), get_value_addr(value))) return StatusTuple(-1, "Error getting value: %s", std::strerror(errno)); return StatusTuple::OK(); } virtual StatusTuple update_value(const int& sock_fd, const ValueType& value) { if (!this->update(const_cast(&sock_fd), get_value_addr(const_cast(value)))) return StatusTuple(-1, "Error updating value: %s", std::strerror(errno)); return StatusTuple::OK(); } virtual StatusTuple remove_value(const int& sock_fd) { if (!this->remove(const_cast(&sock_fd))) return StatusTuple(-1, "Error removing value: %s", std::strerror(errno)); return StatusTuple::OK(); } }; template class BPFInodeStorageTable : public BPFTableBase { public: BPFInodeStorageTable(const TableDesc& desc) : BPFTableBase(desc) { if (desc.type != BPF_MAP_TYPE_INODE_STORAGE) throw std::invalid_argument("Table '" + desc.name + "' is not a inode_storage table"); } virtual StatusTuple get_value(const int& fd, ValueType& value) { if (!this->lookup(const_cast(&fd), get_value_addr(value))) return StatusTuple(-1, "Error getting value: %s", std::strerror(errno)); return StatusTuple::OK(); } virtual StatusTuple update_value(const int& fd, const ValueType& value) { if (!this->update(const_cast(&fd), get_value_addr(const_cast(value)))) return StatusTuple(-1, "Error updating value: %s", std::strerror(errno)); return StatusTuple::OK(); } virtual StatusTuple remove_value(const int& fd) { if (!this->remove(const_cast(&fd))) return StatusTuple(-1, "Error removing value: %s", std::strerror(errno)); return StatusTuple::OK(); } }; template class BPFTaskStorageTable : public BPFTableBase { public: BPFTaskStorageTable(const TableDesc& desc) : BPFTableBase(desc) { if (desc.type != BPF_MAP_TYPE_TASK_STORAGE) throw std::invalid_argument("Table '" + desc.name + "' is not a task_storage table"); } virtual StatusTuple get_value(const int& fd, ValueType& value) { if (!this->lookup(const_cast(&fd), get_value_addr(value))) return StatusTuple(-1, "Error getting value: %s", std::strerror(errno)); return StatusTuple::OK(); } virtual StatusTuple update_value(const int& fd, const ValueType& value) { if (!this->update(const_cast(&fd), get_value_addr(const_cast(value)))) return StatusTuple(-1, "Error updating value: %s", std::strerror(errno)); return StatusTuple::OK(); } virtual StatusTuple remove_value(const int& fd) { if (!this->remove(const_cast(&fd))) return StatusTuple(-1, "Error removing value: %s", std::strerror(errno)); return StatusTuple::OK(); } }; template class BPFCgStorageTable : public BPFTableBase { public: BPFCgStorageTable(const TableDesc& desc) : BPFTableBase(desc) { if (desc.type != BPF_MAP_TYPE_CGROUP_STORAGE) throw std::invalid_argument("Table '" + desc.name + "' is not a cgroup_storage table"); } virtual StatusTuple get_value(struct bpf_cgroup_storage_key& key, ValueType& value) { if (!this->lookup(const_cast(&key), get_value_addr(value))) return StatusTuple(-1, "Error getting value: %s", std::strerror(errno)); return StatusTuple::OK(); } virtual StatusTuple update_value(struct bpf_cgroup_storage_key& key, const ValueType& value) { if (!this->update(const_cast(&key), get_value_addr(const_cast(value)))) return StatusTuple(-1, "Error updating value: %s", std::strerror(errno)); return StatusTuple::OK(); } }; template class BPFPercpuCgStorageTable : public BPFTableBase> { public: BPFPercpuCgStorageTable(const TableDesc& desc) : BPFTableBase>(desc) { if (desc.type != BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE) throw std::invalid_argument("Table '" + desc.name + "' is not a percpu_cgroup_storage table"); if (sizeof(ValueType) % 8) throw std::invalid_argument("leaf must be aligned to 8 bytes"); ncpus = BPFTable::get_possible_cpu_count(); } virtual StatusTuple get_value(struct bpf_cgroup_storage_key& key, std::vector& value) { value.resize(ncpus); if (!this->lookup(const_cast(&key), get_value_addr(value))) return StatusTuple(-1, "Error getting value: %s", std::strerror(errno)); return StatusTuple::OK(); } virtual StatusTuple update_value(struct bpf_cgroup_storage_key& key, std::vector& value) { value.resize(ncpus); if (!this->update(const_cast(&key), get_value_addr(const_cast&>(value)))) return StatusTuple(-1, "Error updating value: %s", std::strerror(errno)); return StatusTuple::OK(); } private: unsigned int ncpus; }; } // namespace ebpf