cc/api/BPFTable.cc

/*
 * Copyright (c) 2016 Facebook, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#include <fcntl.h>
#include <linux/elf.h>
#include <linux/perf_event.h>
#include <sys/epoll.h>
#include <unistd.h>
#include <cerrno>
#include <cinttypes>
#include <cstdint>
#include <cstring>
#include <iostream>
#include <memory>

#include "BPFTable.h"

#include "bcc_exception.h"
#include "bcc_syms.h"
#include "common.h"
#include "file_desc.h"
#include "libbpf.h"
#include "perf_reader.h"

namespace ebpf {

BPFTable::BPFTable(const TableDesc& desc) : BPFTableBase<void, void>(desc) {}

StatusTuple BPFTable::get_value(const std::string& key_str,
                                std::string& value_str) {
  char key[desc.key_size];
  char value[desc.leaf_size];

  StatusTuple r(0);

  r = string_to_key(key_str, key);
  if (r.code() != 0)
    return r;

  if (!lookup(key, value))
    return StatusTuple(-1, "error getting value");

  return leaf_to_string(value, value_str);
}

StatusTuple BPFTable::get_value(const std::string& key_str,
                                std::vector<std::string>& value_str) {
  size_t ncpus = get_possible_cpus().size();
  char key[desc.key_size];
  char value[desc.leaf_size * ncpus];

  StatusTuple r(0);

  r = string_to_key(key_str, key);
  if (r.code() != 0)
    return r;

  if (!lookup(key, value))
    return StatusTuple(-1, "error getting value");

  value_str.resize(ncpus);

  for (size_t i = 0; i < ncpus; i++) {
    r = leaf_to_string(value + i * desc.leaf_size, value_str.at(i));
    if (r.code() != 0)
      return r;
  }
  return StatusTuple(0);
}

StatusTuple BPFTable::update_value(const std::string& key_str,
                                   const std::string& value_str) {
  char key[desc.key_size];
  char value[desc.leaf_size];

  StatusTuple r(0);

  r = string_to_key(key_str, key);
  if (r.code() != 0)
    return r;

  r = string_to_leaf(value_str, value);
  if (r.code() != 0)
    return r;

  if (!update(key, value))
    return StatusTuple(-1, "error updating element");

  return StatusTuple(0);
}

StatusTuple BPFTable::update_value(const std::string& key_str,
                                   const std::vector<std::string>& value_str) {
  size_t ncpus = get_possible_cpus().size();
  char key[desc.key_size];
  char value[desc.leaf_size * ncpus];

  StatusTuple r(0);

  r = string_to_key(key_str, key);
  if (r.code() != 0)
    return r;

  if (value_str.size() != ncpus)
    return StatusTuple(-1, "bad value size");

  for (size_t i = 0; i < ncpus; i++) {
    r = string_to_leaf(value_str.at(i), value + i * desc.leaf_size);
    if (r.code() != 0)
      return r;
  }

  if (!update(key, value))
    return StatusTuple(-1, "error updating element");

  return StatusTuple(0);
}

StatusTuple BPFTable::remove_value(const std::string& key_str) {
  char key[desc.key_size];

  StatusTuple r(0);

  r = string_to_key(key_str, key);
  if (r.code() != 0)
    return r;

  if (!remove(key))
    return StatusTuple(-1, "error removing element");

  return StatusTuple(0);
}

StatusTuple BPFTable::clear_table_non_atomic() {
  if (desc.type == BPF_MAP_TYPE_HASH || desc.type == BPF_MAP_TYPE_PERCPU_HASH ||
      desc.type == BPF_MAP_TYPE_LRU_HASH ||
      desc.type == BPF_MAP_TYPE_PERCPU_HASH ||
      desc.type == BPF_MAP_TYPE_HASH_OF_MAPS) {
    // For hash maps, use the first() interface (which uses get_next_key) to
    // iterate through the map and clear elements
    auto key = std::unique_ptr<void, decltype(::free)*>(::malloc(desc.key_size),
                                                        ::free);

    while (this->first(key.get()))
      if (!this->remove(key.get())) {
        return StatusTuple(-1,
                           "Failed to delete element when clearing table %s",
                           desc.name.c_str());
      }
  } else if (desc.type == BPF_MAP_TYPE_ARRAY ||
             desc.type == BPF_MAP_TYPE_PERCPU_ARRAY) {
    return StatusTuple(-1, "Array map %s do not support clearing elements",
                       desc.name.c_str());
  } else if (desc.type == BPF_MAP_TYPE_PROG_ARRAY ||
             desc.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY ||
             desc.type == BPF_MAP_TYPE_STACK_TRACE ||
             desc.type == BPF_MAP_TYPE_ARRAY_OF_MAPS) {
    // For Stack-trace and FD arrays, just iterate over all indices
    for (size_t i = 0; i < desc.max_entries; i++) {
      this->remove(&i);
    }
  } else {
    return StatusTuple(-1, "Clearing for map type of %s not supported yet",
                       desc.name.c_str());
  }

  return StatusTuple(0);
}

StatusTuple BPFTable::get_table_offline(
  std::vector<std::pair<std::string, std::string>> &res) {
  StatusTuple r(0);
  int err;

  auto key = std::unique_ptr<void, decltype(::free)*>(::malloc(desc.key_size),
                                                      ::free);
  auto value = std::unique_ptr<void, decltype(::free)*>(::malloc(desc.leaf_size),
                                                      ::free);
  std::string key_str;
  std::string value_str;

  if (desc.type == BPF_MAP_TYPE_ARRAY ||
      desc.type == BPF_MAP_TYPE_PROG_ARRAY ||
      desc.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY ||
      desc.type == BPF_MAP_TYPE_PERCPU_ARRAY ||
      desc.type == BPF_MAP_TYPE_CGROUP_ARRAY ||
      desc.type == BPF_MAP_TYPE_ARRAY_OF_MAPS ||
      desc.type == BPF_MAP_TYPE_DEVMAP ||
      desc.type == BPF_MAP_TYPE_CPUMAP ||
      desc.type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY) {
    // For arrays, just iterate over all indices
    for (size_t i = 0; i < desc.max_entries; i++) {
      err = bpf_lookup_elem(desc.fd, &i, value.get());
      if (err < 0 && errno == ENOENT) {
        // Element is not present, skip it
        continue;
      } else if (err < 0) {
        // Other error, abort
        return StatusTuple(-1, "Error looking up value: %s", std::strerror(errno));
      }

      r = key_to_string(&i, key_str);
      if (r.code() != 0)
        return r;

      r = leaf_to_string(value.get(), value_str);
      if (r.code() != 0)
        return r;
      res.emplace_back(key_str, value_str);
    }
  } else {
    res.clear();
    // For other maps, try to use the first() and next() interfaces
    if (!this->first(key.get()))
      return StatusTuple(0);

    while (true) {
      if (!this->lookup(key.get(), value.get()))
        break;
      r = key_to_string(key.get(), key_str);
      if (r.code() != 0)
        return r;

      r = leaf_to_string(value.get(), value_str);
      if (r.code() != 0)
        return r;
      res.emplace_back(key_str, value_str);
      if (!this->next(key.get(), key.get()))
        break;
    }
  }

  return StatusTuple(0);
}

size_t BPFTable::get_possible_cpu_count() { return get_possible_cpus().size(); }

BPFStackTable::BPFStackTable(const TableDesc& desc, bool use_debug_file,
                             bool check_debug_file_crc)
    : BPFTableBase<int, stacktrace_t>(desc) {
  if (desc.type != BPF_MAP_TYPE_STACK_TRACE)
    throw std::invalid_argument("Table '" + desc.name +
                                "' is not a stack table");

  symbol_option_ = {.use_debug_file = use_debug_file,
                    .check_debug_file_crc = check_debug_file_crc,
                    .use_symbol_type = (1 << STT_FUNC) | (1 << STT_GNU_IFUNC)};
}

BPFStackTable::BPFStackTable(BPFStackTable&& that)
    : BPFTableBase<int, stacktrace_t>(that.desc),
      symbol_option_(std::move(that.symbol_option_)),
      pid_sym_(std::move(that.pid_sym_)) {
  that.pid_sym_.clear();
}

BPFStackTable::~BPFStackTable() {
  for (auto it : pid_sym_)
    bcc_free_symcache(it.second, it.first);
}

void BPFStackTable::clear_table_non_atomic() {
  for (int i = 0; size_t(i) < capacity(); i++) {
    remove(&i);
  }
}

std::vector<uintptr_t> BPFStackTable::get_stack_addr(int stack_id) {
  std::vector<uintptr_t> res;
  stacktrace_t stack;
  if (stack_id < 0)
    return res;
  if (!lookup(&stack_id, &stack))
    return res;
  for (int i = 0; (i < BPF_MAX_STACK_DEPTH) && (stack.ip[i] != 0); i++)
    res.push_back(stack.ip[i]);
  return res;
}

std::vector<std::string> BPFStackTable::get_stack_symbol(int stack_id,
                                                         int pid) {
  auto addresses = get_stack_addr(stack_id);
  std::vector<std::string> res;
  if (addresses.empty())
    return res;
  res.reserve(addresses.size());

  if (pid < 0)
    pid = -1;
  if (pid_sym_.find(pid) == pid_sym_.end())
    pid_sym_[pid] = bcc_symcache_new(pid, &symbol_option_);
  void* cache = pid_sym_[pid];

  bcc_symbol symbol;
  for (auto addr : addresses)
    if (bcc_symcache_resolve(cache, addr, &symbol) != 0)
      res.emplace_back("[UNKNOWN]");
    else {
      res.push_back(symbol.demangle_name);
      bcc_symbol_free_demangle_name(&symbol);
    }

  return res;
}

BPFPerfBuffer::BPFPerfBuffer(const TableDesc& desc)
    : BPFTableBase<int, int>(desc), epfd_(-1) {
  if (desc.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY)
    throw std::invalid_argument("Table '" + desc.name +
                                "' is not a perf buffer");
}

StatusTuple BPFPerfBuffer::open_on_cpu(perf_reader_raw_cb cb,
                                       perf_reader_lost_cb lost_cb, int cpu,
                                       void* cb_cookie, int page_cnt) {
  if (cpu_readers_.find(cpu) != cpu_readers_.end())
    return StatusTuple(-1, "Perf buffer already open on CPU %d", cpu);

  auto reader = static_cast<perf_reader*>(
      bpf_open_perf_buffer(cb, lost_cb, cb_cookie, -1, cpu, page_cnt));
  if (reader == nullptr)
    return StatusTuple(-1, "Unable to construct perf reader");

  int reader_fd = perf_reader_fd(reader);
  if (!update(&cpu, &reader_fd)) {
    perf_reader_free(static_cast<void*>(reader));
    return StatusTuple(-1, "Unable to open perf buffer on CPU %d: %s", cpu,
                       std::strerror(errno));
  }

  struct epoll_event event = {};
  event.events = EPOLLIN;
  event.data.ptr = static_cast<void*>(reader);
  if (epoll_ctl(epfd_, EPOLL_CTL_ADD, reader_fd, &event) != 0) {
    perf_reader_free(static_cast<void*>(reader));
    return StatusTuple(-1, "Unable to add perf_reader FD to epoll: %s",
                       std::strerror(errno));
  }

  cpu_readers_[cpu] = reader;
  return StatusTuple(0);
}

StatusTuple BPFPerfBuffer::open_all_cpu(perf_reader_raw_cb cb,
                                        perf_reader_lost_cb lost_cb,
                                        void* cb_cookie, int page_cnt) {
  if (cpu_readers_.size() != 0 || epfd_ != -1)
    return StatusTuple(-1, "Previously opened perf buffer not cleaned");

  std::vector<int> cpus = get_online_cpus();
  ep_events_.reset(new epoll_event[cpus.size()]);
  epfd_ = epoll_create1(EPOLL_CLOEXEC);

  for (int i : cpus) {
    auto res = open_on_cpu(cb, lost_cb, i, cb_cookie, page_cnt);
    if (res.code() != 0) {
      TRY2(close_all_cpu());
      return res;
    }
  }
  return StatusTuple(0);
}

StatusTuple BPFPerfBuffer::close_on_cpu(int cpu) {
  auto it = cpu_readers_.find(cpu);
  if (it == cpu_readers_.end())
    return StatusTuple(0);
  perf_reader_free(static_cast<void*>(it->second));
  if (!remove(const_cast<int*>(&(it->first))))
    return StatusTuple(-1, "Unable to close perf buffer on CPU %d", it->first);
  cpu_readers_.erase(it);
  return StatusTuple(0);
}

StatusTuple BPFPerfBuffer::close_all_cpu() {
  std::string errors;
  bool has_error = false;

  if (epfd_ >= 0) {
    int close_res = close(epfd_);
    epfd_ = -1;
    ep_events_.reset();
    if (close_res != 0) {
      has_error = true;
      errors += std::string(std::strerror(errno)) + "\n";
    }
  }

  std::vector<int> opened_cpus;
  for (auto it : cpu_readers_)
    opened_cpus.push_back(it.first);
  for (int i : opened_cpus) {
    auto res = close_on_cpu(i);
    if (res.code() != 0) {
      errors += "Failed to close CPU" + std::to_string(i) + " perf buffer: ";
      errors += res.msg() + "\n";
      has_error = true;
    }
  }

  if (has_error)
    return StatusTuple(-1, errors);
  return StatusTuple(0);
}

int BPFPerfBuffer::poll(int timeout_ms) {
  if (epfd_ < 0)
    return -1;
  int cnt =
      epoll_wait(epfd_, ep_events_.get(), cpu_readers_.size(), timeout_ms);
  for (int i = 0; i < cnt; i++)
    perf_reader_event_read(static_cast<perf_reader*>(ep_events_[i].data.ptr));
  return cnt;
}

BPFPerfBuffer::~BPFPerfBuffer() {
  auto res = close_all_cpu();
  if (res.code() != 0)
    std::cerr << "Failed to close all perf buffer on destruction: " << res.msg()
              << std::endl;
}

BPFPerfEventArray::BPFPerfEventArray(const TableDesc& desc)
    : BPFTableBase<int, int>(desc) {
  if (desc.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY)
    throw std::invalid_argument("Table '" + desc.name +
                                "' is not a perf event array");
}

StatusTuple BPFPerfEventArray::open_all_cpu(uint32_t type, uint64_t config) {
  if (cpu_fds_.size() != 0)
    return StatusTuple(-1, "Previously opened perf event not cleaned");

  std::vector<int> cpus = get_online_cpus();

  for (int i : cpus) {
    auto res = open_on_cpu(i, type, config);
    if (res.code() != 0) {
      TRY2(close_all_cpu());
      return res;
    }
  }
  return StatusTuple(0);
}

StatusTuple BPFPerfEventArray::close_all_cpu() {
  std::string errors;
  bool has_error = false;

  std::vector<int> opened_cpus;
  for (auto it : cpu_fds_)
    opened_cpus.push_back(it.first);
  for (int i : opened_cpus) {
    auto res = close_on_cpu(i);
    if (res.code() != 0) {
      errors += "Failed to close CPU" + std::to_string(i) + " perf event: ";
      errors += res.msg() + "\n";
      has_error = true;
    }
  }

  if (has_error)
    return StatusTuple(-1, errors);
  return StatusTuple(0);
}

StatusTuple BPFPerfEventArray::open_on_cpu(int cpu, uint32_t type,
                                           uint64_t config) {
  if (cpu_fds_.find(cpu) != cpu_fds_.end())
    return StatusTuple(-1, "Perf event already open on CPU %d", cpu);
  int fd = bpf_open_perf_event(type, config, -1, cpu);
  if (fd < 0) {
    return StatusTuple(-1, "Error constructing perf event %" PRIu32 ":%" PRIu64,
                       type, config);
  }
  if (!update(&cpu, &fd)) {
    bpf_close_perf_event_fd(fd);
    return StatusTuple(-1, "Unable to open perf event on CPU %d: %s", cpu,
                       std::strerror(errno));
  }
  cpu_fds_[cpu] = fd;
  return StatusTuple(0);
}

StatusTuple BPFPerfEventArray::close_on_cpu(int cpu) {
  auto it = cpu_fds_.find(cpu);
  if (it == cpu_fds_.end()) {
    return StatusTuple(0);
  }
  bpf_close_perf_event_fd(it->second);
  cpu_fds_.erase(it);
  return StatusTuple(0);
}

BPFPerfEventArray::~BPFPerfEventArray() {
  auto res = close_all_cpu();
  if (res.code() != 0) {
    std::cerr << "Failed to close all perf buffer on destruction: " << res.msg()
              << std::endl;
  }
}

BPFProgTable::BPFProgTable(const TableDesc& desc)
    : BPFTableBase<int, int>(desc) {
  if (desc.type != BPF_MAP_TYPE_PROG_ARRAY)
    throw std::invalid_argument("Table '" + desc.name +
                                "' is not a prog table");
}

StatusTuple BPFProgTable::update_value(const int& index, const int& prog_fd) {
  if (!this->update(const_cast<int*>(&index), const_cast<int*>(&prog_fd)))
    return StatusTuple(-1, "Error updating value: %s", std::strerror(errno));
  return StatusTuple(0);
}

StatusTuple BPFProgTable::remove_value(const int& index) {
  if (!this->remove(const_cast<int*>(&index)))
    return StatusTuple(-1, "Error removing value: %s", std::strerror(errno));
  return StatusTuple(0);
}

BPFCgroupArray::BPFCgroupArray(const TableDesc& desc)
    : BPFTableBase<int, int>(desc) {
  if (desc.type != BPF_MAP_TYPE_CGROUP_ARRAY)
    throw std::invalid_argument("Table '" + desc.name +
                                "' is not a cgroup array");
}

StatusTuple BPFCgroupArray::update_value(const int& index,
                                         const int& cgroup2_fd) {
  if (!this->update(const_cast<int*>(&index), const_cast<int*>(&cgroup2_fd)))
    return StatusTuple(-1, "Error updating value: %s", std::strerror(errno));
  return StatusTuple(0);
}

StatusTuple BPFCgroupArray::update_value(const int& index,
                                         const std::string& cgroup2_path) {
  FileDesc f(::open(cgroup2_path.c_str(), O_RDONLY | O_CLOEXEC));
  if ((int)f < 0)
    return StatusTuple(-1, "Unable to open %s", cgroup2_path.c_str());
  TRY2(update_value(index, (int)f));
  return StatusTuple(0);
}

StatusTuple BPFCgroupArray::remove_value(const int& index) {
  if (!this->remove(const_cast<int*>(&index)))
    return StatusTuple(-1, "Error removing value: %s", std::strerror(errno));
  return StatusTuple(0);
}

BPFDevmapTable::BPFDevmapTable(const TableDesc& desc)
    : BPFTableBase<int, int>(desc) {
    if(desc.type != BPF_MAP_TYPE_DEVMAP)
      throw std::invalid_argument("Table '" + desc.name +
                                  "' is not a devmap table");
}

StatusTuple BPFDevmapTable::update_value(const int& index,
                                         const int& value) {
    if (!this->update(const_cast<int*>(&index), const_cast<int*>(&value)))
      return StatusTuple(-1, "Error updating value: %s", std::strerror(errno));
    return StatusTuple(0);
}

StatusTuple BPFDevmapTable::get_value(const int& index,
                                      int& value) {
    if (!this->lookup(const_cast<int*>(&index), &value))
      return StatusTuple(-1, "Error getting value: %s", std::strerror(errno));
    return StatusTuple(0);
}

StatusTuple BPFDevmapTable::remove_value(const int& index) {
    if (!this->remove(const_cast<int*>(&index)))
      return StatusTuple(-1, "Error removing value: %s", std::strerror(errno));
    return StatusTuple(0);
}

}  // namespace ebpf