1 /*
2 * Copyright (c) 2015 PLUMgrid, Inc.
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16 #include "bpf_module.h"
17
18 #include <fcntl.h>
19 #include <linux/bpf.h>
20 #include <llvm-c/Transforms/IPO.h>
21 #include <llvm/ExecutionEngine/MCJIT.h>
22 #include <llvm/ExecutionEngine/SectionMemoryManager.h>
23 #include <llvm/IR/IRPrintingPasses.h>
24 #include <llvm/IR/LLVMContext.h>
25 #include <llvm/IR/LegacyPassManager.h>
26 #include <llvm/IR/Module.h>
27
28 #if LLVM_MAJOR_VERSION >= 15
29 #include <llvm/Pass.h>
30 #endif
31
32 #include <llvm/IR/Verifier.h>
33 #include <llvm/Object/ObjectFile.h>
34 #include <llvm/Object/ELFObjectFile.h>
35 #include <llvm/Object/SymbolSize.h>
36 #include <llvm/Support/TargetSelect.h>
37 #include <llvm/Transforms/IPO.h>
38 #include <llvm/Transforms/IPO/PassManagerBuilder.h>
39 #include <net/if.h>
40 #include <sys/stat.h>
41 #include <unistd.h>
42
43 #include <map>
44 #include <set>
45 #include <string>
46 #include <iostream>
47 #include <vector>
48
49 #include "bcc_btf.h"
50 #include "bcc_debug.h"
51 #include "bcc_elf.h"
52 #include "bcc_libbpf_inc.h"
53 #include "common.h"
54 #include "exported_files.h"
55 #include "frontends/clang/b_frontend_action.h"
56 #include "frontends/clang/loader.h"
57 #include "libbpf.h"
58
59 namespace ebpf {
60
61 using std::get;
62 using std::make_tuple;
63 using std::map;
64 using std::move;
65 using std::string;
66 using std::tuple;
67 using std::unique_ptr;
68 using std::vector;
69 using namespace llvm;
70
71 // Snooping class to remember the sections as the JIT creates them
72 class MyMemoryManager : public SectionMemoryManager {
73 public:
MyMemoryManager(sec_map_def * sections,ProgFuncInfo * prog_func_info)74 explicit MyMemoryManager(sec_map_def *sections, ProgFuncInfo *prog_func_info)
75 : sections_(sections), prog_func_info_(prog_func_info) {}
76
~MyMemoryManager()77 virtual ~MyMemoryManager() {}
allocateCodeSection(uintptr_t Size,unsigned Alignment,unsigned SectionID,StringRef SectionName)78 uint8_t *allocateCodeSection(uintptr_t Size, unsigned Alignment,
79 unsigned SectionID,
80 StringRef SectionName) override {
81 // The programs need to change from fake fd to real map fd, so not allocate ReadOnly regions.
82 uint8_t *Addr = SectionMemoryManager::allocateDataSection(Size, Alignment, SectionID, SectionName, false);
83 (*sections_)[SectionName.str()] = make_tuple(Addr, Size, SectionID);
84 return Addr;
85 }
allocateDataSection(uintptr_t Size,unsigned Alignment,unsigned SectionID,StringRef SectionName,bool isReadOnly)86 uint8_t *allocateDataSection(uintptr_t Size, unsigned Alignment,
87 unsigned SectionID, StringRef SectionName,
88 bool isReadOnly) override {
89 // The lines in .BTF.ext line_info, if corresponding to remapped files, will have empty source line.
90 // The line_info will be fixed in place, so not allocate ReadOnly regions.
91 uint8_t *Addr = SectionMemoryManager::allocateDataSection(Size, Alignment, SectionID, SectionName, false);
92 (*sections_)[SectionName.str()] = make_tuple(Addr, Size, SectionID);
93 return Addr;
94 }
95
notifyObjectLoaded(ExecutionEngine * EE,const object::ObjectFile & o)96 void notifyObjectLoaded(ExecutionEngine *EE,
97 const object::ObjectFile &o) override {
98 auto sizes = llvm::object::computeSymbolSizes(o);
99 for (auto ss : sizes) {
100 auto maybe_name = ss.first.getName();
101 if (!maybe_name)
102 continue;
103
104 std::string name = maybe_name->str();
105 auto info = prog_func_info_->get_func(name);
106 if (!info)
107 continue;
108
109 auto section = ss.first.getSection();
110 if (!section)
111 continue;
112
113 auto sec_name = section.get()->getName();
114 if (!sec_name)
115 continue;
116
117 info->section_ = sec_name->str();
118 info->size_ = ss.second;
119 }
120 }
121
122 sec_map_def *sections_;
123 ProgFuncInfo *prog_func_info_;
124 };
125
BPFModule(unsigned flags,TableStorage * ts,bool rw_engine_enabled,const std::string & maps_ns,bool allow_rlimit,const char * dev_name)126 BPFModule::BPFModule(unsigned flags, TableStorage *ts, bool rw_engine_enabled,
127 const std::string &maps_ns, bool allow_rlimit,
128 const char *dev_name)
129 : flags_(flags),
130 rw_engine_enabled_(rw_engine_enabled && bpf_module_rw_engine_enabled()),
131 used_b_loader_(false),
132 allow_rlimit_(allow_rlimit),
133 ctx_(new LLVMContext),
134 id_(std::to_string((uintptr_t)this)),
135 maps_ns_(maps_ns),
136 ts_(ts), btf_(nullptr) {
137 ifindex_ = dev_name ? if_nametoindex(dev_name) : 0;
138 initialize_rw_engine();
139 LLVMInitializeBPFTarget();
140 LLVMInitializeBPFTargetMC();
141 LLVMInitializeBPFTargetInfo();
142 LLVMInitializeBPFAsmPrinter();
143 #if LLVM_MAJOR_VERSION >= 6
144 LLVMInitializeBPFAsmParser();
145 if (flags & DEBUG_SOURCE)
146 LLVMInitializeBPFDisassembler();
147 #endif
148 LLVMLinkInMCJIT(); /* call empty function to force linking of MCJIT */
149 if (!ts_) {
150 local_ts_ = createSharedTableStorage();
151 ts_ = &*local_ts_;
152 }
153 prog_func_info_ = ebpf::make_unique<ProgFuncInfo>();
154 }
155
unimplemented_sscanf(const char *,void *)156 static StatusTuple unimplemented_sscanf(const char *, void *) {
157 return StatusTuple(-1, "sscanf unimplemented");
158 }
unimplemented_snprintf(char *,size_t,const void *)159 static StatusTuple unimplemented_snprintf(char *, size_t, const void *) {
160 return StatusTuple(-1, "snprintf unimplemented");
161 }
162
~BPFModule()163 BPFModule::~BPFModule() {
164 for (auto &v : tables_) {
165 v->key_sscanf = unimplemented_sscanf;
166 v->leaf_sscanf = unimplemented_sscanf;
167 v->key_snprintf = unimplemented_snprintf;
168 v->leaf_snprintf = unimplemented_snprintf;
169 }
170
171 if (!rw_engine_enabled_) {
172 prog_func_info_->for_each_func(
173 [&](std::string name, FuncInfo &info) {
174 if (!info.start_)
175 return;
176 delete[] info.start_;
177 });
178 }
179
180 engine_.reset();
181 cleanup_rw_engine();
182 ctx_.reset();
183 prog_func_info_.reset();
184
185 if (btf_)
186 delete btf_;
187
188 ts_->DeletePrefix(Path({id_}));
189 }
190
free_bcc_memory()191 int BPFModule::free_bcc_memory() {
192 return bcc_free_memory();
193 }
194
195 // load an entire c file as a module
load_cfile(const string & file,bool in_memory,const char * cflags[],int ncflags)196 int BPFModule::load_cfile(const string &file, bool in_memory, const char *cflags[], int ncflags) {
197 ClangLoader clang_loader(&*ctx_, flags_);
198 if (clang_loader.parse(&mod_, *ts_, file, in_memory, cflags, ncflags, id_,
199 *prog_func_info_, mod_src_, maps_ns_, fake_fd_map_,
200 perf_events_))
201 return -1;
202 return 0;
203 }
204
205 // NOTE: this is a duplicate of the above, but planning to deprecate if we
206 // settle on clang as the frontend
207
208 // Load in a pre-built list of functions into the initial Module object, then
209 // build an ExecutionEngine.
load_includes(const string & text)210 int BPFModule::load_includes(const string &text) {
211 ClangLoader clang_loader(&*ctx_, flags_);
212 const char *cflags[] = {"-DB_WORKAROUND"};
213 if (clang_loader.parse(&mod_, *ts_, text, true, cflags, 1, "",
214 *prog_func_info_, mod_src_, "", fake_fd_map_,
215 perf_events_))
216 return -1;
217 return 0;
218 }
219
annotate_light()220 void BPFModule::annotate_light() {
221 for (auto fn = mod_->getFunctionList().begin(); fn != mod_->getFunctionList().end(); ++fn)
222 if (!fn->hasFnAttribute(Attribute::NoInline))
223 fn->addFnAttr(Attribute::AlwaysInline);
224
225 size_t id = 0;
226 Path path({id_});
227 for (auto it = ts_->lower_bound(path), up = ts_->upper_bound(path); it != up; ++it) {
228 TableDesc &table = it->second;
229 tables_.push_back(&it->second);
230 table_names_[table.name] = id++;
231 }
232 }
233
dump_ir(Module & mod)234 void BPFModule::dump_ir(Module &mod) {
235 legacy::PassManager PM;
236 PM.add(createPrintModulePass(errs()));
237 PM.run(mod);
238 }
239
run_pass_manager(Module & mod)240 int BPFModule::run_pass_manager(Module &mod) {
241 if (verifyModule(mod, &errs())) {
242 if (flags_ & DEBUG_LLVM_IR)
243 dump_ir(mod);
244 return -1;
245 }
246
247 legacy::PassManager PM;
248 PassManagerBuilder PMB;
249 PMB.OptLevel = 3;
250 PM.add(createFunctionInliningPass());
251 /*
252 * llvm < 4.0 needs
253 * PM.add(createAlwaysInlinerPass());
254 * llvm >= 4.0 needs
255 * PM.add(createAlwaysInlinerLegacyPass());
256 * use below 'stable' workaround
257 */
258 LLVMAddAlwaysInlinerPass(reinterpret_cast<LLVMPassManagerRef>(&PM));
259 PMB.populateModulePassManager(PM);
260 if (flags_ & DEBUG_LLVM_IR)
261 PM.add(createPrintModulePass(outs()));
262 PM.run(mod);
263 return 0;
264 }
265
load_btf(sec_map_def & sections)266 void BPFModule::load_btf(sec_map_def §ions) {
267 uint8_t *btf_sec = nullptr, *btf_ext_sec = nullptr;
268 uintptr_t btf_sec_size = 0, btf_ext_sec_size = 0;
269
270 for (auto section: sections) {
271 auto sname = section.first;
272 uint8_t *addr = get<0>(section.second);
273 uintptr_t size = get<1>(section.second);
274
275 if (strcmp(".BTF", sname.c_str()) == 0) {
276 btf_sec = addr;
277 btf_sec_size = size;
278 }
279
280 if (strcmp(".BTF.ext", sname.c_str()) == 0) {
281 btf_ext_sec = addr;
282 btf_ext_sec_size = size;
283 }
284 }
285
286 if (btf_sec == nullptr || btf_ext_sec == nullptr)
287 return;
288
289 // Both .BTF and .BTF.ext ELF sections are present.
290 // The remapped files (the main file and /virtual/include/bcc/helpers.h)
291 // will provide missing source codes in the .BTF.ext line_info table.
292 auto helpers_h = ExportedFiles::headers().find("/virtual/include/bcc/helpers.h");
293 if (helpers_h == ExportedFiles::headers().end()) {
294 fprintf(stderr, "Internal error: missing bcc/helpers.h");
295 return;
296 }
297 std::map<std::string, std::string> remapped_sources;
298 remapped_sources["/virtual/main.c"] = mod_src_;
299 remapped_sources["/virtual/include/bcc/helpers.h"] = helpers_h->second;
300
301 BTF *btf = new BTF(flags_ & DEBUG_BTF, sections);
302 int ret = btf->load(btf_sec, btf_sec_size, btf_ext_sec, btf_ext_sec_size,
303 remapped_sources);
304 if (ret) {
305 delete btf;
306 return;
307 }
308 btf_ = btf;
309 }
310
create_maps(std::map<std::string,std::pair<int,int>> & map_tids,std::map<int,int> & map_fds,std::map<std::string,int> & inner_map_fds,bool for_inner_map)311 int BPFModule::create_maps(std::map<std::string, std::pair<int, int>> &map_tids,
312 std::map<int, int> &map_fds,
313 std::map<std::string, int> &inner_map_fds,
314 bool for_inner_map) {
315 std::set<std::string> inner_maps;
316 if (for_inner_map) {
317 for (auto map : fake_fd_map_) {
318 std::string inner_map_name = get<7>(map.second);
319 if (inner_map_name.size())
320 inner_maps.insert(inner_map_name);
321 }
322 }
323
324 for (auto map : fake_fd_map_) {
325 int fd, fake_fd, map_type, key_size, value_size, max_entries, map_flags;
326 int pinned_id;
327 const char *map_name;
328 const char *pinned;
329 std::string inner_map_name;
330 int inner_map_fd = 0;
331
332 fake_fd = map.first;
333 map_type = get<0>(map.second);
334 map_name = get<1>(map.second).c_str();
335 key_size = get<2>(map.second);
336 value_size = get<3>(map.second);
337 max_entries = get<4>(map.second);
338 map_flags = get<5>(map.second);
339 pinned_id = get<6>(map.second);
340 inner_map_name = get<7>(map.second);
341
342 if (for_inner_map) {
343 if (inner_maps.find(map_name) == inner_maps.end())
344 continue;
345 if (inner_map_name.size()) {
346 fprintf(stderr, "inner map %s has inner map %s\n",
347 map_name, inner_map_name.c_str());
348 return -1;
349 }
350 } else {
351 if (inner_map_fds.find(map_name) != inner_map_fds.end())
352 continue;
353 if (inner_map_name.size())
354 inner_map_fd = inner_map_fds[inner_map_name];
355 }
356
357 if (pinned_id <= 0) {
358 struct bpf_create_map_attr attr = {};
359 attr.map_type = (enum bpf_map_type)map_type;
360 attr.name = map_name;
361 attr.key_size = key_size;
362 attr.value_size = value_size;
363 attr.max_entries = max_entries;
364 attr.map_flags = map_flags;
365 attr.map_ifindex = ifindex_;
366 attr.inner_map_fd = inner_map_fd;
367
368 if (map_tids.find(map_name) != map_tids.end()) {
369 attr.btf_fd = btf_->get_fd();
370 attr.btf_key_type_id = map_tids[map_name].first;
371 attr.btf_value_type_id = map_tids[map_name].second;
372 }
373
374 fd = bcc_create_map_xattr(&attr, allow_rlimit_);
375 } else {
376 fd = bpf_map_get_fd_by_id(pinned_id);
377 }
378
379 if (fd < 0) {
380 fprintf(stderr, "could not open bpf map: %s, error: %s\n",
381 map_name, strerror(errno));
382 return -1;
383 }
384
385 if (pinned_id == -1) {
386 pinned = get<8>(map.second).c_str();
387 if (bpf_obj_pin(fd, pinned)) {
388 fprintf(stderr, "failed to pin map: %s, error: %s\n",
389 pinned, strerror(errno));
390 return -1;
391 }
392 }
393
394 if (for_inner_map)
395 inner_map_fds[map_name] = fd;
396
397 map_fds[fake_fd] = fd;
398 }
399
400 return 0;
401 }
402
load_maps(sec_map_def & sections)403 int BPFModule::load_maps(sec_map_def §ions) {
404 // find .maps.<table_name> sections and retrieve all map key/value type id's
405 std::map<std::string, std::pair<int, int>> map_tids;
406 if (btf_) {
407 for (auto section : sections) {
408 auto sec_name = section.first;
409 if (strncmp(".maps.", sec_name.c_str(), 6) == 0) {
410 std::string map_name = sec_name.substr(6);
411 unsigned key_tid = 0, value_tid = 0;
412 unsigned expected_ksize = 0, expected_vsize = 0;
413
414 // skip extern maps, which won't be in fake_fd_map_ as they do not
415 // require explicit bpf_create_map.
416 bool is_extern = false;
417 for (auto &t : tables_) {
418 if (t->name == map_name) {
419 is_extern = t->is_extern;
420 break;
421 }
422 }
423 if (is_extern)
424 continue;
425
426 for (auto map : fake_fd_map_) {
427 std::string name;
428
429 name = get<1>(map.second);
430 if (map_name == name) {
431 expected_ksize = get<2>(map.second);
432 expected_vsize = get<3>(map.second);
433 break;
434 }
435 }
436
437 int ret = btf_->get_map_tids(map_name, expected_ksize,
438 expected_vsize, &key_tid, &value_tid);
439 if (ret)
440 continue;
441
442 map_tids[map_name] = std::make_pair(key_tid, value_tid);
443 }
444 }
445 }
446
447 // create maps
448 std::map<std::string, int> inner_map_fds;
449 std::map<int, int> map_fds;
450 if (create_maps(map_tids, map_fds, inner_map_fds, true) < 0)
451 return -1;
452 if (create_maps(map_tids, map_fds, inner_map_fds, false) < 0)
453 return -1;
454
455 // update map table fd's
456 for (auto it = ts_->begin(), up = ts_->end(); it != up; ++it) {
457 TableDesc &table = it->second;
458 if (map_fds.find(table.fake_fd) != map_fds.end()) {
459 table.fd = map_fds[table.fake_fd];
460 table.fake_fd = 0;
461 }
462 }
463
464 // update instructions
465 prog_func_info_->for_each_func([&](std::string name, FuncInfo &info) {
466 struct bpf_insn *insns = (struct bpf_insn *)info.start_;
467 uint32_t i, num_insns = info.size_ / sizeof(struct bpf_insn);
468 for (i = 0; i < num_insns; i++) {
469 if (insns[i].code == (BPF_LD | BPF_DW | BPF_IMM)) {
470 // change map_fd is it is a ld_pseudo
471 if (insns[i].src_reg == BPF_PSEUDO_MAP_FD &&
472 map_fds.find(insns[i].imm) != map_fds.end())
473 insns[i].imm = map_fds[insns[i].imm];
474 i++;
475 }
476 }
477 });
478
479 return 0;
480 }
481
finalize()482 int BPFModule::finalize() {
483 Module *mod = &*mod_;
484 sec_map_def tmp_sections,
485 *sections_p;
486
487 mod->setTargetTriple("bpf-pc-linux");
488 #if LLVM_MAJOR_VERSION >= 11
489 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
490 mod->setDataLayout("e-m:e-p:64:64-i64:64-i128:128-n32:64-S128");
491 #else
492 mod->setDataLayout("E-m:e-p:64:64-i64:64-i128:128-n32:64-S128");
493 #endif
494 #else
495 #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
496 mod->setDataLayout("e-m:e-p:64:64-i64:64-n32:64-S128");
497 #else
498 mod->setDataLayout("E-m:e-p:64:64-i64:64-n32:64-S128");
499 #endif
500 #endif
501 sections_p = rw_engine_enabled_ ? §ions_ : &tmp_sections;
502
503 string err;
504 EngineBuilder builder(move(mod_));
505 builder.setErrorStr(&err);
506 builder.setMCJITMemoryManager(
507 ebpf::make_unique<MyMemoryManager>(sections_p, &*prog_func_info_));
508 builder.setMArch("bpf");
509 #if LLVM_MAJOR_VERSION <= 11
510 builder.setUseOrcMCJITReplacement(false);
511 #endif
512 engine_ = unique_ptr<ExecutionEngine>(builder.create());
513 if (!engine_) {
514 fprintf(stderr, "Could not create ExecutionEngine: %s\n", err.c_str());
515 return -1;
516 }
517
518 engine_->setProcessAllSections(true);
519
520 if (int rc = run_pass_manager(*mod))
521 return rc;
522
523 engine_->finalizeObject();
524 prog_func_info_->for_each_func([&](std::string name, FuncInfo &info) {
525 info.start_ = (uint8_t *)engine_->getFunctionAddress(name);
526 });
527 finalize_prog_func_info();
528
529 if (flags_ & DEBUG_SOURCE) {
530 SourceDebugger src_debugger(mod, *sections_p, *prog_func_info_, mod_src_,
531 src_dbg_fmap_);
532 src_debugger.dump();
533 }
534
535 load_btf(*sections_p);
536 if (load_maps(*sections_p))
537 return -1;
538
539 if (!rw_engine_enabled_) {
540 // Setup sections_ correctly and then free llvm internal memory
541 for (auto section : tmp_sections) {
542 auto fname = section.first;
543 uintptr_t size = get<1>(section.second);
544 uint8_t *tmp_p = NULL;
545 // Only copy data for non-map sections
546 if (strncmp("maps/", section.first.c_str(), 5)) {
547 uint8_t *addr = get<0>(section.second);
548 tmp_p = new uint8_t[size];
549 memcpy(tmp_p, addr, size);
550 }
551 sections_[fname] = make_tuple(tmp_p, size, get<2>(section.second));
552 }
553
554 prog_func_info_->for_each_func([](std::string name, FuncInfo &info) {
555 uint8_t *tmp_p = new uint8_t[info.size_];
556 memcpy(tmp_p, info.start_, info.size_);
557 info.start_ = tmp_p;
558 });
559 engine_.reset();
560 ctx_.reset();
561 }
562
563 return 0;
564 }
565
finalize_prog_func_info()566 void BPFModule::finalize_prog_func_info() {
567 // prog_func_info_'s FuncInfo data is gradually populated (first in frontend
568 // action, then bpf_module). It's possible for a FuncInfo to have been
569 // created by FrontendAction but no corresponding start location found in
570 // bpf_module - filter out these functions
571 //
572 // The numeric function ids in the new prog_func_info_ are considered
573 // canonical
574 std::unique_ptr<ProgFuncInfo> finalized = ebpf::make_unique<ProgFuncInfo>();
575 prog_func_info_->for_each_func([&](std::string name, FuncInfo &info) {
576 if(info.start_) {
577 auto i = finalized->add_func(name);
578 if (i) { // should always be true
579 *i = info;
580 }
581 }
582 });
583 prog_func_info_.swap(finalized);
584 }
585
num_functions() const586 size_t BPFModule::num_functions() const { return prog_func_info_->num_funcs(); }
587
function_name(size_t id) const588 const char * BPFModule::function_name(size_t id) const {
589 auto name = prog_func_info_->func_name(id);
590 if (name)
591 return name->c_str();
592 return nullptr;
593 }
594
function_start(size_t id) const595 uint8_t * BPFModule::function_start(size_t id) const {
596 auto fn = prog_func_info_->get_func(id);
597 if (fn)
598 return fn->start_;
599 return nullptr;
600 }
601
function_start(const string & name) const602 uint8_t * BPFModule::function_start(const string &name) const {
603 auto fn = prog_func_info_->get_func(name);
604 if (fn)
605 return fn->start_;
606 return nullptr;
607 }
608
function_source(const string & name) const609 const char * BPFModule::function_source(const string &name) const {
610 auto fn = prog_func_info_->get_func(name);
611 if (fn)
612 return fn->src_.c_str();
613 return "";
614 }
615
function_source_rewritten(const string & name) const616 const char * BPFModule::function_source_rewritten(const string &name) const {
617 auto fn = prog_func_info_->get_func(name);
618 if (fn)
619 return fn->src_rewritten_.c_str();
620 return "";
621 }
622
annotate_prog_tag(const string & name,int prog_fd,struct bpf_insn * insns,int prog_len)623 int BPFModule::annotate_prog_tag(const string &name, int prog_fd,
624 struct bpf_insn *insns, int prog_len) {
625 unsigned long long tag1, tag2;
626 int err;
627
628 err = bpf_prog_compute_tag(insns, prog_len, &tag1);
629 if (err)
630 return err;
631 err = bpf_prog_get_tag(prog_fd, &tag2);
632 if (err)
633 return err;
634 if (tag1 != tag2) {
635 fprintf(stderr, "prog tag mismatch %llx %llx\n", tag1, tag2);
636 return -1;
637 }
638
639 err = mkdir(BCC_PROG_TAG_DIR, 0777);
640 if (err && errno != EEXIST) {
641 fprintf(stderr, "cannot create " BCC_PROG_TAG_DIR "\n");
642 return -1;
643 }
644
645 char buf[128];
646 ::snprintf(buf, sizeof(buf), BCC_PROG_TAG_DIR "/bpf_prog_%llx", tag1);
647 err = mkdir(buf, 0777);
648 if (err && errno != EEXIST) {
649 fprintf(stderr, "cannot create %s\n", buf);
650 return -1;
651 }
652
653 ::snprintf(buf, sizeof(buf), BCC_PROG_TAG_DIR "/bpf_prog_%llx/%s.c",
654 tag1, name.data());
655 FileDesc fd(open(buf, O_CREAT | O_WRONLY | O_TRUNC, 0644));
656 if (fd < 0) {
657 fprintf(stderr, "cannot create %s\n", buf);
658 return -1;
659 }
660
661 const char *src = function_source(name);
662 write(fd, src, strlen(src));
663
664 ::snprintf(buf, sizeof(buf), BCC_PROG_TAG_DIR "/bpf_prog_%llx/%s.rewritten.c",
665 tag1, name.data());
666 fd = open(buf, O_CREAT | O_WRONLY | O_TRUNC, 0644);
667 if (fd < 0) {
668 fprintf(stderr, "cannot create %s\n", buf);
669 return -1;
670 }
671
672 src = function_source_rewritten(name);
673 write(fd, src, strlen(src));
674
675 if (!src_dbg_fmap_[name].empty()) {
676 ::snprintf(buf, sizeof(buf), BCC_PROG_TAG_DIR "/bpf_prog_%llx/%s.dis.txt",
677 tag1, name.data());
678 fd = open(buf, O_CREAT | O_WRONLY | O_TRUNC, 0644);
679 if (fd < 0) {
680 fprintf(stderr, "cannot create %s\n", buf);
681 return -1;
682 }
683
684 const char *src = src_dbg_fmap_[name].c_str();
685 write(fd, src, strlen(src));
686 }
687
688 return 0;
689 }
690
function_size(size_t id) const691 size_t BPFModule::function_size(size_t id) const {
692 auto fn = prog_func_info_->get_func(id);
693 if (fn)
694 return fn->size_;
695 return 0;
696 }
697
function_size(const string & name) const698 size_t BPFModule::function_size(const string &name) const {
699 auto fn = prog_func_info_->get_func(name);
700 if (fn)
701 return fn->size_;
702 return 0;
703 }
704
license() const705 char * BPFModule::license() const {
706 auto section = sections_.find("license");
707 if (section == sections_.end())
708 return nullptr;
709
710 return (char *)get<0>(section->second);
711 }
712
kern_version() const713 unsigned BPFModule::kern_version() const {
714 auto section = sections_.find("version");
715 if (section == sections_.end())
716 return 0;
717
718 return *(unsigned *)get<0>(section->second);
719 }
720
num_tables() const721 size_t BPFModule::num_tables() const { return tables_.size(); }
722
perf_event_fields(const char * event) const723 size_t BPFModule::perf_event_fields(const char *event) const {
724 auto it = perf_events_.find(event);
725 if (it == perf_events_.end())
726 return 0;
727 return it->second.size();
728 }
729
perf_event_field(const char * event,size_t i) const730 const char * BPFModule::perf_event_field(const char *event, size_t i) const {
731 auto it = perf_events_.find(event);
732 if (it == perf_events_.end() || i >= it->second.size())
733 return nullptr;
734 return it->second[i].c_str();
735 }
736
table_id(const string & name) const737 size_t BPFModule::table_id(const string &name) const {
738 auto it = table_names_.find(name);
739 if (it == table_names_.end()) return ~0ull;
740 return it->second;
741 }
742
table_fd(const string & name) const743 int BPFModule::table_fd(const string &name) const {
744 return table_fd(table_id(name));
745 }
746
table_fd(size_t id) const747 int BPFModule::table_fd(size_t id) const {
748 if (id >= tables_.size())
749 return -1;
750 return tables_[id]->fd;
751 }
752
table_type(const string & name) const753 int BPFModule::table_type(const string &name) const {
754 return table_type(table_id(name));
755 }
756
table_type(size_t id) const757 int BPFModule::table_type(size_t id) const {
758 if (id >= tables_.size())
759 return -1;
760 return tables_[id]->type;
761 }
762
table_max_entries(const string & name) const763 size_t BPFModule::table_max_entries(const string &name) const {
764 return table_max_entries(table_id(name));
765 }
766
table_max_entries(size_t id) const767 size_t BPFModule::table_max_entries(size_t id) const {
768 if (id >= tables_.size())
769 return 0;
770 return tables_[id]->max_entries;
771 }
772
table_flags(const string & name) const773 int BPFModule::table_flags(const string &name) const {
774 return table_flags(table_id(name));
775 }
776
table_flags(size_t id) const777 int BPFModule::table_flags(size_t id) const {
778 if (id >= tables_.size())
779 return -1;
780 return tables_[id]->flags;
781 }
782
table_name(size_t id) const783 const char * BPFModule::table_name(size_t id) const {
784 if (id >= tables_.size())
785 return nullptr;
786 return tables_[id]->name.c_str();
787 }
788
table_key_desc(size_t id) const789 const char * BPFModule::table_key_desc(size_t id) const {
790 if (used_b_loader_) return nullptr;
791 if (id >= tables_.size())
792 return nullptr;
793 return tables_[id]->key_desc.c_str();
794 }
795
table_key_desc(const string & name) const796 const char * BPFModule::table_key_desc(const string &name) const {
797 return table_key_desc(table_id(name));
798 }
799
table_leaf_desc(size_t id) const800 const char * BPFModule::table_leaf_desc(size_t id) const {
801 if (used_b_loader_) return nullptr;
802 if (id >= tables_.size())
803 return nullptr;
804 return tables_[id]->leaf_desc.c_str();
805 }
806
table_leaf_desc(const string & name) const807 const char * BPFModule::table_leaf_desc(const string &name) const {
808 return table_leaf_desc(table_id(name));
809 }
table_key_size(size_t id) const810 size_t BPFModule::table_key_size(size_t id) const {
811 if (id >= tables_.size())
812 return 0;
813 return tables_[id]->key_size;
814 }
table_key_size(const string & name) const815 size_t BPFModule::table_key_size(const string &name) const {
816 return table_key_size(table_id(name));
817 }
818
table_leaf_size(size_t id) const819 size_t BPFModule::table_leaf_size(size_t id) const {
820 if (id >= tables_.size())
821 return 0;
822 return tables_[id]->leaf_size;
823 }
table_leaf_size(const string & name) const824 size_t BPFModule::table_leaf_size(const string &name) const {
825 return table_leaf_size(table_id(name));
826 }
827
828 struct TableIterator {
TableIteratorebpf::TableIterator829 TableIterator(size_t key_size, size_t leaf_size)
830 : key(new uint8_t[key_size]), leaf(new uint8_t[leaf_size]) {
831 }
832 unique_ptr<uint8_t[]> key;
833 unique_ptr<uint8_t[]> leaf;
834 uint8_t keyb[512];
835 };
836
table_key_printf(size_t id,char * buf,size_t buflen,const void * key)837 int BPFModule::table_key_printf(size_t id, char *buf, size_t buflen, const void *key) {
838 if (id >= tables_.size())
839 return -1;
840 const TableDesc &desc = *tables_[id];
841 StatusTuple rc = desc.key_snprintf(buf, buflen, key);
842 if (rc.code() < 0) {
843 fprintf(stderr, "%s\n", rc.msg().c_str());
844 return -1;
845 }
846 return 0;
847 }
848
table_leaf_printf(size_t id,char * buf,size_t buflen,const void * leaf)849 int BPFModule::table_leaf_printf(size_t id, char *buf, size_t buflen, const void *leaf) {
850 if (id >= tables_.size())
851 return -1;
852 const TableDesc &desc = *tables_[id];
853 StatusTuple rc = desc.leaf_snprintf(buf, buflen, leaf);
854 if (rc.code() < 0) {
855 fprintf(stderr, "%s\n", rc.msg().c_str());
856 return -1;
857 }
858 return 0;
859 }
860
table_key_scanf(size_t id,const char * key_str,void * key)861 int BPFModule::table_key_scanf(size_t id, const char *key_str, void *key) {
862 if (id >= tables_.size())
863 return -1;
864 const TableDesc &desc = *tables_[id];
865 StatusTuple rc = desc.key_sscanf(key_str, key);
866 if (rc.code() < 0) {
867 fprintf(stderr, "%s\n", rc.msg().c_str());
868 return -1;
869 }
870 return 0;
871 }
872
table_leaf_scanf(size_t id,const char * leaf_str,void * leaf)873 int BPFModule::table_leaf_scanf(size_t id, const char *leaf_str, void *leaf) {
874 if (id >= tables_.size())
875 return -1;
876 const TableDesc &desc = *tables_[id];
877 StatusTuple rc = desc.leaf_sscanf(leaf_str, leaf);
878 if (rc.code() < 0) {
879 fprintf(stderr, "%s\n", rc.msg().c_str());
880 return -1;
881 }
882 return 0;
883 }
884
885 // load a C file
load_c(const string & filename,const char * cflags[],int ncflags)886 int BPFModule::load_c(const string &filename, const char *cflags[], int ncflags) {
887 if (!sections_.empty()) {
888 fprintf(stderr, "Program already initialized\n");
889 return -1;
890 }
891 if (filename.empty()) {
892 fprintf(stderr, "Invalid filename\n");
893 return -1;
894 }
895 if (int rc = load_cfile(filename, false, cflags, ncflags))
896 return rc;
897 if (rw_engine_enabled_) {
898 if (int rc = annotate())
899 return rc;
900 } else {
901 annotate_light();
902 }
903 if (int rc = finalize())
904 return rc;
905 return 0;
906 }
907
908 // load a C text string
load_string(const string & text,const char * cflags[],int ncflags)909 int BPFModule::load_string(const string &text, const char *cflags[], int ncflags) {
910 if (!sections_.empty()) {
911 fprintf(stderr, "Program already initialized\n");
912 return -1;
913 }
914 if (int rc = load_cfile(text, true, cflags, ncflags))
915 return rc;
916 if (rw_engine_enabled_) {
917 if (int rc = annotate())
918 return rc;
919 } else {
920 annotate_light();
921 }
922
923 if (int rc = finalize())
924 return rc;
925 return 0;
926 }
927
bcc_func_load(int prog_type,const char * name,const struct bpf_insn * insns,int prog_len,const char * license,unsigned kern_version,int log_level,char * log_buf,unsigned log_buf_size,const char * dev_name,unsigned flags)928 int BPFModule::bcc_func_load(int prog_type, const char *name,
929 const struct bpf_insn *insns, int prog_len,
930 const char *license, unsigned kern_version,
931 int log_level, char *log_buf, unsigned log_buf_size,
932 const char *dev_name, unsigned flags) {
933 struct bpf_load_program_attr attr = {};
934 unsigned func_info_cnt, line_info_cnt, finfo_rec_size, linfo_rec_size;
935 void *func_info = NULL, *line_info = NULL;
936 int ret;
937
938 attr.prog_type = (enum bpf_prog_type)prog_type;
939 attr.name = name;
940 attr.insns = insns;
941 attr.license = license;
942 if (attr.prog_type != BPF_PROG_TYPE_TRACING &&
943 attr.prog_type != BPF_PROG_TYPE_EXT) {
944 attr.kern_version = kern_version;
945 }
946 attr.prog_flags = flags;
947 attr.log_level = log_level;
948 if (dev_name)
949 attr.prog_ifindex = if_nametoindex(dev_name);
950
951 if (btf_) {
952 int btf_fd = btf_->get_fd();
953 char secname[256];
954
955 ::snprintf(secname, sizeof(secname), "%s%s", BPF_FN_PREFIX, name);
956 ret = btf_->get_btf_info(secname, &func_info, &func_info_cnt,
957 &finfo_rec_size, &line_info,
958 &line_info_cnt, &linfo_rec_size);
959 if (!ret) {
960 attr.prog_btf_fd = btf_fd;
961 attr.func_info = func_info;
962 attr.func_info_cnt = func_info_cnt;
963 attr.func_info_rec_size = finfo_rec_size;
964 attr.line_info = line_info;
965 attr.line_info_cnt = line_info_cnt;
966 attr.line_info_rec_size = linfo_rec_size;
967 }
968 }
969
970 ret = bcc_prog_load_xattr(&attr, prog_len, log_buf, log_buf_size, allow_rlimit_);
971 if (btf_) {
972 free(func_info);
973 free(line_info);
974 }
975
976 return ret;
977 }
978
bcc_func_attach(int prog_fd,int attachable_fd,int attach_type,unsigned int flags)979 int BPFModule::bcc_func_attach(int prog_fd, int attachable_fd,
980 int attach_type, unsigned int flags) {
981 return bpf_prog_attach(prog_fd, attachable_fd,
982 (enum bpf_attach_type)attach_type, flags);
983 }
984
bcc_func_detach(int prog_fd,int attachable_fd,int attach_type)985 int BPFModule::bcc_func_detach(int prog_fd, int attachable_fd,
986 int attach_type) {
987 return bpf_prog_detach2(prog_fd, attachable_fd,
988 (enum bpf_attach_type)attach_type);
989 }
990
991 } // namespace ebpf
992