1 /*
2 * Copyright (C) 2017 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #ifndef LOG_TAG
18 #define LOG_TAG "bpfloader"
19 #endif
20
21 #include <arpa/inet.h>
22 #include <elf.h>
23 #include <error.h>
24 #include <fcntl.h>
25 #include <inttypes.h>
26 #include <linux/bpf.h>
27 #include <linux/unistd.h>
28 #include <net/if.h>
29 #include <stdint.h>
30 #include <stdio.h>
31 #include <stdlib.h>
32 #include <string.h>
33 #include <unistd.h>
34
35 #include <sys/mman.h>
36 #include <sys/socket.h>
37 #include <sys/stat.h>
38 #include <sys/types.h>
39
40 #include <android-base/stringprintf.h>
41 #include <android-base/unique_fd.h>
42 #include <cutils/log.h>
43
44 #include <netdutils/MemBlock.h>
45 #include <netdutils/Misc.h>
46 #include <netdutils/Slice.h>
47 #include "bpf/BpfUtils.h"
48 #include "bpf/bpf_shared.h"
49
50 using android::base::unique_fd;
51 using android::netdutils::MemBlock;
52 using android::netdutils::Slice;
53
54 #define BPF_PROG_PATH "/system/etc/bpf"
55 #define BPF_PROG_SRC BPF_PROG_PATH "/bpf_kern.o"
56 #define MAP_LD_CMD_HEAD 0x18
57
58 #define FAIL(...) \
59 do { \
60 ((void)ALOG(LOG_ERROR, LOG_TAG, __VA_ARGS__)); \
61 exit(-1); \
62 } while (0)
63
64 // The BPF instruction bytes that we need to replace. x is a placeholder (e.g., COOKIE_TAG_MAP).
65 #define MAP_SEARCH_PATTERN(x) \
66 { \
67 0x18, 0x01, 0x00, 0x00, \
68 (x)[0], (x)[1], (x)[2], (x)[3], \
69 0x00, 0x00, 0x00, 0x00, \
70 (x)[4], (x)[5], (x)[6], (x)[7] \
71 }
72
73 // The bytes we'll replace them with. x is the actual fd number for the map at runtime.
74 // The second byte is changed from 0x01 to 0x11 since 0x11 is the special command used
75 // for bpf map fd loading. The original 0x01 is only a normal load command.
76 #define MAP_REPLACE_PATTERN(x) \
77 { \
78 0x18, 0x11, 0x00, 0x00, \
79 (x)[0], (x)[1], (x)[2], (x)[3], \
80 0x00, 0x00, 0x00, 0x00, \
81 (x)[4], (x)[5], (x)[6], (x)[7] \
82 }
83
84 #define DECLARE_MAP(_mapFd, _mapPath) \
85 unique_fd _mapFd(android::bpf::mapRetrieve((_mapPath), 0)); \
86 if (_mapFd < 0) { \
87 FAIL("Failed to get map from %s", (_mapPath)); \
88 }
89
90 #define MAP_CMD_SIZE 16
91 #define LOG_BUF_SIZE 65536
92
93 namespace android {
94 namespace bpf {
95
96 struct ReplacePattern {
97 std::array<uint8_t, MAP_CMD_SIZE> search;
98 std::array<uint8_t, MAP_CMD_SIZE> replace;
99
ReplacePatternandroid::bpf::ReplacePattern100 ReplacePattern(uint64_t dummyFd, int realFd) {
101 // Ensure that the fd numbers are big-endian.
102 uint8_t beDummyFd[sizeof(uint64_t)];
103 uint8_t beRealFd[sizeof(uint64_t)];
104 for (size_t i = 0; i < sizeof(uint64_t); i++) {
105 beDummyFd[i] = (dummyFd >> (i * 8)) & 0xFF;
106 beRealFd[i] = (realFd >> (i * 8)) & 0xFF;
107 }
108 search = MAP_SEARCH_PATTERN(beDummyFd);
109 replace = MAP_REPLACE_PATTERN(beRealFd);
110 }
111 };
112
113 MemBlock cgroupIngressProg;
114 MemBlock cgroupEgressProg;
115 MemBlock xtIngressProg;
116 MemBlock xtEgressProg;
117
getProgFromMem(Slice buffer,Elf64_Shdr * section)118 MemBlock getProgFromMem(Slice buffer, Elf64_Shdr* section) {
119 uint64_t progSize = (uint64_t)section->sh_size;
120 Slice progSection = take(drop(buffer, section->sh_offset), progSize);
121 if (progSection.size() < progSize) FAIL("programSection out of bound\n");
122
123 MemBlock progCopy(progSection);
124 if (progCopy.get().size() != progSize) {
125 FAIL("program cannot be extracted");
126 }
127 return progCopy;
128 }
129
parseProgramsFromFile(const char * path)130 void parseProgramsFromFile(const char* path) {
131 int fd = open(path, O_RDONLY);
132 if (fd == -1) {
133 FAIL("Failed to open %s program: %s", path, strerror(errno));
134 }
135
136 struct stat stat;
137 if (fstat(fd, &stat)) FAIL("Fail to get file size");
138
139 off_t fileLen = stat.st_size;
140 char* baseAddr = (char*)mmap(NULL, fileLen, PROT_READ, MAP_PRIVATE, fd, 0);
141 if (baseAddr == MAP_FAILED) FAIL("Failed to map the program into memory");
142
143 if ((uint32_t)fileLen < sizeof(Elf64_Ehdr)) FAIL("file size too small for Elf64_Ehdr");
144
145 Slice buffer(baseAddr, fileLen);
146
147 Slice elfHeader = take(buffer, sizeof(Elf64_Ehdr));
148
149 if (elfHeader.size() < sizeof(Elf64_Ehdr)) FAIL("bpf buffer does not have complete elf header");
150
151 Elf64_Ehdr* elf = (Elf64_Ehdr*)elfHeader.base();
152 // Find section names string table. This is the section whose index is e_shstrndx.
153 if (elf->e_shstrndx == SHN_UNDEF) {
154 FAIL("cannot locate namesSection\n");
155 }
156 size_t totalSectionSize = (elf->e_shnum) * sizeof(Elf64_Shdr);
157 Slice sections = take(drop(buffer, elf->e_shoff), totalSectionSize);
158 if (sections.size() < totalSectionSize) {
159 FAIL("sections corrupted");
160 }
161
162 Slice namesSection = take(drop(sections, elf->e_shstrndx * sizeof(Elf64_Shdr)),
163 sizeof(Elf64_Shdr));
164 if (namesSection.size() != sizeof(Elf64_Shdr)) {
165 FAIL("namesSection corrupted");
166 }
167 size_t strTabOffset = ((Elf64_Shdr*) namesSection.base())->sh_offset;
168 size_t strTabSize = ((Elf64_Shdr*) namesSection.base())->sh_size;
169
170 Slice strTab = take(drop(buffer, strTabOffset), strTabSize);
171 if (strTab.size() < strTabSize) {
172 FAIL("string table out of bound\n");
173 }
174
175 for (int i = 0; i < elf->e_shnum; i++) {
176 Slice section = take(drop(sections, i * sizeof(Elf64_Shdr)), sizeof(Elf64_Shdr));
177 if (section.size() < sizeof(Elf64_Shdr)) {
178 FAIL("section %d is out of bound, section size: %zu, header size: %zu, total size: %zu",
179 i, section.size(), sizeof(Elf64_Shdr), sections.size());
180 }
181 Elf64_Shdr* sectionPtr = (Elf64_Shdr*)section.base();
182 Slice nameSlice = drop(strTab, sectionPtr->sh_name);
183 if (nameSlice.size() == 0) {
184 FAIL("nameSlice out of bound, i: %d, strTabSize: %zu, sh_name: %u", i, strTabSize,
185 sectionPtr->sh_name);
186 }
187 if (!strcmp((char *)nameSlice.base(), BPF_CGROUP_INGRESS_PROG_NAME)) {
188 cgroupIngressProg = getProgFromMem(buffer, sectionPtr);
189 } else if (!strcmp((char *)nameSlice.base(), BPF_CGROUP_EGRESS_PROG_NAME)) {
190 cgroupEgressProg = getProgFromMem(buffer, sectionPtr);
191 } else if (!strcmp((char *)nameSlice.base(), XT_BPF_INGRESS_PROG_NAME)) {
192 xtIngressProg = getProgFromMem(buffer, sectionPtr);
193 } else if (!strcmp((char *)nameSlice.base(), XT_BPF_EGRESS_PROG_NAME)) {
194 xtEgressProg = getProgFromMem(buffer, sectionPtr);
195 }
196 }
197 }
198
loadProg(Slice prog,bpf_prog_type type,const std::vector<ReplacePattern> & mapPatterns)199 int loadProg(Slice prog, bpf_prog_type type, const std::vector<ReplacePattern>& mapPatterns) {
200 if (prog.size() == 0) {
201 FAIL("Couldn't find or parse program type %d", type);
202 }
203 Slice remaining = prog;
204 while (remaining.size() >= MAP_CMD_SIZE) {
205 // Scan the program, examining all possible places that might be the start of a map load
206 // operation (i.e., all bytes of value MAP_LD_CMD_HEAD).
207 // In each of these places, check whether it is the start of one of the patterns we want to
208 // replace, and if so, replace it.
209 Slice mapHead = findFirstMatching(remaining, MAP_LD_CMD_HEAD);
210 if (mapHead.size() < MAP_CMD_SIZE) break;
211 bool replaced = false;
212 for (const auto& pattern : mapPatterns) {
213 if (!memcmp(mapHead.base(), pattern.search.data(), MAP_CMD_SIZE)) {
214 memcpy(mapHead.base(), pattern.replace.data(), MAP_CMD_SIZE);
215 replaced = true;
216 break;
217 }
218 }
219 remaining = drop(mapHead, replaced ? MAP_CMD_SIZE : sizeof(uint8_t));
220 }
221 char bpf_log_buf[LOG_BUF_SIZE];
222 Slice bpfLog = Slice(bpf_log_buf, sizeof(bpf_log_buf));
223 return bpfProgLoad(type, prog, "Apache 2.0", 0, bpfLog);
224 }
225
loadAndAttachProgram(bpf_attach_type type,const char * path,const char * name,std::vector<ReplacePattern> mapPatterns)226 int loadAndAttachProgram(bpf_attach_type type, const char* path, const char* name,
227 std::vector<ReplacePattern> mapPatterns) {
228
229 unique_fd fd;
230 if (type == BPF_CGROUP_INET_INGRESS) {
231 fd.reset(loadProg(cgroupIngressProg, BPF_PROG_TYPE_CGROUP_SKB, mapPatterns));
232 } else if (type == BPF_CGROUP_INET_EGRESS) {
233 fd.reset(loadProg(cgroupEgressProg, BPF_PROG_TYPE_CGROUP_SKB, mapPatterns));
234 } else if (!strcmp(name, XT_BPF_INGRESS_PROG_NAME)) {
235 fd.reset(loadProg(xtIngressProg, BPF_PROG_TYPE_SOCKET_FILTER, mapPatterns));
236 } else if (!strcmp(name, XT_BPF_EGRESS_PROG_NAME)) {
237 fd.reset(loadProg(xtEgressProg, BPF_PROG_TYPE_SOCKET_FILTER, mapPatterns));
238 } else {
239 FAIL("Unrecognized program type: %s", name);
240 }
241
242 if (fd < 0) {
243 FAIL("load %s failed: %s", name, strerror(errno));
244 }
245 int ret = 0;
246 if (type == BPF_CGROUP_INET_EGRESS || type == BPF_CGROUP_INET_INGRESS) {
247 unique_fd cg_fd(open(CGROUP_ROOT_PATH, O_DIRECTORY | O_RDONLY | O_CLOEXEC));
248 if (cg_fd < 0) {
249 FAIL("Failed to open the cgroup directory");
250 }
251 ret = attachProgram(type, fd, cg_fd);
252 if (ret) {
253 FAIL("%s attach failed: %s", name, strerror(errno));
254 }
255 }
256
257 ret = mapPin(fd, path);
258 if (ret) {
259 FAIL("Pin %s as file %s failed: %s", name, path, strerror(errno));
260 }
261 return 0;
262 }
263
264 } // namespace bpf
265 } // namespace android
266
267 using android::bpf::APP_UID_STATS_MAP_PATH;
268 using android::bpf::BPF_EGRESS_PROG_PATH;
269 using android::bpf::BPF_INGRESS_PROG_PATH;
270 using android::bpf::COOKIE_TAG_MAP_PATH;
271 using android::bpf::DOZABLE_UID_MAP_PATH;
272 using android::bpf::IFACE_STATS_MAP_PATH;
273 using android::bpf::POWERSAVE_UID_MAP_PATH;
274 using android::bpf::STANDBY_UID_MAP_PATH;
275 using android::bpf::TAG_STATS_MAP_PATH;
276 using android::bpf::UID_COUNTERSET_MAP_PATH;
277 using android::bpf::UID_STATS_MAP_PATH;
278 using android::bpf::XT_BPF_EGRESS_PROG_PATH;
279 using android::bpf::XT_BPF_INGRESS_PROG_PATH;
280
281 using android::bpf::ReplacePattern;
282 using android::bpf::loadAndAttachProgram;
283
usage(void)284 static void usage(void) {
285 ALOGE( "Usage: ./bpfloader [-i] [-e]\n"
286 " -i load ingress bpf program\n"
287 " -e load egress bpf program\n"
288 " -p load prerouting xt_bpf program\n"
289 " -m load mangle xt_bpf program\n");
290 }
291
main(int argc,char ** argv)292 int main(int argc, char** argv) {
293 int ret = 0;
294 DECLARE_MAP(cookieTagMap, COOKIE_TAG_MAP_PATH);
295 DECLARE_MAP(uidCounterSetMap, UID_COUNTERSET_MAP_PATH);
296 DECLARE_MAP(appUidStatsMap, APP_UID_STATS_MAP_PATH);
297 DECLARE_MAP(uidStatsMap, UID_STATS_MAP_PATH);
298 DECLARE_MAP(tagStatsMap, TAG_STATS_MAP_PATH);
299 DECLARE_MAP(ifaceStatsMap, IFACE_STATS_MAP_PATH);
300 DECLARE_MAP(dozableUidMap, DOZABLE_UID_MAP_PATH);
301 DECLARE_MAP(standbyUidMap, STANDBY_UID_MAP_PATH);
302 DECLARE_MAP(powerSaveUidMap, POWERSAVE_UID_MAP_PATH);
303
304 const std::vector<ReplacePattern> mapPatterns = {
305 ReplacePattern(COOKIE_TAG_MAP, cookieTagMap.get()),
306 ReplacePattern(UID_COUNTERSET_MAP, uidCounterSetMap.get()),
307 ReplacePattern(APP_UID_STATS_MAP, appUidStatsMap.get()),
308 ReplacePattern(UID_STATS_MAP, uidStatsMap.get()),
309 ReplacePattern(TAG_STATS_MAP, tagStatsMap.get()),
310 ReplacePattern(IFACE_STATS_MAP, ifaceStatsMap.get()),
311 ReplacePattern(DOZABLE_UID_MAP, dozableUidMap.get()),
312 ReplacePattern(STANDBY_UID_MAP, standbyUidMap.get()),
313 ReplacePattern(POWERSAVE_UID_MAP, powerSaveUidMap.get()),
314 };
315
316 int opt;
317 bool doIngress = false, doEgress = false, doPrerouting = false, doMangle = false;
318 while ((opt = getopt(argc, argv, "iepm")) != -1) {
319 switch (opt) {
320 case 'i':
321 doIngress = true;
322 break;
323 case 'e':
324 doEgress = true;
325 break;
326 case 'p':
327 doPrerouting = true;
328 break;
329 case 'm':
330 doMangle = true;
331 break;
332 default:
333 usage();
334 FAIL("unknown argument %c", opt);
335 }
336 }
337 android::bpf::parseProgramsFromFile(BPF_PROG_SRC);
338
339 if (doIngress) {
340 ret = loadAndAttachProgram(BPF_CGROUP_INET_INGRESS, BPF_INGRESS_PROG_PATH,
341 BPF_CGROUP_INGRESS_PROG_NAME, mapPatterns);
342 if (ret) {
343 FAIL("Failed to set up ingress program");
344 }
345 }
346 if (doEgress) {
347 ret = loadAndAttachProgram(BPF_CGROUP_INET_EGRESS, BPF_EGRESS_PROG_PATH,
348 BPF_CGROUP_EGRESS_PROG_NAME, mapPatterns);
349 if (ret) {
350 FAIL("Failed to set up ingress program");
351 }
352 }
353 if (doPrerouting) {
354 ret = loadAndAttachProgram(MAX_BPF_ATTACH_TYPE, XT_BPF_INGRESS_PROG_PATH,
355 XT_BPF_INGRESS_PROG_NAME, mapPatterns);
356 if (ret) {
357 FAIL("Failed to set up xt_bpf program");
358 }
359 }
360 if (doMangle) {
361 ret = loadAndAttachProgram(MAX_BPF_ATTACH_TYPE, XT_BPF_EGRESS_PROG_PATH,
362 XT_BPF_EGRESS_PROG_NAME, mapPatterns);
363 if (ret) {
364 FAIL("Failed to set up xt_bpf program");
365 }
366 }
367 return ret;
368 }
369