1#!/usr/bin/env python 2# Copyright (c) PLUMgrid, Inc. 3# Licensed under the Apache License, Version 2.0 (the "License") 4 5# This program implements a topology likes below: 6# pem: physical endpoint manager, implemented as a bpf program 7# 8# vm1 <--------+ +----> bridge1 <----+ 9# V V V 10# pem router 11# ^ ^ ^ 12# vm2 <--------+ +----> bridge2 <----+ 13# 14# The vm1, vm2 and router are implemented as namespaces. 15# The bridge is implemented with limited functionality in bpf program. 16# 17# vm1 and vm2 are in different subnet. For vm1 to communicate to vm2, 18# the packet will have to travel from vm1 to pem, bridge1, router, bridge2, pem, and 19# then come to vm2. 20# 21# When this test is run with verbose mode (ctest -R <test_name> -V), 22# the following printout is observed on my local box: 23# 24# ...... 25# 8: ARPING 100.1.1.254 from 100.1.1.1 eth0 26# 8: Unicast reply from 100.1.1.254 [76:62:B5:5C:8C:6F] 0.533ms 27# 8: Sent 1 probes (1 broadcast(s)) 28# 8: Received 1 response(s) 29# 8: ARPING 200.1.1.254 from 200.1.1.1 eth0 30# 8: Unicast reply from 200.1.1.254 [F2:F0:B4:ED:7B:1B] 0.524ms 31# 8: Sent 1 probes (1 broadcast(s)) 32# 8: Received 1 response(s) 33# 8: PING 200.1.1.1 (200.1.1.1) 56(84) bytes of data. 34# 8: 64 bytes from 200.1.1.1: icmp_req=1 ttl=63 time=0.074 ms 35# 8: 64 bytes from 200.1.1.1: icmp_req=2 ttl=63 time=0.061 ms 36# 8: 37# 8: --- 200.1.1.1 ping statistics --- 38# 8: 2 packets transmitted, 2 received, 0% packet loss, time 999ms 39# 8: rtt min/avg/max/mdev = 0.061/0.067/0.074/0.010 ms 40# 8: [ ID] Interval Transfer Bandwidth 41# 8: [ 5] 0.0- 1.0 sec 4.00 GBytes 34.3 Gbits/sec 42# 8: Starting netserver with host 'IN(6)ADDR_ANY' port '12865' and family AF_UNSPEC 43# 8: MIGRATED TCP STREAM TEST from 0.0.0.0 (0.0.0.0) port 0 AF_INET to 200.1.1.1 (200.1.1.1) port 0 AF_INET : demo 44# 8: Recv Send Send 45# 8: Socket Socket Message Elapsed 46# 8: Size Size Size Time Throughput 47# 8: bytes bytes bytes secs. 10^6bits/sec 48# 8: 49# 8: 87380 16384 65160 1.00 41991.68 50# 8: MIGRATED TCP REQUEST/RESPONSE TEST from 0.0.0.0 (0.0.0.0) port 0 AF_INET to 200.1.1.1 (200.1.1.1) port 0 AF_INET : demo : first burst 0 51# 8: Local /Remote 52# 8: Socket Size Request Resp. Elapsed Trans. 53# 8: Send Recv Size Size Time Rate 54# 8: bytes Bytes bytes bytes secs. per sec 55# 8: 56# 8: 16384 87380 1 1 1.00 48645.53 57# 8: 16384 87380 58# 8: . 59# 8: ---------------------------------------------------------------------- 60# 8: Ran 1 test in 11.296s 61# 8: 62# 8: OK 63 64from ctypes import c_uint 65from netaddr import IPAddress, EUI 66from bcc import BPF 67from pyroute2 import IPRoute, NetNS, IPDB, NSPopen 68from utils import NSPopenWithCheck 69import sys 70from time import sleep 71from unittest import main, TestCase 72from simulation import Simulation 73 74arg1 = sys.argv.pop(1) 75ipr = IPRoute() 76ipdb = IPDB(nl=ipr) 77sim = Simulation(ipdb) 78 79class TestBPFSocket(TestCase): 80 def set_default_const(self): 81 self.ns1 = "ns1" 82 self.ns2 = "ns2" 83 self.ns_router = "ns_router" 84 self.vm1_ip = "100.1.1.1" 85 self.vm2_ip = "200.1.1.1" 86 self.vm1_rtr_ip = "100.1.1.254" 87 self.vm2_rtr_ip = "200.1.1.254" 88 self.vm1_rtr_mask = "100.1.1.0/24" 89 self.vm2_rtr_mask = "200.1.1.0/24" 90 91 def get_table(self, b): 92 self.jump = b.get_table("jump") 93 94 self.pem_dest = b.get_table("pem_dest") 95 self.pem_port = b.get_table("pem_port") 96 self.pem_ifindex = b.get_table("pem_ifindex") 97 self.pem_stats = b.get_table("pem_stats") 98 99 self.br1_dest = b.get_table("br1_dest") 100 self.br1_mac = b.get_table("br1_mac") 101 self.br1_rtr = b.get_table("br1_rtr") 102 103 self.br2_dest = b.get_table("br2_dest") 104 self.br2_mac = b.get_table("br2_mac") 105 self.br2_rtr = b.get_table("br2_rtr") 106 107 def connect_ports(self, prog_id_pem, prog_id_br, curr_pem_pid, curr_br_pid, 108 br_dest_map, br_mac_map, ifindex, vm_mac, vm_ip): 109 self.pem_dest[c_uint(curr_pem_pid)] = self.pem_dest.Leaf(prog_id_br, curr_br_pid) 110 br_dest_map[c_uint(curr_br_pid)] = br_dest_map.Leaf(prog_id_pem, curr_pem_pid) 111 self.pem_port[c_uint(curr_pem_pid)] = c_uint(ifindex) 112 self.pem_ifindex[c_uint(ifindex)] = c_uint(curr_pem_pid) 113 mac_addr = br_mac_map.Key(int(EUI(vm_mac))) 114 br_mac_map[mac_addr] = c_uint(curr_br_pid) 115 116 def config_maps(self): 117 # program id 118 prog_id_pem = 1 119 prog_id_br1 = 2 120 prog_id_br2 = 3 121 122 # initial port id and table pointers 123 curr_pem_pid = 0 124 curr_br1_pid = 0 125 curr_br2_pid = 0 126 127 # configure jump table 128 self.jump[c_uint(prog_id_pem)] = c_uint(self.pem_fn.fd) 129 self.jump[c_uint(prog_id_br1)] = c_uint(self.br1_fn.fd) 130 self.jump[c_uint(prog_id_br2)] = c_uint(self.br2_fn.fd) 131 132 # connect pem and br1 133 curr_pem_pid = curr_pem_pid + 1 134 curr_br1_pid = curr_br1_pid + 1 135 self.connect_ports(prog_id_pem, prog_id_br1, curr_pem_pid, curr_br1_pid, 136 self.br1_dest, self.br1_mac, 137 self.ns1_eth_out.index, self.vm1_mac, self.vm1_ip) 138 139 # connect pem and br2 140 curr_pem_pid = curr_pem_pid + 1 141 curr_br2_pid = curr_br2_pid + 1 142 self.connect_ports(prog_id_pem, prog_id_br2, curr_pem_pid, curr_br2_pid, 143 self.br2_dest, self.br2_mac, 144 self.ns2_eth_out.index, self.vm2_mac, self.vm2_ip) 145 146 # connect <br1, rtr> and <br2, rtr> 147 self.br1_rtr[c_uint(0)] = c_uint(self.nsrtr_eth0_out.index) 148 self.br2_rtr[c_uint(0)] = c_uint(self.nsrtr_eth1_out.index) 149 150 def test_brb(self): 151 try: 152 b = BPF(src_file=arg1, debug=0) 153 self.pem_fn = b.load_func("pem", BPF.SCHED_CLS) 154 self.br1_fn = b.load_func("br1", BPF.SCHED_CLS) 155 self.br2_fn = b.load_func("br2", BPF.SCHED_CLS) 156 self.get_table(b) 157 158 # set up the topology 159 self.set_default_const() 160 (ns1_ipdb, self.ns1_eth_out, _) = sim._create_ns(self.ns1, ipaddr=self.vm1_ip+'/24', 161 fn=self.pem_fn, action='drop', 162 disable_ipv6=True) 163 (ns2_ipdb, self.ns2_eth_out, _) = sim._create_ns(self.ns2, ipaddr=self.vm2_ip+'/24', 164 fn=self.pem_fn, action='drop', 165 disable_ipv6=True) 166 ns1_ipdb.routes.add({'dst': self.vm2_rtr_mask, 'gateway': self.vm1_rtr_ip}).commit() 167 ns2_ipdb.routes.add({'dst': self.vm1_rtr_mask, 'gateway': self.vm2_rtr_ip}).commit() 168 self.vm1_mac = ns1_ipdb.interfaces['eth0'].address 169 self.vm2_mac = ns2_ipdb.interfaces['eth0'].address 170 171 (_, self.nsrtr_eth0_out, _) = sim._create_ns(self.ns_router, ipaddr=self.vm1_rtr_ip+'/24', 172 fn=self.br1_fn, action='drop', 173 disable_ipv6=True) 174 (rt_ipdb, self.nsrtr_eth1_out, _) = sim._ns_add_ifc(self.ns_router, "eth1", "ns_router2", 175 ipaddr=self.vm2_rtr_ip+'/24', 176 fn=self.br2_fn, action='drop', 177 disable_ipv6=True) 178 nsp = NSPopen(rt_ipdb.nl.netns, ["sysctl", "-w", "net.ipv4.ip_forward=1"]) 179 nsp.wait(); nsp.release() 180 181 # configure maps 182 self.config_maps() 183 184 # our bridge is not smart enough, so send arping for router learning to prevent router 185 # from sending out arp request 186 nsp = NSPopen(ns1_ipdb.nl.netns, 187 ["arping", "-w", "1", "-c", "1", "-I", "eth0", self.vm1_rtr_ip]) 188 nsp.wait(); nsp.release() 189 nsp = NSPopen(ns2_ipdb.nl.netns, 190 ["arping", "-w", "1", "-c", "1", "-I", "eth0", self.vm2_rtr_ip]) 191 nsp.wait(); nsp.release() 192 193 # ping 194 nsp = NSPopen(ns1_ipdb.nl.netns, ["ping", self.vm2_ip, "-c", "2"]) 195 nsp.wait(); nsp.release() 196 # pem_stats only counts pem->bridge traffic, each VM has 4: arping/arp request/2 icmp request 197 # total 8 packets should be counted 198 self.assertEqual(self.pem_stats[c_uint(0)].value, 8) 199 200 nsp_server = NSPopenWithCheck(ns2_ipdb.nl.netns, ["iperf", "-s", "-xSC"]) 201 sleep(1) 202 nsp = NSPopen(ns1_ipdb.nl.netns, ["iperf", "-c", self.vm2_ip, "-t", "1", "-xSC"]) 203 nsp.wait(); nsp.release() 204 nsp_server.kill(); nsp_server.wait(); nsp_server.release() 205 206 nsp_server = NSPopenWithCheck(ns2_ipdb.nl.netns, ["netserver", "-D"]) 207 sleep(1) 208 nsp = NSPopenWithCheck(ns1_ipdb.nl.netns, ["netperf", "-l", "1", "-H", self.vm2_ip, "--", "-m", "65160"]) 209 nsp.wait(); nsp.release() 210 nsp = NSPopen(ns1_ipdb.nl.netns, ["netperf", "-l", "1", "-H", self.vm2_ip, "-t", "TCP_RR"]) 211 nsp.wait(); nsp.release() 212 nsp_server.kill(); nsp_server.wait(); nsp_server.release() 213 214 finally: 215 sim.release() 216 ipdb.release() 217 218 219if __name__ == "__main__": 220 main() 221