1 /* 2 * Copyright (C) 2022 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package com.android.server.connectivity; 18 19 import static android.net.INetd.IF_STATE_UP; 20 import static android.net.INetd.PERMISSION_NETWORK; 21 import static android.net.INetd.PERMISSION_SYSTEM; 22 import static android.system.OsConstants.ETH_P_IP; 23 import static android.system.OsConstants.ETH_P_IPV6; 24 25 import static com.android.net.module.util.NetworkStackConstants.IPV6_MIN_MTU; 26 27 import android.annotation.NonNull; 28 import android.annotation.Nullable; 29 import android.net.INetd; 30 import android.net.InetAddresses; 31 import android.net.InterfaceConfigurationParcel; 32 import android.net.IpPrefix; 33 import android.os.Build; 34 import android.os.ParcelFileDescriptor; 35 import android.os.RemoteException; 36 import android.os.ServiceSpecificException; 37 import android.system.ErrnoException; 38 import android.util.Log; 39 40 import androidx.annotation.RequiresApi; 41 42 import com.android.internal.annotations.VisibleForTesting; 43 import com.android.internal.util.IndentingPrintWriter; 44 import com.android.net.module.util.BpfDump; 45 import com.android.net.module.util.BpfMap; 46 import com.android.net.module.util.IBpfMap; 47 import com.android.net.module.util.InterfaceParams; 48 import com.android.net.module.util.TcUtils; 49 import com.android.net.module.util.bpf.ClatEgress4Key; 50 import com.android.net.module.util.bpf.ClatEgress4Value; 51 import com.android.net.module.util.bpf.ClatIngress6Key; 52 import com.android.net.module.util.bpf.ClatIngress6Value; 53 import com.android.net.module.util.bpf.CookieTagMapKey; 54 import com.android.net.module.util.bpf.CookieTagMapValue; 55 56 import java.io.FileDescriptor; 57 import java.io.IOException; 58 import java.net.Inet4Address; 59 import java.net.Inet6Address; 60 import java.net.InetAddress; 61 import java.nio.ByteBuffer; 62 import java.util.Objects; 63 64 /** 65 * This coordinator is responsible for providing clat relevant functionality. 66 * 67 * {@hide} 68 */ 69 @RequiresApi(Build.VERSION_CODES.TIRAMISU) 70 public class ClatCoordinator { 71 private static final String TAG = ClatCoordinator.class.getSimpleName(); 72 73 // Sync from system/core/libcutils/include/private/android_filesystem_config.h 74 @VisibleForTesting 75 static final int AID_CLAT = 1029; 76 77 // Sync from external/android-clat/clatd.c 78 // 40 bytes IPv6 header - 20 bytes IPv4 header + 8 bytes fragment header. 79 @VisibleForTesting 80 static final int MTU_DELTA = 28; 81 @VisibleForTesting 82 static final int CLAT_MAX_MTU = 1500 + MTU_DELTA; 83 84 // This must match the interface prefix in clatd.c. 85 private static final String CLAT_PREFIX = "v4-"; 86 87 // For historical reasons, start with 192.0.0.4, and after that, use all subsequent addresses 88 // in 192.0.0.0/29 (RFC 7335). 89 @VisibleForTesting 90 static final String INIT_V4ADDR_STRING = "192.0.0.4"; 91 @VisibleForTesting 92 static final int INIT_V4ADDR_PREFIX_LEN = 29; 93 private static final InetAddress GOOGLE_DNS_4 = InetAddress.parseNumericAddress("8.8.8.8"); 94 95 private static final int INVALID_IFINDEX = 0; 96 97 // For better code clarity when used for 'bool ingress' parameter. 98 @VisibleForTesting 99 static final boolean EGRESS = false; 100 @VisibleForTesting 101 static final boolean INGRESS = true; 102 103 // For better code clarity when used for 'bool ether' parameter. 104 static final boolean RAWIP = false; 105 static final boolean ETHER = true; 106 107 // The priority of clat hook - must be after tethering. 108 @VisibleForTesting 109 static final int PRIO_CLAT = 4; 110 111 private static final String COOKIE_TAG_MAP_PATH = 112 "/sys/fs/bpf/netd_shared/map_netd_cookie_tag_map"; 113 private static final String CLAT_EGRESS4_MAP_PATH = makeMapPath("egress4"); 114 private static final String CLAT_INGRESS6_MAP_PATH = makeMapPath("ingress6"); 115 makeMapPath(String which)116 private static String makeMapPath(String which) { 117 return "/sys/fs/bpf/net_shared/map_clatd_clat_" + which + "_map"; 118 } 119 120 private static final String CLAT_EGRESS4_RAWIP_PROG_PATH = 121 "/sys/fs/bpf/net_shared/prog_clatd_schedcls_egress4_clat_rawip"; 122 makeIngressProgPath(boolean ether)123 private static String makeIngressProgPath(boolean ether) { 124 return "/sys/fs/bpf/net_shared/prog_clatd_schedcls_ingress6_clat_" 125 + (ether ? "ether" : "rawip"); 126 } 127 128 @NonNull 129 private final INetd mNetd; 130 @NonNull 131 private final Dependencies mDeps; 132 @Nullable 133 private final IBpfMap<ClatIngress6Key, ClatIngress6Value> mIngressMap; 134 @Nullable 135 private final IBpfMap<ClatEgress4Key, ClatEgress4Value> mEgressMap; 136 @Nullable 137 private final IBpfMap<CookieTagMapKey, CookieTagMapValue> mCookieTagMap; 138 @Nullable 139 private ClatdTracker mClatdTracker = null; 140 141 /** 142 * Dependencies of ClatCoordinator which makes ConnectivityService injection 143 * in tests. 144 */ 145 @VisibleForTesting 146 public abstract static class Dependencies { 147 /** 148 * Get netd. 149 */ 150 @NonNull getNetd()151 public abstract INetd getNetd(); 152 153 /** 154 * @see ParcelFileDescriptor#adoptFd(int). 155 */ 156 @NonNull adoptFd(int fd)157 public ParcelFileDescriptor adoptFd(int fd) { 158 return ParcelFileDescriptor.adoptFd(fd); 159 } 160 161 /** 162 * Get interface index for a given interface. 163 */ getInterfaceIndex(String ifName)164 public int getInterfaceIndex(String ifName) { 165 final InterfaceParams params = InterfaceParams.getByName(ifName); 166 return params != null ? params.index : INVALID_IFINDEX; 167 } 168 169 /** 170 * Create tun interface for a given interface name. 171 */ createTunInterface(@onNull String tuniface)172 public int createTunInterface(@NonNull String tuniface) throws IOException { 173 return native_createTunInterface(tuniface); 174 } 175 176 /** 177 * Pick an IPv4 address for clat. 178 */ 179 @NonNull selectIpv4Address(@onNull String v4addr, int prefixlen)180 public String selectIpv4Address(@NonNull String v4addr, int prefixlen) 181 throws IOException { 182 return native_selectIpv4Address(v4addr, prefixlen); 183 } 184 185 /** 186 * Generate a checksum-neutral IID. 187 */ 188 @NonNull generateIpv6Address(@onNull String iface, @NonNull String v4, @NonNull String prefix64, int mark)189 public String generateIpv6Address(@NonNull String iface, @NonNull String v4, 190 @NonNull String prefix64, int mark) throws IOException { 191 return native_generateIpv6Address(iface, v4, prefix64, mark); 192 } 193 194 /** 195 * Detect MTU. 196 */ detectMtu(@onNull String platSubnet, int platSuffix, int mark)197 public int detectMtu(@NonNull String platSubnet, int platSuffix, int mark) 198 throws IOException { 199 return native_detectMtu(platSubnet, platSuffix, mark); 200 } 201 202 /** 203 * Open packet socket. 204 */ openPacketSocket()205 public int openPacketSocket() throws IOException { 206 return native_openPacketSocket(); 207 } 208 209 /** 210 * Open IPv6 raw socket and set SO_MARK. 211 */ openRawSocket6(int mark)212 public int openRawSocket6(int mark) throws IOException { 213 return native_openRawSocket6(mark); 214 } 215 216 /** 217 * Add anycast setsockopt. 218 */ addAnycastSetsockopt(@onNull FileDescriptor sock, String v6, int ifindex)219 public void addAnycastSetsockopt(@NonNull FileDescriptor sock, String v6, int ifindex) 220 throws IOException { 221 native_addAnycastSetsockopt(sock, v6, ifindex); 222 } 223 224 /** 225 * Configure packet socket. 226 */ configurePacketSocket(@onNull FileDescriptor sock, String v6, int ifindex)227 public void configurePacketSocket(@NonNull FileDescriptor sock, String v6, int ifindex) 228 throws IOException { 229 native_configurePacketSocket(sock, v6, ifindex); 230 } 231 232 /** 233 * Start clatd. 234 */ startClatd(@onNull FileDescriptor tunfd, @NonNull FileDescriptor readsock6, @NonNull FileDescriptor writesock6, @NonNull String iface, @NonNull String pfx96, @NonNull String v4, @NonNull String v6)235 public int startClatd(@NonNull FileDescriptor tunfd, @NonNull FileDescriptor readsock6, 236 @NonNull FileDescriptor writesock6, @NonNull String iface, @NonNull String pfx96, 237 @NonNull String v4, @NonNull String v6) throws IOException { 238 return native_startClatd(tunfd, readsock6, writesock6, iface, pfx96, v4, v6); 239 } 240 241 /** 242 * Stop clatd. 243 */ stopClatd(int pid)244 public void stopClatd(int pid) throws IOException { 245 native_stopClatd(pid); 246 } 247 248 /** 249 * Get socket cookie. 250 */ getSocketCookie(@onNull FileDescriptor sock)251 public long getSocketCookie(@NonNull FileDescriptor sock) throws IOException { 252 return native_getSocketCookie(sock); 253 } 254 255 /** Get ingress6 BPF map. */ 256 @Nullable getBpfIngress6Map()257 public IBpfMap<ClatIngress6Key, ClatIngress6Value> getBpfIngress6Map() { 258 try { 259 // written from clatd.c 260 return new BpfMap<>(CLAT_INGRESS6_MAP_PATH, 261 ClatIngress6Key.class, ClatIngress6Value.class); 262 } catch (ErrnoException e) { 263 Log.e(TAG, "Cannot create ingress6 map: " + e); 264 return null; 265 } 266 } 267 268 /** Get egress4 BPF map. */ 269 @Nullable getBpfEgress4Map()270 public IBpfMap<ClatEgress4Key, ClatEgress4Value> getBpfEgress4Map() { 271 try { 272 // written from clatd.c 273 return new BpfMap<>(CLAT_EGRESS4_MAP_PATH, 274 ClatEgress4Key.class, ClatEgress4Value.class); 275 } catch (ErrnoException e) { 276 Log.e(TAG, "Cannot create egress4 map: " + e); 277 return null; 278 } 279 } 280 281 /** Get cookie tag map */ 282 @Nullable getBpfCookieTagMap()283 public IBpfMap<CookieTagMapKey, CookieTagMapValue> getBpfCookieTagMap() { 284 try { 285 // also read and written from other locations 286 return new BpfMap<>(COOKIE_TAG_MAP_PATH, 287 CookieTagMapKey.class, CookieTagMapValue.class); 288 } catch (ErrnoException e) { 289 Log.wtf(TAG, "Cannot open cookie tag map: " + e); 290 return null; 291 } 292 } 293 294 /** Checks if the network interface uses an ethernet L2 header. */ isEthernet(String iface)295 public boolean isEthernet(String iface) throws IOException { 296 return TcUtils.isEthernet(iface); 297 } 298 299 /** Add a clsact qdisc. */ tcQdiscAddDevClsact(int ifIndex)300 public void tcQdiscAddDevClsact(int ifIndex) throws IOException { 301 TcUtils.tcQdiscAddDevClsact(ifIndex); 302 } 303 304 /** Attach a tc bpf filter. */ tcFilterAddDevBpf(int ifIndex, boolean ingress, short prio, short proto, String bpfProgPath)305 public void tcFilterAddDevBpf(int ifIndex, boolean ingress, short prio, short proto, 306 String bpfProgPath) throws IOException { 307 TcUtils.tcFilterAddDevBpf(ifIndex, ingress, prio, proto, bpfProgPath); 308 } 309 310 /** Delete a tc filter. */ tcFilterDelDev(int ifIndex, boolean ingress, short prio, short proto)311 public void tcFilterDelDev(int ifIndex, boolean ingress, short prio, short proto) 312 throws IOException { 313 TcUtils.tcFilterDelDev(ifIndex, ingress, prio, proto); 314 } 315 } 316 317 @VisibleForTesting 318 static class ClatdTracker { 319 @NonNull 320 public final String iface; 321 public final int ifIndex; 322 @NonNull 323 public final String v4iface; 324 public final int v4ifIndex; 325 @NonNull 326 public final Inet4Address v4; 327 @NonNull 328 public final Inet6Address v6; 329 @NonNull 330 public final Inet6Address pfx96; 331 public final int pid; 332 public final long cookie; 333 ClatdTracker(@onNull String iface, int ifIndex, @NonNull String v4iface, int v4ifIndex, @NonNull Inet4Address v4, @NonNull Inet6Address v6, @NonNull Inet6Address pfx96, int pid, long cookie)334 ClatdTracker(@NonNull String iface, int ifIndex, @NonNull String v4iface, 335 int v4ifIndex, @NonNull Inet4Address v4, @NonNull Inet6Address v6, 336 @NonNull Inet6Address pfx96, int pid, long cookie) { 337 this.iface = iface; 338 this.ifIndex = ifIndex; 339 this.v4iface = v4iface; 340 this.v4ifIndex = v4ifIndex; 341 this.v4 = v4; 342 this.v6 = v6; 343 this.pfx96 = pfx96; 344 this.pid = pid; 345 this.cookie = cookie; 346 } 347 348 @Override equals(Object o)349 public boolean equals(Object o) { 350 if (!(o instanceof ClatdTracker)) return false; 351 ClatdTracker that = (ClatdTracker) o; 352 return Objects.equals(this.iface, that.iface) 353 && this.ifIndex == that.ifIndex 354 && Objects.equals(this.v4iface, that.v4iface) 355 && this.v4ifIndex == that.v4ifIndex 356 && Objects.equals(this.v4, that.v4) 357 && Objects.equals(this.v6, that.v6) 358 && Objects.equals(this.pfx96, that.pfx96) 359 && this.pid == that.pid 360 && this.cookie == that.cookie; 361 } 362 363 @Override toString()364 public String toString() { 365 return "iface: " + iface 366 + " (" + ifIndex + ")" 367 + ", v4iface: " + v4iface 368 + " (" + v4ifIndex + ")" 369 + ", v4: " + v4 370 + ", v6: " + v6 371 + ", pfx96: " + pfx96 372 + ", pid: " + pid 373 + ", cookie: " + cookie; 374 } 375 }; 376 377 @VisibleForTesting getFwmark(int netId)378 static int getFwmark(int netId) { 379 // See union Fwmark in system/netd/include/Fwmark.h 380 return (netId & 0xffff) 381 | 0x1 << 16 // explicitlySelected: true 382 | 0x1 << 17 // protectedFromVpn: true 383 | ((PERMISSION_NETWORK | PERMISSION_SYSTEM) & 0x3) << 18; // 2 permission bits = 3 384 } 385 386 @VisibleForTesting adjustMtu(int mtu)387 static int adjustMtu(int mtu) { 388 // clamp to minimum ipv6 mtu - this probably cannot ever trigger 389 if (mtu < IPV6_MIN_MTU) mtu = IPV6_MIN_MTU; 390 // clamp to buffer size 391 if (mtu > CLAT_MAX_MTU) mtu = CLAT_MAX_MTU; 392 // decrease by ipv6(40) + ipv6 fragmentation header(8) vs ipv4(20) overhead of 28 bytes 393 mtu -= MTU_DELTA; 394 395 return mtu; 396 } 397 ClatCoordinator(@onNull Dependencies deps)398 public ClatCoordinator(@NonNull Dependencies deps) { 399 mDeps = deps; 400 mNetd = mDeps.getNetd(); 401 mIngressMap = mDeps.getBpfIngress6Map(); 402 mEgressMap = mDeps.getBpfEgress4Map(); 403 mCookieTagMap = mDeps.getBpfCookieTagMap(); 404 } 405 406 // Note that this may only be called on a brand new v4-* interface, 407 // because it uses bpfmap.insertEntry() which fails if entry exists, 408 // and because the value includes (initialized to 0) byte/packet 409 // counters, so a replace (instead of insert) would wipe those stats. maybeStartBpf(final ClatdTracker tracker)410 private void maybeStartBpf(final ClatdTracker tracker) { 411 if (mIngressMap == null || mEgressMap == null) return; 412 413 final boolean isEthernet; 414 try { 415 isEthernet = mDeps.isEthernet(tracker.iface); 416 } catch (IOException e) { 417 Log.e(TAG, "Fail to call isEthernet for interface " + tracker.iface); 418 return; 419 } 420 421 final ClatEgress4Key txKey = new ClatEgress4Key(tracker.v4ifIndex, tracker.v4); 422 final ClatEgress4Value txValue = new ClatEgress4Value(tracker.ifIndex, tracker.v6, 423 tracker.pfx96, (short) (isEthernet ? 1 /* ETHER */ : 0 /* RAWIP */)); 424 try { 425 mEgressMap.insertEntry(txKey, txValue); 426 } catch (ErrnoException | IllegalStateException e) { 427 Log.e(TAG, "Could not insert entry (" + txKey + ", " + txValue + ") on egress map: " 428 + e); 429 return; 430 } 431 432 final ClatIngress6Key rxKey = new ClatIngress6Key(tracker.ifIndex, tracker.pfx96, 433 tracker.v6); 434 final ClatIngress6Value rxValue = new ClatIngress6Value(tracker.v4ifIndex, 435 tracker.v4); 436 try { 437 mIngressMap.insertEntry(rxKey, rxValue); 438 } catch (ErrnoException | IllegalStateException e) { 439 Log.e(TAG, "Could not insert entry (" + rxKey + ", " + rxValue + ") ingress map: " 440 + e); 441 try { 442 mEgressMap.deleteEntry(txKey); 443 } catch (ErrnoException | IllegalStateException e2) { 444 Log.e(TAG, "Could not delete entry (" + txKey + ") from egress map: " + e2); 445 } 446 return; 447 } 448 449 // Usually the clsact will be added in netd RouteController::addInterfaceToPhysicalNetwork. 450 // But clat is started before the v4- interface is added to the network. The clat startup 451 // have to add clsact of v4- tun interface first for adding bpf filter in maybeStartBpf. 452 try { 453 // tc qdisc add dev .. clsact 454 mDeps.tcQdiscAddDevClsact(tracker.v4ifIndex); 455 } catch (IOException e) { 456 Log.e(TAG, "tc qdisc add dev (" + tracker.v4ifIndex + "[" + tracker.v4iface 457 + "]) failure: " + e); 458 try { 459 mEgressMap.deleteEntry(txKey); 460 } catch (ErrnoException | IllegalStateException e2) { 461 Log.e(TAG, "Could not delete entry (" + txKey + ") from egress map: " + e2); 462 } 463 try { 464 mIngressMap.deleteEntry(rxKey); 465 } catch (ErrnoException | IllegalStateException e3) { 466 Log.e(TAG, "Could not delete entry (" + rxKey + ") from ingress map: " + e3); 467 } 468 return; 469 } 470 471 // This program will be attached to the v4-* interface which is a TUN and thus always rawip. 472 try { 473 // tc filter add dev .. egress prio 4 protocol ip bpf object-pinned /sys/fs/bpf/... 474 // direct-action 475 mDeps.tcFilterAddDevBpf(tracker.v4ifIndex, EGRESS, (short) PRIO_CLAT, (short) ETH_P_IP, 476 CLAT_EGRESS4_RAWIP_PROG_PATH); 477 } catch (IOException e) { 478 Log.e(TAG, "tc filter add dev (" + tracker.v4ifIndex + "[" + tracker.v4iface 479 + "]) egress prio PRIO_CLAT protocol ip failure: " + e); 480 481 // The v4- interface clsact is not deleted for unwinding error because once it is 482 // created with interface addition, the lifetime is till interface deletion. Moreover, 483 // the clsact has no clat filter now. It should not break anything. 484 485 try { 486 mEgressMap.deleteEntry(txKey); 487 } catch (ErrnoException | IllegalStateException e2) { 488 Log.e(TAG, "Could not delete entry (" + txKey + ") from egress map: " + e2); 489 } 490 try { 491 mIngressMap.deleteEntry(rxKey); 492 } catch (ErrnoException | IllegalStateException e3) { 493 Log.e(TAG, "Could not delete entry (" + rxKey + ") from ingress map: " + e3); 494 } 495 return; 496 } 497 498 try { 499 // tc filter add dev .. ingress prio 4 protocol ipv6 bpf object-pinned /sys/fs/bpf/... 500 // direct-action 501 mDeps.tcFilterAddDevBpf(tracker.ifIndex, INGRESS, (short) PRIO_CLAT, 502 (short) ETH_P_IPV6, makeIngressProgPath(isEthernet)); 503 } catch (IOException e) { 504 Log.e(TAG, "tc filter add dev (" + tracker.ifIndex + "[" + tracker.iface 505 + "]) ingress prio PRIO_CLAT protocol ipv6 failure: " + e); 506 507 // The v4- interface clsact is not deleted. See the reason in the error unwinding code 508 // of the egress filter attaching of v4- tun interface. 509 510 try { 511 mDeps.tcFilterDelDev(tracker.v4ifIndex, EGRESS, (short) PRIO_CLAT, 512 (short) ETH_P_IP); 513 } catch (IOException e2) { 514 Log.e(TAG, "tc filter del dev (" + tracker.v4ifIndex + "[" + tracker.v4iface 515 + "]) egress prio PRIO_CLAT protocol ip failure: " + e2); 516 } 517 try { 518 mEgressMap.deleteEntry(txKey); 519 } catch (ErrnoException | IllegalStateException e3) { 520 Log.e(TAG, "Could not delete entry (" + txKey + ") from egress map: " + e3); 521 } 522 try { 523 mIngressMap.deleteEntry(rxKey); 524 } catch (ErrnoException | IllegalStateException e4) { 525 Log.e(TAG, "Could not delete entry (" + rxKey + ") from ingress map: " + e4); 526 } 527 return; 528 } 529 } 530 tagSocketAsClat(long cookie)531 private void tagSocketAsClat(long cookie) throws IOException { 532 if (mCookieTagMap == null) { 533 throw new IOException("Cookie tag map is not initialized"); 534 } 535 536 // Tag raw socket with uid AID_CLAT and set tag as zero because tag is unused in bpf 537 // program for counting data usage in netd.c. Tagging socket is used to avoid counting 538 // duplicated clat traffic in bpf stat. 539 final CookieTagMapKey key = new CookieTagMapKey(cookie); 540 final CookieTagMapValue value = new CookieTagMapValue(AID_CLAT, 0 /* tag, unused */); 541 try { 542 mCookieTagMap.insertEntry(key, value); 543 } catch (ErrnoException | IllegalStateException e) { 544 throw new IOException("Could not insert entry (" + key + ", " + value 545 + ") on cookie tag map: " + e); 546 } 547 Log.i(TAG, "tag socket cookie " + cookie); 548 } 549 untagSocket(long cookie)550 private void untagSocket(long cookie) throws IOException { 551 if (mCookieTagMap == null) { 552 throw new IOException("Cookie tag map is not initialized"); 553 } 554 555 // The reason that deleting entry from cookie tag map directly is that the tag socket 556 // destroy listener only monitors on group INET_TCP, INET_UDP, INET6_TCP, INET6_UDP. 557 // The other socket types, ex: raw, are not able to be removed automatically by the 558 // listener. See TrafficController::makeSkDestroyListener. 559 final CookieTagMapKey key = new CookieTagMapKey(cookie); 560 try { 561 mCookieTagMap.deleteEntry(key); 562 } catch (ErrnoException | IllegalStateException e) { 563 throw new IOException("Could not delete entry (" + key + ") on cookie tag map: " + e); 564 } 565 Log.i(TAG, "untag socket cookie " + cookie); 566 } 567 isStarted()568 private boolean isStarted() { 569 return mClatdTracker != null; 570 } 571 572 /** 573 * Start clatd for a given interface and NAT64 prefix. 574 */ clatStart(final String iface, final int netId, @NonNull final IpPrefix nat64Prefix)575 public String clatStart(final String iface, final int netId, 576 @NonNull final IpPrefix nat64Prefix) 577 throws IOException { 578 if (isStarted()) { 579 throw new IOException("Clatd is already running on " + mClatdTracker.iface 580 + " (pid " + mClatdTracker.pid + ")"); 581 } 582 if (nat64Prefix.getPrefixLength() != 96) { 583 throw new IOException("Prefix must be 96 bits long: " + nat64Prefix); 584 } 585 586 // Initialize all required file descriptors with null pointer. This makes the following 587 // error handling easier. Simply always call #maybeCleanUp for closing file descriptors, 588 // if any valid ones, in error handling. 589 ParcelFileDescriptor tunFd = null; 590 ParcelFileDescriptor readSock6 = null; 591 ParcelFileDescriptor writeSock6 = null; 592 593 long cookie = 0; 594 boolean isSocketTagged = false; 595 596 try { 597 // [1] Pick an IPv4 address from 192.0.0.4, 192.0.0.5, 192.0.0.6 .. 598 final String v4Str = mDeps.selectIpv4Address(INIT_V4ADDR_STRING, 599 INIT_V4ADDR_PREFIX_LEN); 600 final Inet4Address v4 = (Inet4Address) InetAddresses.parseNumericAddress(v4Str); 601 602 // [2] Generate a checksum-neutral IID. 603 final Integer fwmark = getFwmark(netId); 604 final String pfx96Str = nat64Prefix.getAddress().getHostAddress(); 605 final String v6Str = mDeps.generateIpv6Address(iface, v4Str, pfx96Str, fwmark); 606 final Inet6Address pfx96 = (Inet6Address) nat64Prefix.getAddress(); 607 final Inet6Address v6 = (Inet6Address) InetAddresses.parseNumericAddress(v6Str); 608 609 // [3] Open and configure local 464xlat read/write sockets. 610 // Opens a packet socket to receive IPv6 packets in clatd. 611 612 // Use a JNI call to get native file descriptor instead of Os.socket() because we would 613 // like to use ParcelFileDescriptor to manage file descriptor. But ctor 614 // ParcelFileDescriptor(FileDescriptor fd) is a @hide function. Need to use native file 615 // descriptor to initialize ParcelFileDescriptor object instead. 616 readSock6 = mDeps.adoptFd(mDeps.openPacketSocket()); 617 618 // Opens a raw socket with a given fwmark to send IPv6 packets in clatd. 619 // Use a JNI call to get native file descriptor instead of Os.socket(). See above 620 // reason why we use jniOpenPacketSocket6(). 621 writeSock6 = mDeps.adoptFd(mDeps.openRawSocket6(fwmark)); 622 623 final int ifIndex = mDeps.getInterfaceIndex(iface); 624 if (ifIndex == INVALID_IFINDEX) { 625 throw new IOException("Fail to get interface index for interface " + iface); 626 } 627 628 // Start translating packets to the new prefix. 629 mDeps.addAnycastSetsockopt(writeSock6.getFileDescriptor(), v6Str, ifIndex); 630 // Tag socket as AID_CLAT to avoid duplicated CLAT data usage accounting. 631 cookie = mDeps.getSocketCookie(writeSock6.getFileDescriptor()); 632 tagSocketAsClat(cookie); 633 isSocketTagged = true; 634 // Update our packet socket filter to reflect the new 464xlat IP address. 635 mDeps.configurePacketSocket(readSock6.getFileDescriptor(), v6Str, ifIndex); 636 637 // [4] Open, configure and bring up the tun interface. 638 // Create the v4-... tun interface. 639 final String tunIface = CLAT_PREFIX + iface; 640 tunFd = mDeps.adoptFd(mDeps.createTunInterface(tunIface)); 641 final int tunIfIndex = mDeps.getInterfaceIndex(tunIface); 642 if (tunIfIndex == INVALID_IFINDEX) { 643 throw new IOException("Fail to get interface index for interface " + tunIface); 644 } 645 // disable IPv6 on it - failing to do so is not a critical error 646 mNetd.interfaceSetEnableIPv6(tunIface, false /* enabled */); 647 // Detect ipv4 mtu. 648 final int detectedMtu = mDeps.detectMtu(pfx96Str, 649 ByteBuffer.wrap(GOOGLE_DNS_4.getAddress()).getInt(), fwmark); 650 final int mtu = adjustMtu(detectedMtu); 651 Log.i(TAG, "detected ipv4 mtu of " + detectedMtu + " adjusted to " + mtu); 652 // Config tun interface mtu, address and bring up. 653 mNetd.interfaceSetMtu(tunIface, mtu); 654 final InterfaceConfigurationParcel ifConfig = new InterfaceConfigurationParcel(); 655 ifConfig.ifName = tunIface; 656 ifConfig.ipv4Addr = v4Str; 657 ifConfig.prefixLength = 32; 658 ifConfig.hwAddr = ""; 659 ifConfig.flags = new String[] {IF_STATE_UP}; 660 mNetd.interfaceSetCfg(ifConfig); 661 662 // [5] Start clatd. 663 final int pid = mDeps.startClatd(tunFd.getFileDescriptor(), 664 readSock6.getFileDescriptor(), writeSock6.getFileDescriptor(), iface, pfx96Str, 665 v4Str, v6Str); 666 // The file descriptors have been duplicated (dup2) to clatd in native_startClatd(). 667 // Close these file descriptor stubs in finally block. 668 669 // [6] Initialize and store clatd tracker object. 670 mClatdTracker = new ClatdTracker(iface, ifIndex, tunIface, tunIfIndex, v4, v6, pfx96, 671 pid, cookie); 672 673 // [7] Start BPF 674 maybeStartBpf(mClatdTracker); 675 676 return v6Str; 677 } catch (IOException | RemoteException | ServiceSpecificException | ClassCastException 678 | IllegalArgumentException | NullPointerException e) { 679 if (isSocketTagged) { 680 try { 681 untagSocket(cookie); 682 } catch (IOException e2) { 683 Log.e(TAG, "untagSocket cookie " + cookie + " failed: " + e2); 684 } 685 } 686 throw new IOException("Failed to start clat ", e); 687 } finally { 688 if (tunFd != null) { 689 try { 690 tunFd.close(); 691 } catch (IOException e) { 692 Log.e(TAG, "Fail to close tun file descriptor " + e); 693 } 694 } 695 if (readSock6 != null) { 696 try { 697 readSock6.close(); 698 } catch (IOException e) { 699 Log.e(TAG, "Fail to close read socket " + e); 700 } 701 } 702 if (writeSock6 != null) { 703 try { 704 writeSock6.close(); 705 } catch (IOException e) { 706 Log.e(TAG, "Fail to close write socket " + e); 707 } 708 } 709 } 710 } 711 maybeStopBpf(final ClatdTracker tracker)712 private void maybeStopBpf(final ClatdTracker tracker) { 713 if (mIngressMap == null || mEgressMap == null) return; 714 715 try { 716 mDeps.tcFilterDelDev(tracker.ifIndex, INGRESS, (short) PRIO_CLAT, (short) ETH_P_IPV6); 717 } catch (IOException e) { 718 Log.e(TAG, "tc filter del dev (" + tracker.ifIndex + "[" + tracker.iface 719 + "]) ingress prio PRIO_CLAT protocol ipv6 failure: " + e); 720 } 721 722 try { 723 mDeps.tcFilterDelDev(tracker.v4ifIndex, EGRESS, (short) PRIO_CLAT, (short) ETH_P_IP); 724 } catch (IOException e) { 725 Log.e(TAG, "tc filter del dev (" + tracker.v4ifIndex + "[" + tracker.v4iface 726 + "]) egress prio PRIO_CLAT protocol ip failure: " + e); 727 } 728 729 // We cleanup the maps last, so scanning through them can be used to 730 // determine what still needs cleanup. 731 732 final ClatEgress4Key txKey = new ClatEgress4Key(tracker.v4ifIndex, tracker.v4); 733 try { 734 mEgressMap.deleteEntry(txKey); 735 } catch (ErrnoException | IllegalStateException e) { 736 Log.e(TAG, "Could not delete entry (" + txKey + "): " + e); 737 } 738 739 final ClatIngress6Key rxKey = new ClatIngress6Key(tracker.ifIndex, tracker.pfx96, 740 tracker.v6); 741 try { 742 mIngressMap.deleteEntry(rxKey); 743 } catch (ErrnoException | IllegalStateException e) { 744 Log.e(TAG, "Could not delete entry (" + rxKey + "): " + e); 745 } 746 } 747 748 /** 749 * Stop clatd 750 */ clatStop()751 public void clatStop() throws IOException { 752 if (!isStarted()) { 753 throw new IOException("Clatd has not started"); 754 } 755 Log.i(TAG, "Stopping clatd pid=" + mClatdTracker.pid + " on " + mClatdTracker.iface); 756 757 maybeStopBpf(mClatdTracker); 758 mDeps.stopClatd(mClatdTracker.pid); 759 untagSocket(mClatdTracker.cookie); 760 761 Log.i(TAG, "clatd on " + mClatdTracker.iface + " stopped"); 762 mClatdTracker = null; 763 } 764 dumpBpfIngress(@onNull IndentingPrintWriter pw)765 private void dumpBpfIngress(@NonNull IndentingPrintWriter pw) { 766 if (mIngressMap == null) { 767 pw.println("No BPF ingress6 map"); 768 return; 769 } 770 771 try { 772 if (mIngressMap.isEmpty()) { 773 pw.println("<empty>"); 774 } 775 pw.println("BPF ingress map: iif nat64Prefix v6Addr -> v4Addr oif (packets bytes)"); 776 pw.increaseIndent(); 777 mIngressMap.forEach((k, v) -> { 778 // TODO: print interface name 779 pw.println(String.format("%d %s/96 %s -> %s %d (%d %d)", k.iif, k.pfx96, k.local6, 780 v.local4, v.oif, v.packets, v.bytes)); 781 }); 782 pw.decreaseIndent(); 783 } catch (ErrnoException e) { 784 pw.println("Error dumping BPF ingress6 map: " + e); 785 } 786 } 787 dumpBpfEgress(@onNull IndentingPrintWriter pw)788 private void dumpBpfEgress(@NonNull IndentingPrintWriter pw) { 789 if (mEgressMap == null) { 790 pw.println("No BPF egress4 map"); 791 return; 792 } 793 794 try { 795 if (mEgressMap.isEmpty()) { 796 pw.println("<empty>"); 797 } 798 pw.println("BPF egress map: iif v4Addr -> v6Addr nat64Prefix oif (packets bytes)"); 799 pw.increaseIndent(); 800 mEgressMap.forEach((k, v) -> { 801 // TODO: print interface name 802 pw.println(String.format("%d %s -> %s %s/96 %d %s (%d %d)", k.iif, k.local4, 803 v.local6, v.pfx96, v.oif, v.oifIsEthernet != 0 ? "ether" : "rawip", 804 v.packets, v.bytes)); 805 }); 806 pw.decreaseIndent(); 807 } catch (ErrnoException e) { 808 pw.println("Error dumping BPF egress4 map: " + e); 809 } 810 } 811 812 /** 813 * Dump raw BPF map into base64 encoded strings {@literal "<base64 key>,<base64 value>"}. 814 * Allow to dump only one map in each call. For test only. 815 * 816 * @param pw print writer. 817 * @param isEgress4Map whether to dump the egress4 map (true) or the ingress6 map (false). 818 * 819 * Usage: 820 * $ dumpsys connectivity {clatEgress4RawBpfMap|clatIngress6RawBpfMap} 821 * 822 * Output: 823 * {@literal <base64 encoded key #1>,<base64 encoded value #1>} 824 * {@literal <base64 encoded key #2>,<base64 encoded value #2>} 825 * .. 826 */ dumpRawMap(@onNull IndentingPrintWriter pw, boolean isEgress4Map)827 public void dumpRawMap(@NonNull IndentingPrintWriter pw, boolean isEgress4Map) { 828 if (isEgress4Map) { 829 BpfDump.dumpRawMap(mEgressMap, pw); 830 } else { 831 BpfDump.dumpRawMap(mIngressMap, pw); 832 } 833 } 834 835 /** 836 * Dump the coordinator information. 837 * 838 * @param pw print writer. 839 */ dump(@onNull IndentingPrintWriter pw)840 public void dump(@NonNull IndentingPrintWriter pw) { 841 // TODO: move map dump to a global place to avoid duplicate dump while there are two or 842 // more IPv6 only networks. 843 if (isStarted()) { 844 pw.println("CLAT tracker: " + mClatdTracker); 845 pw.println("Forwarding rules:"); 846 pw.increaseIndent(); 847 dumpBpfIngress(pw); 848 dumpBpfEgress(pw); 849 pw.decreaseIndent(); 850 } else { 851 pw.println("<not started>"); 852 } 853 pw.println(); 854 } 855 856 /** 857 * Get clatd tracker. For test only. 858 */ 859 @VisibleForTesting 860 @Nullable getClatdTrackerForTesting()861 ClatdTracker getClatdTrackerForTesting() { 862 return mClatdTracker; 863 } 864 native_selectIpv4Address(String v4addr, int prefixlen)865 private static native String native_selectIpv4Address(String v4addr, int prefixlen) 866 throws IOException; native_generateIpv6Address(String iface, String v4, String prefix64, int mark)867 private static native String native_generateIpv6Address(String iface, String v4, 868 String prefix64, int mark) throws IOException; native_createTunInterface(String tuniface)869 private static native int native_createTunInterface(String tuniface) throws IOException; native_detectMtu(String platSubnet, int platSuffix, int mark)870 private static native int native_detectMtu(String platSubnet, int platSuffix, int mark) 871 throws IOException; native_openPacketSocket()872 private static native int native_openPacketSocket() throws IOException; native_openRawSocket6(int mark)873 private static native int native_openRawSocket6(int mark) throws IOException; native_addAnycastSetsockopt(FileDescriptor sock, String v6, int ifindex)874 private static native void native_addAnycastSetsockopt(FileDescriptor sock, String v6, 875 int ifindex) throws IOException; native_configurePacketSocket(FileDescriptor sock, String v6, int ifindex)876 private static native void native_configurePacketSocket(FileDescriptor sock, String v6, 877 int ifindex) throws IOException; native_startClatd(FileDescriptor tunfd, FileDescriptor readsock6, FileDescriptor writesock6, String iface, String pfx96, String v4, String v6)878 private static native int native_startClatd(FileDescriptor tunfd, FileDescriptor readsock6, 879 FileDescriptor writesock6, String iface, String pfx96, String v4, String v6) 880 throws IOException; native_stopClatd(int pid)881 private static native void native_stopClatd(int pid) throws IOException; native_getSocketCookie(FileDescriptor sock)882 private static native long native_getSocketCookie(FileDescriptor sock) throws IOException; 883 } 884