1 /* 2 * Copyright 2021 The gRPC Authors 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package io.grpc.xds; 18 19 import static com.google.common.base.Preconditions.checkNotNull; 20 import static io.grpc.xds.InternalXdsAttributes.ATTR_DRAIN_GRACE_NANOS; 21 import static io.grpc.xds.InternalXdsAttributes.ATTR_FILTER_CHAIN_SELECTOR_MANAGER; 22 import static io.grpc.xds.XdsServerWrapper.ATTR_SERVER_ROUTING_CONFIG; 23 import static io.grpc.xds.internal.security.SecurityProtocolNegotiators.ATTR_SERVER_SSL_CONTEXT_PROVIDER_SUPPLIER; 24 25 import com.google.common.annotations.VisibleForTesting; 26 import com.google.common.base.MoreObjects; 27 import com.google.common.base.Strings; 28 import com.google.common.collect.Iterables; 29 import com.google.protobuf.UInt32Value; 30 import io.grpc.Attributes; 31 import io.grpc.internal.ObjectPool; 32 import io.grpc.netty.GrpcHttp2ConnectionHandler; 33 import io.grpc.netty.InternalGracefulServerCloseCommand; 34 import io.grpc.netty.InternalProtocolNegotiationEvent; 35 import io.grpc.netty.InternalProtocolNegotiator; 36 import io.grpc.netty.InternalProtocolNegotiator.ProtocolNegotiator; 37 import io.grpc.netty.ProtocolNegotiationEvent; 38 import io.grpc.xds.EnvoyServerProtoData.CidrRange; 39 import io.grpc.xds.EnvoyServerProtoData.ConnectionSourceType; 40 import io.grpc.xds.EnvoyServerProtoData.FilterChain; 41 import io.grpc.xds.EnvoyServerProtoData.FilterChainMatch; 42 import io.grpc.xds.XdsServerWrapper.ServerRoutingConfig; 43 import io.grpc.xds.internal.Matchers.CidrMatcher; 44 import io.grpc.xds.internal.security.SslContextProviderSupplier; 45 import io.netty.channel.Channel; 46 import io.netty.channel.ChannelFuture; 47 import io.netty.channel.ChannelFutureListener; 48 import io.netty.channel.ChannelHandler; 49 import io.netty.channel.ChannelHandlerContext; 50 import io.netty.channel.ChannelInboundHandlerAdapter; 51 import io.netty.util.AsciiString; 52 import java.net.Inet6Address; 53 import java.net.InetAddress; 54 import java.net.InetSocketAddress; 55 import java.util.ArrayList; 56 import java.util.Collection; 57 import java.util.Collections; 58 import java.util.List; 59 import java.util.Map; 60 import java.util.concurrent.Executor; 61 import java.util.concurrent.TimeUnit; 62 import java.util.concurrent.atomic.AtomicReference; 63 import java.util.logging.Level; 64 import java.util.logging.Logger; 65 import javax.annotation.Nullable; 66 67 68 /** 69 * Handles L4 filter chain match for the connection based on the xds configuration. 70 * */ 71 @SuppressWarnings("FutureReturnValueIgnored") // Netty doesn't follow this pattern 72 final class FilterChainMatchingProtocolNegotiators { 73 private static final Logger log = Logger.getLogger( 74 FilterChainMatchingProtocolNegotiators.class.getName()); 75 76 private static final AsciiString SCHEME = AsciiString.of("http"); 77 FilterChainMatchingProtocolNegotiators()78 private FilterChainMatchingProtocolNegotiators() { 79 } 80 81 @VisibleForTesting 82 static final class FilterChainMatchingHandler extends ChannelInboundHandlerAdapter { 83 84 private final GrpcHttp2ConnectionHandler grpcHandler; 85 private final FilterChainSelectorManager filterChainSelectorManager; 86 private final ProtocolNegotiator delegate; 87 FilterChainMatchingHandler( GrpcHttp2ConnectionHandler grpcHandler, FilterChainSelectorManager filterChainSelectorManager, ProtocolNegotiator delegate)88 FilterChainMatchingHandler( 89 GrpcHttp2ConnectionHandler grpcHandler, 90 FilterChainSelectorManager filterChainSelectorManager, 91 ProtocolNegotiator delegate) { 92 this.grpcHandler = checkNotNull(grpcHandler, "grpcHandler"); 93 this.filterChainSelectorManager = 94 checkNotNull(filterChainSelectorManager, "filterChainSelectorManager"); 95 this.delegate = checkNotNull(delegate, "delegate"); 96 } 97 98 @Override userEventTriggered(ChannelHandlerContext ctx, Object evt)99 public void userEventTriggered(ChannelHandlerContext ctx, Object evt) throws Exception { 100 if (!(evt instanceof ProtocolNegotiationEvent)) { 101 super.userEventTriggered(ctx, evt); 102 return; 103 } 104 long drainGraceTime = 0; 105 TimeUnit drainGraceTimeUnit = null; 106 Long drainGraceNanosObj = grpcHandler.getEagAttributes().get(ATTR_DRAIN_GRACE_NANOS); 107 if (drainGraceNanosObj != null) { 108 drainGraceTime = drainGraceNanosObj; 109 drainGraceTimeUnit = TimeUnit.NANOSECONDS; 110 } 111 FilterChainSelectorManager.Closer closer = new FilterChainSelectorManager.Closer( 112 new GracefullyShutdownChannelRunnable(ctx.channel(), drainGraceTime, drainGraceTimeUnit)); 113 FilterChainSelector selector = filterChainSelectorManager.register(closer); 114 ctx.channel().closeFuture().addListener( 115 new FilterChainSelectorManagerDeregister(filterChainSelectorManager, closer)); 116 checkNotNull(selector, "selector"); 117 SelectedConfig config = selector.select( 118 (InetSocketAddress) ctx.channel().localAddress(), 119 (InetSocketAddress) ctx.channel().remoteAddress()); 120 if (config == null) { 121 log.log(Level.WARNING, "Connection from {0} to {1} has no matching filter chain. Closing", 122 new Object[] {ctx.channel().remoteAddress(), ctx.channel().localAddress()}); 123 ctx.close().addListener(ChannelFutureListener.FIRE_EXCEPTION_ON_FAILURE); 124 return; 125 } 126 ProtocolNegotiationEvent pne = (ProtocolNegotiationEvent) evt; 127 // TODO(zivy): merge into one key and take care of this outer class visibility. 128 Attributes attr = InternalProtocolNegotiationEvent.getAttributes(pne).toBuilder() 129 .set(ATTR_SERVER_SSL_CONTEXT_PROVIDER_SUPPLIER, config.sslContextProviderSupplier) 130 .set(ATTR_SERVER_ROUTING_CONFIG, config.routingConfig) 131 .build(); 132 pne = InternalProtocolNegotiationEvent.withAttributes(pne, attr); 133 ctx.pipeline().replace(this, null, delegate.newHandler(grpcHandler)); 134 ctx.fireUserEventTriggered(pne); 135 } 136 137 static final class FilterChainSelector { 138 public static final FilterChainSelector NO_FILTER_CHAIN = new FilterChainSelector( 139 Collections.<FilterChain, AtomicReference<ServerRoutingConfig>>emptyMap(), 140 null, new AtomicReference<ServerRoutingConfig>()); 141 private final Map<FilterChain, AtomicReference<ServerRoutingConfig>> routingConfigs; 142 @Nullable 143 private final SslContextProviderSupplier defaultSslContextProviderSupplier; 144 private final AtomicReference<ServerRoutingConfig> defaultRoutingConfig; 145 FilterChainSelector(Map<FilterChain, AtomicReference<ServerRoutingConfig>> routingConfigs, @Nullable SslContextProviderSupplier defaultSslContextProviderSupplier, AtomicReference<ServerRoutingConfig> defaultRoutingConfig)146 FilterChainSelector(Map<FilterChain, AtomicReference<ServerRoutingConfig>> routingConfigs, 147 @Nullable SslContextProviderSupplier defaultSslContextProviderSupplier, 148 AtomicReference<ServerRoutingConfig> defaultRoutingConfig) { 149 this.routingConfigs = checkNotNull(routingConfigs, "routingConfigs"); 150 this.defaultSslContextProviderSupplier = defaultSslContextProviderSupplier; 151 this.defaultRoutingConfig = checkNotNull(defaultRoutingConfig, "defaultRoutingConfig"); 152 } 153 154 @VisibleForTesting getRoutingConfigs()155 Map<FilterChain, AtomicReference<ServerRoutingConfig>> getRoutingConfigs() { 156 return routingConfigs; 157 } 158 159 @VisibleForTesting getDefaultRoutingConfig()160 AtomicReference<ServerRoutingConfig> getDefaultRoutingConfig() { 161 return defaultRoutingConfig; 162 } 163 164 @VisibleForTesting getDefaultSslContextProviderSupplier()165 SslContextProviderSupplier getDefaultSslContextProviderSupplier() { 166 return defaultSslContextProviderSupplier; 167 } 168 169 /** 170 * Throws IllegalStateException when no exact one match, and we should close the connection. 171 */ select(InetSocketAddress localAddr, InetSocketAddress remoteAddr)172 SelectedConfig select(InetSocketAddress localAddr, InetSocketAddress remoteAddr) { 173 Collection<FilterChain> filterChains = routingConfigs.keySet(); 174 filterChains = filterOnDestinationPort(filterChains); 175 filterChains = filterOnIpAddress(filterChains, localAddr.getAddress(), true); 176 filterChains = filterOnServerNames(filterChains); 177 filterChains = filterOnTransportProtocol(filterChains); 178 filterChains = filterOnApplicationProtocols(filterChains); 179 filterChains = 180 filterOnSourceType(filterChains, remoteAddr.getAddress(), localAddr.getAddress()); 181 filterChains = filterOnIpAddress(filterChains, remoteAddr.getAddress(), false); 182 filterChains = filterOnSourcePort(filterChains, remoteAddr.getPort()); 183 184 if (filterChains.size() > 1) { 185 throw new IllegalStateException("Found more than one matching filter chains. This should " 186 + "not be possible as ClientXdsClient validated the chains for uniqueness."); 187 } 188 if (filterChains.size() == 1) { 189 FilterChain selected = Iterables.getOnlyElement(filterChains); 190 return new SelectedConfig( 191 routingConfigs.get(selected), selected.sslContextProviderSupplier()); 192 } 193 if (defaultRoutingConfig.get() != null) { 194 return new SelectedConfig(defaultRoutingConfig, defaultSslContextProviderSupplier); 195 } 196 return null; 197 } 198 199 // reject if filer-chain-match has non-empty application_protocols filterOnApplicationProtocols( Collection<FilterChain> filterChains)200 private static Collection<FilterChain> filterOnApplicationProtocols( 201 Collection<FilterChain> filterChains) { 202 ArrayList<FilterChain> filtered = new ArrayList<>(filterChains.size()); 203 for (FilterChain filterChain : filterChains) { 204 FilterChainMatch filterChainMatch = filterChain.filterChainMatch(); 205 206 if (filterChainMatch.applicationProtocols().isEmpty()) { 207 filtered.add(filterChain); 208 } 209 } 210 return filtered; 211 } 212 213 // reject if filer-chain-match has non-empty transport protocol other than "raw_buffer" filterOnTransportProtocol( Collection<FilterChain> filterChains)214 private static Collection<FilterChain> filterOnTransportProtocol( 215 Collection<FilterChain> filterChains) { 216 ArrayList<FilterChain> filtered = new ArrayList<>(filterChains.size()); 217 for (FilterChain filterChain : filterChains) { 218 FilterChainMatch filterChainMatch = filterChain.filterChainMatch(); 219 220 String transportProtocol = filterChainMatch.transportProtocol(); 221 if (Strings.isNullOrEmpty(transportProtocol) || "raw_buffer".equals(transportProtocol)) { 222 filtered.add(filterChain); 223 } 224 } 225 return filtered; 226 } 227 228 // reject if filer-chain-match has server_name(s) filterOnServerNames( Collection<FilterChain> filterChains)229 private static Collection<FilterChain> filterOnServerNames( 230 Collection<FilterChain> filterChains) { 231 ArrayList<FilterChain> filtered = new ArrayList<>(filterChains.size()); 232 for (FilterChain filterChain : filterChains) { 233 FilterChainMatch filterChainMatch = filterChain.filterChainMatch(); 234 235 if (filterChainMatch.serverNames().isEmpty()) { 236 filtered.add(filterChain); 237 } 238 } 239 return filtered; 240 } 241 242 // destination_port present => Always fail match filterOnDestinationPort( Collection<FilterChain> filterChains)243 private static Collection<FilterChain> filterOnDestinationPort( 244 Collection<FilterChain> filterChains) { 245 ArrayList<FilterChain> filtered = new ArrayList<>(filterChains.size()); 246 for (FilterChain filterChain : filterChains) { 247 FilterChainMatch filterChainMatch = filterChain.filterChainMatch(); 248 249 if (filterChainMatch.destinationPort() 250 == UInt32Value.getDefaultInstance().getValue()) { 251 filtered.add(filterChain); 252 } 253 } 254 return filtered; 255 } 256 filterOnSourcePort( Collection<FilterChain> filterChains, int sourcePort)257 private static Collection<FilterChain> filterOnSourcePort( 258 Collection<FilterChain> filterChains, int sourcePort) { 259 ArrayList<FilterChain> filteredOnMatch = new ArrayList<>(filterChains.size()); 260 ArrayList<FilterChain> filteredOnEmpty = new ArrayList<>(filterChains.size()); 261 for (FilterChain filterChain : filterChains) { 262 FilterChainMatch filterChainMatch = filterChain.filterChainMatch(); 263 264 List<Integer> sourcePortsToMatch = filterChainMatch.sourcePorts(); 265 if (sourcePortsToMatch.isEmpty()) { 266 filteredOnEmpty.add(filterChain); 267 } else if (sourcePortsToMatch.contains(sourcePort)) { 268 filteredOnMatch.add(filterChain); 269 } 270 } 271 // match against source port is more specific than match against empty list 272 return filteredOnMatch.isEmpty() ? filteredOnEmpty : filteredOnMatch; 273 } 274 filterOnSourceType( Collection<FilterChain> filterChains, InetAddress sourceAddress, InetAddress destAddress)275 private static Collection<FilterChain> filterOnSourceType( 276 Collection<FilterChain> filterChains, InetAddress sourceAddress, 277 InetAddress destAddress) { 278 ArrayList<FilterChain> filtered = new ArrayList<>(filterChains.size()); 279 for (FilterChain filterChain : filterChains) { 280 FilterChainMatch filterChainMatch = filterChain.filterChainMatch(); 281 ConnectionSourceType sourceType = 282 filterChainMatch.connectionSourceType(); 283 284 boolean matching = false; 285 if (sourceType == ConnectionSourceType.SAME_IP_OR_LOOPBACK) { 286 matching = 287 sourceAddress.isLoopbackAddress() 288 || sourceAddress.isAnyLocalAddress() 289 || sourceAddress.equals(destAddress); 290 } else if (sourceType == ConnectionSourceType.EXTERNAL) { 291 matching = !sourceAddress.isLoopbackAddress() && !sourceAddress.isAnyLocalAddress(); 292 } else { // ANY or null 293 matching = true; 294 } 295 if (matching) { 296 filtered.add(filterChain); 297 } 298 } 299 return filtered; 300 } 301 getMatchingPrefixLength( FilterChainMatch filterChainMatch, InetAddress address, boolean forDestination)302 private static int getMatchingPrefixLength( 303 FilterChainMatch filterChainMatch, InetAddress address, boolean forDestination) { 304 boolean isIPv6 = address instanceof Inet6Address; 305 List<CidrRange> cidrRanges = 306 forDestination 307 ? filterChainMatch.prefixRanges() 308 : filterChainMatch.sourcePrefixRanges(); 309 int matchingPrefixLength; 310 if (cidrRanges.isEmpty()) { // if there is no CidrRange assume 0-length match 311 matchingPrefixLength = 0; 312 } else { 313 matchingPrefixLength = -1; 314 for (CidrRange cidrRange : cidrRanges) { 315 InetAddress cidrAddr = cidrRange.addressPrefix(); 316 boolean cidrIsIpv6 = cidrAddr instanceof Inet6Address; 317 if (isIPv6 == cidrIsIpv6) { 318 int prefixLen = cidrRange.prefixLen(); 319 CidrMatcher matcher = CidrMatcher.create(cidrAddr, prefixLen); 320 if (matcher.matches(address) && prefixLen > matchingPrefixLength) { 321 matchingPrefixLength = prefixLen; 322 } 323 } 324 } 325 } 326 return matchingPrefixLength; 327 } 328 329 // use prefix_ranges (CIDR) and get the most specific matches filterOnIpAddress( Collection<FilterChain> filterChains, InetAddress address, boolean forDestination)330 private static Collection<FilterChain> filterOnIpAddress( 331 Collection<FilterChain> filterChains, InetAddress address, boolean forDestination) { 332 // curent list of top ones 333 ArrayList<FilterChain> topOnes = new ArrayList<>(filterChains.size()); 334 int topMatchingPrefixLen = -1; 335 for (FilterChain filterChain : filterChains) { 336 int currentMatchingPrefixLen = getMatchingPrefixLength( 337 filterChain.filterChainMatch(), address, forDestination); 338 339 if (currentMatchingPrefixLen >= 0) { 340 if (currentMatchingPrefixLen < topMatchingPrefixLen) { 341 continue; 342 } 343 if (currentMatchingPrefixLen > topMatchingPrefixLen) { 344 topMatchingPrefixLen = currentMatchingPrefixLen; 345 topOnes.clear(); 346 } 347 topOnes.add(filterChain); 348 } 349 } 350 return topOnes; 351 } 352 353 @Override toString()354 public String toString() { 355 return MoreObjects.toStringHelper(this) 356 .add("routingConfigs", routingConfigs) 357 .add("defaultSslContextProviderSupplier", defaultSslContextProviderSupplier) 358 .add("defaultRoutingConfig", defaultRoutingConfig) 359 .toString(); 360 } 361 } 362 } 363 364 static final class FilterChainMatchingNegotiatorServerFactory 365 implements InternalProtocolNegotiator.ServerFactory { 366 private final InternalProtocolNegotiator.ServerFactory delegate; 367 FilterChainMatchingNegotiatorServerFactory( InternalProtocolNegotiator.ServerFactory delegate)368 public FilterChainMatchingNegotiatorServerFactory( 369 InternalProtocolNegotiator.ServerFactory delegate) { 370 this.delegate = checkNotNull(delegate, "delegate"); 371 } 372 373 @Override newNegotiator( final ObjectPool<? extends Executor> offloadExecutorPool)374 public ProtocolNegotiator newNegotiator( 375 final ObjectPool<? extends Executor> offloadExecutorPool) { 376 377 class FilterChainMatchingNegotiator implements ProtocolNegotiator { 378 379 @Override 380 public AsciiString scheme() { 381 return SCHEME; 382 } 383 384 @Override 385 public ChannelHandler newHandler(GrpcHttp2ConnectionHandler grpcHandler) { 386 FilterChainSelectorManager filterChainSelectorManager = 387 grpcHandler.getEagAttributes().get(ATTR_FILTER_CHAIN_SELECTOR_MANAGER); 388 checkNotNull(filterChainSelectorManager, "filterChainSelectorManager"); 389 return new FilterChainMatchingHandler(grpcHandler, filterChainSelectorManager, 390 delegate.newNegotiator(offloadExecutorPool)); 391 } 392 393 @Override 394 public void close() { 395 } 396 } 397 398 return new FilterChainMatchingNegotiator(); 399 } 400 } 401 402 /** 403 * The FilterChain level configuration. 404 */ 405 private static final class SelectedConfig { 406 private final AtomicReference<ServerRoutingConfig> routingConfig; 407 @Nullable 408 private final SslContextProviderSupplier sslContextProviderSupplier; 409 SelectedConfig(AtomicReference<ServerRoutingConfig> routingConfig, @Nullable SslContextProviderSupplier sslContextProviderSupplier)410 private SelectedConfig(AtomicReference<ServerRoutingConfig> routingConfig, 411 @Nullable SslContextProviderSupplier sslContextProviderSupplier) { 412 this.routingConfig = checkNotNull(routingConfig, "routingConfig"); 413 this.sslContextProviderSupplier = sslContextProviderSupplier; 414 } 415 } 416 417 private static class FilterChainSelectorManagerDeregister implements ChannelFutureListener { 418 private final FilterChainSelectorManager filterChainSelectorManager; 419 private final FilterChainSelectorManager.Closer closer; 420 FilterChainSelectorManagerDeregister( FilterChainSelectorManager filterChainSelectorManager, FilterChainSelectorManager.Closer closer)421 public FilterChainSelectorManagerDeregister( 422 FilterChainSelectorManager filterChainSelectorManager, 423 FilterChainSelectorManager.Closer closer) { 424 this.filterChainSelectorManager = 425 checkNotNull(filterChainSelectorManager, "filterChainSelectorManager"); 426 this.closer = checkNotNull(closer, "closer"); 427 } 428 operationComplete(ChannelFuture future)429 @Override public void operationComplete(ChannelFuture future) throws Exception { 430 filterChainSelectorManager.deregister(closer); 431 } 432 } 433 434 private static class GracefullyShutdownChannelRunnable implements Runnable { 435 private final Channel channel; 436 private final long drainGraceTime; 437 @Nullable 438 private final TimeUnit drainGraceTimeUnit; 439 GracefullyShutdownChannelRunnable( Channel channel, long drainGraceTime, @Nullable TimeUnit drainGraceTimeUnit)440 public GracefullyShutdownChannelRunnable( 441 Channel channel, long drainGraceTime, @Nullable TimeUnit drainGraceTimeUnit) { 442 this.channel = checkNotNull(channel, "channel"); 443 this.drainGraceTime = drainGraceTime; 444 this.drainGraceTimeUnit = drainGraceTimeUnit; 445 } 446 run()447 @Override public void run() { 448 Object gracefulCloseCommand = InternalGracefulServerCloseCommand.create( 449 "xds_drain", drainGraceTime, drainGraceTimeUnit); 450 channel.writeAndFlush(gracefulCloseCommand) 451 .addListener(ChannelFutureListener.CLOSE_ON_FAILURE); 452 } 453 } 454 } 455