1 //
2 // Copyright 2017 gRPC authors.
3 //
4 // Licensed under the Apache License, Version 2.0 (the "License");
5 // you may not use this file except in compliance with the License.
6 // You may obtain a copy of the License at
7 //
8 // http://www.apache.org/licenses/LICENSE-2.0
9 //
10 // Unless required by applicable law or agreed to in writing, software
11 // distributed under the License is distributed on an "AS IS" BASIS,
12 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 // See the License for the specific language governing permissions and
14 // limitations under the License.
15 //
16
17 #include <gmock/gmock.h>
18 #include <grpc/grpc.h>
19 #include <grpc/grpc_security.h>
20 #include <grpcpp/channel.h>
21 #include <grpcpp/client_context.h>
22 #include <grpcpp/create_channel.h>
23 #include <grpcpp/server.h>
24 #include <grpcpp/server_builder.h>
25 #include <grpcpp/xds_server_builder.h>
26 #include <gtest/gtest.h>
27
28 #include <memory>
29
30 #include "absl/strings/str_cat.h"
31 #include "absl/time/time.h"
32 #include "absl/types/optional.h"
33 #include "envoy/config/listener/v3/listener.pb.h"
34 #include "envoy/config/route/v3/route.pb.h"
35 #include "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.pb.h"
36 #include "src/core/config/config_vars.h"
37 #include "src/core/util/env.h"
38 #include "src/core/util/time.h"
39 #include "src/proto/grpc/testing/echo.pb.h"
40 #include "test/core/test_util/port.h"
41 #include "test/core/test_util/resolve_localhost_ip46.h"
42 #include "test/core/test_util/test_config.h"
43 #include "test/cpp/end2end/xds/xds_end2end_test_lib.h"
44
45 namespace grpc {
46 namespace testing {
47 namespace {
48
49 using ::envoy::config::listener::v3::FilterChainMatch;
50
51 //
52 // Basic xDS-enabled server tests
53 //
54
55 class XdsEnabledServerTest : public XdsEnd2endTest {
56 protected:
SetUp()57 void SetUp() override {} // No-op -- individual tests do this themselves.
58
DoSetUp(const absl::optional<XdsBootstrapBuilder> & builder=absl::nullopt)59 void DoSetUp(
60 const absl::optional<XdsBootstrapBuilder>& builder = absl::nullopt) {
61 // We use insecure creds here as a convenience to be able to easily
62 // create new channels in some of the tests below. None of the
63 // tests here actually depend on the channel creds anyway.
64 InitClient(builder, /*lb_expected_authority=*/"",
65 // Using a low timeout to quickly end negative tests.
66 // Prefer using WaitOnServingStatusChange() or a similar
67 // loop on the client side to wait on status changes
68 // instead of increasing this timeout.
69 /*xds_resource_does_not_exist_timeout_ms=*/500,
70 /*balancer_authority_override=*/"", /*args=*/nullptr,
71 InsecureChannelCredentials());
72 CreateBackends(1, /*xds_enabled=*/true, InsecureServerCredentials());
73 EdsResourceArgs args({{"locality0", CreateEndpointsForBackends(0, 1)}});
74 balancer_->ads_service()->SetEdsResource(BuildEdsResource(args));
75 }
76 };
77
78 // We are only testing the server here.
79 // Run with bootstrap from env var, so that we use a global XdsClient
80 // instance. Otherwise, we would need to use a separate fake resolver
81 // result generator on the client and server sides.
82 INSTANTIATE_TEST_SUITE_P(XdsTest, XdsEnabledServerTest,
83 ::testing::Values(XdsTestType().set_bootstrap_source(
84 XdsTestType::kBootstrapFromEnvVar)),
85 &XdsTestType::Name);
86
TEST_P(XdsEnabledServerTest,Basic)87 TEST_P(XdsEnabledServerTest, Basic) {
88 DoSetUp();
89 StartBackend(0);
90 ASSERT_TRUE(backends_[0]->WaitOnServingStatusChange(grpc::StatusCode::OK));
91 WaitForBackend(DEBUG_LOCATION, 0);
92 }
93
TEST_P(XdsEnabledServerTest,ListenerDeletionIgnored)94 TEST_P(XdsEnabledServerTest, ListenerDeletionIgnored) {
95 DoSetUp(MakeBootstrapBuilder().SetIgnoreResourceDeletion());
96 StartBackend(0);
97 ASSERT_TRUE(backends_[0]->WaitOnServingStatusChange(grpc::StatusCode::OK));
98 WaitForBackend(DEBUG_LOCATION, 0);
99 // Check that we ACKed.
100 // TODO(roth): There may be multiple entries in the resource state response
101 // queue, because the client doesn't necessarily subscribe to all resources
102 // in a single message, and the server currently (I suspect incorrectly?)
103 // thinks that each subscription message is an ACK. So for now, we
104 // drain the entire LDS resource state response queue, ensuring that
105 // all responses are ACKs. Need to look more closely at the protocol
106 // semantics here and make sure the server is doing the right thing,
107 // in which case we may be able to avoid this.
108 while (true) {
109 auto response_state = balancer_->ads_service()->lds_response_state();
110 if (!response_state.has_value()) break;
111 ASSERT_TRUE(response_state.has_value());
112 EXPECT_EQ(response_state->state, AdsServiceImpl::ResponseState::ACKED);
113 }
114 // Now unset the resource.
115 balancer_->ads_service()->UnsetResource(
116 kLdsTypeUrl, GetServerListenerName(backends_[0]->port()));
117 // Wait for update to be ACKed.
118 absl::Time deadline =
119 absl::Now() + (absl::Seconds(10) * grpc_test_slowdown_factor());
120 while (true) {
121 auto response_state = balancer_->ads_service()->lds_response_state();
122 if (!response_state.has_value()) {
123 gpr_sleep_until(grpc_timeout_seconds_to_deadline(1));
124 continue;
125 }
126 EXPECT_EQ(response_state->state, AdsServiceImpl::ResponseState::ACKED);
127 ASSERT_LT(absl::Now(), deadline);
128 break;
129 }
130 // Make sure server is still serving.
131 CheckRpcSendOk(DEBUG_LOCATION);
132 }
133
134 // Testing just one example of an invalid resource here.
135 // Unit tests for XdsListenerResourceType have exhaustive tests for all
136 // of the invalid cases.
TEST_P(XdsEnabledServerTest,BadLdsUpdateNoApiListenerNorAddress)137 TEST_P(XdsEnabledServerTest, BadLdsUpdateNoApiListenerNorAddress) {
138 DoSetUp();
139 Listener listener = default_server_listener_;
140 listener.clear_address();
141 listener.set_name(GetServerListenerName(backends_[0]->port()));
142 balancer_->ads_service()->SetLdsResource(listener);
143 StartBackend(0);
144 const auto response_state = WaitForLdsNack(DEBUG_LOCATION);
145 ASSERT_TRUE(response_state.has_value()) << "timed out waiting for NACK";
146 EXPECT_THAT(response_state->error_message,
147 ::testing::EndsWith(absl::StrCat(
148 GetServerListenerName(backends_[0]->port()),
149 ": INVALID_ARGUMENT: Listener has neither address nor "
150 "ApiListener]")));
151 }
152
153 // Verify that a non-TCP listener results in "not serving" status.
TEST_P(XdsEnabledServerTest,NonTcpListener)154 TEST_P(XdsEnabledServerTest, NonTcpListener) {
155 DoSetUp();
156 Listener listener = default_listener_; // Client-side listener.
157 listener = PopulateServerListenerNameAndPort(listener, backends_[0]->port());
158 auto hcm = ClientHcmAccessor().Unpack(listener);
159 auto* rds = hcm.mutable_rds();
160 rds->set_route_config_name(kDefaultRouteConfigurationName);
161 rds->mutable_config_source()->mutable_self();
162 ClientHcmAccessor().Pack(hcm, &listener);
163 balancer_->ads_service()->SetLdsResource(listener);
164 StartBackend(0);
165 ASSERT_TRUE(backends_[0]->WaitOnServingStatusChange(
166 grpc::StatusCode::FAILED_PRECONDITION));
167 }
168
169 // Verify that a mismatch of listening address results in "not serving"
170 // status.
TEST_P(XdsEnabledServerTest,ListenerAddressMismatch)171 TEST_P(XdsEnabledServerTest, ListenerAddressMismatch) {
172 DoSetUp();
173 Listener listener = default_server_listener_;
174 // Set a different listening address in the LDS update
175 listener.mutable_address()->mutable_socket_address()->set_address(
176 "192.168.1.1");
177 SetServerListenerNameAndRouteConfiguration(balancer_.get(), listener,
178 backends_[0]->port(),
179 default_server_route_config_);
180 StartBackend(0);
181 ASSERT_TRUE(backends_[0]->WaitOnServingStatusChange(
182 grpc::StatusCode::FAILED_PRECONDITION));
183 }
184
185 //
186 // server status notification tests
187 //
188
189 class XdsEnabledServerStatusNotificationTest : public XdsEnabledServerTest {
190 protected:
SetUp()191 void SetUp() override { DoSetUp(); }
192
SetValidLdsUpdate()193 void SetValidLdsUpdate() {
194 SetServerListenerNameAndRouteConfiguration(
195 balancer_.get(), default_server_listener_, backends_[0]->port(),
196 default_server_route_config_);
197 }
198
SetInvalidLdsUpdate()199 void SetInvalidLdsUpdate() {
200 Listener listener = default_server_listener_;
201 listener.clear_address();
202 listener.set_name(GetServerListenerName(backends_[0]->port()));
203 balancer_->ads_service()->SetLdsResource(listener);
204 }
205
UnsetLdsUpdate()206 void UnsetLdsUpdate() {
207 balancer_->ads_service()->UnsetResource(
208 kLdsTypeUrl, GetServerListenerName(backends_[0]->port()));
209 }
210 };
211
212 INSTANTIATE_TEST_SUITE_P(XdsTest, XdsEnabledServerStatusNotificationTest,
213 ::testing::Values(XdsTestType()), &XdsTestType::Name);
214
TEST_P(XdsEnabledServerStatusNotificationTest,ServingStatus)215 TEST_P(XdsEnabledServerStatusNotificationTest, ServingStatus) {
216 StartBackend(0);
217 ASSERT_TRUE(backends_[0]->WaitOnServingStatusChange(grpc::StatusCode::OK));
218 CheckRpcSendOk(DEBUG_LOCATION, 1, RpcOptions().set_wait_for_ready(true));
219 }
220
TEST_P(XdsEnabledServerStatusNotificationTest,NotServingStatus)221 TEST_P(XdsEnabledServerStatusNotificationTest, NotServingStatus) {
222 SetInvalidLdsUpdate();
223 StartBackend(0);
224 ASSERT_TRUE(backends_[0]->WaitOnServingStatusChange(
225 grpc::StatusCode::INVALID_ARGUMENT));
226 CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::UNAVAILABLE,
227 MakeConnectionFailureRegex(
228 "connections to all backends failing; last error: "));
229 }
230
TEST_P(XdsEnabledServerStatusNotificationTest,ErrorUpdateWhenAlreadyServing)231 TEST_P(XdsEnabledServerStatusNotificationTest, ErrorUpdateWhenAlreadyServing) {
232 StartBackend(0);
233 ASSERT_TRUE(backends_[0]->WaitOnServingStatusChange(grpc::StatusCode::OK));
234 CheckRpcSendOk(DEBUG_LOCATION, 1, RpcOptions().set_wait_for_ready(true));
235 // Invalid update does not lead to a change in the serving status.
236 SetInvalidLdsUpdate();
237 auto response_state =
238 WaitForLdsNack(DEBUG_LOCATION, RpcOptions(), StatusCode::OK);
239 ASSERT_TRUE(response_state.has_value()) << "timed out waiting for NACK";
240 EXPECT_THAT(response_state->error_message,
241 ::testing::EndsWith(absl::StrCat(
242 GetServerListenerName(backends_[0]->port()),
243 ": INVALID_ARGUMENT: Listener has neither address nor "
244 "ApiListener]")));
245 ASSERT_TRUE(backends_[0]->WaitOnServingStatusChange(grpc::StatusCode::OK));
246 CheckRpcSendOk(DEBUG_LOCATION);
247 }
248
TEST_P(XdsEnabledServerStatusNotificationTest,NotServingStatusToServingStatusTransition)249 TEST_P(XdsEnabledServerStatusNotificationTest,
250 NotServingStatusToServingStatusTransition) {
251 SetInvalidLdsUpdate();
252 StartBackend(0);
253 ASSERT_TRUE(backends_[0]->WaitOnServingStatusChange(
254 grpc::StatusCode::INVALID_ARGUMENT));
255 CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::UNAVAILABLE,
256 MakeConnectionFailureRegex(
257 "connections to all backends failing; last error: "));
258 // Send a valid LDS update to change to serving status
259 SetValidLdsUpdate();
260 ASSERT_TRUE(backends_[0]->WaitOnServingStatusChange(grpc::StatusCode::OK));
261 CheckRpcSendOk(DEBUG_LOCATION, 1, RpcOptions().set_wait_for_ready(true));
262 }
263
264 // This test verifies that the resource getting deleted when already serving
265 // results in future connections being dropped.
TEST_P(XdsEnabledServerStatusNotificationTest,ServingStatusToNonServingStatusTransition)266 TEST_P(XdsEnabledServerStatusNotificationTest,
267 ServingStatusToNonServingStatusTransition) {
268 SetValidLdsUpdate();
269 StartBackend(0);
270 ASSERT_TRUE(backends_[0]->WaitOnServingStatusChange(grpc::StatusCode::OK));
271 CheckRpcSendOk(DEBUG_LOCATION, 1, RpcOptions().set_wait_for_ready(true));
272 // Deleting the resource should result in a non-serving status.
273 UnsetLdsUpdate();
274 ASSERT_TRUE(
275 backends_[0]->WaitOnServingStatusChange(grpc::StatusCode::NOT_FOUND));
276 SendRpcsUntilFailure(
277 DEBUG_LOCATION, StatusCode::UNAVAILABLE,
278 MakeConnectionFailureRegex(
279 "connections to all backends failing; last error: "));
280 }
281
TEST_P(XdsEnabledServerStatusNotificationTest,RepeatedServingStatusChanges)282 TEST_P(XdsEnabledServerStatusNotificationTest, RepeatedServingStatusChanges) {
283 StartBackend(0);
284 for (int i = 0; i < 5; ++i) {
285 // Send a valid LDS update to get the server to start listening
286 SetValidLdsUpdate();
287 ASSERT_TRUE(backends_[0]->WaitOnServingStatusChange(grpc::StatusCode::OK));
288 CheckRpcSendOk(DEBUG_LOCATION, 1, RpcOptions().set_wait_for_ready(true));
289 // Deleting the resource will make the server start rejecting connections
290 UnsetLdsUpdate();
291 ASSERT_TRUE(
292 backends_[0]->WaitOnServingStatusChange(grpc::StatusCode::NOT_FOUND));
293 SendRpcsUntilFailure(
294 DEBUG_LOCATION, StatusCode::UNAVAILABLE,
295 MakeConnectionFailureRegex(
296 "connections to all backends failing; last error: "));
297 }
298 }
299
TEST_P(XdsEnabledServerStatusNotificationTest,ExistingRpcsOnResourceDeletion)300 TEST_P(XdsEnabledServerStatusNotificationTest, ExistingRpcsOnResourceDeletion) {
301 StartBackend(0);
302 ASSERT_TRUE(backends_[0]->WaitOnServingStatusChange(grpc::StatusCode::OK));
303 constexpr int kNumChannels = 10;
304 struct StreamingRpc {
305 std::shared_ptr<Channel> channel;
306 std::unique_ptr<grpc::testing::EchoTestService::Stub> stub;
307 ClientContext context;
308 std::unique_ptr<ClientReaderWriter<EchoRequest, EchoResponse>> stream;
309 } streaming_rpcs[kNumChannels];
310 EchoRequest request;
311 EchoResponse response;
312 request.set_message("Hello");
313 ChannelArguments args;
314 args.SetInt(GRPC_ARG_USE_LOCAL_SUBCHANNEL_POOL, 1);
315 for (int i = 0; i < kNumChannels; i++) {
316 streaming_rpcs[i].channel =
317 CreateCustomChannel(grpc_core::LocalIpUri(backends_[0]->port()),
318 InsecureChannelCredentials(), args);
319 streaming_rpcs[i].stub =
320 grpc::testing::EchoTestService::NewStub(streaming_rpcs[i].channel);
321 streaming_rpcs[i].context.set_wait_for_ready(true);
322 streaming_rpcs[i].stream =
323 streaming_rpcs[i].stub->BidiStream(&streaming_rpcs[i].context);
324 EXPECT_TRUE(streaming_rpcs[i].stream->Write(request));
325 streaming_rpcs[i].stream->Read(&response);
326 EXPECT_EQ(request.message(), response.message());
327 }
328 // Deleting the resource will make the server start rejecting connections
329 UnsetLdsUpdate();
330 ASSERT_TRUE(
331 backends_[0]->WaitOnServingStatusChange(grpc::StatusCode::NOT_FOUND));
332 SendRpcsUntilFailure(
333 DEBUG_LOCATION, StatusCode::UNAVAILABLE,
334 MakeConnectionFailureRegex(
335 "connections to all backends failing; last error: "));
336 for (int i = 0; i < kNumChannels; i++) {
337 EXPECT_TRUE(streaming_rpcs[i].stream->Write(request));
338 streaming_rpcs[i].stream->Read(&response);
339 EXPECT_EQ(request.message(), response.message());
340 EXPECT_TRUE(streaming_rpcs[i].stream->WritesDone());
341 auto status = streaming_rpcs[i].stream->Finish();
342 EXPECT_TRUE(status.ok())
343 << status.error_message() << ", " << status.error_details() << ", "
344 << streaming_rpcs[i].context.debug_error_string();
345 // New RPCs on the existing channels should fail.
346 ClientContext new_context;
347 new_context.set_deadline(grpc_timeout_milliseconds_to_deadline(1000));
348 EXPECT_FALSE(
349 streaming_rpcs[i].stub->Echo(&new_context, request, &response).ok());
350 }
351 }
352
TEST_P(XdsEnabledServerStatusNotificationTest,ExistingRpcsFailOnResourceUpdateAfterDrainGraceTimeExpires)353 TEST_P(XdsEnabledServerStatusNotificationTest,
354 ExistingRpcsFailOnResourceUpdateAfterDrainGraceTimeExpires) {
355 constexpr int kDrainGraceTimeMs = 100;
356 xds_drain_grace_time_ms_ = kDrainGraceTimeMs;
357 StartBackend(0);
358 ASSERT_TRUE(backends_[0]->WaitOnServingStatusChange(grpc::StatusCode::OK));
359 constexpr int kNumChannels = 10;
360 struct StreamingRpc {
361 std::shared_ptr<Channel> channel;
362 std::unique_ptr<grpc::testing::EchoTestService::Stub> stub;
363 ClientContext context;
364 std::unique_ptr<ClientReaderWriter<EchoRequest, EchoResponse>> stream;
365 } streaming_rpcs[kNumChannels];
366 EchoRequest request;
367 EchoResponse response;
368 request.set_message("Hello");
369 ChannelArguments args;
370 args.SetInt(GRPC_ARG_USE_LOCAL_SUBCHANNEL_POOL, 1);
371 for (int i = 0; i < kNumChannels; i++) {
372 streaming_rpcs[i].channel =
373 CreateCustomChannel(grpc_core::LocalIpUri(backends_[0]->port()),
374 InsecureChannelCredentials(), args);
375 streaming_rpcs[i].stub =
376 grpc::testing::EchoTestService::NewStub(streaming_rpcs[i].channel);
377 streaming_rpcs[i].context.set_wait_for_ready(true);
378 streaming_rpcs[i].stream =
379 streaming_rpcs[i].stub->BidiStream(&streaming_rpcs[i].context);
380 EXPECT_TRUE(streaming_rpcs[i].stream->Write(request));
381 streaming_rpcs[i].stream->Read(&response);
382 EXPECT_EQ(request.message(), response.message());
383 }
384 grpc_core::Timestamp update_time = NowFromCycleCounter();
385 // Update the resource. We modify the route with an invalid entry, so
386 // that we can tell from the RPC failure messages when the server has
387 // seen the change.
388 auto route_config = default_server_route_config_;
389 route_config.mutable_virtual_hosts(0)->mutable_routes(0)->mutable_redirect();
390 SetServerListenerNameAndRouteConfiguration(
391 balancer_.get(), default_server_listener_, backends_[0]->port(),
392 route_config);
393 SendRpcsUntilFailure(DEBUG_LOCATION, StatusCode::UNAVAILABLE,
394 "UNAVAILABLE:matching route has unsupported action");
395 // After the drain grace time expires, the existing RPCs should all fail.
396 for (int i = 0; i < kNumChannels; i++) {
397 // Wait for the drain grace time to expire
398 EXPECT_FALSE(streaming_rpcs[i].stream->Read(&response));
399 // Make sure that the drain grace interval is honored.
400 EXPECT_GE(NowFromCycleCounter() - update_time,
401 grpc_core::Duration::Milliseconds(kDrainGraceTimeMs));
402 auto status = streaming_rpcs[i].stream->Finish();
403 EXPECT_EQ(status.error_code(), grpc::StatusCode::UNAVAILABLE)
404 << status.error_code() << ", " << status.error_message() << ", "
405 << status.error_details() << ", "
406 << streaming_rpcs[i].context.debug_error_string();
407 EXPECT_EQ(status.error_message(),
408 "Drain grace time expired. Closing connection immediately.");
409 }
410 }
411
412 //
413 // filter chain matching tests
414 //
415
416 class XdsServerFilterChainMatchTest : public XdsEnabledServerTest {
417 public:
SetUp()418 void SetUp() override { DoSetUp(); }
419
GetHttpConnectionManager(const Listener & listener)420 HttpConnectionManager GetHttpConnectionManager(const Listener& listener) {
421 HttpConnectionManager http_connection_manager =
422 ServerHcmAccessor().Unpack(listener);
423 *http_connection_manager.mutable_route_config() =
424 default_server_route_config_;
425 return http_connection_manager;
426 }
427 };
428
429 // Run with bootstrap from env var so that we use one XdsClient.
430 INSTANTIATE_TEST_SUITE_P(XdsTest, XdsServerFilterChainMatchTest,
431 ::testing::Values(XdsTestType().set_bootstrap_source(
432 XdsTestType::kBootstrapFromEnvVar)),
433 &XdsTestType::Name);
434
TEST_P(XdsServerFilterChainMatchTest,DefaultFilterChainUsedWhenNoFilterChainMentioned)435 TEST_P(XdsServerFilterChainMatchTest,
436 DefaultFilterChainUsedWhenNoFilterChainMentioned) {
437 StartBackend(0);
438 ASSERT_TRUE(backends_[0]->WaitOnServingStatusChange(grpc::StatusCode::OK));
439 CheckRpcSendOk(DEBUG_LOCATION, 1, RpcOptions().set_wait_for_ready(true));
440 }
441
TEST_P(XdsServerFilterChainMatchTest,DefaultFilterChainUsedWhenOtherFilterChainsDontMatch)442 TEST_P(XdsServerFilterChainMatchTest,
443 DefaultFilterChainUsedWhenOtherFilterChainsDontMatch) {
444 Listener listener = default_server_listener_;
445 // Add a filter chain that will never get matched
446 auto* filter_chain = listener.add_filter_chains();
447 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
448 GetHttpConnectionManager(listener));
449 filter_chain->mutable_filter_chain_match()
450 ->mutable_destination_port()
451 ->set_value(8080);
452 SetServerListenerNameAndRouteConfiguration(balancer_.get(), listener,
453 backends_[0]->port(),
454 default_server_route_config_);
455 StartBackend(0);
456 ASSERT_TRUE(backends_[0]->WaitOnServingStatusChange(grpc::StatusCode::OK));
457 CheckRpcSendOk(DEBUG_LOCATION, 1, RpcOptions().set_wait_for_ready(true));
458 }
459
TEST_P(XdsServerFilterChainMatchTest,FilterChainsWithDestinationPortDontMatch)460 TEST_P(XdsServerFilterChainMatchTest,
461 FilterChainsWithDestinationPortDontMatch) {
462 Listener listener = default_server_listener_;
463 // Add filter chain with destination port that should never get matched
464 auto* filter_chain = listener.add_filter_chains();
465 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
466 GetHttpConnectionManager(listener));
467 filter_chain->mutable_filter_chain_match()
468 ->mutable_destination_port()
469 ->set_value(8080);
470 listener.clear_default_filter_chain();
471 balancer_->ads_service()->SetLdsResource(
472 PopulateServerListenerNameAndPort(listener, backends_[0]->port()));
473 StartBackend(0);
474 ASSERT_TRUE(backends_[0]->WaitOnServingStatusChange(grpc::StatusCode::OK));
475 // RPC should fail since no matching filter chain was found and no default
476 // filter chain is configured.
477 CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::UNAVAILABLE,
478 MakeConnectionFailureRegex(
479 "connections to all backends failing; last error: "));
480 }
481
TEST_P(XdsServerFilterChainMatchTest,FilterChainsWithServerNamesDontMatch)482 TEST_P(XdsServerFilterChainMatchTest, FilterChainsWithServerNamesDontMatch) {
483 Listener listener = default_server_listener_;
484 // Add filter chain with server name that should never get matched
485 auto* filter_chain = listener.add_filter_chains();
486 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
487 GetHttpConnectionManager(listener));
488 filter_chain->mutable_filter_chain_match()->add_server_names("server_name");
489 listener.clear_default_filter_chain();
490 balancer_->ads_service()->SetLdsResource(
491 PopulateServerListenerNameAndPort(listener, backends_[0]->port()));
492 StartBackend(0);
493 ASSERT_TRUE(backends_[0]->WaitOnServingStatusChange(grpc::StatusCode::OK));
494 // RPC should fail since no matching filter chain was found and no default
495 // filter chain is configured.
496 CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::UNAVAILABLE,
497 MakeConnectionFailureRegex(
498 "connections to all backends failing; last error: "));
499 }
500
TEST_P(XdsServerFilterChainMatchTest,FilterChainsWithTransportProtocolsOtherThanRawBufferDontMatch)501 TEST_P(XdsServerFilterChainMatchTest,
502 FilterChainsWithTransportProtocolsOtherThanRawBufferDontMatch) {
503 Listener listener = default_server_listener_;
504 // Add filter chain with transport protocol "tls" that should never match
505 auto* filter_chain = listener.add_filter_chains();
506 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
507 GetHttpConnectionManager(listener));
508 filter_chain->mutable_filter_chain_match()->set_transport_protocol("tls");
509 listener.clear_default_filter_chain();
510 balancer_->ads_service()->SetLdsResource(
511 PopulateServerListenerNameAndPort(listener, backends_[0]->port()));
512 StartBackend(0);
513 ASSERT_TRUE(backends_[0]->WaitOnServingStatusChange(grpc::StatusCode::OK));
514 // RPC should fail since no matching filter chain was found and no default
515 // filter chain is configured.
516 CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::UNAVAILABLE,
517 MakeConnectionFailureRegex(
518 "connections to all backends failing; last error: "));
519 }
520
TEST_P(XdsServerFilterChainMatchTest,FilterChainsWithApplicationProtocolsDontMatch)521 TEST_P(XdsServerFilterChainMatchTest,
522 FilterChainsWithApplicationProtocolsDontMatch) {
523 Listener listener = default_server_listener_;
524 // Add filter chain with application protocol that should never get matched
525 auto* filter_chain = listener.add_filter_chains();
526 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
527 GetHttpConnectionManager(listener));
528 filter_chain->mutable_filter_chain_match()->add_application_protocols("h2");
529 listener.clear_default_filter_chain();
530 balancer_->ads_service()->SetLdsResource(
531 PopulateServerListenerNameAndPort(listener, backends_[0]->port()));
532 StartBackend(0);
533 ASSERT_TRUE(backends_[0]->WaitOnServingStatusChange(grpc::StatusCode::OK));
534 // RPC should fail since no matching filter chain was found and no default
535 // filter chain is configured.
536 CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::UNAVAILABLE,
537 MakeConnectionFailureRegex(
538 "connections to all backends failing; last error: "));
539 }
540
TEST_P(XdsServerFilterChainMatchTest,FilterChainsWithTransportProtocolRawBufferIsPreferred)541 TEST_P(XdsServerFilterChainMatchTest,
542 FilterChainsWithTransportProtocolRawBufferIsPreferred) {
543 Listener listener = default_server_listener_;
544 // Add filter chain with "raw_buffer" transport protocol
545 auto* filter_chain = listener.add_filter_chains();
546 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
547 GetHttpConnectionManager(listener));
548 filter_chain->mutable_filter_chain_match()->set_transport_protocol(
549 "raw_buffer");
550 // Add another filter chain with no transport protocol set but application
551 // protocol set (fails match)
552 filter_chain = listener.add_filter_chains();
553 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
554 GetHttpConnectionManager(listener));
555 filter_chain->mutable_filter_chain_match()->add_application_protocols("h2");
556 listener.clear_default_filter_chain();
557 balancer_->ads_service()->SetLdsResource(
558 PopulateServerListenerNameAndPort(listener, backends_[0]->port()));
559 StartBackend(0);
560 ASSERT_TRUE(backends_[0]->WaitOnServingStatusChange(grpc::StatusCode::OK));
561 // A successful RPC proves that filter chains that mention "raw_buffer" as
562 // the transport protocol are chosen as the best match in the round.
563 CheckRpcSendOk(DEBUG_LOCATION, 1, RpcOptions().set_wait_for_ready(true));
564 }
565
TEST_P(XdsServerFilterChainMatchTest,FilterChainsWithMoreSpecificDestinationPrefixRangesArePreferred)566 TEST_P(XdsServerFilterChainMatchTest,
567 FilterChainsWithMoreSpecificDestinationPrefixRangesArePreferred) {
568 Listener listener = default_server_listener_;
569 // Add filter chain with prefix range (length 4 and 16) but with server name
570 // mentioned. (Prefix range is matched first.)
571 auto* filter_chain = listener.add_filter_chains();
572 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
573 GetHttpConnectionManager(listener));
574 auto* prefix_range =
575 filter_chain->mutable_filter_chain_match()->add_prefix_ranges();
576 prefix_range->set_address_prefix(grpc_core::LocalIp());
577 prefix_range->mutable_prefix_len()->set_value(4);
578 prefix_range =
579 filter_chain->mutable_filter_chain_match()->add_prefix_ranges();
580 prefix_range->set_address_prefix(grpc_core::LocalIp());
581 prefix_range->mutable_prefix_len()->set_value(16);
582 filter_chain->mutable_filter_chain_match()->add_server_names("server_name");
583 // Add filter chain with two prefix ranges (length 8 and 24). Since 24 is
584 // the highest match, it should be chosen.
585 filter_chain = listener.add_filter_chains();
586 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
587 GetHttpConnectionManager(listener));
588 prefix_range =
589 filter_chain->mutable_filter_chain_match()->add_prefix_ranges();
590 prefix_range->set_address_prefix(grpc_core::LocalIp());
591 prefix_range->mutable_prefix_len()->set_value(8);
592 prefix_range =
593 filter_chain->mutable_filter_chain_match()->add_prefix_ranges();
594 prefix_range->set_address_prefix(grpc_core::LocalIp());
595 prefix_range->mutable_prefix_len()->set_value(24);
596 // Add another filter chain with a non-matching prefix range (with length
597 // 30)
598 filter_chain = listener.add_filter_chains();
599 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
600 GetHttpConnectionManager(listener));
601 prefix_range =
602 filter_chain->mutable_filter_chain_match()->add_prefix_ranges();
603 prefix_range->set_address_prefix("192.168.1.1");
604 prefix_range->mutable_prefix_len()->set_value(30);
605 filter_chain->mutable_filter_chain_match()->add_server_names("server_name");
606 // Add another filter chain with no prefix range mentioned
607 filter_chain = listener.add_filter_chains();
608 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
609 GetHttpConnectionManager(listener));
610 filter_chain->mutable_filter_chain_match()->add_server_names("server_name");
611 listener.clear_default_filter_chain();
612 balancer_->ads_service()->SetLdsResource(
613 PopulateServerListenerNameAndPort(listener, backends_[0]->port()));
614 StartBackend(0);
615 ASSERT_TRUE(backends_[0]->WaitOnServingStatusChange(grpc::StatusCode::OK));
616 // A successful RPC proves that the filter chain with the longest matching
617 // prefix range was the best match.
618 CheckRpcSendOk(DEBUG_LOCATION, 1, RpcOptions().set_wait_for_ready(true));
619 }
620
TEST_P(XdsServerFilterChainMatchTest,FilterChainsThatMentionSourceTypeArePreferred)621 TEST_P(XdsServerFilterChainMatchTest,
622 FilterChainsThatMentionSourceTypeArePreferred) {
623 Listener listener = default_server_listener_;
624 // Add filter chain with the local source type (best match)
625 auto* filter_chain = listener.add_filter_chains();
626 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
627 GetHttpConnectionManager(listener));
628 filter_chain->mutable_filter_chain_match()->set_source_type(
629 FilterChainMatch::SAME_IP_OR_LOOPBACK);
630 // Add filter chain with the external source type but bad source port.
631 // Note that backends_[0]->port() will never be a match for the source port
632 // because it is already being used by a backend.
633 filter_chain = listener.add_filter_chains();
634 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
635 GetHttpConnectionManager(listener));
636 filter_chain->mutable_filter_chain_match()->set_source_type(
637 FilterChainMatch::EXTERNAL);
638 filter_chain->mutable_filter_chain_match()->add_source_ports(
639 backends_[0]->port());
640 // Add filter chain with the default source type (ANY) but bad source port.
641 filter_chain = listener.add_filter_chains();
642 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
643 GetHttpConnectionManager(listener));
644 filter_chain->mutable_filter_chain_match()->add_source_ports(
645 backends_[0]->port());
646 listener.clear_default_filter_chain();
647 balancer_->ads_service()->SetLdsResource(
648 PopulateServerListenerNameAndPort(listener, backends_[0]->port()));
649 StartBackend(0);
650 ASSERT_TRUE(backends_[0]->WaitOnServingStatusChange(grpc::StatusCode::OK));
651 // A successful RPC proves that the filter chain with the longest matching
652 // prefix range was the best match.
653 CheckRpcSendOk(DEBUG_LOCATION, 1, RpcOptions().set_wait_for_ready(true));
654 }
655
TEST_P(XdsServerFilterChainMatchTest,FilterChainsWithMoreSpecificSourcePrefixRangesArePreferred)656 TEST_P(XdsServerFilterChainMatchTest,
657 FilterChainsWithMoreSpecificSourcePrefixRangesArePreferred) {
658 Listener listener = default_server_listener_;
659 // Add filter chain with source prefix range (length 16) but with a bad
660 // source port mentioned. (Prefix range is matched first.) Note that
661 // backends_[0]->port() will never be a match for the source port because it
662 // is already being used by a backend.
663 auto* filter_chain = listener.add_filter_chains();
664 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
665 GetHttpConnectionManager(listener));
666 auto* source_prefix_range =
667 filter_chain->mutable_filter_chain_match()->add_source_prefix_ranges();
668 source_prefix_range->set_address_prefix(grpc_core::LocalIp());
669 source_prefix_range->mutable_prefix_len()->set_value(4);
670 source_prefix_range =
671 filter_chain->mutable_filter_chain_match()->add_source_prefix_ranges();
672 source_prefix_range->set_address_prefix(grpc_core::LocalIp());
673 source_prefix_range->mutable_prefix_len()->set_value(16);
674 filter_chain->mutable_filter_chain_match()->add_source_ports(
675 backends_[0]->port());
676 // Add filter chain with two source prefix ranges (length 8 and 24). Since
677 // 24 is the highest match, it should be chosen.
678 filter_chain = listener.add_filter_chains();
679 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
680 GetHttpConnectionManager(listener));
681 source_prefix_range =
682 filter_chain->mutable_filter_chain_match()->add_source_prefix_ranges();
683 source_prefix_range->set_address_prefix(grpc_core::LocalIp());
684 source_prefix_range->mutable_prefix_len()->set_value(8);
685 source_prefix_range =
686 filter_chain->mutable_filter_chain_match()->add_source_prefix_ranges();
687 source_prefix_range->set_address_prefix(grpc_core::LocalIp());
688 source_prefix_range->mutable_prefix_len()->set_value(24);
689 // Add another filter chain with a non-matching source prefix range (with
690 // length 30) and bad source port
691 filter_chain = listener.add_filter_chains();
692 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
693 GetHttpConnectionManager(listener));
694 source_prefix_range =
695 filter_chain->mutable_filter_chain_match()->add_source_prefix_ranges();
696 source_prefix_range->set_address_prefix("192.168.1.1");
697 source_prefix_range->mutable_prefix_len()->set_value(30);
698 filter_chain->mutable_filter_chain_match()->add_source_ports(
699 backends_[0]->port());
700 // Add another filter chain with no source prefix range mentioned and bad
701 // source port
702 filter_chain = listener.add_filter_chains();
703 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
704 GetHttpConnectionManager(listener));
705 filter_chain->mutable_filter_chain_match()->add_source_ports(
706 backends_[0]->port());
707 listener.clear_default_filter_chain();
708 balancer_->ads_service()->SetLdsResource(
709 PopulateServerListenerNameAndPort(listener, backends_[0]->port()));
710 StartBackend(0);
711 ASSERT_TRUE(backends_[0]->WaitOnServingStatusChange(grpc::StatusCode::OK));
712 // A successful RPC proves that the filter chain with the longest matching
713 // source prefix range was the best match.
714 CheckRpcSendOk(DEBUG_LOCATION, 1, RpcOptions().set_wait_for_ready(true));
715 }
716
TEST_P(XdsServerFilterChainMatchTest,FilterChainsWithMoreSpecificSourcePortArePreferred)717 TEST_P(XdsServerFilterChainMatchTest,
718 FilterChainsWithMoreSpecificSourcePortArePreferred) {
719 Listener listener = default_server_listener_;
720 auto* filter_chain = listener.add_filter_chains();
721 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
722 GetHttpConnectionManager(listener));
723 // Since we don't know which port will be used by the channel, just add all
724 // ports except for 0.
725 for (int i = 1; i < 65536; i++) {
726 filter_chain->mutable_filter_chain_match()->add_source_ports(i);
727 }
728 // Add another filter chain with no source port mentioned whose route
729 // config has a route with an unsupported action.
730 auto hcm = GetHttpConnectionManager(listener);
731 hcm.mutable_route_config()
732 ->mutable_virtual_hosts(0)
733 ->mutable_routes(0)
734 ->mutable_redirect();
735 filter_chain = listener.add_filter_chains();
736 filter_chain->add_filters()->mutable_typed_config()->PackFrom(hcm);
737 listener.clear_default_filter_chain();
738 balancer_->ads_service()->SetLdsResource(
739 PopulateServerListenerNameAndPort(listener, backends_[0]->port()));
740 StartBackend(0);
741 ASSERT_TRUE(backends_[0]->WaitOnServingStatusChange(grpc::StatusCode::OK));
742 // A successful RPC proves that the filter chain with matching source port
743 // was chosen.
744 CheckRpcSendOk(DEBUG_LOCATION, 1, RpcOptions().set_wait_for_ready(true));
745 }
746
747 //
748 // server-side RDS tests
749 //
750
751 class XdsServerRdsTest : public XdsEnabledServerTest {
752 public:
SetUp()753 void SetUp() override { DoSetUp(); }
754 };
755
756 // Test both with and without RDS.
757 // Run with bootstrap from env var so that we use one XdsClient.
758 INSTANTIATE_TEST_SUITE_P(
759 XdsTest, XdsServerRdsTest,
760 ::testing::Values(
761 XdsTestType().set_bootstrap_source(XdsTestType::kBootstrapFromEnvVar),
762 XdsTestType()
763 .set_bootstrap_source(XdsTestType::kBootstrapFromEnvVar)
764 .set_enable_rds_testing()),
765 &XdsTestType::Name);
766
TEST_P(XdsServerRdsTest,Basic)767 TEST_P(XdsServerRdsTest, Basic) {
768 StartBackend(0);
769 ASSERT_TRUE(backends_[0]->WaitOnServingStatusChange(grpc::StatusCode::OK));
770 CheckRpcSendOk(DEBUG_LOCATION, 1, RpcOptions().set_wait_for_ready(true));
771 }
772
TEST_P(XdsServerRdsTest,FailsRouteMatchesOtherThanNonForwardingAction)773 TEST_P(XdsServerRdsTest, FailsRouteMatchesOtherThanNonForwardingAction) {
774 SetServerListenerNameAndRouteConfiguration(
775 balancer_.get(), default_server_listener_, backends_[0]->port(),
776 default_route_config_ /* inappropriate route config for servers */);
777 StartBackend(0);
778 // The server should be ready to serve but RPCs should fail.
779 ASSERT_TRUE(backends_[0]->WaitOnServingStatusChange(grpc::StatusCode::OK));
780 CheckRpcSendFailure(DEBUG_LOCATION, StatusCode::UNAVAILABLE,
781 "UNAVAILABLE:matching route has unsupported action");
782 }
783
784 // Test that non-inline route configuration also works for non-default filter
785 // chains
TEST_P(XdsServerRdsTest,NonInlineRouteConfigurationNonDefaultFilterChain)786 TEST_P(XdsServerRdsTest, NonInlineRouteConfigurationNonDefaultFilterChain) {
787 if (!GetParam().enable_rds_testing()) return;
788 Listener listener = default_server_listener_;
789 auto* filter_chain = listener.add_filter_chains();
790 HttpConnectionManager http_connection_manager =
791 ServerHcmAccessor().Unpack(listener);
792 auto* rds = http_connection_manager.mutable_rds();
793 rds->set_route_config_name(kDefaultServerRouteConfigurationName);
794 rds->mutable_config_source()->mutable_self();
795 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
796 http_connection_manager);
797 SetServerListenerNameAndRouteConfiguration(balancer_.get(), listener,
798 backends_[0]->port(),
799 default_server_route_config_);
800 StartBackend(0);
801 ASSERT_TRUE(backends_[0]->WaitOnServingStatusChange(grpc::StatusCode::OK));
802 CheckRpcSendOk(DEBUG_LOCATION, 1, RpcOptions().set_wait_for_ready(true));
803 }
804
TEST_P(XdsServerRdsTest,NonInlineRouteConfigurationNotAvailable)805 TEST_P(XdsServerRdsTest, NonInlineRouteConfigurationNotAvailable) {
806 if (!GetParam().enable_rds_testing()) return;
807 Listener listener = default_server_listener_;
808 HttpConnectionManager http_connection_manager =
809 ServerHcmAccessor().Unpack(listener);
810 auto* rds = http_connection_manager.mutable_rds();
811 rds->set_route_config_name("unknown_server_route_config");
812 rds->mutable_config_source()->mutable_self();
813 listener.add_filter_chains()->add_filters()->mutable_typed_config()->PackFrom(
814 http_connection_manager);
815 SetServerListenerNameAndRouteConfiguration(balancer_.get(), listener,
816 backends_[0]->port(),
817 default_server_route_config_);
818 StartBackend(0);
819 ASSERT_TRUE(backends_[0]->WaitOnServingStatusChange(grpc::StatusCode::OK));
820 CheckRpcSendFailure(
821 DEBUG_LOCATION, StatusCode::UNAVAILABLE,
822 "RDS resource unknown_server_route_config: does not exist "
823 "\\(node ID:xds_end2end_test\\)");
824 }
825
826 // TODO(yashykt): Once https://github.com/grpc/grpc/issues/24035 is fixed, we
827 // should add tests that make sure that different route configs are used for
828 // incoming connections with a different match.
TEST_P(XdsServerRdsTest,MultipleRouteConfigurations)829 TEST_P(XdsServerRdsTest, MultipleRouteConfigurations) {
830 Listener listener = default_server_listener_;
831 // Set a filter chain with a new route config name
832 auto new_route_config = default_server_route_config_;
833 new_route_config.set_name("new_server_route_config");
834 HttpConnectionManager http_connection_manager =
835 ServerHcmAccessor().Unpack(listener);
836 auto* rds = http_connection_manager.mutable_rds();
837 rds->set_route_config_name(new_route_config.name());
838 rds->mutable_config_source()->mutable_self();
839 listener.add_filter_chains()->add_filters()->mutable_typed_config()->PackFrom(
840 http_connection_manager);
841 // Set another filter chain with another route config name
842 auto another_route_config = default_server_route_config_;
843 another_route_config.set_name("another_server_route_config");
844 http_connection_manager.mutable_rds()->set_route_config_name(
845 another_route_config.name());
846 auto* filter_chain = listener.add_filter_chains();
847 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
848 http_connection_manager);
849 filter_chain->mutable_filter_chain_match()->set_source_type(
850 FilterChainMatch::SAME_IP_OR_LOOPBACK);
851 // Add another filter chain with the same route config name
852 filter_chain = listener.add_filter_chains();
853 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
854 http_connection_manager);
855 filter_chain->mutable_filter_chain_match()->set_source_type(
856 FilterChainMatch::EXTERNAL);
857 // Add another filter chain with an inline route config
858 filter_chain = listener.add_filter_chains();
859 filter_chain->mutable_filter_chain_match()->add_source_ports(1234);
860 http_connection_manager = ServerHcmAccessor().Unpack(listener);
861 *http_connection_manager.mutable_route_config() =
862 default_server_route_config_;
863 filter_chain->add_filters()->mutable_typed_config()->PackFrom(
864 http_connection_manager);
865 // Set resources on the ADS service
866 balancer_->ads_service()->SetRdsResource(new_route_config);
867 balancer_->ads_service()->SetRdsResource(another_route_config);
868 SetServerListenerNameAndRouteConfiguration(balancer_.get(), listener,
869 backends_[0]->port(),
870 default_server_route_config_);
871 StartBackend(0);
872 ASSERT_TRUE(backends_[0]->WaitOnServingStatusChange(grpc::StatusCode::OK));
873 CheckRpcSendOk(DEBUG_LOCATION, 1, RpcOptions().set_wait_for_ready(true));
874 }
875
876 } // namespace
877 } // namespace testing
878 } // namespace grpc
879
main(int argc,char ** argv)880 int main(int argc, char** argv) {
881 grpc::testing::TestEnvironment env(&argc, argv);
882 ::testing::InitGoogleTest(&argc, argv);
883 // Make the backup poller poll very frequently in order to pick up
884 // updates from all the subchannels's FDs.
885 grpc_core::ConfigVars::Overrides overrides;
886 overrides.client_channel_backup_poll_interval_ms = 1;
887 overrides.trace =
888 "call,channel,client_channel,client_channel_call,client_channel_lb_call,"
889 "handshaker";
890 grpc_core::ConfigVars::SetOverrides(overrides);
891 #if TARGET_OS_IPHONE
892 // Workaround Apple CFStream bug
893 grpc_core::SetEnv("grpc_cfstream", "0");
894 #endif
895 grpc_init();
896 const auto result = RUN_ALL_TESTS();
897 grpc_shutdown();
898 return result;
899 }
900