1 // Copyright 2017 gRPC authors.
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14 //
15
16 #include "test/cpp/end2end/xds/xds_utils.h"
17
18 #include <grpcpp/security/tls_certificate_provider.h>
19
20 #include <functional>
21 #include <map>
22 #include <memory>
23 #include <set>
24 #include <string>
25 #include <thread>
26 #include <vector>
27
28 #include "absl/memory/memory.h"
29 #include "absl/strings/numbers.h"
30 #include "absl/strings/str_cat.h"
31 #include "absl/strings/str_format.h"
32 #include "absl/strings/str_join.h"
33 #include "absl/strings/str_replace.h"
34 #include "absl/strings/string_view.h"
35 #include "absl/types/optional.h"
36 #include "envoy/extensions/filters/http/router/v3/router.pb.h"
37 #include "src/core/ext/filters/http/server/http_server_filter.h"
38 #include "src/core/server/server.h"
39 #include "src/core/util/env.h"
40 #include "src/core/util/tmpfile.h"
41 #include "src/core/xds/grpc/xds_client_grpc.h"
42 #include "src/core/xds/xds_client/xds_channel_args.h"
43 #include "src/cpp/client/secure_credentials.h"
44 #include "test/core/test_util/resolve_localhost_ip46.h"
45
46 namespace grpc {
47 namespace testing {
48
49 using ::envoy::config::cluster::v3::Cluster;
50 using ::envoy::config::core::v3::HealthStatus;
51 using ::envoy::config::endpoint::v3::ClusterLoadAssignment;
52 using ::envoy::config::listener::v3::Listener;
53 using ::envoy::config::route::v3::RouteConfiguration;
54 using ::envoy::extensions::filters::network::http_connection_manager::v3::
55 HttpConnectionManager;
56
57 //
58 // XdsBootstrapBuilder
59 //
60
Build()61 std::string XdsBootstrapBuilder::Build() {
62 std::vector<std::string> fields;
63 fields.push_back(MakeXdsServersText(servers_));
64 if (!client_default_listener_resource_name_template_.empty()) {
65 fields.push_back(
66 absl::StrCat(" \"client_default_listener_resource_name_template\": \"",
67 client_default_listener_resource_name_template_, "\""));
68 }
69 fields.push_back(MakeNodeText());
70 if (!server_listener_resource_name_template_.empty()) {
71 fields.push_back(
72 absl::StrCat(" \"server_listener_resource_name_template\": \"",
73 server_listener_resource_name_template_, "\""));
74 }
75 fields.push_back(MakeCertificateProviderText());
76 fields.push_back(MakeAuthorityText());
77 return absl::StrCat("{", absl::StrJoin(fields, ",\n"), "}");
78 }
79
MakeXdsServersText(absl::Span<const std::string> server_uris)80 std::string XdsBootstrapBuilder::MakeXdsServersText(
81 absl::Span<const std::string> server_uris) {
82 constexpr char kXdsServerTemplate[] =
83 " {\n"
84 " \"server_uri\": \"<SERVER_URI>\",\n"
85 " \"channel_creds\": [\n"
86 " {\n"
87 " \"type\": \"<SERVER_CREDS_TYPE>\"<SERVER_CREDS_CONFIG>\n"
88 " }\n"
89 " ],\n"
90 " \"server_features\": [<SERVER_FEATURES>]\n"
91 " }";
92 std::vector<std::string> server_features;
93 if (ignore_resource_deletion_) {
94 server_features.push_back("\"ignore_resource_deletion\"");
95 }
96 if (trusted_xds_server_) {
97 server_features.push_back("\"trusted_xds_server\"");
98 }
99 std::vector<std::string> servers;
100 for (absl::string_view server_uri : server_uris) {
101 servers.emplace_back(absl::StrReplaceAll(
102 kXdsServerTemplate,
103 {{"<SERVER_URI>", server_uri},
104 {"<SERVER_CREDS_TYPE>", xds_channel_creds_type_},
105 {"<SERVER_CREDS_CONFIG>",
106 xds_channel_creds_config_.empty()
107 ? ""
108 : absl::StrCat(",\n \"config\": ",
109 xds_channel_creds_config_)},
110 {"<SERVER_FEATURES>", absl::StrJoin(server_features, ", ")}}));
111 }
112 return absl::StrCat(" \"xds_servers\": [\n",
113 absl::StrJoin(servers, ",\n"), "\n ]");
114 }
115
MakeNodeText()116 std::string XdsBootstrapBuilder::MakeNodeText() {
117 constexpr char kXdsNode[] =
118 " \"node\": {\n"
119 " \"id\": \"xds_end2end_test\",\n"
120 " \"cluster\": \"test\",\n"
121 " \"metadata\": {\n"
122 " \"foo\": \"bar\"\n"
123 " },\n"
124 " \"locality\": {\n"
125 " \"region\": \"corp\",\n"
126 " \"zone\": \"svl\",\n"
127 " \"sub_zone\": \"mp3\"\n"
128 " }\n"
129 " }";
130 return kXdsNode;
131 }
132
MakeCertificateProviderText()133 std::string XdsBootstrapBuilder::MakeCertificateProviderText() {
134 std::vector<std::string> entries;
135 for (const auto& p : plugins_) {
136 const std::string& key = p.first;
137 const PluginInfo& plugin_info = p.second;
138 std::vector<std::string> fields;
139 fields.push_back(absl::StrFormat(" \"%s\": {", key));
140 if (!plugin_info.plugin_config.empty()) {
141 fields.push_back(
142 absl::StrFormat(" \"plugin_name\": \"%s\",", plugin_info.name));
143 fields.push_back(absl::StrCat(" \"config\": {\n",
144 plugin_info.plugin_config, "\n }"));
145 } else {
146 fields.push_back(
147 absl::StrFormat(" \"plugin_name\": \"%s\"", plugin_info.name));
148 }
149 fields.push_back(" }");
150 entries.push_back(absl::StrJoin(fields, "\n"));
151 }
152 return absl::StrCat(" \"certificate_providers\": {\n",
153 absl::StrJoin(entries, ",\n"), " \n}");
154 }
155
MakeAuthorityText()156 std::string XdsBootstrapBuilder::MakeAuthorityText() {
157 std::vector<std::string> entries;
158 for (const auto& p : authorities_) {
159 const std::string& name = p.first;
160 const AuthorityInfo& authority_info = p.second;
161 std::vector<std::string> fields = {
162 MakeXdsServersText(authority_info.servers)};
163 if (!authority_info.client_listener_resource_name_template.empty()) {
164 fields.push_back(absl::StrCat(
165 "\"client_listener_resource_name_template\": \"",
166 authority_info.client_listener_resource_name_template, "\""));
167 }
168 entries.push_back(absl::StrCat(absl::StrFormat("\"%s\": {\n ", name),
169 absl::StrJoin(fields, ",\n"), "\n}"));
170 }
171 return absl::StrCat("\"authorities\": {\n", absl::StrJoin(entries, ",\n"),
172 "\n}");
173 }
174
175 //
176 // XdsResourceUtils::ClientHcmAccessor
177 //
178
Unpack(const Listener & listener) const179 HttpConnectionManager XdsResourceUtils::ClientHcmAccessor::Unpack(
180 const Listener& listener) const {
181 HttpConnectionManager http_connection_manager;
182 listener.api_listener().api_listener().UnpackTo(&http_connection_manager);
183 return http_connection_manager;
184 }
185
Pack(const HttpConnectionManager & hcm,Listener * listener) const186 void XdsResourceUtils::ClientHcmAccessor::Pack(const HttpConnectionManager& hcm,
187 Listener* listener) const {
188 auto* api_listener = listener->mutable_api_listener()->mutable_api_listener();
189 api_listener->PackFrom(hcm);
190 }
191
192 //
193 // XdsResourceUtils::ServerHcmAccessor
194 //
195
Unpack(const Listener & listener) const196 HttpConnectionManager XdsResourceUtils::ServerHcmAccessor::Unpack(
197 const Listener& listener) const {
198 HttpConnectionManager http_connection_manager;
199 listener.default_filter_chain().filters().at(0).typed_config().UnpackTo(
200 &http_connection_manager);
201 return http_connection_manager;
202 }
203
Pack(const HttpConnectionManager & hcm,Listener * listener) const204 void XdsResourceUtils::ServerHcmAccessor::Pack(const HttpConnectionManager& hcm,
205 Listener* listener) const {
206 auto* filters = listener->mutable_default_filter_chain()->mutable_filters();
207 if (filters->empty()) filters->Add();
208 filters->at(0).mutable_typed_config()->PackFrom(hcm);
209 }
210
211 //
212 // XdsResourceUtils
213 //
214
215 const char XdsResourceUtils::kDefaultLocalityRegion[] =
216 "xds_default_locality_region";
217 const char XdsResourceUtils::kDefaultLocalityZone[] =
218 "xds_default_locality_zone";
219
220 const char XdsResourceUtils::kServerName[] = "server.example.com";
221 const char XdsResourceUtils::kDefaultRouteConfigurationName[] =
222 "route_config_name";
223 const char XdsResourceUtils::kDefaultClusterName[] = "cluster_name";
224 const char XdsResourceUtils::kDefaultEdsServiceName[] = "eds_service_name";
225 const char XdsResourceUtils::kDefaultServerRouteConfigurationName[] =
226 "default_server_route_config_name";
227
DefaultListener()228 Listener XdsResourceUtils::DefaultListener() {
229 Listener listener;
230 listener.set_name(kServerName);
231 ClientHcmAccessor().Pack(DefaultHcm(), &listener);
232 return listener;
233 }
234
DefaultRouteConfig()235 RouteConfiguration XdsResourceUtils::DefaultRouteConfig() {
236 RouteConfiguration route_config;
237 route_config.set_name(kDefaultRouteConfigurationName);
238 auto* virtual_host = route_config.add_virtual_hosts();
239 virtual_host->add_domains("*");
240 auto* route = virtual_host->add_routes();
241 route->mutable_match()->set_prefix("");
242 route->mutable_route()->set_cluster(kDefaultClusterName);
243 return route_config;
244 }
245
DefaultCluster()246 Cluster XdsResourceUtils::DefaultCluster() {
247 Cluster cluster;
248 cluster.set_name(kDefaultClusterName);
249 cluster.set_type(Cluster::EDS);
250 auto* eds_config = cluster.mutable_eds_cluster_config();
251 eds_config->mutable_eds_config()->mutable_self();
252 eds_config->set_service_name(kDefaultEdsServiceName);
253 cluster.set_lb_policy(Cluster::ROUND_ROBIN);
254 return cluster;
255 }
256
DefaultServerListener()257 Listener XdsResourceUtils::DefaultServerListener() {
258 Listener listener;
259 listener.mutable_address()->mutable_socket_address()->set_address(
260 grpc_core::LocalIp());
261 ServerHcmAccessor().Pack(DefaultHcm(), &listener);
262 return listener;
263 }
264
DefaultServerRouteConfig()265 RouteConfiguration XdsResourceUtils::DefaultServerRouteConfig() {
266 RouteConfiguration route_config;
267 route_config.set_name(kDefaultServerRouteConfigurationName);
268 auto* virtual_host = route_config.add_virtual_hosts();
269 virtual_host->add_domains("*");
270 auto* route = virtual_host->add_routes();
271 route->mutable_match()->set_prefix("");
272 route->mutable_non_forwarding_action();
273 return route_config;
274 }
275
DefaultHcm()276 HttpConnectionManager XdsResourceUtils::DefaultHcm() {
277 HttpConnectionManager http_connection_manager;
278 auto* filter = http_connection_manager.add_http_filters();
279 filter->set_name("router");
280 filter->mutable_typed_config()->PackFrom(
281 envoy::extensions::filters::http::router::v3::Router());
282 return http_connection_manager;
283 }
284
GetServerListenerName(int port)285 std::string XdsResourceUtils::GetServerListenerName(int port) {
286 return absl::StrCat("grpc/server?xds.resource.listening_address=",
287 grpc_core::LocalIp(), ":", port);
288 }
289
PopulateServerListenerNameAndPort(const Listener & listener_template,int port)290 Listener XdsResourceUtils::PopulateServerListenerNameAndPort(
291 const Listener& listener_template, int port) {
292 Listener listener = listener_template;
293 listener.set_name(GetServerListenerName(port));
294 listener.mutable_address()->mutable_socket_address()->set_port_value(port);
295 return listener;
296 }
297
SetListenerAndRouteConfiguration(AdsServiceImpl * ads_service,Listener listener,const RouteConfiguration & route_config,bool use_rds,const HcmAccessor & hcm_accessor)298 void XdsResourceUtils::SetListenerAndRouteConfiguration(
299 AdsServiceImpl* ads_service, Listener listener,
300 const RouteConfiguration& route_config, bool use_rds,
301 const HcmAccessor& hcm_accessor) {
302 HttpConnectionManager http_connection_manager = hcm_accessor.Unpack(listener);
303 if (use_rds) {
304 auto* rds = http_connection_manager.mutable_rds();
305 rds->set_route_config_name(route_config.name());
306 rds->mutable_config_source()->mutable_self();
307 ads_service->SetRdsResource(route_config);
308 } else {
309 *http_connection_manager.mutable_route_config() = route_config;
310 }
311 hcm_accessor.Pack(http_connection_manager, &listener);
312 ads_service->SetLdsResource(listener);
313 }
314
SetRouteConfiguration(AdsServiceImpl * ads_service,const RouteConfiguration & route_config,bool use_rds,const Listener * listener_to_copy)315 void XdsResourceUtils::SetRouteConfiguration(
316 AdsServiceImpl* ads_service, const RouteConfiguration& route_config,
317 bool use_rds, const Listener* listener_to_copy) {
318 if (use_rds) {
319 ads_service->SetRdsResource(route_config);
320 } else {
321 Listener listener(listener_to_copy == nullptr ? DefaultListener()
322 : *listener_to_copy);
323 HttpConnectionManager http_connection_manager =
324 ClientHcmAccessor().Unpack(listener);
325 *(http_connection_manager.mutable_route_config()) = route_config;
326 ClientHcmAccessor().Pack(http_connection_manager, &listener);
327 ads_service->SetLdsResource(listener);
328 }
329 }
330
LocalityNameString(absl::string_view sub_zone)331 std::string XdsResourceUtils::LocalityNameString(absl::string_view sub_zone) {
332 return absl::StrFormat("{region=\"%s\", zone=\"%s\", sub_zone=\"%s\"}",
333 kDefaultLocalityRegion, kDefaultLocalityZone,
334 sub_zone);
335 }
336
BuildEdsResource(const EdsResourceArgs & args,absl::string_view eds_service_name)337 ClusterLoadAssignment XdsResourceUtils::BuildEdsResource(
338 const EdsResourceArgs& args, absl::string_view eds_service_name) {
339 ClusterLoadAssignment assignment;
340 assignment.set_cluster_name(eds_service_name);
341 for (const auto& locality : args.locality_list) {
342 auto* endpoints = assignment.add_endpoints();
343 endpoints->mutable_load_balancing_weight()->set_value(locality.lb_weight);
344 endpoints->set_priority(locality.priority);
345 endpoints->mutable_locality()->set_region(kDefaultLocalityRegion);
346 endpoints->mutable_locality()->set_zone(kDefaultLocalityZone);
347 endpoints->mutable_locality()->set_sub_zone(locality.sub_zone);
348 for (size_t i = 0; i < locality.endpoints.size(); ++i) {
349 const auto& endpoint = locality.endpoints[i];
350 auto* lb_endpoints = endpoints->add_lb_endpoints();
351 if (locality.endpoints.size() > i &&
352 locality.endpoints[i].health_status != HealthStatus::UNKNOWN) {
353 lb_endpoints->set_health_status(endpoint.health_status);
354 }
355 if (locality.endpoints.size() > i && endpoint.lb_weight >= 1) {
356 lb_endpoints->mutable_load_balancing_weight()->set_value(
357 endpoint.lb_weight);
358 }
359 auto* endpoint_proto = lb_endpoints->mutable_endpoint();
360 auto* socket_address =
361 endpoint_proto->mutable_address()->mutable_socket_address();
362 socket_address->set_address(grpc_core::LocalIp());
363 socket_address->set_port_value(endpoint.port);
364 for (int port : endpoint.additional_ports) {
365 socket_address = endpoint_proto->add_additional_addresses()
366 ->mutable_address()
367 ->mutable_socket_address();
368 socket_address->set_address(grpc_core::LocalIp());
369 socket_address->set_port_value(port);
370 }
371 if (!endpoint.hostname.empty()) {
372 endpoint_proto->set_hostname(endpoint.hostname);
373 }
374 if (!endpoint.metadata.empty()) {
375 auto& filter_map =
376 *lb_endpoints->mutable_metadata()->mutable_filter_metadata();
377 for (const auto& p : endpoint.metadata) {
378 absl::Status status = grpc::protobuf::json::JsonStringToMessage(
379 p.second, &filter_map[p.first],
380 grpc::protobuf::json::JsonParseOptions());
381 CHECK(status.ok()) << status;
382 }
383 }
384 }
385 }
386 if (!args.drop_categories.empty()) {
387 auto* policy = assignment.mutable_policy();
388 for (const auto& p : args.drop_categories) {
389 const std::string& name = p.first;
390 const uint32_t parts_per_million = p.second;
391 auto* drop_overload = policy->add_drop_overloads();
392 drop_overload->set_category(name);
393 auto* drop_percentage = drop_overload->mutable_drop_percentage();
394 drop_percentage->set_numerator(parts_per_million);
395 drop_percentage->set_denominator(args.drop_denominator);
396 }
397 }
398 return assignment;
399 }
400
401 } // namespace testing
402 } // namespace grpc
403