test/cpp/end2end/xds_end2end_test.cc

   1 /*
   2  *
   3  * Copyright 2017 gRPC authors.
   4  *
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  *
  17  */
  18
  19 #include <deque>
  20 #include <memory>
  21 #include <mutex>
  22 #include <numeric>
  23 #include <set>
  24 #include <sstream>
  25 #include <string>
  26 #include <thread>
  27 #include <vector>
  28
  29 #include <gmock/gmock.h>
  30 #include <gtest/gtest.h>
  31
  32 #include "absl/functional/bind_front.h"
  33 #include "absl/memory/memory.h"
  34 #include "absl/strings/match.h"
  35 #include "absl/strings/str_cat.h"
  36 #include "absl/strings/str_format.h"
  37 #include "absl/strings/str_join.h"
  38 #include "absl/types/optional.h"
  39
  40 #include <grpc/grpc.h>
  41 #include <grpc/grpc_security.h>
  42 #include <grpc/support/alloc.h>
  43 #include <grpc/support/log.h>
  44 #include <grpc/support/time.h>
  45 #include <grpcpp/channel.h>
  46 #include <grpcpp/client_context.h>
  47 #include <grpcpp/create_channel.h>
  48 #include <grpcpp/security/tls_certificate_provider.h>
  49 #include <grpcpp/server.h>
  50 #include <grpcpp/server_builder.h>
  51 #include <grpcpp/xds_server_builder.h>
  52
  53 #include "src/core/ext/filters/client_channel/backup_poller.h"
  54 #include "src/core/ext/filters/client_channel/lb_policy/xds/xds_channel_args.h"
  55 #include "src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h"
  56 #include "src/core/ext/filters/client_channel/server_address.h"
  57 #include "src/core/ext/xds/certificate_provider_registry.h"
  58 #include "src/core/ext/xds/xds_api.h"
  59 #include "src/core/ext/xds/xds_channel_args.h"
  60 #include "src/core/ext/xds/xds_client.h"
  61 #include "src/core/lib/address_utils/parse_address.h"
  62 #include "src/core/lib/channel/channel_args.h"
  63 #include "src/core/lib/gpr/env.h"
  64 #include "src/core/lib/gpr/string.h"
  65 #include "src/core/lib/gpr/time_precise.h"
  66 #include "src/core/lib/gpr/tmpfile.h"
  67 #include "src/core/lib/gprpp/ref_counted_ptr.h"
  68 #include "src/core/lib/gprpp/sync.h"
  69 #include "src/core/lib/gprpp/time_util.h"
  70 #include "src/core/lib/iomgr/load_file.h"
  71 #include "src/core/lib/iomgr/sockaddr.h"
  72 #include "src/core/lib/security/credentials/fake/fake_credentials.h"
  73 #include "src/cpp/client/secure_credentials.h"
  74 #include "src/cpp/server/secure_server_credentials.h"
  75 #include "src/proto/grpc/testing/echo.grpc.pb.h"
  76 #include "src/proto/grpc/testing/xds/ads_for_test.grpc.pb.h"
  77 #include "src/proto/grpc/testing/xds/cds_for_test.grpc.pb.h"
  78 #include "src/proto/grpc/testing/xds/eds_for_test.grpc.pb.h"
  79 #include "src/proto/grpc/testing/xds/lds_rds_for_test.grpc.pb.h"
  80 #include "src/proto/grpc/testing/xds/lrs_for_test.grpc.pb.h"
  81 #include "src/proto/grpc/testing/xds/v3/ads.grpc.pb.h"
  82 #include "src/proto/grpc/testing/xds/v3/aggregate_cluster.grpc.pb.h"
  83 #include "src/proto/grpc/testing/xds/v3/cluster.grpc.pb.h"
  84 #include "src/proto/grpc/testing/xds/v3/discovery.grpc.pb.h"
  85 #include "src/proto/grpc/testing/xds/v3/endpoint.grpc.pb.h"
  86 #include "src/proto/grpc/testing/xds/v3/fault.grpc.pb.h"
  87 #include "src/proto/grpc/testing/xds/v3/http_connection_manager.grpc.pb.h"
  88 #include "src/proto/grpc/testing/xds/v3/listener.grpc.pb.h"
  89 #include "src/proto/grpc/testing/xds/v3/lrs.grpc.pb.h"
  90 #include "src/proto/grpc/testing/xds/v3/route.grpc.pb.h"
  91 #include "src/proto/grpc/testing/xds/v3/router.grpc.pb.h"
  92 #include "src/proto/grpc/testing/xds/v3/tls.grpc.pb.h"
  93 #include "test/core/util/port.h"
  94 #include "test/core/util/resolve_localhost_ip46.h"
  95 #include "test/core/util/test_config.h"
  96 #include "test/cpp/end2end/test_service_impl.h"
  97
  98 #ifndef DISABLED_XDS_PROTO_IN_CC
  99 #include "src/cpp/server/csds/csds.h"
 100 #include "src/proto/grpc/testing/xds/v3/csds.grpc.pb.h"
 101 #endif  // DISABLED_XDS_PROTO_IN_CC
 102
 103 namespace grpc {
 104 namespace testing {
 105 namespace {
 106
 107 using std::chrono::system_clock;
 108
 109 #ifndef DISABLED_XDS_PROTO_IN_CC
 110 using ::envoy::admin::v3::ClientResourceStatus;
 111 #endif  // DISABLED_XDS_PROTO_IN_CC
 112 using ::envoy::config::cluster::v3::CircuitBreakers;
 113 using ::envoy::config::cluster::v3::Cluster;
 114 using ::envoy::config::cluster::v3::CustomClusterType;
 115 using ::envoy::config::cluster::v3::RoutingPriority;
 116 using ::envoy::config::endpoint::v3::ClusterLoadAssignment;
 117 using ::envoy::config::endpoint::v3::HealthStatus;
 118 using ::envoy::config::listener::v3::FilterChainMatch;
 119 using ::envoy::config::listener::v3::Listener;
 120 using ::envoy::config::route::v3::RouteConfiguration;
 121 using ::envoy::extensions::clusters::aggregate::v3::ClusterConfig;
 122 using ::envoy::extensions::filters::http::fault::v3::HTTPFault;
 123 using ::envoy::extensions::filters::network::http_connection_manager::v3::
 124     HttpConnectionManager;
 125 using ::envoy::extensions::filters::network::http_connection_manager::v3::
 126     HttpFilter;
 127 using ::envoy::extensions::transport_sockets::tls::v3::DownstreamTlsContext;
 128 using ::envoy::extensions::transport_sockets::tls::v3::UpstreamTlsContext;
 129 using ::envoy::type::matcher::v3::StringMatcher;
 130 using ::envoy::type::v3::FractionalPercent;
 131
 132 constexpr char kLdsTypeUrl[] =
 133     "type.googleapis.com/envoy.config.listener.v3.Listener";
 134 constexpr char kRdsTypeUrl[] =
 135     "type.googleapis.com/envoy.config.route.v3.RouteConfiguration";
 136 constexpr char kCdsTypeUrl[] =
 137     "type.googleapis.com/envoy.config.cluster.v3.Cluster";
 138 constexpr char kEdsTypeUrl[] =
 139     "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment";
 140
 141 constexpr char kLdsV2TypeUrl[] = "type.googleapis.com/envoy.api.v2.Listener";
 142 constexpr char kRdsV2TypeUrl[] =
 143     "type.googleapis.com/envoy.api.v2.RouteConfiguration";
 144 constexpr char kCdsV2TypeUrl[] = "type.googleapis.com/envoy.api.v2.Cluster";
 145 constexpr char kEdsV2TypeUrl[] =
 146     "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment";
 147
 148 constexpr char kDefaultLocalityRegion[] = "xds_default_locality_region";
 149 constexpr char kDefaultLocalityZone[] = "xds_default_locality_zone";
 150 constexpr char kLbDropType[] = "lb";
 151 constexpr char kThrottleDropType[] = "throttle";
 152 constexpr char kServerName[] = "server.example.com";
 153 constexpr char kDefaultRouteConfigurationName[] = "route_config_name";
 154 constexpr char kDefaultClusterName[] = "cluster_name";
 155 constexpr char kDefaultEdsServiceName[] = "eds_service_name";
 156 constexpr int kDefaultLocalityWeight = 3;
 157 constexpr int kDefaultLocalityPriority = 0;
 158
 159 constexpr char kRequestMessage[] = "Live long and prosper.";
 160 constexpr char kDefaultServiceConfig[] =
 161     "{\n"
 162     "  \"loadBalancingConfig\":[\n"
 163     "    { \"does_not_exist\":{} },\n"
 164     "    { \"xds_cluster_resolver_experimental\":{\n"
 165     "      \"discoveryMechanisms\": [\n"
 166     "      { \"clusterName\": \"server.example.com\",\n"
 167     "        \"type\": \"EDS\",\n"
 168     "        \"lrsLoadReportingServerName\": \"\"\n"
 169     "      } ]\n"
 170     "    } }\n"
 171     "  ]\n"
 172     "}";
 173 constexpr char kDefaultServiceConfigWithoutLoadReporting[] =
 174     "{\n"
 175     "  \"loadBalancingConfig\":[\n"
 176     "    { \"does_not_exist\":{} },\n"
 177     "    { \"xds_cluster_resolver_experimental\":{\n"
 178     "      \"discoveryMechanisms\": [\n"
 179     "      { \"clusterName\": \"server.example.com\",\n"
 180     "        \"type\": \"EDS\"\n"
 181     "      } ]\n"
 182     "    } }\n"
 183     "  ]\n"
 184     "}";
 185
 186 constexpr char kBootstrapFileV3[] =
 187     "{\n"
 188     "  \"xds_servers\": [\n"
 189     "    {\n"
 190     "      \"server_uri\": \"fake:///xds_server\",\n"
 191     "      \"channel_creds\": [\n"
 192     "        {\n"
 193     "          \"type\": \"fake\"\n"
 194     "        }\n"
 195     "      ],\n"
 196     "      \"server_features\": [\"xds_v3\"]\n"
 197     "    }\n"
 198     "  ],\n"
 199     "  \"node\": {\n"
 200     "    \"id\": \"xds_end2end_test\",\n"
 201     "    \"cluster\": \"test\",\n"
 202     "    \"metadata\": {\n"
 203     "      \"foo\": \"bar\"\n"
 204     "    },\n"
 205     "    \"locality\": {\n"
 206     "      \"region\": \"corp\",\n"
 207     "      \"zone\": \"svl\",\n"
 208     "      \"sub_zone\": \"mp3\"\n"
 209     "    }\n"
 210     "  },\n"
 211     "  \"server_listener_resource_name_template\": "
 212     "\"grpc/server?xds.resource.listening_address=%s\",\n"
 213     "  \"certificate_providers\": {\n"
 214     "    \"fake_plugin1\": {\n"
 215     "      \"plugin_name\": \"fake1\"\n"
 216     "    },\n"
 217     "    \"fake_plugin2\": {\n"
 218     "      \"plugin_name\": \"fake2\"\n"
 219     "    },\n"
 220     "    \"file_plugin\": {\n"
 221     "      \"plugin_name\": \"file_watcher\",\n"
 222     "      \"config\": {\n"
 223     "        \"certificate_file\": \"src/core/tsi/test_creds/client.pem\",\n"
 224     "        \"private_key_file\": \"src/core/tsi/test_creds/client.key\",\n"
 225     "        \"ca_certificate_file\": \"src/core/tsi/test_creds/ca.pem\"\n"
 226     "      }"
 227     "    }\n"
 228     "  }\n"
 229     "}\n";
 230
 231 constexpr char kBootstrapFileV2[] =
 232     "{\n"
 233     "  \"xds_servers\": [\n"
 234     "    {\n"
 235     "      \"server_uri\": \"fake:///xds_server\",\n"
 236     "      \"channel_creds\": [\n"
 237     "        {\n"
 238     "          \"type\": \"fake\"\n"
 239     "        }\n"
 240     "      ]\n"
 241     "    }\n"
 242     "  ],\n"
 243     "  \"node\": {\n"
 244     "    \"id\": \"xds_end2end_test\",\n"
 245     "    \"cluster\": \"test\",\n"
 246     "    \"metadata\": {\n"
 247     "      \"foo\": \"bar\"\n"
 248     "    },\n"
 249     "    \"locality\": {\n"
 250     "      \"region\": \"corp\",\n"
 251     "      \"zone\": \"svl\",\n"
 252     "      \"sub_zone\": \"mp3\"\n"
 253     "    }\n"
 254     "  }\n"
 255     "}\n";
 256 constexpr char kCaCertPath[] = "src/core/tsi/test_creds/ca.pem";
 257 constexpr char kServerCertPath[] = "src/core/tsi/test_creds/server1.pem";
 258 constexpr char kServerKeyPath[] = "src/core/tsi/test_creds/server1.key";
 259 constexpr char kClientCertPath[] = "src/core/tsi/test_creds/client.pem";
 260 constexpr char kClientKeyPath[] = "src/core/tsi/test_creds/client.key";
 261 constexpr char kBadClientCertPath[] = "src/core/tsi/test_creds/badclient.pem";
 262 constexpr char kBadClientKeyPath[] = "src/core/tsi/test_creds/badclient.key";
 263
 264 char* g_bootstrap_file_v3;
 265 char* g_bootstrap_file_v2;
 266
 267 void WriteBootstrapFiles() {
 268   char* bootstrap_file;
 269   FILE* out = gpr_tmpfile("xds_bootstrap_v3", &bootstrap_file);
 270   fputs(kBootstrapFileV3, out);
 271   fclose(out);
 272   g_bootstrap_file_v3 = bootstrap_file;
 273   out = gpr_tmpfile("xds_bootstrap_v2", &bootstrap_file);
 274   fputs(kBootstrapFileV2, out);
 275   fclose(out);
 276   g_bootstrap_file_v2 = bootstrap_file;
 277 }
 278
 279 template <typename ServiceType>
 280 class CountedService : public ServiceType {
 281  public:
 282   size_t request_count() {
 283     grpc_core::MutexLock lock(&mu_);
 284     return request_count_;
 285   }
 286
 287   size_t response_count() {
 288     grpc_core::MutexLock lock(&mu_);
 289     return response_count_;
 290   }
 291
 292   void IncreaseResponseCount() {
 293     grpc_core::MutexLock lock(&mu_);
 294     ++response_count_;
 295   }
 296   void IncreaseRequestCount() {
 297     grpc_core::MutexLock lock(&mu_);
 298     ++request_count_;
 299   }
 300
 301   void ResetCounters() {
 302     grpc_core::MutexLock lock(&mu_);
 303     request_count_ = 0;
 304     response_count_ = 0;
 305   }
 306
 307  private:
 308   grpc_core::Mutex mu_;
 309   size_t request_count_ = 0;
 310   size_t response_count_ = 0;
 311 };
 312
 313 template <typename RpcService>
 314 class BackendServiceImpl
 315     : public CountedService<TestMultipleServiceImpl<RpcService>> {
 316  public:
 317   BackendServiceImpl() {}
 318
 319   Status Echo(ServerContext* context, const EchoRequest* request,
 320               EchoResponse* response) override {
 321     auto peer_identity = context->auth_context()->GetPeerIdentity();
 322     CountedService<TestMultipleServiceImpl<RpcService>>::IncreaseRequestCount();
 323     const auto status =
 324         TestMultipleServiceImpl<RpcService>::Echo(context, request, response);
 325     CountedService<
 326         TestMultipleServiceImpl<RpcService>>::IncreaseResponseCount();
 327     {
 328       grpc_core::MutexLock lock(&mu_);
 329       clients_.insert(context->peer());
 330       last_peer_identity_.clear();
 331       for (const auto& entry : peer_identity) {
 332         last_peer_identity_.emplace_back(entry.data(), entry.size());
 333       }
 334     }
 335     return status;
 336   }
 337
 338   Status Echo1(ServerContext* context, const EchoRequest* request,
 339                EchoResponse* response) override {
 340     return Echo(context, request, response);
 341   }
 342
 343   Status Echo2(ServerContext* context, const EchoRequest* request,
 344                EchoResponse* response) override {
 345     return Echo(context, request, response);
 346   }
 347
 348   void Start() {}
 349   void Shutdown() {}
 350
 351   std::set<std::string> clients() {
 352     grpc_core::MutexLock lock(&mu_);
 353     return clients_;
 354   }
 355
 356   const std::vector<std::string>& last_peer_identity() {
 357     grpc_core::MutexLock lock(&mu_);
 358     return last_peer_identity_;
 359   }
 360
 361  private:
 362   grpc_core::Mutex mu_;
 363   std::set<std::string> clients_;
 364   std::vector<std::string> last_peer_identity_;
 365 };
 366
 367 class ClientStats {
 368  public:
 369   struct LocalityStats {
 370     LocalityStats() {}
 371
 372     // Converts from proto message class.
 373     template <class UpstreamLocalityStats>
 374     explicit LocalityStats(const UpstreamLocalityStats& upstream_locality_stats)
 375         : total_successful_requests(
 376               upstream_locality_stats.total_successful_requests()),
 377           total_requests_in_progress(
 378               upstream_locality_stats.total_requests_in_progress()),
 379           total_error_requests(upstream_locality_stats.total_error_requests()),
 380           total_issued_requests(
 381               upstream_locality_stats.total_issued_requests()) {}
 382
 383     LocalityStats& operator+=(const LocalityStats& other) {
 384       total_successful_requests += other.total_successful_requests;
 385       total_requests_in_progress += other.total_requests_in_progress;
 386       total_error_requests += other.total_error_requests;
 387       total_issued_requests += other.total_issued_requests;
 388       return *this;
 389     }
 390
 391     uint64_t total_successful_requests = 0;
 392     uint64_t total_requests_in_progress = 0;
 393     uint64_t total_error_requests = 0;
 394     uint64_t total_issued_requests = 0;
 395   };
 396
 397   ClientStats() {}
 398
 399   // Converts from proto message class.
 400   template <class ClusterStats>
 401   explicit ClientStats(const ClusterStats& cluster_stats)
 402       : cluster_name_(cluster_stats.cluster_name()),
 403         total_dropped_requests_(cluster_stats.total_dropped_requests()) {
 404     for (const auto& input_locality_stats :
 405          cluster_stats.upstream_locality_stats()) {
 406       locality_stats_.emplace(input_locality_stats.locality().sub_zone(),
 407                               LocalityStats(input_locality_stats));
 408     }
 409     for (const auto& input_dropped_requests :
 410          cluster_stats.dropped_requests()) {
 411       dropped_requests_.emplace(input_dropped_requests.category(),
 412                                 input_dropped_requests.dropped_count());
 413     }
 414   }
 415
 416   const std::string& cluster_name() const { return cluster_name_; }
 417
 418   const std::map<std::string, LocalityStats>& locality_stats() const {
 419     return locality_stats_;
 420   }
 421   uint64_t total_successful_requests() const {
 422     uint64_t sum = 0;
 423     for (auto& p : locality_stats_) {
 424       sum += p.second.total_successful_requests;
 425     }
 426     return sum;
 427   }
 428   uint64_t total_requests_in_progress() const {
 429     uint64_t sum = 0;
 430     for (auto& p : locality_stats_) {
 431       sum += p.second.total_requests_in_progress;
 432     }
 433     return sum;
 434   }
 435   uint64_t total_error_requests() const {
 436     uint64_t sum = 0;
 437     for (auto& p : locality_stats_) {
 438       sum += p.second.total_error_requests;
 439     }
 440     return sum;
 441   }
 442   uint64_t total_issued_requests() const {
 443     uint64_t sum = 0;
 444     for (auto& p : locality_stats_) {
 445       sum += p.second.total_issued_requests;
 446     }
 447     return sum;
 448   }
 449
 450   uint64_t total_dropped_requests() const { return total_dropped_requests_; }
 451
 452   uint64_t dropped_requests(const std::string& category) const {
 453     auto iter = dropped_requests_.find(category);
 454     GPR_ASSERT(iter != dropped_requests_.end());
 455     return iter->second;
 456   }
 457
 458   ClientStats& operator+=(const ClientStats& other) {
 459     for (const auto& p : other.locality_stats_) {
 460       locality_stats_[p.first] += p.second;
 461     }
 462     total_dropped_requests_ += other.total_dropped_requests_;
 463     for (const auto& p : other.dropped_requests_) {
 464       dropped_requests_[p.first] += p.second;
 465     }
 466     return *this;
 467   }
 468
 469  private:
 470   std::string cluster_name_;
 471   std::map<std::string, LocalityStats> locality_stats_;
 472   uint64_t total_dropped_requests_ = 0;
 473   std::map<std::string, uint64_t> dropped_requests_;
 474 };
 475
 476 class AdsServiceImpl : public std::enable_shared_from_this<AdsServiceImpl> {
 477  public:
 478   struct ResponseState {
 479     enum State { NOT_SENT, SENT, ACKED, NACKED };
 480     State state = NOT_SENT;
 481     std::string error_message;
 482   };
 483
 484   struct EdsResourceArgs {
 485     struct Endpoint {
 486       explicit Endpoint(int port,
 487                         HealthStatus health_status = HealthStatus::UNKNOWN,
 488                         int lb_weight = 1)
 489           : port(port), health_status(health_status), lb_weight(lb_weight) {}
 490
 491       int port;
 492       HealthStatus health_status;
 493       int lb_weight;
 494     };
 495
 496     struct Locality {
 497       Locality(std::string sub_zone, std::vector<Endpoint> endpoints,
 498                int lb_weight = kDefaultLocalityWeight,
 499                int priority = kDefaultLocalityPriority)
 500           : sub_zone(std::move(sub_zone)),
 501             endpoints(std::move(endpoints)),
 502             lb_weight(lb_weight),
 503             priority(priority) {}
 504
 505       const std::string sub_zone;
 506       std::vector<Endpoint> endpoints;
 507       int lb_weight;
 508       int priority;
 509     };
 510
 511     EdsResourceArgs() = default;
 512     explicit EdsResourceArgs(std::vector<Locality> locality_list)
 513         : locality_list(std::move(locality_list)) {}
 514
 515     std::vector<Locality> locality_list;
 516     std::map<std::string, uint32_t> drop_categories;
 517     FractionalPercent::DenominatorType drop_denominator =
 518         FractionalPercent::MILLION;
 519   };
 520
 521   AdsServiceImpl()
 522       : v2_rpc_service_(this, /*is_v2=*/true),
 523         v3_rpc_service_(this, /*is_v2=*/false) {}
 524
 525   bool seen_v2_client() const { return seen_v2_client_; }
 526   bool seen_v3_client() const { return seen_v3_client_; }
 527
 528   ::envoy::service::discovery::v2::AggregatedDiscoveryService::Service*
 529   v2_rpc_service() {
 530     return &v2_rpc_service_;
 531   }
 532
 533   ::envoy::service::discovery::v3::AggregatedDiscoveryService::Service*
 534   v3_rpc_service() {
 535     return &v3_rpc_service_;
 536   }
 537
 538   ResponseState lds_response_state() {
 539     grpc_core::MutexLock lock(&ads_mu_);
 540     return resource_type_response_state_[kLdsTypeUrl];
 541   }
 542
 543   ResponseState rds_response_state() {
 544     grpc_core::MutexLock lock(&ads_mu_);
 545     return resource_type_response_state_[kRdsTypeUrl];
 546   }
 547
 548   ResponseState cds_response_state() {
 549     grpc_core::MutexLock lock(&ads_mu_);
 550     return resource_type_response_state_[kCdsTypeUrl];
 551   }
 552
 553   ResponseState eds_response_state() {
 554     grpc_core::MutexLock lock(&ads_mu_);
 555     return resource_type_response_state_[kEdsTypeUrl];
 556   }
 557
 558   void SetResourceIgnore(const std::string& type_url) {
 559     grpc_core::MutexLock lock(&ads_mu_);
 560     resource_types_to_ignore_.emplace(type_url);
 561   }
 562
 563   void SetResourceMinVersion(const std::string& type_url, int version) {
 564     grpc_core::MutexLock lock(&ads_mu_);
 565     resource_type_min_versions_[type_url] = version;
 566   }
 567
 568   void UnsetResource(const std::string& type_url, const std::string& name) {
 569     grpc_core::MutexLock lock(&ads_mu_);
 570     ResourceTypeState& resource_type_state = resource_map_[type_url];
 571     ++resource_type_state.resource_type_version;
 572     ResourceState& resource_state = resource_type_state.resource_name_map[name];
 573     resource_state.resource_type_version =
 574         resource_type_state.resource_type_version;
 575     resource_state.resource.reset();
 576     gpr_log(GPR_INFO,
 577             "ADS[%p]: Unsetting %s resource %s; resource_type_version now %u",
 578             this, type_url.c_str(), name.c_str(),
 579             resource_type_state.resource_type_version);
 580     for (SubscriptionState* subscription : resource_state.subscriptions) {
 581       subscription->update_queue->emplace_back(type_url, name);
 582     }
 583   }
 584
 585   void SetResource(google::protobuf::Any resource, const std::string& type_url,
 586                    const std::string& name) {
 587     grpc_core::MutexLock lock(&ads_mu_);
 588     ResourceTypeState& resource_type_state = resource_map_[type_url];
 589     ++resource_type_state.resource_type_version;
 590     ResourceState& resource_state = resource_type_state.resource_name_map[name];
 591     resource_state.resource_type_version =
 592         resource_type_state.resource_type_version;
 593     resource_state.resource = std::move(resource);
 594     gpr_log(GPR_INFO,
 595             "ADS[%p]: Updating %s resource %s; resource_type_version now %u",
 596             this, type_url.c_str(), name.c_str(),
 597             resource_type_state.resource_type_version);
 598     for (SubscriptionState* subscription : resource_state.subscriptions) {
 599       subscription->update_queue->emplace_back(type_url, name);
 600     }
 601   }
 602
 603   void SetLdsResource(const Listener& listener) {
 604     google::protobuf::Any resource;
 605     resource.PackFrom(listener);
 606     SetResource(std::move(resource), kLdsTypeUrl, listener.name());
 607   }
 608
 609   void SetRdsResource(const RouteConfiguration& route) {
 610     google::protobuf::Any resource;
 611     resource.PackFrom(route);
 612     SetResource(std::move(resource), kRdsTypeUrl, route.name());
 613   }
 614
 615   void SetCdsResource(const Cluster& cluster) {
 616     google::protobuf::Any resource;
 617     resource.PackFrom(cluster);
 618     SetResource(std::move(resource), kCdsTypeUrl, cluster.name());
 619   }
 620
 621   void SetEdsResource(const ClusterLoadAssignment& assignment) {
 622     google::protobuf::Any resource;
 623     resource.PackFrom(assignment);
 624     SetResource(std::move(resource), kEdsTypeUrl, assignment.cluster_name());
 625   }
 626
 627   void Start() {
 628     grpc_core::MutexLock lock(&ads_mu_);
 629     ads_done_ = false;
 630   }
 631
 632   void Shutdown() {
 633     {
 634       grpc_core::MutexLock lock(&ads_mu_);
 635       NotifyDoneWithAdsCallLocked();
 636       resource_type_response_state_.clear();
 637     }
 638     gpr_log(GPR_INFO, "ADS[%p]: shut down", this);
 639   }
 640
 641   void NotifyDoneWithAdsCall() {
 642     grpc_core::MutexLock lock(&ads_mu_);
 643     NotifyDoneWithAdsCallLocked();
 644   }
 645
 646   void NotifyDoneWithAdsCallLocked() {
 647     if (!ads_done_) {
 648       ads_done_ = true;
 649       ads_cond_.SignalAll();
 650     }
 651   }
 652
 653   std::set<std::string> clients() {
 654     grpc_core::MutexLock lock(&clients_mu_);
 655     return clients_;
 656   }
 657
 658  private:
 659   // A queue of resource type/name pairs that have changed since the client
 660   // subscribed to them.
 661   using UpdateQueue = std::deque<
 662       std::pair<std::string /* type url */, std::string /* resource name */>>;
 663
 664   // A struct representing a client's subscription to a particular resource.
 665   struct SubscriptionState {
 666     // The queue upon which to place updates when the resource is updated.
 667     UpdateQueue* update_queue;
 668   };
 669
 670   // A struct representing the a client's subscription to all the resources.
 671   using SubscriptionNameMap =
 672       std::map<std::string /* resource_name */, SubscriptionState>;
 673   using SubscriptionMap =
 674       std::map<std::string /* type_url */, SubscriptionNameMap>;
 675
 676   // Sent state for a given resource type.
 677   struct SentState {
 678     int nonce = 0;
 679     int resource_type_version = 0;
 680   };
 681
 682   // A struct representing the current state for an individual resource.
 683   struct ResourceState {
 684     // The resource itself, if present.
 685     absl::optional<google::protobuf::Any> resource;
 686     // The resource type version that this resource was last updated in.
 687     int resource_type_version = 0;
 688     // A list of subscriptions to this resource.
 689     std::set<SubscriptionState*> subscriptions;
 690   };
 691
 692   // The current state for all individual resources of a given type.
 693   using ResourceNameMap =
 694       std::map<std::string /* resource_name */, ResourceState>;
 695
 696   struct ResourceTypeState {
 697     int resource_type_version = 0;
 698     ResourceNameMap resource_name_map;
 699   };
 700
 701   using ResourceMap = std::map<std::string /* type_url */, ResourceTypeState>;
 702
 703   template <class RpcApi, class DiscoveryRequest, class DiscoveryResponse>
 704   class RpcService : public RpcApi::Service {
 705    public:
 706     using Stream = ServerReaderWriter<DiscoveryResponse, DiscoveryRequest>;
 707
 708     RpcService(AdsServiceImpl* parent, bool is_v2)
 709         : parent_(parent), is_v2_(is_v2) {}
 710
 711     Status StreamAggregatedResources(ServerContext* context,
 712                                      Stream* stream) override {
 713       gpr_log(GPR_INFO, "ADS[%p]: StreamAggregatedResources starts", this);
 714       parent_->AddClient(context->peer());
 715       if (is_v2_) {
 716         parent_->seen_v2_client_ = true;
 717       } else {
 718         parent_->seen_v3_client_ = true;
 719       }
 720       // Take a reference of the AdsServiceImpl object, which will go
 721       // out of scope when this request handler returns.  This ensures
 722       // that the parent won't be destroyed until this stream is complete.
 723       std::shared_ptr<AdsServiceImpl> ads_service_impl =
 724           parent_->shared_from_this();
 725       // Resources (type/name pairs) that have changed since the client
 726       // subscribed to them.
 727       UpdateQueue update_queue;
 728       // Resources that the client will be subscribed to keyed by resource type
 729       // url.
 730       SubscriptionMap subscription_map;
 731       // Sent state for each resource type.
 732       std::map<std::string /*type_url*/, SentState> sent_state_map;
 733       // Spawn a thread to read requests from the stream.
 734       // Requests will be delivered to this thread in a queue.
 735       std::deque<DiscoveryRequest> requests;
 736       bool stream_closed = false;
 737       std::thread reader(std::bind(&RpcService::BlockingRead, this, stream,
 738                                    &requests, &stream_closed));
 739       // Main loop to process requests and updates.
 740       while (true) {
 741         // Boolean to keep track if the loop received any work to do: a
 742         // request or an update; regardless whether a response was actually
 743         // sent out.
 744         bool did_work = false;
 745         // Look for new requests and and decide what to handle.
 746         absl::optional<DiscoveryResponse> response;
 747         {
 748           grpc_core::MutexLock lock(&parent_->ads_mu_);
 749           // If the stream has been closed or our parent is being shut
 750           // down, stop immediately.
 751           if (stream_closed || parent_->ads_done_) break;
 752           // Otherwise, see if there's a request to read from the queue.
 753           if (!requests.empty()) {
 754             DiscoveryRequest request = std::move(requests.front());
 755             requests.pop_front();
 756             did_work = true;
 757             gpr_log(GPR_INFO,
 758                     "ADS[%p]: Received request for type %s with content %s",
 759                     this, request.type_url().c_str(),
 760                     request.DebugString().c_str());
 761             const std::string v3_resource_type =
 762                 TypeUrlToV3(request.type_url());
 763             SentState& sent_state = sent_state_map[v3_resource_type];
 764             // Process request.
 765             ProcessRequest(request, v3_resource_type, &update_queue,
 766                            &subscription_map, &sent_state, &response);
 767           }
 768         }
 769         if (response.has_value()) {
 770           gpr_log(GPR_INFO, "ADS[%p]: Sending response: %s", this,
 771                   response->DebugString().c_str());
 772           stream->Write(response.value());
 773         }
 774         response.reset();
 775         // Look for updates and decide what to handle.
 776         {
 777           grpc_core::MutexLock lock(&parent_->ads_mu_);
 778           if (!update_queue.empty()) {
 779             const std::string resource_type =
 780                 std::move(update_queue.front().first);
 781             const std::string resource_name =
 782                 std::move(update_queue.front().second);
 783             update_queue.pop_front();
 784             did_work = true;
 785             SentState& sent_state = sent_state_map[resource_type];
 786             ProcessUpdate(resource_type, resource_name, &subscription_map,
 787                           &sent_state, &response);
 788           }
 789         }
 790         if (response.has_value()) {
 791           gpr_log(GPR_INFO, "ADS[%p]: Sending update response: %s", this,
 792                   response->DebugString().c_str());
 793           stream->Write(response.value());
 794         }
 795         {
 796           grpc_core::MutexLock lock(&parent_->ads_mu_);
 797           if (parent_->ads_done_) {
 798             break;
 799           }
 800         }
 801         // If we didn't find anything to do, delay before the next loop
 802         // iteration; otherwise, check whether we should exit and then
 803         // immediately continue.
 804         gpr_sleep_until(
 805             grpc_timeout_milliseconds_to_deadline(did_work ? 0 : 10));
 806       }
 807       // Done with main loop.  Clean up before returning.
 808       // Join reader thread.
 809       reader.join();
 810       // Clean up any subscriptions that were still active when the call
 811       // finished.
 812       {
 813         grpc_core::MutexLock lock(&parent_->ads_mu_);
 814         for (auto& p : subscription_map) {
 815           const std::string& type_url = p.first;
 816           SubscriptionNameMap& subscription_name_map = p.second;
 817           for (auto& q : subscription_name_map) {
 818             const std::string& resource_name = q.first;
 819             SubscriptionState& subscription_state = q.second;
 820             ResourceNameMap& resource_name_map =
 821                 parent_->resource_map_[type_url].resource_name_map;
 822             ResourceState& resource_state = resource_name_map[resource_name];
 823             resource_state.subscriptions.erase(&subscription_state);
 824           }
 825         }
 826       }
 827       gpr_log(GPR_INFO, "ADS[%p]: StreamAggregatedResources done", this);
 828       parent_->RemoveClient(context->peer());
 829       return Status::OK;
 830     }
 831
 832    private:
 833     // Processes a response read from the client.
 834     // Populates response if needed.
 835     void ProcessRequest(const DiscoveryRequest& request,
 836                         const std::string& v3_resource_type,
 837                         UpdateQueue* update_queue,
 838                         SubscriptionMap* subscription_map,
 839                         SentState* sent_state,
 840                         absl::optional<DiscoveryResponse>* response) {
 841       // Check the nonce sent by the client, if any.
 842       // (This will be absent on the first request on a stream.)
 843       if (request.response_nonce().empty()) {
 844         int client_resource_type_version = 0;
 845         if (!request.version_info().empty()) {
 846           GPR_ASSERT(absl::SimpleAtoi(request.version_info(),
 847                                       &client_resource_type_version));
 848         }
 849         EXPECT_GE(client_resource_type_version,
 850                   parent_->resource_type_min_versions_[v3_resource_type])
 851             << "resource_type: " << v3_resource_type;
 852       } else {
 853         int client_nonce;
 854         GPR_ASSERT(absl::SimpleAtoi(request.response_nonce(), &client_nonce));
 855         // Ignore requests with stale nonces.
 856         if (client_nonce < sent_state->nonce) return;
 857         // Check for ACK or NACK.
 858         auto it = parent_->resource_type_response_state_.find(v3_resource_type);
 859         if (it != parent_->resource_type_response_state_.end()) {
 860           if (!request.has_error_detail()) {
 861             it->second.state = ResponseState::ACKED;
 862             it->second.error_message.clear();
 863             gpr_log(GPR_INFO,
 864                     "ADS[%p]: client ACKed resource_type=%s version=%s", this,
 865                     request.type_url().c_str(), request.version_info().c_str());
 866           } else {
 867             it->second.state = ResponseState::NACKED;
 868             EXPECT_EQ(request.error_detail().code(),
 869                       GRPC_STATUS_INVALID_ARGUMENT);
 870             it->second.error_message = request.error_detail().message();
 871             gpr_log(GPR_INFO,
 872                     "ADS[%p]: client NACKed resource_type=%s version=%s: %s",
 873                     this, request.type_url().c_str(),
 874                     request.version_info().c_str(),
 875                     it->second.error_message.c_str());
 876           }
 877         }
 878       }
 879       // Ignore resource types as requested by tests.
 880       if (parent_->resource_types_to_ignore_.find(v3_resource_type) !=
 881           parent_->resource_types_to_ignore_.end()) {
 882         return;
 883       }
 884       // Look at all the resource names in the request.
 885       auto& subscription_name_map = (*subscription_map)[v3_resource_type];
 886       auto& resource_type_state = parent_->resource_map_[v3_resource_type];
 887       auto& resource_name_map = resource_type_state.resource_name_map;
 888       std::set<std::string> resources_in_current_request;
 889       std::set<std::string> resources_added_to_response;
 890       for (const std::string& resource_name : request.resource_names()) {
 891         resources_in_current_request.emplace(resource_name);
 892         auto& subscription_state = subscription_name_map[resource_name];
 893         auto& resource_state = resource_name_map[resource_name];
 894         // Subscribe if needed.
 895         // Send the resource in the response if either (a) this is
 896         // a new subscription or (b) there is an updated version of
 897         // this resource to send.
 898         if (parent_->MaybeSubscribe(v3_resource_type, resource_name,
 899                                     &subscription_state, &resource_state,
 900                                     update_queue) ||
 901             ClientNeedsResourceUpdate(resource_type_state, resource_state,
 902                                       sent_state->resource_type_version)) {
 903           gpr_log(GPR_INFO, "ADS[%p]: Sending update for type=%s name=%s", this,
 904                   request.type_url().c_str(), resource_name.c_str());
 905           resources_added_to_response.emplace(resource_name);
 906           if (!response->has_value()) response->emplace();
 907           if (resource_state.resource.has_value()) {
 908             auto* resource = (*response)->add_resources();
 909             resource->CopyFrom(resource_state.resource.value());
 910             if (is_v2_) {
 911               resource->set_type_url(request.type_url());
 912             }
 913           }
 914         } else {
 915           gpr_log(GPR_INFO,
 916                   "ADS[%p]: client does not need update for type=%s name=%s",
 917                   this, request.type_url().c_str(), resource_name.c_str());
 918         }
 919       }
 920       // Process unsubscriptions for any resource no longer
 921       // present in the request's resource list.
 922       parent_->ProcessUnsubscriptions(
 923           v3_resource_type, resources_in_current_request,
 924           &subscription_name_map, &resource_name_map);
 925       // Construct response if needed.
 926       if (!resources_added_to_response.empty()) {
 927         CompleteBuildingDiscoveryResponse(
 928             v3_resource_type, request.type_url(),
 929             resource_type_state.resource_type_version, subscription_name_map,
 930             resources_added_to_response, sent_state, &response->value());
 931       }
 932     }
 933
 934     // Processes a resource update from the test.
 935     // Populates response if needed.
 936     void ProcessUpdate(const std::string& resource_type,
 937                        const std::string& resource_name,
 938                        SubscriptionMap* subscription_map, SentState* sent_state,
 939                        absl::optional<DiscoveryResponse>* response) {
 940       const std::string v2_resource_type = TypeUrlToV2(resource_type);
 941       gpr_log(GPR_INFO, "ADS[%p]: Received update for type=%s name=%s", this,
 942               resource_type.c_str(), resource_name.c_str());
 943       auto& subscription_name_map = (*subscription_map)[resource_type];
 944       auto& resource_type_state = parent_->resource_map_[resource_type];
 945       auto& resource_name_map = resource_type_state.resource_name_map;
 946       auto it = subscription_name_map.find(resource_name);
 947       if (it != subscription_name_map.end()) {
 948         ResourceState& resource_state = resource_name_map[resource_name];
 949         if (ClientNeedsResourceUpdate(resource_type_state, resource_state,
 950                                       sent_state->resource_type_version)) {
 951           gpr_log(GPR_INFO, "ADS[%p]: Sending update for type=%s name=%s", this,
 952                   resource_type.c_str(), resource_name.c_str());
 953           response->emplace();
 954           if (resource_state.resource.has_value()) {
 955             auto* resource = (*response)->add_resources();
 956             resource->CopyFrom(resource_state.resource.value());
 957             if (is_v2_) {
 958               resource->set_type_url(v2_resource_type);
 959             }
 960           }
 961           CompleteBuildingDiscoveryResponse(
 962               resource_type, v2_resource_type,
 963               resource_type_state.resource_type_version, subscription_name_map,
 964               {resource_name}, sent_state, &response->value());
 965         }
 966       }
 967     }
 968
 969     // Starting a thread to do blocking read on the stream until cancel.
 970     void BlockingRead(Stream* stream, std::deque<DiscoveryRequest>* requests,
 971                       bool* stream_closed) {
 972       DiscoveryRequest request;
 973       bool seen_first_request = false;
 974       while (stream->Read(&request)) {
 975         if (!seen_first_request) {
 976           EXPECT_TRUE(request.has_node());
 977           ASSERT_FALSE(request.node().client_features().empty());
 978           EXPECT_EQ(request.node().client_features(0),
 979                     "envoy.lb.does_not_support_overprovisioning");
 980           CheckBuildVersion(request);
 981           seen_first_request = true;
 982         }
 983         {
 984           grpc_core::MutexLock lock(&parent_->ads_mu_);
 985           requests->emplace_back(std::move(request));
 986         }
 987       }
 988       gpr_log(GPR_INFO, "ADS[%p]: Null read, stream closed", this);
 989       grpc_core::MutexLock lock(&parent_->ads_mu_);
 990       *stream_closed = true;
 991     }
 992
 993     // Completing the building a DiscoveryResponse by adding common information
 994     // for all resources and by adding all subscribed resources for LDS and CDS.
 995     void CompleteBuildingDiscoveryResponse(
 996         const std::string& resource_type, const std::string& v2_resource_type,
 997         const int version, const SubscriptionNameMap& subscription_name_map,
 998         const std::set<std::string>& resources_added_to_response,
 999         SentState* sent_state, DiscoveryResponse* response) {
1000       auto& response_state =
1001           parent_->resource_type_response_state_[resource_type];
1002       if (response_state.state == ResponseState::NOT_SENT) {
1003         response_state.state = ResponseState::SENT;
1004       }
1005       response->set_type_url(is_v2_ ? v2_resource_type : resource_type);
1006       response->set_version_info(std::to_string(version));
1007       response->set_nonce(std::to_string(++sent_state->nonce));
1008       if (resource_type == kLdsTypeUrl || resource_type == kCdsTypeUrl) {
1009         // For LDS and CDS we must send back all subscribed resources
1010         // (even the unchanged ones)
1011         for (const auto& p : subscription_name_map) {
1012           const std::string& resource_name = p.first;
1013           if (resources_added_to_response.find(resource_name) ==
1014               resources_added_to_response.end()) {
1015             ResourceNameMap& resource_name_map =
1016                 parent_->resource_map_[resource_type].resource_name_map;
1017             const ResourceState& resource_state =
1018                 resource_name_map[resource_name];
1019             if (resource_state.resource.has_value()) {
1020               auto* resource = response->add_resources();
1021               resource->CopyFrom(resource_state.resource.value());
1022               if (is_v2_) {
1023                 resource->set_type_url(v2_resource_type);
1024               }
1025             }
1026           }
1027         }
1028       }
1029       sent_state->resource_type_version = version;
1030     }
1031
1032     static std::string TypeUrlToV2(const std::string& resource_type) {
1033       if (resource_type == kLdsTypeUrl) return kLdsV2TypeUrl;
1034       if (resource_type == kRdsTypeUrl) return kRdsV2TypeUrl;
1035       if (resource_type == kCdsTypeUrl) return kCdsV2TypeUrl;
1036       if (resource_type == kEdsTypeUrl) return kEdsV2TypeUrl;
1037       return resource_type;
1038     }
1039
1040     static std::string TypeUrlToV3(const std::string& resource_type) {
1041       if (resource_type == kLdsV2TypeUrl) return kLdsTypeUrl;
1042       if (resource_type == kRdsV2TypeUrl) return kRdsTypeUrl;
1043       if (resource_type == kCdsV2TypeUrl) return kCdsTypeUrl;
1044       if (resource_type == kEdsV2TypeUrl) return kEdsTypeUrl;
1045       return resource_type;
1046     }
1047
1048     static void CheckBuildVersion(
1049         const ::envoy::api::v2::DiscoveryRequest& request) {
1050       EXPECT_FALSE(request.node().build_version().empty());
1051     }
1052
1053     static void CheckBuildVersion(
1054         const ::envoy::service::discovery::v3::DiscoveryRequest& /*request*/) {}
1055
1056     AdsServiceImpl* parent_;
1057     const bool is_v2_;
1058   };
1059
1060   // Checks whether the client needs to receive a newer version of
1061   // the resource.
1062   static bool ClientNeedsResourceUpdate(
1063       const ResourceTypeState& resource_type_state,
1064       const ResourceState& resource_state, int client_resource_type_version) {
1065     return client_resource_type_version <
1066                resource_type_state.resource_type_version &&
1067            resource_state.resource_type_version <=
1068                resource_type_state.resource_type_version;
1069   }
1070
1071   // Subscribes to a resource if not already subscribed:
1072   // 1. Sets the update_queue field in subscription_state.
1073   // 2. Adds subscription_state to resource_state->subscriptions.
1074   bool MaybeSubscribe(const std::string& resource_type,
1075                       const std::string& resource_name,
1076                       SubscriptionState* subscription_state,
1077                       ResourceState* resource_state,
1078                       UpdateQueue* update_queue) {
1079     // The update_queue will be null if we were not previously subscribed.
1080     if (subscription_state->update_queue != nullptr) return false;
1081     subscription_state->update_queue = update_queue;
1082     resource_state->subscriptions.emplace(subscription_state);
1083     gpr_log(GPR_INFO, "ADS[%p]: subscribe to resource type %s name %s state %p",
1084             this, resource_type.c_str(), resource_name.c_str(),
1085             &subscription_state);
1086     return true;
1087   }
1088
1089   // Removes subscriptions for resources no longer present in the
1090   // current request.
1091   void ProcessUnsubscriptions(
1092       const std::string& resource_type,
1093       const std::set<std::string>& resources_in_current_request,
1094       SubscriptionNameMap* subscription_name_map,
1095       ResourceNameMap* resource_name_map) {
1096     for (auto it = subscription_name_map->begin();
1097          it != subscription_name_map->end();) {
1098       const std::string& resource_name = it->first;
1099       SubscriptionState& subscription_state = it->second;
1100       if (resources_in_current_request.find(resource_name) !=
1101           resources_in_current_request.end()) {
1102         ++it;
1103         continue;
1104       }
1105       gpr_log(GPR_INFO, "ADS[%p]: Unsubscribe to type=%s name=%s state=%p",
1106               this, resource_type.c_str(), resource_name.c_str(),
1107               &subscription_state);
1108       auto resource_it = resource_name_map->find(resource_name);
1109       GPR_ASSERT(resource_it != resource_name_map->end());
1110       auto& resource_state = resource_it->second;
1111       resource_state.subscriptions.erase(&subscription_state);
1112       if (resource_state.subscriptions.empty() &&
1113           !resource_state.resource.has_value()) {
1114         resource_name_map->erase(resource_it);
1115       }
1116       it = subscription_name_map->erase(it);
1117     }
1118   }
1119
1120   void AddClient(const std::string& client) {
1121     grpc_core::MutexLock lock(&clients_mu_);
1122     clients_.insert(client);
1123   }
1124
1125   void RemoveClient(const std::string& client) {
1126     grpc_core::MutexLock lock(&clients_mu_);
1127     clients_.erase(client);
1128   }
1129
1130   RpcService<::envoy::service::discovery::v2::AggregatedDiscoveryService,
1131              ::envoy::api::v2::DiscoveryRequest,
1132              ::envoy::api::v2::DiscoveryResponse>
1133       v2_rpc_service_;
1134   RpcService<::envoy::service::discovery::v3::AggregatedDiscoveryService,
1135              ::envoy::service::discovery::v3::DiscoveryRequest,
1136              ::envoy::service::discovery::v3::DiscoveryResponse>
1137       v3_rpc_service_;
1138
1139   std::atomic_bool seen_v2_client_{false};
1140   std::atomic_bool seen_v3_client_{false};
1141
1142   grpc_core::CondVar ads_cond_;
1143   // Protect the members below.
1144   grpc_core::Mutex ads_mu_;
1145   bool ads_done_ = false;
1146   std::map<std::string /* type_url */, ResponseState>
1147       resource_type_response_state_;
1148   std::set<std::string /*resource_type*/> resource_types_to_ignore_;
1149   std::map<std::string /*resource_type*/, int> resource_type_min_versions_;
1150   // An instance data member containing the current state of all resources.
1151   // Note that an entry will exist whenever either of the following is true:
1152   // - The resource exists (i.e., has been created by SetResource() and has not
1153   //   yet been destroyed by UnsetResource()).
1154   // - There is at least one subscription for the resource.
1155   ResourceMap resource_map_;
1156
1157   grpc_core::Mutex clients_mu_;
1158   std::set<std::string> clients_;
1159 };
1160
1161 class LrsServiceImpl : public std::enable_shared_from_this<LrsServiceImpl> {
1162  public:
1163   explicit LrsServiceImpl(int client_load_reporting_interval_seconds)
1164       : v2_rpc_service_(this),
1165         v3_rpc_service_(this),
1166         client_load_reporting_interval_seconds_(
1167             client_load_reporting_interval_seconds),
1168         cluster_names_({kDefaultClusterName}) {}
1169
1170   ::envoy::service::load_stats::v2::LoadReportingService::Service*
1171   v2_rpc_service() {
1172     return &v2_rpc_service_;
1173   }
1174
1175   ::envoy::service::load_stats::v3::LoadReportingService::Service*
1176   v3_rpc_service() {
1177     return &v3_rpc_service_;
1178   }
1179
1180   size_t request_count() {
1181     return v2_rpc_service_.request_count() + v3_rpc_service_.request_count();
1182   }
1183
1184   size_t response_count() {
1185     return v2_rpc_service_.response_count() + v3_rpc_service_.response_count();
1186   }
1187
1188   // Must be called before the LRS call is started.
1189   void set_send_all_clusters(bool send_all_clusters) {
1190     send_all_clusters_ = send_all_clusters;
1191   }
1192   void set_cluster_names(const std::set<std::string>& cluster_names) {
1193     cluster_names_ = cluster_names;
1194   }
1195
1196   void Start() {
1197     lrs_done_ = false;
1198     result_queue_.clear();
1199   }
1200
1201   void Shutdown() {
1202     {
1203       grpc_core::MutexLock lock(&lrs_mu_);
1204       NotifyDoneWithLrsCallLocked();
1205     }
1206     gpr_log(GPR_INFO, "LRS[%p]: shut down", this);
1207   }
1208
1209   std::vector<ClientStats> WaitForLoadReport() {
1210     grpc_core::MutexLock lock(&load_report_mu_);
1211     grpc_core::CondVar cv;
1212     if (result_queue_.empty()) {
1213       load_report_cond_ = &cv;
1214       while (result_queue_.empty()) {
1215         cv.Wait(&load_report_mu_);
1216       }
1217       load_report_cond_ = nullptr;
1218     }
1219     std::vector<ClientStats> result = std::move(result_queue_.front());
1220     result_queue_.pop_front();
1221     return result;
1222   }
1223
1224   void NotifyDoneWithLrsCall() {
1225     grpc_core::MutexLock lock(&lrs_mu_);
1226     NotifyDoneWithLrsCallLocked();
1227   }
1228
1229  private:
1230   template <class RpcApi, class LoadStatsRequest, class LoadStatsResponse>
1231   class RpcService : public CountedService<typename RpcApi::Service> {
1232    public:
1233     using Stream = ServerReaderWriter<LoadStatsResponse, LoadStatsRequest>;
1234
1235     explicit RpcService(LrsServiceImpl* parent) : parent_(parent) {}
1236
1237     Status StreamLoadStats(ServerContext* /*context*/,
1238                            Stream* stream) override {
1239       gpr_log(GPR_INFO, "LRS[%p]: StreamLoadStats starts", this);
1240       EXPECT_GT(parent_->client_load_reporting_interval_seconds_, 0);
1241       // Take a reference of the LrsServiceImpl object, reference will go
1242       // out of scope after this method exits.
1243       std::shared_ptr<LrsServiceImpl> lrs_service_impl =
1244           parent_->shared_from_this();
1245       // Read initial request.
1246       LoadStatsRequest request;
1247       if (stream->Read(&request)) {
1248         CountedService<typename RpcApi::Service>::IncreaseRequestCount();
1249         // Verify client features.
1250         EXPECT_THAT(
1251             request.node().client_features(),
1252             ::testing::Contains("envoy.lrs.supports_send_all_clusters"));
1253         // Send initial response.
1254         LoadStatsResponse response;
1255         if (parent_->send_all_clusters_) {
1256           response.set_send_all_clusters(true);
1257         } else {
1258           for (const std::string& cluster_name : parent_->cluster_names_) {
1259             response.add_clusters(cluster_name);
1260           }
1261         }
1262         response.mutable_load_reporting_interval()->set_seconds(
1263             parent_->client_load_reporting_interval_seconds_);
1264         stream->Write(response);
1265         CountedService<typename RpcApi::Service>::IncreaseResponseCount();
1266         // Wait for report.
1267         request.Clear();
1268         while (stream->Read(&request)) {
1269           gpr_log(GPR_INFO, "LRS[%p]: received client load report message: %s",
1270                   this, request.DebugString().c_str());
1271           std::vector<ClientStats> stats;
1272           for (const auto& cluster_stats : request.cluster_stats()) {
1273             stats.emplace_back(cluster_stats);
1274           }
1275           grpc_core::MutexLock lock(&parent_->load_report_mu_);
1276           parent_->result_queue_.emplace_back(std::move(stats));
1277           if (parent_->load_report_cond_ != nullptr) {
1278             parent_->load_report_cond_->Signal();
1279           }
1280         }
1281         // Wait until notified done.
1282         grpc_core::MutexLock lock(&parent_->lrs_mu_);
1283         while (!parent_->lrs_done_) {
1284           parent_->lrs_cv_.Wait(&parent_->lrs_mu_);
1285         }
1286       }
1287       gpr_log(GPR_INFO, "LRS[%p]: StreamLoadStats done", this);
1288       return Status::OK;
1289     }
1290
1291    private:
1292     LrsServiceImpl* parent_;
1293   };
1294
1295   void NotifyDoneWithLrsCallLocked() {
1296     if (!lrs_done_) {
1297       lrs_done_ = true;
1298       lrs_cv_.SignalAll();
1299     }
1300   }
1301
1302   RpcService<::envoy::service::load_stats::v2::LoadReportingService,
1303              ::envoy::service::load_stats::v2::LoadStatsRequest,
1304              ::envoy::service::load_stats::v2::LoadStatsResponse>
1305       v2_rpc_service_;
1306   RpcService<::envoy::service::load_stats::v3::LoadReportingService,
1307              ::envoy::service::load_stats::v3::LoadStatsRequest,
1308              ::envoy::service::load_stats::v3::LoadStatsResponse>
1309       v3_rpc_service_;
1310
1311   const int client_load_reporting_interval_seconds_;
1312   bool send_all_clusters_ = false;
1313   std::set<std::string> cluster_names_;
1314
1315   grpc_core::CondVar lrs_cv_;
1316   grpc_core::Mutex lrs_mu_;  // Protects lrs_done_.
1317   bool lrs_done_ = false;
1318
1319   grpc_core::Mutex load_report_mu_;  // Protects the members below.
1320   grpc_core::CondVar* load_report_cond_ = nullptr;
1321   std::deque<std::vector<ClientStats>> result_queue_;
1322 };
1323
1324 class TestType {
1325  public:
1326   enum FilterConfigSetup {
1327     // Set the fault injection filter directly from LDS
1328     kHTTPConnectionManagerOriginal,
1329     // Enable the fault injection filter in LDS, but override the filter config
1330     // in route.
1331     kRouteOverride,
1332   };
1333
1334   enum BootstrapSource {
1335     kBootstrapFromChannelArg,
1336     kBootstrapFromFile,
1337     kBootstrapFromEnvVar,
1338   };
1339
1340   TestType& set_use_fake_resolver() {
1341     use_fake_resolver_ = true;
1342     return *this;
1343   }
1344
1345   TestType& set_enable_load_reporting() {
1346     enable_load_reporting_ = true;
1347     return *this;
1348   }
1349
1350   TestType& set_enable_rds_testing() {
1351     enable_rds_testing_ = true;
1352     return *this;
1353   }
1354
1355   TestType& set_use_v2() {
1356     use_v2_ = true;
1357     return *this;
1358   }
1359
1360   TestType& set_use_xds_credentials() {
1361     use_xds_credentials_ = true;
1362     return *this;
1363   }
1364
1365   TestType& set_use_csds_streaming() {
1366     use_csds_streaming_ = true;
1367     return *this;
1368   }
1369
1370   TestType& set_filter_config_setup(FilterConfigSetup setup) {
1371     filter_config_setup_ = setup;
1372     return *this;
1373   }
1374
1375   TestType& set_bootstrap_source(BootstrapSource bootstrap_source) {
1376     bootstrap_source_ = bootstrap_source;
1377     return *this;
1378   }
1379
1380   bool use_fake_resolver() const { return use_fake_resolver_; }
1381   bool enable_load_reporting() const { return enable_load_reporting_; }
1382   bool enable_rds_testing() const { return enable_rds_testing_; }
1383   bool use_v2() const { return use_v2_; }
1384   bool use_xds_credentials() const { return use_xds_credentials_; }
1385   bool use_csds_streaming() const { return use_csds_streaming_; }
1386   FilterConfigSetup filter_config_setup() const { return filter_config_setup_; }
1387   BootstrapSource bootstrap_source() const { return bootstrap_source_; }
1388
1389   std::string AsString() const {
1390     std::string retval = (use_fake_resolver_ ? "FakeResolver" : "XdsResolver");
1391     retval += (use_v2_ ? "V2" : "V3");
1392     if (enable_load_reporting_) retval += "WithLoadReporting";
1393     if (enable_rds_testing_) retval += "Rds";
1394     if (use_xds_credentials_) retval += "XdsCreds";
1395     if (use_csds_streaming_) retval += "CsdsStreaming";
1396     if (filter_config_setup_ == kRouteOverride) {
1397       retval += "FilterPerRouteOverride";
1398     }
1399     if (bootstrap_source_ == kBootstrapFromFile) {
1400       retval += "BootstrapFromFile";
1401     } else if (bootstrap_source_ == kBootstrapFromEnvVar) {
1402       retval += "BootstrapFromEnvVar";
1403     }
1404     return retval;
1405   }
1406
1407  private:
1408   bool use_fake_resolver_ = false;
1409   bool enable_load_reporting_ = false;
1410   bool enable_rds_testing_ = false;
1411   bool use_v2_ = false;
1412   bool use_xds_credentials_ = false;
1413   bool use_csds_streaming_ = false;
1414   FilterConfigSetup filter_config_setup_ = kHTTPConnectionManagerOriginal;
1415   BootstrapSource bootstrap_source_ = kBootstrapFromChannelArg;
1416 };
1417
1418 std::string ReadFile(const char* file_path) {
1419   grpc_slice slice;
1420   GPR_ASSERT(
1421       GRPC_LOG_IF_ERROR("load_file", grpc_load_file(file_path, 0, &slice)));
1422   std::string file_contents(grpc_core::StringViewFromSlice(slice));
1423   grpc_slice_unref(slice);
1424   return file_contents;
1425 }
1426
1427 grpc_core::PemKeyCertPairList ReadTlsIdentityPair(const char* key_path,
1428                                                   const char* cert_path) {
1429   return grpc_core::PemKeyCertPairList{
1430       grpc_core::PemKeyCertPair(ReadFile(key_path), ReadFile(cert_path))};
1431 }
1432
1433 // Based on StaticDataCertificateProvider, but provides alternate certificates
1434 // if the certificate name is not empty.
1435 class FakeCertificateProvider final : public grpc_tls_certificate_provider {
1436  public:
1437   struct CertData {
1438     std::string root_certificate;
1439     grpc_core::PemKeyCertPairList identity_key_cert_pairs;
1440   };
1441
1442   using CertDataMap = std::map<std::string /*cert_name */, CertData>;
1443
1444   explicit FakeCertificateProvider(CertDataMap cert_data_map)
1445       : distributor_(
1446             grpc_core::MakeRefCounted<grpc_tls_certificate_distributor>()),
1447         cert_data_map_(std::move(cert_data_map)) {
1448     distributor_->SetWatchStatusCallback([this](std::string cert_name,
1449                                                 bool root_being_watched,
1450                                                 bool identity_being_watched) {
1451       if (!root_being_watched && !identity_being_watched) return;
1452       auto it = cert_data_map_.find(cert_name);
1453       if (it == cert_data_map_.end()) {
1454         grpc_error_handle error = GRPC_ERROR_CREATE_FROM_COPIED_STRING(
1455             absl::StrCat("No certificates available for cert_name \"",
1456                          cert_name, "\"")
1457                 .c_str());
1458         distributor_->SetErrorForCert(cert_name, GRPC_ERROR_REF(error),
1459                                       GRPC_ERROR_REF(error));
1460         GRPC_ERROR_UNREF(error);
1461       } else {
1462         absl::optional<std::string> root_certificate;
1463         absl::optional<grpc_core::PemKeyCertPairList> pem_key_cert_pairs;
1464         if (root_being_watched) {
1465           root_certificate = it->second.root_certificate;
1466         }
1467         if (identity_being_watched) {
1468           pem_key_cert_pairs = it->second.identity_key_cert_pairs;
1469         }
1470         distributor_->SetKeyMaterials(cert_name, std::move(root_certificate),
1471                                       std::move(pem_key_cert_pairs));
1472       }
1473     });
1474   }
1475
1476   ~FakeCertificateProvider() override {
1477     distributor_->SetWatchStatusCallback(nullptr);
1478   }
1479
1480   grpc_core::RefCountedPtr<grpc_tls_certificate_distributor> distributor()
1481       const override {
1482     return distributor_;
1483   }
1484
1485  private:
1486   grpc_core::RefCountedPtr<grpc_tls_certificate_distributor> distributor_;
1487   CertDataMap cert_data_map_;
1488 };
1489
1490 class FakeCertificateProviderFactory
1491     : public grpc_core::CertificateProviderFactory {
1492  public:
1493   class Config : public grpc_core::CertificateProviderFactory::Config {
1494    public:
1495     explicit Config(const char* name) : name_(name) {}
1496
1497     const char* name() const override { return name_; }
1498
1499     std::string ToString() const override { return "{}"; }
1500
1501    private:
1502     const char* name_;
1503   };
1504
1505   FakeCertificateProviderFactory(
1506       const char* name, FakeCertificateProvider::CertDataMap** cert_data_map)
1507       : name_(name), cert_data_map_(cert_data_map) {
1508     GPR_ASSERT(cert_data_map != nullptr);
1509   }
1510
1511   const char* name() const override { return name_; }
1512
1513   grpc_core::RefCountedPtr<grpc_core::CertificateProviderFactory::Config>
1514   CreateCertificateProviderConfig(const grpc_core::Json& /*config_json*/,
1515                                   grpc_error_handle* /*error*/) override {
1516     return grpc_core::MakeRefCounted<Config>(name_);
1517   }
1518
1519   grpc_core::RefCountedPtr<grpc_tls_certificate_provider>
1520   CreateCertificateProvider(
1521       grpc_core::RefCountedPtr<grpc_core::CertificateProviderFactory::Config>
1522       /*config*/) override {
1523     if (*cert_data_map_ == nullptr) return nullptr;
1524     return grpc_core::MakeRefCounted<FakeCertificateProvider>(**cert_data_map_);
1525   }
1526
1527  private:
1528   const char* name_;
1529   FakeCertificateProvider::CertDataMap** cert_data_map_;
1530 };
1531
1532 // Global variables for each provider.
1533 FakeCertificateProvider::CertDataMap* g_fake1_cert_data_map = nullptr;
1534 FakeCertificateProvider::CertDataMap* g_fake2_cert_data_map = nullptr;
1535
1536 int ServerAuthCheckSchedule(void* /* config_user_data */,
1537                             grpc_tls_server_authorization_check_arg* arg) {
1538   arg->success = 1;
1539   arg->status = GRPC_STATUS_OK;
1540   return 0; /* synchronous check */
1541 }
1542
1543 std::shared_ptr<ChannelCredentials> CreateTlsFallbackCredentials() {
1544   // TODO(yashykt): Switch to using C++ API once b/173823806 is fixed.
1545   grpc_tls_credentials_options* options = grpc_tls_credentials_options_create();
1546   grpc_tls_credentials_options_set_server_verification_option(
1547       options, GRPC_TLS_SKIP_HOSTNAME_VERIFICATION);
1548   grpc_tls_credentials_options_set_certificate_provider(
1549       options,
1550       grpc_core::MakeRefCounted<grpc_core::StaticDataCertificateProvider>(
1551           ReadFile(kCaCertPath),
1552           ReadTlsIdentityPair(kServerKeyPath, kServerCertPath))
1553           .get());
1554   grpc_tls_credentials_options_watch_root_certs(options);
1555   grpc_tls_credentials_options_watch_identity_key_cert_pairs(options);
1556   grpc_tls_server_authorization_check_config* check_config =
1557       grpc_tls_server_authorization_check_config_create(
1558           nullptr, ServerAuthCheckSchedule, nullptr, nullptr);
1559   grpc_tls_credentials_options_set_server_authorization_check_config(
1560       options, check_config);
1561   auto channel_creds = std::make_shared<SecureChannelCredentials>(
1562       grpc_tls_credentials_create(options));
1563   grpc_tls_server_authorization_check_config_release(check_config);
1564   return channel_creds;
1565 }
1566
1567 // A No-op HTTP filter used for verifying parsing logic.
1568 class NoOpHttpFilter : public grpc_core::XdsHttpFilterImpl {
1569  public:
1570   NoOpHttpFilter(std::string name, bool supported_on_clients,
1571                  bool supported_on_servers)
1572       : name_(std::move(name)),
1573         supported_on_clients_(supported_on_clients),
1574         supported_on_servers_(supported_on_servers) {}
1575
1576   void PopulateSymtab(upb_symtab* /* symtab */) const override {}
1577
1578   absl::StatusOr<grpc_core::XdsHttpFilterImpl::FilterConfig>
1579   GenerateFilterConfig(upb_strview /* serialized_filter_config */,
1580                        upb_arena* /* arena */) const override {
1581     return grpc_core::XdsHttpFilterImpl::FilterConfig{name_, grpc_core::Json()};
1582   }
1583
1584   absl::StatusOr<grpc_core::XdsHttpFilterImpl::FilterConfig>
1585   GenerateFilterConfigOverride(upb_strview /*serialized_filter_config*/,
1586                                upb_arena* /*arena*/) const override {
1587     return grpc_core::XdsHttpFilterImpl::FilterConfig{name_, grpc_core::Json()};
1588   }
1589
1590   const grpc_channel_filter* channel_filter() const override { return nullptr; }
1591
1592   absl::StatusOr<grpc_core::XdsHttpFilterImpl::ServiceConfigJsonEntry>
1593   GenerateServiceConfig(
1594       const FilterConfig& /*hcm_filter_config*/,
1595       const FilterConfig* /*filter_config_override*/) const override {
1596     return grpc_core::XdsHttpFilterImpl::ServiceConfigJsonEntry{name_, ""};
1597   }
1598
1599   bool IsSupportedOnClients() const override { return supported_on_clients_; }
1600
1601   bool IsSupportedOnServers() const override { return supported_on_servers_; }
1602
1603  private:
1604   const std::string name_;
1605   const bool supported_on_clients_;
1606   const bool supported_on_servers_;
1607 };
1608
1609 // There is slight difference between time fetched by GPR and by C++ system
1610 // clock API. It's unclear if they are using the same syscall, but we do know
1611 // GPR round the number at millisecond-level. This creates a 1ms difference,
1612 // which could cause flake.
1613 grpc_millis NowFromCycleCounter() {
1614   gpr_cycle_counter now = gpr_get_cycle_counter();
1615   return grpc_cycle_counter_to_millis_round_up(now);
1616 }
1617
1618 // Returns the number of RPCs needed to pass error_tolerance at 99.99994%
1619 // chance. Rolling dices in drop/fault-injection generates a binomial
1620 // distribution (if our code is not horribly wrong). Let's make "n" the number
1621 // of samples, "p" the probability. If we have np>5 & n(1-p)>5, we can
1622 // approximately treat the binomial distribution as a normal distribution.
1623 //
1624 // For normal distribution, we can easily look up how many standard deviation we
1625 // need to reach 99.995%. Based on Wiki's table
1626 // https://en.wikipedia.org/wiki/68%E2%80%9395%E2%80%9399.7_rule, we need 5.00
1627 // sigma (standard deviation) to cover the probability area of 99.99994%. In
1628 // another word, for a sample with size "n" probability "p" error-tolerance "k",
1629 // we want the error always land within 5.00 sigma. The sigma of binominal
1630 // distribution and be computed as sqrt(np(1-p)). Hence, we have the equation:
1631 //
1632 //   kn <= 5.00 * sqrt(np(1-p))
1633 size_t ComputeIdealNumRpcs(double p, double error_tolerance) {
1634   GPR_ASSERT(p >= 0 && p <= 1);
1635   size_t num_rpcs =
1636       ceil(p * (1 - p) * 5.00 * 5.00 / error_tolerance / error_tolerance);
1637   gpr_log(GPR_INFO,
1638           "Sending %" PRIuPTR " RPCs for percentage=%.3f error_tolerance=%.3f",
1639           num_rpcs, p, error_tolerance);
1640   return num_rpcs;
1641 }
1642
1643 // Channel arg pointer vtable for storing xDS channel args in the parent
1644 // channel's channel args.
1645 void* ChannelArgsArgCopy(void* p) {
1646   auto* args = static_cast<grpc_channel_args*>(p);
1647   return grpc_channel_args_copy(args);
1648 }
1649 void ChannelArgsArgDestroy(void* p) {
1650   auto* args = static_cast<grpc_channel_args*>(p);
1651   grpc_channel_args_destroy(args);
1652 }
1653 int ChannelArgsArgCmp(void* a, void* b) {
1654   auto* args_a = static_cast<grpc_channel_args*>(a);
1655   auto* args_b = static_cast<grpc_channel_args*>(b);
1656   return grpc_channel_args_compare(args_a, args_b);
1657 }
1658 const grpc_arg_pointer_vtable kChannelArgsArgVtable = {
1659     ChannelArgsArgCopy, ChannelArgsArgDestroy, ChannelArgsArgCmp};
1660
1661 class XdsEnd2endTest : public ::testing::TestWithParam<TestType> {
1662  protected:
1663   // TODO(roth): We currently set the number of backends and number of
1664   // balancers on a per-test-suite basis, not a per-test-case basis.
1665   // However, not every individual test case in a given test suite uses
1666   // the same number of backends or balancers, so we wind up having to
1667   // set the numbers for the test suite to the max number needed by any
1668   // one test case in that test suite.  This results in starting more
1669   // servers (and using more ports) than we actually need.  When we have
1670   // time, change each test to directly start the number of backends and
1671   // balancers that it needs, so that we aren't wasting resources.
1672   XdsEnd2endTest(size_t num_backends, size_t num_balancers,
1673                  int client_load_reporting_interval_seconds = 100,
1674                  bool use_xds_enabled_server = false)
1675       : num_backends_(num_backends),
1676         num_balancers_(num_balancers),
1677         client_load_reporting_interval_seconds_(
1678             client_load_reporting_interval_seconds),
1679         use_xds_enabled_server_(use_xds_enabled_server) {}
1680
1681   void SetUp() override {
1682     bool localhost_resolves_to_ipv4 = false;
1683     bool localhost_resolves_to_ipv6 = false;
1684     grpc_core::LocalhostResolves(&localhost_resolves_to_ipv4,
1685                                  &localhost_resolves_to_ipv6);
1686     ipv6_only_ = !localhost_resolves_to_ipv4 && localhost_resolves_to_ipv6;
1687     // Initialize default xDS resources.
1688     // Construct LDS resource.
1689     default_listener_.set_name(kServerName);
1690     HttpConnectionManager http_connection_manager;
1691     if (!GetParam().use_v2()) {
1692       auto* filter = http_connection_manager.add_http_filters();
1693       filter->set_name("router");
1694       filter->mutable_typed_config()->PackFrom(
1695           envoy::extensions::filters::http::router::v3::Router());
1696     }
1697     default_listener_.mutable_api_listener()->mutable_api_listener()->PackFrom(
1698         http_connection_manager);
1699     // Construct RDS resource.
1700     default_route_config_.set_name(kDefaultRouteConfigurationName);
1701     auto* virtual_host = default_route_config_.add_virtual_hosts();
1702     virtual_host->add_domains("*");
1703     auto* route = virtual_host->add_routes();
1704     route->mutable_match()->set_prefix("");
1705     route->mutable_route()->set_cluster(kDefaultClusterName);
1706     // Construct CDS resource.
1707     default_cluster_.set_name(kDefaultClusterName);
1708     default_cluster_.set_type(Cluster::EDS);
1709     auto* eds_config = default_cluster_.mutable_eds_cluster_config();
1710     eds_config->mutable_eds_config()->mutable_ads();
1711     eds_config->set_service_name(kDefaultEdsServiceName);
1712     default_cluster_.set_lb_policy(Cluster::ROUND_ROBIN);
1713     if (GetParam().enable_load_reporting()) {
1714       default_cluster_.mutable_lrs_server()->mutable_self();
1715     }
1716     // Start the load balancers.
1717     for (size_t i = 0; i < num_balancers_; ++i) {
1718       balancers_.emplace_back(new BalancerServerThread(
1719           this, GetParam().enable_load_reporting()
1720                     ? client_load_reporting_interval_seconds_
1721                     : 0));
1722       balancers_.back()->Start();
1723       // Initialize resources.
1724       SetListenerAndRouteConfiguration(i, default_listener_,
1725                                        default_route_config_);
1726       balancers_.back()->ads_service()->SetCdsResource(default_cluster_);
1727     }
1728     // Create fake resolver response generators used by client.
1729     if (GetParam().use_fake_resolver()) {
1730       response_generator_ =
1731           grpc_core::MakeRefCounted<grpc_core::FakeResolverResponseGenerator>();
1732     }
1733     logical_dns_cluster_resolver_response_generator_ =
1734         grpc_core::MakeRefCounted<grpc_core::FakeResolverResponseGenerator>();
1735     lb_channel_response_generator_ =
1736         grpc_core::MakeRefCounted<grpc_core::FakeResolverResponseGenerator>();
1737     // Construct channel args for XdsClient.
1738     xds_channel_args_to_add_.emplace_back(
1739         grpc_core::FakeResolverResponseGenerator::MakeChannelArg(
1740             lb_channel_response_generator_.get()));
1741     if (xds_resource_does_not_exist_timeout_ms_ > 0) {
1742       xds_channel_args_to_add_.emplace_back(grpc_channel_arg_integer_create(
1743           const_cast<char*>(GRPC_ARG_XDS_RESOURCE_DOES_NOT_EXIST_TIMEOUT_MS),
1744           xds_resource_does_not_exist_timeout_ms_));
1745     }
1746     xds_channel_args_.num_args = xds_channel_args_to_add_.size();
1747     xds_channel_args_.args = xds_channel_args_to_add_.data();
1748     // Initialize XdsClient state.
1749     // TODO(roth): Consider changing this to dynamically generate the
1750     // bootstrap config in each individual test instead of hard-coding
1751     // the contents here.  That would allow us to use an ipv4: or ipv6:
1752     // URI for the xDS server instead of using the fake resolver.
1753     if (GetParam().bootstrap_source() == TestType::kBootstrapFromEnvVar) {
1754       gpr_setenv("GRPC_XDS_BOOTSTRAP_CONFIG",
1755                  GetParam().use_v2() ? kBootstrapFileV2 : kBootstrapFileV3);
1756     } else if (GetParam().bootstrap_source() == TestType::kBootstrapFromFile) {
1757       gpr_setenv("GRPC_XDS_BOOTSTRAP", GetParam().use_v2()
1758                                            ? g_bootstrap_file_v2
1759                                            : g_bootstrap_file_v3);
1760     }
1761     if (GetParam().bootstrap_source() != TestType::kBootstrapFromChannelArg) {
1762       // If getting bootstrap from channel arg, we'll pass these args in
1763       // via the parent channel args in CreateChannel() instead.
1764       grpc_core::internal::SetXdsChannelArgsForTest(&xds_channel_args_);
1765       // Make sure each test creates a new XdsClient instance rather than
1766       // reusing the one from the previous test.  This avoids spurious failures
1767       // caused when a load reporting test runs after a non-load reporting test
1768       // and the XdsClient is still talking to the old LRS server, which fails
1769       // because it's not expecting the client to connect.  It also
1770       // ensures that each test can independently set the global channel
1771       // args for the xDS channel.
1772       grpc_core::internal::UnsetGlobalXdsClientForTest();
1773     }
1774     // Start the backends.
1775     for (size_t i = 0; i < num_backends_; ++i) {
1776       backends_.emplace_back(
1777           new BackendServerThread(this, use_xds_enabled_server_));
1778       backends_.back()->Start();
1779     }
1780     // Create channel and stub.
1781     ResetStub();
1782   }
1783
1784   const char* DefaultEdsServiceName() const {
1785     return GetParam().use_fake_resolver() ? kServerName
1786                                           : kDefaultEdsServiceName;
1787   }
1788
1789   void TearDown() override {
1790     ShutdownAllBackends();
1791     for (auto& balancer : balancers_) balancer->Shutdown();
1792     // Clear global xDS channel args, since they will go out of scope
1793     // when this test object is destroyed.
1794     grpc_core::internal::SetXdsChannelArgsForTest(nullptr);
1795     gpr_unsetenv("GRPC_XDS_BOOTSTRAP");
1796     gpr_unsetenv("GRPC_XDS_BOOTSTRAP_CONFIG");
1797   }
1798
1799   void StartAllBackends() {
1800     for (auto& backend : backends_) backend->Start();
1801   }
1802
1803   void StartBackend(size_t index) { backends_[index]->Start(); }
1804
1805   void ShutdownAllBackends() {
1806     for (auto& backend : backends_) backend->Shutdown();
1807   }
1808
1809   void ShutdownBackend(size_t index) { backends_[index]->Shutdown(); }
1810
1811   void ResetStub(int failover_timeout = 0) {
1812     channel_ = CreateChannel(failover_timeout);
1813     stub_ = grpc::testing::EchoTestService::NewStub(channel_);
1814     stub1_ = grpc::testing::EchoTest1Service::NewStub(channel_);
1815     stub2_ = grpc::testing::EchoTest2Service::NewStub(channel_);
1816   }
1817
1818   std::shared_ptr<Channel> CreateChannel(
1819       int failover_timeout = 0, const char* server_name = kServerName,
1820       grpc_core::FakeResolverResponseGenerator* response_generator = nullptr,
1821       grpc_channel_args* xds_channel_args = nullptr) {
1822     ChannelArguments args;
1823     if (failover_timeout > 0) {
1824       args.SetInt(GRPC_ARG_PRIORITY_FAILOVER_TIMEOUT_MS, failover_timeout);
1825     }
1826     // If the parent channel is using the fake resolver, we inject the
1827     // response generator here.
1828     if (GetParam().use_fake_resolver()) {
1829       if (response_generator == nullptr) {
1830         response_generator = response_generator_.get();
1831       }
1832       args.SetPointerWithVtable(
1833           GRPC_ARG_FAKE_RESOLVER_RESPONSE_GENERATOR, response_generator,
1834           &grpc_core::FakeResolverResponseGenerator::kChannelArgPointerVtable);
1835     }
1836     if (GetParam().bootstrap_source() == TestType::kBootstrapFromChannelArg) {
1837       // We're getting the bootstrap from a channel arg, so we do the
1838       // same thing for the response generator to use for the xDS
1839       // channel and the xDS resource-does-not-exist timeout value.
1840       args.SetString(GRPC_ARG_TEST_ONLY_DO_NOT_USE_IN_PROD_XDS_BOOTSTRAP_CONFIG,
1841                      GetParam().use_v2() ? kBootstrapFileV2 : kBootstrapFileV3);
1842       if (xds_channel_args == nullptr) xds_channel_args = &xds_channel_args_;
1843       args.SetPointerWithVtable(
1844           GRPC_ARG_TEST_ONLY_DO_NOT_USE_IN_PROD_XDS_CLIENT_CHANNEL_ARGS,
1845           xds_channel_args, &kChannelArgsArgVtable);
1846     }
1847     args.SetPointerWithVtable(
1848         GRPC_ARG_XDS_LOGICAL_DNS_CLUSTER_FAKE_RESOLVER_RESPONSE_GENERATOR,
1849         logical_dns_cluster_resolver_response_generator_.get(),
1850         &grpc_core::FakeResolverResponseGenerator::kChannelArgPointerVtable);
1851     std::string uri = absl::StrCat(
1852         GetParam().use_fake_resolver() ? "fake" : "xds", ":///", server_name);
1853     std::shared_ptr<ChannelCredentials> channel_creds =
1854         GetParam().use_xds_credentials()
1855             ? experimental::XdsCredentials(CreateTlsFallbackCredentials())
1856             : std::make_shared<SecureChannelCredentials>(
1857                   grpc_fake_transport_security_credentials_create());
1858     return ::grpc::CreateCustomChannel(uri, channel_creds, args);
1859   }
1860
1861   enum RpcService {
1862     SERVICE_ECHO,
1863     SERVICE_ECHO1,
1864     SERVICE_ECHO2,
1865   };
1866
1867   enum RpcMethod {
1868     METHOD_ECHO,
1869     METHOD_ECHO1,
1870     METHOD_ECHO2,
1871   };
1872
1873   struct RpcOptions {
1874     RpcService service = SERVICE_ECHO;
1875     RpcMethod method = METHOD_ECHO;
1876     int timeout_ms = 1000;
1877     bool wait_for_ready = false;
1878     bool server_fail = false;
1879     std::vector<std::pair<std::string, std::string>> metadata;
1880     int server_sleep_us = 0;
1881     int client_cancel_after_us = 0;
1882     bool skip_cancelled_check = false;
1883     StatusCode server_expected_error = StatusCode::OK;
1884
1885     RpcOptions() {}
1886
1887     RpcOptions& set_rpc_service(RpcService rpc_service) {
1888       service = rpc_service;
1889       return *this;
1890     }
1891
1892     RpcOptions& set_rpc_method(RpcMethod rpc_method) {
1893       method = rpc_method;
1894       return *this;
1895     }
1896
1897     RpcOptions& set_timeout_ms(int rpc_timeout_ms) {
1898       timeout_ms = rpc_timeout_ms;
1899       return *this;
1900     }
1901
1902     RpcOptions& set_wait_for_ready(bool rpc_wait_for_ready) {
1903       wait_for_ready = rpc_wait_for_ready;
1904       return *this;
1905     }
1906
1907     RpcOptions& set_server_fail(bool rpc_server_fail) {
1908       server_fail = rpc_server_fail;
1909       return *this;
1910     }
1911
1912     RpcOptions& set_skip_cancelled_check(bool rpc_skip_cancelled_check) {
1913       skip_cancelled_check = rpc_skip_cancelled_check;
1914       return *this;
1915     }
1916
1917     RpcOptions& set_metadata(
1918         std::vector<std::pair<std::string, std::string>> rpc_metadata) {
1919       metadata = std::move(rpc_metadata);
1920       return *this;
1921     }
1922
1923     RpcOptions& set_server_sleep_us(int rpc_server_sleep_us) {
1924       server_sleep_us = rpc_server_sleep_us;
1925       return *this;
1926     }
1927
1928     RpcOptions& set_client_cancel_after_us(int rpc_client_cancel_after_us) {
1929       client_cancel_after_us = rpc_client_cancel_after_us;
1930       return *this;
1931     }
1932
1933     RpcOptions& set_server_expected_error(StatusCode code) {
1934       server_expected_error = code;
1935       return *this;
1936     }
1937
1938     // Populates context and request.
1939     void SetupRpc(ClientContext* context, EchoRequest* request) const {
1940       for (const auto& item : metadata) {
1941         context->AddMetadata(item.first, item.second);
1942       }
1943       if (timeout_ms != 0) {
1944         context->set_deadline(
1945             grpc_timeout_milliseconds_to_deadline(timeout_ms));
1946       }
1947       if (wait_for_ready) context->set_wait_for_ready(true);
1948       request->set_message(kRequestMessage);
1949       if (server_fail) {
1950         request->mutable_param()->mutable_expected_error()->set_code(
1951             GRPC_STATUS_FAILED_PRECONDITION);
1952       }
1953       if (server_sleep_us != 0) {
1954         request->mutable_param()->set_server_sleep_us(server_sleep_us);
1955       }
1956       if (client_cancel_after_us != 0) {
1957         request->mutable_param()->set_client_cancel_after_us(
1958             client_cancel_after_us);
1959       }
1960       if (skip_cancelled_check) {
1961         request->mutable_param()->set_skip_cancelled_check(true);
1962       }
1963     }
1964   };
1965
1966   template <typename Stub>
1967   Status SendRpcMethod(Stub* stub, const RpcOptions& rpc_options,
1968                        ClientContext* context, EchoRequest& request,
1969                        EchoResponse* response) {
1970     switch (rpc_options.method) {
1971       case METHOD_ECHO:
1972         return (*stub)->Echo(context, request, response);
1973       case METHOD_ECHO1:
1974         return (*stub)->Echo1(context, request, response);
1975       case METHOD_ECHO2:
1976         return (*stub)->Echo2(context, request, response);
1977     }
1978     GPR_UNREACHABLE_CODE();
1979   }
1980
1981   void ResetBackendCounters(size_t start_index = 0, size_t stop_index = 0) {
1982     if (stop_index == 0) stop_index = backends_.size();
1983     for (size_t i = start_index; i < stop_index; ++i) {
1984       backends_[i]->backend_service()->ResetCounters();
1985       backends_[i]->backend_service1()->ResetCounters();
1986       backends_[i]->backend_service2()->ResetCounters();
1987     }
1988   }
1989
1990   bool SeenBackend(size_t backend_idx,
1991                    const RpcService rpc_service = SERVICE_ECHO) {
1992     switch (rpc_service) {
1993       case SERVICE_ECHO:
1994         if (backends_[backend_idx]->backend_service()->request_count() == 0) {
1995           return false;
1996         }
1997         break;
1998       case SERVICE_ECHO1:
1999         if (backends_[backend_idx]->backend_service1()->request_count() == 0) {
2000           return false;
2001         }
2002         break;
2003       case SERVICE_ECHO2:
2004         if (backends_[backend_idx]->backend_service2()->request_count() == 0) {
2005           return false;
2006         }
2007         break;
2008     }
2009     return true;
2010   }
2011
2012   bool SeenAllBackends(size_t start_index = 0, size_t stop_index = 0,
2013                        const RpcService rpc_service = SERVICE_ECHO) {
2014     if (stop_index == 0) stop_index = backends_.size();
2015     for (size_t i = start_index; i < stop_index; ++i) {
2016       if (!SeenBackend(i, rpc_service)) {
2017         return false;
2018       }
2019     }
2020     return true;
2021   }
2022
2023   void SendRpcAndCount(
2024       int* num_total, int* num_ok, int* num_failure, int* num_drops,
2025       const RpcOptions& rpc_options = RpcOptions(),
2026       const char* drop_error_message_prefix = "EDS-configured drop: ") {
2027     const Status status = SendRpc(rpc_options);
2028     if (status.ok()) {
2029       ++*num_ok;
2030     } else {
2031       if (absl::StartsWith(status.error_message(), drop_error_message_prefix)) {
2032         ++*num_drops;
2033       } else {
2034         ++*num_failure;
2035       }
2036     }
2037     ++*num_total;
2038   }
2039
2040   struct WaitForBackendOptions {
2041     bool reset_counters = true;
2042     bool allow_failures = false;
2043
2044     WaitForBackendOptions() {}
2045
2046     WaitForBackendOptions& set_reset_counters(bool enable) {
2047       reset_counters = enable;
2048       return *this;
2049     }
2050
2051     WaitForBackendOptions& set_allow_failures(bool enable) {
2052       allow_failures = enable;
2053       return *this;
2054     }
2055   };
2056
2057   std::tuple<int, int, int> WaitForAllBackends(
2058       size_t start_index = 0, size_t stop_index = 0,
2059       const WaitForBackendOptions& wait_options = WaitForBackendOptions(),
2060       const RpcOptions& rpc_options = RpcOptions()) {
2061     int num_ok = 0;
2062     int num_failure = 0;
2063     int num_drops = 0;
2064     int num_total = 0;
2065     while (!SeenAllBackends(start_index, stop_index, rpc_options.service)) {
2066       SendRpcAndCount(&num_total, &num_ok, &num_failure, &num_drops,
2067                       rpc_options);
2068     }
2069     if (wait_options.reset_counters) ResetBackendCounters();
2070     gpr_log(GPR_INFO,
2071             "Performed %d warm up requests against the backends. "
2072             "%d succeeded, %d failed, %d dropped.",
2073             num_total, num_ok, num_failure, num_drops);
2074     if (!wait_options.allow_failures) EXPECT_EQ(num_failure, 0);
2075     return std::make_tuple(num_ok, num_failure, num_drops);
2076   }
2077
2078   void WaitForBackend(
2079       size_t backend_idx,
2080       const WaitForBackendOptions& wait_options = WaitForBackendOptions(),
2081       const RpcOptions& rpc_options = RpcOptions()) {
2082     gpr_log(GPR_INFO, "========= WAITING FOR BACKEND %lu ==========",
2083             static_cast<unsigned long>(backend_idx));
2084     do {
2085       Status status = SendRpc(rpc_options);
2086       if (!wait_options.allow_failures) {
2087         EXPECT_TRUE(status.ok()) << "code=" << status.error_code()
2088                                  << " message=" << status.error_message();
2089       }
2090     } while (!SeenBackend(backend_idx, rpc_options.service));
2091     if (wait_options.reset_counters) ResetBackendCounters();
2092     gpr_log(GPR_INFO, "========= BACKEND %lu READY ==========",
2093             static_cast<unsigned long>(backend_idx));
2094   }
2095
2096   grpc_core::ServerAddressList CreateAddressListFromPortList(
2097       const std::vector<int>& ports) {
2098     grpc_core::ServerAddressList addresses;
2099     for (int port : ports) {
2100       absl::StatusOr<grpc_core::URI> lb_uri = grpc_core::URI::Parse(
2101           absl::StrCat(ipv6_only_ ? "ipv6:[::1]:" : "ipv4:127.0.0.1:", port));
2102       GPR_ASSERT(lb_uri.ok());
2103       grpc_resolved_address address;
2104       GPR_ASSERT(grpc_parse_uri(*lb_uri, &address));
2105       addresses.emplace_back(address.addr, address.len, nullptr);
2106     }
2107     return addresses;
2108   }
2109
2110   std::string CreateMetadataValueThatHashesToBackendPort(int port) {
2111     return absl::StrCat(ipv6_only_ ? "[::1]" : "127.0.0.1", ":", port, "_0");
2112   }
2113
2114   std::string CreateMetadataValueThatHashesToBackend(int index) {
2115     return CreateMetadataValueThatHashesToBackendPort(backends_[index]->port());
2116   }
2117
2118   void SetNextResolution(
2119       const std::vector<int>& ports,
2120       grpc_core::FakeResolverResponseGenerator* response_generator = nullptr) {
2121     if (!GetParam().use_fake_resolver()) return;  // Not used with xds resolver.
2122     grpc_core::ExecCtx exec_ctx;
2123     grpc_core::Resolver::Result result;
2124     result.addresses = CreateAddressListFromPortList(ports);
2125     grpc_error_handle error = GRPC_ERROR_NONE;
2126     const char* service_config_json =
2127         GetParam().enable_load_reporting()
2128             ? kDefaultServiceConfig
2129             : kDefaultServiceConfigWithoutLoadReporting;
2130     result.service_config =
2131         grpc_core::ServiceConfig::Create(nullptr, service_config_json, &error);
2132     ASSERT_EQ(error, GRPC_ERROR_NONE) << grpc_error_std_string(error);
2133     ASSERT_NE(result.service_config.get(), nullptr);
2134     if (response_generator == nullptr) {
2135       response_generator = response_generator_.get();
2136     }
2137     response_generator->SetResponse(std::move(result));
2138   }
2139
2140   void SetNextResolutionForLbChannelAllBalancers(
2141       const char* service_config_json = nullptr,
2142       const char* expected_targets = nullptr,
2143       grpc_core::FakeResolverResponseGenerator* response_generator = nullptr) {
2144     std::vector<int> ports;
2145     for (size_t i = 0; i < balancers_.size(); ++i) {
2146       ports.emplace_back(balancers_[i]->port());
2147     }
2148     SetNextResolutionForLbChannel(ports, service_config_json, expected_targets,
2149                                   response_generator);
2150   }
2151
2152   void SetNextResolutionForLbChannel(
2153       const std::vector<int>& ports, const char* service_config_json = nullptr,
2154       const char* expected_targets = nullptr,
2155       grpc_core::FakeResolverResponseGenerator* response_generator = nullptr) {
2156     grpc_core::ExecCtx exec_ctx;
2157     grpc_core::Resolver::Result result;
2158     result.addresses = CreateAddressListFromPortList(ports);
2159     if (service_config_json != nullptr) {
2160       grpc_error_handle error = GRPC_ERROR_NONE;
2161       result.service_config = grpc_core::ServiceConfig::Create(
2162           nullptr, service_config_json, &error);
2163       ASSERT_NE(result.service_config.get(), nullptr);
2164       ASSERT_EQ(error, GRPC_ERROR_NONE) << grpc_error_std_string(error);
2165     }
2166     if (expected_targets != nullptr) {
2167       grpc_arg expected_targets_arg = grpc_channel_arg_string_create(
2168           const_cast<char*>(GRPC_ARG_FAKE_SECURITY_EXPECTED_TARGETS),
2169           const_cast<char*>(expected_targets));
2170       result.args =
2171           grpc_channel_args_copy_and_add(nullptr, &expected_targets_arg, 1);
2172     }
2173     if (response_generator == nullptr) {
2174       response_generator = lb_channel_response_generator_.get();
2175     }
2176     response_generator->SetResponse(std::move(result));
2177   }
2178
2179   void SetNextReresolutionResponse(const std::vector<int>& ports) {
2180     grpc_core::ExecCtx exec_ctx;
2181     grpc_core::Resolver::Result result;
2182     result.addresses = CreateAddressListFromPortList(ports);
2183     response_generator_->SetReresolutionResponse(std::move(result));
2184   }
2185
2186   std::vector<int> GetBackendPorts(size_t start_index = 0,
2187                                    size_t stop_index = 0) const {
2188     if (stop_index == 0) stop_index = backends_.size();
2189     std::vector<int> backend_ports;
2190     for (size_t i = start_index; i < stop_index; ++i) {
2191       backend_ports.push_back(backends_[i]->port());
2192     }
2193     return backend_ports;
2194   }
2195
2196   Status SendRpc(const RpcOptions& rpc_options = RpcOptions(),
2197                  EchoResponse* response = nullptr) {
2198     const bool local_response = (response == nullptr);
2199     if (local_response) response = new EchoResponse;
2200     ClientContext context;
2201     EchoRequest request;
2202     if (rpc_options.server_expected_error != StatusCode::OK) {
2203       auto* error = request.mutable_param()->mutable_expected_error();
2204       error->set_code(rpc_options.server_expected_error);
2205     }
2206     rpc_options.SetupRpc(&context, &request);
2207     Status status;
2208     switch (rpc_options.service) {
2209       case SERVICE_ECHO:
2210         status =
2211             SendRpcMethod(&stub_, rpc_options, &context, request, response);
2212         break;
2213       case SERVICE_ECHO1:
2214         status =
2215             SendRpcMethod(&stub1_, rpc_options, &context, request, response);
2216         break;
2217       case SERVICE_ECHO2:
2218         status =
2219             SendRpcMethod(&stub2_, rpc_options, &context, request, response);
2220         break;
2221     }
2222     if (local_response) delete response;
2223     return status;
2224   }
2225
2226   void CheckRpcSendOk(const size_t times = 1,
2227                       const RpcOptions& rpc_options = RpcOptions()) {
2228     for (size_t i = 0; i < times; ++i) {
2229       EchoResponse response;
2230       const Status status = SendRpc(rpc_options, &response);
2231       EXPECT_TRUE(status.ok()) << "code=" << status.error_code()
2232                                << " message=" << status.error_message();
2233       EXPECT_EQ(response.message(), kRequestMessage);
2234     }
2235   }
2236
2237   struct CheckRpcSendFailureOptions {
2238     std::function<bool(size_t)> continue_predicate = [](size_t i) {
2239       return i < 1;
2240     };
2241     RpcOptions rpc_options;
2242     StatusCode expected_error_code = StatusCode::OK;
2243
2244     CheckRpcSendFailureOptions() {}
2245
2246     CheckRpcSendFailureOptions& set_times(size_t times) {
2247       continue_predicate = [times](size_t i) { return i < times; };
2248       return *this;
2249     }
2250
2251     CheckRpcSendFailureOptions& set_continue_predicate(
2252         std::function<bool(size_t)> pred) {
2253       continue_predicate = std::move(pred);
2254       return *this;
2255     }
2256
2257     CheckRpcSendFailureOptions& set_rpc_options(const RpcOptions& options) {
2258       rpc_options = options;
2259       return *this;
2260     }
2261
2262     CheckRpcSendFailureOptions& set_expected_error_code(StatusCode code) {
2263       expected_error_code = code;
2264       return *this;
2265     }
2266   };
2267
2268   void CheckRpcSendFailure(const CheckRpcSendFailureOptions& options =
2269                                CheckRpcSendFailureOptions()) {
2270     for (size_t i = 0; options.continue_predicate(i); ++i) {
2271       const Status status = SendRpc(options.rpc_options);
2272       EXPECT_FALSE(status.ok());
2273       if (options.expected_error_code != StatusCode::OK) {
2274         EXPECT_EQ(options.expected_error_code, status.error_code());
2275       }
2276     }
2277   }
2278
2279   bool WaitForNack(
2280       std::function<AdsServiceImpl::ResponseState::State()> get_state,
2281       StatusCode expected_status = StatusCode::UNAVAILABLE) {
2282     auto deadline = absl::Now() + absl::Seconds(30);
2283     bool success = true;
2284     CheckRpcSendFailure(CheckRpcSendFailureOptions()
2285                             .set_continue_predicate([&](size_t) {
2286                               if (absl::Now() >= deadline) {
2287                                 success = false;
2288                                 return false;
2289                               }
2290                               return get_state() !=
2291                                      AdsServiceImpl::ResponseState::NACKED;
2292                             })
2293                             .set_expected_error_code(expected_status));
2294     return success;
2295   }
2296
2297   bool WaitForLdsNack(StatusCode expected_status = StatusCode::UNAVAILABLE) {
2298     return WaitForNack(
2299         [&]() {
2300           return balancers_[0]->ads_service()->lds_response_state().state;
2301         },
2302         expected_status);
2303   }
2304
2305   bool WaitForRdsNack() {
2306     return WaitForNack(
2307         [&]() { return RouteConfigurationResponseState(0).state; });
2308   }
2309
2310   bool WaitForCdsNack() {
2311     return WaitForNack([&]() {
2312       return balancers_[0]->ads_service()->cds_response_state().state;
2313     });
2314   }
2315
2316   bool WaitForEdsNack() {
2317     return WaitForNack([&]() {
2318       return balancers_[0]->ads_service()->eds_response_state().state;
2319     });
2320   }
2321
2322   static Listener BuildListener(const RouteConfiguration& route_config) {
2323     HttpConnectionManager http_connection_manager;
2324     *(http_connection_manager.mutable_route_config()) = route_config;
2325     auto* filter = http_connection_manager.add_http_filters();
2326     filter->set_name("router");
2327     filter->mutable_typed_config()->PackFrom(
2328         envoy::extensions::filters::http::router::v3::Router());
2329     Listener listener;
2330     listener.set_name(kServerName);
2331     listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
2332         http_connection_manager);
2333     return listener;
2334   }
2335
2336   AdsServiceImpl::EdsResourceArgs::Endpoint CreateEndpoint(
2337       size_t backend_idx, HealthStatus health_status = HealthStatus::UNKNOWN,
2338       int lb_weight = 1) {
2339     return AdsServiceImpl::EdsResourceArgs::Endpoint(
2340         backends_[backend_idx]->port(), health_status, lb_weight);
2341   }
2342
2343   std::vector<AdsServiceImpl::EdsResourceArgs::Endpoint>
2344   CreateEndpointsForBackends(size_t start_index = 0, size_t stop_index = 0,
2345                              HealthStatus health_status = HealthStatus::UNKNOWN,
2346                              int lb_weight = 1) {
2347     if (stop_index == 0) stop_index = backends_.size();
2348     std::vector<AdsServiceImpl::EdsResourceArgs::Endpoint> endpoints;
2349     for (size_t i = start_index; i < stop_index; ++i) {
2350       endpoints.emplace_back(CreateEndpoint(i, health_status, lb_weight));
2351     }
2352     return endpoints;
2353   }
2354
2355   AdsServiceImpl::EdsResourceArgs::Endpoint MakeNonExistantEndpoint() {
2356     return AdsServiceImpl::EdsResourceArgs::Endpoint(
2357         grpc_pick_unused_port_or_die());
2358   }
2359
2360   ClusterLoadAssignment BuildEdsResource(
2361       const AdsServiceImpl::EdsResourceArgs& args,
2362       const char* eds_service_name = kDefaultEdsServiceName) {
2363     ClusterLoadAssignment assignment;
2364     assignment.set_cluster_name(eds_service_name);
2365     for (const auto& locality : args.locality_list) {
2366       auto* endpoints = assignment.add_endpoints();
2367       endpoints->mutable_load_balancing_weight()->set_value(locality.lb_weight);
2368       endpoints->set_priority(locality.priority);
2369       endpoints->mutable_locality()->set_region(kDefaultLocalityRegion);
2370       endpoints->mutable_locality()->set_zone(kDefaultLocalityZone);
2371       endpoints->mutable_locality()->set_sub_zone(locality.sub_zone);
2372       for (size_t i = 0; i < locality.endpoints.size(); ++i) {
2373         const int& port = locality.endpoints[i].port;
2374         auto* lb_endpoints = endpoints->add_lb_endpoints();
2375         if (locality.endpoints.size() > i &&
2376             locality.endpoints[i].health_status != HealthStatus::UNKNOWN) {
2377           lb_endpoints->set_health_status(locality.endpoints[i].health_status);
2378         }
2379         if (locality.endpoints.size() > i &&
2380             locality.endpoints[i].lb_weight >= 1) {
2381           lb_endpoints->mutable_load_balancing_weight()->set_value(
2382               locality.endpoints[i].lb_weight);
2383         }
2384         auto* endpoint = lb_endpoints->mutable_endpoint();
2385         auto* address = endpoint->mutable_address();
2386         auto* socket_address = address->mutable_socket_address();
2387         socket_address->set_address(ipv6_only_ ? "::1" : "127.0.0.1");
2388         socket_address->set_port_value(port);
2389       }
2390     }
2391     if (!args.drop_categories.empty()) {
2392       auto* policy = assignment.mutable_policy();
2393       for (const auto& p : args.drop_categories) {
2394         const std::string& name = p.first;
2395         const uint32_t parts_per_million = p.second;
2396         auto* drop_overload = policy->add_drop_overloads();
2397         drop_overload->set_category(name);
2398         auto* drop_percentage = drop_overload->mutable_drop_percentage();
2399         drop_percentage->set_numerator(parts_per_million);
2400         drop_percentage->set_denominator(args.drop_denominator);
2401       }
2402     }
2403     return assignment;
2404   }
2405
2406   void SetListenerAndRouteConfiguration(
2407       int idx, Listener listener, const RouteConfiguration& route_config) {
2408     auto* api_listener =
2409         listener.mutable_api_listener()->mutable_api_listener();
2410     HttpConnectionManager http_connection_manager;
2411     api_listener->UnpackTo(&http_connection_manager);
2412     if (GetParam().enable_rds_testing()) {
2413       auto* rds = http_connection_manager.mutable_rds();
2414       rds->set_route_config_name(kDefaultRouteConfigurationName);
2415       rds->mutable_config_source()->mutable_ads();
2416       balancers_[idx]->ads_service()->SetRdsResource(route_config);
2417     } else {
2418       *http_connection_manager.mutable_route_config() = route_config;
2419     }
2420     api_listener->PackFrom(http_connection_manager);
2421     balancers_[idx]->ads_service()->SetLdsResource(listener);
2422   }
2423
2424   void SetRouteConfiguration(int idx, const RouteConfiguration& route_config) {
2425     if (GetParam().enable_rds_testing()) {
2426       balancers_[idx]->ads_service()->SetRdsResource(route_config);
2427     } else {
2428       balancers_[idx]->ads_service()->SetLdsResource(
2429           BuildListener(route_config));
2430     }
2431   }
2432
2433   AdsServiceImpl::ResponseState RouteConfigurationResponseState(int idx) const {
2434     AdsServiceImpl* ads_service = balancers_[idx]->ads_service();
2435     if (GetParam().enable_rds_testing()) {
2436       return ads_service->rds_response_state();
2437     }
2438     return ads_service->lds_response_state();
2439   }
2440
2441  public:
2442   // This method could benefit test subclasses; to make it accessible
2443   // via bind with a qualified name, it needs to be public.
2444   void SetEdsResourceWithDelay(size_t i,
2445                                const ClusterLoadAssignment& assignment,
2446                                int delay_ms) {
2447     GPR_ASSERT(delay_ms > 0);
2448     gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(delay_ms));
2449     balancers_[i]->ads_service()->SetEdsResource(assignment);
2450   }
2451
2452  protected:
2453   class XdsServingStatusNotifier
2454       : public grpc::experimental::XdsServerServingStatusNotifierInterface {
2455    public:
2456     void OnServingStatusUpdate(std::string uri, grpc::Status status) override {
2457       grpc_core::MutexLock lock(&mu_);
2458       status_map[uri] = status;
2459       cond_.Signal();
2460     }
2461
2462     void WaitOnServingStatusChange(std::string uri,
2463                                    grpc::StatusCode expected_status) {
2464       grpc_core::MutexLock lock(&mu_);
2465       std::map<std::string, grpc::Status>::iterator it;
2466       while ((it = status_map.find(uri)) == status_map.end() ||
2467              it->second.error_code() != expected_status) {
2468         cond_.Wait(&mu_);
2469       }
2470     }
2471
2472    private:
2473     grpc_core::Mutex mu_;
2474     grpc_core::CondVar cond_;
2475     std::map<std::string, grpc::Status> status_map;
2476   };
2477
2478   class ServerThread {
2479    public:
2480     explicit ServerThread(XdsEnd2endTest* test_obj,
2481                           bool use_xds_enabled_server = false)
2482         : test_obj_(test_obj),
2483           port_(grpc_pick_unused_port_or_die()),
2484           use_xds_enabled_server_(use_xds_enabled_server) {}
2485     virtual ~ServerThread(){};
2486
2487     void Start() {
2488       gpr_log(GPR_INFO, "starting %s server on port %d", Type(), port_);
2489       GPR_ASSERT(!running_);
2490       running_ = true;
2491       StartAllServices();
2492       grpc_core::Mutex mu;
2493       // We need to acquire the lock here in order to prevent the notify_one
2494       // by ServerThread::Serve from firing before the wait below is hit.
2495       grpc_core::MutexLock lock(&mu);
2496       grpc_core::CondVar cond;
2497       thread_ = absl::make_unique<std::thread>(
2498           std::bind(&ServerThread::Serve, this, &mu, &cond));
2499       cond.Wait(&mu);
2500       gpr_log(GPR_INFO, "%s server startup complete", Type());
2501     }
2502
2503     void Serve(grpc_core::Mutex* mu, grpc_core::CondVar* cond) {
2504       // We need to acquire the lock here in order to prevent the notify_one
2505       // below from firing before its corresponding wait is executed.
2506       grpc_core::MutexLock lock(mu);
2507       std::ostringstream server_address;
2508       server_address << "localhost:" << port_;
2509       if (use_xds_enabled_server_) {
2510         experimental::XdsServerBuilder builder;
2511         if (GetParam().bootstrap_source() ==
2512             TestType::kBootstrapFromChannelArg) {
2513           builder.SetOption(
2514               absl::make_unique<XdsChannelArgsServerBuilderOption>(test_obj_));
2515         }
2516         builder.set_status_notifier(&notifier_);
2517         builder.AddListeningPort(server_address.str(), Credentials());
2518         RegisterAllServices(&builder);
2519         server_ = builder.BuildAndStart();
2520       } else {
2521         ServerBuilder builder;
2522         builder.AddListeningPort(server_address.str(), Credentials());
2523         RegisterAllServices(&builder);
2524         server_ = builder.BuildAndStart();
2525       }
2526       cond->Signal();
2527     }
2528
2529     void Shutdown() {
2530       if (!running_) return;
2531       gpr_log(GPR_INFO, "%s about to shutdown", Type());
2532       ShutdownAllServices();
2533       server_->Shutdown(grpc_timeout_milliseconds_to_deadline(0));
2534       thread_->join();
2535       gpr_log(GPR_INFO, "%s shutdown completed", Type());
2536       running_ = false;
2537     }
2538
2539     virtual std::shared_ptr<ServerCredentials> Credentials() {
2540       return std::make_shared<SecureServerCredentials>(
2541           grpc_fake_transport_security_server_credentials_create());
2542     }
2543
2544     int port() const { return port_; }
2545
2546     bool use_xds_enabled_server() const { return use_xds_enabled_server_; }
2547
2548     XdsServingStatusNotifier* notifier() { return &notifier_; }
2549
2550    private:
2551     class XdsChannelArgsServerBuilderOption
2552         : public ::grpc::ServerBuilderOption {
2553      public:
2554       explicit XdsChannelArgsServerBuilderOption(XdsEnd2endTest* test_obj)
2555           : test_obj_(test_obj) {}
2556
2557       void UpdateArguments(grpc::ChannelArguments* args) override {
2558         args->SetString(
2559             GRPC_ARG_TEST_ONLY_DO_NOT_USE_IN_PROD_XDS_BOOTSTRAP_CONFIG,
2560             GetParam().use_v2() ? kBootstrapFileV2 : kBootstrapFileV3);
2561         args->SetPointerWithVtable(
2562             GRPC_ARG_TEST_ONLY_DO_NOT_USE_IN_PROD_XDS_CLIENT_CHANNEL_ARGS,
2563             &test_obj_->xds_channel_args_, &kChannelArgsArgVtable);
2564       }
2565
2566       void UpdatePlugins(
2567           std::vector<std::unique_ptr<grpc::ServerBuilderPlugin>>* /*plugins*/)
2568           override {}
2569
2570      private:
2571       XdsEnd2endTest* test_obj_;
2572     };
2573
2574     virtual void RegisterAllServices(ServerBuilder* builder) = 0;
2575     virtual void StartAllServices() = 0;
2576     virtual void ShutdownAllServices() = 0;
2577
2578     virtual const char* Type() = 0;
2579
2580     XdsEnd2endTest* test_obj_;
2581     const int port_;
2582     std::unique_ptr<Server> server_;
2583     XdsServingStatusNotifier notifier_;
2584     std::unique_ptr<std::thread> thread_;
2585     bool running_ = false;
2586     const bool use_xds_enabled_server_;
2587   };
2588
2589   class BackendServerThread : public ServerThread {
2590    public:
2591     explicit BackendServerThread(XdsEnd2endTest* test_obj,
2592                                  bool use_xds_enabled_server)
2593         : ServerThread(test_obj, use_xds_enabled_server) {}
2594
2595     BackendServiceImpl<::grpc::testing::EchoTestService::Service>*
2596     backend_service() {
2597       return &backend_service_;
2598     }
2599     BackendServiceImpl<::grpc::testing::EchoTest1Service::Service>*
2600     backend_service1() {
2601       return &backend_service1_;
2602     }
2603     BackendServiceImpl<::grpc::testing::EchoTest2Service::Service>*
2604     backend_service2() {
2605       return &backend_service2_;
2606     }
2607
2608     std::shared_ptr<ServerCredentials> Credentials() override {
2609       if (GetParam().use_xds_credentials()) {
2610         if (use_xds_enabled_server()) {
2611           // We are testing server's use of XdsServerCredentials
2612           return experimental::XdsServerCredentials(
2613               InsecureServerCredentials());
2614         } else {
2615           // We are testing client's use of XdsCredentials
2616           std::string root_cert = ReadFile(kCaCertPath);
2617           std::string identity_cert = ReadFile(kServerCertPath);
2618           std::string private_key = ReadFile(kServerKeyPath);
2619           std::vector<experimental::IdentityKeyCertPair>
2620               identity_key_cert_pairs = {{private_key, identity_cert}};
2621           auto certificate_provider = std::make_shared<
2622               grpc::experimental::StaticDataCertificateProvider>(
2623               root_cert, identity_key_cert_pairs);
2624           grpc::experimental::TlsServerCredentialsOptions options(
2625               certificate_provider);
2626           options.watch_root_certs();
2627           options.watch_identity_key_cert_pairs();
2628           options.set_cert_request_type(
2629               GRPC_SSL_REQUEST_CLIENT_CERTIFICATE_AND_VERIFY);
2630           return grpc::experimental::TlsServerCredentials(options);
2631         }
2632       }
2633       return ServerThread::Credentials();
2634     }
2635
2636    private:
2637     void RegisterAllServices(ServerBuilder* builder) override {
2638       builder->RegisterService(&backend_service_);
2639       builder->RegisterService(&backend_service1_);
2640       builder->RegisterService(&backend_service2_);
2641     }
2642
2643     void StartAllServices() override {
2644       backend_service_.Start();
2645       backend_service1_.Start();
2646       backend_service2_.Start();
2647     }
2648
2649     void ShutdownAllServices() override {
2650       backend_service_.Shutdown();
2651       backend_service1_.Shutdown();
2652       backend_service2_.Shutdown();
2653     }
2654
2655     const char* Type() override { return "Backend"; }
2656
2657     BackendServiceImpl<::grpc::testing::EchoTestService::Service>
2658         backend_service_;
2659     BackendServiceImpl<::grpc::testing::EchoTest1Service::Service>
2660         backend_service1_;
2661     BackendServiceImpl<::grpc::testing::EchoTest2Service::Service>
2662         backend_service2_;
2663   };
2664
2665   class BalancerServerThread : public ServerThread {
2666    public:
2667     explicit BalancerServerThread(XdsEnd2endTest* test_obj,
2668                                   int client_load_reporting_interval = 0)
2669         : ServerThread(test_obj),
2670           ads_service_(new AdsServiceImpl()),
2671           lrs_service_(new LrsServiceImpl(client_load_reporting_interval)) {}
2672
2673     AdsServiceImpl* ads_service() { return ads_service_.get(); }
2674     LrsServiceImpl* lrs_service() { return lrs_service_.get(); }
2675
2676    private:
2677     void RegisterAllServices(ServerBuilder* builder) override {
2678       builder->RegisterService(ads_service_->v2_rpc_service());
2679       builder->RegisterService(ads_service_->v3_rpc_service());
2680       builder->RegisterService(lrs_service_->v2_rpc_service());
2681       builder->RegisterService(lrs_service_->v3_rpc_service());
2682     }
2683
2684     void StartAllServices() override {
2685       ads_service_->Start();
2686       lrs_service_->Start();
2687     }
2688
2689     void ShutdownAllServices() override {
2690       ads_service_->Shutdown();
2691       lrs_service_->Shutdown();
2692     }
2693
2694     const char* Type() override { return "Balancer"; }
2695
2696     std::shared_ptr<AdsServiceImpl> ads_service_;
2697     std::shared_ptr<LrsServiceImpl> lrs_service_;
2698   };
2699
2700 #ifndef DISABLED_XDS_PROTO_IN_CC
2701   class AdminServerThread : public ServerThread {
2702    public:
2703     explicit AdminServerThread(XdsEnd2endTest* test_obj)
2704         : ServerThread(test_obj) {}
2705
2706    private:
2707     void RegisterAllServices(ServerBuilder* builder) override {
2708       builder->RegisterService(&csds_service_);
2709     }
2710     void StartAllServices() override {}
2711     void ShutdownAllServices() override {}
2712
2713     const char* Type() override { return "Admin"; }
2714
2715     grpc::xds::experimental::ClientStatusDiscoveryService csds_service_;
2716   };
2717 #endif  // DISABLED_XDS_PROTO_IN_CC
2718
2719   class LongRunningRpc {
2720    public:
2721     void StartRpc(grpc::testing::EchoTestService::Stub* stub,
2722                   const RpcOptions& rpc_options =
2723                       RpcOptions().set_timeout_ms(0).set_client_cancel_after_us(
2724                           1 * 1000 * 1000)) {
2725       sender_thread_ = std::thread([this, stub, rpc_options]() {
2726         EchoRequest request;
2727         EchoResponse response;
2728         rpc_options.SetupRpc(&context_, &request);
2729         status_ = stub->Echo(&context_, request, &response);
2730       });
2731     }
2732
2733     void CancelRpc() {
2734       context_.TryCancel();
2735       if (sender_thread_.joinable()) sender_thread_.join();
2736     }
2737
2738     Status GetStatus() {
2739       if (sender_thread_.joinable()) sender_thread_.join();
2740       return status_;
2741     }
2742
2743    private:
2744     std::thread sender_thread_;
2745     ClientContext context_;
2746     Status status_;
2747   };
2748
2749   struct ConcurrentRpc {
2750     ClientContext context;
2751     Status status;
2752     grpc_millis elapsed_time;
2753     EchoResponse response;
2754   };
2755
2756   std::vector<ConcurrentRpc> SendConcurrentRpcs(
2757       grpc::testing::EchoTestService::Stub* stub, size_t num_rpcs,
2758       const RpcOptions& rpc_options) {
2759     // Variables for RPCs.
2760     std::vector<ConcurrentRpc> rpcs(num_rpcs);
2761     EchoRequest request;
2762     // Variables for synchronization
2763     absl::Mutex mu;
2764     absl::CondVar cv;
2765     size_t completed = 0;
2766     // Set-off callback RPCs
2767     for (size_t i = 0; i < num_rpcs; i++) {
2768       ConcurrentRpc* rpc = &rpcs[i];
2769       rpc_options.SetupRpc(&rpc->context, &request);
2770       grpc_millis t0 = NowFromCycleCounter();
2771       stub->async()->Echo(&rpc->context, &request, &rpc->response,
2772                           [rpc, &mu, &completed, &cv, num_rpcs, t0](Status s) {
2773                             rpc->status = s;
2774                             rpc->elapsed_time = NowFromCycleCounter() - t0;
2775                             bool done;
2776                             {
2777                               absl::MutexLock lock(&mu);
2778                               done = (++completed) == num_rpcs;
2779                             }
2780                             if (done) cv.Signal();
2781                           });
2782     }
2783     {
2784       absl::MutexLock lock(&mu);
2785       cv.Wait(&mu);
2786     }
2787     EXPECT_EQ(completed, num_rpcs);
2788     return rpcs;
2789   }
2790
2791   const size_t num_backends_;
2792   const size_t num_balancers_;
2793   const int client_load_reporting_interval_seconds_;
2794   bool ipv6_only_ = false;
2795   std::shared_ptr<Channel> channel_;
2796   std::unique_ptr<grpc::testing::EchoTestService::Stub> stub_;
2797   std::unique_ptr<grpc::testing::EchoTest1Service::Stub> stub1_;
2798   std::unique_ptr<grpc::testing::EchoTest2Service::Stub> stub2_;
2799   std::vector<std::unique_ptr<BackendServerThread>> backends_;
2800   std::vector<std::unique_ptr<BalancerServerThread>> balancers_;
2801   grpc_core::RefCountedPtr<grpc_core::FakeResolverResponseGenerator>
2802       response_generator_;
2803   grpc_core::RefCountedPtr<grpc_core::FakeResolverResponseGenerator>
2804       lb_channel_response_generator_;
2805   grpc_core::RefCountedPtr<grpc_core::FakeResolverResponseGenerator>
2806       logical_dns_cluster_resolver_response_generator_;
2807   int xds_resource_does_not_exist_timeout_ms_ = 0;
2808   absl::InlinedVector<grpc_arg, 2> xds_channel_args_to_add_;
2809   grpc_channel_args xds_channel_args_;
2810
2811   Listener default_listener_;
2812   RouteConfiguration default_route_config_;
2813   Cluster default_cluster_;
2814   bool use_xds_enabled_server_;
2815   bool bootstrap_contents_from_env_var_;
2816 };
2817
2818 class BasicTest : public XdsEnd2endTest {
2819  public:
2820   BasicTest() : XdsEnd2endTest(4, 1) {}
2821 };
2822
2823 // Tests that the balancer sends the correct response to the client, and the
2824 // client sends RPCs to the backends using the default child policy.
2825 TEST_P(BasicTest, Vanilla) {
2826   SetNextResolution({});
2827   SetNextResolutionForLbChannelAllBalancers();
2828   const size_t kNumRpcsPerAddress = 100;
2829   AdsServiceImpl::EdsResourceArgs args({
2830       {"locality0", CreateEndpointsForBackends()},
2831   });
2832   balancers_[0]->ads_service()->SetEdsResource(
2833       BuildEdsResource(args, DefaultEdsServiceName()));
2834   // Make sure that trying to connect works without a call.
2835   channel_->GetState(true /* try_to_connect */);
2836   // We need to wait for all backends to come online.
2837   WaitForAllBackends();
2838   // Send kNumRpcsPerAddress RPCs per server.
2839   CheckRpcSendOk(kNumRpcsPerAddress * num_backends_);
2840   // Each backend should have gotten 100 requests.
2841   for (size_t i = 0; i < backends_.size(); ++i) {
2842     EXPECT_EQ(kNumRpcsPerAddress,
2843               backends_[i]->backend_service()->request_count());
2844   }
2845   // Check LB policy name for the channel.
2846   EXPECT_EQ(
2847       (GetParam().use_fake_resolver() ? "xds_cluster_resolver_experimental"
2848                                       : "xds_cluster_manager_experimental"),
2849       channel_->GetLoadBalancingPolicyName());
2850 }
2851
2852 TEST_P(BasicTest, IgnoresUnhealthyEndpoints) {
2853   SetNextResolution({});
2854   SetNextResolutionForLbChannelAllBalancers();
2855   const size_t kNumRpcsPerAddress = 100;
2856   auto endpoints = CreateEndpointsForBackends();
2857   endpoints[0].health_status = HealthStatus::DRAINING;
2858   AdsServiceImpl::EdsResourceArgs args({
2859       {"locality0", std::move(endpoints), kDefaultLocalityWeight,
2860        kDefaultLocalityPriority},
2861   });
2862   balancers_[0]->ads_service()->SetEdsResource(
2863       BuildEdsResource(args, DefaultEdsServiceName()));
2864   // Make sure that trying to connect works without a call.
2865   channel_->GetState(true /* try_to_connect */);
2866   // We need to wait for all backends to come online.
2867   WaitForAllBackends(/*start_index=*/1);
2868   // Send kNumRpcsPerAddress RPCs per server.
2869   CheckRpcSendOk(kNumRpcsPerAddress * (num_backends_ - 1));
2870   // Each backend should have gotten 100 requests.
2871   for (size_t i = 1; i < backends_.size(); ++i) {
2872     EXPECT_EQ(kNumRpcsPerAddress,
2873               backends_[i]->backend_service()->request_count());
2874   }
2875 }
2876
2877 // Tests that subchannel sharing works when the same backend is listed
2878 // multiple times.
2879 TEST_P(BasicTest, SameBackendListedMultipleTimes) {
2880   SetNextResolution({});
2881   SetNextResolutionForLbChannelAllBalancers();
2882   // Same backend listed twice.
2883   auto endpoints = CreateEndpointsForBackends(0, 1);
2884   endpoints.push_back(endpoints.front());
2885   AdsServiceImpl::EdsResourceArgs args({
2886       {"locality0", endpoints},
2887   });
2888   const size_t kNumRpcsPerAddress = 10;
2889   balancers_[0]->ads_service()->SetEdsResource(
2890       BuildEdsResource(args, DefaultEdsServiceName()));
2891   // We need to wait for the backend to come online.
2892   WaitForBackend(0);
2893   // Send kNumRpcsPerAddress RPCs per server.
2894   CheckRpcSendOk(kNumRpcsPerAddress * endpoints.size());
2895   // Backend should have gotten 20 requests.
2896   EXPECT_EQ(kNumRpcsPerAddress * endpoints.size(),
2897             backends_[0]->backend_service()->request_count());
2898   // And they should have come from a single client port, because of
2899   // subchannel sharing.
2900   EXPECT_EQ(1UL, backends_[0]->backend_service()->clients().size());
2901 }
2902
2903 // Tests that RPCs will be blocked until a non-empty serverlist is received.
2904 TEST_P(BasicTest, InitiallyEmptyServerlist) {
2905   SetNextResolution({});
2906   SetNextResolutionForLbChannelAllBalancers();
2907   const int kServerlistDelayMs = 500 * grpc_test_slowdown_factor();
2908   const int kCallDeadlineMs = kServerlistDelayMs * 2;
2909   // First response is an empty serverlist, sent right away.
2910   AdsServiceImpl::EdsResourceArgs::Locality empty_locality("locality0", {});
2911   AdsServiceImpl::EdsResourceArgs args({
2912       empty_locality,
2913   });
2914   balancers_[0]->ads_service()->SetEdsResource(
2915       BuildEdsResource(args, DefaultEdsServiceName()));
2916   // Send non-empty serverlist only after kServerlistDelayMs.
2917   args = AdsServiceImpl::EdsResourceArgs({
2918       {"locality0", CreateEndpointsForBackends()},
2919   });
2920   std::thread delayed_resource_setter(std::bind(
2921       &BasicTest::SetEdsResourceWithDelay, this, 0,
2922       BuildEdsResource(args, DefaultEdsServiceName()), kServerlistDelayMs));
2923   const auto t0 = system_clock::now();
2924   // Client will block: LB will initially send empty serverlist.
2925   CheckRpcSendOk(
2926       1, RpcOptions().set_timeout_ms(kCallDeadlineMs).set_wait_for_ready(true));
2927   const auto ellapsed_ms =
2928       std::chrono::duration_cast<std::chrono::milliseconds>(
2929           system_clock::now() - t0);
2930   // but eventually, the LB sends a serverlist update that allows the call to
2931   // proceed. The call delay must be larger than the delay in sending the
2932   // populated serverlist but under the call's deadline (which is enforced by
2933   // the call's deadline).
2934   EXPECT_GT(ellapsed_ms.count(), kServerlistDelayMs);
2935   delayed_resource_setter.join();
2936 }
2937
2938 // Tests that RPCs will fail with UNAVAILABLE instead of DEADLINE_EXCEEDED if
2939 // all the servers are unreachable.
2940 TEST_P(BasicTest, AllServersUnreachableFailFast) {
2941   // Set Rpc timeout to 5 seconds to ensure there is enough time
2942   // for communication with the xDS server to take place upon test start up.
2943   const uint32_t kRpcTimeoutMs = 5000;
2944   SetNextResolution({});
2945   SetNextResolutionForLbChannelAllBalancers();
2946   const size_t kNumUnreachableServers = 5;
2947   std::vector<AdsServiceImpl::EdsResourceArgs::Endpoint> endpoints;
2948   for (size_t i = 0; i < kNumUnreachableServers; ++i) {
2949     endpoints.emplace_back(grpc_pick_unused_port_or_die());
2950   }
2951   AdsServiceImpl::EdsResourceArgs args({
2952       {"locality0", endpoints},
2953   });
2954   balancers_[0]->ads_service()->SetEdsResource(
2955       BuildEdsResource(args, DefaultEdsServiceName()));
2956   const Status status = SendRpc(RpcOptions().set_timeout_ms(kRpcTimeoutMs));
2957   // The error shouldn't be DEADLINE_EXCEEDED because timeout is set to 5
2958   // seconds, and we should disocver in that time that the target backend is
2959   // down.
2960   EXPECT_EQ(StatusCode::UNAVAILABLE, status.error_code());
2961 }
2962
2963 // Tests that RPCs fail when the backends are down, and will succeed again
2964 // after the backends are restarted.
2965 TEST_P(BasicTest, BackendsRestart) {
2966   SetNextResolution({});
2967   SetNextResolutionForLbChannelAllBalancers();
2968   AdsServiceImpl::EdsResourceArgs args({
2969       {"locality0", CreateEndpointsForBackends()},
2970   });
2971   balancers_[0]->ads_service()->SetEdsResource(
2972       BuildEdsResource(args, DefaultEdsServiceName()));
2973   WaitForAllBackends();
2974   // Stop backends.  RPCs should fail.
2975   ShutdownAllBackends();
2976   // Sending multiple failed requests instead of just one to ensure that the
2977   // client notices that all backends are down before we restart them. If we
2978   // didn't do this, then a single RPC could fail here due to the race
2979   // condition between the LB pick and the GOAWAY from the chosen backend
2980   // being shut down, which would not actually prove that the client noticed
2981   // that all of the backends are down. Then, when we send another request
2982   // below (which we expect to succeed), if the callbacks happen in the wrong
2983   // order, the same race condition could happen again due to the client not
2984   // yet having noticed that the backends were all down.
2985   CheckRpcSendFailure(CheckRpcSendFailureOptions().set_times(num_backends_));
2986   // Restart all backends.  RPCs should start succeeding again.
2987   StartAllBackends();
2988   CheckRpcSendOk(1, RpcOptions().set_timeout_ms(2000).set_wait_for_ready(true));
2989 }
2990
2991 TEST_P(BasicTest, IgnoresDuplicateUpdates) {
2992   const size_t kNumRpcsPerAddress = 100;
2993   SetNextResolution({});
2994   SetNextResolutionForLbChannelAllBalancers();
2995   AdsServiceImpl::EdsResourceArgs args({
2996       {"locality0", CreateEndpointsForBackends()},
2997   });
2998   balancers_[0]->ads_service()->SetEdsResource(
2999       BuildEdsResource(args, DefaultEdsServiceName()));
3000   // Wait for all backends to come online.
3001   WaitForAllBackends();
3002   // Send kNumRpcsPerAddress RPCs per server, but send an EDS update in
3003   // between.  If the update is not ignored, this will cause the
3004   // round_robin policy to see an update, which will randomly reset its
3005   // position in the address list.
3006   for (size_t i = 0; i < kNumRpcsPerAddress; ++i) {
3007     CheckRpcSendOk(2);
3008     balancers_[0]->ads_service()->SetEdsResource(
3009         BuildEdsResource(args, DefaultEdsServiceName()));
3010     CheckRpcSendOk(2);
3011   }
3012   // Each backend should have gotten the right number of requests.
3013   for (size_t i = 1; i < backends_.size(); ++i) {
3014     EXPECT_EQ(kNumRpcsPerAddress,
3015               backends_[i]->backend_service()->request_count());
3016   }
3017 }
3018
3019 using XdsResolverOnlyTest = BasicTest;
3020
3021 TEST_P(XdsResolverOnlyTest, ResourceTypeVersionPersistsAcrossStreamRestarts) {
3022   SetNextResolution({});
3023   SetNextResolutionForLbChannelAllBalancers();
3024   AdsServiceImpl::EdsResourceArgs args({
3025       {"locality0", CreateEndpointsForBackends(0, 1)},
3026   });
3027   balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
3028   // Wait for backends to come online.
3029   WaitForAllBackends(0, 1);
3030   // Stop balancer.
3031   balancers_[0]->Shutdown();
3032   // Tell balancer to require minimum version 1 for all resource types.
3033   balancers_[0]->ads_service()->SetResourceMinVersion(kLdsTypeUrl, 1);
3034   balancers_[0]->ads_service()->SetResourceMinVersion(kRdsTypeUrl, 1);
3035   balancers_[0]->ads_service()->SetResourceMinVersion(kCdsTypeUrl, 1);
3036   balancers_[0]->ads_service()->SetResourceMinVersion(kEdsTypeUrl, 1);
3037   // Update backend, just so we can be sure that the client has
3038   // reconnected to the balancer.
3039   AdsServiceImpl::EdsResourceArgs args2({
3040       {"locality0", CreateEndpointsForBackends(1, 2)},
3041   });
3042   balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args2));
3043   // Restart balancer.
3044   balancers_[0]->Start();
3045   // Make sure client has reconnected.
3046   WaitForAllBackends(1, 2);
3047 }
3048
3049 // Tests switching over from one cluster to another.
3050 TEST_P(XdsResolverOnlyTest, ChangeClusters) {
3051   const char* kNewClusterName = "new_cluster_name";
3052   const char* kNewEdsServiceName = "new_eds_service_name";
3053   SetNextResolution({});
3054   SetNextResolutionForLbChannelAllBalancers();
3055   AdsServiceImpl::EdsResourceArgs args({
3056       {"locality0", CreateEndpointsForBackends(0, 2)},
3057   });
3058   balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
3059   // We need to wait for all backends to come online.
3060   WaitForAllBackends(0, 2);
3061   // Populate new EDS resource.
3062   AdsServiceImpl::EdsResourceArgs args2({
3063       {"locality0", CreateEndpointsForBackends(2, 4)},
3064   });
3065   balancers_[0]->ads_service()->SetEdsResource(
3066       BuildEdsResource(args2, kNewEdsServiceName));
3067   // Populate new CDS resource.
3068   Cluster new_cluster = default_cluster_;
3069   new_cluster.set_name(kNewClusterName);
3070   new_cluster.mutable_eds_cluster_config()->set_service_name(
3071       kNewEdsServiceName);
3072   balancers_[0]->ads_service()->SetCdsResource(new_cluster);
3073   // Change RDS resource to point to new cluster.
3074   RouteConfiguration new_route_config = default_route_config_;
3075   new_route_config.mutable_virtual_hosts(0)
3076       ->mutable_routes(0)
3077       ->mutable_route()
3078       ->set_cluster(kNewClusterName);
3079   SetListenerAndRouteConfiguration(0, default_listener_, new_route_config);
3080   // Wait for all new backends to be used.
3081   std::tuple<int, int, int> counts = WaitForAllBackends(2, 4);
3082   // Make sure no RPCs failed in the transition.
3083   EXPECT_EQ(0, std::get<1>(counts));
3084 }
3085
3086 // Tests that we go into TRANSIENT_FAILURE if the Cluster disappears.
3087 TEST_P(XdsResolverOnlyTest, ClusterRemoved) {
3088   SetNextResolution({});
3089   SetNextResolutionForLbChannelAllBalancers();
3090   AdsServiceImpl::EdsResourceArgs args({
3091       {"locality0", CreateEndpointsForBackends()},
3092   });
3093   balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
3094   // We need to wait for all backends to come online.
3095   WaitForAllBackends();
3096   // Unset CDS resource.
3097   balancers_[0]->ads_service()->UnsetResource(kCdsTypeUrl, kDefaultClusterName);
3098   // Wait for RPCs to start failing.
3099   do {
3100   } while (SendRpc(RpcOptions(), nullptr).ok());
3101   // Make sure RPCs are still failing.
3102   CheckRpcSendFailure(CheckRpcSendFailureOptions().set_times(1000));
3103   // Make sure we ACK'ed the update.
3104   EXPECT_EQ(balancers_[0]->ads_service()->cds_response_state().state,
3105             AdsServiceImpl::ResponseState::ACKED);
3106 }
3107
3108 // Tests that we restart all xDS requests when we reestablish the ADS call.
3109 TEST_P(XdsResolverOnlyTest, RestartsRequestsUponReconnection) {
3110   // Manually configure use of RDS.
3111   auto listener = default_listener_;
3112   HttpConnectionManager http_connection_manager;
3113   listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
3114       &http_connection_manager);
3115   auto* rds = http_connection_manager.mutable_rds();
3116   rds->set_route_config_name(kDefaultRouteConfigurationName);
3117   rds->mutable_config_source()->mutable_ads();
3118   listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
3119       http_connection_manager);
3120   balancers_[0]->ads_service()->SetLdsResource(listener);
3121   balancers_[0]->ads_service()->SetRdsResource(default_route_config_);
3122   const char* kNewClusterName = "new_cluster_name";
3123   const char* kNewEdsServiceName = "new_eds_service_name";
3124   SetNextResolution({});
3125   SetNextResolutionForLbChannelAllBalancers();
3126   AdsServiceImpl::EdsResourceArgs args({
3127       {"locality0", CreateEndpointsForBackends(0, 2)},
3128   });
3129   balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
3130   // We need to wait for all backends to come online.
3131   WaitForAllBackends(0, 2);
3132   // Now shut down and restart the balancer.  When the client
3133   // reconnects, it should automatically restart the requests for all
3134   // resource types.
3135   balancers_[0]->Shutdown();
3136   balancers_[0]->Start();
3137   // Make sure things are still working.
3138   CheckRpcSendOk(100);
3139   // Populate new EDS resource.
3140   AdsServiceImpl::EdsResourceArgs args2({
3141       {"locality0", CreateEndpointsForBackends(2, 4)},
3142   });
3143   balancers_[0]->ads_service()->SetEdsResource(
3144       BuildEdsResource(args2, kNewEdsServiceName));
3145   // Populate new CDS resource.
3146   Cluster new_cluster = default_cluster_;
3147   new_cluster.set_name(kNewClusterName);
3148   new_cluster.mutable_eds_cluster_config()->set_service_name(
3149       kNewEdsServiceName);
3150   balancers_[0]->ads_service()->SetCdsResource(new_cluster);
3151   // Change RDS resource to point to new cluster.
3152   RouteConfiguration new_route_config = default_route_config_;
3153   new_route_config.mutable_virtual_hosts(0)
3154       ->mutable_routes(0)
3155       ->mutable_route()
3156       ->set_cluster(kNewClusterName);
3157   balancers_[0]->ads_service()->SetRdsResource(new_route_config);
3158   // Wait for all new backends to be used.
3159   std::tuple<int, int, int> counts = WaitForAllBackends(2, 4);
3160   // Make sure no RPCs failed in the transition.
3161   EXPECT_EQ(0, std::get<1>(counts));
3162 }
3163
3164 TEST_P(XdsResolverOnlyTest, DefaultRouteSpecifiesSlashPrefix) {
3165   RouteConfiguration route_config = default_route_config_;
3166   route_config.mutable_virtual_hosts(0)
3167       ->mutable_routes(0)
3168       ->mutable_match()
3169       ->set_prefix("/");
3170   SetListenerAndRouteConfiguration(0, default_listener_, route_config);
3171   SetNextResolution({});
3172   SetNextResolutionForLbChannelAllBalancers();
3173   AdsServiceImpl::EdsResourceArgs args({
3174       {"locality0", CreateEndpointsForBackends()},
3175   });
3176   balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
3177   // We need to wait for all backends to come online.
3178   WaitForAllBackends();
3179 }
3180
3181 TEST_P(XdsResolverOnlyTest, CircuitBreaking) {
3182   constexpr size_t kMaxConcurrentRequests = 10;
3183   SetNextResolution({});
3184   SetNextResolutionForLbChannelAllBalancers();
3185   // Populate new EDS resources.
3186   AdsServiceImpl::EdsResourceArgs args({
3187       {"locality0", CreateEndpointsForBackends(0, 1)},
3188   });
3189   balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
3190   // Update CDS resource to set max concurrent request.
3191   CircuitBreakers circuit_breaks;
3192   Cluster cluster = default_cluster_;
3193   auto* threshold = cluster.mutable_circuit_breakers()->add_thresholds();
3194   threshold->set_priority(RoutingPriority::DEFAULT);
3195   threshold->mutable_max_requests()->set_value(kMaxConcurrentRequests);
3196   balancers_[0]->ads_service()->SetCdsResource(cluster);
3197   // Send exactly max_concurrent_requests long RPCs.
3198   LongRunningRpc rpcs[kMaxConcurrentRequests];
3199   for (size_t i = 0; i < kMaxConcurrentRequests; ++i) {
3200     rpcs[i].StartRpc(stub_.get());
3201   }
3202   // Wait for all RPCs to be in flight.
3203   while (backends_[0]->backend_service()->RpcsWaitingForClientCancel() <
3204          kMaxConcurrentRequests) {
3205     gpr_sleep_until(gpr_time_add(gpr_now(GPR_CLOCK_REALTIME),
3206                                  gpr_time_from_micros(1 * 1000, GPR_TIMESPAN)));
3207   }
3208   // Sending a RPC now should fail, the error message should tell us
3209   // we hit the max concurrent requests limit and got dropped.
3210   Status status = SendRpc();
3211   EXPECT_FALSE(status.ok());
3212   EXPECT_EQ(status.error_message(), "circuit breaker drop");
3213   // Cancel one RPC to allow another one through
3214   rpcs[0].CancelRpc();
3215   status = SendRpc();
3216   EXPECT_TRUE(status.ok());
3217   for (size_t i = 1; i < kMaxConcurrentRequests; ++i) {
3218     rpcs[i].CancelRpc();
3219   }
3220   // Make sure RPCs go to the correct backend:
3221   EXPECT_EQ(kMaxConcurrentRequests + 1,
3222             backends_[0]->backend_service()->request_count());
3223 }
3224
3225 TEST_P(XdsResolverOnlyTest, CircuitBreakingMultipleChannelsShareCallCounter) {
3226   constexpr size_t kMaxConcurrentRequests = 10;
3227   // Populate new EDS resources.
3228   AdsServiceImpl::EdsResourceArgs args({
3229       {"locality0", CreateEndpointsForBackends(0, 1)},
3230   });
3231   balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
3232   // Update CDS resource to set max concurrent request.
3233   CircuitBreakers circuit_breaks;
3234   Cluster cluster = default_cluster_;
3235   auto* threshold = cluster.mutable_circuit_breakers()->add_thresholds();
3236   threshold->set_priority(RoutingPriority::DEFAULT);
3237   threshold->mutable_max_requests()->set_value(kMaxConcurrentRequests);
3238   balancers_[0]->ads_service()->SetCdsResource(cluster);
3239   // Create second channel.
3240   auto response_generator2 =
3241       grpc_core::MakeRefCounted<grpc_core::FakeResolverResponseGenerator>();
3242   auto lb_response_generator2 =
3243       grpc_core::MakeRefCounted<grpc_core::FakeResolverResponseGenerator>();
3244   grpc_arg xds_arg = grpc_core::FakeResolverResponseGenerator::MakeChannelArg(
3245       lb_response_generator2.get());
3246   grpc_channel_args xds_channel_args2 = {1, &xds_arg};
3247   auto channel2 = CreateChannel(
3248       /*failover_timeout=*/0, /*server_name=*/kServerName,
3249       response_generator2.get(), &xds_channel_args2);
3250   auto stub2 = grpc::testing::EchoTestService::NewStub(channel2);
3251   // Set resolution results for both channels and for the xDS channel.
3252   SetNextResolution({});
3253   SetNextResolution({}, response_generator2.get());
3254   SetNextResolutionForLbChannelAllBalancers();
3255   SetNextResolutionForLbChannelAllBalancers(nullptr, nullptr,
3256                                             lb_response_generator2.get());
3257   // Send exactly max_concurrent_requests long RPCs, alternating between
3258   // the two channels.
3259   LongRunningRpc rpcs[kMaxConcurrentRequests];
3260   for (size_t i = 0; i < kMaxConcurrentRequests; ++i) {
3261     rpcs[i].StartRpc(i % 2 == 0 ? stub_.get() : stub2.get());
3262   }
3263   // Wait for all RPCs to be in flight.
3264   while (backends_[0]->backend_service()->RpcsWaitingForClientCancel() <
3265          kMaxConcurrentRequests) {
3266     gpr_sleep_until(gpr_time_add(gpr_now(GPR_CLOCK_REALTIME),
3267                                  gpr_time_from_micros(1 * 1000, GPR_TIMESPAN)));
3268   }
3269   // Sending a RPC now should fail, the error message should tell us
3270   // we hit the max concurrent requests limit and got dropped.
3271   Status status = SendRpc();
3272   EXPECT_FALSE(status.ok());
3273   EXPECT_EQ(status.error_message(), "circuit breaker drop");
3274   // Cancel one RPC to allow another one through
3275   rpcs[0].CancelRpc();
3276   status = SendRpc();
3277   EXPECT_TRUE(status.ok());
3278   for (size_t i = 1; i < kMaxConcurrentRequests; ++i) {
3279     rpcs[i].CancelRpc();
3280   }
3281   // Make sure RPCs go to the correct backend:
3282   EXPECT_EQ(kMaxConcurrentRequests + 1,
3283             backends_[0]->backend_service()->request_count());
3284 }
3285
3286 TEST_P(XdsResolverOnlyTest, ClusterChangeAfterAdsCallFails) {
3287   const char* kNewEdsResourceName = "new_eds_resource_name";
3288   // Populate EDS resources.
3289   AdsServiceImpl::EdsResourceArgs args({
3290       {"locality0", CreateEndpointsForBackends(0, 1)},
3291   });
3292   balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
3293   SetNextResolutionForLbChannelAllBalancers();
3294   // Check that the channel is working.
3295   CheckRpcSendOk();
3296   // Stop and restart the balancer.
3297   balancers_[0]->Shutdown();
3298   balancers_[0]->Start();
3299   // Create new EDS resource.
3300   AdsServiceImpl::EdsResourceArgs args2({
3301       {"locality0", CreateEndpointsForBackends(1, 2)},
3302   });
3303   balancers_[0]->ads_service()->SetEdsResource(
3304       BuildEdsResource(args2, kNewEdsResourceName));
3305   // Change CDS resource to point to new EDS resource.
3306   auto cluster = default_cluster_;
3307   cluster.mutable_eds_cluster_config()->set_service_name(kNewEdsResourceName);
3308   balancers_[0]->ads_service()->SetCdsResource(cluster);
3309   // Make sure client sees the change.
3310   // TODO(roth): This should not be allowing errors.  The errors are
3311   // being caused by a bug that triggers in the following situation:
3312   //
3313   // 1. xDS call fails.
3314   // 2. When xDS call is restarted, the server sends the updated CDS
3315   //    resource that points to the new EDS resource name.
3316   // 3. When the client receives the CDS update, it does two things:
3317   //    - Sends the update to the CDS LB policy, which creates a new
3318   //      xds_cluster_resolver policy using the new EDS service name.
3319   //    - Notices that the CDS update no longer refers to the old EDS
3320   //      service name, so removes that resource, notifying the old
3321   //      xds_cluster_resolver policy that the resource no longer exists.
3322   //
3323   // Need to figure out a way to fix this bug, and then change this to
3324   // not allow failures.
3325   WaitForBackend(1, WaitForBackendOptions().set_allow_failures(true));
3326 }
3327
3328 using GlobalXdsClientTest = BasicTest;
3329
3330 TEST_P(GlobalXdsClientTest, MultipleChannelsShareXdsClient) {
3331   const char* kNewServerName = "new-server.example.com";
3332   Listener listener = default_listener_;
3333   listener.set_name(kNewServerName);
3334   SetListenerAndRouteConfiguration(0, listener, default_route_config_);
3335   SetNextResolution({});
3336   SetNextResolutionForLbChannelAllBalancers();
3337   AdsServiceImpl::EdsResourceArgs args({
3338       {"locality0", CreateEndpointsForBackends()},
3339   });
3340   balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
3341   WaitForAllBackends();
3342   // Create second channel and tell it to connect to kNewServerName.
3343   auto channel2 = CreateChannel(/*failover_timeout=*/0, kNewServerName);
3344   channel2->GetState(/*try_to_connect=*/true);
3345   ASSERT_TRUE(
3346       channel2->WaitForConnected(grpc_timeout_milliseconds_to_deadline(100)));
3347   // Make sure there's only one client connected.
3348   EXPECT_EQ(1UL, balancers_[0]->ads_service()->clients().size());
3349 }
3350
3351 // Tests that the NACK for multiple bad LDS resources includes both errors.
3352 TEST_P(GlobalXdsClientTest, MultipleBadResources) {
3353   constexpr char kServerName2[] = "server.other.com";
3354   auto listener = default_listener_;
3355   listener.clear_api_listener();
3356   balancers_[0]->ads_service()->SetLdsResource(listener);
3357   listener.set_name(kServerName2);
3358   balancers_[0]->ads_service()->SetLdsResource(listener);
3359   SetNextResolutionForLbChannelAllBalancers();
3360   CheckRpcSendFailure();
3361   // Need to create a second channel to subscribe to a second LDS resource.
3362   auto channel2 = CreateChannel(0, kServerName2);
3363   auto stub2 = grpc::testing::EchoTestService::NewStub(channel2);
3364   ClientContext context;
3365   EchoRequest request;
3366   request.set_message(kRequestMessage);
3367   EchoResponse response;
3368   grpc::Status status = stub2->Echo(&context, request, &response);
3369   EXPECT_FALSE(status.ok());
3370   // Wait for second NACK to be reported to xDS server.
3371   auto deadline = absl::Now() + absl::Seconds(30);
3372   bool timed_out = false;
3373   CheckRpcSendFailure(
3374       CheckRpcSendFailureOptions().set_continue_predicate([&](size_t) {
3375         if (absl::Now() >= deadline) {
3376           timed_out = true;
3377           return false;
3378         }
3379         const auto response_state =
3380             balancers_[0]->ads_service()->lds_response_state();
3381         return response_state.state != AdsServiceImpl::ResponseState::NACKED ||
3382                !absl::StrContains(
3383                    response_state.error_message,
3384                    absl::StrCat(
3385                        kServerName,
3386                        ": Listener has neither address nor ApiListener")) ||
3387                !absl::StrContains(
3388                    response_state.error_message,
3389                    absl::StrCat(
3390                        kServerName2,
3391                        ": Listener has neither address nor ApiListener"));
3392       }));
3393   ASSERT_FALSE(timed_out);
3394 }
3395
3396 class XdsResolverLoadReportingOnlyTest : public XdsEnd2endTest {
3397  public:
3398   XdsResolverLoadReportingOnlyTest() : XdsEnd2endTest(4, 1, 3) {}
3399 };
3400
3401 // Tests load reporting when switching over from one cluster to another.
3402 TEST_P(XdsResolverLoadReportingOnlyTest, ChangeClusters) {
3403   const char* kNewClusterName = "new_cluster_name";
3404   const char* kNewEdsServiceName = "new_eds_service_name";
3405   balancers_[0]->lrs_service()->set_cluster_names(
3406       {kDefaultClusterName, kNewClusterName});
3407   SetNextResolution({});
3408   SetNextResolutionForLbChannelAllBalancers();
3409   // cluster kDefaultClusterName -> locality0 -> backends 0 and 1
3410   AdsServiceImpl::EdsResourceArgs args({
3411       {"locality0", CreateEndpointsForBackends(0, 2)},
3412   });
3413   balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
3414   // cluster kNewClusterName -> locality1 -> backends 2 and 3
3415   AdsServiceImpl::EdsResourceArgs args2({
3416       {"locality1", CreateEndpointsForBackends(2, 4)},
3417   });
3418   balancers_[0]->ads_service()->SetEdsResource(
3419       BuildEdsResource(args2, kNewEdsServiceName));
3420   // CDS resource for kNewClusterName.
3421   Cluster new_cluster = default_cluster_;
3422   new_cluster.set_name(kNewClusterName);
3423   new_cluster.mutable_eds_cluster_config()->set_service_name(
3424       kNewEdsServiceName);
3425   balancers_[0]->ads_service()->SetCdsResource(new_cluster);
3426   // Wait for all backends to come online.
3427   int num_ok = 0;
3428   int num_failure = 0;
3429   int num_drops = 0;
3430   std::tie(num_ok, num_failure, num_drops) = WaitForAllBackends(0, 2);
3431   // The load report received at the balancer should be correct.
3432   std::vector<ClientStats> load_report =
3433       balancers_[0]->lrs_service()->WaitForLoadReport();
3434   EXPECT_THAT(
3435       load_report,
3436       ::testing::ElementsAre(::testing::AllOf(
3437           ::testing::Property(&ClientStats::cluster_name, kDefaultClusterName),
3438           ::testing::Property(
3439               &ClientStats::locality_stats,
3440               ::testing::ElementsAre(::testing::Pair(
3441                   "locality0",
3442                   ::testing::AllOf(
3443                       ::testing::Field(&ClientStats::LocalityStats::
3444                                            total_successful_requests,
3445                                        num_ok),
3446                       ::testing::Field(&ClientStats::LocalityStats::
3447                                            total_requests_in_progress,
3448                                        0UL),
3449                       ::testing::Field(
3450                           &ClientStats::LocalityStats::total_error_requests,
3451                           num_failure),
3452                       ::testing::Field(
3453                           &ClientStats::LocalityStats::total_issued_requests,
3454                           num_failure + num_ok))))),
3455           ::testing::Property(&ClientStats::total_dropped_requests,
3456                               num_drops))));
3457   // Change RDS resource to point to new cluster.
3458   RouteConfiguration new_route_config = default_route_config_;
3459   new_route_config.mutable_virtual_hosts(0)
3460       ->mutable_routes(0)
3461       ->mutable_route()
3462       ->set_cluster(kNewClusterName);
3463   SetListenerAndRouteConfiguration(0, default_listener_, new_route_config);
3464   // Wait for all new backends to be used.
3465   std::tie(num_ok, num_failure, num_drops) = WaitForAllBackends(2, 4);
3466   // The load report received at the balancer should be correct.
3467   load_report = balancers_[0]->lrs_service()->WaitForLoadReport();
3468   EXPECT_THAT(
3469       load_report,
3470       ::testing::ElementsAre(
3471           ::testing::AllOf(
3472               ::testing::Property(&ClientStats::cluster_name,
3473                                   kDefaultClusterName),
3474               ::testing::Property(
3475                   &ClientStats::locality_stats,
3476                   ::testing::ElementsAre(::testing::Pair(
3477                       "locality0",
3478                       ::testing::AllOf(
3479                           ::testing::Field(&ClientStats::LocalityStats::
3480                                                total_successful_requests,
3481                                            ::testing::Lt(num_ok)),
3482                           ::testing::Field(&ClientStats::LocalityStats::
3483                                                total_requests_in_progress,
3484                                            0UL),
3485                           ::testing::Field(
3486                               &ClientStats::LocalityStats::total_error_requests,
3487                               ::testing::Le(num_failure)),
3488                           ::testing::Field(
3489                               &ClientStats::LocalityStats::
3490                                   total_issued_requests,
3491                               ::testing::Le(num_failure + num_ok)))))),
3492               ::testing::Property(&ClientStats::total_dropped_requests,
3493                                   num_drops)),
3494           ::testing::AllOf(
3495               ::testing::Property(&ClientStats::cluster_name, kNewClusterName),
3496               ::testing::Property(
3497                   &ClientStats::locality_stats,
3498                   ::testing::ElementsAre(::testing::Pair(
3499                       "locality1",
3500                       ::testing::AllOf(
3501                           ::testing::Field(&ClientStats::LocalityStats::
3502                                                total_successful_requests,
3503                                            ::testing::Le(num_ok)),
3504                           ::testing::Field(&ClientStats::LocalityStats::
3505                                                total_requests_in_progress,
3506                                            0UL),
3507                           ::testing::Field(
3508                               &ClientStats::LocalityStats::total_error_requests,
3509                               ::testing::Le(num_failure)),
3510                           ::testing::Field(
3511                               &ClientStats::LocalityStats::
3512                                   total_issued_requests,
3513                               ::testing::Le(num_failure + num_ok)))))),
3514               ::testing::Property(&ClientStats::total_dropped_requests,
3515                                   num_drops))));
3516   int total_ok = 0;
3517   int total_failure = 0;
3518   for (const ClientStats& client_stats : load_report) {
3519     total_ok += client_stats.total_successful_requests();
3520     total_failure += client_stats.total_error_requests();
3521   }
3522   EXPECT_EQ(total_ok, num_ok);
3523   EXPECT_EQ(total_failure, num_failure);
3524   // The LRS service got a single request, and sent a single response.
3525   EXPECT_EQ(1U, balancers_[0]->lrs_service()->request_count());
3526   EXPECT_EQ(1U, balancers_[0]->lrs_service()->response_count());
3527 }
3528
3529 using SecureNamingTest = BasicTest;
3530
3531 // Tests that secure naming check passes if target name is expected.
3532 TEST_P(SecureNamingTest, TargetNameIsExpected) {
3533   SetNextResolution({});
3534   SetNextResolutionForLbChannel({balancers_[0]->port()}, nullptr, "xds_server");
3535   AdsServiceImpl::EdsResourceArgs args({
3536       {"locality0", CreateEndpointsForBackends()},
3537   });
3538   balancers_[0]->ads_service()->SetEdsResource(
3539       BuildEdsResource(args, DefaultEdsServiceName()));
3540   CheckRpcSendOk();
3541 }
3542
3543 // Tests that secure naming check fails if target name is unexpected.
3544 TEST_P(SecureNamingTest, TargetNameIsUnexpected) {
3545   ::testing::FLAGS_gtest_death_test_style = "threadsafe";
3546   SetNextResolution({});
3547   SetNextResolutionForLbChannel({balancers_[0]->port()}, nullptr,
3548                                 "incorrect_server_name");
3549   AdsServiceImpl::EdsResourceArgs args({
3550       {"locality0", CreateEndpointsForBackends()},
3551   });
3552   balancers_[0]->ads_service()->SetEdsResource(
3553       BuildEdsResource(args, DefaultEdsServiceName()));
3554   // Make sure that we blow up (via abort() from the security connector) when
3555   // the name from the balancer doesn't match expectations.
3556   ASSERT_DEATH_IF_SUPPORTED({ CheckRpcSendOk(); }, "");
3557 }
3558
3559 using LdsTest = BasicTest;
3560
3561 // Tests that LDS client should send a NACK if there is no API listener in the
3562 // Listener in the LDS response.
3563 TEST_P(LdsTest, NoApiListener) {
3564   auto listener = default_listener_;
3565   listener.clear_api_listener();
3566   balancers_[0]->ads_service()->SetLdsResource(listener);
3567   SetNextResolutionForLbChannelAllBalancers();
3568   ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK";
3569   const auto response_state =
3570       balancers_[0]->ads_service()->lds_response_state();
3571   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3572   EXPECT_THAT(
3573       response_state.error_message,
3574       ::testing::HasSubstr("Listener has neither address nor ApiListener"));
3575 }
3576
3577 // Tests that LDS client should send a NACK if the route_specifier in the
3578 // http_connection_manager is neither inlined route_config nor RDS.
3579 TEST_P(LdsTest, WrongRouteSpecifier) {
3580   auto listener = default_listener_;
3581   HttpConnectionManager http_connection_manager;
3582   listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
3583       &http_connection_manager);
3584   http_connection_manager.mutable_scoped_routes();
3585   listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
3586       http_connection_manager);
3587   balancers_[0]->ads_service()->SetLdsResource(listener);
3588   SetNextResolutionForLbChannelAllBalancers();
3589   ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK";
3590   const auto response_state =
3591       balancers_[0]->ads_service()->lds_response_state();
3592   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3593   EXPECT_THAT(
3594       response_state.error_message,
3595       ::testing::HasSubstr(
3596           "HttpConnectionManager neither has inlined route_config nor RDS."));
3597 }
3598
3599 // Tests that LDS client should send a NACK if the rds message in the
3600 // http_connection_manager is missing the config_source field.
3601 TEST_P(LdsTest, RdsMissingConfigSource) {
3602   auto listener = default_listener_;
3603   HttpConnectionManager http_connection_manager;
3604   listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
3605       &http_connection_manager);
3606   http_connection_manager.mutable_rds()->set_route_config_name(
3607       kDefaultRouteConfigurationName);
3608   listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
3609       http_connection_manager);
3610   balancers_[0]->ads_service()->SetLdsResource(listener);
3611   SetNextResolutionForLbChannelAllBalancers();
3612   ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK";
3613   const auto response_state =
3614       balancers_[0]->ads_service()->lds_response_state();
3615   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3616   EXPECT_THAT(response_state.error_message,
3617               ::testing::HasSubstr(
3618                   "HttpConnectionManager missing config_source for RDS."));
3619 }
3620
3621 // Tests that LDS client should send a NACK if the rds message in the
3622 // http_connection_manager has a config_source field that does not specify
3623 // ADS.
3624 TEST_P(LdsTest, RdsConfigSourceDoesNotSpecifyAds) {
3625   auto listener = default_listener_;
3626   HttpConnectionManager http_connection_manager;
3627   listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
3628       &http_connection_manager);
3629   auto* rds = http_connection_manager.mutable_rds();
3630   rds->set_route_config_name(kDefaultRouteConfigurationName);
3631   rds->mutable_config_source()->mutable_self();
3632   listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
3633       http_connection_manager);
3634   balancers_[0]->ads_service()->SetLdsResource(listener);
3635   SetNextResolutionForLbChannelAllBalancers();
3636   ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK";
3637   const auto response_state =
3638       balancers_[0]->ads_service()->lds_response_state();
3639   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3640   EXPECT_THAT(response_state.error_message,
3641               ::testing::HasSubstr("HttpConnectionManager ConfigSource for "
3642                                    "RDS does not specify ADS."));
3643 }
3644
3645 // Tests that we NACK non-terminal filters at the end of the list.
3646 TEST_P(LdsTest, NacksNonTerminalHttpFilterAtEndOfList) {
3647   SetNextResolutionForLbChannelAllBalancers();
3648   auto listener = default_listener_;
3649   HttpConnectionManager http_connection_manager;
3650   listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
3651       &http_connection_manager);
3652   auto* filter = http_connection_manager.mutable_http_filters(0);
3653   filter->set_name("unknown");
3654   filter->mutable_typed_config()->set_type_url(
3655       "grpc.testing.client_only_http_filter");
3656   listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
3657       http_connection_manager);
3658   SetListenerAndRouteConfiguration(0, listener, default_route_config_);
3659   SetNextResolutionForLbChannelAllBalancers();
3660   ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK";
3661   const auto response_state =
3662       balancers_[0]->ads_service()->lds_response_state();
3663   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3664   EXPECT_THAT(response_state.error_message,
3665               ::testing::HasSubstr(
3666                   "non-terminal filter for config type grpc.testing"
3667                   ".client_only_http_filter is the last filter in the chain"));
3668 }
3669
3670 // Test that we NACK terminal filters that are not at the end of the list.
3671 TEST_P(LdsTest, NacksTerminalFilterBeforeEndOfList) {
3672   SetNextResolutionForLbChannelAllBalancers();
3673   auto listener = default_listener_;
3674   HttpConnectionManager http_connection_manager;
3675   listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
3676       &http_connection_manager);
3677   *http_connection_manager.add_http_filters() =
3678       http_connection_manager.http_filters(0);
3679   listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
3680       http_connection_manager);
3681   SetListenerAndRouteConfiguration(0, listener, default_route_config_);
3682   SetNextResolutionForLbChannelAllBalancers();
3683   ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK";
3684   const auto response_state =
3685       balancers_[0]->ads_service()->lds_response_state();
3686   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3687   EXPECT_THAT(
3688       response_state.error_message,
3689       ::testing::HasSubstr(
3690           "terminal filter for config type envoy.extensions.filters.http"
3691           ".router.v3.Router must be the last filter in the chain"));
3692 }
3693
3694 // Test that we NACK empty filter names.
3695 TEST_P(LdsTest, RejectsEmptyHttpFilterName) {
3696   auto listener = default_listener_;
3697   HttpConnectionManager http_connection_manager;
3698   listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
3699       &http_connection_manager);
3700   *http_connection_manager.add_http_filters() =
3701       http_connection_manager.http_filters(0);
3702   auto* filter = http_connection_manager.mutable_http_filters(0);
3703   filter->Clear();
3704   filter->mutable_typed_config()->PackFrom(Listener());
3705   listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
3706       http_connection_manager);
3707   SetListenerAndRouteConfiguration(0, listener, default_route_config_);
3708   SetNextResolutionForLbChannelAllBalancers();
3709   ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK";
3710   const auto response_state =
3711       balancers_[0]->ads_service()->lds_response_state();
3712   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3713   EXPECT_THAT(response_state.error_message,
3714               ::testing::HasSubstr("empty filter name at index 0"));
3715 }
3716
3717 // Test that we NACK duplicate HTTP filter names.
3718 TEST_P(LdsTest, RejectsDuplicateHttpFilterName) {
3719   auto listener = default_listener_;
3720   HttpConnectionManager http_connection_manager;
3721   listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
3722       &http_connection_manager);
3723   *http_connection_manager.add_http_filters() =
3724       http_connection_manager.http_filters(0);
3725   http_connection_manager.mutable_http_filters(0)
3726       ->mutable_typed_config()
3727       ->PackFrom(HTTPFault());
3728   listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
3729       http_connection_manager);
3730   SetListenerAndRouteConfiguration(0, listener, default_route_config_);
3731   SetNextResolutionForLbChannelAllBalancers();
3732   ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK";
3733   const auto response_state =
3734       balancers_[0]->ads_service()->lds_response_state();
3735   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3736   EXPECT_THAT(response_state.error_message,
3737               ::testing::HasSubstr("duplicate HTTP filter name: router"));
3738 }
3739
3740 // Test that we NACK unknown filter types.
3741 TEST_P(LdsTest, RejectsUnknownHttpFilterType) {
3742   auto listener = default_listener_;
3743   HttpConnectionManager http_connection_manager;
3744   listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
3745       &http_connection_manager);
3746   *http_connection_manager.add_http_filters() =
3747       http_connection_manager.http_filters(0);
3748   auto* filter = http_connection_manager.mutable_http_filters(0);
3749   filter->set_name("unknown");
3750   filter->mutable_typed_config()->PackFrom(Listener());
3751   listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
3752       http_connection_manager);
3753   SetListenerAndRouteConfiguration(0, listener, default_route_config_);
3754   SetNextResolutionForLbChannelAllBalancers();
3755   ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK";
3756   const auto response_state =
3757       balancers_[0]->ads_service()->lds_response_state();
3758   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3759   EXPECT_THAT(response_state.error_message,
3760               ::testing::HasSubstr("no filter registered for config type "
3761                                    "envoy.config.listener.v3.Listener"));
3762 }
3763
3764 // Test that we ignore optional unknown filter types.
3765 TEST_P(LdsTest, IgnoresOptionalUnknownHttpFilterType) {
3766   auto listener = default_listener_;
3767   HttpConnectionManager http_connection_manager;
3768   listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
3769       &http_connection_manager);
3770   *http_connection_manager.add_http_filters() =
3771       http_connection_manager.http_filters(0);
3772   auto* filter = http_connection_manager.mutable_http_filters(0);
3773   filter->set_name("unknown");
3774   filter->mutable_typed_config()->PackFrom(Listener());
3775   filter->set_is_optional(true);
3776   listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
3777       http_connection_manager);
3778   SetListenerAndRouteConfiguration(0, listener, default_route_config_);
3779   AdsServiceImpl::EdsResourceArgs args({
3780       {"locality0", CreateEndpointsForBackends()},
3781   });
3782   balancers_[0]->ads_service()->SetEdsResource(
3783       BuildEdsResource(args, DefaultEdsServiceName()));
3784   SetNextResolutionForLbChannelAllBalancers();
3785   WaitForAllBackends();
3786   EXPECT_EQ(balancers_[0]->ads_service()->lds_response_state().state,
3787             AdsServiceImpl::ResponseState::ACKED);
3788 }
3789
3790 // Test that we NACK filters without configs.
3791 TEST_P(LdsTest, RejectsHttpFilterWithoutConfig) {
3792   auto listener = default_listener_;
3793   HttpConnectionManager http_connection_manager;
3794   listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
3795       &http_connection_manager);
3796   *http_connection_manager.add_http_filters() =
3797       http_connection_manager.http_filters(0);
3798   auto* filter = http_connection_manager.mutable_http_filters(0);
3799   filter->Clear();
3800   filter->set_name("unknown");
3801   listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
3802       http_connection_manager);
3803   SetListenerAndRouteConfiguration(0, listener, default_route_config_);
3804   SetNextResolutionForLbChannelAllBalancers();
3805   ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK";
3806   const auto response_state =
3807       balancers_[0]->ads_service()->lds_response_state();
3808   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3809   EXPECT_THAT(response_state.error_message,
3810               ::testing::HasSubstr(
3811                   "no filter config specified for filter name unknown"));
3812 }
3813
3814 // Test that we ignore optional filters without configs.
3815 TEST_P(LdsTest, IgnoresOptionalHttpFilterWithoutConfig) {
3816   auto listener = default_listener_;
3817   HttpConnectionManager http_connection_manager;
3818   listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
3819       &http_connection_manager);
3820   *http_connection_manager.add_http_filters() =
3821       http_connection_manager.http_filters(0);
3822   auto* filter = http_connection_manager.mutable_http_filters(0);
3823   filter->Clear();
3824   filter->set_name("unknown");
3825   filter->set_is_optional(true);
3826   listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
3827       http_connection_manager);
3828   SetListenerAndRouteConfiguration(0, listener, default_route_config_);
3829   AdsServiceImpl::EdsResourceArgs args({
3830       {"locality0", CreateEndpointsForBackends()},
3831   });
3832   balancers_[0]->ads_service()->SetEdsResource(
3833       BuildEdsResource(args, DefaultEdsServiceName()));
3834   SetNextResolutionForLbChannelAllBalancers();
3835   WaitForAllBackends();
3836   EXPECT_EQ(balancers_[0]->ads_service()->lds_response_state().state,
3837             AdsServiceImpl::ResponseState::ACKED);
3838 }
3839
3840 // Test that we NACK unparseable filter configs.
3841 TEST_P(LdsTest, RejectsUnparseableHttpFilterType) {
3842   auto listener = default_listener_;
3843   HttpConnectionManager http_connection_manager;
3844   listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
3845       &http_connection_manager);
3846   *http_connection_manager.add_http_filters() =
3847       http_connection_manager.http_filters(0);
3848   auto* filter = http_connection_manager.mutable_http_filters(0);
3849   filter->set_name("unknown");
3850   filter->mutable_typed_config()->PackFrom(listener);
3851   filter->mutable_typed_config()->set_type_url(
3852       "type.googleapis.com/envoy.extensions.filters.http.fault.v3.HTTPFault");
3853   listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
3854       http_connection_manager);
3855   SetListenerAndRouteConfiguration(0, listener, default_route_config_);
3856   SetNextResolutionForLbChannelAllBalancers();
3857   ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK";
3858   const auto response_state =
3859       balancers_[0]->ads_service()->lds_response_state();
3860   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3861   EXPECT_THAT(
3862       response_state.error_message,
3863       ::testing::HasSubstr(
3864           "filter config for type "
3865           "envoy.extensions.filters.http.fault.v3.HTTPFault failed to parse"));
3866 }
3867
3868 // Test that we NACK HTTP filters unsupported on client-side.
3869 TEST_P(LdsTest, RejectsHttpFiltersNotSupportedOnClients) {
3870   auto listener = default_listener_;
3871   HttpConnectionManager http_connection_manager;
3872   listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
3873       &http_connection_manager);
3874   *http_connection_manager.add_http_filters() =
3875       http_connection_manager.http_filters(0);
3876   auto* filter = http_connection_manager.mutable_http_filters(0);
3877   filter->set_name("grpc.testing.server_only_http_filter");
3878   filter->mutable_typed_config()->set_type_url(
3879       "grpc.testing.server_only_http_filter");
3880   listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
3881       http_connection_manager);
3882   SetListenerAndRouteConfiguration(0, listener, default_route_config_);
3883   SetNextResolutionForLbChannelAllBalancers();
3884   ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK";
3885   const auto response_state =
3886       balancers_[0]->ads_service()->lds_response_state();
3887   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
3888   EXPECT_THAT(
3889       response_state.error_message,
3890       ::testing::HasSubstr("Filter grpc.testing.server_only_http_filter is not "
3891                            "supported on clients"));
3892 }
3893
3894 // Test that we ignore optional HTTP filters unsupported on client-side.
3895 TEST_P(LdsTest, IgnoresOptionalHttpFiltersNotSupportedOnClients) {
3896   auto listener = default_listener_;
3897   HttpConnectionManager http_connection_manager;
3898   listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
3899       &http_connection_manager);
3900   *http_connection_manager.add_http_filters() =
3901       http_connection_manager.http_filters(0);
3902   auto* filter = http_connection_manager.mutable_http_filters(0);
3903   filter->set_name("grpc.testing.server_only_http_filter");
3904   filter->mutable_typed_config()->set_type_url(
3905       "grpc.testing.server_only_http_filter");
3906   filter->set_is_optional(true);
3907   listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
3908       http_connection_manager);
3909   SetListenerAndRouteConfiguration(0, listener, default_route_config_);
3910   AdsServiceImpl::EdsResourceArgs args({
3911       {"locality0", CreateEndpointsForBackends(0, 1)},
3912   });
3913   balancers_[0]->ads_service()->SetEdsResource(
3914       BuildEdsResource(args, DefaultEdsServiceName()));
3915   SetNextResolutionForLbChannelAllBalancers();
3916   WaitForBackend(0);
3917   EXPECT_EQ(balancers_[0]->ads_service()->lds_response_state().state,
3918             AdsServiceImpl::ResponseState::ACKED);
3919 }
3920
3921 using LdsV2Test = LdsTest;
3922
3923 // Tests that we ignore the HTTP filter list in v2.
3924 // TODO(roth): The test framework is not set up to allow us to test
3925 // the server sending v2 resources when the client requests v3, so this
3926 // just tests a pure v2 setup.  When we have time, fix this.
3927 TEST_P(LdsV2Test, IgnoresHttpFilters) {
3928   auto listener = default_listener_;
3929   HttpConnectionManager http_connection_manager;
3930   listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
3931       &http_connection_manager);
3932   auto* filter = http_connection_manager.add_http_filters();
3933   filter->set_name("unknown");
3934   filter->mutable_typed_config()->PackFrom(Listener());
3935   listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
3936       http_connection_manager);
3937   SetListenerAndRouteConfiguration(0, listener, default_route_config_);
3938   AdsServiceImpl::EdsResourceArgs args({
3939       {"locality0", CreateEndpointsForBackends(0, 1)},
3940   });
3941   balancers_[0]->ads_service()->SetEdsResource(
3942       BuildEdsResource(args, DefaultEdsServiceName()));
3943   SetNextResolutionForLbChannelAllBalancers();
3944   CheckRpcSendOk();
3945 }
3946
3947 using LdsRdsTest = BasicTest;
3948
3949 // Tests that LDS client should send an ACK upon correct LDS response (with
3950 // inlined RDS result).
3951 TEST_P(LdsRdsTest, Vanilla) {
3952   SetNextResolution({});
3953   SetNextResolutionForLbChannelAllBalancers();
3954   (void)SendRpc();
3955   EXPECT_EQ(RouteConfigurationResponseState(0).state,
3956             AdsServiceImpl::ResponseState::ACKED);
3957   // Make sure we actually used the RPC service for the right version of xDS.
3958   EXPECT_EQ(balancers_[0]->ads_service()->seen_v2_client(),
3959             GetParam().use_v2());
3960   EXPECT_NE(balancers_[0]->ads_service()->seen_v3_client(),
3961             GetParam().use_v2());
3962 }
3963
3964 // Tests that we go into TRANSIENT_FAILURE if the Listener is removed.
3965 TEST_P(LdsRdsTest, ListenerRemoved) {
3966   SetNextResolution({});
3967   SetNextResolutionForLbChannelAllBalancers();
3968   AdsServiceImpl::EdsResourceArgs args({
3969       {"locality0", CreateEndpointsForBackends()},
3970   });
3971   balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
3972   // We need to wait for all backends to come online.
3973   WaitForAllBackends();
3974   // Unset LDS resource.
3975   balancers_[0]->ads_service()->UnsetResource(kLdsTypeUrl, kServerName);
3976   // Wait for RPCs to start failing.
3977   do {
3978   } while (SendRpc(RpcOptions(), nullptr).ok());
3979   // Make sure RPCs are still failing.
3980   CheckRpcSendFailure(CheckRpcSendFailureOptions().set_times(1000));
3981   // Make sure we ACK'ed the update.
3982   EXPECT_EQ(balancers_[0]->ads_service()->lds_response_state().state,
3983             AdsServiceImpl::ResponseState::ACKED);
3984 }
3985
3986 // Tests that LDS client ACKs but fails if matching domain can't be found in
3987 // the LDS response.
3988 TEST_P(LdsRdsTest, NoMatchedDomain) {
3989   RouteConfiguration route_config = default_route_config_;
3990   route_config.mutable_virtual_hosts(0)->clear_domains();
3991   route_config.mutable_virtual_hosts(0)->add_domains("unmatched_domain");
3992   SetRouteConfiguration(0, route_config);
3993   SetNextResolution({});
3994   SetNextResolutionForLbChannelAllBalancers();
3995   CheckRpcSendFailure();
3996   // Do a bit of polling, to allow the ACK to get to the ADS server.
3997   channel_->WaitForConnected(grpc_timeout_milliseconds_to_deadline(100));
3998   const auto response_state = RouteConfigurationResponseState(0);
3999   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::ACKED);
4000 }
4001
4002 // Tests that LDS client should choose the virtual host with matching domain
4003 // if multiple virtual hosts exist in the LDS response.
4004 TEST_P(LdsRdsTest, ChooseMatchedDomain) {
4005   RouteConfiguration route_config = default_route_config_;
4006   *(route_config.add_virtual_hosts()) = route_config.virtual_hosts(0);
4007   route_config.mutable_virtual_hosts(0)->clear_domains();
4008   route_config.mutable_virtual_hosts(0)->add_domains("unmatched_domain");
4009   SetRouteConfiguration(0, route_config);
4010   SetNextResolution({});
4011   SetNextResolutionForLbChannelAllBalancers();
4012   (void)SendRpc();
4013   EXPECT_EQ(RouteConfigurationResponseState(0).state,
4014             AdsServiceImpl::ResponseState::ACKED);
4015 }
4016
4017 // Tests that LDS client should choose the last route in the virtual host if
4018 // multiple routes exist in the LDS response.
4019 TEST_P(LdsRdsTest, ChooseLastRoute) {
4020   RouteConfiguration route_config = default_route_config_;
4021   *(route_config.mutable_virtual_hosts(0)->add_routes()) =
4022       route_config.virtual_hosts(0).routes(0);
4023   route_config.mutable_virtual_hosts(0)
4024       ->mutable_routes(0)
4025       ->mutable_route()
4026       ->mutable_cluster_header();
4027   SetRouteConfiguration(0, route_config);
4028   SetNextResolution({});
4029   SetNextResolutionForLbChannelAllBalancers();
4030   (void)SendRpc();
4031   EXPECT_EQ(RouteConfigurationResponseState(0).state,
4032             AdsServiceImpl::ResponseState::ACKED);
4033 }
4034
4035 // Tests that LDS client should ignore route which has query_parameters.
4036 TEST_P(LdsRdsTest, RouteMatchHasQueryParameters) {
4037   RouteConfiguration route_config = default_route_config_;
4038   auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4039   route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
4040   route1->mutable_match()->add_query_parameters();
4041   SetRouteConfiguration(0, route_config);
4042   SetNextResolution({});
4043   SetNextResolutionForLbChannelAllBalancers();
4044   ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
4045   const auto response_state = RouteConfigurationResponseState(0);
4046   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
4047   EXPECT_THAT(response_state.error_message,
4048               ::testing::HasSubstr("No valid routes specified."));
4049 }
4050
4051 // Tests that LDS client should send a ACK if route match has a prefix
4052 // that is either empty or a single slash
4053 TEST_P(LdsRdsTest, RouteMatchHasValidPrefixEmptyOrSingleSlash) {
4054   RouteConfiguration route_config = default_route_config_;
4055   auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4056   route1->mutable_match()->set_prefix("");
4057   auto* default_route = route_config.mutable_virtual_hosts(0)->add_routes();
4058   default_route->mutable_match()->set_prefix("/");
4059   default_route->mutable_route()->set_cluster(kDefaultClusterName);
4060   SetRouteConfiguration(0, route_config);
4061   SetNextResolution({});
4062   SetNextResolutionForLbChannelAllBalancers();
4063   (void)SendRpc();
4064   const auto response_state = RouteConfigurationResponseState(0);
4065   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::ACKED);
4066 }
4067
4068 // Tests that LDS client should ignore route which has a path
4069 // prefix string does not start with "/".
4070 TEST_P(LdsRdsTest, RouteMatchHasInvalidPrefixNoLeadingSlash) {
4071   RouteConfiguration route_config = default_route_config_;
4072   auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4073   route1->mutable_match()->set_prefix("grpc.testing.EchoTest1Service/");
4074   SetRouteConfiguration(0, route_config);
4075   SetNextResolution({});
4076   SetNextResolutionForLbChannelAllBalancers();
4077   ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
4078   const auto response_state = RouteConfigurationResponseState(0);
4079   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
4080   EXPECT_THAT(response_state.error_message,
4081               ::testing::HasSubstr("No valid routes specified."));
4082 }
4083
4084 // Tests that LDS client should ignore route which has a prefix
4085 // string with more than 2 slashes.
4086 TEST_P(LdsRdsTest, RouteMatchHasInvalidPrefixExtraContent) {
4087   RouteConfiguration route_config = default_route_config_;
4088   auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4089   route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/Echo1/");
4090   SetRouteConfiguration(0, route_config);
4091   SetNextResolution({});
4092   SetNextResolutionForLbChannelAllBalancers();
4093   ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
4094   const auto response_state = RouteConfigurationResponseState(0);
4095   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
4096   EXPECT_THAT(response_state.error_message,
4097               ::testing::HasSubstr("No valid routes specified."));
4098 }
4099
4100 // Tests that LDS client should ignore route which has a prefix
4101 // string "//".
4102 TEST_P(LdsRdsTest, RouteMatchHasInvalidPrefixDoubleSlash) {
4103   RouteConfiguration route_config = default_route_config_;
4104   auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4105   route1->mutable_match()->set_prefix("//");
4106   SetRouteConfiguration(0, route_config);
4107   SetNextResolution({});
4108   SetNextResolutionForLbChannelAllBalancers();
4109   ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
4110   const auto response_state = RouteConfigurationResponseState(0);
4111   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
4112   EXPECT_THAT(response_state.error_message,
4113               ::testing::HasSubstr("No valid routes specified."));
4114 }
4115
4116 // Tests that LDS client should ignore route which has path
4117 // but it's empty.
4118 TEST_P(LdsRdsTest, RouteMatchHasInvalidPathEmptyPath) {
4119   RouteConfiguration route_config = default_route_config_;
4120   auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4121   route1->mutable_match()->set_path("");
4122   SetRouteConfiguration(0, route_config);
4123   SetNextResolution({});
4124   SetNextResolutionForLbChannelAllBalancers();
4125   ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
4126   const auto response_state = RouteConfigurationResponseState(0);
4127   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
4128   EXPECT_THAT(response_state.error_message,
4129               ::testing::HasSubstr("No valid routes specified."));
4130 }
4131
4132 // Tests that LDS client should ignore route which has path
4133 // string does not start with "/".
4134 TEST_P(LdsRdsTest, RouteMatchHasInvalidPathNoLeadingSlash) {
4135   RouteConfiguration route_config = default_route_config_;
4136   auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4137   route1->mutable_match()->set_path("grpc.testing.EchoTest1Service/Echo1");
4138   SetRouteConfiguration(0, route_config);
4139   SetNextResolution({});
4140   SetNextResolutionForLbChannelAllBalancers();
4141   ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
4142   const auto response_state = RouteConfigurationResponseState(0);
4143   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
4144   EXPECT_THAT(response_state.error_message,
4145               ::testing::HasSubstr("No valid routes specified."));
4146 }
4147
4148 // Tests that LDS client should ignore route which has path
4149 // string that has too many slashes; for example, ends with "/".
4150 TEST_P(LdsRdsTest, RouteMatchHasInvalidPathTooManySlashes) {
4151   RouteConfiguration route_config = default_route_config_;
4152   auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4153   route1->mutable_match()->set_path("/grpc.testing.EchoTest1Service/Echo1/");
4154   SetRouteConfiguration(0, route_config);
4155   SetNextResolution({});
4156   SetNextResolutionForLbChannelAllBalancers();
4157   ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
4158   const auto response_state = RouteConfigurationResponseState(0);
4159   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
4160   EXPECT_THAT(response_state.error_message,
4161               ::testing::HasSubstr("No valid routes specified."));
4162 }
4163
4164 // Tests that LDS client should ignore route which has path
4165 // string that has only 1 slash: missing "/" between service and method.
4166 TEST_P(LdsRdsTest, RouteMatchHasInvalidPathOnlyOneSlash) {
4167   RouteConfiguration route_config = default_route_config_;
4168   auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4169   route1->mutable_match()->set_path("/grpc.testing.EchoTest1Service.Echo1");
4170   SetRouteConfiguration(0, route_config);
4171   SetNextResolution({});
4172   SetNextResolutionForLbChannelAllBalancers();
4173   ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
4174   const auto response_state = RouteConfigurationResponseState(0);
4175   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
4176   EXPECT_THAT(response_state.error_message,
4177               ::testing::HasSubstr("No valid routes specified."));
4178 }
4179
4180 // Tests that LDS client should ignore route which has path
4181 // string that is missing service.
4182 TEST_P(LdsRdsTest, RouteMatchHasInvalidPathMissingService) {
4183   RouteConfiguration route_config = default_route_config_;
4184   auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4185   route1->mutable_match()->set_path("//Echo1");
4186   SetRouteConfiguration(0, route_config);
4187   SetNextResolution({});
4188   SetNextResolutionForLbChannelAllBalancers();
4189   ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
4190   const auto response_state = RouteConfigurationResponseState(0);
4191   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
4192   EXPECT_THAT(response_state.error_message,
4193               ::testing::HasSubstr("No valid routes specified."));
4194 }
4195
4196 // Tests that LDS client should ignore route which has path
4197 // string that is missing method.
4198 TEST_P(LdsRdsTest, RouteMatchHasInvalidPathMissingMethod) {
4199   RouteConfiguration route_config = default_route_config_;
4200   auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4201   route1->mutable_match()->set_path("/grpc.testing.EchoTest1Service/");
4202   SetRouteConfiguration(0, route_config);
4203   SetNextResolution({});
4204   SetNextResolutionForLbChannelAllBalancers();
4205   ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
4206   const auto response_state = RouteConfigurationResponseState(0);
4207   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
4208   EXPECT_THAT(response_state.error_message,
4209               ::testing::HasSubstr("No valid routes specified."));
4210 }
4211
4212 // Test that LDS client should reject route which has invalid path regex.
4213 TEST_P(LdsRdsTest, RouteMatchHasInvalidPathRegex) {
4214   const char* kNewCluster1Name = "new_cluster_1";
4215   RouteConfiguration route_config = default_route_config_;
4216   auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4217   route1->mutable_match()->mutable_safe_regex()->set_regex("a[z-a]");
4218   route1->mutable_route()->set_cluster(kNewCluster1Name);
4219   SetRouteConfiguration(0, route_config);
4220   SetNextResolution({});
4221   SetNextResolutionForLbChannelAllBalancers();
4222   ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
4223   const auto response_state = RouteConfigurationResponseState(0);
4224   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
4225   EXPECT_THAT(response_state.error_message,
4226               ::testing::HasSubstr(
4227                   "path matcher: Invalid regex string specified in matcher."));
4228 }
4229
4230 // Tests that LDS client should send a NACK if route has an action other than
4231 // RouteAction in the LDS response.
4232 TEST_P(LdsRdsTest, RouteHasNoRouteAction) {
4233   RouteConfiguration route_config = default_route_config_;
4234   route_config.mutable_virtual_hosts(0)->mutable_routes(0)->mutable_redirect();
4235   SetRouteConfiguration(0, route_config);
4236   SetNextResolution({});
4237   SetNextResolutionForLbChannelAllBalancers();
4238   ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
4239   const auto response_state = RouteConfigurationResponseState(0);
4240   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
4241   EXPECT_THAT(response_state.error_message,
4242               ::testing::HasSubstr("No RouteAction found in route."));
4243 }
4244
4245 TEST_P(LdsRdsTest, RouteActionClusterHasEmptyClusterName) {
4246   RouteConfiguration route_config = default_route_config_;
4247   auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4248   route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
4249   route1->mutable_route()->set_cluster("");
4250   auto* default_route = route_config.mutable_virtual_hosts(0)->add_routes();
4251   default_route->mutable_match()->set_prefix("");
4252   default_route->mutable_route()->set_cluster(kDefaultClusterName);
4253   SetRouteConfiguration(0, route_config);
4254   SetNextResolution({});
4255   SetNextResolutionForLbChannelAllBalancers();
4256   ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
4257   const auto response_state = RouteConfigurationResponseState(0);
4258   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
4259   EXPECT_THAT(
4260       response_state.error_message,
4261       ::testing::HasSubstr("RouteAction cluster contains empty cluster name."));
4262 }
4263
4264 TEST_P(LdsRdsTest, RouteActionWeightedTargetHasIncorrectTotalWeightSet) {
4265   const size_t kWeight75 = 75;
4266   const char* kNewCluster1Name = "new_cluster_1";
4267   RouteConfiguration route_config = default_route_config_;
4268   auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4269   route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
4270   auto* weighted_cluster1 =
4271       route1->mutable_route()->mutable_weighted_clusters()->add_clusters();
4272   weighted_cluster1->set_name(kNewCluster1Name);
4273   weighted_cluster1->mutable_weight()->set_value(kWeight75);
4274   route1->mutable_route()
4275       ->mutable_weighted_clusters()
4276       ->mutable_total_weight()
4277       ->set_value(kWeight75 + 1);
4278   auto* default_route = route_config.mutable_virtual_hosts(0)->add_routes();
4279   default_route->mutable_match()->set_prefix("");
4280   default_route->mutable_route()->set_cluster(kDefaultClusterName);
4281   SetRouteConfiguration(0, route_config);
4282   SetNextResolution({});
4283   SetNextResolutionForLbChannelAllBalancers();
4284   ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
4285   const auto response_state = RouteConfigurationResponseState(0);
4286   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
4287   EXPECT_THAT(response_state.error_message,
4288               ::testing::HasSubstr(
4289                   "RouteAction weighted_cluster has incorrect total weight"));
4290 }
4291
4292 TEST_P(LdsRdsTest, RouteActionWeightedClusterHasZeroTotalWeight) {
4293   const char* kNewCluster1Name = "new_cluster_1";
4294   RouteConfiguration route_config = default_route_config_;
4295   auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4296   route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
4297   auto* weighted_cluster1 =
4298       route1->mutable_route()->mutable_weighted_clusters()->add_clusters();
4299   weighted_cluster1->set_name(kNewCluster1Name);
4300   weighted_cluster1->mutable_weight()->set_value(0);
4301   route1->mutable_route()
4302       ->mutable_weighted_clusters()
4303       ->mutable_total_weight()
4304       ->set_value(0);
4305   auto* default_route = route_config.mutable_virtual_hosts(0)->add_routes();
4306   default_route->mutable_match()->set_prefix("");
4307   default_route->mutable_route()->set_cluster(kDefaultClusterName);
4308   SetRouteConfiguration(0, route_config);
4309   SetNextResolution({});
4310   SetNextResolutionForLbChannelAllBalancers();
4311   ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
4312   const auto response_state = RouteConfigurationResponseState(0);
4313   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
4314   EXPECT_THAT(
4315       response_state.error_message,
4316       ::testing::HasSubstr(
4317           "RouteAction weighted_cluster has no valid clusters specified."));
4318 }
4319
4320 TEST_P(LdsRdsTest, RouteActionWeightedTargetClusterHasEmptyClusterName) {
4321   const size_t kWeight75 = 75;
4322   RouteConfiguration route_config = default_route_config_;
4323   auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4324   route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
4325   auto* weighted_cluster1 =
4326       route1->mutable_route()->mutable_weighted_clusters()->add_clusters();
4327   weighted_cluster1->set_name("");
4328   weighted_cluster1->mutable_weight()->set_value(kWeight75);
4329   route1->mutable_route()
4330       ->mutable_weighted_clusters()
4331       ->mutable_total_weight()
4332       ->set_value(kWeight75);
4333   auto* default_route = route_config.mutable_virtual_hosts(0)->add_routes();
4334   default_route->mutable_match()->set_prefix("");
4335   default_route->mutable_route()->set_cluster(kDefaultClusterName);
4336   SetRouteConfiguration(0, route_config);
4337   SetNextResolution({});
4338   SetNextResolutionForLbChannelAllBalancers();
4339   ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
4340   const auto response_state = RouteConfigurationResponseState(0);
4341   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
4342   EXPECT_THAT(response_state.error_message,
4343               ::testing::HasSubstr("RouteAction weighted_cluster cluster "
4344                                    "contains empty cluster name."));
4345 }
4346
4347 TEST_P(LdsRdsTest, RouteActionWeightedTargetClusterHasNoWeight) {
4348   const size_t kWeight75 = 75;
4349   const char* kNewCluster1Name = "new_cluster_1";
4350   RouteConfiguration route_config = default_route_config_;
4351   auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4352   route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
4353   auto* weighted_cluster1 =
4354       route1->mutable_route()->mutable_weighted_clusters()->add_clusters();
4355   weighted_cluster1->set_name(kNewCluster1Name);
4356   route1->mutable_route()
4357       ->mutable_weighted_clusters()
4358       ->mutable_total_weight()
4359       ->set_value(kWeight75);
4360   auto* default_route = route_config.mutable_virtual_hosts(0)->add_routes();
4361   default_route->mutable_match()->set_prefix("");
4362   default_route->mutable_route()->set_cluster(kDefaultClusterName);
4363   SetRouteConfiguration(0, route_config);
4364   SetNextResolution({});
4365   SetNextResolutionForLbChannelAllBalancers();
4366   ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
4367   const auto response_state = RouteConfigurationResponseState(0);
4368   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
4369   EXPECT_THAT(response_state.error_message,
4370               ::testing::HasSubstr(
4371                   "RouteAction weighted_cluster cluster missing weight"));
4372 }
4373
4374 TEST_P(LdsRdsTest, RouteHeaderMatchInvalidRegex) {
4375   const char* kNewCluster1Name = "new_cluster_1";
4376   RouteConfiguration route_config = default_route_config_;
4377   auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4378   route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
4379   auto* header_matcher1 = route1->mutable_match()->add_headers();
4380   header_matcher1->set_name("header1");
4381   header_matcher1->mutable_safe_regex_match()->set_regex("a[z-a]");
4382   route1->mutable_route()->set_cluster(kNewCluster1Name);
4383   SetRouteConfiguration(0, route_config);
4384   SetNextResolution({});
4385   SetNextResolutionForLbChannelAllBalancers();
4386   ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
4387   const auto response_state = RouteConfigurationResponseState(0);
4388   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
4389   EXPECT_THAT(
4390       response_state.error_message,
4391       ::testing::HasSubstr(
4392           "header matcher: Invalid regex string specified in matcher."));
4393 }
4394
4395 TEST_P(LdsRdsTest, RouteHeaderMatchInvalidRange) {
4396   const char* kNewCluster1Name = "new_cluster_1";
4397   RouteConfiguration route_config = default_route_config_;
4398   auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4399   route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
4400   auto* header_matcher1 = route1->mutable_match()->add_headers();
4401   header_matcher1->set_name("header1");
4402   header_matcher1->mutable_range_match()->set_start(1001);
4403   header_matcher1->mutable_range_match()->set_end(1000);
4404   route1->mutable_route()->set_cluster(kNewCluster1Name);
4405   SetRouteConfiguration(0, route_config);
4406   SetNextResolution({});
4407   SetNextResolutionForLbChannelAllBalancers();
4408   ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
4409   const auto response_state = RouteConfigurationResponseState(0);
4410   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
4411   EXPECT_THAT(
4412       response_state.error_message,
4413       ::testing::HasSubstr(
4414           "header matcher: Invalid range specifier specified: end cannot be "
4415           "smaller than start."));
4416 }
4417
4418 // Tests that LDS client should choose the default route (with no matching
4419 // specified) after unable to find a match with previous routes.
4420 TEST_P(LdsRdsTest, XdsRoutingPathMatching) {
4421   const char* kNewCluster1Name = "new_cluster_1";
4422   const char* kNewEdsService1Name = "new_eds_service_name_1";
4423   const char* kNewCluster2Name = "new_cluster_2";
4424   const char* kNewEdsService2Name = "new_eds_service_name_2";
4425   const size_t kNumEcho1Rpcs = 10;
4426   const size_t kNumEcho2Rpcs = 20;
4427   const size_t kNumEchoRpcs = 30;
4428   SetNextResolution({});
4429   SetNextResolutionForLbChannelAllBalancers();
4430   // Populate new EDS resources.
4431   AdsServiceImpl::EdsResourceArgs args({
4432       {"locality0", CreateEndpointsForBackends(0, 2)},
4433   });
4434   AdsServiceImpl::EdsResourceArgs args1({
4435       {"locality0", CreateEndpointsForBackends(2, 3)},
4436   });
4437   AdsServiceImpl::EdsResourceArgs args2({
4438       {"locality0", CreateEndpointsForBackends(3, 4)},
4439   });
4440   balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
4441   balancers_[0]->ads_service()->SetEdsResource(
4442       BuildEdsResource(args1, kNewEdsService1Name));
4443   balancers_[0]->ads_service()->SetEdsResource(
4444       BuildEdsResource(args2, kNewEdsService2Name));
4445   // Populate new CDS resources.
4446   Cluster new_cluster1 = default_cluster_;
4447   new_cluster1.set_name(kNewCluster1Name);
4448   new_cluster1.mutable_eds_cluster_config()->set_service_name(
4449       kNewEdsService1Name);
4450   balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
4451   Cluster new_cluster2 = default_cluster_;
4452   new_cluster2.set_name(kNewCluster2Name);
4453   new_cluster2.mutable_eds_cluster_config()->set_service_name(
4454       kNewEdsService2Name);
4455   balancers_[0]->ads_service()->SetCdsResource(new_cluster2);
4456   // Populating Route Configurations for LDS.
4457   RouteConfiguration new_route_config = default_route_config_;
4458   auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4459   route1->mutable_match()->set_path("/grpc.testing.EchoTest1Service/Echo1");
4460   route1->mutable_route()->set_cluster(kNewCluster1Name);
4461   auto* route2 = new_route_config.mutable_virtual_hosts(0)->add_routes();
4462   route2->mutable_match()->set_path("/grpc.testing.EchoTest2Service/Echo2");
4463   route2->mutable_route()->set_cluster(kNewCluster2Name);
4464   auto* route3 = new_route_config.mutable_virtual_hosts(0)->add_routes();
4465   route3->mutable_match()->set_path("/grpc.testing.EchoTest3Service/Echo3");
4466   route3->mutable_route()->set_cluster(kDefaultClusterName);
4467   auto* default_route = new_route_config.mutable_virtual_hosts(0)->add_routes();
4468   default_route->mutable_match()->set_prefix("");
4469   default_route->mutable_route()->set_cluster(kDefaultClusterName);
4470   SetRouteConfiguration(0, new_route_config);
4471   WaitForAllBackends(0, 2);
4472   CheckRpcSendOk(kNumEchoRpcs, RpcOptions().set_wait_for_ready(true));
4473   CheckRpcSendOk(kNumEcho1Rpcs, RpcOptions()
4474                                     .set_rpc_service(SERVICE_ECHO1)
4475                                     .set_rpc_method(METHOD_ECHO1)
4476                                     .set_wait_for_ready(true));
4477   CheckRpcSendOk(kNumEcho2Rpcs, RpcOptions()
4478                                     .set_rpc_service(SERVICE_ECHO2)
4479                                     .set_rpc_method(METHOD_ECHO2)
4480                                     .set_wait_for_ready(true));
4481   // Make sure RPCs all go to the correct backend.
4482   for (size_t i = 0; i < 2; ++i) {
4483     EXPECT_EQ(kNumEchoRpcs / 2,
4484               backends_[i]->backend_service()->request_count());
4485     EXPECT_EQ(0, backends_[i]->backend_service1()->request_count());
4486     EXPECT_EQ(0, backends_[i]->backend_service2()->request_count());
4487   }
4488   EXPECT_EQ(0, backends_[2]->backend_service()->request_count());
4489   EXPECT_EQ(kNumEcho1Rpcs, backends_[2]->backend_service1()->request_count());
4490   EXPECT_EQ(0, backends_[2]->backend_service2()->request_count());
4491   EXPECT_EQ(0, backends_[3]->backend_service()->request_count());
4492   EXPECT_EQ(0, backends_[3]->backend_service1()->request_count());
4493   EXPECT_EQ(kNumEcho2Rpcs, backends_[3]->backend_service2()->request_count());
4494 }
4495
4496 TEST_P(LdsRdsTest, XdsRoutingPathMatchingCaseInsensitive) {
4497   const char* kNewCluster1Name = "new_cluster_1";
4498   const char* kNewEdsService1Name = "new_eds_service_name_1";
4499   const char* kNewCluster2Name = "new_cluster_2";
4500   const char* kNewEdsService2Name = "new_eds_service_name_2";
4501   const size_t kNumEcho1Rpcs = 10;
4502   const size_t kNumEchoRpcs = 30;
4503   SetNextResolution({});
4504   SetNextResolutionForLbChannelAllBalancers();
4505   // Populate new EDS resources.
4506   AdsServiceImpl::EdsResourceArgs args({
4507       {"locality0", CreateEndpointsForBackends(0, 1)},
4508   });
4509   AdsServiceImpl::EdsResourceArgs args1({
4510       {"locality0", CreateEndpointsForBackends(1, 2)},
4511   });
4512   AdsServiceImpl::EdsResourceArgs args2({
4513       {"locality0", CreateEndpointsForBackends(2, 3)},
4514   });
4515   balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
4516   balancers_[0]->ads_service()->SetEdsResource(
4517       BuildEdsResource(args1, kNewEdsService1Name));
4518   balancers_[0]->ads_service()->SetEdsResource(
4519       BuildEdsResource(args2, kNewEdsService2Name));
4520   // Populate new CDS resources.
4521   Cluster new_cluster1 = default_cluster_;
4522   new_cluster1.set_name(kNewCluster1Name);
4523   new_cluster1.mutable_eds_cluster_config()->set_service_name(
4524       kNewEdsService1Name);
4525   balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
4526   Cluster new_cluster2 = default_cluster_;
4527   new_cluster2.set_name(kNewCluster2Name);
4528   new_cluster2.mutable_eds_cluster_config()->set_service_name(
4529       kNewEdsService2Name);
4530   balancers_[0]->ads_service()->SetCdsResource(new_cluster2);
4531   // Populating Route Configurations for LDS.
4532   RouteConfiguration new_route_config = default_route_config_;
4533   // First route will not match, since it's case-sensitive.
4534   // Second route will match with same path.
4535   auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4536   route1->mutable_match()->set_path("/GrPc.TeStInG.EcHoTeSt1SErViCe/EcHo1");
4537   route1->mutable_route()->set_cluster(kNewCluster1Name);
4538   auto* route2 = new_route_config.mutable_virtual_hosts(0)->add_routes();
4539   route2->mutable_match()->set_path("/GrPc.TeStInG.EcHoTeSt1SErViCe/EcHo1");
4540   route2->mutable_match()->mutable_case_sensitive()->set_value(false);
4541   route2->mutable_route()->set_cluster(kNewCluster2Name);
4542   auto* default_route = new_route_config.mutable_virtual_hosts(0)->add_routes();
4543   default_route->mutable_match()->set_prefix("");
4544   default_route->mutable_route()->set_cluster(kDefaultClusterName);
4545   SetRouteConfiguration(0, new_route_config);
4546   CheckRpcSendOk(kNumEchoRpcs, RpcOptions().set_wait_for_ready(true));
4547   CheckRpcSendOk(kNumEcho1Rpcs, RpcOptions()
4548                                     .set_rpc_service(SERVICE_ECHO1)
4549                                     .set_rpc_method(METHOD_ECHO1)
4550                                     .set_wait_for_ready(true));
4551   // Make sure RPCs all go to the correct backend.
4552   EXPECT_EQ(kNumEchoRpcs, backends_[0]->backend_service()->request_count());
4553   EXPECT_EQ(0, backends_[0]->backend_service1()->request_count());
4554   EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
4555   EXPECT_EQ(0, backends_[1]->backend_service1()->request_count());
4556   EXPECT_EQ(0, backends_[2]->backend_service()->request_count());
4557   EXPECT_EQ(kNumEcho1Rpcs, backends_[2]->backend_service1()->request_count());
4558 }
4559
4560 TEST_P(LdsRdsTest, XdsRoutingPrefixMatching) {
4561   const char* kNewCluster1Name = "new_cluster_1";
4562   const char* kNewEdsService1Name = "new_eds_service_name_1";
4563   const char* kNewCluster2Name = "new_cluster_2";
4564   const char* kNewEdsService2Name = "new_eds_service_name_2";
4565   const size_t kNumEcho1Rpcs = 10;
4566   const size_t kNumEcho2Rpcs = 20;
4567   const size_t kNumEchoRpcs = 30;
4568   SetNextResolution({});
4569   SetNextResolutionForLbChannelAllBalancers();
4570   // Populate new EDS resources.
4571   AdsServiceImpl::EdsResourceArgs args({
4572       {"locality0", CreateEndpointsForBackends(0, 2)},
4573   });
4574   AdsServiceImpl::EdsResourceArgs args1({
4575       {"locality0", CreateEndpointsForBackends(2, 3)},
4576   });
4577   AdsServiceImpl::EdsResourceArgs args2({
4578       {"locality0", CreateEndpointsForBackends(3, 4)},
4579   });
4580   balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
4581   balancers_[0]->ads_service()->SetEdsResource(
4582       BuildEdsResource(args1, kNewEdsService1Name));
4583   balancers_[0]->ads_service()->SetEdsResource(
4584       BuildEdsResource(args2, kNewEdsService2Name));
4585   // Populate new CDS resources.
4586   Cluster new_cluster1 = default_cluster_;
4587   new_cluster1.set_name(kNewCluster1Name);
4588   new_cluster1.mutable_eds_cluster_config()->set_service_name(
4589       kNewEdsService1Name);
4590   balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
4591   Cluster new_cluster2 = default_cluster_;
4592   new_cluster2.set_name(kNewCluster2Name);
4593   new_cluster2.mutable_eds_cluster_config()->set_service_name(
4594       kNewEdsService2Name);
4595   balancers_[0]->ads_service()->SetCdsResource(new_cluster2);
4596   // Populating Route Configurations for LDS.
4597   RouteConfiguration new_route_config = default_route_config_;
4598   auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4599   route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
4600   route1->mutable_route()->set_cluster(kNewCluster1Name);
4601   auto* route2 = new_route_config.mutable_virtual_hosts(0)->add_routes();
4602   route2->mutable_match()->set_prefix("/grpc.testing.EchoTest2Service/");
4603   route2->mutable_route()->set_cluster(kNewCluster2Name);
4604   auto* default_route = new_route_config.mutable_virtual_hosts(0)->add_routes();
4605   default_route->mutable_match()->set_prefix("");
4606   default_route->mutable_route()->set_cluster(kDefaultClusterName);
4607   SetRouteConfiguration(0, new_route_config);
4608   WaitForAllBackends(0, 2);
4609   CheckRpcSendOk(kNumEchoRpcs, RpcOptions().set_wait_for_ready(true));
4610   CheckRpcSendOk(
4611       kNumEcho1Rpcs,
4612       RpcOptions().set_rpc_service(SERVICE_ECHO1).set_wait_for_ready(true));
4613   CheckRpcSendOk(
4614       kNumEcho2Rpcs,
4615       RpcOptions().set_rpc_service(SERVICE_ECHO2).set_wait_for_ready(true));
4616   // Make sure RPCs all go to the correct backend.
4617   for (size_t i = 0; i < 2; ++i) {
4618     EXPECT_EQ(kNumEchoRpcs / 2,
4619               backends_[i]->backend_service()->request_count());
4620     EXPECT_EQ(0, backends_[i]->backend_service1()->request_count());
4621     EXPECT_EQ(0, backends_[i]->backend_service2()->request_count());
4622   }
4623   EXPECT_EQ(0, backends_[2]->backend_service()->request_count());
4624   EXPECT_EQ(kNumEcho1Rpcs, backends_[2]->backend_service1()->request_count());
4625   EXPECT_EQ(0, backends_[2]->backend_service2()->request_count());
4626   EXPECT_EQ(0, backends_[3]->backend_service()->request_count());
4627   EXPECT_EQ(0, backends_[3]->backend_service1()->request_count());
4628   EXPECT_EQ(kNumEcho2Rpcs, backends_[3]->backend_service2()->request_count());
4629 }
4630
4631 TEST_P(LdsRdsTest, XdsRoutingPrefixMatchingCaseInsensitive) {
4632   const char* kNewCluster1Name = "new_cluster_1";
4633   const char* kNewEdsService1Name = "new_eds_service_name_1";
4634   const char* kNewCluster2Name = "new_cluster_2";
4635   const char* kNewEdsService2Name = "new_eds_service_name_2";
4636   const size_t kNumEcho1Rpcs = 10;
4637   const size_t kNumEchoRpcs = 30;
4638   SetNextResolution({});
4639   SetNextResolutionForLbChannelAllBalancers();
4640   // Populate new EDS resources.
4641   AdsServiceImpl::EdsResourceArgs args({
4642       {"locality0", CreateEndpointsForBackends(0, 1)},
4643   });
4644   AdsServiceImpl::EdsResourceArgs args1({
4645       {"locality0", CreateEndpointsForBackends(1, 2)},
4646   });
4647   AdsServiceImpl::EdsResourceArgs args2({
4648       {"locality0", CreateEndpointsForBackends(2, 3)},
4649   });
4650   balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
4651   balancers_[0]->ads_service()->SetEdsResource(
4652       BuildEdsResource(args1, kNewEdsService1Name));
4653   balancers_[0]->ads_service()->SetEdsResource(
4654       BuildEdsResource(args2, kNewEdsService2Name));
4655   // Populate new CDS resources.
4656   Cluster new_cluster1 = default_cluster_;
4657   new_cluster1.set_name(kNewCluster1Name);
4658   new_cluster1.mutable_eds_cluster_config()->set_service_name(
4659       kNewEdsService1Name);
4660   balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
4661   Cluster new_cluster2 = default_cluster_;
4662   new_cluster2.set_name(kNewCluster2Name);
4663   new_cluster2.mutable_eds_cluster_config()->set_service_name(
4664       kNewEdsService2Name);
4665   balancers_[0]->ads_service()->SetCdsResource(new_cluster2);
4666   // Populating Route Configurations for LDS.
4667   RouteConfiguration new_route_config = default_route_config_;
4668   // First route will not match, since it's case-sensitive.
4669   // Second route will match with same path.
4670   auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4671   route1->mutable_match()->set_prefix("/GrPc.TeStInG.EcHoTeSt1SErViCe");
4672   route1->mutable_route()->set_cluster(kNewCluster1Name);
4673   auto* route2 = new_route_config.mutable_virtual_hosts(0)->add_routes();
4674   route2->mutable_match()->set_prefix("/GrPc.TeStInG.EcHoTeSt1SErViCe");
4675   route2->mutable_match()->mutable_case_sensitive()->set_value(false);
4676   route2->mutable_route()->set_cluster(kNewCluster2Name);
4677   auto* default_route = new_route_config.mutable_virtual_hosts(0)->add_routes();
4678   default_route->mutable_match()->set_prefix("");
4679   default_route->mutable_route()->set_cluster(kDefaultClusterName);
4680   SetRouteConfiguration(0, new_route_config);
4681   CheckRpcSendOk(kNumEchoRpcs, RpcOptions().set_wait_for_ready(true));
4682   CheckRpcSendOk(kNumEcho1Rpcs, RpcOptions()
4683                                     .set_rpc_service(SERVICE_ECHO1)
4684                                     .set_rpc_method(METHOD_ECHO1)
4685                                     .set_wait_for_ready(true));
4686   // Make sure RPCs all go to the correct backend.
4687   EXPECT_EQ(kNumEchoRpcs, backends_[0]->backend_service()->request_count());
4688   EXPECT_EQ(0, backends_[0]->backend_service1()->request_count());
4689   EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
4690   EXPECT_EQ(0, backends_[1]->backend_service1()->request_count());
4691   EXPECT_EQ(0, backends_[2]->backend_service()->request_count());
4692   EXPECT_EQ(kNumEcho1Rpcs, backends_[2]->backend_service1()->request_count());
4693 }
4694
4695 TEST_P(LdsRdsTest, XdsRoutingPathRegexMatching) {
4696   const char* kNewCluster1Name = "new_cluster_1";
4697   const char* kNewEdsService1Name = "new_eds_service_name_1";
4698   const char* kNewCluster2Name = "new_cluster_2";
4699   const char* kNewEdsService2Name = "new_eds_service_name_2";
4700   const size_t kNumEcho1Rpcs = 10;
4701   const size_t kNumEcho2Rpcs = 20;
4702   const size_t kNumEchoRpcs = 30;
4703   SetNextResolution({});
4704   SetNextResolutionForLbChannelAllBalancers();
4705   // Populate new EDS resources.
4706   AdsServiceImpl::EdsResourceArgs args({
4707       {"locality0", CreateEndpointsForBackends(0, 2)},
4708   });
4709   AdsServiceImpl::EdsResourceArgs args1({
4710       {"locality0", CreateEndpointsForBackends(2, 3)},
4711   });
4712   AdsServiceImpl::EdsResourceArgs args2({
4713       {"locality0", CreateEndpointsForBackends(3, 4)},
4714   });
4715   balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
4716   balancers_[0]->ads_service()->SetEdsResource(
4717       BuildEdsResource(args1, kNewEdsService1Name));
4718   balancers_[0]->ads_service()->SetEdsResource(
4719       BuildEdsResource(args2, kNewEdsService2Name));
4720   // Populate new CDS resources.
4721   Cluster new_cluster1 = default_cluster_;
4722   new_cluster1.set_name(kNewCluster1Name);
4723   new_cluster1.mutable_eds_cluster_config()->set_service_name(
4724       kNewEdsService1Name);
4725   balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
4726   Cluster new_cluster2 = default_cluster_;
4727   new_cluster2.set_name(kNewCluster2Name);
4728   new_cluster2.mutable_eds_cluster_config()->set_service_name(
4729       kNewEdsService2Name);
4730   balancers_[0]->ads_service()->SetCdsResource(new_cluster2);
4731   // Populating Route Configurations for LDS.
4732   RouteConfiguration new_route_config = default_route_config_;
4733   auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4734   // Will match "/grpc.testing.EchoTest1Service/"
4735   route1->mutable_match()->mutable_safe_regex()->set_regex(".*1.*");
4736   route1->mutable_route()->set_cluster(kNewCluster1Name);
4737   auto* route2 = new_route_config.mutable_virtual_hosts(0)->add_routes();
4738   // Will match "/grpc.testing.EchoTest2Service/"
4739   route2->mutable_match()->mutable_safe_regex()->set_regex(".*2.*");
4740   route2->mutable_route()->set_cluster(kNewCluster2Name);
4741   auto* default_route = new_route_config.mutable_virtual_hosts(0)->add_routes();
4742   default_route->mutable_match()->set_prefix("");
4743   default_route->mutable_route()->set_cluster(kDefaultClusterName);
4744   SetRouteConfiguration(0, new_route_config);
4745   WaitForAllBackends(0, 2);
4746   CheckRpcSendOk(kNumEchoRpcs, RpcOptions().set_wait_for_ready(true));
4747   CheckRpcSendOk(
4748       kNumEcho1Rpcs,
4749       RpcOptions().set_rpc_service(SERVICE_ECHO1).set_wait_for_ready(true));
4750   CheckRpcSendOk(
4751       kNumEcho2Rpcs,
4752       RpcOptions().set_rpc_service(SERVICE_ECHO2).set_wait_for_ready(true));
4753   // Make sure RPCs all go to the correct backend.
4754   for (size_t i = 0; i < 2; ++i) {
4755     EXPECT_EQ(kNumEchoRpcs / 2,
4756               backends_[i]->backend_service()->request_count());
4757     EXPECT_EQ(0, backends_[i]->backend_service1()->request_count());
4758     EXPECT_EQ(0, backends_[i]->backend_service2()->request_count());
4759   }
4760   EXPECT_EQ(0, backends_[2]->backend_service()->request_count());
4761   EXPECT_EQ(kNumEcho1Rpcs, backends_[2]->backend_service1()->request_count());
4762   EXPECT_EQ(0, backends_[2]->backend_service2()->request_count());
4763   EXPECT_EQ(0, backends_[3]->backend_service()->request_count());
4764   EXPECT_EQ(0, backends_[3]->backend_service1()->request_count());
4765   EXPECT_EQ(kNumEcho2Rpcs, backends_[3]->backend_service2()->request_count());
4766 }
4767
4768 TEST_P(LdsRdsTest, XdsRoutingWeightedCluster) {
4769   const char* kNewCluster1Name = "new_cluster_1";
4770   const char* kNewEdsService1Name = "new_eds_service_name_1";
4771   const char* kNewCluster2Name = "new_cluster_2";
4772   const char* kNewEdsService2Name = "new_eds_service_name_2";
4773   const char* kNotUsedClusterName = "not_used_cluster";
4774   const size_t kNumEchoRpcs = 10;  // RPCs that will go to a fixed backend.
4775   const size_t kWeight75 = 75;
4776   const size_t kWeight25 = 25;
4777   const double kErrorTolerance = 0.05;
4778   const double kWeight75Percent = static_cast<double>(kWeight75) / 100;
4779   const double kWeight25Percent = static_cast<double>(kWeight25) / 100;
4780   const size_t kNumEcho1Rpcs =
4781       ComputeIdealNumRpcs(kWeight75Percent, kErrorTolerance);
4782   SetNextResolution({});
4783   SetNextResolutionForLbChannelAllBalancers();
4784   // Populate new EDS resources.
4785   AdsServiceImpl::EdsResourceArgs args({
4786       {"locality0", CreateEndpointsForBackends(0, 1)},
4787   });
4788   AdsServiceImpl::EdsResourceArgs args1({
4789       {"locality0", CreateEndpointsForBackends(1, 2)},
4790   });
4791   AdsServiceImpl::EdsResourceArgs args2({
4792       {"locality0", CreateEndpointsForBackends(2, 3)},
4793   });
4794   balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
4795   balancers_[0]->ads_service()->SetEdsResource(
4796       BuildEdsResource(args1, kNewEdsService1Name));
4797   balancers_[0]->ads_service()->SetEdsResource(
4798       BuildEdsResource(args2, kNewEdsService2Name));
4799   // Populate new CDS resources.
4800   Cluster new_cluster1 = default_cluster_;
4801   new_cluster1.set_name(kNewCluster1Name);
4802   new_cluster1.mutable_eds_cluster_config()->set_service_name(
4803       kNewEdsService1Name);
4804   balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
4805   Cluster new_cluster2 = default_cluster_;
4806   new_cluster2.set_name(kNewCluster2Name);
4807   new_cluster2.mutable_eds_cluster_config()->set_service_name(
4808       kNewEdsService2Name);
4809   balancers_[0]->ads_service()->SetCdsResource(new_cluster2);
4810   // Populating Route Configurations for LDS.
4811   RouteConfiguration new_route_config = default_route_config_;
4812   auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4813   route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
4814   auto* weighted_cluster1 =
4815       route1->mutable_route()->mutable_weighted_clusters()->add_clusters();
4816   weighted_cluster1->set_name(kNewCluster1Name);
4817   weighted_cluster1->mutable_weight()->set_value(kWeight75);
4818   auto* weighted_cluster2 =
4819       route1->mutable_route()->mutable_weighted_clusters()->add_clusters();
4820   weighted_cluster2->set_name(kNewCluster2Name);
4821   weighted_cluster2->mutable_weight()->set_value(kWeight25);
4822   // Cluster with weight 0 will not be used.
4823   auto* weighted_cluster3 =
4824       route1->mutable_route()->mutable_weighted_clusters()->add_clusters();
4825   weighted_cluster3->set_name(kNotUsedClusterName);
4826   weighted_cluster3->mutable_weight()->set_value(0);
4827   route1->mutable_route()
4828       ->mutable_weighted_clusters()
4829       ->mutable_total_weight()
4830       ->set_value(kWeight75 + kWeight25);
4831   auto* default_route = new_route_config.mutable_virtual_hosts(0)->add_routes();
4832   default_route->mutable_match()->set_prefix("");
4833   default_route->mutable_route()->set_cluster(kDefaultClusterName);
4834   SetRouteConfiguration(0, new_route_config);
4835   WaitForAllBackends(0, 1);
4836   WaitForAllBackends(1, 3, WaitForBackendOptions(),
4837                      RpcOptions().set_rpc_service(SERVICE_ECHO1));
4838   CheckRpcSendOk(kNumEchoRpcs);
4839   CheckRpcSendOk(kNumEcho1Rpcs, RpcOptions().set_rpc_service(SERVICE_ECHO1));
4840   // Make sure RPCs all go to the correct backend.
4841   EXPECT_EQ(kNumEchoRpcs, backends_[0]->backend_service()->request_count());
4842   EXPECT_EQ(0, backends_[0]->backend_service1()->request_count());
4843   EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
4844   const int weight_75_request_count =
4845       backends_[1]->backend_service1()->request_count();
4846   EXPECT_EQ(0, backends_[2]->backend_service()->request_count());
4847   const int weight_25_request_count =
4848       backends_[2]->backend_service1()->request_count();
4849   gpr_log(GPR_INFO, "target_75 received %d rpcs and target_25 received %d rpcs",
4850           weight_75_request_count, weight_25_request_count);
4851   EXPECT_THAT(static_cast<double>(weight_75_request_count) / kNumEcho1Rpcs,
4852               ::testing::DoubleNear(kWeight75Percent, kErrorTolerance));
4853   EXPECT_THAT(static_cast<double>(weight_25_request_count) / kNumEcho1Rpcs,
4854               ::testing::DoubleNear(kWeight25Percent, kErrorTolerance));
4855 }
4856
4857 TEST_P(LdsRdsTest, RouteActionWeightedTargetDefaultRoute) {
4858   const char* kNewCluster1Name = "new_cluster_1";
4859   const char* kNewEdsService1Name = "new_eds_service_name_1";
4860   const char* kNewCluster2Name = "new_cluster_2";
4861   const char* kNewEdsService2Name = "new_eds_service_name_2";
4862   const size_t kWeight75 = 75;
4863   const size_t kWeight25 = 25;
4864   const double kErrorTolerance = 0.05;
4865   const double kWeight75Percent = static_cast<double>(kWeight75) / 100;
4866   const double kWeight25Percent = static_cast<double>(kWeight25) / 100;
4867   const size_t kNumEchoRpcs =
4868       ComputeIdealNumRpcs(kWeight75Percent, kErrorTolerance);
4869   SetNextResolution({});
4870   SetNextResolutionForLbChannelAllBalancers();
4871   // Populate new EDS resources.
4872   AdsServiceImpl::EdsResourceArgs args({
4873       {"locality0", CreateEndpointsForBackends(0, 1)},
4874   });
4875   AdsServiceImpl::EdsResourceArgs args1({
4876       {"locality0", CreateEndpointsForBackends(1, 2)},
4877   });
4878   AdsServiceImpl::EdsResourceArgs args2({
4879       {"locality0", CreateEndpointsForBackends(2, 3)},
4880   });
4881   balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
4882   balancers_[0]->ads_service()->SetEdsResource(
4883       BuildEdsResource(args1, kNewEdsService1Name));
4884   balancers_[0]->ads_service()->SetEdsResource(
4885       BuildEdsResource(args2, kNewEdsService2Name));
4886   // Populate new CDS resources.
4887   Cluster new_cluster1 = default_cluster_;
4888   new_cluster1.set_name(kNewCluster1Name);
4889   new_cluster1.mutable_eds_cluster_config()->set_service_name(
4890       kNewEdsService1Name);
4891   balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
4892   Cluster new_cluster2 = default_cluster_;
4893   new_cluster2.set_name(kNewCluster2Name);
4894   new_cluster2.mutable_eds_cluster_config()->set_service_name(
4895       kNewEdsService2Name);
4896   balancers_[0]->ads_service()->SetCdsResource(new_cluster2);
4897   // Populating Route Configurations for LDS.
4898   RouteConfiguration new_route_config = default_route_config_;
4899   auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4900   route1->mutable_match()->set_prefix("");
4901   auto* weighted_cluster1 =
4902       route1->mutable_route()->mutable_weighted_clusters()->add_clusters();
4903   weighted_cluster1->set_name(kNewCluster1Name);
4904   weighted_cluster1->mutable_weight()->set_value(kWeight75);
4905   auto* weighted_cluster2 =
4906       route1->mutable_route()->mutable_weighted_clusters()->add_clusters();
4907   weighted_cluster2->set_name(kNewCluster2Name);
4908   weighted_cluster2->mutable_weight()->set_value(kWeight25);
4909   route1->mutable_route()
4910       ->mutable_weighted_clusters()
4911       ->mutable_total_weight()
4912       ->set_value(kWeight75 + kWeight25);
4913   SetRouteConfiguration(0, new_route_config);
4914   WaitForAllBackends(1, 3);
4915   CheckRpcSendOk(kNumEchoRpcs);
4916   // Make sure RPCs all go to the correct backend.
4917   EXPECT_EQ(0, backends_[0]->backend_service()->request_count());
4918   const int weight_75_request_count =
4919       backends_[1]->backend_service()->request_count();
4920   const int weight_25_request_count =
4921       backends_[2]->backend_service()->request_count();
4922   gpr_log(GPR_INFO, "target_75 received %d rpcs and target_25 received %d rpcs",
4923           weight_75_request_count, weight_25_request_count);
4924   EXPECT_THAT(static_cast<double>(weight_75_request_count) / kNumEchoRpcs,
4925               ::testing::DoubleNear(kWeight75Percent, kErrorTolerance));
4926   EXPECT_THAT(static_cast<double>(weight_25_request_count) / kNumEchoRpcs,
4927               ::testing::DoubleNear(kWeight25Percent, kErrorTolerance));
4928 }
4929
4930 TEST_P(LdsRdsTest, XdsRoutingWeightedClusterUpdateWeights) {
4931   const char* kNewCluster1Name = "new_cluster_1";
4932   const char* kNewEdsService1Name = "new_eds_service_name_1";
4933   const char* kNewCluster2Name = "new_cluster_2";
4934   const char* kNewEdsService2Name = "new_eds_service_name_2";
4935   const char* kNewCluster3Name = "new_cluster_3";
4936   const char* kNewEdsService3Name = "new_eds_service_name_3";
4937   const size_t kNumEchoRpcs = 10;
4938   const size_t kWeight75 = 75;
4939   const size_t kWeight25 = 25;
4940   const size_t kWeight50 = 50;
4941   const double kErrorTolerance = 0.05;
4942   const double kWeight75Percent = static_cast<double>(kWeight75) / 100;
4943   const double kWeight25Percent = static_cast<double>(kWeight25) / 100;
4944   const double kWeight50Percent = static_cast<double>(kWeight50) / 100;
4945   const size_t kNumEcho1Rpcs7525 =
4946       ComputeIdealNumRpcs(kWeight75Percent, kErrorTolerance);
4947   const size_t kNumEcho1Rpcs5050 =
4948       ComputeIdealNumRpcs(kWeight50Percent, kErrorTolerance);
4949   SetNextResolution({});
4950   SetNextResolutionForLbChannelAllBalancers();
4951   // Populate new EDS resources.
4952   AdsServiceImpl::EdsResourceArgs args({
4953       {"locality0", CreateEndpointsForBackends(0, 1)},
4954   });
4955   AdsServiceImpl::EdsResourceArgs args1({
4956       {"locality0", CreateEndpointsForBackends(1, 2)},
4957   });
4958   AdsServiceImpl::EdsResourceArgs args2({
4959       {"locality0", CreateEndpointsForBackends(2, 3)},
4960   });
4961   AdsServiceImpl::EdsResourceArgs args3({
4962       {"locality0", CreateEndpointsForBackends(3, 4)},
4963   });
4964   balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
4965   balancers_[0]->ads_service()->SetEdsResource(
4966       BuildEdsResource(args1, kNewEdsService1Name));
4967   balancers_[0]->ads_service()->SetEdsResource(
4968       BuildEdsResource(args2, kNewEdsService2Name));
4969   balancers_[0]->ads_service()->SetEdsResource(
4970       BuildEdsResource(args3, kNewEdsService3Name));
4971   // Populate new CDS resources.
4972   Cluster new_cluster1 = default_cluster_;
4973   new_cluster1.set_name(kNewCluster1Name);
4974   new_cluster1.mutable_eds_cluster_config()->set_service_name(
4975       kNewEdsService1Name);
4976   balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
4977   Cluster new_cluster2 = default_cluster_;
4978   new_cluster2.set_name(kNewCluster2Name);
4979   new_cluster2.mutable_eds_cluster_config()->set_service_name(
4980       kNewEdsService2Name);
4981   balancers_[0]->ads_service()->SetCdsResource(new_cluster2);
4982   Cluster new_cluster3 = default_cluster_;
4983   new_cluster3.set_name(kNewCluster3Name);
4984   new_cluster3.mutable_eds_cluster_config()->set_service_name(
4985       kNewEdsService3Name);
4986   balancers_[0]->ads_service()->SetCdsResource(new_cluster3);
4987   // Populating Route Configurations.
4988   RouteConfiguration new_route_config = default_route_config_;
4989   auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
4990   route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
4991   auto* weighted_cluster1 =
4992       route1->mutable_route()->mutable_weighted_clusters()->add_clusters();
4993   weighted_cluster1->set_name(kNewCluster1Name);
4994   weighted_cluster1->mutable_weight()->set_value(kWeight75);
4995   auto* weighted_cluster2 =
4996       route1->mutable_route()->mutable_weighted_clusters()->add_clusters();
4997   weighted_cluster2->set_name(kNewCluster2Name);
4998   weighted_cluster2->mutable_weight()->set_value(kWeight25);
4999   route1->mutable_route()
5000       ->mutable_weighted_clusters()
5001       ->mutable_total_weight()
5002       ->set_value(kWeight75 + kWeight25);
5003   auto* default_route = new_route_config.mutable_virtual_hosts(0)->add_routes();
5004   default_route->mutable_match()->set_prefix("");
5005   default_route->mutable_route()->set_cluster(kDefaultClusterName);
5006   SetRouteConfiguration(0, new_route_config);
5007   WaitForAllBackends(0, 1);
5008   WaitForAllBackends(1, 3, WaitForBackendOptions(),
5009                      RpcOptions().set_rpc_service(SERVICE_ECHO1));
5010   CheckRpcSendOk(kNumEchoRpcs);
5011   CheckRpcSendOk(kNumEcho1Rpcs7525,
5012                  RpcOptions().set_rpc_service(SERVICE_ECHO1));
5013   // Make sure RPCs all go to the correct backend.
5014   EXPECT_EQ(kNumEchoRpcs, backends_[0]->backend_service()->request_count());
5015   EXPECT_EQ(0, backends_[0]->backend_service1()->request_count());
5016   EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
5017   const int weight_75_request_count =
5018       backends_[1]->backend_service1()->request_count();
5019   EXPECT_EQ(0, backends_[1]->backend_service2()->request_count());
5020   EXPECT_EQ(0, backends_[2]->backend_service()->request_count());
5021   const int weight_25_request_count =
5022       backends_[2]->backend_service1()->request_count();
5023   EXPECT_EQ(0, backends_[3]->backend_service()->request_count());
5024   EXPECT_EQ(0, backends_[3]->backend_service1()->request_count());
5025   gpr_log(GPR_INFO, "target_75 received %d rpcs and target_25 received %d rpcs",
5026           weight_75_request_count, weight_25_request_count);
5027   EXPECT_THAT(static_cast<double>(weight_75_request_count) / kNumEcho1Rpcs7525,
5028               ::testing::DoubleNear(kWeight75Percent, kErrorTolerance));
5029   EXPECT_THAT(static_cast<double>(weight_25_request_count) / kNumEcho1Rpcs7525,
5030               ::testing::DoubleNear(kWeight25Percent, kErrorTolerance));
5031   // Change Route Configurations: same clusters different weights.
5032   weighted_cluster1->mutable_weight()->set_value(kWeight50);
5033   weighted_cluster2->mutable_weight()->set_value(kWeight50);
5034   // Change default route to a new cluster to help to identify when new
5035   // polices are seen by the client.
5036   default_route->mutable_route()->set_cluster(kNewCluster3Name);
5037   SetRouteConfiguration(0, new_route_config);
5038   ResetBackendCounters();
5039   WaitForAllBackends(3, 4);
5040   CheckRpcSendOk(kNumEchoRpcs);
5041   CheckRpcSendOk(kNumEcho1Rpcs5050,
5042                  RpcOptions().set_rpc_service(SERVICE_ECHO1));
5043   // Make sure RPCs all go to the correct backend.
5044   EXPECT_EQ(0, backends_[0]->backend_service()->request_count());
5045   EXPECT_EQ(0, backends_[0]->backend_service1()->request_count());
5046   EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
5047   const int weight_50_request_count_1 =
5048       backends_[1]->backend_service1()->request_count();
5049   EXPECT_EQ(0, backends_[2]->backend_service()->request_count());
5050   const int weight_50_request_count_2 =
5051       backends_[2]->backend_service1()->request_count();
5052   EXPECT_EQ(kNumEchoRpcs, backends_[3]->backend_service()->request_count());
5053   EXPECT_EQ(0, backends_[3]->backend_service1()->request_count());
5054   EXPECT_THAT(
5055       static_cast<double>(weight_50_request_count_1) / kNumEcho1Rpcs5050,
5056       ::testing::DoubleNear(kWeight50Percent, kErrorTolerance));
5057   EXPECT_THAT(
5058       static_cast<double>(weight_50_request_count_2) / kNumEcho1Rpcs5050,
5059       ::testing::DoubleNear(kWeight50Percent, kErrorTolerance));
5060 }
5061
5062 TEST_P(LdsRdsTest, XdsRoutingWeightedClusterUpdateClusters) {
5063   const char* kNewCluster1Name = "new_cluster_1";
5064   const char* kNewEdsService1Name = "new_eds_service_name_1";
5065   const char* kNewCluster2Name = "new_cluster_2";
5066   const char* kNewEdsService2Name = "new_eds_service_name_2";
5067   const char* kNewCluster3Name = "new_cluster_3";
5068   const char* kNewEdsService3Name = "new_eds_service_name_3";
5069   const size_t kNumEchoRpcs = 10;
5070   const size_t kWeight75 = 75;
5071   const size_t kWeight25 = 25;
5072   const size_t kWeight50 = 50;
5073   const double kErrorTolerance = 0.05;
5074   const double kWeight75Percent = static_cast<double>(kWeight75) / 100;
5075   const double kWeight25Percent = static_cast<double>(kWeight25) / 100;
5076   const double kWeight50Percent = static_cast<double>(kWeight50) / 100;
5077   const size_t kNumEcho1Rpcs7525 =
5078       ComputeIdealNumRpcs(kWeight75Percent, kErrorTolerance);
5079   const size_t kNumEcho1Rpcs5050 =
5080       ComputeIdealNumRpcs(kWeight50Percent, kErrorTolerance);
5081   SetNextResolution({});
5082   SetNextResolutionForLbChannelAllBalancers();
5083   // Populate new EDS resources.
5084   AdsServiceImpl::EdsResourceArgs args({
5085       {"locality0", CreateEndpointsForBackends(0, 1)},
5086   });
5087   AdsServiceImpl::EdsResourceArgs args1({
5088       {"locality0", CreateEndpointsForBackends(1, 2)},
5089   });
5090   AdsServiceImpl::EdsResourceArgs args2({
5091       {"locality0", CreateEndpointsForBackends(2, 3)},
5092   });
5093   AdsServiceImpl::EdsResourceArgs args3({
5094       {"locality0", CreateEndpointsForBackends(3, 4)},
5095   });
5096   balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
5097   balancers_[0]->ads_service()->SetEdsResource(
5098       BuildEdsResource(args1, kNewEdsService1Name));
5099   balancers_[0]->ads_service()->SetEdsResource(
5100       BuildEdsResource(args2, kNewEdsService2Name));
5101   balancers_[0]->ads_service()->SetEdsResource(
5102       BuildEdsResource(args3, kNewEdsService3Name));
5103   // Populate new CDS resources.
5104   Cluster new_cluster1 = default_cluster_;
5105   new_cluster1.set_name(kNewCluster1Name);
5106   new_cluster1.mutable_eds_cluster_config()->set_service_name(
5107       kNewEdsService1Name);
5108   balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
5109   Cluster new_cluster2 = default_cluster_;
5110   new_cluster2.set_name(kNewCluster2Name);
5111   new_cluster2.mutable_eds_cluster_config()->set_service_name(
5112       kNewEdsService2Name);
5113   balancers_[0]->ads_service()->SetCdsResource(new_cluster2);
5114   Cluster new_cluster3 = default_cluster_;
5115   new_cluster3.set_name(kNewCluster3Name);
5116   new_cluster3.mutable_eds_cluster_config()->set_service_name(
5117       kNewEdsService3Name);
5118   balancers_[0]->ads_service()->SetCdsResource(new_cluster3);
5119   // Populating Route Configurations.
5120   RouteConfiguration new_route_config = default_route_config_;
5121   auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
5122   route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
5123   auto* weighted_cluster1 =
5124       route1->mutable_route()->mutable_weighted_clusters()->add_clusters();
5125   weighted_cluster1->set_name(kNewCluster1Name);
5126   weighted_cluster1->mutable_weight()->set_value(kWeight75);
5127   auto* weighted_cluster2 =
5128       route1->mutable_route()->mutable_weighted_clusters()->add_clusters();
5129   weighted_cluster2->set_name(kDefaultClusterName);
5130   weighted_cluster2->mutable_weight()->set_value(kWeight25);
5131   route1->mutable_route()
5132       ->mutable_weighted_clusters()
5133       ->mutable_total_weight()
5134       ->set_value(kWeight75 + kWeight25);
5135   auto* default_route = new_route_config.mutable_virtual_hosts(0)->add_routes();
5136   default_route->mutable_match()->set_prefix("");
5137   default_route->mutable_route()->set_cluster(kDefaultClusterName);
5138   SetRouteConfiguration(0, new_route_config);
5139   WaitForBackend(0);
5140   WaitForBackend(1, WaitForBackendOptions(),
5141                  RpcOptions().set_rpc_service(SERVICE_ECHO1));
5142   CheckRpcSendOk(kNumEchoRpcs);
5143   CheckRpcSendOk(kNumEcho1Rpcs7525,
5144                  RpcOptions().set_rpc_service(SERVICE_ECHO1));
5145   // Make sure RPCs all go to the correct backend.
5146   EXPECT_EQ(kNumEchoRpcs, backends_[0]->backend_service()->request_count());
5147   int weight_25_request_count =
5148       backends_[0]->backend_service1()->request_count();
5149   EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
5150   int weight_75_request_count =
5151       backends_[1]->backend_service1()->request_count();
5152   EXPECT_EQ(0, backends_[2]->backend_service()->request_count());
5153   EXPECT_EQ(0, backends_[2]->backend_service1()->request_count());
5154   EXPECT_EQ(0, backends_[3]->backend_service()->request_count());
5155   EXPECT_EQ(0, backends_[3]->backend_service1()->request_count());
5156   gpr_log(GPR_INFO, "target_75 received %d rpcs and target_25 received %d rpcs",
5157           weight_75_request_count, weight_25_request_count);
5158   EXPECT_THAT(static_cast<double>(weight_75_request_count) / kNumEcho1Rpcs7525,
5159               ::testing::DoubleNear(kWeight75Percent, kErrorTolerance));
5160   EXPECT_THAT(static_cast<double>(weight_25_request_count) / kNumEcho1Rpcs7525,
5161               ::testing::DoubleNear(kWeight25Percent, kErrorTolerance));
5162   // Change Route Configurations: new set of clusters with different weights.
5163   weighted_cluster1->mutable_weight()->set_value(kWeight50);
5164   weighted_cluster2->set_name(kNewCluster2Name);
5165   weighted_cluster2->mutable_weight()->set_value(kWeight50);
5166   SetRouteConfiguration(0, new_route_config);
5167   ResetBackendCounters();
5168   WaitForBackend(2, WaitForBackendOptions(),
5169                  RpcOptions().set_rpc_service(SERVICE_ECHO1));
5170   CheckRpcSendOk(kNumEchoRpcs);
5171   CheckRpcSendOk(kNumEcho1Rpcs5050,
5172                  RpcOptions().set_rpc_service(SERVICE_ECHO1));
5173   // Make sure RPCs all go to the correct backend.
5174   EXPECT_EQ(kNumEchoRpcs, backends_[0]->backend_service()->request_count());
5175   EXPECT_EQ(0, backends_[0]->backend_service1()->request_count());
5176   EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
5177   const int weight_50_request_count_1 =
5178       backends_[1]->backend_service1()->request_count();
5179   EXPECT_EQ(0, backends_[2]->backend_service()->request_count());
5180   const int weight_50_request_count_2 =
5181       backends_[2]->backend_service1()->request_count();
5182   EXPECT_EQ(0, backends_[3]->backend_service()->request_count());
5183   EXPECT_EQ(0, backends_[3]->backend_service1()->request_count());
5184   EXPECT_THAT(
5185       static_cast<double>(weight_50_request_count_1) / kNumEcho1Rpcs5050,
5186       ::testing::DoubleNear(kWeight50Percent, kErrorTolerance));
5187   EXPECT_THAT(
5188       static_cast<double>(weight_50_request_count_2) / kNumEcho1Rpcs5050,
5189       ::testing::DoubleNear(kWeight50Percent, kErrorTolerance));
5190   // Change Route Configurations.
5191   weighted_cluster1->mutable_weight()->set_value(kWeight75);
5192   weighted_cluster2->set_name(kNewCluster3Name);
5193   weighted_cluster2->mutable_weight()->set_value(kWeight25);
5194   SetRouteConfiguration(0, new_route_config);
5195   ResetBackendCounters();
5196   WaitForBackend(3, WaitForBackendOptions(),
5197                  RpcOptions().set_rpc_service(SERVICE_ECHO1));
5198   CheckRpcSendOk(kNumEchoRpcs);
5199   CheckRpcSendOk(kNumEcho1Rpcs7525,
5200                  RpcOptions().set_rpc_service(SERVICE_ECHO1));
5201   // Make sure RPCs all go to the correct backend.
5202   EXPECT_EQ(kNumEchoRpcs, backends_[0]->backend_service()->request_count());
5203   EXPECT_EQ(0, backends_[0]->backend_service1()->request_count());
5204   EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
5205   weight_75_request_count = backends_[1]->backend_service1()->request_count();
5206   EXPECT_EQ(0, backends_[2]->backend_service()->request_count());
5207   EXPECT_EQ(0, backends_[2]->backend_service1()->request_count());
5208   EXPECT_EQ(0, backends_[3]->backend_service()->request_count());
5209   weight_25_request_count = backends_[3]->backend_service1()->request_count();
5210   gpr_log(GPR_INFO, "target_75 received %d rpcs and target_25 received %d rpcs",
5211           weight_75_request_count, weight_25_request_count);
5212   EXPECT_THAT(static_cast<double>(weight_75_request_count) / kNumEcho1Rpcs7525,
5213               ::testing::DoubleNear(kWeight75Percent, kErrorTolerance));
5214   EXPECT_THAT(static_cast<double>(weight_25_request_count) / kNumEcho1Rpcs7525,
5215               ::testing::DoubleNear(kWeight25Percent, kErrorTolerance));
5216 }
5217
5218 TEST_P(LdsRdsTest, XdsRoutingClusterUpdateClusters) {
5219   const char* kNewClusterName = "new_cluster";
5220   const char* kNewEdsServiceName = "new_eds_service_name";
5221   const size_t kNumEchoRpcs = 5;
5222   SetNextResolution({});
5223   SetNextResolutionForLbChannelAllBalancers();
5224   // Populate new EDS resources.
5225   AdsServiceImpl::EdsResourceArgs args({
5226       {"locality0", CreateEndpointsForBackends(0, 1)},
5227   });
5228   AdsServiceImpl::EdsResourceArgs args1({
5229       {"locality0", CreateEndpointsForBackends(1, 2)},
5230   });
5231   balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
5232   balancers_[0]->ads_service()->SetEdsResource(
5233       BuildEdsResource(args1, kNewEdsServiceName));
5234   // Populate new CDS resources.
5235   Cluster new_cluster = default_cluster_;
5236   new_cluster.set_name(kNewClusterName);
5237   new_cluster.mutable_eds_cluster_config()->set_service_name(
5238       kNewEdsServiceName);
5239   balancers_[0]->ads_service()->SetCdsResource(new_cluster);
5240   // Send Route Configuration.
5241   RouteConfiguration new_route_config = default_route_config_;
5242   SetRouteConfiguration(0, new_route_config);
5243   WaitForAllBackends(0, 1);
5244   CheckRpcSendOk(kNumEchoRpcs);
5245   // Make sure RPCs all go to the correct backend.
5246   EXPECT_EQ(kNumEchoRpcs, backends_[0]->backend_service()->request_count());
5247   // Change Route Configurations: new default cluster.
5248   auto* default_route =
5249       new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
5250   default_route->mutable_route()->set_cluster(kNewClusterName);
5251   SetRouteConfiguration(0, new_route_config);
5252   WaitForAllBackends(1, 2);
5253   CheckRpcSendOk(kNumEchoRpcs);
5254   // Make sure RPCs all go to the correct backend.
5255   EXPECT_EQ(kNumEchoRpcs, backends_[1]->backend_service()->request_count());
5256 }
5257
5258 TEST_P(LdsRdsTest, XdsRoutingClusterUpdateClustersWithPickingDelays) {
5259   const char* kNewClusterName = "new_cluster";
5260   const char* kNewEdsServiceName = "new_eds_service_name";
5261   SetNextResolution({});
5262   SetNextResolutionForLbChannelAllBalancers();
5263   // Populate new EDS resources.
5264   AdsServiceImpl::EdsResourceArgs args({
5265       {"locality0", CreateEndpointsForBackends(0, 1)},
5266   });
5267   AdsServiceImpl::EdsResourceArgs args1({
5268       {"locality0", CreateEndpointsForBackends(1, 2)},
5269   });
5270   balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
5271   balancers_[0]->ads_service()->SetEdsResource(
5272       BuildEdsResource(args1, kNewEdsServiceName));
5273   // Populate new CDS resources.
5274   Cluster new_cluster = default_cluster_;
5275   new_cluster.set_name(kNewClusterName);
5276   new_cluster.mutable_eds_cluster_config()->set_service_name(
5277       kNewEdsServiceName);
5278   balancers_[0]->ads_service()->SetCdsResource(new_cluster);
5279   // Bring down the current backend: 0, this will delay route picking time,
5280   // resulting in un-committed RPCs.
5281   ShutdownBackend(0);
5282   // Send a RouteConfiguration with a default route that points to
5283   // backend 0.
5284   RouteConfiguration new_route_config = default_route_config_;
5285   SetRouteConfiguration(0, new_route_config);
5286   // Send exactly one RPC with no deadline and with wait_for_ready=true.
5287   // This RPC will not complete until after backend 0 is started.
5288   std::thread sending_rpc([this]() {
5289     CheckRpcSendOk(1, RpcOptions().set_wait_for_ready(true).set_timeout_ms(0));
5290   });
5291   // Send a non-wait_for_ready RPC which should fail, this will tell us
5292   // that the client has received the update and attempted to connect.
5293   const Status status = SendRpc(RpcOptions().set_timeout_ms(0));
5294   EXPECT_FALSE(status.ok());
5295   // Send a update RouteConfiguration to use backend 1.
5296   auto* default_route =
5297       new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
5298   default_route->mutable_route()->set_cluster(kNewClusterName);
5299   SetRouteConfiguration(0, new_route_config);
5300   // Wait for RPCs to go to the new backend: 1, this ensures that the client
5301   // has processed the update.
5302   WaitForBackend(
5303       1, WaitForBackendOptions().set_reset_counters(false).set_allow_failures(
5304              true));
5305   // Bring up the previous backend: 0, this will allow the delayed RPC to
5306   // finally call on_call_committed upon completion.
5307   StartBackend(0);
5308   sending_rpc.join();
5309   // Make sure RPCs go to the correct backend:
5310   EXPECT_EQ(1, backends_[0]->backend_service()->request_count());
5311   EXPECT_EQ(1, backends_[1]->backend_service()->request_count());
5312 }
5313
5314 TEST_P(LdsRdsTest, XdsRoutingApplyXdsTimeout) {
5315   const int64_t kTimeoutMillis = 500;
5316   const int64_t kTimeoutNano = kTimeoutMillis * 1000000;
5317   const int64_t kTimeoutGrpcTimeoutHeaderMaxSecond = 1;
5318   const int64_t kTimeoutMaxStreamDurationSecond = 2;
5319   const int64_t kTimeoutHttpMaxStreamDurationSecond = 3;
5320   const int64_t kTimeoutApplicationSecond = 4;
5321   const char* kNewCluster1Name = "new_cluster_1";
5322   const char* kNewEdsService1Name = "new_eds_service_name_1";
5323   const char* kNewCluster2Name = "new_cluster_2";
5324   const char* kNewEdsService2Name = "new_eds_service_name_2";
5325   const char* kNewCluster3Name = "new_cluster_3";
5326   const char* kNewEdsService3Name = "new_eds_service_name_3";
5327   SetNextResolution({});
5328   SetNextResolutionForLbChannelAllBalancers();
5329   // Populate new EDS resources.
5330   AdsServiceImpl::EdsResourceArgs args(
5331       {{"locality0", {MakeNonExistantEndpoint()}}});
5332   AdsServiceImpl::EdsResourceArgs args1(
5333       {{"locality0", {MakeNonExistantEndpoint()}}});
5334   AdsServiceImpl::EdsResourceArgs args2(
5335       {{"locality0", {MakeNonExistantEndpoint()}}});
5336   AdsServiceImpl::EdsResourceArgs args3(
5337       {{"locality0", {MakeNonExistantEndpoint()}}});
5338   balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
5339   balancers_[0]->ads_service()->SetEdsResource(
5340       BuildEdsResource(args1, kNewEdsService1Name));
5341   balancers_[0]->ads_service()->SetEdsResource(
5342       BuildEdsResource(args2, kNewEdsService2Name));
5343   balancers_[0]->ads_service()->SetEdsResource(
5344       BuildEdsResource(args3, kNewEdsService3Name));
5345   // Populate new CDS resources.
5346   Cluster new_cluster1 = default_cluster_;
5347   new_cluster1.set_name(kNewCluster1Name);
5348   new_cluster1.mutable_eds_cluster_config()->set_service_name(
5349       kNewEdsService1Name);
5350   balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
5351   Cluster new_cluster2 = default_cluster_;
5352   new_cluster2.set_name(kNewCluster2Name);
5353   new_cluster2.mutable_eds_cluster_config()->set_service_name(
5354       kNewEdsService2Name);
5355   balancers_[0]->ads_service()->SetCdsResource(new_cluster2);
5356   Cluster new_cluster3 = default_cluster_;
5357   new_cluster3.set_name(kNewCluster3Name);
5358   new_cluster3.mutable_eds_cluster_config()->set_service_name(
5359       kNewEdsService3Name);
5360   balancers_[0]->ads_service()->SetCdsResource(new_cluster3);
5361   // Construct listener.
5362   auto listener = default_listener_;
5363   HttpConnectionManager http_connection_manager;
5364   listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
5365       &http_connection_manager);
5366   // Set up HTTP max_stream_duration of 3.5 seconds
5367   auto* duration =
5368       http_connection_manager.mutable_common_http_protocol_options()
5369           ->mutable_max_stream_duration();
5370   duration->set_seconds(kTimeoutHttpMaxStreamDurationSecond);
5371   duration->set_nanos(kTimeoutNano);
5372   listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
5373       http_connection_manager);
5374   // Construct route config.
5375   RouteConfiguration new_route_config = default_route_config_;
5376   // route 1: Set max_stream_duration of 2.5 seconds, Set
5377   // grpc_timeout_header_max of 1.5
5378   auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
5379   route1->mutable_match()->set_path("/grpc.testing.EchoTest1Service/Echo1");
5380   route1->mutable_route()->set_cluster(kNewCluster1Name);
5381   auto* max_stream_duration =
5382       route1->mutable_route()->mutable_max_stream_duration();
5383   duration = max_stream_duration->mutable_max_stream_duration();
5384   duration->set_seconds(kTimeoutMaxStreamDurationSecond);
5385   duration->set_nanos(kTimeoutNano);
5386   duration = max_stream_duration->mutable_grpc_timeout_header_max();
5387   duration->set_seconds(kTimeoutGrpcTimeoutHeaderMaxSecond);
5388   duration->set_nanos(kTimeoutNano);
5389   // route 2: Set max_stream_duration of 2.5 seconds
5390   auto* route2 = new_route_config.mutable_virtual_hosts(0)->add_routes();
5391   route2->mutable_match()->set_path("/grpc.testing.EchoTest2Service/Echo2");
5392   route2->mutable_route()->set_cluster(kNewCluster2Name);
5393   max_stream_duration = route2->mutable_route()->mutable_max_stream_duration();
5394   duration = max_stream_duration->mutable_max_stream_duration();
5395   duration->set_seconds(kTimeoutMaxStreamDurationSecond);
5396   duration->set_nanos(kTimeoutNano);
5397   // route 3: No timeout values in route configuration
5398   auto* route3 = new_route_config.mutable_virtual_hosts(0)->add_routes();
5399   route3->mutable_match()->set_path("/grpc.testing.EchoTestService/Echo");
5400   route3->mutable_route()->set_cluster(kNewCluster3Name);
5401   // Set listener and route config.
5402   SetListenerAndRouteConfiguration(0, std::move(listener), new_route_config);
5403   // Test grpc_timeout_header_max of 1.5 seconds applied
5404   grpc_millis t0 = NowFromCycleCounter();
5405   grpc_millis t1 =
5406       t0 + kTimeoutGrpcTimeoutHeaderMaxSecond * 1000 + kTimeoutMillis;
5407   grpc_millis t2 = t0 + kTimeoutMaxStreamDurationSecond * 1000 + kTimeoutMillis;
5408   CheckRpcSendFailure(
5409       CheckRpcSendFailureOptions()
5410           .set_rpc_options(
5411               RpcOptions()
5412                   .set_rpc_service(SERVICE_ECHO1)
5413                   .set_rpc_method(METHOD_ECHO1)
5414                   .set_wait_for_ready(true)
5415                   .set_timeout_ms(kTimeoutApplicationSecond * 1000))
5416           .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
5417   t0 = NowFromCycleCounter();
5418   EXPECT_GE(t0, t1);
5419   EXPECT_LT(t0, t2);
5420   // Test max_stream_duration of 2.5 seconds applied
5421   t0 = NowFromCycleCounter();
5422   t1 = t0 + kTimeoutMaxStreamDurationSecond * 1000 + kTimeoutMillis;
5423   t2 = t0 + kTimeoutHttpMaxStreamDurationSecond * 1000 + kTimeoutMillis;
5424   CheckRpcSendFailure(
5425       CheckRpcSendFailureOptions()
5426           .set_rpc_options(
5427               RpcOptions()
5428                   .set_rpc_service(SERVICE_ECHO2)
5429                   .set_rpc_method(METHOD_ECHO2)
5430                   .set_wait_for_ready(true)
5431                   .set_timeout_ms(kTimeoutApplicationSecond * 1000))
5432           .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
5433   t0 = NowFromCycleCounter();
5434   EXPECT_GE(t0, t1);
5435   EXPECT_LT(t0, t2);
5436   // Test http_stream_duration of 3.5 seconds applied
5437   t0 = NowFromCycleCounter();
5438   t1 = t0 + kTimeoutHttpMaxStreamDurationSecond * 1000 + kTimeoutMillis;
5439   t2 = t0 + kTimeoutApplicationSecond * 1000 + kTimeoutMillis;
5440   CheckRpcSendFailure(
5441       CheckRpcSendFailureOptions()
5442           .set_rpc_options(RpcOptions().set_wait_for_ready(true).set_timeout_ms(
5443               kTimeoutApplicationSecond * 1000))
5444           .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
5445   t0 = NowFromCycleCounter();
5446   EXPECT_GE(t0, t1);
5447   EXPECT_LT(t0, t2);
5448 }
5449
5450 TEST_P(LdsRdsTest, XdsRoutingApplyApplicationTimeoutWhenXdsTimeoutExplicit0) {
5451   const int64_t kTimeoutNano = 500000000;
5452   const int64_t kTimeoutMaxStreamDurationSecond = 2;
5453   const int64_t kTimeoutHttpMaxStreamDurationSecond = 3;
5454   const int64_t kTimeoutApplicationSecond = 4;
5455   const char* kNewCluster1Name = "new_cluster_1";
5456   const char* kNewEdsService1Name = "new_eds_service_name_1";
5457   const char* kNewCluster2Name = "new_cluster_2";
5458   const char* kNewEdsService2Name = "new_eds_service_name_2";
5459   SetNextResolution({});
5460   SetNextResolutionForLbChannelAllBalancers();
5461   // Populate new EDS resources.
5462   AdsServiceImpl::EdsResourceArgs args(
5463       {{"locality0", {MakeNonExistantEndpoint()}}});
5464   AdsServiceImpl::EdsResourceArgs args1(
5465       {{"locality0", {MakeNonExistantEndpoint()}}});
5466   AdsServiceImpl::EdsResourceArgs args2(
5467       {{"locality0", {MakeNonExistantEndpoint()}}});
5468   balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
5469   balancers_[0]->ads_service()->SetEdsResource(
5470       BuildEdsResource(args1, kNewEdsService1Name));
5471   balancers_[0]->ads_service()->SetEdsResource(
5472       BuildEdsResource(args2, kNewEdsService2Name));
5473   // Populate new CDS resources.
5474   Cluster new_cluster1 = default_cluster_;
5475   new_cluster1.set_name(kNewCluster1Name);
5476   new_cluster1.mutable_eds_cluster_config()->set_service_name(
5477       kNewEdsService1Name);
5478   balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
5479   Cluster new_cluster2 = default_cluster_;
5480   new_cluster2.set_name(kNewCluster2Name);
5481   new_cluster2.mutable_eds_cluster_config()->set_service_name(
5482       kNewEdsService2Name);
5483   balancers_[0]->ads_service()->SetCdsResource(new_cluster2);
5484   // Construct listener.
5485   auto listener = default_listener_;
5486   HttpConnectionManager http_connection_manager;
5487   listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
5488       &http_connection_manager);
5489   // Set up HTTP max_stream_duration of 3.5 seconds
5490   auto* duration =
5491       http_connection_manager.mutable_common_http_protocol_options()
5492           ->mutable_max_stream_duration();
5493   duration->set_seconds(kTimeoutHttpMaxStreamDurationSecond);
5494   duration->set_nanos(kTimeoutNano);
5495   listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
5496       http_connection_manager);
5497   // Construct route config.
5498   RouteConfiguration new_route_config = default_route_config_;
5499   // route 1: Set max_stream_duration of 2.5 seconds, Set
5500   // grpc_timeout_header_max of 0
5501   auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
5502   route1->mutable_match()->set_path("/grpc.testing.EchoTest1Service/Echo1");
5503   route1->mutable_route()->set_cluster(kNewCluster1Name);
5504   auto* max_stream_duration =
5505       route1->mutable_route()->mutable_max_stream_duration();
5506   duration = max_stream_duration->mutable_max_stream_duration();
5507   duration->set_seconds(kTimeoutMaxStreamDurationSecond);
5508   duration->set_nanos(kTimeoutNano);
5509   duration = max_stream_duration->mutable_grpc_timeout_header_max();
5510   duration->set_seconds(0);
5511   duration->set_nanos(0);
5512   // route 2: Set max_stream_duration to 0
5513   auto* route2 = new_route_config.mutable_virtual_hosts(0)->add_routes();
5514   route2->mutable_match()->set_path("/grpc.testing.EchoTest2Service/Echo2");
5515   route2->mutable_route()->set_cluster(kNewCluster2Name);
5516   max_stream_duration = route2->mutable_route()->mutable_max_stream_duration();
5517   duration = max_stream_duration->mutable_max_stream_duration();
5518   duration->set_seconds(0);
5519   duration->set_nanos(0);
5520   // Set listener and route config.
5521   SetListenerAndRouteConfiguration(0, std::move(listener), new_route_config);
5522   // Test application timeout is applied for route 1
5523   auto t0 = system_clock::now();
5524   CheckRpcSendFailure(
5525       CheckRpcSendFailureOptions()
5526           .set_rpc_options(
5527               RpcOptions()
5528                   .set_rpc_service(SERVICE_ECHO1)
5529                   .set_rpc_method(METHOD_ECHO1)
5530                   .set_wait_for_ready(true)
5531                   .set_timeout_ms(kTimeoutApplicationSecond * 1000))
5532           .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
5533   auto ellapsed_nano_seconds =
5534       std::chrono::duration_cast<std::chrono::nanoseconds>(system_clock::now() -
5535                                                            t0);
5536   EXPECT_GT(ellapsed_nano_seconds.count(),
5537             kTimeoutApplicationSecond * 1000000000);
5538   // Test application timeout is applied for route 2
5539   t0 = system_clock::now();
5540   CheckRpcSendFailure(
5541       CheckRpcSendFailureOptions()
5542           .set_rpc_options(
5543               RpcOptions()
5544                   .set_rpc_service(SERVICE_ECHO2)
5545                   .set_rpc_method(METHOD_ECHO2)
5546                   .set_wait_for_ready(true)
5547                   .set_timeout_ms(kTimeoutApplicationSecond * 1000))
5548           .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
5549   ellapsed_nano_seconds = std::chrono::duration_cast<std::chrono::nanoseconds>(
5550       system_clock::now() - t0);
5551   EXPECT_GT(ellapsed_nano_seconds.count(),
5552             kTimeoutApplicationSecond * 1000000000);
5553 }
5554
5555 TEST_P(LdsRdsTest, XdsRoutingApplyApplicationTimeoutWhenHttpTimeoutExplicit0) {
5556   const int64_t kTimeoutApplicationSecond = 4;
5557   SetNextResolution({});
5558   SetNextResolutionForLbChannelAllBalancers();
5559   // Populate new EDS resources.
5560   AdsServiceImpl::EdsResourceArgs args(
5561       {{"locality0", {MakeNonExistantEndpoint()}}});
5562   balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
5563   auto listener = default_listener_;
5564   HttpConnectionManager http_connection_manager;
5565   listener.mutable_api_listener()->mutable_api_listener()->UnpackTo(
5566       &http_connection_manager);
5567   // Set up HTTP max_stream_duration to be explicit 0
5568   auto* duration =
5569       http_connection_manager.mutable_common_http_protocol_options()
5570           ->mutable_max_stream_duration();
5571   duration->set_seconds(0);
5572   duration->set_nanos(0);
5573   listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
5574       http_connection_manager);
5575   // Set listener and route config.
5576   SetListenerAndRouteConfiguration(0, std::move(listener),
5577                                    default_route_config_);
5578   // Test application timeout is applied for route 1
5579   auto t0 = system_clock::now();
5580   CheckRpcSendFailure(
5581       CheckRpcSendFailureOptions()
5582           .set_rpc_options(RpcOptions().set_wait_for_ready(true).set_timeout_ms(
5583               kTimeoutApplicationSecond * 1000))
5584           .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
5585   auto ellapsed_nano_seconds =
5586       std::chrono::duration_cast<std::chrono::nanoseconds>(system_clock::now() -
5587                                                            t0);
5588   EXPECT_GT(ellapsed_nano_seconds.count(),
5589             kTimeoutApplicationSecond * 1000000000);
5590 }
5591
5592 // Test to ensure application-specified deadline won't be affected when
5593 // the xDS config does not specify a timeout.
5594 TEST_P(LdsRdsTest, XdsRoutingWithOnlyApplicationTimeout) {
5595   const int64_t kTimeoutApplicationSecond = 4;
5596   SetNextResolution({});
5597   SetNextResolutionForLbChannelAllBalancers();
5598   // Populate new EDS resources.
5599   AdsServiceImpl::EdsResourceArgs args(
5600       {{"locality0", {MakeNonExistantEndpoint()}}});
5601   balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
5602   auto t0 = system_clock::now();
5603   CheckRpcSendFailure(
5604       CheckRpcSendFailureOptions()
5605           .set_rpc_options(RpcOptions().set_wait_for_ready(true).set_timeout_ms(
5606               kTimeoutApplicationSecond * 1000))
5607           .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
5608   auto ellapsed_nano_seconds =
5609       std::chrono::duration_cast<std::chrono::nanoseconds>(system_clock::now() -
5610                                                            t0);
5611   EXPECT_GT(ellapsed_nano_seconds.count(),
5612             kTimeoutApplicationSecond * 1000000000);
5613 }
5614
5615 TEST_P(LdsRdsTest, XdsRetryPolicyNumRetries) {
5616   const size_t kNumRetries = 3;
5617   SetNextResolution({});
5618   SetNextResolutionForLbChannelAllBalancers();
5619   // Populate new EDS resources.
5620   AdsServiceImpl::EdsResourceArgs args({
5621       {"locality0", CreateEndpointsForBackends(0, 1)},
5622   });
5623   balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
5624   // Construct route config to set retry policy.
5625   RouteConfiguration new_route_config = default_route_config_;
5626   auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
5627   auto* retry_policy = route1->mutable_route()->mutable_retry_policy();
5628   retry_policy->set_retry_on(
5629       "5xx,cancelled,deadline-exceeded,internal,resource-exhausted,"
5630       "unavailable");
5631   retry_policy->mutable_num_retries()->set_value(kNumRetries);
5632   SetRouteConfiguration(0, new_route_config);
5633   // Ensure we retried the correct number of times on all supported status.
5634   CheckRpcSendFailure(
5635       CheckRpcSendFailureOptions()
5636           .set_rpc_options(
5637               RpcOptions().set_server_expected_error(StatusCode::CANCELLED))
5638           .set_expected_error_code(StatusCode::CANCELLED));
5639   EXPECT_EQ(kNumRetries + 1, backends_[0]->backend_service()->request_count());
5640   ResetBackendCounters();
5641   CheckRpcSendFailure(
5642       CheckRpcSendFailureOptions()
5643           .set_rpc_options(RpcOptions().set_server_expected_error(
5644               StatusCode::DEADLINE_EXCEEDED))
5645           .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
5646   EXPECT_EQ(kNumRetries + 1, backends_[0]->backend_service()->request_count());
5647   ResetBackendCounters();
5648   CheckRpcSendFailure(
5649       CheckRpcSendFailureOptions()
5650           .set_rpc_options(
5651               RpcOptions().set_server_expected_error(StatusCode::INTERNAL))
5652           .set_expected_error_code(StatusCode::INTERNAL));
5653   EXPECT_EQ(kNumRetries + 1, backends_[0]->backend_service()->request_count());
5654   ResetBackendCounters();
5655   CheckRpcSendFailure(
5656       CheckRpcSendFailureOptions()
5657           .set_rpc_options(RpcOptions().set_server_expected_error(
5658               StatusCode::RESOURCE_EXHAUSTED))
5659           .set_expected_error_code(StatusCode::RESOURCE_EXHAUSTED));
5660   EXPECT_EQ(kNumRetries + 1, backends_[0]->backend_service()->request_count());
5661   ResetBackendCounters();
5662   CheckRpcSendFailure(
5663       CheckRpcSendFailureOptions()
5664           .set_rpc_options(
5665               RpcOptions().set_server_expected_error(StatusCode::UNAVAILABLE))
5666           .set_expected_error_code(StatusCode::UNAVAILABLE));
5667   EXPECT_EQ(kNumRetries + 1, backends_[0]->backend_service()->request_count());
5668   ResetBackendCounters();
5669   // Ensure we don't retry on an unsupported status.
5670   CheckRpcSendFailure(
5671       CheckRpcSendFailureOptions()
5672           .set_rpc_options(RpcOptions().set_server_expected_error(
5673               StatusCode::UNAUTHENTICATED))
5674           .set_expected_error_code(StatusCode::UNAUTHENTICATED));
5675   EXPECT_EQ(1, backends_[0]->backend_service()->request_count());
5676 }
5677
5678 TEST_P(LdsRdsTest, XdsRetryPolicyAtVirtualHostLevel) {
5679   const size_t kNumRetries = 3;
5680   SetNextResolution({});
5681   SetNextResolutionForLbChannelAllBalancers();
5682   // Populate new EDS resources.
5683   AdsServiceImpl::EdsResourceArgs args({
5684       {"locality0", CreateEndpointsForBackends(0, 1)},
5685   });
5686   balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
5687   // Construct route config to set retry policy.
5688   RouteConfiguration new_route_config = default_route_config_;
5689   auto* retry_policy =
5690       new_route_config.mutable_virtual_hosts(0)->mutable_retry_policy();
5691   retry_policy->set_retry_on(
5692       "cancelled,deadline-exceeded,internal,resource-exhausted,unavailable");
5693   retry_policy->mutable_num_retries()->set_value(kNumRetries);
5694   SetRouteConfiguration(0, new_route_config);
5695   // Ensure we retried the correct number of times on a supported status.
5696   CheckRpcSendFailure(
5697       CheckRpcSendFailureOptions()
5698           .set_rpc_options(RpcOptions().set_server_expected_error(
5699               StatusCode::DEADLINE_EXCEEDED))
5700           .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
5701   EXPECT_EQ(kNumRetries + 1, backends_[0]->backend_service()->request_count());
5702 }
5703
5704 TEST_P(LdsRdsTest, XdsRetryPolicyLongBackOff) {
5705   // Set num retries to 3, but due to longer back off, we expect only 1 retry
5706   // will take place.
5707   const size_t kNumRetries = 3;
5708   SetNextResolution({});
5709   SetNextResolutionForLbChannelAllBalancers();
5710   // Populate new EDS resources.
5711   AdsServiceImpl::EdsResourceArgs args({
5712       {"locality0", CreateEndpointsForBackends(0, 1)},
5713   });
5714   balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
5715   // Construct route config to set retry policy.
5716   RouteConfiguration new_route_config = default_route_config_;
5717   auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
5718   auto* retry_policy = route1->mutable_route()->mutable_retry_policy();
5719   retry_policy->set_retry_on(
5720       "5xx,cancelled,deadline-exceeded,internal,resource-exhausted,"
5721       "unavailable");
5722   retry_policy->mutable_num_retries()->set_value(kNumRetries);
5723   auto base_interval =
5724       retry_policy->mutable_retry_back_off()->mutable_base_interval();
5725   // Set backoff to 1 second, 1/2 of rpc timeout of 2 second.
5726   base_interval->set_seconds(1 * grpc_test_slowdown_factor());
5727   base_interval->set_nanos(0);
5728   SetRouteConfiguration(0, new_route_config);
5729   // No need to set max interval and just let it be the default of 10x of base.
5730   // We expect 1 retry before the RPC times out with DEADLINE_EXCEEDED.
5731   CheckRpcSendFailure(
5732       CheckRpcSendFailureOptions()
5733           .set_rpc_options(
5734               RpcOptions().set_timeout_ms(2500).set_server_expected_error(
5735                   StatusCode::CANCELLED))
5736           .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
5737   EXPECT_EQ(1 + 1, backends_[0]->backend_service()->request_count());
5738 }
5739
5740 TEST_P(LdsRdsTest, XdsRetryPolicyMaxBackOff) {
5741   // Set num retries to 3, but due to longer back off, we expect only 2 retry
5742   // will take place, while the 2nd one will obey the max backoff.
5743   const size_t kNumRetries = 3;
5744   SetNextResolution({});
5745   SetNextResolutionForLbChannelAllBalancers();
5746   // Populate new EDS resources.
5747   AdsServiceImpl::EdsResourceArgs args({
5748       {"locality0", CreateEndpointsForBackends(0, 1)},
5749   });
5750   balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
5751   // Construct route config to set retry policy.
5752   RouteConfiguration new_route_config = default_route_config_;
5753   auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
5754   auto* retry_policy = route1->mutable_route()->mutable_retry_policy();
5755   retry_policy->set_retry_on(
5756       "5xx,cancelled,deadline-exceeded,internal,resource-exhausted,"
5757       "unavailable");
5758   retry_policy->mutable_num_retries()->set_value(kNumRetries);
5759   auto base_interval =
5760       retry_policy->mutable_retry_back_off()->mutable_base_interval();
5761   // Set backoff to 1 second.
5762   base_interval->set_seconds(1 * grpc_test_slowdown_factor());
5763   base_interval->set_nanos(0);
5764   auto max_interval =
5765       retry_policy->mutable_retry_back_off()->mutable_max_interval();
5766   // Set max interval to be the same as base, so 2 retries will take 2 seconds
5767   // and both retries will take place before the 2.5 seconds rpc timeout.
5768   // Tested to ensure if max is not set, this test will be the same as
5769   // XdsRetryPolicyLongBackOff and we will only see 1 retry in that case.
5770   max_interval->set_seconds(1 * grpc_test_slowdown_factor());
5771   max_interval->set_nanos(0);
5772   SetRouteConfiguration(0, new_route_config);
5773   // We expect 2 retry before the RPC times out with DEADLINE_EXCEEDED.
5774   CheckRpcSendFailure(
5775       CheckRpcSendFailureOptions()
5776           .set_rpc_options(
5777               RpcOptions().set_timeout_ms(2500).set_server_expected_error(
5778                   StatusCode::CANCELLED))
5779           .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
5780   EXPECT_EQ(2 + 1, backends_[0]->backend_service()->request_count());
5781 }
5782
5783 TEST_P(LdsRdsTest, XdsRetryPolicyUnsupportedStatusCode) {
5784   const size_t kNumRetries = 3;
5785   SetNextResolution({});
5786   SetNextResolutionForLbChannelAllBalancers();
5787   // Populate new EDS resources.
5788   AdsServiceImpl::EdsResourceArgs args({
5789       {"locality0", CreateEndpointsForBackends(0, 1)},
5790   });
5791   balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
5792   // Construct route config to set retry policy.
5793   RouteConfiguration new_route_config = default_route_config_;
5794   auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
5795   auto* retry_policy = route1->mutable_route()->mutable_retry_policy();
5796   retry_policy->set_retry_on("5xx");
5797   retry_policy->mutable_num_retries()->set_value(kNumRetries);
5798   SetRouteConfiguration(0, new_route_config);
5799   // We expect no retry.
5800   CheckRpcSendFailure(
5801       CheckRpcSendFailureOptions()
5802           .set_rpc_options(RpcOptions().set_server_expected_error(
5803               StatusCode::DEADLINE_EXCEEDED))
5804           .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
5805   EXPECT_EQ(1, backends_[0]->backend_service()->request_count());
5806 }
5807
5808 TEST_P(LdsRdsTest, XdsRetryPolicyInvalidNumRetriesZero) {
5809   SetNextResolution({});
5810   SetNextResolutionForLbChannelAllBalancers();
5811   // Populate new EDS resources.
5812   AdsServiceImpl::EdsResourceArgs args({
5813       {"locality0", CreateEndpointsForBackends(0, 1)},
5814   });
5815   balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
5816   // Construct route config to set retry policy.
5817   RouteConfiguration new_route_config = default_route_config_;
5818   auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
5819   auto* retry_policy = route1->mutable_route()->mutable_retry_policy();
5820   retry_policy->set_retry_on("deadline-exceeded");
5821   // Setting num_retries to zero is not valid.
5822   retry_policy->mutable_num_retries()->set_value(0);
5823   SetRouteConfiguration(0, new_route_config);
5824   ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
5825   const auto response_state = RouteConfigurationResponseState(0);
5826   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
5827   EXPECT_THAT(
5828       response_state.error_message,
5829       ::testing::HasSubstr(
5830           "RouteAction RetryPolicy num_retries set to invalid value 0."));
5831 }
5832
5833 TEST_P(LdsRdsTest, XdsRetryPolicyRetryBackOffMissingBaseInterval) {
5834   SetNextResolution({});
5835   SetNextResolutionForLbChannelAllBalancers();
5836   // Populate new EDS resources.
5837   AdsServiceImpl::EdsResourceArgs args({
5838       {"locality0", CreateEndpointsForBackends(0, 1)},
5839   });
5840   balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
5841   // Construct route config to set retry policy.
5842   RouteConfiguration new_route_config = default_route_config_;
5843   auto* route1 = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
5844   auto* retry_policy = route1->mutable_route()->mutable_retry_policy();
5845   retry_policy->set_retry_on("deadline-exceeded");
5846   retry_policy->mutable_num_retries()->set_value(1);
5847   // RetryBackoff is there but base interval is missing.
5848   auto max_interval =
5849       retry_policy->mutable_retry_back_off()->mutable_max_interval();
5850   max_interval->set_seconds(0);
5851   max_interval->set_nanos(250000000);
5852   SetRouteConfiguration(0, new_route_config);
5853   ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
5854   const auto response_state = RouteConfigurationResponseState(0);
5855   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
5856   EXPECT_THAT(
5857       response_state.error_message,
5858       ::testing::HasSubstr(
5859           "RouteAction RetryPolicy RetryBackoff missing base interval."));
5860 }
5861
5862 TEST_P(LdsRdsTest, XdsRoutingHeadersMatching) {
5863   const char* kNewClusterName = "new_cluster";
5864   const char* kNewEdsServiceName = "new_eds_service_name";
5865   const size_t kNumEcho1Rpcs = 100;
5866   const size_t kNumEchoRpcs = 5;
5867   SetNextResolution({});
5868   SetNextResolutionForLbChannelAllBalancers();
5869   // Populate new EDS resources.
5870   AdsServiceImpl::EdsResourceArgs args({
5871       {"locality0", CreateEndpointsForBackends(0, 1)},
5872   });
5873   AdsServiceImpl::EdsResourceArgs args1({
5874       {"locality0", CreateEndpointsForBackends(1, 2)},
5875   });
5876   balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
5877   balancers_[0]->ads_service()->SetEdsResource(
5878       BuildEdsResource(args1, kNewEdsServiceName));
5879   // Populate new CDS resources.
5880   Cluster new_cluster = default_cluster_;
5881   new_cluster.set_name(kNewClusterName);
5882   new_cluster.mutable_eds_cluster_config()->set_service_name(
5883       kNewEdsServiceName);
5884   balancers_[0]->ads_service()->SetCdsResource(new_cluster);
5885   // Populating Route Configurations for LDS.
5886   RouteConfiguration route_config = default_route_config_;
5887   auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
5888   route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
5889   auto* header_matcher1 = route1->mutable_match()->add_headers();
5890   header_matcher1->set_name("header1");
5891   header_matcher1->set_exact_match("POST,PUT,GET");
5892   auto* header_matcher2 = route1->mutable_match()->add_headers();
5893   header_matcher2->set_name("header2");
5894   header_matcher2->mutable_safe_regex_match()->set_regex("[a-z]*");
5895   auto* header_matcher3 = route1->mutable_match()->add_headers();
5896   header_matcher3->set_name("header3");
5897   header_matcher3->mutable_range_match()->set_start(1);
5898   header_matcher3->mutable_range_match()->set_end(1000);
5899   auto* header_matcher4 = route1->mutable_match()->add_headers();
5900   header_matcher4->set_name("header4");
5901   header_matcher4->set_present_match(false);
5902   auto* header_matcher5 = route1->mutable_match()->add_headers();
5903   header_matcher5->set_name("header5");
5904   header_matcher5->set_present_match(true);
5905   auto* header_matcher6 = route1->mutable_match()->add_headers();
5906   header_matcher6->set_name("header6");
5907   header_matcher6->set_prefix_match("/grpc");
5908   auto* header_matcher7 = route1->mutable_match()->add_headers();
5909   header_matcher7->set_name("header7");
5910   header_matcher7->set_suffix_match(".cc");
5911   header_matcher7->set_invert_match(true);
5912   route1->mutable_route()->set_cluster(kNewClusterName);
5913   auto* default_route = route_config.mutable_virtual_hosts(0)->add_routes();
5914   default_route->mutable_match()->set_prefix("");
5915   default_route->mutable_route()->set_cluster(kDefaultClusterName);
5916   SetRouteConfiguration(0, route_config);
5917   std::vector<std::pair<std::string, std::string>> metadata = {
5918       {"header1", "POST"},
5919       {"header2", "blah"},
5920       {"header3", "1"},
5921       {"header5", "anything"},
5922       {"header6", "/grpc.testing.EchoTest1Service/"},
5923       {"header1", "PUT"},
5924       {"header7", "grpc.java"},
5925       {"header1", "GET"},
5926   };
5927   const auto header_match_rpc_options = RpcOptions()
5928                                             .set_rpc_service(SERVICE_ECHO1)
5929                                             .set_rpc_method(METHOD_ECHO1)
5930                                             .set_metadata(std::move(metadata));
5931   // Make sure all backends are up.
5932   WaitForBackend(0);
5933   WaitForBackend(1, WaitForBackendOptions(), header_match_rpc_options);
5934   // Send RPCs.
5935   CheckRpcSendOk(kNumEchoRpcs);
5936   CheckRpcSendOk(kNumEcho1Rpcs, header_match_rpc_options);
5937   EXPECT_EQ(kNumEchoRpcs, backends_[0]->backend_service()->request_count());
5938   EXPECT_EQ(0, backends_[0]->backend_service1()->request_count());
5939   EXPECT_EQ(0, backends_[0]->backend_service2()->request_count());
5940   EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
5941   EXPECT_EQ(kNumEcho1Rpcs, backends_[1]->backend_service1()->request_count());
5942   EXPECT_EQ(0, backends_[1]->backend_service2()->request_count());
5943   const auto response_state = RouteConfigurationResponseState(0);
5944   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::ACKED);
5945 }
5946
5947 TEST_P(LdsRdsTest, XdsRoutingHeadersMatchingSpecialHeaderContentType) {
5948   const char* kNewClusterName = "new_cluster";
5949   const char* kNewEdsServiceName = "new_eds_service_name";
5950   const size_t kNumEchoRpcs = 100;
5951   SetNextResolution({});
5952   SetNextResolutionForLbChannelAllBalancers();
5953   // Populate new EDS resources.
5954   AdsServiceImpl::EdsResourceArgs args({
5955       {"locality0", CreateEndpointsForBackends(0, 1)},
5956   });
5957   AdsServiceImpl::EdsResourceArgs args1({
5958       {"locality0", CreateEndpointsForBackends(1, 2)},
5959   });
5960   balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
5961   balancers_[0]->ads_service()->SetEdsResource(
5962       BuildEdsResource(args1, kNewEdsServiceName));
5963   // Populate new CDS resources.
5964   Cluster new_cluster = default_cluster_;
5965   new_cluster.set_name(kNewClusterName);
5966   new_cluster.mutable_eds_cluster_config()->set_service_name(
5967       kNewEdsServiceName);
5968   balancers_[0]->ads_service()->SetCdsResource(new_cluster);
5969   // Populating Route Configurations for LDS.
5970   RouteConfiguration route_config = default_route_config_;
5971   auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
5972   route1->mutable_match()->set_prefix("");
5973   auto* header_matcher1 = route1->mutable_match()->add_headers();
5974   header_matcher1->set_name("content-type");
5975   header_matcher1->set_exact_match("notapplication/grpc");
5976   route1->mutable_route()->set_cluster(kNewClusterName);
5977   auto* default_route = route_config.mutable_virtual_hosts(0)->add_routes();
5978   default_route->mutable_match()->set_prefix("");
5979   auto* header_matcher2 = default_route->mutable_match()->add_headers();
5980   header_matcher2->set_name("content-type");
5981   header_matcher2->set_exact_match("application/grpc");
5982   default_route->mutable_route()->set_cluster(kDefaultClusterName);
5983   SetRouteConfiguration(0, route_config);
5984   // Make sure the backend is up.
5985   WaitForAllBackends(0, 1);
5986   // Send RPCs.
5987   CheckRpcSendOk(kNumEchoRpcs);
5988   EXPECT_EQ(kNumEchoRpcs, backends_[0]->backend_service()->request_count());
5989   EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
5990   const auto response_state = RouteConfigurationResponseState(0);
5991   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::ACKED);
5992 }
5993
5994 TEST_P(LdsRdsTest, XdsRoutingHeadersMatchingSpecialCasesToIgnore) {
5995   const char* kNewCluster1Name = "new_cluster_1";
5996   const char* kNewEdsService1Name = "new_eds_service_name_1";
5997   const size_t kNumEchoRpcs = 100;
5998   SetNextResolution({});
5999   SetNextResolutionForLbChannelAllBalancers();
6000   // Populate new EDS resources.
6001   AdsServiceImpl::EdsResourceArgs args({
6002       {"locality0", CreateEndpointsForBackends(0, 1)},
6003   });
6004   AdsServiceImpl::EdsResourceArgs args1({
6005       {"locality0", CreateEndpointsForBackends(1, 2)},
6006   });
6007   balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
6008   balancers_[0]->ads_service()->SetEdsResource(
6009       BuildEdsResource(args1, kNewEdsService1Name));
6010   // Populate new CDS resources.
6011   Cluster new_cluster1 = default_cluster_;
6012   new_cluster1.set_name(kNewCluster1Name);
6013   new_cluster1.mutable_eds_cluster_config()->set_service_name(
6014       kNewEdsService1Name);
6015   balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
6016   // Populating Route Configurations for LDS.
6017   RouteConfiguration route_config = default_route_config_;
6018   auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
6019   route1->mutable_match()->set_prefix("");
6020   auto* header_matcher1 = route1->mutable_match()->add_headers();
6021   header_matcher1->set_name("grpc-foo-bin");
6022   header_matcher1->set_present_match(true);
6023   route1->mutable_route()->set_cluster(kNewCluster1Name);
6024   auto* default_route = route_config.mutable_virtual_hosts(0)->add_routes();
6025   default_route->mutable_match()->set_prefix("");
6026   default_route->mutable_route()->set_cluster(kDefaultClusterName);
6027   SetRouteConfiguration(0, route_config);
6028   // Send headers which will mismatch each route
6029   std::vector<std::pair<std::string, std::string>> metadata = {
6030       {"grpc-foo-bin", "grpc-foo-bin"},
6031   };
6032   WaitForAllBackends(0, 1);
6033   CheckRpcSendOk(kNumEchoRpcs, RpcOptions().set_metadata(metadata));
6034   // Verify that only the default backend got RPCs since all previous routes
6035   // were mismatched.
6036   EXPECT_EQ(kNumEchoRpcs, backends_[0]->backend_service()->request_count());
6037   EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
6038   const auto response_state = RouteConfigurationResponseState(0);
6039   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::ACKED);
6040 }
6041
6042 TEST_P(LdsRdsTest, XdsRoutingRuntimeFractionMatching) {
6043   const char* kNewClusterName = "new_cluster";
6044   const char* kNewEdsServiceName = "new_eds_service_name";
6045   const double kErrorTolerance = 0.05;
6046   const size_t kRouteMatchNumerator = 25;
6047   const double kRouteMatchPercent =
6048       static_cast<double>(kRouteMatchNumerator) / 100;
6049   const size_t kNumRpcs =
6050       ComputeIdealNumRpcs(kRouteMatchPercent, kErrorTolerance);
6051   SetNextResolution({});
6052   SetNextResolutionForLbChannelAllBalancers();
6053   // Populate new EDS resources.
6054   AdsServiceImpl::EdsResourceArgs args({
6055       {"locality0", CreateEndpointsForBackends(0, 1)},
6056   });
6057   AdsServiceImpl::EdsResourceArgs args1({
6058       {"locality0", CreateEndpointsForBackends(1, 2)},
6059   });
6060   balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
6061   balancers_[0]->ads_service()->SetEdsResource(
6062       BuildEdsResource(args1, kNewEdsServiceName));
6063   // Populate new CDS resources.
6064   Cluster new_cluster = default_cluster_;
6065   new_cluster.set_name(kNewClusterName);
6066   new_cluster.mutable_eds_cluster_config()->set_service_name(
6067       kNewEdsServiceName);
6068   balancers_[0]->ads_service()->SetCdsResource(new_cluster);
6069   // Populating Route Configurations for LDS.
6070   RouteConfiguration route_config = default_route_config_;
6071   auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
6072   route1->mutable_match()
6073       ->mutable_runtime_fraction()
6074       ->mutable_default_value()
6075       ->set_numerator(kRouteMatchNumerator);
6076   route1->mutable_route()->set_cluster(kNewClusterName);
6077   auto* default_route = route_config.mutable_virtual_hosts(0)->add_routes();
6078   default_route->mutable_match()->set_prefix("");
6079   default_route->mutable_route()->set_cluster(kDefaultClusterName);
6080   SetRouteConfiguration(0, route_config);
6081   WaitForAllBackends(0, 2);
6082   CheckRpcSendOk(kNumRpcs);
6083   const int default_backend_count =
6084       backends_[0]->backend_service()->request_count();
6085   const int matched_backend_count =
6086       backends_[1]->backend_service()->request_count();
6087   EXPECT_THAT(static_cast<double>(default_backend_count) / kNumRpcs,
6088               ::testing::DoubleNear(1 - kRouteMatchPercent, kErrorTolerance));
6089   EXPECT_THAT(static_cast<double>(matched_backend_count) / kNumRpcs,
6090               ::testing::DoubleNear(kRouteMatchPercent, kErrorTolerance));
6091   const auto response_state = RouteConfigurationResponseState(0);
6092   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::ACKED);
6093 }
6094
6095 TEST_P(LdsRdsTest, XdsRoutingHeadersMatchingUnmatchCases) {
6096   const char* kNewCluster1Name = "new_cluster_1";
6097   const char* kNewEdsService1Name = "new_eds_service_name_1";
6098   const char* kNewCluster2Name = "new_cluster_2";
6099   const char* kNewEdsService2Name = "new_eds_service_name_2";
6100   const char* kNewCluster3Name = "new_cluster_3";
6101   const char* kNewEdsService3Name = "new_eds_service_name_3";
6102   const size_t kNumEcho1Rpcs = 100;
6103   const size_t kNumEchoRpcs = 5;
6104   SetNextResolution({});
6105   SetNextResolutionForLbChannelAllBalancers();
6106   // Populate new EDS resources.
6107   AdsServiceImpl::EdsResourceArgs args({
6108       {"locality0", CreateEndpointsForBackends(0, 1)},
6109   });
6110   AdsServiceImpl::EdsResourceArgs args1({
6111       {"locality0", CreateEndpointsForBackends(1, 2)},
6112   });
6113   AdsServiceImpl::EdsResourceArgs args2({
6114       {"locality0", CreateEndpointsForBackends(2, 3)},
6115   });
6116   AdsServiceImpl::EdsResourceArgs args3({
6117       {"locality0", CreateEndpointsForBackends(3, 4)},
6118   });
6119   balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
6120   balancers_[0]->ads_service()->SetEdsResource(
6121       BuildEdsResource(args1, kNewEdsService1Name));
6122   balancers_[0]->ads_service()->SetEdsResource(
6123       BuildEdsResource(args2, kNewEdsService2Name));
6124   balancers_[0]->ads_service()->SetEdsResource(
6125       BuildEdsResource(args3, kNewEdsService3Name));
6126   // Populate new CDS resources.
6127   Cluster new_cluster1 = default_cluster_;
6128   new_cluster1.set_name(kNewCluster1Name);
6129   new_cluster1.mutable_eds_cluster_config()->set_service_name(
6130       kNewEdsService1Name);
6131   balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
6132   Cluster new_cluster2 = default_cluster_;
6133   new_cluster2.set_name(kNewCluster2Name);
6134   new_cluster2.mutable_eds_cluster_config()->set_service_name(
6135       kNewEdsService2Name);
6136   balancers_[0]->ads_service()->SetCdsResource(new_cluster2);
6137   Cluster new_cluster3 = default_cluster_;
6138   new_cluster3.set_name(kNewCluster3Name);
6139   new_cluster3.mutable_eds_cluster_config()->set_service_name(
6140       kNewEdsService3Name);
6141   balancers_[0]->ads_service()->SetCdsResource(new_cluster3);
6142   // Populating Route Configurations for LDS.
6143   RouteConfiguration route_config = default_route_config_;
6144   auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
6145   route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
6146   auto* header_matcher1 = route1->mutable_match()->add_headers();
6147   header_matcher1->set_name("header1");
6148   header_matcher1->set_exact_match("POST");
6149   route1->mutable_route()->set_cluster(kNewCluster1Name);
6150   auto route2 = route_config.mutable_virtual_hosts(0)->add_routes();
6151   route2->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
6152   auto* header_matcher2 = route2->mutable_match()->add_headers();
6153   header_matcher2->set_name("header2");
6154   header_matcher2->mutable_range_match()->set_start(1);
6155   header_matcher2->mutable_range_match()->set_end(1000);
6156   route2->mutable_route()->set_cluster(kNewCluster2Name);
6157   auto route3 = route_config.mutable_virtual_hosts(0)->add_routes();
6158   route3->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
6159   auto* header_matcher3 = route3->mutable_match()->add_headers();
6160   header_matcher3->set_name("header3");
6161   header_matcher3->mutable_safe_regex_match()->set_regex("[a-z]*");
6162   route3->mutable_route()->set_cluster(kNewCluster3Name);
6163   auto* default_route = route_config.mutable_virtual_hosts(0)->add_routes();
6164   default_route->mutable_match()->set_prefix("");
6165   default_route->mutable_route()->set_cluster(kDefaultClusterName);
6166   SetRouteConfiguration(0, route_config);
6167   // Send headers which will mismatch each route
6168   std::vector<std::pair<std::string, std::string>> metadata = {
6169       {"header1", "POST"},
6170       {"header2", "1000"},
6171       {"header3", "123"},
6172       {"header1", "GET"},
6173   };
6174   WaitForAllBackends(0, 1);
6175   CheckRpcSendOk(kNumEchoRpcs, RpcOptions().set_metadata(metadata));
6176   CheckRpcSendOk(kNumEcho1Rpcs, RpcOptions()
6177                                     .set_rpc_service(SERVICE_ECHO1)
6178                                     .set_rpc_method(METHOD_ECHO1)
6179                                     .set_metadata(metadata));
6180   // Verify that only the default backend got RPCs since all previous routes
6181   // were mismatched.
6182   for (size_t i = 1; i < 4; ++i) {
6183     EXPECT_EQ(0, backends_[i]->backend_service()->request_count());
6184     EXPECT_EQ(0, backends_[i]->backend_service1()->request_count());
6185     EXPECT_EQ(0, backends_[i]->backend_service2()->request_count());
6186   }
6187   EXPECT_EQ(kNumEchoRpcs, backends_[0]->backend_service()->request_count());
6188   EXPECT_EQ(kNumEcho1Rpcs, backends_[0]->backend_service1()->request_count());
6189   EXPECT_EQ(0, backends_[0]->backend_service2()->request_count());
6190   const auto response_state = RouteConfigurationResponseState(0);
6191   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::ACKED);
6192 }
6193
6194 TEST_P(LdsRdsTest, XdsRoutingChangeRoutesWithoutChangingClusters) {
6195   const char* kNewClusterName = "new_cluster";
6196   const char* kNewEdsServiceName = "new_eds_service_name";
6197   SetNextResolution({});
6198   SetNextResolutionForLbChannelAllBalancers();
6199   // Populate new EDS resources.
6200   AdsServiceImpl::EdsResourceArgs args({
6201       {"locality0", CreateEndpointsForBackends(0, 1)},
6202   });
6203   AdsServiceImpl::EdsResourceArgs args1({
6204       {"locality0", CreateEndpointsForBackends(1, 2)},
6205   });
6206   balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
6207   balancers_[0]->ads_service()->SetEdsResource(
6208       BuildEdsResource(args1, kNewEdsServiceName));
6209   // Populate new CDS resources.
6210   Cluster new_cluster = default_cluster_;
6211   new_cluster.set_name(kNewClusterName);
6212   new_cluster.mutable_eds_cluster_config()->set_service_name(
6213       kNewEdsServiceName);
6214   balancers_[0]->ads_service()->SetCdsResource(new_cluster);
6215   // Populating Route Configurations for LDS.
6216   RouteConfiguration route_config = default_route_config_;
6217   auto* route1 = route_config.mutable_virtual_hosts(0)->mutable_routes(0);
6218   route1->mutable_match()->set_prefix("/grpc.testing.EchoTest1Service/");
6219   route1->mutable_route()->set_cluster(kNewClusterName);
6220   auto* default_route = route_config.mutable_virtual_hosts(0)->add_routes();
6221   default_route->mutable_match()->set_prefix("");
6222   default_route->mutable_route()->set_cluster(kDefaultClusterName);
6223   SetRouteConfiguration(0, route_config);
6224   // Make sure all backends are up and that requests for each RPC
6225   // service go to the right backends.
6226   WaitForBackend(0, WaitForBackendOptions().set_reset_counters(false));
6227   WaitForBackend(1, WaitForBackendOptions().set_reset_counters(false),
6228                  RpcOptions().set_rpc_service(SERVICE_ECHO1));
6229   WaitForBackend(0, WaitForBackendOptions().set_reset_counters(false),
6230                  RpcOptions().set_rpc_service(SERVICE_ECHO2));
6231   // Requests for services Echo and Echo2 should have gone to backend 0.
6232   EXPECT_EQ(1, backends_[0]->backend_service()->request_count());
6233   EXPECT_EQ(0, backends_[0]->backend_service1()->request_count());
6234   EXPECT_EQ(1, backends_[0]->backend_service2()->request_count());
6235   // Requests for service Echo1 should have gone to backend 1.
6236   EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
6237   EXPECT_EQ(1, backends_[1]->backend_service1()->request_count());
6238   EXPECT_EQ(0, backends_[1]->backend_service2()->request_count());
6239   // Now send an update that changes the first route to match a
6240   // different RPC service, and wait for the client to make the change.
6241   route1->mutable_match()->set_prefix("/grpc.testing.EchoTest2Service/");
6242   SetRouteConfiguration(0, route_config);
6243   WaitForBackend(1, WaitForBackendOptions(),
6244                  RpcOptions().set_rpc_service(SERVICE_ECHO2));
6245   // Now repeat the earlier test, making sure all traffic goes to the
6246   // right place.
6247   WaitForBackend(0, WaitForBackendOptions().set_reset_counters(false));
6248   WaitForBackend(0, WaitForBackendOptions().set_reset_counters(false),
6249                  RpcOptions().set_rpc_service(SERVICE_ECHO1));
6250   WaitForBackend(1, WaitForBackendOptions().set_reset_counters(false),
6251                  RpcOptions().set_rpc_service(SERVICE_ECHO2));
6252   // Requests for services Echo and Echo1 should have gone to backend 0.
6253   EXPECT_EQ(1, backends_[0]->backend_service()->request_count());
6254   EXPECT_EQ(1, backends_[0]->backend_service1()->request_count());
6255   EXPECT_EQ(0, backends_[0]->backend_service2()->request_count());
6256   // Requests for service Echo2 should have gone to backend 1.
6257   EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
6258   EXPECT_EQ(0, backends_[1]->backend_service1()->request_count());
6259   EXPECT_EQ(1, backends_[1]->backend_service2()->request_count());
6260 }
6261
6262 // Test that we NACK unknown filter types in VirtualHost.
6263 TEST_P(LdsRdsTest, RejectsUnknownHttpFilterTypeInVirtualHost) {
6264   if (GetParam().use_v2()) return;  // Filters supported in v3 only.
6265   RouteConfiguration route_config = default_route_config_;
6266   auto* per_filter_config =
6267       route_config.mutable_virtual_hosts(0)->mutable_typed_per_filter_config();
6268   (*per_filter_config)["unknown"].PackFrom(Listener());
6269   SetListenerAndRouteConfiguration(0, default_listener_, route_config);
6270   SetNextResolution({});
6271   SetNextResolutionForLbChannelAllBalancers();
6272   ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
6273   const auto response_state = RouteConfigurationResponseState(0);
6274   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
6275   EXPECT_THAT(response_state.error_message,
6276               ::testing::HasSubstr("no filter registered for config type "
6277                                    "envoy.config.listener.v3.Listener"));
6278 }
6279
6280 // Test that we ignore optional unknown filter types in VirtualHost.
6281 TEST_P(LdsRdsTest, IgnoresOptionalUnknownHttpFilterTypeInVirtualHost) {
6282   if (GetParam().use_v2()) return;  // Filters supported in v3 only.
6283   RouteConfiguration route_config = default_route_config_;
6284   auto* per_filter_config =
6285       route_config.mutable_virtual_hosts(0)->mutable_typed_per_filter_config();
6286   ::envoy::config::route::v3::FilterConfig filter_config;
6287   filter_config.mutable_config()->PackFrom(Listener());
6288   filter_config.set_is_optional(true);
6289   (*per_filter_config)["unknown"].PackFrom(filter_config);
6290   SetListenerAndRouteConfiguration(0, default_listener_, route_config);
6291   AdsServiceImpl::EdsResourceArgs args({
6292       {"locality0", CreateEndpointsForBackends()},
6293   });
6294   balancers_[0]->ads_service()->SetEdsResource(
6295       BuildEdsResource(args, DefaultEdsServiceName()));
6296   SetNextResolution({});
6297   SetNextResolutionForLbChannelAllBalancers();
6298   WaitForAllBackends();
6299   EXPECT_EQ(RouteConfigurationResponseState(0).state,
6300             AdsServiceImpl::ResponseState::ACKED);
6301 }
6302
6303 // Test that we NACK filters without configs in VirtualHost.
6304 TEST_P(LdsRdsTest, RejectsHttpFilterWithoutConfigInVirtualHost) {
6305   if (GetParam().use_v2()) return;  // Filters supported in v3 only.
6306   RouteConfiguration route_config = default_route_config_;
6307   auto* per_filter_config =
6308       route_config.mutable_virtual_hosts(0)->mutable_typed_per_filter_config();
6309   (*per_filter_config)["unknown"];
6310   SetListenerAndRouteConfiguration(0, default_listener_, route_config);
6311   SetNextResolution({});
6312   SetNextResolutionForLbChannelAllBalancers();
6313   ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
6314   const auto response_state = RouteConfigurationResponseState(0);
6315   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
6316   EXPECT_THAT(response_state.error_message,
6317               ::testing::HasSubstr(
6318                   "no filter config specified for filter name unknown"));
6319 }
6320
6321 // Test that we NACK filters without configs in FilterConfig in VirtualHost.
6322 TEST_P(LdsRdsTest, RejectsHttpFilterWithoutConfigInFilterConfigInVirtualHost) {
6323   if (GetParam().use_v2()) return;  // Filters supported in v3 only.
6324   RouteConfiguration route_config = default_route_config_;
6325   auto* per_filter_config =
6326       route_config.mutable_virtual_hosts(0)->mutable_typed_per_filter_config();
6327   (*per_filter_config)["unknown"].PackFrom(
6328       ::envoy::config::route::v3::FilterConfig());
6329   SetListenerAndRouteConfiguration(0, default_listener_, route_config);
6330   SetNextResolution({});
6331   SetNextResolutionForLbChannelAllBalancers();
6332   ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
6333   const auto response_state = RouteConfigurationResponseState(0);
6334   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
6335   EXPECT_THAT(response_state.error_message,
6336               ::testing::HasSubstr(
6337                   "no filter config specified for filter name unknown"));
6338 }
6339
6340 // Test that we ignore optional filters without configs in VirtualHost.
6341 TEST_P(LdsRdsTest, IgnoresOptionalHttpFilterWithoutConfigInVirtualHost) {
6342   if (GetParam().use_v2()) return;  // Filters supported in v3 only.
6343   RouteConfiguration route_config = default_route_config_;
6344   auto* per_filter_config =
6345       route_config.mutable_virtual_hosts(0)->mutable_typed_per_filter_config();
6346   ::envoy::config::route::v3::FilterConfig filter_config;
6347   filter_config.set_is_optional(true);
6348   (*per_filter_config)["unknown"].PackFrom(filter_config);
6349   SetListenerAndRouteConfiguration(0, default_listener_, route_config);
6350   AdsServiceImpl::EdsResourceArgs args({
6351       {"locality0", CreateEndpointsForBackends()},
6352   });
6353   balancers_[0]->ads_service()->SetEdsResource(
6354       BuildEdsResource(args, DefaultEdsServiceName()));
6355   SetNextResolution({});
6356   SetNextResolutionForLbChannelAllBalancers();
6357   WaitForAllBackends();
6358   EXPECT_EQ(RouteConfigurationResponseState(0).state,
6359             AdsServiceImpl::ResponseState::ACKED);
6360 }
6361
6362 // Test that we NACK unparseable filter types in VirtualHost.
6363 TEST_P(LdsRdsTest, RejectsUnparseableHttpFilterTypeInVirtualHost) {
6364   if (GetParam().use_v2()) return;  // Filters supported in v3 only.
6365   RouteConfiguration route_config = default_route_config_;
6366   auto* per_filter_config =
6367       route_config.mutable_virtual_hosts(0)->mutable_typed_per_filter_config();
6368   (*per_filter_config)["unknown"].PackFrom(
6369       envoy::extensions::filters::http::router::v3::Router());
6370   SetListenerAndRouteConfiguration(0, default_listener_, route_config);
6371   SetNextResolution({});
6372   SetNextResolutionForLbChannelAllBalancers();
6373   ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
6374   const auto response_state = RouteConfigurationResponseState(0);
6375   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
6376   EXPECT_THAT(
6377       response_state.error_message,
6378       ::testing::HasSubstr("router filter does not support config override"));
6379 }
6380
6381 // Test that we NACK unknown filter types in Route.
6382 TEST_P(LdsRdsTest, RejectsUnknownHttpFilterTypeInRoute) {
6383   if (GetParam().use_v2()) return;  // Filters supported in v3 only.
6384   RouteConfiguration route_config = default_route_config_;
6385   auto* per_filter_config = route_config.mutable_virtual_hosts(0)
6386                                 ->mutable_routes(0)
6387                                 ->mutable_typed_per_filter_config();
6388   (*per_filter_config)["unknown"].PackFrom(Listener());
6389   SetListenerAndRouteConfiguration(0, default_listener_, route_config);
6390   SetNextResolution({});
6391   SetNextResolutionForLbChannelAllBalancers();
6392   ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
6393   const auto response_state = RouteConfigurationResponseState(0);
6394   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
6395   EXPECT_THAT(response_state.error_message,
6396               ::testing::HasSubstr("no filter registered for config type "
6397                                    "envoy.config.listener.v3.Listener"));
6398 }
6399
6400 // Test that we ignore optional unknown filter types in Route.
6401 TEST_P(LdsRdsTest, IgnoresOptionalUnknownHttpFilterTypeInRoute) {
6402   if (GetParam().use_v2()) return;  // Filters supported in v3 only.
6403   RouteConfiguration route_config = default_route_config_;
6404   auto* per_filter_config = route_config.mutable_virtual_hosts(0)
6405                                 ->mutable_routes(0)
6406                                 ->mutable_typed_per_filter_config();
6407   ::envoy::config::route::v3::FilterConfig filter_config;
6408   filter_config.mutable_config()->PackFrom(Listener());
6409   filter_config.set_is_optional(true);
6410   (*per_filter_config)["unknown"].PackFrom(filter_config);
6411   SetListenerAndRouteConfiguration(0, default_listener_, route_config);
6412   AdsServiceImpl::EdsResourceArgs args({
6413       {"locality0", CreateEndpointsForBackends()},
6414   });
6415   balancers_[0]->ads_service()->SetEdsResource(
6416       BuildEdsResource(args, DefaultEdsServiceName()));
6417   SetNextResolution({});
6418   SetNextResolutionForLbChannelAllBalancers();
6419   WaitForAllBackends();
6420   EXPECT_EQ(RouteConfigurationResponseState(0).state,
6421             AdsServiceImpl::ResponseState::ACKED);
6422 }
6423
6424 // Test that we NACK filters without configs in Route.
6425 TEST_P(LdsRdsTest, RejectsHttpFilterWithoutConfigInRoute) {
6426   if (GetParam().use_v2()) return;  // Filters supported in v3 only.
6427   RouteConfiguration route_config = default_route_config_;
6428   auto* per_filter_config = route_config.mutable_virtual_hosts(0)
6429                                 ->mutable_routes(0)
6430                                 ->mutable_typed_per_filter_config();
6431   (*per_filter_config)["unknown"];
6432   SetListenerAndRouteConfiguration(0, default_listener_, route_config);
6433   SetNextResolution({});
6434   SetNextResolutionForLbChannelAllBalancers();
6435   ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
6436   const auto response_state = RouteConfigurationResponseState(0);
6437   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
6438   EXPECT_THAT(response_state.error_message,
6439               ::testing::HasSubstr(
6440                   "no filter config specified for filter name unknown"));
6441 }
6442
6443 // Test that we NACK filters without configs in FilterConfig in Route.
6444 TEST_P(LdsRdsTest, RejectsHttpFilterWithoutConfigInFilterConfigInRoute) {
6445   if (GetParam().use_v2()) return;  // Filters supported in v3 only.
6446   RouteConfiguration route_config = default_route_config_;
6447   auto* per_filter_config = route_config.mutable_virtual_hosts(0)
6448                                 ->mutable_routes(0)
6449                                 ->mutable_typed_per_filter_config();
6450   (*per_filter_config)["unknown"].PackFrom(
6451       ::envoy::config::route::v3::FilterConfig());
6452   SetListenerAndRouteConfiguration(0, default_listener_, route_config);
6453   SetNextResolution({});
6454   SetNextResolutionForLbChannelAllBalancers();
6455   ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
6456   const auto response_state = RouteConfigurationResponseState(0);
6457   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
6458   EXPECT_THAT(response_state.error_message,
6459               ::testing::HasSubstr(
6460                   "no filter config specified for filter name unknown"));
6461 }
6462
6463 // Test that we ignore optional filters without configs in Route.
6464 TEST_P(LdsRdsTest, IgnoresOptionalHttpFilterWithoutConfigInRoute) {
6465   if (GetParam().use_v2()) return;  // Filters supported in v3 only.
6466   RouteConfiguration route_config = default_route_config_;
6467   auto* per_filter_config = route_config.mutable_virtual_hosts(0)
6468                                 ->mutable_routes(0)
6469                                 ->mutable_typed_per_filter_config();
6470   ::envoy::config::route::v3::FilterConfig filter_config;
6471   filter_config.set_is_optional(true);
6472   (*per_filter_config)["unknown"].PackFrom(filter_config);
6473   SetListenerAndRouteConfiguration(0, default_listener_, route_config);
6474   AdsServiceImpl::EdsResourceArgs args({
6475       {"locality0", CreateEndpointsForBackends()},
6476   });
6477   balancers_[0]->ads_service()->SetEdsResource(
6478       BuildEdsResource(args, DefaultEdsServiceName()));
6479   SetNextResolution({});
6480   SetNextResolutionForLbChannelAllBalancers();
6481   WaitForAllBackends();
6482   EXPECT_EQ(RouteConfigurationResponseState(0).state,
6483             AdsServiceImpl::ResponseState::ACKED);
6484 }
6485
6486 // Test that we NACK unparseable filter types in Route.
6487 TEST_P(LdsRdsTest, RejectsUnparseableHttpFilterTypeInRoute) {
6488   if (GetParam().use_v2()) return;  // Filters supported in v3 only.
6489   RouteConfiguration route_config = default_route_config_;
6490   auto* per_filter_config = route_config.mutable_virtual_hosts(0)
6491                                 ->mutable_routes(0)
6492                                 ->mutable_typed_per_filter_config();
6493   (*per_filter_config)["unknown"].PackFrom(
6494       envoy::extensions::filters::http::router::v3::Router());
6495   SetListenerAndRouteConfiguration(0, default_listener_, route_config);
6496   SetNextResolution({});
6497   SetNextResolutionForLbChannelAllBalancers();
6498   ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
6499   const auto response_state = RouteConfigurationResponseState(0);
6500   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
6501   EXPECT_THAT(
6502       response_state.error_message,
6503       ::testing::HasSubstr("router filter does not support config override"));
6504 }
6505
6506 // Test that we NACK unknown filter types in ClusterWeight.
6507 TEST_P(LdsRdsTest, RejectsUnknownHttpFilterTypeInClusterWeight) {
6508   if (GetParam().use_v2()) return;  // Filters supported in v3 only.
6509   RouteConfiguration route_config = default_route_config_;
6510   auto* cluster_weight = route_config.mutable_virtual_hosts(0)
6511                              ->mutable_routes(0)
6512                              ->mutable_route()
6513                              ->mutable_weighted_clusters()
6514                              ->add_clusters();
6515   cluster_weight->set_name(kDefaultClusterName);
6516   cluster_weight->mutable_weight()->set_value(100);
6517   auto* per_filter_config = cluster_weight->mutable_typed_per_filter_config();
6518   (*per_filter_config)["unknown"].PackFrom(Listener());
6519   SetListenerAndRouteConfiguration(0, default_listener_, route_config);
6520   SetNextResolution({});
6521   SetNextResolutionForLbChannelAllBalancers();
6522   ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
6523   const auto response_state = RouteConfigurationResponseState(0);
6524   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
6525   EXPECT_THAT(response_state.error_message,
6526               ::testing::HasSubstr("no filter registered for config type "
6527                                    "envoy.config.listener.v3.Listener"));
6528 }
6529
6530 // Test that we ignore optional unknown filter types in ClusterWeight.
6531 TEST_P(LdsRdsTest, IgnoresOptionalUnknownHttpFilterTypeInClusterWeight) {
6532   if (GetParam().use_v2()) return;  // Filters supported in v3 only.
6533   RouteConfiguration route_config = default_route_config_;
6534   auto* cluster_weight = route_config.mutable_virtual_hosts(0)
6535                              ->mutable_routes(0)
6536                              ->mutable_route()
6537                              ->mutable_weighted_clusters()
6538                              ->add_clusters();
6539   cluster_weight->set_name(kDefaultClusterName);
6540   cluster_weight->mutable_weight()->set_value(100);
6541   auto* per_filter_config = cluster_weight->mutable_typed_per_filter_config();
6542   ::envoy::config::route::v3::FilterConfig filter_config;
6543   filter_config.mutable_config()->PackFrom(Listener());
6544   filter_config.set_is_optional(true);
6545   (*per_filter_config)["unknown"].PackFrom(filter_config);
6546   SetListenerAndRouteConfiguration(0, default_listener_, route_config);
6547   AdsServiceImpl::EdsResourceArgs args({
6548       {"locality0", CreateEndpointsForBackends()},
6549   });
6550   balancers_[0]->ads_service()->SetEdsResource(
6551       BuildEdsResource(args, DefaultEdsServiceName()));
6552   SetNextResolution({});
6553   SetNextResolutionForLbChannelAllBalancers();
6554   WaitForAllBackends();
6555   EXPECT_EQ(RouteConfigurationResponseState(0).state,
6556             AdsServiceImpl::ResponseState::ACKED);
6557 }
6558
6559 // Test that we NACK filters without configs in ClusterWeight.
6560 TEST_P(LdsRdsTest, RejectsHttpFilterWithoutConfigInClusterWeight) {
6561   if (GetParam().use_v2()) return;  // Filters supported in v3 only.
6562   RouteConfiguration route_config = default_route_config_;
6563   auto* cluster_weight = route_config.mutable_virtual_hosts(0)
6564                              ->mutable_routes(0)
6565                              ->mutable_route()
6566                              ->mutable_weighted_clusters()
6567                              ->add_clusters();
6568   cluster_weight->set_name(kDefaultClusterName);
6569   cluster_weight->mutable_weight()->set_value(100);
6570   auto* per_filter_config = cluster_weight->mutable_typed_per_filter_config();
6571   (*per_filter_config)["unknown"];
6572   SetListenerAndRouteConfiguration(0, default_listener_, route_config);
6573   SetNextResolution({});
6574   SetNextResolutionForLbChannelAllBalancers();
6575   ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
6576   const auto response_state = RouteConfigurationResponseState(0);
6577   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
6578   EXPECT_THAT(response_state.error_message,
6579               ::testing::HasSubstr(
6580                   "no filter config specified for filter name unknown"));
6581 }
6582
6583 // Test that we NACK filters without configs in FilterConfig in ClusterWeight.
6584 TEST_P(LdsRdsTest,
6585        RejectsHttpFilterWithoutConfigInFilterConfigInClusterWeight) {
6586   if (GetParam().use_v2()) return;  // Filters supported in v3 only.
6587   RouteConfiguration route_config = default_route_config_;
6588   auto* cluster_weight = route_config.mutable_virtual_hosts(0)
6589                              ->mutable_routes(0)
6590                              ->mutable_route()
6591                              ->mutable_weighted_clusters()
6592                              ->add_clusters();
6593   cluster_weight->set_name(kDefaultClusterName);
6594   cluster_weight->mutable_weight()->set_value(100);
6595   auto* per_filter_config = cluster_weight->mutable_typed_per_filter_config();
6596   (*per_filter_config)["unknown"].PackFrom(
6597       ::envoy::config::route::v3::FilterConfig());
6598   SetListenerAndRouteConfiguration(0, default_listener_, route_config);
6599   SetNextResolution({});
6600   SetNextResolutionForLbChannelAllBalancers();
6601   ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
6602   const auto response_state = RouteConfigurationResponseState(0);
6603   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
6604   EXPECT_THAT(response_state.error_message,
6605               ::testing::HasSubstr(
6606                   "no filter config specified for filter name unknown"));
6607 }
6608
6609 // Test that we ignore optional filters without configs in ClusterWeight.
6610 TEST_P(LdsRdsTest, IgnoresOptionalHttpFilterWithoutConfigInClusterWeight) {
6611   if (GetParam().use_v2()) return;  // Filters supported in v3 only.
6612   RouteConfiguration route_config = default_route_config_;
6613   auto* cluster_weight = route_config.mutable_virtual_hosts(0)
6614                              ->mutable_routes(0)
6615                              ->mutable_route()
6616                              ->mutable_weighted_clusters()
6617                              ->add_clusters();
6618   cluster_weight->set_name(kDefaultClusterName);
6619   cluster_weight->mutable_weight()->set_value(100);
6620   auto* per_filter_config = cluster_weight->mutable_typed_per_filter_config();
6621   ::envoy::config::route::v3::FilterConfig filter_config;
6622   filter_config.set_is_optional(true);
6623   (*per_filter_config)["unknown"].PackFrom(filter_config);
6624   SetListenerAndRouteConfiguration(0, default_listener_, route_config);
6625   AdsServiceImpl::EdsResourceArgs args({
6626       {"locality0", CreateEndpointsForBackends()},
6627   });
6628   balancers_[0]->ads_service()->SetEdsResource(
6629       BuildEdsResource(args, DefaultEdsServiceName()));
6630   SetNextResolution({});
6631   SetNextResolutionForLbChannelAllBalancers();
6632   WaitForAllBackends();
6633   EXPECT_EQ(RouteConfigurationResponseState(0).state,
6634             AdsServiceImpl::ResponseState::ACKED);
6635 }
6636
6637 // Test that we NACK unparseable filter types in ClusterWeight.
6638 TEST_P(LdsRdsTest, RejectsUnparseableHttpFilterTypeInClusterWeight) {
6639   if (GetParam().use_v2()) return;  // Filters supported in v3 only.
6640   RouteConfiguration route_config = default_route_config_;
6641   auto* cluster_weight = route_config.mutable_virtual_hosts(0)
6642                              ->mutable_routes(0)
6643                              ->mutable_route()
6644                              ->mutable_weighted_clusters()
6645                              ->add_clusters();
6646   cluster_weight->set_name(kDefaultClusterName);
6647   cluster_weight->mutable_weight()->set_value(100);
6648   auto* per_filter_config = cluster_weight->mutable_typed_per_filter_config();
6649   (*per_filter_config)["unknown"].PackFrom(
6650       envoy::extensions::filters::http::router::v3::Router());
6651   SetListenerAndRouteConfiguration(0, default_listener_, route_config);
6652   SetNextResolution({});
6653   SetNextResolutionForLbChannelAllBalancers();
6654   ASSERT_TRUE(WaitForRdsNack()) << "timed out waiting for NACK";
6655   const auto response_state = RouteConfigurationResponseState(0);
6656   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
6657   EXPECT_THAT(
6658       response_state.error_message,
6659       ::testing::HasSubstr("router filter does not support config override"));
6660 }
6661
6662 using CdsTest = BasicTest;
6663
6664 // Tests that CDS client should send an ACK upon correct CDS response.
6665 TEST_P(CdsTest, Vanilla) {
6666   SetNextResolution({});
6667   SetNextResolutionForLbChannelAllBalancers();
6668   (void)SendRpc();
6669   EXPECT_EQ(balancers_[0]->ads_service()->cds_response_state().state,
6670             AdsServiceImpl::ResponseState::ACKED);
6671 }
6672
6673 TEST_P(CdsTest, LogicalDNSClusterType) {
6674   gpr_setenv("GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER",
6675              "true");
6676   SetNextResolution({});
6677   SetNextResolutionForLbChannelAllBalancers();
6678   // Create Logical DNS Cluster
6679   auto cluster = default_cluster_;
6680   cluster.set_type(Cluster::LOGICAL_DNS);
6681   auto* address = cluster.mutable_load_assignment()
6682                       ->add_endpoints()
6683                       ->add_lb_endpoints()
6684                       ->mutable_endpoint()
6685                       ->mutable_address()
6686                       ->mutable_socket_address();
6687   address->set_address(kServerName);
6688   address->set_port_value(443);
6689   balancers_[0]->ads_service()->SetCdsResource(cluster);
6690   // Set Logical DNS result
6691   {
6692     grpc_core::ExecCtx exec_ctx;
6693     grpc_core::Resolver::Result result;
6694     result.addresses = CreateAddressListFromPortList(GetBackendPorts(1, 2));
6695     logical_dns_cluster_resolver_response_generator_->SetResponse(
6696         std::move(result));
6697   }
6698   // Wait for traffic to go to backend 1.
6699   WaitForBackend(1);
6700   gpr_unsetenv(
6701       "GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER");
6702 }
6703
6704 TEST_P(CdsTest, LogicalDNSClusterTypeMissingLoadAssignment) {
6705   gpr_setenv("GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER",
6706              "true");
6707   SetNextResolution({});
6708   SetNextResolutionForLbChannelAllBalancers();
6709   // Create Logical DNS Cluster
6710   auto cluster = default_cluster_;
6711   cluster.set_type(Cluster::LOGICAL_DNS);
6712   balancers_[0]->ads_service()->SetCdsResource(cluster);
6713   ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
6714   const auto response_state =
6715       balancers_[0]->ads_service()->cds_response_state();
6716   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
6717   EXPECT_THAT(response_state.error_message,
6718               ::testing::HasSubstr(
6719                   "load_assignment not present for LOGICAL_DNS cluster"));
6720   gpr_unsetenv(
6721       "GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER");
6722 }
6723
6724 TEST_P(CdsTest, LogicalDNSClusterTypeMissingLocalities) {
6725   gpr_setenv("GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER",
6726              "true");
6727   SetNextResolution({});
6728   SetNextResolutionForLbChannelAllBalancers();
6729   // Create Logical DNS Cluster
6730   auto cluster = default_cluster_;
6731   cluster.set_type(Cluster::LOGICAL_DNS);
6732   cluster.mutable_load_assignment();
6733   balancers_[0]->ads_service()->SetCdsResource(cluster);
6734   ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
6735   const auto response_state =
6736       balancers_[0]->ads_service()->cds_response_state();
6737   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
6738   EXPECT_THAT(
6739       response_state.error_message,
6740       ::testing::HasSubstr("load_assignment for LOGICAL_DNS cluster must have "
6741                            "exactly one locality, found 0"));
6742   gpr_unsetenv(
6743       "GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER");
6744 }
6745
6746 TEST_P(CdsTest, LogicalDNSClusterTypeMultipleLocalities) {
6747   gpr_setenv("GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER",
6748              "true");
6749   SetNextResolution({});
6750   SetNextResolutionForLbChannelAllBalancers();
6751   // Create Logical DNS Cluster
6752   auto cluster = default_cluster_;
6753   cluster.set_type(Cluster::LOGICAL_DNS);
6754   auto* load_assignment = cluster.mutable_load_assignment();
6755   load_assignment->add_endpoints();
6756   load_assignment->add_endpoints();
6757   balancers_[0]->ads_service()->SetCdsResource(cluster);
6758   ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
6759   const auto response_state =
6760       balancers_[0]->ads_service()->cds_response_state();
6761   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
6762   EXPECT_THAT(
6763       response_state.error_message,
6764       ::testing::HasSubstr("load_assignment for LOGICAL_DNS cluster must have "
6765                            "exactly one locality, found 2"));
6766   gpr_unsetenv(
6767       "GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER");
6768 }
6769
6770 TEST_P(CdsTest, LogicalDNSClusterTypeMissingEndpoints) {
6771   gpr_setenv("GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER",
6772              "true");
6773   SetNextResolution({});
6774   SetNextResolutionForLbChannelAllBalancers();
6775   // Create Logical DNS Cluster
6776   auto cluster = default_cluster_;
6777   cluster.set_type(Cluster::LOGICAL_DNS);
6778   cluster.mutable_load_assignment()->add_endpoints();
6779   balancers_[0]->ads_service()->SetCdsResource(cluster);
6780   ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
6781   const auto response_state =
6782       balancers_[0]->ads_service()->cds_response_state();
6783   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
6784   EXPECT_THAT(response_state.error_message,
6785               ::testing::HasSubstr(
6786                   "locality for LOGICAL_DNS cluster must have exactly one "
6787                   "endpoint, found 0"));
6788   gpr_unsetenv(
6789       "GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER");
6790 }
6791
6792 TEST_P(CdsTest, LogicalDNSClusterTypeMultipleEndpoints) {
6793   gpr_setenv("GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER",
6794              "true");
6795   SetNextResolution({});
6796   SetNextResolutionForLbChannelAllBalancers();
6797   // Create Logical DNS Cluster
6798   auto cluster = default_cluster_;
6799   cluster.set_type(Cluster::LOGICAL_DNS);
6800   auto* locality = cluster.mutable_load_assignment()->add_endpoints();
6801   locality->add_lb_endpoints();
6802   locality->add_lb_endpoints();
6803   balancers_[0]->ads_service()->SetCdsResource(cluster);
6804   ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
6805   const auto response_state =
6806       balancers_[0]->ads_service()->cds_response_state();
6807   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
6808   EXPECT_THAT(response_state.error_message,
6809               ::testing::HasSubstr(
6810                   "locality for LOGICAL_DNS cluster must have exactly one "
6811                   "endpoint, found 2"));
6812   gpr_unsetenv(
6813       "GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER");
6814 }
6815
6816 TEST_P(CdsTest, LogicalDNSClusterTypeEmptyEndpoint) {
6817   gpr_setenv("GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER",
6818              "true");
6819   SetNextResolution({});
6820   SetNextResolutionForLbChannelAllBalancers();
6821   // Create Logical DNS Cluster
6822   auto cluster = default_cluster_;
6823   cluster.set_type(Cluster::LOGICAL_DNS);
6824   cluster.mutable_load_assignment()->add_endpoints()->add_lb_endpoints();
6825   balancers_[0]->ads_service()->SetCdsResource(cluster);
6826   ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
6827   const auto response_state =
6828       balancers_[0]->ads_service()->cds_response_state();
6829   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
6830   EXPECT_THAT(response_state.error_message,
6831               ::testing::HasSubstr("LbEndpoint endpoint field not set"));
6832   gpr_unsetenv(
6833       "GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER");
6834 }
6835
6836 TEST_P(CdsTest, LogicalDNSClusterTypeEndpointMissingAddress) {
6837   gpr_setenv("GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER",
6838              "true");
6839   SetNextResolution({});
6840   SetNextResolutionForLbChannelAllBalancers();
6841   // Create Logical DNS Cluster
6842   auto cluster = default_cluster_;
6843   cluster.set_type(Cluster::LOGICAL_DNS);
6844   cluster.mutable_load_assignment()
6845       ->add_endpoints()
6846       ->add_lb_endpoints()
6847       ->mutable_endpoint();
6848   balancers_[0]->ads_service()->SetCdsResource(cluster);
6849   ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
6850   const auto response_state =
6851       balancers_[0]->ads_service()->cds_response_state();
6852   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
6853   EXPECT_THAT(response_state.error_message,
6854               ::testing::HasSubstr("Endpoint address field not set"));
6855   gpr_unsetenv(
6856       "GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER");
6857 }
6858
6859 TEST_P(CdsTest, LogicalDNSClusterTypeAddressMissingSocketAddress) {
6860   gpr_setenv("GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER",
6861              "true");
6862   SetNextResolution({});
6863   SetNextResolutionForLbChannelAllBalancers();
6864   // Create Logical DNS Cluster
6865   auto cluster = default_cluster_;
6866   cluster.set_type(Cluster::LOGICAL_DNS);
6867   cluster.mutable_load_assignment()
6868       ->add_endpoints()
6869       ->add_lb_endpoints()
6870       ->mutable_endpoint()
6871       ->mutable_address();
6872   balancers_[0]->ads_service()->SetCdsResource(cluster);
6873   ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
6874   const auto response_state =
6875       balancers_[0]->ads_service()->cds_response_state();
6876   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
6877   EXPECT_THAT(response_state.error_message,
6878               ::testing::HasSubstr("Address socket_address field not set"));
6879   gpr_unsetenv(
6880       "GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER");
6881 }
6882
6883 TEST_P(CdsTest, LogicalDNSClusterTypeSocketAddressHasResolverName) {
6884   gpr_setenv("GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER",
6885              "true");
6886   SetNextResolution({});
6887   SetNextResolutionForLbChannelAllBalancers();
6888   // Create Logical DNS Cluster
6889   auto cluster = default_cluster_;
6890   cluster.set_type(Cluster::LOGICAL_DNS);
6891   cluster.mutable_load_assignment()
6892       ->add_endpoints()
6893       ->add_lb_endpoints()
6894       ->mutable_endpoint()
6895       ->mutable_address()
6896       ->mutable_socket_address()
6897       ->set_resolver_name("foo");
6898   balancers_[0]->ads_service()->SetCdsResource(cluster);
6899   ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
6900   const auto response_state =
6901       balancers_[0]->ads_service()->cds_response_state();
6902   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
6903   EXPECT_THAT(response_state.error_message,
6904               ::testing::HasSubstr("LOGICAL_DNS clusters must NOT have a "
6905                                    "custom resolver name set"));
6906   gpr_unsetenv(
6907       "GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER");
6908 }
6909
6910 TEST_P(CdsTest, LogicalDNSClusterTypeSocketAddressMissingAddress) {
6911   gpr_setenv("GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER",
6912              "true");
6913   SetNextResolution({});
6914   SetNextResolutionForLbChannelAllBalancers();
6915   // Create Logical DNS Cluster
6916   auto cluster = default_cluster_;
6917   cluster.set_type(Cluster::LOGICAL_DNS);
6918   cluster.mutable_load_assignment()
6919       ->add_endpoints()
6920       ->add_lb_endpoints()
6921       ->mutable_endpoint()
6922       ->mutable_address()
6923       ->mutable_socket_address();
6924   balancers_[0]->ads_service()->SetCdsResource(cluster);
6925   ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
6926   const auto response_state =
6927       balancers_[0]->ads_service()->cds_response_state();
6928   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
6929   EXPECT_THAT(response_state.error_message,
6930               ::testing::HasSubstr("SocketAddress address field not set"));
6931   gpr_unsetenv(
6932       "GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER");
6933 }
6934
6935 TEST_P(CdsTest, LogicalDNSClusterTypeSocketAddressMissingPort) {
6936   gpr_setenv("GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER",
6937              "true");
6938   SetNextResolution({});
6939   SetNextResolutionForLbChannelAllBalancers();
6940   // Create Logical DNS Cluster
6941   auto cluster = default_cluster_;
6942   cluster.set_type(Cluster::LOGICAL_DNS);
6943   cluster.mutable_load_assignment()
6944       ->add_endpoints()
6945       ->add_lb_endpoints()
6946       ->mutable_endpoint()
6947       ->mutable_address()
6948       ->mutable_socket_address()
6949       ->set_address(kServerName);
6950   balancers_[0]->ads_service()->SetCdsResource(cluster);
6951   ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
6952   const auto response_state =
6953       balancers_[0]->ads_service()->cds_response_state();
6954   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
6955   EXPECT_THAT(response_state.error_message,
6956               ::testing::HasSubstr("SocketAddress port_value field not set"));
6957   gpr_unsetenv(
6958       "GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER");
6959 }
6960
6961 TEST_P(CdsTest, AggregateClusterType) {
6962   gpr_setenv("GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER",
6963              "true");
6964   const char* kNewCluster1Name = "new_cluster_1";
6965   const char* kNewEdsService1Name = "new_eds_service_name_1";
6966   const char* kNewCluster2Name = "new_cluster_2";
6967   const char* kNewEdsService2Name = "new_eds_service_name_2";
6968   SetNextResolution({});
6969   SetNextResolutionForLbChannelAllBalancers();
6970   // Populate new EDS resources.
6971   AdsServiceImpl::EdsResourceArgs args1({
6972       {"locality0", CreateEndpointsForBackends(1, 2)},
6973   });
6974   AdsServiceImpl::EdsResourceArgs args2({
6975       {"locality0", CreateEndpointsForBackends(2, 3)},
6976   });
6977   balancers_[0]->ads_service()->SetEdsResource(
6978       BuildEdsResource(args1, kNewEdsService1Name));
6979   balancers_[0]->ads_service()->SetEdsResource(
6980       BuildEdsResource(args2, kNewEdsService2Name));
6981   // Populate new CDS resources.
6982   Cluster new_cluster1 = default_cluster_;
6983   new_cluster1.set_name(kNewCluster1Name);
6984   new_cluster1.mutable_eds_cluster_config()->set_service_name(
6985       kNewEdsService1Name);
6986   balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
6987   Cluster new_cluster2 = default_cluster_;
6988   new_cluster2.set_name(kNewCluster2Name);
6989   new_cluster2.mutable_eds_cluster_config()->set_service_name(
6990       kNewEdsService2Name);
6991   balancers_[0]->ads_service()->SetCdsResource(new_cluster2);
6992   // Create Aggregate Cluster
6993   auto cluster = default_cluster_;
6994   CustomClusterType* custom_cluster = cluster.mutable_cluster_type();
6995   custom_cluster->set_name("envoy.clusters.aggregate");
6996   ClusterConfig cluster_config;
6997   cluster_config.add_clusters(kNewCluster1Name);
6998   cluster_config.add_clusters(kNewCluster2Name);
6999   custom_cluster->mutable_typed_config()->PackFrom(cluster_config);
7000   balancers_[0]->ads_service()->SetCdsResource(cluster);
7001   // Wait for traffic to go to backend 1.
7002   WaitForBackend(1);
7003   // Shutdown backend 1 and wait for all traffic to go to backend 2.
7004   ShutdownBackend(1);
7005   WaitForBackend(2, WaitForBackendOptions().set_allow_failures(true));
7006   EXPECT_EQ(balancers_[0]->ads_service()->cds_response_state().state,
7007             AdsServiceImpl::ResponseState::ACKED);
7008   // Bring backend 1 back and ensure all traffic go back to it.
7009   StartBackend(1);
7010   WaitForBackend(1);
7011   gpr_unsetenv(
7012       "GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER");
7013 }
7014
7015 TEST_P(CdsTest, AggregateClusterEdsToLogicalDns) {
7016   gpr_setenv("GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER",
7017              "true");
7018   SetNextResolution({});
7019   SetNextResolutionForLbChannelAllBalancers();
7020   const char* kNewCluster1Name = "new_cluster_1";
7021   const char* kNewEdsService1Name = "new_eds_service_name_1";
7022   const char* kLogicalDNSClusterName = "logical_dns_cluster";
7023   // Populate new EDS resources.
7024   AdsServiceImpl::EdsResourceArgs args1({
7025       {"locality0", CreateEndpointsForBackends(1, 2)},
7026   });
7027   balancers_[0]->ads_service()->SetEdsResource(
7028       BuildEdsResource(args1, kNewEdsService1Name));
7029   // Populate new CDS resources.
7030   Cluster new_cluster1 = default_cluster_;
7031   new_cluster1.set_name(kNewCluster1Name);
7032   new_cluster1.mutable_eds_cluster_config()->set_service_name(
7033       kNewEdsService1Name);
7034   balancers_[0]->ads_service()->SetCdsResource(new_cluster1);
7035   // Create Logical DNS Cluster
7036   auto logical_dns_cluster = default_cluster_;
7037   logical_dns_cluster.set_name(kLogicalDNSClusterName);
7038   logical_dns_cluster.set_type(Cluster::LOGICAL_DNS);
7039   auto* address = logical_dns_cluster.mutable_load_assignment()
7040                       ->add_endpoints()
7041                       ->add_lb_endpoints()
7042                       ->mutable_endpoint()
7043                       ->mutable_address()
7044                       ->mutable_socket_address();
7045   address->set_address(kServerName);
7046   address->set_port_value(443);
7047   balancers_[0]->ads_service()->SetCdsResource(logical_dns_cluster);
7048   // Create Aggregate Cluster
7049   auto cluster = default_cluster_;
7050   CustomClusterType* custom_cluster = cluster.mutable_cluster_type();
7051   custom_cluster->set_name("envoy.clusters.aggregate");
7052   ClusterConfig cluster_config;
7053   cluster_config.add_clusters(kNewCluster1Name);
7054   cluster_config.add_clusters(kLogicalDNSClusterName);
7055   custom_cluster->mutable_typed_config()->PackFrom(cluster_config);
7056   balancers_[0]->ads_service()->SetCdsResource(cluster);
7057   // Set Logical DNS result
7058   {
7059     grpc_core::ExecCtx exec_ctx;
7060     grpc_core::Resolver::Result result;
7061     result.addresses = CreateAddressListFromPortList(GetBackendPorts(2, 3));
7062     logical_dns_cluster_resolver_response_generator_->SetResponse(
7063         std::move(result));
7064   }
7065   // Wait for traffic to go to backend 1.
7066   WaitForBackend(1);
7067   // Shutdown backend 1 and wait for all traffic to go to backend 2.
7068   ShutdownBackend(1);
7069   WaitForBackend(2, WaitForBackendOptions().set_allow_failures(true));
7070   EXPECT_EQ(balancers_[0]->ads_service()->cds_response_state().state,
7071             AdsServiceImpl::ResponseState::ACKED);
7072   // Bring backend 1 back and ensure all traffic go back to it.
7073   StartBackend(1);
7074   WaitForBackend(1);
7075   gpr_unsetenv(
7076       "GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER");
7077 }
7078
7079 TEST_P(CdsTest, AggregateClusterLogicalDnsToEds) {
7080   gpr_setenv("GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER",
7081              "true");
7082   SetNextResolution({});
7083   SetNextResolutionForLbChannelAllBalancers();
7084   const char* kNewCluster2Name = "new_cluster_2";
7085   const char* kNewEdsService2Name = "new_eds_service_name_2";
7086   const char* kLogicalDNSClusterName = "logical_dns_cluster";
7087   // Populate new EDS resources.
7088   AdsServiceImpl::EdsResourceArgs args2({
7089       {"locality0", CreateEndpointsForBackends(2, 3)},
7090   });
7091   balancers_[0]->ads_service()->SetEdsResource(
7092       BuildEdsResource(args2, kNewEdsService2Name));
7093   // Populate new CDS resources.
7094   Cluster new_cluster2 = default_cluster_;
7095   new_cluster2.set_name(kNewCluster2Name);
7096   new_cluster2.mutable_eds_cluster_config()->set_service_name(
7097       kNewEdsService2Name);
7098   balancers_[0]->ads_service()->SetCdsResource(new_cluster2);
7099   // Create Logical DNS Cluster
7100   auto logical_dns_cluster = default_cluster_;
7101   logical_dns_cluster.set_name(kLogicalDNSClusterName);
7102   logical_dns_cluster.set_type(Cluster::LOGICAL_DNS);
7103   auto* address = logical_dns_cluster.mutable_load_assignment()
7104                       ->add_endpoints()
7105                       ->add_lb_endpoints()
7106                       ->mutable_endpoint()
7107                       ->mutable_address()
7108                       ->mutable_socket_address();
7109   address->set_address(kServerName);
7110   address->set_port_value(443);
7111   balancers_[0]->ads_service()->SetCdsResource(logical_dns_cluster);
7112   // Create Aggregate Cluster
7113   auto cluster = default_cluster_;
7114   CustomClusterType* custom_cluster = cluster.mutable_cluster_type();
7115   custom_cluster->set_name("envoy.clusters.aggregate");
7116   ClusterConfig cluster_config;
7117   cluster_config.add_clusters(kLogicalDNSClusterName);
7118   cluster_config.add_clusters(kNewCluster2Name);
7119   custom_cluster->mutable_typed_config()->PackFrom(cluster_config);
7120   balancers_[0]->ads_service()->SetCdsResource(cluster);
7121   // Set Logical DNS result
7122   {
7123     grpc_core::ExecCtx exec_ctx;
7124     grpc_core::Resolver::Result result;
7125     result.addresses = CreateAddressListFromPortList(GetBackendPorts(1, 2));
7126     logical_dns_cluster_resolver_response_generator_->SetResponse(
7127         std::move(result));
7128   }
7129   // Wait for traffic to go to backend 1.
7130   WaitForBackend(1);
7131   // Shutdown backend 1 and wait for all traffic to go to backend 2.
7132   ShutdownBackend(1);
7133   WaitForBackend(2, WaitForBackendOptions().set_allow_failures(true));
7134   EXPECT_EQ(balancers_[0]->ads_service()->cds_response_state().state,
7135             AdsServiceImpl::ResponseState::ACKED);
7136   // Bring backend 1 back and ensure all traffic go back to it.
7137   StartBackend(1);
7138   WaitForBackend(1);
7139   gpr_unsetenv(
7140       "GRPC_XDS_EXPERIMENTAL_ENABLE_AGGREGATE_AND_LOGICAL_DNS_CLUSTER");
7141 }
7142
7143 // Test that CDS client should send a NACK if cluster type is Logical DNS but
7144 // the feature is not yet supported.
7145 TEST_P(CdsTest, LogicalDNSClusterTypeDisabled) {
7146   auto cluster = default_cluster_;
7147   cluster.set_type(Cluster::LOGICAL_DNS);
7148   balancers_[0]->ads_service()->SetCdsResource(cluster);
7149   SetNextResolution({});
7150   SetNextResolutionForLbChannelAllBalancers();
7151   ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
7152   const auto response_state =
7153       balancers_[0]->ads_service()->cds_response_state();
7154   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
7155   EXPECT_THAT(response_state.error_message,
7156               ::testing::HasSubstr("DiscoveryType is not valid."));
7157 }
7158
7159 // Test that CDS client should send a NACK if cluster type is AGGREGATE but
7160 // the feature is not yet supported.
7161 TEST_P(CdsTest, AggregateClusterTypeDisabled) {
7162   auto cluster = default_cluster_;
7163   CustomClusterType* custom_cluster = cluster.mutable_cluster_type();
7164   custom_cluster->set_name("envoy.clusters.aggregate");
7165   ClusterConfig cluster_config;
7166   cluster_config.add_clusters("cluster1");
7167   cluster_config.add_clusters("cluster2");
7168   custom_cluster->mutable_typed_config()->PackFrom(cluster_config);
7169   cluster.set_type(Cluster::LOGICAL_DNS);
7170   balancers_[0]->ads_service()->SetCdsResource(cluster);
7171   SetNextResolution({});
7172   SetNextResolutionForLbChannelAllBalancers();
7173   ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
7174   const auto response_state =
7175       balancers_[0]->ads_service()->cds_response_state();
7176   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
7177   EXPECT_THAT(response_state.error_message,
7178               ::testing::HasSubstr("DiscoveryType is not valid."));
7179 }
7180
7181 // Tests that CDS client should send a NACK if the cluster type in CDS
7182 // response is unsupported.
7183 TEST_P(CdsTest, UnsupportedClusterType) {
7184   auto cluster = default_cluster_;
7185   cluster.set_type(Cluster::STATIC);
7186   balancers_[0]->ads_service()->SetCdsResource(cluster);
7187   SetNextResolution({});
7188   SetNextResolutionForLbChannelAllBalancers();
7189   ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
7190   const auto response_state =
7191       balancers_[0]->ads_service()->cds_response_state();
7192   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
7193   EXPECT_THAT(response_state.error_message,
7194               ::testing::HasSubstr("DiscoveryType is not valid."));
7195 }
7196
7197 // Tests that the NACK for multiple bad resources includes both errors.
7198 TEST_P(CdsTest, MultipleBadResources) {
7199   constexpr char kClusterName2[] = "cluster_name_2";
7200   // Use unsupported type for default cluster.
7201   auto cluster = default_cluster_;
7202   cluster.set_type(Cluster::STATIC);
7203   balancers_[0]->ads_service()->SetCdsResource(cluster);
7204   // Add second cluster with the same error.
7205   cluster.set_name(kClusterName2);
7206   balancers_[0]->ads_service()->SetCdsResource(cluster);
7207   // Change RouteConfig to point to both clusters.
7208   RouteConfiguration route_config = default_route_config_;
7209   auto* route = route_config.mutable_virtual_hosts(0)->add_routes();
7210   route->mutable_match()->set_prefix("");
7211   route->mutable_route()->set_cluster(kClusterName2);
7212   SetRouteConfiguration(0, route_config);
7213   // Send RPC.
7214   SetNextResolution({});
7215   SetNextResolutionForLbChannelAllBalancers();
7216   ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
7217   const auto response_state =
7218       balancers_[0]->ads_service()->cds_response_state();
7219   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
7220   EXPECT_THAT(response_state.error_message,
7221               ::testing::AllOf(
7222                   ::testing::HasSubstr(absl::StrCat(
7223                       kDefaultClusterName, ": DiscoveryType is not valid.")),
7224                   ::testing::HasSubstr(absl::StrCat(
7225                       kClusterName2, ": DiscoveryType is not valid."))));
7226 }
7227
7228 // Tests that CDS client should send a NACK if the eds_config in CDS response
7229 // is other than ADS.
7230 TEST_P(CdsTest, WrongEdsConfig) {
7231   auto cluster = default_cluster_;
7232   cluster.mutable_eds_cluster_config()->mutable_eds_config()->mutable_self();
7233   balancers_[0]->ads_service()->SetCdsResource(cluster);
7234   SetNextResolution({});
7235   SetNextResolutionForLbChannelAllBalancers();
7236   ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
7237   const auto response_state =
7238       balancers_[0]->ads_service()->cds_response_state();
7239   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
7240   EXPECT_THAT(response_state.error_message,
7241               ::testing::HasSubstr("EDS ConfigSource is not ADS."));
7242 }
7243
7244 // Tests that CDS client should send a NACK if the lb_policy in CDS response
7245 // is other than ROUND_ROBIN.
7246 TEST_P(CdsTest, WrongLbPolicy) {
7247   auto cluster = default_cluster_;
7248   cluster.set_lb_policy(Cluster::LEAST_REQUEST);
7249   balancers_[0]->ads_service()->SetCdsResource(cluster);
7250   SetNextResolution({});
7251   SetNextResolutionForLbChannelAllBalancers();
7252   ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
7253   const auto response_state =
7254       balancers_[0]->ads_service()->cds_response_state();
7255   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
7256   EXPECT_THAT(response_state.error_message,
7257               ::testing::HasSubstr("LB policy is not supported."));
7258 }
7259
7260 // Tests that CDS client should send a NACK if the lrs_server in CDS response
7261 // is other than SELF.
7262 TEST_P(CdsTest, WrongLrsServer) {
7263   auto cluster = default_cluster_;
7264   cluster.mutable_lrs_server()->mutable_ads();
7265   balancers_[0]->ads_service()->SetCdsResource(cluster);
7266   SetNextResolution({});
7267   SetNextResolutionForLbChannelAllBalancers();
7268   ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
7269   const auto response_state =
7270       balancers_[0]->ads_service()->cds_response_state();
7271   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
7272   EXPECT_THAT(response_state.error_message,
7273               ::testing::HasSubstr("LRS ConfigSource is not self."));
7274 }
7275
7276 // Tests that ring hash policy that hashes using channel id ensures all RPCs
7277 // to go 1 particular backend.
7278 TEST_P(CdsTest, RingHashChannelIdHashing) {
7279   auto cluster = default_cluster_;
7280   cluster.set_lb_policy(Cluster::RING_HASH);
7281   balancers_[0]->ads_service()->SetCdsResource(cluster);
7282   auto new_route_config = default_route_config_;
7283   auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
7284   auto* hash_policy = route->mutable_route()->add_hash_policy();
7285   hash_policy->mutable_filter_state()->set_key("io.grpc.channel_id");
7286   SetListenerAndRouteConfiguration(0, default_listener_, new_route_config);
7287   AdsServiceImpl::EdsResourceArgs args({
7288       {"locality0", CreateEndpointsForBackends()},
7289   });
7290   balancers_[0]->ads_service()->SetEdsResource(
7291       BuildEdsResource(args, DefaultEdsServiceName()));
7292   SetNextResolutionForLbChannelAllBalancers();
7293   CheckRpcSendOk(100);
7294   bool found = false;
7295   for (size_t i = 0; i < backends_.size(); ++i) {
7296     if (backends_[i]->backend_service()->request_count() > 0) {
7297       EXPECT_EQ(backends_[i]->backend_service()->request_count(), 100)
7298           << "backend " << i;
7299       EXPECT_FALSE(found) << "backend " << i;
7300       found = true;
7301     }
7302   }
7303   EXPECT_TRUE(found);
7304 }
7305
7306 // Tests that ring hash policy that hashes using a header value can spread
7307 // RPCs across all the backends.
7308 TEST_P(CdsTest, RingHashHeaderHashing) {
7309   auto cluster = default_cluster_;
7310   cluster.set_lb_policy(Cluster::RING_HASH);
7311   balancers_[0]->ads_service()->SetCdsResource(cluster);
7312   auto new_route_config = default_route_config_;
7313   auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
7314   auto* hash_policy = route->mutable_route()->add_hash_policy();
7315   hash_policy->mutable_header()->set_header_name("address_hash");
7316   SetListenerAndRouteConfiguration(0, default_listener_, new_route_config);
7317   AdsServiceImpl::EdsResourceArgs args({
7318       {"locality0", CreateEndpointsForBackends()},
7319   });
7320   balancers_[0]->ads_service()->SetEdsResource(
7321       BuildEdsResource(args, DefaultEdsServiceName()));
7322   SetNextResolutionForLbChannelAllBalancers();
7323   // Note each type of RPC will contains a header value that will always be
7324   // hashed to a specific backend as the header value matches the value used
7325   // to create the entry in the ring.
7326   std::vector<std::pair<std::string, std::string>> metadata = {
7327       {"address_hash", CreateMetadataValueThatHashesToBackend(0)}};
7328   std::vector<std::pair<std::string, std::string>> metadata1 = {
7329       {"address_hash", CreateMetadataValueThatHashesToBackend(1)}};
7330   std::vector<std::pair<std::string, std::string>> metadata2 = {
7331       {"address_hash", CreateMetadataValueThatHashesToBackend(2)}};
7332   std::vector<std::pair<std::string, std::string>> metadata3 = {
7333       {"address_hash", CreateMetadataValueThatHashesToBackend(3)}};
7334   const auto rpc_options = RpcOptions().set_metadata(std::move(metadata));
7335   const auto rpc_options1 = RpcOptions().set_metadata(std::move(metadata1));
7336   const auto rpc_options2 = RpcOptions().set_metadata(std::move(metadata2));
7337   const auto rpc_options3 = RpcOptions().set_metadata(std::move(metadata3));
7338   WaitForBackend(0, WaitForBackendOptions(), rpc_options);
7339   WaitForBackend(1, WaitForBackendOptions(), rpc_options1);
7340   WaitForBackend(2, WaitForBackendOptions(), rpc_options2);
7341   WaitForBackend(3, WaitForBackendOptions(), rpc_options3);
7342   CheckRpcSendOk(100, rpc_options);
7343   CheckRpcSendOk(100, rpc_options1);
7344   CheckRpcSendOk(100, rpc_options2);
7345   CheckRpcSendOk(100, rpc_options3);
7346   for (size_t i = 0; i < backends_.size(); ++i) {
7347     EXPECT_EQ(100, backends_[i]->backend_service()->request_count());
7348   }
7349 }
7350
7351 // Tests that ring hash policy that hashes using a header value and regex
7352 // rewrite to aggregate RPCs to 1 backend.
7353 TEST_P(CdsTest, RingHashHeaderHashingWithRegexRewrite) {
7354   auto cluster = default_cluster_;
7355   cluster.set_lb_policy(Cluster::RING_HASH);
7356   balancers_[0]->ads_service()->SetCdsResource(cluster);
7357   auto new_route_config = default_route_config_;
7358   auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
7359   auto* hash_policy = route->mutable_route()->add_hash_policy();
7360   hash_policy->mutable_header()->set_header_name("address_hash");
7361   hash_policy->mutable_header()
7362       ->mutable_regex_rewrite()
7363       ->mutable_pattern()
7364       ->set_regex("[0-9]+");
7365   hash_policy->mutable_header()->mutable_regex_rewrite()->set_substitution(
7366       "foo");
7367   SetListenerAndRouteConfiguration(0, default_listener_, new_route_config);
7368   AdsServiceImpl::EdsResourceArgs args({
7369       {"locality0", CreateEndpointsForBackends()},
7370   });
7371   balancers_[0]->ads_service()->SetEdsResource(
7372       BuildEdsResource(args, DefaultEdsServiceName()));
7373   SetNextResolutionForLbChannelAllBalancers();
7374   std::vector<std::pair<std::string, std::string>> metadata = {
7375       {"address_hash", CreateMetadataValueThatHashesToBackend(0)}};
7376   std::vector<std::pair<std::string, std::string>> metadata1 = {
7377       {"address_hash", CreateMetadataValueThatHashesToBackend(1)}};
7378   std::vector<std::pair<std::string, std::string>> metadata2 = {
7379       {"address_hash", CreateMetadataValueThatHashesToBackend(2)}};
7380   std::vector<std::pair<std::string, std::string>> metadata3 = {
7381       {"address_hash", CreateMetadataValueThatHashesToBackend(3)}};
7382   const auto rpc_options = RpcOptions().set_metadata(std::move(metadata));
7383   const auto rpc_options1 = RpcOptions().set_metadata(std::move(metadata1));
7384   const auto rpc_options2 = RpcOptions().set_metadata(std::move(metadata2));
7385   const auto rpc_options3 = RpcOptions().set_metadata(std::move(metadata3));
7386   CheckRpcSendOk(100, rpc_options);
7387   CheckRpcSendOk(100, rpc_options1);
7388   CheckRpcSendOk(100, rpc_options2);
7389   CheckRpcSendOk(100, rpc_options3);
7390   bool found = false;
7391   for (size_t i = 0; i < backends_.size(); ++i) {
7392     if (backends_[i]->backend_service()->request_count() > 0) {
7393       EXPECT_EQ(backends_[i]->backend_service()->request_count(), 400)
7394           << "backend " << i;
7395       EXPECT_FALSE(found) << "backend " << i;
7396       found = true;
7397     }
7398   }
7399   EXPECT_TRUE(found);
7400 }
7401
7402 // Tests that ring hash policy that hashes using a random value.
7403 TEST_P(CdsTest, RingHashNoHashPolicy) {
7404   const double kDistribution50Percent = 0.5;
7405   const double kErrorTolerance = 0.05;
7406   const uint32_t kRpcTimeoutMs = 10000;
7407   const size_t kNumRpcs =
7408       ComputeIdealNumRpcs(kDistribution50Percent, kErrorTolerance);
7409   auto cluster = default_cluster_;
7410   // Increasing min ring size for random distribution.
7411   cluster.mutable_ring_hash_lb_config()->mutable_minimum_ring_size()->set_value(
7412       100000);
7413   cluster.set_lb_policy(Cluster::RING_HASH);
7414   balancers_[0]->ads_service()->SetCdsResource(cluster);
7415   AdsServiceImpl::EdsResourceArgs args(
7416       {{"locality0", CreateEndpointsForBackends(0, 2)}});
7417   balancers_[0]->ads_service()->SetEdsResource(
7418       BuildEdsResource(args, DefaultEdsServiceName()));
7419   SetNextResolutionForLbChannelAllBalancers();
7420   // TODO(donnadionne): remove extended timeout after ring creation
7421   // optimization.
7422   WaitForAllBackends(0, 2, WaitForBackendOptions(),
7423                      RpcOptions().set_timeout_ms(kRpcTimeoutMs));
7424   CheckRpcSendOk(kNumRpcs);
7425   const int request_count_1 = backends_[0]->backend_service()->request_count();
7426   const int request_count_2 = backends_[1]->backend_service()->request_count();
7427   EXPECT_THAT(static_cast<double>(request_count_1) / kNumRpcs,
7428               ::testing::DoubleNear(kDistribution50Percent, kErrorTolerance));
7429   EXPECT_THAT(static_cast<double>(request_count_2) / kNumRpcs,
7430               ::testing::DoubleNear(kDistribution50Percent, kErrorTolerance));
7431 }
7432
7433 // Test that ring hash policy evaluation will continue past the terminal
7434 // policy if no results are produced yet.
7435 TEST_P(CdsTest, RingHashContinuesPastTerminalPolicyThatDoesNotProduceResult) {
7436   auto cluster = default_cluster_;
7437   cluster.set_lb_policy(Cluster::RING_HASH);
7438   balancers_[0]->ads_service()->SetCdsResource(cluster);
7439   auto new_route_config = default_route_config_;
7440   auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
7441   auto* hash_policy = route->mutable_route()->add_hash_policy();
7442   hash_policy->mutable_header()->set_header_name("header_not_present");
7443   hash_policy->set_terminal(true);
7444   auto* hash_policy2 = route->mutable_route()->add_hash_policy();
7445   hash_policy2->mutable_header()->set_header_name("address_hash");
7446   SetListenerAndRouteConfiguration(0, default_listener_, new_route_config);
7447   AdsServiceImpl::EdsResourceArgs args(
7448       {{"locality0", CreateEndpointsForBackends(0, 2)}});
7449   balancers_[0]->ads_service()->SetEdsResource(
7450       BuildEdsResource(args, DefaultEdsServiceName()));
7451   SetNextResolutionForLbChannelAllBalancers();
7452   std::vector<std::pair<std::string, std::string>> metadata = {
7453       {"address_hash", CreateMetadataValueThatHashesToBackend(0)}};
7454   const auto rpc_options = RpcOptions().set_metadata(std::move(metadata));
7455   CheckRpcSendOk(100, rpc_options);
7456   EXPECT_EQ(backends_[0]->backend_service()->request_count(), 100);
7457   EXPECT_EQ(backends_[1]->backend_service()->request_count(), 0);
7458 }
7459
7460 // Test random hash is used when header hashing specified a header field that
7461 // the RPC did not have.
7462 TEST_P(CdsTest, RingHashOnHeaderThatIsNotPresent) {
7463   const double kDistribution50Percent = 0.5;
7464   const double kErrorTolerance = 0.05;
7465   const uint32_t kRpcTimeoutMs = 10000;
7466   const size_t kNumRpcs =
7467       ComputeIdealNumRpcs(kDistribution50Percent, kErrorTolerance);
7468   auto cluster = default_cluster_;
7469   // Increasing min ring size for random distribution.
7470   cluster.mutable_ring_hash_lb_config()->mutable_minimum_ring_size()->set_value(
7471       100000);
7472   cluster.set_lb_policy(Cluster::RING_HASH);
7473   balancers_[0]->ads_service()->SetCdsResource(cluster);
7474   auto new_route_config = default_route_config_;
7475   auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
7476   auto* hash_policy = route->mutable_route()->add_hash_policy();
7477   hash_policy->mutable_header()->set_header_name("header_not_present");
7478   SetListenerAndRouteConfiguration(0, default_listener_, new_route_config);
7479   AdsServiceImpl::EdsResourceArgs args(
7480       {{"locality0", CreateEndpointsForBackends(0, 2)}});
7481   balancers_[0]->ads_service()->SetEdsResource(
7482       BuildEdsResource(args, DefaultEdsServiceName()));
7483   SetNextResolutionForLbChannelAllBalancers();
7484   std::vector<std::pair<std::string, std::string>> metadata = {
7485       {"unmatched_header", absl::StrFormat("%" PRIu32, rand())},
7486   };
7487   const auto rpc_options = RpcOptions().set_metadata(std::move(metadata));
7488   // TODO(donnadionne): remove extended timeout after ring creation
7489   // optimization.
7490   WaitForAllBackends(0, 2, WaitForBackendOptions(),
7491                      RpcOptions().set_timeout_ms(kRpcTimeoutMs));
7492   CheckRpcSendOk(kNumRpcs, rpc_options);
7493   const int request_count_1 = backends_[0]->backend_service()->request_count();
7494   const int request_count_2 = backends_[1]->backend_service()->request_count();
7495   EXPECT_THAT(static_cast<double>(request_count_1) / kNumRpcs,
7496               ::testing::DoubleNear(kDistribution50Percent, kErrorTolerance));
7497   EXPECT_THAT(static_cast<double>(request_count_2) / kNumRpcs,
7498               ::testing::DoubleNear(kDistribution50Percent, kErrorTolerance));
7499 }
7500
7501 // Test random hash is used when only unsupported hash policies are
7502 // configured.
7503 TEST_P(CdsTest, RingHashUnsupportedHashPolicyDefaultToRandomHashing) {
7504   const double kDistribution50Percent = 0.5;
7505   const double kErrorTolerance = 0.05;
7506   const uint32_t kRpcTimeoutMs = 10000;
7507   const size_t kNumRpcs =
7508       ComputeIdealNumRpcs(kDistribution50Percent, kErrorTolerance);
7509   auto cluster = default_cluster_;
7510   // Increasing min ring size for random distribution.
7511   cluster.mutable_ring_hash_lb_config()->mutable_minimum_ring_size()->set_value(
7512       100000);
7513   cluster.set_lb_policy(Cluster::RING_HASH);
7514   balancers_[0]->ads_service()->SetCdsResource(cluster);
7515   auto new_route_config = default_route_config_;
7516   auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
7517   auto* hash_policy_unsupported_1 = route->mutable_route()->add_hash_policy();
7518   hash_policy_unsupported_1->mutable_cookie()->set_name("cookie");
7519   auto* hash_policy_unsupported_2 = route->mutable_route()->add_hash_policy();
7520   hash_policy_unsupported_2->mutable_connection_properties()->set_source_ip(
7521       true);
7522   auto* hash_policy_unsupported_3 = route->mutable_route()->add_hash_policy();
7523   hash_policy_unsupported_3->mutable_query_parameter()->set_name(
7524       "query_parameter");
7525   SetListenerAndRouteConfiguration(0, default_listener_, new_route_config);
7526   AdsServiceImpl::EdsResourceArgs args(
7527       {{"locality0", CreateEndpointsForBackends(0, 2)}});
7528   balancers_[0]->ads_service()->SetEdsResource(
7529       BuildEdsResource(args, DefaultEdsServiceName()));
7530   SetNextResolutionForLbChannelAllBalancers();
7531   // TODO(donnadionne): remove extended timeout after ring creation
7532   // optimization.
7533   WaitForAllBackends(0, 2, WaitForBackendOptions(),
7534                      RpcOptions().set_timeout_ms(kRpcTimeoutMs));
7535   CheckRpcSendOk(kNumRpcs);
7536   const int request_count_1 = backends_[0]->backend_service()->request_count();
7537   const int request_count_2 = backends_[1]->backend_service()->request_count();
7538   EXPECT_THAT(static_cast<double>(request_count_1) / kNumRpcs,
7539               ::testing::DoubleNear(kDistribution50Percent, kErrorTolerance));
7540   EXPECT_THAT(static_cast<double>(request_count_2) / kNumRpcs,
7541               ::testing::DoubleNear(kDistribution50Percent, kErrorTolerance));
7542 }
7543
7544 // Tests that ring hash policy that hashes using a random value can spread
7545 // RPCs across all the backends according to locality weight.
7546 TEST_P(CdsTest, RingHashRandomHashingDistributionAccordingToEndpointWeight) {
7547   const size_t kWeight1 = 1;
7548   const size_t kWeight2 = 2;
7549   const size_t kWeightTotal = kWeight1 + kWeight2;
7550   const double kWeight33Percent = static_cast<double>(kWeight1) / kWeightTotal;
7551   const double kWeight66Percent = static_cast<double>(kWeight2) / kWeightTotal;
7552   const double kErrorTolerance = 0.05;
7553   const uint32_t kRpcTimeoutMs = 10000;
7554   const size_t kNumRpcs =
7555       ComputeIdealNumRpcs(kWeight33Percent, kErrorTolerance);
7556   auto cluster = default_cluster_;
7557   // Increasing min ring size for random distribution.
7558   cluster.mutable_ring_hash_lb_config()->mutable_minimum_ring_size()->set_value(
7559       100000);
7560   cluster.set_lb_policy(Cluster::RING_HASH);
7561   balancers_[0]->ads_service()->SetCdsResource(cluster);
7562   AdsServiceImpl::EdsResourceArgs args(
7563       {{"locality0",
7564         {CreateEndpoint(0, HealthStatus::UNKNOWN, 1),
7565          CreateEndpoint(1, HealthStatus::UNKNOWN, 2)}}});
7566   balancers_[0]->ads_service()->SetEdsResource(
7567       BuildEdsResource(args, DefaultEdsServiceName()));
7568   SetNextResolutionForLbChannelAllBalancers();
7569   // TODO(donnadionne): remove extended timeout after ring creation
7570   // optimization.
7571   WaitForAllBackends(0, 2, WaitForBackendOptions(),
7572                      RpcOptions().set_timeout_ms(kRpcTimeoutMs));
7573   CheckRpcSendOk(kNumRpcs);
7574   const int weight_33_request_count =
7575       backends_[0]->backend_service()->request_count();
7576   const int weight_66_request_count =
7577       backends_[1]->backend_service()->request_count();
7578   EXPECT_THAT(static_cast<double>(weight_33_request_count) / kNumRpcs,
7579               ::testing::DoubleNear(kWeight33Percent, kErrorTolerance));
7580   EXPECT_THAT(static_cast<double>(weight_66_request_count) / kNumRpcs,
7581               ::testing::DoubleNear(kWeight66Percent, kErrorTolerance));
7582 }
7583
7584 // Tests that ring hash policy that hashes using a random value can spread
7585 // RPCs across all the backends according to locality weight.
7586 TEST_P(CdsTest,
7587        RingHashRandomHashingDistributionAccordingToLocalityAndEndpointWeight) {
7588   const size_t kWeight1 = 1 * 1;
7589   const size_t kWeight2 = 2 * 2;
7590   const size_t kWeightTotal = kWeight1 + kWeight2;
7591   const double kWeight20Percent = static_cast<double>(kWeight1) / kWeightTotal;
7592   const double kWeight80Percent = static_cast<double>(kWeight2) / kWeightTotal;
7593   const double kErrorTolerance = 0.05;
7594   const uint32_t kRpcTimeoutMs = 10000;
7595   const size_t kNumRpcs =
7596       ComputeIdealNumRpcs(kWeight20Percent, kErrorTolerance);
7597   auto cluster = default_cluster_;
7598   // Increasing min ring size for random distribution.
7599   cluster.mutable_ring_hash_lb_config()->mutable_minimum_ring_size()->set_value(
7600       100000);
7601   cluster.set_lb_policy(Cluster::RING_HASH);
7602   balancers_[0]->ads_service()->SetCdsResource(cluster);
7603   AdsServiceImpl::EdsResourceArgs args(
7604       {{"locality0", {CreateEndpoint(0, HealthStatus::UNKNOWN, 1)}, 1},
7605        {"locality1", {CreateEndpoint(1, HealthStatus::UNKNOWN, 2)}, 2}});
7606   balancers_[0]->ads_service()->SetEdsResource(
7607       BuildEdsResource(args, DefaultEdsServiceName()));
7608   SetNextResolutionForLbChannelAllBalancers();
7609   // TODO(donnadionne): remove extended timeout after ring creation
7610   // optimization.
7611   WaitForAllBackends(0, 2, WaitForBackendOptions(),
7612                      RpcOptions().set_timeout_ms(kRpcTimeoutMs));
7613   CheckRpcSendOk(kNumRpcs);
7614   const int weight_20_request_count =
7615       backends_[0]->backend_service()->request_count();
7616   const int weight_80_request_count =
7617       backends_[1]->backend_service()->request_count();
7618   EXPECT_THAT(static_cast<double>(weight_20_request_count) / kNumRpcs,
7619               ::testing::DoubleNear(kWeight20Percent, kErrorTolerance));
7620   EXPECT_THAT(static_cast<double>(weight_80_request_count) / kNumRpcs,
7621               ::testing::DoubleNear(kWeight80Percent, kErrorTolerance));
7622 }
7623
7624 // Tests round robin is not implacted by the endpoint weight, and that the
7625 // localities in a locality map are picked according to their weights.
7626 TEST_P(CdsTest, RingHashEndpointWeightDoesNotImpactWeightedRoundRobin) {
7627   SetNextResolution({});
7628   SetNextResolutionForLbChannelAllBalancers();
7629   const int kLocalityWeight0 = 2;
7630   const int kLocalityWeight1 = 8;
7631   const int kTotalLocalityWeight = kLocalityWeight0 + kLocalityWeight1;
7632   const double kLocalityWeightRate0 =
7633       static_cast<double>(kLocalityWeight0) / kTotalLocalityWeight;
7634   const double kLocalityWeightRate1 =
7635       static_cast<double>(kLocalityWeight1) / kTotalLocalityWeight;
7636   const double kErrorTolerance = 0.05;
7637   const size_t kNumRpcs =
7638       ComputeIdealNumRpcs(kLocalityWeightRate0, kErrorTolerance);
7639   // ADS response contains 2 localities, each of which contains 1 backend.
7640   AdsServiceImpl::EdsResourceArgs args({
7641       {"locality0",
7642        {CreateEndpoint(0, HealthStatus::UNKNOWN, 8)},
7643        kLocalityWeight0},
7644       {"locality1",
7645        {CreateEndpoint(1, HealthStatus::UNKNOWN, 2)},
7646        kLocalityWeight1},
7647   });
7648   balancers_[0]->ads_service()->SetEdsResource(
7649       BuildEdsResource(args, DefaultEdsServiceName()));
7650   // Wait for both backends to be ready.
7651   WaitForAllBackends(0, 2);
7652   // Send kNumRpcs RPCs.
7653   CheckRpcSendOk(kNumRpcs);
7654   // The locality picking rates should be roughly equal to the expectation.
7655   const double locality_picked_rate_0 =
7656       static_cast<double>(backends_[0]->backend_service()->request_count()) /
7657       kNumRpcs;
7658   const double locality_picked_rate_1 =
7659       static_cast<double>(backends_[1]->backend_service()->request_count()) /
7660       kNumRpcs;
7661   EXPECT_THAT(locality_picked_rate_0,
7662               ::testing::DoubleNear(kLocalityWeightRate0, kErrorTolerance));
7663   EXPECT_THAT(locality_picked_rate_1,
7664               ::testing::DoubleNear(kLocalityWeightRate1, kErrorTolerance));
7665 }
7666
7667 // Tests that ring hash policy that hashes using a fixed string ensures all
7668 // RPCs to go 1 particular backend; and that subsequent hashing policies are
7669 // ignored due to the setting of terminal.
7670 TEST_P(CdsTest, RingHashFixedHashingTerminalPolicy) {
7671   auto cluster = default_cluster_;
7672   cluster.set_lb_policy(Cluster::RING_HASH);
7673   balancers_[0]->ads_service()->SetCdsResource(cluster);
7674   auto new_route_config = default_route_config_;
7675   auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
7676   auto* hash_policy = route->mutable_route()->add_hash_policy();
7677   hash_policy->mutable_header()->set_header_name("fixed_string");
7678   hash_policy->set_terminal(true);
7679   auto* hash_policy_to_be_ignored = route->mutable_route()->add_hash_policy();
7680   hash_policy_to_be_ignored->mutable_header()->set_header_name("random_string");
7681   SetListenerAndRouteConfiguration(0, default_listener_, new_route_config);
7682   AdsServiceImpl::EdsResourceArgs args({
7683       {"locality0", CreateEndpointsForBackends()},
7684   });
7685   balancers_[0]->ads_service()->SetEdsResource(
7686       BuildEdsResource(args, DefaultEdsServiceName()));
7687   SetNextResolutionForLbChannelAllBalancers();
7688   std::vector<std::pair<std::string, std::string>> metadata = {
7689       {"fixed_string", "fixed_value"},
7690       {"random_string", absl::StrFormat("%" PRIu32, rand())},
7691   };
7692   const auto rpc_options = RpcOptions().set_metadata(std::move(metadata));
7693   CheckRpcSendOk(100, rpc_options);
7694   bool found = false;
7695   for (size_t i = 0; i < backends_.size(); ++i) {
7696     if (backends_[i]->backend_service()->request_count() > 0) {
7697       EXPECT_EQ(backends_[i]->backend_service()->request_count(), 100)
7698           << "backend " << i;
7699       EXPECT_FALSE(found) << "backend " << i;
7700       found = true;
7701     }
7702   }
7703   EXPECT_TRUE(found);
7704 }
7705
7706 // Test that the channel will go from idle to ready via connecting;
7707 // (tho it is not possible to catch the connecting state before moving to
7708 // ready)
7709 TEST_P(CdsTest, RingHashIdleToReady) {
7710   auto cluster = default_cluster_;
7711   cluster.set_lb_policy(Cluster::RING_HASH);
7712   balancers_[0]->ads_service()->SetCdsResource(cluster);
7713   auto new_route_config = default_route_config_;
7714   auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
7715   auto* hash_policy = route->mutable_route()->add_hash_policy();
7716   hash_policy->mutable_filter_state()->set_key("io.grpc.channel_id");
7717   SetListenerAndRouteConfiguration(0, default_listener_, new_route_config);
7718   AdsServiceImpl::EdsResourceArgs args({
7719       {"locality0", CreateEndpointsForBackends()},
7720   });
7721   balancers_[0]->ads_service()->SetEdsResource(
7722       BuildEdsResource(args, DefaultEdsServiceName()));
7723   SetNextResolutionForLbChannelAllBalancers();
7724   EXPECT_EQ(GRPC_CHANNEL_IDLE, channel_->GetState(false));
7725   CheckRpcSendOk();
7726   EXPECT_EQ(GRPC_CHANNEL_READY, channel_->GetState(false));
7727 }
7728
7729 // Test that when the first pick is down leading to a transient failure, we
7730 // will move on to the next ring hash entry.
7731 TEST_P(CdsTest, RingHashTransientFailureCheckNextOne) {
7732   auto cluster = default_cluster_;
7733   cluster.set_lb_policy(Cluster::RING_HASH);
7734   balancers_[0]->ads_service()->SetCdsResource(cluster);
7735   auto new_route_config = default_route_config_;
7736   auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
7737   auto* hash_policy = route->mutable_route()->add_hash_policy();
7738   hash_policy->mutable_header()->set_header_name("address_hash");
7739   SetListenerAndRouteConfiguration(0, default_listener_, new_route_config);
7740   std::vector<AdsServiceImpl::EdsResourceArgs::Endpoint> endpoints;
7741   const int unused_port = grpc_pick_unused_port_or_die();
7742   endpoints.emplace_back(unused_port);
7743   endpoints.emplace_back(backends_[1]->port());
7744   AdsServiceImpl::EdsResourceArgs args({
7745       {"locality0", std::move(endpoints)},
7746   });
7747   balancers_[0]->ads_service()->SetEdsResource(
7748       BuildEdsResource(args, DefaultEdsServiceName()));
7749   SetNextResolutionForLbChannelAllBalancers();
7750   std::vector<std::pair<std::string, std::string>> metadata = {
7751       {"address_hash",
7752        CreateMetadataValueThatHashesToBackendPort(unused_port)}};
7753   const auto rpc_options = RpcOptions().set_metadata(std::move(metadata));
7754   WaitForBackend(1, WaitForBackendOptions(), rpc_options);
7755   CheckRpcSendOk(100, rpc_options);
7756   EXPECT_EQ(0, backends_[0]->backend_service()->request_count());
7757   EXPECT_EQ(100, backends_[1]->backend_service()->request_count());
7758 }
7759
7760 // Test that when a backend goes down, we will move on to the next subchannel
7761 // (with a lower priority).  When the backend comes back up, traffic will move
7762 // back.
7763 TEST_P(CdsTest, RingHashSwitchToLowerPrioirtyAndThenBack) {
7764   auto cluster = default_cluster_;
7765   cluster.set_lb_policy(Cluster::RING_HASH);
7766   balancers_[0]->ads_service()->SetCdsResource(cluster);
7767   auto new_route_config = default_route_config_;
7768   auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
7769   auto* hash_policy = route->mutable_route()->add_hash_policy();
7770   hash_policy->mutable_header()->set_header_name("address_hash");
7771   SetListenerAndRouteConfiguration(0, default_listener_, new_route_config);
7772   AdsServiceImpl::EdsResourceArgs args({
7773       {"locality0", CreateEndpointsForBackends(0, 1), kDefaultLocalityWeight,
7774        0},
7775       {"locality1", CreateEndpointsForBackends(1, 2), kDefaultLocalityWeight,
7776        1},
7777   });
7778   balancers_[0]->ads_service()->SetEdsResource(
7779       BuildEdsResource(args, DefaultEdsServiceName()));
7780   SetNextResolutionForLbChannelAllBalancers();
7781   std::vector<std::pair<std::string, std::string>> metadata = {
7782       {"address_hash", CreateMetadataValueThatHashesToBackend(0)}};
7783   const auto rpc_options = RpcOptions().set_metadata(std::move(metadata));
7784   WaitForBackend(0, WaitForBackendOptions(), rpc_options);
7785   ShutdownBackend(0);
7786   WaitForBackend(1, WaitForBackendOptions().set_allow_failures(true),
7787                  rpc_options);
7788   StartBackend(0);
7789   WaitForBackend(0, WaitForBackendOptions(), rpc_options);
7790   CheckRpcSendOk(100, rpc_options);
7791   EXPECT_EQ(100, backends_[0]->backend_service()->request_count());
7792   EXPECT_EQ(0, backends_[1]->backend_service()->request_count());
7793 }
7794
7795 // Test that when all backends are down, we will keep reattempting.
7796 TEST_P(CdsTest, RingHashAllFailReattempt) {
7797   const uint32_t kConnectionTimeoutMilliseconds = 5000;
7798   auto cluster = default_cluster_;
7799   cluster.set_lb_policy(Cluster::RING_HASH);
7800   balancers_[0]->ads_service()->SetCdsResource(cluster);
7801   auto new_route_config = default_route_config_;
7802   auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
7803   auto* hash_policy = route->mutable_route()->add_hash_policy();
7804   hash_policy->mutable_header()->set_header_name("address_hash");
7805   SetListenerAndRouteConfiguration(0, default_listener_, new_route_config);
7806   std::vector<AdsServiceImpl::EdsResourceArgs::Endpoint> endpoints;
7807   endpoints.emplace_back(grpc_pick_unused_port_or_die());
7808   endpoints.emplace_back(backends_[1]->port());
7809   AdsServiceImpl::EdsResourceArgs args({
7810       {"locality0", std::move(endpoints)},
7811   });
7812   balancers_[0]->ads_service()->SetEdsResource(
7813       BuildEdsResource(args, DefaultEdsServiceName()));
7814   SetNextResolutionForLbChannelAllBalancers();
7815   std::vector<std::pair<std::string, std::string>> metadata = {
7816       {"address_hash", CreateMetadataValueThatHashesToBackend(0)}};
7817   EXPECT_EQ(GRPC_CHANNEL_IDLE, channel_->GetState(false));
7818   ShutdownBackend(1);
7819   CheckRpcSendFailure(CheckRpcSendFailureOptions().set_rpc_options(
7820       RpcOptions().set_metadata(std::move(metadata))));
7821   StartBackend(1);
7822   // Ensure we are actively connecting without any traffic.
7823   EXPECT_TRUE(channel_->WaitForConnected(
7824       grpc_timeout_milliseconds_to_deadline(kConnectionTimeoutMilliseconds)));
7825 }
7826
7827 // Test that when all backends are down and then up, we may pick a TF backend
7828 // and we will then jump to ready backend.
7829 TEST_P(CdsTest, RingHashTransientFailureSkipToAvailableReady) {
7830   const uint32_t kConnectionTimeoutMilliseconds = 5000;
7831   auto cluster = default_cluster_;
7832   cluster.set_lb_policy(Cluster::RING_HASH);
7833   balancers_[0]->ads_service()->SetCdsResource(cluster);
7834   auto new_route_config = default_route_config_;
7835   auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
7836   auto* hash_policy = route->mutable_route()->add_hash_policy();
7837   hash_policy->mutable_header()->set_header_name("address_hash");
7838   SetListenerAndRouteConfiguration(0, default_listener_, new_route_config);
7839   std::vector<AdsServiceImpl::EdsResourceArgs::Endpoint> endpoints;
7840   // Make sure we include some unused ports to fill the ring.
7841   endpoints.emplace_back(backends_[0]->port());
7842   endpoints.emplace_back(backends_[1]->port());
7843   endpoints.emplace_back(grpc_pick_unused_port_or_die());
7844   endpoints.emplace_back(grpc_pick_unused_port_or_die());
7845   AdsServiceImpl::EdsResourceArgs args({
7846       {"locality0", std::move(endpoints)},
7847   });
7848   balancers_[0]->ads_service()->SetEdsResource(
7849       BuildEdsResource(args, DefaultEdsServiceName()));
7850   SetNextResolutionForLbChannelAllBalancers();
7851   std::vector<std::pair<std::string, std::string>> metadata = {
7852       {"address_hash", CreateMetadataValueThatHashesToBackend(0)}};
7853   const auto rpc_options = RpcOptions().set_metadata(std::move(metadata));
7854   EXPECT_EQ(GRPC_CHANNEL_IDLE, channel_->GetState(false));
7855   ShutdownBackend(0);
7856   ShutdownBackend(1);
7857   CheckRpcSendFailure(
7858       CheckRpcSendFailureOptions().set_rpc_options(rpc_options));
7859   EXPECT_EQ(GRPC_CHANNEL_TRANSIENT_FAILURE, channel_->GetState(false));
7860   // Bring up 0, should be picked as the RPC is hashed to it.
7861   StartBackend(0);
7862   EXPECT_TRUE(channel_->WaitForConnected(
7863       grpc_timeout_milliseconds_to_deadline(kConnectionTimeoutMilliseconds)));
7864   WaitForBackend(0, WaitForBackendOptions(), rpc_options);
7865   // Bring down 0 and bring up 1.
7866   // Note the RPC contains a header value that will always be hashed to
7867   // backend 0. So by purposely bring down backend 0 and bring up another
7868   // backend, this will ensure Picker's first choice of backend 0 will fail
7869   // and it will
7870   // 1. reattempt backend 0 and
7871   // 2. go through the remaining subchannels to find one in READY.
7872   // Since the the entries in the ring is pretty distributed and we have
7873   // unused ports to fill the ring, it is almost guaranteed that the Picker
7874   // will go through some non-READY entries and skip them as per design.
7875   ShutdownBackend(0);
7876   CheckRpcSendFailure(
7877       CheckRpcSendFailureOptions().set_rpc_options(rpc_options));
7878   StartBackend(1);
7879   EXPECT_TRUE(channel_->WaitForConnected(
7880       grpc_timeout_milliseconds_to_deadline(kConnectionTimeoutMilliseconds)));
7881   WaitForBackend(1, WaitForBackendOptions(), rpc_options);
7882 }
7883
7884 // Test unspported hash policy types are all ignored before a supported
7885 // policy.
7886 TEST_P(CdsTest, RingHashUnsupportedHashPolicyUntilChannelIdHashing) {
7887   auto cluster = default_cluster_;
7888   cluster.set_lb_policy(Cluster::RING_HASH);
7889   balancers_[0]->ads_service()->SetCdsResource(cluster);
7890   auto new_route_config = default_route_config_;
7891   auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
7892   auto* hash_policy_unsupported_1 = route->mutable_route()->add_hash_policy();
7893   hash_policy_unsupported_1->mutable_cookie()->set_name("cookie");
7894   auto* hash_policy_unsupported_2 = route->mutable_route()->add_hash_policy();
7895   hash_policy_unsupported_2->mutable_connection_properties()->set_source_ip(
7896       true);
7897   auto* hash_policy_unsupported_3 = route->mutable_route()->add_hash_policy();
7898   hash_policy_unsupported_3->mutable_query_parameter()->set_name(
7899       "query_parameter");
7900   auto* hash_policy = route->mutable_route()->add_hash_policy();
7901   hash_policy->mutable_filter_state()->set_key("io.grpc.channel_id");
7902   SetListenerAndRouteConfiguration(0, default_listener_, new_route_config);
7903   AdsServiceImpl::EdsResourceArgs args({
7904       {"locality0", CreateEndpointsForBackends()},
7905   });
7906   balancers_[0]->ads_service()->SetEdsResource(
7907       BuildEdsResource(args, DefaultEdsServiceName()));
7908   SetNextResolutionForLbChannelAllBalancers();
7909   CheckRpcSendOk(100);
7910   bool found = false;
7911   for (size_t i = 0; i < backends_.size(); ++i) {
7912     if (backends_[i]->backend_service()->request_count() > 0) {
7913       EXPECT_EQ(backends_[i]->backend_service()->request_count(), 100)
7914           << "backend " << i;
7915       EXPECT_FALSE(found) << "backend " << i;
7916       found = true;
7917     }
7918   }
7919   EXPECT_TRUE(found);
7920 }
7921
7922 // Test we nack when ring hash policy has invalid hash function (something
7923 // other than XX_HASH.
7924 TEST_P(CdsTest, RingHashPolicyHasInvalidHashFunction) {
7925   auto cluster = default_cluster_;
7926   cluster.set_lb_policy(Cluster::RING_HASH);
7927   cluster.mutable_ring_hash_lb_config()->set_hash_function(
7928       Cluster::RingHashLbConfig::MURMUR_HASH_2);
7929   balancers_[0]->ads_service()->SetCdsResource(cluster);
7930   auto new_route_config = default_route_config_;
7931   auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
7932   auto* hash_policy = route->mutable_route()->add_hash_policy();
7933   hash_policy->mutable_filter_state()->set_key("io.grpc.channel_id");
7934   SetListenerAndRouteConfiguration(0, default_listener_, new_route_config);
7935   AdsServiceImpl::EdsResourceArgs args({
7936       {"locality0", CreateEndpointsForBackends()},
7937   });
7938   balancers_[0]->ads_service()->SetEdsResource(
7939       BuildEdsResource(args, DefaultEdsServiceName()));
7940   SetNextResolutionForLbChannelAllBalancers();
7941   ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
7942   const auto response_state =
7943       balancers_[0]->ads_service()->cds_response_state();
7944   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
7945   EXPECT_THAT(
7946       response_state.error_message,
7947       ::testing::HasSubstr("ring hash lb config has invalid hash function."));
7948 }
7949
7950 // Test we nack when ring hash policy has invalid ring size.
7951 TEST_P(CdsTest, RingHashPolicyHasInvalidMinimumRingSize) {
7952   auto cluster = default_cluster_;
7953   cluster.set_lb_policy(Cluster::RING_HASH);
7954   cluster.mutable_ring_hash_lb_config()->mutable_minimum_ring_size()->set_value(
7955       0);
7956   balancers_[0]->ads_service()->SetCdsResource(cluster);
7957   auto new_route_config = default_route_config_;
7958   auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
7959   auto* hash_policy = route->mutable_route()->add_hash_policy();
7960   hash_policy->mutable_filter_state()->set_key("io.grpc.channel_id");
7961   SetListenerAndRouteConfiguration(0, default_listener_, new_route_config);
7962   AdsServiceImpl::EdsResourceArgs args({
7963       {"locality0", CreateEndpointsForBackends()},
7964   });
7965   balancers_[0]->ads_service()->SetEdsResource(
7966       BuildEdsResource(args, DefaultEdsServiceName()));
7967   SetNextResolutionForLbChannelAllBalancers();
7968   ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
7969   const auto response_state =
7970       balancers_[0]->ads_service()->cds_response_state();
7971   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
7972   EXPECT_THAT(response_state.error_message,
7973               ::testing::HasSubstr(
7974                   "min_ring_size is not in the range of 1 to 8388608."));
7975 }
7976
7977 // Test we nack when ring hash policy has invalid ring size.
7978 TEST_P(CdsTest, RingHashPolicyHasInvalidMaxmumRingSize) {
7979   auto cluster = default_cluster_;
7980   cluster.set_lb_policy(Cluster::RING_HASH);
7981   cluster.mutable_ring_hash_lb_config()->mutable_maximum_ring_size()->set_value(
7982       8388609);
7983   balancers_[0]->ads_service()->SetCdsResource(cluster);
7984   auto new_route_config = default_route_config_;
7985   auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
7986   auto* hash_policy = route->mutable_route()->add_hash_policy();
7987   hash_policy->mutable_filter_state()->set_key("io.grpc.channel_id");
7988   SetListenerAndRouteConfiguration(0, default_listener_, new_route_config);
7989   AdsServiceImpl::EdsResourceArgs args({
7990       {"locality0", CreateEndpointsForBackends()},
7991   });
7992   balancers_[0]->ads_service()->SetEdsResource(
7993       BuildEdsResource(args, DefaultEdsServiceName()));
7994   SetNextResolutionForLbChannelAllBalancers();
7995   ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
7996   const auto response_state =
7997       balancers_[0]->ads_service()->cds_response_state();
7998   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
7999   EXPECT_THAT(response_state.error_message,
8000               ::testing::HasSubstr(
8001                   "max_ring_size is not in the range of 1 to 8388608."));
8002 }
8003
8004 // Test we nack when ring hash policy has invalid ring size.
8005 TEST_P(CdsTest, RingHashPolicyHasInvalidRingSizeMinGreaterThanMax) {
8006   auto cluster = default_cluster_;
8007   cluster.set_lb_policy(Cluster::RING_HASH);
8008   cluster.mutable_ring_hash_lb_config()->mutable_maximum_ring_size()->set_value(
8009       5000);
8010   cluster.mutable_ring_hash_lb_config()->mutable_minimum_ring_size()->set_value(
8011       5001);
8012   balancers_[0]->ads_service()->SetCdsResource(cluster);
8013   auto new_route_config = default_route_config_;
8014   auto* route = new_route_config.mutable_virtual_hosts(0)->mutable_routes(0);
8015   auto* hash_policy = route->mutable_route()->add_hash_policy();
8016   hash_policy->mutable_filter_state()->set_key("io.grpc.channel_id");
8017   SetListenerAndRouteConfiguration(0, default_listener_, new_route_config);
8018   AdsServiceImpl::EdsResourceArgs args({
8019       {"locality0", CreateEndpointsForBackends()},
8020   });
8021   balancers_[0]->ads_service()->SetEdsResource(
8022       BuildEdsResource(args, DefaultEdsServiceName()));
8023   SetNextResolutionForLbChannelAllBalancers();
8024   ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
8025   const auto response_state =
8026       balancers_[0]->ads_service()->cds_response_state();
8027   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
8028   EXPECT_THAT(response_state.error_message,
8029               ::testing::HasSubstr(
8030                   "min_ring_size cannot be greater than max_ring_size."));
8031 }
8032
8033 class XdsSecurityTest : public BasicTest {
8034  protected:
8035   static void SetUpTestCase() {
8036     gpr_setenv("GRPC_XDS_EXPERIMENTAL_SECURITY_SUPPORT", "true");
8037     BasicTest::SetUpTestCase();
8038   }
8039
8040   static void TearDownTestCase() {
8041     BasicTest::TearDownTestCase();
8042     gpr_unsetenv("GRPC_XDS_EXPERIMENTAL_SECURITY_SUPPORT");
8043   }
8044
8045   void SetUp() override {
8046     BasicTest::SetUp();
8047     root_cert_ = ReadFile(kCaCertPath);
8048     bad_root_cert_ = ReadFile(kBadClientCertPath);
8049     identity_pair_ = ReadTlsIdentityPair(kClientKeyPath, kClientCertPath);
8050     // TODO(yashykt): Use different client certs here instead of reusing
8051     // server certs after https://github.com/grpc/grpc/pull/24876 is merged
8052     fallback_identity_pair_ =
8053         ReadTlsIdentityPair(kServerKeyPath, kServerCertPath);
8054     bad_identity_pair_ =
8055         ReadTlsIdentityPair(kBadClientKeyPath, kBadClientCertPath);
8056     server_san_exact_.set_exact("*.test.google.fr");
8057     server_san_prefix_.set_prefix("waterzooi.test.google");
8058     server_san_suffix_.set_suffix("google.fr");
8059     server_san_contains_.set_contains("google");
8060     server_san_regex_.mutable_safe_regex()->mutable_google_re2();
8061     server_san_regex_.mutable_safe_regex()->set_regex(
8062         "(foo|waterzooi).test.google.(fr|be)");
8063     bad_san_1_.set_exact("192.168.1.4");
8064     bad_san_2_.set_exact("foo.test.google.in");
8065     authenticated_identity_ = {"testclient"};
8066     fallback_authenticated_identity_ = {"*.test.google.fr",
8067                                         "waterzooi.test.google.be",
8068                                         "*.test.youtube.com", "192.168.1.3"};
8069     AdsServiceImpl::EdsResourceArgs args({
8070         {"locality0", CreateEndpointsForBackends(0, 1)},
8071     });
8072     balancers_[0]->ads_service()->SetEdsResource(
8073         BuildEdsResource(args, DefaultEdsServiceName()));
8074     SetNextResolutionForLbChannelAllBalancers();
8075   }
8076
8077   void TearDown() override {
8078     g_fake1_cert_data_map = nullptr;
8079     g_fake2_cert_data_map = nullptr;
8080     BasicTest::TearDown();
8081   }
8082
8083   // Sends CDS updates with the new security configuration and verifies that
8084   // after propagation, this new configuration is used for connections. If \a
8085   // identity_instance_name and \a root_instance_name are both empty,
8086   // connections are expected to use fallback credentials.
8087   void UpdateAndVerifyXdsSecurityConfiguration(
8088       absl::string_view root_instance_name,
8089       absl::string_view root_certificate_name,
8090       absl::string_view identity_instance_name,
8091       absl::string_view identity_certificate_name,
8092       const std::vector<StringMatcher>& san_matchers,
8093       const std::vector<std::string>& expected_authenticated_identity,
8094       bool test_expects_failure = false) {
8095     auto cluster = default_cluster_;
8096     if (!identity_instance_name.empty() || !root_instance_name.empty()) {
8097       auto* transport_socket = cluster.mutable_transport_socket();
8098       transport_socket->set_name("envoy.transport_sockets.tls");
8099       UpstreamTlsContext upstream_tls_context;
8100       if (!identity_instance_name.empty()) {
8101         upstream_tls_context.mutable_common_tls_context()
8102             ->mutable_tls_certificate_certificate_provider_instance()
8103             ->set_instance_name(std::string(identity_instance_name));
8104         upstream_tls_context.mutable_common_tls_context()
8105             ->mutable_tls_certificate_certificate_provider_instance()
8106             ->set_certificate_name(std::string(identity_certificate_name));
8107       }
8108       if (!root_instance_name.empty()) {
8109         upstream_tls_context.mutable_common_tls_context()
8110             ->mutable_combined_validation_context()
8111             ->mutable_validation_context_certificate_provider_instance()
8112             ->set_instance_name(std::string(root_instance_name));
8113         upstream_tls_context.mutable_common_tls_context()
8114             ->mutable_combined_validation_context()
8115             ->mutable_validation_context_certificate_provider_instance()
8116             ->set_certificate_name(std::string(root_certificate_name));
8117       }
8118       if (!san_matchers.empty()) {
8119         auto* validation_context =
8120             upstream_tls_context.mutable_common_tls_context()
8121                 ->mutable_combined_validation_context()
8122                 ->mutable_default_validation_context();
8123         for (const auto& san_matcher : san_matchers) {
8124           *validation_context->add_match_subject_alt_names() = san_matcher;
8125         }
8126       }
8127       transport_socket->mutable_typed_config()->PackFrom(upstream_tls_context);
8128     }
8129     balancers_[0]->ads_service()->SetCdsResource(cluster);
8130     // The updates might take time to have an effect, so use a retry loop.
8131     constexpr int kRetryCount = 100;
8132     int num_tries = 0;
8133     for (; num_tries < kRetryCount; num_tries++) {
8134       // Give some time for the updates to propagate.
8135       gpr_sleep_until(grpc_timeout_milliseconds_to_deadline(100));
8136       if (test_expects_failure) {
8137         // Restart the servers to force a reconnection so that previously
8138         // connected subchannels are not used for the RPC.
8139         ShutdownBackend(0);
8140         StartBackend(0);
8141         if (SendRpc().ok()) {
8142           gpr_log(GPR_ERROR, "RPC succeeded. Failure expected. Trying again.");
8143           continue;
8144         }
8145       } else {
8146         WaitForBackend(0, WaitForBackendOptions().set_allow_failures(true));
8147         Status status = SendRpc();
8148         if (!status.ok()) {
8149           gpr_log(GPR_ERROR, "RPC failed. code=%d message=%s Trying again.",
8150                   status.error_code(), status.error_message().c_str());
8151           continue;
8152         }
8153         if (backends_[0]->backend_service()->last_peer_identity() !=
8154             expected_authenticated_identity) {
8155           gpr_log(
8156               GPR_ERROR,
8157               "Expected client identity does not match. (actual) %s vs "
8158               "(expected) %s Trying again.",
8159               absl::StrJoin(
8160                   backends_[0]->backend_service()->last_peer_identity(), ",")
8161                   .c_str(),
8162               absl::StrJoin(expected_authenticated_identity, ",").c_str());
8163           continue;
8164         }
8165       }
8166       break;
8167     }
8168     EXPECT_LT(num_tries, kRetryCount);
8169   }
8170
8171   std::string root_cert_;
8172   std::string bad_root_cert_;
8173   grpc_core::PemKeyCertPairList identity_pair_;
8174   grpc_core::PemKeyCertPairList fallback_identity_pair_;
8175   grpc_core::PemKeyCertPairList bad_identity_pair_;
8176   StringMatcher server_san_exact_;
8177   StringMatcher server_san_prefix_;
8178   StringMatcher server_san_suffix_;
8179   StringMatcher server_san_contains_;
8180   StringMatcher server_san_regex_;
8181   StringMatcher bad_san_1_;
8182   StringMatcher bad_san_2_;
8183   std::vector<std::string> authenticated_identity_;
8184   std::vector<std::string> fallback_authenticated_identity_;
8185 };
8186
8187 TEST_P(XdsSecurityTest, UnknownTransportSocket) {
8188   auto cluster = default_cluster_;
8189   auto* transport_socket = cluster.mutable_transport_socket();
8190   transport_socket->set_name("unknown_transport_socket");
8191   balancers_[0]->ads_service()->SetCdsResource(cluster);
8192   ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
8193   const auto response_state =
8194       balancers_[0]->ads_service()->cds_response_state();
8195   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
8196   EXPECT_THAT(response_state.error_message,
8197               ::testing::HasSubstr(
8198                   "Unrecognized transport socket: unknown_transport_socket"));
8199 }
8200
8201 TEST_P(XdsSecurityTest,
8202        TLSConfigurationWithoutValidationContextCertificateProviderInstance) {
8203   auto cluster = default_cluster_;
8204   auto* transport_socket = cluster.mutable_transport_socket();
8205   transport_socket->set_name("envoy.transport_sockets.tls");
8206   balancers_[0]->ads_service()->SetCdsResource(cluster);
8207   ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
8208   const auto response_state =
8209       balancers_[0]->ads_service()->cds_response_state();
8210   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
8211   EXPECT_THAT(response_state.error_message,
8212               ::testing::HasSubstr(
8213                   "TLS configuration provided but no "
8214                   "validation_context_certificate_provider_instance found."));
8215 }
8216
8217 TEST_P(
8218     XdsSecurityTest,
8219     MatchSubjectAltNamesProvidedWithoutValidationContextCertificateProviderInstance) {
8220   auto cluster = default_cluster_;
8221   auto* transport_socket = cluster.mutable_transport_socket();
8222   transport_socket->set_name("envoy.transport_sockets.tls");
8223   UpstreamTlsContext upstream_tls_context;
8224   auto* validation_context = upstream_tls_context.mutable_common_tls_context()
8225                                  ->mutable_combined_validation_context()
8226                                  ->mutable_default_validation_context();
8227   *validation_context->add_match_subject_alt_names() = server_san_exact_;
8228   transport_socket->mutable_typed_config()->PackFrom(upstream_tls_context);
8229   balancers_[0]->ads_service()->SetCdsResource(cluster);
8230   ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
8231   const auto response_state =
8232       balancers_[0]->ads_service()->cds_response_state();
8233   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
8234   EXPECT_THAT(response_state.error_message,
8235               ::testing::HasSubstr(
8236                   "TLS configuration provided but no "
8237                   "validation_context_certificate_provider_instance found."));
8238 }
8239
8240 TEST_P(
8241     XdsSecurityTest,
8242     TlsCertificateCertificateProviderInstanceWithoutValidationContextCertificateProviderInstance) {
8243   auto cluster = default_cluster_;
8244   auto* transport_socket = cluster.mutable_transport_socket();
8245   transport_socket->set_name("envoy.transport_sockets.tls");
8246   UpstreamTlsContext upstream_tls_context;
8247   upstream_tls_context.mutable_common_tls_context()
8248       ->mutable_tls_certificate_certificate_provider_instance()
8249       ->set_instance_name(std::string("fake_plugin1"));
8250   transport_socket->mutable_typed_config()->PackFrom(upstream_tls_context);
8251   balancers_[0]->ads_service()->SetCdsResource(cluster);
8252   ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
8253   const auto response_state =
8254       balancers_[0]->ads_service()->cds_response_state();
8255   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
8256   EXPECT_THAT(response_state.error_message,
8257               ::testing::HasSubstr(
8258                   "TLS configuration provided but no "
8259                   "validation_context_certificate_provider_instance found."));
8260 }
8261
8262 TEST_P(XdsSecurityTest, RegexSanMatcherDoesNotAllowIgnoreCase) {
8263   auto cluster = default_cluster_;
8264   auto* transport_socket = cluster.mutable_transport_socket();
8265   transport_socket->set_name("envoy.transport_sockets.tls");
8266   UpstreamTlsContext upstream_tls_context;
8267   upstream_tls_context.mutable_common_tls_context()
8268       ->mutable_combined_validation_context()
8269       ->mutable_validation_context_certificate_provider_instance()
8270       ->set_instance_name(std::string("fake_plugin1"));
8271   auto* validation_context = upstream_tls_context.mutable_common_tls_context()
8272                                  ->mutable_combined_validation_context()
8273                                  ->mutable_default_validation_context();
8274   StringMatcher matcher;
8275   matcher.mutable_safe_regex()->mutable_google_re2();
8276   matcher.mutable_safe_regex()->set_regex(
8277       "(foo|waterzooi).test.google.(fr|be)");
8278   matcher.set_ignore_case(true);
8279   *validation_context->add_match_subject_alt_names() = matcher;
8280   transport_socket->mutable_typed_config()->PackFrom(upstream_tls_context);
8281   balancers_[0]->ads_service()->SetCdsResource(cluster);
8282   ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
8283   const auto response_state =
8284       balancers_[0]->ads_service()->cds_response_state();
8285   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
8286   EXPECT_THAT(response_state.error_message,
8287               ::testing::HasSubstr(
8288                   "StringMatcher: ignore_case has no effect for SAFE_REGEX."));
8289 }
8290
8291 TEST_P(XdsSecurityTest, UnknownRootCertificateProvider) {
8292   auto cluster = default_cluster_;
8293   auto* transport_socket = cluster.mutable_transport_socket();
8294   transport_socket->set_name("envoy.transport_sockets.tls");
8295   UpstreamTlsContext upstream_tls_context;
8296   upstream_tls_context.mutable_common_tls_context()
8297       ->mutable_combined_validation_context()
8298       ->mutable_validation_context_certificate_provider_instance()
8299       ->set_instance_name("unknown");
8300   transport_socket->mutable_typed_config()->PackFrom(upstream_tls_context);
8301   balancers_[0]->ads_service()->SetCdsResource(cluster);
8302   ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
8303   const auto response_state =
8304       balancers_[0]->ads_service()->cds_response_state();
8305   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
8306   EXPECT_THAT(response_state.error_message,
8307               ::testing::HasSubstr(
8308                   "Unrecognized certificate provider instance name: unknown"));
8309 }
8310
8311 TEST_P(XdsSecurityTest, UnknownIdentityCertificateProvider) {
8312   FakeCertificateProvider::CertDataMap fake1_cert_map = {
8313       {"", {root_cert_, identity_pair_}}};
8314   g_fake1_cert_data_map = &fake1_cert_map;
8315   auto cluster = default_cluster_;
8316   auto* transport_socket = cluster.mutable_transport_socket();
8317   transport_socket->set_name("envoy.transport_sockets.tls");
8318   UpstreamTlsContext upstream_tls_context;
8319   upstream_tls_context.mutable_common_tls_context()
8320       ->mutable_tls_certificate_certificate_provider_instance()
8321       ->set_instance_name("unknown");
8322   upstream_tls_context.mutable_common_tls_context()
8323       ->mutable_combined_validation_context()
8324       ->mutable_validation_context_certificate_provider_instance()
8325       ->set_instance_name("fake_plugin1");
8326   transport_socket->mutable_typed_config()->PackFrom(upstream_tls_context);
8327   balancers_[0]->ads_service()->SetCdsResource(cluster);
8328   ASSERT_TRUE(WaitForCdsNack()) << "timed out waiting for NACK";
8329   const auto response_state =
8330       balancers_[0]->ads_service()->cds_response_state();
8331   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
8332   EXPECT_THAT(response_state.error_message,
8333               ::testing::HasSubstr(
8334                   "Unrecognized certificate provider instance name: unknown"));
8335   g_fake1_cert_data_map = nullptr;
8336 }
8337
8338 TEST_P(XdsSecurityTest, TestMtlsConfigurationWithNoSanMatchers) {
8339   FakeCertificateProvider::CertDataMap fake1_cert_map = {
8340       {"", {root_cert_, identity_pair_}}};
8341   g_fake1_cert_data_map = &fake1_cert_map;
8342   UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin1",
8343                                           "", {}, authenticated_identity_);
8344   g_fake1_cert_data_map = nullptr;
8345 }
8346
8347 TEST_P(XdsSecurityTest, TestMtlsConfigurationWithExactSanMatcher) {
8348   FakeCertificateProvider::CertDataMap fake1_cert_map = {
8349       {"", {root_cert_, identity_pair_}}};
8350   g_fake1_cert_data_map = &fake1_cert_map;
8351   UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin1",
8352                                           "", {server_san_exact_},
8353                                           authenticated_identity_);
8354   g_fake1_cert_data_map = nullptr;
8355 }
8356
8357 TEST_P(XdsSecurityTest, TestMtlsConfigurationWithPrefixSanMatcher) {
8358   FakeCertificateProvider::CertDataMap fake1_cert_map = {
8359       {"", {root_cert_, identity_pair_}}};
8360   g_fake1_cert_data_map = &fake1_cert_map;
8361   UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin1",
8362                                           "", {server_san_prefix_},
8363                                           authenticated_identity_);
8364   g_fake1_cert_data_map = nullptr;
8365 }
8366
8367 TEST_P(XdsSecurityTest, TestMtlsConfigurationWithSuffixSanMatcher) {
8368   FakeCertificateProvider::CertDataMap fake1_cert_map = {
8369       {"", {root_cert_, identity_pair_}}};
8370   g_fake1_cert_data_map = &fake1_cert_map;
8371   UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin1",
8372                                           "", {server_san_suffix_},
8373                                           authenticated_identity_);
8374   g_fake1_cert_data_map = nullptr;
8375 }
8376
8377 TEST_P(XdsSecurityTest, TestMtlsConfigurationWithContainsSanMatcher) {
8378   FakeCertificateProvider::CertDataMap fake1_cert_map = {
8379       {"", {root_cert_, identity_pair_}}};
8380   g_fake1_cert_data_map = &fake1_cert_map;
8381   UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin1",
8382                                           "", {server_san_contains_},
8383                                           authenticated_identity_);
8384   g_fake1_cert_data_map = nullptr;
8385 }
8386
8387 TEST_P(XdsSecurityTest, TestMtlsConfigurationWithRegexSanMatcher) {
8388   FakeCertificateProvider::CertDataMap fake1_cert_map = {
8389       {"", {root_cert_, identity_pair_}}};
8390   g_fake1_cert_data_map = &fake1_cert_map;
8391   UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin1",
8392                                           "", {server_san_regex_},
8393                                           authenticated_identity_);
8394   g_fake1_cert_data_map = nullptr;
8395 }
8396
8397 TEST_P(XdsSecurityTest, TestMtlsConfigurationWithSanMatchersUpdate) {
8398   FakeCertificateProvider::CertDataMap fake1_cert_map = {
8399       {"", {root_cert_, identity_pair_}}};
8400   g_fake1_cert_data_map = &fake1_cert_map;
8401   UpdateAndVerifyXdsSecurityConfiguration(
8402       "fake_plugin1", "", "fake_plugin1", "",
8403       {server_san_exact_, server_san_prefix_}, authenticated_identity_);
8404   UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin1",
8405                                           "", {bad_san_1_, bad_san_2_}, {},
8406                                           true /* failure */);
8407   UpdateAndVerifyXdsSecurityConfiguration(
8408       "fake_plugin1", "", "fake_plugin1", "",
8409       {server_san_prefix_, server_san_regex_}, authenticated_identity_);
8410   g_fake1_cert_data_map = nullptr;
8411 }
8412
8413 TEST_P(XdsSecurityTest, TestMtlsConfigurationWithRootPluginUpdate) {
8414   FakeCertificateProvider::CertDataMap fake1_cert_map = {
8415       {"", {root_cert_, identity_pair_}}};
8416   g_fake1_cert_data_map = &fake1_cert_map;
8417   FakeCertificateProvider::CertDataMap fake2_cert_map = {
8418       {"", {bad_root_cert_, bad_identity_pair_}}};
8419   g_fake2_cert_data_map = &fake2_cert_map;
8420   UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin1",
8421                                           "", {server_san_exact_},
8422                                           authenticated_identity_);
8423   UpdateAndVerifyXdsSecurityConfiguration("fake_plugin2" /* bad root */, "",
8424                                           "fake_plugin1", "", {}, {},
8425                                           true /* failure */);
8426   UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin1",
8427                                           "", {server_san_exact_},
8428                                           authenticated_identity_);
8429   g_fake1_cert_data_map = nullptr;
8430   g_fake2_cert_data_map = nullptr;
8431 }
8432
8433 TEST_P(XdsSecurityTest, TestMtlsConfigurationWithIdentityPluginUpdate) {
8434   FakeCertificateProvider::CertDataMap fake1_cert_map = {
8435       {"", {root_cert_, identity_pair_}}};
8436   g_fake1_cert_data_map = &fake1_cert_map;
8437   FakeCertificateProvider::CertDataMap fake2_cert_map = {
8438       {"", {root_cert_, fallback_identity_pair_}}};
8439   g_fake2_cert_data_map = &fake2_cert_map;
8440   UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin1",
8441                                           "", {server_san_exact_},
8442                                           authenticated_identity_);
8443   UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin2",
8444                                           "", {server_san_exact_},
8445                                           fallback_authenticated_identity_);
8446   g_fake1_cert_data_map = nullptr;
8447   g_fake2_cert_data_map = nullptr;
8448 }
8449
8450 TEST_P(XdsSecurityTest, TestMtlsConfigurationWithBothPluginsUpdated) {
8451   FakeCertificateProvider::CertDataMap fake1_cert_map = {
8452       {"", {root_cert_, identity_pair_}}};
8453   g_fake1_cert_data_map = &fake1_cert_map;
8454   FakeCertificateProvider::CertDataMap fake2_cert_map = {
8455       {"", {bad_root_cert_, bad_identity_pair_}},
8456       {"good", {root_cert_, fallback_identity_pair_}}};
8457   g_fake2_cert_data_map = &fake2_cert_map;
8458   UpdateAndVerifyXdsSecurityConfiguration("fake_plugin2", "", "fake_plugin2",
8459                                           "", {}, {}, true /* failure */);
8460   UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin1",
8461                                           "", {server_san_prefix_},
8462                                           authenticated_identity_);
8463   UpdateAndVerifyXdsSecurityConfiguration(
8464       "fake_plugin2", "good", "fake_plugin2", "good", {server_san_prefix_},
8465       fallback_authenticated_identity_);
8466   g_fake1_cert_data_map = nullptr;
8467   g_fake2_cert_data_map = nullptr;
8468 }
8469
8470 TEST_P(XdsSecurityTest, TestMtlsConfigurationWithRootCertificateNameUpdate) {
8471   FakeCertificateProvider::CertDataMap fake1_cert_map = {
8472       {"", {root_cert_, identity_pair_}},
8473       {"bad", {bad_root_cert_, bad_identity_pair_}}};
8474   g_fake1_cert_data_map = &fake1_cert_map;
8475   UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin1",
8476                                           "", {server_san_regex_},
8477                                           authenticated_identity_);
8478   UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "bad", "fake_plugin1",
8479                                           "", {server_san_regex_}, {},
8480                                           true /* failure */);
8481   g_fake1_cert_data_map = nullptr;
8482 }
8483
8484 TEST_P(XdsSecurityTest,
8485        TestMtlsConfigurationWithIdentityCertificateNameUpdate) {
8486   FakeCertificateProvider::CertDataMap fake1_cert_map = {
8487       {"", {root_cert_, identity_pair_}},
8488       {"bad", {bad_root_cert_, bad_identity_pair_}}};
8489   g_fake1_cert_data_map = &fake1_cert_map;
8490   UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin1",
8491                                           "", {server_san_exact_},
8492                                           authenticated_identity_);
8493   UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin1",
8494                                           "bad", {server_san_exact_}, {},
8495                                           true /* failure */);
8496   g_fake1_cert_data_map = nullptr;
8497 }
8498
8499 TEST_P(XdsSecurityTest,
8500        TestMtlsConfigurationWithIdentityCertificateNameUpdateGoodCerts) {
8501   FakeCertificateProvider::CertDataMap fake1_cert_map = {
8502       {"", {root_cert_, identity_pair_}},
8503       {"good", {root_cert_, fallback_identity_pair_}}};
8504   g_fake1_cert_data_map = &fake1_cert_map;
8505   UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin1",
8506                                           "", {server_san_exact_},
8507                                           authenticated_identity_);
8508   UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin1",
8509                                           "good", {server_san_exact_},
8510                                           fallback_authenticated_identity_);
8511   g_fake1_cert_data_map = nullptr;
8512 }
8513
8514 TEST_P(XdsSecurityTest, TestMtlsConfigurationWithBothCertificateNamesUpdated) {
8515   FakeCertificateProvider::CertDataMap fake1_cert_map = {
8516       {"", {root_cert_, identity_pair_}},
8517       {"bad", {bad_root_cert_, bad_identity_pair_}}};
8518   g_fake1_cert_data_map = &fake1_cert_map;
8519   UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "bad", "fake_plugin1",
8520                                           "bad", {server_san_prefix_}, {},
8521                                           true /* failure */);
8522   UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin1",
8523                                           "", {server_san_prefix_},
8524                                           authenticated_identity_);
8525   g_fake1_cert_data_map = nullptr;
8526 }
8527
8528 TEST_P(XdsSecurityTest, TestTlsConfigurationWithNoSanMatchers) {
8529   FakeCertificateProvider::CertDataMap fake1_cert_map = {
8530       {"", {root_cert_, identity_pair_}}};
8531   g_fake1_cert_data_map = &fake1_cert_map;
8532   UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "", "", {},
8533                                           {} /* unauthenticated */);
8534   g_fake1_cert_data_map = nullptr;
8535 }
8536
8537 TEST_P(XdsSecurityTest, TestTlsConfigurationWithSanMatchers) {
8538   FakeCertificateProvider::CertDataMap fake1_cert_map = {
8539       {"", {root_cert_, identity_pair_}}};
8540   g_fake1_cert_data_map = &fake1_cert_map;
8541   UpdateAndVerifyXdsSecurityConfiguration(
8542       "fake_plugin1", "", "", "",
8543       {server_san_exact_, server_san_prefix_, server_san_regex_},
8544       {} /* unauthenticated */);
8545   g_fake1_cert_data_map = nullptr;
8546 }
8547
8548 TEST_P(XdsSecurityTest, TestTlsConfigurationWithSanMatchersUpdate) {
8549   FakeCertificateProvider::CertDataMap fake1_cert_map = {
8550       {"", {root_cert_, identity_pair_}}};
8551   g_fake1_cert_data_map = &fake1_cert_map;
8552   UpdateAndVerifyXdsSecurityConfiguration(
8553       "fake_plugin1", "", "", "", {server_san_exact_, server_san_prefix_},
8554       {} /* unauthenticated */);
8555   UpdateAndVerifyXdsSecurityConfiguration(
8556       "fake_plugin1", "", "", "", {bad_san_1_, bad_san_2_},
8557       {} /* unauthenticated */, true /* failure */);
8558   UpdateAndVerifyXdsSecurityConfiguration(
8559       "fake_plugin1", "", "", "", {server_san_prefix_, server_san_regex_},
8560       {} /* unauthenticated */);
8561   g_fake1_cert_data_map = nullptr;
8562 }
8563
8564 TEST_P(XdsSecurityTest, TestTlsConfigurationWithRootCertificateNameUpdate) {
8565   FakeCertificateProvider::CertDataMap fake1_cert_map = {
8566       {"", {root_cert_, identity_pair_}},
8567       {"bad", {bad_root_cert_, bad_identity_pair_}}};
8568   g_fake1_cert_data_map = &fake1_cert_map;
8569   UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "", "",
8570                                           {server_san_exact_},
8571                                           {} /* unauthenticated */);
8572   UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "bad", "", "",
8573                                           {server_san_exact_}, {},
8574                                           true /* failure */);
8575   g_fake1_cert_data_map = nullptr;
8576 }
8577
8578 TEST_P(XdsSecurityTest, TestTlsConfigurationWithRootPluginUpdate) {
8579   FakeCertificateProvider::CertDataMap fake1_cert_map = {
8580       {"", {root_cert_, identity_pair_}}};
8581   g_fake1_cert_data_map = &fake1_cert_map;
8582   FakeCertificateProvider::CertDataMap fake2_cert_map = {
8583       {"", {bad_root_cert_, bad_identity_pair_}}};
8584   g_fake2_cert_data_map = &fake2_cert_map;
8585   UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "", "",
8586                                           {server_san_exact_},
8587                                           {} /* unauthenticated */);
8588   UpdateAndVerifyXdsSecurityConfiguration(
8589       "fake_plugin2", "", "", "", {server_san_exact_}, {}, true /* failure */);
8590   g_fake1_cert_data_map = nullptr;
8591   g_fake2_cert_data_map = nullptr;
8592 }
8593
8594 TEST_P(XdsSecurityTest, TestFallbackConfiguration) {
8595   UpdateAndVerifyXdsSecurityConfiguration("", "", "", "", {},
8596                                           fallback_authenticated_identity_);
8597   g_fake1_cert_data_map = nullptr;
8598 }
8599
8600 TEST_P(XdsSecurityTest, TestMtlsToTls) {
8601   FakeCertificateProvider::CertDataMap fake1_cert_map = {
8602       {"", {root_cert_, identity_pair_}}};
8603   g_fake1_cert_data_map = &fake1_cert_map;
8604   UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin1",
8605                                           "", {server_san_exact_},
8606                                           authenticated_identity_);
8607   UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "", "",
8608                                           {server_san_exact_},
8609                                           {} /* unauthenticated */);
8610   g_fake1_cert_data_map = nullptr;
8611 }
8612
8613 TEST_P(XdsSecurityTest, TestMtlsToFallback) {
8614   FakeCertificateProvider::CertDataMap fake1_cert_map = {
8615       {"", {root_cert_, identity_pair_}}};
8616   g_fake1_cert_data_map = &fake1_cert_map;
8617   UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin1",
8618                                           "", {server_san_exact_},
8619                                           authenticated_identity_);
8620   UpdateAndVerifyXdsSecurityConfiguration("", "", "", "", {},
8621                                           fallback_authenticated_identity_);
8622   g_fake1_cert_data_map = nullptr;
8623 }
8624
8625 TEST_P(XdsSecurityTest, TestTlsToMtls) {
8626   FakeCertificateProvider::CertDataMap fake1_cert_map = {
8627       {"", {root_cert_, identity_pair_}}};
8628   g_fake1_cert_data_map = &fake1_cert_map;
8629   UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "", "",
8630                                           {server_san_exact_},
8631                                           {} /* unauthenticated */);
8632   UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin1",
8633                                           "", {server_san_exact_},
8634                                           authenticated_identity_);
8635   g_fake1_cert_data_map = nullptr;
8636 }
8637
8638 TEST_P(XdsSecurityTest, TestTlsToFallback) {
8639   FakeCertificateProvider::CertDataMap fake1_cert_map = {
8640       {"", {root_cert_, identity_pair_}}};
8641   g_fake1_cert_data_map = &fake1_cert_map;
8642   UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "", "",
8643                                           {server_san_exact_},
8644                                           {} /* unauthenticated */);
8645   UpdateAndVerifyXdsSecurityConfiguration("", "", "", "", {},
8646                                           fallback_authenticated_identity_);
8647   g_fake1_cert_data_map = nullptr;
8648 }
8649
8650 TEST_P(XdsSecurityTest, TestFallbackToMtls) {
8651   FakeCertificateProvider::CertDataMap fake1_cert_map = {
8652       {"", {root_cert_, identity_pair_}}};
8653   g_fake1_cert_data_map = &fake1_cert_map;
8654   UpdateAndVerifyXdsSecurityConfiguration("", "", "", "", {},
8655                                           fallback_authenticated_identity_);
8656   UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "fake_plugin1",
8657                                           "", {server_san_exact_},
8658                                           authenticated_identity_);
8659   g_fake1_cert_data_map = nullptr;
8660 }
8661
8662 TEST_P(XdsSecurityTest, TestFallbackToTls) {
8663   FakeCertificateProvider::CertDataMap fake1_cert_map = {
8664       {"", {root_cert_, identity_pair_}}};
8665   g_fake1_cert_data_map = &fake1_cert_map;
8666   UpdateAndVerifyXdsSecurityConfiguration("", "", "", "", {},
8667                                           fallback_authenticated_identity_);
8668   UpdateAndVerifyXdsSecurityConfiguration("fake_plugin1", "", "", "",
8669                                           {server_san_exact_},
8670                                           {} /* unauthenticated */);
8671   g_fake1_cert_data_map = nullptr;
8672 }
8673
8674 TEST_P(XdsSecurityTest, TestFileWatcherCertificateProvider) {
8675   UpdateAndVerifyXdsSecurityConfiguration("file_plugin", "", "file_plugin", "",
8676                                           {server_san_exact_},
8677                                           authenticated_identity_);
8678 }
8679
8680 class XdsEnabledServerTest : public XdsEnd2endTest {
8681  protected:
8682   XdsEnabledServerTest()
8683       : XdsEnd2endTest(1, 1, 100, true /* use_xds_enabled_server */) {}
8684
8685   void SetUp() override {
8686     XdsEnd2endTest::SetUp();
8687     AdsServiceImpl::EdsResourceArgs args({
8688         {"locality0", CreateEndpointsForBackends(0, 1)},
8689     });
8690     balancers_[0]->ads_service()->SetEdsResource(
8691         BuildEdsResource(args, DefaultEdsServiceName()));
8692     SetNextResolution({});
8693     SetNextResolutionForLbChannelAllBalancers();
8694   }
8695 };
8696
8697 TEST_P(XdsEnabledServerTest, Basic) {
8698   Listener listener;
8699   listener.set_name(
8700       absl::StrCat("grpc/server?xds.resource.listening_address=",
8701                    ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()));
8702   listener.mutable_address()->mutable_socket_address()->set_address(
8703       ipv6_only_ ? "::1" : "127.0.0.1");
8704   listener.mutable_address()->mutable_socket_address()->set_port_value(
8705       backends_[0]->port());
8706   listener.add_filter_chains()->add_filters()->mutable_typed_config()->PackFrom(
8707       HttpConnectionManager());
8708   balancers_[0]->ads_service()->SetLdsResource(listener);
8709   WaitForBackend(0);
8710 }
8711
8712 TEST_P(XdsEnabledServerTest, BadLdsUpdateNoApiListenerNorAddress) {
8713   Listener listener;
8714   listener.set_name(
8715       absl::StrCat("grpc/server?xds.resource.listening_address=",
8716                    ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()));
8717   listener.add_filter_chains()->add_filters()->mutable_typed_config()->PackFrom(
8718       HttpConnectionManager());
8719   balancers_[0]->ads_service()->SetLdsResource(listener);
8720   ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK";
8721   const auto response_state =
8722       balancers_[0]->ads_service()->lds_response_state();
8723   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
8724   EXPECT_THAT(
8725       response_state.error_message,
8726       ::testing::HasSubstr("Listener has neither address nor ApiListener"));
8727 }
8728
8729 TEST_P(XdsEnabledServerTest, BadLdsUpdateBothApiListenerAndAddress) {
8730   Listener listener;
8731   listener.set_name(
8732       absl::StrCat("grpc/server?xds.resource.listening_address=",
8733                    ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()));
8734   balancers_[0]->ads_service()->SetLdsResource(listener);
8735   listener.mutable_address()->mutable_socket_address()->set_address(
8736       ipv6_only_ ? "::1" : "127.0.0.1");
8737   listener.mutable_address()->mutable_socket_address()->set_port_value(
8738       backends_[0]->port());
8739   auto* filter_chain = listener.add_filter_chains();
8740   filter_chain->add_filters()->mutable_typed_config()->PackFrom(
8741       HttpConnectionManager());
8742   listener.mutable_api_listener();
8743   balancers_[0]->ads_service()->SetLdsResource(listener);
8744   ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK";
8745   const auto response_state =
8746       balancers_[0]->ads_service()->lds_response_state();
8747   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
8748   EXPECT_THAT(
8749       response_state.error_message,
8750       ::testing::HasSubstr("Listener has both address and ApiListener"));
8751 }
8752
8753 TEST_P(XdsEnabledServerTest, UnsupportedL4Filter) {
8754   Listener listener;
8755   listener.set_name(
8756       absl::StrCat("grpc/server?xds.resource.listening_address=",
8757                    ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()));
8758   balancers_[0]->ads_service()->SetLdsResource(listener);
8759   listener.mutable_address()->mutable_socket_address()->set_address(
8760       ipv6_only_ ? "::1" : "127.0.0.1");
8761   listener.mutable_address()->mutable_socket_address()->set_port_value(
8762       backends_[0]->port());
8763   listener.add_filter_chains()->add_filters()->mutable_typed_config()->PackFrom(default_listener_ /* any proto object other than HttpConnectionManager */);
8764   balancers_[0]->ads_service()->SetLdsResource(listener);
8765   ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK";
8766   const auto response_state =
8767       balancers_[0]->ads_service()->lds_response_state();
8768   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
8769   EXPECT_THAT(response_state.error_message,
8770               ::testing::HasSubstr("Unsupported filter type"));
8771 }
8772
8773 TEST_P(XdsEnabledServerTest, UnsupportedHttpFilter) {
8774   Listener listener;
8775   listener.set_name(
8776       absl::StrCat("grpc/server?xds.resource.listening_address=",
8777                    ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()));
8778   listener.mutable_address()->mutable_socket_address()->set_address(
8779       ipv6_only_ ? "::1" : "127.0.0.1");
8780   listener.mutable_address()->mutable_socket_address()->set_port_value(
8781       backends_[0]->port());
8782   HttpConnectionManager http_connection_manager;
8783   auto* http_filter = http_connection_manager.add_http_filters();
8784   http_filter->set_name("grpc.testing.unsupported_http_filter");
8785   http_filter->mutable_typed_config()->set_type_url(
8786       "grpc.testing.unsupported_http_filter");
8787   listener.add_filter_chains()->add_filters()->mutable_typed_config()->PackFrom(
8788       http_connection_manager);
8789   balancers_[0]->ads_service()->SetLdsResource(listener);
8790   listener.set_name(
8791       absl::StrCat("grpc/server?xds.resource.listening_address=[::1]:",
8792                    backends_[0]->port()));
8793   balancers_[0]->ads_service()->SetLdsResource(listener);
8794   ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK";
8795   const auto response_state =
8796       balancers_[0]->ads_service()->lds_response_state();
8797   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
8798   EXPECT_THAT(response_state.error_message,
8799               ::testing::HasSubstr("no filter registered for config type "
8800                                    "grpc.testing.unsupported_http_filter"));
8801 }
8802
8803 TEST_P(XdsEnabledServerTest, HttpFilterNotSupportedOnServer) {
8804   Listener listener;
8805   listener.set_name(
8806       absl::StrCat("grpc/server?xds.resource.listening_address=",
8807                    ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()));
8808   listener.mutable_address()->mutable_socket_address()->set_address(
8809       ipv6_only_ ? "::1" : "127.0.0.1");
8810   listener.mutable_address()->mutable_socket_address()->set_port_value(
8811       backends_[0]->port());
8812   HttpConnectionManager http_connection_manager;
8813   auto* http_filter = http_connection_manager.add_http_filters();
8814   http_filter->set_name("grpc.testing.client_only_http_filter");
8815   http_filter->mutable_typed_config()->set_type_url(
8816       "grpc.testing.client_only_http_filter");
8817   http_filter = http_connection_manager.add_http_filters();
8818   http_filter->set_name("router");
8819   http_filter->mutable_typed_config()->PackFrom(
8820       envoy::extensions::filters::http::router::v3::Router());
8821   listener.add_filter_chains()->add_filters()->mutable_typed_config()->PackFrom(
8822       http_connection_manager);
8823   balancers_[0]->ads_service()->SetLdsResource(listener);
8824   listener.set_name(
8825       absl::StrCat("grpc/server?xds.resource.listening_address=[::1]:",
8826                    backends_[0]->port()));
8827   balancers_[0]->ads_service()->SetLdsResource(listener);
8828   ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK";
8829   const auto response_state =
8830       balancers_[0]->ads_service()->lds_response_state();
8831   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
8832   EXPECT_THAT(
8833       response_state.error_message,
8834       ::testing::HasSubstr("Filter grpc.testing.client_only_http_filter is not "
8835                            "supported on servers"));
8836 }
8837
8838 TEST_P(XdsEnabledServerTest,
8839        HttpFilterNotSupportedOnServerIgnoredWhenOptional) {
8840   Listener listener;
8841   listener.set_name(
8842       absl::StrCat("grpc/server?xds.resource.listening_address=",
8843                    ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()));
8844   listener.mutable_address()->mutable_socket_address()->set_address(
8845       ipv6_only_ ? "::1" : "127.0.0.1");
8846   listener.mutable_address()->mutable_socket_address()->set_port_value(
8847       backends_[0]->port());
8848   HttpConnectionManager http_connection_manager;
8849   auto* http_filter = http_connection_manager.add_http_filters();
8850   http_filter->set_name("grpc.testing.client_only_http_filter");
8851   http_filter->mutable_typed_config()->set_type_url(
8852       "grpc.testing.client_only_http_filter");
8853   http_filter->set_is_optional(true);
8854   listener.add_filter_chains()->add_filters()->mutable_typed_config()->PackFrom(
8855       http_connection_manager);
8856   balancers_[0]->ads_service()->SetLdsResource(listener);
8857   listener.set_name(
8858       absl::StrCat("grpc/server?xds.resource.listening_address=[::1]:",
8859                    backends_[0]->port()));
8860   balancers_[0]->ads_service()->SetLdsResource(listener);
8861   WaitForBackend(0);
8862   const auto response_state =
8863       balancers_[0]->ads_service()->lds_response_state();
8864   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::ACKED);
8865 }
8866
8867 // Verify that a mismatch of listening address results in "not serving"
8868 // status.
8869 TEST_P(XdsEnabledServerTest, ListenerAddressMismatch) {
8870   Listener listener;
8871   listener.set_name(
8872       absl::StrCat("grpc/server?xds.resource.listening_address=",
8873                    ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()));
8874   listener.mutable_address()->mutable_socket_address()->set_address(
8875       ipv6_only_ ? "::1" : "127.0.0.1");
8876   listener.mutable_address()->mutable_socket_address()->set_port_value(
8877       backends_[0]->port());
8878   listener.add_filter_chains()->add_filters()->mutable_typed_config()->PackFrom(
8879       HttpConnectionManager());
8880   balancers_[0]->ads_service()->SetLdsResource(listener);
8881   WaitForBackend(0);
8882   // Set a different listening address in the LDS update
8883   listener.mutable_address()->mutable_socket_address()->set_address(
8884       "192.168.1.1");
8885   balancers_[0]->ads_service()->SetLdsResource(listener);
8886   backends_[0]->notifier()->WaitOnServingStatusChange(
8887       absl::StrCat(ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()),
8888       grpc::StatusCode::FAILED_PRECONDITION);
8889 }
8890
8891 TEST_P(XdsEnabledServerTest, UseOriginalDstNotSupported) {
8892   Listener listener;
8893   listener.set_name(
8894       absl::StrCat("grpc/server?xds.resource.listening_address=",
8895                    ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()));
8896   balancers_[0]->ads_service()->SetLdsResource(listener);
8897   listener.mutable_address()->mutable_socket_address()->set_address(
8898       ipv6_only_ ? "::1" : "127.0.0.1");
8899   listener.mutable_address()->mutable_socket_address()->set_port_value(
8900       backends_[0]->port());
8901   listener.add_filter_chains()->add_filters()->mutable_typed_config()->PackFrom(
8902       HttpConnectionManager());
8903   listener.mutable_use_original_dst()->set_value(true);
8904   balancers_[0]->ads_service()->SetLdsResource(listener);
8905   ASSERT_TRUE(WaitForLdsNack()) << "timed out waiting for NACK";
8906   const auto response_state =
8907       balancers_[0]->ads_service()->lds_response_state();
8908   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
8909   EXPECT_THAT(
8910       response_state.error_message,
8911       ::testing::HasSubstr("Field \'use_original_dst\' is not supported."));
8912 }
8913
8914 class XdsServerSecurityTest : public XdsEnd2endTest {
8915  protected:
8916   XdsServerSecurityTest()
8917       : XdsEnd2endTest(1, 1, 100, true /* use_xds_enabled_server */) {}
8918
8919   static void SetUpTestCase() {
8920     gpr_setenv("GRPC_XDS_EXPERIMENTAL_SECURITY_SUPPORT", "true");
8921     XdsEnd2endTest::SetUpTestCase();
8922   }
8923
8924   static void TearDownTestCase() {
8925     XdsEnd2endTest::TearDownTestCase();
8926     gpr_unsetenv("GRPC_XDS_EXPERIMENTAL_SECURITY_SUPPORT");
8927   }
8928
8929   void SetUp() override {
8930     XdsEnd2endTest::SetUp();
8931     root_cert_ = ReadFile(kCaCertPath);
8932     bad_root_cert_ = ReadFile(kBadClientCertPath);
8933     identity_pair_ = ReadTlsIdentityPair(kServerKeyPath, kServerCertPath);
8934     bad_identity_pair_ =
8935         ReadTlsIdentityPair(kBadClientKeyPath, kBadClientCertPath);
8936     identity_pair_2_ = ReadTlsIdentityPair(kClientKeyPath, kClientCertPath);
8937     server_authenticated_identity_ = {"*.test.google.fr",
8938                                       "waterzooi.test.google.be",
8939                                       "*.test.youtube.com", "192.168.1.3"};
8940     server_authenticated_identity_2_ = {"testclient"};
8941     client_authenticated_identity_ = {"*.test.google.fr",
8942                                       "waterzooi.test.google.be",
8943                                       "*.test.youtube.com", "192.168.1.3"};
8944     AdsServiceImpl::EdsResourceArgs args({
8945         {"locality0", CreateEndpointsForBackends(0, 1)},
8946     });
8947     balancers_[0]->ads_service()->SetEdsResource(
8948         BuildEdsResource(args, DefaultEdsServiceName()));
8949     SetNextResolution({});
8950     SetNextResolutionForLbChannelAllBalancers();
8951   }
8952
8953   void TearDown() override {
8954     g_fake1_cert_data_map = nullptr;
8955     g_fake2_cert_data_map = nullptr;
8956     XdsEnd2endTest::TearDown();
8957   }
8958
8959   void SetLdsUpdate(absl::string_view root_instance_name,
8960                     absl::string_view root_certificate_name,
8961                     absl::string_view identity_instance_name,
8962                     absl::string_view identity_certificate_name,
8963                     bool require_client_certificates) {
8964     Listener listener;
8965     listener.set_name(absl::StrCat(
8966         ipv6_only_ ? "grpc/server?xds.resource.listening_address=[::1]:"
8967                    : "grpc/server?xds.resource.listening_address=127.0.0.1:",
8968         backends_[0]->port()));
8969     listener.mutable_address()->mutable_socket_address()->set_address(
8970         ipv6_only_ ? "[::1]" : "127.0.0.1");
8971     listener.mutable_address()->mutable_socket_address()->set_port_value(
8972         backends_[0]->port());
8973     auto* filter_chain = listener.add_filter_chains();
8974     filter_chain->add_filters()->mutable_typed_config()->PackFrom(
8975         HttpConnectionManager());
8976     if (!identity_instance_name.empty()) {
8977       auto* transport_socket = filter_chain->mutable_transport_socket();
8978       transport_socket->set_name("envoy.transport_sockets.tls");
8979       DownstreamTlsContext downstream_tls_context;
8980       downstream_tls_context.mutable_common_tls_context()
8981           ->mutable_tls_certificate_certificate_provider_instance()
8982           ->set_instance_name(std::string(identity_instance_name));
8983       downstream_tls_context.mutable_common_tls_context()
8984           ->mutable_tls_certificate_certificate_provider_instance()
8985           ->set_certificate_name(std::string(identity_certificate_name));
8986       if (!root_instance_name.empty()) {
8987         downstream_tls_context.mutable_common_tls_context()
8988             ->mutable_combined_validation_context()
8989             ->mutable_validation_context_certificate_provider_instance()
8990             ->set_instance_name(std::string(root_instance_name));
8991         downstream_tls_context.mutable_common_tls_context()
8992             ->mutable_combined_validation_context()
8993             ->mutable_validation_context_certificate_provider_instance()
8994             ->set_certificate_name(std::string(root_certificate_name));
8995         downstream_tls_context.mutable_require_client_certificate()->set_value(
8996             require_client_certificates);
8997       }
8998       transport_socket->mutable_typed_config()->PackFrom(
8999           downstream_tls_context);
9000     }
9001     balancers_[0]->ads_service()->SetLdsResource(listener);
9002   }
9003
9004   std::shared_ptr<grpc::Channel> CreateMtlsChannel() {
9005     ChannelArguments args;
9006     // Override target name for host name check
9007     args.SetString(GRPC_SSL_TARGET_NAME_OVERRIDE_ARG,
9008                    ipv6_only_ ? "::1" : "127.0.0.1");
9009     args.SetInt(GRPC_ARG_USE_LOCAL_SUBCHANNEL_POOL, 1);
9010     std::string uri = absl::StrCat(
9011         ipv6_only_ ? "ipv6:[::1]:" : "ipv4:127.0.0.1:", backends_[0]->port());
9012     // TODO(yashykt): Switch to using C++ API once b/173823806 is fixed.
9013     grpc_tls_credentials_options* options =
9014         grpc_tls_credentials_options_create();
9015     grpc_tls_credentials_options_set_server_verification_option(
9016         options, GRPC_TLS_SKIP_HOSTNAME_VERIFICATION);
9017     grpc_tls_credentials_options_set_certificate_provider(
9018         options,
9019         grpc_core::MakeRefCounted<grpc_core::StaticDataCertificateProvider>(
9020             ReadFile(kCaCertPath),
9021             ReadTlsIdentityPair(kServerKeyPath, kServerCertPath))
9022             .get());
9023     grpc_tls_credentials_options_watch_root_certs(options);
9024     grpc_tls_credentials_options_watch_identity_key_cert_pairs(options);
9025     grpc_tls_server_authorization_check_config* check_config =
9026         grpc_tls_server_authorization_check_config_create(
9027             nullptr, ServerAuthCheckSchedule, nullptr, nullptr);
9028     grpc_tls_credentials_options_set_server_authorization_check_config(
9029         options, check_config);
9030     auto channel_creds = std::make_shared<SecureChannelCredentials>(
9031         grpc_tls_credentials_create(options));
9032     grpc_tls_server_authorization_check_config_release(check_config);
9033     return CreateCustomChannel(uri, channel_creds, args);
9034   }
9035
9036   std::shared_ptr<grpc::Channel> CreateTlsChannel() {
9037     ChannelArguments args;
9038     // Override target name for host name check
9039     args.SetString(GRPC_SSL_TARGET_NAME_OVERRIDE_ARG,
9040                    ipv6_only_ ? "::1" : "127.0.0.1");
9041     args.SetInt(GRPC_ARG_USE_LOCAL_SUBCHANNEL_POOL, 1);
9042     std::string uri = absl::StrCat(
9043         ipv6_only_ ? "ipv6:[::1]:" : "ipv4:127.0.0.1:", backends_[0]->port());
9044     // TODO(yashykt): Switch to using C++ API once b/173823806 is fixed.
9045     grpc_tls_credentials_options* options =
9046         grpc_tls_credentials_options_create();
9047     grpc_tls_credentials_options_set_server_verification_option(
9048         options, GRPC_TLS_SKIP_HOSTNAME_VERIFICATION);
9049     grpc_tls_credentials_options_set_certificate_provider(
9050         options,
9051         grpc_core::MakeRefCounted<grpc_core::StaticDataCertificateProvider>(
9052             ReadFile(kCaCertPath),
9053             ReadTlsIdentityPair(kServerKeyPath, kServerCertPath))
9054             .get());
9055     grpc_tls_credentials_options_watch_root_certs(options);
9056     grpc_tls_server_authorization_check_config* check_config =
9057         grpc_tls_server_authorization_check_config_create(
9058             nullptr, ServerAuthCheckSchedule, nullptr, nullptr);
9059     grpc_tls_credentials_options_set_server_authorization_check_config(
9060         options, check_config);
9061     auto channel_creds = std::make_shared<SecureChannelCredentials>(
9062         grpc_tls_credentials_create(options));
9063     grpc_tls_server_authorization_check_config_release(check_config);
9064     return CreateCustomChannel(uri, channel_creds, args);
9065   }
9066
9067   std::shared_ptr<grpc::Channel> CreateInsecureChannel() {
9068     ChannelArguments args;
9069     // Override target name for host name check
9070     args.SetString(GRPC_SSL_TARGET_NAME_OVERRIDE_ARG,
9071                    ipv6_only_ ? "::1" : "127.0.0.1");
9072     args.SetInt(GRPC_ARG_USE_LOCAL_SUBCHANNEL_POOL, 1);
9073     std::string uri = absl::StrCat(
9074         ipv6_only_ ? "ipv6:[::1]:" : "ipv4:127.0.0.1:", backends_[0]->port());
9075     return CreateCustomChannel(uri, InsecureChannelCredentials(), args);
9076   }
9077
9078   void SendRpc(std::function<std::shared_ptr<grpc::Channel>()> channel_creator,
9079                std::vector<std::string> expected_server_identity,
9080                std::vector<std::string> expected_client_identity,
9081                bool test_expects_failure = false) {
9082     gpr_log(GPR_INFO, "Sending RPC");
9083     int num_tries = 0;
9084     constexpr int kRetryCount = 100;
9085     for (; num_tries < kRetryCount; num_tries++) {
9086       auto channel = channel_creator();
9087       auto stub = grpc::testing::EchoTestService::NewStub(channel);
9088       ClientContext context;
9089       context.set_wait_for_ready(true);
9090       context.set_deadline(grpc_timeout_milliseconds_to_deadline(2000));
9091       EchoRequest request;
9092       request.set_message(kRequestMessage);
9093       EchoResponse response;
9094       Status status = stub->Echo(&context, request, &response);
9095       if (test_expects_failure) {
9096         if (status.ok()) {
9097           gpr_log(GPR_ERROR, "RPC succeeded. Failure expected. Trying again.");
9098           continue;
9099         }
9100       } else {
9101         if (!status.ok()) {
9102           gpr_log(GPR_ERROR, "RPC failed. code=%d message=%s Trying again.",
9103                   status.error_code(), status.error_message().c_str());
9104           continue;
9105         }
9106         EXPECT_EQ(response.message(), kRequestMessage);
9107         std::vector<std::string> peer_identity;
9108         for (const auto& entry : context.auth_context()->GetPeerIdentity()) {
9109           peer_identity.emplace_back(
9110               std::string(entry.data(), entry.size()).c_str());
9111         }
9112         if (peer_identity != expected_server_identity) {
9113           gpr_log(GPR_ERROR,
9114                   "Expected server identity does not match. (actual) %s vs "
9115                   "(expected) %s Trying again.",
9116                   absl::StrJoin(peer_identity, ",").c_str(),
9117                   absl::StrJoin(expected_server_identity, ",").c_str());
9118           continue;
9119         }
9120         if (backends_[0]->backend_service()->last_peer_identity() !=
9121             expected_client_identity) {
9122           gpr_log(
9123               GPR_ERROR,
9124               "Expected client identity does not match. (actual) %s vs "
9125               "(expected) %s Trying again.",
9126               absl::StrJoin(
9127                   backends_[0]->backend_service()->last_peer_identity(), ",")
9128                   .c_str(),
9129               absl::StrJoin(expected_client_identity, ",").c_str());
9130           continue;
9131         }
9132       }
9133       break;
9134     }
9135     EXPECT_LT(num_tries, kRetryCount);
9136   }
9137
9138   std::string root_cert_;
9139   std::string bad_root_cert_;
9140   grpc_core::PemKeyCertPairList identity_pair_;
9141   grpc_core::PemKeyCertPairList bad_identity_pair_;
9142   grpc_core::PemKeyCertPairList identity_pair_2_;
9143   std::vector<std::string> server_authenticated_identity_;
9144   std::vector<std::string> server_authenticated_identity_2_;
9145   std::vector<std::string> client_authenticated_identity_;
9146 };
9147
9148 TEST_P(XdsServerSecurityTest, UnknownTransportSocket) {
9149   Listener listener;
9150   listener.set_name(
9151       absl::StrCat("grpc/server?xds.resource.listening_address=",
9152                    ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()));
9153   auto* socket_address = listener.mutable_address()->mutable_socket_address();
9154   socket_address->set_address(ipv6_only_ ? "::1" : "127.0.0.1");
9155   socket_address->set_port_value(backends_[0]->port());
9156   auto* filter_chain = listener.add_filter_chains();
9157   filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9158       HttpConnectionManager());
9159   auto* transport_socket = filter_chain->mutable_transport_socket();
9160   transport_socket->set_name("unknown_transport_socket");
9161   balancers_[0]->ads_service()->SetLdsResource(listener);
9162   ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED))
9163       << "timed out waiting for NACK";
9164   const auto response_state =
9165       balancers_[0]->ads_service()->lds_response_state();
9166   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
9167   EXPECT_THAT(response_state.error_message,
9168               ::testing::HasSubstr(
9169                   "Unrecognized transport socket: unknown_transport_socket"));
9170 }
9171
9172 TEST_P(
9173     XdsServerSecurityTest,
9174     NacksRequiringClientCertificateWithoutValidationCertificateProviderInstance) {
9175   Listener listener;
9176   listener.set_name(
9177       absl::StrCat("grpc/server?xds.resource.listening_address=",
9178                    ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()));
9179   auto* socket_address = listener.mutable_address()->mutable_socket_address();
9180   socket_address->set_address(ipv6_only_ ? "::1" : "127.0.0.1");
9181   socket_address->set_port_value(backends_[0]->port());
9182   auto* filter_chain = listener.add_filter_chains();
9183   filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9184       HttpConnectionManager());
9185   auto* transport_socket = filter_chain->mutable_transport_socket();
9186   transport_socket->set_name("envoy.transport_sockets.tls");
9187   DownstreamTlsContext downstream_tls_context;
9188   downstream_tls_context.mutable_common_tls_context()
9189       ->mutable_tls_certificate_certificate_provider_instance()
9190       ->set_instance_name("fake_plugin1");
9191   downstream_tls_context.mutable_require_client_certificate()->set_value(true);
9192   transport_socket->mutable_typed_config()->PackFrom(downstream_tls_context);
9193   balancers_[0]->ads_service()->SetLdsResource(listener);
9194   ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED))
9195       << "timed out waiting for NACK";
9196   const auto response_state =
9197       balancers_[0]->ads_service()->lds_response_state();
9198   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
9199   EXPECT_THAT(response_state.error_message,
9200               ::testing::HasSubstr(
9201                   "TLS configuration requires client certificates but no "
9202                   "certificate provider instance specified for validation."));
9203 }
9204
9205 TEST_P(XdsServerSecurityTest,
9206        NacksTlsConfigurationWithoutIdentityProviderInstance) {
9207   Listener listener;
9208   listener.set_name(
9209       absl::StrCat("grpc/server?xds.resource.listening_address=",
9210                    ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()));
9211   auto* socket_address = listener.mutable_address()->mutable_socket_address();
9212   socket_address->set_address(ipv6_only_ ? "::1" : "127.0.0.1");
9213   socket_address->set_port_value(backends_[0]->port());
9214   auto* filter_chain = listener.add_filter_chains();
9215   filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9216       HttpConnectionManager());
9217   auto* transport_socket = filter_chain->mutable_transport_socket();
9218   transport_socket->set_name("envoy.transport_sockets.tls");
9219   DownstreamTlsContext downstream_tls_context;
9220   transport_socket->mutable_typed_config()->PackFrom(downstream_tls_context);
9221   balancers_[0]->ads_service()->SetLdsResource(listener);
9222   ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED))
9223       << "timed out waiting for NACK";
9224   const auto response_state =
9225       balancers_[0]->ads_service()->lds_response_state();
9226   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
9227   EXPECT_THAT(response_state.error_message,
9228               ::testing::HasSubstr(
9229                   "TLS configuration provided but no "
9230                   "tls_certificate_certificate_provider_instance found."));
9231 }
9232
9233 TEST_P(XdsServerSecurityTest, UnknownIdentityCertificateProvider) {
9234   SetLdsUpdate("", "", "unknown", "", false);
9235   SendRpc([this]() { return CreateTlsChannel(); }, {}, {},
9236           true /* test_expects_failure */);
9237   ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED))
9238       << "timed out waiting for NACK";
9239   const auto response_state =
9240       balancers_[0]->ads_service()->lds_response_state();
9241   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
9242   EXPECT_THAT(response_state.error_message,
9243               ::testing::HasSubstr(
9244                   "Unrecognized certificate provider instance name: unknown"));
9245 }
9246
9247 TEST_P(XdsServerSecurityTest, UnknownRootCertificateProvider) {
9248   FakeCertificateProvider::CertDataMap fake1_cert_map = {
9249       {"", {root_cert_, identity_pair_}}};
9250   SetLdsUpdate("unknown", "", "fake_plugin1", "", false);
9251   ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED))
9252       << "timed out waiting for NACK";
9253   const auto response_state =
9254       balancers_[0]->ads_service()->lds_response_state();
9255   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
9256   EXPECT_THAT(response_state.error_message,
9257               ::testing::HasSubstr(
9258                   "Unrecognized certificate provider instance name: unknown"));
9259 }
9260
9261 TEST_P(XdsServerSecurityTest, CertificatesNotAvailable) {
9262   FakeCertificateProvider::CertDataMap fake1_cert_map;
9263   g_fake1_cert_data_map = &fake1_cert_map;
9264   SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", true);
9265   SendRpc([this]() { return CreateMtlsChannel(); }, {}, {},
9266           true /* test_expects_failure */);
9267 }
9268
9269 TEST_P(XdsServerSecurityTest, TestMtls) {
9270   FakeCertificateProvider::CertDataMap fake1_cert_map = {
9271       {"", {root_cert_, identity_pair_}}};
9272   g_fake1_cert_data_map = &fake1_cert_map;
9273   SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", true);
9274   SendRpc([this]() { return CreateMtlsChannel(); },
9275           server_authenticated_identity_, client_authenticated_identity_);
9276 }
9277
9278 TEST_P(XdsServerSecurityTest, TestMtlsWithRootPluginUpdate) {
9279   FakeCertificateProvider::CertDataMap fake1_cert_map = {
9280       {"", {root_cert_, identity_pair_}}};
9281   g_fake1_cert_data_map = &fake1_cert_map;
9282   FakeCertificateProvider::CertDataMap fake2_cert_map = {
9283       {"", {bad_root_cert_, bad_identity_pair_}}};
9284   g_fake2_cert_data_map = &fake2_cert_map;
9285   SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", true);
9286   SendRpc([this]() { return CreateMtlsChannel(); },
9287           server_authenticated_identity_, client_authenticated_identity_);
9288   SetLdsUpdate("fake_plugin2", "", "fake_plugin1", "", true);
9289   SendRpc([this]() { return CreateMtlsChannel(); }, {}, {},
9290           true /* test_expects_failure */);
9291 }
9292
9293 TEST_P(XdsServerSecurityTest, TestMtlsWithIdentityPluginUpdate) {
9294   FakeCertificateProvider::CertDataMap fake1_cert_map = {
9295       {"", {root_cert_, identity_pair_}}};
9296   g_fake1_cert_data_map = &fake1_cert_map;
9297   FakeCertificateProvider::CertDataMap fake2_cert_map = {
9298       {"", {root_cert_, identity_pair_2_}}};
9299   g_fake2_cert_data_map = &fake2_cert_map;
9300   SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", true);
9301   SendRpc([this]() { return CreateMtlsChannel(); },
9302           server_authenticated_identity_, client_authenticated_identity_);
9303   SetLdsUpdate("fake_plugin1", "", "fake_plugin2", "", true);
9304   SendRpc([this]() { return CreateMtlsChannel(); },
9305           server_authenticated_identity_2_, client_authenticated_identity_);
9306 }
9307
9308 TEST_P(XdsServerSecurityTest, TestMtlsWithBothPluginsUpdated) {
9309   FakeCertificateProvider::CertDataMap fake1_cert_map = {
9310       {"", {root_cert_, identity_pair_}}};
9311   g_fake1_cert_data_map = &fake1_cert_map;
9312   FakeCertificateProvider::CertDataMap fake2_cert_map = {
9313       {"good", {root_cert_, identity_pair_2_}},
9314       {"", {bad_root_cert_, bad_identity_pair_}}};
9315   g_fake2_cert_data_map = &fake2_cert_map;
9316   SetLdsUpdate("fake_plugin2", "", "fake_plugin2", "", true);
9317   SendRpc([this]() { return CreateMtlsChannel(); }, {}, {},
9318           true /* test_expects_failure */);
9319   SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", true);
9320   SendRpc([this]() { return CreateMtlsChannel(); },
9321           server_authenticated_identity_, client_authenticated_identity_);
9322   SetLdsUpdate("fake_plugin2", "good", "fake_plugin2", "good", true);
9323   SendRpc([this]() { return CreateMtlsChannel(); },
9324           server_authenticated_identity_2_, client_authenticated_identity_);
9325 }
9326
9327 TEST_P(XdsServerSecurityTest, TestMtlsWithRootCertificateNameUpdate) {
9328   FakeCertificateProvider::CertDataMap fake1_cert_map = {
9329       {"", {root_cert_, identity_pair_}},
9330       {"bad", {bad_root_cert_, bad_identity_pair_}}};
9331   g_fake1_cert_data_map = &fake1_cert_map;
9332   SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", true);
9333   SendRpc([this]() { return CreateMtlsChannel(); },
9334           server_authenticated_identity_, client_authenticated_identity_);
9335   SetLdsUpdate("fake_plugin1", "bad", "fake_plugin1", "", true);
9336   SendRpc([this]() { return CreateMtlsChannel(); }, {}, {},
9337           true /* test_expects_failure */);
9338 }
9339
9340 TEST_P(XdsServerSecurityTest, TestMtlsWithIdentityCertificateNameUpdate) {
9341   FakeCertificateProvider::CertDataMap fake1_cert_map = {
9342       {"", {root_cert_, identity_pair_}},
9343       {"good", {root_cert_, identity_pair_2_}}};
9344   g_fake1_cert_data_map = &fake1_cert_map;
9345   SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", true);
9346   SendRpc([this]() { return CreateMtlsChannel(); },
9347           server_authenticated_identity_, client_authenticated_identity_);
9348   SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "good", true);
9349   SendRpc([this]() { return CreateMtlsChannel(); },
9350           server_authenticated_identity_2_, client_authenticated_identity_);
9351 }
9352
9353 TEST_P(XdsServerSecurityTest, TestMtlsWithBothCertificateNamesUpdated) {
9354   FakeCertificateProvider::CertDataMap fake1_cert_map = {
9355       {"", {root_cert_, identity_pair_}},
9356       {"good", {root_cert_, identity_pair_2_}}};
9357   g_fake1_cert_data_map = &fake1_cert_map;
9358   SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", true);
9359   SendRpc([this]() { return CreateMtlsChannel(); },
9360           server_authenticated_identity_, client_authenticated_identity_);
9361   SetLdsUpdate("fake_plugin1", "good", "fake_plugin1", "good", true);
9362   SendRpc([this]() { return CreateMtlsChannel(); },
9363           server_authenticated_identity_2_, client_authenticated_identity_);
9364 }
9365
9366 TEST_P(XdsServerSecurityTest, TestMtlsNotRequiringButProvidingClientCerts) {
9367   FakeCertificateProvider::CertDataMap fake1_cert_map = {
9368       {"", {root_cert_, identity_pair_}}};
9369   g_fake1_cert_data_map = &fake1_cert_map;
9370   SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", false);
9371   SendRpc([this]() { return CreateMtlsChannel(); },
9372           server_authenticated_identity_, client_authenticated_identity_);
9373 }
9374
9375 TEST_P(XdsServerSecurityTest, TestMtlsNotRequiringAndNotProvidingClientCerts) {
9376   FakeCertificateProvider::CertDataMap fake1_cert_map = {
9377       {"", {root_cert_, identity_pair_}}};
9378   g_fake1_cert_data_map = &fake1_cert_map;
9379   SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", false);
9380   SendRpc([this]() { return CreateTlsChannel(); },
9381           server_authenticated_identity_, {});
9382 }
9383
9384 TEST_P(XdsServerSecurityTest, TestTls) {
9385   FakeCertificateProvider::CertDataMap fake1_cert_map = {
9386       {"", {root_cert_, identity_pair_}}};
9387   g_fake1_cert_data_map = &fake1_cert_map;
9388   SetLdsUpdate("", "", "fake_plugin1", "", false);
9389   SendRpc([this]() { return CreateTlsChannel(); },
9390           server_authenticated_identity_, {});
9391 }
9392
9393 TEST_P(XdsServerSecurityTest, TestTlsWithIdentityPluginUpdate) {
9394   FakeCertificateProvider::CertDataMap fake1_cert_map = {
9395       {"", {root_cert_, identity_pair_}}};
9396   g_fake1_cert_data_map = &fake1_cert_map;
9397   FakeCertificateProvider::CertDataMap fake2_cert_map = {
9398       {"", {root_cert_, identity_pair_2_}}};
9399   g_fake2_cert_data_map = &fake2_cert_map;
9400   SetLdsUpdate("", "", "fake_plugin1", "", false);
9401   SendRpc([this]() { return CreateTlsChannel(); },
9402           server_authenticated_identity_, {});
9403   SetLdsUpdate("", "", "fake_plugin2", "", false);
9404   SendRpc([this]() { return CreateTlsChannel(); },
9405           server_authenticated_identity_2_, {});
9406 }
9407
9408 TEST_P(XdsServerSecurityTest, TestTlsWithIdentityCertificateNameUpdate) {
9409   FakeCertificateProvider::CertDataMap fake1_cert_map = {
9410       {"", {root_cert_, identity_pair_}},
9411       {"good", {root_cert_, identity_pair_2_}}};
9412   g_fake1_cert_data_map = &fake1_cert_map;
9413   SetLdsUpdate("", "", "fake_plugin1", "", false);
9414   SendRpc([this]() { return CreateTlsChannel(); },
9415           server_authenticated_identity_, {});
9416   SetLdsUpdate("", "", "fake_plugin1", "good", false);
9417   SendRpc([this]() { return CreateTlsChannel(); },
9418           server_authenticated_identity_2_, {});
9419 }
9420
9421 TEST_P(XdsServerSecurityTest, TestFallback) {
9422   FakeCertificateProvider::CertDataMap fake1_cert_map = {
9423       {"", {root_cert_, identity_pair_}}};
9424   g_fake1_cert_data_map = &fake1_cert_map;
9425   SetLdsUpdate("", "", "", "", false);
9426   SendRpc([this]() { return CreateInsecureChannel(); }, {}, {});
9427 }
9428
9429 TEST_P(XdsServerSecurityTest, TestMtlsToTls) {
9430   FakeCertificateProvider::CertDataMap fake1_cert_map = {
9431       {"", {root_cert_, identity_pair_}}};
9432   g_fake1_cert_data_map = &fake1_cert_map;
9433   SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", true);
9434   SendRpc([this]() { return CreateTlsChannel(); }, {}, {},
9435           true /* test_expects_failure */);
9436   SetLdsUpdate("", "", "fake_plugin1", "", false);
9437   SendRpc([this]() { return CreateTlsChannel(); },
9438           server_authenticated_identity_, {});
9439 }
9440
9441 TEST_P(XdsServerSecurityTest, TestTlsToMtls) {
9442   FakeCertificateProvider::CertDataMap fake1_cert_map = {
9443       {"", {root_cert_, identity_pair_}}};
9444   g_fake1_cert_data_map = &fake1_cert_map;
9445   SetLdsUpdate("", "", "fake_plugin1", "", false);
9446   SendRpc([this]() { return CreateTlsChannel(); },
9447           server_authenticated_identity_, {});
9448   SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", true);
9449   SendRpc([this]() { return CreateTlsChannel(); }, {}, {},
9450           true /* test_expects_failure */);
9451 }
9452
9453 TEST_P(XdsServerSecurityTest, TestMtlsToFallback) {
9454   FakeCertificateProvider::CertDataMap fake1_cert_map = {
9455       {"", {root_cert_, identity_pair_}}};
9456   g_fake1_cert_data_map = &fake1_cert_map;
9457   SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", false);
9458   SendRpc([this]() { return CreateMtlsChannel(); },
9459           server_authenticated_identity_, client_authenticated_identity_);
9460   SetLdsUpdate("", "", "", "", false);
9461   SendRpc([this]() { return CreateInsecureChannel(); }, {}, {});
9462 }
9463
9464 TEST_P(XdsServerSecurityTest, TestFallbackToMtls) {
9465   FakeCertificateProvider::CertDataMap fake1_cert_map = {
9466       {"", {root_cert_, identity_pair_}}};
9467   g_fake1_cert_data_map = &fake1_cert_map;
9468   SetLdsUpdate("", "", "", "", false);
9469   SendRpc([this]() { return CreateInsecureChannel(); }, {}, {});
9470   SetLdsUpdate("fake_plugin1", "", "fake_plugin1", "", true);
9471   SendRpc([this]() { return CreateMtlsChannel(); },
9472           server_authenticated_identity_, client_authenticated_identity_);
9473 }
9474
9475 TEST_P(XdsServerSecurityTest, TestTlsToFallback) {
9476   FakeCertificateProvider::CertDataMap fake1_cert_map = {
9477       {"", {root_cert_, identity_pair_}}};
9478   g_fake1_cert_data_map = &fake1_cert_map;
9479   SetLdsUpdate("", "", "fake_plugin1", "", false);
9480   SendRpc([this]() { return CreateTlsChannel(); },
9481           server_authenticated_identity_, {});
9482   SetLdsUpdate("", "", "", "", false);
9483   SendRpc([this]() { return CreateInsecureChannel(); }, {}, {});
9484 }
9485
9486 TEST_P(XdsServerSecurityTest, TestFallbackToTls) {
9487   FakeCertificateProvider::CertDataMap fake1_cert_map = {
9488       {"", {root_cert_, identity_pair_}}};
9489   g_fake1_cert_data_map = &fake1_cert_map;
9490   SetLdsUpdate("", "", "", "", false);
9491   SendRpc([this]() { return CreateInsecureChannel(); }, {}, {});
9492   SetLdsUpdate("", "", "fake_plugin1", "", false);
9493   SendRpc([this]() { return CreateTlsChannel(); },
9494           server_authenticated_identity_, {});
9495 }
9496
9497 class XdsEnabledServerStatusNotificationTest : public XdsServerSecurityTest {
9498  protected:
9499   void SetValidLdsUpdate() { SetLdsUpdate("", "", "", "", false); }
9500
9501   void SetInvalidLdsUpdate() {
9502     Listener listener;
9503     listener.set_name(absl::StrCat(
9504         "grpc/server?xds.resource.listening_address=",
9505         ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()));
9506     auto* socket_address = listener.mutable_address()->mutable_socket_address();
9507     socket_address->set_address(ipv6_only_ ? "::1" : "127.0.0.1");
9508     socket_address->set_port_value(backends_[0]->port());
9509     balancers_[0]->ads_service()->SetLdsResource(listener);
9510   }
9511
9512   void UnsetLdsUpdate() {
9513     balancers_[0]->ads_service()->UnsetResource(
9514         kLdsTypeUrl, absl::StrCat("grpc/server?xds.resource.listening_address=",
9515                                   ipv6_only_ ? "[::1]:" : "127.0.0.1:",
9516                                   backends_[0]->port()));
9517   }
9518 };
9519
9520 TEST_P(XdsEnabledServerStatusNotificationTest, ServingStatus) {
9521   SetValidLdsUpdate();
9522   backends_[0]->notifier()->WaitOnServingStatusChange(
9523       absl::StrCat(ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()),
9524       grpc::StatusCode::OK);
9525   SendRpc([this]() { return CreateInsecureChannel(); }, {}, {});
9526 }
9527
9528 TEST_P(XdsEnabledServerStatusNotificationTest, NotServingStatus) {
9529   SetInvalidLdsUpdate();
9530   backends_[0]->notifier()->WaitOnServingStatusChange(
9531       absl::StrCat(ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()),
9532       grpc::StatusCode::UNAVAILABLE);
9533   SendRpc([this]() { return CreateInsecureChannel(); }, {}, {},
9534           true /* test_expects_failure */);
9535 }
9536
9537 TEST_P(XdsEnabledServerStatusNotificationTest, ErrorUpdateWhenAlreadyServing) {
9538   SetValidLdsUpdate();
9539   backends_[0]->notifier()->WaitOnServingStatusChange(
9540       absl::StrCat(ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()),
9541       grpc::StatusCode::OK);
9542   SendRpc([this]() { return CreateInsecureChannel(); }, {}, {});
9543   // Invalid update does not lead to a change in the serving status.
9544   SetInvalidLdsUpdate();
9545   do {
9546     SendRpc([this]() { return CreateInsecureChannel(); }, {}, {});
9547   } while (balancers_[0]->ads_service()->lds_response_state().state ==
9548            AdsServiceImpl::ResponseState::SENT);
9549   backends_[0]->notifier()->WaitOnServingStatusChange(
9550       absl::StrCat(ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()),
9551       grpc::StatusCode::OK);
9552   SendRpc([this]() { return CreateInsecureChannel(); }, {}, {});
9553 }
9554
9555 TEST_P(XdsEnabledServerStatusNotificationTest,
9556        NotServingStatusToServingStatusTransition) {
9557   SetInvalidLdsUpdate();
9558   backends_[0]->notifier()->WaitOnServingStatusChange(
9559       absl::StrCat(ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()),
9560       grpc::StatusCode::UNAVAILABLE);
9561   SendRpc([this]() { return CreateInsecureChannel(); }, {}, {},
9562           true /* test_expects_failure */);
9563   // Send a valid LDS update to change to serving status
9564   SetValidLdsUpdate();
9565   backends_[0]->notifier()->WaitOnServingStatusChange(
9566       absl::StrCat(ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()),
9567       grpc::StatusCode::OK);
9568   SendRpc([this]() { return CreateInsecureChannel(); }, {}, {});
9569 }
9570
9571 // This test verifies that the resource getting deleted when already serving
9572 // results in future connections being dropped.
9573 TEST_P(XdsEnabledServerStatusNotificationTest,
9574        ServingStatusToNonServingStatusTransition) {
9575   SetValidLdsUpdate();
9576   backends_[0]->notifier()->WaitOnServingStatusChange(
9577       absl::StrCat(ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()),
9578       grpc::StatusCode::OK);
9579   SendRpc([this]() { return CreateInsecureChannel(); }, {}, {});
9580   // Deleting the resource should result in a non-serving status.
9581   UnsetLdsUpdate();
9582   backends_[0]->notifier()->WaitOnServingStatusChange(
9583       absl::StrCat(ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()),
9584       grpc::StatusCode::NOT_FOUND);
9585   SendRpc([this]() { return CreateInsecureChannel(); }, {}, {},
9586           true /* test_expects_failure */);
9587 }
9588
9589 TEST_P(XdsEnabledServerStatusNotificationTest, RepeatedServingStatusChanges) {
9590   for (int i = 0; i < 5; i++) {
9591     // Send a valid LDS update to get the server to start listening
9592     SetValidLdsUpdate();
9593     backends_[0]->notifier()->WaitOnServingStatusChange(
9594         absl::StrCat(ipv6_only_ ? "[::1]:" : "127.0.0.1:",
9595                      backends_[0]->port()),
9596         grpc::StatusCode::OK);
9597     SendRpc([this]() { return CreateInsecureChannel(); }, {}, {});
9598     // Deleting the resource will make the server start rejecting connections
9599     UnsetLdsUpdate();
9600     backends_[0]->notifier()->WaitOnServingStatusChange(
9601         absl::StrCat(ipv6_only_ ? "[::1]:" : "127.0.0.1:",
9602                      backends_[0]->port()),
9603         grpc::StatusCode::NOT_FOUND);
9604     SendRpc([this]() { return CreateInsecureChannel(); }, {}, {},
9605             true /* test_expects_failure */);
9606   }
9607 }
9608
9609 TEST_P(XdsEnabledServerStatusNotificationTest, ExistingRpcsOnResourceDeletion) {
9610   // Send a valid LDS update to get the server to start listening
9611   SetValidLdsUpdate();
9612   backends_[0]->notifier()->WaitOnServingStatusChange(
9613       absl::StrCat(ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()),
9614       grpc::StatusCode::OK);
9615   constexpr int kNumChannels = 10;
9616   struct StreamingRpc {
9617     std::shared_ptr<Channel> channel;
9618     std::unique_ptr<grpc::testing::EchoTestService::Stub> stub;
9619     ClientContext context;
9620     std::unique_ptr<ClientReaderWriter<EchoRequest, EchoResponse>> stream;
9621   } streaming_rpcs[kNumChannels];
9622   EchoRequest request;
9623   EchoResponse response;
9624   request.set_message("Hello");
9625   for (int i = 0; i < kNumChannels; i++) {
9626     streaming_rpcs[i].channel = CreateInsecureChannel();
9627     streaming_rpcs[i].stub =
9628         grpc::testing::EchoTestService::NewStub(streaming_rpcs[i].channel);
9629     streaming_rpcs[i].context.set_wait_for_ready(true);
9630     streaming_rpcs[i].stream =
9631         streaming_rpcs[i].stub->BidiStream(&streaming_rpcs[i].context);
9632     EXPECT_TRUE(streaming_rpcs[i].stream->Write(request));
9633     streaming_rpcs[i].stream->Read(&response);
9634     EXPECT_EQ(request.message(), response.message());
9635   }
9636   // Deleting the resource will make the server start rejecting connections
9637   UnsetLdsUpdate();
9638   backends_[0]->notifier()->WaitOnServingStatusChange(
9639       absl::StrCat(ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()),
9640       grpc::StatusCode::NOT_FOUND);
9641   SendRpc([this]() { return CreateInsecureChannel(); }, {}, {},
9642           true /* test_expects_failure */);
9643   for (int i = 0; i < kNumChannels; i++) {
9644     EXPECT_TRUE(streaming_rpcs[i].stream->Write(request));
9645     streaming_rpcs[i].stream->Read(&response);
9646     EXPECT_EQ(request.message(), response.message());
9647     EXPECT_TRUE(streaming_rpcs[i].stream->WritesDone());
9648     auto status = streaming_rpcs[i].stream->Finish();
9649     EXPECT_TRUE(status.ok())
9650         << status.error_message() << ", " << status.error_details() << ", "
9651         << streaming_rpcs[i].context.debug_error_string();
9652     // New RPCs on the existing channels should fail.
9653     ClientContext new_context;
9654     new_context.set_deadline(grpc_timeout_milliseconds_to_deadline(1000));
9655     EXPECT_FALSE(
9656         streaming_rpcs[i].stub->Echo(&new_context, request, &response).ok());
9657   }
9658 }
9659
9660 using XdsServerFilterChainMatchTest = XdsServerSecurityTest;
9661
9662 TEST_P(XdsServerFilterChainMatchTest,
9663        DefaultFilterChainUsedWhenNoFilterChainMentioned) {
9664   Listener listener;
9665   listener.set_name(
9666       absl::StrCat("grpc/server?xds.resource.listening_address=",
9667                    ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()));
9668   auto* socket_address = listener.mutable_address()->mutable_socket_address();
9669   socket_address->set_address(ipv6_only_ ? "::1" : "127.0.0.1");
9670   socket_address->set_port_value(backends_[0]->port());
9671   listener.mutable_default_filter_chain()
9672       ->add_filters()
9673       ->mutable_typed_config()
9674       ->PackFrom(HttpConnectionManager());
9675   balancers_[0]->ads_service()->SetLdsResource(listener);
9676   SendRpc([this]() { return CreateInsecureChannel(); }, {}, {});
9677 }
9678
9679 TEST_P(XdsServerFilterChainMatchTest,
9680        DefaultFilterChainUsedWhenOtherFilterChainsDontMatch) {
9681   Listener listener;
9682   listener.set_name(
9683       absl::StrCat("grpc/server?xds.resource.listening_address=",
9684                    ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()));
9685   auto* socket_address = listener.mutable_address()->mutable_socket_address();
9686   socket_address->set_address(ipv6_only_ ? "::1" : "127.0.0.1");
9687   socket_address->set_port_value(backends_[0]->port());
9688   // Add a filter chain that will never get matched
9689   auto* filter_chain = listener.add_filter_chains();
9690   filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9691       HttpConnectionManager());
9692   filter_chain->mutable_filter_chain_match()
9693       ->mutable_destination_port()
9694       ->set_value(8080);
9695   // Add default filter chain that should get used
9696   listener.mutable_default_filter_chain()
9697       ->add_filters()
9698       ->mutable_typed_config()
9699       ->PackFrom(HttpConnectionManager());
9700   balancers_[0]->ads_service()->SetLdsResource(listener);
9701   SendRpc([this]() { return CreateInsecureChannel(); }, {}, {});
9702 }
9703
9704 TEST_P(XdsServerFilterChainMatchTest,
9705        FilterChainsWithDestinationPortDontMatch) {
9706   Listener listener;
9707   listener.set_name(
9708       absl::StrCat("grpc/server?xds.resource.listening_address=",
9709                    ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()));
9710   auto* socket_address = listener.mutable_address()->mutable_socket_address();
9711   socket_address->set_address(ipv6_only_ ? "::1" : "127.0.0.1");
9712   socket_address->set_port_value(backends_[0]->port());
9713   // Add filter chain with destination port that should never get matched
9714   auto* filter_chain = listener.add_filter_chains();
9715   filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9716       HttpConnectionManager());
9717   filter_chain->mutable_filter_chain_match()
9718       ->mutable_destination_port()
9719       ->set_value(8080);
9720   balancers_[0]->ads_service()->SetLdsResource(listener);
9721   // RPC should fail since no matching filter chain was found and no default
9722   // filter chain is configured.
9723   SendRpc([this]() { return CreateInsecureChannel(); }, {}, {},
9724           true /* test_expects_failure */);
9725 }
9726
9727 TEST_P(XdsServerFilterChainMatchTest, FilterChainsWithServerNamesDontMatch) {
9728   Listener listener;
9729   listener.set_name(
9730       absl::StrCat("grpc/server?xds.resource.listening_address=",
9731                    ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()));
9732   auto* socket_address = listener.mutable_address()->mutable_socket_address();
9733   socket_address->set_address(ipv6_only_ ? "::1" : "127.0.0.1");
9734   socket_address->set_port_value(backends_[0]->port());
9735   // Add filter chain with server name that should never get matched
9736   auto* filter_chain = listener.add_filter_chains();
9737   filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9738       HttpConnectionManager());
9739   filter_chain->mutable_filter_chain_match()->add_server_names("server_name");
9740   balancers_[0]->ads_service()->SetLdsResource(listener);
9741   // RPC should fail since no matching filter chain was found and no default
9742   // filter chain is configured.
9743   SendRpc([this]() { return CreateInsecureChannel(); }, {}, {},
9744           true /* test_expects_failure */);
9745 }
9746
9747 TEST_P(XdsServerFilterChainMatchTest,
9748        FilterChainsWithTransportProtocolsOtherThanRawBufferDontMatch) {
9749   Listener listener;
9750   listener.set_name(
9751       absl::StrCat("grpc/server?xds.resource.listening_address=",
9752                    ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()));
9753   auto* socket_address = listener.mutable_address()->mutable_socket_address();
9754   socket_address->set_address(ipv6_only_ ? "::1" : "127.0.0.1");
9755   socket_address->set_port_value(backends_[0]->port());
9756   // Add filter chain with transport protocol "tls" that should never match
9757   auto* filter_chain = listener.add_filter_chains();
9758   filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9759       HttpConnectionManager());
9760   filter_chain->mutable_filter_chain_match()->set_transport_protocol("tls");
9761   balancers_[0]->ads_service()->SetLdsResource(listener);
9762   // RPC should fail since no matching filter chain was found and no default
9763   // filter chain is configured.
9764   SendRpc([this]() { return CreateInsecureChannel(); }, {}, {},
9765           true /* test_expects_failure */);
9766 }
9767
9768 TEST_P(XdsServerFilterChainMatchTest,
9769        FilterChainsWithApplicationProtocolsDontMatch) {
9770   Listener listener;
9771   listener.set_name(
9772       absl::StrCat("grpc/server?xds.resource.listening_address=",
9773                    ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()));
9774   auto* socket_address = listener.mutable_address()->mutable_socket_address();
9775   socket_address->set_address(ipv6_only_ ? "::1" : "127.0.0.1");
9776   socket_address->set_port_value(backends_[0]->port());
9777   // Add filter chain with application protocol that should never get matched
9778   auto* filter_chain = listener.add_filter_chains();
9779   filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9780       HttpConnectionManager());
9781   filter_chain->mutable_filter_chain_match()->add_application_protocols("h2");
9782   balancers_[0]->ads_service()->SetLdsResource(listener);
9783   // RPC should fail since no matching filter chain was found and no default
9784   // filter chain is configured.
9785   SendRpc([this]() { return CreateInsecureChannel(); }, {}, {},
9786           true /* test_expects_failure */);
9787 }
9788
9789 TEST_P(XdsServerFilterChainMatchTest,
9790        FilterChainsWithTransportProtocolRawBufferIsPreferred) {
9791   Listener listener;
9792   listener.set_name(
9793       absl::StrCat("grpc/server?xds.resource.listening_address=",
9794                    ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()));
9795   auto* socket_address = listener.mutable_address()->mutable_socket_address();
9796   socket_address->set_address(ipv6_only_ ? "::1" : "127.0.0.1");
9797   socket_address->set_port_value(backends_[0]->port());
9798   // Add filter chain with "raw_buffer" transport protocol
9799   auto* filter_chain = listener.add_filter_chains();
9800   filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9801       HttpConnectionManager());
9802   filter_chain->mutable_filter_chain_match()->set_transport_protocol(
9803       "raw_buffer");
9804   // Add another filter chain with no transport protocol set but application
9805   // protocol set (fails match)
9806   filter_chain = listener.add_filter_chains();
9807   filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9808       HttpConnectionManager());
9809   filter_chain->mutable_filter_chain_match()->add_application_protocols("h2");
9810   balancers_[0]->ads_service()->SetLdsResource(listener);
9811   // A successful RPC proves that filter chains that mention "raw_buffer" as
9812   // the transport protocol are chosen as the best match in the round.
9813   SendRpc([this]() { return CreateInsecureChannel(); }, {}, {});
9814 }
9815
9816 TEST_P(XdsServerFilterChainMatchTest,
9817        FilterChainsWithMoreSpecificDestinationPrefixRangesArePreferred) {
9818   Listener listener;
9819   listener.set_name(
9820       absl::StrCat("grpc/server?xds.resource.listening_address=",
9821                    ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()));
9822   auto* socket_address = listener.mutable_address()->mutable_socket_address();
9823   socket_address->set_address(ipv6_only_ ? "::1" : "127.0.0.1");
9824   socket_address->set_port_value(backends_[0]->port());
9825   // Add filter chain with prefix range (length 4 and 16) but with server name
9826   // mentioned. (Prefix range is matched first.)
9827   auto* filter_chain = listener.add_filter_chains();
9828   filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9829       HttpConnectionManager());
9830   auto* prefix_range =
9831       filter_chain->mutable_filter_chain_match()->add_prefix_ranges();
9832   prefix_range->set_address_prefix(ipv6_only_ ? "::1" : "127.0.0.1");
9833   prefix_range->mutable_prefix_len()->set_value(4);
9834   prefix_range =
9835       filter_chain->mutable_filter_chain_match()->add_prefix_ranges();
9836   prefix_range->set_address_prefix(ipv6_only_ ? "::1" : "127.0.0.1");
9837   prefix_range->mutable_prefix_len()->set_value(16);
9838   filter_chain->mutable_filter_chain_match()->add_server_names("server_name");
9839   // Add filter chain with two prefix ranges (length 8 and 24). Since 24 is
9840   // the highest match, it should be chosen.
9841   filter_chain = listener.add_filter_chains();
9842   filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9843       HttpConnectionManager());
9844   prefix_range =
9845       filter_chain->mutable_filter_chain_match()->add_prefix_ranges();
9846   prefix_range->set_address_prefix(ipv6_only_ ? "::1" : "127.0.0.1");
9847   prefix_range->mutable_prefix_len()->set_value(8);
9848   prefix_range =
9849       filter_chain->mutable_filter_chain_match()->add_prefix_ranges();
9850   prefix_range->set_address_prefix(ipv6_only_ ? "::1" : "127.0.0.1");
9851   prefix_range->mutable_prefix_len()->set_value(24);
9852   // Add another filter chain with a non-matching prefix range (with length
9853   // 30)
9854   filter_chain = listener.add_filter_chains();
9855   filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9856       HttpConnectionManager());
9857   prefix_range =
9858       filter_chain->mutable_filter_chain_match()->add_prefix_ranges();
9859   prefix_range->set_address_prefix("192.168.1.1");
9860   prefix_range->mutable_prefix_len()->set_value(30);
9861   filter_chain->mutable_filter_chain_match()->add_server_names("server_name");
9862   // Add another filter chain with no prefix range mentioned
9863   filter_chain = listener.add_filter_chains();
9864   filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9865       HttpConnectionManager());
9866   filter_chain->mutable_filter_chain_match()->add_server_names("server_name");
9867   balancers_[0]->ads_service()->SetLdsResource(listener);
9868   // A successful RPC proves that the filter chain with the longest matching
9869   // prefix range was the best match.
9870   SendRpc([this]() { return CreateInsecureChannel(); }, {}, {});
9871 }
9872
9873 TEST_P(XdsServerFilterChainMatchTest,
9874        FilterChainsThatMentionSourceTypeArePreferred) {
9875   Listener listener;
9876   listener.set_name(
9877       absl::StrCat("grpc/server?xds.resource.listening_address=",
9878                    ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()));
9879   auto* socket_address = listener.mutable_address()->mutable_socket_address();
9880   socket_address->set_address(ipv6_only_ ? "::1" : "127.0.0.1");
9881   socket_address->set_port_value(backends_[0]->port());
9882   // Add filter chain with the local source type (best match)
9883   auto* filter_chain = listener.add_filter_chains();
9884   filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9885       HttpConnectionManager());
9886   filter_chain->mutable_filter_chain_match()->set_source_type(
9887       FilterChainMatch::SAME_IP_OR_LOOPBACK);
9888   // Add filter chain with the external source type but bad source port.
9889   // Note that backends_[0]->port() will never be a match for the source port
9890   // because it is already being used by a backend.
9891   filter_chain = listener.add_filter_chains();
9892   filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9893       HttpConnectionManager());
9894   filter_chain->mutable_filter_chain_match()->set_source_type(
9895       FilterChainMatch::EXTERNAL);
9896   filter_chain->mutable_filter_chain_match()->add_source_ports(
9897       backends_[0]->port());
9898   // Add filter chain with the default source type (ANY) but bad source port.
9899   filter_chain = listener.add_filter_chains();
9900   filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9901       HttpConnectionManager());
9902   filter_chain->mutable_filter_chain_match()->add_source_ports(
9903       backends_[0]->port());
9904   balancers_[0]->ads_service()->SetLdsResource(listener);
9905   // A successful RPC proves that the filter chain with the longest matching
9906   // prefix range was the best match.
9907   SendRpc([this]() { return CreateInsecureChannel(); }, {}, {});
9908 }
9909
9910 TEST_P(XdsServerFilterChainMatchTest,
9911        FilterChainsWithMoreSpecificSourcePrefixRangesArePreferred) {
9912   Listener listener;
9913   listener.set_name(
9914       absl::StrCat("grpc/server?xds.resource.listening_address=",
9915                    ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()));
9916   auto* socket_address = listener.mutable_address()->mutable_socket_address();
9917   socket_address->set_address(ipv6_only_ ? "::1" : "127.0.0.1");
9918   socket_address->set_port_value(backends_[0]->port());
9919   // Add filter chain with source prefix range (length 16) but with a bad
9920   // source port mentioned. (Prefix range is matched first.) Note that
9921   // backends_[0]->port() will never be a match for the source port because it
9922   // is already being used by a backend.
9923   auto* filter_chain = listener.add_filter_chains();
9924   filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9925       HttpConnectionManager());
9926   auto* source_prefix_range =
9927       filter_chain->mutable_filter_chain_match()->add_source_prefix_ranges();
9928   source_prefix_range->set_address_prefix(ipv6_only_ ? "::1" : "127.0.0.1");
9929   source_prefix_range->mutable_prefix_len()->set_value(4);
9930   source_prefix_range =
9931       filter_chain->mutable_filter_chain_match()->add_source_prefix_ranges();
9932   source_prefix_range->set_address_prefix(ipv6_only_ ? "::1" : "127.0.0.1");
9933   source_prefix_range->mutable_prefix_len()->set_value(16);
9934   filter_chain->mutable_filter_chain_match()->add_source_ports(
9935       backends_[0]->port());
9936   // Add filter chain with two source prefix ranges (length 8 and 24). Since
9937   // 24 is the highest match, it should be chosen.
9938   filter_chain = listener.add_filter_chains();
9939   filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9940       HttpConnectionManager());
9941   source_prefix_range =
9942       filter_chain->mutable_filter_chain_match()->add_source_prefix_ranges();
9943   source_prefix_range->set_address_prefix(ipv6_only_ ? "::1" : "127.0.0.1");
9944   source_prefix_range->mutable_prefix_len()->set_value(8);
9945   source_prefix_range =
9946       filter_chain->mutable_filter_chain_match()->add_source_prefix_ranges();
9947   source_prefix_range->set_address_prefix(ipv6_only_ ? "::1" : "127.0.0.1");
9948   source_prefix_range->mutable_prefix_len()->set_value(24);
9949   // Add another filter chain with a non-matching source prefix range (with
9950   // length 30) and bad source port
9951   filter_chain = listener.add_filter_chains();
9952   filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9953       HttpConnectionManager());
9954   source_prefix_range =
9955       filter_chain->mutable_filter_chain_match()->add_source_prefix_ranges();
9956   source_prefix_range->set_address_prefix("192.168.1.1");
9957   source_prefix_range->mutable_prefix_len()->set_value(30);
9958   filter_chain->mutable_filter_chain_match()->add_source_ports(
9959       backends_[0]->port());
9960   // Add another filter chain with no source prefix range mentioned and bad
9961   // source port
9962   filter_chain = listener.add_filter_chains();
9963   filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9964       HttpConnectionManager());
9965   filter_chain->mutable_filter_chain_match()->add_source_ports(
9966       backends_[0]->port());
9967   balancers_[0]->ads_service()->SetLdsResource(listener);
9968   // A successful RPC proves that the filter chain with the longest matching
9969   // source prefix range was the best match.
9970   SendRpc([this]() { return CreateInsecureChannel(); }, {}, {});
9971 }
9972
9973 TEST_P(XdsServerFilterChainMatchTest,
9974        FilterChainsWithMoreSpecificSourcePortArePreferred) {
9975   Listener listener;
9976   listener.set_name(
9977       absl::StrCat("grpc/server?xds.resource.listening_address=",
9978                    ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()));
9979   auto* socket_address = listener.mutable_address()->mutable_socket_address();
9980   socket_address->set_address(ipv6_only_ ? "::1" : "127.0.0.1");
9981   socket_address->set_port_value(backends_[0]->port());
9982   auto* filter_chain = listener.add_filter_chains();
9983   filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9984       HttpConnectionManager());
9985   // Since we don't know which port will be used by the channel, just add all
9986   // ports except for 0.
9987   for (int i = 1; i < 65536; i++) {
9988     filter_chain->mutable_filter_chain_match()->add_source_ports(i);
9989   }
9990   // Add another filter chain with no source port mentioned with a bad
9991   // DownstreamTlsContext configuration.
9992   filter_chain = listener.add_filter_chains();
9993   filter_chain->add_filters()->mutable_typed_config()->PackFrom(
9994       HttpConnectionManager());
9995   auto* transport_socket = filter_chain->mutable_transport_socket();
9996   transport_socket->set_name("envoy.transport_sockets.tls");
9997   DownstreamTlsContext downstream_tls_context;
9998   downstream_tls_context.mutable_common_tls_context()
9999       ->mutable_tls_certificate_certificate_provider_instance()
10000       ->set_instance_name("fake_plugin1");
10001   transport_socket->mutable_typed_config()->PackFrom(downstream_tls_context);
10002   balancers_[0]->ads_service()->SetLdsResource(listener);
10003   // A successful RPC proves that the filter chain with matching source port
10004   // was chosen.
10005   SendRpc([this]() { return CreateInsecureChannel(); }, {}, {});
10006 }
10007
10008 TEST_P(XdsServerFilterChainMatchTest, DuplicateMatchNacked) {
10009   Listener listener;
10010   listener.set_name(
10011       absl::StrCat("grpc/server?xds.resource.listening_address=",
10012                    ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()));
10013   auto* socket_address = listener.mutable_address()->mutable_socket_address();
10014   socket_address->set_address(ipv6_only_ ? "::1" : "127.0.0.1");
10015   socket_address->set_port_value(backends_[0]->port());
10016   // Add filter chain
10017   auto* filter_chain = listener.add_filter_chains();
10018   filter_chain->add_filters()->mutable_typed_config()->PackFrom(
10019       HttpConnectionManager());
10020   // Add a duplicate filter chain
10021   filter_chain = listener.add_filter_chains();
10022   filter_chain->add_filters()->mutable_typed_config()->PackFrom(
10023       HttpConnectionManager());
10024   balancers_[0]->ads_service()->SetLdsResource(listener);
10025   ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED))
10026       << "timed out waiting for NACK";
10027   EXPECT_THAT(
10028       balancers_[0]->ads_service()->lds_response_state().error_message,
10029       ::testing::HasSubstr(
10030           "Duplicate matching rules detected when adding filter chain: {}"));
10031 }
10032
10033 TEST_P(XdsServerFilterChainMatchTest, DuplicateMatchOnPrefixRangesNacked) {
10034   Listener listener;
10035   listener.set_name(
10036       absl::StrCat("grpc/server?xds.resource.listening_address=",
10037                    ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()));
10038   auto* socket_address = listener.mutable_address()->mutable_socket_address();
10039   socket_address->set_address(ipv6_only_ ? "::1" : "127.0.0.1");
10040   socket_address->set_port_value(backends_[0]->port());
10041   // Add filter chain with prefix range
10042   auto* filter_chain = listener.add_filter_chains();
10043   filter_chain->add_filters()->mutable_typed_config()->PackFrom(
10044       HttpConnectionManager());
10045   auto* prefix_range =
10046       filter_chain->mutable_filter_chain_match()->add_prefix_ranges();
10047   prefix_range->set_address_prefix(ipv6_only_ ? "::1" : "127.0.0.1");
10048   prefix_range->mutable_prefix_len()->set_value(16);
10049   prefix_range =
10050       filter_chain->mutable_filter_chain_match()->add_prefix_ranges();
10051   prefix_range->set_address_prefix(ipv6_only_ ? "::1" : "127.0.0.1");
10052   prefix_range->mutable_prefix_len()->set_value(24);
10053   // Add a filter chain with a duplicate prefix range entry
10054   filter_chain = listener.add_filter_chains();
10055   filter_chain->add_filters()->mutable_typed_config()->PackFrom(
10056       HttpConnectionManager());
10057   prefix_range =
10058       filter_chain->mutable_filter_chain_match()->add_prefix_ranges();
10059   prefix_range->set_address_prefix(ipv6_only_ ? "::1" : "127.0.0.1");
10060   prefix_range->mutable_prefix_len()->set_value(16);
10061   prefix_range =
10062       filter_chain->mutable_filter_chain_match()->add_prefix_ranges();
10063   prefix_range->set_address_prefix(ipv6_only_ ? "::1" : "127.0.0.1");
10064   prefix_range->mutable_prefix_len()->set_value(32);
10065   balancers_[0]->ads_service()->SetLdsResource(listener);
10066   ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED))
10067       << "timed out waiting for NACK";
10068   if (ipv6_only_) {
10069     EXPECT_THAT(
10070         balancers_[0]->ads_service()->lds_response_state().error_message,
10071         ::testing::HasSubstr(
10072             "Duplicate matching rules detected when adding filter chain: "
10073             "{prefix_ranges={{address_prefix=[::]:0, prefix_len=16}, "
10074             "{address_prefix=[::]:0, prefix_len=32}}}"));
10075   } else {
10076     EXPECT_THAT(
10077         balancers_[0]->ads_service()->lds_response_state().error_message,
10078         ::testing::HasSubstr(
10079             "Duplicate matching rules detected when adding filter chain: "
10080             "{prefix_ranges={{address_prefix=127.0.0.0:0, prefix_len=16}, "
10081             "{address_prefix=127.0.0.1:0, prefix_len=32}}}"));
10082   }
10083 }
10084
10085 TEST_P(XdsServerFilterChainMatchTest, DuplicateMatchOnTransportProtocolNacked) {
10086   Listener listener;
10087   listener.set_name(
10088       absl::StrCat("grpc/server?xds.resource.listening_address=",
10089                    ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()));
10090   auto* socket_address = listener.mutable_address()->mutable_socket_address();
10091   socket_address->set_address(ipv6_only_ ? "::1" : "127.0.0.1");
10092   socket_address->set_port_value(backends_[0]->port());
10093   // Add filter chain with "raw_buffer" transport protocol
10094   auto* filter_chain = listener.add_filter_chains();
10095   filter_chain->add_filters()->mutable_typed_config()->PackFrom(
10096       HttpConnectionManager());
10097   filter_chain->mutable_filter_chain_match()->set_transport_protocol(
10098       "raw_buffer");
10099   // Add a duplicate filter chain with the same "raw_buffer" transport
10100   // protocol entry
10101   filter_chain = listener.add_filter_chains();
10102   filter_chain->add_filters()->mutable_typed_config()->PackFrom(
10103       HttpConnectionManager());
10104   filter_chain->mutable_filter_chain_match()->set_transport_protocol(
10105       "raw_buffer");
10106   balancers_[0]->ads_service()->SetLdsResource(listener);
10107   ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED))
10108       << "timed out waiting for NACK";
10109   EXPECT_THAT(
10110       balancers_[0]->ads_service()->lds_response_state().error_message,
10111       ::testing::HasSubstr("Duplicate matching rules detected when adding "
10112                            "filter chain: {transport_protocol=raw_buffer}"));
10113 }
10114
10115 TEST_P(XdsServerFilterChainMatchTest, DuplicateMatchOnLocalSourceTypeNacked) {
10116   Listener listener;
10117   listener.set_name(
10118       absl::StrCat("grpc/server?xds.resource.listening_address=",
10119                    ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()));
10120   auto* socket_address = listener.mutable_address()->mutable_socket_address();
10121   socket_address->set_address(ipv6_only_ ? "::1" : "127.0.0.1");
10122   socket_address->set_port_value(backends_[0]->port());
10123   // Add filter chain with the local source type
10124   auto* filter_chain = listener.add_filter_chains();
10125   filter_chain->add_filters()->mutable_typed_config()->PackFrom(
10126       HttpConnectionManager());
10127   filter_chain->mutable_filter_chain_match()->set_source_type(
10128       FilterChainMatch::SAME_IP_OR_LOOPBACK);
10129   // Add a duplicate filter chain with the same local source type entry
10130   filter_chain = listener.add_filter_chains();
10131   filter_chain->add_filters()->mutable_typed_config()->PackFrom(
10132       HttpConnectionManager());
10133   filter_chain->mutable_filter_chain_match()->set_source_type(
10134       FilterChainMatch::SAME_IP_OR_LOOPBACK);
10135   balancers_[0]->ads_service()->SetLdsResource(listener);
10136   ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED))
10137       << "timed out waiting for NACK";
10138   EXPECT_THAT(
10139       balancers_[0]->ads_service()->lds_response_state().error_message,
10140       ::testing::HasSubstr("Duplicate matching rules detected when adding "
10141                            "filter chain: {source_type=SAME_IP_OR_LOOPBACK}"));
10142 }
10143
10144 TEST_P(XdsServerFilterChainMatchTest,
10145        DuplicateMatchOnExternalSourceTypeNacked) {
10146   Listener listener;
10147   listener.set_name(
10148       absl::StrCat("grpc/server?xds.resource.listening_address=",
10149                    ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()));
10150   auto* socket_address = listener.mutable_address()->mutable_socket_address();
10151   socket_address->set_address(ipv6_only_ ? "::1" : "127.0.0.1");
10152   socket_address->set_port_value(backends_[0]->port());
10153   // Add filter chain with the external source type
10154   auto* filter_chain = listener.add_filter_chains();
10155   filter_chain->add_filters()->mutable_typed_config()->PackFrom(
10156       HttpConnectionManager());
10157   filter_chain->mutable_filter_chain_match()->set_source_type(
10158       FilterChainMatch::EXTERNAL);
10159   // Add a duplicate filter chain with the same external source type entry
10160   filter_chain = listener.add_filter_chains();
10161   filter_chain->add_filters()->mutable_typed_config()->PackFrom(
10162       HttpConnectionManager());
10163   filter_chain->mutable_filter_chain_match()->set_source_type(
10164       FilterChainMatch::EXTERNAL);
10165   balancers_[0]->ads_service()->SetLdsResource(listener);
10166   ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED))
10167       << "timed out waiting for NACK";
10168   EXPECT_THAT(
10169       balancers_[0]->ads_service()->lds_response_state().error_message,
10170       ::testing::HasSubstr("Duplicate matching rules detected when adding "
10171                            "filter chain: {source_type=EXTERNAL}"));
10172 }
10173
10174 TEST_P(XdsServerFilterChainMatchTest,
10175        DuplicateMatchOnSourcePrefixRangesNacked) {
10176   Listener listener;
10177   listener.set_name(
10178       absl::StrCat("grpc/server?xds.resource.listening_address=",
10179                    ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()));
10180   auto* socket_address = listener.mutable_address()->mutable_socket_address();
10181   socket_address->set_address(ipv6_only_ ? "::1" : "127.0.0.1");
10182   socket_address->set_port_value(backends_[0]->port());
10183   // Add filter chain with source prefix range
10184   auto* filter_chain = listener.add_filter_chains();
10185   filter_chain->add_filters()->mutable_typed_config()->PackFrom(
10186       HttpConnectionManager());
10187   auto* prefix_range =
10188       filter_chain->mutable_filter_chain_match()->add_source_prefix_ranges();
10189   prefix_range->set_address_prefix(ipv6_only_ ? "::1" : "127.0.0.1");
10190   prefix_range->mutable_prefix_len()->set_value(16);
10191   prefix_range =
10192       filter_chain->mutable_filter_chain_match()->add_source_prefix_ranges();
10193   prefix_range->set_address_prefix(ipv6_only_ ? "::1" : "127.0.0.1");
10194   prefix_range->mutable_prefix_len()->set_value(24);
10195   // Add a filter chain with a duplicate source prefix range entry
10196   filter_chain = listener.add_filter_chains();
10197   filter_chain->add_filters()->mutable_typed_config()->PackFrom(
10198       HttpConnectionManager());
10199   prefix_range =
10200       filter_chain->mutable_filter_chain_match()->add_source_prefix_ranges();
10201   prefix_range->set_address_prefix(ipv6_only_ ? "::1" : "127.0.0.1");
10202   prefix_range->mutable_prefix_len()->set_value(16);
10203   prefix_range =
10204       filter_chain->mutable_filter_chain_match()->add_source_prefix_ranges();
10205   prefix_range->set_address_prefix(ipv6_only_ ? "::1" : "127.0.0.1");
10206   prefix_range->mutable_prefix_len()->set_value(32);
10207   balancers_[0]->ads_service()->SetLdsResource(listener);
10208   ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED))
10209       << "timed out waiting for NACK";
10210   if (ipv6_only_) {
10211     EXPECT_THAT(
10212         balancers_[0]->ads_service()->lds_response_state().error_message,
10213         ::testing::HasSubstr(
10214             "Duplicate matching rules detected when adding filter chain: "
10215             "{source_prefix_ranges={{address_prefix=[::]:0, prefix_len=16}, "
10216             "{address_prefix=[::]:0, prefix_len=32}}}"));
10217   } else {
10218     EXPECT_THAT(
10219         balancers_[0]->ads_service()->lds_response_state().error_message,
10220         ::testing::HasSubstr(
10221             "Duplicate matching rules detected when adding filter chain: "
10222             "{source_prefix_ranges={{address_prefix=127.0.0.0:0, "
10223             "prefix_len=16}, "
10224             "{address_prefix=127.0.0.1:0, prefix_len=32}}}"));
10225   }
10226 }
10227
10228 TEST_P(XdsServerFilterChainMatchTest, DuplicateMatchOnSourcePortNacked) {
10229   Listener listener;
10230   listener.set_name(
10231       absl::StrCat("grpc/server?xds.resource.listening_address=",
10232                    ipv6_only_ ? "[::1]:" : "127.0.0.1:", backends_[0]->port()));
10233   auto* socket_address = listener.mutable_address()->mutable_socket_address();
10234   socket_address->set_address(ipv6_only_ ? "::1" : "127.0.0.1");
10235   socket_address->set_port_value(backends_[0]->port());
10236   // Add filter chain with the external source type
10237   auto* filter_chain = listener.add_filter_chains();
10238   filter_chain->add_filters()->mutable_typed_config()->PackFrom(
10239       HttpConnectionManager());
10240   filter_chain->mutable_filter_chain_match()->add_source_ports(8080);
10241   // Add a duplicate filter chain with the same source port entry
10242   filter_chain = listener.add_filter_chains();
10243   filter_chain->add_filters()->mutable_typed_config()->PackFrom(
10244       HttpConnectionManager());
10245   filter_chain->mutable_filter_chain_match()->add_source_ports(8080);
10246   balancers_[0]->ads_service()->SetLdsResource(listener);
10247   ASSERT_TRUE(WaitForLdsNack(StatusCode::DEADLINE_EXCEEDED))
10248       << "timed out waiting for NACK";
10249   EXPECT_THAT(
10250       balancers_[0]->ads_service()->lds_response_state().error_message,
10251       ::testing::HasSubstr("Duplicate matching rules detected when adding "
10252                            "filter chain: {source_ports={8080}}"));
10253 }
10254
10255 using EdsTest = BasicTest;
10256
10257 // Tests that EDS client should send a NACK if the EDS update contains
10258 // sparse priorities.
10259 TEST_P(EdsTest, NacksSparsePriorityList) {
10260   SetNextResolution({});
10261   SetNextResolutionForLbChannelAllBalancers();
10262   AdsServiceImpl::EdsResourceArgs args({
10263       {"locality0", CreateEndpointsForBackends(), kDefaultLocalityWeight, 1},
10264   });
10265   balancers_[0]->ads_service()->SetEdsResource(BuildEdsResource(args));
10266   ASSERT_TRUE(WaitForEdsNack()) << "timed out waiting for NACK";
10267   const auto response_state =
10268       balancers_[0]->ads_service()->eds_response_state();
10269   EXPECT_EQ(response_state.state, AdsServiceImpl::ResponseState::NACKED);
10270   EXPECT_THAT(response_state.error_message,
10271               ::testing::HasSubstr("sparse priority list"));
10272 }
10273
10274 // In most of our tests, we use different names for different resource
10275 // types, to make sure that there are no cut-and-paste errors in the code
10276 // that cause us to look at data for the wrong resource type.  So we add
10277 // this test to make sure that the EDS resource name defaults to the
10278 // cluster name if not specified in the CDS resource.
10279 TEST_P(EdsTest, EdsServiceNameDefaultsToClusterName) {
10280   AdsServiceImpl::EdsResourceArgs args({
10281       {"locality0", CreateEndpointsForBackends()},
10282   });
10283   balancers_[0]->ads_service()->SetEdsResource(
10284       BuildEdsResource(args, kDefaultClusterName));
10285   Cluster cluster = default_cluster_;
10286   cluster.mutable_eds_cluster_config()->clear_service_name();
10287   balancers_[0]->ads_service()->SetCdsResource(cluster);
10288   SetNextResolution({});
10289   SetNextResolutionForLbChannelAllBalancers();
10290   CheckRpcSendOk();
10291 }
10292
10293 class TimeoutTest : public BasicTest {
10294  protected:
10295   void SetUp() override {
10296     xds_resource_does_not_exist_timeout_ms_ = 500;
10297     BasicTest::SetUp();
10298   }
10299 };
10300
10301 // Tests that LDS client times out when no response received.
10302 TEST_P(TimeoutTest, Lds) {
10303   balancers_[0]->ads_service()->SetResourceIgnore(kLdsTypeUrl);
10304   SetNextResolution({});
10305   SetNextResolutionForLbChannelAllBalancers();
10306   CheckRpcSendFailure();
10307 }
10308
10309 TEST_P(TimeoutTest, Rds) {
10310   balancers_[0]->ads_service()->SetResourceIgnore(kRdsTypeUrl);
10311   SetNextResolution({});
10312   SetNextResolutionForLbChannelAllBalancers();
10313   CheckRpcSendFailure();
10314 }
10315
10316 // Tests that CDS client times out when no response received.
10317 TEST_P(TimeoutTest, Cds) {
10318   balancers_[0]->ads_service()->SetResourceIgnore(kCdsTypeUrl);
10319   SetNextResolution({});
10320   SetNextResolutionForLbChannelAllBalancers();
10321   CheckRpcSendFailure();
10322 }
10323
10324 TEST_P(TimeoutTest, Eds) {
10325   balancers_[0]->ads_service()->SetResourceIgnore(kEdsTypeUrl);
10326   SetNextResolution({});
10327   SetNextResolutionForLbChannelAllBalancers();
10328   CheckRpcSendFailure();
10329 }
10330
10331 using LocalityMapTest = BasicTest;
10332
10333 // Tests that the localities in a locality map are picked according to their
10334 // weights.
10335 TEST_P(LocalityMapTest, WeightedRoundRobin) {
10336   SetNextResolution({});
10337   SetNextResolutionForLbChannelAllBalancers();
10338   const int kLocalityWeight0 = 2;
10339   const int kLocalityWeight1 = 8;
10340   const int kTotalLocalityWeight = kLocalityWeight0 + kLocalityWeight1;
10341   const double kLocalityWeightRate0 =
10342       static_cast<double>(kLocalityWeight0) / kTotalLocalityWeight;
10343   const double kLocalityWeightRate1 =
10344       static_cast<double>(kLocalityWeight1) / kTotalLocalityWeight;
10345   const double kErrorTolerance = 0.05;
10346   const size_t kNumRpcs =
10347       ComputeIdealNumRpcs(kLocalityWeightRate0, kErrorTolerance);
10348   // ADS response contains 2 localities, each of which contains 1 backend.
10349   AdsServiceImpl::EdsResourceArgs args({
10350       {"locality0", CreateEndpointsForBackends(0, 1), kLocalityWeight0},
10351       {"locality1", CreateEndpointsForBackends(1, 2), kLocalityWeight1},
10352   });
10353   balancers_[0]->ads_service()->SetEdsResource(
10354       BuildEdsResource(args, DefaultEdsServiceName()));
10355   // Wait for both backends to be ready.
10356   WaitForAllBackends(0, 2);
10357   // Send kNumRpcs RPCs.
10358   CheckRpcSendOk(kNumRpcs);
10359   // The locality picking rates should be roughly equal to the expectation.
10360   const double locality_picked_rate_0 =
10361       static_cast<double>(backends_[0]->backend_service()->request_count()) /
10362       kNumRpcs;
10363   const double locality_picked_rate_1 =
10364       static_cast<double>(backends_[1]->backend_service()->request_count()) /
10365       kNumRpcs;
10366   EXPECT_THAT(locality_picked_rate_0,
10367               ::testing::DoubleNear(kLocalityWeightRate0, kErrorTolerance));
10368   EXPECT_THAT(locality_picked_rate_1,
10369               ::testing::DoubleNear(kLocalityWeightRate1, kErrorTolerance));
10370 }
10371
10372 // Tests that we correctly handle a locality containing no endpoints.
10373 TEST_P(LocalityMapTest, LocalityContainingNoEndpoints) {
10374   SetNextResolution({});
10375   SetNextResolutionForLbChannelAllBalancers();
10376   const size_t kNumRpcs = 5000;
10377   // EDS response contains 2 localities, one with no endpoints.
10378   AdsServiceImpl::EdsResourceArgs args({
10379       {"locality0", CreateEndpointsForBackends()},
10380       {"locality1", {}},
10381   });
10382   balancers_[0]->ads_service()->SetEdsResource(
10383       BuildEdsResource(args, DefaultEdsServiceName()));
10384   // Wait for both backends to be ready.
10385   WaitForAllBackends();
10386   // Send kNumRpcs RPCs.
10387   CheckRpcSendOk(kNumRpcs);
10388   // All traffic should go to the reachable locality.
10389   EXPECT_EQ(backends_[0]->backend_service()->request_count(),
10390             kNumRpcs / backends_.size());
10391   EXPECT_EQ(backends_[1]->backend_service()->request_count(),
10392             kNumRpcs / backends_.size());
10393   EXPECT_EQ(backends_[2]->backend_service()->request_count(),
10394             kNumRpcs / backends_.size());
10395   EXPECT_EQ(backends_[3]->backend_service()->request_count(),
10396             kNumRpcs / backends_.size());
10397 }
10398
10399 // EDS update with no localities.
10400 TEST_P(LocalityMapTest, NoLocalities) {
10401   SetNextResolution({});
10402   SetNextResolutionForLbChannelAllBalancers();
10403   balancers_[0]->ads_service()->SetEdsResource(
10404       BuildEdsResource({}, DefaultEdsServiceName()));
10405   Status status = SendRpc();
10406   EXPECT_FALSE(status.ok());
10407   EXPECT_EQ(status.error_code(), StatusCode::UNAVAILABLE);
10408 }
10409
10410 // Tests that the locality map can work properly even when it contains a large
10411 // number of localities.
10412 TEST_P(LocalityMapTest, StressTest) {
10413   SetNextResolution({});
10414   SetNextResolutionForLbChannelAllBalancers();
10415   const size_t kNumLocalities = 100;
10416   const uint32_t kRpcTimeoutMs = 5000;
10417   // The first ADS response contains kNumLocalities localities, each of which
10418   // contains backend 0.
10419   AdsServiceImpl::EdsResourceArgs args;
10420   for (size_t i = 0; i < kNumLocalities; ++i) {
10421     std::string name = absl::StrCat("locality", i);
10422     AdsServiceImpl::EdsResourceArgs::Locality locality(
10423         name, CreateEndpointsForBackends(0, 1));
10424     args.locality_list.emplace_back(std::move(locality));
10425   }
10426   balancers_[0]->ads_service()->SetEdsResource(
10427       BuildEdsResource(args, DefaultEdsServiceName()));
10428   // The second ADS response contains 1 locality, which contains backend 1.
10429   args = AdsServiceImpl::EdsResourceArgs({
10430       {"locality0", CreateEndpointsForBackends(1, 2)},
10431   });
10432   std::thread delayed_resource_setter(
10433       std::bind(&BasicTest::SetEdsResourceWithDelay, this, 0,
10434                 BuildEdsResource(args, DefaultEdsServiceName()), 60 * 1000));
10435   // Wait until backend 0 is ready, before which kNumLocalities localities are
10436   // received and handled by the xds policy.
10437   WaitForBackend(0, WaitForBackendOptions().set_reset_counters(false),
10438                  RpcOptions().set_timeout_ms(kRpcTimeoutMs));
10439   EXPECT_EQ(0U, backends_[1]->backend_service()->request_count());
10440   // Wait until backend 1 is ready, before which kNumLocalities localities are
10441   // removed by the xds policy.
10442   WaitForBackend(1);
10443   delayed_resource_setter.join();
10444 }
10445
10446 // Tests that the localities in a locality map are picked correctly after
10447 // update (addition, modification, deletion).
10448 TEST_P(LocalityMapTest, UpdateMap) {
10449   SetNextResolution({});
10450   SetNextResolutionForLbChannelAllBalancers();
10451   const size_t kNumRpcs = 3000;
10452   // The locality weight for the first 3 localities.
10453   const std::vector<int> kLocalityWeights0 = {2, 3, 4};
10454   const double kTotalLocalityWeight0 =
10455       std::accumulate(kLocalityWeights0.begin(), kLocalityWeights0.end(), 0);
10456   std::vector<double> locality_weight_rate_0;
10457   locality_weight_rate_0.reserve(kLocalityWeights0.size());
10458   for (int weight : kLocalityWeights0) {
10459     locality_weight_rate_0.push_back(weight / kTotalLocalityWeight0);
10460   }
10461   // Delete the first locality, keep the second locality, change the third
10462   // locality's weight from 4 to 2, and add a new locality with weight 6.
10463   const std::vector<int> kLocalityWeights1 = {3, 2, 6};
10464   const double kTotalLocalityWeight1 =
10465       std::accumulate(kLocalityWeights1.begin(), kLocalityWeights1.end(), 0);
10466   std::vector<double> locality_weight_rate_1 = {
10467       0 /* placeholder for locality 0 */};
10468   for (int weight : kLocalityWeights1) {
10469     locality_weight_rate_1.push_back(weight / kTotalLocalityWeight1);
10470   }
10471   AdsServiceImpl::EdsResourceArgs args({
10472       {"locality0", CreateEndpointsForBackends(0, 1), 2},
10473       {"locality1", CreateEndpointsForBackends(1, 2), 3},
10474       {"locality2", CreateEndpointsForBackends(2, 3), 4},
10475   });
10476   balancers_[0]->ads_service()->SetEdsResource(
10477       BuildEdsResource(args, DefaultEdsServiceName()));
10478   // Wait for the first 3 backends to be ready.
10479   WaitForAllBackends(0, 3);
10480   gpr_log(GPR_INFO, "========= BEFORE FIRST BATCH ==========");
10481   // Send kNumRpcs RPCs.
10482   CheckRpcSendOk(kNumRpcs);
10483   gpr_log(GPR_INFO, "========= DONE WITH FIRST BATCH ==========");
10484   // The picking rates of the first 3 backends should be roughly equal to the
10485   // expectation.
10486   std::vector<double> locality_picked_rates;
10487   for (size_t i = 0; i < 3; ++i) {
10488     locality_picked_rates.push_back(
10489         static_cast<double>(backends_[i]->backend_service()->request_count()) /
10490         kNumRpcs);
10491   }
10492   const double kErrorTolerance = 0.2;
10493   for (size_t i = 0; i < 3; ++i) {
10494     gpr_log(GPR_INFO, "Locality %" PRIuPTR " rate %f", i,
10495             locality_picked_rates[i]);
10496     EXPECT_THAT(
10497         locality_picked_rates[i],
10498         ::testing::AllOf(
10499             ::testing::Ge(locality_weight_rate_0[i] * (1 - kErrorTolerance)),
10500             ::testing::Le(locality_weight_rate_0[i] * (1 + kErrorTolerance))));
10501   }
10502   args = AdsServiceImpl::EdsResourceArgs({
10503       {"locality1", CreateEndpointsForBackends(1, 2), 3},
10504       {"locality2", CreateEndpointsForBackends(2, 3), 2},
10505       {"locality3", CreateEndpointsForBackends(3, 4), 6},
10506   });
10507   balancers_[0]->ads_service()->SetEdsResource(
10508       BuildEdsResource(args, DefaultEdsServiceName()));
10509   // Backend 3 hasn't received any request.
10510   EXPECT_EQ(0U, backends_[3]->backend_service()->request_count());
10511   // Wait until the locality update has been processed, as signaled by backend
10512   // 3 receiving a request.
10513   WaitForAllBackends(3, 4);
10514   gpr_log(GPR_INFO, "========= BEFORE SECOND BATCH ==========");
10515   // Send kNumRpcs RPCs.
10516   CheckRpcSendOk(kNumRpcs);
10517   gpr_log(GPR_INFO, "========= DONE WITH SECOND BATCH ==========");
10518   // Backend 0 no longer receives any request.
10519   EXPECT_EQ(0U, backends_[0]->backend_service()->request_count());
10520   // The picking rates of the last 3 backends should be roughly equal to the
10521   // expectation.
10522   locality_picked_rates = {0 /* placeholder for backend 0 */};
10523   for (size_t i = 1; i < 4; ++i) {
10524     locality_picked_rates.push_back(
10525         static_cast<double>(backends_[i]->backend_service()->request_count()) /
10526         kNumRpcs);
10527   }
10528   for (size_t i = 1; i < 4; ++i) {
10529     gpr_log(GPR_INFO, "Locality %" PRIuPTR " rate %f", i,
10530             locality_picked_rates[i]);
10531     EXPECT_THAT(
10532         locality_picked_rates[i],
10533         ::testing::AllOf(
10534             ::testing::Ge(locality_weight_rate_1[i] * (1 - kErrorTolerance)),
10535             ::testing::Le(locality_weight_rate_1[i] * (1 + kErrorTolerance))));
10536   }
10537 }
10538
10539 // Tests that we don't fail RPCs when replacing all of the localities in
10540 // a given priority.
10541 TEST_P(LocalityMapTest, ReplaceAllLocalitiesInPriority) {
10542   SetNextResolution({});
10543   SetNextResolutionForLbChannelAllBalancers();
10544   AdsServiceImpl::EdsResourceArgs args({
10545       {"locality0", CreateEndpointsForBackends(0, 1)},
10546   });
10547   balancers_[0]->ads_service()->SetEdsResource(
10548       BuildEdsResource(args, DefaultEdsServiceName()));
10549   args = AdsServiceImpl::EdsResourceArgs({
10550       {"locality1", CreateEndpointsForBackends(1, 2)},
10551   });
10552   std::thread delayed_resource_setter(
10553       std::bind(&BasicTest::SetEdsResourceWithDelay, this, 0,
10554                 BuildEdsResource(args, DefaultEdsServiceName()), 5000));
10555   // Wait for the first backend to be ready.
10556   WaitForBackend(0);
10557   // Keep sending RPCs until we switch over to backend 1, which tells us
10558   // that we received the update.  No RPCs should fail during this
10559   // transition.
10560   WaitForBackend(1);
10561   delayed_resource_setter.join();
10562 }
10563
10564 class FailoverTest : public BasicTest {
10565  public:
10566   void SetUp() override {
10567     BasicTest::SetUp();
10568     ResetStub(500);
10569   }
10570 };
10571
10572 // Localities with the highest priority are used when multiple priority exist.
10573 TEST_P(FailoverTest, ChooseHighestPriority) {
10574   SetNextResolution({});
10575   SetNextResolutionForLbChannelAllBalancers();
10576   AdsServiceImpl::EdsResourceArgs args({
10577       {"locality0", CreateEndpointsForBackends(0, 1), kDefaultLocalityWeight,
10578        1},
10579       {"locality1", CreateEndpointsForBackends(1, 2), kDefaultLocalityWeight,
10580        2},
10581       {"locality2", CreateEndpointsForBackends(2, 3), kDefaultLocalityWeight,
10582        3},
10583       {"locality3", CreateEndpointsForBackends(3, 4), kDefaultLocalityWeight,
10584        0},
10585   });
10586   balancers_[0]->ads_service()->SetEdsResource(
10587       BuildEdsResource(args, DefaultEdsServiceName()));
10588   WaitForBackend(3, WaitForBackendOptions().set_reset_counters(false));
10589   for (size_t i = 0; i < 3; ++i) {
10590     EXPECT_EQ(0U, backends_[i]->backend_service()->request_count());
10591   }
10592 }
10593
10594 // Does not choose priority with no endpoints.
10595 TEST_P(FailoverTest, DoesNotUsePriorityWithNoEndpoints) {
10596   SetNextResolution({});
10597   SetNextResolutionForLbChannelAllBalancers();
10598   AdsServiceImpl::EdsResourceArgs args({
10599       {"locality0", CreateEndpointsForBackends(0, 1), kDefaultLocalityWeight,
10600        1},
10601       {"locality1", CreateEndpointsForBackends(1, 2), kDefaultLocalityWeight,
10602        2},
10603       {"locality2", CreateEndpointsForBackends(2, 3), kDefaultLocalityWeight,
10604        3},
10605       {"locality3", {}, kDefaultLocalityWeight, 0},
10606   });
10607   balancers_[0]->ads_service()->SetEdsResource(
10608       BuildEdsResource(args, DefaultEdsServiceName()));
10609   WaitForBackend(0, WaitForBackendOptions().set_reset_counters(false));
10610   for (size_t i = 1; i < 3; ++i) {
10611     EXPECT_EQ(0U, backends_[i]->backend_service()->request_count());
10612   }
10613 }
10614
10615 // Does not choose locality with no endpoints.
10616 TEST_P(FailoverTest, DoesNotUseLocalityWithNoEndpoints) {
10617   SetNextResolution({});
10618   SetNextResolutionForLbChannelAllBalancers();
10619   AdsServiceImpl::EdsResourceArgs args({
10620       {"locality0", {}, kDefaultLocalityWeight, 0},
10621       {"locality1", CreateEndpointsForBackends(), kDefaultLocalityWeight, 0},
10622   });
10623   balancers_[0]->ads_service()->SetEdsResource(
10624       BuildEdsResource(args, DefaultEdsServiceName()));
10625   // Wait for all backends to be used.
10626   std::tuple<int, int, int> counts = WaitForAllBackends();
10627   // Make sure no RPCs failed in the transition.
10628   EXPECT_EQ(0, std::get<1>(counts));
10629 }
10630
10631 // If the higher priority localities are not reachable, failover to the
10632 // highest priority among the rest.
10633 TEST_P(FailoverTest, Failover) {
10634   SetNextResolution({});
10635   SetNextResolutionForLbChannelAllBalancers();
10636   AdsServiceImpl::EdsResourceArgs args({
10637       {"locality0", CreateEndpointsForBackends(0, 1), kDefaultLocalityWeight,
10638        1},
10639       {"locality1", CreateEndpointsForBackends(1, 2), kDefaultLocalityWeight,
10640        2},
10641       {"locality2", CreateEndpointsForBackends(2, 3), kDefaultLocalityWeight,
10642        3},
10643       {"locality3", CreateEndpointsForBackends(3, 4), kDefaultLocalityWeight,
10644        0},
10645   });
10646   ShutdownBackend(3);
10647   ShutdownBackend(0);
10648   balancers_[0]->ads_service()->SetEdsResource(
10649       BuildEdsResource(args, DefaultEdsServiceName()));
10650   WaitForBackend(1, WaitForBackendOptions().set_reset_counters(false));
10651   for (size_t i = 0; i < 4; ++i) {
10652     if (i == 1) continue;
10653     EXPECT_EQ(0U, backends_[i]->backend_service()->request_count());
10654   }
10655 }
10656
10657 // If a locality with higher priority than the current one becomes ready,
10658 // switch to it.
10659 TEST_P(FailoverTest, SwitchBackToHigherPriority) {
10660   SetNextResolution({});
10661   SetNextResolutionForLbChannelAllBalancers();
10662   const size_t kNumRpcs = 100;
10663   AdsServiceImpl::EdsResourceArgs args({
10664       {"locality0", CreateEndpointsForBackends(0, 1), kDefaultLocalityWeight,
10665        1},
10666       {"locality1", CreateEndpointsForBackends(1, 2), kDefaultLocalityWeight,
10667        2},
10668       {"locality2", CreateEndpointsForBackends(2, 3), kDefaultLocalityWeight,
10669        3},
10670       {"locality3", CreateEndpointsForBackends(3, 4), kDefaultLocalityWeight,
10671        0},
10672   });
10673   balancers_[0]->ads_service()->SetEdsResource(
10674       BuildEdsResource(args, DefaultEdsServiceName()));
10675   WaitForBackend(3);
10676   ShutdownBackend(3);
10677   ShutdownBackend(0);
10678   WaitForBackend(
10679       1, WaitForBackendOptions().set_reset_counters(false).set_allow_failures(
10680              true));
10681   for (size_t i = 0; i < 4; ++i) {
10682     if (i == 1) continue;
10683     EXPECT_EQ(0U, backends_[i]->backend_service()->request_count());
10684   }
10685   StartBackend(0);
10686   WaitForBackend(0);
10687   CheckRpcSendOk(kNumRpcs);
10688   EXPECT_EQ(kNumRpcs, backends_[0]->backend_service()->request_count());
10689 }
10690
10691 // The first update only contains unavailable priorities. The second update
10692 // contains available priorities.
10693 TEST_P(FailoverTest, UpdateInitialUnavailable) {
10694   SetNextResolution({});
10695   SetNextResolutionForLbChannelAllBalancers();
10696   AdsServiceImpl::EdsResourceArgs args({
10697       {"locality0", CreateEndpointsForBackends(0, 1), kDefaultLocalityWeight,
10698        0},
10699       {"locality1", CreateEndpointsForBackends(1, 2), kDefaultLocalityWeight,
10700        1},
10701   });
10702   balancers_[0]->ads_service()->SetEdsResource(
10703       BuildEdsResource(args, DefaultEdsServiceName()));
10704   args = AdsServiceImpl::EdsResourceArgs({
10705       {"locality0", CreateEndpointsForBackends(0, 1), kDefaultLocalityWeight,
10706        0},
10707       {"locality1", CreateEndpointsForBackends(1, 2), kDefaultLocalityWeight,
10708        1},
10709       {"locality2", CreateEndpointsForBackends(2, 3), kDefaultLocalityWeight,
10710        2},
10711       {"locality3", CreateEndpointsForBackends(3, 4), kDefaultLocalityWeight,
10712        3},
10713   });
10714   ShutdownBackend(0);
10715   ShutdownBackend(1);
10716   std::thread delayed_resource_setter(
10717       std::bind(&BasicTest::SetEdsResourceWithDelay, this, 0,
10718                 BuildEdsResource(args, DefaultEdsServiceName()), 1000));
10719   gpr_timespec deadline = gpr_time_add(gpr_now(GPR_CLOCK_REALTIME),
10720                                        gpr_time_from_millis(500, GPR_TIMESPAN));
10721   // Send 0.5 second worth of RPCs.
10722   do {
10723     CheckRpcSendFailure();
10724   } while (gpr_time_cmp(gpr_now(GPR_CLOCK_REALTIME), deadline) < 0);
10725   WaitForBackend(
10726       2, WaitForBackendOptions().set_reset_counters(false).set_allow_failures(
10727              true));
10728   for (size_t i = 0; i < 4; ++i) {
10729     if (i == 2) continue;
10730     EXPECT_EQ(0U, backends_[i]->backend_service()->request_count());
10731   }
10732   delayed_resource_setter.join();
10733 }
10734
10735 // Tests that after the localities' priorities are updated, we still choose
10736 // the highest READY priority with the updated localities.
10737 TEST_P(FailoverTest, UpdatePriority) {
10738   SetNextResolution({});
10739   SetNextResolutionForLbChannelAllBalancers();
10740   const size_t kNumRpcs = 100;
10741   AdsServiceImpl::EdsResourceArgs args({
10742       {"locality0", CreateEndpointsForBackends(0, 1), kDefaultLocalityWeight,
10743        1},
10744       {"locality1", CreateEndpointsForBackends(1, 2), kDefaultLocalityWeight,
10745        2},
10746       {"locality2", CreateEndpointsForBackends(2, 3), kDefaultLocalityWeight,
10747        3},
10748       {"locality3", CreateEndpointsForBackends(3, 4), kDefaultLocalityWeight,
10749        0},
10750   });
10751   balancers_[0]->ads_service()->SetEdsResource(
10752       BuildEdsResource(args, DefaultEdsServiceName()));
10753   args = AdsServiceImpl::EdsResourceArgs({
10754       {"locality0", CreateEndpointsForBackends(0, 1), kDefaultLocalityWeight,
10755        2},
10756       {"locality1", CreateEndpointsForBackends(1, 2), kDefaultLocalityWeight,
10757        0},
10758       {"locality2", CreateEndpointsForBackends(2, 3), kDefaultLocalityWeight,
10759        1},
10760       {"locality3", CreateEndpointsForBackends(3, 4), kDefaultLocalityWeight,
10761        3},
10762   });
10763   std::thread delayed_resource_setter(
10764       std::bind(&BasicTest::SetEdsResourceWithDelay, this, 0,
10765                 BuildEdsResource(args, DefaultEdsServiceName()), 1000));
10766   WaitForBackend(3, WaitForBackendOptions().set_reset_counters(false));
10767   for (size_t i = 0; i < 3; ++i) {
10768     EXPECT_EQ(0U, backends_[i]->backend_service()->request_count());
10769   }
10770   WaitForBackend(1);
10771   CheckRpcSendOk(kNumRpcs);
10772   EXPECT_EQ(kNumRpcs, backends_[1]->backend_service()->request_count());
10773   delayed_resource_setter.join();
10774 }
10775
10776 // Moves all localities in the current priority to a higher priority.
10777 TEST_P(FailoverTest, MoveAllLocalitiesInCurrentPriorityToHigherPriority) {
10778   SetNextResolution({});
10779   SetNextResolutionForLbChannelAllBalancers();
10780   // First update:
10781   // - Priority 0 is locality 0, containing backend 0, which is down.
10782   // - Priority 1 is locality 1, containing backends 1 and 2, which are up.
10783   ShutdownBackend(0);
10784   AdsServiceImpl::EdsResourceArgs args({
10785       {"locality0", CreateEndpointsForBackends(0, 1), kDefaultLocalityWeight,
10786        0},
10787       {"locality1", CreateEndpointsForBackends(1, 3), kDefaultLocalityWeight,
10788        1},
10789   });
10790   balancers_[0]->ads_service()->SetEdsResource(
10791       BuildEdsResource(args, DefaultEdsServiceName()));
10792   // Second update:
10793   // - Priority 0 contains both localities 0 and 1.
10794   // - Priority 1 is not present.
10795   // - We add backend 3 to locality 1, just so we have a way to know
10796   //   when the update has been seen by the client.
10797   args = AdsServiceImpl::EdsResourceArgs({
10798       {"locality0", CreateEndpointsForBackends(0, 1), kDefaultLocalityWeight,
10799        0},
10800       {"locality1", CreateEndpointsForBackends(1, 4), kDefaultLocalityWeight,
10801        0},
10802   });
10803   std::thread delayed_resource_setter(
10804       std::bind(&BasicTest::SetEdsResourceWithDelay, this, 0,
10805                 BuildEdsResource(args, DefaultEdsServiceName()), 1000));
10806   // When we get the first update, all backends in priority 0 are down,
10807   // so we will create priority 1.  Backends 1 and 2 should have traffic,
10808   // but backend 3 should not.
10809   WaitForAllBackends(1, 3, WaitForBackendOptions().set_reset_counters(false));
10810   EXPECT_EQ(0UL, backends_[3]->backend_service()->request_count());
10811   // When backend 3 gets traffic, we know the second update has been seen.
10812   WaitForBackend(3);
10813   // The ADS service of balancer 0 got at least 1 response.
10814   EXPECT_GT(balancers_[0]->ads_service()->eds_response_state().state,
10815             AdsServiceImpl::ResponseState::NOT_SENT);
10816   delayed_resource_setter.join();
10817 }
10818
10819 using DropTest = BasicTest;
10820
10821 // Tests that RPCs are dropped according to the drop config.
10822 TEST_P(DropTest, Vanilla) {
10823   SetNextResolution({});
10824   SetNextResolutionForLbChannelAllBalancers();
10825   const uint32_t kDropPerMillionForLb = 100000;
10826   const uint32_t kDropPerMillionForThrottle = 200000;
10827   const double kDropRateForLb = kDropPerMillionForLb / 1000000.0;
10828   const double kDropRateForThrottle = kDropPerMillionForThrottle / 1000000.0;
10829   const double kDropRateForLbAndThrottle =
10830       kDropRateForLb + (1 - kDropRateForLb) * kDropRateForThrottle;
10831   const double kErrorTolerance = 0.05;
10832   const size_t kNumRpcs =
10833       ComputeIdealNumRpcs(kDropRateForLbAndThrottle, kErrorTolerance);
10834   // The ADS response contains two drop categories.
10835   AdsServiceImpl::EdsResourceArgs args({
10836       {"locality0", CreateEndpointsForBackends()},
10837   });
10838   args.drop_categories = {{kLbDropType, kDropPerMillionForLb},
10839                           {kThrottleDropType, kDropPerMillionForThrottle}};
10840   balancers_[0]->ads_service()->SetEdsResource(
10841       BuildEdsResource(args, DefaultEdsServiceName()));
10842   WaitForAllBackends();
10843   // Send kNumRpcs RPCs and count the drops.
10844   size_t num_drops = 0;
10845   for (size_t i = 0; i < kNumRpcs; ++i) {
10846     EchoResponse response;
10847     const Status status = SendRpc(RpcOptions(), &response);
10848     if (!status.ok() &&
10849         absl::StartsWith(status.error_message(), "EDS-configured drop: ")) {
10850       ++num_drops;
10851     } else {
10852       EXPECT_TRUE(status.ok()) << "code=" << status.error_code()
10853                                << " message=" << status.error_message();
10854       EXPECT_EQ(response.message(), kRequestMessage);
10855     }
10856   }
10857   // The drop rate should be roughly equal to the expectation.
10858   const double seen_drop_rate = static_cast<double>(num_drops) / kNumRpcs;
10859   EXPECT_THAT(seen_drop_rate, ::testing::DoubleNear(kDropRateForLbAndThrottle,
10860                                                     kErrorTolerance));
10861 }
10862
10863 // Tests that drop config is converted correctly from per hundred.
10864 TEST_P(DropTest, DropPerHundred) {
10865   SetNextResolution({});
10866   SetNextResolutionForLbChannelAllBalancers();
10867   const uint32_t kDropPerHundredForLb = 10;
10868   const double kDropRateForLb = kDropPerHundredForLb / 100.0;
10869   const double kErrorTolerance = 0.05;
10870   const size_t kNumRpcs = ComputeIdealNumRpcs(kDropRateForLb, kErrorTolerance);
10871   // The ADS response contains one drop category.
10872   AdsServiceImpl::EdsResourceArgs args({
10873       {"locality0", CreateEndpointsForBackends()},
10874   });
10875   args.drop_categories = {{kLbDropType, kDropPerHundredForLb}};
10876   args.drop_denominator = FractionalPercent::HUNDRED;
10877   balancers_[0]->ads_service()->SetEdsResource(
10878       BuildEdsResource(args, DefaultEdsServiceName()));
10879   WaitForAllBackends();
10880   // Send kNumRpcs RPCs and count the drops.
10881   size_t num_drops = 0;
10882   for (size_t i = 0; i < kNumRpcs; ++i) {
10883     EchoResponse response;
10884     const Status status = SendRpc(RpcOptions(), &response);
10885     if (!status.ok() &&
10886         absl::StartsWith(status.error_message(), "EDS-configured drop: ")) {
10887       ++num_drops;
10888     } else {
10889       EXPECT_TRUE(status.ok()) << "code=" << status.error_code()
10890                                << " message=" << status.error_message();
10891       EXPECT_EQ(response.message(), kRequestMessage);
10892     }
10893   }
10894   // The drop rate should be roughly equal to the expectation.
10895   const double seen_drop_rate = static_cast<double>(num_drops) / kNumRpcs;
10896   EXPECT_THAT(seen_drop_rate,
10897               ::testing::DoubleNear(kDropRateForLb, kErrorTolerance));
10898 }
10899
10900 // Tests that drop config is converted correctly from per ten thousand.
10901 TEST_P(DropTest, DropPerTenThousand) {
10902   SetNextResolution({});
10903   SetNextResolutionForLbChannelAllBalancers();
10904   const uint32_t kDropPerTenThousandForLb = 1000;
10905   const double kDropRateForLb = kDropPerTenThousandForLb / 10000.0;
10906   const double kErrorTolerance = 0.05;
10907   const size_t kNumRpcs = ComputeIdealNumRpcs(kDropRateForLb, kErrorTolerance);
10908   // The ADS response contains one drop category.
10909   AdsServiceImpl::EdsResourceArgs args({
10910       {"locality0", CreateEndpointsForBackends()},
10911   });
10912   args.drop_categories = {{kLbDropType, kDropPerTenThousandForLb}};
10913   args.drop_denominator = FractionalPercent::TEN_THOUSAND;
10914   balancers_[0]->ads_service()->SetEdsResource(
10915       BuildEdsResource(args, DefaultEdsServiceName()));
10916   WaitForAllBackends();
10917   // Send kNumRpcs RPCs and count the drops.
10918   size_t num_drops = 0;
10919   for (size_t i = 0; i < kNumRpcs; ++i) {
10920     EchoResponse response;
10921     const Status status = SendRpc(RpcOptions(), &response);
10922     if (!status.ok() &&
10923         absl::StartsWith(status.error_message(), "EDS-configured drop: ")) {
10924       ++num_drops;
10925     } else {
10926       EXPECT_TRUE(status.ok()) << "code=" << status.error_code()
10927                                << " message=" << status.error_message();
10928       EXPECT_EQ(response.message(), kRequestMessage);
10929     }
10930   }
10931   // The drop rate should be roughly equal to the expectation.
10932   const double seen_drop_rate = static_cast<double>(num_drops) / kNumRpcs;
10933   EXPECT_THAT(seen_drop_rate,
10934               ::testing::DoubleNear(kDropRateForLb, kErrorTolerance));
10935 }
10936
10937 // Tests that drop is working correctly after update.
10938 TEST_P(DropTest, Update) {
10939   SetNextResolution({});
10940   SetNextResolutionForLbChannelAllBalancers();
10941   const uint32_t kDropPerMillionForLb = 100000;
10942   const uint32_t kDropPerMillionForThrottle = 200000;
10943   const double kErrorTolerance = 0.05;
10944   const double kDropRateForLb = kDropPerMillionForLb / 1000000.0;
10945   const double kDropRateForThrottle = kDropPerMillionForThrottle / 1000000.0;
10946   const double kDropRateForLbAndThrottle =
10947       kDropRateForLb + (1 - kDropRateForLb) * kDropRateForThrottle;
10948   const size_t kNumRpcsLbOnly =
10949       ComputeIdealNumRpcs(kDropRateForLb, kErrorTolerance);
10950   const size_t kNumRpcsBoth =
10951       ComputeIdealNumRpcs(kDropRateForLbAndThrottle, kErrorTolerance);
10952   // The first ADS response contains one drop category.
10953   AdsServiceImpl::EdsResourceArgs args({
10954       {"locality0", CreateEndpointsForBackends()},
10955   });
10956   args.drop_categories = {{kLbDropType, kDropPerMillionForLb}};
10957   balancers_[0]->ads_service()->SetEdsResource(
10958       BuildEdsResource(args, DefaultEdsServiceName()));
10959   WaitForAllBackends();
10960   // Send kNumRpcsLbOnly RPCs and count the drops.
10961   size_t num_drops = 0;
10962   gpr_log(GPR_INFO, "========= BEFORE FIRST BATCH ==========");
10963   for (size_t i = 0; i < kNumRpcsLbOnly; ++i) {
10964     EchoResponse response;
10965     const Status status = SendRpc(RpcOptions(), &response);
10966     if (!status.ok() &&
10967         absl::StartsWith(status.error_message(), "EDS-configured drop: ")) {
10968       ++num_drops;
10969     } else {
10970       EXPECT_TRUE(status.ok()) << "code=" << status.error_code()
10971                                << " message=" << status.error_message();
10972       EXPECT_EQ(response.message(), kRequestMessage);
10973     }
10974   }
10975   gpr_log(GPR_INFO, "========= DONE WITH FIRST BATCH ==========");
10976   // The drop rate should be roughly equal to the expectation.
10977   double seen_drop_rate = static_cast<double>(num_drops) / kNumRpcsLbOnly;
10978   gpr_log(GPR_INFO, "First batch drop rate %f", seen_drop_rate);
10979   EXPECT_THAT(seen_drop_rate,
10980               ::testing::DoubleNear(kDropRateForLb, kErrorTolerance));
10981   // The second ADS response contains two drop categories, send an update EDS
10982   // response.
10983   args.drop_categories = {{kLbDropType, kDropPerMillionForLb},
10984                           {kThrottleDropType, kDropPerMillionForThrottle}};
10985   balancers_[0]->ads_service()->SetEdsResource(
10986       BuildEdsResource(args, DefaultEdsServiceName()));
10987   // Wait until the drop rate increases to the middle of the two configs,
10988   // which implies that the update has been in effect.
10989   const double kDropRateThreshold =
10990       (kDropRateForLb + kDropRateForLbAndThrottle) / 2;
10991   size_t num_rpcs = kNumRpcsBoth;
10992   while (seen_drop_rate < kDropRateThreshold) {
10993     EchoResponse response;
10994     const Status status = SendRpc(RpcOptions(), &response);
10995     ++num_rpcs;
10996     if (!status.ok() &&
10997         absl::StartsWith(status.error_message(), "EDS-configured drop: ")) {
10998       ++num_drops;
10999     } else {
11000       EXPECT_TRUE(status.ok()) << "code=" << status.error_code()
11001                                << " message=" << status.error_message();
11002       EXPECT_EQ(response.message(), kRequestMessage);
11003     }
11004     seen_drop_rate = static_cast<double>(num_drops) / num_rpcs;
11005   }
11006   // Send kNumRpcsBoth RPCs and count the drops.
11007   num_drops = 0;
11008   gpr_log(GPR_INFO, "========= BEFORE SECOND BATCH ==========");
11009   for (size_t i = 0; i < kNumRpcsBoth; ++i) {
11010     EchoResponse response;
11011     const Status status = SendRpc(RpcOptions(), &response);
11012     if (!status.ok() &&
11013         absl::StartsWith(status.error_message(), "EDS-configured drop: ")) {
11014       ++num_drops;
11015     } else {
11016       EXPECT_TRUE(status.ok()) << "code=" << status.error_code()
11017                                << " message=" << status.error_message();
11018       EXPECT_EQ(response.message(), kRequestMessage);
11019     }
11020   }
11021   gpr_log(GPR_INFO, "========= DONE WITH SECOND BATCH ==========");
11022   // The new drop rate should be roughly equal to the expectation.
11023   seen_drop_rate = static_cast<double>(num_drops) / kNumRpcsBoth;
11024   gpr_log(GPR_INFO, "Second batch drop rate %f", seen_drop_rate);
11025   EXPECT_THAT(seen_drop_rate, ::testing::DoubleNear(kDropRateForLbAndThrottle,
11026                                                     kErrorTolerance));
11027 }
11028
11029 // Tests that all the RPCs are dropped if any drop category drops 100%.
11030 TEST_P(DropTest, DropAll) {
11031   SetNextResolution({});
11032   SetNextResolutionForLbChannelAllBalancers();
11033   const size_t kNumRpcs = 1000;
11034   const uint32_t kDropPerMillionForLb = 100000;
11035   const uint32_t kDropPerMillionForThrottle = 1000000;
11036   // The ADS response contains two drop categories.
11037   AdsServiceImpl::EdsResourceArgs args;
11038   args.drop_categories = {{kLbDropType, kDropPerMillionForLb},
11039                           {kThrottleDropType, kDropPerMillionForThrottle}};
11040   balancers_[0]->ads_service()->SetEdsResource(
11041       BuildEdsResource(args, DefaultEdsServiceName()));
11042   // Send kNumRpcs RPCs and all of them are dropped.
11043   for (size_t i = 0; i < kNumRpcs; ++i) {
11044     EchoResponse response;
11045     const Status status = SendRpc(RpcOptions(), &response);
11046     EXPECT_EQ(status.error_code(), StatusCode::UNAVAILABLE);
11047     EXPECT_THAT(status.error_message(),
11048                 ::testing::StartsWith("EDS-configured drop: "));
11049   }
11050 }
11051
11052 class BalancerUpdateTest : public XdsEnd2endTest {
11053  public:
11054   BalancerUpdateTest() : XdsEnd2endTest(4, 3) {}
11055 };
11056
11057 // Tests that the old LB call is still used after the balancer address update
11058 // as long as that call is still alive.
11059 TEST_P(BalancerUpdateTest, UpdateBalancersButKeepUsingOriginalBalancer) {
11060   SetNextResolution({});
11061   SetNextResolutionForLbChannelAllBalancers();
11062   AdsServiceImpl::EdsResourceArgs args(
11063       {{"locality0", CreateEndpointsForBackends(0, 1)}});
11064   balancers_[0]->ads_service()->SetEdsResource(
11065       BuildEdsResource(args, DefaultEdsServiceName()));
11066   args = AdsServiceImpl::EdsResourceArgs(
11067       {{"locality0", CreateEndpointsForBackends(1, 2)}});
11068   balancers_[1]->ads_service()->SetEdsResource(
11069       BuildEdsResource(args, DefaultEdsServiceName()));
11070   // Wait until the first backend is ready.
11071   WaitForBackend(0);
11072   // Send 10 requests.
11073   gpr_log(GPR_INFO, "========= BEFORE FIRST BATCH ==========");
11074   CheckRpcSendOk(10);
11075   gpr_log(GPR_INFO, "========= DONE WITH FIRST BATCH ==========");
11076   // All 10 requests should have gone to the first backend.
11077   EXPECT_EQ(10U, backends_[0]->backend_service()->request_count());
11078   // The ADS service of balancer 0 sent at least 1 response.
11079   EXPECT_GT(balancers_[0]->ads_service()->eds_response_state().state,
11080             AdsServiceImpl::ResponseState::NOT_SENT);
11081   EXPECT_EQ(balancers_[1]->ads_service()->eds_response_state().state,
11082             AdsServiceImpl::ResponseState::NOT_SENT)
11083       << "Error Message:"
11084       << balancers_[1]->ads_service()->eds_response_state().error_message;
11085   EXPECT_EQ(balancers_[2]->ads_service()->eds_response_state().state,
11086             AdsServiceImpl::ResponseState::NOT_SENT)
11087       << "Error Message:"
11088       << balancers_[2]->ads_service()->eds_response_state().error_message;
11089   gpr_log(GPR_INFO, "========= ABOUT TO UPDATE 1 ==========");
11090   SetNextResolutionForLbChannel({balancers_[1]->port()});
11091   gpr_log(GPR_INFO, "========= UPDATE 1 DONE ==========");
11092   EXPECT_EQ(0U, backends_[1]->backend_service()->request_count());
11093   gpr_timespec deadline = gpr_time_add(
11094       gpr_now(GPR_CLOCK_REALTIME), gpr_time_from_millis(10000, GPR_TIMESPAN));
11095   // Send 10 seconds worth of RPCs
11096   do {
11097     CheckRpcSendOk();
11098   } while (gpr_time_cmp(gpr_now(GPR_CLOCK_REALTIME), deadline) < 0);
11099   // The current LB call is still working, so xds continued using it to the
11100   // first balancer, which doesn't assign the second backend.
11101   EXPECT_EQ(0U, backends_[1]->backend_service()->request_count());
11102   // The ADS service of balancer 0 sent at least 1 response.
11103   EXPECT_GT(balancers_[0]->ads_service()->eds_response_state().state,
11104             AdsServiceImpl::ResponseState::NOT_SENT);
11105   EXPECT_EQ(balancers_[1]->ads_service()->eds_response_state().state,
11106             AdsServiceImpl::ResponseState::NOT_SENT)
11107       << "Error Message:"
11108       << balancers_[1]->ads_service()->eds_response_state().error_message;
11109   EXPECT_EQ(balancers_[2]->ads_service()->eds_response_state().state,
11110             AdsServiceImpl::ResponseState::NOT_SENT)
11111       << "Error Message:"
11112       << balancers_[2]->ads_service()->eds_response_state().error_message;
11113 }
11114
11115 // Tests that the old LB call is still used after multiple balancer address
11116 // updates as long as that call is still alive. Send an update with the same
11117 // set of LBs as the one in SetUp() in order to verify that the LB channel
11118 // inside xds keeps the initial connection (which by definition is also
11119 // present in the update).
11120 TEST_P(BalancerUpdateTest, Repeated) {
11121   SetNextResolution({});
11122   SetNextResolutionForLbChannelAllBalancers();
11123   AdsServiceImpl::EdsResourceArgs args(
11124       {{"locality0", CreateEndpointsForBackends(0, 1)}});
11125   balancers_[0]->ads_service()->SetEdsResource(
11126       BuildEdsResource(args, DefaultEdsServiceName()));
11127   args = AdsServiceImpl::EdsResourceArgs(
11128       {{"locality0", CreateEndpointsForBackends(1, 2)}});
11129   balancers_[1]->ads_service()->SetEdsResource(
11130       BuildEdsResource(args, DefaultEdsServiceName()));
11131   // Wait until the first backend is ready.
11132   WaitForBackend(0);
11133   // Send 10 requests.
11134   gpr_log(GPR_INFO, "========= BEFORE FIRST BATCH ==========");
11135   CheckRpcSendOk(10);
11136   gpr_log(GPR_INFO, "========= DONE WITH FIRST BATCH ==========");
11137   // All 10 requests should have gone to the first backend.
11138   EXPECT_EQ(10U, backends_[0]->backend_service()->request_count());
11139   // The ADS service of balancer 0 sent at least 1 response.
11140   EXPECT_GT(balancers_[0]->ads_service()->eds_response_state().state,
11141             AdsServiceImpl::ResponseState::NOT_SENT);
11142   EXPECT_EQ(balancers_[1]->ads_service()->eds_response_state().state,
11143             AdsServiceImpl::ResponseState::NOT_SENT)
11144       << "Error Message:"
11145       << balancers_[1]->ads_service()->eds_response_state().error_message;
11146   EXPECT_EQ(balancers_[2]->ads_service()->eds_response_state().state,
11147             AdsServiceImpl::ResponseState::NOT_SENT)
11148       << "Error Message:"
11149       << balancers_[2]->ads_service()->eds_response_state().error_message;
11150   std::vector<int> ports;
11151   ports.emplace_back(balancers_[0]->port());
11152   ports.emplace_back(balancers_[1]->port());
11153   ports.emplace_back(balancers_[2]->port());
11154   gpr_log(GPR_INFO, "========= ABOUT TO UPDATE 1 ==========");
11155   SetNextResolutionForLbChannel(ports);
11156   gpr_log(GPR_INFO, "========= UPDATE 1 DONE ==========");
11157   EXPECT_EQ(0U, backends_[1]->backend_service()->request_count());
11158   gpr_timespec deadline = gpr_time_add(
11159       gpr_now(GPR_CLOCK_REALTIME), gpr_time_from_millis(10000, GPR_TIMESPAN));
11160   // Send 10 seconds worth of RPCs
11161   do {
11162     CheckRpcSendOk();
11163   } while (gpr_time_cmp(gpr_now(GPR_CLOCK_REALTIME), deadline) < 0);
11164   // xds continued using the original LB call to the first balancer, which
11165   // doesn't assign the second backend.
11166   EXPECT_EQ(0U, backends_[1]->backend_service()->request_count());
11167   ports.clear();
11168   ports.emplace_back(balancers_[0]->port());
11169   ports.emplace_back(balancers_[1]->port());
11170   gpr_log(GPR_INFO, "========= ABOUT TO UPDATE 2 ==========");
11171   SetNextResolutionForLbChannel(ports);
11172   gpr_log(GPR_INFO, "========= UPDATE 2 DONE ==========");
11173   EXPECT_EQ(0U, backends_[1]->backend_service()->request_count());
11174   deadline = gpr_time_add(gpr_now(GPR_CLOCK_REALTIME),
11175                           gpr_time_from_millis(10000, GPR_TIMESPAN));
11176   // Send 10 seconds worth of RPCs
11177   do {
11178     CheckRpcSendOk();
11179   } while (gpr_time_cmp(gpr_now(GPR_CLOCK_REALTIME), deadline) < 0);
11180   // xds continued using the original LB call to the first balancer, which
11181   // doesn't assign the second backend.
11182   EXPECT_EQ(0U, backends_[1]->backend_service()->request_count());
11183 }
11184
11185 // Tests that if the balancer is down, the RPCs will still be sent to the
11186 // backends according to the last balancer response, until a new balancer is
11187 // reachable.
11188 TEST_P(BalancerUpdateTest, DeadUpdate) {
11189   SetNextResolution({});
11190   SetNextResolutionForLbChannel({balancers_[0]->port()});
11191   AdsServiceImpl::EdsResourceArgs args(
11192       {{"locality0", CreateEndpointsForBackends(0, 1)}});
11193   balancers_[0]->ads_service()->SetEdsResource(
11194       BuildEdsResource(args, DefaultEdsServiceName()));
11195   args = AdsServiceImpl::EdsResourceArgs(
11196       {{"locality0", CreateEndpointsForBackends(1, 2)}});
11197   balancers_[1]->ads_service()->SetEdsResource(
11198       BuildEdsResource(args, DefaultEdsServiceName()));
11199   // Start servers and send 10 RPCs per server.
11200   gpr_log(GPR_INFO, "========= BEFORE FIRST BATCH ==========");
11201   CheckRpcSendOk(10);
11202   gpr_log(GPR_INFO, "========= DONE WITH FIRST BATCH ==========");
11203   // All 10 requests should have gone to the first backend.
11204   EXPECT_EQ(10U, backends_[0]->backend_service()->request_count());
11205   // The ADS service of balancer 0 sent at least 1 response.
11206   EXPECT_GT(balancers_[0]->ads_service()->eds_response_state().state,
11207             AdsServiceImpl::ResponseState::NOT_SENT);
11208   EXPECT_EQ(balancers_[1]->ads_service()->eds_response_state().state,
11209             AdsServiceImpl::ResponseState::NOT_SENT)
11210       << "Error Message:"
11211       << balancers_[1]->ads_service()->eds_response_state().error_message;
11212   EXPECT_EQ(balancers_[2]->ads_service()->eds_response_state().state,
11213             AdsServiceImpl::ResponseState::NOT_SENT)
11214       << "Error Message:"
11215       << balancers_[2]->ads_service()->eds_response_state().error_message;
11216   // Kill balancer 0
11217   gpr_log(GPR_INFO, "********** ABOUT TO KILL BALANCER 0 *************");
11218   balancers_[0]->Shutdown();
11219   gpr_log(GPR_INFO, "********** KILLED BALANCER 0 *************");
11220   // This is serviced by the existing child policy.
11221   gpr_log(GPR_INFO, "========= BEFORE SECOND BATCH ==========");
11222   CheckRpcSendOk(10);
11223   gpr_log(GPR_INFO, "========= DONE WITH SECOND BATCH ==========");
11224   // All 10 requests should again have gone to the first backend.
11225   EXPECT_EQ(20U, backends_[0]->backend_service()->request_count());
11226   EXPECT_EQ(0U, backends_[1]->backend_service()->request_count());
11227   // The ADS service of no balancers sent anything
11228   EXPECT_EQ(balancers_[0]->ads_service()->eds_response_state().state,
11229             AdsServiceImpl::ResponseState::NOT_SENT)
11230       << "Error Message:"
11231       << balancers_[0]->ads_service()->eds_response_state().error_message;
11232   EXPECT_EQ(balancers_[1]->ads_service()->eds_response_state().state,
11233             AdsServiceImpl::ResponseState::NOT_SENT)
11234       << "Error Message:"
11235       << balancers_[1]->ads_service()->eds_response_state().error_message;
11236   EXPECT_EQ(balancers_[2]->ads_service()->eds_response_state().state,
11237             AdsServiceImpl::ResponseState::NOT_SENT)
11238       << "Error Message:"
11239       << balancers_[2]->ads_service()->eds_response_state().error_message;
11240   gpr_log(GPR_INFO, "========= ABOUT TO UPDATE 1 ==========");
11241   SetNextResolutionForLbChannel({balancers_[1]->port()});
11242   gpr_log(GPR_INFO, "========= UPDATE 1 DONE ==========");
11243   // Wait until update has been processed, as signaled by the second backend
11244   // receiving a request. In the meantime, the client continues to be serviced
11245   // (by the first backend) without interruption.
11246   EXPECT_EQ(0U, backends_[1]->backend_service()->request_count());
11247   WaitForBackend(1);
11248   // This is serviced by the updated RR policy
11249   backends_[1]->backend_service()->ResetCounters();
11250   gpr_log(GPR_INFO, "========= BEFORE THIRD BATCH ==========");
11251   CheckRpcSendOk(10);
11252   gpr_log(GPR_INFO, "========= DONE WITH THIRD BATCH ==========");
11253   // All 10 requests should have gone to the second backend.
11254   EXPECT_EQ(10U, backends_[1]->backend_service()->request_count());
11255   // The ADS service of balancer 1 sent at least 1 response.
11256   EXPECT_EQ(balancers_[0]->ads_service()->eds_response_state().state,
11257             AdsServiceImpl::ResponseState::NOT_SENT)
11258       << "Error Message:"
11259       << balancers_[0]->ads_service()->eds_response_state().error_message;
11260   EXPECT_GT(balancers_[1]->ads_service()->eds_response_state().state,
11261             AdsServiceImpl::ResponseState::NOT_SENT);
11262   EXPECT_EQ(balancers_[2]->ads_service()->eds_response_state().state,
11263             AdsServiceImpl::ResponseState::NOT_SENT)
11264       << "Error Message:"
11265       << balancers_[2]->ads_service()->eds_response_state().error_message;
11266 }
11267
11268 class ClientLoadReportingTest : public XdsEnd2endTest {
11269  public:
11270   ClientLoadReportingTest() : XdsEnd2endTest(4, 1, 3) {}
11271 };
11272
11273 // Tests that the load report received at the balancer is correct.
11274 TEST_P(ClientLoadReportingTest, Vanilla) {
11275   if (GetParam().use_fake_resolver()) {
11276     balancers_[0]->lrs_service()->set_cluster_names({kServerName});
11277   }
11278   SetNextResolution({});
11279   SetNextResolutionForLbChannel({balancers_[0]->port()});
11280   const size_t kNumRpcsPerAddress = 10;
11281   const size_t kNumFailuresPerAddress = 3;
11282   // TODO(juanlishen): Partition the backends after multiple localities is
11283   // tested.
11284   AdsServiceImpl::EdsResourceArgs args({
11285       {"locality0", CreateEndpointsForBackends()},
11286   });
11287   balancers_[0]->ads_service()->SetEdsResource(
11288       BuildEdsResource(args, DefaultEdsServiceName()));
11289   // Wait until all backends are ready.
11290   int num_ok = 0;
11291   int num_failure = 0;
11292   int num_drops = 0;
11293   std::tie(num_ok, num_failure, num_drops) = WaitForAllBackends();
11294   // Send kNumRpcsPerAddress RPCs per server.
11295   CheckRpcSendOk(kNumRpcsPerAddress * num_backends_);
11296   CheckRpcSendFailure(CheckRpcSendFailureOptions()
11297                           .set_times(kNumFailuresPerAddress * num_backends_)
11298                           .set_rpc_options(RpcOptions().set_server_fail(true)));
11299   // Check that each backend got the right number of requests.
11300   for (size_t i = 0; i < backends_.size(); ++i) {
11301     EXPECT_EQ(kNumRpcsPerAddress + kNumFailuresPerAddress,
11302               backends_[i]->backend_service()->request_count());
11303   }
11304   // The load report received at the balancer should be correct.
11305   std::vector<ClientStats> load_report =
11306       balancers_[0]->lrs_service()->WaitForLoadReport();
11307   ASSERT_EQ(load_report.size(), 1UL);
11308   ClientStats& client_stats = load_report.front();
11309   EXPECT_EQ(kNumRpcsPerAddress * num_backends_ + num_ok,
11310             client_stats.total_successful_requests());
11311   EXPECT_EQ(0U, client_stats.total_requests_in_progress());
11312   EXPECT_EQ((kNumRpcsPerAddress + kNumFailuresPerAddress) * num_backends_ +
11313                 num_ok + num_failure,
11314             client_stats.total_issued_requests());
11315   EXPECT_EQ(kNumFailuresPerAddress * num_backends_ + num_failure,
11316             client_stats.total_error_requests());
11317   EXPECT_EQ(0U, client_stats.total_dropped_requests());
11318   // The LRS service got a single request, and sent a single response.
11319   EXPECT_EQ(1U, balancers_[0]->lrs_service()->request_count());
11320   EXPECT_EQ(1U, balancers_[0]->lrs_service()->response_count());
11321 }
11322
11323 // Tests send_all_clusters.
11324 TEST_P(ClientLoadReportingTest, SendAllClusters) {
11325   balancers_[0]->lrs_service()->set_send_all_clusters(true);
11326   SetNextResolution({});
11327   SetNextResolutionForLbChannel({balancers_[0]->port()});
11328   const size_t kNumRpcsPerAddress = 10;
11329   const size_t kNumFailuresPerAddress = 3;
11330   // TODO(juanlishen): Partition the backends after multiple localities is
11331   // tested.
11332   AdsServiceImpl::EdsResourceArgs args({
11333       {"locality0", CreateEndpointsForBackends()},
11334   });
11335   balancers_[0]->ads_service()->SetEdsResource(
11336       BuildEdsResource(args, DefaultEdsServiceName()));
11337   // Wait until all backends are ready.
11338   int num_ok = 0;
11339   int num_failure = 0;
11340   int num_drops = 0;
11341   std::tie(num_ok, num_failure, num_drops) = WaitForAllBackends();
11342   // Send kNumRpcsPerAddress RPCs per server.
11343   CheckRpcSendOk(kNumRpcsPerAddress * num_backends_);
11344   CheckRpcSendFailure(CheckRpcSendFailureOptions()
11345                           .set_times(kNumFailuresPerAddress * num_backends_)
11346                           .set_rpc_options(RpcOptions().set_server_fail(true)));
11347   // Check that each backend got the right number of requests.
11348   for (size_t i = 0; i < backends_.size(); ++i) {
11349     EXPECT_EQ(kNumRpcsPerAddress + kNumFailuresPerAddress,
11350               backends_[i]->backend_service()->request_count());
11351   }
11352   // The load report received at the balancer should be correct.
11353   std::vector<ClientStats> load_report =
11354       balancers_[0]->lrs_service()->WaitForLoadReport();
11355   ASSERT_EQ(load_report.size(), 1UL);
11356   ClientStats& client_stats = load_report.front();
11357   EXPECT_EQ(kNumRpcsPerAddress * num_backends_ + num_ok,
11358             client_stats.total_successful_requests());
11359   EXPECT_EQ(0U, client_stats.total_requests_in_progress());
11360   EXPECT_EQ((kNumRpcsPerAddress + kNumFailuresPerAddress) * num_backends_ +
11361                 num_ok + num_failure,
11362             client_stats.total_issued_requests());
11363   EXPECT_EQ(kNumFailuresPerAddress * num_backends_ + num_failure,
11364             client_stats.total_error_requests());
11365   EXPECT_EQ(0U, client_stats.total_dropped_requests());
11366   // The LRS service got a single request, and sent a single response.
11367   EXPECT_EQ(1U, balancers_[0]->lrs_service()->request_count());
11368   EXPECT_EQ(1U, balancers_[0]->lrs_service()->response_count());
11369 }
11370
11371 // Tests that we don't include stats for clusters that are not requested
11372 // by the LRS server.
11373 TEST_P(ClientLoadReportingTest, HonorsClustersRequestedByLrsServer) {
11374   balancers_[0]->lrs_service()->set_cluster_names({"bogus"});
11375   SetNextResolution({});
11376   SetNextResolutionForLbChannel({balancers_[0]->port()});
11377   const size_t kNumRpcsPerAddress = 100;
11378   AdsServiceImpl::EdsResourceArgs args({
11379       {"locality0", CreateEndpointsForBackends()},
11380   });
11381   balancers_[0]->ads_service()->SetEdsResource(
11382       BuildEdsResource(args, DefaultEdsServiceName()));
11383   // Wait until all backends are ready.
11384   int num_ok = 0;
11385   int num_failure = 0;
11386   int num_drops = 0;
11387   std::tie(num_ok, num_failure, num_drops) = WaitForAllBackends();
11388   // Send kNumRpcsPerAddress RPCs per server.
11389   CheckRpcSendOk(kNumRpcsPerAddress * num_backends_);
11390   // Each backend should have gotten 100 requests.
11391   for (size_t i = 0; i < backends_.size(); ++i) {
11392     EXPECT_EQ(kNumRpcsPerAddress,
11393               backends_[i]->backend_service()->request_count());
11394   }
11395   // The LRS service got a single request, and sent a single response.
11396   EXPECT_EQ(1U, balancers_[0]->lrs_service()->request_count());
11397   EXPECT_EQ(1U, balancers_[0]->lrs_service()->response_count());
11398   // The load report received at the balancer should be correct.
11399   std::vector<ClientStats> load_report =
11400       balancers_[0]->lrs_service()->WaitForLoadReport();
11401   ASSERT_EQ(load_report.size(), 0UL);
11402 }
11403
11404 // Tests that if the balancer restarts, the client load report contains the
11405 // stats before and after the restart correctly.
11406 TEST_P(ClientLoadReportingTest, BalancerRestart) {
11407   if (GetParam().use_fake_resolver()) {
11408     balancers_[0]->lrs_service()->set_cluster_names({kServerName});
11409   }
11410   SetNextResolution({});
11411   SetNextResolutionForLbChannel({balancers_[0]->port()});
11412   const size_t kNumBackendsFirstPass = backends_.size() / 2;
11413   const size_t kNumBackendsSecondPass =
11414       backends_.size() - kNumBackendsFirstPass;
11415   AdsServiceImpl::EdsResourceArgs args({
11416       {"locality0", CreateEndpointsForBackends(0, kNumBackendsFirstPass)},
11417   });
11418   balancers_[0]->ads_service()->SetEdsResource(
11419       BuildEdsResource(args, DefaultEdsServiceName()));
11420   // Wait until all backends returned by the balancer are ready.
11421   int num_ok = 0;
11422   int num_failure = 0;
11423   int num_drops = 0;
11424   std::tie(num_ok, num_failure, num_drops) =
11425       WaitForAllBackends(/* start_index */ 0,
11426                          /* stop_index */ kNumBackendsFirstPass);
11427   std::vector<ClientStats> load_report =
11428       balancers_[0]->lrs_service()->WaitForLoadReport();
11429   ASSERT_EQ(load_report.size(), 1UL);
11430   ClientStats client_stats = std::move(load_report.front());
11431   EXPECT_EQ(static_cast<size_t>(num_ok),
11432             client_stats.total_successful_requests());
11433   EXPECT_EQ(0U, client_stats.total_requests_in_progress());
11434   EXPECT_EQ(0U, client_stats.total_error_requests());
11435   EXPECT_EQ(0U, client_stats.total_dropped_requests());
11436   // Shut down the balancer.
11437   balancers_[0]->Shutdown();
11438   // We should continue using the last EDS response we received from the
11439   // balancer before it was shut down.
11440   // Note: We need to use WaitForAllBackends() here instead of just
11441   // CheckRpcSendOk(kNumBackendsFirstPass), because when the balancer
11442   // shuts down, the XdsClient will generate an error to the
11443   // ServiceConfigWatcher, which will cause the xds resolver to send a
11444   // no-op update to the LB policy.  When this update gets down to the
11445   // round_robin child policy for the locality, it will generate a new
11446   // subchannel list, which resets the start index randomly.  So we need
11447   // to be a little more permissive here to avoid spurious failures.
11448   ResetBackendCounters();
11449   int num_started = std::get<0>(WaitForAllBackends(
11450       /* start_index */ 0, /* stop_index */ kNumBackendsFirstPass));
11451   // Now restart the balancer, this time pointing to the new backends.
11452   balancers_[0]->Start();
11453   args = AdsServiceImpl::EdsResourceArgs({
11454       {"locality0", CreateEndpointsForBackends(kNumBackendsFirstPass)},
11455   });
11456   balancers_[0]->ads_service()->SetEdsResource(
11457       BuildEdsResource(args, DefaultEdsServiceName()));
11458   // Wait for queries to start going to one of the new backends.
11459   // This tells us that we're now using the new serverlist.
11460   std::tie(num_ok, num_failure, num_drops) =
11461       WaitForAllBackends(/* start_index */ kNumBackendsFirstPass);
11462   num_started += num_ok + num_failure + num_drops;
11463   // Send one RPC per backend.
11464   CheckRpcSendOk(kNumBackendsSecondPass);
11465   num_started += kNumBackendsSecondPass;
11466   // Check client stats.
11467   load_report = balancers_[0]->lrs_service()->WaitForLoadReport();
11468   ASSERT_EQ(load_report.size(), 1UL);
11469   client_stats = std::move(load_report.front());
11470   EXPECT_EQ(num_started, client_stats.total_successful_requests());
11471   EXPECT_EQ(0U, client_stats.total_requests_in_progress());
11472   EXPECT_EQ(0U, client_stats.total_error_requests());
11473   EXPECT_EQ(0U, client_stats.total_dropped_requests());
11474 }
11475
11476 class ClientLoadReportingWithDropTest : public XdsEnd2endTest {
11477  public:
11478   ClientLoadReportingWithDropTest() : XdsEnd2endTest(4, 1, 20) {}
11479 };
11480
11481 // Tests that the drop stats are correctly reported by client load reporting.
11482 TEST_P(ClientLoadReportingWithDropTest, Vanilla) {
11483   if (GetParam().use_fake_resolver()) {
11484     balancers_[0]->lrs_service()->set_cluster_names({kServerName});
11485   }
11486   SetNextResolution({});
11487   SetNextResolutionForLbChannelAllBalancers();
11488   const uint32_t kDropPerMillionForLb = 100000;
11489   const uint32_t kDropPerMillionForThrottle = 200000;
11490   const double kErrorTolerance = 0.05;
11491   const double kDropRateForLb = kDropPerMillionForLb / 1000000.0;
11492   const double kDropRateForThrottle = kDropPerMillionForThrottle / 1000000.0;
11493   const double kDropRateForLbAndThrottle =
11494       kDropRateForLb + (1 - kDropRateForLb) * kDropRateForThrottle;
11495   const size_t kNumRpcs =
11496       ComputeIdealNumRpcs(kDropRateForLbAndThrottle, kErrorTolerance);
11497   // The ADS response contains two drop categories.
11498   AdsServiceImpl::EdsResourceArgs args({
11499       {"locality0", CreateEndpointsForBackends()},
11500   });
11501   args.drop_categories = {{kLbDropType, kDropPerMillionForLb},
11502                           {kThrottleDropType, kDropPerMillionForThrottle}};
11503   balancers_[0]->ads_service()->SetEdsResource(
11504       BuildEdsResource(args, DefaultEdsServiceName()));
11505   int num_ok = 0;
11506   int num_failure = 0;
11507   int num_drops = 0;
11508   std::tie(num_ok, num_failure, num_drops) = WaitForAllBackends();
11509   const size_t num_warmup = num_ok + num_failure + num_drops;
11510   // Send kNumRpcs RPCs and count the drops.
11511   for (size_t i = 0; i < kNumRpcs; ++i) {
11512     EchoResponse response;
11513     const Status status = SendRpc(RpcOptions(), &response);
11514     if (!status.ok() &&
11515         absl::StartsWith(status.error_message(), "EDS-configured drop: ")) {
11516       ++num_drops;
11517     } else {
11518       EXPECT_TRUE(status.ok()) << "code=" << status.error_code()
11519                                << " message=" << status.error_message();
11520       EXPECT_EQ(response.message(), kRequestMessage);
11521     }
11522   }
11523   // The drop rate should be roughly equal to the expectation.
11524   const double seen_drop_rate = static_cast<double>(num_drops) / kNumRpcs;
11525   EXPECT_THAT(seen_drop_rate, ::testing::DoubleNear(kDropRateForLbAndThrottle,
11526                                                     kErrorTolerance));
11527   // Check client stats.
11528   const size_t total_rpc = num_warmup + kNumRpcs;
11529   ClientStats client_stats;
11530   do {
11531     std::vector<ClientStats> load_reports =
11532         balancers_[0]->lrs_service()->WaitForLoadReport();
11533     for (const auto& load_report : load_reports) {
11534       client_stats += load_report;
11535     }
11536   } while (client_stats.total_issued_requests() +
11537                client_stats.total_dropped_requests() <
11538            total_rpc);
11539   EXPECT_EQ(num_drops, client_stats.total_dropped_requests());
11540   EXPECT_THAT(static_cast<double>(client_stats.dropped_requests(kLbDropType)) /
11541                   total_rpc,
11542               ::testing::DoubleNear(kDropRateForLb, kErrorTolerance));
11543   EXPECT_THAT(
11544       static_cast<double>(client_stats.dropped_requests(kThrottleDropType)) /
11545           (total_rpc * (1 - kDropRateForLb)),
11546       ::testing::DoubleNear(kDropRateForThrottle, kErrorTolerance));
11547 }
11548
11549 class FaultInjectionTest : public XdsEnd2endTest {
11550  public:
11551   FaultInjectionTest() : XdsEnd2endTest(1, 1) {}
11552
11553   // Builds a Listener with Fault Injection filter config. If the http_fault
11554   // is nullptr, then assign an empty filter config. This filter config is
11555   // required to enable the fault injection features.
11556   static Listener BuildListenerWithFaultInjection(
11557       const HTTPFault& http_fault = HTTPFault()) {
11558     HttpConnectionManager http_connection_manager;
11559     Listener listener;
11560     listener.set_name(kServerName);
11561     HttpFilter* fault_filter = http_connection_manager.add_http_filters();
11562     fault_filter->set_name("envoy.fault");
11563     fault_filter->mutable_typed_config()->PackFrom(http_fault);
11564     HttpFilter* router_filter = http_connection_manager.add_http_filters();
11565     router_filter->set_name("router");
11566     router_filter->mutable_typed_config()->PackFrom(
11567         envoy::extensions::filters::http::router::v3::Router());
11568     listener.mutable_api_listener()->mutable_api_listener()->PackFrom(
11569         http_connection_manager);
11570     return listener;
11571   }
11572
11573   RouteConfiguration BuildRouteConfigurationWithFaultInjection(
11574       const HTTPFault& http_fault) {
11575     // Package as Any
11576     google::protobuf::Any filter_config;
11577     filter_config.PackFrom(http_fault);
11578     // Plug into the RouteConfiguration
11579     RouteConfiguration new_route_config = default_route_config_;
11580     auto* config_map = new_route_config.mutable_virtual_hosts(0)
11581                            ->mutable_routes(0)
11582                            ->mutable_typed_per_filter_config();
11583     (*config_map)["envoy.fault"] = std::move(filter_config);
11584     return new_route_config;
11585   }
11586
11587   void SetFilterConfig(HTTPFault& http_fault) {
11588     switch (GetParam().filter_config_setup()) {
11589       case TestType::FilterConfigSetup::kRouteOverride: {
11590         Listener listener = BuildListenerWithFaultInjection();
11591         RouteConfiguration route =
11592             BuildRouteConfigurationWithFaultInjection(http_fault);
11593         SetListenerAndRouteConfiguration(0, listener, route);
11594         break;
11595       }
11596       case TestType::FilterConfigSetup::kHTTPConnectionManagerOriginal: {
11597         Listener listener = BuildListenerWithFaultInjection(http_fault);
11598         SetListenerAndRouteConfiguration(0, listener, default_route_config_);
11599       }
11600     };
11601   }
11602 };
11603
11604 // Test to ensure the most basic fault injection config works.
11605 TEST_P(FaultInjectionTest, XdsFaultInjectionAlwaysAbort) {
11606   const uint32_t kAbortPercentagePerHundred = 100;
11607   SetNextResolution({});
11608   SetNextResolutionForLbChannelAllBalancers();
11609   // Construct the fault injection filter config
11610   HTTPFault http_fault;
11611   auto* abort_percentage = http_fault.mutable_abort()->mutable_percentage();
11612   abort_percentage->set_numerator(kAbortPercentagePerHundred);
11613   abort_percentage->set_denominator(FractionalPercent::HUNDRED);
11614   http_fault.mutable_abort()->set_grpc_status(
11615       static_cast<uint32_t>(StatusCode::ABORTED));
11616   // Config fault injection via different setup
11617   SetFilterConfig(http_fault);
11618   // Fire several RPCs, and expect all of them to be aborted.
11619   CheckRpcSendFailure(
11620       CheckRpcSendFailureOptions()
11621           .set_times(5)
11622           .set_rpc_options(RpcOptions().set_wait_for_ready(true))
11623           .set_expected_error_code(StatusCode::ABORTED));
11624 }
11625
11626 // Without the listener config, the fault injection won't be enabled.
11627 TEST_P(FaultInjectionTest, XdsFaultInjectionWithoutListenerFilter) {
11628   const uint32_t kAbortPercentagePerHundred = 100;
11629   SetNextResolution({});
11630   SetNextResolutionForLbChannelAllBalancers();
11631   // Create an EDS resource
11632   AdsServiceImpl::EdsResourceArgs args({
11633       {"locality0", CreateEndpointsForBackends()},
11634   });
11635   balancers_[0]->ads_service()->SetEdsResource(
11636       BuildEdsResource(args, DefaultEdsServiceName()));
11637   // Construct the fault injection filter config
11638   HTTPFault http_fault;
11639   auto* abort_percentage = http_fault.mutable_abort()->mutable_percentage();
11640   abort_percentage->set_numerator(kAbortPercentagePerHundred);
11641   abort_percentage->set_denominator(FractionalPercent::HUNDRED);
11642   http_fault.mutable_abort()->set_grpc_status(
11643       static_cast<uint32_t>(StatusCode::ABORTED));
11644   // Turn on fault injection
11645   RouteConfiguration route =
11646       BuildRouteConfigurationWithFaultInjection(http_fault);
11647   SetListenerAndRouteConfiguration(0, default_listener_, route);
11648   // Fire several RPCs, and expect all of them to be pass.
11649   CheckRpcSendOk(5, RpcOptions().set_wait_for_ready(true));
11650 }
11651
11652 TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageAbort) {
11653   const uint32_t kAbortPercentagePerHundred = 50;
11654   const double kAbortRate = kAbortPercentagePerHundred / 100.0;
11655   const double kErrorTolerance = 0.05;
11656   const size_t kNumRpcs = ComputeIdealNumRpcs(kAbortRate, kErrorTolerance);
11657   SetNextResolution({});
11658   SetNextResolutionForLbChannelAllBalancers();
11659   // Create an EDS resource
11660   AdsServiceImpl::EdsResourceArgs args({
11661       {"locality0", CreateEndpointsForBackends()},
11662   });
11663   balancers_[0]->ads_service()->SetEdsResource(
11664       BuildEdsResource(args, DefaultEdsServiceName()));
11665   // Construct the fault injection filter config
11666   HTTPFault http_fault;
11667   auto* abort_percentage = http_fault.mutable_abort()->mutable_percentage();
11668   abort_percentage->set_numerator(kAbortPercentagePerHundred);
11669   abort_percentage->set_denominator(FractionalPercent::HUNDRED);
11670   http_fault.mutable_abort()->set_grpc_status(
11671       static_cast<uint32_t>(StatusCode::ABORTED));
11672   // Config fault injection via different setup
11673   SetFilterConfig(http_fault);
11674   // Send kNumRpcs RPCs and count the aborts.
11675   int num_total = 0, num_ok = 0, num_failure = 0, num_aborted = 0;
11676   for (size_t i = 0; i < kNumRpcs; ++i) {
11677     SendRpcAndCount(&num_total, &num_ok, &num_failure, &num_aborted,
11678                     RpcOptions(), "Fault injected");
11679   }
11680   EXPECT_EQ(kNumRpcs, num_total);
11681   EXPECT_EQ(0, num_failure);
11682   // The abort rate should be roughly equal to the expectation.
11683   const double seen_abort_rate = static_cast<double>(num_aborted) / kNumRpcs;
11684   EXPECT_THAT(seen_abort_rate,
11685               ::testing::DoubleNear(kAbortRate, kErrorTolerance));
11686 }
11687
11688 TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageAbortViaHeaders) {
11689   const uint32_t kAbortPercentageCap = 100;
11690   const uint32_t kAbortPercentage = 50;
11691   const double kAbortRate = kAbortPercentage / 100.0;
11692   const double kErrorTolerance = 0.05;
11693   const size_t kNumRpcs = ComputeIdealNumRpcs(kAbortRate, kErrorTolerance);
11694   SetNextResolution({});
11695   SetNextResolutionForLbChannelAllBalancers();
11696   // Create an EDS resource
11697   AdsServiceImpl::EdsResourceArgs args({
11698       {"locality0", CreateEndpointsForBackends()},
11699   });
11700   balancers_[0]->ads_service()->SetEdsResource(
11701       BuildEdsResource(args, DefaultEdsServiceName()));
11702   // Construct the fault injection filter config
11703   HTTPFault http_fault;
11704   http_fault.mutable_abort()->mutable_header_abort();
11705   http_fault.mutable_abort()->mutable_percentage()->set_numerator(
11706       kAbortPercentageCap);
11707   // Config fault injection via different setup
11708   SetFilterConfig(http_fault);
11709   // Send kNumRpcs RPCs and count the aborts.
11710   std::vector<std::pair<std::string, std::string>> metadata = {
11711       {"x-envoy-fault-abort-grpc-request", "10"},
11712       {"x-envoy-fault-abort-percentage", std::to_string(kAbortPercentage)},
11713   };
11714   int num_total = 0, num_ok = 0, num_failure = 0, num_aborted = 0;
11715   RpcOptions options = RpcOptions().set_metadata(metadata);
11716   for (size_t i = 0; i < kNumRpcs; ++i) {
11717     SendRpcAndCount(&num_total, &num_ok, &num_failure, &num_aborted, options,
11718                     "Fault injected");
11719   }
11720   EXPECT_EQ(kNumRpcs, num_total);
11721   EXPECT_EQ(0, num_failure);
11722   // The abort rate should be roughly equal to the expectation.
11723   const double seen_abort_rate = static_cast<double>(num_aborted) / kNumRpcs;
11724   EXPECT_THAT(seen_abort_rate,
11725               ::testing::DoubleNear(kAbortRate, kErrorTolerance));
11726 }
11727
11728 TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageDelay) {
11729   const uint32_t kRpcTimeoutMilliseconds = grpc_test_slowdown_factor() * 3000;
11730   const uint32_t kFixedDelaySeconds = 100;
11731   const uint32_t kDelayPercentagePerHundred = 50;
11732   const double kDelayRate = kDelayPercentagePerHundred / 100.0;
11733   const double kErrorTolerance = 0.05;
11734   const size_t kNumRpcs = ComputeIdealNumRpcs(kDelayRate, kErrorTolerance);
11735   const size_t kMaxConcurrentRequests = kNumRpcs;
11736   SetNextResolution({});
11737   SetNextResolutionForLbChannelAllBalancers();
11738   // Create an EDS resource
11739   AdsServiceImpl::EdsResourceArgs args({
11740       {"locality0", CreateEndpointsForBackends()},
11741   });
11742   balancers_[0]->ads_service()->SetEdsResource(
11743       BuildEdsResource(args, DefaultEdsServiceName()));
11744   // Loosen the max concurrent request limit
11745   Cluster cluster = default_cluster_;
11746   auto* threshold = cluster.mutable_circuit_breakers()->add_thresholds();
11747   threshold->set_priority(RoutingPriority::DEFAULT);
11748   threshold->mutable_max_requests()->set_value(kMaxConcurrentRequests);
11749   balancers_[0]->ads_service()->SetCdsResource(cluster);
11750   // Construct the fault injection filter config
11751   HTTPFault http_fault;
11752   auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage();
11753   delay_percentage->set_numerator(kDelayPercentagePerHundred);
11754   delay_percentage->set_denominator(FractionalPercent::HUNDRED);
11755   auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay();
11756   fixed_delay->set_seconds(kFixedDelaySeconds);
11757   // Config fault injection via different setup
11758   SetFilterConfig(http_fault);
11759   // Send kNumRpcs RPCs and count the delays.
11760   RpcOptions rpc_options = RpcOptions()
11761                                .set_timeout_ms(kRpcTimeoutMilliseconds)
11762                                .set_skip_cancelled_check(true);
11763   std::vector<ConcurrentRpc> rpcs =
11764       SendConcurrentRpcs(stub_.get(), kNumRpcs, rpc_options);
11765   size_t num_delayed = 0;
11766   for (auto& rpc : rpcs) {
11767     if (rpc.status.error_code() == StatusCode::OK) continue;
11768     EXPECT_EQ(StatusCode::DEADLINE_EXCEEDED, rpc.status.error_code());
11769     ++num_delayed;
11770   }
11771   // The delay rate should be roughly equal to the expectation.
11772   const double seen_delay_rate = static_cast<double>(num_delayed) / kNumRpcs;
11773   EXPECT_THAT(seen_delay_rate,
11774               ::testing::DoubleNear(kDelayRate, kErrorTolerance));
11775 }
11776
11777 TEST_P(FaultInjectionTest, XdsFaultInjectionPercentageDelayViaHeaders) {
11778   const uint32_t kFixedDelayMilliseconds = 100000;
11779   const uint32_t kRpcTimeoutMilliseconds = grpc_test_slowdown_factor() * 3000;
11780   const uint32_t kDelayPercentageCap = 100;
11781   const uint32_t kDelayPercentage = 50;
11782   const double kDelayRate = kDelayPercentage / 100.0;
11783   const double kErrorTolerance = 0.05;
11784   const size_t kNumRpcs = ComputeIdealNumRpcs(kDelayRate, kErrorTolerance);
11785   const size_t kMaxConcurrentRequests = kNumRpcs;
11786   SetNextResolution({});
11787   SetNextResolutionForLbChannelAllBalancers();
11788   // Create an EDS resource
11789   AdsServiceImpl::EdsResourceArgs args({
11790       {"locality0", CreateEndpointsForBackends()},
11791   });
11792   balancers_[0]->ads_service()->SetEdsResource(
11793       BuildEdsResource(args, DefaultEdsServiceName()));
11794   // Loosen the max concurrent request limit
11795   Cluster cluster = default_cluster_;
11796   auto* threshold = cluster.mutable_circuit_breakers()->add_thresholds();
11797   threshold->set_priority(RoutingPriority::DEFAULT);
11798   threshold->mutable_max_requests()->set_value(kMaxConcurrentRequests);
11799   balancers_[0]->ads_service()->SetCdsResource(cluster);
11800   // Construct the fault injection filter config
11801   HTTPFault http_fault;
11802   http_fault.mutable_delay()->mutable_header_delay();
11803   http_fault.mutable_delay()->mutable_percentage()->set_numerator(
11804       kDelayPercentageCap);
11805   // Config fault injection via different setup
11806   SetFilterConfig(http_fault);
11807   // Send kNumRpcs RPCs and count the delays.
11808   std::vector<std::pair<std::string, std::string>> metadata = {
11809       {"x-envoy-fault-delay-request", std::to_string(kFixedDelayMilliseconds)},
11810       {"x-envoy-fault-delay-request-percentage",
11811        std::to_string(kDelayPercentage)},
11812   };
11813   RpcOptions rpc_options = RpcOptions()
11814                                .set_metadata(metadata)
11815                                .set_timeout_ms(kRpcTimeoutMilliseconds)
11816                                .set_skip_cancelled_check(true);
11817   std::vector<ConcurrentRpc> rpcs =
11818       SendConcurrentRpcs(stub_.get(), kNumRpcs, rpc_options);
11819   size_t num_delayed = 0;
11820   for (auto& rpc : rpcs) {
11821     if (rpc.status.error_code() == StatusCode::OK) continue;
11822     EXPECT_EQ(StatusCode::DEADLINE_EXCEEDED, rpc.status.error_code());
11823     ++num_delayed;
11824   }
11825   // The delay rate should be roughly equal to the expectation.
11826   const double seen_delay_rate = static_cast<double>(num_delayed) / kNumRpcs;
11827   EXPECT_THAT(seen_delay_rate,
11828               ::testing::DoubleNear(kDelayRate, kErrorTolerance));
11829 }
11830
11831 TEST_P(FaultInjectionTest, XdsFaultInjectionAlwaysDelayPercentageAbort) {
11832   const uint32_t kAbortPercentagePerHundred = 50;
11833   const double kAbortRate = kAbortPercentagePerHundred / 100.0;
11834   const uint32_t kFixedDelaySeconds = 1;
11835   const uint32_t kRpcTimeoutMilliseconds = 100 * 1000;  // 100s should not reach
11836   const uint32_t kConnectionTimeoutMilliseconds =
11837       10 * 1000;  // 10s should not reach
11838   const double kErrorTolerance = 0.05;
11839   const size_t kNumRpcs = ComputeIdealNumRpcs(kAbortRate, kErrorTolerance);
11840   const size_t kMaxConcurrentRequests = kNumRpcs;
11841   SetNextResolution({});
11842   SetNextResolutionForLbChannelAllBalancers();
11843   // Create an EDS resource
11844   AdsServiceImpl::EdsResourceArgs args({
11845       {"locality0", CreateEndpointsForBackends()},
11846   });
11847   balancers_[0]->ads_service()->SetEdsResource(
11848       BuildEdsResource(args, DefaultEdsServiceName()));
11849   // Loosen the max concurrent request limit
11850   Cluster cluster = default_cluster_;
11851   auto* threshold = cluster.mutable_circuit_breakers()->add_thresholds();
11852   threshold->set_priority(RoutingPriority::DEFAULT);
11853   threshold->mutable_max_requests()->set_value(kMaxConcurrentRequests);
11854   balancers_[0]->ads_service()->SetCdsResource(cluster);
11855   // Construct the fault injection filter config
11856   HTTPFault http_fault;
11857   auto* abort_percentage = http_fault.mutable_abort()->mutable_percentage();
11858   abort_percentage->set_numerator(kAbortPercentagePerHundred);
11859   abort_percentage->set_denominator(FractionalPercent::HUNDRED);
11860   http_fault.mutable_abort()->set_grpc_status(
11861       static_cast<uint32_t>(StatusCode::ABORTED));
11862   auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage();
11863   delay_percentage->set_numerator(1000000);  // Always inject DELAY!
11864   delay_percentage->set_denominator(FractionalPercent::MILLION);
11865   auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay();
11866   fixed_delay->set_seconds(kFixedDelaySeconds);
11867   // Config fault injection via different setup
11868   SetFilterConfig(http_fault);
11869   // Allow the channel to connect to one backends, so the herd of queued RPCs
11870   // won't be executed on the same ExecCtx object and using the cached Now()
11871   // value, which causes millisecond level delay error.
11872   channel_->WaitForConnected(
11873       grpc_timeout_milliseconds_to_deadline(kConnectionTimeoutMilliseconds));
11874   // Send kNumRpcs RPCs and count the aborts.
11875   int num_aborted = 0;
11876   RpcOptions rpc_options = RpcOptions().set_timeout_ms(kRpcTimeoutMilliseconds);
11877   std::vector<ConcurrentRpc> rpcs =
11878       SendConcurrentRpcs(stub_.get(), kNumRpcs, rpc_options);
11879   for (auto& rpc : rpcs) {
11880     EXPECT_GE(rpc.elapsed_time, kFixedDelaySeconds * 1000);
11881     if (rpc.status.error_code() == StatusCode::OK) continue;
11882     EXPECT_EQ("Fault injected", rpc.status.error_message());
11883     ++num_aborted;
11884   }
11885   // The abort rate should be roughly equal to the expectation.
11886   const double seen_abort_rate = static_cast<double>(num_aborted) / kNumRpcs;
11887   EXPECT_THAT(seen_abort_rate,
11888               ::testing::DoubleNear(kAbortRate, kErrorTolerance));
11889 }
11890
11891 // This test and the above test apply different denominators to delay and
11892 // abort. This ensures that we are using the right denominator for each
11893 // injected fault in our code.
11894 TEST_P(FaultInjectionTest,
11895        XdsFaultInjectionAlwaysDelayPercentageAbortSwitchDenominator) {
11896   const uint32_t kAbortPercentagePerMillion = 500000;
11897   const double kAbortRate = kAbortPercentagePerMillion / 1000000.0;
11898   const uint32_t kFixedDelaySeconds = 1;                // 1s
11899   const uint32_t kRpcTimeoutMilliseconds = 100 * 1000;  // 100s should not reach
11900   const uint32_t kConnectionTimeoutMilliseconds =
11901       10 * 1000;  // 10s should not reach
11902   const double kErrorTolerance = 0.05;
11903   const size_t kNumRpcs = ComputeIdealNumRpcs(kAbortRate, kErrorTolerance);
11904   const size_t kMaxConcurrentRequests = kNumRpcs;
11905   SetNextResolution({});
11906   SetNextResolutionForLbChannelAllBalancers();
11907   // Create an EDS resource
11908   AdsServiceImpl::EdsResourceArgs args({
11909       {"locality0", CreateEndpointsForBackends()},
11910   });
11911   balancers_[0]->ads_service()->SetEdsResource(
11912       BuildEdsResource(args, DefaultEdsServiceName()));
11913   // Loosen the max concurrent request limit
11914   Cluster cluster = default_cluster_;
11915   auto* threshold = cluster.mutable_circuit_breakers()->add_thresholds();
11916   threshold->set_priority(RoutingPriority::DEFAULT);
11917   threshold->mutable_max_requests()->set_value(kMaxConcurrentRequests);
11918   balancers_[0]->ads_service()->SetCdsResource(cluster);
11919   // Construct the fault injection filter config
11920   HTTPFault http_fault;
11921   auto* abort_percentage = http_fault.mutable_abort()->mutable_percentage();
11922   abort_percentage->set_numerator(kAbortPercentagePerMillion);
11923   abort_percentage->set_denominator(FractionalPercent::MILLION);
11924   http_fault.mutable_abort()->set_grpc_status(
11925       static_cast<uint32_t>(StatusCode::ABORTED));
11926   auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage();
11927   delay_percentage->set_numerator(100);  // Always inject DELAY!
11928   delay_percentage->set_denominator(FractionalPercent::HUNDRED);
11929   auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay();
11930   fixed_delay->set_seconds(kFixedDelaySeconds);
11931   // Config fault injection via different setup
11932   SetFilterConfig(http_fault);
11933   // Allow the channel to connect to one backends, so the herd of queued RPCs
11934   // won't be executed on the same ExecCtx object and using the cached Now()
11935   // value, which causes millisecond level delay error.
11936   channel_->WaitForConnected(
11937       grpc_timeout_milliseconds_to_deadline(kConnectionTimeoutMilliseconds));
11938   // Send kNumRpcs RPCs and count the aborts.
11939   int num_aborted = 0;
11940   RpcOptions rpc_options = RpcOptions().set_timeout_ms(kRpcTimeoutMilliseconds);
11941   std::vector<ConcurrentRpc> rpcs =
11942       SendConcurrentRpcs(stub_.get(), kNumRpcs, rpc_options);
11943   for (auto& rpc : rpcs) {
11944     EXPECT_GE(rpc.elapsed_time, kFixedDelaySeconds * 1000);
11945     if (rpc.status.error_code() == StatusCode::OK) continue;
11946     EXPECT_EQ("Fault injected", rpc.status.error_message());
11947     ++num_aborted;
11948   }
11949   // The abort rate should be roughly equal to the expectation.
11950   const double seen_abort_rate = static_cast<double>(num_aborted) / kNumRpcs;
11951   EXPECT_THAT(seen_abort_rate,
11952               ::testing::DoubleNear(kAbortRate, kErrorTolerance));
11953 }
11954
11955 TEST_P(FaultInjectionTest, XdsFaultInjectionMaxFault) {
11956   const uint32_t kMaxFault = 10;
11957   const uint32_t kNumRpcs = 30;  // kNumRpcs should be bigger than kMaxFault
11958   const uint32_t kRpcTimeoutMs = 4000;     // 4 seconds
11959   const uint32_t kLongDelaySeconds = 100;  // 100 seconds
11960   const uint32_t kAlwaysDelayPercentage = 100;
11961   SetNextResolution({});
11962   SetNextResolutionForLbChannelAllBalancers();
11963   // Create an EDS resource
11964   AdsServiceImpl::EdsResourceArgs args({
11965       {"locality0", CreateEndpointsForBackends()},
11966   });
11967   balancers_[0]->ads_service()->SetEdsResource(
11968       BuildEdsResource(args, DefaultEdsServiceName()));
11969   // Construct the fault injection filter config
11970   HTTPFault http_fault;
11971   auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage();
11972   delay_percentage->set_numerator(
11973       kAlwaysDelayPercentage);  // Always inject DELAY!
11974   delay_percentage->set_denominator(FractionalPercent::HUNDRED);
11975   auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay();
11976   fixed_delay->set_seconds(kLongDelaySeconds);
11977   http_fault.mutable_max_active_faults()->set_value(kMaxFault);
11978   // Config fault injection via different setup
11979   SetFilterConfig(http_fault);
11980   // Sends a batch of long running RPCs with long timeout to consume all
11981   // active faults quota.
11982   int num_delayed = 0;
11983   RpcOptions rpc_options = RpcOptions().set_timeout_ms(kRpcTimeoutMs);
11984   std::vector<ConcurrentRpc> rpcs =
11985       SendConcurrentRpcs(stub_.get(), kNumRpcs, rpc_options);
11986   for (auto& rpc : rpcs) {
11987     if (rpc.status.error_code() == StatusCode::OK) continue;
11988     EXPECT_EQ(StatusCode::DEADLINE_EXCEEDED, rpc.status.error_code());
11989     ++num_delayed;
11990   }
11991   // Only kMaxFault number of RPC should be fault injected..
11992   EXPECT_EQ(kMaxFault, num_delayed);
11993 }
11994
11995 TEST_P(FaultInjectionTest, XdsFaultInjectionBidiStreamDelayOk) {
11996   // kRpcTimeoutMilliseconds is 10s should never be reached.
11997   const uint32_t kRpcTimeoutMilliseconds = grpc_test_slowdown_factor() * 10000;
11998   const uint32_t kFixedDelaySeconds = 1;
11999   const uint32_t kDelayPercentagePerHundred = 100;
12000   SetNextResolution({});
12001   SetNextResolutionForLbChannelAllBalancers();
12002   // Create an EDS resource
12003   AdsServiceImpl::EdsResourceArgs args({
12004       {"locality0", CreateEndpointsForBackends()},
12005   });
12006   balancers_[0]->ads_service()->SetEdsResource(
12007       BuildEdsResource(args, DefaultEdsServiceName()));
12008   // Construct the fault injection filter config
12009   HTTPFault http_fault;
12010   auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage();
12011   delay_percentage->set_numerator(kDelayPercentagePerHundred);
12012   delay_percentage->set_denominator(FractionalPercent::HUNDRED);
12013   auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay();
12014   fixed_delay->set_seconds(kFixedDelaySeconds);
12015   // Config fault injection via different setup
12016   SetFilterConfig(http_fault);
12017   ClientContext context;
12018   context.set_deadline(
12019       grpc_timeout_milliseconds_to_deadline(kRpcTimeoutMilliseconds));
12020   auto stream = stub_->BidiStream(&context);
12021   stream->WritesDone();
12022   auto status = stream->Finish();
12023   EXPECT_TRUE(status.ok()) << status.error_message() << ", "
12024                            << status.error_details() << ", "
12025                            << context.debug_error_string();
12026 }
12027
12028 // This case catches a bug in the retry code that was triggered by a bad
12029 // interaction with the FI code.  See https://github.com/grpc/grpc/pull/27217
12030 // for description.
12031 TEST_P(FaultInjectionTest, XdsFaultInjectionBidiStreamDelayError) {
12032   const uint32_t kRpcTimeoutMilliseconds = grpc_test_slowdown_factor() * 500;
12033   const uint32_t kFixedDelaySeconds = 100;
12034   const uint32_t kDelayPercentagePerHundred = 100;
12035   SetNextResolution({});
12036   SetNextResolutionForLbChannelAllBalancers();
12037   // Create an EDS resource
12038   AdsServiceImpl::EdsResourceArgs args({
12039       {"locality0", CreateEndpointsForBackends()},
12040   });
12041   balancers_[0]->ads_service()->SetEdsResource(
12042       BuildEdsResource(args, DefaultEdsServiceName()));
12043   // Construct the fault injection filter config
12044   HTTPFault http_fault;
12045   auto* delay_percentage = http_fault.mutable_delay()->mutable_percentage();
12046   delay_percentage->set_numerator(kDelayPercentagePerHundred);
12047   delay_percentage->set_denominator(FractionalPercent::HUNDRED);
12048   auto* fixed_delay = http_fault.mutable_delay()->mutable_fixed_delay();
12049   fixed_delay->set_seconds(kFixedDelaySeconds);
12050   // Config fault injection via different setup
12051   SetFilterConfig(http_fault);
12052   ClientContext context;
12053   context.set_deadline(
12054       grpc_timeout_milliseconds_to_deadline(kRpcTimeoutMilliseconds));
12055   auto stream = stub_->BidiStream(&context);
12056   stream->WritesDone();
12057   auto status = stream->Finish();
12058   EXPECT_EQ(StatusCode::DEADLINE_EXCEEDED, status.error_code())
12059       << status.error_message() << ", " << status.error_details() << ", "
12060       << context.debug_error_string();
12061 }
12062
12063 class BootstrapSourceTest : public XdsEnd2endTest {
12064  public:
12065   BootstrapSourceTest() : XdsEnd2endTest(4, 1) {}
12066 };
12067
12068 TEST_P(BootstrapSourceTest, Vanilla) {
12069   SetNextResolution({});
12070   SetNextResolutionForLbChannelAllBalancers();
12071   AdsServiceImpl::EdsResourceArgs args({
12072       {"locality0", CreateEndpointsForBackends()},
12073   });
12074   balancers_[0]->ads_service()->SetEdsResource(
12075       BuildEdsResource(args, DefaultEdsServiceName()));
12076   WaitForAllBackends();
12077 }
12078
12079 #ifndef DISABLED_XDS_PROTO_IN_CC
12080 class ClientStatusDiscoveryServiceTest : public XdsEnd2endTest {
12081  public:
12082   ClientStatusDiscoveryServiceTest() : XdsEnd2endTest(1, 1) {}
12083
12084   void SetUp() override {
12085     XdsEnd2endTest::SetUp();
12086     admin_server_thread_ = absl::make_unique<AdminServerThread>(this);
12087     admin_server_thread_->Start();
12088     std::string admin_server_address = absl::StrCat(
12089         ipv6_only_ ? "[::1]:" : "127.0.0.1:", admin_server_thread_->port());
12090     admin_channel_ = grpc::CreateChannel(
12091         admin_server_address,
12092         std::make_shared<SecureChannelCredentials>(
12093             grpc_fake_transport_security_credentials_create()));
12094     csds_stub_ =
12095         envoy::service::status::v3::ClientStatusDiscoveryService::NewStub(
12096             admin_channel_);
12097     if (GetParam().use_csds_streaming()) {
12098       stream_ = csds_stub_->StreamClientStatus(&stream_context_);
12099     }
12100   }
12101
12102   void TearDown() override {
12103     if (stream_ != nullptr) {
12104       EXPECT_TRUE(stream_->WritesDone());
12105       Status status = stream_->Finish();
12106       EXPECT_TRUE(status.ok()) << "code=" << status.error_code()
12107                                << " message=" << status.error_message();
12108     }
12109     admin_server_thread_->Shutdown();
12110     XdsEnd2endTest::TearDown();
12111   }
12112
12113   envoy::service::status::v3::ClientStatusResponse FetchCsdsResponse() {
12114     envoy::service::status::v3::ClientStatusResponse response;
12115     if (!GetParam().use_csds_streaming()) {
12116       // Fetch through unary pulls
12117       ClientContext context;
12118       Status status = csds_stub_->FetchClientStatus(
12119           &context, envoy::service::status::v3::ClientStatusRequest(),
12120           &response);
12121       EXPECT_TRUE(status.ok()) << "code=" << status.error_code()
12122                                << " message=" << status.error_message();
12123     } else {
12124       // Fetch through streaming pulls
12125       EXPECT_TRUE(
12126           stream_->Write(envoy::service::status::v3::ClientStatusRequest()));
12127       EXPECT_TRUE(stream_->Read(&response));
12128     }
12129     return response;
12130   }
12131
12132  private:
12133   std::unique_ptr<AdminServerThread> admin_server_thread_;
12134   std::shared_ptr<Channel> admin_channel_;
12135   std::unique_ptr<
12136       envoy::service::status::v3::ClientStatusDiscoveryService::Stub>
12137       csds_stub_;
12138   ClientContext stream_context_;
12139   std::unique_ptr<
12140       ClientReaderWriter<envoy::service::status::v3::ClientStatusRequest,
12141                          envoy::service::status::v3::ClientStatusResponse>>
12142       stream_;
12143 };
12144
12145 MATCHER_P4(EqNode, id, user_agent_name, user_agent_version, client_features,
12146            "equals Node") {
12147   bool ok = true;
12148   ok &= ::testing::ExplainMatchResult(id, arg.id(), result_listener);
12149   ok &= ::testing::ExplainMatchResult(user_agent_name, arg.user_agent_name(),
12150                                       result_listener);
12151   ok &= ::testing::ExplainMatchResult(
12152       user_agent_version, arg.user_agent_version(), result_listener);
12153   ok &= ::testing::ExplainMatchResult(client_features, arg.client_features(),
12154                                       result_listener);
12155   return ok;
12156 }
12157
12158 MATCHER_P2(EqListenersConfigDump, version_info, dynamic_listeners,
12159            "equals ListenerConfigDump") {
12160   bool ok = true;
12161   ok &= ::testing::ExplainMatchResult(::testing::ElementsAre(),
12162                                       arg.static_listeners(), result_listener);
12163   ok &= ::testing::ExplainMatchResult(version_info, arg.version_info(),
12164                                       result_listener);
12165   ok &= ::testing::ExplainMatchResult(dynamic_listeners,
12166                                       arg.dynamic_listeners(), result_listener);
12167   return ok;
12168 }
12169
12170 MATCHER_P2(EqDynamicListenerState, version_info, listener,
12171            "equals DynamicListenerState") {
12172   bool ok = true;
12173   ok &= ::testing::ExplainMatchResult(version_info, arg.version_info(),
12174                                       result_listener);
12175   ok &=
12176       ::testing::ExplainMatchResult(listener, arg.listener(), result_listener);
12177   return ok;
12178 }
12179
12180 MATCHER_P2(EqListener, name, api_listener, "equals Listener") {
12181   bool ok = true;
12182   ok &= ::testing::ExplainMatchResult(name, arg.name(), result_listener);
12183   ok &= ::testing::ExplainMatchResult(
12184       api_listener, arg.api_listener().api_listener(), result_listener);
12185   return ok;
12186 }
12187
12188 MATCHER_P(EqHttpConnectionManagerNotRds, route_config,
12189           "equals HttpConnectionManager") {
12190   bool ok = true;
12191   ok &= ::testing::ExplainMatchResult(route_config, arg.route_config(),
12192                                       result_listener);
12193   return ok;
12194 }
12195
12196 MATCHER_P(EqRouteConfigurationName, name, "equals RouteConfiguration") {
12197   bool ok = true;
12198   ok &= ::testing::ExplainMatchResult(name, arg.name(), result_listener);
12199   return ok;
12200 }
12201
12202 MATCHER_P2(EqRouteConfiguration, name, cluster_name,
12203            "equals RouteConfiguration") {
12204   bool ok = true;
12205   ok &= ::testing::ExplainMatchResult(name, arg.name(), result_listener);
12206   ok &= ::testing::ExplainMatchResult(
12207       ::testing::ElementsAre(::testing::Property(
12208           &envoy::config::route::v3::VirtualHost::routes,
12209           ::testing::ElementsAre(::testing::Property(
12210               &envoy::config::route::v3::Route::route,
12211               ::testing::Property(
12212                   &envoy::config::route::v3::RouteAction::cluster,
12213                   cluster_name))))),
12214       arg.virtual_hosts(), result_listener);
12215   return ok;
12216 }
12217
12218 MATCHER_P(EqRoutesConfigDump, dynamic_route_configs,
12219           "equals RoutesConfigDump") {
12220   bool ok = true;
12221   ok &= ::testing::ExplainMatchResult(
12222       ::testing::ElementsAre(), arg.static_route_configs(), result_listener);
12223   ok &= ::testing::ExplainMatchResult(
12224       dynamic_route_configs, arg.dynamic_route_configs(), result_listener);
12225   return ok;
12226 }
12227
12228 MATCHER_P2(EqClustersConfigDump, version_info, dynamic_active_clusters,
12229            "equals ClustersConfigDump") {
12230   bool ok = true;
12231   ok &= ::testing::ExplainMatchResult(::testing::ElementsAre(),
12232                                       arg.static_clusters(), result_listener);
12233   ok &= ::testing::ExplainMatchResult(::testing::ElementsAre(),
12234                                       arg.dynamic_warming_clusters(),
12235                                       result_listener);
12236   ok &= ::testing::ExplainMatchResult(version_info, arg.version_info(),
12237                                       result_listener);
12238   ok &= ::testing::ExplainMatchResult(
12239       dynamic_active_clusters, arg.dynamic_active_clusters(), result_listener);
12240   return ok;
12241 }
12242
12243 MATCHER_P(EqCluster, name, "equals Cluster") {
12244   bool ok = true;
12245   ok &= ::testing::ExplainMatchResult(name, arg.name(), result_listener);
12246   return ok;
12247 }
12248
12249 MATCHER_P(EqEndpointsConfigDump, dynamic_endpoint_configs,
12250           "equals EndpointsConfigDump") {
12251   bool ok = true;
12252   ok &= ::testing::ExplainMatchResult(dynamic_endpoint_configs,
12253                                       arg.dynamic_endpoint_configs(),
12254                                       result_listener);
12255   return ok;
12256 }
12257
12258 MATCHER_P(EqEndpoint, port, "equals Endpoint") {
12259   bool ok = true;
12260   ok &= ::testing::ExplainMatchResult(
12261       port, arg.address().socket_address().port_value(), result_listener);
12262   return ok;
12263 }
12264
12265 MATCHER_P2(EqLocalityLbEndpoints, port, weight, "equals LocalityLbEndpoints") {
12266   bool ok = true;
12267   ok &= ::testing::ExplainMatchResult(
12268       ::testing::ElementsAre(::testing::Property(
12269           &envoy::config::endpoint::v3::LbEndpoint::endpoint,
12270           EqEndpoint(port))),
12271       arg.lb_endpoints(), result_listener);
12272   ok &= ::testing::ExplainMatchResult(
12273       weight, arg.load_balancing_weight().value(), result_listener);
12274   return ok;
12275 }
12276
12277 MATCHER_P(EqClusterLoadAssignmentName, cluster_name,
12278           "equals ClusterLoadAssignment") {
12279   bool ok = true;
12280   ok &= ::testing::ExplainMatchResult(cluster_name, arg.cluster_name(),
12281                                       result_listener);
12282   return ok;
12283 }
12284
12285 MATCHER_P3(EqClusterLoadAssignment, cluster_name, port, weight,
12286            "equals ClusterLoadAssignment") {
12287   bool ok = true;
12288   ok &= ::testing::ExplainMatchResult(cluster_name, arg.cluster_name(),
12289                                       result_listener);
12290   ok &= ::testing::ExplainMatchResult(
12291       ::testing::ElementsAre(EqLocalityLbEndpoints(port, weight)),
12292       arg.endpoints(), result_listener);
12293   return ok;
12294 }
12295
12296 MATCHER_P2(EqUpdateFailureState, details, version_info,
12297            "equals UpdateFailureState") {
12298   bool ok = true;
12299   ok &= ::testing::ExplainMatchResult(details, arg.details(), result_listener);
12300   ok &= ::testing::ExplainMatchResult(version_info, arg.version_info(),
12301                                       result_listener);
12302   return ok;
12303 }
12304
12305 MATCHER_P(UnpackListener, matcher, "is a Listener") {
12306   Listener config;
12307   if (!::testing::ExplainMatchResult(true, arg.UnpackTo(&config),
12308                                      result_listener)) {
12309     return false;
12310   }
12311   return ::testing::ExplainMatchResult(matcher, config, result_listener);
12312 }
12313
12314 MATCHER_P(UnpackRouteConfiguration, matcher, "is a RouteConfiguration") {
12315   RouteConfiguration config;
12316   if (!::testing::ExplainMatchResult(true, arg.UnpackTo(&config),
12317                                      result_listener)) {
12318     return false;
12319   }
12320   return ::testing::ExplainMatchResult(matcher, config, result_listener);
12321 }
12322
12323 MATCHER_P(UnpackHttpConnectionManager, matcher, "is a HttpConnectionManager") {
12324   HttpConnectionManager config;
12325   if (!::testing::ExplainMatchResult(true, arg.UnpackTo(&config),
12326                                      result_listener)) {
12327     return false;
12328   }
12329   return ::testing::ExplainMatchResult(matcher, config, result_listener);
12330 }
12331
12332 MATCHER_P(UnpackCluster, matcher, "is a Cluster") {
12333   Cluster config;
12334   if (!::testing::ExplainMatchResult(true, arg.UnpackTo(&config),
12335                                      result_listener)) {
12336     return false;
12337   }
12338   return ::testing::ExplainMatchResult(matcher, config, result_listener);
12339 }
12340
12341 MATCHER_P(UnpackClusterLoadAssignment, matcher, "is a ClusterLoadAssignment") {
12342   ClusterLoadAssignment config;
12343   if (!::testing::ExplainMatchResult(true, arg.UnpackTo(&config),
12344                                      result_listener)) {
12345     return false;
12346   }
12347   return ::testing::ExplainMatchResult(matcher, config, result_listener);
12348 }
12349
12350 MATCHER_P5(EqDynamicListener, name, version_info, client_status,
12351            api_listener_matcher, error_state, "equals DynamicListener") {
12352   bool ok = true;
12353   ok &= ::testing::ExplainMatchResult(false, arg.has_warming_state(),
12354                                       result_listener);
12355   ok &= ::testing::ExplainMatchResult(false, arg.has_draining_state(),
12356                                       result_listener);
12357   ok &= ::testing::ExplainMatchResult(name, arg.name(), result_listener);
12358   ok &= ::testing::ExplainMatchResult(client_status, arg.client_status(),
12359                                       result_listener);
12360   if (client_status == ClientResourceStatus::ACKED ||
12361       client_status == ClientResourceStatus::NACKED) {
12362     ok &= ::testing::ExplainMatchResult(
12363         EqDynamicListenerState(version_info, UnpackListener(EqListener(
12364                                                  name, api_listener_matcher))),
12365         arg.active_state(), result_listener);
12366   }
12367   ok &= ::testing::ExplainMatchResult(error_state, arg.error_state(),
12368                                       result_listener);
12369   return ok;
12370 }
12371
12372 MATCHER_P5(EqDynamicRouteConfig, name, version_info, client_status,
12373            cluster_name, error_state, "equals DynamicRouteConfig") {
12374   bool ok = true;
12375   ok &= ::testing::ExplainMatchResult(version_info, arg.version_info(),
12376                                       result_listener);
12377   if (client_status == ClientResourceStatus::REQUESTED ||
12378       client_status == ClientResourceStatus::DOES_NOT_EXIST) {
12379     ok &= ::testing::ExplainMatchResult(
12380         UnpackRouteConfiguration(EqRouteConfigurationName(name)),
12381         arg.route_config(), result_listener);
12382   } else {
12383     ok &= ::testing::ExplainMatchResult(
12384         UnpackRouteConfiguration(EqRouteConfiguration(name, cluster_name)),
12385         arg.route_config(), result_listener);
12386   }
12387   ok &= ::testing::ExplainMatchResult(error_state, arg.error_state(),
12388                                       result_listener);
12389   ok &= ::testing::ExplainMatchResult(client_status, arg.client_status(),
12390                                       result_listener);
12391   return ok;
12392 }
12393
12394 MATCHER_P4(EqDynamicCluster, name, version_info, client_status, error_state,
12395            "equals DynamicCluster") {
12396   bool ok = true;
12397   ok &= ::testing::ExplainMatchResult(UnpackCluster(EqCluster(name)),
12398                                       arg.cluster(), result_listener);
12399   ok &= ::testing::ExplainMatchResult(version_info, arg.version_info(),
12400                                       result_listener);
12401   ok &= ::testing::ExplainMatchResult(client_status, arg.client_status(),
12402                                       result_listener);
12403   ok &= ::testing::ExplainMatchResult(error_state, arg.error_state(),
12404                                       result_listener);
12405   return ok;
12406 }
12407
12408 MATCHER_P6(EqDynamicEndpointConfig, name, version_info, client_status, port,
12409            weight, error_state, "equals DynamicEndpointConfig") {
12410   bool ok = true;
12411   if (client_status == ClientResourceStatus::REQUESTED ||
12412       client_status == ClientResourceStatus::DOES_NOT_EXIST) {
12413     ok &= ::testing::ExplainMatchResult(
12414         UnpackClusterLoadAssignment(EqClusterLoadAssignmentName(name)),
12415         arg.endpoint_config(), result_listener);
12416   } else {
12417     ok &= ::testing::ExplainMatchResult(
12418         UnpackClusterLoadAssignment(
12419             EqClusterLoadAssignment(name, port, weight)),
12420         arg.endpoint_config(), result_listener);
12421   }
12422   ok &= ::testing::ExplainMatchResult(version_info, arg.version_info(),
12423                                       result_listener);
12424   ok &= ::testing::ExplainMatchResult(client_status, arg.client_status(),
12425                                       result_listener);
12426   ok &= ::testing::ExplainMatchResult(error_state, arg.error_state(),
12427                                       result_listener);
12428   return ok;
12429 }
12430
12431 MATCHER(IsRdsEnabledHCM, "is a RDS enabled HttpConnectionManager") {
12432   return ::testing::ExplainMatchResult(
12433       UnpackHttpConnectionManager(
12434           ::testing::Property(&HttpConnectionManager::has_rds, true)),
12435       arg, result_listener);
12436 }
12437
12438 MATCHER_P2(EqNoRdsHCM, route_configuration_name, cluster_name,
12439            "equals RDS disabled HttpConnectionManager") {
12440   return ::testing::ExplainMatchResult(
12441       UnpackHttpConnectionManager(EqHttpConnectionManagerNotRds(
12442           EqRouteConfiguration(route_configuration_name, cluster_name))),
12443       arg, result_listener);
12444 }
12445
12446 TEST_P(ClientStatusDiscoveryServiceTest, XdsConfigDumpVanilla) {
12447   const size_t kNumRpcs = 5;
12448   SetNextResolution({});
12449   SetNextResolutionForLbChannelAllBalancers();
12450   AdsServiceImpl::EdsResourceArgs args(
12451       {{"locality0", CreateEndpointsForBackends(0, 1)}});
12452   balancers_[0]->ads_service()->SetEdsResource(
12453       BuildEdsResource(args, DefaultEdsServiceName()));
12454   // Send several RPCs to ensure the xDS setup works
12455   CheckRpcSendOk(kNumRpcs);
12456   // Fetches the client config
12457   auto csds_response = FetchCsdsResponse();
12458   gpr_log(GPR_INFO, "xDS config dump: %s", csds_response.DebugString().c_str());
12459   EXPECT_EQ(1, csds_response.config_size());
12460   const auto& client_config = csds_response.config(0);
12461   // Validate the Node information
12462   EXPECT_THAT(client_config.node(),
12463               EqNode("xds_end2end_test", ::testing::HasSubstr("C-core"),
12464                      ::testing::HasSubstr(grpc_version_string()),
12465                      ::testing::ElementsAre(
12466                          "envoy.lb.does_not_support_overprovisioning")));
12467   // Prepare matches for RDS on or off
12468   ::testing::Matcher<google::protobuf::Any> api_listener_matcher;
12469   ::testing::Matcher<envoy::admin::v3::RoutesConfigDump>
12470       route_config_dump_matcher;
12471   if (GetParam().enable_rds_testing()) {
12472     api_listener_matcher = IsRdsEnabledHCM();
12473     route_config_dump_matcher =
12474         EqRoutesConfigDump(::testing::ElementsAre(EqDynamicRouteConfig(
12475             kDefaultRouteConfigurationName, "1", ClientResourceStatus::ACKED,
12476             kDefaultClusterName, ::testing::_)));
12477   } else {
12478     api_listener_matcher =
12479         EqNoRdsHCM(kDefaultRouteConfigurationName, kDefaultClusterName);
12480     route_config_dump_matcher = EqRoutesConfigDump(::testing::ElementsAre());
12481   }
12482   // Validate the dumped xDS configs
12483   EXPECT_THAT(
12484       client_config.xds_config(),
12485       ::testing::UnorderedElementsAre(
12486           ::testing::Property(
12487               &envoy::service::status::v3::PerXdsConfig::listener_config,
12488               EqListenersConfigDump(
12489                   "1", ::testing::ElementsAre(EqDynamicListener(
12490                            kServerName, "1", ClientResourceStatus::ACKED,
12491                            api_listener_matcher, ::testing::_)))),
12492           ::testing::Property(
12493               &envoy::service::status::v3::PerXdsConfig::route_config,
12494               route_config_dump_matcher),
12495           ::testing::Property(
12496               &envoy::service::status::v3::PerXdsConfig::cluster_config,
12497               EqClustersConfigDump(
12498                   "1", ::testing::ElementsAre(EqDynamicCluster(
12499                            kDefaultClusterName, "1",
12500                            ClientResourceStatus::ACKED, ::testing::_)))),
12501           ::testing::Property(
12502               &envoy::service::status::v3::PerXdsConfig::endpoint_config,
12503               EqEndpointsConfigDump(
12504                   ::testing::ElementsAre(EqDynamicEndpointConfig(
12505                       kDefaultEdsServiceName, "1", ClientResourceStatus::ACKED,
12506                       backends_[0]->port(), kDefaultLocalityWeight,
12507                       ::testing::_))))));
12508 }
12509
12510 TEST_P(ClientStatusDiscoveryServiceTest, XdsConfigDumpEmpty) {
12511   // The CSDS service should not fail if XdsClient is not initialized or there
12512   // is no working xDS configs.
12513   FetchCsdsResponse();
12514 }
12515
12516 TEST_P(ClientStatusDiscoveryServiceTest, XdsConfigDumpListenerError) {
12517   int kFetchConfigRetries = 3;
12518   int kFetchIntervalMilliseconds = 200;
12519   SetNextResolution({});
12520   SetNextResolutionForLbChannelAllBalancers();
12521   AdsServiceImpl::EdsResourceArgs args(
12522       {{"locality0", CreateEndpointsForBackends(0, 1)}});
12523   balancers_[0]->ads_service()->SetEdsResource(
12524       BuildEdsResource(args, DefaultEdsServiceName()));
12525   // Ensure the xDS resolver has working configs.
12526   CheckRpcSendOk();
12527   // Bad Listener should be rejected.
12528   Listener listener;
12529   listener.set_name(kServerName);
12530   balancers_[0]->ads_service()->SetLdsResource(listener);
12531   // The old xDS configs should still be effective.
12532   CheckRpcSendOk();
12533   ::testing::Matcher<google::protobuf::Any> api_listener_matcher;
12534   if (GetParam().enable_rds_testing()) {
12535     api_listener_matcher = IsRdsEnabledHCM();
12536   } else {
12537     api_listener_matcher =
12538         EqNoRdsHCM(kDefaultRouteConfigurationName, kDefaultClusterName);
12539   }
12540   for (int o = 0; o < kFetchConfigRetries; o++) {
12541     auto csds_response = FetchCsdsResponse();
12542     // Check if error state is propagated
12543     bool ok = ::testing::Value(
12544         csds_response.config(0).xds_config(),
12545         ::testing::Contains(::testing::Property(
12546             &envoy::service::status::v3::PerXdsConfig::listener_config,
12547             EqListenersConfigDump(
12548                 "1",
12549                 ::testing::ElementsAre(EqDynamicListener(
12550                     kServerName, "1", ClientResourceStatus::NACKED,
12551                     api_listener_matcher,
12552                     EqUpdateFailureState(
12553                         ::testing::HasSubstr(
12554                             "Listener has neither address nor ApiListener"),
12555                         "2")))))));
12556     if (ok) return;  // TEST PASSED!
12557     gpr_sleep_until(
12558         grpc_timeout_milliseconds_to_deadline(kFetchIntervalMilliseconds));
12559   }
12560   FAIL() << "error_state not seen in CSDS responses";
12561 }
12562
12563 TEST_P(ClientStatusDiscoveryServiceTest, XdsConfigDumpRouteError) {
12564   int kFetchConfigRetries = 3;
12565   int kFetchIntervalMilliseconds = 200;
12566   SetNextResolution({});
12567   SetNextResolutionForLbChannelAllBalancers();
12568   AdsServiceImpl::EdsResourceArgs args(
12569       {{"locality0", CreateEndpointsForBackends(0, 1)}});
12570   balancers_[0]->ads_service()->SetEdsResource(
12571       BuildEdsResource(args, DefaultEdsServiceName()));
12572   // Ensure the xDS resolver has working configs.
12573   CheckRpcSendOk();
12574   // Bad route config will be rejected.
12575   RouteConfiguration route_config;
12576   route_config.set_name(kDefaultRouteConfigurationName);
12577   route_config.add_virtual_hosts();
12578   SetRouteConfiguration(0, route_config);
12579   // The old xDS configs should still be effective.
12580   SetNextResolution({});
12581   SetNextResolutionForLbChannelAllBalancers();
12582   CheckRpcSendOk();
12583   for (int o = 0; o < kFetchConfigRetries; o++) {
12584     auto csds_response = FetchCsdsResponse();
12585     bool ok = false;
12586     if (GetParam().enable_rds_testing()) {
12587       ok = ::testing::Value(
12588           csds_response.config(0).xds_config(),
12589           ::testing::Contains(::testing::Property(
12590               &envoy::service::status::v3::PerXdsConfig::route_config,
12591               EqRoutesConfigDump(::testing::ElementsAre(EqDynamicRouteConfig(
12592                   kDefaultRouteConfigurationName, "1",
12593                   ClientResourceStatus::NACKED, kDefaultClusterName,
12594                   EqUpdateFailureState(
12595                       ::testing::HasSubstr("VirtualHost has no domains"),
12596                       "2")))))));
12597     } else {
12598       ok = ::testing::Value(
12599           csds_response.config(0).xds_config(),
12600           ::testing::Contains(::testing::Property(
12601               &envoy::service::status::v3::PerXdsConfig::listener_config,
12602               EqListenersConfigDump(
12603                   "1",
12604                   ::testing::ElementsAre(EqDynamicListener(
12605                       kServerName, "1", ClientResourceStatus::NACKED,
12606                       EqNoRdsHCM(kDefaultRouteConfigurationName,
12607                                  kDefaultClusterName),
12608                       EqUpdateFailureState(
12609                           ::testing::HasSubstr("VirtualHost has no domains"),
12610                           "2")))))));
12611     }
12612     if (ok) return;  // TEST PASSED!
12613     gpr_sleep_until(
12614         grpc_timeout_milliseconds_to_deadline(kFetchIntervalMilliseconds));
12615   }
12616   FAIL() << "error_state not seen in CSDS responses";
12617 }
12618
12619 TEST_P(ClientStatusDiscoveryServiceTest, XdsConfigDumpClusterError) {
12620   int kFetchConfigRetries = 3;
12621   int kFetchIntervalMilliseconds = 200;
12622   SetNextResolution({});
12623   SetNextResolutionForLbChannelAllBalancers();
12624   AdsServiceImpl::EdsResourceArgs args(
12625       {{"locality0", CreateEndpointsForBackends(0, 1)}});
12626   balancers_[0]->ads_service()->SetEdsResource(
12627       BuildEdsResource(args, DefaultEdsServiceName()));
12628   // Ensure the xDS resolver has working configs.
12629   CheckRpcSendOk();
12630   // Listener without any route, will be rejected.
12631   Cluster cluster;
12632   cluster.set_name(kDefaultClusterName);
12633   balancers_[0]->ads_service()->SetCdsResource(cluster);
12634   // The old xDS configs should still be effective.
12635   SetNextResolution({});
12636   SetNextResolutionForLbChannelAllBalancers();
12637   CheckRpcSendOk();
12638   for (int o = 0; o < kFetchConfigRetries; o++) {
12639     auto csds_response = FetchCsdsResponse();
12640     // Check if error state is propagated
12641     bool ok = ::testing::Value(
12642         csds_response.config(0).xds_config(),
12643         ::testing::Contains(::testing::Property(
12644             &envoy::service::status::v3::PerXdsConfig::cluster_config,
12645             EqClustersConfigDump(
12646                 "1", ::testing::ElementsAre(EqDynamicCluster(
12647                          kDefaultClusterName, "1", ClientResourceStatus::NACKED,
12648                          EqUpdateFailureState(
12649                              ::testing::HasSubstr("DiscoveryType not found"),
12650                              "2")))))));
12651     if (ok) return;  // TEST PASSED!
12652     gpr_sleep_until(
12653         grpc_timeout_milliseconds_to_deadline(kFetchIntervalMilliseconds));
12654   }
12655   FAIL() << "error_state not seen in CSDS responses";
12656 }
12657
12658 TEST_P(ClientStatusDiscoveryServiceTest, XdsConfigDumpEndpointError) {
12659   int kFetchConfigRetries = 3;
12660   int kFetchIntervalMilliseconds = 200;
12661   SetNextResolution({});
12662   SetNextResolutionForLbChannelAllBalancers();
12663   AdsServiceImpl::EdsResourceArgs args(
12664       {{"locality0", CreateEndpointsForBackends(0, 1)}});
12665   balancers_[0]->ads_service()->SetEdsResource(
12666       BuildEdsResource(args, DefaultEdsServiceName()));
12667   // Ensure the xDS resolver has working configs.
12668   CheckRpcSendOk();
12669   // Bad endpoint config will be rejected.
12670   ClusterLoadAssignment cluster_load_assignment;
12671   cluster_load_assignment.set_cluster_name(kDefaultEdsServiceName);
12672   auto* endpoints = cluster_load_assignment.add_endpoints();
12673   endpoints->mutable_load_balancing_weight()->set_value(1);
12674   auto* endpoint = endpoints->add_lb_endpoints()->mutable_endpoint();
12675   endpoint->mutable_address()->mutable_socket_address()->set_port_value(1 << 1);
12676   balancers_[0]->ads_service()->SetEdsResource(cluster_load_assignment);
12677   // The old xDS configs should still be effective.
12678   SetNextResolution({});
12679   SetNextResolutionForLbChannelAllBalancers();
12680   CheckRpcSendOk();
12681   for (int o = 0; o < kFetchConfigRetries; o++) {
12682     auto csds_response = FetchCsdsResponse();
12683
12684     // Check if error state is propagated
12685     bool ok = ::testing::Value(
12686         csds_response.config(0).xds_config(),
12687         ::testing::Contains(::testing::Property(
12688             &envoy::service::status::v3::PerXdsConfig::endpoint_config,
12689             EqEndpointsConfigDump(
12690                 ::testing::ElementsAre(EqDynamicEndpointConfig(
12691                     kDefaultEdsServiceName, "1", ClientResourceStatus::NACKED,
12692                     backends_[0]->port(), kDefaultLocalityWeight,
12693                     EqUpdateFailureState(::testing::HasSubstr("Empty locality"),
12694                                          "2")))))));
12695     if (ok) return;  // TEST PASSED!
12696     gpr_sleep_until(
12697         grpc_timeout_milliseconds_to_deadline(kFetchIntervalMilliseconds));
12698   }
12699   FAIL() << "error_state not seen in CSDS responses";
12700 }
12701
12702 TEST_P(ClientStatusDiscoveryServiceTest, XdsConfigDumpListenerRequested) {
12703   int kTimeoutMillisecond = 1000;
12704   balancers_[0]->ads_service()->UnsetResource(kLdsTypeUrl, kServerName);
12705   CheckRpcSendFailure(
12706       CheckRpcSendFailureOptions()
12707           .set_rpc_options(RpcOptions().set_timeout_ms(kTimeoutMillisecond))
12708           .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
12709   auto csds_response = FetchCsdsResponse();
12710   EXPECT_THAT(csds_response.config(0).xds_config(),
12711               ::testing::Contains(::testing::Property(
12712                   &envoy::service::status::v3::PerXdsConfig::listener_config,
12713                   EqListenersConfigDump(
12714                       ::testing::_, ::testing::ElementsAre(EqDynamicListener(
12715                                         kServerName, ::testing::_,
12716                                         ClientResourceStatus::REQUESTED,
12717                                         ::testing::_, ::testing::_))))));
12718 }
12719
12720 TEST_P(ClientStatusDiscoveryServiceTest, XdsConfigDumpClusterRequested) {
12721   int kTimeoutMillisecond = 1000;
12722   std::string kClusterName1 = "cluster-1";
12723   std::string kClusterName2 = "cluster-2";
12724   SetNextResolution({});
12725   SetNextResolutionForLbChannelAllBalancers();
12726   // Create a route config requesting two non-existing clusters
12727   RouteConfiguration route_config;
12728   route_config.set_name(kDefaultRouteConfigurationName);
12729   auto* vh = route_config.add_virtual_hosts();
12730   // The VirtualHost must match the domain name, otherwise will cause resolver
12731   // transient failure.
12732   vh->add_domains("*");
12733   auto* routes1 = vh->add_routes();
12734   routes1->mutable_match()->set_prefix("");
12735   routes1->mutable_route()->set_cluster(kClusterName1);
12736   auto* routes2 = vh->add_routes();
12737   routes2->mutable_match()->set_prefix("");
12738   routes2->mutable_route()->set_cluster(kClusterName2);
12739   SetRouteConfiguration(0, route_config);
12740   // Try to get the configs plumb through
12741   CheckRpcSendFailure(
12742       CheckRpcSendFailureOptions()
12743           .set_rpc_options(RpcOptions().set_timeout_ms(kTimeoutMillisecond))
12744           .set_expected_error_code(StatusCode::DEADLINE_EXCEEDED));
12745   auto csds_response = FetchCsdsResponse();
12746   EXPECT_THAT(csds_response.config(0).xds_config(),
12747               ::testing::Contains(::testing::Property(
12748                   &envoy::service::status::v3::PerXdsConfig::cluster_config,
12749                   EqClustersConfigDump(
12750                       ::testing::_,
12751                       ::testing::UnorderedElementsAre(
12752                           EqDynamicCluster(kClusterName1, ::testing::_,
12753                                            ClientResourceStatus::REQUESTED,
12754                                            ::testing::_),
12755                           EqDynamicCluster(kClusterName2, ::testing::_,
12756                                            ClientResourceStatus::REQUESTED,
12757                                            ::testing::_))))));
12758 }
12759
12760 class CsdsShortAdsTimeoutTest : public ClientStatusDiscoveryServiceTest {
12761   void SetUp() override {
12762     // Shorten the ADS subscription timeout to speed up the test run.
12763     xds_resource_does_not_exist_timeout_ms_ = 2000;
12764     ClientStatusDiscoveryServiceTest::SetUp();
12765   }
12766 };
12767
12768 TEST_P(CsdsShortAdsTimeoutTest, XdsConfigDumpListenerDoesNotExist) {
12769   int kTimeoutMillisecond = 1000000;  // 1000s wait for the transient failure.
12770   balancers_[0]->ads_service()->UnsetResource(kLdsTypeUrl, kServerName);
12771   CheckRpcSendFailure(
12772       CheckRpcSendFailureOptions()
12773           .set_rpc_options(RpcOptions().set_timeout_ms(kTimeoutMillisecond))
12774           .set_expected_error_code(grpc::UNAVAILABLE));
12775   auto csds_response = FetchCsdsResponse();
12776   EXPECT_THAT(csds_response.config(0).xds_config(),
12777               ::testing::Contains(::testing::Property(
12778                   &envoy::service::status::v3::PerXdsConfig::listener_config,
12779                   EqListenersConfigDump(
12780                       ::testing::_, ::testing::ElementsAre(EqDynamicListener(
12781                                         kServerName, ::testing::_,
12782                                         ClientResourceStatus::DOES_NOT_EXIST,
12783                                         ::testing::_, ::testing::_))))));
12784 }
12785
12786 TEST_P(CsdsShortAdsTimeoutTest, XdsConfigDumpRouteConfigDoesNotExist) {
12787   if (!GetParam().enable_rds_testing()) return;
12788   int kTimeoutMillisecond = 1000000;  // 1000s wait for the transient failure.
12789   SetNextResolution({});
12790   SetNextResolutionForLbChannelAllBalancers();
12791   balancers_[0]->ads_service()->UnsetResource(kRdsTypeUrl,
12792                                               kDefaultRouteConfigurationName);
12793   CheckRpcSendFailure(
12794       CheckRpcSendFailureOptions()
12795           .set_rpc_options(RpcOptions().set_timeout_ms(kTimeoutMillisecond))
12796           .set_expected_error_code(grpc::UNAVAILABLE));
12797   auto csds_response = FetchCsdsResponse();
12798   EXPECT_THAT(
12799       csds_response.config(0).xds_config(),
12800       ::testing::Contains(::testing::Property(
12801           &envoy::service::status::v3::PerXdsConfig::route_config,
12802           EqRoutesConfigDump(::testing::ElementsAre(
12803               EqDynamicRouteConfig(kDefaultRouteConfigurationName, ::testing::_,
12804                                    ClientResourceStatus::DOES_NOT_EXIST,
12805                                    ::testing::_, ::testing::_))))));
12806 }
12807
12808 TEST_P(CsdsShortAdsTimeoutTest, XdsConfigDumpClusterDoesNotExist) {
12809   int kTimeoutMillisecond = 1000000;  // 1000s wait for the transient failure.
12810   SetNextResolution({});
12811   SetNextResolutionForLbChannelAllBalancers();
12812   balancers_[0]->ads_service()->UnsetResource(kCdsTypeUrl, kDefaultClusterName);
12813   CheckRpcSendFailure(
12814       CheckRpcSendFailureOptions()
12815           .set_rpc_options(RpcOptions().set_timeout_ms(kTimeoutMillisecond))
12816           .set_expected_error_code(grpc::UNAVAILABLE));
12817   auto csds_response = FetchCsdsResponse();
12818   EXPECT_THAT(csds_response.config(0).xds_config(),
12819               ::testing::Contains(::testing::Property(
12820                   &envoy::service::status::v3::PerXdsConfig::cluster_config,
12821                   EqClustersConfigDump(::testing::_,
12822                                        ::testing::ElementsAre(EqDynamicCluster(
12823                                            kDefaultClusterName, ::testing::_,
12824                                            ClientResourceStatus::DOES_NOT_EXIST,
12825                                            ::testing::_))))));
12826 }
12827
12828 TEST_P(CsdsShortAdsTimeoutTest, XdsConfigDumpEndpointDoesNotExist) {
12829   int kTimeoutMillisecond = 1000000;  // 1000s wait for the transient failure.
12830   SetNextResolution({});
12831   SetNextResolutionForLbChannelAllBalancers();
12832   balancers_[0]->ads_service()->UnsetResource(kEdsTypeUrl,
12833                                               kDefaultEdsServiceName);
12834   CheckRpcSendFailure(
12835       CheckRpcSendFailureOptions()
12836           .set_rpc_options(RpcOptions().set_timeout_ms(kTimeoutMillisecond))
12837           .set_expected_error_code(grpc::UNAVAILABLE));
12838   auto csds_response = FetchCsdsResponse();
12839   EXPECT_THAT(
12840       csds_response.config(0).xds_config(),
12841       ::testing::Contains(::testing::Property(
12842           &envoy::service::status::v3::PerXdsConfig::endpoint_config,
12843           EqEndpointsConfigDump(::testing::ElementsAre(EqDynamicEndpointConfig(
12844               kDefaultEdsServiceName, ::testing::_,
12845               ClientResourceStatus::DOES_NOT_EXIST, ::testing::_, ::testing::_,
12846               ::testing::_))))));
12847 }
12848 #endif  // DISABLED_XDS_PROTO_IN_CC
12849
12850 std::string TestTypeName(const ::testing::TestParamInfo<TestType>& info) {
12851   return info.param.AsString();
12852 }
12853
12854 // Run with all combinations of xds/fake resolver and enabling load reporting.
12855 INSTANTIATE_TEST_SUITE_P(
12856     XdsTest, BasicTest,
12857     ::testing::Values(
12858         TestType(), TestType().set_enable_load_reporting(),
12859         TestType().set_use_fake_resolver(),
12860         TestType().set_use_fake_resolver().set_enable_load_reporting()),
12861     &TestTypeName);
12862
12863 // Run with both fake resolver and xds resolver.
12864 // Don't run with load reporting or v2 or RDS, since they are irrelevant to
12865 // the tests.
12866 INSTANTIATE_TEST_SUITE_P(XdsTest, SecureNamingTest,
12867                          ::testing::Values(TestType(),
12868                                            TestType().set_use_fake_resolver()),
12869                          &TestTypeName);
12870
12871 // LDS depends on XdsResolver.
12872 INSTANTIATE_TEST_SUITE_P(XdsTest, LdsTest, ::testing::Values(TestType()),
12873                          &TestTypeName);
12874 INSTANTIATE_TEST_SUITE_P(XdsTest, LdsV2Test,
12875                          ::testing::Values(TestType().set_use_v2()),
12876                          &TestTypeName);
12877
12878 // LDS/RDS commmon tests depend on XdsResolver.
12879 INSTANTIATE_TEST_SUITE_P(
12880     XdsTest, LdsRdsTest,
12881     ::testing::Values(TestType(), TestType().set_enable_rds_testing(),
12882                       // Also test with xDS v2.
12883                       TestType().set_enable_rds_testing().set_use_v2()),
12884     &TestTypeName);
12885
12886 // CDS depends on XdsResolver.
12887 INSTANTIATE_TEST_SUITE_P(
12888     XdsTest, CdsTest,
12889     ::testing::Values(TestType(), TestType().set_enable_load_reporting()),
12890     &TestTypeName);
12891
12892 // CDS depends on XdsResolver.
12893 // Security depends on v3.
12894 // Not enabling load reporting or RDS, since those are irrelevant to these
12895 // tests.
12896 INSTANTIATE_TEST_SUITE_P(
12897     XdsTest, XdsSecurityTest,
12898     ::testing::Values(TestType().set_use_xds_credentials()), &TestTypeName);
12899
12900 // We are only testing the server here.
12901 // Run with bootstrap from env var, so that we use a global XdsClient
12902 // instance.  Otherwise, we would need to use a separate fake resolver
12903 // result generator on the client and server sides.
12904 INSTANTIATE_TEST_SUITE_P(XdsTest, XdsEnabledServerTest,
12905                          ::testing::Values(TestType().set_bootstrap_source(
12906                              TestType::kBootstrapFromEnvVar)),
12907                          &TestTypeName);
12908
12909 // We are only testing the server here.
12910 INSTANTIATE_TEST_SUITE_P(XdsTest, XdsServerSecurityTest,
12911                          ::testing::Values(TestType()
12912                                                .set_use_fake_resolver()
12913                                                .set_use_xds_credentials()),
12914                          &TestTypeName);
12915
12916 // We are only testing the server here.
12917 INSTANTIATE_TEST_SUITE_P(XdsTest, XdsEnabledServerStatusNotificationTest,
12918                          ::testing::Values(TestType()
12919                                                .set_use_fake_resolver()
12920                                                .set_use_xds_credentials()),
12921                          &TestTypeName);
12922
12923 // We are only testing the server here.
12924 INSTANTIATE_TEST_SUITE_P(XdsTest, XdsServerFilterChainMatchTest,
12925                          ::testing::Values(TestType()
12926                                                .set_use_fake_resolver()
12927                                                .set_use_xds_credentials()),
12928                          &TestTypeName);
12929
12930 // EDS could be tested with or without XdsResolver, but the tests would
12931 // be the same either way, so we test it only with XdsResolver.
12932 INSTANTIATE_TEST_SUITE_P(
12933     XdsTest, EdsTest,
12934     ::testing::Values(TestType(), TestType().set_enable_load_reporting()),
12935     &TestTypeName);
12936
12937 // Test initial resource timeouts for each resource type.
12938 // Do this only for XdsResolver with RDS enabled, so that we can test
12939 // all resource types.
12940 // Run with V3 only, since the functionality is no different in V2.
12941 INSTANTIATE_TEST_SUITE_P(XdsTest, TimeoutTest,
12942                          ::testing::Values(TestType().set_enable_rds_testing()),
12943                          &TestTypeName);
12944
12945 // XdsResolverOnlyTest depends on XdsResolver.
12946 INSTANTIATE_TEST_SUITE_P(
12947     XdsTest, XdsResolverOnlyTest,
12948     ::testing::Values(TestType(), TestType().set_enable_load_reporting()),
12949     &TestTypeName);
12950
12951 // Runs with bootstrap from env var, so that there's a global XdsClient.
12952 INSTANTIATE_TEST_SUITE_P(
12953     XdsTest, GlobalXdsClientTest,
12954     ::testing::Values(
12955         TestType().set_bootstrap_source(TestType::kBootstrapFromEnvVar),
12956         TestType()
12957             .set_bootstrap_source(TestType::kBootstrapFromEnvVar)
12958             .set_enable_load_reporting()),
12959     &TestTypeName);
12960
12961 // XdsResolverLoadReprtingOnlyTest depends on XdsResolver and load reporting.
12962 INSTANTIATE_TEST_SUITE_P(
12963     XdsTest, XdsResolverLoadReportingOnlyTest,
12964     ::testing::Values(TestType().set_enable_load_reporting()), &TestTypeName);
12965
12966 INSTANTIATE_TEST_SUITE_P(
12967     XdsTest, LocalityMapTest,
12968     ::testing::Values(
12969         TestType(), TestType().set_enable_load_reporting(),
12970         TestType().set_use_fake_resolver(),
12971         TestType().set_use_fake_resolver().set_enable_load_reporting()),
12972     &TestTypeName);
12973
12974 INSTANTIATE_TEST_SUITE_P(
12975     XdsTest, FailoverTest,
12976     ::testing::Values(
12977         TestType(), TestType().set_enable_load_reporting(),
12978         TestType().set_use_fake_resolver(),
12979         TestType().set_use_fake_resolver().set_enable_load_reporting()),
12980     &TestTypeName);
12981
12982 INSTANTIATE_TEST_SUITE_P(
12983     XdsTest, DropTest,
12984     ::testing::Values(
12985         TestType(), TestType().set_enable_load_reporting(),
12986         TestType().set_use_fake_resolver(),
12987         TestType().set_use_fake_resolver().set_enable_load_reporting()),
12988     &TestTypeName);
12989
12990 INSTANTIATE_TEST_SUITE_P(
12991     XdsTest, BalancerUpdateTest,
12992     ::testing::Values(
12993         TestType().set_use_fake_resolver(),
12994         TestType().set_use_fake_resolver().set_enable_load_reporting(),
12995         TestType().set_enable_load_reporting()),
12996     &TestTypeName);
12997
12998 // Load reporting tests are not run with load reporting disabled.
12999 INSTANTIATE_TEST_SUITE_P(
13000     XdsTest, ClientLoadReportingTest,
13001     ::testing::Values(
13002         TestType().set_enable_load_reporting(),
13003         TestType().set_enable_load_reporting().set_use_fake_resolver()),
13004     &TestTypeName);
13005
13006 // Load reporting tests are not run with load reporting disabled.
13007 INSTANTIATE_TEST_SUITE_P(
13008     XdsTest, ClientLoadReportingWithDropTest,
13009     ::testing::Values(
13010         TestType().set_enable_load_reporting(),
13011         TestType().set_enable_load_reporting().set_use_fake_resolver()),
13012     &TestTypeName);
13013
13014 INSTANTIATE_TEST_SUITE_P(
13015     XdsTest, FaultInjectionTest,
13016     ::testing::Values(
13017         TestType(), TestType().set_enable_rds_testing(),
13018         TestType().set_filter_config_setup(
13019             TestType::FilterConfigSetup::kRouteOverride),
13020         TestType().set_enable_rds_testing().set_filter_config_setup(
13021             TestType::FilterConfigSetup::kRouteOverride)),
13022     &TestTypeName);
13023
13024 INSTANTIATE_TEST_SUITE_P(
13025     XdsTest, BootstrapSourceTest,
13026     ::testing::Values(
13027         TestType().set_bootstrap_source(TestType::kBootstrapFromEnvVar),
13028         TestType().set_bootstrap_source(TestType::kBootstrapFromFile)),
13029     &TestTypeName);
13030
13031 #ifndef DISABLED_XDS_PROTO_IN_CC
13032 // Run CSDS tests with RDS enabled and disabled.
13033 // These need to run with the bootstrap from an env var instead of from
13034 // a channel arg, since there needs to be a global XdsClient instance.
13035 INSTANTIATE_TEST_SUITE_P(
13036     XdsTest, ClientStatusDiscoveryServiceTest,
13037     ::testing::Values(
13038         TestType().set_bootstrap_source(TestType::kBootstrapFromEnvVar),
13039         TestType()
13040             .set_bootstrap_source(TestType::kBootstrapFromEnvVar)
13041             .set_enable_rds_testing(),
13042         TestType()
13043             .set_bootstrap_source(TestType::kBootstrapFromEnvVar)
13044             .set_use_csds_streaming(),
13045         TestType()
13046             .set_bootstrap_source(TestType::kBootstrapFromEnvVar)
13047             .set_enable_rds_testing()
13048             .set_use_csds_streaming()),
13049     &TestTypeName);
13050 INSTANTIATE_TEST_SUITE_P(
13051     XdsTest, CsdsShortAdsTimeoutTest,
13052     ::testing::Values(
13053         TestType().set_bootstrap_source(TestType::kBootstrapFromEnvVar),
13054         TestType()
13055             .set_bootstrap_source(TestType::kBootstrapFromEnvVar)
13056             .set_enable_rds_testing(),
13057         TestType()
13058             .set_bootstrap_source(TestType::kBootstrapFromEnvVar)
13059             .set_use_csds_streaming(),
13060         TestType()
13061             .set_bootstrap_source(TestType::kBootstrapFromEnvVar)
13062             .set_enable_rds_testing()
13063             .set_use_csds_streaming()),
13064     &TestTypeName);
13065 #endif  // DISABLED_XDS_PROTO_IN_CC
13066
13067 }  // namespace
13068 }  // namespace testing
13069 }  // namespace grpc
13070
13071 int main(int argc, char** argv) {
13072   grpc::testing::TestEnvironment env(argc, argv);
13073   ::testing::InitGoogleTest(&argc, argv);
13074   grpc::testing::WriteBootstrapFiles();
13075   // Make the backup poller poll very frequently in order to pick up
13076   // updates from all the subchannels's FDs.
13077   GPR_GLOBAL_CONFIG_SET(grpc_client_channel_backup_poll_interval_ms, 1);
13078 #if TARGET_OS_IPHONE
13079   // Workaround Apple CFStream bug
13080   gpr_setenv("grpc_cfstream", "0");
13081 #endif
13082   grpc_core::CertificateProviderRegistry::RegisterCertificateProviderFactory(
13083       absl::make_unique<grpc::testing::FakeCertificateProviderFactory>(
13084           "fake1", &grpc::testing::g_fake1_cert_data_map));
13085   grpc_core::CertificateProviderRegistry::RegisterCertificateProviderFactory(
13086       absl::make_unique<grpc::testing::FakeCertificateProviderFactory>(
13087           "fake2", &grpc::testing::g_fake2_cert_data_map));
13088   grpc_init();
13089   grpc_core::XdsHttpFilterRegistry::RegisterFilter(
13090       absl::make_unique<grpc::testing::NoOpHttpFilter>(
13091           "grpc.testing.client_only_http_filter", true, false),
13092       {"grpc.testing.client_only_http_filter"});
13093   grpc_core::XdsHttpFilterRegistry::RegisterFilter(
13094       absl::make_unique<grpc::testing::NoOpHttpFilter>(
13095           "grpc.testing.server_only_http_filter", false, true),
13096       {"grpc.testing.server_only_http_filter"});
13097   const auto result = RUN_ALL_TESTS();
13098   grpc_shutdown();
13099   return result;
13100 }