test/core/transport/chttp2/too_many_pings_test.cc

   1 /*
   2  *
   3  * Copyright 2020 gRPC authors.
   4  *
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  *
  17  */
  18
  19 #include <grpc/support/port_platform.h>
  20
  21 #include <gmock/gmock.h>
  22 #include <stdlib.h>
  23 #include <string.h>
  24 #include <functional>
  25 #include <set>
  26 #include <thread>
  27
  28 #include "absl/strings/str_cat.h"
  29
  30 #include <grpc/grpc.h>
  31 #include <grpc/grpc_security.h>
  32 #include <grpc/impl/codegen/grpc_types.h>
  33 #include <grpc/slice.h>
  34 #include <grpc/support/alloc.h>
  35 #include <grpc/support/log.h>
  36 #include <grpc/support/string_util.h>
  37 #include <grpc/support/time.h>
  38
  39 #include <grpcpp/impl/codegen/service_type.h>
  40 #include <grpcpp/server_builder.h>
  41
  42 #include "src/core/ext/filters/client_channel/resolver/fake/fake_resolver.h"
  43 #include "src/core/lib/gpr/useful.h"
  44 #include "src/core/lib/gprpp/host_port.h"
  45 #include "src/core/lib/gprpp/thd.h"
  46 #include "src/core/lib/iomgr/error.h"
  47 #include "src/core/lib/iomgr/parse_address.h"
  48 #include "src/core/lib/security/credentials/alts/alts_credentials.h"
  49 #include "src/core/lib/security/credentials/credentials.h"
  50 #include "src/core/lib/security/security_connector/alts/alts_security_connector.h"
  51 #include "src/core/lib/slice/slice_string_helpers.h"
  52 #include "src/core/lib/surface/channel.h"
  53
  54 #include "test/core/util/memory_counters.h"
  55 #include "test/core/util/port.h"
  56 #include "test/core/util/test_config.h"
  57
  58 #include "test/core/end2end/cq_verifier.h"
  59
  60 namespace {
  61
  62 void* tag(int i) { return (void*)static_cast<intptr_t>(i); }
  63
  64 // Perform a simple RPC where the server cancels the request with
  65 // grpc_call_cancel_with_status
  66 grpc_status_code PerformCall(grpc_channel* channel, grpc_server* server,
  67                              grpc_completion_queue* cq) {
  68   grpc_call* c;
  69   grpc_call* s;
  70   cq_verifier* cqv = cq_verifier_create(cq);
  71   grpc_op ops[6];
  72   grpc_op* op;
  73   grpc_metadata_array trailing_metadata_recv;
  74   grpc_metadata_array request_metadata_recv;
  75   grpc_call_details call_details;
  76   grpc_status_code status;
  77   grpc_call_error error;
  78   grpc_slice details;
  79   gpr_timespec deadline = grpc_timeout_seconds_to_deadline(5);
  80   // Start a call
  81   c = grpc_channel_create_call(channel, nullptr, GRPC_PROPAGATE_DEFAULTS, cq,
  82                                grpc_slice_from_static_string("/foo"), nullptr,
  83                                deadline, nullptr);
  84   GPR_ASSERT(c);
  85   grpc_metadata_array_init(&trailing_metadata_recv);
  86   grpc_metadata_array_init(&request_metadata_recv);
  87   grpc_call_details_init(&call_details);
  88   memset(ops, 0, sizeof(ops));
  89   op = ops;
  90   op->op = GRPC_OP_SEND_INITIAL_METADATA;
  91   op->data.send_initial_metadata.count = 0;
  92   op->flags = 0;
  93   op->reserved = nullptr;
  94   op++;
  95   op->op = GRPC_OP_RECV_STATUS_ON_CLIENT;
  96   op->data.recv_status_on_client.trailing_metadata = &trailing_metadata_recv;
  97   op->data.recv_status_on_client.status = &status;
  98   op->data.recv_status_on_client.status_details = &details;
  99   op->flags = 0;
 100   op->reserved = nullptr;
 101   op++;
 102   error = grpc_call_start_batch(c, ops, static_cast<size_t>(op - ops), tag(1),
 103                                 nullptr);
 104   GPR_ASSERT(GRPC_CALL_OK == error);
 105   // Request a call on the server
 106   error = grpc_server_request_call(server, &s, &call_details,
 107                                    &request_metadata_recv, cq, cq, tag(101));
 108   GPR_ASSERT(GRPC_CALL_OK == error);
 109   CQ_EXPECT_COMPLETION(cqv, tag(101), 1);
 110   cq_verify(cqv);
 111   grpc_call_cancel_with_status(s, GRPC_STATUS_PERMISSION_DENIED, "test status",
 112                                nullptr);
 113   CQ_EXPECT_COMPLETION(cqv, tag(1), 1);
 114   cq_verify(cqv);
 115   // cleanup
 116   grpc_slice_unref(details);
 117   grpc_metadata_array_destroy(&trailing_metadata_recv);
 118   grpc_metadata_array_destroy(&request_metadata_recv);
 119   grpc_call_details_destroy(&call_details);
 120   grpc_call_unref(c);
 121   grpc_call_unref(s);
 122   cq_verifier_destroy(cqv);
 123   return status;
 124 }
 125
 126 // Test that sending a lot of RPCs that are cancelled by the server doesn't
 127 // result in too many pings due to the pings sent by BDP.
 128 TEST(TooManyPings, TestLotsOfServerCancelledRpcsDoesntGiveTooManyPings) {
 129   grpc_completion_queue* cq = grpc_completion_queue_create_for_next(nullptr);
 130   // create the server
 131   grpc_server* server = grpc_server_create(nullptr, nullptr);
 132   std::string server_address =
 133       grpc_core::JoinHostPort("localhost", grpc_pick_unused_port_or_die());
 134   grpc_server_register_completion_queue(server, cq, nullptr);
 135   GPR_ASSERT(
 136       grpc_server_add_insecure_http2_port(server, server_address.c_str()));
 137   grpc_server_start(server);
 138   // create the channel (bdp pings are enabled by default)
 139   grpc_channel* channel = grpc_insecure_channel_create(
 140       server_address.c_str(), nullptr /* channel args */, nullptr);
 141   std::map<grpc_status_code, int> statuses_and_counts;
 142   const int kNumTotalRpcs = 1e5;
 143   // perform an RPC
 144   gpr_log(GPR_INFO,
 145           "Performing %d total RPCs and expecting them all to receive status "
 146           "PERMISSION_DENIED (%d)",
 147           kNumTotalRpcs, GRPC_STATUS_PERMISSION_DENIED);
 148   for (int i = 0; i < kNumTotalRpcs; i++) {
 149     grpc_status_code status = PerformCall(channel, server, cq);
 150     statuses_and_counts[status] += 1;
 151   }
 152   int num_not_cancelled = 0;
 153   for (auto itr = statuses_and_counts.begin(); itr != statuses_and_counts.end();
 154        itr++) {
 155     if (itr->first != GRPC_STATUS_PERMISSION_DENIED) {
 156       num_not_cancelled += itr->second;
 157     }
 158     gpr_log(GPR_INFO, "%d / %d RPCs received status code: %d", itr->second,
 159             kNumTotalRpcs, itr->first);
 160   }
 161   if (num_not_cancelled > 0) {
 162     gpr_log(GPR_ERROR,
 163             "Expected all RPCs to receive status PERMISSION_DENIED (%d) but %d "
 164             "received other status codes",
 165             GRPC_STATUS_PERMISSION_DENIED, num_not_cancelled);
 166     FAIL();
 167   }
 168   // shutdown and destroy the client and server
 169   grpc_channel_destroy(channel);
 170   grpc_server_shutdown_and_notify(server, cq, nullptr);
 171   grpc_completion_queue_shutdown(cq);
 172   while (grpc_completion_queue_next(cq, gpr_inf_future(GPR_CLOCK_REALTIME),
 173                                     nullptr)
 174              .type != GRPC_QUEUE_SHUTDOWN)
 175     ;
 176   grpc_server_destroy(server);
 177   grpc_completion_queue_destroy(cq);
 178 }
 179
 180 // Perform a simple RPC where the client makes a request, and both the client
 181 // and server continue reading so that gRPC can send and receive keepalive
 182 // pings.
 183 grpc_status_code PerformWaitingCall(grpc_channel* channel, grpc_server* server,
 184                                     grpc_completion_queue* cq) {
 185   grpc_call* c;
 186   grpc_call* s;
 187   cq_verifier* cqv = cq_verifier_create(cq);
 188   grpc_op ops[6];
 189   grpc_op* op;
 190   grpc_metadata_array trailing_metadata_recv;
 191   grpc_metadata_array request_metadata_recv;
 192   grpc_call_details call_details;
 193   grpc_status_code status;
 194   grpc_call_error error;
 195   grpc_slice details;
 196   gpr_timespec deadline = grpc_timeout_seconds_to_deadline(15);
 197   // Start a call
 198   c = grpc_channel_create_call(channel, nullptr, GRPC_PROPAGATE_DEFAULTS, cq,
 199                                grpc_slice_from_static_string("/foo"), nullptr,
 200                                deadline, nullptr);
 201   GPR_ASSERT(c);
 202   grpc_metadata_array_init(&trailing_metadata_recv);
 203   grpc_metadata_array_init(&request_metadata_recv);
 204   grpc_call_details_init(&call_details);
 205   memset(ops, 0, sizeof(ops));
 206   op = ops;
 207   op->op = GRPC_OP_SEND_INITIAL_METADATA;
 208   op->data.send_initial_metadata.count = 0;
 209   op->flags = 0;
 210   op->reserved = nullptr;
 211   op++;
 212   op->op = GRPC_OP_RECV_STATUS_ON_CLIENT;
 213   op->data.recv_status_on_client.trailing_metadata = &trailing_metadata_recv;
 214   op->data.recv_status_on_client.status = &status;
 215   op->data.recv_status_on_client.status_details = &details;
 216   op->flags = 0;
 217   op->reserved = nullptr;
 218   op++;
 219   error = grpc_call_start_batch(c, ops, static_cast<size_t>(op - ops), tag(1),
 220                                 nullptr);
 221   GPR_ASSERT(GRPC_CALL_OK == error);
 222   // Request a call on the server
 223   error = grpc_server_request_call(server, &s, &call_details,
 224                                    &request_metadata_recv, cq, cq, tag(101));
 225   GPR_ASSERT(GRPC_CALL_OK == error);
 226   CQ_EXPECT_COMPLETION(cqv, tag(101), 1);
 227   cq_verify(cqv);
 228   // Since the server is configured to allow only a single ping strike, it would
 229   // take 3 pings to trigger the GOAWAY frame with "too_many_pings" from the
 230   // server. (The second ping from the client would be the first bad ping sent
 231   // too quickly leading to a ping strike and the third ping would lead to the
 232   // GOAWAY.) If the client settings match with the server's settings, there
 233   // won't be a bad ping, and the call will end due to the deadline expiring
 234   // instead.
 235   CQ_EXPECT_COMPLETION(cqv, tag(1), 1);
 236   // The call will end after this
 237   cq_verify(cqv, 60);
 238   // cleanup
 239   grpc_slice_unref(details);
 240   grpc_metadata_array_destroy(&trailing_metadata_recv);
 241   grpc_metadata_array_destroy(&request_metadata_recv);
 242   grpc_call_details_destroy(&call_details);
 243   grpc_call_unref(c);
 244   grpc_call_unref(s);
 245   cq_verifier_destroy(cqv);
 246   return status;
 247 }
 248
 249 // Shuts down and destroys the server.
 250 void ServerShutdownAndDestroy(grpc_server* server, grpc_completion_queue* cq) {
 251   // Shutdown and destroy server
 252   grpc_server_shutdown_and_notify(server, cq, (void*)(1000));
 253   while (grpc_completion_queue_next(cq, gpr_inf_future(GPR_CLOCK_REALTIME),
 254                                     nullptr)
 255              .tag != (void*)(1000))
 256     ;
 257   grpc_server_destroy(server);
 258 }
 259
 260 void VerifyChannelReady(grpc_channel* channel, grpc_completion_queue* cq) {
 261   grpc_connectivity_state state =
 262       grpc_channel_check_connectivity_state(channel, 1 /* try_to_connect */);
 263   while (state != GRPC_CHANNEL_READY) {
 264     grpc_channel_watch_connectivity_state(
 265         channel, state, grpc_timeout_seconds_to_deadline(5), cq, nullptr);
 266     grpc_completion_queue_next(cq, grpc_timeout_seconds_to_deadline(5),
 267                                nullptr);
 268     state = grpc_channel_check_connectivity_state(channel, 0);
 269   }
 270 }
 271
 272 void VerifyChannelDisconnected(grpc_channel* channel,
 273                                grpc_completion_queue* cq) {
 274   // Verify channel gets disconnected. Use a ping to make sure that clients
 275   // tries sending/receiving bytes if the channel is connected.
 276   grpc_channel_ping(channel, cq, (void*)(2000), nullptr);
 277   grpc_event ev = grpc_completion_queue_next(
 278       cq, grpc_timeout_seconds_to_deadline(5), nullptr);
 279   GPR_ASSERT(ev.type == GRPC_OP_COMPLETE);
 280   GPR_ASSERT(ev.tag == (void*)(2000));
 281   GPR_ASSERT(ev.success == 0);
 282   GPR_ASSERT(grpc_channel_check_connectivity_state(channel, 0) !=
 283              GRPC_CHANNEL_READY);
 284 }
 285
 286 class KeepaliveThrottlingTest : public ::testing::Test {
 287  protected:
 288   // Starts the server and makes sure that the channel is able to get connected.
 289   grpc_server* ServerStart(const char* addr, grpc_completion_queue* cq) {
 290     // Set up server channel args to expect pings at an interval of 5 seconds
 291     // and use a single ping strike
 292     grpc_arg server_args[] = {
 293         grpc_channel_arg_integer_create(
 294             const_cast<char*>(
 295                 GRPC_ARG_HTTP2_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS),
 296             5 * 1000),
 297         grpc_channel_arg_integer_create(
 298             const_cast<char*>(GRPC_ARG_HTTP2_MAX_PING_STRIKES), 1)};
 299     grpc_channel_args server_channel_args = {GPR_ARRAY_SIZE(server_args),
 300                                              server_args};
 301     // Create server
 302     grpc_server* server = grpc_server_create(&server_channel_args, nullptr);
 303     grpc_server_register_completion_queue(server, cq, nullptr);
 304     GPR_ASSERT(grpc_server_add_insecure_http2_port(server, addr));
 305     grpc_server_start(server);
 306     return server;
 307   }
 308 };
 309
 310 TEST_F(KeepaliveThrottlingTest, KeepaliveThrottlingMultipleChannels) {
 311   grpc_completion_queue* cq = grpc_completion_queue_create_for_next(nullptr);
 312   std::string server_address =
 313       grpc_core::JoinHostPort("127.0.0.1", grpc_pick_unused_port_or_die());
 314   grpc_server* server = ServerStart(server_address.c_str(), cq);
 315   // create two channel with a keepalive ping interval of 1 second.
 316   grpc_arg client_args[] = {
 317       grpc_channel_arg_integer_create(
 318           const_cast<char*>(GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA), 0),
 319       grpc_channel_arg_integer_create(
 320           const_cast<char*>(
 321               GRPC_ARG_HTTP2_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS),
 322           0),
 323       grpc_channel_arg_integer_create(
 324           const_cast<char*>(GRPC_ARG_KEEPALIVE_TIME_MS), 1 * 1000),
 325       grpc_channel_arg_integer_create(
 326           const_cast<char*>(GRPC_ARG_HTTP2_BDP_PROBE), 0)};
 327   grpc_channel_args client_channel_args = {GPR_ARRAY_SIZE(client_args),
 328                                            client_args};
 329   grpc_channel* channel = grpc_insecure_channel_create(
 330       server_address.c_str(), &client_channel_args, nullptr);
 331   grpc_channel* channel_dup = grpc_insecure_channel_create(
 332       server_address.c_str(), &client_channel_args, nullptr);
 333   int expected_keepalive_time_sec = 1;
 334   // We need 3 GOAWAY frames to throttle the keepalive time from 1 second to 8
 335   // seconds (> 5sec).
 336   for (int i = 0; i < 3; i++) {
 337     gpr_log(GPR_INFO, "Expected keepalive time : %d",
 338             expected_keepalive_time_sec);
 339     EXPECT_EQ(PerformWaitingCall(channel, server, cq), GRPC_STATUS_UNAVAILABLE);
 340     expected_keepalive_time_sec *= 2;
 341   }
 342   gpr_log(
 343       GPR_INFO,
 344       "Client keepalive time %d should now be in sync with the server settings",
 345       expected_keepalive_time_sec);
 346   EXPECT_EQ(PerformWaitingCall(channel, server, cq),
 347             GRPC_STATUS_DEADLINE_EXCEEDED);
 348   // Since the subchannel is shared, the second channel should also have
 349   // keepalive settings in sync with the server.
 350   gpr_log(GPR_INFO, "Now testing second channel sharing the same subchannel");
 351   EXPECT_EQ(PerformWaitingCall(channel_dup, server, cq),
 352             GRPC_STATUS_DEADLINE_EXCEEDED);
 353   // shutdown and destroy the client and server
 354   grpc_channel_destroy(channel);
 355   grpc_channel_destroy(channel_dup);
 356   ServerShutdownAndDestroy(server, cq);
 357   grpc_completion_queue_shutdown(cq);
 358   while (grpc_completion_queue_next(cq, gpr_inf_future(GPR_CLOCK_REALTIME),
 359                                     nullptr)
 360              .type != GRPC_QUEUE_SHUTDOWN)
 361     ;
 362   grpc_completion_queue_destroy(cq);
 363 }
 364
 365 grpc_core::Resolver::Result BuildResolverResult(
 366     const std::vector<std::string>& addresses) {
 367   grpc_core::Resolver::Result result;
 368   for (const auto& address_str : addresses) {
 369     grpc_uri* uri = grpc_uri_parse(address_str.c_str(), true);
 370     if (uri == nullptr) {
 371       gpr_log(GPR_ERROR, "Failed to parse uri:%s", address_str.c_str());
 372       GPR_ASSERT(0);
 373     }
 374     grpc_resolved_address address;
 375     GPR_ASSERT(grpc_parse_uri(uri, &address));
 376     result.addresses.emplace_back(address.addr, address.len, nullptr);
 377     grpc_uri_destroy(uri);
 378   }
 379   return result;
 380 }
 381
 382 // Tests that when new subchannels are created due to a change in resolved
 383 // addresses, the new subchannels use the updated keepalive time.
 384 TEST_F(KeepaliveThrottlingTest, NewSubchannelsUseUpdatedKeepaliveTime) {
 385   grpc_core::ExecCtx exec_ctx;
 386   grpc_completion_queue* cq = grpc_completion_queue_create_for_next(nullptr);
 387   std::string server_address1 =
 388       grpc_core::JoinHostPort("127.0.0.1", grpc_pick_unused_port_or_die());
 389   std::string server_address2 =
 390       grpc_core::JoinHostPort("127.0.0.1", grpc_pick_unused_port_or_die());
 391   grpc_server* server1 = ServerStart(server_address1.c_str(), cq);
 392   grpc_server* server2 = ServerStart(server_address2.c_str(), cq);
 393   // create a single channel with multiple subchannels with a keepalive ping
 394   // interval of 1 second. To get finer control on subchannel connection times,
 395   // we are using pick_first instead of round_robin and using the fake resolver
 396   // response generator to switch between the two.
 397   auto response_generator =
 398       grpc_core::MakeRefCounted<grpc_core::FakeResolverResponseGenerator>();
 399   grpc_arg client_args[] = {
 400       grpc_channel_arg_integer_create(
 401           const_cast<char*>(GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA), 0),
 402       grpc_channel_arg_integer_create(
 403           const_cast<char*>(
 404               GRPC_ARG_HTTP2_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS),
 405           0),
 406       grpc_channel_arg_integer_create(
 407           const_cast<char*>(GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS), 0),
 408       grpc_channel_arg_integer_create(
 409           const_cast<char*>(GRPC_ARG_KEEPALIVE_TIME_MS), 1 * 1000),
 410       grpc_channel_arg_integer_create(
 411           const_cast<char*>(GRPC_ARG_HTTP2_BDP_PROBE), 0),
 412       grpc_core::FakeResolverResponseGenerator::MakeChannelArg(
 413           response_generator.get())};
 414   grpc_channel_args client_channel_args = {GPR_ARRAY_SIZE(client_args),
 415                                            client_args};
 416   grpc_channel* channel =
 417       grpc_insecure_channel_create("fake:///", &client_channel_args, nullptr);
 418   // For a single subchannel 3 GOAWAYs would be sufficient to increase the
 419   // keepalive time from 1 second to beyond 5 seconds. Even though we are
 420   // alternating between two subchannels, 3 GOAWAYs should still be enough since
 421   // the channel should start all new transports with the new keepalive value
 422   // (even those from a different subchannel).
 423   int expected_keepalive_time_sec = 1;
 424   for (int i = 0; i < 3; i++) {
 425     gpr_log(GPR_INFO, "Expected keepalive time : %d",
 426             expected_keepalive_time_sec);
 427     response_generator->SetResponse(BuildResolverResult({absl::StrCat(
 428         "ipv4:", i % 2 == 0 ? server_address1 : server_address2)}));
 429     // ExecCtx::Flush() might not be enough to make sure that the resolver
 430     // result has been propagated, so sleep for a bit.
 431     grpc_core::ExecCtx::Get()->Flush();
 432     gpr_sleep_until(grpc_timeout_seconds_to_deadline(1));
 433     EXPECT_EQ(PerformWaitingCall(channel, i % 2 == 0 ? server1 : server2, cq),
 434               GRPC_STATUS_UNAVAILABLE);
 435     expected_keepalive_time_sec *= 2;
 436   }
 437   gpr_log(
 438       GPR_INFO,
 439       "Client keepalive time %d should now be in sync with the server settings",
 440       expected_keepalive_time_sec);
 441   response_generator->SetResponse(
 442       BuildResolverResult({absl::StrCat("ipv4:", server_address2)}));
 443   grpc_core::ExecCtx::Get()->Flush();
 444   gpr_sleep_until(grpc_timeout_seconds_to_deadline(1));
 445   EXPECT_EQ(PerformWaitingCall(channel, server2, cq),
 446             GRPC_STATUS_DEADLINE_EXCEEDED);
 447   // shutdown and destroy the client and server
 448   grpc_channel_destroy(channel);
 449   ServerShutdownAndDestroy(server1, cq);
 450   ServerShutdownAndDestroy(server2, cq);
 451   grpc_completion_queue_shutdown(cq);
 452   while (grpc_completion_queue_next(cq, gpr_inf_future(GPR_CLOCK_REALTIME),
 453                                     nullptr)
 454              .type != GRPC_QUEUE_SHUTDOWN)
 455     ;
 456   grpc_completion_queue_destroy(cq);
 457 }
 458
 459 // Tests that when a channel has multiple subchannels and receives a GOAWAY with
 460 // "too_many_pings" on one of them, all subchannels start any new transports
 461 // with an updated keepalive time.
 462 TEST_F(KeepaliveThrottlingTest,
 463        ExistingSubchannelsUseNewKeepaliveTimeWhenReconnecting) {
 464   grpc_completion_queue* cq = grpc_completion_queue_create_for_next(nullptr);
 465   std::string server_address1 =
 466       grpc_core::JoinHostPort("127.0.0.1", grpc_pick_unused_port_or_die());
 467   std::string server_address2 =
 468       grpc_core::JoinHostPort("127.0.0.1", grpc_pick_unused_port_or_die());
 469   // create a single channel with round robin load balancing policy.
 470   auto response_generator =
 471       grpc_core::MakeRefCounted<grpc_core::FakeResolverResponseGenerator>();
 472   grpc_arg client_args[] = {
 473       grpc_channel_arg_integer_create(
 474           const_cast<char*>(GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA), 0),
 475       grpc_channel_arg_integer_create(
 476           const_cast<char*>(
 477               GRPC_ARG_HTTP2_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS),
 478           0),
 479       grpc_channel_arg_integer_create(
 480           const_cast<char*>(GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS), 0),
 481       grpc_channel_arg_integer_create(
 482           const_cast<char*>(GRPC_ARG_KEEPALIVE_TIME_MS), 1 * 1000),
 483       grpc_channel_arg_integer_create(
 484           const_cast<char*>(GRPC_ARG_HTTP2_BDP_PROBE), 0),
 485       grpc_core::FakeResolverResponseGenerator::MakeChannelArg(
 486           response_generator.get())};
 487   grpc_channel_args client_channel_args = {GPR_ARRAY_SIZE(client_args),
 488                                            client_args};
 489   grpc_channel* channel =
 490       grpc_insecure_channel_create("fake:///", &client_channel_args, nullptr);
 491   response_generator->SetResponse(
 492       BuildResolverResult({absl::StrCat("ipv4:", server_address1),
 493                            absl::StrCat("ipv4:", server_address2)}));
 494   // For a single subchannel 3 GOAWAYs would be sufficient to increase the
 495   // keepalive time from 1 second to beyond 5 seconds. Even though we are
 496   // alternating between two subchannels, 3 GOAWAYs should still be enough since
 497   // the channel should start all new transports with the new keepalive value
 498   // (even those from a different subchannel).
 499   int expected_keepalive_time_sec = 1;
 500   for (int i = 0; i < 3; i++) {
 501     gpr_log(GPR_ERROR, "Expected keepalive time : %d",
 502             expected_keepalive_time_sec);
 503     grpc_server* server = ServerStart(
 504         i % 2 == 0 ? server_address1.c_str() : server_address2.c_str(), cq);
 505     VerifyChannelReady(channel, cq);
 506     EXPECT_EQ(PerformWaitingCall(channel, server, cq), GRPC_STATUS_UNAVAILABLE);
 507     ServerShutdownAndDestroy(server, cq);
 508     VerifyChannelDisconnected(channel, cq);
 509     expected_keepalive_time_sec *= 2;
 510   }
 511   gpr_log(
 512       GPR_INFO,
 513       "Client keepalive time %d should now be in sync with the server settings",
 514       expected_keepalive_time_sec);
 515   grpc_server* server = ServerStart(server_address1.c_str(), cq);
 516   VerifyChannelReady(channel, cq);
 517   EXPECT_EQ(PerformWaitingCall(channel, server, cq),
 518             GRPC_STATUS_DEADLINE_EXCEEDED);
 519   ServerShutdownAndDestroy(server, cq);
 520   // shutdown and destroy the client
 521   grpc_channel_destroy(channel);
 522   grpc_completion_queue_shutdown(cq);
 523   while (grpc_completion_queue_next(cq, gpr_inf_future(GPR_CLOCK_REALTIME),
 524                                     nullptr)
 525              .type != GRPC_QUEUE_SHUTDOWN)
 526     ;
 527   grpc_completion_queue_destroy(cq);
 528 }
 529
 530 // Perform a simple RPC where the client makes a request expecting a response
 531 // with payload.
 532 void PerformCallWithResponsePayload(grpc_channel* channel, grpc_server* server,
 533                                     grpc_completion_queue* cq) {
 534   grpc_slice response_payload_slice = grpc_slice_from_static_string("hello");
 535
 536   grpc_call* c;
 537   grpc_call* s;
 538   grpc_byte_buffer* response_payload =
 539       grpc_raw_byte_buffer_create(&response_payload_slice, 1);
 540   cq_verifier* cqv = cq_verifier_create(cq);
 541   grpc_op ops[6];
 542   grpc_op* op;
 543   grpc_metadata_array initial_metadata_recv;
 544   grpc_metadata_array trailing_metadata_recv;
 545   grpc_metadata_array request_metadata_recv;
 546   grpc_byte_buffer* response_payload_recv = nullptr;
 547   grpc_call_details call_details;
 548   grpc_status_code status;
 549   grpc_call_error error;
 550   grpc_slice details;
 551   int was_cancelled = 2;
 552
 553   gpr_timespec deadline = grpc_timeout_seconds_to_deadline(60);
 554   c = grpc_channel_create_call(channel, nullptr, GRPC_PROPAGATE_DEFAULTS, cq,
 555                                grpc_slice_from_static_string("/foo"), nullptr,
 556                                deadline, nullptr);
 557   GPR_ASSERT(c);
 558
 559   grpc_metadata_array_init(&initial_metadata_recv);
 560   grpc_metadata_array_init(&trailing_metadata_recv);
 561   grpc_metadata_array_init(&request_metadata_recv);
 562   grpc_call_details_init(&call_details);
 563
 564   memset(ops, 0, sizeof(ops));
 565   op = ops;
 566   op->op = GRPC_OP_SEND_INITIAL_METADATA;
 567   op->data.send_initial_metadata.count = 0;
 568   op->flags = 0;
 569   op->reserved = nullptr;
 570   op++;
 571   op->op = GRPC_OP_SEND_CLOSE_FROM_CLIENT;
 572   op->flags = 0;
 573   op->reserved = nullptr;
 574   op++;
 575   op->op = GRPC_OP_RECV_INITIAL_METADATA;
 576   op->data.recv_initial_metadata.recv_initial_metadata = &initial_metadata_recv;
 577   op->flags = 0;
 578   op->reserved = nullptr;
 579   op++;
 580   op->op = GRPC_OP_RECV_MESSAGE;
 581   op->data.recv_message.recv_message = &response_payload_recv;
 582   op->flags = 0;
 583   op->reserved = nullptr;
 584   op++;
 585   op->op = GRPC_OP_RECV_STATUS_ON_CLIENT;
 586   op->data.recv_status_on_client.trailing_metadata = &trailing_metadata_recv;
 587   op->data.recv_status_on_client.status = &status;
 588   op->data.recv_status_on_client.status_details = &details;
 589   op->flags = 0;
 590   op->reserved = nullptr;
 591   op++;
 592   error = grpc_call_start_batch(c, ops, static_cast<size_t>(op - ops), tag(1),
 593                                 nullptr);
 594   GPR_ASSERT(GRPC_CALL_OK == error);
 595
 596   error = grpc_server_request_call(server, &s, &call_details,
 597                                    &request_metadata_recv, cq, cq, tag(101));
 598   GPR_ASSERT(GRPC_CALL_OK == error);
 599   CQ_EXPECT_COMPLETION(cqv, tag(101), 1);
 600   cq_verify(cqv);
 601
 602   memset(ops, 0, sizeof(ops));
 603   op = ops;
 604   op->op = GRPC_OP_SEND_INITIAL_METADATA;
 605   op->data.send_initial_metadata.count = 0;
 606   op->flags = 0;
 607   op->reserved = nullptr;
 608   op++;
 609   error = grpc_call_start_batch(s, ops, static_cast<size_t>(op - ops), tag(102),
 610                                 nullptr);
 611   GPR_ASSERT(GRPC_CALL_OK == error);
 612
 613   CQ_EXPECT_COMPLETION(cqv, tag(102), 1);
 614   cq_verify(cqv);
 615
 616   memset(ops, 0, sizeof(ops));
 617   op = ops;
 618   op->op = GRPC_OP_RECV_CLOSE_ON_SERVER;
 619   op->data.recv_close_on_server.cancelled = &was_cancelled;
 620   op->flags = 0;
 621   op->reserved = nullptr;
 622   op++;
 623   op->op = GRPC_OP_SEND_MESSAGE;
 624   op->data.send_message.send_message = response_payload;
 625   op->flags = 0;
 626   op->reserved = nullptr;
 627   op++;
 628   op->op = GRPC_OP_SEND_STATUS_FROM_SERVER;
 629   op->data.send_status_from_server.trailing_metadata_count = 0;
 630   op->data.send_status_from_server.status = GRPC_STATUS_OK;
 631   grpc_slice status_details = grpc_slice_from_static_string("xyz");
 632   op->data.send_status_from_server.status_details = &status_details;
 633   op->flags = 0;
 634   op->reserved = nullptr;
 635   op++;
 636   error = grpc_call_start_batch(s, ops, static_cast<size_t>(op - ops), tag(103),
 637                                 nullptr);
 638   GPR_ASSERT(GRPC_CALL_OK == error);
 639
 640   CQ_EXPECT_COMPLETION(cqv, tag(103), 1);
 641   CQ_EXPECT_COMPLETION(cqv, tag(1), 1);
 642   cq_verify(cqv);
 643
 644   GPR_ASSERT(status == GRPC_STATUS_OK);
 645   GPR_ASSERT(0 == grpc_slice_str_cmp(details, "xyz"));
 646   GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/foo"));
 647   GPR_ASSERT(was_cancelled == 0);
 648   GPR_ASSERT(
 649       byte_buffer_eq_slice(response_payload_recv, response_payload_slice));
 650
 651   grpc_slice_unref(details);
 652   grpc_metadata_array_destroy(&initial_metadata_recv);
 653   grpc_metadata_array_destroy(&trailing_metadata_recv);
 654   grpc_metadata_array_destroy(&request_metadata_recv);
 655   grpc_call_details_destroy(&call_details);
 656
 657   grpc_call_unref(c);
 658   grpc_call_unref(s);
 659
 660   cq_verifier_destroy(cqv);
 661
 662   grpc_byte_buffer_destroy(response_payload);
 663   grpc_byte_buffer_destroy(response_payload_recv);
 664 }
 665
 666 TEST(TooManyPings, BdpPingNotSentWithoutReceiveSideActivity) {
 667   grpc_completion_queue* cq = grpc_completion_queue_create_for_next(nullptr);
 668   // create the server
 669   std::string server_address =
 670       grpc_core::JoinHostPort("localhost", grpc_pick_unused_port_or_die());
 671   grpc_arg server_args[] = {
 672       grpc_channel_arg_integer_create(
 673           const_cast<char*>(
 674               GRPC_ARG_HTTP2_MIN_RECV_PING_INTERVAL_WITHOUT_DATA_MS),
 675           60 * 1000),
 676       grpc_channel_arg_integer_create(
 677           const_cast<char*>(GRPC_ARG_HTTP2_MAX_PING_STRIKES), 1)};
 678   grpc_channel_args server_channel_args = {GPR_ARRAY_SIZE(server_args),
 679                                            server_args};
 680   grpc_server* server = grpc_server_create(&server_channel_args, nullptr);
 681   grpc_server_register_completion_queue(server, cq, nullptr);
 682   GPR_ASSERT(
 683       grpc_server_add_insecure_http2_port(server, server_address.c_str()));
 684   grpc_server_start(server);
 685   // create the channel (bdp pings are enabled by default)
 686   grpc_arg client_args[] = {
 687       grpc_channel_arg_integer_create(
 688           const_cast<char*>(GRPC_ARG_HTTP2_MAX_PINGS_WITHOUT_DATA), 0),
 689       grpc_channel_arg_integer_create(
 690           const_cast<char*>(
 691               GRPC_ARG_HTTP2_MIN_SENT_PING_INTERVAL_WITHOUT_DATA_MS),
 692           0),
 693       grpc_channel_arg_integer_create(
 694           const_cast<char*>(GRPC_ARG_KEEPALIVE_PERMIT_WITHOUT_CALLS), 1)};
 695   grpc_channel_args client_channel_args = {GPR_ARRAY_SIZE(client_args),
 696                                            client_args};
 697   grpc_channel* channel = grpc_insecure_channel_create(
 698       server_address.c_str(), &client_channel_args, nullptr);
 699   VerifyChannelReady(channel, cq);
 700   cq_verifier* cqv = cq_verifier_create(cq);
 701   cq_verify_empty_timeout(cqv, 1);
 702   // Channel should be able to send two pings without disconnect if there was no
 703   // BDP sent.
 704   grpc_channel_ping(channel, cq, tag(1), nullptr);
 705   CQ_EXPECT_COMPLETION(cqv, tag(1), 1);
 706   cq_verify(cqv, 5);
 707   // Second ping
 708   grpc_channel_ping(channel, cq, tag(2), nullptr);
 709   CQ_EXPECT_COMPLETION(cqv, tag(2), 1);
 710   cq_verify(cqv, 5);
 711   ASSERT_EQ(grpc_channel_check_connectivity_state(channel, 0),
 712             GRPC_CHANNEL_READY);
 713   PerformCallWithResponsePayload(channel, server, cq);
 714   // Wait a bit to make sure that the BDP ping goes out.
 715   cq_verify_empty_timeout(cqv, 1);
 716   // The call with a response payload should have triggered a BDP ping.
 717   // Send two more pings to verify. The second ping should cause a disconnect.
 718   // If BDP was not sent, the second ping would not cause a disconnect.
 719   grpc_channel_ping(channel, cq, tag(3), nullptr);
 720   CQ_EXPECT_COMPLETION(cqv, tag(3), 1);
 721   cq_verify(cqv, 5);
 722   // Second ping
 723   grpc_channel_ping(channel, cq, tag(4), nullptr);
 724   CQ_EXPECT_COMPLETION(cqv, tag(4), 1);
 725   cq_verify(cqv, 5);
 726   cq_verify_empty_timeout(cqv, 1);
 727   ASSERT_NE(grpc_channel_check_connectivity_state(channel, 0),
 728             GRPC_CHANNEL_READY);
 729   cq_verifier_destroy(cqv);
 730   // shutdown and destroy the client and server
 731   ServerShutdownAndDestroy(server, cq);
 732   grpc_channel_destroy(channel);
 733   grpc_completion_queue_shutdown(cq);
 734   while (grpc_completion_queue_next(cq, gpr_inf_future(GPR_CLOCK_REALTIME),
 735                                     nullptr)
 736              .type != GRPC_QUEUE_SHUTDOWN)
 737     ;
 738   grpc_completion_queue_destroy(cq);
 739 }
 740
 741 }  // namespace
 742
 743 int main(int argc, char** argv) {
 744   ::testing::InitGoogleTest(&argc, argv);
 745   grpc::testing::TestEnvironment env(argc, argv);
 746   grpc_init();
 747   auto result = RUN_ALL_TESTS();
 748   grpc_shutdown();
 749   return result;
 750 }