test/core/end2end/tests/retry_per_attempt_recv_timeout.cc

   1 //
   2 // Copyright 2017 gRPC authors.
   3 //
   4 // Licensed under the Apache License, Version 2.0 (the "License");
   5 // you may not use this file except in compliance with the License.
   6 // You may obtain a copy of the License at
   7 //
   8 //     http://www.apache.org/licenses/LICENSE-2.0
   9 //
  10 // Unless required by applicable law or agreed to in writing, software
  11 // distributed under the License is distributed on an "AS IS" BASIS,
  12 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 // See the License for the specific language governing permissions and
  14 // limitations under the License.
  15 //
  16
  17 #include "test/core/end2end/end2end_tests.h"
  18
  19 #include <stdio.h>
  20 #include <string.h>
  21
  22 #include <grpc/byte_buffer.h>
  23 #include <grpc/grpc.h>
  24 #include <grpc/support/alloc.h>
  25 #include <grpc/support/log.h>
  26 #include <grpc/support/string_util.h>
  27 #include <grpc/support/time.h>
  28
  29 #include "src/core/lib/channel/channel_args.h"
  30 #include "src/core/lib/gpr/string.h"
  31 #include "src/core/lib/gpr/useful.h"
  32 #include "src/core/lib/iomgr/exec_ctx.h"
  33 #include "src/core/lib/transport/static_metadata.h"
  34
  35 #include "test/core/end2end/cq_verifier.h"
  36 #include "test/core/end2end/tests/cancel_test_helpers.h"
  37
  38 static void* tag(intptr_t t) { return reinterpret_cast<void*>(t); }
  39
  40 static grpc_end2end_test_fixture begin_test(grpc_end2end_test_config config,
  41                                             const char* test_name,
  42                                             grpc_channel_args* client_args,
  43                                             grpc_channel_args* server_args) {
  44   grpc_end2end_test_fixture f;
  45   gpr_log(GPR_INFO, "Running test: %s/%s", test_name, config.name);
  46   f = config.create_fixture(client_args, server_args);
  47   config.init_server(&f, server_args);
  48   config.init_client(&f, client_args);
  49   return f;
  50 }
  51
  52 static gpr_timespec n_seconds_from_now(int n) {
  53   return grpc_timeout_seconds_to_deadline(n);
  54 }
  55
  56 static gpr_timespec five_seconds_from_now(void) {
  57   return n_seconds_from_now(5);
  58 }
  59
  60 static void drain_cq(grpc_completion_queue* cq) {
  61   grpc_event ev;
  62   do {
  63     ev = grpc_completion_queue_next(cq, five_seconds_from_now(), nullptr);
  64   } while (ev.type != GRPC_QUEUE_SHUTDOWN);
  65 }
  66
  67 static void shutdown_server(grpc_end2end_test_fixture* f) {
  68   if (!f->server) return;
  69   grpc_server_shutdown_and_notify(f->server, f->shutdown_cq, tag(1000));
  70   GPR_ASSERT(grpc_completion_queue_pluck(f->shutdown_cq, tag(1000),
  71                                          grpc_timeout_seconds_to_deadline(5),
  72                                          nullptr)
  73                  .type == GRPC_OP_COMPLETE);
  74   grpc_server_destroy(f->server);
  75   f->server = nullptr;
  76 }
  77
  78 static void shutdown_client(grpc_end2end_test_fixture* f) {
  79   if (!f->client) return;
  80   grpc_channel_destroy(f->client);
  81   f->client = nullptr;
  82 }
  83
  84 static void end_test(grpc_end2end_test_fixture* f) {
  85   shutdown_server(f);
  86   shutdown_client(f);
  87
  88   grpc_completion_queue_shutdown(f->cq);
  89   drain_cq(f->cq);
  90   grpc_completion_queue_destroy(f->cq);
  91   grpc_completion_queue_destroy(f->shutdown_cq);
  92 }
  93
  94 // Tests perAttemptRecvTimeout:
  95 // - 2 retries allowed for ABORTED status
  96 // - first attempt does not receive a response until after perAttemptRecvTimeout
  97 // - second attempt returns ABORTED
  98 // - third attempt returns OK
  99 static void test_retry_per_attempt_recv_timeout(
 100     grpc_end2end_test_config config) {
 101   grpc_call* c;
 102   grpc_call* s;
 103   grpc_call* s0;
 104   grpc_op ops[6];
 105   grpc_op* op;
 106   grpc_metadata_array initial_metadata_recv;
 107   grpc_metadata_array trailing_metadata_recv;
 108   grpc_metadata_array request_metadata_recv;
 109   grpc_call_details call_details;
 110   grpc_slice request_payload_slice = grpc_slice_from_static_string("foo");
 111   grpc_slice response_payload_slice = grpc_slice_from_static_string("bar");
 112   grpc_byte_buffer* request_payload =
 113       grpc_raw_byte_buffer_create(&request_payload_slice, 1);
 114   grpc_byte_buffer* response_payload =
 115       grpc_raw_byte_buffer_create(&response_payload_slice, 1);
 116   grpc_byte_buffer* request_payload_recv = nullptr;
 117   grpc_byte_buffer* response_payload_recv = nullptr;
 118   grpc_status_code status;
 119   grpc_call_error error;
 120   grpc_slice details;
 121   int was_cancelled = 2;
 122   char* peer;
 123
 124   grpc_arg args[] = {
 125       grpc_channel_arg_integer_create(
 126           const_cast<char*>(GRPC_ARG_EXPERIMENTAL_ENABLE_HEDGING), 1),
 127       grpc_channel_arg_string_create(
 128           const_cast<char*>(GRPC_ARG_SERVICE_CONFIG),
 129           const_cast<char*>(
 130               "{\n"
 131               "  \"methodConfig\": [ {\n"
 132               "    \"name\": [\n"
 133               "      { \"service\": \"service\", \"method\": \"method\" }\n"
 134               "    ],\n"
 135               "    \"retryPolicy\": {\n"
 136               "      \"maxAttempts\": 3,\n"
 137               "      \"initialBackoff\": \"1s\",\n"
 138               "      \"maxBackoff\": \"120s\",\n"
 139               "      \"backoffMultiplier\": 1.6,\n"
 140               "      \"perAttemptRecvTimeout\": \"2s\",\n"
 141               "      \"retryableStatusCodes\": [ \"ABORTED\" ]\n"
 142               "    }\n"
 143               "  } ]\n"
 144               "}")),
 145   };
 146   grpc_channel_args client_args = {GPR_ARRAY_SIZE(args), args};
 147   grpc_end2end_test_fixture f =
 148       begin_test(config, "retry", &client_args, nullptr);
 149
 150   cq_verifier* cqv = cq_verifier_create(f.cq);
 151
 152   gpr_timespec deadline = five_seconds_from_now();
 153   c = grpc_channel_create_call(f.client, nullptr, GRPC_PROPAGATE_DEFAULTS, f.cq,
 154                                grpc_slice_from_static_string("/service/method"),
 155                                nullptr, deadline, nullptr);
 156   GPR_ASSERT(c);
 157
 158   grpc_metadata_array_init(&initial_metadata_recv);
 159   grpc_metadata_array_init(&trailing_metadata_recv);
 160   grpc_metadata_array_init(&request_metadata_recv);
 161   grpc_call_details_init(&call_details);
 162   grpc_slice status_details = grpc_slice_from_static_string("xyz");
 163
 164   memset(ops, 0, sizeof(ops));
 165   op = ops;
 166   op->op = GRPC_OP_SEND_INITIAL_METADATA;
 167   op->data.send_initial_metadata.count = 0;
 168   op++;
 169   op->op = GRPC_OP_SEND_MESSAGE;
 170   op->data.send_message.send_message = request_payload;
 171   op++;
 172   op->op = GRPC_OP_RECV_MESSAGE;
 173   op->data.recv_message.recv_message = &response_payload_recv;
 174   op++;
 175   op->op = GRPC_OP_SEND_CLOSE_FROM_CLIENT;
 176   op++;
 177   op->op = GRPC_OP_RECV_INITIAL_METADATA;
 178   op->data.recv_initial_metadata.recv_initial_metadata = &initial_metadata_recv;
 179   op++;
 180   op->op = GRPC_OP_RECV_STATUS_ON_CLIENT;
 181   op->data.recv_status_on_client.trailing_metadata = &trailing_metadata_recv;
 182   op->data.recv_status_on_client.status = &status;
 183   op->data.recv_status_on_client.status_details = &details;
 184   op++;
 185   error = grpc_call_start_batch(c, ops, static_cast<size_t>(op - ops), tag(1),
 186                                 nullptr);
 187   GPR_ASSERT(GRPC_CALL_OK == error);
 188
 189   // Server gets a call but does not respond to the call.
 190   error =
 191       grpc_server_request_call(f.server, &s0, &call_details,
 192                                &request_metadata_recv, f.cq, f.cq, tag(101));
 193   GPR_ASSERT(GRPC_CALL_OK == error);
 194   CQ_EXPECT_COMPLETION(cqv, tag(101), true);
 195   cq_verify(cqv);
 196
 197   // Make sure the "grpc-previous-rpc-attempts" header was not sent in the
 198   // initial attempt.
 199   for (size_t i = 0; i < request_metadata_recv.count; ++i) {
 200     GPR_ASSERT(!grpc_slice_eq(request_metadata_recv.metadata[i].key,
 201                               GRPC_MDSTR_GRPC_PREVIOUS_RPC_ATTEMPTS));
 202   }
 203
 204   grpc_metadata_array_destroy(&request_metadata_recv);
 205   grpc_metadata_array_init(&request_metadata_recv);
 206   grpc_call_details_destroy(&call_details);
 207   grpc_call_details_init(&call_details);
 208
 209   // Server gets a second call.
 210   error =
 211       grpc_server_request_call(f.server, &s, &call_details,
 212                                &request_metadata_recv, f.cq, f.cq, tag(201));
 213   GPR_ASSERT(GRPC_CALL_OK == error);
 214   CQ_EXPECT_COMPLETION(cqv, tag(201), true);
 215   cq_verify(cqv);
 216
 217   // Now we can unref the first call.
 218   grpc_call_unref(s0);
 219
 220   // Make sure the "grpc-previous-rpc-attempts" header was sent in the retry.
 221   bool found_retry_header = false;
 222   for (size_t i = 0; i < request_metadata_recv.count; ++i) {
 223     if (grpc_slice_eq(request_metadata_recv.metadata[i].key,
 224                       GRPC_MDSTR_GRPC_PREVIOUS_RPC_ATTEMPTS)) {
 225       GPR_ASSERT(
 226           grpc_slice_eq(request_metadata_recv.metadata[i].value, GRPC_MDSTR_1));
 227       found_retry_header = true;
 228       break;
 229     }
 230   }
 231   GPR_ASSERT(found_retry_header);
 232
 233   peer = grpc_call_get_peer(s);
 234   GPR_ASSERT(peer != nullptr);
 235   gpr_log(GPR_DEBUG, "server_peer=%s", peer);
 236   gpr_free(peer);
 237   peer = grpc_call_get_peer(c);
 238   GPR_ASSERT(peer != nullptr);
 239   gpr_log(GPR_DEBUG, "client_peer=%s", peer);
 240   gpr_free(peer);
 241
 242   // Server sends status ABORTED.
 243   memset(ops, 0, sizeof(ops));
 244   op = ops;
 245   op->op = GRPC_OP_SEND_INITIAL_METADATA;
 246   op->data.send_initial_metadata.count = 0;
 247   op++;
 248   op->op = GRPC_OP_SEND_STATUS_FROM_SERVER;
 249   op->data.send_status_from_server.trailing_metadata_count = 0;
 250   op->data.send_status_from_server.status = GRPC_STATUS_ABORTED;
 251   op->data.send_status_from_server.status_details = &status_details;
 252   op++;
 253   op->op = GRPC_OP_RECV_CLOSE_ON_SERVER;
 254   op->data.recv_close_on_server.cancelled = &was_cancelled;
 255   op++;
 256   error = grpc_call_start_batch(s, ops, static_cast<size_t>(op - ops), tag(202),
 257                                 nullptr);
 258   GPR_ASSERT(GRPC_CALL_OK == error);
 259   CQ_EXPECT_COMPLETION(cqv, tag(202), true);
 260   cq_verify(cqv);
 261
 262   grpc_call_unref(s);
 263   grpc_metadata_array_destroy(&request_metadata_recv);
 264   grpc_metadata_array_init(&request_metadata_recv);
 265   grpc_call_details_destroy(&call_details);
 266   grpc_call_details_init(&call_details);
 267
 268   // Server gets a third call.
 269   error =
 270       grpc_server_request_call(f.server, &s, &call_details,
 271                                &request_metadata_recv, f.cq, f.cq, tag(301));
 272   GPR_ASSERT(GRPC_CALL_OK == error);
 273   CQ_EXPECT_COMPLETION(cqv, tag(301), true);
 274   cq_verify(cqv);
 275
 276   // Make sure the "grpc-previous-rpc-attempts" header was sent in the retry.
 277   found_retry_header = false;
 278   for (size_t i = 0; i < request_metadata_recv.count; ++i) {
 279     if (grpc_slice_eq(request_metadata_recv.metadata[i].key,
 280                       GRPC_MDSTR_GRPC_PREVIOUS_RPC_ATTEMPTS)) {
 281       GPR_ASSERT(
 282           grpc_slice_eq(request_metadata_recv.metadata[i].value, GRPC_MDSTR_2));
 283       found_retry_header = true;
 284       break;
 285     }
 286   }
 287   GPR_ASSERT(found_retry_header);
 288
 289   // Server sends OK status.
 290   memset(ops, 0, sizeof(ops));
 291   op = ops;
 292   op->op = GRPC_OP_SEND_INITIAL_METADATA;
 293   op->data.send_initial_metadata.count = 0;
 294   op++;
 295   op->op = GRPC_OP_RECV_MESSAGE;
 296   op->data.recv_message.recv_message = &request_payload_recv;
 297   op++;
 298   op->op = GRPC_OP_SEND_MESSAGE;
 299   op->data.send_message.send_message = response_payload;
 300   op++;
 301   op->op = GRPC_OP_SEND_STATUS_FROM_SERVER;
 302   op->data.send_status_from_server.trailing_metadata_count = 0;
 303   op->data.send_status_from_server.status = GRPC_STATUS_OK;
 304   op->data.send_status_from_server.status_details = &status_details;
 305   op++;
 306   op->op = GRPC_OP_RECV_CLOSE_ON_SERVER;
 307   op->data.recv_close_on_server.cancelled = &was_cancelled;
 308   op++;
 309   error = grpc_call_start_batch(s, ops, static_cast<size_t>(op - ops), tag(302),
 310                                 nullptr);
 311   GPR_ASSERT(GRPC_CALL_OK == error);
 312
 313   CQ_EXPECT_COMPLETION(cqv, tag(302), true);
 314   CQ_EXPECT_COMPLETION(cqv, tag(1), true);
 315   cq_verify(cqv);
 316
 317   GPR_ASSERT(status == GRPC_STATUS_OK);
 318   GPR_ASSERT(0 == grpc_slice_str_cmp(details, "xyz"));
 319   GPR_ASSERT(0 == grpc_slice_str_cmp(call_details.method, "/service/method"));
 320   GPR_ASSERT(0 == call_details.flags);
 321   GPR_ASSERT(was_cancelled == 0);
 322
 323   grpc_slice_unref(details);
 324   grpc_metadata_array_destroy(&initial_metadata_recv);
 325   grpc_metadata_array_destroy(&trailing_metadata_recv);
 326   grpc_metadata_array_destroy(&request_metadata_recv);
 327   grpc_call_details_destroy(&call_details);
 328   grpc_byte_buffer_destroy(request_payload);
 329   grpc_byte_buffer_destroy(response_payload);
 330   grpc_byte_buffer_destroy(request_payload_recv);
 331   grpc_byte_buffer_destroy(response_payload_recv);
 332
 333   grpc_call_unref(c);
 334   grpc_call_unref(s);
 335
 336   cq_verifier_destroy(cqv);
 337
 338   end_test(&f);
 339   config.tear_down_data(&f);
 340 }
 341
 342 void retry_per_attempt_recv_timeout(grpc_end2end_test_config config) {
 343   GPR_ASSERT(config.feature_mask & FEATURE_MASK_SUPPORTS_CLIENT_CHANNEL);
 344   test_retry_per_attempt_recv_timeout(config);
 345 }
 346
 347 void retry_per_attempt_recv_timeout_pre_init(void) {}