src/shrpx_quic_connection_handler.cc

   1 /*
   2  * nghttp2 - HTTP/2 C Library
   3  *
   4  * Copyright (c) 2021 Tatsuhiro Tsujikawa
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining
   7  * a copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sublicense, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice shall be
  15  * included in all copies or substantial portions of the Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  18  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  19  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  20  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  21  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  22  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  23  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  24  */
  25 #include "shrpx_quic_connection_handler.h"
  26
  27 #include <openssl/rand.h>
  28
  29 #include <ngtcp2/ngtcp2.h>
  30 #include <ngtcp2/ngtcp2_crypto.h>
  31
  32 #include "shrpx_worker.h"
  33 #include "shrpx_client_handler.h"
  34 #include "shrpx_log.h"
  35 #include "shrpx_http3_upstream.h"
  36 #include "shrpx_connection_handler.h"
  37
  38 namespace shrpx {
  39
  40 namespace {
  41 void stateless_reset_bucket_regen_timercb(struct ev_loop *loop, ev_timer *w,
  42                                           int revents) {
  43   auto quic_conn_handler = static_cast<QUICConnectionHandler *>(w->data);
  44
  45   quic_conn_handler->on_stateless_reset_bucket_regen();
  46 }
  47 } // namespace
  48
  49 QUICConnectionHandler::QUICConnectionHandler(Worker *worker)
  50     : worker_{worker},
  51       stateless_reset_bucket_{SHRPX_QUIC_STATELESS_RESET_BURST} {
  52   ev_timer_init(&stateless_reset_bucket_regen_timer_,
  53                 stateless_reset_bucket_regen_timercb, 0., 1.);
  54   stateless_reset_bucket_regen_timer_.data = this;
  55 }
  56
  57 QUICConnectionHandler::~QUICConnectionHandler() {
  58   ev_timer_stop(worker_->get_loop(), &stateless_reset_bucket_regen_timer_);
  59 }
  60
  61 int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr,
  62                                          const Address &remote_addr,
  63                                          const Address &local_addr,
  64                                          const uint8_t *data, size_t datalen) {
  65   int rv;
  66   uint32_t version;
  67   const uint8_t *dcid, *scid;
  68   size_t dcidlen, scidlen;
  69
  70   rv = ngtcp2_pkt_decode_version_cid(&version, &dcid, &dcidlen, &scid, &scidlen,
  71                                      data, datalen, SHRPX_QUIC_SCIDLEN);
  72   switch (rv) {
  73   case 0:
  74     break;
  75   case NGTCP2_ERR_VERSION_NEGOTIATION:
  76     send_version_negotiation(faddr, version, dcid, dcidlen, scid, scidlen,
  77                              remote_addr, local_addr);
  78
  79     return 0;
  80   default:
  81     return 0;
  82   }
  83
  84   auto config = get_config();
  85
  86   ngtcp2_cid dcid_key;
  87   ngtcp2_cid_init(&dcid_key, dcid, dcidlen);
  88
  89   auto conn_handler = worker_->get_connection_handler();
  90
  91   ClientHandler *handler;
  92
  93   auto &quicconf = config->quic;
  94
  95   auto it = connections_.find(dcid_key);
  96   if (it == std::end(connections_)) {
  97     auto cwit = close_waits_.find(dcid_key);
  98     if (cwit != std::end(close_waits_)) {
  99       auto cw = (*cwit).second;
 100
 101       cw->handle_packet(faddr, remote_addr, local_addr, data, datalen);
 102
 103       return 0;
 104     }
 105
 106     if (data[0] & 0x80) {
 107       if (generate_quic_hashed_connection_id(dcid_key, remote_addr, local_addr,
 108                                              dcid_key) != 0) {
 109         return 0;
 110       }
 111
 112       it = connections_.find(dcid_key);
 113       if (it == std::end(connections_)) {
 114         auto cwit = close_waits_.find(dcid_key);
 115         if (cwit != std::end(close_waits_)) {
 116           auto cw = (*cwit).second;
 117
 118           cw->handle_packet(faddr, remote_addr, local_addr, data, datalen);
 119
 120           return 0;
 121         }
 122       }
 123     }
 124   }
 125
 126   if (it == std::end(connections_)) {
 127     std::array<uint8_t, SHRPX_QUIC_DECRYPTED_DCIDLEN> decrypted_dcid;
 128
 129     auto &qkms = conn_handler->get_quic_keying_materials();
 130     const QUICKeyingMaterial *qkm = nullptr;
 131
 132     if (dcidlen == SHRPX_QUIC_SCIDLEN) {
 133       qkm = select_quic_keying_material(*qkms.get(), dcid);
 134
 135       if (decrypt_quic_connection_id(decrypted_dcid.data(),
 136                                      dcid + SHRPX_QUIC_CID_PREFIX_OFFSET,
 137                                      qkm->cid_encryption_key.data()) != 0) {
 138         return 0;
 139       }
 140
 141       if (qkm != &qkms->keying_materials.front() ||
 142           !std::equal(std::begin(decrypted_dcid),
 143                       std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN,
 144                       worker_->get_cid_prefix())) {
 145         auto quic_lwp =
 146             conn_handler->match_quic_lingering_worker_process_cid_prefix(
 147                 decrypted_dcid.data(), decrypted_dcid.size());
 148         if (quic_lwp) {
 149           if (conn_handler->forward_quic_packet_to_lingering_worker_process(
 150                   quic_lwp, remote_addr, local_addr, data, datalen) == 0) {
 151             return 0;
 152           }
 153
 154           return 0;
 155         }
 156       }
 157     }
 158
 159     // new connection
 160
 161     auto &upstreamconf = config->conn.upstream;
 162     if (worker_->get_worker_stat()->num_connections >=
 163         upstreamconf.worker_connections) {
 164       if (LOG_ENABLED(INFO)) {
 165         LOG(INFO) << "Too many connections >="
 166                   << upstreamconf.worker_connections;
 167       }
 168
 169       return 0;
 170     }
 171
 172     ngtcp2_pkt_hd hd;
 173     ngtcp2_cid odcid, *podcid = nullptr;
 174     const uint8_t *token = nullptr;
 175     size_t tokenlen = 0;
 176
 177     switch (ngtcp2_accept(&hd, data, datalen)) {
 178     case 0: {
 179       // If we get Initial and it has the CID prefix of this worker,
 180       // it is likely that client is intentionally use the prefix.
 181       // Just drop it.
 182       if (dcidlen == SHRPX_QUIC_SCIDLEN) {
 183         if (qkm != &qkms->keying_materials.front()) {
 184           qkm = &qkms->keying_materials.front();
 185
 186           if (decrypt_quic_connection_id(decrypted_dcid.data(),
 187                                          dcid + SHRPX_QUIC_CID_PREFIX_OFFSET,
 188                                          qkm->cid_encryption_key.data()) != 0) {
 189             return 0;
 190           }
 191         }
 192
 193         if (std::equal(std::begin(decrypted_dcid),
 194                        std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN,
 195                        worker_->get_cid_prefix())) {
 196           return 0;
 197         }
 198       }
 199
 200       if (worker_->get_graceful_shutdown()) {
 201         send_connection_close(faddr, version, hd.dcid, hd.scid, remote_addr,
 202                               local_addr, NGTCP2_CONNECTION_REFUSED);
 203         return 0;
 204       }
 205
 206       if (hd.token.len == 0) {
 207         if (quicconf.upstream.require_token) {
 208           send_retry(faddr, version, dcid, dcidlen, scid, scidlen, remote_addr,
 209                      local_addr);
 210
 211           return 0;
 212         }
 213
 214         break;
 215       }
 216
 217       if (dcidlen != SHRPX_QUIC_SCIDLEN) {
 218         // Initial packets with token must have DCID chosen by server.
 219         return 0;
 220       }
 221
 222       auto qkm = select_quic_keying_material(*qkms.get(), dcid);
 223
 224       switch (hd.token.base[0]) {
 225       case NGTCP2_CRYPTO_TOKEN_MAGIC_RETRY:
 226         if (verify_retry_token(odcid, hd.token.base, hd.token.len, hd.dcid,
 227                                &remote_addr.su.sa, remote_addr.len,
 228                                qkm->secret.data(), qkm->secret.size()) != 0) {
 229           if (LOG_ENABLED(INFO)) {
 230             LOG(INFO) << "Failed to validate Retry token from remote="
 231                       << util::to_numeric_addr(&remote_addr);
 232           }
 233
 234           // 2nd Retry packet is not allowed, so send CONNECTION_CLOSE
 235           // with INVALID_TOKEN.
 236           send_connection_close(faddr, version, hd.dcid, hd.scid, remote_addr,
 237                                 local_addr, NGTCP2_INVALID_TOKEN);
 238           return 0;
 239         }
 240
 241         if (LOG_ENABLED(INFO)) {
 242           LOG(INFO) << "Successfully validated Retry token from remote="
 243                     << util::to_numeric_addr(&remote_addr);
 244         }
 245
 246         podcid = &odcid;
 247         token = hd.token.base;
 248         tokenlen = hd.token.len;
 249
 250         break;
 251       case NGTCP2_CRYPTO_TOKEN_MAGIC_REGULAR:
 252         if (verify_token(hd.token.base, hd.token.len, &remote_addr.su.sa,
 253                          remote_addr.len, qkm->secret.data(),
 254                          qkm->secret.size()) != 0) {
 255           if (LOG_ENABLED(INFO)) {
 256             LOG(INFO) << "Failed to validate token from remote="
 257                       << util::to_numeric_addr(&remote_addr);
 258           }
 259
 260           if (quicconf.upstream.require_token) {
 261             send_retry(faddr, version, dcid, dcidlen, scid, scidlen,
 262                        remote_addr, local_addr);
 263
 264             return 0;
 265           }
 266
 267           break;
 268         }
 269
 270         if (LOG_ENABLED(INFO)) {
 271           LOG(INFO) << "Successfully validated token from remote="
 272                     << util::to_numeric_addr(&remote_addr);
 273         }
 274
 275         token = hd.token.base;
 276         tokenlen = hd.token.len;
 277
 278         break;
 279       default:
 280         if (quicconf.upstream.require_token) {
 281           send_retry(faddr, version, dcid, dcidlen, scid, scidlen, remote_addr,
 282                      local_addr);
 283
 284           return 0;
 285         }
 286
 287         break;
 288       }
 289
 290       break;
 291     }
 292     case NGTCP2_ERR_RETRY:
 293       if (worker_->get_graceful_shutdown()) {
 294         send_connection_close(faddr, version, hd.dcid, hd.scid, remote_addr,
 295                               local_addr, NGTCP2_CONNECTION_REFUSED);
 296         return 0;
 297       }
 298
 299       send_retry(faddr, version, dcid, dcidlen, scid, scidlen, remote_addr,
 300                  local_addr);
 301       return 0;
 302     case NGTCP2_ERR_VERSION_NEGOTIATION:
 303       send_version_negotiation(faddr, version, dcid, dcidlen, scid, scidlen,
 304                                remote_addr, local_addr);
 305       return 0;
 306     default:
 307       if (!config->single_thread && !(data[0] & 0x80) &&
 308           dcidlen == SHRPX_QUIC_SCIDLEN &&
 309           !std::equal(std::begin(decrypted_dcid),
 310                       std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN,
 311                       worker_->get_cid_prefix())) {
 312         if (conn_handler->forward_quic_packet(faddr, remote_addr, local_addr,
 313                                               decrypted_dcid.data(), data,
 314                                               datalen) == 0) {
 315           return 0;
 316         }
 317       }
 318
 319       if (!(data[0] & 0x80)) {
 320         // TODO Must be rate limited
 321         send_stateless_reset(faddr, dcid, dcidlen, remote_addr, local_addr);
 322       }
 323
 324       return 0;
 325     }
 326
 327     handler = handle_new_connection(faddr, remote_addr, local_addr, hd, podcid,
 328                                     token, tokenlen);
 329     if (handler == nullptr) {
 330       return 0;
 331     }
 332   } else {
 333     handler = (*it).second;
 334   }
 335
 336   if (handler->read_quic(faddr, remote_addr, local_addr, data, datalen) != 0) {
 337     delete handler;
 338     return 0;
 339   }
 340
 341   handler->signal_write();
 342
 343   return 0;
 344 }
 345
 346 ClientHandler *QUICConnectionHandler::handle_new_connection(
 347     const UpstreamAddr *faddr, const Address &remote_addr,
 348     const Address &local_addr, const ngtcp2_pkt_hd &hd, const ngtcp2_cid *odcid,
 349     const uint8_t *token, size_t tokenlen) {
 350   std::array<char, NI_MAXHOST> host;
 351   std::array<char, NI_MAXSERV> service;
 352   int rv;
 353
 354   rv = getnameinfo(&remote_addr.su.sa, remote_addr.len, host.data(),
 355                    host.size(), service.data(), service.size(),
 356                    NI_NUMERICHOST | NI_NUMERICSERV);
 357   if (rv != 0) {
 358     LOG(ERROR) << "getnameinfo() failed: " << gai_strerror(rv);
 359
 360     return nullptr;
 361   }
 362
 363   auto ssl_ctx = worker_->get_quic_sv_ssl_ctx();
 364
 365   assert(ssl_ctx);
 366
 367   auto ssl = tls::create_ssl(ssl_ctx);
 368   if (ssl == nullptr) {
 369     return nullptr;
 370   }
 371
 372 #if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
 373   assert(SSL_is_quic(ssl));
 374 #endif // OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
 375
 376   SSL_set_accept_state(ssl);
 377
 378   auto config = get_config();
 379   auto &quicconf = config->quic;
 380
 381   if (quicconf.upstream.early_data) {
 382 #if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
 383     SSL_set_quic_early_data_enabled(ssl, 1);
 384 #else  // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL))
 385     SSL_set_early_data_enabled(ssl, 1);
 386 #endif // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL))
 387   }
 388
 389   // Disable TLS session ticket if we don't have working ticket
 390   // keys.
 391   if (!worker_->get_ticket_keys()) {
 392     SSL_set_options(ssl, SSL_OP_NO_TICKET);
 393   }
 394
 395   auto handler = std::make_unique<ClientHandler>(
 396       worker_, faddr->fd, ssl, StringRef{host.data()},
 397       StringRef{service.data()}, remote_addr.su.sa.sa_family, faddr);
 398
 399   auto upstream = std::make_unique<Http3Upstream>(handler.get());
 400   if (upstream->init(faddr, remote_addr, local_addr, hd, odcid, token,
 401                      tokenlen) != 0) {
 402     return nullptr;
 403   }
 404
 405   handler->setup_http3_upstream(std::move(upstream));
 406
 407   return handler.release();
 408 }
 409
 410 namespace {
 411 uint32_t generate_reserved_version(const Address &addr, uint32_t version) {
 412   uint32_t h = 0x811C9DC5u;
 413   const uint8_t *p = reinterpret_cast<const uint8_t *>(&addr.su.sa);
 414   const uint8_t *ep = p + addr.len;
 415
 416   for (; p != ep; ++p) {
 417     h ^= *p;
 418     h *= 0x01000193u;
 419   }
 420
 421   version = htonl(version);
 422   p = (const uint8_t *)&version;
 423   ep = p + sizeof(version);
 424
 425   for (; p != ep; ++p) {
 426     h ^= *p;
 427     h *= 0x01000193u;
 428   }
 429
 430   h &= 0xf0f0f0f0u;
 431   h |= 0x0a0a0a0au;
 432
 433   return h;
 434 }
 435 } // namespace
 436
 437 int QUICConnectionHandler::send_retry(
 438     const UpstreamAddr *faddr, uint32_t version, const uint8_t *ini_dcid,
 439     size_t ini_dcidlen, const uint8_t *ini_scid, size_t ini_scidlen,
 440     const Address &remote_addr, const Address &local_addr) {
 441   std::array<char, NI_MAXHOST> host;
 442   std::array<char, NI_MAXSERV> port;
 443
 444   if (getnameinfo(&remote_addr.su.sa, remote_addr.len, host.data(), host.size(),
 445                   port.data(), port.size(),
 446                   NI_NUMERICHOST | NI_NUMERICSERV) != 0) {
 447     return -1;
 448   }
 449
 450   auto config = get_config();
 451   auto &quicconf = config->quic;
 452
 453   auto conn_handler = worker_->get_connection_handler();
 454   auto &qkms = conn_handler->get_quic_keying_materials();
 455   auto &qkm = qkms->keying_materials.front();
 456
 457   ngtcp2_cid retry_scid;
 458
 459   if (generate_quic_retry_connection_id(retry_scid, SHRPX_QUIC_SCIDLEN,
 460                                         quicconf.server_id.data(), qkm.id,
 461                                         qkm.cid_encryption_key.data()) != 0) {
 462     return -1;
 463   }
 464
 465   std::array<uint8_t, NGTCP2_CRYPTO_MAX_RETRY_TOKENLEN> token;
 466   size_t tokenlen;
 467
 468   ngtcp2_cid idcid, iscid;
 469   ngtcp2_cid_init(&idcid, ini_dcid, ini_dcidlen);
 470   ngtcp2_cid_init(&iscid, ini_scid, ini_scidlen);
 471
 472   if (generate_retry_token(token.data(), tokenlen, &remote_addr.su.sa,
 473                            remote_addr.len, retry_scid, idcid,
 474                            qkm.secret.data(), qkm.secret.size()) != 0) {
 475     return -1;
 476   }
 477
 478   std::vector<uint8_t> buf;
 479   buf.resize(256);
 480
 481   auto nwrite =
 482       ngtcp2_crypto_write_retry(buf.data(), buf.size(), version, &iscid,
 483                                 &retry_scid, &idcid, token.data(), tokenlen);
 484   if (nwrite < 0) {
 485     LOG(ERROR) << "ngtcp2_crypto_write_retry: " << ngtcp2_strerror(nwrite);
 486     return -1;
 487   }
 488
 489   buf.resize(nwrite);
 490
 491   quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len,
 492                    &local_addr.su.sa, local_addr.len, buf.data(), buf.size(),
 493                    0);
 494
 495   if (generate_quic_hashed_connection_id(idcid, remote_addr, local_addr,
 496                                          idcid) != 0) {
 497     return -1;
 498   }
 499
 500   auto d =
 501       static_cast<ev_tstamp>(NGTCP2_DEFAULT_INITIAL_RTT * 3) / NGTCP2_SECONDS;
 502
 503   if (LOG_ENABLED(INFO)) {
 504     LOG(INFO) << "Enter close-wait period " << d << "s with " << buf.size()
 505               << " bytes sentinel packet";
 506   }
 507
 508   auto cw = std::make_unique<CloseWait>(worker_, std::vector<ngtcp2_cid>{idcid},
 509                                         std::move(buf), d);
 510
 511   add_close_wait(cw.get());
 512
 513   cw.release();
 514
 515   return 0;
 516 }
 517
 518 int QUICConnectionHandler::send_version_negotiation(
 519     const UpstreamAddr *faddr, uint32_t version, const uint8_t *ini_dcid,
 520     size_t ini_dcidlen, const uint8_t *ini_scid, size_t ini_scidlen,
 521     const Address &remote_addr, const Address &local_addr) {
 522   std::array<uint32_t, 2> sv;
 523
 524   sv[0] = generate_reserved_version(remote_addr, version);
 525   sv[1] = NGTCP2_PROTO_VER_V1;
 526
 527   std::array<uint8_t, NGTCP2_MAX_UDP_PAYLOAD_SIZE> buf;
 528
 529   uint8_t rand_byte;
 530   util::random_bytes(&rand_byte, &rand_byte + 1, worker_->get_randgen());
 531
 532   auto nwrite = ngtcp2_pkt_write_version_negotiation(
 533       buf.data(), buf.size(), rand_byte, ini_scid, ini_scidlen, ini_dcid,
 534       ini_dcidlen, sv.data(), sv.size());
 535   if (nwrite < 0) {
 536     LOG(ERROR) << "ngtcp2_pkt_write_version_negotiation: "
 537                << ngtcp2_strerror(nwrite);
 538     return -1;
 539   }
 540
 541   return quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len,
 542                           &local_addr.su.sa, local_addr.len, buf.data(), nwrite,
 543                           0);
 544 }
 545
 546 int QUICConnectionHandler::send_stateless_reset(const UpstreamAddr *faddr,
 547                                                 const uint8_t *dcid,
 548                                                 size_t dcidlen,
 549                                                 const Address &remote_addr,
 550                                                 const Address &local_addr) {
 551   if (stateless_reset_bucket_ == 0) {
 552     if (LOG_ENABLED(INFO)) {
 553       LOG(INFO) << "Stateless Reset bucket has been depleted";
 554     }
 555
 556     return 0;
 557   }
 558
 559   --stateless_reset_bucket_;
 560
 561   if (!ev_is_active(&stateless_reset_bucket_regen_timer_)) {
 562     ev_timer_again(worker_->get_loop(), &stateless_reset_bucket_regen_timer_);
 563   }
 564
 565   int rv;
 566   std::array<uint8_t, NGTCP2_STATELESS_RESET_TOKENLEN> token;
 567   ngtcp2_cid cid;
 568
 569   ngtcp2_cid_init(&cid, dcid, dcidlen);
 570
 571   auto conn_handler = worker_->get_connection_handler();
 572   auto &qkms = conn_handler->get_quic_keying_materials();
 573   auto &qkm = qkms->keying_materials.front();
 574
 575   rv = generate_quic_stateless_reset_token(token.data(), cid, qkm.secret.data(),
 576                                            qkm.secret.size());
 577   if (rv != 0) {
 578     return -1;
 579   }
 580
 581   std::array<uint8_t, NGTCP2_MIN_STATELESS_RESET_RANDLEN> rand_bytes;
 582
 583   if (RAND_bytes(rand_bytes.data(), rand_bytes.size()) != 1) {
 584     return -1;
 585   }
 586
 587   std::array<uint8_t, NGTCP2_MAX_UDP_PAYLOAD_SIZE> buf;
 588
 589   auto nwrite =
 590       ngtcp2_pkt_write_stateless_reset(buf.data(), buf.size(), token.data(),
 591                                        rand_bytes.data(), rand_bytes.size());
 592   if (nwrite < 0) {
 593     LOG(ERROR) << "ngtcp2_pkt_write_stateless_reset: "
 594                << ngtcp2_strerror(nwrite);
 595     return -1;
 596   }
 597
 598   if (LOG_ENABLED(INFO)) {
 599     LOG(INFO) << "Send stateless_reset to remote="
 600               << util::to_numeric_addr(&remote_addr)
 601               << " dcid=" << util::format_hex(dcid, dcidlen);
 602   }
 603
 604   return quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len,
 605                           &local_addr.su.sa, local_addr.len, buf.data(), nwrite,
 606                           0);
 607 }
 608
 609 int QUICConnectionHandler::send_connection_close(
 610     const UpstreamAddr *faddr, uint32_t version, const ngtcp2_cid &ini_dcid,
 611     const ngtcp2_cid &ini_scid, const Address &remote_addr,
 612     const Address &local_addr, uint64_t error_code) {
 613   std::array<uint8_t, NGTCP2_MAX_UDP_PAYLOAD_SIZE> buf;
 614
 615   auto nwrite = ngtcp2_crypto_write_connection_close(
 616       buf.data(), buf.size(), version, &ini_scid, &ini_dcid, error_code);
 617   if (nwrite < 0) {
 618     LOG(ERROR) << "ngtcp2_crypto_write_connection_close failed";
 619     return -1;
 620   }
 621
 622   if (LOG_ENABLED(INFO)) {
 623     LOG(INFO) << "Send Initial CONNECTION_CLOSE with error_code=" << log::hex
 624               << error_code << log::dec
 625               << " to remote=" << util::to_numeric_addr(&remote_addr)
 626               << " dcid=" << util::format_hex(ini_scid.data, ini_scid.datalen)
 627               << " scid=" << util::format_hex(ini_dcid.data, ini_dcid.datalen);
 628   }
 629
 630   return quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len,
 631                           &local_addr.su.sa, local_addr.len, buf.data(), nwrite,
 632                           0);
 633 }
 634
 635 void QUICConnectionHandler::add_connection_id(const ngtcp2_cid &cid,
 636                                               ClientHandler *handler) {
 637   connections_.emplace(cid, handler);
 638 }
 639
 640 void QUICConnectionHandler::remove_connection_id(const ngtcp2_cid &cid) {
 641   connections_.erase(cid);
 642 }
 643
 644 void QUICConnectionHandler::add_close_wait(CloseWait *cw) {
 645   for (auto &cid : cw->scids) {
 646     close_waits_.emplace(cid, cw);
 647   }
 648 }
 649
 650 void QUICConnectionHandler::remove_close_wait(const CloseWait *cw) {
 651   for (auto &cid : cw->scids) {
 652     close_waits_.erase(cid);
 653   }
 654 }
 655
 656 void QUICConnectionHandler::on_stateless_reset_bucket_regen() {
 657   assert(stateless_reset_bucket_ < SHRPX_QUIC_STATELESS_RESET_BURST);
 658
 659   if (++stateless_reset_bucket_ == SHRPX_QUIC_STATELESS_RESET_BURST) {
 660     ev_timer_stop(worker_->get_loop(), &stateless_reset_bucket_regen_timer_);
 661   }
 662 }
 663
 664 static void close_wait_timeoutcb(struct ev_loop *loop, ev_timer *w,
 665                                  int revents) {
 666   auto cw = static_cast<CloseWait *>(w->data);
 667
 668   if (LOG_ENABLED(INFO)) {
 669     LOG(INFO) << "close-wait period finished";
 670   }
 671
 672   auto quic_conn_handler = cw->worker->get_quic_connection_handler();
 673   quic_conn_handler->remove_close_wait(cw);
 674
 675   delete cw;
 676 }
 677
 678 CloseWait::CloseWait(Worker *worker, std::vector<ngtcp2_cid> scids,
 679                      std::vector<uint8_t> pkt, ev_tstamp period)
 680     : worker{worker},
 681       scids{std::move(scids)},
 682       pkt{std::move(pkt)},
 683       bytes_recv{0},
 684       bytes_sent{0},
 685       num_pkts_recv{0},
 686       next_pkts_recv{1} {
 687   ++worker->get_worker_stat()->num_close_waits;
 688
 689   ev_timer_init(&timer, close_wait_timeoutcb, period, 0.);
 690   timer.data = this;
 691
 692   ev_timer_start(worker->get_loop(), &timer);
 693 }
 694
 695 CloseWait::~CloseWait() {
 696   auto loop = worker->get_loop();
 697
 698   ev_timer_stop(loop, &timer);
 699
 700   auto worker_stat = worker->get_worker_stat();
 701   --worker_stat->num_close_waits;
 702
 703   if (worker->get_graceful_shutdown() && worker_stat->num_connections == 0 &&
 704       worker_stat->num_close_waits == 0) {
 705     ev_break(loop);
 706   }
 707 }
 708
 709 int CloseWait::handle_packet(const UpstreamAddr *faddr,
 710                              const Address &remote_addr,
 711                              const Address &local_addr, const uint8_t *data,
 712                              size_t datalen) {
 713   if (pkt.empty()) {
 714     return 0;
 715   }
 716
 717   ++num_pkts_recv;
 718   bytes_recv += datalen;
 719
 720   if (bytes_sent + pkt.size() > 3 * bytes_recv ||
 721       next_pkts_recv > num_pkts_recv) {
 722     return 0;
 723   }
 724
 725   if (quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len,
 726                        &local_addr.su.sa, local_addr.len, pkt.data(),
 727                        pkt.size(), 0) != 0) {
 728     return -1;
 729   }
 730
 731   next_pkts_recv *= 2;
 732   bytes_sent += pkt.size();
 733
 734   return 0;
 735 }
 736
 737 } // namespace shrpx