2 * nghttp2 - HTTP/2 C Library
4 * Copyright (c) 2021 Tatsuhiro Tsujikawa
6 * Permission is hereby granted, free of charge, to any person obtaining
7 * a copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sublicense, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice shall be
15 * included in all copies or substantial portions of the Software.
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
20 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25 #include "shrpx_quic_connection_handler.h"
27 #include <openssl/rand.h>
29 #include <ngtcp2/ngtcp2.h>
30 #include <ngtcp2/ngtcp2_crypto.h>
32 #include "shrpx_worker.h"
33 #include "shrpx_client_handler.h"
34 #include "shrpx_log.h"
35 #include "shrpx_http3_upstream.h"
36 #include "shrpx_connection_handler.h"
41 void stateless_reset_bucket_regen_timercb(struct ev_loop *loop, ev_timer *w,
43 auto quic_conn_handler = static_cast<QUICConnectionHandler *>(w->data);
45 quic_conn_handler->on_stateless_reset_bucket_regen();
49 QUICConnectionHandler::QUICConnectionHandler(Worker *worker)
51 stateless_reset_bucket_{SHRPX_QUIC_STATELESS_RESET_BURST} {
52 ev_timer_init(&stateless_reset_bucket_regen_timer_,
53 stateless_reset_bucket_regen_timercb, 0., 1.);
54 stateless_reset_bucket_regen_timer_.data = this;
57 QUICConnectionHandler::~QUICConnectionHandler() {
58 ev_timer_stop(worker_->get_loop(), &stateless_reset_bucket_regen_timer_);
61 int QUICConnectionHandler::handle_packet(const UpstreamAddr *faddr,
62 const Address &remote_addr,
63 const Address &local_addr,
64 const uint8_t *data, size_t datalen) {
67 const uint8_t *dcid, *scid;
68 size_t dcidlen, scidlen;
70 rv = ngtcp2_pkt_decode_version_cid(&version, &dcid, &dcidlen, &scid, &scidlen,
71 data, datalen, SHRPX_QUIC_SCIDLEN);
75 case NGTCP2_ERR_VERSION_NEGOTIATION:
76 send_version_negotiation(faddr, version, dcid, dcidlen, scid, scidlen,
77 remote_addr, local_addr);
84 auto config = get_config();
87 ngtcp2_cid_init(&dcid_key, dcid, dcidlen);
89 auto conn_handler = worker_->get_connection_handler();
91 ClientHandler *handler;
93 auto &quicconf = config->quic;
95 auto it = connections_.find(dcid_key);
96 if (it == std::end(connections_)) {
97 auto cwit = close_waits_.find(dcid_key);
98 if (cwit != std::end(close_waits_)) {
99 auto cw = (*cwit).second;
101 cw->handle_packet(faddr, remote_addr, local_addr, data, datalen);
106 if (data[0] & 0x80) {
107 if (generate_quic_hashed_connection_id(dcid_key, remote_addr, local_addr,
112 it = connections_.find(dcid_key);
113 if (it == std::end(connections_)) {
114 auto cwit = close_waits_.find(dcid_key);
115 if (cwit != std::end(close_waits_)) {
116 auto cw = (*cwit).second;
118 cw->handle_packet(faddr, remote_addr, local_addr, data, datalen);
126 if (it == std::end(connections_)) {
127 std::array<uint8_t, SHRPX_QUIC_DECRYPTED_DCIDLEN> decrypted_dcid;
129 auto &qkms = conn_handler->get_quic_keying_materials();
130 const QUICKeyingMaterial *qkm = nullptr;
132 if (dcidlen == SHRPX_QUIC_SCIDLEN) {
133 qkm = select_quic_keying_material(*qkms.get(), dcid);
135 if (decrypt_quic_connection_id(decrypted_dcid.data(),
136 dcid + SHRPX_QUIC_CID_PREFIX_OFFSET,
137 qkm->cid_encryption_key.data()) != 0) {
141 if (qkm != &qkms->keying_materials.front() ||
142 !std::equal(std::begin(decrypted_dcid),
143 std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN,
144 worker_->get_cid_prefix())) {
146 conn_handler->match_quic_lingering_worker_process_cid_prefix(
147 decrypted_dcid.data(), decrypted_dcid.size());
149 if (conn_handler->forward_quic_packet_to_lingering_worker_process(
150 quic_lwp, remote_addr, local_addr, data, datalen) == 0) {
161 auto &upstreamconf = config->conn.upstream;
162 if (worker_->get_worker_stat()->num_connections >=
163 upstreamconf.worker_connections) {
164 if (LOG_ENABLED(INFO)) {
165 LOG(INFO) << "Too many connections >="
166 << upstreamconf.worker_connections;
173 ngtcp2_cid odcid, *podcid = nullptr;
174 const uint8_t *token = nullptr;
177 switch (ngtcp2_accept(&hd, data, datalen)) {
179 // If we get Initial and it has the CID prefix of this worker,
180 // it is likely that client is intentionally use the prefix.
182 if (dcidlen == SHRPX_QUIC_SCIDLEN) {
183 if (qkm != &qkms->keying_materials.front()) {
184 qkm = &qkms->keying_materials.front();
186 if (decrypt_quic_connection_id(decrypted_dcid.data(),
187 dcid + SHRPX_QUIC_CID_PREFIX_OFFSET,
188 qkm->cid_encryption_key.data()) != 0) {
193 if (std::equal(std::begin(decrypted_dcid),
194 std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN,
195 worker_->get_cid_prefix())) {
200 if (worker_->get_graceful_shutdown()) {
201 send_connection_close(faddr, version, hd.dcid, hd.scid, remote_addr,
202 local_addr, NGTCP2_CONNECTION_REFUSED);
206 if (hd.token.len == 0) {
207 if (quicconf.upstream.require_token) {
208 send_retry(faddr, version, dcid, dcidlen, scid, scidlen, remote_addr,
217 if (dcidlen != SHRPX_QUIC_SCIDLEN) {
218 // Initial packets with token must have DCID chosen by server.
222 auto qkm = select_quic_keying_material(*qkms.get(), dcid);
224 switch (hd.token.base[0]) {
225 case NGTCP2_CRYPTO_TOKEN_MAGIC_RETRY:
226 if (verify_retry_token(odcid, hd.token.base, hd.token.len, hd.dcid,
227 &remote_addr.su.sa, remote_addr.len,
228 qkm->secret.data(), qkm->secret.size()) != 0) {
229 if (LOG_ENABLED(INFO)) {
230 LOG(INFO) << "Failed to validate Retry token from remote="
231 << util::to_numeric_addr(&remote_addr);
234 // 2nd Retry packet is not allowed, so send CONNECTION_CLOSE
235 // with INVALID_TOKEN.
236 send_connection_close(faddr, version, hd.dcid, hd.scid, remote_addr,
237 local_addr, NGTCP2_INVALID_TOKEN);
241 if (LOG_ENABLED(INFO)) {
242 LOG(INFO) << "Successfully validated Retry token from remote="
243 << util::to_numeric_addr(&remote_addr);
247 token = hd.token.base;
248 tokenlen = hd.token.len;
251 case NGTCP2_CRYPTO_TOKEN_MAGIC_REGULAR:
252 if (verify_token(hd.token.base, hd.token.len, &remote_addr.su.sa,
253 remote_addr.len, qkm->secret.data(),
254 qkm->secret.size()) != 0) {
255 if (LOG_ENABLED(INFO)) {
256 LOG(INFO) << "Failed to validate token from remote="
257 << util::to_numeric_addr(&remote_addr);
260 if (quicconf.upstream.require_token) {
261 send_retry(faddr, version, dcid, dcidlen, scid, scidlen,
262 remote_addr, local_addr);
270 if (LOG_ENABLED(INFO)) {
271 LOG(INFO) << "Successfully validated token from remote="
272 << util::to_numeric_addr(&remote_addr);
275 token = hd.token.base;
276 tokenlen = hd.token.len;
280 if (quicconf.upstream.require_token) {
281 send_retry(faddr, version, dcid, dcidlen, scid, scidlen, remote_addr,
292 case NGTCP2_ERR_RETRY:
293 if (worker_->get_graceful_shutdown()) {
294 send_connection_close(faddr, version, hd.dcid, hd.scid, remote_addr,
295 local_addr, NGTCP2_CONNECTION_REFUSED);
299 send_retry(faddr, version, dcid, dcidlen, scid, scidlen, remote_addr,
302 case NGTCP2_ERR_VERSION_NEGOTIATION:
303 send_version_negotiation(faddr, version, dcid, dcidlen, scid, scidlen,
304 remote_addr, local_addr);
307 if (!config->single_thread && !(data[0] & 0x80) &&
308 dcidlen == SHRPX_QUIC_SCIDLEN &&
309 !std::equal(std::begin(decrypted_dcid),
310 std::begin(decrypted_dcid) + SHRPX_QUIC_CID_PREFIXLEN,
311 worker_->get_cid_prefix())) {
312 if (conn_handler->forward_quic_packet(faddr, remote_addr, local_addr,
313 decrypted_dcid.data(), data,
319 if (!(data[0] & 0x80)) {
320 // TODO Must be rate limited
321 send_stateless_reset(faddr, dcid, dcidlen, remote_addr, local_addr);
327 handler = handle_new_connection(faddr, remote_addr, local_addr, hd, podcid,
329 if (handler == nullptr) {
333 handler = (*it).second;
336 if (handler->read_quic(faddr, remote_addr, local_addr, data, datalen) != 0) {
341 handler->signal_write();
346 ClientHandler *QUICConnectionHandler::handle_new_connection(
347 const UpstreamAddr *faddr, const Address &remote_addr,
348 const Address &local_addr, const ngtcp2_pkt_hd &hd, const ngtcp2_cid *odcid,
349 const uint8_t *token, size_t tokenlen) {
350 std::array<char, NI_MAXHOST> host;
351 std::array<char, NI_MAXSERV> service;
354 rv = getnameinfo(&remote_addr.su.sa, remote_addr.len, host.data(),
355 host.size(), service.data(), service.size(),
356 NI_NUMERICHOST | NI_NUMERICSERV);
358 LOG(ERROR) << "getnameinfo() failed: " << gai_strerror(rv);
363 auto ssl_ctx = worker_->get_quic_sv_ssl_ctx();
367 auto ssl = tls::create_ssl(ssl_ctx);
368 if (ssl == nullptr) {
372 #if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
373 assert(SSL_is_quic(ssl));
374 #endif // OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
376 SSL_set_accept_state(ssl);
378 auto config = get_config();
379 auto &quicconf = config->quic;
381 if (quicconf.upstream.early_data) {
382 #if OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL)
383 SSL_set_quic_early_data_enabled(ssl, 1);
384 #else // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL))
385 SSL_set_early_data_enabled(ssl, 1);
386 #endif // !(OPENSSL_1_1_1_API && !defined(OPENSSL_IS_BORINGSSL))
389 // Disable TLS session ticket if we don't have working ticket
391 if (!worker_->get_ticket_keys()) {
392 SSL_set_options(ssl, SSL_OP_NO_TICKET);
395 auto handler = std::make_unique<ClientHandler>(
396 worker_, faddr->fd, ssl, StringRef{host.data()},
397 StringRef{service.data()}, remote_addr.su.sa.sa_family, faddr);
399 auto upstream = std::make_unique<Http3Upstream>(handler.get());
400 if (upstream->init(faddr, remote_addr, local_addr, hd, odcid, token,
405 handler->setup_http3_upstream(std::move(upstream));
407 return handler.release();
411 uint32_t generate_reserved_version(const Address &addr, uint32_t version) {
412 uint32_t h = 0x811C9DC5u;
413 const uint8_t *p = reinterpret_cast<const uint8_t *>(&addr.su.sa);
414 const uint8_t *ep = p + addr.len;
416 for (; p != ep; ++p) {
421 version = htonl(version);
422 p = (const uint8_t *)&version;
423 ep = p + sizeof(version);
425 for (; p != ep; ++p) {
437 int QUICConnectionHandler::send_retry(
438 const UpstreamAddr *faddr, uint32_t version, const uint8_t *ini_dcid,
439 size_t ini_dcidlen, const uint8_t *ini_scid, size_t ini_scidlen,
440 const Address &remote_addr, const Address &local_addr) {
441 std::array<char, NI_MAXHOST> host;
442 std::array<char, NI_MAXSERV> port;
444 if (getnameinfo(&remote_addr.su.sa, remote_addr.len, host.data(), host.size(),
445 port.data(), port.size(),
446 NI_NUMERICHOST | NI_NUMERICSERV) != 0) {
450 auto config = get_config();
451 auto &quicconf = config->quic;
453 auto conn_handler = worker_->get_connection_handler();
454 auto &qkms = conn_handler->get_quic_keying_materials();
455 auto &qkm = qkms->keying_materials.front();
457 ngtcp2_cid retry_scid;
459 if (generate_quic_retry_connection_id(retry_scid, SHRPX_QUIC_SCIDLEN,
460 quicconf.server_id.data(), qkm.id,
461 qkm.cid_encryption_key.data()) != 0) {
465 std::array<uint8_t, NGTCP2_CRYPTO_MAX_RETRY_TOKENLEN> token;
468 ngtcp2_cid idcid, iscid;
469 ngtcp2_cid_init(&idcid, ini_dcid, ini_dcidlen);
470 ngtcp2_cid_init(&iscid, ini_scid, ini_scidlen);
472 if (generate_retry_token(token.data(), tokenlen, &remote_addr.su.sa,
473 remote_addr.len, retry_scid, idcid,
474 qkm.secret.data(), qkm.secret.size()) != 0) {
478 std::vector<uint8_t> buf;
482 ngtcp2_crypto_write_retry(buf.data(), buf.size(), version, &iscid,
483 &retry_scid, &idcid, token.data(), tokenlen);
485 LOG(ERROR) << "ngtcp2_crypto_write_retry: " << ngtcp2_strerror(nwrite);
491 quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len,
492 &local_addr.su.sa, local_addr.len, buf.data(), buf.size(),
495 if (generate_quic_hashed_connection_id(idcid, remote_addr, local_addr,
501 static_cast<ev_tstamp>(NGTCP2_DEFAULT_INITIAL_RTT * 3) / NGTCP2_SECONDS;
503 if (LOG_ENABLED(INFO)) {
504 LOG(INFO) << "Enter close-wait period " << d << "s with " << buf.size()
505 << " bytes sentinel packet";
508 auto cw = std::make_unique<CloseWait>(worker_, std::vector<ngtcp2_cid>{idcid},
511 add_close_wait(cw.get());
518 int QUICConnectionHandler::send_version_negotiation(
519 const UpstreamAddr *faddr, uint32_t version, const uint8_t *ini_dcid,
520 size_t ini_dcidlen, const uint8_t *ini_scid, size_t ini_scidlen,
521 const Address &remote_addr, const Address &local_addr) {
522 std::array<uint32_t, 2> sv;
524 sv[0] = generate_reserved_version(remote_addr, version);
525 sv[1] = NGTCP2_PROTO_VER_V1;
527 std::array<uint8_t, NGTCP2_MAX_UDP_PAYLOAD_SIZE> buf;
530 util::random_bytes(&rand_byte, &rand_byte + 1, worker_->get_randgen());
532 auto nwrite = ngtcp2_pkt_write_version_negotiation(
533 buf.data(), buf.size(), rand_byte, ini_scid, ini_scidlen, ini_dcid,
534 ini_dcidlen, sv.data(), sv.size());
536 LOG(ERROR) << "ngtcp2_pkt_write_version_negotiation: "
537 << ngtcp2_strerror(nwrite);
541 return quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len,
542 &local_addr.su.sa, local_addr.len, buf.data(), nwrite,
546 int QUICConnectionHandler::send_stateless_reset(const UpstreamAddr *faddr,
549 const Address &remote_addr,
550 const Address &local_addr) {
551 if (stateless_reset_bucket_ == 0) {
552 if (LOG_ENABLED(INFO)) {
553 LOG(INFO) << "Stateless Reset bucket has been depleted";
559 --stateless_reset_bucket_;
561 if (!ev_is_active(&stateless_reset_bucket_regen_timer_)) {
562 ev_timer_again(worker_->get_loop(), &stateless_reset_bucket_regen_timer_);
566 std::array<uint8_t, NGTCP2_STATELESS_RESET_TOKENLEN> token;
569 ngtcp2_cid_init(&cid, dcid, dcidlen);
571 auto conn_handler = worker_->get_connection_handler();
572 auto &qkms = conn_handler->get_quic_keying_materials();
573 auto &qkm = qkms->keying_materials.front();
575 rv = generate_quic_stateless_reset_token(token.data(), cid, qkm.secret.data(),
581 std::array<uint8_t, NGTCP2_MIN_STATELESS_RESET_RANDLEN> rand_bytes;
583 if (RAND_bytes(rand_bytes.data(), rand_bytes.size()) != 1) {
587 std::array<uint8_t, NGTCP2_MAX_UDP_PAYLOAD_SIZE> buf;
590 ngtcp2_pkt_write_stateless_reset(buf.data(), buf.size(), token.data(),
591 rand_bytes.data(), rand_bytes.size());
593 LOG(ERROR) << "ngtcp2_pkt_write_stateless_reset: "
594 << ngtcp2_strerror(nwrite);
598 if (LOG_ENABLED(INFO)) {
599 LOG(INFO) << "Send stateless_reset to remote="
600 << util::to_numeric_addr(&remote_addr)
601 << " dcid=" << util::format_hex(dcid, dcidlen);
604 return quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len,
605 &local_addr.su.sa, local_addr.len, buf.data(), nwrite,
609 int QUICConnectionHandler::send_connection_close(
610 const UpstreamAddr *faddr, uint32_t version, const ngtcp2_cid &ini_dcid,
611 const ngtcp2_cid &ini_scid, const Address &remote_addr,
612 const Address &local_addr, uint64_t error_code) {
613 std::array<uint8_t, NGTCP2_MAX_UDP_PAYLOAD_SIZE> buf;
615 auto nwrite = ngtcp2_crypto_write_connection_close(
616 buf.data(), buf.size(), version, &ini_scid, &ini_dcid, error_code);
618 LOG(ERROR) << "ngtcp2_crypto_write_connection_close failed";
622 if (LOG_ENABLED(INFO)) {
623 LOG(INFO) << "Send Initial CONNECTION_CLOSE with error_code=" << log::hex
624 << error_code << log::dec
625 << " to remote=" << util::to_numeric_addr(&remote_addr)
626 << " dcid=" << util::format_hex(ini_scid.data, ini_scid.datalen)
627 << " scid=" << util::format_hex(ini_dcid.data, ini_dcid.datalen);
630 return quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len,
631 &local_addr.su.sa, local_addr.len, buf.data(), nwrite,
635 void QUICConnectionHandler::add_connection_id(const ngtcp2_cid &cid,
636 ClientHandler *handler) {
637 connections_.emplace(cid, handler);
640 void QUICConnectionHandler::remove_connection_id(const ngtcp2_cid &cid) {
641 connections_.erase(cid);
644 void QUICConnectionHandler::add_close_wait(CloseWait *cw) {
645 for (auto &cid : cw->scids) {
646 close_waits_.emplace(cid, cw);
650 void QUICConnectionHandler::remove_close_wait(const CloseWait *cw) {
651 for (auto &cid : cw->scids) {
652 close_waits_.erase(cid);
656 void QUICConnectionHandler::on_stateless_reset_bucket_regen() {
657 assert(stateless_reset_bucket_ < SHRPX_QUIC_STATELESS_RESET_BURST);
659 if (++stateless_reset_bucket_ == SHRPX_QUIC_STATELESS_RESET_BURST) {
660 ev_timer_stop(worker_->get_loop(), &stateless_reset_bucket_regen_timer_);
664 static void close_wait_timeoutcb(struct ev_loop *loop, ev_timer *w,
666 auto cw = static_cast<CloseWait *>(w->data);
668 if (LOG_ENABLED(INFO)) {
669 LOG(INFO) << "close-wait period finished";
672 auto quic_conn_handler = cw->worker->get_quic_connection_handler();
673 quic_conn_handler->remove_close_wait(cw);
678 CloseWait::CloseWait(Worker *worker, std::vector<ngtcp2_cid> scids,
679 std::vector<uint8_t> pkt, ev_tstamp period)
681 scids{std::move(scids)},
687 ++worker->get_worker_stat()->num_close_waits;
689 ev_timer_init(&timer, close_wait_timeoutcb, period, 0.);
692 ev_timer_start(worker->get_loop(), &timer);
695 CloseWait::~CloseWait() {
696 auto loop = worker->get_loop();
698 ev_timer_stop(loop, &timer);
700 auto worker_stat = worker->get_worker_stat();
701 --worker_stat->num_close_waits;
703 if (worker->get_graceful_shutdown() && worker_stat->num_connections == 0 &&
704 worker_stat->num_close_waits == 0) {
709 int CloseWait::handle_packet(const UpstreamAddr *faddr,
710 const Address &remote_addr,
711 const Address &local_addr, const uint8_t *data,
718 bytes_recv += datalen;
720 if (bytes_sent + pkt.size() > 3 * bytes_recv ||
721 next_pkts_recv > num_pkts_recv) {
725 if (quic_send_packet(faddr, &remote_addr.su.sa, remote_addr.len,
726 &local_addr.su.sa, local_addr.len, pkt.data(),
727 pkt.size(), 0) != 0) {
732 bytes_sent += pkt.size();