src/shrpx_http2_session.cc

   1 /*
   2  * nghttp2 - HTTP/2 C Library
   3  *
   4  * Copyright (c) 2012 Tatsuhiro Tsujikawa
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining
   7  * a copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sublicense, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice shall be
  15  * included in all copies or substantial portions of the Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  18  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  19  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  20  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
  21  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  22  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  23  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  24  */
  25 #include "shrpx_http2_session.h"
  26
  27 #include <netinet/tcp.h>
  28 #ifdef HAVE_UNISTD_H
  29 #  include <unistd.h>
  30 #endif // HAVE_UNISTD_H
  31
  32 #include <vector>
  33
  34 #include <openssl/err.h>
  35
  36 #include "shrpx_upstream.h"
  37 #include "shrpx_downstream.h"
  38 #include "shrpx_config.h"
  39 #include "shrpx_error.h"
  40 #include "shrpx_http2_downstream_connection.h"
  41 #include "shrpx_client_handler.h"
  42 #include "shrpx_tls.h"
  43 #include "shrpx_http.h"
  44 #include "shrpx_worker.h"
  45 #include "shrpx_connect_blocker.h"
  46 #include "shrpx_log.h"
  47 #include "http2.h"
  48 #include "util.h"
  49 #include "base64.h"
  50 #include "tls.h"
  51
  52 using namespace nghttp2;
  53
  54 namespace shrpx {
  55
  56 namespace {
  57 constexpr ev_tstamp CONNCHK_TIMEOUT = 5.;
  58 constexpr ev_tstamp CONNCHK_PING_TIMEOUT = 1.;
  59 } // namespace
  60
  61 namespace {
  62 constexpr size_t MAX_BUFFER_SIZE = 32_k;
  63 } // namespace
  64
  65 namespace {
  66 void connchk_timeout_cb(struct ev_loop *loop, ev_timer *w, int revents) {
  67   auto http2session = static_cast<Http2Session *>(w->data);
  68
  69   ev_timer_stop(loop, w);
  70
  71   switch (http2session->get_connection_check_state()) {
  72   case ConnectionCheck::STARTED:
  73     // ping timeout; disconnect
  74     if (LOG_ENABLED(INFO)) {
  75       SSLOG(INFO, http2session) << "ping timeout";
  76     }
  77
  78     delete http2session;
  79
  80     return;
  81   default:
  82     if (LOG_ENABLED(INFO)) {
  83       SSLOG(INFO, http2session) << "connection check required";
  84     }
  85     http2session->set_connection_check_state(ConnectionCheck::REQUIRED);
  86   }
  87 }
  88 } // namespace
  89
  90 namespace {
  91 void settings_timeout_cb(struct ev_loop *loop, ev_timer *w, int revents) {
  92   auto http2session = static_cast<Http2Session *>(w->data);
  93
  94   if (LOG_ENABLED(INFO)) {
  95     SSLOG(INFO, http2session) << "SETTINGS timeout";
  96   }
  97
  98   downstream_failure(http2session->get_addr(), http2session->get_raddr());
  99
 100   if (http2session->terminate_session(NGHTTP2_SETTINGS_TIMEOUT) != 0) {
 101     delete http2session;
 102
 103     return;
 104   }
 105   http2session->signal_write();
 106 }
 107 } // namespace
 108
 109 namespace {
 110 void timeoutcb(struct ev_loop *loop, ev_timer *w, int revents) {
 111   auto conn = static_cast<Connection *>(w->data);
 112   auto http2session = static_cast<Http2Session *>(conn->data);
 113
 114   if (w == &conn->rt && !conn->expired_rt()) {
 115     return;
 116   }
 117
 118   if (LOG_ENABLED(INFO)) {
 119     SSLOG(INFO, http2session) << "Timeout";
 120   }
 121
 122   http2session->on_timeout();
 123
 124   delete http2session;
 125 }
 126 } // namespace
 127
 128 namespace {
 129 void readcb(struct ev_loop *loop, ev_io *w, int revents) {
 130   int rv;
 131   auto conn = static_cast<Connection *>(w->data);
 132   auto http2session = static_cast<Http2Session *>(conn->data);
 133   rv = http2session->do_read();
 134   if (rv != 0) {
 135     delete http2session;
 136
 137     return;
 138   }
 139   http2session->connection_alive();
 140 }
 141 } // namespace
 142
 143 namespace {
 144 void writecb(struct ev_loop *loop, ev_io *w, int revents) {
 145   int rv;
 146   auto conn = static_cast<Connection *>(w->data);
 147   auto http2session = static_cast<Http2Session *>(conn->data);
 148   rv = http2session->do_write();
 149   if (rv != 0) {
 150     delete http2session;
 151
 152     return;
 153   }
 154   http2session->reset_connection_check_timer_if_not_checking();
 155 }
 156 } // namespace
 157
 158 namespace {
 159 void initiate_connection_cb(struct ev_loop *loop, ev_timer *w, int revents) {
 160   auto http2session = static_cast<Http2Session *>(w->data);
 161   ev_timer_stop(loop, w);
 162   if (http2session->initiate_connection() != 0) {
 163     if (LOG_ENABLED(INFO)) {
 164       SSLOG(INFO, http2session) << "Could not initiate backend connection";
 165     }
 166
 167     delete http2session;
 168
 169     return;
 170   }
 171 }
 172 } // namespace
 173
 174 namespace {
 175 void prepare_cb(struct ev_loop *loop, ev_prepare *w, int revents) {
 176   auto http2session = static_cast<Http2Session *>(w->data);
 177   http2session->check_retire();
 178 }
 179 } // namespace
 180
 181 Http2Session::Http2Session(struct ev_loop *loop, SSL_CTX *ssl_ctx,
 182                            Worker *worker,
 183                            const std::shared_ptr<DownstreamAddrGroup> &group,
 184                            DownstreamAddr *addr)
 185     : dlnext(nullptr),
 186       dlprev(nullptr),
 187       conn_(loop, -1, nullptr, worker->get_mcpool(),
 188             group->shared_addr->timeout.write, group->shared_addr->timeout.read,
 189             {}, {}, writecb, readcb, timeoutcb, this,
 190             get_config()->tls.dyn_rec.warmup_threshold,
 191             get_config()->tls.dyn_rec.idle_timeout, Proto::HTTP2),
 192       wb_(worker->get_mcpool()),
 193       worker_(worker),
 194       ssl_ctx_(ssl_ctx),
 195       group_(group),
 196       addr_(addr),
 197       session_(nullptr),
 198       raddr_(nullptr),
 199       state_(Http2SessionState::DISCONNECTED),
 200       connection_check_state_(ConnectionCheck::NONE),
 201       freelist_zone_(FreelistZone::NONE),
 202       settings_recved_(false),
 203       allow_connect_proto_(false) {
 204   read_ = write_ = &Http2Session::noop;
 205
 206   on_read_ = &Http2Session::read_noop;
 207   on_write_ = &Http2Session::write_noop;
 208
 209   // We will resuse this many times, so use repeat timeout value.  The
 210   // timeout value is set later.
 211   ev_timer_init(&connchk_timer_, connchk_timeout_cb, 0., 0.);
 212
 213   connchk_timer_.data = this;
 214
 215   // SETTINGS ACK timeout is 10 seconds for now.  We will resuse this
 216   // many times, so use repeat timeout value.
 217   ev_timer_init(&settings_timer_, settings_timeout_cb, 0., 0.);
 218
 219   settings_timer_.data = this;
 220
 221   ev_timer_init(&initiate_connection_timer_, initiate_connection_cb, 0., 0.);
 222   initiate_connection_timer_.data = this;
 223
 224   ev_prepare_init(&prep_, prepare_cb);
 225   prep_.data = this;
 226   ev_prepare_start(loop, &prep_);
 227 }
 228
 229 Http2Session::~Http2Session() {
 230   exclude_from_scheduling();
 231   disconnect(should_hard_fail());
 232 }
 233
 234 int Http2Session::disconnect(bool hard) {
 235   if (LOG_ENABLED(INFO)) {
 236     SSLOG(INFO, this) << "Disconnecting";
 237   }
 238   nghttp2_session_del(session_);
 239   session_ = nullptr;
 240
 241   wb_.reset();
 242
 243   if (dns_query_) {
 244     auto dns_tracker = worker_->get_dns_tracker();
 245     dns_tracker->cancel(dns_query_.get());
 246   }
 247
 248   conn_.rlimit.stopw();
 249   conn_.wlimit.stopw();
 250
 251   ev_prepare_stop(conn_.loop, &prep_);
 252
 253   ev_timer_stop(conn_.loop, &initiate_connection_timer_);
 254   ev_timer_stop(conn_.loop, &settings_timer_);
 255   ev_timer_stop(conn_.loop, &connchk_timer_);
 256
 257   read_ = write_ = &Http2Session::noop;
 258
 259   on_read_ = &Http2Session::read_noop;
 260   on_write_ = &Http2Session::write_noop;
 261
 262   conn_.disconnect();
 263
 264   if (proxy_htp_) {
 265     proxy_htp_.reset();
 266   }
 267
 268   connection_check_state_ = ConnectionCheck::NONE;
 269   state_ = Http2SessionState::DISCONNECTED;
 270
 271   // When deleting Http2DownstreamConnection, it calls this object's
 272   // remove_downstream_connection().  The multiple
 273   // Http2DownstreamConnection objects belong to the same
 274   // ClientHandler object if upstream is h2.  So be careful when you
 275   // delete ClientHandler here.
 276   //
 277   // We allow creating new pending Http2DownstreamConnection with this
 278   // object.  Upstream::on_downstream_reset() may add
 279   // Http2DownstreamConnection to another Http2Session.
 280
 281   for (auto dc = dconns_.head; dc;) {
 282     auto next = dc->dlnext;
 283     auto downstream = dc->get_downstream();
 284     auto upstream = downstream->get_upstream();
 285
 286     // Failure is allowed only for HTTP/1 upstream where upstream is
 287     // not shared by multiple Downstreams.
 288     if (upstream->on_downstream_reset(downstream, hard) != 0) {
 289       delete upstream->get_client_handler();
 290     }
 291
 292     // dc was deleted
 293     dc = next;
 294   }
 295
 296   auto streams = std::move(streams_);
 297   for (auto s = streams.head; s;) {
 298     auto next = s->dlnext;
 299     delete s;
 300     s = next;
 301   }
 302
 303   return 0;
 304 }
 305
 306 int Http2Session::resolve_name() {
 307   auto dns_query = std::make_unique<DNSQuery>(
 308       addr_->host, [this](DNSResolverStatus status, const Address *result) {
 309         int rv;
 310
 311         if (status == DNSResolverStatus::OK) {
 312           *resolved_addr_ = *result;
 313           util::set_port(*this->resolved_addr_, this->addr_->port);
 314         }
 315
 316         rv = this->initiate_connection();
 317         if (rv != 0) {
 318           delete this;
 319         }
 320       });
 321   resolved_addr_ = std::make_unique<Address>();
 322   auto dns_tracker = worker_->get_dns_tracker();
 323   switch (dns_tracker->resolve(resolved_addr_.get(), dns_query.get())) {
 324   case DNSResolverStatus::ERROR:
 325     return -1;
 326   case DNSResolverStatus::RUNNING:
 327     dns_query_ = std::move(dns_query);
 328     state_ = Http2SessionState::RESOLVING_NAME;
 329     return 0;
 330   case DNSResolverStatus::OK:
 331     util::set_port(*resolved_addr_, addr_->port);
 332     return 0;
 333   default:
 334     assert(0);
 335     abort();
 336   }
 337 }
 338
 339 namespace {
 340 int htp_hdrs_completecb(llhttp_t *htp);
 341 } // namespace
 342
 343 namespace {
 344 constexpr llhttp_settings_t htp_hooks = {
 345     nullptr,             // llhttp_cb      on_message_begin;
 346     nullptr,             // llhttp_data_cb on_url;
 347     nullptr,             // llhttp_data_cb on_status;
 348     nullptr,             // llhttp_data_cb on_header_field;
 349     nullptr,             // llhttp_data_cb on_header_value;
 350     htp_hdrs_completecb, // llhttp_cb      on_headers_complete;
 351     nullptr,             // llhttp_data_cb on_body;
 352     nullptr,             // llhttp_cb      on_message_complete;
 353     nullptr,             // llhttp_cb      on_chunk_header
 354     nullptr,             // llhttp_cb      on_chunk_complete
 355 };
 356 } // namespace
 357
 358 int Http2Session::initiate_connection() {
 359   int rv = 0;
 360
 361   auto worker_blocker = worker_->get_connect_blocker();
 362
 363   if (state_ == Http2SessionState::DISCONNECTED ||
 364       state_ == Http2SessionState::RESOLVING_NAME) {
 365     if (worker_blocker->blocked()) {
 366       if (LOG_ENABLED(INFO)) {
 367         SSLOG(INFO, this)
 368             << "Worker wide backend connection was blocked temporarily";
 369       }
 370       return -1;
 371     }
 372   }
 373
 374   auto &downstreamconf = *get_config()->conn.downstream;
 375
 376   const auto &proxy = get_config()->downstream_http_proxy;
 377   if (!proxy.host.empty() && state_ == Http2SessionState::DISCONNECTED) {
 378     if (LOG_ENABLED(INFO)) {
 379       SSLOG(INFO, this) << "Connecting to the proxy " << proxy.host << ":"
 380                         << proxy.port;
 381     }
 382
 383     conn_.fd = util::create_nonblock_socket(proxy.addr.su.storage.ss_family);
 384
 385     if (conn_.fd == -1) {
 386       auto error = errno;
 387       SSLOG(WARN, this) << "Backend proxy socket() failed; addr="
 388                         << util::to_numeric_addr(&proxy.addr)
 389                         << ", errno=" << error;
 390
 391       worker_blocker->on_failure();
 392       return -1;
 393     }
 394
 395     rv = connect(conn_.fd, &proxy.addr.su.sa, proxy.addr.len);
 396     if (rv != 0 && errno != EINPROGRESS) {
 397       auto error = errno;
 398       SSLOG(WARN, this) << "Backend proxy connect() failed; addr="
 399                         << util::to_numeric_addr(&proxy.addr)
 400                         << ", errno=" << error;
 401
 402       worker_blocker->on_failure();
 403
 404       return -1;
 405     }
 406
 407     raddr_ = &proxy.addr;
 408
 409     worker_blocker->on_success();
 410
 411     ev_io_set(&conn_.rev, conn_.fd, EV_READ);
 412     ev_io_set(&conn_.wev, conn_.fd, EV_WRITE);
 413
 414     conn_.wlimit.startw();
 415
 416     conn_.wt.repeat = downstreamconf.timeout.connect;
 417     ev_timer_again(conn_.loop, &conn_.wt);
 418
 419     write_ = &Http2Session::connected;
 420
 421     on_read_ = &Http2Session::downstream_read_proxy;
 422     on_write_ = &Http2Session::downstream_connect_proxy;
 423
 424     proxy_htp_ = std::make_unique<llhttp_t>();
 425     llhttp_init(proxy_htp_.get(), HTTP_RESPONSE, &htp_hooks);
 426     proxy_htp_->data = this;
 427
 428     state_ = Http2SessionState::PROXY_CONNECTING;
 429
 430     return 0;
 431   }
 432
 433   if (state_ == Http2SessionState::DISCONNECTED ||
 434       state_ == Http2SessionState::PROXY_CONNECTED ||
 435       state_ == Http2SessionState::RESOLVING_NAME) {
 436     if (LOG_ENABLED(INFO)) {
 437       if (state_ != Http2SessionState::RESOLVING_NAME) {
 438         SSLOG(INFO, this) << "Connecting to downstream server";
 439       }
 440     }
 441     if (addr_->tls) {
 442       assert(ssl_ctx_);
 443
 444       if (state_ != Http2SessionState::RESOLVING_NAME) {
 445         auto ssl = tls::create_ssl(ssl_ctx_);
 446         if (!ssl) {
 447           return -1;
 448         }
 449
 450         tls::setup_downstream_http2_alpn(ssl);
 451
 452         conn_.set_ssl(ssl);
 453         conn_.tls.client_session_cache = &addr_->tls_session_cache;
 454
 455         auto sni_name =
 456             addr_->sni.empty() ? StringRef{addr_->host} : StringRef{addr_->sni};
 457
 458         if (!util::numeric_host(sni_name.c_str())) {
 459           // TLS extensions: SNI. There is no documentation about the return
 460           // code for this function (actually this is macro wrapping SSL_ctrl
 461           // at the time of this writing).
 462           SSL_set_tlsext_host_name(conn_.tls.ssl, sni_name.c_str());
 463         }
 464
 465         auto tls_session = tls::reuse_tls_session(addr_->tls_session_cache);
 466         if (tls_session) {
 467           SSL_set_session(conn_.tls.ssl, tls_session);
 468           SSL_SESSION_free(tls_session);
 469         }
 470       }
 471
 472       if (state_ == Http2SessionState::DISCONNECTED) {
 473         if (addr_->dns) {
 474           rv = resolve_name();
 475           if (rv != 0) {
 476             downstream_failure(addr_, nullptr);
 477             return -1;
 478           }
 479           if (state_ == Http2SessionState::RESOLVING_NAME) {
 480             return 0;
 481           }
 482           raddr_ = resolved_addr_.get();
 483         } else {
 484           raddr_ = &addr_->addr;
 485         }
 486       }
 487
 488       if (state_ == Http2SessionState::RESOLVING_NAME) {
 489         if (dns_query_->status == DNSResolverStatus::ERROR) {
 490           downstream_failure(addr_, nullptr);
 491           return -1;
 492         }
 493         assert(dns_query_->status == DNSResolverStatus::OK);
 494         state_ = Http2SessionState::DISCONNECTED;
 495         dns_query_.reset();
 496         raddr_ = resolved_addr_.get();
 497       }
 498
 499       // If state_ == Http2SessionState::PROXY_CONNECTED, we have
 500       // connected to the proxy using conn_.fd and tunnel has been
 501       // established.
 502       if (state_ == Http2SessionState::DISCONNECTED) {
 503         assert(conn_.fd == -1);
 504
 505         conn_.fd = util::create_nonblock_socket(raddr_->su.storage.ss_family);
 506         if (conn_.fd == -1) {
 507           auto error = errno;
 508           SSLOG(WARN, this)
 509               << "socket() failed; addr=" << util::to_numeric_addr(raddr_)
 510               << ", errno=" << error;
 511
 512           worker_blocker->on_failure();
 513           return -1;
 514         }
 515
 516         worker_blocker->on_success();
 517
 518         rv = connect(conn_.fd,
 519                      // TODO maybe not thread-safe?
 520                      const_cast<sockaddr *>(&raddr_->su.sa), raddr_->len);
 521         if (rv != 0 && errno != EINPROGRESS) {
 522           auto error = errno;
 523           SSLOG(WARN, this)
 524               << "connect() failed; addr=" << util::to_numeric_addr(raddr_)
 525               << ", errno=" << error;
 526
 527           downstream_failure(addr_, raddr_);
 528           return -1;
 529         }
 530
 531         ev_io_set(&conn_.rev, conn_.fd, EV_READ);
 532         ev_io_set(&conn_.wev, conn_.fd, EV_WRITE);
 533       }
 534
 535       conn_.prepare_client_handshake();
 536     } else {
 537       if (state_ == Http2SessionState::DISCONNECTED) {
 538         // Without TLS and proxy.
 539         if (addr_->dns) {
 540           rv = resolve_name();
 541           if (rv != 0) {
 542             downstream_failure(addr_, nullptr);
 543             return -1;
 544           }
 545           if (state_ == Http2SessionState::RESOLVING_NAME) {
 546             return 0;
 547           }
 548           raddr_ = resolved_addr_.get();
 549         } else {
 550           raddr_ = &addr_->addr;
 551         }
 552       }
 553
 554       if (state_ == Http2SessionState::RESOLVING_NAME) {
 555         if (dns_query_->status == DNSResolverStatus::ERROR) {
 556           downstream_failure(addr_, nullptr);
 557           return -1;
 558         }
 559         assert(dns_query_->status == DNSResolverStatus::OK);
 560         state_ = Http2SessionState::DISCONNECTED;
 561         dns_query_.reset();
 562         raddr_ = resolved_addr_.get();
 563       }
 564
 565       if (state_ == Http2SessionState::DISCONNECTED) {
 566         // Without TLS and proxy.
 567         assert(conn_.fd == -1);
 568
 569         conn_.fd = util::create_nonblock_socket(raddr_->su.storage.ss_family);
 570
 571         if (conn_.fd == -1) {
 572           auto error = errno;
 573           SSLOG(WARN, this)
 574               << "socket() failed; addr=" << util::to_numeric_addr(raddr_)
 575               << ", errno=" << error;
 576
 577           worker_blocker->on_failure();
 578           return -1;
 579         }
 580
 581         worker_blocker->on_success();
 582
 583         rv = connect(conn_.fd, const_cast<sockaddr *>(&raddr_->su.sa),
 584                      raddr_->len);
 585         if (rv != 0 && errno != EINPROGRESS) {
 586           auto error = errno;
 587           SSLOG(WARN, this)
 588               << "connect() failed; addr=" << util::to_numeric_addr(raddr_)
 589               << ", errno=" << error;
 590
 591           downstream_failure(addr_, raddr_);
 592           return -1;
 593         }
 594
 595         ev_io_set(&conn_.rev, conn_.fd, EV_READ);
 596         ev_io_set(&conn_.wev, conn_.fd, EV_WRITE);
 597       }
 598     }
 599
 600     // We have been already connected when no TLS and proxy is used.
 601     if (state_ == Http2SessionState::PROXY_CONNECTED) {
 602       on_read_ = &Http2Session::read_noop;
 603       on_write_ = &Http2Session::write_noop;
 604
 605       return connected();
 606     }
 607
 608     write_ = &Http2Session::connected;
 609
 610     state_ = Http2SessionState::CONNECTING;
 611     conn_.wlimit.startw();
 612
 613     conn_.wt.repeat = downstreamconf.timeout.connect;
 614     ev_timer_again(conn_.loop, &conn_.wt);
 615
 616     return 0;
 617   }
 618
 619   // Unreachable
 620   assert(0);
 621
 622   return 0;
 623 }
 624
 625 namespace {
 626 int htp_hdrs_completecb(llhttp_t *htp) {
 627   auto http2session = static_cast<Http2Session *>(htp->data);
 628
 629   // We only read HTTP header part.  If tunneling succeeds, response
 630   // body is a different protocol (HTTP/2 in this case), we don't read
 631   // them here.
 632
 633   // We just check status code here
 634   if (htp->status_code / 100 == 2) {
 635     if (LOG_ENABLED(INFO)) {
 636       SSLOG(INFO, http2session) << "Tunneling success";
 637     }
 638     http2session->set_state(Http2SessionState::PROXY_CONNECTED);
 639
 640     return HPE_PAUSED;
 641   }
 642
 643   SSLOG(WARN, http2session) << "Tunneling failed: " << htp->status_code;
 644   http2session->set_state(Http2SessionState::PROXY_FAILED);
 645
 646   return HPE_PAUSED;
 647 }
 648 } // namespace
 649
 650 int Http2Session::downstream_read_proxy(const uint8_t *data, size_t datalen) {
 651   auto htperr = llhttp_execute(proxy_htp_.get(),
 652                                reinterpret_cast<const char *>(data), datalen);
 653   if (htperr == HPE_PAUSED) {
 654     switch (state_) {
 655     case Http2SessionState::PROXY_CONNECTED:
 656       // Initiate SSL/TLS handshake through established tunnel.
 657       if (initiate_connection() != 0) {
 658         return -1;
 659       }
 660       return 0;
 661     case Http2SessionState::PROXY_FAILED:
 662       return -1;
 663     default:
 664       break;
 665     }
 666     // should not be here
 667     assert(0);
 668   }
 669
 670   if (htperr != HPE_OK) {
 671     return -1;
 672   }
 673
 674   return 0;
 675 }
 676
 677 int Http2Session::downstream_connect_proxy() {
 678   if (LOG_ENABLED(INFO)) {
 679     SSLOG(INFO, this) << "Connected to the proxy";
 680   }
 681
 682   std::string req = "CONNECT ";
 683   req.append(addr_->hostport.c_str(), addr_->hostport.size());
 684   if (addr_->port == 80 || addr_->port == 443) {
 685     req += ':';
 686     req += util::utos(addr_->port);
 687   }
 688   req += " HTTP/1.1\r\nHost: ";
 689   req += addr_->host;
 690   req += "\r\n";
 691   const auto &proxy = get_config()->downstream_http_proxy;
 692   if (!proxy.userinfo.empty()) {
 693     req += "Proxy-Authorization: Basic ";
 694     req += base64::encode(std::begin(proxy.userinfo), std::end(proxy.userinfo));
 695     req += "\r\n";
 696   }
 697   req += "\r\n";
 698   if (LOG_ENABLED(INFO)) {
 699     SSLOG(INFO, this) << "HTTP proxy request headers\n" << req;
 700   }
 701   wb_.append(req);
 702
 703   on_write_ = &Http2Session::write_noop;
 704
 705   signal_write();
 706   return 0;
 707 }
 708
 709 void Http2Session::add_downstream_connection(Http2DownstreamConnection *dconn) {
 710   dconns_.append(dconn);
 711   ++addr_->num_dconn;
 712 }
 713
 714 void Http2Session::remove_downstream_connection(
 715     Http2DownstreamConnection *dconn) {
 716   --addr_->num_dconn;
 717   dconns_.remove(dconn);
 718   dconn->detach_stream_data();
 719
 720   if (LOG_ENABLED(INFO)) {
 721     SSLOG(INFO, this) << "Remove downstream";
 722   }
 723
 724   if (freelist_zone_ == FreelistZone::NONE && !max_concurrency_reached()) {
 725     if (LOG_ENABLED(INFO)) {
 726       SSLOG(INFO, this) << "Append to http2_extra_freelist, addr=" << addr_
 727                         << ", freelist.size="
 728                         << addr_->http2_extra_freelist.size();
 729     }
 730
 731     add_to_extra_freelist();
 732   }
 733 }
 734
 735 void Http2Session::remove_stream_data(StreamData *sd) {
 736   streams_.remove(sd);
 737   if (sd->dconn) {
 738     sd->dconn->detach_stream_data();
 739   }
 740   delete sd;
 741 }
 742
 743 int Http2Session::submit_request(Http2DownstreamConnection *dconn,
 744                                  const nghttp2_nv *nva, size_t nvlen,
 745                                  const nghttp2_data_provider *data_prd) {
 746   assert(state_ == Http2SessionState::CONNECTED);
 747   auto sd = std::make_unique<StreamData>();
 748   sd->dlnext = sd->dlprev = nullptr;
 749   // TODO Specify nullptr to pri_spec for now
 750   auto stream_id =
 751       nghttp2_submit_request(session_, nullptr, nva, nvlen, data_prd, sd.get());
 752   if (stream_id < 0) {
 753     SSLOG(FATAL, this) << "nghttp2_submit_request() failed: "
 754                        << nghttp2_strerror(stream_id);
 755     return -1;
 756   }
 757
 758   dconn->attach_stream_data(sd.get());
 759   dconn->get_downstream()->set_downstream_stream_id(stream_id);
 760   streams_.append(sd.release());
 761
 762   return 0;
 763 }
 764
 765 int Http2Session::submit_rst_stream(int32_t stream_id, uint32_t error_code) {
 766   assert(state_ == Http2SessionState::CONNECTED);
 767   if (LOG_ENABLED(INFO)) {
 768     SSLOG(INFO, this) << "RST_STREAM stream_id=" << stream_id
 769                       << " with error_code=" << error_code;
 770   }
 771   int rv = nghttp2_submit_rst_stream(session_, NGHTTP2_FLAG_NONE, stream_id,
 772                                      error_code);
 773   if (rv != 0) {
 774     SSLOG(FATAL, this) << "nghttp2_submit_rst_stream() failed: "
 775                        << nghttp2_strerror(rv);
 776     return -1;
 777   }
 778   return 0;
 779 }
 780
 781 nghttp2_session *Http2Session::get_session() const { return session_; }
 782
 783 int Http2Session::resume_data(Http2DownstreamConnection *dconn) {
 784   assert(state_ == Http2SessionState::CONNECTED);
 785   auto downstream = dconn->get_downstream();
 786   int rv = nghttp2_session_resume_data(session_,
 787                                        downstream->get_downstream_stream_id());
 788   switch (rv) {
 789   case 0:
 790   case NGHTTP2_ERR_INVALID_ARGUMENT:
 791     return 0;
 792   default:
 793     SSLOG(FATAL, this) << "nghttp2_resume_session() failed: "
 794                        << nghttp2_strerror(rv);
 795     return -1;
 796   }
 797 }
 798
 799 namespace {
 800 void call_downstream_readcb(Http2Session *http2session,
 801                             Downstream *downstream) {
 802   auto upstream = downstream->get_upstream();
 803   if (!upstream) {
 804     return;
 805   }
 806   if (upstream->downstream_read(downstream->get_downstream_connection()) != 0) {
 807     delete upstream->get_client_handler();
 808   }
 809 }
 810 } // namespace
 811
 812 namespace {
 813 int on_stream_close_callback(nghttp2_session *session, int32_t stream_id,
 814                              uint32_t error_code, void *user_data) {
 815   auto http2session = static_cast<Http2Session *>(user_data);
 816   if (LOG_ENABLED(INFO)) {
 817     SSLOG(INFO, http2session)
 818         << "Stream stream_id=" << stream_id
 819         << " is being closed with error code " << error_code;
 820   }
 821   auto sd = static_cast<StreamData *>(
 822       nghttp2_session_get_stream_user_data(session, stream_id));
 823   if (sd == 0) {
 824     // We might get this close callback when pushed streams are
 825     // closed.
 826     return 0;
 827   }
 828   auto dconn = sd->dconn;
 829   if (dconn) {
 830     auto downstream = dconn->get_downstream();
 831     auto upstream = downstream->get_upstream();
 832
 833     if (downstream->get_downstream_stream_id() % 2 == 0 &&
 834         downstream->get_request_state() == DownstreamState::INITIAL) {
 835       // Downstream is canceled in backend before it is submitted in
 836       // frontend session.
 837
 838       // This will avoid to send RST_STREAM to backend
 839       downstream->set_response_state(DownstreamState::MSG_RESET);
 840       upstream->cancel_premature_downstream(downstream);
 841     } else {
 842       if (downstream->get_upgraded() && downstream->get_response_state() ==
 843                                             DownstreamState::HEADER_COMPLETE) {
 844         // For tunneled connection, we have to submit RST_STREAM to
 845         // upstream *after* whole response body is sent. We just set
 846         // MSG_COMPLETE here. Upstream will take care of that.
 847         downstream->get_upstream()->on_downstream_body_complete(downstream);
 848         downstream->set_response_state(DownstreamState::MSG_COMPLETE);
 849       } else if (error_code == NGHTTP2_NO_ERROR) {
 850         switch (downstream->get_response_state()) {
 851         case DownstreamState::MSG_COMPLETE:
 852         case DownstreamState::MSG_BAD_HEADER:
 853           break;
 854         default:
 855           downstream->set_response_state(DownstreamState::MSG_RESET);
 856         }
 857       } else if (downstream->get_response_state() !=
 858                  DownstreamState::MSG_BAD_HEADER) {
 859         downstream->set_response_state(DownstreamState::MSG_RESET);
 860       }
 861       if (downstream->get_response_state() == DownstreamState::MSG_RESET &&
 862           downstream->get_response_rst_stream_error_code() ==
 863               NGHTTP2_NO_ERROR) {
 864         downstream->set_response_rst_stream_error_code(error_code);
 865       }
 866       call_downstream_readcb(http2session, downstream);
 867     }
 868     // dconn may be deleted
 869   }
 870   // The life time of StreamData ends here
 871   http2session->remove_stream_data(sd);
 872   return 0;
 873 }
 874 } // namespace
 875
 876 void Http2Session::start_settings_timer() {
 877   auto &downstreamconf = get_config()->http2.downstream;
 878
 879   ev_timer_set(&settings_timer_, downstreamconf.timeout.settings, 0.);
 880   ev_timer_start(conn_.loop, &settings_timer_);
 881 }
 882
 883 void Http2Session::stop_settings_timer() {
 884   ev_timer_stop(conn_.loop, &settings_timer_);
 885 }
 886
 887 namespace {
 888 int on_header_callback2(nghttp2_session *session, const nghttp2_frame *frame,
 889                         nghttp2_rcbuf *name, nghttp2_rcbuf *value,
 890                         uint8_t flags, void *user_data) {
 891   auto http2session = static_cast<Http2Session *>(user_data);
 892   auto sd = static_cast<StreamData *>(
 893       nghttp2_session_get_stream_user_data(session, frame->hd.stream_id));
 894   if (!sd || !sd->dconn) {
 895     return 0;
 896   }
 897   auto downstream = sd->dconn->get_downstream();
 898
 899   auto namebuf = nghttp2_rcbuf_get_buf(name);
 900   auto valuebuf = nghttp2_rcbuf_get_buf(value);
 901
 902   auto &resp = downstream->response();
 903   auto &httpconf = get_config()->http;
 904
 905   switch (frame->hd.type) {
 906   case NGHTTP2_HEADERS: {
 907     auto trailer = frame->headers.cat == NGHTTP2_HCAT_HEADERS &&
 908                    !downstream->get_expect_final_response();
 909
 910     if (resp.fs.buffer_size() + namebuf.len + valuebuf.len >
 911             httpconf.response_header_field_buffer ||
 912         resp.fs.num_fields() >= httpconf.max_response_header_fields) {
 913       if (LOG_ENABLED(INFO)) {
 914         DLOG(INFO, downstream)
 915             << "Too large or many header field size="
 916             << resp.fs.buffer_size() + namebuf.len + valuebuf.len
 917             << ", num=" << resp.fs.num_fields() + 1;
 918       }
 919
 920       if (trailer) {
 921         // We don't care trailer part exceeds header size limit; just
 922         // discard it.
 923         return 0;
 924       }
 925
 926       return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE;
 927     }
 928
 929     auto token = http2::lookup_token(namebuf.base, namebuf.len);
 930     auto no_index = flags & NGHTTP2_NV_FLAG_NO_INDEX;
 931
 932     downstream->add_rcbuf(name);
 933     downstream->add_rcbuf(value);
 934
 935     if (trailer) {
 936       // just store header fields for trailer part
 937       resp.fs.add_trailer_token(StringRef{namebuf.base, namebuf.len},
 938                                 StringRef{valuebuf.base, valuebuf.len},
 939                                 no_index, token);
 940       return 0;
 941     }
 942
 943     resp.fs.add_header_token(StringRef{namebuf.base, namebuf.len},
 944                              StringRef{valuebuf.base, valuebuf.len}, no_index,
 945                              token);
 946     return 0;
 947   }
 948   case NGHTTP2_PUSH_PROMISE: {
 949     auto promised_stream_id = frame->push_promise.promised_stream_id;
 950     auto promised_sd = static_cast<StreamData *>(
 951         nghttp2_session_get_stream_user_data(session, promised_stream_id));
 952     if (!promised_sd || !promised_sd->dconn) {
 953       http2session->submit_rst_stream(promised_stream_id, NGHTTP2_CANCEL);
 954       return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE;
 955     }
 956
 957     auto promised_downstream = promised_sd->dconn->get_downstream();
 958
 959     auto namebuf = nghttp2_rcbuf_get_buf(name);
 960     auto valuebuf = nghttp2_rcbuf_get_buf(value);
 961
 962     assert(promised_downstream);
 963
 964     auto &promised_req = promised_downstream->request();
 965
 966     // We use request header limit for PUSH_PROMISE
 967     if (promised_req.fs.buffer_size() + namebuf.len + valuebuf.len >
 968             httpconf.request_header_field_buffer ||
 969         promised_req.fs.num_fields() >= httpconf.max_request_header_fields) {
 970       if (LOG_ENABLED(INFO)) {
 971         DLOG(INFO, downstream)
 972             << "Too large or many header field size="
 973             << promised_req.fs.buffer_size() + namebuf.len + valuebuf.len
 974             << ", num=" << promised_req.fs.num_fields() + 1;
 975       }
 976
 977       return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE;
 978     }
 979
 980     promised_downstream->add_rcbuf(name);
 981     promised_downstream->add_rcbuf(value);
 982
 983     auto token = http2::lookup_token(namebuf.base, namebuf.len);
 984     promised_req.fs.add_header_token(StringRef{namebuf.base, namebuf.len},
 985                                      StringRef{valuebuf.base, valuebuf.len},
 986                                      flags & NGHTTP2_NV_FLAG_NO_INDEX, token);
 987
 988     return 0;
 989   }
 990   }
 991
 992   return 0;
 993 }
 994 } // namespace
 995
 996 namespace {
 997 int on_invalid_header_callback2(nghttp2_session *session,
 998                                 const nghttp2_frame *frame, nghttp2_rcbuf *name,
 999                                 nghttp2_rcbuf *value, uint8_t flags,
1000                                 void *user_data) {
1001   auto http2session = static_cast<Http2Session *>(user_data);
1002   auto sd = static_cast<StreamData *>(
1003       nghttp2_session_get_stream_user_data(session, frame->hd.stream_id));
1004   if (!sd || !sd->dconn) {
1005     return 0;
1006   }
1007
1008   int32_t stream_id;
1009
1010   if (frame->hd.type == NGHTTP2_PUSH_PROMISE) {
1011     stream_id = frame->push_promise.promised_stream_id;
1012   } else {
1013     stream_id = frame->hd.stream_id;
1014   }
1015
1016   if (LOG_ENABLED(INFO)) {
1017     auto namebuf = nghttp2_rcbuf_get_buf(name);
1018     auto valuebuf = nghttp2_rcbuf_get_buf(value);
1019
1020     SSLOG(INFO, http2session)
1021         << "Invalid header field for stream_id=" << stream_id
1022         << " in frame type=" << static_cast<uint32_t>(frame->hd.type)
1023         << ": name=[" << StringRef{namebuf.base, namebuf.len} << "], value=["
1024         << StringRef{valuebuf.base, valuebuf.len} << "]";
1025   }
1026
1027   http2session->submit_rst_stream(stream_id, NGHTTP2_PROTOCOL_ERROR);
1028
1029   return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE;
1030 }
1031 } // namespace
1032
1033 namespace {
1034 int on_begin_headers_callback(nghttp2_session *session,
1035                               const nghttp2_frame *frame, void *user_data) {
1036   auto http2session = static_cast<Http2Session *>(user_data);
1037
1038   switch (frame->hd.type) {
1039   case NGHTTP2_HEADERS: {
1040     if (frame->headers.cat != NGHTTP2_HCAT_RESPONSE &&
1041         frame->headers.cat != NGHTTP2_HCAT_PUSH_RESPONSE) {
1042       return 0;
1043     }
1044     auto sd = static_cast<StreamData *>(
1045         nghttp2_session_get_stream_user_data(session, frame->hd.stream_id));
1046     if (!sd || !sd->dconn) {
1047       http2session->submit_rst_stream(frame->hd.stream_id,
1048                                       NGHTTP2_INTERNAL_ERROR);
1049       return 0;
1050     }
1051     return 0;
1052   }
1053   case NGHTTP2_PUSH_PROMISE: {
1054     auto promised_stream_id = frame->push_promise.promised_stream_id;
1055     auto sd = static_cast<StreamData *>(
1056         nghttp2_session_get_stream_user_data(session, frame->hd.stream_id));
1057     if (!sd || !sd->dconn) {
1058       http2session->submit_rst_stream(promised_stream_id, NGHTTP2_CANCEL);
1059       return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE;
1060     }
1061
1062     auto downstream = sd->dconn->get_downstream();
1063
1064     assert(downstream);
1065     assert(downstream->get_downstream_stream_id() == frame->hd.stream_id);
1066
1067     if (http2session->handle_downstream_push_promise(downstream,
1068                                                      promised_stream_id) != 0) {
1069       http2session->submit_rst_stream(promised_stream_id, NGHTTP2_CANCEL);
1070       return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE;
1071     }
1072
1073     return 0;
1074   }
1075   }
1076
1077   return 0;
1078 }
1079 } // namespace
1080
1081 namespace {
1082 int on_response_headers(Http2Session *http2session, Downstream *downstream,
1083                         nghttp2_session *session, const nghttp2_frame *frame) {
1084   int rv;
1085
1086   auto upstream = downstream->get_upstream();
1087   auto handler = upstream->get_client_handler();
1088   const auto &req = downstream->request();
1089   auto &resp = downstream->response();
1090
1091   auto &nva = resp.fs.headers();
1092
1093   auto config = get_config();
1094   auto &loggingconf = config->logging;
1095
1096   downstream->set_expect_final_response(false);
1097
1098   auto status = resp.fs.header(http2::HD__STATUS);
1099   // libnghttp2 guarantees this exists and can be parsed
1100   assert(status);
1101   auto status_code = http2::parse_http_status_code(status->value);
1102
1103   resp.http_status = status_code;
1104   resp.http_major = 2;
1105   resp.http_minor = 0;
1106
1107   downstream->set_downstream_addr_group(
1108       http2session->get_downstream_addr_group());
1109   downstream->set_addr(http2session->get_addr());
1110
1111   if (LOG_ENABLED(INFO)) {
1112     std::stringstream ss;
1113     for (auto &nv : nva) {
1114       ss << TTY_HTTP_HD << nv.name << TTY_RST << ": " << nv.value << "\n";
1115     }
1116     SSLOG(INFO, http2session)
1117         << "HTTP response headers. stream_id=" << frame->hd.stream_id << "\n"
1118         << ss.str();
1119   }
1120
1121   if (downstream->get_non_final_response()) {
1122
1123     if (LOG_ENABLED(INFO)) {
1124       SSLOG(INFO, http2session) << "This is non-final response.";
1125     }
1126
1127     downstream->set_expect_final_response(true);
1128     rv = upstream->on_downstream_header_complete(downstream);
1129
1130     // Now Dowstream's response headers are erased.
1131
1132     if (rv != 0) {
1133       http2session->submit_rst_stream(frame->hd.stream_id,
1134                                       NGHTTP2_PROTOCOL_ERROR);
1135       downstream->set_response_state(DownstreamState::MSG_RESET);
1136     }
1137
1138     return 0;
1139   }
1140
1141   downstream->set_response_state(DownstreamState::HEADER_COMPLETE);
1142   downstream->check_upgrade_fulfilled_http2();
1143
1144   if (downstream->get_upgraded()) {
1145     resp.connection_close = true;
1146     // On upgrade success, both ends can send data
1147     if (upstream->resume_read(SHRPX_NO_BUFFER, downstream, 0) != 0) {
1148       // If resume_read fails, just drop connection. Not ideal.
1149       delete handler;
1150       return -1;
1151     }
1152     downstream->set_request_state(DownstreamState::HEADER_COMPLETE);
1153     if (LOG_ENABLED(INFO)) {
1154       SSLOG(INFO, http2session)
1155           << "HTTP upgrade success. stream_id=" << frame->hd.stream_id;
1156     }
1157   } else {
1158     auto content_length = resp.fs.header(http2::HD_CONTENT_LENGTH);
1159     if (content_length) {
1160       // libnghttp2 guarantees this can be parsed
1161       resp.fs.content_length = util::parse_uint(content_length->value);
1162     }
1163
1164     if (resp.fs.content_length == -1 && downstream->expect_response_body()) {
1165       // Here we have response body but Content-Length is not known in
1166       // advance.
1167       if (req.http_major <= 0 || (req.http_major == 1 && req.http_minor == 0)) {
1168         // We simply close connection for pre-HTTP/1.1 in this case.
1169         resp.connection_close = true;
1170       } else {
1171         // Otherwise, use chunked encoding to keep upstream connection
1172         // open.  In HTTP2, we are supposed not to receive
1173         // transfer-encoding.
1174         resp.fs.add_header_token(StringRef::from_lit("transfer-encoding"),
1175                                  StringRef::from_lit("chunked"), false,
1176                                  http2::HD_TRANSFER_ENCODING);
1177         downstream->set_chunked_response(true);
1178       }
1179     }
1180   }
1181
1182   if (frame->hd.flags & NGHTTP2_FLAG_END_STREAM) {
1183     resp.headers_only = true;
1184   }
1185
1186   if (loggingconf.access.write_early && downstream->accesslog_ready()) {
1187     handler->write_accesslog(downstream);
1188     downstream->set_accesslog_written(true);
1189   }
1190
1191   rv = upstream->on_downstream_header_complete(downstream);
1192   if (rv != 0) {
1193     // Handling early return (in other words, response was hijacked by
1194     // mruby scripting).
1195     if (downstream->get_response_state() == DownstreamState::MSG_COMPLETE) {
1196       http2session->submit_rst_stream(frame->hd.stream_id, NGHTTP2_CANCEL);
1197     } else {
1198       http2session->submit_rst_stream(frame->hd.stream_id,
1199                                       NGHTTP2_INTERNAL_ERROR);
1200       downstream->set_response_state(DownstreamState::MSG_RESET);
1201     }
1202   }
1203
1204   return 0;
1205 }
1206 } // namespace
1207
1208 namespace {
1209 int on_frame_recv_callback(nghttp2_session *session, const nghttp2_frame *frame,
1210                            void *user_data) {
1211   int rv;
1212   auto http2session = static_cast<Http2Session *>(user_data);
1213
1214   switch (frame->hd.type) {
1215   case NGHTTP2_DATA: {
1216     auto sd = static_cast<StreamData *>(
1217         nghttp2_session_get_stream_user_data(session, frame->hd.stream_id));
1218     if (!sd || !sd->dconn) {
1219       return 0;
1220     }
1221     auto downstream = sd->dconn->get_downstream();
1222     auto upstream = downstream->get_upstream();
1223     rv = upstream->on_downstream_body(downstream, nullptr, 0, true);
1224     if (rv != 0) {
1225       http2session->submit_rst_stream(frame->hd.stream_id,
1226                                       NGHTTP2_INTERNAL_ERROR);
1227       downstream->set_response_state(DownstreamState::MSG_RESET);
1228
1229     } else if (frame->hd.flags & NGHTTP2_FLAG_END_STREAM) {
1230
1231       downstream->disable_downstream_rtimer();
1232
1233       if (downstream->get_response_state() ==
1234           DownstreamState::HEADER_COMPLETE) {
1235
1236         downstream->set_response_state(DownstreamState::MSG_COMPLETE);
1237
1238         rv = upstream->on_downstream_body_complete(downstream);
1239
1240         if (rv != 0) {
1241           downstream->set_response_state(DownstreamState::MSG_RESET);
1242         }
1243       }
1244     }
1245
1246     call_downstream_readcb(http2session, downstream);
1247     return 0;
1248   }
1249   case NGHTTP2_HEADERS: {
1250     auto sd = static_cast<StreamData *>(
1251         nghttp2_session_get_stream_user_data(session, frame->hd.stream_id));
1252     if (!sd || !sd->dconn) {
1253       return 0;
1254     }
1255     auto downstream = sd->dconn->get_downstream();
1256
1257     if (frame->headers.cat == NGHTTP2_HCAT_RESPONSE ||
1258         frame->headers.cat == NGHTTP2_HCAT_PUSH_RESPONSE) {
1259       rv = on_response_headers(http2session, downstream, session, frame);
1260
1261       if (rv != 0) {
1262         return 0;
1263       }
1264     } else if (frame->headers.cat == NGHTTP2_HCAT_HEADERS) {
1265       if (downstream->get_expect_final_response()) {
1266         rv = on_response_headers(http2session, downstream, session, frame);
1267
1268         if (rv != 0) {
1269           return 0;
1270         }
1271       }
1272     }
1273
1274     if (frame->hd.flags & NGHTTP2_FLAG_END_STREAM) {
1275       downstream->disable_downstream_rtimer();
1276
1277       if (downstream->get_response_state() ==
1278           DownstreamState::HEADER_COMPLETE) {
1279         downstream->set_response_state(DownstreamState::MSG_COMPLETE);
1280
1281         auto upstream = downstream->get_upstream();
1282
1283         rv = upstream->on_downstream_body_complete(downstream);
1284
1285         if (rv != 0) {
1286           downstream->set_response_state(DownstreamState::MSG_RESET);
1287         }
1288       }
1289     } else {
1290       downstream->reset_downstream_rtimer();
1291     }
1292
1293     // This may delete downstream
1294     call_downstream_readcb(http2session, downstream);
1295
1296     return 0;
1297   }
1298   case NGHTTP2_RST_STREAM: {
1299     auto sd = static_cast<StreamData *>(
1300         nghttp2_session_get_stream_user_data(session, frame->hd.stream_id));
1301     if (sd && sd->dconn) {
1302       auto downstream = sd->dconn->get_downstream();
1303       downstream->set_response_rst_stream_error_code(
1304           frame->rst_stream.error_code);
1305       call_downstream_readcb(http2session, downstream);
1306     }
1307     return 0;
1308   }
1309   case NGHTTP2_SETTINGS: {
1310     if ((frame->hd.flags & NGHTTP2_FLAG_ACK) == 0) {
1311       http2session->on_settings_received(frame);
1312       return 0;
1313     }
1314
1315     http2session->stop_settings_timer();
1316
1317     auto addr = http2session->get_addr();
1318     auto &connect_blocker = addr->connect_blocker;
1319
1320     connect_blocker->on_success();
1321
1322     return 0;
1323   }
1324   case NGHTTP2_PING:
1325     if (frame->hd.flags & NGHTTP2_FLAG_ACK) {
1326       if (LOG_ENABLED(INFO)) {
1327         LOG(INFO) << "PING ACK received";
1328       }
1329       http2session->connection_alive();
1330     }
1331     return 0;
1332   case NGHTTP2_PUSH_PROMISE: {
1333     auto promised_stream_id = frame->push_promise.promised_stream_id;
1334
1335     if (LOG_ENABLED(INFO)) {
1336       SSLOG(INFO, http2session)
1337           << "Received downstream PUSH_PROMISE stream_id="
1338           << frame->hd.stream_id
1339           << ", promised_stream_id=" << promised_stream_id;
1340     }
1341
1342     auto sd = static_cast<StreamData *>(
1343         nghttp2_session_get_stream_user_data(session, frame->hd.stream_id));
1344     if (!sd || !sd->dconn) {
1345       http2session->submit_rst_stream(promised_stream_id, NGHTTP2_CANCEL);
1346       return 0;
1347     }
1348
1349     auto downstream = sd->dconn->get_downstream();
1350
1351     assert(downstream);
1352     assert(downstream->get_downstream_stream_id() == frame->hd.stream_id);
1353
1354     auto promised_sd = static_cast<StreamData *>(
1355         nghttp2_session_get_stream_user_data(session, promised_stream_id));
1356     if (!promised_sd || !promised_sd->dconn) {
1357       http2session->submit_rst_stream(promised_stream_id, NGHTTP2_CANCEL);
1358       return 0;
1359     }
1360
1361     auto promised_downstream = promised_sd->dconn->get_downstream();
1362
1363     assert(promised_downstream);
1364
1365     if (http2session->handle_downstream_push_promise_complete(
1366             downstream, promised_downstream) != 0) {
1367       http2session->submit_rst_stream(promised_stream_id, NGHTTP2_CANCEL);
1368       return 0;
1369     }
1370
1371     return 0;
1372   }
1373   case NGHTTP2_GOAWAY:
1374     if (LOG_ENABLED(INFO)) {
1375       auto debug_data = util::ascii_dump(frame->goaway.opaque_data,
1376                                          frame->goaway.opaque_data_len);
1377
1378       SSLOG(INFO, http2session)
1379           << "GOAWAY received: last-stream-id=" << frame->goaway.last_stream_id
1380           << ", error_code=" << frame->goaway.error_code
1381           << ", debug_data=" << debug_data;
1382     }
1383     return 0;
1384   default:
1385     return 0;
1386   }
1387 }
1388 } // namespace
1389
1390 namespace {
1391 int on_data_chunk_recv_callback(nghttp2_session *session, uint8_t flags,
1392                                 int32_t stream_id, const uint8_t *data,
1393                                 size_t len, void *user_data) {
1394   int rv;
1395   auto http2session = static_cast<Http2Session *>(user_data);
1396   auto sd = static_cast<StreamData *>(
1397       nghttp2_session_get_stream_user_data(session, stream_id));
1398   if (!sd || !sd->dconn) {
1399     http2session->submit_rst_stream(stream_id, NGHTTP2_INTERNAL_ERROR);
1400
1401     if (http2session->consume(stream_id, len) != 0) {
1402       return NGHTTP2_ERR_CALLBACK_FAILURE;
1403     }
1404
1405     return 0;
1406   }
1407   auto downstream = sd->dconn->get_downstream();
1408   if (!downstream->expect_response_body()) {
1409     http2session->submit_rst_stream(stream_id, NGHTTP2_INTERNAL_ERROR);
1410
1411     if (http2session->consume(stream_id, len) != 0) {
1412       return NGHTTP2_ERR_CALLBACK_FAILURE;
1413     }
1414
1415     return 0;
1416   }
1417
1418   // We don't want DATA after non-final response, which is illegal in
1419   // HTTP.
1420   if (downstream->get_non_final_response()) {
1421     http2session->submit_rst_stream(stream_id, NGHTTP2_PROTOCOL_ERROR);
1422
1423     if (http2session->consume(stream_id, len) != 0) {
1424       return NGHTTP2_ERR_CALLBACK_FAILURE;
1425     }
1426
1427     return 0;
1428   }
1429
1430   downstream->reset_downstream_rtimer();
1431
1432   auto &resp = downstream->response();
1433
1434   resp.recv_body_length += len;
1435   resp.unconsumed_body_length += len;
1436
1437   auto upstream = downstream->get_upstream();
1438   rv = upstream->on_downstream_body(downstream, data, len, false);
1439   if (rv != 0) {
1440     http2session->submit_rst_stream(stream_id, NGHTTP2_INTERNAL_ERROR);
1441
1442     if (http2session->consume(stream_id, len) != 0) {
1443       return NGHTTP2_ERR_CALLBACK_FAILURE;
1444     }
1445
1446     downstream->set_response_state(DownstreamState::MSG_RESET);
1447   }
1448
1449   call_downstream_readcb(http2session, downstream);
1450   return 0;
1451 }
1452 } // namespace
1453
1454 namespace {
1455 int on_frame_send_callback(nghttp2_session *session, const nghttp2_frame *frame,
1456                            void *user_data) {
1457   auto http2session = static_cast<Http2Session *>(user_data);
1458
1459   if (frame->hd.type == NGHTTP2_DATA || frame->hd.type == NGHTTP2_HEADERS) {
1460     auto sd = static_cast<StreamData *>(
1461         nghttp2_session_get_stream_user_data(session, frame->hd.stream_id));
1462
1463     if (!sd || !sd->dconn) {
1464       return 0;
1465     }
1466
1467     auto downstream = sd->dconn->get_downstream();
1468
1469     if (frame->hd.type == NGHTTP2_HEADERS &&
1470         frame->headers.cat == NGHTTP2_HCAT_REQUEST) {
1471       downstream->set_request_header_sent(true);
1472       auto src = downstream->get_blocked_request_buf();
1473       if (src->rleft()) {
1474         auto dest = downstream->get_request_buf();
1475         src->remove(*dest);
1476         if (http2session->resume_data(sd->dconn) != 0) {
1477           return NGHTTP2_ERR_CALLBACK_FAILURE;
1478         }
1479         downstream->ensure_downstream_wtimer();
1480       }
1481     }
1482
1483     if ((frame->hd.flags & NGHTTP2_FLAG_END_STREAM) == 0) {
1484       return 0;
1485     }
1486
1487     downstream->reset_downstream_rtimer();
1488
1489     return 0;
1490   }
1491
1492   if (frame->hd.type == NGHTTP2_SETTINGS &&
1493       (frame->hd.flags & NGHTTP2_FLAG_ACK) == 0) {
1494     http2session->start_settings_timer();
1495   }
1496   return 0;
1497 }
1498 } // namespace
1499
1500 namespace {
1501 int on_frame_not_send_callback(nghttp2_session *session,
1502                                const nghttp2_frame *frame, int lib_error_code,
1503                                void *user_data) {
1504   auto http2session = static_cast<Http2Session *>(user_data);
1505   if (LOG_ENABLED(INFO)) {
1506     SSLOG(INFO, http2session) << "Failed to send control frame type="
1507                               << static_cast<uint32_t>(frame->hd.type)
1508                               << ", lib_error_code=" << lib_error_code << ": "
1509                               << nghttp2_strerror(lib_error_code);
1510   }
1511   if (frame->hd.type != NGHTTP2_HEADERS ||
1512       lib_error_code == NGHTTP2_ERR_STREAM_CLOSED ||
1513       lib_error_code == NGHTTP2_ERR_STREAM_CLOSING) {
1514     return 0;
1515   }
1516
1517   auto sd = static_cast<StreamData *>(
1518       nghttp2_session_get_stream_user_data(session, frame->hd.stream_id));
1519   if (!sd) {
1520     return 0;
1521   }
1522   if (!sd->dconn) {
1523     return 0;
1524   }
1525   auto downstream = sd->dconn->get_downstream();
1526
1527   if (lib_error_code == NGHTTP2_ERR_START_STREAM_NOT_ALLOWED) {
1528     // Migrate to another downstream connection.
1529     auto upstream = downstream->get_upstream();
1530
1531     if (upstream->on_downstream_reset(downstream, false)) {
1532       // This should be done for h1 upstream only.  Deleting
1533       // ClientHandler for h2 upstream may lead to crash.
1534       delete upstream->get_client_handler();
1535     }
1536
1537     return 0;
1538   }
1539
1540   // To avoid stream hanging around, flag DownstreamState::MSG_RESET.
1541   downstream->set_response_state(DownstreamState::MSG_RESET);
1542   call_downstream_readcb(http2session, downstream);
1543
1544   return 0;
1545 }
1546 } // namespace
1547
1548 namespace {
1549 constexpr auto PADDING = std::array<uint8_t, 256>{};
1550 } // namespace
1551
1552 namespace {
1553 int send_data_callback(nghttp2_session *session, nghttp2_frame *frame,
1554                        const uint8_t *framehd, size_t length,
1555                        nghttp2_data_source *source, void *user_data) {
1556   auto http2session = static_cast<Http2Session *>(user_data);
1557   auto sd = static_cast<StreamData *>(
1558       nghttp2_session_get_stream_user_data(session, frame->hd.stream_id));
1559
1560   if (sd == nullptr) {
1561     return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE;
1562   }
1563
1564   auto dconn = sd->dconn;
1565   auto downstream = dconn->get_downstream();
1566   auto input = downstream->get_request_buf();
1567   auto wb = http2session->get_request_buf();
1568
1569   size_t padlen = 0;
1570
1571   wb->append(framehd, 9);
1572   if (frame->data.padlen > 0) {
1573     padlen = frame->data.padlen - 1;
1574     wb->append(static_cast<uint8_t>(padlen));
1575   }
1576
1577   input->remove(*wb, length);
1578
1579   wb->append(PADDING.data(), padlen);
1580
1581   if (input->rleft() == 0) {
1582     downstream->disable_downstream_wtimer();
1583   } else {
1584     downstream->reset_downstream_wtimer();
1585   }
1586
1587   if (length > 0) {
1588     // This is important because it will handle flow control
1589     // stuff.
1590     if (downstream->get_upstream()->resume_read(SHRPX_NO_BUFFER, downstream,
1591                                                 length) != 0) {
1592       // In this case, downstream may be deleted.
1593       return NGHTTP2_ERR_TEMPORAL_CALLBACK_FAILURE;
1594     }
1595
1596     // Here sd->dconn could be nullptr, because
1597     // Upstream::resume_read() may delete downstream which will delete
1598     // dconn.  Is this still really true?
1599   }
1600
1601   return 0;
1602 }
1603 } // namespace
1604
1605 nghttp2_session_callbacks *create_http2_downstream_callbacks() {
1606   int rv;
1607   nghttp2_session_callbacks *callbacks;
1608
1609   rv = nghttp2_session_callbacks_new(&callbacks);
1610
1611   if (rv != 0) {
1612     return nullptr;
1613   }
1614
1615   nghttp2_session_callbacks_set_on_stream_close_callback(
1616       callbacks, on_stream_close_callback);
1617
1618   nghttp2_session_callbacks_set_on_frame_recv_callback(callbacks,
1619                                                        on_frame_recv_callback);
1620
1621   nghttp2_session_callbacks_set_on_data_chunk_recv_callback(
1622       callbacks, on_data_chunk_recv_callback);
1623
1624   nghttp2_session_callbacks_set_on_frame_send_callback(callbacks,
1625                                                        on_frame_send_callback);
1626
1627   nghttp2_session_callbacks_set_on_frame_not_send_callback(
1628       callbacks, on_frame_not_send_callback);
1629
1630   nghttp2_session_callbacks_set_on_header_callback2(callbacks,
1631                                                     on_header_callback2);
1632
1633   nghttp2_session_callbacks_set_on_invalid_header_callback2(
1634       callbacks, on_invalid_header_callback2);
1635
1636   nghttp2_session_callbacks_set_on_begin_headers_callback(
1637       callbacks, on_begin_headers_callback);
1638
1639   nghttp2_session_callbacks_set_send_data_callback(callbacks,
1640                                                    send_data_callback);
1641
1642   if (get_config()->padding) {
1643     nghttp2_session_callbacks_set_select_padding_callback(
1644         callbacks, http::select_padding_callback);
1645   }
1646
1647   return callbacks;
1648 }
1649
1650 int Http2Session::connection_made() {
1651   int rv;
1652
1653   state_ = Http2SessionState::CONNECTED;
1654
1655   on_write_ = &Http2Session::downstream_write;
1656   on_read_ = &Http2Session::downstream_read;
1657
1658   if (addr_->tls) {
1659     const unsigned char *next_proto = nullptr;
1660     unsigned int next_proto_len = 0;
1661
1662 #ifndef OPENSSL_NO_NEXTPROTONEG
1663     SSL_get0_next_proto_negotiated(conn_.tls.ssl, &next_proto, &next_proto_len);
1664 #endif // !OPENSSL_NO_NEXTPROTONEG
1665 #if OPENSSL_VERSION_NUMBER >= 0x10002000L
1666     if (!next_proto) {
1667       SSL_get0_alpn_selected(conn_.tls.ssl, &next_proto, &next_proto_len);
1668     }
1669 #endif // OPENSSL_VERSION_NUMBER >= 0x10002000L
1670
1671     if (!next_proto) {
1672       downstream_failure(addr_, raddr_);
1673       return -1;
1674     }
1675
1676     auto proto = StringRef{next_proto, next_proto_len};
1677     if (LOG_ENABLED(INFO)) {
1678       SSLOG(INFO, this) << "Negotiated next protocol: " << proto;
1679     }
1680     if (!util::check_h2_is_selected(proto)) {
1681       downstream_failure(addr_, raddr_);
1682       return -1;
1683     }
1684   }
1685
1686   auto config = get_config();
1687   auto &http2conf = config->http2;
1688
1689   rv = nghttp2_session_client_new2(&session_, http2conf.downstream.callbacks,
1690                                    this, http2conf.downstream.option);
1691
1692   if (rv != 0) {
1693     return -1;
1694   }
1695
1696   std::array<nghttp2_settings_entry, 4> entry;
1697   size_t nentry = 2;
1698   entry[0].settings_id = NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS;
1699   entry[0].value = http2conf.downstream.max_concurrent_streams;
1700
1701   entry[1].settings_id = NGHTTP2_SETTINGS_INITIAL_WINDOW_SIZE;
1702   entry[1].value = http2conf.downstream.window_size;
1703
1704   if (http2conf.no_server_push || config->http2_proxy) {
1705     entry[nentry].settings_id = NGHTTP2_SETTINGS_ENABLE_PUSH;
1706     entry[nentry].value = 0;
1707     ++nentry;
1708   }
1709
1710   if (http2conf.downstream.decoder_dynamic_table_size !=
1711       NGHTTP2_DEFAULT_HEADER_TABLE_SIZE) {
1712     entry[nentry].settings_id = NGHTTP2_SETTINGS_HEADER_TABLE_SIZE;
1713     entry[nentry].value = http2conf.downstream.decoder_dynamic_table_size;
1714     ++nentry;
1715   }
1716
1717   rv = nghttp2_submit_settings(session_, NGHTTP2_FLAG_NONE, entry.data(),
1718                                nentry);
1719   if (rv != 0) {
1720     return -1;
1721   }
1722
1723   rv = nghttp2_session_set_local_window_size(
1724       session_, NGHTTP2_FLAG_NONE, 0,
1725       http2conf.downstream.connection_window_size);
1726   if (rv != 0) {
1727     return -1;
1728   }
1729
1730   reset_connection_check_timer(CONNCHK_TIMEOUT);
1731
1732   submit_pending_requests();
1733
1734   signal_write();
1735   return 0;
1736 }
1737
1738 int Http2Session::do_read() { return read_(*this); }
1739 int Http2Session::do_write() { return write_(*this); }
1740
1741 int Http2Session::on_read(const uint8_t *data, size_t datalen) {
1742   return on_read_(*this, data, datalen);
1743 }
1744
1745 int Http2Session::on_write() { return on_write_(*this); }
1746
1747 int Http2Session::downstream_read(const uint8_t *data, size_t datalen) {
1748   ssize_t rv;
1749
1750   rv = nghttp2_session_mem_recv(session_, data, datalen);
1751   if (rv < 0) {
1752     SSLOG(ERROR, this) << "nghttp2_session_mem_recv() returned error: "
1753                        << nghttp2_strerror(rv);
1754     return -1;
1755   }
1756
1757   if (nghttp2_session_want_read(session_) == 0 &&
1758       nghttp2_session_want_write(session_) == 0 && wb_.rleft() == 0) {
1759     if (LOG_ENABLED(INFO)) {
1760       SSLOG(INFO, this) << "No more read/write for this HTTP2 session";
1761     }
1762     return -1;
1763   }
1764
1765   signal_write();
1766   return 0;
1767 }
1768
1769 int Http2Session::downstream_write() {
1770   for (;;) {
1771     const uint8_t *data;
1772     auto datalen = nghttp2_session_mem_send(session_, &data);
1773     if (datalen < 0) {
1774       SSLOG(ERROR, this) << "nghttp2_session_mem_send() returned error: "
1775                          << nghttp2_strerror(datalen);
1776       return -1;
1777     }
1778     if (datalen == 0) {
1779       break;
1780     }
1781     wb_.append(data, datalen);
1782
1783     if (wb_.rleft() >= MAX_BUFFER_SIZE) {
1784       break;
1785     }
1786   }
1787
1788   if (nghttp2_session_want_read(session_) == 0 &&
1789       nghttp2_session_want_write(session_) == 0 && wb_.rleft() == 0) {
1790     if (LOG_ENABLED(INFO)) {
1791       SSLOG(INFO, this) << "No more read/write for this session";
1792     }
1793     return -1;
1794   }
1795
1796   return 0;
1797 }
1798
1799 void Http2Session::signal_write() {
1800   switch (state_) {
1801   case Http2SessionState::DISCONNECTED:
1802     if (!ev_is_active(&initiate_connection_timer_)) {
1803       if (LOG_ENABLED(INFO)) {
1804         LOG(INFO) << "Start connecting to backend server";
1805       }
1806       // Since the timer is set to 0., these will feed 2 events.  We
1807       // will stop the timer in the initiate_connection_timer_ to void
1808       // 2nd event.
1809       ev_timer_start(conn_.loop, &initiate_connection_timer_);
1810       ev_feed_event(conn_.loop, &initiate_connection_timer_, 0);
1811     }
1812     break;
1813   case Http2SessionState::CONNECTED:
1814     conn_.wlimit.startw();
1815     break;
1816   default:
1817     break;
1818   }
1819 }
1820
1821 struct ev_loop *Http2Session::get_loop() const {
1822   return conn_.loop;
1823 }
1824
1825 ev_io *Http2Session::get_wev() { return &conn_.wev; }
1826
1827 Http2SessionState Http2Session::get_state() const { return state_; }
1828
1829 void Http2Session::set_state(Http2SessionState state) { state_ = state; }
1830
1831 int Http2Session::terminate_session(uint32_t error_code) {
1832   int rv;
1833   rv = nghttp2_session_terminate_session(session_, error_code);
1834   if (rv != 0) {
1835     return -1;
1836   }
1837   return 0;
1838 }
1839
1840 SSL *Http2Session::get_ssl() const { return conn_.tls.ssl; }
1841
1842 int Http2Session::consume(int32_t stream_id, size_t len) {
1843   int rv;
1844
1845   if (!session_) {
1846     return 0;
1847   }
1848
1849   rv = nghttp2_session_consume(session_, stream_id, len);
1850
1851   if (rv != 0) {
1852     SSLOG(WARN, this) << "nghttp2_session_consume() returned error: "
1853                       << nghttp2_strerror(rv);
1854
1855     return -1;
1856   }
1857
1858   return 0;
1859 }
1860
1861 bool Http2Session::can_push_request(const Downstream *downstream) const {
1862   auto &req = downstream->request();
1863   return state_ == Http2SessionState::CONNECTED &&
1864          connection_check_state_ == ConnectionCheck::NONE &&
1865          (req.connect_proto == ConnectProto::NONE || settings_recved_);
1866 }
1867
1868 void Http2Session::start_checking_connection() {
1869   if (state_ != Http2SessionState::CONNECTED ||
1870       connection_check_state_ != ConnectionCheck::REQUIRED) {
1871     return;
1872   }
1873   connection_check_state_ = ConnectionCheck::STARTED;
1874
1875   SSLOG(INFO, this) << "Start checking connection";
1876   // If connection is down, we may get error when writing data.  Issue
1877   // ping frame to see whether connection is alive.
1878   nghttp2_submit_ping(session_, NGHTTP2_FLAG_NONE, NULL);
1879
1880   // set ping timeout and start timer again
1881   reset_connection_check_timer(CONNCHK_PING_TIMEOUT);
1882
1883   signal_write();
1884 }
1885
1886 void Http2Session::reset_connection_check_timer(ev_tstamp t) {
1887   connchk_timer_.repeat = t;
1888   ev_timer_again(conn_.loop, &connchk_timer_);
1889 }
1890
1891 void Http2Session::reset_connection_check_timer_if_not_checking() {
1892   if (connection_check_state_ != ConnectionCheck::NONE) {
1893     return;
1894   }
1895
1896   reset_connection_check_timer(CONNCHK_TIMEOUT);
1897 }
1898
1899 void Http2Session::connection_alive() {
1900   reset_connection_check_timer(CONNCHK_TIMEOUT);
1901
1902   if (connection_check_state_ == ConnectionCheck::NONE) {
1903     return;
1904   }
1905
1906   if (LOG_ENABLED(INFO)) {
1907     SSLOG(INFO, this) << "Connection alive";
1908   }
1909
1910   connection_check_state_ = ConnectionCheck::NONE;
1911
1912   submit_pending_requests();
1913 }
1914
1915 void Http2Session::submit_pending_requests() {
1916   for (auto dconn = dconns_.head; dconn; dconn = dconn->dlnext) {
1917     auto downstream = dconn->get_downstream();
1918
1919     if (!downstream->get_request_pending() ||
1920         !downstream->request_submission_ready()) {
1921       continue;
1922     }
1923
1924     auto &req = downstream->request();
1925     if (req.connect_proto != ConnectProto::NONE && !settings_recved_) {
1926       continue;
1927     }
1928
1929     auto upstream = downstream->get_upstream();
1930
1931     if (dconn->push_request_headers() != 0) {
1932       if (LOG_ENABLED(INFO)) {
1933         SSLOG(INFO, this) << "backend request failed";
1934       }
1935
1936       upstream->on_downstream_abort_request(downstream, 400);
1937
1938       continue;
1939     }
1940
1941     upstream->resume_read(SHRPX_NO_BUFFER, downstream, 0);
1942   }
1943 }
1944
1945 void Http2Session::set_connection_check_state(ConnectionCheck state) {
1946   connection_check_state_ = state;
1947 }
1948
1949 ConnectionCheck Http2Session::get_connection_check_state() const {
1950   return connection_check_state_;
1951 }
1952
1953 int Http2Session::noop() { return 0; }
1954
1955 int Http2Session::read_noop(const uint8_t *data, size_t datalen) { return 0; }
1956
1957 int Http2Session::write_noop() { return 0; }
1958
1959 int Http2Session::connected() {
1960   auto sock_error = util::get_socket_error(conn_.fd);
1961   if (sock_error != 0) {
1962     SSLOG(WARN, this) << "Backend connect failed; addr="
1963                       << util::to_numeric_addr(raddr_)
1964                       << ": errno=" << sock_error;
1965
1966     downstream_failure(addr_, raddr_);
1967
1968     return -1;
1969   }
1970
1971   if (LOG_ENABLED(INFO)) {
1972     SSLOG(INFO, this) << "Connection established";
1973   }
1974
1975   // Reset timeout for write.  Previously, we set timeout for connect.
1976   conn_.wt.repeat = group_->shared_addr->timeout.write;
1977   ev_timer_again(conn_.loop, &conn_.wt);
1978
1979   conn_.rlimit.startw();
1980   conn_.again_rt();
1981
1982   read_ = &Http2Session::read_clear;
1983   write_ = &Http2Session::write_clear;
1984
1985   if (state_ == Http2SessionState::PROXY_CONNECTING) {
1986     return do_write();
1987   }
1988
1989   if (conn_.tls.ssl) {
1990     read_ = &Http2Session::tls_handshake;
1991     write_ = &Http2Session::tls_handshake;
1992
1993     return do_write();
1994   }
1995
1996   if (connection_made() != 0) {
1997     state_ = Http2SessionState::CONNECT_FAILING;
1998     return -1;
1999   }
2000
2001   return 0;
2002 }
2003
2004 int Http2Session::read_clear() {
2005   conn_.last_read = ev_now(conn_.loop);
2006
2007   std::array<uint8_t, 16_k> buf;
2008
2009   for (;;) {
2010     auto nread = conn_.read_clear(buf.data(), buf.size());
2011
2012     if (nread == 0) {
2013       return write_clear();
2014     }
2015
2016     if (nread < 0) {
2017       return nread;
2018     }
2019
2020     if (on_read(buf.data(), nread) != 0) {
2021       return -1;
2022     }
2023   }
2024 }
2025
2026 int Http2Session::write_clear() {
2027   conn_.last_read = ev_now(conn_.loop);
2028
2029   std::array<struct iovec, MAX_WR_IOVCNT> iov;
2030
2031   for (;;) {
2032     if (wb_.rleft() > 0) {
2033       auto iovcnt = wb_.riovec(iov.data(), iov.size());
2034       auto nwrite = conn_.writev_clear(iov.data(), iovcnt);
2035
2036       if (nwrite == 0) {
2037         return 0;
2038       }
2039
2040       if (nwrite < 0) {
2041         // We may have pending data in receive buffer which may
2042         // contain part of response body.  So keep reading.  Invoke
2043         // read event to get read(2) error just in case.
2044         ev_feed_event(conn_.loop, &conn_.rev, EV_READ);
2045         write_ = &Http2Session::write_void;
2046         break;
2047       }
2048
2049       wb_.drain(nwrite);
2050       continue;
2051     }
2052
2053     if (on_write() != 0) {
2054       return -1;
2055     }
2056     if (wb_.rleft() == 0) {
2057       break;
2058     }
2059   }
2060
2061   conn_.wlimit.stopw();
2062   ev_timer_stop(conn_.loop, &conn_.wt);
2063
2064   return 0;
2065 }
2066
2067 int Http2Session::tls_handshake() {
2068   conn_.last_read = ev_now(conn_.loop);
2069
2070   ERR_clear_error();
2071
2072   auto rv = conn_.tls_handshake();
2073
2074   if (rv == SHRPX_ERR_INPROGRESS) {
2075     return 0;
2076   }
2077
2078   if (rv < 0) {
2079     downstream_failure(addr_, raddr_);
2080
2081     return rv;
2082   }
2083
2084   if (LOG_ENABLED(INFO)) {
2085     SSLOG(INFO, this) << "SSL/TLS handshake completed";
2086   }
2087
2088   if (!get_config()->tls.insecure &&
2089       tls::check_cert(conn_.tls.ssl, addr_, raddr_) != 0) {
2090     downstream_failure(addr_, raddr_);
2091
2092     return -1;
2093   }
2094
2095   read_ = &Http2Session::read_tls;
2096   write_ = &Http2Session::write_tls;
2097
2098   if (connection_made() != 0) {
2099     state_ = Http2SessionState::CONNECT_FAILING;
2100     return -1;
2101   }
2102
2103   return 0;
2104 }
2105
2106 int Http2Session::read_tls() {
2107   conn_.last_read = ev_now(conn_.loop);
2108
2109   std::array<uint8_t, 16_k> buf;
2110
2111   ERR_clear_error();
2112
2113   for (;;) {
2114     auto nread = conn_.read_tls(buf.data(), buf.size());
2115
2116     if (nread == 0) {
2117       return write_tls();
2118     }
2119
2120     if (nread < 0) {
2121       return nread;
2122     }
2123
2124     if (on_read(buf.data(), nread) != 0) {
2125       return -1;
2126     }
2127   }
2128 }
2129
2130 int Http2Session::write_tls() {
2131   conn_.last_read = ev_now(conn_.loop);
2132
2133   ERR_clear_error();
2134
2135   struct iovec iov;
2136
2137   for (;;) {
2138     if (wb_.rleft() > 0) {
2139       auto iovcnt = wb_.riovec(&iov, 1);
2140       if (iovcnt != 1) {
2141         assert(0);
2142         return -1;
2143       }
2144       auto nwrite = conn_.write_tls(iov.iov_base, iov.iov_len);
2145
2146       if (nwrite == 0) {
2147         return 0;
2148       }
2149
2150       if (nwrite < 0) {
2151         // We may have pending data in receive buffer which may
2152         // contain part of response body.  So keep reading.  Invoke
2153         // read event to get read(2) error just in case.
2154         ev_feed_event(conn_.loop, &conn_.rev, EV_READ);
2155         write_ = &Http2Session::write_void;
2156         break;
2157       }
2158
2159       wb_.drain(nwrite);
2160
2161       continue;
2162     }
2163
2164     if (on_write() != 0) {
2165       return -1;
2166     }
2167     if (wb_.rleft() == 0) {
2168       conn_.start_tls_write_idle();
2169       break;
2170     }
2171   }
2172
2173   conn_.wlimit.stopw();
2174   ev_timer_stop(conn_.loop, &conn_.wt);
2175
2176   return 0;
2177 }
2178
2179 int Http2Session::write_void() {
2180   conn_.wlimit.stopw();
2181   return 0;
2182 }
2183
2184 bool Http2Session::should_hard_fail() const {
2185   switch (state_) {
2186   case Http2SessionState::PROXY_CONNECTING:
2187   case Http2SessionState::PROXY_FAILED:
2188     return true;
2189   case Http2SessionState::DISCONNECTED: {
2190     const auto &proxy = get_config()->downstream_http_proxy;
2191     return !proxy.host.empty();
2192   }
2193   default:
2194     return false;
2195   }
2196 }
2197
2198 DownstreamAddr *Http2Session::get_addr() const { return addr_; }
2199
2200 int Http2Session::handle_downstream_push_promise(Downstream *downstream,
2201                                                  int32_t promised_stream_id) {
2202   auto upstream = downstream->get_upstream();
2203   if (!upstream->push_enabled()) {
2204     return -1;
2205   }
2206
2207   auto promised_downstream =
2208       upstream->on_downstream_push_promise(downstream, promised_stream_id);
2209   if (!promised_downstream) {
2210     return -1;
2211   }
2212
2213   // Now we have Downstream object for pushed stream.
2214   // promised_downstream->get_stream() still returns 0.
2215
2216   auto handler = upstream->get_client_handler();
2217
2218   auto promised_dconn = std::make_unique<Http2DownstreamConnection>(this);
2219   promised_dconn->set_client_handler(handler);
2220
2221   auto ptr = promised_dconn.get();
2222
2223   if (promised_downstream->attach_downstream_connection(
2224           std::move(promised_dconn)) != 0) {
2225     return -1;
2226   }
2227
2228   auto promised_sd = std::make_unique<StreamData>();
2229
2230   nghttp2_session_set_stream_user_data(session_, promised_stream_id,
2231                                        promised_sd.get());
2232
2233   ptr->attach_stream_data(promised_sd.get());
2234   streams_.append(promised_sd.release());
2235
2236   return 0;
2237 }
2238
2239 int Http2Session::handle_downstream_push_promise_complete(
2240     Downstream *downstream, Downstream *promised_downstream) {
2241   auto &promised_req = promised_downstream->request();
2242
2243   auto &promised_balloc = promised_downstream->get_block_allocator();
2244
2245   auto authority = promised_req.fs.header(http2::HD__AUTHORITY);
2246   auto path = promised_req.fs.header(http2::HD__PATH);
2247   auto method = promised_req.fs.header(http2::HD__METHOD);
2248   auto scheme = promised_req.fs.header(http2::HD__SCHEME);
2249
2250   if (!authority) {
2251     authority = promised_req.fs.header(http2::HD_HOST);
2252   }
2253
2254   auto method_token = http2::lookup_method_token(method->value);
2255   if (method_token == -1) {
2256     if (LOG_ENABLED(INFO)) {
2257       SSLOG(INFO, this) << "Unrecognized method: " << method->value;
2258     }
2259
2260     return -1;
2261   }
2262
2263   // TODO Rewrite authority if we enabled rewrite host.  But we
2264   // really don't know how to rewrite host.  Should we use the same
2265   // host in associated stream?
2266   if (authority) {
2267     promised_req.authority = authority->value;
2268   }
2269   promised_req.method = method_token;
2270   // libnghttp2 ensures that we don't have CONNECT method in
2271   // PUSH_PROMISE, and guarantees that :scheme exists.
2272   if (scheme) {
2273     promised_req.scheme = scheme->value;
2274   }
2275
2276   // For server-wide OPTIONS request, path is empty.
2277   if (method_token != HTTP_OPTIONS || path->value != "*") {
2278     promised_req.path = http2::rewrite_clean_path(promised_balloc, path->value);
2279   }
2280
2281   promised_downstream->inspect_http2_request();
2282
2283   auto upstream = promised_downstream->get_upstream();
2284
2285   promised_downstream->set_request_state(DownstreamState::MSG_COMPLETE);
2286   promised_downstream->set_request_header_sent(true);
2287
2288   if (upstream->on_downstream_push_promise_complete(downstream,
2289                                                     promised_downstream) != 0) {
2290     return -1;
2291   }
2292
2293   return 0;
2294 }
2295
2296 size_t Http2Session::get_num_dconns() const { return dconns_.size(); }
2297
2298 bool Http2Session::max_concurrency_reached(size_t extra) const {
2299   if (!session_) {
2300     return dconns_.size() + extra >= 100;
2301   }
2302
2303   // If session does not allow further requests, it effectively means
2304   // that maximum concurrency is reached.
2305   return !nghttp2_session_check_request_allowed(session_) ||
2306          dconns_.size() + extra >=
2307              nghttp2_session_get_remote_settings(
2308                  session_, NGHTTP2_SETTINGS_MAX_CONCURRENT_STREAMS);
2309 }
2310
2311 const std::shared_ptr<DownstreamAddrGroup> &
2312 Http2Session::get_downstream_addr_group() const {
2313   return group_;
2314 }
2315
2316 void Http2Session::add_to_extra_freelist() {
2317   if (freelist_zone_ != FreelistZone::NONE) {
2318     return;
2319   }
2320
2321   if (LOG_ENABLED(INFO)) {
2322     SSLOG(INFO, this) << "Append to http2_extra_freelist, addr=" << addr_
2323                       << ", freelist.size="
2324                       << addr_->http2_extra_freelist.size();
2325   }
2326
2327   freelist_zone_ = FreelistZone::EXTRA;
2328   addr_->http2_extra_freelist.append(this);
2329 }
2330
2331 void Http2Session::remove_from_freelist() {
2332   switch (freelist_zone_) {
2333   case FreelistZone::NONE:
2334     return;
2335   case FreelistZone::EXTRA:
2336     if (LOG_ENABLED(INFO)) {
2337       SSLOG(INFO, this) << "Remove from http2_extra_freelist, addr=" << addr_
2338                         << ", freelist.size="
2339                         << addr_->http2_extra_freelist.size();
2340     }
2341     addr_->http2_extra_freelist.remove(this);
2342     break;
2343   case FreelistZone::GONE:
2344     return;
2345   }
2346
2347   freelist_zone_ = FreelistZone::NONE;
2348 }
2349
2350 void Http2Session::exclude_from_scheduling() {
2351   remove_from_freelist();
2352   freelist_zone_ = FreelistZone::GONE;
2353 }
2354
2355 DefaultMemchunks *Http2Session::get_request_buf() { return &wb_; }
2356
2357 void Http2Session::on_timeout() {
2358   switch (state_) {
2359   case Http2SessionState::PROXY_CONNECTING: {
2360     auto worker_blocker = worker_->get_connect_blocker();
2361     worker_blocker->on_failure();
2362     break;
2363   }
2364   case Http2SessionState::CONNECTING:
2365     SSLOG(WARN, this) << "Connect time out; addr="
2366                       << util::to_numeric_addr(raddr_);
2367
2368     downstream_failure(addr_, raddr_);
2369     break;
2370   default:
2371     break;
2372   }
2373 }
2374
2375 void Http2Session::check_retire() {
2376   if (!group_->retired) {
2377     return;
2378   }
2379
2380   ev_prepare_stop(conn_.loop, &prep_);
2381
2382   if (!session_) {
2383     return;
2384   }
2385
2386   auto last_stream_id = nghttp2_session_get_last_proc_stream_id(session_);
2387   nghttp2_submit_goaway(session_, NGHTTP2_FLAG_NONE, last_stream_id,
2388                         NGHTTP2_NO_ERROR, nullptr, 0);
2389
2390   signal_write();
2391 }
2392
2393 const Address *Http2Session::get_raddr() const { return raddr_; }
2394
2395 void Http2Session::on_settings_received(const nghttp2_frame *frame) {
2396   // TODO This effectively disallows nghttpx to change its behaviour
2397   // based on the 2nd SETTINGS.
2398   if (settings_recved_) {
2399     return;
2400   }
2401
2402   settings_recved_ = true;
2403
2404   for (size_t i = 0; i < frame->settings.niv; ++i) {
2405     auto &ent = frame->settings.iv[i];
2406     if (ent.settings_id == NGHTTP2_SETTINGS_ENABLE_CONNECT_PROTOCOL) {
2407       allow_connect_proto_ = true;
2408       break;
2409     }
2410   }
2411
2412   submit_pending_requests();
2413 }
2414
2415 bool Http2Session::get_allow_connect_proto() const {
2416   return allow_connect_proto_;
2417 }
2418
2419 } // namespace shrpx