2 * nghttp2 - HTTP/2 C Library
4 * Copyright (c) 2015 Tatsuhiro Tsujikawa
6 * Permission is hereby granted, free of charge, to any person obtaining
7 * a copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sublicense, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice shall be
15 * included in all copies or substantial portions of the Software.
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
20 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25 #include "shrpx_worker_process.h"
27 #include <sys/types.h>
30 #endif // HAVE_UNISTD_H
31 #include <sys/resource.h>
38 #include <openssl/rand.h>
44 #include "shrpx_config.h"
45 #include "shrpx_connection_handler.h"
46 #include "shrpx_log_config.h"
47 #include "shrpx_worker.h"
48 #include "shrpx_accept_handler.h"
49 #include "shrpx_http2_upstream.h"
50 #include "shrpx_http2_session.h"
51 #include "shrpx_memcached_dispatcher.h"
52 #include "shrpx_memcached_request.h"
53 #include "shrpx_process.h"
54 #include "shrpx_tls.h"
55 #include "shrpx_log.h"
57 #include "app_helper.h"
59 #include "xsi_strerror.h"
61 using namespace nghttp2;
67 #ifdef HAVE_NEVERBLEED
69 #endif // HAVE_NEVERBLEED
71 std::array<char, STRERROR_BUFSIZE> errbuf;
72 auto config = get_config();
74 if (getuid() == 0 && config->uid != 0) {
75 #ifdef HAVE_NEVERBLEED
77 neverbleed_setuidgid(nb, config->user.c_str(), 1);
79 #endif // HAVE_NEVERBLEED
81 if (initgroups(config->user.c_str(), config->gid) != 0) {
83 LOG(FATAL) << "Could not change supplementary groups: "
84 << xsi_strerror(error, errbuf.data(), errbuf.size());
87 if (setgid(config->gid) != 0) {
89 LOG(FATAL) << "Could not change gid: "
90 << xsi_strerror(error, errbuf.data(), errbuf.size());
93 if (setuid(config->uid) != 0) {
95 LOG(FATAL) << "Could not change uid: "
96 << xsi_strerror(error, errbuf.data(), errbuf.size());
99 if (setuid(0) != -1) {
100 LOG(FATAL) << "Still have root privileges?";
108 void graceful_shutdown(ConnectionHandler *conn_handler) {
109 if (conn_handler->get_graceful_shutdown()) {
113 LOG(NOTICE) << "Graceful shutdown signal received";
115 conn_handler->set_graceful_shutdown(true);
117 // TODO What happens for the connections not established in the
119 conn_handler->accept_pending_connection();
120 conn_handler->delete_acceptor();
122 conn_handler->graceful_shutdown_worker();
124 auto single_worker = conn_handler->get_single_worker();
126 auto worker_stat = single_worker->get_worker_stat();
127 if (worker_stat->num_connections == 0 &&
128 worker_stat->num_close_waits == 0) {
129 ev_break(conn_handler->get_loop());
138 void reopen_log(ConnectionHandler *conn_handler) {
139 LOG(NOTICE) << "Reopening log files: worker process (thread main)";
141 auto config = get_config();
142 auto &loggingconf = config->logging;
144 (void)reopen_log_files(loggingconf);
145 redirect_stderr_to_errorlog(loggingconf);
147 conn_handler->worker_reopen_log_files();
152 void ipc_readcb(struct ev_loop *loop, ev_io *w, int revents) {
153 auto conn_handler = static_cast<ConnectionHandler *>(w->data);
154 std::array<uint8_t, 1024> buf;
156 while ((nread = read(w->fd, buf.data(), buf.size())) == -1 && errno == EINTR)
160 LOG(ERROR) << "Failed to read data from ipc channel: errno=" << error;
165 // IPC socket closed. Perform immediate shutdown.
166 LOG(FATAL) << "IPC socket is closed. Perform immediate shutdown.";
167 nghttp2_Exit(EXIT_FAILURE);
170 for (ssize_t i = 0; i < nread; ++i) {
172 case SHRPX_IPC_GRACEFUL_SHUTDOWN:
173 graceful_shutdown(conn_handler);
175 case SHRPX_IPC_REOPEN_LOG:
176 reopen_log(conn_handler);
185 void quic_ipc_readcb(struct ev_loop *loop, ev_io *w, int revents) {
186 auto conn_handler = static_cast<ConnectionHandler *>(w->data);
188 if (conn_handler->quic_ipc_read() != 0) {
189 LOG(ERROR) << "Failed to read data from QUIC IPC channel";
195 #endif // ENABLE_HTTP3
198 int generate_ticket_key(TicketKey &ticket_key) {
199 ticket_key.cipher = get_config()->tls.ticket.cipher;
200 ticket_key.hmac = EVP_sha256();
201 ticket_key.hmac_keylen = EVP_MD_size(ticket_key.hmac);
203 assert(static_cast<size_t>(EVP_CIPHER_key_length(ticket_key.cipher)) <=
204 ticket_key.data.enc_key.size());
205 assert(ticket_key.hmac_keylen <= ticket_key.data.hmac_key.size());
207 if (LOG_ENABLED(INFO)) {
208 LOG(INFO) << "enc_keylen=" << EVP_CIPHER_key_length(ticket_key.cipher)
209 << ", hmac_keylen=" << ticket_key.hmac_keylen;
212 if (RAND_bytes(reinterpret_cast<unsigned char *>(&ticket_key.data),
213 sizeof(ticket_key.data)) == 0) {
222 void renew_ticket_key_cb(struct ev_loop *loop, ev_timer *w, int revents) {
223 auto conn_handler = static_cast<ConnectionHandler *>(w->data);
224 const auto &old_ticket_keys = conn_handler->get_ticket_keys();
226 auto ticket_keys = std::make_shared<TicketKeys>();
227 LOG(NOTICE) << "Renew new ticket keys";
229 // If old_ticket_keys is not empty, it should contain at least 2
230 // keys: one for encryption, and last one for the next encryption
231 // key but decryption only. The keys in between are old keys and
232 // decryption only. The next key is provided to ensure to mitigate
233 // possible problem when one worker encrypt new key, but one worker,
234 // which did not take the that key yet, and cannot decrypt it.
236 // We keep keys for get_config()->tls_session_timeout seconds. The
237 // default is 12 hours. Thus the maximum ticket vector size is 12.
238 if (old_ticket_keys) {
239 auto &old_keys = old_ticket_keys->keys;
240 auto &new_keys = ticket_keys->keys;
242 assert(!old_keys.empty());
245 static_cast<size_t>(std::chrono::duration_cast<std::chrono::hours>(
246 get_config()->tls.session_timeout)
249 new_keys.resize(std::min(max_tickets, old_keys.size() + 1));
250 std::copy_n(std::begin(old_keys), new_keys.size() - 1,
251 std::begin(new_keys) + 1);
253 ticket_keys->keys.resize(1);
256 auto &new_key = ticket_keys->keys[0];
258 if (generate_ticket_key(new_key) != 0) {
259 if (LOG_ENABLED(INFO)) {
260 LOG(INFO) << "failed to generate ticket key";
262 conn_handler->set_ticket_keys(nullptr);
263 conn_handler->set_ticket_keys_to_worker(nullptr);
267 if (LOG_ENABLED(INFO)) {
268 LOG(INFO) << "ticket keys generation done";
269 assert(ticket_keys->keys.size() >= 1);
270 LOG(INFO) << 0 << " enc+dec: "
271 << util::format_hex(ticket_keys->keys[0].data.name);
272 for (size_t i = 1; i < ticket_keys->keys.size(); ++i) {
273 auto &key = ticket_keys->keys[i];
274 LOG(INFO) << i << " dec: " << util::format_hex(key.data.name);
278 conn_handler->set_ticket_keys(ticket_keys);
279 conn_handler->set_ticket_keys_to_worker(ticket_keys);
284 void memcached_get_ticket_key_cb(struct ev_loop *loop, ev_timer *w,
286 auto conn_handler = static_cast<ConnectionHandler *>(w->data);
287 auto dispatcher = conn_handler->get_tls_ticket_key_memcached_dispatcher();
289 auto req = std::make_unique<MemcachedRequest>();
290 req->key = "nghttpx:tls-ticket-key";
291 req->op = MemcachedOp::GET;
292 req->cb = [conn_handler, w](MemcachedRequest *req, MemcachedResult res) {
293 switch (res.status_code) {
294 case MemcachedStatusCode::NO_ERROR:
296 case MemcachedStatusCode::EXT_NETWORK_ERROR:
297 conn_handler->on_tls_ticket_key_network_error(w);
300 conn_handler->on_tls_ticket_key_not_found(w);
304 // |version (4bytes)|len (2bytes)|key (variable length)|...
305 // (len, key) pairs are repeated as necessary.
307 auto &value = res.value;
308 if (value.size() < 4) {
309 LOG(WARN) << "Memcached: tls ticket key value is too small: got "
311 conn_handler->on_tls_ticket_key_not_found(w);
314 auto p = value.data();
315 auto version = util::get_uint32(p);
316 // Currently supported version is 1.
318 LOG(WARN) << "Memcached: tls ticket key version: want 1, got " << version;
319 conn_handler->on_tls_ticket_key_not_found(w);
323 auto end = p + value.size();
326 auto &ticketconf = get_config()->tls.ticket;
331 if (ticketconf.cipher == EVP_aes_128_cbc()) {
335 } else if (ticketconf.cipher == EVP_aes_256_cbc()) {
343 auto ticket_keys = std::make_shared<TicketKeys>();
347 LOG(WARN) << "Memcached: tls ticket key data is too small";
348 conn_handler->on_tls_ticket_key_not_found(w);
351 auto len = util::get_uint16(p);
353 if (len != expectedlen) {
354 LOG(WARN) << "Memcached: wrong tls ticket key size: want "
355 << expectedlen << ", got " << len;
356 conn_handler->on_tls_ticket_key_not_found(w);
360 LOG(WARN) << "Memcached: too short tls ticket key payload: want " << len
361 << ", got " << (end - p);
362 conn_handler->on_tls_ticket_key_not_found(w);
365 auto key = TicketKey();
366 key.cipher = ticketconf.cipher;
367 key.hmac = EVP_sha256();
368 key.hmac_keylen = hmac_keylen;
370 std::copy_n(p, key.data.name.size(), std::begin(key.data.name));
371 p += key.data.name.size();
373 std::copy_n(p, enc_keylen, std::begin(key.data.enc_key));
376 std::copy_n(p, hmac_keylen, std::begin(key.data.hmac_key));
379 ticket_keys->keys.push_back(std::move(key));
382 conn_handler->on_tls_ticket_key_get_success(ticket_keys, w);
385 if (LOG_ENABLED(INFO)) {
386 LOG(INFO) << "Memcached: tls ticket key get request sent";
389 dispatcher->add_request(std::move(req));
394 #ifdef HAVE_NEVERBLEED
396 void nb_child_cb(struct ev_loop *loop, ev_child *w, int revents) {
397 log_chld(w->rpid, w->rstatus, "neverbleed process");
399 ev_child_stop(loop, w);
401 LOG(FATAL) << "neverbleed process exitted; aborting now";
403 nghttp2_Exit(EXIT_FAILURE);
406 #endif // HAVE_NEVERBLEED
408 int worker_process_event_loop(WorkerProcessConfig *wpconf) {
410 std::array<char, STRERROR_BUFSIZE> errbuf;
413 auto config = get_config();
415 if (reopen_log_files(config->logging) != 0) {
416 LOG(FATAL) << "Failed to open log file";
420 rv = ares_library_init(ARES_LIB_INIT_ALL);
422 LOG(FATAL) << "ares_library_init failed: " << ares_strerror(rv);
426 auto loop = EV_DEFAULT;
428 auto gen = util::make_mt19937();
430 #ifdef HAVE_NEVERBLEED
431 std::array<char, NEVERBLEED_ERRBUF_SIZE> nb_errbuf;
432 auto nb = std::make_unique<neverbleed_t>();
433 if (neverbleed_init(nb.get(), nb_errbuf.data()) != 0) {
434 LOG(FATAL) << "neverbleed_init failed: " << nb_errbuf.data();
438 LOG(NOTICE) << "neverbleed process [" << nb->daemon_pid << "] spawned";
442 ev_child_init(&nb_childev, nb_child_cb, nb->daemon_pid, 0);
443 nb_childev.data = nullptr;
444 ev_child_start(loop, &nb_childev);
445 #endif // HAVE_NEVERBLEED
447 auto conn_handler = std::make_unique<ConnectionHandler>(loop, gen);
449 #ifdef HAVE_NEVERBLEED
450 conn_handler->set_neverbleed(nb.get());
451 #endif // HAVE_NEVERBLEED
454 conn_handler->set_quic_ipc_fd(wpconf->quic_ipc_fd);
455 conn_handler->set_quic_lingering_worker_processes(
456 wpconf->quic_lingering_worker_processes);
457 #endif // ENABLE_HTTP3
459 for (auto &addr : config->conn.listener.addrs) {
460 conn_handler->add_acceptor(
461 std::make_unique<AcceptHandler>(&addr, conn_handler.get()));
466 ev_timer renew_ticket_key_timer;
467 if (tls::upstream_tls_enabled(config->conn)) {
468 auto &ticketconf = config->tls.ticket;
469 auto &memcachedconf = ticketconf.memcached;
471 if (!memcachedconf.host.empty()) {
472 SSL_CTX *ssl_ctx = nullptr;
474 if (memcachedconf.tls) {
475 ssl_ctx = conn_handler->create_tls_ticket_key_memcached_ssl_ctx();
478 conn_handler->set_tls_ticket_key_memcached_dispatcher(
479 std::make_unique<MemcachedDispatcher>(
480 &ticketconf.memcached.addr, loop, ssl_ctx,
481 StringRef{memcachedconf.host}, &mcpool, gen));
483 ev_timer_init(&renew_ticket_key_timer, memcached_get_ticket_key_cb, 0.,
485 renew_ticket_key_timer.data = conn_handler.get();
486 // Get first ticket keys.
487 memcached_get_ticket_key_cb(loop, &renew_ticket_key_timer, 0);
489 bool auto_tls_ticket_key = true;
490 if (!ticketconf.files.empty()) {
491 if (!ticketconf.cipher_given) {
493 << "It is strongly recommended to specify "
494 "--tls-ticket-key-cipher=aes-128-cbc (or "
495 "tls-ticket-key-cipher=aes-128-cbc in configuration file) "
496 "when --tls-ticket-key-file is used for the smooth "
497 "transition when the default value of --tls-ticket-key-cipher "
498 "becomes aes-256-cbc";
500 auto ticket_keys = read_tls_ticket_key_file(
501 ticketconf.files, ticketconf.cipher, EVP_sha256());
503 LOG(WARN) << "Use internal session ticket key generator";
505 conn_handler->set_ticket_keys(std::move(ticket_keys));
506 auto_tls_ticket_key = false;
509 if (auto_tls_ticket_key) {
510 // Generate new ticket key every 1hr.
511 ev_timer_init(&renew_ticket_key_timer, renew_ticket_key_cb, 0., 1_h);
512 renew_ticket_key_timer.data = conn_handler.get();
513 ev_timer_again(loop, &renew_ticket_key_timer);
515 // Generate first session ticket key before running workers.
516 renew_ticket_key_cb(loop, &renew_ticket_key_timer, 0);
522 auto &quicconf = config->quic;
524 std::shared_ptr<QUICKeyingMaterials> qkms;
526 if (!quicconf.upstream.secret_file.empty()) {
527 qkms = read_quic_secret_file(quicconf.upstream.secret_file);
529 LOG(WARN) << "Use QUIC keying materials generated internally";
534 qkms = std::make_shared<QUICKeyingMaterials>();
535 qkms->keying_materials.resize(1);
537 auto &qkm = qkms->keying_materials.front();
539 if (RAND_bytes(qkm.reserved.data(), qkm.reserved.size()) != 1) {
540 LOG(ERROR) << "Failed to generate QUIC secret reserved data";
544 if (RAND_bytes(qkm.secret.data(), qkm.secret.size()) != 1) {
545 LOG(ERROR) << "Failed to generate QUIC secret";
549 if (RAND_bytes(qkm.salt.data(), qkm.salt.size()) != 1) {
550 LOG(ERROR) << "Failed to generate QUIC salt";
555 for (auto &qkm : qkms->keying_materials) {
556 if (generate_quic_connection_id_encryption_key(
557 qkm.cid_encryption_key.data(), qkm.cid_encryption_key.size(),
558 qkm.secret.data(), qkm.secret.size(), qkm.salt.data(),
559 qkm.salt.size()) != 0) {
560 LOG(ERROR) << "Failed to generate QUIC Connection ID encryption key";
565 conn_handler->set_quic_keying_materials(std::move(qkms));
567 conn_handler->set_cid_prefixes(wpconf->cid_prefixes);
568 conn_handler->set_quic_lingering_worker_processes(
569 wpconf->quic_lingering_worker_processes);
570 #endif // ENABLE_HTTP3
572 if (config->single_thread) {
573 rv = conn_handler->create_single_worker();
581 sigaddset(&set, SIGCHLD);
583 rv = pthread_sigmask(SIG_BLOCK, &set, nullptr);
585 LOG(ERROR) << "Blocking SIGCHLD failed: "
586 << xsi_strerror(rv, errbuf.data(), errbuf.size());
591 rv = conn_handler->create_worker_thread(config->num_worker);
597 rv = pthread_sigmask(SIG_UNBLOCK, &set, nullptr);
599 LOG(ERROR) << "Unblocking SIGCHLD failed: "
600 << xsi_strerror(rv, errbuf.data(), errbuf.size());
606 #if defined(ENABLE_HTTP3) && defined(HAVE_LIBBPF)
607 conn_handler->unload_bpf_objects();
608 #endif // defined(ENABLE_HTTP3) && defined(HAVE_LIBBPF)
611 #ifdef HAVE_NEVERBLEED
613 #endif // HAVE_NEVERBLEED
617 ev_io_init(&ipcev, ipc_readcb, wpconf->ipc_fd, EV_READ);
618 ipcev.data = conn_handler.get();
619 ev_io_start(loop, &ipcev);
623 ev_io_init(&quic_ipcev, quic_ipc_readcb, wpconf->quic_ipc_fd, EV_READ);
624 quic_ipcev.data = conn_handler.get();
625 ev_io_start(loop, &quic_ipcev);
626 #endif // ENABLE_HTTP3
628 if (tls::upstream_tls_enabled(config->conn) && !config->tls.ocsp.disabled) {
629 if (config->tls.ocsp.startup) {
630 conn_handler->set_enable_acceptor_on_ocsp_completion(true);
631 conn_handler->disable_acceptor();
634 conn_handler->proceed_next_cert_ocsp();
637 if (LOG_ENABLED(INFO)) {
638 LOG(INFO) << "Entering event loop";
643 conn_handler->cancel_ocsp_update();
645 // Destroy SSL_CTX held in conn_handler before killing neverbleed
646 // daemon. Otherwise priv_rsa_finish yields "write error" and
647 // worker process aborts.
648 conn_handler.reset();
650 #ifdef HAVE_NEVERBLEED
651 assert(nb->daemon_pid > 0);
653 rv = kill(nb->daemon_pid, SIGTERM);
656 LOG(ERROR) << "Could not send signal to neverbleed daemon: errno=" << error;
659 while ((rv = waitpid(nb->daemon_pid, nullptr, 0)) == -1 && errno == EINTR)
663 LOG(ERROR) << "Error occurred while we were waiting for the completion "
664 "of neverbleed process: errno="
667 #endif // HAVE_NEVERBLEED
669 ares_library_cleanup();