2 * nghttp2 - HTTP/2 C Library
4 * Copyright (c) 2015 Tatsuhiro Tsujikawa
6 * Permission is hereby granted, free of charge, to any person obtaining
7 * a copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sublicense, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice shall be
15 * included in all copies or substantial portions of the Software.
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
20 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25 #include "shrpx_worker_process.h"
27 #include <sys/types.h>
30 #endif // HAVE_UNISTD_H
31 #include <sys/resource.h>
38 #include <openssl/rand.h>
44 #include "shrpx_config.h"
45 #include "shrpx_connection_handler.h"
46 #include "shrpx_log_config.h"
47 #include "shrpx_worker.h"
48 #include "shrpx_accept_handler.h"
49 #include "shrpx_http2_upstream.h"
50 #include "shrpx_http2_session.h"
51 #include "shrpx_memcached_dispatcher.h"
52 #include "shrpx_memcached_request.h"
53 #include "shrpx_process.h"
54 #include "shrpx_tls.h"
55 #include "shrpx_log.h"
57 #include "app_helper.h"
59 #include "xsi_strerror.h"
61 using namespace nghttp2;
67 #ifdef HAVE_NEVERBLEED
69 #endif // HAVE_NEVERBLEED
71 std::array<char, STRERROR_BUFSIZE> errbuf;
72 auto config = get_config();
74 if (getuid() == 0 && config->uid != 0) {
75 #ifdef HAVE_NEVERBLEED
77 neverbleed_setuidgid(nb, config->user.c_str(), 1);
79 #endif // HAVE_NEVERBLEED
81 if (initgroups(config->user.c_str(), config->gid) != 0) {
83 LOG(FATAL) << "Could not change supplementary groups: "
84 << xsi_strerror(error, errbuf.data(), errbuf.size());
87 if (setgid(config->gid) != 0) {
89 LOG(FATAL) << "Could not change gid: "
90 << xsi_strerror(error, errbuf.data(), errbuf.size());
93 if (setuid(config->uid) != 0) {
95 LOG(FATAL) << "Could not change uid: "
96 << xsi_strerror(error, errbuf.data(), errbuf.size());
99 if (setuid(0) != -1) {
100 LOG(FATAL) << "Still have root privileges?";
108 void graceful_shutdown(ConnectionHandler *conn_handler) {
109 if (conn_handler->get_graceful_shutdown()) {
113 LOG(NOTICE) << "Graceful shutdown signal received";
115 conn_handler->set_graceful_shutdown(true);
117 // TODO What happens for the connections not established in the
119 conn_handler->accept_pending_connection();
120 conn_handler->delete_acceptor();
122 conn_handler->graceful_shutdown_worker();
124 auto single_worker = conn_handler->get_single_worker();
126 if (single_worker->get_worker_stat()->num_connections == 0) {
127 ev_break(conn_handler->get_loop());
136 void reopen_log(ConnectionHandler *conn_handler) {
137 LOG(NOTICE) << "Reopening log files: worker process (thread main)";
139 auto config = get_config();
140 auto &loggingconf = config->logging;
142 (void)reopen_log_files(loggingconf);
143 redirect_stderr_to_errorlog(loggingconf);
145 conn_handler->worker_reopen_log_files();
150 void ipc_readcb(struct ev_loop *loop, ev_io *w, int revents) {
151 auto conn_handler = static_cast<ConnectionHandler *>(w->data);
152 std::array<uint8_t, 1024> buf;
154 while ((nread = read(w->fd, buf.data(), buf.size())) == -1 && errno == EINTR)
158 LOG(ERROR) << "Failed to read data from ipc channel: errno=" << error;
163 // IPC socket closed. Perform immediate shutdown.
164 LOG(FATAL) << "IPC socket is closed. Perform immediate shutdown.";
165 nghttp2_Exit(EXIT_FAILURE);
168 for (ssize_t i = 0; i < nread; ++i) {
170 case SHRPX_IPC_GRACEFUL_SHUTDOWN:
171 graceful_shutdown(conn_handler);
173 case SHRPX_IPC_REOPEN_LOG:
174 reopen_log(conn_handler);
182 int generate_ticket_key(TicketKey &ticket_key) {
183 ticket_key.cipher = get_config()->tls.ticket.cipher;
184 ticket_key.hmac = EVP_sha256();
185 ticket_key.hmac_keylen = EVP_MD_size(ticket_key.hmac);
187 assert(static_cast<size_t>(EVP_CIPHER_key_length(ticket_key.cipher)) <=
188 ticket_key.data.enc_key.size());
189 assert(ticket_key.hmac_keylen <= ticket_key.data.hmac_key.size());
191 if (LOG_ENABLED(INFO)) {
192 LOG(INFO) << "enc_keylen=" << EVP_CIPHER_key_length(ticket_key.cipher)
193 << ", hmac_keylen=" << ticket_key.hmac_keylen;
196 if (RAND_bytes(reinterpret_cast<unsigned char *>(&ticket_key.data),
197 sizeof(ticket_key.data)) == 0) {
206 void renew_ticket_key_cb(struct ev_loop *loop, ev_timer *w, int revents) {
207 auto conn_handler = static_cast<ConnectionHandler *>(w->data);
208 const auto &old_ticket_keys = conn_handler->get_ticket_keys();
210 auto ticket_keys = std::make_shared<TicketKeys>();
211 LOG(NOTICE) << "Renew new ticket keys";
213 // If old_ticket_keys is not empty, it should contain at least 2
214 // keys: one for encryption, and last one for the next encryption
215 // key but decryption only. The keys in between are old keys and
216 // decryption only. The next key is provided to ensure to mitigate
217 // possible problem when one worker encrypt new key, but one worker,
218 // which did not take the that key yet, and cannot decrypt it.
220 // We keep keys for get_config()->tls_session_timeout seconds. The
221 // default is 12 hours. Thus the maximum ticket vector size is 12.
222 if (old_ticket_keys) {
223 auto &old_keys = old_ticket_keys->keys;
224 auto &new_keys = ticket_keys->keys;
226 assert(!old_keys.empty());
229 static_cast<size_t>(std::chrono::duration_cast<std::chrono::hours>(
230 get_config()->tls.session_timeout)
233 new_keys.resize(std::min(max_tickets, old_keys.size() + 1));
234 std::copy_n(std::begin(old_keys), new_keys.size() - 1,
235 std::begin(new_keys) + 1);
237 ticket_keys->keys.resize(1);
240 auto &new_key = ticket_keys->keys[0];
242 if (generate_ticket_key(new_key) != 0) {
243 if (LOG_ENABLED(INFO)) {
244 LOG(INFO) << "failed to generate ticket key";
246 conn_handler->set_ticket_keys(nullptr);
247 conn_handler->set_ticket_keys_to_worker(nullptr);
251 if (LOG_ENABLED(INFO)) {
252 LOG(INFO) << "ticket keys generation done";
253 assert(ticket_keys->keys.size() >= 1);
254 LOG(INFO) << 0 << " enc+dec: "
255 << util::format_hex(ticket_keys->keys[0].data.name);
256 for (size_t i = 1; i < ticket_keys->keys.size(); ++i) {
257 auto &key = ticket_keys->keys[i];
258 LOG(INFO) << i << " dec: " << util::format_hex(key.data.name);
262 conn_handler->set_ticket_keys(ticket_keys);
263 conn_handler->set_ticket_keys_to_worker(ticket_keys);
268 void memcached_get_ticket_key_cb(struct ev_loop *loop, ev_timer *w,
270 auto conn_handler = static_cast<ConnectionHandler *>(w->data);
271 auto dispatcher = conn_handler->get_tls_ticket_key_memcached_dispatcher();
273 auto req = std::make_unique<MemcachedRequest>();
274 req->key = "nghttpx:tls-ticket-key";
275 req->op = MemcachedOp::GET;
276 req->cb = [conn_handler, w](MemcachedRequest *req, MemcachedResult res) {
277 switch (res.status_code) {
278 case MemcachedStatusCode::NO_ERROR:
280 case MemcachedStatusCode::EXT_NETWORK_ERROR:
281 conn_handler->on_tls_ticket_key_network_error(w);
284 conn_handler->on_tls_ticket_key_not_found(w);
288 // |version (4bytes)|len (2bytes)|key (variable length)|...
289 // (len, key) pairs are repeated as necessary.
291 auto &value = res.value;
292 if (value.size() < 4) {
293 LOG(WARN) << "Memcached: tls ticket key value is too small: got "
295 conn_handler->on_tls_ticket_key_not_found(w);
298 auto p = value.data();
299 auto version = util::get_uint32(p);
300 // Currently supported version is 1.
302 LOG(WARN) << "Memcached: tls ticket key version: want 1, got " << version;
303 conn_handler->on_tls_ticket_key_not_found(w);
307 auto end = p + value.size();
310 auto &ticketconf = get_config()->tls.ticket;
315 if (ticketconf.cipher == EVP_aes_128_cbc()) {
319 } else if (ticketconf.cipher == EVP_aes_256_cbc()) {
327 auto ticket_keys = std::make_shared<TicketKeys>();
331 LOG(WARN) << "Memcached: tls ticket key data is too small";
332 conn_handler->on_tls_ticket_key_not_found(w);
335 auto len = util::get_uint16(p);
337 if (len != expectedlen) {
338 LOG(WARN) << "Memcached: wrong tls ticket key size: want "
339 << expectedlen << ", got " << len;
340 conn_handler->on_tls_ticket_key_not_found(w);
344 LOG(WARN) << "Memcached: too short tls ticket key payload: want " << len
345 << ", got " << (end - p);
346 conn_handler->on_tls_ticket_key_not_found(w);
349 auto key = TicketKey();
350 key.cipher = ticketconf.cipher;
351 key.hmac = EVP_sha256();
352 key.hmac_keylen = hmac_keylen;
354 std::copy_n(p, key.data.name.size(), std::begin(key.data.name));
355 p += key.data.name.size();
357 std::copy_n(p, enc_keylen, std::begin(key.data.enc_key));
360 std::copy_n(p, hmac_keylen, std::begin(key.data.hmac_key));
363 ticket_keys->keys.push_back(std::move(key));
366 conn_handler->on_tls_ticket_key_get_success(ticket_keys, w);
369 if (LOG_ENABLED(INFO)) {
370 LOG(INFO) << "Memcached: tls ticket key get request sent";
373 dispatcher->add_request(std::move(req));
378 #ifdef HAVE_NEVERBLEED
380 void nb_child_cb(struct ev_loop *loop, ev_child *w, int revents) {
381 log_chld(w->rpid, w->rstatus, "neverbleed process");
383 ev_child_stop(loop, w);
385 LOG(FATAL) << "neverbleed process exitted; aborting now";
387 nghttp2_Exit(EXIT_FAILURE);
390 #endif // HAVE_NEVERBLEED
392 int worker_process_event_loop(WorkerProcessConfig *wpconf) {
394 std::array<char, STRERROR_BUFSIZE> errbuf;
397 auto config = get_config();
399 if (reopen_log_files(config->logging) != 0) {
400 LOG(FATAL) << "Failed to open log file";
404 rv = ares_library_init(ARES_LIB_INIT_ALL);
406 LOG(FATAL) << "ares_library_init failed: " << ares_strerror(rv);
410 auto loop = EV_DEFAULT;
412 auto gen = util::make_mt19937();
414 auto conn_handler = std::make_unique<ConnectionHandler>(loop, gen);
416 for (auto &addr : config->conn.listener.addrs) {
417 conn_handler->add_acceptor(
418 std::make_unique<AcceptHandler>(&addr, conn_handler.get()));
421 #ifdef HAVE_NEVERBLEED
422 std::array<char, NEVERBLEED_ERRBUF_SIZE> nb_errbuf;
423 auto nb = std::make_unique<neverbleed_t>();
424 if (neverbleed_init(nb.get(), nb_errbuf.data()) != 0) {
425 LOG(FATAL) << "neverbleed_init failed: " << nb_errbuf.data();
429 LOG(NOTICE) << "neverbleed process [" << nb->daemon_pid << "] spawned";
431 conn_handler->set_neverbleed(nb.get());
435 ev_child_init(&nb_childev, nb_child_cb, nb->daemon_pid, 0);
436 nb_childev.data = nullptr;
437 ev_child_start(loop, &nb_childev);
438 #endif // HAVE_NEVERBLEED
442 ev_timer renew_ticket_key_timer;
443 if (tls::upstream_tls_enabled(config->conn)) {
444 auto &ticketconf = config->tls.ticket;
445 auto &memcachedconf = ticketconf.memcached;
447 if (!memcachedconf.host.empty()) {
448 SSL_CTX *ssl_ctx = nullptr;
450 if (memcachedconf.tls) {
451 ssl_ctx = conn_handler->create_tls_ticket_key_memcached_ssl_ctx();
454 conn_handler->set_tls_ticket_key_memcached_dispatcher(
455 std::make_unique<MemcachedDispatcher>(
456 &ticketconf.memcached.addr, loop, ssl_ctx,
457 StringRef{memcachedconf.host}, &mcpool, gen));
459 ev_timer_init(&renew_ticket_key_timer, memcached_get_ticket_key_cb, 0.,
461 renew_ticket_key_timer.data = conn_handler.get();
462 // Get first ticket keys.
463 memcached_get_ticket_key_cb(loop, &renew_ticket_key_timer, 0);
465 bool auto_tls_ticket_key = true;
466 if (!ticketconf.files.empty()) {
467 if (!ticketconf.cipher_given) {
469 << "It is strongly recommended to specify "
470 "--tls-ticket-key-cipher=aes-128-cbc (or "
471 "tls-ticket-key-cipher=aes-128-cbc in configuration file) "
472 "when --tls-ticket-key-file is used for the smooth "
473 "transition when the default value of --tls-ticket-key-cipher "
474 "becomes aes-256-cbc";
476 auto ticket_keys = read_tls_ticket_key_file(
477 ticketconf.files, ticketconf.cipher, EVP_sha256());
479 LOG(WARN) << "Use internal session ticket key generator";
481 conn_handler->set_ticket_keys(std::move(ticket_keys));
482 auto_tls_ticket_key = false;
485 if (auto_tls_ticket_key) {
486 // Generate new ticket key every 1hr.
487 ev_timer_init(&renew_ticket_key_timer, renew_ticket_key_cb, 0., 1_h);
488 renew_ticket_key_timer.data = conn_handler.get();
489 ev_timer_again(loop, &renew_ticket_key_timer);
491 // Generate first session ticket key before running workers.
492 renew_ticket_key_cb(loop, &renew_ticket_key_timer, 0);
497 if (config->single_thread) {
498 rv = conn_handler->create_single_worker();
506 sigaddset(&set, SIGCHLD);
508 rv = pthread_sigmask(SIG_BLOCK, &set, nullptr);
510 LOG(ERROR) << "Blocking SIGCHLD failed: "
511 << xsi_strerror(rv, errbuf.data(), errbuf.size());
516 rv = conn_handler->create_worker_thread(config->num_worker);
522 rv = pthread_sigmask(SIG_UNBLOCK, &set, nullptr);
524 LOG(ERROR) << "Unblocking SIGCHLD failed: "
525 << xsi_strerror(rv, errbuf.data(), errbuf.size());
532 #ifdef HAVE_NEVERBLEED
534 #endif // HAVE_NEVERBLEED
538 ev_io_init(&ipcev, ipc_readcb, wpconf->ipc_fd, EV_READ);
539 ipcev.data = conn_handler.get();
540 ev_io_start(loop, &ipcev);
542 if (tls::upstream_tls_enabled(config->conn) && !config->tls.ocsp.disabled) {
543 if (config->tls.ocsp.startup) {
544 conn_handler->set_enable_acceptor_on_ocsp_completion(true);
545 conn_handler->disable_acceptor();
548 conn_handler->proceed_next_cert_ocsp();
551 if (LOG_ENABLED(INFO)) {
552 LOG(INFO) << "Entering event loop";
557 conn_handler->cancel_ocsp_update();
559 // Destroy SSL_CTX held in conn_handler before killing neverbleed
560 // daemon. Otherwise priv_rsa_finish yields "write error" and
561 // worker process aborts.
562 conn_handler.reset();
564 #ifdef HAVE_NEVERBLEED
565 assert(nb->daemon_pid > 0);
567 rv = kill(nb->daemon_pid, SIGTERM);
570 LOG(ERROR) << "Could not send signal to neverbleed daemon: errno=" << error;
573 while ((rv = waitpid(nb->daemon_pid, nullptr, 0)) == -1 && errno == EINTR)
577 LOG(ERROR) << "Error occurred while we were waiting for the completion "
578 "of neverbleed process: errno="
581 #endif // HAVE_NEVERBLEED
583 ares_library_cleanup();