valgrind: stop openssl still reachable complaints
[platform/upstream/libwebsockets.git] / lib / ssl.c
index 54d86c8..46b4f7f 100644 (file)
--- a/lib/ssl.c
+++ b/lib/ssl.c
@@ -1,7 +1,7 @@
 /*
  * libwebsockets - small server side websockets and web server implementation
  *
- * Copyright (C) 2010-2014 Andy Green <andy@warmcat.com>
+ * Copyright (C) 2010-2017 Andy Green <andy@warmcat.com>
  *
  *  This library is free software; you can redistribute it and/or
  *  modify it under the terms of the GNU Lesser General Public
 
 #include "private-libwebsockets.h"
 
-int openssl_websocket_private_data_index;
+/* workaround for mingw */
+#if !defined(ECONNABORTED)
+#define ECONNABORTED 103
+#endif
 
-#ifndef LWS_NO_SERVER
-static int
-OpenSSL_verify_callback(int preverify_ok, X509_STORE_CTX *x509_ctx)
+int lws_alloc_vfs_file(struct lws_context *context, const char *filename, uint8_t **buf,
+               lws_filepos_t *amount)
 {
-       SSL *ssl;
+       lws_filepos_t len;
+       lws_fop_flags_t flags = LWS_O_RDONLY;
+       lws_fop_fd_t fops_fd = lws_vfs_file_open(
+                               lws_get_fops(context), filename, &flags);
+       int ret = 1;
+
+       if (!fops_fd)
+               return 1;
+
+       len = lws_vfs_get_length(fops_fd);
+
+       *buf = malloc((size_t)len);
+       if (!buf)
+               goto bail;
+
+       if (lws_vfs_file_read(fops_fd, amount, *buf, len))
+               goto bail;
+
+       ret = 0;
+bail:
+       lws_vfs_file_close(&fops_fd);
+
+       return ret;
+}
+
+#if defined(LWS_WITH_ESP32)
+int alloc_file(struct lws_context *context, const char *filename, uint8_t **buf,
+              lws_filepos_t *amount)
+{
+       nvs_handle nvh;
+       size_t s;
+       int n = 0;
+
+       ESP_ERROR_CHECK(nvs_open("lws-station", NVS_READWRITE, &nvh));
+       if (nvs_get_blob(nvh, filename, NULL, &s) != ESP_OK) {
+               n = 1;
+               goto bail;
+       }
+       *buf = malloc(s);
+       if (!*buf) {
+               n = 2;
+               goto bail;
+       }
+       if (nvs_get_blob(nvh, filename, (char *)*buf, &s) != ESP_OK) {
+               free(*buf);
+               n = 1;
+               goto bail;
+       }
+
+       *amount = s;
+
+bail:
+       nvs_close(nvh);
+
+       return n;
+}
+int alloc_pem_to_der_file(struct lws_context *context, const char *filename, uint8_t **buf,
+              lws_filepos_t *amount)
+{
+       uint8_t *pem, *p, *q, *end;
+       lws_filepos_t len;
        int n;
-       struct libwebsocket_context *context;
 
-       ssl = X509_STORE_CTX_get_ex_data(x509_ctx,
-               SSL_get_ex_data_X509_STORE_CTX_idx());
+       n = alloc_file(context, filename, &pem, &len);
+       if (n)
+               return n;
+
+       /* trim the first line */
 
-       /*
-        * !!! nasty openssl requires the index to come as a library-scope
-        * static
-        */
-       context = SSL_get_ex_data(ssl, openssl_websocket_private_data_index);
+       p = pem;
+       end = p + len;
+       if (strncmp((char *)p, "-----", 5))
+               goto bail;
+       p += 5;
+       while (p < end && *p != '\n' && *p != '-')
+               p++;
+
+       if (*p != '-')
+               goto bail;
+
+       while (p < end && *p != '\n')
+               p++;
+
+       if (p >= end)
+               goto bail;
+
+       p++;
+
+       /* trim the last line */
+
+       q = end - 2;
+
+       while (q > pem && *q != '\n')
+               q--;
+
+       if (*q != '\n')
+               goto bail;
+
+       *q = '\0';
 
-       n = context->protocols[0].callback(NULL, NULL,
-               LWS_CALLBACK_OPENSSL_PERFORM_CLIENT_CERT_VERIFICATION,
-                                                  x509_ctx, ssl, preverify_ok);
+       *amount = lws_b64_decode_string((char *)p, (char *)pem, len);
+       *buf = pem;
 
-       /* convert return code from 0 = OK to 1 = OK */
-       return !n;
+       return 0;
+
+bail:
+       free(pem);
+
+       return 4;
+}
+#endif
+
+int openssl_websocket_private_data_index,
+    openssl_SSL_CTX_private_data_index;
+
+int lws_ssl_get_error(struct lws *wsi, int n)
+{
+       if (!wsi->ssl)
+               return 99;
+       lwsl_debug("%s: %p %d\n", __func__, wsi->ssl, n);
+       return SSL_get_error(wsi->ssl, n);
 }
 
-static int lws_context_init_ssl_pem_passwd_cb(char * buf, int size, int rwflag, void *userdata)
+/* Copies a string describing the code returned by lws_ssl_get_error(),
+ * which may also contain system error information in the case of SSL_ERROR_SYSCALL,
+ * into buf up to len.
+ * Returns a pointer to buf.
+ *
+ * Note: the lws_ssl_get_error() code is *not* an error code that can be passed
+ * to ERR_error_string(),
+ *
+ * ret is the return value originally passed to lws_ssl_get_error(), needed to disambiguate
+ * SYS_ERROR_SYSCALL.
+ *
+ * See man page for SSL_get_error().
+ *
+ * Not thread safe, uses strerror()
+ */
+char* lws_ssl_get_error_string(int status, int ret, char *buf, size_t len) {
+       switch (status) {
+       case SSL_ERROR_NONE: return strncpy(buf, "SSL_ERROR_NONE", len);
+       case SSL_ERROR_ZERO_RETURN: return strncpy(buf, "SSL_ERROR_ZERO_RETURN", len);
+       case SSL_ERROR_WANT_READ: return strncpy(buf, "SSL_ERROR_WANT_READ", len);
+       case SSL_ERROR_WANT_WRITE: return strncpy(buf, "SSL_ERROR_WANT_WRITE", len);
+       case SSL_ERROR_WANT_CONNECT: return strncpy(buf, "SSL_ERROR_WANT_CONNECT", len);
+       case SSL_ERROR_WANT_ACCEPT: return strncpy(buf, "SSL_ERROR_WANT_ACCEPT", len);
+       case SSL_ERROR_WANT_X509_LOOKUP: return strncpy(buf, "SSL_ERROR_WANT_X509_LOOKUP", len);
+       case SSL_ERROR_SYSCALL:
+               switch (ret) {
+                case 0:
+                        lws_snprintf(buf, len, "SSL_ERROR_SYSCALL: EOF");
+                        return buf;
+                case -1:
+#ifndef LWS_PLAT_OPTEE
+                       lws_snprintf(buf, len, "SSL_ERROR_SYSCALL: %s", strerror(errno));
+#else
+                       lws_snprintf(buf, len, "SSL_ERROR_SYSCALL: %d", errno);
+#endif
+                       return buf;
+                default:
+                        return strncpy(buf, "SSL_ERROR_SYSCALL", len);
+       }
+       case SSL_ERROR_SSL: return "SSL_ERROR_SSL";
+       default: return "SSL_ERROR_UNKNOWN";
+       }
+}
+
+void
+lws_ssl_elaborate_error(void)
+{
+#if defined(LWS_WITH_ESP32)
+#else
+       char buf[256];
+       u_long err;
+
+       while ((err = ERR_get_error()) != 0) {
+               ERR_error_string_n(err, buf, sizeof(buf));
+               lwsl_err("*** %s\n", buf);
+       }
+#endif
+}
+
+#if !defined(LWS_WITH_ESP32)
+
+static int
+lws_context_init_ssl_pem_passwd_cb(char * buf, int size, int rwflag, void *userdata)
 {
-       struct lws_context_creation_info * info = (struct lws_context_creation_info *)userdata;
+       struct lws_context_creation_info * info =
+                       (struct lws_context_creation_info *)userdata;
 
        strncpy(buf, info->ssl_private_key_password, size);
        buf[size - 1] = '\0';
@@ -58,8 +225,8 @@ static int lws_context_init_ssl_pem_passwd_cb(char * buf, int size, int rwflag,
        return strlen(buf);
 }
 
-static void lws_ssl_bind_passphrase(struct libwebsocket_context *context,
-                                   struct lws_context_creation_info *info)
+void
+lws_ssl_bind_passphrase(SSL_CTX *ssl_ctx, struct lws_context_creation_info *info)
 {
        if (!info->ssl_private_key_password)
                return;
@@ -68,435 +235,461 @@ static void lws_ssl_bind_passphrase(struct libwebsocket_context *context,
         * for checking password which will be trigered during
         * SSL_CTX_use_PrivateKey_file function
         */
-       SSL_CTX_set_default_passwd_cb_userdata(context->ssl_client_ctx,
-                                              (void *)info);
-       SSL_CTX_set_default_passwd_cb(context->ssl_client_ctx,
-                                     lws_context_init_ssl_pem_passwd_cb);
+       SSL_CTX_set_default_passwd_cb_userdata(ssl_ctx, (void *)info);
+       SSL_CTX_set_default_passwd_cb(ssl_ctx, lws_context_init_ssl_pem_passwd_cb);
 }
+#endif
 
-LWS_VISIBLE int
-lws_context_init_server_ssl(struct lws_context_creation_info *info,
-                    struct libwebsocket_context *context)
+int
+lws_context_init_ssl_library(struct lws_context_creation_info *info)
 {
-       SSL_METHOD *method;
-       int error;
-       int n;
-
-       if (info->port != CONTEXT_PORT_NO_LISTEN) {
-
-               context->use_ssl = info->ssl_cert_filepath != NULL &&
-                                        info->ssl_private_key_filepath != NULL;
-#ifdef USE_CYASSL
-               lwsl_notice(" Compiled with CYASSL support\n");
+#ifdef USE_WOLFSSL
+#ifdef USE_OLD_CYASSL
+       lwsl_notice(" Compiled with CyaSSL support\n");
 #else
-               lwsl_notice(" Compiled with OpenSSL support\n");
+       lwsl_notice(" Compiled with wolfSSL support\n");
 #endif
-               
-               if (info->ssl_cipher_list)
-                       lwsl_notice(" SSL ciphers: '%s'\n", info->ssl_cipher_list);
-
-               if (context->use_ssl)
-                       lwsl_notice(" Using SSL mode\n");
-               else
-                       lwsl_notice(" Using non-SSL mode\n");
+#else
+#if defined(LWS_USE_BORINGSSL)
+       lwsl_notice(" Compiled with BoringSSL support\n");
+#else
+       lwsl_notice(" Compiled with OpenSSL support\n");
+#endif
+#endif
+       if (!lws_check_opt(info->options, LWS_SERVER_OPTION_DO_SSL_GLOBAL_INIT)) {
+               lwsl_notice(" SSL disabled: no LWS_SERVER_OPTION_DO_SSL_GLOBAL_INIT\n");
+               return 0;
        }
 
        /* basic openssl init */
 
-       SSL_library_init();
+       lwsl_notice("Doing SSL library init\n");
 
+#if !defined(LWS_WITH_ESP32)
+       SSL_library_init();
        OpenSSL_add_all_algorithms();
        SSL_load_error_strings();
 
        openssl_websocket_private_data_index =
-               SSL_get_ex_new_index(0, "libwebsockets", NULL, NULL, NULL);
-
-       /*
-        * Firefox insists on SSLv23 not SSLv3
-        * Konq disables SSLv2 by default now, SSLv23 works
-        */
+               SSL_get_ex_new_index(0, "lws", NULL, NULL, NULL);
 
-       method = (SSL_METHOD *)SSLv23_server_method();
-       if (!method) {
-               error = ERR_get_error();
-               lwsl_err("problem creating ssl method %lu: %s\n", 
-                       error, ERR_error_string(error,
-                                             (char *)context->service_buffer));
-               return 1;
-       }
-       context->ssl_ctx = SSL_CTX_new(method); /* create context */
-       if (!context->ssl_ctx) {
-               error = ERR_get_error();
-               lwsl_err("problem creating ssl context %lu: %s\n",
-                       error, ERR_error_string(error,
-                                             (char *)context->service_buffer));
-               return 1;
-       }
-
-#ifdef SSL_OP_NO_COMPRESSION
-       SSL_CTX_set_options(context->ssl_ctx, SSL_OP_NO_COMPRESSION);
+       openssl_SSL_CTX_private_data_index = SSL_CTX_get_ex_new_index(0,
+                       NULL, NULL, NULL, NULL);
 #endif
-       SSL_CTX_set_options(context->ssl_ctx, SSL_OP_CIPHER_SERVER_PREFERENCE);
-       if (info->ssl_cipher_list)
-               SSL_CTX_set_cipher_list(context->ssl_ctx,
-                                               info->ssl_cipher_list);
-
-       /* as a server, are we requiring clients to identify themselves? */
-
-       if (info->options &
-                       LWS_SERVER_OPTION_REQUIRE_VALID_OPENSSL_CLIENT_CERT) {
-
-               /* absolutely require the client cert */
-
-               SSL_CTX_set_verify(context->ssl_ctx,
-                      SSL_VERIFY_PEER | SSL_VERIFY_FAIL_IF_NO_PEER_CERT,
-                                                      OpenSSL_verify_callback);
-
-               /*
-                * give user code a chance to load certs into the server
-                * allowing it to verify incoming client certs
-                */
-
-               context->protocols[0].callback(context, NULL,
-                       LWS_CALLBACK_OPENSSL_LOAD_EXTRA_SERVER_VERIFY_CERTS,
-                                                    context->ssl_ctx, NULL, 0);
-       }
-
-       if (info->options & LWS_SERVER_OPTION_ALLOW_NON_SSL_ON_SSL_PORT) {
-               /* Normally SSL listener rejects non-ssl, optionally allow */
-               context->allow_non_ssl_on_ssl_port = 1;
-       }
 
-       if (context->use_ssl) {
-
-               /* openssl init for server sockets */
-
-               /* set the local certificate from CertFile */
-               n = SSL_CTX_use_certificate_chain_file(context->ssl_ctx,
-                                       info->ssl_cert_filepath);
-               if (n != 1) {
-                       error = ERR_get_error();
-                       lwsl_err("problem getting cert '%s' %lu: %s\n",
-                               info->ssl_cert_filepath,
-                               error,
-                               ERR_error_string(error,
-                                             (char *)context->service_buffer));
-                       return 1;
-               }
-               lws_ssl_bind_passphrase(context, info);
-               /* set the private key from KeyFile */
-               if (SSL_CTX_use_PrivateKey_file(context->ssl_ctx,
-                            info->ssl_private_key_filepath,
-                                                      SSL_FILETYPE_PEM) != 1) {
-                       error = ERR_get_error();
-                       lwsl_err("ssl problem getting key '%s' %lu: %s\n",
-                               info->ssl_private_key_filepath,
-                                       error,
-                                       ERR_error_string(error,
-                                             (char *)context->service_buffer));
-                       return 1;
-               }
-               /* verify private key */
-               if (!SSL_CTX_check_private_key(context->ssl_ctx)) {
-                       lwsl_err("Private SSL key doesn't match cert\n");
-                       return 1;
-               }
-
-               /*
-                * SSL is happy and has a cert it's content with
-                * If we're supporting HTTP2, initialize that
-                */
-               
-               lws_context_init_http2_ssl(context);
-       }
-       
        return 0;
 }
-#endif
 
 LWS_VISIBLE void
-lws_ssl_destroy(struct libwebsocket_context *context)
+lws_ssl_destroy(struct lws_vhost *vhost)
 {
-       if (context->ssl_ctx)
-               SSL_CTX_free(context->ssl_ctx);
-       if (!context->user_supplied_ssl_ctx && context->ssl_client_ctx)
-               SSL_CTX_free(context->ssl_client_ctx);
+       if (!lws_check_opt(vhost->context->options,
+                          LWS_SERVER_OPTION_DO_SSL_GLOBAL_INIT))
+               return;
+
+       if (vhost->ssl_ctx)
+               SSL_CTX_free(vhost->ssl_ctx);
+       if (!vhost->user_supplied_ssl_ctx && vhost->ssl_client_ctx)
+               SSL_CTX_free(vhost->ssl_client_ctx);
+#if !defined(LWS_WITH_ESP32)
 
-#ifdef USE_CYASSL
+// after 1.1.0 no need
+#if (OPENSSL_VERSION_NUMBER <  0x10100000)
+// <= 1.0.1f = old api, 1.0.1g+ = new api
+#if (OPENSSL_VERSION_NUMBER <= 0x1000106f) || defined(USE_WOLFSSL)
        ERR_remove_state(0);
 #else
+#if OPENSSL_VERSION_NUMBER >= 0x1010005f && \
+    !defined(LIBRESSL_VERSION_NUMBER) && \
+    !defined(OPENSSL_IS_BORINGSSL)
+       ERR_remove_thread_state();
+#else
        ERR_remove_thread_state(NULL);
 #endif
+#endif
+       // after 1.1.0 no need
+#if  (OPENSSL_VERSION_NUMBER >= 0x10002000) && (OPENSSL_VERSION_NUMBER <= 0x10100000)
+       SSL_COMP_free_compression_methods();
+#endif
        ERR_free_strings();
        EVP_cleanup();
        CRYPTO_cleanup_all_ex_data();
+#endif
+#endif
 }
 
 LWS_VISIBLE void
-libwebsockets_decode_ssl_error(void)
+lws_ssl_remove_wsi_from_buffered_list(struct lws *wsi)
 {
-       char buf[256];
-       u_long err;
+       struct lws_context *context = wsi->context;
+       struct lws_context_per_thread *pt = &context->pt[(int)wsi->tsi];
 
-       while ((err = ERR_get_error()) != 0) {
-               ERR_error_string_n(err, buf, sizeof(buf));
-               lwsl_err("*** %lu %s\n", err, buf);
-       }
-}
+       if (!wsi->pending_read_list_prev &&
+           !wsi->pending_read_list_next &&
+           pt->pending_read_list != wsi)
+               /* we are not on the list */
+               return;
+
+       /* point previous guy's next to our next */
+       if (!wsi->pending_read_list_prev)
+               pt->pending_read_list = wsi->pending_read_list_next;
+       else
+               wsi->pending_read_list_prev->pending_read_list_next =
+                       wsi->pending_read_list_next;
+
+       /* point next guy's previous to our previous */
+       if (wsi->pending_read_list_next)
+               wsi->pending_read_list_next->pending_read_list_prev =
+                       wsi->pending_read_list_prev;
 
-#ifndef LWS_NO_CLIENT
+       wsi->pending_read_list_prev = NULL;
+       wsi->pending_read_list_next = NULL;
+}
 
-int lws_context_init_client_ssl(struct lws_context_creation_info *info,
-                           struct libwebsocket_context *context)
+LWS_VISIBLE int
+lws_ssl_capable_read(struct lws *wsi, unsigned char *buf, int len)
 {
-       int error;
-       int n;
-       SSL_METHOD *method;
+       struct lws_context *context = wsi->context;
+       struct lws_context_per_thread *pt = &context->pt[(int)wsi->tsi];
+       int n = 0;
+#if !defined(LWS_WITH_ESP32)
+       int ssl_read_errno = 0;
+#endif
 
-       if (info->provided_client_ssl_ctx) {
-               /* use the provided OpenSSL context if given one */
-               context->ssl_client_ctx = info->provided_client_ssl_ctx;
-               /* nothing for lib to delete */
-               context->user_supplied_ssl_ctx = 1;
-               return 0;
-       }
+       if (!wsi->ssl)
+               return lws_ssl_capable_read_no_ssl(wsi, buf, len);
 
-       if (info->port != CONTEXT_PORT_NO_LISTEN)
-               return 0;
+       lws_stats_atomic_bump(context, pt, LWSSTATS_C_API_READ, 1);
 
-       /* basic openssl init */
+       errno = 0;
+       n = SSL_read(wsi->ssl, buf, len);
+#if defined(LWS_WITH_ESP32)
+       if (!n && errno == ENOTCONN) {
+               lwsl_debug("%p: SSL_read ENOTCONN\n", wsi);
+               return LWS_SSL_CAPABLE_ERROR;
+       }
+#endif
+#if defined(LWS_WITH_STATS)
+       if (!wsi->seen_rx) {
+                lws_stats_atomic_bump(wsi->context, pt, LWSSTATS_MS_SSL_RX_DELAY,
+                               time_in_microseconds() - wsi->accept_start_us);
+                lws_stats_atomic_bump(wsi->context, pt, LWSSTATS_C_SSL_CONNS_HAD_RX, 1);
+               wsi->seen_rx = 1;
+       }
+#endif
 
-       SSL_library_init();
 
-       OpenSSL_add_all_algorithms();
-       SSL_load_error_strings();
+       lwsl_debug("%p: SSL_read says %d\n", wsi, n);
+       /* manpage: returning 0 means connection shut down */
+       if (!n) {
+               n = lws_ssl_get_error(wsi, n);
+               lwsl_debug("%p: ssl err %d errno %d\n", wsi, n, errno);
+               if (n == SSL_ERROR_ZERO_RETURN)
+                       return LWS_SSL_CAPABLE_ERROR;
+
+               if (n == SSL_ERROR_SYSCALL) {
+#if !defined(LWS_WITH_ESP32)
+                       int err = ERR_get_error();
+                       if (err == 0 && (ssl_read_errno == EPIPE ||
+                                        ssl_read_errno == ECONNABORTED ||
+                                        ssl_read_errno == 0))
+                               return LWS_SSL_CAPABLE_ERROR;
+#endif
+               }
 
-       method = (SSL_METHOD *)SSLv23_client_method();
-       if (!method) {
-               error = ERR_get_error();
-               lwsl_err("problem creating ssl method %lu: %s\n",
-                       error, ERR_error_string(error,
-                                     (char *)context->service_buffer));
-               return 1;
+               lwsl_err("%s failed: %s\n",__func__,
+                        ERR_error_string(lws_ssl_get_error(wsi, 0), NULL));
+               lws_ssl_elaborate_error();
+
+               return LWS_SSL_CAPABLE_ERROR;
        }
-       /* create context */
-       context->ssl_client_ctx = SSL_CTX_new(method);
-       if (!context->ssl_client_ctx) {
-               error = ERR_get_error();
-               lwsl_err("problem creating ssl context %lu: %s\n",
-                       error, ERR_error_string(error,
-                                     (char *)context->service_buffer));
-               return 1;
+
+       if (n < 0) {
+               n = lws_ssl_get_error(wsi, n);
+               // lwsl_notice("get_ssl_err result %d\n", n);
+               if (n ==  SSL_ERROR_WANT_READ || SSL_want_read(wsi->ssl)) {
+                       lwsl_debug("%s: WANT_READ\n", __func__);
+                       lwsl_debug("%p: LWS_SSL_CAPABLE_MORE_SERVICE\n", wsi);
+                       return LWS_SSL_CAPABLE_MORE_SERVICE;
+               }
+               if (n ==  SSL_ERROR_WANT_WRITE || SSL_want_write(wsi->ssl)) {
+                       lwsl_debug("%s: WANT_WRITE\n", __func__);
+                       lwsl_debug("%p: LWS_SSL_CAPABLE_MORE_SERVICE\n", wsi);
+                       return LWS_SSL_CAPABLE_MORE_SERVICE;
+               }
+
+
+               lwsl_err("%s failed2: %s\n",__func__,
+                                ERR_error_string(lws_ssl_get_error(wsi, 0), NULL));
+                       lws_ssl_elaborate_error();
+
+               return LWS_SSL_CAPABLE_ERROR;
        }
 
-#ifdef SSL_OP_NO_COMPRESSION
-       SSL_CTX_set_options(context->ssl_client_ctx,
-                                                SSL_OP_NO_COMPRESSION);
-#endif
-       SSL_CTX_set_options(context->ssl_client_ctx,
-                                      SSL_OP_CIPHER_SERVER_PREFERENCE);
-       if (info->ssl_cipher_list)
-               SSL_CTX_set_cipher_list(context->ssl_client_ctx,
-                                               info->ssl_cipher_list);
-
-#ifdef LWS_SSL_CLIENT_USE_OS_CA_CERTS
-       if (!(info->options & LWS_SERVER_OPTION_DISABLE_OS_CA_CERTS))
-               /* loads OS default CA certs */
-               SSL_CTX_set_default_verify_paths(context->ssl_client_ctx);
-#endif
-
-       /* openssl init for cert verification (for client sockets) */
-       if (!info->ssl_ca_filepath) {
-               if (!SSL_CTX_load_verify_locations(
-                       context->ssl_client_ctx, NULL,
-                                            LWS_OPENSSL_CLIENT_CERTS))
-                       lwsl_err(
-                           "Unable to load SSL Client certs from %s "
-                           "(set by --with-client-cert-dir= "
-                           "in configure) --  client ssl isn't "
-                           "going to work", LWS_OPENSSL_CLIENT_CERTS);
-       } else
-               if (!SSL_CTX_load_verify_locations(
-                       context->ssl_client_ctx, info->ssl_ca_filepath,
-                                                         NULL))
-                       lwsl_err(
-                               "Unable to load SSL Client certs "
-                               "file from %s -- client ssl isn't "
-                               "going to work", info->ssl_ca_filepath);
+       lws_stats_atomic_bump(context, pt, LWSSTATS_B_READ, n);
+
+       if (wsi->vhost)
+               wsi->vhost->conn_stats.rx += n;
+
+       lws_restart_ws_ping_pong_timer(wsi);
 
        /*
-        * callback allowing user code to load extra verification certs
-        * helping the client to verify server identity
+        * if it was our buffer that limited what we read,
+        * check if SSL has additional data pending inside SSL buffers.
+        *
+        * Because these won't signal at the network layer with POLLIN
+        * and if we don't realize, this data will sit there forever
         */
+       if (n != len)
+               goto bail;
+       if (!wsi->ssl)
+               goto bail;
 
-       /* support for client-side certificate authentication */
-       if (info->ssl_cert_filepath) {
-               n = SSL_CTX_use_certificate_chain_file(
-                       context->ssl_client_ctx,
-                                       info->ssl_cert_filepath);
-               if (n != 1) {
-                       lwsl_err("problem getting cert '%s' %lu: %s\n",
-                               info->ssl_cert_filepath,
-                               ERR_get_error(),
-                               ERR_error_string(ERR_get_error(),
-                               (char *)context->service_buffer));
-                       return 1;
-               }
-       } 
-       if (info->ssl_private_key_filepath) {
-               lws_ssl_bind_passphrase(context, info);
-               /* set the private key from KeyFile */
-               if (SSL_CTX_use_PrivateKey_file(context->ssl_client_ctx,
-                   info->ssl_private_key_filepath, SSL_FILETYPE_PEM) != 1) {
-                       lwsl_err("use_PrivateKey_file '%s' %lu: %s\n",
-                               info->ssl_private_key_filepath,
-                               ERR_get_error(),
-                               ERR_error_string(ERR_get_error(),
-                                     (char *)context->service_buffer));
-                       return 1;
-               }
+       if (!SSL_pending(wsi->ssl))
+               goto bail;
 
-               /* verify private key */
-               if (!SSL_CTX_check_private_key(
-                                       context->ssl_client_ctx)) {
-                       lwsl_err("Private SSL key doesn't match cert\n");
-                       return 1;
-               }
-       } 
+       if (wsi->pending_read_list_next)
+               return n;
+       if (wsi->pending_read_list_prev)
+               return n;
+       if (pt->pending_read_list == wsi)
+               return n;
 
-       context->protocols[0].callback(context, NULL,
-               LWS_CALLBACK_OPENSSL_LOAD_EXTRA_CLIENT_VERIFY_CERTS,
-               context->ssl_client_ctx, NULL, 0);
-       
-       return 0;
+       /* add us to the linked list of guys with pending ssl */
+       if (pt->pending_read_list)
+               pt->pending_read_list->pending_read_list_prev = wsi;
+
+       wsi->pending_read_list_next = pt->pending_read_list;
+       wsi->pending_read_list_prev = NULL;
+       pt->pending_read_list = wsi;
+
+       return n;
+bail:
+       lws_ssl_remove_wsi_from_buffered_list(wsi);
+
+       return n;
 }
-#endif
 
 LWS_VISIBLE int
-lws_ssl_capable_read(struct libwebsocket_context *context,
-                    struct libwebsocket *wsi, unsigned char *buf, int len)
+lws_ssl_pending(struct lws *wsi)
 {
-       int n;
-
        if (!wsi->ssl)
-               return lws_ssl_capable_read_no_ssl(context, wsi, buf, len);
-       
-       wsi->buffered_reads_pending = 0;
-
-       n = SSL_read(wsi->ssl, buf, len);
-       if (n >= 0) {
-               /* 
-                * if it was our buffer that limited what we read,
-                * check if SSL has additional data pending inside SSL buffers.
-                * 
-                * Because these won't signal at the network layer with POLLIN
-                * and if we don't realize, this data will sit there forever
-                */
-               if (n == len && wsi->ssl && SSL_pending(wsi->ssl)) {
-                       context->ssl_flag_buffered_reads = 1;
-                       wsi->buffered_reads_pending = 1;
-               }
-               
-               return n;
-       }
-       n = SSL_get_error(wsi->ssl, n);
-       if (n ==  SSL_ERROR_WANT_READ || n ==  SSL_ERROR_WANT_WRITE)
-               return LWS_SSL_CAPABLE_MORE_SERVICE;
+               return 0;
 
-       return LWS_SSL_CAPABLE_ERROR; 
+       return SSL_pending(wsi->ssl);
 }
 
 LWS_VISIBLE int
-lws_ssl_capable_write(struct libwebsocket *wsi, unsigned char *buf, int len)
+lws_ssl_capable_write(struct lws *wsi, unsigned char *buf, int len)
 {
        int n;
+#if !defined(LWS_WITH_ESP32)
+               int ssl_read_errno = 0;
+#endif
 
        if (!wsi->ssl)
                return lws_ssl_capable_write_no_ssl(wsi, buf, len);
-       
+
        n = SSL_write(wsi->ssl, buf, len);
-       if (n >= 0)
+       if (n > 0)
                return n;
 
-       n = SSL_get_error(wsi->ssl, n);
+       n = lws_ssl_get_error(wsi, n);
        if (n == SSL_ERROR_WANT_READ || n == SSL_ERROR_WANT_WRITE) {
-               if (n == SSL_ERROR_WANT_WRITE)
+               if (n == SSL_ERROR_WANT_WRITE) {
+                       lwsl_debug("%s: WANT_WRITE\n", __func__);
                        lws_set_blocking_send(wsi);
+               }
                return LWS_SSL_CAPABLE_MORE_SERVICE;
        }
 
+ if (n == SSL_ERROR_ZERO_RETURN)
+  return LWS_SSL_CAPABLE_ERROR;
+
+#if !defined(LWS_WITH_ESP32)
+ if (n == SSL_ERROR_SYSCALL) {
+
+  int err = ERR_get_error();
+  if (err == 0
+    && (ssl_read_errno == EPIPE
+     || ssl_read_errno == ECONNABORTED
+     || ssl_read_errno == 0))
+    return LWS_SSL_CAPABLE_ERROR;
+ }
+#endif
+
+ lwsl_err("%s failed: %s\n",__func__,
+   ERR_error_string(lws_ssl_get_error(wsi, 0), NULL));
+       lws_ssl_elaborate_error();
+
        return LWS_SSL_CAPABLE_ERROR;
 }
 
+static int
+lws_gate_accepts(struct lws_context *context, int on)
+{
+       struct lws_vhost *v = context->vhost_list;
+
+       lwsl_info("gating accepts %d\n", on);
+       context->ssl_gate_accepts = !on;
+#if defined(LWS_WITH_STATS)
+       context->updated = 1;
+#endif
+
+       while (v) {
+               if (v->use_ssl &&  v->lserv_wsi) /* gate ability to accept incoming connections */
+                       if (lws_change_pollfd(v->lserv_wsi, (LWS_POLLIN) * !on, (LWS_POLLIN) * on))
+                               lwsl_err("Unable to set accept POLLIN %d\n", on);
+
+               v = v->vhost_next;
+       }
+
+       return 0;
+}
+
+void
+lws_ssl_info_callback(const SSL *ssl, int where, int ret)
+{
+       struct lws *wsi;
+       struct lws_context *context;
+       struct lws_ssl_info si;
+
+       context = (struct lws_context *)SSL_CTX_get_ex_data(
+                                       SSL_get_SSL_CTX(ssl),
+                                       openssl_SSL_CTX_private_data_index);
+       if (!context)
+               return;
+       wsi = wsi_from_fd(context, SSL_get_fd(ssl));
+       if (!wsi)
+               return;
+
+       if (!(where & wsi->vhost->ssl_info_event_mask))
+               return;
+
+       si.where = where;
+       si.ret = ret;
+
+       if (user_callback_handle_rxflow(wsi->protocol->callback,
+                                                  wsi, LWS_CALLBACK_SSL_INFO,
+                                                  wsi->user_space, &si, 0))
+               lws_set_timeout(wsi, PENDING_TIMEOUT_KILLED_BY_SSL_INFO, -1);
+}
+
+
 LWS_VISIBLE int
-lws_ssl_close(struct libwebsocket *wsi)
+lws_ssl_close(struct lws *wsi)
 {
-       int n;
+       lws_sockfd_type n;
 
        if (!wsi->ssl)
                return 0; /* not handled */
 
+#if defined (LWS_HAVE_SSL_SET_INFO_CALLBACK)
+       /* kill ssl callbacks, becausse we will remove the fd from the
+        * table linking it to the wsi
+        */
+       if (wsi->vhost->ssl_info_event_mask)
+               SSL_set_info_callback(wsi->ssl, NULL);
+#endif
+
        n = SSL_get_fd(wsi->ssl);
        SSL_shutdown(wsi->ssl);
        compatible_close(n);
        SSL_free(wsi->ssl);
+       wsi->ssl = NULL;
+
+       if (wsi->context->simultaneous_ssl_restriction &&
+           wsi->context->simultaneous_ssl-- ==
+                           wsi->context->simultaneous_ssl_restriction)
+               /* we made space and can do an accept */
+               lws_gate_accepts(wsi->context, 1);
+#if defined(LWS_WITH_STATS)
+       wsi->context->updated = 1;
+#endif
 
        return 1; /* handled */
 }
 
+/* leave all wsi close processing to the caller */
+
 LWS_VISIBLE int
-lws_server_socket_service_ssl(struct libwebsocket_context *context,
-               struct libwebsocket **pwsi, struct libwebsocket *new_wsi,
-                       int accept_fd, struct libwebsocket_pollfd *pollfd)
+lws_server_socket_service_ssl(struct lws *wsi, lws_sockfd_type accept_fd)
 {
+       struct lws_context *context = wsi->context;
+       struct lws_context_per_thread *pt = &context->pt[(int)wsi->tsi];
        int n, m;
-       struct libwebsocket *wsi = *pwsi;
-#ifndef USE_CYASSL
+#if !defined(USE_WOLFSSL) && !defined(LWS_WITH_ESP32)
        BIO *bio;
 #endif
+        char buf[256];
 
-       if (!LWS_SSL_ENABLED(context))
+       if (!LWS_SSL_ENABLED(wsi->vhost))
                return 0;
 
        switch (wsi->mode) {
-       case LWS_CONNMODE_SERVER_LISTENER:
-
-               if (!new_wsi) {
-                       lwsl_err("no new_wsi\n");
-                       return 0;
+       case LWSCM_SSL_INIT:
+       case LWSCM_SSL_INIT_RAW:
+               if (wsi->ssl)
+                       lwsl_err("%s: leaking ssl\n", __func__);
+               if (accept_fd == LWS_SOCK_INVALID)
+                       assert(0);
+               if (context->simultaneous_ssl_restriction &&
+                   context->simultaneous_ssl >= context->simultaneous_ssl_restriction) {
+                       lwsl_notice("unable to deal with SSL connection\n");
+                       return 1;
                }
-
-               new_wsi->ssl = SSL_new(context->ssl_ctx);
-               if (new_wsi->ssl == NULL) {
-                       lwsl_err("SSL_new failed: %s\n",
-                           ERR_error_string(SSL_get_error(
-                           new_wsi->ssl, 0), NULL));
-                           libwebsockets_decode_ssl_error();
-                       free(new_wsi);
-                       compatible_close(accept_fd);
-                       break;
+               errno = 0;
+               wsi->ssl = SSL_new(wsi->vhost->ssl_ctx);
+               if (wsi->ssl == NULL) {
+                       lwsl_err("SSL_new failed: %d (errno %d)\n",
+                                lws_ssl_get_error(wsi, 0), errno);
+
+                       lws_ssl_elaborate_error();
+                       if (accept_fd != LWS_SOCK_INVALID)
+                               compatible_close(accept_fd);
+                       goto fail;
                }
+#if defined (LWS_HAVE_SSL_SET_INFO_CALLBACK)
+               if (wsi->vhost->ssl_info_event_mask)
+                       SSL_set_info_callback(wsi->ssl, lws_ssl_info_callback);
+#endif
+               if (context->simultaneous_ssl_restriction &&
+                   ++context->simultaneous_ssl == context->simultaneous_ssl_restriction)
+                       /* that was the last allowed SSL connection */
+                       lws_gate_accepts(context, 0);
+#if defined(LWS_WITH_STATS)
+       context->updated = 1;
+#endif
 
-               SSL_set_ex_data(new_wsi->ssl,
-                       openssl_websocket_private_data_index, context);
-
-               SSL_set_fd(new_wsi->ssl, accept_fd);
+#if !defined(LWS_WITH_ESP32)
+               SSL_set_ex_data(wsi->ssl,
+                       openssl_websocket_private_data_index, wsi);
+#endif
+               SSL_set_fd(wsi->ssl, accept_fd);
 
-#ifdef USE_CYASSL
-               CyaSSL_set_using_nonblock(new_wsi->ssl, 1);
+#ifdef USE_WOLFSSL
+#ifdef USE_OLD_CYASSL
+               CyaSSL_set_using_nonblock(wsi->ssl, 1);
 #else
-               SSL_set_mode(new_wsi->ssl, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER);
-               bio = SSL_get_rbio(new_wsi->ssl);
+               wolfSSL_set_using_nonblock(wsi->ssl, 1);
+#endif
+#else
+#if defined(LWS_WITH_ESP32)
+               lws_plat_set_socket_options(wsi->vhost, accept_fd);
+#else
+               SSL_set_mode(wsi->ssl, SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER);
+               bio = SSL_get_rbio(wsi->ssl);
                if (bio)
                        BIO_set_nbio(bio, 1); /* nonblocking */
                else
                        lwsl_notice("NULL rbio\n");
-               bio = SSL_get_wbio(new_wsi->ssl);
+               bio = SSL_get_wbio(wsi->ssl);
                if (bio)
                        BIO_set_nbio(bio, 1); /* nonblocking */
                else
                        lwsl_notice("NULL rbio\n");
 #endif
+#endif
 
                /*
                 * we are not accepted yet, but we need to enter ourselves
@@ -504,29 +697,36 @@ lws_server_socket_service_ssl(struct libwebsocket_context *context,
                 * pieces come if we're not sorted yet
                 */
 
-               *pwsi = new_wsi;
-               wsi = *pwsi;
-               wsi->mode = LWS_CONNMODE_SSL_ACK_PENDING;
-               insert_wsi_socket_into_fds(context, wsi);
+               if (wsi->mode == LWSCM_SSL_INIT)
+                       wsi->mode = LWSCM_SSL_ACK_PENDING;
+               else
+                       wsi->mode = LWSCM_SSL_ACK_PENDING_RAW;
 
-               libwebsocket_set_timeout(wsi, PENDING_TIMEOUT_SSL_ACCEPT,
-                                                       AWAITING_TIMEOUT);
+               if (insert_wsi_socket_into_fds(context, wsi)) {
+                       lwsl_err("%s: failed to insert into fds\n", __func__);
+                       goto fail;
+               }
 
-               lwsl_info("inserted SSL accept into fds, trying SSL_accept\n");
+               lws_set_timeout(wsi, PENDING_TIMEOUT_SSL_ACCEPT,
+                               context->timeout_secs);
 
-               /* fallthru */
+               lwsl_debug("inserted SSL accept into fds, trying SSL_accept\n");
 
-       case LWS_CONNMODE_SSL_ACK_PENDING:
+               /* fallthru */
 
-               if (lws_change_pollfd(wsi, LWS_POLLOUT, 0))
+       case LWSCM_SSL_ACK_PENDING:
+       case LWSCM_SSL_ACK_PENDING_RAW:
+               if (lws_change_pollfd(wsi, LWS_POLLOUT, 0)) {
+                       lwsl_err("%s: lws_change_pollfd failed\n", __func__);
                        goto fail;
-
-               lws_libev_io(context, wsi, LWS_EV_STOP | LWS_EV_WRITE);
+               }
 
                lws_latency_pre(context, wsi);
 
-               n = recv(wsi->sock, context->service_buffer,
-                       sizeof(context->service_buffer), MSG_PEEK);
+               if (wsi->vhost->allow_non_ssl_on_ssl_port) {
+
+                       n = recv(wsi->desc.sockfd, (char *)pt->serv_buf, context->pt_serv_buf_size,
+                                MSG_PEEK);
 
                /*
                 * optionally allow non-SSL connect on SSL listening socket
@@ -535,66 +735,108 @@ lws_server_socket_service_ssl(struct libwebsocket_context *context,
                 * it disabled unless you know it's not a problem for you
                 */
 
-               if (context->allow_non_ssl_on_ssl_port && n >= 1 &&
-                                       context->service_buffer[0] >= ' ') {
-                       /*
-                        * TLS content-type for Handshake is 0x16
-                        * TLS content-type for ChangeCipherSpec Record is 0x14
-                        *
-                        * A non-ssl session will start with the HTTP method in
-                        * ASCII.  If we see it's not a legit SSL handshake
-                        * kill the SSL for this connection and try to handle
-                        * as a HTTP connection upgrade directly.
-                        */
-                       wsi->use_ssl = 0;
-                       SSL_shutdown(wsi->ssl);
-                       SSL_free(wsi->ssl);
-                       wsi->ssl = NULL;
-                       goto accepted;
+                       if (n >= 1 && pt->serv_buf[0] >= ' ') {
+                               /*
+                               * TLS content-type for Handshake is 0x16, and
+                               * for ChangeCipherSpec Record, it's 0x14
+                               *
+                               * A non-ssl session will start with the HTTP
+                               * method in ASCII.  If we see it's not a legit
+                               * SSL handshake kill the SSL for this
+                               * connection and try to handle as a HTTP
+                               * connection upgrade directly.
+                               */
+                               wsi->use_ssl = 0;
+
+                               SSL_shutdown(wsi->ssl);
+                               SSL_free(wsi->ssl);
+                               wsi->ssl = NULL;
+                               if (lws_check_opt(context->options,
+                                   LWS_SERVER_OPTION_REDIRECT_HTTP_TO_HTTPS))
+                                       wsi->redirect_to_https = 1;
+                               goto accepted;
+                       }
+                       if (!n) /*
+                                * connection is gone, or nothing to read
+                                * if it's gone, we will timeout on
+                                * PENDING_TIMEOUT_SSL_ACCEPT
+                                */
+                               break;
+                       if (n < 0 && (LWS_ERRNO == LWS_EAGAIN ||
+                                     LWS_ERRNO == LWS_EWOULDBLOCK)) {
+                               /*
+                                * well, we get no way to know ssl or not
+                                * so go around again waiting for something
+                                * to come and give us a hint, or timeout the
+                                * connection.
+                                */
+                               m = SSL_ERROR_WANT_READ;
+                               goto go_again;
+                       }
                }
 
                /* normal SSL connection processing path */
 
+#if defined(LWS_WITH_STATS)
+               if (!wsi->accept_start_us)
+                       wsi->accept_start_us = time_in_microseconds();
+#endif
+
                n = SSL_accept(wsi->ssl);
                lws_latency(context, wsi,
-                       "SSL_accept LWS_CONNMODE_SSL_ACK_PENDING\n", n, n == 1);
-
+                       "SSL_accept LWSCM_SSL_ACK_PENDING\n", n, n == 1);
+               lwsl_info("SSL_accept says %d\n", n);
                if (n == 1)
                        goto accepted;
 
-               m = SSL_get_error(wsi->ssl, n);
-               lwsl_debug("SSL_accept failed %d / %s\n",
-                                                 m, ERR_error_string(m, NULL));
+               m = lws_ssl_get_error(wsi, n);
 
-               if (m == SSL_ERROR_WANT_READ) {
-                       if (lws_change_pollfd(wsi, 0, LWS_POLLIN))
-                               goto fail;
+#if defined(LWS_WITH_ESP32)
+               if (m == 5 && errno == 11)
+                       m = SSL_ERROR_WANT_READ;
+#endif
 
-                       lws_libev_io(context, wsi, LWS_EV_START | LWS_EV_READ);
+go_again:
+               if (m == SSL_ERROR_WANT_READ || SSL_want_read(wsi->ssl)) {
+                       if (lws_change_pollfd(wsi, 0, LWS_POLLIN)) {
+                               lwsl_err("%s: WANT_READ change_pollfd failed\n", __func__);
+                               goto fail;
+                       }
 
                        lwsl_info("SSL_ERROR_WANT_READ\n");
                        break;
                }
-               if (m == SSL_ERROR_WANT_WRITE) {
-                       if (lws_change_pollfd(wsi, 0, LWS_POLLOUT))
+               if (m == SSL_ERROR_WANT_WRITE || SSL_want_write(wsi->ssl)) {
+                       lwsl_debug("%s: WANT_WRITE\n", __func__);
+
+                       if (lws_change_pollfd(wsi, 0, LWS_POLLOUT)) {
+                               lwsl_err("%s: WANT_WRITE change_pollfd failed\n", __func__);
                                goto fail;
+                       }
 
-                       lws_libev_io(context, wsi, LWS_EV_START | LWS_EV_WRITE);
                        break;
                }
-               lwsl_debug("SSL_accept failed skt %u: %s\n",
-                                        pollfd->fd, ERR_error_string(m, NULL));
-               libwebsocket_close_and_free_session(context, wsi,
-                                                    LWS_CLOSE_STATUS_NOSTATUS);
-               break;
+               lws_stats_atomic_bump(wsi->context, pt, LWSSTATS_C_SSL_CONNECTIONS_FAILED, 1);
+                lwsl_err("SSL_accept failed socket %u: %s\n", wsi->desc.sockfd,
+                         lws_ssl_get_error_string(m, n, buf, sizeof(buf)));
+               lws_ssl_elaborate_error();
+               goto fail;
 
 accepted:
+               lws_stats_atomic_bump(wsi->context, pt, LWSSTATS_C_SSL_CONNECTIONS_ACCEPTED, 1);
+#if defined(LWS_WITH_STATS)
+               lws_stats_atomic_bump(wsi->context, pt, LWSSTATS_MS_SSL_CONNECTIONS_ACCEPTED_DELAY, time_in_microseconds() - wsi->accept_start_us);
+               wsi->accept_start_us = time_in_microseconds();
+#endif
+
                /* OK, we are accepted... give him some time to negotiate */
-               libwebsocket_set_timeout(wsi,
-                       PENDING_TIMEOUT_ESTABLISH_WITH_SERVER,
-                                                       AWAITING_TIMEOUT);
+               lws_set_timeout(wsi, PENDING_TIMEOUT_ESTABLISH_WITH_SERVER,
+                               context->timeout_secs);
 
-               wsi->mode = LWS_CONNMODE_HTTP_SERVING;
+               if (wsi->mode == LWSCM_SSL_ACK_PENDING_RAW)
+                       wsi->mode = LWSCM_RAW;
+               else
+                       wsi->mode = LWSCM_HTTP_SERVING;
 
                lws_http2_configure_if_upgraded(wsi);
 
@@ -603,25 +845,48 @@ accepted:
        }
 
        return 0;
-       
+
 fail:
        return 1;
 }
 
-LWS_VISIBLE void
-lws_ssl_context_destroy(struct libwebsocket_context *context)
+void
+lws_ssl_SSL_CTX_destroy(struct lws_vhost *vhost)
+{
+       if (vhost->ssl_ctx)
+               SSL_CTX_free(vhost->ssl_ctx);
+
+       if (!vhost->user_supplied_ssl_ctx && vhost->ssl_client_ctx)
+               SSL_CTX_free(vhost->ssl_client_ctx);
+}
+
+void
+lws_ssl_context_destroy(struct lws_context *context)
 {
-       if (context->ssl_ctx)
-               SSL_CTX_free(context->ssl_ctx);
-       if (!context->user_supplied_ssl_ctx && context->ssl_client_ctx)
-               SSL_CTX_free(context->ssl_client_ctx);
 
-#ifdef USE_CYASSL
+#if !defined(LWS_WITH_ESP32)
+
+// after 1.1.0 no need
+#if (OPENSSL_VERSION_NUMBER <  0x10100000)
+// <= 1.0.1f = old api, 1.0.1g+ = new api
+#if (OPENSSL_VERSION_NUMBER <= 0x1000106f) || defined(USE_WOLFSSL)
        ERR_remove_state(0);
 #else
+#if OPENSSL_VERSION_NUMBER >= 0x1010005f && \
+    !defined(LIBRESSL_VERSION_NUMBER) && \
+    !defined(OPENSSL_IS_BORINGSSL)
+       ERR_remove_thread_state();
+#else
        ERR_remove_thread_state(NULL);
 #endif
+#endif
+       // after 1.1.0 no need
+#if  (OPENSSL_VERSION_NUMBER >= 0x10002000) && (OPENSSL_VERSION_NUMBER <= 0x10100000)
+       SSL_COMP_free_compression_methods();
+#endif
        ERR_free_strings();
        EVP_cleanup();
        CRYPTO_cleanup_all_ex_data();
+#endif
+#endif
 }