HTTP_PROXY: make usable
[platform/upstream/libwebsockets.git] / lib / parsers.c
index 17059af..9cd70e0 100644 (file)
 
 #include "private-libwebsockets.h"
 
-unsigned char lextable[] = {
+const unsigned char lextable[] = {
        #include "lextable.h"
 };
 
 #define FAIL_CHAR 0x08
 
-int lextable_decode(int pos, char c)
+int LWS_WARN_UNUSED_RESULT
+lextable_decode(int pos, char c)
 {
-       c = tolower(c);
+       if (c >= 'A' && c <= 'Z')
+               c += 'a' - 'A';
 
        while (1) {
                if (lextable[pos] & (1 << 7)) { /* 1-byte, fail on mismatch */
@@ -58,94 +60,291 @@ int lextable_decode(int pos, char c)
        }
 }
 
-int lws_allocate_header_table(struct lws *wsi)
+void
+_lws_header_table_reset(struct allocated_headers *ah)
+{
+       /* init the ah to reflect no headers or data have appeared yet */
+       memset(ah->frag_index, 0, sizeof(ah->frag_index));
+       ah->nfrag = 0;
+       ah->pos = 0;
+       ah->http_response = 0;
+}
+
+// doesn't scrub the ah rxbuffer by default, parent must do if needed
+
+void
+lws_header_table_reset(struct lws *wsi, int autoservice)
+{
+       struct allocated_headers *ah = wsi->u.hdr.ah;
+       struct lws_context_per_thread *pt;
+       struct lws_pollfd *pfd;
+
+       /* if we have the idea we're resetting 'our' ah, must be bound to one */
+       assert(ah);
+       /* ah also concurs with ownership */
+       assert(ah->wsi == wsi);
+
+       _lws_header_table_reset(ah);
+
+       /* since we will restart the ah, our new headers are not completed */
+       // wsi->hdr_parsing_completed = 0;
+
+       /*
+        * if we inherited pending rx (from socket adoption deferred
+        * processing), apply and free it.
+        */
+       if (wsi->u.hdr.preamble_rx) {
+               memcpy(ah->rx, wsi->u.hdr.preamble_rx,
+                      wsi->u.hdr.preamble_rx_len);
+               ah->rxlen = wsi->u.hdr.preamble_rx_len;
+               lws_free_set_NULL(wsi->u.hdr.preamble_rx);
+
+               if (autoservice) {
+                       lwsl_notice("%s: calling service on readbuf ah\n", __func__);
+
+                       pt = &wsi->context->pt[(int)wsi->tsi];
+
+                       /* unlike a normal connect, we have the headers already
+                        * (or the first part of them anyway)
+                        */
+                       pfd = &pt->fds[wsi->position_in_fds_table];
+                       pfd->revents |= LWS_POLLIN;
+                       lwsl_err("%s: calling service\n", __func__);
+                       lws_service_fd_tsi(wsi->context, pfd, wsi->tsi);
+               }
+       }
+}
+
+int LWS_WARN_UNUSED_RESULT
+lws_header_table_attach(struct lws *wsi, int autoservice)
 {
        struct lws_context *context = wsi->context;
+       struct lws_context_per_thread *pt = &context->pt[(int)wsi->tsi];
+       struct lws_pollargs pa;
+       struct lws **pwsi;
        int n;
 
-       lwsl_debug("%s: wsi %p: ah %p\n", __func__, (void *)wsi,
-                (void *)wsi->u.hdr.ah);
+       lwsl_info("%s: wsi %p: ah %p (tsi %d, count = %d) in\n", __func__, (void *)wsi,
+                (void *)wsi->u.hdr.ah, wsi->tsi, pt->ah_count_in_use);
 
        /* if we are already bound to one, just clear it down */
-       if (wsi->u.hdr.ah)
+       if (wsi->u.hdr.ah) {
+               lwsl_info("cleardown\n");
                goto reset;
+       }
+
+       lws_pt_lock(pt);
+       pwsi = &pt->ah_wait_list;
+       while (*pwsi) {
+               if (*pwsi == wsi) {
+                       /* if already waiting on list, if no new ah just ret */
+                       if (pt->ah_count_in_use ==
+                           context->max_http_header_pool) {
+                               lwsl_notice("%s: no free ah to attach\n", __func__);
+                               goto bail;
+                       }
+                       /* new ah.... remove ourselves from waiting list */
+                       *pwsi = wsi->u.hdr.ah_wait_list; /* set our prev to our next */
+                       wsi->u.hdr.ah_wait_list = NULL; /* no next any more */
+                       pt->ah_wait_list_length--;
+                       break;
+               }
+               pwsi = &(*pwsi)->u.hdr.ah_wait_list;
+       }
        /*
-        * server should have suppressed the accept of a new wsi before this
-        * became the case.  If initiating multiple client connects, make sure
-        * the ah pool is big enough to cope, or be prepared to retry
+        * pool is all busy... add us to waiting list and return that we
+        * weren't able to deliver it right now
         */
-       if (context->ah_count_in_use == context->max_http_header_pool) {
-               lwsl_err("No free ah\n");
-               return -1;
+       if (pt->ah_count_in_use == context->max_http_header_pool) {
+               lwsl_info("%s: adding %p to ah waiting list\n", __func__, wsi);
+               wsi->u.hdr.ah_wait_list = pt->ah_wait_list;
+               pt->ah_wait_list = wsi;
+               pt->ah_wait_list_length++;
+
+               /* we cannot accept input then */
+
+               _lws_change_pollfd(wsi, LWS_POLLIN, 0, &pa);
+               goto bail;
        }
 
        for (n = 0; n < context->max_http_header_pool; n++)
-               if (!context->ah_pool[n].in_use)
+               if (!pt->ah_pool[n].in_use)
                        break;
 
        /* if the count of in use said something free... */
        assert(n != context->max_http_header_pool);
 
-       wsi->u.hdr.ah = &context->ah_pool[n];
+       wsi->u.hdr.ah = &pt->ah_pool[n];
        wsi->u.hdr.ah->in_use = 1;
+       pt->ah_pool[n].wsi = wsi; /* mark our owner */
+       pt->ah_count_in_use++;
 
-       context->ah_count_in_use++;
-       /* if we used up all the ah, defeat accepting new server connections */
-       if (context->ah_count_in_use == context->max_http_header_pool)
-               if (_lws_server_listen_accept_flow_control(context, 0))
-                       return 1;
+       _lws_change_pollfd(wsi, 0, LWS_POLLIN, &pa);
 
-       lwsl_debug("%s: wsi %p: ah %p: count %d (on exit)\n",
-                __func__, (void *)wsi, (void *)wsi->u.hdr.ah,
-                context->ah_count_in_use);
+       lwsl_info("%s: did attach wsi %p: ah %p: count %d (on exit)\n", __func__,
+                 (void *)wsi, (void *)wsi->u.hdr.ah, pt->ah_count_in_use);
+
+       lws_pt_unlock(pt);
 
 reset:
-       /* init the ah to reflect no headers or data have appeared yet */
-       memset(wsi->u.hdr.ah->frag_index, 0, sizeof(wsi->u.hdr.ah->frag_index));
-       wsi->u.hdr.ah->nfrag = 0;
-       wsi->u.hdr.ah->pos = 0;
+
+       /* and reset the rx state */
+       wsi->u.hdr.ah->rxpos = 0;
+       wsi->u.hdr.ah->rxlen = 0;
+
+       lws_header_table_reset(wsi, autoservice);
+       time(&wsi->u.hdr.ah->assigned);
+
+#ifndef LWS_NO_CLIENT
+       if (wsi->state == LWSS_CLIENT_UNCONNECTED)
+               if (!lws_client_connect_via_info2(wsi))
+                       /* our client connect has failed, the wsi
+                        * has been closed
+                        */
+                       return -1;
+#endif
 
        return 0;
+
+bail:
+       lws_pt_unlock(pt);
+
+       return 1;
 }
 
-int lws_free_header_table(struct lws *wsi)
+int lws_header_table_detach(struct lws *wsi, int autoservice)
 {
        struct lws_context *context = wsi->context;
+       struct allocated_headers *ah = wsi->u.hdr.ah;
+       struct lws_context_per_thread *pt = &context->pt[(int)wsi->tsi];
+       struct lws_pollargs pa;
+       struct lws **pwsi;
+       time_t now;
 
-       lwsl_debug("%s: wsi %p: ah %p (count = %d)\n", __func__, (void *)wsi,
-                (void *)wsi->u.hdr.ah, context->ah_count_in_use);
+       lwsl_info("%s: wsi %p: ah %p (tsi=%d, count = %d)\n", __func__,
+                 (void *)wsi, (void *)ah, wsi->tsi,
+                 pt->ah_count_in_use);
 
-       assert(wsi->u.hdr.ah);
-       if (!wsi->u.hdr.ah)
-               return 0;
+       if (wsi->u.hdr.preamble_rx)
+               lws_free_set_NULL(wsi->u.hdr.preamble_rx);
 
-       /* if we think we're freeing one, there should be one to free */
-       assert(context->ah_count_in_use > 0);
+       /* may not be detached while he still has unprocessed rx */
+       if (ah && ah->rxpos != ah->rxlen) {
+               lwsl_err("%s: %p: CANNOT DETACH rxpos:%d, rxlen:%d\n", __func__, wsi,
+                               ah->rxpos, ah->rxlen);
+               assert(ah->rxpos == ah->rxlen);
 
-       assert(wsi->u.hdr.ah->in_use);
-       wsi->u.hdr.ah->in_use = 0;
+               return 0;
+       }
+
+       lws_pt_lock(pt);
+
+       pwsi = &pt->ah_wait_list;
+       if (!ah) { /* remove from wait list if none attached */
+               while (*pwsi) {
+                       if (*pwsi == wsi) {
+                               lwsl_info("%s: wsi %p, remv wait\n",
+                                         __func__, wsi);
+                               *pwsi = wsi->u.hdr.ah_wait_list;
+                               wsi->u.hdr.ah_wait_list = NULL;
+                               pt->ah_wait_list_length--;
+                               goto bail;
+                       }
+                       pwsi = &(*pwsi)->u.hdr.ah_wait_list;
+               }
+               /* no ah, not on list... no more business here */
+               goto bail;
+       }
+       /* we did have an ah attached */
+       time(&now);
+       if (ah->assigned && now - ah->assigned > 3) {
+               /*
+                * we're detaching the ah, but it was held an
+                * unreasonably long time
+                */
+               lwsl_notice("%s: wsi %p: ah held %ds, "
+                           "ah.rxpos %d, ah.rxlen %d, mode/state %d %d,"
+                           "wsi->more_rx_waiting %d\n", __func__, wsi,
+                           (int)(now - ah->assigned),
+                           ah->rxpos, ah->rxlen, wsi->mode, wsi->state,
+                           wsi->more_rx_waiting);
+       }
 
-       /* if we just freed up one ah, allow new server connection */
-       if (context->ah_count_in_use == context->max_http_header_pool)
-               if (_lws_server_listen_accept_flow_control(context, 1))
-                       return 1;
+       ah->assigned = 0;
 
-       context->ah_count_in_use--;
+       /* if we think we're detaching one, there should be one in use */
+       assert(pt->ah_count_in_use > 0);
+       /* and this specific one should have been in use */
+       assert(ah->in_use);
        wsi->u.hdr.ah = NULL;
+       ah->wsi = NULL; /* no owner */
+
+       /* oh there is nobody on the waiting list... leave it at that then */
+       if (!*pwsi) {
+               ah->in_use = 0;
+               pt->ah_count_in_use--;
+
+               goto bail;
+       }
+
+       /* somebody else on same tsi is waiting, give it to oldest guy */
+
+       lwsl_info("pt wait list %p\n", *pwsi);
+       while ((*pwsi)->u.hdr.ah_wait_list)
+               pwsi = &(*pwsi)->u.hdr.ah_wait_list;
+
+       wsi = *pwsi;
+       lwsl_info("last wsi in wait list %p\n", wsi);
+
+       wsi->u.hdr.ah = ah;
+       ah->wsi = wsi; /* new owner */
+       /* and reset the rx state */
+       ah->rxpos = 0;
+       ah->rxlen = 0;
+       lws_header_table_reset(wsi, autoservice);
+       time(&wsi->u.hdr.ah->assigned);
+
+       /* clients acquire the ah and then insert themselves in fds table... */
+       if (wsi->position_in_fds_table != -1) {
+               lwsl_info("%s: Enabling %p POLLIN\n", __func__, wsi);
+
+               /* he has been stuck waiting for an ah, but now his wait is over,
+                * let him progress
+                */
+               _lws_change_pollfd(wsi, 0, LWS_POLLIN, &pa);
+       }
+
+       /* point prev guy to next guy in list instead */
+       *pwsi = wsi->u.hdr.ah_wait_list;
+       /* the guy who got one is out of the list */
+       wsi->u.hdr.ah_wait_list = NULL;
+       pt->ah_wait_list_length--;
+
+#ifndef LWS_NO_CLIENT
+       if (wsi->state == LWSS_CLIENT_UNCONNECTED) {
+               lws_pt_unlock(pt);
+
+               if (!lws_client_connect_via_info2(wsi)) {
+                       /* our client connect has failed, the wsi
+                        * has been closed
+                        */
+
+                       return -1;
+               }
+               return 0;
+       }
+#endif
+
+       assert(!!pt->ah_wait_list_length == !!(int)(long)pt->ah_wait_list);
+bail:
+       lwsl_info("%s: wsi %p: ah %p (tsi=%d, count = %d)\n", __func__,
+         (void *)wsi, (void *)ah, wsi->tsi,
+         pt->ah_count_in_use);
+       lws_pt_unlock(pt);
 
        return 0;
 }
 
-/**
- * lws_hdr_fragment_length: report length of a single fragment of a header
- *             The returned length does not include the space for a
- *             terminating '\0'
- *
- * @wsi: websocket connection
- * @h: which header index we are interested in
- * @frag_idx: which fragment of @h we want to get the length of
- */
-
 LWS_VISIBLE int
 lws_hdr_fragment_length(struct lws *wsi, enum lws_token_indexes h, int frag_idx)
 {
@@ -163,15 +362,6 @@ lws_hdr_fragment_length(struct lws *wsi, enum lws_token_indexes h, int frag_idx)
        return 0;
 }
 
-/**
- * lws_hdr_total_length: report length of all fragments of a header totalled up
- *             The returned length does not include the space for a
- *             terminating '\0'
- *
- * @wsi: websocket connection
- * @h: which header index we are interested in
- */
-
 LWS_VISIBLE int lws_hdr_total_length(struct lws *wsi, enum lws_token_indexes h)
 {
        int n;
@@ -188,26 +378,15 @@ LWS_VISIBLE int lws_hdr_total_length(struct lws *wsi, enum lws_token_indexes h)
        return len;
 }
 
-/**
- * lws_hdr_copy_fragment: copy a single fragment of the given header to a buffer
- *             The buffer length @len must include space for an additional
- *             terminating '\0', or it will fail returning -1.
- *             If the requested fragment index is not present, it fails
- *             returning -1.
- *
- * @wsi: websocket connection
- * @dst: destination buffer
- * @len: length of destination buffer
- * @h: which header index we are interested in
- * @frag_index: which fragment of @h we want to copy
- */
-
 LWS_VISIBLE int lws_hdr_copy_fragment(struct lws *wsi, char *dst, int len,
                                      enum lws_token_indexes h, int frag_idx)
 {
        int n = 0;
        int f = wsi->u.hdr.ah->frag_index[h];
 
+       if (!f)
+               return -1;
+
        while (n < frag_idx) {
                f = wsi->u.hdr.ah->frags[f].nfrag;
                if (!f)
@@ -225,17 +404,6 @@ LWS_VISIBLE int lws_hdr_copy_fragment(struct lws *wsi, char *dst, int len,
        return wsi->u.hdr.ah->frags[f].len;
 }
 
-/**
- * lws_hdr_copy: copy a single fragment of the given header to a buffer
- *             The buffer length @len must include space for an additional
- *             terminating '\0', or it will fail returning -1.
- *
- * @wsi: websocket connection
- * @dst: destination buffer
- * @len: length of destination buffer
- * @h: which header index we are interested in
- */
-
 LWS_VISIBLE int lws_hdr_copy(struct lws *wsi, char *dst, int len,
                             enum lws_token_indexes h)
 {
@@ -269,8 +437,31 @@ char *lws_hdr_simple_ptr(struct lws *wsi, enum lws_token_indexes h)
        return wsi->u.hdr.ah->data + wsi->u.hdr.ah->frags[n].offset;
 }
 
-int lws_hdr_simple_create(struct lws *wsi, enum lws_token_indexes h,
-                         const char *s)
+int LWS_WARN_UNUSED_RESULT
+lws_pos_in_bounds(struct lws *wsi)
+{
+       if (wsi->u.hdr.ah->pos < (unsigned int)wsi->context->max_http_header_data)
+               return 0;
+
+       if (wsi->u.hdr.ah->pos == wsi->context->max_http_header_data) {
+               lwsl_err("Ran out of header data space\n");
+               return 1;
+       }
+
+       /*
+        * with these tests everywhere, it should never be able to exceed
+        * the limit, only meet the limit
+        */
+
+       lwsl_err("%s: pos %d, limit %d\n", __func__, wsi->u.hdr.ah->pos,
+                wsi->context->max_http_header_data);
+       assert(0);
+
+       return 1;
+}
+
+int LWS_WARN_UNUSED_RESULT
+lws_hdr_simple_create(struct lws *wsi, enum lws_token_indexes h, const char *s)
 {
        wsi->u.hdr.ah->nfrag++;
        if (wsi->u.hdr.ah->nfrag == ARRAY_SIZE(wsi->u.hdr.ah->frags)) {
@@ -285,10 +476,9 @@ int lws_hdr_simple_create(struct lws *wsi, enum lws_token_indexes h,
        wsi->u.hdr.ah->frags[wsi->u.hdr.ah->nfrag].nfrag = 0;
 
        do {
-               if (wsi->u.hdr.ah->pos == wsi->context->max_http_header_data) {
-                       lwsl_err("Ran out of header data space\n");
+               if (lws_pos_in_bounds(wsi))
                        return -1;
-               }
+
                wsi->u.hdr.ah->data[wsi->u.hdr.ah->pos++] = *s;
                if (*s)
                        wsi->u.hdr.ah->frags[wsi->u.hdr.ah->nfrag].len++;
@@ -297,7 +487,7 @@ int lws_hdr_simple_create(struct lws *wsi, enum lws_token_indexes h,
        return 0;
 }
 
-static signed char char_to_hex(const char c)
+signed char char_to_hex(const char c)
 {
        if (c >= '0' && c <= '9')
                return c - '0';
@@ -311,14 +501,13 @@ static signed char char_to_hex(const char c)
        return -1;
 }
 
-static int issue_char(struct lws *wsi, unsigned char c)
+static int LWS_WARN_UNUSED_RESULT
+issue_char(struct lws *wsi, unsigned char c)
 {
        unsigned short frag_len;
 
-       if (wsi->u.hdr.ah->pos == wsi->context->max_http_header_data) {
-               lwsl_warn("excessive header content\n");
+       if (lws_pos_in_bounds(wsi))
                return -1;
-       }
 
        frag_len = wsi->u.hdr.ah->frags[wsi->u.hdr.ah->nfrag].len;
        /*
@@ -334,15 +523,19 @@ static int issue_char(struct lws *wsi, unsigned char c)
 
        /* Insert a null character when we *hit* the limit: */
        if (frag_len == wsi->u.hdr.current_token_limit) {
+               if (lws_pos_in_bounds(wsi))
+                       return -1;
                wsi->u.hdr.ah->data[wsi->u.hdr.ah->pos++] = '\0';
                lwsl_warn("header %i exceeds limit %d\n",
-                         wsi->u.hdr.parser_state, wsi->u.hdr.current_token_limit);
+                         wsi->u.hdr.parser_state,
+                         wsi->u.hdr.current_token_limit);
        }
 
        return 1;
 }
 
-int lws_parse(struct lws *wsi, unsigned char c)
+int LWS_WARN_UNUSED_RESULT
+lws_parse(struct lws *wsi, unsigned char c)
 {
        static const unsigned char methods[] = {
                WSI_TOKEN_GET_URI,
@@ -351,11 +544,14 @@ int lws_parse(struct lws *wsi, unsigned char c)
                WSI_TOKEN_PUT_URI,
                WSI_TOKEN_PATCH_URI,
                WSI_TOKEN_DELETE_URI,
+               WSI_TOKEN_CONNECT,
        };
        struct allocated_headers *ah = wsi->u.hdr.ah;
        struct lws_context *context = wsi->context;
        unsigned int n, m, enc = 0;
 
+       assert(wsi->u.hdr.ah);
+
        switch (wsi->u.hdr.parser_state) {
        default:
 
@@ -363,8 +559,8 @@ int lws_parse(struct lws *wsi, unsigned char c)
 
                /* collect into malloc'd buffers */
                /* optional initial space swallow */
-               if (!ah->frags[ah->frag_index[
-                                     wsi->u.hdr.parser_state]].len && c == ' ')
+               if (!ah->frags[ah->frag_index[wsi->u.hdr.parser_state]].len &&
+                   c == ' ')
                        break;
 
                for (m = 0; m < ARRAY_SIZE(methods); m++)
@@ -382,6 +578,23 @@ int lws_parse(struct lws *wsi, unsigned char c)
                                if (issue_char(wsi, '/') < 0)
                                        return -1;
 
+                       if (wsi->u.hdr.ups == URIPS_SEEN_SLASH_DOT_DOT) {
+                               /*
+                                * back up one dir level if possible
+                                * safe against header fragmentation because
+                                * the method URI can only be in 1 fragment
+                                */
+                               if (ah->frags[ah->nfrag].len > 2) {
+                                       ah->pos--;
+                                       ah->frags[ah->nfrag].len--;
+                                       do {
+                                               ah->pos--;
+                                               ah->frags[ah->nfrag].len--;
+                                       } while (ah->frags[ah->nfrag].len > 1 &&
+                                                ah->data[ah->pos] != '/');
+                               }
+                       }
+
                        /* begin parsing HTTP version: */
                        if (issue_char(wsi, '\0') < 0)
                                return -1;
@@ -389,7 +602,10 @@ int lws_parse(struct lws *wsi, unsigned char c)
                        goto start_fragment;
                }
 
-               /* special URI processing... convert %xx */
+               /*
+                * PRIORITY 1
+                * special URI processing... convert %xx
+                */
 
                switch (wsi->u.hdr.ues) {
                case URIES_IDLE:
@@ -399,29 +615,19 @@ int lws_parse(struct lws *wsi, unsigned char c)
                        }
                        break;
                case URIES_SEEN_PERCENT:
-                       if (char_to_hex(c) < 0) {
-                               /* regurgitate */
-                               if (issue_char(wsi, '%') < 0)
-                                       return -1;
-                               wsi->u.hdr.ues = URIES_IDLE;
-                               /* continue on to assess c */
-                               break;
-                       }
+                       if (char_to_hex(c) < 0)
+                               /* illegal post-% char */
+                               goto forbid;
+
                        wsi->u.hdr.esc_stash = c;
                        wsi->u.hdr.ues = URIES_SEEN_PERCENT_H1;
                        goto swallow;
 
                case URIES_SEEN_PERCENT_H1:
-                       if (char_to_hex(c) < 0) {
-                               /* regurgitate */
-                               issue_char(wsi, '%');
-                               wsi->u.hdr.ues = URIES_IDLE;
-                               /* regurgitate + assess */
-                               if (lws_parse(wsi, wsi->u.hdr.esc_stash) < 0)
-                                       return -1;
-                               /* continue on to assess c */
-                               break;
-                       }
+                       if (char_to_hex(c) < 0)
+                               /* illegal post-% char */
+                               goto forbid;
+
                        c = (char_to_hex(wsi->u.hdr.esc_stash) << 4) |
                                        char_to_hex(c);
                        enc = 1;
@@ -430,6 +636,7 @@ int lws_parse(struct lws *wsi, unsigned char c)
                }
 
                /*
+                * PRIORITY 2
                 * special URI processing...
                 *  convert /.. or /... or /../ etc to /
                 *  convert /./ to /
@@ -439,9 +646,12 @@ int lws_parse(struct lws *wsi, unsigned char c)
 
                switch (wsi->u.hdr.ups) {
                case URIPS_IDLE:
+                       if (!c)
+                               return -1;
                        /* genuine delimiter */
                        if ((c == '&' || c == ';') && !enc) {
-                               issue_char(wsi, c);
+                               if (issue_char(wsi, c) < 0)
+                                       return -1;
                                /* swallow the terminator */
                                ah->frags[ah->nfrag].len--;
                                /* link to next fragment */
@@ -457,7 +667,9 @@ int lws_parse(struct lws *wsi, unsigned char c)
                                goto swallow;
                        }
                        /* uriencoded = in the name part, disallow */
-                       if (c == '=' && enc && !wsi->u.hdr.post_literal_equal)
+                       if (c == '=' && enc &&
+                           ah->frag_index[WSI_TOKEN_HTTP_URI_ARGS] &&
+                           !wsi->u.hdr.post_literal_equal)
                                c = '_';
 
                        /* after the real =, we don't care how many = */
@@ -485,6 +697,24 @@ int lws_parse(struct lws *wsi, unsigned char c)
                case URIPS_SEEN_SLASH_DOT:
                        /* swallow second . */
                        if (c == '.') {
+                               wsi->u.hdr.ups = URIPS_SEEN_SLASH_DOT_DOT;
+                               goto swallow;
+                       }
+                       /* change /./ to / */
+                       if (c == '/') {
+                               wsi->u.hdr.ups = URIPS_SEEN_SLASH;
+                               goto swallow;
+                       }
+                       /* it was like /.dir ... regurgitate the . */
+                       wsi->u.hdr.ups = URIPS_IDLE;
+                       if (issue_char(wsi, '.') < 0)
+                               return -1;
+                       break;
+
+               case URIPS_SEEN_SLASH_DOT_DOT:
+
+                       /* /../ or /..[End of URI] --> backup to last / */
+                       if (c == '/' || c == '?') {
                                /*
                                 * back up one dir level if possible
                                 * safe against header fragmentation because
@@ -499,36 +729,30 @@ int lws_parse(struct lws *wsi, unsigned char c)
                                        } while (ah->frags[ah->nfrag].len > 1 &&
                                                 ah->data[ah->pos] != '/');
                                }
-                               wsi->u.hdr.ups = URIPS_SEEN_SLASH_DOT_DOT;
-                               goto swallow;
-                       }
-                       /* change /./ to / */
-                       if (c == '/') {
                                wsi->u.hdr.ups = URIPS_SEEN_SLASH;
+                               if (ah->frags[ah->nfrag].len > 1)
+                                       break;
                                goto swallow;
                        }
-                       /* it was like /.dir ... regurgitate the . */
-                       wsi->u.hdr.ups = URIPS_IDLE;
+
+                       /*  /..[^/] ... regurgitate and allow */
+
                        if (issue_char(wsi, '.') < 0)
                                return -1;
-                       break;
-
-               case URIPS_SEEN_SLASH_DOT_DOT:
-                       /* swallow prior .. chars and any subsequent . */
-                       if (c == '.')
-                               goto swallow;
-                       /* last issued was /, so another / == // */
-                       if (c == '/')
-                               goto swallow;
-                       /* last we issued was / so SEEN_SLASH */
-                       wsi->u.hdr.ups = URIPS_SEEN_SLASH;
+                       if (issue_char(wsi, '.') < 0)
+                               return -1;
+                       wsi->u.hdr.ups = URIPS_IDLE;
                        break;
                }
 
                if (c == '?' && !enc &&
                    !ah->frag_index[WSI_TOKEN_HTTP_URI_ARGS]) { /* start of URI arguments */
+                       if (wsi->u.hdr.ues != URIES_IDLE)
+                               goto forbid;
+
                        /* seal off uri header */
-                       ah->data[ah->pos++] = '\0';
+                       if (issue_char(wsi, '\0') < 0)
+                               return -1;
 
                        /* move to using WSI_TOKEN_HTTP_URI_ARGS */
                        ah->nfrag++;
@@ -545,10 +769,12 @@ int lws_parse(struct lws *wsi, unsigned char c)
                }
 
 check_eol:
-
                /* bail at EOL */
                if (wsi->u.hdr.parser_state != WSI_TOKEN_CHALLENGE &&
-                                                                 c == '\x0d') {
+                   c == '\x0d') {
+                       if (wsi->u.hdr.ues != URIES_IDLE)
+                               goto forbid;
+
                        c = '\0';
                        wsi->u.hdr.parser_state = WSI_TOKEN_SKIPPING_SAW_CR;
                        lwsl_parser("*\n");
@@ -590,11 +816,18 @@ swallow:
                                }
                        /*
                         * hm it's an unknown http method from a client in fact,
-                        * treat as dangerous
+                        * it cannot be valid http
                         */
                        if (m == ARRAY_SIZE(methods)) {
+                               /*
+                                * are we set up to accept raw in these cases?
+                                */
+                               if (lws_check_opt(wsi->vhost->options,
+                                          LWS_SERVER_OPTION_FALLBACK_TO_RAW))
+                                       return 2; /* transition to raw */
+
                                lwsl_info("Unknown method - dropping\n");
-                               return -1;
+                               goto forbid;
                        }
                        break;
                }
@@ -616,8 +849,7 @@ swallow:
                        lwsl_parser("known hdr %d\n", n);
                        for (m = 0; m < ARRAY_SIZE(methods); m++)
                                if (n == methods[m] &&
-                                               ah->frag_index[
-                                                       methods[m]]) {
+                                   ah->frag_index[methods[m]]) {
                                        lwsl_warn("Duplicated method\n");
                                        return -1;
                                }
@@ -657,7 +889,7 @@ excessive:
 
                ah->frags[ah->nfrag].offset = ah->pos;
                ah->frags[ah->nfrag].len = 0;
-               ah->frags[ ah->nfrag].nfrag = 0;
+               ah->frags[ah->nfrag].nfrag = 0;
 
                n = ah->frag_index[wsi->u.hdr.parser_state];
                if (!n) { /* first fragment */
@@ -666,16 +898,11 @@ excessive:
                }
                /* continuation */
                while (ah->frags[n].nfrag)
-                               n = ah->frags[n].nfrag;
+                       n = ah->frags[n].nfrag;
                ah->frags[n].nfrag = ah->nfrag;
 
-               if (ah->pos == wsi->context->max_http_header_data) {
-                       lwsl_warn("excessive header content\n");
+               if (issue_char(wsi, ' ') < 0)
                        return -1;
-               }
-
-               ah->data[ah->pos++] = ' ';
-               ah->frags[ah->nfrag].len++;
                break;
 
                /* skipping arg part of a name we didn't recognize */
@@ -688,6 +915,8 @@ excessive:
 
        case WSI_TOKEN_SKIPPING_SAW_CR:
                lwsl_parser("WSI_TOKEN_SKIPPING_SAW_CR '%c'\n", c);
+               if (wsi->u.hdr.ues != URIES_IDLE)
+                       goto forbid;
                if (c == '\x0a') {
                        wsi->u.hdr.parser_state = WSI_TOKEN_NAME_PART;
                        wsi->u.hdr.lextable_pos = 0;
@@ -704,7 +933,8 @@ excessive:
        return 0;
 
 set_parsing_complete:
-
+       if (wsi->u.hdr.ues != URIES_IDLE)
+               goto forbid;
        if (lws_hdr_total_length(wsi, WSI_TOKEN_UPGRADE)) {
                if (lws_hdr_total_length(wsi, WSI_TOKEN_VERSION))
                        wsi->ietf_spec_revision =
@@ -716,53 +946,83 @@ set_parsing_complete:
        wsi->hdr_parsing_completed = 1;
 
        return 0;
-}
 
-
-/**
- * lws_frame_is_binary: true if the current frame was sent in binary mode
- *
- * @wsi: the connection we are inquiring about
- *
- * This is intended to be called from the LWS_CALLBACK_RECEIVE callback if
- * it's interested to see if the frame it's dealing with was sent in binary
- * mode.
- */
+forbid:
+       lwsl_notice(" forbidding on uri sanitation\n");
+       lws_return_http_status(wsi, HTTP_STATUS_FORBIDDEN, NULL);
+       return -1;
+}
 
 LWS_VISIBLE int lws_frame_is_binary(struct lws *wsi)
 {
        return wsi->u.ws.frame_is_binary;
 }
 
+void
+lws_add_wsi_to_draining_ext_list(struct lws *wsi)
+{
+       struct lws_context_per_thread *pt = &wsi->context->pt[(int)wsi->tsi];
+
+       if (wsi->u.ws.rx_draining_ext)
+               return;
+
+       lwsl_ext("%s: RX EXT DRAINING: Adding to list\n", __func__);
+
+       wsi->u.ws.rx_draining_ext = 1;
+       wsi->u.ws.rx_draining_ext_list = pt->rx_draining_ext_list;
+       pt->rx_draining_ext_list = wsi;
+}
+
+void
+lws_remove_wsi_from_draining_ext_list(struct lws *wsi)
+{
+       struct lws_context_per_thread *pt = &wsi->context->pt[(int)wsi->tsi];
+       struct lws **w = &pt->rx_draining_ext_list;
+
+       if (!wsi->u.ws.rx_draining_ext)
+               return;
+
+       lwsl_ext("%s: RX EXT DRAINING: Removing from list\n", __func__);
+
+       wsi->u.ws.rx_draining_ext = 0;
+
+       /* remove us from context draining ext list */
+       while (*w) {
+               if (*w == wsi) {
+                       /* if us, point it instead to who we were pointing to */
+                       *w = wsi->u.ws.rx_draining_ext_list;
+                       break;
+               }
+               w = &((*w)->u.ws.rx_draining_ext_list);
+       }
+       wsi->u.ws.rx_draining_ext_list = NULL;
+}
+
+/*
+ * client-parser.c: lws_client_rx_sm() needs to be roughly kept in
+ *   sync with changes here, esp related to ext draining
+ */
+
 int
 lws_rx_sm(struct lws *wsi, unsigned char c)
 {
-       struct lws_context_per_thread *pt = &wsi->context->pt[(int)wsi->tsi];
-       struct lws_tokens eff_buf;
-       int ret = 0, n, rx_draining_ext = 0;
        int callback_action = LWS_CALLBACK_RECEIVE;
+       int ret = 0, n, rx_draining_ext = 0;
+       struct lws_tokens eff_buf;
+
+       eff_buf.token = NULL;
+       eff_buf.token_len = 0;
        if (wsi->socket_is_permanently_unusable)
                return -1;
 
        switch (wsi->lws_rx_parse_state) {
        case LWS_RXPS_NEW:
                if (wsi->u.ws.rx_draining_ext) {
-                       struct lws **w = &pt->rx_draining_ext_list;
-
                        eff_buf.token = NULL;
                        eff_buf.token_len = 0;
-                       wsi->u.ws.rx_draining_ext = 0;
-                       /* remove us from context draining ext list */
-                       while (*w) {
-                               if (*w == wsi) {
-                                       *w = wsi->u.ws.rx_draining_ext_list;
-                                       break;
-                               }
-                               w = &((*w)->u.ws.rx_draining_ext_list);
-                       }
-                       wsi->u.ws.rx_draining_ext_list = NULL;
+                       lws_remove_wsi_from_draining_ext_list(wsi);
                        rx_draining_ext = 1;
-                       lwsl_err("%s: doing draining flow\n", __func__);
+                       lwsl_debug("%s: doing draining flow\n", __func__);
 
                        goto drain_extension;
                }
@@ -1014,9 +1274,11 @@ handle_first:
 
 
        case LWS_RXPS_PAYLOAD_UNTIL_LENGTH_EXHAUSTED:
-
                assert(wsi->u.ws.rx_ubuf);
 
+               if (wsi->u.ws.rx_draining_ext)
+                       goto drain_extension;
+
                if (wsi->u.ws.rx_ubuf_head + LWS_PRE >=
                    wsi->u.ws.rx_ubuf_alloc) {
                        lwsl_err("Attempted overflow \n");
@@ -1039,12 +1301,11 @@ handle_first:
 
                /*
                 * if there's no protocol max frame size given, we are
-                * supposed to default to LWS_MAX_SOCKET_IO_BUF
+                * supposed to default to context->pt_serv_buf_size
                 */
 
                if (!wsi->protocol->rx_buffer_size &&
-                                       wsi->u.ws.rx_ubuf_head !=
-                                                         LWS_MAX_SOCKET_IO_BUF)
+                   wsi->u.ws.rx_ubuf_head != wsi->context->pt_serv_buf_size)
                        break;
                else
                        if (wsi->protocol->rx_buffer_size &&
@@ -1077,6 +1338,17 @@ spill:
                                /* if he sends us 2 CLOSE, kill him */
                                return -1;
 
+                       if (lws_partial_buffered(wsi)) {
+                               /*
+                                * if we're in the middle of something,
+                                * we can't do a normal close response and
+                                * have to just close our end.
+                                */
+                               wsi->socket_is_permanently_unusable = 1;
+                               lwsl_parser("Closing on peer close due to Pending tx\n");
+                               return -1;
+                       }
+
                        if (user_callback_handle_rxflow(
                                        wsi->protocol->callback, wsi,
                                        LWS_CALLBACK_WS_PEER_INITIATED_CLOSE,
@@ -1129,6 +1401,11 @@ ping_drop:
                        lwsl_hexdump(&wsi->u.ws.rx_ubuf[LWS_PRE],
                                     wsi->u.ws.rx_ubuf_head);
 
+                       if (wsi->pending_timeout == PENDING_TIMEOUT_WS_PONG_CHECK_GET_PONG) {
+                               lwsl_info("received expected PONG on wsi %p\n", wsi);
+                               lws_set_timeout(wsi, NO_PENDING_TIMEOUT, 0);
+                       }
+
                        /* issue it */
                        callback_action = LWS_CALLBACK_RECEIVE_PONG;
                        break;
@@ -1151,9 +1428,10 @@ ping_drop:
                        eff_buf.token_len = wsi->u.ws.rx_ubuf_head;
 
                        if (lws_ext_cb_active(wsi, LWS_EXT_CB_EXTENDED_PAYLOAD_RX,
-                                       &eff_buf, 0) <= 0) /* not handle or fail */
+                                             &eff_buf, 0) <= 0)
+                               /* not handle or fail */
                                lwsl_ext("ext opc opcode 0x%x unknown\n",
-                                                             wsi->u.ws.opcode);
+                                        wsi->u.ws.opcode);
 
                        wsi->u.ws.rx_ubuf_head = 0;
                        return 0;
@@ -1176,6 +1454,9 @@ drain_extension:
                        goto already_done;
 
                n = lws_ext_cb_active(wsi, LWS_EXT_CB_PAYLOAD_RX, &eff_buf, 0);
+               /* eff_buf may be pointing somewhere completely different now,
+                * it's the output
+                */
                if (n < 0) {
                        /*
                         * we may rely on this to get RX, just drop connection
@@ -1189,10 +1470,9 @@ drain_extension:
 
                if (n && eff_buf.token_len) {
                        /* extension had more... main loop will come back */
-                       wsi->u.ws.rx_draining_ext = 1;
-                       wsi->u.ws.rx_draining_ext_list = pt->rx_draining_ext_list;
-                       pt->rx_draining_ext_list = wsi;
-               }
+                       lws_add_wsi_to_draining_ext_list(wsi);
+               } else
+                       lws_remove_wsi_from_draining_ext_list(wsi);
 
                if (eff_buf.token_len > 0 ||
                    callback_action == LWS_CALLBACK_RECEIVE_PONG) {
@@ -1229,26 +1509,71 @@ illegal_ctl_length:
        return -1;
 }
 
-
-/**
- * lws_remaining_packet_payload() - Bytes to come before "overall"
- *                                           rx packet is complete
- * @wsi:               Websocket instance (available from user callback)
- *
- *     This function is intended to be called from the callback if the
- *  user code is interested in "complete packets" from the client.
- *  libwebsockets just passes through payload as it comes and issues a buffer
- *  additionally when it hits a built-in limit.  The LWS_CALLBACK_RECEIVE
- *  callback handler can use this API to find out if the buffer it has just
- *  been given is the last piece of a "complete packet" from the client --
- *  when that is the case lws_remaining_packet_payload() will return
- *  0.
- *
- *  Many protocols won't care becuse their packets are always small.
- */
-
 LWS_VISIBLE size_t
 lws_remaining_packet_payload(struct lws *wsi)
 {
        return wsi->u.ws.rx_packet_length;
 }
+
+/* Once we reach LWS_RXPS_PAYLOAD_UNTIL_LENGTH_EXHAUSTED, we know how much
+ * to expect in that state and can deal with it in bulk more efficiently.
+ */
+
+int
+lws_payload_until_length_exhausted(struct lws *wsi, unsigned char **buf,
+                                  size_t *len)
+{
+       unsigned char *buffer = *buf, mask[4];
+       int buffer_size, n;
+       unsigned int avail;
+       char *rx_ubuf;
+
+       if (wsi->protocol->rx_buffer_size)
+               buffer_size = wsi->protocol->rx_buffer_size;
+       else
+               buffer_size = wsi->context->pt_serv_buf_size;
+       avail = buffer_size - wsi->u.ws.rx_ubuf_head;
+
+       /* do not consume more than we should */
+       if (avail > wsi->u.ws.rx_packet_length)
+               avail = wsi->u.ws.rx_packet_length;
+
+       /* do not consume more than what is in the buffer */
+       if (avail > *len)
+               avail = *len;
+
+       /* we want to leave 1 byte for the parser to handle properly */
+       if (avail <= 1)
+               return 0;
+
+       avail--;
+       rx_ubuf = wsi->u.ws.rx_ubuf + LWS_PRE + wsi->u.ws.rx_ubuf_head;
+       if (wsi->u.ws.all_zero_nonce)
+               memcpy(rx_ubuf, buffer, avail);
+       else {
+
+               for (n = 0; n < 4; n++)
+                       mask[n] = wsi->u.ws.mask[(wsi->u.ws.mask_idx + n) & 3];
+
+               /* deal with 4-byte chunks using unwrapped loop */
+               n = avail >> 2;
+               while (n--) {
+                       *(rx_ubuf++) = *(buffer++) ^ mask[0];
+                       *(rx_ubuf++) = *(buffer++) ^ mask[1];
+                       *(rx_ubuf++) = *(buffer++) ^ mask[2];
+                       *(rx_ubuf++) = *(buffer++) ^ mask[3];
+               }
+               /* and the remaining bytes bytewise */
+               for (n = 0; n < (int)(avail & 3); n++)
+                       *(rx_ubuf++) = *(buffer++) ^ mask[n];
+
+               wsi->u.ws.mask_idx = (wsi->u.ws.mask_idx + avail) & 3;
+       }
+
+       (*buf) += avail;
+       wsi->u.ws.rx_ubuf_head += avail;
+       wsi->u.ws.rx_packet_length -= avail;
+       *len -= avail;
+
+       return avail;
+}