replace per header mallocs with single malloc 3 level struct
authorAndy Green <andy.green@linaro.org>
Sun, 10 Feb 2013 10:02:31 +0000 (18:02 +0800)
committerAndy Green <andy.green@linaro.org>
Mon, 11 Feb 2013 03:10:57 +0000 (11:10 +0800)
This big patch replaces the malloc / realloc per header
approach used until now with a single three-level struct
that gets malloc'd during the header union phase and freed
in one go when we transition to a different union phase.

It's more expensive in that we malloc a bit more than 4Kbytes,
but it's a lot cheaper in terms of malloc, frees, heap fragmentation,
no reallocs, nothing to configure.  It also moves from arrays of
pointers (8 bytes on x86_64) to unsigned short offsets into the
data array, (2 bytes on all platforms).

The 3-level thing is all in one struct

 - array indexed by the header enum, pointing to first "fragment" index
(ie, header type to fragment lookup, or 0 for none)

 - array of fragments indexes, enough for 2 x the number of known headers
(fragment array... note that fragments can point to a "next"
fragment if the same header is spread across multiple entries)

 - linear char array where the known header payload gets written
(fragments point into null-terminated strings stored in here,
only the known header content is stored)

http headers can legally be split over multiple headers of the same
name which should be concatenated.  This scheme does not linearly
conatenate them but uses a linked list in the fragment structs to
link them.  There are apis to get the total length and copy out a
linear, concatenated version to a buffer.

Signed-off-by: Andy Green <andy.green@linaro.org>
12 files changed:
README.build
changelog
lib/client-handshake.c
lib/client.c
lib/handshake.c
lib/libwebsockets.c
lib/libwebsockets.h
lib/parsers.c
lib/private-libwebsockets.h
lib/server-handshake.c
lib/server.c
test-server/test-server.c

index fd9c8e8..c856393 100644 (file)
@@ -14,6 +14,7 @@ You need to regenerate the autotools and libtoolize stuff for your system
 
 $ ./autogen.sh
 
+Then,
 
 ------Fedora x86_64
 
@@ -22,7 +23,6 @@ $ ./autogen.sh
 ------Apple
 
 Christopher Baker reported that this is needed
-(and I was told separately enabling openssl makes trouble somehow)
 
 ./configure CC="gcc -arch i386 -arch x86_64" CXX="g++ -arch i386 -arch
 x86_64" CPP="gcc -E" CXXCPP="g++ -E" --enable-nofork
@@ -148,21 +148,19 @@ constrained, so you only need to take care about them if you want to tune them
 to the amount of memory available.
 
  - LWS_MAX_HEADER_NAME_LENGTH default 64: max characters in an HTTP header
-name that libwebsockets can cope with
+name that libwebsockets can cope with, if a header arrives bigger than this
+it's ignored until the next header is seen
 
- - LWS_MAX_HEADER_LEN default 4096: largest HTTP header value string length
-libwebsockets can cope with
+ - LWS_MAX_HEADER_LEN default 1024: allocated area to copy http headers that
+libwebsockets knows about into.  You only need to think about increasing this
+if your application might have monster length URLs for example, or some other
+header that lws cares about will be abnormally large (headers it does not
+know about are skipped).
 
- - LWS_INITIAL_HDR_ALLOC default 256: amount of memory to allocate initially,
-tradeoff between taking too much and needless realloc
-
- - LWS_ADDITIONAL_HDR_ALLOC default 64: how much to additionally realloc if
-the header value string keeps coming
-
- - LWS_MAX_PROTOCOLS default 10: largest amount of different protocols the
+ - LWS_MAX_PROTOCOLS default 5: largest amount of different protocols the
 server can serve
 
- - LWS_MAX_EXTENSIONS_ACTIVE default 10: largest amount of extensions we can
+ - LWS_MAX_EXTENSIONS_ACTIVE default 3: largest amount of extensions we can
 choose to have active on one connection
 
  - SPEC_LATEST_SUPPORTED default 13: only change if you want to remove support
index a6e0b58..442ab6a 100644 (file)
--- a/changelog
+++ b/changelog
@@ -54,21 +54,30 @@ User api changes
        By correctly setting this, you can save a lot of memory when your
        protocol has small frames (see the test server and client sources).
 
+ - LWS_MAX_HEADER_LEN now defaults to 1024 and is the total amount of known
+       header payload lws can cope with, that includes the GET URL, origin
+       etc.  Headers not understood by lws are ignored and their payload
+       not included in this.
+
 
 User api removals
 -----------------
 
-The configuration-time option MAX_USER_RX_BUFFER has been replaced by a
-buffer size chosen per-protocol.  For compatibility, there's a default of
-4096 rx buffer, but user code should set the appropriate size for the
-protocol frames.
+ - The configuration-time option MAX_USER_RX_BUFFER has been replaced by a
+       buffer size chosen per-protocol.  For compatibility, there's a default
+       of 4096 rx buffer, but user code should set the appropriate size for
+       the protocol frames.
+
+ - LWS_INITIAL_HDR_ALLOC and LWS_ADDITIONAL_HDR_ALLOC are no longer needed
+       and have been removed.  There's a new header management scheme that
+       handles them in a much more compact way.
 
 
 New features
 ------------
 
  - Cmake project file added, aimed initially at Windows support: this replaces
-the visual studio project files that were in the tree until now.
+       the visual studio project files that were in the tree until now.
 
  - PATH_MAX or MAX_PATH no longer needed
 
@@ -84,6 +93,15 @@ the visual studio project files that were in the tree until now.
        below the threshold, so it's removed.  Veto the compression extension
        in your user callback if you will typically have very small frames.
 
+ - There are many memory usage improvements, both a reduction in malloc/
+       realloc and architectural changes.  A websocket connection now
+       consumes only 296 bytes with SSL or 272 bytes without on x86_64,
+       during header processing an additional 1262 bytes is allocated in a
+       single malloc, but is freed when the websocket connection starts.
+       The RX frame buffer defined by the protocol in user
+       code is also allocated per connection, this represents the largest
+       frame you can receive atomically in that protocol.
+
 
 v1.1-chrome26-firefox18
 =======================
index 75e0b41..01aae4d 100644 (file)
@@ -253,10 +253,8 @@ libwebsocket_client_connect(struct libwebsocket_context *context,
        if (!wsi->c_callback)
                wsi->c_callback = context->protocols[0].callback;
 
-       for (n = 0; n < WSI_TOKEN_COUNT; n++) {
-               wsi->u.hdr.hdrs[n].token = NULL;
-               wsi->u.hdr.hdrs[n].token_len = 0;
-       }
+       if (lws_allocate_header_table(wsi))
+               goto oom3;
 
 #ifndef LWS_NO_EXTENSIONS
        /*
index 9c98059..200b3ad 100644 (file)
@@ -350,6 +350,8 @@ lws_client_interpret_server_handshake(struct libwebsocket_context *context,
 {
        const char *pc;
        int okay = 0;
+       char *p;
+       int len;
 #ifndef LWS_NO_EXTENSIONS
        char ext_name[128];
        struct libwebsocket_extension *ext;
@@ -363,44 +365,36 @@ lws_client_interpret_server_handshake(struct libwebsocket_context *context,
         * well, what the server sent looked reasonable for syntax.
         * Now let's confirm it sent all the necessary headers
         */
-#if 0
-       lwsl_parser("WSI_TOKEN_HTTP: %d\n",
-                                   wsi->u.hdr.hdrs[WSI_TOKEN_HTTP].token_len);
-       lwsl_parser("WSI_TOKEN_UPGRADE: %d\n",
-                                wsi->u.hdr.hdrs[WSI_TOKEN_UPGRADE].token_len);
-       lwsl_parser("WSI_TOKEN_CONNECTION: %d\n",
-                             wsi->u.hdr.hdrs[WSI_TOKEN_CONNECTION].token_len);
-       lwsl_parser("WSI_TOKEN_ACCEPT: %d\n",
-                                 wsi->u.hdr.hdrs[WSI_TOKEN_ACCEPT].token_len);
-       lwsl_parser("WSI_TOKEN_NONCE: %d\n",
-                                  wsi->u.hdr.hdrs[WSI_TOKEN_NONCE].token_len);
-       lwsl_parser("WSI_TOKEN_PROTOCOL: %d\n",
-                               wsi->u.hdr.hdrs[WSI_TOKEN_PROTOCOL].token_len);
-#endif
 
-       strtolower(wsi->u.hdr.hdrs[WSI_TOKEN_HTTP].token);
-       if (strncmp(wsi->u.hdr.hdrs[WSI_TOKEN_HTTP].token, "101", 3)) {
+       if (lws_hdr_total_length(wsi, WSI_TOKEN_ACCEPT) == 0)
+               goto bail3;
+
+       p = lws_hdr_simple_ptr(wsi, WSI_TOKEN_HTTP);
+       if (!p)
+               goto bail3;
+       if (p && strncmp(p, "101", 3)) {
                lwsl_warn("libwebsocket_client_handshake "
-                               "server sent bad HTTP response '%s'\n",
-                                wsi->u.hdr.hdrs[WSI_TOKEN_HTTP].token);
+                               "server sent bad HTTP response '%s'\n", p);
                goto bail3;
        }
 
-       strtolower(wsi->u.hdr.hdrs[WSI_TOKEN_UPGRADE].token);
-       if (strcmp(wsi->u.hdr.hdrs[WSI_TOKEN_UPGRADE].token,
-                                                        "websocket")) {
+       p = lws_hdr_simple_ptr(wsi, WSI_TOKEN_UPGRADE);
+       if (!p)
+               goto bail3;
+       strtolower(p);
+       if (strcmp(p, "websocket")) {
                lwsl_warn("libwebsocket_client_handshake server "
-                               "sent bad Upgrade header '%s'\n",
-                                 wsi->u.hdr.hdrs[WSI_TOKEN_UPGRADE].token);
+                               "sent bad Upgrade header '%s'\n", p);
                goto bail3;
        }
 
-       strtolower(wsi->u.hdr.hdrs[WSI_TOKEN_CONNECTION].token);
-       if (strcmp(wsi->u.hdr.hdrs[WSI_TOKEN_CONNECTION].token,
-                                                          "upgrade")) {
+       p = lws_hdr_simple_ptr(wsi, WSI_TOKEN_CONNECTION);
+       if (!p)
+               goto bail3;
+       strtolower(p);
+       if (strcmp(p, "upgrade")) {
                lwsl_warn("libwebsocket_client_handshake server "
-                               "sent bad Connection hdr '%s'\n",
-                          wsi->u.hdr.hdrs[WSI_TOKEN_CONNECTION].token);
+                               "sent bad Connection hdr '%s'\n", p);
                goto bail3;
        }
 
@@ -417,7 +411,8 @@ lws_client_interpret_server_handshake(struct libwebsocket_context *context,
         * of protocols we offered
         */
 
-       if (!wsi->u.hdr.hdrs[WSI_TOKEN_PROTOCOL].token_len) {
+       len = lws_hdr_total_length(wsi, WSI_TOKEN_PROTOCOL);
+       if (!len) {
 
                lwsl_info("lws_client_interpret_server_handshake "
                                               "WSI_TOKEN_PROTOCOL is null\n");
@@ -432,11 +427,11 @@ lws_client_interpret_server_handshake(struct libwebsocket_context *context,
                goto check_extensions;
        }
 
+       p = lws_hdr_simple_ptr(wsi, WSI_TOKEN_PROTOCOL);
+       len = strlen(p);
+
        while (*pc && !okay) {
-               if ((!strncmp(pc, wsi->u.hdr.hdrs[WSI_TOKEN_PROTOCOL].token,
-                wsi->u.hdr.hdrs[WSI_TOKEN_PROTOCOL].token_len)) &&
-                (pc[wsi->u.hdr.hdrs[WSI_TOKEN_PROTOCOL].token_len] == ',' ||
-                 pc[wsi->u.hdr.hdrs[WSI_TOKEN_PROTOCOL].token_len] == '\0')) {
+               if (!strncmp(pc, p, len) && (pc[len] == ',' || pc[len] == '\0')) {
                        okay = 1;
                        continue;
                }
@@ -453,8 +448,7 @@ lws_client_interpret_server_handshake(struct libwebsocket_context *context,
 
        if (!okay) {
                lwsl_err("libwebsocket_client_handshake server "
-                                       "sent bad protocol '%s'\n",
-                                wsi->u.hdr.hdrs[WSI_TOKEN_PROTOCOL].token);
+                                       "sent bad protocol '%s'\n", p);
                goto bail2;
        }
 
@@ -463,11 +457,11 @@ lws_client_interpret_server_handshake(struct libwebsocket_context *context,
         */
        n = 0;
        wsi->protocol = NULL;
-       while (context->protocols[n].callback && !wsi->protocol) {  /* Stop after finding first one?? */
-               if (strcmp(wsi->u.hdr.hdrs[WSI_TOKEN_PROTOCOL].token,
-                                          context->protocols[n].name) == 0) {
+       while (context->protocols[n].callback && !wsi->protocol) {
+               if (strcmp(p, context->protocols[n].name) == 0) {
                        wsi->protocol = &context->protocols[n];
                        wsi->c_callback = wsi->protocol->callback;
+                       break;
                }
                n++;
        }
@@ -475,8 +469,7 @@ lws_client_interpret_server_handshake(struct libwebsocket_context *context,
        if (wsi->protocol == NULL) {
                lwsl_err("libwebsocket_client_handshake server "
                                "requested protocol '%s', which we "
-                               "said we supported but we don't!\n",
-                                wsi->u.hdr.hdrs[WSI_TOKEN_PROTOCOL].token);
+                               "said we supported but we don't!\n", p);
                goto bail2;
        }
 
@@ -485,7 +478,7 @@ check_extensions:
 #ifndef LWS_NO_EXTENSIONS
        /* instantiate the accepted extensions */
 
-       if (!wsi->u.hdr.hdrs[WSI_TOKEN_EXTENSIONS].token_len) {
+       if (!lws_hdr_total_length(wsi, WSI_TOKEN_EXTENSIONS)) {
                lwsl_ext("no client extenstions allowed by server\n");
                goto check_accept;
        }
@@ -495,7 +488,10 @@ check_extensions:
         * and go through matching them or identifying bogons
         */
 
-       c = wsi->u.hdr.hdrs[WSI_TOKEN_EXTENSIONS].token;
+       if (lws_hdr_copy(wsi, (char *)context->service_buffer, sizeof(context->service_buffer), WSI_TOKEN_EXTENSIONS) < 0)
+               goto bail2;
+
+       c = (char *)context->service_buffer;
        n = 0;
        while (more) {
 
@@ -577,12 +573,11 @@ check_accept:
         * Confirm his accept token is the one we precomputed
         */
 
-       if (strcmp(wsi->u.hdr.hdrs[WSI_TOKEN_ACCEPT].token,
-                                 wsi->u.hdr.initial_handshake_hash_base64)) {
+       p = lws_hdr_simple_ptr(wsi, WSI_TOKEN_ACCEPT);
+       if (strcmp(p, wsi->u.hdr.initial_handshake_hash_base64)) {
                lwsl_warn("libwebsocket_client_handshake server "
-                       "sent bad ACCEPT '%s' vs computed '%s'\n",
-                       wsi->u.hdr.hdrs[WSI_TOKEN_ACCEPT].token,
-                                       wsi->u.hdr.initial_handshake_hash_base64);
+                       "sent bad ACCEPT '%s' vs computed '%s'\n", p,
+                                     wsi->u.hdr.initial_handshake_hash_base64);
                goto bail2;
        }
 
@@ -605,10 +600,8 @@ check_accept:
        libwebsocket_set_timeout(wsi, NO_PENDING_TIMEOUT, 0);
 
        /* free up his parsing allocations */
-
-       for (n = 0; n < WSI_TOKEN_COUNT; n++)
-               if (wsi->u.hdr.hdrs[n].token)
-                       free(wsi->u.hdr.hdrs[n].token);
+       if (wsi->u.hdr.ah)
+               free(wsi->u.hdr.ah);
 
        /* mark him as being alive */
 
@@ -616,6 +609,7 @@ check_accept:
        wsi->mode = LWS_CONNMODE_WS_CLIENT;
 
        /* union transition */
+
        memset(&wsi->u, 0, sizeof wsi->u);
 
        /*
@@ -669,16 +663,17 @@ bail3:
                free(wsi->c_protocol);
 
 bail2:
-       if (wsi->c_callback) wsi->c_callback(context, wsi,
-       LWS_CALLBACK_CLIENT_CONNECTION_ERROR,
-                        wsi->user_space,
-                        NULL, 0);
+       if (wsi->c_callback)
+               wsi->c_callback(context, wsi,
+                       LWS_CALLBACK_CLIENT_CONNECTION_ERROR,
+                                                     wsi->user_space, NULL, 0);
+
        lwsl_info("closing connection due to bail2 connection error\n");
+
        /* free up his parsing allocations */
 
-       for (n = 0; n < WSI_TOKEN_COUNT; n++)
-               if (wsi->u.hdr.hdrs[n].token)
-                       free(wsi->u.hdr.hdrs[n].token);
+       if (wsi->u.hdr.ah)
+               free(wsi->u.hdr.ah);
 
        libwebsocket_close_and_free_session(context, wsi,
                                                 LWS_CLOSE_STATUS_PROTOCOL_ERR);
index 2ac7e76..96e1aba 100644 (file)
@@ -113,15 +113,15 @@ libwebsocket_read(struct libwebsocket_context *context,
 
                /* is this websocket protocol or normal http 1.0? */
 
-               if (!wsi->u.hdr.hdrs[WSI_TOKEN_UPGRADE].token_len ||
-                            !wsi->u.hdr.hdrs[WSI_TOKEN_CONNECTION].token_len) {
+               if (!lws_hdr_total_length(wsi, WSI_TOKEN_UPGRADE) ||
+                            !lws_hdr_total_length(wsi, WSI_TOKEN_CONNECTION)) {
                        wsi->state = WSI_STATE_HTTP;
                        if (wsi->protocol->callback)
                                if (wsi->protocol->callback(context, wsi,
                                                LWS_CALLBACK_HTTP,
                                                wsi->user_space,
-                                               wsi->u.hdr.hdrs[WSI_TOKEN_GET_URI].token,
-                                               wsi->u.hdr.hdrs[WSI_TOKEN_GET_URI].token_len)) {
+                                               lws_hdr_simple_ptr(wsi, WSI_TOKEN_GET_URI),
+                                               lws_hdr_total_length(wsi, WSI_TOKEN_GET_URI))) {
                                        lwsl_info("LWS_CALLBACK_HTTP wanted to close\n");
                                        goto bail;
                                }
@@ -139,12 +139,12 @@ libwebsocket_read(struct libwebsocket_context *context,
 
                while (wsi->protocol->callback) {
 
-                       if (wsi->u.hdr.hdrs[WSI_TOKEN_PROTOCOL].token == NULL) {
+                       if (!lws_hdr_total_length(wsi, WSI_TOKEN_PROTOCOL)) {
                                if (wsi->protocol->name == NULL)
                                        break;
                        } else
                                if (wsi->protocol->name && strcmp(
-                                    wsi->u.hdr.hdrs[WSI_TOKEN_PROTOCOL].token,
+                                       lws_hdr_simple_ptr(wsi, WSI_TOKEN_PROTOCOL),
                                                      wsi->protocol->name) == 0)
                                        break;
 
@@ -154,14 +154,14 @@ libwebsocket_read(struct libwebsocket_context *context,
                /* we didn't find a protocol he wanted? */
 
                if (wsi->protocol->callback == NULL) {
-                       if (wsi->u.hdr.hdrs[WSI_TOKEN_PROTOCOL].token == NULL) {
+                       if (lws_hdr_simple_ptr(wsi, WSI_TOKEN_PROTOCOL) == NULL) {
                                lwsl_info("[no protocol] "
                                        "mapped to protocol 0 handler\n");
                                wsi->protocol = &context->protocols[0];
                        } else {
                                lwsl_err("Requested protocol %s "
                                                "not supported\n",
-                                    wsi->u.hdr.hdrs[WSI_TOKEN_PROTOCOL].token);
+                                               lws_hdr_simple_ptr(wsi, WSI_TOKEN_PROTOCOL));
                                goto bail;
                        }
                }
@@ -173,7 +173,8 @@ libwebsocket_read(struct libwebsocket_context *context,
 
                if ((wsi->protocol->callback)(wsi->protocol->owning_server, wsi,
                                LWS_CALLBACK_FILTER_PROTOCOL_CONNECTION,
-                                               &wsi->u.hdr.hdrs[0], NULL, 0)) {
+                               lws_hdr_simple_ptr(wsi, WSI_TOKEN_PROTOCOL),
+                                                                    NULL, 0)) {
                        lwsl_warn("User code denied connection\n");
                        goto bail;
                }
@@ -201,11 +202,8 @@ libwebsocket_read(struct libwebsocket_context *context,
 
                /* drop the header info */
 
-               /* free up his parsing allocations... these are gone... */
-
-               for (n = 0; n < WSI_TOKEN_COUNT; n++)
-                       if (wsi->u.hdr.hdrs[n].token)
-                               free(wsi->u.hdr.hdrs[n].token);
+               if (wsi->u.hdr.ah)
+                       free(wsi->u.hdr.ah);
 
                wsi->mode = LWS_CONNMODE_WS_SERVING;
 
index 32f4a8d..6b1e217 100644 (file)
@@ -911,6 +911,7 @@ libwebsocket_service_fd(struct libwebsocket_context *context,
                                            wsi->state == WSI_STATE_ESTABLISHED)
                        if (lws_handle_POLLOUT_event(context, wsi,
                                                                  pollfd) < 0) {
+                               lwsl_info("libwebsocket_service_fd: closing");
                                libwebsocket_close_and_free_session(
                                         context, wsi, LWS_CLOSE_STATUS_NORMAL);
                                return 0;
@@ -1565,8 +1566,6 @@ libwebsocket_create_context(struct lws_context_creation_info *info)
        lwsl_notice("Library version: %s\n", library_version);
        lwsl_info(" LWS_MAX_HEADER_NAME_LENGTH: %u\n", LWS_MAX_HEADER_NAME_LENGTH);
        lwsl_info(" LWS_MAX_HEADER_LEN: %u\n", LWS_MAX_HEADER_LEN);
-       lwsl_info(" LWS_INITIAL_HDR_ALLOC: %u\n", LWS_INITIAL_HDR_ALLOC);
-       lwsl_info(" LWS_ADDITIONAL_HDR_ALLOC: %u\n", LWS_ADDITIONAL_HDR_ALLOC);
        lwsl_info(" LWS_MAX_PROTOCOLS: %u\n", LWS_MAX_PROTOCOLS);
 #ifndef LWS_NO_EXTENSIONS
        lwsl_info(" LWS_MAX_EXTENSIONS_ACTIVE: %u\n", LWS_MAX_EXTENSIONS_ACTIVE);
@@ -1760,7 +1759,7 @@ libwebsocket_create_context(struct lws_context_creation_info *info)
                                                       "serving unencrypted\n");
 #endif
 
-               lwsl_notice(" per-connection allocation: %u + headers during handshake + frame buffer set by protocol\n", sizeof(struct libwebsocket));
+               lwsl_notice(" per-connection allocation: %u + %u headers (only during handshake) + frame buffer set by protocol\n", sizeof(struct libwebsocket), sizeof(struct allocated_headers));
        }
 #endif
 
index 5e2681c..c39d13a 100644 (file)
@@ -930,6 +930,14 @@ lws_b64_decode_string(const char *in, char *out, int out_size);
 LWS_EXTERN const char *
 lws_get_library_version(void);
 
+/* access to headers... only valid while headers valid */
+
+extern int
+lws_hdr_total_length(struct libwebsocket *wsi, enum lws_token_indexes h);
+
+extern int
+lws_hdr_copy(struct libwebsocket *wsi, char *dest, int len, enum lws_token_indexes h);
+
 /*
  * Note: this is not normally needed as a user api.  It's provided in case it is
  * useful when integrating with other app poll loop service code.
index 438f103..e445aac 100644 (file)
@@ -307,7 +307,68 @@ int lextable_decode(int pos, char c)
        return pos;
 }
 
+int lws_allocate_header_table(struct libwebsocket *wsi)
+{
+       wsi->u.hdr.ah = malloc(sizeof *wsi->u.hdr.ah);
+       if (wsi->u.hdr.ah == NULL) {
+               lwsl_err("Out of memory\n");
+               return -1;
+       }
+       memset(wsi->u.hdr.ah->frag_index, 0, sizeof wsi->u.hdr.ah->frag_index);
+       wsi->u.hdr.ah->next_frag_index = 0;
+       wsi->u.hdr.ah->pos = 0;
+
+       return 0;
+}
+
+int lws_hdr_total_length(struct libwebsocket *wsi, enum lws_token_indexes h)
+{
+       int n;
+       int len = 0;
+
+       n = wsi->u.hdr.ah->frag_index[h];
+       if (n == 0)
+               return 0;
+
+       do {
+               len += wsi->u.hdr.ah->frags[n].len;
+               n = wsi->u.hdr.ah->frags[n].next_frag_index;
+       } while (n);
+
+       return len;
+}
+
+int lws_hdr_copy(struct libwebsocket *wsi, char *dest, int len, enum lws_token_indexes h)
+{
+       int toklen = lws_hdr_total_length(wsi, h);
+       int n;
+
+       if (toklen >= len)
+               return -1;
+
+       n = wsi->u.hdr.ah->frag_index[h];
+       if (n == 0)
+               return 0;
+
+       do {
+               strcpy(dest, &wsi->u.hdr.ah->data[wsi->u.hdr.ah->frags[n].offset]);
+               dest += wsi->u.hdr.ah->frags[n].len;
+               n = wsi->u.hdr.ah->frags[n].next_frag_index;
+       } while (n);
+
+       return toklen;
+}
+
+char * lws_hdr_simple_ptr(struct libwebsocket *wsi, enum lws_token_indexes h)
+{
+       int n;
 
+       n = wsi->u.hdr.ah->frag_index[h];
+       if (!n)
+               return NULL;
+
+       return &wsi->u.hdr.ah->data[wsi->u.hdr.ah->frags[n].offset];
+}
 
 int libwebsocket_parse(struct libwebsocket *wsi, unsigned char c)
 {
@@ -331,75 +392,39 @@ int libwebsocket_parse(struct libwebsocket *wsi, unsigned char c)
        case WSI_TOKEN_NONCE:
        case WSI_TOKEN_EXTENSIONS:
        case WSI_TOKEN_HTTP:
-       case WSI_TOKEN_MUXURL:
 
                lwsl_parser("WSI_TOKEN_(%d) '%c'\n", wsi->u.hdr.parser_state, c);
 
                /* collect into malloc'd buffers */
-               /* optional space swallow */
-               if (!wsi->u.hdr.hdrs[wsi->u.hdr.parser_state].token_len && c == ' ')
+               /* optional initial space swallow */
+               if (!wsi->u.hdr.ah->frags[wsi->u.hdr.ah->frag_index[wsi->u.hdr.parser_state]].len && c == ' ')
                        break;
 
                /* special case space terminator for get-uri */
                if (wsi->u.hdr.parser_state == WSI_TOKEN_GET_URI && c == ' ') {
-                       wsi->u.hdr.hdrs[wsi->u.hdr.parser_state].token[
-                          wsi->u.hdr.hdrs[wsi->u.hdr.parser_state].token_len] = '\0';
-//                     lwsl_parser("uri '%s'\n", wsi->u.hdr.hdrs[wsi->u.hdr.parser_state].token);
+                       c = '\0';
                        wsi->u.hdr.parser_state = WSI_TOKEN_SKIPPING;
-                       break;
-               }
-
-               /* allocate appropriate memory */
-               if (wsi->u.hdr.hdrs[wsi->u.hdr.parser_state].token_len ==
-                                            wsi->u.hdr.current_alloc_len - 1) {
-                       /* need to extend */
-                       wsi->u.hdr.current_alloc_len += LWS_ADDITIONAL_HDR_ALLOC;
-                       if (wsi->u.hdr.current_alloc_len >= LWS_MAX_HEADER_LEN) {
-                               /* it's waaay to much payload, fail it */
-                               strcpy(wsi->u.hdr.hdrs[wsi->u.hdr.parser_state].token,
-                                  "!!! Length exceeded maximum supported !!!");
-                               wsi->u.hdr.parser_state = WSI_TOKEN_SKIPPING;
-                               break;
-                       }
-                       wsi->u.hdr.hdrs[wsi->u.hdr.parser_state].token = (char *)
-                              realloc(wsi->u.hdr.hdrs[wsi->u.hdr.parser_state].token,
-                                                       wsi->u.hdr.current_alloc_len);
-                       if (wsi->u.hdr.hdrs[wsi->u.hdr.parser_state].token == NULL) {
-                               lwsl_err("Out of mem\n");
-                               return -1;
-                       }
                }
 
                /* bail at EOL */
                if (wsi->u.hdr.parser_state != WSI_TOKEN_CHALLENGE && c == '\x0d') {
-                       wsi->u.hdr.hdrs[wsi->u.hdr.parser_state].token[
-                          wsi->u.hdr.hdrs[wsi->u.hdr.parser_state].token_len] = '\0';
+                       c = '\0';
                        wsi->u.hdr.parser_state = WSI_TOKEN_SKIPPING_SAW_CR;
                        lwsl_parser("*\n");
-                       break;
                }
 
-               wsi->u.hdr.hdrs[wsi->u.hdr.parser_state].token[
-                           wsi->u.hdr.hdrs[wsi->u.hdr.parser_state].token_len++] = c;
+               if (wsi->u.hdr.ah->pos == sizeof wsi->u.hdr.ah->data) {
+                       lwsl_warn("excessive header content\n");
+                       return -1;
+               }
+               wsi->u.hdr.ah->data[wsi->u.hdr.ah->pos++] = c;
+               if (c)
+                       wsi->u.hdr.ah->frags[wsi->u.hdr.ah->next_frag_index].len++;
 
                /* per-protocol end of headers management */
 
-               if (wsi->u.hdr.parser_state != WSI_TOKEN_CHALLENGE)
-                       break;
-
-               goto set_parsing_complete;
-
-       case WSI_INIT_TOKEN_MUXURL:
-               wsi->u.hdr.parser_state = WSI_TOKEN_MUXURL;
-               wsi->u.hdr.current_alloc_len = LWS_INITIAL_HDR_ALLOC;
-
-               wsi->u.hdr.hdrs[wsi->u.hdr.parser_state].token = (char *)
-                                        malloc(wsi->u.hdr.current_alloc_len);
-               if (wsi->u.hdr.hdrs[wsi->u.hdr.parser_state].token == NULL) {
-                       lwsl_err("Out of mem\n");
-                       return -1;
-               }
-               wsi->u.hdr.hdrs[wsi->u.hdr.parser_state].token_len = 0;
+               if (wsi->u.hdr.parser_state == WSI_TOKEN_CHALLENGE)
+                       goto set_parsing_complete;
                break;
 
                /* collecting and checking a name part */
@@ -415,10 +440,11 @@ int libwebsocket_parse(struct libwebsocket *wsi, unsigned char c)
                wsi->u.hdr.name_buffer[wsi->u.hdr.name_buffer_pos] = '\0';
 
                wsi->u.hdr.lextable_pos = lextable_decode(wsi->u.hdr.lextable_pos, c);
+
                if (wsi->u.hdr.lextable_pos < 0) {
                        /* this is not a header we know about */
-                       if (wsi->u.hdr.hdrs[WSI_TOKEN_GET_URI].token_len) {
-                               /* if not the method, just skip it all */
+                       if (wsi->u.hdr.ah->frag_index[WSI_TOKEN_GET_URI]) {
+                               /* altready had the method, no idea what this crap is, ignore */
                                wsi->u.hdr.parser_state = WSI_TOKEN_SKIPPING;
                                break;
                        }
@@ -427,14 +453,7 @@ int libwebsocket_parse(struct libwebsocket *wsi, unsigned char c)
                                lwsl_info("Unknown method %s\n", wsi->u.hdr.name_buffer);
                                /* treat it as GET */
                                wsi->u.hdr.parser_state = WSI_TOKEN_GET_URI;
-                               wsi->u.hdr.current_alloc_len = LWS_INITIAL_HDR_ALLOC;
-                               wsi->u.hdr.hdrs[WSI_TOKEN_GET_URI].token =
-                                       (char *)malloc(wsi->u.hdr.current_alloc_len);
-                               if (wsi->u.hdr.hdrs[WSI_TOKEN_GET_URI].token == NULL) {
-                                       lwsl_err("Out of mem\n");
-                                       return -1;
-                               }
-                               break;
+                               goto start_fragment;
                        }
                }
                if (lextable[wsi->u.hdr.lextable_pos + 1] == 0) {
@@ -452,35 +471,48 @@ int libwebsocket_parse(struct libwebsocket *wsi, unsigned char c)
                        if (n == WSI_TOKEN_SWORIGIN)
                                n = WSI_TOKEN_ORIGIN;
 
-                       wsi->u.hdr.parser_state = (enum lws_token_indexes) (WSI_TOKEN_GET_URI + n);
+                       wsi->u.hdr.parser_state = (enum lws_token_indexes)
+                                                       (WSI_TOKEN_GET_URI + n);
+                       if (wsi->u.hdr.parser_state == WSI_TOKEN_CHALLENGE)
+                               goto set_parsing_complete;
 
-                       n = WSI_TOKEN_COUNT;
-
-                       /*  If the header has been seen already, just append */
-                       if (!wsi->u.hdr.hdrs[wsi->u.hdr.parser_state].token) {
+                       goto start_fragment;
+               }
+               break;
 
-                               wsi->u.hdr.current_alloc_len = LWS_INITIAL_HDR_ALLOC;
-                               wsi->u.hdr.hdrs[wsi->u.hdr.parser_state].token = (char *)
-                                                        malloc(wsi->u.hdr.current_alloc_len);
-                               if (wsi->u.hdr.hdrs[wsi->u.hdr.parser_state].token == NULL) {
-                                       lwsl_err("Out of mem\n");
-                                       return -1;
-                               }
-                               wsi->u.hdr.hdrs[wsi->u.hdr.parser_state].token_len = 0;
-                       }
+start_fragment:
+               wsi->u.hdr.ah->next_frag_index++;
+               if (wsi->u.hdr.ah->next_frag_index == sizeof(wsi->u.hdr.ah->frags) / sizeof(wsi->u.hdr.ah->frags[0])) {
+                       lwsl_warn("More header fragments than we can deal with, dropping\n");
+                       return -1;
                }
 
-               if (wsi->u.hdr.parser_state == WSI_TOKEN_CHALLENGE) {
-                       if (wsi->u.hdr.hdrs[WSI_TOKEN_CHALLENGE].token) {
-                               free(wsi->u.hdr.hdrs[WSI_TOKEN_CHALLENGE].token);
-                               wsi->u.hdr.hdrs[WSI_TOKEN_CHALLENGE].token = NULL;
+               wsi->u.hdr.ah->frags[wsi->u.hdr.ah->next_frag_index].offset = wsi->u.hdr.ah->pos;
+               wsi->u.hdr.ah->frags[wsi->u.hdr.ah->next_frag_index].len = 0;
+               wsi->u.hdr.ah->frags[wsi->u.hdr.ah->next_frag_index].next_frag_index = 0;
+
+               n = wsi->u.hdr.ah->frag_index[wsi->u.hdr.parser_state];
+               if (!n) { /* first fragment */
+                       wsi->u.hdr.ah->frag_index[wsi->u.hdr.parser_state] =
+                                                wsi->u.hdr.ah->next_frag_index;
+               } else { /* continuation */
+                       while (wsi->u.hdr.ah->frags[n].next_frag_index)
+                               n = wsi->u.hdr.ah->frags[n].next_frag_index;
+                       wsi->u.hdr.ah->frags[n].next_frag_index =
+                                                wsi->u.hdr.ah->next_frag_index;
+
+                       if (wsi->u.hdr.ah->pos == sizeof wsi->u.hdr.ah->data) {
+                               lwsl_warn("excessive header content\n");
+                               return -1;
                        }
-                       wsi->u.hdr.hdrs[WSI_TOKEN_CHALLENGE].token_len = 0;
-                       goto set_parsing_complete;
+
+                       wsi->u.hdr.ah->data[wsi->u.hdr.ah->pos++] = ' ';
+                       wsi->u.hdr.ah->frags[wsi->u.hdr.ah->next_frag_index].len++;
                }
 
                break;
 
+
                /* skipping arg part of a name we didn't recognize */
        case WSI_TOKEN_SKIPPING:
                lwsl_parser("WSI_TOKEN_SKIPPING '%c'\n", c);
@@ -510,13 +542,13 @@ int libwebsocket_parse(struct libwebsocket *wsi, unsigned char c)
 
 set_parsing_complete:
 
-       if (wsi->u.hdr.hdrs[WSI_TOKEN_UPGRADE].token_len) {
-               if (!wsi->u.hdr.hdrs[WSI_TOKEN_VERSION].token_len) {
+       if (lws_hdr_total_length(wsi, WSI_TOKEN_UPGRADE)) {
+               if (!lws_hdr_total_length(wsi, WSI_TOKEN_VERSION)) {
 //                     lwsl_info("Missing Version Header\n");
 //                     return 1;
                } else
                        wsi->ietf_spec_revision =
-                               atoi(wsi->u.hdr.hdrs[WSI_TOKEN_VERSION].token);
+                              atoi(lws_hdr_simple_ptr(wsi, WSI_TOKEN_VERSION));
 
                lwsl_parser("v%02d headers completed\n", wsi->ietf_spec_revision);
        }
index b05f25b..a24d584 100644 (file)
@@ -117,13 +117,7 @@ SHA1(const unsigned char *d, size_t n, unsigned char *md);
 #define LWS_MAX_HEADER_NAME_LENGTH 64
 #endif
 #ifndef LWS_MAX_HEADER_LEN
-#define LWS_MAX_HEADER_LEN 4096
-#endif
-#ifndef LWS_INITIAL_HDR_ALLOC
-#define LWS_INITIAL_HDR_ALLOC 256
-#endif
-#ifndef LWS_ADDITIONAL_HDR_ALLOC
-#define LWS_ADDITIONAL_HDR_ALLOC 64
+#define LWS_MAX_HEADER_LEN 1024
 #endif
 #ifndef LWS_MAX_PROTOCOLS
 #define LWS_MAX_PROTOCOLS 5
@@ -313,10 +307,24 @@ struct _lws_http_mode_related {
        unsigned long filelen;
 };
 
+struct lws_fragments {
+       unsigned short offset;
+       unsigned short len;
+       unsigned char next_frag_index;
+};
+
+struct allocated_headers {
+       unsigned short next_frag_index;
+       unsigned short pos;
+       unsigned char frag_index[WSI_TOKEN_COUNT];
+       struct lws_fragments frags[WSI_TOKEN_COUNT * 2];
+       char data[LWS_MAX_HEADER_LEN];
+};
+
 struct _lws_header_related {
        char name_buffer[LWS_MAX_HEADER_NAME_LENGTH];
        unsigned char name_buffer_pos;
-       struct lws_tokens hdrs[WSI_TOKEN_COUNT];
+       struct allocated_headers *ah;
        int lextable_pos;
        unsigned char parser_state; /* enum lws_token_indexes */
        int current_alloc_len;
@@ -490,6 +498,12 @@ user_callback_handle_rxflow(callback_function, struct libwebsocket_context * con
 extern int
 lws_set_socket_options(struct libwebsocket_context *context, int fd);
 
+extern int
+lws_allocate_header_table(struct libwebsocket *wsi);
+
+extern char *
+lws_hdr_simple_ptr(struct libwebsocket *wsi, enum lws_token_indexes h);
+
 #ifndef LWS_OPENSSL_SUPPORT
 
 unsigned char *
index ce6cb55..8b4260c 100644 (file)
@@ -22,8 +22,6 @@
 #include "private-libwebsockets.h"
 
 #define LWS_CPYAPP(ptr, str) { strcpy(ptr, str); ptr += strlen(str); }
-#define LWS_CPYAPP_TOKEN(ptr, tok) { strcpy(p, wsi->u.hdr.hdrs[tok].token); \
-               p += wsi->u.hdr.hdrs[tok].token_len; }
 
 /*
  * Perform the newer BASE64-encoded handshake scheme
@@ -45,24 +43,23 @@ handshake_0405(struct libwebsocket_context *context, struct libwebsocket *wsi)
        int more = 1;
 #endif
 
-       if (!wsi->u.hdr.hdrs[WSI_TOKEN_HOST].token_len ||
-           !wsi->u.hdr.hdrs[WSI_TOKEN_KEY].token_len) {
+       if (!lws_hdr_total_length(wsi, WSI_TOKEN_HOST) ||
+                               !lws_hdr_total_length(wsi, WSI_TOKEN_KEY)) {
                lwsl_parser("handshake_04 missing pieces\n");
                /* completed header processing, but missing some bits */
                goto bail;
        }
 
-       if (wsi->u.hdr.hdrs[WSI_TOKEN_KEY].token_len >=
-                                                    MAX_WEBSOCKET_04_KEY_LEN) {
+       if (lws_hdr_total_length(wsi, WSI_TOKEN_KEY) >= MAX_WEBSOCKET_04_KEY_LEN) {
                lwsl_warn("Client sent handshake key longer "
                           "than max supported %d\n", MAX_WEBSOCKET_04_KEY_LEN);
                goto bail;
        }
 
        n = snprintf((char *)context->service_buffer,
-               sizeof context->service_buffer,
-               "%s258EAFA5-E914-47DA-95CA-C5AB0DC85B11",
-               wsi->u.hdr.hdrs[WSI_TOKEN_KEY].token);
+                       sizeof context->service_buffer,
+                               "%s258EAFA5-E914-47DA-95CA-C5AB0DC85B11",
+                               lws_hdr_simple_ptr(wsi, WSI_TOKEN_KEY));
 
        SHA1(context->service_buffer, n, hash);
 
@@ -92,9 +89,12 @@ handshake_0405(struct libwebsocket_context *context, struct libwebsocket *wsi)
        strcpy(p, (char *)context->service_buffer);
        p += accept_len;
 
-       if (wsi->u.hdr.hdrs[WSI_TOKEN_PROTOCOL].token) {
+       if (lws_hdr_total_length(wsi, WSI_TOKEN_PROTOCOL)) {
                LWS_CPYAPP(p, "\x0d\x0aSec-WebSocket-Protocol: ");
-               LWS_CPYAPP_TOKEN(p, WSI_TOKEN_PROTOCOL);
+               n = lws_hdr_copy(wsi, p, 128, WSI_TOKEN_PROTOCOL);
+               if (n < 0)
+                       goto bail;
+               p += n;
        }
 
 #ifndef LWS_NO_EXTENSIONS
@@ -103,16 +103,18 @@ handshake_0405(struct libwebsocket_context *context, struct libwebsocket *wsi)
         * enable on this connection, and give him back the list
         */
 
-       if (wsi->u.hdr.hdrs[WSI_TOKEN_EXTENSIONS].token_len) {
+       if (lws_hdr_total_length(wsi, WSI_TOKEN_EXTENSIONS)) {
 
                /*
                 * break down the list of client extensions
                 * and go through them
                 */
 
-               c = wsi->u.hdr.hdrs[WSI_TOKEN_EXTENSIONS].token;
-               lwsl_parser("wsi->u.hdr.hdrs[WSI_TOKEN_EXTENSIONS].token = %s\n",
-                                 wsi->u.hdr.hdrs[WSI_TOKEN_EXTENSIONS].token);
+               if (lws_hdr_copy(wsi, (char *)context->service_buffer, sizeof context->service_buffer, WSI_TOKEN_EXTENSIONS) < 0)
+                       goto bail;
+
+               c = (char *)context->service_buffer;
+               lwsl_parser("WSI_TOKEN_EXTENSIONS = '%s'\n", c);
                wsi->count_active_extensions = 0;
                n = 0;
                while (more) {
@@ -259,9 +261,8 @@ handshake_0405(struct libwebsocket_context *context, struct libwebsocket *wsi)
 bail:
        /* free up his parsing allocations */
 
-       for (n = 0; n < WSI_TOKEN_COUNT; n++)
-               if (wsi->u.hdr.hdrs[n].token)
-                       free(wsi->u.hdr.hdrs[n].token);
+       if (wsi->u.hdr.ah)
+               free(wsi->u.hdr.ah);
 
        return -1;
 }
index 5f746f5..c99d8fa 100644 (file)
@@ -85,7 +85,6 @@ struct libwebsocket *
 libwebsocket_create_new_server_wsi(struct libwebsocket_context *context)
 {
        struct libwebsocket *new_wsi;
-       int n;
 
        new_wsi = (struct libwebsocket *)malloc(sizeof(struct libwebsocket));
        if (new_wsi == NULL) {
@@ -105,9 +104,9 @@ libwebsocket_create_new_server_wsi(struct libwebsocket_context *context)
        new_wsi->u.hdr.name_buffer_pos = 0;
        new_wsi->mode = LWS_CONNMODE_HTTP_SERVING;
 
-       for (n = 0; n < WSI_TOKEN_COUNT; n++) {
-               new_wsi->u.hdr.hdrs[n].token = NULL;
-               new_wsi->u.hdr.hdrs[n].token_len = 0;
+       if (lws_allocate_header_table(new_wsi)) {
+               free(new_wsi);
+               return NULL;
        }
 
        /*
@@ -164,6 +163,7 @@ int lws_server_socket_service(struct libwebsocket_context *context,
                                return 0;
                        }
                        if (!len) {
+                               lwsl_info("lws_server_socket_service: closing on zero length read\n");
                                libwebsocket_close_and_free_session(context, wsi,
                                                            LWS_CLOSE_STATUS_NOSTATUS);
                                return 0;
@@ -186,9 +186,11 @@ int lws_server_socket_service(struct libwebsocket_context *context,
                if (wsi->state != WSI_STATE_HTTP_ISSUING_FILE)
                        break;
 
-               if (libwebsockets_serve_http_file_fragment(context, wsi)) /* nonzero for completion or error */
+               if (libwebsockets_serve_http_file_fragment(context, wsi)) { /* nonzero for completion or error */
+                       lwsl_info("lws_server_socket_service: libwebsockets_serve_http_file_fragment says to close\n");
                        libwebsocket_close_and_free_session(context, wsi,
                                               LWS_CLOSE_STATUS_NOSTATUS);
+               }
                break;
 
        case LWS_CONNMODE_SERVER_LISTENER:
index e272cd3..d786f4b 100644 (file)
@@ -203,7 +203,7 @@ static int callback_http(struct libwebsocket_context *context,
  */
 
 static void
-dump_handshake_info(struct lws_tokens *lwst)
+dump_handshake_info(struct libwebsocket *wsi)
 {
        int n;
        static const char *token_names[WSI_TOKEN_COUNT] = {
@@ -232,12 +232,15 @@ dump_handshake_info(struct lws_tokens *lwst)
                /*[WSI_TOKEN_HTTP]              =*/ "Http",
                /*[WSI_TOKEN_MUXURL]    =*/ "MuxURL",
        };
+       char buf[256];
 
        for (n = 0; n < WSI_TOKEN_COUNT; n++) {
-               if (lwst[n].token == NULL)
+               if (!lws_hdr_total_length(wsi, n))
                        continue;
 
-               fprintf(stderr, "    %s = %s\n", token_names[n], lwst[n].token);
+               lws_hdr_copy(wsi, buf, sizeof buf, n);
+
+               fprintf(stderr, "    %s = %s\n", token_names[n], buf);
        }
 }
 
@@ -303,7 +306,7 @@ callback_dumb_increment(struct libwebsocket_context *context,
         */
 
        case LWS_CALLBACK_FILTER_PROTOCOL_CONNECTION:
-               dump_handshake_info((struct lws_tokens *)(long)user);
+               dump_handshake_info(wsi);
                /* you could return non-zero here and kill the connection */
                break;
 
@@ -433,7 +436,7 @@ done:
         */
 
        case LWS_CALLBACK_FILTER_PROTOCOL_CONNECTION:
-               dump_handshake_info((struct lws_tokens *)(long)user);
+               dump_handshake_info(wsi);
                /* you could return non-zero here and kill the connection */
                break;