From 7e598ef92e5b9efec01d901966008d95c8cd499f Mon Sep 17 00:00:00 2001 From: Dan Winship Date: Thu, 7 Feb 2008 02:30:00 +0000 Subject: [PATCH] New method that lets the application set a callback function to use to * libsoup/soup-message.c (soup_message_set_chunk_allocator): New method that lets the application set a callback function to use to allocate SoupBuffers for reading into, so as to avoid needing extra copies. * libsoup/soup-message-body.c (soup_buffer_new_with_owner): new, to create a SoupBuffer pointing to memory owned by another object, with a GDestroyNotify to unref/free that object when the SoupBuffer is freed. (soup_buffer_get_owner): Returns the owner of a buffer created with soup_buffer_new_with_owner. (soup_buffer_free, etc): update SoupBuffer code for owned buffers. Suggested by Wouter Cloetens, #513810. * tests/simple-httpd.c (do_get): Use mmap() and soup_buffer_new_with_owner(), as a demo/test. svn path=/trunk/; revision=1075 --- ChangeLog | 20 +++++++ configure.in | 1 + libsoup/soup-message-body.c | 131 +++++++++++++++++++++++++++++++++-------- libsoup/soup-message-body.h | 24 +++++--- libsoup/soup-message-io.c | 42 +++++++++---- libsoup/soup-message-private.h | 4 ++ libsoup/soup-message.c | 81 +++++++++++++++++++++++++ libsoup/soup-message.h | 9 +++ tests/simple-httpd.c | 40 +++++++++++-- 9 files changed, 302 insertions(+), 50 deletions(-) diff --git a/ChangeLog b/ChangeLog index b42981a..d562166 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,5 +1,25 @@ 2008-02-06 Dan Winship + * libsoup/soup-message.c (soup_message_set_chunk_allocator): New + method that lets the application set a callback function to use to + allocate SoupBuffers for reading into, so as to avoid needing + extra copies. + + * libsoup/soup-message-body.c (soup_buffer_new_with_owner): new, + to create a SoupBuffer pointing to memory owned by another object, + with a GDestroyNotify to unref/free that object when the + SoupBuffer is freed. + (soup_buffer_get_owner): Returns the owner of a buffer created + with soup_buffer_new_with_owner. + (soup_buffer_free, etc): update SoupBuffer code for owned buffers. + + Suggested by Wouter Cloetens, #513810. + + * tests/simple-httpd.c (do_get): Use mmap() and + soup_buffer_new_with_owner(), as a demo/test. + +2008-02-06 Dan Winship + * libsoup/soup-date.c (soup_date_to_time_t): clamp the result to the time_t range, and document that. Remove the #ifdef HAVE_TIMEGM branch. diff --git a/configure.in b/configure.in index 8dfc91b..338e0af 100644 --- a/configure.in +++ b/configure.in @@ -109,6 +109,7 @@ dnl ******************* dnl *** Misc checks *** dnl ******************* AC_CHECK_FUNCS(gmtime_r) +AC_CHECK_FUNCS(mmap) dnl ********************************* dnl *** Networking library checks *** diff --git a/libsoup/soup-message-body.c b/libsoup/soup-message-body.c index 60e1271..3fa481c 100644 --- a/libsoup/soup-message-body.c +++ b/libsoup/soup-message-body.c @@ -43,8 +43,17 @@ * * Describes how #SoupBuffer should use the data passed in by the * caller. + * + * See also soup_buffer_new_with_owner(), which allows to you create a + * buffer containing data which is owned by another object. **/ +/* Internal SoupMemoryUse values */ +enum { + SOUP_MEMORY_SUBBUFFER = SOUP_MEMORY_TEMPORARY + 1, + SOUP_MEMORY_OWNED +}; + /** * SoupBuffer: * @data: the data @@ -62,12 +71,8 @@ typedef struct { SoupMemoryUse use; guint refcount; - /* @other is used in subbuffers to store a reference to - * the parent buffer, or in TEMPORARY buffers to store a - * reference to a copy (see soup_buffer_copy()). Either - * way, we hold a ref. - */ - SoupBuffer *other; + gpointer owner; + GDestroyNotify owner_dnotify; } SoupBufferPrivate; /** @@ -86,15 +91,20 @@ soup_buffer_new (SoupMemoryUse use, gconstpointer data, gsize length) SoupBufferPrivate *priv = g_slice_new0 (SoupBufferPrivate); if (use == SOUP_MEMORY_COPY) { - priv->buffer.data = g_memdup (data, length); - priv->use = SOUP_MEMORY_TAKE; - } else { - priv->buffer.data = data; - priv->use = use; + data = g_memdup (data, length); + use = SOUP_MEMORY_TAKE; } + + priv->buffer.data = data; priv->buffer.length = length; + priv->use = use; priv->refcount = 1; + if (use == SOUP_MEMORY_TAKE) { + priv->owner = (gpointer)data; + priv->owner_dnotify = g_free; + } + return (SoupBuffer *)priv; } @@ -116,16 +126,85 @@ soup_buffer_new_subbuffer (SoupBuffer *parent, gsize offset, gsize length) { SoupBufferPrivate *priv; + /* Normally this is just a ref, but if @parent is TEMPORARY, + * it will do an actual copy. + */ + parent = soup_buffer_copy (parent); + priv = g_slice_new0 (SoupBufferPrivate); - priv->other = soup_buffer_copy (parent); - priv->buffer.data = priv->other->data + offset; + priv->buffer.data = parent->data + offset; + priv->buffer.length = length; + priv->use = SOUP_MEMORY_SUBBUFFER; + priv->owner = parent; + priv->owner_dnotify = (GDestroyNotify)soup_buffer_free; + priv->refcount = 1; + + return (SoupBuffer *)priv; +} + +/** + * soup_buffer_new_with_owner: + * @data: data + * @length: length of @data + * @owner: pointer to an object that owns @data + * @owner_dnotify: a function to free/unref @owner when the buffer is + * freed + * + * Creates a new #SoupBuffer containing @length bytes from @data. When + * the #SoupBuffer is freed, it will call @owner_dnotify, passing + * @owner to it. You must ensure that @data will remain valid until + * @owner_dnotify is called. + * + * For example, you could use this to create a buffer containing data + * returned from libxml without needing to do an extra copy: + * + * + * xmlDocDumpMemory (doc, &xmlbody, &len); + * return soup_buffer_new_with_owner (xmlbody, len, xmlbody, + * (GDestroyNotify)xmlFree); + * + * + * In this example, @data and @owner are the same, but in other cases + * they would be different (eg, @owner would be a object, and @data + * would be a pointer to one of the object's fields). + * + * Return value: the new #SoupBuffer. + **/ +SoupBuffer * +soup_buffer_new_with_owner (gconstpointer data, gsize length, + gpointer owner, GDestroyNotify owner_dnotify) +{ + SoupBufferPrivate *priv = g_slice_new0 (SoupBufferPrivate); + + priv->buffer.data = data; priv->buffer.length = length; - priv->use = SOUP_MEMORY_STATIC; + priv->use = SOUP_MEMORY_OWNED; + priv->owner = owner; + priv->owner_dnotify = owner_dnotify; priv->refcount = 1; + return (SoupBuffer *)priv; } /** + * soup_buffer_get_owner: + * @buffer: a #SoupBuffer created with soup_buffer_new_with_owner() + * + * Gets the "owner" object for a buffer created with + * soup_buffer_new_with_owner(). + * + * Return value: the owner pointer + **/ +gpointer +soup_buffer_get_owner (SoupBuffer *buffer) +{ + SoupBufferPrivate *priv = (SoupBufferPrivate *)buffer; + + g_return_val_if_fail (priv->use == SOUP_MEMORY_OWNED, NULL); + return priv->owner; +} + +/** * soup_buffer_copy: * @buffer: a #SoupBuffer * @@ -149,16 +228,20 @@ soup_buffer_copy (SoupBuffer *buffer) return buffer; } - /* For TEMPORARY buffers, we need to do a real copy the - * first time, and then after that, we just keep returning - * the copy. Use priv->other to store the copy. + /* For TEMPORARY buffers, we need to do a real copy the first + * time, and then after that, we just keep returning the copy. + * We store the copy in priv->owner, which is technically + * backwards, but it saves us from having to keep an extra + * pointer in SoupBufferPrivate. */ - if (!priv->other) { - priv->other = soup_buffer_new (SOUP_MEMORY_COPY, - buffer->data, buffer->length); + if (!priv->owner) { + priv->owner = soup_buffer_new (SOUP_MEMORY_COPY, + buffer->data, + buffer->length); + priv->owner_dnotify = (GDestroyNotify)soup_buffer_free; } - return soup_buffer_copy (priv->other); + return soup_buffer_copy (priv->owner); } /** @@ -175,10 +258,8 @@ soup_buffer_free (SoupBuffer *buffer) SoupBufferPrivate *priv = (SoupBufferPrivate *)buffer; if (!--priv->refcount) { - if (priv->use == SOUP_MEMORY_TAKE) - g_free ((gpointer)buffer->data); - if (priv->other) - soup_buffer_free (priv->other); + if (priv->owner_dnotify) + priv->owner_dnotify (priv->owner); g_slice_free (SoupBufferPrivate, priv); } } diff --git a/libsoup/soup-message-body.h b/libsoup/soup-message-body.h index cbb9c6f..5c36a80 100644 --- a/libsoup/soup-message-body.h +++ b/libsoup/soup-message-body.h @@ -25,15 +25,21 @@ typedef struct { GType soup_buffer_get_type (void); #define SOUP_TYPE_BUFFER (soup_buffer_get_type ()) -SoupBuffer *soup_buffer_new (SoupMemoryUse use, - gconstpointer data, - gsize length); -SoupBuffer *soup_buffer_new_subbuffer (SoupBuffer *parent, - gsize offset, - gsize length); - -SoupBuffer *soup_buffer_copy (SoupBuffer *buffer); -void soup_buffer_free (SoupBuffer *buffer); +SoupBuffer *soup_buffer_new (SoupMemoryUse use, + gconstpointer data, + gsize length); +SoupBuffer *soup_buffer_new_subbuffer (SoupBuffer *parent, + gsize offset, + gsize length); + +SoupBuffer *soup_buffer_new_with_owner (gconstpointer data, + gsize length, + gpointer owner, + GDestroyNotify owner_dnotify); +gpointer soup_buffer_get_owner (SoupBuffer *buffer); + +SoupBuffer *soup_buffer_copy (SoupBuffer *buffer); +void soup_buffer_free (SoupBuffer *buffer); typedef struct { const char *data; diff --git a/libsoup/soup-message-io.c b/libsoup/soup-message-io.c index f10d4c3..a10d37d 100644 --- a/libsoup/soup-message-io.c +++ b/libsoup/soup-message-io.c @@ -266,27 +266,39 @@ read_body_chunk (SoupMessage *msg) SoupMessagePrivate *priv = SOUP_MESSAGE_GET_PRIVATE (msg); SoupMessageIOData *io = priv->io_data; SoupSocketIOStatus status; - guchar read_buf[RESPONSE_BLOCK_SIZE]; - guint len = sizeof (read_buf); + guchar *stack_buf = NULL; + gsize len; gboolean read_to_eof = (io->read_encoding == SOUP_ENCODING_EOF); gsize nread; GError *error = NULL; SoupBuffer *buffer; while (read_to_eof || io->read_length > 0) { - if (!read_to_eof) - len = MIN (len, io->read_length); + if (priv->chunk_allocator) { + buffer = priv->chunk_allocator (msg, io->read_length, priv->chunk_allocator_data); + if (!buffer) { + soup_message_io_pause (msg); + return FALSE; + } + } else { + if (!stack_buf) + stack_buf = alloca (RESPONSE_BLOCK_SIZE); + buffer = soup_buffer_new (SOUP_MEMORY_TEMPORARY, + stack_buf, + RESPONSE_BLOCK_SIZE); + } - status = soup_socket_read (io->sock, read_buf, len, - &nread, NULL, &error); + if (read_to_eof) + len = buffer->length; + else + len = MIN (buffer->length, io->read_length); - switch (status) { - case SOUP_SOCKET_OK: - if (!nread) - break; + status = soup_socket_read (io->sock, + (guchar *)buffer->data, len, + &nread, NULL, &error); - buffer = soup_buffer_new (SOUP_MEMORY_TEMPORARY, - read_buf, nread); + if (status == SOUP_SOCKET_OK && nread) { + buffer->length = nread; if (!(priv->msg_flags & SOUP_MESSAGE_OVERWRITE_CHUNKS)) soup_message_body_append_buffer (io->read_body, buffer); @@ -296,6 +308,12 @@ read_body_chunk (SoupMessage *msg) soup_message_got_chunk (msg, buffer); soup_buffer_free (buffer); SOUP_MESSAGE_IO_RETURN_VAL_IF_CANCELLED_OR_PAUSED (FALSE); + continue; + } + + soup_buffer_free (buffer); + switch (status) { + case SOUP_SOCKET_OK: break; case SOUP_SOCKET_EOF: diff --git a/libsoup/soup-message-private.h b/libsoup/soup-message-private.h index 1a22ac6..fd67a3d 100644 --- a/libsoup/soup-message-private.h +++ b/libsoup/soup-message-private.h @@ -22,6 +22,10 @@ typedef struct { gpointer io_data; SoupMessageIOStatus io_status; + SoupChunkAllocator chunk_allocator; + gpointer chunk_allocator_data; + GDestroyNotify chunk_allocator_dnotify; + guint msg_flags; SoupHTTPVersion http_version; diff --git a/libsoup/soup-message.c b/libsoup/soup-message.c index 7797f5f..68bff59 100644 --- a/libsoup/soup-message.c +++ b/libsoup/soup-message.c @@ -117,6 +117,8 @@ finalize (GObject *object) SoupMessagePrivate *priv = SOUP_MESSAGE_GET_PRIVATE (msg); soup_message_io_cleanup (msg); + if (priv->chunk_allocator_dnotify) + priv->chunk_allocator_dnotify (priv->chunk_allocator_data); if (priv->uri) soup_uri_free (priv->uri); @@ -1265,3 +1267,82 @@ soup_message_get_io_status (SoupMessage *msg) return priv->io_status; } + +/** + * SoupChunkAllocator: + * @msg: the #SoupMessage the chunk is being allocated for + * @max_len: the maximum length that will be read, or 0. + * @user_data: the data passed to soup_message_set_chunk_allocator() + * + * The prototype for a chunk allocation callback. This should allocate + * a new #SoupBuffer and return it for the I/O layer to read message + * body data off the network into. + * + * If @max_len is non-0, it indicates the maximum number of bytes that + * could be read, based on what is known about the message size. Note + * that this might be a very large number, and you should not simply + * try to allocate that many bytes blindly. If @max_len is 0, that + * means that libsoup does not know how many bytes remain to be read, + * and the allocator should return a buffer of a size that it finds + * convenient. + * + * If the allocator returns %NULL, the message will be paused. It is + * up to the application to make sure that it gets unpaused when it + * becomes possible to allocate a new buffer. + * + * Return value: the new buffer (or %NULL) + **/ + +/** + * soup_message_set_chunk_allocator: + * @msg: a #SoupMessage + * @allocator: the chunk allocator callback + * @user_data: data to pass to @allocator + * @destroy_notify: destroy notifier to free @user_data when @msg is + * destroyed + * + * Sets an alternate chunk-allocation function to use when reading + * @msg's body. Every time data is available to read, libsoup will + * call @allocator, which should return a #SoupBuffer. (See + * #SoupChunkAllocator for additional details.) Libsoup will then read + * data from the network into that buffer, and update the buffer's + * %length to indicate how much data it read. + * + * Generally, a custom chunk allocator would be used in conjunction + * with %SOUP_MESSAGE_OVERWRITE_CHUNKS and #SoupMessage::got_chunk, as + * part of a strategy to avoid unnecessary copying of data. However, + * you cannot assume that every call to the allocator will be followed + * by a call to your %got_chunk handler; if an I/O error occurs, then + * the buffer will be unreffed without ever having been used. If your + * buffer-allocation strategy requires special cleanup, use + * soup_buffer_new_with_owner() rather than doing the cleanup from the + * %got_chunk handler. + * + * The other thing to remember when using + * %SOUP_MESSAGE_OVERWRITE_CHUNKS is that the buffer passed to the + * %got_chunk handler will be unreffed after the handler returns, just + * as it would be in the non-custom-allocated case. If you want to + * hand the chunk data off to some other part of your program to use + * later, you'll need to ref the #SoupBuffer (or its owner, in the + * soup_buffer_new_with_owner() case) to ensure that the data remains + * valid. + **/ +void +soup_message_set_chunk_allocator (SoupMessage *msg, + SoupChunkAllocator allocator, + gpointer user_data, + GDestroyNotify destroy_notify) +{ + SoupMessagePrivate *priv; + + g_return_if_fail (SOUP_IS_MESSAGE (msg)); + + priv = SOUP_MESSAGE_GET_PRIVATE (msg); + + if (priv->chunk_allocator_dnotify) + priv->chunk_allocator_dnotify (priv->chunk_allocator_data); + + priv->chunk_allocator = allocator; + priv->chunk_allocator_data = user_data; + priv->chunk_allocator_dnotify = destroy_notify; +} diff --git a/libsoup/soup-message.h b/libsoup/soup-message.h index 6f38a31..d6ee5c8 100644 --- a/libsoup/soup-message.h +++ b/libsoup/soup-message.h @@ -132,6 +132,15 @@ void soup_message_set_status_full (SoupMessage *msg, guint status_code, const char *reason_phrase); +/* I/O */ +typedef SoupBuffer * (*SoupChunkAllocator) (SoupMessage *msg, + gsize max_len, + gpointer user_data); + +void soup_message_set_chunk_allocator (SoupMessage *msg, + SoupChunkAllocator allocator, + gpointer user_data, + GDestroyNotify destroy_notify); void soup_message_wrote_informational (SoupMessage *msg); void soup_message_wrote_headers (SoupMessage *msg); diff --git a/tests/simple-httpd.c b/tests/simple-httpd.c index bacd6d3..cf06bee 100644 --- a/tests/simple-httpd.c +++ b/tests/simple-httpd.c @@ -3,6 +3,8 @@ * Copyright (C) 2001-2003, Ximian, Inc. */ +#include "config.h" + #include #include #include @@ -13,10 +15,26 @@ #include #include -#include -#include -#include -#include +#ifdef HAVE_MMAP +#include +#endif + +#include + +#ifdef HAVE_MMAP +struct mapping { + void *start; + size_t length; +}; + +static void +free_mapping (gpointer data) +{ + struct mapping *mapping = data; + munmap (mapping->start, mapping->length); + g_slice_free (struct mapping, mapping); +} +#endif static void do_get (SoupServer *server, SoupMessage *msg, const char *path) @@ -65,6 +83,19 @@ do_get (SoupServer *server, SoupMessage *msg, const char *path) } if (msg->method == SOUP_METHOD_GET) { +#ifdef HAVE_MMAP + struct mapping *mapping = g_slice_new (struct mapping); + SoupBuffer *buffer; + + mapping->start = mmap (NULL, st.st_size, PROT_READ, + MAP_PRIVATE, fd, 0); + mapping->length = st.st_size; + buffer = soup_buffer_new_with_owner (mapping->start, + mapping->length, + mapping, free_mapping); + soup_message_body_append_buffer (msg->response_body, buffer); + soup_buffer_free (buffer); +#else char *buf; buf = g_malloc (st.st_size); @@ -72,6 +103,7 @@ do_get (SoupServer *server, SoupMessage *msg, const char *path) close (fd); soup_message_body_append (msg->response_body, SOUP_MEMORY_TAKE, buf, st.st_size); +#endif } else /* msg->method == SOUP_METHOD_HEAD */ { char *length; -- 2.7.4