/*****************************************************************************
- * _ _ ____ _
- * Project ___| | | | _ \| |
- * / __| | | | |_) | |
- * | (__| |_| | _ <| |___
- * \___|\___/|_| \_\_____|
*
- * $Id$
+ * This example source code introduces a c library buffered I/O interface to
+ * URL reads it supports fopen(), fread(), fgets(), feof(), fclose(),
+ * rewind(). Supported functions have identical prototypes to their normal c
+ * lib namesakes and are preceaded by url_ .
*
- * This example source code introduces an fopen()/fread()/fclose() emulation
- * for URL reads. Using an approach similar to this, you could replace your
- * program's fopen() with this url_fopen() and fread() with url_fread() and
- * it should be possible to read remote streams instead of (only) local files.
+ * Using this code you can replace your program's fopen() with url_fopen()
+ * and fread() with url_fread() and it become possible to read remote streams
+ * instead of (only) local files. Local files (ie those that can be directly
+ * fopened) will drop back to using the underlying clib implementations
*
- * See the main() function at the bottom that shows a tiny app in action.
+ * See the main() function at the bottom that shows an app that retrives from a
+ * specified url using fgets() and fread() and saves as two output files.
*
- * This source code is a proof of concept. It will need further attention to
- * become production-use useful and solid.
+ * Copyright (c) 2003 Simtec Electronics
+ *
+ * Re-implemented by Vincent Sanders <vince@kyllikki.org> with extensive
+ * reference to original curl example code
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products
+ * derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* This example requires libcurl 7.9.7 or later.
*/
+
#include <stdio.h>
#include <string.h>
-#include <sys/time.h>
+#ifndef WIN32
+# include <sys/time.h>
+#endif
#include <stdlib.h>
+#include <errno.h>
#include <curl/curl.h>
-#if (LIBCURL_VERSION_NUM < 0x070907)
-#error "too old libcurl version, get the latest!"
-#endif
+enum fcurl_type_e {
+ CFTYPE_NONE=0,
+ CFTYPE_FILE=1,
+ CFTYPE_CURL=2
+};
-struct data {
- int type;
+struct fcurl_data
+{
+ enum fcurl_type_e type; /* type of handle */
union {
CURL *curl;
FILE *file;
- } handle;
+ } handle; /* handle */
- /* This is the documented biggest possible buffer chunk we can get from
- libcurl in one single callback! */
- char buffer[CURL_MAX_WRITE_SIZE];
-
- char *readptr; /* read from here */
- int bytes; /* bytes available from read pointer */
-
- CURLMcode m; /* stored from a previous url_fread() */
+ char *buffer; /* buffer to store cached data*/
+ size_t buffer_len; /* currently allocated buffers length */
+ size_t buffer_pos; /* end of data in buffer*/
+ int still_running; /* Is background url fetch still in progress */
};
-typedef struct data URL_FILE;
+typedef struct fcurl_data URL_FILE;
+
+/* exported functions */
+URL_FILE *url_fopen(const char *url,const char *operation);
+int url_fclose(URL_FILE *file);
+int url_feof(URL_FILE *file);
+size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file);
+char * url_fgets(char *ptr, size_t size, URL_FILE *file);
+void url_rewind(URL_FILE *file);
/* we use a global one for convenience */
CURLM *multi_handle;
-static
-size_t write_callback(char *buffer,
- size_t size,
- size_t nitems,
- void *userp)
+/* curl calls this routine to get more data */
+static size_t write_callback(char *buffer,
+ size_t size,
+ size_t nitems,
+ void *userp)
{
+ char *newbuff;
+ size_t rembuff;
+
URL_FILE *url = (URL_FILE *)userp;
size *= nitems;
- memcpy(url->readptr, buffer, size);
- url->readptr += size;
- url->bytes += size;
+ rembuff=url->buffer_len - url->buffer_pos; /* remaining space in buffer */
+
+ if(size > rembuff) {
+ /* not enough space in buffer */
+ newbuff=realloc(url->buffer,url->buffer_len + (size - rembuff));
+ if(newbuff==NULL) {
+ fprintf(stderr,"callback buffer grow failed\n");
+ size=rembuff;
+ }
+ else {
+ /* realloc suceeded increase buffer size*/
+ url->buffer_len+=size - rembuff;
+ url->buffer=newbuff;
+ }
+ }
- fprintf(stderr, "callback %d size bytes\n", size);
+ memcpy(&url->buffer[url->buffer_pos], buffer, size);
+ url->buffer_pos += size;
return size;
}
-URL_FILE *url_fopen(char *url, char *operation)
+/* use to attempt to fill the read buffer up to requested number of bytes */
+static int fill_buffer(URL_FILE *file, size_t want)
+{
+ fd_set fdread;
+ fd_set fdwrite;
+ fd_set fdexcep;
+ struct timeval timeout;
+ int rc;
+
+ /* only attempt to fill buffer if transactions still running and buffer
+ * doesnt exceed required size already
+ */
+ if((!file->still_running) || (file->buffer_pos > want))
+ return 0;
+
+ /* attempt to fill buffer */
+ do {
+ int maxfd = -1;
+ long curl_timeo = -1;
+
+ FD_ZERO(&fdread);
+ FD_ZERO(&fdwrite);
+ FD_ZERO(&fdexcep);
+
+ /* set a suitable timeout to fail on */
+ timeout.tv_sec = 60; /* 1 minute */
+ timeout.tv_usec = 0;
+
+ curl_multi_timeout(multi_handle, &curl_timeo);
+ if(curl_timeo >= 0) {
+ timeout.tv_sec = curl_timeo / 1000;
+ if(timeout.tv_sec > 1)
+ timeout.tv_sec = 1;
+ else
+ timeout.tv_usec = (curl_timeo % 1000) * 1000;
+ }
+
+ /* get file descriptors from the transfers */
+ curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd);
+
+ /* In a real-world program you OF COURSE check the return code of the
+ function calls. On success, the value of maxfd is guaranteed to be
+ greater or equal than -1. We call select(maxfd + 1, ...), specially
+ in case of (maxfd == -1), we call select(0, ...), which is basically
+ equal to sleep. */
+
+ rc = select(maxfd+1, &fdread, &fdwrite, &fdexcep, &timeout);
+
+ switch(rc) {
+ case -1:
+ /* select error */
+ break;
+
+ case 0:
+ default:
+ /* timeout or readable/writable sockets */
+ curl_multi_perform(multi_handle, &file->still_running);
+ break;
+ }
+ } while(file->still_running && (file->buffer_pos < want));
+ return 1;
+}
+
+/* use to remove want bytes from the front of a files buffer */
+static int use_buffer(URL_FILE *file,int want)
+{
+ /* sort out buffer */
+ if((file->buffer_pos - want) <=0) {
+ /* ditch buffer - write will recreate */
+ if(file->buffer)
+ free(file->buffer);
+
+ file->buffer=NULL;
+ file->buffer_pos=0;
+ file->buffer_len=0;
+ }
+ else {
+ /* move rest down make it available for later */
+ memmove(file->buffer,
+ &file->buffer[want],
+ (file->buffer_pos - want));
+
+ file->buffer_pos -= want;
+ }
+ return 0;
+}
+
+URL_FILE *url_fopen(const char *url,const char *operation)
{
/* this code could check for URLs or types in the 'url' and
basicly use the real fopen() for standard files */
URL_FILE *file;
- int still_running;
(void)operation;
- file = (URL_FILE *)malloc(sizeof(URL_FILE));
+ file = malloc(sizeof(URL_FILE));
if(!file)
return NULL;
memset(file, 0, sizeof(URL_FILE));
- file->type = 1; /* marked as URL, use 0 for plain file */
- file->handle.curl = curl_easy_init();
+ if((file->handle.file=fopen(url,operation)))
+ file->type = CFTYPE_FILE; /* marked as URL */
+
+ else {
+ file->type = CFTYPE_CURL; /* marked as URL */
+ file->handle.curl = curl_easy_init();
+
+ curl_easy_setopt(file->handle.curl, CURLOPT_URL, url);
+ curl_easy_setopt(file->handle.curl, CURLOPT_WRITEDATA, file);
+ curl_easy_setopt(file->handle.curl, CURLOPT_VERBOSE, 0L);
+ curl_easy_setopt(file->handle.curl, CURLOPT_WRITEFUNCTION, write_callback);
+
+ if(!multi_handle)
+ multi_handle = curl_multi_init();
+
+ curl_multi_add_handle(multi_handle, file->handle.curl);
- curl_easy_setopt(file->handle.curl, CURLOPT_URL, url);
- curl_easy_setopt(file->handle.curl, CURLOPT_FILE, file);
- curl_easy_setopt(file->handle.curl, CURLOPT_VERBOSE, FALSE);
- curl_easy_setopt(file->handle.curl, CURLOPT_WRITEFUNCTION, write_callback);
+ /* lets start the fetch */
+ curl_multi_perform(multi_handle, &file->still_running);
- if(!multi_handle)
- multi_handle = curl_multi_init();
+ if((file->buffer_pos == 0) && (!file->still_running)) {
+ /* if still_running is 0 now, we should return NULL */
- curl_multi_add_handle(multi_handle, file->handle.curl);
+ /* make sure the easy handle is not in the multi handle anymore */
+ curl_multi_remove_handle(multi_handle, file->handle.curl);
- while(CURLM_CALL_MULTI_PERFORM ==
- curl_multi_perform(multi_handle, &still_running));
+ /* cleanup */
+ curl_easy_cleanup(file->handle.curl);
- /* if still_running would be 0 now, we should return NULL */
+ free(file);
+ file = NULL;
+ }
+ }
return file;
}
-void url_fclose(URL_FILE *file)
+int url_fclose(URL_FILE *file)
{
- /* make sure the easy handle is not in the multi handle anymore */
- curl_multi_remove_handle(multi_handle, file->handle.curl);
+ int ret=0;/* default is good return */
- /* cleanup */
- curl_easy_cleanup(file->handle.curl);
-}
+ switch(file->type) {
+ case CFTYPE_FILE:
+ ret=fclose(file->handle.file); /* passthrough */
+ break;
+
+ case CFTYPE_CURL:
+ /* make sure the easy handle is not in the multi handle anymore */
+ curl_multi_remove_handle(multi_handle, file->handle.curl);
+
+ /* cleanup */
+ curl_easy_cleanup(file->handle.curl);
+ break;
+
+ default: /* unknown or supported type - oh dear */
+ ret=EOF;
+ errno=EBADF;
+ break;
+ }
+ if(file->buffer)
+ free(file->buffer);/* free any allocated buffer space */
+ free(file);
+
+ return ret;
+}
+
+int url_feof(URL_FILE *file)
+{
+ int ret=0;
+
+ switch(file->type) {
+ case CFTYPE_FILE:
+ ret=feof(file->handle.file);
+ break;
+
+ case CFTYPE_CURL:
+ if((file->buffer_pos == 0) && (!file->still_running))
+ ret = 1;
+ break;
+
+ default: /* unknown or supported type - oh dear */
+ ret=-1;
+ errno=EBADF;
+ break;
+ }
+ return ret;
+}
size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file)
{
- fd_set fdread;
- fd_set fdwrite;
- fd_set fdexcep;
- int maxfd;
- struct timeval timeout;
- int rc;
- int still_running = 0;
+ size_t want;
- if(!file->bytes) { /* no data available at this point */
+ switch(file->type) {
+ case CFTYPE_FILE:
+ want=fread(ptr,size,nmemb,file->handle.file);
+ break;
- file->readptr = file->buffer; /* reset read pointer */
+ case CFTYPE_CURL:
+ want = nmemb * size;
- if(CURLM_CALL_MULTI_PERFORM == file->m) {
- while(CURLM_CALL_MULTI_PERFORM ==
- curl_multi_perform(multi_handle, &still_running)) {
- if(file->bytes) {
- printf("(fread) WOAH! THis happened!\n");
- break;
- }
- }
- if(!still_running) {
- printf("DONE RUNNING AROUND!\n");
- return 0;
- }
- }
+ fill_buffer(file,want);
- do {
+ /* check if theres data in the buffer - if not fill_buffer()
+ * either errored or EOF */
+ if(!file->buffer_pos)
+ return 0;
- FD_ZERO(&fdread);
- FD_ZERO(&fdwrite);
- FD_ZERO(&fdexcep);
-
- /* set a suitable timeout to fail on */
- timeout.tv_sec = 500; /* 5 minutes */
- timeout.tv_usec = 0;
+ /* ensure only available data is considered */
+ if(file->buffer_pos < want)
+ want = file->buffer_pos;
- /* get file descriptors from the transfers */
- curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd);
+ /* xfer data to caller */
+ memcpy(ptr, file->buffer, want);
- rc = select(maxfd+1, &fdread, &fdwrite, &fdexcep, &timeout);
+ use_buffer(file,want);
- switch(rc) {
- case -1:
- /* select error */
- break;
- case 0:
- break;
- default:
- /* timeout or readable/writable sockets */
- printf("select() returned %d!\n", rc);
- do {
- file->m = curl_multi_perform(multi_handle, &still_running);
-
- if(file->bytes)
- /* we have received data, return that now */
- break;
-
- } while(CURLM_CALL_MULTI_PERFORM == file->m);
-
-
- if(!still_running)
- printf("DONE RUNNING AROUND!\n");
-
+ want = want / size; /* number of items */
+ break;
+
+ default: /* unknown or supported type - oh dear */
+ want=0;
+ errno=EBADF;
+ break;
+
+ }
+ return want;
+}
+
+char *url_fgets(char *ptr, size_t size, URL_FILE *file)
+{
+ size_t want = size - 1;/* always need to leave room for zero termination */
+ size_t loop;
+
+ switch(file->type) {
+ case CFTYPE_FILE:
+ ptr = fgets(ptr,size,file->handle.file);
+ break;
+
+ case CFTYPE_CURL:
+ fill_buffer(file,want);
+
+ /* check if theres data in the buffer - if not fill either errored or
+ * EOF */
+ if(!file->buffer_pos)
+ return NULL;
+
+ /* ensure only available data is considered */
+ if(file->buffer_pos < want)
+ want = file->buffer_pos;
+
+ /*buffer contains data */
+ /* look for newline or eof */
+ for(loop=0;loop < want;loop++) {
+ if(file->buffer[loop] == '\n') {
+ want=loop+1;/* include newline */
break;
}
- } while(still_running && (file->bytes <= 0));
+ }
+
+ /* xfer data to caller */
+ memcpy(ptr, file->buffer, want);
+ ptr[want]=0;/* allways null terminate */
+
+ use_buffer(file,want);
+
+ break;
+
+ default: /* unknown or supported type - oh dear */
+ ptr=NULL;
+ errno=EBADF;
+ break;
}
- else
- printf("(fread) Skip network read\n");
- if(file->bytes) {
- /* data already available, return that */
- int want = size * nmemb;
+ return ptr;/*success */
+}
+
+void url_rewind(URL_FILE *file)
+{
+ switch(file->type) {
+ case CFTYPE_FILE:
+ rewind(file->handle.file); /* passthrough */
+ break;
+
+ case CFTYPE_CURL:
+ /* halt transaction */
+ curl_multi_remove_handle(multi_handle, file->handle.curl);
+
+ /* restart */
+ curl_multi_add_handle(multi_handle, file->handle.curl);
- if(file->bytes < want)
- want = file->bytes;
+ /* ditch buffer - write will recreate - resets stream pos*/
+ if(file->buffer)
+ free(file->buffer);
- memcpy(ptr, file->readptr, want);
- file->readptr += want;
- file->bytes -= want;
+ file->buffer=NULL;
+ file->buffer_pos=0;
+ file->buffer_len=0;
- printf("(fread) return %d bytes\n", want);
+ break;
- return want;
+ default: /* unknown or supported type - oh dear */
+ break;
}
- return 0; /* no data available to return */
}
-
+/* Small main program to retrive from a url using fgets and fread saving the
+ * output to two test files (note the fgets method will corrupt binary files if
+ * they contain 0 chars */
int main(int argc, char *argv[])
{
URL_FILE *handle;
+ FILE *outf;
+
int nread;
char buffer[256];
+ const char *url;
+
+ if(argc < 2)
+ url="http://192.168.7.3/testfile";/* default to testurl */
+ else
+ url=argv[1];/* use passed url */
+
+ /* copy from url line by line with fgets */
+ outf=fopen("fgets.test","w+");
+ if(!outf) {
+ perror("couldn't open fgets output file\n");
+ return 1;
+ }
+
+ handle = url_fopen(url, "r");
+ if(!handle) {
+ printf("couldn't url_fopen() %s\n", url);
+ fclose(outf);
+ return 2;
+ }
- (void)argc;
- (void)argv;
+ while(!url_feof(handle)) {
+ url_fgets(buffer,sizeof(buffer),handle);
+ fwrite(buffer,1,strlen(buffer),outf);
+ }
- handle = url_fopen("http://curl.haxx.se/", "r");
+ url_fclose(handle);
+ fclose(outf);
+
+
+ /* Copy from url with fread */
+ outf=fopen("fread.test","w+");
+ if(!outf) {
+ perror("couldn't open fread output file\n");
+ return 1;
+ }
+
+ handle = url_fopen("testfile", "r");
if(!handle) {
- printf("couldn't url_fopen()\n");
+ printf("couldn't url_fopen() testfile\n");
+ fclose(outf);
+ return 2;
}
do {
- nread = url_fread(buffer, sizeof(buffer), 1, handle);
-
- printf("We got: %d bytes\n", nread);
+ nread = url_fread(buffer, 1,sizeof(buffer), handle);
+ fwrite(buffer,1,nread,outf);
} while(nread);
url_fclose(handle);
- return 0;
+ fclose(outf);
+
+
+ /* Test rewind */
+ outf=fopen("rewind.test","w+");
+ if(!outf) {
+ perror("couldn't open fread output file\n");
+ return 1;
+ }
+
+ handle = url_fopen("testfile", "r");
+ if(!handle) {
+ printf("couldn't url_fopen() testfile\n");
+ fclose(outf);
+ return 2;
+ }
+
+ nread = url_fread(buffer, 1,sizeof(buffer), handle);
+ fwrite(buffer,1,nread,outf);
+ url_rewind(handle);
+
+ buffer[0]='\n';
+ fwrite(buffer,1,1,outf);
+
+ nread = url_fread(buffer, 1,sizeof(buffer), handle);
+ fwrite(buffer,1,nread,outf);
+
+
+ url_fclose(handle);
+
+ fclose(outf);
+
+
+ return 0;/* all done */
}
projects / platform / upstream / curl.git / blobdiff