1 /*****************************************************************************
3 * This example source code introduces a c library buffered I/O interface to
4 * URL reads it supports fopen(), fread(), fgets(), feof(), fclose(),
5 * rewind(). Supported functions have identical prototypes to their normal c
6 * lib namesakes and are preceaded by url_ .
8 * Using this code you can replace your program's fopen() with url_fopen()
9 * and fread() with url_fread() and it become possible to read remote streams
10 * instead of (only) local files. Local files (ie those that can be directly
11 * fopened) will drop back to using the underlying clib implementations
13 * See the main() function at the bottom that shows an app that retrives from a
14 * specified url using fgets() and fread() and saves as two output files.
16 * Copyright (c) 2003 Simtec Electronics
18 * Re-implemented by Vincent Sanders <vince@kyllikki.org> with extensive
19 * reference to original curl example code
21 * Redistribution and use in source and binary forms, with or without
22 * modification, are permitted provided that the following conditions
24 * 1. Redistributions of source code must retain the above copyright
25 * notice, this list of conditions and the following disclaimer.
26 * 2. Redistributions in binary form must reproduce the above copyright
27 * notice, this list of conditions and the following disclaimer in the
28 * documentation and/or other materials provided with the distribution.
29 * 3. The name of the author may not be used to endorse or promote products
30 * derived from this software without specific prior written permission.
32 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
33 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
34 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
35 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
36 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
37 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
38 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
39 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
40 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
41 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
43 * This example requires libcurl 7.9.7 or later.
46 * implements an fopen() abstraction allowing reading from URLs
53 # include <sys/time.h>
58 #include <curl/curl.h>
68 enum fcurl_type_e type; /* type of handle */
72 } handle; /* handle */
74 char *buffer; /* buffer to store cached data*/
75 size_t buffer_len; /* currently allocated buffers length */
76 size_t buffer_pos; /* end of data in buffer*/
77 int still_running; /* Is background url fetch still in progress */
80 typedef struct fcurl_data URL_FILE;
82 /* exported functions */
83 URL_FILE *url_fopen(const char *url, const char *operation);
84 int url_fclose(URL_FILE *file);
85 int url_feof(URL_FILE *file);
86 size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file);
87 char *url_fgets(char *ptr, size_t size, URL_FILE *file);
88 void url_rewind(URL_FILE *file);
90 /* we use a global one for convenience */
93 /* curl calls this routine to get more data */
94 static size_t write_callback(char *buffer,
102 URL_FILE *url = (URL_FILE *)userp;
105 rembuff=url->buffer_len - url->buffer_pos; /* remaining space in buffer */
108 /* not enough space in buffer */
109 newbuff=realloc(url->buffer, url->buffer_len + (size - rembuff));
111 fprintf(stderr, "callback buffer grow failed\n");
115 /* realloc succeeded increase buffer size*/
116 url->buffer_len+=size - rembuff;
121 memcpy(&url->buffer[url->buffer_pos], buffer, size);
122 url->buffer_pos += size;
127 /* use to attempt to fill the read buffer up to requested number of bytes */
128 static int fill_buffer(URL_FILE *file, size_t want)
133 struct timeval timeout;
135 CURLMcode mc; /* curl_multi_fdset() return code */
137 /* only attempt to fill buffer if transactions still running and buffer
138 * doesn't exceed required size already
140 if((!file->still_running) || (file->buffer_pos > want))
143 /* attempt to fill buffer */
146 long curl_timeo = -1;
152 /* set a suitable timeout to fail on */
153 timeout.tv_sec = 60; /* 1 minute */
156 curl_multi_timeout(multi_handle, &curl_timeo);
157 if(curl_timeo >= 0) {
158 timeout.tv_sec = curl_timeo / 1000;
159 if(timeout.tv_sec > 1)
162 timeout.tv_usec = (curl_timeo % 1000) * 1000;
165 /* get file descriptors from the transfers */
166 mc = curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd);
169 fprintf(stderr, "curl_multi_fdset() failed, code %d.\n", mc);
173 /* On success the value of maxfd is guaranteed to be >= -1. We call
174 select(maxfd + 1, ...); specially in case of (maxfd == -1) there are
175 no fds ready yet so we call select(0, ...) --or Sleep() on Windows--
176 to sleep 100ms, which is the minimum suggested value in the
177 curl_multi_fdset() doc. */
184 /* Portable sleep for platforms other than Windows. */
185 struct timeval wait = { 0, 100 * 1000 }; /* 100ms */
186 rc = select(0, NULL, NULL, NULL, &wait);
190 /* Note that on some platforms 'timeout' may be modified by select().
191 If you need access to the original value save a copy beforehand. */
192 rc = select(maxfd+1, &fdread, &fdwrite, &fdexcep, &timeout);
202 /* timeout or readable/writable sockets */
203 curl_multi_perform(multi_handle, &file->still_running);
206 } while(file->still_running && (file->buffer_pos < want));
210 /* use to remove want bytes from the front of a files buffer */
211 static int use_buffer(URL_FILE *file, size_t want)
213 /* sort out buffer */
214 if((file->buffer_pos - want) <=0) {
215 /* ditch buffer - write will recreate */
222 /* move rest down make it available for later */
223 memmove(file->buffer,
225 (file->buffer_pos - want));
227 file->buffer_pos -= want;
232 URL_FILE *url_fopen(const char *url, const char *operation)
234 /* this code could check for URLs or types in the 'url' and
235 basically use the real fopen() for standard files */
240 file = malloc(sizeof(URL_FILE));
244 memset(file, 0, sizeof(URL_FILE));
246 if((file->handle.file=fopen(url, operation)))
247 file->type = CFTYPE_FILE; /* marked as URL */
250 file->type = CFTYPE_CURL; /* marked as URL */
251 file->handle.curl = curl_easy_init();
253 curl_easy_setopt(file->handle.curl, CURLOPT_URL, url);
254 curl_easy_setopt(file->handle.curl, CURLOPT_WRITEDATA, file);
255 curl_easy_setopt(file->handle.curl, CURLOPT_VERBOSE, 0L);
256 curl_easy_setopt(file->handle.curl, CURLOPT_WRITEFUNCTION, write_callback);
259 multi_handle = curl_multi_init();
261 curl_multi_add_handle(multi_handle, file->handle.curl);
263 /* lets start the fetch */
264 curl_multi_perform(multi_handle, &file->still_running);
266 if((file->buffer_pos == 0) && (!file->still_running)) {
267 /* if still_running is 0 now, we should return NULL */
269 /* make sure the easy handle is not in the multi handle anymore */
270 curl_multi_remove_handle(multi_handle, file->handle.curl);
273 curl_easy_cleanup(file->handle.curl);
283 int url_fclose(URL_FILE *file)
285 int ret=0;/* default is good return */
289 ret=fclose(file->handle.file); /* passthrough */
293 /* make sure the easy handle is not in the multi handle anymore */
294 curl_multi_remove_handle(multi_handle, file->handle.curl);
297 curl_easy_cleanup(file->handle.curl);
300 default: /* unknown or supported type - oh dear */
306 free(file->buffer);/* free any allocated buffer space */
312 int url_feof(URL_FILE *file)
318 ret=feof(file->handle.file);
322 if((file->buffer_pos == 0) && (!file->still_running))
326 default: /* unknown or supported type - oh dear */
334 size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file)
340 want=fread(ptr, size, nmemb, file->handle.file);
346 fill_buffer(file, want);
348 /* check if theres data in the buffer - if not fill_buffer()
349 * either errored or EOF */
350 if(!file->buffer_pos)
353 /* ensure only available data is considered */
354 if(file->buffer_pos < want)
355 want = file->buffer_pos;
357 /* xfer data to caller */
358 memcpy(ptr, file->buffer, want);
360 use_buffer(file, want);
362 want = want / size; /* number of items */
365 default: /* unknown or supported type - oh dear */
374 char *url_fgets(char *ptr, size_t size, URL_FILE *file)
376 size_t want = size - 1;/* always need to leave room for zero termination */
381 ptr = fgets(ptr, (int)size, file->handle.file);
385 fill_buffer(file, want);
387 /* check if theres data in the buffer - if not fill either errored or
389 if(!file->buffer_pos)
392 /* ensure only available data is considered */
393 if(file->buffer_pos < want)
394 want = file->buffer_pos;
396 /*buffer contains data */
397 /* look for newline or eof */
398 for(loop=0;loop < want;loop++) {
399 if(file->buffer[loop] == '\n') {
400 want=loop+1;/* include newline */
405 /* xfer data to caller */
406 memcpy(ptr, file->buffer, want);
407 ptr[want]=0;/* allways null terminate */
409 use_buffer(file, want);
413 default: /* unknown or supported type - oh dear */
419 return ptr;/*success */
422 void url_rewind(URL_FILE *file)
426 rewind(file->handle.file); /* passthrough */
430 /* halt transaction */
431 curl_multi_remove_handle(multi_handle, file->handle.curl);
434 curl_multi_add_handle(multi_handle, file->handle.curl);
436 /* ditch buffer - write will recreate - resets stream pos*/
444 default: /* unknown or supported type - oh dear */
449 #define FGETSFILE "fgets.test"
450 #define FREADFILE "fread.test"
451 #define REWINDFILE "rewind.test"
453 /* Small main program to retrive from a url using fgets and fread saving the
454 * output to two test files (note the fgets method will corrupt binary files if
455 * they contain 0 chars */
456 int main(int argc, char *argv[])
466 url="http://192.168.7.3/testfile";/* default to testurl */
468 url=argv[1];/* use passed url */
470 /* copy from url line by line with fgets */
471 outf=fopen(FGETSFILE, "wb+");
473 perror("couldn't open fgets output file\n");
477 handle = url_fopen(url, "r");
479 printf("couldn't url_fopen() %s\n", url);
484 while(!url_feof(handle)) {
485 url_fgets(buffer, sizeof(buffer), handle);
486 fwrite(buffer, 1, strlen(buffer), outf);
494 /* Copy from url with fread */
495 outf=fopen(FREADFILE, "wb+");
497 perror("couldn't open fread output file\n");
501 handle = url_fopen("testfile", "r");
503 printf("couldn't url_fopen() testfile\n");
509 nread = url_fread(buffer, 1, sizeof(buffer), handle);
510 fwrite(buffer, 1, nread, outf);
519 outf=fopen(REWINDFILE, "wb+");
521 perror("couldn't open fread output file\n");
525 handle = url_fopen("testfile", "r");
527 printf("couldn't url_fopen() testfile\n");
532 nread = url_fread(buffer, 1, sizeof(buffer), handle);
533 fwrite(buffer, 1, nread, outf);
537 fwrite(buffer, 1, 1, outf);
539 nread = url_fread(buffer, 1, sizeof(buffer), handle);
540 fwrite(buffer, 1, nread, outf);
546 return 0;/* all done */