1 /*****************************************************************************
3 * Project ___| | | | _ \| |
5 * | (__| |_| | _ <| |___
6 * \___|\___/|_| \_\_____|
10 * Example application source code using the multi socket interface to
11 * download many files at once.
13 * Written by Jeff Pohlmeyer
15 Requires libevent and a (POSIX?) system that has mkfifo().
17 This is an adaptation of libcurl's "hipev.c" and libevent's "event-test.c"
20 When running, the program creates the named pipe "hiper.fifo"
22 Whenever there is input into the fifo, the program reads the input as a list
23 of URL's and creates some new easy handles to fetch each URL via the
24 curl_multi "hiper" API.
27 Thus, you can try a single URL:
28 % echo http://www.yahoo.com > hiper.fifo
30 Or a whole bunch of them:
31 % cat my-url-list > hiper.fifo
33 The fifo buffer is handled almost instantly, so you can even add more URL's
34 while the previous requests are still being downloaded.
37 For the sake of simplicity, URL length is limited to 1023 char's !
39 This is purely a demo app, all retrieved data is simply discarded by the write
51 #include <curl/curl.h>
58 #define MSG_OUT stdout /* Send info to stdout, change to stderr if you want */
61 /* Global information, common to all connections */
62 typedef struct _GlobalInfo {
63 struct event fifo_event;
64 struct event timer_event;
72 /* Information associated with a specific easy handle */
73 typedef struct _ConnInfo {
77 char error[CURL_ERROR_SIZE];
81 /* Information associated with a specific socket */
82 typedef struct _SockInfo {
94 /* Update the event timer after curl_multi library calls */
95 static int multi_timer_cb(CURLM *multi, long timeout_ms, GlobalInfo *g)
97 struct timeval timeout;
98 (void)multi; /* unused */
100 timeout.tv_sec = timeout_ms/1000;
101 timeout.tv_usec = (timeout_ms%1000)*1000;
102 fprintf(MSG_OUT, "multi_timer_cb: Setting timeout to %ld ms\n", timeout_ms);
103 evtimer_add(&g->timer_event, &timeout);
107 /* Die if we get a bad CURLMcode somewhere */
108 static void mcode_or_die(const char *where, CURLMcode code)
110 if ( CURLM_OK != code ) {
113 case CURLM_CALL_MULTI_PERFORM: s="CURLM_CALL_MULTI_PERFORM"; break;
114 case CURLM_OK: s="CURLM_OK"; break;
115 case CURLM_BAD_HANDLE: s="CURLM_BAD_HANDLE"; break;
116 case CURLM_BAD_EASY_HANDLE: s="CURLM_BAD_EASY_HANDLE"; break;
117 case CURLM_OUT_OF_MEMORY: s="CURLM_OUT_OF_MEMORY"; break;
118 case CURLM_INTERNAL_ERROR: s="CURLM_INTERNAL_ERROR"; break;
119 case CURLM_UNKNOWN_OPTION: s="CURLM_UNKNOWN_OPTION"; break;
120 case CURLM_LAST: s="CURLM_LAST"; break;
121 default: s="CURLM_unknown";
123 case CURLM_BAD_SOCKET: s="CURLM_BAD_SOCKET";
124 fprintf(MSG_OUT, "ERROR: %s returns %s\n", where, s);
125 /* ignore this error */
128 fprintf(MSG_OUT, "ERROR: %s returns %s\n", where, s);
135 /* Check for completed transfers, and remove their easy handles */
136 static void check_run_count(GlobalInfo *g)
138 if (g->prev_running > g->still_running) {
146 fprintf(MSG_OUT, "REMAINING: %d\n", g->still_running);
148 I am still uncertain whether it is safe to remove an easy handle
149 from inside the curl_multi_info_read loop, so here I will search
150 for completed transfers in the inner "while" loop, and then remove
151 them in the outer "do-while" loop...
155 while ((msg = curl_multi_info_read(g->multi, &msgs_left))) {
156 if (msg->msg == CURLMSG_DONE) {
157 easy=msg->easy_handle;
158 res=msg->data.result;
163 curl_easy_getinfo(easy, CURLINFO_PRIVATE, &conn);
164 curl_easy_getinfo(easy, CURLINFO_EFFECTIVE_URL, &eff_url);
165 fprintf(MSG_OUT, "DONE: %s => (%d) %s\n", eff_url, res, conn->error);
166 curl_multi_remove_handle(g->multi, easy);
168 curl_easy_cleanup(easy);
173 g->prev_running = g->still_running;
178 /* Called by libevent when we get action on a multi socket */
179 static void event_cb(int fd, short kind, void *userp)
181 GlobalInfo *g = (GlobalInfo*) userp;
185 (kind&EV_READ?CURL_CSELECT_IN:0)|
186 (kind&EV_WRITE?CURL_CSELECT_OUT:0);
189 rc = curl_multi_socket_action(g->multi, fd, action, &g->still_running);
190 } while (rc == CURLM_CALL_MULTI_PERFORM);
192 mcode_or_die("event_cb: curl_multi_socket_action", rc);
195 if ( g->still_running <= 0 ) {
196 fprintf(MSG_OUT, "last transfer done, kill timeout\n");
197 if (evtimer_pending(&g->timer_event, NULL)) {
198 evtimer_del(&g->timer_event);
205 /* Called by libevent when our timeout expires */
206 static void timer_cb(int fd, short kind, void *userp)
208 GlobalInfo *g = (GlobalInfo *)userp;
214 rc = curl_multi_socket_action(g->multi,
215 CURL_SOCKET_TIMEOUT, 0, &g->still_running);
216 } while (rc == CURLM_CALL_MULTI_PERFORM);
217 mcode_or_die("timer_cb: curl_multi_socket_action", rc);
223 /* Clean up the SockInfo structure */
224 static void remsock(SockInfo *f)
235 /* Assign information to a SockInfo structure */
236 static void setsock(SockInfo*f, curl_socket_t s, CURL*e, int act, GlobalInfo*g)
239 (act&CURL_POLL_IN?EV_READ:0)|(act&CURL_POLL_OUT?EV_WRITE:0)|EV_PERSIST;
246 event_set(&f->ev, f->sockfd, kind, event_cb, g);
248 event_add(&f->ev, NULL);
253 /* Initialize a new SockInfo structure */
254 static void addsock(curl_socket_t s, CURL *easy, int action, GlobalInfo *g) {
255 SockInfo *fdp = calloc(sizeof(SockInfo), 1);
258 setsock(fdp, s, easy, action, g);
259 curl_multi_assign(g->multi, s, fdp);
262 /* CURLMOPT_SOCKETFUNCTION */
263 static int sock_cb(CURL *e, curl_socket_t s, int what, void *cbp, void *sockp)
265 GlobalInfo *g = (GlobalInfo*) cbp;
266 SockInfo *fdp = (SockInfo*) sockp;
267 const char *whatstr[]={ "none", "IN", "OUT", "INOUT", "REMOVE" };
270 "socket callback: s=%d e=%p what=%s ", s, e, whatstr[what]);
271 if (what == CURL_POLL_REMOVE) {
272 fprintf(MSG_OUT, "\n");
277 fprintf(MSG_OUT, "Adding data: %s\n", whatstr[what]);
278 addsock(s, e, what, g);
282 "Changing action from %s to %s\n",
283 whatstr[fdp->action], whatstr[what]);
284 setsock(fdp, s, e, what, g);
292 /* CURLOPT_WRITEFUNCTION */
293 static size_t write_cb(void *ptr, size_t size, size_t nmemb, void *data)
295 size_t realsize = size * nmemb;
296 ConnInfo *conn = (ConnInfo*) data;
303 /* CURLOPT_PROGRESSFUNCTION */
304 static int prog_cb (void *p, double dltotal, double dlnow, double ult,
307 ConnInfo *conn = (ConnInfo *)p;
311 fprintf(MSG_OUT, "Progress: %s (%g/%g)\n", conn->url, dlnow, dltotal);
316 /* Create a new easy handle, and add it to the global curl_multi */
317 static void new_conn(char *url, GlobalInfo *g )
322 conn = calloc(1, sizeof(ConnInfo));
323 memset(conn, 0, sizeof(ConnInfo));
326 conn->easy = curl_easy_init();
328 fprintf(MSG_OUT, "curl_easy_init() failed, exiting!\n");
332 conn->url = strdup(url);
333 curl_easy_setopt(conn->easy, CURLOPT_URL, conn->url);
334 curl_easy_setopt(conn->easy, CURLOPT_WRITEFUNCTION, write_cb);
335 curl_easy_setopt(conn->easy, CURLOPT_WRITEDATA, &conn);
336 curl_easy_setopt(conn->easy, CURLOPT_VERBOSE, 1L);
337 curl_easy_setopt(conn->easy, CURLOPT_ERRORBUFFER, conn->error);
338 curl_easy_setopt(conn->easy, CURLOPT_PRIVATE, conn);
339 curl_easy_setopt(conn->easy, CURLOPT_NOPROGRESS, 0L);
340 curl_easy_setopt(conn->easy, CURLOPT_PROGRESSFUNCTION, prog_cb);
341 curl_easy_setopt(conn->easy, CURLOPT_PROGRESSDATA, conn);
343 "Adding easy %p to multi %p (%s)\n", conn->easy, g->multi, url);
344 rc =curl_multi_add_handle(g->multi, conn->easy);
345 mcode_or_die("new_conn: curl_multi_add_handle", rc);
347 /* note that the add_handle() will set a time-out to trigger very soon so
348 that the necessary socket_action() call will be called by this app */
351 /* This gets called whenever data is received from the fifo */
352 static void fifo_cb(int fd, short event, void *arg)
357 GlobalInfo *g = (GlobalInfo *)arg;
358 (void)fd; /* unused */
359 (void)event; /* unused */
363 rv=fscanf(g->input, "%1023s%n", s, &n);
366 new_conn(s,arg); /* if we read a URL, go get it! */
368 } while ( rv != EOF);
371 /* Create a named pipe and tell libevent to monitor it */
372 static int init_fifo (GlobalInfo *g)
375 static const char *fifo = "hiper.fifo";
378 fprintf(MSG_OUT, "Creating named pipe \"%s\"\n", fifo);
379 if (lstat (fifo, &st) == 0) {
380 if ((st.st_mode & S_IFMT) == S_IFREG) {
387 if (mkfifo (fifo, 0600) == -1) {
391 sockfd = open(fifo, O_RDWR | O_NONBLOCK, 0);
396 g->input = fdopen(sockfd, "r");
398 fprintf(MSG_OUT, "Now, pipe some URL's into > %s\n", fifo);
399 event_set(&g->fifo_event, sockfd, EV_READ | EV_PERSIST, fifo_cb, g);
400 event_add(&g->fifo_event, NULL);
404 int main(int argc, char **argv)
410 memset(&g, 0, sizeof(GlobalInfo));
413 g.multi = curl_multi_init();
414 evtimer_set(&g.timer_event, timer_cb, &g);
416 /* setup the generic multi interface options we want */
417 curl_multi_setopt(g.multi, CURLMOPT_SOCKETFUNCTION, sock_cb);
418 curl_multi_setopt(g.multi, CURLMOPT_SOCKETDATA, &g);
419 curl_multi_setopt(g.multi, CURLMOPT_TIMERFUNCTION, multi_timer_cb);
420 curl_multi_setopt(g.multi, CURLMOPT_TIMERDATA, &g);
422 /* we don't call any curl_multi_socket*() function yet as we have no handles
426 curl_multi_cleanup(g.multi);