3 /***************************************************************************
5 * Project ___| | | | _ \| |
7 * | (__| |_| | _ <| |___
8 * \___|\___/|_| \_\_____|
10 * Copyright (C) 1998 - 2003, Daniel Stenberg, <daniel@haxx.se>, et al.
12 * This software is licensed as described in the file COPYING, which
13 * you should have received as part of this distribution. The terms
14 * are also available at http://curl.haxx.se/docs/copyright.html.
16 * You may opt to use, copy, modify, merge, publish, distribute and/or sell
17 * copies of the Software, and permit persons to whom the Software is
18 * furnished to do so, under the terms of the COPYING file.
20 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
21 * KIND, either express or implied.
24 ***************************************************************************/
26 /* This file is for lib internal stuff */
31 #define PORT_TELNET 23
32 #define PORT_GOPHER 70
34 #define PORT_HTTPS 443
35 #define PORT_DICT 2628
38 #define DICT_MATCH "/MATCH:"
39 #define DICT_MATCH2 "/M:"
40 #define DICT_MATCH3 "/FIND:"
41 #define DICT_DEFINE "/DEFINE:"
42 #define DICT_DEFINE2 "/D:"
43 #define DICT_DEFINE3 "/LOOKUP:"
45 #define CURL_DEFAULT_USER "anonymous"
46 #define CURL_DEFAULT_PASSWORD "curl_by_daniel@haxx.se"
52 /* SSLeay stuff usually in /usr/local/ssl/include */
54 #include "openssl/rsa.h"
55 #include "openssl/crypto.h"
56 #include "openssl/x509.h"
57 #include "openssl/pem.h"
58 #include "openssl/ssl.h"
59 #include "openssl/err.h"
60 #ifdef HAVE_OPENSSL_ENGINE_H
61 #include <openssl/engine.h>
73 #ifdef HAVE_NETINET_IN_H
74 #include <netinet/in.h>
79 #include <curl/curl.h>
81 #include "http_chunks.h" /* for the structs and enum stuff */
86 #include <zlib.h> /* for content-encoding 08/28/02 jhrg */
89 /* Download buffer size, keep it fairly big for speed reasons */
90 #define BUFSIZE CURL_MAX_WRITE_SIZE
92 /* Initial size of the buffer to store headers in, it'll be enlarged in case
94 #define HEADERSIZE 256
96 /* Just a convenience macro to get the larger value out of two given */
98 #define MAX(x,y) ((x)>(y)?(x):(y))
102 /* Types needed for krb4-ftp connections */
109 enum protection_level {
117 /* struct for data related to SSL and SSL connections */
118 struct ssl_connect_data {
119 bool use; /* use ssl encrypted communications TRUE/FALSE */
121 /* these ones requires specific SSL-types */
125 #endif /* USE_SSLEAY */
128 struct ssl_config_data {
129 long version; /* what version the client wants to use */
130 long certverifyresult; /* result from the certificate verification */
131 long verifypeer; /* set TRUE if this is desired */
132 long verifyhost; /* 0: no verify
133 1: check that CN exists
134 2: CN must match hostname */
135 char *CApath; /* DOES NOT WORK ON WINDOWS */
136 char *CAfile; /* cerficate to verify peer against */
137 char *random_file; /* path to file containing "random" data */
138 char *egdsocket; /* path to file containing the EGD daemon socket */
139 char *cipher_list; /* list of ciphers to use */
140 long numsessions; /* SSL session id cache size */
143 /* information stored about one single SSL session */
144 struct curl_ssl_session {
145 char *name; /* host name for which this ID was used */
146 void *sessionid; /* as returned from the SSL layer */
147 long age; /* just a number, the higher the more recent */
148 unsigned short remote_port; /* remote port to connect to */
149 struct ssl_config_data ssl_config; /* setup for this session */
153 /****************************************************************************
155 ***************************************************************************/
157 struct FormData *sendit;
161 const char *p_pragma; /* Pragma: string */
162 const char *p_accept; /* Accept: string */
166 /* For FORM posting */
168 struct Curl_chunker chunk;
171 curl_read_callback fread; /* backup storage for fread pointer */
172 void *fread_in; /* backup storage for fread_in pointer */
178 HTTPSEND_NADA, /* init */
179 HTTPSEND_REQUEST, /* sending a request */
180 HTTPSEND_BODY, /* sending body */
181 HTTPSEND_LAST /* never use this */
184 void *send_buffer; /* used if the request couldn't be sent in one chunk,
185 points to an allocated send_buffer struct */
188 /****************************************************************************
190 ***************************************************************************/
193 char *user; /* user name string */
194 char *passwd; /* password string */
195 char *urlpath; /* the originally given path part of the URL */
196 char *dir; /* decoded directory */
197 char *file; /* decoded file */
199 char *entrypath; /* the PWD reply when we logged on */
201 char *cache; /* data cache between getresponse()-calls */
202 size_t cache_size; /* size of cache in bytes */
203 bool dont_check; /* Set to TRUE to prevent the final (post-transfer)
204 file size and 226/250 status check. It should still
205 read the line, just ignore the result. */
206 bool no_transfer; /* nothing was transfered, (possibly because a resumed
207 transfer already was complete) */
208 long response_time; /* When no timeout is given, this is the amount of
209 seconds we await for an FTP response. Initialized
210 in Curl_ftp_connect() */
213 /****************************************************************************
215 ***************************************************************************/
217 int fd; /* open file descriptor to read from! */
221 * Boolean values that concerns this connection.
224 bool close; /* if set, we close the connection after this request */
225 bool reuse; /* if set, this is a re-used connection */
226 bool chunk; /* if set, this is a chunked transfer-encoding */
227 bool httpproxy; /* if set, this transfer is done through a http proxy */
228 bool user_passwd; /* do we use user+password for this connection? */
229 bool proxy_user_passwd; /* user+password for the proxy? */
230 bool ipv6_ip; /* we communicate with a remove site specified with pure IPv6
233 bool rangestringalloc; /* the range string is malloc()'ed */
235 bool do_more; /* this is set TRUE if the ->curl_do_more() function is
236 supposed to be called, after ->curl_do() */
238 bool upload_chunky; /* set TRUE if we are doing chunked transfer-encoding
240 bool getheader; /* TRUE if header parsing is wanted */
242 bool forbidchunk; /* used only to explicitly forbid chunk-upload for
243 specific upload buffers. See readmoredata() in
244 http.c for details. */
245 bool tcpconnect; /* the tcp stream (or simimlar) is connected, this
246 is set the first time on the first connect function
251 * This struct is all the previously local variables from Curl_perform() moved
252 * to struct to allow the function to return and get re-invoked better without
256 struct Curl_transfer_keeper {
257 int bytecount; /* total number of bytes read */
258 int writebytecount; /* number of bytes written */
259 long contentlength; /* size of incoming data */
260 struct timeval start; /* transfer started at this time */
261 struct timeval now; /* current time */
262 bool header; /* incoming data has HTTP header */
264 HEADER_NORMAL, /* no bad header at all */
265 HEADER_PARTHEADER, /* part of the chunk is a bad header, the rest is
267 HEADER_ALLBAD /* all was believed to be header */
268 } badheader; /* the header was deemed bad and will be
270 int headerline; /* counts header lines to better track the
272 char *hbufp; /* points at *end* of header line */
274 char *str; /* within buf */
275 char *str_start; /* within buf */
276 char *end_ptr; /* within buf */
277 char *p; /* within headerbuff */
278 bool content_range; /* set TRUE if Content-Range: was found */
279 int offset; /* possible resume offset read from the
280 Content-Range: header */
281 int httpcode; /* error code from the 'HTTP/1.? XXX' line */
282 int httpversion; /* the HTTP version*10 */
283 struct timeval start100; /* time stamp to wait for the 100 code from */
284 bool write_after_100_header; /* TRUE = we enable the write after we
285 received a 100-continue/timeout or
287 bool wait100_after_headers; /* TRUE = after the request-headers have been
288 sent off properly, we go into the wait100
289 state, FALSE = don't */
290 int content_encoding; /* What content encoding. sec 3.5, RFC2616. */
292 #define IDENTITY 0 /* No encoding */
293 #define DEFLATE 1 /* zlib delfate [RFC 1950 & 1951] */
294 #define GZIP 2 /* gzip algorithm [RFC 1952] */
295 #define COMPRESS 3 /* Not handled, added for completeness */
298 bool zlib_init; /* True if zlib already initialized;
299 undefined if Content-Encoding header. */
300 z_stream z; /* State structure for zlib. */
303 /* for the low speed checks: */
312 /* pointers to the actual descriptors we check */
316 /* the file descriptors to play with */
323 bool upload_done; /* set to TRUE when doing chunked transfer-encoding upload
324 and we're uploading the last chunk */
329 * The connectdata struct contains all fields and variables that should be
330 * unique for an entire connection.
333 /**** Fields set when inited and not modified again */
334 struct SessionHandle *data; /* link to the root CURL struct */
335 int connectindex; /* what index in the connects index this particular
338 long protocol; /* PROT_* flags concerning the protocol set */
339 #define PROT_MISSING (1<<0)
340 #define PROT_GOPHER (1<<1)
341 #define PROT_HTTP (1<<2)
342 #define PROT_HTTPS (1<<3)
343 #define PROT_FTP (1<<4)
344 #define PROT_TELNET (1<<5)
345 #define PROT_DICT (1<<6)
346 #define PROT_LDAP (1<<7)
347 #define PROT_FILE (1<<8)
348 #define PROT_FTPS (1<<9)
349 #define PROT_SSL (1<<10) /* protocol requires SSL */
351 /* the particular host we use, in two different ways */
352 struct Curl_dns_entry *connect_addr;
355 struct addrinfo *serv_addr;
357 struct sockaddr_in serv_addr;
359 char protostr[64]; /* store the protocol string in this buffer */
360 char gname[513]; /* store the hostname in this buffer */
361 char *name; /* host name pointer to fool around with */
362 char *path; /* allocated buffer to store the URL's path part in */
363 char *hostname; /* hostname to connect, as parsed from url */
364 long port; /* which port to use locally */
365 unsigned short remote_port; /* what remote port to connect to,
366 not the proxy port! */
369 long headerbytecount; /* only count received headers */
371 char *range; /* range, if used. See README for detailed specification on
373 ssize_t resume_from; /* continue [ftp] transfer from here */
375 char *proxyhost; /* name of the http proxy host */
377 struct timeval now; /* "current" time */
378 struct timeval created; /* creation time */
379 int firstsocket; /* the main socket to use */
380 int secondarysocket; /* for i.e ftp transfers */
381 long maxdownload; /* in bytes, the maximum amount of data to fetch, 0
384 struct ssl_connect_data ssl; /* this is for ssl-stuff */
385 struct ssl_config_data ssl_config;
387 struct ConnectBits bits; /* various state-flags for this connection */
389 /* These two functions MUST be set by the curl_connect() function to be
390 be protocol dependent */
391 CURLcode (*curl_do)(struct connectdata *connect);
392 CURLcode (*curl_done)(struct connectdata *connect);
394 /* If the curl_do() function is better made in two halves, this
395 * curl_do_more() function will be called afterwards, if set. For example
396 * for doing the FTP stuff after the PASV/PORT command.
398 CURLcode (*curl_do_more)(struct connectdata *connect);
400 /* This function *MAY* be set to a protocol-dependent function that is run
401 * after the connect() and everything is done, as a step in the connection.
403 CURLcode (*curl_connect)(struct connectdata *connect);
405 /* This function *MAY* be set to a protocol-dependent function that is run
406 * by the curl_disconnect(), as a step in the disconnection.
408 CURLcode (*curl_disconnect)(struct connectdata *connect);
410 /* This function *MAY* be set to a protocol-dependent function that is run
411 * in the curl_close() function if protocol-specific cleanups are required.
413 CURLcode (*curl_close)(struct connectdata *connect);
415 /**** curl_get() phase fields */
418 int sockfd; /* socket to read from or -1 */
419 int size; /* -1 if unknown at this point */
420 long *bytecountp; /* return number of bytes read or NULL */
423 int writesockfd; /* socket to write to, it may very well be
424 the same we read from. -1 disables */
425 long *writebytecountp; /* return number of bytes written or NULL */
427 /** Dynamicly allocated strings, may need to be freed before this **/
428 /** struct is killed. **/
429 struct dynamically_allocated_data {
430 char *proxyuserpwd; /* free later if not NULL! */
431 char *uagent; /* free later if not NULL! */
432 char *accept_encoding; /* free later if not NULL! 08/28/02 jhrg */
433 char *userpwd; /* free later if not NULL! */
434 char *rangeline; /* free later if not NULL! */
435 char *ref; /* free later if not NULL! */
436 char *cookie; /* free later if not NULL! */
437 char *host; /* free later if not NULL */
440 char *newurl; /* This can only be set if a Location: was in the
444 enum protection_level command_prot;
445 enum protection_level data_prot;
446 enum protection_level request_data_prot;
450 struct krb4buffer in_buffer, out_buffer;
454 struct Curl_sec_client_mech *mech;
455 struct sockaddr_in local_addr;
459 /*************** Request - specific items ************/
460 /* previously this was in the urldata struct */
463 struct HTTP *gopher; /* alias, just for the sake of being more readable */
464 struct HTTP *https; /* alias, just for the sake of being more readable */
467 void *telnet; /* private for telnet.c-eyes only */
468 #if 0 /* no need for special ones for these: */
475 /* This struct is inited when needed */
476 struct Curl_transfer_keeper keep;
478 /* 'upload_present' is used to keep a byte counter of how much data there is
479 still left in the buffer, aimed for upload. */
482 /* 'upload_fromhere' is used as a read-pointer when we uploaded parts of a
483 buffer, so the next read should read from where this pointer points to,
484 and the 'upload_present' contains the number of bytes available at this
486 char *upload_fromhere;
488 curl_read_callback fread; /* function that reads the input */
489 void *fread_in; /* pointer to pass to the fread() above */
492 /* The end of connectdata. 08/27/02 jhrg */
495 * Struct to keep statistical and informational data.
500 long filetime; /* If requested, this is might get set. Set to -1 if
501 the time was unretrievable */
502 long header_size; /* size of read header(s) in bytes */
503 long request_size; /* the amount of bytes sent in the request(s) */
505 char *contenttype; /* the content type of the object */
510 long lastshow; /* time() of the last displayed progress meter or NULL to
511 force redraw at next call */
517 double current_speed; /* uses the currently fastest transfer */
519 bool callback; /* set when progress callback is used */
520 int width; /* screen width at download start */
521 int flags; /* see progress.h */
530 double t_pretransfer;
531 double t_starttransfer;
534 struct timeval start;
535 struct timeval t_startsingle;
536 #define CURR_TIME (5+1) /* 6 entries for 5 seconds */
538 double speeder[ CURR_TIME ];
539 struct timeval speeder_time[ CURR_TIME ];
544 HTTPREQ_NONE, /* first in list */
547 HTTPREQ_POST_FORM, /* we make a difference internally */
550 HTTPREQ_LAST /* last in list */
554 * Values that are generated, temporary or calculated internally for a
555 * "session handle" must be defined within the 'struct urlstate'. This struct
556 * will be used within the SessionHandle struct. When the 'SessionHandle'
557 * struct is cloned, this data MUST NOT be copied.
559 * Remember that any "state" information goes globally for the curl handle.
560 * Session-data MUST be put in the connectdata struct and here. */
561 #define MAX_CURL_USER_LENGTH 256
562 #define MAX_CURL_PASSWORD_LENGTH 256
571 /* buffers to store authentication data in, as parsed from input options */
572 char user[MAX_CURL_USER_LENGTH];
573 char passwd[MAX_CURL_PASSWORD_LENGTH];
574 char proxyuser[MAX_CURL_USER_LENGTH];
575 char proxypasswd[MAX_CURL_PASSWORD_LENGTH];
577 bool passwdgiven; /* set TRUE if an application-provided password has been
580 struct timeval keeps_speed; /* for the progress meter really */
582 /* 'connects' will be an allocated array with pointers. If the pointer is
583 set, it holds an allocated connection. */
584 struct connectdata **connects;
585 long numconnects; /* size of the 'connects' array */
587 char *headerbuff; /* allocated buffer to store headers in */
588 int headersize; /* size of the allocation */
590 char buffer[BUFSIZE+1]; /* download buffer */
591 char uploadbuffer[BUFSIZE+1]; /* upload buffer */
592 double current_speed; /* the ProgressShow() funcion sets this */
594 bool this_is_a_follow; /* this is a followed Location: request */
596 char *auth_host; /* if set, this should be the host name that we will
597 sent authorization to, no else. Used to make Location:
598 following not keep sending user+password... This is
602 struct curl_ssl_session *session; /* array of 'numsessions' size */
603 long sessionage; /* number of the most recent session */
605 char *scratch; /* huge buffer[BUFSIZE*2] when doing upload CRLF replacing */
606 bool errorbuf; /* Set to TRUE if the error buffer is already filled in.
607 This must be set to FALSE every time _easy_perform() is
611 /* storage for the previous bag^H^H^HSIGPIPE signal handler :-) */
612 void (*prev_signal)(int sig);
614 bool allow_port; /* Is set.use_port allowed to take effect or not. This
615 is always set TRUE when curl_easy_perform() is called. */
620 * This 'DynamicStatic' struct defines dynamic states that actually change
621 * values in the 'UserDefined' area, which MUST be taken into consideration
622 * if the UserDefined struct is cloned or similar. You can probably just
623 * copy these, but each one indicate a special action on other data.
626 struct DynamicStatic {
627 char *url; /* work URL, copied from UserDefined */
628 bool url_alloc; /* URL string is malloc()'ed */
629 char *proxy; /* work proxy, copied from UserDefined */
630 bool proxy_alloc; /* http proxy string is malloc()'ed */
631 char *referer; /* referer string */
632 bool referer_alloc; /* referer sting is malloc()ed */
633 struct curl_slist *cookielist; /* list of cookie files set by
634 curl_easy_setopt(COOKIEFILE) calls */
638 * This 'UserDefined' struct must only contain data that is set once to go
639 * for many (perhaps) independent connections. Values that are generated or
640 * calculated internally for the "session handle" MUST be defined within the
641 * 'struct urlstate' instead. The only exceptions MUST note the changes in
642 * the 'DynamicStatic' struct.
646 FILE *err; /* the stderr user data goes here */
647 void *debugdata; /* the data that will be passed to fdebug */
648 char *errorbuffer; /* store failure messages in here */
649 char *proxyuserpwd; /* Proxy <user:password>, if used */
650 long proxyport; /* If non-zero, use this port number by default. If the
651 proxy string features a ":[port]" that one will override
653 void *out; /* the fetched file goes here */
654 void *in; /* the uploaded file is read from here */
655 void *writeheader; /* write the header to this is non-NULL */
656 char *set_url; /* what original URL to work on */
657 char *set_proxy; /* proxy to use */
658 long use_port; /* which port to use (when not using default) */
659 char *userpwd; /* <user:password>, if used */
660 char *set_range; /* range, if used. See README for detailed specification
662 long followlocation; /* as in HTTP Location: */
663 long maxredirs; /* maximum no. of http(s) redirects to follow */
664 char *set_referer; /* custom string */
665 bool free_referer; /* set TRUE if 'referer' points to a string we
667 char *useragent; /* User-Agent string */
668 char *encoding; /* Accept-Encoding string */
669 char *postfields; /* if POST, set the fields' values here */
670 size_t postfieldsize; /* if POST, this might have a size to use instead of
671 strlen(), and then the data *may* be binary (contain
673 char *ftpport; /* port to send with the FTP PORT command */
674 char *device; /* network interface to use */
675 curl_write_callback fwrite; /* function that stores the output */
676 curl_write_callback fwrite_header; /* function that stores headers */
677 curl_read_callback fread; /* function that reads the input */
678 curl_progress_callback fprogress; /* function for progress information */
679 curl_debug_callback fdebug; /* function that write informational data */
680 void *progress_client; /* pointer to pass to the progress callback */
681 curl_passwd_callback fpasswd; /* call for password */
682 void *passwd_client; /* pass to the passwd callback */
683 long timeout; /* in seconds, 0 means no timeout */
684 long connecttimeout; /* in seconds, 0 means no timeout */
685 long infilesize; /* size of file to upload, -1 means unknown */
686 long low_speed_limit; /* bytes/second */
687 long low_speed_time; /* number of seconds */
688 int set_resume_from; /* continue [ftp] transfer from here */
689 char *cookie; /* HTTP cookie string to send */
690 struct curl_slist *headers; /* linked list of extra headers */
691 struct HttpPost *httppost; /* linked list of POST data */
692 char *cert; /* certificate */
693 char *cert_type; /* format for certificate (default: PEM) */
694 char *key; /* private key */
695 char *key_type; /* format for private key (default: PEM) */
696 char *key_passwd; /* plain text private key password */
697 char *crypto_engine; /* name of the crypto engine to use */
698 char *cookiejar; /* dump all cookies to this file */
699 bool cookiesession; /* new cookie session? */
700 bool crlf; /* convert crlf on ftp upload(?) */
701 struct curl_slist *quote; /* after connection is established */
702 struct curl_slist *postquote; /* after the transfer */
703 struct curl_slist *prequote; /* before the transfer, after type (Wesley Laxton)*/
704 struct curl_slist *telnet_options; /* linked list of telnet options */
705 curl_TimeCond timecondition; /* kind of time/date comparison */
706 time_t timevalue; /* what time to compare with */
707 curl_closepolicy closepolicy; /* connection cache close concept */
708 Curl_HttpReq httpreq; /* what kind of HTTP request (if any) is this */
709 char *customrequest; /* HTTP/FTP request to use */
710 long httpversion; /* when non-zero, a specific HTTP version requested to
711 be used in the library's request(s) */
712 char *auth_host; /* if set, this is the allocated string to the host name
713 * to which to send the authorization data to, and no other
714 * host (which location-following otherwise could lead to)
716 char *krb4_level; /* what security level */
717 struct ssl_config_data ssl; /* user defined SSL stuff */
719 curl_proxytype proxytype; /* what kind of proxy that is in use */
721 int dns_cache_timeout; /* DNS cache timeout */
722 long buffer_size; /* size of receive buffer to use */
724 char *private; /* Private data */
726 struct curl_slist *http200aliases; /* linked list of aliases for http200 */
728 /* Here follows boolean settings that define how to behave during
729 this session. They are STATIC, set by libcurl users or at least initially
730 and they don't change during operations. */
733 bool tunnel_thru_httpproxy;
739 bool http_fail_on_error;
740 bool http_follow_location;
741 bool http_disable_hostname_check_before_authentication;
743 #define http_include_header include_header /* former name */
745 bool http_set_referer;
746 bool http_auto_referer; /* set "correct" referer when following location: */
750 enum CURL_NETRC_OPTION
751 use_netrc; /* defined in include/curl.h */
753 bool krb4; /* kerberos4 connection requested */
754 bool reuse_forbid; /* forbidden to be reused, close after use */
755 bool reuse_fresh; /* do not re-use an existing connection */
756 bool expect100header; /* TRUE if we added Expect: 100-continue */
757 bool ftp_use_epsv; /* if EPSV is to be attempted or not */
758 bool no_signal; /* do not use any signal/alarm handler */
760 bool global_dns_cache;
764 * In August 2001, this struct was redesigned and is since stricter than
765 * before. The 'connectdata' struct MUST have all the connection oriented
766 * stuff as we may now have several simultaneous connections and connection
769 * From now on, the 'SessionHandle' must only contain data that is set once to
770 * go for many (perhaps) independent connections. Values that are generated or
771 * calculated internally for the "session handle" must be defined within the
772 * 'struct urlstate' instead. */
774 struct SessionHandle {
775 curl_hash *hostcache;
776 struct Curl_share *share; /* Share, handles global variable mutexing */
777 struct UserDefined set; /* values set by the libcurl user */
778 struct DynamicStatic change; /* possibly modified userdefined data */
780 struct CookieInfo *cookies; /* the cookies, read from files and servers */
781 struct Progress progress; /* for all the progress meter data */
782 struct UrlState state; /* struct for fields used for state info and
783 other dynamic purposes */
784 struct PureInfo info; /* stats, reports and info data */
785 #if defined(USE_SSLEAY) && defined(HAVE_OPENSSL_ENGINE_H)
787 #endif /* USE_SSLEAY */
790 #define LIBCURL_NAME "libcurl"