3 /***************************************************************************
5 * Project ___| | | | _ \| |
7 * | (__| |_| | _ <| |___
8 * \___|\___/|_| \_\_____|
10 * Copyright (C) 1998 - 2003, Daniel Stenberg, <daniel@haxx.se>, et al.
12 * This software is licensed as described in the file COPYING, which
13 * you should have received as part of this distribution. The terms
14 * are also available at http://curl.haxx.se/docs/copyright.html.
16 * You may opt to use, copy, modify, merge, publish, distribute and/or sell
17 * copies of the Software, and permit persons to whom the Software is
18 * furnished to do so, under the terms of the COPYING file.
20 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
21 * KIND, either express or implied.
24 ***************************************************************************/
26 /* This file is for lib internal stuff */
31 #define PORT_TELNET 23
32 #define PORT_GOPHER 70
34 #define PORT_HTTPS 443
35 #define PORT_DICT 2628
38 #define DICT_MATCH "/MATCH:"
39 #define DICT_MATCH2 "/M:"
40 #define DICT_MATCH3 "/FIND:"
41 #define DICT_DEFINE "/DEFINE:"
42 #define DICT_DEFINE2 "/D:"
43 #define DICT_DEFINE3 "/LOOKUP:"
45 #define CURL_DEFAULT_USER "anonymous"
46 #define CURL_DEFAULT_PASSWORD "curl_by_daniel@haxx.se"
52 /* SSLeay stuff usually in /usr/local/ssl/include */
54 #include "openssl/rsa.h"
55 #include "openssl/crypto.h"
56 #include "openssl/x509.h"
57 #include "openssl/pem.h"
58 #include "openssl/ssl.h"
59 #include "openssl/err.h"
60 #ifdef HAVE_OPENSSL_ENGINE_H
61 #include <openssl/engine.h>
73 #ifdef HAVE_NETINET_IN_H
74 #include <netinet/in.h>
79 #include <curl/curl.h>
81 #include "http_chunks.h" /* for the structs and enum stuff */
86 #include <zlib.h> /* for content-encoding 08/28/02 jhrg */
89 /* Download buffer size, keep it fairly big for speed reasons */
90 #define BUFSIZE CURL_MAX_WRITE_SIZE
92 /* Initial size of the buffer to store headers in, it'll be enlarged in case
94 #define HEADERSIZE 256
96 /* Maximum number of dirs supported by libcurl in a FTP dir hierarchy */
97 #define CURL_MAX_FTP_DIRDEPTH 100
99 /* Just a convenience macro to get the larger value out of two given */
101 #define MAX(x,y) ((x)>(y)?(x):(y))
105 /* Types needed for krb4-ftp connections */
112 enum protection_level {
120 /* struct for data related to SSL and SSL connections */
121 struct ssl_connect_data {
122 bool use; /* use ssl encrypted communications TRUE/FALSE */
124 /* these ones requires specific SSL-types */
128 #endif /* USE_SSLEAY */
131 struct ssl_config_data {
132 long version; /* what version the client wants to use */
133 long certverifyresult; /* result from the certificate verification */
134 long verifypeer; /* set TRUE if this is desired */
135 long verifyhost; /* 0: no verify
136 1: check that CN exists
137 2: CN must match hostname */
138 char *CApath; /* DOES NOT WORK ON WINDOWS */
139 char *CAfile; /* cerficate to verify peer against */
140 char *random_file; /* path to file containing "random" data */
141 char *egdsocket; /* path to file containing the EGD daemon socket */
142 char *cipher_list; /* list of ciphers to use */
143 long numsessions; /* SSL session id cache size */
146 /* information stored about one single SSL session */
147 struct curl_ssl_session {
148 char *name; /* host name for which this ID was used */
149 void *sessionid; /* as returned from the SSL layer */
150 long age; /* just a number, the higher the more recent */
151 unsigned short remote_port; /* remote port to connect to */
152 struct ssl_config_data ssl_config; /* setup for this session */
156 /****************************************************************************
158 ***************************************************************************/
160 struct FormData *sendit;
164 const char *p_pragma; /* Pragma: string */
165 const char *p_accept; /* Accept: string */
169 /* For FORM posting */
171 struct Curl_chunker chunk;
174 curl_read_callback fread; /* backup storage for fread pointer */
175 void *fread_in; /* backup storage for fread_in pointer */
181 HTTPSEND_NADA, /* init */
182 HTTPSEND_REQUEST, /* sending a request */
183 HTTPSEND_BODY, /* sending body */
184 HTTPSEND_LAST /* never use this */
187 void *send_buffer; /* used if the request couldn't be sent in one chunk,
188 points to an allocated send_buffer struct */
191 /****************************************************************************
193 ***************************************************************************/
196 char *user; /* user name string */
197 char *passwd; /* password string */
198 char *urlpath; /* the originally given path part of the URL */
199 char *dirs[CURL_MAX_FTP_DIRDEPTH]; /* path components */
200 char *file; /* decoded file */
202 char *entrypath; /* the PWD reply when we logged on */
204 char *cache; /* data cache between getresponse()-calls */
205 size_t cache_size; /* size of cache in bytes */
206 bool dont_check; /* Set to TRUE to prevent the final (post-transfer)
207 file size and 226/250 status check. It should still
208 read the line, just ignore the result. */
209 bool no_transfer; /* nothing was transfered, (possibly because a resumed
210 transfer already was complete) */
211 long response_time; /* When no timeout is given, this is the amount of
212 seconds we await for an FTP response. Initialized
213 in Curl_ftp_connect() */
216 /****************************************************************************
218 ***************************************************************************/
220 int fd; /* open file descriptor to read from! */
224 * Boolean values that concerns this connection.
227 bool close; /* if set, we close the connection after this request */
228 bool reuse; /* if set, this is a re-used connection */
229 bool chunk; /* if set, this is a chunked transfer-encoding */
230 bool httpproxy; /* if set, this transfer is done through a http proxy */
231 bool user_passwd; /* do we use user+password for this connection? */
232 bool proxy_user_passwd; /* user+password for the proxy? */
233 bool ipv6_ip; /* we communicate with a remove site specified with pure IPv6
236 bool rangestringalloc; /* the range string is malloc()'ed */
238 bool do_more; /* this is set TRUE if the ->curl_do_more() function is
239 supposed to be called, after ->curl_do() */
241 bool upload_chunky; /* set TRUE if we are doing chunked transfer-encoding
243 bool getheader; /* TRUE if header parsing is wanted */
245 bool forbidchunk; /* used only to explicitly forbid chunk-upload for
246 specific upload buffers. See readmoredata() in
247 http.c for details. */
248 bool tcpconnect; /* the tcp stream (or simimlar) is connected, this
249 is set the first time on the first connect function
254 * This struct is all the previously local variables from Curl_perform() moved
255 * to struct to allow the function to return and get re-invoked better without
259 struct Curl_transfer_keeper {
260 int bytecount; /* total number of bytes read */
261 int writebytecount; /* number of bytes written */
262 long contentlength; /* size of incoming data */
263 struct timeval start; /* transfer started at this time */
264 struct timeval now; /* current time */
265 bool header; /* incoming data has HTTP header */
267 HEADER_NORMAL, /* no bad header at all */
268 HEADER_PARTHEADER, /* part of the chunk is a bad header, the rest is
270 HEADER_ALLBAD /* all was believed to be header */
271 } badheader; /* the header was deemed bad and will be
273 int headerline; /* counts header lines to better track the
275 char *hbufp; /* points at *end* of header line */
277 char *str; /* within buf */
278 char *str_start; /* within buf */
279 char *end_ptr; /* within buf */
280 char *p; /* within headerbuff */
281 bool content_range; /* set TRUE if Content-Range: was found */
282 int offset; /* possible resume offset read from the
283 Content-Range: header */
284 int httpcode; /* error code from the 'HTTP/1.? XXX' line */
285 int httpversion; /* the HTTP version*10 */
286 struct timeval start100; /* time stamp to wait for the 100 code from */
287 bool write_after_100_header; /* TRUE = we enable the write after we
288 received a 100-continue/timeout or
290 bool wait100_after_headers; /* TRUE = after the request-headers have been
291 sent off properly, we go into the wait100
292 state, FALSE = don't */
293 int content_encoding; /* What content encoding. sec 3.5, RFC2616. */
295 #define IDENTITY 0 /* No encoding */
296 #define DEFLATE 1 /* zlib delfate [RFC 1950 & 1951] */
297 #define GZIP 2 /* gzip algorithm [RFC 1952] */
298 #define COMPRESS 3 /* Not handled, added for completeness */
301 bool zlib_init; /* True if zlib already initialized;
302 undefined if Content-Encoding header. */
303 z_stream z; /* State structure for zlib. */
306 /* for the low speed checks: */
315 /* pointers to the actual descriptors we check */
319 /* the file descriptors to play with */
326 bool upload_done; /* set to TRUE when doing chunked transfer-encoding upload
327 and we're uploading the last chunk */
332 * The connectdata struct contains all fields and variables that should be
333 * unique for an entire connection.
336 /**** Fields set when inited and not modified again */
337 struct SessionHandle *data; /* link to the root CURL struct */
338 int connectindex; /* what index in the connects index this particular
341 long protocol; /* PROT_* flags concerning the protocol set */
342 #define PROT_MISSING (1<<0)
343 #define PROT_GOPHER (1<<1)
344 #define PROT_HTTP (1<<2)
345 #define PROT_HTTPS (1<<3)
346 #define PROT_FTP (1<<4)
347 #define PROT_TELNET (1<<5)
348 #define PROT_DICT (1<<6)
349 #define PROT_LDAP (1<<7)
350 #define PROT_FILE (1<<8)
351 #define PROT_FTPS (1<<9)
352 #define PROT_SSL (1<<10) /* protocol requires SSL */
354 /* the particular host we use, in two different ways */
355 struct Curl_dns_entry *connect_addr;
358 struct addrinfo *serv_addr;
360 struct sockaddr_in serv_addr;
362 char protostr[64]; /* store the protocol string in this buffer */
363 char gname[513]; /* store the hostname in this buffer */
364 char *name; /* host name pointer to fool around with */
365 char *path; /* allocated buffer to store the URL's path part in */
366 char *hostname; /* hostname to connect, as parsed from url */
367 long port; /* which port to use locally */
368 unsigned short remote_port; /* what remote port to connect to,
369 not the proxy port! */
372 long headerbytecount; /* only count received headers */
374 char *range; /* range, if used. See README for detailed specification on
376 ssize_t resume_from; /* continue [ftp] transfer from here */
378 char *proxyhost; /* name of the http proxy host */
380 struct timeval now; /* "current" time */
381 struct timeval created; /* creation time */
382 int firstsocket; /* the main socket to use */
383 int secondarysocket; /* for i.e ftp transfers */
384 long maxdownload; /* in bytes, the maximum amount of data to fetch, 0
387 struct ssl_connect_data ssl; /* this is for ssl-stuff */
388 struct ssl_config_data ssl_config;
390 struct ConnectBits bits; /* various state-flags for this connection */
392 /* These two functions MUST be set by the curl_connect() function to be
393 be protocol dependent */
394 CURLcode (*curl_do)(struct connectdata *connect);
395 CURLcode (*curl_done)(struct connectdata *connect);
397 /* If the curl_do() function is better made in two halves, this
398 * curl_do_more() function will be called afterwards, if set. For example
399 * for doing the FTP stuff after the PASV/PORT command.
401 CURLcode (*curl_do_more)(struct connectdata *connect);
403 /* This function *MAY* be set to a protocol-dependent function that is run
404 * after the connect() and everything is done, as a step in the connection.
406 CURLcode (*curl_connect)(struct connectdata *connect);
408 /* This function *MAY* be set to a protocol-dependent function that is run
409 * by the curl_disconnect(), as a step in the disconnection.
411 CURLcode (*curl_disconnect)(struct connectdata *connect);
413 /* This function *MAY* be set to a protocol-dependent function that is run
414 * in the curl_close() function if protocol-specific cleanups are required.
416 CURLcode (*curl_close)(struct connectdata *connect);
418 /**** curl_get() phase fields */
421 int sockfd; /* socket to read from or -1 */
422 int size; /* -1 if unknown at this point */
423 long *bytecountp; /* return number of bytes read or NULL */
426 int writesockfd; /* socket to write to, it may very well be
427 the same we read from. -1 disables */
428 long *writebytecountp; /* return number of bytes written or NULL */
430 /** Dynamicly allocated strings, may need to be freed before this **/
431 /** struct is killed. **/
432 struct dynamically_allocated_data {
433 char *proxyuserpwd; /* free later if not NULL! */
434 char *uagent; /* free later if not NULL! */
435 char *accept_encoding; /* free later if not NULL! 08/28/02 jhrg */
436 char *userpwd; /* free later if not NULL! */
437 char *rangeline; /* free later if not NULL! */
438 char *ref; /* free later if not NULL! */
439 char *cookie; /* free later if not NULL! */
440 char *host; /* free later if not NULL */
441 char *cookiehost; /* free later if not NULL */
444 char *newurl; /* This can only be set if a Location: was in the
448 enum protection_level command_prot;
449 enum protection_level data_prot;
450 enum protection_level request_data_prot;
454 struct krb4buffer in_buffer, out_buffer;
458 struct Curl_sec_client_mech *mech;
459 struct sockaddr_in local_addr;
463 /*************** Request - specific items ************/
464 /* previously this was in the urldata struct */
467 struct HTTP *gopher; /* alias, just for the sake of being more readable */
468 struct HTTP *https; /* alias, just for the sake of being more readable */
471 void *telnet; /* private for telnet.c-eyes only */
472 #if 0 /* no need for special ones for these: */
479 /* This struct is inited when needed */
480 struct Curl_transfer_keeper keep;
482 /* 'upload_present' is used to keep a byte counter of how much data there is
483 still left in the buffer, aimed for upload. */
486 /* 'upload_fromhere' is used as a read-pointer when we uploaded parts of a
487 buffer, so the next read should read from where this pointer points to,
488 and the 'upload_present' contains the number of bytes available at this
490 char *upload_fromhere;
492 curl_read_callback fread; /* function that reads the input */
493 void *fread_in; /* pointer to pass to the fread() above */
496 /* The end of connectdata. 08/27/02 jhrg */
499 * Struct to keep statistical and informational data.
504 long filetime; /* If requested, this is might get set. Set to -1 if
505 the time was unretrievable */
506 long header_size; /* size of read header(s) in bytes */
507 long request_size; /* the amount of bytes sent in the request(s) */
509 char *contenttype; /* the content type of the object */
514 long lastshow; /* time() of the last displayed progress meter or NULL to
515 force redraw at next call */
521 double current_speed; /* uses the currently fastest transfer */
523 bool callback; /* set when progress callback is used */
524 int width; /* screen width at download start */
525 int flags; /* see progress.h */
534 double t_pretransfer;
535 double t_starttransfer;
538 struct timeval start;
539 struct timeval t_startsingle;
540 #define CURR_TIME (5+1) /* 6 entries for 5 seconds */
542 double speeder[ CURR_TIME ];
543 struct timeval speeder_time[ CURR_TIME ];
548 HTTPREQ_NONE, /* first in list */
551 HTTPREQ_POST_FORM, /* we make a difference internally */
554 HTTPREQ_LAST /* last in list */
558 * Values that are generated, temporary or calculated internally for a
559 * "session handle" must be defined within the 'struct urlstate'. This struct
560 * will be used within the SessionHandle struct. When the 'SessionHandle'
561 * struct is cloned, this data MUST NOT be copied.
563 * Remember that any "state" information goes globally for the curl handle.
564 * Session-data MUST be put in the connectdata struct and here. */
565 #define MAX_CURL_USER_LENGTH 256
566 #define MAX_CURL_PASSWORD_LENGTH 256
575 /* buffers to store authentication data in, as parsed from input options */
576 char user[MAX_CURL_USER_LENGTH];
577 char passwd[MAX_CURL_PASSWORD_LENGTH];
578 char proxyuser[MAX_CURL_USER_LENGTH];
579 char proxypasswd[MAX_CURL_PASSWORD_LENGTH];
581 bool passwdgiven; /* set TRUE if an application-provided password has been
584 struct timeval keeps_speed; /* for the progress meter really */
586 /* 'connects' will be an allocated array with pointers. If the pointer is
587 set, it holds an allocated connection. */
588 struct connectdata **connects;
589 long numconnects; /* size of the 'connects' array */
591 char *headerbuff; /* allocated buffer to store headers in */
592 int headersize; /* size of the allocation */
594 char buffer[BUFSIZE+1]; /* download buffer */
595 char uploadbuffer[BUFSIZE+1]; /* upload buffer */
596 double current_speed; /* the ProgressShow() funcion sets this */
598 bool this_is_a_follow; /* this is a followed Location: request */
600 char *auth_host; /* if set, this should be the host name that we will
601 sent authorization to, no else. Used to make Location:
602 following not keep sending user+password... This is
606 struct curl_ssl_session *session; /* array of 'numsessions' size */
607 long sessionage; /* number of the most recent session */
609 char *scratch; /* huge buffer[BUFSIZE*2] when doing upload CRLF replacing */
610 bool errorbuf; /* Set to TRUE if the error buffer is already filled in.
611 This must be set to FALSE every time _easy_perform() is
615 /* storage for the previous bag^H^H^HSIGPIPE signal handler :-) */
616 void (*prev_signal)(int sig);
618 bool allow_port; /* Is set.use_port allowed to take effect or not. This
619 is always set TRUE when curl_easy_perform() is called. */
624 * This 'DynamicStatic' struct defines dynamic states that actually change
625 * values in the 'UserDefined' area, which MUST be taken into consideration
626 * if the UserDefined struct is cloned or similar. You can probably just
627 * copy these, but each one indicate a special action on other data.
630 struct DynamicStatic {
631 char *url; /* work URL, copied from UserDefined */
632 bool url_alloc; /* URL string is malloc()'ed */
633 char *proxy; /* work proxy, copied from UserDefined */
634 bool proxy_alloc; /* http proxy string is malloc()'ed */
635 char *referer; /* referer string */
636 bool referer_alloc; /* referer sting is malloc()ed */
637 struct curl_slist *cookielist; /* list of cookie files set by
638 curl_easy_setopt(COOKIEFILE) calls */
642 * This 'UserDefined' struct must only contain data that is set once to go
643 * for many (perhaps) independent connections. Values that are generated or
644 * calculated internally for the "session handle" MUST be defined within the
645 * 'struct urlstate' instead. The only exceptions MUST note the changes in
646 * the 'DynamicStatic' struct.
650 FILE *err; /* the stderr user data goes here */
651 void *debugdata; /* the data that will be passed to fdebug */
652 char *errorbuffer; /* store failure messages in here */
653 char *proxyuserpwd; /* Proxy <user:password>, if used */
654 long proxyport; /* If non-zero, use this port number by default. If the
655 proxy string features a ":[port]" that one will override
657 void *out; /* the fetched file goes here */
658 void *in; /* the uploaded file is read from here */
659 void *writeheader; /* write the header to this is non-NULL */
660 char *set_url; /* what original URL to work on */
661 char *set_proxy; /* proxy to use */
662 long use_port; /* which port to use (when not using default) */
663 char *userpwd; /* <user:password>, if used */
664 char *set_range; /* range, if used. See README for detailed specification
666 long followlocation; /* as in HTTP Location: */
667 long maxredirs; /* maximum no. of http(s) redirects to follow */
668 char *set_referer; /* custom string */
669 bool free_referer; /* set TRUE if 'referer' points to a string we
671 char *useragent; /* User-Agent string */
672 char *encoding; /* Accept-Encoding string */
673 char *postfields; /* if POST, set the fields' values here */
674 size_t postfieldsize; /* if POST, this might have a size to use instead of
675 strlen(), and then the data *may* be binary (contain
677 char *ftpport; /* port to send with the FTP PORT command */
678 char *device; /* network interface to use */
679 curl_write_callback fwrite; /* function that stores the output */
680 curl_write_callback fwrite_header; /* function that stores headers */
681 curl_read_callback fread; /* function that reads the input */
682 curl_progress_callback fprogress; /* function for progress information */
683 curl_debug_callback fdebug; /* function that write informational data */
684 void *progress_client; /* pointer to pass to the progress callback */
685 curl_passwd_callback fpasswd; /* call for password */
686 void *passwd_client; /* pass to the passwd callback */
687 long timeout; /* in seconds, 0 means no timeout */
688 long connecttimeout; /* in seconds, 0 means no timeout */
689 long infilesize; /* size of file to upload, -1 means unknown */
690 long low_speed_limit; /* bytes/second */
691 long low_speed_time; /* number of seconds */
692 int set_resume_from; /* continue [ftp] transfer from here */
693 char *cookie; /* HTTP cookie string to send */
694 struct curl_slist *headers; /* linked list of extra headers */
695 struct HttpPost *httppost; /* linked list of POST data */
696 char *cert; /* certificate */
697 char *cert_type; /* format for certificate (default: PEM) */
698 char *key; /* private key */
699 char *key_type; /* format for private key (default: PEM) */
700 char *key_passwd; /* plain text private key password */
701 char *crypto_engine; /* name of the crypto engine to use */
702 char *cookiejar; /* dump all cookies to this file */
703 bool cookiesession; /* new cookie session? */
704 bool crlf; /* convert crlf on ftp upload(?) */
705 struct curl_slist *quote; /* after connection is established */
706 struct curl_slist *postquote; /* after the transfer */
707 struct curl_slist *prequote; /* before the transfer, after type (Wesley Laxton)*/
708 struct curl_slist *telnet_options; /* linked list of telnet options */
709 curl_TimeCond timecondition; /* kind of time/date comparison */
710 time_t timevalue; /* what time to compare with */
711 curl_closepolicy closepolicy; /* connection cache close concept */
712 Curl_HttpReq httpreq; /* what kind of HTTP request (if any) is this */
713 char *customrequest; /* HTTP/FTP request to use */
714 long httpversion; /* when non-zero, a specific HTTP version requested to
715 be used in the library's request(s) */
716 char *auth_host; /* if set, this is the allocated string to the host name
717 * to which to send the authorization data to, and no other
718 * host (which location-following otherwise could lead to)
720 char *krb4_level; /* what security level */
721 struct ssl_config_data ssl; /* user defined SSL stuff */
723 curl_proxytype proxytype; /* what kind of proxy that is in use */
725 int dns_cache_timeout; /* DNS cache timeout */
726 long buffer_size; /* size of receive buffer to use */
728 char *private; /* Private data */
730 struct curl_slist *http200aliases; /* linked list of aliases for http200 */
732 /* Here follows boolean settings that define how to behave during
733 this session. They are STATIC, set by libcurl users or at least initially
734 and they don't change during operations. */
737 bool tunnel_thru_httpproxy;
743 bool http_fail_on_error;
744 bool http_follow_location;
745 bool http_disable_hostname_check_before_authentication;
747 #define http_include_header include_header /* former name */
749 bool http_set_referer;
750 bool http_auto_referer; /* set "correct" referer when following location: */
754 enum CURL_NETRC_OPTION
755 use_netrc; /* defined in include/curl.h */
757 bool krb4; /* kerberos4 connection requested */
758 bool reuse_forbid; /* forbidden to be reused, close after use */
759 bool reuse_fresh; /* do not re-use an existing connection */
760 bool expect100header; /* TRUE if we added Expect: 100-continue */
761 bool ftp_use_epsv; /* if EPSV is to be attempted or not */
762 bool no_signal; /* do not use any signal/alarm handler */
764 bool global_dns_cache;
768 * In August 2001, this struct was redesigned and is since stricter than
769 * before. The 'connectdata' struct MUST have all the connection oriented
770 * stuff as we may now have several simultaneous connections and connection
773 * From now on, the 'SessionHandle' must only contain data that is set once to
774 * go for many (perhaps) independent connections. Values that are generated or
775 * calculated internally for the "session handle" must be defined within the
776 * 'struct urlstate' instead. */
778 struct SessionHandle {
779 curl_hash *hostcache;
780 struct Curl_share *share; /* Share, handles global variable mutexing */
781 struct UserDefined set; /* values set by the libcurl user */
782 struct DynamicStatic change; /* possibly modified userdefined data */
784 struct CookieInfo *cookies; /* the cookies, read from files and servers */
785 struct Progress progress; /* for all the progress meter data */
786 struct UrlState state; /* struct for fields used for state info and
787 other dynamic purposes */
788 struct PureInfo info; /* stats, reports and info data */
789 #if defined(USE_SSLEAY) && defined(HAVE_OPENSSL_ENGINE_H)
791 #endif /* USE_SSLEAY */
794 #define LIBCURL_NAME "libcurl"