1 /***************************************************************************
3 * Project ___| | | | _ \| |
5 * | (__| |_| | _ <| |___
6 * \___|\___/|_| \_\_____|
8 * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al.
10 * This software is licensed as described in the file COPYING, which
11 * you should have received as part of this distribution. The terms
12 * are also available at https://curl.haxx.se/docs/copyright.html.
14 * You may opt to use, copy, modify, merge, publish, distribute and/or sell
15 * copies of the Software, and permit persons to whom the Software is
16 * furnished to do so, under the terms of the COPYING file.
18 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
19 * KIND, either express or implied.
21 ***************************************************************************/
23 #include "curl_setup.h"
24 #include "strtoofft.h"
26 #ifdef HAVE_NETINET_IN_H
27 #include <netinet/in.h>
32 #ifdef HAVE_ARPA_INET_H
33 #include <arpa/inet.h>
38 #ifdef HAVE_SYS_IOCTL_H
39 #include <sys/ioctl.h>
45 #ifdef HAVE_SYS_PARAM_H
46 #include <sys/param.h>
49 #ifdef HAVE_SYS_SELECT_H
50 #include <sys/select.h>
54 #error "We can't compile without socket() support!"
58 #include <curl/curl.h>
61 #include "content_encoding.h"
65 #include "speedcheck.h"
70 #include "vtls/vtls.h"
74 #include "non-ascii.h"
79 /* The last 3 #include files should be in this order */
80 #include "curl_printf.h"
81 #include "curl_memory.h"
84 #if !defined(CURL_DISABLE_HTTP) || !defined(CURL_DISABLE_SMTP) || \
85 !defined(CURL_DISABLE_IMAP)
87 * checkheaders() checks the linked list of custom headers for a
88 * particular header (prefix). Provide the prefix without colon!
90 * Returns a pointer to the first matching header or NULL if none matched.
92 char *Curl_checkheaders(const struct connectdata *conn,
93 const char *thisheader)
95 struct curl_slist *head;
96 size_t thislen = strlen(thisheader);
97 struct Curl_easy *data = conn->data;
99 for(head = data->set.headers; head; head = head->next) {
100 if(strncasecompare(head->data, thisheader, thislen) &&
101 Curl_headersep(head->data[thislen]) )
110 * This function will call the read callback to fill our buffer with data
113 CURLcode Curl_fillreadbuffer(struct connectdata *conn, int bytes, int *nreadp)
115 struct Curl_easy *data = conn->data;
116 size_t buffersize = (size_t)bytes;
118 #ifdef CURL_DOES_CONVERSIONS
119 bool sending_http_headers = FALSE;
121 if(conn->handler->protocol&(PROTO_FAMILY_HTTP|CURLPROTO_RTSP)) {
122 const struct HTTP *http = data->req.protop;
124 if(http->sending == HTTPSEND_REQUEST)
125 /* We're sending the HTTP request headers, not the data.
126 Remember that so we don't re-translate them into garbage. */
127 sending_http_headers = TRUE;
131 if(data->req.upload_chunky) {
132 /* if chunked Transfer-Encoding */
133 buffersize -= (8 + 2 + 2); /* 32bit hex + CRLF + CRLF */
134 data->req.upload_fromhere += (8 + 2); /* 32bit hex + CRLF */
137 /* this function returns a size_t, so we typecast to int to prevent warnings
138 with picky compilers */
139 Curl_set_in_callback(data, true);
140 nread = (int)data->state.fread_func(data->req.upload_fromhere, 1,
141 buffersize, data->state.in);
142 Curl_set_in_callback(data, false);
144 if(nread == CURL_READFUNC_ABORT) {
145 failf(data, "operation aborted by callback");
147 return CURLE_ABORTED_BY_CALLBACK;
149 if(nread == CURL_READFUNC_PAUSE) {
150 struct SingleRequest *k = &data->req;
152 if(conn->handler->flags & PROTOPT_NONETWORK) {
153 /* protocols that work without network cannot be paused. This is
154 actually only FILE:// just now, and it can't pause since the transfer
155 isn't done using the "normal" procedure. */
156 failf(data, "Read callback asked for PAUSE when not supported!");
157 return CURLE_READ_ERROR;
160 /* CURL_READFUNC_PAUSE pauses read callbacks that feed socket writes */
161 k->keepon |= KEEP_SEND_PAUSE; /* mark socket send as paused */
162 if(data->req.upload_chunky) {
163 /* Back out the preallocation done above */
164 data->req.upload_fromhere -= (8 + 2);
168 return CURLE_OK; /* nothing was read */
170 else if((size_t)nread > buffersize) {
171 /* the read function returned a too large value */
173 failf(data, "read function returned funny value");
174 return CURLE_READ_ERROR;
177 if(!data->req.forbidchunk && data->req.upload_chunky) {
178 /* if chunked Transfer-Encoding
184 /* On non-ASCII platforms the <DATA> may or may not be
185 translated based on set.prefer_ascii while the protocol
186 portion must always be translated to the network encoding.
187 To further complicate matters, line end conversion might be
188 done later on, so we need to prevent CRLFs from becoming
189 CRCRLFs if that's the case. To do this we use bare LFs
190 here, knowing they'll become CRLFs later on.
194 const char *endofline_native;
195 const char *endofline_network;
199 #ifdef CURL_DO_LINEEND_CONV
200 (data->set.prefer_ascii) ||
203 /* \n will become \r\n later on */
204 endofline_native = "\n";
205 endofline_network = "\x0a";
208 endofline_native = "\r\n";
209 endofline_network = "\x0d\x0a";
211 hexlen = snprintf(hexbuffer, sizeof(hexbuffer),
212 "%x%s", nread, endofline_native);
214 /* move buffer pointer */
215 data->req.upload_fromhere -= hexlen;
218 /* copy the prefix to the buffer, leaving out the NUL */
219 memcpy(data->req.upload_fromhere, hexbuffer, hexlen);
221 /* always append ASCII CRLF to the data */
222 memcpy(data->req.upload_fromhere + nread,
224 strlen(endofline_network));
226 #ifdef CURL_DOES_CONVERSIONS
230 if(data->set.prefer_ascii)
231 /* translate the protocol and data */
234 /* just translate the protocol portion */
235 length = (int)strlen(hexbuffer);
236 result = Curl_convert_to_network(data, data->req.upload_fromhere,
238 /* Curl_convert_to_network calls failf if unsuccessful */
242 #endif /* CURL_DOES_CONVERSIONS */
244 if((nread - hexlen) == 0) {
245 /* mark this as done once this chunk is transferred */
246 data->req.upload_done = TRUE;
247 infof(data, "Signaling end of chunked upload via terminating chunk.\n");
250 nread += (int)strlen(endofline_native); /* for the added end of line */
252 #ifdef CURL_DOES_CONVERSIONS
253 else if((data->set.prefer_ascii) && (!sending_http_headers)) {
255 result = Curl_convert_to_network(data, data->req.upload_fromhere, nread);
256 /* Curl_convert_to_network calls failf if unsuccessful */
260 #endif /* CURL_DOES_CONVERSIONS */
269 * Curl_readrewind() rewinds the read stream. This is typically used for HTTP
270 * POST/PUT with multi-pass authentication when a sending was denied and a
271 * resend is necessary.
273 CURLcode Curl_readrewind(struct connectdata *conn)
275 struct Curl_easy *data = conn->data;
276 curl_mimepart *mimepart = &data->set.mimepost;
278 conn->bits.rewindaftersend = FALSE; /* we rewind now */
280 /* explicitly switch off sending data on this connection now since we are
281 about to restart a new transfer and thus we want to avoid inadvertently
282 sending more data on the existing connection until the next transfer
284 data->req.keepon &= ~KEEP_SEND;
286 /* We have sent away data. If not using CURLOPT_POSTFIELDS or
287 CURLOPT_HTTPPOST, call app to rewind
289 if(conn->handler->protocol & PROTO_FAMILY_HTTP) {
290 struct HTTP *http = data->req.protop;
293 mimepart = http->sendit;
295 if(data->set.postfields)
297 else if(data->set.httpreq == HTTPREQ_POST_MIME ||
298 data->set.httpreq == HTTPREQ_POST_FORM) {
299 if(Curl_mime_rewind(mimepart)) {
300 failf(data, "Cannot rewind mime/post data");
301 return CURLE_SEND_FAIL_REWIND;
305 if(data->set.seek_func) {
308 Curl_set_in_callback(data, true);
309 err = (data->set.seek_func)(data->set.seek_client, 0, SEEK_SET);
310 Curl_set_in_callback(data, false);
312 failf(data, "seek callback returned error %d", (int)err);
313 return CURLE_SEND_FAIL_REWIND;
316 else if(data->set.ioctl_func) {
319 Curl_set_in_callback(data, true);
320 err = (data->set.ioctl_func)(data, CURLIOCMD_RESTARTREAD,
321 data->set.ioctl_client);
322 Curl_set_in_callback(data, false);
323 infof(data, "the ioctl callback returned %d\n", (int)err);
326 /* FIXME: convert to a human readable error message */
327 failf(data, "ioctl callback returned error %d", (int)err);
328 return CURLE_SEND_FAIL_REWIND;
332 /* If no CURLOPT_READFUNCTION is used, we know that we operate on a
333 given FILE * stream and we can actually attempt to rewind that
334 ourselves with fseek() */
335 if(data->state.fread_func == (curl_read_callback)fread) {
336 if(-1 != fseek(data->state.in, 0, SEEK_SET))
337 /* successful rewind */
341 /* no callback set or failure above, makes us fail at once */
342 failf(data, "necessary data rewind wasn't possible");
343 return CURLE_SEND_FAIL_REWIND;
349 static int data_pending(const struct connectdata *conn)
351 /* in the case of libssh2, we can never be really sure that we have emptied
352 its internal buffers so we MUST always try until we get EAGAIN back */
353 return conn->handler->protocol&(CURLPROTO_SCP|CURLPROTO_SFTP) ||
354 #if defined(USE_NGHTTP2)
355 Curl_ssl_data_pending(conn, FIRSTSOCKET) ||
356 /* For HTTP/2, we may read up everything including responde body
357 with header fields in Curl_http_readwrite_headers. If no
358 content-length is provided, curl waits for the connection
359 close, which we emulate it using conn->proto.httpc.closed =
360 TRUE. The thing is if we read everything, then http2_recv won't
361 be called and we cannot signal the HTTP/2 stream has closed. As
362 a workaround, we return nonzero here to call http2_recv. */
363 ((conn->handler->protocol&PROTO_FAMILY_HTTP) && conn->httpversion == 20);
365 Curl_ssl_data_pending(conn, FIRSTSOCKET);
369 static void read_rewind(struct connectdata *conn,
372 DEBUGASSERT(conn->read_pos >= thismuch);
374 conn->read_pos -= thismuch;
375 conn->bits.stream_was_rewound = TRUE;
382 show = CURLMIN(conn->buf_len - conn->read_pos, sizeof(buf)-1);
383 if(conn->master_buffer) {
384 memcpy(buf, conn->master_buffer + conn->read_pos, show);
391 DEBUGF(infof(conn->data,
392 "Buffer after stream rewind (read_pos = %zu): [%s]\n",
393 conn->read_pos, buf));
399 * Check to see if CURLOPT_TIMECONDITION was met by comparing the time of the
400 * remote document with the time provided by CURLOPT_TIMEVAL
402 bool Curl_meets_timecondition(struct Curl_easy *data, time_t timeofdoc)
404 if((timeofdoc == 0) || (data->set.timevalue == 0))
407 switch(data->set.timecondition) {
408 case CURL_TIMECOND_IFMODSINCE:
410 if(timeofdoc <= data->set.timevalue) {
412 "The requested document is not new enough\n");
413 data->info.timecond = TRUE;
417 case CURL_TIMECOND_IFUNMODSINCE:
418 if(timeofdoc >= data->set.timevalue) {
420 "The requested document is not old enough\n");
421 data->info.timecond = TRUE;
431 * Go ahead and do a read if we have a readable socket or if
432 * the stream was rewound (in which case we have data in a
435 * return '*comeback' TRUE if we didn't properly drain the socket so this
436 * function should get called again without select() or similar in between!
438 static CURLcode readwrite_data(struct Curl_easy *data,
439 struct connectdata *conn,
440 struct SingleRequest *k,
441 int *didwhat, bool *done,
444 CURLcode result = CURLE_OK;
445 ssize_t nread; /* number of bytes read */
446 size_t excess = 0; /* excess bytes read */
447 bool is_empty_data = FALSE;
448 bool readmore = FALSE; /* used by RTP to signal for more data */
454 /* This is where we loop until we have read everything there is to
455 read or we get a CURLE_AGAIN */
457 size_t buffersize = data->set.buffer_size;
458 size_t bytestoread = buffersize;
461 #if defined(USE_NGHTTP2)
462 /* For HTTP/2, read data without caring about the content
463 length. This is safe because body in HTTP/2 is always
464 segmented thanks to its framing layer. Meanwhile, we have to
465 call Curl_read to ensure that http2_handle_stream_close is
466 called when we read all incoming bytes for a particular
468 !((conn->handler->protocol & PROTO_FAMILY_HTTP) &&
469 conn->httpversion == 20) &&
471 k->size != -1 && !k->header) {
472 /* make sure we don't read "too much" if we can help it since we
473 might be pipelining and then someone else might want to read what
475 curl_off_t totalleft = k->size - k->bytecount;
476 if(totalleft < (curl_off_t)bytestoread)
477 bytestoread = (size_t)totalleft;
481 /* receive data from the network! */
482 result = Curl_read(conn, conn->sockfd, k->buf, bytestoread, &nread);
484 /* read would've blocked */
485 if(CURLE_AGAIN == result)
486 break; /* get out of loop */
492 /* read nothing but since we wanted nothing we consider this an OK
493 situation to proceed from */
494 DEBUGF(infof(data, "readwrite_data: we're done!\n"));
498 if((k->bytecount == 0) && (k->writebytecount == 0)) {
499 Curl_pgrsTime(data, TIMER_STARTTRANSFER);
500 if(k->exp100 > EXP100_SEND_DATA)
501 /* set time stamp to compare with when waiting for the 100 */
502 k->start100 = Curl_now();
505 *didwhat |= KEEP_RECV;
506 /* indicates data of zero size, i.e. empty file */
507 is_empty_data = ((nread == 0) && (k->bodywrites == 0)) ? TRUE : FALSE;
509 /* NUL terminate, allowing string ops to be used */
510 if(0 < nread || is_empty_data) {
513 else if(0 >= nread) {
514 /* if we receive 0 or less here, the server closed the connection
515 and we bail out from this! */
516 DEBUGF(infof(data, "nread <= 0, server closed connection, bailing\n"));
517 k->keepon &= ~KEEP_RECV;
521 /* Default buffer to use when we write the buffer, it may be changed
522 in the flow below before the actual storing is done. */
525 if(conn->handler->readwrite) {
526 result = conn->handler->readwrite(data, conn, &nread, &readmore);
533 #ifndef CURL_DISABLE_HTTP
534 /* Since this is a two-state thing, we check if we are parsing
535 headers at the moment or not. */
537 /* we are in parse-the-header-mode */
538 bool stop_reading = FALSE;
539 result = Curl_http_readwrite_headers(data, conn, &nread, &stop_reading);
543 if(conn->handler->readwrite &&
544 (k->maxdownload <= 0 && nread > 0)) {
545 result = conn->handler->readwrite(data, conn, &nread, &readmore);
553 /* We've stopped dealing with input, get out of the do-while loop */
556 if(Curl_pipeline_wanted(conn->data->multi, CURLPIPE_HTTP1)) {
558 "Rewinding stream by : %zd"
559 " bytes on url %s (zero-length body)\n",
560 nread, data->state.path);
561 read_rewind(conn, (size_t)nread);
565 "Excess found in a non pipelined read:"
567 " url = %s (zero-length body)\n",
568 nread, data->state.path);
575 #endif /* CURL_DISABLE_HTTP */
578 /* This is not an 'else if' since it may be a rest from the header
579 parsing, where the beginning of the buffer is headers and the end
581 if(k->str && !k->header && (nread > 0 || is_empty_data)) {
583 if(data->set.opt_no_body) {
584 /* data arrives although we want none, bail out */
585 streamclose(conn, "ignoring body");
587 return CURLE_WEIRD_SERVER_REPLY;
590 #ifndef CURL_DISABLE_HTTP
591 if(0 == k->bodywrites && !is_empty_data) {
592 /* These checks are only made the first time we are about to
593 write a piece of the body */
594 if(conn->handler->protocol&(PROTO_FAMILY_HTTP|CURLPROTO_RTSP)) {
595 /* HTTP-only checks */
597 if(data->req.newurl) {
598 if(conn->bits.close) {
599 /* Abort after the headers if "follow Location" is set
600 and we're set to close anyway. */
601 k->keepon &= ~KEEP_RECV;
605 /* We have a new url to load, but since we want to be able
606 to re-use this connection properly, we read the full
607 response in "ignore more" */
608 k->ignorebody = TRUE;
609 infof(data, "Ignoring the response-body\n");
611 if(data->state.resume_from && !k->content_range &&
612 (data->set.httpreq == HTTPREQ_GET) &&
615 if(k->size == data->state.resume_from) {
616 /* The resume point is at the end of file, consider this fine
617 even if it doesn't allow resume from here. */
618 infof(data, "The entire document is already downloaded");
619 connclose(conn, "already downloaded");
621 k->keepon &= ~KEEP_RECV;
626 /* we wanted to resume a download, although the server doesn't
627 * seem to support this and we did this with a GET (if it
628 * wasn't a GET we did a POST or PUT resume) */
629 failf(data, "HTTP server doesn't seem to support "
630 "byte ranges. Cannot resume.");
631 return CURLE_RANGE_ERROR;
634 if(data->set.timecondition && !data->state.range) {
635 /* A time condition has been set AND no ranges have been
636 requested. This seems to be what chapter 13.3.4 of
637 RFC 2616 defines to be the correct action for a
640 if(!Curl_meets_timecondition(data, k->timeofdoc)) {
642 /* We're simulating a http 304 from server so we return
643 what should have been returned from the server */
644 data->info.httpcode = 304;
645 infof(data, "Simulate a HTTP 304 response!\n");
646 /* we abort the transfer before it is completed == we ruin the
647 re-use ability. Close the connection */
648 connclose(conn, "Simulated 304 handling");
651 } /* we have a time condition */
653 } /* this is HTTP or RTSP */
654 } /* this is the first time we write a body part */
655 #endif /* CURL_DISABLE_HTTP */
659 /* pass data to the debug function before it gets "dechunked" */
660 if(data->set.verbose) {
662 Curl_debug(data, CURLINFO_DATA_IN, data->state.headerbuff,
663 (size_t)k->hbuflen, conn);
664 if(k->badheader == HEADER_PARTHEADER)
665 Curl_debug(data, CURLINFO_DATA_IN,
666 k->str, (size_t)nread, conn);
669 Curl_debug(data, CURLINFO_DATA_IN,
670 k->str, (size_t)nread, conn);
673 #ifndef CURL_DISABLE_HTTP
676 * Here comes a chunked transfer flying and we need to decode this
677 * properly. While the name says read, this function both reads
678 * and writes away the data. The returned 'nread' holds the number
679 * of actual data it wrote to the client.
683 Curl_httpchunk_read(conn, k->str, nread, &nread);
685 if(CHUNKE_OK < res) {
686 if(CHUNKE_WRITE_ERROR == res) {
687 failf(data, "Failed writing data");
688 return CURLE_WRITE_ERROR;
690 failf(data, "%s in chunked-encoding", Curl_chunked_strerror(res));
691 return CURLE_RECV_ERROR;
693 if(CHUNKE_STOP == res) {
695 /* we're done reading chunks! */
696 k->keepon &= ~KEEP_RECV; /* read no more */
698 /* There are now possibly N number of bytes at the end of the
699 str buffer that weren't written to the client.
701 We DO care about this data if we are pipelining.
702 Push it back to be read on the next pass. */
704 dataleft = conn->chunk.dataleft;
706 infof(conn->data, "Leftovers after chunking: %zu bytes\n",
708 if(Curl_pipeline_wanted(conn->data->multi, CURLPIPE_HTTP1)) {
709 /* only attempt the rewind if we truly are pipelining */
710 infof(conn->data, "Rewinding %zu bytes\n",dataleft);
711 read_rewind(conn, dataleft);
715 /* If it returned OK, we just keep going */
717 #endif /* CURL_DISABLE_HTTP */
719 /* Account for body content stored in the header buffer */
720 if(k->badheader && !k->ignorebody) {
721 DEBUGF(infof(data, "Increasing bytecount by %zu from hbuflen\n",
723 k->bytecount += k->hbuflen;
726 if((-1 != k->maxdownload) &&
727 (k->bytecount + nread >= k->maxdownload)) {
729 excess = (size_t)(k->bytecount + nread - k->maxdownload);
730 if(excess > 0 && !k->ignorebody) {
731 if(Curl_pipeline_wanted(conn->data->multi, CURLPIPE_HTTP1)) {
733 "Rewinding stream by : %zu"
734 " bytes on url %s (size = %" CURL_FORMAT_CURL_OFF_T
735 ", maxdownload = %" CURL_FORMAT_CURL_OFF_T
736 ", bytecount = %" CURL_FORMAT_CURL_OFF_T ", nread = %zd)\n",
737 excess, data->state.path,
738 k->size, k->maxdownload, k->bytecount, nread);
739 read_rewind(conn, excess);
743 "Excess found in a non pipelined read:"
745 ", size = %" CURL_FORMAT_CURL_OFF_T
746 ", maxdownload = %" CURL_FORMAT_CURL_OFF_T
747 ", bytecount = %" CURL_FORMAT_CURL_OFF_T "\n",
748 excess, k->size, k->maxdownload, k->bytecount);
752 nread = (ssize_t) (k->maxdownload - k->bytecount);
753 if(nread < 0) /* this should be unusual */
756 k->keepon &= ~KEEP_RECV; /* we're done reading */
759 k->bytecount += nread;
761 Curl_pgrsSetDownloadCounter(data, k->bytecount);
763 if(!k->chunk && (nread || k->badheader || is_empty_data)) {
764 /* If this is chunky transfer, it was already written */
766 if(k->badheader && !k->ignorebody) {
767 /* we parsed a piece of data wrongly assuming it was a header
768 and now we output it as body instead */
770 /* Don't let excess data pollute body writes */
771 if(k->maxdownload == -1 || (curl_off_t)k->hbuflen <= k->maxdownload)
772 result = Curl_client_write(conn, CLIENTWRITE_BODY,
773 data->state.headerbuff,
776 result = Curl_client_write(conn, CLIENTWRITE_BODY,
777 data->state.headerbuff,
778 (size_t)k->maxdownload);
783 if(k->badheader < HEADER_ALLBAD) {
784 /* This switch handles various content encodings. If there's an
785 error here, be sure to check over the almost identical code
787 Make sure that ALL_CONTENT_ENCODINGS contains all the
788 encodings handled here. */
789 if(conn->data->set.http_ce_skip || !k->writer_stack) {
791 #ifndef CURL_DISABLE_POP3
792 if(conn->handler->protocol & PROTO_FAMILY_POP3)
793 result = Curl_pop3_write(conn, k->str, nread);
795 #endif /* CURL_DISABLE_POP3 */
796 result = Curl_client_write(conn, CLIENTWRITE_BODY, k->str,
801 result = Curl_unencode_write(conn, k->writer_stack, k->str, nread);
803 k->badheader = HEADER_NORMAL; /* taken care of now */
809 } /* if(!header and data to read) */
811 if(conn->handler->readwrite && excess && !conn->bits.stream_was_rewound) {
812 /* Parse the excess data */
815 if(&k->str[excess] > &k->buf[data->set.buffer_size]) {
816 /* the excess amount was too excessive(!), make sure
817 it doesn't read out of buffer */
818 excess = &k->buf[data->set.buffer_size] - k->str;
820 nread = (ssize_t)excess;
822 result = conn->handler->readwrite(data, conn, &nread, &readmore);
827 k->keepon |= KEEP_RECV; /* we're not done reading */
832 /* if we received nothing, the server closed the connection and we
834 k->keepon &= ~KEEP_RECV;
837 } while(data_pending(conn) && maxloops--);
840 /* we mark it as read-again-please */
841 conn->cselect_bits = CURL_CSELECT_IN;
845 if(((k->keepon & (KEEP_RECV|KEEP_SEND)) == KEEP_SEND) &&
847 /* When we've read the entire thing and the close bit is set, the server
848 may now close the connection. If there's now any kind of sending going
849 on from our side, we need to stop that immediately. */
850 infof(data, "we are done reading and this is set to close, stop send\n");
851 k->keepon &= ~KEEP_SEND; /* no writing anymore either */
857 static CURLcode done_sending(struct connectdata *conn,
858 struct SingleRequest *k)
860 k->keepon &= ~KEEP_SEND; /* we're done writing */
862 Curl_http2_done_sending(conn);
864 if(conn->bits.rewindaftersend) {
865 CURLcode result = Curl_readrewind(conn);
874 * Send data to upload to the server, when the socket is writable.
876 static CURLcode readwrite_upload(struct Curl_easy *data,
877 struct connectdata *conn,
881 ssize_t bytes_written;
883 ssize_t nread; /* number of bytes read */
884 bool sending_http_headers = FALSE;
885 struct SingleRequest *k = &data->req;
887 if((k->bytecount == 0) && (k->writebytecount == 0))
888 Curl_pgrsTime(data, TIMER_STARTTRANSFER);
890 *didwhat |= KEEP_SEND;
894 /* only read more data if there's no upload data already
895 present in the upload buffer */
896 if(0 == k->upload_present) {
897 /* init the "upload from here" pointer */
898 k->upload_fromhere = data->state.uploadbuffer;
900 if(!k->upload_done) {
901 /* HTTP pollution, this should be written nicer to become more
902 protocol agnostic. */
904 struct HTTP *http = k->protop;
906 if((k->exp100 == EXP100_SENDING_REQUEST) &&
907 (http->sending == HTTPSEND_BODY)) {
908 /* If this call is to send body data, we must take some action:
909 We have sent off the full HTTP 1.1 request, and we shall now
910 go into the Expect: 100 state and await such a header */
911 k->exp100 = EXP100_AWAITING_CONTINUE; /* wait for the header */
912 k->keepon &= ~KEEP_SEND; /* disable writing */
913 k->start100 = Curl_now(); /* timeout count starts now */
914 *didwhat &= ~KEEP_SEND; /* we didn't write anything actually */
916 /* set a timeout for the multi interface */
917 Curl_expire(data, data->set.expect_100_timeout, EXPIRE_100_TIMEOUT);
921 if(conn->handler->protocol&(PROTO_FAMILY_HTTP|CURLPROTO_RTSP)) {
922 if(http->sending == HTTPSEND_REQUEST)
923 /* We're sending the HTTP request headers, not the data.
924 Remember that so we don't change the line endings. */
925 sending_http_headers = TRUE;
927 sending_http_headers = FALSE;
930 result = Curl_fillreadbuffer(conn, UPLOAD_BUFSIZE, &fillcount);
934 nread = (ssize_t)fillcount;
937 nread = 0; /* we're done uploading/reading */
939 if(!nread && (k->keepon & KEEP_SEND_PAUSE)) {
940 /* this is a paused transfer */
944 result = done_sending(conn, k);
950 /* store number of bytes available for upload */
951 k->upload_present = nread;
953 /* convert LF to CRLF if so asked */
954 if((!sending_http_headers) && (
955 #ifdef CURL_DO_LINEEND_CONV
956 /* always convert if we're FTPing in ASCII mode */
957 (data->set.prefer_ascii) ||
960 /* Do we need to allocate a scratch buffer? */
961 if(!data->state.scratch) {
962 data->state.scratch = malloc(2 * data->set.buffer_size);
963 if(!data->state.scratch) {
964 failf(data, "Failed to alloc scratch buffer!");
966 return CURLE_OUT_OF_MEMORY;
971 * ASCII/EBCDIC Note: This is presumably a text (not binary)
972 * transfer so the data should already be in ASCII.
973 * That means the hex values for ASCII CR (0x0d) & LF (0x0a)
974 * must be used instead of the escape sequences \r & \n.
976 for(i = 0, si = 0; i < nread; i++, si++) {
977 if(k->upload_fromhere[i] == 0x0a) {
978 data->state.scratch[si++] = 0x0d;
979 data->state.scratch[si] = 0x0a;
980 if(!data->set.crlf) {
981 /* we're here only because FTP is in ASCII mode...
982 bump infilesize for the LF we just added */
983 if(data->state.infilesize != -1)
984 data->state.infilesize++;
988 data->state.scratch[si] = k->upload_fromhere[i];
992 /* only perform the special operation if we really did replace
996 /* upload from the new (replaced) buffer instead */
997 k->upload_fromhere = data->state.scratch;
999 /* set the new amount too */
1000 k->upload_present = nread;
1004 #ifndef CURL_DISABLE_SMTP
1005 if(conn->handler->protocol & PROTO_FAMILY_SMTP) {
1006 result = Curl_smtp_escape_eob(conn, nread);
1010 #endif /* CURL_DISABLE_SMTP */
1011 } /* if 0 == k->upload_present */
1013 /* We have a partial buffer left from a previous "round". Use
1014 that instead of reading more data */
1017 /* write to socket (send away data) */
1018 result = Curl_write(conn,
1019 conn->writesockfd, /* socket to send to */
1020 k->upload_fromhere, /* buffer pointer */
1021 k->upload_present, /* buffer size */
1022 &bytes_written); /* actually sent */
1027 if(data->set.verbose)
1028 /* show the data before we change the pointer upload_fromhere */
1029 Curl_debug(data, CURLINFO_DATA_OUT, k->upload_fromhere,
1030 (size_t)bytes_written, conn);
1032 k->writebytecount += bytes_written;
1034 if((!k->upload_chunky || k->forbidchunk) &&
1035 (k->writebytecount == data->state.infilesize)) {
1036 /* we have sent all data we were supposed to */
1037 k->upload_done = TRUE;
1038 infof(data, "We are completely uploaded and fine\n");
1041 if(k->upload_present != bytes_written) {
1042 /* we only wrote a part of the buffer (if anything), deal with it! */
1044 /* store the amount of bytes left in the buffer to write */
1045 k->upload_present -= bytes_written;
1047 /* advance the pointer where to find the buffer when the next send
1049 k->upload_fromhere += bytes_written;
1052 /* we've uploaded that buffer now */
1053 k->upload_fromhere = data->state.uploadbuffer;
1054 k->upload_present = 0; /* no more bytes left */
1056 if(k->upload_done) {
1057 result = done_sending(conn, k);
1063 Curl_pgrsSetUploadCounter(data, k->writebytecount);
1065 } WHILE_FALSE; /* just to break out from! */
1071 * Curl_readwrite() is the low-level function to be called when data is to
1072 * be read and written to/from the connection.
1074 * return '*comeback' TRUE if we didn't properly drain the socket so this
1075 * function should get called again without select() or similar in between!
1077 CURLcode Curl_readwrite(struct connectdata *conn,
1078 struct Curl_easy *data,
1082 struct SingleRequest *k = &data->req;
1086 curl_socket_t fd_read;
1087 curl_socket_t fd_write;
1088 int select_res = conn->cselect_bits;
1090 conn->cselect_bits = 0;
1092 /* only use the proper socket if the *_HOLD bit is not set simultaneously as
1093 then we are in rate limiting state in that transfer direction */
1095 if((k->keepon & KEEP_RECVBITS) == KEEP_RECV)
1096 fd_read = conn->sockfd;
1098 fd_read = CURL_SOCKET_BAD;
1100 if((k->keepon & KEEP_SENDBITS) == KEEP_SEND)
1101 fd_write = conn->writesockfd;
1103 fd_write = CURL_SOCKET_BAD;
1105 if(conn->data->state.drain) {
1106 select_res |= CURL_CSELECT_IN;
1107 DEBUGF(infof(data, "Curl_readwrite: forcibly told to drain data\n"));
1110 if(!select_res) /* Call for select()/poll() only, if read/write/error
1111 status is not known. */
1112 select_res = Curl_socket_check(fd_read, CURL_SOCKET_BAD, fd_write, 0);
1114 if(select_res == CURL_CSELECT_ERR) {
1115 failf(data, "select/poll returned error");
1116 return CURLE_SEND_ERROR;
1119 /* We go ahead and do a read if we have a readable socket or if
1120 the stream was rewound (in which case we have data in a
1122 if((k->keepon & KEEP_RECV) &&
1123 ((select_res & CURL_CSELECT_IN) || conn->bits.stream_was_rewound)) {
1125 result = readwrite_data(data, conn, k, &didwhat, done, comeback);
1130 /* If we still have writing to do, we check if we have a writable socket. */
1131 if((k->keepon & KEEP_SEND) && (select_res & CURL_CSELECT_OUT)) {
1134 result = readwrite_upload(data, conn, &didwhat);
1139 k->now = Curl_now();
1141 /* Update read/write counters */
1143 *k->bytecountp = k->bytecount; /* read count */
1144 if(k->writebytecountp)
1145 *k->writebytecountp = k->writebytecount; /* write count */
1148 /* no read no write, this is a timeout? */
1149 if(k->exp100 == EXP100_AWAITING_CONTINUE) {
1150 /* This should allow some time for the header to arrive, but only a
1151 very short time as otherwise it'll be too much wasted time too
1154 /* Quoting RFC2616, section "8.2.3 Use of the 100 (Continue) Status":
1156 Therefore, when a client sends this header field to an origin server
1157 (possibly via a proxy) from which it has never seen a 100 (Continue)
1158 status, the client SHOULD NOT wait for an indefinite period before
1159 sending the request body.
1163 timediff_t ms = Curl_timediff(k->now, k->start100);
1164 if(ms >= data->set.expect_100_timeout) {
1165 /* we've waited long enough, continue anyway */
1166 k->exp100 = EXP100_SEND_DATA;
1167 k->keepon |= KEEP_SEND;
1168 Curl_expire_done(data, EXPIRE_100_TIMEOUT);
1169 infof(data, "Done waiting for 100-continue\n");
1174 if(Curl_pgrsUpdate(conn))
1175 result = CURLE_ABORTED_BY_CALLBACK;
1177 result = Curl_speedcheck(data, k->now);
1182 if(0 > Curl_timeleft(data, &k->now, FALSE)) {
1184 failf(data, "Operation timed out after %ld milliseconds with %"
1185 CURL_FORMAT_CURL_OFF_T " out of %"
1186 CURL_FORMAT_CURL_OFF_T " bytes received",
1187 Curl_timediff(k->now, data->progress.t_startsingle),
1188 k->bytecount, k->size);
1191 failf(data, "Operation timed out after %ld milliseconds with %"
1192 CURL_FORMAT_CURL_OFF_T " bytes received",
1193 Curl_timediff(k->now, data->progress.t_startsingle),
1196 return CURLE_OPERATION_TIMEDOUT;
1201 * The transfer has been performed. Just make some general checks before
1205 if(!(data->set.opt_no_body) && (k->size != -1) &&
1206 (k->bytecount != k->size) &&
1207 #ifdef CURL_DO_LINEEND_CONV
1208 /* Most FTP servers don't adjust their file SIZE response for CRLFs,
1209 so we'll check to see if the discrepancy can be explained
1210 by the number of CRLFs we've changed to LFs.
1212 (k->bytecount != (k->size + data->state.crlf_conversions)) &&
1213 #endif /* CURL_DO_LINEEND_CONV */
1215 failf(data, "transfer closed with %" CURL_FORMAT_CURL_OFF_T
1216 " bytes remaining to read", k->size - k->bytecount);
1217 return CURLE_PARTIAL_FILE;
1219 if(!(data->set.opt_no_body) && k->chunk &&
1220 (conn->chunk.state != CHUNK_STOP)) {
1222 * In chunked mode, return an error if the connection is closed prior to
1223 * the empty (terminating) chunk is read.
1225 * The condition above used to check for
1226 * conn->proto.http->chunk.datasize != 0 which is true after reading
1227 * *any* chunk, not just the empty chunk.
1230 failf(data, "transfer closed with outstanding read data remaining");
1231 return CURLE_PARTIAL_FILE;
1233 if(Curl_pgrsUpdate(conn))
1234 return CURLE_ABORTED_BY_CALLBACK;
1237 /* Now update the "done" boolean we return */
1238 *done = (0 == (k->keepon&(KEEP_RECV|KEEP_SEND|
1239 KEEP_RECV_PAUSE|KEEP_SEND_PAUSE))) ? TRUE : FALSE;
1245 * Curl_single_getsock() gets called by the multi interface code when the app
1246 * has requested to get the sockets for the current connection. This function
1247 * will then be called once for every connection that the multi interface
1248 * keeps track of. This function will only be called for connections that are
1249 * in the proper state to have this information available.
1251 int Curl_single_getsock(const struct connectdata *conn,
1252 curl_socket_t *sock, /* points to numsocks number
1256 const struct Curl_easy *data = conn->data;
1257 int bitmap = GETSOCK_BLANK;
1258 unsigned sockindex = 0;
1260 if(conn->handler->perform_getsock)
1261 return conn->handler->perform_getsock(conn, sock, numsocks);
1264 /* simple check but we might need two slots */
1265 return GETSOCK_BLANK;
1267 /* don't include HOLD and PAUSE connections */
1268 if((data->req.keepon & KEEP_RECVBITS) == KEEP_RECV) {
1270 DEBUGASSERT(conn->sockfd != CURL_SOCKET_BAD);
1272 bitmap |= GETSOCK_READSOCK(sockindex);
1273 sock[sockindex] = conn->sockfd;
1276 /* don't include HOLD and PAUSE connections */
1277 if((data->req.keepon & KEEP_SENDBITS) == KEEP_SEND) {
1279 if((conn->sockfd != conn->writesockfd) ||
1280 bitmap == GETSOCK_BLANK) {
1281 /* only if they are not the same socket and we have a readable
1282 one, we increase index */
1283 if(bitmap != GETSOCK_BLANK)
1284 sockindex++; /* increase index if we need two entries */
1286 DEBUGASSERT(conn->writesockfd != CURL_SOCKET_BAD);
1288 sock[sockindex] = conn->writesockfd;
1291 bitmap |= GETSOCK_WRITESOCK(sockindex);
1297 /* Curl_init_CONNECT() gets called each time the handle switches to CONNECT
1298 which means this gets called once for each subsequent redirect etc */
1299 void Curl_init_CONNECT(struct Curl_easy *data)
1301 data->state.fread_func = data->set.fread_func_set;
1302 data->state.in = data->set.in_set;
1306 * Curl_pretransfer() is called immediately before a transfer starts, and only
1307 * once for one transfer no matter if it has redirects or do multi-pass
1308 * authentication etc.
1310 CURLcode Curl_pretransfer(struct Curl_easy *data)
1313 if(!data->change.url) {
1314 /* we can't do anything without URL */
1315 failf(data, "No URL set!");
1316 return CURLE_URL_MALFORMAT;
1318 /* since the URL may have been redirected in a previous use of this handle */
1319 if(data->change.url_alloc) {
1320 /* the already set URL is allocated, free it first! */
1321 Curl_safefree(data->change.url);
1322 data->change.url_alloc = FALSE;
1324 data->change.url = data->set.str[STRING_SET_URL];
1326 /* Init the SSL session ID cache here. We do it here since we want to do it
1327 after the *_setopt() calls (that could specify the size of the cache) but
1328 before any transfer takes place. */
1329 result = Curl_ssl_initsessions(data, data->set.general_ssl.max_ssl_sessions);
1333 data->state.wildcardmatch = data->set.wildcard_enabled;
1334 data->set.followlocation = 0; /* reset the location-follow counter */
1335 data->state.this_is_a_follow = FALSE; /* reset this */
1336 data->state.errorbuf = FALSE; /* no error has occurred */
1337 data->state.httpversion = 0; /* don't assume any particular server version */
1339 data->state.authproblem = FALSE;
1340 data->state.authhost.want = data->set.httpauth;
1341 data->state.authproxy.want = data->set.proxyauth;
1342 Curl_safefree(data->info.wouldredirect);
1343 data->info.wouldredirect = NULL;
1345 if(data->set.httpreq == HTTPREQ_PUT)
1346 data->state.infilesize = data->set.filesize;
1348 data->state.infilesize = data->set.postfieldsize;
1349 if(data->set.postfields && (data->state.infilesize == -1))
1350 data->state.infilesize = (curl_off_t)strlen(data->set.postfields);
1353 /* If there is a list of cookie files to read, do it now! */
1354 if(data->change.cookielist)
1355 Curl_cookie_loadfiles(data);
1357 /* If there is a list of host pairs to deal with */
1358 if(data->change.resolve)
1359 result = Curl_loadhostpairs(data);
1362 /* Allow data->set.use_port to set which port to use. This needs to be
1363 * disabled for example when we follow Location: headers to URLs using
1364 * different ports! */
1365 data->state.allow_port = TRUE;
1367 #if defined(HAVE_SIGNAL) && defined(SIGPIPE) && !defined(HAVE_MSG_NOSIGNAL)
1368 /*************************************************************
1369 * Tell signal handler to ignore SIGPIPE
1370 *************************************************************/
1371 if(!data->set.no_signal)
1372 data->state.prev_signal = signal(SIGPIPE, SIG_IGN);
1375 Curl_initinfo(data); /* reset session-specific information "variables" */
1376 Curl_pgrsResetTransferSizes(data);
1377 Curl_pgrsStartNow(data);
1379 if(data->set.timeout)
1380 Curl_expire(data, data->set.timeout, EXPIRE_TIMEOUT);
1382 if(data->set.connecttimeout)
1383 Curl_expire(data, data->set.connecttimeout, EXPIRE_CONNECTTIMEOUT);
1385 /* In case the handle is re-used and an authentication method was picked
1386 in the session we need to make sure we only use the one(s) we now
1387 consider to be fine */
1388 data->state.authhost.picked &= data->state.authhost.want;
1389 data->state.authproxy.picked &= data->state.authproxy.want;
1391 if(data->state.wildcardmatch) {
1392 struct WildcardData *wc = &data->wildcard;
1393 if(wc->state < CURLWC_INIT) {
1394 result = Curl_wildcard_init(wc); /* init wildcard structures */
1396 return CURLE_OUT_OF_MEMORY;
1405 * Curl_posttransfer() is called immediately after a transfer ends
1407 CURLcode Curl_posttransfer(struct Curl_easy *data)
1409 #if defined(HAVE_SIGNAL) && defined(SIGPIPE) && !defined(HAVE_MSG_NOSIGNAL)
1410 /* restore the signal handler for SIGPIPE before we get back */
1411 if(!data->set.no_signal)
1412 signal(SIGPIPE, data->state.prev_signal);
1414 (void)data; /* unused parameter */
1420 #ifndef CURL_DISABLE_HTTP
1422 * Find the separator at the end of the host name, or the '?' in cases like
1423 * http://www.url.com?id=2380
1425 static const char *find_host_sep(const char *url)
1430 /* Find the start of the hostname */
1431 sep = strstr(url, "//");
1437 query = strchr(sep, '?');
1438 sep = strchr(sep, '/');
1441 sep = url + strlen(url);
1444 query = url + strlen(url);
1446 return sep < query ? sep : query;
1450 * strlen_url() returns the length of the given URL if the spaces within the
1451 * URL were properly URL encoded.
1452 * URL encoding should be skipped for host names, otherwise IDN resolution
1455 static size_t strlen_url(const char *url, bool relative)
1457 const unsigned char *ptr;
1459 bool left = TRUE; /* left side of the ? */
1460 const unsigned char *host_sep = (const unsigned char *) url;
1463 host_sep = (const unsigned char *) find_host_sep(url);
1465 for(ptr = (unsigned char *)url; *ptr; ptr++) {
1467 if(ptr < host_sep) {
1492 /* strcpy_url() copies a url to a output buffer and URL-encodes the spaces in
1493 * the source URL accordingly.
1494 * URL encoding should be skipped for host names, otherwise IDN resolution
1497 static void strcpy_url(char *output, const char *url, bool relative)
1499 /* we must add this with whitespace-replacing */
1501 const unsigned char *iptr;
1502 char *optr = output;
1503 const unsigned char *host_sep = (const unsigned char *) url;
1506 host_sep = (const unsigned char *) find_host_sep(url);
1508 for(iptr = (unsigned char *)url; /* read from here */
1509 *iptr; /* until zero byte */
1512 if(iptr < host_sep) {
1523 snprintf(optr, 4, "%%%02x", *iptr);
1531 *optr++='%'; /* add a '%' */
1532 *optr++='2'; /* add a '2' */
1533 *optr++='0'; /* add a '0' */
1536 *optr++='+'; /* add a '+' here */
1540 *optr = 0; /* zero terminate output buffer */
1545 * Returns true if the given URL is absolute (as opposed to relative)
1547 static bool is_absolute_url(const char *url)
1549 char prot[16]; /* URL protocol string storage */
1550 char letter; /* used for a silly sscanf */
1552 return (2 == sscanf(url, "%15[^?&/:]://%c", prot, &letter)) ? TRUE : FALSE;
1556 * Concatenate a relative URL to a base URL making it absolute.
1557 * URL-encodes any spaces.
1558 * The returned pointer must be freed by the caller unless NULL
1559 * (returns NULL on out of memory).
1561 static char *concat_url(const char *base, const char *relurl)
1564 TRY to append this new path to the old URL
1565 to the right of the host part. Oh crap, this is doomed to cause
1566 problems in the future...
1572 bool host_changed = FALSE;
1574 const char *useurl = relurl;
1577 /* we must make our own copy of the URL to play with, as it may
1578 point to read-only data */
1579 char *url_clone = strdup(base);
1582 return NULL; /* skip out of this NOW */
1584 /* protsep points to the start of the host name */
1585 protsep = strstr(url_clone, "//");
1587 protsep = url_clone;
1589 protsep += 2; /* pass the slashes */
1591 if('/' != relurl[0]) {
1594 /* First we need to find out if there's a ?-letter in the URL,
1595 and cut it and the right-side of that off */
1596 pathsep = strchr(protsep, '?');
1600 /* we have a relative path to append to the last slash if there's one
1601 available, or if the new URL is just a query string (starts with a
1602 '?') we append the new one at the end of the entire currently worked
1604 if(useurl[0] != '?') {
1605 pathsep = strrchr(protsep, '/');
1610 /* Check if there's any slash after the host name, and if so, remember
1611 that position instead */
1612 pathsep = strchr(protsep, '/');
1614 protsep = pathsep + 1;
1618 /* now deal with one "./" or any amount of "../" in the newurl
1619 and act accordingly */
1621 if((useurl[0] == '.') && (useurl[1] == '/'))
1622 useurl += 2; /* just skip the "./" */
1624 while((useurl[0] == '.') &&
1625 (useurl[1] == '.') &&
1626 (useurl[2] == '/')) {
1628 useurl += 3; /* pass the "../" */
1633 /* cut off one more level from the right of the original URL */
1634 pathsep = strrchr(protsep, '/');
1645 /* We got a new absolute path for this server */
1647 if((relurl[0] == '/') && (relurl[1] == '/')) {
1648 /* the new URL starts with //, just keep the protocol part from the
1651 useurl = &relurl[2]; /* we keep the slashes from the original, so we
1652 skip the new ones */
1653 host_changed = TRUE;
1656 /* cut off the original URL from the first slash, or deal with URLs
1658 pathsep = strchr(protsep, '/');
1660 /* When people use badly formatted URLs, such as
1661 "http://www.url.com?dir=/home/daniel" we must not use the first
1662 slash, if there's a ?-letter before it! */
1663 char *sep = strchr(protsep, '?');
1664 if(sep && (sep < pathsep))
1669 /* There was no slash. Now, since we might be operating on a badly
1670 formatted URL, such as "http://www.url.com?id=2380" which doesn't
1671 use a slash separator as it is supposed to, we need to check for a
1672 ?-letter as well! */
1673 pathsep = strchr(protsep, '?');
1680 /* If the new part contains a space, this is a mighty stupid redirect
1681 but we still make an effort to do "right". To the left of a '?'
1682 letter we replace each space with %20 while it is replaced with '+'
1683 on the right side of the '?' letter.
1685 newlen = strlen_url(useurl, !host_changed);
1687 urllen = strlen(url_clone);
1689 newest = malloc(urllen + 1 + /* possible slash */
1690 newlen + 1 /* zero byte */);
1693 free(url_clone); /* don't leak this */
1697 /* copy over the root url part */
1698 memcpy(newest, url_clone, urllen);
1700 /* check if we need to append a slash */
1701 if(('/' == useurl[0]) || (protsep && !*protsep) || ('?' == useurl[0]))
1704 newest[urllen++]='/';
1706 /* then append the new piece on the right side */
1707 strcpy_url(&newest[urllen], useurl, !host_changed);
1713 #endif /* CURL_DISABLE_HTTP */
1716 * Curl_follow() handles the URL redirect magic. Pass in the 'newurl' string
1717 * as given by the remote server and set up the new URL to request.
1719 CURLcode Curl_follow(struct Curl_easy *data,
1720 char *newurl, /* the Location: string */
1721 followtype type) /* see transfer.h */
1723 #ifdef CURL_DISABLE_HTTP
1727 /* Location: following will not happen when HTTP is disabled */
1728 return CURLE_TOO_MANY_REDIRECTS;
1731 /* Location: redirect */
1732 bool disallowport = FALSE;
1733 bool reachedmax = FALSE;
1735 if(type == FOLLOW_REDIR) {
1736 if((data->set.maxredirs != -1) &&
1737 (data->set.followlocation >= data->set.maxredirs)) {
1739 type = FOLLOW_FAKE; /* switch to fake to store the would-be-redirected
1743 /* mark the next request as a followed location: */
1744 data->state.this_is_a_follow = TRUE;
1746 data->set.followlocation++; /* count location-followers */
1748 if(data->set.http_auto_referer) {
1749 /* We are asked to automatically set the previous URL as the referer
1750 when we get the next URL. We pick the ->url field, which may or may
1751 not be 100% correct */
1753 if(data->change.referer_alloc) {
1754 Curl_safefree(data->change.referer);
1755 data->change.referer_alloc = FALSE;
1758 data->change.referer = strdup(data->change.url);
1759 if(!data->change.referer)
1760 return CURLE_OUT_OF_MEMORY;
1761 data->change.referer_alloc = TRUE; /* yes, free this later */
1766 if(!is_absolute_url(newurl)) {
1768 *DANG* this is an RFC 2068 violation. The URL is supposed
1769 to be absolute and this doesn't seem to be that!
1771 char *absolute = concat_url(data->change.url, newurl);
1773 return CURLE_OUT_OF_MEMORY;
1777 /* The new URL MAY contain space or high byte values, that means a mighty
1778 stupid redirect URL but we still make an effort to do "right". */
1780 size_t newlen = strlen_url(newurl, FALSE);
1782 /* This is an absolute URL, don't allow the custom port number */
1783 disallowport = TRUE;
1785 newest = malloc(newlen + 1); /* get memory for this */
1787 return CURLE_OUT_OF_MEMORY;
1789 strcpy_url(newest, newurl, FALSE); /* create a space-free URL */
1790 newurl = newest; /* use this instead now */
1794 if(type == FOLLOW_FAKE) {
1795 /* we're only figuring out the new url if we would've followed locations
1796 but now we're done so we can get out! */
1797 data->info.wouldredirect = newurl;
1800 failf(data, "Maximum (%ld) redirects followed", data->set.maxredirs);
1801 return CURLE_TOO_MANY_REDIRECTS;
1807 data->state.allow_port = FALSE;
1809 if(data->change.url_alloc) {
1810 Curl_safefree(data->change.url);
1811 data->change.url_alloc = FALSE;
1814 data->change.url = newurl;
1815 data->change.url_alloc = TRUE;
1817 infof(data, "Issue another request to this URL: '%s'\n", data->change.url);
1820 * We get here when the HTTP code is 300-399 (and 401). We need to perform
1821 * differently based on exactly what return code there was.
1823 * News from 7.10.6: we can also get here on a 401 or 407, in case we act on
1824 * a HTTP (proxy-) authentication scheme other than Basic.
1826 switch(data->info.httpcode) {
1827 /* 401 - Act on a WWW-Authenticate, we keep on moving and do the
1828 Authorization: XXXX header in the HTTP request code snippet */
1829 /* 407 - Act on a Proxy-Authenticate, we keep on moving and do the
1830 Proxy-Authorization: XXXX header in the HTTP request code snippet */
1831 /* 300 - Multiple Choices */
1832 /* 306 - Not used */
1833 /* 307 - Temporary Redirect */
1834 default: /* for all above (and the unknown ones) */
1835 /* Some codes are explicitly mentioned since I've checked RFC2616 and they
1836 * seem to be OK to POST to.
1839 case 301: /* Moved Permanently */
1840 /* (quote from RFC7231, section 6.4.2)
1842 * Note: For historical reasons, a user agent MAY change the request
1843 * method from POST to GET for the subsequent request. If this
1844 * behavior is undesired, the 307 (Temporary Redirect) status code
1845 * can be used instead.
1849 * Many webservers expect this, so these servers often answers to a POST
1850 * request with an error page. To be sure that libcurl gets the page that
1851 * most user agents would get, libcurl has to force GET.
1853 * This behaviour is forbidden by RFC1945 and the obsolete RFC2616, and
1854 * can be overridden with CURLOPT_POSTREDIR.
1856 if((data->set.httpreq == HTTPREQ_POST
1857 || data->set.httpreq == HTTPREQ_POST_FORM
1858 || data->set.httpreq == HTTPREQ_POST_MIME)
1859 && !(data->set.keep_post & CURL_REDIR_POST_301)) {
1860 infof(data, "Switch from POST to GET\n");
1861 data->set.httpreq = HTTPREQ_GET;
1864 case 302: /* Found */
1865 /* (quote from RFC7231, section 6.4.3)
1867 * Note: For historical reasons, a user agent MAY change the request
1868 * method from POST to GET for the subsequent request. If this
1869 * behavior is undesired, the 307 (Temporary Redirect) status code
1870 * can be used instead.
1874 * Many webservers expect this, so these servers often answers to a POST
1875 * request with an error page. To be sure that libcurl gets the page that
1876 * most user agents would get, libcurl has to force GET.
1878 * This behaviour is forbidden by RFC1945 and the obsolete RFC2616, and
1879 * can be overridden with CURLOPT_POSTREDIR.
1881 if((data->set.httpreq == HTTPREQ_POST
1882 || data->set.httpreq == HTTPREQ_POST_FORM
1883 || data->set.httpreq == HTTPREQ_POST_MIME)
1884 && !(data->set.keep_post & CURL_REDIR_POST_302)) {
1885 infof(data, "Switch from POST to GET\n");
1886 data->set.httpreq = HTTPREQ_GET;
1890 case 303: /* See Other */
1891 /* Disable both types of POSTs, unless the user explicitly
1892 asks for POST after POST */
1893 if(data->set.httpreq != HTTPREQ_GET
1894 && !(data->set.keep_post & CURL_REDIR_POST_303)) {
1895 data->set.httpreq = HTTPREQ_GET; /* enforce GET request */
1896 infof(data, "Disables POST, goes with %s\n",
1897 data->set.opt_no_body?"HEAD":"GET");
1900 case 304: /* Not Modified */
1901 /* 304 means we did a conditional request and it was "Not modified".
1902 * We shouldn't get any Location: header in this response!
1905 case 305: /* Use Proxy */
1906 /* (quote from RFC2616, section 10.3.6):
1907 * "The requested resource MUST be accessed through the proxy given
1908 * by the Location field. The Location field gives the URI of the
1909 * proxy. The recipient is expected to repeat this single request
1910 * via the proxy. 305 responses MUST only be generated by origin
1915 Curl_pgrsTime(data, TIMER_REDIRECT);
1916 Curl_pgrsResetTransferSizes(data);
1919 #endif /* CURL_DISABLE_HTTP */
1922 /* Returns CURLE_OK *and* sets '*url' if a request retry is wanted.
1924 NOTE: that the *url is malloc()ed. */
1925 CURLcode Curl_retry_request(struct connectdata *conn,
1928 struct Curl_easy *data = conn->data;
1932 /* if we're talking upload, we can't do the checks below, unless the protocol
1933 is HTTP as when uploading over HTTP we will still get a response */
1934 if(data->set.upload &&
1935 !(conn->handler->protocol&(PROTO_FAMILY_HTTP|CURLPROTO_RTSP)))
1938 if((data->req.bytecount + data->req.headerbytecount == 0) &&
1940 (!data->set.opt_no_body
1941 || (conn->handler->protocol & PROTO_FAMILY_HTTP)) &&
1942 (data->set.rtspreq != RTSPREQ_RECEIVE)) {
1943 /* We got no data, we attempted to re-use a connection. For HTTP this
1944 can be a retry so we try again regardless if we expected a body.
1945 For other protocols we only try again only if we expected a body.
1947 This might happen if the connection was left alive when we were
1948 done using it before, but that was closed when we wanted to read from
1949 it again. Bad luck. Retry the same request on a fresh connect! */
1950 infof(conn->data, "Connection died, retrying a fresh connect\n");
1951 *url = strdup(conn->data->change.url);
1953 return CURLE_OUT_OF_MEMORY;
1955 connclose(conn, "retry"); /* close this connection */
1956 conn->bits.retry = TRUE; /* mark this as a connection we're about
1957 to retry. Marking it this way should
1958 prevent i.e HTTP transfers to return
1959 error just because nothing has been
1963 if(conn->handler->protocol&PROTO_FAMILY_HTTP) {
1964 struct HTTP *http = data->req.protop;
1965 if(http->writebytecount)
1966 return Curl_readrewind(conn);
1973 * Curl_setup_transfer() is called to setup some basic properties for the
1974 * upcoming transfer.
1977 Curl_setup_transfer(
1978 struct connectdata *conn, /* connection data */
1979 int sockindex, /* socket index to read from or -1 */
1980 curl_off_t size, /* -1 if unknown at this point */
1981 bool getheader, /* TRUE if header parsing is wanted */
1982 curl_off_t *bytecountp, /* return number of bytes read or NULL */
1983 int writesockindex, /* socket index to write to, it may very well be
1984 the same we read from. -1 disables */
1985 curl_off_t *writecountp /* return number of bytes written or NULL */
1988 struct Curl_easy *data;
1989 struct SingleRequest *k;
1991 DEBUGASSERT(conn != NULL);
1996 DEBUGASSERT((sockindex <= 1) && (sockindex >= -1));
1998 /* now copy all input parameters */
1999 conn->sockfd = sockindex == -1 ?
2000 CURL_SOCKET_BAD : conn->sock[sockindex];
2001 conn->writesockfd = writesockindex == -1 ?
2002 CURL_SOCKET_BAD:conn->sock[writesockindex];
2003 k->getheader = getheader;
2006 k->bytecountp = bytecountp;
2007 k->writebytecountp = writecountp;
2009 /* The code sequence below is placed in this function just because all
2010 necessary input is not always known in do_complete() as this function may
2011 be called after that */
2016 Curl_pgrsSetDownloadSize(data, size);
2018 /* we want header and/or body, if neither then don't do this! */
2019 if(k->getheader || !data->set.opt_no_body) {
2021 if(conn->sockfd != CURL_SOCKET_BAD)
2022 k->keepon |= KEEP_RECV;
2024 if(conn->writesockfd != CURL_SOCKET_BAD) {
2025 struct HTTP *http = data->req.protop;
2028 Even if we require a 100-return code before uploading data, we might
2029 need to write data before that since the REQUEST may not have been
2030 finished sent off just yet.
2032 Thus, we must check if the request has been sent before we set the
2033 state info where we wait for the 100-return code
2035 if((data->state.expect100header) &&
2036 (conn->handler->protocol&PROTO_FAMILY_HTTP) &&
2037 (http->sending == HTTPSEND_BODY)) {
2038 /* wait with write until we either got 100-continue or a timeout */
2039 k->exp100 = EXP100_AWAITING_CONTINUE;
2040 k->start100 = Curl_now();
2042 /* Set a timeout for the multi interface. Add the inaccuracy margin so
2043 that we don't fire slightly too early and get denied to run. */
2044 Curl_expire(data, data->set.expect_100_timeout, EXPIRE_100_TIMEOUT);
2047 if(data->state.expect100header)
2048 /* when we've sent off the rest of the headers, we must await a
2049 100-continue but first finish sending the request */
2050 k->exp100 = EXP100_SENDING_REQUEST;
2052 /* enable the write bit when we're not waiting for continue */
2053 k->keepon |= KEEP_SEND;
2055 } /* if(conn->writesockfd != CURL_SOCKET_BAD) */
2056 } /* if(k->getheader || !data->set.opt_no_body) */