1 /***************************************************************************
3 * Project ___| | | | _ \| |
5 * | (__| |_| | _ <| |___
6 * \___|\___/|_| \_\_____|
8 * Copyright (C) 1998 - 2022, Daniel Stenberg, <daniel@haxx.se>, et al.
10 * This software is licensed as described in the file COPYING, which
11 * you should have received as part of this distribution. The terms
12 * are also available at https://curl.se/docs/copyright.html.
14 * You may opt to use, copy, modify, merge, publish, distribute and/or sell
15 * copies of the Software, and permit persons to whom the Software is
16 * furnished to do so, under the terms of the COPYING file.
18 * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
19 * KIND, either express or implied.
21 * SPDX-License-Identifier: curl
23 ***************************************************************************/
25 #include "curl_setup.h"
26 #include "strtoofft.h"
28 #ifdef HAVE_NETINET_IN_H
29 #include <netinet/in.h>
34 #ifdef HAVE_ARPA_INET_H
35 #include <arpa/inet.h>
40 #ifdef HAVE_SYS_IOCTL_H
41 #include <sys/ioctl.h>
47 #ifdef HAVE_SYS_PARAM_H
48 #include <sys/param.h>
51 #ifdef HAVE_SYS_SELECT_H
52 #include <sys/select.h>
53 #elif defined(HAVE_UNISTD_H)
58 #error "We can't compile without socket() support!"
62 #include <curl/curl.h>
65 #include "content_encoding.h"
69 #include "speedcheck.h"
74 #include "vtls/vtls.h"
81 #include "urlapi-int.h"
86 /* The last 3 #include files should be in this order */
87 #include "curl_printf.h"
88 #include "curl_memory.h"
91 #if !defined(CURL_DISABLE_HTTP) || !defined(CURL_DISABLE_SMTP) || \
92 !defined(CURL_DISABLE_IMAP)
94 * checkheaders() checks the linked list of custom headers for a
95 * particular header (prefix). Provide the prefix without colon!
97 * Returns a pointer to the first matching header or NULL if none matched.
99 char *Curl_checkheaders(const struct Curl_easy *data,
100 const char *thisheader,
101 const size_t thislen)
103 struct curl_slist *head;
104 DEBUGASSERT(thislen);
105 DEBUGASSERT(thisheader[thislen-1] != ':');
107 for(head = data->set.headers; head; head = head->next) {
108 if(strncasecompare(head->data, thisheader, thislen) &&
109 Curl_headersep(head->data[thislen]) )
117 CURLcode Curl_get_upload_buffer(struct Curl_easy *data)
119 if(!data->state.ulbuf) {
120 data->state.ulbuf = malloc(data->set.upload_buffer_size);
121 if(!data->state.ulbuf)
122 return CURLE_OUT_OF_MEMORY;
127 #ifndef CURL_DISABLE_HTTP
129 * This function will be called to loop through the trailers buffer
130 * until no more data is available for sending.
132 static size_t trailers_read(char *buffer, size_t size, size_t nitems,
135 struct Curl_easy *data = (struct Curl_easy *)raw;
136 struct dynbuf *trailers_buf = &data->state.trailers_buf;
137 size_t bytes_left = Curl_dyn_len(trailers_buf) -
138 data->state.trailers_bytes_sent;
139 size_t to_copy = (size*nitems < bytes_left) ? size*nitems : bytes_left;
142 Curl_dyn_ptr(trailers_buf) + data->state.trailers_bytes_sent,
144 data->state.trailers_bytes_sent += to_copy;
149 static size_t trailers_left(void *raw)
151 struct Curl_easy *data = (struct Curl_easy *)raw;
152 struct dynbuf *trailers_buf = &data->state.trailers_buf;
153 return Curl_dyn_len(trailers_buf) - data->state.trailers_bytes_sent;
158 * This function will call the read callback to fill our buffer with data
161 CURLcode Curl_fillreadbuffer(struct Curl_easy *data, size_t bytes,
164 size_t buffersize = bytes;
167 curl_read_callback readfunc = NULL;
168 void *extra_data = NULL;
170 #ifndef CURL_DISABLE_HTTP
171 if(data->state.trailers_state == TRAILERS_INITIALIZED) {
172 struct curl_slist *trailers = NULL;
174 int trailers_ret_code;
176 /* at this point we already verified that the callback exists
177 so we compile and store the trailers buffer, then proceed */
179 "Moving trailers state machine from initialized to sending.");
180 data->state.trailers_state = TRAILERS_SENDING;
181 Curl_dyn_init(&data->state.trailers_buf, DYN_TRAILERS);
183 data->state.trailers_bytes_sent = 0;
184 Curl_set_in_callback(data, true);
185 trailers_ret_code = data->set.trailer_callback(&trailers,
186 data->set.trailer_data);
187 Curl_set_in_callback(data, false);
188 if(trailers_ret_code == CURL_TRAILERFUNC_OK) {
189 result = Curl_http_compile_trailers(trailers, &data->state.trailers_buf,
193 failf(data, "operation aborted by trailing headers callback");
195 result = CURLE_ABORTED_BY_CALLBACK;
198 Curl_dyn_free(&data->state.trailers_buf);
199 curl_slist_free_all(trailers);
202 infof(data, "Successfully compiled trailers.");
203 curl_slist_free_all(trailers);
207 #ifndef CURL_DISABLE_HTTP
208 /* if we are transmitting trailing data, we don't need to write
209 a chunk size so we skip this */
210 if(data->req.upload_chunky &&
211 data->state.trailers_state == TRAILERS_NONE) {
212 /* if chunked Transfer-Encoding */
213 buffersize -= (8 + 2 + 2); /* 32bit hex + CRLF + CRLF */
214 data->req.upload_fromhere += (8 + 2); /* 32bit hex + CRLF */
217 if(data->state.trailers_state == TRAILERS_SENDING) {
218 /* if we're here then that means that we already sent the last empty chunk
219 but we didn't send a final CR LF, so we sent 0 CR LF. We then start
220 pulling trailing data until we have no more at which point we
221 simply return to the previous point in the state machine as if
224 readfunc = trailers_read;
225 extra_data = (void *)data;
230 readfunc = data->state.fread_func;
231 extra_data = data->state.in;
234 Curl_set_in_callback(data, true);
235 nread = readfunc(data->req.upload_fromhere, 1,
236 buffersize, extra_data);
237 Curl_set_in_callback(data, false);
239 if(nread == CURL_READFUNC_ABORT) {
240 failf(data, "operation aborted by callback");
242 return CURLE_ABORTED_BY_CALLBACK;
244 if(nread == CURL_READFUNC_PAUSE) {
245 struct SingleRequest *k = &data->req;
247 if(data->conn->handler->flags & PROTOPT_NONETWORK) {
248 /* protocols that work without network cannot be paused. This is
249 actually only FILE:// just now, and it can't pause since the transfer
250 isn't done using the "normal" procedure. */
251 failf(data, "Read callback asked for PAUSE when not supported");
252 return CURLE_READ_ERROR;
255 /* CURL_READFUNC_PAUSE pauses read callbacks that feed socket writes */
256 k->keepon |= KEEP_SEND_PAUSE; /* mark socket send as paused */
257 if(data->req.upload_chunky) {
258 /* Back out the preallocation done above */
259 data->req.upload_fromhere -= (8 + 2);
263 return CURLE_OK; /* nothing was read */
265 else if(nread > buffersize) {
266 /* the read function returned a too large value */
268 failf(data, "read function returned funny value");
269 return CURLE_READ_ERROR;
272 #ifndef CURL_DISABLE_HTTP
273 if(!data->req.forbidchunk && data->req.upload_chunky) {
274 /* if chunked Transfer-Encoding
280 /* On non-ASCII platforms the <DATA> may or may not be
281 translated based on state.prefer_ascii while the protocol
282 portion must always be translated to the network encoding.
283 To further complicate matters, line end conversion might be
284 done later on, so we need to prevent CRLFs from becoming
285 CRCRLFs if that's the case. To do this we use bare LFs
286 here, knowing they'll become CRLFs later on.
289 bool added_crlf = FALSE;
291 const char *endofline_native;
292 const char *endofline_network;
295 #ifdef CURL_DO_LINEEND_CONV
296 (data->state.prefer_ascii) ||
299 /* \n will become \r\n later on */
300 endofline_native = "\n";
301 endofline_network = "\x0a";
304 endofline_native = "\r\n";
305 endofline_network = "\x0d\x0a";
308 /* if we're not handling trailing data, proceed as usual */
309 if(data->state.trailers_state != TRAILERS_SENDING) {
310 char hexbuffer[11] = "";
311 hexlen = msnprintf(hexbuffer, sizeof(hexbuffer),
312 "%zx%s", nread, endofline_native);
314 /* move buffer pointer */
315 data->req.upload_fromhere -= hexlen;
318 /* copy the prefix to the buffer, leaving out the NUL */
319 memcpy(data->req.upload_fromhere, hexbuffer, hexlen);
321 /* always append ASCII CRLF to the data unless
322 we have a valid trailer callback */
323 if((nread-hexlen) == 0 &&
324 data->set.trailer_callback != NULL &&
325 data->state.trailers_state == TRAILERS_NONE) {
326 data->state.trailers_state = TRAILERS_INITIALIZED;
329 memcpy(data->req.upload_fromhere + nread,
331 strlen(endofline_network));
336 if(data->state.trailers_state == TRAILERS_SENDING &&
337 !trailers_left(data)) {
338 Curl_dyn_free(&data->state.trailers_buf);
339 data->state.trailers_state = TRAILERS_DONE;
340 data->set.trailer_data = NULL;
341 data->set.trailer_callback = NULL;
342 /* mark the transfer as done */
343 data->req.upload_done = TRUE;
344 infof(data, "Signaling end of chunked upload after trailers.");
347 if((nread - hexlen) == 0 &&
348 data->state.trailers_state != TRAILERS_INITIALIZED) {
349 /* mark this as done once this chunk is transferred */
350 data->req.upload_done = TRUE;
352 "Signaling end of chunked upload via terminating chunk.");
356 nread += strlen(endofline_network); /* for the added end of line */
367 * Curl_readrewind() rewinds the read stream. This is typically used for HTTP
368 * POST/PUT with multi-pass authentication when a sending was denied and a
369 * resend is necessary.
371 CURLcode Curl_readrewind(struct Curl_easy *data)
373 struct connectdata *conn = data->conn;
374 curl_mimepart *mimepart = &data->set.mimepost;
376 conn->bits.rewindaftersend = FALSE; /* we rewind now */
378 /* explicitly switch off sending data on this connection now since we are
379 about to restart a new transfer and thus we want to avoid inadvertently
380 sending more data on the existing connection until the next transfer
382 data->req.keepon &= ~KEEP_SEND;
384 /* We have sent away data. If not using CURLOPT_POSTFIELDS or
385 CURLOPT_HTTPPOST, call app to rewind
387 if(conn->handler->protocol & PROTO_FAMILY_HTTP) {
388 struct HTTP *http = data->req.p.http;
391 mimepart = http->sendit;
393 if(data->set.postfields)
395 else if(data->state.httpreq == HTTPREQ_POST_MIME ||
396 data->state.httpreq == HTTPREQ_POST_FORM) {
397 CURLcode result = Curl_mime_rewind(mimepart);
399 failf(data, "Cannot rewind mime/post data");
404 if(data->set.seek_func) {
407 Curl_set_in_callback(data, true);
408 err = (data->set.seek_func)(data->set.seek_client, 0, SEEK_SET);
409 Curl_set_in_callback(data, false);
411 failf(data, "seek callback returned error %d", (int)err);
412 return CURLE_SEND_FAIL_REWIND;
415 else if(data->set.ioctl_func) {
418 Curl_set_in_callback(data, true);
419 err = (data->set.ioctl_func)(data, CURLIOCMD_RESTARTREAD,
420 data->set.ioctl_client);
421 Curl_set_in_callback(data, false);
422 infof(data, "the ioctl callback returned %d", (int)err);
425 failf(data, "ioctl callback returned error %d", (int)err);
426 return CURLE_SEND_FAIL_REWIND;
430 /* If no CURLOPT_READFUNCTION is used, we know that we operate on a
431 given FILE * stream and we can actually attempt to rewind that
432 ourselves with fseek() */
433 if(data->state.fread_func == (curl_read_callback)fread) {
434 if(-1 != fseek(data->state.in, 0, SEEK_SET))
435 /* successful rewind */
439 /* no callback set or failure above, makes us fail at once */
440 failf(data, "necessary data rewind wasn't possible");
441 return CURLE_SEND_FAIL_REWIND;
447 static int data_pending(const struct Curl_easy *data)
449 struct connectdata *conn = data->conn;
452 if(conn->transport == TRNSPRT_QUIC)
453 return Curl_quic_data_pending(data);
456 if(conn->handler->protocol&PROTO_FAMILY_FTP)
457 return Curl_ssl_data_pending(conn, SECONDARYSOCKET);
459 /* in the case of libssh2, we can never be really sure that we have emptied
460 its internal buffers so we MUST always try until we get EAGAIN back */
461 return conn->handler->protocol&(CURLPROTO_SCP|CURLPROTO_SFTP) ||
463 /* For HTTP/2, we may read up everything including response body
464 with header fields in Curl_http_readwrite_headers. If no
465 content-length is provided, curl waits for the connection
466 close, which we emulate it using conn->proto.httpc.closed =
467 TRUE. The thing is if we read everything, then http2_recv won't
468 be called and we cannot signal the HTTP/2 stream has closed. As
469 a workaround, we return nonzero here to call http2_recv. */
470 ((conn->handler->protocol&PROTO_FAMILY_HTTP) && conn->httpversion >= 20) ||
472 Curl_ssl_data_pending(conn, FIRSTSOCKET);
476 * Check to see if CURLOPT_TIMECONDITION was met by comparing the time of the
477 * remote document with the time provided by CURLOPT_TIMEVAL
479 bool Curl_meets_timecondition(struct Curl_easy *data, time_t timeofdoc)
481 if((timeofdoc == 0) || (data->set.timevalue == 0))
484 switch(data->set.timecondition) {
485 case CURL_TIMECOND_IFMODSINCE:
487 if(timeofdoc <= data->set.timevalue) {
489 "The requested document is not new enough");
490 data->info.timecond = TRUE;
494 case CURL_TIMECOND_IFUNMODSINCE:
495 if(timeofdoc >= data->set.timevalue) {
497 "The requested document is not old enough");
498 data->info.timecond = TRUE;
508 * Go ahead and do a read if we have a readable socket or if
509 * the stream was rewound (in which case we have data in a
512 * return '*comeback' TRUE if we didn't properly drain the socket so this
513 * function should get called again without select() or similar in between!
515 static CURLcode readwrite_data(struct Curl_easy *data,
516 struct connectdata *conn,
517 struct SingleRequest *k,
518 int *didwhat, bool *done,
521 CURLcode result = CURLE_OK;
522 ssize_t nread; /* number of bytes read */
523 size_t excess = 0; /* excess bytes read */
524 bool readmore = FALSE; /* used by RTP to signal for more data */
526 char *buf = data->state.buffer;
532 /* This is where we loop until we have read everything there is to
533 read or we get a CURLE_AGAIN */
535 bool is_empty_data = FALSE;
536 size_t buffersize = data->set.buffer_size;
537 size_t bytestoread = buffersize;
539 bool is_http2 = ((conn->handler->protocol & PROTO_FAMILY_HTTP) &&
540 (conn->httpversion == 20));
544 ((conn->handler->protocol & PROTO_FAMILY_HTTP) &&
545 (conn->httpversion == 30));
552 /* For HTTP/2, read data without caring about the content length. This
553 is safe because body in HTTP/2 is always segmented thanks to its
554 framing layer. Meanwhile, we have to call Curl_read to ensure that
555 http2_handle_stream_close is called when we read all incoming bytes
556 for a particular stream. */
559 !is_http3 && /* Same reason mentioned above. */
560 k->size != -1 && !k->header) {
561 /* make sure we don't read too much */
562 curl_off_t totalleft = k->size - k->bytecount;
563 if(totalleft < (curl_off_t)bytestoread)
564 bytestoread = (size_t)totalleft;
568 /* receive data from the network! */
569 result = Curl_read(data, conn->sockfd, buf, bytestoread, &nread);
571 /* read would've blocked */
572 if(CURLE_AGAIN == result)
573 break; /* get out of loop */
579 /* read nothing but since we wanted nothing we consider this an OK
580 situation to proceed from */
581 DEBUGF(infof(data, "readwrite_data: we're done"));
586 Curl_pgrsTime(data, TIMER_STARTTRANSFER);
587 if(k->exp100 > EXP100_SEND_DATA)
588 /* set time stamp to compare with when waiting for the 100 */
589 k->start100 = Curl_now();
592 *didwhat |= KEEP_RECV;
593 /* indicates data of zero size, i.e. empty file */
594 is_empty_data = ((nread == 0) && (k->bodywrites == 0)) ? TRUE : FALSE;
596 if(0 < nread || is_empty_data) {
600 /* if we receive 0 or less here, either the http2 stream is closed or the
601 server closed the connection and we bail out from this! */
603 if(is_http2 && !nread)
604 DEBUGF(infof(data, "nread == 0, stream closed, bailing"));
607 if(is_http3 && !nread)
608 DEBUGF(infof(data, "nread == 0, stream closed, bailing"));
610 DEBUGF(infof(data, "nread <= 0, server closed connection, bailing"));
611 k->keepon &= ~KEEP_RECV;
615 /* Default buffer to use when we write the buffer, it may be changed
616 in the flow below before the actual storing is done. */
619 if(conn->handler->readwrite) {
620 result = conn->handler->readwrite(data, conn, &nread, &readmore);
627 #ifndef CURL_DISABLE_HTTP
628 /* Since this is a two-state thing, we check if we are parsing
629 headers at the moment or not. */
631 /* we are in parse-the-header-mode */
632 bool stop_reading = FALSE;
633 result = Curl_http_readwrite_headers(data, conn, &nread, &stop_reading);
637 if(conn->handler->readwrite &&
638 (k->maxdownload <= 0 && nread > 0)) {
639 result = conn->handler->readwrite(data, conn, &nread, &readmore);
647 /* We've stopped dealing with input, get out of the do-while loop */
653 " url = %s (zero-length body)",
654 nread, data->state.up.path);
660 #endif /* CURL_DISABLE_HTTP */
663 /* This is not an 'else if' since it may be a rest from the header
664 parsing, where the beginning of the buffer is headers and the end
666 if(!k->header && (nread > 0 || is_empty_data)) {
668 if(data->set.opt_no_body) {
669 /* data arrives although we want none, bail out */
670 streamclose(conn, "ignoring body");
672 return CURLE_WEIRD_SERVER_REPLY;
675 #ifndef CURL_DISABLE_HTTP
676 if(0 == k->bodywrites && !is_empty_data) {
677 /* These checks are only made the first time we are about to
678 write a piece of the body */
679 if(conn->handler->protocol&(PROTO_FAMILY_HTTP|CURLPROTO_RTSP)) {
680 /* HTTP-only checks */
681 result = Curl_http_firstwrite(data, conn, done);
685 } /* this is the first time we write a body part */
686 #endif /* CURL_DISABLE_HTTP */
690 /* pass data to the debug function before it gets "dechunked" */
691 if(data->set.verbose) {
693 Curl_debug(data, CURLINFO_DATA_IN,
694 Curl_dyn_ptr(&data->state.headerb),
695 Curl_dyn_len(&data->state.headerb));
696 if(k->badheader == HEADER_PARTHEADER)
697 Curl_debug(data, CURLINFO_DATA_IN,
698 k->str, (size_t)nread);
701 Curl_debug(data, CURLINFO_DATA_IN,
702 k->str, (size_t)nread);
705 #ifndef CURL_DISABLE_HTTP
708 * Here comes a chunked transfer flying and we need to decode this
709 * properly. While the name says read, this function both reads
710 * and writes away the data. The returned 'nread' holds the number
711 * of actual data it wrote to the client.
715 Curl_httpchunk_read(data, k->str, nread, &nread, &extra);
717 if(CHUNKE_OK < res) {
718 if(CHUNKE_PASSTHRU_ERROR == res) {
719 failf(data, "Failed reading the chunked-encoded stream");
722 failf(data, "%s in chunked-encoding", Curl_chunked_strerror(res));
723 return CURLE_RECV_ERROR;
725 if(CHUNKE_STOP == res) {
726 /* we're done reading chunks! */
727 k->keepon &= ~KEEP_RECV; /* read no more */
729 /* N number of bytes at the end of the str buffer that weren't
730 written to the client. */
731 if(conn->chunk.datasize) {
732 infof(data, "Leftovers after chunking: % "
733 CURL_FORMAT_CURL_OFF_T "u bytes",
734 conn->chunk.datasize);
737 /* If it returned OK, we just keep going */
739 #endif /* CURL_DISABLE_HTTP */
741 /* Account for body content stored in the header buffer */
742 if((k->badheader == HEADER_PARTHEADER) && !k->ignorebody) {
743 size_t headlen = Curl_dyn_len(&data->state.headerb);
744 DEBUGF(infof(data, "Increasing bytecount by %zu", headlen));
745 k->bytecount += headlen;
748 if((-1 != k->maxdownload) &&
749 (k->bytecount + nread >= k->maxdownload)) {
751 excess = (size_t)(k->bytecount + nread - k->maxdownload);
752 if(excess > 0 && !k->ignorebody) {
754 "Excess found in a read:"
756 ", size = %" CURL_FORMAT_CURL_OFF_T
757 ", maxdownload = %" CURL_FORMAT_CURL_OFF_T
758 ", bytecount = %" CURL_FORMAT_CURL_OFF_T,
759 excess, k->size, k->maxdownload, k->bytecount);
760 connclose(conn, "excess found in a read");
763 nread = (ssize_t) (k->maxdownload - k->bytecount);
764 if(nread < 0) /* this should be unusual */
767 /* HTTP/3 over QUIC should keep reading until QUIC connection
768 is closed. In contrast to HTTP/2 which can stop reading
769 from TCP connection, HTTP/3 over QUIC needs ACK from server
770 to ensure stream closure. It should keep reading. */
772 k->keepon &= ~KEEP_RECV; /* we're done reading */
776 k->bytecount += nread;
778 Curl_pgrsSetDownloadCounter(data, k->bytecount);
780 if(!k->chunk && (nread || k->badheader || is_empty_data)) {
781 /* If this is chunky transfer, it was already written */
783 if(k->badheader && !k->ignorebody) {
784 /* we parsed a piece of data wrongly assuming it was a header
785 and now we output it as body instead */
786 size_t headlen = Curl_dyn_len(&data->state.headerb);
788 /* Don't let excess data pollute body writes */
789 if(k->maxdownload == -1 || (curl_off_t)headlen <= k->maxdownload)
790 result = Curl_client_write(data, CLIENTWRITE_BODY,
791 Curl_dyn_ptr(&data->state.headerb),
794 result = Curl_client_write(data, CLIENTWRITE_BODY,
795 Curl_dyn_ptr(&data->state.headerb),
796 (size_t)k->maxdownload);
801 if(k->badheader < HEADER_ALLBAD) {
802 /* This switch handles various content encodings. If there's an
803 error here, be sure to check over the almost identical code
805 Make sure that ALL_CONTENT_ENCODINGS contains all the
806 encodings handled here. */
807 if(data->set.http_ce_skip || !k->writer_stack) {
808 if(!k->ignorebody && nread) {
809 #ifndef CURL_DISABLE_POP3
810 if(conn->handler->protocol & PROTO_FAMILY_POP3)
811 result = Curl_pop3_write(data, k->str, nread);
813 #endif /* CURL_DISABLE_POP3 */
814 result = Curl_client_write(data, CLIENTWRITE_BODY, k->str,
818 else if(!k->ignorebody && nread)
819 result = Curl_unencode_write(data, k->writer_stack, k->str, nread);
821 k->badheader = HEADER_NORMAL; /* taken care of now */
827 } /* if(!header and data to read) */
829 if(conn->handler->readwrite && excess) {
830 /* Parse the excess data */
833 if(&k->str[excess] > &buf[data->set.buffer_size]) {
834 /* the excess amount was too excessive(!), make sure
835 it doesn't read out of buffer */
836 excess = &buf[data->set.buffer_size] - k->str;
838 nread = (ssize_t)excess;
840 result = conn->handler->readwrite(data, conn, &nread, &readmore);
845 k->keepon |= KEEP_RECV; /* we're not done reading */
850 /* if we received nothing, the server closed the connection and we
852 k->keepon &= ~KEEP_RECV;
855 if(k->keepon & KEEP_RECV_PAUSE) {
856 /* this is a paused transfer */
860 } while(data_pending(data) && maxloops--);
863 /* we mark it as read-again-please */
864 conn->cselect_bits = CURL_CSELECT_IN;
868 if(((k->keepon & (KEEP_RECV|KEEP_SEND)) == KEEP_SEND) &&
870 /* When we've read the entire thing and the close bit is set, the server
871 may now close the connection. If there's now any kind of sending going
872 on from our side, we need to stop that immediately. */
873 infof(data, "we are done reading and this is set to close, stop send");
874 k->keepon &= ~KEEP_SEND; /* no writing anymore either */
880 CURLcode Curl_done_sending(struct Curl_easy *data,
881 struct SingleRequest *k)
883 struct connectdata *conn = data->conn;
884 k->keepon &= ~KEEP_SEND; /* we're done writing */
886 /* These functions should be moved into the handler struct! */
887 Curl_http2_done_sending(data, conn);
888 Curl_quic_done_sending(data);
890 if(conn->bits.rewindaftersend) {
891 CURLcode result = Curl_readrewind(data);
898 #if defined(WIN32) && defined(USE_WINSOCK)
899 #ifndef SIO_IDEAL_SEND_BACKLOG_QUERY
900 #define SIO_IDEAL_SEND_BACKLOG_QUERY 0x4004747B
903 static void win_update_buffer_size(curl_socket_t sockfd)
908 result = WSAIoctl(sockfd, SIO_IDEAL_SEND_BACKLOG_QUERY, 0, 0,
909 &ideal, sizeof(ideal), &ideallen, 0, 0);
911 setsockopt(sockfd, SOL_SOCKET, SO_SNDBUF,
912 (const char *)&ideal, sizeof(ideal));
916 #define win_update_buffer_size(x)
919 #define curl_upload_refill_watermark(data) \
920 ((ssize_t)((data)->set.upload_buffer_size >> 5))
923 * Send data to upload to the server, when the socket is writable.
925 static CURLcode readwrite_upload(struct Curl_easy *data,
926 struct connectdata *conn,
930 ssize_t bytes_written;
932 ssize_t nread; /* number of bytes read */
933 bool sending_http_headers = FALSE;
934 struct SingleRequest *k = &data->req;
936 if((k->bytecount == 0) && (k->writebytecount == 0))
937 Curl_pgrsTime(data, TIMER_STARTTRANSFER);
939 *didwhat |= KEEP_SEND;
945 if(0 != k->upload_present &&
946 k->upload_present < curl_upload_refill_watermark(data) &&
947 !k->upload_chunky &&/*(variable sized chunked header; append not safe)*/
948 !k->upload_done && /*!(k->upload_done once k->upload_present sent)*/
949 !(k->writebytecount + k->upload_present - k->pendingheader ==
950 data->state.infilesize)) {
951 offset = k->upload_present;
954 /* only read more data if there's no upload data already
955 present in the upload buffer, or if appending to upload buffer */
956 if(0 == k->upload_present || offset) {
957 result = Curl_get_upload_buffer(data);
960 if(offset && k->upload_fromhere != data->state.ulbuf)
961 memmove(data->state.ulbuf, k->upload_fromhere, offset);
962 /* init the "upload from here" pointer */
963 k->upload_fromhere = data->state.ulbuf;
965 if(!k->upload_done) {
966 /* HTTP pollution, this should be written nicer to become more
967 protocol agnostic. */
969 struct HTTP *http = k->p.http;
971 if((k->exp100 == EXP100_SENDING_REQUEST) &&
972 (http->sending == HTTPSEND_BODY)) {
973 /* If this call is to send body data, we must take some action:
974 We have sent off the full HTTP 1.1 request, and we shall now
975 go into the Expect: 100 state and await such a header */
976 k->exp100 = EXP100_AWAITING_CONTINUE; /* wait for the header */
977 k->keepon &= ~KEEP_SEND; /* disable writing */
978 k->start100 = Curl_now(); /* timeout count starts now */
979 *didwhat &= ~KEEP_SEND; /* we didn't write anything actually */
980 /* set a timeout for the multi interface */
981 Curl_expire(data, data->set.expect_100_timeout, EXPIRE_100_TIMEOUT);
985 if(conn->handler->protocol&(PROTO_FAMILY_HTTP|CURLPROTO_RTSP)) {
986 if(http->sending == HTTPSEND_REQUEST)
987 /* We're sending the HTTP request headers, not the data.
988 Remember that so we don't change the line endings. */
989 sending_http_headers = TRUE;
991 sending_http_headers = FALSE;
994 k->upload_fromhere += offset;
995 result = Curl_fillreadbuffer(data, data->set.upload_buffer_size-offset,
997 k->upload_fromhere -= offset;
1001 nread = offset + fillcount;
1004 nread = 0; /* we're done uploading/reading */
1006 if(!nread && (k->keepon & KEEP_SEND_PAUSE)) {
1007 /* this is a paused transfer */
1011 result = Curl_done_sending(data, k);
1017 /* store number of bytes available for upload */
1018 k->upload_present = nread;
1020 /* convert LF to CRLF if so asked */
1021 if((!sending_http_headers) && (
1022 #ifdef CURL_DO_LINEEND_CONV
1023 /* always convert if we're FTPing in ASCII mode */
1024 (data->state.prefer_ascii) ||
1026 (data->set.crlf))) {
1027 /* Do we need to allocate a scratch buffer? */
1028 if(!data->state.scratch) {
1029 data->state.scratch = malloc(2 * data->set.upload_buffer_size);
1030 if(!data->state.scratch) {
1031 failf(data, "Failed to alloc scratch buffer");
1033 return CURLE_OUT_OF_MEMORY;
1038 * ASCII/EBCDIC Note: This is presumably a text (not binary)
1039 * transfer so the data should already be in ASCII.
1040 * That means the hex values for ASCII CR (0x0d) & LF (0x0a)
1041 * must be used instead of the escape sequences \r & \n.
1044 memcpy(data->state.scratch, k->upload_fromhere, offset);
1045 for(i = offset, si = offset; i < nread; i++, si++) {
1046 if(k->upload_fromhere[i] == 0x0a) {
1047 data->state.scratch[si++] = 0x0d;
1048 data->state.scratch[si] = 0x0a;
1049 if(!data->set.crlf) {
1050 /* we're here only because FTP is in ASCII mode...
1051 bump infilesize for the LF we just added */
1052 if(data->state.infilesize != -1)
1053 data->state.infilesize++;
1057 data->state.scratch[si] = k->upload_fromhere[i];
1061 /* only perform the special operation if we really did replace
1065 /* upload from the new (replaced) buffer instead */
1066 k->upload_fromhere = data->state.scratch;
1068 /* set the new amount too */
1069 k->upload_present = nread;
1073 #ifndef CURL_DISABLE_SMTP
1074 if(conn->handler->protocol & PROTO_FAMILY_SMTP) {
1075 result = Curl_smtp_escape_eob(data, nread, offset);
1079 #endif /* CURL_DISABLE_SMTP */
1080 } /* if 0 == k->upload_present or appended to upload buffer */
1082 /* We have a partial buffer left from a previous "round". Use
1083 that instead of reading more data */
1086 /* write to socket (send away data) */
1087 result = Curl_write(data,
1088 conn->writesockfd, /* socket to send to */
1089 k->upload_fromhere, /* buffer pointer */
1090 k->upload_present, /* buffer size */
1091 &bytes_written); /* actually sent */
1095 win_update_buffer_size(conn->writesockfd);
1097 if(k->pendingheader) {
1098 /* parts of what was sent was header */
1099 curl_off_t n = CURLMIN(k->pendingheader, bytes_written);
1100 /* show the data before we change the pointer upload_fromhere */
1101 Curl_debug(data, CURLINFO_HEADER_OUT, k->upload_fromhere, (size_t)n);
1102 k->pendingheader -= n;
1103 nbody = bytes_written - n; /* size of the written body part */
1106 nbody = bytes_written;
1109 /* show the data before we change the pointer upload_fromhere */
1110 Curl_debug(data, CURLINFO_DATA_OUT,
1111 &k->upload_fromhere[bytes_written - nbody],
1114 k->writebytecount += nbody;
1115 Curl_pgrsSetUploadCounter(data, k->writebytecount);
1118 if((!k->upload_chunky || k->forbidchunk) &&
1119 (k->writebytecount == data->state.infilesize)) {
1120 /* we have sent all data we were supposed to */
1121 k->upload_done = TRUE;
1122 infof(data, "We are completely uploaded and fine");
1125 if(k->upload_present != bytes_written) {
1126 /* we only wrote a part of the buffer (if anything), deal with it! */
1128 /* store the amount of bytes left in the buffer to write */
1129 k->upload_present -= bytes_written;
1131 /* advance the pointer where to find the buffer when the next send
1133 k->upload_fromhere += bytes_written;
1136 /* we've uploaded that buffer now */
1137 result = Curl_get_upload_buffer(data);
1140 k->upload_fromhere = data->state.ulbuf;
1141 k->upload_present = 0; /* no more bytes left */
1143 if(k->upload_done) {
1144 result = Curl_done_sending(data, k);
1151 } while(0); /* just to break out from! */
1157 * Curl_readwrite() is the low-level function to be called when data is to
1158 * be read and written to/from the connection.
1160 * return '*comeback' TRUE if we didn't properly drain the socket so this
1161 * function should get called again without select() or similar in between!
1163 CURLcode Curl_readwrite(struct connectdata *conn,
1164 struct Curl_easy *data,
1168 struct SingleRequest *k = &data->req;
1172 curl_socket_t fd_read;
1173 curl_socket_t fd_write;
1174 int select_res = conn->cselect_bits;
1176 conn->cselect_bits = 0;
1178 /* only use the proper socket if the *_HOLD bit is not set simultaneously as
1179 then we are in rate limiting state in that transfer direction */
1181 if((k->keepon & KEEP_RECVBITS) == KEEP_RECV)
1182 fd_read = conn->sockfd;
1184 fd_read = CURL_SOCKET_BAD;
1186 if((k->keepon & KEEP_SENDBITS) == KEEP_SEND)
1187 fd_write = conn->writesockfd;
1189 fd_write = CURL_SOCKET_BAD;
1191 #if defined(USE_HTTP2) || defined(USE_HTTP3)
1192 if(data->state.drain) {
1193 select_res |= CURL_CSELECT_IN;
1194 DEBUGF(infof(data, "Curl_readwrite: forcibly told to drain data"));
1198 if(!select_res) /* Call for select()/poll() only, if read/write/error
1199 status is not known. */
1200 select_res = Curl_socket_check(fd_read, CURL_SOCKET_BAD, fd_write, 0);
1202 if(select_res == CURL_CSELECT_ERR) {
1203 failf(data, "select/poll returned error");
1204 return CURLE_SEND_ERROR;
1208 if(conn->datastream) {
1209 result = conn->datastream(data, conn, &didwhat, done, select_res);
1215 /* We go ahead and do a read if we have a readable socket or if
1216 the stream was rewound (in which case we have data in a
1218 if((k->keepon & KEEP_RECV) && (select_res & CURL_CSELECT_IN)) {
1219 result = readwrite_data(data, conn, k, &didwhat, done, comeback);
1224 /* If we still have writing to do, we check if we have a writable socket. */
1225 if((k->keepon & KEEP_SEND) && (select_res & CURL_CSELECT_OUT)) {
1228 result = readwrite_upload(data, conn, &didwhat);
1236 k->now = Curl_now();
1238 /* no read no write, this is a timeout? */
1239 if(k->exp100 == EXP100_AWAITING_CONTINUE) {
1240 /* This should allow some time for the header to arrive, but only a
1241 very short time as otherwise it'll be too much wasted time too
1244 /* Quoting RFC2616, section "8.2.3 Use of the 100 (Continue) Status":
1246 Therefore, when a client sends this header field to an origin server
1247 (possibly via a proxy) from which it has never seen a 100 (Continue)
1248 status, the client SHOULD NOT wait for an indefinite period before
1249 sending the request body.
1253 timediff_t ms = Curl_timediff(k->now, k->start100);
1254 if(ms >= data->set.expect_100_timeout) {
1255 /* we've waited long enough, continue anyway */
1256 k->exp100 = EXP100_SEND_DATA;
1257 k->keepon |= KEEP_SEND;
1258 Curl_expire_done(data, EXPIRE_100_TIMEOUT);
1259 infof(data, "Done waiting for 100-continue");
1264 if(conn->transport == TRNSPRT_QUIC) {
1265 result = Curl_quic_idle(data);
1272 if(Curl_pgrsUpdate(data))
1273 result = CURLE_ABORTED_BY_CALLBACK;
1275 result = Curl_speedcheck(data, k->now);
1280 if(0 > Curl_timeleft(data, &k->now, FALSE)) {
1282 failf(data, "Operation timed out after %" CURL_FORMAT_TIMEDIFF_T
1283 " milliseconds with %" CURL_FORMAT_CURL_OFF_T " out of %"
1284 CURL_FORMAT_CURL_OFF_T " bytes received",
1285 Curl_timediff(k->now, data->progress.t_startsingle),
1286 k->bytecount, k->size);
1289 failf(data, "Operation timed out after %" CURL_FORMAT_TIMEDIFF_T
1290 " milliseconds with %" CURL_FORMAT_CURL_OFF_T " bytes received",
1291 Curl_timediff(k->now, data->progress.t_startsingle),
1294 return CURLE_OPERATION_TIMEDOUT;
1299 * The transfer has been performed. Just make some general checks before
1303 if(!(data->set.opt_no_body) && (k->size != -1) &&
1304 (k->bytecount != k->size) &&
1305 #ifdef CURL_DO_LINEEND_CONV
1306 /* Most FTP servers don't adjust their file SIZE response for CRLFs,
1307 so we'll check to see if the discrepancy can be explained
1308 by the number of CRLFs we've changed to LFs.
1310 (k->bytecount != (k->size + data->state.crlf_conversions)) &&
1311 #endif /* CURL_DO_LINEEND_CONV */
1313 failf(data, "transfer closed with %" CURL_FORMAT_CURL_OFF_T
1314 " bytes remaining to read", k->size - k->bytecount);
1315 return CURLE_PARTIAL_FILE;
1317 if(!(data->set.opt_no_body) && k->chunk &&
1318 (conn->chunk.state != CHUNK_STOP)) {
1320 * In chunked mode, return an error if the connection is closed prior to
1321 * the empty (terminating) chunk is read.
1323 * The condition above used to check for
1324 * conn->proto.http->chunk.datasize != 0 which is true after reading
1325 * *any* chunk, not just the empty chunk.
1328 failf(data, "transfer closed with outstanding read data remaining");
1329 return CURLE_PARTIAL_FILE;
1331 if(Curl_pgrsUpdate(data))
1332 return CURLE_ABORTED_BY_CALLBACK;
1335 /* Now update the "done" boolean we return */
1336 *done = (0 == (k->keepon&(KEEP_RECV|KEEP_SEND|
1337 KEEP_RECV_PAUSE|KEEP_SEND_PAUSE))) ? TRUE : FALSE;
1343 * Curl_single_getsock() gets called by the multi interface code when the app
1344 * has requested to get the sockets for the current connection. This function
1345 * will then be called once for every connection that the multi interface
1346 * keeps track of. This function will only be called for connections that are
1347 * in the proper state to have this information available.
1349 int Curl_single_getsock(struct Curl_easy *data,
1350 struct connectdata *conn,
1351 curl_socket_t *sock)
1353 int bitmap = GETSOCK_BLANK;
1354 unsigned sockindex = 0;
1356 if(conn->handler->perform_getsock)
1357 return conn->handler->perform_getsock(data, conn, sock);
1359 /* don't include HOLD and PAUSE connections */
1360 if((data->req.keepon & KEEP_RECVBITS) == KEEP_RECV) {
1362 DEBUGASSERT(conn->sockfd != CURL_SOCKET_BAD);
1364 bitmap |= GETSOCK_READSOCK(sockindex);
1365 sock[sockindex] = conn->sockfd;
1368 /* don't include HOLD and PAUSE connections */
1369 if((data->req.keepon & KEEP_SENDBITS) == KEEP_SEND) {
1371 if((conn->sockfd != conn->writesockfd) ||
1372 bitmap == GETSOCK_BLANK) {
1373 /* only if they are not the same socket and we have a readable
1374 one, we increase index */
1375 if(bitmap != GETSOCK_BLANK)
1376 sockindex++; /* increase index if we need two entries */
1378 DEBUGASSERT(conn->writesockfd != CURL_SOCKET_BAD);
1380 sock[sockindex] = conn->writesockfd;
1383 bitmap |= GETSOCK_WRITESOCK(sockindex);
1389 /* Curl_init_CONNECT() gets called each time the handle switches to CONNECT
1390 which means this gets called once for each subsequent redirect etc */
1391 void Curl_init_CONNECT(struct Curl_easy *data)
1393 data->state.fread_func = data->set.fread_func_set;
1394 data->state.in = data->set.in_set;
1398 * Curl_pretransfer() is called immediately before a transfer starts, and only
1399 * once for one transfer no matter if it has redirects or do multi-pass
1400 * authentication etc.
1402 CURLcode Curl_pretransfer(struct Curl_easy *data)
1406 if(!data->state.url && !data->set.uh) {
1407 /* we can't do anything without URL */
1408 failf(data, "No URL set");
1409 return CURLE_URL_MALFORMAT;
1412 /* since the URL may have been redirected in a previous use of this handle */
1413 if(data->state.url_alloc) {
1414 /* the already set URL is allocated, free it first! */
1415 Curl_safefree(data->state.url);
1416 data->state.url_alloc = FALSE;
1419 if(!data->state.url && data->set.uh) {
1421 free(data->set.str[STRING_SET_URL]);
1422 uc = curl_url_get(data->set.uh,
1423 CURLUPART_URL, &data->set.str[STRING_SET_URL], 0);
1425 failf(data, "No URL set");
1426 return CURLE_URL_MALFORMAT;
1430 data->state.prefer_ascii = data->set.prefer_ascii;
1431 data->state.list_only = data->set.list_only;
1432 data->state.httpreq = data->set.method;
1433 data->state.url = data->set.str[STRING_SET_URL];
1435 /* Init the SSL session ID cache here. We do it here since we want to do it
1436 after the *_setopt() calls (that could specify the size of the cache) but
1437 before any transfer takes place. */
1438 result = Curl_ssl_initsessions(data, data->set.general_ssl.max_ssl_sessions);
1442 data->state.requests = 0;
1443 data->state.followlocation = 0; /* reset the location-follow counter */
1444 data->state.this_is_a_follow = FALSE; /* reset this */
1445 data->state.errorbuf = FALSE; /* no error has occurred */
1446 data->state.httpwant = data->set.httpwant;
1447 data->state.httpversion = 0;
1448 data->state.authproblem = FALSE;
1449 data->state.authhost.want = data->set.httpauth;
1450 data->state.authproxy.want = data->set.proxyauth;
1451 Curl_safefree(data->info.wouldredirect);
1453 if(data->state.httpreq == HTTPREQ_PUT)
1454 data->state.infilesize = data->set.filesize;
1455 else if((data->state.httpreq != HTTPREQ_GET) &&
1456 (data->state.httpreq != HTTPREQ_HEAD)) {
1457 data->state.infilesize = data->set.postfieldsize;
1458 if(data->set.postfields && (data->state.infilesize == -1))
1459 data->state.infilesize = (curl_off_t)strlen(data->set.postfields);
1462 data->state.infilesize = 0;
1464 #ifndef CURL_DISABLE_COOKIES
1465 /* If there is a list of cookie files to read, do it now! */
1466 if(data->state.cookielist)
1467 Curl_cookie_loadfiles(data);
1469 /* If there is a list of host pairs to deal with */
1470 if(data->state.resolve)
1471 result = Curl_loadhostpairs(data);
1474 /* Allow data->set.use_port to set which port to use. This needs to be
1475 * disabled for example when we follow Location: headers to URLs using
1476 * different ports! */
1477 data->state.allow_port = TRUE;
1479 #if defined(HAVE_SIGNAL) && defined(SIGPIPE) && !defined(HAVE_MSG_NOSIGNAL)
1480 /*************************************************************
1481 * Tell signal handler to ignore SIGPIPE
1482 *************************************************************/
1483 if(!data->set.no_signal)
1484 data->state.prev_signal = signal(SIGPIPE, SIG_IGN);
1487 Curl_initinfo(data); /* reset session-specific information "variables" */
1488 Curl_pgrsResetTransferSizes(data);
1489 Curl_pgrsStartNow(data);
1491 /* In case the handle is re-used and an authentication method was picked
1492 in the session we need to make sure we only use the one(s) we now
1493 consider to be fine */
1494 data->state.authhost.picked &= data->state.authhost.want;
1495 data->state.authproxy.picked &= data->state.authproxy.want;
1497 #ifndef CURL_DISABLE_FTP
1498 data->state.wildcardmatch = data->set.wildcard_enabled;
1499 if(data->state.wildcardmatch) {
1500 struct WildcardData *wc = &data->wildcard;
1501 if(wc->state < CURLWC_INIT) {
1502 result = Curl_wildcard_init(wc); /* init wildcard structures */
1504 return CURLE_OUT_OF_MEMORY;
1508 Curl_http2_init_state(&data->state);
1509 result = Curl_hsts_loadcb(data, data->hsts);
1513 * Set user-agent. Used for HTTP, but since we can attempt to tunnel
1514 * basically anything through a http proxy we can't limit this based on
1517 if(data->set.str[STRING_USERAGENT]) {
1518 Curl_safefree(data->state.aptr.uagent);
1519 data->state.aptr.uagent =
1520 aprintf("User-Agent: %s\r\n", data->set.str[STRING_USERAGENT]);
1521 if(!data->state.aptr.uagent)
1522 return CURLE_OUT_OF_MEMORY;
1526 result = Curl_setstropt(&data->state.aptr.user,
1527 data->set.str[STRING_USERNAME]);
1529 result = Curl_setstropt(&data->state.aptr.passwd,
1530 data->set.str[STRING_PASSWORD]);
1532 result = Curl_setstropt(&data->state.aptr.proxyuser,
1533 data->set.str[STRING_PROXYUSERNAME]);
1535 result = Curl_setstropt(&data->state.aptr.proxypasswd,
1536 data->set.str[STRING_PROXYPASSWORD]);
1538 data->req.headerbytecount = 0;
1539 Curl_headers_cleanup(data);
1544 * Curl_posttransfer() is called immediately after a transfer ends
1546 CURLcode Curl_posttransfer(struct Curl_easy *data)
1548 #if defined(HAVE_SIGNAL) && defined(SIGPIPE) && !defined(HAVE_MSG_NOSIGNAL)
1549 /* restore the signal handler for SIGPIPE before we get back */
1550 if(!data->set.no_signal)
1551 signal(SIGPIPE, data->state.prev_signal);
1553 (void)data; /* unused parameter */
1560 * Curl_follow() handles the URL redirect magic. Pass in the 'newurl' string
1561 * as given by the remote server and set up the new URL to request.
1563 * This function DOES NOT FREE the given url.
1565 CURLcode Curl_follow(struct Curl_easy *data,
1566 char *newurl, /* the Location: string */
1567 followtype type) /* see transfer.h */
1569 #ifdef CURL_DISABLE_HTTP
1573 /* Location: following will not happen when HTTP is disabled */
1574 return CURLE_TOO_MANY_REDIRECTS;
1577 /* Location: redirect */
1578 bool disallowport = FALSE;
1579 bool reachedmax = FALSE;
1582 DEBUGASSERT(type != FOLLOW_NONE);
1584 if(type != FOLLOW_FAKE)
1585 data->state.requests++; /* count all real follows */
1586 if(type == FOLLOW_REDIR) {
1587 if((data->set.maxredirs != -1) &&
1588 (data->state.followlocation >= data->set.maxredirs)) {
1590 type = FOLLOW_FAKE; /* switch to fake to store the would-be-redirected
1594 /* mark the next request as a followed location: */
1595 data->state.this_is_a_follow = TRUE;
1597 data->state.followlocation++; /* count location-followers */
1599 if(data->set.http_auto_referer) {
1601 char *referer = NULL;
1603 /* We are asked to automatically set the previous URL as the referer
1604 when we get the next URL. We pick the ->url field, which may or may
1605 not be 100% correct */
1607 if(data->state.referer_alloc) {
1608 Curl_safefree(data->state.referer);
1609 data->state.referer_alloc = FALSE;
1612 /* Make a copy of the URL without credentials and fragment */
1615 return CURLE_OUT_OF_MEMORY;
1617 uc = curl_url_set(u, CURLUPART_URL, data->state.url, 0);
1619 uc = curl_url_set(u, CURLUPART_FRAGMENT, NULL, 0);
1621 uc = curl_url_set(u, CURLUPART_USER, NULL, 0);
1623 uc = curl_url_set(u, CURLUPART_PASSWORD, NULL, 0);
1625 uc = curl_url_get(u, CURLUPART_URL, &referer, 0);
1627 curl_url_cleanup(u);
1630 return CURLE_OUT_OF_MEMORY;
1632 data->state.referer = referer;
1633 data->state.referer_alloc = TRUE; /* yes, free this later */
1638 if((type != FOLLOW_RETRY) &&
1639 (data->req.httpcode != 401) && (data->req.httpcode != 407) &&
1640 Curl_is_absolute_url(newurl, NULL, 0, FALSE))
1641 /* If this is not redirect due to a 401 or 407 response and an absolute
1642 URL: don't allow a custom port number */
1643 disallowport = TRUE;
1645 DEBUGASSERT(data->state.uh);
1646 uc = curl_url_set(data->state.uh, CURLUPART_URL, newurl,
1647 (type == FOLLOW_FAKE) ? CURLU_NON_SUPPORT_SCHEME :
1648 ((type == FOLLOW_REDIR) ? CURLU_URLENCODE : 0) |
1650 (data->set.path_as_is ? CURLU_PATH_AS_IS : 0));
1652 if(type != FOLLOW_FAKE) {
1653 failf(data, "The redirect target URL could not be parsed: %s",
1654 curl_url_strerror(uc));
1655 return Curl_uc_to_curlcode(uc);
1658 /* the URL could not be parsed for some reason, but since this is FAKE
1659 mode, just duplicate the field as-is */
1660 newurl = strdup(newurl);
1662 return CURLE_OUT_OF_MEMORY;
1665 uc = curl_url_get(data->state.uh, CURLUPART_URL, &newurl, 0);
1667 return Curl_uc_to_curlcode(uc);
1669 /* Clear auth if this redirects to a different port number or protocol,
1671 if(!data->set.allow_auth_to_other_hosts && (type != FOLLOW_FAKE)) {
1676 if(data->set.use_port && data->state.allow_port)
1677 /* a custom port is used */
1678 port = (int)data->set.use_port;
1680 uc = curl_url_get(data->state.uh, CURLUPART_PORT, &portnum,
1681 CURLU_DEFAULT_PORT);
1684 return Curl_uc_to_curlcode(uc);
1686 port = atoi(portnum);
1689 if(port != data->info.conn_remote_port) {
1690 infof(data, "Clear auth, redirects to port from %u to %u",
1691 data->info.conn_remote_port, port);
1696 const struct Curl_handler *p;
1697 uc = curl_url_get(data->state.uh, CURLUPART_SCHEME, &scheme, 0);
1700 return Curl_uc_to_curlcode(uc);
1703 p = Curl_builtin_scheme(scheme, CURL_ZERO_TERMINATED);
1704 if(p && (p->protocol != data->info.conn_protocol)) {
1705 infof(data, "Clear auth, redirects scheme from %s to %s",
1706 data->info.conn_scheme, scheme);
1712 Curl_safefree(data->state.aptr.user);
1713 Curl_safefree(data->state.aptr.passwd);
1718 if(type == FOLLOW_FAKE) {
1719 /* we're only figuring out the new url if we would've followed locations
1720 but now we're done so we can get out! */
1721 data->info.wouldredirect = newurl;
1724 failf(data, "Maximum (%ld) redirects followed", data->set.maxredirs);
1725 return CURLE_TOO_MANY_REDIRECTS;
1731 data->state.allow_port = FALSE;
1733 if(data->state.url_alloc)
1734 Curl_safefree(data->state.url);
1736 data->state.url = newurl;
1737 data->state.url_alloc = TRUE;
1739 infof(data, "Issue another request to this URL: '%s'", data->state.url);
1742 * We get here when the HTTP code is 300-399 (and 401). We need to perform
1743 * differently based on exactly what return code there was.
1745 * News from 7.10.6: we can also get here on a 401 or 407, in case we act on
1746 * a HTTP (proxy-) authentication scheme other than Basic.
1748 switch(data->info.httpcode) {
1749 /* 401 - Act on a WWW-Authenticate, we keep on moving and do the
1750 Authorization: XXXX header in the HTTP request code snippet */
1751 /* 407 - Act on a Proxy-Authenticate, we keep on moving and do the
1752 Proxy-Authorization: XXXX header in the HTTP request code snippet */
1753 /* 300 - Multiple Choices */
1754 /* 306 - Not used */
1755 /* 307 - Temporary Redirect */
1756 default: /* for all above (and the unknown ones) */
1757 /* Some codes are explicitly mentioned since I've checked RFC2616 and they
1758 * seem to be OK to POST to.
1761 case 301: /* Moved Permanently */
1762 /* (quote from RFC7231, section 6.4.2)
1764 * Note: For historical reasons, a user agent MAY change the request
1765 * method from POST to GET for the subsequent request. If this
1766 * behavior is undesired, the 307 (Temporary Redirect) status code
1767 * can be used instead.
1771 * Many webservers expect this, so these servers often answers to a POST
1772 * request with an error page. To be sure that libcurl gets the page that
1773 * most user agents would get, libcurl has to force GET.
1775 * This behavior is forbidden by RFC1945 and the obsolete RFC2616, and
1776 * can be overridden with CURLOPT_POSTREDIR.
1778 if((data->state.httpreq == HTTPREQ_POST
1779 || data->state.httpreq == HTTPREQ_POST_FORM
1780 || data->state.httpreq == HTTPREQ_POST_MIME)
1781 && !(data->set.keep_post & CURL_REDIR_POST_301)) {
1782 infof(data, "Switch from POST to GET");
1783 data->state.httpreq = HTTPREQ_GET;
1786 case 302: /* Found */
1787 /* (quote from RFC7231, section 6.4.3)
1789 * Note: For historical reasons, a user agent MAY change the request
1790 * method from POST to GET for the subsequent request. If this
1791 * behavior is undesired, the 307 (Temporary Redirect) status code
1792 * can be used instead.
1796 * Many webservers expect this, so these servers often answers to a POST
1797 * request with an error page. To be sure that libcurl gets the page that
1798 * most user agents would get, libcurl has to force GET.
1800 * This behavior is forbidden by RFC1945 and the obsolete RFC2616, and
1801 * can be overridden with CURLOPT_POSTREDIR.
1803 if((data->state.httpreq == HTTPREQ_POST
1804 || data->state.httpreq == HTTPREQ_POST_FORM
1805 || data->state.httpreq == HTTPREQ_POST_MIME)
1806 && !(data->set.keep_post & CURL_REDIR_POST_302)) {
1807 infof(data, "Switch from POST to GET");
1808 data->state.httpreq = HTTPREQ_GET;
1812 case 303: /* See Other */
1813 /* 'See Other' location is not the resource but a substitute for the
1814 * resource. In this case we switch the method to GET/HEAD, unless the
1815 * method is POST and the user specified to keep it as POST.
1816 * https://github.com/curl/curl/issues/5237#issuecomment-614641049
1818 if(data->state.httpreq != HTTPREQ_GET &&
1819 ((data->state.httpreq != HTTPREQ_POST &&
1820 data->state.httpreq != HTTPREQ_POST_FORM &&
1821 data->state.httpreq != HTTPREQ_POST_MIME) ||
1822 !(data->set.keep_post & CURL_REDIR_POST_303))) {
1823 data->state.httpreq = HTTPREQ_GET;
1824 data->set.upload = false;
1825 infof(data, "Switch to %s",
1826 data->set.opt_no_body?"HEAD":"GET");
1829 case 304: /* Not Modified */
1830 /* 304 means we did a conditional request and it was "Not modified".
1831 * We shouldn't get any Location: header in this response!
1834 case 305: /* Use Proxy */
1835 /* (quote from RFC2616, section 10.3.6):
1836 * "The requested resource MUST be accessed through the proxy given
1837 * by the Location field. The Location field gives the URI of the
1838 * proxy. The recipient is expected to repeat this single request
1839 * via the proxy. 305 responses MUST only be generated by origin
1844 Curl_pgrsTime(data, TIMER_REDIRECT);
1845 Curl_pgrsResetTransferSizes(data);
1848 #endif /* CURL_DISABLE_HTTP */
1851 /* Returns CURLE_OK *and* sets '*url' if a request retry is wanted.
1853 NOTE: that the *url is malloc()ed. */
1854 CURLcode Curl_retry_request(struct Curl_easy *data, char **url)
1856 struct connectdata *conn = data->conn;
1860 /* if we're talking upload, we can't do the checks below, unless the protocol
1861 is HTTP as when uploading over HTTP we will still get a response */
1862 if(data->set.upload &&
1863 !(conn->handler->protocol&(PROTO_FAMILY_HTTP|CURLPROTO_RTSP)))
1866 if((data->req.bytecount + data->req.headerbytecount == 0) &&
1868 (!data->set.opt_no_body || (conn->handler->protocol & PROTO_FAMILY_HTTP))
1869 #ifndef CURL_DISABLE_RTSP
1870 && (data->set.rtspreq != RTSPREQ_RECEIVE)
1873 /* We got no data, we attempted to re-use a connection. For HTTP this
1874 can be a retry so we try again regardless if we expected a body.
1875 For other protocols we only try again only if we expected a body.
1877 This might happen if the connection was left alive when we were
1878 done using it before, but that was closed when we wanted to read from
1879 it again. Bad luck. Retry the same request on a fresh connect! */
1881 else if(data->state.refused_stream &&
1882 (data->req.bytecount + data->req.headerbytecount == 0) ) {
1883 /* This was sent on a refused stream, safe to rerun. A refused stream
1884 error can typically only happen on HTTP/2 level if the stream is safe
1885 to issue again, but the nghttp2 API can deliver the message to other
1886 streams as well, which is why this adds the check the data counters
1888 infof(data, "REFUSED_STREAM, retrying a fresh connect");
1889 data->state.refused_stream = FALSE; /* clear again */
1893 #define CONN_MAX_RETRIES 5
1894 if(data->state.retrycount++ >= CONN_MAX_RETRIES) {
1895 failf(data, "Connection died, tried %d times before giving up",
1897 data->state.retrycount = 0;
1898 return CURLE_SEND_ERROR;
1900 infof(data, "Connection died, retrying a fresh connect (retry count: %d)",
1901 data->state.retrycount);
1902 *url = strdup(data->state.url);
1904 return CURLE_OUT_OF_MEMORY;
1906 connclose(conn, "retry"); /* close this connection */
1907 conn->bits.retry = TRUE; /* mark this as a connection we're about
1908 to retry. Marking it this way should
1909 prevent i.e HTTP transfers to return
1910 error just because nothing has been
1914 if(conn->handler->protocol&PROTO_FAMILY_HTTP) {
1915 if(data->req.writebytecount) {
1916 CURLcode result = Curl_readrewind(data);
1918 Curl_safefree(*url);
1928 * Curl_setup_transfer() is called to setup some basic properties for the
1929 * upcoming transfer.
1932 Curl_setup_transfer(
1933 struct Curl_easy *data, /* transfer */
1934 int sockindex, /* socket index to read from or -1 */
1935 curl_off_t size, /* -1 if unknown at this point */
1936 bool getheader, /* TRUE if header parsing is wanted */
1937 int writesockindex /* socket index to write to, it may very well be
1938 the same we read from. -1 disables */
1941 struct SingleRequest *k = &data->req;
1942 struct connectdata *conn = data->conn;
1943 struct HTTP *http = data->req.p.http;
1946 DEBUGASSERT(conn != NULL);
1947 DEBUGASSERT((sockindex <= 1) && (sockindex >= -1));
1949 httpsending = ((conn->handler->protocol&PROTO_FAMILY_HTTP) &&
1950 (http->sending == HTTPSEND_REQUEST));
1952 if(conn->bits.multiplex || conn->httpversion == 20 || httpsending) {
1953 /* when multiplexing, the read/write sockets need to be the same! */
1954 conn->sockfd = sockindex == -1 ?
1955 ((writesockindex == -1 ? CURL_SOCKET_BAD : conn->sock[writesockindex])) :
1956 conn->sock[sockindex];
1957 conn->writesockfd = conn->sockfd;
1959 /* special and very HTTP-specific */
1960 writesockindex = FIRSTSOCKET;
1963 conn->sockfd = sockindex == -1 ?
1964 CURL_SOCKET_BAD : conn->sock[sockindex];
1965 conn->writesockfd = writesockindex == -1 ?
1966 CURL_SOCKET_BAD:conn->sock[writesockindex];
1968 k->getheader = getheader;
1972 /* The code sequence below is placed in this function just because all
1973 necessary input is not always known in do_complete() as this function may
1974 be called after that */
1979 Curl_pgrsSetDownloadSize(data, size);
1981 /* we want header and/or body, if neither then don't do this! */
1982 if(k->getheader || !data->set.opt_no_body) {
1985 k->keepon |= KEEP_RECV;
1987 if(writesockindex != -1) {
1990 Even if we require a 100-return code before uploading data, we might
1991 need to write data before that since the REQUEST may not have been
1992 finished sent off just yet.
1994 Thus, we must check if the request has been sent before we set the
1995 state info where we wait for the 100-return code
1997 if((data->state.expect100header) &&
1998 (conn->handler->protocol&PROTO_FAMILY_HTTP) &&
1999 (http->sending == HTTPSEND_BODY)) {
2000 /* wait with write until we either got 100-continue or a timeout */
2001 k->exp100 = EXP100_AWAITING_CONTINUE;
2002 k->start100 = Curl_now();
2004 /* Set a timeout for the multi interface. Add the inaccuracy margin so
2005 that we don't fire slightly too early and get denied to run. */
2006 Curl_expire(data, data->set.expect_100_timeout, EXPIRE_100_TIMEOUT);
2009 if(data->state.expect100header)
2010 /* when we've sent off the rest of the headers, we must await a
2011 100-continue but first finish sending the request */
2012 k->exp100 = EXP100_SENDING_REQUEST;
2014 /* enable the write bit when we're not waiting for continue */
2015 k->keepon |= KEEP_SEND;
2017 } /* if(writesockindex != -1) */
2018 } /* if(k->getheader || !data->set.opt_no_body) */