• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /***************************************************************************
2  *                                  _   _ ____  _
3  *  Project                     ___| | | |  _ \| |
4  *                             / __| | | | |_) | |
5  *                            | (__| |_| |  _ <| |___
6  *                             \___|\___/|_| \_\_____|
7  *
8  * Copyright (C) 1998 - 2016, Daniel Stenberg, <daniel@haxx.se>, et al.
9  *
10  * This software is licensed as described in the file COPYING, which
11  * you should have received as part of this distribution. The terms
12  * are also available at https://curl.haxx.se/docs/copyright.html.
13  *
14  * You may opt to use, copy, modify, merge, publish, distribute and/or sell
15  * copies of the Software, and permit persons to whom the Software is
16  * furnished to do so, under the terms of the COPYING file.
17  *
18  * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
19  * KIND, either express or implied.
20  *
21  ***************************************************************************/
22 
23 #include "curl_setup.h"
24 #include "strtoofft.h"
25 
26 #ifdef HAVE_NETINET_IN_H
27 #include <netinet/in.h>
28 #endif
29 #ifdef HAVE_NETDB_H
30 #include <netdb.h>
31 #endif
32 #ifdef HAVE_ARPA_INET_H
33 #include <arpa/inet.h>
34 #endif
35 #ifdef HAVE_NET_IF_H
36 #include <net/if.h>
37 #endif
38 #ifdef HAVE_SYS_IOCTL_H
39 #include <sys/ioctl.h>
40 #endif
41 #ifdef HAVE_SIGNAL_H
42 #include <signal.h>
43 #endif
44 
45 #ifdef HAVE_SYS_PARAM_H
46 #include <sys/param.h>
47 #endif
48 
49 #ifdef HAVE_SYS_SELECT_H
50 #include <sys/select.h>
51 #endif
52 
53 #ifndef HAVE_SOCKET
54 #error "We can't compile without socket() support!"
55 #endif
56 
57 #include "urldata.h"
58 #include <curl/curl.h>
59 #include "netrc.h"
60 
61 #include "content_encoding.h"
62 #include "hostip.h"
63 #include "transfer.h"
64 #include "sendf.h"
65 #include "speedcheck.h"
66 #include "progress.h"
67 #include "http.h"
68 #include "url.h"
69 #include "getinfo.h"
70 #include "vtls/vtls.h"
71 #include "select.h"
72 #include "multiif.h"
73 #include "connect.h"
74 #include "non-ascii.h"
75 #include "http2.h"
76 
77 /* The last 3 #include files should be in this order */
78 #include "curl_printf.h"
79 #include "curl_memory.h"
80 #include "memdebug.h"
81 
82 /*
83  * This function will call the read callback to fill our buffer with data
84  * to upload.
85  */
Curl_fillreadbuffer(struct connectdata * conn,int bytes,int * nreadp)86 CURLcode Curl_fillreadbuffer(struct connectdata *conn, int bytes, int *nreadp)
87 {
88   struct Curl_easy *data = conn->data;
89   size_t buffersize = (size_t)bytes;
90   int nread;
91 #ifdef CURL_DOES_CONVERSIONS
92   bool sending_http_headers = FALSE;
93 
94   if(conn->handler->protocol&(PROTO_FAMILY_HTTP|CURLPROTO_RTSP)) {
95     const struct HTTP *http = data->req.protop;
96 
97     if(http->sending == HTTPSEND_REQUEST)
98       /* We're sending the HTTP request headers, not the data.
99          Remember that so we don't re-translate them into garbage. */
100       sending_http_headers = TRUE;
101   }
102 #endif
103 
104   if(data->req.upload_chunky) {
105     /* if chunked Transfer-Encoding */
106     buffersize -= (8 + 2 + 2);   /* 32bit hex + CRLF + CRLF */
107     data->req.upload_fromhere += (8 + 2); /* 32bit hex + CRLF */
108   }
109 
110   /* this function returns a size_t, so we typecast to int to prevent warnings
111      with picky compilers */
112   nread = (int)data->state.fread_func(data->req.upload_fromhere, 1,
113                                       buffersize, data->state.in);
114 
115   if(nread == CURL_READFUNC_ABORT) {
116     failf(data, "operation aborted by callback");
117     *nreadp = 0;
118     return CURLE_ABORTED_BY_CALLBACK;
119   }
120   else if(nread == CURL_READFUNC_PAUSE) {
121 
122     if(conn->handler->flags & PROTOPT_NONETWORK) {
123       /* protocols that work without network cannot be paused. This is
124          actually only FILE:// just now, and it can't pause since the transfer
125          isn't done using the "normal" procedure. */
126       failf(data, "Read callback asked for PAUSE when not supported!");
127       return CURLE_READ_ERROR;
128     }
129     else {
130       struct SingleRequest *k = &data->req;
131       /* CURL_READFUNC_PAUSE pauses read callbacks that feed socket writes */
132       k->keepon |= KEEP_SEND_PAUSE; /* mark socket send as paused */
133       if(data->req.upload_chunky) {
134         /* Back out the preallocation done above */
135         data->req.upload_fromhere -= (8 + 2);
136       }
137       *nreadp = 0;
138     }
139     return CURLE_OK; /* nothing was read */
140   }
141   else if((size_t)nread > buffersize) {
142     /* the read function returned a too large value */
143     *nreadp = 0;
144     failf(data, "read function returned funny value");
145     return CURLE_READ_ERROR;
146   }
147 
148   if(!data->req.forbidchunk && data->req.upload_chunky) {
149     /* if chunked Transfer-Encoding
150      *    build chunk:
151      *
152      *        <HEX SIZE> CRLF
153      *        <DATA> CRLF
154      */
155     /* On non-ASCII platforms the <DATA> may or may not be
156        translated based on set.prefer_ascii while the protocol
157        portion must always be translated to the network encoding.
158        To further complicate matters, line end conversion might be
159        done later on, so we need to prevent CRLFs from becoming
160        CRCRLFs if that's the case.  To do this we use bare LFs
161        here, knowing they'll become CRLFs later on.
162      */
163 
164     char hexbuffer[11];
165     const char *endofline_native;
166     const char *endofline_network;
167     int hexlen;
168 
169     if(
170 #ifdef CURL_DO_LINEEND_CONV
171        (data->set.prefer_ascii) ||
172 #endif
173        (data->set.crlf)) {
174       /* \n will become \r\n later on */
175       endofline_native  = "\n";
176       endofline_network = "\x0a";
177     }
178     else {
179       endofline_native  = "\r\n";
180       endofline_network = "\x0d\x0a";
181     }
182     hexlen = snprintf(hexbuffer, sizeof(hexbuffer),
183                       "%x%s", nread, endofline_native);
184 
185     /* move buffer pointer */
186     data->req.upload_fromhere -= hexlen;
187     nread += hexlen;
188 
189     /* copy the prefix to the buffer, leaving out the NUL */
190     memcpy(data->req.upload_fromhere, hexbuffer, hexlen);
191 
192     /* always append ASCII CRLF to the data */
193     memcpy(data->req.upload_fromhere + nread,
194            endofline_network,
195            strlen(endofline_network));
196 
197 #ifdef CURL_DOES_CONVERSIONS
198     CURLcode result;
199     int length;
200     if(data->set.prefer_ascii) {
201       /* translate the protocol and data */
202       length = nread;
203     }
204     else {
205       /* just translate the protocol portion */
206       length = strlen(hexbuffer);
207     }
208     result = Curl_convert_to_network(data, data->req.upload_fromhere, length);
209     /* Curl_convert_to_network calls failf if unsuccessful */
210     if(result)
211       return result;
212 #endif /* CURL_DOES_CONVERSIONS */
213 
214     if((nread - hexlen) == 0)
215       /* mark this as done once this chunk is transferred */
216       data->req.upload_done = TRUE;
217 
218     nread+=(int)strlen(endofline_native); /* for the added end of line */
219   }
220 #ifdef CURL_DOES_CONVERSIONS
221   else if((data->set.prefer_ascii) && (!sending_http_headers)) {
222     CURLcode result;
223     result = Curl_convert_to_network(data, data->req.upload_fromhere, nread);
224     /* Curl_convert_to_network calls failf if unsuccessful */
225     if(result)
226       return result;
227   }
228 #endif /* CURL_DOES_CONVERSIONS */
229 
230   *nreadp = nread;
231 
232   return CURLE_OK;
233 }
234 
235 
236 /*
237  * Curl_readrewind() rewinds the read stream. This is typically used for HTTP
238  * POST/PUT with multi-pass authentication when a sending was denied and a
239  * resend is necessary.
240  */
Curl_readrewind(struct connectdata * conn)241 CURLcode Curl_readrewind(struct connectdata *conn)
242 {
243   struct Curl_easy *data = conn->data;
244 
245   conn->bits.rewindaftersend = FALSE; /* we rewind now */
246 
247   /* explicitly switch off sending data on this connection now since we are
248      about to restart a new transfer and thus we want to avoid inadvertently
249      sending more data on the existing connection until the next transfer
250      starts */
251   data->req.keepon &= ~KEEP_SEND;
252 
253   /* We have sent away data. If not using CURLOPT_POSTFIELDS or
254      CURLOPT_HTTPPOST, call app to rewind
255   */
256   if(data->set.postfields ||
257      (data->set.httpreq == HTTPREQ_POST_FORM))
258     ; /* do nothing */
259   else {
260     if(data->set.seek_func) {
261       int err;
262 
263       err = (data->set.seek_func)(data->set.seek_client, 0, SEEK_SET);
264       if(err) {
265         failf(data, "seek callback returned error %d", (int)err);
266         return CURLE_SEND_FAIL_REWIND;
267       }
268     }
269     else if(data->set.ioctl_func) {
270       curlioerr err;
271 
272       err = (data->set.ioctl_func)(data, CURLIOCMD_RESTARTREAD,
273                                    data->set.ioctl_client);
274       infof(data, "the ioctl callback returned %d\n", (int)err);
275 
276       if(err) {
277         /* FIXME: convert to a human readable error message */
278         failf(data, "ioctl callback returned error %d", (int)err);
279         return CURLE_SEND_FAIL_REWIND;
280       }
281     }
282     else {
283       /* If no CURLOPT_READFUNCTION is used, we know that we operate on a
284          given FILE * stream and we can actually attempt to rewind that
285          ourselves with fseek() */
286       if(data->state.fread_func == (curl_read_callback)fread) {
287         if(-1 != fseek(data->state.in, 0, SEEK_SET))
288           /* successful rewind */
289           return CURLE_OK;
290       }
291 
292       /* no callback set or failure above, makes us fail at once */
293       failf(data, "necessary data rewind wasn't possible");
294       return CURLE_SEND_FAIL_REWIND;
295     }
296   }
297   return CURLE_OK;
298 }
299 
data_pending(const struct connectdata * conn)300 static int data_pending(const struct connectdata *conn)
301 {
302   /* in the case of libssh2, we can never be really sure that we have emptied
303      its internal buffers so we MUST always try until we get EAGAIN back */
304   return conn->handler->protocol&(CURLPROTO_SCP|CURLPROTO_SFTP) ||
305 #if defined(USE_NGHTTP2)
306     Curl_ssl_data_pending(conn, FIRSTSOCKET) ||
307     /* For HTTP/2, we may read up everything including responde body
308        with header fields in Curl_http_readwrite_headers. If no
309        content-length is provided, curl waits for the connection
310        close, which we emulate it using conn->proto.httpc.closed =
311        TRUE. The thing is if we read everything, then http2_recv won't
312        be called and we cannot signal the HTTP/2 stream has closed. As
313        a workaround, we return nonzero here to call http2_recv. */
314     ((conn->handler->protocol&PROTO_FAMILY_HTTP) && conn->httpversion == 20);
315 #else
316     Curl_ssl_data_pending(conn, FIRSTSOCKET);
317 #endif
318 }
319 
read_rewind(struct connectdata * conn,size_t thismuch)320 static void read_rewind(struct connectdata *conn,
321                         size_t thismuch)
322 {
323   DEBUGASSERT(conn->read_pos >= thismuch);
324 
325   conn->read_pos -= thismuch;
326   conn->bits.stream_was_rewound = TRUE;
327 
328 #ifdef DEBUGBUILD
329   {
330     char buf[512 + 1];
331     size_t show;
332 
333     show = CURLMIN(conn->buf_len - conn->read_pos, sizeof(buf)-1);
334     if(conn->master_buffer) {
335       memcpy(buf, conn->master_buffer + conn->read_pos, show);
336       buf[show] = '\0';
337     }
338     else {
339       buf[0] = '\0';
340     }
341 
342     DEBUGF(infof(conn->data,
343                  "Buffer after stream rewind (read_pos = %zu): [%s]\n",
344                  conn->read_pos, buf));
345   }
346 #endif
347 }
348 
349 /*
350  * Check to see if CURLOPT_TIMECONDITION was met by comparing the time of the
351  * remote document with the time provided by CURLOPT_TIMEVAL
352  */
Curl_meets_timecondition(struct Curl_easy * data,time_t timeofdoc)353 bool Curl_meets_timecondition(struct Curl_easy *data, time_t timeofdoc)
354 {
355   if((timeofdoc == 0) || (data->set.timevalue == 0))
356     return TRUE;
357 
358   switch(data->set.timecondition) {
359   case CURL_TIMECOND_IFMODSINCE:
360   default:
361     if(timeofdoc <= data->set.timevalue) {
362       infof(data,
363             "The requested document is not new enough\n");
364       data->info.timecond = TRUE;
365       return FALSE;
366     }
367     break;
368   case CURL_TIMECOND_IFUNMODSINCE:
369     if(timeofdoc >= data->set.timevalue) {
370       infof(data,
371             "The requested document is not old enough\n");
372       data->info.timecond = TRUE;
373       return FALSE;
374     }
375     break;
376   }
377 
378   return TRUE;
379 }
380 
381 /*
382  * Go ahead and do a read if we have a readable socket or if
383  * the stream was rewound (in which case we have data in a
384  * buffer)
385  *
386  * return '*comeback' TRUE if we didn't properly drain the socket so this
387  * function should get called again without select() or similar in between!
388  */
readwrite_data(struct Curl_easy * data,struct connectdata * conn,struct SingleRequest * k,int * didwhat,bool * done,bool * comeback)389 static CURLcode readwrite_data(struct Curl_easy *data,
390                                struct connectdata *conn,
391                                struct SingleRequest *k,
392                                int *didwhat, bool *done,
393                                bool *comeback)
394 {
395   CURLcode result = CURLE_OK;
396   ssize_t nread; /* number of bytes read */
397   size_t excess = 0; /* excess bytes read */
398   bool is_empty_data = FALSE;
399   bool readmore = FALSE; /* used by RTP to signal for more data */
400   int maxloops = 100;
401 
402   *done = FALSE;
403   *comeback = FALSE;
404 
405   /* This is where we loop until we have read everything there is to
406      read or we get a CURLE_AGAIN */
407   do {
408     size_t buffersize = data->set.buffer_size?
409       data->set.buffer_size : BUFSIZE;
410     size_t bytestoread = buffersize;
411 
412     if(
413 #if defined(USE_NGHTTP2)
414        /* For HTTP/2, read data without caring about the content
415           length. This is safe because body in HTTP/2 is always
416           segmented thanks to its framing layer. Meanwhile, we have to
417           call Curl_read to ensure that http2_handle_stream_close is
418           called when we read all incoming bytes for a particular
419           stream. */
420        !((conn->handler->protocol & PROTO_FAMILY_HTTP) &&
421          conn->httpversion == 20) &&
422 #endif
423        k->size != -1 && !k->header) {
424       /* make sure we don't read "too much" if we can help it since we
425          might be pipelining and then someone else might want to read what
426          follows! */
427       curl_off_t totalleft = k->size - k->bytecount;
428       if(totalleft < (curl_off_t)bytestoread)
429         bytestoread = (size_t)totalleft;
430     }
431 
432     if(bytestoread) {
433       /* receive data from the network! */
434       result = Curl_read(conn, conn->sockfd, k->buf, bytestoread, &nread);
435 
436       /* read would've blocked */
437       if(CURLE_AGAIN == result)
438         break; /* get out of loop */
439 
440       if(result>0)
441         return result;
442     }
443     else {
444       /* read nothing but since we wanted nothing we consider this an OK
445          situation to proceed from */
446       DEBUGF(infof(data, "readwrite_data: we're done!\n"));
447       nread = 0;
448     }
449 
450     if((k->bytecount == 0) && (k->writebytecount == 0)) {
451       Curl_pgrsTime(data, TIMER_STARTTRANSFER);
452       if(k->exp100 > EXP100_SEND_DATA)
453         /* set time stamp to compare with when waiting for the 100 */
454         k->start100 = Curl_tvnow();
455     }
456 
457     *didwhat |= KEEP_RECV;
458     /* indicates data of zero size, i.e. empty file */
459     is_empty_data = ((nread == 0) && (k->bodywrites == 0)) ? TRUE : FALSE;
460 
461     /* NUL terminate, allowing string ops to be used */
462     if(0 < nread || is_empty_data) {
463       k->buf[nread] = 0;
464     }
465     else if(0 >= nread) {
466       /* if we receive 0 or less here, the server closed the connection
467          and we bail out from this! */
468       DEBUGF(infof(data, "nread <= 0, server closed connection, bailing\n"));
469       k->keepon &= ~KEEP_RECV;
470       break;
471     }
472 
473     /* Default buffer to use when we write the buffer, it may be changed
474        in the flow below before the actual storing is done. */
475     k->str = k->buf;
476 
477     if(conn->handler->readwrite) {
478       result = conn->handler->readwrite(data, conn, &nread, &readmore);
479       if(result)
480         return result;
481       if(readmore)
482         break;
483     }
484 
485 #ifndef CURL_DISABLE_HTTP
486     /* Since this is a two-state thing, we check if we are parsing
487        headers at the moment or not. */
488     if(k->header) {
489       /* we are in parse-the-header-mode */
490       bool stop_reading = FALSE;
491       result = Curl_http_readwrite_headers(data, conn, &nread, &stop_reading);
492       if(result)
493         return result;
494 
495       if(conn->handler->readwrite &&
496          (k->maxdownload <= 0 && nread > 0)) {
497         result = conn->handler->readwrite(data, conn, &nread, &readmore);
498         if(result)
499           return result;
500         if(readmore)
501           break;
502       }
503 
504       if(stop_reading) {
505         /* We've stopped dealing with input, get out of the do-while loop */
506 
507         if(nread > 0) {
508           if(Curl_pipeline_wanted(conn->data->multi, CURLPIPE_HTTP1)) {
509             infof(data,
510                   "Rewinding stream by : %zd"
511                   " bytes on url %s (zero-length body)\n",
512                   nread, data->state.path);
513             read_rewind(conn, (size_t)nread);
514           }
515           else {
516             infof(data,
517                   "Excess found in a non pipelined read:"
518                   " excess = %zd"
519                   " url = %s (zero-length body)\n",
520                   nread, data->state.path);
521           }
522         }
523 
524         break;
525       }
526     }
527 #endif /* CURL_DISABLE_HTTP */
528 
529 
530     /* This is not an 'else if' since it may be a rest from the header
531        parsing, where the beginning of the buffer is headers and the end
532        is non-headers. */
533     if(k->str && !k->header && (nread > 0 || is_empty_data)) {
534 
535       if(data->set.opt_no_body) {
536         /* data arrives although we want none, bail out */
537         streamclose(conn, "ignoring body");
538         *done = TRUE;
539         return CURLE_WEIRD_SERVER_REPLY;
540       }
541 
542 #ifndef CURL_DISABLE_HTTP
543       if(0 == k->bodywrites && !is_empty_data) {
544         /* These checks are only made the first time we are about to
545            write a piece of the body */
546         if(conn->handler->protocol&(PROTO_FAMILY_HTTP|CURLPROTO_RTSP)) {
547           /* HTTP-only checks */
548 
549           if(data->req.newurl) {
550             if(conn->bits.close) {
551               /* Abort after the headers if "follow Location" is set
552                  and we're set to close anyway. */
553               k->keepon &= ~KEEP_RECV;
554               *done = TRUE;
555               return CURLE_OK;
556             }
557             /* We have a new url to load, but since we want to be able
558                to re-use this connection properly, we read the full
559                response in "ignore more" */
560             k->ignorebody = TRUE;
561             infof(data, "Ignoring the response-body\n");
562           }
563           if(data->state.resume_from && !k->content_range &&
564              (data->set.httpreq==HTTPREQ_GET) &&
565              !k->ignorebody) {
566 
567             if(k->size == data->state.resume_from) {
568               /* The resume point is at the end of file, consider this fine
569                  even if it doesn't allow resume from here. */
570               infof(data, "The entire document is already downloaded");
571               connclose(conn, "already downloaded");
572               /* Abort download */
573               k->keepon &= ~KEEP_RECV;
574               *done = TRUE;
575               return CURLE_OK;
576             }
577 
578             /* we wanted to resume a download, although the server doesn't
579              * seem to support this and we did this with a GET (if it
580              * wasn't a GET we did a POST or PUT resume) */
581             failf(data, "HTTP server doesn't seem to support "
582                   "byte ranges. Cannot resume.");
583             return CURLE_RANGE_ERROR;
584           }
585 
586           if(data->set.timecondition && !data->state.range) {
587             /* A time condition has been set AND no ranges have been
588                requested. This seems to be what chapter 13.3.4 of
589                RFC 2616 defines to be the correct action for a
590                HTTP/1.1 client */
591 
592             if(!Curl_meets_timecondition(data, k->timeofdoc)) {
593               *done = TRUE;
594               /* We're simulating a http 304 from server so we return
595                  what should have been returned from the server */
596               data->info.httpcode = 304;
597               infof(data, "Simulate a HTTP 304 response!\n");
598               /* we abort the transfer before it is completed == we ruin the
599                  re-use ability. Close the connection */
600               connclose(conn, "Simulated 304 handling");
601               return CURLE_OK;
602             }
603           } /* we have a time condition */
604 
605         } /* this is HTTP or RTSP */
606       } /* this is the first time we write a body part */
607 #endif /* CURL_DISABLE_HTTP */
608 
609       k->bodywrites++;
610 
611       /* pass data to the debug function before it gets "dechunked" */
612       if(data->set.verbose) {
613         if(k->badheader) {
614           Curl_debug(data, CURLINFO_DATA_IN, data->state.headerbuff,
615                      (size_t)k->hbuflen, conn);
616           if(k->badheader == HEADER_PARTHEADER)
617             Curl_debug(data, CURLINFO_DATA_IN,
618                        k->str, (size_t)nread, conn);
619         }
620         else
621           Curl_debug(data, CURLINFO_DATA_IN,
622                      k->str, (size_t)nread, conn);
623       }
624 
625 #ifndef CURL_DISABLE_HTTP
626       if(k->chunk) {
627         /*
628          * Here comes a chunked transfer flying and we need to decode this
629          * properly.  While the name says read, this function both reads
630          * and writes away the data. The returned 'nread' holds the number
631          * of actual data it wrote to the client.
632          */
633 
634         CHUNKcode res =
635           Curl_httpchunk_read(conn, k->str, nread, &nread);
636 
637         if(CHUNKE_OK < res) {
638           if(CHUNKE_WRITE_ERROR == res) {
639             failf(data, "Failed writing data");
640             return CURLE_WRITE_ERROR;
641           }
642           failf(data, "%s in chunked-encoding", Curl_chunked_strerror(res));
643           return CURLE_RECV_ERROR;
644         }
645         else if(CHUNKE_STOP == res) {
646           size_t dataleft;
647           /* we're done reading chunks! */
648           k->keepon &= ~KEEP_RECV; /* read no more */
649 
650           /* There are now possibly N number of bytes at the end of the
651              str buffer that weren't written to the client.
652 
653              We DO care about this data if we are pipelining.
654              Push it back to be read on the next pass. */
655 
656           dataleft = conn->chunk.dataleft;
657           if(dataleft != 0) {
658             infof(conn->data, "Leftovers after chunking: %zu bytes\n",
659                   dataleft);
660             if(Curl_pipeline_wanted(conn->data->multi, CURLPIPE_HTTP1)) {
661               /* only attempt the rewind if we truly are pipelining */
662               infof(conn->data, "Rewinding %zu bytes\n",dataleft);
663               read_rewind(conn, dataleft);
664             }
665           }
666         }
667         /* If it returned OK, we just keep going */
668       }
669 #endif   /* CURL_DISABLE_HTTP */
670 
671       /* Account for body content stored in the header buffer */
672       if(k->badheader && !k->ignorebody) {
673         DEBUGF(infof(data, "Increasing bytecount by %zu from hbuflen\n",
674                      k->hbuflen));
675         k->bytecount += k->hbuflen;
676       }
677 
678       if((-1 != k->maxdownload) &&
679          (k->bytecount + nread >= k->maxdownload)) {
680 
681         excess = (size_t)(k->bytecount + nread - k->maxdownload);
682         if(excess > 0 && !k->ignorebody) {
683           if(Curl_pipeline_wanted(conn->data->multi, CURLPIPE_HTTP1)) {
684             /* The 'excess' amount below can't be more than BUFSIZE which
685                always will fit in a size_t */
686             infof(data,
687                   "Rewinding stream by : %zu"
688                   " bytes on url %s (size = %" CURL_FORMAT_CURL_OFF_T
689                   ", maxdownload = %" CURL_FORMAT_CURL_OFF_T
690                   ", bytecount = %" CURL_FORMAT_CURL_OFF_T ", nread = %zd)\n",
691                   excess, data->state.path,
692                   k->size, k->maxdownload, k->bytecount, nread);
693             read_rewind(conn, excess);
694           }
695           else {
696             infof(data,
697                   "Excess found in a non pipelined read:"
698                   " excess = %zu"
699                   ", size = %" CURL_FORMAT_CURL_OFF_T
700                   ", maxdownload = %" CURL_FORMAT_CURL_OFF_T
701                   ", bytecount = %" CURL_FORMAT_CURL_OFF_T "\n",
702                   excess, k->size, k->maxdownload, k->bytecount);
703           }
704         }
705 
706         nread = (ssize_t) (k->maxdownload - k->bytecount);
707         if(nread < 0) /* this should be unusual */
708           nread = 0;
709 
710         k->keepon &= ~KEEP_RECV; /* we're done reading */
711       }
712 
713       k->bytecount += nread;
714 
715       Curl_pgrsSetDownloadCounter(data, k->bytecount);
716 
717       if(!k->chunk && (nread || k->badheader || is_empty_data)) {
718         /* If this is chunky transfer, it was already written */
719 
720         if(k->badheader && !k->ignorebody) {
721           /* we parsed a piece of data wrongly assuming it was a header
722              and now we output it as body instead */
723 
724           /* Don't let excess data pollute body writes */
725           if(k->maxdownload == -1 || (curl_off_t)k->hbuflen <= k->maxdownload)
726             result = Curl_client_write(conn, CLIENTWRITE_BODY,
727                                        data->state.headerbuff,
728                                        k->hbuflen);
729           else
730             result = Curl_client_write(conn, CLIENTWRITE_BODY,
731                                        data->state.headerbuff,
732                                        (size_t)k->maxdownload);
733 
734           if(result)
735             return result;
736         }
737         if(k->badheader < HEADER_ALLBAD) {
738           /* This switch handles various content encodings. If there's an
739              error here, be sure to check over the almost identical code
740              in http_chunks.c.
741              Make sure that ALL_CONTENT_ENCODINGS contains all the
742              encodings handled here. */
743 #ifdef HAVE_LIBZ
744           switch (conn->data->set.http_ce_skip ?
745                   IDENTITY : k->auto_decoding) {
746           case IDENTITY:
747 #endif
748             /* This is the default when the server sends no
749                Content-Encoding header. See Curl_readwrite_init; the
750                memset() call initializes k->auto_decoding to zero. */
751             if(!k->ignorebody) {
752 
753 #ifndef CURL_DISABLE_POP3
754               if(conn->handler->protocol&PROTO_FAMILY_POP3)
755                 result = Curl_pop3_write(conn, k->str, nread);
756               else
757 #endif /* CURL_DISABLE_POP3 */
758 
759                 result = Curl_client_write(conn, CLIENTWRITE_BODY, k->str,
760                                            nread);
761             }
762 #ifdef HAVE_LIBZ
763             break;
764 
765           case DEFLATE:
766             /* Assume CLIENTWRITE_BODY; headers are not encoded. */
767             if(!k->ignorebody)
768               result = Curl_unencode_deflate_write(conn, k, nread);
769             break;
770 
771           case GZIP:
772             /* Assume CLIENTWRITE_BODY; headers are not encoded. */
773             if(!k->ignorebody)
774               result = Curl_unencode_gzip_write(conn, k, nread);
775             break;
776 
777           default:
778             failf (data, "Unrecognized content encoding type. "
779                    "libcurl understands `identity', `deflate' and `gzip' "
780                    "content encodings.");
781             result = CURLE_BAD_CONTENT_ENCODING;
782             break;
783           }
784 #endif
785         }
786         k->badheader = HEADER_NORMAL; /* taken care of now */
787 
788         if(result)
789           return result;
790       }
791 
792     } /* if(!header and data to read) */
793 
794     if(conn->handler->readwrite &&
795        (excess > 0 && !conn->bits.stream_was_rewound)) {
796       /* Parse the excess data */
797       k->str += nread;
798       nread = (ssize_t)excess;
799 
800       result = conn->handler->readwrite(data, conn, &nread, &readmore);
801       if(result)
802         return result;
803 
804       if(readmore)
805         k->keepon |= KEEP_RECV; /* we're not done reading */
806       break;
807     }
808 
809     if(is_empty_data) {
810       /* if we received nothing, the server closed the connection and we
811          are done */
812       k->keepon &= ~KEEP_RECV;
813     }
814 
815   } while(data_pending(conn) && maxloops--);
816 
817   if(maxloops <= 0) {
818     /* we mark it as read-again-please */
819     conn->cselect_bits = CURL_CSELECT_IN;
820     *comeback = TRUE;
821   }
822 
823   if(((k->keepon & (KEEP_RECV|KEEP_SEND)) == KEEP_SEND) &&
824      conn->bits.close) {
825     /* When we've read the entire thing and the close bit is set, the server
826        may now close the connection. If there's now any kind of sending going
827        on from our side, we need to stop that immediately. */
828     infof(data, "we are done reading and this is set to close, stop send\n");
829     k->keepon &= ~KEEP_SEND; /* no writing anymore either */
830   }
831 
832   return CURLE_OK;
833 }
834 
done_sending(struct connectdata * conn,struct SingleRequest * k)835 static CURLcode done_sending(struct connectdata *conn,
836                              struct SingleRequest *k)
837 {
838   k->keepon &= ~KEEP_SEND; /* we're done writing */
839 
840   Curl_http2_done_sending(conn);
841 
842   if(conn->bits.rewindaftersend) {
843     CURLcode result = Curl_readrewind(conn);
844     if(result)
845       return result;
846   }
847   return CURLE_OK;
848 }
849 
850 
851 /*
852  * Send data to upload to the server, when the socket is writable.
853  */
readwrite_upload(struct Curl_easy * data,struct connectdata * conn,struct SingleRequest * k,int * didwhat)854 static CURLcode readwrite_upload(struct Curl_easy *data,
855                                  struct connectdata *conn,
856                                  struct SingleRequest *k,
857                                  int *didwhat)
858 {
859   ssize_t i, si;
860   ssize_t bytes_written;
861   CURLcode result;
862   ssize_t nread; /* number of bytes read */
863   bool sending_http_headers = FALSE;
864 
865   if((k->bytecount == 0) && (k->writebytecount == 0))
866     Curl_pgrsTime(data, TIMER_STARTTRANSFER);
867 
868   *didwhat |= KEEP_SEND;
869 
870   do {
871 
872     /* only read more data if there's no upload data already
873        present in the upload buffer */
874     if(0 == data->req.upload_present) {
875       /* init the "upload from here" pointer */
876       data->req.upload_fromhere = k->uploadbuf;
877 
878       if(!k->upload_done) {
879         /* HTTP pollution, this should be written nicer to become more
880            protocol agnostic. */
881         int fillcount;
882         struct HTTP *http = data->req.protop;
883 
884         if((k->exp100 == EXP100_SENDING_REQUEST) &&
885            (http->sending == HTTPSEND_BODY)) {
886           /* If this call is to send body data, we must take some action:
887              We have sent off the full HTTP 1.1 request, and we shall now
888              go into the Expect: 100 state and await such a header */
889           k->exp100 = EXP100_AWAITING_CONTINUE; /* wait for the header */
890           k->keepon &= ~KEEP_SEND;         /* disable writing */
891           k->start100 = Curl_tvnow();       /* timeout count starts now */
892           *didwhat &= ~KEEP_SEND;  /* we didn't write anything actually */
893 
894           /* set a timeout for the multi interface */
895           Curl_expire(data, data->set.expect_100_timeout);
896           break;
897         }
898 
899         if(conn->handler->protocol&(PROTO_FAMILY_HTTP|CURLPROTO_RTSP)) {
900           if(http->sending == HTTPSEND_REQUEST)
901             /* We're sending the HTTP request headers, not the data.
902                Remember that so we don't change the line endings. */
903             sending_http_headers = TRUE;
904           else
905             sending_http_headers = FALSE;
906         }
907 
908         result = Curl_fillreadbuffer(conn, BUFSIZE, &fillcount);
909         if(result)
910           return result;
911 
912         nread = (ssize_t)fillcount;
913       }
914       else
915         nread = 0; /* we're done uploading/reading */
916 
917       if(!nread && (k->keepon & KEEP_SEND_PAUSE)) {
918         /* this is a paused transfer */
919         break;
920       }
921       else if(nread<=0) {
922         result = done_sending(conn, k);
923         if(result)
924           return result;
925         break;
926       }
927 
928       /* store number of bytes available for upload */
929       data->req.upload_present = nread;
930 
931       /* convert LF to CRLF if so asked */
932       if((!sending_http_headers) && (
933 #ifdef CURL_DO_LINEEND_CONV
934          /* always convert if we're FTPing in ASCII mode */
935          (data->set.prefer_ascii) ||
936 #endif
937          (data->set.crlf))) {
938         /* Do we need to allocate a scratch buffer? */
939         if(!data->state.scratch) {
940           data->state.scratch = malloc(2 * BUFSIZE);
941           if(!data->state.scratch) {
942             failf(data, "Failed to alloc scratch buffer!");
943 
944             return CURLE_OUT_OF_MEMORY;
945           }
946         }
947 
948         /*
949          * ASCII/EBCDIC Note: This is presumably a text (not binary)
950          * transfer so the data should already be in ASCII.
951          * That means the hex values for ASCII CR (0x0d) & LF (0x0a)
952          * must be used instead of the escape sequences \r & \n.
953          */
954         for(i = 0, si = 0; i < nread; i++, si++) {
955           if(data->req.upload_fromhere[i] == 0x0a) {
956             data->state.scratch[si++] = 0x0d;
957             data->state.scratch[si] = 0x0a;
958             if(!data->set.crlf) {
959               /* we're here only because FTP is in ASCII mode...
960                  bump infilesize for the LF we just added */
961               if(data->state.infilesize != -1)
962                 data->state.infilesize++;
963             }
964           }
965           else
966             data->state.scratch[si] = data->req.upload_fromhere[i];
967         }
968 
969         if(si != nread) {
970           /* only perform the special operation if we really did replace
971              anything */
972           nread = si;
973 
974           /* upload from the new (replaced) buffer instead */
975           data->req.upload_fromhere = data->state.scratch;
976 
977           /* set the new amount too */
978           data->req.upload_present = nread;
979         }
980       }
981 
982 #ifndef CURL_DISABLE_SMTP
983       if(conn->handler->protocol & PROTO_FAMILY_SMTP) {
984         result = Curl_smtp_escape_eob(conn, nread);
985         if(result)
986           return result;
987       }
988 #endif /* CURL_DISABLE_SMTP */
989     } /* if 0 == data->req.upload_present */
990     else {
991       /* We have a partial buffer left from a previous "round". Use
992          that instead of reading more data */
993     }
994 
995     /* write to socket (send away data) */
996     result = Curl_write(conn,
997                         conn->writesockfd,     /* socket to send to */
998                         data->req.upload_fromhere, /* buffer pointer */
999                         data->req.upload_present,  /* buffer size */
1000                         &bytes_written);           /* actually sent */
1001 
1002     if(result)
1003       return result;
1004 
1005     if(data->set.verbose)
1006       /* show the data before we change the pointer upload_fromhere */
1007       Curl_debug(data, CURLINFO_DATA_OUT, data->req.upload_fromhere,
1008                  (size_t)bytes_written, conn);
1009 
1010     k->writebytecount += bytes_written;
1011 
1012     if(k->writebytecount == data->state.infilesize) {
1013       /* we have sent all data we were supposed to */
1014       k->upload_done = TRUE;
1015       infof(data, "We are completely uploaded and fine\n");
1016     }
1017 
1018     if(data->req.upload_present != bytes_written) {
1019       /* we only wrote a part of the buffer (if anything), deal with it! */
1020 
1021       /* store the amount of bytes left in the buffer to write */
1022       data->req.upload_present -= bytes_written;
1023 
1024       /* advance the pointer where to find the buffer when the next send
1025          is to happen */
1026       data->req.upload_fromhere += bytes_written;
1027     }
1028     else {
1029       /* we've uploaded that buffer now */
1030       data->req.upload_fromhere = k->uploadbuf;
1031       data->req.upload_present = 0; /* no more bytes left */
1032 
1033       if(k->upload_done) {
1034         result = done_sending(conn, k);
1035         if(result)
1036           return result;
1037       }
1038     }
1039 
1040     Curl_pgrsSetUploadCounter(data, k->writebytecount);
1041 
1042   } WHILE_FALSE; /* just to break out from! */
1043 
1044   return CURLE_OK;
1045 }
1046 
1047 /*
1048  * Curl_readwrite() is the low-level function to be called when data is to
1049  * be read and written to/from the connection.
1050  *
1051  * return '*comeback' TRUE if we didn't properly drain the socket so this
1052  * function should get called again without select() or similar in between!
1053  */
Curl_readwrite(struct connectdata * conn,struct Curl_easy * data,bool * done,bool * comeback)1054 CURLcode Curl_readwrite(struct connectdata *conn,
1055                         struct Curl_easy *data,
1056                         bool *done,
1057                         bool *comeback)
1058 {
1059   struct SingleRequest *k = &data->req;
1060   CURLcode result;
1061   int didwhat=0;
1062 
1063   curl_socket_t fd_read;
1064   curl_socket_t fd_write;
1065   int select_res = conn->cselect_bits;
1066 
1067   conn->cselect_bits = 0;
1068 
1069   /* only use the proper socket if the *_HOLD bit is not set simultaneously as
1070      then we are in rate limiting state in that transfer direction */
1071 
1072   if((k->keepon & KEEP_RECVBITS) == KEEP_RECV)
1073     fd_read = conn->sockfd;
1074   else
1075     fd_read = CURL_SOCKET_BAD;
1076 
1077   if((k->keepon & KEEP_SENDBITS) == KEEP_SEND)
1078     fd_write = conn->writesockfd;
1079   else
1080     fd_write = CURL_SOCKET_BAD;
1081 
1082   if(conn->data->state.drain) {
1083     select_res |= CURL_CSELECT_IN;
1084     DEBUGF(infof(data, "Curl_readwrite: forcibly told to drain data\n"));
1085   }
1086 
1087   if(!select_res) /* Call for select()/poll() only, if read/write/error
1088                      status is not known. */
1089     select_res = Curl_socket_check(fd_read, CURL_SOCKET_BAD, fd_write, 0);
1090 
1091   if(select_res == CURL_CSELECT_ERR) {
1092     failf(data, "select/poll returned error");
1093     return CURLE_SEND_ERROR;
1094   }
1095 
1096   /* We go ahead and do a read if we have a readable socket or if
1097      the stream was rewound (in which case we have data in a
1098      buffer) */
1099   if((k->keepon & KEEP_RECV) &&
1100      ((select_res & CURL_CSELECT_IN) || conn->bits.stream_was_rewound)) {
1101 
1102     result = readwrite_data(data, conn, k, &didwhat, done, comeback);
1103     if(result || *done)
1104       return result;
1105   }
1106 
1107   /* If we still have writing to do, we check if we have a writable socket. */
1108   if((k->keepon & KEEP_SEND) && (select_res & CURL_CSELECT_OUT)) {
1109     /* write */
1110 
1111     result = readwrite_upload(data, conn, k, &didwhat);
1112     if(result)
1113       return result;
1114   }
1115 
1116   k->now = Curl_tvnow();
1117   if(didwhat) {
1118     /* Update read/write counters */
1119     if(k->bytecountp)
1120       *k->bytecountp = k->bytecount; /* read count */
1121     if(k->writebytecountp)
1122       *k->writebytecountp = k->writebytecount; /* write count */
1123   }
1124   else {
1125     /* no read no write, this is a timeout? */
1126     if(k->exp100 == EXP100_AWAITING_CONTINUE) {
1127       /* This should allow some time for the header to arrive, but only a
1128          very short time as otherwise it'll be too much wasted time too
1129          often. */
1130 
1131       /* Quoting RFC2616, section "8.2.3 Use of the 100 (Continue) Status":
1132 
1133          Therefore, when a client sends this header field to an origin server
1134          (possibly via a proxy) from which it has never seen a 100 (Continue)
1135          status, the client SHOULD NOT wait for an indefinite period before
1136          sending the request body.
1137 
1138       */
1139 
1140       time_t ms = Curl_tvdiff(k->now, k->start100);
1141       if(ms >= data->set.expect_100_timeout) {
1142         /* we've waited long enough, continue anyway */
1143         k->exp100 = EXP100_SEND_DATA;
1144         k->keepon |= KEEP_SEND;
1145         infof(data, "Done waiting for 100-continue\n");
1146       }
1147     }
1148   }
1149 
1150   if(Curl_pgrsUpdate(conn))
1151     result = CURLE_ABORTED_BY_CALLBACK;
1152   else
1153     result = Curl_speedcheck(data, k->now);
1154   if(result)
1155     return result;
1156 
1157   if(k->keepon) {
1158     if(0 > Curl_timeleft(data, &k->now, FALSE)) {
1159       if(k->size != -1) {
1160         failf(data, "Operation timed out after %ld milliseconds with %"
1161               CURL_FORMAT_CURL_OFF_T " out of %"
1162               CURL_FORMAT_CURL_OFF_T " bytes received",
1163               Curl_tvdiff(k->now, data->progress.t_startsingle), k->bytecount,
1164               k->size);
1165       }
1166       else {
1167         failf(data, "Operation timed out after %ld milliseconds with %"
1168               CURL_FORMAT_CURL_OFF_T " bytes received",
1169               Curl_tvdiff(k->now, data->progress.t_startsingle), k->bytecount);
1170       }
1171       return CURLE_OPERATION_TIMEDOUT;
1172     }
1173   }
1174   else {
1175     /*
1176      * The transfer has been performed. Just make some general checks before
1177      * returning.
1178      */
1179 
1180     if(!(data->set.opt_no_body) && (k->size != -1) &&
1181        (k->bytecount != k->size) &&
1182 #ifdef CURL_DO_LINEEND_CONV
1183        /* Most FTP servers don't adjust their file SIZE response for CRLFs,
1184           so we'll check to see if the discrepancy can be explained
1185           by the number of CRLFs we've changed to LFs.
1186        */
1187        (k->bytecount != (k->size + data->state.crlf_conversions)) &&
1188 #endif /* CURL_DO_LINEEND_CONV */
1189        !data->req.newurl) {
1190       failf(data, "transfer closed with %" CURL_FORMAT_CURL_OFF_T
1191             " bytes remaining to read",
1192             k->size - k->bytecount);
1193       return CURLE_PARTIAL_FILE;
1194     }
1195     else if(!(data->set.opt_no_body) &&
1196             k->chunk &&
1197             (conn->chunk.state != CHUNK_STOP)) {
1198       /*
1199        * In chunked mode, return an error if the connection is closed prior to
1200        * the empty (terminating) chunk is read.
1201        *
1202        * The condition above used to check for
1203        * conn->proto.http->chunk.datasize != 0 which is true after reading
1204        * *any* chunk, not just the empty chunk.
1205        *
1206        */
1207       failf(data, "transfer closed with outstanding read data remaining");
1208       return CURLE_PARTIAL_FILE;
1209     }
1210     if(Curl_pgrsUpdate(conn))
1211       return CURLE_ABORTED_BY_CALLBACK;
1212   }
1213 
1214   /* Now update the "done" boolean we return */
1215   *done = (0 == (k->keepon&(KEEP_RECV|KEEP_SEND|
1216                             KEEP_RECV_PAUSE|KEEP_SEND_PAUSE))) ? TRUE : FALSE;
1217 
1218   return CURLE_OK;
1219 }
1220 
1221 /*
1222  * Curl_single_getsock() gets called by the multi interface code when the app
1223  * has requested to get the sockets for the current connection. This function
1224  * will then be called once for every connection that the multi interface
1225  * keeps track of. This function will only be called for connections that are
1226  * in the proper state to have this information available.
1227  */
Curl_single_getsock(const struct connectdata * conn,curl_socket_t * sock,int numsocks)1228 int Curl_single_getsock(const struct connectdata *conn,
1229                         curl_socket_t *sock, /* points to numsocks number
1230                                                 of sockets */
1231                         int numsocks)
1232 {
1233   const struct Curl_easy *data = conn->data;
1234   int bitmap = GETSOCK_BLANK;
1235   unsigned sockindex = 0;
1236 
1237   if(conn->handler->perform_getsock)
1238     return conn->handler->perform_getsock(conn, sock, numsocks);
1239 
1240   if(numsocks < 2)
1241     /* simple check but we might need two slots */
1242     return GETSOCK_BLANK;
1243 
1244   /* don't include HOLD and PAUSE connections */
1245   if((data->req.keepon & KEEP_RECVBITS) == KEEP_RECV) {
1246 
1247     DEBUGASSERT(conn->sockfd != CURL_SOCKET_BAD);
1248 
1249     bitmap |= GETSOCK_READSOCK(sockindex);
1250     sock[sockindex] = conn->sockfd;
1251   }
1252 
1253   /* don't include HOLD and PAUSE connections */
1254   if((data->req.keepon & KEEP_SENDBITS) == KEEP_SEND) {
1255 
1256     if((conn->sockfd != conn->writesockfd) ||
1257        bitmap == GETSOCK_BLANK) {
1258       /* only if they are not the same socket and we have a readable
1259          one, we increase index */
1260       if(bitmap != GETSOCK_BLANK)
1261         sockindex++; /* increase index if we need two entries */
1262 
1263       DEBUGASSERT(conn->writesockfd != CURL_SOCKET_BAD);
1264 
1265       sock[sockindex] = conn->writesockfd;
1266     }
1267 
1268     bitmap |= GETSOCK_WRITESOCK(sockindex);
1269   }
1270 
1271   return bitmap;
1272 }
1273 
1274 /* Curl_init_CONNECT() gets called each time the handle switches to CONNECT
1275    which means this gets called once for each subsequent redirect etc */
Curl_init_CONNECT(struct Curl_easy * data)1276 void Curl_init_CONNECT(struct Curl_easy *data)
1277 {
1278   data->state.fread_func = data->set.fread_func_set;
1279   data->state.in = data->set.in_set;
1280 }
1281 
1282 /*
1283  * Curl_pretransfer() is called immediately before a transfer starts, and only
1284  * once for one transfer no matter if it has redirects or do multi-pass
1285  * authentication etc.
1286  */
Curl_pretransfer(struct Curl_easy * data)1287 CURLcode Curl_pretransfer(struct Curl_easy *data)
1288 {
1289   CURLcode result;
1290   if(!data->change.url) {
1291     /* we can't do anything without URL */
1292     failf(data, "No URL set!");
1293     return CURLE_URL_MALFORMAT;
1294   }
1295 
1296   /* Init the SSL session ID cache here. We do it here since we want to do it
1297      after the *_setopt() calls (that could specify the size of the cache) but
1298      before any transfer takes place. */
1299   result = Curl_ssl_initsessions(data, data->set.general_ssl.max_ssl_sessions);
1300   if(result)
1301     return result;
1302 
1303   data->set.followlocation=0; /* reset the location-follow counter */
1304   data->state.this_is_a_follow = FALSE; /* reset this */
1305   data->state.errorbuf = FALSE; /* no error has occurred */
1306   data->state.httpversion = 0; /* don't assume any particular server version */
1307 
1308   data->state.authproblem = FALSE;
1309   data->state.authhost.want = data->set.httpauth;
1310   data->state.authproxy.want = data->set.proxyauth;
1311   Curl_safefree(data->info.wouldredirect);
1312   data->info.wouldredirect = NULL;
1313 
1314   if(data->set.httpreq == HTTPREQ_PUT)
1315     data->state.infilesize = data->set.filesize;
1316   else
1317     data->state.infilesize = data->set.postfieldsize;
1318 
1319   /* If there is a list of cookie files to read, do it now! */
1320   if(data->change.cookielist)
1321     Curl_cookie_loadfiles(data);
1322 
1323   /* If there is a list of host pairs to deal with */
1324   if(data->change.resolve)
1325     result = Curl_loadhostpairs(data);
1326 
1327   if(!result) {
1328     /* Allow data->set.use_port to set which port to use. This needs to be
1329      * disabled for example when we follow Location: headers to URLs using
1330      * different ports! */
1331     data->state.allow_port = TRUE;
1332 
1333 #if defined(HAVE_SIGNAL) && defined(SIGPIPE) && !defined(HAVE_MSG_NOSIGNAL)
1334     /*************************************************************
1335      * Tell signal handler to ignore SIGPIPE
1336      *************************************************************/
1337     if(!data->set.no_signal)
1338       data->state.prev_signal = signal(SIGPIPE, SIG_IGN);
1339 #endif
1340 
1341     Curl_initinfo(data); /* reset session-specific information "variables" */
1342     Curl_pgrsResetTimesSizes(data);
1343     Curl_pgrsStartNow(data);
1344 
1345     if(data->set.timeout)
1346       Curl_expire(data, data->set.timeout);
1347 
1348     if(data->set.connecttimeout)
1349       Curl_expire(data, data->set.connecttimeout);
1350 
1351     /* In case the handle is re-used and an authentication method was picked
1352        in the session we need to make sure we only use the one(s) we now
1353        consider to be fine */
1354     data->state.authhost.picked &= data->state.authhost.want;
1355     data->state.authproxy.picked &= data->state.authproxy.want;
1356 
1357     if(data->set.wildcardmatch) {
1358       struct WildcardData *wc = &data->wildcard;
1359       if(!wc->filelist) {
1360         result = Curl_wildcard_init(wc); /* init wildcard structures */
1361         if(result)
1362           return CURLE_OUT_OF_MEMORY;
1363       }
1364     }
1365 
1366   }
1367 
1368   return result;
1369 }
1370 
1371 /*
1372  * Curl_posttransfer() is called immediately after a transfer ends
1373  */
Curl_posttransfer(struct Curl_easy * data)1374 CURLcode Curl_posttransfer(struct Curl_easy *data)
1375 {
1376 #if defined(HAVE_SIGNAL) && defined(SIGPIPE) && !defined(HAVE_MSG_NOSIGNAL)
1377   /* restore the signal handler for SIGPIPE before we get back */
1378   if(!data->set.no_signal)
1379     signal(SIGPIPE, data->state.prev_signal);
1380 #else
1381   (void)data; /* unused parameter */
1382 #endif
1383 
1384   return CURLE_OK;
1385 }
1386 
1387 #ifndef CURL_DISABLE_HTTP
1388 /*
1389  * strlen_url() returns the length of the given URL if the spaces within the
1390  * URL were properly URL encoded.
1391  */
strlen_url(const char * url)1392 static size_t strlen_url(const char *url)
1393 {
1394   const unsigned char *ptr;
1395   size_t newlen=0;
1396   bool left=TRUE; /* left side of the ? */
1397 
1398   for(ptr=(unsigned char *)url; *ptr; ptr++) {
1399     switch(*ptr) {
1400     case '?':
1401       left=FALSE;
1402       /* fall through */
1403     default:
1404       if(*ptr >= 0x80)
1405         newlen += 2;
1406       newlen++;
1407       break;
1408     case ' ':
1409       if(left)
1410         newlen+=3;
1411       else
1412         newlen++;
1413       break;
1414     }
1415   }
1416   return newlen;
1417 }
1418 
1419 /* strcpy_url() copies a url to a output buffer and URL-encodes the spaces in
1420  * the source URL accordingly.
1421  */
strcpy_url(char * output,const char * url)1422 static void strcpy_url(char *output, const char *url)
1423 {
1424   /* we must add this with whitespace-replacing */
1425   bool left=TRUE;
1426   const unsigned char *iptr;
1427   char *optr = output;
1428   for(iptr = (unsigned char *)url;    /* read from here */
1429       *iptr;         /* until zero byte */
1430       iptr++) {
1431     switch(*iptr) {
1432     case '?':
1433       left=FALSE;
1434       /* fall through */
1435     default:
1436       if(*iptr >= 0x80) {
1437         snprintf(optr, 4, "%%%02x", *iptr);
1438         optr += 3;
1439       }
1440       else
1441         *optr++=*iptr;
1442       break;
1443     case ' ':
1444       if(left) {
1445         *optr++='%'; /* add a '%' */
1446         *optr++='2'; /* add a '2' */
1447         *optr++='0'; /* add a '0' */
1448       }
1449       else
1450         *optr++='+'; /* add a '+' here */
1451       break;
1452     }
1453   }
1454   *optr=0; /* zero terminate output buffer */
1455 
1456 }
1457 
1458 /*
1459  * Returns true if the given URL is absolute (as opposed to relative)
1460  */
is_absolute_url(const char * url)1461 static bool is_absolute_url(const char *url)
1462 {
1463   char prot[16]; /* URL protocol string storage */
1464   char letter;   /* used for a silly sscanf */
1465 
1466   return (2 == sscanf(url, "%15[^?&/:]://%c", prot, &letter)) ? TRUE : FALSE;
1467 }
1468 
1469 /*
1470  * Concatenate a relative URL to a base URL making it absolute.
1471  * URL-encodes any spaces.
1472  * The returned pointer must be freed by the caller unless NULL
1473  * (returns NULL on out of memory).
1474  */
concat_url(const char * base,const char * relurl)1475 static char *concat_url(const char *base, const char *relurl)
1476 {
1477   /***
1478    TRY to append this new path to the old URL
1479    to the right of the host part. Oh crap, this is doomed to cause
1480    problems in the future...
1481   */
1482   char *newest;
1483   char *protsep;
1484   char *pathsep;
1485   size_t newlen;
1486 
1487   const char *useurl = relurl;
1488   size_t urllen;
1489 
1490   /* we must make our own copy of the URL to play with, as it may
1491      point to read-only data */
1492   char *url_clone=strdup(base);
1493 
1494   if(!url_clone)
1495     return NULL; /* skip out of this NOW */
1496 
1497   /* protsep points to the start of the host name */
1498   protsep=strstr(url_clone, "//");
1499   if(!protsep)
1500     protsep=url_clone;
1501   else
1502     protsep+=2; /* pass the slashes */
1503 
1504   if('/' != relurl[0]) {
1505     int level=0;
1506 
1507     /* First we need to find out if there's a ?-letter in the URL,
1508        and cut it and the right-side of that off */
1509     pathsep = strchr(protsep, '?');
1510     if(pathsep)
1511       *pathsep=0;
1512 
1513     /* we have a relative path to append to the last slash if there's one
1514        available, or if the new URL is just a query string (starts with a
1515        '?')  we append the new one at the end of the entire currently worked
1516        out URL */
1517     if(useurl[0] != '?') {
1518       pathsep = strrchr(protsep, '/');
1519       if(pathsep)
1520         *pathsep=0;
1521     }
1522 
1523     /* Check if there's any slash after the host name, and if so, remember
1524        that position instead */
1525     pathsep = strchr(protsep, '/');
1526     if(pathsep)
1527       protsep = pathsep+1;
1528     else
1529       protsep = NULL;
1530 
1531     /* now deal with one "./" or any amount of "../" in the newurl
1532        and act accordingly */
1533 
1534     if((useurl[0] == '.') && (useurl[1] == '/'))
1535       useurl+=2; /* just skip the "./" */
1536 
1537     while((useurl[0] == '.') &&
1538           (useurl[1] == '.') &&
1539           (useurl[2] == '/')) {
1540       level++;
1541       useurl+=3; /* pass the "../" */
1542     }
1543 
1544     if(protsep) {
1545       while(level--) {
1546         /* cut off one more level from the right of the original URL */
1547         pathsep = strrchr(protsep, '/');
1548         if(pathsep)
1549           *pathsep=0;
1550         else {
1551           *protsep=0;
1552           break;
1553         }
1554       }
1555     }
1556   }
1557   else {
1558     /* We got a new absolute path for this server */
1559 
1560     if((relurl[0] == '/') && (relurl[1] == '/')) {
1561       /* the new URL starts with //, just keep the protocol part from the
1562          original one */
1563       *protsep=0;
1564       useurl = &relurl[2]; /* we keep the slashes from the original, so we
1565                               skip the new ones */
1566     }
1567     else {
1568       /* cut off the original URL from the first slash, or deal with URLs
1569          without slash */
1570       pathsep = strchr(protsep, '/');
1571       if(pathsep) {
1572         /* When people use badly formatted URLs, such as
1573            "http://www.url.com?dir=/home/daniel" we must not use the first
1574            slash, if there's a ?-letter before it! */
1575         char *sep = strchr(protsep, '?');
1576         if(sep && (sep < pathsep))
1577           pathsep = sep;
1578         *pathsep=0;
1579       }
1580       else {
1581         /* There was no slash. Now, since we might be operating on a badly
1582            formatted URL, such as "http://www.url.com?id=2380" which doesn't
1583            use a slash separator as it is supposed to, we need to check for a
1584            ?-letter as well! */
1585         pathsep = strchr(protsep, '?');
1586         if(pathsep)
1587           *pathsep=0;
1588       }
1589     }
1590   }
1591 
1592   /* If the new part contains a space, this is a mighty stupid redirect
1593      but we still make an effort to do "right". To the left of a '?'
1594      letter we replace each space with %20 while it is replaced with '+'
1595      on the right side of the '?' letter.
1596   */
1597   newlen = strlen_url(useurl);
1598 
1599   urllen = strlen(url_clone);
1600 
1601   newest = malloc(urllen + 1 + /* possible slash */
1602                   newlen + 1 /* zero byte */);
1603 
1604   if(!newest) {
1605     free(url_clone); /* don't leak this */
1606     return NULL;
1607   }
1608 
1609   /* copy over the root url part */
1610   memcpy(newest, url_clone, urllen);
1611 
1612   /* check if we need to append a slash */
1613   if(('/' == useurl[0]) || (protsep && !*protsep) || ('?' == useurl[0]))
1614     ;
1615   else
1616     newest[urllen++]='/';
1617 
1618   /* then append the new piece on the right side */
1619   strcpy_url(&newest[urllen], useurl);
1620 
1621   free(url_clone);
1622 
1623   return newest;
1624 }
1625 #endif /* CURL_DISABLE_HTTP */
1626 
1627 /*
1628  * Curl_follow() handles the URL redirect magic. Pass in the 'newurl' string
1629  * as given by the remote server and set up the new URL to request.
1630  */
Curl_follow(struct Curl_easy * data,char * newurl,followtype type)1631 CURLcode Curl_follow(struct Curl_easy *data,
1632                      char *newurl, /* this 'newurl' is the Location: string,
1633                                       and it must be malloc()ed before passed
1634                                       here */
1635                      followtype type) /* see transfer.h */
1636 {
1637 #ifdef CURL_DISABLE_HTTP
1638   (void)data;
1639   (void)newurl;
1640   (void)type;
1641   /* Location: following will not happen when HTTP is disabled */
1642   return CURLE_TOO_MANY_REDIRECTS;
1643 #else
1644 
1645   /* Location: redirect */
1646   bool disallowport = FALSE;
1647 
1648   if(type == FOLLOW_REDIR) {
1649     if((data->set.maxredirs != -1) &&
1650         (data->set.followlocation >= data->set.maxredirs)) {
1651       failf(data, "Maximum (%ld) redirects followed", data->set.maxredirs);
1652       return CURLE_TOO_MANY_REDIRECTS;
1653     }
1654 
1655     /* mark the next request as a followed location: */
1656     data->state.this_is_a_follow = TRUE;
1657 
1658     data->set.followlocation++; /* count location-followers */
1659 
1660     if(data->set.http_auto_referer) {
1661       /* We are asked to automatically set the previous URL as the referer
1662          when we get the next URL. We pick the ->url field, which may or may
1663          not be 100% correct */
1664 
1665       if(data->change.referer_alloc) {
1666         Curl_safefree(data->change.referer);
1667         data->change.referer_alloc = FALSE;
1668       }
1669 
1670       data->change.referer = strdup(data->change.url);
1671       if(!data->change.referer)
1672         return CURLE_OUT_OF_MEMORY;
1673       data->change.referer_alloc = TRUE; /* yes, free this later */
1674     }
1675   }
1676 
1677   if(!is_absolute_url(newurl))  {
1678     /***
1679      *DANG* this is an RFC 2068 violation. The URL is supposed
1680      to be absolute and this doesn't seem to be that!
1681      */
1682     char *absolute = concat_url(data->change.url, newurl);
1683     if(!absolute)
1684       return CURLE_OUT_OF_MEMORY;
1685     free(newurl);
1686     newurl = absolute;
1687   }
1688   else {
1689     /* The new URL MAY contain space or high byte values, that means a mighty
1690        stupid redirect URL but we still make an effort to do "right". */
1691     char *newest;
1692     size_t newlen = strlen_url(newurl);
1693 
1694     /* This is an absolute URL, don't allow the custom port number */
1695     disallowport = TRUE;
1696 
1697     newest = malloc(newlen+1); /* get memory for this */
1698     if(!newest)
1699       return CURLE_OUT_OF_MEMORY;
1700     strcpy_url(newest, newurl); /* create a space-free URL */
1701 
1702     free(newurl); /* that was no good */
1703     newurl = newest; /* use this instead now */
1704 
1705   }
1706 
1707   if(type == FOLLOW_FAKE) {
1708     /* we're only figuring out the new url if we would've followed locations
1709        but now we're done so we can get out! */
1710     data->info.wouldredirect = newurl;
1711     return CURLE_OK;
1712   }
1713 
1714   if(disallowport)
1715     data->state.allow_port = FALSE;
1716 
1717   if(data->change.url_alloc) {
1718     Curl_safefree(data->change.url);
1719     data->change.url_alloc = FALSE;
1720   }
1721 
1722   data->change.url = newurl;
1723   data->change.url_alloc = TRUE;
1724   newurl = NULL; /* don't free! */
1725 
1726   infof(data, "Issue another request to this URL: '%s'\n", data->change.url);
1727 
1728   /*
1729    * We get here when the HTTP code is 300-399 (and 401). We need to perform
1730    * differently based on exactly what return code there was.
1731    *
1732    * News from 7.10.6: we can also get here on a 401 or 407, in case we act on
1733    * a HTTP (proxy-) authentication scheme other than Basic.
1734    */
1735   switch(data->info.httpcode) {
1736     /* 401 - Act on a WWW-Authenticate, we keep on moving and do the
1737        Authorization: XXXX header in the HTTP request code snippet */
1738     /* 407 - Act on a Proxy-Authenticate, we keep on moving and do the
1739        Proxy-Authorization: XXXX header in the HTTP request code snippet */
1740     /* 300 - Multiple Choices */
1741     /* 306 - Not used */
1742     /* 307 - Temporary Redirect */
1743   default:  /* for all above (and the unknown ones) */
1744     /* Some codes are explicitly mentioned since I've checked RFC2616 and they
1745      * seem to be OK to POST to.
1746      */
1747     break;
1748   case 301: /* Moved Permanently */
1749     /* (quote from RFC7231, section 6.4.2)
1750      *
1751      * Note: For historical reasons, a user agent MAY change the request
1752      * method from POST to GET for the subsequent request.  If this
1753      * behavior is undesired, the 307 (Temporary Redirect) status code
1754      * can be used instead.
1755      *
1756      * ----
1757      *
1758      * Many webservers expect this, so these servers often answers to a POST
1759      * request with an error page. To be sure that libcurl gets the page that
1760      * most user agents would get, libcurl has to force GET.
1761      *
1762      * This behaviour is forbidden by RFC1945 and the obsolete RFC2616, and
1763      * can be overridden with CURLOPT_POSTREDIR.
1764      */
1765     if((data->set.httpreq == HTTPREQ_POST
1766         || data->set.httpreq == HTTPREQ_POST_FORM)
1767        && !(data->set.keep_post & CURL_REDIR_POST_301)) {
1768       infof(data, "Switch from POST to GET\n");
1769       data->set.httpreq = HTTPREQ_GET;
1770     }
1771     break;
1772   case 302: /* Found */
1773     /* (quote from RFC7231, section 6.4.3)
1774      *
1775      * Note: For historical reasons, a user agent MAY change the request
1776      * method from POST to GET for the subsequent request.  If this
1777      * behavior is undesired, the 307 (Temporary Redirect) status code
1778      * can be used instead.
1779      *
1780      * ----
1781      *
1782      * Many webservers expect this, so these servers often answers to a POST
1783      * request with an error page. To be sure that libcurl gets the page that
1784      * most user agents would get, libcurl has to force GET.
1785      *
1786      * This behaviour is forbidden by RFC1945 and the obsolete RFC2616, and
1787      * can be overridden with CURLOPT_POSTREDIR.
1788      */
1789     if((data->set.httpreq == HTTPREQ_POST
1790         || data->set.httpreq == HTTPREQ_POST_FORM)
1791        && !(data->set.keep_post & CURL_REDIR_POST_302)) {
1792       infof(data, "Switch from POST to GET\n");
1793       data->set.httpreq = HTTPREQ_GET;
1794     }
1795     break;
1796 
1797   case 303: /* See Other */
1798     /* Disable both types of POSTs, unless the user explicitely
1799        asks for POST after POST */
1800     if(data->set.httpreq != HTTPREQ_GET
1801       && !(data->set.keep_post & CURL_REDIR_POST_303)) {
1802       data->set.httpreq = HTTPREQ_GET; /* enforce GET request */
1803       infof(data, "Disables POST, goes with %s\n",
1804             data->set.opt_no_body?"HEAD":"GET");
1805     }
1806     break;
1807   case 304: /* Not Modified */
1808     /* 304 means we did a conditional request and it was "Not modified".
1809      * We shouldn't get any Location: header in this response!
1810      */
1811     break;
1812   case 305: /* Use Proxy */
1813     /* (quote from RFC2616, section 10.3.6):
1814      * "The requested resource MUST be accessed through the proxy given
1815      * by the Location field. The Location field gives the URI of the
1816      * proxy.  The recipient is expected to repeat this single request
1817      * via the proxy. 305 responses MUST only be generated by origin
1818      * servers."
1819      */
1820     break;
1821   }
1822   Curl_pgrsTime(data, TIMER_REDIRECT);
1823   Curl_pgrsResetTimesSizes(data);
1824 
1825   return CURLE_OK;
1826 #endif /* CURL_DISABLE_HTTP */
1827 }
1828 
1829 /* Returns CURLE_OK *and* sets '*url' if a request retry is wanted.
1830 
1831    NOTE: that the *url is malloc()ed. */
Curl_retry_request(struct connectdata * conn,char ** url)1832 CURLcode Curl_retry_request(struct connectdata *conn,
1833                             char **url)
1834 {
1835   struct Curl_easy *data = conn->data;
1836 
1837   *url = NULL;
1838 
1839   /* if we're talking upload, we can't do the checks below, unless the protocol
1840      is HTTP as when uploading over HTTP we will still get a response */
1841   if(data->set.upload &&
1842      !(conn->handler->protocol&(PROTO_FAMILY_HTTP|CURLPROTO_RTSP)))
1843     return CURLE_OK;
1844 
1845   if((data->req.bytecount + data->req.headerbytecount == 0) &&
1846      conn->bits.reuse &&
1847      (data->set.rtspreq != RTSPREQ_RECEIVE)) {
1848     /* We didn't get a single byte when we attempted to re-use a
1849        connection. This might happen if the connection was left alive when we
1850        were done using it before, but that was closed when we wanted to use it
1851        again. Bad luck. Retry the same request on a fresh connect! */
1852     infof(conn->data, "Connection died, retrying a fresh connect\n");
1853     *url = strdup(conn->data->change.url);
1854     if(!*url)
1855       return CURLE_OUT_OF_MEMORY;
1856 
1857     connclose(conn, "retry"); /* close this connection */
1858     conn->bits.retry = TRUE; /* mark this as a connection we're about
1859                                 to retry. Marking it this way should
1860                                 prevent i.e HTTP transfers to return
1861                                 error just because nothing has been
1862                                 transferred! */
1863 
1864 
1865     if(conn->handler->protocol&PROTO_FAMILY_HTTP) {
1866       struct HTTP *http = data->req.protop;
1867       if(http->writebytecount)
1868         return Curl_readrewind(conn);
1869     }
1870   }
1871   return CURLE_OK;
1872 }
1873 
1874 /*
1875  * Curl_setup_transfer() is called to setup some basic properties for the
1876  * upcoming transfer.
1877  */
1878 void
Curl_setup_transfer(struct connectdata * conn,int sockindex,curl_off_t size,bool getheader,curl_off_t * bytecountp,int writesockindex,curl_off_t * writecountp)1879 Curl_setup_transfer(
1880   struct connectdata *conn, /* connection data */
1881   int sockindex,            /* socket index to read from or -1 */
1882   curl_off_t size,          /* -1 if unknown at this point */
1883   bool getheader,           /* TRUE if header parsing is wanted */
1884   curl_off_t *bytecountp,   /* return number of bytes read or NULL */
1885   int writesockindex,       /* socket index to write to, it may very well be
1886                                the same we read from. -1 disables */
1887   curl_off_t *writecountp   /* return number of bytes written or NULL */
1888   )
1889 {
1890   struct Curl_easy *data;
1891   struct SingleRequest *k;
1892 
1893   DEBUGASSERT(conn != NULL);
1894 
1895   data = conn->data;
1896   k = &data->req;
1897 
1898   DEBUGASSERT((sockindex <= 1) && (sockindex >= -1));
1899 
1900   /* now copy all input parameters */
1901   conn->sockfd = sockindex == -1 ?
1902       CURL_SOCKET_BAD : conn->sock[sockindex];
1903   conn->writesockfd = writesockindex == -1 ?
1904       CURL_SOCKET_BAD:conn->sock[writesockindex];
1905   k->getheader = getheader;
1906 
1907   k->size = size;
1908   k->bytecountp = bytecountp;
1909   k->writebytecountp = writecountp;
1910 
1911   /* The code sequence below is placed in this function just because all
1912      necessary input is not always known in do_complete() as this function may
1913      be called after that */
1914 
1915   if(!k->getheader) {
1916     k->header = FALSE;
1917     if(size > 0)
1918       Curl_pgrsSetDownloadSize(data, size);
1919   }
1920   /* we want header and/or body, if neither then don't do this! */
1921   if(k->getheader || !data->set.opt_no_body) {
1922 
1923     if(conn->sockfd != CURL_SOCKET_BAD)
1924       k->keepon |= KEEP_RECV;
1925 
1926     if(conn->writesockfd != CURL_SOCKET_BAD) {
1927       struct HTTP *http = data->req.protop;
1928       /* HTTP 1.1 magic:
1929 
1930          Even if we require a 100-return code before uploading data, we might
1931          need to write data before that since the REQUEST may not have been
1932          finished sent off just yet.
1933 
1934          Thus, we must check if the request has been sent before we set the
1935          state info where we wait for the 100-return code
1936       */
1937       if((data->state.expect100header) &&
1938          (conn->handler->protocol&PROTO_FAMILY_HTTP) &&
1939          (http->sending == HTTPSEND_BODY)) {
1940         /* wait with write until we either got 100-continue or a timeout */
1941         k->exp100 = EXP100_AWAITING_CONTINUE;
1942         k->start100 = Curl_tvnow();
1943 
1944         /* Set a timeout for the multi interface. Add the inaccuracy margin so
1945            that we don't fire slightly too early and get denied to run. */
1946         Curl_expire(data, data->set.expect_100_timeout);
1947       }
1948       else {
1949         if(data->state.expect100header)
1950           /* when we've sent off the rest of the headers, we must await a
1951              100-continue but first finish sending the request */
1952           k->exp100 = EXP100_SENDING_REQUEST;
1953 
1954         /* enable the write bit when we're not waiting for continue */
1955         k->keepon |= KEEP_SEND;
1956       }
1957     } /* if(conn->writesockfd != CURL_SOCKET_BAD) */
1958   } /* if(k->getheader || !data->set.opt_no_body) */
1959 
1960 }
1961