1 /*****************************************************************************
2 *
3 * This example source code introduces a c library buffered I/O interface to
4 * URL reads it supports fopen(), fread(), fgets(), feof(), fclose(),
5 * rewind(). Supported functions have identical prototypes to their normal c
6 * lib namesakes and are preceaded by url_ .
7 *
8 * Using this code you can replace your program's fopen() with url_fopen()
9 * and fread() with url_fread() and it become possible to read remote streams
10 * instead of (only) local files. Local files (ie those that can be directly
11 * fopened) will drop back to using the underlying clib implementations
12 *
13 * See the main() function at the bottom that shows an app that retrieves from
14 * a specified url using fgets() and fread() and saves as two output files.
15 *
16 * Copyright (c) 2003, 2017 Simtec Electronics
17 *
18 * Re-implemented by Vincent Sanders <vince@kyllikki.org> with extensive
19 * reference to original curl example code
20 *
21 * Redistribution and use in source and binary forms, with or without
22 * modification, are permitted provided that the following conditions
23 * are met:
24 * 1. Redistributions of source code must retain the above copyright
25 * notice, this list of conditions and the following disclaimer.
26 * 2. Redistributions in binary form must reproduce the above copyright
27 * notice, this list of conditions and the following disclaimer in the
28 * documentation and/or other materials provided with the distribution.
29 * 3. The name of the author may not be used to endorse or promote products
30 * derived from this software without specific prior written permission.
31 *
32 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
33 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
34 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
35 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
36 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
37 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
38 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
39 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
40 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
41 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
42 *
43 * This example requires libcurl 7.9.7 or later.
44 */
45 /* <DESC>
46 * implements an fopen() abstraction allowing reading from URLs
47 * </DESC>
48 */
49
50 #include <stdio.h>
51 #include <string.h>
52 #ifndef WIN32
53 # include <sys/time.h>
54 #endif
55 #include <stdlib.h>
56 #include <errno.h>
57
58 #include <curl/curl.h>
59
60 enum fcurl_type_e {
61 CFTYPE_NONE = 0,
62 CFTYPE_FILE = 1,
63 CFTYPE_CURL = 2
64 };
65
66 struct fcurl_data
67 {
68 enum fcurl_type_e type; /* type of handle */
69 union {
70 CURL *curl;
71 FILE *file;
72 } handle; /* handle */
73
74 char *buffer; /* buffer to store cached data*/
75 size_t buffer_len; /* currently allocated buffers length */
76 size_t buffer_pos; /* end of data in buffer*/
77 int still_running; /* Is background url fetch still in progress */
78 };
79
80 typedef struct fcurl_data URL_FILE;
81
82 /* exported functions */
83 URL_FILE *url_fopen(const char *url, const char *operation);
84 int url_fclose(URL_FILE *file);
85 int url_feof(URL_FILE *file);
86 size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file);
87 char *url_fgets(char *ptr, size_t size, URL_FILE *file);
88 void url_rewind(URL_FILE *file);
89
90 /* we use a global one for convenience */
91 static CURLM *multi_handle;
92
93 /* curl calls this routine to get more data */
write_callback(char * buffer,size_t size,size_t nitems,void * userp)94 static size_t write_callback(char *buffer,
95 size_t size,
96 size_t nitems,
97 void *userp)
98 {
99 char *newbuff;
100 size_t rembuff;
101
102 URL_FILE *url = (URL_FILE *)userp;
103 size *= nitems;
104
105 rembuff = url->buffer_len - url->buffer_pos; /* remaining space in buffer */
106
107 if(size > rembuff) {
108 /* not enough space in buffer */
109 newbuff = realloc(url->buffer, url->buffer_len + (size - rembuff));
110 if(newbuff == NULL) {
111 fprintf(stderr, "callback buffer grow failed\n");
112 size = rembuff;
113 }
114 else {
115 /* realloc succeeded increase buffer size*/
116 url->buffer_len += size - rembuff;
117 url->buffer = newbuff;
118 }
119 }
120
121 memcpy(&url->buffer[url->buffer_pos], buffer, size);
122 url->buffer_pos += size;
123
124 return size;
125 }
126
127 /* use to attempt to fill the read buffer up to requested number of bytes */
fill_buffer(URL_FILE * file,size_t want)128 static int fill_buffer(URL_FILE *file, size_t want)
129 {
130 fd_set fdread;
131 fd_set fdwrite;
132 fd_set fdexcep;
133 struct timeval timeout;
134 int rc;
135 CURLMcode mc; /* curl_multi_fdset() return code */
136
137 /* only attempt to fill buffer if transactions still running and buffer
138 * doesn't exceed required size already
139 */
140 if((!file->still_running) || (file->buffer_pos > want))
141 return 0;
142
143 /* attempt to fill buffer */
144 do {
145 int maxfd = -1;
146 long curl_timeo = -1;
147
148 FD_ZERO(&fdread);
149 FD_ZERO(&fdwrite);
150 FD_ZERO(&fdexcep);
151
152 /* set a suitable timeout to fail on */
153 timeout.tv_sec = 60; /* 1 minute */
154 timeout.tv_usec = 0;
155
156 curl_multi_timeout(multi_handle, &curl_timeo);
157 if(curl_timeo >= 0) {
158 timeout.tv_sec = curl_timeo / 1000;
159 if(timeout.tv_sec > 1)
160 timeout.tv_sec = 1;
161 else
162 timeout.tv_usec = (curl_timeo % 1000) * 1000;
163 }
164
165 /* get file descriptors from the transfers */
166 mc = curl_multi_fdset(multi_handle, &fdread, &fdwrite, &fdexcep, &maxfd);
167
168 if(mc != CURLM_OK) {
169 fprintf(stderr, "curl_multi_fdset() failed, code %d.\n", mc);
170 break;
171 }
172
173 /* On success the value of maxfd is guaranteed to be >= -1. We call
174 select(maxfd + 1, ...); specially in case of (maxfd == -1) there are
175 no fds ready yet so we call select(0, ...) --or Sleep() on Windows--
176 to sleep 100ms, which is the minimum suggested value in the
177 curl_multi_fdset() doc. */
178
179 if(maxfd == -1) {
180 #ifdef _WIN32
181 Sleep(100);
182 rc = 0;
183 #else
184 /* Portable sleep for platforms other than Windows. */
185 struct timeval wait = { 0, 100 * 1000 }; /* 100ms */
186 rc = select(0, NULL, NULL, NULL, &wait);
187 #endif
188 }
189 else {
190 /* Note that on some platforms 'timeout' may be modified by select().
191 If you need access to the original value save a copy beforehand. */
192 rc = select(maxfd + 1, &fdread, &fdwrite, &fdexcep, &timeout);
193 }
194
195 switch(rc) {
196 case -1:
197 /* select error */
198 break;
199
200 case 0:
201 default:
202 /* timeout or readable/writable sockets */
203 curl_multi_perform(multi_handle, &file->still_running);
204 break;
205 }
206 } while(file->still_running && (file->buffer_pos < want));
207 return 1;
208 }
209
210 /* use to remove want bytes from the front of a files buffer */
use_buffer(URL_FILE * file,size_t want)211 static int use_buffer(URL_FILE *file, size_t want)
212 {
213 /* sort out buffer */
214 if((file->buffer_pos - want) <= 0) {
215 /* ditch buffer - write will recreate */
216 free(file->buffer);
217 file->buffer = NULL;
218 file->buffer_pos = 0;
219 file->buffer_len = 0;
220 }
221 else {
222 /* move rest down make it available for later */
223 memmove(file->buffer,
224 &file->buffer[want],
225 (file->buffer_pos - want));
226
227 file->buffer_pos -= want;
228 }
229 return 0;
230 }
231
url_fopen(const char * url,const char * operation)232 URL_FILE *url_fopen(const char *url, const char *operation)
233 {
234 /* this code could check for URLs or types in the 'url' and
235 basically use the real fopen() for standard files */
236
237 URL_FILE *file;
238 (void)operation;
239
240 file = malloc(sizeof(URL_FILE));
241 if(!file)
242 return NULL;
243
244 memset(file, 0, sizeof(URL_FILE));
245
246 file->handle.file = fopen(url, operation);
247 if(file->handle.file)
248 file->type = CFTYPE_FILE; /* marked as URL */
249
250 else {
251 file->type = CFTYPE_CURL; /* marked as URL */
252 file->handle.curl = curl_easy_init();
253
254 curl_easy_setopt(file->handle.curl, CURLOPT_URL, url);
255 curl_easy_setopt(file->handle.curl, CURLOPT_WRITEDATA, file);
256 curl_easy_setopt(file->handle.curl, CURLOPT_VERBOSE, 0L);
257 curl_easy_setopt(file->handle.curl, CURLOPT_WRITEFUNCTION, write_callback);
258
259 if(!multi_handle)
260 multi_handle = curl_multi_init();
261
262 curl_multi_add_handle(multi_handle, file->handle.curl);
263
264 /* lets start the fetch */
265 curl_multi_perform(multi_handle, &file->still_running);
266
267 if((file->buffer_pos == 0) && (!file->still_running)) {
268 /* if still_running is 0 now, we should return NULL */
269
270 /* make sure the easy handle is not in the multi handle anymore */
271 curl_multi_remove_handle(multi_handle, file->handle.curl);
272
273 /* cleanup */
274 curl_easy_cleanup(file->handle.curl);
275
276 free(file);
277
278 file = NULL;
279 }
280 }
281 return file;
282 }
283
url_fclose(URL_FILE * file)284 int url_fclose(URL_FILE *file)
285 {
286 int ret = 0;/* default is good return */
287
288 switch(file->type) {
289 case CFTYPE_FILE:
290 ret = fclose(file->handle.file); /* passthrough */
291 break;
292
293 case CFTYPE_CURL:
294 /* make sure the easy handle is not in the multi handle anymore */
295 curl_multi_remove_handle(multi_handle, file->handle.curl);
296
297 /* cleanup */
298 curl_easy_cleanup(file->handle.curl);
299 break;
300
301 default: /* unknown or supported type - oh dear */
302 ret = EOF;
303 errno = EBADF;
304 break;
305 }
306
307 free(file->buffer);/* free any allocated buffer space */
308 free(file);
309
310 return ret;
311 }
312
url_feof(URL_FILE * file)313 int url_feof(URL_FILE *file)
314 {
315 int ret = 0;
316
317 switch(file->type) {
318 case CFTYPE_FILE:
319 ret = feof(file->handle.file);
320 break;
321
322 case CFTYPE_CURL:
323 if((file->buffer_pos == 0) && (!file->still_running))
324 ret = 1;
325 break;
326
327 default: /* unknown or supported type - oh dear */
328 ret = -1;
329 errno = EBADF;
330 break;
331 }
332 return ret;
333 }
334
url_fread(void * ptr,size_t size,size_t nmemb,URL_FILE * file)335 size_t url_fread(void *ptr, size_t size, size_t nmemb, URL_FILE *file)
336 {
337 size_t want;
338
339 switch(file->type) {
340 case CFTYPE_FILE:
341 want = fread(ptr, size, nmemb, file->handle.file);
342 break;
343
344 case CFTYPE_CURL:
345 want = nmemb * size;
346
347 fill_buffer(file, want);
348
349 /* check if there's data in the buffer - if not fill_buffer()
350 * either errored or EOF */
351 if(!file->buffer_pos)
352 return 0;
353
354 /* ensure only available data is considered */
355 if(file->buffer_pos < want)
356 want = file->buffer_pos;
357
358 /* xfer data to caller */
359 memcpy(ptr, file->buffer, want);
360
361 use_buffer(file, want);
362
363 want = want / size; /* number of items */
364 break;
365
366 default: /* unknown or supported type - oh dear */
367 want = 0;
368 errno = EBADF;
369 break;
370
371 }
372 return want;
373 }
374
url_fgets(char * ptr,size_t size,URL_FILE * file)375 char *url_fgets(char *ptr, size_t size, URL_FILE *file)
376 {
377 size_t want = size - 1;/* always need to leave room for zero termination */
378 size_t loop;
379
380 switch(file->type) {
381 case CFTYPE_FILE:
382 ptr = fgets(ptr, (int)size, file->handle.file);
383 break;
384
385 case CFTYPE_CURL:
386 fill_buffer(file, want);
387
388 /* check if there's data in the buffer - if not fill either errored or
389 * EOF */
390 if(!file->buffer_pos)
391 return NULL;
392
393 /* ensure only available data is considered */
394 if(file->buffer_pos < want)
395 want = file->buffer_pos;
396
397 /*buffer contains data */
398 /* look for newline or eof */
399 for(loop = 0; loop < want; loop++) {
400 if(file->buffer[loop] == '\n') {
401 want = loop + 1;/* include newline */
402 break;
403 }
404 }
405
406 /* xfer data to caller */
407 memcpy(ptr, file->buffer, want);
408 ptr[want] = 0;/* always null terminate */
409
410 use_buffer(file, want);
411
412 break;
413
414 default: /* unknown or supported type - oh dear */
415 ptr = NULL;
416 errno = EBADF;
417 break;
418 }
419
420 return ptr;/*success */
421 }
422
url_rewind(URL_FILE * file)423 void url_rewind(URL_FILE *file)
424 {
425 switch(file->type) {
426 case CFTYPE_FILE:
427 rewind(file->handle.file); /* passthrough */
428 break;
429
430 case CFTYPE_CURL:
431 /* halt transaction */
432 curl_multi_remove_handle(multi_handle, file->handle.curl);
433
434 /* restart */
435 curl_multi_add_handle(multi_handle, file->handle.curl);
436
437 /* ditch buffer - write will recreate - resets stream pos*/
438 free(file->buffer);
439 file->buffer = NULL;
440 file->buffer_pos = 0;
441 file->buffer_len = 0;
442
443 break;
444
445 default: /* unknown or supported type - oh dear */
446 break;
447 }
448 }
449
450 #define FGETSFILE "fgets.test"
451 #define FREADFILE "fread.test"
452 #define REWINDFILE "rewind.test"
453
454 /* Small main program to retrieve from a url using fgets and fread saving the
455 * output to two test files (note the fgets method will corrupt binary files if
456 * they contain 0 chars */
main(int argc,char * argv[])457 int main(int argc, char *argv[])
458 {
459 URL_FILE *handle;
460 FILE *outf;
461
462 size_t nread;
463 char buffer[256];
464 const char *url;
465
466 if(argc < 2)
467 url = "http://192.168.7.3/testfile";/* default to testurl */
468 else
469 url = argv[1];/* use passed url */
470
471 /* copy from url line by line with fgets */
472 outf = fopen(FGETSFILE, "wb+");
473 if(!outf) {
474 perror("couldn't open fgets output file\n");
475 return 1;
476 }
477
478 handle = url_fopen(url, "r");
479 if(!handle) {
480 printf("couldn't url_fopen() %s\n", url);
481 fclose(outf);
482 return 2;
483 }
484
485 while(!url_feof(handle)) {
486 url_fgets(buffer, sizeof(buffer), handle);
487 fwrite(buffer, 1, strlen(buffer), outf);
488 }
489
490 url_fclose(handle);
491
492 fclose(outf);
493
494
495 /* Copy from url with fread */
496 outf = fopen(FREADFILE, "wb+");
497 if(!outf) {
498 perror("couldn't open fread output file\n");
499 return 1;
500 }
501
502 handle = url_fopen("testfile", "r");
503 if(!handle) {
504 printf("couldn't url_fopen() testfile\n");
505 fclose(outf);
506 return 2;
507 }
508
509 do {
510 nread = url_fread(buffer, 1, sizeof(buffer), handle);
511 fwrite(buffer, 1, nread, outf);
512 } while(nread);
513
514 url_fclose(handle);
515
516 fclose(outf);
517
518
519 /* Test rewind */
520 outf = fopen(REWINDFILE, "wb+");
521 if(!outf) {
522 perror("couldn't open fread output file\n");
523 return 1;
524 }
525
526 handle = url_fopen("testfile", "r");
527 if(!handle) {
528 printf("couldn't url_fopen() testfile\n");
529 fclose(outf);
530 return 2;
531 }
532
533 nread = url_fread(buffer, 1, sizeof(buffer), handle);
534 fwrite(buffer, 1, nread, outf);
535 url_rewind(handle);
536
537 buffer[0]='\n';
538 fwrite(buffer, 1, 1, outf);
539
540 nread = url_fread(buffer, 1, sizeof(buffer), handle);
541 fwrite(buffer, 1, nread, outf);
542
543 url_fclose(handle);
544
545 fclose(outf);
546
547 return 0;/* all done */
548 }
549