9  * Copyright (C) 2018 - 2020 Jeroen Ooms <jeroenooms@gmail.com>
13  *   gcc crawler.c $(pkg-config --cflags --libs libxml-2.0 libcurl)
17  * Web crawler based on curl and libxml2 to stress-test curl with
55   char *ptr = realloc(mem->buf, mem->size + realsize);  in grow_buffer()
61   mem->buf = ptr;  in grow_buffer()
62   memcpy(&(mem->buf[mem->size]), contents, realsize);  in grow_buffer()
63   mem->size += realsize;  in grow_buffer()
69   CURL *handle = curl_easy_init();  in make_handle()  local
72   curl_easy_setopt(handle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2TLS);  in make_handle()
73   curl_easy_setopt(handle, CURLOPT_URL, url);  in make_handle()
77   mem->size = 0;  in make_handle()
78   mem->buf = malloc(1);  in make_handle()
79   curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, grow_buffer);  in make_handle()
80   curl_easy_setopt(handle, CURLOPT_WRITEDATA, mem);  in make_handle()
81   curl_easy_setopt(handle, CURLOPT_PRIVATE, mem);  in make_handle()
84   curl_easy_setopt(handle, CURLOPT_ACCEPT_ENCODING, "");  in make_handle()
85   curl_easy_setopt(handle, CURLOPT_TIMEOUT, 5L);  in make_handle()
86   curl_easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 1L);  in make_handle()
87   curl_easy_setopt(handle, CURLOPT_MAXREDIRS, 10L);  in make_handle()
88   curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT, 2L);  in make_handle()
89   curl_easy_setopt(handle, CURLOPT_COOKIEFILE, "");  in make_handle()
90   curl_easy_setopt(handle, CURLOPT_FILETIME, 1L);  in make_handle()
91   curl_easy_setopt(handle, CURLOPT_USERAGENT, "mini crawler");  in make_handle()
92   curl_easy_setopt(handle, CURLOPT_HTTPAUTH, CURLAUTH_ANY);  in make_handle()
93   curl_easy_setopt(handle, CURLOPT_UNRESTRICTED_AUTH, 1L);  in make_handle()
94   curl_easy_setopt(handle, CURLOPT_PROXYAUTH, CURLAUTH_ANY);  in make_handle()
95   curl_easy_setopt(handle, CURLOPT_EXPECT_100_TIMEOUT_MS, 0L);  in make_handle()
96   return handle;  in make_handle()
104   htmlDocPtr doc = htmlReadMemory(mem->buf, mem->size, url, NULL, opts);  in follow_links()
113   xmlNodeSetPtr nodeset = result->nodesetval;  in follow_links()
120   for(i = 0; i < nodeset->nodeNr; i++) {  in follow_links()
122     int x = r * nodeset->nodeNr / RAND_MAX;  in follow_links()
123     const xmlNode *node = nodeset->nodeTab[x]->xmlChildrenNode;  in follow_links()
151   signal(SIGINT, sighandler);  in main()
178       if(m->msg == CURLMSG_DONE) {  in main()
179         CURL *handle = m->easy_handle;  in main()  local
182         curl_easy_getinfo(handle, CURLINFO_PRIVATE, &mem);  in main()
183         curl_easy_getinfo(handle, CURLINFO_EFFECTIVE_URL, &url);  in main()
184         if(m->data.result == CURLE_OK) {  in main()
186           curl_easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &res_status);  in main()
189             curl_easy_getinfo(handle, CURLINFO_CONTENT_TYPE, &ctype);  in main()
191             if(is_html(ctype) && mem->size > 100) {  in main()
205         curl_multi_remove_handle(multi_handle, handle);  in main()
206         curl_easy_cleanup(handle);  in main()
207         free(mem->buf);  in main()
210         pending--;  in main()