1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3#*************************************************************************** 4# _ _ ____ _ 5# Project ___| | | | _ \| | 6# / __| | | | |_) | | 7# | (__| |_| | _ <| |___ 8# \___|\___/|_| \_\_____| 9# 10# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al. 11# 12# This software is licensed as described in the file COPYING, which 13# you should have received as part of this distribution. The terms 14# are also available at https://curl.se/docs/copyright.html. 15# 16# You may opt to use, copy, modify, merge, publish, distribute and/or sell 17# copies of the Software, and permit persons to whom the Software is 18# furnished to do so, under the terms of the COPYING file. 19# 20# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY 21# KIND, either express or implied. 22# 23# SPDX-License-Identifier: curl 24# 25########################################################################### 26# 27import difflib 28import filecmp 29import logging 30import math 31import os 32import re 33from datetime import timedelta 34import pytest 35 36from testenv import Env, CurlClient, LocalClient 37 38 39log = logging.getLogger(__name__) 40 41 42class TestDownload: 43 44 @pytest.fixture(autouse=True, scope='class') 45 def _class_scope(self, env, httpd, nghttpx): 46 if env.have_h3(): 47 nghttpx.start_if_needed() 48 httpd.clear_extra_configs() 49 httpd.reload() 50 51 @pytest.fixture(autouse=True, scope='class') 52 def _class_scope(self, env, httpd): 53 indir = httpd.docs_dir 54 env.make_data_file(indir=indir, fname="data-10k", fsize=10*1024) 55 env.make_data_file(indir=indir, fname="data-100k", fsize=100*1024) 56 env.make_data_file(indir=indir, fname="data-1m", fsize=1024*1024) 57 env.make_data_file(indir=indir, fname="data-10m", fsize=10*1024*1024) 58 env.make_data_file(indir=indir, fname="data-50m", fsize=50*1024*1024) 59 60 # download 1 file 61 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 62 def test_02_01_download_1(self, env: Env, httpd, nghttpx, proto): 63 if proto == 'h3' and not env.have_h3(): 64 pytest.skip("h3 not supported") 65 curl = CurlClient(env=env) 66 url = f'https://{env.authority_for(env.domain1, proto)}/data.json' 67 r = curl.http_download(urls=[url], alpn_proto=proto) 68 r.check_response(http_status=200) 69 70 # download 2 files 71 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 72 def test_02_02_download_2(self, env: Env, httpd, nghttpx, proto): 73 if proto == 'h3' and not env.have_h3(): 74 pytest.skip("h3 not supported") 75 curl = CurlClient(env=env) 76 url = f'https://{env.authority_for(env.domain1, proto)}/data.json?[0-1]' 77 r = curl.http_download(urls=[url], alpn_proto=proto) 78 r.check_response(http_status=200, count=2) 79 80 # download 100 files sequentially 81 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 82 def test_02_03_download_sequential(self, env: Env, httpd, nghttpx, proto): 83 if proto == 'h3' and not env.have_h3(): 84 pytest.skip("h3 not supported") 85 count = 10 86 curl = CurlClient(env=env) 87 urln = f'https://{env.authority_for(env.domain1, proto)}/data.json?[0-{count-1}]' 88 r = curl.http_download(urls=[urln], alpn_proto=proto) 89 r.check_response(http_status=200, count=count, connect_count=1) 90 91 # download 100 files parallel 92 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 93 def test_02_04_download_parallel(self, env: Env, httpd, nghttpx, proto): 94 if proto == 'h3' and not env.have_h3(): 95 pytest.skip("h3 not supported") 96 count = 10 97 max_parallel = 5 98 curl = CurlClient(env=env) 99 urln = f'https://{env.authority_for(env.domain1, proto)}/data.json?[0-{count-1}]' 100 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[ 101 '--parallel', '--parallel-max', f'{max_parallel}' 102 ]) 103 r.check_response(http_status=200, count=count) 104 if proto == 'http/1.1': 105 # http/1.1 parallel transfers will open multiple connections 106 assert r.total_connects > 1, r.dump_logs() 107 else: 108 # http2 parallel transfers will use one connection (common limit is 100) 109 assert r.total_connects == 1, r.dump_logs() 110 111 # download 500 files sequential 112 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 113 def test_02_05_download_many_sequential(self, env: Env, httpd, nghttpx, proto): 114 if proto == 'h3' and not env.have_h3(): 115 pytest.skip("h3 not supported") 116 if proto == 'h3' and env.curl_uses_lib('msh3'): 117 pytest.skip("msh3 shaky here") 118 count = 200 119 curl = CurlClient(env=env) 120 urln = f'https://{env.authority_for(env.domain1, proto)}/data.json?[0-{count-1}]' 121 r = curl.http_download(urls=[urln], alpn_proto=proto) 122 r.check_response(http_status=200, count=count) 123 if proto == 'http/1.1': 124 # http/1.1 parallel transfers will open multiple connections 125 assert r.total_connects > 1, r.dump_logs() 126 else: 127 # http2 parallel transfers will use one connection (common limit is 100) 128 assert r.total_connects == 1, r.dump_logs() 129 130 # download 500 files parallel 131 @pytest.mark.parametrize("proto", ['h2', 'h3']) 132 def test_02_06_download_many_parallel(self, env: Env, httpd, nghttpx, proto): 133 if proto == 'h3' and not env.have_h3(): 134 pytest.skip("h3 not supported") 135 count = 200 136 max_parallel = 50 137 curl = CurlClient(env=env) 138 urln = f'https://{env.authority_for(env.domain1, proto)}/data.json?[000-{count-1}]' 139 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[ 140 '--parallel', '--parallel-max', f'{max_parallel}' 141 ]) 142 r.check_response(http_status=200, count=count, connect_count=1) 143 144 # download files parallel, check connection reuse/multiplex 145 @pytest.mark.parametrize("proto", ['h2', 'h3']) 146 def test_02_07_download_reuse(self, env: Env, httpd, nghttpx, proto): 147 if proto == 'h3' and not env.have_h3(): 148 pytest.skip("h3 not supported") 149 count = 200 150 curl = CurlClient(env=env) 151 urln = f'https://{env.authority_for(env.domain1, proto)}/data.json?[0-{count-1}]' 152 r = curl.http_download(urls=[urln], alpn_proto=proto, 153 with_stats=True, extra_args=[ 154 '--parallel', '--parallel-max', '200' 155 ]) 156 r.check_response(http_status=200, count=count) 157 # should have used at most 2 connections only (test servers allow 100 req/conn) 158 # it may be just 1 on slow systems where request are answered faster than 159 # curl can exhaust the capacity or if curl runs with address-sanitizer speed 160 assert r.total_connects <= 2, "h2 should use fewer connections here" 161 162 # download files parallel with http/1.1, check connection not reused 163 @pytest.mark.parametrize("proto", ['http/1.1']) 164 def test_02_07b_download_reuse(self, env: Env, httpd, nghttpx, proto): 165 count = 6 166 curl = CurlClient(env=env) 167 urln = f'https://{env.authority_for(env.domain1, proto)}/data.json?[0-{count-1}]' 168 r = curl.http_download(urls=[urln], alpn_proto=proto, 169 with_stats=True, extra_args=[ 170 '--parallel' 171 ]) 172 r.check_response(count=count, http_status=200) 173 # http/1.1 should have used count connections 174 assert r.total_connects == count, "http/1.1 should use this many connections" 175 176 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 177 def test_02_08_1MB_serial(self, env: Env, httpd, nghttpx, proto): 178 if proto == 'h3' and not env.have_h3(): 179 pytest.skip("h3 not supported") 180 count = 5 181 urln = f'https://{env.authority_for(env.domain1, proto)}/data-1m?[0-{count-1}]' 182 curl = CurlClient(env=env) 183 r = curl.http_download(urls=[urln], alpn_proto=proto) 184 r.check_response(count=count, http_status=200) 185 186 @pytest.mark.parametrize("proto", ['h2', 'h3']) 187 def test_02_09_1MB_parallel(self, env: Env, httpd, nghttpx, proto): 188 if proto == 'h3' and not env.have_h3(): 189 pytest.skip("h3 not supported") 190 count = 5 191 urln = f'https://{env.authority_for(env.domain1, proto)}/data-1m?[0-{count-1}]' 192 curl = CurlClient(env=env) 193 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[ 194 '--parallel' 195 ]) 196 r.check_response(count=count, http_status=200) 197 198 @pytest.mark.skipif(condition=Env().slow_network, reason="not suitable for slow network tests") 199 @pytest.mark.skipif(condition=Env().ci_run, reason="not suitable for CI runs") 200 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 201 def test_02_10_10MB_serial(self, env: Env, httpd, nghttpx, proto): 202 if proto == 'h3' and not env.have_h3(): 203 pytest.skip("h3 not supported") 204 count = 3 205 urln = f'https://{env.authority_for(env.domain1, proto)}/data-10m?[0-{count-1}]' 206 curl = CurlClient(env=env) 207 r = curl.http_download(urls=[urln], alpn_proto=proto) 208 r.check_response(count=count, http_status=200) 209 210 @pytest.mark.skipif(condition=Env().slow_network, reason="not suitable for slow network tests") 211 @pytest.mark.skipif(condition=Env().ci_run, reason="not suitable for CI runs") 212 @pytest.mark.parametrize("proto", ['h2', 'h3']) 213 def test_02_11_10MB_parallel(self, env: Env, httpd, nghttpx, proto): 214 if proto == 'h3' and not env.have_h3(): 215 pytest.skip("h3 not supported") 216 if proto == 'h3' and env.curl_uses_lib('msh3'): 217 pytest.skip("msh3 stalls here") 218 count = 3 219 urln = f'https://{env.authority_for(env.domain1, proto)}/data-10m?[0-{count-1}]' 220 curl = CurlClient(env=env) 221 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[ 222 '--parallel' 223 ]) 224 r.check_response(count=count, http_status=200) 225 226 @pytest.mark.parametrize("proto", ['h2', 'h3']) 227 def test_02_12_head_serial_https(self, env: Env, httpd, nghttpx, proto): 228 if proto == 'h3' and not env.have_h3(): 229 pytest.skip("h3 not supported") 230 count = 5 231 urln = f'https://{env.authority_for(env.domain1, proto)}/data-10m?[0-{count-1}]' 232 curl = CurlClient(env=env) 233 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[ 234 '--head' 235 ]) 236 r.check_response(count=count, http_status=200) 237 238 @pytest.mark.parametrize("proto", ['h2']) 239 def test_02_13_head_serial_h2c(self, env: Env, httpd, nghttpx, proto): 240 if proto == 'h3' and not env.have_h3(): 241 pytest.skip("h3 not supported") 242 count = 5 243 urln = f'http://{env.domain1}:{env.http_port}/data-10m?[0-{count-1}]' 244 curl = CurlClient(env=env) 245 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[ 246 '--head', '--http2-prior-knowledge', '--fail-early' 247 ]) 248 r.check_response(count=count, http_status=200) 249 250 @pytest.mark.parametrize("proto", ['h2', 'h3']) 251 def test_02_14_not_found(self, env: Env, httpd, nghttpx, proto): 252 if proto == 'h3' and not env.have_h3(): 253 pytest.skip("h3 not supported") 254 if proto == 'h3' and env.curl_uses_lib('msh3'): 255 pytest.skip("msh3 stalls here") 256 count = 5 257 urln = f'https://{env.authority_for(env.domain1, proto)}/not-found?[0-{count-1}]' 258 curl = CurlClient(env=env) 259 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[ 260 '--parallel' 261 ]) 262 r.check_stats(count=count, http_status=404, exitcode=0, 263 remote_port=env.port_for(alpn_proto=proto), 264 remote_ip='127.0.0.1') 265 266 @pytest.mark.parametrize("proto", ['h2', 'h3']) 267 def test_02_15_fail_not_found(self, env: Env, httpd, nghttpx, proto): 268 if proto == 'h3' and not env.have_h3(): 269 pytest.skip("h3 not supported") 270 if proto == 'h3' and env.curl_uses_lib('msh3'): 271 pytest.skip("msh3 stalls here") 272 count = 5 273 urln = f'https://{env.authority_for(env.domain1, proto)}/not-found?[0-{count-1}]' 274 curl = CurlClient(env=env) 275 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[ 276 '--fail' 277 ]) 278 r.check_stats(count=count, http_status=404, exitcode=22, 279 remote_port=env.port_for(alpn_proto=proto), 280 remote_ip='127.0.0.1') 281 282 @pytest.mark.skipif(condition=Env().slow_network, reason="not suitable for slow network tests") 283 def test_02_20_h2_small_frames(self, env: Env, httpd): 284 # Test case to reproduce content corruption as observed in 285 # https://github.com/curl/curl/issues/10525 286 # To reliably reproduce, we need an Apache httpd that supports 287 # setting smaller frame sizes. This is not released yet, we 288 # test if it works and back out if not. 289 httpd.set_extra_config(env.domain1, lines=[ 290 'H2MaxDataFrameLen 1024', 291 ]) 292 assert httpd.stop() 293 if not httpd.start(): 294 # no, not supported, bail out 295 httpd.set_extra_config(env.domain1, lines=None) 296 assert httpd.start() 297 pytest.skip('H2MaxDataFrameLen not supported') 298 # ok, make 100 downloads with 2 parallel running and they 299 # are expected to stumble into the issue when using `lib/http2.c` 300 # from curl 7.88.0 301 count = 5 302 urln = f'https://{env.authority_for(env.domain1, "h2")}/data-1m?[0-{count-1}]' 303 curl = CurlClient(env=env) 304 r = curl.http_download(urls=[urln], alpn_proto="h2", extra_args=[ 305 '--parallel', '--parallel-max', '2' 306 ]) 307 r.check_response(count=count, http_status=200) 308 srcfile = os.path.join(httpd.docs_dir, 'data-1m') 309 self.check_downloads(curl, srcfile, count) 310 # restore httpd defaults 311 httpd.set_extra_config(env.domain1, lines=None) 312 assert httpd.stop() 313 assert httpd.start() 314 315 # download via lib client, 1 at a time, pause/resume at different offsets 316 @pytest.mark.parametrize("pause_offset", [0, 10*1024, 100*1023, 640000]) 317 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 318 def test_02_21_lib_serial(self, env: Env, httpd, nghttpx, proto, pause_offset): 319 if proto == 'h3' and not env.have_h3(): 320 pytest.skip("h3 not supported") 321 count = 2 322 docname = 'data-10m' 323 url = f'https://localhost:{env.https_port}/{docname}' 324 client = LocalClient(name='hx-download', env=env) 325 if not client.exists(): 326 pytest.skip(f'example client not built: {client.name}') 327 r = client.run(args=[ 328 '-n', f'{count}', '-P', f'{pause_offset}', '-V', proto, url 329 ]) 330 r.check_exit_code(0) 331 srcfile = os.path.join(httpd.docs_dir, docname) 332 self.check_downloads(client, srcfile, count) 333 334 # download via lib client, several at a time, pause/resume 335 @pytest.mark.parametrize("pause_offset", [100*1023]) 336 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 337 def test_02_22_lib_parallel_resume(self, env: Env, httpd, nghttpx, proto, pause_offset): 338 if proto == 'h3' and not env.have_h3(): 339 pytest.skip("h3 not supported") 340 count = 2 341 max_parallel = 5 342 docname = 'data-10m' 343 url = f'https://localhost:{env.https_port}/{docname}' 344 client = LocalClient(name='hx-download', env=env) 345 if not client.exists(): 346 pytest.skip(f'example client not built: {client.name}') 347 r = client.run(args=[ 348 '-n', f'{count}', '-m', f'{max_parallel}', 349 '-P', f'{pause_offset}', '-V', proto, url 350 ]) 351 r.check_exit_code(0) 352 srcfile = os.path.join(httpd.docs_dir, docname) 353 self.check_downloads(client, srcfile, count) 354 355 # download, several at a time, pause and abort paused 356 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 357 def test_02_23a_lib_abort_paused(self, env: Env, httpd, nghttpx, proto): 358 if proto == 'h3' and not env.have_h3(): 359 pytest.skip("h3 not supported") 360 if proto == 'h3' and env.curl_uses_ossl_quic(): 361 pytest.skip('OpenSSL QUIC fails here') 362 if proto == 'h3' and env.ci_run and env.curl_uses_lib('quiche'): 363 pytest.skip("fails in CI, but works locally for unknown reasons") 364 count = 10 365 max_parallel = 5 366 if proto in ['h2', 'h3']: 367 pause_offset = 64 * 1024 368 else: 369 pause_offset = 12 * 1024 370 docname = 'data-1m' 371 url = f'https://localhost:{env.https_port}/{docname}' 372 client = LocalClient(name='hx-download', env=env) 373 if not client.exists(): 374 pytest.skip(f'example client not built: {client.name}') 375 r = client.run(args=[ 376 '-n', f'{count}', '-m', f'{max_parallel}', '-a', 377 '-P', f'{pause_offset}', '-V', proto, url 378 ]) 379 r.check_exit_code(0) 380 srcfile = os.path.join(httpd.docs_dir, docname) 381 # downloads should be there, but not necessarily complete 382 self.check_downloads(client, srcfile, count, complete=False) 383 384 # download, several at a time, abort after n bytes 385 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 386 def test_02_23b_lib_abort_offset(self, env: Env, httpd, nghttpx, proto): 387 if proto == 'h3' and not env.have_h3(): 388 pytest.skip("h3 not supported") 389 if proto == 'h3' and env.curl_uses_ossl_quic(): 390 pytest.skip('OpenSSL QUIC fails here') 391 if proto == 'h3' and env.ci_run and env.curl_uses_lib('quiche'): 392 pytest.skip("fails in CI, but works locally for unknown reasons") 393 count = 10 394 max_parallel = 5 395 if proto in ['h2', 'h3']: 396 abort_offset = 64 * 1024 397 else: 398 abort_offset = 12 * 1024 399 docname = 'data-1m' 400 url = f'https://localhost:{env.https_port}/{docname}' 401 client = LocalClient(name='hx-download', env=env) 402 if not client.exists(): 403 pytest.skip(f'example client not built: {client.name}') 404 r = client.run(args=[ 405 '-n', f'{count}', '-m', f'{max_parallel}', '-a', 406 '-A', f'{abort_offset}', '-V', proto, url 407 ]) 408 r.check_exit_code(0) 409 srcfile = os.path.join(httpd.docs_dir, docname) 410 # downloads should be there, but not necessarily complete 411 self.check_downloads(client, srcfile, count, complete=False) 412 413 # download, several at a time, abort after n bytes 414 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 415 def test_02_23c_lib_fail_offset(self, env: Env, httpd, nghttpx, proto): 416 if proto == 'h3' and not env.have_h3(): 417 pytest.skip("h3 not supported") 418 if proto == 'h3' and env.curl_uses_ossl_quic(): 419 pytest.skip('OpenSSL QUIC fails here') 420 if proto == 'h3' and env.ci_run and env.curl_uses_lib('quiche'): 421 pytest.skip("fails in CI, but works locally for unknown reasons") 422 count = 10 423 max_parallel = 5 424 if proto in ['h2', 'h3']: 425 fail_offset = 64 * 1024 426 else: 427 fail_offset = 12 * 1024 428 docname = 'data-1m' 429 url = f'https://localhost:{env.https_port}/{docname}' 430 client = LocalClient(name='hx-download', env=env) 431 if not client.exists(): 432 pytest.skip(f'example client not built: {client.name}') 433 r = client.run(args=[ 434 '-n', f'{count}', '-m', f'{max_parallel}', '-a', 435 '-F', f'{fail_offset}', '-V', proto, url 436 ]) 437 r.check_exit_code(0) 438 srcfile = os.path.join(httpd.docs_dir, docname) 439 # downloads should be there, but not necessarily complete 440 self.check_downloads(client, srcfile, count, complete=False) 441 442 # speed limited download 443 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 444 def test_02_24_speed_limit(self, env: Env, httpd, nghttpx, proto): 445 if proto == 'h3' and not env.have_h3(): 446 pytest.skip("h3 not supported") 447 count = 1 448 url = f'https://{env.authority_for(env.domain1, proto)}/data-1m' 449 curl = CurlClient(env=env) 450 speed_limit = 384 * 1024 451 min_duration = math.floor((1024 * 1024)/speed_limit) 452 r = curl.http_download(urls=[url], alpn_proto=proto, extra_args=[ 453 '--limit-rate', f'{speed_limit}' 454 ]) 455 r.check_response(count=count, http_status=200) 456 assert r.duration > timedelta(seconds=min_duration), \ 457 f'rate limited transfer should take more than {min_duration}s, '\ 458 f'not {r.duration}' 459 460 # make extreme parallel h2 upgrades, check invalid conn reuse 461 # before protocol switch has happened 462 def test_02_25_h2_upgrade_x(self, env: Env, httpd): 463 url = f'http://localhost:{env.http_port}/data-100k' 464 client = LocalClient(name='h2-upgrade-extreme', env=env, timeout=15) 465 if not client.exists(): 466 pytest.skip(f'example client not built: {client.name}') 467 r = client.run(args=[url]) 468 assert r.exit_code == 0, f'{client.dump_logs()}' 469 470 # Special client that tests TLS session reuse in parallel transfers 471 # TODO: just uses a single connection for h2/h3. Not sure how to prevent that 472 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 473 def test_02_26_session_shared_reuse(self, env: Env, proto, httpd, nghttpx): 474 url = f'https://{env.authority_for(env.domain1, proto)}/data-100k' 475 client = LocalClient(name='tls-session-reuse', env=env) 476 if not client.exists(): 477 pytest.skip(f'example client not built: {client.name}') 478 r = client.run(args=[proto, url]) 479 r.check_exit_code(0) 480 481 # test on paused transfers, based on issue #11982 482 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 483 def test_02_27a_paused_no_cl(self, env: Env, httpd, nghttpx, proto): 484 url = f'https://{env.authority_for(env.domain1, proto)}' \ 485 '/curltest/tweak/?&chunks=6&chunk_size=8000' 486 client = LocalClient(env=env, name='h2-pausing') 487 r = client.run(args=['-V', proto, url]) 488 r.check_exit_code(0) 489 490 # test on paused transfers, based on issue #11982 491 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 492 def test_02_27b_paused_no_cl(self, env: Env, httpd, nghttpx, proto): 493 url = f'https://{env.authority_for(env.domain1, proto)}' \ 494 '/curltest/tweak/?error=502' 495 client = LocalClient(env=env, name='h2-pausing') 496 r = client.run(args=['-V', proto, url]) 497 r.check_exit_code(0) 498 499 # test on paused transfers, based on issue #11982 500 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 501 def test_02_27c_paused_no_cl(self, env: Env, httpd, nghttpx, proto): 502 url = f'https://{env.authority_for(env.domain1, proto)}' \ 503 '/curltest/tweak/?status=200&chunks=1&chunk_size=100' 504 client = LocalClient(env=env, name='h2-pausing') 505 r = client.run(args=['-V', proto, url]) 506 r.check_exit_code(0) 507 508 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 509 def test_02_28_get_compressed(self, env: Env, httpd, nghttpx, proto): 510 if proto == 'h3' and not env.have_h3(): 511 pytest.skip("h3 not supported") 512 count = 1 513 urln = f'https://{env.authority_for(env.domain1brotli, proto)}/data-100k?[0-{count-1}]' 514 curl = CurlClient(env=env) 515 r = curl.http_download(urls=[urln], alpn_proto=proto, extra_args=[ 516 '--compressed' 517 ]) 518 r.check_exit_code(code=0) 519 r.check_response(count=count, http_status=200) 520 521 def check_downloads(self, client, srcfile: str, count: int, 522 complete: bool = True): 523 for i in range(count): 524 dfile = client.download_file(i) 525 assert os.path.exists(dfile) 526 if complete and not filecmp.cmp(srcfile, dfile, shallow=False): 527 diff = "".join(difflib.unified_diff(a=open(srcfile).readlines(), 528 b=open(dfile).readlines(), 529 fromfile=srcfile, 530 tofile=dfile, 531 n=1)) 532 assert False, f'download {dfile} differs:\n{diff}' 533 534 # download via lib client, 1 at a time, pause/resume at different offsets 535 @pytest.mark.parametrize("pause_offset", [0, 10*1024, 100*1023, 640000]) 536 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 537 def test_02_29_h2_lib_serial(self, env: Env, httpd, nghttpx, proto, pause_offset): 538 count = 2 539 docname = 'data-10m' 540 url = f'https://localhost:{env.https_port}/{docname}' 541 client = LocalClient(name='hx-download', env=env) 542 if not client.exists(): 543 pytest.skip(f'example client not built: {client.name}') 544 r = client.run(args=[ 545 '-n', f'{count}', '-P', f'{pause_offset}', '-V', proto, url 546 ]) 547 r.check_exit_code(0) 548 srcfile = os.path.join(httpd.docs_dir, docname) 549 self.check_downloads(client, srcfile, count) 550 551 # download parallel with prior knowledge 552 def test_02_30_parallel_prior_knowledge(self, env: Env, httpd): 553 count = 3 554 curl = CurlClient(env=env) 555 urln = f'http://{env.domain1}:{env.http_port}/data.json?[0-{count-1}]' 556 r = curl.http_download(urls=[urln], extra_args=[ 557 '--parallel', '--http2-prior-knowledge' 558 ]) 559 r.check_response(http_status=200, count=count) 560 assert r.total_connects == 1, r.dump_logs() 561 562 # download parallel with h2 "Upgrade:" 563 def test_02_31_parallel_upgrade(self, env: Env, httpd, nghttpx): 564 count = 3 565 curl = CurlClient(env=env) 566 urln = f'http://{env.domain1}:{env.http_port}/data.json?[0-{count-1}]' 567 r = curl.http_download(urls=[urln], extra_args=[ 568 '--parallel', '--http2' 569 ]) 570 r.check_response(http_status=200, count=count) 571 # we see 3 connections, because Apache only every serves a single 572 # request via Upgrade: and then closed the connection. 573 assert r.total_connects == 3, r.dump_logs() 574 575 # nghttpx is the only server we have that supports TLS early data 576 @pytest.mark.skipif(condition=not Env.have_nghttpx(), reason="no nghttpx") 577 @pytest.mark.parametrize("proto", ['http/1.1', 'h2', 'h3']) 578 def test_02_32_earlydata(self, env: Env, httpd, nghttpx, proto): 579 if not env.curl_uses_lib('gnutls'): 580 pytest.skip('TLS earlydata only implemented in GnuTLS') 581 if proto == 'h3' and not env.have_h3(): 582 pytest.skip("h3 not supported") 583 count = 2 584 docname = 'data-10k' 585 # we want this test to always connect to nghttpx, since it is 586 # the only server we have that supports TLS earlydata 587 port = env.port_for(proto) 588 if proto != 'h3': 589 port = env.nghttpx_https_port 590 url = f'https://{env.domain1}:{port}/{docname}' 591 client = LocalClient(name='hx-download', env=env) 592 if not client.exists(): 593 pytest.skip(f'example client not built: {client.name}') 594 r = client.run(args=[ 595 '-n', f'{count}', 596 '-e', # use TLS earlydata 597 '-f', # forbid reuse of connections 598 '-r', f'{env.domain1}:{port}:127.0.0.1', 599 '-V', proto, url 600 ]) 601 r.check_exit_code(0) 602 srcfile = os.path.join(httpd.docs_dir, docname) 603 self.check_downloads(client, srcfile, count) 604 # check that TLS earlydata worked as expected 605 earlydata = {} 606 reused_session = False 607 for line in r.trace_lines: 608 m = re.match(r'^\[t-(\d+)] EarlyData: (-?\d+)', line) 609 if m: 610 earlydata[int(m.group(1))] = int(m.group(2)) 611 continue 612 m = re.match(r'\[1-1] \* SSL reusing session.*', line) 613 if m: 614 reused_session = True 615 assert reused_session, 'session was not reused for 2nd transfer' 616 assert earlydata[0] == 0, f'{earlydata}' 617 if proto == 'http/1.1': 618 assert earlydata[1] == 69, f'{earlydata}' 619 elif proto == 'h2': 620 assert earlydata[1] == 107, f'{earlydata}' 621 elif proto == 'h3': 622 assert earlydata[1] == 67, f'{earlydata}' 623 624 @pytest.mark.parametrize("proto", ['http/1.1', 'h2']) 625 @pytest.mark.parametrize("max_host_conns", [0, 1, 5]) 626 def test_02_33_max_host_conns(self, env: Env, httpd, nghttpx, proto, max_host_conns): 627 if proto == 'h3' and not env.have_h3(): 628 pytest.skip("h3 not supported") 629 count = 50 630 max_parallel = 50 631 docname = 'data-10k' 632 port = env.port_for(proto) 633 url = f'https://{env.domain1}:{port}/{docname}' 634 client = LocalClient(name='hx-download', env=env) 635 if not client.exists(): 636 pytest.skip(f'example client not built: {client.name}') 637 r = client.run(args=[ 638 '-n', f'{count}', 639 '-m', f'{max_parallel}', 640 '-x', # always use a fresh connection 641 '-M', str(max_host_conns), # limit conns per host 642 '-r', f'{env.domain1}:{port}:127.0.0.1', 643 '-V', proto, url 644 ]) 645 r.check_exit_code(0) 646 srcfile = os.path.join(httpd.docs_dir, docname) 647 self.check_downloads(client, srcfile, count) 648 if max_host_conns > 0: 649 matched_lines = 0 650 for line in r.trace_lines: 651 m = re.match(r'.*The cache now contains (\d+) members.*', line) 652 if m: 653 matched_lines += 1 654 n = int(m.group(1)) 655 assert n <= max_host_conns 656 assert matched_lines > 0 657 658 @pytest.mark.parametrize("proto", ['http/1.1', 'h2']) 659 @pytest.mark.parametrize("max_total_conns", [0, 1, 5]) 660 def test_02_34_max_total_conns(self, env: Env, httpd, nghttpx, proto, max_total_conns): 661 if proto == 'h3' and not env.have_h3(): 662 pytest.skip("h3 not supported") 663 count = 50 664 max_parallel = 50 665 docname = 'data-10k' 666 port = env.port_for(proto) 667 url = f'https://{env.domain1}:{port}/{docname}' 668 client = LocalClient(name='hx-download', env=env) 669 if not client.exists(): 670 pytest.skip(f'example client not built: {client.name}') 671 r = client.run(args=[ 672 '-n', f'{count}', 673 '-m', f'{max_parallel}', 674 '-x', # always use a fresh connection 675 '-T', str(max_total_conns), # limit total connections 676 '-r', f'{env.domain1}:{port}:127.0.0.1', 677 '-V', proto, url 678 ]) 679 r.check_exit_code(0) 680 srcfile = os.path.join(httpd.docs_dir, docname) 681 self.check_downloads(client, srcfile, count) 682 if max_total_conns > 0: 683 matched_lines = 0 684 for line in r.trace_lines: 685 m = re.match(r'.*The cache now contains (\d+) members.*', line) 686 if m: 687 matched_lines += 1 688 n = int(m.group(1)) 689 assert n <= max_total_conns 690 assert matched_lines > 0 691