1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3#*************************************************************************** 4# _ _ ____ _ 5# Project ___| | | | _ \| | 6# / __| | | | |_) | | 7# | (__| |_| | _ <| |___ 8# \___|\___/|_| \_\_____| 9# 10# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al. 11# 12# This software is licensed as described in the file COPYING, which 13# you should have received as part of this distribution. The terms 14# are also available at https://curl.se/docs/copyright.html. 15# 16# You may opt to use, copy, modify, merge, publish, distribute and/or sell 17# copies of the Software, and permit persons to whom the Software is 18# furnished to do so, under the terms of the COPYING file. 19# 20# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY 21# KIND, either express or implied. 22# 23# SPDX-License-Identifier: curl 24# 25########################################################################### 26# 27import logging 28import os 29import re 30import shutil 31import socket 32import subprocess 33import tempfile 34from configparser import ConfigParser, ExtendedInterpolation 35from datetime import timedelta 36from typing import Optional 37 38from .certs import CertificateSpec, Credentials, TestCA 39from .ports import alloc_ports 40 41 42log = logging.getLogger(__name__) 43 44 45def init_config_from(conf_path): 46 if os.path.isfile(conf_path): 47 config = ConfigParser(interpolation=ExtendedInterpolation()) 48 config.read(conf_path) 49 return config 50 return None 51 52 53TESTS_HTTPD_PATH = os.path.dirname(os.path.dirname(__file__)) 54TOP_PATH = os.path.join(os.getcwd(), os.path.pardir) 55DEF_CONFIG = init_config_from(os.path.join(TOP_PATH, 'tests', 'http', 'config.ini')) 56CURL = os.path.join(TOP_PATH, 'src', 'curl') 57 58 59class EnvConfig: 60 61 def __init__(self): 62 self.tests_dir = TESTS_HTTPD_PATH 63 self.gen_dir = os.path.join(self.tests_dir, 'gen') 64 self.project_dir = os.path.dirname(os.path.dirname(self.tests_dir)) 65 self.build_dir = TOP_PATH 66 self.config = DEF_CONFIG 67 # check cur and its features 68 self.curl = CURL 69 if 'CURL' in os.environ: 70 self.curl = os.environ['CURL'] 71 self.curl_props = { 72 'version_string': '', 73 'version': '', 74 'os': '', 75 'fullname': '', 76 'features_string': '', 77 'features': set(), 78 'protocols_string': '', 79 'protocols': set(), 80 'libs': set(), 81 'lib_versions': set(), 82 } 83 self.curl_is_debug = False 84 self.curl_protos = [] 85 p = subprocess.run(args=[self.curl, '-V'], 86 capture_output=True, text=True) 87 if p.returncode != 0: 88 raise RuntimeError(f'{self.curl} -V failed with exit code: {p.returncode}') 89 if p.stderr.startswith('WARNING:'): 90 self.curl_is_debug = True 91 for line in p.stdout.splitlines(keepends=False): 92 if line.startswith('curl '): 93 self.curl_props['version_string'] = line 94 m = re.match(r'^curl (?P<version>\S+) (?P<os>\S+) (?P<libs>.*)$', line) 95 if m: 96 self.curl_props['fullname'] = m.group(0) 97 self.curl_props['version'] = m.group('version') 98 self.curl_props['os'] = m.group('os') 99 self.curl_props['lib_versions'] = { 100 lib.lower() for lib in m.group('libs').split(' ') 101 } 102 self.curl_props['libs'] = { 103 re.sub(r'/[a-z0-9.-]*', '', lib) for lib in self.curl_props['lib_versions'] 104 } 105 if line.startswith('Features: '): 106 self.curl_props['features_string'] = line[10:] 107 self.curl_props['features'] = { 108 feat.lower() for feat in line[10:].split(' ') 109 } 110 if line.startswith('Protocols: '): 111 self.curl_props['protocols_string'] = line[11:] 112 self.curl_props['protocols'] = { 113 prot.lower() for prot in line[11:].split(' ') 114 } 115 116 self.ports = alloc_ports(port_specs={ 117 'ftp': socket.SOCK_STREAM, 118 'ftps': socket.SOCK_STREAM, 119 'http': socket.SOCK_STREAM, 120 'https': socket.SOCK_STREAM, 121 'nghttpx_https': socket.SOCK_STREAM, 122 'proxy': socket.SOCK_STREAM, 123 'proxys': socket.SOCK_STREAM, 124 'h2proxys': socket.SOCK_STREAM, 125 'caddy': socket.SOCK_STREAM, 126 'caddys': socket.SOCK_STREAM, 127 'ws': socket.SOCK_STREAM, 128 }) 129 self.httpd = self.config['httpd']['httpd'] 130 self.apxs = self.config['httpd']['apxs'] 131 if len(self.apxs) == 0: 132 self.apxs = None 133 self._httpd_version = None 134 135 self.examples_pem = { 136 'key': 'xxx', 137 'cert': 'xxx', 138 } 139 self.htdocs_dir = os.path.join(self.gen_dir, 'htdocs') 140 self.tld = 'http.curl.se' 141 self.domain1 = f"one.{self.tld}" 142 self.domain1brotli = f"brotli.one.{self.tld}" 143 self.domain2 = f"two.{self.tld}" 144 self.ftp_domain = f"ftp.{self.tld}" 145 self.proxy_domain = f"proxy.{self.tld}" 146 self.expired_domain = f"expired.{self.tld}" 147 self.cert_specs = [ 148 CertificateSpec(domains=[self.domain1, self.domain1brotli, 'localhost', '127.0.0.1'], key_type='rsa2048'), 149 CertificateSpec(domains=[self.domain2], key_type='rsa2048'), 150 CertificateSpec(domains=[self.ftp_domain], key_type='rsa2048'), 151 CertificateSpec(domains=[self.proxy_domain, '127.0.0.1'], key_type='rsa2048'), 152 CertificateSpec(domains=[self.expired_domain], key_type='rsa2048', 153 valid_from=timedelta(days=-100), valid_to=timedelta(days=-10)), 154 CertificateSpec(name="clientsX", sub_specs=[ 155 CertificateSpec(name="user1", client=True), 156 ]), 157 ] 158 159 self.nghttpx = self.config['nghttpx']['nghttpx'] 160 if len(self.nghttpx.strip()) == 0: 161 self.nghttpx = None 162 self._nghttpx_version = None 163 self.nghttpx_with_h3 = False 164 if self.nghttpx is not None: 165 p = subprocess.run(args=[self.nghttpx, '-v'], 166 capture_output=True, text=True) 167 if p.returncode != 0: 168 # not a working nghttpx 169 self.nghttpx = None 170 else: 171 self._nghttpx_version = re.sub(r'^nghttpx\s*', '', p.stdout.strip()) 172 self.nghttpx_with_h3 = re.match(r'.* nghttp3/.*', p.stdout.strip()) is not None 173 log.debug(f'nghttpx -v: {p.stdout}') 174 175 self.caddy = self.config['caddy']['caddy'] 176 self._caddy_version = None 177 if len(self.caddy.strip()) == 0: 178 self.caddy = None 179 if self.caddy is not None: 180 try: 181 p = subprocess.run(args=[self.caddy, 'version'], 182 capture_output=True, text=True) 183 if p.returncode != 0: 184 # not a working caddy 185 self.caddy = None 186 m = re.match(r'v?(\d+\.\d+\.\d+).*', p.stdout) 187 if m: 188 self._caddy_version = m.group(1) 189 else: 190 raise RuntimeError(f'Unable to determine cadd version from: {p.stdout}') 191 # TODO: specify specific exceptions here 192 except: # noqa: E722 193 self.caddy = None 194 195 self.vsftpd = self.config['vsftpd']['vsftpd'] 196 self._vsftpd_version = None 197 if self.vsftpd is not None: 198 try: 199 with tempfile.TemporaryFile('w+') as tmp: 200 p = subprocess.run(args=[self.vsftpd, '-v'], 201 capture_output=True, text=True, stdin=tmp) 202 if p.returncode != 0: 203 # not a working vsftpd 204 self.vsftpd = None 205 if p.stderr: 206 ver_text = p.stderr 207 else: 208 # Oddly, some versions of vsftpd write to stdin (!) 209 # instead of stderr, which is odd but works. If there 210 # is nothing on stderr, read the file on stdin and use 211 # any data there instead. 212 tmp.seek(0) 213 ver_text = tmp.read() 214 m = re.match(r'vsftpd: version (\d+\.\d+\.\d+)', ver_text) 215 if m: 216 self._vsftpd_version = m.group(1) 217 elif len(p.stderr) == 0: 218 # vsftp does not use stdout or stderr for printing its version... -.- 219 self._vsftpd_version = 'unknown' 220 else: 221 raise Exception(f'Unable to determine VsFTPD version from: {p.stderr}') 222 except Exception: 223 self.vsftpd = None 224 225 self._tcpdump = shutil.which('tcpdump') 226 227 @property 228 def httpd_version(self): 229 if self._httpd_version is None and self.apxs is not None: 230 try: 231 p = subprocess.run(args=[self.apxs, '-q', 'HTTPD_VERSION'], 232 capture_output=True, text=True) 233 if p.returncode != 0: 234 log.error(f'{self.apxs} failed to query HTTPD_VERSION: {p}') 235 else: 236 self._httpd_version = p.stdout.strip() 237 except Exception: 238 log.exception(f'{self.apxs} failed to run') 239 return self._httpd_version 240 241 def versiontuple(self, v): 242 v = re.sub(r'(\d+\.\d+(\.\d+)?)(-\S+)?', r'\1', v) 243 return tuple(map(int, v.split('.'))) 244 245 def httpd_is_at_least(self, minv): 246 if self.httpd_version is None: 247 return False 248 hv = self.versiontuple(self.httpd_version) 249 return hv >= self.versiontuple(minv) 250 251 def caddy_is_at_least(self, minv): 252 if self.caddy_version is None: 253 return False 254 hv = self.versiontuple(self.caddy_version) 255 return hv >= self.versiontuple(minv) 256 257 def is_complete(self) -> bool: 258 return os.path.isfile(self.httpd) and \ 259 self.apxs is not None and \ 260 os.path.isfile(self.apxs) 261 262 def get_incomplete_reason(self) -> Optional[str]: 263 if self.httpd is None or len(self.httpd.strip()) == 0: 264 return 'httpd not configured, see `--with-test-httpd=<path>`' 265 if not os.path.isfile(self.httpd): 266 return f'httpd ({self.httpd}) not found' 267 if self.apxs is None: 268 return "command apxs not found (commonly provided in apache2-dev)" 269 if not os.path.isfile(self.apxs): 270 return f"apxs ({self.apxs}) not found" 271 return None 272 273 @property 274 def nghttpx_version(self): 275 return self._nghttpx_version 276 277 @property 278 def caddy_version(self): 279 return self._caddy_version 280 281 @property 282 def vsftpd_version(self): 283 return self._vsftpd_version 284 285 @property 286 def tcpdmp(self) -> Optional[str]: 287 return self._tcpdump 288 289 290class Env: 291 292 CONFIG = EnvConfig() 293 294 @staticmethod 295 def setup_incomplete() -> bool: 296 return not Env.CONFIG.is_complete() 297 298 @staticmethod 299 def incomplete_reason() -> Optional[str]: 300 return Env.CONFIG.get_incomplete_reason() 301 302 @staticmethod 303 def have_nghttpx() -> bool: 304 return Env.CONFIG.nghttpx is not None 305 306 @staticmethod 307 def have_h3_server() -> bool: 308 return Env.CONFIG.nghttpx_with_h3 309 310 @staticmethod 311 def have_ssl_curl() -> bool: 312 return Env.curl_has_feature('ssl') or Env.curl_has_feature('multissl') 313 314 @staticmethod 315 def have_h2_curl() -> bool: 316 return 'http2' in Env.CONFIG.curl_props['features'] 317 318 @staticmethod 319 def have_h3_curl() -> bool: 320 return 'http3' in Env.CONFIG.curl_props['features'] 321 322 @staticmethod 323 def curl_uses_lib(libname: str) -> bool: 324 return libname.lower() in Env.CONFIG.curl_props['libs'] 325 326 @staticmethod 327 def curl_uses_ossl_quic() -> bool: 328 if Env.have_h3_curl(): 329 return not Env.curl_uses_lib('ngtcp2') and Env.curl_uses_lib('nghttp3') 330 return False 331 332 @staticmethod 333 def curl_version_string() -> str: 334 return Env.CONFIG.curl_props['version_string'] 335 336 @staticmethod 337 def curl_features_string() -> str: 338 return Env.CONFIG.curl_props['features_string'] 339 340 @staticmethod 341 def curl_has_feature(feature: str) -> bool: 342 return feature.lower() in Env.CONFIG.curl_props['features'] 343 344 @staticmethod 345 def curl_protocols_string() -> str: 346 return Env.CONFIG.curl_props['protocols_string'] 347 348 @staticmethod 349 def curl_has_protocol(protocol: str) -> bool: 350 return protocol.lower() in Env.CONFIG.curl_props['protocols'] 351 352 @staticmethod 353 def curl_lib_version(libname: str) -> str: 354 prefix = f'{libname.lower()}/' 355 for lversion in Env.CONFIG.curl_props['lib_versions']: 356 if lversion.startswith(prefix): 357 return lversion[len(prefix):] 358 return 'unknown' 359 360 @staticmethod 361 def curl_lib_version_at_least(libname: str, min_version) -> bool: 362 lversion = Env.curl_lib_version(libname) 363 if lversion != 'unknown': 364 return Env.CONFIG.versiontuple(min_version) <= \ 365 Env.CONFIG.versiontuple(lversion) 366 return False 367 368 @staticmethod 369 def curl_os() -> str: 370 return Env.CONFIG.curl_props['os'] 371 372 @staticmethod 373 def curl_fullname() -> str: 374 return Env.CONFIG.curl_props['fullname'] 375 376 @staticmethod 377 def curl_version() -> str: 378 return Env.CONFIG.curl_props['version'] 379 380 @staticmethod 381 def curl_is_debug() -> bool: 382 return Env.CONFIG.curl_is_debug 383 384 @staticmethod 385 def have_h3() -> bool: 386 return Env.have_h3_curl() and Env.have_h3_server() 387 388 @staticmethod 389 def httpd_version() -> str: 390 return Env.CONFIG.httpd_version 391 392 @staticmethod 393 def nghttpx_version() -> str: 394 return Env.CONFIG.nghttpx_version 395 396 @staticmethod 397 def caddy_version() -> str: 398 return Env.CONFIG.caddy_version 399 400 @staticmethod 401 def caddy_is_at_least(minv) -> bool: 402 return Env.CONFIG.caddy_is_at_least(minv) 403 404 @staticmethod 405 def httpd_is_at_least(minv) -> bool: 406 return Env.CONFIG.httpd_is_at_least(minv) 407 408 @staticmethod 409 def has_caddy() -> bool: 410 return Env.CONFIG.caddy is not None 411 412 @staticmethod 413 def has_vsftpd() -> bool: 414 return Env.CONFIG.vsftpd is not None 415 416 @staticmethod 417 def vsftpd_version() -> str: 418 return Env.CONFIG.vsftpd_version 419 420 @staticmethod 421 def tcpdump() -> Optional[str]: 422 return Env.CONFIG.tcpdmp 423 424 def __init__(self, pytestconfig=None): 425 self._verbose = pytestconfig.option.verbose \ 426 if pytestconfig is not None else 0 427 self._ca = None 428 self._test_timeout = 300.0 if self._verbose > 1 else 60.0 # seconds 429 430 def issue_certs(self): 431 if self._ca is None: 432 ca_dir = os.path.join(self.CONFIG.gen_dir, 'ca') 433 self._ca = TestCA.create_root(name=self.CONFIG.tld, 434 store_dir=ca_dir, 435 key_type="rsa2048") 436 self._ca.issue_certs(self.CONFIG.cert_specs) 437 438 def setup(self): 439 os.makedirs(self.gen_dir, exist_ok=True) 440 os.makedirs(self.htdocs_dir, exist_ok=True) 441 self.issue_certs() 442 443 def get_credentials(self, domain) -> Optional[Credentials]: 444 creds = self.ca.get_credentials_for_name(domain) 445 if len(creds) > 0: 446 return creds[0] 447 return None 448 449 @property 450 def verbose(self) -> int: 451 return self._verbose 452 453 @property 454 def test_timeout(self) -> Optional[float]: 455 return self._test_timeout 456 457 @test_timeout.setter 458 def test_timeout(self, val: Optional[float]): 459 self._test_timeout = val 460 461 @property 462 def gen_dir(self) -> str: 463 return self.CONFIG.gen_dir 464 465 @property 466 def project_dir(self) -> str: 467 return self.CONFIG.project_dir 468 469 @property 470 def build_dir(self) -> str: 471 return self.CONFIG.build_dir 472 473 @property 474 def ca(self): 475 return self._ca 476 477 @property 478 def htdocs_dir(self) -> str: 479 return self.CONFIG.htdocs_dir 480 481 @property 482 def tld(self) -> str: 483 return self.CONFIG.tld 484 485 @property 486 def domain1(self) -> str: 487 return self.CONFIG.domain1 488 489 @property 490 def domain1brotli(self) -> str: 491 return self.CONFIG.domain1brotli 492 493 @property 494 def domain2(self) -> str: 495 return self.CONFIG.domain2 496 497 @property 498 def ftp_domain(self) -> str: 499 return self.CONFIG.ftp_domain 500 501 @property 502 def proxy_domain(self) -> str: 503 return self.CONFIG.proxy_domain 504 505 @property 506 def expired_domain(self) -> str: 507 return self.CONFIG.expired_domain 508 509 @property 510 def http_port(self) -> int: 511 return self.CONFIG.ports['http'] 512 513 @property 514 def https_port(self) -> int: 515 return self.CONFIG.ports['https'] 516 517 @property 518 def nghttpx_https_port(self) -> int: 519 return self.CONFIG.ports['nghttpx_https'] 520 521 @property 522 def h3_port(self) -> int: 523 return self.https_port 524 525 @property 526 def proxy_port(self) -> int: 527 return self.CONFIG.ports['proxy'] 528 529 @property 530 def proxys_port(self) -> int: 531 return self.CONFIG.ports['proxys'] 532 533 @property 534 def ftp_port(self) -> int: 535 return self.CONFIG.ports['ftp'] 536 537 @property 538 def ftps_port(self) -> int: 539 return self.CONFIG.ports['ftps'] 540 541 @property 542 def h2proxys_port(self) -> int: 543 return self.CONFIG.ports['h2proxys'] 544 545 def pts_port(self, proto: str = 'http/1.1') -> int: 546 # proxy tunnel port 547 return self.CONFIG.ports['h2proxys' if proto == 'h2' else 'proxys'] 548 549 @property 550 def caddy(self) -> str: 551 return self.CONFIG.caddy 552 553 @property 554 def caddy_https_port(self) -> int: 555 return self.CONFIG.ports['caddys'] 556 557 @property 558 def caddy_http_port(self) -> int: 559 return self.CONFIG.ports['caddy'] 560 561 @property 562 def vsftpd(self) -> str: 563 return self.CONFIG.vsftpd 564 565 @property 566 def ws_port(self) -> int: 567 return self.CONFIG.ports['ws'] 568 569 @property 570 def curl(self) -> str: 571 return self.CONFIG.curl 572 573 @property 574 def httpd(self) -> str: 575 return self.CONFIG.httpd 576 577 @property 578 def apxs(self) -> str: 579 return self.CONFIG.apxs 580 581 @property 582 def nghttpx(self) -> Optional[str]: 583 return self.CONFIG.nghttpx 584 585 @property 586 def slow_network(self) -> bool: 587 return "CURL_DBG_SOCK_WBLOCK" in os.environ or \ 588 "CURL_DBG_SOCK_WPARTIAL" in os.environ 589 590 @property 591 def ci_run(self) -> bool: 592 return "CURL_CI" in os.environ 593 594 def port_for(self, alpn_proto: Optional[str] = None): 595 if alpn_proto is None or \ 596 alpn_proto in ['h2', 'http/1.1', 'http/1.0', 'http/0.9']: 597 return self.https_port 598 if alpn_proto in ['h3']: 599 return self.h3_port 600 return self.http_port 601 602 def authority_for(self, domain: str, alpn_proto: Optional[str] = None): 603 return f'{domain}:{self.port_for(alpn_proto=alpn_proto)}' 604 605 def make_data_file(self, indir: str, fname: str, fsize: int, 606 line_length: int = 1024) -> str: 607 if line_length < 11: 608 raise RuntimeError('line_length less than 11 not supported') 609 fpath = os.path.join(indir, fname) 610 s10 = "0123456789" 611 s = round((line_length / 10) + 1) * s10 612 s = s[0:line_length-11] 613 with open(fpath, 'w') as fd: 614 for i in range(int(fsize / line_length)): 615 fd.write(f"{i:09d}-{s}\n") 616 remain = int(fsize % line_length) 617 if remain != 0: 618 i = int(fsize / line_length) + 1 619 fd.write(f"{i:09d}-{s}"[0:remain-1] + "\n") 620 return fpath 621