1#!/usr/bin/env python3 2# -*- coding: utf-8 -*- 3#*************************************************************************** 4# _ _ ____ _ 5# Project ___| | | | _ \| | 6# / __| | | | |_) | | 7# | (__| |_| | _ <| |___ 8# \___|\___/|_| \_\_____| 9# 10# Copyright (C) Daniel Stenberg, <daniel@haxx.se>, et al. 11# 12# This software is licensed as described in the file COPYING, which 13# you should have received as part of this distribution. The terms 14# are also available at https://curl.se/docs/copyright.html. 15# 16# You may opt to use, copy, modify, merge, publish, distribute and/or sell 17# copies of the Software, and permit persons to whom the Software is 18# furnished to do so, under the terms of the COPYING file. 19# 20# This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY 21# KIND, either express or implied. 22# 23# SPDX-License-Identifier: curl 24# 25########################################################################### 26# 27import inspect 28import logging 29import os 30import subprocess 31from datetime import timedelta, datetime 32from json import JSONEncoder 33import time 34from typing import List, Union, Optional 35import copy 36 37from .curl import CurlClient, ExecResult 38from .env import Env 39 40 41log = logging.getLogger(__name__) 42 43 44class Httpd: 45 46 MODULES = [ 47 'log_config', 'logio', 'unixd', 'version', 'watchdog', 48 'authn_core', 'authn_file', 49 'authz_user', 'authz_core', 'authz_host', 50 'auth_basic', 'auth_digest', 51 'alias', 'env', 'filter', 'headers', 'mime', 'setenvif', 52 'socache_shmcb', 53 'rewrite', 'http2', 'ssl', 'proxy', 'proxy_http', 'proxy_connect', 54 'brotli', 55 'mpm_event', 56 ] 57 COMMON_MODULES_DIRS = [ 58 '/usr/lib/apache2/modules', # debian 59 '/usr/libexec/apache2/', # macos 60 ] 61 62 MOD_CURLTEST = None 63 64 def __init__(self, env: Env, proxy_auth: bool = False): 65 self.env = env 66 self._apache_dir = os.path.join(env.gen_dir, 'apache') 67 self._run_dir = os.path.join(self._apache_dir, 'run') 68 self._lock_dir = os.path.join(self._apache_dir, 'locks') 69 self._docs_dir = os.path.join(self._apache_dir, 'docs') 70 self._conf_dir = os.path.join(self._apache_dir, 'conf') 71 self._conf_file = os.path.join(self._conf_dir, 'test.conf') 72 self._logs_dir = os.path.join(self._apache_dir, 'logs') 73 self._error_log = os.path.join(self._logs_dir, 'error_log') 74 self._tmp_dir = os.path.join(self._apache_dir, 'tmp') 75 self._basic_passwords = os.path.join(self._conf_dir, 'basic.passwords') 76 self._digest_passwords = os.path.join(self._conf_dir, 'digest.passwords') 77 self._mods_dir = None 78 self._auth_digest = True 79 self._proxy_auth_basic = proxy_auth 80 self._extra_configs = {} 81 self._loaded_extra_configs = None 82 assert env.apxs 83 p = subprocess.run(args=[env.apxs, '-q', 'libexecdir'], 84 capture_output=True, text=True) 85 if p.returncode != 0: 86 raise Exception(f'{env.apxs} failed to query libexecdir: {p}') 87 self._mods_dir = p.stdout.strip() 88 if self._mods_dir is None: 89 raise Exception('apache modules dir cannot be found') 90 if not os.path.exists(self._mods_dir): 91 raise Exception(f'apache modules dir does not exist: {self._mods_dir}') 92 self._process = None 93 self._rmf(self._error_log) 94 self._init_curltest() 95 96 @property 97 def docs_dir(self): 98 return self._docs_dir 99 100 def clear_logs(self): 101 self._rmf(self._error_log) 102 103 def exists(self): 104 return os.path.exists(self.env.httpd) 105 106 def set_extra_config(self, domain: str, lines: Optional[Union[str, List[str]]]): 107 if lines is None: 108 self._extra_configs.pop(domain, None) 109 else: 110 self._extra_configs[domain] = lines 111 112 def clear_extra_configs(self): 113 self._extra_configs = {} 114 115 def set_proxy_auth(self, active: bool): 116 self._proxy_auth_basic = active 117 118 def _run(self, args, intext=''): 119 env = os.environ.copy() 120 env['APACHE_RUN_DIR'] = self._run_dir 121 env['APACHE_RUN_USER'] = os.environ['USER'] 122 env['APACHE_LOCK_DIR'] = self._lock_dir 123 env['APACHE_CONFDIR'] = self._apache_dir 124 p = subprocess.run(args, stderr=subprocess.PIPE, stdout=subprocess.PIPE, 125 cwd=self.env.gen_dir, 126 input=intext.encode() if intext else None, 127 env=env) 128 start = datetime.now() 129 return ExecResult(args=args, exit_code=p.returncode, 130 stdout=p.stdout.decode().splitlines(), 131 stderr=p.stderr.decode().splitlines(), 132 duration=datetime.now() - start) 133 134 def _cmd_httpd(self, cmd: str): 135 args = [self.env.httpd, 136 "-d", self._apache_dir, 137 "-f", self._conf_file, 138 "-k", cmd] 139 return self._run(args=args) 140 141 def start(self): 142 if self._process: 143 self.stop() 144 self._write_config() 145 with open(self._error_log, 'a') as fd: 146 fd.write('start of server\n') 147 with open(os.path.join(self._apache_dir, 'xxx'), 'a') as fd: 148 fd.write('start of server\n') 149 r = self._cmd_httpd('start') 150 if r.exit_code != 0: 151 log.error(f'failed to start httpd: {r}') 152 return False 153 self._loaded_extra_configs = copy.deepcopy(self._extra_configs) 154 return self.wait_live(timeout=timedelta(seconds=5)) 155 156 def stop(self): 157 r = self._cmd_httpd('stop') 158 self._loaded_extra_configs = None 159 if r.exit_code == 0: 160 return self.wait_dead(timeout=timedelta(seconds=5)) 161 log.fatal(f'stopping httpd failed: {r}') 162 return r.exit_code == 0 163 164 def restart(self): 165 self.stop() 166 return self.start() 167 168 def reload(self): 169 self._write_config() 170 r = self._cmd_httpd("graceful") 171 self._loaded_extra_configs = None 172 if r.exit_code != 0: 173 log.error(f'failed to reload httpd: {r}') 174 self._loaded_extra_configs = copy.deepcopy(self._extra_configs) 175 return self.wait_live(timeout=timedelta(seconds=5)) 176 177 def reload_if_config_changed(self): 178 if self._loaded_extra_configs == self._extra_configs: 179 return True 180 return self.reload() 181 182 def wait_dead(self, timeout: timedelta): 183 curl = CurlClient(env=self.env, run_dir=self._tmp_dir) 184 try_until = datetime.now() + timeout 185 while datetime.now() < try_until: 186 r = curl.http_get(url=f'http://{self.env.domain1}:{self.env.http_port}/') 187 if r.exit_code != 0: 188 return True 189 time.sleep(.1) 190 log.debug(f"Server still responding after {timeout}") 191 return False 192 193 def wait_live(self, timeout: timedelta): 194 curl = CurlClient(env=self.env, run_dir=self._tmp_dir, 195 timeout=timeout.total_seconds()) 196 try_until = datetime.now() + timeout 197 while datetime.now() < try_until: 198 r = curl.http_get(url=f'http://{self.env.domain1}:{self.env.http_port}/') 199 if r.exit_code == 0: 200 return True 201 time.sleep(.1) 202 log.debug(f"Server still not responding after {timeout}") 203 return False 204 205 def _rmf(self, path): 206 if os.path.exists(path): 207 return os.remove(path) 208 209 def _mkpath(self, path): 210 if not os.path.exists(path): 211 return os.makedirs(path) 212 213 def _write_config(self): 214 domain1 = self.env.domain1 215 domain1brotli = self.env.domain1brotli 216 creds1 = self.env.get_credentials(domain1) 217 assert creds1 # convince pytype this isn't None 218 domain2 = self.env.domain2 219 creds2 = self.env.get_credentials(domain2) 220 assert creds2 # convince pytype this isn't None 221 exp_domain = self.env.expired_domain 222 exp_creds = self.env.get_credentials(exp_domain) 223 assert exp_creds # convince pytype this isn't None 224 proxy_domain = self.env.proxy_domain 225 proxy_creds = self.env.get_credentials(proxy_domain) 226 assert proxy_creds # convince pytype this isn't None 227 self._mkpath(self._conf_dir) 228 self._mkpath(self._logs_dir) 229 self._mkpath(self._tmp_dir) 230 self._mkpath(os.path.join(self._docs_dir, 'two')) 231 with open(os.path.join(self._docs_dir, 'data.json'), 'w') as fd: 232 data = { 233 'server': f'{domain1}', 234 } 235 fd.write(JSONEncoder().encode(data)) 236 with open(os.path.join(self._docs_dir, 'two/data.json'), 'w') as fd: 237 data = { 238 'server': f'{domain2}', 239 } 240 fd.write(JSONEncoder().encode(data)) 241 if self._proxy_auth_basic: 242 with open(self._basic_passwords, 'w') as fd: 243 fd.write('proxy:$apr1$FQfeInbs$WQZbODJlVg60j0ogEIlTW/\n') 244 if self._auth_digest: 245 with open(self._digest_passwords, 'w') as fd: 246 fd.write('test:restricted area:57123e269fd73d71ae0656594e938e2f\n') 247 self._mkpath(os.path.join(self.docs_dir, 'restricted/digest')) 248 with open(os.path.join(self.docs_dir, 'restricted/digest/data.json'), 'w') as fd: 249 fd.write('{"area":"digest"}\n') 250 with open(self._conf_file, 'w') as fd: 251 for m in self.MODULES: 252 if os.path.exists(os.path.join(self._mods_dir, f'mod_{m}.so')): 253 fd.write(f'LoadModule {m}_module "{self._mods_dir}/mod_{m}.so"\n') 254 if Httpd.MOD_CURLTEST is not None: 255 fd.write(f'LoadModule curltest_module "{Httpd.MOD_CURLTEST}"\n') 256 conf = [ # base server config 257 f'ServerRoot "{self._apache_dir}"', 258 'DefaultRuntimeDir logs', 259 'PidFile httpd.pid', 260 f'ErrorLog {self._error_log}', 261 f'LogLevel {self._get_log_level()}', 262 'StartServers 4', 263 'ReadBufferSize 16000', 264 'H2MinWorkers 16', 265 'H2MaxWorkers 256', 266 f'Listen {self.env.http_port}', 267 f'Listen {self.env.https_port}', 268 f'Listen {self.env.proxy_port}', 269 f'Listen {self.env.proxys_port}', 270 f'TypesConfig "{self._conf_dir}/mime.types', 271 'SSLSessionCache "shmcb:ssl_gcache_data(32000)"', 272 ] 273 if 'base' in self._extra_configs: 274 conf.extend(self._extra_configs['base']) 275 conf.extend([ # plain http host for domain1 276 f'<VirtualHost *:{self.env.http_port}>', 277 f' ServerName {domain1}', 278 ' ServerAlias localhost', 279 f' DocumentRoot "{self._docs_dir}"', 280 ' Protocols h2c http/1.1', 281 ' H2Direct on', 282 ]) 283 conf.extend(self._curltest_conf(domain1)) 284 conf.extend([ 285 '</VirtualHost>', 286 '', 287 ]) 288 conf.extend([ # https host for domain1, h1 + h2 289 f'<VirtualHost *:{self.env.https_port}>', 290 f' ServerName {domain1}', 291 ' ServerAlias localhost', 292 ' Protocols h2 http/1.1', 293 ' SSLEngine on', 294 f' SSLCertificateFile {creds1.cert_file}', 295 f' SSLCertificateKeyFile {creds1.pkey_file}', 296 f' DocumentRoot "{self._docs_dir}"', 297 ]) 298 conf.extend(self._curltest_conf(domain1)) 299 if domain1 in self._extra_configs: 300 conf.extend(self._extra_configs[domain1]) 301 conf.extend([ 302 '</VirtualHost>', 303 '', 304 ]) 305 # Alternate to domain1 with BROTLI compression 306 conf.extend([ # https host for domain1, h1 + h2 307 f'<VirtualHost *:{self.env.https_port}>', 308 f' ServerName {domain1brotli}', 309 ' Protocols h2 http/1.1', 310 ' SSLEngine on', 311 f' SSLCertificateFile {creds1.cert_file}', 312 f' SSLCertificateKeyFile {creds1.pkey_file}', 313 f' DocumentRoot "{self._docs_dir}"', 314 ' SetOutputFilter BROTLI_COMPRESS', 315 ]) 316 conf.extend(self._curltest_conf(domain1)) 317 if domain1 in self._extra_configs: 318 conf.extend(self._extra_configs[domain1]) 319 conf.extend([ 320 '</VirtualHost>', 321 '', 322 ]) 323 conf.extend([ # plain http host for domain2 324 f'<VirtualHost *:{self.env.http_port}>', 325 f' ServerName {domain2}', 326 ' ServerAlias localhost', 327 f' DocumentRoot "{self._docs_dir}"', 328 ' Protocols h2c http/1.1', 329 ]) 330 conf.extend(self._curltest_conf(domain2)) 331 conf.extend([ 332 '</VirtualHost>', 333 '', 334 ]) 335 conf.extend([ # https host for domain2, no h2 336 f'<VirtualHost *:{self.env.https_port}>', 337 f' ServerName {domain2}', 338 ' Protocols http/1.1', 339 ' SSLEngine on', 340 f' SSLCertificateFile {creds2.cert_file}', 341 f' SSLCertificateKeyFile {creds2.pkey_file}', 342 f' DocumentRoot "{self._docs_dir}/two"', 343 ]) 344 conf.extend(self._curltest_conf(domain2)) 345 if domain2 in self._extra_configs: 346 conf.extend(self._extra_configs[domain2]) 347 conf.extend([ 348 '</VirtualHost>', 349 '', 350 ]) 351 conf.extend([ # https host for expired domain 352 f'<VirtualHost *:{self.env.https_port}>', 353 f' ServerName {exp_domain}', 354 ' Protocols h2 http/1.1', 355 ' SSLEngine on', 356 f' SSLCertificateFile {exp_creds.cert_file}', 357 f' SSLCertificateKeyFile {exp_creds.pkey_file}', 358 f' DocumentRoot "{self._docs_dir}/expired"', 359 ]) 360 conf.extend(self._curltest_conf(exp_domain)) 361 if exp_domain in self._extra_configs: 362 conf.extend(self._extra_configs[exp_domain]) 363 conf.extend([ 364 '</VirtualHost>', 365 '', 366 ]) 367 conf.extend([ # http forward proxy 368 f'<VirtualHost *:{self.env.proxy_port}>', 369 f' ServerName {proxy_domain}', 370 ' Protocols h2c http/1.1', 371 ' ProxyRequests On', 372 ' H2ProxyRequests On', 373 ' ProxyVia On', 374 f' AllowCONNECT {self.env.http_port} {self.env.https_port}', 375 ]) 376 conf.extend(self._get_proxy_conf()) 377 conf.extend([ 378 '</VirtualHost>', 379 '', 380 ]) 381 conf.extend([ # https forward proxy 382 f'<VirtualHost *:{self.env.proxys_port}>', 383 f' ServerName {proxy_domain}', 384 ' Protocols h2 http/1.1', 385 ' SSLEngine on', 386 f' SSLCertificateFile {proxy_creds.cert_file}', 387 f' SSLCertificateKeyFile {proxy_creds.pkey_file}', 388 ' ProxyRequests On', 389 ' H2ProxyRequests On', 390 ' ProxyVia On', 391 f' AllowCONNECT {self.env.http_port} {self.env.https_port}', 392 ]) 393 conf.extend(self._get_proxy_conf()) 394 conf.extend([ 395 '</VirtualHost>', 396 '', 397 ]) 398 399 fd.write("\n".join(conf)) 400 with open(os.path.join(self._conf_dir, 'mime.types'), 'w') as fd: 401 fd.write("\n".join([ 402 'text/html html', 403 'application/json json', 404 '' 405 ])) 406 407 def _get_proxy_conf(self): 408 if self._proxy_auth_basic: 409 return [ 410 ' <Proxy "*">', 411 ' AuthType Basic', 412 ' AuthName "Restricted Proxy"', 413 ' AuthBasicProvider file', 414 f' AuthUserFile "{self._basic_passwords}"', 415 ' Require user proxy', 416 ' </Proxy>', 417 ] 418 else: 419 return [ 420 ' <Proxy "*">', 421 ' Require ip 127.0.0.1', 422 ' </Proxy>', 423 ] 424 425 def _get_log_level(self): 426 if self.env.verbose > 3: 427 return 'trace2' 428 if self.env.verbose > 2: 429 return 'trace1' 430 if self.env.verbose > 1: 431 return 'debug' 432 return 'info' 433 434 def _curltest_conf(self, servername) -> List[str]: 435 lines = [] 436 if Httpd.MOD_CURLTEST is not None: 437 lines.extend([ 438 ' Redirect 302 /data.json.302 /data.json', 439 ' Redirect 301 /curltest/echo301 /curltest/echo', 440 ' Redirect 302 /curltest/echo302 /curltest/echo', 441 ' Redirect 303 /curltest/echo303 /curltest/echo', 442 ' Redirect 307 /curltest/echo307 /curltest/echo', 443 ' <Location /curltest/sslinfo>', 444 ' SSLOptions StdEnvVars', 445 ' SetHandler curltest-sslinfo', 446 ' </Location>', 447 ' <Location /curltest/echo>', 448 ' SetHandler curltest-echo', 449 ' </Location>', 450 ' <Location /curltest/put>', 451 ' SetHandler curltest-put', 452 ' </Location>', 453 ' <Location /curltest/tweak>', 454 ' SetHandler curltest-tweak', 455 ' </Location>', 456 ' Redirect 302 /tweak /curltest/tweak', 457 ' <Location /curltest/1_1>', 458 ' SetHandler curltest-1_1-required', 459 ' </Location>', 460 ' <Location /curltest/shutdown_unclean>', 461 ' SetHandler curltest-tweak', 462 ' SetEnv force-response-1.0 1', 463 ' </Location>', 464 ' SetEnvIf Request_URI "/shutdown_unclean" ssl-unclean=1', 465 ]) 466 if self._auth_digest: 467 lines.extend([ 468 f' <Directory {self.docs_dir}/restricted/digest>', 469 ' AuthType Digest', 470 ' AuthName "restricted area"', 471 f' AuthDigestDomain "https://{servername}"', 472 ' AuthBasicProvider file', 473 f' AuthUserFile "{self._digest_passwords}"', 474 ' Require valid-user', 475 ' </Directory>', 476 477 ]) 478 return lines 479 480 def _init_curltest(self): 481 if Httpd.MOD_CURLTEST is not None: 482 return 483 local_dir = os.path.dirname(inspect.getfile(Httpd)) 484 p = subprocess.run([self.env.apxs, '-c', 'mod_curltest.c'], 485 capture_output=True, 486 cwd=os.path.join(local_dir, 'mod_curltest')) 487 rv = p.returncode 488 if rv != 0: 489 log.error(f"compiling mod_curltest failed: {p.stderr}") 490 raise Exception(f"compiling mod_curltest failed: {p.stderr}") 491 Httpd.MOD_CURLTEST = os.path.join( 492 local_dir, 'mod_curltest/.libs/mod_curltest.so') 493